V4.2021.12.2753 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-360Spider
-404enemy
+01h4x.com
+404checker
 80legs
-ASPSeek
-AfD-Verbotsverfahren
+Aboundexbot
+Acunetix
 AhrefsBot
-Alexibot
 AlphaBot
-Anarchie
-Anarchy99
 Ankit
-Anthill
-Asterias
-AwarioRssBot
+Aspiegel
+Attach
 AwarioSmartBot
-BDCbot
-Badass
-BatchFTP
-Bigfoot
-BlackWidow
+BBBike
+BDFetch
+BackDoorBot
+BackStreet
+BackWeb
+BacklinkCrawler
+Barkrowler
+Battleztar Bazinga
+Bitacle
 BlowFish
-Buck
+Bolt
 BuiltWith
-Bullseye
+BunnySlippers
+BuzzSumo
+CODE87
 Calculon
-CazoodleBot
-CensysInspect
-CheeseBot
+ChinaClaw
+Claritybot
 Cliqzbot
-Cocolyzebot
+Cloud mapping
 Cogentbot
 Copier
-CopyRightCheck
+Copyscape
 Cosmos
-Craftbot
-Crescent
-CyotekWebCopy
+Crawling at Home Project
+Custo
 DBLBot
 DIIbot
 DSearch
-DataCha0s
+DatabaseDriverMysqli
+Deusu
 Devil
 Digincore
+DigitalPebble
 Dirbuster
-Discobot
+Disco
+Discoverybot
+Dispatch
+DittoSpyder
 DnBCrawler-Analytics
-DomainStatsBot
-Dotbot
-Dragonfly
-Drip
-ECCP/1.0
-EMail Siphon
-EirGrabber
-Extreme Picture Finder
+DomainAppender
+Domains Project
+Download Wonder
+EroCrawler
+Exabot
+Ezooms
+FDM
+FemtosearchBot
+Fimap
 FlashGet
-Fuzz
-FyberSpider
+Flunky
 Fyrebot
-GalaxyBot
-GermCrawler
+G-i-g-a-b-o-t
+Genieo
+GetRight
 Go!Zilla
-Grabber
+Gotit
 Grafula
+GrapeshotCrawler
 GridBot
-HEADMasterSEO
-HMView
+HTMLparser
+HTTrack
 Haansoft
+Harvest
 Havij
-Humanlinks
-InfoTekies
-InterGET
-Iria
-Iskanie
+IRLbot
+IndeedBot
+Internet Ninja
+InternetSeer
+IstellaBot
 JOC Web Spider
 JamesBOT
-JetCar
-Kenjin Spider
-Keyword Density
-Kinza
+JikeSpider
+JustView
 Kozmosbot
-LWP::Simple
-Lanshanbot
+Leap
 LeechFTP
+Lftp
 Libwhisker
-Lightspeedsystems
-Likse
-LinkWalker
 Linkbot
 LinkextractorPro
 LinqiaMetadataDownloaderBot
 LinqiaScrapeBot
-Lipperhey
-Litemage_walker
-Ltx71
-MTRobot
+Lmspider
+MFC_Tear_Sample
+MIDown tool
+MJ12bot
+Mag-Net
+Magnet
 Majestic SEO
 Majestic12
-Mass Downloader
 MauiBot
-Mb2345Browser
-MeanPath Bot
-MegaIndex.ru
-Metauri
-Microsoft URL Control
-Mojeek
-Mojolicious
-Morfeus Fucking Scanner
+Meanpathbot
+Mediatoolkitbot
+MicroMessenger
+Mozlila
 Mr.4x3
 Msrabot
-NICErsPRO
+NPbot
 Name Intelligence
+NearSite
 Nessus
-NetMechanic
-Netcraft
-NextGenSearchBot
+Net Vampire
+NetAnts
 Niki-bot
 NimbleCrawler
-Nimbostratus
-Nmap
-Nutch
-Octopus
 Offline Explorer
-OpenVAS
-OutclicksBot
+Offline Navigator
+OnCrawl
+Openfind
+Openvas
+OrangeBot
+OutfoxBot
 PECL::HTTP
-Panscient
-Pixray
-ProPowerBot
-ProWebWalker
+PHPCrawl
+Page Analyzer
+Pandalytics
+PeoplePal
+Petalbot
+Pi-Monster
+Picscout
+PleaseCrawl
 Probethenet
-Psbot
-RankActiveLinkBot
+Pu_iN
+Pump
+RankActive
 Rankivabot
-Re-re
-ReGet
-Recorder
+RealDownload
+Reaper
+RedesScrapy
+RepoMonkey
 RocketCrawler
 Rogerbot
 SBIder
-SEOkicks-Robot
+SEOlyticsCrawler
+SEOprofiler
 SEOstats
-SMTBot
-ScanAlert
+SISTRIX
+SalesIntelligent
+ScoutJet
+Screaming
+ScreenerBot
 Searchestate
-SearchmetricsBot
+SemanticJuice
 Semrush
 SentiBot
 SeobilityBot
 Shodan
 Siphon
 Site Sucker
-SiteCheckerBotCrawler
-SiteExplorer
 SiteLockSpider
-SiteSnagger
 SiteSucker
 Sitebeam
-Sitevigil
 Snake
+SocialRankIOBot
 Sociscraper
-Sosospider
-Sottopop
-Spammen
-SpankBot
-Spbot
-Spinn3r
-SputnikBot
-Sucuri
-Surfbot
-Szukacz
+Sqlmap
+Stripper
+Sucker
+SurveyBot
+Suzuran
+Swiftbot
 T0PHackTeam
-T8Abot
-Telesoft
+Teleport
 Telesphoreo
-Titan
+Telesphorep
+The Intraformant
+Thumbor
+TightTwatBot
+Toweyabot
+Tracemyfile
+Trendiction
 Trendictionbot
 True_Robot
-Turingos
-UnisterBot
-Upflow
-VCI
+Vacuum
 Vagabondo
-VelenPublicWebCrawler
-Virusdie
+VeriCiteCrawler
+VidibleScraper
 VoidEYE
-Voil
+WASALive-Bot
+WBSearchBot
 WISENutbot
-WPScan
-WWW-Collector-E
-WWW-Mechanize
-WWWOFFLE
-Wallpapers
-Wallpapers/3.0
+WWW::Mechanize
 WeSEE
 Web Auto
-Web Collage
-Web Enhancer
-Web Fetch
-Web Fuck
 Web Pix
-WebCollage
-WebEnhancer
-WebFuck
+Web Sucker
+WebAuto
+WebGo IS
 WebLeacher
-WebPix
 WebReaper
+WebSucker
+WebWhacker
+WebZIP
 Webalta
 Website Quester
 WebsiteQuester
-Webster
+Whack
+Whacker
+Whatweb
 WinHTTrack
-Wprecon
+Wonderbot
+Xaldon_WebSpider
+Xenu
 Zade
 Zauba
-Zermelo
+Zeus
 Zitebot
-ZmEu
+ZoomBot
 ZumBot
-archive.org_bot
-arquivo-web-crawler
-autoemailspider
-clark-crawler
+arquivo.pt
+backlink-check
 coccocbot-web
 cognitiveseo
-crawler.feedback
-dataforseo.com
-demandbase-bot
-evc-batch
-facebookscraper
-internetVista monitor
+com.plumanalytics
+crawler4j
+domainsproject.org
+eCatch
+gopher
+heritrix
 isitwp.com
 lwp-trivial
 meanpathbot
 muhstik-scan
-oBot
 pcBrowser
-probe-image-size
+satoristudio.net
 scan.lol
-seobility
+seocompany.store
+seoscanners
+seostar
 sexsearcher
-sogouspider
-sp_auditbot
+sitechecker.pro
+siteripz
 spyfu
-trendiction.de
-webgains-bot
-webpros.com
+voyagerx.com
+webmeup-crawler
 webprosbot
-x09Mozilla
 xpymep1.exe
 zgrab