V4.2021.08.2510 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-360Spider
+404checker
 AIBOT
 ALittle Client
-Aboundex
+ASPSeek
+Abonti
 Acunetix
-Alexibot
-AllSubmitter
-Anarchy99
-Ankit
-Anthill
-AwarioRssBot
+AfD-Verbotsverfahren
+AiHitBot
+Anarchie
+Aspiegel
+Asterias
+AwarioSmartBot
+BBBike
 BDCbot
-BackWeb
-Backlink-Ceck
-Badass
-Bandit
+BDFetch
+BackDoorBot
+BacklinkCrawler
 Barkrowler
-Black Hole
-BlackWidow
-Bolt
+BatchFTP
+Bitacle
+Blackboard
 BotALot
-Buck
-BuzzSumo
-CATExplorador
+Brandprotect
+Buddy
 CCBot
+CSHttp
+Calculon
+CazoodleBot
 Cegbfeieh
-ChinaClaw
-Chlooe
-Cliqzbot
+CensysInspect
+CheTeam
+CheeseBot
+Claritybot
+Cloud mapping
 Cogentbot
-Collector
 Copier
 CopyRightCheck
+Cosmos
 Craftbot
-CrazyWebCrawler
-Curious
+CrunchBot
+Custo
 DBLBot
-DSearch
-DTS Agent
-DataCha0s
 DatabaseDriverMysqli
+Demon
+Devil
 Digincore
-DigitalPebble
-Dirbuster
 Discoverybot
-DittoSpyder
-DnyzBot
 DomCopBot
+DomainAppender
+DomainCrawler
 DomainSigmaCrawler
+DomainStatsBot
 Domains Project
-Dotbot
-Drip
 ECCP/1.0
-Ebingbong
-EroCrawler
-Evil
-Extreme Picture Finder
-EyeNetIE
+EMail Siphon
+EirGrabber
+Exabot
+Express WebPictures
+Extractor
 Ezooms
-FHscan
-Firefox/7.0
+FDM
+FemtosearchBot
+FlashGet
+Foobot
 Freeuploader
-FrontPage
-Fuzz
-Fyrebot
+FyberSpider
+G-i-g-a-b-o-t
+GT::WWW
+GermCrawler
+GetRight
+GetWeb
 Getintent
 Gigablast
+GoZilla
+Gotit
 GrabNet
-Grabber
-GridBot
-HEADMasterSEO
-HTTrack
+Haansoft
+HaosouSpider
 Harvest
+Havij
 Hloader
-HybridBot
-IDBTE4M
-IDBot
-Image Sucker
+IRLbot
+Image Fetch
+IndeedBot
+Indy Library
+InfoNaviRobot
 Intelliseek
 InternetSeer
-Iskanie
-JOC Web Spider
-Jbrofuzz
+Iria
+IstellaBot
+JetCar
 Jetty
-JustView
-Keyword Density
-LNSpiderguy
-LWP::Simple
-Lanshanbot
-Larbin
+JikeSpider
+Joomla
+Jorgee
 Leap
-LeechGet
-LieBaoFast
-LinkScan
+LeechFTP
+LexiBot
+Linkbot
+LinkextractorPro
+LinkpadBot
 LinksManager
-LinqiaMetadataDownloaderBot
 LinqiaRSSBot
 Lipperhey
 Litemage_walker
-Ltx71
 MFC_Tear_Sample
-MIDown tool
 MIIxpc
-MJ12bot
 MQQBrowser
 Mag-Net
-Majestic-SEO
+Magnet
+Majestic SEO
 Majestic12
-MarkMonitor
 Mass Downloader
-Mata Hari
 MeanPath Bot
-Meanpathbot
-Microsoft URL Control
-Mojeek
+Metauri
+MicroMessenger
+Microsoft Data Access
+Mister PiX
+Moblie Safari
 Mojolicious
 Mr.4x3
-Msrabot
 Musobot
 NICErsPRO
-NPbot
-Navroad
-Net Vampire
+Nameprotect
+Nessus
 NetMechanic
-NetSpider
-Netvibes
+Netcraft
+Nibbler
+Niki-bot
 NimbleCrawler
-OnCrawl
-OpenLinkProfiler
-OpenVAS
+Nimbostratus
+Ninja
+Nutch
+Octopus
+Offline Explorer
 OrangeBot
-OutfoxBot
-PHPCrawl
-POE-Component-Client-HTTP
-PageScorer
+OutclicksBot
+PECL::HTTP
+PageAnalyzer
+PageGrabber
 PageThing.com
-Pandalytics
 Pavuk
-Picsearch
-PictureFinder
-ProWebWalker
-Probethenet
+Petalbot
+Piepmatz
+Pixray
+Psbot
 Pu_iN
-Pump
 PxBroker
-PyCurl
-QueryN Metasearch
-RSSingBot
 RankActive
 RankActiveLinkBot
-RankFlex
-Reaper
+RankurBot
+ReGet
+RebelMouse
+Recorder
 RedesScrapy
-Rogerbot
-SBIder
-SEOkicks
+RocketCrawler
 SEOstats
 SalesIntelligent
 Screaming
 ScrepyBot
 SearchmetricsBot
-SentiBot
 SeoSiteCheckup
-Siphon
-SiteExplorer
-SiteLockSpider
+Shodan
+Site Sucker
+SiteCheckerBotCrawler
 SiteSnagger
 Sitebeam
 Siteimprove
-SocialRankIOBot
-Sociscraper
-Sogou web spider
-Sosospider
-Sottopop
+Sitevigil
+SmartDownload
+Snapbot
+Spbot
 Spinn3r
+SputnikBot
 Sqlmap
-Steeler
-Sucuri
+Stripper
 SuperHTTP
+SurveyBot
 Suzuran
+Swiftbot
+Szukacz
 T0PHackTeam
 Teleport
 Telesphoreo
-The Intraformant
-Thumbor
 TightTwatBot
-Titan
-Toweyabot
-Trendictionbot
+Toata
 True_Robot
-Turnitin
+Turingos
+TurnitinBot
+Twice
 URLy Warning
+UnisterBot
 V-BOT
 VB Project
-VCI
 Vacuum
-VeriCiteCrawler
+VelenPublicWebCrawler
 VidibleScraper
 Virusdie
-Wallpapers/3.0
+VoidEYE
+Voil
+WASALive-Bot
+WEBDAV
+WISENutbot
 WallpapersHD
 Web Auto
 Web Collage
+Web Fuck
+Web Pix
 Web Sucker
-WebAuto
-WebFetch
-WebFuck
-WebImageCollector
+WebBandit
+WebCollage
+WebGo IS
 WebLeacher
-WebReaper
-WebSauger
-WebWhacker
+WebPix
 Webalta
-WebmasterWorldForumBot
 Website Quester
-WebsiteExtractor
 Whack
+Whacker
+Whatweb
 Who.is Bot
 WinHTTrack
-WiseGuys Robot
-Wprecon
+Xaldon WebSpider
 YoudaoBot
-ZoominfoBot
-coccocbot-web
-crawl.sogou.com
-evc-batch
-instabid
-internetVista monitor
+Zermelo
+Zeus
+ZoomBot
+ZumBot
+archive.org_bot
+backlink-check
+check1.exe
+crawler4j
+demandbase-bot
+facebookscraper
+heritrix
 isitwp.com
-lwp-request
 magpie-crawler
 mediawords
-muhstik-scan
-oBot
-plumanalytics
+netEstate NE Crawler
+pcBrowser
 probe-image-size
+s1z.ru
 satoristudio.net
-scalaj-http
-scan.lol
-seobility
-seostar
-sexsearcher
+seoscanners
+serpstatbot
 sogouspider
-sysscan
-tAkeOut
-trendiction.de
-voyagerx.com
+sp_auditbot
+trendiction.com
+webmeup-crawler
+xpymep1.exe
 zauba.io
-zgrab