V4.2022.08.3296 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-360Spider
-404checker
 404enemy
-ALittle Client
-Aboundexbot
-AiHitBot
+AIBOT
+ASPSeek
+Abonti
+AfD-Verbotsverfahren
+Aipbot
 Alexibot
-AllSubmitter
-Anarchy
-Anarchy99
-Anthill
+Alligator
+AlphaBot
+Anarchie
 Apexoo
-Attach
-AwarioRssBot
-BDCbot
-BLEXBot
-BacklinkCrawler
+Aspiegel
+Asterias
+BDFetch
+BackStreet
+Bandit
 Barkrowler
+BatchFTP
 Battleztar Bazinga
-BlackWidow
-Blackboard
-BlowFish
+BetaBot
+Bigfoot
+Bitacle
+Black Hole
 Boardreader
-Bolt
-Brandprotect
-Brandwatch
-Buck
-CATExplorador
+BotALot
+Buddy
+BunnySlippers
 CCBot
 Calculon
-Cegbfeieh
+ChinaClaw
+Cogentbot
 Copier
-CopyRightCheck
-Craftbot
-DBLBot
-DSearch
-DataCha0s
-Dispatch
+Copyscape
+Curious
+Custo
+CyotekWebCopy
+DIIbot
+Demon
+Deusu
 DittoSpyder
-DnyzBot
-DomainCrawler
-DomainSigmaCrawler
+DomCopBot
+DomainAppender
 DomainStatsBot
-Drip
+Download Wonder
 ECCP/1.0
+EMail Siphon
 EasyDL
-Ecxi
+EroCrawler
+Evil
 Exabot
-Express WebPictures
 ExtLinksBot
-Extreme Picture Finder
-EyeNetIE
 FHscan
 FemtosearchBot
-Firefox/7.0
-FyberSpider
-G-i-g-a-b-o-t
-GT::WWW
+Fimap
+FlashGet
+Fuzz
 GalaxyBot
-Genieo
-Gigabot
+GermCrawler
+GetRight
+GetWeb
+Go!Zilla
 Go-Ahead-Got-It
-Gotit
-GrabNet
-Grafula
+Grabber
+GrapeFX
+GrapeshotCrawler
 HEADMasterSEO
-HTMLparser
-HTTrack
+HMView
+HTTP::Lite
 Harvest
 Havij
 Heritrix
 Hloader
-Humanlinks
-Iblog
-Id-search
+Image Fetch
 IndeedBot
 Indy Library
+InfoNaviRobot
 InfoTekies
-Internet Ninja
-JOC Web Spider
-Jbrofuzz
+InterGET
+InternetSeer
+JamesBOT
+JennyBot
 JetCar
-Jetty
-JikeSpider
-JustView
-Jyxobot
+Joomla
 Kenjin Spider
 Keyword Density
-LNSpiderguy
-Larbin
-Libwhisker
-LieBaoFast
-Lightspeedsystems
-Likse
-Linkbot
+Kozmosbot
+LWP::Simple
+LexiBot
+LibWeb
+LinkWalker
 LinqiaScrapeBot
+Lipperhey
 Lipperhey Spider
-Lmspider
-MIDown tool
-MIIxpc
+Litemage_walker
+MFC_Tear_Sample
+MJ12bot
 MQQBrowser
-Mag-Net
-Magnet
-Majestic SEO
-Mass Downloader
-MauiBot
+MTRobot
+Mail.RU_Bot
+Majestic12
 Mb2345Browser
-Microsoft Data Access
-Microsoft URL Control
-Mister PiX
-Moblie Safari
+Metauri
+MicroMessenger
+Minefield
 Mojeek
-Mojolicious
-MolokaiBot
-Mr.4x3
+Msrabot
+Musobot
 NPbot
-Name Intelligence
-Nessus
-Net Vampire
-NetLyzer
-Netcraft
-Nettrack
-NextGenSearchBot
+Nameprotect
+NearSite
+NetSpider
+NetZIP
+Netvibes
 Nibbler
-NimbleCrawler
-Nimbostratus
-Nmap
+Niki-bot
+Ninja
 Nutch
+Octopus
 Offline Navigator
 OnCrawl
-Openfind
+Openvas
+OrangeBot
 OrangeSpider
 OutclicksBot
-PECL::HTTP
-PHPCrawl
-Page Analyzer
-PageAnalyzer
+OutfoxBot
+POE-Component-Client-HTTP
+PageGrabber
+PageScorer
+PageThing.com
+Papa Foto
 PeoplePal
-Petalbot
+Picsearch
+PictureFinder
+Pimonster
 Pixray
-PleaseCrawl
 Pockey
 ProWebWalker
-Psbot
 Pu_iN
-PxBroker
-PyCurl
+Pump
 QueryN Metasearch
-RSSingBot
 RankActive
+RankActiveLinkBot
+RankFlex
+RankurBot
 Re-re
-RebelMouse
+ReGet
 RepoMonkey
 Ripper
 RocketCrawler
+Rogerbot
+SBIder
+SEOkicks
+SEOlyticsCrawler
 SISTRIX
-SalesIntelligent
-ScanAlert
-Screaming
+ScoutJet
+Scrapy
 ScrepyBot
-Searchestate
+SearchmetricsBot
 Seekport
-SemanticJuice
+SentiBot
+Seomoz
 Siphon
-SiteLockSpider
+Site Sucker
+SiteSnagger
+SiteSucker
+Sitebeam
 Siteimprove
-Snoopy
+Sitevigil
+SmartDownload
 Sociscraper
+Sogou web spider
 Sosospider
 SpaceBison
-Spammen
 SpankBot
-SputnikBot
+Spbot
+Sqlworm
 Sqworm
-SuperBot
-SuperHTTP
-Surfbot
-T8Abot
-TeleportPro
-Telesoft
-Telesphorep
-The Intraformant
-TheNomad
-TightTwatBot
-Toweyabot
+Stripper
+Telesphoreo
 Tracemyfile
-True_Robot
-Turingos
+Trendiction
+Turnitin
 TurnitinBot
-Twice
+Typhoeus
+URLy Warning
 UnisterBot
-V-BOT
-Vacuum
-Vagabondo
-VelenPublicWebCrawler
-VidibleScraper
+Upflow
+VB Project
+VCI
 VoidEYE
-WBSearchBot
-WISENutbot
-WPScan
-WWW-Mechanize
-Web Collage
+Voil
+Wallpapers
+Web Auto
 Web Enhancer
-Web Pix
+Web Sauger
 WebBandit
 WebCopier
+WebFetch
 WebFuck
-WebReaper
-WebSauger
+WebGo IS
+WebImageCollector
+WebStripper
 WebSucker
-WebWhacker
 Webalta
 WebmasterWorldForumBot
-WebsiteQuester
-Whacker
+Website Quester
+Whatweb
 Who.is Bot
+Widow
 WinHTTrack
 Wonderbot
-Woobot
+Xaldon WebSpider
+Xaldon_WebSpider
+Xenu
+YoudaoBot
 Zade
 Zermelo
-Zitebot
 ZmEu
-ZyBorg
-arquivo.pt
+ZumBot
+adscanner
+arquivo-web-crawler
 backlink-check
 cah.io.community
-clark-crawler
-crawl.sogou.com
+com.plumanalytics
+crawler.feedback
 crawler4j
-domainsproject.org
-facebookscraper
-gopher
-isitwp.com
-iubenda-radar
-muhstik-scan
-netEstate NE Crawler
-probe-image-size
-s1z.ru
+dataforseo.com
+evc-batch
+heritrix
+instabid
+lwp-trivial
+magpie-crawler
+meanpathbot
+mediawords
+pcBrowser
+polaris version
+scalaj-http
 scan.lol
+seocompany.store
 seoscanners
-seostar
+serpstatbot
 siteripz
-trendiction.de
-ubermetrics-technologies.com
-webgains-bot
+spyfu
+sysscan
+trendiction.com
+webpros.com
+webprosbot
 x09Mozilla
-xpymep1.exe
 zgrab