V4.2022.03.3011 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-ADmantX
+80legs
+AIBOT
+ALittle Client
 ASPSeek
-Abonti
-Acunetix
-AhrefsBot
+Aboundexbot
+AfD-Verbotsverfahren
 AiHitBot
 AllSubmitter
-Alligator
-Anthill
+AlphaBot
+Anarchie
+Anarchy
 Atomseobot
-AwarioRssBot
+Attach
 AwarioSmartBot
 BBBike
+BDCbot
+BLEXBot
+BackDoorBot
+BackStreet
+BackWeb
+Backlink-Ceck
+BacklinkCrawler
 Badass
-Barkrowler
+Bigfoot
 Bitacle
-Black Hole
-BlackWidow
-Bolt
-BotALot
+Boardreader
+Brandprotect
 Buck
+BuiltBotTough
 BuiltWith
-CODE87
-CSHttp
-Calculon
-CazoodleBot
+BunnySlippers
+CATExplorador
+CCBot
 CensysInspect
-Cocolyzebot
+CheeseBot
+ChinaClaw
+Chlooe
+Cliqzbot
+Cloud mapping
 Cogentbot
 Collector
-Copier
-CopyRightCheck
 Copyscape
-Cosmos
-Craftbot
-CrazyWebCrawler
+Crawling at Home Project
 CrunchBot
 Curious
-Custo
 CyotekWebCopy
+DSearch
 Demon
-Digincore
-DigitalPebble
-DomCopBot
-EMail Wolf
-Ebingbong
-EroCrawler
-Extreme Picture Finder
+Deusu
+Dirbuster
+Disco
+Discobot
+DomainSigmaCrawler
+Domains Project
+Dotbot
+Download Wonder
+Dragonfly
+Drip
+EMail Siphon
 FHscan
-Firefox/7.0
+Fimap
 FlashGet
-Flunky
 Foobot
-FyberSpider
-Fyrebot
+Freeuploader
+FrontPage
 G-i-g-a-b-o-t
-GalaxyBot
+Genieo
 GermCrawler
-GetRight
-GetWeb
-Go-Ahead-Got-It
-Gotit
+Grabber
 GrapeFX
 GrapeshotCrawler
-HMView
+HEADMasterSEO
 HTTP::Lite
 HTTrack
-Haansoft
+Harvest
 Havij
-IDBTE4M
-IDBot
-IRLbot
+Heritrix
+HonoluluBot
+HybridBot
 Iblog
-Id-search
+IlseBot
 Image Fetch
-InterGET
+Image Sucker
+Indy Library
+InfoNaviRobot
+InfoTekies
+Intelliseek
+Internet Ninja
+Iria
+Iskanie
+JamesBOT
 JennyBot
-JetCar
-JikeSpider
+Joomla
 Jorgee
-Jyxobot
-Keybot Translation-Search-Machine
+JustView
+Kenjin Spider
 Keyword Density
-Kinza
-Kozmosbot
-LNSpiderguy
 Lanshanbot
-Larbin
-LeechFTP
-LexiBot
-LieBaoFast
-Lightspeedsystems
-Likse
-Linkdexbot
-LinkextractorPro
-LinkpadBot
+Leap
+Libwhisker
+LinkScan
 LinksManager
-LinqiaRSSBot
-LinqiaScrapeBot
 Lipperhey
-Lipperhey Spider
-Litemage_walker
-MIDown tool
+Lmspider
 MJ12bot
-MQQBrowser
-Magnet
-Majestic SEO
-Majestic-SEO
-MauiBot
+MSFrontPage
+MSIECrawler
+MTRobot
+MarkMonitor
+Mata Hari
 Mb2345Browser
-MeanPath Bot
-Meanpathbot
 MegaIndex.ru
 Metauri
-Minefield
-Moblie Safari
-Mojeek
+Microsoft Data Access
+Microsoft URL Control
+Mister PiX
 MolokaiBot
-Msrabot
-Musobot
-Navroad
+Morfeus Fucking Scanner
+Mozlila
+Mr.4x3
+Nameprotect
+NearSite
 Needle
-Net Vampire
-NetAnts
-NetLyzer
+Nessus
 NetSpider
 Netcraft
 Nettrack
+Netvibes
 NextGenSearchBot
 Nibbler
 Niki-bot
+Nmap
 Not
+Nutch
 Octopus
 Offline Navigator
-OpenLinkProfiler
-OpenVAS
-POE-Component-Client-HTTP
+OnCrawl
+Openvas
+OrangeBot
+OrangeSpider
+OutfoxBot
 Page Analyzer
 PageAnalyzer
+PageGrabber
 PageThing.com
+Pandalytics
 Panscient
 Petalbot
-Pi-Monster
-PictureFinder
+Picsearch
 Piepmatz
 Pockey
-ProPowerBot
 Probethenet
-Pump
+Psbot
+PxBroker
 QueryN Metasearch
 Quick-Crawler
-Rankivabot
-Re-re
-ReGet
-RealDownload
+RSSingBot
+RankFlex
+RankingBot
+RankingBot2
+RebelMouse
+Recorder
 RedesScrapy
-RepoMonkey
 RocketCrawler
-SBIder
-SEOkicks-Robot
+SEOkicks
 SEOlyticsCrawler
-SEOstats
 SISTRIX
+SalesIntelligent
 ScanAlert
-Screaming
-Seomoz
-Shodan
-Site Sucker
-SiteExplorer
-SiteSucker
-Sitebeam
+Scrapy
+SemanticJuice
+SeoSiteCheckup
+Siphon
+Siteimprove
+SmartDownload
+Snake
 Snapbot
-Sociscraper
-Spbot
+SocialRankIOBot
+SpaceBison
+Spammen
+SpankBot
+Spanner
 SputnikBot
-Sqworm
-Steeler
-Sucker
+Sqlworm
+Stripper
 Sucuri
-SuperBot
 SurveyBot
-Suzuran
-Swiftbot
-TeleportPro
+T0PHackTeam
+Teleport
 Telesphoreo
-Telesphorep
-TightTwatBot
-Titan
+TheNomad
+Thumbor
 Toweyabot
+Tracemyfile
 Trendictionbot
-True_Robot
 Turnitin
-UnisterBot
-Vacuum
-VidibleScraper
+URLy Warning
+VB Project
+Vagabondo
+VelenPublicWebCrawler
+Virusdie
 VoidEYE
-WWW-Collector-E
-WWWOFFLE
+Voltron
+WISENutbot
 Wallpapers
-WeSEE
+Wallpapers/3.0
+WallpapersHD
 Web Fetch
-Web Fuck
-Web Pix
 Web Sauger
 Web Sucker
-WebBandit
 WebCollage
+WebFetch
 WebPix
-WebSauger
 WebStripper
-WebZIP
-Webalta
-Webshag
-WebsiteQuester
-Whack
+WebmasterWorldForumBot
+Whatweb
 Widow
 WiseGuys Robot
+Woobot
 Wotbox
-Xaldon WebSpider
 Xaldon_WebSpider
 YoudaoBot
-Zauba
 Zeus
-ZumBot
-ZyBorg
-arquivo-web-crawler
-arquivo.pt
+ZoominfoBot
+archive.org_bot
 autoemailspider
-crawler.feedback
 dataforseo.com
-domainsproject.org
-eCatch
-facebookscraper
+demandbase-bot
+evc-batch
+heritrix
 instabid
-meanpathbot
-oBot
-polaris version
-probe-image-size
+internetVista monitor
+isitwp.com
+iubenda-radar
+lwp-request
+page scorer
 ripz
-scan.lol
+seobility
+seocompany.store
 seoscanners
-seostar
 sexsearcher
 sogouspider
-sysscan
+sp_auditbot
 trendiction.com
 trendiction.de
-voyagerx.com
-zgrab
+webgains-bot
+webpros.com
+webprosbot
+x09Mozilla
+x22Mozilla