V4.2022.02.2947 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
 360Spider
 404checker
-AIBOT
-ALittle Client
-Abonti
-Aboundexbot
-Acunetix
-AhrefsBot
-Aipbot
-AllSubmitter
-Atomseobot
-Attach
+80legs
+Aboundex
+AiHitBot
+Anarchie
+Anarchy
+Anarchy99
+Anthill
+Asterias
+BBBike
 BDFetch
-BLEXBot
-BackStreet
-Backlink-Ceck
+BackDoorBot
+BackWeb
 Badass
-BatchFTP
+Bandit
+Barkrowler
+Bigfoot
+Bitacle
 Black Hole
-Brandwatch
-Buck
+Blow
+Boardreader
+Bolt
+BotALot
 Buddy
 BuiltWith
 BunnySlippers
-CODE87
+CCBot
 CSHttp
-CazoodleBot
+Calculon
 Cegbfeieh
-CherryPicker
+CheTeam
 Claritybot
 Cloud mapping
-CopyRightCheck
-Copyscape
+Cogentbot
 Crawling at Home Project
-Devil
-Discoverybot
-DnyzBot
-DomCopBot
-DomainCrawler
+CrazyWebCrawler
+Crescent
+CyotekWebCopy
+DTS Agent
+DataCha0s
+Deusu
+Digincore
+DigitalPebble
+Disco
+Discobot
+DittoSpyder
+DomainAppender
 DomainSigmaCrawler
-Domains Project
-Dotbot
-Drip
-ECCP/1.0
+Download Wonder
 EMail Siphon
-Ecxi
+Ebingbong
 EroCrawler
+Evil
 Exabot
 ExtLinksBot
 ExtractorPro
-FDM
+Ezooms
+Fimap
 Firefox/7.0
+FlashGet
 Flunky
+Foobot
 Freeuploader
-GT::WWW
-GermCrawler
-Gigabot
+GalaxyBot
+GetRight
+GetWeb
 Go!Zilla
+Go-Ahead-Got-It
 GoZilla
-Grabber
-HEADMasterSEO
-HTTP::Lite
-HybridBot
-IRLbot
-Iblog
+GrabNet
+GridBot
+Haansoft
+HaosouSpider
+Havij
+Hloader
+Id-search
 Image Fetch
-Image Sucker
-Indy Library
 InfoNaviRobot
-Intelliseek
-Internet Ninja
+InfoTekies
 InternetSeer
-IstellaBot
-JOC Web Spider
-JamesBOT
-Jbrofuzz
-JetCar
-JustView
-Jyxobot
-Kenjin Spider
-LNSpiderguy
+Iskanie
+Joomla
+Keyword Density
+Kozmosbot
+Lanshanbot
 LeechFTP
+LeechGet
 LexiBot
 Lftp
-LinkWalker
-Linkbot
+LieBaoFast
+LinkextractorPro
 LinkpadBot
 LinksManager
 LinqiaMetadataDownloaderBot
 Lipperhey Spider
 Ltx71
 MFC_Tear_Sample
+MJ12bot
 MQQBrowser
-MSFrontPage
+MTRobot
 Mag-Net
-Magnet
-MarkMonitor
-Mass Downloader
+Majestic SEO
+Majestic12
 Masscan
 Mata Hari
-MeanPath Bot
+Mb2345Browser
+Meanpathbot
 Mediatoolkitbot
+MegaIndex.ru
 Metauri
-MicroMessenger
-Microsoft Data Access
-Microsoft URL Control
 Minefield
-Mister PiX
-Morfeus Fucking Scanner
+Mr.4x3
 Msrabot
-Musobot
-NICErsPRO
+Name Intelligence
+Nameprotect
+NearSite
+Net Vampire
 NetLyzer
-NetMechanic
-NetSpider
 Netcraft
+Nettrack
+NextGenSearchBot
+Niki-bot
+Nikto
+Nimbostratus
 Nmap
-Nuclei
+Not
+Octopus
 Offline Explorer
-Openvas
+Offline Navigator
+OpenLinkProfiler
 OrangeBot
-OrangeSpider
+OutclicksBot
 OutfoxBot
-PHPCrawl
 POE-Component-Client-HTTP
-PageAnalyzer
-PageScorer
+Page Analyzer
 Pandalytics
-Picsearch
-PictureFinder
-Pixray
+PeoplePal
+Pi-Monster
+Picscout
+PleaseCrawl
 Pockey
-ProPowerBot
-Probethenet
-Pump
+ProWebWalker
+Psbot
 PxBroker
+QueryN Metasearch
+Quick-Crawler
 RSSingBot
-RankingBot
-RankingBot2
-Reaper
-SEOkicks-Robot
+RankActiveLinkBot
+RankurBot
+Re-re
+ReGet
+RedesScrapy
+Ripper
+RocketCrawler
+SEOkicks
 SEOprofiler
-SISTRIX
+SEOstats
+ScoutJet
+Scrapy
+ScrepyBot
 Searchestate
 SearchmetricsBot
 Seekport
 Semrush
 SentiBot
-SeoSiteCheckup
-Site Sucker
-SiteCheckerBotCrawler
-SiteExplorer
+SiteLockSpider
 SiteSnagger
 SiteSucker
 Sitevigil
 SlySearch
-Snake
+Snapbot
 Snoopy
-Sociscraper
-SpaceBison
-Spammen
+Spbot
 Spinn3r
-Sqlmap
-Steeler
-Sucker
-SuperBot
-SuperHTTP
+Sqlworm
+Sucuri
 SurveyBot
-T8Abot
 Teleport
-TeleportPro
-Telesphorep
-Thumbor
-Toata
+Telesoft
+Tracemyfile
+Trendiction
 Trendictionbot
-Turnitin
+True_Robot
+Turingos
+TwengaBot
+Twice
+Typhoeus
 URLy.Warning
-Upflow
+V-BOT
 VB Project
-VCI
 Vacuum
 VidibleScraper
-Voil
-WASALive-Bot
-WWW-Collector-E
+Voltron
+WISENutbot
 WWW::Mechanize
-WWWOFFLE
 Wallpapers
-WallpapersHD
-Web Collage
-Web Fetch
-Web Fuck
-Web Sauger
-WebAuto
-WebCollage
+WeSEE
+Web Auto
+Web Sucker
+WebBandit
+WebCopier
+WebEnhancer
+WebFuck
 WebGo IS
-WebImageCollector
-WebSucker
+WebLeacher
+WebStripper
 WebWhacker
 WebZIP
+Webalta
 WebmasterWorldForumBot
-Webshag
-Webster
+Website Quester
+WebsiteExtractor
+WebsiteQuester
 Whack
-Whacker
-Widow
+Who.is Bot
+WiseGuys Robot
 Woobot
-Xaldon WebSpider
-Xaldon_WebSpider
-Xenu
-YoudaoBot
-Zauba
 Zermelo
+Zitebot
+ZmEu
 ZoomBot
 ZoominfoBot
+ZumBot
 ZyBorg
-archive.org_bot
+adscanner
 autoemailspider
-cah.io.community
-cognitiveseo
-crawl.sogou.com
-crawler4j
+clark-crawler
+crawler.feedback
 dataforseo.com
-demandbase-bot
-eCatch
+domainsproject.org
 evc-batch
+heritrix
 instabid
-ips-agent
+isitwp.com
 iubenda-radar
 lwp-trivial
 muhstik-scan
 netEstate NE Crawler
-plumanalytics
-probe-image-size
-scan.lol
-seobility
-serpstatbot
-sexsearcher
-sitechecker.pro
-sogouspider
+page scorer
+s1z.ru
+scalaj-http
+seocompany.store
+seoscanners
 sysscan
-tAkeOut
+trendiction.com
 ubermetrics-technologies.com
-voyagerx.com
 webgains-bot
-webpros.com
-webprosbot
-x09Mozilla
-zauba.io
+x22Mozilla