V4.2021.09.2577 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-404enemy
-80legs
+404checker
+ASPSeek
+Abonti
 Aboundex
+AfD-Verbotsverfahren
 AhrefsBot
 AiHitBot
-AllSubmitter
-Alligator
+Aipbot
 AlphaBot
-Anarchy
-Anarchy99
+Apexoo
 Aspiegel
-Asterias
 Attach
-AwarioSmartBot
-BDCbot
-BDFetch
+AwarioRssBot
+BBBike
+BackStreet
+BackWeb
+Backlink-Ceck
 BacklinkCrawler
-Badass
-Bandit
 BatchFTP
-Battleztar Bazinga
-Bigfoot
-Blackboard
-Blow
+BlowFish
+BotALot
+Brandprotect
 Brandwatch
-BuiltBotTough
-Bullseye
+BunnySlippers
 BuzzSumo
 CATExplorador
 CazoodleBot
 CensysInspect
-CheTeam
-CherryPicker
-ChinaClaw
-Claritybot
-Cliqzbot
+CheeseBot
 Cocolyzebot
-Collector
-Copier
 CopyRightCheck
-Copyscape
-Craftbot
+Cosmos
+Crawling at Home Project
 CrazyWebCrawler
-Crescent
-CyotekWebCopy
-DBLBot
-DSearch
-Demon
-Devil
+Curious
+DTS Agent
+DataCha0s
+Deusu
+Digincore
 DigitalPebble
-Dirbuster
-Discobot
-Dispatch
-DnyzBot
-DomCopBot
-DomainCrawler
+Disco
+Discoverybot
+DittoSpyder
 DomainSigmaCrawler
 DomainStatsBot
-Dotbot
+Domains Project
 Download Wonder
-Drip
 ECCP/1.0
+Ebingbong
 EirGrabber
-ExtLinksBot
-ExtractorPro
+Express WebPictures
+Extractor
 Extreme Picture Finder
-Fuzz
-Fyrebot
+FemtosearchBot
+Fimap
+Firefox/7.0
+Flunky
+Foobot
+FrontPage
 G-i-g-a-b-o-t
-GalaxyBot
-Genieo
-Getintent
+GT::WWW
+GetRight
+GetWeb
 Gigabot
 Go!Zilla
+GoZilla
 GrabNet
-Grabber
-Grafula
+GrapeshotCrawler
 HEADMasterSEO
-HTMLparser
-Havij
-Hloader
-Humanlinks
-IDBTE4M
-Iblog
-Id-search
-IndeedBot
+HTTrack
+Harvest
+Heritrix
+HybridBot
 Indy Library
-InfoNaviRobot
 InfoTekies
-InternetSeer
+InterGET
 Iskanie
-JOC Web Spider
-JamesBOT
+Jbrofuzz
 JennyBot
-JetCar
+Jetty
 JikeSpider
-Joomla
-Kenjin Spider
+Jorgee
+JustView
 Kozmosbot
-Larbin
-Leap
-LeechGet
+LNSpiderguy
+LeechFTP
+LexiBot
 Libwhisker
-LieBaoFast
+Likse
 Linkbot
-LinkpadBot
-LinksManager
-LinqiaMetadataDownloaderBot
+LinkextractorPro
 LinqiaScrapeBot
-Lmspider
-MJ12bot
+Lipperhey
+Lipperhey Spider
+MIDown tool
+MIIxpc
+MTRobot
 Mag-Net
-Magnet
-MarkMonitor
+Mail.RU_Bot
+Majestic-SEO
 Mass Downloader
-Mata Hari
-MauiBot
+Mb2345Browser
 MeanPath Bot
-MegaIndex.ru
+Mediatoolkitbot
+Microsoft URL Control
 Mojeek
-Mojolicious
-Mozlila
-Mr.4x3
-Msrabot
+Musobot
 NICErsPRO
+NPbot
 Nameprotect
 Navroad
 NearSite
-NetAnts
+NetLyzer
+NetMechanic
 NetSpider
-Netcraft
+NetZIP
 Nettrack
+NextGenSearchBot
+Nibbler
 Nikto
-NimbleCrawler
-Nutch
-Offline Navigator
-OpenVAS
-Openvas
+Nimbostratus
+Octopus
+OnCrawl
 OrangeBot
-PECL::HTTP
-PHPCrawl
-POE-Component-Client-HTTP
-PageGrabber
-Papa Foto
-Petalbot
-Pimonster
-Pixray
-PleaseCrawl
+OutclicksBot
+Page Analyzer
+PageAnalyzer
+PageThing.com
+Pi-Monster
 ProPowerBot
+ProWebWalker
 Psbot
 Pu_iN
-PxBroker
 PyCurl
 QueryN Metasearch
+Quick-Crawler
 RSSingBot
-RankActive
+RankActiveLinkBot
 RankFlex
-ReGet
+RankingBot
 Reaper
+RepoMonkey
+Rogerbot
 SEOkicks
 SEOstats
 SISTRIX
-SMTBot
 ScanAlert
-Scanbot
-ScoutJet
 Screaming
-Searchestate
+ScrepyBot
 SearchmetricsBot
-SentiBot
-SeobilityBot
-Shodan
+Semrush
+Seomoz
+SiteCheckerBotCrawler
+SiteExplorer
 SiteLockSpider
+SiteSucker
 Siteimprove
-Sitevigil
 SlySearch
-Snake
+SmartDownload
 Snapbot
-Sociscraper
+SocialRankIOBot
 Sosospider
 SpaceBison
-SpankBot
-Spanner
-Spbot
-Sucker
-SuperBot
+Spinn3r
+Sqlworm
+Sucuri
+Surfbot
+Szukacz
+T0PHackTeam
 T8Abot
-TightTwatBot
+Telesoft
+Telesphoreo
+The Intraformant
+TheNomad
+Toata
+Toweyabot
 Tracemyfile
+Trendictionbot
 True_Robot
 Turingos
-Twice
-URLy.Warning
+TurnitinBot
+URLy Warning
 UnisterBot
 Upflow
+VB Project
 Vacuum
-Vagabondo
-VeriCiteCrawler
+VelenPublicWebCrawler
 VidibleScraper
 Virusdie
 Voil
-Voltron
-WISENutbot
-WWWOFFLE
-Web Auto
+WASALive-Bot
+WPScan
+WWW-Mechanize
+WWW::Mechanize
+Wallpapers/3.0
+WeSEE
+Web Enhancer
 Web Fetch
-WebAuto
+Web Fuck
+Web Pix
+Web Sucker
 WebEnhancer
 WebFetch
-WebLeacher
+WebFuck
+WebImageCollector
 WebReaper
-WebStripper
 WebSucker
-WebWhacker
-Webalta
-WebmasterWorldForumBot
+WebZIP
+WebsiteExtractor
 WebsiteQuester
-Whack
-Wotbox
+Webster
+WinHTTrack
+WiseGuys Robot
+Wonderbot
+Woobot
 Xaldon WebSpider
+Xaldon_WebSpider
+Xenu
 YoudaoBot
-Zade
-Zauba
+Zitebot
 ZmEu
+ZoominfoBot
+ZumBot
 archive.org_bot
-arquivo-web-crawler
+autoemailspider
+cah.io.community
+check1.exe
+cognitiveseo
+com.plumanalytics
 crawl.sogou.com
-crawler4j
+crawler.feedback
+dataforseo.com
 domainsproject.org
-eCatch
-facebookscraper
 gopher
-internetVista monitor
-isitwp.com
+heritrix
+ips-agent
+lwp-request
 lwp-trivial
 mediawords
-netEstate NE Crawler
-plumanalytics
-satoristudio.net
-seobility
-seoscanners
-serpstatbot
-sexsearcher
-siteripz
-sp_auditbot
+pcBrowser
+s1z.ru
+scalaj-http
+sitechecker.pro
 spyfu
-tAkeOut
-voyagerx.com
+webmeup-crawler
 x09Mozilla
+x22Mozilla
+xpymep1.exe
+zauba.io
+zgrab