V4.2021.12.2799 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-360Spider
-404enemy
+404checker
+80legs
+AIBOT
+ALittle Client
+Abonti
 Aboundex
 Acunetix
-AfD-Verbotsverfahren
 AhrefsBot
-AiHitBot
-Aipbot
-AllSubmitter
 Alligator
-Anarchie
-Anarchy99
+AlphaBot
+Anarchy
+Anthill
 Apexoo
+Aspiegel
 Attach
-BBBike
+AwarioRssBot
+BLEXBot
 BackStreet
 Badass
 Barkrowler
-Black Hole
-Blow
-Boardreader
+Battleztar Bazinga
+Bigfoot
+Blackboard
+Bolt
 BotALot
+Brandwatch
 Buck
+Buddy
 BuiltWith
 Bullseye
 BunnySlippers
-CCBot
-CODE87
 Calculon
 CazoodleBot
 Cegbfeieh
-CensysInspect
-CheTeam
-Citoid
+ChinaClaw
+Chlooe
 Claritybot
+Cloud mapping
 Copier
-Crawling at Home Project
+Copyscape
+Craftbot
 CrazyWebCrawler
-Crescent
-Custo
 CyotekWebCopy
-DBLBot
-DIIbot
+DSearch
 DTS Agent
-Demon
-Deusu
-DigitalPebble
-Discobot
+DataCha0s
+Dirbuster
+Disco
 Dispatch
-DittoSpyder
 DnBCrawler-Analytics
-DomCopBot
-Download Wonder
+DomainSigmaCrawler
+Dotbot
 ECCP/1.0
-EMail Siphon
 EMail Wolf
 EasyDL
-Ecxi
+Ebingbong
+EirGrabber
 EroCrawler
-Exabot
 Express WebPictures
-Ezooms
-FDM
-FHscan
-FemtosearchBot
 Fimap
 Firefox/7.0
-FlashGet
-Foobot
+Flunky
 Freeuploader
+FrontPage
+Fuzz
+FyberSpider
+Fyrebot
+GT::WWW
+GalaxyBot
+GetWeb
 Getintent
 Gigabot
+Go!Zilla
 Go-Ahead-Got-It
+GoZilla
 GrabNet
-Grabber
-Grafula
 GrapeshotCrawler
 HEADMasterSEO
-HTMLparser
 Haansoft
+HaosouSpider
 Harvest
+Humanlinks
+IDBot
 IRLbot
 IlseBot
-Intelliseek
-InterGET
+IndeedBot
+InfoNaviRobot
+InfoTekies
+InternetSeer
 Iria
-JOC Web Spider
-Jetty
+JennyBot
+JetCar
+JikeSpider
 Joomla
-Jyxobot
+Jorgee
 Kenjin Spider
 Kozmosbot
-LNSpiderguy
-LWP::Simple
-Lanshanbot
+LeechFTP
 LeechGet
-LibWeb
+LexiBot
 Libwhisker
-Lightspeedsystems
 Linkdexbot
+LinkextractorPro
 LinksManager
 LinqiaRSSBot
-LinqiaScrapeBot
-Lipperhey Spider
+Lipperhey
 Lmspider
 Ltx71
 MFC_Tear_Sample
-MIIxpc
-MQQBrowser
+MJ12bot
+MSIECrawler
 MTRobot
+Mag-Net
+Magnet
 Majestic SEO
-MarkMonitor
+Majestic12
 MarkWatch
-Mass Downloader
+Masscan
+Mata Hari
 MauiBot
 MeanPath Bot
-MegaIndex.ru
+MicroMessenger
+Microsoft Data Access
+Microsoft URL Control
 Mister PiX
-Moblie Safari
-Morfeus Fucking Scanner
+Mojeek
+Mojolicious
+Mr.4x3
 Msrabot
 NICErsPRO
+NPbot
 Nameprotect
 NearSite
-Net Vampire
+Needle
+NetLyzer
+NetMechanic
+NetSpider
 NetZIP
-Netcraft
-Nettrack
 NextGenSearchBot
 Nikto
+NimbleCrawler
 Nimbostratus
-Ninja
-Not
-Nutch
-Octopus
-Offline Navigator
+Openfind
 OutclicksBot
-OutfoxBot
-PageAnalyzer
-PageScorer
-Pandalytics
-Papa Foto
-PeoplePal
+POE-Component-Client-HTTP
+Page Analyzer
 Pi-Monster
+Picscout
+Picsearch
 PictureFinder
+Pimonster
 Pixray
+PleaseCrawl
+Pockey
 ProPowerBot
 Probethenet
+Psbot
 PxBroker
-PyCurl
-Quick-Crawler
-RSSingBot
-RankFlex
-RankingBot2
-Rankivabot
-RocketCrawler
-SEOkicks
-SEOkicks-Robot
-SMTBot
-Scanbot
-ScoutJet
-SearchmetricsBot
+QueryN Metasearch
+RankActive
+RankurBot
+Re-re
+ReGet
+RepoMonkey
+Ripper
+Rogerbot
+SEOlyticsCrawler
+SEOstats
+Scrapy
+Screaming
+ScreenerBot
+ScrepyBot
 SemanticJuice
 Semrush
-SemrushBot
-SeoSiteCheckup
-Siphon
-Site Sucker
-SiteCheckerBotCrawler
+Seomoz
 SiteLockSpider
-SiteSnagger
-Sitevigil
-SlySearch
-Snake
+Siteimprove
+SmartDownload
 Snoopy
-SocialRankIOBot
-Sogou web spider
-SpankBot
+Sociscraper
+Sottopop
+Spanner
 Spbot
-Spinn3r
-Sqworm
-Stripper
+SputnikBot
+Sqlmap
 SuperBot
+Surfbot
 SurveyBot
 Szukacz
 Teleport
-TheNomad
-Thumbor
+The Intraformant
 TightTwatBot
-Titan
 Tracemyfile
+Trendiction
 Trendictionbot
-Vacuum
+True_Robot
+TurnitinBot
+TwengaBot
+Twice
+URLy Warning
+URLy.Warning
+Upflow
+V-BOT
+VB Project
+VelenPublicWebCrawler
+VidibleScraper
 Virusdie
+WBSearchBot
+WEBDAV
+WWW-Collector-E
+WWW-Mechanize
 WWW::Mechanize
-Wallpapers/3.0
-WallpapersHD
-Web Enhancer
-Web Fuck
-Web Sucker
-WebAuto
-WebCollage
-WebLeacher
-WebStripper
+WWWOFFLE
+Wallpapers
+WeSEE
+Web Auto
+Web Sauger
+WebImageCollector
+WebPix
+WebReaper
+WebSauger
+WebZIP
 Website Quester
-WebsiteExtractor
-WebsiteQuester
-Whack
-Who.is Bot
 Widow
-Woobot
+WinHTTrack
 Wotbox
-Wprecon
-Xaldon WebSpider
 Xaldon_WebSpider
 YoudaoBot
 Zade
-Zauba
 Zeus
+ZmEu
+ZoomBot
 ZumBot
-check1.exe
-clark-crawler
-coccocbot-web
-cognitiveseo
-crawler.feedback
-domainsproject.org
-facebookscraper
-heritrix
-isitwp.com
-lwp-request
-meanpathbot
-mediawords
+autoemailspider
+backlink-check
+com.plumanalytics
+dataforseo.com
+eCatch
+evc-batch
+magpie-crawler
 muhstik-scan
+netEstate NE Crawler
 plumanalytics
-ripz
-s1z.ru
-scalaj-http
-seoscanners
-seostar
-serpstatbot
-sexsearcher
-siteripz
-sogouspider
-sp_auditbot
-trendiction.de
+scan.lol
+seocompany.store
+spyfu
+tAkeOut
 ubermetrics-technologies.com
-voyagerx.com
 webgains-bot
 webmeup-crawler
-x09Mozilla