V4.2021.04.2328 [ci skip]

.dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
 360Spider
 404checker
-80legs
-Aboundex
+404enemy
+Abonti
 Aboundexbot
-AIBOT
-AiHitBot
+AfD-Verbotsverfahren
 Alexibot
 Alligator
 AllSubmitter
-Anarchy99
+Anarchie
 Ankit
-arquivo.pt
-arquivo-web-crawler
-autoemailspider
-AwarioRssBot
-AwarioSmartBot
+Anthill
+archive.org_bot
+Asterias
 BackDoorBot
-backlink-check
-Barkrowler
-Battleztar Bazinga
-BDFetch
-Brandwatch
-Buck
+BacklinkCrawler
+Badass
+BetaBot
+Bigfoot
+Bitacle
+Blackboard
+BLEXBot
+BlowFish
 Buddy
-BuiltWith
 BunnySlippers
 BuzzSumo
 Calculon
-CazoodleBot
+CATExplorador
 CCBot
-CherryPicker
+CheeseBot
+CheTeam
 Chlooe
-Claritybot
 Cloud mapping
-coccocbot-web
-CODE87
-Cogentbot
+Collector
+com.plumanalytics
 CopyRightCheck
-Craftbot
-CrazyWebCrawler
+Cosmos
+crawler4j
+crawler.feedback
 Crescent
-CSHttp
 Curious
-demandbase-bot
+CyotekWebCopy
+DataCha0s
+DBLBot
+Deusu
 Devil
-DigitalPebble
-DIIbot
-Dispatch
+Dirbuster
+Discoverybot
+DittoSpyder
+DnyzBot
 DomainCrawler
+DomainSigmaCrawler
+Domains Project
 domainsproject.org
-Dotbot
-Dragonfly
-Ebingbong
-Ecxi
-EirGrabber
+DomainStatsBot
+DomCopBot
+Drip
+ECCP/1.0
+EMail Wolf
 EroCrawler
 evc-batch
-ExtractorPro
-Extreme Picture Finder
+Evil
+Extractor
+facebookscraper
 FDM
+FemtosearchBot
 FHscan
-Foobot
-Fyrebot
-GalaxyBot
-Genieo
+Fimap
+Firefox/7.0
+FlashGet
+Flunky
+FrontPage
+Fuzz
 GermCrawler
-Getintent
 GetRight
 Gigabot
-G-i-g-a-b-o-t
+gopher
 Gotit
+Go!Zilla
 GrabNet
 Grafula
-GrapeFX
 GrapeshotCrawler
-Haansoft
 Harvest
-Havij
+Heritrix
 Hloader
+HMView
 HTTP::Lite
 Humanlinks
-HybridBot
-Iblog
-Image Fetch
+IDBot
+IDBTE4M
+Id-search
+IlseBot
 IndeedBot
 Indy Library
+InfoNaviRobot
+InfoTekies
+instabid
+Intelliseek
+InterGET
 InternetSeer
-ips-agent
-IRLbot
-isitwp.com
-Iskanie
-JikeSpider
-Kenjin Spider
-Kozmosbot
+internetVista monitor
+Iria
+IstellaBot
+JennyBot
+JetCar
+JOC Web Spider
+Jorgee
+Jyxobot
+Kinza
 Larbin
-LexiBot
-Lftp
+LeechFTP
+LeechGet
+Lightspeedsystems
 LinkextractorPro
-LinksManager
+LinkScan
 LinkWalker
 LinqiaMetadataDownloaderBot
-LinqiaRSSBot
-Lipperhey
-Lmspider
-LNSpiderguy
-Ltx71
-LWP::Simple
 Magnet
-Mag-Net
-Majestic12
+magpie-crawler
 Majestic SEO
 Majestic-SEO
-MarkMonitor
 MarkWatch
-Mata Hari
-meanpathbot
-MeanPath Bot
+MauiBot
+Meanpathbot
 Mediatoolkitbot
-MicroMessenger
-Microsoft Data Access
-Microsoft URL Control
+mediawords
+MegaIndex.ru
 MIDown tool
 MIIxpc
-Morfeus Fucking Scanner
-MQQBrowser
+Mister PiX
+MJ12bot
+Mozlila
 MSFrontPage
-MTRobot
+muhstik-scan
 Musobot
 Navroad
+NearSite
+Needle
 NetAnts
 Netcraft
-Net Vampire
-NetZIP
-NextGenSearchBot
+NetSpider
 Nibbler
-NimbleCrawler
-Nmap
+Niki-bot
+Nimbostratus
 NPbot
+oBot
 Offline Explorer
-Openfind
-OPPO A33
-OrangeSpider
+Offline Navigator
+OpenLinkProfiler
+OrangeBot
+PageAnalyzer
+Page Analyzer
+PageGrabber
+PageScorer
 PageThing.com
 Pandalytics
+Pavuk
 Petalbot
 PHPCrawl
-Picsearch
-PictureFinder
+Piepmatz
 Pimonster
-Pi-Monster
-Pixray
-PleaseCrawl
-plumanalytics
-Probethenet
-Psbot
+Pockey
+ProPowerBot
 Pu_iN
-Pump
-PyCurl
+Quick-Crawler
 RankActive
-RankActiveLinkBot
 RankFlex
-Rankivabot
-Reaper
-Recorder
-RedesScrapy
+RankingBot2
+RebelMouse
 RepoMonkey
 Re-re
-Ripper
 Rogerbot
 RSSingBot
 s1z.ru
 satoristudio.net
-Scanbot
 scan.lol
+Semrush
 SentiBot
-seoscanners
+SEOkicks-Robot
+SEOprofiler
 SEOstats
-SISTRIX
+serpstatbot
+Siphon
+SiteCheckerBotCrawler
 sitechecker.pro
-SiteExplorer
 Siteimprove
+SiteSucker
 Sitevigil
 SlySearch
-SmartDownload
 SMTBot
+Snapbot
 Snoopy
 SocialRankIOBot
-Sociscraper
-Sogou web spider
-Sottopop
+Sosospider
 Spammen
 Spanner
-sp_auditbot
-Spinn3r
-Sqlworm
+SputnikBot
 Sqworm
-Steeler
 Stripper
+Sucker
+Sucuri
+SuperBot
 Surfbot
-SurveyBot
-Swiftbot
-Szukacz
+Suzuran
+sysscan
 T0PHackTeam
 T8Abot
+tAkeOut
 Telesoft
+Telesphorep
+The Intraformant
+Titan
 Toweyabot
-Trendiction
 trendiction.de
 Turingos
-Turnitin
-TwengaBot
-Twice
 UnisterBot
-URLy Warning
+Upflow
 URLy.Warning
 Vacuum
-VCI
 VelenPublicWebCrawler
 VeriCiteCrawler
+Virusdie
 VoidEYE
+Voltron
 Web Auto
-Web Collage
-WEBDAV
-WebEnhancer
+WebCollage
+WebFetch
 Web Fuck
 WebGo IS
+WebImageCollector
 WebmasterWorldForumBot
-webmeup-crawler
-WebReaper
-WebSauger
+WebPix
+Web Pix
 WebsiteQuester
 Website Quester
 Webster
-WebStripper
-WebWhacker
+WebSucker
+Web Sucker
 WeSEE
-Whatweb
 Who.is Bot
 WiseGuys Robot
-Wonderbot
-Wprecon
+WISENutbot
+Wotbox
 WWW::Mechanize
-x09Mozilla
-x22Mozilla
-Xaldon_WebSpider
 YoudaoBot
-Zitebot
+Zade
+zauba.io
+Zermelo
 ZoomBot
+ZyBorg