V4.2021.12.2757 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-404enemy
+360Spider
+404checker
 80legs
-AIBOT
-ASPSeek
 Abonti
-AiHitBot
+Acunetix
+AfD-Verbotsverfahren
+AllSubmitter
+Alligator
 AlphaBot
-Anarchie
+Ankit
 Anthill
-Aspiegel
 Attach
-AwarioSmartBot
 BBBike
-BDFetch
+BDCbot
 BLEXBot
 BacklinkCrawler
-Bandit
+Barkrowler
 Battleztar Bazinga
-BetaBot
 Bitacle
+BlackWidow
 Blackboard
-Blow
+BlowFish
 Boardreader
-Bolt
-BotALot
-Buddy
+Brandprotect
 BuiltWith
-BunnySlippers
-BuzzSumo
-CATExplorador
-CazoodleBot
+Cegbfeieh
+CheeseBot
+Chlooe
 Citoid
+Claritybot
+Cloud mapping
+Cocolyzebot
 Cogentbot
+Collector
 Copyscape
-Crawling at Home Project
+Crescent
 CrunchBot
+Custo
+CyotekWebCopy
+DBLBot
+DIIbot
 Deusu
-DomCopBot
-DomainCrawler
+Digincore
+Discobot
+Dispatch
+DittoSpyder
+DnBCrawler-Analytics
+DomainSigmaCrawler
 Domains Project
+Dotbot
 Dragonfly
-EMail Siphon
+ECCP/1.0
 EMail Wolf
+Ebingbong
+EirGrabber
+EroCrawler
+Evil
 ExtLinksBot
-ExtractorPro
 EyeNetIE
-FDM
+Ezooms
 Fimap
 Firefox/7.0
 FlashGet
-Flunky
-Foobot
-Fuzz
-Genieo
-Getintent
+Fyrebot
+G-i-g-a-b-o-t
+GermCrawler
+Gigabot
 Go-Ahead-Got-It
-GoZilla
 Gotit
-GrabNet
+GridBot
 HMView
-HTMLparser
-HTTrack
-HaosouSpider
+HTTP::Lite
 Harvest
 Havij
+Hloader
 Humanlinks
-HybridBot
-IDBTE4M
-IRLbot
-Iblog
-Id-search
-Intelliseek
+IlseBot
+Image Fetch
+IndeedBot
+InfoNaviRobot
+InfoTekies
 InterGET
 Internet Ninja
+InternetSeer
 JOC Web Spider
-JamesBOT
-JetCar
-JikeSpider
+Jbrofuzz
+Jetty
 Jyxobot
+Keyword Density
 LNSpiderguy
+LWP::Simple
 Lanshanbot
-LeechFTP
 LeechGet
 Lftp
+Libwhisker
+LieBaoFast
+Lightspeedsystems
+Likse
 LinkScan
-LinkWalker
-LinkextractorPro
 LinkpadBot
-LinksManager
 LinqiaMetadataDownloaderBot
-Ltx71
-MIDown tool
+LinqiaScrapeBot
+Lipperhey
+Lipperhey Spider
+MFC_Tear_Sample
+MQQBrowser
 MSFrontPage
-MTRobot
+MSIECrawler
+Mag-Net
+Magnet
 Mail.RU_Bot
-Majestic12
-MarkMonitor
-Mata Hari
 Mb2345Browser
-MicroMessenger
+Meanpathbot
+MegaIndex.ru
 Microsoft Data Access
+Microsoft URL Control
+Minefield
 Mister PiX
 Moblie Safari
-Mojolicious
-Morfeus Fucking Scanner
-Msrabot
-NPbot
-Name Intelligence
-NearSite
-Needle
-Nessus
-NetLyzer
-NetMechanic
-NetSpider
+Mozlila
+Mr.4x3
+Nameprotect
+Net Vampire
+Netcraft
 Netvibes
-NextGenSearchBot
-Niki-bot
-Nikto
+Nibbler
 Nuclei
-Offline Navigator
-OpenLinkProfiler
-PECL::HTTP
-PHPCrawl
+Offline Explorer
+OnCrawl
+OpenVAS
+Openfind
+OrangeSpider
+OutclicksBot
+OutfoxBot
+Page Analyzer
 PageGrabber
 PageScorer
-Papa Foto
-PeoplePal
+PageThing.com
+Panscient
+Pavuk
 Pi-Monster
-Picscout
-PleaseCrawl
+Pixray
 ProPowerBot
-PxBroker
+Probethenet
+QueryN Metasearch
+Quick-Crawler
+RSSingBot
+RankFlex
 RankingBot
 RankingBot2
-Rankivabot
-Re-re
-RealDownload
-RebelMouse
-RedesScrapy
-Rogerbot
-SBIder
-SEOkicks
+RankurBot
+Recorder
 SEOkicks-Robot
-SEOprofiler
 SEOstats
 SalesIntelligent
-Screaming
-ScreenerBot
+Scanbot
+ScoutJet
+Scrapy
 ScrepyBot
-Searchestate
+SearchmetricsBot
+SemrushBot
 SentiBot
 SeobilityBot
-Site Sucker
-SiteCheckerBotCrawler
+Seomoz
+Siphon
 SiteExplorer
-SiteSnagger
+SiteLockSpider
 SiteSucker
-Snapbot
-Snoopy
+Siteimprove
+SmartDownload
+SocialRankIOBot
+Sosospider
+Spammen
 SpankBot
 Spanner
+Spinn3r
 SputnikBot
-Sqworm
-Surfbot
-Suzuran
-Swiftbot
-T8Abot
-Teleport
-Telesphorep
+Sqlmap
+Sqlworm
+SuperBot
+SurveyBot
+Telesoft
 The Intraformant
-Toata
-Toweyabot
-Turingos
+Tracemyfile
 Turnitin
 TurnitinBot
 TwengaBot
-Twice
 Typhoeus
-V-BOT
-VelenPublicWebCrawler
+VCI
 VidibleScraper
-VoidEYE
+Virusdie
 WBSearchBot
 WEBDAV
-WWW-Collector-E
-WWWOFFLE
-Wallpapers
-WeSEE
-Web Fuck
-Web Pix
-WebBandit
+WISENutbot
+Web Enhancer
+Web Fetch
+WebAuto
+WebCollage
+WebCopier
 WebFuck
+WebImageCollector
+WebLeacher
 WebPix
-WebStripper
-WebSucker
-WebWhacker
 WebZIP
-WebmasterWorldForumBot
-Webshag
+Webalta
 Website Quester
-WebsiteExtractor
+WebsiteQuester
 Webster
 Whacker
-WinHTTrack
+WiseGuys Robot
 Wonderbot
-Wotbox
-Xaldon WebSpider
+Woobot
 Xaldon_WebSpider
 Xenu
-YoudaoBot
-Zade
 Zauba
 Zermelo
-ZoomBot
+ZmEu
 ZoominfoBot
-ZyBorg
-adscanner
+archive.org_bot
 arquivo-web-crawler
+arquivo.pt
 autoemailspider
+cah.io.community
 check1.exe
 clark-crawler
-crawl.sogou.com
-crawler.feedback
-demandbase-bot
+coccocbot-web
 domainsproject.org
 eCatch
 evc-batch
-gopher
+heritrix
 instabid
 ips-agent
-isitwp.com
-lwp-request
+magpie-crawler
 meanpathbot
-netEstate NE Crawler
-oBot
+muhstik-scan
+page scorer
 pcBrowser
-probe-image-size
-satoristudio.net
+ripz
+s1z.ru
 scalaj-http
+seobility
 seostar
-serpstatbot
-sexsearcher
+siteripz
+sogouspider
 spyfu
-sysscan
 trendiction.com
-trendiction.de
-webgains-bot
-webmeup-crawler
-webpros.com
+x09Mozilla
+zauba.io
+zgrab