V4.2022.01.2832 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-80legs
+360Spider
+404checker
+404enemy
 ADmantX
-Abonti
-Aboundex
-AfD-Verbotsverfahren
+ALittle Client
 AhrefsBot
 AiHitBot
-Anarchie
+Alexibot
+AllSubmitter
+AlphaBot
 Anarchy
-Anthill
-Attach
 AwarioRssBot
+BBBike
+BDCbot
+BDFetch
 BackDoorBot
 Backlink-Ceck
-Barkrowler
-Bitacle
-Black Hole
-BlackWidow
-Blackboard
-Brandprotect
-Buck
-CCBot
-CODE87
+Bigfoot
+Blow
+Brandwatch
+BuiltBotTough
+BuiltWith
+BuzzSumo
+CSHttp
 CheTeam
-CheeseBot
-Chlooe
-Citoid
-Cloud mapping
+Claritybot
 Cocolyzebot
-Copier
-Copyscape
+CopyRightCheck
 Cosmos
-Crawling at Home Project
-CrazyWebCrawler
+Craftbot
 Crescent
 Curious
 Custo
 CyotekWebCopy
 DBLBot
-DSearch
-Discobot
-Discoverybot
-DittoSpyder
-DnyzBot
-DomainCrawler
-DomainSigmaCrawler
+DIIbot
+DTS Agent
+Deusu
+Devil
+Dirbuster
+Disco
+Dispatch
+DnBCrawler-Analytics
+DomCopBot
+DomainStatsBot
+Domains Project
 Dotbot
+Download Wonder
 Drip
 ECCP/1.0
-EMail Siphon
-EMail Wolf
-Ecxi
-EirGrabber
-EroCrawler
+EasyDL
+Ebingbong
+Evil
 Extractor
+Extreme Picture Finder
 EyeNetIE
-Ezooms
-FDM
+FHscan
 FemtosearchBot
-FlashGet
-Foobot
 FrontPage
-GT::WWW
-GalaxyBot
-Genieo
-GermCrawler
+GetRight
 GetWeb
-Getintent
-Gigabot
 Go-Ahead-Got-It
-GoZilla
-Gotit
 Grafula
-GrapeFX
-GrapeshotCrawler
-GridBot
-HEADMasterSEO
-HMView
 HTMLparser
+Haansoft
+HaosouSpider
+Harvest
+Havij
 Humanlinks
+HybridBot
+IDBTE4M
 IDBot
-IlseBot
-Image Sucker
+Iblog
+Image Fetch
+IndeedBot
+Indy Library
+InfoNaviRobot
 InfoTekies
-Intelliseek
 Internet Ninja
+InternetSeer
+Iskanie
 JOC Web Spider
 JamesBOT
+Jbrofuzz
 JennyBot
-Jetty
-Joomla
+JetCar
+JikeSpider
 JustView
+Jyxobot
 Kenjin Spider
-Kozmosbot
 LNSpiderguy
-Lanshanbot
+Leap
 LeechGet
-LinkScan
-Linkbot
-Linkdexbot
-LinqiaRSSBot
+LieBaoFast
+Likse
+LinkextractorPro
+LinkpadBot
+LinksManager
 LinqiaScrapeBot
+Lipperhey
 Lipperhey Spider
 Ltx71
 MFC_Tear_Sample
 MIIxpc
-MQQBrowser
+MSFrontPage
 MSIECrawler
+Mag-Net
 Majestic SEO
+Majestic-SEO
 Majestic12
-MarkMonitor
+MarkWatch
 Masscan
 Mata Hari
-MeanPath Bot
-Microsoft Data Access
+Meanpathbot
+Metauri
+Microsoft URL Control
+Mojolicious
 Morfeus Fucking Scanner
-Mozlila
 Mr.4x3
-NPbot
-Name Intelligence
-Nameprotect
-Nessus
-NetMechanic
+Msrabot
+NICErsPRO
+NetSpider
 NetZIP
-Netcraft
-NextGenSearchBot
-Nimbostratus
-Offline Explorer
-Offline Navigator
-OnCrawl
+Niki-bot
+Nikto
+NimbleCrawler
+Octopus
 OpenLinkProfiler
+OpenVAS
 Openfind
 OrangeSpider
 OutclicksBot
+PHPCrawl
 POE-Component-Client-HTTP
-PageThing.com
-Pandalytics
+PageGrabber
+Panscient
 Pavuk
 PeoplePal
-Petalbot
+Picscout
+Picsearch
+PictureFinder
 Piepmatz
 Pimonster
-PleaseCrawl
 ProPowerBot
-ProWebWalker
-Psbot
+Pump
 PyCurl
-RankFlex
+QueryN Metasearch
+RankActive
+RankActiveLinkBot
 RankingBot2
-Rankivabot
-RealDownload
+Re-re
+Reaper
+RebelMouse
+RepoMonkey
 RocketCrawler
+Rogerbot
 SBIder
+SEOlyticsCrawler
 SEOprofiler
-SalesIntelligent
-ScanAlert
+SMTBot
+Scrapy
 ScreenerBot
 ScrepyBot
 Searchestate
-Semrush
+SemanticJuice
 SemrushBot
 SentiBot
-Seomoz
-SiteLockSpider
-SiteSucker
-Siteimprove
-Sitevigil
-Snoopy
-Sociscraper
+Shodan
+SiteSnagger
+Sitebeam
+Snapbot
 Sogou web spider
 Sottopop
-Spammen
+SpaceBison
+Spbot
 Spinn3r
+Sqlmap
 Sqworm
-Stripper
-Sucker
+Steeler
 Sucuri
-Surfbot
-Szukacz
-Telesphoreo
+SurveyBot
+Swiftbot
+Telesoft
+Telesphorep
+The Intraformant
 TheNomad
 Thumbor
-Toata
+Toweyabot
+Tracemyfile
 True_Robot
-URLy Warning
-URLy.Warning
+TwengaBot
 UnisterBot
 V-BOT
-VCI
-Vagabondo
-VelenPublicWebCrawler
-VeriCiteCrawler
 VidibleScraper
-WISENutbot
-WPScan
+VoidEYE
+Voil
+WASALive-Bot
+WEBDAV
 WWW-Collector-E
 WWW-Mechanize
 WWWOFFLE
 Wallpapers/3.0
+WallpapersHD
 Web Auto
-Web Fetch
-Web Fuck
-Web Sauger
-WebCollage
-WebCopier
-WebEnhancer
+Web Collage
+Web Enhancer
+WebBandit
 WebFetch
 WebFuck
-WebImageCollector
-WebSucker
+WebLeacher
+WebPix
+WebReaper
+WebSauger
+WebStripper
 WebWhacker
-WebmasterWorldForumBot
-Webshag
-Website Quester
-Whack
-Whatweb
-WinHTTrack
-Xaldon WebSpider
-Xenu
-YoudaoBot
+Webalta
+Whacker
+Widow
+WiseGuys Robot
+Wonderbot
+Zauba
+Zermelo
 Zeus
-ZmEu
-ZoomBot
-ZoominfoBot
-backlink-check
+ZumBot
+ZyBorg
+arquivo.pt
+autoemailspider
 cah.io.community
-crawl.sogou.com
+check1.exe
+cognitiveseo
 crawler.feedback
+crawler4j
+demandbase-bot
+eCatch
 gopher
 heritrix
-ips-agent
+isitwp.com
+lwp-request
 meanpathbot
 mediawords
-page scorer
-probe-image-size
-ripz
-satoristudio.net
+muhstik-scan
+netEstate NE Crawler
 seoscanners
 seostar
-sogouspider
-spyfu
-sysscan
-tAkeOut
-ubermetrics-technologies.com
-voyagerx.com
+serpstatbot
+sexsearcher
+sp_auditbot
+webmeup-crawler
 webpros.com
-x09Mozilla
-x22Mozilla
-zgrab
+zauba.io