V4.2021.10.2642 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-360Spider
-404enemy
+01h4x.com
+404checker
+80legs
+ADmantX
+AIBOT
 ALittle Client
+ASPSeek
 AfD-Verbotsverfahren
-Alexibot
-Alligator
-Apexoo
-Aspiegel
-BDCbot
-BackStreet
+AhrefsBot
+Aipbot
+AlphaBot
+Anarchie
+Anarchy99
+Anthill
+Attach
+BBBike
+BLEXBot
+BackDoorBot
+BackWeb
 Backlink-Ceck
-Badass
 Bandit
-BatchFTP
-Bigfoot
-Blackboard
+Barkrowler
+Bitacle
+BlackWidow
+BlowFish
 Bolt
-Brandwatch
-BuiltBotTough
+BuiltWith
 Bullseye
-CATExplorador
+BuzzSumo
+CCBot
 CODE87
 CSHttp
-CheTeam
+CheeseBot
 CherryPicker
-Claritybot
+Chlooe
 Cliqzbot
-Cocolyzebot
+Collector
 Copier
+Crawling at Home Project
+CrazyWebCrawler
+Crescent
 Curious
-Custo
-DBLBot
 DSearch
-DTS Agent
-DatabaseDriverMysqli
-Demon
 Deusu
-DigitalPebble
-Disco
-Discobot
-DnyzBot
-DomCopBot
-DomainSigmaCrawler
-Dotbot
-Download Wonder
-ECCP/1.0
-EasyDL
+Dirbuster
+Discoverybot
+DomainCrawler
+DomainStatsBot
 Ecxi
-EirGrabber
-Extractor
+EroCrawler
+Exabot
+Express WebPictures
+ExtLinksBot
+EyeNetIE
 Ezooms
-FDM
+FHscan
 FemtosearchBot
 Firefox/7.0
-FlashGet
-FyberSpider
-GermCrawler
-Go!Zilla
-GrabNet
+Flunky
+Fuzz
+Fyrebot
+GalaxyBot
+Genieo
+GetRight
+GetWeb
+Getintent
+Go-Ahead-Got-It
+GoZilla
+Gotit
 Grabber
-Grafula
 GrapeFX
+GridBot
 HEADMasterSEO
-HMView
 HTMLparser
-HTTP::Lite
 HTTrack
+Haansoft
 Harvest
 Havij
+Heritrix
+Hloader
 IRLbot
-Id-search
 IlseBot
-Image Fetch
-Image Sucker
-Indy Library
-InfoTekies
+IndeedBot
 Intelliseek
-InterGET
-IstellaBot
+Internet Ninja
+InternetSeer
 JOC Web Spider
-Jbrofuzz
-JennyBot
-Jetty
-Joomla
+JamesBOT
+JetCar
 Jorgee
-JustView
 Keyword Density
-LNSpiderguy
-Lanshanbot
-Leap
-LeechGet
+Larbin
+Lftp
 LibWeb
+LieBaoFast
 Likse
-Linkdexbot
-LinkpadBot
+Linkbot
+LinqiaMetadataDownloaderBot
 LinqiaScrapeBot
+Lipperhey Spider
+Litemage_walker
+Ltx71
 MFC_Tear_Sample
 MIDown tool
-MIIxpc
-MJ12bot
-MQQBrowser
-MSIECrawler
-Magnet
-Majestic SEO
-MarkMonitor
+MTRobot
+Majestic12
 MarkWatch
-Mass Downloader
+Masscan
+Mata Hari
+MauiBot
+Mb2345Browser
 MeanPath Bot
 MegaIndex.ru
-MicroMessenger
-Microsoft URL Control
+Mister PiX
 Moblie Safari
-Mozlila
-Musobot
-Name Intelligence
-Nameprotect
+Mojeek
+Mr.4x3
+NPbot
 Navroad
 NearSite
 Needle
-NetLyzer
+Net Vampire
+NetSpider
 NetZIP
+Netcraft
 Nettrack
-Nibbler
-NimbleCrawler
+Netvibes
+Niki-bot
 Nimbostratus
-Ninja
-Nuclei
+Nmap
 Nutch
-OpenLinkProfiler
-Openfind
-Openvas
-OrangeSpider
-OutfoxBot
-PageAnalyzer
+Offline Navigator
+OutclicksBot
+POE-Component-Client-HTTP
+PageGrabber
+PageScorer
 Panscient
 Papa Foto
-Pavuk
 Petalbot
-Picsearch
-Pimonster
+Pi-Monster
+PictureFinder
 ProPowerBot
-ProWebWalker
 Pump
-RankActiveLinkBot
+PxBroker
+QueryN Metasearch
+Quick-Crawler
+RSSingBot
 RankingBot2
+Rankivabot
+RankurBot
+Re-re
+ReGet
 RealDownload
-Reaper
-RebelMouse
-RedesScrapy
+Recorder
+RepoMonkey
 Ripper
-RocketCrawler
-Rogerbot
-SEOkicks-Robot
-SEOlyticsCrawler
-SEOprofiler
-SISTRIX
-ScanAlert
-ScoutJet
 Scrapy
-SeobilityBot
+ScrepyBot
+Searchestate
+SearchmetricsBot
+Semrush
 Shodan
-Siphon
 SiteCheckerBotCrawler
-SiteExplorer
-SiteSnagger
-SiteSucker
-SlySearch
-SmartDownload
+Snapbot
+Snoopy
 Sociscraper
-SpaceBison
+Sosospider
+SpankBot
 Spanner
-Spbot
 Spinn3r
 Sqlmap
 Stripper
 Sucker
-SuperHTTP
-Surfbot
+Sucuri
 Suzuran
-Swiftbot
-Szukacz
-T8Abot
-Teleport
-Telesphorep
+T0PHackTeam
+TeleportPro
+Telesoft
 The Intraformant
-Thumbor
-Toata
-Tracemyfile
-TwengaBot
-URLy Warning
+TightTwatBot
+Titan
+Trendiction
+TurnitinBot
 UnisterBot
+VB Project
+Vacuum
 Vagabondo
-VelenPublicWebCrawler
 VeriCiteCrawler
+VidibleScraper
+Virusdie
 VoidEYE
-Voil
-WASALive-Bot
-WBSearchBot
+Voltron
 WEBDAV
+WISENutbot
 WWW-Collector-E
-WWW-Mechanize
+WWW::Mechanize
 WWWOFFLE
 Wallpapers
 Wallpapers/3.0
-WeSEE
-Web Fetch
+WallpapersHD
+Web Auto
+Web Fuck
 Web Pix
-Web Sucker
-WebEnhancer
-WebGo IS
-WebImageCollector
-WebLeacher
+WebCopier
+WebFetch
+WebReaper
+WebStripper
+WebSucker
 WebWhacker
-Webalta
-Whacker
+WebmasterWorldForumBot
+Website Quester
+WebsiteExtractor
+Widow
 WiseGuys Robot
-Wonderbot
-Woobot
 Wotbox
-Xaldon_WebSpider
-Zermelo
-ZmEu
+Zeus
+Zitebot
 ZoomBot
 ZumBot
 ZyBorg
-autoemailspider
+backlink-check
 check1.exe
-coccocbot-web
+clark-crawler
+com.plumanalytics
+crawl.sogou.com
 crawler.feedback
-crawler4j
 dataforseo.com
+eCatch
 facebookscraper
-gopher
-instabid
+heritrix
+internetVista monitor
+isitwp.com
+lwp-trivial
+magpie-crawler
 meanpathbot
-oBot
+muhstik-scan
+netEstate NE Crawler
 page scorer
+pcBrowser
 polaris version
 probe-image-size
+ripz
+satoristudio.net
 scan.lol
-seoscanners
+seobility
 seostar
 sogouspider
-sp_auditbot
-spyfu
-sysscan
 ubermetrics-technologies.com
-voyagerx.com
-webmeup-crawler
-x22Mozilla
+webpros.com
+xpymep1.exe