V4.2021.12.2804 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-360Spider
-80legs
-ADmantX
-AIBOT
-ALittle Client
 Aboundex
-Acunetix
-AfD-Verbotsverfahren
-AhrefsBot
-AiHitBot
 AllSubmitter
-Alligator
-Anarchie
+Anthill
 Asterias
 Attach
 AwarioSmartBot
-BDFetch
+BDCbot
 BackDoorBot
-BetaBot
-Black Hole
+BackWeb
+Backlink-Ceck
+Barkrowler
+BatchFTP
+Battleztar Bazinga
+Bitacle
+BlackWidow
 Blackboard
-BlowFish
-BotALot
-Brandprotect
-Buck
+Blow
+Bolt
+Brandwatch
 Buddy
+BuiltBotTough
+BuiltWith
+Bullseye
 BunnySlippers
 BuzzSumo
 CCBot
-Calculon
-CazoodleBot
-Cegbfeieh
-CheTeam
+CSHttp
 CherryPicker
-Claritybot
-Cocolyzebot
+ChinaClaw
+Cliqzbot
+Cogentbot
+Collector
 Crawling at Home Project
 CrunchBot
 Curious
-Custo
 CyotekWebCopy
+DBLBot
+DIIbot
 DSearch
-DatabaseDriverMysqli
-Devil
-DigitalPebble
+Demon
+Digincore
 Disco
-Dispatch
+DittoSpyder
+DnBCrawler-Analytics
+DomCopBot
+DomainCrawler
 DomainSigmaCrawler
-DomainStatsBot
 Dotbot
 Download Wonder
-Dragonfly
-ECCP/1.0
 EasyDL
-Ebingbong
-Ecxi
 EirGrabber
 EroCrawler
-Evil
 Exabot
-Extractor
-ExtractorPro
+ExtLinksBot
+Extreme Picture Finder
 FDM
 FHscan
+FlashGet
+Flunky
+FrontPage
+Fuzz
+FyberSpider
+Fyrebot
 G-i-g-a-b-o-t
+GT::WWW
 GalaxyBot
 Genieo
+GermCrawler
 GetWeb
-Gotit
-GrabNet
-Grabber
+Getintent
+GoZilla
+GrapeshotCrawler
 GridBot
-HMView
-HaosouSpider
+HEADMasterSEO
+HTTP::Lite
+HTTrack
+Haansoft
+Havij
+Heritrix
+Hloader
 IDBTE4M
-IDBot
-Id-search
-Image Fetch
-IstellaBot
-JennyBot
+IRLbot
+InfoTekies
+Jbrofuzz
+Jetty
 JikeSpider
 Joomla
+Jorgee
 JustView
-Jyxobot
-Keyword Density
+Kenjin Spider
+Kinza
 Kozmosbot
-LWP::Simple
-Lanshanbot
+Leap
 LeechFTP
-Libwhisker
-Lightspeedsystems
+LeechGet
 Likse
+LinkScan
 LinkWalker
-Linkdexbot
-LinkpadBot
-LinksManager
-Lipperhey Spider
-Lmspider
-MFC_Tear_Sample
-MIIxpc
+Linkbot
+LinqiaRSSBot
 MJ12bot
 MSFrontPage
-MSIECrawler
-Mag-Net
 Magnet
-Majestic12
+Mail.RU_Bot
+Majestic SEO
+Majestic-SEO
+MarkWatch
+Masscan
+MeanPath Bot
+Mediatoolkitbot
+MegaIndex.ru
 Metauri
 Microsoft Data Access
-Microsoft URL Control
 Minefield
-Mister PiX
-Morfeus Fucking Scanner
-Mr.4x3
-Msrabot
-Musobot
-NPbot
+Moblie Safari
+Mojolicious
+Mozlila
+NICErsPRO
+Name Intelligence
+Navroad
 NearSite
-Nessus
-Net Vampire
-NetAnts
-NetZIP
-Netcraft
-Netvibes
+NetMechanic
+NetSpider
+Nettrack
 NextGenSearchBot
 Niki-bot
 Nikto
 NimbleCrawler
-Nuclei
-Octopus
 Offline Navigator
-OnCrawl
 OpenLinkProfiler
-Openvas
-OrangeBot
 OutclicksBot
 PHPCrawl
+Page Analyzer
 PageAnalyzer
-PageGrabber
 PageThing.com
 Pandalytics
 Pavuk
+PeoplePal
 Petalbot
-Picscout
+Pi-Monster
 PictureFinder
-ProPowerBot
+Piepmatz
+Pimonster
+Pixray
+PleaseCrawl
+Probethenet
 Psbot
 Pu_iN
-PyCurl
-RSSingBot
-Rankivabot
-RankurBot
+Pump
+PxBroker
+RankingBot
 Re-re
-RealDownload
 Recorder
-SEOkicks
+RedesScrapy
+SBIder
 SEOkicks-Robot
-SEOprofiler
+SEOlyticsCrawler
+SEOstats
 SMTBot
-Scanbot
+SalesIntelligent
 ScoutJet
+Scrapy
+Screaming
 ScreenerBot
-SearchmetricsBot
 SemanticJuice
-SemrushBot
-SentiBot
-SeoSiteCheckup
 Shodan
+Site Sucker
 SiteCheckerBotCrawler
-SiteLockSpider
 SiteSnagger
 SiteSucker
-Siteimprove
+SlySearch
 Snake
+Sogou web spider
 Sosospider
-Spanner
-Spbot
-Sqworm
-Stripper
-SuperHTTP
+Sottopop
+SpaceBison
+SputnikBot
+Sqlmap
+Steeler
+Sucuri
+SuperBot
+Surfbot
+Szukacz
 TeleportPro
+Telesphoreo
 Telesphorep
-The Intraformant
-Titan
-Toweyabot
-True_Robot
-Turnitin
-Twice
-URLy Warning
-UnisterBot
-Upflow
-V-BOT
-VB Project
+TheNomad
+Thumbor
+Toata
+TurnitinBot
+TwengaBot
+Typhoeus
 Vacuum
-VeriCiteCrawler
-Voil
-WASALive-Bot
-WBSearchBot
-WEBDAV
-WWW-Collector-E
-Wallpapers/3.0
-WallpapersHD
-Web Fetch
+VelenPublicWebCrawler
+VidibleScraper
+Virusdie
+WPScan
+WWW-Mechanize
+WWWOFFLE
+Web Auto
+Web Collage
+Web Fuck
 Web Pix
-WebEnhancer
-WebImageCollector
-WebPix
-WebStripper
-WebWhacker
-WebZIP
+Web Sucker
+WebGo IS
+WebReaper
+WebSauger
 WebmasterWorldForumBot
 Website Quester
-Webster
+WebsiteQuester
 Who.is Bot
+Widow
 WinHTTrack
 Wonderbot
+Woobot
 Wotbox
 Wprecon
+Xaldon WebSpider
+Xaldon_WebSpider
 Xenu
-YoudaoBot
-Zeus
-ZoominfoBot
-adscanner
-arquivo-web-crawler
-cah.io.community
+Zade
+Zauba
+ZoomBot
+arquivo.pt
+backlink-check
+check1.exe
+clark-crawler
+coccocbot-web
 com.plumanalytics
-crawler.feedback
+crawl.sogou.com
+dataforseo.com
 demandbase-bot
-domainsproject.org
 instabid
 internetVista monitor
 ips-agent
-isitwp.com
 lwp-request
-magpie-crawler
 meanpathbot
+netEstate NE Crawler
 oBot
 pcBrowser
-plumanalytics
-polaris version
-seoscanners
-serpstatbot
-sexsearcher
-sitechecker.pro
+probe-image-size
+ripz
+scalaj-http
+siteripz
+sogouspider
+sp_auditbot
 spyfu
-sysscan
-trendiction.com
-webmeup-crawler
-x22Mozilla
-zgrab
+tAkeOut
+ubermetrics-technologies.com
+x09Mozilla