V4.2021.12.2782 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-360Spider
+404checker
 404enemy
 ADmantX
-AIBOT
 ALittle Client
-ASPSeek
 Aboundex
-Aboundexbot
-Acunetix
 AfD-Verbotsverfahren
-AllSubmitter
-Alligator
+AhrefsBot
+AiHitBot
+Aipbot
 Anarchie
-Anarchy99
-Anthill
-AwarioRssBot
+Anarchy
+Ankit
 BBBike
-BDCbot
-BDFetch
-BLEXBot
-BackDoorBot
-BacklinkCrawler
-Bandit
-Battleztar Bazinga
-Boardreader
-Brandwatch
-Buck
+Badass
+BatchFTP
+Bigfoot
+Bitacle
+Black Hole
+BotALot
 BuiltBotTough
 BuiltWith
-Bullseye
 CCBot
-CazoodleBot
-CheTeam
+CODE87
+CensysInspect
+Chlooe
 Cliqzbot
-Cloud mapping
 Copier
+Copyscape
 Cosmos
-Craftbot
-CrazyWebCrawler
-Crescent
-CrunchBot
-Curious
-CyotekWebCopy
 DBLBot
 DSearch
+DTS Agent
 DataCha0s
+Deusu
+Devil
 Digincore
-DigitalPebble
-Discobot
-Discoverybot
-DnyzBot
+Disco
+DittoSpyder
+DnBCrawler-Analytics
 DomCopBot
-DomainSigmaCrawler
+DomainCrawler
 Domains Project
-Download Wonder
-Dragonfly
-Drip
-ECCP/1.0
 EMail Wolf
+EasyDL
+Ebingbong
 Ecxi
-Evil
+EroCrawler
+Exabot
 ExtLinksBot
-Extractor
-ExtractorPro
 Extreme Picture Finder
-EyeNetIE
-FHscan
+FDM
 FemtosearchBot
-Firefox/7.0
+Flunky
 Foobot
 Freeuploader
-GalaxyBot
-GermCrawler
+FrontPage
+FyberSpider
+Fyrebot
+Genieo
 GetRight
+GetWeb
+Getintent
 Gigabot
 Go!Zilla
 Go-Ahead-Got-It
-GoZilla
-Gotit
-Grabber
-GrapeshotCrawler
-GridBot
+Grafula
 HEADMasterSEO
-HMView
 HTTP::Lite
-Humanlinks
-IDBTE4M
-IDBot
+HTTrack
+Haansoft
+Harvest
+Havij
+Hloader
+Iblog
+Id-search
+IlseBot
 Image Fetch
-IndeedBot
-Indy Library
-InfoTekies
+Image Sucker
+Intelliseek
 InterGET
-Internet Ninja
+InternetSeer
+Iskanie
 JOC Web Spider
+JetCar
+JikeSpider
 Joomla
-Kinza
-LWP::Simple
-Lanshanbot
+JustView
+Jyxobot
+Kenjin Spider
+Larbin
 Leap
-LeechFTP
-LeechGet
-LexiBot
 Lftp
-LibWeb
-LinkextractorPro
-LinksManager
+Libwhisker
+LieBaoFast
+LinkScan
+Linkdexbot
 LinqiaMetadataDownloaderBot
 LinqiaScrapeBot
+Lipperhey Spider
 Lmspider
 MIDown tool
 MJ12bot
-MSIECrawler
-Mag-Net
-Mail.RU_Bot
+Magnet
+Majestic SEO
+MarkMonitor
+Mass Downloader
 Mata Hari
-MauiBot
 Mb2345Browser
-MicroMessenger
-Microsoft URL Control
+Meanpathbot
+Microsoft Data Access
 Mister PiX
-Moblie Safari
-Morfeus Fucking Scanner
-Mozlila
-Mr.4x3
-NPbot
-Name Intelligence
+Mojolicious
+NICErsPRO
+Nameprotect
 Navroad
-Nessus
-Net Vampire
-NetAnts
+NearSite
 NetLyzer
+NetSpider
+NetZIP
+Netvibes
 Niki-bot
-Nmap
+Nikto
+Not
 Nuclei
-Octopus
 Offline Explorer
 OpenVAS
-Openfind
-Openvas
-OrangeBot
-OrangeSpider
-OutfoxBot
+OutclicksBot
+PECL::HTTP
+PHPCrawl
+POE-Component-Client-HTTP
+PageScorer
+Pavuk
 PeoplePal
 Petalbot
-Pi-Monster
-Picscout
+Picsearch
+Piepmatz
 Pimonster
 PleaseCrawl
+ProWebWalker
+Probethenet
 Psbot
-PyCurl
+Pu_iN
 QueryN Metasearch
-RSSingBot
-RankActive
-RankActiveLinkBot
-RankurBot
-Reaper
+Quick-Crawler
+RankFlex
+RankingBot2
+ReGet
+RealDownload
+RebelMouse
 Recorder
-RedesScrapy
-Ripper
+SBIder
 SEOkicks
+SEOkicks-Robot
+SEOstats
+SISTRIX
 SMTBot
 ScanAlert
 Scrapy
 Screaming
-ScreenerBot
+Searchestate
 SearchmetricsBot
 SemanticJuice
-SemrushBot
+Semrush
+SentiBot
 SeobilityBot
-Shodan
+Seomoz
 Siphon
-SiteLockSpider
+Site Sucker
+SiteCheckerBotCrawler
+SiteExplorer
 SiteSnagger
-SmartDownload
+SlySearch
 Snake
 Snapbot
+Sociscraper
+Sosospider
 SpaceBison
-Spbot
-Sqlworm
-Sqworm
-Steeler
-Sucker
-Swiftbot
-T0PHackTeam
-Telesphoreo
-Thumbor
-Trendictionbot
+Spinn3r
+SuperBot
+Surfbot
+SurveyBot
+T8Abot
+Teleport
+TeleportPro
+Telesphorep
+TightTwatBot
+Titan
+Toata
+Toweyabot
+Trendiction
 True_Robot
-Turingos
 Turnitin
-TwengaBot
-Typhoeus
-UnisterBot
+Twice
+URLy.Warning
 Upflow
-V-BOT
+Vacuum
+WASALive-Bot
+WISENutbot
 WPScan
 WWW-Collector-E
 WWW-Mechanize
+Web Auto
 Web Collage
-Web Enhancer
-Web Sauger
-Web Sucker
-WebBandit
-WebCopier
-WebEnhancer
-WebFetch
+Web Pix
 WebImageCollector
-WebSucker
+WebLeacher
+WebPix
+WebStripper
+WebmasterWorldForumBot
+Webshag
 Website Quester
+WebsiteExtractor
 Webster
+Whack
 WinHTTrack
+WiseGuys Robot
 Wonderbot
-Woobot
-Wotbox
 Xaldon WebSpider
-Xenu
+Xaldon_WebSpider
+YoudaoBot
 Zade
 Zermelo
-Zeus
 ZmEu
-ZoomBot
 ZoominfoBot
-arquivo.pt
-clark-crawler
+ZyBorg
+archive.org_bot
+autoemailspider
 coccocbot-web
-crawler4j
-gopher
-heritrix
-magpie-crawler
-mediawords
-muhstik-scan
-page scorer
-plumanalytics
+cognitiveseo
+dataforseo.com
+facebookscraper
+ips-agent
+isitwp.com
+lwp-trivial
+meanpathbot
+oBot
 ripz
-scalaj-http
-scan.lol
+satoristudio.net
 seobility
-serpstatbot
-sexsearcher
-sitechecker.pro
-sp_auditbot
+seoscanners
+seostar
+siteripz
+sogouspider
 spyfu
+sysscan
+ubermetrics-technologies.com
 voyagerx.com
-webgains-bot
-webmeup-crawler
 webpros.com
-webprosbot
-x09Mozilla
+xpymep1.exe
+zauba.io