V4.2021.11.2738 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
 360Spider
-404enemy
 80legs
-ALittle Client
+ADmantX
+ASPSeek
 Abonti
-Acunetix
-AfD-Verbotsverfahren
 AiHitBot
-AlphaBot
+Aipbot
+Alligator
+Anarchie
+Anarchy
+Anarchy99
 Ankit
+Anthill
 Attach
-AwarioRssBot
 AwarioSmartBot
-BDFetch
 BackDoorBot
-BackWeb
 BacklinkCrawler
 Bandit
-BatchFTP
-BetaBot
-Black Hole
+Battleztar Bazinga
+Bigfoot
 BlackWidow
 Blackboard
-Blow
 BlowFish
-Boardreader
-Bolt
-Buddy
+BotALot
+Brandprotect
+Buck
 BunnySlippers
+CATExplorador
 CCBot
-CensysInspect
-ChinaClaw
-Claritybot
-Cogentbot
-Collector
-Cosmos
+Calculon
+CheTeam
+Cliqzbot
+Cloud mapping
+Copyscape
+Craftbot
 CrunchBot
 Curious
-Custo
-DTS Agent
-DataCha0s
+DBLBot
+DIIbot
 DatabaseDriverMysqli
-Digincore
-Dirbuster
-Disco
-DnBCrawler-Analytics
+Discoverybot
 DomCopBot
-DomainAppender
-DomainSigmaCrawler
-Dotbot
-Download Wonder
-Dragonfly
+DomainStatsBot
+Drip
 ECCP/1.0
-EMail Siphon
-Ecxi
+EMail Wolf
 EroCrawler
 Exabot
-Extractor
+ExtLinksBot
+Extreme Picture Finder
 FDM
-FemtosearchBot
+FHscan
+Fimap
 FlashGet
 Flunky
-FrontPage
+Freeuploader
 Fuzz
-Fyrebot
-Genieo
+GalaxyBot
 GermCrawler
+GetWeb
+Getintent
 Gigabot
-Grafula
-GrapeshotCrawler
+GridBot
 HEADMasterSEO
+HMView
+HTMLparser
 HTTP::Lite
-Harvest
+Havij
+Heritrix
 Hloader
 Humanlinks
 HybridBot
-IRLbot
+IDBTE4M
+Iblog
+Id-search
+InfoNaviRobot
+InfoTekies
 Intelliseek
-Internet Ninja
-IstellaBot
-JOC Web Spider
+Iskanie
 JamesBOT
 JetCar
 Jetty
-Joomla
-Keyword Density
+JikeSpider
+Kenjin Spider
 Kinza
-Kozmosbot
-Lanshanbot
-LeechFTP
-LeechGet
-LibWeb
-Libwhisker
-LieBaoFast
-Likse
+LNSpiderguy
+LWP::Simple
+Lightspeedsystems
 LinkScan
+LinkWalker
+Linkdexbot
 LinkextractorPro
-LinkpadBot
-LinksManager
-LinqiaRSSBot
-LinqiaScrapeBot
+Litemage_walker
+Ltx71
 MIDown tool
+MIIxpc
+MJ12bot
+MQQBrowser
+MSFrontPage
 MSIECrawler
-Magnet
+Mag-Net
 Mail.RU_Bot
-Mass Downloader
-Mb2345Browser
-Moblie Safari
-Mojolicious
-Morfeus Fucking Scanner
-Msrabot
+Majestic-SEO
+MarkMonitor
+MeanPath Bot
+Meanpathbot
+MicroMessenger
+Microsoft Data Access
+Minefield
+Mister PiX
+Mojeek
+Mozlila
+Mr.4x3
 NICErsPRO
-Nameprotect
-NearSite
-Needle
-NetLyzer
+NPbot
+Navroad
 NetSpider
-Netcraft
-Nettrack
-Niki-bot
+NextGenSearchBot
+Nibbler
+Nikto
 NimbleCrawler
-Nimbostratus
-Ninja
 Nmap
-OpenLinkProfiler
 Openfind
-OutclicksBot
-OutfoxBot
+PECL::HTTP
 PHPCrawl
 Page Analyzer
-PeoplePal
-Pi-Monster
+PageAnalyzer
+PageScorer
+Panscient
 Picsearch
 PictureFinder
-Pockey
-ProWebWalker
-Probethenet
+Pimonster
+Pu_iN
+Pump
+PyCurl
 QueryN Metasearch
+Quick-Crawler
 RankActive
-RankActiveLinkBot
+RankingBot
+RankingBot2
 Rankivabot
-RankurBot
-Re-re
 RealDownload
-Reaper
+RebelMouse
 RedesScrapy
-RepoMonkey
-Ripper
-SEOkicks
-SEOprofiler
+RocketCrawler
+SBIder
+SEOlyticsCrawler
+SEOstats
+SISTRIX
+SMTBot
 SalesIntelligent
-Scanbot
 Screaming
-Searchestate
-SemrushBot
+ScrepyBot
+SemanticJuice
+Semrush
+SentiBot
 SeoSiteCheckup
-SiteCheckerBotCrawler
-SiteLockSpider
-Sitevigil
-SlySearch
-Snoopy
-Sogou web spider
-Sottopop
-Spammen
+SeobilityBot
+Seomoz
+Site Sucker
+SiteSucker
+Sitebeam
+SmartDownload
+Snapbot
+Sociscraper
+SpaceBison
 SpankBot
-Spanner
+Spbot
+Spinn3r
 Sqlworm
-Stripper
-Sucuri
-SuperBot
+SuperHTTP
 Surfbot
 SurveyBot
-Suzuran
-Teleport
+Szukacz
 TeleportPro
-Telesoft
-TheNomad
+Telesphoreo
 Thumbor
-TightTwatBot
 Titan
-Turingos
+Toata
+Toweyabot
+Tracemyfile
+Turnitin
 TurnitinBot
-TwengaBot
 Twice
-URLy Warning
-Upflow
+URLy.Warning
+UnisterBot
 VB Project
-Vacuum
-VeriCiteCrawler
-Voil
+VelenPublicWebCrawler
+Virusdie
+WASALive-Bot
 WISENutbot
 WWW-Collector-E
+WWWOFFLE
 Wallpapers/3.0
 WeSEE
-Web Fuck
+Web Auto
+Web Fetch
 Web Pix
+Web Sauger
 Web Sucker
-WebImageCollector
+WebAuto
+WebCollage
 WebLeacher
 WebPix
-WebStripper
-WebSucker
-WebZIP
 Webalta
+Webshag
 Website Quester
-WebsiteExtractor
 WebsiteQuester
+Webster
 Whack
-Whacker
 Widow
 WinHTTrack
 Wonderbot
+Wotbox
+Xaldon WebSpider
 YoudaoBot
 Zade
 Zauba
 Zermelo
+Zeus
 Zitebot
-adscanner
-archive.org_bot
+arquivo-web-crawler
 arquivo.pt
 autoemailspider
-check1.exe
-clark-crawler
+backlink-check
 coccocbot-web
+cognitiveseo
 crawl.sogou.com
-crawler4j
-dataforseo.com
 demandbase-bot
-eCatch
-evc-batch
-instabid
-internetVista monitor
-magpie-crawler
-oBot
-plumanalytics
-probe-image-size
-ripz
-s1z.ru
+ips-agent
+lwp-trivial
+meanpathbot
+mediawords
+page scorer
+scalaj-http
+scan.lol
 seobility
-seocompany.store
+seoscanners
 sexsearcher
-siteripz
 sogouspider
 spyfu
+tAkeOut
+trendiction.com
+ubermetrics-technologies.com
+webgains-bot
 webmeup-crawler
-webpros.com
 webprosbot
 x09Mozilla
-x22Mozilla
+xpymep1.exe
+zgrab