V4.2021.09.2599 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-360Spider
-ADmantX
-ASPSeek
-Abonti
+80legs
+AIBOT
 Aboundex
+AfD-Verbotsverfahren
 AiHitBot
+Aipbot
+Alexibot
+AllSubmitter
+Alligator
 AlphaBot
-Anarchie
-Anarchy
-Ankit
+Anarchy99
+Apexoo
 Aspiegel
-Attach
-BDFetch
+BBBike
+BDCbot
+BLEXBot
 BackWeb
+BacklinkCrawler
 Badass
-Bandit
 Barkrowler
 BatchFTP
-BetaBot
 BlackWidow
+Blackboard
+Blow
 BotALot
-Brandprotect
-Bullseye
-BunnySlippers
-CATExplorador
+Buck
+BuiltBotTough
 CCBot
 CSHttp
 CazoodleBot
 Cegbfeieh
 CensysInspect
-CheeseBot
 CherryPicker
-ChinaClaw
-Cloud mapping
-Cocolyzebot
+Chlooe
+Cliqzbot
 Cogentbot
 CopyRightCheck
-CrazyWebCrawler
+Copyscape
+Craftbot
+Crawling at Home Project
 Crescent
-CrunchBot
-Demon
+Custo
+DSearch
+DataCha0s
+DatabaseDriverMysqli
 Deusu
-DigitalPebble
+Devil
+Digincore
 Disco
 Discobot
-Dispatch
-DomainCrawler
+DittoSpyder
+DnyzBot
+DomainAppender
 DomainStatsBot
-ECCP/1.0
+Download Wonder
 EMail Siphon
-Express WebPictures
-ExtLinksBot
-ExtractorPro
-Ezooms
-FDM
-FemtosearchBot
+EasyDL
+Ecxi
+EroCrawler
+Evil
+Exabot
+Extreme Picture Finder
+FHscan
 FlashGet
-Foobot
-Fuzz
-FyberSpider
+FrontPage
+Fyrebot
+GT::WWW
 GalaxyBot
-GetWeb
+Getintent
 Gigabot
+GoZilla
+Gotit
+GrabNet
 Grabber
 Grafula
-GrapeshotCrawler
-GridBot
-HTTP::Lite
-HaosouSpider
+HTMLparser
+Haansoft
 Harvest
-Havij
-Hloader
+Heritrix
+IDBot
+Iblog
 Id-search
+IlseBot
+Image Fetch
 Indy Library
-Internet Ninja
+InfoNaviRobot
+Intelliseek
+InterGET
+Iskanie
 JOC Web Spider
-JamesBOT
+Jbrofuzz
 JennyBot
-Jetty
+JetCar
 Joomla
-Keyword Density
-Kozmosbot
+Jorgee
+JustView
+LNSpiderguy
 Lanshanbot
-Larbin
-LeechFTP
-Lftp
-LibWeb
-Libwhisker
-Lightspeedsystems
+LieBaoFast
 Likse
 LinkScan
-Linkbot
-LinkextractorPro
-LinksManager
-Lipperhey
-Litemage_walker
-Ltx71
+LinkWalker
+LinqiaRSSBot
 MIIxpc
 MQQBrowser
-MSFrontPage
-MSIECrawler
 MTRobot
 Mag-Net
 Magnet
-Majestic-SEO
+Majestic SEO
+Majestic12
 MarkWatch
-Mass Downloader
-MeanPath Bot
+Masscan
 Meanpathbot
 Microsoft Data Access
-Mister PiX
+Mojeek
 Morfeus Fucking Scanner
+Mozlila
 Mr.4x3
 Msrabot
+Musobot
 NICErsPRO
-Navroad
 NearSite
-Net Vampire
+Needle
+Nessus
+NetAnts
 NetLyzer
 NetSpider
 Netcraft
-Nettrack
-NimbleCrawler
-Offline Explorer
-Offline Navigator
-OrangeBot
+NextGenSearchBot
+Nibbler
+Niki-bot
+Nimbostratus
+Octopus
+OnCrawl
 OrangeSpider
 OutclicksBot
-OutfoxBot
-PECL::HTTP
+PHPCrawl
 Page Analyzer
-PageScorer
-PageThing.com
+PageGrabber
 PeoplePal
-Petalbot
 Pi-Monster
-Piepmatz
-Pixray
-Pockey
+PictureFinder
 ProPowerBot
 Probethenet
 Psbot
+Pu_iN
 PxBroker
-PyCurl
-Quick-Crawler
-RankActiveLinkBot
-RankingBot
+RSSingBot
+RankFlex
 RankingBot2
 Rankivabot
-Re-re
-ReGet
-RealDownload
-RepoMonkey
-SEOkicks
-SEOkicks-Robot
+Reaper
+RedesScrapy
 SEOprofiler
+SMTBot
 SalesIntelligent
 ScanAlert
-ScoutJet
+Scanbot
 Scrapy
+Screaming
 Searchestate
-SearchmetricsBot
-Seomoz
-SiteCheckerBotCrawler
-SiteSnagger
-SiteSucker
+Shodan
+Siphon
+SiteExplorer
+SiteLockSpider
+Sitebeam
 Siteimprove
 Sitevigil
-SmartDownload
+Snake
 Snapbot
-Sogou web spider
-SpaceBison
+Snoopy
+SocialRankIOBot
+Sosospider
 SpankBot
-Spanner
-Spbot
 SputnikBot
-Sucuri
-SuperHTTP
-T0PHackTeam
-Telesoft
-TheNomad
-TightTwatBot
-Trendictionbot
+Sqlworm
+Sqworm
+Steeler
+Stripper
+Suzuran
+Telesphoreo
+Toata
+Tracemyfile
 True_Robot
-Turingos
-V-BOT
+TwengaBot
+URLy.Warning
+Upflow
+VB Project
+VCI
+Vacuum
 Vagabondo
+VelenPublicWebCrawler
 Virusdie
-VoidEYE
 Voil
+WASALive-Bot
 WEBDAV
-WPScan
-WWW-Mechanize
-Wallpapers
+WWW::Mechanize
+Wallpapers/3.0
 WeSEE
-Web Auto
-Web Sucker
 WebAuto
-WebBandit
-WebEnhancer
 WebFetch
-WebFuck
-WebGo IS
 WebPix
-WebReaper
 WebSauger
-WebWhacker
-WebZIP
-Website Quester
+WebStripper
+WebSucker
+Webalta
+WebsiteExtractor
 WebsiteQuester
-Who.is Bot
-WiseGuys Robot
-Wotbox
-Wprecon
-Xenu
+Whatweb
+Widow
+WinHTTrack
 Zade
+Zermelo
 Zeus
-ZmEu
 ZoomBot
-archive.org_bot
+ZoominfoBot
+ZyBorg
+arquivo-web-crawler
 arquivo.pt
-cah.io.community
-check1.exe
-coccocbot-web
-cognitiveseo
+crawl.sogou.com
+crawler.feedback
 crawler4j
-demandbase-bot
-domainsproject.org
+evc-batch
 facebookscraper
-heritrix
 internetVista monitor
 ips-agent
-lwp-request
+isitwp.com
+meanpathbot
 mediawords
 muhstik-scan
 oBot
-pcBrowser
-plumanalytics
+polaris version
 ripz
 s1z.ru
 scan.lol
-seoscanners
+seobility
 seostar
-siteripz
+sexsearcher
+sitechecker.pro
+sp_auditbot
 spyfu
+sysscan
+tAkeOut
+trendiction.com
 trendiction.de
-webprosbot
-x09Mozilla
+voyagerx.com
+webmeup-crawler
 x22Mozilla
-xpymep1.exe
+zauba.io
+zgrab