V4.2021.07.2478 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-360Spider
+01h4x.com
 404checker
 404enemy
-AIBOT
+ADmantX
 ASPSeek
-Abonti
 Aboundex
+Acunetix
 AfD-Verbotsverfahren
+AhrefsBot
 Aipbot
 Alexibot
+Anarchy99
 Ankit
-BDCbot
-BackWeb
+Anthill
+Apexoo
+Aspiegel
+Attach
+AwarioSmartBot
+BackStreet
+Badass
 Battleztar Bazinga
-Bigfoot
 Bitacle
-Black Hole
+BlackWidow
+Blackboard
+Blow
 BlowFish
-Bolt
-BotALot
-Brandprotect
-Buck
-Buddy
+Boardreader
+Brandwatch
 BuiltBotTough
 BuiltWith
+Bullseye
+BunnySlippers
 CCBot
-Calculon
-CheTeam
+CSHttp
+CheeseBot
+Chlooe
 Claritybot
-Cloud mapping
+Cocolyzebot
 Cogentbot
+Collector
 Copier
 CopyRightCheck
 Copyscape
+Craftbot
+CrazyWebCrawler
+CyotekWebCopy
 DBLBot
-DatabaseDriverMysqli
-Demon
-Digincore
+DSearch
+DTS Agent
+DataCha0s
 DigitalPebble
-Disco
-Discoverybot
+Dirbuster
 Dispatch
-DnyzBot
+DittoSpyder
+DomCopBot
 DomainCrawler
-DomainStatsBot
+Domains Project
+Dotbot
+Download Wonder
 Dragonfly
 Drip
 ECCP/1.0
-EMail Siphon
 EMail Wolf
-Ecxi
-EroCrawler
-ExtLinksBot
-Ezooms
-FDM
+EasyDL
+EirGrabber
+Evil
+Exabot
+ExtractorPro
 FHscan
 Firefox/7.0
-FlashGet
+Freeuploader
 FyberSpider
 GT::WWW
-GalaxyBot
+Genieo
+GetWeb
 Getintent
-Go!Zilla
+Gigabot
 Gotit
-GrabNet
 Grafula
-GrapeFX
+GrapeshotCrawler
 GridBot
 HEADMasterSEO
-HMView
-HTTP::Lite
-Harvest
-Havij
+HTTrack
 Heritrix
-Humanlinks
 IDBTE4M
-Id-search
+IRLbot
+IlseBot
 Image Sucker
-InfoNaviRobot
 InfoTekies
-InterGET
-Internet Ninja
+Intelliseek
 InternetSeer
+Iskanie
 IstellaBot
 JOC Web Spider
 JamesBOT
 Jbrofuzz
-JetCar
-JikeSpider
-Joomla
-JustView
-Kozmosbot
+JennyBot
+Jorgee
+Jyxobot
+Kenjin Spider
+Keyword Density
+Kinza
 LNSpiderguy
+Lanshanbot
+LeechFTP
 LeechGet
-Likse
+LibWeb
+Libwhisker
+LinkScan
 LinkextractorPro
+LinkpadBot
 LinksManager
-LinqiaRSSBot
-Lipperhey
-Litemage_walker
-Ltx71
+LinqiaMetadataDownloaderBot
+LinqiaScrapeBot
 MFC_Tear_Sample
 MJ12bot
 MQQBrowser
-MSIECrawler
 Magnet
+Mail.RU_Bot
 Majestic SEO
 Majestic-SEO
+Majestic12
 MarkWatch
-Mass Downloader
 Masscan
-MegaIndex.ru
 MicroMessenger
-Mister PiX
-Mr.4x3
+Moblie Safari
+Msrabot
+Musobot
 NPbot
 Name Intelligence
-Nameprotect
-Navroad
-Nessus
+NearSite
+Needle
 Net Vampire
-NetAnts
-NetLyzer
 NetMechanic
-NetSpider
-NimbleCrawler
+Netcraft
+Niki-bot
+Nikto
+Nmap
 Nuclei
 Octopus
-Offline Navigator
-Openfind
-Openvas
-OrangeSpider
-OutfoxBot
-PHPCrawl
+OutclicksBot
 POE-Component-Client-HTTP
-PageAnalyzer
-PageThing.com
 Pandalytics
-Papa Foto
-PeoplePal
-Picscout
-Pixray
-ProWebWalker
-Psbot
-Pump
-PyCurl
+Panscient
+Pi-Monster
+PictureFinder
+PleaseCrawl
+Probethenet
+PxBroker
 QueryN Metasearch
-RSSingBot
-RankActive
-RankingBot
+Quick-Crawler
+RankingBot2
+Rankivabot
 RankurBot
-RedesScrapy
-RepoMonkey
-RocketCrawler
+Re-re
+ReGet
+RealDownload
+Reaper
+RebelMouse
+Recorder
+Ripper
+SEOkicks
 SEOlyticsCrawler
+SMTBot
+SalesIntelligent
+ScoutJet
+Scrapy
+Screaming
+Searchestate
 SearchmetricsBot
-SemrushBot
-SentiBot
 SeoSiteCheckup
-Seomoz
+Siphon
 Site Sucker
-SiteCheckerBotCrawler
-SiteSnagger
 Sitevigil
-Snoopy
-Sottopop
-SpaceBison
-Spbot
-Spinn3r
-Sqlmap
-Sqlworm
-Sqworm
+SocialRankIOBot
+Sociscraper
+SpankBot
 Steeler
 Sucuri
-SuperHTTP
-Surfbot
-SurveyBot
-Suzuran
-T0PHackTeam
-T8Abot
 TeleportPro
-Telesoft
-Telesphorep
-The Intraformant
+TheNomad
 Thumbor
 Titan
-Toweyabot
-Trendiction
-Trendictionbot
-True_Robot
-Typhoeus
+Toata
+Tracemyfile
+Turingos
+TwengaBot
+UnisterBot
 Upflow
-VCI
-Virusdie
-VoidEYE
-Voil
+V-BOT
+VB Project
+Vacuum
+Vagabondo
+VeriCiteCrawler
 WPScan
+WWW-Mechanize
 WWW::Mechanize
-WeSEE
-Web Collage
-Web Enhancer
-Web Sucker
+Wallpapers/3.0
+WallpapersHD
+WebCollage
 WebCopier
-WebGo IS
+WebImageCollector
 WebLeacher
 WebPix
+WebReaper
 WebWhacker
+WebZIP
+WebmasterWorldForumBot
+Webshag
+Website Quester
+WebsiteExtractor
 WebsiteQuester
-Webster
 Whack
-WinHTTrack
+Whacker
+Whatweb
+Widow
 Wonderbot
+Woobot
 Wotbox
-Wprecon
-Xaldon_WebSpider
+Xaldon WebSpider
 YoudaoBot
-Zade
-Zauba
-Zermelo
+ZmEu
 ZoominfoBot
-arquivo.pt
-check1.exe
+ZyBorg
+arquivo-web-crawler
+backlink-check
 coccocbot-web
 crawler.feedback
-demandbase-bot
+crawler4j
+domainsproject.org
 eCatch
-gopher
-internetVista monitor
-ips-agent
+evc-batch
+instabid
+isitwp.com
 lwp-request
 lwp-trivial
 mediawords
 muhstik-scan
-page scorer
-pcBrowser
-plumanalytics
-polaris version
-ripz
+probe-image-size
 s1z.ru
 scalaj-http
-scan.lol
-seobility
-seoscanners
+seostar
+sexsearcher
 siteripz
 spyfu
+tAkeOut
 trendiction.com
 trendiction.de
-webmeup-crawler
-xpymep1.exe
-zauba.io
+x09Mozilla
+zgrab