V4.2022.01.2850 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
+01h4x.com
+360Spider
 404checker
-404enemy
+ADmantX
+AIBOT
 ASPSeek
-Abonti
 Aboundex
-Acunetix
+Aboundexbot
+AfD-Verbotsverfahren
 AiHitBot
-Aipbot
-AllSubmitter
+Alexibot
+AlphaBot
 Anarchie
+Anarchy99
 Ankit
-Asterias
-AwarioSmartBot
-BDCbot
-BLEXBot
-Badass
+Attach
+AwarioRssBot
+Backlink-Ceck
+BatchFTP
+Battleztar Bazinga
 Bigfoot
-Black Hole
-BlackWidow
+Bitacle
 Blackboard
-Boardreader
 Bolt
-BotALot
 Brandprotect
-BuiltBotTough
-CSHttp
-Cegbfeieh
+Buck
+Bullseye
+CODE87
+CazoodleBot
 CensysInspect
 CheTeam
 CherryPicker
-ChinaClaw
-Citoid
-Claritybot
-Cliqzbot
 Cloud mapping
 Cocolyzebot
-Cogentbot
-CopyRightCheck
+Copier
 Copyscape
-Cosmos
+Crawling at Home Project
+CrazyWebCrawler
 Crescent
 Custo
-DIIbot
+DBLBot
 DataCha0s
 Devil
 DigitalPebble
-Dirbuster
+Discoverybot
+Dispatch
+DittoSpyder
 DnBCrawler-Analytics
-DnyzBot
 DomCopBot
-DomainAppender
+DomainCrawler
+Download Wonder
 Dragonfly
-Drip
+ECCP/1.0
 EMail Siphon
-EMail Wolf
 EasyDL
 Ebingbong
 Ecxi
-EirGrabber
+EroCrawler
+Evil
 Exabot
-ExtractorPro
-FDM
+Extreme Picture Finder
 FHscan
-Foobot
-FrontPage
-FyberSpider
-Genieo
+Fimap
+Freeuploader
+G-i-g-a-b-o-t
+GalaxyBot
 GermCrawler
-Gigabot
-GoZilla
+GetWeb
+Getintent
+Gotit
 Grabber
-GrapeshotCrawler
+GrapeFX
 HEADMasterSEO
-HTMLparser
+HMView
 HTTP::Lite
-Haansoft
-HaosouSpider
-Hloader
 Humanlinks
 HybridBot
-IDBot
-Image Sucker
+IDBTE4M
+IRLbot
+Image Fetch
 IndeedBot
-InfoTekies
-InterGET
+Indy Library
+InfoNaviRobot
+Intelliseek
 Iskanie
 JOC Web Spider
 JamesBOT
 Jbrofuzz
-JetCar
-Joomla
-Jorgee
-JustView
-Keyword Density
-Kinza
-Lanshanbot
+JennyBot
+Jyxobot
+LNSpiderguy
+LWP::Simple
 Leap
+LeechGet
+LexiBot
 LibWeb
-LieBaoFast
-Lightspeedsystems
 Likse
+Linkdexbot
 LinkextractorPro
-LinqiaScrapeBot
-Lipperhey Spider
+LinqiaRSSBot
 Litemage_walker
-MFC_Tear_Sample
+MIDown tool
 MIIxpc
-MSFrontPage
 MTRobot
-Majestic SEO
-MarkMonitor
-MarkWatch
-Mata Hari
-Mediatoolkitbot
+Mag-Net
+Magnet
+Mail.RU_Bot
+Majestic-SEO
+Mass Downloader
+Masscan
+MeanPath Bot
+Meanpathbot
 MegaIndex.ru
 Metauri
+MicroMessenger
+Minefield
 Mister PiX
-Mozlila
+Moblie Safari
+Mojeek
+Morfeus Fucking Scanner
 Mr.4x3
-Musobot
+Msrabot
+Name Intelligence
+Nameprotect
+Navroad
+Needle
 NetAnts
+NetMechanic
 NetSpider
-NextGenSearchBot
-Nikto
-Offline Explorer
-Openfind
-OrangeSpider
-OutclicksBot
-PECL::HTTP
-PHPCrawl
-Pi-Monster
+NetZIP
+Nimbostratus
+Ninja
+Not
+Nutch
+OnCrawl
+OpenVAS
+Openvas
+Pavuk
+PeoplePal
 Picscout
-Pixray
-PleaseCrawl
-Pockey
-ProPowerBot
-Probethenet
-Psbot
-Pump
-PxBroker
+Piepmatz
 PyCurl
-RankActive
+QueryN Metasearch
+Quick-Crawler
+RSSingBot
 RankActiveLinkBot
-RankFlex
-RankingBot2
-Re-re
-Reaper
+RankurBot
+RealDownload
 RebelMouse
-RepoMonkey
+Ripper
+Rogerbot
 SEOprofiler
 SEOstats
-SalesIntelligent
+SISTRIX
 ScanAlert
 Scanbot
 ScoutJet
+Scrapy
+Screaming
 ScreenerBot
 ScrepyBot
 SearchmetricsBot
+Seekport
+SemanticJuice
 Semrush
-SemrushBot
-SentiBot
-SeoSiteCheckup
 Seomoz
+Shodan
 Siphon
-Site Sucker
+SiteCheckerBotCrawler
+SiteExplorer
+SiteLockSpider
+Siteimprove
 Sitevigil
-SmartDownload
 Snake
 Snoopy
-Sociscraper
-Spammen
-SpankBot
-Spanner
+Sosospider
+SpaceBison
 Spbot
-Sqlworm
 Sqworm
-Sucuri
-SuperHTTP
+Stripper
+SuperBot
 Surfbot
-SurveyBot
-T0PHackTeam
-TeleportPro
-Toata
-Turingos
-TurnitinBot
-Twice
-URLy Warning
-URLy.Warning
+Suzuran
+Swiftbot
+Szukacz
+Telesphoreo
+Telesphorep
+TheNomad
+Titan
+Tracemyfile
+Trendictionbot
+True_Robot
+UnisterBot
 Upflow
 V-BOT
-VCI
-VelenPublicWebCrawler
+VB Project
+Vacuum
+Vagabondo
 Virusdie
-VoidEYE
-WASALive-Bot
-WISENutbot
+WEBDAV
 WPScan
 WWW-Mechanize
-WWW::Mechanize
 Wallpapers
-WallpapersHD
+Wallpapers/3.0
 WeSEE
-WebCollage
-WebFetch
-WebFuck
-WebImageCollector
-WebPix
-Webalta
+Web Enhancer
+Web Fetch
+Web Fuck
+Web Pix
+WebGo IS
+WebLeacher
+WebWhacker
 WebmasterWorldForumBot
 Webshag
-WebsiteQuester
-Whatweb
-Who.is Bot
-Widow
+WebsiteExtractor
+Webster
+WinHTTrack
+Wprecon
 Xaldon WebSpider
-Xaldon_WebSpider
-Xenu
-Zade
-Zermelo
+Zitebot
 ZoomBot
-archive.org_bot
-arquivo-web-crawler
-autoemailspider
-check1.exe
-crawl.sogou.com
+ZumBot
+arquivo.pt
+cah.io.community
 crawler.feedback
-crawler4j
 dataforseo.com
 domainsproject.org
+evc-batch
 facebookscraper
-gopher
-instabid
+heritrix
+ips-agent
+isitwp.com
 magpie-crawler
-muhstik-scan
-netEstate NE Crawler
-pcBrowser
-plumanalytics
-polaris version
-probe-image-size
+oBot
+page scorer
 ripz
+satoristudio.net
+scalaj-http
+scan.lol
+seobility
 seocompany.store
-serpstatbot
-sexsearcher
-sysscan
-trendiction.com
-voyagerx.com
-webmeup-crawler
-webprosbot
-x09Mozilla
+seoscanners
+seostar
+siteripz
+sp_auditbot
+tAkeOut
+webgains-bot
 x22Mozilla
+xpymep1.exe
+zgrab