V4.2022.01.2843 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-ASPSeek
-AhrefsBot
-Anarchie
+360Spider
+404checker
+404enemy
+AIBOT
+Abonti
+Aboundexbot
+AiHitBot
+AllSubmitter
+AlphaBot
+Anarchy99
 Anthill
-Aspiegel
-BBBike
+AwarioSmartBot
+BDCbot
 BDFetch
-BackStreet
-BackWeb
+BLEXBot
 BacklinkCrawler
-Badass
-Bandit
-BatchFTP
-BetaBot
-Bigfoot
-Boardreader
-Bolt
-BotALot
-Buck
-BuiltBotTough
+Bitacle
+Black Hole
+BlackWidow
+BuiltWith
 Bullseye
 BunnySlippers
-BuzzSumo
 CSHttp
-Calculon
 CazoodleBot
 Cegbfeieh
-CensysInspect
-CheeseBot
 ChinaClaw
+Citoid
 Claritybot
-Cloud mapping
-Crawling at Home Project
-Crescent
-CrunchBot
+Cocolyzebot
+Collector
 Curious
-Custo
+CyotekWebCopy
+DIIbot
+DTS Agent
 DataCha0s
-DatabaseDriverMysqli
-Demon
-Deusu
-Digincore
-Dispatch
+Devil
+Discoverybot
 DittoSpyder
-DnyzBot
+DnBCrawler-Analytics
 DomCopBot
-DomainAppender
-DomainCrawler
+DomainSigmaCrawler
 DomainStatsBot
 Domains Project
-Dotbot
-Dragonfly
-EMail Siphon
+Download Wonder
+Drip
+EasyDL
 Ebingbong
-EirGrabber
-EroCrawler
+Ecxi
 Evil
 Exabot
 Extractor
-ExtractorPro
+Extreme Picture Finder
 EyeNetIE
-FDM
-FemtosearchBot
-FlashGet
-FyberSpider
-G-i-g-a-b-o-t
-GalaxyBot
+Firefox/7.0
+Foobot
+Fuzz
+Fyrebot
+GT::WWW
 Genieo
 GermCrawler
-GetRight
+GetWeb
 Getintent
+Gigabot
 Go!Zilla
+Go-Ahead-Got-It
+GoZilla
 Gotit
-Grafula
+GrabNet
+Grabber
 GrapeFX
+GridBot
+HTMLparser
 HTTP::Lite
-HaosouSpider
-Humanlinks
-IlseBot
-IndeedBot
-Indy Library
-InfoTekies
-InterGET
+Harvest
+Havij
+IDBTE4M
+IDBot
+IRLbot
+Image Sucker
+Intelliseek
+InternetSeer
+JOC Web Spider
 JetCar
-Jorgee
-JustView
+Jetty
+Joomla
 Jyxobot
-Kenjin Spider
-Keyword Density
 Kozmosbot
-LNSpiderguy
 LWP::Simple
 Lanshanbot
 Larbin
-Leap
-LeechFTP
-LibWeb
-Libwhisker
-LinkScan
+LeechGet
+LexiBot
+Lftp
+Lightspeedsystems
+Likse
 LinkWalker
-LinkpadBot
-LinksManager
+Linkbot
+Linkdexbot
+LinqiaMetadataDownloaderBot
 LinqiaRSSBot
-Lmspider
-MFC_Tear_Sample
+Lipperhey
+Lipperhey Spider
+Litemage_walker
+Ltx71
+MIDown tool
+MIIxpc
 MJ12bot
-MQQBrowser
-MSFrontPage
 Magnet
+Majestic SEO
 MarkMonitor
 MarkWatch
 Mass Downloader
 Mata Hari
 MauiBot
 MeanPath Bot
-Meanpathbot
+Mediatoolkitbot
 MegaIndex.ru
-Metauri
-Moblie Safari
+MicroMessenger
+Mojeek
 Mojolicious
-Morfeus Fucking Scanner
-Mozlila
-Mr.4x3
 Msrabot
-Musobot
 NICErsPRO
-Name Intelligence
-Navroad
-Nessus
+Net Vampire
 NetAnts
-NetLyzer
-NetSpider
+NetMechanic
 NetZIP
 Netvibes
-Nibbler
-Nikto
+NextGenSearchBot
 NimbleCrawler
 Nimbostratus
 Nmap
-Not
 Nuclei
-Nutch
 Offline Explorer
-Offline Navigator
-OpenLinkProfiler
-PECL::HTTP
-PHPCrawl
+OrangeBot
+OrangeSpider
+OutfoxBot
+POE-Component-Client-HTTP
 PageAnalyzer
+PageGrabber
+PageScorer
+Pandalytics
+Papa Foto
+Pavuk
 Pi-Monster
-Picscout
-Picsearch
 Piepmatz
-ProPowerBot
+Pixray
 ProWebWalker
-Psbot
-Pump
+Probethenet
+Pu_iN
+PxBroker
 PyCurl
 Quick-Crawler
 RSSingBot
-RankActive
+RankFlex
 Rankivabot
-Reaper
+RealDownload
 RebelMouse
-Recorder
-RedesScrapy
-Ripper
-RocketCrawler
 SBIder
-SEOkicks-Robot
 SEOlyticsCrawler
-SEOprofiler
-SEOstats
-SalesIntelligent
 Scanbot
-ScrepyBot
-Searchestate
-SearchmetricsBot
+Scrapy
+Screaming
 Seekport
-SemanticJuice
-Semrush
-SeoSiteCheckup
-Shodan
+SeobilityBot
+Seomoz
 SiteLockSpider
-Snoopy
+SiteSnagger
+Sitevigil
+Snapbot
+SocialRankIOBot
+Sociscraper
 Sosospider
 Sottopop
 Spinn3r
-SurveyBot
+Sqlworm
+Steeler
+Stripper
+Sucker
+SuperHTTP
+Surfbot
+T8Abot
 Teleport
+Telesoft
+Telesphoreo
+Telesphorep
 Thumbor
-TightTwatBot
 Titan
-Toweyabot
+Tracemyfile
+Trendiction
 Trendictionbot
-Turnitin
-TwengaBot
+TurnitinBot
+Twice
 Typhoeus
-URLy.Warning
-Upflow
-Vagabondo
-VidibleScraper
-Virusdie
-Voltron
-WASALive-Bot
-WBSearchBot
+URLy Warning
+V-BOT
+VB Project
+Vacuum
+VeriCiteCrawler
+VoidEYE
+Voil
 WEBDAV
-WeSEE
-Web Fetch
-Web Sauger
-WebFetch
-WebPix
+WWW-Collector-E
+WallpapersHD
+Web Enhancer
+Web Sucker
+WebGo IS
+WebLeacher
 WebReaper
 WebSauger
-WebStripper
-WebWhacker
-WebmasterWorldForumBot
-WebsiteExtractor
-Webster
+WebSucker
+WebZIP
+Webalta
+WebsiteQuester
+Whack
+Whacker
 Whatweb
-Who.is Bot
-Woobot
+WinHTTrack
+WiseGuys Robot
+Wonderbot
 Wotbox
-Xaldon WebSpider
+Xaldon_WebSpider
 Xenu
-ZmEu
+YoudaoBot
+adscanner
 archive.org_bot
+arquivo.pt
 coccocbot-web
-com.plumanalytics
-crawler4j
+crawler.feedback
 domainsproject.org
 eCatch
-evc-batch
-heritrix
+facebookscraper
 instabid
 isitwp.com
-meanpathbot
-plumanalytics
-ripz
+lwp-trivial
+mediawords
+muhstik-scan
+oBot
+polaris version
+s1z.ru
 seocompany.store
-seoscanners
+serpstatbot
 sexsearcher
 sitechecker.pro
-sogouspider
-sp_auditbot
+siteripz
 spyfu
-trendiction.com
-webprosbot
+tAkeOut
+trendiction.de
+voyagerx.com
+x09Mozilla
 xpymep1.exe
-zauba.io