V4.2021.07.2459 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-360Spider
+01h4x.com
 404checker
-ADmantX
+ASPSeek
 Aboundex
-AhrefsBot
-AiHitBot
+Aboundexbot
+Aipbot
 Alexibot
 AllSubmitter
-Alligator
-Apexoo
-Aspiegel
+AlphaBot
+Anarchie
+Anarchy
+Ankit
 Attach
+AwarioRssBot
+AwarioSmartBot
+BDCbot
 BDFetch
-BLEXBot
+Backlink-Ceck
 BacklinkCrawler
-Badass
-BetaBot
-BlowFish
+Barkrowler
+BatchFTP
+Bitacle
+BlackWidow
+Blow
+Boardreader
 Bolt
-BotALot
-Brandprotect
 Buck
 Buddy
-BuiltBotTough
-BuiltWith
-BuzzSumo
-CATExplorador
 CCBot
 CODE87
 CSHttp
-Calculon
-ChinaClaw
-Copyscape
+CazoodleBot
+CensysInspect
+CherryPicker
+Claritybot
+Cogentbot
+Cosmos
 Craftbot
-CrazyWebCrawler
 Crescent
+CrunchBot
 Curious
 CyotekWebCopy
-DSearch
+DBLBot
 DTS Agent
 DataCha0s
+Devil
+Digincore
 DigitalPebble
+Disco
 Discobot
-DittoSpyder
-DomainCrawler
+Discoverybot
+Dispatch
 DomainSigmaCrawler
+DomainStatsBot
 Domains Project
+Dotbot
 Download Wonder
+EMail Siphon
+Ecxi
+EroCrawler
 Evil
 Express WebPictures
 ExtLinksBot
-Extractor
-ExtractorPro
-Extreme Picture Finder
-EyeNetIE
-Ezooms
-FemtosearchBot
-Firefox/7.0
-Foobot
-Freeuploader
-G-i-g-a-b-o-t
-GalaxyBot
-GermCrawler
-GetRight
-Gigabot
-Go!Zilla
+FHscan
+Fimap
+FrontPage
+Fuzz
+GT::WWW
 Go-Ahead-Got-It
-Grabber
-HMView
-HTTP::Lite
+Grafula
+GrapeFX
+HTMLparser
 HTTrack
 Haansoft
-Humanlinks
-HybridBot
-IDBot
-IlseBot
-Image Fetch
-Image Sucker
-Intelliseek
+HaosouSpider
+Harvest
+IDBTE4M
+IRLbot
+Iblog
+Id-search
 InterGET
-JOC Web Spider
-Jbrofuzz
-JikeSpider
+Iskanie
+IstellaBot
+JamesBOT
+Jetty
 Jorgee
+Kenjin Spider
 Keyword Density
-Kinza
-Kozmosbot
+LNSpiderguy
 Lanshanbot
-Larbin
 LeechGet
 Lftp
 Libwhisker
 LieBaoFast
-Lightspeedsystems
-Linkdexbot
-LinkextractorPro
+LinkpadBot
 LinksManager
-LinqiaMetadataDownloaderBot
-LinqiaScrapeBot
 Lipperhey
 Lipperhey Spider
-Litemage_walker
 Lmspider
-MIIxpc
 MJ12bot
 MQQBrowser
+MTRobot
 Mag-Net
-Magnet
+Mail.RU_Bot
+Majestic SEO
 Majestic-SEO
-MarkMonitor
-Mata Hari
-Mb2345Browser
+Majestic12
+Mass Downloader
+MeanPath Bot
 Meanpathbot
 Mediatoolkitbot
+MegaIndex.ru
 Metauri
-Moblie Safari
-Mojeek
-Mojolicious
+MicroMessenger
+Microsoft URL Control
+Mozlila
+Mr.4x3
+Msrabot
 NICErsPRO
-Navroad
+Name Intelligence
 Nessus
-NetAnts
 NetLyzer
+NetZIP
 Netvibes
 Nibbler
+Niki-bot
+Nikto
 Ninja
-Nutch
+Nuclei
+Octopus
+Offline Explorer
+OnCrawl
 OpenLinkProfiler
 Openvas
-OrangeBot
-OrangeSpider
-OutclicksBot
+OutfoxBot
 PECL::HTTP
-PageGrabber
+POE-Component-Client-HTTP
+PageAnalyzer
 PageScorer
+Pandalytics
 Panscient
-Pavuk
-Pi-Monster
-Pimonster
-Pockey
-ProPowerBot
+Papa Foto
+PictureFinder
+Pixray
+PleaseCrawl
 Pu_iN
 Pump
 Quick-Crawler
 RSSingBot
-RankActiveLinkBot
-RankFlex
-RankingBot
+RankActive
 RankurBot
-RealDownload
-Recorder
-Ripper
-Rogerbot
+RebelMouse
+SBIder
+SEOkicks
 SEOkicks-Robot
-SISTRIX
-ScanAlert
-Scanbot
-Scrapy
-Screaming
-SeoSiteCheckup
-Site Sucker
-SiteExplorer
+SEOprofiler
+SEOstats
+ScreenerBot
+Searchestate
+SemrushBot
+Seomoz
 SiteSnagger
 SiteSucker
-Sitebeam
-SmartDownload
+Snake
 Snoopy
-Sociscraper
+SocialRankIOBot
 Sogou web spider
-Sottopop
+Spammen
+Spinn3r
+SputnikBot
 Sqlmap
+Sqworm
 Stripper
-SuperBot
+Sucuri
+SuperHTTP
 Surfbot
+Suzuran
 Swiftbot
-T0PHackTeam
-T8Abot
+Teleport
 Telesoft
-The Intraformant
+TheNomad
 TightTwatBot
-Tracemyfile
-Trendictionbot
-TurnitinBot
-Twice
+Titan
+True_Robot
+TwengaBot
+Typhoeus
+URLy Warning
+URLy.Warning
 UnisterBot
-Upflow
 V-BOT
-VCI
-Vacuum
-Vagabondo
-VeriCiteCrawler
+VB Project
+VelenPublicWebCrawler
+Virusdie
 VoidEYE
-WBSearchBot
+Voltron
 WEBDAV
 WISENutbot
 WWW-Collector-E
 WWW-Mechanize
 WWW::Mechanize
-Wallpapers/3.0
-WeSEE
-Web Enhancer
+WWWOFFLE
+WallpapersHD
+Web Auto
+Web Fetch
 Web Pix
+WebAuto
 WebCollage
-WebFetch
+WebFuck
 WebImageCollector
-WebPix
-WebSauger
-WebStripper
-WebSucker
-WebsiteExtractor
+WebReaper
+WebWhacker
+WebZIP
+WebmasterWorldForumBot
 WebsiteQuester
 Webster
 Whack
-Whacker
-Whatweb
 Who.is Bot
+Widow
+WinHTTrack
 Wonderbot
+Woobot
 Wotbox
-Xaldon WebSpider
-Xenu
+Wprecon
+Xaldon_WebSpider
 YoudaoBot
 Zade
-Zeus
-Zitebot
+Zauba
+ZmEu
+ZoominfoBot
 ZumBot
 ZyBorg
-autoemailspider
-backlink-check
-coccocbot-web
+arquivo.pt
+cognitiveseo
 crawler.feedback
-crawler4j
-demandbase-bot
-domainsproject.org
-eCatch
-facebookscraper
-gopher
-instabid
-ips-agent
+internetVista monitor
 lwp-trivial
-page scorer
+netEstate NE Crawler
 pcBrowser
 polaris version
-satoristudio.net
+s1z.ru
+scan.lol
+seostar
+serpstatbot
 sexsearcher
-sitechecker.pro
-spyfu
+siteripz
+sogouspider
 sysscan
-trendiction.com
 x09Mozilla
 x22Mozilla
+zauba.io