V4.2021.09.2584 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-360Spider
-ALittle Client
+404enemy
+AIBOT
 ASPSeek
 Abonti
-Acunetix
-AfD-Verbotsverfahren
-AhrefsBot
-AiHitBot
-Aipbot
-Alligator
-Aspiegel
-Asterias
-AwarioRssBot
+Aboundexbot
+AlphaBot
+Anarchy
+Ankit
+Apexoo
+Attach
+BBBike
+BDFetch
 BLEXBot
-BackStreet
+BackDoorBot
 Backlink-Ceck
-BacklinkCrawler
-Badass
-Bandit
-Barkrowler
-Black Hole
-Blow
-BotALot
-Bullseye
-BunnySlippers
-BuzzSumo
+BatchFTP
+BetaBot
+Bigfoot
+Blackboard
+BlowFish
+Boardreader
+Bolt
+Buddy
+BuiltWith
 CCBot
+CODE87
 CSHttp
-Calculon
-CheeseBot
+Cegbfeieh
 ChinaClaw
-Chlooe
 Claritybot
+Cliqzbot
+Cloud mapping
 Collector
 Copier
-Copyscape
-Craftbot
+Cosmos
 Crawling at Home Project
-Crescent
-Curious
+CrunchBot
+CyotekWebCopy
+DBLBot
+DIIbot
+DSearch
 DTS Agent
+DataCha0s
+DatabaseDriverMysqli
+Demon
 Deusu
 DigitalPebble
-Dirbuster
 Dispatch
-DomainAppender
+DittoSpyder
+DomCopBot
+DomainCrawler
 DomainStatsBot
-Domains Project
 Dotbot
-Drip
-ECCP/1.0
-EasyDL
-Ebingbong
-EirGrabber
-Evil
-Express WebPictures
-Extractor
-ExtractorPro
-FlashGet
+Download Wonder
+EMail Siphon
+Ecxi
+Exabot
+Extreme Picture Finder
+FHscan
+FemtosearchBot
 Flunky
-Foobot
 Freeuploader
 FyberSpider
-G-i-g-a-b-o-t
-Genieo
+GT::WWW
+GalaxyBot
 GermCrawler
-GetRight
-GetWeb
-Getintent
-GoZilla
-Gotit
-Grafula
-GrapeFX
-GrapeshotCrawler
-HMView
-HTTP::Lite
+Gigabot
+Go-Ahead-Got-It
+GrabNet
+Grabber
+HTMLparser
+HTTrack
 HaosouSpider
 Havij
-Heritrix
 Hloader
+HybridBot
 IDBTE4M
 IDBot
-IRLbot
+Iblog
 IlseBot
 Image Sucker
-IndeedBot
-InterGET
+Intelliseek
 Internet Ninja
-InternetSeer
-Iria
-JOC Web Spider
-Jbrofuzz
-JennyBot
+Iskanie
+JamesBOT
 JetCar
+JikeSpider
 Jorgee
-Kozmosbot
-LWP::Simple
-Lanshanbot
+Jyxobot
+Kenjin Spider
+Keyword Density
+LNSpiderguy
 Larbin
 Leap
 LeechGet
-LibWeb
-Libwhisker
-LinqiaMetadataDownloaderBot
-LinqiaRSSBot
-Litemage_walker
+LieBaoFast
+LinkextractorPro
+LinkpadBot
+LinksManager
+LinqiaScrapeBot
+Lipperhey
+MIDown tool
 MIIxpc
-MJ12bot
+MSFrontPage
 MSIECrawler
-MTRobot
-Mag-Net
-Majestic-SEO
+Magnet
 MarkWatch
-Mb2345Browser
-Metauri
+Mata Hari
+MauiBot
+MeanPath Bot
+MegaIndex.ru
 MicroMessenger
-Microsoft Data Access
-Moblie Safari
-Mojeek
-Mozlila
-Mr.4x3
+Microsoft URL Control
+Mister PiX
+Mojolicious
 Musobot
-Needle
-Nessus
-Net Vampire
-Nettrack
-Niki-bot
-Nuclei
+NICErsPRO
+NPbot
+Name Intelligence
+NearSite
+NetAnts
+NetMechanic
+NetSpider
+NetZIP
+Netcraft
+NimbleCrawler
+Nimbostratus
+Ninja
+Octopus
 Offline Explorer
-OnCrawl
+Offline Navigator
 Openvas
 OrangeBot
 OutclicksBot
-OutfoxBot
-PECL::HTTP
+PHPCrawl
+Page Analyzer
+PageThing.com
 Papa Foto
-PeoplePal
+Pavuk
 Petalbot
-Picsearch
-Piepmatz
-Pixray
-Probethenet
+Pi-Monster
+PleaseCrawl
+ProWebWalker
 Psbot
-PyCurl
-RSSingBot
-RankActive
-RankFlex
-RankurBot
+Pu_iN
+PxBroker
+QueryN Metasearch
+RankingBot2
 Re-re
 RealDownload
+Reaper
+RebelMouse
+RedesScrapy
 RepoMonkey
-RocketCrawler
 Rogerbot
 SBIder
-SEOkicks
-SEOkicks-Robot
 SEOstats
-SISTRIX
-SalesIntelligent
+SMTBot
 Scanbot
-Scrapy
-SemrushBot
-SentiBot
-Seomoz
-Siphon
-SiteExplorer
-SiteLockSpider
+Screaming
+ScreenerBot
+ScrepyBot
+SearchmetricsBot
+Semrush
+SiteCheckerBotCrawler
 SiteSnagger
-SiteSucker
-Siteimprove
-SmartDownload
-Snake
 Sociscraper
 Sogou web spider
-Spbot
-Spinn3r
-Stripper
+SpaceBison
+Spammen
+Sqlworm
 SuperBot
-Szukacz
-Teleport
-TeleportPro
+Surfbot
+Suzuran
+T0PHackTeam
 Telesphoreo
-The Intraformant
 TightTwatBot
 Titan
-Toweyabot
-Tracemyfile
-Trendiction
-True_Robot
-URLy Warning
+Toata
+Turingos
+Turnitin
+TurnitinBot
 UnisterBot
 VB Project
-VeriCiteCrawler
+VCI
+Vacuum
+Vagabondo
 Virusdie
-VoidEYE
+Voil
 Voltron
-WASALive-Bot
-WPScan
-WWW::Mechanize
-Wallpapers
-WeSEE
+WEBDAV
+WWWOFFLE
+WallpapersHD
+Web Collage
+Web Fetch
+Web Fuck
 Web Sucker
-WebBandit
 WebCollage
-WebEnhancer
-WebFuck
-WebGo IS
+WebCopier
 WebImageCollector
-WebPix
+WebLeacher
 WebReaper
 WebStripper
-WebSucker
 WebWhacker
 Webalta
-Webshag
 Website Quester
-Whack
+WebsiteQuester
 Whacker
+Whatweb
 Widow
+WinHTTrack
 WiseGuys Robot
-Wotbox
-Xaldon WebSpider
+Wprecon
+Xaldon_WebSpider
 YoudaoBot
-Zitebot
+Zauba
+ZmEu
+ZoomBot
+ZoominfoBot
 ZumBot
-ZyBorg
+archive.org_bot
 arquivo-web-crawler
-coccocbot-web
-crawl.sogou.com
-dataforseo.com
+arquivo.pt
+autoemailspider
+cah.io.community
+clark-crawler
+cognitiveseo
+com.plumanalytics
+crawler.feedback
 domainsproject.org
-eCatch
-facebookscraper
+gopher
 heritrix
-internetVista monitor
-lwp-trivial
+ips-agent
+magpie-crawler
 mediawords
-netEstate NE Crawler
 oBot
 page scorer
-plumanalytics
-probe-image-size
-satoristudio.net
-seoscanners
-serpstatbot
+polaris version
+s1z.ru
+scan.lol
+seobility
+sexsearcher
 sitechecker.pro
+sysscan
 tAkeOut
+trendiction.de
 ubermetrics-technologies.com
-webpros.com
-zauba.io
+voyagerx.com
+webmeup-crawler
+x22Mozilla