V4.2021.08.2520 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404checker
-ADmantX
-AIBOT
-ALittle Client
-Abonti
+01h4x.com
+360Spider
 Aboundex
 Aboundexbot
+Acunetix
+AfD-Verbotsverfahren
+AhrefsBot
+AiHitBot
 Aipbot
+Alexibot
+AllSubmitter
 Alligator
-Anarchy
-Anthill
-Apexoo
+Anarchie
 Asterias
-Attach
-AwarioRssBot
 BDCbot
 BDFetch
 BackStreet
 BackWeb
-Backlink-Ceck
-Badass
 Barkrowler
-BatchFTP
 BetaBot
 Bigfoot
-Black Hole
-BlackWidow
-BlowFish
-Bolt
-BotALot
 Buck
+Buddy
+BuiltBotTough
+BuiltWith
 Bullseye
 BuzzSumo
-CATExplorador
-Calculon
+CCBot
+CSHttp
+CazoodleBot
 CheeseBot
 CherryPicker
 ChinaClaw
-Chlooe
+Cliqzbot
 Cloud mapping
-Cocolyzebot
 Collector
 CopyRightCheck
-Craftbot
-CrunchBot
-DIIbot
-Demon
-Devil
+Cosmos
+CyotekWebCopy
+DTS Agent
+DataCha0s
+Digincore
 Disco
-Discoverybot
-Dispatch
-DomainAppender
+DnyzBot
+DomCopBot
+DomainCrawler
+DomainStatsBot
 Domains Project
-Dotbot
 Download Wonder
 Dragonfly
-Drip
 ECCP/1.0
-EMail Wolf
-Evil
-Exabot
-ExtLinksBot
+Ebingbong
+EroCrawler
+Express WebPictures
+Extreme Picture Finder
 EyeNetIE
 Ezooms
 FDM
-FemtosearchBot
 FlashGet
-Freeuploader
-FyberSpider
+Foobot
+FrontPage
+G-i-g-a-b-o-t
+GT::WWW
 GalaxyBot
 Genieo
 GermCrawler
-Gigabot
+Getintent
+Go!Zilla
 Go-Ahead-Got-It
-GoZilla
+Grabber
+Grafula
 GrapeFX
-GridBot
+GrapeshotCrawler
+HEADMasterSEO
 HMView
-HTMLparser
-HTTrack
-Haansoft
-Heritrix
+HTTP::Lite
+Harvest
 Hloader
-HybridBot
-Iblog
-IlseBot
-InfoNaviRobot
-Intelliseek
-InternetSeer
-Iria
+IDBot
+IndeedBot
+Internet Ninja
 JOC Web Spider
-JennyBot
 JetCar
+Jetty
 JikeSpider
-Joomla
-Jorgee
-JustView
+Kenjin Spider
 Keyword Density
+Kozmosbot
 LNSpiderguy
-LWP::Simple
 Lanshanbot
-Lightspeedsystems
-LinkWalker
-Linkdexbot
+LeechFTP
+LeechGet
+LexiBot
+LibWeb
+LinkScan
 LinkextractorPro
 LinkpadBot
 LinksManager
 LinqiaMetadataDownloaderBot
-LinqiaScrapeBot
-Lipperhey Spider
-Litemage_walker
-Ltx71
+LinqiaRSSBot
+Lipperhey
 MIIxpc
+MQQBrowser
+MSFrontPage
 MSIECrawler
-Majestic SEO
-MarkMonitor
-MauiBot
-MeanPath Bot
-Meanpathbot
-MegaIndex.ru
+Mail.RU_Bot
+Majestic-SEO
+Majestic12
+Mata Hari
+Mediatoolkitbot
 Metauri
 MicroMessenger
 Microsoft Data Access
-Mister PiX
-Moblie Safari
+Mojeek
+Mojolicious
 Morfeus Fucking Scanner
-Mozlila
-NPbot
+Musobot
+NICErsPRO
+Name Intelligence
+NearSite
 Needle
-NetLyzer
-NetMechanic
-NetZIP
-Netvibes
-NextGenSearchBot
-Nibbler
-Nutch
+Nessus
+NetAnts
+Nettrack
+Nimbostratus
+Nmap
 Octopus
-OpenLinkProfiler
-OpenVAS
-OrangeBot
-OrangeSpider
-OutfoxBot
+Offline Navigator
+Openfind
+Openvas
 PHPCrawl
-PageAnalyzer
+POE-Component-Client-HTTP
 PageGrabber
-PageThing.com
+Pandalytics
+Panscient
 Papa Foto
-Pavuk
-Petalbot
-Picscout
+Pi-Monster
+Picsearch
 Pixray
+PleaseCrawl
+Pockey
+ProWebWalker
 Psbot
-Pu_iN
-RSSingBot
-RankActiveLinkBot
-RankingBot
-RankingBot2
-Rankivabot
+PxBroker
+QueryN Metasearch
 RankurBot
-Reaper
-RebelMouse
-RedesScrapy
-RepoMonkey
+Re-re
+ReGet
+Recorder
+Ripper
+RocketCrawler
 Rogerbot
 SBIder
-SEOkicks
 SEOkicks-Robot
-SISTRIX
-SMTBot
-ScanAlert
-Scrapy
 Screaming
-SearchmetricsBot
-Semrush
-SeobilityBot
+ScreenerBot
+ScrepyBot
+Searchestate
+SemrushBot
+Site Sucker
 SiteExplorer
-SiteSucker
-Siteimprove
-Sitevigil
+SiteSnagger
 SlySearch
 Snake
 Snoopy
+SocialRankIOBot
 Sociscraper
-Sottopop
+Spammen
 Spanner
 SputnikBot
-Sucker
-Sucuri
-SuperHTTP
-Swiftbot
-T0PHackTeam
+Sqlworm
+Sqworm
+Steeler
+Suzuran
+T8Abot
+TeleportPro
+Telesoft
 Telesphoreo
+Telesphorep
 The Intraformant
 TheNomad
-Thumbor
-Trendiction
+TightTwatBot
+Toweyabot
+Tracemyfile
+TurnitinBot
+TwengaBot
+Twice
 Typhoeus
-Upflow
-VCI
+URLy Warning
+V-BOT
+Vacuum
+VeriCiteCrawler
 VidibleScraper
-Virusdie
 VoidEYE
-Voil
-WEBDAV
-WWW::Mechanize
+WBSearchBot
+WISENutbot
+WPScan
+WWW-Collector-E
 Wallpapers/3.0
-WallpapersHD
+Web Auto
 Web Enhancer
 Web Pix
-Web Sucker
-WebCollage
-WebCopier
-WebFuck
-WebImageCollector
+WebBandit
+WebEnhancer
 WebPix
-WebStripper
+WebReaper
 WebSucker
+WebmasterWorldForumBot
 Website Quester
-WebsiteExtractor
+WebsiteQuester
 Who.is Bot
 WiseGuys Robot
-Wonderbot
 Wotbox
 Wprecon
-Xaldon_WebSpider
-YoudaoBot
+Xaldon WebSpider
+Zauba
+Zeus
+Zitebot
 ZmEu
-ZoominfoBot
-ZyBorg
-autoemailspider
-backlink-check
-cah.io.community
-cognitiveseo
+ZoomBot
+arquivo-web-crawler
+arquivo.pt
+coccocbot-web
+com.plumanalytics
 crawl.sogou.com
-crawler.feedback
-domainsproject.org
+crawler4j
+demandbase-bot
+evc-batch
+ips-agent
+lwp-trivial
 magpie-crawler
 mediawords
-muhstik-scan
-oBot
+plumanalytics
 probe-image-size
+s1z.ru
+satoristudio.net
 scalaj-http
-sitechecker.pro
-sogouspider
+scan.lol
+seobility
+seostar
+serpstatbot
+sexsearcher
 sp_auditbot
 spyfu
 sysscan
-trendiction.de
-x22Mozilla
+tAkeOut
+voyagerx.com
+webmeup-crawler
+zauba.io
 zgrab