V4.2022.04.3044 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-360Spider
 404checker
-AIBOT
-Aboundex
-AfD-Verbotsverfahren
+80legs
+ASPSeek
 AiHitBot
-Alexibot
 AllSubmitter
-Alligator
-AlphaBot
+Anarchy
+Anarchy99
 Ankit
-Apexoo
+Asterias
 Attach
-BDCbot
+AwarioRssBot
 BLEXBot
+BackDoorBot
+BackStreet
+BackWeb
+BacklinkCrawler
+Badass
 Bandit
-Barkrowler
+BatchFTP
 Battleztar Bazinga
 BetaBot
-Bitacle
-Black Hole
-BlackWidow
-BlowFish
-Boardreader
+Bigfoot
+Blackboard
+Blow
+Brandwatch
 Buck
-Bullseye
-BuzzSumo
-CCBot
-CazoodleBot
-Cegbfeieh
+Buddy
+CATExplorador
+CSHttp
+CheTeam
 CheeseBot
-CherryPicker
-Chlooe
+ChinaClaw
 Citoid
 Cliqzbot
 Collector
-Crawling at Home Project
-CrunchBot
-Custo
-DBLBot
+CopyRightCheck
+Copyscape
+CrazyWebCrawler
+Crescent
+Curious
+CyotekWebCopy
+DIIbot
+DSearch
 DTS Agent
 Deusu
-Dirbuster
+Digincore
+DigitalPebble
 Disco
 Discoverybot
-DnBCrawler-Analytics
 DnyzBot
 DomCopBot
 DomainStatsBot
-Domains Project
-Dotbot
 Dragonfly
 EMail Siphon
 EMail Wolf
-Ebingbong
-Evil
-Exabot
-Firefox/7.0
+EasyDL
+Ecxi
+EirGrabber
+Express WebPictures
+ExtLinksBot
+ExtractorPro
+EyeNetIE
+FDM
+FemtosearchBot
+Fimap
 FlashGet
 Foobot
-Freeuploader
 FrontPage
+Fyrebot
+GT::WWW
+GermCrawler
 Go!Zilla
 Go-Ahead-Got-It
+GoZilla
 GrabNet
-Grafula
+GrapeshotCrawler
 GridBot
-HTTrack
-Haansoft
-HaosouSpider
-HybridBot
+HMView
+HTTP::Lite
+Harvest
+Havij
+Heritrix
+Hloader
+HonoluluBot
+Humanlinks
+IDBTE4M
 IDBot
-IRLbot
-Indy Library
+Id-search
+IlseBot
+IndeedBot
 InfoNaviRobot
-Intelliseek
 Internet Ninja
-InternetSeer
+Iria
+Iskanie
 IstellaBot
-JamesBOT
-JennyBot
 JikeSpider
-JustView
-LNSpiderguy
-Leap
+Jyxobot
+Kozmosbot
+Lanshanbot
+Larbin
+LeechFTP
+LeechGet
+Lftp
+Lightspeedsystems
 Likse
-LinkScan
 LinkWalker
 LinkextractorPro
 LinkpadBot
+LinksManager
 LinqiaMetadataDownloaderBot
 LinqiaRSSBot
-Lipperhey
-Lipperhey Spider
+LinqiaScrapeBot
 Lmspider
-Ltx71
-MIDown tool
-MJ12bot
-MSIECrawler
-Magnet
-Mail.RU_Bot
-Majestic SEO
-Majestic-SEO
-Mass Downloader
-Mb2345Browser
+MSFrontPage
+Majestic12
+MarkWatch
+Masscan
+Mata Hari
+MauiBot
+MeanPath Bot
 Meanpathbot
-Mediatoolkitbot
-MegaIndex.ru
 Metauri
 MicroMessenger
-Microsoft Data Access
 Microsoft URL Control
-Minefield
-Mojeek
+Mister PiX
 Mojolicious
+MolokaiBot
 Morfeus Fucking Scanner
-Mozlila
-Mr.4x3
-NPbot
-Name Intelligence
 Navroad
-Needle
+Nessus
 NetLyzer
-NetSpider
-Niki-bot
-Nikto
 NimbleCrawler
-Octopus
+Nmap
+Not
 Offline Explorer
+Offline Navigator
+OnCrawl
 OpenLinkProfiler
 OpenVAS
-OrangeSpider
+Openfind
+OrangeBot
+OutclicksBot
+OutfoxBot
+POE-Component-Client-HTTP
+PageAnalyzer
 PageGrabber
-PageThing.com
-Pavuk
-Pi-Monster
+PageScorer
+Papa Foto
+Petalbot
 Picscout
-PictureFinder
 Piepmatz
-Pimonster
 Pixray
 PleaseCrawl
 Pockey
-Psbot
-Pu_iN
-Pump
+ProPowerBot
+PxBroker
+Quick-Crawler
 RSSingBot
+RankActive
+RankActiveLinkBot
+RankFlex
 RankingBot
-Rankivabot
-RankurBot
-Re-re
-ReGet
-RebelMouse
-RedesScrapy
-RepoMonkey
-SEOkicks
-SEOlyticsCrawler
-SEOstats
+Recorder
+RocketCrawler
+Rogerbot
 SISTRIX
 SMTBot
-Scanbot
+SalesIntelligent
+Scrapy
 ScreenerBot
-SemanticJuice
-Semrush
-SemrushBot
-SeobilityBot
-Siphon
-SiteCheckerBotCrawler
+SentiBot
+Seomoz
+Site Sucker
 SiteLockSpider
-SiteSucker
 Sitebeam
+Siteimprove
 Sitevigil
 SmartDownload
+Snake
 Snapbot
-Snoopy
-SocialRankIOBot
-SpaceBison
-SpankBot
+Sosospider
+Sottopop
+Spammen
+Spbot
+Sqlworm
+Sqworm
 Steeler
 Sucuri
-SuperBot
+SuperHTTP
+Surfbot
 SurveyBot
-Teleport
-Telesoft
-Telesphoreo
-Thumbor
-Titan
-Trendictionbot
-True_Robot
-TwengaBot
-URLy.Warning
+Szukacz
+T0PHackTeam
+TeleportPro
+The Intraformant
+Trendiction
+Turingos
+URLy Warning
 UnisterBot
-V-BOT
-VCI
+Vagabondo
+VeriCiteCrawler
+VidibleScraper
 Voil
-Voltron
 WISENutbot
-WPScan
-Wallpapers/3.0
-Web Fuck
+WWW-Collector-E
+Web Auto
+Web Collage
+Web Enhancer
 Web Sauger
 Web Sucker
 WebCopier
 WebFuck
-WebGo IS
 WebImageCollector
-WebPix
 WebReaper
-WebSucker
-WebZIP
-Webshag
-WebsiteExtractor
+WebWhacker
 WebsiteQuester
-Whacker
-Who.is Bot
-Widow
-WiseGuys Robot
-Wonderbot
-Woobot
-Xaldon_WebSpider
+Webster
+Whack
+Wotbox
+Wprecon
+Xaldon WebSpider
 Xenu
-YoudaoBot
-Zauba
-Zermelo
-Zeus
+Zitebot
 ZmEu
 ZyBorg
-arquivo.pt
-cah.io.community
-coccocbot
-eCatch
-gopher
-ips-agent
-isitwp.com
+adscanner
+backlink-check
+clark-crawler
+crawl.sogou.com
+dataforseo.com
+domainsproject.org
+facebookscraper
+lwp-trivial
 meanpathbot
-plumanalytics
-polaris version
-satoristudio.net
+mediawords
+oBot
+pcBrowser
+ripz
+scalaj-http
+scan.lol
+seobility
 seocompany.store
-seoscanners
-seostar
-serpstatbot
-sitechecker.pro
 siteripz
-spyfu
-sysscan
-webgains-bot
-x09Mozilla
+trendiction.de
+ubermetrics-technologies.com
+voyagerx.com
+webmeup-crawler
+webpros.com
 xpymep1.exe
+zgrab