V4.2022.01.2830 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-80legs
+360Spider
+404checker
+404enemy
+AIBOT
 ALittle Client
 ASPSeek
 Abonti
+Aboundexbot
 Acunetix
 AfD-Verbotsverfahren
 AiHitBot
+Aipbot
+Alexibot
+AllSubmitter
 Anarchy99
-BDCbot
+Anthill
+Apexoo
 BDFetch
-BackStreet
-Backlink-Ceck
-Bandit
+BLEXBot
+BacklinkCrawler
 Barkrowler
+BatchFTP
 Battleztar Bazinga
 Bigfoot
-Bitacle
-Black Hole
+Blackboard
+Blow
 BlowFish
-Boardreader
-Bolt
 Brandwatch
-Buck
-BuiltBotTough
 BuiltWith
+BunnySlippers
+BuzzSumo
 CATExplorador
+CCBot
+CazoodleBot
 CheTeam
-ChinaClaw
 Chlooe
-Cliqzbot
+Claritybot
+Cloud mapping
 Cocolyzebot
-Cogentbot
 Copier
-Copyscape
-Craftbot
+CopyRightCheck
+Crawling at Home Project
+Crescent
 CrunchBot
-DTS Agent
-Deusu
-Devil
-DigitalPebble
+Curious
+Custo
+CyotekWebCopy
+DBLBot
+DIIbot
+DSearch
+Demon
+Digincore
 Dirbuster
-Discoverybot
+DittoSpyder
+DomCopBot
 DomainAppender
-Dotbot
-Download Wonder
+DomainSigmaCrawler
 EMail Siphon
-EMail Wolf
-EasyDL
-Ebingbong
-EroCrawler
-ExtLinksBot
-FHscan
-FlashGet
-Foobot
+EirGrabber
+Express WebPictures
+Extractor
+ExtractorPro
+Extreme Picture Finder
+FDM
+Fimap
+Firefox/7.0
 FrontPage
+Fuzz
 FyberSpider
 G-i-g-a-b-o-t
-GetRight
-Go-Ahead-Got-It
-GrapeFX
-HTMLparser
-HaosouSpider
+GT::WWW
+GetWeb
+Getintent
+Go!Zilla
+Gotit
+Grabber
+Grafula
+GrapeshotCrawler
+HMView
+Harvest
+Heritrix
+Humanlinks
+HybridBot
 IRLbot
+Iblog
 IndeedBot
 InfoNaviRobot
-InterGET
-JOC Web Spider
-JennyBot
-JetCar
-Jetty
+Intelliseek
+InternetSeer
+Iria
+IstellaBot
+JamesBOT
+Joomla
 Jorgee
-JustView
-Kenjin Spider
+Keyword Density
 Kinza
+Kozmosbot
 LNSpiderguy
-LWP::Simple
-Lanshanbot
-LibWeb
+Larbin
+Lftp
+Libwhisker
+LieBaoFast
 Lightspeedsystems
-Likse
+LinkScan
 LinkWalker
-Linkbot
-LinkpadBot
-LinqiaScrapeBot
-Lmspider
-MFC_Tear_Sample
-MQQBrowser
-Mag-Net
+LinkextractorPro
+LinqiaRSSBot
+Lipperhey
+Lipperhey Spider
+MIDown tool
+MIIxpc
+MSFrontPage
+MSIECrawler
+MTRobot
 Mail.RU_Bot
 Majestic-SEO
-Majestic12
-Masscan
-Mata Hari
-Mb2345Browser
+MarkWatch
+MauiBot
 MeanPath Bot
-Meanpathbot
 Mediatoolkitbot
-Microsoft Data Access
-Minefield
+Metauri
+MicroMessenger
 Mister PiX
-Moblie Safari
 Mojolicious
-Mozlila
-Mr.4x3
-Name Intelligence
-NearSite
-Nessus
-NetLyzer
-NetZIP
+Musobot
+NetSpider
 Nettrack
-Netvibes
 NextGenSearchBot
-Nibbler
-Niki-bot
 NimbleCrawler
 Ninja
-Nutch
+Offline Explorer
 Offline Navigator
-OpenVAS
-Openfind
 Openvas
-OrangeBot
 PECL::HTTP
-PHPCrawl
-POE-Component-Client-HTTP
-PageScorer
-Panscient
+Page Analyzer
+PageAnalyzer
+Pandalytics
+PeoplePal
 PictureFinder
-PleaseCrawl
-ProWebWalker
-Psbot
-Pump
-PyCurl
-Quick-Crawler
+Pu_iN
+RSSingBot
 RankActive
 RankActiveLinkBot
+RankFlex
+RankingBot2
+RankurBot
 Re-re
-ReGet
+RealDownload
 Reaper
 RebelMouse
 Recorder
 RepoMonkey
 Ripper
-RocketCrawler
-Rogerbot
+SBIder
 SEOkicks
 SEOkicks-Robot
 SEOstats
 SISTRIX
 SMTBot
-SalesIntelligent
 ScanAlert
-Scrapy
-ScreenerBot
-Searchestate
-SemanticJuice
-SentiBot
+ScrepyBot
+SeobilityBot
 Seomoz
-Siphon
-SiteLockSpider
-SiteSnagger
-Sitebeam
-SmartDownload
+SiteExplorer
+Snoopy
 SocialRankIOBot
-Sosospider
-Spanner
-Spbot
+Sociscraper
+SpaceBison
+Spammen
 Spinn3r
-Stripper
-Sucker
-SuperBot
-Surfbot
-SurveyBot
-Swiftbot
+Sqlmap
+SuperHTTP
 Szukacz
+T8Abot
+TeleportPro
+Telesoft
 Telesphoreo
-Thumbor
-Titan
-Trendiction
-True_Robot
-Turingos
-TwengaBot
+The Intraformant
+TheNomad
+TightTwatBot
+Toata
+Tracemyfile
+Trendictionbot
 Twice
-UnisterBot
-Upflow
 VB Project
-VelenPublicWebCrawler
 VeriCiteCrawler
-Voltron
-WASALive-Bot
+VidibleScraper
+VoidEYE
+Voil
 WBSearchBot
 WISENutbot
 WPScan
-WWW-Mechanize
 WWW::Mechanize
-WWWOFFLE
 Wallpapers
-Wallpapers/3.0
-WallpapersHD
-WeSEE
+Web Auto
+Web Enhancer
+Web Fetch
 Web Fuck
-WebAuto
+WebBandit
 WebEnhancer
 WebFuck
+WebGo IS
 WebImageCollector
+WebPix
+WebReaper
 WebSauger
-WebStripper
 WebWhacker
-WebZIP
-Webalta
+WebmasterWorldForumBot
 Webshag
-WebsiteQuester
-Whacker
-Woobot
-Wprecon
-Xaldon WebSpider
+Website Quester
+Webster
+Whatweb
+Who.is Bot
+WinHTTrack
+WiseGuys Robot
+Wotbox
 Xaldon_WebSpider
 Xenu
+YoudaoBot
+Zade
+Zauba
+Zermelo
 Zitebot
-ZoomBot
+ZumBot
 ZyBorg
+adscanner
+arquivo.pt
 autoemailspider
 backlink-check
 cah.io.community
-check1.exe
-clark-crawler
-coccocbot-web
-com.plumanalytics
+crawl.sogou.com
+crawler.feedback
 crawler4j
-eCatch
-facebookscraper
-gopher
-heritrix
-isitwp.com
-magpie-crawler
-page scorer
-ripz
-s1z.ru
-scan.lol
-seobility
+demandbase-bot
+evc-batch
+instabid
+meanpathbot
 seocompany.store
 seoscanners
-serpstatbot
+sitechecker.pro
 siteripz
 sp_auditbot
-tAkeOut
+spyfu
 trendiction.com
+voyagerx.com
 webgains-bot
 webmeup-crawler
-xpymep1.exe
+webpros.com
+webprosbot
+zgrab