V4.2022.02.2940 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
+360Spider
 404enemy
 AIBOT
-Aboundex
+ALittle Client
 AfD-Verbotsverfahren
-AhrefsBot
-AiHitBot
-Alexibot
+Aipbot
+AllSubmitter
+Alligator
 Anarchie
-Apexoo
-Attach
-BBBike
+Anarchy
+Ankit
+Aspiegel
 BDFetch
 BLEXBot
-BackStreet
-Backlink-Ceck
-Badass
+BackWeb
 Barkrowler
+BatchFTP
 Battleztar Bazinga
-BetaBot
 Bitacle
-Black Hole
 BlackWidow
 Blackboard
-Blow
-Bolt
-Brandwatch
-Buddy
+Boardreader
+BotALot
 BuzzSumo
 CATExplorador
-CODE87
-CazoodleBot
-Cegbfeieh
+CCBot
+Calculon
+CensysInspect
 CherryPicker
 ChinaClaw
-Citoid
-Claritybot
-Cocolyzebot
+Cloud mapping
 Cogentbot
 Collector
-CopyRightCheck
+Copier
+Copyscape
 Crawling at Home Project
-CrazyWebCrawler
 Crescent
-DSearch
+CrunchBot
+Curious
+CyotekWebCopy
 DataCha0s
+Demon
 Deusu
+Digincore
 Discoverybot
-Dispatch
 DnBCrawler-Analytics
-DomainCrawler
+DomCopBot
 DomainStatsBot
-Download Wonder
+Dotbot
 Dragonfly
-Drip
-EMail Siphon
-Evil
-Exabot
-Express WebPictures
+ECCP/1.0
+EMail Wolf
+EirGrabber
 ExtLinksBot
 Extractor
+ExtractorPro
+EyeNetIE
 Ezooms
-FHscan
-Firefox/7.0
+FDM
+Fimap
+Flunky
 Freeuploader
-FrontPage
-Fuzz
-FyberSpider
-Genieo
+GermCrawler
 GetRight
-Getintent
-Go!Zilla
+GetWeb
+Gigabot
 Go-Ahead-Got-It
 GoZilla
-Gotit
-Grabber
-GrapeFX
-HTTP::Lite
+Grafula
+GridBot
 Haansoft
-Heritrix
-IDBTE4M
+Hloader
 IDBot
 IRLbot
+Iblog
 Id-search
-Image Fetch
-IndeedBot
-Indy Library
-InterGET
+InfoNaviRobot
+InfoTekies
 InternetSeer
-Iria
+IstellaBot
 Jbrofuzz
 JennyBot
 JetCar
-Jetty
-JikeSpider
-Joomla
-Jorgee
-Jyxobot
-Kinza
+JustView
 LWP::Simple
-Lanshanbot
-Larbin
-LeechFTP
-LeechGet
 LexiBot
 LibWeb
-Libwhisker
-Linkdexbot
-LinqiaMetadataDownloaderBot
-LinqiaScrapeBot
+LieBaoFast
+Likse
+LinksManager
+LinqiaRSSBot
 Lipperhey
 Lipperhey Spider
+Lmspider
 MIDown tool
 MIIxpc
 MJ12bot
+MSFrontPage
 Magnet
-Majestic SEO
-Majestic-SEO
 MarkMonitor
-Masscan
-MauiBot
-MeanPath Bot
-MicroMessenger
-Mojolicious
-MolokaiBot
+Mass Downloader
+Mediatoolkitbot
+MegaIndex.ru
+Microsoft URL Control
+Moblie Safari
+Mojeek
 Mr.4x3
+Msrabot
 NPbot
 Name Intelligence
-Nameprotect
+Navroad
 NearSite
-Nessus
-NetMechanic
-Netcraft
+Needle
+Net Vampire
+NetSpider
 Nettrack
-Nibbler
-Niki-bot
 NimbleCrawler
-Nmap
+Nimbostratus
 Not
-Offline Navigator
+Offline Explorer
 OpenLinkProfiler
 OpenVAS
 Openfind
 Openvas
-OrangeBot
-PHPCrawl
+OutclicksBot
+OutfoxBot
 PageGrabber
-PageScorer
-PageThing.com
-Pandalytics
-Papa Foto
-Picscout
 PictureFinder
-Pimonster
-Pockey
+PleaseCrawl
 ProWebWalker
-Pu_iN
-PyCurl
-Quick-Crawler
+Psbot
+Pump
+RankActive
+RankActiveLinkBot
 RankFlex
-RankurBot
-ReGet
-SBIder
+RankingBot
+RankingBot2
+RealDownload
+Reaper
+RedesScrapy
+RepoMonkey
+Ripper
 SEOkicks
-SEOkicks-Robot
-SEOlyticsCrawler
-SISTRIX
-ScreenerBot
+Screaming
 ScrepyBot
 SearchmetricsBot
-SemanticJuice
-SemrushBot
-Siphon
+Seekport
+SentiBot
+SeobilityBot
+Seomoz
 Site Sucker
-SiteSucker
-Sitebeam
+SiteExplorer
+SiteLockSpider
 SlySearch
-Snapbot
-SocialRankIOBot
-Sociscraper
-SpankBot
-Spinn3r
+Snake
+Sosospider
+SpaceBison
+Spanner
+SputnikBot
+Sqlmap
+Sqlworm
+Steeler
 Sucker
+SuperBot
+SuperHTTP
 Surfbot
 SurveyBot
 Suzuran
-Swiftbot
 Szukacz
+T8Abot
+TeleportPro
+Telesoft
+Telesphoreo
+Telesphorep
+The Intraformant
 TheNomad
-TightTwatBot
 Toata
-Toweyabot
-Turingos
-Turnitin
+TwengaBot
 Typhoeus
-UnisterBot
+URLy Warning
+URLy.Warning
+VB Project
+VCI
+VelenPublicWebCrawler
 VeriCiteCrawler
+VidibleScraper
 Virusdie
+VoidEYE
+Voltron
 WASALive-Bot
-WBSearchBot
-WEBDAV
-WPScan
+WWW-Collector-E
+WWW-Mechanize
 WWW::Mechanize
 WWWOFFLE
+Wallpapers
+Wallpapers/3.0
 WallpapersHD
 Web Auto
-Web Collage
-Web Fetch
+Web Fuck
+Web Pix
+WebCopier
+WebEnhancer
 WebFetch
-WebLeacher
-WebPix
 WebStripper
-WebWhacker
-Webalta
+WebSucker
+WebZIP
 Webshag
-WebsiteExtractor
-Whacker
+Webster
 Whatweb
-Who.is Bot
-WinHTTrack
-Xaldon_WebSpider
+WiseGuys Robot
+Woobot
+Wprecon
 Xenu
-YoudaoBot
 Zauba
+Zitebot
+ZmEu
+ZoominfoBot
 ZyBorg
+adscanner
 arquivo-web-crawler
-backlink-check
-cah.io.community
-cognitiveseo
-domainsproject.org
-eCatch
-heritrix
-instabid
+arquivo.pt
+clark-crawler
+coccocbot
+crawl.sogou.com
+crawler4j
+dataforseo.com
+demandbase-bot
 isitwp.com
-mediawords
+iubenda-radar
+lwp-trivial
 pcBrowser
-polaris version
+plumanalytics
 probe-image-size
+ripz
 s1z.ru
 satoristudio.net
-scalaj-http
-seobility
-serpstatbot
+scan.lol
+seostar
+sexsearcher
+sitechecker.pro
 siteripz
 sogouspider
-trendiction.com
+sp_auditbot
+spyfu
+trendiction.de
 ubermetrics-technologies.com
-x09Mozilla
-x22Mozilla
-xpymep1.exe
+webgains-bot
+webprosbot
+zgrab