V4.2022.03.3003 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
+360Spider
 80legs
-AIBOT
-ALittle Client
+ASPSeek
+Abonti
 Aboundexbot
-Acunetix
 AfD-Verbotsverfahren
+AhrefsBot
+AllSubmitter
+Alligator
 AlphaBot
-Asterias
+Anarchy99
 Attach
+AwarioRssBot
 AwarioSmartBot
-BBBike
+BDCbot
 BDFetch
-BackStreet
-BackWeb
+BackDoorBot
 BacklinkCrawler
+Badass
 Battleztar Bazinga
+BetaBot
 Bigfoot
 BlackWidow
 Boardreader
+Bolt
 BotALot
-Buck
-BuiltBotTough
+Brandprotect
+Buddy
+BuiltWith
 BunnySlippers
-CherryPicker
+Calculon
+CazoodleBot
+ChinaClaw
+Chlooe
 Citoid
-Claritybot
-Cocolyzebot
-Cogentbot
-Copier
+Cliqzbot
+Cloud mapping
 Craftbot
 Crawling at Home Project
-CrazyWebCrawler
 Crescent
-CyotekWebCopy
-DBLBot
+Curious
+Custo
+DIIbot
 Demon
-Devil
 DigitalPebble
 Disco
-DittoSpyder
-DnyzBot
-DomainAppender
+Discobot
+DnBCrawler-Analytics
+DomCopBot
+DomainSigmaCrawler
+Dragonfly
+Drip
 EMail Siphon
-Ecxi
-Exabot
-Express WebPictures
-Ezooms
-FyberSpider
+EMail Wolf
+EroCrawler
+Evil
+Extractor
+ExtractorPro
+Extreme Picture Finder
+Fimap
+Firefox/7.0
+Flunky
 Fyrebot
-GT::WWW
+G-i-g-a-b-o-t
 GalaxyBot
-GetRight
-GetWeb
-GoZilla
-GrabNet
-GrapeshotCrawler
+Go-Ahead-Got-It
+Grafula
 GridBot
-HEADMasterSEO
+HMView
 HTMLparser
-HTTP::Lite
 HTTrack
+Havij
 Hloader
 Humanlinks
-IDBTE4M
-IndeedBot
-InfoNaviRobot
+IDBot
+IRLbot
+IlseBot
+Image Sucker
 Intelliseek
-Iria
+InterGET
 Iskanie
-JOC Web Spider
+Jbrofuzz
+JetCar
 JikeSpider
 JustView
 Kenjin Spider
-Keybot Translation-Search-Machine
-Keyword Density
-LNSpiderguy
-LWP::Simple
-Larbin
-LeechFTP
-LexiBot
-LibWeb
+Libwhisker
 Lightspeedsystems
-Likse
-Linkdexbot
-LinkpadBot
+LinkScan
+Linkbot
 LinksManager
-LinqiaRSSBot
 LinqiaScrapeBot
-Lipperhey Spider
-Ltx71
-MQQBrowser
-MTRobot
+Lipperhey
+Lmspider
+MIDown tool
+MJ12bot
+MSFrontPage
 Majestic SEO
-Masscan
-MauiBot
-Mb2345Browser
-Meanpathbot
+Majestic-SEO
+Majestic12
+MarkMonitor
+MeanPath Bot
+Mediatoolkitbot
+Metauri
+MicroMessenger
+Microsoft URL Control
 Minefield
 Mister PiX
+Moblie Safari
 Mojeek
-Mojolicious
-MolokaiBot
+Morfeus Fucking Scanner
+Mr.4x3
 Musobot
 NICErsPRO
 NPbot
-Nameprotect
-NearSite
-Nessus
+NetAnts
 NetLyzer
 NetSpider
-Netvibes
+Nettrack
 NextGenSearchBot
 Nibbler
 Niki-bot
-Ninja
 Nuclei
-OnCrawl
-OpenLinkProfiler
-OrangeBot
-POE-Component-Client-HTTP
+Octopus
+Offline Explorer
+OutclicksBot
 PageAnalyzer
+PageScorer
 PageThing.com
 Pandalytics
 Pavuk
 PeoplePal
 Pi-Monster
 Picsearch
+Piepmatz
 Pimonster
 PleaseCrawl
 Pockey
 ProPowerBot
 ProWebWalker
-Probethenet
-Quick-Crawler
+Psbot
+Pu_iN
+PxBroker
+QueryN Metasearch
 RankingBot
-RankurBot
+ReGet
 Reaper
 RebelMouse
-Recorder
-RocketCrawler
+RedesScrapy
+RepoMonkey
+Ripper
+Rogerbot
 SBIder
-SEOlyticsCrawler
-SEOprofiler
-SMTBot
-SalesIntelligent
-ScanAlert
-ScreenerBot
-Searchestate
-SearchmetricsBot
+SEOkicks-Robot
+SISTRIX
+Scrapy
+ScrepyBot
+Seekport
 Semrush
-Site Sucker
-SiteSnagger
-SiteSucker
-SlySearch
-Snapbot
-Snoopy
-SocialRankIOBot
-Sociscraper
-Sogou web spider
-Spammen
-Spinn3r
-SputnikBot
-Steeler
+SeobilityBot
+Shodan
+Siteimprove
+Spanner
+Sucuri
 SuperHTTP
-Suzuran
+Surfbot
+SurveyBot
 Swiftbot
+Szukacz
+T0PHackTeam
 T8Abot
-TeleportPro
 Telesoft
 Telesphoreo
-Telesphorep
-The Intraformant
+Thumbor
+TightTwatBot
 Titan
+Toata
 Toweyabot
 Tracemyfile
 Trendiction
-True_Robot
-Turnitin
+Turingos
 TurnitinBot
+Typhoeus
 URLy Warning
 URLy.Warning
-VeriCiteCrawler
+Upflow
+VB Project
+VCI
+VidibleScraper
 Virusdie
-VoidEYE
 Voil
-Voltron
 WBSearchBot
-WEBDAV
-WWW-Collector-E
-Wallpapers
-Wallpapers/3.0
+WWW-Mechanize
+WWWOFFLE
+WallpapersHD
+Web Collage
+Web Fetch
+Web Fuck
 Web Sauger
-Web Sucker
-WebAuto
-WebCollage
-WebCopier
-WebEnhancer
+WebFetch
 WebGo IS
 WebImageCollector
-WebReaper
+WebLeacher
+WebStripper
 WebWhacker
-WebmasterWorldForumBot
+WebZIP
 Webshag
-WebsiteExtractor
-Whacker
-Whatweb
+WebsiteQuester
+Webster
 Widow
-WiseGuys Robot
+WinHTTrack
 Woobot
+Wotbox
 Wprecon
-Zade
+Xenu
+Zermelo
+Zitebot
 ZmEu
-ZoomBot
+ZoominfoBot
+ZumBot
+ZyBorg
 adscanner
 arquivo-web-crawler
-autoemailspider
-backlink-check
+arquivo.pt
 check1.exe
-cognitiveseo
+coccocbot
 crawler.feedback
 crawler4j
 dataforseo.com
-domainsproject.org
-eCatch
 evc-batch
 facebookscraper
-gopher
-instabid
-lwp-request
-lwp-trivial
+heritrix
+iubenda-radar
+magpie-crawler
 muhstik-scan
+netEstate NE Crawler
 oBot
 plumanalytics
-probe-image-size
+polaris version
 s1z.ru
 satoristudio.net
 scalaj-http
-seobility
-seocompany.store
-sexsearcher
 sitechecker.pro
-sogouspider
+sysscan
+trendiction.com
 voyagerx.com
-webgains-bot
 webmeup-crawler
-webpros.com
-x09Mozilla
-x22Mozilla
 xpymep1.exe
+zgrab