V4.2022.05.3134 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
+404checker
 404enemy
 80legs
-ADmantX
-ALittle Client
-Abonti
-Aboundex
-Aboundexbot
-Acunetix
-AhrefsBot
+AIBOT
 AiHitBot
 Aipbot
-Anarchie
 Anarchy
-Ankit
-Anthill
+Anarchy99
 Apexoo
-AwarioSmartBot
+Attach
+AwarioRssBot
+BBBike
+BDCbot
+BDFetch
 BLEXBot
-BatchFTP
-Bitacle
-Black Hole
+BackDoorBot
+Badass
+Battleztar Bazinga
+Bigfoot
+Blackboard
 BlowFish
+Boardreader
 Bolt
 BotALot
+Brandwatch
+Buck
 Buddy
-CATExplorador
+BuiltWith
+BunnySlippers
+BuzzSumo
 CSHttp
-CheeseBot
-Chlooe
-Citoid
-Cliqzbot
-Cloud mapping
-Copier
-Copyscape
+Calculon
+CazoodleBot
+CheTeam
+Cocolyzebot
+Collector
+CopyRightCheck
 Craftbot
-Crawling at Home Project
 CrazyWebCrawler
-Custo
+Crescent
+CyotekWebCopy
+DBLBot
 DIIbot
-DSearch
-Demon
+DataCha0s
 Deusu
-Digincore
-Dirbuster
-Discoverybot
+Devil
+DigitalPebble
+Discobot
 Dispatch
-DnBCrawler-Analytics
 DomCopBot
-DomainAppender
-DomainCrawler
-DomainStatsBot
-Dragonfly
 Drip
-EasyDL
-Exabot
-ExtLinksBot
-Extractor
-ExtractorPro
+EroCrawler
 Extreme Picture Finder
-EyeNetIE
-Ezooms
-Flunky
-Genieo
+FDM
+Fimap
+FlashGet
+FyberSpider
+GT::WWW
+GalaxyBot
 GermCrawler
 GetRight
+GetWeb
+Gigabot
 GoZilla
-HEADMasterSEO
-HMView
+Gotit
+Grabber
 HTMLparser
 HTTP::Lite
 Haansoft
-HaosouSpider
+Harvest
+Havij
+Heritrix
+Hloader
+Humanlinks
+HybridBot
+IDBTE4M
+IDBot
+IRLbot
 Id-search
-InterGET
+InfoNaviRobot
+InfoTekies
+Intelliseek
 Internet Ninja
-InternetSeer
-Iria
+JOC Web Spider
 JetCar
-Jetty
-Joomla
 Jorgee
+JustView
+Jyxobot
+Kenjin Spider
+Keybot Translation-Search-Machine
 Keyword Density
-Kozmosbot
-LNSpiderguy
 Lanshanbot
-Larbin
 Leap
-LeechFTP
+LeechGet
 LexiBot
+Libwhisker
 Lightspeedsystems
+Likse
+LinkWalker
+Linkbot
 LinkpadBot
-LinqiaScrapeBot
-Lipperhey
-Lipperhey Spider
+LinksManager
+LinqiaMetadataDownloaderBot
+LinqiaRSSBot
+Litemage_walker
 Lmspider
+Ltx71
 MFC_Tear_Sample
+MIDown tool
+MQQBrowser
 MSFrontPage
-MSIECrawler
-Majestic SEO
+Magnet
+Majestic-SEO
+Majestic12
 MarkWatch
-Mass Downloader
-Mb2345Browser
-MeanPath Bot
+Masscan
 MegaIndex.ru
 Microsoft URL Control
 Minefield
 Mister PiX
-Mojeek
 MolokaiBot
-Msrabot
+Mr.4x3
+NPbot
 Name Intelligence
-Navroad
-Needle
-NetMechanic
-NetZIP
+Nameprotect
+NetSpider
 Nettrack
-Nikto
-NimbleCrawler
-Not
-Nuclei
-Nutch
+Netvibes
+Nibbler
+Nimbostratus
 OnCrawl
-OpenLinkProfiler
-PHPCrawl
+Openvas
+PECL::HTTP
+Page Analyzer
 PageAnalyzer
-PageThing.com
+PageScorer
+Papa Foto
+Pavuk
+PeoplePal
 Pi-Monster
-PictureFinder
-Pixray
-PleaseCrawl
+Picscout
+Piepmatz
+ProPowerBot
 ProWebWalker
 Pu_iN
-PxBroker
 PyCurl
-Quick-Crawler
-RankActiveLinkBot
-RankingBot2
+QueryN Metasearch
+RankingBot
 RankurBot
-Reaper
-RedesScrapy
-SBIder
+Re-re
+ReGet
+Recorder
+Ripper
 SEOkicks
 SISTRIX
 SMTBot
-Screaming
+ScanAlert
+ScoutJet
 Searchestate
-SearchmetricsBot
 SemanticJuice
-Semrush
-SemrushBot
-SentiBot
-SeoSiteCheckup
-SeobilityBot
-Seomoz
 Shodan
-Siphon
 Site Sucker
 SiteExplorer
-SiteLockSpider
 SiteSnagger
+Sitebeam
 Siteimprove
 Sitevigil
-Snoopy
-Sociscraper
-Sogou web spider
-Sottopop
-Sqlworm
+SlySearch
+SmartDownload
+Snake
+SocialRankIOBot
+Spammen
+SpankBot
+Spanner
+Spbot
+Spinn3r
+SputnikBot
 Stripper
-Sucuri
-SuperHTTP
 Surfbot
 SurveyBot
-Suzuran
-TeleportPro
-TightTwatBot
-Titan
+Szukacz
+T8Abot
+Teleport
+Telesphorep
 Toweyabot
 Trendiction
+Trendictionbot
 True_Robot
-Upflow
-VB Project
+Turnitin
+Typhoeus
+URLy.Warning
+UnisterBot
+V-BOT
 VCI
-Vacuum
-VidibleScraper
 Voil
-WBSearchBot
-WISENutbot
+Voltron
 WPScan
+WWW-Mechanize
+WWW::Mechanize
 WWWOFFLE
-Web Collage
-Web Enhancer
-Web Fetch
-Web Pix
+WeSEE
+Web Auto
+Web Sauger
 Web Sucker
-WebAuto
-WebBandit
-WebGo IS
-WebImageCollector
+WebCopier
+WebEnhancer
+WebLeacher
+WebPix
 WebSauger
-Webalta
-Webshag
+WebStripper
+WebWhacker
 Webster
-Whacker
+Whack
+Whatweb
 WinHTTrack
 WiseGuys Robot
 Wonderbot
-Woobot
+Wotbox
+Wprecon
 Xaldon_WebSpider
 Xenu
-Zermelo
-Zeus
+Zauba
 ZoominfoBot
-ZumBot
-ZyBorg
-arquivo-web-crawler
-autoemailspider
-backlink-check
-coccocbot
+adscanner
+arquivo.pt
+check1.exe
+clark-crawler
+cognitiveseo
+crawl.sogou.com
 crawler.feedback
-crawler4j
-demandbase-bot
-domainsproject.org
-eCatch
-evc-batch
 facebookscraper
 heritrix
-ips-agent
-lwp-request
-lwp-trivial
-netEstate NE Crawler
+instabid
+isitwp.com
 oBot
 probe-image-size
+ripz
+s1z.ru
 satoristudio.net
-scalaj-http
-seobility
-seocompany.store
 seoscanners
-sexsearcher
-sitechecker.pro
+seostar
+sogouspider
 spyfu
 sysscan
-tAkeOut
-trendiction.com
 ubermetrics-technologies.com
-voyagerx.com
 webgains-bot
-webmeup-crawler
-webpros.com
 webprosbot
+x09Mozilla
+zauba.io
+zgrab