V4.2021.12.2794 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
+404checker
 80legs
+AIBOT
+ALittle Client
+Aboundex
+Aboundexbot
 AhrefsBot
-AiHitBot
 Aipbot
-Alexibot
-AllSubmitter
-Alligator
 AlphaBot
-Anarchie
-Ankit
-Asterias
-AwarioSmartBot
-BDFetch
-BackDoorBot
+Anthill
+Aspiegel
+Attach
+BackStreet
 BackWeb
-Bandit
-BatchFTP
+BacklinkCrawler
+Barkrowler
+Battleztar Bazinga
 Bitacle
-BlackWidow
-BlowFish
-Boardreader
+Black Hole
+Blackboard
+Bolt
 BotALot
-Brandwatch
-BuiltWith
-CODE87
+CCBot
 CSHttp
-Cegbfeieh
-CherryPicker
+Calculon
+CensysInspect
+CheeseBot
 Chlooe
-Cloud mapping
+Citoid
+Cliqzbot
+Cocolyzebot
 Cogentbot
-Collector
 CopyRightCheck
+Cosmos
 Craftbot
 Crawling at Home Project
 CrazyWebCrawler
-Custo
-DIIbot
-DTS Agent
+CyotekWebCopy
+DSearch
 DataCha0s
-Digincore
 Dirbuster
 Dispatch
+DittoSpyder
+DnBCrawler-Analytics
+DnyzBot
+DomCopBot
 DomainAppender
-DomainStatsBot
+DomainSigmaCrawler
+Download Wonder
 ECCP/1.0
-EMail Wolf
-Ebingbong
+EMail Siphon
+EasyDL
 Ecxi
+EirGrabber
+Evil
+Exabot
 ExtLinksBot
-ExtractorPro
+Extractor
 FDM
-FHscan
-Fimap
+FemtosearchBot
 Flunky
 FrontPage
-Fyrebot
-G-i-g-a-b-o-t
-GalaxyBot
+Fuzz
+FyberSpider
+GermCrawler
+GetRight
 GetWeb
 Gigabot
 Go!Zilla
-GoZilla
-Grabber
-GrapeshotCrawler
-HEADMasterSEO
-HTMLparser
-HTTrack
-Haansoft
+GridBot
+HMView
+HaosouSpider
+Harvest
 Havij
 Heritrix
-IDBTE4M
-IlseBot
-Image Sucker
+Humanlinks
+HybridBot
+Id-search
 IndeedBot
-InfoNaviRobot
-InfoTekies
+Indy Library
 InterGET
-Internet Ninja
 InternetSeer
 IstellaBot
+JOC Web Spider
 JamesBOT
 JetCar
-JikeSpider
+Jetty
+Jorgee
 JustView
+Jyxobot
+Kenjin Spider
+Keyword Density
+Kinza
 Kozmosbot
-LWP::Simple
-Lanshanbot
 Larbin
-Leap
-LeechFTP
-LibWeb
+LeechGet
+LexiBot
 Libwhisker
 LieBaoFast
-Lightspeedsystems
 Likse
+LinkScan
+LinkWalker
 Linkbot
-Linkdexbot
-LinqiaScrapeBot
-MIIxpc
+LinkpadBot
+LinksManager
+LinqiaRSSBot
+Lipperhey Spider
+Litemage_walker
+Lmspider
+Ltx71
 MJ12bot
-Majestic SEO
-Majestic-SEO
-MarkMonitor
-Masscan
+MSIECrawler
+MTRobot
+Mag-Net
+Mail.RU_Bot
+Mass Downloader
 Mata Hari
-MauiBot
-Microsoft Data Access
+Mediatoolkitbot
+MicroMessenger
+Microsoft URL Control
 Minefield
-Mister PiX
-Moblie Safari
-Mojolicious
+Mozlila
+Mr.4x3
+Msrabot
 NICErsPRO
 Nameprotect
-Navroad
 Nessus
 Net Vampire
-NetAnts
-NetSpider
-NextGenSearchBot
+NetMechanic
+Niki-bot
 NimbleCrawler
-Ninja
-Nmap
+Nimbostratus
+Nutch
 Offline Explorer
-Offline Navigator
-OnCrawl
 OpenLinkProfiler
 OpenVAS
 Openfind
-OrangeBot
+OrangeSpider
 PECL::HTTP
-Page Analyzer
-PageAnalyzer
+POE-Component-Client-HTTP
+PageGrabber
 PageScorer
 Pandalytics
+Panscient
 Papa Foto
-PeoplePal
 Picscout
 PictureFinder
-Piepmatz
 Pimonster
+Pixray
 Pockey
-ProPowerBot
-ProWebWalker
-Psbot
-PyCurl
-Quick-Crawler
-RankActive
-RankActiveLinkBot
+PxBroker
+RSSingBot
+Rankivabot
+Re-re
 RealDownload
-RebelMouse
 Recorder
-RedesScrapy
 RepoMonkey
 Ripper
 Rogerbot
-SEOkicks
+SBIder
 SEOkicks-Robot
+SEOlyticsCrawler
 SEOstats
-ScanAlert
+SISTRIX
 Scanbot
-ScoutJet
-ScrepyBot
-Searchestate
+Screaming
 SearchmetricsBot
-SeoSiteCheckup
+SemanticJuice
+SemrushBot
+SentiBot
 SeobilityBot
-Seomoz
 Siphon
-SiteExplorer
+SiteCheckerBotCrawler
 SiteLockSpider
 Sitebeam
-Sitevigil
-Snake
-Sociscraper
+Siteimprove
+SmartDownload
+SocialRankIOBot
+Sogou web spider
+Sottopop
 SpaceBison
 Spanner
-Sqlmap
-Sqlworm
-Sucuri
-SuperHTTP
+SputnikBot
+Surfbot
 SurveyBot
-Swiftbot
-Szukacz
-T8Abot
 Teleport
+TeleportPro
 Telesoft
-The Intraformant
-TheNomad
-TightTwatBot
-Toweyabot
-Trendiction
-True_Robot
+Telesphoreo
+Trendictionbot
+Turnitin
+Typhoeus
 URLy.Warning
 UnisterBot
 Upflow
 V-BOT
 VCI
-Vagabondo
-Virusdie
-Voil
+Vacuum
+VelenPublicWebCrawler
+VidibleScraper
+Voltron
 WASALive-Bot
-WBSearchBot
-WWW-Mechanize
-WallpapersHD
+WWW::Mechanize
+WWWOFFLE
+Wallpapers
+WeSEE
 Web Collage
-Web Enhancer
 Web Fetch
+Web Fuck
+Web Sauger
 WebCollage
-WebFuck
+WebCopier
+WebEnhancer
+WebGo IS
 WebImageCollector
-WebLeacher
+WebPix
+WebSauger
+WebSucker
 WebZIP
-Webalta
 Webshag
-Website Quester
-Whack
-WiseGuys Robot
-Wotbox
-Wprecon
-Xaldon WebSpider
+WebsiteExtractor
+Wonderbot
 Xenu
-Zermelo
+YoudaoBot
+Zade
 Zitebot
-ZmEu
-ZoomBot
-cah.io.community
-cognitiveseo
-com.plumanalytics
-crawler4j
+ZyBorg
+arquivo.pt
+backlink-check
+check1.exe
+crawler.feedback
 dataforseo.com
+demandbase-bot
 domainsproject.org
-evc-batch
-facebookscraper
 gopher
-instabid
-lwp-request
-lwp-trivial
+heritrix
+ips-agent
 magpie-crawler
-mediawords
-oBot
-pcBrowser
+netEstate NE Crawler
+probe-image-size
+ripz
+scan.lol
+seocompany.store
 seoscanners
 sexsearcher
-sysscan
+sogouspider
+sp_auditbot
 webgains-bot
-webpros.com
 x09Mozilla
-xpymep1.exe
+x22Mozilla