V4.2024.05.4492 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-Aboundex
-AhrefsBot
+01h4x.com
+Aboundexbot
 AiHitBot
-Alexibot
-AlphaBot
 Apexoo
 AwarioRssBot
 BLEXBot
-Barkrowler
+BackStreet
+BacklinkCrawler
 BunnySlippers
-Calculon
+CATExplorador
 CheeseBot
+CherryPicker
 ChinaClaw
-Cliqzbot
-CopyRightCheck
-Craftbot
-DTS Agent
+CyotekWebCopy
+DSearch
+DataCha0s
+Devil
 DomCopBot
-DomainStatsBot
-Freeuploader
-FyberSpider
-GPTBot
+DomainSigmaCrawler
+EasyDL
+Ebingbong
+EirGrabber
+Evil
+Exabot
+Ezooms
+FHscan
+FemtosearchBot
+GetRight
+Gigabot
+GoZilla
 Google-Extended
-GrapeFX
-JamesBOT
-Jorgee
-LNSpiderguy
+HTMLparser
+Havij
+Id-search
+Indy Library
+InfraSec Scanner
+Intelliseek
+Internet Ninja
+InternetSeer
+JennyBot
 Lanshanbot
-Larbin
-LibWeb
-MJ12bot
-MTRobot
-MarkWatch
-Masscan
-MegaIndex.ru
-MolokaiBot
-Netcraft
-NimbleCrawler
-Ninja
+MIIxpc
+Magnet
+MauiBot
+Mediatoolkitbot
+Metauri
+MicroMessenger
+Minefield
+Mister PiX
+Msrabot
+Musobot
+Needle
+NextGenSearchBot
+Nikto
 Nuclei
-Octopus
-Offline Explorer
 Offline Navigator
-OrangeSpider
+Openvas
+PageScorer
+Pandalytics
 Panscient
-PictureFinder
-Pixray
-ProPowerBot
-ProWebWalker
+Pavuk
+Picscout
+Picsearch
+Piepmatz
+Probethenet
+Pu_iN
 RSSingBot
-Re-re
-Ripper
-ScoutJet
-SeobilityBot
-Siphon
-SiteCheckerBotCrawler
-Sitevigil
-SocialRankIOBot
+SemrushBot
+Sosospider
+Spammen
+Spanner
 Spbot
-Spinn3r
-Telesphoreo
-Tracemyfile
-TurnitinBot
-URLy.Warning
-UnisterBot
+Suzuran
+Telesoft
+Telesphorep
+Toata
+Twice
 VB Project
-Vagabondo
-Virusdie
-VoidEYE
 Voltron
-WASALive-Bot
 WWW-Collector-E
-WWW-Mechanize
-WebCopier
-WebFetch
-WebLeacher
+Web Enhancer
+Web Pix
+WebsiteExtractor
 WebsiteQuester
-Whacker
-Widow
-Woobot
-adscanner
-anthropic-ai
-arquivo-web-crawler
-cah.io.community
-cognitiveseo
-com.plumanalytics
+Webster
+Xaldon_WebSpider
+arquivo.pt
+autoemailspider
+clark-crawler
+cohere-ai
 crawl.sogou.com
 crawler4j
-dataforseobot
-isitwp.com
-page scorer
-polaris version
-probe-image-size
-s1z.ru
-scan.lol
-seocompany.store
-spyfu
+facebookscraper
+heritrix
+imagesift.com
+mediawords
+scalaj-http
+siteripz
 tAkeOut
-ubermetrics-technologies.com
-webgains-bot
+x22Mozilla