V4.2024.07.4635 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-01h4x.com
 360Spider
-AIBOT
 AdsTxtCrawlerTP
 AfD-Verbotsverfahren
-AiHitBot
-Aipbot
-Anarchy99
-Ankit
-BBBike
-BDFetch
-BackDoorBot
-Bitacle
-BlowFish
-Brandprotect
-Brandwatch
-BuiltBotTough
-BuzzSumo
-CCBot
+AhrefsBot
+Apexoo
+Attach
+Backlink-Ceck
+Bandit
 Calculon
-ChatGPT-User
+CheTeam
 CheeseBot
+Citoid
 Claritybot
-Copier
-DomainSigmaCrawler
-Ebingbong
-Evil
-Express WebPictures
-FrontPage
+Copyscape
+CrunchBot
+DTS Agent
+Dispatch
+DnyzBot
+EasyDL
+EirGrabber
+Exabot
+FacebookBot
+FlashGet
+Fyrebot
 GPTBot
-Gigabot
-GridBot
-HMView
+Google-Extended
+Grafula
+GrapeFX
 Haansoft
-IlseBot
-Image Sucker
-Indy Library
-Iskanie
-Lanshanbot
-Leap
-Lightspeedsystems
+Havij
+Heritrix
+IDBTE4M
+InternetSeer
+IstellaBot
+JOC Web Spider
+JamesBOT
+JikeSpider
+Keybot Translation-Search-Machine
+Kinza
+LexiBot
+Likse
+LinkWalker
 Linkbot
-Lipperhey Spider
-Ltx71
-MFC_Tear_Sample
-MIDown tool
+LinqiaRSSBot
+MJ12bot
 MSIECrawler
-Majestic12
+MTRobot
 Mass Downloader
-Mata Hari
-Niki-bot
-Ninja
-OrangeSpider
-Picsearch
+Mb2345Browser
+Mojeek
+Msrabot
+Musobot
+NextGenSearchBot
+Nikto
+Nmap
+OnCrawl
+PageScorer
+Pavuk
+Picscout
+ProPowerBot
 Pu_iN
-Quick-Crawler
-RankingBot2
-Re-re
-RepoMonkey
-SEOkicks-Robot
-SalesIntelligent
-ScoutJet
-Seekport
-SeoSiteCheckup
-Shodan
-SmartDownload
-SocialRankIOBot
-Sqlworm
-Steeler
-T8Abot
-Telesphoreo
-Thumbor
-True_Robot
-Twice
-URLy Warning
-VeriCiteCrawler
-Voil
-WEBDAV
-Web Collage
-Web Sucker
-WebCopier
-WebGo IS
-WebPix
-WebReaper
-WebStripper
-Webshag
-Who.is Bot
-ZmEu
-ZoominfoBot
-adscanner
+PyCurl
+Reaper
+Recorder
+RedesScrapy
+SBIder
+SiteSnagger
+SiteSucker
+Sosospider
+Sqworm
+Teleport
+Toata
+TwengaBot
+URLy.Warning
+Voltron
+Web Fetch
+Web Fuck
+WebFetch
+WebsiteExtractor
+Whatweb
+Xaldon WebSpider
+YaK
+Zermelo
+Zitebot
+cah.io.community
+check1.exe
 coccocbot
-cohere-ai
-evc-batch
-imagesift.com
+cognitiveseo
+dataforseobot
+heritrix
+instabid
+ips-agent
+iubenda-radar
 linkdexbot
-lwp-request
-polaris version
-seoscanners
-sysscan
-x22Mozilla
+linkfluence
+magpie-crawler
+scan.lol
+sogouspider
+webprosbot
+zauba.io