V4.2024.05.4479 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-AdsTxtCrawlerTP
+404enemy
+Acunetix
+AhrefsBot
+Anarchy
+Atomseobot
 AwarioBot
-BLEXBot
-Black Hole
-BlowFish
-Cegbfeieh
-CheTeam
-CheeseBot
-Crawling at Home Project
+Brandprotect
+Buddy
+BuiltBotTough
+Bullseye
+CSHttp
+CazoodleBot
+CopyRightCheck
+Copyscape
 CrunchBot
-DataCha0s
-Digincore
-Discoverybot
+DSearch
+Dirbuster
 Dispatch
-Ebingbong
-ExtractorPro
-Extreme Picture Finder
+DomCopBot
+Dotbot
+EasyDL
+Extractor
+Firefox/7.0
 FlashGet
-Flunky
-Fyrebot
+FyberSpider
 G-i-g-a-b-o-t
 GermCrawler
-IDBTE4M
-IRLbot
-Iblog
-IndeedBot
+Getintent
+Gotit
+HEADMasterSEO
+Haansoft
+HybridBot
+Image Fetch
 Iria
-JOC Web Spider
-Kinza
-LibWeb
-Libwhisker
-Lightspeedsystems
-Linkbot
+Jbrofuzz
+Jorgee
+LeechFTP
+LexiBot
+LieBaoFast
+LinqiaRSSBot
+Lipperhey
+Lipperhey Spider
+MSFrontPage
+MSIECrawler
 Majestic SEO
-Minefield
 Msrabot
-NICErsPRO
 NPbot
-Needle
 Net Vampire
 NetSpider
-Nibbler
-NimbleCrawler
-Nmap
-Offline Navigator
-Openfind
-OrangeBot
-PageScorer
-Papa Foto
-PictureFinder
-Psbot
-RSSingBot
+Octopus
+Offline Explorer
+Page Analyzer
+PageAnalyzer
+Picscout
+QueryN Metasearch
 RankingBot
-RealDownload
-Reaper
-RocketCrawler
-SMTBot
-ScanAlert
-Screaming
+RankingBot2
+Re-re
+RebelMouse
+SEOstats
+ScreenerBot
+ScrepyBot
 SeekportBot
-Seomoz
-Site Sucker
-SiteSnagger
-SlySearch
+Siteimprove
 SmartDownload
-SocialRankIOBot
-Sociscraper
-Telesoft
+Sottopop
+Spbot
+Steeler
+Telesphoreo
 The Intraformant
-Turingos
-Turnitin
-TurnitinBot
+Thumbor
+Tracemyfile
+URLy Warning
+URLy.Warning
+V-BOT
 VCI
-VidibleScraper
-Virusdie
-WBSearchBot
-Web Collage
-Web Enhancer
-Web Pix
-WebStripper
+VeriCiteCrawler
+WWW-Mechanize
+WebFetch
+WebGo IS
+WebPix
+WebSucker
 Webshag
-Website Quester
-Webster
-Whack
-Wonderbot
-Xenu
-Zauba
-Zermelo
-Zitebot
-anthropic-ai
-arquivo.pt
-clark-crawler
-crawler4j
+Whatweb
+ZyBorg
+awario.com
+crawl.sogou.com
+crawler.feedback
+facebookscraper
+heritrix
 imagesift.com
-internetVista monitor
-mediawords
-sexsearcher
-sogouspider
-spyfu
-webprosbot
+isitwp.com
+lwp-trivial
+netEstate NE Crawler
+oBot
+pcBrowser
+plumanalytics
+sp_auditbot
+webmeup-crawler
+xpymep1.exe