V4.2022.09.3373 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-404enemy
+404checker
+AIBOT
 ASPSeek
-Abonti
 Aboundex
-Aipbot
-Anarchy
-Asterias
-Atomseobot
+AfD-Verbotsverfahren
+AhrefsBot
+AllSubmitter
+Anthill
 Attach
-BBBike
+AwarioSmartBot
+BDCbot
+BackStreet
+BackWeb
+BacklinkCrawler
 Badass
 Battleztar Bazinga
+Bigfoot
+Bitacle
 BlackWidow
-Blackboard
-Blow
-BlowFish
-Boardreader
-Bolt
 BotALot
-Bullseye
+Buck
+Buddy
+BuiltBotTough
+BuiltWith
+BuzzSumo
+CCBot
+CODE87
 CSHttp
+Calculon
 CazoodleBot
-CheeseBot
+Cegbfeieh
+CherryPicker
 ChinaClaw
-Citoid
-Claritybot
+Chlooe
 Cloud mapping
-Cogentbot
-Copyscape
-Cosmos
-Crawling at Home Project
+Cocolyzebot
+Collector
+Copier
+Craftbot
+CrunchBot
 Curious
 Custo
-CyotekWebCopy
 DSearch
-Demon
-Digincore
+DTS Agent
+DataCha0s
+Devil
 DigitalPebble
-Disco
+Dirbuster
+Dispatch
 DittoSpyder
+DnBCrawler-Analytics
+DomCopBot
 DomainAppender
-Domains Project
-Dotbot
-Drip
+DomainStatsBot
+Dragonfly
+EasyDL
+Ecxi
+EirGrabber
 EroCrawler
-Exabot
 Express WebPictures
-FDM
-Firefox/7.0
+ExtractorPro
+EyeNetIE
+Ezooms
 FlashGet
-Freeuploader
-FrontPage
+Flunky
 G-i-g-a-b-o-t
-GetRight
-Getintent
-Gigabot
-Go!Zilla
+GalaxyBot
+GetWeb
 Go-Ahead-Got-It
-Gotit
-Grabber
-GrapeFX
-HTMLparser
+GoZilla
+Grafula
+GrapeshotCrawler
+HEADMasterSEO
+HTTrack
+Haansoft
+HaosouSpider
 Harvest
 Havij
 Heritrix
-Hloader
-IDBTE4M
-IRLbot
-Id-search
+HonoluluBot
+HybridBot
+IlseBot
 Image Fetch
-InfoNaviRobot
-Intelliseek
-InternetSeer
+Image Sucker
+InfoTekies
+Iria
 Iskanie
-JOC Web Spider
-Jbrofuzz
-JetCar
-Jetty
+IstellaBot
+JamesBOT
 JikeSpider
-Keybot Translation-Search-Machine
-LNSpiderguy
+Joomla
+Jorgee
+Kenjin Spider
+Keyword Density
+Kinza
 Larbin
-LeechGet
-Lftp
-Libwhisker
-Likse
-LinkScan
-Linkbot
-LinkextractorPro
-LinqiaMetadataDownloaderBot
+Leap
+LibWeb
+LieBaoFast
+Lightspeedsystems
+LinkpadBot
 LinqiaRSSBot
-LinqiaScrapeBot
-Lipperhey
+Lipperhey Spider
 Lmspider
 MFC_Tear_Sample
-MIDown tool
-MIIxpc
-MJ12bot
-MSFrontPage
 MSIECrawler
 Magnet
 Mail.RU_Bot
-Mass Downloader
+Majestic-SEO
 MauiBot
+Mb2345Browser
 MegaIndex.ru
-Moblie Safari
+MicroMessenger
+Mister PiX
+Mojeek
 MolokaiBot
 Mr.4x3
-Msrabot
 Musobot
 NPbot
 Nameprotect
-Navroad
-Net Vampire
-NetAnts
 NetLyzer
-NetSpider
-Netcraft
-NextGenSearchBot
+NetZIP
 Niki-bot
-NimbleCrawler
-Nimbostratus
-Not
-Nuclei
-Nutch
-OpenLinkProfiler
-Openvas
 OrangeBot
-OrangeSpider
-OutfoxBot
-POE-Component-Client-HTTP
-PageThing.com
-Pavuk
+PECL::HTTP
+PHPCrawl
+PageScorer
+Pandalytics
+Papa Foto
 PeoplePal
-Petalbot
+Picscout
+Picsearch
 Piepmatz
-Pockey
-ProWebWalker
-Psbot
-PyCurl
-RSSingBot
-RankActiveLinkBot
-RankFlex
-RankingBot2
-Re-re
-ReGet
+Pimonster
+Pixray
+ProPowerBot
+Probethenet
+Pump
+PxBroker
+Quick-Crawler
 Reaper
-Rogerbot
+RedesScrapy
+RepoMonkey
+Ripper
+SBIder
 SEOkicks
-SEOprofiler
-SMTBot
-ScanAlert
-Screaming
-ScreenerBot
-ScrepyBot
-Searchestate
-SearchmetricsBot
+SEOkicks-Robot
+SEOlyticsCrawler
+SISTRIX
 Semrush
 SentiBot
+SeoSiteCheckup
 SeobilityBot
 Seomoz
 Siphon
-SiteCheckerBotCrawler
-SiteSnagger
-SlySearch
-SocialRankIOBot
-Sogou web spider
+Site Sucker
+Sitevigil
+SmartDownload
 Spammen
 SpankBot
-Spanner
+Spbot
+Spinn3r
 SputnikBot
-Sqlworm
+Sqlmap
+Steeler
+Stripper
+Sucuri
 SuperBot
-SurveyBot
+SuperHTTP
 Suzuran
-Swiftbot
-T0PHackTeam
 Teleport
-TeleportPro
-Telesoft
+Telesphoreo
+Telesphorep
+The Intraformant
 TightTwatBot
-Titan
 Toata
-Tracemyfile
 Trendiction
-True_Robot
-TwengaBot
-Typhoeus
-URLy Warning
-UnisterBot
+Turingos
+Turnitin
 V-BOT
-Vacuum
-VeriCiteCrawler
-Virusdie
+VB Project
+VCI
+VidibleScraper
 VoidEYE
+Voil
+Voltron
 WASALive-Bot
+WBSearchBot
 WEBDAV
 WISENutbot
 WWW::Mechanize
+WWWOFFLE
 Wallpapers
-WallpapersHD
+Wallpapers/3.0
 Web Auto
-Web Fetch
+Web Collage
+Web Enhancer
 Web Fuck
-Web Sucker
+Web Pix
+WebAuto
 WebBandit
-WebCollage
-WebCopier
-WebFetch
-WebGo IS
-WebLeacher
 WebReaper
-WebSauger
-WebStripper
-WebWhacker
+WebSucker
 WebZIP
-Who.is Bot
+Webalta
+Webshag
+Website Quester
+WebsiteExtractor
+WebsiteQuester
+Whatweb
 Widow
-Wonderbot
+WiseGuys Robot
 Wotbox
 Wprecon
+Xaldon WebSpider
 Xaldon_WebSpider
-YoudaoBot
-Zauba
-ZmEu
+Zitebot
 ZoomBot
-ZyBorg
+ZoominfoBot
+ZumBot
 adscanner
-backlink-check
 cah.io.community
+check1.exe
+cognitiveseo
 com.plumanalytics
-crawl.sogou.com
 crawler.feedback
+crawler4j
 dataforseo.com
-domainsproject.org
-ips-agent
+demandbase-bot
+eCatch
+evc-batch
+heritrix
 isitwp.com
 linkdexbot
-magpie-crawler
+mediawords
 muhstik-scan
+netEstate NE Crawler
 oBot
-plumanalytics
+ripz
 s1z.ru
-satoristudio.net
-sexsearcher
+scan.lol
+seoscanners
+seostar
+sitechecker.pro
 sogouspider
 sp_auditbot
-sysscan
+spyfu
 webmeup-crawler
+zauba.io