V4.2022.11.3479 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
+404checker
 ADmantX
-ALittle Client
+AIBOT
+ASPSeek
 Abonti
-Aboundex
-Aboundexbot
-AhrefsBot
-Aipbot
+AfD-Verbotsverfahren
+AiHitBot
 AlphaBot
 Anarchie
+Anarchy99
+Ankit
 Anthill
-Apexoo
 Aspiegel
-Asterias
+Atomseobot
 Attach
-BDFetch
-BLEXBot
-BackDoorBot
-BatchFTP
-Bigfoot
-Blackboard
+BDCbot
+BackStreet
+BacklinkCrawler
+Bandit
+BetaBot
+Black Hole
+BlackWidow
 Boardreader
-BotALot
-Brandprotect
-Buddy
-CCBot
-CSHttp
-CensysInspect
-Cliqzbot
-Cloud mapping
-Cocolyzebot
+BuiltWith
+Cegbfeieh
+CheeseBot
+CherryPicker
+Citoid
+Claritybot
 Cogentbot
+Copier
 CopyRightCheck
-Copyscape
-Craftbot
+Cosmos
 Crawling at Home Project
-Crescent
-CrunchBot
-Curious
 CyotekWebCopy
-DBLBot
-DIIbot
 DSearch
-DTS Agent
-Demon
+DatabaseDriverMysqli
 Deusu
+Devil
+Digincore
 DigitalPebble
-Dirbuster
 Disco
+DittoSpyder
 DnBCrawler-Analytics
-DomCopBot
-DomainAppender
+DnyzBot
+DomainCrawler
+DomainSigmaCrawler
 DomainStatsBot
 Domains Project
-Dragonfly
+Dotbot
+Download Wonder
 Drip
-EMail Siphon
 EasyDL
-Ebingbong
-EirGrabber
+Ecxi
 Evil
-ExtractorPro
+Exabot
+Extractor
 Extreme Picture Finder
-Ezooms
+EyeNetIE
 FDM
-FemtosearchBot
-Firefox/7.0
-Gigabot
+GT::WWW
+GalaxyBot
+GermCrawler
 Go!Zilla
+Go-Ahead-Got-It
+GoZilla
 Gotit
 GrabNet
 GrapeshotCrawler
+HEADMasterSEO
 HTTP::Lite
-Haansoft
 HaosouSpider
+Harvest
 Hloader
-HonoluluBot
-Humanlinks
-HybridBot
-IDBot
-IlseBot
+IDBTE4M
+Iblog
 Image Fetch
 IndeedBot
-InfoNaviRobot
+Indy Library
 Intelliseek
+InterGET
 Internet Ninja
 InternetSeer
-Iria
-Iskanie
-IstellaBot
-Kenjin Spider
+JamesBOT
+JennyBot
+Jorgee
+JustView
+Jyxobot
 Keyword Density
 LNSpiderguy
-LWP::Simple
 Lanshanbot
-LeechGet
-LexiBot
 Lftp
 LibWeb
-LieBaoFast
-LinkScan
-LinkWalker
-Linkbot
+Lightspeedsystems
 LinkextractorPro
-MJ12bot
-MQQBrowser
-MSIECrawler
+LinkpadBot
+LinqiaMetadataDownloaderBot
+Lipperhey
+Lmspider
+MSFrontPage
 MTRobot
-Mail.RU_Bot
-Majestic-SEO
-MarkWatch
-Mass Downloader
-Masscan
-Mb2345Browser
-MeanPath Bot
-Meanpathbot
+Mag-Net
+Magnet
+Mediatoolkitbot
 Microsoft URL Control
 Mister PiX
-Mojolicious
+Mojeek
 MolokaiBot
+Morfeus Fucking Scanner
 Mozlila
-Msrabot
 Musobot
-NICErsPRO
-Name Intelligence
 Nameprotect
-NetAnts
-NetSpider
-NetZIP
-Netvibes
-Nibbler
-Nikto
-Nimbostratus
+Navroad
+NearSite
+Nessus
+NetMechanic
+Netcraft
+NextGenSearchBot
+NimbleCrawler
 Ninja
-Nmap
+Not
 Nutch
-Openvas
+Octopus
+Offline Navigator
+OnCrawl
+Openfind
 OrangeSpider
-OutclicksBot
+OutfoxBot
 PHPCrawl
-PageAnalyzer
 PageScorer
-PeoplePal
-Picscout
-Picsearch
+Pandalytics
+Panscient
+Papa Foto
+Pavuk
 Piepmatz
+Pixray
+PleaseCrawl
+Pockey
 ProPowerBot
-Probethenet
-Psbot
+ProWebWalker
+Pu_iN
 Pump
-PxBroker
-QueryN Metasearch
-RankingBot
+Quick-Crawler
+RSSingBot
+RankFlex
 RankingBot2
+Rankivabot
 ReGet
-RealDownload
 Reaper
-RepoMonkey
-SEOstats
-SISTRIX
-SMTBot
+Recorder
+SEOkicks-Robot
+ScanAlert
 Scanbot
-ScoutJet
-ScrepyBot
+Screaming
+Searchestate
+SearchmetricsBot
 Seekport
 Semrush
-SentiBot
-SiteLockSpider
-SiteSnagger
-SlySearch
-Snake
+SemrushBot
+SeoSiteCheckup
+Seomoz
+Siphon
+Site Sucker
+SiteCheckerBotCrawler
+Sitebeam
+SmartDownload
 Snoopy
-SocialRankIOBot
-Sociscraper
-Spanner
-Spinn3r
-Steeler
-Stripper
+Sosospider
+Sqlworm
 Sucker
 SuperBot
+Swiftbot
 T0PHackTeam
-Teleport
-TeleportPro
-Titan
-Toweyabot
+T8Abot
+TheNomad
+Thumbor
 Trendictionbot
 True_Robot
-V-BOT
+Turnitin
+TurnitinBot
+TwengaBot
+URLy.Warning
+Upflow
 VCI
+Vagabondo
 VeriCiteCrawler
-Voltron
-WEBDAV
 WISENutbot
-WWW-Collector-E
-WWWOFFLE
-Wallpapers
-Wallpapers/3.0
+WWW::Mechanize
+WallpapersHD
+WeSEE
+Web Auto
 Web Collage
-Web Pix
+Web Enhancer
+Web Sucker
+WebAuto
 WebBandit
+WebCollage
 WebCopier
+WebEnhancer
+WebFetch
 WebFuck
-WebGo IS
+WebImageCollector
+WebLeacher
 WebReaper
-WebSauger
 WebStripper
-WebSucker
 WebWhacker
-WebmasterWorldForumBot
-Webshag
-Website Quester
+WebsiteQuester
+Webster
 Whatweb
-Widow
-WiseGuys Robot
+Who.is Bot
 Wonderbot
-Xaldon_WebSpider
-Zermelo
-Zeus
-ZmEu
+Woobot
+Wotbox
+Xaldon WebSpider
+Zauba
+Zitebot
 ZoomBot
-autoemailspider
-cah.io.community
-clark-crawler
-crawler.feedback
-dataforseo.com
+ZyBorg
+arquivo-web-crawler
+backlink-check
+coccocbot
+crawl.sogou.com
+crawler4j
+domainsproject.org
 eCatch
-evc-batch
 gopher
-instabid
 internetVista monitor
+iubenda-radar
 lwp-request
+meanpathbot
+muhstik-scan
+oBot
+page scorer
 pcBrowser
-probe-image-size
+ripz
 s1z.ru
-satoristudio.net
-scalaj-http
-seocompany.store
+scan.lol
+seobility
 seoscanners
-seostar
+serpstatbot
 sexsearcher
-sitechecker.pro
-siteripz
+sogouspider
+spyfu
 sysscan
+tAkeOut
+trendiction.com
 trendiction.de
-webgains-bot
-webmeup-crawler
-webprosbot
-x09Mozilla
+voyagerx.com
 zauba.io
-zgrab