V4.2022.02.2907 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-360Spider
-80legs
-ADmantX
+01h4x.com
+404enemy
 ALittle Client
 ASPSeek
 Abonti
+Aboundexbot
 Acunetix
-AiHitBot
+AfD-Verbotsverfahren
 Aipbot
-Ankit
+Alexibot
+Alligator
+Anarchie
 Anthill
-Apexoo
+Aspiegel
+Asterias
 AwarioRssBot
-AwarioSmartBot
-BLEXBot
+BBBike
+BDFetch
 BackDoorBot
-BackStreet
-Bandit
-Barkrowler
-Bitacle
+Backlink-Ceck
+BacklinkCrawler
+BetaBot
 Black Hole
 BlowFish
-Bolt
-BotALot
+Boardreader
 Brandwatch
 BuiltWith
-BunnySlippers
 CATExplorador
-CODE87
-CSHttp
-Calculon
 CazoodleBot
-Cegbfeieh
-CensysInspect
-Citoid
-CopyRightCheck
+CheeseBot
+CherryPicker
+ChinaClaw
+Chlooe
+Claritybot
+Cocolyzebot
+Collector
+Copier
 Copyscape
+Cosmos
 Craftbot
-Crawling at Home Project
 CrazyWebCrawler
 CrunchBot
-DBLBot
-DSearch
-DatabaseDriverMysqli
+DIIbot
+DTS Agent
+DataCha0s
 Deusu
-DigitalPebble
-Dirbuster
-DnyzBot
+Devil
+Disco
+Dispatch
 DomainAppender
-DomainCrawler
-Domains Project
+DomainStatsBot
+Dotbot
 Download Wonder
-Dragonfly
-ECCP/1.0
-EMail Siphon
 EMail Wolf
+EirGrabber
 EroCrawler
-Exabot
+Evil
+Express WebPictures
 ExtLinksBot
-EyeNetIE
 Ezooms
-FDM
-Firefox/7.0
+Fimap
+Flunky
+Foobot
 Freeuploader
 FrontPage
-Fuzz
+Fyrebot
 G-i-g-a-b-o-t
-GT::WWW
-Genieo
+GalaxyBot
 Gigabot
 Go!Zilla
-Go-Ahead-Got-It
+Gotit
+Grabber
 Grafula
-GrapeFX
-HMView
-HTMLparser
+HEADMasterSEO
 HTTP::Lite
-Humanlinks
-IDBot
+HTTrack
+Haansoft
+Havij
+Iblog
+IndeedBot
 Indy Library
 InfoTekies
+InterGET
 InternetSeer
-JamesBOT
+Iskanie
+Jbrofuzz
 JennyBot
-JetCar
-Jyxobot
-Kinza
+Jetty
+JikeSpider
+Jorgee
+JustView
+Kenjin Spider
+Keyword Density
 LNSpiderguy
-Larbin
+Leap
 LeechGet
-Lftp
+Libwhisker
+LieBaoFast
 Lightspeedsystems
+Likse
 LinkScan
-LinkpadBot
-LinksManager
-Litemage_walker
-Lmspider
-Ltx71
+LinkWalker
+Linkdexbot
+LinkextractorPro
+LinqiaMetadataDownloaderBot
+LinqiaRSSBot
 MFC_Tear_Sample
-MIDown tool
-MIIxpc
 MJ12bot
+Magnet
 Mail.RU_Bot
 Majestic SEO
 Majestic-SEO
+MarkMonitor
+Mass Downloader
 Masscan
+Mata Hari
 MauiBot
-Meanpathbot
-MegaIndex.ru
-MicroMessenger
-Microsoft Data Access
-Mister PiX
+Mb2345Browser
+MeanPath Bot
+Metauri
+Minefield
 Moblie Safari
 Mojolicious
-MolokaiBot
-Mozlila
-NICErsPRO
+Mr.4x3
+Msrabot
+NPbot
 Name Intelligence
+NearSite
+Needle
 Nessus
 NetMechanic
+NetZIP
+Netcraft
 Nettrack
-Nimbostratus
-Nuclei
+Nibbler
+Nmap
+Not
+Nutch
 Octopus
 Offline Explorer
 OpenLinkProfiler
-OpenVAS
-OrangeBot
-OrangeSpider
+Openfind
 OutfoxBot
-PECL::HTTP
-PHPCrawl
 POE-Component-Client-HTTP
 Page Analyzer
-PageScorer
 Panscient
-PeoplePal
-Pi-Monster
-PictureFinder
-PleaseCrawl
-ProWebWalker
+Picsearch
+Piepmatz
 Probethenet
-Pu_iN
-Pump
+PxBroker
+PyCurl
 QueryN Metasearch
 RSSingBot
-Rankivabot
-ReGet
-RealDownload
+RankActiveLinkBot
+RankingBot2
+Re-re
 RebelMouse
-Recorder
+RedesScrapy
 RepoMonkey
 Ripper
+Rogerbot
+SBIder
 SEOkicks-Robot
-SEOprofiler
-ScanAlert
-ScreenerBot
-Searchestate
+SEOlyticsCrawler
+Scanbot
+Scrapy
+ScrepyBot
 SearchmetricsBot
+Seekport
 SemanticJuice
 Semrush
-SentiBot
+Shodan
 Siphon
 Site Sucker
-SiteExplorer
 SiteLockSpider
-SiteSnagger
+Sitebeam
 Siteimprove
-SlySearch
-Snoopy
-SocialRankIOBot
-Sosospider
+Snapbot
+Sogou web spider
+Spammen
 SpankBot
-Spanner
-SputnikBot
-Sqworm
-Steeler
 Sucker
 SuperBot
-SuperHTTP
+SurveyBot
 Suzuran
+Szukacz
 T0PHackTeam
-T8Abot
-TeleportPro
+Telesoft
 Telesphoreo
+Telesphorep
 The Intraformant
-Thumbor
-TightTwatBot
-Titan
-Toweyabot
-Tracemyfile
-Trendiction
+Toata
 Trendictionbot
-Turingos
-Turnitin
-TwengaBot
 Typhoeus
-V-BOT
+URLy Warning
+UnisterBot
+Upflow
 VCI
-Vagabondo
-VidibleScraper
-WEBDAV
+Virusdie
+Voil
 WISENutbot
-WPScan
-WWW::Mechanize
-Wallpapers
-WeSEE
-WebBandit
+WWW-Collector-E
+Wallpapers/3.0
+WallpapersHD
+Web Fetch
+Web Sauger
+WebAuto
 WebCollage
 WebCopier
-WebFetch
-WebImageCollector
-WebPix
-WebWhacker
-WebZIP
-WebmasterWorldForumBot
+WebLeacher
+WebSucker
+Webalta
+Webshag
 Website Quester
-Webster
-Whack
-Whatweb
-WinHTTrack
-Xenu
-Zermelo
-ZoominfoBot
+WebsiteExtractor
+WebsiteQuester
+Who.is Bot
+Wonderbot
+Woobot
+Wotbox
+Xaldon WebSpider
+YoudaoBot
+Zade
+Zeus
+ZoomBot
 adscanner
+archive.org_bot
 autoemailspider
-backlink-check
-check1.exe
-com.plumanalytics
-dataforseo.com
-demandbase-bot
-heritrix
-ips-agent
-lwp-request
+clark-crawler
+crawl.sogou.com
+crawler.feedback
+crawler4j
+eCatch
+instabid
 magpie-crawler
-muhstik-scan
-netEstate NE Crawler
-pcBrowser
+mediawords
+oBot
 probe-image-size
 ripz
+s1z.ru
 satoristudio.net
-seoscanners
+serpstatbot
 sexsearcher
-siteripz
 sogouspider
-sysscan
+sp_auditbot
 voyagerx.com
-webpros.com
-x09Mozilla
-zgrab
+webgains-bot
+webmeup-crawler
+x22Mozilla
+xpymep1.exe