V4.2021.11.2716 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
 404enemy
-ADmantX
-ALittle Client
 ASPSeek
-Abonti
+Aboundexbot
 Acunetix
 AhrefsBot
-AllSubmitter
+Aipbot
+Alexibot
+Alligator
 AlphaBot
-Anarchy
-Anarchy99
-Ankit
+Anarchie
+Anthill
 Apexoo
-Asterias
 AwarioRssBot
-BDFetch
 BackDoorBot
-BackStreet
 BackWeb
-BacklinkCrawler
-Badass
+Bandit
+Battleztar Bazinga
 BetaBot
+Black Hole
 BlackWidow
-Blow
-Boardreader
+BlowFish
 Bolt
-BotALot
-Buck
 BuiltBotTough
-CCBot
-CSHttp
-Calculon
+BuiltWith
+CATExplorador
+CODE87
+CazoodleBot
 CherryPicker
+ChinaClaw
+Chlooe
 Claritybot
-Cliqzbot
-Collector
-Copier
-Cosmos
-CrazyWebCrawler
-Crescent
+Cogentbot
+Crawling at Home Project
 CrunchBot
-DBLBot
-DSearch
 DTS Agent
-DataCha0s
-Devil
-Digincore
-Dirbuster
+DatabaseDriverMysqli
+Demon
+Deusu
+DigitalPebble
 Disco
+Discobot
 Discoverybot
-DittoSpyder
-DnBCrawler-Analytics
-DnyzBot
+Dispatch
 DomCopBot
+DomainAppender
+DomainStatsBot
 Domains Project
-Download Wonder
-Drip
+Dragonfly
 EMail Siphon
-EasyDL
 Ecxi
 EroCrawler
-Express WebPictures
+Extractor
+ExtractorPro
+Extreme Picture Finder
 EyeNetIE
 FHscan
-Fimap
-Firefox/7.0
-FrontPage
+FlashGet
+Foobot
+Freeuploader
+FyberSpider
 G-i-g-a-b-o-t
-Genieo
 GermCrawler
-GetWeb
-Getintent
-Go!Zilla
-GrabNet
-Grabber
-GrapeFX
+Gotit
 GrapeshotCrawler
 HMView
 HTMLparser
-Harvest
+Havij
+Heritrix
+Humanlinks
+HybridBot
+IDBot
 Iblog
-IlseBot
-Image Fetch
+Image Sucker
 IndeedBot
-InfoNaviRobot
+Indy Library
+InfoTekies
 Intelliseek
+InterGET
 Internet Ninja
-IstellaBot
-JikeSpider
-Kenjin Spider
+Iskanie
+JOC Web Spider
+JamesBOT
+Jbrofuzz
+JetCar
+Jetty
+JustView
+Jyxobot
 Keyword Density
+Kinza
+LNSpiderguy
+Lanshanbot
 LeechGet
-Lftp
+LibWeb
+Libwhisker
 LieBaoFast
 LinkScan
 LinkWalker
-LinksManager
-LinqiaMetadataDownloaderBot
-LinqiaRSSBot
-LinqiaScrapeBot
-Lipperhey
 Lipperhey Spider
-Ltx71
-MFC_Tear_Sample
-Mail.RU_Bot
+MIDown tool
+MIIxpc
+MJ12bot
+MSIECrawler
+MTRobot
+Mag-Net
+Majestic SEO
+Majestic-SEO
+MarkMonitor
+MarkWatch
 Mass Downloader
 Meanpathbot
-MicroMessenger
+MegaIndex.ru
 Microsoft URL Control
-Moblie Safari
+Mister PiX
 Mojeek
-Morfeus Fucking Scanner
-Mr.4x3
-Msrabot
 Musobot
 NICErsPRO
-Nessus
-Net Vampire
-NetZIP
-Nibbler
+Name Intelligence
+Nameprotect
+Navroad
+NearSite
+NetAnts
+NetLyzer
+Netvibes
+NextGenSearchBot
 Niki-bot
 Nikto
 Ninja
 Nmap
+Nuclei
 Octopus
-OnCrawl
+Offline Explorer
+Offline Navigator
 OpenVAS
 Openfind
-OrangeBot
-OutclicksBot
-PECL::HTTP
+Openvas
+OrangeSpider
+OutfoxBot
+POE-Component-Client-HTTP
+Page Analyzer
 PageGrabber
+PageScorer
+PageThing.com
 Pandalytics
-Pavuk
-PeoplePal
+Papa Foto
+PictureFinder
 Piepmatz
-PleaseCrawl
+Pimonster
 ProPowerBot
-Pump
-QueryN Metasearch
+Psbot
 RSSingBot
-RankActive
 RankActiveLinkBot
-Re-re
-Reaper
+RankFlex
 Recorder
-RepoMonkey
-Rogerbot
+RedesScrapy
+Ripper
 SBIder
+SEOkicks-Robot
 SEOlyticsCrawler
+SEOstats
 SISTRIX
-ScanAlert
-Scrapy
+Scanbot
 ScreenerBot
 ScrepyBot
+Searchestate
+SearchmetricsBot
+SemrushBot
 SeoSiteCheckup
 SeobilityBot
-Site Sucker
-SiteLockSpider
+Shodan
+SiteCheckerBotCrawler
+SiteExplorer
 SiteSnagger
-SiteSucker
 Siteimprove
 SlySearch
-Sogou web spider
-Sottopop
-Spinn3r
-SputnikBot
+Sosospider
+SpaceBison
+SpankBot
 Sqlmap
 Sqlworm
-Steeler
+Sqworm
 Stripper
-Sucker
-SuperBot
+SuperHTTP
 Surfbot
 SurveyBot
 Suzuran
-Szukacz
 T0PHackTeam
-Titan
+T8Abot
+Telesphoreo
+Telesphorep
+Thumbor
 Toata
+Trendiction
 Trendictionbot
 Turnitin
+TurnitinBot
+Twice
 Typhoeus
-URLy Warning
-URLy.Warning
 UnisterBot
-Upflow
+VB Project
 VCI
-Vagabondo
+VelenPublicWebCrawler
 VeriCiteCrawler
 Virusdie
-Voltron
 WASALive-Bot
-WBSearchBot
+WISENutbot
 WPScan
-WWW-Collector-E
-WWWOFFLE
-Wallpapers
-Wallpapers/3.0
-WallpapersHD
+WWW::Mechanize
 Web Collage
-Web Fuck
-Web Pix
+Web Enhancer
+Web Fetch
 WebBandit
-WebCollage
-WebFetch
-WebGo IS
+WebCopier
+WebEnhancer
+WebFuck
 WebImageCollector
-WebLeacher
-WebPix
-WebWhacker
-Webalta
 Webshag
 WebsiteExtractor
-Whacker
-WinHTTrack
+Whatweb
+Who.is Bot
 WiseGuys Robot
-Woobot
 Wotbox
 Wprecon
-Xenu
-Zauba
+Xaldon WebSpider
+Xaldon_WebSpider
+YoudaoBot
+Zade
+Zermelo
 ZoomBot
-ZyBorg
-archive.org_bot
-autoemailspider
+ZoominfoBot
+adscanner
 backlink-check
 clark-crawler
-coccocbot-web
+cognitiveseo
 com.plumanalytics
 crawl.sogou.com
-crawler4j
 domainsproject.org
 eCatch
-instabid
-internetVista monitor
+isitwp.com
 meanpathbot
-mediawords
-page scorer
-plumanalytics
-probe-image-size
+netEstate NE Crawler
+s1z.ru
+seobility
+seocompany.store
 seostar
-sp_auditbot
+sogouspider
 spyfu
-voyagerx.com
-webmeup-crawler
-webpros.com
-xpymep1.exe
-zgrab
+trendiction.com
+trendiction.de
+ubermetrics-technologies.com
+x22Mozilla