V4.2021.09.2597 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-AIBOT
+360Spider
+ADmantX
 ALittle Client
+ASPSeek
+Abonti
+Aboundex
 Aboundexbot
 Acunetix
-Aipbot
+AllSubmitter
 Alligator
 AlphaBot
 Anarchy
 Anarchy99
-Ankit
 Anthill
 Aspiegel
 Asterias
 Attach
 AwarioRssBot
-AwarioSmartBot
 BBBike
 BDCbot
+BDFetch
 BLEXBot
 BacklinkCrawler
 Badass
 Bandit
-Barkrowler
-BatchFTP
-BlackWidow
+Bigfoot
+Bitacle
+Black Hole
+BlowFish
 BotALot
-Buck
+Brandprotect
+BuiltWith
+Bullseye
 BunnySlippers
-BuzzSumo
+CATExplorador
+CCBot
+Calculon
+Cegbfeieh
 CensysInspect
+CheTeam
+CheeseBot
 CherryPicker
-Cliqzbot
+ChinaClaw
+Cloud mapping
+Cocolyzebot
 Copier
 CopyRightCheck
+Cosmos
 Craftbot
 Crawling at Home Project
-CrazyWebCrawler
-CrunchBot
+Crescent
+DataCha0s
 DatabaseDriverMysqli
 Demon
 Devil
-Disco
-Discoverybot
-DomainCrawler
+DigitalPebble
+Discobot
+Dispatch
+DnyzBot
 DomainStatsBot
-Domains Project
-Dotbot
-Dragonfly
-EMail Siphon
+Download Wonder
 EMail Wolf
-Evil
+EasyDL
+Ebingbong
+EirGrabber
 Exabot
-Express WebPictures
+ExtLinksBot
+ExtractorPro
 EyeNetIE
+FHscan
+Fimap
+Firefox/7.0
+Foobot
 Freeuploader
-FrontPage
-Fyrebot
+Fuzz
+GT::WWW
+GalaxyBot
 GetWeb
-Gigabot
 Go!Zilla
-Grabber
 Grafula
-GrapeFX
+GridBot
+HEADMasterSEO
 HMView
-HTTrack
-Haansoft
-HybridBot
-IDBot
+HTTP::Lite
+Heritrix
+Humanlinks
 IRLbot
 Iblog
+Id-search
+IlseBot
 IndeedBot
-Indy Library
 InfoNaviRobot
+InfoTekies
 InterGET
-Internet Ninja
-Iskanie
+InternetSeer
+IstellaBot
 JOC Web Spider
-Jbrofuzz
-Jetty
-JikeSpider
+JamesBOT
+JetCar
 Jorgee
+JustView
+Jyxobot
 Kenjin Spider
-Keyword Density
-Kinza
-LNSpiderguy
-Larbin
-Leap
+LWP::Simple
+Lanshanbot
 LibWeb
-Lightspeedsystems
 Likse
-LinkScan
-LinkWalker
-Linkbot
-LinkextractorPro
-LinkpadBot
-LinqiaScrapeBot
-MIIxpc
+Linkdexbot
+LinksManager
+LinqiaRSSBot
+Lmspider
+MIDown tool
 MJ12bot
-MSFrontPage
 MTRobot
-Magnet
+Mag-Net
+MarkMonitor
 MarkWatch
-Mata Hari
-MauiBot
+Mass Downloader
+Mediatoolkitbot
+MegaIndex.ru
+MicroMessenger
+Moblie Safari
+Mojeek
 Mozlila
-Msrabot
-Musobot
-NPbot
+NICErsPRO
+Name Intelligence
 Nameprotect
-Nessus
+Navroad
+NearSite
+Needle
 NetAnts
 NetLyzer
-NetMechanic
-NetZIP
-Netcraft
-Netvibes
-NimbleCrawler
-Nimbostratus
-Ninja
-Nuclei
+NetSpider
+Nmap
+Octopus
+Offline Explorer
 OnCrawl
+OpenLinkProfiler
 OpenVAS
 Openfind
+OutfoxBot
 PECL::HTTP
 PHPCrawl
-Page Analyzer
-PageAnalyzer
-PageThing.com
-Panscient
-Pavuk
-Petalbot
-Pi-Monster
 Picscout
-PictureFinder
-ProWebWalker
+Pimonster
+PleaseCrawl
+Pockey
+Probethenet
 Pump
-QueryN Metasearch
-RankActive
+Quick-Crawler
+RSSingBot
 RankActiveLinkBot
-RankFlex
-RankingBot2
-Re-re
-ReGet
+RankurBot
 RealDownload
+Reaper
+Recorder
+RepoMonkey
 Ripper
-SBIder
 SEOkicks
-SEOkicks-Robot
-SMTBot
+SEOlyticsCrawler
+SalesIntelligent
 Scanbot
-ScoutJet
-Screaming
-ScrepyBot
-Semrush
-SemrushBot
-SentiBot
+SearchmetricsBot
 SeoSiteCheckup
-SeobilityBot
-SiteCheckerBotCrawler
-SiteSucker
-Siteimprove
-Sitevigil
-SlySearch
-Snake
-Snapbot
-Sociscraper
-Sogou web spider
-SpankBot
+SiteSnagger
+Sitebeam
+SocialRankIOBot
+SpaceBison
 Spbot
-Spinn3r
-Sqlworm
 Steeler
-Sucuri
+Stripper
+Sucker
+SuperBot
 SuperHTTP
-Surfbot
-SurveyBot
-T8Abot
+Swiftbot
+Szukacz
 Teleport
-Telesoft
-Telesphoreo
+The Intraformant
 TheNomad
 Thumbor
-TightTwatBot
-Trendiction
-Trendictionbot
+Toata
+Tracemyfile
+True_Robot
 Turingos
-Turnitin
-TwengaBot
 Twice
 Typhoeus
-URLy.Warning
-Upflow
+URLy Warning
+UnisterBot
 V-BOT
-VB Project
-Vacuum
+VCI
+VidibleScraper
 Voil
-Voltron
 WASALive-Bot
+WEBDAV
+WPScan
 WWW-Collector-E
-WWW::Mechanize
-Wallpapers
+WWW-Mechanize
+WWWOFFLE
+WallpapersHD
 WeSEE
-Web Auto
-Web Collage
+Web Enhancer
 Web Fetch
 Web Fuck
-Web Sauger
-Web Sucker
-WebAuto
+Web Pix
 WebCopier
+WebFetch
 WebFuck
 WebLeacher
-WebPix
-WebReaper
-Whatweb
+WebSucker
+WebZIP
+WebsiteExtractor
 WiseGuys Robot
 Wonderbot
-Woobot
+Xaldon WebSpider
+Xaldon_WebSpider
 Xenu
 YoudaoBot
-Zade
+Zauba
 Zermelo
 Zitebot
 ZmEu
 ZoomBot
-ZyBorg
+ZoominfoBot
+ZumBot
+arquivo-web-crawler
+autoemailspider
+cah.io.community
+clark-crawler
 coccocbot-web
+com.plumanalytics
+crawler.feedback
+dataforseo.com
+demandbase-bot
+eCatch
 evc-batch
-gopher
-internetVista monitor
-magpie-crawler
-meanpathbot
-muhstik-scan
-netEstate NE Crawler
+heritrix
+instabid
 oBot
-polaris version
 probe-image-size
-scalaj-http
+s1z.ru
 seobility
-seoscanners
-serpstatbot
 sexsearcher
-sogouspider
+sitechecker.pro
+siteripz
+spyfu
 tAkeOut
-trendiction.com
-ubermetrics-technologies.com
-webmeup-crawler
-x22Mozilla
+webpros.com
+xpymep1.exe
 zauba.io
+zgrab