V4.2021.10.2680 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
+360Spider
 404checker
-AIBOT
+ADmantX
+Abonti
+AfD-Verbotsverfahren
+AhrefsBot
+AiHitBot
 Aipbot
 Alexibot
 AllSubmitter
+Alligator
+Ankit
 Anthill
 Asterias
 Attach
-BDFetch
+AwarioRssBot
+AwarioSmartBot
 BLEXBot
-BacklinkCrawler
-Battleztar Bazinga
+BackDoorBot
+Backlink-Ceck
+Bandit
+Bigfoot
 Black Hole
 Blow
 BotALot
-Brandwatch
-BuiltWith
-BunnySlippers
-BuzzSumo
-CATExplorador
-CCBot
-CazoodleBot
-CheTeam
-Chlooe
-Cliqzbot
-Cloud mapping
-Cogentbot
+Brandprotect
+Buddy
+CODE87
+CherryPicker
+ChinaClaw
 Collector
-Copier
 CopyRightCheck
-Copyscape
-Craftbot
-Custo
+Crawling at Home Project
+CrazyWebCrawler
+Crescent
+CrunchBot
 CyotekWebCopy
 DBLBot
-Deusu
-Dirbuster
-Disco
-DomCopBot
-DomainStatsBot
-Domains Project
-Dragonfly
+DIIbot
+DataCha0s
+Devil
+Dispatch
+DittoSpyder
+DnyzBot
+DomainAppender
+DomainSigmaCrawler
 Drip
-EasyDL
+EMail Wolf
+Ebingbong
+Ecxi
 EirGrabber
-EroCrawler
-Extractor
+Evil
+Express WebPictures
 ExtractorPro
 Extreme Picture Finder
-FDM
+EyeNetIE
+FemtosearchBot
 Firefox/7.0
-Freeuploader
-Fuzz
-FyberSpider
+FrontPage
+G-i-g-a-b-o-t
 Genieo
+GetRight
 Gigabot
-Go!Zilla
 Go-Ahead-Got-It
 GoZilla
+GrabNet
 Grabber
 GrapeFX
 GrapeshotCrawler
 GridBot
-HMView
-HTTrack
-Harvest
-Havij
-Heritrix
-Hloader
-IRLbot
+HaosouSpider
+HybridBot
+Iblog
 IlseBot
-InfoNaviRobot
+IndeedBot
+Intelliseek
+InterGET
 Internet Ninja
-InternetSeer
+Iria
 IstellaBot
 JOC Web Spider
+Jbrofuzz
 JetCar
-Jetty
-JikeSpider
+Joomla
 Jorgee
-LWP::Simple
-Lanshanbot
+Keyword Density
+Kinza
+LNSpiderguy
+LeechFTP
 LeechGet
 LibWeb
-LieBaoFast
-Likse
-LinkWalker
-LinkpadBot
-LinqiaMetadataDownloaderBot
-LinqiaScrapeBot
-Lipperhey
-Lipperhey Spider
+LinkScan
+Linkdexbot
+LinkextractorPro
 Litemage_walker
-MIDown tool
-MQQBrowser
-Mag-Net
+Lmspider
+Ltx71
 Mail.RU_Bot
-MarkMonitor
-MarkWatch
-Mediatoolkitbot
-Microsoft URL Control
+Majestic SEO
+Majestic12
+MauiBot
+Mb2345Browser
+MeanPath Bot
+MegaIndex.ru
+Metauri
+Microsoft Data Access
+Mister PiX
 Moblie Safari
-Mojolicious
-Morfeus Fucking Scanner
-Mr.4x3
+Musobot
+NICErsPRO
 NPbot
 Name Intelligence
+Navroad
 NearSite
-Nessus
 NetMechanic
-Niki-bot
-NimbleCrawler
+Netcraft
+Nettrack
+Nibbler
 Nimbostratus
-Nutch
-Octopus
+Nmap
+Nuclei
+Offline Explorer
 OnCrawl
-Openfind
-OrangeBot
 OutfoxBot
-PECL::HTTP
 POE-Component-Client-HTTP
 PageAnalyzer
 PageGrabber
-PageThing.com
-Pandalytics
-PeoplePal
-Petalbot
+PageScorer
+Piepmatz
 Pimonster
+Pixray
+Pockey
 ProPowerBot
-ProWebWalker
-Psbot
-Pu_iN
-PxBroker
-PyCurl
+Probethenet
+Pump
+Quick-Crawler
 RSSingBot
+RankActive
 RankFlex
 RankingBot
-Rankivabot
-Re-re
-RealDownload
+RankingBot2
+RankurBot
+ReGet
 RebelMouse
 Recorder
-RocketCrawler
 Rogerbot
-SEOkicks
 SEOkicks-Robot
-SEOstats
-SMTBot
-ScoutJet
-Screaming
-SearchmetricsBot
-SemrushBot
-SeobilityBot
+SEOprofiler
+ScanAlert
+Scanbot
+ScreenerBot
+Searchestate
+SentiBot
 Seomoz
 Shodan
-SiteLockSpider
+SiteSucker
 Sitebeam
 Siteimprove
 Sitevigil
-Snake
-Snapbot
-Sociscraper
+SocialRankIOBot
+Sogou web spider
+Sosospider
 Sottopop
 Spammen
+Spanner
 Spbot
-SputnikBot
-Sqlmap
-Sqlworm
 Steeler
 Stripper
-Sucuri
-SuperBot
-SuperHTTP
 Surfbot
-Szukacz
-T0PHackTeam
+Suzuran
+Swiftbot
 TeleportPro
-Telesphoreo
-The Intraformant
-TheNomad
-Titan
-Toata
+Telesoft
+Thumbor
+Tracemyfile
 Trendiction
-Turingos
-Turnitin
-TurnitinBot
-Typhoeus
-URLy.Warning
+Trendictionbot
+TwengaBot
 VB Project
 VCI
+Vacuum
+Vagabondo
+VelenPublicWebCrawler
 VeriCiteCrawler
-Virusdie
+VidibleScraper
 VoidEYE
-Voltron
-WASALive-Bot
+Voil
+WBSearchBot
+WEBDAV
+WISENutbot
 WPScan
-WWW-Mechanize
-Wallpapers
 Wallpapers/3.0
+WallpapersHD
+WeSEE
+Web Auto
 Web Collage
-Web Fetch
-WebAuto
-WebImageCollector
+Web Enhancer
+Web Sucker
+WebCopier
+WebEnhancer
+WebFuck
+WebPix
 WebStripper
 WebSucker
-WebWhacker
 WebZIP
-WebmasterWorldForumBot
-Webshag
+Webalta
 WebsiteQuester
-Webster
+Who.is Bot
 WinHTTrack
-Xaldon_WebSpider
-Xenu
+WiseGuys Robot
+Xaldon WebSpider
 YoudaoBot
-ZoomBot
+Zade
+Zeus
+ZoominfoBot
+ZumBot
 ZyBorg
-archive.org_bot
+arquivo-web-crawler
 arquivo.pt
 clark-crawler
-cognitiveseo
-com.plumanalytics
-dataforseo.com
+crawl.sogou.com
+crawler4j
 demandbase-bot
 evc-batch
-gopher
+heritrix
+instabid
 internetVista monitor
-ips-agent
 lwp-request
-magpie-crawler
-mediawords
+lwp-trivial
 muhstik-scan
-page scorer
+netEstate NE Crawler
+pcBrowser
 polaris version
 probe-image-size
 ripz
-satoristudio.net
-scalaj-http
 scan.lol
 seobility
-sogouspider
+seostar
+serpstatbot
+sexsearcher
 spyfu
 tAkeOut
-ubermetrics-technologies.com
+trendiction.com
+trendiction.de
 voyagerx.com
-webprosbot
+webmeup-crawler
+webpros.com
+x22Mozilla
 xpymep1.exe
 zauba.io
-zgrab