V4.2021.08.2560 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-360Spider
+404checker
+404enemy
 80legs
-ADmantX
+AIBOT
 Abonti
-Aboundex
-Aboundexbot
+AhrefsBot
+AiHitBot
+Aipbot
 Alexibot
-Alligator
+AllSubmitter
 AlphaBot
-Anthill
-Asterias
-Attach
+Anarchy
+Apexoo
+AwarioRssBot
 BLEXBot
-BacklinkCrawler
-Bandit
-Battleztar Bazinga
+BackStreet
+Backlink-Ceck
+Badass
+Barkrowler
+BatchFTP
 Bigfoot
-BlackWidow
+Blackboard
 BlowFish
-Bolt
-Buddy
-BuiltBotTough
+Brandprotect
+Bullseye
 BunnySlippers
-CATExplorador
-ChinaClaw
-Claritybot
-Cliqzbot
+CODE87
+CSHttp
+Calculon
+Cegbfeieh
+CensysInspect
+CherryPicker
+Chlooe
+Cloud mapping
 Cogentbot
-Collector
-CopyRightCheck
+Copyscape
+Craftbot
+Crescent
 CrunchBot
-Curious
-CyotekWebCopy
-DBLBot
-DIIbot
-DSearch
+DTS Agent
 DataCha0s
 Demon
 Deusu
-DigitalPebble
+Dirbuster
+Discobot
 Dispatch
-DnyzBot
-Dotbot
-Download Wonder
+DittoSpyder
+DomainCrawler
+DomainSigmaCrawler
+DomainStatsBot
 Drip
 ECCP/1.0
 EMail Siphon
+EMail Wolf
 Ebingbong
 Ecxi
-EirGrabber
 EroCrawler
-Evil
-Exabot
-Extractor
-FemtosearchBot
-Freeuploader
-Fuzz
-GalaxyBot
+ExtractorPro
+EyeNetIE
+Foobot
+GT::WWW
 GetRight
 GetWeb
 Gigabot
-Go!Zilla
+Go-Ahead-Got-It
+GoZilla
 Gotit
-GrapeFX
 HEADMasterSEO
-HMView
-Haansoft
+HTMLparser
+HTTP::Lite
+HTTrack
 HaosouSpider
-Harvest
+Havij
+Hloader
 IDBTE4M
-IDBot
-IlseBot
+Id-search
+Image Fetch
+Indy Library
 InfoNaviRobot
-InfoTekies
-InterGET
+Internet Ninja
+Iria
+Iskanie
+IstellaBot
 JOC Web Spider
 JamesBOT
-Jyxobot
+JennyBot
+Jetty
 Kenjin Spider
-Keyword Density
 Kinza
-Larbin
-Leap
-LeechGet
+LNSpiderguy
 LexiBot
-LibWeb
-Linkbot
+LieBaoFast
+Lightspeedsystems
 Linkdexbot
-LinksManager
-LinqiaMetadataDownloaderBot
-LinqiaRSSBot
-Mag-Net
-Mail.RU_Bot
+LinkextractorPro
+LinqiaScrapeBot
+Lipperhey
+Litemage_walker
+Ltx71
+MFC_Tear_Sample
+MJ12bot
+MQQBrowser
+MSFrontPage
+MSIECrawler
+MTRobot
 Majestic-SEO
-Majestic12
 MarkMonitor
-MarkWatch
-Masscan
+Mass Downloader
 Mata Hari
-Mb2345Browser
-MeanPath Bot
-Mediatoolkitbot
-MegaIndex.ru
+MauiBot
+Metauri
 MicroMessenger
 Mister PiX
-Mojeek
+Moblie Safari
 Mozlila
+Mr.4x3
 Msrabot
-Musobot
 NICErsPRO
-NPbot
-Nameprotect
-Navroad
-NearSite
-Nessus
-Net Vampire
-NetLyzer
+Name Intelligence
+Needle
 NetSpider
 NetZIP
-NextGenSearchBot
+Netcraft
+Netvibes
 Nibbler
-NimbleCrawler
-Nimbostratus
-Ninja
 Nmap
 Nutch
 Offline Explorer
 OnCrawl
-OrangeBot
+OpenLinkProfiler
+OpenVAS
+Openfind
 OrangeSpider
 OutclicksBot
-OutfoxBot
-PECL::HTTP
-POE-Component-Client-HTTP
-Page Analyzer
-Pandalytics
+PageThing.com
 Panscient
-Pavuk
-PeoplePal
+Papa Foto
 Petalbot
+Picscout
 PictureFinder
 Pixray
 PleaseCrawl
-ProWebWalker
-Psbot
 Pu_iN
-Pump
 PxBroker
 PyCurl
-Quick-Crawler
-RankActive
+QueryN Metasearch
+RSSingBot
+RankActiveLinkBot
 RankingBot
-Rankivabot
 RankurBot
 Re-re
-ReGet
 RealDownload
 Reaper
-RebelMouse
-RepoMonkey
-Rogerbot
+Ripper
 SEOkicks
-SEOstats
-SMTBot
-Scanbot
+SEOkicks-Robot
+SEOprofiler
 Screaming
-ScreenerBot
-ScrepyBot
-SearchmetricsBot
-SentiBot
+Searchestate
+SemrushBot
 Shodan
 SiteCheckerBotCrawler
 SiteExplorer
-SiteSucker
-Sitevigil
+SiteLockSpider
 SmartDownload
 Snapbot
+Sogou web spider
 Sosospider
+Sottopop
+Spanner
 Spbot
 Spinn3r
-Sqworm
-Sucuri
-SuperHTTP
-Szukacz
+Sqlmap
+Sqlworm
+T0PHackTeam
 Teleport
 TeleportPro
-Telesoft
+Telesphoreo
+Telesphorep
 The Intraformant
-Thumbor
-Titan
-TwengaBot
+TightTwatBot
+Toweyabot
+Tracemyfile
+Trendictionbot
+Turingos
+Turnitin
+Twice
+URLy Warning
 Upflow
-VB Project
-Vacuum
-Vagabondo
-VelenPublicWebCrawler
-VeriCiteCrawler
-VoidEYE
-WBSearchBot
-WEBDAV
-WWW-Mechanize
-WWW::Mechanize
-WeSEE
+V-BOT
+VidibleScraper
+Voil
+WASALive-Bot
+Wallpapers
+Wallpapers/3.0
+WallpapersHD
+Web Auto
+Web Enhancer
+Web Fuck
+Web Pix
 Web Sauger
 Web Sucker
-WebAuto
-WebCollage
-WebGo IS
-WebLeacher
-WebReaper
-WebStripper
+WebBandit
+WebFuck
+WebImageCollector
 WebSucker
 WebZIP
-WebmasterWorldForumBot
-Webshag
-Website Quester
 WebsiteQuester
-Whack
+Webster
+Whatweb
 Who.is Bot
-WiseGuys Robot
+Wonderbot
+Wotbox
 Wprecon
-Zauba
+Xaldon_WebSpider
+Zade
 Zermelo
 Zeus
-ZmEu
-ZoominfoBot
+archive.org_bot
+arquivo.pt
 clark-crawler
-coccocbot-web
-com.plumanalytics
+cognitiveseo
 crawl.sogou.com
-crawler4j
+crawler.feedback
+dataforseo.com
 demandbase-bot
-gopher
-instabid
-ips-agent
+domainsproject.org
+heritrix
+internetVista monitor
 isitwp.com
+lwp-trivial
 magpie-crawler
-meanpathbot
-oBot
+mediawords
+page scorer
 pcBrowser
-polaris version
-probe-image-size
 ripz
-s1z.ru
+satoristudio.net
+scalaj-http
+seobility
+serpstatbot
+sexsearcher
 sysscan
 tAkeOut
-trendiction.de
-voyagerx.com
+trendiction.com
+webpros.com
+webprosbot
+x09Mozilla
 x22Mozilla
+zauba.io