V4.2021.09.2571 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-360Spider
-404enemy
+01h4x.com
+404checker
 ALittle Client
-Aboundex
-AhrefsBot
+ASPSeek
+Acunetix
 AiHitBot
-Aipbot
 AllSubmitter
 AlphaBot
-Anarchie
-Anarchy99
-Ankit
+Anarchy
 Anthill
-Apexoo
-Aspiegel
-Attach
-AwarioRssBot
-BLEXBot
 BackDoorBot
-BackWeb
-Badass
-Battleztar Bazinga
-Bigfoot
-Blackboard
-Boardreader
+BackStreet
+BacklinkCrawler
+Bandit
+Barkrowler
+BlackWidow
+Blow
 Bolt
-Brandprotect
+Brandwatch
 Buddy
-BuiltBotTough
-BuiltWith
-CATExplorador
-CCBot
-CSHttp
+BunnySlippers
+Calculon
+CazoodleBot
+Cegbfeieh
 CensysInspect
 CherryPicker
-ChinaClaw
 Chlooe
 Claritybot
+Cliqzbot
 Cloud mapping
-Cocolyzebot
 Collector
-Cosmos
-Crawling at Home Project
+Copyscape
 CrunchBot
 Curious
-CyotekWebCopy
-DSearch
-DataCha0s
-Demon
-DigitalPebble
-Disco
+DBLBot
+Digincore
+Dispatch
 DittoSpyder
+DnyzBot
+DomCopBot
 DomainAppender
-Dragonfly
-Drip
+DomainStatsBot
+Domains Project
+Dotbot
+Download Wonder
+EMail Wolf
+EasyDL
 EirGrabber
-Extreme Picture Finder
-FDM
+Evil
+Exabot
+ExtLinksBot
+ExtractorPro
+EyeNetIE
+FHscan
 FemtosearchBot
+FlashGet
+Flunky
+Foobot
 Freeuploader
+FrontPage
 Fyrebot
+G-i-g-a-b-o-t
 GT::WWW
-GalaxyBot
-GermCrawler
+Genieo
 GetRight
-GetWeb
-Gigabot
+Getintent
+Go-Ahead-Got-It
 Gotit
 GrabNet
-Grafula
-GrapeFX
-HTTP::Lite
-Heritrix
+Grabber
+GridBot
+HEADMasterSEO
+HTTrack
+HaosouSpider
+Harvest
 Humanlinks
 IDBTE4M
+IDBot
+IRLbot
 Iblog
 IlseBot
 Image Fetch
-Image Sucker
-InfoNaviRobot
-InfoTekies
+Indy Library
+Intelliseek
+Internet Ninja
 InternetSeer
-Iria
 Iskanie
 JOC Web Spider
-Jetty
-Jorgee
+JamesBOT
+JustView
 Kenjin Spider
 Keyword Density
-Kinza
-Leap
-LeechFTP
-Lftp
+Kozmosbot
+LNSpiderguy
+Larbin
+LibWeb
 Libwhisker
 LieBaoFast
-LinqiaMetadataDownloaderBot
-Lipperhey Spider
+LinkScan
+LinkWalker
+Linkbot
+LinkpadBot
+LinqiaScrapeBot
+Lipperhey
+Litemage_walker
 Ltx71
-MIDown tool
-MQQBrowser
-MSIECrawler
-MTRobot
-Mail.RU_Bot
+MFC_Tear_Sample
+Mag-Net
 Majestic SEO
 Majestic12
-MarkWatch
-MauiBot
-Mediatoolkitbot
+Masscan
+MeanPath Bot
 MegaIndex.ru
-Metauri
-MicroMessenger
+Moblie Safari
+Mojolicious
 Mozlila
-Mr.4x3
 Msrabot
 Musobot
-NPbot
+NICErsPRO
+NearSite
+Needle
 Nessus
-NetZIP
+NetMechanic
 Netcraft
-Nettrack
 Netvibes
-NextGenSearchBot
 Niki-bot
 NimbleCrawler
+Nimbostratus
 Nuclei
 Octopus
+Offline Explorer
+Offline Navigator
 OpenVAS
-Openfind
+Openvas
+OrangeBot
 OrangeSpider
+PECL::HTTP
 PHPCrawl
-Page Analyzer
+POE-Component-Client-HTTP
 PageAnalyzer
-PeoplePal
+PageGrabber
+PageScorer
+PageThing.com
+Pandalytics
+Panscient
+Pavuk
+Pi-Monster
 Picsearch
-PictureFinder
-Pu_iN
-QueryN Metasearch
-RankActiveLinkBot
-RankingBot
-Rankivabot
-Re-re
-ReGet
-RebelMouse
-RepoMonkey
-Ripper
-SBIder
+PleaseCrawl
+Pockey
+ProWebWalker
+Probethenet
+Psbot
+Pump
+PyCurl
+Quick-Crawler
+RSSingBot
+RankingBot2
+Reaper
+Recorder
+RocketCrawler
 SEOkicks
-SEOkicks-Robot
-SEOstats
+SISTRIX
+SMTBot
 SalesIntelligent
-Scanbot
-ScoutJet
+Scrapy
 SearchmetricsBot
 Semrush
 SemrushBot
-SeoSiteCheckup
-SeobilityBot
 Seomoz
-Shodan
-Siphon
-SiteLockSpider
-SiteSnagger
-SiteSucker
+Site Sucker
 Sitebeam
-Sitevigil
-SlySearch
-SmartDownload
-Snoopy
-SocialRankIOBot
+Siteimprove
+Sociscraper
 Sogou web spider
 Sosospider
+Spammen
 SpankBot
-Spanner
-Spinn3r
-Sucker
-SuperBot
-Suzuran
-T8Abot
-TeleportPro
-The Intraformant
-Titan
-Toweyabot
+Spbot
+SputnikBot
+Sqworm
+Steeler
+SurveyBot
+Swiftbot
+Teleport
+Telesphoreo
+TheNomad
+Thumbor
+Toata
 Tracemyfile
-Trendictionbot
-Turingos
+True_Robot
+Turnitin
 TurnitinBot
-Twice
+Typhoeus
 URLy.Warning
-Upflow
+Vacuum
 Vagabondo
+VelenPublicWebCrawler
+VidibleScraper
 Voil
-WBSearchBot
+WASALive-Bot
 WEBDAV
-Wallpapers
-Wallpapers/3.0
-WeSEE
-Web Auto
-Web Fuck
-Web Pix
+WPScan
+WWW-Collector-E
+Web Enhancer
 Web Sucker
-WebAuto
-WebCollage
-WebCopier
-WebEnhancer
-WebFetch
-WebPix
+WebGo IS
 WebReaper
 WebSucker
 WebWhacker
-WebZIP
-WebsiteExtractor
-WebsiteQuester
 Whacker
-Whatweb
-WinHTTrack
-WiseGuys Robot
-Wotbox
-Xaldon WebSpider
-Xaldon_WebSpider
-Zeus
+Who.is Bot
+Widow
+Wonderbot
 Zitebot
 ZmEu
-ZoomBot
-archive.org_bot
+ZoominfoBot
+ZumBot
+ZyBorg
 arquivo.pt
-backlink-check
+autoemailspider
 cah.io.community
-crawl.sogou.com
+check1.exe
+clark-crawler
+coccocbot-web
 crawler4j
+dataforseo.com
+demandbase-bot
+eCatch
 ips-agent
-isitwp.com
+lwp-request
+lwp-trivial
+magpie-crawler
 meanpathbot
-mediawords
 muhstik-scan
-oBot
+page scorer
+pcBrowser
+plumanalytics
 polaris version
-probe-image-size
 ripz
-s1z.ru
-scan.lol
-seobility
-seoscanners
-sitechecker.pro
+satoristudio.net
 siteripz
-sogouspider
+spyfu
 sysscan
+trendiction.de
+voyagerx.com
 webpros.com
 webprosbot
 x09Mozilla
-zauba.io
+zgrab