V4.2021.11.2689 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-80legs
+ADmantX
+AIBOT
 ALittle Client
 ASPSeek
+Aboundex
 Aboundexbot
 Acunetix
-AhrefsBot
-AiHitBot
+AfD-Verbotsverfahren
+Alexibot
 Anarchy
-Ankit
 Anthill
-Attach
+Asterias
+AwarioRssBot
 BBBike
-BDFetch
+BLEXBot
+BackStreet
 BackWeb
 BacklinkCrawler
 Badass
-Black Hole
+Bandit
+BetaBot
+Blackboard
 Blow
-BlowFish
-Buck
-Buddy
-Bullseye
-BunnySlippers
-Calculon
+BotALot
+BuiltWith
+BuzzSumo
+CCBot
 CazoodleBot
-Cegbfeieh
-CheTeam
-CherryPicker
+CensysInspect
+CheeseBot
 Chlooe
-Cliqzbot
-Cloud mapping
 Cocolyzebot
+Cogentbot
 Collector
-Craftbot
+CopyRightCheck
+Cosmos
 Crawling at Home Project
-CrunchBot
-Curious
-CyotekWebCopy
-DIIbot
+DBLBot
+DSearch
+DTS Agent
 DataCha0s
-Digincore
+Deusu
+Devil
+DigitalPebble
+Dirbuster
+Discobot
+Discoverybot
 Dispatch
-DittoSpyder
-DnBCrawler-Analytics
-DomainAppender
-DomainCrawler
+DomCopBot
+Dotbot
 Download Wonder
-EMail Siphon
 EMail Wolf
-EasyDL
-Ecxi
-EirGrabber
+Ebingbong
+Evil
 Express WebPictures
 ExtLinksBot
 ExtractorPro
-Ezooms
-FHscan
-FemtosearchBot
-FlashGet
-Flunky
-Foobot
-Fyrebot
-Gigabot
-Go!Zilla
+FDM
+Freeuploader
+Fuzz
+FyberSpider
+GermCrawler
+GetRight
+Getintent
 GrabNet
-Grafula
+Grabber
+GrapeFX
+GridBot
 HEADMasterSEO
-HTTrack
-Havij
+HMView
+HTTP::Lite
+Haansoft
+HaosouSpider
+Harvest
+Humanlinks
+HybridBot
+IDBTE4M
 IRLbot
 Iblog
 Id-search
 IlseBot
 Image Fetch
 Image Sucker
-IndeedBot
+Indy Library
+InfoNaviRobot
+InfoTekies
+InterGET
+Internet Ninja
 Iskanie
-IstellaBot
 JOC Web Spider
 JamesBOT
 JennyBot
 JetCar
-Joomla
-Jyxobot
-Kinza
-Larbin
+JikeSpider
+JustView
+Kozmosbot
 LeechGet
-Lftp
+LieBaoFast
 Lightspeedsystems
 LinkScan
-Linkdexbot
+LinkWalker
+Linkbot
 LinkextractorPro
-LinqiaMetadataDownloaderBot
-LinqiaScrapeBot
+LinkpadBot
+Lipperhey Spider
+Litemage_walker
 MFC_Tear_Sample
 MIDown tool
+MJ12bot
 MSIECrawler
-Mag-Net
-Majestic SEO
-Majestic12
+MTRobot
+Magnet
+Mail.RU_Bot
 MarkWatch
-MeanPath Bot
-Mediatoolkitbot
-Microsoft Data Access
+Mass Downloader
+Mata Hari
+MauiBot
+Mb2345Browser
+Metauri
 Microsoft URL Control
+Mister PiX
 Mojeek
-Mojolicious
 Musobot
-NPbot
+Nameprotect
+Navroad
+Nessus
 Net Vampire
-NetMechanic
+NetAnts
+NetZIP
+Netcraft
 Nettrack
-Netvibes
-NextGenSearchBot
-NimbleCrawler
 Nimbostratus
-Ninja
-Octopus
-Offline Explorer
+Nmap
+Nuclei
+OnCrawl
+OpenLinkProfiler
 OpenVAS
-OrangeBot
-PageGrabber
+OrangeSpider
+OutclicksBot
+POE-Component-Client-HTTP
+Page Analyzer
 Panscient
 Papa Foto
-Pavuk
 Petalbot
-PictureFinder
-Piepmatz
-ProWebWalker
-Probethenet
+Picscout
+Pixray
+PleaseCrawl
+Pockey
 Pu_iN
-PxBroker
+Pump
 PyCurl
+Quick-Crawler
 RSSingBot
 RankActive
-RankActiveLinkBot
-RankingBot
+RankFlex
 RankingBot2
-ReGet
-RealDownload
+Re-re
 Reaper
-Recorder
-RedesScrapy
-Ripper
-RocketCrawler
-Rogerbot
+RebelMouse
+RepoMonkey
+SEOkicks
+SEOkicks-Robot
 SEOprofiler
 SEOstats
-SISTRIX
+SMTBot
+SalesIntelligent
 ScanAlert
+Scanbot
 Screaming
-ScrepyBot
-Searchestate
 Semrush
+SentiBot
 Seomoz
 Shodan
-Site Sucker
+Siphon
 SiteCheckerBotCrawler
-SiteLockSpider
-SiteSnagger
-Snake
-Sociscraper
+SiteExplorer
+SiteSucker
+Sitebeam
+Siteimprove
+Sitevigil
+SmartDownload
+SocialRankIOBot
 Sogou web spider
 Sottopop
-SpaceBison
+Spammen
 Spanner
-Sqlworm
-Steeler
+SputnikBot
+Stripper
 Sucker
-SuperBot
+Sucuri
+SuperHTTP
 SurveyBot
-Szukacz
-Teleport
+T8Abot
+TeleportPro
 Telesoft
-Telesphoreo
-Thumbor
-Tracemyfile
-Trendictionbot
+TheNomad
+TightTwatBot
+Trendiction
 True_Robot
-Turingos
-TwengaBot
-Twice
+TurnitinBot
 URLy Warning
 URLy.Warning
-VB Project
-Vacuum
-Vagabondo
-VidibleScraper
+UnisterBot
 Voil
-WASALive-Bot
-WEBDAV
-WPScan
-WWW-Collector-E
-WallpapersHD
-WeSEE
+WWW-Mechanize
+WWW::Mechanize
 Web Auto
-Web Enhancer
-Web Sauger
+Web Collage
+Web Pix
 WebBandit
 WebCollage
-WebCopier
-WebFuck
-WebImageCollector
-WebLeacher
 WebReaper
 WebSauger
-WebStripper
-Webalta
-WebmasterWorldForumBot
+WebWhacker
 Webshag
-WebsiteExtractor
-Whack
 Whatweb
+WinHTTrack
+WiseGuys Robot
 Woobot
-Wotbox
-Wprecon
-Xaldon_WebSpider
-Zade
+YoudaoBot
 Zauba
-Zermelo
-Zitebot
-ZoomBot
-ZyBorg
-archive.org_bot
-arquivo-web-crawler
-backlink-check
-check1.exe
+ZmEu
+arquivo.pt
+autoemailspider
 clark-crawler
-facebookscraper
+cognitiveseo
+crawler.feedback
 heritrix
 instabid
 internetVista monitor
-isitwp.com
-lwp-trivial
+lwp-request
+magpie-crawler
 mediawords
-pcBrowser
-ripz
+muhstik-scan
+oBot
+plumanalytics
+polaris version
+probe-image-size
+s1z.ru
 satoristudio.net
-seoscanners
-serpstatbot
-sexsearcher
+scan.lol
 sitechecker.pro
-siteripz
+sogouspider
 sp_auditbot
-trendiction.de
+tAkeOut
+trendiction.com
 ubermetrics-technologies.com
 voyagerx.com
-x09Mozilla
-x22Mozilla
+xpymep1.exe
+zauba.io