V4.2022.04.3057 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
 360Spider
 404enemy
+80legs
 ADmantX
 ALittle Client
-ASPSeek
-Abonti
-Aboundex
 Aboundexbot
-AfD-Verbotsverfahren
-AhrefsBot
-AiHitBot
 Aipbot
+Alligator
 AlphaBot
-Asterias
-Attach
-AwarioSmartBot
+Anarchie
+Anarchy99
+Ankit
+Anthill
+Atomseobot
+BDCbot
 BDFetch
-BackDoorBot
-BackStreet
 Backlink-Ceck
-BacklinkCrawler
+Bandit
+Barkrowler
 BetaBot
-Bitacle
-Blackboard
-Blow
-BlowFish
+BlackWidow
 Boardreader
-Buck
+BotALot
+Brandwatch
+Buddy
 BuiltBotTough
 BuiltWith
-Bullseye
 BunnySlippers
 BuzzSumo
-CATExplorador
+CCBot
 CODE87
-Calculon
+CazoodleBot
+Cegbfeieh
+CheTeam
 CheeseBot
-CherryPicker
-Chlooe
-Citoid
-Cliqzbot
-Cloud mapping
-Cocolyzebot
-CopyRightCheck
+Copier
 Copyscape
-Crescent
-Curious
+Craftbot
+Custo
+DBLBot
 DIIbot
+DSearch
+DataCha0s
 Deusu
-Digincore
 DigitalPebble
-Disco
+Discobot
+Discoverybot
+Dispatch
 DnBCrawler-Analytics
+DnyzBot
 DomainAppender
-DomainSigmaCrawler
+DomainCrawler
 DomainStatsBot
 Domains Project
-Dotbot
-Download Wonder
-EMail Siphon
-Ecxi
-EirGrabber
+Drip
+EasyDL
 EroCrawler
-Express WebPictures
-Extractor
-Extreme Picture Finder
-Ezooms
-FDM
-Freeuploader
-Fuzz
+ExtLinksBot
+FlashGet
+FrontPage
 Fyrebot
-GT::WWW
-GermCrawler
-Getintent
-GrabNet
+G-i-g-a-b-o-t
+GetWeb
+Gigabot
+Gotit
 Grabber
-Grafula
-GrapeshotCrawler
-HMView
-Haansoft
+GridBot
+HTMLparser
+HTTrack
+HaosouSpider
 Harvest
-Havij
+Heritrix
+HonoluluBot
 Humanlinks
 Iblog
+Id-search
+Image Sucker
 Indy Library
-InfoNaviRobot
 InfoTekies
-Intelliseek
 InterGET
-Internet Ninja
-InternetSeer
 Iskanie
-IstellaBot
 JamesBOT
+Jbrofuzz
+Jetty
+JikeSpider
+Joomla
+Jorgee
 JustView
-Kenjin Spider
+Jyxobot
 Keybot Translation-Search-Machine
-LNSpiderguy
-LWP::Simple
-Lanshanbot
+Keyword Density
+Kinza
 LexiBot
 Lftp
 LibWeb
-Libwhisker
-Lightspeedsystems
-LinkextractorPro
+LieBaoFast
 LinkpadBot
-LinqiaScrapeBot
-Lmspider
+LinksManager
+Lipperhey Spider
+Litemage_walker
 MIIxpc
 MSFrontPage
-Mag-Net
 Mail.RU_Bot
-Mass Downloader
-Mata Hari
+Majestic-SEO
+Majestic12
+Masscan
 MauiBot
 Mb2345Browser
-Meanpathbot
 MegaIndex.ru
 Metauri
-Moblie Safari
-Mojeek
+MicroMessenger
+Microsoft URL Control
+Minefield
+Mojolicious
 MolokaiBot
-Msrabot
+Mozlila
+Mr.4x3
+Musobot
 NICErsPRO
-NPbot
-Navroad
+Nameprotect
+Needle
+Net Vampire
 NetAnts
-NetMechanic
-NetSpider
-Netcraft
+NetZIP
+Nettrack
+Netvibes
+NextGenSearchBot
+Niki-bot
+NimbleCrawler
+Nimbostratus
 Nuclei
-Octopus
+Nutch
+Offline Explorer
+OnCrawl
+OpenLinkProfiler
 Openfind
-OrangeBot
+Openvas
+OrangeSpider
 OutclicksBot
-POE-Component-Client-HTTP
+OutfoxBot
 Pandalytics
 Panscient
-Papa Foto
-Pavuk
 PeoplePal
-Pi-Monster
-Picsearch
-Piepmatz
+Petalbot
 ProWebWalker
 Probethenet
-Psbot
-Pu_iN
-QueryN Metasearch
-Quick-Crawler
-RankActiveLinkBot
+Pump
+RankingBot
 RankingBot2
-ReGet
-RealDownload
+RankurBot
+Reaper
 RebelMouse
-RedesScrapy
-RepoMonkey
-SBIder
-SISTRIX
+Rogerbot
+SEOkicks-Robot
+SEOprofiler
 ScanAlert
-Scanbot
 ScoutJet
-SemanticJuice
-SemrushBot
+Scrapy
+Screaming
+Seekport
+SentiBot
 Siphon
-SiteExplorer
-SiteLockSpider
-Sitebeam
+Siteimprove
+Snake
+SocialRankIOBot
 Sociscraper
-Sottopop
-SputnikBot
+SpankBot
+Sqlmap
 Steeler
-Sucker
-Sucuri
+SuperHTTP
+SurveyBot
 Suzuran
-Swiftbot
-T0PHackTeam
+Szukacz
 T8Abot
-Telesphoreo
+Telesoft
 The Intraformant
 TheNomad
 Thumbor
 Toata
-Trendiction
-True_Robot
+Trendictionbot
 Turingos
-TurnitinBot
+TwengaBot
 Twice
+Typhoeus
 URLy Warning
-V-BOT
-Vagabondo
+URLy.Warning
+VB Project
+Vacuum
+VelenPublicWebCrawler
 VeriCiteCrawler
+VidibleScraper
+Virusdie
+Voil
 WASALive-Bot
 WBSearchBot
-WWW-Collector-E
-WWWOFFLE
-Wallpapers
+WWW-Mechanize
 WallpapersHD
 WeSEE
-Web Auto
-Web Collage
-Web Sauger
+Web Fetch
+Web Pix
+Web Sucker
 WebAuto
-WebEnhancer
+WebBandit
 WebFetch
 WebGo IS
-WebReaper
-Webalta
+WebSauger
+WebSucker
+WebWhacker
+WebZIP
 WebmasterWorldForumBot
 Webshag
 WebsiteExtractor
 Webster
-Whack
 Whacker
 Whatweb
-Who.is Bot
 Widow
-WinHTTrack
+WiseGuys Robot
 Wonderbot
-Woobot
-Wotbox
 Xaldon WebSpider
-Xaldon_WebSpider
-Zermelo
+YoudaoBot
+Zeus
+Zitebot
 ZyBorg
-arquivo-web-crawler
-cah.io.community
-coccocbot
-com.plumanalytics
+adscanner
+autoemailspider
+backlink-check
+cognitiveseo
+crawler.feedback
 dataforseo.com
-eCatch
-evc-batch
 facebookscraper
 gopher
-iubenda-radar
-linkdexbot
+instabid
+ips-agent
+isitwp.com
+lwp-trivial
 magpie-crawler
+oBot
 page scorer
 plumanalytics
-probe-image-size
-s1z.ru
+ripz
+scan.lol
 seobility
 seostar
-sitechecker.pro
-voyagerx.com
-webpros.com
-x22Mozilla
+siteripz
+sp_auditbot
+spyfu
+webgains-bot
+zauba.io
 zgrab