V4.2021.11.2702 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 404checker
+80legs
 AIBOT
-ASPSeek
+Aboundexbot
 Acunetix
-AfD-Verbotsverfahren
-Aipbot
+AhrefsBot
 AlphaBot
 Anarchie
-Anarchy
-Anarchy99
-Apexoo
-Aspiegel
-BBBike
-BDCbot
+BDFetch
 BackDoorBot
 BackStreet
-Backlink-Ceck
 BatchFTP
-BetaBot
-BlackWidow
-BlowFish
+Bigfoot
+Bitacle
+Black Hole
+Blackboard
+Blow
 Boardreader
-Bolt
-Buck
-Buddy
-BuiltWith
-CCBot
-CODE87
+BotALot
+CATExplorador
 CSHttp
-CensysInspect
-CheTeam
+CazoodleBot
+Cegbfeieh
 CherryPicker
-ChinaClaw
 Chlooe
 Cliqzbot
-Cloud mapping
-Collector
-Crescent
+Cogentbot
+Copyscape
+Craftbot
+Crawling at Home Project
 DSearch
 DTS Agent
-DatabaseDriverMysqli
-Deusu
+DataCha0s
+DigitalPebble
 Disco
 Discobot
-Discoverybot
-Dispatch
-DomainAppender
-Drip
-EMail Wolf
-Ecxi
-EirGrabber
+DnBCrawler-Analytics
+DnyzBot
+DomainStatsBot
+Domains Project
+Dotbot
+ECCP/1.0
+Ebingbong
+EroCrawler
+Evil
 ExtLinksBot
-Extreme Picture Finder
+Extractor
 Ezooms
-FDM
+FemtosearchBot
 Fimap
 FlashGet
-Flunky
 Foobot
 Freeuploader
-Fuzz
+FrontPage
 FyberSpider
-G-i-g-a-b-o-t
 GT::WWW
 GalaxyBot
-Genieo
-Go!Zilla
+GetRight
+Getintent
+Gigabot
 Go-Ahead-Got-It
-Gotit
+GrabNet
 Grafula
 GrapeFX
 HTMLparser
-HTTP::Lite
-HaosouSpider
-Harvest
-Image Sucker
+Haansoft
+Humanlinks
+Iblog
+Id-search
+IndeedBot
 Indy Library
 Intelliseek
-InterGET
 InternetSeer
-Iria
 Iskanie
 IstellaBot
 JOC Web Spider
 JamesBOT
-JennyBot
-Jorgee
-Keyword Density
+JustView
+Jyxobot
 Kinza
-LNSpiderguy
-Leap
+Lanshanbot
+Larbin
 LeechFTP
-Lftp
 LibWeb
 Likse
 LinkScan
+LinkWalker
+Linkbot
 LinksManager
 LinqiaMetadataDownloaderBot
-LinqiaScrapeBot
+LinqiaRSSBot
 Lipperhey
 Lipperhey Spider
-Litemage_walker
+Lmspider
 Ltx71
-MIDown tool
+MFC_Tear_Sample
+MJ12bot
 MQQBrowser
 MTRobot
-Majestic12
+Magnet
+Majestic SEO
+Mass Downloader
 Masscan
 MauiBot
 Mb2345Browser
-Metauri
+MeanPath Bot
+Mediatoolkitbot
+MicroMessenger
+Microsoft URL Control
 Mister PiX
-Mojeek
 Mojolicious
-Morfeus Fucking Scanner
-Mozlila
-Msrabot
-NICErsPRO
+NPbot
 Name Intelligence
 Nameprotect
 Navroad
-Net Vampire
-NetAnts
-NetLyzer
-NetSpider
+NearSite
+Needle
+Nessus
+NetZIP
 NextGenSearchBot
-Nibbler
-Niki-bot
 Nikto
 NimbleCrawler
-Nimbostratus
+Ninja
 Nmap
-Octopus
 Offline Explorer
-OnCrawl
-OpenLinkProfiler
-OutclicksBot
+Offline Navigator
+OpenVAS
+Openvas
+OrangeBot
 OutfoxBot
 POE-Component-Client-HTTP
-PageThing.com
-Pandalytics
+PageAnalyzer
 Papa Foto
-Pavuk
 PeoplePal
-Piepmatz
+Petalbot
+Picscout
 Pimonster
-Pockey
 ProPowerBot
 ProWebWalker
+Probethenet
 Pu_iN
 PyCurl
-Quick-Crawler
-RSSingBot
-RankurBot
-Re-re
+QueryN Metasearch
+RankingBot
+ReGet
 RealDownload
 Reaper
 Recorder
-Ripper
-SBIder
+RocketCrawler
 SEOkicks
-SEOprofiler
-SMTBot
-ScanAlert
+SEOkicks-Robot
+SISTRIX
 Scanbot
 ScoutJet
-Scrapy
+Screaming
+ScreenerBot
 ScrepyBot
-SeoSiteCheckup
-Seomoz
-Site Sucker
-SiteCheckerBotCrawler
-SiteExplorer
+SearchmetricsBot
+Semrush
+Siphon
 SiteSucker
+Sitebeam
 Siteimprove
-Sitevigil
-Snake
-Snapbot
-SpaceBison
-SputnikBot
+SlySearch
+Sociscraper
+Sogou web spider
+Spanner
+Spbot
 Sqworm
+Stripper
+Sucker
+SuperBot
+Surfbot
 SurveyBot
-Suzuran
 Swiftbot
-Telesoft
-The Intraformant
-Toata
+Telesphorep
+Toweyabot
+Trendictionbot
+True_Robot
 Turingos
-TurnitinBot
+Typhoeus
+URLy Warning
+UnisterBot
 V-BOT
+VB Project
 VCI
-VeriCiteCrawler
-Virusdie
-VoidEYE
-WBSearchBot
+Voltron
+WASALive-Bot
+WEBDAV
 WPScan
-Wallpapers
-Wallpapers/3.0
-WallpapersHD
-Web Fuck
+WWW-Collector-E
+WWWOFFLE
+Web Enhancer
+Web Fetch
 Web Sucker
-WebBandit
-WebCopier
-WebFuck
+WebAuto
+WebCollage
+WebFetch
+WebImageCollector
+WebPix
 WebReaper
+WebSauger
 WebStripper
+WebSucker
+WebWhacker
 WebZIP
-WebsiteQuester
-Widow
+WebmasterWorldForumBot
+WebsiteExtractor
+Webster
+Whack
+Whacker
+Who.is Bot
 WiseGuys Robot
-Woobot
-Wotbox
-Xenu
+Wonderbot
+Wprecon
+Xaldon WebSpider
+Xaldon_WebSpider
 YoudaoBot
 Zade
 Zermelo
-ZumBot
-autoemailspider
+Zeus
+Zitebot
+ZmEu
+ZyBorg
+arquivo-web-crawler
 backlink-check
-cah.io.community
-check1.exe
 coccocbot-web
-com.plumanalytics
-crawler.feedback
-crawler4j
+cognitiveseo
+crawl.sogou.com
 demandbase-bot
-ips-agent
-lwp-trivial
-magpie-crawler
-meanpathbot
+eCatch
+facebookscraper
+heritrix
+internetVista monitor
+lwp-request
 muhstik-scan
+oBot
+page scorer
 plumanalytics
-satoristudio.net
-scalaj-http
+polaris version
+scan.lol
 seoscanners
-serpstatbot
-sexsearcher
-siteripz
-sp_auditbot
-sysscan
+sogouspider
+spyfu
 tAkeOut
 trendiction.com
-voyagerx.com
-webmeup-crawler
-webprosbot
+trendiction.de
+ubermetrics-technologies.com
+webpros.com
 x09Mozilla
-x22Mozilla
-xpymep1.exe
-zauba.io
-zgrab