V4.2021.11.2726 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 360Spider
-404checker
+404enemy
 80legs
-AIBOT
-ALittle Client
-Aboundexbot
-AfD-Verbotsverfahren
-AhrefsBot
-AlphaBot
-Anarchie
-Anarchy
+ASPSeek
+Abonti
+Alexibot
+Alligator
 Anarchy99
 Ankit
+Anthill
 Aspiegel
 Attach
-AwarioRssBot
+BBBike
+BDCbot
 BDFetch
 BackStreet
 BacklinkCrawler
-Bandit
+BatchFTP
+Battleztar Bazinga
+BetaBot
+Bigfoot
 Black Hole
 BlackWidow
-Blackboard
-BlowFish
-Boardreader
+Bolt
 BotALot
-Brandwatch
-BuiltWith
+Buck
 BunnySlippers
 BuzzSumo
 CATExplorador
+CCBot
 CODE87
-CSHttp
-CazoodleBot
-CensysInspect
-CheTeam
-CheeseBot
-Collector
+Cegbfeieh
+Claritybot
+Cliqzbot
+Copyscape
 Cosmos
 Crawling at Home Project
-CrazyWebCrawler
 Crescent
 CrunchBot
-Curious
-CyotekWebCopy
-DIIbot
+Custo
+DBLBot
 DTS Agent
-DatabaseDriverMysqli
-Demon
-Digincore
 Dirbuster
-Dispatch
 DittoSpyder
-DnBCrawler-Analytics
-DomCopBot
-DomainStatsBot
-EMail Siphon
+DomainAppender
+DomainCrawler
+DomainSigmaCrawler
+Download Wonder
+Dragonfly
 EasyDL
 Ebingbong
-Ecxi
+EirGrabber
+Exabot
 Express WebPictures
 Extractor
-ExtractorPro
-Extreme Picture Finder
-Freeuploader
-FrontPage
-Fuzz
-GalaxyBot
-Gigabot
-Go!Zilla
-GoZilla
+FDM
+Flunky
+FyberSpider
+Fyrebot
+G-i-g-a-b-o-t
+GermCrawler
+GetWeb
 Gotit
+Grabber
+GrapeFX
 GrapeshotCrawler
-HMView
-Haansoft
-Heritrix
+GridBot
+HEADMasterSEO
+HTMLparser
+Havij
 Hloader
-HybridBot
 IDBTE4M
-IRLbot
 Id-search
-IlseBot
-JamesBOT
-Jbrofuzz
-JennyBot
-JetCar
-Jorgee
-Kozmosbot
-LNSpiderguy
-Lanshanbot
+Image Sucker
+InfoTekies
+Iria
+Iskanie
+IstellaBot
+JOC Web Spider
+Jetty
 Leap
 LeechFTP
-LexiBot
+LeechGet
 LieBaoFast
-Likse
 LinkScan
-LinkpadBot
+LinkWalker
+Linkbot
+LinkextractorPro
+LinksManager
 LinqiaRSSBot
-Lipperhey Spider
+LinqiaScrapeBot
+Litemage_walker
 Lmspider
+MFC_Tear_Sample
 MIIxpc
-MJ12bot
-Mag-Net
-Magnet
 Majestic-SEO
 Majestic12
-Mass Downloader
 Mb2345Browser
 MeanPath Bot
-Metauri
+Meanpathbot
+Mediatoolkitbot
+MicroMessenger
 Microsoft Data Access
-Morfeus Fucking Scanner
-Mr.4x3
+Mister PiX
+Moblie Safari
+Mojolicious
 Msrabot
-Musobot
-NICErsPRO
-NPbot
 Nameprotect
-Navroad
-Needle
-NetMechanic
-NetSpider
+NearSite
+Net Vampire
+NetAnts
+NetLyzer
+NetZIP
 Netcraft
+Nettrack
 Netvibes
-Niki-bot
 Nimbostratus
 Ninja
 Nmap
-Nuclei
-OnCrawl
-OpenLinkProfiler
-OpenVAS
-OrangeBot
-OutclicksBot
+Openfind
 POE-Component-Client-HTTP
 PageAnalyzer
 PageGrabber
-PageScorer
+PageThing.com
+Papa Foto
+Petalbot
+Pi-Monster
+PictureFinder
+Piepmatz
 Pimonster
 Pixray
-ProPowerBot
-ProWebWalker
-Psbot
+PleaseCrawl
 PxBroker
-PyCurl
-RSSingBot
-RankingBot2
-Re-re
-ReGet
-RealDownload
+RankFlex
+RankingBot
+Rankivabot
+Reaper
 RebelMouse
-Ripper
+Recorder
 RocketCrawler
 SBIder
 SEOkicks
-SEOlyticsCrawler
-SEOprofiler
-SEOstats
-SISTRIX
-SMTBot
-ScoutJet
-ScrepyBot
-SearchmetricsBot
-SemrushBot
+Scanbot
+Screaming
+ScreenerBot
 SeoSiteCheckup
 Siphon
-SiteLockSpider
+SiteCheckerBotCrawler
 SiteSnagger
-SiteSucker
-Sitebeam
-Sitevigil
+Siteimprove
 SmartDownload
 Snake
-Snapbot
 Snoopy
-Sociscraper
 Sogou web spider
+Sosospider
 Sottopop
 SpaceBison
-Spammen
-Spanner
+Spbot
+Spinn3r
 SputnikBot
-Stripper
 Sucker
+SuperBot
+SuperHTTP
+Surfbot
+SurveyBot
 Suzuran
+Swiftbot
+Szukacz
 T0PHackTeam
+Teleport
+TeleportPro
 Telesoft
 Telesphorep
-The Intraformant
-Toata
-Tracemyfile
+Thumbor
 Turingos
-TurnitinBot
-Twice
+Turnitin
+TwengaBot
+URLy Warning
 URLy.Warning
-V-BOT
+Upflow
+VB Project
+VCI
+Vacuum
 Vagabondo
-VelenPublicWebCrawler
 VeriCiteCrawler
-Virusdie
-Voil
+VoidEYE
+Voltron
 WASALive-Bot
-WPScan
-WallpapersHD
-WeSEE
+WEBDAV
+WWW-Mechanize
+WWW::Mechanize
+Wallpapers
 Web Auto
 Web Collage
 Web Enhancer
 Web Fuck
-Web Pix
+Web Sauger
+Web Sucker
 WebBandit
 WebCollage
-WebPix
-WebSucker
+WebGo IS
+WebReaper
+WebSauger
+WebWhacker
+Website Quester
 WebsiteQuester
 Webster
 Whack
-Whacker
 Whatweb
-Who.is Bot
-Wonderbot
+WinHTTrack
+WiseGuys Robot
 Wotbox
-Xaldon WebSpider
 Xaldon_WebSpider
+Zauba
 Zeus
+ZmEu
+ZoomBot
+ZumBot
 ZyBorg
+adscanner
+archive.org_bot
+arquivo-web-crawler
 autoemailspider
+backlink-check
+cah.io.community
+coccocbot-web
+com.plumanalytics
 crawler.feedback
-evc-batch
-facebookscraper
+crawler4j
+dataforseo.com
 gopher
-instabid
+heritrix
+internetVista monitor
+isitwp.com
+lwp-trivial
 magpie-crawler
-mediawords
+meanpathbot
 muhstik-scan
-netEstate NE Crawler
 pcBrowser
 plumanalytics
-ripz
-s1z.ru
-seocompany.store
-seostar
+polaris version
+satoristudio.net
+sitechecker.pro
+sogouspider
+sp_auditbot
 trendiction.com
-trendiction.de
-ubermetrics-technologies.com
-voyagerx.com
 webmeup-crawler
+webpros.com
 webprosbot
-x22Mozilla
 zauba.io