V4.2021.12.2752 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
+360Spider
 404enemy
 80legs
-AIBOT
-Aboundex
-Aboundexbot
+ASPSeek
+AfD-Verbotsverfahren
 AhrefsBot
-Aipbot
-Alligator
+Alexibot
+AlphaBot
 Anarchie
+Anarchy99
+Ankit
 Anthill
-Apexoo
-Aspiegel
-BackWeb
-Backlink-Ceck
-Barkrowler
-BetaBot
-BotALot
-Brandprotect
+Asterias
+AwarioRssBot
+AwarioSmartBot
+BDCbot
+Badass
+BatchFTP
+Bigfoot
+BlackWidow
+BlowFish
 Buck
-Buddy
-BuzzSumo
-CATExplorador
-CCBot
-Cegbfeieh
-CherryPicker
-Chlooe
-Claritybot
+BuiltWith
+Bullseye
+Calculon
+CazoodleBot
+CensysInspect
+CheeseBot
+Cliqzbot
+Cocolyzebot
+Cogentbot
+Copier
 CopyRightCheck
 Cosmos
 Craftbot
-Crawling at Home Project
-CrazyWebCrawler
 Crescent
-Curious
+CyotekWebCopy
 DBLBot
 DIIbot
 DSearch
-Deusu
+DataCha0s
 Devil
 Digincore
-Discoverybot
-DnyzBot
-DomCopBot
-DomainAppender
-DomainSigmaCrawler
+Dirbuster
+Discobot
+DnBCrawler-Analytics
 DomainStatsBot
 Dotbot
 Dragonfly
 Drip
-EMail Wolf
-EasyDL
-Ebingbong
-EroCrawler
-Evil
-Exabot
-ExtLinksBot
+ECCP/1.0
+EMail Siphon
+EirGrabber
 Extreme Picture Finder
-EyeNetIE
-FDM
+FlashGet
+Fuzz
 FyberSpider
-G-i-g-a-b-o-t
-GT::WWW
-Genieo
-GoZilla
+Fyrebot
+GalaxyBot
+GermCrawler
+Go!Zilla
+Grabber
 Grafula
-GrapeFX
-GrapeshotCrawler
-HTTrack
-Harvest
-Hloader
+GridBot
+HEADMasterSEO
+HMView
+Haansoft
+Havij
 Humanlinks
-HybridBot
-IDBTE4M
-Image Fetch
-Image Sucker
-IndeedBot
-InfoNaviRobot
-Intelliseek
+InfoTekies
 InterGET
-Internet Ninja
-InternetSeer
+Iria
+Iskanie
 JOC Web Spider
 JamesBOT
-Jbrofuzz
 JetCar
-Jetty
-Jorgee
 Kenjin Spider
 Keyword Density
-LNSpiderguy
-Leap
-LeechGet
-LexiBot
-Lftp
-LieBaoFast
-LinkScan
+Kinza
+Kozmosbot
+LWP::Simple
+Lanshanbot
+LeechFTP
+Libwhisker
+Lightspeedsystems
+Likse
+LinkWalker
+Linkbot
+LinkextractorPro
+LinqiaMetadataDownloaderBot
 LinqiaScrapeBot
 Lipperhey
 Litemage_walker
-Lmspider
 Ltx71
-MSFrontPage
 MTRobot
-Mag-Net
-Majestic-SEO
+Majestic SEO
+Majestic12
+Mass Downloader
+MauiBot
+Mb2345Browser
 MeanPath Bot
 MegaIndex.ru
 Metauri
-MicroMessenger
 Microsoft URL Control
-Minefield
+Mojeek
 Mojolicious
+Morfeus Fucking Scanner
+Mr.4x3
 Msrabot
-NPbot
+NICErsPRO
 Name Intelligence
-Needle
-Net Vampire
-NetAnts
-NetSpider
+Nessus
+NetMechanic
+Netcraft
 NextGenSearchBot
-Nibbler
-Nikto
+Niki-bot
+NimbleCrawler
 Nimbostratus
 Nmap
-Nuclei
+Nutch
+Octopus
+Offline Explorer
 OpenVAS
-OrangeSpider
+OutclicksBot
 PECL::HTTP
-PHPCrawl
-Page Analyzer
-PageAnalyzer
-PageGrabber
 Panscient
-PeoplePal
-Pi-Monster
-PictureFinder
-Piepmatz
 Pixray
+ProPowerBot
+ProWebWalker
+Probethenet
 Psbot
-Pu_iN
-PxBroker
-RankingBot
+RankActiveLinkBot
+Rankivabot
 Re-re
 ReGet
-RealDownload
-Reaper
 Recorder
-RepoMonkey
-Ripper
+RocketCrawler
+Rogerbot
 SBIder
 SEOkicks-Robot
 SEOstats
-SISTRIX
 SMTBot
-Scanbot
-ScoutJet
-Screaming
+ScanAlert
 Searchestate
+SearchmetricsBot
 Semrush
-SemrushBot
-Seomoz
+SentiBot
+SeobilityBot
 Shodan
+Siphon
 Site Sucker
+SiteCheckerBotCrawler
+SiteExplorer
+SiteLockSpider
 SiteSnagger
-Siteimprove
-SlySearch
-SmartDownload
+SiteSucker
+Sitebeam
+Sitevigil
 Snake
 Sociscraper
 Sosospider
-Spanner
-Sqlmap
-Steeler
-Stripper
-Sucker
+Sottopop
+Spammen
+SpankBot
+Spbot
+Spinn3r
+SputnikBot
 Sucuri
-SuperHTTP
-Suzuran
-TeleportPro
+Surfbot
+Szukacz
+T0PHackTeam
+T8Abot
 Telesoft
 Telesphoreo
-Telesphorep
-Thumbor
-TightTwatBot
-Toweyabot
+Titan
 Trendictionbot
-Turnitin
-TurnitinBot
-TwengaBot
-Twice
-URLy Warning
+True_Robot
+Turingos
+UnisterBot
 Upflow
-V-BOT
 VCI
-Vacuum
+Vagabondo
 VelenPublicWebCrawler
 Virusdie
 VoidEYE
-Voltron
-WASALive-Bot
-WEBDAV
+Voil
+WISENutbot
 WPScan
+WWW-Collector-E
+WWW-Mechanize
 WWWOFFLE
 Wallpapers
-WallpapersHD
-WebAuto
-WebBandit
+Wallpapers/3.0
+WeSEE
+Web Auto
+Web Collage
+Web Enhancer
+Web Fetch
+Web Fuck
+Web Pix
 WebCollage
-WebCopier
-WebImageCollector
-WebStripper
+WebEnhancer
+WebFuck
+WebLeacher
+WebPix
+WebReaper
 Webalta
-WebmasterWorldForumBot
 Website Quester
 WebsiteQuester
 Webster
-Whack
-Wotbox
-Xenu
+WinHTTrack
+Wprecon
 Zade
-Zeus
-ZoomBot
-ZoominfoBot
-ZyBorg
-adscanner
+Zauba
+Zermelo
+Zitebot
+ZmEu
+ZumBot
 archive.org_bot
-arquivo.pt
-backlink-check
-cah.io.community
+arquivo-web-crawler
+autoemailspider
 clark-crawler
 coccocbot-web
-com.plumanalytics
-crawler4j
+cognitiveseo
+crawler.feedback
 dataforseo.com
+demandbase-bot
 evc-batch
 facebookscraper
-heritrix
-mediawords
+internetVista monitor
+isitwp.com
+lwp-trivial
+meanpathbot
 muhstik-scan
-polaris version
-seoscanners
-sitechecker.pro
+oBot
+pcBrowser
+probe-image-size
+scan.lol
+seobility
+sexsearcher
 sogouspider
 sp_auditbot
-sysscan
-ubermetrics-technologies.com
+spyfu
+trendiction.de
+webgains-bot
 webpros.com
-x22Mozilla
+webprosbot
+x09Mozilla
 xpymep1.exe
+zgrab