V4.2021.09.2590 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-360Spider
+404enemy
 80legs
+ALittle Client
 ASPSeek
-Aboundex
-Acunetix
-AiHitBot
-Alexibot
-Alligator
+AhrefsBot
+AlphaBot
 Anarchie
-Asterias
-Attach
-BDCbot
-BDFetch
-BLEXBot
-BackStreet
-BackWeb
+Anarchy99
+Apexoo
+Aspiegel
+AwarioSmartBot
+BackDoorBot
 Backlink-Ceck
 BacklinkCrawler
-Badass
-Bandit
-Barkrowler
 Battleztar Bazinga
-BlackWidow
-Blow
-Boardreader
-Brandprotect
-Buck
-BuiltWith
+BlowFish
+BotALot
+Buddy
+BuiltBotTough
 BunnySlippers
-BuzzSumo
 CATExplorador
-CCBot
-CODE87
 Cegbfeieh
-CheTeam
-ChinaClaw
+CensysInspect
 Chlooe
+Claritybot
 Cliqzbot
+Cloud mapping
 Cocolyzebot
-Collector
+Cogentbot
 Copier
-CrazyWebCrawler
+Copyscape
+Cosmos
+Craftbot
+Crawling at Home Project
+CrunchBot
 Curious
-CyotekWebCopy
-DBLBot
-Demon
+Custo
+DIIbot
+DTS Agent
+DatabaseDriverMysqli
 Deusu
+Digincore
+DigitalPebble
+Dirbuster
 Discobot
-DomainAppender
+Discoverybot
+DomainCrawler
 Domains Project
-Dotbot
-Drip
-ECCP/1.0
+Dragonfly
 EasyDL
-EirGrabber
 EroCrawler
 ExtLinksBot
-Extreme Picture Finder
-FDM
+ExtractorPro
+FHscan
+FemtosearchBot
 Fimap
-Firefox/7.0
-FlashGet
 Flunky
-Foobot
-Freeuploader
-Fyrebot
-G-i-g-a-b-o-t
-GalaxyBot
 Genieo
-GermCrawler
-GoZilla
-GrabNet
-Grabber
+Go-Ahead-Got-It
 GrapeFX
-GrapeshotCrawler
+GridBot
 HEADMasterSEO
 HMView
 HTMLparser
-HTTP::Lite
-HaosouSpider
-Hloader
-HybridBot
-IRLbot
-Id-search
-Image Sucker
-Indy Library
-InfoTekies
-Intelliseek
-InternetSeer
+Havij
+Humanlinks
+IDBot
+Iblog
+IlseBot
+Image Fetch
+Internet Ninja
 Iria
+Iskanie
 IstellaBot
-JOC Web Spider
-JennyBot
-Joomla
+Jbrofuzz
+JetCar
+JikeSpider
 Jorgee
-Keyword Density
-Kozmosbot
-LeechFTP
+Kinza
+LWP::Simple
+Larbin
 LeechGet
 LexiBot
+Lftp
 LibWeb
-Likse
-LinkWalker
+LieBaoFast
+LinkScan
 LinkextractorPro
+LinkpadBot
 LinqiaMetadataDownloaderBot
 LinqiaRSSBot
-Lipperhey Spider
-MIDown tool
+LinqiaScrapeBot
+Litemage_walker
+MFC_Tear_Sample
 MIIxpc
-MTRobot
+MJ12bot
+Mag-Net
+Mail.RU_Bot
 Majestic SEO
 Majestic-SEO
-MarkMonitor
+Majestic12
+MarkWatch
+Mass Downloader
+Masscan
 Mata Hari
-MeanPath Bot
-MegaIndex.ru
+Mb2345Browser
 MicroMessenger
 Microsoft Data Access
-Mister PiX
-Moblie Safari
-Mojolicious
-Mozlila
+Morfeus Fucking Scanner
+Mr.4x3
 Msrabot
-Musobot
 NICErsPRO
-NPbot
 Name Intelligence
 Nameprotect
 Navroad
-NearSite
-Nessus
-NetAnts
+Net Vampire
 NetLyzer
-NetSpider
-Netcraft
+NetMechanic
 Nettrack
-Netvibes
+NextGenSearchBot
 Nibbler
-Nimbostratus
-Nmap
+Niki-bot
+Ninja
 Nuclei
-Offline Explorer
+Offline Navigator
+OpenLinkProfiler
+OpenVAS
+OutclicksBot
 OutfoxBot
 PHPCrawl
-POE-Component-Client-HTTP
-PageGrabber
-PageScorer
-PageThing.com
-Pandalytics
-Panscient
-Pavuk
+Page Analyzer
+Papa Foto
+Petalbot
+Pi-Monster
 Picsearch
-PictureFinder
 Piepmatz
+Pimonster
 Pixray
 PleaseCrawl
 ProWebWalker
-Pu_iN
-Pump
-QueryN Metasearch
-Quick-Crawler
-RankFlex
+RankActiveLinkBot
+RankingBot2
+RankurBot
+Re-re
 RealDownload
-RebelMouse
+Reaper
+Recorder
+RedesScrapy
 RepoMonkey
+Ripper
 RocketCrawler
+Rogerbot
+SISTRIX
 SMTBot
-Scanbot
-ScoutJet
-ScreenerBot
-Semrush
-SeoSiteCheckup
-SeobilityBot
+ScanAlert
+Scrapy
+SearchmetricsBot
+SemrushBot
 Shodan
 Siphon
-SmartDownload
-SocialRankIOBot
+Site Sucker
+SiteExplorer
+Snapbot
+Snoopy
 Sociscraper
-Sosospider
-Sottopop
-Spammen
+SpankBot
+Spbot
 SputnikBot
+Sqworm
+Steeler
 Stripper
 Sucuri
-T0PHackTeam
-T8Abot
+SuperHTTP
+SurveyBot
+Suzuran
 Teleport
-Telesphoreo
 Telesphorep
-Tracemyfile
+The Intraformant
+TheNomad
+Thumbor
+Toata
 Trendiction
+True_Robot
+Turingos
+Turnitin
+TurnitinBot
 TwengaBot
+Twice
 Typhoeus
-URLy Warning
-URLy.Warning
 Vacuum
+Vagabondo
 VelenPublicWebCrawler
-VoidEYE
-WPScan
+VidibleScraper
+Voltron
+WWW-Collector-E
 WWW-Mechanize
 WWW::Mechanize
-WWWOFFLE
 Web Auto
-Web Collage
-Web Enhancer
-Web Fuck
-WebAuto
-WebBandit
-WebCollage
+Web Sauger
+Web Sucker
 WebEnhancer
-WebGo IS
+WebFetch
 WebImageCollector
-WebPix
-WebSauger
+WebLeacher
 WebStripper
 WebSucker
 WebZIP
 Webalta
+WebmasterWorldForumBot
+WebsiteExtractor
 WebsiteQuester
+Whack
 Widow
 WinHTTrack
 Wonderbot
-Woobot
-Wotbox
-Zauba
 Zeus
+ZoomBot
 ZyBorg
 archive.org_bot
-arquivo-web-crawler
+cah.io.community
+check1.exe
+cognitiveseo
+com.plumanalytics
 crawl.sogou.com
-ips-agent
+crawler.feedback
+dataforseo.com
+demandbase-bot
+domainsproject.org
+facebookscraper
+gopher
+heritrix
+internetVista monitor
 isitwp.com
-lwp-trivial
+magpie-crawler
+meanpathbot
 mediawords
-netEstate NE Crawler
 oBot
-scalaj-http
+page scorer
+pcBrowser
+satoristudio.net
+scan.lol
 seobility
 seostar
-serpstatbot
-sitechecker.pro
+sexsearcher
 sp_auditbot
 sysscan
 ubermetrics-technologies.com
-voyagerx.com
-webmeup-crawler
-webpros.com
-webprosbot
 x09Mozilla
+zauba.io