V4.2022.04.3021 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-404checker
-ADmantX
+404enemy
+80legs
 AIBOT
+ALittle Client
 ASPSeek
-Abonti
-Acunetix
+AhrefsBot
 AiHitBot
 Aipbot
 Alexibot
-AllSubmitter
-AlphaBot
-Anarchy99
-Apexoo
-Aspiegel
-AwarioRssBot
-AwarioSmartBot
-BBBike
-BDCbot
+Alligator
+Anarchie
+Anarchy
+Anthill
+Asterias
+Attach
+BDFetch
+BLEXBot
 BackStreet
-BackWeb
-Backlink-Ceck
 BacklinkCrawler
-Badass
-Bandit
-BlackWidow
-Blackboard
-Blow
-Bolt
+Battleztar Bazinga
+BetaBot
+Black Hole
 BotALot
+Buck
 BuiltBotTough
 BunnySlippers
-CATExplorador
-CSHttp
-Calculon
-Cegbfeieh
-CensysInspect
-ChinaClaw
+BuzzSumo
+CCBot
+CODE87
+CheTeam
+CheeseBot
 Chlooe
+Claritybot
 Cloud mapping
 Cocolyzebot
 Collector
-CopyRightCheck
-Cosmos
-Craftbot
+Copier
+Crawling at Home Project
 Crescent
-DTS Agent
+DIIbot
 DatabaseDriverMysqli
-Demon
-DnyzBot
-DomainStatsBot
-Download Wonder
+Devil
+Digincore
+Dirbuster
+Disco
+Discobot
+Discoverybot
+Dispatch
+DomainAppender
+Domains Project
+Dotbot
+Dragonfly
+ECCP/1.0
 EMail Siphon
 EasyDL
-Exabot
+Ecxi
 Express WebPictures
-FlashGet
+Extractor
+ExtractorPro
+EyeNetIE
+Ezooms
+FHscan
+Firefox/7.0
+Flunky
+Foobot
+Freeuploader
 FrontPage
-Fuzz
-Genieo
-GermCrawler
+G-i-g-a-b-o-t
+GalaxyBot
 GetWeb
-Go!Zilla
-HEADMasterSEO
+Getintent
+Gigabot
+GoZilla
+Gotit
+GrabNet
+GrapeshotCrawler
 HMView
 HTMLparser
 HTTP::Lite
+HTTrack
 Haansoft
 HaosouSpider
-Harvest
+Heritrix
+HonoluluBot
 Humanlinks
-IDBTE4M
+HybridBot
 IDBot
-Iblog
 IlseBot
-Image Fetch
-InterGET
-Internet Ninja
+InfoTekies
+Intelliseek
+JamesBOT
+JennyBot
 JetCar
-Joomla
-JustView
+Jetty
 Kenjin Spider
-Keyword Density
-LeechFTP
+Keybot Translation-Search-Machine
+Kinza
+LNSpiderguy
+Lanshanbot
+LexiBot
 Lftp
-Libwhisker
-Lightspeedsystems
+LibWeb
+LieBaoFast
+LinkScan
+Linkbot
 Linkdexbot
-LinqiaMetadataDownloaderBot
-LinqiaRSSBot
-LinqiaScrapeBot
-Lipperhey Spider
-Litemage_walker
+LinksManager
+Lipperhey
 MFC_Tear_Sample
 MIDown tool
 MIIxpc
-MJ12bot
-MQQBrowser
 MSFrontPage
-MSIECrawler
-MTRobot
 Mail.RU_Bot
-Majestic SEO
-Majestic-SEO
-Mass Downloader
+MarkMonitor
 Mata Hari
-MauiBot
-Mb2345Browser
-Meanpathbot
-Moblie Safari
-Mr.4x3
+MeanPath Bot
+Mediatoolkitbot
+Metauri
+Minefield
+MolokaiBot
 NICErsPRO
 NPbot
-Nessus
+Name Intelligence
+Needle
 Net Vampire
+NetLyzer
 NetMechanic
 NetSpider
-NetZIP
-Netvibes
+Netcraft
+Nettrack
 NextGenSearchBot
 Nibbler
-Ninja
+Niki-bot
+Nimbostratus
 Nmap
+Not
 Nutch
+Offline Explorer
 Offline Navigator
-OnCrawl
-OpenLinkProfiler
-OrangeBot
-OutfoxBot
-PECL::HTTP
-PHPCrawl
-Page Analyzer
-PageAnalyzer
 Panscient
-Papa Foto
 Pavuk
-PictureFinder
+Piepmatz
 Pimonster
-Pockey
-ProPowerBot
+PleaseCrawl
+ProWebWalker
 Probethenet
-PxBroker
 PyCurl
-Quick-Crawler
-RankingBot
-RankurBot
+QueryN Metasearch
+RankActive
+RankActiveLinkBot
+RankFlex
+Re-re
 ReGet
-RealDownload
-Reaper
 RebelMouse
-Recorder
-RedesScrapy
-Ripper
-SBIder
-ScanAlert
-Scrapy
+Rogerbot
+SEOkicks
+SEOkicks-Robot
+SEOlyticsCrawler
+SMTBot
+Screaming
+ScreenerBot
+Searchestate
 SemanticJuice
+Semrush
 SeoSiteCheckup
-SeobilityBot
-Site Sucker
 SiteCheckerBotCrawler
 SiteExplorer
-SiteLockSpider
+SiteSucker
 Siteimprove
 SlySearch
-Snake
-Snapbot
-Snoopy
+SmartDownload
 Sociscraper
-Sosospider
+Sogou web spider
+Sottopop
+Spbot
+Spinn3r
 SputnikBot
+Sqlmap
+Sqlworm
+Stripper
 Sucuri
-Surfbot
-Swiftbot
+SuperBot
+SurveyBot
+Szukacz
 T0PHackTeam
 T8Abot
-Telesoft
-TightTwatBot
+Thumbor
 Titan
 Toata
-Trendiction
-Turnitin
-TwengaBot
-URLy Warning
-UnisterBot
-Upflow
+Tracemyfile
+Trendictionbot
+True_Robot
+Twice
 VB Project
 Vacuum
-Vagabondo
-WWW-Mechanize
+VelenPublicWebCrawler
+VidibleScraper
+Voil
+WBSearchBot
+WEBDAV
+WISENutbot
+WWWOFFLE
 Wallpapers
-WallpapersHD
-WeSEE
+Web Auto
 Web Collage
-Web Enhancer
-Web Fetch
-Web Fuck
-Web Sucker
+Web Pix
+Web Sauger
+WebAuto
 WebCollage
-WebCopier
-WebFetch
-WebFuck
-WebGo IS
-WebLeacher
 WebPix
-WebWhacker
-Webalta
-WebmasterWorldForumBot
-WebsiteExtractor
+WebSauger
+Website Quester
+Webster
 Whacker
-Whatweb
 Who.is Bot
+Widow
 Wotbox
-Wprecon
-Zade
+Xaldon_WebSpider
+Xenu
+Zeus
 Zitebot
+ZoomBot
 ZyBorg
+adscanner
 archive.org_bot
 arquivo-web-crawler
-arquivo.pt
 backlink-check
-clark-crawler
-coccocbot
-com.plumanalytics
+check1.exe
+crawl.sogou.com
+crawler.feedback
 crawler4j
 demandbase-bot
-eCatch
 gopher
+instabid
 iubenda-radar
-lwp-request
-magpie-crawler
 meanpathbot
+mediawords
 page scorer
 plumanalytics
 polaris version
-s1z.ru
-satoristudio.net
 scalaj-http
 seocompany.store
 serpstatbot
-sitechecker.pro
-sogouspider
+sexsearcher
 spyfu
-tAkeOut
 trendiction.de
+ubermetrics-technologies.com
 voyagerx.com
-webmeup-crawler
-webprosbot
 x09Mozilla
-x22Mozilla
+zgrab