V4.2021.08.2519 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-80legs
+404checker
+ADmantX
 AIBOT
 ALittle Client
-ASPSeek
+Abonti
 Aboundex
 Aboundexbot
-Acunetix
-AhrefsBot
 Aipbot
 Alligator
-AlphaBot
-Anarchie
 Anarchy
-Anarchy99
-Ankit
 Anthill
+Apexoo
 Asterias
+Attach
 AwarioRssBot
-BLEXBot
+BDCbot
+BDFetch
 BackStreet
+BackWeb
+Backlink-Ceck
+Badass
+Barkrowler
+BatchFTP
 BetaBot
 Bigfoot
+Black Hole
+BlackWidow
 BlowFish
-Boardreader
+Bolt
 BotALot
-BuiltBotTough
+Buck
 Bullseye
-BunnySlippers
 BuzzSumo
-CODE87
-CSHttp
-CazoodleBot
-Claritybot
-Cliqzbot
+CATExplorador
+Calculon
+CheeseBot
+CherryPicker
+ChinaClaw
+Chlooe
+Cloud mapping
+Cocolyzebot
 Collector
-Cosmos
+CopyRightCheck
+Craftbot
 CrunchBot
-Custo
-DBLBot
-DSearch
-DTS Agent
-DataCha0s
+DIIbot
+Demon
 Devil
-DigitalPebble
+Disco
+Discoverybot
 Dispatch
-DittoSpyder
-DomCopBot
 DomainAppender
-DomainSigmaCrawler
 Domains Project
 Dotbot
-EMail Siphon
+Download Wonder
+Dragonfly
+Drip
+ECCP/1.0
 EMail Wolf
-EroCrawler
 Evil
-Express WebPictures
+Exabot
+ExtLinksBot
+EyeNetIE
 Ezooms
 FDM
-FHscan
 FemtosearchBot
-Fimap
-Firefox/7.0
+FlashGet
 Freeuploader
-FrontPage
-Fyrebot
+FyberSpider
+GalaxyBot
 Genieo
+GermCrawler
 Gigabot
+Go-Ahead-Got-It
 GoZilla
-GrabNet
-Grabber
-Grafula
-HEADMasterSEO
+GrapeFX
+GridBot
 HMView
-HTTP::Lite
+HTMLparser
+HTTrack
 Haansoft
-HaosouSpider
-Harvest
 Heritrix
 Hloader
-IDBTE4M
-IDBot
-IRLbot
+HybridBot
 Iblog
-Id-search
 IlseBot
-Image Sucker
-IndeedBot
-Indy Library
-IstellaBot
-Jbrofuzz
+InfoNaviRobot
+Intelliseek
+InternetSeer
+Iria
+JOC Web Spider
+JennyBot
 JetCar
+JikeSpider
 Joomla
 Jorgee
 JustView
-Jyxobot
-Kenjin Spider
 Keyword Density
-Kozmosbot
+LNSpiderguy
 LWP::Simple
-Leap
-LeechFTP
-Lftp
-LibWeb
-Libwhisker
+Lanshanbot
+Lightspeedsystems
+LinkWalker
+Linkdexbot
 LinkextractorPro
-LinqiaRSSBot
-MIDown tool
+LinkpadBot
+LinksManager
+LinqiaMetadataDownloaderBot
+LinqiaScrapeBot
+Lipperhey Spider
+Litemage_walker
+Ltx71
 MIIxpc
-MJ12bot
-MSFrontPage
 MSIECrawler
-MTRobot
-Mag-Net
-Majestic-SEO
-MarkWatch
-Mass Downloader
-Mata Hari
+Majestic SEO
+MarkMonitor
 MauiBot
-Mb2345Browser
+MeanPath Bot
+Meanpathbot
+MegaIndex.ru
 Metauri
-Microsoft URL Control
+MicroMessenger
+Microsoft Data Access
+Mister PiX
+Moblie Safari
 Morfeus Fucking Scanner
-NICErsPRO
-Name Intelligence
-Nameprotect
-Nessus
-Netcraft
+Mozlila
+NPbot
+Needle
+NetLyzer
+NetMechanic
+NetZIP
+Netvibes
 NextGenSearchBot
-Niki-bot
-NimbleCrawler
-Ninja
-Nmap
-Offline Explorer
-Offline Navigator
+Nibbler
+Nutch
+Octopus
 OpenLinkProfiler
+OpenVAS
+OrangeBot
 OrangeSpider
-OutclicksBot
-PECL::HTTP
-POE-Component-Client-HTTP
+OutfoxBot
+PHPCrawl
 PageAnalyzer
-PageScorer
+PageGrabber
 PageThing.com
+Papa Foto
+Pavuk
 Petalbot
-PictureFinder
+Picscout
 Pixray
-Pockey
+Psbot
 Pu_iN
-Pump
-PyCurl
-Quick-Crawler
 RSSingBot
-RankActive
-RankFlex
+RankActiveLinkBot
+RankingBot
 RankingBot2
-RealDownload
+Rankivabot
+RankurBot
 Reaper
-Recorder
+RebelMouse
 RedesScrapy
-Ripper
+RepoMonkey
 Rogerbot
 SBIder
 SEOkicks
-SEOlyticsCrawler
-SEOprofiler
-SEOstats
+SEOkicks-Robot
 SISTRIX
-ScoutJet
-Searchestate
+SMTBot
+ScanAlert
+Scrapy
+Screaming
 SearchmetricsBot
-SentiBot
-SeoSiteCheckup
+Semrush
 SeobilityBot
-Shodan
+SiteExplorer
 SiteSucker
-Sitebeam
 Siteimprove
+Sitevigil
 SlySearch
 Snake
-SocialRankIOBot
+Snoopy
 Sociscraper
-Sosospider
 Sottopop
 Spanner
-Sqlmap
-Sqlworm
-Surfbot
-SurveyBot
+SputnikBot
+Sucker
+Sucuri
+SuperHTTP
+Swiftbot
 T0PHackTeam
-Titan
-Turnitin
-TurnitinBot
-URLy.Warning
-UnisterBot
-Vacuum
-Vagabondo
-VelenPublicWebCrawler
+Telesphoreo
+The Intraformant
+TheNomad
+Thumbor
+Trendiction
+Typhoeus
+Upflow
+VCI
 VidibleScraper
-WASALive-Bot
-WISENutbot
-WWW-Collector-E
-WWW-Mechanize
+Virusdie
+VoidEYE
+Voil
+WEBDAV
+WWW::Mechanize
 Wallpapers/3.0
-Web Auto
+WallpapersHD
+Web Enhancer
 Web Pix
-Web Sauger
+Web Sucker
+WebCollage
+WebCopier
 WebFuck
-WebLeacher
-WebReaper
+WebImageCollector
+WebPix
+WebStripper
 WebSucker
-WebmasterWorldForumBot
-Webshag
-Widow
-WinHTTrack
+Website Quester
+WebsiteExtractor
+Who.is Bot
+WiseGuys Robot
+Wonderbot
+Wotbox
 Wprecon
+Xaldon_WebSpider
 YoudaoBot
-Zade
-Zauba
-Zeus
 ZmEu
-ZumBot
+ZoominfoBot
+ZyBorg
 autoemailspider
-coccocbot-web
-com.plumanalytics
+backlink-check
+cah.io.community
+cognitiveseo
+crawl.sogou.com
 crawler.feedback
-crawler4j
-demandbase-bot
 domainsproject.org
-eCatch
-evc-batch
-gopher
-heritrix
-instabid
-internetVista monitor
 magpie-crawler
-meanpathbot
-netEstate NE Crawler
-page scorer
-pcBrowser
-polaris version
+mediawords
+muhstik-scan
+oBot
 probe-image-size
-ripz
 scalaj-http
-sexsearcher
-siteripz
+sitechecker.pro
 sogouspider
+sp_auditbot
+spyfu
+sysscan
 trendiction.de
-voyagerx.com
-x09Mozilla
+x22Mozilla
 zgrab