V4.2021.10.2632 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-360Spider
-80legs
-Aboundexbot
-Acunetix
-AiHitBot
+404enemy
+Abonti
+Aboundex
 AllSubmitter
 AlphaBot
-Anarchie
-Anarchy99
-Ankit
-Anthill
-Apexoo
-Aspiegel
-AwarioRssBot
+Anarchy
+Asterias
+Attach
 AwarioSmartBot
-BDCbot
+BBBike
+BDFetch
+BLEXBot
+BackDoorBot
 BackStreet
+BackWeb
 Backlink-Ceck
-BacklinkCrawler
 Bandit
 Bigfoot
 Bitacle
-Black Hole
-BlowFish
-Bolt
+BlackWidow
+BotALot
 Brandprotect
+Brandwatch
 Buck
-BuiltBotTough
 Bullseye
-CATExplorador
+CCBot
+CODE87
+Calculon
+CazoodleBot
 CheTeam
-ChinaClaw
-Cliqzbot
-Cloud mapping
-Cocolyzebot
+CheeseBot
+Claritybot
 Cogentbot
-Copyscape
-Craftbot
-Crawling at Home Project
-CrazyWebCrawler
-CrunchBot
-Custo
+Copier
+Cosmos
+Crescent
 CyotekWebCopy
-DataCha0s
-DomCopBot
+DSearch
+DatabaseDriverMysqli
+Digincore
+Discoverybot
+Dispatch
+DnyzBot
 DomainAppender
-Domains Project
+DomainCrawler
 Dotbot
-Dragonfly
+Drip
+ECCP/1.0
 EMail Siphon
+Ebingbong
 EirGrabber
+EroCrawler
 Evil
-Extractor
+Exabot
+Express WebPictures
+Extreme Picture Finder
+EyeNetIE
+Ezooms
+FDM
 FHscan
-FemtosearchBot
-Fimap
-Firefox/7.0
+Flunky
 Foobot
-Freeuploader
-FyberSpider
-Fyrebot
-G-i-g-a-b-o-t
 GT::WWW
-Genieo
-Getintent
+GalaxyBot
 Go!Zilla
-Go-Ahead-Got-It
+GrabNet
 Grabber
 Grafula
-GrapeFX
-HMView
-Haansoft
-Harvest
+GrapeshotCrawler
+HTTP::Lite
 Id-search
 IlseBot
-IndeedBot
-InfoTekies
+Indy Library
+InfoNaviRobot
+Internet Ninja
 InternetSeer
-Iskanie
+Iria
+JamesBOT
+Jbrofuzz
+JennyBot
 JikeSpider
 Jorgee
 Jyxobot
 Keyword Density
-Lanshanbot
-Larbin
-LeechFTP
-Libwhisker
-Likse
-Linkdexbot
-LinkextractorPro
-LinkpadBot
+Kozmosbot
+LNSpiderguy
+LeechGet
+LibWeb
+LieBaoFast
+Lightspeedsystems
+Linkbot
 LinksManager
 LinqiaMetadataDownloaderBot
-LinqiaRSSBot
-Lipperhey
-Lipperhey Spider
-MFC_Tear_Sample
-MIDown tool
-MIIxpc
-Majestic-SEO
+MSFrontPage
+MSIECrawler
+MTRobot
+Majestic12
 MarkWatch
 Mass Downloader
-Masscan
 Mata Hari
-Mb2345Browser
-MeanPath Bot
-Mediatoolkitbot
 MegaIndex.ru
-MicroMessenger
-Mojeek
+Microsoft Data Access
+Mister PiX
+Moblie Safari
+Morfeus Fucking Scanner
+Mr.4x3
 Msrabot
+Musobot
 NPbot
-Needle
-Net Vampire
+Navroad
+NearSite
+Nessus
 NetAnts
-NetLyzer
+NetMechanic
+NetZIP
+Nettrack
 Netvibes
-Nibbler
-Nikto
-Nimbostratus
+Niki-bot
 Ninja
-Nmap
 Nutch
 Octopus
 OpenLinkProfiler
-OpenVAS
+Openvas
 OutfoxBot
+PHPCrawl
 POE-Component-Client-HTTP
 Page Analyzer
-PageAnalyzer
-PageScorer
-PageThing.com
-Pandalytics
 Panscient
-Pavuk
-PeoplePal
-Picscout
+Papa Foto
+PictureFinder
 Piepmatz
-Pimonster
-Pockey
-Pu_iN
+Probethenet
+PxBroker
 PyCurl
-RSSingBot
-RankFlex
-RankingBot
+Quick-Crawler
+RankActiveLinkBot
 RankingBot2
-Rankivabot
-RankurBot
 RealDownload
-Reaper
 Recorder
-RedesScrapy
 RepoMonkey
-RocketCrawler
-Rogerbot
+Ripper
 SEOkicks-Robot
-SEOlyticsCrawler
+SEOprofiler
+SEOstats
 SMTBot
 SalesIntelligent
-Scanbot
-Screaming
+ScoutJet
 ScreenerBot
-ScrepyBot
 Semrush
+SemrushBot
+SentiBot
+SeoSiteCheckup
+Siphon
+Site Sucker
 SiteCheckerBotCrawler
-SiteExplorer
 SiteLockSpider
-SiteSucker
+SiteSnagger
 Sitevigil
 SmartDownload
-Snoopy
+Snapbot
 SocialRankIOBot
-Sogou web spider
-Sosospider
-Sottopop
-SpaceBison
 SpankBot
-Sqlmap
-Sqworm
+Spanner
+Spbot
 Steeler
-Stripper
 Sucuri
-SuperHTTP
-Surfbot
-SurveyBot
-Teleport
-TeleportPro
-Telesoft
-Telesphorep
+SuperBot
+Swiftbot
+T0PHackTeam
+T8Abot
+The Intraformant
 TheNomad
-TightTwatBot
-Titan
-Toweyabot
-Twice
-Upflow
-VeriCiteCrawler
+Toata
+Tracemyfile
+Trendictionbot
+TurnitinBot
+TwengaBot
+URLy Warning
+UnisterBot
+VB Project
+Vacuum
+Vagabondo
+VelenPublicWebCrawler
 Virusdie
 Voltron
+WASALive-Bot
 WEBDAV
-WISENutbot
-WWW::Mechanize
-Web Pix
-WebCopier
-WebEnhancer
-WebImageCollector
+Wallpapers
+Wallpapers/3.0
+Web Auto
+Web Enhancer
+Web Fetch
+WebAuto
+WebBandit
+WebFetch
+WebGo IS
 WebLeacher
-WebReaper
-WebSauger
-WebWhacker
+WebPix
+WebStripper
+WebZIP
+Webalta
 Webshag
-Webster
-Whatweb
+Who.is Bot
 Widow
+WinHTTrack
 WiseGuys Robot
-Wotbox
-Wprecon
+Wonderbot
+Woobot
+Xaldon WebSpider
+Xaldon_WebSpider
 YoudaoBot
-Zauba
-Zermelo
+Zeus
 Zitebot
 ZoomBot
-ZumBot
 ZyBorg
-arquivo-web-crawler
-arquivo.pt
 backlink-check
+check1.exe
 coccocbot-web
 cognitiveseo
 com.plumanalytics
+crawl.sogou.com
 dataforseo.com
+demandbase-bot
 domainsproject.org
-gopher
+eCatch
+evc-batch
+facebookscraper
+instabid
+internetVista monitor
 isitwp.com
-meanpathbot
+lwp-request
 mediawords
 netEstate NE Crawler
 page scorer
-pcBrowser
-plumanalytics
-polaris version
 probe-image-size
+ripz
 scalaj-http
+scan.lol
 seobility
-siteripz
-spyfu
+serpstatbot
+sexsearcher
+sogouspider
+sp_auditbot
+sysscan
 tAkeOut
-trendiction.de
-ubermetrics-technologies.com
+trendiction.com
+voyagerx.com
 webmeup-crawler
 webpros.com
+webprosbot