V4.2022.09.3372 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
+01h4x.com
+404enemy
 ASPSeek
 Abonti
-Aboundexbot
-AhrefsBot
-AllSubmitter
-Anarchie
-Anthill
-Apexoo
-Aspiegel
-BDCbot
-BDFetch
-BLEXBot
-BackStreet
-Backlink-Ceck
-Barkrowler
-Bigfoot
-Bitacle
-Black Hole
+Aboundex
+Aipbot
+Anarchy
+Asterias
+Atomseobot
+Attach
+BBBike
+Badass
+Battleztar Bazinga
+BlackWidow
+Blackboard
 Blow
+BlowFish
 Boardreader
-Brandprotect
-Brandwatch
-BuiltBotTough
-BuiltWith
-BunnySlippers
-BuzzSumo
-CATExplorador
+Bolt
+BotALot
+Bullseye
 CSHttp
-Calculon
+CazoodleBot
 CheeseBot
 ChinaClaw
-Chlooe
-Cliqzbot
-CrazyWebCrawler
-Crescent
-CrunchBot
-DBLBot
-DIIbot
+Citoid
+Claritybot
+Cloud mapping
+Cogentbot
+Copyscape
+Cosmos
+Crawling at Home Project
+Curious
+Custo
+CyotekWebCopy
 DSearch
-DTS Agent
-DataCha0s
-Devil
+Demon
+Digincore
 DigitalPebble
-Dirbuster
 Disco
-DnBCrawler-Analytics
-DnyzBot
-DomainSigmaCrawler
+DittoSpyder
+DomainAppender
 Domains Project
 Dotbot
 Drip
-ECCP/1.0
-EMail Wolf
-EasyDL
-EirGrabber
-Evil
+EroCrawler
 Exabot
 Express WebPictures
-Extractor
-EyeNetIE
-Foobot
-Fuzz
-Fyrebot
-GT::WWW
-Genieo
+FDM
+Firefox/7.0
+FlashGet
+Freeuploader
+FrontPage
+G-i-g-a-b-o-t
 GetRight
+Getintent
+Gigabot
 Go!Zilla
+Go-Ahead-Got-It
+Gotit
 Grabber
 GrapeFX
-GridBot
-HTTrack
+HTMLparser
+Harvest
 Havij
-HybridBot
+Heritrix
+Hloader
 IDBTE4M
+IRLbot
+Id-search
 Image Fetch
-Image Sucker
-Indy Library
-InfoTekies
-InterGET
-JamesBOT
-Joomla
-JustView
-Kozmosbot
+InfoNaviRobot
+Intelliseek
+InternetSeer
+Iskanie
+JOC Web Spider
+Jbrofuzz
+JetCar
+Jetty
+JikeSpider
+Keybot Translation-Search-Machine
+LNSpiderguy
 Larbin
 LeechGet
-LexiBot
 Lftp
-LieBaoFast
+Libwhisker
 Likse
+LinkScan
 Linkbot
+LinkextractorPro
+LinqiaMetadataDownloaderBot
+LinqiaRSSBot
+LinqiaScrapeBot
+Lipperhey
 Lmspider
-Ltx71
 MFC_Tear_Sample
 MIDown tool
+MIIxpc
+MJ12bot
 MSFrontPage
-Mag-Net
-Majestic SEO
+MSIECrawler
+Magnet
+Mail.RU_Bot
 Mass Downloader
 MauiBot
-Metauri
-Microsoft URL Control
-Mojeek
-Mojolicious
+MegaIndex.ru
+Moblie Safari
 MolokaiBot
-NearSite
-Needle
-Nessus
+Mr.4x3
+Msrabot
+Musobot
+NPbot
+Nameprotect
+Navroad
 Net Vampire
 NetAnts
-NetMechanic
+NetLyzer
 NetSpider
-NetZIP
-Nettrack
-Netvibes
+Netcraft
+NextGenSearchBot
 Niki-bot
-Ninja
+NimbleCrawler
+Nimbostratus
+Not
+Nuclei
 Nutch
-Octopus
-OnCrawl
 OpenLinkProfiler
+Openvas
 OrangeBot
 OrangeSpider
-OutclicksBot
-Page Analyzer
+OutfoxBot
+POE-Component-Client-HTTP
 PageThing.com
-Panscient
-Papa Foto
+Pavuk
 PeoplePal
-Pi-Monster
-PictureFinder
+Petalbot
 Piepmatz
-PleaseCrawl
 Pockey
-Pu_iN
-Quick-Crawler
-RankActive
+ProWebWalker
+Psbot
+PyCurl
+RSSingBot
+RankActiveLinkBot
 RankFlex
 RankingBot2
-RealDownload
-RedesScrapy
-Ripper
-RocketCrawler
+Re-re
+ReGet
+Reaper
 Rogerbot
 SEOkicks
 SEOprofiler
-SISTRIX
-SalesIntelligent
-Scanbot
+SMTBot
+ScanAlert
+Screaming
 ScreenerBot
+ScrepyBot
+Searchestate
 SearchmetricsBot
-Seekport
-SemanticJuice
 Semrush
-SeoSiteCheckup
+SentiBot
 SeobilityBot
-Shodan
-Site Sucker
-Sitevigil
-Snake
-Sottopop
+Seomoz
+Siphon
+SiteCheckerBotCrawler
+SiteSnagger
+SlySearch
+SocialRankIOBot
+Sogou web spider
 Spammen
 SpankBot
+Spanner
 SputnikBot
-Sqlmap
 Sqlworm
-Sqworm
-Stripper
-Sucker
-Sucuri
-Surfbot
+SuperBot
+SurveyBot
 Suzuran
-Szukacz
-T8Abot
+Swiftbot
+T0PHackTeam
+Teleport
 TeleportPro
 Telesoft
-The Intraformant
+TightTwatBot
 Titan
 Toata
-Toweyabot
-Trendictionbot
-Turingos
-Turnitin
-TurnitinBot
+Tracemyfile
+Trendiction
+True_Robot
+TwengaBot
+Typhoeus
+URLy Warning
 UnisterBot
-VelenPublicWebCrawler
+V-BOT
+Vacuum
 VeriCiteCrawler
-VidibleScraper
 Virusdie
+VoidEYE
 WASALive-Bot
-WBSearchBot
+WEBDAV
 WISENutbot
-WWW-Collector-E
+WWW::Mechanize
+Wallpapers
 WallpapersHD
 Web Auto
-Web Collage
-Web Enhancer
+Web Fetch
 Web Fuck
-Web Pix
 Web Sucker
+WebBandit
 WebCollage
+WebCopier
+WebFetch
 WebGo IS
 WebLeacher
-WebPix
+WebReaper
+WebSauger
 WebStripper
-Webshag
-Webster
-Whack
-Whacker
-WinHTTrack
-WiseGuys Robot
-Woobot
+WebWhacker
+WebZIP
+Who.is Bot
+Widow
+Wonderbot
 Wotbox
 Wprecon
 Xaldon_WebSpider
+YoudaoBot
 Zauba
-Zitebot
 ZmEu
 ZoomBot
+ZyBorg
 adscanner
-arquivo-web-crawler
 backlink-check
 cah.io.community
-coccocbot
-cognitiveseo
 com.plumanalytics
-crawler4j
-demandbase-bot
-heritrix
-instabid
-internetVista monitor
+crawl.sogou.com
+crawler.feedback
+dataforseo.com
+domainsproject.org
 ips-agent
-iubenda-radar
+isitwp.com
 linkdexbot
-lwp-request
-pcBrowser
+magpie-crawler
+muhstik-scan
+oBot
 plumanalytics
-ripz
 s1z.ru
-scalaj-http
-scan.lol
-seocompany.store
-seostar
-serpstatbot
-sitechecker.pro
+satoristudio.net
+sexsearcher
+sogouspider
 sp_auditbot
-ubermetrics-technologies.com
+sysscan
 webmeup-crawler
-webpros.com
-x22Mozilla
-zgrab