V4.2022.11.3493 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404enemy
-80legs
-AIBOT
 Aboundex
 Aboundexbot
 AhrefsBot
-Alexibot
 Alligator
-Anthill
 Aspiegel
-Asterias
-Atomseobot
 Attach
-AwarioSmartBot
 BDCbot
-BLEXBot
+BDFetch
+BackWeb
 Backlink-Ceck
-Badass
+Bandit
 Battleztar Bazinga
+BetaBot
 Bigfoot
 Bitacle
+BlackWidow
 Blackboard
+Blow
+Bolt
 Brandprotect
 Brandwatch
-Buck
-Buddy
+BuiltBotTough
+BuiltWith
+Bullseye
 BunnySlippers
 CATExplorador
-CCBot
-Cegbfeieh
-CheTeam
+CODE87
+CSHttp
 CheeseBot
 CherryPicker
-Claritybot
+Citoid
 Cliqzbot
 Cloud mapping
+Cocolyzebot
+Cogentbot
+CopyRightCheck
 Copyscape
 Craftbot
-CyotekWebCopy
+Crawling at Home Project
+CrazyWebCrawler
+Crescent
+CrunchBot
+DBLBot
+DIIbot
 DSearch
-DataCha0s
+DTS Agent
 DatabaseDriverMysqli
-Devil
+Demon
+Deusu
+Digincore
+Dirbuster
+Disco
+Dispatch
+DittoSpyder
 DnBCrawler-Analytics
-DnyzBot
+DomCopBot
 DomainAppender
 DomainCrawler
-Domains Project
-Download Wonder
-Dragonfly
+Drip
 EMail Siphon
 EMail Wolf
+EasyDL
+Ebingbong
+Ecxi
 ExtLinksBot
-FemtosearchBot
-FlashGet
-Flunky
-FrontPage
-GT::WWW
+ExtractorPro
+EyeNetIE
+Fimap
+Freeuploader
+Fuzz
+Fyrebot
+Genieo
+GetRight
 GetWeb
+Gigabot
 Go!Zilla
-GoZilla
-Gotit
-GrapeFX
 GrapeshotCrawler
-GridBot
-HMView
-Haansoft
-Harvest
-Heritrix
-IDBTE4M
-Id-search
-IndeedBot
-Indy Library
+Hloader
+Humanlinks
+Iblog
+Image Fetch
+InfoNaviRobot
 InfoTekies
 Intelliseek
 InterGET
-InternetSeer
-Iskanie
+Iria
 IstellaBot
 Jbrofuzz
-JennyBot
-Jetty
+JikeSpider
 Joomla
-JustView
-Jyxobot
 Kinza
 LNSpiderguy
 LWP::Simple
-Larbin
-Leap
-LeechGet
-LibWeb
-LinkScan
-Linkbot
-LinksManager
+LeechFTP
+Likse
+LinkWalker
 LinqiaMetadataDownloaderBot
 LinqiaScrapeBot
 Lipperhey
 MFC_Tear_Sample
-MIDown tool
+MIIxpc
+MSFrontPage
+MSIECrawler
 MTRobot
-Mag-Net
 Mail.RU_Bot
-Majestic SEO
 Majestic-SEO
 Majestic12
-MarkMonitor
 MarkWatch
 Mass Downloader
 Masscan
 Mata Hari
-Mediatoolkitbot
-Microsoft Data Access
-Microsoft URL Control
+MegaIndex.ru
+Metauri
 Mojeek
-Mozlila
+MolokaiBot
+Morfeus Fucking Scanner
 Mr.4x3
-Navroad
-NearSite
 Needle
 Nessus
 Net Vampire
 NetAnts
-NetLyzer
-NetMechanic
+NetSpider
+NetZIP
+Netcraft
+Nettrack
 Niki-bot
-Nikto
-Not
+Nimbostratus
+Ninja
+Nmap
+Octopus
 Offline Explorer
-Offline Navigator
-OnCrawl
-OpenLinkProfiler
 OpenVAS
-Openfind
-OrangeBot
+OrangeSpider
+OutfoxBot
+PECL::HTTP
 PHPCrawl
+POE-Component-Client-HTTP
 Page Analyzer
-PageAnalyzer
+PageGrabber
 PageThing.com
+Pandalytics
 Panscient
 Pi-Monster
-Picsearch
+Picscout
+Pixray
 PleaseCrawl
-Psbot
-Pump
-PyCurl
-QueryN Metasearch
-RSSingBot
-RankingBot
-Rankivabot
+Pockey
+ProPowerBot
+ProWebWalker
+Probethenet
+Pu_iN
+Quick-Crawler
+RankFlex
 RankurBot
+Re-re
+ReGet
+RealDownload
+Reaper
+RebelMouse
 Recorder
-RedesScrapy
-RepoMonkey
-RocketCrawler
 Rogerbot
-SMTBot
+SEOprofiler
 Scanbot
-ScoutJet
-Searchestate
+ScrepyBot
+SearchmetricsBot
 Seekport
-SemanticJuice
-SentiBot
-SeoSiteCheckup
-SeobilityBot
-Seomoz
+SemrushBot
+Site Sucker
 SiteCheckerBotCrawler
 SiteExplorer
-SiteSnagger
-Sitebeam
+SiteLockSpider
+SiteSucker
 Sitevigil
-SmartDownload
-SocialRankIOBot
-SpankBot
+Snoopy
+Spanner
 Spbot
-Sqlmap
-Sucker
+Steeler
 Sucuri
-SuperHTTP
+SuperBot
+SurveyBot
 Swiftbot
+Szukacz
 T0PHackTeam
-T8Abot
+Telesoft
 Telesphoreo
-TheNomad
-Toata
-Tracemyfile
+Telesphorep
+Thumbor
 Trendiction
+True_Robot
+Turingos
 Turnitin
 TwengaBot
-Twice
+V-BOT
+VB Project
 VCI
+Vagabondo
 VelenPublicWebCrawler
-VidibleScraper
-Virusdie
-WBSearchBot
-WEBDAV
-WISENutbot
-WPScan
-WWW-Collector-E
-WWW-Mechanize
+WWW::Mechanize
+Wallpapers/3.0
+WallpapersHD
 WeSEE
+Web Fetch
 Web Fuck
-Web Sucker
 WebBandit
 WebCollage
+WebEnhancer
 WebFetch
 WebImageCollector
-WebLeacher
-WebReaper
-WebSauger
-WebZIP
+WebSucker
 Webshag
+Website Quester
 WebsiteExtractor
-Widow
+Whacker
 WiseGuys Robot
-Wonderbot
-Wotbox
+Woobot
+Xaldon_WebSpider
 YoudaoBot
 Zauba
 Zeus
+Zitebot
+ZmEu
+ZoomBot
+ZoominfoBot
+archive.org_bot
+arquivo-web-crawler
 arquivo.pt
-autoemailspider
 check1.exe
 cognitiveseo
-com.plumanalytics
+crawl.sogou.com
 domainsproject.org
 eCatch
-facebookscraper
-gopher
+heritrix
 internetVista monitor
 ips-agent
 isitwp.com
-lwp-trivial
+iubenda-radar
+lwp-request
 magpie-crawler
-oBot
+meanpathbot
 polaris version
-s1z.ru
+probe-image-size
 satoristudio.net
-scan.lol
-seoscanners
-seostar
-sexsearcher
-sitechecker.pro
-sogouspider
-spyfu
+scalaj-http
+siteripz
 tAkeOut
+trendiction.com
+webgains-bot
 webmeup-crawler
-webpros.com
-webprosbot
 zauba.io
+zgrab