V4.2021.08.2546 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
+360Spider
 404checker
 80legs
-ALittle Client
 ASPSeek
-Abonti
 Aboundex
-AhrefsBot
+Acunetix
+AfD-Verbotsverfahren
 AiHitBot
+Aipbot
+AllSubmitter
 Alligator
-Anarchie
-Anarchy
-Apexoo
-Attach
+Ankit
 AwarioRssBot
 AwarioSmartBot
-BBBike
+BDCbot
+BackDoorBot
 BackStreet
 Backlink-Ceck
-BacklinkCrawler
+Badass
+Bandit
 Barkrowler
-Bigfoot
 Black Hole
-Blackboard
-Blow
-BlowFish
 Bolt
-BotALot
-BuiltBotTough
-Bullseye
-CODE87
-Calculon
-CazoodleBot
-CheTeam
+Buddy
+BunnySlippers
+CCBot
+Cegbfeieh
 CheeseBot
+CherryPicker
+ChinaClaw
 Chlooe
-Cliqzbot
+Claritybot
+Cloud mapping
 Cogentbot
 Collector
-Copyscape
+Cosmos
 Crawling at Home Project
-Crescent
 CrunchBot
 CyotekWebCopy
 DSearch
 DTS Agent
-DataCha0s
-DatabaseDriverMysqli
-Devil
 Digincore
-DigitalPebble
+Dirbuster
 Discobot
+Discoverybot
 Dispatch
 DnyzBot
 DomainStatsBot
 Drip
+ECCP/1.0
 EMail Siphon
 EasyDL
-Ecxi
-EroCrawler
-Evil
-Exabot
-Express WebPictures
-ExtLinksBot
-Extractor
-Ezooms
+FDM
 FHscan
-FemtosearchBot
-FlashGet
-Freeuploader
+Fimap
+Firefox/7.0
+Flunky
+FrontPage
 Fuzz
-G-i-g-a-b-o-t
-GT::WWW
-GalaxyBot
-Grabber
-Grafula
+GermCrawler
+GetRight
+GetWeb
+Getintent
+Go!Zilla
+GrabNet
 GrapeshotCrawler
+GridBot
 HTMLparser
-HTTrack
 Haansoft
-Havij
-Iblog
-Id-search
+Heritrix
+Humanlinks
+HybridBot
+IRLbot
+IlseBot
+Image Sucker
+IndeedBot
 Indy Library
-InfoNaviRobot
 InterGET
-Internet Ninja
-Iria
-JamesBOT
+IstellaBot
+JOC Web Spider
+JennyBot
 JetCar
-Jetty
 Joomla
-JustView
-Kenjin Spider
 Kozmosbot
-LeechFTP
+LNSpiderguy
 LeechGet
-LibWeb
-Likse
+Lftp
+Libwhisker
+LinkScan
 LinkWalker
 Linkdexbot
-LinksManager
-LinqiaMetadataDownloaderBot
+LinkpadBot
 LinqiaRSSBot
-Lipperhey Spider
-Litemage_walker
+LinqiaScrapeBot
 Lmspider
 MFC_Tear_Sample
-MIDown tool
 MQQBrowser
-MSFrontPage
 Mail.RU_Bot
+Majestic-SEO
+MarkWatch
 Mass Downloader
-Masscan
-Mata Hari
+Mb2345Browser
 Meanpathbot
 MegaIndex.ru
 Metauri
-MicroMessenger
 Microsoft URL Control
-Mojeek
-Morfeus Fucking Scanner
-Mr.4x3
-NICErsPRO
-Nessus
+Moblie Safari
+Mojolicious
+Mozlila
+Msrabot
+Nameprotect
+Navroad
+NearSite
 Net Vampire
+NetAnts
+NetLyzer
+NetMechanic
 NetSpider
-NextGenSearchBot
+Nettrack
+Netvibes
 Niki-bot
-NimbleCrawler
 Nimbostratus
 Nmap
 Nuclei
 Nutch
-Octopus
 Offline Navigator
-OnCrawl
-OpenLinkProfiler
+OrangeBot
 OrangeSpider
-PECL::HTTP
-PHPCrawl
+OutclicksBot
+POE-Component-Client-HTTP
 Page Analyzer
 PageAnalyzer
 PageGrabber
 PageThing.com
-Papa Foto
+Panscient
 Petalbot
-Pi-Monster
-Piepmatz
-Pimonster
+PictureFinder
+PleaseCrawl
 ProPowerBot
 ProWebWalker
-Probethenet
-Psbot
+Pump
+PxBroker
+Quick-Crawler
+RSSingBot
+RankActive
+RankActiveLinkBot
 RankFlex
+Rankivabot
 RankurBot
-Reaper
-RepoMonkey
-SBIder
-SEOprofiler
+Re-re
+RealDownload
+RebelMouse
+Recorder
+RedesScrapy
+RocketCrawler
+Rogerbot
+SEOkicks
+SEOkicks-Robot
+SEOstats
 SISTRIX
-SalesIntelligent
-ScoutJet
+Scrapy
+Screaming
+ScrepyBot
 Searchestate
-SemrushBot
+SearchmetricsBot
+Semrush
 SeobilityBot
+Shodan
 Siphon
 SiteSnagger
+Sitebeam
+Siteimprove
 SlySearch
-Sociscraper
+Snapbot
+SocialRankIOBot
 Sogou web spider
 Spanner
-Spinn3r
+SputnikBot
+Sqworm
 Stripper
-SuperHTTP
-Suzuran
+Sucker
 Swiftbot
+Szukacz
+T0PHackTeam
+T8Abot
+Teleport
 TeleportPro
-Telesphorep
 The Intraformant
+TheNomad
+Titan
+Tracemyfile
 Trendiction
-True_Robot
 Turingos
-TurnitinBot
-Typhoeus
+Turnitin
+URLy.Warning
 UnisterBot
-VB Project
+VCI
 VelenPublicWebCrawler
-Virusdie
-Voil
+VeriCiteCrawler
+VidibleScraper
+VoidEYE
 Voltron
 WBSearchBot
-WWW-Collector-E
-WWW-Mechanize
-WWWOFFLE
+WISENutbot
+WPScan
+WWW::Mechanize
 Wallpapers
-Wallpapers/3.0
 WallpapersHD
 Web Auto
-Web Collage
 Web Fetch
-Web Pix
-Web Sauger
-WebCollage
+WebBandit
 WebCopier
+WebEnhancer
 WebFetch
-WebReaper
-Webalta
-Webster
-Whacker
+WebFuck
+WebGo IS
+WebWhacker
+WebsiteExtractor
+WebsiteQuester
+WiseGuys Robot
+Woobot
 Xaldon_WebSpider
+YoudaoBot
 Zade
-Zauba
-Zermelo
+Zitebot
 ZmEu
-ZoomBot
-ZoominfoBot
-ZyBorg
-arquivo.pt
+ZumBot
 autoemailspider
-backlink-check
-cah.io.community
-check1.exe
-coccocbot-web
+com.plumanalytics
 crawl.sogou.com
-demandbase-bot
+crawler4j
 eCatch
+evc-batch
 facebookscraper
 gopher
 internetVista monitor
-isitwp.com
-lwp-request
-magpie-crawler
-meanpathbot
-muhstik-scan
-page scorer
+ips-agent
+mediawords
+oBot
 plumanalytics
 ripz
-s1z.ru
 satoristudio.net
-scalaj-http
-scan.lol
 seostar
-serpstatbot
-siteripz
+sexsearcher
 spyfu
-tAkeOut
 voyagerx.com
+webmeup-crawler
+webpros.com
 x09Mozilla
-xpymep1.exe
-zauba.io
+x22Mozilla