V4.2021.11.2703 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 404checker
 80legs
-AIBOT
-Aboundexbot
-Acunetix
+ALittle Client
 AhrefsBot
-AlphaBot
-Anarchie
-BDFetch
-BackDoorBot
-BackStreet
-BatchFTP
+Alligator
+Anarchy
+Apexoo
+Aspiegel
+AwarioSmartBot
+BBBike
+BDCbot
+Backlink-Ceck
+BacklinkCrawler
+Barkrowler
 Bigfoot
-Bitacle
 Black Hole
-Blackboard
-Blow
-Boardreader
+BlowFish
 BotALot
+Buck
+BuiltBotTough
+BuiltWith
+BunnySlippers
 CATExplorador
-CSHttp
+CCBot
 CazoodleBot
-Cegbfeieh
-CherryPicker
-Chlooe
+ChinaClaw
+Claritybot
 Cliqzbot
+Cloud mapping
+Cocolyzebot
 Cogentbot
 Copyscape
+Cosmos
 Craftbot
 Crawling at Home Project
-DSearch
-DTS Agent
+Crescent
+CrunchBot
+CyotekWebCopy
+DBLBot
 DataCha0s
-DigitalPebble
-Disco
-Discobot
-DnBCrawler-Analytics
+DatabaseDriverMysqli
+Deusu
+Digincore
+Discoverybot
+Dispatch
 DnyzBot
+DomainAppender
+DomainCrawler
 DomainStatsBot
 Domains Project
 Dotbot
-ECCP/1.0
-Ebingbong
-EroCrawler
+Download Wonder
+Drip
+EirGrabber
 Evil
-ExtLinksBot
-Extractor
-Ezooms
+Extreme Picture Finder
 FemtosearchBot
 Fimap
-FlashGet
-Foobot
-Freeuploader
-FrontPage
-FyberSpider
-GT::WWW
-GalaxyBot
-GetRight
-Getintent
+Firefox/7.0
 Gigabot
-Go-Ahead-Got-It
-GrabNet
-Grafula
-GrapeFX
+GoZilla
+Grabber
+GridBot
+HMView
 HTMLparser
-Haansoft
+HTTrack
 Humanlinks
+HybridBot
+IDBot
 Iblog
-Id-search
-IndeedBot
-Indy Library
-Intelliseek
-InternetSeer
-Iskanie
-IstellaBot
-JOC Web Spider
+IlseBot
+InterGET
+Iria
 JamesBOT
-JustView
+Jbrofuzz
+Jetty
+Jorgee
 Jyxobot
 Kinza
+Kozmosbot
+LNSpiderguy
 Lanshanbot
-Larbin
-LeechFTP
-LibWeb
-Likse
-LinkScan
-LinkWalker
+LexiBot
+Lightspeedsystems
 Linkbot
-LinksManager
-LinqiaMetadataDownloaderBot
-LinqiaRSSBot
 Lipperhey
-Lipperhey Spider
-Lmspider
-Ltx71
-MFC_Tear_Sample
-MJ12bot
-MQQBrowser
+MSIECrawler
 MTRobot
+Mag-Net
 Magnet
-Majestic SEO
+Mail.RU_Bot
+MarkWatch
 Mass Downloader
 Masscan
-MauiBot
-Mb2345Browser
-MeanPath Bot
-Mediatoolkitbot
-MicroMessenger
-Microsoft URL Control
-Mister PiX
+Meanpathbot
+MegaIndex.ru
+Microsoft Data Access
 Mojolicious
+Mr.4x3
 NPbot
-Name Intelligence
 Nameprotect
-Navroad
-NearSite
-Needle
 Nessus
-NetZIP
+NetAnts
+NetMechanic
+NetSpider
+Netcraft
+Nettrack
 NextGenSearchBot
-Nikto
+Nibbler
 NimbleCrawler
-Ninja
 Nmap
-Offline Explorer
+Nutch
+Octopus
 Offline Navigator
+OnCrawl
 OpenVAS
-Openvas
 OrangeBot
-OutfoxBot
-POE-Component-Client-HTTP
-PageAnalyzer
-Papa Foto
-PeoplePal
-Petalbot
+OutclicksBot
+PECL::HTTP
+PageScorer
+Pandalytics
+Pavuk
+Pi-Monster
 Picscout
+Picsearch
+PictureFinder
 Pimonster
-ProPowerBot
-ProWebWalker
-Probethenet
+PleaseCrawl
+Pockey
 Pu_iN
 PyCurl
-QueryN Metasearch
-RankingBot
+RankActiveLinkBot
+RankFlex
+Rankivabot
+RankurBot
+Re-re
 ReGet
 RealDownload
 Reaper
 Recorder
+RedesScrapy
+RepoMonkey
 RocketCrawler
-SEOkicks
+SBIder
 SEOkicks-Robot
+SEOstats
 SISTRIX
-Scanbot
 ScoutJet
+Scrapy
 Screaming
-ScreenerBot
 ScrepyBot
-SearchmetricsBot
+Searchestate
 Semrush
+SentiBot
+SeobilityBot
 Siphon
-SiteSucker
-Sitebeam
-Siteimprove
-SlySearch
+Site Sucker
+SiteExplorer
+SiteLockSpider
+SmartDownload
+Snake
 Sociscraper
-Sogou web spider
+Sosospider
+Spammen
+SpankBot
 Spanner
-Spbot
-Sqworm
 Stripper
-Sucker
+Sucuri
 SuperBot
 Surfbot
-SurveyBot
-Swiftbot
+T0PHackTeam
+Teleport
+TeleportPro
+Telesphoreo
 Telesphorep
+The Intraformant
+TheNomad
+Thumbor
+TightTwatBot
+Titan
 Toweyabot
-Trendictionbot
-True_Robot
-Turingos
-Typhoeus
-URLy Warning
-UnisterBot
+URLy.Warning
+Upflow
 V-BOT
-VB Project
-VCI
+VeriCiteCrawler
+VidibleScraper
+VoidEYE
 Voltron
 WASALive-Bot
 WEBDAV
-WPScan
-WWW-Collector-E
 WWWOFFLE
+Wallpapers
+Wallpapers/3.0
 Web Enhancer
-Web Fetch
-Web Sucker
-WebAuto
-WebCollage
-WebFetch
+Web Fuck
+Web Pix
+Web Sauger
+WebBandit
+WebFuck
+WebGo IS
 WebImageCollector
-WebPix
-WebReaper
+WebLeacher
 WebSauger
-WebStripper
-WebSucker
-WebWhacker
 WebZIP
 WebmasterWorldForumBot
 WebsiteExtractor
+WebsiteQuester
 Webster
 Whack
-Whacker
+Whatweb
 Who.is Bot
-WiseGuys Robot
 Wonderbot
-Wprecon
-Xaldon WebSpider
+Woobot
+Wotbox
 Xaldon_WebSpider
+Xenu
 YoudaoBot
-Zade
+Zauba
 Zermelo
-Zeus
 Zitebot
 ZmEu
-ZyBorg
 arquivo-web-crawler
-backlink-check
-coccocbot-web
+arquivo.pt
+autoemailspider
+clark-crawler
 cognitiveseo
-crawl.sogou.com
-demandbase-bot
-eCatch
-facebookscraper
-heritrix
+crawler4j
+dataforseo.com
 internetVista monitor
-lwp-request
-muhstik-scan
+meanpathbot
+mediawords
+netEstate NE Crawler
 oBot
 page scorer
 plumanalytics
 polaris version
-scan.lol
+ripz
+s1z.ru
+scalaj-http
+seobility
 seoscanners
-sogouspider
-spyfu
+serpstatbot
+sexsearcher
+siteripz
+sp_auditbot
 tAkeOut
-trendiction.com
-trendiction.de
 ubermetrics-technologies.com
-webpros.com
+webmeup-crawler
+webprosbot
 x09Mozilla