V4.2022.03.2965 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-80legs
-AIBOT
-Aboundex
-AiHitBot
+ALittle Client
+ASPSeek
+Abonti
+Aboundexbot
+AfD-Verbotsverfahren
+AhrefsBot
+Aipbot
 Alexibot
+Alligator
+Anarchie
 Anarchy
-Anarchy99
 Anthill
+Apexoo
+Aspiegel
+Asterias
 Atomseobot
-Attach
-AwarioRssBot
-BDCbot
+AwarioSmartBot
 BDFetch
-BacklinkCrawler
+BLEXBot
+Backlink-Ceck
 Badass
 Bandit
-BatchFTP
+Barkrowler
 Battleztar Bazinga
 BetaBot
-Black Hole
-Blow
+Bigfoot
+Blackboard
 BlowFish
-Boardreader
+BotALot
 Brandprotect
-Brandwatch
+BuiltWith
 Bullseye
-BuzzSumo
+CATExplorador
 CODE87
-CazoodleBot
 CensysInspect
-CheeseBot
-ChinaClaw
+CheTeam
 Chlooe
 Citoid
+Cliqzbot
+Cocolyzebot
 Collector
 Copier
+CopyRightCheck
+Copyscape
+Crawling at Home Project
 CrunchBot
-Custo
-DIIbot
-DTS Agent
+Curious
+DBLBot
 Deusu
-Devil
-Dirbuster
+Digincore
+DigitalPebble
 Discoverybot
 Dispatch
-DittoSpyder
-DomainSigmaCrawler
+DnBCrawler-Analytics
+DnyzBot
+DomainAppender
+DomainCrawler
 Dotbot
-EMail Wolf
-Ebingbong
-Ecxi
+Dragonfly
+EasyDL
 EirGrabber
-Evil
+EroCrawler
 Exabot
 Express WebPictures
-ExtLinksBot
 ExtractorPro
+EyeNetIE
 Ezooms
-FHscan
-Fimap
+FDM
+FemtosearchBot
+FlashGet
 Flunky
-FrontPage
+Freeuploader
+Fuzz
+Fyrebot
 G-i-g-a-b-o-t
-GT::WWW
 GalaxyBot
-Genieo
-GermCrawler
-GetRight
+GetWeb
+Getintent
+Gigabot
 GoZilla
-Gotit
+GrapeFX
+HMView
 HTMLparser
-HTTrack
-Haansoft
-Harvest
-Humanlinks
-HybridBot
 IDBot
 Iblog
-IlseBot
-Indy Library
+Id-search
+IndeedBot
 InfoNaviRobot
-InterGET
+InfoTekies
+Intelliseek
 Iria
-JetCar
-Joomla
-JustView
+Iskanie
+IstellaBot
+JamesBOT
+Jbrofuzz
+Jetty
+Jorgee
 Jyxobot
-Kenjin Spider
-Keybot Translation-Search-Machine
-Kinza
-Lftp
-LibWeb
+Larbin
+Leap
+LeechGet
+LexiBot
 Libwhisker
 LieBaoFast
 Lightspeedsystems
+Likse
 LinkScan
+LinkWalker
 Linkbot
+Linkdexbot
 LinkextractorPro
-LinkpadBot
+LinqiaMetadataDownloaderBot
 LinqiaRSSBot
-Lipperhey
-MSIECrawler
+LinqiaScrapeBot
+Litemage_walker
+MSFrontPage
 MTRobot
-Magnet
 Mail.RU_Bot
+Majestic-SEO
 Masscan
-Mata Hari
-Mb2345Browser
-MeanPath Bot
 MegaIndex.ru
+Microsoft Data Access
 Microsoft URL Control
 Mister PiX
-Moblie Safari
-Mojeek
-Mojolicious
 MolokaiBot
 Morfeus Fucking Scanner
+Mr.4x3
+Msrabot
 Musobot
 NPbot
-NearSite
-Nessus
-Net Vampire
-NetMechanic
-NetZIP
-Niki-bot
+Name Intelligence
+NetAnts
+NetSpider
+Nettrack
+Nibbler
 Nikto
 Nimbostratus
-Ninja
-Nmap
-Not
 Nuclei
-Offline Navigator
-OpenLinkProfiler
-OpenVAS
-OrangeSpider
-POE-Component-Client-HTTP
-PageAnalyzer
+Octopus
+Offline Explorer
+OnCrawl
+Openvas
+OutfoxBot
 PageGrabber
-Pandalytics
-Pi-Monster
-Picscout
-Picsearch
-Probethenet
+PageScorer
+Pavuk
+PictureFinder
+Piepmatz
+ProWebWalker
+Pu_iN
 Pump
 PyCurl
-QueryN Metasearch
+RankActive
 RankingBot
-Rankivabot
+RankingBot2
+RankurBot
 ReGet
-RepoMonkey
-SEOlyticsCrawler
-SEOstats
-SMTBot
-ScanAlert
-Seekport
-SemanticJuice
-SeobilityBot
-Seomoz
-SiteSucker
+RealDownload
+Ripper
+Rogerbot
+SBIder
+SEOkicks-Robot
+SISTRIX
+Scanbot
+Screaming
+ScreenerBot
+ScrepyBot
+Semrush
+SeoSiteCheckup
+Siphon
+Site Sucker
+SiteExplorer
+Sitebeam
 Sitevigil
 SmartDownload
-SocialRankIOBot
-Sociscraper
-Sosospider
-Sqworm
+Snapbot
+Snoopy
+Sottopop
+Spammen
+Spanner
+Spbot
+Spinn3r
 Steeler
-Sucker
 Sucuri
-SuperBot
-SuperHTTP
 Surfbot
-Suzuran
-Swiftbot
-Szukacz
-Teleport
 Telesoft
-Telesphoreo
+Telesphorep
 Thumbor
-Toweyabot
-Tracemyfile
-Trendiction
+True_Robot
 Turingos
-Turnitin
-Twice
 Typhoeus
-Vagabondo
-VoidEYE
-Voil
+URLy.Warning
+Upflow
+VB Project
+Vacuum
+VidibleScraper
+Virusdie
 Voltron
 WASALive-Bot
-WBSearchBot
-WPScan
-WWW-Mechanize
-Wallpapers/3.0
-WallpapersHD
-WeSEE
-Web Auto
-Web Enhancer
-Web Sucker
+WEBDAV
+WISENutbot
+Web Fuck
 WebAuto
 WebBandit
-WebEnhancer
-WebFetch
-WebFuck
-WebGo IS
+WebCollage
+WebCopier
+WebImageCollector
 WebLeacher
 WebSauger
-WebStripper
-WebZIP
-WebmasterWorldForumBot
-Webshag
+Webalta
 WebsiteExtractor
 Who.is Bot
-Widow
-WinHTTrack
+Woobot
 Wprecon
-Xaldon WebSpider
+Xaldon_WebSpider
+Xenu
+YoudaoBot
+Zauba
+Zermelo
+Zeus
 ZmEu
+ZumBot
+archive.org_bot
 arquivo.pt
+cah.io.community
+check1.exe
 clark-crawler
 coccocbot
-cognitiveseo
-com.plumanalytics
-crawler.feedback
-crawler4j
-evc-batch
+crawl.sogou.com
+dataforseo.com
+demandbase-bot
+facebookscraper
+gopher
+instabid
+internetVista monitor
 ips-agent
-iubenda-radar
-lwp-trivial
 magpie-crawler
-muhstik-scan
-netEstate NE Crawler
 oBot
-page scorer
-plumanalytics
+pcBrowser
 polaris version
+probe-image-size
+ripz
 s1z.ru
-scan.lol
 seobility
-serpstatbot
+seoscanners
 sexsearcher
-sitechecker.pro
-siteripz
-sogouspider
-spyfu
+sysscan
 trendiction.de
 ubermetrics-technologies.com
-webgains-bot
-webmeup-crawler
-xpymep1.exe
+x09Mozilla