V4.2021.07.2479 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-404checker
 404enemy
 ADmantX
 ASPSeek
+Abonti
 Aboundex
+Aboundexbot
 Acunetix
-AfD-Verbotsverfahren
 AhrefsBot
-Aipbot
+AiHitBot
 Alexibot
-Anarchy99
+AlphaBot
+Anarchy
 Ankit
-Anthill
-Apexoo
-Aspiegel
+Asterias
 Attach
-AwarioSmartBot
-BackStreet
-Badass
-Battleztar Bazinga
+Backlink-Ceck
+BacklinkCrawler
+Bandit
+BatchFTP
+BetaBot
+Bigfoot
 Bitacle
 BlackWidow
-Blackboard
-Blow
 BlowFish
-Boardreader
-Brandwatch
-BuiltBotTough
+BotALot
+Buck
 BuiltWith
 Bullseye
-BunnySlippers
+CATExplorador
 CCBot
-CSHttp
+Calculon
+Cegbfeieh
+CheTeam
 CheeseBot
 Chlooe
 Claritybot
+Cliqzbot
+Cloud mapping
 Cocolyzebot
-Cogentbot
-Collector
-Copier
-CopyRightCheck
 Copyscape
 Craftbot
-CrazyWebCrawler
-CyotekWebCopy
+Crescent
+CrunchBot
+Curious
+Custo
 DBLBot
-DSearch
-DTS Agent
-DataCha0s
+Demon
+Deusu
+Digincore
 DigitalPebble
 Dirbuster
+Discobot
 Dispatch
 DittoSpyder
-DomCopBot
+DomainAppender
 DomainCrawler
-Domains Project
-Dotbot
-Download Wonder
-Dragonfly
-Drip
-ECCP/1.0
-EMail Wolf
-EasyDL
 EirGrabber
-Evil
 Exabot
+Express WebPictures
+Extractor
 ExtractorPro
-FHscan
+Extreme Picture Finder
+Ezooms
+FDM
 Firefox/7.0
+FlashGet
 Freeuploader
-FyberSpider
+FrontPage
+Fuzz
+G-i-g-a-b-o-t
 GT::WWW
-Genieo
-GetWeb
-Getintent
-Gigabot
-Gotit
-Grafula
-GrapeshotCrawler
-GridBot
+GrabNet
+Grabber
 HEADMasterSEO
-HTTrack
-Heritrix
-IDBTE4M
-IRLbot
+HTMLparser
+HaosouSpider
+Havij
+Humanlinks
 IlseBot
-Image Sucker
+IndeedBot
 InfoTekies
-Intelliseek
+InterGET
 InternetSeer
-Iskanie
-IstellaBot
 JOC Web Spider
-JamesBOT
-Jbrofuzz
 JennyBot
 Jorgee
+JustView
 Jyxobot
 Kenjin Spider
-Keyword Density
 Kinza
 LNSpiderguy
-Lanshanbot
 LeechFTP
-LeechGet
 LibWeb
-Libwhisker
-LinkScan
-LinkextractorPro
+LieBaoFast
+LinkWalker
 LinkpadBot
-LinksManager
 LinqiaMetadataDownloaderBot
 LinqiaScrapeBot
-MFC_Tear_Sample
-MJ12bot
+Lipperhey Spider
+MIDown tool
+MIIxpc
 MQQBrowser
+MSFrontPage
+MSIECrawler
+MTRobot
+Mag-Net
 Magnet
 Mail.RU_Bot
-Majestic SEO
 Majestic-SEO
-Majestic12
-MarkWatch
+MarkMonitor
 Masscan
-MicroMessenger
+Mata Hari
+MeanPath Bot
+Mediatoolkitbot
+Microsoft URL Control
 Moblie Safari
-Msrabot
+Morfeus Fucking Scanner
 Musobot
-NPbot
-Name Intelligence
-NearSite
-Needle
-Net Vampire
+Navroad
+NetLyzer
 NetMechanic
-Netcraft
+Nettrack
+Netvibes
+NextGenSearchBot
+Nibbler
 Niki-bot
 Nikto
-Nmap
+NimbleCrawler
 Nuclei
-Octopus
-OutclicksBot
+Offline Navigator
+Openfind
+Openvas
+OrangeSpider
+OutfoxBot
 POE-Component-Client-HTTP
+PageGrabber
+PageThing.com
 Pandalytics
 Panscient
+Pavuk
 Pi-Monster
+Picscout
 PictureFinder
-PleaseCrawl
+Pimonster
 Probethenet
-PxBroker
+Psbot
+Pump
 QueryN Metasearch
-Quick-Crawler
-RankingBot2
+RankActiveLinkBot
 Rankivabot
-RankurBot
-Re-re
 ReGet
 RealDownload
-Reaper
-RebelMouse
 Recorder
-Ripper
-SEOkicks
-SEOlyticsCrawler
-SMTBot
-SalesIntelligent
-ScoutJet
-Scrapy
-Screaming
-Searchestate
-SearchmetricsBot
-SeoSiteCheckup
-Siphon
+RepoMonkey
+SEOprofiler
+SEOstats
+ScrepyBot
+Semrush
+SentiBot
+Shodan
 Site Sucker
-Sitevigil
+SiteLockSpider
+SiteSnagger
+SiteSucker
+Siteimprove
+SlySearch
+SmartDownload
+Snake
+Snapbot
 SocialRankIOBot
-Sociscraper
-SpankBot
-Steeler
-Sucuri
+Sosospider
+Spanner
+Spinn3r
+SputnikBot
+Sqlmap
+Stripper
+Surfbot
+Suzuran
+T8Abot
 TeleportPro
-TheNomad
+Telesphoreo
+Telesphorep
 Thumbor
+TightTwatBot
 Titan
-Toata
-Tracemyfile
+True_Robot
 Turingos
-TwengaBot
-UnisterBot
-Upflow
+Twice
+Typhoeus
+URLy.Warning
 V-BOT
-VB Project
-Vacuum
-Vagabondo
-VeriCiteCrawler
-WPScan
+VidibleScraper
+VoidEYE
+Voil
+WBSearchBot
 WWW-Mechanize
-WWW::Mechanize
-Wallpapers/3.0
-WallpapersHD
+WWWOFFLE
+WeSEE
+Web Collage
+Web Fetch
+WebAuto
 WebCollage
-WebCopier
-WebImageCollector
-WebLeacher
-WebPix
-WebReaper
-WebWhacker
-WebZIP
+WebFuck
+Webalta
 WebmasterWorldForumBot
 Webshag
-Website Quester
-WebsiteExtractor
 WebsiteQuester
-Whack
-Whacker
-Whatweb
+Webster
 Widow
-Wonderbot
-Woobot
+WinHTTrack
 Wotbox
 Xaldon WebSpider
+Xaldon_WebSpider
 YoudaoBot
+Zade
+Zauba
 ZmEu
-ZoominfoBot
 ZyBorg
-arquivo-web-crawler
+archive.org_bot
 backlink-check
+check1.exe
 coccocbot-web
+cognitiveseo
+com.plumanalytics
+crawl.sogou.com
 crawler.feedback
-crawler4j
+demandbase-bot
 domainsproject.org
-eCatch
-evc-batch
+gopher
 instabid
+internetVista monitor
 isitwp.com
 lwp-request
 lwp-trivial
 mediawords
-muhstik-scan
+oBot
+plumanalytics
 probe-image-size
-s1z.ru
-scalaj-http
+seobility
+seoscanners
 seostar
-sexsearcher
-siteripz
-spyfu
-tAkeOut
+sitechecker.pro
+sp_auditbot
+sysscan
 trendiction.com
-trendiction.de
+webmeup-crawler
 x09Mozilla
+zauba.io
 zgrab