V4.2021.11.2700 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
+ALittle Client
 ASPSeek
 Abonti
+Aboundex
+Aboundexbot
 Acunetix
-AfD-Verbotsverfahren
-Anarchy
-Ankit
-Apexoo
-Aspiegel
-Asterias
+AhrefsBot
+AiHitBot
 Attach
+AwarioRssBot
 BBBike
-BDCbot
-Badass
+BDFetch
+BackStreet
 Bandit
-Barkrowler
-BetaBot
-Bigfoot
-Bitacle
-Blackboard
-Boardreader
-Buck
-CODE87
-Calculon
-CazoodleBot
-CheeseBot
-CherryPicker
+BatchFTP
+Black Hole
+BlowFish
+Bolt
+BotALot
+Brandprotect
+Buddy
+BuiltWith
+BuzzSumo
+CATExplorador
+CCBot
+CSHttp
+Cegbfeieh
+CensysInspect
+ChinaClaw
+Chlooe
 Claritybot
+Cliqzbot
 Cloud mapping
-Copier
-CopyRightCheck
-Copyscape
-Cosmos
-Crawling at Home Project
-CrazyWebCrawler
-Crescent
+CrunchBot
 Custo
+CyotekWebCopy
+DIIbot
 DSearch
-DataCha0s
-DatabaseDriverMysqli
+Demon
+Devil
 Discobot
 Discoverybot
 Dispatch
 DnBCrawler-Analytics
-DnyzBot
 DomCopBot
-DomainStatsBot
+DomainAppender
+DomainCrawler
 Domains Project
 Dotbot
-Drip
-EMail Siphon
 EMail Wolf
+EasyDL
+Ecxi
+EroCrawler
 Exabot
-Express WebPictures
-Extractor
-ExtractorPro
 EyeNetIE
+FDM
 FHscan
-FemtosearchBot
 Fimap
 FlashGet
+Flunky
 Foobot
 Freeuploader
-FrontPage
-Fuzz
-FyberSpider
-Fyrebot
-GT::WWW
-GermCrawler
+Genieo
 GetWeb
 Getintent
 Gigabot
-Go-Ahead-Got-It
 GoZilla
-GrabNet
-Grafula
+Grabber
+GrapeFX
 HEADMasterSEO
-HTMLparser
-HTTP::Lite
-Haansoft
-Heritrix
+HTTrack
+HaosouSpider
+Havij
 HybridBot
 IDBTE4M
-Id-search
-Image Fetch
-IndeedBot
-Indy Library
+IDBot
+IRLbot
+IlseBot
 InfoTekies
-Intelliseek
-Internet Ninja
-Iria
-IstellaBot
-JamesBOT
-Jorgee
+InterGET
+Jbrofuzz
+JikeSpider
+JustView
+Jyxobot
+Kenjin Spider
 Kinza
-Kozmosbot
-LWP::Simple
+LNSpiderguy
+Larbin
 LeechFTP
 LeechGet
 Likse
 LinkScan
-Linkbot
-Linkdexbot
 LinkextractorPro
-LinkpadBot
-LinqiaRSSBot
-LinqiaScrapeBot
-Lipperhey
-Lipperhey Spider
-MIIxpc
+LinqiaMetadataDownloaderBot
+Litemage_walker
+MIDown tool
 MJ12bot
 MSIECrawler
 MTRobot
-Mag-Net
 Magnet
-Mail.RU_Bot
-MauiBot
+Majestic SEO
+Majestic-SEO
+MarkMonitor
+MarkWatch
+Mata Hari
+MicroMessenger
+Microsoft Data Access
+Microsoft URL Control
 Moblie Safari
-Mozlila
-Mr.4x3
-Musobot
-NearSite
+Mojeek
+NPbot
+Name Intelligence
+Needle
 Nessus
-Net Vampire
-NetLyzer
-Nettrack
+NetSpider
+Netcraft
 NextGenSearchBot
 Nibbler
-Niki-bot
-Nikto
-NimbleCrawler
-Ninja
-Nutch
-OpenLinkProfiler
-OutfoxBot
+Nuclei
+Offline Explorer
+Offline Navigator
+Openfind
+Openvas
+OrangeSpider
 PHPCrawl
 Page Analyzer
-Pandalytics
+PageAnalyzer
+PageGrabber
 Panscient
 Papa Foto
 Pavuk
-PeoplePal
 Pi-Monster
-Picscout
-ProWebWalker
-Psbot
-Pu_iN
-PxBroker
-PyCurl
-RSSingBot
-RankActiveLinkBot
-RankingBot
+PictureFinder
+Pimonster
+ProPowerBot
+Pump
+QueryN Metasearch
+Quick-Crawler
+RankFlex
+RankingBot2
+Rankivabot
 RankurBot
+ReGet
 RedesScrapy
+Rogerbot
 SBIder
 SEOkicks
+SEOkicks-Robot
 SEOprofiler
+SEOstats
 SISTRIX
-ScanAlert
 ScoutJet
 Scrapy
 ScreenerBot
 ScrepyBot
-SeoSiteCheckup
+Searchestate
+Semrush
 SeobilityBot
 Seomoz
+Shodan
 Site Sucker
 SiteLockSpider
+SiteSnagger
 Siteimprove
-Snapbot
-Sociscraper
-Sottopop
-Spanner
+SocialRankIOBot
+SpaceBison
+Spammen
 Spinn3r
-SputnikBot
+Sqworm
 Steeler
 Stripper
-Sucker
 SuperHTTP
 Suzuran
 T0PHackTeam
+T8Abot
 Teleport
-TeleportPro
+Telesphoreo
 Telesphorep
-TheNomad
-Titan
-Tracemyfile
+Thumbor
+TightTwatBot
 Trendiction
 Trendictionbot
+Turnitin
 URLy Warning
 Upflow
-V-BOT
+Vagabondo
 VelenPublicWebCrawler
-VidibleScraper
-Voil
+Virusdie
+Voltron
 WBSearchBot
-WWW::Mechanize
+WWWOFFLE
+Wallpapers
+Wallpapers/3.0
+WallpapersHD
+WeSEE
 Web Auto
-Web Fuck
-Web Pix
+Web Fetch
 WebCollage
 WebCopier
 WebFetch
-WebFuck
-WebGo IS
-WebImageCollector
-WebPix
-WebSauger
+WebLeacher
+WebStripper
+WebSucker
 WebWhacker
-WebZIP
 Webalta
+WebmasterWorldForumBot
 WebsiteExtractor
-Webster
+Whack
 Whacker
-Whatweb
-Wonderbot
-Wotbox
+WiseGuys Robot
+Woobot
 Wprecon
 Xaldon_WebSpider
-Zade
-ZoomBot
-ZumBot
+Xenu
+YoudaoBot
+Zitebot
+ZmEu
+archive.org_bot
+arquivo-web-crawler
 arquivo.pt
 autoemailspider
+cah.io.community
+check1.exe
+clark-crawler
 coccocbot-web
 cognitiveseo
 com.plumanalytics
-crawler4j
-evc-batch
+domainsproject.org
+gopher
 heritrix
 instabid
+internetVista monitor
 isitwp.com
-lwp-trivial
-magpie-crawler
+lwp-request
+meanpathbot
+muhstik-scan
+netEstate NE Crawler
 page scorer
-plumanalytics
-polaris version
-probe-image-size
-satoristudio.net
-scalaj-http
+ripz
+scan.lol
 seoscanners
-siteripz
-sp_auditbot
-trendiction.de
-ubermetrics-technologies.com
-webpros.com
+seostar
+sitechecker.pro
+sogouspider
+trendiction.com
+webmeup-crawler
+webprosbot
 x09Mozilla
-x22Mozilla
-xpymep1.exe
-zgrab
+zauba.io