V4.2021.08.2561 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-404checker
-404enemy
-80legs
-AIBOT
-Abonti
-AhrefsBot
-AiHitBot
-Aipbot
-Alexibot
-AllSubmitter
-AlphaBot
+ADmantX
+AfD-Verbotsverfahren
+Alligator
 Anarchy
-Apexoo
-AwarioRssBot
-BLEXBot
+Asterias
+AwarioSmartBot
+BDCbot
 BackStreet
-Backlink-Ceck
-Badass
+Bandit
 Barkrowler
 BatchFTP
+Battleztar Bazinga
 Bigfoot
+BlackWidow
 Blackboard
-BlowFish
+Blow
+BotALot
 Brandprotect
-Bullseye
+Brandwatch
+Buck
+Buddy
+BuiltBotTough
 BunnySlippers
-CODE87
-CSHttp
-Calculon
-Cegbfeieh
-CensysInspect
+BuzzSumo
+CazoodleBot
 CherryPicker
-Chlooe
+ChinaClaw
+Claritybot
+Cliqzbot
 Cloud mapping
 Cogentbot
-Copyscape
-Craftbot
-Crescent
+Copier
+Cosmos
+Crawling at Home Project
+CrazyWebCrawler
 CrunchBot
+Curious
+Custo
+DIIbot
+DSearch
 DTS Agent
 DataCha0s
 Demon
-Deusu
-Dirbuster
-Discobot
-Dispatch
-DittoSpyder
-DomainCrawler
-DomainSigmaCrawler
-DomainStatsBot
+DigitalPebble
+Discoverybot
+Dotbot
+Dragonfly
 Drip
-ECCP/1.0
 EMail Siphon
-EMail Wolf
-Ebingbong
-Ecxi
-EroCrawler
-ExtractorPro
-EyeNetIE
-Foobot
+EirGrabber
+Evil
+FHscan
+Flunky
+Freeuploader
+FrontPage
 GT::WWW
+GalaxyBot
 GetRight
-GetWeb
 Gigabot
-Go-Ahead-Got-It
-GoZilla
 Gotit
-HEADMasterSEO
-HTMLparser
-HTTP::Lite
-HTTrack
+GrabNet
+HMView
 HaosouSpider
-Havij
-Hloader
-IDBTE4M
-Id-search
-Image Fetch
-Indy Library
-InfoNaviRobot
-Internet Ninja
+Heritrix
+Humanlinks
+HybridBot
+IDBot
+IRLbot
+InfoTekies
+Intelliseek
+InterGET
 Iria
 Iskanie
 IstellaBot
-JOC Web Spider
 JamesBOT
-JennyBot
-Jetty
-Kenjin Spider
+JikeSpider
 Kinza
-LNSpiderguy
+Kozmosbot
+Lanshanbot
+LeechFTP
 LexiBot
-LieBaoFast
-Lightspeedsystems
-Linkdexbot
-LinkextractorPro
-LinqiaScrapeBot
-Lipperhey
-Litemage_walker
-Ltx71
-MFC_Tear_Sample
-MJ12bot
-MQQBrowser
-MSFrontPage
-MSIECrawler
-MTRobot
+LibWeb
+LinkpadBot
+LinqiaMetadataDownloaderBot
+Lmspider
+MIDown tool
+Mag-Net
+Mail.RU_Bot
 Majestic-SEO
+Majestic12
 MarkMonitor
-Mass Downloader
-Mata Hari
 MauiBot
+MeanPath Bot
+MegaIndex.ru
 Metauri
 MicroMessenger
-Mister PiX
-Moblie Safari
-Mozlila
+Mojolicious
 Mr.4x3
-Msrabot
 NICErsPRO
-Name Intelligence
+NPbot
+Nameprotect
+NearSite
 Needle
+Nessus
+NetMechanic
 NetSpider
-NetZIP
-Netcraft
-Netvibes
 Nibbler
-Nmap
+Niki-bot
+Nimbostratus
+Ninja
 Nutch
 Offline Explorer
-OnCrawl
-OpenLinkProfiler
 OpenVAS
 Openfind
+Openvas
+OrangeBot
 OrangeSpider
 OutclicksBot
+OutfoxBot
+PECL::HTTP
+POE-Component-Client-HTTP
+PageGrabber
+PageScorer
 PageThing.com
-Panscient
+Pandalytics
 Papa Foto
+Pavuk
 Petalbot
-Picscout
-PictureFinder
-Pixray
+Pi-Monster
+Picsearch
+Piepmatz
 PleaseCrawl
+ProWebWalker
+Psbot
 Pu_iN
 PxBroker
 PyCurl
 QueryN Metasearch
 RSSingBot
-RankActiveLinkBot
 RankingBot
-RankurBot
+RankingBot2
 Re-re
+ReGet
 RealDownload
-Reaper
+RebelMouse
 Ripper
+RocketCrawler
 SEOkicks
-SEOkicks-Robot
-SEOprofiler
-Screaming
+SISTRIX
+SMTBot
+Scrapy
+ScrepyBot
 Searchestate
-SemrushBot
-Shodan
-SiteCheckerBotCrawler
-SiteExplorer
-SiteLockSpider
+Semrush
+SentiBot
+SeoSiteCheckup
+Seomoz
+SiteSucker
 SmartDownload
-Snapbot
-Sogou web spider
-Sosospider
+SocialRankIOBot
+Sociscraper
 Sottopop
+SpankBot
 Spanner
-Spbot
 Spinn3r
-Sqlmap
-Sqlworm
+Stripper
+Sucker
+SuperBot
+SuperHTTP
+Surfbot
+Suzuran
+Swiftbot
+Szukacz
 T0PHackTeam
-Teleport
-TeleportPro
 Telesphoreo
-Telesphorep
-The Intraformant
 TightTwatBot
+Toata
 Toweyabot
-Tracemyfile
 Trendictionbot
-Turingos
-Turnitin
-Twice
-URLy Warning
-Upflow
+True_Robot
+TurnitinBot
+TwengaBot
+UnisterBot
 V-BOT
-VidibleScraper
-Voil
+VelenPublicWebCrawler
 WASALive-Bot
+WPScan
+WWW::Mechanize
+WWWOFFLE
 Wallpapers
 Wallpapers/3.0
-WallpapersHD
-Web Auto
+Web Collage
 Web Enhancer
+Web Fetch
 Web Fuck
-Web Pix
 Web Sauger
 Web Sucker
-WebBandit
+WebCollage
+WebCopier
+WebEnhancer
 WebFuck
 WebImageCollector
-WebSucker
-WebZIP
+Webalta
+Webshag
+Website Quester
 WebsiteQuester
 Webster
-Whatweb
+Whacker
 Who.is Bot
+Widow
 Wonderbot
-Wotbox
 Wprecon
+Xaldon WebSpider
 Xaldon_WebSpider
-Zade
+Xenu
 Zermelo
-Zeus
-archive.org_bot
+Zitebot
+ZumBot
 arquivo.pt
-clark-crawler
-cognitiveseo
-crawl.sogou.com
-crawler.feedback
+autoemailspider
+backlink-check
+cah.io.community
+coccocbot-web
+com.plumanalytics
 dataforseo.com
-demandbase-bot
 domainsproject.org
+gopher
 heritrix
-internetVista monitor
+ips-agent
 isitwp.com
 lwp-trivial
-magpie-crawler
+meanpathbot
 mediawords
+oBot
 page scorer
-pcBrowser
-ripz
-satoristudio.net
-scalaj-http
-seobility
+probe-image-size
+s1z.ru
+seoscanners
+seostar
 serpstatbot
 sexsearcher
+siteripz
+spyfu
 sysscan
-tAkeOut
-trendiction.com
-webpros.com
+trendiction.de
+ubermetrics-technologies.com
 webprosbot
-x09Mozilla
 x22Mozilla
 zauba.io