V4.2021.09.2578 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404checker
+360Spider
+404enemy
+AIBOT
+ALittle Client
 ASPSeek
-Abonti
 Aboundex
-AfD-Verbotsverfahren
+Aboundexbot
+Acunetix
 AhrefsBot
-AiHitBot
-Aipbot
+Alexibot
 AlphaBot
-Apexoo
-Aspiegel
+Ankit
 Attach
-AwarioRssBot
 BBBike
+BLEXBot
 BackStreet
-BackWeb
-Backlink-Ceck
-BacklinkCrawler
-BatchFTP
+Badass
+Battleztar Bazinga
+BetaBot
+Bigfoot
+Bitacle
+Black Hole
+BlackWidow
+Blackboard
+Blow
 BlowFish
-BotALot
+Boardreader
+Bolt
 Brandprotect
-Brandwatch
-BunnySlippers
-BuzzSumo
+Buddy
+BuiltBotTough
+BuiltWith
 CATExplorador
+CODE87
+Calculon
 CazoodleBot
-CensysInspect
-CheeseBot
-Cocolyzebot
-CopyRightCheck
-Cosmos
-Crawling at Home Project
-CrazyWebCrawler
-Curious
-DTS Agent
-DataCha0s
+Cegbfeieh
+CheTeam
+CherryPicker
+ChinaClaw
+Cloud mapping
+Copyscape
+CrunchBot
+Custo
+DBLBot
+DSearch
+Demon
 Deusu
+Devil
 Digincore
-DigitalPebble
 Disco
 Discoverybot
 DittoSpyder
-DomainSigmaCrawler
+DnyzBot
+DomCopBot
+DomainAppender
+DomainCrawler
 DomainStatsBot
-Domains Project
-Download Wonder
+Drip
 ECCP/1.0
+EasyDL
 Ebingbong
+Ecxi
 EirGrabber
-Express WebPictures
+Evil
 Extractor
-Extreme Picture Finder
-FemtosearchBot
-Fimap
-Firefox/7.0
-Flunky
+EyeNetIE
+FHscan
 Foobot
 FrontPage
-G-i-g-a-b-o-t
+FyberSpider
 GT::WWW
-GetRight
-GetWeb
-Gigabot
+GalaxyBot
 Go!Zilla
+Go-Ahead-Got-It
 GoZilla
 GrabNet
-GrapeshotCrawler
-HEADMasterSEO
-HTTrack
-Harvest
-Heritrix
-HybridBot
+GridBot
+HTTP::Lite
+Havij
+IDBTE4M
+Image Sucker
 Indy Library
-InfoTekies
 InterGET
+Iria
 Iskanie
-Jbrofuzz
-JennyBot
 Jetty
 JikeSpider
-Jorgee
 JustView
+Jyxobot
+Kenjin Spider
+Keyword Density
 Kozmosbot
-LNSpiderguy
-LeechFTP
-LexiBot
+Lanshanbot
+Larbin
+LeechGet
+LibWeb
 Libwhisker
-Likse
-Linkbot
+LieBaoFast
+Lightspeedsystems
+LinkWalker
+Linkdexbot
 LinkextractorPro
+LinqiaRSSBot
 LinqiaScrapeBot
-Lipperhey
-Lipperhey Spider
-MIDown tool
+Litemage_walker
 MIIxpc
-MTRobot
+MJ12bot
+MQQBrowser
+MSIECrawler
 Mag-Net
-Mail.RU_Bot
 Majestic-SEO
+Majestic12
+MarkWatch
 Mass Downloader
-Mb2345Browser
-MeanPath Bot
+Masscan
+Mata Hari
 Mediatoolkitbot
-Microsoft URL Control
-Mojeek
-Musobot
-NICErsPRO
-NPbot
+Mister PiX
+Moblie Safari
 Nameprotect
-Navroad
-NearSite
-NetLyzer
+Needle
+Nessus
+Net Vampire
+NetAnts
 NetMechanic
-NetSpider
-NetZIP
+Netcraft
 Nettrack
 NextGenSearchBot
 Nibbler
-Nikto
-Nimbostratus
+Niki-bot
+NimbleCrawler
+Nutch
 Octopus
-OnCrawl
+Offline Explorer
+Offline Navigator
+OpenLinkProfiler
+Openvas
 OrangeBot
+OrangeSpider
 OutclicksBot
-Page Analyzer
-PageAnalyzer
-PageThing.com
+Panscient
 Pi-Monster
-ProPowerBot
-ProWebWalker
-Psbot
+Picscout
+Pimonster
+Pixray
 Pu_iN
-PyCurl
-QueryN Metasearch
+Pump
 Quick-Crawler
-RSSingBot
+RankActive
 RankActiveLinkBot
 RankFlex
-RankingBot
+RankingBot2
+RankurBot
 Reaper
+Recorder
+RedesScrapy
 RepoMonkey
-Rogerbot
+Ripper
+RocketCrawler
 SEOkicks
+SEOkicks-Robot
 SEOstats
-SISTRIX
+SalesIntelligent
 ScanAlert
-Screaming
-ScrepyBot
-SearchmetricsBot
+Scrapy
+ScreenerBot
 Semrush
+SentiBot
 Seomoz
-SiteCheckerBotCrawler
+Siphon
 SiteExplorer
-SiteLockSpider
 SiteSucker
 Siteimprove
-SlySearch
-SmartDownload
-Snapbot
+Sitevigil
 SocialRankIOBot
+Sociscraper
+Sogou web spider
 Sosospider
+Sottopop
 SpaceBison
-Spinn3r
-Sqlworm
+Spbot
+Sucker
 Sucuri
 Surfbot
+Swiftbot
 Szukacz
 T0PHackTeam
 T8Abot
-Telesoft
-Telesphoreo
+Teleport
+TeleportPro
 The Intraformant
-TheNomad
-Toata
+Titan
 Toweyabot
-Tracemyfile
-Trendictionbot
-True_Robot
-Turingos
-TurnitinBot
+TwengaBot
+Twice
 URLy Warning
-UnisterBot
-Upflow
-VB Project
-Vacuum
+URLy.Warning
 VelenPublicWebCrawler
-VidibleScraper
-Virusdie
-Voil
+VoidEYE
 WASALive-Bot
-WPScan
+WWW-Collector-E
 WWW-Mechanize
 WWW::Mechanize
-Wallpapers/3.0
-WeSEE
-Web Enhancer
 Web Fetch
 Web Fuck
-Web Pix
 Web Sucker
-WebEnhancer
+WebBandit
+WebCopier
 WebFetch
 WebFuck
+WebGo IS
 WebImageCollector
-WebReaper
-WebSucker
-WebZIP
-WebsiteExtractor
-WebsiteQuester
+WebLeacher
+WebPix
+WebStripper
+WebWhacker
+Webalta
+WebmasterWorldForumBot
 Webster
-WinHTTrack
-WiseGuys Robot
-Wonderbot
-Woobot
-Xaldon WebSpider
+Whack
+Who.is Bot
+Wotbox
 Xaldon_WebSpider
 Xenu
-YoudaoBot
-Zitebot
 ZmEu
-ZoominfoBot
-ZumBot
-archive.org_bot
-autoemailspider
-cah.io.community
-check1.exe
+ZyBorg
+backlink-check
+clark-crawler
 cognitiveseo
 com.plumanalytics
-crawl.sogou.com
 crawler.feedback
-dataforseo.com
 domainsproject.org
-gopher
+evc-batch
 heritrix
-ips-agent
+instabid
+internetVista monitor
 lwp-request
-lwp-trivial
-mediawords
+meanpathbot
+netEstate NE Crawler
 pcBrowser
 s1z.ru
 scalaj-http
-sitechecker.pro
-spyfu
-webmeup-crawler
-x09Mozilla
-x22Mozilla
-xpymep1.exe
+seoscanners
+serpstatbot
+sysscan
+trendiction.com
+trendiction.de
+ubermetrics-technologies.com
 zauba.io
-zgrab