--
-- (C) 2013-18 - ntop.org
--
require "lua_utils"
require "db_utils"
require "historical_utils"
require "rrd_paths"
local dkjson = require("dkjson")
local host_pools_utils = require "host_pools_utils"
local os_utils = require "os_utils"
local have_nedge = ntop.isnEdge()
local ts_utils = require("ts_utils")
-- ########################################################
if(ntop.isPro()) then
package.path = dirs.installdir .. "/pro/scripts/lua/modules/?.lua;" .. package.path
require "nv_graph_utils"
end
-- ########################################################
local graph_colors = {
'#1f77b4',
'#ff7f0e',
'#2ca02c',
'#d62728',
'#9467bd',
'#8c564b',
'#e377c2',
'#7f7f7f',
'#bcbd22',
'#17becf',
-- https://github.com/mbostock/d3/wiki/Ordinal-Scales
'#ff7f0e',
'#ffbb78',
'#1f77b4',
'#aec7e8',
'#2ca02c',
'#98df8a',
'#d62728',
'#ff9896',
'#9467bd',
'#c5b0d5',
'#8c564b',
'#c49c94',
'#e377c2',
'#f7b6d2',
'#7f7f7f',
'#c7c7c7',
'#bcbd22',
'#dbdb8d',
'#17becf',
'#9edae5'
}
-- ########################################################
function queryEpochData(schema, tags, selectedEpoch, zoomLevel, options)
if(zoomLevel == nil) then zoomLevel = "1h" end
local d = getZoomDuration(zoomLevel)
local end_time
local start_time
if((selectedEpoch == nil) or (selectedEpoch == "")) then
selectedEpoch = os.time()
end_time = tonumber(selectedEpoch)
start_time = end_time-d
else
end_time = tonumber(selectedEpoch) + d/2
start_time = tonumber(selectedEpoch) - d/2
end
return ts_utils.query(schema, tags, start_time, end_time, options)
end
-- ########################################################
function getProtoVolume(ifName, start_time, end_time)
ifId = getInterfaceId(ifName)
local series = ts_utils.listSeries("iface:ndpi", {ifid=ifId}, start_time)
ret = { }
for _, tags in ipairs(series or {}) do
-- NOTE: this could be optimized via a dedicated driver call
local data = ts_utils.query("iface:ndpi", tags, start_time, end_time)
if(data ~= nil) and (data.statistics.total > 0) then
ret[tags.protocol] = data.statistics.total
end
end
return(ret)
end
-- ########################################################
function breakdownBar(sent, sentLabel, rcvd, rcvdLabel, thresholdLow, thresholdHigh)
if((sent+rcvd) > 0) then
sent2rcvd = round((sent * 100) / (sent+rcvd), 0)
-- io.write("****>> "..sent.."/"..rcvd.."/"..sent2rcvd.."\n")
if((thresholdLow == nil) or (thresholdLow < 0)) then thresholdLow = 0 end
if((thresholdHigh == nil) or (thresholdHigh > 100)) then thresholdHigh = 100 end
if(sent2rcvd < thresholdLow) then sentLabel = ' '..sentLabel
elseif(sent2rcvd > thresholdHigh) then rcvdLabel = ' '..rcvdLabel end
print('
'..sentLabel)
print('
' .. rcvdLabel .. '
')
else
print(' ')
end
end
-- ########################################################
function percentageBar(total, value, valueLabel)
-- io.write("****>> "..total.."/"..value.."\n")
if((total ~= nil) and (total > 0)) then
pctg = round((value * 100) / total, 0)
print('
'..valueLabel)
print('
')
else
print(' ')
end
end
-- ########################################################
-- label, relative_difference, seconds, graph_tick_step
zoom_vals = {
{ "1m", "now-60s", 60, (60)/12 },
{ "5m", "now-300s", 60*5, (60*5)/10 },
{ "10m", "now-600s", 60*10, (60*10)/10 },
{ "1h", "now-1h", 60*60*1, (60*60*1)/12 },
{ "3h", "now-3h", 60*60*3, (60*60*3)/12 },
{ "6h", "now-6h", 60*60*6, (60*60*6)/12 },
{ "12h", "now-12h", 60*60*12, (60*60*12)/12 },
{ "1d", "now-1d", 60*60*24, (60*60*24)/12 },
{ "1w", "now-1w", 60*60*24*7, (60*60*24*7)/7 },
{ "2w", "now-2w", 60*60*24*14, (60*60*24*14)/14 },
{ "1M", "now-1mon", 60*60*24*31, (60*60*24*31)/15 },
{ "6M", "now-6mon", 60*60*24*31*6, (60*60*24*31*6)/18 },
{ "1Y", "now-1y", 60*60*24*366, (60*60*24*366)/12 }
}
function getZoomAtPos(cur_zoom, pos_offset)
local pos = 1
local new_zoom_level = cur_zoom
for k,v in pairs(zoom_vals) do
if(zoom_vals[k][1] == cur_zoom) then
if (pos+pos_offset >= 1 and pos+pos_offset < 13) then
new_zoom_level = zoom_vals[pos+pos_offset][1]
break
end
end
pos = pos + 1
end
return new_zoom_level
end
-- ########################################################
function getZoomDuration(cur_zoom)
for k,v in pairs(zoom_vals) do
if(zoom_vals[k][1] == cur_zoom) then
return(zoom_vals[k][3])
end
end
return(180)
end
-- ########################################################
function getZoomTicksInterval(cur_zoom)
for k,v in pairs(zoom_vals) do
if(zoom_vals[k][1] == cur_zoom) then
return(zoom_vals[k][4])
end
end
return(12)
end
-- ########################################################
function getZoomTicksJsArray(start_time, end_time, zoom)
local parts = {}
local step = getZoomTicksInterval(zoom)
for t=start_time,end_time,step do
parts[#parts+1] = t
end
return "[" .. table.concat(parts, ', ') .. "]"
end
-- ########################################################
local graph_menu_entries = {}
function populateGraphMenuEntry(label, base_url, params, tab_id)
local url = getPageUrl(base_url, params)
local parts = {}
parts[#parts + 1] = [[
]]
local entry_str = table.concat(parts, "")
local entry_params = table.clone(params)
for k, v in pairs(splitUrl(base_url).params) do
entry_params[k] = v
end
graph_menu_entries[#graph_menu_entries + 1] = {
html = entry_str,
label = label,
schema = params.ts_schema,
params = entry_params, -- for graphMenuGetTitle
}
end
function graphMenuDivider()
graph_menu_entries[#graph_menu_entries + 1] = {html=''}
end
function graphMenuGetTitle(schema, params)
for _, entry in pairs(graph_menu_entries) do
if entry.schema == schema and entry.params then
for k, v in pairs(params) do
if tostring(entry.params[k]) ~= tostring(v) then
goto continue
end
end
return entry.label
end
::continue::
end
return i18n("prefs.timeseries")
end
function printGraphMenuEntries()
for _, entry in ipairs(graph_menu_entries) do
print(entry.html)
end
end
-- ########################################################
function printSeries(options, tags, start_time, base_url, params)
local series = options.timeseries
local needs_separator = false
for _,top in ipairs(series) do
if (have_nedge and top.nedge_exclude) or (not have_nedge and top.nedge_only) then
goto continue
end
if top.separator then
needs_separator = true
else
local k = top.schema
local v = top.label
-- only show if there has been an update within the specified time frame
local res = ts_utils.listSeries(k, tags, start_time)
if not table.empty(res) then
if needs_separator then
-- Only add the separator if there are actually some entries in the group
graphMenuDivider()
needs_separator = false
end
populateGraphMenuEntry(v, base_url, table.merge(params, {ts_schema=k}))
end
end
::continue::
end
-- nDPI applications
if options.top_protocols then
local schema = split(options.top_protocols, "top:")[2]
local proto_tags = table.clone(tags)
proto_tags.protocol = nil
local series = ts_utils.listSeries(schema, proto_tags, start_time)
if not table.empty(series) then
graphMenuDivider()
local by_protocol = {}
for _, serie in pairs(series) do
by_protocol[serie.protocol] = 1
end
for protocol in pairsByKeys(by_protocol, asc) do
local proto_id = protocol
populateGraphMenuEntry(protocol, base_url, table.merge(params, {ts_schema=schema, protocol=proto_id}))
end
end
end
-- nDPI application categories
if options.top_categories then
local schema = split(options.top_categories, "top:")[2]
local cat_tags = table.clone(tags)
cat_tags.category = nil
local series = ts_utils.listSeries(schema, cat_tags, start_time)
if not table.empty(series) then
graphMenuDivider()
local by_category = {}
for _, serie in pairs(series) do
by_category[serie.category] = 1
end
for category in pairsByKeys(by_category, asc) do
populateGraphMenuEntry(category, base_url, table.merge(params, {ts_schema=schema, category=category}))
end
end
end
end
-- ########################################################
function getMinZoomResolution(schema)
local schema_obj = ts_utils.getSchema(schema)
if schema_obj then
if schema_obj.options.step >= 300 then
return '10m'
elseif schema_obj.options.step >= 60 then
return '5m'
end
end
return '1m'
end
-- ########################################################
function drawGraphs(ifid, schema, tags, zoomLevel, baseurl, selectedEpoch, options)
local debug_rrd = false
options = options or {}
if(zoomLevel == nil) then zoomLevel = "1h" end
if((selectedEpoch == nil) or (selectedEpoch == "")) then
-- Refresh the page every minute unless:
-- ** a specific epoch has been selected or
-- ** the user is browsing historical top talkers and protocols
print[[
]]
end
if ntop.isPro() then
_ifstats = interface.getStats()
drawProGraph(ifid, schema, tags, zoomLevel, baseurl, selectedEpoch, options)
return
end
nextZoomLevel = zoomLevel;
epoch = tonumber(selectedEpoch);
for k,v in ipairs(zoom_vals) do
if(zoom_vals[k][1] == zoomLevel) then
if(k > 1) then
nextZoomLevel = zoom_vals[k-1][1]
end
if(epoch ~= nil) then
start_time = epoch - zoom_vals[k][3]/2
end_time = epoch + zoom_vals[k][3]/2
else
end_time = os.time()
start_time = end_time - zoom_vals[k][3]
end
end
end
local data = ts_utils.query(schema, tags, start_time, end_time)
if(data) then
print [[
\n')
local min_zoom = getMinZoomResolution(schema)
for k,v in ipairs(zoom_vals) do
-- display 1 minute button only for networks and interface stats
-- but exclude applications. Application statistics are gathered
-- every 5 minutes
if zoom_vals[k][1] == '1m' and min_zoom ~= '1m' then
goto continue
elseif zoom_vals[k][1] == '5m' and min_zoom ~= '1m' and min_zoom ~= '5m' then
goto continue
end
print('\n')
::continue::
end
print [[
NOTE: Click on the graph to zoom.
]]
local format_as_bps = true
local formatter_fctn
local label = data.series[1].label
if string.contains(label, "packets") or string.contains(label, "flows") or label:starts("num_") then
format_as_bps = false
formatter_fctn = "fint"
else
formatter_fctn = "fbits"
end
print [[
]]
print('
Time
Value
\n')
local stats = data.statistics
local minval_time = stats.min_val_idx and (data.start + data.step * stats.min_val_idx) or ""
local maxval_time = stats.max_val_idx and (data.start + data.step * stats.max_val_idx) or ""
local lastval_time = data.start + data.step * (data.count-1)
local lastval = 0
for _, serie in pairs(data.series) do
lastval = lastval + serie.data[data.count]
end
if(not format_as_bps) then
print('
]]
if show_historical_tabs then
local host = tags.host -- can be nil
local l7proto = tags.protocol or ""
local k2info = hostkey2hostinfo(host)
print('
')
if tonumber(start_time) ~= nil and tonumber(end_time) ~= nil then
-- if both start_time and end_time are vaid epoch we can print finer-grained top flows
historicalFlowsTab(ifid, k2info["host"] or '', start_time, end_time, l7proto, '', '', '', k2info["vlan"])
else
printGraphTopFlows(ifid, k2info["host"] or '', _GET["epoch"], zoomLevel, l7proto, k2info["vlan"])
end
print('
')
end
print[[
]]
else
print("
No data found
")
end -- if(data)
end
function printGraphTopFlows(ifId, host, epoch, zoomLevel, l7proto, vlan)
-- Check if the DB is enabled
rsp = interface.execSQLQuery("show tables")
if(rsp == nil) then return end
if((epoch == nil) or (epoch == "")) then epoch = os.time() end
local d = getZoomDuration(zoomLevel)
epoch_end = epoch
epoch_begin = epoch-d
historicalFlowsTab(ifId, host, epoch_begin, epoch_end, l7proto, '', '', '', vlan)
end
-- #################################################
--
-- proto table should contain the following information:
-- string traffic_quota
-- string time_quota
-- string protoName
--
-- ndpi_stats or category_stats can be nil if they are not relevant for the proto
--
-- quotas_to_show can contain:
-- bool traffic
-- bool time
--
function printProtocolQuota(proto, ndpi_stats, category_stats, quotas_to_show, show_td, hide_limit)
local total_bytes = 0
local total_duration = 0
local output = {}
if ndpi_stats ~= nil then
-- This is a single protocol
local proto_stats = ndpi_stats[proto.protoName]
if proto_stats ~= nil then
total_bytes = proto_stats["bytes.sent"] + proto_stats["bytes.rcvd"]
total_duration = proto_stats["duration"]
end
else
-- This is a category
local cat_stats = category_stats[proto.protoName]
if cat_stats ~= nil then
total_bytes = cat_stats["bytes"]
total_duration = cat_stats["duration"]
end
end
if quotas_to_show.traffic then
local bytes_exceeded = ((proto.traffic_quota ~= "0") and (total_bytes >= tonumber(proto.traffic_quota)))
local lb_bytes = bytesToSize(total_bytes)
local lb_bytes_quota = ternary(proto.traffic_quota ~= "0", bytesToSize(tonumber(proto.traffic_quota)), i18n("unlimited"))
local traffic_taken = ternary(proto.traffic_quota ~= "0", math.min(total_bytes, proto.traffic_quota), 0)
local traffic_remaining = math.max(proto.traffic_quota - traffic_taken, 0)
local traffic_quota_ratio = round(traffic_taken * 100 / (traffic_taken+traffic_remaining), 0)
if show_td then
output[#output + 1] = [[
") end
end
if quotas_to_show.time then
local time_exceeded = ((proto.time_quota ~= "0") and (total_duration >= tonumber(proto.time_quota)))
local lb_duration = secondsToTime(total_duration)
local lb_duration_quota = ternary(proto.time_quota ~= "0", secondsToTime(tonumber(proto.time_quota)), i18n("unlimited"))
local duration_taken = ternary(proto.time_quota ~= "0", math.min(total_duration, proto.time_quota), 0)
local duration_remaining = math.max(proto.time_quota - duration_taken, 0)
local duration_quota_ratio = round(duration_taken * 100 / (duration_taken+duration_remaining), 0)
if show_td then
output[#output + 1] = [[
") end
end
return table.concat(output, '')
end
-- #################################################
function poolDropdown(ifId, pool_id, exclude)
local output = {}
--exclude = exclude or {[host_pools_utils.DEFAULT_POOL_ID]=true}
exclude = exclude or {}
for _,pool in ipairs(host_pools_utils.getPoolsList(ifId)) do
if (not exclude[pool.id]) or (pool.id == pool_id) then
output[#output + 1] = ''
end
end
return table.concat(output, '')
end
function printPoolChangeDropdown(ifId, pool_id, have_nedge)
local output = {}
output[#output + 1] = [[