-- -- (C) 2013-18 - ntop.org -- require "lua_utils" require "db_utils" require "historical_utils" require "rrd_paths" local dkjson = require("dkjson") local host_pools_utils = require "host_pools_utils" local os_utils = require "os_utils" local have_nedge = ntop.isnEdge() local ts_utils = require("ts_utils") -- ######################################################## if(ntop.isPro()) then package.path = dirs.installdir .. "/pro/scripts/lua/modules/?.lua;" .. package.path require "nv_graph_utils" end -- ######################################################## local graph_colors = { '#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf', -- https://github.com/mbostock/d3/wiki/Ordinal-Scales '#ff7f0e', '#ffbb78', '#1f77b4', '#aec7e8', '#2ca02c', '#98df8a', '#d62728', '#ff9896', '#9467bd', '#c5b0d5', '#8c564b', '#c49c94', '#e377c2', '#f7b6d2', '#7f7f7f', '#c7c7c7', '#bcbd22', '#dbdb8d', '#17becf', '#9edae5' } -- ######################################################## function queryEpochData(schema, tags, selectedEpoch, zoomLevel, options) if(zoomLevel == nil) then zoomLevel = "1h" end local d = getZoomDuration(zoomLevel) local end_time local start_time if((selectedEpoch == nil) or (selectedEpoch == "")) then selectedEpoch = os.time() end_time = tonumber(selectedEpoch) start_time = end_time-d else end_time = tonumber(selectedEpoch) + d/2 start_time = tonumber(selectedEpoch) - d/2 end return ts_utils.query(schema, tags, start_time, end_time, options) end -- ######################################################## function getProtoVolume(ifName, start_time, end_time) ifId = getInterfaceId(ifName) local series = ts_utils.listSeries("iface:ndpi", {ifid=ifId}, start_time) ret = { } for _, tags in ipairs(series or {}) do -- NOTE: this could be optimized via a dedicated driver call local data = ts_utils.query("iface:ndpi", tags, start_time, end_time) if(data ~= nil) and (data.statistics.total > 0) then ret[tags.protocol] = data.statistics.total end end return(ret) end -- ######################################################## function breakdownBar(sent, sentLabel, rcvd, rcvdLabel, thresholdLow, thresholdHigh) if((sent+rcvd) > 0) then sent2rcvd = round((sent * 100) / (sent+rcvd), 0) -- io.write("****>> "..sent.."/"..rcvd.."/"..sent2rcvd.."\n") if((thresholdLow == nil) or (thresholdLow < 0)) then thresholdLow = 0 end if((thresholdHigh == nil) or (thresholdHigh > 100)) then thresholdHigh = 100 end if(sent2rcvd < thresholdLow) then sentLabel = ' '..sentLabel elseif(sent2rcvd > thresholdHigh) then rcvdLabel = ' '..rcvdLabel end print('
'..sentLabel) print('
' .. rcvdLabel .. '
') else print(' ') end end -- ######################################################## function percentageBar(total, value, valueLabel) -- io.write("****>> "..total.."/"..value.."\n") if((total ~= nil) and (total > 0)) then pctg = round((value * 100) / total, 0) print('
'..valueLabel) print('
') else print(' ') end end -- ######################################################## -- label, relative_difference, seconds zoom_vals = { { "1m", "now-60s", 60}, { "5m", "now-300s", 60*5}, { "30m", "now-1800s", 60*30}, { "1h", "now-1h", 60*60*1}, { "3h", "now-3h", 60*60*3}, { "6h", "now-6h", 60*60*6}, { "12h", "now-12h", 60*60*12}, { "1d", "now-1d", 60*60*24}, { "1w", "now-1w", 60*60*24*7}, { "2w", "now-2w", 60*60*24*14}, { "1M", "now-1mon", 60*60*24*31}, { "6M", "now-6mon", 60*60*24*31*6}, { "1Y", "now-1y", 60*60*24*366} } function getZoomAtPos(cur_zoom, pos_offset) local pos = 1 local new_zoom_level = cur_zoom for k,v in pairs(zoom_vals) do if(zoom_vals[k][1] == cur_zoom) then if (pos+pos_offset >= 1 and pos+pos_offset < 13) then new_zoom_level = zoom_vals[pos+pos_offset][1] break end end pos = pos + 1 end return new_zoom_level end -- ######################################################## function getZoomDuration(cur_zoom) for k,v in pairs(zoom_vals) do if(zoom_vals[k][1] == cur_zoom) then return(zoom_vals[k][3]) end end return(180) end -- ######################################################## local graph_menu_entries = {} function populateGraphMenuEntry(label, base_url, params, tab_id) local url = getPageUrl(base_url, params) local parts = {} parts[#parts + 1] = [[
  • ]] .. label .. [[
  • ]] local entry_str = table.concat(parts, "") local entry_params = table.clone(params) for k, v in pairs(splitUrl(base_url).params) do entry_params[k] = v end local entry = { html = entry_str, label = label, schema = params.ts_schema, params = entry_params, -- for graphMenuGetActive } graph_menu_entries[#graph_menu_entries + 1] = entry return entry end function graphMenuDivider() graph_menu_entries[#graph_menu_entries + 1] = {html='
  • '} end function graphMenuGetActive(schema, params) for _, entry in pairs(graph_menu_entries) do if entry.schema == schema and entry.params then for k, v in pairs(params) do if (k ~= "zoom") and tostring(entry.params[k]) ~= tostring(v) then goto continue end end return entry end ::continue:: end return nil end function printGraphMenuEntries() for _, entry in ipairs(graph_menu_entries) do print(entry.html) end end -- ######################################################## function printSeries(options, tags, start_time, base_url, params) local series = options.timeseries local needs_separator = false for _, serie in ipairs(series) do if (have_nedge and serie.nedge_exclude) or (not have_nedge and serie.nedge_only) then goto continue end if serie.separator then needs_separator = true else local k = serie.schema local v = serie.label local exists = false if starts(k, "custom:") then if not ntop.isPro() then goto continue end -- exists by default, otherwise specify a serie.check below exists = true end if serie.check ~= nil then exists = true for _, serie in pairs(serie.check) do exists = exists and not table.empty(ts_utils.listSeries(serie, tags, start_time)) if not exists then break end end elseif not exists then -- only show if there has been an update within the specified time frame exists = not table.empty(ts_utils.listSeries(k, tags, start_time)) end if exists then if needs_separator then -- Only add the separator if there are actually some entries in the group graphMenuDivider() needs_separator = false end populateGraphMenuEntry(v, base_url, table.merge(params, {ts_schema=k})) end end ::continue:: end -- nDPI applications if options.top_protocols then local schema = split(options.top_protocols, "top:")[2] local proto_tags = table.clone(tags) proto_tags.protocol = nil local series = ts_utils.listSeries(schema, proto_tags, start_time) if not table.empty(series) then graphMenuDivider() local by_protocol = {} for _, serie in pairs(series) do by_protocol[serie.protocol] = 1 end for protocol in pairsByKeys(by_protocol, asc) do local proto_id = protocol populateGraphMenuEntry(protocol, base_url, table.merge(params, {ts_schema=schema, protocol=proto_id})) end end end -- nDPI application categories if options.top_categories then local schema = split(options.top_categories, "top:")[2] local cat_tags = table.clone(tags) cat_tags.category = nil local series = ts_utils.listSeries(schema, cat_tags, start_time) if not table.empty(series) then graphMenuDivider() local by_category = {} for _, serie in pairs(series) do by_category[serie.category] = 1 end for category in pairsByKeys(by_category, asc) do populateGraphMenuEntry(category, base_url, table.merge(params, {ts_schema=schema, category=category})) end end end end -- ######################################################## function getMinZoomResolution(schema) local schema_obj = ts_utils.getSchema(schema) if schema_obj then if schema_obj.options.step >= 300 then return '10m' elseif schema_obj.options.step >= 60 then return '5m' end end return '1m' end -- ######################################################## function drawGraphs(ifid, schema, tags, zoomLevel, baseurl, selectedEpoch, options) local debug_rrd = false options = options or {} if(zoomLevel == nil) then zoomLevel = "5m" end if((selectedEpoch == nil) or (selectedEpoch == "")) then -- Refresh the page every minute unless: -- ** a specific epoch has been selected or -- ** the user is browsing historical top talkers and protocols print[[ ]] end if ntop.isPro() then _ifstats = interface.getStats() drawProGraph(ifid, schema, tags, zoomLevel, baseurl, options) return end nextZoomLevel = zoomLevel; epoch = tonumber(selectedEpoch); for k,v in ipairs(zoom_vals) do if(zoom_vals[k][1] == zoomLevel) then if(k > 1) then nextZoomLevel = zoom_vals[k-1][1] end if(epoch ~= nil) then start_time = epoch - zoom_vals[k][3]/2 end_time = epoch + zoom_vals[k][3]/2 else end_time = os.time() start_time = end_time - zoom_vals[k][3] end end end local data = ts_utils.query(schema, tags, start_time, end_time) if(data) then print [[

    ]] local page_params = { ts_schema = schema, zoom = zoomLevel or '', epoch = selectedEpoch or '', } if(options.timeseries) then print [[
    ]] end -- options.timeseries print(' Timeframe:
    \n') local min_zoom = getMinZoomResolution(schema) for k,v in ipairs(zoom_vals) do -- display 1 minute button only for networks and interface stats -- but exclude applications. Application statistics are gathered -- every 5 minutes if zoom_vals[k][1] == '1m' and min_zoom ~= '1m' then goto continue elseif zoom_vals[k][1] == '5m' and min_zoom ~= '1m' and min_zoom ~= '5m' then goto continue end print('\n') ::continue:: end print [[

    NOTE: Click on the graph to zoom.

    ]] local format_as_bps = true local formatter_fctn local label = data.series[1].label if string.contains(label, "packets") or string.contains(label, "flows") or label:starts("num_") then format_as_bps = false formatter_fctn = "fint" else formatter_fctn = "fbits" end print [[ ]] print(' \n') local stats = data.statistics local minval_time = stats.min_val_idx and (data.start + data.step * stats.min_val_idx) or "" local maxval_time = stats.max_val_idx and (data.start + data.step * stats.max_val_idx) or "" local lastval_time = data.start + data.step * (data.count-1) local lastval = 0 for _, serie in pairs(data.series) do lastval = lastval + serie.data[data.count] end if(not format_as_bps) then print(' \n') print(' \n') print(' \n') print(' \n') print(' \n') print(' \n') else print(' \n') print(' \n') print(' \n') print(' \n') print(' \n') print(' \n') end print(' \n') print(' \n') print [[
     TimeValue
    Min' .. os.date("%x %X", minval_time) .. '' .. formatValue(stats.min_val or "") .. '
    Max' .. os.date("%x %X", maxval_time) .. '' .. formatValue(stats.max_val or "") .. '
    Last' .. os.date("%x %X", lastval_time) .. '' .. formatValue(round(lastval), 1) .. '
    Average' .. formatValue(round(stats.average, 2)) .. '
    95th Percentile' .. formatValue(round(stats["95th_percentile"], 2)) .. '
    Total Number' .. formatValue(round(stats.total)) .. '
    Min' .. os.date("%x %X", minval_time) .. '' .. bitsToSize(stats.min_val or "") .. '
    Max' .. os.date("%x %X", maxval_time) .. '' .. bitsToSize(stats.max_val or "") .. '
    Last' .. os.date("%x %X", lastval_time) .. '' .. bitsToSize(lastval) .. '
    Average' .. bitsToSize(stats.average*8) .. '
    95th Percentile' .. bitsToSize(stats["95th_percentile"]) .. '
    Total Traffic' .. bytesToSize(stats.total) .. '
    Selection Time
    Minute
    Interface
    Top Talkers
    ]] print[[
    ]] if show_historical_tabs then local host = tags.host -- can be nil local l7proto = tags.protocol or "" local k2info = hostkey2hostinfo(host) print('
    ') if tonumber(start_time) ~= nil and tonumber(end_time) ~= nil then -- if both start_time and end_time are vaid epoch we can print finer-grained top flows historicalFlowsTab(ifid, k2info["host"] or '', start_time, end_time, l7proto, '', '', '', k2info["vlan"]) else printGraphTopFlows(ifid, k2info["host"] or '', _GET["epoch"], zoomLevel, l7proto, k2info["vlan"]) end print('
    ') end print[[
    ]] else print("
    No data found
    ") end -- if(data) end function printGraphTopFlows(ifId, host, epoch, zoomLevel, l7proto, vlan) -- Check if the DB is enabled rsp = interface.execSQLQuery("show tables") if(rsp == nil) then return end if((epoch == nil) or (epoch == "")) then epoch = os.time() end local d = getZoomDuration(zoomLevel) epoch_end = epoch epoch_begin = epoch-d historicalFlowsTab(ifId, host, epoch_begin, epoch_end, l7proto, '', '', '', vlan) end -- ################################################# -- -- proto table should contain the following information: -- string traffic_quota -- string time_quota -- string protoName -- -- ndpi_stats or category_stats can be nil if they are not relevant for the proto -- -- quotas_to_show can contain: -- bool traffic -- bool time -- function printProtocolQuota(proto, ndpi_stats, category_stats, quotas_to_show, show_td, hide_limit) local total_bytes = 0 local total_duration = 0 local output = {} if ndpi_stats ~= nil then -- This is a single protocol local proto_stats = ndpi_stats[proto.protoName] if proto_stats ~= nil then total_bytes = proto_stats["bytes.sent"] + proto_stats["bytes.rcvd"] total_duration = proto_stats["duration"] end else -- This is a category local cat_stats = category_stats[proto.protoName] if cat_stats ~= nil then total_bytes = cat_stats["bytes"] total_duration = cat_stats["duration"] end end if quotas_to_show.traffic then local bytes_exceeded = ((proto.traffic_quota ~= "0") and (total_bytes >= tonumber(proto.traffic_quota))) local lb_bytes = bytesToSize(total_bytes) local lb_bytes_quota = ternary(proto.traffic_quota ~= "0", bytesToSize(tonumber(proto.traffic_quota)), i18n("unlimited")) local traffic_taken = ternary(proto.traffic_quota ~= "0", math.min(total_bytes, proto.traffic_quota), 0) local traffic_remaining = math.max(proto.traffic_quota - traffic_taken, 0) local traffic_quota_ratio = round(traffic_taken * 100 / (traffic_taken+traffic_remaining), 0) if show_td then output[#output + 1] = [["..lb_bytes..ternary(hide_limit, "", " / "..lb_bytes_quota).."" end output[#output + 1] = [[
    '.. ternary(traffic_quota_ratio == traffic_quota_ratio --[[nan check]], traffic_quota_ratio, 0)..[[%
    ]] if show_td then output[#output + 1] = ("") end end if quotas_to_show.time then local time_exceeded = ((proto.time_quota ~= "0") and (total_duration >= tonumber(proto.time_quota))) local lb_duration = secondsToTime(total_duration) local lb_duration_quota = ternary(proto.time_quota ~= "0", secondsToTime(tonumber(proto.time_quota)), i18n("unlimited")) local duration_taken = ternary(proto.time_quota ~= "0", math.min(total_duration, proto.time_quota), 0) local duration_remaining = math.max(proto.time_quota - duration_taken, 0) local duration_quota_ratio = round(duration_taken * 100 / (duration_taken+duration_remaining), 0) if show_td then output[#output + 1] = [["..lb_duration..ternary(hide_limit, "", " / "..lb_duration_quota).."" end output[#output + 1] = ([[
    '.. ternary(duration_quota_ratio == duration_quota_ratio --[[nan check]], duration_quota_ratio, 0)..[[%
    ]]) if show_td then output[#output + 1] = ("") end end return table.concat(output, '') end -- ################################################# function poolDropdown(ifId, pool_id, exclude) local output = {} --exclude = exclude or {[host_pools_utils.DEFAULT_POOL_ID]=true} exclude = exclude or {} for _,pool in ipairs(host_pools_utils.getPoolsList(ifId)) do if (not exclude[pool.id]) or (pool.id == pool_id) then output[#output + 1] = '' end end return table.concat(output, '') end function printPoolChangeDropdown(ifId, pool_id, have_nedge) local output = {} output[#output + 1] = [[ ]] .. i18n(ternary(have_nedge, "nedge.user", "host_config.host_pool")) .. [[   ]] .. i18n(ternary(have_nedge, "nedge.edit_users", "host_pools.edit_host_pools")) .. [[ ]] print(table.concat(output, '')) end