Reworked assets table

This commit is contained in:
Matteo Biscosi 2024-12-30 12:37:05 +01:00
parent 24c3745810
commit a63dec0dcc
17 changed files with 544 additions and 486 deletions

View file

@ -6,141 +6,49 @@ package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path
require "ntop_utils"
require "check_redis_prefs"
local os_utils = require "os_utils"
local json = require "dkjson"
-- ##############################################
local asset_management_utils = {}
local table_name = "asset_management"
local table_name = "assets"
-- ##############################################
function get_mac_serialization_key(mac, ifid)
return tostring(ifid) .. "_" .. mac
end
-- @brief insert assetkey
function asset_management_utils.insert_host(entry)
local insert_host = ""
if hasClickHouseSupport() then
insert_host = string.format(
"INSERT INTO %s " ..
"(type, key, ip, mac, vlan, network, name, device_type, manufacturer, first_seen, last_seen) " ..
"SELECT '%s','%s','%s','%s', %u, %u, %s, %u, %s, %u, %u "..
"WHERE NOT EXISTS ( SELECT 1 FROM %s WHERE key = '%s' )",
table_name,
entry["type"],
entry["key"],
entry["ip"] or "",
entry["mac"] or "",
entry["vlan"] or 0,
entry["network"] or 0,
ternary(not isEmptyString(entry["name"]),string.format("'%s'",entry["name"]),"NULL"),
entry["device_type"],
ternary(not isEmptyString(entry["manufacturer"]),string.format("'%s'",entry["manufacturer"]), "NULL"),
entry["first_seen"],
entry["last_seen"],
table_name,
entry["key"]
)
local update_host = string.format("ALTER TABLE `%s` UPDATE `last_seen` = %u WHERE `key`='%s'",
table_name,
entry["last_seen"],
entry["key"]
)
interface.alert_store_query(insert_host)
return interface.alert_store_query(update_host)
else
insert_host = string.format(
"INSERT INTO %s " ..
"(type, key, ip, mac, vlan, network, name, device_type, manufacturer, first_seen, last_seen) " ..
"VALUES ('%s','%s','%s','%s', %u, %u, %s, %u, %s, %u, %u) "..
"ON CONFLICT(key) DO UPDATE SET last_seen = %u;",
table_name,
entry["type"],
entry["key"],
entry["ip"],
entry["mac"] or "",
entry["vlan"] or 0,
entry["network"] or 0,
ternary(not isEmptyString(entry["name"]),string.format("'%s'",entry["name"]),"NULL"),
entry["device_type"],
ternary(not isEmptyString(entry["manufacturer"]),string.format("'%s'",entry["manufacturer"]), "NULL"),
entry["first_seen"],
entry["last_seen"],
entry["last_seen"]
)
-- traceError(TRACE_NORMAL, TRACE_CONSOLE, insert_host)
return interface.alert_store_query(insert_host)
local function getAssetInfo(ifid, key, type)
if isEmptyString(key) then
return nil
end
end
function asset_management_utils.insert_mac(entry)
if hasClickHouseSupport() then
local insert_mac = string.format(
"INSERT INTO %s " ..
"(type, key, mac, manufacturer, vlan, device_type, first_seen, last_seen, device_status, trigger_alert) " ..
"SELECT '%s','%s','%s','%s','%d', %u, %u, %u, '%s', %u "..
"WHERE NOT EXISTS ( SELECT 1 FROM %s WHERE key = '%s' )",
table_name,
entry["type"],
entry["key"],
entry["mac"],
entry["manufacturer"],
0, -- VLAN
entry["device_type"],
entry["first_seen"],
entry["last_seen"],
"allowed",
0,
table_name,
entry["mac"]
)
local update_mac = string.format("ALTER TABLE `%s` UPDATE `last_seen` = %u WHERE `key`='%s'",
table_name,
entry["last_seen"],
entry["key"]
)
--tprint(insert_mac)
interface.alert_store_query(insert_mac)
return interface.alert_store_query(update_mac)
else
local insert_mac = string.format(
"INSERT INTO %s " ..
"(type, key, mac, manufacturer, vlan, device_type, first_seen, last_seen, device_status, trigger_alert) " ..
"VALUES ('%s','%s','%s','%s', %u, %u, %u, %u, '%s', %u) "..
"ON CONFLICT(key) DO UPDATE SET last_seen = %u ;",
table_name,
entry["type"],
entry["key"],
entry["mac"],
entry["manufacturer"],
0,
entry["device_type"],
entry["first_seen"],
entry["last_seen"],
"allowed",
0,
entry["last_seen"]
)
return interface.alert_store_query(insert_mac)
end
-- traceError(TRACE_NORMAL, TRACE_CONSOLE, insert_mac)
local query = string.format(
"SELECT type, key, ifid, ip, mac, vlan, network, name, device_type, manufacturer, %s , %s, gateway_mac, json_info %s FROM %s WHERE key='%s' AND ifid=%d AND type='%s'",
ternary(hasClickHouseSupport(), "toUnixTimestamp(last_seen) as last_seen", "last_seen"),
ternary(hasClickHouseSupport(), "toUnixTimestamp(first_seen) as first_seen", "first_seen"),
ternary(hasClickHouseSupport(), ", version", ""),
table_name,
key,
ifid,
type
)
local res = interface.alert_store_query(query)
return res
end
-- ##############################################
-- Return the lists of inactive hosts from the DB
function asset_management_utils.get_inactive_hosts(ifid, order, sort, start, length, filters)
local function updateJsonField(fields, new_fields)
if fields then
local json_info = json.decode(fields.json_info) or {}
for field_name, field_value in pairs(new_fields or {}) do
json_info[field_name] = field_value
end
fields.json_info = json.encode(json_info)
end
return fields
end
-- ##############################################
local function getAssetData(ifid, order, sort, start, length, filters, asset_type, check_last_seen)
if not ifid then
ifid = interface.getId()
end
@ -149,7 +57,7 @@ function asset_management_utils.get_inactive_hosts(ifid, order, sort, start, len
sort = "IPv4StringToNum(ip)"
end
local where = ""
for key, value in pairs(filters or {}) do
where = where .. "AND"
if tonumber(value) then
@ -157,40 +65,61 @@ function asset_management_utils.get_inactive_hosts(ifid, order, sort, start, len
else
value = string.format("'%s'", value)
end
where = string.format("%s %s=%s ", where, key, value)
end
local sort_query = ""
local limit_query = ""
if sort and order then
where = string.format("%s ORDER BY %s %s", where, sort, order)
sort_query = string.format("ORDER BY %s %s", sort, order)
end
if start and length then
where = string.format("%s LIMIT %s, %s", where, start, length)
limit_query = string.format("LIMIT %s, %s", start, length)
end
local query = string.format("SELECT key, ip, mac, vlan, network, name, device_type, manufacturer, %s, %s " ..
"FROM %s WHERE type='%s' AND last_seen!=%d %s",
ternary(hasClickHouseSupport(), "toUnixTimestamp(last_seen) as last_seen", "last_seen"),
ternary(hasClickHouseSupport(), "toUnixTimestamp(first_seen) as first_seen", "first_seen"),
table_name,
"host", -- Only hosts here
0, -- 0 Because by default an host that is still in memory has a last_seen 0
where
)
local res = interface.alert_store_query(query)
return res
local query = nil
if hasClickHouseSupport() then
query = string.format("SELECT a.type, a.key, a.ifid, a.ip, a.mac, a.vlan, a.network, a.name, a.device_type, a.manufacturer, %s, %s, a.gateway_mac, a.json_info, a.version" ..
" FROM %s a INNER JOIN (SELECT type, key, MAX(version) AS max_version FROM %s WHERE type='%s' %s AND ifid=%d %s GROUP BY type, key) AS latest" ..
" ON a.type = latest.type AND a.key = latest.key AND a.version = latest.max_version %s %s",
ternary(hasClickHouseSupport(), "toUnixTimestamp(a.last_seen) as last_seen", "a.last_seen"),
ternary(hasClickHouseSupport(), "toUnixTimestamp(a.first_seen) as first_seen", "a.first_seen"),
table_name,
table_name,
asset_type, -- Only hosts here
ternary(check_last_seen, 'AND last_seen!=0', ''), -- 0 Because by default an host that is still in memory has a last_seen 0
tonumber(ifid),
where,
sort_query,
limit_query
)
else
query = string.format("SELECT type, key, ifid, ip, mac, vlan, network, name, device_type, manufacturer, last_seen, first_seen, gateway_mac, json_info" ..
" FROM %s WHERE type='%s' %s AND ifid=%d %s %s %s",
table_name,
asset_type, -- Only hosts here
ternary(check_last_seen, 'AND last_seen!=0', ''), -- 0 Because by default an host that is still in memory has a last_seen 0
tonumber(ifid),
where,
sort_query,
limit_query
)
end
return interface.alert_store_query(query)
end
-- ##############################################
-- Return the lists of inactive hosts from the DB
function asset_management_utils.get_total_inactive_hosts(ifid, filters)
local function getNumAssets(ifid, filters, asset_type, check_last_seen)
if not ifid then
ifid = interface.getId()
end
local where = ""
for key, value in pairs(filters) do
where = where .. "AND"
if tonumber(value) then
@ -198,44 +127,208 @@ function asset_management_utils.get_total_inactive_hosts(ifid, filters)
else
value = string.format("'%s'", value)
end
where = string.format("%s %s=%s ", where, key, value)
end
local query = nil
if hasClickHouseSupport() then
query = string.format("SELECT count(*) as count FROM %s a INNER JOIN (SELECT type, key, MAX(version) AS max_version FROM %s WHERE type='%s' %s AND ifid=%d %s GROUP BY type, key) AS latest" ..
" ON a.type = latest.type AND a.key = latest.key AND a.version = latest.max_version",
table_name,
table_name,
asset_type, -- Only hosts here
ternary(check_last_seen, 'AND last_seen!=0', ''), -- 0 Because by default an host that is still in memory has a last_seen 0
tonumber(ifid),
where
)
else
query = string.format("SELECT COUNT(*) as count " ..
"FROM %s WHERE type='%s' %s %s AND ifid=%d",
table_name,
asset_type,
where,
ternary(check_last_seen, 'AND last_seen!=0', ''), -- 0 Because by default an host that is still in memory has a last_seen 0
ifid
)
end
local query = string.format("SELECT COUNT(*) as count " ..
"FROM %s WHERE type='%s' %s AND last_seen!=%d",
table_name,
"host", -- Only hosts here,
where,
0 -- 0 Because by default an host that is still in memory has a last_seen 0
)
local res = interface.alert_store_query(query)
return res
return interface.alert_store_query(query)
end
-- ##############################################
-- Return the lists of inactive hosts from the DB
function asset_management_utils.get_filters(ifid)
local function get_mac_serialization_key(mac, ifid)
return tostring(ifid) .. "_" .. mac
end
-- ##############################################
-- @brief insert assetkey
function asset_management_utils.getLastVersion(ifid)
local query = string.format("SELECT version FROM %s WHERE ifid=%d ORDER BY version DESC LIMIT 1", table_name, ifid)
local last_version = interface.alert_store_query(query)
if table.len(last_version) == 0 then
last_version = 0
else
last_version = last_version[1].version
end
return last_version
end
-- ##############################################
-- @brief insert assetkey
function asset_management_utils.insertHost(entry, version, ifid)
local query = nil
if hasClickHouseSupport() then
query = string.format(
"INSERT INTO %s " ..
"(type, key, ifid, ip, mac, vlan, network, name, device_type, manufacturer, first_seen, last_seen, version, json_info) " ..
"VALUES ('%s','%s', %u, '%s', '%s', %u, %u, %s, %u, %s, %u, %u, %u, '%s')",
table_name,
entry["type"],
entry["key"],
ifid,
entry["ip"] or "",
entry["mac"] or "",
entry["vlan"] or 0,
entry["network"] or 0,
ternary(not isEmptyString(entry["name"]), string.format("'%s'", entry["name"]), "NULL"),
entry["device_type"],
ternary(not isEmptyString(entry["manufacturer"]), string.format("'%s'", entry["manufacturer"]), "NULL"),
entry["first_seen"],
entry["last_seen"] or 0,
version,
entry["json_info"] or ""
)
else
query = string.format(
"INSERT INTO %s " ..
"(type, key, ifid, ip, mac, vlan, network, name, device_type, manufacturer, first_seen, last_seen, json_info) " ..
"VALUES ('%s','%s', %u, '%s','%s', %u, %u, %s, %u, %s, %u, %u, '%s') " ..
"ON CONFLICT(key) DO UPDATE SET last_seen = %u, first_seen = %u;",
table_name,
entry["type"],
entry["key"],
ifid,
entry["ip"],
entry["mac"] or "",
entry["vlan"] or 0,
entry["network"] or 0,
ternary(not isEmptyString(entry["name"]), string.format("'%s'", entry["name"]), "NULL"),
entry["device_type"],
ternary(not isEmptyString(entry["manufacturer"]), string.format("'%s'", entry["manufacturer"]), "NULL"),
entry["first_seen"],
entry["last_seen"] or 0,
entry["json_info"] or "",
entry["last_seen"] or 0,
entry["first_seen"] or 0
)
end
return interface.alert_store_query(query)
end
function asset_management_utils.insertMac(entry, version, ifid)
local query = nil
if hasClickHouseSupport() then
query = string.format(
"INSERT INTO %s " ..
"(type, key, ifid, mac, manufacturer, vlan, device_type, first_seen, last_seen, version, json_info) " ..
"SELECT '%s','%s', %u, '%s','%s', %u, %u, %u, %u, %u, '%s'",
table_name,
entry["type"],
entry["key"],
tonumber(ifid),
entry["mac"],
entry["manufacturer"],
0, -- VLAN
tonumber(entry["device_type"]),
tonumber(entry["first_seen"]),
tonumber(entry["last_seen"] or 0),
tonumber(version),
entry["json_info"] or ""
)
else
query = string.format(
"INSERT INTO %s " ..
"(type, key, ifid, mac, manufacturer, device_type, first_seen, last_seen, json_info) " ..
"VALUES ('%s','%s', %u, '%s','%s', %u, %u, %u, '%s') " ..
"ON CONFLICT(key) DO UPDATE SET last_seen = %u, first_seen = %u;",
table_name,
entry["type"],
entry["key"],
tonumber(ifid),
entry["mac"],
entry["manufacturer"],
tonumber(entry["device_type"] or 0),
tonumber(entry["first_seen"] or 0),
tonumber(entry["last_seen"] or 0),
entry["json_info"] or "",
tonumber(entry["last_seen"] or 0),
tonumber(entry["first_seen"])
)
end
return interface.alert_store_query(query)
end
-- ##############################################
function asset_management_utils.getDevices(ifid, order, sort, start, length, filters)
return getAssetData(ifid, order, sort, start, length, filters, "mac" --[[ Asset Type ]], false)
end
-- ##############################################
-- Return the lists of inactive hosts from the DB
function asset_management_utils.getInactiveHosts(ifid, order, sort, start, length, filters)
return getAssetData(ifid, order, sort, start, length, filters, "host" --[[ Asset Type ]], true)
end
-- ##############################################
-- Return the lists of inactive hosts from the DB
function asset_management_utils.getNumDevices(ifid, filters)
return getNumAssets(ifid, filters, "mac", false)
end
-- ##############################################
-- Return the lists of inactive hosts from the DB
function asset_management_utils.getNumInactiveHosts(ifid, filters)
return getNumAssets(ifid, filters, "host", true)
end
-- ##############################################
-- Return the lists of inactive hosts from the DB
function asset_management_utils.getFilters(ifid)
if not ifid then
ifid = interface.getId()
end
local query = string.format("SELECT 'manufacturer' AS filter, manufacturer AS value, COUNT(*) AS count " ..
"FROM %s where type='host' GROUP BY manufacturer UNION ALL " ..
"FROM %s where type='host' AND ifid=%d GROUP BY manufacturer UNION ALL " ..
"SELECT 'device_type' AS filter, %s AS value, COUNT(*) AS count " ..
"FROM %s where type='host' GROUP BY device_type UNION ALL " ..
"FROM %s where type='host' AND ifid=%d GROUP BY device_type UNION ALL " ..
"SELECT 'vlan' AS filter, %s AS value, COUNT(*) AS count " ..
"FROM %s where type='host' GROUP BY vlan UNION ALL " ..
"FROM %s where type='host' AND ifid=%d GROUP BY vlan UNION ALL " ..
"SELECT 'network' AS filter, %s AS value, COUNT(*) AS count " ..
"FROM %s where type='host' GROUP BY network",
"FROM %s where type='host' AND ifid=%d GROUP BY network",
table_name,
ifid,
ternary(hasClickHouseSupport(), "CAST(device_type, 'String')", "CAST(device_type AS CHAR)"),
table_name,
ifid,
ternary(hasClickHouseSupport(), "CAST(vlan, 'String')", "CAST(vlan AS CHAR)"),
table_name,
ifid,
ternary(hasClickHouseSupport(), "CAST(network, 'String')", "CAST(network AS CHAR)"),
table_name
table_name,
ifid
)
local res = interface.alert_store_query(query)
return res
@ -243,100 +336,70 @@ end
-- ##############################################
-- Return the lists of inactive hosts from the DB
function asset_management_utils.get_inactive_host_info(ifid, key)
if isEmptyString(key) then
return nil
end
local query = string.format("SELECT key, ip, mac, vlan, network, name, device_type, manufacturer, %s , %s FROM %s WHERE key='%s'",
ternary(hasClickHouseSupport(), "toUnixTimestamp(last_seen) as last_seen", "last_seen"),
ternary(hasClickHouseSupport(), "toUnixTimestamp(first_seen) as first_seen", "first_seen"),
table_name,
key
)
local res = interface.alert_store_query(query)
return res
function asset_management_utils.getInactiveHostInfo(ifid, key)
return getAssetInfo(ifid, key, "host")
end
-- ##############################################
function asset_management_utils.get_devices()
local query = string.format("SELECT mac, name, manufacturer, device_status, trigger_alert, %s, %s FROM '%s' WHERE type = 'mac'",
ternary(hasClickHouseSupport(), "toUnixTimestamp(last_seen) as last_seen", "last_seen"),
ternary(hasClickHouseSupport(), "toUnixTimestamp(first_seen) as first_seen", "first_seen"),
table_name)
local res = interface.alert_store_query(query)
local data = {}
for _, val in pairs(res) do
local info = {}
info["device_status"] = val["device_status"]
info["trigger_alert"] = ternary(val["trigger_alert"] == "1", true, false)
info["first_seen"] = val["first_seen"]
info["last_seen"] = val["last_seen"]
info["name"] = val["name"]
info["manufacturer"] = val["manufacturer"]
local mac = val["mac"]
data[mac] = info
end
return data
function asset_management_utils.getMacInfo(ifid, key)
return getAssetInfo(ifid, key, "mac")
end
-- ##############################################
-- Edit a list of macs with the specified trigger_alert value
function asset_management_utils.edit_mac_list(device_list, trigger_alert, ifid)
function asset_management_utils.editMacList(device_list, trigger_alert, ifid)
for _, device in pairs(device_list) do
asset_management_utils.edit_mac(device, trigger_alert, "allowed", ifid)
asset_management_utils.editMac(device, trigger_alert, "allowed", ifid)
end
end
function asset_management_utils.edit_mac(device, trigger_alert, mac_status, ifid)
-- ##############################################
function asset_management_utils.editMac(device, trigger_alert, mac_status, ifid)
if isMacAddress(device) then
local key = get_mac_serialization_key(device, ifid)
local query = ""
if hasClickHouseSupport() then
query = string.format("ALTER TABLE %s UPDATE ", table_name)
else
query = string.format("UPDATE %s SET", table_name)
local fields = asset_management_utils.getMacInfo(ifid, key)
if fields and table.len(fields) > 0 then
fields = fields[1]
fields = updateJsonField(fields, { device_status = mac_status, trigger_alert = trigger_alert })
if hasClickHouseSupport() then
asset_management_utils.insertMac(fields, tonumber(fields.version) + 1, tonumber(ifid))
else
local update_query = string.format("UPDATE %s SET `json_info`='%s' WHERE type='mac' AND ifid=%d AND key='%s'", table_name, fields.json_info, fields.ifid, fields.key)
interface.alert_store_query(update_query)
end
end
if not isEmptyString(mac_status) and mac_status ~= nil then
query = query .. "`device_status` = '" .. mac_status .. "', "
end
query = query .."`trigger_alert` = ".. ternary(trigger_alert, "1", "0") .." "
query = query .. "WHERE `key`='" .. key .. "'"
interface.alert_store_query(query)
end
end
function asset_management_utils.delete_all()
-- ##############################################
function asset_management_utils.deleteAll(ifid)
local query = ""
if hasClickHouseSupport() then
query = string.format("ALTER TABLE %s DELETE WHERE type = 'mac'", table_name)
query = string.format("ALTER TABLE %s DELETE WHERE type='mac' and ifid=%d", table_name, tonumber(ifid))
else
query = string.format("DELETE FROM %s WHERE type = 'mac'", table_name)
query = string.format("DELETE FROM %s WHERE type='mac' and ifid=%d", table_name, tonumber(ifid))
end
interface.alert_store_query(query)
end
function asset_management_utils.delete_mac(device, ifid)
-- ##############################################
function asset_management_utils.deleteMac(device, ifid)
local key = get_mac_serialization_key(device, ifid)
local query = ""
if hasClickHouseSupport() then
query = string.format("ALTER TABLE %s DELETE WHERE key = '%s'", table_name, key)
query = string.format("ALTER TABLE %s DELETE WHERE key='%s' and type='mac'", table_name, key)
else
query = string.format("DELETE FROM %s WHERE key = '%s'", table_name, key)
query = string.format("DELETE FROM %s WHERE key='%s' and type='mac'", table_name, key)
end
interface.alert_store_query(query)
end
return asset_management_utils