From 33a89e993e097718c61a34cee32b2e9f238e2524 Mon Sep 17 00:00:00 2001 From: lucapruneti <30806719+lucapruneti@users.noreply.github.com> Date: Fri, 4 Jun 2021 18:10:26 +0200 Subject: [PATCH] CSV Export (#5445) * Change host alert function name to json * Refactored *_alert_store function with format_json, added csv format tohost alert * Added .vscode directory to gitignore * Refactor alert_store:to_csv function to single loop format * Removed no_data value for no records in alert csv download * Implemented csv export for alerts * Merge with dev * Removed row_id alert field to be exported in csv * Refactored alert/*/list.lua with simple decision value format construct * Refactored alerts csv export * Fix removed output test files * Added alert csv export for subdocuments * Modified gitignore * Change MSG values to export in alerts * Added flow information in alert export * Added export msg.description for alert --- .../lua/modules/alert_store/alert_store.lua | 39 ++++++++++++++++--- .../modules/alert_store/flow_alert_store.lua | 11 ++++-- .../modules/alert_store/host_alert_store.lua | 6 ++- .../alert_store/interface_alert_store.lua | 6 ++- .../modules/alert_store/mac_alert_store.lua | 6 ++- .../alert_store/network_alert_store.lua | 6 ++- 6 files changed, 61 insertions(+), 13 deletions(-) diff --git a/scripts/lua/modules/alert_store/alert_store.lua b/scripts/lua/modules/alert_store/alert_store.lua index cf718ac349..a1288be61a 100644 --- a/scripts/lua/modules/alert_store/alert_store.lua +++ b/scripts/lua/modules/alert_store/alert_store.lua @@ -844,7 +844,7 @@ function alert_store:build_csv_row_header(rnames) row = row .. CSV_SEPARATOR .. value.name else for _, element in ipairs(value.elements) do - row = row .. CSV_SEPARATOR .. value.name .. "_" .. element + row = row .. CSV_SEPARATOR .. value.name .. "_" .. string.gsub(element, "%.", "_") end end end @@ -860,14 +860,12 @@ function alert_store:build_csv_row(rnames, document) for _, rname in pairsByKeys(rnames) do local doc_value = document[rname.name] if type(doc_value) ~= "table" then - row = row .. CSV_SEPARATOR .. self:escape_csv(tostring(doc_value)) + row = row .. self:build_csv_row_single_element(doc_value) else if rname["elements"] ~= nil then - for _, element in ipairs(rname.elements) do - row = row .. CSV_SEPARATOR .. self:escape_csv(tostring(doc_value[element])) - end + row = row .. self:build_csv_row_multiple_elements(doc_value, rname.elements) else - row = row .. CSV_SEPARATOR .. self:escape_csv(tostring(doc_value.value)) + row = row .. self:build_csv_row_single_element(doc_value.value) end end end @@ -877,6 +875,35 @@ function alert_store:build_csv_row(rnames, document) return row end +function alert_store:build_csv_row_single_element(value) + return CSV_SEPARATOR .. self:escape_csv(tostring(value)) +end + +function alert_store:build_csv_row_multiple_elements(value, elements) + local row = "" + for _, element in ipairs(elements) do + local splitted = string.split(element, "%.") + if(splitted == nil) then + row = row .. CSV_SEPARATOR .. self:escape_csv(tostring(value[element])) + else + if #splitted > 2 then + row = row .. self:build_csv_row_multiple_elements(value[splitted[1]], self:rebuild_sub_elements(splitted)) + else + row = row .. CSV_SEPARATOR .. self:escape_csv(tostring(value[splitted[1]][splitted[2]])) + end + end + end + return row +end + +function alert_store:rebuild_sub_elements(splitted) + local tmp_elements = {} + for i = 2, #splitted, 1 do + tmp_elements[#tmp_elements+1] = splitted[i] + end + return { table.concat(tmp_elements, ".") } +end + -- Used to escape "'s by to_csv function alert_store:escape_csv(s) if string.find(s, '[,"|\n]') then diff --git a/scripts/lua/modules/alert_store/flow_alert_store.lua b/scripts/lua/modules/alert_store/flow_alert_store.lua index 8ceef1d54e..b3c00ba021 100644 --- a/scripts/lua/modules/alert_store/flow_alert_store.lua +++ b/scripts/lua/modules/alert_store/flow_alert_store.lua @@ -387,7 +387,8 @@ local RNAME = { ADDITIONAL_ALERTS = { name = "additional_alerts", export = true}, ALERT_NAME = { name = "alert_name", export = true}, DESCRIPTION = { name = "description", export = true}, - MSG = { name = "msg", export = true, elements = {"name", "value"}}, + MSG = { name = "msg", export = true, elements = {"name", "value", "description"}}, + FLOW = { name = "flow", export = true, elements = {"srv_ip.label", "srv_ip.value", "srv_port", "cli_ip.label", "cli_ip.value", "cli_port"}}, VLAN_ID = { name = "vlan_id", export = true}, PROTO = { name = "proto", export = true}, L7_PROTO = { name = "l7_proto", export = true} @@ -480,6 +481,10 @@ function flow_alert_store:format_record(value, no_html) msg = "" end + if no_html then + msg = noHtml(msg) + end + record[RNAME.MSG.name] = { name = noHtml(alert_name), value = tonumber(value["alert_id"]), @@ -530,13 +535,13 @@ function flow_alert_store:format_record(value, no_html) local flow_cli_port = value["cli_port"] local flow_srv_port = value["srv_port"] - record["flow"] = { + record[RNAME.FLOW.name] = { cli_ip = flow_cli_ip, srv_ip = flow_srv_ip, cli_port = flow_cli_port, srv_port = flow_srv_port, historical_url = historical_url, - active_url = active_url, + active_url = active_url } record[RNAME.VLAN_ID.name] = value["vlan_id"] diff --git a/scripts/lua/modules/alert_store/host_alert_store.lua b/scripts/lua/modules/alert_store/host_alert_store.lua index ff6b484fb9..f671f3c3ca 100644 --- a/scripts/lua/modules/alert_store/host_alert_store.lua +++ b/scripts/lua/modules/alert_store/host_alert_store.lua @@ -205,7 +205,7 @@ local RNAME = { IS_ATTACKER = { name = "is_attacker", export = true}, VLAN_ID = { name = "vlan_id", export = true}, ALERT_NAME = { name = "alert_name", export = true}, - MSG = { name = "msg", export = true, elements = {"name", "value"}} + MSG = { name = "msg", export = true, elements = {"name", "value", "description"}} } function host_alert_store:get_rnames() @@ -280,6 +280,10 @@ function host_alert_store:format_record(value, no_html) msg = "" end + if no_html then + msg = noHtml(msg) + end + record[RNAME.MSG.name] = { name = noHtml(alert_name), value = tonumber(value["alert_id"]), diff --git a/scripts/lua/modules/alert_store/interface_alert_store.lua b/scripts/lua/modules/alert_store/interface_alert_store.lua index 8b52a2e8ef..a70fd8c82e 100644 --- a/scripts/lua/modules/alert_store/interface_alert_store.lua +++ b/scripts/lua/modules/alert_store/interface_alert_store.lua @@ -66,7 +66,7 @@ end local RNAME = { ALERT_NAME = { name = "alert_name", export = true}, - MSG = { name = "msg", export = true, elements = {"name", "value"}} + MSG = { name = "msg", export = true, elements = {"name", "value", "description"}} } function interface_alert_store:get_rnames() @@ -87,6 +87,10 @@ function interface_alert_store:format_record(value, no_html) msg = "" end + if no_html then + msg = noHtml(msg) + end + record[RNAME.MSG.name] = { name = noHtml(alert_name), value = tonumber(value["alert_id"]), diff --git a/scripts/lua/modules/alert_store/mac_alert_store.lua b/scripts/lua/modules/alert_store/mac_alert_store.lua index 451ac92a10..10b3c54365 100644 --- a/scripts/lua/modules/alert_store/mac_alert_store.lua +++ b/scripts/lua/modules/alert_store/mac_alert_store.lua @@ -93,7 +93,7 @@ local RNAME = { ADDRESS = { name = "address", export = true}, DEVICE_TYPE = { name = "device_type", export = true}, NAME = { name = "name", export = true}, - MSG = { name = "msg", export = true, elements = {"name", "value"}} + MSG = { name = "msg", export = true, elements = {"name", "value", "description"}} } function mac_alert_store:get_rnames() @@ -120,6 +120,10 @@ function mac_alert_store:format_record(value, no_html) msg = "" end + if no_html then + msg = noHtml(msg) + end + record[RNAME.MSG.name] = { name = noHtml(alert_name), value = tonumber(value["alert_id"]), diff --git a/scripts/lua/modules/alert_store/network_alert_store.lua b/scripts/lua/modules/alert_store/network_alert_store.lua index 60e2c8231d..7eef4eadc5 100644 --- a/scripts/lua/modules/alert_store/network_alert_store.lua +++ b/scripts/lua/modules/alert_store/network_alert_store.lua @@ -94,7 +94,7 @@ local RNAME = { LOCAL_NETWORK_ID = { name = "local_network_id", export = true}, NETWORK = { name = "network", export = true}, ALERT_NAME = { name = "alert_name", export = true}, - MSG = { name = "msg", export = true, elements = {"name", "value"}} + MSG = { name = "msg", export = true, elements = {"name", "value", "description"}} } function network_alert_store:get_rnames() @@ -120,6 +120,10 @@ function network_alert_store:format_record(value, no_html) msg = "" end + if no_html then + msg = noHtml(msg) + end + record[RNAME.MSG.name] = { name = noHtml(alert_name), value = tonumber(value["alert_id"]),