From 76759a7d0372df10dce088bc552e26395a714ea7 Mon Sep 17 00:00:00 2001 From: Alfredo Cardigliano Date: Thu, 10 Oct 2019 15:56:53 +0200 Subject: [PATCH] Suricata events are now processed by the Lua script --- include/SyslogParserInterface.h | 7 - scripts/callbacks/syslog/suricata.lua | 139 +++++++++++- scripts/lua/modules/flow_utils.lua | 36 +++ src/ParsedFlow.cpp | 6 + src/SyslogParserInterface.cpp | 303 +------------------------- 5 files changed, 174 insertions(+), 317 deletions(-) diff --git a/include/SyslogParserInterface.h b/include/SyslogParserInterface.h index eb82c9e5f5..ad7ae82c18 100644 --- a/include/SyslogParserInterface.h +++ b/include/SyslogParserInterface.h @@ -28,13 +28,6 @@ class SyslogParserInterface : public ParserInterface { private: SyslogLuaEngine *le; - void parseSuricataFlow(json_object *f, ParsedFlow *flow); - void parseSuricataNetflow(json_object *f, ParsedFlow *flow); - void parseSuricataHTTP(json_object *h, ParsedFlow *flow); - void parseSuricataDNS(json_object *d, ParsedFlow *flow); - void parseSuricataTLS(json_object *t, ParsedFlow *flow); - void parseSuricataAlert(json_object *a, ParsedFlow *flow, ICMPinfo *icmp_info, bool flow_alert); - public: SyslogParserInterface(const char *endpoint, const char *custom_interface_type = NULL); ~SyslogParserInterface(); diff --git a/scripts/callbacks/syslog/suricata.lua b/scripts/callbacks/syslog/suricata.lua index d41243490d..957e1d1acb 100644 --- a/scripts/callbacks/syslog/suricata.lua +++ b/scripts/callbacks/syslog/suricata.lua @@ -5,10 +5,8 @@ local dirs = ntop.getDirs() package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path require "lua_utils" -require "alert_utils" +require "flow_utils" local json = require ("dkjson") -local alerts_api = require("alerts_api") -local alert_consts = require("alert_consts") local syslog_module = { key = "suricata", @@ -24,13 +22,138 @@ end -- ################################################################# +local function parseFiveTuple(event, flow) + flow.vlan_id = tonumber(event.vlan) + flow.src_ip = event.src_ip + flow.dst_ip = event.dest_ip + flow.src_port = tonumber(event.src_port) + flow.dst_port = tonumber(event.dest_port) + flow.l4_proto = event.proto +end + +-- ################################################################# + +local function parseFlowMetadata(event_flow, flow) + flow.first_switched_iso8601 = event_flow.start + flow.last_switched_iso8601 = event_flow['end'] + flow.in_pkts = tonumber(event_flow.pkts_toserver) + flow.out_pkts = tonumber(event_flow.pkts_toclient) + flow.in_bytes = tonumber(event_flow.bytes_toserver) + flow.out_bytes = tonumber(event_flow.bytes_toclient) +end + +-- ################################################################# + +local function parseAlertMetadata(event_alert, flow) + flow.external_alert_severity = tonumber(event_alert.severity) + flow.external_alert = json.encode(event_alert) +end + +-- ################################################################# + +local function parseNetflowMetadata(event_flow, flow) + flow.first_switched_iso8601 = event_flow.start + flow.last_switched_iso8601 = event_flow['end'] + flow.in_pkts = tonumber(event_flow.pkts) + flow.in_bytes = tonumber(event_flow.bytes) +end + +-- ################################################################# + +local function parseHTTPMetadata(event_http, flow) + + -- Additional fields: + -- event_http.protocol + -- event_http.http_refer + -- event_http.http_content_type + -- event_http.length + + flow.http_method = event_http.http_method + flow.http_ret_code = tonumber(event_http.status) + flow.http_site = event_http.hostname + if event_http.hostname ~= nil and event_http.url ~= nil then + flow.http_url = event_http.hostname..event_http.url + end +end + +-- ################################################################# + +local function parseDNSMetadata(event_dns, flow) + + -- Additional fields: + -- event_dns.id + -- event_dns.tx_id + + if event_dns.type == "query" then + flow.dns_query = event_dns.rrname + flow.dns_query_type = get_dns_type(event_dns.rrtype) + end +end + +-- ################################################################# + +local function parseTLSMetadata(event_tls, flow) + + -- Additional fields: + -- event_tls.version + -- event_tls.session_resumed + -- event_tls.ja3 + -- event_tls.ja3s + + flow.ssl_server_name = event_tls.sni +end + +-- ################################################################# + -- The function below is called for each received alert function syslog_module.hooks.handleEvent(message) - -- Example: printing the Suricata alert - -- local alert = json.decode(message) - -- if alert ~= nil then - -- tprint(alert) - -- end + local event = json.decode(message) + if event == nil then + return + end + + -- Additional fields: + -- event.timestamp + -- event.event_type + -- event.flow_id + -- event.community_id + -- event.app_proto + + local flow = {} + parseFiveTuple(event, flow) + + if event.event_type == "alert" then + + if event.flow ~= nil then + parseFlowMetadata(event.flow, flow) + if flow.last_switched_iso8601 == nil then + flow.last_switched_iso8601 = event.timestamp + end + parseAlertMetadata(event.alert, flow) + else + flow = nil + end + + elseif event.event_type == "netflow" then + parseNetflowMetadata(event.netflow, flow) + + elseif event.event_type == "http" then + parseHTTPMetadata(event.http, flow) + + elseif event.event_type == "dns" then + parseDNSMetadata(event.dns, flow) + + elseif event.event_type == "tls" then + parseTLSMetadata(event.tls, flow) + + else + -- traceError(TRACE_NORMAL, TRACE_CONSOLE, "Unsupported Suricata event '"..event.event_type.."'") + flow = nil + end + + if flow ~= nil then + interface.processFlow(flow) + end end -- ################################################################# diff --git a/scripts/lua/modules/flow_utils.lua b/scripts/lua/modules/flow_utils.lua index c4b3865bed..937d02d0fd 100644 --- a/scripts/lua/modules/flow_utils.lua +++ b/scripts/lua/modules/flow_utils.lua @@ -354,6 +354,42 @@ function getL4ProtoName(proto_id) end -- ####################### + +local dns_types = { + ['A'] = 1, + ['NS'] = 2, + ['MD'] = 3, + ['MF'] = 4, + ['CNAME'] = 5, + ['SOA'] = 6, + ['MB'] = 7, + ['MG'] = 8, + ['MR'] = 9, + ['NULL'] = 10, + ['WKS'] = 11, + ['PTR'] = 12, + ['HINFO'] = 13, + ['MINFO'] = 14, + ['MX'] = 15, + ['TXT'] = 16, + ['AAAA'] = 28, + ['A6'] = 38, + ['SPF'] = 99, + ['AXFR'] = 252, + ['MAILB'] = 253, + ['MAILA'] = 254, + ['ANY'] = 255, +} + +function get_dns_type(dns_type_name) + if dns_types[dns_type_name] then + return dns_types[dns_type_name] + else + return 0 + end +end + + -- ####################### local icmp_v4_msgs = { { 0, 0, i18n("icmp_v4_msgs.type_0_0_echo_reply") }, diff --git a/src/ParsedFlow.cpp b/src/ParsedFlow.cpp index 117ef0c2ed..f1feb3a900 100644 --- a/src/ParsedFlow.cpp +++ b/src/ParsedFlow.cpp @@ -110,6 +110,12 @@ void ParsedFlow::fromLua(lua_State *L, int index) { } else if(!strcmp(key, "external_alert")) { if(external_alert) free(external_alert); external_alert = strdup(lua_tostring(L, -1)); + } else if(!strcmp(key, "first_switched_iso8601")) { + first_switched = Utils::str2epoch(lua_tostring(L, -1)); + } else if(!strcmp(key, "last_switched_iso8601")) { + last_switched = Utils::str2epoch(lua_tostring(L, -1)); + } else if(!strcmp(key, "l4_proto")) { + l4_proto = Utils::l4name2proto(lua_tostring(L, -1)); } else { ntop->getTrace()->traceEvent(TRACE_ERROR, "Invalid string type (%s) for %s", lua_tostring(L, -1), key); } diff --git a/src/SyslogParserInterface.cpp b/src/SyslogParserInterface.cpp index 1f392fa1e4..c317e03ae0 100644 --- a/src/SyslogParserInterface.cpp +++ b/src/SyslogParserInterface.cpp @@ -23,7 +23,6 @@ #ifndef HAVE_NEDGE -#define USE_SURICATA_NETFLOW //#define SYSLOG_DEBUG /* **************************************************** */ @@ -41,175 +40,8 @@ SyslogParserInterface::~SyslogParserInterface() { /* **************************************************** */ -void SyslogParserInterface::parseSuricataNetflow(json_object *f, ParsedFlow *flow) { - json_object *w; - - if(json_object_object_get_ex(f, "start", &w)) - flow->first_switched = Utils::str2epoch(json_object_get_string(w)); - - if(json_object_object_get_ex(f, "end", &w)) - flow->last_switched = Utils::str2epoch(json_object_get_string(w)); - - if(json_object_object_get_ex(f, "pkts", &w)) - flow->in_pkts = json_object_get_int(w); - - if(json_object_object_get_ex(f, "bytes", &w)) - flow->in_bytes = json_object_get_int(w); -} - -/* **************************************************** */ - -void SyslogParserInterface::parseSuricataFlow(json_object *f, ParsedFlow *flow) { - json_object *w; - - if(json_object_object_get_ex(f, "start", &w)) - flow->first_switched = Utils::str2epoch(json_object_get_string(w)); - - if(json_object_object_get_ex(f, "end", &w)) - flow->last_switched = Utils::str2epoch(json_object_get_string(w)); - - if(json_object_object_get_ex(f, "pkts_toserver", &w)) - flow->in_pkts = json_object_get_int(w); - - if(json_object_object_get_ex(f, "pkts_toclient", &w)) - flow->out_pkts = json_object_get_int(w); - - if(json_object_object_get_ex(f, "bytes_toserver", &w)) - flow->in_bytes = json_object_get_int(w); - - if(json_object_object_get_ex(f, "bytes_client", &w)) - flow->out_bytes = json_object_get_int(w); -} - -/* **************************************************** */ - -void SyslogParserInterface::parseSuricataHTTP(json_object *h, ParsedFlow *flow) { - json_object *w; - - /* Other available fields: - * protocol (string) - * http_refer (string) - * http_content_type (string) - * length (int) - */ - - if(json_object_object_get_ex(h, "http_method", &w)) - flow->http_method = strdup(json_object_get_string(w)); - - if(json_object_object_get_ex(h, "hostname", &w)) { - flow->http_site = strdup(json_object_get_string(w)); - - if(json_object_object_get_ex(h, "url", &w)) { - const char *url = json_object_get_string(w); - int url_size = strlen(flow->http_site) + strlen(url) + 1; - flow->http_url = (char *) malloc(url_size); - if (flow->http_url) - snprintf(flow->http_url, url_size, "%s%s", flow->http_site, url); - } - } - - if(json_object_object_get_ex(h, "status", &w)) - flow->http_ret_code = json_object_get_int(w); -} - -/* **************************************************** */ - -void SyslogParserInterface::parseSuricataDNS(json_object *d, ParsedFlow *flow) { - json_object *w; - - /* Other available fields: - * id (int) - * tx_id (int) - */ - - if(json_object_object_get_ex(d, "type", &w)) { - const char *type = json_object_get_string(w); - if (strcmp(type, "query") == 0) { - if(json_object_object_get_ex(d, "rrname", &w)) - flow->dns_query = strdup(json_object_get_string(w)); - if(json_object_object_get_ex(d, "rrtype", &w)) { - const char *query_type = json_object_get_string(w); - flow->dns_query_type = Utils::queryname2type(query_type); - } - } - } -} - -/* **************************************************** */ - -void SyslogParserInterface::parseSuricataTLS(json_object *t, ParsedFlow *flow) { - json_object *w; - - /* Other available fields: - * version (string) - * session_resumed (bool) - * ja3 (obj) - * ja3s (obj) - */ - - if(json_object_object_get_ex(t, "sni", &w)) - flow->ssl_server_name = strdup(json_object_get_string(w)); -} - -/* **************************************************** */ - -void SyslogParserInterface::parseSuricataAlert(json_object *a, ParsedFlow *flow, ICMPinfo *icmp_info, bool flow_alert) { - json_object *w; - u_int8_t severity; - - if (json_object_object_get_ex(a, "severity", &w)) - severity = json_object_get_int(w); - - if (flow_alert) { - Flow *f; - bool src2dst_direction, new_flow; -#ifdef SYSLOG_DEBUG - char src_ip_buf[64], dst_ip_buf[64]; - - ntop->getTrace()->traceEvent(TRACE_NORMAL, "[Suricata] Flow Alert for %s:%u <-> %s:%u JSON: %s", - flow->src_ip.print(src_ip_buf, sizeof(src_ip_buf)), ntohs(flow->src_port), - flow->dst_ip.print(dst_ip_buf, sizeof(dst_ip_buf)), ntohs(flow->dst_port), - json_object_to_json_string(a)); -#endif - - f = getFlow(NULL, NULL, flow->vlan_id, 0, 0, 0, - icmp_info, - &flow->src_ip, &flow->dst_ip, - flow->src_port, flow->dst_port, - flow->l4_proto, &src2dst_direction, - flow->first_switched, flow->last_switched, - 0, &new_flow, - true /* create it if we didn't receive netflow yet */); - - if (f) { - f->setExternalAlert(json_object_get(a), severity); - } else { -#ifdef SYSLOG_DEBUG - ntop->getTrace()->traceEvent(TRACE_INFO, "[Suricata] Flow matching the alert not found (ignored)", - json_object_to_json_string(a)); -#endif - } - - if (companionsEnabled()) { - flow->external_alert = strdup(json_object_to_json_string(a)); - flow->external_alert_severity = severity; - deliverFlowToCompanions(flow); - } - - } else { - /* Other alert types? (e.g. host) */ -#ifdef SYSLOG_DEBUG - ntop->getTrace()->traceEvent(TRACE_NORMAL, "[Suricata] Alert JSON: %s (ignored)", - json_object_to_json_string(a)); -#endif - } -} - -/* **************************************************** */ - u_int8_t SyslogParserInterface::parseLog(char *log_line) { char *tmp, *content, *application; - enum json_tokener_error jerr = json_tokener_success; int num_flows = 0; #ifdef SYSLOG_DEBUG @@ -234,138 +66,7 @@ u_int8_t SyslogParserInterface::parseLog(char *log_line) { application, content); #endif - if(strstr(log_line, "suricata") != NULL) { - json_object *o; - ParsedFlow flow; - ICMPinfo icmp_info; - - /* Suricata Log */ - -#ifdef SYSLOG_DEBUG - ntop->getTrace()->traceEvent(TRACE_DEBUG, "[Suricata] JSON: %s", content); -#endif - - o = json_tokener_parse_verbose(content, &jerr); - - if(o) { - json_object *w, *f, *a; - const char *timestamp = ""; - const char *event_type = ""; -#ifdef SYSLOG_DEBUG - char src_ip_buf[64], dst_ip_buf[64]; -#endif - - if(json_object_object_get_ex(o, "timestamp", &w)) timestamp = json_object_get_string(w); - if(json_object_object_get_ex(o, "event_type", &w)) event_type = json_object_get_string(w); - - //if(json_object_object_get_ex(o, "flow_id", &w)) flow_id = json_object_get_string(w); - //if(json_object_object_get_ex(o, "community_id", &w)) community_id = json_object_get_string(w); - //if(json_object_object_get_ex(o, "app_proto", &w)) app_proto = json_object_get_string(w); - if(json_object_object_get_ex(o, "vlan", &w)) flow.vlan_id = json_object_get_int(w); - if(json_object_object_get_ex(o, "src_ip", &w)) flow.src_ip.set((char *) json_object_get_string(w)); - if(json_object_object_get_ex(o, "dest_ip", &w)) flow.dst_ip.set((char *) json_object_get_string(w)); - if(json_object_object_get_ex(o, "src_port", &w)) flow.src_port = htons(json_object_get_int(w)); - if(json_object_object_get_ex(o, "dest_port", &w)) flow.dst_port = htons(json_object_get_int(w)); - if(json_object_object_get_ex(o, "proto", &w)) flow.l4_proto = Utils::l4name2proto((char *) json_object_get_string(w)); - - if(flow.l4_proto == 1 /* ICMP */) { - if(json_object_object_get_ex(o, "icmp_type", &w)) - icmp_info.setType(json_object_get_int(w)); - if(json_object_object_get_ex(o, "icmp_code", &w)) - icmp_info.setCode(json_object_get_int(w)); - } - - if(strcmp(event_type, "alert") == 0 && json_object_object_get_ex(o, "alert", &a)) { - bool flow_alert = false; - - /* Suricata Alert */ - -#ifdef SYSLOG_DEBUG - ntop->getTrace()->traceEvent(TRACE_NORMAL, "[Suricata] Alert JSON: %s", content); -#endif - - if (le) le->handleEvent(application, content); - - if(json_object_object_get_ex(o, "flow", &f)) { - parseSuricataFlow(f, &flow); - - if (!flow.last_switched) - flow.last_switched = Utils::str2epoch(timestamp); - - flow_alert = true; - } - - parseSuricataAlert(a, &flow, &icmp_info, flow_alert); - - } else if(strcmp(event_type, "netflow") == 0 && json_object_object_get_ex(o, "netflow", &f)) { -#ifdef USE_SURICATA_NETFLOW - - /* Suricata Flow (Unidirectional "netflow") */ - - parseSuricataNetflow(f, &flow); - -#ifdef SYSLOG_DEBUG - ntop->getTrace()->traceEvent(TRACE_DEBUG, "[Suricata] Netflow %s:%u <-> %s:%u [start=%u][end=%u][%u pkts][%u bytes]", - flow.src_ip.print(src_ip_buf, sizeof(src_ip_buf)), ntohs(flow.src_port), - flow.dst_ip.print(dst_ip_buf, sizeof(dst_ip_buf)), ntohs(flow.dst_port), - flow.first_switched, flow.last_switched, - flow.in_pkts, flow.in_bytes); -#endif - - processFlow(&flow); - num_flows++; -#endif /* USE_SURICATA_NETFLOW */ - } else if(strcmp(event_type, "flow") == 0 && json_object_object_get_ex(o, "flow", &f)) { -#ifndef USE_SURICATA_NETFLOW - - /* Suricata Flow (Bidirectional) */ - - parseSuricataFlow(f, &flow); - -#ifdef SYSLOG_DEBUG - ntop->getTrace()->traceEvent(TRACE_DEBUG, "[Suricata] Flow %s:%u <-> %s:%u [start=%u][end=%u][%u/%u pkts][%u/%u bytes]", - flow.src_ip.print(src_ip_buf, sizeof(src_ip_buf)), ntohs(flow.src_port), - flow.dst_ip.print(dst_ip_buf, sizeof(dst_ip_buf)), ntohs(flow.dst_port), - flow.first_switched, flow.last_switched, - flow.in_pkts, flow.out_pkts, flow.in_bytes, flow.out_bytes); -#endif - - processFlow(&flow, false); - num_flows++; -#endif /* USE_SURICATA_NETFLOW */ - - } else if(strcmp(event_type, "http") == 0 && json_object_object_get_ex(o, "http", &f)) { - /* Suricata HTTP metadata */ - parseSuricataHTTP(f, &flow); - processFlow(&flow); - - } else if(strcmp(event_type, "dns") == 0 && json_object_object_get_ex(o, "dns", &f)) { - /* Suricata DNS metadata */ - parseSuricataDNS(f, &flow); - processFlow(&flow); - - } else if(strcmp(event_type, "tls") == 0 && json_object_object_get_ex(o, "tls", &f)) { - /* Suricata DNS metadata */ - parseSuricataTLS(f, &flow); - processFlow(&flow); - -#ifdef SYSLOG_DEBUG - } else { - /* Other Events */ - ntop->getTrace()->traceEvent(TRACE_NORMAL, "[Suricata] Event '%s' [%s] JSON: %s (ignored)", event_type, timestamp, content); -#endif - } - - json_object_put(o); - } - -#ifdef SYSLOG_DEBUG - } else { - /* System Log */ - ntop->getTrace()->traceEvent(TRACE_DEBUG, "[SYSLOG] System Event (%s): %s (ignored)", log_line, content); - if (le) le->handleEvent(application, content); -#endif - } + if (le) le->handleEvent(application, content); return num_flows; } @@ -373,9 +74,7 @@ u_int8_t SyslogParserInterface::parseLog(char *log_line) { /* **************************************************** */ void SyslogParserInterface::lua(lua_State* vm) { - NetworkInterface::lua(vm); - } /* **************************************************** */