diff --git a/doc/src/api/lua_c/flow/flow.lua b/doc/src/api/lua_c/flow/flow.lua index 7fbc6d15be..d32815d7fa 100644 --- a/doc/src/api/lua_c/flow/flow.lua +++ b/doc/src/api/lua_c/flow/flow.lua @@ -1,10 +1,23 @@ ---! @brief Get the status of the flow ---! @return table:
---! flow.status: the most relevant flow status, see flow_consts.lua
---! flow.idle: true if the flow is idle
---! status_map: the flow status bitmap +--! @brief Check if the flow is blacklisted +--! @return true if blacklisted, false otherwise +function flow.isBlacklistedFlow() + +--! @brief Get the status bitmap of the flow +--! @return the flow status bitmap function flow.getStatus() +--! @brief Sets a bit into the flow status +--! @param status_bit the status bit to set, see flow_consts.lua +--! @notes This is used to indicate that the Flow has a possible problem. +function flow.addStatus(status_bit) + +--! @brief Trigger an alert on the current flow +--! @param alerted_status the flow status which is causing the alert generation +--! @param alert_type the alert_id of the alert to generate (see alert_consts.alert_types) +--! @param alert_severity the severity_id of the alert to generate (see alert_consts.alert_types) +--! @notes alert_json an optional string message or json to store into the alert +function flow.triggerAlert(alerted_status, alert_type, alert_severity, alert_json = nil) + --! @brief Get the Layer-4 and the Layer-7 protocols --! @return table:
--! proto.ndpi_id: the L7 nDPI protocol ID
diff --git a/doc/src/api/user_scripts/flow_hooks.rst b/doc/src/api/user_scripts/flow_hooks.rst index 59980aa776..0539907b30 100644 --- a/doc/src/api/user_scripts/flow_hooks.rst +++ b/doc/src/api/user_scripts/flow_hooks.rst @@ -73,7 +73,7 @@ The `flow` object keeps a context of the currently processed flow. The `flow.getClientGeolocation` and `flow.getServerGeolocation` functions extract the peers country information. The country code is then processed to determine if any of the peers is located in China (country code `CN`). -An easier way to access all the flow information would be to call `flow.getStatus()`, but this should not be used in +An easier way to access all the flow information would be to call `flow.getInfo()`, but this should not be used in production as it's a very expensive call. See the `Flow API`_ for a documentation of the available functions. diff --git a/include/Flow.h b/include/Flow.h index 6d8a58883f..365017a343 100644 --- a/include/Flow.h +++ b/include/Flow.h @@ -54,9 +54,13 @@ class Flow : public GenericHashEntry { u_int32_t vrfId; u_int8_t protocol, src2dst_tcp_flags, dst2src_tcp_flags; u_int16_t alert_score; - Bitmap last_notified_status_map; + Bitmap status_map, last_notified_status_map; time_t performed_lua_calls[FLOW_LUA_CALL_MAX_VAL]; struct ndpi_flow_struct *ndpiFlow; + FlowStatus alerted_status; + AlertType alert_type; + AlertLevel alert_level; + char *tmp_alert_json; /* When the interface isViewed(), the corresponding view needs to acknowledge the purge before the flow can actually be deleted from memory. This guarantees the view has @@ -219,7 +223,6 @@ class Flow : public GenericHashEntry { bool isLowGoodput() const; static void updatePacketStats(InterarrivalStats *stats, const struct timeval *when, bool update_iat); bool isReadyToBeMarkedAsIdle(); - bool isBlacklistedFlow() const; inline bool isDeviceAllowedProtocol() const { return(!cli_host || !srv_host || ((cli_host->getDeviceAllowedProtocolStatus(ndpiDetectedProtocol, true) == device_proto_allowed) && @@ -251,7 +254,13 @@ class Flow : public GenericHashEntry { time_t _first_seen, time_t _last_seen); ~Flow(); + inline Bitmap getStatusBitmap() { return(status_map); } + inline void addStatus(FlowStatus status) { status_map.setBit(status); } FlowStatus getFlowStatus(Bitmap *status_map) const; + void triggerAlert(AlertType atype, AlertLevel severity, const char*alert_json); + inline void setAlertedStatus(FlowStatus status) { alerted_status = status; }; + + bool isBlacklistedFlow() const; struct site_categories* getFlowCategory(bool force_categorization); void freeDPIMemory(); static const ndpi_protocol ndpiUnknownProtocol; diff --git a/scripts/callbacks/interface/flow.lua b/scripts/callbacks/interface/flow.lua index 8174604aab..493ff2b650 100644 --- a/scripts/callbacks/interface/flow.lua +++ b/scripts/callbacks/interface/flow.lua @@ -11,7 +11,10 @@ package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path require "lua_utils" require "flow_utils" +require "alert_utils" local user_scripts = require("user_scripts") +local alert_consts = require("alert_consts") +local flow_consts = require("flow_consts") if ntop.isPro() then package.path = dirs.installdir .. "/pro/scripts/lua/modules/?.lua;" .. package.path @@ -94,8 +97,10 @@ end -- modules, calling them one after one. -- @param l4_proto the L4 protocol of the flow -- @param mod_fn the callback to call +-- @return true if some module was called, false otherwise local function call_modules(l4_proto, mod_fn) local hooks = available_modules.l4_hooks[l4_proto] + local rv = false if(hooks ~= nil) then hooks = hooks[mod_fn] @@ -103,7 +108,7 @@ local function call_modules(l4_proto, mod_fn) if(hooks == nil) then if do_trace then print(string.format("No flow.lua modules, skipping %s(%d) for %s\n", mod_fn, l4_proto, shortFlowLabel(flow.getInfo()))) end - return + return(false) end -- TODO too expensive, remove @@ -118,6 +123,35 @@ local function call_modules(l4_proto, mod_fn) if do_trace then print(string.format("%s() [check: %s]: %s\n", mod_fn, mod_key, shortFlowLabel(info))) end hook_fn(params) + rv = true + end + + return(rv) +end + +-- ################################################################# + +-- @brief This function determines the most critical problem of the flow +-- and possibly triggers an alert +-- @params status the flow status bitmap +local function checkFlowStatus(status) + local alert_type = nil + local alerted_status = nil + local severity = alert_consts.alert_severities.error.severity_id + + if(status == 0) then + return + end + + if(ntop.bitmapIsSet(status, flow_consts.status_blacklisted)) then + alerted_status = flow_consts.status_blacklisted + alert_type = alert_consts.alert_types.flow_blacklisted.alert_id + end + + if(alert_type ~= nil) then + if do_trace then print(string.format("flow.triggerAlert(type=%s, severity=%s)\n", alertTypeRaw(alert_type), alertSeverityRaw(severity))) end + + flow.triggerAlert(alerted_status, alert_type, severity) end end @@ -126,13 +160,23 @@ end -- Given an L4 protocol, we must call both the hooks registered for that protocol and -- the hooks registered for any L4 protocol (id 255) function protocolDetected(l4_proto) - call_modules(l4_proto, "protocolDetected") + local initial_status = flow.getStatus() + + if call_modules(l4_proto, "protocolDetected") then + local new_status = flow.getStatus() + + if(new_status ~= initial_status) then + -- The flow status has changed, possibly generate the alert + checkFlowStatus(new_status) + end + end end -- ################################################################# function statusChanged(l4_proto) call_modules(l4_proto, "statusChanged") + checkFlowStatus(flow.getStatus()) end -- ################################################################# diff --git a/scripts/callbacks/interface/flow/blacklisted.lua b/scripts/callbacks/interface/flow/blacklisted.lua index 706294dfe9..4bea2414c3 100644 --- a/scripts/callbacks/interface/flow/blacklisted.lua +++ b/scripts/callbacks/interface/flow/blacklisted.lua @@ -2,10 +2,8 @@ -- (C) 2019 - ntop.org -- -local alerts_api = require "alerts_api" -local alert_consts = require "alert_consts" +local flow_consts = require("flow_consts") local user_scripts = require("user_scripts") -local do_trace = false -- ################################################################# @@ -24,15 +22,9 @@ local script = { -- ################################################################# -function script.setup() - return false -- TODO: activate when migration to lua flow alerts completed -end - --- ################################################################# - -function script.hooks.protocolDetected(flow_info) - if flow_info["cli.blacklisted"] or flow_info["srv.blacklisted"] then - alerts_api.storeFlowAlert(alert_consts.alert_types.flow_blacklisted, alert_consts.alert_severities.error, flow_info) +function script.hooks.protocolDetected(params) + if flow.isBlacklisted() then + flow.addStatus(flow_consts.status_blacklisted) end end diff --git a/src/Flow.cpp b/src/Flow.cpp index b2c55d2ee7..af1a6bf6bb 100644 --- a/src/Flow.cpp +++ b/src/Flow.cpp @@ -47,6 +47,11 @@ Flow::Flow(NetworkInterface *_iface, flow_dropped_counts_increased = false, vrfId = 0; alert_score = CONST_NO_SCORE_SET; + tmp_alert_json = NULL; + alert_type = alert_none; + alert_level = alert_level_none; + alerted_status = status_normal; + alert_rowid = -1; last_notified_status_map.setBit(status_normal); purge_acknowledged_mark = detection_completed = update_flow_port_stats = false; @@ -295,6 +300,7 @@ Flow::~Flow() { freeDPIMemory(); if(icmp_info) delete(icmp_info); if(external_alert) json_object_put(external_alert); + if(tmp_alert_json) free(tmp_alert_json); } /* *************************************** */ @@ -329,10 +335,12 @@ bool Flow::triggerAlerts() const { /* *************************************** */ +// TODO: refactor void Flow::dumpFlowAlert() { time_t when; FlowStatus status; Bitmap status_map; + bool is_from_lua = false; if(!triggerAlerts()) return; @@ -340,7 +348,13 @@ void Flow::dumpFlowAlert() { status = getFlowStatus(&status_map); if(!isFlowAlerted()) { - bool do_dump = Utils::dumpFlowStatus(status); + bool do_dump; + is_from_lua = (alert_type != alert_none); + + if(is_from_lua) + do_dump = true; + else + do_dump = Utils::dumpFlowStatus(status); #ifdef HAVE_NEDGE /* NOTE: this must be explicitly re-checked as a more specific alert @@ -373,7 +387,12 @@ void Flow::dumpFlowAlert() { } if(do_dump) { - iface->getAlertsManager()->storeFlowAlert(this); + if(is_from_lua) { + iface->getAlertsManager()->storeFlowAlert(this, alert_type, alert_level, tmp_alert_json); + + if(tmp_alert_json) free(tmp_alert_json); + } else + iface->getAlertsManager()->storeFlowAlert(this); setFlowAlerted(); @@ -3681,6 +3700,10 @@ FlowStatus Flow::getFlowStatus(Bitmap *status_map) const { #endif u_int16_t l7proto = ndpi_get_lower_proto(ndpiDetectedProtocol); + if(alerted_status != status_normal) + // TODO refactor + return(alerted_status); + status_map->reset(); if(iface->isPacketInterface() && iface->is_purge_idle_interface() @@ -3834,9 +3857,6 @@ FlowStatus Flow::getFlowStatus(Bitmap *status_map) const { if(get_protocol_breed() == NDPI_PROTOCOL_DANGEROUS) status_map->setBit(status = status_potentially_dangerous); - if(isBlacklistedFlow()) - status_map->setBit(status = status_blacklisted); - if(status == status_normal) status_map->setBit(status_normal); @@ -4620,3 +4640,16 @@ bool Flow::hasDissectedTooManyPackets() { return(num_packets >= NDPI_MIN_NUM_PACKETS); } + +/* ***************************************************** */ + +void Flow::triggerAlert(AlertType atype, AlertLevel severity, const char*alert_json) { + if((alert_type != alert_none) || isFlowAlerted()) { + /* Triggering multiple alerts is not supported */ + return; + } + + tmp_alert_json = alert_json ? strdup(alert_json) : NULL; + alert_level = severity; + alert_type = atype; /* set this as the last thing as "a notification" to avoid concurrency issues */ +} diff --git a/src/LuaEngine.cpp b/src/LuaEngine.cpp index 5789137bed..216e6de8ea 100644 --- a/src/LuaEngine.cpp +++ b/src/LuaEngine.cpp @@ -8549,12 +8549,72 @@ static int ntop_flow_get_server_mud_pref(lua_State* vm) { // ***API*** static int ntop_flow_get_status(lua_State* vm) { Flow *f = ntop_flow_get_context_flow(vm); + if(!f) return(CONST_LUA_ERROR); - lua_newtable(vm); + lua_pushinteger(vm, f->getStatusBitmap().get()); - if(f) f->lua_get_status(vm); + return(CONST_LUA_OK); +} - return CONST_LUA_OK; +/* ****************************************** */ + +// ***API*** +static int ntop_flow_add_status(lua_State* vm) { + FlowStatus new_status; + Flow *f = ntop_flow_get_context_flow(vm); + + if(!f) return(CONST_LUA_ERROR); + + if(ntop_lua_check(vm, __FUNCTION__, 1, LUA_TNUMBER) != CONST_LUA_OK) return(CONST_LUA_ERROR); + new_status = (FlowStatus)lua_tonumber(vm, 1); + + f->addStatus(new_status); + lua_pushnil(vm); + + return(CONST_LUA_OK); +} + +/* ****************************************** */ + +// ***API*** +static int ntop_flow_trigger_alert(lua_State* vm) { + Flow *f = ntop_flow_get_context_flow(vm); + FlowStatus status; + AlertType atype; + AlertLevel severity; + const char *status_info = NULL; + + if(!f) return(CONST_LUA_ERROR); + + if(ntop_lua_check(vm, __FUNCTION__, 1, LUA_TNUMBER) != CONST_LUA_OK) return(CONST_LUA_ERROR); + status = (FlowStatus)lua_tonumber(vm, 1); + + if(ntop_lua_check(vm, __FUNCTION__, 2, LUA_TNUMBER) != CONST_LUA_OK) return(CONST_LUA_ERROR); + atype = (AlertType)lua_tonumber(vm, 2); + + if(ntop_lua_check(vm, __FUNCTION__, 3, LUA_TNUMBER) != CONST_LUA_OK) return(CONST_LUA_ERROR); + severity = (AlertLevel)lua_tonumber(vm, 3); + + if(lua_type(vm, 3) == LUA_TSTRING) + status_info = lua_tostring(vm, 3); + + f->setAlertedStatus(status); + f->triggerAlert(atype, severity, status_info); + lua_pushnil(vm); + + return(CONST_LUA_OK); +} + +/* ****************************************** */ + +// ***API*** +static int ntop_flow_is_blacklisted(lua_State* vm) { + Flow *f = ntop_flow_get_context_flow(vm); + + if(!f) return(CONST_LUA_ERROR); + + lua_pushboolean(vm, f->isBlacklistedFlow()); + return(CONST_LUA_OK); } /* ****************************************** */ @@ -8572,7 +8632,6 @@ static int ntop_flow_get_protocols(lua_State* vm) { /* ****************************************** */ -// ***API*** static int ntop_flow_get_bytes(lua_State* vm) { Flow *f = ntop_flow_get_context_flow(vm); @@ -8585,7 +8644,6 @@ static int ntop_flow_get_bytes(lua_State* vm) { /* ****************************************** */ -// ***API*** static int ntop_flow_get_dir_traffic(lua_State* vm, bool cli2srv) { Flow *f = ntop_flow_get_context_flow(vm); @@ -8598,21 +8656,18 @@ static int ntop_flow_get_dir_traffic(lua_State* vm, bool cli2srv) { /* ****************************************** */ -// ***API*** static int ntop_flow_get_cli2srv_traffic(lua_State* vm) { return ntop_flow_get_dir_traffic(vm, true /* Client to Server */); } /* ****************************************** */ -// ***API*** static int ntop_flow_get_srv2cli_traffic(lua_State* vm) { return ntop_flow_get_dir_traffic(vm, false /* Server to Client */); } /* ****************************************** */ -// ***API*** static int ntop_flow_get_dir_iat(lua_State* vm, bool cli2srv) { Flow *f = ntop_flow_get_context_flow(vm); @@ -8625,21 +8680,18 @@ static int ntop_flow_get_dir_iat(lua_State* vm, bool cli2srv) { /* ****************************************** */ -// ***API*** static int ntop_flow_get_cli2srv_iat(lua_State* vm) { return ntop_flow_get_dir_iat(vm, true /* Client to Server */); } /* ****************************************** */ -// ***API*** static int ntop_flow_get_srv2cli_iat(lua_State* vm) { return ntop_flow_get_dir_iat(vm, false /* Server to Client */); } /* ****************************************** */ -// ***API*** static int ntop_flow_get_packets(lua_State* vm) { Flow *f = ntop_flow_get_context_flow(vm); @@ -8652,7 +8704,6 @@ static int ntop_flow_get_packets(lua_State* vm) { /* ****************************************** */ -// ***API*** static int ntop_flow_get_time(lua_State* vm) { Flow *f = ntop_flow_get_context_flow(vm); @@ -8665,7 +8716,6 @@ static int ntop_flow_get_time(lua_State* vm) { /* ****************************************** */ -// ***API*** static int ntop_flow_get_ip(lua_State* vm, bool client) { Flow *f = ntop_flow_get_context_flow(vm); @@ -8678,21 +8728,18 @@ static int ntop_flow_get_ip(lua_State* vm, bool client) { /* ****************************************** */ -// ***API*** static int ntop_flow_get_cli_ip(lua_State* vm) { return ntop_flow_get_ip(vm, true /* Client */); } /* ****************************************** */ -// ***API*** static int ntop_flow_get_srv_ip(lua_State* vm) { return ntop_flow_get_ip(vm, false /* Server */); } /* ****************************************** */ -// ***API*** static int ntop_flow_get_info(lua_State* vm, bool client) { Flow *f = ntop_flow_get_context_flow(vm); @@ -8705,21 +8752,18 @@ static int ntop_flow_get_info(lua_State* vm, bool client) { /* ****************************************** */ -// ***API*** static int ntop_flow_get_cli_info(lua_State* vm) { return ntop_flow_get_info(vm, true /* Client */); } /* ****************************************** */ -// ***API*** static int ntop_flow_get_srv_info(lua_State* vm) { return ntop_flow_get_info(vm, false /* Server */); } /* ****************************************** */ -// ***API*** static int ntop_flow_get_ssl_info(lua_State* vm) { Flow *f = ntop_flow_get_context_flow(vm); @@ -8732,7 +8776,6 @@ static int ntop_flow_get_ssl_info(lua_State* vm) { /* ****************************************** */ -// ***API*** static int ntop_flow_get_ssh_info(lua_State* vm) { Flow *f = ntop_flow_get_context_flow(vm); @@ -8745,7 +8788,6 @@ static int ntop_flow_get_ssh_info(lua_State* vm) { /* ****************************************** */ -// ***API*** static int ntop_flow_get_http_info(lua_State* vm) { Flow *f = ntop_flow_get_context_flow(vm); @@ -8758,7 +8800,6 @@ static int ntop_flow_get_http_info(lua_State* vm) { /* ****************************************** */ -// ***API*** static int ntop_flow_get_dns_info(lua_State* vm) { Flow *f = ntop_flow_get_context_flow(vm); @@ -8771,7 +8812,6 @@ static int ntop_flow_get_dns_info(lua_State* vm) { /* ****************************************** */ -// ***API*** static int ntop_flow_get_icmp_info(lua_State* vm) { Flow *f = ntop_flow_get_context_flow(vm); @@ -8784,7 +8824,6 @@ static int ntop_flow_get_icmp_info(lua_State* vm) { /* ****************************************** */ -// ***API*** static int ntop_flow_get_tcp_info(lua_State* vm) { Flow *f = ntop_flow_get_context_flow(vm); @@ -8797,7 +8836,6 @@ static int ntop_flow_get_tcp_info(lua_State* vm) { /* ****************************************** */ -// ***API*** static int ntop_flow_get_port(lua_State* vm, bool client) { Flow *f = ntop_flow_get_context_flow(vm); @@ -8810,21 +8848,18 @@ static int ntop_flow_get_port(lua_State* vm, bool client) { /* ****************************************** */ -// ***API*** static int ntop_flow_get_cli_port(lua_State* vm) { return ntop_flow_get_port(vm, true /* Client */); } /* ****************************************** */ -// ***API*** static int ntop_flow_get_srv_port(lua_State* vm) { return ntop_flow_get_port(vm, false /* Server */); } /* ****************************************** */ -// ***API*** static int ntop_flow_get_geoloc(lua_State* vm, bool client) { Flow *f = ntop_flow_get_context_flow(vm); @@ -8837,14 +8872,12 @@ static int ntop_flow_get_geoloc(lua_State* vm, bool client) { /* ****************************************** */ -// ***API*** static int ntop_flow_get_cli_geoloc(lua_State* vm) { return ntop_flow_get_geoloc(vm, true /* Client */); } /* ****************************************** */ -// ***API*** static int ntop_flow_get_srv_geoloc(lua_State* vm) { return ntop_flow_get_geoloc(vm, false /* Server */); } @@ -10201,7 +10234,10 @@ static const luaL_Reg ntop_flow_reg[] = { { "serializeServerByMac", ntop_flow_serialize_server_by_mac }, { "storeAlert", ntop_flow_store_alert }, + { "isBlacklisted", ntop_flow_is_blacklisted }, { "getStatus", ntop_flow_get_status }, + { "addStatus", ntop_flow_add_status }, + { "triggerAlert", ntop_flow_trigger_alert }, { "getProtocols", ntop_flow_get_protocols }, { "getBytes", ntop_flow_get_bytes }, { "getClient2ServerTraffic", ntop_flow_get_cli2srv_traffic },