diff --git a/httpdocs/misc/db_schema_clickhouse.sql b/httpdocs/misc/db_schema_clickhouse.sql index 9a2b82bc74..936e28ae1a 100644 --- a/httpdocs/misc/db_schema_clickhouse.sql +++ b/httpdocs/misc/db_schema_clickhouse.sql @@ -2636,3 +2636,25 @@ ALTER TABLE `flow_l4_map` MODIFY COMMENT 'Startup-populated lookup: flows.PROTOC ALTER TABLE `flow_l4_map` MODIFY COLUMN `PROTOCOL` COMMENT 'IANA IP protocol number (matches flows.PROTOCOL, e.g. 6=TCP 17=UDP 1=ICMP)'; @ ALTER TABLE `flow_l4_map` MODIFY COLUMN `NAME` COMMENT 'Protocol name (e.g. "TCP", "UDP", "ICMP")'; + +@ + +/* AS data populated on startup */ +CREATE TABLE IF NOT EXISTS `asn_info` ( + `asn` UInt32, + `handle` String, + `description` String, + `country_code` String +) ENGINE = ReplacingMergeTree() + PRIMARY KEY (asn) + ORDER BY (asn); +@ +ALTER TABLE `asn_info` MODIFY COMMENT 'AS (Autonomous System) data loaded from geoip/as.csv on startup. Maps ASN to handle, description, and ISO 3166-1 country code. Refreshed on every ntopng startup via TRUNCATE + bulk INSERT.'; +@ +ALTER TABLE `asn_info` MODIFY COLUMN `asn` COMMENT 'Autonomous System Number (matches flows.SRC_ASN / DST_ASN)'; +@ +ALTER TABLE `asn_info` MODIFY COLUMN `handle` COMMENT 'BGP handle / RIR registry object name for this AS (e.g. LVLT-1)'; +@ +ALTER TABLE `asn_info` MODIFY COLUMN `description` COMMENT 'Human-readable organization name for this AS'; +@ +ALTER TABLE `asn_info` MODIFY COLUMN `country_code` COMMENT 'ISO 3166-1 alpha-2 country code of the AS registrant'; diff --git a/httpdocs/misc/db_schema_clickhouse_cluster.sql b/httpdocs/misc/db_schema_clickhouse_cluster.sql index 95329827b3..ab4aa6884e 100644 --- a/httpdocs/misc/db_schema_clickhouse_cluster.sql +++ b/httpdocs/misc/db_schema_clickhouse_cluster.sql @@ -2319,3 +2319,25 @@ ALTER TABLE `hourly_asn` ON CLUSTER '$CLUSTER' MODIFY COLUMN `PROBE_IP` COMMENT ALTER TABLE `hourly_asn` ON CLUSTER '$CLUSTER' MODIFY COLUMN `INPUT_SNMP` COMMENT 'SNMP input interface index from NetFlow/IPFIX'; @ ALTER TABLE `hourly_asn` ON CLUSTER '$CLUSTER' MODIFY COLUMN `OUTPUT_SNMP` COMMENT 'SNMP output interface index from NetFlow/IPFIX'; + +@ + +/* AS data (local, non replicated) populated on startup */ +CREATE TABLE IF NOT EXISTS `asn_info` ( + `asn` UInt32, + `handle` String, + `description` String, + `country_code` String +) ENGINE = ReplacingMergeTree() + PRIMARY KEY (asn) + ORDER BY (asn); +@ +ALTER TABLE `asn_info` MODIFY COMMENT 'AS (Autonomous System) reference data loaded from geoip/as.csv on startup. Maps ASN to handle, description, and ISO 3166-1 country code. Refreshed on every ntopng startup via TRUNCATE + bulk INSERT.'; +@ +ALTER TABLE `asn_info` MODIFY COLUMN `asn` COMMENT 'Autonomous System Number (matches flows.SRC_ASN / DST_ASN)'; +@ +ALTER TABLE `asn_info` MODIFY COLUMN `handle` COMMENT 'BGP handle / RIR registry object name for this AS (e.g. LVLT-1)'; +@ +ALTER TABLE `asn_info` MODIFY COLUMN `description` COMMENT 'Human-readable organization name for this AS'; +@ +ALTER TABLE `asn_info` MODIFY COLUMN `country_code` COMMENT 'ISO 3166-1 alpha-2 country code of the AS registrant'; diff --git a/include/Geolocation.h b/include/Geolocation.h index 83ef493b3b..de20da73a8 100644 --- a/include/Geolocation.h +++ b/include/Geolocation.h @@ -24,6 +24,14 @@ #include "ntop_includes.h" +struct asn_info_query_ctx_t { + char* handle_buf; + u_int handle_len; + char* desc_buf; + u_int desc_len; + bool found; +}; + class Geolocation { private: #ifdef HAVE_MAXMINDDB @@ -52,7 +60,8 @@ class Geolocation { bool getInfo(IpAddress* addr, char** continent_code, char** country_code, char** city, float* latitude, float* longitude); static void freeInfo(char** continent_code, char** country_code, char** city); - bool getASName(u_int32_t asn, char* asname, u_int asname_len); + bool getASName(u_int32_t asn, char* handle, u_int handle_len, + char* description, u_int description_len); }; #endif /* _GEOLOCATION_H_ */ diff --git a/include/ntop_defines.h b/include/ntop_defines.h index 184d405114..8f6d69261b 100644 --- a/include/ntop_defines.h +++ b/include/ntop_defines.h @@ -259,7 +259,7 @@ #define DOMAIN_TO_CATEGORIZE "ntopng.domain.tocategorize" #define DOMAIN_WHITELIST_CAT "ntopng.domain.whitelist" #define DNS_CACHE "ntopng.dns.cache" -#define AS_NANE_CACHE "ntopng.as.cache" +#define AS_NAME_CACHE "ntopng.as.cache" #define DHCP_CACHE "ntopng.dhcp.%d.cache.%s" #define DHCP_STORM_QUEUE_NAME "ntopng.dhcp.storm.%d" #define ASSET_SERVICE_KEY "ntopng.asset.%d.%s" /* ifId.host */ diff --git a/scripts/lua/flow_details.lua b/scripts/lua/flow_details.lua index 3853303cb6..6b456bbe8a 100644 --- a/scripts/lua/flow_details.lua +++ b/scripts/lua/flow_details.lua @@ -86,7 +86,8 @@ function formatASN(v, peer_as, ip, is_client_as) local peer_as_name if peer_as and tonumber(peer_as) then - peer_as_name = shortenString(ntop.getASNameFromASN(tonumber(peer_as)), max_len) + local as_info = ntop.getASNameFromASN(tonumber(peer_as)) + peer_as_name = shortenString(as_info and (as_info.description or as_info.handle) or "", max_len) end if (peer_as_name == nil) then diff --git a/scripts/lua/modules/bgp_utils.lua b/scripts/lua/modules/bgp_utils.lua index 4edede37e4..1ea54d12e9 100644 --- a/scripts/lua/modules/bgp_utils.lua +++ b/scripts/lua/modules/bgp_utils.lua @@ -73,8 +73,9 @@ function bgp_utils.formatBgpBmpInfo(bgp_info) is_best_path = (peer.info.best_entry or false), } + local as_info = ntop.getASNameFromASN(tonumber(peer.info.asn)) asn_list[#asn_list + 1] = { - name = string.format("%s (%s)", peer.info.asn, ntop.getASNameFromASN(tonumber(peer.info.asn))), + name = string.format("%s (%s)", peer.info.asn, as_info and (as_info.description or as_info.handle) or ""), url = string.format("%s/lua/hosts_stats.lua?asn=%s", ntop.getHttpPrefix(), peer.info.asn), value = peer.info.asn, } @@ -100,8 +101,9 @@ function bgp_utils.formatBgpBmpInfo(bgp_info) -- Format AS path entries as clickable ASN links. if peer.info["as_path"] and #peer.info["as_path"] > 0 then for _, asn in ipairs(peer.info["as_path"]) do + local as_info = ntop.getASNameFromASN(tonumber(asn)) as_path[#as_path + 1] = { - name = string.format("%d (%s)", tonumber(asn), ntop.getASNameFromASN(tonumber(asn))), + name = string.format("%d (%s)", tonumber(asn), as_info and (as_info.description or as_info.handle) or ""), url = string.format("%s/lua/hosts_stats.lua?asn=%s", ntop.getHttpPrefix(), asn), value = asn, } diff --git a/scripts/lua/modules/format_utils.lua b/scripts/lua/modules/format_utils.lua index ca5269833f..f86177279d 100644 --- a/scripts/lua/modules/format_utils.lua +++ b/scripts/lua/modules/format_utils.lua @@ -6,6 +6,13 @@ local format_utils = {} local clock_start = os.clock() +-- Returns the AS display name (description, or handle as fallback) +local function getASDisplayName(asn) + local info = ntop.getASNameFromASN(asn) + if not info then return nil end + return info.description or info.handle +end + function format_utils.formatBgpBmpInfo(bgp_data) if bgp_data == nil then return " " end @@ -34,7 +41,7 @@ function format_utils.formatBgpBmpInfo(bgp_data) for _, peer in ipairs(peer_list) do out[#out + 1] = "" .. "" .. peer.info.asn .. " (" .. shortenString(ntop.getASNameFromASN(tonumber(peer.info.asn)), max_len) .. ")" + .. "\">" .. peer.info.asn .. " (" .. shortenString(getASDisplayName(tonumber(peer.info.asn)) or "", max_len) .. ")" end out[#out + 1] = "" @@ -54,7 +61,7 @@ function format_utils.formatBgpBmpInfo(bgp_data) local parts = {} for _, asn in ipairs(peer.info["as_path"]) do parts[#parts + 1] = "" .. asn .. " (" .. shortenString(ntop.getASNameFromASN(tonumber(asn)), max_len) .. ")" + .. "\">" .. asn .. " (" .. shortenString(getASDisplayName(tonumber(asn)) or "", max_len) .. ")" end as_path_string = (#parts == 0) and "Local" or table.concat(parts, "
  • ") end @@ -912,7 +919,7 @@ function format_utils.formatASN(asn, short_version, shorten_stringing) name = as_info.asname else -- if no asn info is present, curl to get ASN name - name = ntop.getASNameFromASN(asn) + name = getASDisplayName(asn) end if (shorten_stringing) then name = shortenString(name) @@ -960,7 +967,7 @@ function format_utils.formatASN_transit(v, peer_as, ip, is_client_as) -- Process peer/transit ASN if valid, non-zero, and different from main ASN if peer_as and peer_as ~= 0 and v ~= peer_as then local peer_asn = string.format('%s', ntop.getHttpPrefix(), peer_as, peer_as) - local peer_as_name = shortenString(ntop.getASNameFromASN(peer_as), max_len) + local peer_as_name = shortenString(getASDisplayName(peer_as) or "", max_len) local via = "" if peer_as_name and peer_as_name ~= "" then diff --git a/scripts/lua/rest/v2/get/asn/get_as_data.lua b/scripts/lua/rest/v2/get/asn/get_as_data.lua index 24303822f8..044ca32107 100644 --- a/scripts/lua/rest/v2/get/asn/get_as_data.lua +++ b/scripts/lua/rest/v2/get/asn/get_as_data.lua @@ -103,7 +103,8 @@ for key, value in pairs(ases_info or {}) do end record["asname"] = value["asname"] else - record["asname"] = ntop.getASNameFromASN(tonumber(record["asn"])) + local as_info = ntop.getASNameFromASN(tonumber(record["asn"])) + record["asname"] = as_info and (as_info.description or as_info.handle) or nil end if asn == 0 then diff --git a/scripts/lua/rest/v2/get/host/host_filters.lua b/scripts/lua/rest/v2/get/host/host_filters.lua index 1ed876ba60..e2475d05ce 100644 --- a/scripts/lua/rest/v2/get/host/host_filters.lua +++ b/scripts/lua/rest/v2/get/host/host_filters.lua @@ -235,9 +235,10 @@ if (not isEmptyString(country)) then end if (not isEmptyString(asn)) then + local as_info = ntop.getASNameFromASN(tonumber(asn)) or {} local as_filter = { {key = "asn", value = "", label = i18n("all")}, - {key = "asn", value = asn, label = ntop.getASNameFromASN(tonumber(asn))} + {key = "asn", value = asn, label = as_info.description or as_info.handle or nil} } rsp[#rsp + 1] = { action = "asn", diff --git a/src/Geolocation.cpp b/src/Geolocation.cpp index f7690d311d..632a3f0bd9 100644 --- a/src/Geolocation.cpp +++ b/src/Geolocation.cpp @@ -402,12 +402,11 @@ void Geolocation::testme() { for (u_int32_t i = 0; i < sizeof(ips) / sizeof(char*); i++) { char* ip = ips[i]; - ntop->getTrace()->traceEvent(TRACE_NORMAL, "Geolocating [%s]", ip); + ntop->getTrace()->traceEvent(TRACE_INFO, "Geolocating [%s]", ip); test.set(ip); if (test.get_sockaddr(&sa, &sa_len)) { - ntop->getTrace()->traceEvent(TRACE_NORMAL, - "Autonomous System Information", ip); + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Autonomous System Information", ip); /* TEST Autonomous Systems database */ result = MMDB_lookup_sockaddr(&geo_ip_asn_mmdb, sa, &mmdb_error); @@ -431,8 +430,7 @@ void Geolocation::testme() { "autonomous_system_number", NULL)) == MMDB_SUCCESS) { if (entry_data.has_data && entry_data.type == MMDB_DATA_TYPE_UINT32) - ntop->getTrace()->traceEvent(TRACE_NORMAL, "ASN: %d", - entry_data.uint32); + ntop->getTrace()->traceEvent(TRACE_NORMAL, "ASN: %d", entry_data.uint32); } else ntop->getTrace()->traceEvent( @@ -448,8 +446,7 @@ void Geolocation::testme() { if ((org = (char*)calloc(1, entry_data.data_size + 1))) { memcpy(org, entry_data.utf8_string, entry_data.data_size); - ntop->getTrace()->traceEvent(TRACE_NORMAL, "Organization: %s", - org); + ntop->getTrace()->traceEvent(TRACE_NORMAL, "Organization: %s", org); free(org); } } @@ -461,7 +458,6 @@ void Geolocation::testme() { MMDB_strerror(status)); } - /* TEST City database */ ntop->getTrace()->traceEvent(TRACE_NORMAL, "City Information", ip); result = MMDB_lookup_sockaddr(&geo_ip_city_mmdb, sa, &mmdb_error); @@ -491,19 +487,71 @@ void Geolocation::testme() { /* *************************************** */ -bool Geolocation::getASName(u_int32_t asn, char* asname, u_int asname_len) { +/* Callback for getASName(): reads handle (col 0) and description (col 1) from asn_info */ +static int asn_info_cb(std::vector* row, + std::vector* /* columns */, + void* user_data) { + asn_info_query_ctx_t* out = (asn_info_query_ctx_t*)user_data; + if (!out->found && row->size() >= 2) { + snprintf(out->handle_buf, out->handle_len, "%s", (*row)[0].c_str()); + snprintf(out->desc_buf, out->desc_len, "%s", (*row)[1].c_str()); + out->found = true; + } + return 0; +} + +bool Geolocation::getASName(u_int32_t asn, char* handle, u_int handle_len, + char* description, u_int description_len) { char buf[256], val[16], json[2048]; long rc; snprintf(val, sizeof(val), "%u", asn); - if (ntop->getRedis()->hashGet(AS_NANE_CACHE, val, asname, asname_len) == 0) { - return (true); /* Found on cache */ + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "ASN Lookup (%u)", asn); + + /* Redis cache (AS data stored as "handle|description", old entries have just "description") */ + char cached[512]; + if (ntop->getRedis()->hashGet(AS_NAME_CACHE, val, cached, sizeof(cached)) == 0) { + if (cached[0] == '\0') return (false); /* previously not found */ + char* pipe = strchr(cached, '|'); + if (pipe) { + snprintf(handle, handle_len, "%.*s", (int)(pipe - cached), cached); + snprintf(description, description_len, "%s", pipe + 1); + } else { + snprintf(handle, handle_len, "%s", cached); + snprintf(description, description_len, "%s", cached); + } + + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "AS INFO From Redis [%s]", cached); + + return (true); } - snprintf(buf, sizeof(buf), - "https://stat.ripe.net/data/as-overview/data.json?resource=AS%u", - asn); + /* Check asn_info table on DB before checking RIPE */ + NetworkInterface* iface = ntop->getFirstInterface(); + if (iface) { + DB* db = iface->getFlowsDB(); /* asn_info lives only in ClickHouse */ + if (db) { + asn_info_query_ctx_t ctx = {handle, handle_len, description, description_len, false}; + snprintf(buf, sizeof(buf), + "SELECT handle, description FROM asn_info WHERE asn = %u LIMIT 1", asn); + db->execSQLQuery(buf, true, true, asn_info_cb, &ctx); + if (ctx.found) { + /* Note: if either field is empty, use the other */ + if (handle[0] == '\0') snprintf(handle, handle_len, "%s", description); + if (description[0] == '\0') snprintf(description, description_len, "%s", handle); + snprintf(cached, sizeof(cached), "%s|%s", handle, description); + ntop->getRedis()->hashSet(AS_NAME_CACHE, val, cached); + + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "AS INFO From DB [%s]", cached); + + return (true); + } + } + } + + /* Last resort: get AS info from RIPE stat */ + snprintf(buf, sizeof(buf), "https://stat.ripe.net/data/as-overview/data.json?resource=AS%u", asn); json[0] = '\0'; rc = Utils::httpGet(buf, NULL, NULL, NULL, 5 /* connect timeout */, 10 /* download timeout */, json, sizeof(json)); @@ -533,10 +581,16 @@ bool Geolocation::getASName(u_int32_t asn, char* asname, u_int asname_len) { res = &res[1]; /* Skip space */ #endif - snprintf(asname, asname_len, "%s", res); + /* RIPE provides a single holder string — use it for both fields */ + snprintf(description, description_len, "%s", res); + snprintf(handle, handle_len, "%s", res); - ntop->getRedis()->hashSet(AS_NANE_CACHE, val, asname); + snprintf(cached, sizeof(cached), "%s|%s", handle, description); + ntop->getRedis()->hashSet(AS_NAME_CACHE, val, cached); json_object_put(o); + + //ntop->getTrace()->traceEvent(TRACE_NORMAL, "AS INFO From RIPE [%s]", cached); + return (true); } @@ -545,7 +599,7 @@ bool Geolocation::getASName(u_int32_t asn, char* asname, u_int asname_len) { } /* Not found */ - ntop->getRedis()->hashSet(AS_NANE_CACHE, val, ""); + ntop->getRedis()->hashSet(AS_NAME_CACHE, val, ""); return (false); } diff --git a/src/LuaEngineNtop.cpp b/src/LuaEngineNtop.cpp index e6180e5ff6..daea5e73ba 100644 --- a/src/LuaEngineNtop.cpp +++ b/src/LuaEngineNtop.cpp @@ -8638,20 +8638,23 @@ static int ntop_get_asn_name(lua_State* vm) { /* ****************************************** */ -/* @brief Returns the organization name for a given autonomous system number. Lua: ntop.getASNameFromASN(asn_number) → string */ +/* @brief Returns handle and description for a given ASN. Lua: ntop.getASNameFromASN(asn_number) → table{handle, description} or nil */ static int ntop_get_as_name_from_asn(lua_State* vm) { u_int32_t asn; - char as_name[128]; + char handle[128], description[256]; if (ntop_lua_check(vm, __FUNCTION__, 1, LUA_TNUMBER) != CONST_LUA_OK) return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_NO_RETURN_VALUE)); else asn = (u_int32_t)lua_tonumber(vm, 1); - if (ntop->getGeolocation()->getASName(asn, as_name, sizeof(as_name))) - lua_pushstring(vm, as_name); - else + if (ntop->getGeolocation()->getASName(asn, handle, sizeof(handle), description, sizeof(description))) { + lua_newtable(vm); + lua_push_str_table_entry(vm, "handle", handle); + lua_push_str_table_entry(vm, "description", description); + } else { lua_pushnil(vm); + } return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_ONE_RETURN_VALUE)); } diff --git a/third-party/clickhouse-cpp b/third-party/clickhouse-cpp index 620ec41bff..506248dd04 160000 --- a/third-party/clickhouse-cpp +++ b/third-party/clickhouse-cpp @@ -1 +1 @@ -Subproject commit 620ec41bff2fcaf01c4e72eb64fe47302a41b781 +Subproject commit 506248dd049318226805b99ddaf873008762bef4