vrr/ntopng
2
0
Fork 0
mirror of https://github.com/ntop/ntopng.git synced 2026-07-09 16:00:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Add AS information to the database from geoip/as.csv when available. Cache the information on redis. Use RIPE service as last resort.

Browse source
This commit is contained in:
Alfredo Cardigliano 2026-06-09 10:36:54 +02:00
parent 2073342042
commit 07e23d469a
12 changed files with 156 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

22
httpdocs/misc/db_schema_clickhouse.sql
View file

@ -2636,3 +2636,25 @@ ALTER TABLE `flow_l4_map` MODIFY COMMENT 'Startup-populated lookup: flows.PROTOC
ALTER TABLE `flow_l4_map` MODIFY COLUMN `PROTOCOL` COMMENT 'IANA IP protocol number (matches flows.PROTOCOL, e.g. 6=TCP 17=UDP 1=ICMP)';
@
ALTER TABLE `flow_l4_map` MODIFY COLUMN `NAME` COMMENT 'Protocol name (e.g. "TCP", "UDP", "ICMP")';
@
/* AS data populated on startup */
CREATE TABLE IF NOT EXISTS `asn_info` (
`asn` UInt32,
`handle` String,
`description` String,
`country_code` String
) ENGINE = ReplacingMergeTree()
PRIMARY KEY (asn)
ORDER BY (asn);
@
ALTER TABLE `asn_info` MODIFY COMMENT 'AS (Autonomous System) data loaded from geoip/as.csv on startup. Maps ASN to handle, description, and ISO 3166-1 country code. Refreshed on every ntopng startup via TRUNCATE + bulk INSERT.';
@
ALTER TABLE `asn_info` MODIFY COLUMN `asn` COMMENT 'Autonomous System Number (matches flows.SRC_ASN / DST_ASN)';
@
ALTER TABLE `asn_info` MODIFY COLUMN `handle` COMMENT 'BGP handle / RIR registry object name for this AS (e.g. LVLT-1)';
@
ALTER TABLE `asn_info` MODIFY COLUMN `description` COMMENT 'Human-readable organization name for this AS';
@
ALTER TABLE `asn_info` MODIFY COLUMN `country_code` COMMENT 'ISO 3166-1 alpha-2 country code of the AS registrant';

22
httpdocs/misc/db_schema_clickhouse_cluster.sql
View file

@ -2319,3 +2319,25 @@ ALTER TABLE `hourly_asn` ON CLUSTER '$CLUSTER' MODIFY COLUMN `PROBE_IP` COMMENT
ALTER TABLE `hourly_asn` ON CLUSTER '$CLUSTER' MODIFY COLUMN `INPUT_SNMP` COMMENT 'SNMP input interface index from NetFlow/IPFIX';
@
ALTER TABLE `hourly_asn` ON CLUSTER '$CLUSTER' MODIFY COLUMN `OUTPUT_SNMP` COMMENT 'SNMP output interface index from NetFlow/IPFIX';
@
/* AS data (local, non replicated) populated on startup */
CREATE TABLE IF NOT EXISTS `asn_info` (
`asn` UInt32,
`handle` String,
`description` String,
`country_code` String
) ENGINE = ReplacingMergeTree()
PRIMARY KEY (asn)
ORDER BY (asn);
@
ALTER TABLE `asn_info` MODIFY COMMENT 'AS (Autonomous System) reference data loaded from geoip/as.csv on startup. Maps ASN to handle, description, and ISO 3166-1 country code. Refreshed on every ntopng startup via TRUNCATE + bulk INSERT.';
@
ALTER TABLE `asn_info` MODIFY COLUMN `asn` COMMENT 'Autonomous System Number (matches flows.SRC_ASN / DST_ASN)';
@
ALTER TABLE `asn_info` MODIFY COLUMN `handle` COMMENT 'BGP handle / RIR registry object name for this AS (e.g. LVLT-1)';
@
ALTER TABLE `asn_info` MODIFY COLUMN `description` COMMENT 'Human-readable organization name for this AS';
@
ALTER TABLE `asn_info` MODIFY COLUMN `country_code` COMMENT 'ISO 3166-1 alpha-2 country code of the AS registrant';

11
include/Geolocation.h
View file

@ -24,6 +24,14 @@
#include "ntop_includes.h"
struct asn_info_query_ctx_t {
char* handle_buf;
u_int handle_len;
char* desc_buf;
u_int desc_len;
bool found;
};
class Geolocation {
private:
#ifdef HAVE_MAXMINDDB
@ -52,7 +60,8 @@ class Geolocation {
bool getInfo(IpAddress* addr, char** continent_code, char** country_code,
char** city, float* latitude, float* longitude);
static void freeInfo(char** continent_code, char** country_code, char** city);
bool getASName(u_int32_t asn, char* asname, u_int asname_len);
bool getASName(u_int32_t asn, char* handle, u_int handle_len,
char* description, u_int description_len);
};
#endif /* _GEOLOCATION_H_ */

2
include/ntop_defines.h
View file

@ -259,7 +259,7 @@
#define DOMAIN_TO_CATEGORIZE "ntopng.domain.tocategorize"
#define DOMAIN_WHITELIST_CAT "ntopng.domain.whitelist"
#define DNS_CACHE "ntopng.dns.cache"
#define AS_NANE_CACHE "ntopng.as.cache"
#define AS_NAME_CACHE "ntopng.as.cache"
#define DHCP_CACHE "ntopng.dhcp.%d.cache.%s"
#define DHCP_STORM_QUEUE_NAME "ntopng.dhcp.storm.%d"
#define ASSET_SERVICE_KEY "ntopng.asset.%d.%s" /* ifId.host */

3
scripts/lua/flow_details.lua
View file

@ -86,7 +86,8 @@ function formatASN(v, peer_as, ip, is_client_as)
local peer_as_name
if peer_as and tonumber(peer_as) then
peer_as_name = shortenString(ntop.getASNameFromASN(tonumber(peer_as)), max_len)
local as_info = ntop.getASNameFromASN(tonumber(peer_as))
peer_as_name = shortenString(as_info and (as_info.description or as_info.handle) or "", max_len)
end
if (peer_as_name == nil) then

6
scripts/lua/modules/bgp_utils.lua
View file

@ -73,8 +73,9 @@ function bgp_utils.formatBgpBmpInfo(bgp_info)
is_best_path = (peer.info.best_entry or false),
}
local as_info = ntop.getASNameFromASN(tonumber(peer.info.asn))
asn_list[#asn_list + 1] = {
name = string.format("%s (%s)", peer.info.asn, ntop.getASNameFromASN(tonumber(peer.info.asn))),
name = string.format("%s (%s)", peer.info.asn, as_info and (as_info.description or as_info.handle) or ""),
url = string.format("%s/lua/hosts_stats.lua?asn=%s", ntop.getHttpPrefix(), peer.info.asn),
value = peer.info.asn,
}
@ -100,8 +101,9 @@ function bgp_utils.formatBgpBmpInfo(bgp_info)
-- Format AS path entries as clickable ASN links.
if peer.info["as_path"] and #peer.info["as_path"] > 0 then
for _, asn in ipairs(peer.info["as_path"]) do
local as_info = ntop.getASNameFromASN(tonumber(asn))
as_path[#as_path + 1] = {
name = string.format("%d (%s)", tonumber(asn), ntop.getASNameFromASN(tonumber(asn))),
name = string.format("%d (%s)", tonumber(asn), as_info and (as_info.description or as_info.handle) or ""),
url = string.format("%s/lua/hosts_stats.lua?asn=%s", ntop.getHttpPrefix(), asn),
value = asn,
}

15
scripts/lua/modules/format_utils.lua
View file

@ -6,6 +6,13 @@ local format_utils = {}
local clock_start = os.clock()
-- Returns the AS display name (description, or handle as fallback)
local function getASDisplayName(asn)
local info = ntop.getASNameFromASN(asn)
if not info then return nil end
return info.description or info.handle
end
function format_utils.formatBgpBmpInfo(bgp_data)
if bgp_data == nil then return " " end
@ -34,7 +41,7 @@ function format_utils.formatBgpBmpInfo(bgp_data)
for _, peer in ipairs(peer_list) do
out[#out + 1] = "<td>"
.. "<A HREF=\"" .. ntop.getHttpPrefix() .. "/lua/hosts_stats.lua?asn=" .. peer.info.asn
.. "\">" .. peer.info.asn .. " (" .. shortenString(ntop.getASNameFromASN(tonumber(peer.info.asn)), max_len) .. ")</A>"
.. "\">" .. peer.info.asn .. " (" .. shortenString(getASDisplayName(tonumber(peer.info.asn)) or "", max_len) .. ")</A>"
end
out[#out + 1] = "</tr>"
@ -54,7 +61,7 @@ function format_utils.formatBgpBmpInfo(bgp_data)
local parts = {}
for _, asn in ipairs(peer.info["as_path"]) do
parts[#parts + 1] = "<A HREF=\"" .. ntop.getHttpPrefix() .. "/lua/hosts_stats.lua?asn=" .. asn
.. "\">" .. asn .. " (" .. shortenString(ntop.getASNameFromASN(tonumber(asn)), max_len) .. ")</A>"
.. "\">" .. asn .. " (" .. shortenString(getASDisplayName(tonumber(asn)) or "", max_len) .. ")</A>"
end
as_path_string = (#parts == 0) and "Local" or table.concat(parts, "<li>")
end
@ -912,7 +919,7 @@ function format_utils.formatASN(asn, short_version, shorten_stringing)
name = as_info.asname
else
-- if no asn info is present, curl to get ASN name
name = ntop.getASNameFromASN(asn)
name = getASDisplayName(asn)
end
if (shorten_stringing) then
name = shortenString(name)
@ -960,7 +967,7 @@ function format_utils.formatASN_transit(v, peer_as, ip, is_client_as)
-- Process peer/transit ASN if valid, non-zero, and different from main ASN
if peer_as and peer_as ~= 0 and v ~= peer_as then
local peer_asn = string.format('<A HREF="%s/lua/hosts_stats.lua?asn=%s">%s', ntop.getHttpPrefix(), peer_as, peer_as)
local peer_as_name = shortenString(ntop.getASNameFromASN(peer_as), max_len)
local peer_as_name = shortenString(getASDisplayName(peer_as) or "", max_len)
local via = ""
if peer_as_name and peer_as_name ~= "" then

3
scripts/lua/rest/v2/get/asn/get_as_data.lua
View file

@ -103,7 +103,8 @@ for key, value in pairs(ases_info or {}) do
end
record["asname"] = value["asname"]
else
record["asname"] = ntop.getASNameFromASN(tonumber(record["asn"]))
local as_info = ntop.getASNameFromASN(tonumber(record["asn"]))
record["asname"] = as_info and (as_info.description or as_info.handle) or nil
end
if asn == 0 then

3
scripts/lua/rest/v2/get/host/host_filters.lua
View file

@ -235,9 +235,10 @@ if (not isEmptyString(country)) then
end
if (not isEmptyString(asn)) then
local as_info = ntop.getASNameFromASN(tonumber(asn)) or {}
local as_filter = {
{key = "asn", value = "", label = i18n("all")},
{key = "asn", value = asn, label = ntop.getASNameFromASN(tonumber(asn))}
{key = "asn", value = asn, label = as_info.description or as_info.handle or nil}
}
rsp[#rsp + 1] = {
action = "asn",

88
src/Geolocation.cpp
View file

@ -402,12 +402,11 @@ void Geolocation::testme() {
for (u_int32_t i = 0; i < sizeof(ips) / sizeof(char*); i++) {
char* ip = ips[i];
ntop->getTrace()->traceEvent(TRACE_NORMAL, "Geolocating [%s]", ip);
ntop->getTrace()->traceEvent(TRACE_INFO, "Geolocating [%s]", ip);
test.set(ip);
if (test.get_sockaddr(&sa, &sa_len)) {
ntop->getTrace()->traceEvent(TRACE_NORMAL,
"Autonomous System Information", ip);
ntop->getTrace()->traceEvent(TRACE_NORMAL, "Autonomous System Information", ip);
/* TEST Autonomous Systems database */
result = MMDB_lookup_sockaddr(&geo_ip_asn_mmdb, sa, &mmdb_error);
@ -431,8 +430,7 @@ void Geolocation::testme() {
"autonomous_system_number", NULL)) ==
MMDB_SUCCESS) {
if (entry_data.has_data && entry_data.type == MMDB_DATA_TYPE_UINT32)
ntop->getTrace()->traceEvent(TRACE_NORMAL, "ASN: %d",
entry_data.uint32);
ntop->getTrace()->traceEvent(TRACE_NORMAL, "ASN: %d", entry_data.uint32);
} else
ntop->getTrace()->traceEvent(
@ -448,8 +446,7 @@ void Geolocation::testme() {
if ((org = (char*)calloc(1, entry_data.data_size + 1))) {
memcpy(org, entry_data.utf8_string, entry_data.data_size);
ntop->getTrace()->traceEvent(TRACE_NORMAL, "Organization: %s",
org);
ntop->getTrace()->traceEvent(TRACE_NORMAL, "Organization: %s", org);
free(org);
}
}
@ -461,7 +458,6 @@ void Geolocation::testme() {
MMDB_strerror(status));
}
/* TEST City database */
ntop->getTrace()->traceEvent(TRACE_NORMAL, "City Information", ip);
result = MMDB_lookup_sockaddr(&geo_ip_city_mmdb, sa, &mmdb_error);
@ -491,19 +487,71 @@ void Geolocation::testme() {
/* *************************************** */
bool Geolocation::getASName(u_int32_t asn, char* asname, u_int asname_len) {
/* Callback for getASName(): reads handle (col 0) and description (col 1) from asn_info */
static int asn_info_cb(std::vector<std::string>* row,
std::vector<std::string>* /* columns */,
void* user_data) {
asn_info_query_ctx_t* out = (asn_info_query_ctx_t*)user_data;
if (!out->found && row->size() >= 2) {
snprintf(out->handle_buf, out->handle_len, "%s", (*row)[0].c_str());
snprintf(out->desc_buf, out->desc_len, "%s", (*row)[1].c_str());
out->found = true;
}
return 0;
}
bool Geolocation::getASName(u_int32_t asn, char* handle, u_int handle_len,
char* description, u_int description_len) {
char buf[256], val[16], json[2048];
long rc;
snprintf(val, sizeof(val), "%u", asn);
if (ntop->getRedis()->hashGet(AS_NANE_CACHE, val, asname, asname_len) == 0) {
return (true); /* Found on cache */
//ntop->getTrace()->traceEvent(TRACE_NORMAL, "ASN Lookup (%u)", asn);
/* Redis cache (AS data stored as "handle|description", old entries have just "description") */
char cached[512];
if (ntop->getRedis()->hashGet(AS_NAME_CACHE, val, cached, sizeof(cached)) == 0) {
if (cached[0] == '\0') return (false); /* previously not found */
char* pipe = strchr(cached, '|');
if (pipe) {
snprintf(handle, handle_len, "%.*s", (int)(pipe - cached), cached);
snprintf(description, description_len, "%s", pipe + 1);
} else {
snprintf(handle, handle_len, "%s", cached);
snprintf(description, description_len, "%s", cached);
}
//ntop->getTrace()->traceEvent(TRACE_NORMAL, "AS INFO From Redis [%s]", cached);
return (true);
}
snprintf(buf, sizeof(buf),
"https://stat.ripe.net/data/as-overview/data.json?resource=AS%u",
asn);
/* Check asn_info table on DB before checking RIPE */
NetworkInterface* iface = ntop->getFirstInterface();
if (iface) {
DB* db = iface->getFlowsDB(); /* asn_info lives only in ClickHouse */
if (db) {
asn_info_query_ctx_t ctx = {handle, handle_len, description, description_len, false};
snprintf(buf, sizeof(buf),
"SELECT handle, description FROM asn_info WHERE asn = %u LIMIT 1", asn);
db->execSQLQuery(buf, true, true, asn_info_cb, &ctx);
if (ctx.found) {
/* Note: if either field is empty, use the other */
if (handle[0] == '\0') snprintf(handle, handle_len, "%s", description);
if (description[0] == '\0') snprintf(description, description_len, "%s", handle);
snprintf(cached, sizeof(cached), "%s|%s", handle, description);
ntop->getRedis()->hashSet(AS_NAME_CACHE, val, cached);
//ntop->getTrace()->traceEvent(TRACE_NORMAL, "AS INFO From DB [%s]", cached);
return (true);
}
}
}
/* Last resort: get AS info from RIPE stat */
snprintf(buf, sizeof(buf), "https://stat.ripe.net/data/as-overview/data.json?resource=AS%u", asn);
json[0] = '\0';
rc = Utils::httpGet(buf, NULL, NULL, NULL, 5 /* connect timeout */,
10 /* download timeout */, json, sizeof(json));
@ -533,10 +581,16 @@ bool Geolocation::getASName(u_int32_t asn, char* asname, u_int asname_len) {
res = &res[1]; /* Skip space */
#endif
snprintf(asname, asname_len, "%s", res);
/* RIPE provides a single holder string — use it for both fields */
snprintf(description, description_len, "%s", res);
snprintf(handle, handle_len, "%s", res);
ntop->getRedis()->hashSet(AS_NANE_CACHE, val, asname);
snprintf(cached, sizeof(cached), "%s|%s", handle, description);
ntop->getRedis()->hashSet(AS_NAME_CACHE, val, cached);
json_object_put(o);
//ntop->getTrace()->traceEvent(TRACE_NORMAL, "AS INFO From RIPE [%s]", cached);
return (true);
}
@ -545,7 +599,7 @@ bool Geolocation::getASName(u_int32_t asn, char* asname, u_int asname_len) {
}
/* Not found */
ntop->getRedis()->hashSet(AS_NANE_CACHE, val, "");
ntop->getRedis()->hashSet(AS_NAME_CACHE, val, "");
return (false);
}

13
src/LuaEngineNtop.cpp
View file

@ -8638,20 +8638,23 @@ static int ntop_get_asn_name(lua_State* vm) {
/* ****************************************** */
/* @brief Returns the organization name for a given autonomous system number. Lua: ntop.getASNameFromASN(asn_number) → string */
/* @brief Returns handle and description for a given ASN. Lua: ntop.getASNameFromASN(asn_number) → table{handle, description} or nil */
static int ntop_get_as_name_from_asn(lua_State* vm) {
u_int32_t asn;
char as_name[128];
char handle[128], description[256];
if (ntop_lua_check(vm, __FUNCTION__, 1, LUA_TNUMBER) != CONST_LUA_OK)
return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_NO_RETURN_VALUE));
else
asn = (u_int32_t)lua_tonumber(vm, 1);
if (ntop->getGeolocation()->getASName(asn, as_name, sizeof(as_name)))
lua_pushstring(vm, as_name);
else
if (ntop->getGeolocation()->getASName(asn, handle, sizeof(handle), description, sizeof(description))) {
lua_newtable(vm);
lua_push_str_table_entry(vm, "handle", handle);
lua_push_str_table_entry(vm, "description", description);
} else {
lua_pushnil(vm);
}
return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_ONE_RETURN_VALUE));
}

2
third-party/clickhouse-cpp vendored

@ -1 +1 @@
Subproject commit 620ec41bff2fcaf01c4e72eb64fe47302a41b781
Subproject commit 506248dd049318226805b99ddaf873008762bef4