Implements global host pools

Implements #4086

Refactors host pools to global

Implements host pools migration

Migration fixes

nEdge changes for host pools migration

Create README.host_pools_migration.md
This commit is contained in:
Simone Mainardi 2020-07-01 11:56:35 +02:00
parent a03c2158d8
commit 5ade224d85
34 changed files with 1056 additions and 229 deletions

View file

@ -25,8 +25,8 @@ function host_pools_utils.usernameToPoolId(username)
end
function host_pools_utils.poolIdToUsername(pool_id)
local ifid = getInterfaceId(ifname)
return host_pools_utils.getPoolName(ifid, pool_id)
local ifid = getInterfaceId(ifname) -- in nEdge this always takes one interface
return host_pools_utils.getPoolName(pool_id)
end
function host_pools_utils.getUserUrl(pool_id)
@ -43,16 +43,16 @@ host_pools_utils.LIMITED_NUMBER_TOTAL_HOST_POOLS = ntop_info["constants.max_num_
-- this does not take into account the special pools
host_pools_utils.LIMITED_NUMBER_USER_HOST_POOLS = host_pools_utils.LIMITED_NUMBER_TOTAL_HOST_POOLS - 1
local function get_pool_members_key(ifid, pool_id)
return "ntopng.prefs." .. ifid .. ".host_pools.members." .. pool_id
local function get_pool_members_key(pool_id)
return "ntopng.prefs.host_pools.members." .. pool_id
end
local function get_pool_ids_key(ifid)
return "ntopng.prefs." .. ifid .. ".host_pools.pool_ids"
local function get_pool_ids_key()
return "ntopng.prefs.host_pools.pool_ids"
end
local function get_pool_details_key(ifid, pool_id)
return "ntopng.prefs." .. ifid .. ".host_pools.details." .. pool_id
local function get_pool_details_key(pool_id)
return "ntopng.prefs.host_pools.details." .. pool_id
end
local function get_pools_serialized_key(ifid)
@ -60,18 +60,18 @@ local function get_pools_serialized_key(ifid)
end
-- It is safe to call this multiple times
local function initInterfacePools(ifid)
host_pools_utils.createPool(ifid, host_pools_utils.DEFAULT_POOL_ID, host_pools_utils.DEFAULT_POOL_NAME)
local function initInterfacePools()
host_pools_utils.createPool(host_pools_utils.DEFAULT_POOL_ID, host_pools_utils.DEFAULT_POOL_NAME)
end
function host_pools_utils.getPoolDetail(ifid, pool_id, detail)
local details_key = get_pool_details_key(ifid, pool_id)
function host_pools_utils.getPoolDetail(pool_id, detail)
local details_key = get_pool_details_key(pool_id)
return ntop.getHashCache(details_key, detail)
end
function host_pools_utils.setPoolDetail(ifid, pool_id, detail, value)
local details_key = get_pool_details_key(ifid, pool_id)
function host_pools_utils.setPoolDetail(pool_id, detail, value)
local details_key = get_pool_details_key(pool_id)
return ntop.setHashCache(details_key, detail, tostring(value))
end
@ -89,7 +89,7 @@ local function traceHostPoolEvent(severity, event)
traceError(severity, TRACE_CONSOLE, string.format("%s%s", f_name or '', event))
end
local function addMemberToRedisPool(ifid, pool_id, member_key)
local function addMemberToRedisPool(pool_id, member_key)
if pool_id == host_pools_utils.DEFAULT_POOL_ID then
-- avoid adding default pool members explicitly
traceHostPoolEvent(TRACE_NORMAL,
@ -98,7 +98,7 @@ local function addMemberToRedisPool(ifid, pool_id, member_key)
return true
end
local members_key = get_pool_members_key(ifid, pool_id)
local members_key = get_pool_members_key(pool_id)
local n = table.len(ntop.getMembersCache(members_key) or {})
if n >= host_pools_utils.LIMITED_NUMBER_POOL_MEMBERS then
@ -113,17 +113,17 @@ end
--------------------------------------------------------------------------------
function host_pools_utils.getPoolMembersRaw(ifid, pool_id)
local members_key = get_pool_members_key(ifid, pool_id)
function host_pools_utils.getPoolMembersRaw(pool_id)
local members_key = get_pool_members_key(pool_id)
return ntop.getMembersCache(members_key) or {}
end
-- Export host pools
function host_pools_utils.export(ifid)
function host_pools_utils.export()
local pools = {}
for _,pool in pairs(host_pools_utils.getPoolsList(ifid)) do
pool.members = host_pools_utils.getPoolMembersRaw(ifid, pool.id)
for _,pool in pairs(host_pools_utils.getPoolsList()) do
pool.members = host_pools_utils.getPoolMembersRaw(pool.id)
pools[pool.id] = pool
end
@ -131,8 +131,8 @@ function host_pools_utils.export(ifid)
end
-- Import host pools, in case of conflict (same name) the pool is replaced
function host_pools_utils.import(pools, ifid)
local existing_pools = host_pools_utils.getPoolsList(ifid)
function host_pools_utils.import(pools)
local existing_pools = host_pools_utils.getPoolsList()
local retval = true
-- Import pools
@ -142,19 +142,19 @@ function host_pools_utils.import(pools, ifid)
if pool.name == existing_pool.name then
-- Same name, delete the old pool and reuse the id
pool.id = existing_pool.id
host_pools_utils.emptyPool(ifid, existing_pool.id)
host_pools_utils.deletePool(ifid, existing_pool.id)
host_pools_utils.emptyPool(existing_pool.id)
host_pools_utils.deletePool(existing_pool.id)
end
end
-- Add pool
host_pools_utils.createPool(ifid, pool.id, pool.name, pool.children_safe,
host_pools_utils.createPool(pool.id, pool.name, pool.children_safe,
pool.enforce_quotas_per_pool_member, pool. enforce_shapers_per_pool_member,
true)
-- Add members
for _,member in ipairs(pool.members) do
local success = addMemberToRedisPool(ifid, pool.id, member)
local success = addMemberToRedisPool(pool.id, member)
if not success then
retval = false
end
@ -166,10 +166,10 @@ end
--------------------------------------------------------------------------------
function host_pools_utils.createPool(ifid, pool_id, pool_name, children_safe,
function host_pools_utils.createPool(pool_id, pool_name, children_safe,
enforce_quotas_per_pool_member, enforce_shapers_per_pool_member, ignore_exist)
local details_key = get_pool_details_key(ifid, pool_id)
local ids_key = get_pool_ids_key(ifid)
local details_key = get_pool_details_key(pool_id)
local ids_key = get_pool_ids_key()
local members = ntop.getMembersCache(ids_key) or {}
local n = table.len(members)
@ -195,20 +195,23 @@ function host_pools_utils.createPool(ifid, pool_id, pool_name, children_safe,
return true
end
function host_pools_utils.deletePool(ifid, pool_id)
function host_pools_utils.deletePool(pool_id)
local ts_utils = require "ts_utils"
local rrd_base = host_pools_utils.getRRDBase(ifid, pool_id)
local ids_key = get_pool_ids_key(ifid)
local details_key = get_pool_details_key(ifid, pool_id)
local members_key = get_pool_members_key(ifid, pool_id)
local serialized_key = get_pools_serialized_key(ifid)
local ids_key = get_pool_ids_key()
local details_key = get_pool_details_key(pool_id)
local members_key = get_pool_members_key(pool_id)
host_pools_utils.emptyPool(ifid, pool_id)
host_pools_utils.emptyPool(pool_id)
ntop.delMembersCache(ids_key, pool_id)
ntop.delCache(details_key)
ntop.delCache(members_key)
ntop.delHashCache(serialized_key, pool_id)
ts_utils.delete("host_pool", {ifid = ifid, pool = pool_id})
-- Delete serialized values and timeseries across all interfaces
for ifid, ifname in pairs(interface.getIfNames()) do
local serialized_key = get_pools_serialized_key(ifid)
ntop.delHashCache(serialized_key, pool_id)
ts_utils.delete("host_pool", {ifid = tonumber(ifid), pool = pool_id})
end
end
function getMembershipInfo(member_and_vlan)
@ -255,7 +258,7 @@ end
-- MAC address changed instead of the IP address when their MAC address is already bound to
-- a pool.
--
function host_pools_utils.changeMemberPool(ifid, member_and_vlan, new_pool, info --[[optional]], strict_host_mode --[[optional]])
function host_pools_utils.changeMemberPool(member_and_vlan, new_pool, info --[[optional]], strict_host_mode --[[optional]])
traceHostPoolEvent(TRACE_NORMAL,
string.format("Pool change requested. [member: %s][new_pool: %s][strict_host_mode: %s]",
member_and_vlan, new_pool, tostring(strict_host_mode)))
@ -307,12 +310,12 @@ function host_pools_utils.changeMemberPool(ifid, member_and_vlan, new_pool, info
string.format("Pool change prepared. [member: %s][info.key: %s][prev_pool: %s][new_pool: %s]",
member_and_vlan, tostring(info.key), prev_pool, new_pool))
host_pools_utils.deletePoolMember(ifid, prev_pool, info.key)
addMemberToRedisPool(ifid, new_pool, info.key)
host_pools_utils.deletePoolMember(prev_pool, info.key)
addMemberToRedisPool(new_pool, info.key)
return true
end
function host_pools_utils.addPoolMember(ifid, pool_id, member_and_vlan)
function host_pools_utils.addPoolMember(pool_id, member_and_vlan)
traceHostPoolEvent(TRACE_NORMAL,
string.format("Pool member addition requested. [member: %s][pool_id: %s]",
member_and_vlan, pool_id))
@ -323,17 +326,17 @@ function host_pools_utils.addPoolMember(ifid, pool_id, member_and_vlan)
traceHostPoolEvent(TRACE_NORMAL, string.format("Member already in pool. [pool_id: %d] [member: %s]", pool_id, member_and_vlan))
return false, info
else
local rv = addMemberToRedisPool(ifid, pool_id, info.key)
local rv = addMemberToRedisPool(pool_id, info.key)
return rv, info
end
end
function host_pools_utils.deletePoolMember(ifid, pool_id, member_and_vlan)
function host_pools_utils.deletePoolMember(pool_id, member_and_vlan)
traceHostPoolEvent(TRACE_NORMAL,
string.format("Pool member deletion requested. [member: %s][pool_id: %s]",
member_and_vlan, pool_id))
local members_key = get_pool_members_key(ifid, pool_id)
local members_key = get_pool_members_key(pool_id)
-- Possible delete volatile member
if ntop.isPro() then
@ -344,8 +347,8 @@ function host_pools_utils.deletePoolMember(ifid, pool_id, member_and_vlan)
ntop.delMembersCache(members_key, member_and_vlan)
end
function host_pools_utils.getPoolsList(ifid, without_info)
local ids_key = get_pool_ids_key(ifid)
function host_pools_utils.getPoolsList(without_info)
local ids_key = get_pool_ids_key()
local ids = ntop.getMembersCache(ids_key)
if not ids then ids = {} end
@ -355,7 +358,7 @@ function host_pools_utils.getPoolsList(ifid, without_info)
local pools = {}
initInterfacePools(ifid)
initInterfacePools()
for _, pool_id in pairsByValues(ids, asc) do
pool_id = tostring(pool_id)
@ -366,10 +369,10 @@ function host_pools_utils.getPoolsList(ifid, without_info)
else
pool = {
id = pool_id,
name = host_pools_utils.getPoolName(ifid, pool_id),
children_safe = host_pools_utils.getChildrenSafe(ifid, pool_id),
enforce_quotas_per_pool_member = host_pools_utils.getEnforceQuotasPerPoolMember(ifid, pool_id),
enforce_shapers_per_pool_member = host_pools_utils.getEnforceShapersPerPoolMember(ifid, pool_id),
name = host_pools_utils.getPoolName(pool_id),
children_safe = host_pools_utils.getChildrenSafe(pool_id),
enforce_quotas_per_pool_member = host_pools_utils.getEnforceQuotasPerPoolMember(pool_id),
enforce_shapers_per_pool_member = host_pools_utils.getEnforceShapersPerPoolMember(pool_id),
}
end
@ -379,8 +382,8 @@ function host_pools_utils.getPoolsList(ifid, without_info)
return pools
end
function host_pools_utils.getPoolMembers(ifid, pool_id)
local members_key = get_pool_members_key(ifid, pool_id)
function host_pools_utils.getPoolMembers(pool_id)
local members_key = get_pool_members_key(pool_id)
local members = {}
local all_members = ntop.getMembersCache(members_key) or {}
@ -435,46 +438,38 @@ function host_pools_utils.getMemberKey(member)
return host_key, is_network
end
function host_pools_utils.getPoolName(ifid, pool_id)
return host_pools_utils.getPoolDetail(ifid, pool_id, "name")
function host_pools_utils.getPoolName(pool_id)
return host_pools_utils.getPoolDetail(pool_id, "name")
end
function host_pools_utils.getChildrenSafe(ifid, pool_id)
return toboolean(host_pools_utils.getPoolDetail(ifid, pool_id, "children_safe"))
function host_pools_utils.getChildrenSafe(pool_id)
return toboolean(host_pools_utils.getPoolDetail(pool_id, "children_safe"))
end
function host_pools_utils.setChildrenSafe(ifid, pool_id, value)
host_pools_utils.setPoolDetail(ifid, pool_id, "children_safe", ternary(value, "true", "false"))
function host_pools_utils.setChildrenSafe(pool_id, value)
host_pools_utils.setPoolDetail(pool_id, "children_safe", ternary(value, "true", "false"))
end
function host_pools_utils.getForgeGlobalDNS(ifid, pool_id)
return toboolean(host_pools_utils.getPoolDetail(ifid, pool_id, "forge_global_dns"))
end
function host_pools_utils.setForgeGlobalDNS(ifid, pool_id, value)
host_pools_utils.setPoolDetail(ifid, pool_id, "forge_global_dns", ternary(value, "true", "false"))
end
function host_pools_utils.getRoutingPolicyId(ifid, pool_id)
local routing_policy_id = host_pools_utils.getPoolDetail(ifid, pool_id, "routing_policy_id")
function host_pools_utils.getRoutingPolicyId(pool_id)
local routing_policy_id = host_pools_utils.getPoolDetail(pool_id, "routing_policy_id")
if isEmptyString(routing_policy_id) then routing_policy_id = host_pools_utils.DEFAULT_ROUTING_POLICY_ID end
return routing_policy_id
end
function host_pools_utils.setRoutingPolicyId(ifid, pool_id, routing_policy_id)
return host_pools_utils.setPoolDetail(ifid, pool_id, "routing_policy_id", routing_policy_id)
function host_pools_utils.setRoutingPolicyId(pool_id, routing_policy_id)
return host_pools_utils.setPoolDetail(pool_id, "routing_policy_id", routing_policy_id)
end
function host_pools_utils.getEnforceQuotasPerPoolMember(ifid, pool_id)
return toboolean(host_pools_utils.getPoolDetail(ifid, pool_id, "enforce_quotas_per_pool_member"))
function host_pools_utils.getEnforceQuotasPerPoolMember(pool_id)
return toboolean(host_pools_utils.getPoolDetail(pool_id, "enforce_quotas_per_pool_member"))
end
function host_pools_utils.getEnforceShapersPerPoolMember(ifid, pool_id)
return toboolean(host_pools_utils.getPoolDetail(ifid, pool_id, "enforce_shapers_per_pool_member"))
function host_pools_utils.getEnforceShapersPerPoolMember(pool_id)
return toboolean(host_pools_utils.getPoolDetail(pool_id, "enforce_shapers_per_pool_member"))
end
function host_pools_utils.emptyPool(ifid, pool_id)
local members_key = get_pool_members_key(ifid, pool_id)
function host_pools_utils.emptyPool(pool_id)
local members_key = get_pool_members_key(pool_id)
if ntop.isPro() then
-- Remove volatile members
@ -492,9 +487,9 @@ function host_pools_utils.emptyPools()
local ifid = getInterfaceId(ifname)
local ifstats = interface.getStats()
local pools_list = host_pools_utils.getPoolsList(ifid)
local pools_list = host_pools_utils.getPoolsList()
for _, pool in pairs(pools_list) do
host_pools_utils.emptyPool(ifid, pool["id"])
host_pools_utils.emptyPool(pool["id"])
end
end
end
@ -505,7 +500,7 @@ function host_pools_utils.initPools()
local ifstats = interface.getStats()
-- Note: possible shapers are initialized in shaper_utils::initShapers
initInterfacePools(ifid)
initInterfacePools()
end
end
@ -518,7 +513,7 @@ function host_pools_utils.getMacPool(mac_address)
end
end
function host_pools_utils.getUndeletablePools(ifid)
function host_pools_utils.getUndeletablePools()
local pools = {}
for user_key,_ in pairs(ntop.getKeysCache("ntopng.user.*.host_pool_id") or {}) do
@ -538,23 +533,6 @@ function host_pools_utils.getUndeletablePools(ifid)
return pools
end
function host_pools_utils.purgeExpiredPoolsMembers()
local ifnames = interface.getIfNames()
for _, ifname in pairs(ifnames) do
interface.select(ifname)
-- Currently, volatile pool members are no longer used,
-- so there's no need to purge them.
-- interface.purgeExpiredPoolsMembers()
end
end
function host_pools_utils.getRRDBase(ifid, pool_id)
local dirs = ntop.getDirs()
return os_utils.fixPath(dirs.workingdir .. "/" .. ifid .. "/host_pools/" .. pool_id)
end
function host_pools_utils.updateRRDs(ifid, dump_ndpi, verbose)
local ts_utils = require "ts_utils"
require "ts_5min"
@ -592,7 +570,7 @@ function host_pools_utils.hostpool2record(ifid, pool_id, pool)
local record = {}
record["key"] = tostring(pool_id)
local pool_name = host_pools_utils.getPoolName(ifid, pool_id)
local pool_name = host_pools_utils.getPoolName(pool_id)
local pool_link = "<A HREF='"..ntop.getHttpPrefix()..'/lua/hosts_stats.lua?pool='..pool_id.."' title='"..pool_name.."'>"..pool_name..'</A>'
record["column_id"] = pool_link
@ -627,6 +605,7 @@ function host_pools_utils.printQuotas(pool_id, host, page_params)
local ndpi_stats = pool_stats.ndpi
local category_stats = pool_stats.ndpi_categories
-- ifId is a global variable here
local quota_and_protos = shaper_utils.getPoolProtoShapers(ifId, pool_id)
local cross_traffic_quota, cross_time_quota = shaper_utils.getCrossApplicationQuotas(ifId, pool_id)
@ -686,8 +665,8 @@ function host_pools_utils.printQuotas(pool_id, host, page_params)
end
function host_pools_utils.getFirstAvailablePoolId(ifid)
local ids_key = get_pool_ids_key(ifid)
function host_pools_utils.getFirstAvailablePoolId()
local ids_key = get_pool_ids_key()
local ids = ntop.getMembersCache(ids_key) or {}
for i, id in pairs(ids) do
@ -707,8 +686,85 @@ function host_pools_utils.getFirstAvailablePoolId(ifid)
return tostring(host_pool_id)
end
function host_pools_utils.resetPoolsQuotas(ifid, pool_filter)
local serialized_key = get_pools_serialized_key(ifid)
-- @brief Perform migration from the old host pools which were configured per-interface
-- to global host pools which are now system-wide
function host_pools_utils.migrateHostPools()
-- Migration is only performed when host pools are configured for 1 and only 1 interface (this always cover the case of nEdge)
-- https://github.com/ntop/ntopng/issues/4086
if ntop.getKeysCache(get_pool_ids_key()) then
-- Already migrated
return
end
local host_pools_migration = require "host_pools_migration"
local delete_data_utils = require "delete_data_utils"
local json = require "dkjson"
local migration_ifid
-- The migration is only done when there's 1 active interface
if table.len(interface.getIfNames()) == 1 then
-- Avoid calling interface.getId() as this migration function
-- can be executed too early
for ifid, ifname in pairs(interface.getIfNames()) do
migration_ifid = ifid
break
end
end
-- Do the actual migration
if migration_ifid then
-- Call the `new` import with the `old` export which takes ifid as argument
host_pools_utils.import(host_pools_migration.export(migration_ifid))
traceError(TRACE_WARNING, TRACE_CONSOLE, "Host pools configuration migrated.")
end
for ifid, ifname in pairs(delete_data_utils.list_all_interfaces()) do
if ifid == migration_ifid then
-- Don't delete migrated elements
goto continue
end
local if_pools = host_pools_migration.getPoolsList(ifid)
if table.len(if_pools) <= 1 then
-- Nothing to migrate, only the default pool is present
goto continue
end
-- Copy the pool configuration
local base_dir = os_utils.fixPath(dirs.workingdir .. "/" .. ifid .. "/migration/host_pools/")
ntop.mkdir(base_dir)
local config_filename = os_utils.fixPath(base_dir.."/".."pools_configuration.json")
local config_file = assert(io.open(config_filename, "w"))
config_file:write(json.encode(host_pools_migration.export(ifid)))
config_file:close()
-- Delete data
for _, pool in ipairs(if_pools) do
host_pools_migration.deletePool(ifid, pool.id)
end
-- Print out a message
traceError(TRACE_WARNING, TRACE_CONSOLE, string.format("[%s] host pools configuration backed up to file %s due to major changes. It is possible to take the file and re-import it manually.", ifname, config_filename))
::continue::
end
-- Cleanup old host-pool related keys
local old_keys_pattern = string.format("ntopng.prefs.*.host_pools.*")
local old_keys = ntop.getKeysCache(old_keys_pattern)
for old_key in pairs(old_keys) do
ntop.delCache(old_key)
end
ntop.reloadHostPools()
end
function host_pools_utils.resetPoolsQuotas(pool_filter)
local serialized_key = get_pools_serialized_key(tostring(interface.getId()))
local keys_to_del
if pool_filter ~= nil then