vrr/ntopng
2
0
Fork 0
mirror of https://github.com/ntop/ntopng.git synced 2026-05-01 00:19:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Removed not necessary clickhouse code

Browse source
This commit is contained in:
Luca Deri 2023-11-20 21:08:38 +01:00
parent a0fb58c7b0
commit dc19d2aada
1 changed files with 0 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

52
scripts/lua/modules/db_utils.lua
View file

@ -889,58 +889,6 @@ end
-- ########################################################
-- Delete partitions older than data_retention (epoch)
function db_utils.clickhouseDeleteOldPartitions(data_retention, aggregated_retention)
traceError(TRACE_NORMAL, TRACE_CONSOLE, "Performing Clickhouse cleanup")
-- [1] Non Aggregated Retention
local day_aligned_retention = data_retention - (data_retention % 86400)
-- Create a string that identifies the PARTITIONs name of the most recent partition that will be deleted
local retention_yyyymmdd = os.date("%Y%m%d", day_aligned_retention)
-- Deletion is done directly on partitions (Clickhouse database has daily partitions)
-- Query the partitions that need to be deleted. Convert YYYYMMDD strings into integers so that
-- only relevant partitions can be queried and deleted
-- The last condition > 999999 prevents old partitions created as YYYMM to be deleted
local partitions_q = string.format("SELECT DISTINCT database, table, toUInt32(partition) drop_part FROM system.parts WHERE active AND database='%s' AND table != 'hourly_flows' AND table = 'l4_protocols' AND table = 'l7_protocols' AND table = 'l7_categories' AND table = 'alert_severities' AND drop_part <= %u AND drop_part > 999999", ntop.getPrefs().mysql_dbname or 'ntopng', retention_yyyymmdd)
if ntop.getCache("ntopng.debug.retention_query") == "1" then
traceError(TRACE_NORMAL, TRACE_CONSOLE, "Clickhouse retention query: " .. partitions_q)
end
local partitions_res = interface.execSQLQuery(partitions_q)
if(partitions_res ~= nil) then
-- Iterate queried partitions and delete them (nil is returned if there is nothing to delete)
for _, partition_info in ipairs(partitions_res) do
local delete_partition_q = string.format("ALTER TABLE %s.%s DROP PARTITION '%s'",
partition_info["database"], partition_info["table"], partition_info["drop_part"])
local delete_partition_res = interface.execSQLQuery(delete_partition_q)
end
end
-- [2] Aggregated flows
day_aligned_retention = aggregated_retention - (aggregated_retention % 86400)
retention_yyyymmdd = os.date("%Y%m%d", day_aligned_retention)
partitions_q = string.format("SELECT DISTINCT database, table, toUInt32(partition) drop_part FROM system.parts WHERE active AND database='%s' AND table = 'hourly_flows' AND table = 'l4_protocols' AND table = 'l7_protocols' AND table = 'l7_categories' AND table = 'alert_severities' AND drop_part <= %u AND drop_part > 999999", ntop.getPrefs().mysql_dbname or 'ntopng', retention_yyyymmdd)
partitions_res = interface.execSQLQuery(partitions_q)
if(partitions_res ~= nil) then
-- Iterate queried partitions and delete them (nil is returned if there is nothing to delete)
for _, partition_info in ipairs(partitions_res) do
local delete_partition_q = string.format("ALTER TABLE %s.%s DROP PARTITION '%s'",
partition_info["database"], partition_info["table"], partition_info["drop_part"])
local delete_partition_res = interface.execSQLQuery(delete_partition_q)
end
end
traceError(TRACE_NORMAL, TRACE_CONSOLE, "Clickhouse cleanup completed")
end
-- ########################################################
local function _harvest_expired_mysql_flows(ifname, mysql_retention, verbose)
local dbtables = {"flowsv4", "flowsv6"}