Fix schema. Add DOMAIN_NAME.

This commit is contained in:
Alfredo Cardigliano 2024-10-28 11:22:36 +01:00
parent 42a6d071f5
commit fb30f7592f
2 changed files with 149 additions and 25 deletions

View file

@ -55,7 +55,36 @@ CREATE TABLE IF NOT EXISTS `flows` ON CLUSTER '$CLUSTER' (
`SRC_PROC_NAME` String,
`DST_PROC_NAME` String,
`SRC_PROC_USER_NAME` String,
`DST_PROC_USER_NAME` String
`DST_PROC_USER_NAME` String,
`ALERTS_MAP` String,
`SEVERITY` UInt8,
`IS_CLI_ATTACKER` UInt8,
`IS_CLI_VICTIM` UInt8,
`IS_CLI_BLACKLISTED` UInt8,
`IS_SRV_ATTACKER` UInt8,
`IS_SRV_VICTIM` UInt8,
`IS_SRV_BLACKLISTED` UInt8,
`ALERT_STATUS` UInt8,
`USER_LABEL` String,
`USER_LABEL_TSTAMP` DateTime,
`ALERT_JSON` String, /* Extra flow metadata (not alert only) */
`IS_ALERT_DELETED` UInt8,
`SRC2DST_PACKETS` UInt32,
`DST2SRC_PACKETS` UInt32,
`ALERT_CATEGORY` UInt8,
`MINOR_CONNECTION_STATE` UInt8,
`MAJOR_CONNECTION_STATE` UInt8,
`PRE_NAT_IPV4_SRC_ADDR` UInt32,
`PRE_NAT_SRC_PORT` UInt32,
`PRE_NAT_IPV4_DST_ADDR` UInt32,
`PRE_NAT_DST_PORT` UInt32,
`POST_NAT_IPV4_SRC_ADDR` UInt32,
`POST_NAT_SRC_PORT` UInt32,
`POST_NAT_IPV4_DST_ADDR` UInt32,
`POST_NAT_DST_PORT` UInt32,
`WLAN_SSID` String,
`WTP_MAC_ADDRESS` UInt64,
`DOMAIN_NAME` String
) ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}', '{replica}') PARTITION BY toYYYYMMDD(FIRST_SEEN) ORDER BY (IPV4_SRC_ADDR, IPV4_DST_ADDR, FIRST_SEEN);
@
ALTER TABLE `flows` ON CLUSTER '$CLUSTER' ADD COLUMN IF NOT EXISTS `FLOW_ID` UInt64;
@ -143,6 +172,8 @@ ALTER TABLE `flows` ON CLUSTER '$CLUSTER' ADD COLUMN IF NOT EXISTS `POST_NAT_DST
ALTER TABLE `flows` ON CLUSTER '$CLUSTER' ADD COLUMN IF NOT EXISTS `WLAN_SSID` String;
@
ALTER TABLE `flows` ON CLUSTER '$CLUSTER' ADD COLUMN IF NOT EXISTS `WTP_MAC_ADDRESS` UInt64;
@
ALTER TABLE `flows` ON CLUSTER '$CLUSTER' ADD COLUMN IF NOT EXISTS `DOMAIN_NAME` String;
@
@ -164,7 +195,8 @@ CREATE TABLE IF NOT EXISTS `active_monitoring_alerts` ON CLUSTER '$CLUSTER' (
`description` String,
`json` String,
`user_label` String,
`user_label_tstamp` DateTime NULL
`user_label_tstamp` DateTime NULL,
`alert_category` UInt8
) ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}', '{replica}') PARTITION BY toYYYYMMDD(tstamp) ORDER BY (tstamp);
@
ALTER TABLE `active_monitoring_alerts` ON CLUSTER '$CLUSTER' ADD COLUMN IF NOT EXISTS alert_category UInt8;
@ -211,7 +243,18 @@ CREATE TABLE IF NOT EXISTS `flow_alerts` ON CLUSTER '$CLUSTER' (
`alerts_map` String, -- An HEX bitmap of all flow statuses
`flow_risk_bitmap` UInt64 NOT NULL,
`user_label` String,
`user_label_tstamp` DateTime
`user_label_tstamp` DateTime,
`cli_host_pool_id` UInt16,
`srv_host_pool_id` UInt16,
`cli_network` UInt16,
`srv_network` UInt16,
`info` String,
`cli_location` UInt8,
`srv_location` UInt8,
`probe_ip` String,
`input_snmp` UInt32,
`output_snmp` UInt32,
`alert_category` UInt8
) ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}', '{replica}') PARTITION BY toYYYYMMDD(first_seen) ORDER BY (first_seen);
@
ALTER TABLE `flow_alerts` ON CLUSTER '$CLUSTER' ADD COLUMN IF NOT EXISTS cli_host_pool_id UInt16;
@ -260,7 +303,11 @@ CREATE TABLE IF NOT EXISTS `host_alerts` ON CLUSTER '$CLUSTER' (
`description` String,
`json` String,
`user_label` String,
`user_label_tstamp` DateTime
`user_label_tstamp` DateTime,
`host_pool_id` UInt16,
`network` UInt16,
`country` String,
`alert_category` UInt8
) ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}', '{replica}') PARTITION BY toYYYYMMDD(tstamp) ORDER BY (tstamp);
@
ALTER TABLE `host_alerts` ON CLUSTER '$CLUSTER' ADD COLUMN IF NOT EXISTS host_pool_id UInt16;
@ -292,7 +339,8 @@ CREATE TABLE IF NOT EXISTS `mac_alerts` ON CLUSTER '$CLUSTER' (
`description` String,
`json` String,
`user_label` String,
`user_label_tstamp` DateTime
`user_label_tstamp` DateTime,
`alert_category` UInt8
) ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}', '{replica}') PARTITION BY toYYYYMMDD(tstamp) ORDER BY (tstamp);
@
ALTER TABLE `mac_alerts` ON CLUSTER '$CLUSTER' ADD COLUMN IF NOT EXISTS alert_category UInt8;
@ -317,7 +365,8 @@ CREATE TABLE IF NOT EXISTS `snmp_alerts` ON CLUSTER '$CLUSTER' (
`description` String,
`json` String,
`user_label` String,
`user_label_tstamp` DateTime
`user_label_tstamp` DateTime,
`alert_category` UInt8
) ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}', '{replica}') PARTITION BY toYYYYMMDD(tstamp) ORDER BY (tstamp);
@
ALTER TABLE `snmp_alerts` MODIFY COLUMN `port` UInt32;
@ -343,7 +392,8 @@ CREATE TABLE IF NOT EXISTS `network_alerts` ON CLUSTER '$CLUSTER' (
`description` String,
`json` String,
`user_label` String,
`user_label_tstamp` DateTime
`user_label_tstamp` DateTime,
`alert_category` UInt8
) ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}', '{replica}') PARTITION BY toYYYYMMDD(tstamp) ORDER BY (tstamp);
@
ALTER TABLE `network_alerts` ON CLUSTER '$CLUSTER' ADD COLUMN IF NOT EXISTS alert_category UInt8;
@ -368,7 +418,8 @@ CREATE TABLE IF NOT EXISTS `interface_alerts` ON CLUSTER '$CLUSTER' (
`description` String,
`json` String,
`user_label` String,
`user_label_tstamp` DateTime
`user_label_tstamp` DateTime,
`alert_category` UInt8
) ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}', '{replica}') PARTITION BY toYYYYMMDD(tstamp) ORDER BY (tstamp);
@
ALTER TABLE `interface_alerts` ON CLUSTER '$CLUSTER' ADD COLUMN IF NOT EXISTS alert_category UInt8;
@ -390,7 +441,8 @@ CREATE TABLE IF NOT EXISTS `user_alerts` ON CLUSTER '$CLUSTER' (
`description` String,
`json` String,
`user_label` String,
`user_label_tstamp` DateTime
`user_label_tstamp` DateTime,
`alert_category` UInt8
) ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}', '{replica}') PARTITION BY toYYYYMMDD(tstamp) ORDER BY (tstamp);
@
ALTER TABLE `user_alerts` ON CLUSTER '$CLUSTER' ADD COLUMN IF NOT EXISTS alert_category UInt8;
@ -412,7 +464,8 @@ CREATE TABLE IF NOT EXISTS `system_alerts` ON CLUSTER '$CLUSTER' (
`description` String,
`json` String,
`user_label` String,
`user_label_tstamp` DateTime
`user_label_tstamp` DateTime,
`alert_category` UInt8
) ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}', '{replica}') PARTITION BY toYYYYMMDD(tstamp) ORDER BY (tstamp);
@
ALTER TABLE `system_alerts` ON CLUSTER '$CLUSTER' ADD COLUMN IF NOT EXISTS alert_category UInt8;
@ -477,7 +530,14 @@ CREATE TABLE IF NOT EXISTS `hourly_flows` ON CLUSTER '$CLUSTER' (
`INPUT_SNMP` UInt32,
`OUTPUT_SNMP` UInt32,
`SRC_NETWORK_ID` UInt16,
`DST_NETWORK_ID` UInt16
`DST_NETWORK_ID` UInt16,
`SRC_LABEL` String,
`DST_LABEL` String,
`INTERFACE_ID` UInt16,
`WLAN_SSID` String,
`WTP_MAC_ADDRESS` UInt64,
`CLIENT_LOCATION` UInt8,
`SERVER_LOCATION` UInt8
) ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}', '{replica}') PARTITION BY toYYYYMMDD(FIRST_SEEN) ORDER BY (IPV4_SRC_ADDR, IPV4_DST_ADDR, FIRST_SEEN);
@
ALTER TABLE `hourly_flows` ON CLUSTER '$CLUSTER' ADD COLUMN IF NOT EXISTS SRC_LABEL String;
@ -489,6 +549,10 @@ ALTER TABLE `hourly_flows` ON CLUSTER '$CLUSTER' ADD COLUMN IF NOT EXISTS INTERF
ALTER TABLE `hourly_flows` ON CLUSTER '$CLUSTER' ADD COLUMN IF NOT EXISTS WLAN_SSID String;
@
ALTER TABLE `hourly_flows` ON CLUSTER '$CLUSTER' ADD COLUMN IF NOT EXISTS WTP_MAC_ADDRESS UInt64;
@
ALTER TABLE `hourly_flows` ON CLUSTER '$CLUSTER' ADD COLUMN IF NOT EXISTS CLIENT_LOCATION UInt8;
@
ALTER TABLE `hourly_flows` ON CLUSTER '$CLUSTER' ADD COLUMN IF NOT EXISTS SERVER_LOCATION UInt8;
@