diff --git a/httpdocs/misc/alert_store_schema_clickhouse.sql b/httpdocs/misc/alert_store_schema_clickhouse.sql new file mode 100644 index 0000000000..1ab774d296 --- /dev/null +++ b/httpdocs/misc/alert_store_schema_clickhouse.sql @@ -0,0 +1,243 @@ +-- ----------------------------------------------------- +-- Table `active_monitoring_alerts` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `active_monitoring_alerts` ( +`alert_id` UInt32 NOT NULL, +`alert_status` UInt8 NOT NULL, +`resolved_ip` String, +`resolved_name` String, +`interface_id` UInt16 NULL, +`measurement` String, +`measure_threshold` UInt32 NULL, +`measure_value` REAL NULL, +`tstamp` DateTime NOT NULL, +`tstamp_end` DateTime NULL, +`severity` UInt8 NOT NULL, +`score` UInt16 NOT NULL, +`counter` UInt32 NOT NULL, +`description` String, +`json` String, +`user_label` String, +`user_label_tstamp` DateTime NULL +) ENGINE = MergeTree PARTITION BY toYYYYMM(tstamp) ORDER BY (tstamp); + +-- ----------------------------------------------------- +-- Table `flow_alerts` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `flow_alerts` ( +`alert_id` UInt32 NOT NULL, +`alert_status` UInt8 NOT NULL, +`tstamp` DateTime NOT NULL, +`tstamp_end` DateTime, +`severity` UInt8 NOT NULL, +`score` UInt16 NOT NULL, +`counter` UInt32 NOT NULL, +`json` String, +`ip_version` UInt8 NOT NULL, +`cli_ip` String NOT NULL, +`srv_ip` String NOT NULL, +`cli_port` UInt16 NOT NULL, +`srv_port` UInt16 NOT NULL, +`vlan_id` UInt16 NOT NULL, +`is_cli_attacker` UInt8 NOT NULL, +`is_cli_victim` UInt8 NOT NULL, +`is_srv_attacker` UInt8 NOT NULL, +`is_srv_victim` UInt8 NOT NULL, +`proto` UInt8 NOT NULL, +`l7_proto` UInt16 NOT NULL, +`l7_master_proto` UInt16 NOT NULL, +`l7_cat` UInt16 NOT NULL, +`cli_name` String, +`srv_name` String, +`cli_country` String, +`srv_country` String, +`cli_blacklisted` UInt8 NOT NULL, +`srv_blacklisted` UInt8 NOT NULL, +`cli2srv_bytes` UInt8 NOT NULL, +`srv2cli_bytes` UInt8 NOT NULL, +`cli2srv_pkts` UInt8 NOT NULL, +`srv2cli_pkts` UInt8 NOT NULL, +`first_seen` DateTime NOT NULL, +`community_id` String, +`alerts_map` String, -- An HEX bitmap of all flow statuses +`flow_risk_bitmap` UInt128 NOT NULL, +`user_label` String, +`user_label_tstamp` DateTime +) ENGINE = MergeTree PARTITION BY toYYYYMM(first_seen) ORDER BY (first_seen); + +-- ----------------------------------------------------- +-- Table `host_alerts` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `host_alerts` ( +`alert_id` UInt32 NOT NULL, +`alert_status` UInt8 NOT NULL, +`ip_version` UInt8 NOT NULL, +`ip` String NOT NULL, +`vlan_id` UInt16, +`name` String, +`is_attacker` UInt8, +`is_victim` UInt8, +`is_client` UInt8, +`is_server` UInt8, +`tstamp` DateTime NOT NULL, +`tstamp_end` DateTime, +`severity` UInt8 NOT NULL, +`score` UInt16 NOT NULL, +`granularity` UInt8 NOT NULL, +`counter` UInt32 NOT NULL, +`description` String, +`json` String, +`user_label` String, +`user_label_tstamp` DateTime +) ENGINE = MergeTree PARTITION BY toYYYYMM(tstamp) ORDER BY (tstamp); + +-- ----------------------------------------------------- +-- Table `mac_alerts` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `mac_alerts` ( +`alert_id` UInt32 NOT NULL, +`alert_status` UInt8 NOT NULL, +`address` String, +`device_type` UInt8 NULL, +`name` String, +`is_attacker` UInt8, +`is_victim` UInt8, +`tstamp` DateTime NOT NULL, +`tstamp_end` DateTime, +`severity` UInt8 NOT NULL, +`score` UInt16 NOT NULL, +`granularity` UInt8 NOT NULL, +`counter` UInt32 NOT NULL, +`description` String, +`json` String, +`user_label` String, +`user_label_tstamp` DateTime +) ENGINE = MergeTree PARTITION BY toYYYYMM(tstamp) ORDER BY (tstamp); + +-- ----------------------------------------------------- +-- Table `snmp_alerts` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `snmp_alerts` ( +`alert_id` UInt32 NOT NULL, +`alert_status` UInt8 NOT NULL, +`ip` String NOT NULL, +`port` UInt16, +`name` String, +`port_name` String, +`tstamp` DateTime NOT NULL, +`tstamp_end` DateTime, +`severity` UInt8 NOT NULL, +`score` UInt16 NOT NULL, +`granularity` UInt8 NOT NULL, +`counter` UInt32 NOT NULL, +`description` String, +`json` String, +`user_label` String, +`user_label_tstamp` DateTime +) ENGINE = MergeTree PARTITION BY toYYYYMM(tstamp) ORDER BY (tstamp); + +-- ----------------------------------------------------- +-- Table `network_alerts` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `network_alerts` ( +`local_network_id` UInt16 NOT NULL, +`alert_id` UInt32 NOT NULL, +`alert_status` UInt8 NOT NULL, +`name` String, +`alias` String, +`tstamp` DateTime NOT NULL, +`tstamp_end` DateTime, +`severity` UInt8 NOT NULL, +`score` UInt16 NOT NULL, +`granularity` UInt8 NOT NULL, +`counter` UInt32 NOT NULL, +`description` String, +`json` String, +`user_label` String, +`user_label_tstamp` DateTime +) ENGINE = MergeTree PARTITION BY toYYYYMM(tstamp) ORDER BY (tstamp); + +-- ----------------------------------------------------- +-- Table `interface_alerts` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `interface_alerts` ( +`ifid` UInt8 NOT NULL, +`alert_id` UInt32 NOT NULL, +`alert_status` UInt8 NOT NULL, +`subtype` String, +`name` String, +`alias` String, +`tstamp` DateTime NOT NULL, +`tstamp_end` DateTime, +`severity` UInt8 NOT NULL, +`score` UInt16 NOT NULL, +`granularity` UInt8 NOT NULL, +`counter` UInt32 NOT NULL, +`description` String, +`json` String, +`user_label` String, +`user_label_tstamp` DateTime +) ENGINE = MergeTree PARTITION BY toYYYYMM(tstamp) ORDER BY (tstamp); + +-- ----------------------------------------------------- +-- Table `user_alerts` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `user_alerts` ( +`alert_id` UInt32 NOT NULL, +`alert_status` UInt8 NOT NULL, +`user` String, +`tstamp` DateTime NOT NULL, +`tstamp_end` DateTime, +`severity` UInt8 NOT NULL, +`score` UInt16 NOT NULL, +`granularity` UInt8 NOT NULL, +`counter` UInt32 NOT NULL, +`description` String, +`json` String, +`user_label` String, +`user_label_tstamp` DateTime +) ENGINE = MergeTree PARTITION BY toYYYYMM(tstamp) ORDER BY (tstamp); + +-- ----------------------------------------------------- +-- Table `system_alerts` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `system_alerts` ( +`alert_id` UInt32 NOT NULL, +`alert_status` UInt8 NOT NULL, +`name` String, +`tstamp` DateTime NOT NULL, +`tstamp_end` DateTime, +`severity` UInt8 NOT NULL, +`score` UInt16 NOT NULL, +`granularity` UInt8 NOT NULL, +`counter` UInt32 NOT NULL, +`description` String, +`json` String, +`user_label` String, +`user_label_tstamp` DateTime +) ENGINE = MergeTree PARTITION BY toYYYYMM(tstamp) ORDER BY (tstamp); + +-- ----------------------------------------------------- +-- View that merges all tables together +-- NOTE: integer entity_id MUST BE KEPT IN SYNC WITH IDS in alert_entities.lua +-- ----------------------------------------------------- +DROP VIEW IF EXISTS `all_alerts`; +CREATE VIEW IF NOT EXISTS `all_alerts` AS +SELECT 8 entity_id, alert_id, alert_status, tstamp, tstamp_end, severity, score FROM `active_monitoring_alerts` +UNION ALL +SELECT 4 entity_id, alert_id, alert_status, tstamp, tstamp_end, severity, score FROM `flow_alerts` +UNION ALL +SELECT 1 entity_id, alert_id, alert_status, tstamp, tstamp_end, severity, score FROM `host_alerts` +UNION ALL +SELECT 5 entity_id, alert_id, alert_status, tstamp, tstamp_end, severity, score FROM `mac_alerts` +UNION ALL +SELECT 3 entity_id, alert_id, alert_status, tstamp, tstamp_end, severity, score FROM `snmp_alerts` +UNION ALL +SELECT 2 entity_id, alert_id, alert_status, tstamp, tstamp_end, severity, score FROM `network_alerts` +UNION ALL +SELECT 0 entity_id, alert_id, alert_status, tstamp, tstamp_end, severity, score FROM `interface_alerts` +UNION ALL +SELECT 7 entity_id, alert_id, alert_status, tstamp, tstamp_end, severity, score FROM `user_alerts` +UNION ALL +SELECT 9 entity_id, alert_id, alert_status, tstamp, tstamp_end, severity, score FROM `system_alerts` +; diff --git a/include/AlertStore.h b/include/AlertStore.h index 6e12f5bd99..ab102330a1 100644 --- a/include/AlertStore.h +++ b/include/AlertStore.h @@ -26,7 +26,7 @@ class Flow; -class AlertStore : public StoreManager { +class AlertStore : public SQLiteStoreManager { private: bool store_opened, store_initialized; int openStore(); diff --git a/include/StoreManager.h b/include/SQLiteStoreManager.h similarity index 84% rename from include/StoreManager.h rename to include/SQLiteStoreManager.h index d2b8077999..d48cb8cac4 100644 --- a/include/StoreManager.h +++ b/include/SQLiteStoreManager.h @@ -19,12 +19,12 @@ * */ -#ifndef _STORE_MANAGER_H_ -#define _STORE_MANAGER_H_ +#ifndef _SQLITE_STORE_MANAGER_H_ +#define _SQLITE_STORE_MANAGER_H_ #include "ntop_includes.h" -class StoreManager { +class SQLiteStoreManager { private: protected: int ifid; @@ -37,12 +37,13 @@ class StoreManager { int (*callback)(void *, int, char **, char **), void *payload); int exec_statement(sqlite3_stmt *stmt); + public: - StoreManager(int interface_id); - virtual ~StoreManager(); + SQLiteStoreManager(int interface_id); + virtual ~SQLiteStoreManager(); NetworkInterface* getNetworkInterface(); int optimizeStore(); }; -#endif /* _STORE_MANAGER_H_ */ +#endif /* _SQLITE_STORE_MANAGER_H_ */ diff --git a/include/StatsManager.h b/include/StatsManager.h index 7c81879433..cde1f50022 100644 --- a/include/StatsManager.h +++ b/include/StatsManager.h @@ -31,7 +31,7 @@ struct statsManagerRetrieval { statsManagerRetrieval(): num_vals(0) {} }; -class StatsManager : protected StoreManager { +class StatsManager : protected SQLiteStoreManager { public: StatsManager(int interface_id, const char *db_filename); ~StatsManager() {}; diff --git a/include/ntop_includes.h b/include/ntop_includes.h index 65dcaddbe9..14de819ca3 100644 --- a/include/ntop_includes.h +++ b/include/ntop_includes.h @@ -299,7 +299,7 @@ using namespace std; #include "PacketDumperTuntap.h" #include "TimelineExtract.h" #include "TcpFlowStats.h" -#include "StoreManager.h" +#include "SQLiteStoreManager.h" #include "StatsManager.h" #include "AlertStore.h" #include "DB.h" diff --git a/src/AlertStore.cpp b/src/AlertStore.cpp index 79f3fdf59c..9c30f5d94b 100644 --- a/src/AlertStore.cpp +++ b/src/AlertStore.cpp @@ -23,7 +23,7 @@ /* **************************************************** */ -AlertStore::AlertStore(int interface_id, const char *filename) : StoreManager(interface_id) { +AlertStore::AlertStore(int interface_id, const char *filename) : SQLiteStoreManager(interface_id) { char filePath[MAX_PATH]; /* Create the directories needed to keep the alerts database */ diff --git a/src/StoreManager.cpp b/src/SQLiteStoreManager.cpp similarity index 85% rename from src/StoreManager.cpp rename to src/SQLiteStoreManager.cpp index a40f56f5fe..43f7ea2769 100644 --- a/src/StoreManager.cpp +++ b/src/SQLiteStoreManager.cpp @@ -21,13 +21,17 @@ #include "ntop_includes.h" -StoreManager::StoreManager(int interface_id) { +/* **************************************************** */ + +SQLiteStoreManager::SQLiteStoreManager(int interface_id) { ifid = interface_id; iface = ntop->getInterfaceById(interface_id); db = NULL; }; -int StoreManager::init(const char *db_file_full_path) { +/* **************************************************** */ + +int SQLiteStoreManager::init(const char *db_file_full_path) { // db_file_full_path = (char*)":memory:"; if(sqlite3_open(db_file_full_path, &db)) { @@ -40,16 +44,22 @@ int StoreManager::init(const char *db_file_full_path) { return 0; } -NetworkInterface* StoreManager::getNetworkInterface() { +/* **************************************************** */ + +NetworkInterface* SQLiteStoreManager::getNetworkInterface() { if(!iface) iface = ntop->getInterfaceById(ifid); return iface; } -StoreManager::~StoreManager() { +/* **************************************************** */ + +SQLiteStoreManager::~SQLiteStoreManager() { if(db) sqlite3_close(db); } +/* **************************************************** */ + /** * @brief Executes a database query on an already opened SQLite3 DB * @brief This function implements handling of a direct query on @@ -64,7 +74,7 @@ StoreManager::~StoreManager() { * * @return Zero in case of success, nonzero in case of failure. */ -int StoreManager::exec_query(const char * const db_query, +int SQLiteStoreManager::exec_query(const char * const db_query, int (*callback)(void *, int, char **, char **), void *payload) { char *zErrMsg = 0; @@ -91,7 +101,7 @@ int StoreManager::exec_query(const char * const db_query, See https://www.sqlite.org/rescode.html */ -int StoreManager::exec_statement(sqlite3_stmt *stmt) { +int SQLiteStoreManager::exec_statement(sqlite3_stmt *stmt) { int rc; int max_retries = 5; bool retry = true; @@ -130,7 +140,7 @@ int StoreManager::exec_statement(sqlite3_stmt *stmt) { Should be called as disk space and defragmentation are not run automatically by sqlite. */ -int StoreManager::optimizeStore() { +int SQLiteStoreManager::optimizeStore() { char query[STORE_MANAGER_MAX_QUERY]; int step; bool rc = false; diff --git a/src/StatsManager.cpp b/src/StatsManager.cpp index 9408787392..e35077c5f2 100644 --- a/src/StatsManager.cpp +++ b/src/StatsManager.cpp @@ -21,8 +21,9 @@ #include "ntop_includes.h" +/* ******************************************* */ -StatsManager::StatsManager(int interface_id, const char *filename) : StoreManager(interface_id) { +StatsManager::StatsManager(int interface_id, const char *filename) : SQLiteStoreManager(interface_id) { char filePath[MAX_PATH], fileFullPath[MAX_PATH], fileName[MAX_PATH]; MINUTE_CACHE_NAME = "MINUTE_STATS"; @@ -47,6 +48,8 @@ StatsManager::StatsManager(int interface_id, const char *filename) : StoreManage init(fileFullPath); } +/* ******************************************* */ + /** * @brief Opens a new cache to be used to store statistics. * @brief This function implements opening a new cache to store stats @@ -85,6 +88,8 @@ int StatsManager::openStore(const char *cache_name) return rc; } +/* ******************************************* */ + /** * @brief Database interface to implement stats purging. * @details This function implements the database-specific code @@ -119,6 +124,8 @@ int StatsManager::deleteStatsOlderThan(const char * const cache_name, const time return rc; } +/* ******************************************* */ + /** * @brief Minute stats interface to database purging. * @details This function hides cache-specific details (e.g. building the key @@ -137,6 +144,8 @@ int StatsManager::deleteMinuteStatsOlderThan(unsigned num_days) { return deleteStatsOlderThan(MINUTE_CACHE_NAME, rawtime); } +/* ******************************************* */ + /** * @brief Hour stats interface to database purging. * @details This function hides cache-specific details (e.g. building the key @@ -155,6 +164,8 @@ int StatsManager::deleteHourStatsOlderThan(unsigned num_days) { return deleteStatsOlderThan(HOUR_CACHE_NAME, rawtime); } +/* ******************************************* */ + /** * @brief Day stats interface to database purging. * @details This function hides cache-specific details (e.g. building the key @@ -173,6 +184,8 @@ int StatsManager::deleteDayStatsOlderThan(unsigned num_days) { return deleteStatsOlderThan(DAY_CACHE_NAME, rawtime); } +/* ******************************************* */ + /** * @brief Callback for completion of retrieval of an interval of stats * @@ -195,6 +208,8 @@ static int get_samplings_db(void *data, int argc, return 0; } +/* ******************************************* */ + /** * @brief Retrieve an interval of samplings from a database * @details This function implements the database-specific code @@ -239,6 +254,8 @@ int StatsManager::retrieveStatsInterval(struct statsManagerRetrieval *retvals, return rc; } +/* ******************************************* */ + /** * @brief Retrieve an interval of samplings from the minute stats cache * @details This function implements the database-specific code @@ -261,6 +278,8 @@ int StatsManager::retrieveMinuteStatsInterval(time_t epoch_start, return retrieveStatsInterval(retvals, MINUTE_CACHE_NAME, epoch_start, epoch_end); } +/* ******************************************* */ + /** * @brief Retrieve an interval of samplings from the hour stats cache * @details This function implements the database-specific code @@ -283,6 +302,8 @@ int StatsManager::retrieveHourStatsInterval(time_t epoch_start, return retrieveStatsInterval(retvals, HOUR_CACHE_NAME, epoch_start, epoch_end); } +/* ******************************************* */ + /** * @brief Retrieve an interval of samplings from the day stats cache * @details This function implements the database-specific code @@ -305,6 +326,8 @@ int StatsManager::retrieveDayStatsInterval(time_t epoch_start, return retrieveStatsInterval(retvals, DAY_CACHE_NAME, epoch_start, epoch_end); } +/* ******************************************* */ + /** * @brief Database interface to add a new stats sampling * @details This function implements the database-specific layer for @@ -352,6 +375,8 @@ int StatsManager::insertSampling(const char * const sampling, const char * const return rc; } +/* ******************************************* */ + /** * @brief Interface function for insertion of a new minute stats sampling * @details This public method implements insertion of a new sampling, @@ -369,6 +394,8 @@ int StatsManager::insertMinuteSampling(time_t epoch, const char * const sampling return insertSampling(sampling, MINUTE_CACHE_NAME, static_cast(epoch)); } +/* ******************************************* */ + /** * @brief Interface function for insertion of a new hour stats sampling * @details This public method implements insertion of a new sampling, @@ -386,6 +413,8 @@ int StatsManager::insertHourSampling(time_t epoch, const char * const sampling) return insertSampling(sampling, HOUR_CACHE_NAME, static_cast(epoch)); } +/* ******************************************* */ + /** * @brief Interface function for insertion of a new day stats sampling * @details This public method implements insertion of a new sampling, @@ -428,6 +457,8 @@ static int get_sampling_db_callback(void *data, int argc, return 0; } +/* ******************************************* */ + /** * @brief Database interface to retrieve a stats sampling * @details This function implements the database-specific layer for @@ -468,6 +499,8 @@ int StatsManager::getSampling(string * sampling, const char * const cache_name, return rc; } +/* ******************************************* */ + /** * @brief Interface function for retrieval of a minute stats sampling * @details This public method implements retrieval of an existing @@ -489,3 +522,5 @@ int StatsManager::getMinuteSampling(time_t epoch, string * sampling) { return getSampling(sampling, MINUTE_CACHE_NAME, epoch_start, epoch_end); } + +/* ******************************************* */