diff --git a/doc/src/api/rest/rest-api-v2.json b/doc/src/api/rest/rest-api-v2.json index c918ef7a9f..3887cb9873 100644 --- a/doc/src/api/rest/rest-api-v2.json +++ b/doc/src/api/rest/rest-api-v2.json @@ -680,174 +680,6 @@ } } }, - "/lua/rest/v2/get/alert/data.lua": { - "get": { - "tags": [ - "Alerts" - ], - "summary": "Get alerts data", - "description": "Alerts are returned", - "operationId": "get_alert_data", - "produces": [ - "application/json" - ], - "parameters": [{ - "name": "ifid", - "in": "query", - "description": "Interface identifier", - "required": true, - "type": "integer", - "format": "int32" - }, - { - "name": "status", - "in": "query", - "description": "Status filter (historical, historical-flows, engaged)", - "required": true, - "type": "string" - }, - { - "name": "epoch_begin", - "in": "query", - "description": "Start time (epoch) (optional)", - "required": true, - "type": "integer", - "format": "int32" - }, - { - "name": "epoch_end", - "in": "query", - "description": "Start time (epoch) (optional)", - "required": true, - "type": "integer", - "format": "int32" - }, - { - "name": "alert_type", - "in": "query", - "description": "Alert Type (optional)", - "required": false, - "type": "integer", - "format": "int32" - }, - { - "name": "alert_severity", - "in": "query", - "description": "Alert Severity (optional)", - "required": false, - "type": "integer", - "format": "int32" - } - ], - "responses": { - "0": { - "description": "OK", - "schema": { - "$ref": "#/definitions/Host" - } - }, - "-1": { - "description": "NOT_FOUND" - }, - "-2": { - "description": "INVALID_INTERFACE" - }, - "-3": { - "description": "NOT_GRANTED" - }, - "-4": { - "description": "INVALID_HOST" - }, - "-5": { - "description": "INVALID_ARGUMENTS" - }, - "-6": { - "description": "INTERNAL_ERROR" - }, - "-7": { - "description": "BAD_FORMAT" - }, - "-8": { - "description": "BAD_CONTENT" - } - } - } - }, - "/lua/rest/v2/get/alert/ts.lua": { - "get": { - "tags": [ - "Alerts" - ], - "summary": "Get alerts timeseries", - "description": "Number of alerts per hour divided by day are returned", - "operationId": "get_alert_ts", - "produces": [ - "application/json" - ], - "parameters": [{ - "name": "ifid", - "in": "query", - "description": "Interface identifier", - "required": true, - "type": "integer", - "format": "int32" - }, - { - "name": "status", - "in": "query", - "description": "Status filter (historical, historical-flows)", - "required": true, - "type": "string" - }, - { - "name": "epoch_begin", - "in": "query", - "description": "Start time (epoch)", - "required": true, - "type": "integer", - "format": "int32" - }, - { - "name": "epoch_end", - "in": "query", - "description": "Start time (epoch)", - "required": true, - "type": "integer", - "format": "int32" - }, - { - "name": "alert_type", - "in": "query", - "description": "Alert Type (optional)", - "required": false, - "type": "integer", - "format": "int32" - }, - { - "name": "alert_severity", - "in": "query", - "description": "Alert Severity (optional)", - "required": false, - "type": "integer", - "format": "int32" - } - ], - "responses": { - "0": { - "description": "OK" - }, - "-2": { - "description": "INVALID_INTERFACE" - }, - "-5": { - "description": "INVALID_ARGUMENTS" - }, - "-6": { - "description": "INTERNAL_ERROR" - } - } - } - }, "/lua/rest/v2/get/alert/type/consts.lua": { "get": { "tags": [ @@ -6615,36 +6447,6 @@ } } }, - "/lua/pro/rest/v2/get/interface/top/remote/talkers_ng.lua": { - "get": { - "tags": [ - "Top" - ], - "summary": "Get interface top talkers stats", - "description": "Get interface top talkers stats", - "operationId": "get_interface_top_talkers", - "produces": [ - "application/json" - ], - "parameters": [{ - "name": "ifid", - "in": "query", - "description": "Interface identifier", - "required": true, - "type": "integer", - "format": "int32" - } - ], - "responses": { - "0": { - "description": "OK" - }, - "-2": { - "description": "INVALID_INTERFACE" - } - } - } - }, "/lua/pro/rest/v2/get/interface/alert/general_stats.lua": { "get": { "tags": [ diff --git a/doc/src/clickhouse/clickhouse.rst b/doc/src/clickhouse/clickhouse.rst index d8ea85f4b9..448066a41e 100644 --- a/doc/src/clickhouse/clickhouse.rst +++ b/doc/src/clickhouse/clickhouse.rst @@ -36,27 +36,6 @@ IPv4 and IPv6 flows are stored in table :code:`flows`. A column :code:`INTERFACE Alerts are stored in several tables, all ending with suffix :code:`_alerts`. The table prefix indicates the alert family, e.g. :code:`host_alerts` table contains alerts for hosts, :code:`flow_alerts` table contains alerts for flows, and so on. -Health ------- - -The health of the currently used ClickHouse database can be checked under the System Interface, "Health" page. The status of the database is shown, along with the space used by flow and alert tables. - - -.. figure:: ../img/clickhouse_health.png - :align: center - :alt: ClickHouse Health - - ClickHouse Health - -Similarly, the number of exported and dropped flows can be checked, on a per-interface basis, under the interface stats page. Flow export drops should normally stay at zero or be a tiny fraction of the total amount of exported flows. Drops can occur due to the database being temporarily unreachable or not able to received additional data. - -.. figure:: ../img/clickhouse_exported_flows.png - :align: center - :alt: ClickHouse Exported Flows - - ClickHouse Exported Flows - - Migration from nIndex --------------------- @@ -67,7 +46,7 @@ The migration script requires as prerequisites executable :code:`clickhouse-clie Here are the steps to perform the migration. -- Install ClickHouse as described in the guide_. This will automatically pull in :code:`clickhouse-client`. +- Install ClickHouse as described in the `guide `_. This will automatically pull in :code:`clickhouse-client`. - Update the installed ntopng with the most recent development package (version :code:`5.1.211119` or above). This will automatically pull in :code:`nindex`. For packages already installed it suffices to run :code:`apt-get update && apt-get install ntopng` (on Debian/Ubuntu) or :code:`yum install ntopng` (on Centos). - Open a shell and download the script @@ -105,7 +84,7 @@ An ntopng instance was running with nIndex enabled for interface :code:`eno1`. To replace nIndex with ClickHouse, the following steps are performed. -- ClickHouse is installed locally following this guide_. :code:`clickhouse-client` can be used to verify connections can be successfully established to the local database +- ClickHouse is installed locally following this `guide `_. :code:`clickhouse-client` can be used to verify connections can be successfully established to the local database .. code:: bash diff --git a/doc/src/img/alert_monitor_home.png b/doc/src/img/alert_monitor_home.png new file mode 100644 index 0000000000..d6401b1df4 Binary files /dev/null and b/doc/src/img/alert_monitor_home.png differ diff --git a/doc/src/img/checks_gui_items_list.png b/doc/src/img/checks_gui_items_list.png index 5975b54d6d..89e8aeaf4f 100644 Binary files a/doc/src/img/checks_gui_items_list.png and b/doc/src/img/checks_gui_items_list.png differ diff --git a/doc/src/img/checks_gui_threshold_cross.png b/doc/src/img/checks_gui_threshold_cross.png index bfe8affd59..5959a42981 100644 Binary files a/doc/src/img/checks_gui_threshold_cross.png and b/doc/src/img/checks_gui_threshold_cross.png differ diff --git a/doc/src/img/clickhouse_monitor_home.png b/doc/src/img/clickhouse_monitor_home.png new file mode 100644 index 0000000000..9eca290913 Binary files /dev/null and b/doc/src/img/clickhouse_monitor_home.png differ diff --git a/doc/src/img/web_gui_network_discovery_page.png b/doc/src/img/web_gui_network_discovery_page.png new file mode 100644 index 0000000000..3b86347dba Binary files /dev/null and b/doc/src/img/web_gui_network_discovery_page.png differ diff --git a/doc/src/plugins/examples.rst b/doc/src/plugins/examples.rst index 39fbc9312f..def26c20ee 100644 --- a/doc/src/plugins/examples.rst +++ b/doc/src/plugins/examples.rst @@ -6,159 +6,6 @@ Examples A couple of examples are presented to give the reader a quick and direct overview of ntopng plugins. -.. _Blacklisted Flows: - -Blacklisted Flows ------------------ - -Aim of this plugin is to trigger an alert every time a flow is found -to have its client or server (or both) in a blacklist. ntopng loads -custom and predefined blacklists as explained in :ref:`Category -Lists`. This plugin tests each flow for its client and server, and -possibly create an alert when they are found to be blacklisted. - -Full plugin sources are available on `GitHub blacklisted flows plugin -page -`_. - -The complete structure of the plugin is as follows: - -.. code:: bash - - blacklisted - |-- manifest.lua - |-- checks - `-- flow - `-- blacklisted.lua - |-- alert_definitions - `-- alert_flow_blacklisted.lua - |-- status_definitions - `-- status_blacklisted.lua - -As it can be seen from the file system tree, a plugin is a set of Lua -files, placed in predefined sub-directories. - -The root directory, :code:`blacklisted`, carries a name which is -representative for the plugin. This directory contains other -sub-directories and a :code:`manifest.lua` (see :ref:`Manifest`) file containing basic plugin information: - -.. code:: lua - - -- - -- (C) 2019-20 - ntop.org - -- - - return { - title = "Blacklisted Hosts", - description = "Detects blacklisted hosts and triggers alerts", - author = "ntop", - dependencies = {}, - } - -Sub-directories -define the alerts the plugin is going to trigger, whereas :ref:`Flow Definitions` define flow statues the plugin is going to set. In this specific plugin, -:code:`alert_flow_blacklisted.lua` tells ntopng the plugin is willing -to create an alert for blacklisted flows. Similarly, -:code:`status_blacklisted.lua` tells ntopng the plugin is going to set -a blacklisted status for certain flows. Those two directories, as said -by their names, contain just definitions of alerts and flow status, -the actual logic which sets the status and trigger the alert resides in directory :code:`checks`. - -As this plugin requires flows to carry on its task, directory -:code:`checks` (see :ref:`Checks`) with the logic must contain a sub-directory -:code:`flow`, which, in turn, contains file -:code:`blacklisted.lua`. ntopng knows it has to execute -:code:`blacklisted.lua` against each flow it sees because -:code:`blacklisted.lua` is found under the :code:`flow` directory. - -Let's have a look at the -contents of :code:`blacklisted.lua`: - -.. code:: lua - - -- - -- (C) 2019-20 - ntop.org - -- - - local flow_consts = require("flow_consts") - - -- ################################################################# - - local script = { - -- NOTE: hooks defined below - hooks = {}, - - gui = { - i18n_title = "flow_checks_config.blacklisted", - i18n_description = "flow_checks_config.blacklisted_description", - } - } - - -- ################################################################# - - function script.hooks.protocolDetected(now) - if flow.isBlacklisted() then - local info = flow.getBlacklistedInfo() - local flow_score = 100 - local cli_score, srv_score - - if info["blacklisted.srv"] then - cli_score = 100 - srv_score = 5 - else - cli_score = 5 - srv_score = 10 - end - - flow.triggerStatus(flow_consts.status_types.status_blacklisted.status_key, info, - flow_score, cli_score, srv_score) - end - end - - -- ################################################################# - - return script - - -The first thing to observe, is that :code:`blacklisted.lua` contains a -single :code:`function` with a predefined -name :code:`script.hooks.protocolDetected`. This name tells -ntopng to execute the plugin for every flow, as soon as the flow has -its :code:`protocolDetected`, which is one of the several :ref:`Check Hooks` -a plugin can attach to. - -The body of the function has access to a :code:`flow` Lua table, with -several methods available to be called, among which -:code:`flow.isBlacklisted()`. Method :code:`flow.isBlacklisted()` -returns a boolean which is either true or false, depending on whether -any of the client or server is blacklisted. As this plugin wants to -trigger an alert then the flow is blacklisted, method is called and -tested in the first :code:`if`. When the flow is blacklisted and the -method returns true, a couple of scores are computed. **Scores** are -numbers associated to the client and server of the flow and attempt to -summarize how critical is the issue for both the client and the -server. - -The client score is much higher when the server is blacklisted because in this case it is assumed that the client is infected and -attempting to contact malicious hosts. When is the client to be -blacklisted, then it may just be a scan attempt by a malicious host -and thus the score is lower. - -Once the scores have been computed, the function calls -:code:`flow.triggerStatus`. This is the actual call that causes -ntopng to set the blacklisted status and trigger an alert! This call -wants the scores as parameters, along with the flow status defined -in :code:`status_definitions` and an info table which contains certain -extra details and description of the flow blacklisted peers. - -From this point on, the flow will appear as alerted and with status -blacklisted in the ntopng Web GUI, along with the scores specified for -its client and server. That is pretty much all to create a flow script. - -A quick note on the :ref:`Web GUI` section. It has just a title and a -description that will be used by ntopng in the web GUI, to allow a user -to enable/disable the plugin. - .. _Flow Flooders: Flow Flooders @@ -212,10 +59,7 @@ representative for the plugin. The :code:`manifest.lua` (see :ref:`Manifest`) sc dependencies = {}, } -This plugin doesn't work on flows, so no :code:`flow` directory is -present under :code:`checks` and no :code:`status_definitions` -is necessary as it has been seen for the `Blacklisted -Flows`_. However, as this plugin generates alerts, +However, as this plugin generates alerts, :code:`alert_flows_flood.lua` is needed under :code:`alert_definitions` to tell ntopng about this. diff --git a/doc/src/plugins/overview.rst b/doc/src/plugins/overview.rst index 020b8d44f9..f2b926d6b3 100644 --- a/doc/src/plugins/overview.rst +++ b/doc/src/plugins/overview.rst @@ -36,16 +36,6 @@ What is a Plugin A plugin is a collection of Lua scripts with a predefined structure. -Examples -~~~~~~~~ - -Examples of plugins are: - -- A `monitor for the disk space - `_ - which continuously observes free disk space and triggers alerts when the - space available is below a certain threshold - Availability ------------ diff --git a/doc/src/self_monitoring/alert_monitor.rst b/doc/src/self_monitoring/alert_monitor.rst new file mode 100644 index 0000000000..a8f81b53a8 --- /dev/null +++ b/doc/src/self_monitoring/alert_monitor.rst @@ -0,0 +1,16 @@ +.. _Alert Monitor: + +Alert Monitor +============= + +This monitor is used to understand if there is some alert loss and how much is the internal queue filled. + +.. figure:: ../img/alert_monitor_home.png + :align: center + :alt: The Alert Monitor Home + + The Alert Monitor Home + +The overview has the following items: + +- :code:`Fill Level/Dropped`: indicates how much is the internal queue filled and the number of alerts dropped. diff --git a/doc/src/self_monitoring/clickhouse_monitor.rst b/doc/src/self_monitoring/clickhouse_monitor.rst new file mode 100644 index 0000000000..bc85ae3ec2 --- /dev/null +++ b/doc/src/self_monitoring/clickhouse_monitor.rst @@ -0,0 +1,21 @@ +.. _ClickHouse Monitor: + +ClickHouse Monitor +================== + +ntopng features ClickHouse to export flows to `ClickHouse `_. Enabling the export into ClickHouse Database is going to make the :doc:`Historical Flow Explorer <../clickhouse/historical_flow_explorer>` page available. + +.. note:: + ClickHouse export is available only with Enterprise license + +.. figure:: ../img/clickhouse_monitor_home.png + :align: center + :alt: The ClickHouse Monitor Home + + The ClickHouse Monitor Home + +The overview has the following items: + +- :code:`Health`: A badge which is either green, yellow or red, depending on the status of ClickHouse. The badge is green when ntopng is correctly exporting to ClickHouse, yellow when there are export errors that are recoverable and didn't cause any data loss, red when export errors are persistent and data loss is occurring. +- :code:`Tables Size`: Total disk space used by ClickHouse. +- :code:`Last Errors`: A log trace used to check if some error occurred during the export of the flows diff --git a/doc/src/self_monitoring/index.rst b/doc/src/self_monitoring/index.rst index e2650b5dab..71c182c011 100644 --- a/doc/src/self_monitoring/index.rst +++ b/doc/src/self_monitoring/index.rst @@ -11,3 +11,5 @@ ntopng implements checks for its own status and health as well as for services u internals redis_monitor influxdb_monitor + alert_monitor + clickhouse_monitor diff --git a/doc/src/web_gui/index.rst b/doc/src/web_gui/index.rst index 793ec30918..28863445cd 100644 --- a/doc/src/web_gui/index.rst +++ b/doc/src/web_gui/index.rst @@ -58,6 +58,7 @@ Each individual menu bar entry will be discussed below. help_menu dashboard_menu dashboard + network_discovery historical report flows diff --git a/doc/src/web_gui/network_discovery.rst b/doc/src/web_gui/network_discovery.rst new file mode 100644 index 0000000000..91dd2275c3 --- /dev/null +++ b/doc/src/web_gui/network_discovery.rst @@ -0,0 +1,26 @@ +Network Discovery +================= + +Network Discovery is available under the Dashboard sub-menu. + +.. note:: + This feature is available only for `Packet Interfaces `_. + + +.. figure:: ../img/web_gui_network_discovery_page.png + :align: center + :alt: Network Discovery + Network Discovery + + +By clicking the `reload` button next to the page name, a Network Discovery is going to be launched. +ntopng now tries to contact all the available devices inside the `local network` by using different protocols (like `SSDP`, `MDNS` and other discovery protocols) and after few seconds, all the available devices are going to be displayed into the table. +The table contains: + +- `IP Address`: The IP Address of the device. +- `Name`: The name of the device (if available). +- `Manufacturer`: The manufacturer of the device. +- `MAC Address`: The MAC Address of the device. +- `OS`: The Operting System of the device (if available). +- `Info`: The services made available from the device. +- `Device`: The device type (Router, PC, ...). diff --git a/scripts/lua/rest/v2/get/alert/type/counters.lua b/scripts/lua/rest/v2/get/alert/type/counters.lua index a634664216..8f053d427e 100644 --- a/scripts/lua/rest/v2/get/alert/type/counters.lua +++ b/scripts/lua/rest/v2/get/alert/type/counters.lua @@ -19,7 +19,7 @@ local all_alert_store = require "all_alert_store".new() -- local rc = rest_utils.consts.success.ok -local ifid = _GET["ifid"] +local ifid = _GET["ifid"] or "" if not auth.has_capability(auth.capabilities.alerts) then rest_utils.answer(rest_utils.consts.err.not_granted)