Replaced ndpiReader's libjson-c support with libnDPI's internal serialization interface. (#1535)

* Fixes #1528
 * Serialization Interface should also fuzzed
 * libjson-c may only be used in the unit test to verify the internal serialization interface
 * Serialization Interface supports tlv(broken), csv and json
 * Unit test does work again and requires libjson-c

Signed-off-by: lns <matzeton@googlemail.com>
This commit is contained in:
Toni 2022-05-07 09:26:09 +02:00 committed by GitHub
parent 2e0dedbaae
commit 87f93ea4fd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 591 additions and 1123 deletions

View file

@ -99,7 +99,11 @@ AC_DEFINE_UNQUOTED(NDPI_GIT_DATE, "${GIT_DATE}", [Last GIT change])
if ! test "${with_only_libndpi+set}" = set; then :
dnl> used by json-c for unit tests
PKG_CHECK_MODULES([JSONC], [json-c], [JSONC_LIBS="${pkg_cv_JSONC_LIBS}" JSONC_CFLAGS="${pkg_cv_JSONC_CFLAGS}"], [AC_MSG_WARN([JSON-C not available. Disabled unit test.])])
PKG_CHECK_MODULES([JSONC], [json-c], [
AC_DEFINE(HAVE_LIBJSON_C, 1, [libjson-c is present])
JSONC_LIBS="${pkg_cv_JSONC_LIBS}"
JSONC_CFLAGS="${pkg_cv_JSONC_CFLAGS}"
], [AC_MSG_WARN([JSON-C not available. Disabled unit test.])])
AC_CHECK_LIB([json-c], [json_object_put], [EXTRA_TARGETS="$EXTRA_TARGETS tests/unit"], [
AC_MSG_WARN([JSON-C not available. Disabled unit test.])
JSONC_LIBS=""

File diff suppressed because it is too large Load diff

View file

@ -394,7 +394,8 @@ extern char *_debug_protocols;
static int _debug_protocols_ok = 0;
struct ndpi_workflow* ndpi_workflow_init(const struct ndpi_workflow_prefs * prefs,
pcap_t * pcap_handle, int do_init_flows_root) {
pcap_t * pcap_handle, int do_init_flows_root,
ndpi_serialization_format serialization_format) {
struct ndpi_detection_module_struct * module;
struct ndpi_workflow * workflow;
@ -433,6 +434,14 @@ struct ndpi_workflow* ndpi_workflow_init(const struct ndpi_workflow_prefs * pref
if(do_init_flows_root)
workflow->ndpi_flows_root = ndpi_calloc(workflow->prefs.num_roots, sizeof(void *));
if (serialization_format != ndpi_serialization_format_unknown &&
ndpi_init_serializer(&workflow->ndpi_serializer,
serialization_format) != 0)
{
LOG(NDPI_LOG_ERROR, "serializer initialization failed\n");
exit(-1);
}
return workflow;
}
@ -539,6 +548,8 @@ void ndpi_flow_info_free_data(struct ndpi_flow_info *flow) {
void ndpi_workflow_free(struct ndpi_workflow * workflow) {
u_int i;
ndpi_term_serializer(&workflow->ndpi_serializer);
for(i=0; i<workflow->prefs.num_roots; i++)
ndpi_tdestroy(workflow->ndpi_flows_root[i], ndpi_flow_info_freer);
@ -857,12 +868,14 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
inet_ntop(AF_INET, &newflow->src_ip, newflow->src_name, sizeof(newflow->src_name));
inet_ntop(AF_INET, &newflow->dst_ip, newflow->dst_name, sizeof(newflow->dst_name));
} else {
struct in6_addr addr = *(struct in6_addr *)&iph6->ip6_src;
inet_ntop(AF_INET6, &addr, newflow->src_name, sizeof(newflow->src_name));
addr = *(struct in6_addr *)&iph6->ip6_dst;
inet_ntop(AF_INET6, &addr, newflow->dst_name, sizeof(newflow->dst_name));
/* For consistency across platforms replace :0: with :: */
ndpi_patchIPv6Address(newflow->src_name), ndpi_patchIPv6Address(newflow->dst_name);
newflow->src_ip6 = *(struct ndpi_in6_addr *)&iph6->ip6_src;
inet_ntop(AF_INET6, &newflow->src_ip6,
newflow->src_name, sizeof(newflow->src_name));
newflow->dst_ip6 = *(struct ndpi_in6_addr *)&iph6->ip6_dst;
inet_ntop(AF_INET6, &newflow->dst_ip6,
newflow->dst_name, sizeof(newflow->dst_name));
/* For consistency across platforms replace :0: with :: */
ndpi_patchIPv6Address(newflow->src_name), ndpi_patchIPv6Address(newflow->dst_name);
}
if((newflow->ndpi_flow = ndpi_flow_malloc(SIZEOF_FLOW_STRUCT)) == NULL) {
@ -1021,11 +1034,13 @@ u_int8_t plen2slot(u_int16_t plen) {
/* ****************************************************** */
void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow, FILE * csv_fp) {
void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow) {
u_int i, is_quic = 0;
if(!flow->ndpi_flow) return;
flow->info_type = INFO_INVALID;
flow->confidence = flow->ndpi_flow->confidence;
ndpi_snprintf(flow->host_server_name, sizeof(flow->host_server_name), "%s",
@ -1059,8 +1074,11 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
/* DNS */
else if(is_ndpi_proto(flow, NDPI_PROTOCOL_DNS)) {
if(flow->ndpi_flow->protos.dns.rsp_type == 0x1)
{
flow->info_type = INFO_GENERIC;
inet_ntop(AF_INET, &flow->ndpi_flow->protos.dns.rsp_addr.ipv4, flow->info, sizeof(flow->info));
else {
} else {
flow->info_type = INFO_GENERIC;
inet_ntop(AF_INET6, &flow->ndpi_flow->protos.dns.rsp_addr.ipv6, flow->info, sizeof(flow->info));
/* For consistency across platforms replace :0: with :: */
@ -1069,10 +1087,12 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
}
/* MDNS */
else if(is_ndpi_proto(flow, NDPI_PROTOCOL_MDNS)) {
flow->info_type = INFO_GENERIC;
ndpi_snprintf(flow->info, sizeof(flow->info), "%s", flow->ndpi_flow->host_server_name);
}
/* UBNTAC2 */
else if(is_ndpi_proto(flow, NDPI_PROTOCOL_UBNTAC2)) {
flow->info_type = INFO_GENERIC;
ndpi_snprintf(flow->info, sizeof(flow->info), "%s", flow->ndpi_flow->protos.ubntac2.version);
}
/* FTP */
@ -1080,33 +1100,28 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
|| /* IMAP */ is_ndpi_proto(flow, NDPI_PROTOCOL_MAIL_IMAP)
|| /* POP */ is_ndpi_proto(flow, NDPI_PROTOCOL_MAIL_POP)
|| /* SMTP */ is_ndpi_proto(flow, NDPI_PROTOCOL_MAIL_SMTP)) {
if(flow->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.username[0] != '\0')
ndpi_snprintf(flow->info, sizeof(flow->info), "User: %s][Pwd: %s%s",
flow->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.username,
flow->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.password,
flow->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.auth_failed ? "][Auth Failed" : "");
flow->info_type = INFO_FTP_IMAP_POP_SMTP;
ndpi_snprintf(flow->ftp_imap_pop_smtp.username,
sizeof(flow->ftp_imap_pop_smtp.username),
"%s", flow->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.username);
ndpi_snprintf(flow->ftp_imap_pop_smtp.password,
sizeof(flow->ftp_imap_pop_smtp.password),
"%s", flow->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.password);
flow->ftp_imap_pop_smtp.auth_failed =
flow->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.auth_failed;
}
/* KERBEROS */
else if(is_ndpi_proto(flow, NDPI_PROTOCOL_KERBEROS)) {
if((flow->ndpi_flow->protos.kerberos.hostname[0] != '\0')
|| (flow->ndpi_flow->protos.kerberos.username[0] != '\0')) {
ndpi_snprintf(flow->info, sizeof(flow->info), "%s%s%s%s",
flow->ndpi_flow->protos.kerberos.domain /* = realm */,
flow->ndpi_flow->protos.kerberos.domain[0] != '\0' ? "\\" : "",
flow->ndpi_flow->protos.kerberos.hostname,
flow->ndpi_flow->protos.kerberos.username);
} else if(flow->ndpi_flow->protos.kerberos.domain[0] != '\0')
ndpi_snprintf(flow->info, sizeof(flow->info), "%s",
flow->ndpi_flow->protos.kerberos.domain);
#if 0
if(flow->info[0] != '\0')
printf("->> (%d) [%s][%s][%s]<<--\n",
htons(flow->src_port),
flow->ndpi_flow->protos.kerberos.domain,
flow->ndpi_flow->protos.kerberos.hostname,
flow->ndpi_flow->protos.kerberos.username);
#endif
flow->info_type = INFO_KERBEROS;
ndpi_snprintf(flow->kerberos.domain,
sizeof(flow->kerberos.domain),
"%s", flow->ndpi_flow->protos.kerberos.domain);
ndpi_snprintf(flow->kerberos.hostname,
sizeof(flow->kerberos.hostname),
"%s", flow->ndpi_flow->protos.kerberos.hostname);
ndpi_snprintf(flow->kerberos.username,
sizeof(flow->kerberos.username),
"%s", flow->ndpi_flow->protos.kerberos.username);
}
/* HTTP */
else if((flow->detected_protocol.master_protocol == NDPI_PROTOCOL_HTTP)
@ -1187,22 +1202,18 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
correct_csv_data_field(flow->ndpi_flow->protos.tls_quic.alpn);
correct_csv_data_field(flow->ndpi_flow->protos.tls_quic.tls_supported_versions);
if(csv_fp)
ndpi_snprintf(flow->info, sizeof(flow->info), "%s",
flow->ndpi_flow->protos.tls_quic.alpn);
else
ndpi_snprintf(flow->info, sizeof(flow->info), "ALPN: %s][TLS Supported Versions: %s",
flow->ndpi_flow->protos.tls_quic.alpn,
flow->ndpi_flow->protos.tls_quic.tls_supported_versions);
flow->info_type = INFO_TLS_QUIC_ALPN_VERSION;
ndpi_snprintf(flow->tls_quic.alpn, sizeof(flow->tls_quic.alpn), "%s",
flow->ndpi_flow->protos.tls_quic.alpn);
ndpi_snprintf(flow->tls_quic.tls_supported_versions,
sizeof(flow->tls_quic.tls_supported_versions),
"%s", flow->ndpi_flow->protos.tls_quic.tls_supported_versions);
} else if(flow->ndpi_flow->protos.tls_quic.alpn) {
correct_csv_data_field(flow->ndpi_flow->protos.tls_quic.alpn);
if(csv_fp)
ndpi_snprintf(flow->info, sizeof(flow->info), "%s,",
flow->ndpi_flow->protos.tls_quic.alpn);
else
ndpi_snprintf(flow->info, sizeof(flow->info), "ALPN: %s",
flow->ndpi_flow->protos.tls_quic.alpn);
flow->info_type = INFO_TLS_QUIC_ALPN_ONLY;
ndpi_snprintf(flow->tls_quic.alpn, sizeof(flow->tls_quic.alpn), "%s",
flow->ndpi_flow->protos.tls_quic.alpn);
}
if(enable_doh_dot_detection) {
@ -1305,8 +1316,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
const struct pcap_pkthdr *header,
const u_char *packet,
pkt_timeval when,
ndpi_risk *flow_risk,
FILE * csv_fp) {
ndpi_risk *flow_risk) {
struct ndpi_flow_info *flow = NULL;
struct ndpi_flow_struct *ndpi_flow = NULL;
u_int8_t proto;
@ -1546,7 +1556,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
if(enable_protocol_guess) workflow->stats.guessed_flow_protocols++;
}
process_ndpi_collected_info(workflow, flow, csv_fp);
process_ndpi_collected_info(workflow, flow);
}
}
}
@ -1598,8 +1608,7 @@ int ndpi_is_datalink_supported(int datalink_type) {
struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
const struct pcap_pkthdr *header,
const u_char *packet,
ndpi_risk *flow_risk,
FILE * csv_fp) {
ndpi_risk *flow_risk) {
/*
* Declare pointers to packet headers
*/
@ -2093,7 +2102,7 @@ struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
return(packet_processing(workflow, time_ms, vlan_id, tunnel_type, iph, iph6,
ip_offset, header->caplen - ip_offset,
header->caplen, header, packet, header->ts,
flow_risk, csv_fp));
flow_risk));
}
/* ********************************************************** */

View file

@ -90,6 +90,7 @@ extern int dpdk_port_deinit(int port);
#define MAX_TABLE_SIZE_1 4096
#define MAX_TABLE_SIZE_2 8192
#define INIT_VAL -1
#define SERIALIZATION_BUFSIZ (8192 * 2)
// inner hash table (ja3 -> security state)
@ -157,12 +158,23 @@ struct ndpi_entropy {
float score;
};
enum info_type {
INFO_INVALID = 0,
INFO_GENERIC,
INFO_KERBEROS,
INFO_FTP_IMAP_POP_SMTP,
INFO_TLS_QUIC_ALPN_VERSION,
INFO_TLS_QUIC_ALPN_ONLY,
};
// flow tracking
typedef struct ndpi_flow_info {
u_int32_t flow_id;
u_int32_t hashval;
u_int32_t src_ip; /* network order */
u_int32_t dst_ip; /* network order */
struct ndpi_in6_addr src_ip6; /* network order */
struct ndpi_in6_addr dst_ip6; /* network order */
u_int16_t src_port; /* network order */
u_int16_t dst_port; /* network order */
u_int8_t detection_completed, protocol, bidirectional, check_extra_packets;
@ -196,7 +208,25 @@ typedef struct ndpi_flow_info {
struct ndpi_analyze_struct *iat_c_to_s, *iat_s_to_c, *iat_flow,
*pktlen_c_to_s, *pktlen_s_to_c;
char info[255];
enum info_type info_type;
union {
char info[256];
struct {
char alpn[128];
char tls_supported_versions[128];
} tls_quic;
struct {
unsigned char auth_failed;
char username[127];
char password[128];
} ftp_imap_pop_smtp;
struct {
char domain[85];
char hostname[85];
char username[86];
} kerberos;
};
char flow_extra_info[16];
char host_server_name[80]; /* Hostname/SNI */
char *bittorent_hash;
@ -302,11 +332,14 @@ typedef struct ndpi_workflow {
void **ndpi_flows_root;
struct ndpi_detection_module_struct *ndpi_struct;
u_int32_t num_allocated_flows;
/* CSV,TLV,JSON serialization interface */
ndpi_serializer ndpi_serializer;
} ndpi_workflow_t;
/* TODO: remove wrappers parameters and use ndpi global, when their initialization will be fixed... */
struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * prefs, pcap_t * pcap_handle, int do_init_flows_root);
struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * prefs, pcap_t * pcap_handle, int do_init_flows_root, ndpi_serialization_format serialization_format);
/* workflow main free function */
@ -324,8 +357,7 @@ void ndpi_free_flow_info_half(struct ndpi_flow_info *flow);
struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
const struct pcap_pkthdr *header,
const u_char *packet,
ndpi_risk *flow_risk,
FILE * csv_fp);
ndpi_risk *flow_risk);
int ndpi_is_datalink_supported(int datalink_type);
@ -345,7 +377,7 @@ static inline void ndpi_workflow_set_flow_giveup_callback(struct ndpi_workflow *
/* compare two nodes in workflow */
int ndpi_workflow_node_cmp(const void *a, const void *b);
void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow, FILE * csv_fp);
void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow);
u_int32_t ethernet_crc32(const void* data, size_t n_bytes);
void ndpi_flow_info_free_data(struct ndpi_flow_info *flow);
void ndpi_flow_info_freer(void *node);

View file

@ -60,7 +60,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
prefs->max_ndpi_flows = 1024 * 1024;
prefs->quiet_mode = 0;
workflow = ndpi_workflow_init(prefs, NULL /* pcap handler will be set later */, 0);
workflow = ndpi_workflow_init(prefs, NULL /* pcap handler will be set later */, 0, ndpi_serialization_format_json);
// enable all protocols
NDPI_BITMASK_SET_ALL(all);
ndpi_set_protocol_detection_bitmask2(workflow->ndpi_struct, &all);
@ -79,6 +79,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
if (pkts == NULL) {
remove(pcap_path);
free(pcap_path);
ndpi_term_serializer(&workflow->ndpi_serializer);
return 0;
}
if (ndpi_is_datalink_supported(pcap_datalink(pkts)) == 0)
@ -87,6 +88,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
pcap_close(pkts);
remove(pcap_path);
free(pcap_path);
ndpi_term_serializer(&workflow->ndpi_serializer);
return 0;
}
@ -104,7 +106,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
ndpi_risk flow_risk;
memcpy(packet_checked, pkt, header->caplen);
ndpi_workflow_process_packet(workflow, header, packet_checked, &flow_risk, NULL);
ndpi_workflow_process_packet(workflow, header, packet_checked, &flow_risk);
free(packet_checked);
}

View file

@ -4,6 +4,8 @@
#include <stdio.h>
struct ndpi_detection_module_struct *ndpi_info_mod = NULL;
static ndpi_serializer json_serializer = {};
static ndpi_serializer csv_serializer = {};
int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
uint8_t protocol_was_guessed;
@ -17,12 +19,27 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
ndpi_set_log_level(ndpi_info_mod, 4);
ndpi_set_debug_bitmask(ndpi_info_mod, debug_bitmask);
ndpi_finalize_initialization(ndpi_info_mod);
ndpi_init_serializer(&json_serializer, ndpi_serialization_format_json);
ndpi_init_serializer(&csv_serializer, ndpi_serialization_format_csv);
}
struct ndpi_flow_struct *ndpi_flow = ndpi_flow_malloc(SIZEOF_FLOW_STRUCT);
memset(ndpi_flow, 0, SIZEOF_FLOW_STRUCT);
ndpi_detection_process_packet(ndpi_info_mod, ndpi_flow, Data, Size, 0);
ndpi_detection_giveup(ndpi_info_mod, ndpi_flow, 1, &protocol_was_guessed);
ndpi_protocol detected_protocol =
ndpi_detection_process_packet(ndpi_info_mod, ndpi_flow, Data, Size, 0);
ndpi_protocol guessed_protocol =
ndpi_detection_giveup(ndpi_info_mod, ndpi_flow, 1, &protocol_was_guessed);
ndpi_reset_serializer(&json_serializer);
ndpi_reset_serializer(&csv_serializer);
if (protocol_was_guessed == 0)
{
ndpi_dpi2json(ndpi_info_mod, ndpi_flow, detected_protocol, &json_serializer);
ndpi_dpi2json(ndpi_info_mod, ndpi_flow, detected_protocol, &csv_serializer);
} else {
ndpi_dpi2json(ndpi_info_mod, ndpi_flow, guessed_protocol, &json_serializer);
ndpi_dpi2json(ndpi_info_mod, ndpi_flow, guessed_protocol, &csv_serializer);
}
ndpi_free_flow(ndpi_flow);
return 0;

View file

@ -55,9 +55,7 @@
#include "ndpi_api.h"
#include "ndpi_define.h"
#ifdef HAVE_LIBJSON_C
#include "json.h" /* JSON-C */
#endif
static struct ndpi_detection_module_struct *ndpi_info_mod = NULL;
static int verbose = 0;
@ -65,7 +63,6 @@ static int verbose = 0;
/* *********************************************** */
int serializerUnitTest() {
#ifdef HAVE_LIBJSON_C
ndpi_serializer serializer, deserializer;
int i, loop_id;
ndpi_serialization_format fmt = {0};
@ -229,7 +226,6 @@ int serializerUnitTest() {
}
printf("%30s OK\n", __FUNCTION__);
#endif
return 0;
}
@ -237,7 +233,6 @@ int serializerUnitTest() {
int serializeProtoUnitTest(void)
{
#ifdef HAVE_LIBJSON_C
ndpi_serializer serializer;
int loop_id;
ndpi_serialization_format fmt = {0};
@ -277,7 +272,7 @@ int serializeProtoUnitTest(void)
{
buffer_len = 0;
buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len);
char const * const expected_json_str = "{\"ndpi\": {\"flow_risk\": {\"6\": {\"risk\":\"Self-signed Certificate\",\"severity\":\"High\",\"risk_score\": {\"total\":500,\"client\":450,\"server\":50}},\"7\": {\"risk\":\"Obsolete TLS Version (1.1 or older)\",\"severity\":\"High\",\"risk_score\": {\"total\":510,\"client\":455,\"server\":55}},\"8\": {\"risk\":\"Weak TLS Cipher\",\"severity\":\"High\",\"risk_score\": {\"total\":250,\"client\":225,\"server\":25}},\"17\": {\"risk\":\"Malformed Packet\",\"severity\":\"Low\",\"risk_score\": {\"total\":260,\"client\":130,\"server\":130}}},\"confidence\": {\"4\":\"DPI\"},\"proto\":\"TLS.Facebook\",\"breed\":\"Fun\",\"category\":\"SocialNetwork\"}}";
char const * const expected_json_str = "{\"ndpi\": {\"flow_risk\": {\"6\": {\"risk\":\"Self-signed Cert\",\"severity\":\"High\",\"risk_score\": {\"total\":500,\"client\":450,\"server\":50}},\"7\": {\"risk\":\"Obsolete TLS (v1.1 or older)\",\"severity\":\"High\",\"risk_score\": {\"total\":510,\"client\":455,\"server\":55}},\"8\": {\"risk\":\"Weak TLS Cipher\",\"severity\":\"High\",\"risk_score\": {\"total\":250,\"client\":225,\"server\":25}},\"17\": {\"risk\":\"Malformed Packet\",\"severity\":\"Low\",\"risk_score\": {\"total\":260,\"client\":130,\"server\":130}}},\"confidence\": {\"4\":\"DPI\"},\"proto\":\"TLS.Facebook\",\"breed\":\"Fun\",\"category\":\"SocialNetwork\"}}";
if (strncmp(buffer, expected_json_str, buffer_len) != 0)
{
@ -316,7 +311,6 @@ int serializeProtoUnitTest(void)
}
printf("%30s OK\n", __FUNCTION__);
#endif
return 0;
}