diff --git a/include/Flow.h b/include/Flow.h index 8a48d3b4c1..5470b78432 100644 --- a/include/Flow.h +++ b/include/Flow.h @@ -84,6 +84,7 @@ class Flow : public GenericHashEntry { custom_app_t custom_app; void *cli_id, *srv_id; json_object *json_info; + ndpi_serializer *tlv_info; char *host_server_name, *bt_hash; #ifdef HAVE_NEDGE u_int32_t last_conntrack_update; @@ -314,6 +315,7 @@ class Flow : public GenericHashEntry { }; u_int16_t getStatsProtocol() const; void setJSONInfo(json_object *json); + void setTLVInfo(ndpi_serializer *tlv); #ifdef NTOPNG_PRO inline bool is_status_counted_in_aggregated_flow() const { return(status_counted_in_aggregated_flow); }; inline bool is_counted_in_aggregated_flow() const { return(counted_in_aggregated_flow); }; @@ -369,6 +371,7 @@ class Flow : public GenericHashEntry { inline const IpAddress* get_cli_ip_addr() const { return(cli_ip_addr); }; inline const IpAddress* get_srv_ip_addr() const { return(srv_ip_addr); }; inline json_object* get_json_info() const { return(json_info); }; + inline ndpi_serializer* get_tlv_info() const { return(tlv_info); }; inline bool has_long_icmp_payload() const { return(protos.icmp.has_long_icmp_payload); }; inline void set_long_icmp_payload() { protos.icmp.has_long_icmp_payload = true; }; inline ndpi_protocol_breed_t get_protocol_breed() const { diff --git a/include/ParsedFlow.h b/include/ParsedFlow.h index 60bf995ea7..7cfabe050b 100644 --- a/include/ParsedFlow.h +++ b/include/ParsedFlow.h @@ -28,7 +28,8 @@ class ParsedFlow : public ParsedFlowCore, public ParsedeBPF { private: bool parsed_flow_free_memory; bool has_parsed_ebpf; - json_object *additional_fields; + json_object *additional_fields_json; + ndpi_serializer *additional_fields_tlv; public: char *http_url, *http_site; @@ -46,10 +47,25 @@ class ParsedFlow : public ParsedFlowCore, public ParsedeBPF { ParsedFlow(); ParsedFlow(const ParsedFlow &pf); inline void addAdditionalField(const char *key, json_object *field) { - if (!additional_fields) additional_fields = json_object_new_object(); - if (additional_fields) json_object_object_add(additional_fields, key, field); + if (!additional_fields_json) additional_fields_json = json_object_new_object(); + if (additional_fields_json) json_object_object_add(additional_fields_json, key, field); } - inline json_object* getAdditionalFields() { return additional_fields; }; + inline void addAdditionalField(ndpi_deserializer *deserializer) { + if (!additional_fields_tlv) { + additional_fields_tlv = (ndpi_serializer *) calloc(1, sizeof(ndpi_serializer)); + if (additional_fields_tlv) + ndpi_init_serializer(additional_fields_tlv, + ndpi_serialization_format_tlv); + } + if (additional_fields_tlv) + ndpi_deserialize_clone_item(deserializer, additional_fields_tlv); + } + inline json_object *getAdditionalFieldsJSON() { return additional_fields_json; }; + inline ndpi_serializer *getAdditionalFieldsTLV() { + ndpi_serializer *tlv = additional_fields_tlv; + additional_fields_tlv = NULL; + return tlv; + }; inline bool hasParsedeBPF() const { return has_parsed_ebpf; }; inline void setParsedeBPF() { has_parsed_ebpf = true; }; virtual ~ParsedFlow(); diff --git a/src/Flow.cpp b/src/Flow.cpp index 09ca5316da..798e2678a1 100644 --- a/src/Flow.cpp +++ b/src/Flow.cpp @@ -62,7 +62,7 @@ Flow::Flow(NetworkInterface *_iface, ndpiFlow = NULL, cli_id = srv_id = NULL; cli_ebpf = srv_ebpf = NULL; - json_info = NULL, cli2srv_direction = true, twh_over = twh_ok = false, + json_info = NULL, tlv_info = NULL, cli2srv_direction = true, twh_over = twh_ok = false, dissect_next_http_packet = false, check_tor = false, host_server_name = NULL, diff_num_http_requests = 0, bt_hash = NULL; @@ -241,6 +241,10 @@ Flow::~Flow() { if(last_partial) free(last_partial); if(last_db_dump.partial) free(last_db_dump.partial); if(json_info) json_object_put(json_info); + if(tlv_info) { + ndpi_term_serializer(tlv_info); + free(tlv_info); + } if(host_server_name) free(host_server_name); if(cli_ebpf) delete cli_ebpf; @@ -664,6 +668,19 @@ void Flow::setJSONInfo(json_object *json) { /* *************************************** */ +void Flow::setTLVInfo(ndpi_serializer *tlv) { + if(tlv == NULL) return; + + if(tlv_info != NULL) { + ndpi_term_serializer(tlv_info); + free(tlv_info); + } + + tlv_info = tlv; +} + +/* *************************************** */ + /* * A faster replacement for inet_ntoa(). */ @@ -1632,6 +1649,7 @@ void Flow::lua(lua_State* vm, AddressTree * ptree, const IpAddress *src_ip = get_cli_ip_addr(), *dst_ip = get_srv_ip_addr(); bool src_match = true, dst_match = true; bool mask_cli_host = true, mask_dst_host = true, mask_flow; + bool has_json_info = false; FlowStatusMap status_map; if(ptree) { @@ -1907,7 +1925,28 @@ void Flow::lua(lua_State* vm, AddressTree * ptree, } } - lua_push_str_table_entry(vm, "moreinfo.json", get_json_info() ? json_object_to_json_string(get_json_info()) : "{}"); + if (get_json_info()) { + lua_push_str_table_entry(vm, "moreinfo.json", json_object_to_json_string(get_json_info())); + has_json_info = true; + } else if (get_tlv_info()) { + ndpi_deserializer deserializer; + if (ndpi_init_deserializer(&deserializer, get_tlv_info()) == 0) { + ndpi_serializer serializer; + if (ndpi_init_serializer(&serializer, ndpi_serialization_format_json) >= 0) { + char *buffer; + u_int32_t buffer_len; + ndpi_deserialize_clone_all(&deserializer, &serializer); + buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len); + if (buffer) { + lua_push_str_table_entry(vm, "moreinfo.json", buffer); + has_json_info = true; + } + ndpi_term_serializer(&serializer); + } + } + } + if (!has_json_info) + lua_push_str_table_entry(vm, "moreinfo.json", "{}"); if(cli_ebpf) processLua(vm, cli_ebpf, true); if(srv_ebpf) processLua(vm, srv_ebpf, false); diff --git a/src/NetworkInterface.cpp b/src/NetworkInterface.cpp index 15a8042d29..e763851bd5 100644 --- a/src/NetworkInterface.cpp +++ b/src/NetworkInterface.cpp @@ -1360,7 +1360,8 @@ void NetworkInterface::processFlow(ParsedFlow *zflow, bool zmq_flow) { p.category = NDPI_PROTOCOL_CATEGORY_UNSPECIFIED; flow->setDetectedProtocol(p, true); - flow->setJSONInfo(zflow->getAdditionalFields()); + flow->setJSONInfo(zflow->getAdditionalFieldsJSON()); + flow->setTLVInfo(zflow->getAdditionalFieldsTLV()); flow->updateInterfaceLocalStats(src2dst_direction, zflow->pkt_sampling_rate*(zflow->in_pkts+zflow->out_pkts), diff --git a/src/ParsedFlow.cpp b/src/ParsedFlow.cpp index 8da6ad057c..27ea9c6b88 100644 --- a/src/ParsedFlow.cpp +++ b/src/ParsedFlow.cpp @@ -24,7 +24,8 @@ /* *************************************** */ ParsedFlow::ParsedFlow() : ParsedFlowCore(), ParsedeBPF() { - additional_fields = NULL; + additional_fields_json = NULL; + additional_fields_tlv = NULL; http_url = http_site = NULL; dns_query = ssl_server_name = NULL; ja3c_hash = ja3s_hash = NULL; @@ -44,7 +45,10 @@ ParsedFlow::ParsedFlow() : ParsedFlowCore(), ParsedeBPF() { /* *************************************** */ ParsedFlow::ParsedFlow(const ParsedFlow &pf) : ParsedFlowCore(pf), ParsedeBPF(pf) { - additional_fields = NULL; /* Currently we avoid additional fields in the copy constructor */ + + /* Currently we avoid additional fields in the copy constructor */ + additional_fields_json = NULL; + additional_fields_tlv = NULL; if(pf.http_url) http_url = strdup(pf.http_url); else http_url = NULL; if(pf.http_site) http_site = strdup(pf.http_site); else http_site = NULL; @@ -72,8 +76,13 @@ ParsedFlow::ParsedFlow(const ParsedFlow &pf) : ParsedFlowCore(pf), ParsedeBPF(pf /* *************************************** */ ParsedFlow::~ParsedFlow() { - if(additional_fields) - json_object_put(additional_fields); + if(additional_fields_json) + json_object_put(additional_fields_json); + + if(additional_fields_tlv) { + ndpi_term_serializer(additional_fields_tlv); + free(additional_fields_tlv); + } if(parsed_flow_free_memory) { if(http_url) free(http_url); diff --git a/src/ZMQParserInterface.cpp b/src/ZMQParserInterface.cpp index 9d585c3b3f..698abcc086 100755 --- a/src/ZMQParserInterface.cpp +++ b/src/ZMQParserInterface.cpp @@ -982,8 +982,6 @@ int ZMQParserInterface::parseSingleTLVFlow(ndpi_deserializer *deserializer, goto error; } - ndpi_deserialize_next(deserializer); - if(key_is_string) { u_int8_t kbkp = key.str[key.str_len]; key.str[key.str_len] = '\0'; @@ -1069,13 +1067,20 @@ int ZMQParserInterface::parseSingleTLVFlow(ndpi_deserializer *deserializer, if(add_to_additional_fields) { //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Additional field: %s (Key-ID: %u PEN: %u)", key_str, key_id, pen); - flow.addAdditionalField(key_str, +#if 0 + flow.addAdditionalField(deserializer); +#else + flow.addAdditionalField(key_str, value_is_string ? json_object_new_string(value.string) : json_object_new_int64(value.uint_num)); +#endif } /* Restoring backed up character at the end of the string in place of '\0' */ if(value_is_string) vs.str[vs.str_len] = vbkp; - + + /* Move to the next element */ + ndpi_deserialize_next(deserializer); + } /* while */ end_of_record: