fuzz: improve fuzzing coverage (#3020)

We should pay attention to tell ndpiReader configuration files and
libnDPI configuration files!! Better solution?

Be sure that configuration files are located where they are expected.
In oss-fuzz enviroment we can't make any assumptions about the current
working directory of your fuzz target.

example/config_only_classification.txt

@@ -0,0 +1,48 @@
+#Example of configuration if you are interested ONLY in flow (sub)-classification
+#(i.e. no metadata at all and no flow risks)
+
+#No flow risks
+flow_risk.all,0
+
+#General metadata
+metadata.tcp_fingerprint,0
+metadata.ndpi_fingerprint,0
+dpi.compute_entropy,0
+#BITTORRENT
+bittorrent,metadata.hash,0
+#SSDP
+ssdp,metadata,0
+#TLS (we might need ja4c for subclassification)
+tls,metadata.sha1_fingerprint,0
+tls,metadata.ja3s_fingerprint,0
+tls,metadata.cert_server_names,0
+tls,metadata.cert_validity,0
+tls,metadata.cert_issuer,0
+tls,metadata.cert_subject,0
+tls,metadata.alpn_negotiated,0
+tls,metadata.versions_supported,0
+tls,metadata.cipher,0
+tls,metadata.browser,0
+#SIP
+sip,metadata.attribute.from,0
+sip,metadata.attribute.to,0
+#STUN
+stun,metadata.attribute.mapped_address,0
+stun,metadata.attribute.peer_address,0
+stun,metadata.attribute.relayed_address,0
+stun,metadata.attribute.response_origin,0
+stun,metadata.attribute.other_address,0
+#HTTP
+http,metadata.req.content_type,0
+http,metadata.req.referer,0
+http,metadata.req.host,0
+http,metadata.req.username,0
+http,metadata.req.password,0
+http,metadata.resp.content_type,0
+http,metadata.resp.server,0
+
+#DNS:we need only the request for sub-classification
+dns,process_response,0
+
+#RTP
+rtp,max_packets_extra_dissection,0

example/only_classification.conf

@@ -13,14 +13,33 @@
 #SSDP
 --cfg=ssdp,metadata,0
 #TLS (we might need ja4c for subclassification)
---cfg=tls,metadata.sha1_fingerprint,0 --cfg=tls,metadata.ja3s_fingerprint,0 --cfg=tls,metadata.cert_server_names,0 --cfg=tls,metadata.cert_validity,0 --cfg=tls,metadata.cert_issuer,0 --cfg=tls,metadata.cert_subject,0 --cfg=tls,metadata.alpn_negotiated,0 --cfg=tls,metadata.versions_supported,0 --cfg=tls,metadata.cipher,0 --cfg=tls,metadata.browser,0
+--cfg=tls,metadata.sha1_fingerprint,0
+--cfg=tls,metadata.ja3s_fingerprint,0
+--cfg=tls,metadata.cert_server_names,0
+--cfg=tls,metadata.cert_validity,0
+--cfg=tls,metadata.cert_issuer,0
+--cfg=tls,metadata.cert_subject,0
+--cfg=tls,metadata.alpn_negotiated,0
+--cfg=tls,metadata.versions_supported,0
+--cfg=tls,metadata.cipher,0
+--cfg=tls,metadata.browser,0
 #SIP
---cfg=sip,metadata.attribute.from,0 --cfg=sip,metadata.attribute.to,0
+--cfg=sip,metadata.attribute.from,0
+--cfg=sip,metadata.attribute.to,0
 #STUN
---cfg=stun,metadata.attribute.mapped_address,0 --cfg=stun,metadata.attribute.peer_address,0 --cfg=stun,metadata.attribute.relayed_address,0 --cfg=stun,metadata.attribute.response_origin,0 --cfg=stun,metadata.attribute.other_address,0
+--cfg=stun,metadata.attribute.mapped_address,0
+--cfg=stun,metadata.attribute.peer_address,0
+--cfg=stun,metadata.attribute.relayed_address,0
+--cfg=stun,metadata.attribute.response_origin,0
+--cfg=stun,metadata.attribute.other_address,0
 #HTTP
---cfg=http,metadata.req.content_type,0 --cfg=http,metadata.req.referer,0 --cfg=http,metadata.req.host,0 --cfg=http,metadata.req.username,0 --cfg=http,metadata.req.password,0
---cfg=http,metadata.resp.content_type,0  --cfg=http,metadata.resp.server,0 
+--cfg=http,metadata.req.content_type,0
+--cfg=http,metadata.req.referer,0
+--cfg=http,metadata.req.host,0
+--cfg=http,metadata.req.username,0
+--cfg=http,metadata.req.password,0
+--cfg=http,metadata.resp.content_type,0
+--cfg=http,metadata.resp.server,0
 
 #DNS:we need only the request for sub-classification
 --cfg=dns,process_response,0 #Note that this option has an huge impact on FPC!

example/reader_util.c

@@ -2108,8 +2108,6 @@ static inline u_int ndpi_skip_vxlan(u_int16_t ip_offset, u_int16_t ip_len){
 static uint32_t ndpi_is_valid_gre_tunnel(const struct pcap_pkthdr *header,
                 const u_char *packet, const u_int16_t ip_offset,
                 const u_int16_t ip_len) {
-  if(header->caplen < ip_offset + ip_len + sizeof(struct ndpi_gre_basehdr))
-    return 0; /* Too short for GRE header*/
   uint32_t offset = ip_offset + ip_len;
   struct ndpi_gre_basehdr *grehdr = (struct ndpi_gre_basehdr*)&packet[offset];
   offset += sizeof(struct ndpi_gre_basehdr);

fuzz/fuzz_ndpi_reader.c

@@ -7,6 +7,8 @@
 #include <errno.h>
 #include <stdint.h>
 #include <stdio.h>
+#include <assert.h>
+#include <libgen.h>
 
 #ifdef ENABLE_PCAP_L7_MUTATOR
 #include "pl7m.h"
@@ -26,6 +28,8 @@ char *addr_dump_path = NULL;
 int monitoring_enabled = 1;
 u_int8_t enable_doh_dot_detection = 0;
 
+static char *path = NULL;
+
 extern void ndpi_report_payload_stats(FILE *out);
 
 #ifdef CRYPT_FORCE_NO_AESNI
@@ -39,6 +43,13 @@ size_t LLVMFuzzerCustomMutator(uint8_t *Data, size_t Size,
 }
 #endif
 
+int LLVMFuzzerInitialize(int *argc, char ***argv) {
+  (void)argc;
+
+  path = dirname(strdup(*argv[0])); /* No errors; no free! */
+  return 0;
+}
+
 static void node_cleanup_walker(const void *node, ndpi_VISIT which, int depth, void *user_data) {
   struct ndpi_flow_info *flow = *(struct ndpi_flow_info **) node;
 
@@ -55,6 +66,13 @@ static void node_cleanup_walker(const void *node, ndpi_VISIT which, int depth, v
   }
 }
 
+static void fn_flow_callback(struct ndpi_workflow *w, struct ndpi_flow_info *f, void *d)
+{
+  (void)w;
+  (void)f;
+  (void)d;
+}
+
 int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   pcap_t * pkts;
   const u_char *pkt;
@@ -63,6 +81,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   char errbuf[PCAP_ERRBUF_SIZE];
   u_int i;
   FILE *fd;
+  char name[256];
 
   if (prefs == NULL) {
     prefs = calloc(sizeof(struct ndpi_workflow_prefs), 1); /* No failure here */
@@ -79,23 +98,33 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
 
     workflow = ndpi_workflow_init(prefs, NULL /* pcap handler will be set later */, 0, ndpi_serialization_format_json, g_ctx);
 
-    ndpi_workflow_set_flow_callback(workflow, NULL, NULL); /* No real callback */
+    ndpi_workflow_set_flow_callback(workflow, fn_flow_callback, NULL);
 
     ndpi_set_config(workflow->ndpi_struct, NULL, "log.level", "3");
     ndpi_set_config(workflow->ndpi_struct, "all", "log", "1");
 
-    ndpi_load_domain_suffixes(workflow->ndpi_struct, "public_suffix_list.dat");
-    ndpi_load_categories_dir(workflow->ndpi_struct, "./lists/");
-    ndpi_load_protocols_dir(workflow->ndpi_struct, "./lists/protocols/");
-    ndpi_load_protocols_file(workflow->ndpi_struct, "protos.txt");
-    ndpi_load_categories_file(workflow->ndpi_struct, "categories.txt", NULL);
-    ndpi_load_risk_domain_file(workflow->ndpi_struct, "risky_domains.txt");
-    ndpi_load_malicious_ja4_file(workflow->ndpi_struct, "ja4_fingerprints.csv");
-    ndpi_load_tcp_fingerprint_file(workflow->ndpi_struct, "tcp_fingerprints.csv");
-    ndpi_load_malicious_sha1_file(workflow->ndpi_struct, "sha1_fingerprints.csv");
+    sprintf(name, "%s/public_suffix_list.dat", path);
+    assert(ndpi_load_domain_suffixes(workflow->ndpi_struct, name) >= 0);
+    sprintf(name, "%s/lists/", path);
+    assert(ndpi_load_categories_dir(workflow->ndpi_struct, name) >= 0);
+    sprintf(name, "%s/lists/protocols/", path);
+    assert(ndpi_load_protocols_dir(workflow->ndpi_struct, name) >= 0);
+    sprintf(name, "%s/protos.txt", path);
+    assert(ndpi_load_protocols_file(workflow->ndpi_struct, name) >= 0);
+    sprintf(name, "%s/categories.txt", path);
+    assert(ndpi_load_categories_file(workflow->ndpi_struct, name, NULL) >= 0);
+    sprintf(name, "%s/risky_domains.txt", path);
+    assert(ndpi_load_risk_domain_file(workflow->ndpi_struct, name) >= 0);
+    sprintf(name, "%s/ja4_fingerprints.csv", path);
+    assert(ndpi_load_malicious_ja4_file(workflow->ndpi_struct, name) >= 0);
+    sprintf(name, "%s/tcp_fingerprints.csv", path);
+    assert(ndpi_load_tcp_fingerprint_file(workflow->ndpi_struct, name) >= 0);
+    sprintf(name, "%s/sha1_fingerprints.csv", path);
+    assert(ndpi_load_malicious_sha1_file(workflow->ndpi_struct, name) >= 0);
 
 #ifdef ENABLE_ONLY_SUBCLASSIFICATION
-    ndpi_set_config(workflow->ndpi_struct, NULL, "filename.config", "only_classification.conf");
+    sprintf(name, "%s/config_only_classification.txt", path);
+    assert(ndpi_set_config(workflow->ndpi_struct, NULL, "filename.config", name) == 0);
 #else
 
     ndpi_set_config(workflow->ndpi_struct, NULL, "packets_limit_per_flow", "255");
@@ -122,6 +151,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
     ndpi_set_config(workflow->ndpi_struct, NULL, "flow_risk.all.info", "0");
     ndpi_set_config(workflow->ndpi_struct, NULL, "metadata.tcp_fingerprint_format", "1");
     ndpi_set_config(workflow->ndpi_struct, NULL, "metadata.ndpi_fingerprint_format", "1");
+    ndpi_set_config(workflow->ndpi_struct, "tls", "blocks_analysis", "1");
+
+    addr_dump_path = "/tmp/";
 #endif
 
 #endif /* ENABLE_ONLY_SUBCLASSIFICATION */

src/lib/ndpi_fingerprint.c

@@ -36,7 +36,9 @@
 /* ************************************************************** */
 
 void ndpi_load_tcp_fingerprints(struct ndpi_detection_module_struct *ndpi_str) {
-  if(ndpi_hash_init(&ndpi_str->tcp_fingerprint_hashmap) == 0) {
+
+  if(ndpi_str->tcp_fingerprint_hashmap ||
+     ndpi_hash_init(&ndpi_str->tcp_fingerprint_hashmap) == 0) {
     u_int i;
     
     for(i=0; tcp_fps[i].fingerprint != NULL; i++)

src/lib/ndpi_main.c

@@ -6184,11 +6184,11 @@ int ndpi_load_categories_dir(struct ndpi_detection_module_struct *ndpi_str,
   int num_loaded = 0;
 
   if(!ndpi_str || !dir_path)
-    return(0);
+    return(-1);
 
   dirp = opendir(dir_path);
   if (dirp == NULL)
-    return(0);
+    return(-1);
 
   while((dp = readdir(dirp)) != NULL) {
     char *underscore, *extn;
@@ -6248,11 +6248,11 @@ int ndpi_load_protocols_dir(struct ndpi_detection_module_struct *ndpi_str,
   int num_loaded = 0;
 
   if(!ndpi_str || !dir_path)
-    return(0);
+    return(-1);
 
   dirp = opendir(dir_path);
   if (dirp == NULL)
-    return(0);
+    return(-1);
 
   while((dp = readdir(dirp)) != NULL) {
     char *underscore, *extn;

tests/do.sh.in

@@ -109,13 +109,21 @@ fi
 
 fuzzy_testing() {
     if [ -f ${ABS_BUILDDIR}/fuzz/fuzz_ndpi_reader ]; then
-	cp ${TOP_SRCDIR}/example/protos.txt .
-	cp ${TOP_SRCDIR}/example/categories.txt .
-	cp ${TOP_SRCDIR}/example/risky_domains.txt .
-	cp ${TOP_SRCDIR}/example/ja4_fingerprints.csv .
-	cp ${TOP_SRCDIR}/example/sha1_fingerprints.csv .
-	${ABS_BUILDDIR}/fuzz/fuzz_ndpi_reader -dict=${TOP_SRCDIR}/fuzz/dictionary.dict -max_total_time="${MAX_TOTAL_TIME:-592}" -print_pcs=1 -workers="${FUZZY_WORKERS:-0}" -jobs="${FUZZY_JOBS:-0}" ${ABS_SRCDIR}/tests/cfgs/default/pcap/
-	rm -f protos.txt categories.txt risky_domains.txt ja4_fingerprints.csv sha1_fingerprints.csv
+	cd ${ABS_BUILDDIR}/fuzz
+
+	cp ${TOP_SRCDIR}/example/*.txt .
+	cp ${TOP_SRCDIR}/example/*csv .
+	cp ${TOP_SRCDIR}/example/*.conf .
+	cp ${TOP_SRCDIR}/lists/*.dat .
+	mkdir -p lists
+	find ${TOP_SRCDIR}/lists/*.list ! -name 100_malware.list -exec cp -t lists/ {} +
+	mkdir -p lists/protocols
+	find ${TOP_SRCDIR}/lists/protocols/*.list -exec cp -t lists/protocols/ {} +
+	./fuzz_ndpi_reader -dict=${TOP_SRCDIR}/fuzz/dictionary.dict -max_total_time="${MAX_TOTAL_TIME:-592}" -print_pcs=1 -workers="${FUZZY_WORKERS:-0}" -jobs="${FUZZY_JOBS:-0}" ${ABS_SRCDIR}/tests/cfgs/default/pcap/
+	rm -f *.txt *csv *.conf *.dat
+	rm -rf lists
+
+	cd -
     fi
 }
 

tests/ossfuzz.sh

@@ -54,13 +54,8 @@ cp fuzz/*.zip "$OUT"/
 # Copy options
 cp fuzz/*.options "$OUT"/
 # Copy configuration files
-cp example/protos.txt "$OUT"/
-cp example/categories.txt "$OUT"/
-cp example/risky_domains.txt "$OUT"/
-cp example/ja4_fingerprints.csv "$OUT"/
-cp example/tcp_fingerprints.csv "$OUT"/
-cp example/sha1_fingerprints.csv "$OUT"/
-cp example/config.txt "$OUT"/
+cp example/*.txt "$OUT"/
+cp example/*.csv "$OUT"/
 cp example/*.conf "$OUT"/
 cp lists/public_suffix_list.dat "$OUT"/
 cp fuzz/ipv*_addresses.txt "$OUT"/