diff --git a/example/sha1_fingerprints.csv b/example/sha1_fingerprints.csv index 0df111cf8..4d2be4e2b 100644 --- a/example/sha1_fingerprints.csv +++ b/example/sha1_fingerprints.csv @@ -7,6 +7,8 @@ ################################################################ # # Listingdate,SHA1,Listingreason +#The first entry (and only the first one) is fake. It is used to trigger the risk in the tests +1970-01-01 00:00:00,0DDB34F875632C7E1EC09D75827F82D2336DFEB6, Fake 2022-07-02 07:02:11,3784422d67a6ae0c82d60eeaf3850960a9896cb2,AsyncRAT C&C 2022-07-01 18:28:10,a0d78b3ddb27cd4f110a0292cab800beadaf9bd2,AsyncRAT C&C 2022-07-01 09:44:14,4c319ec8b5a66a256a0137d70cd0f9a76bb23fd7,AsyncRAT C&C @@ -4878,4 +4880,4 @@ 2014-05-04 08:10:31,86d6aade4ba1414a91b1e7fb3cdd7d503692f410,Shylock C&C 2014-05-04 08:10:26,5afc236d1dd00c9c45457b75226b501b815a59c7,Shylock C&C 2014-05-04 08:09:56,b08a4939fb88f375a2757eaddc47b1fb8b554439,Shylock C&C -# END (4871) entries \ No newline at end of file +# END (4871) entries diff --git a/fuzz/fuzz_alg_hw_rsi_outliers_da.cpp b/fuzz/fuzz_alg_hw_rsi_outliers_da.cpp index 6e4f2af17..9e55a22e5 100644 --- a/fuzz/fuzz_alg_hw_rsi_outliers_da.cpp +++ b/fuzz/fuzz_alg_hw_rsi_outliers_da.cpp @@ -74,6 +74,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { ndpi_predict_linear(values, num_values, predict_periods, &prediction); /* Data analysis stuff */ + ndpi_data_print_window_values(a); ndpi_data_average(a); ndpi_data_mean(a); ndpi_data_variance(a); diff --git a/fuzz/fuzz_alg_hw_rsi_outliers_da.options b/fuzz/fuzz_alg_hw_rsi_outliers_da.options new file mode 100644 index 000000000..1c815b33f --- /dev/null +++ b/fuzz/fuzz_alg_hw_rsi_outliers_da.options @@ -0,0 +1,2 @@ +[libfuzzer] +close_fd_mask=1 diff --git a/fuzz/fuzz_common_code.c b/fuzz/fuzz_common_code.c index d13b519ce..b48fa9133 100644 --- a/fuzz/fuzz_common_code.c +++ b/fuzz/fuzz_common_code.c @@ -42,8 +42,6 @@ void fuzz_init_detection_module(struct ndpi_detection_module_struct **ndpi_info_ if(*ndpi_info_mod == NULL) { *ndpi_info_mod = ndpi_init_detection_module(prefs); - NDPI_BITMASK_SET_ALL(all); - ndpi_set_protocol_detection_bitmask2(*ndpi_info_mod, &all); NDPI_BITMASK_SET_ALL(debug_bitmask); ndpi_set_log_level(*ndpi_info_mod, 4); @@ -56,6 +54,9 @@ void fuzz_init_detection_module(struct ndpi_detection_module_struct **ndpi_info_ ndpi_load_malicious_ja3_file(*ndpi_info_mod, "ja3_fingerprints.csv"); ndpi_load_malicious_sha1_file(*ndpi_info_mod, "sha1_fingerprints.csv"); + NDPI_BITMASK_SET_ALL(all); + ndpi_set_protocol_detection_bitmask2(*ndpi_info_mod, &all); + ndpi_finalize_initialization(*ndpi_info_mod); } } diff --git a/fuzz/fuzz_gcrypt_aes.cpp b/fuzz/fuzz_gcrypt_aes.cpp index 1469ab0ce..527a501f9 100644 --- a/fuzz/fuzz_gcrypt_aes.cpp +++ b/fuzz/fuzz_gcrypt_aes.cpp @@ -3,7 +3,7 @@ #include "fuzzer/FuzzedDataProvider.h" #define MBEDTLS_CHECK_RETURN_TYPICAL -#include "../src/lib/third_party/include/gcrypt/aes.h" +#include "gcrypt/aes.h" extern int force_no_aesni; diff --git a/fuzz/fuzz_gcrypt_cipher.cpp b/fuzz/fuzz_gcrypt_cipher.cpp index 270c583ea..703e480a7 100644 --- a/fuzz/fuzz_gcrypt_cipher.cpp +++ b/fuzz/fuzz_gcrypt_cipher.cpp @@ -5,8 +5,8 @@ #define MBEDTLS_CHECK_RETURN_TYPICAL #define MBEDTLS_INTERNAL_VALIDATE_RET( cond, ret ) do { } while( 0 ) -#include "../src/lib/third_party/include/gcrypt/cipher.h" -#include "../src/lib/third_party/include/gcrypt/aes.h" +#include "gcrypt/cipher.h" +#include "gcrypt/aes.h" extern int force_no_aesni; diff --git a/fuzz/fuzz_gcrypt_gcm.cpp b/fuzz/fuzz_gcrypt_gcm.cpp index 3e0b85cca..bf8311dd6 100644 --- a/fuzz/fuzz_gcrypt_gcm.cpp +++ b/fuzz/fuzz_gcrypt_gcm.cpp @@ -5,9 +5,9 @@ #define MBEDTLS_CHECK_RETURN_TYPICAL #define MBEDTLS_INTERNAL_VALIDATE_RET( cond, ret ) do { } while( 0 ) -#include "../src/lib/third_party/include/gcrypt/aes.h" -#include "../src/lib/third_party/include/gcrypt/cipher.h" -#include "../src/lib/third_party/include/gcrypt/gcm.h" +#include "gcrypt/aes.h" +#include "gcrypt/cipher.h" +#include "gcrypt/gcm.h" extern int force_no_aesni; diff --git a/fuzz/fuzz_ndpi_reader.c b/fuzz/fuzz_ndpi_reader.c index 9a3eec625..5f2d980c5 100644 --- a/fuzz/fuzz_ndpi_reader.c +++ b/fuzz/fuzz_ndpi_reader.c @@ -57,9 +57,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { #endif workflow = ndpi_workflow_init(prefs, NULL /* pcap handler will be set later */, 0, ndpi_serialization_format_json); - // enable all protocols - NDPI_BITMASK_SET_ALL(all); - ndpi_set_protocol_detection_bitmask2(workflow->ndpi_struct, &all); NDPI_BITMASK_SET_ALL(debug_bitmask); ndpi_set_log_level(workflow->ndpi_struct, 4); @@ -71,6 +68,10 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { ndpi_load_malicious_ja3_file(workflow->ndpi_struct, "ja3_fingerprints.csv"); ndpi_load_malicious_sha1_file(workflow->ndpi_struct, "sha1_fingerprints.csv"); + // enable all protocols + NDPI_BITMASK_SET_ALL(all); + ndpi_set_protocol_detection_bitmask2(workflow->ndpi_struct, &all); + ndpi_set_detection_preferences(workflow->ndpi_struct, ndpi_pref_enable_tls_block_dissection, 0 /* unused */); ndpi_set_monitoring_state(workflow->ndpi_struct, NDPI_PROTOCOL_STUN, diff --git a/src/lib/ndpi_analyze.c b/src/lib/ndpi_analyze.c index 7cb6b368c..bb0b74fd4 100644 --- a/src/lib/ndpi_analyze.c +++ b/src/lib/ndpi_analyze.c @@ -708,6 +708,7 @@ float ndpi_bin_similarity(struct ndpi_bin *b1, struct ndpi_bin *b2, /* ********************************************************************************* */ +//#define DEBUG_CLUSTER_BINS #define MAX_NUM_CLUSTERS 128 /* @@ -723,7 +724,7 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins, u_int8_t num_clusters, u_int16_t *cluster_ids, struct ndpi_bin *centroids) { u_int16_t i, j, max_iterations = 25, num_iterations, num_moves; - u_int8_t verbose = 0, alloc_centroids = 0; + u_int8_t alloc_centroids = 0; char out_buf[256]; float *bin_score; u_int16_t num_cluster_elems[MAX_NUM_CLUSTERS] = { 0 }; @@ -737,8 +738,9 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins, if(num_clusters > num_bins) num_clusters = num_bins; if(num_clusters > MAX_NUM_CLUSTERS) num_clusters = MAX_NUM_CLUSTERS; - if(verbose) - printf("Distributing %u bins over %u clusters\n", num_bins, num_clusters); +#ifdef DEBUG_CLUSTER_BINS + printf("Distributing %u bins over %u clusters\n", num_bins, num_clusters); +#endif if((bin_score = (float*)ndpi_calloc(num_bins, sizeof(float))) == NULL) return(-2); @@ -764,10 +766,11 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins, cluster_ids[i] = cluster_id; - if(verbose) - printf("Initializing cluster %u for bin %u: %s\n", - cluster_id, i, - ndpi_print_bin(&bins[i], 0, out_buf, sizeof(out_buf))); +#ifdef DEBUG_CLUSTER_BINS + printf("Initializing cluster %u for bin %u: %s\n", + cluster_id, i, + ndpi_print_bin(&bins[i], 0, out_buf, sizeof(out_buf))); +#endif num_cluster_elems[cluster_id]++; } @@ -780,12 +783,12 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins, /* Compute the centroids for each cluster */ memset(bin_score, 0, num_bins*sizeof(float)); - if(verbose) { - printf("\nIteration %u\n", num_iterations); +#ifdef DEBUG_CLUSTER_BINS + printf("\nIteration %u\n", num_iterations); - for(j=0; j best_similarity) { @@ -857,9 +863,10 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins, bin_score[i] = best_similarity; if(cluster_ids[i] != cluster_id) { - if(verbose) - printf("Moved bin %u from cluster %u -> %u [similarity: %f]\n", - i, cluster_ids[i], cluster_id, best_similarity); +#ifdef DEBUG_CLUSTER_BINS + printf("Moved bin %u from cluster %u -> %u [similarity: %f]\n", + i, cluster_ids[i], cluster_id, best_similarity); +#endif num_cluster_elems[cluster_ids[i]]--; num_cluster_elems[cluster_id]++; @@ -872,10 +879,10 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins, if(num_moves == 0) break; - if(verbose) { - for(j=0; jprotos.tls_quic.advertised_alpns) - return; - if(strlen(flow->protos.tls_quic.advertised_alpns) > NDPI_STATICSTRING_LEN("anydesk/") && strncmp(flow->protos.tls_quic.advertised_alpns, "anydesk/", NDPI_STATICSTRING_LEN("anydesk/")) == 0) { #ifdef DEBUG_TLS diff --git a/tests/cfgs/default/pcap/tls_malicious_sha1.pcapng b/tests/cfgs/default/pcap/tls_malicious_sha1.pcapng new file mode 100644 index 000000000..5be4ffa1f Binary files /dev/null and b/tests/cfgs/default/pcap/tls_malicious_sha1.pcapng differ diff --git a/tests/cfgs/default/result/tls_malicious_sha1.pcapng.out b/tests/cfgs/default/result/tls_malicious_sha1.pcapng.out new file mode 100644 index 000000000..9ecc48407 --- /dev/null +++ b/tests/cfgs/default/result/tls_malicious_sha1.pcapng.out @@ -0,0 +1,33 @@ +Guessed flow protos: 0 + +DPI Packets (TCP): 8 (8.00 pkts/flow) +Confidence DPI : 1 (flows) +Num dissector calls: 1 (1.00 diss/flow) +LRU cache ookla: 0/0/0 (insert/search/found) +LRU cache bittorrent: 0/0/0 (insert/search/found) +LRU cache zoom: 0/0/0 (insert/search/found) +LRU cache stun: 0/0/0 (insert/search/found) +LRU cache tls_cert: 0/2/0 (insert/search/found) +LRU cache mining: 0/0/0 (insert/search/found) +LRU cache msteams: 0/0/0 (insert/search/found) +LRU cache stun_zoom: 0/0/0 (insert/search/found) +Automa host: 2/0 (search/found) +Automa domain: 2/0 (search/found) +Automa tls cert: 1/0 (search/found) +Automa risk mask: 0/0 (search/found) +Automa common alpns: 2/2 (search/found) +Patricia risk mask: 0/0 (search/found) +Patricia risk mask IPv6: 0/0 (search/found) +Patricia risk: 0/0 (search/found) +Patricia risk IPv6: 1/0 (search/found) +Patricia protocols: 0/0 (search/found) +Patricia protocols IPv6: 1/1 (search/found) + +TLS 22 7204 1 + +JA3 Host Stats: + IP Address # JA3C + 1 2001:b07:a3d:c112:9726:f643:a838:b0c4 1 + + + 1 TCP [2001:b07:a3d:c112:9726:f643:a838:b0c4]:40294 <-> [2a00:1450:4002:414::2013]:443 [proto: 91/TLS][IP: 126/Google][Encrypted][Confidence: DPI][DPI packets: 8][cat: Web/5][12 pkts/1574 bytes <-> 10 pkts/5630 bytes][Goodput ratio: 34/85][0.12 sec][Hostname/SNI: www.prbtest.dev][(Advertised) ALPNs: h2;http/1.1][(Negotiated) ALPN: h2][bytes ratio: -0.563 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 4/6 23/20 7/7][Pkt Len c2s/s2c min/avg/max/stddev: 86/86 131/563 316/2502 62/920][Risk: ** Malicious SSL Cert/SHA1 Fingerp. **][Risk Score: 50][Risk Info: 0DDB34F875632C7E1EC09D75827F82D2336DFEB6][TLSv1.2][JA3C: 00bcd759cb8ad485fdbf1e7a0c5b94b4][ServerNames: www.prbtest.dev][JA3S: e2bc06b738d7e5d2b0cec5d2196b1d80][Issuer: C=US, O=Google Trust Services LLC, CN=GTS CA 1D4][Subject: CN=www.prbtest.dev][Certificate SHA-1: 0D:DB:34:F8:75:63:2C:7E:1E:C0:9D:75:82:7F:82:D2:33:6D:FE:B6][Firefox][Validity: 2023-11-28 12:50:11 - 2024-02-26 13:39:22][Cipher: TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256][Plen Bins: 16,51,8,0,0,0,0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,16]