Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add infrastructure for explicit support of First Packet Classification #2488

Merged
merged 1 commit into from
Jul 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
18 changes: 18 additions & 0 deletions example/ndpiReader.c
Original file line number Diff line number Diff line change
Expand Up @@ -1747,6 +1747,24 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
flow->detected_protocol) ? "Encrypted" : "ClearText");

fprintf(out, "[Confidence: %s]", ndpi_confidence_get_name(flow->confidence));

if(flow->fpc.master_protocol == NDPI_PROTOCOL_UNKNOWN) {
fprintf(out, "[FPC: %u/%s, ",
flow->fpc.app_protocol,
ndpi_get_proto_name(ndpi_thread_info[thread_id].workflow->ndpi_struct,
flow->fpc.app_protocol));
} else {
fprintf(out, "[FPC: %u.%u/%s.%s, ",
flow->fpc.master_protocol,
flow->fpc.app_protocol,
ndpi_get_proto_name(ndpi_thread_info[thread_id].workflow->ndpi_struct,
flow->fpc.master_protocol),
ndpi_get_proto_name(ndpi_thread_info[thread_id].workflow->ndpi_struct,
flow->fpc.app_protocol));
}
fprintf(out, "Confidence: %s]",
ndpi_fpc_confidence_get_name(flow->fpc.confidence));

/* If someone wants to have the num_dissector_calls variable per flow, he can print it here.
Disabled by default to avoid too many diffs in the unit tests...
*/
Expand Down
3 changes: 3 additions & 0 deletions example/reader_util.c
Original file line number Diff line number Diff line change
Expand Up @@ -1055,6 +1055,9 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
flow->risk_str = ndpi_strdup(s);

flow->confidence = flow->ndpi_flow->confidence;

flow->fpc = flow->ndpi_flow->fpc;

flow->num_dissector_calls = flow->ndpi_flow->num_dissector_calls;

ndpi_snprintf(flow->host_server_name, sizeof(flow->host_server_name), "%s",
Expand Down
1 change: 1 addition & 0 deletions example/reader_util.h
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,7 @@ typedef struct ndpi_flow_info {
// result only, not used for flow identification
ndpi_protocol detected_protocol;
ndpi_confidence_t confidence;
struct ndpi_fpc_info fpc;
u_int16_t num_dissector_calls;
u_int16_t dpi_packets;

Expand Down
1 change: 1 addition & 0 deletions fuzz/fuzz_config.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -519,6 +519,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
ndpi_risk2score(static_cast<ndpi_risk_enum>(fuzzed_data.ConsumeIntegral<u_int64_t>()), &unused1, &unused2);
ndpi_http_method2str(static_cast<ndpi_http_method>(fuzzed_data.ConsumeIntegral<u_int8_t>()));
ndpi_confidence_get_name(static_cast<ndpi_confidence_t>(fuzzed_data.ConsumeIntegral<u_int8_t>()));
ndpi_fpc_confidence_get_name(static_cast<ndpi_fpc_confidence_t>(fuzzed_data.ConsumeIntegral<u_int8_t>()));
ndpi_get_proto_breed_name(static_cast<ndpi_protocol_breed_t>(fuzzed_data.ConsumeIntegral<u_int8_t>()));
ndpi_get_l4_proto_name(static_cast<ndpi_l4_proto_info>(fuzzed_data.ConsumeIntegral<u_int8_t>()));

Expand Down
9 changes: 9 additions & 0 deletions src/include/ndpi_api.h
Original file line number Diff line number Diff line change
Expand Up @@ -627,6 +627,15 @@ extern "C" {
*/
const char* ndpi_confidence_get_name(ndpi_confidence_t confidence);

/**
* Get FPC confidence as string
*
* @par confidence = the confidence value
* @return the string name of the confidence result
*
*/
const char* ndpi_fpc_confidence_get_name(ndpi_fpc_confidence_t fpc_confidence);

/**
* Set protocol category string
*
Expand Down
30 changes: 28 additions & 2 deletions src/include/ndpi_typedefs.h
Original file line number Diff line number Diff line change
Expand Up @@ -995,6 +995,23 @@ typedef enum {
NDPI_CONFIDENCE_MAX,
} ndpi_confidence_t;

typedef enum {
NDPI_FPC_CONFIDENCE_UNKNOWN = 0, /* Unknown First Packet Classification */
NDPI_FPC_CONFIDENCE_IP, /* FPC based on IP address */
NDPI_FPC_CONFIDENCE_DNS, /* FPC based on DNS information */

/*
IMPORTANT

Please keep in sync with
ndpi_fpc_confidence_get_name()
in ndpi_main.c
*/

/* Last one */
NDPI_FPC_CONFIDENCE_MAX,
} ndpi_fpc_confidence_t;

typedef enum {
NDPI_PROTOCOL_SAFE = 0, /* Surely doesn't provide risks for the network. (e.g., a news site) */
NDPI_PROTOCOL_ACCEPTABLE, /* Probably doesn't provide risks, but could be malicious (e.g., Dropbox) */
Expand Down Expand Up @@ -1119,6 +1136,12 @@ typedef struct _ndpi_automa {

typedef void ndpi_str_hash;

struct ndpi_fpc_info {
u_int16_t master_protocol;
u_int16_t app_protocol;
ndpi_fpc_confidence_t confidence;
};

typedef struct ndpi_proto {
/*
Note
Expand Down Expand Up @@ -1195,6 +1218,9 @@ struct ndpi_flow_struct {
u_int16_t num_dissector_calls;
ndpi_confidence_t confidence; /* ndpi_confidence_t */

/* First Packet Classification info */
struct ndpi_fpc_info fpc;

/*
if ndpi_struct->direction_detect_disable == 1
tcp sequence number connection tracking
Expand Down Expand Up @@ -1510,8 +1536,8 @@ struct ndpi_flow_struct {
_Static_assert(sizeof(((struct ndpi_flow_struct *)0)->protos) <= 256,
"Size of the struct member protocols increased to more than 256 bytes, "
"please check if this change is necessary.");
_Static_assert(sizeof(struct ndpi_flow_struct) <= 1112,
"Size of the flow struct increased to more than 1112 bytes, "
_Static_assert(sizeof(struct ndpi_flow_struct) <= 1120,
"Size of the flow struct increased to more than 1120 bytes, "
"please check if this change is necessary.");
#endif
#endif
Expand Down
48 changes: 48 additions & 0 deletions src/lib/ndpi_main.c
Original file line number Diff line number Diff line change
Expand Up @@ -8287,6 +8287,33 @@ static int ndpi_do_guess(struct ndpi_detection_module_struct *ndpi_str, struct n

/* ********************************************************************************* */

static void fpc_update(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow,
u_int16_t fpc_master, u_int16_t fpc_app,
ndpi_fpc_confidence_t fpc_confidence)
{
NDPI_LOG_DBG(ndpi_str, "FPC %d.%d/%s -> %d.%d/%s\n",
flow->fpc.master_protocol, flow->fpc.app_protocol,
ndpi_fpc_confidence_get_name(flow->fpc.confidence),
fpc_master, fpc_app,
ndpi_fpc_confidence_get_name(fpc_confidence));
flow->fpc.master_protocol = fpc_master;
flow->fpc.app_protocol = fpc_app;
flow->fpc.confidence = fpc_confidence;
}

/* ********************************************************************************* */

static void fpc_check_ip(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow)
{
if(flow->guessed_protocol_id_by_ip != NDPI_PROTOCOL_UNKNOWN)
fpc_update(ndpi_str, flow, NDPI_PROTOCOL_UNKNOWN,
flow->guessed_protocol_id_by_ip, NDPI_FPC_CONFIDENCE_IP);
}

/* ********************************************************************************* */

static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow,
const unsigned char *packet_data,
Expand Down Expand Up @@ -8421,6 +8448,8 @@ static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detectio

if(ndpi_do_guess(ndpi_str, flow, &ret) == -1)
return(ret);

fpc_check_ip(ndpi_str, flow);
}

num_calls = ndpi_check_flow_func(ndpi_str, flow, &ndpi_selection_packet);
Expand Down Expand Up @@ -9383,6 +9412,25 @@ const char *ndpi_confidence_get_name(ndpi_confidence_t confidence)

/* ****************************************************** */

const char *ndpi_fpc_confidence_get_name(ndpi_fpc_confidence_t fpc_confidence)
{
switch(fpc_confidence) {
case NDPI_FPC_CONFIDENCE_UNKNOWN:
return "Unknown";

case NDPI_FPC_CONFIDENCE_IP:
return "IP address";

case NDPI_FPC_CONFIDENCE_DNS:
return "DNS";

default:
return "Invalid"; /* Out of sync with ndpi_fpc_confidence_t definition */
}
}

/* ****************************************************** */

const char *ndpi_category_get_name(struct ndpi_detection_module_struct *ndpi_str,
ndpi_protocol_category_t category) {
if((!ndpi_str) || (category >= NDPI_PROTOCOL_NUM_CATEGORIES)) {
Expand Down
12 changes: 6 additions & 6 deletions tests/cfgs/caches_cfg/result/ookla.pcap.out
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,9 @@ JA3 Host Stats:
1 192.168.1.128 2


1 TCP 192.168.1.128:35830 <-> 89.96.108.170:8080 [proto: 91/TLS][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 6][cat: Web/5][21 pkts/21216 bytes <-> 8 pkts/1950 bytes][Goodput ratio: 93/72][0.32 sec][Hostname/SNI: spd-pub-mi-01-01.fastwebnet.it][(Advertised) ALPNs: h2;http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2][bytes ratio: 0.832 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 17/61 274/280 62/109][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 1010/244 1514/387 612/138][Risk: ** Known Proto on Non Std Port **][Risk Score: 50][Risk Info: Expected on port 443][TLSv1.3][JA3C: c279b0189edb9269da7bc43dea5e0c36][JA4: t13d1714h2_5b57614c22b0_8f66f9ee9c6c][JA3S: fcb2d4d0991292272fcb1e464eedfd43][Firefox][Cipher: TLS_AES_128_GCM_SHA256][Plen Bins: 0,0,4,0,0,0,0,4,9,0,9,0,0,0,0,0,4,0,0,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,55,0,0]
2 TCP 192.168.1.128:48854 <-> 104.16.209.12:443 [proto: 91.191/TLS.Ookla][IP: 220/Cloudflare][Encrypted][Confidence: DPI][DPI packets: 6][cat: Network/14][8 pkts/1620 bytes <-> 6 pkts/3818 bytes][Goodput ratio: 67/89][0.06 sec][Hostname/SNI: www.speedtest.net][(Advertised) ALPNs: h2;http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2][bytes ratio: -0.404 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 7/5 18/15 7/6][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 202/636 583/1514 181/646][TLSv1.3][JA3C: 579ccef312d18482fc42e2b822ca2430][JA4: t13d1715h2_5b57614c22b0_3d5424432f57][JA3S: eb1d94daa7e0344597e756a1fb6e7054][Firefox][Cipher: TLS_AES_128_GCM_SHA256][PLAIN TEXT (@oTAgOeedtest.net)][Plen Bins: 0,0,14,0,0,14,0,0,0,0,14,0,0,0,0,0,28,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,28,0,0]
3 TCP 192.168.1.7:51207 <-> 46.44.253.187:80 [proto: 7.191/HTTP.Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 6][cat: Network/14][12 pkts/2238 bytes <-> 8 pkts/2082 bytes][Goodput ratio: 64/74][5.33 sec][Hostname/SNI: massarosa-1.speedtest.welcomeitalia.it][bytes ratio: 0.036 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/4 528/47 5005/84 1493/28][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 186/260 430/523 168/194][URL: massarosa-1.speedtest.welcomeitalia.it/crossdomain.xml][StatusCode: 200][Content-Type: application/xml][Server: Apache/2.2.22 (Ubuntu)][User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/602.4.8 (KHTML, like Gecko) Version/10.0.3 Safari/602.4.8][Risk: ** HTTP Obsolete Server **][Risk Score: 50][Risk Info: Obsolete Apache server 2.2.22][PLAIN TEXT (GET /crossdomain.xml HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,12,75,0,0,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
4 TCP 192.168.1.192:51156 <-> 89.96.108.170:8080 [proto: 131/HTTP_Proxy][IP: 0/Unknown][ClearText][Confidence: Match by port][DPI packets: 10][cat: Web/5][6 pkts/591 bytes <-> 4 pkts/1784 bytes][Goodput ratio: 32/85][0.05 sec][bytes ratio: -0.502 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 9/10 15/20 6/8][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 98/446 143/1514 31/617][Risk: ** Fully encrypted flow **][Risk Score: 50][PLAIN TEXT (gKRZvA)][Plen Bins: 0,40,40,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20,0,0]
5 TCP 192.168.1.7:51215 <-> 46.44.253.187:8080 [proto: 191/Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 6][cat: Network/14][19 pkts/1421 bytes <-> 11 pkts/920 bytes][Goodput ratio: 11/20][0.80 sec][bytes ratio: 0.214 (Upload)][IAT c2s/s2c min/avg/max/stddev: 26/0 44/75 103/137 23/41][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 75/84 85/100 9/8][PLAIN TEXT ( 6HELLO 2.4 2016)][Plen Bins: 94,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
6 TCP 192.168.1.192:37790 <-> 185.157.229.246:8080 [proto: 191/Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 6][cat: Network/14][6 pkts/454 bytes <-> 4 pkts/317 bytes][Goodput ratio: 11/14][0.06 sec][bytes ratio: 0.178 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 12/5 46/9 17/4][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 76/79 106/108 14/17][PLAIN TEXT (HELLO 2.9 )][Plen Bins: 50,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
1 TCP 192.168.1.128:35830 <-> 89.96.108.170:8080 [proto: 91/TLS][IP: 0/Unknown][Encrypted][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 6][cat: Web/5][21 pkts/21216 bytes <-> 8 pkts/1950 bytes][Goodput ratio: 93/72][0.32 sec][Hostname/SNI: spd-pub-mi-01-01.fastwebnet.it][(Advertised) ALPNs: h2;http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2][bytes ratio: 0.832 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 17/61 274/280 62/109][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 1010/244 1514/387 612/138][Risk: ** Known Proto on Non Std Port **][Risk Score: 50][Risk Info: Expected on port 443][TLSv1.3][JA3C: c279b0189edb9269da7bc43dea5e0c36][JA4: t13d1714h2_5b57614c22b0_8f66f9ee9c6c][JA3S: fcb2d4d0991292272fcb1e464eedfd43][Firefox][Cipher: TLS_AES_128_GCM_SHA256][Plen Bins: 0,0,4,0,0,0,0,4,9,0,9,0,0,0,0,0,4,0,0,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,55,0,0]
2 TCP 192.168.1.128:48854 <-> 104.16.209.12:443 [proto: 91.191/TLS.Ookla][IP: 220/Cloudflare][Encrypted][Confidence: DPI][FPC: 220/Cloudflare, Confidence: IP address][DPI packets: 6][cat: Network/14][8 pkts/1620 bytes <-> 6 pkts/3818 bytes][Goodput ratio: 67/89][0.06 sec][Hostname/SNI: www.speedtest.net][(Advertised) ALPNs: h2;http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2][bytes ratio: -0.404 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 7/5 18/15 7/6][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 202/636 583/1514 181/646][TLSv1.3][JA3C: 579ccef312d18482fc42e2b822ca2430][JA4: t13d1715h2_5b57614c22b0_3d5424432f57][JA3S: eb1d94daa7e0344597e756a1fb6e7054][Firefox][Cipher: TLS_AES_128_GCM_SHA256][PLAIN TEXT (@oTAgOeedtest.net)][Plen Bins: 0,0,14,0,0,14,0,0,0,0,14,0,0,0,0,0,28,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,28,0,0]
3 TCP 192.168.1.7:51207 <-> 46.44.253.187:80 [proto: 7.191/HTTP.Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 6][cat: Network/14][12 pkts/2238 bytes <-> 8 pkts/2082 bytes][Goodput ratio: 64/74][5.33 sec][Hostname/SNI: massarosa-1.speedtest.welcomeitalia.it][bytes ratio: 0.036 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/4 528/47 5005/84 1493/28][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 186/260 430/523 168/194][URL: massarosa-1.speedtest.welcomeitalia.it/crossdomain.xml][StatusCode: 200][Content-Type: application/xml][Server: Apache/2.2.22 (Ubuntu)][User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/602.4.8 (KHTML, like Gecko) Version/10.0.3 Safari/602.4.8][Risk: ** HTTP Obsolete Server **][Risk Score: 50][Risk Info: Obsolete Apache server 2.2.22][PLAIN TEXT (GET /crossdomain.xml HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,12,75,0,0,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
4 TCP 192.168.1.192:51156 <-> 89.96.108.170:8080 [proto: 131/HTTP_Proxy][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 10][cat: Web/5][6 pkts/591 bytes <-> 4 pkts/1784 bytes][Goodput ratio: 32/85][0.05 sec][bytes ratio: -0.502 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 9/10 15/20 6/8][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 98/446 143/1514 31/617][Risk: ** Fully encrypted flow **][Risk Score: 50][PLAIN TEXT (gKRZvA)][Plen Bins: 0,40,40,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20,0,0]
5 TCP 192.168.1.7:51215 <-> 46.44.253.187:8080 [proto: 191/Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 6][cat: Network/14][19 pkts/1421 bytes <-> 11 pkts/920 bytes][Goodput ratio: 11/20][0.80 sec][bytes ratio: 0.214 (Upload)][IAT c2s/s2c min/avg/max/stddev: 26/0 44/75 103/137 23/41][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 75/84 85/100 9/8][PLAIN TEXT ( 6HELLO 2.4 2016)][Plen Bins: 94,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
6 TCP 192.168.1.192:37790 <-> 185.157.229.246:8080 [proto: 191/Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 6][cat: Network/14][6 pkts/454 bytes <-> 4 pkts/317 bytes][Goodput ratio: 11/14][0.06 sec][bytes ratio: 0.178 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 12/5 46/9 17/4][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 76/79 106/108 14/17][PLAIN TEXT (HELLO 2.9 )][Plen Bins: 50,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
Loading
Loading