Skip to content

Commit 5bb5bec

Browse files
IvanNardiIvanNardi
authored
Remove struct ndpi_id_struct (#1427)
Remove the last uses of `struct ndpi_id_struct`. That code is not really used and it has not been updated for a very long time: see #1279 for details. Correlation among flows is achieved via LRU caches. This change allows to further reduce memory consumption (see also 91bb77a). At nDPI 4.0 (more precisly, at a6b10cf, because memory stats were wrong until that commit): ``` nDPI Memory statistics: nDPI Memory (once): 221.15 KB Flow Memory (per flow): 2.94 KB ``` Now: ``` nDPI Memory statistics: nDPI Memory (once): 235.27 KB Flow Memory (per flow): 688 B <-------- ``` i.e. memory usage per flow has been reduced by 77%. Close #1279
1 parent 1fdcddb commit 5bb5bec

File tree

14 files changed

+31
-843
lines changed

14 files changed

+31
-843
lines changed

‎example/ndpiReader.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2911,7 +2911,7 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us
29112911
if(!quiet_mode) {
29122912
printf("\nnDPI Memory statistics:\n");
29132913
printf("\tnDPI Memory (once): %-13s\n", formatBytes(ndpi_get_ndpi_detection_module_size(), buf, sizeof(buf)));
2914-
printf("\tFlow Memory (per flow): %-13s\n", formatBytes( ndpi_detection_get_sizeof_ndpi_flow_struct() + 2*ndpi_detection_get_sizeof_ndpi_id_struct(), buf, sizeof(buf)));
2914+
printf("\tFlow Memory (per flow): %-13s\n", formatBytes( ndpi_detection_get_sizeof_ndpi_flow_struct(), buf, sizeof(buf)));
29152915
printf("\tActual Memory: %-13s\n", formatBytes(current_ndpi_memory, buf, sizeof(buf)));
29162916
printf("\tPeak Memory: %-13s\n", formatBytes(max_ndpi_memory, buf, sizeof(buf)));
29172917
printf("\tSetup Time: %lu msec\n", (unsigned long)(setup_time_usec/1000));

‎example/ndpiSimpleIntegration.c

Lines changed: 1 addition & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -85,8 +85,6 @@ struct nDPI_flow_info {
8585
struct ndpi_proto guessed_protocol;
8686

8787
struct ndpi_flow_struct * ndpi_flow;
88-
struct ndpi_id_struct * ndpi_src;
89-
struct ndpi_id_struct * ndpi_dst;
9088
};
9189

9290
struct nDPI_workflow {
@@ -190,8 +188,6 @@ static void ndpi_flow_info_freer(void * const node)
190188
{
191189
struct nDPI_flow_info * const flow = (struct nDPI_flow_info *)node;
192190

193-
ndpi_free(flow->ndpi_dst);
194-
ndpi_free(flow->ndpi_src);
195191
ndpi_flow_free(flow->ndpi_flow);
196192
ndpi_free(flow);
197193
}
@@ -505,10 +501,6 @@ static void ndpi_process_packet(uint8_t * const args,
505501
void * tree_result;
506502
struct nDPI_flow_info * flow_to_process;
507503

508-
int direction_changed = 0;
509-
struct ndpi_id_struct * ndpi_src;
510-
struct ndpi_id_struct * ndpi_dst;
511-
512504
const struct ndpi_ethhdr * ethernet;
513505
const struct ndpi_iphdr * ip;
514506
struct ndpi_ipv6hdr * ip6;
@@ -753,9 +745,6 @@ static void ndpi_process_packet(uint8_t * const args,
753745
flow.dst_port = orig_src_port;
754746

755747
tree_result = ndpi_tfind(&flow, &workflow->ndpi_flows_active[hashed_index], ndpi_workflow_node_cmp);
756-
if (tree_result != NULL) {
757-
direction_changed = 1;
758-
}
759748

760749
flow.ip_tuple.u32.src[0] = orig_src_ip[0];
761750
flow.ip_tuple.u32.src[1] = orig_src_ip[1];
@@ -800,40 +789,15 @@ static void ndpi_process_packet(uint8_t * const args,
800789
}
801790
memset(flow_to_process->ndpi_flow, 0, SIZEOF_FLOW_STRUCT);
802791

803-
flow_to_process->ndpi_src = (struct ndpi_id_struct *)ndpi_calloc(1, SIZEOF_ID_STRUCT);
804-
if (flow_to_process->ndpi_src == NULL) {
805-
fprintf(stderr, "[%8llu, %d, %4u] Not enough memory for src id struct\n",
806-
workflow->packets_captured, reader_thread->array_index, flow_to_process->flow_id);
807-
return;
808-
}
809-
810-
flow_to_process->ndpi_dst = (struct ndpi_id_struct *)ndpi_calloc(1, SIZEOF_ID_STRUCT);
811-
if (flow_to_process->ndpi_dst == NULL) {
812-
fprintf(stderr, "[%8llu, %d, %4u] Not enough memory for dst id struct\n",
813-
workflow->packets_captured, reader_thread->array_index, flow_to_process->flow_id);
814-
return;
815-
}
816-
817792
printf("[%8llu, %d, %4u] new %sflow\n", workflow->packets_captured, thread_index,
818793
flow_to_process->flow_id,
819794
(flow_to_process->is_midstream_flow != 0 ? "midstream-" : ""));
820795
if (ndpi_tsearch(flow_to_process, &workflow->ndpi_flows_active[hashed_index], ndpi_workflow_node_cmp) == NULL) {
821796
/* Possible Leak, but should not happen as we'd abort earlier. */
822797
return;
823798
}
824-
825-
ndpi_src = flow_to_process->ndpi_src;
826-
ndpi_dst = flow_to_process->ndpi_dst;
827799
} else {
828800
flow_to_process = *(struct nDPI_flow_info **)tree_result;
829-
830-
if (direction_changed != 0) {
831-
ndpi_src = flow_to_process->ndpi_dst;
832-
ndpi_dst = flow_to_process->ndpi_src;
833-
} else {
834-
ndpi_src = flow_to_process->ndpi_src;
835-
ndpi_dst = flow_to_process->ndpi_dst;
836-
}
837801
}
838802

839803
flow_to_process->packets_processed++;
@@ -884,7 +848,7 @@ static void ndpi_process_packet(uint8_t * const args,
884848
flow_to_process->detected_l7_protocol =
885849
ndpi_detection_process_packet(workflow->ndpi_struct, flow_to_process->ndpi_flow,
886850
ip != NULL ? (uint8_t *)ip : (uint8_t *)ip6,
887-
ip_size, time_ms, ndpi_src, ndpi_dst);
851+
ip_size, time_ms);
888852

889853
if (ndpi_is_protocol_detected(workflow->ndpi_struct,
890854
flow_to_process->detected_l7_protocol) != 0 &&

‎example/reader_util.c

Lines changed: 8 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -300,8 +300,6 @@ void ndpi_report_payload_stats() {
300300

301301
void ndpi_free_flow_info_half(struct ndpi_flow_info *flow) {
302302
if(flow->ndpi_flow) { ndpi_flow_free(flow->ndpi_flow); flow->ndpi_flow = NULL; }
303-
if(flow->src_id) { ndpi_free(flow->src_id); flow->src_id = NULL; }
304-
if(flow->dst_id) { ndpi_free(flow->dst_id); flow->dst_id = NULL; }
305303
}
306304

307305
/* ***************************************************** */
@@ -699,8 +697,6 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
699697
struct ndpi_tcphdr **tcph,
700698
struct ndpi_udphdr **udph,
701699
u_int16_t *sport, u_int16_t *dport,
702-
struct ndpi_id_struct **src,
703-
struct ndpi_id_struct **dst,
704700
u_int8_t *proto,
705701
u_int8_t **payload,
706702
u_int16_t *payload_len,
@@ -878,30 +874,6 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
878874
} else
879875
memset(newflow->ndpi_flow, 0, SIZEOF_FLOW_STRUCT);
880876

881-
if((newflow->src_id = ndpi_malloc(SIZEOF_ID_STRUCT)) == NULL) {
882-
LOG(NDPI_LOG_ERROR, "[NDPI] %s(3): not enough memory\n", __FUNCTION__);
883-
#ifdef DIRECTION_BINS
884-
ndpi_free_bin(&newflow->payload_len_bin_src2dst), ndpi_free_bin(&newflow->payload_len_bin_dst2src);
885-
#else
886-
ndpi_free_bin(&newflow->payload_len_bin);
887-
#endif
888-
ndpi_free(newflow);
889-
return(NULL);
890-
} else
891-
memset(newflow->src_id, 0, SIZEOF_ID_STRUCT);
892-
893-
if((newflow->dst_id = ndpi_malloc(SIZEOF_ID_STRUCT)) == NULL) {
894-
LOG(NDPI_LOG_ERROR, "[NDPI] %s(4): not enough memory\n", __FUNCTION__);
895-
#ifdef DIRECTION_BINS
896-
ndpi_free_bin(&newflow->payload_len_bin_src2dst), ndpi_free_bin(&newflow->payload_len_bin_dst2src);
897-
#else
898-
ndpi_free_bin(&newflow->payload_len_bin);
899-
#endif
900-
ndpi_free(newflow);
901-
return(NULL);
902-
} else
903-
memset(newflow->dst_id, 0, SIZEOF_ID_STRUCT);
904-
905877
ndpi_tsearch(newflow, &workflow->ndpi_flows_root[idx], ndpi_workflow_node_cmp); /* Add */
906878
workflow->stats.ndpi_flow_count++;
907879
if(*proto == IPPROTO_TCP)
@@ -911,8 +883,6 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
911883
else
912884
workflow->stats.flow_count[2]++;
913885

914-
*src = newflow->src_id, *dst = newflow->dst_id;
915-
916886
if(enable_flow_stats) {
917887
newflow->entropy = ndpi_calloc(1, sizeof(struct ndpi_entropy));
918888
newflow->last_entropy = ndpi_calloc(1, sizeof(struct ndpi_entropy));
@@ -939,19 +909,19 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
939909
&& rflow->src_port == htons(*sport)
940910
&& rflow->dst_port == htons(*dport)
941911
)
942-
*src = rflow->dst_id, *dst = rflow->src_id, *src_to_dst_direction = 0, rflow->bidirectional = 1;
912+
*src_to_dst_direction = 0, rflow->bidirectional = 1;
943913
else
944-
*src = rflow->src_id, *dst = rflow->dst_id, *src_to_dst_direction = 1;
914+
*src_to_dst_direction = 1;
945915
}
946916
else {
947917
if(rflow->src_ip == iph->saddr
948918
&& rflow->dst_ip == iph->daddr
949919
&& rflow->src_port == htons(*sport)
950920
&& rflow->dst_port == htons(*dport)
951921
)
952-
*src = rflow->src_id, *dst = rflow->dst_id, *src_to_dst_direction = 1;
922+
*src_to_dst_direction = 1;
953923
else
954-
*src = rflow->dst_id, *dst = rflow->src_id, *src_to_dst_direction = 0, rflow->bidirectional = 1;
924+
*src_to_dst_direction = 0, rflow->bidirectional = 1;
955925
}
956926
if(enable_flow_stats) {
957927
if(src_to_dst_direction) {
@@ -997,8 +967,6 @@ static struct ndpi_flow_info *get_ndpi_flow_info6(struct ndpi_workflow * workflo
997967
struct ndpi_tcphdr **tcph,
998968
struct ndpi_udphdr **udph,
999969
u_int16_t *sport, u_int16_t *dport,
1000-
struct ndpi_id_struct **src,
1001-
struct ndpi_id_struct **dst,
1002970
u_int8_t *proto,
1003971
u_int8_t **payload,
1004972
u_int16_t *payload_len,
@@ -1024,7 +992,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info6(struct ndpi_workflow * workflo
1024992
&iph, iph6, ip_offset, ipsize,
1025993
ip_len, l4ptr - (const u_int8_t *)iph6,
1026994
tcph, udph, sport, dport,
1027-
src, dst, proto, payload,
995+
proto, payload,
1028996
payload_len, src_to_dst_direction, when));
1029997
}
1030998

@@ -1347,7 +1315,6 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
13471315
pkt_timeval when,
13481316
ndpi_risk *flow_risk,
13491317
FILE * csv_fp) {
1350-
struct ndpi_id_struct *src, *dst;
13511318
struct ndpi_flow_info *flow = NULL;
13521319
struct ndpi_flow_struct *ndpi_flow = NULL;
13531320
u_int8_t proto;
@@ -1369,13 +1336,13 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
13691336
ntohs(iph->tot_len) - (iph->ihl * 4),
13701337
iph->ihl * 4,
13711338
&tcph, &udph, &sport, &dport,
1372-
&src, &dst, &proto,
1339+
&proto,
13731340
&payload, &payload_len, &src_to_dst_direction, when);
13741341
else
13751342
flow = get_ndpi_flow_info6(workflow, vlan_id,
13761343
tunnel_type, iph6, ip_offset, ipsize,
13771344
&tcph, &udph, &sport, &dport,
1378-
&src, &dst, &proto,
1345+
&proto,
13791346
&payload, &payload_len, &src_to_dst_direction, when);
13801347

13811348
if(flow != NULL) {
@@ -1563,7 +1530,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
15631530

15641531
flow->detected_protocol = ndpi_detection_process_packet(workflow->ndpi_struct, ndpi_flow,
15651532
iph ? (uint8_t *)iph : (uint8_t *)iph6,
1566-
ipsize, time_ms, src, dst);
1533+
ipsize, time_ms);
15671534

15681535
if(enough_packets || (flow->detected_protocol.app_protocol != NDPI_PROTOCOL_UNKNOWN)) {
15691536
if((!enough_packets)

‎fuzz/fuzz_process_packet.c

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,6 @@
44
#include <stdio.h>
55

66
struct ndpi_detection_module_struct *ndpi_info_mod = NULL;
7-
struct ndpi_id_struct *src;
8-
struct ndpi_id_struct *dst;
97

108
int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
119
uint8_t protocol_was_guessed;
@@ -16,18 +14,14 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
1614
NDPI_BITMASK_SET_ALL(all);
1715
NDPI_BITMASK_SET_ALL(debug_bitmask);
1816
ndpi_set_protocol_detection_bitmask2(ndpi_info_mod, &all);
19-
src = ndpi_malloc(SIZEOF_ID_STRUCT);
20-
dst = ndpi_malloc(SIZEOF_ID_STRUCT);
2117
ndpi_set_log_level(ndpi_info_mod, 4);
2218
ndpi_set_debug_bitmask(ndpi_info_mod, debug_bitmask);
2319
ndpi_finalize_initialization(ndpi_info_mod);
2420
}
2521

2622
struct ndpi_flow_struct *ndpi_flow = ndpi_flow_malloc(SIZEOF_FLOW_STRUCT);
2723
memset(ndpi_flow, 0, SIZEOF_FLOW_STRUCT);
28-
memset(src, 0, SIZEOF_ID_STRUCT);
29-
memset(dst, 0, SIZEOF_ID_STRUCT);
30-
ndpi_detection_process_packet(ndpi_info_mod, ndpi_flow, Data, Size, 0, src, dst);
24+
ndpi_detection_process_packet(ndpi_info_mod, ndpi_flow, Data, Size, 0);
3125
ndpi_detection_giveup(ndpi_info_mod, ndpi_flow, 1, &protocol_was_guessed);
3226
ndpi_free_flow(ndpi_flow);
3327

‎src/include/ndpi_api.h.in

Lines changed: 2 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ extern "C" {
3636
*/
3737
#define NDPI_API_VERSION @NDPI_API_VERSION@
3838

39-
#define SIZEOF_ID_STRUCT ( sizeof(struct ndpi_id_struct) )
4039
#define SIZEOF_FLOW_STRUCT ( sizeof(struct ndpi_flow_struct) )
4140

4241
#define NDPI_DETECTION_ONLY_IPV4 ( 1 << 0 )
@@ -75,15 +74,6 @@ extern "C" {
7574
u_int32_t ndpi_detection_get_sizeof_ndpi_flow_struct(void);
7675

7776

78-
/**
79-
* Get the size of the id struct
80-
*
81-
* @return the size of the id struct
82-
*
83-
*/
84-
u_int32_t ndpi_detection_get_sizeof_ndpi_id_struct(void);
85-
86-
8777
/**
8878
* Get the size of the flow tcp struct
8979
*
@@ -303,18 +293,14 @@ extern "C" {
303293
* @par packet = unsigned char pointer to the Layer 3 (IP header)
304294
* @par packetlen = the length of the packet
305295
* @par packet_time_ms = the current timestamp for the packet (expressed in msec)
306-
* @par src = pointer to the source subscriber state machine
307-
* @par dst = pointer to the destination subscriber state machine
308296
* @return void
309297
*
310298
*/
311299
void ndpi_process_extra_packet(struct ndpi_detection_module_struct *ndpi_struct,
312300
struct ndpi_flow_struct *flow,
313301
const unsigned char *packet,
314302
const unsigned short packetlen,
315-
const u_int64_t packet_time_ms,
316-
struct ndpi_id_struct *src,
317-
struct ndpi_id_struct *dst);
303+
const u_int64_t packet_time_ms);
318304

319305
/**
320306
* Processes one packet and returns the ID of the detected protocol.
@@ -325,18 +311,14 @@ extern "C" {
325311
* @par packet = unsigned char pointer to the Layer 3 (IP header)
326312
* @par packetlen = the length of the packet
327313
* @par packet_time_ms = the current timestamp for the packet (expressed in msec)
328-
* @par src = pointer to the source subscriber state machine
329-
* @par dst = pointer to the destination subscriber state machine
330314
* @return the detected ID of the protocol
331315
*
332316
*/
333317
ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct *ndpi_struct,
334318
struct ndpi_flow_struct *flow,
335319
const unsigned char *packet,
336320
const unsigned short packetlen,
337-
const u_int64_t packet_time_ms,
338-
struct ndpi_id_struct *src,
339-
struct ndpi_id_struct *dst);
321+
const u_int64_t packet_time_ms);
340322
/**
341323
* Get the main protocol of the passed flows for the detected module
342324
*

‎src/include/ndpi_typedefs.h

Lines changed: 0 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -530,52 +530,6 @@ struct ndpi_lru_cache {
530530
struct ndpi_lru_cache_entry *entries;
531531
};
532532

533-
struct ndpi_id_struct {
534-
/**
535-
detected_protocol_bitmask:
536-
access this bitmask to find out whether an id has used skype or not
537-
if a flag is set here, it will not be reset
538-
to compare this, use:
539-
**/
540-
NDPI_PROTOCOL_BITMASK detected_protocol_bitmask;
541-
542-
/* NDPI_PROTOCOL_IRC_MAXPORT % 2 must be 0 */
543-
/* NDPI_PROTOCOL_IRC */
544-
#define NDPI_PROTOCOL_IRC_MAXPORT 8
545-
u_int16_t irc_port[NDPI_PROTOCOL_IRC_MAXPORT];
546-
u_int32_t last_time_port_used[NDPI_PROTOCOL_IRC_MAXPORT];
547-
u_int32_t irc_ts;
548-
549-
/* NDPI_PROTOCOL_GNUTELLA */
550-
u_int32_t gnutella_ts;
551-
552-
/* NDPI_PROTOCOL_JABBER */
553-
u_int32_t jabber_stun_or_ft_ts;
554-
555-
/* NDPI_PROTOCOL_DIRECTCONNECT */
556-
u_int32_t directconnect_last_safe_access_time;
557-
558-
/* NDPI_PROTOCOL_DIRECTCONNECT */
559-
u_int16_t detected_directconnect_port;
560-
u_int16_t detected_directconnect_udp_port;
561-
u_int16_t detected_directconnect_ssl_port;
562-
563-
/* NDPI_PROTOCOL_JABBER */
564-
#define JABBER_MAX_STUN_PORTS 6
565-
u_int16_t jabber_voice_stun_port[JABBER_MAX_STUN_PORTS];
566-
u_int16_t jabber_file_transfer_port[2];
567-
568-
/* NDPI_PROTOCOL_GNUTELLA */
569-
u_int16_t detected_gnutella_udp_port1;
570-
u_int16_t detected_gnutella_udp_port2;
571-
572-
/* NDPI_PROTOCOL_IRC */
573-
u_int8_t irc_number_of_port;
574-
575-
/* NDPI_PROTOCOL_JABBER */
576-
u_int8_t jabber_voice_stun_used_ports;
577-
};
578-
579533
/* ************************************************** */
580534

581535
struct ndpi_flow_tcp_struct {
@@ -850,7 +804,6 @@ typedef enum {
850804
NDPI_CONFIDENCE_UNKNOWN = 0, /* Unknown classification */
851805
NDPI_CONFIDENCE_MATCH_BY_PORT, /* Classification obtained looking only at the L4 ports */
852806
NDPI_CONFIDENCE_MATCH_BY_IP, /* Classification obtained looking only at the L3 addresses */
853-
NDPI_CONFIDENCE_DPI_SRC_DST_ID, /* Classification results based on ndpi_id_struct structures */
854807
NDPI_CONFIDENCE_DPI_CACHE, /* Classification results based on same LRU cache (i.e. correlation among sessions) */
855808
NDPI_CONFIDENCE_DPI, /* Deep packet inspection */
856809

@@ -1399,9 +1352,6 @@ struct ndpi_flow_struct {
13991352
/* NDPI_PROTOCOL_TINC */
14001353
u_int8_t tinc_state;
14011354
struct tinc_cache_entry tinc_cache_entry;
1402-
1403-
struct ndpi_id_struct *src;
1404-
struct ndpi_id_struct *dst;
14051355
};
14061356

14071357
#define NDPI_PROTOCOL_DEFAULT_LEVEL 0

0 commit comments

Comments
 (0)