4.1.5

src/collate.c

@@ -652,19 +652,21 @@ int ldb_collate_load_tuples_to_delete(job_delete_tuples_t * job, char * buffer,
 
     while (line != NULL) 
 	{
-		tuples = realloc(tuples, ((tuples_index+1) * sizeof(tuple_t*)));
-		tuples[tuples_index] = calloc(1, sizeof(tuple_t));
-		ldb_hex_to_bin(line, key_len * 2, tuples[tuples_index]->key);
-
-		if (strchr(line, ','))
+		if (line && strlen(line) >= key_len * 2)
 		{
-			char * data = strdup(line + key_len * 2 + 1);
-			if (data && *data)
-				tuples[tuples_index]->data = data;
+			tuples = realloc(tuples, ((tuples_index+1) * sizeof(tuple_t*)));
+			tuples[tuples_index] = calloc(1, sizeof(tuple_t));
+			ldb_hex_to_bin(line, key_len * 2, tuples[tuples_index]->key);
+
+			if (strchr(line, ','))
+			{
+				char * data = strdup(line + key_len * 2 + 1);
+				if (data && *data)
+					tuples[tuples_index]->data = data;
+			}
+			tuples_index++;
 		}
-
         line = strtok(NULL, delimiter);
-		tuples_index++;
     }
 	job->tuples = tuples;
 	job->tuples_number = tuples_index;
@@ -715,12 +717,12 @@ bool ldb_collate_init(struct ldb_collate_data * collate, struct ldb_table table,
 	}
 
 	/* Reserve space for collate data */
-	collate->data = (char *)calloc(LDB_MAX_RECORDS * collate->rec_width, 1);
+	collate->data = (char *)calloc(LDB_MAX_RECORDS * (collate->rec_width+10), 1);
 
 	if (!collate->data)
 		return false;
 
-	collate->tmp_data = (char *)calloc(LDB_MAX_RECORDS * collate->rec_width, 1);
+	collate->tmp_data = (char *)calloc(LDB_MAX_RECORDS * (collate->rec_width+10), 1);
 
 	if (!collate->tmp_data)
 		return false;
@@ -736,12 +738,12 @@ bool ldb_collate_init(struct ldb_collate_data * collate, struct ldb_table table,
 	return true;
 }
 
-void ldb_collate_sector(struct ldb_collate_data *collate, uint8_t sector, uint8_t *sector_mem)
+void ldb_collate_sector(struct ldb_collate_data *collate, ldb_sector_t * sector)
 {
-	log_info("Collating %s/%s - sector %02x - %s\n", collate->in_table.db, collate->in_table.table, sector, sector_mem == NULL ? "On disk" : "On RAM");
+	log_info("Collating %s/%s - sector %02x - %s\n", collate->in_table.db, collate->in_table.table, sector->id, sector->data == NULL ? "On disk" : "On RAM");
 	/* Read each one of the (256 ^ 3) list pointers from the map */
 	uint8_t k[LDB_KEY_LN];
-	k[0] = sector;
+	k[0] = sector->id;
 	for (int k1 = 0; k1 < 256; k1++)
 		for (int k2 = 0; k2 < 256; k2++)
 			for (int k3 = 0; k3 < 256; k3++)
@@ -751,7 +753,7 @@ void ldb_collate_sector(struct ldb_collate_data *collate, uint8_t sector, uint8_
 				k[3] = k3;
 				/* If there is a pointer, read the list */
 				/* Process records */
-				ldb_fetch_recordset(sector_mem, collate->in_table, k, true, ldb_collate_handler, collate);
+				ldb_fetch_recordset_v2(sector, collate->in_table, k, true, ldb_collate_handler, collate);
 			}
 
 	/* Process last record/s */
@@ -771,13 +773,13 @@ void ldb_collate_sector(struct ldb_collate_data *collate, uint8_t sector, uint8_
 		ldb_sector_update(collate->out_table, k);
 
 	if (collate->del_count)
-		log_info("%s - sector %02X: %'ld records deleted\n", collate->in_table.table, sector, collate->del_count);
+		log_info("%s - sector %02X: %'ld records deleted\n", collate->in_table.table, sector->id, collate->del_count);
 
-	log_info("Table %s - sector %2x: collate completed with %'ld records\n", collate->in_table.table , sector, collate->rec_count);
+	log_info("Table %s - sector %2x: collate completed with %'ld records\n", collate->in_table.table , sector->id, collate->rec_count);
 
 	free(collate->data);
-	free(collate->tmp_data);
-	free(sector_mem);
+	free(sector->data);
+	sector->data = NULL;
 }
 
 /**
@@ -812,12 +814,12 @@ void ldb_collate(struct ldb_table table, struct ldb_table out_table, int max_rec
 			/* Load collate data structure */
 			collate.handler = handler;
 			collate.del_tuples = NULL;
-			uint8_t *sector  = ldb_load_sector(table, &k0);
+			ldb_sector_t sector  = ldb_load_sector_v2(table, &k0);
 			//skip unexistent sector.
-			if (!sector)
+			if (!sector.size)
 				continue;
 
-			ldb_collate_sector(&collate, k0, sector);
+			ldb_collate_sector(&collate, &sector);
 		}
 
 		if (p_sector >=0)
@@ -867,22 +869,23 @@ void ldb_collate_delete(struct ldb_table table, struct ldb_table out_table, job_
 	{
 		k0 = *delete->tuples[i]->key;
 		struct ldb_collate_data collate;
-
 		if (k0 != k0_last && ldb_collate_init(&collate, table, out_table, 2048, false, k0))
 		{
 			log_info("Removing keys from Table %s - Reading sector %02x\n", table.table, k0);
 			/* Load collate data structure */
 			collate.handler = handler;
 			collate.del_tuples = delete;
 			collate.del_count = 0;
-			uint8_t * sector  = ldb_load_sector(table, &k0);
-			ldb_collate_sector(&collate, k0, sector);
+			ldb_sector_t sector = ldb_load_sector_v2(table, &k0);
+			ldb_collate_sector(&collate, &sector);
 			total_records += collate.del_count;
-			k0_last = k0;
 		}
+		k0_last = k0;
+		/* Exit here if it is a delete command, otherwise move to the next sector */
 	} 
 
 	/* Show processed totals */
 	log_info("Table %s: cleanup completed with %'ld records\n", table.table, total_records);
 	fflush(stdout);
+
 }

src/import.c

@@ -1430,42 +1430,44 @@ int import_collate_sector(ldb_importation_config_t *config)
 		char *ptr = NULL;
 		char *filename = basename(config->csv_path);
 		//extract the sector from the file name. If there is no sector (example license.csv) process the compleate table.
-		long sector = strtol(filename, &ptr, 16);
+		long sector_number = strtol(filename, &ptr, 16);
 		if (ptr - filename < 2 || (ptr && *ptr != '.'))
-			sector = -1;
+			sector_number = -1;
 
 		logger_basic("Collating - %s", dbtable);
 
-		if (sector < 0)
+		if (sector_number < 0)
 		{
-			log_info("Collating table %s - all sectors, Max record size: %d\n", dbtable, sector, max_rec_len);
-			ldb_collate(ldbtable, tmptable, max_rec_len, false, sector, NULL);
+			log_info("Collating table %s - all sectors, Max record size: %d\n", dbtable, sector_number, max_rec_len);
+			ldb_collate(ldbtable, tmptable, max_rec_len, false, sector_number, NULL);
 			return 0;
 		}
 
-		log_info("Collating table %s - sector %02x, Max record size: %d\n", dbtable, sector, max_rec_len);
+		log_info("Collating table %s - sector %02x, Max record size: %d\n", dbtable, sector_number, max_rec_len);
 		if ((ldbtable.definitions > 0 && ldbtable.definitions & LDB_TABLE_DEFINITION_MZ) ||
 			 (ldbtable.definitions == LDB_TABLE_DEFINITION_UNDEFINED && config->opt.params.is_mz_table))
 		{
-			ldb_mz_collate(ldbtable, sector);
+			ldb_mz_collate(ldbtable, sector_number);
 		}
-		else if (sector >= 0)
+		else if (sector_number >= 0)
 		{
 			pthread_mutex_lock(&lock);
 			struct ldb_collate_data collate;
-			uint8_t k0 = sector;
-			uint8_t *sector_mem = NULL;
+			uint8_t k0 = sector_number;
+			//uint8_t *sector_mem = NULL;
+			ldb_sector_t sector = {.data=NULL, .size = 0, .id = sector_number};
 
 			bool init_ok = ldb_collate_init(&collate, ldbtable, tmptable, max_rec_len, false, k0);
 			if (!init_ok)
 				log_info("Collate init failed for sector %d\n", k0);
 
 			if (init_ok && check_system_available_ram(ldbtable, k0, config->opt.params.collate_max_ram_percent))
-				sector_mem = ldb_load_sector(ldbtable, &k0);
+				//sector_mem = ldb_load_sector(ldbtable, &k0);
+				sector = ldb_load_sector_v2(ldbtable, &k0);
 
 			pthread_mutex_unlock(&lock);
 			if (init_ok)
-				ldb_collate_sector(&collate, sector, sector_mem);
+				ldb_collate_sector(&collate, &sector);
 			else
 			{
 				log_info("ERROR: failed to allocate memory to collate sector %02x\n", k0);

src/ldb.h

@@ -27,7 +27,7 @@
 #include "./ldb/types.h"
 #include "./ldb/mz.h"
 
-#define LDB_VERSION "4.1.4"
+#define LDB_VERSION "4.1.5"
 
 #define LDB_TABLE_DEFINITION_UNDEFINED -1
 #define LDB_TABLE_DEFINITION_STANDARD 0
@@ -62,6 +62,7 @@ uint64_t ldb_last_node_pointer(FILE *ldb_sector, uint64_t list_pointer);
 void ldb_update_list_pointers(FILE *ldb_sector, uint8_t *key, uint64_t list, uint64_t new_node);
 int ldb_node_write (struct ldb_table table, FILE *ldb_sector, uint8_t *key, uint8_t *data, uint32_t dataln, uint16_t records);
 uint64_t ldb_node_read (uint8_t *sector, struct ldb_table table, FILE *ldb_sector, uint64_t ptr, uint8_t *key, uint32_t *bytes_read, uint8_t **out, int max_node_size);
+uint64_t ldb_node_read_v2(ldb_sector_t *sector, struct ldb_table table, uint64_t ptr, uint8_t *key, uint32_t *bytes_read, uint8_t **out, int max_node_size);
 char *ldb_sector_path (struct ldb_table table, uint8_t *key, char *mode);
 FILE *ldb_open (struct ldb_table table, uint8_t *key, char *mode);
 bool ldb_close(FILE * sector);
@@ -84,11 +85,13 @@ bool ldb_create_database(char *database);
 struct ldb_recordset ldb_recordset_init(char *db, char *table, uint8_t *key);
 void ldb_list_unlink(FILE *ldb_sector, uint8_t *key);
 uint8_t *ldb_load_sector (struct ldb_table table, uint8_t *key);
+ldb_sector_t ldb_load_sector_v2(struct ldb_table table, uint8_t *key);
 bool ldb_validate_node(uint8_t *node, uint32_t node_size, int subkey_ln);
 //bool uint32_is_zero(uint8_t *n);
 bool ldb_key_exists(struct ldb_table table, uint8_t *key);
 bool ldb_key_in_recordset(uint8_t *rs, uint32_t rs_len, uint8_t *subkey, uint8_t subkey_ln);
 uint32_t ldb_fetch_recordset(uint8_t *sector, struct ldb_table table, uint8_t* key, bool skip_subkey, bool (*ldb_record_handler) (uint8_t *, uint8_t *, int, uint8_t *, uint32_t, int, void *), void *void_ptr);
+uint32_t ldb_fetch_recordset_v2(ldb_sector_t * sector, struct ldb_table table, uint8_t* key, bool skip_subkey, bool (*ldb_record_handler) (uint8_t *, uint8_t *, int, uint8_t *, uint32_t, int, void *), void *void_ptr);
 bool ldb_hexprint_width(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *data, uint32_t size, int iteration, void *ptr);
 void ldb_sector_update(struct ldb_table table, uint8_t *key);
 void ldb_sector_erase(struct ldb_table table, uint8_t *key);

src/ldb/collate.h

@@ -43,7 +43,7 @@ struct ldb_collate_data
 	collate_handler handler;
 };
 bool ldb_collate_init(struct ldb_collate_data * collate, struct ldb_table table, struct ldb_table out_table, int max_rec_ln, bool merge, uint8_t sector);
-void ldb_collate_sector(struct ldb_collate_data *collate, uint8_t sector, uint8_t *sector_mem);
+void ldb_collate_sector(struct ldb_collate_data *collate, ldb_sector_t * sector);
 int ldb_collate_load_tuples_to_delete(job_delete_tuples_t* job, char * buffer, char * d, struct ldb_table table);
 void ldb_collate(struct ldb_table table, struct ldb_table out_table, int max_rec_ln, bool merge, int p_sector, collate_handler handler);
 void ldb_collate_delete(struct ldb_table table, struct ldb_table out_table, job_delete_tuples_t * delete, collate_handler handler);

src/ldb/types.h

@@ -47,6 +47,14 @@ struct ldb_recordset
 	uint64_t last_node; // Location of last node of the list
     uint8_t ts_ln;      // 2 or 4 (16-bit or 32-bit reserved for total sector size)
 };
+typedef struct ldb_sector_t
+{
+	uint8_t id;
+	size_t size;
+	uint8_t * data;
+	FILE * file;
+	bool failure;
+} ldb_sector_t;
 
 typedef bool (*ldb_record_handler) (uint8_t *, uint8_t *, int, uint8_t *, uint32_t, int, void *);
 

src/md5.c

@@ -22,32 +22,48 @@ void md5_string(const unsigned char *input, int len, unsigned char output[16])
  * @param path string path
  * @return pointer to file md5 array
  */
-uint8_t * md5_file(char *path) 
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <gcrypt.h>
+
+static void __attribute__((constructor)) init_libgcrypt(void)
 {
-    uint8_t *c = calloc(1, gcry_md_get_algo_dlen(GCRY_MD_MD5));  // Allocate memory for MD5 hash
-    FILE *fp = fopen(path, "rb");
+    if (!gcry_check_version(GCRYPT_VERSION)) {
+        fprintf(stderr, "Libgcrypt version mismatch\n");
+        return;
+    }
+
+    gcry_control(GCRYCTL_DISABLE_SECMEM_WARN);
+    gcry_control(GCRYCTL_INIT_SECMEM, 16384, 0);
+    gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0);
+}
 
+uint8_t * md5_file(char *path)
+{ 
+    uint8_t *c = calloc(1, gcry_md_get_algo_dlen(GCRY_MD_MD5)); // Allocate memory for MD5 hash
+    FILE *fp = fopen(path, "rb");
     if (!fp) {
         fprintf(stderr, "Unable to open file for reading.\n");
+        free(c);
         return NULL;
     }
-
+    
     gcry_md_hd_t md5_hd;
     gcry_md_open(&md5_hd, GCRY_MD_MD5, GCRY_MD_FLAG_SECURE);
-
     uint8_t *buffer = malloc(BUFFER_SIZE);
     size_t bytes;
-
+    
     while ((bytes = fread(buffer, 1, BUFFER_SIZE, fp)) > 0) {
         gcry_md_write(md5_hd, buffer, bytes);
     }
-
+    
     fclose(fp);
     free(buffer);
-
+    
     const uint8_t *digest = gcry_md_read(md5_hd, GCRY_MD_MD5);
     memcpy(c, digest, gcry_md_get_algo_dlen(GCRY_MD_MD5));
-
     gcry_md_close(md5_hd);
+
     return c;
 }