diff --git a/inc/attributions.h b/inc/attributions.h index 27831b1..a41f890 100644 --- a/inc/attributions.h +++ b/inc/attributions.h @@ -3,6 +3,6 @@ #include "scanoss.h" -int attribution_notices(); +int attribution_notices(char * components); #endif diff --git a/inc/match_list.h b/inc/match_list.h index 401f343..91b3b32 100644 --- a/inc/match_list.h +++ b/inc/match_list.h @@ -80,7 +80,6 @@ #define MATCH_LIST_TOLERANCE 98.5 typedef struct match_data_t match_data_t; /* Forward declaration */ -typedef struct scan_data_t scan_data_t; /* Forward declaration*/ /** * @brief Define a list of component_data_t diff --git a/inc/mz.h b/inc/mz.h index 64da06e..ae49f90 100644 --- a/inc/mz.h +++ b/inc/mz.h @@ -3,6 +3,5 @@ #include "scanoss.h" -void mz_file_contents(char *key, char * dbtable); -void mz_get_key(struct mz_job *job, char *key); +void mz_get_key(struct ldb_table kb, char *key); #endif diff --git a/inc/scan.h b/inc/scan.h index bb76cba..51aaca3 100644 --- a/inc/scan.h +++ b/inc/scan.h @@ -62,6 +62,7 @@ typedef struct scan_data_t int max_components_to_process; /* Max component to retrieve during snippet scanning */ int max_snippets_to_show; //TODO int max_components_to_show; //TODO + int max_matchmap_size; bool printed_succed; } scan_data_t; diff --git a/inc/scanoss.h b/inc/scanoss.h index bc274f5..9c29e89 100644 --- a/inc/scanoss.h +++ b/inc/scanoss.h @@ -40,7 +40,7 @@ #define WFP_REC_LN 18 /* Log files */ -#define SCANOSS_VERSION "5.4.4" +#define SCANOSS_VERSION "5.4.5" #define SCAN_LOG "/tmp/scanoss_scan.log" #define MAP_DUMP "/tmp/scanoss_map.dump" #define SLOW_QUERY_LOG "/tmp/scanoss_slow_query.log" @@ -119,10 +119,6 @@ extern long microseconds_start; extern int map_rec_len; extern bool match_extensions; -/* File tracing -qi */ -extern uint8_t trace_id[MD5_LEN]; -extern bool trace_on; - /*component hint hold the last component matched/guessed */ extern char * component_hint; @@ -141,6 +137,8 @@ extern struct ldb_table oss_license; extern struct ldb_table oss_attribution; extern struct ldb_table oss_cryptography; extern struct ldb_table oss_sources; +extern struct ldb_table oss_notices; + extern bool first_file; extern int max_vulnerabilities; diff --git a/src/attributions.c b/src/attributions.c index a17e3d6..e9a85b4 100644 --- a/src/attributions.c +++ b/src/attributions.c @@ -59,35 +59,14 @@ uint8_t *data, uint32_t datalen, int iteration, void *ptr) memcpy(hexkey, data, MD5_LEN * 2); hexkey[MD5_LEN * 2] = 0; - /* Define mz_job values */ - char *src = calloc(MZ_MAX_FILE + 1, 1); - uint8_t *zsrc = calloc((MZ_MAX_FILE + 1) * 2, 1); - struct mz_job job; - sprintf(job.path, "%s/oss/notices", ldb_root); - memset(job.mz_id, 0, 2); - job.mz = NULL; - job.mz_ln = 0; - job.id = NULL; - job.ln = 0; - job.data = src; // Uncompressed data - job.data_ln = 0; - job.zdata = zsrc; // Compressed data - job.zdata_ln = 0; - job.md5[MD5_LEN] = 0; - job.key = NULL; - /* Print attribution notice header */ char *component = (char *) ptr; printf("[%s]\n\n", component); /* Print attribution notice */ - //mz_cat(&job, hexkey); - mz_get_key(&job,hexkey); + mz_get_key(oss_notices, hexkey); printf("\n"); - free(src); - free(zsrc); - return false; } @@ -304,11 +283,11 @@ void print_purl_attribution_notices(struct ldb_table oss_attributions, char * li * @brief //Validate the declared SBOM and print the attribution noticies in stdout * @return //TODO */ -int attribution_notices() +int attribution_notices(char * components) { char * licenses_json = notices_load_file(); /* Validate SBOM */ - declared_components = get_components(optarg); + declared_components = get_components(components); if (check_purl_attributions(oss_attribution, licenses_json) && !debug_on) /* Print attribution notices */ print_purl_attribution_notices(oss_attribution, licenses_json); diff --git a/src/help.c b/src/help.c index 9d1c752..8916f1a 100644 --- a/src/help.c +++ b/src/help.c @@ -65,7 +65,6 @@ Options:\n\ -h Display this help and exit.\n\ -d Save debugging information to disk (/tmp).\n\ -q Produces no JSON output. Only debugging info via STDERR.\n\ --i Trace specific file id when debugging (requires -q).\n\ \n\ Enviroment variables:\n\ SCANOSS_MATCHMAP_MAX: define the snippet scanning match map size, %d by default.\n\ diff --git a/src/license.c b/src/license.c index 141662c..6e56cea 100644 --- a/src/license.c +++ b/src/license.c @@ -117,7 +117,6 @@ bool osadl_load_file(void) scanlog("Warning: Cannot find OSADL definition. Please check that %s is present\n", path); else result = true; - free(path); return result; } @@ -172,10 +171,14 @@ int osadl_print_license(char *output, const char *license, bool more_keys_after) void print_osadl_license_data(char *license) { char output[MAX_FIELD_LN]; - osadl_print_license(output, license, false); - printf("{\"%s\": {%s}}", license, output); + int len = osadl_print_license(output, license, false); + if (len >0) + printf("{\"%s\": {%s}}", license, output); + else + printf("{\"%s\": {}}", license); } + static char *json_from_license(uint32_t *crclist, char *buffer, char *license, int src, bool *first_record) { clean_license(license); diff --git a/src/main.c b/src/main.c index 08cae91..9a488aa 100644 --- a/src/main.c +++ b/src/main.c @@ -59,13 +59,10 @@ struct ldb_table oss_license; struct ldb_table oss_attribution; struct ldb_table oss_cryptography; struct ldb_table oss_sources; - +struct ldb_table oss_notices; component_item *ignore_components; component_item *declared_components; -/* File tracing -qi */ -uint8_t trace_id[MD5_LEN]; -bool trace_on; bool lib_encoder_present = false; #define LDB_VER_MIN "4.1.0" @@ -168,6 +165,9 @@ void initialize_ldb_tables(char *name) snprintf(dbtable, MAX_ARGLN * 2, "%s/%s", oss_db_name, "sources"); oss_sources = ldb_read_cfg(dbtable); + snprintf(dbtable, MAX_ARGLN * 2, "%s/%s", oss_db_name, "notices"); + oss_notices = ldb_read_cfg(dbtable); + kb_version_get(); osadl_load_file(); @@ -272,10 +272,6 @@ int main(int argc, char **argv) //global var initialization - it must be improved debug_on = false; quiet = false; - - /* File tracing with -qi */ - trace_on = false; - memset(trace_id, 0 ,16); if (argc <= 1) { @@ -295,7 +291,7 @@ int main(int argc, char **argv) int option; bool invalid_argument = false; char * ldb_db_name = NULL; - while ((option = getopt(argc, argv, ":f:s:b:B:c:k:a:F:l:n:i:M:N:wtvhedqH")) != -1) + while ((option = getopt(argc, argv, ":f:s:b:B:c:k:a:F:l:n:M:N:wtvhedqH")) != -1) { /* Check valid alpha is entered */ if (optarg) @@ -325,12 +321,17 @@ int main(int argc, char **argv) case 'k': initialize_ldb_tables(ldb_db_name); - mz_file_contents(optarg, oss_file.db); + mz_get_key(oss_sources, optarg); exit(EXIT_SUCCESS); break; case 'a': - if (declared_components) printf("Cannot combine -s and -a\n"); + if (declared_components) + { + printf("Cannot combine -s and -a\n"); + break; + } + initialize_ldb_tables(ldb_db_name); exit(attribution_notices(optarg)); break; @@ -340,6 +341,7 @@ int main(int argc, char **argv) break; case 'l': + initialize_ldb_tables(ldb_db_name); print_osadl_license_data(optarg); exit(EXIT_SUCCESS); break; @@ -353,15 +355,6 @@ int main(int argc, char **argv) case 'N': scan_max_components = atol(optarg); break; - case 'i': - if (strlen(optarg) == (MD5_LEN * 2)) - { - ldb_hex_to_bin(optarg, MD5_LEN * 2, trace_id); - trace_on = true; - } - else fprintf(stderr, "Ignoring -i due to invalid length\n"); - break; - case 'w': force_wfp = true; break; @@ -370,6 +363,7 @@ int main(int argc, char **argv) force_snippet_scan = true; break; case 't': + initialize_ldb_tables(ldb_db_name); scan_benchmark(); exit(EXIT_SUCCESS); break; @@ -417,7 +411,7 @@ int main(int argc, char **argv) else { printf("'libhpsm.so' must be present in the system to execute this command\n"); - exit(1); + exit(EXIT_FAILURE); } break; } diff --git a/src/match.c b/src/match.c index 1b9eb98..7c7e5a2 100644 --- a/src/match.c +++ b/src/match.c @@ -399,7 +399,7 @@ bool add_component_from_urlid(component_list_t *component_list, uint8_t *url_id, * @return false */ -file_recordset *files = NULL; +static file_recordset *files = NULL; bool component_from_file(uint8_t *key, uint8_t *subkey, int subkey_ln, uint8_t *raw_data, uint32_t datalen, int iteration, void *ptr) { diff --git a/src/mz.c b/src/mz.c index 5532888..c3ab15a 100644 --- a/src/mz.c +++ b/src/mz.c @@ -44,15 +44,16 @@ * @param job input mz job * @param key key to be found */ -void mz_get_key(struct mz_job *job, char *key) +void mz_get_key(struct ldb_table kb, char *key) { /* Calculate mz file path */ char mz_path[LDB_MAX_PATH + MD5_LEN] = "\0"; char mz_file_id[5] = "\0\0\0\0\0"; + struct mz_job job; memcpy(mz_file_id, key, 4); + sprintf(mz_path, "%s/%s/%s/%s.mz", ldb_root, kb.db, kb.table,mz_file_id); - sprintf(mz_path, "%s/%s.mz", job->path, mz_file_id); - if (oss_sources.definitions & LDB_TABLE_DEFINITION_ENCRYPTED) + if (kb.definitions & LDB_TABLE_DEFINITION_ENCRYPTED) { if (decrypt_mz) strcat(mz_path, ".enc"); @@ -65,88 +66,52 @@ void mz_get_key(struct mz_job *job, char *key) scanlog("MZ path: %s \n", mz_path); /* Save path and key on job */ - job->key = calloc(MD5_LEN, 1); - ldb_hex_to_bin(key, MD5_LEN * 2, job->key); + job.key = calloc(MD5_LEN, 1); + ldb_hex_to_bin(key, MD5_LEN * 2, job.key); /* Read source mz file into memory */ - job->mz = file_read(mz_path, &job->mz_ln); + job.mz = file_read(mz_path, &job.mz_ln); /* Search and display "key" file contents */ /* Recurse mz contents */ uint64_t ptr = 0; - while (ptr < job->mz_ln) + while (ptr < job.mz_ln) { /* Position pointers */ - job->id = job->mz + ptr; - uint8_t *file_ln = job->id + MZ_MD5; - job->zdata = file_ln + MZ_SIZE; + job.id = job.mz + ptr; + uint8_t *file_ln = job.id + MZ_MD5; + job.zdata = file_ln + MZ_SIZE; /* Get compressed data size */ uint32_t tmpln; memcpy((uint8_t*)&tmpln, file_ln, MZ_SIZE); - job->zdata_ln = tmpln; + job.zdata_ln = tmpln; /* Get total mz record length */ - job->ln = MZ_MD5 + MZ_SIZE + job->zdata_ln; + job.ln = MZ_MD5 + MZ_SIZE + job.zdata_ln; /* Pass job to handler */ - if (!memcmp(job->id, job->key + 2, MZ_MD5)) + if (!memcmp(job.id, job.key + 2, MZ_MD5)) { - if (decrypt_mz) + if (kb.definitions & LDB_TABLE_DEFINITION_ENCRYPTED) { - decrypt_mz(job->id, job->zdata_ln); + decrypt_mz(job.id, job.zdata_ln); } /* Decompress */ - MZ_DEFLATE(job); + MZ_DEFLATE(&job); - job->data[job->data_ln] = 0; - printf("%s", job->data); + job.data[job.data_ln] = 0; + printf("%s", job.data); return; } /* Increment pointer */ - ptr += job->ln; - if (ptr > job->mz_ln) + ptr += job.ln; + if (ptr > job.mz_ln) { - printf("%s integrity failed\n", job->path); + printf("%s integrity failed\n", job.path); exit(EXIT_FAILURE); } } - free(job->key); - free(job->mz); -} - - -/** - * @brief uncompress the file contents of a given md5 key - * @param key md5 key - */ -void mz_file_contents(char *key, char * db) -{ - /* Extract values from command */ - char dbtable[64]; - sprintf(dbtable,"%s/sources",db); - - /* Reserve memory for compressed and uncompressed data */ - char *src = calloc(MZ_MAX_FILE + 1, 1); - uint8_t *zsrc = calloc((MZ_MAX_FILE + 1) * 2, 1); - - /* Define mz_job values */ - struct mz_job job; - sprintf(job.path, "%s/%s", ldb_root, dbtable); - memset(job.mz_id, 0, 2); - job.mz = NULL; - job.mz_ln = 0; - job.id = NULL; - job.ln = 0; - job.data = src; // Uncompressed data - job.data_ln = 0; - job.zdata = zsrc; // Compressed data - job.zdata_ln = 0; - job.md5[MD5_LEN] = 0; - job.key = NULL; - - mz_get_key(&job, key); - - free(src); - free(zsrc); + free(job.key); + free(job.mz); } diff --git a/src/scan.c b/src/scan.c index b053f0c..2dad032 100644 --- a/src/scan.c +++ b/src/scan.c @@ -115,7 +115,7 @@ static bool zero_bytes (uint8_t *md5) */ static match_t ldb_scan_file(scan_data_t * scan) { - scanlog("Checking entire file\n"); + scanlog("Checking entire file %s\n", scan->file_path); if (zero_bytes(scan->md5)) return MATCH_NONE; diff --git a/src/snippets.c b/src/snippets.c index 7c3e2ae..36302ce 100644 --- a/src/snippets.c +++ b/src/snippets.c @@ -536,23 +536,25 @@ void wfp_invert(uint32_t wfpint32, uint8_t *out) static void matchmap_setup(scan_data_t * scan) { char * matchmap_env = getenv("SCANOSS_MATCHMAP_MAX"); + scan->max_matchmap_size = DEFAULT_MATCHMAP_FILES; if (matchmap_env) { int matchmap_max_files_aux = atoi(matchmap_env); if (matchmap_max_files_aux > DEFAULT_MATCHMAP_FILES / 4 && matchmap_max_files_aux < DEFAULT_MATCHMAP_FILES * 20) { scanlog("matchmap size changed by env variable to: %d\n", matchmap_max_files_aux); - matchmap_max_files = matchmap_max_files_aux; + scan->max_matchmap_size = matchmap_max_files_aux; } } //If we are looking fow multiple snippets, update the matchmap size - matchmap_max_files = scan->max_snippets_to_process * matchmap_max_files; + scan->max_matchmap_size *= scan->max_snippets_to_process; if (engine_flags & ENABLE_HIGH_ACCURACY) { - matchmap_max_files *=5; - scanlog("matchmap size changed by high accuracy analisys to: %d\n", matchmap_max_files); + scan->max_matchmap_size *=5; + scanlog("matchmap size changed by high accuracy analisys to: %d\n", scan->max_matchmap_size); } + matchmap_max_files = scan->max_matchmap_size; } typedef struct matchmap_entry_t @@ -581,7 +583,7 @@ int add_file_to_matchmap(scan_data_t *scan, matchmap_entry_t *item, uint8_t *md5 for (long t = start_pos; t < scan->matchmap_size; t++) { //The matchmap is sorted, stop if you are comparing against a different sector - if (*scan->matchmap[t].md5 > *md5 && (scan->matchmap_size < matchmap_max_files)) + if (*scan->matchmap[t].md5 > *md5 && (scan->matchmap_size < scan->max_matchmap_size)) { scanlog("skipping: md5 out of range wfp\n"); return -1; @@ -614,7 +616,7 @@ int add_file_to_matchmap(scan_data_t *scan, matchmap_entry_t *item, uint8_t *md5 if (found < 0) { /* Not found. Add MD5 to map */ - if (scan->matchmap_size >= matchmap_max_files) + if (scan->matchmap_size >= scan->max_matchmap_size) { scanlog("skipping: matchmap is full\n"); return -1; @@ -704,11 +706,6 @@ match_t ldb_scan_snippets(scan_data_t *scan) if (engine_flags & DISABLE_SNIPPET_MATCHING) return MATCH_NONE; - if (trace_on) - scanlog("Checking snippets. Traced (-qi) matches marked with *\n"); - else - scanlog("Checking snippets\n"); - matchmap_setup(scan); adjust_tolerance(scan); @@ -753,7 +750,7 @@ match_t ldb_scan_snippets(scan_data_t *scan) memset(map_indirection_index, 0, sizeof(map_indirection_index)); scanlog ("< Snippet scan setup: Total lines: %d ,Matchmap size: %d, Min hits: %d, Min lines: %d, Map max size = %d, Cat N = %d x %d, Cat size = %d >\n", - scan->total_lines, matchmap_max_files, min_match_hits, min_match_lines, map_max_size, MAP_INDIRECTION_CAT_NUMBER, map_indedirection_items_size, MAP_INDIRECTION_CAT_SIZE); + scan->total_lines, scan->max_matchmap_size, min_match_hits, min_match_lines, map_max_size, MAP_INDIRECTION_CAT_NUMBER, map_indedirection_items_size, MAP_INDIRECTION_CAT_SIZE); for (int i =0; i < scan->hash_count; i++) { @@ -802,11 +799,11 @@ match_t ldb_scan_snippets(scan_data_t *scan) map_lines_indirection[map[map_indirection[i][j]].line] = 1; lines_coverage++; } - if (cat_limit > matchmap_max_files) + if (cat_limit > scan->max_matchmap_size) { if ((hashes_to_process < scan->hash_count / 10 || (float) lines_coverage / scan->hash_count < 0.6) && cat_limit < MAX_MATCHMAP_FILES) { - matchmap_max_files += map[map_indirection[i][j]].size; + scan->max_matchmap_size += map[map_indirection[i][j]].size; } else { @@ -843,10 +840,10 @@ match_t ldb_scan_snippets(scan_data_t *scan) } } } - matchmap_max_files = cat_limit; + scan->max_matchmap_size = cat_limit; scanlog("Map limit on %d MD5s at %d of %d caths. Selected hashes: %d/%d - lines coverage %d\n", - matchmap_max_files, cat_limit_index, MAP_INDIRECTION_CAT_NUMBER, hashes_to_process, scan->hash_count, (lines_coverage * 100) / scan->total_lines); - scan->matchmap = calloc(matchmap_max_files, sizeof(matchmap_entry)); + scan->max_matchmap_size, cat_limit_index, MAP_INDIRECTION_CAT_NUMBER, hashes_to_process, scan->hash_count, (lines_coverage * 100) / scan->total_lines); + scan->matchmap = calloc(scan->max_matchmap_size, sizeof(matchmap_entry)); int map_indexes[scan->hash_count]; memset(map_indexes, 0, sizeof(map_indexes));