Merge pull request #3 from BCCDC-PHL/add-ncbi-table

Add ncbi table

Author: sherrie9

bin/custom_html.py

@@ -77,12 +77,15 @@ def build_dbnote(databases_df):
 
 	return f"<p>{DBNOTE}</p>"
 
+def build_table_title(name):
+	return f"""		
+			<h3 class = "first">{name}</h3>
+			"""
 
 def build_table(name, col_names, table_rows, hidden_rows=None):
 	hidden_rows = '' if not hidden_rows else hidden_rows
 	table_header = '</th>\n<th>'.join(col_names)
 	return f'''
-			<h3 class = "first">{name}</h3>
 			<table>
 				<tbody>
 					<tr class="header">

bin/filter_best_bitscore.py

@@ -7,14 +7,14 @@
 import pandas as pd 
 
 
-def filter_best_bitscore(df, group_col, score_col):
+def filter_dataframe(df, group_col, score_col):
     idxmax = df.groupby(group_col)[score_col].idxmax()
     df = df.loc[idxmax].reset_index(drop=True)
     return df
 
 def main(args):
     blast_df = pd.read_csv(args.input)
-    filtered_df = filter_best_bitscore(blast_df, args.group_col, args.score_col)
+    filtered_df = filter_dataframe(blast_df, args.group_col, args.score_col)
     filtered_df.to_csv(args.output, index=False)
 
 

bin/report.py

@@ -4,12 +4,11 @@
 import argparse
 from functools import partial
 import re
-from custom_html import HEAD, build_dbnote, build_table, build_row, FOOT, PAGEBREAK
+from custom_html import HEAD, build_dbnote, build_table,build_table_title, build_row, FOOT, PAGEBREAK
 
 
 EXPR_PRIORITY = re.compile(r'ATCC|LMG|type|NCTC')
-row_names = "subject_accession species bitscore percent_coverage percent_identity database_name extra_info".split()
-build_row_part = partial(build_row, row_names=row_names)
+ROW_NAMES = "subject_accession species bitscore percent_coverage percent_identity database_name extra_info".split()
 
 
 def parse_db_csv(filepath):
@@ -77,50 +76,64 @@ def parse_blast(filepath):
 
 	return df.fillna('N/A')
 
-def main(args):
-	blast_table = parse_blast(args.blast)
+def build_table_string(name, df, limit=20):
+	build_row_partial = partial(build_row, row_names=ROW_NAMES)
+
+	if df.shape[0] < limit:
+		str_rows = df.apply(build_row_partial, axis=1)
+		str_rows = '\n'.join(str_rows)
+		str_table = build_table(name, ROW_NAMES, str_rows)
+
+	else:
+		N = df.shape[0]
+		str_rows = df.iloc[0:limit].apply(build_row_partial, axis=1)
+		str_rows = '\n'.join(str_rows)
+		
+		hidden_str_rows = df.iloc[limit+1:min(N, 300)].apply(build_row_partial, axis=1)
+		hidden_str_rows = '\n'.join(hidden_str_rows)
+
+		str_table = build_table(name, ROW_NAMES, str_rows, hidden_str_rows)
+	
+	return str_table
 
-	outfile = open(args.output, "w")
-	outfile.write(HEAD)
+def main(args):
+	local_blast_table = parse_blast(args.blast)
+	ncbi_blast_table = parse_blast(args.ncbi)
+	ncbi_blast_table['extra_info'] = ''
 
 	database_df = parse_db_csv(args.db)
 	DBNOTE = build_dbnote(database_df)
 
 	extra_info = extract_descriptions(database_df)
 
-	blast_table = blast_table.merge(extra_info, on='subject_accession', how='left')
-	
+	local_blast_table = local_blast_table.merge(extra_info, on='subject_accession', how='left')
 
-	for name, df in blast_table.groupby('query_seq_id'):
-		
-		print(name)
-
-		outfile.write(DBNOTE)
-
-		if df.shape[0] < 20:
-			str_rows = df.apply(build_row_part, axis=1)
-			str_rows = '\n'.join(str_rows)
-			str_table = build_table(name, row_names, str_rows)
-			outfile.write(str_table)
-		else:
-			N = df.shape[0]
-			str_rows = df.iloc[0:20].apply(build_row_part, axis=1)
-			str_rows = '\n'.join(str_rows)
+	local_blast_dict = dict(list(local_blast_table.groupby('query_seq_id')))
+	ncbi_blast_dict = dict(list(ncbi_blast_table.groupby('query_seq_id')))
+
+	with open(args.output, "w") as outfile:
+		outfile.write(HEAD)
+		for name in set(local_blast_dict.keys()).union(ncbi_blast_dict.keys()):
 
-			hidden_str_rows = df.iloc[21:min(N, 300)].apply(build_row_part, axis=1)
-			hidden_str_rows = '\n'.join(hidden_str_rows)
+			print(name)
+			outfile.write(DBNOTE)
+
+			outfile.write(build_table_title(name))
 
-			str_table = build_table(name, row_names, str_rows, hidden_str_rows)
-			outfile.write(str_table)
+			if name in local_blast_dict:
+				outfile.write(build_table_string(name, local_blast_dict[name]))
 
-		outfile.write(PAGEBREAK)
+			if name in ncbi_blast_dict:
+				outfile.write(build_table_string(name, ncbi_blast_dict[name]))
+			
+			outfile.write(PAGEBREAK)
 
-	outfile.write(FOOT)
-	outfile.close()
+		outfile.write(FOOT)
 
 if __name__ == '__main__':
 	parser = argparse.ArgumentParser()
-	parser.add_argument('-b', '--blast', help='A single concatenated BLAST CSV table with hits from multiple samples and multiple database sources.')
+	parser.add_argument('-b', '--blast', required=True, help='A single concatenated BLAST CSV table with hits from multiple samples and multiple database sources.')
+	parser.add_argument('-n', '--ncbi', required=True, help='A single concatenated BLAST CSV table with hits from multiple samples from NCBI core_nt database.')
 	parser.add_argument('-d', '--db', help='Database CSV file containing ID, DBNAME, and PATH columns.')
 	parser.add_argument('-o', '--output', help='Output HTML report filename.')
 	args = parser.parse_args()

main.nf

@@ -17,14 +17,19 @@ println "Current date and time: $formattedDateTime"
 
 nextflow.enable.dsl = 2
 
-include { hash_seqs }                 from './modules/hash_seqs.nf'
-include { seq_qc }                    from './modules/blast.nf'
-include { blastn }                    from './modules/blast.nf'
-include { filter_by_regex }           from './modules/blast.nf'
-include { filter_best_bitscore }      from './modules/blast.nf'
-include { build_report }              from './modules/blast.nf'
-include { collect_provenance }        from './modules/provenance.nf'
-include { pipeline_provenance }       from './modules/provenance.nf'
+include { hash_seqs }                                           from './modules/hash_seqs.nf'
+include { seq_qc }                                              from './modules/blast.nf'
+include { blastn }                                              from './modules/blast.nf'
+include { blastn_ncbi }                                         from './modules/blast.nf'
+include { taxonkit_annotation as taxonkit_annotation_local }    from './modules/blast.nf'
+include { taxonkit_annotation as taxonkit_annotation_ncbi }     from './modules/blast.nf'
+include { filter_by_regex as filter_by_regex_local }            from './modules/blast.nf'
+include { filter_by_regex as filter_by_regex_ncbi }             from './modules/blast.nf'
+include { filter_best_bitscore as filter_best_bitscore_local }  from './modules/blast.nf'
+include { filter_best_bitscore as filter_best_bitscore_ncbi }   from './modules/blast.nf'
+include { build_report }                                        from './modules/blast.nf'
+include { collect_provenance }                                  from './modules/provenance.nf'
+include { pipeline_provenance }                                 from './modules/provenance.nf'
 
 
 workflow {
@@ -49,6 +54,8 @@ workflow {
     ch_db = Channel.of()
   }
 
+  ch_ncbi_db = Channel.fromPath(params.ncbi_db)
+
   ch_seqs = ch_fasta.splitFasta(record: [id: true, seqString: true])
 
   main:
@@ -58,20 +65,31 @@ workflow {
 
     seq_qc(ch_seqs)
     ch_blast = blastn(ch_seqs.combine(ch_db)).blast_report
-    ch_blast_prov = blastn.out.provenance.map{}
+    ch_blast = taxonkit_annotation_local(ch_blast).blast_report
+
+    ch_blast_ncbi = blastn_ncbi(ch_seqs.combine(ch_ncbi_db)).blast_report
+    ch_blast_ncbi = taxonkit_annotation_ncbi(ch_blast_ncbi).blast_report
 
     if (params.filter_regexes != 'NO_FILE') {
       ch_regexes = Channel.fromPath(params.filter_regexes)
-      ch_blast = filter_by_regex(ch_blast.combine(ch_regexes)).blast_filtered
+      ch_blast = filter_by_regex_local(ch_blast.combine(ch_regexes)).blast_filtered
+      ch_blast_ncbi = filter_by_regex_ncbi(ch_blast_ncbi.combine(ch_regexes)).blast_filtered
     }
 
     ch_blast_collect = ch_blast.collectFile(it -> it[2], name: "collected_blast.csv", storeDir: params.outdir, keepHeader: true, skip: 1)
+    
+    ch_blast_ncbi_collect = ch_blast_ncbi.collectFile(it -> it[2], name: "collected_blast_ncbi.csv", storeDir: params.outdir, keepHeader: true, skip: 1)
 
-    filter_best_bitscore(ch_blast)
+    filter_best_bitscore_local(ch_blast)
+
+    filter_best_bitscore_ncbi(ch_blast_ncbi)
 
-    filter_best_bitscore.out.blast_best_bitscore_csv.collectFile(it -> it[1], name: "collected_blast_best_bitscore.csv", storeDir: params.outdir, keepHeader: true, skip: 1)
+    filter_best_bitscore_local.out.blast_best_bitscore_csv.collectFile(it -> it[1], name: "collected_blast_best_bitscore.csv", storeDir: params.outdir, keepHeader: true, skip: 1)
+
+    filter_best_bitscore_ncbi.out.blast_best_bitscore_csv.collectFile(it -> it[1], name: "collected_blast_ncbi_best_bitscore.csv", storeDir: params.outdir, keepHeader: true, skip: 1)
+
 
-    build_report(ch_blast_collect, Channel.fromPath(params.databases))
+    build_report(ch_blast_collect, ch_blast_ncbi_collect, Channel.fromPath(params.databases))
 
     // Build pipeline provenance 
     ch_pipeline_provenance = pipeline_provenance(ch_pipeline_metadata, build_report.out.provenance)
@@ -80,6 +98,7 @@ workflow {
     ch_provenance = hash_seqs.out.provenance
     ch_provenance = ch_provenance.join(seq_qc.out.provenance).map{ it -> [it[0], [it[1]] << it[2]] }
     ch_provenance = ch_provenance.join(blastn.out.provenance.groupTuple()).map{ it -> [it[0], (it[1] + it[2]).flatten() ] } 
+    ch_provenance = ch_provenance.join(blastn_ncbi.out.provenance.groupTuple()).map{ it -> [it[0], (it[1] + it[2]).flatten() ] } 
     //ch_provenance = ch_provenance.join(filter_best_bitscore.out.provenance.groupTuple()).map{ it -> [it[0], (it[1] + it[2]).flatten()] }
     ch_provenance = ch_provenance.join(seq_qc.out.provenance.map{it -> it[0]}.combine(ch_pipeline_provenance)).map{ it -> [it[0], it[1] << it[2]] }
     collect_provenance(ch_provenance)