From 9c8fe6f0f0353da4cd0dff058d283092f000d6af Mon Sep 17 00:00:00 2001 From: Nicolas Brossard Date: Fri, 3 May 2024 10:04:32 -0400 Subject: [PATCH 1/4] Various improvements/bug fixes for the delete_imaging_upload.pl script --- tools/delete_imaging_upload.pl | 141 ++++++++++++++++++++++----------- 1 file changed, 94 insertions(+), 47 deletions(-) diff --git a/tools/delete_imaging_upload.pl b/tools/delete_imaging_upload.pl index 1dee3487b..55ee0dd13 100755 --- a/tools/delete_imaging_upload.pl +++ b/tools/delete_imaging_upload.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/perl -w =pod @@ -10,7 +10,7 @@ =head1 NAME =head1 SYNOPSIS perl delete_imaging_upload.pl [-profile file] [-ignore] [-backup_path basename] [-protocol] [-form] [-uploadID list_of_uploadIDs] - [-type list_of_scan_types] [-defaced] [-basename fileBaseName] [-nosqlbk] [-nofilesbk] + [-type list_of_scan_types] [-defaced] [-basename fileBaseName] [-nosqlbk] [-nofilesbk] [-extra_mysqlcnf file] Available options are: @@ -65,10 +65,19 @@ =head1 SYNOPSIS C, to the backup file is the default behaviour. -dumpOptions : options to add to the mysqldump command. By default, the following options are used - C<--no-create-info --compact --single-transaction --skip-extended-insert --no-tablespaces>. + C<--no-create-info --compact --skip-extended-insert --no-tablespaces>. Example of additional option: C<--column-statistics=0> to disable column statistics flag in mysqldump 8. +-extra_mysqlcnf : MySql config file containing the password to log in the database (see + C for a description of this file's format). + This file should only contain the password stored in the C<@db> array defined in your C file. + You should use that option if you do not have a default C<.mysql.cnf> config file containing the credentials + to log in the database. If this option is used, C will pass the option + C<--defaults-extra-file=file> to the C command when creating a backup of the SQL tables. Note that + only the password is fetched in this file, the host name and user name used to access the database are actually + retrieved from the C file. + =head1 DESCRIPTION This program deletes an imaging upload or specific parts of it from the database and the file system. There are four @@ -114,9 +123,13 @@ =head1 DESCRIPTION The script will also create a file that contains a backup of all the information that was deleted or modified from the database tables. This backup is created using C and contains an C statement for every record erased. -It will be part of the backup archive mentioned above unless option C<-nosqlbk> is used. If sourced back into the database -with C, it should allow the database to be exactly like it was before C was invoked, -provided the database was not modified in the meantime. The SQL backup file will be named C. +When running C the script uses the database credentials in file C<~/.my.cnf> to connect to the database. Option +C<-extra_mysqlcnf> has to be used to specify an alternate credentials file when the default credential file does not exist +(see C for a descrption of MySQL option files format). +The backup produced by C will be part of the backup archive mentioned above unless option C<-nosqlbk> is used. +If sourced back into the database with C, it should allow the database to be exactly like it was before +C was invoked, provided the database was not modified in the meantime. The SQL backup file will +be named C. 2. Delete specific scan types from an archive. The behaviour of the script is identical to the one described above, except that: @@ -167,7 +180,10 @@ =head2 Methods use NeuroDB::objectBroker::ObjectBrokerException; use NeuroDB::objectBroker::ConfigOB; +# Default credentials file used when running mysqldump +use constant DEFAULT_MYSQLDUMP_CNF_FILE => '~/.my.cnf'; +# Default values for various options use constant DEFAULT_PROFILE => 'prod'; use constant DEFAULT_DIE_ON_FILE_ERROR => 1; use constant DEFAULT_NO_FILES_BK => 0; @@ -215,7 +231,8 @@ =head2 Methods BACKUP_PATH => '', UPLOAD_ID => '', BASENAME => '', - DUMP_ADDITIONAL_OPTIONS => '' + DUMP_ADDITIONAL_OPTIONS => '', + EXTRA_MYSQLCNF => '' ); my $scanTypeList = undef; @@ -248,8 +265,11 @@ =head2 Methods 'Delete a file with a specific basename.'], ['-dumpOptions', 'string' , 1, \$options{'DUMP_ADDITIONAL_OPTIONS'}, 'Additional options to use with the mysqldump command. By default, the following options are used when running' - . 'mysqldump: --no-create-info --compact --single-transaction --skip-extended-insert --no-tablespaces.' - . 'Example of additional option: \'--column-statistics=0\' to disable column statistics flag in mysqldump 8.'] + . 'mysqldump: --no-create-info --compact --skip-extended-insert --no-tablespaces.' + . 'Example of additional option: \'--column-statistics=0\' to disable column statistics flag in mysqldump 8.'], + ['-extra_mysqlcnf', 'string' , 1, \$options{'EXTRA_MYSQLCNF'}, + '(OPTIONAL) Path of the extra MySQL configuration file to use'], + ); my $Help = <begin_work; - my $dump_opt = $options{DUMP_ADDITIONAL_OPTIONS}; my(undef, $tmpSQLFile) = $optionsRef->{'NO_SQL_BK'} ? (undef, undef) : tempfile('sql_backup_XXXX', UNLINK => 1); @@ -1717,52 +1745,52 @@ sub deleteUploadsInDatabase { my $nbRecordsDeleted = 0; if(!@$scanTypesToDeleteRef && $optionsRef->{'BASENAME'} eq '') { - $nbRecordsDeleted += &deleteTableData($dbh, 'notification_spool', 'ProcessID', \@IDs, $tmpSQLFile, $dump_opt); + $nbRecordsDeleted += &deleteTableData($dbh, 'notification_spool', 'ProcessID', \@IDs, $tmpSQLFile, $optionsRef); } # If only specific scan types are targeted for deletion, do not delete the entries in # tarchive_files and tarchive_series as these are tied to the archive, not the MINC files if(!@$scanTypesToDeleteRef && $optionsRef->{'BASENAME'} eq '') { my $IDsRef = &getTarchiveSeriesIDs($dbh, $filesRef); - $nbRecordsDeleted += &deleteTableData($dbh, 'tarchive_files', 'TarchiveSeriesID', $IDsRef, $tmpSQLFile, $dump_opt); + $nbRecordsDeleted += &deleteTableData($dbh, 'tarchive_files', 'TarchiveSeriesID', $IDsRef, $tmpSQLFile, $optionsRef); - $nbRecordsDeleted += &deleteTableData($dbh, 'tarchive_series', 'TarchiveSeriesID', $IDsRef, $tmpSQLFile, $dump_opt); + $nbRecordsDeleted += &deleteTableData($dbh, 'tarchive_series', 'TarchiveSeriesID', $IDsRef, $tmpSQLFile, $optionsRef); } @IDs = map { $_->{'ParameterFileID'} } @{ $filesRef->{'parameter_file'} }; - $nbRecordsDeleted += &deleteTableData($dbh, 'parameter_file', 'ParameterFileID', \@IDs, $tmpSQLFile, $dump_opt); + $nbRecordsDeleted += &deleteTableData($dbh, 'parameter_file', 'ParameterFileID', \@IDs, $tmpSQLFile, $optionsRef); @IDs = map { $_->{'IntermedID'} } @{ $filesRef->{'files_intermediary'} }; - $nbRecordsDeleted += &deleteTableData($dbh, 'files_intermediary', 'IntermedID', \@IDs, $tmpSQLFile, $dump_opt); + $nbRecordsDeleted += &deleteTableData($dbh, 'files_intermediary', 'IntermedID', \@IDs, $tmpSQLFile, $optionsRef); # Since all files in files_intermediary are linked to other files in table files, we # have to delete these files first from table files. if(!$optionsRef->{'KEEP_DEFACED'}) { @IDs = map { $_->{'FileID'} } @{ $filesRef->{'files_intermediary'} }; - $nbRecordsDeleted += &deleteTableData($dbh, 'files', 'FileID', \@IDs, $tmpSQLFile, $dump_opt); + $nbRecordsDeleted += &deleteTableData($dbh, 'files', 'FileID', \@IDs, $tmpSQLFile, $optionsRef); } else { &updateFilesIntermediaryTable($dbh, $filesRef, $tmpSQLFile); } @IDs = map { $_->{'BIDSExportedFileID'} } @{ $filesRef->{'bids_export_files'}}; - $nbRecordsDeleted += &deleteTableData($dbh, 'bids_export_files', 'BIDSExportedFileID', \@IDs, $tmpSQLFile, $dump_opt); + $nbRecordsDeleted += &deleteTableData($dbh, 'bids_export_files', 'BIDSExportedFileID', \@IDs, $tmpSQLFile, $optionsRef); @IDs = map { $_->{'FileID'} } @{ $filesRef->{'files'} }; - $nbRecordsDeleted += &deleteTableData($dbh, 'files', 'FileID', \@IDs, $tmpSQLFile, $dump_opt); + $nbRecordsDeleted += &deleteTableData($dbh, 'files', 'FileID', \@IDs, $tmpSQLFile, $optionsRef); @IDs = map { $_->{'ProcessProtocolID'} } @{ $filesRef->{'mri_processing_protocol'} }; - $nbRecordsDeleted += &deleteTableData($dbh, 'mri_processing_protocol', 'ProcessProtocolID', \@IDs, $tmpSQLFile, $dump_opt); + $nbRecordsDeleted += &deleteTableData($dbh, 'mri_processing_protocol', 'ProcessProtocolID', \@IDs, $tmpSQLFile, $optionsRef); @IDs = map { $_->{'ID'} } @{ $filesRef->{'mri_protocol_violated_scans'} }; - $nbRecordsDeleted += &deleteTableData($dbh, 'mri_protocol_violated_scans', 'ID', \@IDs, $tmpSQLFile, $dump_opt); + $nbRecordsDeleted += &deleteTableData($dbh, 'mri_protocol_violated_scans', 'ID', \@IDs, $tmpSQLFile, $optionsRef); @IDs = map { $_->{'LogID'} } @{ $filesRef->{'mri_violations_log'} }; - $nbRecordsDeleted += &deleteTableData($dbh, 'mri_violations_log', 'LogID', \@IDs, $tmpSQLFile, $dump_opt); + $nbRecordsDeleted += &deleteTableData($dbh, 'mri_violations_log', 'LogID', \@IDs, $tmpSQLFile, $optionsRef); @IDs = map { $_->{'ID'} } @{ $filesRef->{'MRICandidateErrors'} }; - $nbRecordsDeleted += &deleteTableData($dbh, 'MRICandidateErrors', 'ID', \@IDs, $tmpSQLFile, $dump_opt); + $nbRecordsDeleted += &deleteTableData($dbh, 'MRICandidateErrors', 'ID', \@IDs, $tmpSQLFile, $optionsRef); - $nbRecordsDeleted += &deleteMriParameterForm($dbh, $filesRef->{'mri_upload'}, $tmpSQLFile, $dump_opt) if $optionsRef->{'DELETE_MRI_PARAMETER_FORM'}; + $nbRecordsDeleted += &deleteMriParameterForm($dbh, $filesRef->{'mri_upload'}, $tmpSQLFile, $optionsRef) if $optionsRef->{'DELETE_MRI_PARAMETER_FORM'}; # Should check instead if the tarchive is not tied to anything # (i.e no associated entries in files, mri_violations_log, etc...) @@ -1772,9 +1800,9 @@ sub deleteUploadsInDatabase { my $tarchiveID = $filesRef->{'mri_upload'}->[0]->{'TarchiveID'}; if(!@$scanTypesToDeleteRef && $optionsRef->{'BASENAME'} eq '') { @IDs = map { $_->{'UploadID'} } @{ $filesRef->{'mri_upload'} }; - $nbRecordsDeleted += &deleteTableData($dbh, 'mri_upload', 'UploadID', \@IDs, $tmpSQLFile, $dump_opt); + $nbRecordsDeleted += &deleteTableData($dbh, 'mri_upload', 'UploadID', \@IDs, $tmpSQLFile, $optionsRef); - $nbRecordsDeleted += &deleteTableData($dbh, 'tarchive', 'TarchiveID', [$tarchiveID], $tmpSQLFile, $dump_opt) if defined $tarchiveID; + $nbRecordsDeleted += &deleteTableData($dbh, 'tarchive', 'TarchiveID', [$tarchiveID], $tmpSQLFile, $optionsRef) if defined $tarchiveID; } &updateSessionTable($dbh, $filesRef->{'mri_upload'}, $tmpSQLFile) unless @$scanTypesToDeleteRef || $optionsRef->{'BASENAME'} ne ''; @@ -1915,7 +1943,7 @@ sub updateFilesIntermediaryTable { =pod -=head3 deleteMriParameterForm($dbh, $mriUploadsRef, $tmpSQLFile, $dump_opt) +=head3 deleteMriParameterForm($dbh, $mriUploadsRef, $tmpSQLFile, $optionsRef) Delete the entries in C (and associated C entry) for the upload(s) passed on the command line. The script also adds an SQL statement in the SQL file whose path is passed as argument to @@ -1928,14 +1956,14 @@ =head3 deleteMriParameterForm($dbh, $mriUploadsRef, $tmpSQLFile, $dump_opt) in the array. The properties stored for each hash are: C, C, C C, C and C. - $tmpSQLFile: path of the SQL file that contains the SQL statements used to restore the deleted records. - - $dump_opt: additional options to use for mysqldump + - $optionsRef: reference to the array that contains the options passed on the command line. RETURNS: - The numbers of records deleted as a result of this operation. =cut sub deleteMriParameterForm { - my($dbh, $mriUploadsRef, $tmpSQLFile, $dump_opt) = @_; + my($dbh, $mriUploadsRef, $tmpSQLFile, $optionsRef) = @_; my @uploadIDs = map { $_->{'UploadID'} } @$mriUploadsRef; my $query = "SELECT f.CommentID FROM flag f " @@ -1951,8 +1979,8 @@ sub deleteMriParameterForm { return if !@commentIDs; my $nbRecordsDeleted = 0; - $nbRecordsDeleted += &deleteTableData($dbh, 'mri_parameter_form', 'CommentID', \@commentIDs, $tmpSQLFile, $dump_opt); - $nbRecordsDeleted += &deleteTableData($dbh, 'flag' , 'CommentID', \@commentIDs, $tmpSQLFile, $dump_opt); + $nbRecordsDeleted += &deleteTableData($dbh, 'mri_parameter_form', 'CommentID', \@commentIDs, $tmpSQLFile, $optionsRef); + $nbRecordsDeleted += &deleteTableData($dbh, 'flag' , 'CommentID', \@commentIDs, $tmpSQLFile, $optionsRef); return $nbRecordsDeleted; } @@ -2062,7 +2090,7 @@ sub getTarchiveSeriesIDs { return [ map { $_->{'TarchiveSeriesID'} } @$tarchiveFilesRef ]; } -=head3 deleteTableData($dbh, $table, $key, $keyValuesRef, $tmpSQLBackupFile, $dump_opt) +=head3 deleteTableData($dbh, $table, $key, $keyValuesRef, $tmpSQLBackupFile, $optionsRef) Deletes records from a database table and adds in a file the SQL statements that allow rewriting the records back in the table. @@ -2074,14 +2102,14 @@ =head3 deleteTableData($dbh, $table, $key, $keyValuesRef, $tmpSQLBackupFile, $du - $key: name of the key used to delete the records. - $keyValuesRef: reference on the list of values that field C<$key> has for the records to delete. - $tmpSQLBackupFile: path of the SQL file that contains the SQL statements used to restore the deleted records. - - $dump_opt: additional options to use for mysqldump + - $optionsRef: reference to the array that contains the options passed on the command line. RETURNS: - The number of records deleted. =cut sub deleteTableData { - my($dbh, $table, $key, $keyValuesRef, $tmpSQLBackupFile, $dump_opt) = @_; + my($dbh, $table, $key, $keyValuesRef, $tmpSQLBackupFile, $optionsRef) = @_; return 0 unless @$keyValuesRef; @@ -2089,14 +2117,14 @@ sub deleteTableData { . join(',', ('?') x @$keyValuesRef) . ')'; - &updateSQLBackupFile($tmpSQLBackupFile, $table, $key, $keyValuesRef, $dump_opt) if $tmpSQLBackupFile; + &updateSQLBackupFile($tmpSQLBackupFile, $table, $key, $keyValuesRef, $optionsRef) if $tmpSQLBackupFile; my $nbRecordsDeleted = $dbh->do($query, undef, @$keyValuesRef); return $nbRecordsDeleted; } -=head3 updateSQLBackupFile($tmpSQLBackupFile, $table, $key, $keyValuesRef, $dump_opt) +=head3 updateSQLBackupFile($tmpSQLBackupFile, $table, $key, $keyValuesRef, $optionsRef) Updates the SQL file with the statements to restore the records whose properties are passed as argument. The block of statements is written at the beginning of the file. @@ -2106,11 +2134,11 @@ =head3 updateSQLBackupFile($tmpSQLBackupFile, $table, $key, $keyValuesRef, $dump - $table: name of the database table. - $key: name of the key used to delete the records. - $keyValuesRef: reference on the list of values that field C<$key> has for the records to delete. - - $dump_opt: additional options to use for mysqldump + - $optionsRef: reference to the array that contains the options passed on the command line. =cut sub updateSQLBackupFile { - my($tmpSQLBackupFile, $table, $key, $keyValuesRef, $dump_opt) = @_; + my($tmpSQLBackupFile, $table, $key, $keyValuesRef, $optionsRef) = @_; # Make sure all keys are quoted before using them in the Unix command my %quotedKeys; @@ -2127,24 +2155,43 @@ sub updateSQLBackupFile { # Run the mysqldump command for the current table and store the # result in $tmpSqlBackupFile (overwrite contents) - my $mysqldumpOptions = "$dump_opt --no-create-info --compact --single-transaction --skip-extended-insert --no-tablespaces"; - - my $warningToIgnore = 'mysqldump: [Warning] Using a password on the command line interface can be insecure.'; + # Note that --defaults-extra-file has to be the first option (it it is provided) + my $mySqldumpOptions = $optionsRef->{'EXTRA_MYSQLCNF'} ne '' + ? sprintf("--defaults-extra-file=%s", quotemeta($optionsRef->{'EXTRA_MYSQLCNF'})) + : ''; + $mySqldumpOptions .= sprintf( + " %s --no-create-info --compact --skip-extended-insert --no-tablespaces", + $optionsRef->{'DUMP_ADDITIONAL_OPTIONS'} + ); + + # Create a temporary file for mySqlDump's stdout and stderr + my($unusedFileHandle, $mySqlDumpOutputFile) = tempfile("mysqlDumpXXXXX", UNLINK => 1); my $cmd = sprintf( - "mysqldump $mysqldumpOptions --where='%s IN (%s)' --result-file=%s -h %s -p%s -u %s %s %s 2>&1 | fgrep -v '$warningToIgnore'", + "mysqldump $mySqldumpOptions --where='%s IN (%s)' --result-file=%s -h %s -u %s %s %s >$mySqlDumpOutputFile 2>&1", $key, join(',', keys %quotedKeys), quotemeta($tmpSQLBackupFile), quotemeta($Settings::db[3]), - quotemeta($Settings::db[2]), quotemeta($Settings::db[1]), quotemeta($Settings::db[0]), $table ); - - system($cmd) != 0 - or die "Cannot run command $cmd. Aborting\n"; - + + + # Execute the command and read the file containing the warnings+errors of the mysqldump command + my $mySqlDumpReturnCode = system($cmd); + open(MYSQLDUMP_OUTPUT, "<$mySqlDumpOutputFile") or die "Cannot read $mySqlDumpOutputFile: $!\n"; + my @mySqlDumpOutput = ; + close(MYSQLDUMP_OUTPUT); + + # Check that command succeeded. Echo the warnings and errors on stdout in all cases. + if ($mySqlDumpReturnCode == 0) { + printf("Backup of table $table succeeded but a warning was issued: %s.\n", join('', @mySqlDumpOutput)) if @mySqlDumpOutput; + } else { + die sprintf("Backup of table $table failed with the following output:\n%sAborting\n", join('', @mySqlDumpOutput)); + } + unlink $mySqlDumpOutputFile or warn "Warning! Cannot delete $mySqlDumpOutputFile: $!.\n"; + # Write back the original lines contained in $tmpSQlBackupFile at the end of the file. # This is so that the mysqldump results are written in the file in the reverse order in # which the mySQL delete statements are made. From 04852830251fddad21c5cec313745fc8482f29d2 Mon Sep 17 00:00:00 2001 From: Nicolas Brossard Date: Fri, 3 May 2024 10:12:30 -0400 Subject: [PATCH 2/4] Update of docs/scripts_md/delete_imaging_upload.md --- docs/scripts_md/delete_imaging_upload.md | 1252 ++++++++++++---------- 1 file changed, 713 insertions(+), 539 deletions(-) diff --git a/docs/scripts_md/delete_imaging_upload.md b/docs/scripts_md/delete_imaging_upload.md index 66700b52f..534578181 100644 --- a/docs/scripts_md/delete_imaging_upload.md +++ b/docs/scripts_md/delete_imaging_upload.md @@ -1,603 +1,777 @@ -# NAME - -delete\_imaging\_upload.pl -- Delete everything that was produced (or part of what was produced) by the imaging pipeline for a given set of imaging uploads - IDs, all associated to the same `tarchive`. - -# SYNOPSIS - -perl delete\_imaging\_upload.pl \[-profile file\] \[-ignore\] \[-backup\_path basename\] \[-protocol\] \[-form\] \[-uploadID list\_of\_uploadIDs\] - \[-type list\_of\_scan\_types\] \[-defaced\] \[-basename fileBaseName\] \[-nosqlbk\] \[-nofilesbk\] - +.\" Automatically generated by Pod::Man 4.09 (Pod::Simple 3.35) +.\" +.\" Standard preamble: +.\" ======================================================================== +.de Sp \" Vertical space (when we can't use .PP) +.if t .sp .5v +.if n .sp +.. +.de Vb \" Begin verbatim text +.ft CW +.nf +.ne \\$1 +.. +.de Ve \" End verbatim text +.ft R +.fi +.. +.\" Set up some character translations and predefined strings. \*(-- will +.\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left +.\" double quote, and \*(R" will give a right double quote. \*(C+ will +.\" give a nicer C++. Capital omega is used to do unbreakable dashes and +.\" therefore won't be available. \*(C` and \*(C' expand to `' in nroff, +.\" nothing in troff, for use with C<>. +.tr \(*W- +.ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p' +.ie n \{\ +. ds -- \(*W- +. ds PI pi +. if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch +. if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\" diablo 12 pitch +. ds L" "" +. ds R" "" +. ds C` "" +. ds C' "" +'br\} +.el\{\ +. ds -- \|\(em\| +. ds PI \(*p +. ds L" `` +. ds R" '' +. ds C` +. ds C' +'br\} +.\" +.\" Escape single quotes in literal strings from groff's Unicode transform. +.ie \n(.g .ds Aq \(aq +.el .ds Aq ' +.\" +.\" If the F register is >0, we'll generate index entries on stderr for +.\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index +.\" entries marked with X<> in POD. Of course, you'll have to process the +.\" output yourself in some meaningful fashion. +.\" +.\" Avoid warning from groff about undefined register 'F'. +.de IX +.. +.if !\nF .nr F 0 +.if \nF>0 \{\ +. de IX +. tm Index:\\$1\t\\n%\t"\\$2" +.. +. if !\nF==2 \{\ +. nr % 0 +. nr F 2 +. \} +.\} +.\" +.\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2). +.\" Fear. Run. Save yourself. No user-serviceable parts. +. \" fudge factors for nroff and troff +.if n \{\ +. ds #H 0 +. ds #V .8m +. ds #F .3m +. ds #[ \f1 +. ds #] \fP +.\} +.if t \{\ +. ds #H ((1u-(\\\\n(.fu%2u))*.13m) +. ds #V .6m +. ds #F 0 +. ds #[ \& +. ds #] \& +.\} +. \" simple accents for nroff and troff +.if n \{\ +. ds ' \& +. ds ` \& +. ds ^ \& +. ds , \& +. ds ~ ~ +. ds / +.\} +.if t \{\ +. ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u" +. ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u' +. ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u' +. ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u' +. ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u' +. ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u' +.\} +. \" troff and (daisy-wheel) nroff accents +.ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V' +.ds 8 \h'\*(#H'\(*b\h'-\*(#H' +.ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#] +.ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H' +.ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u' +.ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#] +.ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#] +.ds ae a\h'-(\w'a'u*4/10)'e +.ds Ae A\h'-(\w'A'u*4/10)'E +. \" corrections for vroff +.if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u' +.if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u' +. \" for low resolution devices (crt and lpr) +.if \n(.H>23 .if \n(.V>19 \ +\{\ +. ds : e +. ds 8 ss +. ds o a +. ds d- d\h'-1'\(ga +. ds D- D\h'-1'\(hy +. ds th \o'bp' +. ds Th \o'LP' +. ds ae ae +. ds Ae AE +.\} +.rm #[ #] #H #V #F C +.\" ======================================================================== +.\" +.IX Title "DELETE_IMAGING_UPLOAD 1" +.TH DELETE_IMAGING_UPLOAD 1 "2024-05-03" "perl v5.26.1" "User Contributed Perl Documentation" +.\" For nroff, turn off justification. Always turn off hyphenation; it makes +.\" way too many mistakes in technical documents. +.if n .ad l +.nh +.SH "NAME" +delete_imaging_upload.pl \-\- Delete everything that was produced (or part of what was produced) by the imaging pipeline for a given set of imaging uploads + IDs, all associated to the same "tarchive". +.SH "SYNOPSIS" +.IX Header "SYNOPSIS" +perl delete_imaging_upload.pl [\-profile file] [\-ignore] [\-backup_path basename] [\-protocol] [\-form] [\-uploadID list_of_uploadIDs] + [\-type list_of_scan_types] [\-defaced] [\-basename fileBaseName] [\-nosqlbk] [\-nofilesbk] [\-extra_mysqlcnf file] +.PP Available options are: - -\-profile : name of the config file in `../dicom-archive/.loris_mri` (defaults to `prod`). - -\-ignore : ignore files whose paths exist in the database but do not exist on the file system. +.PP +\&\-profile : name of the config file in \f(CW\*(C`../dicom\-archive/.loris_mri\*(C'\fR (defaults to \f(CW\*(C`prod\*(C'\fR). +.PP +\&\-ignore : ignore files whose paths exist in the database but do not exist on the file system. Default is to abort if such a file is found, irrespective of whether a backup file will - be created by the script (see `-nofilesbk` and `-nosqlbk`). If this option is used, a + be created by the script (see \f(CW\*(C`\-nofilesbk\*(C'\fR and \f(CW\*(C`\-nosqlbk\*(C'\fR). If this option is used, a warning is issued and program execution continues. - -\-nofilesbk : when creating the backup file for the deleted upload(s), do not backup the files produced by +.PP +\&\-nofilesbk : when creating the backup file for the deleted upload(s), do not backup the files produced by the imaging pipeline (default is to backup these files). - -\-backup\_path <path> : specify the path of the backup file, which by default contains a copy of everything that the - script will delete, both on the file system and in the database (but see `-nofilesbk` and - `-nosqlbk`). The extension `.tar.gz` will be added to this base name to build the name of the final +.PP +\&\-backup_path : specify the path of the backup file, which by default contains a copy of everything that the + script will delete, both on the file system and in the database (but see \f(CW\*(C`\-nofilesbk\*(C'\fR and + \f(CW\*(C`\-nosqlbk\*(C'\fR). The extension \f(CW\*(C`.tar.gz\*(C'\fR will be added to this base name to build the name of the final backup file. If a file with this resulting name already exists, an error message is shown and the script - will abort. Note that `path` can be an absolute path or a path relative to the current directory. A - backup file is always created unless options `-nofilesbk` and `-nosqlbk` are both used. By default, the - backup file name is `imaging_upload_backup.tar.gz` and is written in the current directory. Option - `-backup_path` cannot be used if `-nofilesbk` and `-nosqlbk` are also used. - -\-basename fileBaseName : basename of the file to delete. The file is assumed to either exist in table `files` or table - `parameter_file`. This option should be used when targeting a specific (unique) file for deletion. + will abort. Note that \f(CW\*(C`path\*(C'\fR can be an absolute path or a path relative to the current directory. A + backup file is always created unless options \f(CW\*(C`\-nofilesbk\*(C'\fR and \f(CW\*(C`\-nosqlbk\*(C'\fR are both used. By default, the + backup file name is \f(CW\*(C`imaging_upload_backup.tar.gz\*(C'\fR and is written in the current directory. Option + \f(CW\*(C`\-backup_path\*(C'\fR cannot be used if \f(CW\*(C`\-nofilesbk\*(C'\fR and \f(CW\*(C`\-nosqlbk\*(C'\fR are also used. +.PP +\&\-basename fileBaseName : basename of the file to delete. The file is assumed to either exist in table \f(CW\*(C`files\*(C'\fR or table + \f(CW\*(C`parameter_file\*(C'\fR. This option should be used when targeting a specific (unique) file for deletion. Note that the file will be deleted from both the database and the filesystem. This option cannot be - used with options `-defaced` and `-form`. - -\-uploadID : comma-separated list of upload IDs (found in table `mri_upload`) to delete. The program will - abort if the list contains an upload ID that does not exist. Also, all upload IDs must - have the same `tarchive` ID (which can be `NULL`). - -\-protocol : delete the imaging protocol(s) in table `mri_processing_protocol` associated to either the - upload(s) specified via the `-uploadID` option or any file that was produced using this (these) + used with options \f(CW\*(C`\-defaced\*(C'\fR and \f(CW\*(C`\-form\*(C'\fR. +.PP +\&\-uploadID : comma-separated list of upload IDs (found in table \f(CW\*(C`mri_upload\*(C'\fR) to delete. The program will + abort if the list contains an upload \s-1ID\s0 that does not exist. Also, all upload IDs must + have the same \f(CW\*(C`tarchive\*(C'\fR \s-1ID\s0 (which can be \f(CW\*(C`NULL\*(C'\fR). +.PP +\&\-protocol : delete the imaging protocol(s) in table \f(CW\*(C`mri_processing_protocol\*(C'\fR associated to either the + upload(s) specified via the \f(CW\*(C`\-uploadID\*(C'\fR option or any file that was produced using this (these) upload(s). Let F be the set of files directly or indirectly associated to the upload(s) to delete. - This option must be used if there is at least one record in `mri_processing_protocol` that is tied + This option must be used if there is at least one record in \f(CW\*(C`mri_processing_protocol\*(C'\fR that is tied only to files in F. Protocols that are tied to files not in F are never deleted. If the files in F do not have a protocol associated to them, the switch is ignored if used. - -\-form : delete the entries in `mri_parameter_form` associated to the upload(s) passed on - the command line, if any (default is NOT to delete them). - -\-type : comma-separated list of scan type names to delete. All the names must exist in table `mri_scan_type` or - the script will issue an error. This option cannot be used in conjunction with `-defaced`. - -\-defaced : fetch the scan types listed in config setting `modalities_to_delete` and perform a deletion of these scan - types as if their names were used with option `-type`. Once all deletions are done, set the `SourceFileID` - and `TarchiveSource` of all the defaced files in table <files> to `NULL` and to the tarchive ID of the - upload(s) whose arguments were passed to `-uploadID`, respectively. - -\-nosqlbk : when creating the backup file, do not add to it an SQL file that contains the statements used to restore +.PP +\&\-form : delete the entries in \f(CW\*(C`mri_parameter_form\*(C'\fR associated to the upload(s) passed on + the command line, if any (default is \s-1NOT\s0 to delete them). +.PP +\&\-type : comma-separated list of scan type names to delete. All the names must exist in table \f(CW\*(C`mri_scan_type\*(C'\fR or + the script will issue an error. This option cannot be used in conjunction with \f(CW\*(C`\-defaced\*(C'\fR. +.PP +\&\-defaced : fetch the scan types listed in config setting \f(CW\*(C`modalities_to_delete\*(C'\fR and perform a deletion of these scan + types as if their names were used with option \f(CW\*(C`\-type\*(C'\fR. Once all deletions are done, set the \f(CW\*(C`SourceFileID\*(C'\fR + and \f(CW\*(C`TarchiveSource\*(C'\fR of all the defaced files in table to \f(CW\*(C`NULL\*(C'\fR and to the tarchive \s-1ID\s0 of the + upload(s) whose arguments were passed to \f(CW\*(C`\-uploadID\*(C'\fR, respectively. +.PP +\&\-nosqlbk : when creating the backup file, do not add to it an \s-1SQL\s0 file that contains the statements used to restore the database to the state it had before the script was invoked. Adding this file, which will be named - `imaging_upload_restore.sql`, to the backup file is the default behaviour. - -\-dumpOptions : options to add to the mysqldump command. By default, the following options are used - `--no-create-info --compact --single-transaction --skip-extended-insert --no-tablespaces`. - Example of additional option: `--column-statistics=0` to disable column statistics flag in + \f(CW\*(C`imaging_upload_restore.sql\*(C'\fR, to the backup file is the default behaviour. +.PP +\&\-dumpOptions : options to add to the mysqldump command. By default, the following options are used + \f(CW\*(C`\-\-no\-create\-info \-\-compact \-\-skip\-extended\-insert \-\-no\-tablespaces\*(C'\fR. + Example of additional option: \f(CW\*(C`\-\-column\-statistics=0\*(C'\fR to disable column statistics flag in mysqldump 8. - -# DESCRIPTION - +.PP +\&\-extra_mysqlcnf : MySql config file containing the password to log in the database (see + \f(CW\*(C`https://dev.mysql.com/doc/refman/8.0/en/option\-files.html\*(C'\fR for a description of this file's format). + This file should only contain the password stored in the \f(CW@db\fR array defined in your \f(CW\*(C`prod\*(C'\fR file. + You should use that option if you do not have a default \f(CW\*(C`.mysql.cnf\*(C'\fR config file containing the credentials + to log in the database. If this option is used, \f(CW\*(C`delete_imaging_upload.pl\*(C'\fR will pass the option + \f(CW\*(C`\-\-defaults\-extra\-file=file\*(C'\fR to the \f(CW\*(C`mysqldump\*(C'\fR command when creating a backup of the \s-1SQL\s0 tables. Note that + only the password is fetched in this file, the host name and user name used to access the database are actually + retrieved from the \f(CW\*(C`prod\*(C'\fR file. +.SH "DESCRIPTION" +.IX Header "DESCRIPTION" This program deletes an imaging upload or specific parts of it from the database and the file system. There are four possible ways in which this script can be used: - -1\. Delete everything that is associated to the upload ID(s). Basically, for uploads on which the MRI pipeline was +.PP +1. Delete everything that is associated to the upload \s-1ID\s0(s). Basically, for uploads on which the \s-1MRI\s0 pipeline was successfully run, this removes all records tied to the upload in the following tables: - a) `notification_spool` - b) `files` - c) `tarchive_series` and `tarchive_files` - d) `mri_protocol_violated_scans`, `MRICandidateErrors` and `mri_violations_log` - e) `files_intermediary`. - f) `parameter_file` - g) `tarchive` - h) `mri_upload` - i) `bids_export_files` - j) `mri_processing_protocol` if option `-protocol` is used (see below) - k) `mri_parameter_form` if option `-form` is used (see below) - + a) \f(CW\*(C`notification_spool\*(C'\fR + b) \f(CW\*(C`files\*(C'\fR + c) \f(CW\*(C`tarchive_series\*(C'\fR and \f(CW\*(C`tarchive_files\*(C'\fR + d) \f(CW\*(C`mri_protocol_violated_scans\*(C'\fR, \f(CW\*(C`MRICandidateErrors\*(C'\fR and \f(CW\*(C`mri_violations_log\*(C'\fR + e) \f(CW\*(C`files_intermediary\*(C'\fR. + f) \f(CW\*(C`parameter_file\*(C'\fR + g) \f(CW\*(C`tarchive\*(C'\fR + h) \f(CW\*(C`mri_upload\*(C'\fR + i) \f(CW\*(C`bids_export_files\*(C'\fR + j) \f(CW\*(C`mri_processing_protocol\*(C'\fR if option \f(CW\*(C`\-protocol\*(C'\fR is used (see below) + k) \f(CW\*(C`mri_parameter_form\*(C'\fR if option \f(CW\*(C`\-form\*(C'\fR is used (see below) +.PP All the deletions and modifications performed in the database are done as part of a single transaction, so they either -all succeed or a rollback is performed and the database is not modified in any way. The ID of the upload to delete -is specified via option `-uploadID`. More than one upload can be deleted if they all have the same `TarchiveID` -in table `mri_upload`: option `-uploadID` can take as argument a comma-separated list of upload IDs for this case. -If an upload that is deleted is the only one that was associated to a given session, the script will set the `Scan_done` -value for that session to 'N'. If option `-form` is used, the `mri_parameter_form` and its associated `flag` record -are also deleted, for each deleted upload. If option `-protocol` is used and if there is a record in table -`mri_processing_protocol` that is tied only to the deleted upload(s), then that record is also deleted. - -`delete_imaging_upload.pl` cannot be used to delete an upload that has an associated MINC file that has been QCed. -In other words, if there is a MINC file tied to the upload that is targeted for deletion and if that MINC file has -an associated record in table `files_qcstatus` or `feedback_mri_comments`, the script will issue an error message +all succeed or a rollback is performed and the database is not modified in any way. The \s-1ID\s0 of the upload to delete +is specified via option \f(CW\*(C`\-uploadID\*(C'\fR. More than one upload can be deleted if they all have the same \f(CW\*(C`TarchiveID\*(C'\fR +in table \f(CW\*(C`mri_upload\*(C'\fR: option \f(CW\*(C`\-uploadID\*(C'\fR can take as argument a comma-separated list of upload IDs for this case. +If an upload that is deleted is the only one that was associated to a given session, the script will set the \f(CW\*(C`Scan_done\*(C'\fR +value for that session to 'N'. If option \f(CW\*(C`\-form\*(C'\fR is used, the \f(CW\*(C`mri_parameter_form\*(C'\fR and its associated \f(CW\*(C`flag\*(C'\fR record +are also deleted, for each deleted upload. If option \f(CW\*(C`\-protocol\*(C'\fR is used and if there is a record in table +\&\f(CW\*(C`mri_processing_protocol\*(C'\fR that is tied only to the deleted upload(s), then that record is also deleted. +.PP +\&\f(CW\*(C`delete_imaging_upload.pl\*(C'\fR cannot be used to delete an upload that has an associated \s-1MINC\s0 file that has been QCed. +In other words, if there is a \s-1MINC\s0 file tied to the upload that is targeted for deletion and if that \s-1MINC\s0 file has +an associated record in table \f(CW\*(C`files_qcstatus\*(C'\fR or \f(CW\*(C`feedback_mri_comments\*(C'\fR, the script will issue an error message and exit. - +.PP Before deleting any records in the database, the script will verify that all the records in tables a) through j) that -represent file names (e.g. a record in `files`, a protocol file in `mri_processing_protocol`, etc...) refer to files +represent file names (e.g. a record in \f(CW\*(C`files\*(C'\fR, a protocol file in \f(CW\*(C`mri_processing_protocol\*(C'\fR, etc...) refer to files that actually exist on the file system. If it finds a record that does not meet that criterion, the script issues an -error message and exits, leaving the database untouched. To avoid this check, use option `-ignore`. Each time a file +error message and exits, leaving the database untouched. To avoid this check, use option \f(CW\*(C`\-ignore\*(C'\fR. Each time a file record is deleted, the file it refers to on the file system is also deleted. A backup will be created by -`delete_imaging_upload.pl` of all the files that were deleted during execution. Option `-nofilesbk` can be used to -prevent this. If created, the backup file will be named `imaging_upload_backup.tar.gz`. This name can be changed with -option `-backup_path`. Note that the file paths inside this backup archive are absolute. To restore the files in the archive, -one must use `tar` with option `--absolute-names`. - +\&\f(CW\*(C`delete_imaging_upload.pl\*(C'\fR of all the files that were deleted during execution. Option \f(CW\*(C`\-nofilesbk\*(C'\fR can be used to +prevent this. If created, the backup file will be named \f(CW\*(C`imaging_upload_backup.tar.gz\*(C'\fR. This name can be changed with +option \f(CW\*(C`\-backup_path\*(C'\fR. Note that the file paths inside this backup archive are absolute. To restore the files in the archive, +one must use \f(CW\*(C`tar\*(C'\fR with option \f(CW\*(C`\-\-absolute\-names\*(C'\fR. +.PP The script will also create a file that contains a backup of all the information that was deleted or modified from the -database tables. This backup is created using `mysqldump` and contains an `INSERT` statement for every record erased. -It will be part of the backup archive mentioned above unless option `-nosqlbk` is used. If sourced back into the database -with `mysql`, it should allow the database to be exactly like it was before `delete_imaging_upload.pl` was invoked, -provided the database was not modified in the meantime. The SQL backup file will be named `imaging_upload_restore.sql`. - -2\. Delete specific scan types from an archive. The behaviour of the script is identical to the one described above, except +database tables. This backup is created using \f(CW\*(C`mysqldump\*(C'\fR and contains an \f(CW\*(C`INSERT\*(C'\fR statement for every record erased. +When running \f(CW\*(C`mysqldump\*(C'\fR the script uses the database credentials in file \f(CW\*(C`~/.my.cnf\*(C'\fR to connect to the database. Option +\&\f(CW\*(C`\-extra_mysqlcnf\*(C'\fR has to be used to specify an alternate credentials file when the default credential file does not exist +(see \f(CW\*(C`https://dev.mysql.com/doc/refman/8.0/en/option\-files.html\*(C'\fR for a descrption of MySQL option files format). +The backup produced by \f(CW\*(C`mysqldump\*(C'\fR will be part of the backup archive mentioned above unless option \f(CW\*(C`\-nosqlbk\*(C'\fR is used. +If sourced back into the database with \f(CW\*(C`mysql\*(C'\fR, it should allow the database to be exactly like it was before +\&\f(CW\*(C`delete_imaging_upload.pl\*(C'\fR was invoked, provided the database was not modified in the meantime. The \s-1SQL\s0 backup file will +be named \f(CW\*(C`imaging_upload_restore.sql\*(C'\fR. +.PP +2. Delete specific scan types from an archive. The behaviour of the script is identical to the one described above, except that: - a) the deletions are limited to MINC files of a specific scan type: use option `-type` with a comma-separated list + a) the deletions are limited to \s-1MINC\s0 files of a specific scan type: use option \f(CW\*(C`\-type\*(C'\fR with a comma-separated list of scan type names to specify which ones. - b) everything associated to the MINC files deleted in a) is also deleted: this includes the processed files in - `files_intermediary` and the records in `mri_violations_log`. - c) if `-protocol` is used and there is an entry in table `mri_processing_protocol` that is tied only to the files + b) everything associated to the \s-1MINC\s0 files deleted in a) is also deleted: this includes the processed files in + \f(CW\*(C`files_intermediary\*(C'\fR and the records in \f(CW\*(C`mri_violations_log\*(C'\fR. + c) if \f(CW\*(C`\-protocol\*(C'\fR is used and there is an entry in table \f(CW\*(C`mri_processing_protocol\*(C'\fR that is tied only to the files deleted in a), then that record is also deleted. - d) tables `tarchive_series`, `tarchive_files`, `tarchive`, `mri_upload`, `notification_spool`, `MRICandidateErrors` - and `mri_parameter_form` are never modified. - Note that option `-type` cannot be used in conjunction with either option `-form` or option `-defaced`. - -3\. Replace MINC files with their defaced counterparts. This is the behaviour obtained when option `-defaced` is used. As far as + d) tables \f(CW\*(C`tarchive_series\*(C'\fR, \f(CW\*(C`tarchive_files\*(C'\fR, \f(CW\*(C`tarchive\*(C'\fR, \f(CW\*(C`mri_upload\*(C'\fR, \f(CW\*(C`notification_spool\*(C'\fR, \f(CW\*(C`MRICandidateErrors\*(C'\fR + and \f(CW\*(C`mri_parameter_form\*(C'\fR are never modified. + Note that option \f(CW\*(C`\-type\*(C'\fR cannot be used in conjunction with either option \f(CW\*(C`\-form\*(C'\fR or option \f(CW\*(C`\-defaced\*(C'\fR. +.PP +3. Replace \s-1MINC\s0 files with their defaced counterparts. This is the behaviour obtained when option \f(CW\*(C`\-defaced\*(C'\fR is used. As far as deletions go, the behaviour of the script in this case is identical to the one described in 2), except that the list of - scan types to delete is fetched from the config setting `modalities_to_deface`. Use of option `-defaced` is not permitted - in conjunction with option `-type` or option `-form`. Once all deletions are made, the script will change the `SourceFileID` - of all defaced files to `NULL` and set the `TarchiveSource` of all defaced files to the `TarchiveID` of the upload(s). - This effectively "replaces" the original MINC files with their corresponding defaced versions. Note that the script will issue + scan types to delete is fetched from the config setting \f(CW\*(C`modalities_to_deface\*(C'\fR. Use of option \f(CW\*(C`\-defaced\*(C'\fR is not permitted + in conjunction with option \f(CW\*(C`\-type\*(C'\fR or option \f(CW\*(C`\-form\*(C'\fR. Once all deletions are made, the script will change the \f(CW\*(C`SourceFileID\*(C'\fR + of all defaced files to \f(CW\*(C`NULL\*(C'\fR and set the \f(CW\*(C`TarchiveSource\*(C'\fR of all defaced files to the \f(CW\*(C`TarchiveID\*(C'\fR of the upload(s). + This effectively \*(L"replaces\*(R" the original \s-1MINC\s0 files with their corresponding defaced versions. Note that the script will issue an error message and abort, leaving the database and file system untouched, if: - a) A MINC file should have a corresponding defaced file but does not. - b) A MINC file that has been defaced has an associated file that is not a defaced file. - -4\. Delete a specific file that exists in table `files` or table `parameter_file` (in which case it will be associated to parameter - `check_pic_filename`, `check_nii_filename`, `check_bval_filename` or `check_bvec_filename`). Note that once a file is - deleted, all the files that have been derived/built using this deleted file will also be deleted. Use option `-basename fileBaseName`, - where `fileBaseName` is the basename of the file to delete, along with option `-uploadID` to use the script this way. - -## Methods - -### printExitMessage($filesRef, $scanTypesToDeleteRef, $deleteResultsRef) - -Prints an appropriate message before exiting. - -INPUTS: - - $filesRef: reference to the array that contains the file information for all the files + a) A \s-1MINC\s0 file should have a corresponding defaced file but does not. + b) A \s-1MINC\s0 file that has been defaced has an associated file that is not a defaced file. +.PP +4. Delete a specific file that exists in table \f(CW\*(C`files\*(C'\fR or table \f(CW\*(C`parameter_file\*(C'\fR (in which case it will be associated to parameter + \f(CW\*(C`check_pic_filename\*(C'\fR, \f(CW\*(C`check_nii_filename\*(C'\fR, \f(CW\*(C`check_bval_filename\*(C'\fR or \f(CW\*(C`check_bvec_filename\*(C'\fR). Note that once a file is + deleted, all the files that have been derived/built using this deleted file will also be deleted. Use option \f(CW\*(C`\-basename fileBaseName\*(C'\fR, + where \f(CW\*(C`fileBaseName\*(C'\fR is the basename of the file to delete, along with option \f(CW\*(C`\-uploadID\*(C'\fR to use the script this way. +.SS "Methods" +.IX Subsection "Methods" +\fIprintExitMessage($filesRef, \f(CI$scanTypesToDeleteRef\fI, \f(CI$deleteResultsRef\fI)\fR +.IX Subsection "printExitMessage($filesRef, $scanTypesToDeleteRef, $deleteResultsRef)" +.PP +Prints an appropriate message before exiting. +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$filesRef:\fR reference to the array that contains the file information for all the files that are associated to the upload(s) passed on the command line. - - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - - $deleteResultsRef: reference on the hash that contains the result of the deletion of the records in the - database. - -### prettyListPrint($listRef, $andOr) - -Pretty prints a list in string form (e.g "1, 2, 3 and 4" or "7, 8 or 9"). - -INPUTS: - - $listRef: the list of elements to print, separated by commas. - - $andOr: whether to join the last element with the rest of the elements using an 'and' or an 'or'. - -### getMriUploadFiles($dbh, $uploadIDsRef) - -Finds the list of `mri_upload` that match the upload IDs passed on the command line -via option `-uploadID`. - -INPUTS: - - $dbh: database handle reference. - - $uploadIDsRef: reference to the array that contains the upload IDs passed on the command line. - -RETURNS: - - a reference on an array that contains the `mri_ipload`s retrieved from the database. - -### getTarchiveFiles($dbh, $tarchiveID, $tarchiveLibraryDir) - -Retrieves from the database the `tarchive` associated to the upload ID(s) passed on the command line. - -INPUTS: - - $dbh: database handle reference. - - $tarchiveID: ID of the `tarchive` to retrieve (can be `undef`). - - $tarchiveLibraryDir: config setting `tarchiveLibraryDir` (can be `undef` if none is defined in the + \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. + \- \f(CW$deleteResultsRef:\fR reference on the hash that contains the result of the deletion of the records in the + database. +.PP +\fIprettyListPrint($listRef, \f(CI$andOr\fI)\fR +.IX Subsection "prettyListPrint($listRef, $andOr)" +.PP +Pretty prints a list in string form (e.g \*(L"1, 2, 3 and 4\*(R" or \*(L"7, 8 or 9\*(R"). +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$listRef:\fR the list of elements to print, separated by commas. + \- \f(CW$andOr:\fR whether to join the last element with the rest of the elements using an 'and' or an 'or'. +.PP +\fIgetMriUploadFiles($dbh, \f(CI$uploadIDsRef\fI)\fR +.IX Subsection "getMriUploadFiles($dbh, $uploadIDsRef)" +.PP +Finds the list of \f(CW\*(C`mri_upload\*(C'\fR that match the upload IDs passed on the command line +via option \f(CW\*(C`\-uploadID\*(C'\fR. +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$dbh:\fR database handle reference. + \- \f(CW$uploadIDsRef:\fR reference to the array that contains the upload IDs passed on the command line. +.PP +\&\s-1RETURNS:\s0 + \- a reference on an array that contains the \f(CW\*(C`mri_ipload\*(C'\fRs retrieved from the database. +.PP +\fIgetTarchiveFiles($dbh, \f(CI$tarchiveID\fI, \f(CI$tarchiveLibraryDir\fI)\fR +.IX Subsection "getTarchiveFiles($dbh, $tarchiveID, $tarchiveLibraryDir)" +.PP +Retrieves from the database the \f(CW\*(C`tarchive\*(C'\fR associated to the upload \s-1ID\s0(s) passed on the command line. +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$dbh:\fR database handle reference. + \- \f(CW$tarchiveID:\fR \s-1ID\s0 of the \f(CW\*(C`tarchive\*(C'\fR to retrieve (can be \f(CW\*(C`undef\*(C'\fR). + \- \f(CW$tarchiveLibraryDir:\fR config setting \f(CW\*(C`tarchiveLibraryDir\*(C'\fR (can be \f(CW\*(C`undef\*(C'\fR if none is defined in the database). - -RETURNS: - - a reference on an array that contains the `tarchive` associated to the upload ID(s) passed on the command line. - This array can contain one element (if the uploads are all tied to the same `tarchive`) or be empty (if all the - uploads have a `TarchiveID` set to `NULL`). If the `ArchiveLocation` for the `tarchive` is a relative path and - if config setting `tarchiveLibraryDir`is not defined, the return value is `undef`, indicating that something is +.PP +\&\s-1RETURNS:\s0 + \- a reference on an array that contains the \f(CW\*(C`tarchive\*(C'\fR associated to the upload \s-1ID\s0(s) passed on the command line. + This array can contain one element (if the uploads are all tied to the same \f(CW\*(C`tarchive\*(C'\fR) or be empty (if all the + uploads have a \f(CW\*(C`TarchiveID\*(C'\fR set to \f(CW\*(C`NULL\*(C'\fR). If the \f(CW\*(C`ArchiveLocation\*(C'\fR for the \f(CW\*(C`tarchive\*(C'\fR is a relative path and + if config setting \f(CW\*(C`tarchiveLibraryDir\*(C'\fRis not defined, the return value is \f(CW\*(C`undef\*(C'\fR, indicating that something is wrong. - -### validateMriUploads($mriUploadsRef, $optionsRef, $scanTypeList) - +.PP +\fIvalidateMriUploads($mriUploadsRef, \f(CI$optionsRef\fI, \f(CI$scanTypeList\fI)\fR +.IX Subsection "validateMriUploads($mriUploadsRef, $optionsRef, $scanTypeList)" +.PP Validates that the list of upload IDs passed on the commamnd line are valid arguments for the script. It one of them is invalid, an error message is displayed and the program exits. - -INPUTS: - - $mriUploadsRef: reference on an array of hashes containing the uploads to delete. Accessed like this: - `$mriUploadsRef->[0]->{'TarchiveID'}`(this would return the `TarchiveID` of the first `mri_upload` - in the array. The properties stored for each hash are: `UploadID`, `TarchiveID`, `FullPath` - `Inserting`, `InsertionComplete` and `SessionID`. - - $optionsRef: reference to the array that contains the options passed on the command line. - - $scanTypeList: list of scan types to delete passed on the command line. - -### getMriProcessingProtocolFilesRef($dbh, $filesRef) - -Finds the list of `ProcessingProtocolID`s to delete, namely those in table -`mri_processing_protocol` associated to the files to delete, and \*only\* to +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$mriUploadsRef:\fR reference on an array of hashes containing the uploads to delete. Accessed like this: + \f(CW\*(C`$mriUploadsRef\->[0]\->{\*(AqTarchiveID\*(Aq}\*(C'\fR(this would return the \f(CW\*(C`TarchiveID\*(C'\fR of the first \f(CW\*(C`mri_upload\*(C'\fR + in the array. The properties stored for each hash are: \f(CW\*(C`UploadID\*(C'\fR, \f(CW\*(C`TarchiveID\*(C'\fR, \f(CW\*(C`FullPath\*(C'\fR + \f(CW\*(C`Inserting\*(C'\fR, \f(CW\*(C`InsertionComplete\*(C'\fR and \f(CW\*(C`SessionID\*(C'\fR. + \- \f(CW$optionsRef:\fR reference to the array that contains the options passed on the command line. + \- \f(CW$scanTypeList:\fR list of scan types to delete passed on the command line. +.PP +\fIgetMriProcessingProtocolFilesRef($dbh, \f(CI$filesRef\fI)\fR +.IX Subsection "getMriProcessingProtocolFilesRef($dbh, $filesRef)" +.PP +Finds the list of \f(CW\*(C`ProcessingProtocolID\*(C'\fRs to delete, namely those in table +\&\f(CW\*(C`mri_processing_protocol\*(C'\fR associated to the files to delete, and *only* to those files that are going to be deleted. - -INPUTS: - - $dbh: database handle reference. - - $filesRef: reference to the array that contains the file informations for all the files +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$dbh:\fR database handle reference. + \- \f(CW$filesRef:\fR reference to the array that contains the file informations for all the files that are associated to the upload(s) passed on the command line. - -RETURNS: - - reference on an array that contains the `ProcessProtocolID` in table `mri_processing_protocol` - associated to the files to delete. This array has two keys: `ProcessProtocolID` => the protocol - process ID found in table `mri_processing_protocol` and `FullPath` => the value of `ProtocolFile` +.PP +\&\s-1RETURNS:\s0 + \- reference on an array that contains the \f(CW\*(C`ProcessProtocolID\*(C'\fR in table \f(CW\*(C`mri_processing_protocol\*(C'\fR + associated to the files to delete. This array has two keys: \f(CW\*(C`ProcessProtocolID\*(C'\fR => the protocol + process \s-1ID\s0 found in table \f(CW\*(C`mri_processing_protocol\*(C'\fR and \f(CW\*(C`FullPath\*(C'\fR => the value of \f(CW\*(C`ProtocolFile\*(C'\fR in the same table. - -### hasQcOrComment($dbh, $mriUploadsRef) - -Determines if any of the MINC files associated to the `tarchive` have QC +.PP +\fIhasQcOrComment($dbh, \f(CI$mriUploadsRef\fI)\fR +.IX Subsection "hasQcOrComment($dbh, $mriUploadsRef)" +.PP +Determines if any of the \s-1MINC\s0 files associated to the \f(CW\*(C`tarchive\*(C'\fR have \s-1QC\s0 information associated to them by looking at the contents of tables -`files_qcstatus` and `feedback_mri_comments`. - -INPUTS: - - $dbh: database handle reference. - - $mriUploadsRef: reference on an array of hashes containing the uploads to delete. Accessed like this: - `$mriUploadsRef->[0]->{'TarchiveID'}`(this would return the `TarchiveID` of the first `mri_upload` - in the array. The properties stored for each hash are: `UploadID`, `TarchiveID`, `FullPath` - `Inserting`, `InsertionComplete` and `SessionID`. - -RETURNS: - - 1 if there is QC information associated to the DICOM archive(s), 0 otherwise. - -### getFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef) - -Get the absolute paths of all the files associated to a DICOM archive that are listed in -table `files`. - -INPUTS: - - $dbhr : database handle reference. - - $tarchiveID: ID of the DICOM archive. - - $dataDirBasePath: config value of setting `dataDirBasePath`. - - $scanTypesToDeleteRef: reference to the array that contains the list of names of scan types to delete. - - $optionsRef: reference on the array of command line options. - -RETURNS: - - an array of hash references. Each hash has three keys: `FileID` => ID of a file in table `files`, - `File` => value of column `File` for the file with the given ID and `FullPath` => absolute path - for the file with the given ID. - -### getIntermediaryFilesRef($dbh, $filesRef, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef) - +\&\f(CW\*(C`files_qcstatus\*(C'\fR and \f(CW\*(C`feedback_mri_comments\*(C'\fR. +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$dbh:\fR database handle reference. + \- \f(CW$mriUploadsRef:\fR reference on an array of hashes containing the uploads to delete. Accessed like this: + \f(CW\*(C`$mriUploadsRef\->[0]\->{\*(AqTarchiveID\*(Aq}\*(C'\fR(this would return the \f(CW\*(C`TarchiveID\*(C'\fR of the first \f(CW\*(C`mri_upload\*(C'\fR + in the array. The properties stored for each hash are: \f(CW\*(C`UploadID\*(C'\fR, \f(CW\*(C`TarchiveID\*(C'\fR, \f(CW\*(C`FullPath\*(C'\fR + \f(CW\*(C`Inserting\*(C'\fR, \f(CW\*(C`InsertionComplete\*(C'\fR and \f(CW\*(C`SessionID\*(C'\fR. +.PP +\&\s-1RETURNS:\s0 + \- 1 if there is \s-1QC\s0 information associated to the \s-1DICOM\s0 archive(s), 0 otherwise. +.PP +\fIgetFilesRef($dbh, \f(CI$tarchiveID\fI, \f(CI$dataDirBasePath\fI, \f(CI$scanTypesToDeleteRef\fI, \f(CI$optionsRef\fI)\fR +.IX Subsection "getFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef)" +.PP +Get the absolute paths of all the files associated to a \s-1DICOM\s0 archive that are listed in +table \f(CW\*(C`files\*(C'\fR. +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$dbhr\fR : database handle reference. + \- \f(CW$tarchiveID:\fR \s-1ID\s0 of the \s-1DICOM\s0 archive. + \- \f(CW$dataDirBasePath:\fR config value of setting \f(CW\*(C`dataDirBasePath\*(C'\fR. + \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of names of scan types to delete. + \- \f(CW$optionsRef:\fR reference on the array of command line options. +.PP +\&\s-1RETURNS:\s0 + \- an array of hash references. Each hash has three keys: \f(CW\*(C`FileID\*(C'\fR => \s-1ID\s0 of a file in table \f(CW\*(C`files\*(C'\fR, + \f(CW\*(C`File\*(C'\fR => value of column \f(CW\*(C`File\*(C'\fR for the file with the given \s-1ID\s0 and \f(CW\*(C`FullPath\*(C'\fR => absolute path + for the file with the given \s-1ID.\s0 +.PP +\fIgetIntermediaryFilesRef($dbh, \f(CI$filesRef\fI, \f(CI$tarchiveID\fI, \f(CI$dataDirBasePath\fI, \f(CI$scanTypesToDeleteRef\fI, \f(CI$optionsRef\fI)\fR +.IX Subsection "getIntermediaryFilesRef($dbh, $filesRef, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef)" +.PP Get the absolute paths of all the intermediary files associated to an archive -that are listed in table `files_intermediary`. - -INPUTS: - - $dbhr : database handle reference. - - $filesRef: reference on the hash of all files. - - $tarchiveID: ID of the DICOM archive. - - $dataDirBasePath: config value of setting `dataDirBasePath`. - - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - - $optionsRef: reference on the array of command line options. - -RETURNS: - - an array of hash references. Each hash has seven keys: `IntermedID` => ID of a file in - table `files_intermediary`, `Input_FileID` => ID of the file that was used as input to create - the intermediary file, `Output_FileID` ID of the output file, `FileID` => ID of this file in - table `files`, `File` => value of column `File` in table `files` for the file with the given - ID, `SourceFileID` value of column `SourceFileID` for the intermediary file and - `FullPath` => absolute path of the file with the given ID. - -### getParameterFilesRef($dbh, $filesRef, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef) - -Gets the records to delete from table parameter\_file. - -INPUTS: - - $dbhr : database handle reference. - - $filesRef: reference on the hash of all files. - - $tarchiveID: ID of the DICOM archive. - - $dataDirBasePath: config value of setting `dataDirBasePath`. - - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - - $optionsRef: reference on the array of command line options. - -RETURNS: - - an array of hash references. Each hash has four keys: `FileID` => FileID of a file - in table `parameter_file`, `Value` => value of column `Value` in table `parameter_file` - for the file with the given ID, `Name` => name of the parameter and `FullPath` => absolute - path of the file with the given ID. - -### getMriProtocolViolatedScansFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef) - -Get the absolute paths of all the files associated to a DICOM archive that are listed in -table `mri_protocol_violated_scans`. - -INPUTS: - - $dbhr : database handle reference. - - $tarchiveID: ID of the DICOM archive. - - $dataDirBasePath: config value of setting `dataDirBasePath`. - - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - - $optionsRef: reference on the array of command line options. - -RETURNS: - - an array of hash references. Each hash has three keys: `ID` => ID of the record in table - `mri_protocol_violated_scans`, `minc_location` => value of column `minc_location` in table - `mri_protocol_violated_scans` for the MINC file found and `FullPath` => absolute path of the MINC +that are listed in table \f(CW\*(C`files_intermediary\*(C'\fR. +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$dbhr\fR : database handle reference. + \- \f(CW$filesRef:\fR reference on the hash of all files. + \- \f(CW$tarchiveID:\fR \s-1ID\s0 of the \s-1DICOM\s0 archive. + \- \f(CW$dataDirBasePath:\fR config value of setting \f(CW\*(C`dataDirBasePath\*(C'\fR. + \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. + \- \f(CW$optionsRef:\fR reference on the array of command line options. +.PP +\&\s-1RETURNS:\s0 + \- an array of hash references. Each hash has seven keys: \f(CW\*(C`IntermedID\*(C'\fR => \s-1ID\s0 of a file in + table \f(CW\*(C`files_intermediary\*(C'\fR, \f(CW\*(C`Input_FileID\*(C'\fR => \s-1ID\s0 of the file that was used as input to create + the intermediary file, \f(CW\*(C`Output_FileID\*(C'\fR \s-1ID\s0 of the output file, \f(CW\*(C`FileID\*(C'\fR => \s-1ID\s0 of this file in + table \f(CW\*(C`files\*(C'\fR, \f(CW\*(C`File\*(C'\fR => value of column \f(CW\*(C`File\*(C'\fR in table \f(CW\*(C`files\*(C'\fR for the file with the given + \s-1ID,\s0 \f(CW\*(C`SourceFileID\*(C'\fR value of column \f(CW\*(C`SourceFileID\*(C'\fR for the intermediary file and + \f(CW\*(C`FullPath\*(C'\fR => absolute path of the file with the given \s-1ID.\s0 +.PP +\fIgetParameterFilesRef($dbh, \f(CI$filesRef\fI, \f(CI$tarchiveID\fI, \f(CI$dataDirBasePath\fI, \f(CI$scanTypesToDeleteRef\fI, \f(CI$optionsRef\fI)\fR +.IX Subsection "getParameterFilesRef($dbh, $filesRef, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef)" +.PP +Gets the records to delete from table parameter_file. +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$dbhr\fR : database handle reference. + \- \f(CW$filesRef:\fR reference on the hash of all files. + \- \f(CW$tarchiveID:\fR \s-1ID\s0 of the \s-1DICOM\s0 archive. + \- \f(CW$dataDirBasePath:\fR config value of setting \f(CW\*(C`dataDirBasePath\*(C'\fR. + \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. + \- \f(CW$optionsRef:\fR reference on the array of command line options. +.PP +\&\s-1RETURNS:\s0 + \- an array of hash references. Each hash has four keys: \f(CW\*(C`FileID\*(C'\fR => FileID of a file + in table \f(CW\*(C`parameter_file\*(C'\fR, \f(CW\*(C`Value\*(C'\fR => value of column \f(CW\*(C`Value\*(C'\fR in table \f(CW\*(C`parameter_file\*(C'\fR + for the file with the given \s-1ID,\s0 \f(CW\*(C`Name\*(C'\fR => name of the parameter and \f(CW\*(C`FullPath\*(C'\fR => absolute + path of the file with the given \s-1ID.\s0 +.PP +\fIgetMriProtocolViolatedScansFilesRef($dbh, \f(CI$tarchiveID\fI, \f(CI$dataDirBasePath\fI, \f(CI$scanTypesToDeleteRef\fI, \f(CI$optionsRef\fI)\fR +.IX Subsection "getMriProtocolViolatedScansFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef)" +.PP +Get the absolute paths of all the files associated to a \s-1DICOM\s0 archive that are listed in +table \f(CW\*(C`mri_protocol_violated_scans\*(C'\fR. +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$dbhr\fR : database handle reference. + \- \f(CW$tarchiveID:\fR \s-1ID\s0 of the \s-1DICOM\s0 archive. + \- \f(CW$dataDirBasePath:\fR config value of setting \f(CW\*(C`dataDirBasePath\*(C'\fR. + \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. + \- \f(CW$optionsRef:\fR reference on the array of command line options. +.PP +\&\s-1RETURNS:\s0 + \- an array of hash references. Each hash has three keys: \f(CW\*(C`ID\*(C'\fR => \s-1ID\s0 of the record in table + \f(CW\*(C`mri_protocol_violated_scans\*(C'\fR, \f(CW\*(C`minc_location\*(C'\fR => value of column \f(CW\*(C`minc_location\*(C'\fR in table + \f(CW\*(C`mri_protocol_violated_scans\*(C'\fR for the \s-1MINC\s0 file found and \f(CW\*(C`FullPath\*(C'\fR => absolute path of the \s-1MINC\s0 file found. - -### getMriViolationsLogFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef) - +.PP +\fIgetMriViolationsLogFilesRef($dbh, \f(CI$tarchiveID\fI, \f(CI$dataDirBasePath\fI, \f(CI$scanTypesToDeleteRef\fI, \f(CI$optionsRef\fI)\fR +.IX Subsection "getMriViolationsLogFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef)" +.PP Get the absolute paths of all the files associated to an archive that are listed in -table `mri_violations_log`. - -INPUTS: - - $dbhr : database handle reference. - - $tarchiveID: ID of the DICOM archive. - - $dataDirBasePath: config value of setting `dataDirBasePath`. - - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - - $optionsRef: reference on the array of command line options. - -RETURNS: - an array of hash references. Each hash has three keys: `LogID` => ID of the record in table - `mri_violations_log`, `MincFile` => value of column `MincFile` for the MINC file found in table - `mri_violations_log` and `FullPath` => absolute path of the MINC file. - -### getMRICandidateErrorsFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypeToDeleteRef, $optionsRef) - -Get the absolute paths of all the files associated to a DICOM archive that are listed in -table `MRICandidateErrors`. - -INPUTS: - - $dbh : database handle reference. - - $tarchiveID: ID of the DICOM archive. - - $dataDirBasePath: config value of setting `dataDirBasePath`. - - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - - $optionsRef: reference on the array of command line options. - -RETURNS: - - an array of hash references. Each hash has three keys: `ID` => ID of the record in the - table, `MincFile` => value of column `MincFile` for the MINC file found in table - `MRICandidateErrors` and `FullPath` => absolute path of the MINC file. - -### getBidsExportFilesRef($dbh, $filesRef, $dataDirBasePath) - -Gets the imaging file records to delete from table bids\_export\_files. - -INPUTS: - - $dbhr : database handle reference. - - $filesRef: reference on the hash of all files. - - $tarchiveID: ID of the DICOM archive. - - $dataDirBasePath: config value of setting `dataDirBasePath`. - - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - - $optionsRef: reference on the array of command line options. - -RETURNS: - - an array of hash references. Each hash has three keys: `BIDSExportedFileID` => ID of the record in the - table, `FilePath` => value of column `FilePath` for the BIDS export file found in table - `bids_export_files` and `FullPath` => absolute path of the BIDS export file. - -### getBidsExportFilesSessionRef($dbh, $filesRef, $dataDirBasePath) - -Gets the session level records to delete from table bids\_export\_files (aka scans.tsv and scans.json files). - -INPUTS: - - $dbhr : database handle reference. - - $filesRef: reference on the hash of all files. - - $dataDirBasePath: config value of setting `dataDirBasePath`. - -RETURNS: - - a hash with scans.tsv full path and a boolean specifying whether the session level scans.tsv file needs to be updated - -### updateBidsScansTsvFile() - -Modifies the scans.tsv BIDS file if not all files for the session have been deleted. It will remove the -entries for the files that were deleted from scans.tsv BIDS export file. - -INPUTS: - - $filesRef: reference to the array that contains the file informations for all the files +table \f(CW\*(C`mri_violations_log\*(C'\fR. +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$dbhr\fR : database handle reference. + \- \f(CW$tarchiveID:\fR \s-1ID\s0 of the \s-1DICOM\s0 archive. + \- \f(CW$dataDirBasePath:\fR config value of setting \f(CW\*(C`dataDirBasePath\*(C'\fR. + \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. + \- \f(CW$optionsRef:\fR reference on the array of command line options. +.PP +\&\s-1RETURNS:\s0 + an array of hash references. Each hash has three keys: \f(CW\*(C`LogID\*(C'\fR => \s-1ID\s0 of the record in table + \f(CW\*(C`mri_violations_log\*(C'\fR, \f(CW\*(C`MincFile\*(C'\fR => value of column \f(CW\*(C`MincFile\*(C'\fR for the \s-1MINC\s0 file found in table + \f(CW\*(C`mri_violations_log\*(C'\fR and \f(CW\*(C`FullPath\*(C'\fR => absolute path of the \s-1MINC\s0 file. +.PP +\fIgetMRICandidateErrorsFilesRef($dbh, \f(CI$tarchiveID\fI, \f(CI$dataDirBasePath\fI, \f(CI$scanTypeToDeleteRef\fI, \f(CI$optionsRef\fI)\fR +.IX Subsection "getMRICandidateErrorsFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypeToDeleteRef, $optionsRef)" +.PP +Get the absolute paths of all the files associated to a \s-1DICOM\s0 archive that are listed in +table \f(CW\*(C`MRICandidateErrors\*(C'\fR. +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$dbh\fR : database handle reference. + \- \f(CW$tarchiveID:\fR \s-1ID\s0 of the \s-1DICOM\s0 archive. + \- \f(CW$dataDirBasePath:\fR config value of setting \f(CW\*(C`dataDirBasePath\*(C'\fR. + \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. + \- \f(CW$optionsRef:\fR reference on the array of command line options. +.PP +\&\s-1RETURNS:\s0 + \- an array of hash references. Each hash has three keys: \f(CW\*(C`ID\*(C'\fR => \s-1ID\s0 of the record in the + table, \f(CW\*(C`MincFile\*(C'\fR => value of column \f(CW\*(C`MincFile\*(C'\fR for the \s-1MINC\s0 file found in table + \f(CW\*(C`MRICandidateErrors\*(C'\fR and \f(CW\*(C`FullPath\*(C'\fR => absolute path of the \s-1MINC\s0 file. +.PP +\fIgetBidsExportFilesRef($dbh, \f(CI$filesRef\fI, \f(CI$dataDirBasePath\fI)\fR +.IX Subsection "getBidsExportFilesRef($dbh, $filesRef, $dataDirBasePath)" +.PP +Gets the imaging file records to delete from table bids_export_files. +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$dbhr\fR : database handle reference. + \- \f(CW$filesRef:\fR reference on the hash of all files. + \- \f(CW$tarchiveID:\fR \s-1ID\s0 of the \s-1DICOM\s0 archive. + \- \f(CW$dataDirBasePath:\fR config value of setting \f(CW\*(C`dataDirBasePath\*(C'\fR. + \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. + \- \f(CW$optionsRef:\fR reference on the array of command line options. +.PP +\&\s-1RETURNS:\s0 + \- an array of hash references. Each hash has three keys: \f(CW\*(C`BIDSExportedFileID\*(C'\fR => \s-1ID\s0 of the record in the + table, \f(CW\*(C`FilePath\*(C'\fR => value of column \f(CW\*(C`FilePath\*(C'\fR for the \s-1BIDS\s0 export file found in table + \f(CW\*(C`bids_export_files\*(C'\fR and \f(CW\*(C`FullPath\*(C'\fR => absolute path of the \s-1BIDS\s0 export file. +.PP +\fIgetBidsExportFilesSessionRef($dbh, \f(CI$filesRef\fI, \f(CI$dataDirBasePath\fI)\fR +.IX Subsection "getBidsExportFilesSessionRef($dbh, $filesRef, $dataDirBasePath)" +.PP +Gets the session level records to delete from table bids_export_files (aka scans.tsv and scans.json files). +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$dbhr\fR : database handle reference. + \- \f(CW$filesRef:\fR reference on the hash of all files. + \- \f(CW$dataDirBasePath:\fR config value of setting \f(CW\*(C`dataDirBasePath\*(C'\fR. +.PP +\&\s-1RETURNS:\s0 + \- a hash with scans.tsv full path and a boolean specifying whether the session level scans.tsv file needs to be updated +.PP +\fI\fIupdateBidsScansTsvFile()\fI\fR +.IX Subsection "updateBidsScansTsvFile()" +.PP +Modifies the scans.tsv \s-1BIDS\s0 file if not all files for the session have been deleted. It will remove the +entries for the files that were deleted from scans.tsv \s-1BIDS\s0 export file. +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$filesRef:\fR reference to the array that contains the file informations for all the files that are associated to the upload(s) passed on the command line. - -### setFileExistenceStatus($filesRef) - +.PP +\fIsetFileExistenceStatus($filesRef)\fR +.IX Subsection "setFileExistenceStatus($filesRef)" +.PP Checks the list of all the files related to the upload(s) that were found in the database and determine whether they exist or not on the file system. - -INPUTS: - - $filesRef: reference to the array that contains the file informations for all the files +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$filesRef:\fR reference to the array that contains the file informations for all the files that are associated to the upload(s) passed on the command line. - -RETURNS: - - Reference on the list of files that do not exist on the file system. - -### shouldExist($table, $fileRef) - +.PP +\&\s-1RETURNS:\s0 + \- Reference on the list of files that do not exist on the file system. +.PP +\fIshouldExist($table, \f(CI$fileRef\fI)\fR +.IX Subsection "shouldExist($table, $fileRef)" +.PP Checks whether a file path in the database refers to a file that should exist on the file system. - -INPUTS: - - $table: name of the table in which the file path was found. - - $fileRef: reference to the array that contains the file information for a given file. - -RETURNS: - - 0 or 1 depending on whether the file should exist or not. - -### backupFiles($filesRef, $scanTypesToDeleteRef, $optionsRef) - +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$table:\fR name of the table in which the file path was found. + \- \f(CW$fileRef:\fR reference to the array that contains the file information for a given file. +.PP +\&\s-1RETURNS:\s0 + \- 0 or 1 depending on whether the file should exist or not. +.PP +\fIbackupFiles($filesRef, \f(CI$scanTypesToDeleteRef\fI, \f(CI$optionsRef\fI)\fR +.IX Subsection "backupFiles($filesRef, $scanTypesToDeleteRef, $optionsRef)" +.PP Backs up all the files associated to the archive before deleting them. The backed up files will -be stored in a `.tar.gz` archive in which all paths are absolute. - -INPUTS: - - $filesRef: reference to the array that contains the file informations for all the files +be stored in a \f(CW\*(C`.tar.gz\*(C'\fR archive in which all paths are absolute. +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$filesRef:\fR reference to the array that contains the file informations for all the files that are associated to the upload(s) passed on the command line. - - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - - $optionsRef: reference on the hash array of command line options. - -RETURNS: - - The number of files backed up. - -### shouldDeleteFile($table, $fileRef, $scanTypesToDeleteRef, $keepDefaced, $fileBaseName) - + \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. + \- \f(CW$optionsRef:\fR reference on the hash array of command line options. +.PP +\&\s-1RETURNS:\s0 + \- The number of files backed up. +.PP +\fIshouldDeleteFile($table, \f(CI$fileRef\fI, \f(CI$scanTypesToDeleteRef\fI, \f(CI$keepDefaced\fI, \f(CI$fileBaseName\fI)\fR +.IX Subsection "shouldDeleteFile($table, $fileRef, $scanTypesToDeleteRef, $keepDefaced, $fileBaseName)" +.PP Checks whether a given file should be deleted or not. - -INPUTS: - - $table: name of the table in which the file path was found. - - $fileRef: reference to the array that contains the file information for a given file. - - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - - $keepDefaced: whether the defaced files should be kept or not. - - $fileBaseName: value of option -basename (or '' if -basename was not used). - -RETURNS: - - 0 or 1 depending on whether the file should be deleted or not. - -### deleteUploadsInDatabase($dbh, $filesRef, $scanTypesToDeleteRef, $optionsRef) - +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$table:\fR name of the table in which the file path was found. + \- \f(CW$fileRef:\fR reference to the array that contains the file information for a given file. + \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. + \- \f(CW$keepDefaced:\fR whether the defaced files should be kept or not. + \- \f(CW$fileBaseName:\fR value of option \-basename (or '' if \-basename was not used). +.PP +\&\s-1RETURNS:\s0 + \- 0 or 1 depending on whether the file should be deleted or not. +.PP +\fIdeleteUploadsInDatabase($dbh, \f(CI$filesRef\fI, \f(CI$scanTypesToDeleteRef\fI, \f(CI$optionsRef\fI)\fR +.IX Subsection "deleteUploadsInDatabase($dbh, $filesRef, $scanTypesToDeleteRef, $optionsRef)" +.PP This method deletes all information in the database associated to the given upload(s)/scan type combination. -More specifically, it deletes records from tables `notification_spool`, `tarchive_files`, `tarchive_series` -`files_intermediary`, `parameter_file`, `files`, `mri_protocol_violated_scans`, `mri_violations_log` -`MRICandidateErrors`, `mri_upload`, `tarchive`, `mri_processing_protocol` and `mri_parameter_form` -(the later is done only if requested). It will also set the `Scan_done` value of the scan's session to 'N' for +More specifically, it deletes records from tables \f(CW\*(C`notification_spool\*(C'\fR, \f(CW\*(C`tarchive_files\*(C'\fR, \f(CW\*(C`tarchive_series\*(C'\fR +\&\f(CW\*(C`files_intermediary\*(C'\fR, \f(CW\*(C`parameter_file\*(C'\fR, \f(CW\*(C`files\*(C'\fR, \f(CW\*(C`mri_protocol_violated_scans\*(C'\fR, \f(CW\*(C`mri_violations_log\*(C'\fR +\&\f(CW\*(C`MRICandidateErrors\*(C'\fR, \f(CW\*(C`mri_upload\*(C'\fR, \f(CW\*(C`tarchive\*(C'\fR, \f(CW\*(C`mri_processing_protocol\*(C'\fR and \f(CW\*(C`mri_parameter_form\*(C'\fR +(the later is done only if requested). It will also set the \f(CW\*(C`Scan_done\*(C'\fR value of the scan's session to 'N' for each upload that is the last upload tied to that session. All the delete/update operations are done inside a single transaction so either they all succeed or they all fail (and a rollback is performed). - -INPUTS: - - $dbh : database handle. - - $filesRef : reference to the array that contains the file informations for all the files +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$dbh\fR : database handle. + \- \f(CW$filesRef\fR : reference to the array that contains the file informations for all the files that are associated to the upload(s) passed on the command line. - - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - - $optionsRef: reference on the hash array of the options that were passed on the command line. - -RETURNS: - - A reference to a hash with two keys: - \* SQL\_BACKUP\_DONE => 1 if this method produced a file containing the SQL statements that restore the database state to what it was before calling this + \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. + \- \f(CW$optionsRef:\fR reference on the hash array of the options that were passed on the command line. +.PP +\&\s-1RETURNS:\s0 + \- A reference to a hash with two keys: + * \s-1SQL_BACKUP_DONE\s0 => 1 if this method produced a file containing the \s-1SQL\s0 statements that restore the database state to what it was before calling this method, 0 otherwise. - \* NB\_RECORDS\_DELETED => the number of records effectively deleted by this method. - -### gzipBackupFile($backupPath) - + * \s-1NB_RECORDS_DELETED\s0 => the number of records effectively deleted by this method. +.PP +\fIgzipBackupFile($backupPath)\fR +.IX Subsection "gzipBackupFile($backupPath)" +.PP Compresses the file that contains a backup of everything that was deleted by the script, both -from the file system and the database, using `gzip`. - -INPUTS: - - $backupPath: path of the backup file to compress (without the .tar.gz extension). - -### updateSessionTable($dbh, $mriUploadsRef, $tmpSQLFile) - -Sets to `N` the `Scan_done` column of all `sessions` in the database that do not have an associated upload -after the script has deleted those whose IDs are passed on the command line. The script also adds an SQL statement -in the SQL file whose path is passed as argument to restore the state that the `session` table had before the deletions. - -INPUTS: - - $dbh : database handle. - - $mriUploadsRef: reference on an array of hashes containing the uploads to delete. Accessed like this: - `$mriUploadsRef->[0]->{'TarchiveID'}`(this would return the `TarchiveID` of the first `mri_upload` - in the array. The properties stored for each hash are: `UploadID`, `TarchiveID`, `FullPath` - `Inserting`, `InsertionComplete` and `SessionID`. - - $tmpSQLFile: path of the SQL file that contains the SQL statements used to restore the deleted records. - -### updateFilesIntermediaryTable($dbh, $filesRef, $tmpSQLFile) - -Sets the `TarchiveSource` and `SourceFileID` columns of all the defaced files to `$tarchiveID` and `NULL` -respectively. The script also adds an SQL statement in the SQL file whose path is passed as argument to -restore the state that the defaced files in the `files` table had before the deletions. - -INPUTS: - - $dbh : database handle. - - $filesRef: reference to the array that contains the file informations for all the files +from the file system and the database, using \f(CW\*(C`gzip\*(C'\fR. +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$backupPath:\fR path of the backup file to compress (without the .tar.gz extension). +.PP +\fIupdateSessionTable($dbh, \f(CI$mriUploadsRef\fI, \f(CI$tmpSQLFile\fI)\fR +.IX Subsection "updateSessionTable($dbh, $mriUploadsRef, $tmpSQLFile)" +.PP +Sets to \f(CW\*(C`N\*(C'\fR the \f(CW\*(C`Scan_done\*(C'\fR column of all \f(CW\*(C`sessions\*(C'\fR in the database that do not have an associated upload +after the script has deleted those whose IDs are passed on the command line. The script also adds an \s-1SQL\s0 statement +in the \s-1SQL\s0 file whose path is passed as argument to restore the state that the \f(CW\*(C`session\*(C'\fR table had before the deletions. +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$dbh\fR : database handle. + \- \f(CW$mriUploadsRef:\fR reference on an array of hashes containing the uploads to delete. Accessed like this: + \f(CW\*(C`$mriUploadsRef\->[0]\->{\*(AqTarchiveID\*(Aq}\*(C'\fR(this would return the \f(CW\*(C`TarchiveID\*(C'\fR of the first \f(CW\*(C`mri_upload\*(C'\fR + in the array. The properties stored for each hash are: \f(CW\*(C`UploadID\*(C'\fR, \f(CW\*(C`TarchiveID\*(C'\fR, \f(CW\*(C`FullPath\*(C'\fR + \f(CW\*(C`Inserting\*(C'\fR, \f(CW\*(C`InsertionComplete\*(C'\fR and \f(CW\*(C`SessionID\*(C'\fR. + \- \f(CW$tmpSQLFile:\fR path of the \s-1SQL\s0 file that contains the \s-1SQL\s0 statements used to restore the deleted records. +.PP +\fIupdateFilesIntermediaryTable($dbh, \f(CI$filesRef\fI, \f(CI$tmpSQLFile\fI)\fR +.IX Subsection "updateFilesIntermediaryTable($dbh, $filesRef, $tmpSQLFile)" +.PP +Sets the \f(CW\*(C`TarchiveSource\*(C'\fR and \f(CW\*(C`SourceFileID\*(C'\fR columns of all the defaced files to \f(CW$tarchiveID\fR and \f(CW\*(C`NULL\*(C'\fR +respectively. The script also adds an \s-1SQL\s0 statement in the \s-1SQL\s0 file whose path is passed as argument to +restore the state that the defaced files in the \f(CW\*(C`files\*(C'\fR table had before the deletions. +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$dbh\fR : database handle. + \- \f(CW$filesRef:\fR reference to the array that contains the file informations for all the files that are associated to the upload(s) passed on the command line. - - $tmpSQLFile: path of the SQL file that contains the SQL statements used to restore the deleted records. - -### deleteMriParameterForm($dbh, $mriUploadsRef, $tmpSQLFile, $dump\_opt) - -Delete the entries in `mri_parameter_form` (and associated `flag` entry) for the upload(s) passed on the -command line. The script also adds an SQL statement in the SQL file whose path is passed as argument to -restore the state that the `mri_parameter_form` and `flag` tables had before the deletions. - -INPUTS: - - $dbh : database handle. - - $mriUploadsRef: reference on an array of hashes containing the uploads to delete. Accessed like this: - `$mriUploadsRef->[0]->{'TarchiveID'}`(this would return the `TarchiveID` of the first `mri_upload` - in the array. The properties stored for each hash are: `UploadID`, `TarchiveID`, `FullPath` - `Inserting`, `InsertionComplete` and `SessionID`. - - $tmpSQLFile: path of the SQL file that contains the SQL statements used to restore the deleted records. - - $dump\_opt: additional options to use for mysqldump - -RETURNS: - - The numbers of records deleted as a result of this operation. - -### deleteUploadsOnFileSystem($filesRef, $scanTypesToDeleteRef, $keepDefaced, $fileBaseName) - + \- \f(CW$tmpSQLFile:\fR path of the \s-1SQL\s0 file that contains the \s-1SQL\s0 statements used to restore the deleted records. +.PP +\fIdeleteMriParameterForm($dbh, \f(CI$mriUploadsRef\fI, \f(CI$tmpSQLFile\fI, \f(CI$optionsRef\fI)\fR +.IX Subsection "deleteMriParameterForm($dbh, $mriUploadsRef, $tmpSQLFile, $optionsRef)" +.PP +Delete the entries in \f(CW\*(C`mri_parameter_form\*(C'\fR (and associated \f(CW\*(C`flag\*(C'\fR entry) for the upload(s) passed on the +command line. The script also adds an \s-1SQL\s0 statement in the \s-1SQL\s0 file whose path is passed as argument to +restore the state that the \f(CW\*(C`mri_parameter_form\*(C'\fR and \f(CW\*(C`flag\*(C'\fR tables had before the deletions. +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$dbh\fR : database handle. + \- \f(CW$mriUploadsRef:\fR reference on an array of hashes containing the uploads to delete. Accessed like this: + \f(CW\*(C`$mriUploadsRef\->[0]\->{\*(AqTarchiveID\*(Aq}\*(C'\fR(this would return the \f(CW\*(C`TarchiveID\*(C'\fR of the first \f(CW\*(C`mri_upload\*(C'\fR + in the array. The properties stored for each hash are: \f(CW\*(C`UploadID\*(C'\fR, \f(CW\*(C`TarchiveID\*(C'\fR, \f(CW\*(C`FullPath\*(C'\fR + \f(CW\*(C`Inserting\*(C'\fR, \f(CW\*(C`InsertionComplete\*(C'\fR and \f(CW\*(C`SessionID\*(C'\fR. + \- \f(CW$tmpSQLFile:\fR path of the \s-1SQL\s0 file that contains the \s-1SQL\s0 statements used to restore the deleted records. + \- \f(CW$optionsRef:\fR reference to the array that contains the options passed on the command line. +.PP +\&\s-1RETURNS:\s0 + \- The numbers of records deleted as a result of this operation. +.PP +\fIdeleteUploadsOnFileSystem($filesRef, \f(CI$scanTypesToDeleteRef\fI, \f(CI$keepDefaced\fI, \f(CI$fileBaseName\fI)\fR +.IX Subsection "deleteUploadsOnFileSystem($filesRef, $scanTypesToDeleteRef, $keepDefaced, $fileBaseName)" +.PP This method deletes from the file system all the files associated to the upload(s) passed on the command line that were found on the file system. A warning will be issued for any file that could not be deleted. - -INPUTS: - - $filesRef: reference to the array that contains the file informations for all the files +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$filesRef:\fR reference to the array that contains the file informations for all the files that are associated to the upload(s) passed on the command line. - - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - - $keepDefaced: whether the defaced files should be kept or not. - - $fileBaseName: value of option -basename (or '' if option was not used). - -### getTypesToDelete($dbh, $scanTypeList, $keepDefaced) - -Gets the list of names of the scan types to delete. If `-type` was used, then this list is built -using the argument to this option. If `-defaced` was used, then the list is fetched using the config -setting `modalities_to_deface`. - -INPUTS: - - $dbh: database handle. - - $scanTypeList: comma separated string of scan type names. - - $keepDefaced: whether the defaced files should be kept or not. - -RETURNS: - - A reference on a hash of the names of the scan types to delete: key => scan type name, + \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. + \- \f(CW$keepDefaced:\fR whether the defaced files should be kept or not. + \- \f(CW$fileBaseName:\fR value of option \-basename (or '' if option was not used). +.PP +\fIgetTypesToDelete($dbh, \f(CI$scanTypeList\fI, \f(CI$keepDefaced\fI)\fR +.IX Subsection "getTypesToDelete($dbh, $scanTypeList, $keepDefaced)" +.PP +Gets the list of names of the scan types to delete. If \f(CW\*(C`\-type\*(C'\fR was used, then this list is built +using the argument to this option. If \f(CW\*(C`\-defaced\*(C'\fR was used, then the list is fetched using the config +setting \f(CW\*(C`modalities_to_deface\*(C'\fR. +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$dbh:\fR database handle. + \- \f(CW$scanTypeList:\fR comma separated string of scan type names. + \- \f(CW$keepDefaced:\fR whether the defaced files should be kept or not. +.PP +\&\s-1RETURNS:\s0 + \- A reference on a hash of the names of the scan types to delete: key => scan type name, value => 1 or 0 depending on whether the name is valid or not. - -### getTarchiveSeriesIDs($dbh, $filesRef) - -Gets the list of `TarchiveSeriesID` to delete in table `tarchive_files`. - -INPUTS: - - $dbh: database handle. - - $filesRef: reference to the array that contains the file informations for all the files +.PP +\fIgetTarchiveSeriesIDs($dbh, \f(CI$filesRef\fI)\fR +.IX Subsection "getTarchiveSeriesIDs($dbh, $filesRef)" +.PP +Gets the list of \f(CW\*(C`TarchiveSeriesID\*(C'\fR to delete in table \f(CW\*(C`tarchive_files\*(C'\fR. +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$dbh:\fR database handle. + \- \f(CW$filesRef:\fR reference to the array that contains the file informations for all the files that are associated to the upload(s) passed on the command line. - -RETURNS: - - A reference on an array containing the `TarchiveSeriesID` to delete. - -### deleteTableData($dbh, $table, $key, $keyValuesRef, $tmpSQLBackupFile, $dump\_opt) - -Deletes records from a database table and adds in a file the SQL statements that allow rewriting the -records back in the table. - -INPUTS: - - $dbh: database handle. - - $table: name of the database table. - - $key: name of the key used to delete the records. - - $keyValuesRef: reference on the list of values that field `$key` has for the records to delete. - - $tmpSQLBackupFile: path of the SQL file that contains the SQL statements used to restore the deleted records. - - $dump\_opt: additional options to use for mysqldump - -RETURNS: - - The number of records deleted. - -### updateSQLBackupFile($tmpSQLBackupFile, $table, $key, $keyValuesRef, $dump\_opt) - -Updates the SQL file with the statements to restore the records whose properties are passed as argument. +.PP +\&\s-1RETURNS:\s0 + \- A reference on an array containing the \f(CW\*(C`TarchiveSeriesID\*(C'\fR to delete. +.PP +\fIdeleteTableData($dbh, \f(CI$table\fI, \f(CI$key\fI, \f(CI$keyValuesRef\fI, \f(CI$tmpSQLBackupFile\fI, \f(CI$optionsRef\fI)\fR +.IX Subsection "deleteTableData($dbh, $table, $key, $keyValuesRef, $tmpSQLBackupFile, $optionsRef)" +.PP +Deletes records from a database table and adds in a file the \s-1SQL\s0 statements that allow rewriting the +records back in the table. +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$dbh:\fR database handle. + \- \f(CW$table:\fR name of the database table. + \- \f(CW$key:\fR name of the key used to delete the records. + \- \f(CW$keyValuesRef:\fR reference on the list of values that field \f(CW$key\fR has for the records to delete. + \- \f(CW$tmpSQLBackupFile:\fR path of the \s-1SQL\s0 file that contains the \s-1SQL\s0 statements used to restore the deleted records. + \- \f(CW$optionsRef:\fR reference to the array that contains the options passed on the command line. +.PP +\&\s-1RETURNS:\s0 + \- The number of records deleted. +.PP +\fIupdateSQLBackupFile($tmpSQLBackupFile, \f(CI$table\fI, \f(CI$key\fI, \f(CI$keyValuesRef\fI, \f(CI$optionsRef\fI)\fR +.IX Subsection "updateSQLBackupFile($tmpSQLBackupFile, $table, $key, $keyValuesRef, $optionsRef)" +.PP +Updates the \s-1SQL\s0 file with the statements to restore the records whose properties are passed as argument. The block of statements is written at the beginning of the file. - -INPUTS: - - $tmpSQLBackupFile: path of the SQL file that contains the SQL statements used to restore the deleted records. - - $table: name of the database table. - - $key: name of the key used to delete the records. - - $keyValuesRef: reference on the list of values that field `$key` has for the records to delete. - - $dump\_opt: additional options to use for mysqldump - -### getInvalidDefacedFiles($dbh, $filesRef, $scanTypesToDeleteRef) - -Checks all the MINC files that should have been defaced and makes sure that only the defaced file +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$tmpSQLBackupFile:\fR path of the \s-1SQL\s0 file that contains the \s-1SQL\s0 statements used to restore the deleted records. + \- \f(CW$table:\fR name of the database table. + \- \f(CW$key:\fR name of the key used to delete the records. + \- \f(CW$keyValuesRef:\fR reference on the list of values that field \f(CW$key\fR has for the records to delete. + \- \f(CW$optionsRef:\fR reference to the array that contains the options passed on the command line. +.PP +\fIgetInvalidDefacedFiles($dbh, \f(CI$filesRef\fI, \f(CI$scanTypesToDeleteRef\fI)\fR +.IX Subsection "getInvalidDefacedFiles($dbh, $filesRef, $scanTypesToDeleteRef)" +.PP +Checks all the \s-1MINC\s0 files that should have been defaced and makes sure that only the defaced file is associated to it. - -INPUTS: - - $dbh : database handle. - - $filesRef: reference to the array that contains the information for all the files +.PP +\&\s-1INPUTS:\s0 + \- \f(CW$dbh\fR : database handle. + \- \f(CW$filesRef:\fR reference to the array that contains the information for all the files that are associated to the upload(s) passed on the command line. - - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - -RETURNS: - - The hash of MINC files that were either not defaced (and should have been) or that have more than one + \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. +.PP +\&\s-1RETURNS:\s0 + \- The hash of \s-1MINC\s0 files that were either not defaced (and should have been) or that have more than one processed file associated to them. Key => file path (relative), Value => reference on an array that contains the - list of processed files associated to the MINC file (0, 2, 3 or more entries). - -# LICENSING - + list of processed files associated to the \s-1MINC\s0 file (0, 2, 3 or more entries). +.SH "LICENSING" +.IX Header "LICENSING" License: GPLv3 - -# AUTHORS - -LORIS community and McGill Centre for Integrative +.SH "AUTHORS" +.IX Header "AUTHORS" +\&\s-1LORIS\s0 community and McGill Centre for Integrative Neuroscience From 4d22e49d04e97906d64ef8a5051340021e0c230f Mon Sep 17 00:00:00 2001 From: Nicolas Brossard Date: Fri, 3 May 2024 10:14:47 -0400 Subject: [PATCH 3/4] Update of docs/scripts_md/delete_imaging_upload.md --- docs/scripts_md/delete_imaging_upload.md | 1263 ++++++++++------------ 1 file changed, 551 insertions(+), 712 deletions(-) diff --git a/docs/scripts_md/delete_imaging_upload.md b/docs/scripts_md/delete_imaging_upload.md index 534578181..f26207892 100644 --- a/docs/scripts_md/delete_imaging_upload.md +++ b/docs/scripts_md/delete_imaging_upload.md @@ -1,777 +1,616 @@ -.\" Automatically generated by Pod::Man 4.09 (Pod::Simple 3.35) -.\" -.\" Standard preamble: -.\" ======================================================================== -.de Sp \" Vertical space (when we can't use .PP) -.if t .sp .5v -.if n .sp -.. -.de Vb \" Begin verbatim text -.ft CW -.nf -.ne \\$1 -.. -.de Ve \" End verbatim text -.ft R -.fi -.. -.\" Set up some character translations and predefined strings. \*(-- will -.\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left -.\" double quote, and \*(R" will give a right double quote. \*(C+ will -.\" give a nicer C++. Capital omega is used to do unbreakable dashes and -.\" therefore won't be available. \*(C` and \*(C' expand to `' in nroff, -.\" nothing in troff, for use with C<>. -.tr \(*W- -.ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p' -.ie n \{\ -. ds -- \(*W- -. ds PI pi -. if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch -. if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\" diablo 12 pitch -. ds L" "" -. ds R" "" -. ds C` "" -. ds C' "" -'br\} -.el\{\ -. ds -- \|\(em\| -. ds PI \(*p -. ds L" `` -. ds R" '' -. ds C` -. ds C' -'br\} -.\" -.\" Escape single quotes in literal strings from groff's Unicode transform. -.ie \n(.g .ds Aq \(aq -.el .ds Aq ' -.\" -.\" If the F register is >0, we'll generate index entries on stderr for -.\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index -.\" entries marked with X<> in POD. Of course, you'll have to process the -.\" output yourself in some meaningful fashion. -.\" -.\" Avoid warning from groff about undefined register 'F'. -.de IX -.. -.if !\nF .nr F 0 -.if \nF>0 \{\ -. de IX -. tm Index:\\$1\t\\n%\t"\\$2" -.. -. if !\nF==2 \{\ -. nr % 0 -. nr F 2 -. \} -.\} -.\" -.\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2). -.\" Fear. Run. Save yourself. No user-serviceable parts. -. \" fudge factors for nroff and troff -.if n \{\ -. ds #H 0 -. ds #V .8m -. ds #F .3m -. ds #[ \f1 -. ds #] \fP -.\} -.if t \{\ -. ds #H ((1u-(\\\\n(.fu%2u))*.13m) -. ds #V .6m -. ds #F 0 -. ds #[ \& -. ds #] \& -.\} -. \" simple accents for nroff and troff -.if n \{\ -. ds ' \& -. ds ` \& -. ds ^ \& -. ds , \& -. ds ~ ~ -. ds / -.\} -.if t \{\ -. ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u" -. ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u' -. ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u' -. ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u' -. ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u' -. ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u' -.\} -. \" troff and (daisy-wheel) nroff accents -.ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V' -.ds 8 \h'\*(#H'\(*b\h'-\*(#H' -.ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#] -.ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H' -.ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u' -.ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#] -.ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#] -.ds ae a\h'-(\w'a'u*4/10)'e -.ds Ae A\h'-(\w'A'u*4/10)'E -. \" corrections for vroff -.if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u' -.if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u' -. \" for low resolution devices (crt and lpr) -.if \n(.H>23 .if \n(.V>19 \ -\{\ -. ds : e -. ds 8 ss -. ds o a -. ds d- d\h'-1'\(ga -. ds D- D\h'-1'\(hy -. ds th \o'bp' -. ds Th \o'LP' -. ds ae ae -. ds Ae AE -.\} -.rm #[ #] #H #V #F C -.\" ======================================================================== -.\" -.IX Title "DELETE_IMAGING_UPLOAD 1" -.TH DELETE_IMAGING_UPLOAD 1 "2024-05-03" "perl v5.26.1" "User Contributed Perl Documentation" -.\" For nroff, turn off justification. Always turn off hyphenation; it makes -.\" way too many mistakes in technical documents. -.if n .ad l -.nh -.SH "NAME" -delete_imaging_upload.pl \-\- Delete everything that was produced (or part of what was produced) by the imaging pipeline for a given set of imaging uploads - IDs, all associated to the same "tarchive". -.SH "SYNOPSIS" -.IX Header "SYNOPSIS" -perl delete_imaging_upload.pl [\-profile file] [\-ignore] [\-backup_path basename] [\-protocol] [\-form] [\-uploadID list_of_uploadIDs] - [\-type list_of_scan_types] [\-defaced] [\-basename fileBaseName] [\-nosqlbk] [\-nofilesbk] [\-extra_mysqlcnf file] -.PP +# NAME + +delete\_imaging\_upload.pl -- Delete everything that was produced (or part of what was produced) by the imaging pipeline for a given set of imaging uploads + IDs, all associated to the same `tarchive`. + +# SYNOPSIS + +perl delete\_imaging\_upload.pl \[-profile file\] \[-ignore\] \[-backup\_path basename\] \[-protocol\] \[-form\] \[-uploadID list\_of\_uploadIDs\] + \[-type list\_of\_scan\_types\] \[-defaced\] \[-basename fileBaseName\] \[-nosqlbk\] \[-nofilesbk\] \[-extra\_mysqlcnf file\] + Available options are: -.PP -\&\-profile : name of the config file in \f(CW\*(C`../dicom\-archive/.loris_mri\*(C'\fR (defaults to \f(CW\*(C`prod\*(C'\fR). -.PP -\&\-ignore : ignore files whose paths exist in the database but do not exist on the file system. + +\-profile : name of the config file in `../dicom-archive/.loris_mri` (defaults to `prod`). + +\-ignore : ignore files whose paths exist in the database but do not exist on the file system. Default is to abort if such a file is found, irrespective of whether a backup file will - be created by the script (see \f(CW\*(C`\-nofilesbk\*(C'\fR and \f(CW\*(C`\-nosqlbk\*(C'\fR). If this option is used, a + be created by the script (see `-nofilesbk` and `-nosqlbk`). If this option is used, a warning is issued and program execution continues. -.PP -\&\-nofilesbk : when creating the backup file for the deleted upload(s), do not backup the files produced by + +\-nofilesbk : when creating the backup file for the deleted upload(s), do not backup the files produced by the imaging pipeline (default is to backup these files). -.PP -\&\-backup_path : specify the path of the backup file, which by default contains a copy of everything that the - script will delete, both on the file system and in the database (but see \f(CW\*(C`\-nofilesbk\*(C'\fR and - \f(CW\*(C`\-nosqlbk\*(C'\fR). The extension \f(CW\*(C`.tar.gz\*(C'\fR will be added to this base name to build the name of the final + +\-backup\_path <path> : specify the path of the backup file, which by default contains a copy of everything that the + script will delete, both on the file system and in the database (but see `-nofilesbk` and + `-nosqlbk`). The extension `.tar.gz` will be added to this base name to build the name of the final backup file. If a file with this resulting name already exists, an error message is shown and the script - will abort. Note that \f(CW\*(C`path\*(C'\fR can be an absolute path or a path relative to the current directory. A - backup file is always created unless options \f(CW\*(C`\-nofilesbk\*(C'\fR and \f(CW\*(C`\-nosqlbk\*(C'\fR are both used. By default, the - backup file name is \f(CW\*(C`imaging_upload_backup.tar.gz\*(C'\fR and is written in the current directory. Option - \f(CW\*(C`\-backup_path\*(C'\fR cannot be used if \f(CW\*(C`\-nofilesbk\*(C'\fR and \f(CW\*(C`\-nosqlbk\*(C'\fR are also used. -.PP -\&\-basename fileBaseName : basename of the file to delete. The file is assumed to either exist in table \f(CW\*(C`files\*(C'\fR or table - \f(CW\*(C`parameter_file\*(C'\fR. This option should be used when targeting a specific (unique) file for deletion. + will abort. Note that `path` can be an absolute path or a path relative to the current directory. A + backup file is always created unless options `-nofilesbk` and `-nosqlbk` are both used. By default, the + backup file name is `imaging_upload_backup.tar.gz` and is written in the current directory. Option + `-backup_path` cannot be used if `-nofilesbk` and `-nosqlbk` are also used. + +\-basename fileBaseName : basename of the file to delete. The file is assumed to either exist in table `files` or table + `parameter_file`. This option should be used when targeting a specific (unique) file for deletion. Note that the file will be deleted from both the database and the filesystem. This option cannot be - used with options \f(CW\*(C`\-defaced\*(C'\fR and \f(CW\*(C`\-form\*(C'\fR. -.PP -\&\-uploadID : comma-separated list of upload IDs (found in table \f(CW\*(C`mri_upload\*(C'\fR) to delete. The program will - abort if the list contains an upload \s-1ID\s0 that does not exist. Also, all upload IDs must - have the same \f(CW\*(C`tarchive\*(C'\fR \s-1ID\s0 (which can be \f(CW\*(C`NULL\*(C'\fR). -.PP -\&\-protocol : delete the imaging protocol(s) in table \f(CW\*(C`mri_processing_protocol\*(C'\fR associated to either the - upload(s) specified via the \f(CW\*(C`\-uploadID\*(C'\fR option or any file that was produced using this (these) + used with options `-defaced` and `-form`. + +\-uploadID : comma-separated list of upload IDs (found in table `mri_upload`) to delete. The program will + abort if the list contains an upload ID that does not exist. Also, all upload IDs must + have the same `tarchive` ID (which can be `NULL`). + +\-protocol : delete the imaging protocol(s) in table `mri_processing_protocol` associated to either the + upload(s) specified via the `-uploadID` option or any file that was produced using this (these) upload(s). Let F be the set of files directly or indirectly associated to the upload(s) to delete. - This option must be used if there is at least one record in \f(CW\*(C`mri_processing_protocol\*(C'\fR that is tied + This option must be used if there is at least one record in `mri_processing_protocol` that is tied only to files in F. Protocols that are tied to files not in F are never deleted. If the files in F do not have a protocol associated to them, the switch is ignored if used. -.PP -\&\-form : delete the entries in \f(CW\*(C`mri_parameter_form\*(C'\fR associated to the upload(s) passed on - the command line, if any (default is \s-1NOT\s0 to delete them). -.PP -\&\-type : comma-separated list of scan type names to delete. All the names must exist in table \f(CW\*(C`mri_scan_type\*(C'\fR or - the script will issue an error. This option cannot be used in conjunction with \f(CW\*(C`\-defaced\*(C'\fR. -.PP -\&\-defaced : fetch the scan types listed in config setting \f(CW\*(C`modalities_to_delete\*(C'\fR and perform a deletion of these scan - types as if their names were used with option \f(CW\*(C`\-type\*(C'\fR. Once all deletions are done, set the \f(CW\*(C`SourceFileID\*(C'\fR - and \f(CW\*(C`TarchiveSource\*(C'\fR of all the defaced files in table to \f(CW\*(C`NULL\*(C'\fR and to the tarchive \s-1ID\s0 of the - upload(s) whose arguments were passed to \f(CW\*(C`\-uploadID\*(C'\fR, respectively. -.PP -\&\-nosqlbk : when creating the backup file, do not add to it an \s-1SQL\s0 file that contains the statements used to restore + +\-form : delete the entries in `mri_parameter_form` associated to the upload(s) passed on + the command line, if any (default is NOT to delete them). + +\-type : comma-separated list of scan type names to delete. All the names must exist in table `mri_scan_type` or + the script will issue an error. This option cannot be used in conjunction with `-defaced`. + +\-defaced : fetch the scan types listed in config setting `modalities_to_delete` and perform a deletion of these scan + types as if their names were used with option `-type`. Once all deletions are done, set the `SourceFileID` + and `TarchiveSource` of all the defaced files in table <files> to `NULL` and to the tarchive ID of the + upload(s) whose arguments were passed to `-uploadID`, respectively. + +\-nosqlbk : when creating the backup file, do not add to it an SQL file that contains the statements used to restore the database to the state it had before the script was invoked. Adding this file, which will be named - \f(CW\*(C`imaging_upload_restore.sql\*(C'\fR, to the backup file is the default behaviour. -.PP -\&\-dumpOptions : options to add to the mysqldump command. By default, the following options are used - \f(CW\*(C`\-\-no\-create\-info \-\-compact \-\-skip\-extended\-insert \-\-no\-tablespaces\*(C'\fR. - Example of additional option: \f(CW\*(C`\-\-column\-statistics=0\*(C'\fR to disable column statistics flag in + `imaging_upload_restore.sql`, to the backup file is the default behaviour. + +\-dumpOptions : options to add to the mysqldump command. By default, the following options are used + `--no-create-info --compact --skip-extended-insert --no-tablespaces`. + Example of additional option: `--column-statistics=0` to disable column statistics flag in mysqldump 8. -.PP -\&\-extra_mysqlcnf : MySql config file containing the password to log in the database (see - \f(CW\*(C`https://dev.mysql.com/doc/refman/8.0/en/option\-files.html\*(C'\fR for a description of this file's format). - This file should only contain the password stored in the \f(CW@db\fR array defined in your \f(CW\*(C`prod\*(C'\fR file. - You should use that option if you do not have a default \f(CW\*(C`.mysql.cnf\*(C'\fR config file containing the credentials - to log in the database. If this option is used, \f(CW\*(C`delete_imaging_upload.pl\*(C'\fR will pass the option - \f(CW\*(C`\-\-defaults\-extra\-file=file\*(C'\fR to the \f(CW\*(C`mysqldump\*(C'\fR command when creating a backup of the \s-1SQL\s0 tables. Note that + +\-extra\_mysqlcnf <file> : MySql config file containing the password to log in the database (see + `https://dev.mysql.com/doc/refman/8.0/en/option-files.html` for a description of this file's format). + This file should only contain the password stored in the `@db` array defined in your `prod` file. + You should use that option if you do not have a default `.mysql.cnf` config file containing the credentials + to log in the database. If this option is used, `delete_imaging_upload.pl` will pass the option + `--defaults-extra-file=file` to the `mysqldump` command when creating a backup of the SQL tables. Note that only the password is fetched in this file, the host name and user name used to access the database are actually - retrieved from the \f(CW\*(C`prod\*(C'\fR file. -.SH "DESCRIPTION" -.IX Header "DESCRIPTION" + retrieved from the `prod` file. + +# DESCRIPTION + This program deletes an imaging upload or specific parts of it from the database and the file system. There are four possible ways in which this script can be used: -.PP -1. Delete everything that is associated to the upload \s-1ID\s0(s). Basically, for uploads on which the \s-1MRI\s0 pipeline was + +1\. Delete everything that is associated to the upload ID(s). Basically, for uploads on which the MRI pipeline was successfully run, this removes all records tied to the upload in the following tables: - a) \f(CW\*(C`notification_spool\*(C'\fR - b) \f(CW\*(C`files\*(C'\fR - c) \f(CW\*(C`tarchive_series\*(C'\fR and \f(CW\*(C`tarchive_files\*(C'\fR - d) \f(CW\*(C`mri_protocol_violated_scans\*(C'\fR, \f(CW\*(C`MRICandidateErrors\*(C'\fR and \f(CW\*(C`mri_violations_log\*(C'\fR - e) \f(CW\*(C`files_intermediary\*(C'\fR. - f) \f(CW\*(C`parameter_file\*(C'\fR - g) \f(CW\*(C`tarchive\*(C'\fR - h) \f(CW\*(C`mri_upload\*(C'\fR - i) \f(CW\*(C`bids_export_files\*(C'\fR - j) \f(CW\*(C`mri_processing_protocol\*(C'\fR if option \f(CW\*(C`\-protocol\*(C'\fR is used (see below) - k) \f(CW\*(C`mri_parameter_form\*(C'\fR if option \f(CW\*(C`\-form\*(C'\fR is used (see below) -.PP + a) `notification_spool` + b) `files` + c) `tarchive_series` and `tarchive_files` + d) `mri_protocol_violated_scans`, `MRICandidateErrors` and `mri_violations_log` + e) `files_intermediary`. + f) `parameter_file` + g) `tarchive` + h) `mri_upload` + i) `bids_export_files` + j) `mri_processing_protocol` if option `-protocol` is used (see below) + k) `mri_parameter_form` if option `-form` is used (see below) + All the deletions and modifications performed in the database are done as part of a single transaction, so they either -all succeed or a rollback is performed and the database is not modified in any way. The \s-1ID\s0 of the upload to delete -is specified via option \f(CW\*(C`\-uploadID\*(C'\fR. More than one upload can be deleted if they all have the same \f(CW\*(C`TarchiveID\*(C'\fR -in table \f(CW\*(C`mri_upload\*(C'\fR: option \f(CW\*(C`\-uploadID\*(C'\fR can take as argument a comma-separated list of upload IDs for this case. -If an upload that is deleted is the only one that was associated to a given session, the script will set the \f(CW\*(C`Scan_done\*(C'\fR -value for that session to 'N'. If option \f(CW\*(C`\-form\*(C'\fR is used, the \f(CW\*(C`mri_parameter_form\*(C'\fR and its associated \f(CW\*(C`flag\*(C'\fR record -are also deleted, for each deleted upload. If option \f(CW\*(C`\-protocol\*(C'\fR is used and if there is a record in table -\&\f(CW\*(C`mri_processing_protocol\*(C'\fR that is tied only to the deleted upload(s), then that record is also deleted. -.PP -\&\f(CW\*(C`delete_imaging_upload.pl\*(C'\fR cannot be used to delete an upload that has an associated \s-1MINC\s0 file that has been QCed. -In other words, if there is a \s-1MINC\s0 file tied to the upload that is targeted for deletion and if that \s-1MINC\s0 file has -an associated record in table \f(CW\*(C`files_qcstatus\*(C'\fR or \f(CW\*(C`feedback_mri_comments\*(C'\fR, the script will issue an error message +all succeed or a rollback is performed and the database is not modified in any way. The ID of the upload to delete +is specified via option `-uploadID`. More than one upload can be deleted if they all have the same `TarchiveID` +in table `mri_upload`: option `-uploadID` can take as argument a comma-separated list of upload IDs for this case. +If an upload that is deleted is the only one that was associated to a given session, the script will set the `Scan_done` +value for that session to 'N'. If option `-form` is used, the `mri_parameter_form` and its associated `flag` record +are also deleted, for each deleted upload. If option `-protocol` is used and if there is a record in table +`mri_processing_protocol` that is tied only to the deleted upload(s), then that record is also deleted. + +`delete_imaging_upload.pl` cannot be used to delete an upload that has an associated MINC file that has been QCed. +In other words, if there is a MINC file tied to the upload that is targeted for deletion and if that MINC file has +an associated record in table `files_qcstatus` or `feedback_mri_comments`, the script will issue an error message and exit. -.PP + Before deleting any records in the database, the script will verify that all the records in tables a) through j) that -represent file names (e.g. a record in \f(CW\*(C`files\*(C'\fR, a protocol file in \f(CW\*(C`mri_processing_protocol\*(C'\fR, etc...) refer to files +represent file names (e.g. a record in `files`, a protocol file in `mri_processing_protocol`, etc...) refer to files that actually exist on the file system. If it finds a record that does not meet that criterion, the script issues an -error message and exits, leaving the database untouched. To avoid this check, use option \f(CW\*(C`\-ignore\*(C'\fR. Each time a file +error message and exits, leaving the database untouched. To avoid this check, use option `-ignore`. Each time a file record is deleted, the file it refers to on the file system is also deleted. A backup will be created by -\&\f(CW\*(C`delete_imaging_upload.pl\*(C'\fR of all the files that were deleted during execution. Option \f(CW\*(C`\-nofilesbk\*(C'\fR can be used to -prevent this. If created, the backup file will be named \f(CW\*(C`imaging_upload_backup.tar.gz\*(C'\fR. This name can be changed with -option \f(CW\*(C`\-backup_path\*(C'\fR. Note that the file paths inside this backup archive are absolute. To restore the files in the archive, -one must use \f(CW\*(C`tar\*(C'\fR with option \f(CW\*(C`\-\-absolute\-names\*(C'\fR. -.PP +`delete_imaging_upload.pl` of all the files that were deleted during execution. Option `-nofilesbk` can be used to +prevent this. If created, the backup file will be named `imaging_upload_backup.tar.gz`. This name can be changed with +option `-backup_path`. Note that the file paths inside this backup archive are absolute. To restore the files in the archive, +one must use `tar` with option `--absolute-names`. + The script will also create a file that contains a backup of all the information that was deleted or modified from the -database tables. This backup is created using \f(CW\*(C`mysqldump\*(C'\fR and contains an \f(CW\*(C`INSERT\*(C'\fR statement for every record erased. -When running \f(CW\*(C`mysqldump\*(C'\fR the script uses the database credentials in file \f(CW\*(C`~/.my.cnf\*(C'\fR to connect to the database. Option -\&\f(CW\*(C`\-extra_mysqlcnf\*(C'\fR has to be used to specify an alternate credentials file when the default credential file does not exist -(see \f(CW\*(C`https://dev.mysql.com/doc/refman/8.0/en/option\-files.html\*(C'\fR for a descrption of MySQL option files format). -The backup produced by \f(CW\*(C`mysqldump\*(C'\fR will be part of the backup archive mentioned above unless option \f(CW\*(C`\-nosqlbk\*(C'\fR is used. -If sourced back into the database with \f(CW\*(C`mysql\*(C'\fR, it should allow the database to be exactly like it was before -\&\f(CW\*(C`delete_imaging_upload.pl\*(C'\fR was invoked, provided the database was not modified in the meantime. The \s-1SQL\s0 backup file will -be named \f(CW\*(C`imaging_upload_restore.sql\*(C'\fR. -.PP -2. Delete specific scan types from an archive. The behaviour of the script is identical to the one described above, except +database tables. This backup is created using `mysqldump` and contains an `INSERT` statement for every record erased. +When running `mysqldump` the script uses the database credentials in file `~/.my.cnf` to connect to the database. Option +`-extra_mysqlcnf` has to be used to specify an alternate credentials file when the default credential file does not exist +(see `https://dev.mysql.com/doc/refman/8.0/en/option-files.html` for a descrption of MySQL option files format). +The backup produced by `mysqldump` will be part of the backup archive mentioned above unless option `-nosqlbk` is used. +If sourced back into the database with `mysql`, it should allow the database to be exactly like it was before +`delete_imaging_upload.pl` was invoked, provided the database was not modified in the meantime. The SQL backup file will +be named `imaging_upload_restore.sql`. + +2\. Delete specific scan types from an archive. The behaviour of the script is identical to the one described above, except that: - a) the deletions are limited to \s-1MINC\s0 files of a specific scan type: use option \f(CW\*(C`\-type\*(C'\fR with a comma-separated list + a) the deletions are limited to MINC files of a specific scan type: use option `-type` with a comma-separated list of scan type names to specify which ones. - b) everything associated to the \s-1MINC\s0 files deleted in a) is also deleted: this includes the processed files in - \f(CW\*(C`files_intermediary\*(C'\fR and the records in \f(CW\*(C`mri_violations_log\*(C'\fR. - c) if \f(CW\*(C`\-protocol\*(C'\fR is used and there is an entry in table \f(CW\*(C`mri_processing_protocol\*(C'\fR that is tied only to the files + b) everything associated to the MINC files deleted in a) is also deleted: this includes the processed files in + `files_intermediary` and the records in `mri_violations_log`. + c) if `-protocol` is used and there is an entry in table `mri_processing_protocol` that is tied only to the files deleted in a), then that record is also deleted. - d) tables \f(CW\*(C`tarchive_series\*(C'\fR, \f(CW\*(C`tarchive_files\*(C'\fR, \f(CW\*(C`tarchive\*(C'\fR, \f(CW\*(C`mri_upload\*(C'\fR, \f(CW\*(C`notification_spool\*(C'\fR, \f(CW\*(C`MRICandidateErrors\*(C'\fR - and \f(CW\*(C`mri_parameter_form\*(C'\fR are never modified. - Note that option \f(CW\*(C`\-type\*(C'\fR cannot be used in conjunction with either option \f(CW\*(C`\-form\*(C'\fR or option \f(CW\*(C`\-defaced\*(C'\fR. -.PP -3. Replace \s-1MINC\s0 files with their defaced counterparts. This is the behaviour obtained when option \f(CW\*(C`\-defaced\*(C'\fR is used. As far as + d) tables `tarchive_series`, `tarchive_files`, `tarchive`, `mri_upload`, `notification_spool`, `MRICandidateErrors` + and `mri_parameter_form` are never modified. + Note that option `-type` cannot be used in conjunction with either option `-form` or option `-defaced`. + +3\. Replace MINC files with their defaced counterparts. This is the behaviour obtained when option `-defaced` is used. As far as deletions go, the behaviour of the script in this case is identical to the one described in 2), except that the list of - scan types to delete is fetched from the config setting \f(CW\*(C`modalities_to_deface\*(C'\fR. Use of option \f(CW\*(C`\-defaced\*(C'\fR is not permitted - in conjunction with option \f(CW\*(C`\-type\*(C'\fR or option \f(CW\*(C`\-form\*(C'\fR. Once all deletions are made, the script will change the \f(CW\*(C`SourceFileID\*(C'\fR - of all defaced files to \f(CW\*(C`NULL\*(C'\fR and set the \f(CW\*(C`TarchiveSource\*(C'\fR of all defaced files to the \f(CW\*(C`TarchiveID\*(C'\fR of the upload(s). - This effectively \*(L"replaces\*(R" the original \s-1MINC\s0 files with their corresponding defaced versions. Note that the script will issue + scan types to delete is fetched from the config setting `modalities_to_deface`. Use of option `-defaced` is not permitted + in conjunction with option `-type` or option `-form`. Once all deletions are made, the script will change the `SourceFileID` + of all defaced files to `NULL` and set the `TarchiveSource` of all defaced files to the `TarchiveID` of the upload(s). + This effectively "replaces" the original MINC files with their corresponding defaced versions. Note that the script will issue an error message and abort, leaving the database and file system untouched, if: - a) A \s-1MINC\s0 file should have a corresponding defaced file but does not. - b) A \s-1MINC\s0 file that has been defaced has an associated file that is not a defaced file. -.PP -4. Delete a specific file that exists in table \f(CW\*(C`files\*(C'\fR or table \f(CW\*(C`parameter_file\*(C'\fR (in which case it will be associated to parameter - \f(CW\*(C`check_pic_filename\*(C'\fR, \f(CW\*(C`check_nii_filename\*(C'\fR, \f(CW\*(C`check_bval_filename\*(C'\fR or \f(CW\*(C`check_bvec_filename\*(C'\fR). Note that once a file is - deleted, all the files that have been derived/built using this deleted file will also be deleted. Use option \f(CW\*(C`\-basename fileBaseName\*(C'\fR, - where \f(CW\*(C`fileBaseName\*(C'\fR is the basename of the file to delete, along with option \f(CW\*(C`\-uploadID\*(C'\fR to use the script this way. -.SS "Methods" -.IX Subsection "Methods" -\fIprintExitMessage($filesRef, \f(CI$scanTypesToDeleteRef\fI, \f(CI$deleteResultsRef\fI)\fR -.IX Subsection "printExitMessage($filesRef, $scanTypesToDeleteRef, $deleteResultsRef)" -.PP -Prints an appropriate message before exiting. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$filesRef:\fR reference to the array that contains the file information for all the files + a) A MINC file should have a corresponding defaced file but does not. + b) A MINC file that has been defaced has an associated file that is not a defaced file. + +4\. Delete a specific file that exists in table `files` or table `parameter_file` (in which case it will be associated to parameter + `check_pic_filename`, `check_nii_filename`, `check_bval_filename` or `check_bvec_filename`). Note that once a file is + deleted, all the files that have been derived/built using this deleted file will also be deleted. Use option `-basename fileBaseName`, + where `fileBaseName` is the basename of the file to delete, along with option `-uploadID` to use the script this way. + +## Methods + +### printExitMessage($filesRef, $scanTypesToDeleteRef, $deleteResultsRef) + +Prints an appropriate message before exiting. + +INPUTS: + - $filesRef: reference to the array that contains the file information for all the files that are associated to the upload(s) passed on the command line. - \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. - \- \f(CW$deleteResultsRef:\fR reference on the hash that contains the result of the deletion of the records in the - database. -.PP -\fIprettyListPrint($listRef, \f(CI$andOr\fI)\fR -.IX Subsection "prettyListPrint($listRef, $andOr)" -.PP -Pretty prints a list in string form (e.g \*(L"1, 2, 3 and 4\*(R" or \*(L"7, 8 or 9\*(R"). -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$listRef:\fR the list of elements to print, separated by commas. - \- \f(CW$andOr:\fR whether to join the last element with the rest of the elements using an 'and' or an 'or'. -.PP -\fIgetMriUploadFiles($dbh, \f(CI$uploadIDsRef\fI)\fR -.IX Subsection "getMriUploadFiles($dbh, $uploadIDsRef)" -.PP -Finds the list of \f(CW\*(C`mri_upload\*(C'\fR that match the upload IDs passed on the command line -via option \f(CW\*(C`\-uploadID\*(C'\fR. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$dbh:\fR database handle reference. - \- \f(CW$uploadIDsRef:\fR reference to the array that contains the upload IDs passed on the command line. -.PP -\&\s-1RETURNS:\s0 - \- a reference on an array that contains the \f(CW\*(C`mri_ipload\*(C'\fRs retrieved from the database. -.PP -\fIgetTarchiveFiles($dbh, \f(CI$tarchiveID\fI, \f(CI$tarchiveLibraryDir\fI)\fR -.IX Subsection "getTarchiveFiles($dbh, $tarchiveID, $tarchiveLibraryDir)" -.PP -Retrieves from the database the \f(CW\*(C`tarchive\*(C'\fR associated to the upload \s-1ID\s0(s) passed on the command line. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$dbh:\fR database handle reference. - \- \f(CW$tarchiveID:\fR \s-1ID\s0 of the \f(CW\*(C`tarchive\*(C'\fR to retrieve (can be \f(CW\*(C`undef\*(C'\fR). - \- \f(CW$tarchiveLibraryDir:\fR config setting \f(CW\*(C`tarchiveLibraryDir\*(C'\fR (can be \f(CW\*(C`undef\*(C'\fR if none is defined in the + - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. + - $deleteResultsRef: reference on the hash that contains the result of the deletion of the records in the + database. + +### prettyListPrint($listRef, $andOr) + +Pretty prints a list in string form (e.g "1, 2, 3 and 4" or "7, 8 or 9"). + +INPUTS: + - $listRef: the list of elements to print, separated by commas. + - $andOr: whether to join the last element with the rest of the elements using an 'and' or an 'or'. + +### getMriUploadFiles($dbh, $uploadIDsRef) + +Finds the list of `mri_upload` that match the upload IDs passed on the command line +via option `-uploadID`. + +INPUTS: + - $dbh: database handle reference. + - $uploadIDsRef: reference to the array that contains the upload IDs passed on the command line. + +RETURNS: + - a reference on an array that contains the `mri_ipload`s retrieved from the database. + +### getTarchiveFiles($dbh, $tarchiveID, $tarchiveLibraryDir) + +Retrieves from the database the `tarchive` associated to the upload ID(s) passed on the command line. + +INPUTS: + - $dbh: database handle reference. + - $tarchiveID: ID of the `tarchive` to retrieve (can be `undef`). + - $tarchiveLibraryDir: config setting `tarchiveLibraryDir` (can be `undef` if none is defined in the database). -.PP -\&\s-1RETURNS:\s0 - \- a reference on an array that contains the \f(CW\*(C`tarchive\*(C'\fR associated to the upload \s-1ID\s0(s) passed on the command line. - This array can contain one element (if the uploads are all tied to the same \f(CW\*(C`tarchive\*(C'\fR) or be empty (if all the - uploads have a \f(CW\*(C`TarchiveID\*(C'\fR set to \f(CW\*(C`NULL\*(C'\fR). If the \f(CW\*(C`ArchiveLocation\*(C'\fR for the \f(CW\*(C`tarchive\*(C'\fR is a relative path and - if config setting \f(CW\*(C`tarchiveLibraryDir\*(C'\fRis not defined, the return value is \f(CW\*(C`undef\*(C'\fR, indicating that something is + +RETURNS: + - a reference on an array that contains the `tarchive` associated to the upload ID(s) passed on the command line. + This array can contain one element (if the uploads are all tied to the same `tarchive`) or be empty (if all the + uploads have a `TarchiveID` set to `NULL`). If the `ArchiveLocation` for the `tarchive` is a relative path and + if config setting `tarchiveLibraryDir`is not defined, the return value is `undef`, indicating that something is wrong. -.PP -\fIvalidateMriUploads($mriUploadsRef, \f(CI$optionsRef\fI, \f(CI$scanTypeList\fI)\fR -.IX Subsection "validateMriUploads($mriUploadsRef, $optionsRef, $scanTypeList)" -.PP + +### validateMriUploads($mriUploadsRef, $optionsRef, $scanTypeList) + Validates that the list of upload IDs passed on the commamnd line are valid arguments for the script. It one of them is invalid, an error message is displayed and the program exits. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$mriUploadsRef:\fR reference on an array of hashes containing the uploads to delete. Accessed like this: - \f(CW\*(C`$mriUploadsRef\->[0]\->{\*(AqTarchiveID\*(Aq}\*(C'\fR(this would return the \f(CW\*(C`TarchiveID\*(C'\fR of the first \f(CW\*(C`mri_upload\*(C'\fR - in the array. The properties stored for each hash are: \f(CW\*(C`UploadID\*(C'\fR, \f(CW\*(C`TarchiveID\*(C'\fR, \f(CW\*(C`FullPath\*(C'\fR - \f(CW\*(C`Inserting\*(C'\fR, \f(CW\*(C`InsertionComplete\*(C'\fR and \f(CW\*(C`SessionID\*(C'\fR. - \- \f(CW$optionsRef:\fR reference to the array that contains the options passed on the command line. - \- \f(CW$scanTypeList:\fR list of scan types to delete passed on the command line. -.PP -\fIgetMriProcessingProtocolFilesRef($dbh, \f(CI$filesRef\fI)\fR -.IX Subsection "getMriProcessingProtocolFilesRef($dbh, $filesRef)" -.PP -Finds the list of \f(CW\*(C`ProcessingProtocolID\*(C'\fRs to delete, namely those in table -\&\f(CW\*(C`mri_processing_protocol\*(C'\fR associated to the files to delete, and *only* to + +INPUTS: + - $mriUploadsRef: reference on an array of hashes containing the uploads to delete. Accessed like this: + `$mriUploadsRef->[0]->{'TarchiveID'}`(this would return the `TarchiveID` of the first `mri_upload` + in the array. The properties stored for each hash are: `UploadID`, `TarchiveID`, `FullPath` + `Inserting`, `InsertionComplete` and `SessionID`. + - $optionsRef: reference to the array that contains the options passed on the command line. + - $scanTypeList: list of scan types to delete passed on the command line. + +### getMriProcessingProtocolFilesRef($dbh, $filesRef) + +Finds the list of `ProcessingProtocolID`s to delete, namely those in table +`mri_processing_protocol` associated to the files to delete, and \*only\* to those files that are going to be deleted. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$dbh:\fR database handle reference. - \- \f(CW$filesRef:\fR reference to the array that contains the file informations for all the files + +INPUTS: + - $dbh: database handle reference. + - $filesRef: reference to the array that contains the file informations for all the files that are associated to the upload(s) passed on the command line. -.PP -\&\s-1RETURNS:\s0 - \- reference on an array that contains the \f(CW\*(C`ProcessProtocolID\*(C'\fR in table \f(CW\*(C`mri_processing_protocol\*(C'\fR - associated to the files to delete. This array has two keys: \f(CW\*(C`ProcessProtocolID\*(C'\fR => the protocol - process \s-1ID\s0 found in table \f(CW\*(C`mri_processing_protocol\*(C'\fR and \f(CW\*(C`FullPath\*(C'\fR => the value of \f(CW\*(C`ProtocolFile\*(C'\fR + +RETURNS: + - reference on an array that contains the `ProcessProtocolID` in table `mri_processing_protocol` + associated to the files to delete. This array has two keys: `ProcessProtocolID` => the protocol + process ID found in table `mri_processing_protocol` and `FullPath` => the value of `ProtocolFile` in the same table. -.PP -\fIhasQcOrComment($dbh, \f(CI$mriUploadsRef\fI)\fR -.IX Subsection "hasQcOrComment($dbh, $mriUploadsRef)" -.PP -Determines if any of the \s-1MINC\s0 files associated to the \f(CW\*(C`tarchive\*(C'\fR have \s-1QC\s0 + +### hasQcOrComment($dbh, $mriUploadsRef) + +Determines if any of the MINC files associated to the `tarchive` have QC information associated to them by looking at the contents of tables -\&\f(CW\*(C`files_qcstatus\*(C'\fR and \f(CW\*(C`feedback_mri_comments\*(C'\fR. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$dbh:\fR database handle reference. - \- \f(CW$mriUploadsRef:\fR reference on an array of hashes containing the uploads to delete. Accessed like this: - \f(CW\*(C`$mriUploadsRef\->[0]\->{\*(AqTarchiveID\*(Aq}\*(C'\fR(this would return the \f(CW\*(C`TarchiveID\*(C'\fR of the first \f(CW\*(C`mri_upload\*(C'\fR - in the array. The properties stored for each hash are: \f(CW\*(C`UploadID\*(C'\fR, \f(CW\*(C`TarchiveID\*(C'\fR, \f(CW\*(C`FullPath\*(C'\fR - \f(CW\*(C`Inserting\*(C'\fR, \f(CW\*(C`InsertionComplete\*(C'\fR and \f(CW\*(C`SessionID\*(C'\fR. -.PP -\&\s-1RETURNS:\s0 - \- 1 if there is \s-1QC\s0 information associated to the \s-1DICOM\s0 archive(s), 0 otherwise. -.PP -\fIgetFilesRef($dbh, \f(CI$tarchiveID\fI, \f(CI$dataDirBasePath\fI, \f(CI$scanTypesToDeleteRef\fI, \f(CI$optionsRef\fI)\fR -.IX Subsection "getFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef)" -.PP -Get the absolute paths of all the files associated to a \s-1DICOM\s0 archive that are listed in -table \f(CW\*(C`files\*(C'\fR. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$dbhr\fR : database handle reference. - \- \f(CW$tarchiveID:\fR \s-1ID\s0 of the \s-1DICOM\s0 archive. - \- \f(CW$dataDirBasePath:\fR config value of setting \f(CW\*(C`dataDirBasePath\*(C'\fR. - \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of names of scan types to delete. - \- \f(CW$optionsRef:\fR reference on the array of command line options. -.PP -\&\s-1RETURNS:\s0 - \- an array of hash references. Each hash has three keys: \f(CW\*(C`FileID\*(C'\fR => \s-1ID\s0 of a file in table \f(CW\*(C`files\*(C'\fR, - \f(CW\*(C`File\*(C'\fR => value of column \f(CW\*(C`File\*(C'\fR for the file with the given \s-1ID\s0 and \f(CW\*(C`FullPath\*(C'\fR => absolute path - for the file with the given \s-1ID.\s0 -.PP -\fIgetIntermediaryFilesRef($dbh, \f(CI$filesRef\fI, \f(CI$tarchiveID\fI, \f(CI$dataDirBasePath\fI, \f(CI$scanTypesToDeleteRef\fI, \f(CI$optionsRef\fI)\fR -.IX Subsection "getIntermediaryFilesRef($dbh, $filesRef, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef)" -.PP +`files_qcstatus` and `feedback_mri_comments`. + +INPUTS: + - $dbh: database handle reference. + - $mriUploadsRef: reference on an array of hashes containing the uploads to delete. Accessed like this: + `$mriUploadsRef->[0]->{'TarchiveID'}`(this would return the `TarchiveID` of the first `mri_upload` + in the array. The properties stored for each hash are: `UploadID`, `TarchiveID`, `FullPath` + `Inserting`, `InsertionComplete` and `SessionID`. + +RETURNS: + - 1 if there is QC information associated to the DICOM archive(s), 0 otherwise. + +### getFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef) + +Get the absolute paths of all the files associated to a DICOM archive that are listed in +table `files`. + +INPUTS: + - $dbhr : database handle reference. + - $tarchiveID: ID of the DICOM archive. + - $dataDirBasePath: config value of setting `dataDirBasePath`. + - $scanTypesToDeleteRef: reference to the array that contains the list of names of scan types to delete. + - $optionsRef: reference on the array of command line options. + +RETURNS: + - an array of hash references. Each hash has three keys: `FileID` => ID of a file in table `files`, + `File` => value of column `File` for the file with the given ID and `FullPath` => absolute path + for the file with the given ID. + +### getIntermediaryFilesRef($dbh, $filesRef, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef) + Get the absolute paths of all the intermediary files associated to an archive -that are listed in table \f(CW\*(C`files_intermediary\*(C'\fR. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$dbhr\fR : database handle reference. - \- \f(CW$filesRef:\fR reference on the hash of all files. - \- \f(CW$tarchiveID:\fR \s-1ID\s0 of the \s-1DICOM\s0 archive. - \- \f(CW$dataDirBasePath:\fR config value of setting \f(CW\*(C`dataDirBasePath\*(C'\fR. - \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. - \- \f(CW$optionsRef:\fR reference on the array of command line options. -.PP -\&\s-1RETURNS:\s0 - \- an array of hash references. Each hash has seven keys: \f(CW\*(C`IntermedID\*(C'\fR => \s-1ID\s0 of a file in - table \f(CW\*(C`files_intermediary\*(C'\fR, \f(CW\*(C`Input_FileID\*(C'\fR => \s-1ID\s0 of the file that was used as input to create - the intermediary file, \f(CW\*(C`Output_FileID\*(C'\fR \s-1ID\s0 of the output file, \f(CW\*(C`FileID\*(C'\fR => \s-1ID\s0 of this file in - table \f(CW\*(C`files\*(C'\fR, \f(CW\*(C`File\*(C'\fR => value of column \f(CW\*(C`File\*(C'\fR in table \f(CW\*(C`files\*(C'\fR for the file with the given - \s-1ID,\s0 \f(CW\*(C`SourceFileID\*(C'\fR value of column \f(CW\*(C`SourceFileID\*(C'\fR for the intermediary file and - \f(CW\*(C`FullPath\*(C'\fR => absolute path of the file with the given \s-1ID.\s0 -.PP -\fIgetParameterFilesRef($dbh, \f(CI$filesRef\fI, \f(CI$tarchiveID\fI, \f(CI$dataDirBasePath\fI, \f(CI$scanTypesToDeleteRef\fI, \f(CI$optionsRef\fI)\fR -.IX Subsection "getParameterFilesRef($dbh, $filesRef, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef)" -.PP -Gets the records to delete from table parameter_file. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$dbhr\fR : database handle reference. - \- \f(CW$filesRef:\fR reference on the hash of all files. - \- \f(CW$tarchiveID:\fR \s-1ID\s0 of the \s-1DICOM\s0 archive. - \- \f(CW$dataDirBasePath:\fR config value of setting \f(CW\*(C`dataDirBasePath\*(C'\fR. - \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. - \- \f(CW$optionsRef:\fR reference on the array of command line options. -.PP -\&\s-1RETURNS:\s0 - \- an array of hash references. Each hash has four keys: \f(CW\*(C`FileID\*(C'\fR => FileID of a file - in table \f(CW\*(C`parameter_file\*(C'\fR, \f(CW\*(C`Value\*(C'\fR => value of column \f(CW\*(C`Value\*(C'\fR in table \f(CW\*(C`parameter_file\*(C'\fR - for the file with the given \s-1ID,\s0 \f(CW\*(C`Name\*(C'\fR => name of the parameter and \f(CW\*(C`FullPath\*(C'\fR => absolute - path of the file with the given \s-1ID.\s0 -.PP -\fIgetMriProtocolViolatedScansFilesRef($dbh, \f(CI$tarchiveID\fI, \f(CI$dataDirBasePath\fI, \f(CI$scanTypesToDeleteRef\fI, \f(CI$optionsRef\fI)\fR -.IX Subsection "getMriProtocolViolatedScansFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef)" -.PP -Get the absolute paths of all the files associated to a \s-1DICOM\s0 archive that are listed in -table \f(CW\*(C`mri_protocol_violated_scans\*(C'\fR. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$dbhr\fR : database handle reference. - \- \f(CW$tarchiveID:\fR \s-1ID\s0 of the \s-1DICOM\s0 archive. - \- \f(CW$dataDirBasePath:\fR config value of setting \f(CW\*(C`dataDirBasePath\*(C'\fR. - \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. - \- \f(CW$optionsRef:\fR reference on the array of command line options. -.PP -\&\s-1RETURNS:\s0 - \- an array of hash references. Each hash has three keys: \f(CW\*(C`ID\*(C'\fR => \s-1ID\s0 of the record in table - \f(CW\*(C`mri_protocol_violated_scans\*(C'\fR, \f(CW\*(C`minc_location\*(C'\fR => value of column \f(CW\*(C`minc_location\*(C'\fR in table - \f(CW\*(C`mri_protocol_violated_scans\*(C'\fR for the \s-1MINC\s0 file found and \f(CW\*(C`FullPath\*(C'\fR => absolute path of the \s-1MINC\s0 +that are listed in table `files_intermediary`. + +INPUTS: + - $dbhr : database handle reference. + - $filesRef: reference on the hash of all files. + - $tarchiveID: ID of the DICOM archive. + - $dataDirBasePath: config value of setting `dataDirBasePath`. + - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. + - $optionsRef: reference on the array of command line options. + +RETURNS: + - an array of hash references. Each hash has seven keys: `IntermedID` => ID of a file in + table `files_intermediary`, `Input_FileID` => ID of the file that was used as input to create + the intermediary file, `Output_FileID` ID of the output file, `FileID` => ID of this file in + table `files`, `File` => value of column `File` in table `files` for the file with the given + ID, `SourceFileID` value of column `SourceFileID` for the intermediary file and + `FullPath` => absolute path of the file with the given ID. + +### getParameterFilesRef($dbh, $filesRef, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef) + +Gets the records to delete from table parameter\_file. + +INPUTS: + - $dbhr : database handle reference. + - $filesRef: reference on the hash of all files. + - $tarchiveID: ID of the DICOM archive. + - $dataDirBasePath: config value of setting `dataDirBasePath`. + - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. + - $optionsRef: reference on the array of command line options. + +RETURNS: + - an array of hash references. Each hash has four keys: `FileID` => FileID of a file + in table `parameter_file`, `Value` => value of column `Value` in table `parameter_file` + for the file with the given ID, `Name` => name of the parameter and `FullPath` => absolute + path of the file with the given ID. + +### getMriProtocolViolatedScansFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef) + +Get the absolute paths of all the files associated to a DICOM archive that are listed in +table `mri_protocol_violated_scans`. + +INPUTS: + - $dbhr : database handle reference. + - $tarchiveID: ID of the DICOM archive. + - $dataDirBasePath: config value of setting `dataDirBasePath`. + - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. + - $optionsRef: reference on the array of command line options. + +RETURNS: + - an array of hash references. Each hash has three keys: `ID` => ID of the record in table + `mri_protocol_violated_scans`, `minc_location` => value of column `minc_location` in table + `mri_protocol_violated_scans` for the MINC file found and `FullPath` => absolute path of the MINC file found. -.PP -\fIgetMriViolationsLogFilesRef($dbh, \f(CI$tarchiveID\fI, \f(CI$dataDirBasePath\fI, \f(CI$scanTypesToDeleteRef\fI, \f(CI$optionsRef\fI)\fR -.IX Subsection "getMriViolationsLogFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef)" -.PP + +### getMriViolationsLogFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef) + Get the absolute paths of all the files associated to an archive that are listed in -table \f(CW\*(C`mri_violations_log\*(C'\fR. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$dbhr\fR : database handle reference. - \- \f(CW$tarchiveID:\fR \s-1ID\s0 of the \s-1DICOM\s0 archive. - \- \f(CW$dataDirBasePath:\fR config value of setting \f(CW\*(C`dataDirBasePath\*(C'\fR. - \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. - \- \f(CW$optionsRef:\fR reference on the array of command line options. -.PP -\&\s-1RETURNS:\s0 - an array of hash references. Each hash has three keys: \f(CW\*(C`LogID\*(C'\fR => \s-1ID\s0 of the record in table - \f(CW\*(C`mri_violations_log\*(C'\fR, \f(CW\*(C`MincFile\*(C'\fR => value of column \f(CW\*(C`MincFile\*(C'\fR for the \s-1MINC\s0 file found in table - \f(CW\*(C`mri_violations_log\*(C'\fR and \f(CW\*(C`FullPath\*(C'\fR => absolute path of the \s-1MINC\s0 file. -.PP -\fIgetMRICandidateErrorsFilesRef($dbh, \f(CI$tarchiveID\fI, \f(CI$dataDirBasePath\fI, \f(CI$scanTypeToDeleteRef\fI, \f(CI$optionsRef\fI)\fR -.IX Subsection "getMRICandidateErrorsFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypeToDeleteRef, $optionsRef)" -.PP -Get the absolute paths of all the files associated to a \s-1DICOM\s0 archive that are listed in -table \f(CW\*(C`MRICandidateErrors\*(C'\fR. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$dbh\fR : database handle reference. - \- \f(CW$tarchiveID:\fR \s-1ID\s0 of the \s-1DICOM\s0 archive. - \- \f(CW$dataDirBasePath:\fR config value of setting \f(CW\*(C`dataDirBasePath\*(C'\fR. - \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. - \- \f(CW$optionsRef:\fR reference on the array of command line options. -.PP -\&\s-1RETURNS:\s0 - \- an array of hash references. Each hash has three keys: \f(CW\*(C`ID\*(C'\fR => \s-1ID\s0 of the record in the - table, \f(CW\*(C`MincFile\*(C'\fR => value of column \f(CW\*(C`MincFile\*(C'\fR for the \s-1MINC\s0 file found in table - \f(CW\*(C`MRICandidateErrors\*(C'\fR and \f(CW\*(C`FullPath\*(C'\fR => absolute path of the \s-1MINC\s0 file. -.PP -\fIgetBidsExportFilesRef($dbh, \f(CI$filesRef\fI, \f(CI$dataDirBasePath\fI)\fR -.IX Subsection "getBidsExportFilesRef($dbh, $filesRef, $dataDirBasePath)" -.PP -Gets the imaging file records to delete from table bids_export_files. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$dbhr\fR : database handle reference. - \- \f(CW$filesRef:\fR reference on the hash of all files. - \- \f(CW$tarchiveID:\fR \s-1ID\s0 of the \s-1DICOM\s0 archive. - \- \f(CW$dataDirBasePath:\fR config value of setting \f(CW\*(C`dataDirBasePath\*(C'\fR. - \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. - \- \f(CW$optionsRef:\fR reference on the array of command line options. -.PP -\&\s-1RETURNS:\s0 - \- an array of hash references. Each hash has three keys: \f(CW\*(C`BIDSExportedFileID\*(C'\fR => \s-1ID\s0 of the record in the - table, \f(CW\*(C`FilePath\*(C'\fR => value of column \f(CW\*(C`FilePath\*(C'\fR for the \s-1BIDS\s0 export file found in table - \f(CW\*(C`bids_export_files\*(C'\fR and \f(CW\*(C`FullPath\*(C'\fR => absolute path of the \s-1BIDS\s0 export file. -.PP -\fIgetBidsExportFilesSessionRef($dbh, \f(CI$filesRef\fI, \f(CI$dataDirBasePath\fI)\fR -.IX Subsection "getBidsExportFilesSessionRef($dbh, $filesRef, $dataDirBasePath)" -.PP -Gets the session level records to delete from table bids_export_files (aka scans.tsv and scans.json files). -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$dbhr\fR : database handle reference. - \- \f(CW$filesRef:\fR reference on the hash of all files. - \- \f(CW$dataDirBasePath:\fR config value of setting \f(CW\*(C`dataDirBasePath\*(C'\fR. -.PP -\&\s-1RETURNS:\s0 - \- a hash with scans.tsv full path and a boolean specifying whether the session level scans.tsv file needs to be updated -.PP -\fI\fIupdateBidsScansTsvFile()\fI\fR -.IX Subsection "updateBidsScansTsvFile()" -.PP -Modifies the scans.tsv \s-1BIDS\s0 file if not all files for the session have been deleted. It will remove the -entries for the files that were deleted from scans.tsv \s-1BIDS\s0 export file. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$filesRef:\fR reference to the array that contains the file informations for all the files +table `mri_violations_log`. + +INPUTS: + - $dbhr : database handle reference. + - $tarchiveID: ID of the DICOM archive. + - $dataDirBasePath: config value of setting `dataDirBasePath`. + - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. + - $optionsRef: reference on the array of command line options. + +RETURNS: + an array of hash references. Each hash has three keys: `LogID` => ID of the record in table + `mri_violations_log`, `MincFile` => value of column `MincFile` for the MINC file found in table + `mri_violations_log` and `FullPath` => absolute path of the MINC file. + +### getMRICandidateErrorsFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypeToDeleteRef, $optionsRef) + +Get the absolute paths of all the files associated to a DICOM archive that are listed in +table `MRICandidateErrors`. + +INPUTS: + - $dbh : database handle reference. + - $tarchiveID: ID of the DICOM archive. + - $dataDirBasePath: config value of setting `dataDirBasePath`. + - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. + - $optionsRef: reference on the array of command line options. + +RETURNS: + - an array of hash references. Each hash has three keys: `ID` => ID of the record in the + table, `MincFile` => value of column `MincFile` for the MINC file found in table + `MRICandidateErrors` and `FullPath` => absolute path of the MINC file. + +### getBidsExportFilesRef($dbh, $filesRef, $dataDirBasePath) + +Gets the imaging file records to delete from table bids\_export\_files. + +INPUTS: + - $dbhr : database handle reference. + - $filesRef: reference on the hash of all files. + - $tarchiveID: ID of the DICOM archive. + - $dataDirBasePath: config value of setting `dataDirBasePath`. + - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. + - $optionsRef: reference on the array of command line options. + +RETURNS: + - an array of hash references. Each hash has three keys: `BIDSExportedFileID` => ID of the record in the + table, `FilePath` => value of column `FilePath` for the BIDS export file found in table + `bids_export_files` and `FullPath` => absolute path of the BIDS export file. + +### getBidsExportFilesSessionRef($dbh, $filesRef, $dataDirBasePath) + +Gets the session level records to delete from table bids\_export\_files (aka scans.tsv and scans.json files). + +INPUTS: + - $dbhr : database handle reference. + - $filesRef: reference on the hash of all files. + - $dataDirBasePath: config value of setting `dataDirBasePath`. + +RETURNS: + - a hash with scans.tsv full path and a boolean specifying whether the session level scans.tsv file needs to be updated + +### updateBidsScansTsvFile() + +Modifies the scans.tsv BIDS file if not all files for the session have been deleted. It will remove the +entries for the files that were deleted from scans.tsv BIDS export file. + +INPUTS: + - $filesRef: reference to the array that contains the file informations for all the files that are associated to the upload(s) passed on the command line. -.PP -\fIsetFileExistenceStatus($filesRef)\fR -.IX Subsection "setFileExistenceStatus($filesRef)" -.PP + +### setFileExistenceStatus($filesRef) + Checks the list of all the files related to the upload(s) that were found in the database and determine whether they exist or not on the file system. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$filesRef:\fR reference to the array that contains the file informations for all the files + +INPUTS: + - $filesRef: reference to the array that contains the file informations for all the files that are associated to the upload(s) passed on the command line. -.PP -\&\s-1RETURNS:\s0 - \- Reference on the list of files that do not exist on the file system. -.PP -\fIshouldExist($table, \f(CI$fileRef\fI)\fR -.IX Subsection "shouldExist($table, $fileRef)" -.PP + +RETURNS: + - Reference on the list of files that do not exist on the file system. + +### shouldExist($table, $fileRef) + Checks whether a file path in the database refers to a file that should exist on the file system. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$table:\fR name of the table in which the file path was found. - \- \f(CW$fileRef:\fR reference to the array that contains the file information for a given file. -.PP -\&\s-1RETURNS:\s0 - \- 0 or 1 depending on whether the file should exist or not. -.PP -\fIbackupFiles($filesRef, \f(CI$scanTypesToDeleteRef\fI, \f(CI$optionsRef\fI)\fR -.IX Subsection "backupFiles($filesRef, $scanTypesToDeleteRef, $optionsRef)" -.PP + +INPUTS: + - $table: name of the table in which the file path was found. + - $fileRef: reference to the array that contains the file information for a given file. + +RETURNS: + - 0 or 1 depending on whether the file should exist or not. + +### backupFiles($filesRef, $scanTypesToDeleteRef, $optionsRef) + Backs up all the files associated to the archive before deleting them. The backed up files will -be stored in a \f(CW\*(C`.tar.gz\*(C'\fR archive in which all paths are absolute. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$filesRef:\fR reference to the array that contains the file informations for all the files +be stored in a `.tar.gz` archive in which all paths are absolute. + +INPUTS: + - $filesRef: reference to the array that contains the file informations for all the files that are associated to the upload(s) passed on the command line. - \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. - \- \f(CW$optionsRef:\fR reference on the hash array of command line options. -.PP -\&\s-1RETURNS:\s0 - \- The number of files backed up. -.PP -\fIshouldDeleteFile($table, \f(CI$fileRef\fI, \f(CI$scanTypesToDeleteRef\fI, \f(CI$keepDefaced\fI, \f(CI$fileBaseName\fI)\fR -.IX Subsection "shouldDeleteFile($table, $fileRef, $scanTypesToDeleteRef, $keepDefaced, $fileBaseName)" -.PP + - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. + - $optionsRef: reference on the hash array of command line options. + +RETURNS: + - The number of files backed up. + +### shouldDeleteFile($table, $fileRef, $scanTypesToDeleteRef, $keepDefaced, $fileBaseName) + Checks whether a given file should be deleted or not. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$table:\fR name of the table in which the file path was found. - \- \f(CW$fileRef:\fR reference to the array that contains the file information for a given file. - \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. - \- \f(CW$keepDefaced:\fR whether the defaced files should be kept or not. - \- \f(CW$fileBaseName:\fR value of option \-basename (or '' if \-basename was not used). -.PP -\&\s-1RETURNS:\s0 - \- 0 or 1 depending on whether the file should be deleted or not. -.PP -\fIdeleteUploadsInDatabase($dbh, \f(CI$filesRef\fI, \f(CI$scanTypesToDeleteRef\fI, \f(CI$optionsRef\fI)\fR -.IX Subsection "deleteUploadsInDatabase($dbh, $filesRef, $scanTypesToDeleteRef, $optionsRef)" -.PP + +INPUTS: + - $table: name of the table in which the file path was found. + - $fileRef: reference to the array that contains the file information for a given file. + - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. + - $keepDefaced: whether the defaced files should be kept or not. + - $fileBaseName: value of option -basename (or '' if -basename was not used). + +RETURNS: + - 0 or 1 depending on whether the file should be deleted or not. + +### deleteUploadsInDatabase($dbh, $filesRef, $scanTypesToDeleteRef, $optionsRef) + This method deletes all information in the database associated to the given upload(s)/scan type combination. -More specifically, it deletes records from tables \f(CW\*(C`notification_spool\*(C'\fR, \f(CW\*(C`tarchive_files\*(C'\fR, \f(CW\*(C`tarchive_series\*(C'\fR -\&\f(CW\*(C`files_intermediary\*(C'\fR, \f(CW\*(C`parameter_file\*(C'\fR, \f(CW\*(C`files\*(C'\fR, \f(CW\*(C`mri_protocol_violated_scans\*(C'\fR, \f(CW\*(C`mri_violations_log\*(C'\fR -\&\f(CW\*(C`MRICandidateErrors\*(C'\fR, \f(CW\*(C`mri_upload\*(C'\fR, \f(CW\*(C`tarchive\*(C'\fR, \f(CW\*(C`mri_processing_protocol\*(C'\fR and \f(CW\*(C`mri_parameter_form\*(C'\fR -(the later is done only if requested). It will also set the \f(CW\*(C`Scan_done\*(C'\fR value of the scan's session to 'N' for +More specifically, it deletes records from tables `notification_spool`, `tarchive_files`, `tarchive_series` +`files_intermediary`, `parameter_file`, `files`, `mri_protocol_violated_scans`, `mri_violations_log` +`MRICandidateErrors`, `mri_upload`, `tarchive`, `mri_processing_protocol` and `mri_parameter_form` +(the later is done only if requested). It will also set the `Scan_done` value of the scan's session to 'N' for each upload that is the last upload tied to that session. All the delete/update operations are done inside a single transaction so either they all succeed or they all fail (and a rollback is performed). -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$dbh\fR : database handle. - \- \f(CW$filesRef\fR : reference to the array that contains the file informations for all the files + +INPUTS: + - $dbh : database handle. + - $filesRef : reference to the array that contains the file informations for all the files that are associated to the upload(s) passed on the command line. - \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. - \- \f(CW$optionsRef:\fR reference on the hash array of the options that were passed on the command line. -.PP -\&\s-1RETURNS:\s0 - \- A reference to a hash with two keys: - * \s-1SQL_BACKUP_DONE\s0 => 1 if this method produced a file containing the \s-1SQL\s0 statements that restore the database state to what it was before calling this + - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. + - $optionsRef: reference on the hash array of the options that were passed on the command line. + +RETURNS: + - A reference to a hash with two keys: + \* SQL\_BACKUP\_DONE => 1 if this method produced a file containing the SQL statements that restore the database state to what it was before calling this method, 0 otherwise. - * \s-1NB_RECORDS_DELETED\s0 => the number of records effectively deleted by this method. -.PP -\fIgzipBackupFile($backupPath)\fR -.IX Subsection "gzipBackupFile($backupPath)" -.PP + \* NB\_RECORDS\_DELETED => the number of records effectively deleted by this method. + +### gzipBackupFile($backupPath) + Compresses the file that contains a backup of everything that was deleted by the script, both -from the file system and the database, using \f(CW\*(C`gzip\*(C'\fR. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$backupPath:\fR path of the backup file to compress (without the .tar.gz extension). -.PP -\fIupdateSessionTable($dbh, \f(CI$mriUploadsRef\fI, \f(CI$tmpSQLFile\fI)\fR -.IX Subsection "updateSessionTable($dbh, $mriUploadsRef, $tmpSQLFile)" -.PP -Sets to \f(CW\*(C`N\*(C'\fR the \f(CW\*(C`Scan_done\*(C'\fR column of all \f(CW\*(C`sessions\*(C'\fR in the database that do not have an associated upload -after the script has deleted those whose IDs are passed on the command line. The script also adds an \s-1SQL\s0 statement -in the \s-1SQL\s0 file whose path is passed as argument to restore the state that the \f(CW\*(C`session\*(C'\fR table had before the deletions. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$dbh\fR : database handle. - \- \f(CW$mriUploadsRef:\fR reference on an array of hashes containing the uploads to delete. Accessed like this: - \f(CW\*(C`$mriUploadsRef\->[0]\->{\*(AqTarchiveID\*(Aq}\*(C'\fR(this would return the \f(CW\*(C`TarchiveID\*(C'\fR of the first \f(CW\*(C`mri_upload\*(C'\fR - in the array. The properties stored for each hash are: \f(CW\*(C`UploadID\*(C'\fR, \f(CW\*(C`TarchiveID\*(C'\fR, \f(CW\*(C`FullPath\*(C'\fR - \f(CW\*(C`Inserting\*(C'\fR, \f(CW\*(C`InsertionComplete\*(C'\fR and \f(CW\*(C`SessionID\*(C'\fR. - \- \f(CW$tmpSQLFile:\fR path of the \s-1SQL\s0 file that contains the \s-1SQL\s0 statements used to restore the deleted records. -.PP -\fIupdateFilesIntermediaryTable($dbh, \f(CI$filesRef\fI, \f(CI$tmpSQLFile\fI)\fR -.IX Subsection "updateFilesIntermediaryTable($dbh, $filesRef, $tmpSQLFile)" -.PP -Sets the \f(CW\*(C`TarchiveSource\*(C'\fR and \f(CW\*(C`SourceFileID\*(C'\fR columns of all the defaced files to \f(CW$tarchiveID\fR and \f(CW\*(C`NULL\*(C'\fR -respectively. The script also adds an \s-1SQL\s0 statement in the \s-1SQL\s0 file whose path is passed as argument to -restore the state that the defaced files in the \f(CW\*(C`files\*(C'\fR table had before the deletions. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$dbh\fR : database handle. - \- \f(CW$filesRef:\fR reference to the array that contains the file informations for all the files +from the file system and the database, using `gzip`. + +INPUTS: + - $backupPath: path of the backup file to compress (without the .tar.gz extension). + +### updateSessionTable($dbh, $mriUploadsRef, $tmpSQLFile) + +Sets to `N` the `Scan_done` column of all `sessions` in the database that do not have an associated upload +after the script has deleted those whose IDs are passed on the command line. The script also adds an SQL statement +in the SQL file whose path is passed as argument to restore the state that the `session` table had before the deletions. + +INPUTS: + - $dbh : database handle. + - $mriUploadsRef: reference on an array of hashes containing the uploads to delete. Accessed like this: + `$mriUploadsRef->[0]->{'TarchiveID'}`(this would return the `TarchiveID` of the first `mri_upload` + in the array. The properties stored for each hash are: `UploadID`, `TarchiveID`, `FullPath` + `Inserting`, `InsertionComplete` and `SessionID`. + - $tmpSQLFile: path of the SQL file that contains the SQL statements used to restore the deleted records. + +### updateFilesIntermediaryTable($dbh, $filesRef, $tmpSQLFile) + +Sets the `TarchiveSource` and `SourceFileID` columns of all the defaced files to `$tarchiveID` and `NULL` +respectively. The script also adds an SQL statement in the SQL file whose path is passed as argument to +restore the state that the defaced files in the `files` table had before the deletions. + +INPUTS: + - $dbh : database handle. + - $filesRef: reference to the array that contains the file informations for all the files that are associated to the upload(s) passed on the command line. - \- \f(CW$tmpSQLFile:\fR path of the \s-1SQL\s0 file that contains the \s-1SQL\s0 statements used to restore the deleted records. -.PP -\fIdeleteMriParameterForm($dbh, \f(CI$mriUploadsRef\fI, \f(CI$tmpSQLFile\fI, \f(CI$optionsRef\fI)\fR -.IX Subsection "deleteMriParameterForm($dbh, $mriUploadsRef, $tmpSQLFile, $optionsRef)" -.PP -Delete the entries in \f(CW\*(C`mri_parameter_form\*(C'\fR (and associated \f(CW\*(C`flag\*(C'\fR entry) for the upload(s) passed on the -command line. The script also adds an \s-1SQL\s0 statement in the \s-1SQL\s0 file whose path is passed as argument to -restore the state that the \f(CW\*(C`mri_parameter_form\*(C'\fR and \f(CW\*(C`flag\*(C'\fR tables had before the deletions. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$dbh\fR : database handle. - \- \f(CW$mriUploadsRef:\fR reference on an array of hashes containing the uploads to delete. Accessed like this: - \f(CW\*(C`$mriUploadsRef\->[0]\->{\*(AqTarchiveID\*(Aq}\*(C'\fR(this would return the \f(CW\*(C`TarchiveID\*(C'\fR of the first \f(CW\*(C`mri_upload\*(C'\fR - in the array. The properties stored for each hash are: \f(CW\*(C`UploadID\*(C'\fR, \f(CW\*(C`TarchiveID\*(C'\fR, \f(CW\*(C`FullPath\*(C'\fR - \f(CW\*(C`Inserting\*(C'\fR, \f(CW\*(C`InsertionComplete\*(C'\fR and \f(CW\*(C`SessionID\*(C'\fR. - \- \f(CW$tmpSQLFile:\fR path of the \s-1SQL\s0 file that contains the \s-1SQL\s0 statements used to restore the deleted records. - \- \f(CW$optionsRef:\fR reference to the array that contains the options passed on the command line. -.PP -\&\s-1RETURNS:\s0 - \- The numbers of records deleted as a result of this operation. -.PP -\fIdeleteUploadsOnFileSystem($filesRef, \f(CI$scanTypesToDeleteRef\fI, \f(CI$keepDefaced\fI, \f(CI$fileBaseName\fI)\fR -.IX Subsection "deleteUploadsOnFileSystem($filesRef, $scanTypesToDeleteRef, $keepDefaced, $fileBaseName)" -.PP + - $tmpSQLFile: path of the SQL file that contains the SQL statements used to restore the deleted records. + +### deleteMriParameterForm($dbh, $mriUploadsRef, $tmpSQLFile, $optionsRef) + +Delete the entries in `mri_parameter_form` (and associated `flag` entry) for the upload(s) passed on the +command line. The script also adds an SQL statement in the SQL file whose path is passed as argument to +restore the state that the `mri_parameter_form` and `flag` tables had before the deletions. + +INPUTS: + - $dbh : database handle. + - $mriUploadsRef: reference on an array of hashes containing the uploads to delete. Accessed like this: + `$mriUploadsRef->[0]->{'TarchiveID'}`(this would return the `TarchiveID` of the first `mri_upload` + in the array. The properties stored for each hash are: `UploadID`, `TarchiveID`, `FullPath` + `Inserting`, `InsertionComplete` and `SessionID`. + - $tmpSQLFile: path of the SQL file that contains the SQL statements used to restore the deleted records. + - $optionsRef: reference to the array that contains the options passed on the command line. + +RETURNS: + - The numbers of records deleted as a result of this operation. + +### deleteUploadsOnFileSystem($filesRef, $scanTypesToDeleteRef, $keepDefaced, $fileBaseName) + This method deletes from the file system all the files associated to the upload(s) passed on the command line that were found on the file system. A warning will be issued for any file that could not be deleted. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$filesRef:\fR reference to the array that contains the file informations for all the files + +INPUTS: + - $filesRef: reference to the array that contains the file informations for all the files that are associated to the upload(s) passed on the command line. - \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. - \- \f(CW$keepDefaced:\fR whether the defaced files should be kept or not. - \- \f(CW$fileBaseName:\fR value of option \-basename (or '' if option was not used). -.PP -\fIgetTypesToDelete($dbh, \f(CI$scanTypeList\fI, \f(CI$keepDefaced\fI)\fR -.IX Subsection "getTypesToDelete($dbh, $scanTypeList, $keepDefaced)" -.PP -Gets the list of names of the scan types to delete. If \f(CW\*(C`\-type\*(C'\fR was used, then this list is built -using the argument to this option. If \f(CW\*(C`\-defaced\*(C'\fR was used, then the list is fetched using the config -setting \f(CW\*(C`modalities_to_deface\*(C'\fR. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$dbh:\fR database handle. - \- \f(CW$scanTypeList:\fR comma separated string of scan type names. - \- \f(CW$keepDefaced:\fR whether the defaced files should be kept or not. -.PP -\&\s-1RETURNS:\s0 - \- A reference on a hash of the names of the scan types to delete: key => scan type name, + - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. + - $keepDefaced: whether the defaced files should be kept or not. + - $fileBaseName: value of option -basename (or '' if option was not used). + +### getTypesToDelete($dbh, $scanTypeList, $keepDefaced) + +Gets the list of names of the scan types to delete. If `-type` was used, then this list is built +using the argument to this option. If `-defaced` was used, then the list is fetched using the config +setting `modalities_to_deface`. + +INPUTS: + - $dbh: database handle. + - $scanTypeList: comma separated string of scan type names. + - $keepDefaced: whether the defaced files should be kept or not. + +RETURNS: + - A reference on a hash of the names of the scan types to delete: key => scan type name, value => 1 or 0 depending on whether the name is valid or not. -.PP -\fIgetTarchiveSeriesIDs($dbh, \f(CI$filesRef\fI)\fR -.IX Subsection "getTarchiveSeriesIDs($dbh, $filesRef)" -.PP -Gets the list of \f(CW\*(C`TarchiveSeriesID\*(C'\fR to delete in table \f(CW\*(C`tarchive_files\*(C'\fR. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$dbh:\fR database handle. - \- \f(CW$filesRef:\fR reference to the array that contains the file informations for all the files + +### getTarchiveSeriesIDs($dbh, $filesRef) + +Gets the list of `TarchiveSeriesID` to delete in table `tarchive_files`. + +INPUTS: + - $dbh: database handle. + - $filesRef: reference to the array that contains the file informations for all the files that are associated to the upload(s) passed on the command line. -.PP -\&\s-1RETURNS:\s0 - \- A reference on an array containing the \f(CW\*(C`TarchiveSeriesID\*(C'\fR to delete. -.PP -\fIdeleteTableData($dbh, \f(CI$table\fI, \f(CI$key\fI, \f(CI$keyValuesRef\fI, \f(CI$tmpSQLBackupFile\fI, \f(CI$optionsRef\fI)\fR -.IX Subsection "deleteTableData($dbh, $table, $key, $keyValuesRef, $tmpSQLBackupFile, $optionsRef)" -.PP -Deletes records from a database table and adds in a file the \s-1SQL\s0 statements that allow rewriting the -records back in the table. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$dbh:\fR database handle. - \- \f(CW$table:\fR name of the database table. - \- \f(CW$key:\fR name of the key used to delete the records. - \- \f(CW$keyValuesRef:\fR reference on the list of values that field \f(CW$key\fR has for the records to delete. - \- \f(CW$tmpSQLBackupFile:\fR path of the \s-1SQL\s0 file that contains the \s-1SQL\s0 statements used to restore the deleted records. - \- \f(CW$optionsRef:\fR reference to the array that contains the options passed on the command line. -.PP -\&\s-1RETURNS:\s0 - \- The number of records deleted. -.PP -\fIupdateSQLBackupFile($tmpSQLBackupFile, \f(CI$table\fI, \f(CI$key\fI, \f(CI$keyValuesRef\fI, \f(CI$optionsRef\fI)\fR -.IX Subsection "updateSQLBackupFile($tmpSQLBackupFile, $table, $key, $keyValuesRef, $optionsRef)" -.PP -Updates the \s-1SQL\s0 file with the statements to restore the records whose properties are passed as argument. + +RETURNS: + - A reference on an array containing the `TarchiveSeriesID` to delete. + +### deleteTableData($dbh, $table, $key, $keyValuesRef, $tmpSQLBackupFile, $optionsRef) + +Deletes records from a database table and adds in a file the SQL statements that allow rewriting the +records back in the table. + +INPUTS: + - $dbh: database handle. + - $table: name of the database table. + - $key: name of the key used to delete the records. + - $keyValuesRef: reference on the list of values that field `$key` has for the records to delete. + - $tmpSQLBackupFile: path of the SQL file that contains the SQL statements used to restore the deleted records. + - $optionsRef: reference to the array that contains the options passed on the command line. + +RETURNS: + - The number of records deleted. + +### updateSQLBackupFile($tmpSQLBackupFile, $table, $key, $keyValuesRef, $optionsRef) + +Updates the SQL file with the statements to restore the records whose properties are passed as argument. The block of statements is written at the beginning of the file. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$tmpSQLBackupFile:\fR path of the \s-1SQL\s0 file that contains the \s-1SQL\s0 statements used to restore the deleted records. - \- \f(CW$table:\fR name of the database table. - \- \f(CW$key:\fR name of the key used to delete the records. - \- \f(CW$keyValuesRef:\fR reference on the list of values that field \f(CW$key\fR has for the records to delete. - \- \f(CW$optionsRef:\fR reference to the array that contains the options passed on the command line. -.PP -\fIgetInvalidDefacedFiles($dbh, \f(CI$filesRef\fI, \f(CI$scanTypesToDeleteRef\fI)\fR -.IX Subsection "getInvalidDefacedFiles($dbh, $filesRef, $scanTypesToDeleteRef)" -.PP -Checks all the \s-1MINC\s0 files that should have been defaced and makes sure that only the defaced file + +INPUTS: + - $tmpSQLBackupFile: path of the SQL file that contains the SQL statements used to restore the deleted records. + - $table: name of the database table. + - $key: name of the key used to delete the records. + - $keyValuesRef: reference on the list of values that field `$key` has for the records to delete. + - $optionsRef: reference to the array that contains the options passed on the command line. + +### getInvalidDefacedFiles($dbh, $filesRef, $scanTypesToDeleteRef) + +Checks all the MINC files that should have been defaced and makes sure that only the defaced file is associated to it. -.PP -\&\s-1INPUTS:\s0 - \- \f(CW$dbh\fR : database handle. - \- \f(CW$filesRef:\fR reference to the array that contains the information for all the files + +INPUTS: + - $dbh : database handle. + - $filesRef: reference to the array that contains the information for all the files that are associated to the upload(s) passed on the command line. - \- \f(CW$scanTypesToDeleteRef:\fR reference to the array that contains the list of scan type names to delete. -.PP -\&\s-1RETURNS:\s0 - \- The hash of \s-1MINC\s0 files that were either not defaced (and should have been) or that have more than one + - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. + +RETURNS: + - The hash of MINC files that were either not defaced (and should have been) or that have more than one processed file associated to them. Key => file path (relative), Value => reference on an array that contains the - list of processed files associated to the \s-1MINC\s0 file (0, 2, 3 or more entries). -.SH "LICENSING" -.IX Header "LICENSING" + list of processed files associated to the MINC file (0, 2, 3 or more entries). + +# LICENSING + License: GPLv3 -.SH "AUTHORS" -.IX Header "AUTHORS" -\&\s-1LORIS\s0 community and McGill Centre for Integrative + +# AUTHORS + +LORIS community and McGill Centre for Integrative Neuroscience From 84967887e2d151283f14ee4add0aad8111ee0599 Mon Sep 17 00:00:00 2001 From: Nicolas Brossard Date: Thu, 6 Jun 2024 15:17:55 -0400 Subject: [PATCH 4/4] Fixed bug found in Camille's testing --- tools/delete_imaging_upload.pl | 8 ++++++-- uploadNeuroDB/NeuroDB/Database.pm | 18 +----------------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/tools/delete_imaging_upload.pl b/tools/delete_imaging_upload.pl index 55ee0dd13..a3c6aa936 100755 --- a/tools/delete_imaging_upload.pl +++ b/tools/delete_imaging_upload.pl @@ -1557,9 +1557,13 @@ sub setFileExistenceStatus { my %missingFiles; foreach my $t (@PROCESSED_TABLES) { foreach my $f (@{ $filesRef->{$t} }) { - $f->{'Exists'} = undef, next if !$f->{'FullPath'}; + $f->{'Exists'} = undef, next if !$f->{'FullPath'}; - $f->{'Exists'} = -e $f->{'FullPath'}; + # The -f test is to ensure $f->{'FullPath'} is an actual file + # and not a directory. If $f->{'FullPath'} is a directory, it + # means that it has an incorrect/invalid value and should not be backed up + # later on. + $f->{'Exists'} = -f $f->{'FullPath'} && -e $f->{'FullPath'}; # A file is only considered "missing" if it is expected to be on the file # system but is not. There are some file paths refered to in the database that diff --git a/uploadNeuroDB/NeuroDB/Database.pm b/uploadNeuroDB/NeuroDB/Database.pm index a1866d0a1..d4976103e 100644 --- a/uploadNeuroDB/NeuroDB/Database.pm +++ b/uploadNeuroDB/NeuroDB/Database.pm @@ -280,9 +280,7 @@ sub disconnect { my $self = shift; if($self->dbh) { - try { - $self->dbh->disconnect(); - } catch { + if($self->dbh->disconnect()) { NeuroDB::DatabaseException->throw( statement => 'Call to disconnect failed', args => [], @@ -293,20 +291,6 @@ sub disconnect { } } -=pod - -=head3 C - -Object destructor: terminates the connection previously instantiated to the -database (if any). - -=cut -sub DESTROY { - my $self = shift; - - $self->disconnect(); -} - 1; __END__