Add samtools for file format testing.

.github/workflows/gatk-tests.yml

@@ -118,6 +118,9 @@ jobs:
         if: needs.check-secrets.outputs.google-credentials == 'true'
         uses: google-github-actions/setup-gcloud@v2
 
+      - name: 'Install Samtools'
+        run: scripts/install-samtools.sh
+
       - name: pull lfs files
         run: git lfs pull
 
@@ -190,6 +193,9 @@ jobs:
         if: needs.check-secrets.outputs.google-credentials == 'true'
         uses: google-github-actions/setup-gcloud@v2
 
+      - name: 'Install Samtools'
+        run: scripts/install-samtools.sh
+
       - name: build test jars
         run: ./gradlew clean shadowTestClassJar shadowTestJar
 

scripts/install-samtools.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -ex
+#ubuntu specific
+sudo apt-get update
+sudo apt-get upgrade
+sudo apt-get install -y libncurses-dev libbz2-dev liblzma-dev
+
+#install from the github tar
+export SAMTOOLS_VERSION=1.21
+wget https://github.com/samtools/samtools/releases/download/${SAMTOOLS_VERSION}/samtools-${SAMTOOLS_VERSION}.tar.bz2
+tar -xjvf samtools-${SAMTOOLS_VERSION}.tar.bz2
+cd samtools-${SAMTOOLS_VERSION} && ./configure --prefix=/usr && make && sudo make install

src/test/java/org/broadinstitute/hellbender/testutils/SamtoolsTestUtilsTest.java

@@ -0,0 +1,74 @@
+package org.broadinstitute.hellbender.testutils;
+
+import htsjdk.io.HtsPath;
+import htsjdk.io.IOPath;
+import htsjdk.samtools.SAMRecord;
+import htsjdk.samtools.SamReader;
+import htsjdk.samtools.SamReaderFactory;
+import htsjdk.samtools.ValidationStringency;
+import htsjdk.samtools.util.CloseableIterator;
+import htsjdk.samtools.util.ProcessExecutor;
+import org.broadinstitute.hellbender.GATKBaseTest;
+import org.testng.Assert;
+import org.testng.SkipException;
+import org.testng.annotations.Test;
+
+import java.io.File;
+import java.io.IOException;
+
+public class SamtoolsTestUtilsTest extends GATKBaseTest {
+    private static final File TEST_DATA_DIR = new File("src/test/resources/org/broadinstitute/hellbender/tools/");
+
+    @Test
+    public void testSamtoolsIsAvailable() {
+        Assert.assertTrue(SamtoolsTestUtils.isSamtoolsAvailable());
+    }
+
+    @Test
+    public void testSamtoolsVersion() {
+        if (!SamtoolsTestUtils.isSamtoolsAvailable()) {
+            throw new SkipException("Samtools not available on local device");
+        }
+        // If this test runs, but fails because version validation fails, then the local samtools version is
+        // not the one expected by the htsjdk tests
+        final ProcessExecutor.ExitStatusAndOutput processStatus = SamtoolsTestUtils.executeSamToolsCommand("--version");
+        Assert.assertTrue(processStatus.stdout.contains(SamtoolsTestUtils.expectedSamtoolsVersion));
+    }
+
+    @Test(expectedExceptions = RuntimeException.class)
+    public void testSamtoolsPresentButCommandFails() {
+        if (!SamtoolsTestUtils.isSamtoolsAvailable()) {
+            throw new SkipException("Samtools not available on local device");
+        }
+        SamtoolsTestUtils.executeSamToolsCommand("--notASamtoolsCommand");
+    }
+
+    @Test
+    public void testCRAMConversion()throws IOException {
+        if (!SamtoolsTestUtils.isSamtoolsAvailable()) {
+            throw new SkipException("Samtools not available on local device");
+        }
+
+        // Validates CRAM 3.1 conversion.
+        final File sourceFile = new File(TEST_DATA_DIR, "print_reads.cram");
+        final File cramReference = new File(TEST_DATA_DIR, "print_reads.fasta");
+        // This also validates that any extra command line arguments are passed through to samtools by requesting
+        // that NM/MD values are synthesized in the output file (which is required for the output records to match).
+        final IOPath tempSamtoolsPath = SamtoolsTestUtils.convertToCRAM(
+                new HtsPath(sourceFile.getAbsolutePath()),
+                new HtsPath(cramReference.getAbsolutePath()),
+                "--output-fmt cram,version=3.0,fast");
+        final SamReaderFactory factory = SamReaderFactory.makeDefault()
+                .validationStringency(ValidationStringency.LENIENT)
+                .referenceSequence(cramReference);
+        try (final SamReader originalReader = factory.open(sourceFile);
+             final SamReader samtoolsCopyReader = factory.open(tempSamtoolsPath.toPath());
+             final CloseableIterator<SAMRecord> originalIt = originalReader.iterator();
+             final CloseableIterator<SAMRecord> samtoolsIt = samtoolsCopyReader.iterator()) {
+            while (originalIt.hasNext() && samtoolsIt.hasNext()) {
+                Assert.assertEquals(originalIt.next(), samtoolsIt.next());
+            }
+            Assert.assertEquals(samtoolsIt.hasNext(), originalIt.hasNext());
+        }
+    }
+}

src/testUtils/java/org/broadinstitute/hellbender/testutils/SamtoolsTestUtils.java

@@ -0,0 +1,115 @@
+package org.broadinstitute.hellbender.testutils;
+
+import htsjdk.beta.plugin.IOUtils;
+import htsjdk.io.IOPath;
+import htsjdk.samtools.util.FileExtensions;
+import htsjdk.samtools.util.ProcessExecutor;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+/**
+ * Test utilities for running samtools from GATK tests.
+ */
+public class SamtoolsTestUtils {
+    private static final String SAMTOOLS_BINARY_ENV_VARIABLE = "GATK_SAMTOOLS_BIN";
+    public final static String expectedSamtoolsVersion = "1.21";
+
+    /**
+     * @return true if samtools is available, otherwise false
+     */
+    public static boolean isSamtoolsAvailable() {
+        final String binPath = getSamtoolsBin();
+        final Path binFile = Paths.get(binPath);
+        return Files.exists(binFile);
+    }
+
+    /**
+     * @return true if a local samtools executable is available, otherwise throws a runtimeException
+     */
+    public static void assertSamtoolsAvailable() {
+        if (!isSamtoolsAvailable()) {
+            throw new RuntimeException(
+                    String.format(
+                            "No samtools executable can be found." +
+                                    " The %s environment variable must be set to the name of the local samtools executable.",
+                            SAMTOOLS_BINARY_ENV_VARIABLE));
+        }
+    }
+
+    /**
+     * @return the name and location of the local samtools executable as specified by the environment
+     * variable SAMTOOLS_BINARY_ENV_VARIABLE, or the default value of "/usr/bin/samtools" if the environment
+     * variable is not set
+     */
+    public static String getSamtoolsBin() {
+        final String samtoolsPath = System.getenv(SAMTOOLS_BINARY_ENV_VARIABLE);
+        return samtoolsPath == null ? "/usr/bin/samtools" : samtoolsPath;
+    }
+
+    /**
+     * Execute a samtools command line if a local samtools executable is available see {@link #isSamtoolsAvailable()}.
+     *
+     * @param commandLine samtools command line string, excluding the "samtools" prefix. For example:
+     *                    {@code "view -h -b my.sam -o my.bam"}
+     * @return the {@link ProcessExecutor.ExitStatusAndOutput} resulting from the command execution, if
+     * the command succeeds
+     * @throws RuntimeException if the command fails, or if a local samtools executable is not available.
+     */
+    public static ProcessExecutor.ExitStatusAndOutput executeSamToolsCommand(final String commandLine) {
+        assertSamtoolsAvailable();
+        final String commandString = String.format("%s %s", getSamtoolsBin(), commandLine);
+        final ProcessExecutor.ExitStatusAndOutput processStatus =
+                ProcessExecutor.executeAndReturnInterleavedOutput(commandString);
+        if (processStatus.exitStatus != 0) {
+            // samtools seems to write some errors to stdout
+            throw new RuntimeException(
+                    String.format("Failure code %d returned from samtools command %s\n (stderr: %.500s)\n (stdout: %.500s)\n",
+                            processStatus.exitStatus,
+                            commandString,
+                            processStatus.stderr == null ? "" : processStatus.stderr,
+                            processStatus.stdout == null ? "" : processStatus.stdout));
+        }
+        return processStatus;
+    }
+
+    /**
+     * Convert an input sam/bam/cram file to a temporary CRAM file using the samtools "view" command. The temp
+     * file will be deleted when the process exits. Use {@link #isSamtoolsAvailable()} to determine if its safe
+     * to use this method.
+     *
+     * @param inputSAMBAMCRAMFile input file to convert
+     * @param referenceFile a valid reference file
+     * @param commandLineOptions additional command line options (--input-fmt-option or --output-fmt-option)
+     * @return a temporary file containing the samtools-generated results.
+     */
+    public static final IOPath convertToCRAM(
+            final IOPath inputSAMBAMCRAMFile,
+            final IOPath referenceFile,
+            final String commandLineOptions) {
+        assertSamtoolsAvailable();
+        final IOPath tempCRAMPath = IOUtils.createTempPath("samtoolsTemporaryCRAM", FileExtensions.CRAM);
+        tempCRAMPath.toPath().toFile().deleteOnExit();
+        final String commandString = String.format("view -h -C -T %s %s %s -o %s",
+                referenceFile.toPath().toAbsolutePath(),
+                commandLineOptions == null ? "" : commandLineOptions,
+                inputSAMBAMCRAMFile.toPath().toAbsolutePath(),
+                tempCRAMPath.toPath().toAbsolutePath());
+        executeSamToolsCommand(commandString);
+        return tempCRAMPath;
+}
+
+    public static final void convertToCRAM(
+            final IOPath inputSAMBAMCRAMPath,
+            final IOPath outputPath,
+            final IOPath referencePath,
+            final String commandLineOptions) {
+        assertSamtoolsAvailable();
+        final String commandString = String.format("view -h -C -T %s %s %s -o %s",
+                referencePath.toPath().toAbsolutePath(),
+                commandLineOptions == null ? "" : commandLineOptions,
+                inputSAMBAMCRAMPath.toPath().toAbsolutePath(),
+                outputPath.toPath().toAbsolutePath());
+        executeSamToolsCommand(commandString);
+    }
+}