Features & build cleanups

.github/workflows/rust.yml

@@ -76,6 +76,12 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           path-to-lcov: ./lcov.info
 
+      - name: Test Prebuilt Bindings
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --no-default-features
+
       - name: Test musl build without default features
         env:
            CFLAGS: -I/usr/local/musl/include
@@ -92,5 +98,4 @@ jobs:
         with:
           use-cross: true  
           command: build
-          args: --target x86_64-unknown-linux-musl --all-features
-
+          args: --target x86_64-unknown-linux-musl --all-features

Cargo.toml

@@ -33,10 +33,12 @@ snafu = "0.6.8"
 hts-sys = { version = "^1.10", path = "hts-sys", default-features = false }
 
 [features]
-default = ["bzip2", "lzma", "curl"]
+default = ["bzip2", "lzma", "curl", "bindgen"]
 bzip2 = ["hts-sys/bzip2"]
 lzma = ["hts-sys/lzma"]
+bindgen = ["hts-sys/bindgen"]
 curl = ["hts-sys/curl"]
+libdeflater = ["hts-sys/libdeflater"]
 #openssl = ["hts-sys/openssl"]
 serde = ["serde_base", "serde_bytes"]
 static = []

README.md

@@ -62,6 +62,11 @@ with these compression methods, you can deactivate these features to reduce you
 rust-htslib = { version = "*", default-features = false }
 ```
 
+By default `rust-htslib` uses `bindgen` to generate bindings to htslib. This can slow down the build substantially. Disabling the `bindgen` feature will 
+cause `hts-sys` to use a prebuilt binding.rs file for your architecture.  Note
+Windows is not currently supported & this approach is experimental -- please 
+exercise caution.
+
 `rust-htslib` also has optional support for `serde`, to allow (de)serialization of `bam::Record` via any serde-supported format:
 
 ```toml

hts-sys/Cargo.toml

@@ -18,21 +18,20 @@ tag-message = "Version {{version}} of Rust-HTSlib."
 
 [dependencies]
 libz-sys = { version = "1.0.25", features = ["static"] }
-# https://github.com/alexcrichton/bzip2-rs/issues/56
 bzip2-sys = { version = "0.1.8", optional = true }
 lzma-sys = { version = "0.1.16", optional = true, features = ["static"] }
 curl-sys = { version = "0.4.31", optional = true, features = ["static-curl", "static-ssl"] }
+libdeflater = { git = "https://github.com/pmarks/libdeflater.git", rev = "afb139ee9e9b1d9d1349e5fbf0a54b3079bc5259", optional = true }
 
 [features]
-default = ["bzip2", "lzma"]
+default = ["bzip2", "lzma", "curl", "bindgen"]
 bzip2 = ["bzip2-sys"]
 lzma = ["lzma-sys"]
-#openssl = ["openssl-sys"]
 curl = ["curl-sys"]
 static = []
 
 [build-dependencies]
 fs-utils = "1.1"
-bindgen = { version = "0.53.2", default-features = false, features = ["runtime"] }
-cc = "1.0"
+bindgen = { version = "0.53.2", default-features = false, features = ["runtime"], optional = true }
+cc = { version = "1.0", features = ["parallel"] }
 glob = "0.3.0"

hts-sys/build.rs

@@ -3,34 +3,80 @@
 // This file may not be copied, modified, or distributed
 // except according to those terms.
 
+#[cfg(feature = "serde")]
+use bindgen;
+use cc;
 use fs_utils::copy::copy_directory;
-use glob::glob;
 
 use std::env;
 use std::fs;
+use std::io::Write;
 use std::path::PathBuf;
-use std::process::Command;
-
-fn sed_htslib_makefile(out: &PathBuf, patterns: &[&str], feature: &str) {
-    for pattern in patterns {
-        if !Command::new("sed")
-            .current_dir(out.join("htslib"))
-            .arg("-i")
-            .arg("-e")
-            .arg(pattern)
-            .arg("Makefile")
-            .status()
-            .unwrap()
-            .success()
-        {
-            panic!("failed to strip {} support", feature);
-        }
-    }
-}
+
+// these need to be kept in sync with the htslib Makefile
+const FILES: &[&str] = &[
+    "kfunc.c",
+    "knetfile.c",
+    "kstring.c",
+    "bcf_sr_sort.c",
+    "bgzf.c",
+    "errmod.c",
+    "faidx.c",
+    "header.c",
+    "hfile.c",
+    "hfile_net.c",
+    "hts.c",
+    "hts_os.c",
+    "md5.c",
+    "multipart.c",
+    "probaln.c",
+    "realn.c",
+    "regidx.c",
+    "region.c",
+    "sam.c",
+    "synced_bcf_reader.c",
+    "vcf_sweep.c",
+    "tbx.c",
+    "textutils.c",
+    "thread_pool.c",
+    "vcf.c",
+    "vcfutils.c",
+    "cram/cram_codecs.c",
+    "cram/cram_decode.c",
+    "cram/cram_encode.c",
+    "cram/cram_external.c",
+    "cram/cram_index.c",
+    "cram/cram_io.c",
+    "cram/cram_samtools.c",
+    "cram/cram_stats.c",
+    "cram/mFILE.c",
+    "cram/open_trace_file.c",
+    "cram/pooled_alloc.c",
+    "cram/rANS_static.c",
+    "cram/string_alloc.c",
+];
 
 fn main() {
+
     let out = PathBuf::from(env::var("OUT_DIR").unwrap());
+    if !out.join("htslib").exists() {
+        println!("copying...");
+        copy_directory("htslib", &out).unwrap();
+    }
+
     let mut cfg = cc::Build::new();
+
+    // default files
+    let out_htslib = out.join("htslib");
+    let htslib = PathBuf::from("htslib");
+    for f in FILES {
+        let c_file = out_htslib.join(f);
+        cfg.file(&c_file);
+        println!("cargo:rerun-if-changed={}", htslib.join(f).display());
+    }
+
+    cfg.include(out.join("htslib"));
+
     let want_static = cfg!(feature = "static") || env::var("HTS_STATIC").is_ok();
 
     if want_static {
@@ -43,114 +89,115 @@ fn main() {
         cfg.include(z_inc);
     }
 
-    if !out.join("htslib").exists() {
-        copy_directory("htslib", &out).unwrap();
-    }
-
+
+    // We build a config.h ourselves, rather than rely on Makefile or ./configure
+    let mut config_lines = vec![
+        "/* Default config.h generated by build.rs */",
+        "#define HAVE_DRAND48 1",
+    ];
+
     let use_bzip2 = env::var("CARGO_FEATURE_BZIP2").is_ok();
-    if !use_bzip2 {
-        let bzip2_patterns = vec!["s/ -lbz2//", "/#define HAVE_LIBBZ2/d"];
-        sed_htslib_makefile(&out, &bzip2_patterns, "bzip2");
-    } else if let Ok(inc) = env::var("DEP_BZIP2_ROOT")
+    if use_bzip2 {
+         if let Ok(inc) = env::var("DEP_BZIP2_ROOT")
         .map(PathBuf::from)
         .map(|path| path.join("include"))
-    {
-        cfg.include(inc);
+        {
+            cfg.include(inc);
+            config_lines.push("#define HAVE_LIBBZ2 1");
+        }
+    }
+
+    let use_libdeflater = env::var("CARGO_FEATURE_LIBDEFLATER").is_ok();
+    if use_libdeflater {
+        if let Ok(inc) = env::var("DEP_LIBDEFLATE_INCLUDE").map(PathBuf::from) {
+            cfg.include(inc);
+            config_lines.push("#define HAVE_LIBDEFLATE 1");
+        } else {
+            panic!("no DEP_LIBDEFLATE_INCLUDE");
+        }
     }
 
     let use_lzma = env::var("CARGO_FEATURE_LZMA").is_ok();
-    if !use_lzma && want_static {
-        let lzma_patterns = vec!["s/ -llzma//", "/#define HAVE_LIBLZMA/d"];
-        sed_htslib_makefile(&out, &lzma_patterns, "lzma");
-    } else if let Ok(inc) = env::var("DEP_LZMA_INCLUDE").map(PathBuf::from) {
-        cfg.include(inc);
+    if use_lzma {
+        if let Ok(inc) = env::var("DEP_LZMA_INCLUDE").map(PathBuf::from) {
+            cfg.include(inc);
+            config_lines.push("#define HAVE_LIBBZ2 1");
+            config_lines.push("#ifndef __APPLE__");
+            config_lines.push("#define HAVE_LZMA_H 1");
+            config_lines.push("#endif");
+        }
     }
 
     let use_curl = env::var("CARGO_FEATURE_CURL").is_ok();
-    if !use_curl {
-        let curl_patterns = vec!["s/ -lcurl//", "/#define HAVE_LIBCURL/d"];
-        sed_htslib_makefile(&out, &curl_patterns, "curl");
-    } else if let Ok(inc) = env::var("DEP_CURL_INCLUDE").map(PathBuf::from) {
-        cfg.include(inc);
+    if use_curl {
+        if let Ok(inc) = env::var("DEP_CURL_INCLUDE").map(PathBuf::from) {
+            cfg.include(inc);
+            config_lines.push("#define HAVE_LIBCURL 1");
+            cfg.file("htslib/hfile_libcurl.c");
+            println!("cargo:rerun-if-changed=htslib/hfile_libcurl.c");
+        }
     }
 
-    let tool = cfg.get_compiler();
-    let (cc_path, cflags_env) = (tool.path(), tool.cflags_env());
-    let cc_cflags = cflags_env.to_string_lossy().replace("-O0", "");
-    let host = env::var("HOST").unwrap_or_default();
-    // autoreconf & ./configure (with no args) steps are necessary to include the htslib plugins (hfile_s3.o, hfile_s3_writer.o, etc...)
-    if  // cleanup first
-        // TODO: Have top level "cargo clean" do this instead of in here, see:
-        // https://github.com/rust-lang/cargo/issues/572#issuecomment-632456478
-        !Command::new("make")
-        .current_dir(out.join("htslib"))
-        .arg("clean")
-        .status().unwrap().success()
+    // write out config.h which controls the options htslib will use
+    {
+        let mut f = std::fs::File::create(out.join("htslib").join("config.h")).unwrap();
+        for l in config_lines {
+            writeln!(&mut f, "{}", l).unwrap();
+        };
+    }
 
-        &&
-
-        !Command::new("autoreconf")
-        .current_dir(out.join("htslib"))
-        .env("CFLAGS", &cc_cflags)
-        .status().unwrap().success()
-
-        &&
-
-        !Command::new("./configure")
-        .current_dir(out.join("htslib"))
-        .env("CFLAGS", &cc_cflags)
-        .arg(format!("--host={}", &host))
-        .status().unwrap().success()
+    // write out version.h
     {
-        panic!("could not configure htslib nor any of its plugins")
+        let mut f = std::fs::File::create(out.join("htslib").join("version.h")).unwrap();
+        // FIXME - get version somehow.
+        writeln!(&mut f, "#define HTS_VERSION_TEXT \"1.0\"").unwrap();
     }
 
-    if !Command::new("make")
-        .current_dir(out.join("htslib"))
-        .arg(format!("CC={}", cc_path.display()))
-        .arg(format!("CFLAGS={}", &cc_cflags))
-        .arg("lib-static")
-        .status()
-        .unwrap()
-        .success()
+    cfg.file("wrapper.c");
+    cfg.compile("hts");
+
+    // If bindgen is enabled, use it
+    #[cfg(feature = "bindgen")]
     {
-        panic!("failed to build htslib");
+        bindgen::Builder::default()
+            .header("wrapper.h")
+            .generate_comments(false)
+            .blacklist_function("strtold")
+            .blacklist_type("max_align_t")
+            .generate()
+            .expect("Unable to generate bindings.")
+            .write_to_file(out.join("bindings.rs"))
+            .expect("Could not write bindings.");
     }
 
-    cfg.file("wrapper.c").compile("wrapper");
+    // If no bindgen, use pre-built bindings
+    #[cfg(all(not(feature = "bindgen"), target_os="macos"))]
+    {
+        fs::copy("osx_prebuilt_bindings.rs", out.join("bindings.rs"))
+            .expect("couldn't copy prebuilt bindings");
+        println!("cargo:rerun-if-changed=osx_prebuilt_bindings.rs");
+    }
 
-    bindgen::Builder::default()
-        .header("wrapper.h")
-        .generate_comments(false)
-        .blacklist_function("strtold")
-        .blacklist_type("max_align_t")
-        .generate()
-        .expect("Unable to generate bindings.")
-        .write_to_file(out.join("bindings.rs"))
-        .expect("Could not write bindings.");
+    #[cfg(all(not(feature = "bindgen"), target_os="linux"))]
+    {
+        fs::copy("linux_prebuilt_bindings.rs", out.join("bindings.rs"))
+            .expect("couldn't copy prebuilt bindings");
+        println!("cargo:rerun-if-changed=linux_prebuilt_bindings.rs");
+    }
 
     let include = out.join("include");
     fs::create_dir_all(&include).unwrap();
     if include.join("htslib").exists() {
         fs::remove_dir_all(include.join("htslib")).expect("remove exist include dir");
     }
     copy_directory(out.join("htslib").join("htslib"), &include).unwrap();
-    fs::copy(out.join("htslib").join("libhts.a"), out.join("libhts.a")).unwrap();
 
     println!("cargo:root={}", out.display());
     println!("cargo:include={}", include.display());
     println!("cargo:libdir={}", out.display());
-    println!("cargo:rustc-link-lib=static=hts"); // XXX: only for static, adapt for dynamic?
     println!("cargo:rerun-if-changed=wrapper.c");
     println!("cargo:rerun-if-changed=wrapper.h");
-    println!("cargo:rerun-if-changed=htslib/Makefile");
-    let globs = std::iter::empty()
-        .chain(glob("htslib/*.[ch]").unwrap())
-        .chain(glob("htslib/cram/*.[ch]").unwrap())
-        .chain(glob("htslib/htslib/*.h").unwrap())
-        .chain(glob("htslib/os/*.[ch]").unwrap())
-        .filter_map(Result::ok);
-    for htsfile in globs {
-        println!("cargo:rerun-if-changed={}", htsfile.display());
-    }
-}
+    // Note: config.h is a function of the cargo features. Any feature change will 
+    // cause build.rs to re-run, so don't re-run on that change.
+    //println!("cargo:rerun-if-changed=htslib/config.h");
+}