diff --git a/.cargo/config b/.cargo/config index f04323123..7d7cf0727 100644 --- a/.cargo/config +++ b/.cargo/config @@ -1,12 +1,2 @@ -[target.x86_64-apple-darwin] -rustflags = [ - "-C", "link-arg=-undefined", - "-C", "link-arg=dynamic_lookup", -] - [target.x86_64-unknown-linux-musl] -rustflags = [ - "-C", "link-arg=-undefined", - "-C", "link-arg=dynamic_lookup", -] -linker = "x86_64-linux-musl-gcc" \ No newline at end of file +linker = "x86_64-linux-musl-gcc" diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e20c7a81c..3d98254e8 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -76,16 +76,21 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} path-to-lcov: ./lcov.info -# - name: Test musl build without default features -# uses: actions-rs/cargo@v1 -# with: -# use-cross: true -# command: build -# args: --target x86_64-unknown-linux-musl --no-default-features -# -# - name: Test musl build with all features -# uses: actions-rs/cargo@v1 -# with: -# use-cross: true -# command: build -# args: --target x86_64-unknown-linux-musl --all-features + - name: Test musl build without default features + env: + CFLAGS: -I/usr/local/musl/include + uses: actions-rs/cargo@v1 + with: + use-cross: true + command: build + args: --target x86_64-unknown-linux-musl --no-default-features + + - name: Test musl build with all features + env: + CFLAGS: -I/usr/local/musl/include + uses: actions-rs/cargo@v1 + with: + use-cross: true + command: build + args: --target x86_64-unknown-linux-musl --all-features + diff --git a/Cargo.toml b/Cargo.toml index a265ce52f..c6e0d0677 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ tag-message = "Version {{version}} of Rust-HTSlib." [dependencies] libc = "0.2" -itertools = "0.8" +itertools = "0.9.0" newtype_derive = "0.1" custom_derive = "0.1" url = "2.1" @@ -29,7 +29,7 @@ linear-map = "1.2" serde_base = { version = "^1", optional = true, package = "serde" } serde_bytes = { version = "0.11", optional = true } bio-types = ">=0.6" -snafu = ">= 0.5.0, <= 0.6.0" +snafu = "0.6.8" hts-sys = { version = "^1.10", path = "hts-sys", default-features = false } [features] @@ -37,8 +37,9 @@ default = ["bzip2", "lzma", "curl"] bzip2 = ["hts-sys/bzip2"] lzma = ["hts-sys/lzma"] curl = ["hts-sys/curl"] -openssl = ["hts-sys/openssl"] +#openssl = ["hts-sys/openssl"] serde = ["serde_base", "serde_bytes"] +static = [] [dev-dependencies] tempdir = "0.3" diff --git a/Cross.toml b/Cross.toml index c753b089b..f867314c8 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,4 +1,12 @@ +[build.env] +passthrough = [ + "RUST_DEBUG", + "RUST_BACKTRACE", + "CFLAGS" +] + + [target.x86_64-unknown-linux-musl] -image = "brainstorm/rust_musl_docker:stable-latest-libcurl" +image = "brainstorm/cross-x86_64-unknown-linux-musl:latest" [target.x86_64-unknown-linux-gnu] image = "brainstorm/cross-x86_64-unknown-linux-gnu:libcurl-openssl" diff --git a/README.md b/README.md index 1d6237fec..e2b5b6a5a 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ This library provides HTSlib bindings and a high level Rust API for reading and To clone this repository, issue -``` -git clone --recursive https://github.com/rust-bio/rust-htslib.git +```shell +$ git clone --recursive https://github.com/rust-bio/rust-htslib.git ``` ensuring that the HTSlib submodule is fetched, too. @@ -19,7 +19,21 @@ If you only want to use the library, there is no need to clone the repository. G ## Requirements -To compile this crate you need the development headers of zlib, bzip2 and xz. For instance, in Debian systems one needs the following dependencies: +To compile this crate you need docker and cross: + +```shell +$ cargo install cross +$ cross build # will build with GNU toolchain +``` + +If you want to run rust-htslib code on AWS lambda, you'll need to statically compile it with MUSL as follows: + +```shell +$ export CFLAGS="-I/usr/local/musl/include" +$ cross build --target x86_64-unknown-linux-musl # will build with MUSL toolchain +``` + +Alternatively, you can also install it locally by installing the development headers of zlib, bzip2 and xz. For instance, in Debian systems one needs the following dependencies: ```shell $ sudo apt-get install zlib1g-dev libbz2-dev liblzma-dev clang pkg-config diff --git a/docker/Dockerfile.gnu b/docker/Dockerfile.gnu index 2ec9fafe8..e6b78719e 100644 --- a/docker/Dockerfile.gnu +++ b/docker/Dockerfile.gnu @@ -1,8 +1,9 @@ FROM rustembedded/cross:x86_64-unknown-linux-gnu -ENV LIBCLANG_PATH /usr/lib/x86_64-linux-gnu +#ENV LIBCLANG_PATH /usr/lib/x86_64-linux-musl +ENV LIBCLANG_PATH /usr/lib/llvm-10/lib ENV LLVM_CONFIG_PATH /usr/bin -RUN apt-get update && \ - apt-get install -y libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev clang-8 && \ - ln -sf /usr/bin/llvm-config-8 /usr/bin/llvm-config && \ - ln -sf /usr/lib/x86_64-linux-gnu/libclang-8.so.1 /usr/lib/x86_64-linux-gnu/libclang.so.1 +RUN apt-get update +RUN apt-get install -y wget gnupg lsb-release software-properties-common apt-transport-https ca-certificates # Otherwise LLVM bump below fails +RUN bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" +RUN apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev # htslib deps \ No newline at end of file diff --git a/docker/Dockerfile.musl b/docker/Dockerfile.musl index 55b228447..92880ec0e 100644 --- a/docker/Dockerfile.musl +++ b/docker/Dockerfile.musl @@ -1,7 +1,121 @@ FROM rustembedded/cross:x86_64-unknown-linux-musl +ENV MUSL_CROSS_MAKE_VERSION 0.9.9 ENV PKG_CONFIG_ALLOW_CROSS 1 -ENV OPENSSL_LIB_DIR /usr/lib/x86_64-linux-gnu -ENV OPENSSL_INCLUDE_DIR /usr/include/openssl +ENV LZMA_VERSION 5.2.5 +ENV ZLIB_VERSION 1.2.11 +ENV OPENSSL_VERSION 1_1_1g +ENV CURL_VERSION 7.70.0 + +ENV LIBCLANG_PATH /usr/lib/llvm-10/lib +ENV LLVM_CONFIG_PATH /usr/bin + + +# The default includes and packages from the Ubuntu distro that cross uses will generate all sorts of linux-headers related include errors, see: +# https://github.com/rust-bio/rust-htslib/pull/184#commitcomment-37496651 +# Those are the packages installed, hopefully someone will find a good way to use the distro ones instead of compiling everything under /usr/local :/ +# apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev musl musl-dev musl-tools linux-libc-dev linux-headers-4.15.0-20-generic + +# Install basics to locally compile htslib dependencies RUN apt-get update && \ - apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev musl musl-dev musl-tools linux-libc-dev linux-headers-4.15.0-20-generic + apt-get install -y build-essential autoconf automake autotools-dev git wget + +# Otherwise LLVM bump below fails +RUN apt-get install -y wget gnupg lsb-release software-properties-common apt-transport-https ca-certificates + +# Autodetect and fetch latest LLVM repos for the current distro, avoids LLVM warnings and other issues, might generate slower builds for now though, see: +# https://www.phoronix.com/scan.php?page=news_item&px=Rust-Hurt-On-LLVM-10 +RUN bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" +#RUN apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev musl musl-dev musl-tools # htslib deps + +# Remove pre-installed musl, to avoid cross-musl-make interference +RUN apt-get remove -y musl + +# For now we'll have to go nuts and not only build musl-1.2.0 from scratch but all the other libs too... +# Updated musl-cross toolchain that does not fail on OpenSSL: https://github.com/openssl/openssl/issues/7207 +WORKDIR /root +RUN wget https://github.com/richfelker/musl-cross-make/archive/v$MUSL_CROSS_MAKE_VERSION.tar.gz \ + && tar xvfz v$MUSL_CROSS_MAKE_VERSION.tar.gz +WORKDIR /root/musl-cross-make-$MUSL_CROSS_MAKE_VERSION +COPY config-musl-cross-make.mak config.mak +RUN make -j40 install + +# Now we assume we have a properly configured musl-cross... +ENV PATH "/usr/local/musl/bin:$PATH" +ENV CFLAGS "-fPIC" +ENV CROSS_COMPILE x86_64-linux-musl- +ENV CC ${CROSS_COMPILE}cc +ENV AR ${CROSS_COMPILE}ar +ENV RANLIB ${CROSS_COMPILE}ranlib +ENV CXX ${CROSS_COMPILE}g++ +ENV CPPFLAGS "-I/usr/local/musl/include -I/usr/local/include" +ENV LDFLAGS "-L/usr/local/musl/lib -L/usr/local/lib" + +# .. and carry on with the htslib deps +RUN wget https://www.zlib.net/zlib-$ZLIB_VERSION.tar.gz \ + && tar xvfz zlib-$ZLIB_VERSION.tar.gz \ + && cd zlib-$ZLIB_VERSION \ + && ./configure --static --prefix=/usr/local/musl \ + && make -j40 install +WORKDIR /root +RUN git clone git://sourceware.org/git/bzip2 \ + && cd bzip2 \ + #&& make -j40 CC=$CC AR=$AR RANLIB=$RANLIB CFLAGS=$CFLAGS bzip2 \ + && make PREFIX=/usr/local/musl -j40 bzip2 \ + && make -j40 install +WORKDIR /root +RUN wget https://tukaani.org/xz/xz-$LZMA_VERSION.tar.bz2 \ + && tar xvfj xz-$LZMA_VERSION.tar.bz2 \ + && cd xz-$LZMA_VERSION \ + && ./configure --prefix=/usr/local/musl --enable-static --disable-shared --host x86_64-unknown-linux-musl \ + && make -j40 install +WORKDIR /root + +# A few gems from: https://wiki.openssl.org/index.php/Compilation_and_Installation +# "OpenSSL has been around a long time, and it carries around a lot of cruft" +# "SSLv2 is completely broken, and you should disable it during configuration" +# "You should specify both --prefix and --openssldir to ensure make install works as expected." +# +# And also this: +# https://github.com/openssl/openssl/issues/11362 +# +# And having to redefine AR,CC & RANLIB because otherwise, this: +# +# make[1]: x86_64-linux-musl-x86_64-linux-musl-ar: Command not found +RUN wget https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION.tar.gz \ + && tar xvfz OpenSSL_$OPENSSL_VERSION.tar.gz && cd openssl-OpenSSL_$OPENSSL_VERSION \ + && ./Configure --prefix=/usr/local/musl --openssldir=/usr/local/musl \ + --with-zlib-lib=/usr/local/lib/zlib-$ZLIB_VERSION linux-x86_64 \ + no-shared no-dso no-gost no-engine no-ssl2 no-srp no-srtp no-tests zlib \ + no-weak-ssl-ciphers \ + && make AR=x86_64-linux-musl-ar \ + CC=x86_64-linux-musl-cc \ + -j40 \ + && make RANLIB=x86_64-linux-musl-ranlib -j40 install + +WORKDIR /root +RUN wget https://curl.haxx.se/download/curl-$CURL_VERSION.tar.gz \ + && tar xvfz curl-$CURL_VERSION.tar.gz && cd curl-$CURL_VERSION \ + && ./configure --prefix=/usr/local/musl --host x86_64-linux-musl \ + --with-ssl=/usr/local/musl --with-zlib=/usr/local/musl --enable-static \ + --disable-dict --disable-file --disable-ftp --disable-gopher --disable-imap \ + --disable-pop3 --disable-rtsp --disable-smb --disable-smtp --disable-telnet \ + --disable-tftp --disable-ntlm --disable-ldap \ + && make -j40 install + +# To cater Rust's openssl-sys needs... +#ENV OPENSSL_DIR /usr/local/openssl +ENV OPENSSL_DIR /usr/local/musl + +# Hack to force ld stick to musl on the hts-sys/build.rs side, otherwise: +# = note: /usr/bin/ld: /target/debug/deps/liblibloading-689161fea10b6234.rlib(global_static.o): unable to initialize decompress status for section .debug_info +RUN rm /usr/bin/ld && ln -sf /usr/local/musl/bin/x86_64-linux-musl-ld /usr/bin/ld + +# Prepare rustup and toolchain locally for easy manual intervention in this container +#RUN wget https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init \ +# && chmod +x rustup-init \ +# && ./rustup-init -y \ +# && . $HOME/.cargo/env \ +# && rustup target add x86_64-unknown-linux-musl + +CMD ["bash"] \ No newline at end of file diff --git a/docker/README.md b/docker/README.md index 349b54d58..57612c885 100644 --- a/docker/README.md +++ b/docker/README.md @@ -10,7 +10,7 @@ $ docker build -t brainstorm/cross-x86_64-unknown-linux-musl:libcurl-openssl . - $ docker build -t brainstorm/cross-x86_64-unknown-linux-gnu:libcurl-openssl . -f Dockerfile.gnu ``` -Then to build and test rust-htslib with the above containers, proceed as you would with `cargo`, using `cross` instead, i.e: +Then, to build and test rust-htslib with the above containers, proceed as you would with `cargo`, using `cross` instead, i.e: ```shell $ cross build --target x86_64-unknown-linux-musl diff --git a/docker/config-musl-cross-make.mak b/docker/config-musl-cross-make.mak new file mode 100644 index 000000000..2c3cc25ef --- /dev/null +++ b/docker/config-musl-cross-make.mak @@ -0,0 +1,4 @@ +TARGET = x86_64-linux-musl +OUTPUT = /usr/local/musl +GCC_VER = 9.2.0 +BINUTILS_VER = 2.33.1 \ No newline at end of file diff --git a/hts-sys/Cargo.toml b/hts-sys/Cargo.toml index 73fa34262..296a916df 100644 --- a/hts-sys/Cargo.toml +++ b/hts-sys/Cargo.toml @@ -17,22 +17,22 @@ pre-release-commit-message = "release version {{version}}" tag-message = "Version {{version}} of Rust-HTSlib." [dependencies] -libc = "0.2" -libz-sys = "1.0" -bzip2-sys = { version = "0.1", optional = true } -lzma-sys = { version = "0.1", optional = true } -curl-sys = { version = "0.4.26", optional = true } -openssl-sys = { version = "0.9.54", optional = true } +libz-sys = { version = "1.0.25", features = ["static"] } +# https://github.com/alexcrichton/bzip2-rs/issues/56 +bzip2-sys = { version = "0.1.8", optional = true } +lzma-sys = { version = "0.1.16", optional = true, features = ["static"] } +curl-sys = { version = "0.4.31", optional = true, features = ["static-curl", "static-ssl"] } [features] -default = ["bzip2", "lzma", "curl"] +default = ["bzip2", "lzma"] bzip2 = ["bzip2-sys"] lzma = ["lzma-sys"] -openssl = ["openssl-sys"] +#openssl = ["openssl-sys"] curl = ["curl-sys"] +static = [] [build-dependencies] fs-utils = "1.1" -bindgen = { version = "0.53.1", default-features = false, features = ["runtime"] } +bindgen = { version = "0.53.2", default-features = false, features = ["runtime"] } cc = "1.0" -glob = "0.3.0" +glob = "0.3.0" \ No newline at end of file diff --git a/hts-sys/build.rs b/hts-sys/build.rs index dfb2c70ed..66f745c6c 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -3,8 +3,6 @@ // This file may not be copied, modified, or distributed // except according to those terms. -use bindgen; -use cc; use fs_utils::copy::copy_directory; use glob::glob; @@ -33,7 +31,13 @@ fn sed_htslib_makefile(out: &PathBuf, patterns: &[&str], feature: &str) { fn main() { let out = PathBuf::from(env::var("OUT_DIR").unwrap()); let mut cfg = cc::Build::new(); - cfg.warnings(false).static_flag(true).pic(true); + let want_static = cfg!(feature = "static") || env::var("HTS_STATIC").is_ok(); + + if want_static { + cfg.warnings(true).static_flag(true).pic(true); + } else { + cfg.warnings(true).static_flag(false).pic(true); + } if let Ok(z_inc) = env::var("DEP_Z_INCLUDE") { cfg.include(z_inc); @@ -42,7 +46,7 @@ fn main() { if !out.join("htslib").exists() { copy_directory("htslib", &out).unwrap(); } - + let use_bzip2 = env::var("CARGO_FEATURE_BZIP2").is_ok(); if !use_bzip2 { let bzip2_patterns = vec!["s/ -lbz2//", "/#define HAVE_LIBBZ2/d"]; @@ -55,7 +59,7 @@ fn main() { } let use_lzma = env::var("CARGO_FEATURE_LZMA").is_ok(); - if !use_lzma { + if !use_lzma && want_static { let lzma_patterns = vec!["s/ -llzma//", "/#define HAVE_LIBLZMA/d"]; sed_htslib_makefile(&out, &lzma_patterns, "lzma"); } else if let Ok(inc) = env::var("DEP_LZMA_INCLUDE").map(PathBuf::from) { @@ -73,12 +77,39 @@ fn main() { let tool = cfg.get_compiler(); let (cc_path, cflags_env) = (tool.path(), tool.cflags_env()); let cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); + let host = env::var("HOST").unwrap_or_default(); + // autoreconf & ./configure (with no args) steps are necessary to include the htslib plugins (hfile_s3.o, hfile_s3_writer.o, etc...) + if // cleanup first + // TODO: Have top level "cargo clean" do this instead of in here, see: + // https://github.com/rust-lang/cargo/issues/572#issuecomment-632456478 + !Command::new("make") + .current_dir(out.join("htslib")) + .arg("clean") + .status().unwrap().success() + + && + + !Command::new("autoreconf") + .current_dir(out.join("htslib")) + .env("CFLAGS", &cc_cflags) + .status().unwrap().success() + + && + + !Command::new("./configure") + .current_dir(out.join("htslib")) + .env("CFLAGS", &cc_cflags) + .arg(format!("--host={}", &host)) + .status().unwrap().success() + { + panic!("could not configure htslib nor any of its plugins") + } + if !Command::new("make") .current_dir(out.join("htslib")) .arg(format!("CC={}", cc_path.display())) - .arg(format!("CFLAGS={}", cc_cflags)) + .arg(format!("CFLAGS={}", &cc_cflags)) .arg("lib-static") - .arg("-B") .status() .unwrap() .success() @@ -109,7 +140,7 @@ fn main() { println!("cargo:root={}", out.display()); println!("cargo:include={}", include.display()); println!("cargo:libdir={}", out.display()); - println!("cargo:rustc-link-lib=static=hts"); + println!("cargo:rustc-link-lib=static=hts"); // XXX: only for static, adapt for dynamic? println!("cargo:rerun-if-changed=wrapper.c"); println!("cargo:rerun-if-changed=wrapper.h"); println!("cargo:rerun-if-changed=htslib/Makefile"); diff --git a/hts-sys/src/lib.rs b/hts-sys/src/lib.rs index de6108884..c0774ffe3 100644 --- a/hts-sys/src/lib.rs +++ b/hts-sys/src/lib.rs @@ -5,8 +5,8 @@ #![allow(improper_ctypes)] //! This module exposes the raw Htslib bindings. +#[cfg(feature = "libz")] extern crate libz_sys; - #[cfg(feature = "bzip2")] extern crate bzip2_sys; #[cfg(feature = "lzma")] diff --git a/src/bam/mod.rs b/src/bam/mod.rs index 1c7feefab..fa67b899f 100644 --- a/src/bam/mod.rs +++ b/src/bam/mod.rs @@ -32,6 +32,8 @@ pub use crate::bam::header::Header; pub use crate::bam::record::Record; use std::convert::TryInto; +/// # Safety +/// /// Implementation for `Read::set_threads` and `Writer::set_threads`. pub unsafe fn set_threads(htsfile: *mut htslib::htsFile, n_threads: usize) -> Result<()> { assert!(n_threads != 0, "n_threads must be > 0"); @@ -43,6 +45,8 @@ pub unsafe fn set_threads(htsfile: *mut htslib::htsFile, n_threads: usize) -> Re } } +/// # Safety +/// /// Set the reference FAI index path in a `htslib::htsFile` struct for reading CRAM format. pub unsafe fn set_fai_filename>( htsfile: *mut htslib::htsFile, diff --git a/src/bcf/mod.rs b/src/bcf/mod.rs index 0cefb8965..c00c373a1 100644 --- a/src/bcf/mod.rs +++ b/src/bcf/mod.rs @@ -73,7 +73,8 @@ pub struct Reader { } unsafe impl Send for Reader {} - +/// # Safety +/// /// Implementation for `Reader::set_threads()` and `Writer::set_threads`. pub unsafe fn set_threads(hts_file: *mut htslib::htsFile, n_threads: usize) -> Result<()> { assert!(n_threads > 0, "n_threads must be > 0"); diff --git a/src/tbx/mod.rs b/src/tbx/mod.rs index ee2792b51..0d1efcf50 100644 --- a/src/tbx/mod.rs +++ b/src/tbx/mod.rs @@ -41,7 +41,6 @@ //! } //! ``` -use libc; use std::ffi; use std::path::Path; use std::ptr;