From 938d0d47b6c9d9ad4125fa1d79ea7ce180249a26 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Wed, 3 Jun 2020 14:22:54 +1000 Subject: [PATCH 1/5] That is no longer the case, I encouraged @FiloSottile to bottle it up via a fork in https://github.com/FiloSottile/homebrew-musl-cross/issues/13#issuecomment-602173990, so now it's fast to install (just a bottle pour) :) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 578d921f1..a4b17b33d 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ To compile this crate you need the development headers of zlib, bzip2 and xz. Fo $ sudo apt-get install zlib1g-dev libbz2-dev liblzma-dev clang pkg-config ``` -On OSX, this will take a significant amount of time due to musl cross compiling toolchain: +On OSX: ```shell $ brew install FiloSottile/musl-cross/musl-cross From cd0fa8d2007bc395a5b385b5a25f98bdaeea8149 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Fri, 5 Jun 2020 15:24:53 +1000 Subject: [PATCH 2/5] Update both GNU and MUSL containers, add libdeflate to both --- Cross.toml | 4 ++-- docker/Dockerfile.gnu | 8 +++++--- docker/Dockerfile.musl | 44 +++++++++++++++++++++++++----------------- docker/README.md | 4 ++-- 4 files changed, 35 insertions(+), 25 deletions(-) diff --git a/Cross.toml b/Cross.toml index f867314c8..3d7e29edf 100644 --- a/Cross.toml +++ b/Cross.toml @@ -7,6 +7,6 @@ passthrough = [ [target.x86_64-unknown-linux-musl] -image = "brainstorm/cross-x86_64-unknown-linux-musl:latest" +image = "brainstorm/cross-x86_64-unknown-linux-musl:1.0.0" [target.x86_64-unknown-linux-gnu] -image = "brainstorm/cross-x86_64-unknown-linux-gnu:libcurl-openssl" +image = "brainstorm/cross-x86_64-unknown-linux-gnu:1.0.0" diff --git a/docker/Dockerfile.gnu b/docker/Dockerfile.gnu index e6b78719e..081a1ddf8 100644 --- a/docker/Dockerfile.gnu +++ b/docker/Dockerfile.gnu @@ -1,9 +1,11 @@ FROM rustembedded/cross:x86_64-unknown-linux-gnu -#ENV LIBCLANG_PATH /usr/lib/x86_64-linux-musl ENV LIBCLANG_PATH /usr/lib/llvm-10/lib ENV LLVM_CONFIG_PATH /usr/bin RUN apt-get update -RUN apt-get install -y wget gnupg lsb-release software-properties-common apt-transport-https ca-certificates # Otherwise LLVM bump below fails +RUN apt-get install -y build-essential wget gnupg lsb-release software-properties-common apt-transport-https ca-certificates # Otherwise LLVM bump below fails RUN bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" -RUN apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev # htslib deps \ No newline at end of file +RUN apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev # htslib deps +RUN git clone --depth 1 https://github.com/ebiggers/libdeflate.git && \ + cd libdeflate && make -j40 CFLAGS="-fPIC -O3" install && \ + cd .. && rm -rf libdeflate diff --git a/docker/Dockerfile.musl b/docker/Dockerfile.musl index 92880ec0e..b6cd2776c 100644 --- a/docker/Dockerfile.musl +++ b/docker/Dockerfile.musl @@ -26,7 +26,8 @@ RUN apt-get install -y wget gnupg lsb-release software-properties-common apt-tra # Autodetect and fetch latest LLVM repos for the current distro, avoids LLVM warnings and other issues, might generate slower builds for now though, see: # https://www.phoronix.com/scan.php?page=news_item&px=Rust-Hurt-On-LLVM-10 RUN bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" -#RUN apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev musl musl-dev musl-tools # htslib deps +#RUN apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev \ +# libbz2-dev liblzma-dev musl musl-dev musl-tools # htslib deps # Remove pre-installed musl, to avoid cross-musl-make interference RUN apt-get remove -y musl @@ -38,7 +39,8 @@ RUN wget https://github.com/richfelker/musl-cross-make/archive/v$MUSL_CROSS_MAKE && tar xvfz v$MUSL_CROSS_MAKE_VERSION.tar.gz WORKDIR /root/musl-cross-make-$MUSL_CROSS_MAKE_VERSION COPY config-musl-cross-make.mak config.mak -RUN make -j40 install +RUN make -j40 install && cd .. && rm v$MUSL_CROSS_MAKE_VERSION.tar.gz + #&& rm -rf musl-cross-make-* # Now we assume we have a properly configured musl-cross... ENV PATH "/usr/local/musl/bin:$PATH" @@ -52,24 +54,30 @@ ENV CPPFLAGS "-I/usr/local/musl/include -I/usr/local/include" ENV LDFLAGS "-L/usr/local/musl/lib -L/usr/local/lib" # .. and carry on with the htslib deps -RUN wget https://www.zlib.net/zlib-$ZLIB_VERSION.tar.gz \ - && tar xvfz zlib-$ZLIB_VERSION.tar.gz \ - && cd zlib-$ZLIB_VERSION \ - && ./configure --static --prefix=/usr/local/musl \ - && make -j40 install WORKDIR /root +RUN git clone --depth 1 https://github.com/ebiggers/libdeflate.git && \ + cd libdeflate && make -j40 CFLAGS="-fPIC -O3" PREFIX="/usr/local/musl" install && \ + cd .. && rm -rf libdeflate +RUN git clone https://github.com/cloudflare/zlib cloudflare-zlib && \ + cd cloudflare-zlib && ./configure --static --prefix=/usr/local/musl && \ + make install && cd .. && rm -rf cloudflare-zlib +#RUN wget https://www.zlib.net/zlib-$ZLIB_VERSION.tar.gz \ +# && tar xvfz zlib-$ZLIB_VERSION.tar.gz \ +# && cd zlib-$ZLIB_VERSION \ +# && ./configure --static --prefix=/usr/local/musl \ +# && make -j40 install RUN git clone git://sourceware.org/git/bzip2 \ && cd bzip2 \ #&& make -j40 CC=$CC AR=$AR RANLIB=$RANLIB CFLAGS=$CFLAGS bzip2 \ && make PREFIX=/usr/local/musl -j40 bzip2 \ - && make -j40 install -WORKDIR /root + && make -j40 install && cd .. && rm -rf bzip2 RUN wget https://tukaani.org/xz/xz-$LZMA_VERSION.tar.bz2 \ && tar xvfj xz-$LZMA_VERSION.tar.bz2 \ && cd xz-$LZMA_VERSION \ - && ./configure --prefix=/usr/local/musl --enable-static --disable-shared --host x86_64-unknown-linux-musl \ - && make -j40 install -WORKDIR /root + && ./configure --prefix=/usr/local/musl \ + --enable-static --disable-shared \ + --host x86_64-unknown-linux-musl \ + && make -j40 install && cd .. && rm -rf xz-$LZMA_VERSION xz-$LZMA_VERSION.tar.bz2 # A few gems from: https://wiki.openssl.org/index.php/Compilation_and_Installation # "OpenSSL has been around a long time, and it carries around a lot of cruft" @@ -91,20 +99,20 @@ RUN wget https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION.tar && make AR=x86_64-linux-musl-ar \ CC=x86_64-linux-musl-cc \ -j40 \ - && make RANLIB=x86_64-linux-musl-ranlib -j40 install + && make RANLIB=x86_64-linux-musl-ranlib -j40 install && cd .. \ + && rm -rf OpenSSL_$OPENSSL_VERSION.tar.gz openssl-OpenSSL_$OPENSSL_VERSION -WORKDIR /root RUN wget https://curl.haxx.se/download/curl-$CURL_VERSION.tar.gz \ && tar xvfz curl-$CURL_VERSION.tar.gz && cd curl-$CURL_VERSION \ && ./configure --prefix=/usr/local/musl --host x86_64-linux-musl \ --with-ssl=/usr/local/musl --with-zlib=/usr/local/musl --enable-static \ - --disable-dict --disable-file --disable-ftp --disable-gopher --disable-imap \ + --disable-dict --disable-ftp --disable-gopher --disable-imap \ --disable-pop3 --disable-rtsp --disable-smb --disable-smtp --disable-telnet \ --disable-tftp --disable-ntlm --disable-ldap \ - && make -j40 install + && make -j40 install \ + && rm -rf curl-$CURL_VERSION.tar.gz curl-$CURL_VERSION # To cater Rust's openssl-sys needs... -#ENV OPENSSL_DIR /usr/local/openssl ENV OPENSSL_DIR /usr/local/musl # Hack to force ld stick to musl on the hts-sys/build.rs side, otherwise: @@ -118,4 +126,4 @@ RUN rm /usr/bin/ld && ln -sf /usr/local/musl/bin/x86_64-linux-musl-ld /usr/bin/l # && . $HOME/.cargo/env \ # && rustup target add x86_64-unknown-linux-musl -CMD ["bash"] \ No newline at end of file +CMD ["bash"] diff --git a/docker/README.md b/docker/README.md index 57612c885..8ce0ee1cd 100644 --- a/docker/README.md +++ b/docker/README.md @@ -6,8 +6,8 @@ Allows to compile (rust-)htslib in a variety of environments and architectures v ```shell $ cd docker -$ docker build -t brainstorm/cross-x86_64-unknown-linux-musl:libcurl-openssl . -f Dockerfile.musl -$ docker build -t brainstorm/cross-x86_64-unknown-linux-gnu:libcurl-openssl . -f Dockerfile.gnu +$ docker build -t brainstorm/cross-x86_64-unknown-linux-musl:1.0.0 . -f Dockerfile.musl +$ docker build -t brainstorm/cross-x86_64-unknown-linux-gnu:1.0.0 . -f Dockerfile.gnu ``` Then, to build and test rust-htslib with the above containers, proceed as you would with `cargo`, using `cross` instead, i.e: From 082f6fde347c2c859a78b91a2e098b875bdfa23b Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Fri, 5 Jun 2020 17:00:38 +1000 Subject: [PATCH 3/5] Re-add bzip2, make sure curl is cleaned up --- docker/Dockerfile.musl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.musl b/docker/Dockerfile.musl index b6cd2776c..f2e55c626 100644 --- a/docker/Dockerfile.musl +++ b/docker/Dockerfile.musl @@ -68,7 +68,7 @@ RUN git clone https://github.com/cloudflare/zlib cloudflare-zlib && \ # && make -j40 install RUN git clone git://sourceware.org/git/bzip2 \ && cd bzip2 \ - #&& make -j40 CC=$CC AR=$AR RANLIB=$RANLIB CFLAGS=$CFLAGS bzip2 \ + && make -j40 CC=$CC AR=$AR RANLIB=$RANLIB CFLAGS=$CFLAGS bzip2 \ && make PREFIX=/usr/local/musl -j40 bzip2 \ && make -j40 install && cd .. && rm -rf bzip2 RUN wget https://tukaani.org/xz/xz-$LZMA_VERSION.tar.bz2 \ @@ -109,7 +109,7 @@ RUN wget https://curl.haxx.se/download/curl-$CURL_VERSION.tar.gz \ --disable-dict --disable-ftp --disable-gopher --disable-imap \ --disable-pop3 --disable-rtsp --disable-smb --disable-smtp --disable-telnet \ --disable-tftp --disable-ntlm --disable-ldap \ - && make -j40 install \ + && make -j40 install && cd .. \ && rm -rf curl-$CURL_VERSION.tar.gz curl-$CURL_VERSION # To cater Rust's openssl-sys needs... From 2468ea610789e2912d785417fd30fb201d7fc57f Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Sat, 6 Jun 2020 12:14:04 +1000 Subject: [PATCH 4/5] Add libdeflate sys crate (libdeflater), unsure on whether to make it optional or just ship it enabled by default in light of @juliangehring htslib benchmarks link --- Cargo.toml | 1 + hts-sys/Cargo.toml | 4 +++- hts-sys/build.rs | 9 +++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index c6e0d0677..6490467b2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ default = ["bzip2", "lzma", "curl"] bzip2 = ["hts-sys/bzip2"] lzma = ["hts-sys/lzma"] curl = ["hts-sys/curl"] +libdeflate = ["hts-sys/libdeflate"] #openssl = ["hts-sys/openssl"] serde = ["serde_base", "serde_bytes"] static = [] diff --git a/hts-sys/Cargo.toml b/hts-sys/Cargo.toml index 296a916df..dd881f9ee 100644 --- a/hts-sys/Cargo.toml +++ b/hts-sys/Cargo.toml @@ -22,11 +22,13 @@ libz-sys = { version = "1.0.25", features = ["static"] } bzip2-sys = { version = "0.1.8", optional = true } lzma-sys = { version = "0.1.16", optional = true, features = ["static"] } curl-sys = { version = "0.4.31", optional = true, features = ["static-curl", "static-ssl"] } +libdeflater = { version = "0.2.0", optional = true } [features] default = ["bzip2", "lzma"] bzip2 = ["bzip2-sys"] lzma = ["lzma-sys"] +libdeflate = ["libdeflater"] #openssl = ["openssl-sys"] curl = ["curl-sys"] static = [] @@ -35,4 +37,4 @@ static = [] fs-utils = "1.1" bindgen = { version = "0.53.2", default-features = false, features = ["runtime"] } cc = "1.0" -glob = "0.3.0" \ No newline at end of file +glob = "0.3.0" diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 66f745c6c..8664ef10b 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -73,6 +73,14 @@ fn main() { } else if let Ok(inc) = env::var("DEP_CURL_INCLUDE").map(PathBuf::from) { cfg.include(inc); } + +// let use_libdeflate = env::var("CARGO_FEATURE_LIBDEFLATE").is_ok(); +// if !use_libdeflate { +// let libdeflate_patterns = vec!["s/ -llibdeflate//", "/#define HAVE_LIBDEFLATE/d"]; +// sed_htslib_makefile(&out, &libdeflate_patterns, "libdeflate"); +// } else if let Ok(inc) = env::var("DEP_LIBDEFLATE_INCLUDE").map(PathBuf::from) { +// cfg.include(inc); +// } let tool = cfg.get_compiler(); let (cc_path, cflags_env) = (tool.path(), tool.cflags_env()); @@ -100,6 +108,7 @@ fn main() { .current_dir(out.join("htslib")) .env("CFLAGS", &cc_cflags) .arg(format!("--host={}", &host)) + .arg("--with-libdeflate") // TODO: Make it optional? .status().unwrap().success() { panic!("could not configure htslib nor any of its plugins") From a5cf84f5bf6b4190cd693f77cc3347009fb3c6bb Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Tue, 9 Jun 2020 13:10:10 +1000 Subject: [PATCH 5/5] Reverting changes to build.rs and Cargo.toml related to libdeflate in favor of PR #189. Leaving only docker-related setups --- Cargo.toml | 2 -- hts-sys/Cargo.toml | 3 --- hts-sys/build.rs | 8 -------- 3 files changed, 13 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6490467b2..95afd61ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,8 +37,6 @@ default = ["bzip2", "lzma", "curl"] bzip2 = ["hts-sys/bzip2"] lzma = ["hts-sys/lzma"] curl = ["hts-sys/curl"] -libdeflate = ["hts-sys/libdeflate"] -#openssl = ["hts-sys/openssl"] serde = ["serde_base", "serde_bytes"] static = [] diff --git a/hts-sys/Cargo.toml b/hts-sys/Cargo.toml index dd881f9ee..6180bd912 100644 --- a/hts-sys/Cargo.toml +++ b/hts-sys/Cargo.toml @@ -22,14 +22,11 @@ libz-sys = { version = "1.0.25", features = ["static"] } bzip2-sys = { version = "0.1.8", optional = true } lzma-sys = { version = "0.1.16", optional = true, features = ["static"] } curl-sys = { version = "0.4.31", optional = true, features = ["static-curl", "static-ssl"] } -libdeflater = { version = "0.2.0", optional = true } [features] default = ["bzip2", "lzma"] bzip2 = ["bzip2-sys"] lzma = ["lzma-sys"] -libdeflate = ["libdeflater"] -#openssl = ["openssl-sys"] curl = ["curl-sys"] static = [] diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 8664ef10b..247638c01 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -74,13 +74,6 @@ fn main() { cfg.include(inc); } -// let use_libdeflate = env::var("CARGO_FEATURE_LIBDEFLATE").is_ok(); -// if !use_libdeflate { -// let libdeflate_patterns = vec!["s/ -llibdeflate//", "/#define HAVE_LIBDEFLATE/d"]; -// sed_htslib_makefile(&out, &libdeflate_patterns, "libdeflate"); -// } else if let Ok(inc) = env::var("DEP_LIBDEFLATE_INCLUDE").map(PathBuf::from) { -// cfg.include(inc); -// } let tool = cfg.get_compiler(); let (cc_path, cflags_env) = (tool.path(), tool.cflags_env()); @@ -108,7 +101,6 @@ fn main() { .current_dir(out.join("htslib")) .env("CFLAGS", &cc_cflags) .arg(format!("--host={}", &host)) - .arg("--with-libdeflate") // TODO: Make it optional? .status().unwrap().success() { panic!("could not configure htslib nor any of its plugins")