From 0bb745da06bd8baa19c5987b839e9711bb44e881 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Wed, 18 Mar 2020 15:21:02 +1100 Subject: [PATCH 01/56] Add alternative strategy (local dependencies install) to get musl support shipped into rust-htslib --- docker/Dockerfile.musl | 40 ++++++++++++++++++++++++++++--- docker/config-musl-cross-make.mak | 4 ++++ 2 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 docker/config-musl-cross-make.mak diff --git a/docker/Dockerfile.musl b/docker/Dockerfile.musl index 55b228447..062fad648 100644 --- a/docker/Dockerfile.musl +++ b/docker/Dockerfile.musl @@ -1,7 +1,41 @@ FROM rustembedded/cross:x86_64-unknown-linux-musl ENV PKG_CONFIG_ALLOW_CROSS 1 -ENV OPENSSL_LIB_DIR /usr/lib/x86_64-linux-gnu -ENV OPENSSL_INCLUDE_DIR /usr/include/openssl +ENV LZMA_VERSION 5.2.4 +ENV ZLIB_VERSION 1.2.11 +ENV OPENSSL_VERSION 1.1.1d +ENV CURL_VERSION 7.69.1 + +# Install basics to locally compile htslib dependencies RUN apt-get update && \ - apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev musl musl-dev musl-tools linux-libc-dev linux-headers-4.15.0-20-generic + apt-get install -y build-essential git wget + +# The default includes and packages from the Ubuntu distro that cross uses will generate all sorts of linux-headers related include errors, see: +# https://github.com/rust-bio/rust-htslib/pull/184#commitcomment-37496651 +# Those are the packages installed, hopefully someone will find a good way to use the distro ones instead of compiling everything under /usr/local :/ +# apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev musl musl-dev musl-tools linux-libc-dev linux-headers-4.15.0-20-generic + + + +# For now we'll have to go nuts and not only build musl-1.2.0 from scratch but all the other libs too... +WORKDIR /root + +# Updated musl-cross toolchain that does not fail on OpenSSL: https://github.com/openssl/openssl/issues/7207 +RUN wget https://musl.libc.org/releases/musl-1.2.0.tar.gz && tar xvfz musl-1.2.0.tar.gz +COPY config-musl-cross-make.mak config.mak +WORKDIR /root/musl-1.2.0 +RUN ./configure && make && cd .. + +# Now we assume we have a properly configured musl-cross... +ENV CC=/usr/local/musl/bin/x86_64-linux-musl-cc +ENV CXX_x86_64_unknown_linux_musl=x86_64-linux-musl-g++ + +# .. and carry on with the htslib deps +RUN git clone git://sourceware.org/git/bzip2.git && cd bzip2 && make install && cd .. +RUN wget https://tukaani.org/xz/xz-$LZMA_VERSION.tar.bz2 && tar xvfj xz-$LZMA_VERSION.tar.bz2 && cd xz-$LZMA_VERSION && ./configure && make install && cd .. +RUN wget https://www.zlib.net/zlib-$ZLIB_VERSION.tar.gz && tar xvfz zlib-$ZLIB_VERSION.tar.gz && cd zlib-$ZLIB_VERSION && ./configure && make install && cd .. +RUN wget https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz && tar xvfz openssl-$OPENSSL_VERSION.tar.gz && tar xvfz openssl-$OPENSSL_VERSION.tar.gz && cd openssl-$OPENSSL_VERSION +RUN ./Configure --prefix=/usr/local/openssl linux-x86_64 no-shared && make && cd .. +RUN wget https://curl.haxx.se/download/curl-$CURL_VERSION.tar.gz && tar xvfz curl-$CURL_VERSION.tar.gz && cd curl-$CURL_VERSION && ./configure --host --with-ssl=/usr/local/openssl && make install + +CMD ["bash"] \ No newline at end of file diff --git a/docker/config-musl-cross-make.mak b/docker/config-musl-cross-make.mak new file mode 100644 index 000000000..08c8df062 --- /dev/null +++ b/docker/config-musl-cross-make.mak @@ -0,0 +1,4 @@ +TARGET = x86_64-linux-musl +OUTPUT = /usr/local/musl +GCC_VER = 9.2.0 + From a563f5c7c357e3619d0ba91687a8453fe215fe0e Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Thu, 19 Mar 2020 10:23:15 +1100 Subject: [PATCH 02/56] Tweaking toolchain further for rust-htslib-musl and its dependencies --- docker/Dockerfile.musl | 54 +++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/docker/Dockerfile.musl b/docker/Dockerfile.musl index 062fad648..4639f700b 100644 --- a/docker/Dockerfile.musl +++ b/docker/Dockerfile.musl @@ -1,41 +1,61 @@ FROM rustembedded/cross:x86_64-unknown-linux-musl +ENV MUSL_CROSS_MAKE_VERSION 0.9.9 ENV PKG_CONFIG_ALLOW_CROSS 1 ENV LZMA_VERSION 5.2.4 ENV ZLIB_VERSION 1.2.11 -ENV OPENSSL_VERSION 1.1.1d +ENV OPENSSL_VERSION 1_1_1e ENV CURL_VERSION 7.69.1 -# Install basics to locally compile htslib dependencies -RUN apt-get update && \ - apt-get install -y build-essential git wget - # The default includes and packages from the Ubuntu distro that cross uses will generate all sorts of linux-headers related include errors, see: # https://github.com/rust-bio/rust-htslib/pull/184#commitcomment-37496651 # Those are the packages installed, hopefully someone will find a good way to use the distro ones instead of compiling everything under /usr/local :/ # apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev musl musl-dev musl-tools linux-libc-dev linux-headers-4.15.0-20-generic +# Install basics to locally compile htslib dependencies +RUN apt-get update && \ + apt-get install -y build-essential git wget vim +# Remove pre-installed musl, to avoid cross-musl-make interference +RUN apt-get remove -y musl # For now we'll have to go nuts and not only build musl-1.2.0 from scratch but all the other libs too... WORKDIR /root -# Updated musl-cross toolchain that does not fail on OpenSSL: https://github.com/openssl/openssl/issues/7207 -RUN wget https://musl.libc.org/releases/musl-1.2.0.tar.gz && tar xvfz musl-1.2.0.tar.gz +# Updated musl-cross toolchain that does not fail on OpenSSL: https://github.com/openssl/openssl/issues/7207 +RUN wget https://github.com/richfelker/musl-cross-make/archive/v$MUSL_CROSS_MAKE_VERSION.tar.gz && tar xvfz v$MUSL_CROSS_MAKE_VERSION.tar.gz +WORKDIR /root/musl-cross-make-$MUSL_CROSS_MAKE_VERSION COPY config-musl-cross-make.mak config.mak -WORKDIR /root/musl-1.2.0 -RUN ./configure && make && cd .. +RUN make -j16 install # Now we assume we have a properly configured musl-cross... -ENV CC=/usr/local/musl/bin/x86_64-linux-musl-cc -ENV CXX_x86_64_unknown_linux_musl=x86_64-linux-musl-g++ +ENV PATH "/usr/local/musl/bin:$PATH" +ENV CFLAGS "-I/usr/local/include -L/usr/local/lib -fPIC" +ENV CROSS_COMPILE x86_64-linux-musl- +ENV CC $CROSS_COMPILE-cc +ENV AR $CROSS_COMPILE-ar +ENV RANLIB $CROSS_COMPILE-ranlib +ENV CXX $CROSS_COMPILE-g++ # .. and carry on with the htslib deps -RUN git clone git://sourceware.org/git/bzip2.git && cd bzip2 && make install && cd .. -RUN wget https://tukaani.org/xz/xz-$LZMA_VERSION.tar.bz2 && tar xvfj xz-$LZMA_VERSION.tar.bz2 && cd xz-$LZMA_VERSION && ./configure && make install && cd .. -RUN wget https://www.zlib.net/zlib-$ZLIB_VERSION.tar.gz && tar xvfz zlib-$ZLIB_VERSION.tar.gz && cd zlib-$ZLIB_VERSION && ./configure && make install && cd .. -RUN wget https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz && tar xvfz openssl-$OPENSSL_VERSION.tar.gz && tar xvfz openssl-$OPENSSL_VERSION.tar.gz && cd openssl-$OPENSSL_VERSION -RUN ./Configure --prefix=/usr/local/openssl linux-x86_64 no-shared && make && cd .. -RUN wget https://curl.haxx.se/download/curl-$CURL_VERSION.tar.gz && tar xvfz curl-$CURL_VERSION.tar.gz && cd curl-$CURL_VERSION && ./configure --host --with-ssl=/usr/local/openssl && make install +WORKDIR /root +RUN git clone git://sourceware.org/git/bzip2.git && cd bzip2 && make -j16 install +WORKDIR /root +RUN wget https://tukaani.org/xz/xz-$LZMA_VERSION.tar.bz2 && tar xvfj xz-$LZMA_VERSION.tar.bz2 && cd xz-$LZMA_VERSION && ./configure --host x86_64-unknown-linux-musl && make -j16 install +WORKDIR /root +RUN wget https://www.zlib.net/zlib-$ZLIB_VERSION.tar.gz && tar xvfz zlib-$ZLIB_VERSION.tar.gz && cd zlib-$ZLIB_VERSION && ./configure --static && make -j16 install +WORKDIR /root +# A few gems from: https://wiki.openssl.org/index.php/Compilation_and_Installation +# "OpenSSL has been around a long time, and it carries around a lot of cruft" +# "SSLv2 is completely broken, and you should disable it during configuration" +# "You should specify both --prefix and --openssldir to ensure make install works as expected." +RUN wget https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION.tar.gz && tar xvfz OpenSSL_$OPENSSL_VERSION.tar.gz && tar xvfz OpenSSL_$OPENSSL_VERSION.tar.gz && cd openssl-OpenSSL_$OPENSSL_VERSION && ./Configure --prefix=/usr/local/openssl --openssldir=/usr/local/openssl/etc --with-zlib-lib=/root/zlib-$ZLIB_VERSION linux-x86_64 no-shared no-dso no-gost no-engine no-ssl2 no-srp no-srtp no-tests zlib no-weak-ssl-ciphers && make -j16 install +WORKDIR /root +RUN wget https://curl.haxx.se/download/curl-$CURL_VERSION.tar.gz && tar xvfz curl-$CURL_VERSION.tar.gz && cd curl-$CURL_VERSION && \ + ./configure --host x86_64-unknown-linux-musl --with-ssl=/usr/local/openssl --enable-static \ + --disable-dict --disable-file --disable-ftp --disable-gopher --disable-imap \ + --disable-pop3 --disable-rtsp --disable-smb --disable-smtp --disable-telnet \ + --disable-tftp --disable-ntlm && \ + make -j16 install CMD ["bash"] \ No newline at end of file From 17550f51e2ed73b97a1e55edb23a9b063ed9133e Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Thu, 19 Mar 2020 13:06:17 +1100 Subject: [PATCH 03/56] Circumvent Makefile openssl bug in https://github.com/openssl/openssl/issues/11362 and many other compile time gotchas --- docker/Dockerfile.musl | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/docker/Dockerfile.musl b/docker/Dockerfile.musl index 4639f700b..5c64be90b 100644 --- a/docker/Dockerfile.musl +++ b/docker/Dockerfile.musl @@ -26,36 +26,43 @@ WORKDIR /root RUN wget https://github.com/richfelker/musl-cross-make/archive/v$MUSL_CROSS_MAKE_VERSION.tar.gz && tar xvfz v$MUSL_CROSS_MAKE_VERSION.tar.gz WORKDIR /root/musl-cross-make-$MUSL_CROSS_MAKE_VERSION COPY config-musl-cross-make.mak config.mak -RUN make -j16 install +RUN make install # Now we assume we have a properly configured musl-cross... ENV PATH "/usr/local/musl/bin:$PATH" -ENV CFLAGS "-I/usr/local/include -L/usr/local/lib -fPIC" +ENV CFLAGS "-fPIC" ENV CROSS_COMPILE x86_64-linux-musl- -ENV CC $CROSS_COMPILE-cc -ENV AR $CROSS_COMPILE-ar -ENV RANLIB $CROSS_COMPILE-ranlib -ENV CXX $CROSS_COMPILE-g++ +ENV CC ${CROSS_COMPILE}cc +ENV AR ${CROSS_COMPILE}ar +ENV RANLIB ${CROSS_COMPILE}ranlib +ENV CXX ${CROSS_COMPILE}g++ +ENV CPPFLAGS "-I/usr/local/include" +ENV LDFLAGS "-L/usr/local/lib -static" # .. and carry on with the htslib deps WORKDIR /root -RUN git clone git://sourceware.org/git/bzip2.git && cd bzip2 && make -j16 install +RUN git clone git://sourceware.org/git/bzip2.git && cd bzip2 && make CC=$CC AR=$AR RANLIB=$RANLIB bzip2 && make install WORKDIR /root -RUN wget https://tukaani.org/xz/xz-$LZMA_VERSION.tar.bz2 && tar xvfj xz-$LZMA_VERSION.tar.bz2 && cd xz-$LZMA_VERSION && ./configure --host x86_64-unknown-linux-musl && make -j16 install +RUN wget https://tukaani.org/xz/xz-$LZMA_VERSION.tar.bz2 && tar xvfj xz-$LZMA_VERSION.tar.bz2 && cd xz-$LZMA_VERSION && ./configure --host x86_64-unknown-linux-musl && make install WORKDIR /root -RUN wget https://www.zlib.net/zlib-$ZLIB_VERSION.tar.gz && tar xvfz zlib-$ZLIB_VERSION.tar.gz && cd zlib-$ZLIB_VERSION && ./configure --static && make -j16 install +RUN wget https://www.zlib.net/zlib-$ZLIB_VERSION.tar.gz && tar xvfz zlib-$ZLIB_VERSION.tar.gz && cd zlib-$ZLIB_VERSION && ./configure --static && make install WORKDIR /root # A few gems from: https://wiki.openssl.org/index.php/Compilation_and_Installation # "OpenSSL has been around a long time, and it carries around a lot of cruft" # "SSLv2 is completely broken, and you should disable it during configuration" # "You should specify both --prefix and --openssldir to ensure make install works as expected." -RUN wget https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION.tar.gz && tar xvfz OpenSSL_$OPENSSL_VERSION.tar.gz && tar xvfz OpenSSL_$OPENSSL_VERSION.tar.gz && cd openssl-OpenSSL_$OPENSSL_VERSION && ./Configure --prefix=/usr/local/openssl --openssldir=/usr/local/openssl/etc --with-zlib-lib=/root/zlib-$ZLIB_VERSION linux-x86_64 no-shared no-dso no-gost no-engine no-ssl2 no-srp no-srtp no-tests zlib no-weak-ssl-ciphers && make -j16 install +RUN unset CC && unset AR && unset RANLIB && wget https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION.tar.gz && \ + tar xvfz OpenSSL_$OPENSSL_VERSION.tar.gz && cd openssl-OpenSSL_$OPENSSL_VERSION && \ + ./Configure --prefix=/usr/local/openssl --openssldir=/usr/local/openssl/etc \ + --with-zlib-lib=/root/zlib-$ZLIB_VERSION linux-x86_64 \ + no-shared no-dso no-gost no-engine no-ssl2 no-srp no-srtp no-tests zlib \ + no-weak-ssl-ciphers && make && make install WORKDIR /root RUN wget https://curl.haxx.se/download/curl-$CURL_VERSION.tar.gz && tar xvfz curl-$CURL_VERSION.tar.gz && cd curl-$CURL_VERSION && \ - ./configure --host x86_64-unknown-linux-musl --with-ssl=/usr/local/openssl --enable-static \ + ./configure --host x86_64-linux-musl --with-ssl=/usr/local/openssl --enable-static \ --disable-dict --disable-file --disable-ftp --disable-gopher --disable-imap \ --disable-pop3 --disable-rtsp --disable-smb --disable-smtp --disable-telnet \ --disable-tftp --disable-ntlm && \ - make -j16 install + make install CMD ["bash"] \ No newline at end of file From 3a33b5df184bd1b144d258d5d71ae0d85b925982 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Thu, 19 Mar 2020 13:33:16 +1100 Subject: [PATCH 04/56] Add newly docker baked image with the whole toolchain built statically with and for musl --- Cross.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Cross.toml b/Cross.toml index c753b089b..8e3f98410 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,4 +1,2 @@ [target.x86_64-unknown-linux-musl] -image = "brainstorm/rust_musl_docker:stable-latest-libcurl" -[target.x86_64-unknown-linux-gnu] -image = "brainstorm/cross-x86_64-unknown-linux-gnu:libcurl-openssl" +image = "brainstorm/rust-htslib-musl:latest" From 4f0cbc3b67020b25d3adef316990c1219cb7b83e Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Thu, 19 Mar 2020 13:34:06 +1100 Subject: [PATCH 05/56] Bypass unneeded CI steps for now to focus on musl --- .github/workflows/rust.yml | 124 ++++++++++++++++++------------------- docker/Dockerfile.musl | 24 +++---- 2 files changed, 74 insertions(+), 74 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e20c7a81c..50b9f5bab 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -7,46 +7,46 @@ on: branches: [ master ] jobs: - Formatting: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 + # Formatting: + # runs-on: ubuntu-latest + # steps: + # - name: Checkout repository + # uses: actions/checkout@v2 - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - components: rustfmt + # - name: Install stable toolchain + # uses: actions-rs/toolchain@v1 + # with: + # toolchain: stable + # override: true + # components: rustfmt - - name: Check format - run: cargo fmt -- --check - - Linting: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 + # - name: Check format + # run: cargo fmt -- --check - - name: Checkout submodules - uses: textbook/git-checkout-submodule-action@2.0.0 + # Linting: + # runs-on: ubuntu-latest + # steps: + # - name: Checkout repository + # uses: actions/checkout@v2 - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - components: clippy + # - name: Checkout submodules + # uses: textbook/git-checkout-submodule-action@2.0.0 - - name: Lint with clippy - uses: actions-rs/clippy-check@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} + # - name: Install stable toolchain + # uses: actions-rs/toolchain@v1 + # with: + # toolchain: stable + # override: true + # components: clippy + + # - name: Lint with clippy + # uses: actions-rs/clippy-check@v1 + # with: + # token: ${{ secrets.GITHUB_TOKEN }} Testing: - needs: Formatting + #needs: Formatting runs-on: ubuntu-latest steps: - name: Checkout repository @@ -55,37 +55,37 @@ jobs: - name: Checkout submodules uses: textbook/git-checkout-submodule-action@2.0.0 - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true + # - name: Install stable toolchain + # uses: actions-rs/toolchain@v1 + # with: + # toolchain: stable + # override: true - - name: Install system dependencies - run: | - sudo apt-get install --yes zlib1g-dev libbz2-dev musl musl-dev musl-tools clang libc6-dev + # - name: Install system dependencies + # run: | + # sudo apt-get install --yes zlib1g-dev libbz2-dev musl musl-dev musl-tools clang libc6-dev - - name: Run cargo-tarpaulin - uses: actions-rs/tarpaulin@v0.1 - with: - args: '--out Lcov -- --test-threads 1' + # - name: Run cargo-tarpaulin + # uses: actions-rs/tarpaulin@v0.1 + # with: + # args: '--out Lcov -- --test-threads 1' + + # - name: Upload coverage + # uses: coverallsapp/github-action@master + # with: + # github-token: ${{ secrets.GITHUB_TOKEN }} + # path-to-lcov: ./lcov.info + + # - name: Test musl build without default features + # uses: actions-rs/cargo@v1 + # with: + # use-cross: true + # command: build + # args: --target x86_64-unknown-linux-musl --no-default-features - - name: Upload coverage - uses: coverallsapp/github-action@master + - name: Test musl build with all features + uses: actions-rs/cargo@v1 with: - github-token: ${{ secrets.GITHUB_TOKEN }} - path-to-lcov: ./lcov.info - -# - name: Test musl build without default features -# uses: actions-rs/cargo@v1 -# with: -# use-cross: true -# command: build -# args: --target x86_64-unknown-linux-musl --no-default-features -# -# - name: Test musl build with all features -# uses: actions-rs/cargo@v1 -# with: -# use-cross: true -# command: build -# args: --target x86_64-unknown-linux-musl --all-features + use-cross: true + command: build + args: --target x86_64-unknown-linux-musl --all-features diff --git a/docker/Dockerfile.musl b/docker/Dockerfile.musl index 5c64be90b..f67016079 100644 --- a/docker/Dockerfile.musl +++ b/docker/Dockerfile.musl @@ -1,6 +1,6 @@ FROM rustembedded/cross:x86_64-unknown-linux-musl -ENV MUSL_CROSS_MAKE_VERSION 0.9.9 +ENV MUSL_CROSS_VERSION 0.9.9 ENV PKG_CONFIG_ALLOW_CROSS 1 ENV LZMA_VERSION 5.2.4 ENV ZLIB_VERSION 1.2.11 @@ -14,19 +14,19 @@ ENV CURL_VERSION 7.69.1 # Install basics to locally compile htslib dependencies RUN apt-get update && \ - apt-get install -y build-essential git wget vim + apt-get install -y build-essential git wget -# Remove pre-installed musl, to avoid cross-musl-make interference +# Remove pre-installed musl, to avoid cross-musl-make -j48 interference RUN apt-get remove -y musl # For now we'll have to go nuts and not only build musl-1.2.0 from scratch but all the other libs too... WORKDIR /root # Updated musl-cross toolchain that does not fail on OpenSSL: https://github.com/openssl/openssl/issues/7207 -RUN wget https://github.com/richfelker/musl-cross-make/archive/v$MUSL_CROSS_MAKE_VERSION.tar.gz && tar xvfz v$MUSL_CROSS_MAKE_VERSION.tar.gz -WORKDIR /root/musl-cross-make-$MUSL_CROSS_MAKE_VERSION +RUN wget https://github.com/richfelker/musl-cross-make/archive/v$MUSL_CROSS_VERSION.tar.gz && tar xvfz v$MUSL_CROSS_VERSION.tar.gz +WORKDIR /root/musl-cross-make-$MUSL_CROSS_VERSION COPY config-musl-cross-make.mak config.mak -RUN make install +RUN make -j48 install # Now we assume we have a properly configured musl-cross... ENV PATH "/usr/local/musl/bin:$PATH" @@ -41,28 +41,28 @@ ENV LDFLAGS "-L/usr/local/lib -static" # .. and carry on with the htslib deps WORKDIR /root -RUN git clone git://sourceware.org/git/bzip2.git && cd bzip2 && make CC=$CC AR=$AR RANLIB=$RANLIB bzip2 && make install +RUN git clone git://sourceware.org/git/bzip2.git && cd bzip2 && make -j48 CC=$CC AR=$AR RANLIB=$RANLIB bzip2 && make -j48 install WORKDIR /root -RUN wget https://tukaani.org/xz/xz-$LZMA_VERSION.tar.bz2 && tar xvfj xz-$LZMA_VERSION.tar.bz2 && cd xz-$LZMA_VERSION && ./configure --host x86_64-unknown-linux-musl && make install +RUN wget https://tukaani.org/xz/xz-$LZMA_VERSION.tar.bz2 && tar xvfj xz-$LZMA_VERSION.tar.bz2 && cd xz-$LZMA_VERSION && ./configure --host x86_64-unknown-linux-musl && make -j48 install WORKDIR /root -RUN wget https://www.zlib.net/zlib-$ZLIB_VERSION.tar.gz && tar xvfz zlib-$ZLIB_VERSION.tar.gz && cd zlib-$ZLIB_VERSION && ./configure --static && make install +RUN wget https://www.zlib.net/zlib-$ZLIB_VERSION.tar.gz && tar xvfz zlib-$ZLIB_VERSION.tar.gz && cd zlib-$ZLIB_VERSION && ./configure --static && make -j48 install WORKDIR /root # A few gems from: https://wiki.openssl.org/index.php/Compilation_and_Installation # "OpenSSL has been around a long time, and it carries around a lot of cruft" # "SSLv2 is completely broken, and you should disable it during configuration" -# "You should specify both --prefix and --openssldir to ensure make install works as expected." +# "You should specify both --prefix and --openssldir to ensure make -j48 install works as expected." RUN unset CC && unset AR && unset RANLIB && wget https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION.tar.gz && \ tar xvfz OpenSSL_$OPENSSL_VERSION.tar.gz && cd openssl-OpenSSL_$OPENSSL_VERSION && \ ./Configure --prefix=/usr/local/openssl --openssldir=/usr/local/openssl/etc \ --with-zlib-lib=/root/zlib-$ZLIB_VERSION linux-x86_64 \ no-shared no-dso no-gost no-engine no-ssl2 no-srp no-srtp no-tests zlib \ - no-weak-ssl-ciphers && make && make install + no-weak-ssl-ciphers && make -j48 && make -j48 install WORKDIR /root RUN wget https://curl.haxx.se/download/curl-$CURL_VERSION.tar.gz && tar xvfz curl-$CURL_VERSION.tar.gz && cd curl-$CURL_VERSION && \ ./configure --host x86_64-linux-musl --with-ssl=/usr/local/openssl --enable-static \ --disable-dict --disable-file --disable-ftp --disable-gopher --disable-imap \ --disable-pop3 --disable-rtsp --disable-smb --disable-smtp --disable-telnet \ --disable-tftp --disable-ntlm && \ - make install + make -j48 install CMD ["bash"] \ No newline at end of file From 865e3d17cf213b535ddf3a34864eac2f074e1486 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Thu, 19 Mar 2020 13:55:36 +1100 Subject: [PATCH 06/56] Tell openssl-sys crate where is our custom musl-openssl cross compiled directory... --- docker/Dockerfile.musl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/Dockerfile.musl b/docker/Dockerfile.musl index f67016079..e12d8ab7c 100644 --- a/docker/Dockerfile.musl +++ b/docker/Dockerfile.musl @@ -65,4 +65,7 @@ RUN wget https://curl.haxx.se/download/curl-$CURL_VERSION.tar.gz && tar xvfz cur --disable-tftp --disable-ntlm && \ make -j48 install +# To cater Rust's openssl-sys needs... +ENV OPENSSL_DIR /usr/local/openssl + CMD ["bash"] \ No newline at end of file From 23b89a8281bcac967212002b02db6ae11aea60a0 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Thu, 19 Mar 2020 14:22:00 +1100 Subject: [PATCH 07/56] It is possible that https://github.com/rust-bio/rust-htslib/pull/193/checks?check_run_id=518233498#step:5:253 is just a binutils bug according to https://bbs.archlinux.org/viewtopic.php?id=242682 and other sites... --- docker/config-musl-cross-make.mak | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/config-musl-cross-make.mak b/docker/config-musl-cross-make.mak index 08c8df062..2c3cc25ef 100644 --- a/docker/config-musl-cross-make.mak +++ b/docker/config-musl-cross-make.mak @@ -1,4 +1,4 @@ TARGET = x86_64-linux-musl OUTPUT = /usr/local/musl GCC_VER = 9.2.0 - +BINUTILS_VER = 2.33.1 \ No newline at end of file From 3a3c59187024a63b87d62724e9d15a8c376ccec9 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Thu, 19 Mar 2020 14:53:23 +1100 Subject: [PATCH 08/56] Revert linting/formatting steps, those are done (and needed) anyway for the PR... --- .github/workflows/rust.yml | 118 ++++++++++++++++++------------------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 50b9f5bab..c4ccc5e91 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -7,46 +7,46 @@ on: branches: [ master ] jobs: - # Formatting: - # runs-on: ubuntu-latest - # steps: - # - name: Checkout repository - # uses: actions/checkout@v2 - - # - name: Install stable toolchain - # uses: actions-rs/toolchain@v1 - # with: - # toolchain: stable - # override: true - # components: rustfmt + Formatting: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + components: rustfmt - # - name: Check format - # run: cargo fmt -- --check + - name: Check format + run: cargo fmt -- --check - # Linting: - # runs-on: ubuntu-latest - # steps: - # - name: Checkout repository - # uses: actions/checkout@v2 + Linting: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 - # - name: Checkout submodules - # uses: textbook/git-checkout-submodule-action@2.0.0 + - name: Checkout submodules + uses: textbook/git-checkout-submodule-action@2.0.0 - # - name: Install stable toolchain - # uses: actions-rs/toolchain@v1 - # with: - # toolchain: stable - # override: true - # components: clippy + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + components: clippy - # - name: Lint with clippy - # uses: actions-rs/clippy-check@v1 - # with: - # token: ${{ secrets.GITHUB_TOKEN }} + - name: Lint with clippy + uses: actions-rs/clippy-check@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} Testing: - #needs: Formatting + needs: Formatting runs-on: ubuntu-latest steps: - name: Checkout repository @@ -55,33 +55,33 @@ jobs: - name: Checkout submodules uses: textbook/git-checkout-submodule-action@2.0.0 - # - name: Install stable toolchain - # uses: actions-rs/toolchain@v1 - # with: - # toolchain: stable - # override: true - - # - name: Install system dependencies - # run: | - # sudo apt-get install --yes zlib1g-dev libbz2-dev musl musl-dev musl-tools clang libc6-dev - - # - name: Run cargo-tarpaulin - # uses: actions-rs/tarpaulin@v0.1 - # with: - # args: '--out Lcov -- --test-threads 1' - - # - name: Upload coverage - # uses: coverallsapp/github-action@master - # with: - # github-token: ${{ secrets.GITHUB_TOKEN }} - # path-to-lcov: ./lcov.info - - # - name: Test musl build without default features - # uses: actions-rs/cargo@v1 - # with: - # use-cross: true - # command: build - # args: --target x86_64-unknown-linux-musl --no-default-features + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + + - name: Install system dependencies + run: | + sudo apt-get install --yes zlib1g-dev libbz2-dev musl musl-dev musl-tools clang libc6-dev + + - name: Run cargo-tarpaulin + uses: actions-rs/tarpaulin@v0.1 + with: + args: '--out Lcov -- --test-threads 1' + + - name: Upload coverage + uses: coverallsapp/github-action@master + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + path-to-lcov: ./lcov.info + + - name: Test musl build without default features + uses: actions-rs/cargo@v1 + with: + use-cross: true + command: build + args: --target x86_64-unknown-linux-musl --no-default-features - name: Test musl build with all features uses: actions-rs/cargo@v1 From 511a43298a5dd21b4df97b22594a67a8067a0c6b Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Thu, 19 Mar 2020 15:28:02 +1100 Subject: [PATCH 09/56] Nasty hack with ld, tell musl that its own includes are available via CPPFLAGS and hopefully bypass clang/llvm errors? --- docker/Dockerfile.musl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.musl b/docker/Dockerfile.musl index e12d8ab7c..8c360bfcf 100644 --- a/docker/Dockerfile.musl +++ b/docker/Dockerfile.musl @@ -36,7 +36,7 @@ ENV CC ${CROSS_COMPILE}cc ENV AR ${CROSS_COMPILE}ar ENV RANLIB ${CROSS_COMPILE}ranlib ENV CXX ${CROSS_COMPILE}g++ -ENV CPPFLAGS "-I/usr/local/include" +ENV CPPFLAGS "-I/usr/local/include -I/usr/local/musl/x86_64-linux-musl/include/" ENV LDFLAGS "-L/usr/local/lib -static" # .. and carry on with the htslib deps @@ -68,4 +68,8 @@ RUN wget https://curl.haxx.se/download/curl-$CURL_VERSION.tar.gz && tar xvfz cur # To cater Rust's openssl-sys needs... ENV OPENSSL_DIR /usr/local/openssl -CMD ["bash"] \ No newline at end of file +# XXX: Hack to force ld stick to musl on the hts-sys/build.rs side +RUN rm /usr/bin/ld && ln -sf /usr/local/musl/bin/x86_64-linux-musl-ld /usr/bin/ld +RUN apt-get install -y libclang1-9 + +CMD ["bash"] From 9d68756f4443e18ecae60e93fd3e79934e0dbea3 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Fri, 20 Mar 2020 17:04:48 +1100 Subject: [PATCH 10/56] LDFLAGS and CPPFLAGS need to be passed to hts-sys, plus need to be surrounded by quotes since there are spaces in the variables --- docker/Dockerfile.musl | 15 +++++++++------ hts-sys/build.rs | 4 ++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile.musl b/docker/Dockerfile.musl index 8c360bfcf..70eca1d14 100644 --- a/docker/Dockerfile.musl +++ b/docker/Dockerfile.musl @@ -36,14 +36,14 @@ ENV CC ${CROSS_COMPILE}cc ENV AR ${CROSS_COMPILE}ar ENV RANLIB ${CROSS_COMPILE}ranlib ENV CXX ${CROSS_COMPILE}g++ -ENV CPPFLAGS "-I/usr/local/include -I/usr/local/musl/x86_64-linux-musl/include/" -ENV LDFLAGS "-L/usr/local/lib -static" +ENV CPPFLAGS "-I/usr/local/musl/x86_64-linux-musl/include -I/usr/local/include" +ENV LDFLAGS "-L/usr/local/musl/x86_64-linux-musl/lib -L/usr/local/lib" # .. and carry on with the htslib deps WORKDIR /root -RUN git clone git://sourceware.org/git/bzip2.git && cd bzip2 && make -j48 CC=$CC AR=$AR RANLIB=$RANLIB bzip2 && make -j48 install +RUN git clone git://sourceware.org/git/bzip2.git && cd bzip2 && make -j48 CC=$CC AR=$AR RANLIB=$RANLIB CFLAGS=$CFLAGS bzip2 && make -j48 install WORKDIR /root -RUN wget https://tukaani.org/xz/xz-$LZMA_VERSION.tar.bz2 && tar xvfj xz-$LZMA_VERSION.tar.bz2 && cd xz-$LZMA_VERSION && ./configure --host x86_64-unknown-linux-musl && make -j48 install +RUN wget https://tukaani.org/xz/xz-$LZMA_VERSION.tar.bz2 && tar xvfj xz-$LZMA_VERSION.tar.bz2 && cd xz-$LZMA_VERSION && ./configure --host x86_64-unknown-linux-musl && make -j48 install WORKDIR /root RUN wget https://www.zlib.net/zlib-$ZLIB_VERSION.tar.gz && tar xvfz zlib-$ZLIB_VERSION.tar.gz && cd zlib-$ZLIB_VERSION && ./configure --static && make -j48 install WORKDIR /root @@ -56,7 +56,7 @@ RUN unset CC && unset AR && unset RANLIB && wget https://github.com/openssl/open ./Configure --prefix=/usr/local/openssl --openssldir=/usr/local/openssl/etc \ --with-zlib-lib=/root/zlib-$ZLIB_VERSION linux-x86_64 \ no-shared no-dso no-gost no-engine no-ssl2 no-srp no-srtp no-tests zlib \ - no-weak-ssl-ciphers && make -j48 && make -j48 install + no-weak-ssl-ciphers && make -j48 && make CFLAGS="$CFLAGS" LDFLAGS="$LDFLAGS" -j48 install WORKDIR /root RUN wget https://curl.haxx.se/download/curl-$CURL_VERSION.tar.gz && tar xvfz curl-$CURL_VERSION.tar.gz && cd curl-$CURL_VERSION && \ ./configure --host x86_64-linux-musl --with-ssl=/usr/local/openssl --enable-static \ @@ -68,8 +68,11 @@ RUN wget https://curl.haxx.se/download/curl-$CURL_VERSION.tar.gz && tar xvfz cur # To cater Rust's openssl-sys needs... ENV OPENSSL_DIR /usr/local/openssl -# XXX: Hack to force ld stick to musl on the hts-sys/build.rs side +# Hack to force ld stick to musl on the hts-sys/build.rs side RUN rm /usr/bin/ld && ln -sf /usr/local/musl/bin/x86_64-linux-musl-ld /usr/bin/ld RUN apt-get install -y libclang1-9 +# Prepare rustup and toolchain locally for easy manual intervention in this container +RUN wget https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init && chmod +x rustup-init && ./rustup-init -y && . $HOME/.cargo/env && rustup target add x86_64-unknown-linux-musl + CMD ["bash"] diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 3891084e1..aa21aec30 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -73,10 +73,14 @@ fn main() { let tool = cfg.get_compiler(); let (cc_path, cflags_env) = (tool.path(), tool.cflags_env()); let cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); + let cppflags = env::var("CPPFLAGS").unwrap(); + let ldflags= env::var("CPPFLAGS").unwrap(); if !Command::new("make") .current_dir(out.join("htslib")) .arg(format!("CC={}", cc_path.display())) .arg(format!("CFLAGS={}", cc_cflags)) + .arg(format!("CPPFLAGS=\"{}\"", cppflags)) + .arg(format!("LDFLAGS=\"{}\"", ldflags)) .arg("lib-static") .arg("-B") .status() From da356ee370d6b35140f60c5cae2ef37cce01a871 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Sat, 21 Mar 2020 20:53:07 +1100 Subject: [PATCH 11/56] Be a bit more careful with env variables when clippy is running --- hts-sys/build.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hts-sys/build.rs b/hts-sys/build.rs index aa21aec30..87065615a 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -73,8 +73,8 @@ fn main() { let tool = cfg.get_compiler(); let (cc_path, cflags_env) = (tool.path(), tool.cflags_env()); let cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); - let cppflags = env::var("CPPFLAGS").unwrap(); - let ldflags= env::var("CPPFLAGS").unwrap(); + let cppflags = env::var("CPPFLAGS").unwrap_or("".to_string()); + let ldflags= env::var("CPPFLAGS").unwrap_or("".to_string()); if !Command::new("make") .current_dir(out.join("htslib")) .arg(format!("CC={}", cc_path.display())) From 8d8513b0c4ecda8e5385f0e7e04bc733f7884f3e Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Sat, 21 Mar 2020 21:05:01 +1100 Subject: [PATCH 12/56] A bit more classy to pass clippy? --- hts-sys/build.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 87065615a..0eede273b 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -73,8 +73,8 @@ fn main() { let tool = cfg.get_compiler(); let (cc_path, cflags_env) = (tool.path(), tool.cflags_env()); let cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); - let cppflags = env::var("CPPFLAGS").unwrap_or("".to_string()); - let ldflags= env::var("CPPFLAGS").unwrap_or("".to_string()); + let cppflags = env::var("CPPFLAGS").unwrap_or_default(); + let ldflags= env::var("CPPFLAGS").unwrap_or_default(); if !Command::new("make") .current_dir(out.join("htslib")) .arg(format!("CC={}", cc_path.display())) From cb60f01a9eb3101a134899abd4a99c1b24f45b05 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Sat, 21 Mar 2020 21:13:12 +1100 Subject: [PATCH 13/56] Temporarily try to bypass formatting/linting again, not helping focus on musl --- .github/workflows/rust.yml | 118 ++++++++++++++++++------------------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index c4ccc5e91..dcb3e4e6c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -7,46 +7,46 @@ on: branches: [ master ] jobs: - Formatting: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - components: rustfmt + # Formatting: + # runs-on: ubuntu-latest + # steps: + # - name: Checkout repository + # uses: actions/checkout@v2 + + # - name: Install stable toolchain + # uses: actions-rs/toolchain@v1 + # with: + # toolchain: stable + # override: true + # components: rustfmt - - name: Check format - run: cargo fmt -- --check + # - name: Check format + # run: cargo fmt -- --check - Linting: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 + # Linting: + # runs-on: ubuntu-latest + # steps: + # - name: Checkout repository + # uses: actions/checkout@v2 - - name: Checkout submodules - uses: textbook/git-checkout-submodule-action@2.0.0 + # - name: Checkout submodules + # uses: textbook/git-checkout-submodule-action@2.0.0 - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - components: clippy + # - name: Install stable toolchain + # uses: actions-rs/toolchain@v1 + # with: + # toolchain: stable + # override: true + # components: clippy - - name: Lint with clippy - uses: actions-rs/clippy-check@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} + # - name: Lint with clippy + # uses: actions-rs/clippy-check@v1 + # with: + # token: ${{ secrets.GITHUB_TOKEN }} Testing: - needs: Formatting + # needs: Formatting runs-on: ubuntu-latest steps: - name: Checkout repository @@ -55,33 +55,33 @@ jobs: - name: Checkout submodules uses: textbook/git-checkout-submodule-action@2.0.0 - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - - name: Install system dependencies - run: | - sudo apt-get install --yes zlib1g-dev libbz2-dev musl musl-dev musl-tools clang libc6-dev - - - name: Run cargo-tarpaulin - uses: actions-rs/tarpaulin@v0.1 - with: - args: '--out Lcov -- --test-threads 1' - - - name: Upload coverage - uses: coverallsapp/github-action@master - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - path-to-lcov: ./lcov.info - - - name: Test musl build without default features - uses: actions-rs/cargo@v1 - with: - use-cross: true - command: build - args: --target x86_64-unknown-linux-musl --no-default-features + # - name: Install stable toolchain + # uses: actions-rs/toolchain@v1 + # with: + # toolchain: stable + # override: true + + # - name: Install system dependencies + # run: | + # sudo apt-get install --yes zlib1g-dev libbz2-dev musl musl-dev musl-tools clang libc6-dev + + # - name: Run cargo-tarpaulin + # uses: actions-rs/tarpaulin@v0.1 + # with: + # args: '--out Lcov -- --test-threads 1' + + # - name: Upload coverage + # uses: coverallsapp/github-action@master + # with: + # github-token: ${{ secrets.GITHUB_TOKEN }} + # path-to-lcov: ./lcov.info + + # - name: Test musl build without default features + # uses: actions-rs/cargo@v1 + # with: + # use-cross: true + # command: build + # args: --target x86_64-unknown-linux-musl --no-default-features - name: Test musl build with all features uses: actions-rs/cargo@v1 From b90dd5102ef48270d31e2bb07ab7896f800af9c0 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Sat, 21 Mar 2020 23:27:51 +1100 Subject: [PATCH 14/56] Passing clang_arg() to bindgen to compile the wrapper with -I/usr/local/musl/x86_64-linux-musl/include does not seem to help :/ --- hts-sys/build.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 0eede273b..fad940253 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -79,8 +79,8 @@ fn main() { .current_dir(out.join("htslib")) .arg(format!("CC={}", cc_path.display())) .arg(format!("CFLAGS={}", cc_cflags)) - .arg(format!("CPPFLAGS=\"{}\"", cppflags)) - .arg(format!("LDFLAGS=\"{}\"", ldflags)) + .arg(format!("CPPFLAGS=\"{}\"", &cppflags)) + .arg(format!("LDFLAGS=\"{}\"", &ldflags)) .arg("lib-static") .arg("-B") .status() @@ -97,6 +97,7 @@ fn main() { .generate_comments(false) .blacklist_function("strtold") .blacklist_type("max_align_t") + .clang_arg(&cppflags) .generate() .expect("Unable to generate bindings.") .write_to_file(out.join("bindings.rs")) From 2da44bb1d899234b16b6fa85f48151bc3b7dba45 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Sun, 22 Mar 2020 09:43:35 +1100 Subject: [PATCH 15/56] Hardcoded include path works for both musl and GNU bindgen building, removing .cargo/config, not necessary --- .cargo/config | 12 ------------ hts-sys/build.rs | 3 ++- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/.cargo/config b/.cargo/config index f04323123..e69de29bb 100644 --- a/.cargo/config +++ b/.cargo/config @@ -1,12 +0,0 @@ -[target.x86_64-apple-darwin] -rustflags = [ - "-C", "link-arg=-undefined", - "-C", "link-arg=dynamic_lookup", -] - -[target.x86_64-unknown-linux-musl] -rustflags = [ - "-C", "link-arg=-undefined", - "-C", "link-arg=dynamic_lookup", -] -linker = "x86_64-linux-musl-gcc" \ No newline at end of file diff --git a/hts-sys/build.rs b/hts-sys/build.rs index fad940253..5287644a7 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -97,7 +97,8 @@ fn main() { .generate_comments(false) .blacklist_function("strtold") .blacklist_type("max_align_t") - .clang_arg(&cppflags) + .clang_arg("-I/usr/local/musl/x86_64-linux-musl/include") + //.clang_arg(&cppflags) .generate() .expect("Unable to generate bindings.") .write_to_file(out.join("bindings.rs")) From f7c108fb35bc759c15121762fd81822134f8218c Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Sun, 22 Mar 2020 09:53:57 +1100 Subject: [PATCH 16/56] Re-enabling formatting and linting Github action phases after green build on MUSL testing :) --- .github/workflows/rust.yml | 118 ++++++++++++++++++------------------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index dcb3e4e6c..c4ccc5e91 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -7,46 +7,46 @@ on: branches: [ master ] jobs: - # Formatting: - # runs-on: ubuntu-latest - # steps: - # - name: Checkout repository - # uses: actions/checkout@v2 - - # - name: Install stable toolchain - # uses: actions-rs/toolchain@v1 - # with: - # toolchain: stable - # override: true - # components: rustfmt + Formatting: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + components: rustfmt - # - name: Check format - # run: cargo fmt -- --check + - name: Check format + run: cargo fmt -- --check - # Linting: - # runs-on: ubuntu-latest - # steps: - # - name: Checkout repository - # uses: actions/checkout@v2 + Linting: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 - # - name: Checkout submodules - # uses: textbook/git-checkout-submodule-action@2.0.0 + - name: Checkout submodules + uses: textbook/git-checkout-submodule-action@2.0.0 - # - name: Install stable toolchain - # uses: actions-rs/toolchain@v1 - # with: - # toolchain: stable - # override: true - # components: clippy + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + components: clippy - # - name: Lint with clippy - # uses: actions-rs/clippy-check@v1 - # with: - # token: ${{ secrets.GITHUB_TOKEN }} + - name: Lint with clippy + uses: actions-rs/clippy-check@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} Testing: - # needs: Formatting + needs: Formatting runs-on: ubuntu-latest steps: - name: Checkout repository @@ -55,33 +55,33 @@ jobs: - name: Checkout submodules uses: textbook/git-checkout-submodule-action@2.0.0 - # - name: Install stable toolchain - # uses: actions-rs/toolchain@v1 - # with: - # toolchain: stable - # override: true - - # - name: Install system dependencies - # run: | - # sudo apt-get install --yes zlib1g-dev libbz2-dev musl musl-dev musl-tools clang libc6-dev - - # - name: Run cargo-tarpaulin - # uses: actions-rs/tarpaulin@v0.1 - # with: - # args: '--out Lcov -- --test-threads 1' - - # - name: Upload coverage - # uses: coverallsapp/github-action@master - # with: - # github-token: ${{ secrets.GITHUB_TOKEN }} - # path-to-lcov: ./lcov.info - - # - name: Test musl build without default features - # uses: actions-rs/cargo@v1 - # with: - # use-cross: true - # command: build - # args: --target x86_64-unknown-linux-musl --no-default-features + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + + - name: Install system dependencies + run: | + sudo apt-get install --yes zlib1g-dev libbz2-dev musl musl-dev musl-tools clang libc6-dev + + - name: Run cargo-tarpaulin + uses: actions-rs/tarpaulin@v0.1 + with: + args: '--out Lcov -- --test-threads 1' + + - name: Upload coverage + uses: coverallsapp/github-action@master + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + path-to-lcov: ./lcov.info + + - name: Test musl build without default features + uses: actions-rs/cargo@v1 + with: + use-cross: true + command: build + args: --target x86_64-unknown-linux-musl --no-default-features - name: Test musl build with all features uses: actions-rs/cargo@v1 From 85a9ef42a7513791514c0dfac46007c7acaabe0d Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Thu, 26 Mar 2020 15:06:11 +1100 Subject: [PATCH 17/56] hfile_s3.o is only included on libhts.a when ./configure is run previously according to James and Rob from the htslib team --- docker/Dockerfile.musl | 4 ++-- hts-sys/Cargo.toml | 1 + hts-sys/build.rs | 7 +++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.musl b/docker/Dockerfile.musl index 70eca1d14..d73f6622b 100644 --- a/docker/Dockerfile.musl +++ b/docker/Dockerfile.musl @@ -14,9 +14,9 @@ ENV CURL_VERSION 7.69.1 # Install basics to locally compile htslib dependencies RUN apt-get update && \ - apt-get install -y build-essential git wget + apt-get install -y build-essential autoconf automake autotools-dev git wget -# Remove pre-installed musl, to avoid cross-musl-make -j48 interference +# Remove pre-installed musl, to avoid cross-musl-make interference RUN apt-get remove -y musl # For now we'll have to go nuts and not only build musl-1.2.0 from scratch but all the other libs too... diff --git a/hts-sys/Cargo.toml b/hts-sys/Cargo.toml index 73fa34262..a96a148fe 100644 --- a/hts-sys/Cargo.toml +++ b/hts-sys/Cargo.toml @@ -23,6 +23,7 @@ bzip2-sys = { version = "0.1", optional = true } lzma-sys = { version = "0.1", optional = true } curl-sys = { version = "0.4.26", optional = true } openssl-sys = { version = "0.9.54", optional = true } +clang-sys = { version = "0.29.2" } [features] default = ["bzip2", "lzma", "curl"] diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 5287644a7..110bbd2d8 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -75,6 +75,13 @@ fn main() { let cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); let cppflags = env::var("CPPFLAGS").unwrap_or_default(); let ldflags= env::var("CPPFLAGS").unwrap_or_default(); + // This ./configure step is necessary to include the htslib plugins in the resulting libhts.a (hfile_s3.o, hfile_s3_writer.o, etc...) + if !Command::new("autoreconf").status().unwrap().success() && + !Command::new("./configure").status().unwrap().success() + { + panic!("could not configure htslib") + } + if !Command::new("make") .current_dir(out.join("htslib")) .arg(format!("CC={}", cc_path.display())) From a1d7bdf634fb36f99d9dee094ea873aeeaf83206 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Thu, 26 Mar 2020 15:15:15 +1100 Subject: [PATCH 18/56] Make sure we are in htslib directory, otherwise configure.ac and .in will not be found --- hts-sys/build.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 110bbd2d8..7320b2ba2 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -76,10 +76,10 @@ fn main() { let cppflags = env::var("CPPFLAGS").unwrap_or_default(); let ldflags= env::var("CPPFLAGS").unwrap_or_default(); // This ./configure step is necessary to include the htslib plugins in the resulting libhts.a (hfile_s3.o, hfile_s3_writer.o, etc...) - if !Command::new("autoreconf").status().unwrap().success() && - !Command::new("./configure").status().unwrap().success() + if !Command::new("autoreconf").current_dir(out.join("htslib")).status().unwrap().success() && + !Command::new("./configure").current_dir(out.join("htslib")).status().unwrap().success() { - panic!("could not configure htslib") + panic!("could not configure htslib nor any of its plugins") } if !Command::new("make") From a738213be967ccdf43e638546b9ae290551b0977 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Thu, 26 Mar 2020 15:35:32 +1100 Subject: [PATCH 19/56] Fix up redundant imports according to clippy, switch to alpine linux container, way more lightweight and less messy build process, also includes MUSL as base compiler --- Cross.toml | 2 +- docker/Dockerfile.alpine | 11 +++++++++++ hts-sys/build.rs | 2 -- 3 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 docker/Dockerfile.alpine diff --git a/Cross.toml b/Cross.toml index 8e3f98410..1d15b39e1 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,2 +1,2 @@ [target.x86_64-unknown-linux-musl] -image = "brainstorm/rust-htslib-musl:latest" +image = "brainstorm/rust-htslib-musl-alpine:latest" diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine new file mode 100644 index 000000000..35ceb1802 --- /dev/null +++ b/docker/Dockerfile.alpine @@ -0,0 +1,11 @@ +FROM alpine:edge + +RUN apk add --update wget git g++ build-base automake autoconf openssl openssl-dev curl-dev bzip2-dev xz-dev zlib-dev rustup && \ + rm -rf /var/cache/apk/* + +WORKDIR /root +#RUN git clone --recursive https://github.com/brainstorm/rust-htslib +RUN rustup-init -y + +ENV PATH "/root/.cargo/bin:$PATH" +RUN rustup target add x86_64-unknown-linux-musl \ No newline at end of file diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 7320b2ba2..b4059af86 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -3,8 +3,6 @@ // This file may not be copied, modified, or distributed // except according to those terms. -use bindgen; -use cc; use fs_utils::copy::copy_directory; use glob::glob; From 4e66846e47a7811a533c43379086b079b19c1250 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Thu, 26 Mar 2020 16:42:14 +1100 Subject: [PATCH 20/56] Explicitly cross-compile passing the right ./configure --host flag, to be generalized later --- hts-sys/build.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hts-sys/build.rs b/hts-sys/build.rs index b4059af86..3dc82ea17 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -75,7 +75,10 @@ fn main() { let ldflags= env::var("CPPFLAGS").unwrap_or_default(); // This ./configure step is necessary to include the htslib plugins in the resulting libhts.a (hfile_s3.o, hfile_s3_writer.o, etc...) if !Command::new("autoreconf").current_dir(out.join("htslib")).status().unwrap().success() && - !Command::new("./configure").current_dir(out.join("htslib")).status().unwrap().success() + !Command::new("./configure") + .current_dir(out.join("htslib")) + .arg(format!("--host=x86_64-unknown-linux-musl")) + .status().unwrap().success() { panic!("could not configure htslib nor any of its plugins") } @@ -102,7 +105,7 @@ fn main() { .generate_comments(false) .blacklist_function("strtold") .blacklist_type("max_align_t") - .clang_arg("-I/usr/local/musl/x86_64-linux-musl/include") + //.clang_arg("-I/usr/local/musl/x86_64-linux-musl/include") //.clang_arg(&cppflags) .generate() .expect("Unable to generate bindings.") From 7c7e2831292ca104ecb9d68473f3512b00f570da Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Mon, 30 Mar 2020 16:52:36 +1100 Subject: [PATCH 21/56] LDFLAGS was being pointed to CPPFLAGS, copy&paste hell. Also make cargo clippy happy --- hts-sys/Cargo.toml | 2 +- hts-sys/build.rs | 5 ++--- src/bam/mod.rs | 2 ++ src/bcf/mod.rs | 3 ++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/hts-sys/Cargo.toml b/hts-sys/Cargo.toml index a96a148fe..cef4f7b55 100644 --- a/hts-sys/Cargo.toml +++ b/hts-sys/Cargo.toml @@ -21,7 +21,7 @@ libc = "0.2" libz-sys = "1.0" bzip2-sys = { version = "0.1", optional = true } lzma-sys = { version = "0.1", optional = true } -curl-sys = { version = "0.4.26", optional = true } +curl-sys = { version = "0.4.27", optional = true } openssl-sys = { version = "0.9.54", optional = true } clang-sys = { version = "0.29.2" } diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 3dc82ea17..d99a42705 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -72,11 +72,10 @@ fn main() { let (cc_path, cflags_env) = (tool.path(), tool.cflags_env()); let cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); let cppflags = env::var("CPPFLAGS").unwrap_or_default(); - let ldflags= env::var("CPPFLAGS").unwrap_or_default(); + let ldflags= env::var("LDFLAGS").unwrap_or_default(); // This ./configure step is necessary to include the htslib plugins in the resulting libhts.a (hfile_s3.o, hfile_s3_writer.o, etc...) if !Command::new("autoreconf").current_dir(out.join("htslib")).status().unwrap().success() && - !Command::new("./configure") - .current_dir(out.join("htslib")) + !Command::new("./configure").current_dir(out.join("htslib")) .arg(format!("--host=x86_64-unknown-linux-musl")) .status().unwrap().success() { diff --git a/src/bam/mod.rs b/src/bam/mod.rs index bd371c8f2..cf32882b9 100644 --- a/src/bam/mod.rs +++ b/src/bam/mod.rs @@ -41,6 +41,8 @@ pub unsafe fn set_threads(htsfile: *mut htslib::htsFile, n_threads: usize) -> Re } } +/// # Safety +/// /// Set the reference FAI index path in a `htslib::htsFile` struct for reading CRAM format. pub unsafe fn set_fai_filename>( htsfile: *mut htslib::htsFile, diff --git a/src/bcf/mod.rs b/src/bcf/mod.rs index b72cd1721..da002048f 100644 --- a/src/bcf/mod.rs +++ b/src/bcf/mod.rs @@ -73,7 +73,8 @@ pub struct Reader { } unsafe impl Send for Reader {} - +/// # Safety +/// /// Implementation for `Reader::set_threads()` and `Writer::set_threads`. pub unsafe fn set_threads(hts_file: *mut htslib::htsFile, n_threads: usize) -> Result<()> { assert!(n_threads > 0, "n_threads must be > 0"); From fe0e8a0877fab379cba24d04bb23c296fce42772 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Mon, 30 Mar 2020 17:00:03 +1100 Subject: [PATCH 22/56] More clippy --- src/bam/mod.rs | 4 +++- src/tbx/mod.rs | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/bam/mod.rs b/src/bam/mod.rs index cf32882b9..3f710ae07 100644 --- a/src/bam/mod.rs +++ b/src/bam/mod.rs @@ -30,6 +30,8 @@ pub use crate::bam::header::Header; pub use crate::bam::record::Record; use std::convert::TryInto; +/// # Safety +/// /// Implementation for `Read::set_threads` and `Writer::set_threads`. pub unsafe fn set_threads(htsfile: *mut htslib::htsFile, n_threads: usize) -> Result<()> { assert!(n_threads != 0, "n_threads must be > 0"); @@ -42,7 +44,7 @@ pub unsafe fn set_threads(htsfile: *mut htslib::htsFile, n_threads: usize) -> Re } /// # Safety -/// +/// /// Set the reference FAI index path in a `htslib::htsFile` struct for reading CRAM format. pub unsafe fn set_fai_filename>( htsfile: *mut htslib::htsFile, diff --git a/src/tbx/mod.rs b/src/tbx/mod.rs index ee2792b51..0d1efcf50 100644 --- a/src/tbx/mod.rs +++ b/src/tbx/mod.rs @@ -41,7 +41,6 @@ //! } //! ``` -use libc; use std::ffi; use std::path::Path; use std::ptr; From ed55bb0a9ba66ea9f8bfda7fe0caf2efa6dbfd80 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Mon, 30 Mar 2020 17:07:12 +1100 Subject: [PATCH 23/56] Puzzled as of why this clippy error was not detected locally... also htslib compiles fine locally as well, no cross, just cargo --- hts-sys/build.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hts-sys/build.rs b/hts-sys/build.rs index d99a42705..53dfde526 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -73,10 +73,11 @@ fn main() { let cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); let cppflags = env::var("CPPFLAGS").unwrap_or_default(); let ldflags= env::var("LDFLAGS").unwrap_or_default(); + let host = env::var("HOST").unwrap_or_default(); // This ./configure step is necessary to include the htslib plugins in the resulting libhts.a (hfile_s3.o, hfile_s3_writer.o, etc...) if !Command::new("autoreconf").current_dir(out.join("htslib")).status().unwrap().success() && !Command::new("./configure").current_dir(out.join("htslib")) - .arg(format!("--host=x86_64-unknown-linux-musl")) + .arg(format!("--host={}", &host)) .status().unwrap().success() { panic!("could not configure htslib nor any of its plugins") From b79981f96cf916494eb5475d34dc771a29ee8331 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Mon, 30 Mar 2020 17:16:28 +1100 Subject: [PATCH 24/56] CPPFLAGS as an argument to make is causing trouble on Linux but not on OSX... --- hts-sys/build.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 53dfde526..8f6c7b622 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -71,6 +71,7 @@ fn main() { let tool = cfg.get_compiler(); let (cc_path, cflags_env) = (tool.path(), tool.cflags_env()); let cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); + // Some other flags which can be critical for cross-compiling to targets like MUSL let cppflags = env::var("CPPFLAGS").unwrap_or_default(); let ldflags= env::var("LDFLAGS").unwrap_or_default(); let host = env::var("HOST").unwrap_or_default(); @@ -87,7 +88,7 @@ fn main() { .current_dir(out.join("htslib")) .arg(format!("CC={}", cc_path.display())) .arg(format!("CFLAGS={}", cc_cflags)) - .arg(format!("CPPFLAGS=\"{}\"", &cppflags)) + //.arg(format!("CPPFLAGS=\"{}\"", &cppflags)) .arg(format!("LDFLAGS=\"{}\"", &ldflags)) .arg("lib-static") .arg("-B") From 03d9df4d0344aba509e90cab2a568c7c64a8640c Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Mon, 30 Mar 2020 18:53:55 +1100 Subject: [PATCH 25/56] This include does not affect any build and it is crucial for MUSL ones --- hts-sys/build.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 8f6c7b622..fb523a221 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -106,8 +106,7 @@ fn main() { .generate_comments(false) .blacklist_function("strtold") .blacklist_type("max_align_t") - //.clang_arg("-I/usr/local/musl/x86_64-linux-musl/include") - //.clang_arg(&cppflags) + .clang_arg("-I/usr/local/musl/x86_64-linux-musl/include") .generate() .expect("Unable to generate bindings.") .write_to_file(out.join("bindings.rs")) From bb677c8807f4178bfc666418e5748fe0dbefb3e5 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Mon, 30 Mar 2020 20:14:53 +1100 Subject: [PATCH 26/56] Back to non-alpine container, not sure why cargo is not being found --- Cross.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cross.toml b/Cross.toml index 1d15b39e1..8e3f98410 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,2 +1,2 @@ [target.x86_64-unknown-linux-musl] -image = "brainstorm/rust-htslib-musl-alpine:latest" +image = "brainstorm/rust-htslib-musl:latest" From 038b87b5272b90d7c8736e5e202e5dd1c7a8e734 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Tue, 31 Mar 2020 10:53:39 +1100 Subject: [PATCH 27/56] Disabling --no-default-features building for now, clang-sys not needed, suppress cppflags warnings --- .github/workflows/rust.yml | 12 ++++++------ hts-sys/Cargo.toml | 1 - hts-sys/build.rs | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index c4ccc5e91..c648c1e84 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -76,12 +76,12 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} path-to-lcov: ./lcov.info - - name: Test musl build without default features - uses: actions-rs/cargo@v1 - with: - use-cross: true - command: build - args: --target x86_64-unknown-linux-musl --no-default-features +# - name: Test musl build without default features +# uses: actions-rs/cargo@v1 +# with: +# use-cross: true +# command: build +# args: --target x86_64-unknown-linux-musl --no-default-features - name: Test musl build with all features uses: actions-rs/cargo@v1 diff --git a/hts-sys/Cargo.toml b/hts-sys/Cargo.toml index cef4f7b55..fce34c089 100644 --- a/hts-sys/Cargo.toml +++ b/hts-sys/Cargo.toml @@ -23,7 +23,6 @@ bzip2-sys = { version = "0.1", optional = true } lzma-sys = { version = "0.1", optional = true } curl-sys = { version = "0.4.27", optional = true } openssl-sys = { version = "0.9.54", optional = true } -clang-sys = { version = "0.29.2" } [features] default = ["bzip2", "lzma", "curl"] diff --git a/hts-sys/build.rs b/hts-sys/build.rs index fb523a221..0070e3347 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -72,7 +72,7 @@ fn main() { let (cc_path, cflags_env) = (tool.path(), tool.cflags_env()); let cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); // Some other flags which can be critical for cross-compiling to targets like MUSL - let cppflags = env::var("CPPFLAGS").unwrap_or_default(); + //let cppflags = env::var("CPPFLAGS").unwrap_or_default(); let ldflags= env::var("LDFLAGS").unwrap_or_default(); let host = env::var("HOST").unwrap_or_default(); // This ./configure step is necessary to include the htslib plugins in the resulting libhts.a (hfile_s3.o, hfile_s3_writer.o, etc...) From 7ac72544c662006da0debd82848882f3407fbe95 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Tue, 7 Apr 2020 09:55:33 +1000 Subject: [PATCH 28/56] Following @pmarks advice on libz-sys, bump up both libc anb libz-sys while at it --- hts-sys/Cargo.toml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hts-sys/Cargo.toml b/hts-sys/Cargo.toml index fce34c089..e0a037d8e 100644 --- a/hts-sys/Cargo.toml +++ b/hts-sys/Cargo.toml @@ -17,15 +17,16 @@ pre-release-commit-message = "release version {{version}}" tag-message = "Version {{version}} of Rust-HTSlib." [dependencies] -libc = "0.2" -libz-sys = "1.0" +libc = "0.2.68" +libz-sys = { version = "1.0.25", optional = true } bzip2-sys = { version = "0.1", optional = true } lzma-sys = { version = "0.1", optional = true } curl-sys = { version = "0.4.27", optional = true } openssl-sys = { version = "0.9.54", optional = true } [features] -default = ["bzip2", "lzma", "curl"] +default = ["bzip2", "lzma", "curl", "libz"] +libz = ["libz-sys"] bzip2 = ["bzip2-sys"] lzma = ["lzma-sys"] openssl = ["openssl-sys"] From 3c8433a005f2dd0b50eceda204184a438003bc1d Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Tue, 7 Apr 2020 09:57:11 +1000 Subject: [PATCH 29/56] Enable no-default-features back --- .github/workflows/rust.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index c648c1e84..c4ccc5e91 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -76,12 +76,12 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} path-to-lcov: ./lcov.info -# - name: Test musl build without default features -# uses: actions-rs/cargo@v1 -# with: -# use-cross: true -# command: build -# args: --target x86_64-unknown-linux-musl --no-default-features + - name: Test musl build without default features + uses: actions-rs/cargo@v1 + with: + use-cross: true + command: build + args: --target x86_64-unknown-linux-musl --no-default-features - name: Test musl build with all features uses: actions-rs/cargo@v1 From 92f99ff38f9091c25a28f64e8aeef139cab5ba99 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Tue, 7 Apr 2020 10:13:48 +1000 Subject: [PATCH 30/56] Make clippy happy with libz --- hts-sys/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hts-sys/src/lib.rs b/hts-sys/src/lib.rs index de6108884..c0774ffe3 100644 --- a/hts-sys/src/lib.rs +++ b/hts-sys/src/lib.rs @@ -5,8 +5,8 @@ #![allow(improper_ctypes)] //! This module exposes the raw Htslib bindings. +#[cfg(feature = "libz")] extern crate libz_sys; - #[cfg(feature = "bzip2")] extern crate bzip2_sys; #[cfg(feature = "lzma")] From fed68ab5a6cd3eca428d06413299e2937e565607 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Tue, 7 Apr 2020 10:25:25 +1000 Subject: [PATCH 31/56] libz-sys should not be optional, thanks @pmarks --- hts-sys/Cargo.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hts-sys/Cargo.toml b/hts-sys/Cargo.toml index e0a037d8e..467cc5d45 100644 --- a/hts-sys/Cargo.toml +++ b/hts-sys/Cargo.toml @@ -18,15 +18,14 @@ tag-message = "Version {{version}} of Rust-HTSlib." [dependencies] libc = "0.2.68" -libz-sys = { version = "1.0.25", optional = true } +libz-sys = { version = "1.0.25", optional = false } bzip2-sys = { version = "0.1", optional = true } lzma-sys = { version = "0.1", optional = true } curl-sys = { version = "0.4.27", optional = true } openssl-sys = { version = "0.9.54", optional = true } [features] -default = ["bzip2", "lzma", "curl", "libz"] -libz = ["libz-sys"] +default = ["bzip2", "lzma"] bzip2 = ["bzip2-sys"] lzma = ["lzma-sys"] openssl = ["openssl-sys"] From dd216639809551b267b02ca46d4d03192a0b9f9f Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Tue, 12 May 2020 12:08:01 +1000 Subject: [PATCH 32/56] Cleaning up flags, trying to have libz as static to avoid //usr/lib64/libz.so.1: error adding symbols: DSO missing from command line, linking errors, thansk @wezm for the tips --- hts-sys/Cargo.toml | 2 +- hts-sys/build.rs | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/hts-sys/Cargo.toml b/hts-sys/Cargo.toml index 467cc5d45..aed3444e0 100644 --- a/hts-sys/Cargo.toml +++ b/hts-sys/Cargo.toml @@ -18,7 +18,7 @@ tag-message = "Version {{version}} of Rust-HTSlib." [dependencies] libc = "0.2.68" -libz-sys = { version = "1.0.25", optional = false } +libz-sys = { version = "1.0.25", optional = false, features=["static"] } bzip2-sys = { version = "0.1", optional = true } lzma-sys = { version = "0.1", optional = true } curl-sys = { version = "0.4.27", optional = true } diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 0070e3347..85f984a36 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -70,10 +70,10 @@ fn main() { let tool = cfg.get_compiler(); let (cc_path, cflags_env) = (tool.path(), tool.cflags_env()); - let cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); + let _cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); // Some other flags which can be critical for cross-compiling to targets like MUSL //let cppflags = env::var("CPPFLAGS").unwrap_or_default(); - let ldflags= env::var("LDFLAGS").unwrap_or_default(); + let _ldflags= env::var("LDFLAGS").unwrap_or_default(); let host = env::var("HOST").unwrap_or_default(); // This ./configure step is necessary to include the htslib plugins in the resulting libhts.a (hfile_s3.o, hfile_s3_writer.o, etc...) if !Command::new("autoreconf").current_dir(out.join("htslib")).status().unwrap().success() && @@ -87,9 +87,9 @@ fn main() { if !Command::new("make") .current_dir(out.join("htslib")) .arg(format!("CC={}", cc_path.display())) - .arg(format!("CFLAGS={}", cc_cflags)) + //.arg(format!("CFLAGS={}", cc_cflags)) //.arg(format!("CPPFLAGS=\"{}\"", &cppflags)) - .arg(format!("LDFLAGS=\"{}\"", &ldflags)) + //.arg(format!("LDFLAGS=\"{}\"", &ldflags)) .arg("lib-static") .arg("-B") .status() @@ -106,7 +106,6 @@ fn main() { .generate_comments(false) .blacklist_function("strtold") .blacklist_type("max_align_t") - .clang_arg("-I/usr/local/musl/x86_64-linux-musl/include") .generate() .expect("Unable to generate bindings.") .write_to_file(out.join("bindings.rs")) From 3241e87f7bfd145559334560679166d6714edaa9 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Tue, 12 May 2020 17:04:19 +1000 Subject: [PATCH 33/56] Figured out next steps: need to incorporate static compilation features to both bzip2-sys and lzma-sys, otherwise MUSL will have DSO linker issues. Will implement https://github.com/alexcrichton/xz2-rs/issues/62 and https://github.com/alexcrichton/bzip2-rs/issues/56 next --- hts-sys/Cargo.toml | 12 +++++++----- hts-sys/build.rs | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/hts-sys/Cargo.toml b/hts-sys/Cargo.toml index aed3444e0..a059ecde4 100644 --- a/hts-sys/Cargo.toml +++ b/hts-sys/Cargo.toml @@ -17,11 +17,13 @@ pre-release-commit-message = "release version {{version}}" tag-message = "Version {{version}} of Rust-HTSlib." [dependencies] -libc = "0.2.68" -libz-sys = { version = "1.0.25", optional = false, features=["static"] } -bzip2-sys = { version = "0.1", optional = true } -lzma-sys = { version = "0.1", optional = true } -curl-sys = { version = "0.4.27", optional = true } +#libc = "0.2.68" +libz-sys = { version = "1.0.25", features = ["static"] } +# https://github.com/alexcrichton/bzip2-rs/issues/56 +bzip2-sys = { version = "0.1.8", optional = true } +# https://github.com/alexcrichton/xz2-rs/issues/62 +lzma-sys = { version = "0.1.15", optional = true } +curl-sys = { version = "0.4.27", optional = true, features = ["static-curl", "static-ssl"] } openssl-sys = { version = "0.9.54", optional = true } [features] diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 85f984a36..13ffb49e4 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -123,6 +123,7 @@ fn main() { println!("cargo:include={}", include.display()); println!("cargo:libdir={}", out.display()); println!("cargo:rustc-link-lib=static=hts"); + println!("cargo:rustc-link-lib=static=z"); println!("cargo:rerun-if-changed=wrapper.c"); println!("cargo:rerun-if-changed=wrapper.h"); for htsfile in glob("htslib/**/*").unwrap() { From d8d1c5a8dddf01bb5aba24468bbc444f7035155b Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Thu, 14 May 2020 15:06:57 +1000 Subject: [PATCH 34/56] Add missing static flag to lzma-sys, getting rid of the need to export the env variable, closing into the core of the issue: rust-htslib's build.rs needs a feature-flag revamp as well... in addition to env variables, it should be able to use cfg Cargo features as well... and not sed-modify htslib's makefile if it's a static build --- Cargo.toml | 3 ++- hts-sys/Cargo.toml | 3 ++- hts-sys/build.rs | 15 ++++++++++++--- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a265ce52f..34d65cd5a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ tag-message = "Version {{version}} of Rust-HTSlib." [dependencies] libc = "0.2" -itertools = "0.8" +itertools = "0.9.0" newtype_derive = "0.1" custom_derive = "0.1" url = "2.1" @@ -39,6 +39,7 @@ lzma = ["hts-sys/lzma"] curl = ["hts-sys/curl"] openssl = ["hts-sys/openssl"] serde = ["serde_base", "serde_bytes"] +static = [] [dev-dependencies] tempdir = "0.3" diff --git a/hts-sys/Cargo.toml b/hts-sys/Cargo.toml index a059ecde4..2d1474915 100644 --- a/hts-sys/Cargo.toml +++ b/hts-sys/Cargo.toml @@ -22,7 +22,7 @@ libz-sys = { version = "1.0.25", features = ["static"] } # https://github.com/alexcrichton/bzip2-rs/issues/56 bzip2-sys = { version = "0.1.8", optional = true } # https://github.com/alexcrichton/xz2-rs/issues/62 -lzma-sys = { version = "0.1.15", optional = true } +lzma-sys = { version = "0.1.15", optional = true, features = ["static"], git = "https://github.com/brainstorm/xz2-rs", branch = "static_feature_flag" } curl-sys = { version = "0.4.27", optional = true, features = ["static-curl", "static-ssl"] } openssl-sys = { version = "0.9.54", optional = true } @@ -32,6 +32,7 @@ bzip2 = ["bzip2-sys"] lzma = ["lzma-sys"] openssl = ["openssl-sys"] curl = ["curl-sys"] +static = [] [build-dependencies] fs-utils = "1.1" diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 13ffb49e4..d048f6830 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -31,10 +31,18 @@ fn sed_htslib_makefile(out: &PathBuf, patterns: &[&str], feature: &str) { fn main() { let out = PathBuf::from(env::var("OUT_DIR").unwrap()); let mut cfg = cc::Build::new(); - cfg.warnings(false).static_flag(true).pic(true); + let want_static = cfg!(feature = "static") || env::var("HTS_STATIC").is_ok(); + + if want_static { + cfg.warnings(true).static_flag(true).pic(true); + } else { + cfg.warnings(true).static_flag(false).pic(true); + } if let Ok(z_inc) = env::var("DEP_Z_INCLUDE") { - cfg.include(z_inc); + if !want_static { + cfg.include(z_inc); + } } if !out.join("htslib").exists() { @@ -53,7 +61,7 @@ fn main() { } let use_lzma = env::var("CARGO_FEATURE_LZMA").is_ok(); - if !use_lzma { + if !use_lzma && want_static { let lzma_patterns = vec!["s/ -llzma//", "/#define HAVE_LIBLZMA/d"]; sed_htslib_makefile(&out, &lzma_patterns, "lzma"); } else if let Ok(inc) = env::var("DEP_LZMA_INCLUDE").map(PathBuf::from) { @@ -124,6 +132,7 @@ fn main() { println!("cargo:libdir={}", out.display()); println!("cargo:rustc-link-lib=static=hts"); println!("cargo:rustc-link-lib=static=z"); + println!("cargo:rustc-link-lib=static=lzma"); println!("cargo:rerun-if-changed=wrapper.c"); println!("cargo:rerun-if-changed=wrapper.h"); for htsfile in glob("htslib/**/*").unwrap() { From 52ef91157392197fba7be5df9496eced1c33c47d Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Thu, 14 May 2020 17:12:15 +1000 Subject: [PATCH 35/56] Remove openssl-sys crate in favor of curl-sys, which already includes a vendored openssl-sys (hopefully more static-friendly) --- Cargo.toml | 4 ++-- hts-sys/Cargo.toml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 34d65cd5a..c6e0d0677 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,7 +29,7 @@ linear-map = "1.2" serde_base = { version = "^1", optional = true, package = "serde" } serde_bytes = { version = "0.11", optional = true } bio-types = ">=0.6" -snafu = ">= 0.5.0, <= 0.6.0" +snafu = "0.6.8" hts-sys = { version = "^1.10", path = "hts-sys", default-features = false } [features] @@ -37,7 +37,7 @@ default = ["bzip2", "lzma", "curl"] bzip2 = ["hts-sys/bzip2"] lzma = ["hts-sys/lzma"] curl = ["hts-sys/curl"] -openssl = ["hts-sys/openssl"] +#openssl = ["hts-sys/openssl"] serde = ["serde_base", "serde_bytes"] static = [] diff --git a/hts-sys/Cargo.toml b/hts-sys/Cargo.toml index 2d1474915..2be25fc0d 100644 --- a/hts-sys/Cargo.toml +++ b/hts-sys/Cargo.toml @@ -23,14 +23,14 @@ libz-sys = { version = "1.0.25", features = ["static"] } bzip2-sys = { version = "0.1.8", optional = true } # https://github.com/alexcrichton/xz2-rs/issues/62 lzma-sys = { version = "0.1.15", optional = true, features = ["static"], git = "https://github.com/brainstorm/xz2-rs", branch = "static_feature_flag" } -curl-sys = { version = "0.4.27", optional = true, features = ["static-curl", "static-ssl"] } -openssl-sys = { version = "0.9.54", optional = true } +curl-sys = { version = "0.4.31+curl-7.70.0", optional = true, features = ["static-curl", "static-ssl"] } +#openssl-sys = { version = "0.9.56", optional = true, vendored = true} [features] default = ["bzip2", "lzma"] bzip2 = ["bzip2-sys"] lzma = ["lzma-sys"] -openssl = ["openssl-sys"] +#openssl = ["openssl-sys"] curl = ["curl-sys"] static = [] From c767e0e1a9adb3fd5296dbc7616bbc4fafccc889 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Thu, 14 May 2020 17:13:53 +1000 Subject: [PATCH 36/56] Temporarily comment out sed modifications against htslib's makefile, focusing on static compilation --- hts-sys/build.rs | 99 +++++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 48 deletions(-) diff --git a/hts-sys/build.rs b/hts-sys/build.rs index d048f6830..94ae8ae41 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -11,22 +11,22 @@ use std::fs; use std::path::PathBuf; use std::process::Command; -fn sed_htslib_makefile(out: &PathBuf, patterns: &[&str], feature: &str) { - for pattern in patterns { - if !Command::new("sed") - .current_dir(out.join("htslib")) - .arg("-i") - .arg("-e") - .arg(pattern) - .arg("Makefile") - .status() - .unwrap() - .success() - { - panic!("failed to strip {} support", feature); - } - } -} +// fn sed_htslib_makefile(out: &PathBuf, patterns: &[&str], feature: &str) { +// for pattern in patterns { +// if !Command::new("sed") +// .current_dir(out.join("htslib")) +// .arg("-i") +// .arg("-e") +// .arg(pattern) +// .arg("Makefile") +// .status() +// .unwrap() +// .success() +// { +// panic!("failed to strip {} support", feature); +// } +// } +// } fn main() { let out = PathBuf::from(env::var("OUT_DIR").unwrap()); @@ -39,42 +39,42 @@ fn main() { cfg.warnings(true).static_flag(false).pic(true); } - if let Ok(z_inc) = env::var("DEP_Z_INCLUDE") { - if !want_static { - cfg.include(z_inc); - } - } + // if let Ok(z_inc) = env::var("DEP_Z_INCLUDE") { + // if !want_static { + // cfg.include(z_inc); + // } + // } if !out.join("htslib").exists() { copy_directory("htslib", &out).unwrap(); } - let use_bzip2 = env::var("CARGO_FEATURE_BZIP2").is_ok(); - if !use_bzip2 { - let bzip2_patterns = vec!["s/ -lbz2//", "/#define HAVE_LIBBZ2/d"]; - sed_htslib_makefile(&out, &bzip2_patterns, "bzip2"); - } else if let Ok(inc) = env::var("DEP_BZIP2_ROOT") - .map(PathBuf::from) - .map(|path| path.join("include")) - { - cfg.include(inc); - } - - let use_lzma = env::var("CARGO_FEATURE_LZMA").is_ok(); - if !use_lzma && want_static { - let lzma_patterns = vec!["s/ -llzma//", "/#define HAVE_LIBLZMA/d"]; - sed_htslib_makefile(&out, &lzma_patterns, "lzma"); - } else if let Ok(inc) = env::var("DEP_LZMA_INCLUDE").map(PathBuf::from) { - cfg.include(inc); - } - - let use_curl = env::var("CARGO_FEATURE_CURL").is_ok(); - if !use_curl { - let curl_patterns = vec!["s/ -lcurl//", "/#define HAVE_LIBCURL/d"]; - sed_htslib_makefile(&out, &curl_patterns, "curl"); - } else if let Ok(inc) = env::var("DEP_CURL_INCLUDE").map(PathBuf::from) { - cfg.include(inc); - } + // let use_bzip2 = env::var("CARGO_FEATURE_BZIP2").is_ok(); + // if !use_bzip2 { + // let bzip2_patterns = vec!["s/ -lbz2//", "/#define HAVE_LIBBZ2/d"]; + // sed_htslib_makefile(&out, &bzip2_patterns, "bzip2"); + // } else if let Ok(inc) = env::var("DEP_BZIP2_ROOT") + // .map(PathBuf::from) + // .map(|path| path.join("include")) + // { + // cfg.include(inc); + // } + + // let use_lzma = env::var("CARGO_FEATURE_LZMA").is_ok(); + // if !use_lzma && want_static { + // let lzma_patterns = vec!["s/ -llzma//", "/#define HAVE_LIBLZMA/d"]; + // sed_htslib_makefile(&out, &lzma_patterns, "lzma"); + // } else if let Ok(inc) = env::var("DEP_LZMA_INCLUDE").map(PathBuf::from) { + // cfg.include(inc); + // } + + // let use_curl = env::var("CARGO_FEATURE_CURL").is_ok(); + // if !use_curl { + // let curl_patterns = vec!["s/ -lcurl//", "/#define HAVE_LIBCURL/d"]; + // sed_htslib_makefile(&out, &curl_patterns, "curl"); + // } else if let Ok(inc) = env::var("DEP_CURL_INCLUDE").map(PathBuf::from) { + // cfg.include(inc); + // } let tool = cfg.get_compiler(); let (cc_path, cflags_env) = (tool.path(), tool.cflags_env()); @@ -83,7 +83,7 @@ fn main() { //let cppflags = env::var("CPPFLAGS").unwrap_or_default(); let _ldflags= env::var("LDFLAGS").unwrap_or_default(); let host = env::var("HOST").unwrap_or_default(); - // This ./configure step is necessary to include the htslib plugins in the resulting libhts.a (hfile_s3.o, hfile_s3_writer.o, etc...) + // Those two steps are necessary to include the htslib plugins in the resulting libhts.a (hfile_s3.o, hfile_s3_writer.o, etc...) if !Command::new("autoreconf").current_dir(out.join("htslib")).status().unwrap().success() && !Command::new("./configure").current_dir(out.join("htslib")) .arg(format!("--host={}", &host)) @@ -133,6 +133,9 @@ fn main() { println!("cargo:rustc-link-lib=static=hts"); println!("cargo:rustc-link-lib=static=z"); println!("cargo:rustc-link-lib=static=lzma"); + // println!("cargo:rustc-link-lib=static=ssl"); + // println!("cargo:rustc-link-lib=static=crypto"); + println!("cargo:rustc-link-lib=static=curl"); println!("cargo:rerun-if-changed=wrapper.c"); println!("cargo:rerun-if-changed=wrapper.h"); for htsfile in glob("htslib/**/*").unwrap() { From c88c0689cc16f66d5ebcc5f493cf884b65d0bce6 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Fri, 15 May 2020 15:38:02 +1000 Subject: [PATCH 37/56] Semver + suffix on curl is not needed --- hts-sys/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hts-sys/Cargo.toml b/hts-sys/Cargo.toml index 2be25fc0d..9ce0e185f 100644 --- a/hts-sys/Cargo.toml +++ b/hts-sys/Cargo.toml @@ -23,7 +23,7 @@ libz-sys = { version = "1.0.25", features = ["static"] } bzip2-sys = { version = "0.1.8", optional = true } # https://github.com/alexcrichton/xz2-rs/issues/62 lzma-sys = { version = "0.1.15", optional = true, features = ["static"], git = "https://github.com/brainstorm/xz2-rs", branch = "static_feature_flag" } -curl-sys = { version = "0.4.31+curl-7.70.0", optional = true, features = ["static-curl", "static-ssl"] } +curl-sys = { version = "0.4.31", optional = true, features = ["static-curl", "static-ssl"] } #openssl-sys = { version = "0.9.56", optional = true, vendored = true} [features] From 7e6ca8d17de74fe2f2baade9258cbe76db391568 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Fri, 15 May 2020 15:39:25 +1000 Subject: [PATCH 38/56] make -B, that flag was the current culprit on static compilation, bizarre since it does not seem to be related at all to the libssl errors: https://www.gnu.org/software/make/manual/html_node/Options-Summary.html --- hts-sys/build.rs | 51 ++++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 94ae8ae41..b9c22b8e1 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -11,22 +11,23 @@ use std::fs; use std::path::PathBuf; use std::process::Command; -// fn sed_htslib_makefile(out: &PathBuf, patterns: &[&str], feature: &str) { -// for pattern in patterns { -// if !Command::new("sed") -// .current_dir(out.join("htslib")) -// .arg("-i") -// .arg("-e") -// .arg(pattern) -// .arg("Makefile") -// .status() -// .unwrap() -// .success() -// { -// panic!("failed to strip {} support", feature); -// } -// } -// } +fn sed_htslib_makefile(out: &PathBuf, patterns: &[&str], feature: &str) { + println!("SUBSTITUTING THINGS!"); + for pattern in patterns { + if !Command::new("sed") + .current_dir(out.join("htslib")) + .arg("-i") + .arg("-e") + .arg(pattern) + .arg("Makefile") + .status() + .unwrap() + .success() + { + panic!("failed to strip {} support", feature); + } + } +} fn main() { let out = PathBuf::from(env::var("OUT_DIR").unwrap()); @@ -48,6 +49,10 @@ fn main() { if !out.join("htslib").exists() { copy_directory("htslib", &out).unwrap(); } + + // Nice flags to have + let cflags_patterns = vec!["s/-g -Wall -O2 -fvisibility=hidden/-g -Wall -O2 -fvisibility=hidden -fPIC -pedantic -std=c90 -D_XOPEN_SOURCE=600/"]; + sed_htslib_makefile(&out, &cflags_patterns, "cflags"); // let use_bzip2 = env::var("CARGO_FEATURE_BZIP2").is_ok(); // if !use_bzip2 { @@ -78,7 +83,7 @@ fn main() { let tool = cfg.get_compiler(); let (cc_path, cflags_env) = (tool.path(), tool.cflags_env()); - let _cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); + let cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); // Some other flags which can be critical for cross-compiling to targets like MUSL //let cppflags = env::var("CPPFLAGS").unwrap_or_default(); let _ldflags= env::var("LDFLAGS").unwrap_or_default(); @@ -95,11 +100,11 @@ fn main() { if !Command::new("make") .current_dir(out.join("htslib")) .arg(format!("CC={}", cc_path.display())) - //.arg(format!("CFLAGS={}", cc_cflags)) + .arg(format!("CFLAGS={}", cc_cflags)) //.arg(format!("CPPFLAGS=\"{}\"", &cppflags)) //.arg(format!("LDFLAGS=\"{}\"", &ldflags)) .arg("lib-static") - .arg("-B") + //.arg("-B") .status() .unwrap() .success() @@ -130,12 +135,12 @@ fn main() { println!("cargo:root={}", out.display()); println!("cargo:include={}", include.display()); println!("cargo:libdir={}", out.display()); - println!("cargo:rustc-link-lib=static=hts"); - println!("cargo:rustc-link-lib=static=z"); - println!("cargo:rustc-link-lib=static=lzma"); + // println!("cargo:rustc-link-lib=static=hts"); + // println!("cargo:rustc-link-lib=static=z"); + // println!("cargo:rustc-link-lib=static=lzma"); // println!("cargo:rustc-link-lib=static=ssl"); // println!("cargo:rustc-link-lib=static=crypto"); - println!("cargo:rustc-link-lib=static=curl"); + // println!("cargo:rustc-link-lib=static=curl"); println!("cargo:rerun-if-changed=wrapper.c"); println!("cargo:rerun-if-changed=wrapper.h"); for htsfile in glob("htslib/**/*").unwrap() { From 1ba12f32430dc3dd1df0e6edcc7fa50ed5615e01 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Fri, 15 May 2020 16:02:29 +1000 Subject: [PATCH 39/56] Make extra sure that -fPIC flag is passed during all compile phases --- hts-sys/build.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/hts-sys/build.rs b/hts-sys/build.rs index b9c22b8e1..71889a648 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -84,13 +84,22 @@ fn main() { let tool = cfg.get_compiler(); let (cc_path, cflags_env) = (tool.path(), tool.cflags_env()); let cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); + let extra_cc_cflags = "-g -Wall -O2 -fvisibility=hidden -fPIC".to_string(); // Some other flags which can be critical for cross-compiling to targets like MUSL //let cppflags = env::var("CPPFLAGS").unwrap_or_default(); let _ldflags= env::var("LDFLAGS").unwrap_or_default(); let host = env::var("HOST").unwrap_or_default(); // Those two steps are necessary to include the htslib plugins in the resulting libhts.a (hfile_s3.o, hfile_s3_writer.o, etc...) - if !Command::new("autoreconf").current_dir(out.join("htslib")).status().unwrap().success() && - !Command::new("./configure").current_dir(out.join("htslib")) + if !Command::new("autoreconf") + .current_dir(out.join("htslib")) + .env("CFLAGS", &extra_cc_cflags) + .status().unwrap().success() + + && + + !Command::new("./configure") + .current_dir(out.join("htslib")) + .env("CFLAGS", &extra_cc_cflags) .arg(format!("--host={}", &host)) .status().unwrap().success() { @@ -100,10 +109,11 @@ fn main() { if !Command::new("make") .current_dir(out.join("htslib")) .arg(format!("CC={}", cc_path.display())) - .arg(format!("CFLAGS={}", cc_cflags)) + .env("CFLAGS", &extra_cc_cflags) //.arg(format!("CPPFLAGS=\"{}\"", &cppflags)) //.arg(format!("LDFLAGS=\"{}\"", &ldflags)) .arg("lib-static") + .arg("-j40") //.arg("-B") .status() .unwrap() From 6e587a64634fbad7a451fc45cf50370d1a37c3ce Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Fri, 15 May 2020 22:57:49 +1000 Subject: [PATCH 40/56] Passing println(cargo:rustc-link-lib=static=hts), works, finally :) --- hts-sys/build.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 71889a648..88179ae53 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -83,7 +83,7 @@ fn main() { let tool = cfg.get_compiler(); let (cc_path, cflags_env) = (tool.path(), tool.cflags_env()); - let cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); + let _cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); let extra_cc_cflags = "-g -Wall -O2 -fvisibility=hidden -fPIC".to_string(); // Some other flags which can be critical for cross-compiling to targets like MUSL //let cppflags = env::var("CPPFLAGS").unwrap_or_default(); @@ -145,12 +145,7 @@ fn main() { println!("cargo:root={}", out.display()); println!("cargo:include={}", include.display()); println!("cargo:libdir={}", out.display()); - // println!("cargo:rustc-link-lib=static=hts"); - // println!("cargo:rustc-link-lib=static=z"); - // println!("cargo:rustc-link-lib=static=lzma"); - // println!("cargo:rustc-link-lib=static=ssl"); - // println!("cargo:rustc-link-lib=static=crypto"); - // println!("cargo:rustc-link-lib=static=curl"); + println!("cargo:rustc-link-lib=static=hts"); // XXX: only for static println!("cargo:rerun-if-changed=wrapper.c"); println!("cargo:rerun-if-changed=wrapper.h"); for htsfile in glob("htslib/**/*").unwrap() { From bd2215ab067e131fdbbfd683258ff359634269c0 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Fri, 15 May 2020 23:10:33 +1000 Subject: [PATCH 41/56] Cleanup cflgags_patterns mess --- hts-sys/build.rs | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 88179ae53..6449cfd82 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -11,23 +11,22 @@ use std::fs; use std::path::PathBuf; use std::process::Command; -fn sed_htslib_makefile(out: &PathBuf, patterns: &[&str], feature: &str) { - println!("SUBSTITUTING THINGS!"); - for pattern in patterns { - if !Command::new("sed") - .current_dir(out.join("htslib")) - .arg("-i") - .arg("-e") - .arg(pattern) - .arg("Makefile") - .status() - .unwrap() - .success() - { - panic!("failed to strip {} support", feature); - } - } -} +// fn sed_htslib_makefile(out: &PathBuf, patterns: &[&str], feature: &str) { +// for pattern in patterns { +// if !Command::new("sed") +// .current_dir(out.join("htslib")) +// .arg("-i") +// .arg("-e") +// .arg(pattern) +// .arg("Makefile") +// .status() +// .unwrap() +// .success() +// { +// panic!("failed to strip {} support", feature); +// } +// } +// } fn main() { let out = PathBuf::from(env::var("OUT_DIR").unwrap()); @@ -50,10 +49,6 @@ fn main() { copy_directory("htslib", &out).unwrap(); } - // Nice flags to have - let cflags_patterns = vec!["s/-g -Wall -O2 -fvisibility=hidden/-g -Wall -O2 -fvisibility=hidden -fPIC -pedantic -std=c90 -D_XOPEN_SOURCE=600/"]; - sed_htslib_makefile(&out, &cflags_patterns, "cflags"); - // let use_bzip2 = env::var("CARGO_FEATURE_BZIP2").is_ok(); // if !use_bzip2 { // let bzip2_patterns = vec!["s/ -lbz2//", "/#define HAVE_LIBBZ2/d"]; From d79bc04415976487c260815f45230fdd615457f6 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Mon, 18 May 2020 14:37:11 +1000 Subject: [PATCH 42/56] Works locally, migrating to docker/cross builds, MUSL has regressions after build.rs changes, regular GNU toolchain works though --- Cross.toml | 4 +- docker/Dockerfile.gnu | 11 ++--- docker/Dockerfile.musl | 83 ++++--------------------------------- docker/init_debug_docker.sh | 7 ++++ hts-sys/Cargo.toml | 9 ++-- hts-sys/build.rs | 1 + 6 files changed, 27 insertions(+), 88 deletions(-) create mode 100755 docker/init_debug_docker.sh diff --git a/Cross.toml b/Cross.toml index 8e3f98410..40192e175 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,2 +1,4 @@ [target.x86_64-unknown-linux-musl] -image = "brainstorm/rust-htslib-musl:latest" +image = "brainstorm/cross-x86_64-unknown-linux-musl:libcurl-openssl" +[target.x86_64-unknown-linux-gnu] +image = "brainstorm/cross-x86_64-unknown-linux-gnu:libcurl-openssl" diff --git a/docker/Dockerfile.gnu b/docker/Dockerfile.gnu index 2ec9fafe8..e6b78719e 100644 --- a/docker/Dockerfile.gnu +++ b/docker/Dockerfile.gnu @@ -1,8 +1,9 @@ FROM rustembedded/cross:x86_64-unknown-linux-gnu -ENV LIBCLANG_PATH /usr/lib/x86_64-linux-gnu +#ENV LIBCLANG_PATH /usr/lib/x86_64-linux-musl +ENV LIBCLANG_PATH /usr/lib/llvm-10/lib ENV LLVM_CONFIG_PATH /usr/bin -RUN apt-get update && \ - apt-get install -y libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev clang-8 && \ - ln -sf /usr/bin/llvm-config-8 /usr/bin/llvm-config && \ - ln -sf /usr/lib/x86_64-linux-gnu/libclang-8.so.1 /usr/lib/x86_64-linux-gnu/libclang.so.1 +RUN apt-get update +RUN apt-get install -y wget gnupg lsb-release software-properties-common apt-transport-https ca-certificates # Otherwise LLVM bump below fails +RUN bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" +RUN apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev # htslib deps \ No newline at end of file diff --git a/docker/Dockerfile.musl b/docker/Dockerfile.musl index d73f6622b..334456681 100644 --- a/docker/Dockerfile.musl +++ b/docker/Dockerfile.musl @@ -1,78 +1,9 @@ FROM rustembedded/cross:x86_64-unknown-linux-musl -ENV MUSL_CROSS_VERSION 0.9.9 -ENV PKG_CONFIG_ALLOW_CROSS 1 -ENV LZMA_VERSION 5.2.4 -ENV ZLIB_VERSION 1.2.11 -ENV OPENSSL_VERSION 1_1_1e -ENV CURL_VERSION 7.69.1 - -# The default includes and packages from the Ubuntu distro that cross uses will generate all sorts of linux-headers related include errors, see: -# https://github.com/rust-bio/rust-htslib/pull/184#commitcomment-37496651 -# Those are the packages installed, hopefully someone will find a good way to use the distro ones instead of compiling everything under /usr/local :/ -# apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev musl musl-dev musl-tools linux-libc-dev linux-headers-4.15.0-20-generic - -# Install basics to locally compile htslib dependencies -RUN apt-get update && \ - apt-get install -y build-essential autoconf automake autotools-dev git wget - -# Remove pre-installed musl, to avoid cross-musl-make interference -RUN apt-get remove -y musl - -# For now we'll have to go nuts and not only build musl-1.2.0 from scratch but all the other libs too... -WORKDIR /root - -# Updated musl-cross toolchain that does not fail on OpenSSL: https://github.com/openssl/openssl/issues/7207 -RUN wget https://github.com/richfelker/musl-cross-make/archive/v$MUSL_CROSS_VERSION.tar.gz && tar xvfz v$MUSL_CROSS_VERSION.tar.gz -WORKDIR /root/musl-cross-make-$MUSL_CROSS_VERSION -COPY config-musl-cross-make.mak config.mak -RUN make -j48 install - -# Now we assume we have a properly configured musl-cross... -ENV PATH "/usr/local/musl/bin:$PATH" -ENV CFLAGS "-fPIC" -ENV CROSS_COMPILE x86_64-linux-musl- -ENV CC ${CROSS_COMPILE}cc -ENV AR ${CROSS_COMPILE}ar -ENV RANLIB ${CROSS_COMPILE}ranlib -ENV CXX ${CROSS_COMPILE}g++ -ENV CPPFLAGS "-I/usr/local/musl/x86_64-linux-musl/include -I/usr/local/include" -ENV LDFLAGS "-L/usr/local/musl/x86_64-linux-musl/lib -L/usr/local/lib" - -# .. and carry on with the htslib deps -WORKDIR /root -RUN git clone git://sourceware.org/git/bzip2.git && cd bzip2 && make -j48 CC=$CC AR=$AR RANLIB=$RANLIB CFLAGS=$CFLAGS bzip2 && make -j48 install -WORKDIR /root -RUN wget https://tukaani.org/xz/xz-$LZMA_VERSION.tar.bz2 && tar xvfj xz-$LZMA_VERSION.tar.bz2 && cd xz-$LZMA_VERSION && ./configure --host x86_64-unknown-linux-musl && make -j48 install -WORKDIR /root -RUN wget https://www.zlib.net/zlib-$ZLIB_VERSION.tar.gz && tar xvfz zlib-$ZLIB_VERSION.tar.gz && cd zlib-$ZLIB_VERSION && ./configure --static && make -j48 install -WORKDIR /root -# A few gems from: https://wiki.openssl.org/index.php/Compilation_and_Installation -# "OpenSSL has been around a long time, and it carries around a lot of cruft" -# "SSLv2 is completely broken, and you should disable it during configuration" -# "You should specify both --prefix and --openssldir to ensure make -j48 install works as expected." -RUN unset CC && unset AR && unset RANLIB && wget https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION.tar.gz && \ - tar xvfz OpenSSL_$OPENSSL_VERSION.tar.gz && cd openssl-OpenSSL_$OPENSSL_VERSION && \ - ./Configure --prefix=/usr/local/openssl --openssldir=/usr/local/openssl/etc \ - --with-zlib-lib=/root/zlib-$ZLIB_VERSION linux-x86_64 \ - no-shared no-dso no-gost no-engine no-ssl2 no-srp no-srtp no-tests zlib \ - no-weak-ssl-ciphers && make -j48 && make CFLAGS="$CFLAGS" LDFLAGS="$LDFLAGS" -j48 install -WORKDIR /root -RUN wget https://curl.haxx.se/download/curl-$CURL_VERSION.tar.gz && tar xvfz curl-$CURL_VERSION.tar.gz && cd curl-$CURL_VERSION && \ - ./configure --host x86_64-linux-musl --with-ssl=/usr/local/openssl --enable-static \ - --disable-dict --disable-file --disable-ftp --disable-gopher --disable-imap \ - --disable-pop3 --disable-rtsp --disable-smb --disable-smtp --disable-telnet \ - --disable-tftp --disable-ntlm && \ - make -j48 install - -# To cater Rust's openssl-sys needs... -ENV OPENSSL_DIR /usr/local/openssl - -# Hack to force ld stick to musl on the hts-sys/build.rs side -RUN rm /usr/bin/ld && ln -sf /usr/local/musl/bin/x86_64-linux-musl-ld /usr/bin/ld -RUN apt-get install -y libclang1-9 - -# Prepare rustup and toolchain locally for easy manual intervention in this container -RUN wget https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init && chmod +x rustup-init && ./rustup-init -y && . $HOME/.cargo/env && rustup target add x86_64-unknown-linux-musl - -CMD ["bash"] +#ENV LIBCLANG_PATH /usr/lib/x86_64-linux-musl +ENV LIBCLANG_PATH /usr/lib/llvm-10/lib +ENV LLVM_CONFIG_PATH /usr/bin +RUN apt-get update +RUN apt-get install -y wget gnupg lsb-release software-properties-common apt-transport-https ca-certificates # Otherwise LLVM bump below fails +RUN bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" +RUN apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev musl musl-dev musl-tools # htslib deps diff --git a/docker/init_debug_docker.sh b/docker/init_debug_docker.sh new file mode 100755 index 000000000..5b967957c --- /dev/null +++ b/docker/init_debug_docker.sh @@ -0,0 +1,7 @@ +#!/bin/sh -x + +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +. /root/.cargo/env +rustup target add x86_64-unknown-linux-musl + +echo "Run source $HOME/.cargo/env to continue" diff --git a/hts-sys/Cargo.toml b/hts-sys/Cargo.toml index 9ce0e185f..296a916df 100644 --- a/hts-sys/Cargo.toml +++ b/hts-sys/Cargo.toml @@ -17,14 +17,11 @@ pre-release-commit-message = "release version {{version}}" tag-message = "Version {{version}} of Rust-HTSlib." [dependencies] -#libc = "0.2.68" libz-sys = { version = "1.0.25", features = ["static"] } # https://github.com/alexcrichton/bzip2-rs/issues/56 bzip2-sys = { version = "0.1.8", optional = true } -# https://github.com/alexcrichton/xz2-rs/issues/62 -lzma-sys = { version = "0.1.15", optional = true, features = ["static"], git = "https://github.com/brainstorm/xz2-rs", branch = "static_feature_flag" } +lzma-sys = { version = "0.1.16", optional = true, features = ["static"] } curl-sys = { version = "0.4.31", optional = true, features = ["static-curl", "static-ssl"] } -#openssl-sys = { version = "0.9.56", optional = true, vendored = true} [features] default = ["bzip2", "lzma"] @@ -36,6 +33,6 @@ static = [] [build-dependencies] fs-utils = "1.1" -bindgen = { version = "0.53.1", default-features = false, features = ["runtime"] } +bindgen = { version = "0.53.2", default-features = false, features = ["runtime"] } cc = "1.0" -glob = "0.3.0" +glob = "0.3.0" \ No newline at end of file diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 6449cfd82..2fc40a041 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -80,6 +80,7 @@ fn main() { let (cc_path, cflags_env) = (tool.path(), tool.cflags_env()); let _cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); let extra_cc_cflags = "-g -Wall -O2 -fvisibility=hidden -fPIC".to_string(); + cfg.include("/usr/include"); // Some other flags which can be critical for cross-compiling to targets like MUSL //let cppflags = env::var("CPPFLAGS").unwrap_or_default(); let _ldflags= env::var("LDFLAGS").unwrap_or_default(); From 886ab3eee2b07885dbd1b2999dcf779c81b2a439 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Mon, 18 May 2020 15:43:11 +1000 Subject: [PATCH 43/56] Cleanup build.rs --- .cargo/config | 0 hts-sys/build.rs | 15 ++++----------- 2 files changed, 4 insertions(+), 11 deletions(-) delete mode 100644 .cargo/config diff --git a/.cargo/config b/.cargo/config deleted file mode 100644 index e69de29bb..000000000 diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 2fc40a041..955819e3a 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -78,24 +78,20 @@ fn main() { let tool = cfg.get_compiler(); let (cc_path, cflags_env) = (tool.path(), tool.cflags_env()); - let _cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); - let extra_cc_cflags = "-g -Wall -O2 -fvisibility=hidden -fPIC".to_string(); + let cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); cfg.include("/usr/include"); - // Some other flags which can be critical for cross-compiling to targets like MUSL - //let cppflags = env::var("CPPFLAGS").unwrap_or_default(); - let _ldflags= env::var("LDFLAGS").unwrap_or_default(); let host = env::var("HOST").unwrap_or_default(); // Those two steps are necessary to include the htslib plugins in the resulting libhts.a (hfile_s3.o, hfile_s3_writer.o, etc...) if !Command::new("autoreconf") .current_dir(out.join("htslib")) - .env("CFLAGS", &extra_cc_cflags) + .env("CFLAGS", &cc_cflags) .status().unwrap().success() && !Command::new("./configure") .current_dir(out.join("htslib")) - .env("CFLAGS", &extra_cc_cflags) + .env("CFLAGS", &cc_cflags) .arg(format!("--host={}", &host)) .status().unwrap().success() { @@ -105,12 +101,9 @@ fn main() { if !Command::new("make") .current_dir(out.join("htslib")) .arg(format!("CC={}", cc_path.display())) - .env("CFLAGS", &extra_cc_cflags) - //.arg(format!("CPPFLAGS=\"{}\"", &cppflags)) - //.arg(format!("LDFLAGS=\"{}\"", &ldflags)) + .env("CFLAGS", &cc_cflags) .arg("lib-static") .arg("-j40") - //.arg("-B") .status() .unwrap() .success() From 58ff70eac604d2a7ca731d0438b36ef564ad4f63 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Tue, 19 May 2020 14:55:16 +1000 Subject: [PATCH 44/56] Add clux/muslrust Dockerfile example, as referred on https://users.rust-lang.org/t/musl-cross-compilation-how-to-reuse-curl-sys-libz-sys-bzip2-sys-lzma-sys-and-other-dependency-includes-under-target/42817/4 --- .github/workflows/rust.yml | 3 --- docker/Dockerfile.clux | 9 +++++++++ docker/README.md | 9 ++++++++- hts-sys/build.rs | 4 +--- 4 files changed, 18 insertions(+), 7 deletions(-) create mode 100644 docker/Dockerfile.clux diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index c4ccc5e91..51425871d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -61,9 +61,6 @@ jobs: toolchain: stable override: true - - name: Install system dependencies - run: | - sudo apt-get install --yes zlib1g-dev libbz2-dev musl musl-dev musl-tools clang libc6-dev - name: Run cargo-tarpaulin uses: actions-rs/tarpaulin@v0.1 diff --git a/docker/Dockerfile.clux b/docker/Dockerfile.clux new file mode 100644 index 000000000..0c9ef35f5 --- /dev/null +++ b/docker/Dockerfile.clux @@ -0,0 +1,9 @@ +FROM clux/muslrust:latest + +#ENV LIBCLANG_PATH /usr/lib/x86_64-linux-musl +ENV LIBCLANG_PATH /usr/lib/llvm-10/lib +ENV LLVM_CONFIG_PATH /usr/bin +RUN apt-get update +RUN apt-get install -y wget gnupg lsb-release software-properties-common apt-transport-https ca-certificates # Otherwise LLVM bump below fails +RUN bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" +RUN apt-get install -y zlib1g-dev libbz2-dev liblzma-dev # htslib deps diff --git a/docker/README.md b/docker/README.md index 349b54d58..417f5a2c2 100644 --- a/docker/README.md +++ b/docker/README.md @@ -8,10 +8,17 @@ Allows to compile (rust-)htslib in a variety of environments and architectures v $ cd docker $ docker build -t brainstorm/cross-x86_64-unknown-linux-musl:libcurl-openssl . -f Dockerfile.musl $ docker build -t brainstorm/cross-x86_64-unknown-linux-gnu:libcurl-openssl . -f Dockerfile.gnu +$ docker build -t brainstorm/clux-x86_64-unknown-linux-musl:latest . -f Dockerfile.clux ``` -Then to build and test rust-htslib with the above containers, proceed as you would with `cargo`, using `cross` instead, i.e: +Then, to build and test rust-htslib with the above containers, proceed as you would with `cargo`, using `cross` instead, i.e: ```shell $ cross build --target x86_64-unknown-linux-musl ``` + +Or the following for clux's muslrust: + +```shell +$ docker run -v $PWD:/volume --rm -t brainstorm/clux-x86_64-unknown-linux-musl:latest cargo build +``` diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 955819e3a..ceb5d3b86 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -40,9 +40,7 @@ fn main() { } // if let Ok(z_inc) = env::var("DEP_Z_INCLUDE") { - // if !want_static { - // cfg.include(z_inc); - // } + // cfg.include(z_inc); // } if !out.join("htslib").exists() { From 37aee9baec93e64c1737cde60ce63d737343d3dd Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Tue, 19 May 2020 14:57:21 +1000 Subject: [PATCH 45/56] Revert .github workflow minor changes because 'remote rejected] musl_support -> musl_support (refusing to allow an OAuth App to create or update workflow without scope)' --- .github/workflows/rust.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 51425871d..d654c6318 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -61,6 +61,9 @@ jobs: toolchain: stable override: true + - name: Install system dependencies + run: | + sudo apt-get install --yes zlib1g-dev libbz2-dev musl musl-dev musl-tools clang libc6-dev - name: Run cargo-tarpaulin uses: actions-rs/tarpaulin@v0.1 @@ -86,3 +89,4 @@ jobs: use-cross: true command: build args: --target x86_64-unknown-linux-musl --all-features + From 2f2d91ebb82a39ef295984820f0f72663a2c1039 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Tue, 19 May 2020 15:00:29 +1000 Subject: [PATCH 46/56] Adding rust workflow from master... --- .github/workflows/rust.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index d654c6318..67d93e66e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -22,7 +22,7 @@ jobs: - name: Check format run: cargo fmt -- --check - + Linting: runs-on: ubuntu-latest steps: @@ -76,17 +76,17 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} path-to-lcov: ./lcov.info - - name: Test musl build without default features - uses: actions-rs/cargo@v1 - with: - use-cross: true - command: build - args: --target x86_64-unknown-linux-musl --no-default-features - - - name: Test musl build with all features - uses: actions-rs/cargo@v1 - with: - use-cross: true - command: build - args: --target x86_64-unknown-linux-musl --all-features +# - name: Test musl build without default features +# uses: actions-rs/cargo@v1 +# with: +# use-cross: true +# command: build +# args: --target x86_64-unknown-linux-musl --no-default-features +# +# - name: Test musl build with all features +# uses: actions-rs/cargo@v1 +# with: +# use-cross: true +# command: build +# args: --target x86_64-unknown-linux-musl --all-features From ceac6a92e1b2d3c3bca6df3fb69354a57ad87bfe Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Tue, 19 May 2020 15:22:36 +1000 Subject: [PATCH 47/56] Remove GitHub actions workflows for now since they are blocking my progress --- .github/workflows/rust.yml | 92 -------------------------------------- 1 file changed, 92 deletions(-) delete mode 100644 .github/workflows/rust.yml diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index 67d93e66e..000000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,92 +0,0 @@ -name: CI - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - Formatting: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - components: rustfmt - - - name: Check format - run: cargo fmt -- --check - - Linting: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Checkout submodules - uses: textbook/git-checkout-submodule-action@2.0.0 - - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - components: clippy - - - name: Lint with clippy - uses: actions-rs/clippy-check@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - Testing: - needs: Formatting - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Checkout submodules - uses: textbook/git-checkout-submodule-action@2.0.0 - - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - - name: Install system dependencies - run: | - sudo apt-get install --yes zlib1g-dev libbz2-dev musl musl-dev musl-tools clang libc6-dev - - - name: Run cargo-tarpaulin - uses: actions-rs/tarpaulin@v0.1 - with: - args: '--out Lcov -- --test-threads 1' - - - name: Upload coverage - uses: coverallsapp/github-action@master - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - path-to-lcov: ./lcov.info - -# - name: Test musl build without default features -# uses: actions-rs/cargo@v1 -# with: -# use-cross: true -# command: build -# args: --target x86_64-unknown-linux-musl --no-default-features -# -# - name: Test musl build with all features -# uses: actions-rs/cargo@v1 -# with: -# use-cross: true -# command: build -# args: --target x86_64-unknown-linux-musl --all-features - From 37628ef807bbe26de5f459e9d35b38aa0fb34208 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Wed, 20 May 2020 21:28:50 +1000 Subject: [PATCH 48/56] Thanks @wezm for feedback on MUSL dockerfiles and https://github.com/fornwall/rust-static-builder pointer --- Cross.toml | 2 +- docker/Dockerfile.musl | 120 +++++++++++++++++++++++++++++++++++++++-- hts-sys/build.rs | 2 +- 3 files changed, 118 insertions(+), 6 deletions(-) diff --git a/Cross.toml b/Cross.toml index 40192e175..bd6665dfe 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,4 +1,4 @@ [target.x86_64-unknown-linux-musl] -image = "brainstorm/cross-x86_64-unknown-linux-musl:libcurl-openssl" +image = "brainstorm/cross-x86_64-unknown-linux-musl:latest" [target.x86_64-unknown-linux-gnu] image = "brainstorm/cross-x86_64-unknown-linux-gnu:libcurl-openssl" diff --git a/docker/Dockerfile.musl b/docker/Dockerfile.musl index 334456681..5d24b53a3 100644 --- a/docker/Dockerfile.musl +++ b/docker/Dockerfile.musl @@ -1,9 +1,121 @@ FROM rustembedded/cross:x86_64-unknown-linux-musl -#ENV LIBCLANG_PATH /usr/lib/x86_64-linux-musl +ENV MUSL_CROSS_MAKE_VERSION 0.9.9 +ENV PKG_CONFIG_ALLOW_CROSS 1 +ENV LZMA_VERSION 5.2.5 +ENV ZLIB_VERSION 1.2.11 +ENV OPENSSL_VERSION 1_1_1g +ENV CURL_VERSION 7.70.0 + ENV LIBCLANG_PATH /usr/lib/llvm-10/lib ENV LLVM_CONFIG_PATH /usr/bin -RUN apt-get update -RUN apt-get install -y wget gnupg lsb-release software-properties-common apt-transport-https ca-certificates # Otherwise LLVM bump below fails + + +# The default includes and packages from the Ubuntu distro that cross uses will generate all sorts of linux-headers related include errors, see: +# https://github.com/rust-bio/rust-htslib/pull/184#commitcomment-37496651 +# Those are the packages installed, hopefully someone will find a good way to use the distro ones instead of compiling everything under /usr/local :/ +# apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev musl musl-dev musl-tools linux-libc-dev linux-headers-4.15.0-20-generic + +# Install basics to locally compile htslib dependencies +RUN apt-get update && \ + apt-get install -y build-essential autoconf automake autotools-dev git wget + +# Otherwise LLVM bump below fails +RUN apt-get install -y wget gnupg lsb-release software-properties-common apt-transport-https ca-certificates + +# Autodetect and fetch latest LLVM repos for the current distro, avoids LLVM warnings and other issues, might generate slower builds for now though, see: +# https://www.phoronix.com/scan.php?page=news_item&px=Rust-Hurt-On-LLVM-10 RUN bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" -RUN apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev musl musl-dev musl-tools # htslib deps +#RUN apt-get install -y libssl-dev libcurl4-openssl-dev zlib1g-dev libbz2-dev liblzma-dev musl musl-dev musl-tools # htslib deps + +# Remove pre-installed musl, to avoid cross-musl-make interference +RUN apt-get remove -y musl + +# For now we'll have to go nuts and not only build musl-1.2.0 from scratch but all the other libs too... +# Updated musl-cross toolchain that does not fail on OpenSSL: https://github.com/openssl/openssl/issues/7207 +WORKDIR /root +RUN wget https://github.com/richfelker/musl-cross-make/archive/v$MUSL_CROSS_MAKE_VERSION.tar.gz \ + && tar xvfz v$MUSL_CROSS_MAKE_VERSION.tar.gz +WORKDIR /root/musl-cross-make-$MUSL_CROSS_MAKE_VERSION +COPY config-musl-cross-make.mak config.mak +RUN make -j40 install + +# Now we assume we have a properly configured musl-cross... +ENV PATH "/usr/local/musl/bin:$PATH" +ENV CFLAGS "-fPIC" +ENV CROSS_COMPILE x86_64-linux-musl- +ENV CC ${CROSS_COMPILE}cc +ENV AR ${CROSS_COMPILE}ar +ENV RANLIB ${CROSS_COMPILE}ranlib +ENV CXX ${CROSS_COMPILE}g++ +ENV CPPFLAGS "-I/usr/local/musl/include -I/usr/local/include" +ENV LDFLAGS "-L/usr/local/musl/lib -L/usr/local/lib" + +# .. and carry on with the htslib deps +RUN wget https://www.zlib.net/zlib-$ZLIB_VERSION.tar.gz \ + && tar xvfz zlib-$ZLIB_VERSION.tar.gz \ + && cd zlib-$ZLIB_VERSION \ + && ./configure --static --prefix=/usr/local/musl \ + && make -j40 install +WORKDIR /root +RUN git clone git://sourceware.org/git/bzip2 \ + && cd bzip2 \ + #&& make -j40 CC=$CC AR=$AR RANLIB=$RANLIB CFLAGS=$CFLAGS bzip2 \ + && make PREFIX=/usr/local/musl -j40 bzip2 \ + && make -j40 install +WORKDIR /root +RUN wget https://tukaani.org/xz/xz-$LZMA_VERSION.tar.bz2 \ + && tar xvfj xz-$LZMA_VERSION.tar.bz2 \ + && cd xz-$LZMA_VERSION \ + && ./configure --prefix=/usr/local/musl --enable-static --disable-shared --host x86_64-unknown-linux-musl \ + && make -j40 install +WORKDIR /root + +# A few gems from: https://wiki.openssl.org/index.php/Compilation_and_Installation +# "OpenSSL has been around a long time, and it carries around a lot of cruft" +# "SSLv2 is completely broken, and you should disable it during configuration" +# "You should specify both --prefix and --openssldir to ensure make install works as expected." +# +# And also this: +# https://github.com/openssl/openssl/issues/11362 +# +# And having to redefine AR,CC & RANLIB because otherwise, this: +# +# make[1]: x86_64-linux-musl-x86_64-linux-musl-ar: Command not found +RUN wget https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION.tar.gz \ + && tar xvfz OpenSSL_$OPENSSL_VERSION.tar.gz && cd openssl-OpenSSL_$OPENSSL_VERSION \ + && ./Configure --prefix=/usr/local/musl --openssldir=/usr/local/musl \ + --with-zlib-lib=/usr/local/lib/zlib-$ZLIB_VERSION linux-x86_64 \ + no-shared no-dso no-gost no-engine no-ssl2 no-srp no-srtp no-tests zlib \ + no-weak-ssl-ciphers \ + && make AR=x86_64-linux-musl-ar \ + CC=x86_64-linux-musl-cc \ + -j40 \ + && make RANLIB=x86_64-linux-musl-ranlib -j40 install + +WORKDIR /root +RUN wget https://curl.haxx.se/download/curl-$CURL_VERSION.tar.gz \ + && tar xvfz curl-$CURL_VERSION.tar.gz && cd curl-$CURL_VERSION \ + && ./configure --prefix=/usr/local/musl --host x86_64-linux-musl \ + --with-ssl=/usr/local/musl --with-zlib=/usr/local/musl --enable-static \ + --disable-dict --disable-file --disable-ftp --disable-gopher --disable-imap \ + --disable-pop3 --disable-rtsp --disable-smb --disable-smtp --disable-telnet \ + --disable-tftp --disable-ntlm --disable-ldap \ + && make -j40 install + +# To cater Rust's openssl-sys needs... +#ENV OPENSSL_DIR /usr/local/openssl +ENV OPENSSL_DIR /usr/local/musl + +# Hack to force ld stick to musl on the hts-sys/build.rs side +#RUN rm /usr/bin/ld && ln -sf /usr/local/musl/bin/x86_64-linux-musl-ld /usr/bin/ld +#RUN apt-get install -y libclang1-9 + +# Prepare rustup and toolchain locally for easy manual intervention in this container +#RUN wget https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init \ +# && chmod +x rustup-init \ +# && ./rustup-init -y \ +# && . $HOME/.cargo/env \ +# && rustup target add x86_64-unknown-linux-musl + +CMD ["bash"] \ No newline at end of file diff --git a/hts-sys/build.rs b/hts-sys/build.rs index ceb5d3b86..9c1831213 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -77,7 +77,6 @@ fn main() { let tool = cfg.get_compiler(); let (cc_path, cflags_env) = (tool.path(), tool.cflags_env()); let cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); - cfg.include("/usr/include"); let host = env::var("HOST").unwrap_or_default(); // Those two steps are necessary to include the htslib plugins in the resulting libhts.a (hfile_s3.o, hfile_s3_writer.o, etc...) if !Command::new("autoreconf") @@ -100,6 +99,7 @@ fn main() { .current_dir(out.join("htslib")) .arg(format!("CC={}", cc_path.display())) .env("CFLAGS", &cc_cflags) + .env("CFLAGS_x86_64-unknown-linux-musl", "-I/usr/local/musl") .arg("lib-static") .arg("-j40") .status() From 3cafee823f00d7ccf8ec320dc3606cdb9d95b13e Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Thu, 21 May 2020 11:41:35 +1000 Subject: [PATCH 49/56] Finally compiles MUSL, passing CFLAGS=-I/usr/local/musl/include and using build.env cross to passthrough that var --- .cargo/config | 2 + Cross.toml | 8 ++++ docker/Dockerfile.musl | 6 +-- hts-sys/build.rs | 83 +++++++++++++++++++++--------------------- 4 files changed, 54 insertions(+), 45 deletions(-) create mode 100644 .cargo/config diff --git a/.cargo/config b/.cargo/config new file mode 100644 index 000000000..7d7cf0727 --- /dev/null +++ b/.cargo/config @@ -0,0 +1,2 @@ +[target.x86_64-unknown-linux-musl] +linker = "x86_64-linux-musl-gcc" diff --git a/Cross.toml b/Cross.toml index bd6665dfe..f867314c8 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,3 +1,11 @@ +[build.env] +passthrough = [ + "RUST_DEBUG", + "RUST_BACKTRACE", + "CFLAGS" +] + + [target.x86_64-unknown-linux-musl] image = "brainstorm/cross-x86_64-unknown-linux-musl:latest" [target.x86_64-unknown-linux-gnu] diff --git a/docker/Dockerfile.musl b/docker/Dockerfile.musl index 5d24b53a3..92880ec0e 100644 --- a/docker/Dockerfile.musl +++ b/docker/Dockerfile.musl @@ -107,9 +107,9 @@ RUN wget https://curl.haxx.se/download/curl-$CURL_VERSION.tar.gz \ #ENV OPENSSL_DIR /usr/local/openssl ENV OPENSSL_DIR /usr/local/musl -# Hack to force ld stick to musl on the hts-sys/build.rs side -#RUN rm /usr/bin/ld && ln -sf /usr/local/musl/bin/x86_64-linux-musl-ld /usr/bin/ld -#RUN apt-get install -y libclang1-9 +# Hack to force ld stick to musl on the hts-sys/build.rs side, otherwise: +# = note: /usr/bin/ld: /target/debug/deps/liblibloading-689161fea10b6234.rlib(global_static.o): unable to initialize decompress status for section .debug_info +RUN rm /usr/bin/ld && ln -sf /usr/local/musl/bin/x86_64-linux-musl-ld /usr/bin/ld # Prepare rustup and toolchain locally for easy manual intervention in this container #RUN wget https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init \ diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 9c1831213..58dd2cfea 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -11,22 +11,22 @@ use std::fs; use std::path::PathBuf; use std::process::Command; -// fn sed_htslib_makefile(out: &PathBuf, patterns: &[&str], feature: &str) { -// for pattern in patterns { -// if !Command::new("sed") -// .current_dir(out.join("htslib")) -// .arg("-i") -// .arg("-e") -// .arg(pattern) -// .arg("Makefile") -// .status() -// .unwrap() -// .success() -// { -// panic!("failed to strip {} support", feature); -// } -// } -// } +fn sed_htslib_makefile(out: &PathBuf, patterns: &[&str], feature: &str) { + for pattern in patterns { + if !Command::new("sed") + .current_dir(out.join("htslib")) + .arg("-i") + .arg("-e") + .arg(pattern) + .arg("Makefile") + .status() + .unwrap() + .success() + { + panic!("failed to strip {} support", feature); + } + } +} fn main() { let out = PathBuf::from(env::var("OUT_DIR").unwrap()); @@ -47,32 +47,32 @@ fn main() { copy_directory("htslib", &out).unwrap(); } - // let use_bzip2 = env::var("CARGO_FEATURE_BZIP2").is_ok(); - // if !use_bzip2 { - // let bzip2_patterns = vec!["s/ -lbz2//", "/#define HAVE_LIBBZ2/d"]; - // sed_htslib_makefile(&out, &bzip2_patterns, "bzip2"); - // } else if let Ok(inc) = env::var("DEP_BZIP2_ROOT") - // .map(PathBuf::from) - // .map(|path| path.join("include")) - // { - // cfg.include(inc); - // } + let use_bzip2 = env::var("CARGO_FEATURE_BZIP2").is_ok(); + if !use_bzip2 { + let bzip2_patterns = vec!["s/ -lbz2//", "/#define HAVE_LIBBZ2/d"]; + sed_htslib_makefile(&out, &bzip2_patterns, "bzip2"); + } else if let Ok(inc) = env::var("DEP_BZIP2_ROOT") + .map(PathBuf::from) + .map(|path| path.join("include")) + { + cfg.include(inc); + } - // let use_lzma = env::var("CARGO_FEATURE_LZMA").is_ok(); - // if !use_lzma && want_static { - // let lzma_patterns = vec!["s/ -llzma//", "/#define HAVE_LIBLZMA/d"]; - // sed_htslib_makefile(&out, &lzma_patterns, "lzma"); - // } else if let Ok(inc) = env::var("DEP_LZMA_INCLUDE").map(PathBuf::from) { - // cfg.include(inc); - // } + let use_lzma = env::var("CARGO_FEATURE_LZMA").is_ok(); + if !use_lzma && want_static { + let lzma_patterns = vec!["s/ -llzma//", "/#define HAVE_LIBLZMA/d"]; + sed_htslib_makefile(&out, &lzma_patterns, "lzma"); + } else if let Ok(inc) = env::var("DEP_LZMA_INCLUDE").map(PathBuf::from) { + cfg.include(inc); + } - // let use_curl = env::var("CARGO_FEATURE_CURL").is_ok(); - // if !use_curl { - // let curl_patterns = vec!["s/ -lcurl//", "/#define HAVE_LIBCURL/d"]; - // sed_htslib_makefile(&out, &curl_patterns, "curl"); - // } else if let Ok(inc) = env::var("DEP_CURL_INCLUDE").map(PathBuf::from) { - // cfg.include(inc); - // } + let use_curl = env::var("CARGO_FEATURE_CURL").is_ok(); + if !use_curl { + let curl_patterns = vec!["s/ -lcurl//", "/#define HAVE_LIBCURL/d"]; + sed_htslib_makefile(&out, &curl_patterns, "curl"); + } else if let Ok(inc) = env::var("DEP_CURL_INCLUDE").map(PathBuf::from) { + cfg.include(inc); + } let tool = cfg.get_compiler(); let (cc_path, cflags_env) = (tool.path(), tool.cflags_env()); @@ -98,8 +98,7 @@ fn main() { if !Command::new("make") .current_dir(out.join("htslib")) .arg(format!("CC={}", cc_path.display())) - .env("CFLAGS", &cc_cflags) - .env("CFLAGS_x86_64-unknown-linux-musl", "-I/usr/local/musl") + .arg(format!("CFLAGS={}", &cc_cflags)) .arg("lib-static") .arg("-j40") .status() From b0fa1981ada531b9b99f99285a14cba305ae6824 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Thu, 21 May 2020 11:55:24 +1000 Subject: [PATCH 50/56] Reintroducing the GitHub actions workflow... will I be able to push it now? --- .github/workflows/rust.yml | 92 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 .github/workflows/rust.yml diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 000000000..da51dd60b --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,92 @@ +name: CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + Formatting: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + components: rustfmt + + - name: Check format + run: cargo fmt -- --check + + Linting: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Checkout submodules + uses: textbook/git-checkout-submodule-action@2.0.0 + + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + components: clippy + + - name: Lint with clippy + uses: actions-rs/clippy-check@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + + Testing: + needs: Formatting + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Checkout submodules + uses: textbook/git-checkout-submodule-action@2.0.0 + + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + + - name: Install system dependencies + run: | + sudo apt-get install --yes zlib1g-dev libbz2-dev musl musl-dev musl-tools clang libc6-dev + + - name: Run cargo-tarpaulin + uses: actions-rs/tarpaulin@v0.1 + with: + args: '--out Lcov -- --test-threads 1' + + - name: Upload coverage + uses: coverallsapp/github-action@master + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + path-to-lcov: ./lcov.info + + - name: Test musl build without default features + uses: actions-rs/cargo@v1 + with: + use-cross: true + command: build + args: --target x86_64-unknown-linux-musl --no-default-features + + - name: Test musl build with all features + uses: actions-rs/cargo@v1 + with: + use-cross: true + command: build + args: --target x86_64-unknown-linux-musl --all-features + From f1e3540a1f921eb2a189ede9ef84450c4d18c8af Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Thu, 21 May 2020 12:00:22 +1000 Subject: [PATCH 51/56] Nope, pushes are still blocked by '[remote rejected] musl_support -> musl_support (refusing to allow an OAuth App to create or update workflow...' --- .github/workflows/rust.yml | 92 -------------------------------------- 1 file changed, 92 deletions(-) delete mode 100644 .github/workflows/rust.yml diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index da51dd60b..000000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,92 +0,0 @@ -name: CI - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - Formatting: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - components: rustfmt - - - name: Check format - run: cargo fmt -- --check - - Linting: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Checkout submodules - uses: textbook/git-checkout-submodule-action@2.0.0 - - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - components: clippy - - - name: Lint with clippy - uses: actions-rs/clippy-check@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - Testing: - needs: Formatting - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Checkout submodules - uses: textbook/git-checkout-submodule-action@2.0.0 - - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - - name: Install system dependencies - run: | - sudo apt-get install --yes zlib1g-dev libbz2-dev musl musl-dev musl-tools clang libc6-dev - - - name: Run cargo-tarpaulin - uses: actions-rs/tarpaulin@v0.1 - with: - args: '--out Lcov -- --test-threads 1' - - - name: Upload coverage - uses: coverallsapp/github-action@master - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - path-to-lcov: ./lcov.info - - - name: Test musl build without default features - uses: actions-rs/cargo@v1 - with: - use-cross: true - command: build - args: --target x86_64-unknown-linux-musl --no-default-features - - - name: Test musl build with all features - uses: actions-rs/cargo@v1 - with: - use-cross: true - command: build - args: --target x86_64-unknown-linux-musl --all-features - From da9387037e90eaa7a9c22a3cb1af6965f4c9e479 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Fri, 22 May 2020 13:38:46 +1000 Subject: [PATCH 52/56] Cleanup and update README with the required cross commands. Also cleanup the different Dockerfiles tested but no longer needed. build.rs now runs make clean before reconfiguring htslib for better reproducible builds --- README.md | 20 +++++++++++++++++--- docker/Dockerfile.alpine | 11 ----------- docker/Dockerfile.clux | 9 --------- docker/init_debug_docker.sh | 7 ------- hts-sys/build.rs | 14 ++++++++++++-- 5 files changed, 29 insertions(+), 32 deletions(-) delete mode 100644 docker/Dockerfile.alpine delete mode 100644 docker/Dockerfile.clux delete mode 100755 docker/init_debug_docker.sh diff --git a/README.md b/README.md index 578d921f1..53971a6bf 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ This library provides HTSlib bindings and a high level Rust API for reading and To clone this repository, issue -``` -git clone --recursive https://github.com/rust-bio/rust-htslib.git +```shell +$ git clone --recursive https://github.com/rust-bio/rust-htslib.git ``` ensuring that the HTSlib submodule is fetched, too. @@ -19,7 +19,21 @@ If you only want to use the library, there is no need to clone the repository. G ## Requirements -To compile this crate you need the development headers of zlib, bzip2 and xz. For instance, in Debian systems one needs the following dependencies: +To compile this crate you need docker and cross: + +```shell +$ cargo install cross +$ cross build # will build with GNU toolchain +``` + +If you want to run rust-htslib code on AWS lambda, you'll need to statically compile it with MUSL as follows: + +```shell +$ export CFLAGS="-I/usr/local/musl/include" +$ cross build --target x86_64-unknown-linux-musl # will build with MUSL toolchain +``` + +Alternatively, you can also install it locally by installing the development headers of zlib, bzip2 and xz. For instance, in Debian systems one needs the following dependencies: ```shell $ sudo apt-get install zlib1g-dev libbz2-dev liblzma-dev clang pkg-config diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine deleted file mode 100644 index 35ceb1802..000000000 --- a/docker/Dockerfile.alpine +++ /dev/null @@ -1,11 +0,0 @@ -FROM alpine:edge - -RUN apk add --update wget git g++ build-base automake autoconf openssl openssl-dev curl-dev bzip2-dev xz-dev zlib-dev rustup && \ - rm -rf /var/cache/apk/* - -WORKDIR /root -#RUN git clone --recursive https://github.com/brainstorm/rust-htslib -RUN rustup-init -y - -ENV PATH "/root/.cargo/bin:$PATH" -RUN rustup target add x86_64-unknown-linux-musl \ No newline at end of file diff --git a/docker/Dockerfile.clux b/docker/Dockerfile.clux deleted file mode 100644 index 0c9ef35f5..000000000 --- a/docker/Dockerfile.clux +++ /dev/null @@ -1,9 +0,0 @@ -FROM clux/muslrust:latest - -#ENV LIBCLANG_PATH /usr/lib/x86_64-linux-musl -ENV LIBCLANG_PATH /usr/lib/llvm-10/lib -ENV LLVM_CONFIG_PATH /usr/bin -RUN apt-get update -RUN apt-get install -y wget gnupg lsb-release software-properties-common apt-transport-https ca-certificates # Otherwise LLVM bump below fails -RUN bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" -RUN apt-get install -y zlib1g-dev libbz2-dev liblzma-dev # htslib deps diff --git a/docker/init_debug_docker.sh b/docker/init_debug_docker.sh deleted file mode 100755 index 5b967957c..000000000 --- a/docker/init_debug_docker.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -x - -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -. /root/.cargo/env -rustup target add x86_64-unknown-linux-musl - -echo "Run source $HOME/.cargo/env to continue" diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 58dd2cfea..5f95bfcbb 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -78,8 +78,18 @@ fn main() { let (cc_path, cflags_env) = (tool.path(), tool.cflags_env()); let cc_cflags = cflags_env.to_string_lossy().replace("-O0", ""); let host = env::var("HOST").unwrap_or_default(); - // Those two steps are necessary to include the htslib plugins in the resulting libhts.a (hfile_s3.o, hfile_s3_writer.o, etc...) - if !Command::new("autoreconf") + // autoreconf & ./configure (with no args) steps are necessary to include the htslib plugins (hfile_s3.o, hfile_s3_writer.o, etc...) + if // cleanup first + // TODO: Have top level "cargo clean" do this instead of in here, see: + // https://github.com/rust-lang/cargo/issues/572#issuecomment-632456478 + !Command::new("make") + .current_dir(out.join("htslib")) + .arg("clean") + .status().unwrap().success() + + && + + !Command::new("autoreconf") .current_dir(out.join("htslib")) .env("CFLAGS", &cc_cflags) .status().unwrap().success() From d987c1254e4fbf732d5e3b7c45d9f8b3f64c2483 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Tue, 2 Jun 2020 15:54:13 +1000 Subject: [PATCH 53/56] Add back github workflow, now that I am a project collab/admin --- .github/workflows/rust.yml | 92 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 .github/workflows/rust.yml diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 000000000..d41dbb737 --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,92 @@ +name: CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + Formatting: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + components: rustfmt + + - name: Check format + run: cargo fmt -- --check + + Linting: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Checkout submodules + uses: textbook/git-checkout-submodule-action@2.0.0 + + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + components: clippy + + - name: Lint with clippy + uses: actions-rs/clippy-check@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + + Testing: + needs: Formatting + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Checkout submodules + uses: textbook/git-checkout-submodule-action@2.0.0 + + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + + - name: Install system dependencies + run: | + sudo apt-get install --yes zlib1g-dev libbz2-dev musl musl-dev musl-tools clang libc6-dev + + - name: Run cargo-tarpaulin + uses: actions-rs/tarpaulin@v0.1 + with: + args: '--out Lcov -- --test-threads 1' + + - name: Upload coverage + uses: coverallsapp/github-action@master + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + path-to-lcov: ./lcov.info + + - name: Test musl build without default features + uses: actions-rs/cargo@v1 + with: + use-cross: true + command: build + args: --target x86_64-unknown-linux-musl --no-default-features + + - name: Test musl build with all features + uses: actions-rs/cargo@v1 + with: + use-cross: true + command: build + args: --target x86_64-unknown-linux-musl --all-features + From 5656e4a306803f9c0ec0490a0b8a1fbbecc56363 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Tue, 2 Jun 2020 16:19:39 +1000 Subject: [PATCH 54/56] Make sure MUSL builds find zlib.h et al --- .github/workflows/rust.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index d41dbb737..22df7ebc6 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -77,6 +77,8 @@ jobs: path-to-lcov: ./lcov.info - name: Test musl build without default features + env: + CFLAGS: "$CFLAGS:-I/usr/local/musl/include" uses: actions-rs/cargo@v1 with: use-cross: true @@ -84,6 +86,8 @@ jobs: args: --target x86_64-unknown-linux-musl --no-default-features - name: Test musl build with all features + env: + CFLAGS: "$CFLAGS:-I/usr/local/musl/include" uses: actions-rs/cargo@v1 with: use-cross: true From b32524b84cbd6620db5fcff4d99ff64176cdd838 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Tue, 2 Jun 2020 16:31:24 +1000 Subject: [PATCH 55/56] Sans quotes and let others concatenate CFLAGS --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 22df7ebc6..3d98254e8 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -78,7 +78,7 @@ jobs: - name: Test musl build without default features env: - CFLAGS: "$CFLAGS:-I/usr/local/musl/include" + CFLAGS: -I/usr/local/musl/include uses: actions-rs/cargo@v1 with: use-cross: true @@ -87,7 +87,7 @@ jobs: - name: Test musl build with all features env: - CFLAGS: "$CFLAGS:-I/usr/local/musl/include" + CFLAGS: -I/usr/local/musl/include uses: actions-rs/cargo@v1 with: use-cross: true From 48a329ff07b8231eae3fb0df4163e41ec057007f Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Tue, 2 Jun 2020 17:23:46 +1000 Subject: [PATCH 56/56] Remove clux rust container references and stray -j40 make flag --- docker/README.md | 7 ------- hts-sys/build.rs | 9 ++++----- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/docker/README.md b/docker/README.md index 417f5a2c2..57612c885 100644 --- a/docker/README.md +++ b/docker/README.md @@ -8,7 +8,6 @@ Allows to compile (rust-)htslib in a variety of environments and architectures v $ cd docker $ docker build -t brainstorm/cross-x86_64-unknown-linux-musl:libcurl-openssl . -f Dockerfile.musl $ docker build -t brainstorm/cross-x86_64-unknown-linux-gnu:libcurl-openssl . -f Dockerfile.gnu -$ docker build -t brainstorm/clux-x86_64-unknown-linux-musl:latest . -f Dockerfile.clux ``` Then, to build and test rust-htslib with the above containers, proceed as you would with `cargo`, using `cross` instead, i.e: @@ -16,9 +15,3 @@ Then, to build and test rust-htslib with the above containers, proceed as you wo ```shell $ cross build --target x86_64-unknown-linux-musl ``` - -Or the following for clux's muslrust: - -```shell -$ docker run -v $PWD:/volume --rm -t brainstorm/clux-x86_64-unknown-linux-musl:latest cargo build -``` diff --git a/hts-sys/build.rs b/hts-sys/build.rs index 5d48b38d0..66f745c6c 100644 --- a/hts-sys/build.rs +++ b/hts-sys/build.rs @@ -39,9 +39,9 @@ fn main() { cfg.warnings(true).static_flag(false).pic(true); } - // if let Ok(z_inc) = env::var("DEP_Z_INCLUDE") { - // cfg.include(z_inc); - // } + if let Ok(z_inc) = env::var("DEP_Z_INCLUDE") { + cfg.include(z_inc); + } if !out.join("htslib").exists() { copy_directory("htslib", &out).unwrap(); @@ -110,7 +110,6 @@ fn main() { .arg(format!("CC={}", cc_path.display())) .arg(format!("CFLAGS={}", &cc_cflags)) .arg("lib-static") - .arg("-j40") .status() .unwrap() .success() @@ -141,7 +140,7 @@ fn main() { println!("cargo:root={}", out.display()); println!("cargo:include={}", include.display()); println!("cargo:libdir={}", out.display()); - println!("cargo:rustc-link-lib=static=hts"); // XXX: only for static + println!("cargo:rustc-link-lib=static=hts"); // XXX: only for static, adapt for dynamic? println!("cargo:rerun-if-changed=wrapper.c"); println!("cargo:rerun-if-changed=wrapper.h"); println!("cargo:rerun-if-changed=htslib/Makefile");