From c17498196cf7e18e4d508b2a1c00224c77182bdb Mon Sep 17 00:00:00 2001 From: Sebastian Urban Date: Sun, 23 Feb 2025 09:33:50 +0000 Subject: [PATCH 1/6] Replace xz2 dependency by liblzma liblzma is a drop-in replacement for xz2. It uses a new liblzma version and supports targeting WebAssembly. --- Cargo.toml | 8 ++++---- src/codec/mod.rs | 4 ++-- src/codec/xz2/decoder.rs | 2 +- src/codec/xz2/encoder.rs | 2 +- src/lib.rs | 4 ++-- tests/artifacts/lib.rs | 4 ++-- tests/utils/algos.rs | 12 ++++++------ 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 289ddb41..c44d3f3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,9 +24,9 @@ all-algorithms = ["brotli", "bzip2", "deflate", "gzip", "lzma", "xz", "zlib", "z # algorithms deflate = ["flate2"] gzip = ["flate2"] -lzma = ["xz2"] -xz = ["xz2"] -zlib = ["flate2"] +lzma = ["liblzma"] +xz = ["liblzma"] +zlib = ["liblzma"] zstd = ["libzstd", "zstd-safe"] zstdmt = ["zstd", "zstd-safe/zstdmt"] deflate64 = ["dep:deflate64"] @@ -41,7 +41,7 @@ libzstd = { package = "zstd", version = "0.13.1", optional = true, default-featu memchr = "2" pin-project-lite = "0.2" tokio = { version = "1.24.2", optional = true, default-features = false } -xz2 = { version = "0.1.6", optional = true } +liblzma = { version = "0.3.6", optional = true } zstd-safe = { version = "7", optional = true, default-features = false } deflate64 = { version = "0.1.5", optional = true } diff --git a/src/codec/mod.rs b/src/codec/mod.rs index 23eda276..b37f54ef 100644 --- a/src/codec/mod.rs +++ b/src/codec/mod.rs @@ -17,7 +17,7 @@ mod gzip; mod lzma; #[cfg(feature = "xz")] mod xz; -#[cfg(feature = "xz2")] +#[cfg(feature = "liblzma")] mod xz2; #[cfg(feature = "zlib")] mod zlib; @@ -40,7 +40,7 @@ pub(crate) use self::gzip::{GzipDecoder, GzipEncoder}; pub(crate) use self::lzma::{LzmaDecoder, LzmaEncoder}; #[cfg(feature = "xz")] pub(crate) use self::xz::{XzDecoder, XzEncoder}; -#[cfg(feature = "xz2")] +#[cfg(feature = "liblzma")] pub(crate) use self::xz2::{Xz2Decoder, Xz2Encoder, Xz2FileFormat}; #[cfg(feature = "zlib")] pub(crate) use self::zlib::{ZlibDecoder, ZlibEncoder}; diff --git a/src/codec/xz2/decoder.rs b/src/codec/xz2/decoder.rs index 158976a7..cfcd1cbf 100644 --- a/src/codec/xz2/decoder.rs +++ b/src/codec/xz2/decoder.rs @@ -1,6 +1,6 @@ use std::{fmt, io}; -use xz2::stream::{Action, Status, Stream}; +use liblzma::stream::{Action, Status, Stream}; use crate::{codec::Decode, util::PartialBuffer}; diff --git a/src/codec/xz2/encoder.rs b/src/codec/xz2/encoder.rs index cf5d7cdb..d31070ac 100644 --- a/src/codec/xz2/encoder.rs +++ b/src/codec/xz2/encoder.rs @@ -1,6 +1,6 @@ use std::{fmt, io}; -use xz2::stream::{Action, Check, LzmaOptions, Status, Stream}; +use liblzma::stream::{Action, Check, LzmaOptions, Status, Stream}; use crate::{ codec::{Encode, Xz2FileFormat}, diff --git a/src/lib.rs b/src/lib.rs index 91d906c3..3c691862 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -139,7 +139,7 @@ )] #![cfg_attr(not(all), allow(unused))] -#[cfg(any(feature = "bzip2", feature = "flate2", feature = "xz2"))] +#[cfg(any(feature = "bzip2", feature = "flate2", feature = "liblzma"))] use std::convert::TryInto; #[macro_use] @@ -242,7 +242,7 @@ impl Level { } } - #[cfg(feature = "xz2")] + #[cfg(feature = "liblzma")] fn into_xz2(self) -> u32 { match self { Self::Fastest => 0, diff --git a/tests/artifacts/lib.rs b/tests/artifacts/lib.rs index fcd2e531..2e8b34c1 100644 --- a/tests/artifacts/lib.rs +++ b/tests/artifacts/lib.rs @@ -130,7 +130,7 @@ )] #![cfg_attr(not(all), allow(unused))] -#[cfg(any(feature = "bzip2", feature = "flate2", feature = "xz2"))] +#[cfg(any(feature = "bzip2", feature = "flate2", feature = "liblzma"))] use std::convert::TryInto; #[macro_use] @@ -225,7 +225,7 @@ impl Level { } } - #[cfg(feature = "xz2")] + #[cfg(feature = "liblzma")] fn into_xz2(self) -> u32 { match self { Self::Fastest => 0, diff --git a/tests/utils/algos.rs b/tests/utils/algos.rs index b82bdba5..fbe956d8 100644 --- a/tests/utils/algos.rs +++ b/tests/utils/algos.rs @@ -169,13 +169,13 @@ algos! { pub use crate::utils::impls::sync::to_vec; pub fn compress(bytes: &[u8]) -> Vec { - use xz2::bufread::XzEncoder; + use liblzma::bufread::XzEncoder; to_vec(XzEncoder::new(bytes, 0)) } pub fn decompress(bytes: &[u8]) -> Vec { - use xz2::bufread::XzDecoder; + use liblzma::bufread::XzDecoder; to_vec(XzDecoder::new(bytes)) } @@ -187,8 +187,8 @@ algos! { pub use crate::utils::impls::sync::to_vec; pub fn compress(bytes: &[u8]) -> Vec { - use xz2::bufread::XzEncoder; - use xz2::stream::{LzmaOptions, Stream}; + use liblzma::bufread::XzEncoder; + use liblzma::stream::{LzmaOptions, Stream}; to_vec(XzEncoder::new_stream( bytes, @@ -197,8 +197,8 @@ algos! { } pub fn decompress(bytes: &[u8]) -> Vec { - use xz2::bufread::XzDecoder; - use xz2::stream::Stream; + use liblzma::bufread::XzDecoder; + use liblzma::stream::Stream; to_vec(XzDecoder::new_stream( bytes, From fcd20c843ffac355b7a44ae20c5be409df1b18f2 Mon Sep 17 00:00:00 2001 From: Sebastian Urban Date: Sun, 23 Feb 2025 09:34:35 +0000 Subject: [PATCH 2/6] Use cargo resolver 2 This is required for "cargo build --lib" to not activate Tokio features from dev-dependencies. Otherwise, the build fails on WebAssembly. --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index c44d3f3b..f610cfab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ name = "async-compression" version = "0.4.18" authors = ["Wim Looman ", "Allen Bui "] edition = "2018" +resolver = "2" license = "MIT OR Apache-2.0" keywords = ["compression", "gzip", "zstd", "brotli", "async"] categories = ["compression", "asynchronous"] From 38c387bd0e8cb99012df7cd37f0e720278924fa1 Mon Sep 17 00:00:00 2001 From: Sebastian Urban Date: Sun, 23 Feb 2025 09:58:52 +0000 Subject: [PATCH 3/6] CI: add wasm32-wasip1-threads target (build only) --- .github/workflows/wasi.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/wasi.yml diff --git a/.github/workflows/wasi.yml b/.github/workflows/wasi.yml new file mode 100644 index 00000000..111d44a6 --- /dev/null +++ b/.github/workflows/wasi.yml @@ -0,0 +1,32 @@ +name: wasi + +env: + RUST_BACKTRACE: 1 + +jobs: + build: + name: cargo +nightly build --lib --features all --target wasm32-wasip1-threads + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: nightly + target: wasm32-wasip1-threads + - run: | + curl -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sysroot-25.0.tar.gz -o wasi-sysroot.tar.gz + mkdir -p wasi-sysroot + tar xf wasi-sysroot.tar.gz --strip-components=1 -C wasi-sysroot + - run: | + export "CFLAGS_wasm32_wasip1_threads=--sysroot=${{ github.workspace }}/wasi-sysroot -I${{ github.workspace }}/wasi-sysroot/include/wasm32-wasip1-threads -L-I${{ github.workspace }}/wasi-sysroot/lib/wasm32-wasip1-threads" + cargo +nightly build --lib --features all --target wasm32-wasip1-threads + +on: + merge_group: + types: [checks_requested] + pull_request: + branches: [main] + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: + - main From ebc53e1ad4789de3a06b58201a88ad8308a4076e Mon Sep 17 00:00:00 2001 From: Sebastian Urban Date: Sun, 23 Feb 2025 10:22:37 +0000 Subject: [PATCH 4/6] Fix zlib feature --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index f610cfab..2cb0f8b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ deflate = ["flate2"] gzip = ["flate2"] lzma = ["liblzma"] xz = ["liblzma"] -zlib = ["liblzma"] +zlib = ["flate2"] zstd = ["libzstd", "zstd-safe"] zstdmt = ["zstd", "zstd-safe/zstdmt"] deflate64 = ["dep:deflate64"] From a9c03ac988bfe713387089e9a1d8a1953f305d4f Mon Sep 17 00:00:00 2001 From: Sebastian Urban Date: Thu, 27 Feb 2025 16:41:27 +0100 Subject: [PATCH 5/6] tests: revert lib.rs artifact --- tests/artifacts/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/artifacts/lib.rs b/tests/artifacts/lib.rs index 2e8b34c1..fcd2e531 100644 --- a/tests/artifacts/lib.rs +++ b/tests/artifacts/lib.rs @@ -130,7 +130,7 @@ )] #![cfg_attr(not(all), allow(unused))] -#[cfg(any(feature = "bzip2", feature = "flate2", feature = "liblzma"))] +#[cfg(any(feature = "bzip2", feature = "flate2", feature = "xz2"))] use std::convert::TryInto; #[macro_use] @@ -225,7 +225,7 @@ impl Level { } } - #[cfg(feature = "liblzma")] + #[cfg(feature = "xz2")] fn into_xz2(self) -> u32 { match self { Self::Fastest => 0, From 04b854ee61615560245839fe8e30faa76f9414d7 Mon Sep 17 00:00:00 2001 From: Sebastian Urban Date: Thu, 27 Feb 2025 23:32:35 +0100 Subject: [PATCH 6/6] Apply code review suggestions --- .github/workflows/wasi.yml | 2 +- Cargo.toml | 5 +++-- src/codec/mod.rs | 4 ++-- src/lib.rs | 4 ++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/wasi.yml b/.github/workflows/wasi.yml index 111d44a6..b6a4e0fe 100644 --- a/.github/workflows/wasi.yml +++ b/.github/workflows/wasi.yml @@ -5,7 +5,7 @@ env: jobs: build: - name: cargo +nightly build --lib --features all --target wasm32-wasip1-threads + name: Build for wasm32-wasip1-threads runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/Cargo.toml b/Cargo.toml index 087660e0..57a889e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,8 +25,9 @@ all-algorithms = ["brotli", "bzip2", "deflate", "gzip", "lzma", "xz", "zlib", "z # algorithms deflate = ["flate2"] gzip = ["flate2"] -lzma = ["liblzma"] -xz = ["liblzma"] +lzma = ["dep:liblzma"] +xz = ["lzma"] +xz2 = ["xz"] zlib = ["flate2"] zstd = ["libzstd", "zstd-safe"] zstdmt = ["zstd", "zstd-safe/zstdmt"] diff --git a/src/codec/mod.rs b/src/codec/mod.rs index b37f54ef..79acff41 100644 --- a/src/codec/mod.rs +++ b/src/codec/mod.rs @@ -17,7 +17,7 @@ mod gzip; mod lzma; #[cfg(feature = "xz")] mod xz; -#[cfg(feature = "liblzma")] +#[cfg(feature = "lzma")] mod xz2; #[cfg(feature = "zlib")] mod zlib; @@ -40,7 +40,7 @@ pub(crate) use self::gzip::{GzipDecoder, GzipEncoder}; pub(crate) use self::lzma::{LzmaDecoder, LzmaEncoder}; #[cfg(feature = "xz")] pub(crate) use self::xz::{XzDecoder, XzEncoder}; -#[cfg(feature = "liblzma")] +#[cfg(feature = "lzma")] pub(crate) use self::xz2::{Xz2Decoder, Xz2Encoder, Xz2FileFormat}; #[cfg(feature = "zlib")] pub(crate) use self::zlib::{ZlibDecoder, ZlibEncoder}; diff --git a/src/lib.rs b/src/lib.rs index 3c691862..f21fbd85 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -139,7 +139,7 @@ )] #![cfg_attr(not(all), allow(unused))] -#[cfg(any(feature = "bzip2", feature = "flate2", feature = "liblzma"))] +#[cfg(any(feature = "bzip2", feature = "flate2", feature = "lzma"))] use std::convert::TryInto; #[macro_use] @@ -242,7 +242,7 @@ impl Level { } } - #[cfg(feature = "liblzma")] + #[cfg(feature = "lzma")] fn into_xz2(self) -> u32 { match self { Self::Fastest => 0,