From 123a14d930c5e2c08adad7e115b2f1fdcc0bcbf6 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Wed, 4 Jun 2025 18:27:12 -0700 Subject: [PATCH 1/3] Test Linux Stability --- Tests/SparkConnectTests/CatalogTests.swift | 2 -- .../SparkConnectTests/DataFrameInternalTests.swift | 2 -- Tests/SparkConnectTests/DataFrameTests.swift | 4 ---- Tests/SparkConnectTests/IcebergTests.swift | 14 ++++++-------- Tests/SparkConnectTests/SQLTests.swift | 2 -- Tests/SparkConnectTests/SparkSessionTests.swift | 4 ---- 6 files changed, 6 insertions(+), 22 deletions(-) diff --git a/Tests/SparkConnectTests/CatalogTests.swift b/Tests/SparkConnectTests/CatalogTests.swift index 0888fdd..179b789 100644 --- a/Tests/SparkConnectTests/CatalogTests.swift +++ b/Tests/SparkConnectTests/CatalogTests.swift @@ -25,7 +25,6 @@ import Testing /// A test suite for `Catalog` @Suite(.serialized) struct CatalogTests { -#if !os(Linux) @Test func currentCatalog() async throws { let spark = try await SparkSession.builder.getOrCreate() @@ -306,7 +305,6 @@ struct CatalogTests { #expect(try await spark.catalog.dropGlobalTempView("invalid view name") == false) await spark.stop() } -#endif @Test func cacheTable() async throws { diff --git a/Tests/SparkConnectTests/DataFrameInternalTests.swift b/Tests/SparkConnectTests/DataFrameInternalTests.swift index 96e8fc2..1b79419 100644 --- a/Tests/SparkConnectTests/DataFrameInternalTests.swift +++ b/Tests/SparkConnectTests/DataFrameInternalTests.swift @@ -25,7 +25,6 @@ import Testing @Suite(.serialized) struct DataFrameInternalTests { -#if !os(Linux) @Test func showString() async throws { let spark = try await SparkSession.builder.getOrCreate() @@ -83,5 +82,4 @@ struct DataFrameInternalTests { """) await spark.stop() } -#endif } diff --git a/Tests/SparkConnectTests/DataFrameTests.swift b/Tests/SparkConnectTests/DataFrameTests.swift index f5c6eeb..cdfc10a 100644 --- a/Tests/SparkConnectTests/DataFrameTests.swift +++ b/Tests/SparkConnectTests/DataFrameTests.swift @@ -344,7 +344,6 @@ struct DataFrameTests { await spark.stop() } -#if !os(Linux) @Test func sort() async throws { let spark = try await SparkSession.builder.getOrCreate() @@ -360,7 +359,6 @@ struct DataFrameTests { #expect(try await spark.range(10, 0, -1).orderBy("id").collect() == expected) await spark.stop() } -#endif @Test func table() async throws { @@ -376,7 +374,6 @@ struct DataFrameTests { await spark.stop() } -#if !os(Linux) @Test func collect() async throws { let spark = try await SparkSession.builder.getOrCreate() @@ -912,7 +909,6 @@ struct DataFrameTests { #expect(try await df.collect() == expected) await spark.stop() } -#endif @Test func storageLevel() async throws { diff --git a/Tests/SparkConnectTests/IcebergTests.swift b/Tests/SparkConnectTests/IcebergTests.swift index 94c6a8a..70095a5 100644 --- a/Tests/SparkConnectTests/IcebergTests.swift +++ b/Tests/SparkConnectTests/IcebergTests.swift @@ -99,14 +99,12 @@ struct IcebergTests { WHEN NOT MATCHED BY SOURCE THEN UPDATE SET data = 'invalid' """ ).count() - #if !os(Linux) - let expected = [ - Row(2, "updated"), - Row(3, "invalid"), - Row(4, "new"), - ] - #expect(try await spark.table(t2).collect() == expected) - #endif + let expected = [ + Row(2, "updated"), + Row(3, "invalid"), + Row(4, "new"), + ] + #expect(try await spark.table(t2).collect() == expected) }) await spark.stop() } diff --git a/Tests/SparkConnectTests/SQLTests.swift b/Tests/SparkConnectTests/SQLTests.swift index 5c5efb2..906c338 100644 --- a/Tests/SparkConnectTests/SQLTests.swift +++ b/Tests/SparkConnectTests/SQLTests.swift @@ -90,7 +90,6 @@ struct SQLTests { "variant.sql", ] -#if !os(Linux) @Test func runAll() async throws { let spark = try await SparkSession.builder.getOrCreate() @@ -120,5 +119,4 @@ struct SQLTests { } await spark.stop() } -#endif } diff --git a/Tests/SparkConnectTests/SparkSessionTests.swift b/Tests/SparkConnectTests/SparkSessionTests.swift index 1b4a658..3ffb340 100644 --- a/Tests/SparkConnectTests/SparkSessionTests.swift +++ b/Tests/SparkConnectTests/SparkSessionTests.swift @@ -129,7 +129,6 @@ struct SparkSessionTests { await spark.stop() } -#if !os(Linux) @Test func sql() async throws { await SparkSession.builder.clear() @@ -196,7 +195,6 @@ struct SparkSessionTests { } await spark.stop() } -#endif @Test func table() async throws { @@ -215,10 +213,8 @@ struct SparkSessionTests { await SparkSession.builder.clear() let spark = try await SparkSession.builder.getOrCreate() #expect(try await spark.time(spark.range(1000).count) == 1000) -#if !os(Linux) #expect(try await spark.time(spark.range(1).collect) == [Row(0)]) try await spark.time(spark.range(10).show) -#endif await spark.stop() } From 97e6a3ca188db1af19fac86df7962bfd6df2c853 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Wed, 4 Jun 2025 18:38:19 -0700 Subject: [PATCH 2/3] Enable test pipeline --- .github/workflows/build_and_test.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index ca5d245..80cc149 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -174,3 +174,22 @@ jobs: DEFAULT_BRANCH: main VALIDATE_MARKDOWN: true GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + integration-test-linux: + runs-on: ubuntu-latest + env: + SPARK_REMOTE: "sc://localhost:15003" + services: + spark: + image: apache/spark:4.0.0 + env: + SPARK_NO_DAEMONIZE: 1 + ports: + - 15003:15002 + options: --entrypoint /opt/spark/sbin/start-connect-server.sh + steps: + - uses: actions/checkout@v4 + - name: Build + run: | + docker run swift:6.1 uname -a + docker run --add-host=host.docker.internal:host-gateway -v $PWD:/spark -w /spark -e SPARK_REMOTE='sc://host.docker.internal:15003' swift:6.1 swift test --no-parallel -c release From 4e9e19b68b4f2a4ac1b951e68b26ee4fb82dc3c7 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Thu, 5 Jun 2025 07:18:17 -0700 Subject: [PATCH 3/3] Set SPARK_USER temporarily --- .github/workflows/build_and_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 80cc149..54fa7b1 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -184,6 +184,7 @@ jobs: image: apache/spark:4.0.0 env: SPARK_NO_DAEMONIZE: 1 + SPARK_USER: 185 ports: - 15003:15002 options: --entrypoint /opt/spark/sbin/start-connect-server.sh