diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index ca5d245..54fa7b1 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -174,3 +174,23 @@ jobs: DEFAULT_BRANCH: main VALIDATE_MARKDOWN: true GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + integration-test-linux: + runs-on: ubuntu-latest + env: + SPARK_REMOTE: "sc://localhost:15003" + services: + spark: + image: apache/spark:4.0.0 + env: + SPARK_NO_DAEMONIZE: 1 + SPARK_USER: 185 + ports: + - 15003:15002 + options: --entrypoint /opt/spark/sbin/start-connect-server.sh + steps: + - uses: actions/checkout@v4 + - name: Build + run: | + docker run swift:6.1 uname -a + docker run --add-host=host.docker.internal:host-gateway -v $PWD:/spark -w /spark -e SPARK_REMOTE='sc://host.docker.internal:15003' swift:6.1 swift test --no-parallel -c release diff --git a/Tests/SparkConnectTests/CatalogTests.swift b/Tests/SparkConnectTests/CatalogTests.swift index 0888fdd..179b789 100644 --- a/Tests/SparkConnectTests/CatalogTests.swift +++ b/Tests/SparkConnectTests/CatalogTests.swift @@ -25,7 +25,6 @@ import Testing /// A test suite for `Catalog` @Suite(.serialized) struct CatalogTests { -#if !os(Linux) @Test func currentCatalog() async throws { let spark = try await SparkSession.builder.getOrCreate() @@ -306,7 +305,6 @@ struct CatalogTests { #expect(try await spark.catalog.dropGlobalTempView("invalid view name") == false) await spark.stop() } -#endif @Test func cacheTable() async throws { diff --git a/Tests/SparkConnectTests/DataFrameInternalTests.swift b/Tests/SparkConnectTests/DataFrameInternalTests.swift index 96e8fc2..1b79419 100644 --- a/Tests/SparkConnectTests/DataFrameInternalTests.swift +++ b/Tests/SparkConnectTests/DataFrameInternalTests.swift @@ -25,7 +25,6 @@ import Testing @Suite(.serialized) struct DataFrameInternalTests { -#if !os(Linux) @Test func showString() async throws { let spark = try await SparkSession.builder.getOrCreate() @@ -83,5 +82,4 @@ struct DataFrameInternalTests { """) await spark.stop() } -#endif } diff --git a/Tests/SparkConnectTests/DataFrameTests.swift b/Tests/SparkConnectTests/DataFrameTests.swift index f5c6eeb..cdfc10a 100644 --- a/Tests/SparkConnectTests/DataFrameTests.swift +++ b/Tests/SparkConnectTests/DataFrameTests.swift @@ -344,7 +344,6 @@ struct DataFrameTests { await spark.stop() } -#if !os(Linux) @Test func sort() async throws { let spark = try await SparkSession.builder.getOrCreate() @@ -360,7 +359,6 @@ struct DataFrameTests { #expect(try await spark.range(10, 0, -1).orderBy("id").collect() == expected) await spark.stop() } -#endif @Test func table() async throws { @@ -376,7 +374,6 @@ struct DataFrameTests { await spark.stop() } -#if !os(Linux) @Test func collect() async throws { let spark = try await SparkSession.builder.getOrCreate() @@ -912,7 +909,6 @@ struct DataFrameTests { #expect(try await df.collect() == expected) await spark.stop() } -#endif @Test func storageLevel() async throws { diff --git a/Tests/SparkConnectTests/IcebergTests.swift b/Tests/SparkConnectTests/IcebergTests.swift index 94c6a8a..70095a5 100644 --- a/Tests/SparkConnectTests/IcebergTests.swift +++ b/Tests/SparkConnectTests/IcebergTests.swift @@ -99,14 +99,12 @@ struct IcebergTests { WHEN NOT MATCHED BY SOURCE THEN UPDATE SET data = 'invalid' """ ).count() - #if !os(Linux) - let expected = [ - Row(2, "updated"), - Row(3, "invalid"), - Row(4, "new"), - ] - #expect(try await spark.table(t2).collect() == expected) - #endif + let expected = [ + Row(2, "updated"), + Row(3, "invalid"), + Row(4, "new"), + ] + #expect(try await spark.table(t2).collect() == expected) }) await spark.stop() } diff --git a/Tests/SparkConnectTests/SQLTests.swift b/Tests/SparkConnectTests/SQLTests.swift index 5c5efb2..906c338 100644 --- a/Tests/SparkConnectTests/SQLTests.swift +++ b/Tests/SparkConnectTests/SQLTests.swift @@ -90,7 +90,6 @@ struct SQLTests { "variant.sql", ] -#if !os(Linux) @Test func runAll() async throws { let spark = try await SparkSession.builder.getOrCreate() @@ -120,5 +119,4 @@ struct SQLTests { } await spark.stop() } -#endif } diff --git a/Tests/SparkConnectTests/SparkSessionTests.swift b/Tests/SparkConnectTests/SparkSessionTests.swift index 1b4a658..3ffb340 100644 --- a/Tests/SparkConnectTests/SparkSessionTests.swift +++ b/Tests/SparkConnectTests/SparkSessionTests.swift @@ -129,7 +129,6 @@ struct SparkSessionTests { await spark.stop() } -#if !os(Linux) @Test func sql() async throws { await SparkSession.builder.clear() @@ -196,7 +195,6 @@ struct SparkSessionTests { } await spark.stop() } -#endif @Test func table() async throws { @@ -215,10 +213,8 @@ struct SparkSessionTests { await SparkSession.builder.clear() let spark = try await SparkSession.builder.getOrCreate() #expect(try await spark.time(spark.range(1000).count) == 1000) -#if !os(Linux) #expect(try await spark.time(spark.range(1).collect) == [Row(0)]) try await spark.time(spark.range(10).show) -#endif await spark.stop() }