Test Linux Stability

.github/workflows/build_and_test.yml

@@ -174,3 +174,23 @@ jobs:
         DEFAULT_BRANCH: main
         VALIDATE_MARKDOWN: true
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  integration-test-linux:
+    runs-on: ubuntu-latest
+    env:
+      SPARK_REMOTE: "sc://localhost:15003"
+    services:
+      spark:
+        image: apache/spark:4.0.0
+        env:
+          SPARK_NO_DAEMONIZE: 1
+          SPARK_USER: 185
+        ports:
+          - 15003:15002
+        options: --entrypoint /opt/spark/sbin/start-connect-server.sh
+    steps:
+    - uses: actions/checkout@v4
+    - name: Build
+      run: |
+        docker run swift:6.1 uname -a
+        docker run --add-host=host.docker.internal:host-gateway -v $PWD:/spark -w /spark -e SPARK_REMOTE='sc://host.docker.internal:15003' swift:6.1 swift test --no-parallel -c release

Tests/SparkConnectTests/CatalogTests.swift

@@ -25,7 +25,6 @@ import Testing
 /// A test suite for `Catalog`
 @Suite(.serialized)
 struct CatalogTests {
-#if !os(Linux)
   @Test
   func currentCatalog() async throws {
     let spark = try await SparkSession.builder.getOrCreate()
@@ -306,7 +305,6 @@ struct CatalogTests {
     #expect(try await spark.catalog.dropGlobalTempView("invalid view name") == false)
     await spark.stop()
   }
-#endif
 
   @Test
   func cacheTable() async throws {

Tests/SparkConnectTests/DataFrameInternalTests.swift

@@ -25,7 +25,6 @@ import Testing
 @Suite(.serialized)
 struct DataFrameInternalTests {
 
-#if !os(Linux)
   @Test
   func showString() async throws {
     let spark = try await SparkSession.builder.getOrCreate()
@@ -83,5 +82,4 @@ struct DataFrameInternalTests {
         """)
     await spark.stop()
   }
-#endif
 }

Tests/SparkConnectTests/DataFrameTests.swift

@@ -344,7 +344,6 @@ struct DataFrameTests {
     await spark.stop()
   }
 
-#if !os(Linux)
   @Test
   func sort() async throws {
     let spark = try await SparkSession.builder.getOrCreate()
@@ -360,7 +359,6 @@ struct DataFrameTests {
     #expect(try await spark.range(10, 0, -1).orderBy("id").collect() == expected)
     await spark.stop()
   }
-#endif
 
   @Test
   func table() async throws {
@@ -376,7 +374,6 @@ struct DataFrameTests {
     await spark.stop()
   }
 
-#if !os(Linux)
   @Test
   func collect() async throws {
     let spark = try await SparkSession.builder.getOrCreate()
@@ -912,7 +909,6 @@ struct DataFrameTests {
     #expect(try await df.collect() == expected)
     await spark.stop()
   }
-#endif
 
   @Test
   func storageLevel() async throws {

Tests/SparkConnectTests/IcebergTests.swift

@@ -99,14 +99,12 @@ struct IcebergTests {
         WHEN NOT MATCHED BY SOURCE THEN UPDATE SET data = 'invalid'
         """
       ).count()
-      #if !os(Linux)
-        let expected = [
-          Row(2, "updated"),
-          Row(3, "invalid"),
-          Row(4, "new"),
-        ]
-        #expect(try await spark.table(t2).collect() == expected)
-      #endif
+      let expected = [
+        Row(2, "updated"),
+        Row(3, "invalid"),
+        Row(4, "new"),
+      ]
+      #expect(try await spark.table(t2).collect() == expected)
     })
     await spark.stop()
   }

Tests/SparkConnectTests/SQLTests.swift

@@ -90,7 +90,6 @@ struct SQLTests {
     "variant.sql",
   ]
 
-#if !os(Linux)
   @Test
   func runAll() async throws {
     let spark = try await SparkSession.builder.getOrCreate()
@@ -120,5 +119,4 @@ struct SQLTests {
     }
     await spark.stop()
   }
-#endif
 }

Tests/SparkConnectTests/SparkSessionTests.swift

@@ -129,7 +129,6 @@ struct SparkSessionTests {
     await spark.stop()
   }
 
-#if !os(Linux)
   @Test
   func sql() async throws {
     await SparkSession.builder.clear()
@@ -196,7 +195,6 @@ struct SparkSessionTests {
     }
     await spark.stop()
   }
-#endif
 
   @Test
   func table() async throws {
@@ -215,10 +213,8 @@ struct SparkSessionTests {
     await SparkSession.builder.clear()
     let spark = try await SparkSession.builder.getOrCreate()
     #expect(try await spark.time(spark.range(1000).count) == 1000)
-#if !os(Linux)
     #expect(try await spark.time(spark.range(1).collect) == [Row(0)])
     try await spark.time(spark.range(10).show)
-#endif
     await spark.stop()
   }