feat(cache): KEP-2655 - Add build pipeline and address vulnerabilities for data_cache

.github/workflows/build-and-push-images.yaml

@@ -41,6 +41,9 @@ jobs:
           - component-name: torchtune-trainer
             dockerfile: cmd/trainers/torchtune/Dockerfile
             platforms: linux/amd64,linux/arm64
+          - component-name: data-cache
+            dockerfile: cmd/data_cache/Dockerfile
+            platforms: linux/amd64,linux/arm64
     steps:
       - name: Checkout
         uses: actions/checkout@v4

.github/workflows/test-rust.yaml

@@ -0,0 +1,27 @@
+name: Unit and Integration Test - Rust
+
+on:
+  - pull_request
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    if: ${{ github.repository == 'kubeflow/trainer' }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: "pkg/data_cache"
+
+      - name: Run Rust unit tests
+        run: make test-rust

Makefile

@@ -183,6 +183,10 @@ test-python-integration: ## Run Python integration test.
 
 	PYTHONPATH=$(PROJECT_DIR) pytest ./test/integration/initializers
 
+.PHONY: test-rust
+test-rust: ## Run Rust unit test.
+	cargo test --lib --bins --manifest-path ./pkg/data_cache/Cargo.toml
+
 .PHONY: test-e2e-setup-cluster
 test-e2e-setup-cluster: kind ## Setup Kind cluster for e2e test.
 	KIND=$(KIND) K8S_VERSION=$(K8S_VERSION) ./hack/e2e-setup-cluster.sh

pkg/data_cache/Cargo.toml

@@ -5,8 +5,8 @@ edition = "2024"
 
 [dependencies]
 tracing = "0.1.41"
-tracing-subscriber = "0.3"
-tokio = "1.41.1"
+tracing-subscriber = "0.3.20"
+tokio = "1.44.2"
 datafusion = "47.0.0"
 arrow = "55.0.0"
 arrow-schema = "55.0.0"

pkg/data_cache/src/head/provider.rs

@@ -231,7 +231,7 @@ impl ExecutionPlan for DataFileTableExec {
         &self.plan_properties
     }
 
-    fn children(&self) -> Vec<&Arc<(dyn ExecutionPlan + 'static)>> {
+    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan + 'static>> {
         vec![]
     }
 

pkg/data_cache/src/head/writer.rs

@@ -155,7 +155,7 @@ impl ExecutionPlan for DistributedWriterExec {
         &self.plan_properties
     }
 
-    fn children(&self) -> Vec<&Arc<(dyn ExecutionPlan + 'static)>> {
+    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan + 'static>> {
         vec![]
     }