diff --git a/.github/workflows/admission_webhook_test.yaml b/.github/workflows/admission_webhook_test.yaml index b8a7e3203d..b0bbd7e6bc 100644 --- a/.github/workflows/admission_webhook_test.yaml +++ b/.github/workflows/admission_webhook_test.yaml @@ -2,11 +2,10 @@ name: Test Admission Webhook on: pull_request: paths: - - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + - tests/install_KinD_create_KinD_cluster_install_kustomize.sh - .github/workflows/admission_webhook_test.yaml - apps/admission-webhook/upstream/** - - tests/gh-actions/install_istio*.sh - - tests/gh-actions/install_cert_manager.sh + - tests/istio* - common/cert-manager/** - common/istio*/** @@ -18,13 +17,13 @@ jobs: uses: actions/checkout@v4 - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - name: Install Istio - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/istio-cni_install.sh - name: Install cert-manager - run: ./tests/gh-actions/install_cert_manager.sh + run: ./tests/cert_manager_install.sh - name: Build & Apply manifests run: | diff --git a/.github/workflows/centraldashboard_test.yaml b/.github/workflows/centraldashboard_test.yaml index 9c65b434d2..4addba42d3 100644 --- a/.github/workflows/centraldashboard_test.yaml +++ b/.github/workflows/centraldashboard_test.yaml @@ -2,10 +2,10 @@ name: Test Central Dashboard on: pull_request: paths: - - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + - tests/install_KinD_create_KinD_cluster_install_kustomize.sh - .github/workflows/centraldashboard_test.yaml - apps/centraldashboard/upstream/** - - tests/gh-actions/install_istio*.sh + - tests/istio* - common/istio*/** jobs: @@ -16,10 +16,10 @@ jobs: uses: actions/checkout@v4 - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - name: Install Istio - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/istio-cni_install.sh - name: Create kubeflow namespace run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - diff --git a/.github/workflows/dex_oauth2-proxy_test.yaml b/.github/workflows/dex_oauth2-proxy_test.yaml index 0f3d871bd6..425b18beed 100644 --- a/.github/workflows/dex_oauth2-proxy_test.yaml +++ b/.github/workflows/dex_oauth2-proxy_test.yaml @@ -2,15 +2,15 @@ name: Test Dex and OAuth2 Proxy on: pull_request: paths: - - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh - - .github/workflows/dex_oauth2-proxy.yaml + - tests/install_KinD_create_KinD_cluster_install_kustomize.sh + - .github/workflows/dex_oauth2-proxy_test.yaml - common/cert-manager/** - common/oauth2-proxy/** - common/istio*/** - experimental/security/PSS/* - common/dex/base/** - - tests/gh-actions/install_istio*.sh - - tests/gh-actions/test_dex_login.py + - tests/istio* + - tests/dex_login_test.py jobs: build: @@ -20,16 +20,16 @@ jobs: uses: actions/checkout@v4 - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - name: Install cert-manager - run: ./tests/gh-actions/install_cert_manager.sh + run: ./tests/cert_manager_install.sh - name: Install Istio CNI - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/istio-cni_install.sh - name: Install oauth2-proxy - run: ./tests/gh-actions/install_oauth2-proxy.sh + run: ./tests/oauth2-proxy_install.sh - name: Create kubeflow namespace run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - @@ -38,19 +38,19 @@ jobs: run: kustomize build common/istio-cni-1-24/kubeflow-istio-resources/base | kubectl apply -f - - name: Install KF Multi Tenancy - run: ./tests/gh-actions/install_multi_tenancy.sh + run: ./tests/multi_tenancy_install.sh - name: Install dex - run: ./tests/gh-actions/install_dex.sh + run: ./tests/dex_install.sh - name: Install central-dashboard - run: ./tests/gh-actions/install_central_dashboard.sh + run: ./tests/central_dashboard_install.sh - name: Create KF Profile - run: ./tests/gh-actions/install_kubeflow_profile.sh + run: ./tests/kubeflow_profile_install.sh - name: Port forward the istio-ingress gateway - run: ./tests/gh-actions/port_forward_gateway.sh + run: ./tests/port_forward_gateway.sh - name: Test dex login run: | @@ -58,12 +58,12 @@ jobs: pip3 install -q requests # Run the Dex login test - python3 tests/gh-actions/test_dex_login.py + python3 tests/dex_login_test.py echo "Dex login test completed successfully." - name: Apply Pod Security Standards baseline levels for static namespaces - run: ./tests/gh-actions/enable_baseline_PSS.sh + run: ./tests/PSS_baseline_enable.sh - name: Unapply applied baseline labels run: | @@ -75,4 +75,4 @@ jobs: done - name: Applying Pod Security Standards restricted levels for static namespaces - run: ./tests/gh-actions/enable_restricted_PSS.sh + run: ./tests/PSS_restricted_enable.sh diff --git a/.github/workflows/full_kubeflow_integration_test.yaml b/.github/workflows/full_kubeflow_integration_test.yaml index f0365d8f96..7b4a81e0c5 100644 --- a/.github/workflows/full_kubeflow_integration_test.yaml +++ b/.github/workflows/full_kubeflow_integration_test.yaml @@ -23,46 +23,46 @@ jobs: uses: actions/checkout@v4 - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - name: Install kubectl - run: ./tests/gh-actions/install_kubectl.sh + run: ./tests/kubectl_install.sh - name: Create Kubeflow Namespace run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - - name: Install Certificate Manager - run: ./tests/gh-actions/install_cert_manager.sh + run: ./tests/cert_manager_install.sh - name: Install Istio CNI - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/istio-cni_install.sh - name: Install OAuth2 Proxy - run: ./tests/gh-actions/install_oauth2-proxy.sh + run: ./tests/oauth2-proxy_install.sh - name: Install Kubeflow Istio Resources run: kustomize build common/istio-cni-1-24/kubeflow-istio-resources/base | kubectl apply -f - - name: Install Multi-Tenancy - run: ./tests/gh-actions/install_multi_tenancy.sh + run: ./tests/multi_tenancy_install.sh - name: Install Dex - run: ./tests/gh-actions/install_dex.sh + run: ./tests/dex_install.sh - name: Install Central Dashboard - run: ./tests/gh-actions/install_central_dashboard.sh + run: ./tests/central_dashboard_install.sh - name: Install Knative - run: ./tests/gh-actions/install_knative-cni.sh + run: ./tests/knative-cni_install.sh - name: Install KServe - run: ./tests/gh-actions/install_kserve.sh + run: ./tests/kserve_install.sh - name: Install Pipelines - run: ./tests/gh-actions/install_pipelines.sh + run: ./tests/pipelines_install.sh - name: Create KF Profile - run: ./tests/gh-actions/install_kubeflow_profile.sh + run: ./tests/kubeflow_profile_install.sh - name: Install Jupyter Web Application run: kustomize build apps/jupyter/jupyter-web-app/upstream/overlays/istio/ | kubectl apply -f - @@ -77,13 +77,13 @@ jobs: run: kubectl get crd poddefaults.kubeflow.org || kubectl apply -f https://raw.githubusercontent.com/kubeflow/kubeflow/master/components/admission-webhook/manifests/base/crd.yaml - name: Install Volumes Web Application - run: ./tests/gh-actions/install_volumes_web_application.sh + run: ./tests/volumes_web_application_install.sh - name: Install Katib - run: ./tests/gh-actions/install_katib.sh + run: ./tests/katib_install.sh - name: Install Training Operator - run: ./tests/gh-actions/install_training_operator.sh + run: ./tests/training_operator_install.sh - name: Install Model Registry run: | @@ -95,67 +95,67 @@ jobs: kustomize build apps/model-registry/upstream/options/ui/overlays/istio | kubectl apply -n kubeflow -f - - name: Install Spark - run: chmod u+x tests/gh-actions/*.sh && ./tests/gh-actions/install_spark.sh + run: chmod u+x tests/*.sh && ./tests/spark_install.sh - name: Wait for All Pods to be Ready run: kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout 60s --field-selector=status.phase!=Succeeded - name: Port-forward the istio-ingress gateway - run: ./tests/gh-actions/port_forward_gateway.sh + run: ./tests/port_forward_gateway.sh # name: Setup OAuth2 and Dex Credentials - # run: chmod +x tests/gh-actions/oauth2_dex_credentials.sh && ./tests/gh-actions/oauth2_dex_credentials.sh + # run: chmod +x tests/oauth2_dex_credentials.sh && ./tests/oauth2_dex_credentials.sh - name: Test Dex Login run: | pip3 install -q requests - python3 tests/gh-actions/test_dex_login.py + python3 tests/dex_login_test.py echo "Dex login test completed successfully." - name: V1 Pipeline Test run: | pip3 install "kfp>=1.8.22,<2.0.0" TOKEN="$(kubectl -n $KF_PROFILE create token default-editor)" - python3 tests/gh-actions/test_pipeline_v1.py "${TOKEN}" "${KF_PROFILE}" + python3 tests/pipeline_v1_test.py "${TOKEN}" "${KF_PROFILE}" - name: V2 Pipeline Test run: | pip3 install -U "kfp>=2.13.0" TOKEN="$(kubectl -n $KF_PROFILE create token default-editor)" - python3 tests/gh-actions/test_pipeline_v2.py run_pipeline "${TOKEN}" "${KF_PROFILE}" + python3 tests/pipeline_v2_test.py run_pipeline "${TOKEN}" "${KF_PROFILE}" - name: Test Pipeline Access with Unauthorized Token run: | kubectl create namespace test-unauthorized kubectl create serviceaccount test-unauthorized -n test-unauthorized UNAUTHORIZED_TOKEN=$(kubectl -n test-unauthorized create token test-unauthorized) - python3 tests/gh-actions/test_pipeline_v2.py test_unauthorized_access "$UNAUTHORIZED_TOKEN" "${KF_PROFILE}" + python3 tests/pipeline_v2_test.py test_unauthorized_access "$UNAUTHORIZED_TOKEN" "${KF_PROFILE}" - name: Test Volumes Web Application API - run: ./tests/gh-actions/test_volumes_web_application.sh "${KF_PROFILE}" + run: ./tests/volumes_web_application_test.sh "${KF_PROFILE}" - name: Apply PodDefault for Pipeline Access Token - run: sed "s/kubeflow-user-example-com/$KF_PROFILE/g" tests/gh-actions/kf-objects/poddefaults.access-ml-pipeline.kubeflow-user-example-com.yaml | kubectl apply -f - + run: sed "s/kubeflow-user-example-com/$KF_PROFILE/g" tests/poddefaults.access-ml-pipeline.kubeflow-user-example-com.yaml | kubectl apply -f - - name: Create Test Notebook run: | - sed "s/kubeflow-user-example-com/$KF_PROFILE/g" tests/gh-actions/kf-objects/notebook.test.kubeflow-user-example.com.yaml | kubectl apply -f - + sed "s/kubeflow-user-example-com/$KF_PROFILE/g" tests/notebook.test.kubeflow-user-example.com.yaml | kubectl apply -f - kubectl wait --for=condition=Ready pod -l app=test -n $KF_PROFILE --timeout=300s - name: Copy and execute the pipeline run script in KF Notebook run: | - cp tests/gh-actions/run_and_wait_kubeflow_pipeline.py /tmp/run_pipeline_temp.py + cp tests/pipeline_run_and_wait_kubeflow.py /tmp/run_pipeline_temp.py sed -i "s/experiment_namespace = \"kubeflow-user-example-com\"/experiment_namespace = \"$KF_PROFILE\"/g" /tmp/run_pipeline_temp.py sed -i 's/except Exception:/except Exception as e:/g' /tmp/run_pipeline_temp.py sed -i 's/logger.info("Experiment not found, trying to create experiment.")/logger.info("Experiment not found, trying to create experiment. Error: " + str(e))/g' /tmp/run_pipeline_temp.py - kubectl -n $KF_PROFILE cp /tmp/run_pipeline_temp.py test-0:/home/jovyan/run_and_wait_kubeflow_pipeline.py + kubectl -n $KF_PROFILE cp /tmp/run_pipeline_temp.py test-0:/home/jovyan/pipeline_run_and_wait_kubeflow.py - kubectl -n $KF_PROFILE exec test-0 -- python /home/jovyan/run_and_wait_kubeflow_pipeline.py + kubectl -n $KF_PROFILE exec test-0 -- python /home/jovyan/pipeline_run_and_wait_kubeflow.py - name: Run Katib Test run: | - kubectl apply -f tests/gh-actions/kf-objects/katib_test.yaml + kubectl apply -f tests/katib_test.yaml kubectl wait --for=condition=Running experiments.kubeflow.org -n $KF_PROFILE --all --timeout=300s echo "Waiting for all Trials to be Completed..." kubectl wait --for=condition=Created trials.kubeflow.org -n $KF_PROFILE --all --timeout=60s @@ -164,14 +164,14 @@ jobs: kubectl get trials.kubeflow.org -n $KF_PROFILE - name: Run Training Operator Test - run: ./tests/gh-actions/test_training_operator.sh "${KF_PROFILE}" + run: ./tests/training_operator_test.sh "${KF_PROFILE}" - name: Run KServe Test run: | - ./tests/gh-actions/test_kserve.sh ${KF_PROFILE} + ./tests/kserve_test.sh ${KF_PROFILE} - name: Run Spark Test - run: chmod u+x tests/gh-actions/*.sh && ./tests/gh-actions/test_spark.sh "${KF_PROFILE}" + run: chmod u+x tests/*.sh && ./tests/spark_test.sh "${KF_PROFILE}" - name: Test Model Registry Deployment run: | @@ -210,7 +210,7 @@ jobs: fi - name: Apply Pod Security Standards Baseline - run: ./tests/gh-actions/enable_baseline_PSS.sh + run: ./tests/PSS_baseline_enable.sh - name: Remove Pod Security Labels run: | @@ -220,11 +220,11 @@ jobs: done - name: Apply Pod Security Standards Restricted - run: ./tests/gh-actions/enable_restricted_PSS.sh + run: ./tests/PSS_restricted_enable.sh - name: Run Non-Root Test run: | - [ -f "tests/gh-actions/runasnonroot.sh" ] && chmod +x tests/gh-actions/runasnonroot.sh && ./tests/gh-actions/runasnonroot.sh + [ -f "tests/runasnonroot.sh" ] && chmod +x tests/runasnonroot.sh && ./tests/runasnonroot.sh - name: Verify Components run: kubectl get pods --all-namespaces | grep -E '(Error|CrashLoopBackOff)' && exit 1 || true diff --git a/.github/workflows/jupyter_web_application_test.yaml b/.github/workflows/jupyter_web_application_test.yaml index 72bf92d5e1..772bc25284 100644 --- a/.github/workflows/jupyter_web_application_test.yaml +++ b/.github/workflows/jupyter_web_application_test.yaml @@ -2,10 +2,10 @@ name: Test Jupyter Web Application on: pull_request: paths: - - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + - tests/install_KinD_create_KinD_cluster_install_kustomize.sh - .github/workflows/jupyter_web_application_test.yaml - apps/jupyter/jupyter-web-app/upstream/** - - tests/gh-actions/install_istio*.sh + - tests/istio* - common/istio*/** jobs: @@ -16,10 +16,10 @@ jobs: uses: actions/checkout@v4 - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - name: Install Istio - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/istio-cni_install.sh - name: Build & Apply manifests run: | diff --git a/.github/workflows/katib_test.yaml b/.github/workflows/katib_test.yaml index 62f80e6832..7938ae4908 100644 --- a/.github/workflows/katib_test.yaml +++ b/.github/workflows/katib_test.yaml @@ -2,13 +2,12 @@ name: Deploy and Test Katib on: pull_request: paths: - - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh - - tests/gh-actions/install_katib.sh + - tests/install_KinD_create_KinD_cluster_install_kustomize.sh + - tests/katib_install.sh - .github/workflows/katib_test.yaml - apps/katib/upstream/** - - tests/gh-actions/install_istio-cni.sh - common/istio*/** - - tests/gh-actions/install_cert_manager.sh + - tests/istio* - common/cert-manager/** - experimental/security/PSS/* @@ -23,45 +22,45 @@ jobs: uses: actions/checkout@v4 - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - name: Install kubectl - run: ./tests/gh-actions/install_kubectl.sh + run: ./tests/kubectl_install.sh - name: Create Kubeflow Namespace run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - - name: Install Certificate Manager - run: ./tests/gh-actions/install_cert_manager.sh + run: ./tests/cert_manager_install.sh - name: Install Istio CNI - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/istio-cni_install.sh - name: Install OAuth2 Proxy - run: ./tests/gh-actions/install_oauth2-proxy.sh + run: ./tests/oauth2-proxy_install.sh - name: Install Kubeflow Istio Resources run: kustomize build common/istio-cni-1-24/kubeflow-istio-resources/base | kubectl apply -f - - name: Install Multi-Tenancy - run: ./tests/gh-actions/install_multi_tenancy.sh + run: ./tests/multi_tenancy_install.sh - name: Create KF Profile - run: ./tests/gh-actions/install_kubeflow_profile.sh + run: ./tests/kubeflow_profile_install.sh - name: Install Katib - run: ./tests/gh-actions/install_katib.sh + run: ./tests/katib_install.sh - name: Install Dependencies run: pip install pytest kubernetes kfp==2.13.0 requests - name: Port-forward the istio-ingress gateway - run: ./tests/gh-actions/port_forward_gateway.sh + run: ./tests/port_forward_gateway.sh - name: Run Katib Test run: | - kubectl apply -f tests/gh-actions/kf-objects/katib_test.yaml + kubectl apply -f tests/katib_test.yaml kubectl wait --for=condition=Running experiments.kubeflow.org -n $KF_PROFILE --all --timeout=300s echo "Waiting for all Trials to be Completed..." kubectl wait --for=condition=Created trials.kubeflow.org -n $KF_PROFILE --all --timeout=60s @@ -80,7 +79,7 @@ jobs: kubectl get experiments.kubeflow.org -n $KF_PROFILE --token="$UNAUTHORIZED_TOKEN" >/dev/null - name: Apply Pod Security Standards baseline levels - run: ./tests/gh-actions/enable_baseline_PSS.sh + run: ./tests/PSS_baseline_enable.sh - name: Unapply applied baseline labels run: | @@ -92,4 +91,4 @@ jobs: done - name: Applying Pod Security Standards restricted levels - run: ./tests/gh-actions/enable_restricted_PSS.sh + run: ./tests/PSS_restricted_enable.sh diff --git a/.github/workflows/kserve_test.yaml b/.github/workflows/kserve_test.yaml index 4ba69f0a42..aced9ab0ea 100644 --- a/.github/workflows/kserve_test.yaml +++ b/.github/workflows/kserve_test.yaml @@ -2,20 +2,19 @@ name: Test KServe on: pull_request: paths: - - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + - tests/install_KinD_create_KinD_cluster_install_kustomize.sh - .github/workflows/kserve_m2m_test.yaml - apps/kserve/** - - tests/gh-actions/kserve/** - - tests/gh-actions/test_kserve.sh - - tests/gh-actions/install_kserve.sh + - tests/kserve/** + - tests/kserve_test.sh + - tests/kserve_install.sh - common/istio*/** - - tests/gh-actions/install_istio*.sh - common/oauth2-proxy/** - - tests/gh-actions/install_oauth2-proxy.sh + - tests/oauth2-proxy_install.sh - common/cert-manager/** - - tests/gh-actions/install_cert_manager.sh + - tests/istio* - common/knative/** - - tests/gh-actions/install_knative*.sh + - tests/knative-cni_install.sh jobs: build: @@ -25,37 +24,37 @@ jobs: uses: actions/checkout@v4 - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - name: Install kubectl - run: ./tests/gh-actions/install_kubectl.sh + run: ./tests/kubectl_install.sh - name: Install Istio CNI - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/istio-cni_install.sh - name: Install oauth2-proxy - run: ./tests/gh-actions/install_oauth2-proxy.sh + run: ./tests/oauth2-proxy_install.sh - name: Install cert-manager - run: ./tests/gh-actions/install_cert_manager.sh + run: ./tests/cert_manager_install.sh - name: Create kubeflow namespace run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - - name: Install knative CNI - run: ./tests/gh-actions/install_knative-cni.sh + run: ./tests/knative-cni_install.sh - name: Install KServe - run: ./tests/gh-actions/install_kserve.sh + run: ./tests/kserve_install.sh - name: Install KF Multi Tenancy - run: ./tests/gh-actions/install_multi_tenancy.sh + run: ./tests/multi_tenancy_install.sh - name: Install kubeflow-istio-resources run: kustomize build common/istio-cni-1-24/kubeflow-istio-resources/base | kubectl apply -f - - name: Create KF Profile - run: ./tests/gh-actions/install_kubeflow_profile.sh + run: ./tests/kubeflow_profile_install.sh - name: Setup python 3.12 uses: actions/setup-python@v4 @@ -63,10 +62,10 @@ jobs: python-version: 3.12 - name: Port forward - run: ./tests/gh-actions/port_forward_gateway.sh + run: ./tests/port_forward_gateway.sh - name: Run KServe tests - run: ./tests/gh-actions/test_kserve.sh kubeflow-user-example-com + run: ./tests/kserve_test.sh kubeflow-user-example-com - name: Detailed KServe Access Diagnostics run: | @@ -88,7 +87,7 @@ jobs: #- name: Run and fail kserve tests without kserve m2m token #run: | # export KSERVE_INGRESS_HOST_PORT=localhost:8080 - # cd ./tests/gh-actions/kserve + # cd ./tests/kserve # if pytest . -vs --log-level info; then # echo "This test should fail with an HTTP redirect to oauth2-proxy/dex auth."; exit 1 # else @@ -120,7 +119,7 @@ jobs: kubectl wait --for=condition=Available --timeout=300s -n kubeflow deployment/kserve-models-web-app - name: Apply Pod Security Standards baseline levels - run: ./tests/gh-actions/enable_baseline_PSS.sh + run: ./tests/PSS_baseline_enable.sh - name: Unapply applied baseline labels run: | @@ -132,4 +131,4 @@ jobs: done - name: Applying Pod Security Standards restricted levels - run: ./tests/gh-actions/enable_restricted_PSS.sh + run: ./tests/PSS_restricted_enable.sh diff --git a/.github/workflows/manifests_example_test.yaml b/.github/workflows/manifests_example_test.yaml index 6f4bb32a0a..819a74ca0c 100644 --- a/.github/workflows/manifests_example_test.yaml +++ b/.github/workflows/manifests_example_test.yaml @@ -15,7 +15,7 @@ jobs: uses: actions/checkout@v4 - name: Install kustomize - run: ./tests/gh-actions/install_kustomize.sh + run: ./tests/kustomize_install.sh - name: Unit Test run: | diff --git a/.github/workflows/model_registry_test.yaml b/.github/workflows/model_registry_test.yaml index c3eaac7c4e..5694ab4039 100644 --- a/.github/workflows/model_registry_test.yaml +++ b/.github/workflows/model_registry_test.yaml @@ -4,10 +4,10 @@ name: Test Model Registry on: pull_request: paths: - - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + - tests/install_KinD_create_KinD_cluster_install_kustomize.sh - .github/workflows/model_registry_test.yaml - apps/model-registry/upstream/** - - tests/gh-actions/install_istio*.sh + - tests/istio* - common/istio*/** jobs: @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@v4 - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - name: Remove AppArmor profile for mysql in KinD on GHA # https://github.com/kubeflow/manifests/issues/2507 run: | @@ -27,22 +27,22 @@ jobs: sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - name: Install kubectl - run: ./tests/gh-actions/install_kubectl.sh + run: ./tests/kubectl_install.sh - name: Create kubeflow namespace run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - - name: Install Istio - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/istio-cni_install.sh - name: Install oauth2-proxy - run: ./tests/gh-actions/install_oauth2-proxy.sh + run: ./tests/oauth2-proxy_install.sh - name: Install cert-manager - run: ./tests/gh-actions/install_cert_manager.sh + run: ./tests/cert_manager_install.sh - name: Install KF Multi Tenancy - run: ./tests/gh-actions/install_multi_tenancy.sh + run: ./tests/multi_tenancy_install.sh - name: Install kubeflow-istio-resources run: kustomize build common/istio-1-24/kubeflow-istio-resources/base | kubectl apply -f - diff --git a/.github/workflows/notebook_controller_test.yaml b/.github/workflows/notebook_controller_test.yaml index 76c48deacd..6846597094 100644 --- a/.github/workflows/notebook_controller_test.yaml +++ b/.github/workflows/notebook_controller_test.yaml @@ -2,14 +2,14 @@ name: Test Notebook Controller on: pull_request: paths: - - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + - tests/install_KinD_create_KinD_cluster_install_kustomize.sh - .github/workflows/notebook_controller_m2m_test.yaml - apps/jupyter/** - common/oauth2-proxy/** - common/istio*/** - - tests/gh-actions/install_istio*.sh - - tests/gh-actions/install_oauth2-proxy.sh - - tests/gh-actions/install_multi_tenancy.sh + - tests/istio* + - tests/oauth2-proxy_install.sh + - tests/multi_tenancy_install.sh jobs: build: @@ -19,25 +19,25 @@ jobs: uses: actions/checkout@v4 - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - name: Install kubectl - run: ./tests/gh-actions/install_kubectl.sh + run: ./tests/kubectl_install.sh - name: Create kubeflow namespace run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - - name: Install Istio - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/istio-cni_install.sh - name: Install oauth2-proxy - run: ./tests/gh-actions/install_oauth2-proxy.sh + run: ./tests/oauth2-proxy_install.sh - name: Install kubeflow-istio-resources run: kustomize build common/istio-1-24/kubeflow-istio-resources/base | kubectl apply -f - - name: Install KF Multi Tenancy - run: ./tests/gh-actions/install_multi_tenancy.sh + run: ./tests/multi_tenancy_install.sh - name: Build & Apply manifests run: | diff --git a/.github/workflows/pipeline_run_from_notebook.yaml b/.github/workflows/pipeline_run_from_notebook.yaml index 0bcf3bafae..5a48953c4e 100644 --- a/.github/workflows/pipeline_run_from_notebook.yaml +++ b/.github/workflows/pipeline_run_from_notebook.yaml @@ -2,12 +2,11 @@ name: Test Pipeline run from Jupyterlab on: pull_request: paths: - - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + - tests/install_KinD_create_KinD_cluster_install_kustomize.sh - .github/workflows/pipeline_run_from_notebook.yaml - apps/jupyter/notebook-controller/upstream/** - apps/pipeline/upstream/** - - tests/gh-actions/install_istio*.sh - - tests/gh-actions/install_cert_manager.sh + - tests/istio* - common/cert-manager/** - common/oauth2-proxy/** - common/istio*/** @@ -22,16 +21,16 @@ jobs: uses: actions/checkout@v4 - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - name: Install Istio - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/istio-cni_install.sh - name: Install oauth2-proxy - run: ./tests/gh-actions/install_oauth2-proxy.sh + run: ./tests/oauth2-proxy_install.sh - name: Install cert-manager - run: ./tests/gh-actions/install_cert_manager.sh + run: ./tests/cert_manager_install.sh - name: Create kubeflow namespace run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - @@ -40,10 +39,10 @@ jobs: run: kustomize build common/istio-cni-1-24/kubeflow-istio-resources/base | kubectl apply -f - - name: Install KF Pipelines - run: ./tests/gh-actions/install_pipelines.sh + run: ./tests/pipelines_install.sh - name: Install KF Multi Tenancy - run: ./tests/gh-actions/install_multi_tenancy.sh + run: ./tests/multi_tenancy_install.sh - name: Build & Apply manifests run: | @@ -54,29 +53,29 @@ jobs: --field-selector=status.phase!=Succeeded - name: Create KF Profile - run: ./tests/gh-actions/install_kubeflow_profile.sh + run: ./tests/kubeflow_profile_install.sh - name: Apply PodDefaults to access ml-pipeline with projected token - run: kubectl apply -f tests/gh-actions/kf-objects/poddefaults.access-ml-pipeline.kubeflow-user-example-com.yaml + run: kubectl apply -f tests/poddefaults.access-ml-pipeline.kubeflow-user-example-com.yaml - name: Create Kubeflow Notebook with PodDefaults run: | - kubectl apply -f tests/gh-actions/kf-objects/notebook.test.kubeflow-user-example.com.yaml + kubectl apply -f tests/notebook.test.kubeflow-user-example.com.yaml kubectl wait --for=jsonpath='{.status.readyReplicas}'=1 \ - -f tests/gh-actions/kf-objects/notebook.test.kubeflow-user-example.com.yaml \ + -f tests/notebook.test.kubeflow-user-example.com.yaml \ --timeout 600s - name: Copy and execute the pipeline run script in KF Notebook run: | kubectl -n kubeflow-user-example-com cp \ - ./tests/gh-actions/run_and_wait_kubeflow_pipeline.py \ - test-0:/home/jovyan/run_and_wait_kubeflow_pipeline.py + ./tests/pipeline_run_and_wait_kubeflow.py \ + test-0:/home/jovyan/pipeline_run_and_wait_kubeflow.py kubectl -n kubeflow-user-example-com exec -ti \ - test-0 -- python /home/jovyan/run_and_wait_kubeflow_pipeline.py + test-0 -- python /home/jovyan/pipeline_run_and_wait_kubeflow.py - name: Apply Pod Security Standards baseline levels - run: ./tests/gh-actions/enable_baseline_PSS.sh + run: ./tests/PSS_baseline_enable.sh - name: Unapply applied baseline labels run: | @@ -88,4 +87,4 @@ jobs: done - name: Applying Pod Security Standards restricted levels - run: ./tests/gh-actions/enable_restricted_PSS.sh + run: ./tests/PSS_restricted_enable.sh diff --git a/.github/workflows/pipeline_swfs_test.yaml b/.github/workflows/pipeline_swfs_test.yaml index 3350a802a5..bda6a51f36 100644 --- a/.github/workflows/pipeline_swfs_test.yaml +++ b/.github/workflows/pipeline_swfs_test.yaml @@ -3,18 +3,17 @@ on: workflow_dispatch: pull_request: paths: - - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + - tests/install_KinD_create_KinD_cluster_install_kustomize.sh - .github/workflows/pipeline_swfs_test.yaml - apps/pipeline/upstream/** - - tests/gh-actions/install_istio*.sh - - tests/gh-actions/install_cert_manager.sh - - tests/gh-actions/install_oauth2-proxy.sh + - tests/istio* + - tests/oauth2-proxy_install.sh - common/cert-manager/** - common/oauth2-proxy/** - common/istio*/** - experimental/seaweedfs/** - - tests/gh-actions/test_swfs_namespace_isolation.sh - - tests/gh-actions/s3_test_helper.py + - tests/swfs_namespace_isolation_test.sh + - tests/s3_helper_test.py jobs: build: @@ -37,34 +36,34 @@ jobs: run: docker system prune -a --volumes --force - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - name: Install kubectl - run: ./tests/gh-actions/install_kubectl.sh + run: ./tests/kubectl_install.sh - name: Install Istio - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/istio-cni_install.sh - name: Install oauth2-proxy - run: ./tests/gh-actions/install_oauth2-proxy.sh + run: ./tests/oauth2-proxy_install.sh - name: Install cert-manager - run: ./tests/gh-actions/install_cert_manager.sh + run: ./tests/cert_manager_install.sh - name: Create kubeflow namespace run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - - name: Install KF Pipelines - run: ./tests/gh-actions/install_pipelines_swfs.sh + run: ./tests/pipelines_swfs_install.sh - name: Install KF Multi Tenancy - run: ./tests/gh-actions/install_multi_tenancy.sh + run: ./tests/multi_tenancy_install.sh - name: Install kubeflow-istio-resources run: kustomize build common/istio-cni-1-24/kubeflow-istio-resources/base | kubectl apply -f - - name: Create KF Profile - run: ./tests/gh-actions/install_kubeflow_profile.sh + run: ./tests/kubeflow_profile_install.sh - name: Verify Pipeline Integration run: | @@ -77,14 +76,14 @@ jobs: - name: Port forward - run: ./tests/gh-actions/port_forward_gateway.sh + run: ./tests/port_forward_gateway.sh - name: List and deploy test pipeline with V1 API run: | pip3 install "kfp>=1.8.22,<2.0.0" KF_PROFILE=kubeflow-user-example-com TOKEN="$(kubectl -n $KF_PROFILE create token default-editor)" - python3 tests/gh-actions/test_pipeline_v1.py "${TOKEN}" "${KF_PROFILE}" + python3 tests/pipeline_v1_test.py "${TOKEN}" "${KF_PROFILE}" - name: Prune images inside Kind cluster run: docker exec kind-control-plane bash -c "crictl images prune" @@ -94,21 +93,21 @@ jobs: pip3 install kfp==2.13.0 KF_PROFILE=kubeflow-user-example-com TOKEN="$(kubectl -n $KF_PROFILE create token default-editor)" - python3 tests/gh-actions/test_pipeline_v2.py run_pipeline "${TOKEN}" "${KF_PROFILE}" + python3 tests/pipeline_v2_test.py run_pipeline "${TOKEN}" "${KF_PROFILE}" - name: Fail to list pipelines with unauthorized ServiceAccount Token (V2 API) run: | pip3 install kfp==2.13.0 KF_PROFILE=kubeflow-user-example-com TOKEN="$(kubectl -n default create token default)" - python3 tests/gh-actions/test_pipeline_v2.py test_unauthorized_access "${TOKEN}" "${KF_PROFILE}" + python3 tests/pipeline_v2_test.py test_unauthorized_access "${TOKEN}" "${KF_PROFILE}" echo "Test succeeded. Token from unauthorized ServiceAccount cannot list pipelines in $KF_PROFILE namespace." - name: Test SeaweedFS Namespace Isolation - run: ./tests/gh-actions/test_swfs_namespace_isolation.sh + run: ./tests/swfs_namespace_isolation_test.sh - name: Apply Pod Security Standards baseline levels for static namespaces - run: ./tests/gh-actions/enable_baseline_PSS.sh + run: ./tests/PSS_baseline_enable.sh - name: Unapply applied baseline labels run: | @@ -120,7 +119,7 @@ jobs: done - name: Applying Pod Security Standards restricted levels for static namespaces - run: ./tests/gh-actions/enable_restricted_PSS.sh + run: ./tests/PSS_restricted_enable.sh - name: Collect Logs on Failure if: failure() diff --git a/.github/workflows/pipeline_test.yaml b/.github/workflows/pipeline_test.yaml index 11e37ed3c1..aa6a067dda 100644 --- a/.github/workflows/pipeline_test.yaml +++ b/.github/workflows/pipeline_test.yaml @@ -2,17 +2,16 @@ name: Test Pipelines on: pull_request: paths: - - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + - tests/install_KinD_create_KinD_cluster_install_kustomize.sh - .github/workflows/pipeline_test.yaml - apps/pipeline/upstream/** - - tests/gh-actions/install_istio*.sh - - tests/gh-actions/install_cert_manager.sh - - tests/gh-actions/install_oauth2-proxy.sh + - tests/istio* + - tests/oauth2-proxy_install.sh - common/cert-manager/** - common/oauth2-proxy/** - common/istio*/** - - tests/gh-actions/test_pipeline_v1.py - - tests/gh-actions/test_pipeline_v2.py + - tests/pipeline_v1_test.py + - tests/pipeline_v2_test.py - experimental/security/PSS/* env: @@ -27,34 +26,34 @@ jobs: uses: actions/checkout@v4 - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - name: Install kubectl - run: ./tests/gh-actions/install_kubectl.sh + run: ./tests/kubectl_install.sh - name: Install Istio - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/istio-cni_install.sh - name: Install oauth2-proxy - run: ./tests/gh-actions/install_oauth2-proxy.sh + run: ./tests/oauth2-proxy_install.sh - name: Install cert-manager - run: ./tests/gh-actions/install_cert_manager.sh + run: ./tests/cert_manager_install.sh - name: Create kubeflow namespace run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - - name: Install KF Pipelines - run: ./tests/gh-actions/install_pipelines.sh + run: ./tests/pipelines_install.sh - name: Install KF Multi Tenancy - run: ./tests/gh-actions/install_multi_tenancy.sh + run: ./tests/multi_tenancy_install.sh - name: Install kubeflow-istio-resources run: kustomize build common/istio-cni-1-24/kubeflow-istio-resources/base | kubectl apply -f - - name: Create KF Profile - run: ./tests/gh-actions/install_kubeflow_profile.sh + run: ./tests/kubeflow_profile_install.sh - name: Verify Pipeline Integration run: | @@ -68,28 +67,28 @@ jobs: run: kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout 60s --field-selector=status.phase!=Succeeded - name: Port forward - run: ./tests/gh-actions/port_forward_gateway.sh + run: ./tests/port_forward_gateway.sh - name: V1 Pipeline Test run: | pip3 install "kfp>=1.8.22,<2.0.0" TOKEN="$(kubectl -n $KF_PROFILE create token default-editor)" - python3 tests/gh-actions/test_pipeline_v1.py "${TOKEN}" "${KF_PROFILE}" + python3 tests/pipeline_v1_test.py "${TOKEN}" "${KF_PROFILE}" - name: V2 Pipeline Test run: | pip3 install "kfp>=2.13.0" TOKEN="$(kubectl -n $KF_PROFILE create token default-editor)" - python3 tests/gh-actions/test_pipeline_v2.py run_pipeline "${TOKEN}" "${KF_PROFILE}" + python3 tests/pipeline_v2_test.py run_pipeline "${TOKEN}" "${KF_PROFILE}" - name: Test unauthorized access run: | TOKEN="$(kubectl -n default create token default)" - python3 tests/gh-actions/test_pipeline_v2.py test_unauthorized_access "${TOKEN}" "${KF_PROFILE}" + python3 tests/pipeline_v2_test.py test_unauthorized_access "${TOKEN}" "${KF_PROFILE}" echo "Test succeeded. Token from unauthorized ServiceAccount cannot list pipelines in $KF_PROFILE namespace." - name: Apply Pod Security Standards baseline levels for static namespaces - run: ./tests/gh-actions/enable_baseline_PSS.sh + run: ./tests/PSS_baseline_enable.sh - name: Unapply applied baseline labels run: | @@ -101,4 +100,4 @@ jobs: done - name: Applying Pod Security Standards restricted levels for static namespaces - run: ./tests/gh-actions/enable_restricted_PSS.sh + run: ./tests/PSS_restricted_enable.sh diff --git a/.github/workflows/profiles_test.yaml b/.github/workflows/profiles_test.yaml index a22803fd7f..2dba69aeb4 100644 --- a/.github/workflows/profiles_test.yaml +++ b/.github/workflows/profiles_test.yaml @@ -2,10 +2,10 @@ name: Test Profiles on: pull_request: paths: - - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + - tests/install_KinD_create_KinD_cluster_install_kustomize.sh - .github/workflows/profiles_test.yaml - apps/profiles/upstream/** - - tests/gh-actions/install_istio*.sh + - tests/istio* - common/istio*/** jobs: @@ -16,10 +16,10 @@ jobs: uses: actions/checkout@v4 - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - name: Install Istio - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/istio-cni_install.sh - name: Build & Apply manifests run: | diff --git a/.github/workflows/ray_test.yaml b/.github/workflows/ray_test.yaml index f72be1567a..73248203e2 100644 --- a/.github/workflows/ray_test.yaml +++ b/.github/workflows/ray_test.yaml @@ -2,12 +2,11 @@ name: Test Ray on: pull_request: paths: - - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + - tests/install_KinD_create_KinD_cluster_install_kustomize.sh - .github/workflows/ray_test.yaml - experimental/ray/** - - tests/gh-actions/install_istio*.sh - - tests/gh-actions/install_cert_manager.sh - - tests/gh-actions/install_oauth2-proxy.sh + - tests/istio* + - tests/oauth2-proxy_install.sh - common/cert-manager/** - common/oauth2-proxy/** - common/istio*/** @@ -21,22 +20,22 @@ jobs: uses: actions/checkout@v4 - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - name: Install Istio - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/istio-cni_install.sh - name: Install oauth2-proxy - run: ./tests/gh-actions/install_oauth2-proxy.sh + run: ./tests/oauth2-proxy_install.sh - name: Install cert-manager - run: ./tests/gh-actions/install_cert_manager.sh + run: ./tests/cert_manager_install.sh - name: Create kubeflow namespace run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - - name: Install KF Multi Tenancy - run: ./tests/gh-actions/install_multi_tenancy.sh + run: ./tests/multi_tenancy_install.sh - name: Create KF Profile run: kustomize build common/user-namespace/base | kubectl apply -f - diff --git a/.github/workflows/spark_test.yaml b/.github/workflows/spark_test.yaml index e53e8dfaab..3665554317 100644 --- a/.github/workflows/spark_test.yaml +++ b/.github/workflows/spark_test.yaml @@ -2,13 +2,12 @@ name: Test Spark on: pull_request: paths: - - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + - tests/install_KinD_create_KinD_cluster_install_kustomize.sh - .github/workflows/test_spark.yaml - apps/spark/** - - tests/gh-actions/spark*.sh - - tests/gh-actions/install_istio*.sh - - tests/gh-actions/install_cert_manager.sh - - tests/gh-actions/install_oauth2-proxy.sh + - tests/spark*.sh + - tests/istio* + - tests/oauth2-proxy_install.sh - common/cert-manager/** - common/oauth2-proxy/** - common/istio*/** @@ -22,22 +21,22 @@ jobs: uses: actions/checkout@v4 - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - name: Install Istio - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/istio-cni_install.sh - name: Install oauth2-proxy - run: ./tests/gh-actions/install_oauth2-proxy.sh + run: ./tests/oauth2-proxy_install.sh - name: Install cert-manager - run: ./tests/gh-actions/install_cert_manager.sh + run: ./tests/cert_manager_install.sh - name: Create kubeflow namespace run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - - name: Install KF Multi Tenancy - run: ./tests/gh-actions/install_multi_tenancy.sh + run: ./tests/multi_tenancy_install.sh - name: Create KF Profile run: kustomize build common/user-namespace/base | kubectl apply -f - @@ -46,7 +45,7 @@ jobs: run: | cd apps/spark # TODO remove the debugging lines - ls -lah ../../tests/gh-actions/ - chmod u+x ../../tests/gh-actions/*.sh - ../../tests/gh-actions/install_spark.sh - ../../tests/gh-actions/test_spark.sh "kubeflow-user-example-com" + ls -lah ../../tests/ + chmod u+x ../../tests/*.sh + ../../tests/spark_install.sh + ../../tests/spark_test.sh "kubeflow-user-example-com" diff --git a/.github/workflows/tensorboard_controller_test.yaml b/.github/workflows/tensorboard_controller_test.yaml index f7127854e3..2e5c5664a5 100644 --- a/.github/workflows/tensorboard_controller_test.yaml +++ b/.github/workflows/tensorboard_controller_test.yaml @@ -2,10 +2,10 @@ name: Test Tensorboard Controller on: pull_request: paths: - - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + - tests/install_KinD_create_KinD_cluster_install_kustomize.sh - .github/workflows/tensorboard_controller_test.yaml - apps/tensorboard/tensorboard-controller/upstream/** - - tests/gh-actions/install_istio*.sh + - tests/istio* - common/istio*/** jobs: @@ -17,10 +17,10 @@ jobs: uses: actions/checkout@v4 - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - name: Install Istio - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/istio-cni_install.sh - name: Build & Apply manifests run: | diff --git a/.github/workflows/tensorboards_web_application_test.yaml b/.github/workflows/tensorboards_web_application_test.yaml index 87eb80396e..6152ab15b5 100644 --- a/.github/workflows/tensorboards_web_application_test.yaml +++ b/.github/workflows/tensorboards_web_application_test.yaml @@ -2,10 +2,10 @@ name: Test Tensorboards Web Application on: pull_request: paths: - - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + - tests/install_KinD_create_KinD_cluster_install_kustomize.sh - .github/workflows/tensorboards_web_application_test.yaml - apps/tensorboard/tensorboards-web-app/upstream/** - - tests/gh-actions/install_istio*.sh + - tests/istio* - common/istio*/** jobs: @@ -17,10 +17,10 @@ jobs: uses: actions/checkout@v4 - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - name: Install Istio - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/istio-cni_install.sh - name: Build & Apply manifests run: | diff --git a/.github/workflows/training_operator_test.yaml b/.github/workflows/training_operator_test.yaml index 1de0e81949..2f5d565c30 100644 --- a/.github/workflows/training_operator_test.yaml +++ b/.github/workflows/training_operator_test.yaml @@ -2,13 +2,12 @@ name: Deploy and Test Training Operator on: pull_request: paths: - - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + - tests/install_KinD_create_KinD_cluster_install_kustomize.sh - .github/workflows/training_operator_test.yaml - apps/training-operator/upstream/** - - tests/gh-actions/kf-objects/training_operator_job.yaml - - tests/gh-actions/install_istio-cni.sh - - tests/gh-actions/install_cert_manager.sh - - tests/gh-actions/install_oauth2-proxy.sh + - tests/training_operator_job.yaml + - tests/istio* + - tests/oauth2-proxy_install.sh - common/cert-manager/** - common/oauth2-proxy/** - common/istio*/** @@ -25,34 +24,34 @@ jobs: uses: actions/checkout@v4 - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - name: Install kubectl - run: ./tests/gh-actions/install_kubectl.sh + run: ./tests/kubectl_install.sh - name: Create Kubeflow Namespace run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - - name: Install Certificate Manager - run: ./tests/gh-actions/install_cert_manager.sh + run: ./tests/cert_manager_install.sh - name: Install Istio CNI - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/istio-cni_install.sh - name: Install OAuth2 Proxy - run: ./tests/gh-actions/install_oauth2-proxy.sh + run: ./tests/oauth2-proxy_install.sh - name: Install Kubeflow Istio Resources run: kustomize build common/istio-cni-1-24/kubeflow-istio-resources/base | kubectl apply -f - - name: Install Multi-Tenancy - run: ./tests/gh-actions/install_multi_tenancy.sh + run: ./tests/multi_tenancy_install.sh - name: Create KF Profile - run: ./tests/gh-actions/install_kubeflow_profile.sh + run: ./tests/kubeflow_profile_install.sh - name: Install Training Operator - run: ./tests/gh-actions/install_training_operator.sh + run: ./tests/training_operator_install.sh - name: Verify CRDs are ready run: | @@ -63,10 +62,10 @@ jobs: run: pip install pytest kubernetes requests - name: Port-forward the istio-ingress gateway - run: ./tests/gh-actions/port_forward_gateway.sh + run: ./tests/port_forward_gateway.sh - name: Run Training Operator Test - run: ./tests/gh-actions/test_training_operator.sh "${KF_PROFILE}" + run: ./tests/training_operator_test.sh "${KF_PROFILE}" - name: Test with Authorized Token run: kubectl get pytorchjobs -n $KF_PROFILE --token="$(kubectl -n $KF_PROFILE create token default-editor)" @@ -79,7 +78,7 @@ jobs: kubectl get pytorchjobs -n $KF_PROFILE --token="$UNAUTHORIZED_TOKEN" >/dev/null - name: Apply Pod Security Standards baseline levels - run: ./tests/gh-actions/enable_baseline_PSS.sh + run: ./tests/PSS_baseline_enable.sh - name: Unapply applied baseline labels run: | @@ -91,4 +90,4 @@ jobs: done - name: Applying Pod Security Standards restricted levels - run: ./tests/gh-actions/enable_restricted_PSS.sh + run: ./tests/PSS_restricted_enable.sh diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index e5dc62319e..8ca1e8c63c 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -7,7 +7,7 @@ on: pull_request: paths: - '.github/workflows/trivy.yaml' - - 'tests/gh-actions/trivy_scan.py' + - 'tests/trivy_scan.py' jobs: image-extraction-and-security-scan: @@ -20,12 +20,12 @@ jobs: # Install kustomize - name: Install kustomize run: | - bash tests/gh-actions/install_kustomize.sh + bash tests/kustomize_install.sh # Install trivy - name: Install trivy run: | - bash tests/gh-actions/install_trivy.sh + bash tests/trivy_install.sh # Install Python - name: Setup Python @@ -40,7 +40,7 @@ jobs: - name: Run image extracting and security scanning script run: | - cd tests/gh-actions + cd tests python3 trivy_scan.py - name: Upload trivy scanned_results diff --git a/.github/workflows/volumes_web_application_test.yaml b/.github/workflows/volumes_web_application_test.yaml index 4e960a1323..0d75223404 100644 --- a/.github/workflows/volumes_web_application_test.yaml +++ b/.github/workflows/volumes_web_application_test.yaml @@ -2,16 +2,16 @@ name: Deploy and Test Volumes Web Application on: pull_request: paths: - - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + - tests/install_KinD_create_KinD_cluster_install_kustomize.sh - .github/workflows/volumes_web_application_test.yaml - apps/volumes-web-app/upstream/** - - tests/gh-actions/install_istio-cni.sh + - tests/istio* - common/istio*/** - common/oauth2-proxy/** - - tests/gh-actions/install_oauth2-proxy.sh - - tests/gh-actions/install_multi_tenancy.sh - - tests/gh-actions/install_volumes_web_application.sh - - tests/gh-actions/test_volumes_web_application.sh + - tests/oauth2-proxy_install.sh + - tests/multi_tenancy_install.sh + - tests/volumes_web_application_install.sh + - tests/volumes_web_application_test.sh env: KF_PROFILE: kubeflow-user-example-com @@ -25,34 +25,34 @@ jobs: uses: actions/checkout@v4 - name: Install KinD, Create KinD cluster and Install kustomize - run: ./tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh + run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh - name: Install kubectl - run: ./tests/gh-actions/install_kubectl.sh + run: ./tests/kubectl_install.sh - name: Create Kubeflow Namespace run: kustomize build common/kubeflow-namespace/base | kubectl apply -f - - name: Install Certificate Manager - run: ./tests/gh-actions/install_cert_manager.sh + run: ./tests/cert_manager_install.sh - name: Install Istio CNI - run: ./tests/gh-actions/install_istio-cni.sh + run: ./tests/istio-cni_install.sh - name: Install OAuth2 Proxy - run: ./tests/gh-actions/install_oauth2-proxy.sh + run: ./tests/oauth2-proxy_install.sh - name: Install Kubeflow Istio Resources run: kustomize build common/istio-cni-1-24/kubeflow-istio-resources/base | kubectl apply -f - - name: Install Multi-Tenancy - run: ./tests/gh-actions/install_multi_tenancy.sh + run: ./tests/multi_tenancy_install.sh - name: Create KF Profile - run: ./tests/gh-actions/install_kubeflow_profile.sh + run: ./tests/kubeflow_profile_install.sh - name: Install Volumes Web Application - run: ./tests/gh-actions/install_volumes_web_application.sh + run: ./tests/volumes_web_application_install.sh - name: Wait for VWA Pods run: | @@ -60,7 +60,7 @@ jobs: sleep 15 - name: Port-forward the istio-ingress gateway - run: ./tests/gh-actions/port_forward_gateway.sh + run: ./tests/port_forward_gateway.sh - name: Test Volumes Web Application API - run: ./tests/gh-actions/test_volumes_web_application.sh "${KF_PROFILE}" + run: ./tests/volumes_web_application_test.sh "${KF_PROFILE}" diff --git a/README.md b/README.md index f89cd6bedf..e9c37b5ae8 100644 --- a/README.md +++ b/README.md @@ -715,7 +715,7 @@ pre-commit run - **Q:** What versions of Istio, Knative, Cert-Manager, Argo, ... are compatible with Kubeflow? **A:** Please refer to each individual component's documentation for a dependency compatibility range. For Istio, Knative, Dex, Cert-Manager, and OAuth2 Proxy, the versions in `common` are the ones we have validated. - **Q:** Can I use Kubeflow in an air-gapped environment? - **A:** Yes you can. You just need to to get the list of images from our [trivy CVE scanning script](https://github.com/kubeflow/manifests/blob/master/tests/gh-actions/trivy_scan.py), mirror them and replace the references in the manifests with kustomize components and overlays, see [Upgrading and Extending](#upgrading-and-extending). You could also use a simple kyverno policy to replace the images at runtime, which could be easier to maintain. + **A:** Yes you can. You just need to to get the list of images from our [trivy CVE scanning script](https://github.com/kubeflow/manifests/blob/master/tests/trivy_scan.py), mirror them and replace the references in the manifests with kustomize components and overlays, see [Upgrading and Extending](#upgrading-and-extending). You could also use a simple kyverno policy to replace the images at runtime, which could be easier to maintain. - **Q:** Why does Kubeflow use Istio CNI instead of standard Istio? **A:** Istio CNI provides better security by eliminating the need for privileged init containers, making it more compatible with Pod Security Standards (PSS). It also enables native sidecars support introduced in Kubernetes 1.28, which helps address issues with init containers and application lifecycle management. - **Q:** Why does Istio CNI fail on Google Kubernetes Engine (GKE) with "read-only file system" errors? diff --git a/tests/gh-actions/enable_baseline_PSS.sh b/tests/PSS_baseline_enable.sh similarity index 100% rename from tests/gh-actions/enable_baseline_PSS.sh rename to tests/PSS_baseline_enable.sh diff --git a/tests/gh-actions/enable_restricted_PSS.sh b/tests/PSS_restricted_enable.sh similarity index 100% rename from tests/gh-actions/enable_restricted_PSS.sh rename to tests/PSS_restricted_enable.sh diff --git a/tests/gh-actions/install_argo_cli.sh b/tests/argo_cli_install.sh similarity index 100% rename from tests/gh-actions/install_argo_cli.sh rename to tests/argo_cli_install.sh diff --git a/tests/gh-actions/install_central_dashboard.sh b/tests/central_dashboard_install.sh similarity index 100% rename from tests/gh-actions/install_central_dashboard.sh rename to tests/central_dashboard_install.sh diff --git a/tests/gh-actions/install_cert_manager.sh b/tests/cert_manager_install.sh similarity index 100% rename from tests/gh-actions/install_cert_manager.sh rename to tests/cert_manager_install.sh diff --git a/tests/gh-actions/install_dex.sh b/tests/dex_install.sh similarity index 100% rename from tests/gh-actions/install_dex.sh rename to tests/dex_install.sh diff --git a/tests/gh-actions/test_dex_login.py b/tests/dex_login_test.py similarity index 100% rename from tests/gh-actions/test_dex_login.py rename to tests/dex_login_test.py diff --git a/tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh b/tests/install_KinD_create_KinD_cluster_install_kustomize.sh similarity index 100% rename from tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh rename to tests/install_KinD_create_KinD_cluster_install_kustomize.sh diff --git a/tests/gh-actions/install_istio-cni.sh b/tests/istio-cni_install.sh similarity index 100% rename from tests/gh-actions/install_istio-cni.sh rename to tests/istio-cni_install.sh diff --git a/tests/gh-actions/install_katib.sh b/tests/katib_install.sh similarity index 100% rename from tests/gh-actions/install_katib.sh rename to tests/katib_install.sh diff --git a/tests/gh-actions/kf-objects/katib_test.yaml b/tests/katib_test.yaml similarity index 100% rename from tests/gh-actions/kf-objects/katib_test.yaml rename to tests/katib_test.yaml diff --git a/tests/gh-actions/install_knative-cni.sh b/tests/knative-cni_install.sh similarity index 100% rename from tests/gh-actions/install_knative-cni.sh rename to tests/knative-cni_install.sh diff --git a/tests/gh-actions/kserve/data/iris_input.json b/tests/kserve/data/iris_input.json similarity index 100% rename from tests/gh-actions/kserve/data/iris_input.json rename to tests/kserve/data/iris_input.json diff --git a/tests/gh-actions/kserve/requirements.txt b/tests/kserve/requirements.txt similarity index 100% rename from tests/gh-actions/kserve/requirements.txt rename to tests/kserve/requirements.txt diff --git a/tests/gh-actions/kserve/test_sklearn.py b/tests/kserve/test_sklearn.py similarity index 100% rename from tests/gh-actions/kserve/test_sklearn.py rename to tests/kserve/test_sklearn.py diff --git a/tests/gh-actions/kserve/utils.py b/tests/kserve/utils.py similarity index 100% rename from tests/gh-actions/kserve/utils.py rename to tests/kserve/utils.py diff --git a/tests/gh-actions/install_kserve.sh b/tests/kserve_install.sh similarity index 100% rename from tests/gh-actions/install_kserve.sh rename to tests/kserve_install.sh diff --git a/tests/gh-actions/test_kserve.sh b/tests/kserve_test.sh similarity index 100% rename from tests/gh-actions/test_kserve.sh rename to tests/kserve_test.sh diff --git a/tests/gh-actions/kf-objects/kserve_test.yaml b/tests/kserve_test.yaml similarity index 100% rename from tests/gh-actions/kf-objects/kserve_test.yaml rename to tests/kserve_test.yaml diff --git a/tests/gh-actions/install_kubectl.sh b/tests/kubectl_install.sh similarity index 100% rename from tests/gh-actions/install_kubectl.sh rename to tests/kubectl_install.sh diff --git a/tests/gh-actions/install_kubeflow_profile.sh b/tests/kubeflow_profile_install.sh similarity index 100% rename from tests/gh-actions/install_kubeflow_profile.sh rename to tests/kubeflow_profile_install.sh diff --git a/tests/gh-actions/install_kustomize.sh b/tests/kustomize_install.sh similarity index 100% rename from tests/gh-actions/install_kustomize.sh rename to tests/kustomize_install.sh diff --git a/tests/gh-actions/install_multi_tenancy.sh b/tests/multi_tenancy_install.sh similarity index 100% rename from tests/gh-actions/install_multi_tenancy.sh rename to tests/multi_tenancy_install.sh diff --git a/tests/gh-actions/kf-objects/notebook.test.kubeflow-user-example.com.yaml b/tests/notebook.test.kubeflow-user-example.com.yaml similarity index 100% rename from tests/gh-actions/kf-objects/notebook.test.kubeflow-user-example.com.yaml rename to tests/notebook.test.kubeflow-user-example.com.yaml diff --git a/tests/gh-actions/install_oauth2-proxy.sh b/tests/oauth2-proxy_install.sh similarity index 100% rename from tests/gh-actions/install_oauth2-proxy.sh rename to tests/oauth2-proxy_install.sh diff --git a/tests/gh-actions/oauth2_dex_credentials.sh b/tests/oauth2_dex_credentials.sh similarity index 93% rename from tests/gh-actions/oauth2_dex_credentials.sh rename to tests/oauth2_dex_credentials.sh index 31b0392711..2a36e4a1ab 100755 --- a/tests/gh-actions/oauth2_dex_credentials.sh +++ b/tests/oauth2_dex_credentials.sh @@ -59,7 +59,7 @@ if ! kubectl get deploy -n auth dex -o yaml | grep -q "OIDC_CLIENT_ID"; then --from-literal=OIDC_CLIENT_SECRET=pUBnBOY80SnXgjibTYM9ZWNzY2xreNGQok \ --dry-run=client -o yaml | kubectl apply -f - - ./tests/gh-actions/install_dex.sh + ./tests/dex_install.sh fi if kubectl get deployment -n oauth2-proxy oauth2-proxy &>/dev/null; then @@ -77,12 +77,12 @@ until curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/dex/health 2> sleep 10 done -sed -i 's/raise RuntimeError/print("ERROR:"); exit 1/g' tests/gh-actions/test_dex_login.py +sed -i 's/raise RuntimeError/print("ERROR:"); exit 1/g' tests/dex_login_test.py # Create a temporary python script file instead of using heredoc cat > /tmp/update_dex_login.py << 'PYTHONEOF' import re -with open('tests/gh-actions/test_dex_login.py', 'r') as f: +with open('tests/dex_login_test.py', 'r') as f: content = f.read() content = re.sub('import re', 'import re, time, sys', content, count=1) retry_pattern = r'([ \t]+)session_cookies = dex_session_manager\.get_session_cookies\(\)' @@ -96,7 +96,7 @@ replacement = r"""\1# Try with retries \1 sys.exit(1) \1 time.sleep(5)""" content = re.sub(retry_pattern, replacement, content, count=1) -with open('tests/gh-actions/test_dex_login.py', 'w') as f: +with open('tests/dex_login_test.py', 'w') as f: f.write(content) PYTHONEOF diff --git a/tests/gh-actions/run_and_wait_kubeflow_pipeline.py b/tests/pipeline_run_and_wait_kubeflow.py similarity index 100% rename from tests/gh-actions/run_and_wait_kubeflow_pipeline.py rename to tests/pipeline_run_and_wait_kubeflow.py diff --git a/tests/gh-actions/kf-objects/test_pipeline.py b/tests/pipeline_test.py similarity index 100% rename from tests/gh-actions/kf-objects/test_pipeline.py rename to tests/pipeline_test.py diff --git a/tests/gh-actions/test_pipeline_v1.py b/tests/pipeline_v1_test.py similarity index 100% rename from tests/gh-actions/test_pipeline_v1.py rename to tests/pipeline_v1_test.py diff --git a/tests/gh-actions/test_pipeline_v2.py b/tests/pipeline_v2_test.py similarity index 100% rename from tests/gh-actions/test_pipeline_v2.py rename to tests/pipeline_v2_test.py diff --git a/tests/gh-actions/install_pipelines.sh b/tests/pipelines_install.sh similarity index 100% rename from tests/gh-actions/install_pipelines.sh rename to tests/pipelines_install.sh diff --git a/tests/gh-actions/install_pipelines_swfs.sh b/tests/pipelines_swfs_install.sh similarity index 100% rename from tests/gh-actions/install_pipelines_swfs.sh rename to tests/pipelines_swfs_install.sh diff --git a/tests/gh-actions/kf-objects/poddefaults.access-ml-pipeline.kubeflow-user-example-com.yaml b/tests/poddefaults.access-ml-pipeline.kubeflow-user-example-com.yaml similarity index 100% rename from tests/gh-actions/kf-objects/poddefaults.access-ml-pipeline.kubeflow-user-example-com.yaml rename to tests/poddefaults.access-ml-pipeline.kubeflow-user-example-com.yaml diff --git a/tests/gh-actions/port_forward_gateway.sh b/tests/port_forward_gateway.sh similarity index 100% rename from tests/gh-actions/port_forward_gateway.sh rename to tests/port_forward_gateway.sh diff --git a/tests/gh-actions/runasnonroot.sh b/tests/runasnonroot.sh similarity index 100% rename from tests/gh-actions/runasnonroot.sh rename to tests/runasnonroot.sh diff --git a/tests/gh-actions/s3_test_helper.py b/tests/s3_helper_test.py similarity index 100% rename from tests/gh-actions/s3_test_helper.py rename to tests/s3_helper_test.py diff --git a/tests/gh-actions/install_spark.sh b/tests/spark_install.sh similarity index 100% rename from tests/gh-actions/install_spark.sh rename to tests/spark_install.sh diff --git a/tests/gh-actions/test_spark.sh b/tests/spark_test.sh similarity index 100% rename from tests/gh-actions/test_spark.sh rename to tests/spark_test.sh diff --git a/tests/gh-actions/test_swfs_namespace_isolation.sh b/tests/swfs_namespace_isolation_test.sh similarity index 97% rename from tests/gh-actions/test_swfs_namespace_isolation.sh rename to tests/swfs_namespace_isolation_test.sh index 95fcb69497..03b0af0d28 100755 --- a/tests/gh-actions/test_swfs_namespace_isolation.sh +++ b/tests/swfs_namespace_isolation_test.sh @@ -118,7 +118,7 @@ upload_file() { setup_port_forward - python3 tests/gh-actions/s3_test_helper.py upload \ + python3 tests/s3_helper_test.py upload \ --access-key "$access_key" \ --secret-key "$secret_key" \ --endpoint-url "http://localhost:8333" \ @@ -142,7 +142,7 @@ test_unauthorized_access() { # Try to access the other namespace's file # Note: Python script returns 0 when access is denied (good), 1 when access succeeds (bad) - if python3 tests/gh-actions/s3_test_helper.py download \ + if python3 tests/s3_helper_test.py download \ --access-key "$access_key" \ --secret-key "$secret_key" \ --endpoint-url "http://localhost:8333" \ diff --git a/tests/gh-actions/install_training_operator.sh b/tests/training_operator_install.sh similarity index 100% rename from tests/gh-actions/install_training_operator.sh rename to tests/training_operator_install.sh diff --git a/tests/gh-actions/kf-objects/training_operator_job.yaml b/tests/training_operator_job.yaml similarity index 100% rename from tests/gh-actions/kf-objects/training_operator_job.yaml rename to tests/training_operator_job.yaml diff --git a/tests/gh-actions/test_training_operator.sh b/tests/training_operator_test.sh similarity index 90% rename from tests/gh-actions/test_training_operator.sh rename to tests/training_operator_test.sh index fefe6617ce..8a8c99bb11 100755 --- a/tests/gh-actions/test_training_operator.sh +++ b/tests/training_operator_test.sh @@ -2,7 +2,7 @@ set -euxo pipefail KF_PROFILE=${1:-kubeflow-user-example-com} -cat tests/gh-actions/kf-objects/training_operator_job.yaml | \ +cat tests/training_operator_job.yaml | \ sed 's/name: pytorch-simple/name: pytorch-simple\n namespace: '"$KF_PROFILE"'/g' > /tmp/pytorch-job.yaml kubectl apply -f /tmp/pytorch-job.yaml diff --git a/tests/gh-actions/install_trivy.sh b/tests/trivy_install.sh similarity index 100% rename from tests/gh-actions/install_trivy.sh rename to tests/trivy_install.sh diff --git a/tests/gh-actions/trivy_scan.py b/tests/trivy_scan.py similarity index 87% rename from tests/gh-actions/trivy_scan.py rename to tests/trivy_scan.py index 993aa5e035..846608fa43 100644 --- a/tests/gh-actions/trivy_scan.py +++ b/tests/trivy_scan.py @@ -1,13 +1,13 @@ # The script: # 1. Extract all the images used by the Kubeflow Working Groups -# - The reported image lists are saved in respective files under ../../image_lists directory +# - The reported image lists are saved in respective files under ../image_lists directory # 2. Scan the reported images using Trivy for security vulnerabilities -# - Scanned reports will be saved in JSON format inside ../../image_lists/security_scan_reports/ folder for each Working Group +# - Scanned reports will be saved in JSON format inside ../image_lists/security_scan_reports/ folder for each Working Group # 3. The script will also generate a summary of the security scan reports with severity counts for each Working Group with images -# - Summary of security counts with images a JSON file inside ../../image_lists/summary_of_severity_counts_for_WG folder +# - Summary of security counts with images a JSON file inside ../image_lists/summary_of_severity_counts_for_WG folder # 4. Generate a summary of the security scan reports -# - The summary will be saved in JSON format inside ../../image_lists/summary_of_severity_counts_for_WG folder -# The script must be executed from the tests/gh-actions folder as it uses relative paths +# - The summary will be saved in JSON format inside ../image_lists/summary_of_severity_counts_for_WG folder +# The script must be executed from the tests/ folder as it uses relative paths import os import subprocess @@ -19,17 +19,17 @@ # Dictionary mapping Kubeflow workgroups to directories containing kustomization files wg_dirs = { - "katib": "../../apps/katib/upstream/installs", - "pipelines": "../../apps/pipeline/upstream/env/cert-manager/platform-agnostic-multi-user", - "trainer": "../../apps/training-operator/upstream/overlays", - "manifests": "../../common/cert-manager/cert-manager/base ../../common/cert-manager/kubeflow-issuer/base ../../common/istio-1-24/istio-crds/base ../../common/istio-1-24/istio-namespace/base ../../common/istio-1-24/istio-install/overlays/oauth2-proxy ../../common/oauth2-proxy/overlays/m2m-self-signed ../../common/dex/overlays/oauth2-proxy ../../common/knative/knative-serving/overlays/gateways ../../common/knative/knative-eventing/base ../../common/istio-1-24/cluster-local-gateway/base ../../common/kubeflow-namespace/base ../../common/kubeflow-roles/base ../../common/istio-1-24/kubeflow-istio-resources/base", - "workbenches": "../../apps/pvcviewer-controller/upstream/base ../../apps/admission-webhook/upstream/overlays ../../apps/centraldashboard/overlays ../../apps/jupyter/jupyter-web-app/upstream/overlays ../../apps/volumes-web-app/upstream/overlays ../../apps/tensorboard/tensorboards-web-app/upstream/overlays ../../apps/profiles/upstream/overlays ../../apps/jupyter/notebook-controller/upstream/overlays ../../apps/tensorboard/tensorboard-controller/upstream/overlays", - "kserve": "../../apps/kserve - ../../apps/kserve/models-web-app/overlays/kubeflow", - "model-registry": "../../apps/model-registry/upstream", - "spark": "../../apps/spark/spark-operator/overlays/kubeflow", + "katib": "../apps/katib/upstream/installs", + "pipelines": "../apps/pipeline/upstream/env/cert-manager/platform-agnostic-multi-user", + "trainer": "../apps/training-operator/upstream/overlays", + "manifests": "../common/cert-manager/cert-manager/base ../common/cert-manager/kubeflow-issuer/base ../common/istio-1-24/istio-crds/base ../common/istio-1-24/istio-namespace/base ../common/istio-1-24/istio-install/overlays/oauth2-proxy ../common/oauth2-proxy/overlays/m2m-self-signed ../common/dex/overlays/oauth2-proxy ../common/knative/knative-serving/overlays/gateways ../common/knative/knative-eventing/base ../common/istio-1-24/cluster-local-gateway/base ../common/kubeflow-namespace/base ../common/kubeflow-roles/base ../common/istio-1-24/kubeflow-istio-resources/base", + "workbenches": "../apps/pvcviewer-controller/upstream/base ../apps/admission-webhook/upstream/overlays ../apps/centraldashboard/overlays ../apps/jupyter/jupyter-web-app/upstream/overlays ../apps/volumes-web-app/upstream/overlays ../apps/tensorboard/tensorboards-web-app/upstream/overlays ../apps/profiles/upstream/overlays ../apps/jupyter/notebook-controller/upstream/overlays ../apps/tensorboard/tensorboard-controller/upstream/overlays", + "kserve": "../apps/kserve - ../apps/kserve/models-web-app/overlays/kubeflow", + "model-registry": "../apps/model-registry/upstream", + "spark": "../apps/spark/spark-operator/overlays/kubeflow", } -DIRECTORY = "../../image_lists" +DIRECTORY = "../image_lists" os.makedirs(DIRECTORY, exist_ok=True) SCAN_REPORTS_DIR = os.path.join(DIRECTORY, "security_scan_reports") ALL_SEVERITY_COUNTS = os.path.join(DIRECTORY, "severity_counts_with_images_for_WG") @@ -50,7 +50,7 @@ def log(*args, **kwargs): def save_images(wg, images, version): # Saves a list of container images to a text file named after the workgroup and version. - output_file = f"../../image_lists/kf_{version}_{wg}_images.txt" + output_file = f"../image_lists/kf_{version}_{wg}_images.txt" with open(output_file, "w") as f: f.write("\n".join(images)) log(f"File {output_file} successfully created") diff --git a/tests/gh-actions/install_volumes_web_application.sh b/tests/volumes_web_application_install.sh similarity index 100% rename from tests/gh-actions/install_volumes_web_application.sh rename to tests/volumes_web_application_install.sh diff --git a/tests/gh-actions/test_volumes_web_application.sh b/tests/volumes_web_application_test.sh similarity index 100% rename from tests/gh-actions/test_volumes_web_application.sh rename to tests/volumes_web_application_test.sh