Skip to content

Commit dc7549d

Browse files
xeniapexeniape
authored
ci: Add workflows for pyspark images (#108)
1 parent 7dd08ba commit dc7549d

File tree

4 files changed

+178
-4
lines changed

4 files changed

+178
-4
lines changed

‎.github/workflows/dev_jupyter-pyspark-with-alibi-detect.yaml

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
---
2+
name: Build and publish jupyter-pyspark-with-alibi-detect
3+
4+
env:
5+
IMAGE_NAME: jupyter-pyspark-with-alibi-detect
6+
IMAGE_VERSION: python-3.9
7+
REGISTRY_PATH: stackable
8+
DOCKERFILE_PATH: "demos/signal-processing/Dockerfile-jupyter"
9+
10+
on:
11+
workflow_dispatch:
12+
push:
13+
branches:
14+
- main
15+
paths:
16+
- demos/signal-processing/Dockerfile-jupyter
17+
- demos/signal-processing/requirements.txt
18+
- .github/workflows/dev_jupyter-pyspark-with-alibi-detect.yaml
19+
20+
jobs:
21+
build:
22+
name: Build/Publish ${{ matrix.runner.arch }} Image
23+
permissions:
24+
id-token: write
25+
runs-on: ${{ matrix.runner.name }}
26+
strategy:
27+
matrix:
28+
runner:
29+
- {name: "ubuntu-latest", arch: "amd64"}
30+
- {name: "ubicloud-standard-8-arm", arch: "arm64"}
31+
steps:
32+
- name: Checkout Repository
33+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
34+
35+
- name: Build image
36+
id: build
37+
uses: stackabletech/actions/build-container-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
38+
with:
39+
image-name: ${{ env.IMAGE_NAME }}
40+
image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
41+
container-file: ${{ env.DOCKERFILE_PATH }}
42+
43+
- name: Publish Container Image on docker.stackable.tech
44+
uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
45+
with:
46+
image-registry-uri: docker.stackable.tech
47+
image-registry-username: github
48+
image-registry-password: ${{ secrets.NEXUS_PASSWORD }}
49+
image-repository: demos/${{ env.IMAGE_NAME }}
50+
image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
51+
source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}
52+
53+
- name: Publish Container Image on oci.stackable.tech
54+
uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
55+
with:
56+
image-registry-uri: oci.stackable.tech
57+
image-registry-username: robot$stackable+github-action-build
58+
image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
59+
image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}
60+
image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
61+
source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}
62+
63+
publish_manifests:
64+
name: Build/Publish Manifest
65+
needs: [build]
66+
permissions:
67+
id-token: write
68+
runs-on: ubuntu-latest
69+
steps:
70+
- name: Checkout Repository
71+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
72+
73+
- name: Publish and Sign Image Index Manifest to docker.stackable.tech
74+
uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
75+
with:
76+
image-registry-uri: docker.stackable.tech
77+
image-registry-username: github
78+
image-registry-password: ${{ secrets.NEXUS_PASSWORD }}
79+
image-repository: demos/${{ env.IMAGE_NAME }}
80+
image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
81+
82+
- name: Publish and Sign Image Index Manifest to oci.stackable.tech
83+
uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
84+
with:
85+
image-registry-uri: oci.stackable.tech
86+
image-registry-username: robot$stackable+github-action-build
87+
image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
88+
image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}
89+
image-index-manifest-tag: ${{ env.IMAGE_VERSION }}

‎.github/workflows/dev_pyspark-k8s-with-scikit-learn.yaml

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
---
2+
name: Build and publish pyspark-k8s-with-scikit-learn
3+
4+
env:
5+
IMAGE_NAME: pyspark-k8s-with-scikit-learn
6+
IMAGE_VERSION: 3.4.0-stackable0.0.0-dev
7+
REGISTRY_PATH: stackable
8+
DOCKERFILE_PATH: "demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile"
9+
10+
on:
11+
workflow_dispatch:
12+
push:
13+
branches:
14+
- main
15+
paths:
16+
- demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile
17+
- demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/requirements.txt
18+
- .github/workflows/dev_pyspark-k8s-with-scikit-learn.yaml
19+
20+
jobs:
21+
build:
22+
name: Build/Publish ${{ matrix.runner.arch }} Image
23+
permissions:
24+
id-token: write
25+
runs-on: ${{ matrix.runner.name }}
26+
strategy:
27+
matrix:
28+
runner:
29+
- {name: "ubuntu-latest", arch: "amd64"}
30+
- {name: "ubicloud-standard-8-arm", arch: "arm64"}
31+
steps:
32+
- name: Checkout Repository
33+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
34+
35+
- name: Build image
36+
id: build
37+
uses: stackabletech/actions/build-container-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
38+
with:
39+
image-name: ${{ env.IMAGE_NAME }}
40+
image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
41+
container-file: ${{ env.DOCKERFILE_PATH }}
42+
43+
- name: Publish Container Image on docker.stackable.tech
44+
uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
45+
with:
46+
image-registry-uri: docker.stackable.tech
47+
image-registry-username: github
48+
image-registry-password: ${{ secrets.NEXUS_PASSWORD }}
49+
image-repository: demos/${{ env.IMAGE_NAME }}
50+
image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
51+
source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}
52+
53+
- name: Publish Container Image on oci.stackable.tech
54+
uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
55+
with:
56+
image-registry-uri: oci.stackable.tech
57+
image-registry-username: robot$stackable+github-action-build
58+
image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
59+
image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}
60+
image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
61+
source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}
62+
63+
publish_manifests:
64+
name: Build/Publish Manifest
65+
needs: [build]
66+
permissions:
67+
id-token: write
68+
runs-on: ubuntu-latest
69+
steps:
70+
- name: Checkout Repository
71+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
72+
73+
- name: Publish and Sign Image Index Manifest to docker.stackable.tech
74+
uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
75+
with:
76+
image-registry-uri: docker.stackable.tech
77+
image-registry-username: github
78+
image-registry-password: ${{ secrets.NEXUS_PASSWORD }}
79+
image-repository: demos/${{ env.IMAGE_NAME }}
80+
image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
81+
82+
- name: Publish and Sign Image Index Manifest to oci.stackable.tech
83+
uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
84+
with:
85+
image-registry-uri: oci.stackable.tech
86+
image-registry-username: robot$stackable+github-action-build
87+
image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
88+
image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}
89+
image-index-manifest-tag: ${{ env.IMAGE_VERSION }}

‎demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
# docker build -t docker.stackable.tech/demos/pyspark-k8s-with-scikit-learn:3.4.0-stackable0.0.0-dev .
2-
31
FROM docker.stackable.tech/stackable/pyspark-k8s:3.4.0-stackable23.7.0
42

53
COPY requirements.txt .

‎demos/signal-processing/Dockerfile-jupyter

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
# docker build -f ./Dockerfile-jupyter -t docker.stackable.tech/demos/jupyter-pyspark-with-alibi-detect:python-3.9 .
2-
31
FROM jupyter/pyspark-notebook:python-3.9
42

53
COPY requirements.txt .

0 commit comments

Comments
 (0)