Skip to content

Commit 66a5a95

Browse files
Add parent dependency check on samples (#397)
* ignore cached remote samples Signed-off-by: Michael Valdron <mvaldron@redhat.com> * nightly script for building sample parent stack list Signed-off-by: Michael Valdron <mvaldron@redhat.com> * init check parent stacks script Signed-off-by: Michael Valdron <mvaldron@redhat.com> * convert nightly parents build to on the fly parents build. Signed-off-by: Michael Valdron <mvaldron@redhat.com> * multi version support for build parent stacks Signed-off-by: Michael Valdron <mvaldron@redhat.com> * ignore any local parents file. Signed-off-by: Michael Valdron <mvaldron@redhat.com> * rename check_parent_stacks.sh to build_parents_file.sh, build_parents_file.sh now builds the parents yaml file and returns the children samples of the parent stacks. Signed-off-by: Michael Valdron <mvaldron@redhat.com> * validate devfile schemas test suite parameters now changeable Signed-off-by: Michael Valdron <mvaldron@redhat.com> * validate samples workflow Signed-off-by: Michael Valdron <mvaldron@redhat.com> * add check for if there are samples to be checked under validate samples job. Signed-off-by: Michael Valdron <mvaldron@redhat.com> * add README instruction on dependency check changes Signed-off-by: Michael Valdron <mvaldron@redhat.com> * yq 4.x installation guide link added to testing prerequisites list. Signed-off-by: Michael Valdron <mvaldron@redhat.com> * add missing link to test suite Signed-off-by: Michael Valdron <mvaldron@redhat.com> * validate_devfile_schemas script accepts relative or absolute paths Signed-off-by: Michael Valdron <mvaldron@redhat.com> * use registry-support cache samples Signed-off-by: Michael Valdron <mvaldron@redhat.com> * use yq v4.44.1 Signed-off-by: Michael Valdron <mvaldron@redhat.com> * validate-samples actions match revision from other workflows Signed-off-by: Michael Valdron <mvaldron@redhat.com> * create verbose mode for build_parents_file.sh script, defaults to false Signed-off-by: Michael Valdron <mvaldron@redhat.com> * validate_devfile_schemas.sh script unzips resource files if samples Signed-off-by: Michael Valdron <mvaldron@redhat.com> * update README Signed-off-by: Michael Valdron <mvaldron@redhat.com> * correct plural meaning to singular meaning for operation names Signed-off-by: Michael Valdron <mvaldron@redhat.com> * add verbose mode to validate_devfile_schemas script Signed-off-by: Michael Valdron <mvaldron@redhat.com> * set default sample cache dir to .cache/samples Signed-off-by: Michael Valdron <mvaldron@redhat.com> * remove unused minikube env vars Signed-off-by: Michael Valdron <mvaldron@redhat.com> --------- Signed-off-by: Michael Valdron <mvaldron@redhat.com>
1 parent 56269fc commit 66a5a95

File tree

7 files changed

+393
-5
lines changed

7 files changed

+393
-5
lines changed

.ci/Dockerfile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ FROM registry.access.redhat.com/ubi8/go-toolset:1.19 AS builder
1818
USER root
1919

2020
# Install yq
21-
RUN curl -sL -O https://github.com/mikefarah/yq/releases/download/v4.9.5/yq_linux_amd64 -o /usr/local/bin/yq && mv ./yq_linux_amd64 /usr/local/bin/yq && chmod +x /usr/local/bin/yq
21+
ENV YQ_VERSION=v4.44.1
22+
RUN curl -sL -O https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -o /usr/local/bin/yq && mv ./yq_linux_amd64 /usr/local/bin/yq && chmod +x /usr/local/bin/yq
2223

2324
COPY . /registry
2425

.ci/Dockerfile.offline

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ FROM registry.access.redhat.com/ubi8/go-toolset:1.19 AS builder
1818
USER root
1919

2020
# Install yq
21-
RUN curl -sL -O https://github.com/mikefarah/yq/releases/download/v4.9.5/yq_linux_amd64 -o /usr/local/bin/yq && mv ./yq_linux_amd64 /usr/local/bin/yq && chmod +x /usr/local/bin/yq
21+
ENV YQ_VERSION=v4.44.1
22+
RUN curl -sL -O https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -o /usr/local/bin/yq && mv ./yq_linux_amd64 /usr/local/bin/yq && chmod +x /usr/local/bin/yq
2223

2324
COPY . /registry
2425

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#
2+
# Copyright 2023 Red Hat, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
name: Validate child samples
17+
18+
on:
19+
push:
20+
branches: [main]
21+
pull_request:
22+
branches: [main]
23+
schedule:
24+
- cron: 0 5 * * *
25+
26+
concurrency:
27+
group: ${{ github.workflow }}-${{ github.event.number || github.ref }}
28+
cancel-in-progress: true
29+
30+
env:
31+
YQ_VERSION: "v4.44.1"
32+
TEST_DELTA: false
33+
34+
jobs:
35+
validate-devfile-schema:
36+
name: validate devfile schemas
37+
runs-on: ubuntu-latest
38+
steps:
39+
- name: Checkout
40+
uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
41+
with:
42+
fetch-depth: 0
43+
44+
- name: Install Go
45+
uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
46+
with:
47+
go-version: "1.19"
48+
49+
- name: Install Ginkgo
50+
run: go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo@v2.13.0
51+
52+
- name: Install yq
53+
run: curl -sL -O https://github.com/mikefarah/yq/releases/download/${{ env.YQ_VERSION }}/yq_linux_amd64 -o /usr/local/bin/yq && mv ./yq_linux_amd64 /usr/local/bin/yq && chmod +x /usr/local/bin/yq
54+
55+
- name: Test delta if on a pull request
56+
if: ${{ github.event_name == 'pull_request' }}
57+
run: echo "TEST_DELTA=true" >> $GITHUB_ENV
58+
59+
- name: Build parents file and get child samples
60+
run: echo "STACKS=$(bash tests/build_parents_file.sh)" >> $GITHUB_ENV
61+
62+
- name: Validate samples
63+
if: ${{ env.STACKS != '' }}
64+
run: STACKS_DIR=$(pwd)/samples/.cache bash tests/validate_devfile_schemas.sh --samples

.gitignore

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,6 @@ registry-support/
33
.idea/
44
devfile-web/
55
vendor/
6-
.odo
6+
.odo
7+
.cache
8+
parents.yaml

tests/README.md

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,29 @@
11
# Devfile Registry Testing
22

3+
## Dependency check
4+
5+
### Prerequisites
6+
7+
- Ensure [yq 4.x](https://github.com/mikefarah/yq/#install) is installed
8+
9+
### Running the build script
10+
11+
- This script performs three actions
12+
- Clones samples from provided `extraDevfileEntries.yaml` under `samples/.cache`
13+
- Creates a `parents.yaml` which contains the dependency tree for parent stacks
14+
- Outputs the child sample paths of parent stacks, `TEST_DELTA=true` will result in only outputting child samples which have changed parent stacks
15+
- The build script takes one optional argument and works off of the current working directory
16+
- `bash tests/build_parents_file.sh`, default samples file is `extraDevfileEntries.yaml`
17+
- `bash tests/build_parents_file.sh <path_to_extraDevfileEntries>`
18+
19+
### Use with testing
20+
21+
- One can test the child samples using the [validate_devfile_schemas](./validate_devfile_schemas/) test suite by performing the following:
22+
```sh
23+
export STACKS=$(bash tests/build_parents_file.sh)
24+
STACKS_DIR=samples/.cache bash tests/validate_devfile_schemas.sh --samples
25+
```
26+
327
## Validating non-terminating images
428

529
### Prerequisites

tests/build_parents_file.sh

Lines changed: 251 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,251 @@
1+
#!/bin/bash
2+
3+
POSITIONAL_ARGS=()
4+
VERBOSE="false"
5+
6+
while [[ $# -gt 0 ]]; do
7+
case $1 in
8+
-v|--verbose)
9+
VERBOSE="true"
10+
shift # past argument
11+
;;
12+
-*|--*)
13+
echo "Unknown option $1"
14+
exit 1
15+
;;
16+
*)
17+
POSITIONAL_ARGS+=("$1") # save positional arg
18+
shift # past argument
19+
;;
20+
esac
21+
done
22+
23+
set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters
24+
25+
# default samples file path
26+
samples_file=$(pwd)/extraDevfileEntries.yaml
27+
# Cached remote samples directory
28+
samples_dir=$(pwd)/.cache/samples
29+
# default stacks directory
30+
stacks_dir=${STACKS:-"$(pwd)/stacks"}
31+
parents_file=$(pwd)/parents.yaml
32+
# Base directory path
33+
base_path=$(realpath $(dirname $(dirname $0)))
34+
35+
# YAML query cmd path
36+
YQ_PATH=${YQ_PATH:-yq}
37+
38+
# Read samples file as first argument
39+
# if unset use default samples file path
40+
if [ ! -z "${1}" ]; then
41+
samples_file=${1}
42+
fi
43+
44+
get_parent_version() {
45+
devfile=$1
46+
name=$2
47+
version=$($YQ_PATH eval .parent.version ${devfile})
48+
49+
if [ "${version}" == "null" ] && [ -f "${stacks_dir}/${name}/stack.yaml" ]; then
50+
version=$($YQ_PATH eval '.versions | filter(.default) | .[0].version' ${stacks_dir}/${name}/stack.yaml)
51+
fi
52+
53+
echo ${version}
54+
}
55+
56+
# Get parent index if exists, else returns -1
57+
parent_index() {
58+
name=$1
59+
version=$2
60+
61+
if [ -z "${version}" ]; then
62+
result=$($YQ_PATH eval ".parents | to_entries | filter(.value.name == \"${name}\") | .[0].key" ${parents_file})
63+
else
64+
result=$($YQ_PATH eval ".parents | to_entries | filter(.value.name == \"${name}\" and .value.version == \"${version}\") | .[0].key" ${parents_file})
65+
fi
66+
67+
if [ "${result}" == "null" ]; then
68+
echo "-1"
69+
else
70+
echo ${result}
71+
fi
72+
}
73+
74+
# Get child index if exists, else returns -1
75+
child_index() {
76+
parent_idx=$1
77+
name=$2
78+
version=$3
79+
80+
if [ -z "${version}" ]; then
81+
result=$($YQ_PATH eval ".parents.[${parent_idx}].children | to_entries | filter(.value.name == \"${name}\") | .[0].key" ${parents_file})
82+
else
83+
result=$($YQ_PATH eval ".parents.[${parent_idx}].children | to_entries | filter(.value.name == \"${name}\" and .value.version == \"${version}\") | .[0].key" ${parents_file})
84+
fi
85+
86+
if [ "${result}" == "null" ]; then
87+
echo "-1"
88+
else
89+
echo ${result}
90+
fi
91+
}
92+
93+
# Builds sample parent
94+
build_parent() {
95+
parent_name=$1
96+
parent_version=$2
97+
98+
if [ "${parent_version}" == "null" ]; then
99+
parent_version=""
100+
fi
101+
102+
if [ "${parent_name}" != "null" ]; then
103+
if [ ! -f ${parents_file} ]; then
104+
$YQ_PATH eval -n ".parents[0].name = \"${parent_name}\"" > ${parents_file}
105+
if [ "${parent_version}" != "" ]; then
106+
$YQ_PATH eval ".parents[0].version = \"${parent_version}\"" -i ${parents_file}
107+
fi
108+
109+
return
110+
fi
111+
112+
if [ "$($YQ_PATH eval .parents ${parents_file})" == "null" ]; then
113+
$YQ_PATH eval ".parents[0].name = \"${parent_name}\"" -i ${parents_file}
114+
if [ "${parent_version}" != "" ]; then
115+
$YQ_PATH eval ".parents[0].version = \"${parent_version}\"" -i ${parents_file}
116+
fi
117+
118+
return
119+
fi
120+
121+
parent_idx=$(parent_index ${parent_name} ${parent_version})
122+
if [ "${parent_idx}" == "-1" ]; then
123+
next_idx=$($YQ_PATH eval ".parents | length" ${parents_file})
124+
$YQ_PATH eval ".parents[${next_idx}].name = \"${parent_name}\"" -i ${parents_file}
125+
if [ "${parent_version}" != "" ]; then
126+
$YQ_PATH eval ".parents[${next_idx}].version = \"${parent_version}\"" -i ${parents_file}
127+
fi
128+
fi
129+
else
130+
return 1
131+
fi
132+
}
133+
134+
# Builds a child of a parent stack
135+
build_child() {
136+
parent_name=$1
137+
parent_version=$2
138+
sample_name=$3
139+
sample_version=$4
140+
141+
parent_idx=$(parent_index ${parent_name} ${parent_version})
142+
143+
if [ "$($YQ_PATH eval .parents[${parent_idx}].children ${parents_file})" == "null" ]; then
144+
$YQ_PATH eval ".parents[${parent_idx}].children[0].name = \"${sample_name}\"" -i ${parents_file}
145+
if [ "${sample_version}" != "" ]; then
146+
$YQ_PATH eval ".parents[${parent_idx}].children[0].version = \"${sample_version}\"" -i ${parents_file}
147+
fi
148+
149+
return
150+
fi
151+
152+
child_idx=$(child_index ${parent_idx} ${sample_name} ${sample_version})
153+
if [ "${child_idx}" == "-1" ]; then
154+
next_idx=$($YQ_PATH eval ".parents[${parent_idx}].children | length" ${parents_file})
155+
$YQ_PATH eval ".parents[${parent_idx}].children[${next_idx}].name = \"${sample_name}\"" -i ${parents_file}
156+
if [ "${sample_version}" != "" ]; then
157+
$YQ_PATH eval ".parents[${parent_idx}].children[${next_idx}].version = \"${sample_version}\"" -i ${parents_file}
158+
fi
159+
fi
160+
}
161+
162+
build_parents_file() {
163+
samples_len=$($YQ_PATH eval '.samples | length' ${samples_file})
164+
165+
for ((s_idx=0;s_idx<${samples_len};s_idx++)); do
166+
sample_name=$($YQ_PATH eval .samples.${s_idx}.name ${samples_file})
167+
sample_versions=($($YQ_PATH eval .samples.${s_idx}.versions.[].version ${samples_file}))
168+
169+
# Iterate through sample versions if sample has multi version support
170+
if [ ${#sample_versions[@]} -ne 0 ]; then
171+
for ((v_idx=0;v_idx<${#sample_versions[@]};v_idx++)); do
172+
devfile=${samples_dir}/${sample_name}/${sample_versions[$v_idx]}/devfile.yaml
173+
parent_name=$($YQ_PATH eval .parent.id ${devfile})
174+
parent_version=$(get_parent_version ${devfile} ${parent_name})
175+
build_parent ${parent_name} ${parent_version}
176+
177+
if [ $? -eq 0 ]; then
178+
build_child "${parent_name}" "${parent_version}" "${sample_name}" "${sample_versions[$v_idx]}"
179+
fi
180+
done
181+
else
182+
devfile=${samples_dir}/${sample_name}/devfile.yaml
183+
parent_name=$($YQ_PATH eval .parent.id ${devfile})
184+
parent_version=$(get_parent_version ${devfile} ${parent_name})
185+
build_parent ${parent_name} ${parent_version}
186+
187+
if [ $? -eq 0 ]; then
188+
build_child "${parent_name}" "${parent_version}" "${sample_name}" ""
189+
fi
190+
fi
191+
done
192+
}
193+
194+
# Gets the children sample paths of parents.
195+
# When TEST_DELTA is set to true, only children of parents
196+
# with changes are returned.
197+
get_children_of_parents() {
198+
stack_dirs=$(bash $(pwd)/tests/get_stacks.sh)
199+
children=()
200+
201+
for stack_dir in $stack_dirs; do
202+
if [ "$(basename $(dirname $stack_dir))" == "." ]; then
203+
stack_name=$(basename $stack_dir)
204+
205+
names=($($YQ_PATH eval ".parents | filter(.name == \"${stack_name}\") | .[0].children.[].name" ${parents_file}))
206+
versions=($($YQ_PATH eval ".parents | filter(.name == \"${stack_name}\") | .[0].children.[].version" ${parents_file}))
207+
else
208+
stack_name=$(basename $(dirname $stack_dir))
209+
stack_version=$(basename $stack_dir)
210+
211+
names=($($YQ_PATH eval ".parents | filter(.name == \"${stack_name}\" and .version == \"${stack_version}\") | .[0].children.[].name" ${parents_file}))
212+
versions=($($YQ_PATH eval ".parents | filter(.name == \"${stack_name}\" and .version == \"${stack_version}\") | .[0].children.[].version" ${parents_file}))
213+
fi
214+
215+
216+
for ((c_idx=0;c_idx<${#names[@]};c_idx++)); do
217+
if [ "${versions[$c_idx]}" == "null" ]; then
218+
children+=("${names[$c_idx]}")
219+
else
220+
children+=("${names[$c_idx]}/${versions[$c_idx]}")
221+
fi
222+
done
223+
done
224+
225+
echo ${children[@]}
226+
}
227+
228+
if [ ! -d ${base_path}/registry-support ]
229+
then
230+
if [ "$VERBOSE" == "true" ]
231+
then
232+
git clone https://github.com/devfile/registry-support ${base_path}/registry-support
233+
else
234+
git clone -q https://github.com/devfile/registry-support ${base_path}/registry-support
235+
fi
236+
fi
237+
238+
if [ "$VERBOSE" == "true" ]
239+
then
240+
bash ${base_path}/registry-support/build-tools/cache_samples.sh ${samples_file} ${samples_dir}
241+
else
242+
bash ${base_path}/registry-support/build-tools/cache_samples.sh ${samples_file} ${samples_dir} > /dev/null 2>&- echo
243+
fi
244+
245+
if [ -f ${parents_file} ]; then
246+
rm ${parents_file}
247+
fi
248+
249+
build_parents_file
250+
251+
get_children_of_parents

0 commit comments

Comments
 (0)