From aa0e1dd24f8d47f351935bc67f9587f7bd736e6b Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Fri, 14 Jul 2023 18:50:48 -0400 Subject: [PATCH 01/14] ignore cached remote samples Signed-off-by: Michael Valdron --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a2d3cfd3f..4815f375c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ index-generator registry-support/ .idea/ devfile-web/ +samples/.cache From 81113bfb569793ed17b885aabe1c1cd4b8f77d5d Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Fri, 14 Jul 2023 18:52:02 -0400 Subject: [PATCH 02/14] nightly script for building sample parent stack list Signed-off-by: Michael Valdron --- tests/build_sample_parents_nightly.sh | 53 +++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 tests/build_sample_parents_nightly.sh diff --git a/tests/build_sample_parents_nightly.sh b/tests/build_sample_parents_nightly.sh new file mode 100644 index 000000000..07477284c --- /dev/null +++ b/tests/build_sample_parents_nightly.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +base_path=$(dirname $0)/.. +samples_file=${base_path}/extraDevfileEntries.yaml +# Cached remote samples directory +samples_dir=${base_path}/samples/.cache + +# Clones remote samples into cache directory +clone_samples() { + samples_len=$(yq eval '.samples | length' ${samples_file}) + + # Removes old cached samples directory + if [ -d ${samples_dir} ]; then + rm -rf ${samples_dir} + fi + + for ((idx=0;idx<${samples_len};idx++)); do + name=$(yq eval .samples.${idx}.name ${samples_file}) + remote_url=$(yq eval .samples.${idx}.git.remotes.origin ${samples_file}) + + git clone --depth=1 ${remote_url} ${samples_dir}/${name} + done +} + +# Builds sample parent dependency file +build_parents() { + samples_len=$(yq eval '.samples | length' ${samples_file}) + + if [ -f parents.yaml ]; then + rm parents.yaml + fi + + for ((idx=0;idx<${samples_len};idx++)); do + name=$(yq eval .samples.${idx}.name ${samples_file}) + devfile=${samples_dir}/${name}/devfile.yaml + parent=$(yq eval .parent.id ${devfile}) + + if [ "${parent}" != "null" ]; then + if [ -f parents.yaml ] && [ "$(yq eval .parents.${parent}.children parents.yaml)" != "null" ]; then + next_idx=$(yq eval ".parents.${parent}.children | length" parents.yaml) + yq eval ".parents.${parent}.children[${next_idx}] = \"${name}\"" -i parents.yaml + elif [ -f parents.yaml ]; then + yq eval ".parents.${parent}.children[0] = \"${name}\"" -i parents.yaml + else + yq eval -n ".parents.${parent}.children[0] = \"${name}\"" > parents.yaml + fi + fi + done +} + +clone_samples + +build_parents \ No newline at end of file From 1abda838be0a969e94c5a9fdd814971bd0af9635 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Fri, 14 Jul 2023 18:52:39 -0400 Subject: [PATCH 03/14] init check parent stacks script Signed-off-by: Michael Valdron --- tests/check_parent_stacks.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 tests/check_parent_stacks.sh diff --git a/tests/check_parent_stacks.sh b/tests/check_parent_stacks.sh new file mode 100644 index 000000000..911a9aa68 --- /dev/null +++ b/tests/check_parent_stacks.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +base_path=$(dirname $0)/.. + From 1321736d29447355481b390d78fefac2ed03c9a1 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Tue, 18 Jul 2023 14:46:09 -0400 Subject: [PATCH 04/14] convert nightly parents build to on the fly parents build. Signed-off-by: Michael Valdron --- tests/build_sample_parents_nightly.sh | 53 --------------------------- tests/check_parent_stacks.sh | 49 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 53 deletions(-) delete mode 100644 tests/build_sample_parents_nightly.sh diff --git a/tests/build_sample_parents_nightly.sh b/tests/build_sample_parents_nightly.sh deleted file mode 100644 index 07477284c..000000000 --- a/tests/build_sample_parents_nightly.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -base_path=$(dirname $0)/.. -samples_file=${base_path}/extraDevfileEntries.yaml -# Cached remote samples directory -samples_dir=${base_path}/samples/.cache - -# Clones remote samples into cache directory -clone_samples() { - samples_len=$(yq eval '.samples | length' ${samples_file}) - - # Removes old cached samples directory - if [ -d ${samples_dir} ]; then - rm -rf ${samples_dir} - fi - - for ((idx=0;idx<${samples_len};idx++)); do - name=$(yq eval .samples.${idx}.name ${samples_file}) - remote_url=$(yq eval .samples.${idx}.git.remotes.origin ${samples_file}) - - git clone --depth=1 ${remote_url} ${samples_dir}/${name} - done -} - -# Builds sample parent dependency file -build_parents() { - samples_len=$(yq eval '.samples | length' ${samples_file}) - - if [ -f parents.yaml ]; then - rm parents.yaml - fi - - for ((idx=0;idx<${samples_len};idx++)); do - name=$(yq eval .samples.${idx}.name ${samples_file}) - devfile=${samples_dir}/${name}/devfile.yaml - parent=$(yq eval .parent.id ${devfile}) - - if [ "${parent}" != "null" ]; then - if [ -f parents.yaml ] && [ "$(yq eval .parents.${parent}.children parents.yaml)" != "null" ]; then - next_idx=$(yq eval ".parents.${parent}.children | length" parents.yaml) - yq eval ".parents.${parent}.children[${next_idx}] = \"${name}\"" -i parents.yaml - elif [ -f parents.yaml ]; then - yq eval ".parents.${parent}.children[0] = \"${name}\"" -i parents.yaml - else - yq eval -n ".parents.${parent}.children[0] = \"${name}\"" > parents.yaml - fi - fi - done -} - -clone_samples - -build_parents \ No newline at end of file diff --git a/tests/check_parent_stacks.sh b/tests/check_parent_stacks.sh index 911a9aa68..658c1460f 100644 --- a/tests/check_parent_stacks.sh +++ b/tests/check_parent_stacks.sh @@ -1,4 +1,53 @@ #!/bin/bash base_path=$(dirname $0)/.. +samples_file=${base_path}/extraDevfileEntries.yaml +# Cached remote samples directory +samples_dir=${base_path}/samples/.cache +# Clones remote samples into cache directory +clone_samples() { + samples_len=$(yq eval '.samples | length' ${samples_file}) + + # Removes old cached samples directory + if [ -d ${samples_dir} ]; then + rm -rf ${samples_dir} + fi + + for ((idx=0;idx<${samples_len};idx++)); do + name=$(yq eval .samples.${idx}.name ${samples_file}) + remote_url=$(yq eval .samples.${idx}.git.remotes.origin ${samples_file}) + + git clone --depth=1 ${remote_url} ${samples_dir}/${name} + done +} + +# Builds sample parent dependency file +build_parents() { + samples_len=$(yq eval '.samples | length' ${samples_file}) + + if [ -f parents.yaml ]; then + rm parents.yaml + fi + + for ((idx=0;idx<${samples_len};idx++)); do + name=$(yq eval .samples.${idx}.name ${samples_file}) + devfile=${samples_dir}/${name}/devfile.yaml + parent=$(yq eval .parent.id ${devfile}) + + if [ "${parent}" != "null" ]; then + if [ -f parents.yaml ] && [ "$(yq eval .parents.${parent}.children parents.yaml)" != "null" ]; then + next_idx=$(yq eval ".parents.${parent}.children | length" parents.yaml) + yq eval ".parents.${parent}.children[${next_idx}] = \"${name}\"" -i parents.yaml + elif [ -f parents.yaml ]; then + yq eval ".parents.${parent}.children[0] = \"${name}\"" -i parents.yaml + else + yq eval -n ".parents.${parent}.children[0] = \"${name}\"" > parents.yaml + fi + fi + done +} + +clone_samples + +build_parents From 8d71e6cc6db51c8a332e8fb44446f3d792cb7414 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Tue, 18 Jul 2023 20:09:15 -0400 Subject: [PATCH 05/14] multi version support for build parent stacks Signed-off-by: Michael Valdron --- tests/check_parent_stacks.sh | 72 ++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 20 deletions(-) diff --git a/tests/check_parent_stacks.sh b/tests/check_parent_stacks.sh index 658c1460f..a5bc34764 100644 --- a/tests/check_parent_stacks.sh +++ b/tests/check_parent_stacks.sh @@ -1,9 +1,17 @@ #!/bin/bash base_path=$(dirname $0)/.. +# default samples file path samples_file=${base_path}/extraDevfileEntries.yaml # Cached remote samples directory samples_dir=${base_path}/samples/.cache +parents_file=${base_path}/parents.yaml + +# Read samples file as first argument +# if unset use default samples file path +if [ ! -z "${1}" ]; then + samples_file=${1} +fi # Clones remote samples into cache directory clone_samples() { @@ -14,11 +22,22 @@ clone_samples() { rm -rf ${samples_dir} fi - for ((idx=0;idx<${samples_len};idx++)); do - name=$(yq eval .samples.${idx}.name ${samples_file}) - remote_url=$(yq eval .samples.${idx}.git.remotes.origin ${samples_file}) + for ((s_idx=0;s_idx<${samples_len};s_idx++)); do + name=$(yq eval .samples.${s_idx}.name ${samples_file}) + versions=($(yq eval .samples.${s_idx}.versions.[].version ${samples_file})) + + # Iterate through sample versions if sample has multi version support + if [ ${#versions[@]} -ne 0 ]; then + for ((v_idx=0;v_idx<${#versions[@]};v_idx++)); do + remote_url=$(yq eval .samples.${s_idx}.versions.${v_idx}.git.remotes.origin ${samples_file}) + + git clone --depth=1 ${remote_url} ${samples_dir}/${name}/${versions[$v_idx]} + done + else + remote_url=$(yq eval .samples.${s_idx}.git.remotes.origin ${samples_file}) - git clone --depth=1 ${remote_url} ${samples_dir}/${name} + git clone --depth=1 ${remote_url} ${samples_dir}/${name} + fi done } @@ -26,23 +45,36 @@ clone_samples() { build_parents() { samples_len=$(yq eval '.samples | length' ${samples_file}) - if [ -f parents.yaml ]; then - rm parents.yaml + if [ -f ${parents_file} ]; then + rm ${parents_file} fi - for ((idx=0;idx<${samples_len};idx++)); do - name=$(yq eval .samples.${idx}.name ${samples_file}) - devfile=${samples_dir}/${name}/devfile.yaml - parent=$(yq eval .parent.id ${devfile}) - - if [ "${parent}" != "null" ]; then - if [ -f parents.yaml ] && [ "$(yq eval .parents.${parent}.children parents.yaml)" != "null" ]; then - next_idx=$(yq eval ".parents.${parent}.children | length" parents.yaml) - yq eval ".parents.${parent}.children[${next_idx}] = \"${name}\"" -i parents.yaml - elif [ -f parents.yaml ]; then - yq eval ".parents.${parent}.children[0] = \"${name}\"" -i parents.yaml - else - yq eval -n ".parents.${parent}.children[0] = \"${name}\"" > parents.yaml + for ((s_idx=0;s_idx<${samples_len};s_idx++)); do + name=$(yq eval .samples.${s_idx}.name ${samples_file}) + versions=($(yq eval .samples.${s_idx}.versions.[].version ${samples_file})) + + # Iterate through sample versions if sample has multi version support + if [ ${#versions[@]} -ne 0 ]; then + for ((v_idx=0;v_idx<${#versions[@]};v_idx++)); do + devfile=${samples_dir}/${name}/${versions[$v_idx]}/devfile.yaml + parent=$(yq eval .parent.id ${devfile}) + parent_version=$(yq eval .parent.version ${devfile}) + + # TODO: multi version build control logic + done + else + devfile=${samples_dir}/${name}/devfile.yaml + parent=$(yq eval .parent.id ${devfile}) + + if [ "${parent}" != "null" ]; then + if [ -f ${parents_file} ] && [ "$(yq eval .parents.${parent}.children ${parents_file})" != "null" ]; then + next_idx=$(yq eval ".parents.${parent}.children | length" ${parents_file}) + yq eval ".parents.${parent}.children[${next_idx}] = \"${name}\"" -i ${parents_file} + elif [ -f ${parents_file} ]; then + yq eval ".parents.${parent}.children[0] = \"${name}\"" -i ${parents_file} + else + yq eval -n ".parents.${parent}.children[0] = \"${name}\"" > ${parents_file} + fi fi fi done @@ -50,4 +82,4 @@ build_parents() { clone_samples -build_parents +# build_parents From 9c86d26694f06fd8b5ff5c165adcc841772a6f08 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Tue, 25 Jul 2023 18:23:09 -0400 Subject: [PATCH 06/14] ignore any local parents file. Signed-off-by: Michael Valdron --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4815f375c..99e3e16bb 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ registry-support/ .idea/ devfile-web/ samples/.cache +parents.yaml From 76b1f4e095219a4a33b7874aff0d6de451681b6a Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Tue, 25 Jul 2023 18:29:43 -0400 Subject: [PATCH 07/14] rename check_parent_stacks.sh to build_parents_file.sh, build_parents_file.sh now builds the parents yaml file and returns the children samples of the parent stacks. Signed-off-by: Michael Valdron --- tests/build_parents_file.sh | 240 +++++++++++++++++++++++++++++++++++ tests/check_parent_stacks.sh | 85 ------------- 2 files changed, 240 insertions(+), 85 deletions(-) create mode 100644 tests/build_parents_file.sh delete mode 100644 tests/check_parent_stacks.sh diff --git a/tests/build_parents_file.sh b/tests/build_parents_file.sh new file mode 100644 index 000000000..865c4077d --- /dev/null +++ b/tests/build_parents_file.sh @@ -0,0 +1,240 @@ +#!/bin/bash + +# default samples file path +samples_file=$(pwd)/extraDevfileEntries.yaml +# Cached remote samples directory +samples_dir=$(pwd)/samples/.cache +# default stacks directory +stacks_dir=${STACKS:-"$(pwd)/stacks"} +parents_file=$(pwd)/parents.yaml + +# YAML query cmd path +YQ_PATH=${YQ_PATH:-yq} + +# Read samples file as first argument +# if unset use default samples file path +if [ ! -z "${1}" ]; then + samples_file=${1} +fi + +# Clones remote samples into cache directory +clone_samples() { + samples_len=$($YQ_PATH eval '.samples | length' ${samples_file}) + + # Removes old cached samples directory + if [ -d ${samples_dir} ]; then + rm -rf ${samples_dir} + fi + + for ((s_idx=0;s_idx<${samples_len};s_idx++)); do + name=$($YQ_PATH eval .samples.${s_idx}.name ${samples_file}) + versions=($($YQ_PATH eval .samples.${s_idx}.versions.[].version ${samples_file})) + + # Iterate through sample versions if sample has multi version support + if [ ${#versions[@]} -ne 0 ]; then + for ((v_idx=0;v_idx<${#versions[@]};v_idx++)); do + remote_url=$($YQ_PATH eval .samples.${s_idx}.versions.${v_idx}.git.remotes.origin ${samples_file}) + + git clone --depth=1 ${remote_url} ${samples_dir}/${name}/${versions[$v_idx]} + done + else + remote_url=$($YQ_PATH eval .samples.${s_idx}.git.remotes.origin ${samples_file}) + + git clone --depth=1 ${remote_url} ${samples_dir}/${name} + fi + done +} + +get_parent_version() { + devfile=$1 + name=$2 + version=$($YQ_PATH eval .parent.version ${devfile}) + + if [ "${version}" == "null" ] && [ -f "${stacks_dir}/${name}/stack.yaml" ]; then + version=$($YQ_PATH eval '.versions | filter(.default) | .[0].version' ${stacks_dir}/${name}/stack.yaml) + fi + + echo ${version} +} + +# Get parent index if exists, else returns -1 +parent_index() { + name=$1 + version=$2 + + if [ -z "${version}" ]; then + result=$($YQ_PATH eval ".parents | to_entries | filter(.value.name == \"${name}\") | .[0].key" ${parents_file}) + else + result=$($YQ_PATH eval ".parents | to_entries | filter(.value.name == \"${name}\" and .value.version == \"${version}\") | .[0].key" ${parents_file}) + fi + + if [ "${result}" == "null" ]; then + echo "-1" + else + echo ${result} + fi +} + +# Get child index if exists, else returns -1 +child_index() { + parent_idx=$1 + name=$2 + version=$3 + + if [ -z "${version}" ]; then + result=$($YQ_PATH eval ".parents.[${parent_idx}].children | to_entries | filter(.value.name == \"${name}\") | .[0].key" ${parents_file}) + else + result=$($YQ_PATH eval ".parents.[${parent_idx}].children | to_entries | filter(.value.name == \"${name}\" and .value.version == \"${version}\") | .[0].key" ${parents_file}) + fi + + if [ "${result}" == "null" ]; then + echo "-1" + else + echo ${result} + fi +} + +# Builds sample parents +build_parents() { + parent_name=$1 + parent_version=$2 + + if [ "${parent_version}" == "null" ]; then + parent_version="" + fi + + if [ "${parent_name}" != "null" ]; then + if [ ! -f ${parents_file} ]; then + $YQ_PATH eval -n ".parents[0].name = \"${parent_name}\"" > ${parents_file} + if [ "${parent_version}" != "" ]; then + $YQ_PATH eval ".parents[0].version = \"${parent_version}\"" -i ${parents_file} + fi + + return + fi + + if [ "$($YQ_PATH eval .parents ${parents_file})" == "null" ]; then + $YQ_PATH eval ".parents[0].name = \"${parent_name}\"" -i ${parents_file} + if [ "${parent_version}" != "" ]; then + $YQ_PATH eval ".parents[0].version = \"${parent_version}\"" -i ${parents_file} + fi + + return + fi + + parent_idx=$(parent_index ${parent_name} ${parent_version}) + if [ "${parent_idx}" == "-1" ]; then + next_idx=$($YQ_PATH eval ".parents | length" ${parents_file}) + $YQ_PATH eval ".parents[${next_idx}].name = \"${parent_name}\"" -i ${parents_file} + if [ "${parent_version}" != "" ]; then + $YQ_PATH eval ".parents[${next_idx}].version = \"${parent_version}\"" -i ${parents_file} + fi + fi + else + return 1 + fi +} + +# Builds children of parent stacks +build_children() { + parent_name=$1 + parent_version=$2 + sample_name=$3 + sample_version=$4 + + parent_idx=$(parent_index ${parent_name} ${parent_version}) + + if [ "$($YQ_PATH eval .parents[${parent_idx}].children ${parents_file})" == "null" ]; then + $YQ_PATH eval ".parents[${parent_idx}].children[0].name = \"${sample_name}\"" -i ${parents_file} + if [ "${sample_version}" != "" ]; then + $YQ_PATH eval ".parents[${parent_idx}].children[0].version = \"${sample_version}\"" -i ${parents_file} + fi + + return + fi + + child_idx=$(child_index ${parent_idx} ${sample_name} ${sample_version}) + if [ "${child_idx}" == "-1" ]; then + next_idx=$($YQ_PATH eval ".parents[${parent_idx}].children | length" ${parents_file}) + $YQ_PATH eval ".parents[${parent_idx}].children[${next_idx}].name = \"${sample_name}\"" -i ${parents_file} + if [ "${sample_version}" != "" ]; then + $YQ_PATH eval ".parents[${parent_idx}].children[${next_idx}].version = \"${sample_version}\"" -i ${parents_file} + fi + fi +} + +build_parents_file() { + samples_len=$($YQ_PATH eval '.samples | length' ${samples_file}) + + for ((s_idx=0;s_idx<${samples_len};s_idx++)); do + sample_name=$($YQ_PATH eval .samples.${s_idx}.name ${samples_file}) + sample_versions=($($YQ_PATH eval .samples.${s_idx}.versions.[].version ${samples_file})) + + # Iterate through sample versions if sample has multi version support + if [ ${#sample_versions[@]} -ne 0 ]; then + for ((v_idx=0;v_idx<${#sample_versions[@]};v_idx++)); do + devfile=${samples_dir}/${sample_name}/${sample_versions[$v_idx]}/devfile.yaml + parent_name=$($YQ_PATH eval .parent.id ${devfile}) + parent_version=$(get_parent_version ${devfile} ${parent_name}) + build_parents ${parent_name} ${parent_version} + + if [ $? -eq 0 ]; then + build_children "${parent_name}" "${parent_version}" "${sample_name}" "${sample_versions[$v_idx]}" + fi + done + else + devfile=${samples_dir}/${sample_name}/devfile.yaml + parent_name=$($YQ_PATH eval .parent.id ${devfile}) + parent_version=$(get_parent_version ${devfile} ${parent_name}) + build_parents ${parent_name} ${parent_version} + + if [ $? -eq 0 ]; then + build_children "${parent_name}" "${parent_version}" "${sample_name}" "" + fi + fi + done +} + +# Gets the children sample paths of parents. +# When TEST_DELTA is set to true, only children of parents +# with changes are returned. +get_children_of_parents() { + stack_dirs=$(bash $(pwd)/tests/get_stacks.sh) + children=() + + for stack_dir in $stack_dirs; do + if [ "$(basename $(dirname $stack_dir))" == "." ]; then + stack_name=$(basename $stack_dir) + + names=($($YQ_PATH eval ".parents | filter(.name == \"${stack_name}\") | .[0].children.[].name" ${parents_file})) + versions=($($YQ_PATH eval ".parents | filter(.name == \"${stack_name}\") | .[0].children.[].version" ${parents_file})) + else + stack_name=$(basename $(dirname $stack_dir)) + stack_version=$(basename $stack_dir) + + names=($($YQ_PATH eval ".parents | filter(.name == \"${stack_name}\" and .version == \"${stack_version}\") | .[0].children.[].name" ${parents_file})) + versions=($($YQ_PATH eval ".parents | filter(.name == \"${stack_name}\" and .version == \"${stack_version}\") | .[0].children.[].version" ${parents_file})) + fi + + + for ((c_idx=0;c_idx<${#names[@]};c_idx++)); do + if [ "${versions[$c_idx]}" == "null" ]; then + children+=("${names[$c_idx]}") + else + children+=("${names[$c_idx]}/${versions[$c_idx]}") + fi + done + done + + echo ${children[@]} +} + +clone_samples + +if [ -f ${parents_file} ]; then + rm ${parents_file} +fi + +build_parents_file + +echo $(get_children_of_parents) diff --git a/tests/check_parent_stacks.sh b/tests/check_parent_stacks.sh deleted file mode 100644 index a5bc34764..000000000 --- a/tests/check_parent_stacks.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash - -base_path=$(dirname $0)/.. -# default samples file path -samples_file=${base_path}/extraDevfileEntries.yaml -# Cached remote samples directory -samples_dir=${base_path}/samples/.cache -parents_file=${base_path}/parents.yaml - -# Read samples file as first argument -# if unset use default samples file path -if [ ! -z "${1}" ]; then - samples_file=${1} -fi - -# Clones remote samples into cache directory -clone_samples() { - samples_len=$(yq eval '.samples | length' ${samples_file}) - - # Removes old cached samples directory - if [ -d ${samples_dir} ]; then - rm -rf ${samples_dir} - fi - - for ((s_idx=0;s_idx<${samples_len};s_idx++)); do - name=$(yq eval .samples.${s_idx}.name ${samples_file}) - versions=($(yq eval .samples.${s_idx}.versions.[].version ${samples_file})) - - # Iterate through sample versions if sample has multi version support - if [ ${#versions[@]} -ne 0 ]; then - for ((v_idx=0;v_idx<${#versions[@]};v_idx++)); do - remote_url=$(yq eval .samples.${s_idx}.versions.${v_idx}.git.remotes.origin ${samples_file}) - - git clone --depth=1 ${remote_url} ${samples_dir}/${name}/${versions[$v_idx]} - done - else - remote_url=$(yq eval .samples.${s_idx}.git.remotes.origin ${samples_file}) - - git clone --depth=1 ${remote_url} ${samples_dir}/${name} - fi - done -} - -# Builds sample parent dependency file -build_parents() { - samples_len=$(yq eval '.samples | length' ${samples_file}) - - if [ -f ${parents_file} ]; then - rm ${parents_file} - fi - - for ((s_idx=0;s_idx<${samples_len};s_idx++)); do - name=$(yq eval .samples.${s_idx}.name ${samples_file}) - versions=($(yq eval .samples.${s_idx}.versions.[].version ${samples_file})) - - # Iterate through sample versions if sample has multi version support - if [ ${#versions[@]} -ne 0 ]; then - for ((v_idx=0;v_idx<${#versions[@]};v_idx++)); do - devfile=${samples_dir}/${name}/${versions[$v_idx]}/devfile.yaml - parent=$(yq eval .parent.id ${devfile}) - parent_version=$(yq eval .parent.version ${devfile}) - - # TODO: multi version build control logic - done - else - devfile=${samples_dir}/${name}/devfile.yaml - parent=$(yq eval .parent.id ${devfile}) - - if [ "${parent}" != "null" ]; then - if [ -f ${parents_file} ] && [ "$(yq eval .parents.${parent}.children ${parents_file})" != "null" ]; then - next_idx=$(yq eval ".parents.${parent}.children | length" ${parents_file}) - yq eval ".parents.${parent}.children[${next_idx}] = \"${name}\"" -i ${parents_file} - elif [ -f ${parents_file} ]; then - yq eval ".parents.${parent}.children[0] = \"${name}\"" -i ${parents_file} - else - yq eval -n ".parents.${parent}.children[0] = \"${name}\"" > ${parents_file} - fi - fi - fi - done -} - -clone_samples - -# build_parents From 563d7911baca8d8a2543386a6fd0a1f63b68c651 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Tue, 25 Jul 2023 18:31:05 -0400 Subject: [PATCH 08/14] validate devfile schemas test suite parameters now changeable Signed-off-by: Michael Valdron --- tests/validate_devfile_schemas.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/validate_devfile_schemas.sh b/tests/validate_devfile_schemas.sh index 305dc8898..e967ddc14 100755 --- a/tests/validate_devfile_schemas.sh +++ b/tests/validate_devfile_schemas.sh @@ -2,7 +2,8 @@ set -x -stackDirs=$(bash "$(pwd)/tests/get_stacks.sh") +stacksDir=${STACKS_DIR:-"$(pwd)/stacks"} +stackDirs=${STACKS:-"$(bash "$(pwd)/tests/get_stacks.sh")"} ginkgo run --procs 2 \ - tests/validate_devfile_schemas -- -stacksPath "$(pwd)"/stacks -stackDirs "$stackDirs" + tests/validate_devfile_schemas -- -stacksPath ${stacksDir} -stackDirs "$stackDirs" From 88722d52da3ecea1c14c756231c3661b809a400a Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Tue, 25 Jul 2023 18:32:10 -0400 Subject: [PATCH 09/14] validate samples workflow Signed-off-by: Michael Valdron --- .github/workflows/validate-samples.yaml | 61 +++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .github/workflows/validate-samples.yaml diff --git a/.github/workflows/validate-samples.yaml b/.github/workflows/validate-samples.yaml new file mode 100644 index 000000000..8a54c9581 --- /dev/null +++ b/.github/workflows/validate-samples.yaml @@ -0,0 +1,61 @@ +# +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Validate child samples + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: 0 5 * * * + +concurrency: + group: ${{ github.workflow }}-${{ github.event.number || github.ref }} + cancel-in-progress: true + +env: + MINIKUBE_VERSION: "v1.29.0" + KUBERNETES_VERSION: "v1.25.2" + TEST_DELTA: false + +jobs: + validate-devfile-schema: + name: validate devfile schemas + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: "1.19" + + - name: Install Ginkgo + run: go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo@v2.5.0 + + - name: Test delta if on a pull request + if: ${{ github.event_name == 'pull_request' }} + run: echo "TEST_DELTA=true" >> $GITHUB_ENV + + - name: Build parents file and get child samples + run: echo "STACKS=$(bash tests/build_parents_file.sh)" >> $GITHUB_ENV + + - name: Validate samples + run: STACKS_DIR=$(pwd)/samples/.cache bash tests/validate_devfile_schemas.sh From a5da72eb5f3a182f62b39b62cb187404b4803b90 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Wed, 26 Jul 2023 10:49:07 -0400 Subject: [PATCH 10/14] add check for if there are samples to be checked under validate samples job. Signed-off-by: Michael Valdron --- .github/workflows/validate-samples.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate-samples.yaml b/.github/workflows/validate-samples.yaml index 8a54c9581..fbe215d35 100644 --- a/.github/workflows/validate-samples.yaml +++ b/.github/workflows/validate-samples.yaml @@ -58,4 +58,5 @@ jobs: run: echo "STACKS=$(bash tests/build_parents_file.sh)" >> $GITHUB_ENV - name: Validate samples + if: ${{ env.STACKS != '' }} run: STACKS_DIR=$(pwd)/samples/.cache bash tests/validate_devfile_schemas.sh From 3464e437c294ae59da19e22321d66534ff5c5b30 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Wed, 26 Jul 2023 11:58:01 -0400 Subject: [PATCH 11/14] add README instruction on dependency check changes Signed-off-by: Michael Valdron --- tests/README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/README.md b/tests/README.md index 424249f85..1ac93c4c6 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,5 +1,29 @@ # Devfile Registry Testing +## Dependency check + +### Prerequisites + +- Ensure `yq 4.3x` is installed + +### Running the build script + +- This script performs three actions + - Clones samples from provided `extraDevfileEntries.yaml` under `samples/.cache` + - Creates a `parents.yaml` which contains the dependency tree for parent stacks + - Outputs the child sample paths of parent stacks, `TEST_DELTA=true` will result in only outputting child samples which have changed parent stacks +- The build script takes one optional argument and works off of the current working directory + - `bash tests/build_parents_file.sh`, default samples file is `extraDevfileEntries.yaml` + - `bash tests/build_parents_file.sh ` + +### Use with testing + +- One can test the child samples using the [validate_devfile_schemas]() test suite by performing the following: +```sh +export STACKS=$(bash tests/build_parents_file.sh) +STACKS_DIR=samples/.cache bash tests/validate_devfile_schemas.sh +``` + ## Validating non-terminating images ### Prerequisites From 959de4d8675cda71bd446b7f8c9cc519a676a789 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Mon, 31 Jul 2023 14:50:02 -0400 Subject: [PATCH 12/14] yq 4.x installation guide link added to testing prerequisites list. Signed-off-by: Michael Valdron --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 1ac93c4c6..a7a1a570c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -4,7 +4,7 @@ ### Prerequisites -- Ensure `yq 4.3x` is installed +- Ensure [yq 4.x](https://github.com/mikefarah/yq/#install) is installed ### Running the build script From 1b375fbdf9d3db5739fed64214ef34a83021ed34 Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Mon, 31 Jul 2023 15:04:51 -0400 Subject: [PATCH 13/14] add missing link to test suite Signed-off-by: Michael Valdron --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index a7a1a570c..2b3903b95 100644 --- a/tests/README.md +++ b/tests/README.md @@ -18,7 +18,7 @@ ### Use with testing -- One can test the child samples using the [validate_devfile_schemas]() test suite by performing the following: +- One can test the child samples using the [validate_devfile_schemas](./validate_devfile_schemas/) test suite by performing the following: ```sh export STACKS=$(bash tests/build_parents_file.sh) STACKS_DIR=samples/.cache bash tests/validate_devfile_schemas.sh From 7c0d08c3780917e6905f526be5cb5c6b619d62bb Mon Sep 17 00:00:00 2001 From: Michael Valdron Date: Mon, 31 Jul 2023 16:33:17 -0400 Subject: [PATCH 14/14] validate_devfile_schemas script accepts relative or absolute paths Signed-off-by: Michael Valdron --- tests/validate_devfile_schemas.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/validate_devfile_schemas.sh b/tests/validate_devfile_schemas.sh index e967ddc14..eac6a657a 100755 --- a/tests/validate_devfile_schemas.sh +++ b/tests/validate_devfile_schemas.sh @@ -2,8 +2,13 @@ set -x -stacksDir=${STACKS_DIR:-"$(pwd)/stacks"} +stacksDir=${STACKS_DIR:-stacks} stackDirs=${STACKS:-"$(bash "$(pwd)/tests/get_stacks.sh")"} +# Use pwd if relative path +if [[ ! ${stacksDir} = /* ]]; then + stacksDir=$(pwd)/${stacksDir} +fi + ginkgo run --procs 2 \ tests/validate_devfile_schemas -- -stacksPath ${stacksDir} -stackDirs "$stackDirs"