From 5194209efa422de3b9dae329ac06d6f5eedaa12d Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Fri, 17 Jan 2025 14:58:07 -0700 Subject: [PATCH 01/68] start adding linter --- .github/scripts/lint.py | 27 ++++++++++++++++++++ .github/workflows/lint-pr.yml | 47 +++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 .github/scripts/lint.py create mode 100644 .github/workflows/lint-pr.yml diff --git a/.github/scripts/lint.py b/.github/scripts/lint.py new file mode 100644 index 0000000..44871e6 --- /dev/null +++ b/.github/scripts/lint.py @@ -0,0 +1,27 @@ +import sys +import yaml +import requests + + +KEY_SET = set(['owner', 'name', 'branch', 'docs']) + + +def lint(yml): + # Assert corrrect keys + assert set(yml.keys()) == KEY_SET + + # Get the docs URL assert 200 + response = requests.get(yml['docs']) + assert response.status_code == 200 + + # Put together the owner/name:branch + # ...Do something with it + + +if __name__ == "__main__": + files = sys.argv[1:] + + for file in files: + with open(file, 'r') as fh: + yml = yaml.safe_load(fh) + lint(yml) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml new file mode 100644 index 0000000..16bb308 --- /dev/null +++ b/.github/workflows/lint-pr.yml @@ -0,0 +1,47 @@ +name: lint-pr + +# Run on PR +on: + # Possible to only run on PR that modifies plugins? + pull_request: + branches: + - 'main' + paths: + - '/plugins/**' + workflow_dispatch: + +# Checkout +jobs: + lint: + permissions: + contents: read + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + sparse-checkout: plugins + + # Get new/changed files? + # Not positive this works yet + - name: Added Files + run: | + git fetch origin main + GITHUB_OUTPUT=`git diff --name-only --diff-filter=A main~ main` + + # Lint all added files + - name: Lint + run: | + python /.github/scripts/lint.py $GITHUB_OUTPUT + + +# Lint all files in plugins + # If anything outside of plugins is changed then fail? + +# Grab the target repo and validate it? + # Description? + # README? + # Env files? + From e7eb7d6ace8b38a1a1f3c7ecf39b96cedb7c725f Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 11:42:09 -0700 Subject: [PATCH 02/68] Whitespace error? --- .github/workflows/lint-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 16bb308..c67f57e 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -8,7 +8,7 @@ on: - 'main' paths: - '/plugins/**' - workflow_dispatch: + workflow_dispatch: # Checkout jobs: From f3763cf2e59899519a116c759dfd123017cd69dc Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 11:45:35 -0700 Subject: [PATCH 03/68] Simplify condition --- .github/workflows/lint-pr.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index c67f57e..1dc895d 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -4,11 +4,10 @@ name: lint-pr on: # Possible to only run on PR that modifies plugins? pull_request: - branches: - - 'main' - paths: - - '/plugins/**' - workflow_dispatch: + # branches: + # - 'main' + # paths: + # - '/plugins/**' # Checkout jobs: From 125ca729a6df6f415b1f19bd532121e74ed5e09c Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 11:52:19 -0700 Subject: [PATCH 04/68] Different way to get diff --- .github/workflows/lint-pr.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 1dc895d..b17dfc2 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -27,8 +27,8 @@ jobs: # Not positive this works yet - name: Added Files run: | - git fetch origin main - GITHUB_OUTPUT=`git diff --name-only --diff-filter=A main~ main` + git fetch origin ${{ github.event.pull_request.base.ref }} + git diff --name-only --diff-filter=A origin/${{ github.event.pull_request.base.ref }}...${{ github.sha }} # Lint all added files - name: Lint From 52a4fb45043f0d7c20dd78a1d9b934f6875357ac Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 11:56:18 -0700 Subject: [PATCH 05/68] HEAD not main --- .github/workflows/lint-pr.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index b17dfc2..64b9e28 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -26,9 +26,7 @@ jobs: # Get new/changed files? # Not positive this works yet - name: Added Files - run: | - git fetch origin ${{ github.event.pull_request.base.ref }} - git diff --name-only --diff-filter=A origin/${{ github.event.pull_request.base.ref }}...${{ github.sha }} + run: GITHUB_OUTPUT=`git diff --name-only --diff-filter=A HEAD~ HEAD` # Lint all added files - name: Lint From 2f1155d4393b0e3701e6b322870e2fee36207545 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 11:57:48 -0700 Subject: [PATCH 06/68] need fetch depth 2? --- .github/workflows/lint-pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 64b9e28..194ca0c 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -22,6 +22,7 @@ jobs: uses: actions/checkout@v4 with: sparse-checkout: plugins + fetch-depth: 2 # Get new/changed files? # Not positive this works yet From bb5d79b19c6ac64c9ff85cab54ceff5356eb3891 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 12:02:03 -0700 Subject: [PATCH 07/68] No leading / --- .github/workflows/lint-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 194ca0c..aebe2ba 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -32,7 +32,7 @@ jobs: # Lint all added files - name: Lint run: | - python /.github/scripts/lint.py $GITHUB_OUTPUT + python .github/scripts/lint.py $GITHUB_OUTPUT # Lint all files in plugins From 979daf76b1a203a80ca24f8fed77f162eedb6faf Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 12:10:29 -0700 Subject: [PATCH 08/68] abspath --- .github/workflows/lint-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index aebe2ba..b9b774f 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -32,7 +32,7 @@ jobs: # Lint all added files - name: Lint run: | - python .github/scripts/lint.py $GITHUB_OUTPUT + python /home/runner/work/library-plugins/.github/scripts/lint.py $GITHUB_OUTPUT # Lint all files in plugins From 90406c86f45455ab2ea0b072a4885f84ea40b537 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 12:13:12 -0700 Subject: [PATCH 09/68] put on one line --- .github/workflows/lint-pr.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index b9b774f..acc40ed 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -31,8 +31,7 @@ jobs: # Lint all added files - name: Lint - run: | - python /home/runner/work/library-plugins/.github/scripts/lint.py $GITHUB_OUTPUT + run: python .github/scripts/lint.py $GITHUB_OUTPUT # Lint all files in plugins From a5495d3cfeecc394f46ea213e710b3df3ec3e907 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 13:31:27 -0700 Subject: [PATCH 10/68] find --- .github/workflows/lint-pr.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index acc40ed..82ec2b1 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -31,7 +31,9 @@ jobs: # Lint all added files - name: Lint - run: python .github/scripts/lint.py $GITHUB_OUTPUT + run: | + find / -type f -iname lint.py + python .github/scripts/lint.py $GITHUB_OUTPUT # Lint all files in plugins From 1cc258c5d0aca6403243fd359f1715be45ce7493 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 13:37:07 -0700 Subject: [PATCH 11/68] Make find narrower --- .github/workflows/lint-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 82ec2b1..7d374f6 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -32,7 +32,7 @@ jobs: # Lint all added files - name: Lint run: | - find / -type f -iname lint.py + find /home/runner/work -type f -iname lint.py 2> /dev/null python .github/scripts/lint.py $GITHUB_OUTPUT From 9d971beab5e16fcfd24002db3b07df58f1bde62f Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 13:38:43 -0700 Subject: [PATCH 12/68] no sparse checkout --- .github/workflows/lint-pr.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 7d374f6..df8a897 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -21,7 +21,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - sparse-checkout: plugins fetch-depth: 2 # Get new/changed files? @@ -31,9 +30,7 @@ jobs: # Lint all added files - name: Lint - run: | - find /home/runner/work -type f -iname lint.py 2> /dev/null - python .github/scripts/lint.py $GITHUB_OUTPUT + run: python .github/scripts/lint.py $GITHUB_OUTPUT # Lint all files in plugins From 6e050ffd757554d7f18fd2fea5d101f1d4b27c7f Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 13:40:23 -0700 Subject: [PATCH 13/68] View output --- .github/workflows/lint-pr.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index df8a897..fefd498 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -30,7 +30,9 @@ jobs: # Lint all added files - name: Lint - run: python .github/scripts/lint.py $GITHUB_OUTPUT + run: | + echo $GITHUB_OUTPUT + python .github/scripts/lint.py $GITHUB_OUTPUT # Lint all files in plugins From 1e446e9c850e8f2fbcee57e2e036f8e350b7687b Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 13:43:19 -0700 Subject: [PATCH 14/68] Different syntax to store outputs --- .github/workflows/lint-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index fefd498..0bc290a 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -26,7 +26,7 @@ jobs: # Get new/changed files? # Not positive this works yet - name: Added Files - run: GITHUB_OUTPUT=`git diff --name-only --diff-filter=A HEAD~ HEAD` + run: echo "git diff --name-only --diff-filter=A HEAD~ HEAD" >> $GITHUB_OUTPUT # Lint all added files - name: Lint From 7c6f398d8c5b57f598b2c15a93cf3ec3e01e7647 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 13:49:44 -0700 Subject: [PATCH 15/68] change get outputs syntax --- .github/workflows/lint-pr.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 0bc290a..0e73c4e 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -27,12 +27,13 @@ jobs: # Not positive this works yet - name: Added Files run: echo "git diff --name-only --diff-filter=A HEAD~ HEAD" >> $GITHUB_OUTPUT + id: get_added_files # Lint all added files - name: Lint run: | - echo $GITHUB_OUTPUT - python .github/scripts/lint.py $GITHUB_OUTPUT + echo ${{ steps.get_added_files.outputs }} + python .github/scripts/lint.py ${{ steps.get_added_files.outputs }} # Lint all files in plugins From 394510f583fbe2e82348beec66daf57355f1e786 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 13:52:59 -0700 Subject: [PATCH 16/68] change get outputs syntax --- .github/workflows/lint-pr.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 0e73c4e..477339e 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -26,14 +26,14 @@ jobs: # Get new/changed files? # Not positive this works yet - name: Added Files - run: echo "git diff --name-only --diff-filter=A HEAD~ HEAD" >> $GITHUB_OUTPUT + run: echo "DIFF=`git diff --name-only --diff-filter=A HEAD~ HEAD` >> $GITHUB_ENV id: get_added_files # Lint all added files - name: Lint run: | - echo ${{ steps.get_added_files.outputs }} - python .github/scripts/lint.py ${{ steps.get_added_files.outputs }} + echo $DIFF + python .github/scripts/lint.py $DIFF # Lint all files in plugins From 2916197b8fd05c1d7f1464d3177973a6ab5bd4d0 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 13:54:08 -0700 Subject: [PATCH 17/68] Close quote --- .github/workflows/lint-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 477339e..e4dd2da 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -26,7 +26,7 @@ jobs: # Get new/changed files? # Not positive this works yet - name: Added Files - run: echo "DIFF=`git diff --name-only --diff-filter=A HEAD~ HEAD` >> $GITHUB_ENV + run: echo "DIFF=`git diff --name-only --diff-filter=A HEAD~ HEAD`" >> $GITHUB_ENV id: get_added_files # Lint all added files From 97fb4f12f5afd561124d16e402009d233c70796b Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 13:55:06 -0700 Subject: [PATCH 18/68] cleanup --- .github/workflows/lint-pr.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index e4dd2da..a9464c7 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -27,13 +27,10 @@ jobs: # Not positive this works yet - name: Added Files run: echo "DIFF=`git diff --name-only --diff-filter=A HEAD~ HEAD`" >> $GITHUB_ENV - id: get_added_files # Lint all added files - name: Lint - run: | - echo $DIFF - python .github/scripts/lint.py $DIFF + run: python .github/scripts/lint.py $DIFF # Lint all files in plugins From fc7d6eed886b8acf754483ce2f689d60c322f806 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 14:00:08 -0700 Subject: [PATCH 19/68] Get all files in diff --- .github/workflows/lint-pr.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index a9464c7..3338329 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -24,9 +24,8 @@ jobs: fetch-depth: 2 # Get new/changed files? - # Not positive this works yet - name: Added Files - run: echo "DIFF=`git diff --name-only --diff-filter=A HEAD~ HEAD`" >> $GITHUB_ENV + run: echo "DIFF=`git diff --name-only HEAD~ HEAD`" >> $GITHUB_ENV # Lint all added files - name: Lint From da85e9783b93f5d75f570a9c3a8fa1f493a575bd Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 14:02:36 -0700 Subject: [PATCH 20/68] Add back requirements to run --- .github/workflows/lint-pr.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 3338329..e674759 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -4,10 +4,10 @@ name: lint-pr on: # Possible to only run on PR that modifies plugins? pull_request: - # branches: - # - 'main' - # paths: - # - '/plugins/**' + branches: + - 'main' + paths: + - '/plugins/**' # Checkout jobs: From 37d0cd2380e12f04b8dfe9c98dba1ca45f3c58d9 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 14:03:24 -0700 Subject: [PATCH 21/68] No leading / --- .github/workflows/lint-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index e674759..ce96217 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -7,7 +7,7 @@ on: branches: - 'main' paths: - - '/plugins/**' + - 'plugins/**' # Checkout jobs: From a7d634b9d4122ad60d80f5b11edee0a578bd1fe2 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 14:15:36 -0700 Subject: [PATCH 22/68] Update comments --- .github/workflows/lint-pr.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index ce96217..6e96fb4 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -1,15 +1,13 @@ name: lint-pr -# Run on PR +# Run on PR changing files in plugins dir against main on: - # Possible to only run on PR that modifies plugins? pull_request: branches: - 'main' paths: - 'plugins/**' -# Checkout jobs: lint: permissions: @@ -18,16 +16,17 @@ jobs: runs-on: ubuntu-latest steps: + # Checkout new commit and prior HEAD - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 2 - # Get new/changed files? - - name: Added Files + # Get diff + - name: Get Diff run: echo "DIFF=`git diff --name-only HEAD~ HEAD`" >> $GITHUB_ENV - # Lint all added files + # Lint all plugins/** files in diff (filtering of diff occurs in Python) - name: Lint run: python .github/scripts/lint.py $DIFF From 474b7987944d67c8e19d67e10b079d061faf5607 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 14:29:17 -0700 Subject: [PATCH 23/68] Only lint files in plugins and assert their names match pattern --- .github/scripts/lint.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/scripts/lint.py b/.github/scripts/lint.py index 44871e6..83886a8 100644 --- a/.github/scripts/lint.py +++ b/.github/scripts/lint.py @@ -1,3 +1,4 @@ +import os import sys import yaml import requests @@ -22,6 +23,13 @@ def lint(yml): files = sys.argv[1:] for file in files: - with open(file, 'r') as fh: - yml = yaml.safe_load(fh) - lint(yml) + head, tail = os.path.split(file) + + # We only care about files added to the plugins dir + if head == 'plugins': + if tail[0:3] != 'q2-' or tail[-4:] != '.yml': + raise ValueError('File name must conform to q2-*.yml') + + with open(file, 'r') as fh: + yml = yaml.safe_load(fh) + lint(yml) From d098632a817ed95d51a1c9b45a387f4ff2bc7e43 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 14:41:03 -0700 Subject: [PATCH 24/68] Refactor vars --- .github/scripts/lint.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/scripts/lint.py b/.github/scripts/lint.py index 83886a8..72a407f 100644 --- a/.github/scripts/lint.py +++ b/.github/scripts/lint.py @@ -4,6 +4,10 @@ import requests +DIR = 'plugins' +FILE_START = 'q2-' +FILE_EXT = '.yml' + KEY_SET = set(['owner', 'name', 'branch', 'docs']) @@ -24,10 +28,11 @@ def lint(yml): for file in files: head, tail = os.path.split(file) + file_name, file_ext = os.path.splitext(tail) # We only care about files added to the plugins dir - if head == 'plugins': - if tail[0:3] != 'q2-' or tail[-4:] != '.yml': + if head == DIR: + if file_name[0:3] != FILE_START or file_ext != FILE_EXT: raise ValueError('File name must conform to q2-*.yml') with open(file, 'r') as fh: From cfe7d1ff69dbb3282408dff1f5d9184070d65e68 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 14:49:49 -0700 Subject: [PATCH 25/68] view diff --- .github/workflows/lint-pr.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 6e96fb4..96c2537 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -24,7 +24,9 @@ jobs: # Get diff - name: Get Diff - run: echo "DIFF=`git diff --name-only HEAD~ HEAD`" >> $GITHUB_ENV + run: | + git diff --name-only HEAD~ HEAD + echo "DIFF=`git diff --name-only HEAD~ HEAD`" >> $GITHUB_ENV # Lint all plugins/** files in diff (filtering of diff occurs in Python) - name: Lint From 9c4d07d5a01dfcfe40d9813bd7f47d193e3880e6 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 15:39:02 -0700 Subject: [PATCH 26/68] multi line --- .github/workflows/lint-pr.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 96c2537..d526ac9 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -23,10 +23,16 @@ jobs: fetch-depth: 2 # Get diff + # This feels odd to me got it from + # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#multiline-strings - name: Get Diff run: | git diff --name-only HEAD~ HEAD - echo "DIFF=`git diff --name-only HEAD~ HEAD`" >> $GITHUB_ENV + { + echo 'DIFF<<\n' + git diff --name-only HEAD~ HEAD + echo \n + } >> "$GITUB_ENV" # Lint all plugins/** files in diff (filtering of diff occurs in Python) - name: Lint From f69b6ac460bda79303c51765ac15afc9bf216417 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 15:43:47 -0700 Subject: [PATCH 27/68] Make \n a string --- .github/workflows/lint-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index d526ac9..7bd259c 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -31,7 +31,7 @@ jobs: { echo 'DIFF<<\n' git diff --name-only HEAD~ HEAD - echo \n + echo '\n' } >> "$GITUB_ENV" # Lint all plugins/** files in diff (filtering of diff occurs in Python) From ddad5299b27896b908789774d5b302711e097908 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 15:44:39 -0700 Subject: [PATCH 28/68] typo --- .github/workflows/lint-pr.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 7bd259c..77c48e8 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -31,8 +31,8 @@ jobs: { echo 'DIFF<<\n' git diff --name-only HEAD~ HEAD - echo '\n' - } >> "$GITUB_ENV" + echo \n + } >> "$GITHUB_ENV" # Lint all plugins/** files in diff (filtering of diff occurs in Python) - name: Lint From cc5f3bebc955437c37f2d4572de00e013659a653 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 15:45:50 -0700 Subject: [PATCH 29/68] Get rid of end delimeter --- .github/workflows/lint-pr.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 77c48e8..62f5567 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -31,7 +31,6 @@ jobs: { echo 'DIFF<<\n' git diff --name-only HEAD~ HEAD - echo \n } >> "$GITHUB_ENV" # Lint all plugins/** files in diff (filtering of diff occurs in Python) From b62a4520a40c06c3c2ba04c6e4de1076a2ea50dd Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 15:47:04 -0700 Subject: [PATCH 30/68] EOF --- .github/workflows/lint-pr.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 62f5567..52dd3a6 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -29,8 +29,9 @@ jobs: run: | git diff --name-only HEAD~ HEAD { - echo 'DIFF<<\n' + echo 'DIFF<> "$GITHUB_ENV" # Lint all plugins/** files in diff (filtering of diff occurs in Python) From 67d8af9f41477aef6f4522492dec1a9ec9c840de Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 21 Jan 2025 15:51:40 -0700 Subject: [PATCH 31/68] cleanup --- .github/workflows/lint-pr.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 52dd3a6..ed4f94e 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -27,7 +27,6 @@ jobs: # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#multiline-strings - name: Get Diff run: | - git diff --name-only HEAD~ HEAD { echo 'DIFF< Date: Tue, 21 Jan 2025 16:24:54 -0700 Subject: [PATCH 32/68] github token --- .github/scripts/lint.py | 38 ++++++++++++++++++++++++++++++++++- .github/workflows/lint-pr.yml | 2 +- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/.github/scripts/lint.py b/.github/scripts/lint.py index 72a407f..43483ff 100644 --- a/.github/scripts/lint.py +++ b/.github/scripts/lint.py @@ -10,6 +10,8 @@ KEY_SET = set(['owner', 'name', 'branch', 'docs']) +GITHUB_TOKEN = sys.argv[1] + def lint(yml): # Assert corrrect keys @@ -21,10 +23,44 @@ def lint(yml): # Put together the owner/name:branch # ...Do something with it + base_url = "https://api.github.com" + + def create_repo(access_token, repo_name, repo_descr=None): + url = f"{base_url}/user/repos" + + headers = { + "Authorization": f"token {access_token}", + } + + # create json data to send using the post request + data = { + "name": repo_name, + "description": repo_descr, + } + + response = requests.post(url, headers=headers, json=data) + + if response.status_code == 201: + repo_data = response.json() + return repo_data + else: + return None + + + access_token = "YOUR ACCESS TOKEN" + repo_name = "apify_testing" + repo_descr = "New repo created using the Python GitHub API." + + new_repo = create_repo(access_token, repo_name, repo_descr) + + if new_repo: + print(f"New public repo created successfully!") + else: + print("Failed to create a new repo.") if __name__ == "__main__": - files = sys.argv[1:] + files = sys.argv[2:] for file in files: head, tail = os.path.split(file) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index ed4f94e..f5d89b5 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -35,7 +35,7 @@ jobs: # Lint all plugins/** files in diff (filtering of diff occurs in Python) - name: Lint - run: python .github/scripts/lint.py $DIFF + run: python .github/scripts/lint.py ${{ secrets.GITHUB_TOKEN }} $DIFF # Lint all files in plugins From ad1154bf0ebf2e791b433b486fdaee8d65dfeccd Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 22 Jan 2025 12:07:40 -0700 Subject: [PATCH 33/68] Get environment-files --- .github/scripts/lint.py | 50 +++++++++++++---------------------------- 1 file changed, 15 insertions(+), 35 deletions(-) diff --git a/.github/scripts/lint.py b/.github/scripts/lint.py index 43483ff..1b56392 100644 --- a/.github/scripts/lint.py +++ b/.github/scripts/lint.py @@ -11,7 +11,7 @@ KEY_SET = set(['owner', 'name', 'branch', 'docs']) GITHUB_TOKEN = sys.argv[1] - +GITHUB_BASE_URL = 'https://api.github.com' def lint(yml): # Assert corrrect keys @@ -23,40 +23,20 @@ def lint(yml): # Put together the owner/name:branch # ...Do something with it - base_url = "https://api.github.com" - - def create_repo(access_token, repo_name, repo_descr=None): - url = f"{base_url}/user/repos" - - headers = { - "Authorization": f"token {access_token}", - } - - # create json data to send using the post request - data = { - "name": repo_name, - "description": repo_descr, - } - - response = requests.post(url, headers=headers, json=data) - - if response.status_code == 201: - repo_data = response.json() - return repo_data - else: - return None - - - access_token = "YOUR ACCESS TOKEN" - repo_name = "apify_testing" - repo_descr = "New repo created using the Python GitHub API." - - new_repo = create_repo(access_token, repo_name, repo_descr) - - if new_repo: - print(f"New public repo created successfully!") - else: - print("Failed to create a new repo.") + url = f'{GITHUB_BASE_URL}/repos/{yml['owner']}/{yml['name']}/contents/environment-files' + headers = { + 'Authorization': f'token: {GITHUB_TOKEN}', + 'X-GitHub-Api-Version': '2022-11-28' + } + query_params = { + 'owner': yml['owner'], + 'repo': yml['name'], + 'ref': yml['branch'], + 'path': '/environment-files/' + } + + response = requests.get(url, headers=headers, params=query_params) + print(response.json()) if __name__ == "__main__": From 1205469f6fb812f54903670184a1c3660aa5337f Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 12 Feb 2025 12:29:58 -0700 Subject: [PATCH 34/68] Merge --- .github/scripts/lint.py | 24 ++++++++++++++++++++++++ .github/workflows/lint-pr.yml | 3 --- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.github/scripts/lint.py b/.github/scripts/lint.py index 44871e6..053749a 100644 --- a/.github/scripts/lint.py +++ b/.github/scripts/lint.py @@ -1,3 +1,5 @@ +import os +import re import sys import yaml import requests @@ -6,6 +8,10 @@ KEY_SET = set(['owner', 'name', 'branch', 'docs']) +ENV_FILE_REGEX = '.*-qiime2-.*-20[0-9][0-9]\.([1-9]|1[0-2])\.yml' +env_urls = [] + + def lint(yml): # Assert corrrect keys assert set(yml.keys()) == KEY_SET @@ -16,6 +22,24 @@ def lint(yml): # Put together the owner/name:branch # ...Do something with it + url = f'{GITHUB_BASE_URL}/repos/{yml['owner']}/{yml['name']}/contents/environment-files' + headers = { + 'Authorization': f'token: {GITHUB_TOKEN}', + 'X-GitHub-Api-Version': '2022-11-28' + } + query_params = { + 'owner': yml['owner'], + 'repo': yml['name'], + 'ref': yml['branch'], + 'path': '/environment-files/' + } + + response = requests.get(url, headers=headers, params=query_params) + envs = response.json() + + for env in envs: + if re.search(ENV_FILE_REGEX, env.name) is not None: + env_urls.append(env['download_url']) if __name__ == "__main__": diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index a9464c7..5826c47 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -12,9 +12,6 @@ on: # Checkout jobs: lint: - permissions: - contents: read - runs-on: ubuntu-latest steps: From b5366df142e0cecb85cf5062242c4e87ee9e9616 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 12 Feb 2025 12:47:37 -0700 Subject: [PATCH 35/68] play with getting env files --- .github/scripts/lint.py | 9 ++++++--- .github/workflows/lint-pr.yml | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/scripts/lint.py b/.github/scripts/lint.py index 053749a..5eb8638 100644 --- a/.github/scripts/lint.py +++ b/.github/scripts/lint.py @@ -6,9 +6,9 @@ KEY_SET = set(['owner', 'name', 'branch', 'docs']) - - ENV_FILE_REGEX = '.*-qiime2-.*-20[0-9][0-9]\.([1-9]|1[0-2])\.yml' +GITHUB_BASE_URL = "https://api.github.com" + env_urls = [] @@ -41,9 +41,12 @@ def lint(yml): if re.search(ENV_FILE_REGEX, env.name) is not None: env_urls.append(env['download_url']) + print(env_urls) + if __name__ == "__main__": - files = sys.argv[1:] + GITHUB_TOKEN = sys.argv[1] + files = sys.argv[2:] for file in files: with open(file, 'r') as fh: diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 5826c47..66c58ac 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -27,7 +27,7 @@ jobs: # Lint all added files - name: Lint - run: python .github/scripts/lint.py $DIFF + run: python .github/scripts/lint.py ${{ secrets.GITHUB_TOKEN }} $DIFF # Lint all files in plugins From f0e4c2ced5627ec8141dd3c497b310d457932db5 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 10:39:02 -0700 Subject: [PATCH 36/68] Different debug --- .github/scripts/lint.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/scripts/lint.py b/.github/scripts/lint.py index 5eb8638..c71532a 100644 --- a/.github/scripts/lint.py +++ b/.github/scripts/lint.py @@ -38,11 +38,12 @@ def lint(yml): envs = response.json() for env in envs: - if re.search(ENV_FILE_REGEX, env.name) is not None: + if re.search(ENV_FILE_REGEX, env['name']) is not None: + print(env['name']) + print(env) + print() env_urls.append(env['download_url']) - print(env_urls) - if __name__ == "__main__": GITHUB_TOKEN = sys.argv[1] From d7ce8c8aed69b5fd3f2acf8617cd96597e89af85 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 10:39:16 -0700 Subject: [PATCH 37/68] Maybe the button will work --- .github/workflows/lint-pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 66c58ac..1f24608 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -8,6 +8,7 @@ on: # - 'main' # paths: # - '/plugins/**' + workflow_dispatch: # Checkout jobs: From 2f0943fbe66c79e5cc0189e9a4b1f21d6e223ca7 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 11:45:33 -0700 Subject: [PATCH 38/68] more info --- .github/scripts/lint.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/scripts/lint.py b/.github/scripts/lint.py index 8688df5..8f48781 100644 --- a/.github/scripts/lint.py +++ b/.github/scripts/lint.py @@ -50,6 +50,8 @@ def lint(yml): print() env_urls.append(env['download_url']) + print(env_urls) + if __name__ == "__main__": GITHUB_TOKEN = sys.argv[1] From 5980b1001c3ee5e0dd2b0e14d467806949ff6053 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 11:46:54 -0700 Subject: [PATCH 39/68] Move that --- .github/scripts/lint.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/scripts/lint.py b/.github/scripts/lint.py index 8f48781..f35efb5 100644 --- a/.github/scripts/lint.py +++ b/.github/scripts/lint.py @@ -50,9 +50,6 @@ def lint(yml): print() env_urls.append(env['download_url']) - print(env_urls) - - if __name__ == "__main__": GITHUB_TOKEN = sys.argv[1] files = sys.argv[2:] @@ -69,3 +66,5 @@ def lint(yml): with open(file, 'r') as fh: yml = yaml.safe_load(fh) lint(yml) + + print(env_urls) From 6be59c3a2948b5eca377df2832416ff52516d020 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 11:54:14 -0700 Subject: [PATCH 40/68] Save ENV_FILES --- .github/scripts/lint.py | 6 ++---- .github/workflows/lint-pr.yml | 3 +++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/scripts/lint.py b/.github/scripts/lint.py index f35efb5..5e4c0aa 100644 --- a/.github/scripts/lint.py +++ b/.github/scripts/lint.py @@ -45,9 +45,6 @@ def lint(yml): for env in envs: if re.search(ENV_FILE_REGEX, env['name']) is not None: - print(env['name']) - print(env) - print() env_urls.append(env['download_url']) if __name__ == "__main__": @@ -67,4 +64,5 @@ def lint(yml): yml = yaml.safe_load(fh) lint(yml) - print(env_urls) + with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: + fh.write(f'ENV_FILES={env_urls}\n') diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 02c9a3b..9f2ea5d 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -33,8 +33,11 @@ jobs: # Lint all plugins/** files in diff (filtering of diff occurs in Python) - name: Lint + id: lint run: python .github/scripts/lint.py ${{ secrets.GITHUB_TOKEN }} $DIFF + - name: Lint Env Files + run: echo "${{ steps.lint.outputs.ENV_FILES }}" # Lint all files in plugins # If anything outside of plugins is changed then fail? From c5c1619b3b9b36ed51912c0d245a458991a78f68 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 12:04:35 -0700 Subject: [PATCH 41/68] Try to set up matrix over env files --- .github/workflows/lint-pr.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 9f2ea5d..0a4b345 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -36,8 +36,19 @@ jobs: id: lint run: python .github/scripts/lint.py ${{ secrets.GITHUB_TOKEN }} $DIFF - - name: Lint Env Files - run: echo "${{ steps.lint.outputs.ENV_FILES }}" + lint-envs: + runs-on: ubuntu-latest + needs: lint + strategy: + matrix: + env-file: ${{ needs.lint.outputs.ENV_FILES }} + + steps: + - name: Echo + env: + file: ${{ matrix.env-file }} + + run: echo $color # Lint all files in plugins # If anything outside of plugins is changed then fail? From acb206ad58ade691b85f9242d92a7496947187d1 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 12:07:27 -0700 Subject: [PATCH 42/68] Parse output --- .github/workflows/lint-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 0a4b345..72db2c0 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -41,7 +41,7 @@ jobs: needs: lint strategy: matrix: - env-file: ${{ needs.lint.outputs.ENV_FILES }} + env-file: ${{ fromJSON(needs.lint.outputs.ENV_FILES) }} steps: - name: Echo From 8919aa5c38f4b7dff24fa057d39738c6df8c0bfd Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 12:08:28 -0700 Subject: [PATCH 43/68] Get step? --- .github/workflows/lint-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 72db2c0..5706b0d 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -41,7 +41,7 @@ jobs: needs: lint strategy: matrix: - env-file: ${{ fromJSON(needs.lint.outputs.ENV_FILES) }} + env-file: ${{ fromJSON(needs.lint.lint.outputs.ENV_FILES) }} steps: - name: Echo From c6c1d8c3f64e09d39776aa688898a2bbc9e35765 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 12:10:03 -0700 Subject: [PATCH 44/68] Maybe the step name at end --- .github/workflows/lint-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 5706b0d..6c32d85 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -41,7 +41,7 @@ jobs: needs: lint strategy: matrix: - env-file: ${{ fromJSON(needs.lint.lint.outputs.ENV_FILES) }} + env-file: ${{ needs.lint.outputs.lint }} steps: - name: Echo From 0668149a1703a93b903a5350984a51ffbb77e60d Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 12:11:12 -0700 Subject: [PATCH 45/68] WHY --- .github/workflows/lint-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 6c32d85..e4a3e6f 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -41,7 +41,7 @@ jobs: needs: lint strategy: matrix: - env-file: ${{ needs.lint.outputs.lint }} + env-file: ${{ needs.lint.outputs.lint.ENV_FILES }} steps: - name: Echo From 94ea4fe351e6512d2fc6b898f1dfcf38d925621e Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 12:19:43 -0700 Subject: [PATCH 46/68] Clean up names --- .github/workflows/lint-pr.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index e4a3e6f..bd19feb 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -10,7 +10,7 @@ on: workflow_dispatch: jobs: - lint: + lint-yml: runs-on: ubuntu-latest steps: @@ -32,16 +32,16 @@ jobs: } >> "$GITHUB_ENV" # Lint all plugins/** files in diff (filtering of diff occurs in Python) - - name: Lint + - name: Lint Yml and Get Env Files id: lint run: python .github/scripts/lint.py ${{ secrets.GITHUB_TOKEN }} $DIFF lint-envs: runs-on: ubuntu-latest - needs: lint + needs: lint-yml strategy: matrix: - env-file: ${{ needs.lint.outputs.lint.ENV_FILES }} + env-file: ${{ needs.lint-yml.outputs.ENV_FILES }} steps: - name: Echo From ef73ada47053023d83fd92f1c3440060a2d1e7e1 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 12:22:06 -0700 Subject: [PATCH 47/68] Test --- .github/workflows/lint-pr.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index bd19feb..7c1b13d 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -39,16 +39,20 @@ jobs: lint-envs: runs-on: ubuntu-latest needs: lint-yml - strategy: - matrix: - env-file: ${{ needs.lint-yml.outputs.ENV_FILES }} steps: - - name: Echo - env: - file: ${{ matrix.env-file }} + - name: Test + run: echo ${{needs.lint-yml.outputs.ENV_FILES}} + # strategy: + # matrix: + # env-file: ${{ needs.lint-yml.outputs.ENV_FILES }} - run: echo $color + # steps: + # - name: Echo + # env: + # file: ${{ matrix.env-file }} + + # run: echo $color # Lint all files in plugins # If anything outside of plugins is changed then fail? From c56904f11e2e823a75982da2af1263349a16c40d Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 12:23:07 -0700 Subject: [PATCH 48/68] Just outputs --- .github/workflows/lint-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 7c1b13d..528b6ac 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -42,7 +42,7 @@ jobs: steps: - name: Test - run: echo ${{needs.lint-yml.outputs.ENV_FILES}} + run: echo ${{needs.lint-yml.outputs}} # strategy: # matrix: # env-file: ${{ needs.lint-yml.outputs.ENV_FILES }} From 58c74a146c5bef22ddd0a81bcb61d8bb877d2ecf Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 12:24:58 -0700 Subject: [PATCH 49/68] toJSON --- .github/workflows/lint-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 528b6ac..a14c4cb 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -42,7 +42,7 @@ jobs: steps: - name: Test - run: echo ${{needs.lint-yml.outputs}} + run: echo ${{ toJSON(needs.lint-yml.outputs) }} # strategy: # matrix: # env-file: ${{ needs.lint-yml.outputs.ENV_FILES }} From c0f20cfc72c390819a959e1f40dddf680c4e98ea Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 12:26:18 -0700 Subject: [PATCH 50/68] AAAAAAAAAHHHHHHHHHHHHHHHH --- .github/workflows/lint-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index a14c4cb..edf137a 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -38,7 +38,7 @@ jobs: lint-envs: runs-on: ubuntu-latest - needs: lint-yml + needs: [lint-yml] steps: - name: Test From 817d5a9c591c289b7ea0361aecff776cb9102148 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 12:28:23 -0700 Subject: [PATCH 51/68] Maybe like this --- .github/workflows/lint-pr.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index edf137a..501af42 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -12,6 +12,8 @@ on: jobs: lint-yml: runs-on: ubuntu-latest + outputs: + ENV_FILES: ${{ steps.lint.outputs.ENV_FILES }} steps: # Checkout new commit and prior HEAD @@ -38,11 +40,11 @@ jobs: lint-envs: runs-on: ubuntu-latest - needs: [lint-yml] + needs: lint-yml steps: - name: Test - run: echo ${{ toJSON(needs.lint-yml.outputs) }} + run: echo ${{ needs.lint-yml.outputs.ENV_FILES }} # strategy: # matrix: # env-file: ${{ needs.lint-yml.outputs.ENV_FILES }} From ddf022c23e23f0393ac1b3de49bdc0c4e7f4af38 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 12:56:51 -0700 Subject: [PATCH 52/68] Back to matrix --- .github/workflows/lint-pr.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 501af42..e0f157d 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -41,10 +41,15 @@ jobs: lint-envs: runs-on: ubuntu-latest needs: lint-yml + strategy: + matrix: + env-file: ${{ fromJSON(needs.lint-yml.outputs.ENV_FILES) }} steps: - name: Test - run: echo ${{ needs.lint-yml.outputs.ENV_FILES }} + env: + env-file: ${{ matrix.env-file }} + run: echo ${{ env-file }} # strategy: # matrix: # env-file: ${{ needs.lint-yml.outputs.ENV_FILES }} From 110980e2117a3bc7f875408977a20c457e4b021c Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 13:01:05 -0700 Subject: [PATCH 53/68] Different var name/access --- .github/workflows/lint-pr.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index e0f157d..4da2422 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -43,13 +43,13 @@ jobs: needs: lint-yml strategy: matrix: - env-file: ${{ fromJSON(needs.lint-yml.outputs.ENV_FILES) }} + url: ${{ fromJSON(needs.lint-yml.outputs.ENV_FILES) }} steps: - name: Test env: - env-file: ${{ matrix.env-file }} - run: echo ${{ env-file }} + url: ${{ matrix.url }} + run: echo $url # strategy: # matrix: # env-file: ${{ needs.lint-yml.outputs.ENV_FILES }} From 47e64382ad9541e8bd9927fe92e6fb309c209a12 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 13:07:19 -0700 Subject: [PATCH 54/68] Try making env --- .github/workflows/lint-pr.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 4da2422..0ff710c 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -46,10 +46,22 @@ jobs: url: ${{ fromJSON(needs.lint-yml.outputs.ENV_FILES) }} steps: + - name: 'setup miniconda' + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: '' + architecture: 'x64' + auto-activate-base: true + miniconda-version: 'latest' + - name: Test env: url: ${{ matrix.url }} - run: echo $url + run: | + echo $url + conda create -y -n test $url + conda activate $url + qiime info # strategy: # matrix: # env-file: ${{ needs.lint-yml.outputs.ENV_FILES }} From 9a01a9bb8b5e8fff620f3a4adffc19486c082de8 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 13:09:27 -0700 Subject: [PATCH 55/68] Need -f --- .github/workflows/lint-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 0ff710c..0e100cd 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -59,7 +59,7 @@ jobs: url: ${{ matrix.url }} run: | echo $url - conda create -y -n test $url + conda create -y -n test -f $url conda activate $url qiime info # strategy: From ff172b0c2283b9401dd4248478c4e8d4b67cefb7 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 13:09:42 -0700 Subject: [PATCH 56/68] Job name caps --- .github/workflows/lint-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 0e100cd..870d1e5 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -46,7 +46,7 @@ jobs: url: ${{ fromJSON(needs.lint-yml.outputs.ENV_FILES) }} steps: - - name: 'setup miniconda' + - name: Setup Miniconda uses: conda-incubator/setup-miniconda@v3 with: activate-environment: '' From 2073af436d44c324339b52382bdea94f3ddb64c4 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 13:11:20 -0700 Subject: [PATCH 57/68] env create --- .github/workflows/lint-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 870d1e5..d39d586 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -59,7 +59,7 @@ jobs: url: ${{ matrix.url }} run: | echo $url - conda create -y -n test -f $url + conda env create -y -n test -f $url conda activate $url qiime info # strategy: From 80d4a82f59749b8aeca73cb88a0201c3ba0e924d Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 13:18:04 -0700 Subject: [PATCH 58/68] Conda init --- .github/workflows/lint-pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index d39d586..6422f31 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -60,6 +60,7 @@ jobs: run: | echo $url conda env create -y -n test -f $url + conda init conda activate $url qiime info # strategy: From 37eb9cfe440389d2ff09fa6df195451e7dcfdd9c Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 13:23:52 -0700 Subject: [PATCH 59/68] Steal from ALP --- .github/workflows/lint-pr.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 6422f31..35fc1a2 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -54,14 +54,21 @@ jobs: auto-activate-base: true miniconda-version: 'latest' - - name: Test + - name: 'Set up test environment' env: url: ${{ matrix.url }} run: | - echo $url - conda env create -y -n test -f $url - conda init - conda activate $url + conda env create -y -p test -f $url + mkdir -p test/etc + cat < 'test/etc/activate.sh' + . "$CONDA/etc/profile.d/conda.sh" + conda activate 'test' + EOF + chmod +x 'test/etc/activate.sh' + + - name: Test + run: | + conda activate test qiime info # strategy: # matrix: From 31f327113d9b22d42551743f745c96826a8fb25a Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 13:24:10 -0700 Subject: [PATCH 60/68] step names --- .github/workflows/lint-pr.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 35fc1a2..5637344 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -46,7 +46,7 @@ jobs: url: ${{ fromJSON(needs.lint-yml.outputs.ENV_FILES) }} steps: - - name: Setup Miniconda + - name: Set up Miniconda uses: conda-incubator/setup-miniconda@v3 with: activate-environment: '' @@ -54,7 +54,7 @@ jobs: auto-activate-base: true miniconda-version: 'latest' - - name: 'Set up test environment' + - name: Set up Test Environment env: url: ${{ matrix.url }} run: | From 09af28e19d029a65261d92295dc2f3c276a31ab3 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 13:28:34 -0700 Subject: [PATCH 61/68] Source file --- .github/workflows/lint-pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 5637344..53feb74 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -68,6 +68,7 @@ jobs: - name: Test run: | + source test/etc/activate.sh conda activate test qiime info # strategy: From 8b1f35749f26cbfaa86c423a51adaf5e4eb4eea5 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 13:30:02 -0700 Subject: [PATCH 62/68] ./test as prefix --- .github/workflows/lint-pr.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 53feb74..52c9c0f 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -58,9 +58,9 @@ jobs: env: url: ${{ matrix.url }} run: | - conda env create -y -p test -f $url - mkdir -p test/etc - cat < 'test/etc/activate.sh' + conda env create -y -p ./test -f $url + mkdir -p ./test/etc + cat < './test/etc/activate.sh' . "$CONDA/etc/profile.d/conda.sh" conda activate 'test' EOF @@ -68,8 +68,8 @@ jobs: - name: Test run: | - source test/etc/activate.sh - conda activate test + source ./test/etc/activate.sh + conda activate ./test qiime info # strategy: # matrix: From 99eb7cad9fe899f8c4edd040f7b46a7f7f5a5f9b Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 19 Feb 2025 13:32:48 -0700 Subject: [PATCH 63/68] Use env var --- .github/workflows/lint-pr.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 52c9c0f..7d89b2e 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -44,6 +44,8 @@ jobs: strategy: matrix: url: ${{ fromJSON(needs.lint-yml.outputs.ENV_FILES) }} + env: + prefix: ./test steps: - name: Set up Miniconda @@ -58,18 +60,18 @@ jobs: env: url: ${{ matrix.url }} run: | - conda env create -y -p ./test -f $url - mkdir -p ./test/etc - cat < './test/etc/activate.sh' + conda env create -y -p ${{ env.prefix }} -f $url + mkdir -p ${{ env.prefix }}/etc + cat < '${{ env.prefix }}/etc/activate.sh' . "$CONDA/etc/profile.d/conda.sh" - conda activate 'test' + conda activate '${{ env.prefix }}' EOF chmod +x 'test/etc/activate.sh' - name: Test run: | - source ./test/etc/activate.sh - conda activate ./test + source ${{ env.prefix }}/etc/activate.sh + conda activate ${{ env.prefix }} qiime info # strategy: # matrix: From cf6b439464e74d9a4f4924478495c2beefcc0dd9 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Thu, 20 Feb 2025 13:50:34 -0700 Subject: [PATCH 64/68] matrix the os as well --- .github/workflows/lint-pr.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 7d89b2e..74f6ea1 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -39,11 +39,12 @@ jobs: run: python .github/scripts/lint.py ${{ secrets.GITHUB_TOKEN }} $DIFF lint-envs: - runs-on: ubuntu-latest needs: lint-yml strategy: matrix: + os: [ubuntu-latest, macos-13] url: ${{ fromJSON(needs.lint-yml.outputs.ENV_FILES) }} + runs-on: ${{ matrix.os }} env: prefix: ./test From ce87b0d7f6e39dae9895568b725f1ba7f3dc739e Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Thu, 20 Feb 2025 13:53:37 -0700 Subject: [PATCH 65/68] Setup run conditions --- .github/workflows/lint-pr.yml | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 74f6ea1..6bbc599 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -3,11 +3,10 @@ name: lint-pr # Run on PR changing files in plugins dir against main on: pull_request: - # branches: - # - 'main' - # paths: - # - '/plugins/**' - workflow_dispatch: + branches: + - 'main' + paths: + - '/plugins/**' jobs: lint-yml: @@ -74,16 +73,6 @@ jobs: source ${{ env.prefix }}/etc/activate.sh conda activate ${{ env.prefix }} qiime info - # strategy: - # matrix: - # env-file: ${{ needs.lint-yml.outputs.ENV_FILES }} - - # steps: - # - name: Echo - # env: - # file: ${{ matrix.env-file }} - - # run: echo $color # Lint all files in plugins # If anything outside of plugins is changed then fail? From 4ad0d1661205bcea012a1bb3a911c5b54b8a8757 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Thu, 20 Feb 2025 13:59:02 -0700 Subject: [PATCH 66/68] Comments in yml --- .github/workflows/lint-pr.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 6bbc599..437e669 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -33,11 +33,14 @@ jobs: } >> "$GITHUB_ENV" # Lint all plugins/** files in diff (filtering of diff occurs in Python) + # Also gets all env files in added plugins for env testing - name: Lint Yml and Get Env Files id: lint run: python .github/scripts/lint.py ${{ secrets.GITHUB_TOKEN }} $DIFF - lint-envs: + # Install all envs found and at least make sure `qiime info` runs + # TODO: Make this attempt to run pytest? + test-envs: needs: lint-yml strategy: matrix: @@ -56,6 +59,7 @@ jobs: auto-activate-base: true miniconda-version: 'latest' + # Hacky stff to make it so I can activate the conda env for testing - name: Set up Test Environment env: url: ${{ matrix.url }} @@ -74,11 +78,7 @@ jobs: conda activate ${{ env.prefix }} qiime info -# Lint all files in plugins - # If anything outside of plugins is changed then fail? - # Grab the target repo and validate it? # Description? # README? # Env files? - From 7b5bb9566f507ab9d16ef1d3c48451500059f0bf Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Thu, 20 Feb 2025 13:59:46 -0700 Subject: [PATCH 67/68] Let people have their own stuff in there --- .github/scripts/lint.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/scripts/lint.py b/.github/scripts/lint.py index 5e4c0aa..27191bc 100644 --- a/.github/scripts/lint.py +++ b/.github/scripts/lint.py @@ -6,8 +6,6 @@ DIR = 'plugins' -FILE_START = 'q2-' -FILE_EXT = '.yml' KEY_SET = set(['owner', 'name', 'branch', 'docs']) ENV_FILE_REGEX = '.*-qiime2-.*-20[0-9][0-9]\.([1-9]|1[0-2])\.yml' @@ -57,9 +55,6 @@ def lint(yml): # We only care about files added to the plugins dir if head == DIR: - if file_name[0:3] != FILE_START or file_ext != FILE_EXT: - raise ValueError('File name must conform to q2-*.yml') - with open(file, 'r') as fh: yml = yaml.safe_load(fh) lint(yml) From c0cab9f0e06fb51d085ffb9838dc32702a952f2e Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Thu, 20 Feb 2025 14:01:38 -0700 Subject: [PATCH 68/68] Formatting and comments in lint.py --- .github/scripts/lint.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/scripts/lint.py b/.github/scripts/lint.py index 27191bc..afebe3c 100644 --- a/.github/scripts/lint.py +++ b/.github/scripts/lint.py @@ -16,6 +16,7 @@ GITHUB_TOKEN = sys.argv[1] GITHUB_BASE_URL = 'https://api.github.com' + def lint(yml): # Assert corrrect keys assert set(yml.keys()) == KEY_SET @@ -25,7 +26,6 @@ def lint(yml): assert response.status_code == 200 # Put together the owner/name:branch - # ...Do something with it url = f'{GITHUB_BASE_URL}/repos/{yml['owner']}/{yml['name']}/contents/environment-files' headers = { 'Authorization': f'token: {GITHUB_TOKEN}', @@ -38,13 +38,17 @@ def lint(yml): 'path': '/environment-files/' } + # Get all files in the /environment-files/ folder response = requests.get(url, headers=headers, params=query_params) envs = response.json() + # If the file matches the regex to be a QIIME 2 environment-file then keep + # track of its download URL for env in envs: if re.search(ENV_FILE_REGEX, env['name']) is not None: env_urls.append(env['download_url']) + if __name__ == "__main__": GITHUB_TOKEN = sys.argv[1] files = sys.argv[2:]