Skip to content

Commit 4f6dadc

Browse files
claytonparnellclaytonparnell
authored
Merge branch 'main' into release-2.6.1
2 parents 277453e + 9832085 commit 4f6dadc

File tree

293 files changed

+15472
-419
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

293 files changed

+15472
-419
lines changed

‎.github/pull_request_template.md

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
## Description
2+
[Provide a brief description of the changes]
3+
4+
## Type of Change
5+
- [ ] Image update - Bug fix
6+
- [ ] Image update - New feature
7+
- [ ] Image update - Breaking change
8+
- [ ] SMD image build tool update
9+
- [ ] Documentation update
10+
11+
## Release Information
12+
Does this change need to be included in patch version releases? By default, any pull requests will only be added to the next SMD image minor version release once they are merged in template folder. Only critical bug fix or security update should be applied to new patch versions of existed image minor versions.
13+
- [ ] Yes (Critical bug fix or security update)
14+
- [ ] No (New feature or non-critical change)
15+
- [ ] N/A (Not an image update)
16+
17+
If yes, please explain why:
18+
[Explain the criticality of this change and why it should be included in patch releases]
19+
20+
## How Has This Been Tested?
21+
[Describe the tests you ran]
22+
23+
## Checklist:
24+
- [ ] My code follows the style guidelines of this project
25+
- [ ] I have commented my code, particularly in hard-to-understand areas
26+
- [ ] I have made corresponding changes to the documentation
27+
- [ ] I have added tests that prove my fix is effective or that my feature works
28+
29+
## Test Screenshots (if applicable):
30+
31+
## Related Issues
32+
[Link any related issues here]
33+
34+
## Additional Notes
35+
[Any additional information that might be helpful for reviewers]

‎.github/workflows/add-nightly-builds.yml

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
# Workflow to start nightly build(s) upon new image version release.
2+
# Triggered by release (https://github.com/aws/sagemaker-distribution/releases) publication, for `*-cpu` only so we run once per image version.
3+
# Can also be triggered manually.
4+
# When a minor version (X.y.0) is released, we'll start the next minor (X.(y+1).0) and patch (X.y.1).
5+
# When a patch version (X.y.z) is released, we'll start the next patch (X.y.(z+1)).
6+
name: Add New Nightly Build(s)
7+
on:
8+
release:
9+
types: [published]
10+
workflow_dispatch:
11+
inputs:
12+
version:
13+
description: 'Released version to add next versions for (x.y.z)'
14+
required: true
15+
type: string
16+
concurrency:
17+
group: schedule-update
18+
cancel-in-progress: false
19+
defaults:
20+
run:
21+
shell: bash -l {0}
22+
jobs:
23+
update-nightly-schedule:
24+
name: Update NIGHTLY_BUILD_SCHEDULE with new version(s).
25+
runs-on: ubuntu-latest
26+
# Only run for CPU releases or manual triggers
27+
if: |
28+
github.event_name == 'workflow_dispatch' ||
29+
(github.event_name == 'release' && endsWith(github.event.release.tag_name, '-cpu'))
30+
steps:
31+
- uses: actions/checkout@v4
32+
- uses: mamba-org/setup-micromamba@v2
33+
with:
34+
environment-file: ./environment.lock
35+
environment-name: sagemaker-distribution
36+
init-shell: bash
37+
- name: Set version
38+
id: version
39+
run: |
40+
if [ "${{ github.event_name }}" == "release" ]; then
41+
# Extract x.y.z from x.y.z-cpu
42+
VERSION=$(echo "${{ github.event.release.tag_name }}" | sed 's/-cpu$//')
43+
echo "version=$VERSION" >> $GITHUB_OUTPUT
44+
else
45+
# Use manually provided version
46+
echo "version=${{ inputs.version }}" >> $GITHUB_OUTPUT
47+
fi
48+
- name: Update nightly build schedule
49+
id: update-schedule
50+
env:
51+
GH_TOKEN: ${{ secrets.GH_TOKEN }}
52+
run: |
53+
micromamba activate sagemaker-distribution
54+
python ./.github/workflows/utils/nightly_build_helper.py \
55+
add-next-versions ${{ steps.version.outputs.version }}

‎.github/workflows/build-image.yml

Lines changed: 34 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,11 @@ jobs:
4040
target_version: ${{ steps.calc_target.outputs.target_version }}
4141
steps:
4242
- uses: actions/checkout@v4
43-
- uses: mamba-org/setup-micromamba@v1
4443
with:
45-
environment-file: ./environment.yml
44+
fetch-depth: 0
45+
- uses: mamba-org/setup-micromamba@v2
46+
with:
47+
environment-file: ./environment.lock
4648
environment-name: sagemaker-distribution
4749
init-shell: bash
4850
- name: Free up disk space
@@ -54,30 +56,51 @@ jobs:
5456
run: |
5557
TARGET_VERSION=$(python -c 'import semver; print(semver.bump_${{ inputs.release-type }}("${{ inputs.base-version }}"))')
5658
echo "target_version=$TARGET_VERSION" >> $GITHUB_OUTPUT
57-
- name: Create new branch
59+
- name: Create or merge release branch
5860
# Note - CodeBuild depends on this branch name. Don't change without corresponding backend change.
59-
run: git checkout -b release-${{ steps.calc_target.outputs.target_version }}
61+
id: check_branch
62+
run: |
63+
git config --local user.email "github-actions[bot]@users.noreply.github.com"
64+
git config --local user.name "github-actions[bot]"
65+
if git ls-remote --exit-code --heads origin release-${{ steps.calc_target.outputs.target_version }}; then
66+
echo "Branch exists, merging latest changes in"
67+
git checkout release-${{ steps.calc_target.outputs.target_version }}
68+
git merge main
69+
git push
70+
else
71+
echo "Branch doesn't exist, creating now..."
72+
git checkout -b release-${{ steps.calc_target.outputs.target_version }}
73+
fi
6074
- name: Generate artifacts
6175
run: python ./src/main.py create-${{ inputs.release-type }}-version-artifacts --base-patch-version ${{ inputs.base-version }} --force
6276
- name: Commit .in artifacts to branch
6377
env:
6478
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
79+
# Generate the artifacts every time, only push if there are changes
6580
run: |
6681
git config --local user.email "github-actions[bot]@users.noreply.github.com"
6782
git config --local user.name "github-actions[bot]"
6883
git add ./build_artifacts
69-
git commit -m 'chore: Generate build artifacts for ${{ steps.calc_target.outputs.target_version }} release'
84+
git diff-index --quiet HEAD || git commit -m 'chore: Generate build artifacts for ${{ steps.calc_target.outputs.target_version }} release'
7085
git push --set-upstream origin release-${{ steps.calc_target.outputs.target_version }}
71-
- name: Open pull request
86+
- name: Create or fetch release PR
7287
id: get_pr_id
7388
env:
7489
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
75-
# Note - CodeBuild depends on this PR title. Don't change without corresponding backend change.
90+
# Create PR if it doesn't exist; fetch existing PR number if it does
7691
run: |
77-
URL=$(gh pr create -H release-${{ steps.calc_target.outputs.target_version }} \
78-
--title 'release: v${{ steps.calc_target.outputs.target_version }}' -F ./.github/workflows/PR_TEMPLATE.md)
79-
PR=$(echo $URL | sed 's:.*/::')
80-
echo "pr_id=$PR" >> $GITHUB_OUTPUT
92+
if gh pr list --head release-${{ steps.calc_target.outputs.target_version }} --state open --json number | grep -q '"number":'; then
93+
echo "PR exists already, just fetching ID"
94+
PR_ID=$(gh pr view release-${{ steps.calc_target.outputs.target_version }} --json number | jq -r .number)
95+
echo "pr_id=$PR_ID" >> $GITHUB_OUTPUT
96+
else
97+
echo "PR doesn't exist, creating now..."
98+
git push --set-upstream origin release-${{ steps.calc_target.outputs.target_version }}
99+
URL=$(gh pr create -H release-${{ steps.calc_target.outputs.target_version }} -B main \
100+
--title 'release: v${{ steps.calc_target.outputs.target_version }}' -F ./.github/workflows/PR_TEMPLATE.md)
101+
PR_ID=$(echo $URL | sed 's:.*/::')
102+
echo "pr_id=$PR_ID" >> $GITHUB_OUTPUT
103+
fi
81104
call-codebuild-project:
82105
runs-on: ubuntu-latest
83106
needs: open-pr

‎.github/workflows/check-image-size.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,9 @@ jobs:
3939
contents: write
4040
steps:
4141
- uses: actions/checkout@v4
42-
- uses: mamba-org/setup-micromamba@v1
42+
- uses: mamba-org/setup-micromamba@v2
4343
with:
44-
environment-file: ./environment.yml
44+
environment-file: ./environment.lock
4545
environment-name: sagemaker-distribution
4646
init-shell: bash
4747
- name: Free up disk space

‎.github/workflows/check_code_quality.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,9 @@ jobs:
2929
contents: write
3030
steps:
3131
- uses: actions/checkout@v4
32-
- uses: mamba-org/setup-micromamba@v1
32+
- uses: mamba-org/setup-micromamba@v2
3333
with:
34-
environment-file: ./environment.yml
34+
environment-file: ./environment.lock
3535
environment-name: sagemaker-distribution
3636
init-shell: bash
3737
- name: Free up disk space

‎.github/workflows/monthly-minor-release.yml

Lines changed: 0 additions & 57 deletions
This file was deleted.

‎.github/workflows/nightly-image-build.yml

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: Kickoff Nightly builds
2+
run-name: Kickoff nightly builds
3+
on:
4+
# Run manually
5+
workflow_dispatch:
6+
# Run on Mon-Fri at 4PM PST / 5PM PDT
7+
schedule:
8+
- cron: '0 0 * * 1-5'
9+
jobs:
10+
generate-version-matrix:
11+
name: Generate version matrix
12+
runs-on: ubuntu-latest
13+
if: github.repository == 'aws/sagemaker-distribution'
14+
outputs:
15+
matrix: ${{ steps.set-matrix.outputs.matrix }}
16+
patch-versions: ${{ steps.set-matrix.outputs.patch-versions }}
17+
minor-versions: ${{ steps.set-matrix.outputs.minor-versions }}
18+
steps:
19+
- uses: actions/checkout@v4
20+
- name: Get versions from schedule
21+
id: set-matrix
22+
env:
23+
GH_TOKEN: ${{ secrets.GH_TOKEN }}
24+
run: |
25+
SCHEDULE='${{ vars.NIGHTLY_BUILD_SCHEDULE }}'
26+
echo "Triggering builds for $(echo $SCHEDULE | jq -c '.active_nightly_builds')"
27+
echo "matrix=$(echo $SCHEDULE | jq -c)" >> $GITHUB_OUTPUT
28+
echo "patch-versions=$(echo $SCHEDULE | jq -c '.patch_base_versions')" >> $GITHUB_OUTPUT
29+
echo "minor-versions=$(echo $SCHEDULE | jq -c '.minor_base_versions')" >> $GITHUB_OUTPUT
30+
start-minor-build:
31+
name: Start nightly minor build
32+
needs: generate-version-matrix
33+
permissions:
34+
pull-requests: write
35+
contents: write
36+
id-token: write
37+
strategy:
38+
matrix:
39+
version: ${{ fromJson(needs.generate-version-matrix.outputs.minor-versions) }}
40+
fail-fast: false
41+
uses: aws/sagemaker-distribution/.github/workflows/build-image.yml@main
42+
secrets: inherit
43+
with:
44+
release-type: "minor"
45+
base-version: ${{ matrix.version }}
46+
start-patch-build:
47+
name: Start nightly patch build
48+
needs: generate-version-matrix
49+
permissions:
50+
pull-requests: write
51+
contents: write
52+
id-token: write
53+
strategy:
54+
matrix:
55+
version: ${{ fromJson(needs.generate-version-matrix.outputs.patch-versions) }}
56+
fail-fast: false
57+
uses: aws/sagemaker-distribution/.github/workflows/build-image.yml@main
58+
secrets: inherit
59+
with:
60+
release-type: "patch"
61+
base-version: ${{ matrix.version }}

‎.github/workflows/remove-nightly-build.yml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Workflow manually triggered to stop nightly builds for a given image version.
2+
name: Stop Nightly Build
3+
run-name: Stop Nightly Build (${{ inputs.version }})
4+
on:
5+
workflow_dispatch:
6+
inputs:
7+
version:
8+
description: 'Version to stop building (x.y.z, e.g. 3.0.0)'
9+
required: true
10+
type: string
11+
concurrency:
12+
group: schedule-update
13+
cancel-in-progress: false
14+
defaults:
15+
run:
16+
shell: bash -l {0}
17+
jobs:
18+
update-nightly-schedule:
19+
name: Update nightly build schedule
20+
runs-on: ubuntu-latest
21+
steps:
22+
- uses: actions/checkout@v4
23+
- uses: mamba-org/setup-micromamba@v2
24+
with:
25+
environment-file: ./environment.lock
26+
environment-name: sagemaker-distribution
27+
init-shell: bash
28+
- name: Update nightly_build_schedule.json
29+
id: update-schedule
30+
env:
31+
GH_TOKEN: ${{ secrets.GH_TOKEN }}
32+
run: |
33+
micromamba activate sagemaker-distribution
34+
python ./.github/workflows/utils/nightly_build_helper.py remove-version ${{ inputs.version }}

‎.github/workflows/update-environment-lock.yml

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
name: Update environment.lock
2+
# Update environment.lock weekly, so we can quickly install environment without having
3+
# to solve environment.yml every time.
4+
on:
5+
# Every Monday
6+
schedule:
7+
- cron: '0 0 * * MON'
8+
# Manually call
9+
workflow_dispatch:
10+
defaults:
11+
run:
12+
shell: bash -l {0}
13+
jobs:
14+
update-environment:
15+
name: Install environment.yml, generate environment.lock
16+
runs-on: ubuntu-latest
17+
permissions:
18+
contents: write
19+
steps:
20+
- uses: actions/checkout@v4
21+
- uses: mamba-org/setup-micromamba@v2
22+
with:
23+
environment-file: ./environment.yml
24+
environment-name: sagemaker-distribution
25+
init-shell: bash
26+
- name: Create environment-lock.yml
27+
run: micromamba env export --explicit > environment.lock
28+
- name: Run sagemaker-distribution unit tests to check for regressions
29+
run: pytest -m unit
30+
- name: Commit changes
31+
run: |
32+
git config --local user.email "github-actions[bot]@users.noreply.github.com"
33+
git config --local user.name "github-actions[bot]"
34+
git add environment.lock
35+
git commit -m 'Update environment.lock'
36+
git push

0 commit comments

Comments
 (0)