Simulator Tests #1269
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Simulator Tests | |
| concurrency: | |
| group: ${{ github.workflow }} | |
| on: | |
| schedule: | |
| - cron: '0 9 * * *' # equivalent to 1am PST | |
| - cron: '0 21 * * *' # equivalent to 1pm PST | |
| workflow_dispatch: | |
| inputs: | |
| sui_ref: | |
| description: "Branch / commit to test" | |
| type: string | |
| required: true | |
| default: main | |
| test_num: | |
| description: "MSIM_TEST_NUM (test iterations)" | |
| type: string | |
| required: false | |
| default: "30" | |
| protocol_config_override: | |
| description: "Protocol config override" | |
| type: string | |
| required: false | |
| default: "" | |
| env: | |
| SUI_REF: "${{ github.event.inputs.sui_ref || 'main' }}" | |
| TEST_NUM: "${{ github.event.inputs.test_num || '30' }}" | |
| jobs: | |
| simtest: | |
| # Allow running the job for up to 6 hours, maximum for GitHub Actions. | |
| timeout-minutes: 360 | |
| permissions: | |
| # The "id-token: write" permission is required or Machine ID will not be | |
| # able to authenticate with the cluster. | |
| id-token: write | |
| contents: read | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install Teleport | |
| uses: teleport-actions/setup@176c25dfcd19cd31a252f275d579822b243e7b9c # pin@v1.0.6 | |
| with: | |
| version: 18.2.0 | |
| - name: Authorize against Teleport | |
| id: auth | |
| uses: teleport-actions/auth@d48447391aa28b202c98ae3f3358097025c32511 # pin@v2.0.4 | |
| with: | |
| # Specify the publically accessible address of your Teleport proxy. | |
| proxy: mysten.teleport.sh:443 | |
| # Specify the name of the join token for your bot. | |
| token: sui-simtest-token | |
| # Specify the length of time that the generated credentials should be | |
| # valid for. This is optional and defaults to "1h" | |
| certificate-ttl: 6h | |
| - name: Show Teleport cert details | |
| run: tsh -i ${{ steps.auth.outputs.identity-file }} status | |
| env: | |
| TELEPORT_IDENTITY_FILE: ${{ env.TELEPORT_IDENTITY_FILE }} | |
| # Cargo clean and git restore on any left over files from git checkout, and deletes all remote tracking branches | |
| - name: Environment clean | |
| run: | | |
| tsh -i ${{ steps.auth.outputs.identity-file }} ssh ubuntu@simtest-01 echo 'teleport OK' | |
| tsh -i ${{ steps.auth.outputs.identity-file }} ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && rm -rf ~/sui" | |
| tsh -i ${{ steps.auth.outputs.identity-file }} ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/ && git clone git@github.com:MystenLabs/sui.git" | |
| # Deleting files in tmpfs that usually fill up pretty quickly after each run. Cargo clean to free up space as well. | |
| - name: Tmpfs and cargo clean | |
| run: | | |
| tsh -i ${{ steps.auth.outputs.identity-file }} ssh ubuntu@simtest-01 "sudo rm -rf /tmp/*" | |
| tsh -i ${{ steps.auth.outputs.identity-file }} ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/sui && cargo clean" | |
| # Checkout out the latest sui repo | |
| - name: Checkout sui repo | |
| run: | | |
| tsh -i ${{ steps.auth.outputs.identity-file }} ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/sui && git fetch origin && git checkout ${{ env.SUI_REF }}" | |
| # Setting up cargo and simtest | |
| - name: Install simtest | |
| run: | | |
| tsh -i ${{ steps.auth.outputs.identity-file }} ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/sui && ./scripts/simtest/install.sh" | |
| - name: Set protocol config override based on trigger | |
| # This step determines which protocol config override to use. | |
| # It checks if the trigger was a manual dispatch or a specific schedule. | |
| run: | | |
| message="" | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| if [[ -n "${{ github.event.inputs.protocol_config_override }}" ]]; then | |
| echo "Testing with protocol config override: ${{ github.event.inputs.protocol_config_override }}" | |
| echo "SUI_PROTOCOL_CONFIG_CHAIN_OVERRIDE=${{ github.event.inputs.protocol_config_override }}" >> $GITHUB_ENV | |
| else | |
| echo "Testing latest protocol config for dispatched run" | |
| fi | |
| elif [[ "${{ github.event.schedule }}" == "0 21 * * *" ]]; then | |
| echo "Testing mainnet protocol config" | |
| echo "SUI_PROTOCOL_CONFIG_CHAIN_OVERRIDE=mainnet" >> $GITHUB_ENV | |
| message="(mainnet config) " | |
| else | |
| echo "Testing latest protocol config" | |
| fi | |
| # Set the output for other jobs to use | |
| echo "protocol_message=${message}" >> $GITHUB_OUTPUT | |
| # Run simulator tests | |
| - name: Run simtest | |
| run: | | |
| tsh -i ${{ steps.auth.outputs.identity-file }} ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/sui && RUSTUP_MAX_RETRIES=10 CARGO_TERM_COLOR=always CARGO_INCREMENTAL=0 CARGO_NET_RETRY=10 RUST_BACKTRACE=short RUST_LOG=off NUM_CPUS=24 TEST_NUM=${{ env.TEST_NUM }} ./scripts/simtest/simtest-run.sh" | |
| notify: | |
| name: Notify | |
| needs: [simtest] | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'schedule' && failure() | |
| steps: | |
| - uses: technote-space/workflow-conclusion-action@45ce8e0eb155657ab8ccf346ade734257fd196a5 # Pin v4.1.1 | |
| - name: Checkout sui repo main branch | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 | |
| - name: Get sui commit | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| export sui_sha=$(git rev-parse HEAD) | |
| echo "sui_sha=${sui_sha}" >> $GITHUB_ENV | |
| - name: Get link to logs | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| gh_job_link=$(gh api -X GET 'repos/MystenLabs/sui/actions/runs/${{ github.run_id }}/jobs' --jq '.jobs.[0].html_url') | |
| echo "gh_job_link=${gh_job_link}" >> $GITHUB_ENV | |
| - name: Get current oncall | |
| run: | | |
| export current_oncall=$(curl -s --request GET \ | |
| --url 'https://api.pagerduty.com/oncalls?schedule_ids[]=PGCQ3YS' \ | |
| --header 'Accept: application/json' \ | |
| --header 'Authorization: Token token=${{ secrets.PAGERDUTY_ACCESS_KEY }}' \ | |
| --header 'Content-Type: application/json' \ | |
| | jq '.oncalls[].user.summary' | tr -d '"') | |
| echo "current_oncall=$(echo ${current_oncall})" >> $GITHUB_ENV | |
| export oncall_name=$(curl -s --request GET \ | |
| --url 'https://api.pagerduty.com/oncalls?schedule_ids[]=PGCQ3YS' \ | |
| --header 'Accept: application/json' \ | |
| --header 'Authorization: Token token=${{ secrets.PAGERDUTY_ACCESS_KEY }}' \ | |
| --header 'Content-Type: application/json' \ | |
| | jq '.oncalls[].escalation_policy.summary' | tr -d '"') | |
| echo "oncall_name=$(echo ${oncall_name})" >> $GITHUB_ENV | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # pin v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - name: Get slack id for the oncall | |
| run: | | |
| export slack_id=$(aws s3 cp s3://mysten-employees-dir/employees.json - | jq --arg ONCALL "${{ env.current_oncall }}" '.[] | if .name == $ONCALL then .slack_id else empty end') | |
| echo "slack_id=$(echo ${slack_id} | tr -d '"')" >> $GITHUB_ENV | |
| - name: Post to slack | |
| uses: slackapi/slack-github-action@34c3fd73326693ef04728f8611669d918a2d781d # pin@v1.19.0 | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| SUI_SHA: ${{ env.sui_sha }} | |
| GH_JOB_LINK: ${{ env.gh_job_link }} | |
| SLACK_ID: ${{ env.slack_id }} | |
| ONCALL_NAME: ${{ env.oncall_name }} | |
| with: | |
| channel-id: 'simtest-nightly' | |
| slack-message: | | |
| :sui::error-cross: *${{ github.workflow }}* ${{ needs.simtest.outputs.protocol_message }}workflow status: `${{ env.WORKFLOW_CONCLUSION }}` | |
| Sui commit: <https://github.com/MystenLabs/sui/commit/${{ env.SUI_SHA }}|${{ env.SUI_SHA }}> | |
| Run: <${{ env.GH_JOB_LINK }}|${{ github.run_id }}> | |
| <@${{ env.SLACK_ID }}>, current `${{ env.ONCALL_NAME }}` oncall, please debug failures: `tsh ssh ubuntu@simtest-01` and look in the `/home/ubuntu/simtest_logs/{date}` folder for test results |