chore: Major CI Refactor

Author: mjameswh

.github/workflows/ci.yml

@@ -43,66 +43,103 @@ env:
   TEMPORAL_TESTING_MEM_LOG_DIR: /tmp/worker-mem-logs
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   REUSE_V8_CONTEXT: ${{ inputs.reuse-v8-context }}
+  IS_MAIN_BRANCH: |
+    ${{
+      github.repository == 'temporalio/sdk-typescript'
+        && (github.ref == 'refs/heads/main'
+            || startsWith(github.ref, 'refs/tags/')
+            || startsWith(github.ref, 'refs/heads/releases'))
+        && github.event_name != 'pull_request'
+    }}
 
 jobs:
   stress-test:
     runs-on: buildjet-4vcpu-ubuntu-2204
     steps:
       - name: Print build info
         run: 'echo test-type: ${{ inputs.test-type }}, test-timeout-minutes: ${{ inputs.test-timeout-minutes }}, reuse-v8-context: $REUSE_V8_CONTEXT'
-      - uses: actions/checkout@v2
+
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - uses: actions-rs/toolchain@v1
+
+      - name: Install Node
+        uses: actions/setup-node@v4
         with:
-          toolchain: stable
-      - uses: actions/setup-node@v1
+          node-version: 20
+
+      - name: Get NPM cache directory
+        id: npm-cache-dir
+        run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT}
+
+      - name: Restore NPM cache
+        uses: actions/cache/restore@v4
         with:
-          node-version: ${{ matrix.node }}
+          path: ${{ steps.npm-cache-dir.outputs.dir }}
+          key: npm-main-linux-x64-${{ hashFiles('./package-lock.json') }}
+          restore-keys: |
+            npm-main-linux-x64-
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          target: x86_64-unknown-linux-gnu
+
       - name: Install protoc
-        uses: arduino/setup-protoc@v1
+        uses: arduino/setup-protoc@v3
         with:
-          version: '3.x'
+          # TODO: Upgrade proto once https://github.com/arduino/setup-protoc/issues/99 is fixed
+          version: '23.x'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: Swatinem/rust-cache@v1
+
+      - name: Rust Cargo and Build cache
+        uses: Swatinem/rust-cache@v2
         with:
-          working-directory: packages/core-bridge
-      # Don't build during install phase since we're going to explicitly build
-      - run: npm ci --ignore-scripts
-      - run: npm run build
+          workspaces: packages/core-bridge -> target
+          prefix-key: corebridge-buildcache
+          shared-key: linux-intel
+          env-vars: ''
+          save-if: ${{ env.IS_MAIN_BRANCH }}
+
+      - name: Download dependencies
+        # Make up to 3 attempts to install NPM dependencies, to work around transient NPM errors
+        run: |
+          npm ci --ignore-scripts --verbose || npm ci --ignore-scripts --verbose || npm ci --ignore-scripts --verbose
+
+      - name: Compile code
+        run: npm run build
         env:
           BUILD_CORE_RELEASE: true
 
-      # docker-compose
-      - name: Checkout docker-compose repo
-        uses: actions/checkout@v2
-        with:
-          repository: temporalio/docker-compose
-          path: docker-compose
+      - name: Install Temporal CLI
+        uses: temporalio/setup-temporal@v0
 
-      - name: Start Temporal Server
-        run: docker-compose up -d
-        working-directory: docker-compose
+      - name: Run Temporal CLI
+        shell: bash
+        run: |
+          temporal server start-dev --headless &
 
-      - name: Wait for Temporal Server
-        run: npm run wait-namespace
-      # end:docker-compose
+      - name: Run tests
+        run: |
+          mkdir -p $TEMPORAL_TESTING_LOG_DIR/tails
+          mkdir -p $TEMPORAL_TESTING_MEM_LOG_DIR
+          timeout ${{ inputs.test-timeout-minutes }}m npm run ${{ inputs.test-type }}
 
-      - run: mkdir -p $TEMPORAL_TESTING_LOG_DIR/tails
-      - run: mkdir -p $TEMPORAL_TESTING_MEM_LOG_DIR
-      - run: 'timeout ${{ inputs.test-timeout-minutes }}m npm run ${{ inputs.test-type }}'
       - run: for f in $TEMPORAL_TESTING_LOG_DIR/*.log; do tail -20000 $f > $TEMPORAL_TESTING_LOG_DIR/tails/$(basename $f); done
         if: ${{ always() }}
-      - uses: actions/upload-artifact@v3
+
+      - uses: actions/upload-artifact@v4
         if: ${{ always() }}
         with:
-          name: worker-logs
+          name: worker-logs-${{ inputs.reuse-v8-context && 'reuse-v8' || 'no-reuse-v8' }}
           path: ${{ env.TEMPORAL_TESTING_LOG_DIR }}/tails
-      - uses: actions/upload-artifact@v3
+
+      - uses: actions/upload-artifact@v4
         if: ${{ always() }}
         with:
-          name: worker-mem-logs
+          name: worker-mem-logs-${{ inputs.reuse-v8-context && 'reuse-v8' || 'no-reuse-v8' }}
           path: ${{ env.TEMPORAL_TESTING_MEM_LOG_DIR }}
 
       # TODO: set up alerting
       # TODO: record test durations and other metrics like memory usage / cache utilization / CPU
+      # TODO: uses prebuilt binaries from ci.yml

.github/workflows/stress.yml

@@ -50,9 +50,14 @@ See [sdk-structure.md](./docs/sdk-structure.md)
 
 ### Environment setup
 
-- Install Node 18 and [Temporal Server](https://github.com/temporalio/docker-compose#temporal-server-docker-compose-files)
-- Install the [Rust toolchain](https://rustup.rs/)
-- Install [Protocol Buffers](https://github.com/protocolbuffers/protobuf/releases/)
+- The TS SDK can be executed on 16, 18 and 20. However, for SDK development, we recommend
+  using Node 20. For easier testing, you may want to use a version manager, such as:
+  [nvm](https://github.com/nvm-sh/nvm/blob/master/README.md) or [volta](https://volta.sh/).
+
+- To run tests, you will need access to a local Temporal server, e.g. using the
+  [Temporal CLI's integrated dev server](https://github.com/temporalio/cli#start-the-server).
+- Install the [Rust toolchain](https://rustup.rs/).
+- Install [Protocol Buffers](https://github.com/protocolbuffers/protobuf/releases/).
 - Clone the [sdk-typescript](https://github.com/temporalio/sdk-typescript) repo:
   ```sh
   git clone https://github.com/temporalio/sdk-typescript.git
@@ -62,7 +67,8 @@ See [sdk-structure.md](./docs/sdk-structure.md)
   ```sh
   git submodule update --init --recursive
   ```
-  > If you get a `The authenticity of host 'github.com (192.30.252.123)' can't be established.` error, run `ssh-keyscan github.com >> ~/.ssh/known_hosts` and retry.
+  > If you get a `The authenticity of host 'github.com (192.30.252.123)' can't be established.`
+  > error, run `ssh-keyscan github.com >> ~/.ssh/known_hosts` and retry.
 - Install the dependencies:
   ```sh
   npm ci
@@ -79,11 +85,13 @@ If building fails, resetting your environment may help:
 npx lerna clean -y && npm ci
 ```
 
-If `npm ci` fails in `@temporalio/core-bridge` on the command `node ./scripts/build.js`, you may need to do `rustup update`.
+If `npm ci` fails in `@temporalio/core-bridge` on the command `node ./scripts/build.js`, you may
+need to do `rustup update`.
 
 To update to the latest version of the Core SDK, run `git submodule update` followed by `npm run build` to recompile.
 
-> For cross compilation on MacOS follow [these instructions](https://github.com/temporalio/sdk-typescript/blob/main/docs/building.md) (only required for publishing packages).
+> For cross compilation on MacOS follow [these instructions](https://github.com/temporalio/sdk-typescript/blob/main/docs/building.md)
+> (only required for publishing packages).
 
 ## Development
 
@@ -109,7 +117,7 @@ described [here](https://doc.rust-lang.org/cargo/reference/overriding-dependenci
 
 In order to run integration tests:
 
-1. Run the temporal server using [docker-compose](https://github.com/temporalio/docker-compose).
+1. Run the Temporal server, e.g. using the [Temporal CLI's integrated dev server](https://github.com/temporalio/cli#start-the-server)
 1. Export `RUN_INTEGRATION_TESTS=true`
 
 #### test-npm-init

CONTRIBUTING.md

@@ -267,7 +267,7 @@ export async function encodeScheduleAction(
       },
       input: { payloads: await encodeToPayloads(dataConverter, ...action.args) },
       taskQueue: {
-        kind: temporal.api.enums.v1.TaskQueueKind.TASK_QUEUE_KIND_UNSPECIFIED,
+        kind: temporal.api.enums.v1.TaskQueueKind.TASK_QUEUE_KIND_NORMAL,
         name: action.taskQueue,
       },
       workflowExecutionTimeout: msOptionalToTs(action.workflowExecutionTimeout),

package-lock.json

@@ -901,7 +901,7 @@ export class WorkflowClient extends BaseClient {
       signalName,
       signalInput: { payloads: await encodeToPayloads(this.dataConverter, ...signalArgs) },
       taskQueue: {
-        kind: temporal.api.enums.v1.TaskQueueKind.TASK_QUEUE_KIND_UNSPECIFIED,
+        kind: temporal.api.enums.v1.TaskQueueKind.TASK_QUEUE_KIND_NORMAL,
         name: options.taskQueue,
       },
       workflowExecutionTimeout: options.workflowExecutionTimeout,
@@ -949,7 +949,7 @@ export class WorkflowClient extends BaseClient {
       workflowType: { name: workflowType },
       input: { payloads: await encodeToPayloads(this.dataConverter, ...opts.args) },
       taskQueue: {
-        kind: temporal.api.enums.v1.TaskQueueKind.TASK_QUEUE_KIND_UNSPECIFIED,
+        kind: temporal.api.enums.v1.TaskQueueKind.TASK_QUEUE_KIND_NORMAL,
         name: opts.taskQueue,
       },
       workflowExecutionTimeout: opts.workflowExecutionTimeout,

packages/client/src/schedule-helpers.ts

@@ -10,7 +10,7 @@
     "build-rust-release": "npm run build-rust -- --release",
     "install": "node ./scripts/build.js",
     "format": "cargo fmt",
-    "lint": "cargo clippy --fix",
+    "lint": "cargo clippy --fix --allow-staged",
     "lint.check": "cargo clippy"
   },
   "keywords": [

packages/client/src/workflow-client.ts

@@ -8,7 +8,6 @@ import {
 } from '@temporalio/client';
 import {
   LocalTestWorkflowEnvironmentOptions,
-  TestWorkflowEnvironment,
   workflowInterceptorModules as defaultWorkflowInterceptorModules,
 } from '@temporalio/testing';
 import {
@@ -21,7 +20,13 @@ import {
 } from '@temporalio/worker';
 import * as workflow from '@temporalio/workflow';
 import { ConnectionInjectorInterceptor } from './activities/interceptors';
-import { Worker, test as anyTest, bundlerOptions, registerDefaultCustomSearchAttributes } from './helpers';
+import {
+  Worker,
+  TestWorkflowEnvironment,
+  test as anyTest,
+  bundlerOptions,
+  registerDefaultCustomSearchAttributes,
+} from './helpers';
 
 export interface Context {
   env: TestWorkflowEnvironment;