chore: remove some infrastructure tests (moved to dedicated repository)

.github/workflows/dataset.yml

@@ -0,0 +1,195 @@
+name: Generate dataset
+on:
+  workflow_dispatch:
+    inputs:
+      ledger-version:
+        description: 'The version of the ledger to deploy'
+        required: true
+      preview:
+        description: 'Preview the changes'
+        required: false
+        type: boolean
+      rds-db-subnet-group-name:
+        description: 'The network to deploy the RDS instance to'
+        required: false
+      until-log-id:
+        description: 'The log id to stop at'
+        required: false
+      generator-version:
+        default: 'latest'
+        description: 'The version of the generator to use'
+        required: false
+      namespace:
+        default: 'default'
+        description: 'The namespace to deploy the dataset to'
+        required: false
+      create-snapshot:
+        default: 'false'
+        description: 'Create a snapshot of the dataset'
+        required: false
+      script:
+        required: true
+        description: 'The script to generate the dataset'
+      organization:
+        description: 'The organization to deploy the dataset to'
+        required: false
+        default: formance
+      refresh:
+        default: false
+        description: 'Refresh the stack'
+        required: false
+  workflow_call:
+    secrets:
+      PULUMI_ACCESS_TOKEN:
+        required: true
+      KUBE_APISERVER:
+        required: true
+      KUBE_TOKEN:
+        required: true
+      TS_OAUTH_CLIENT_ID:
+        required: true
+      TS_OAUTH_SECRET:
+        required: true
+    inputs:
+      ledger-version:
+        type: string
+        description: 'The version of the ledger to deploy'
+        required: true
+      preview:
+        description: 'Preview the changes'
+        required: false
+        type: boolean
+      rds-db-subnet-group-name:
+        type: string
+        description: 'The network to deploy the RDS instance to'
+        required: false
+      until-log-id:
+        type: number
+        description: 'The log id to stop at'
+        required: false
+      generator-version:
+        type: string
+        default: 'latest'
+        description: 'The version of the generator to use'
+        required: false
+      namespace:
+        type: string
+        default: 'default'
+        description: 'The namespace to deploy the dataset to'
+        required: false
+      create-snapshot:
+        type: boolean
+        default: false
+        description: 'Create a snapshot of the dataset'
+        required: false
+      script:
+        type: string
+        required: true
+        description: 'The script to generate the dataset'
+      organization:
+        type: string
+        description: 'The organization to deploy the dataset to'
+        required: false
+        default: formance
+      refresh:
+        type: boolean
+        default: false
+        description: 'Refresh the stack'
+        required: false
+      destroy:
+        type: boolean
+        default: true
+        description: 'Destroy the stack'
+        required: false
+
+concurrency:
+  group: dataset-${{ inputs.ledger-version }}
+  cancel-in-progress: true
+
+jobs:
+  Run:
+    name: Generate dataset
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        name: Checkout
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '^1.23'
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::955332203423:role/github-access
+          aws-region: eu-west-1
+      - name: Tailscale
+        uses: tailscale/github-action@v2
+        with:
+          oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
+          oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
+          tags: tag:ci
+      - name: Create kube config file
+        env:
+          KUBE_APISERVER: ${{ secrets.KUBE_APISERVER }}
+          KUBE_TOKEN: ${{ secrets.KUBE_TOKEN }}
+        run: |
+          mkdir .kube;
+
+          cat <<EOF > .kube/config
+          apiVersion: v1
+          kind: Config
+          clusters:
+            - cluster:
+                insecure-skip-tls-verify: true
+                server: $KUBE_APISERVER
+              name: default
+          contexts:
+            - name: default
+              context:
+                cluster: default
+                user: default
+          users:
+            - name: default
+              user:
+                token: $KUBE_TOKEN
+          current-context: default
+          EOF
+      - uses: pulumi/actions@v6
+        name: Run Pulumi
+        with:
+          command: ${{ inputs.preview && 'preview' || 'up' }}
+          stack-name: ${{ inputs.organization }}/ledger-dataset/${{ inputs.ledger-version }}
+          work-dir: './tools/dataset'
+          suppress-progress: 'true'
+          refresh: ${{ inputs.refresh }}
+          config-map: |
+            ledger-version:
+              value: ${{ inputs.ledger-version }}
+            rds-db-subnet-group-name:
+              value: ${{ inputs.rds-db-subnet-group-name }}
+            until-log-id:
+              value: ${{ inputs.until-log-id }}
+            generator-version:
+              value: ${{ inputs.generator-version }}
+            namespace:
+              value: ${{ inputs.namespace }}
+            create-snapshot:
+              value: ${{ inputs.create-snapshot }}
+            script:
+              value: ${{ toJSON(inputs.script) }}
+          upsert: true
+        env:
+          PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
+          KUBECONFIG: /home/runner/work/ledger/ledger/.kube/config
+      - uses: pulumi/actions@v6
+        name: Clean resources
+        if: ${{ inputs.destroy && !inputs.preview }}
+        with:
+          command: destroy
+          stack-name: ${{ inputs.organization }}/ledger-dataset/${{ inputs.ledger-version }}
+          work-dir: './tools/dataset'
+        env:
+          PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
+          KUBECONFIG: /home/runner/work/ledger/ledger/.kube/config

.github/workflows/test_data.yml

@@ -0,0 +1,57 @@
+name: Test Generate V2.1
+on:
+  pull_request:
+    types: [ assigned, opened, synchronize, reopened, labeled ]
+
+jobs:
+  Run:
+    uses: ./.github/workflows/dataset.yml
+    secrets:
+      PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
+      KUBE_APISERVER: ${{ secrets.FORMANCE_DEV_KUBE_API_SERVER_ADDRESS }}
+      KUBE_TOKEN: ${{ secrets.FORMANCE_DEV_KUBE_TOKEN }}
+      TS_OAUTH_CLIENT_ID: ${{ secrets.TS_OAUTH_CLIENT_ID }}
+      TS_OAUTH_SECRET: ${{ secrets.TS_OAUTH_SECRET }}
+    with:
+      ledger-version: 'v2.1'
+      organization: 'formance'
+      rds-db-subnet-group-name: 'staging-eu-west-1-hosting'
+      until-log-id: 100000
+      generator-version: 'latest'
+      namespace: ledger-dataset-v2-1
+      create-snapshot: false
+      preview: false
+      refresh: true
+      destroy: false
+      script: |
+        const plain = `vars {
+          account $order
+          account $seller
+        }
+        send [USD/2 100] (
+          source = @world
+          destination = $order
+        )
+        send [USD/2 1] (
+          source = $order
+          destination = @fees
+        )
+        send [USD/2 99] (
+          source = $order
+          destination = $seller
+        )`
+
+        function next(iteration) {
+          return [{
+            action: 'CREATE_TRANSACTION',
+            data: {
+              script: {
+                plain,
+                vars: {
+                  order: `orders:${uuid()}`,
+                  seller: `sellers:${iteration % 5}`
+                }
+              }
+            }
+          }]
+        }

Earthfile

@@ -144,6 +144,7 @@ pre-commit:
 
     BUILD ./test/*+pre-commit
     BUILD ./tools/*+pre-commit
+    BUILD ./deployments/*+pre-commit
 
 openapi:
     FROM node:20-alpine