Merge branch 'master' into no-envars

Author: 0x2b3bfa0

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,29 @@
+name: CodeQL
+
+on:
+  pull_request:
+    paths-ignore:
+      - '**.md'
+      - assets/**
+  schedule:
+    - cron: '0 0 * * *' # everyday @ 0000 UTC
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_name }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+    steps:
+      - uses: actions/checkout@v3
+      - uses: github/codeql-action/init@v2
+        with:
+          languages: javascript
+      - uses: github/codeql-action/autobuild@v2
+      - uses: github/codeql-action/analyze@v2

.github/workflows/gitlab.yml

@@ -6,8 +6,7 @@ jobs:
     services:
       gitlab:
         image: docker://gitlab/gitlab-ce
-        ports:
-          - 8000:8000
+        ports: ['8000:8000']
         env:
           GITLAB_OMNIBUS_CONFIG: |
             external_url 'http://localhost:8000/gitlab'
@@ -21,8 +20,7 @@ jobs:
               }
               '
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Configure credentials
         run: |
           docker exec ${{ job.services.gitlab.id }} bin/gitlab-rails runner "
@@ -60,8 +58,7 @@ jobs:
           --request GET \
           | jq -r .id \
           | xargs -0 printf "::set-output name=hash::%s"
-      - name: Install dependencies
-        run: npm ci
+      - run: npm ci
       - name: Run cml-send-comment
         run: |
           node bin/cml.js send-comment \

.github/workflows/images.yml

@@ -1,8 +1,8 @@
 on:
   workflow_call:
     inputs:
-      test:
-        required: false
+      release:
+        required: true
         type: boolean
 jobs:
   images:
@@ -27,7 +27,7 @@ jobs:
             base: 1
             dvc: 2
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
           fetch-depth: 0
@@ -73,7 +73,7 @@ jobs:
           )"
           echo ::set-output name=tags::"${TAGS//$'\n'/'%0A'}"
       - uses: docker/setup-buildx-action@v1
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: /tmp/.buildx-cache
           key:
@@ -94,7 +94,7 @@ jobs:
       - uses: docker/build-push-action@v2
         with:
           push:
-            ${{ github.event_name == 'push' || github.event_name == 'schedule'
+            ${{ inputs.release || github.event_name == 'push' || github.event_name == 'schedule'
             || github.event_name == 'workflow_dispatch' }}
           context: ./
           file: ./Dockerfile

.github/workflows/release.yml

@@ -15,23 +15,32 @@ jobs:
     if: github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - run: |
           git config --global user.name Olivaw[bot]
           git config --global user.email 64868532+iterative-olivaw@users.noreply.github.com
           git checkout -b bump/$(npm version ${{ github.event.inputs.bump }})
-          git push --set-upstream origin --follow-tags HEAD
+          git push --set-upstream origin HEAD
           gh pr create --title "Bump version to $(git describe --tags)" --body "Approve me 🤖"
-          gh pr merge --squash --auto
+          gh pr merge --auto --squash
         env:
           GITHUB_TOKEN: ${{ secrets.ADMIN_GITHUB_TOKEN }}
   release:
     if: github.event.pull_request.merged && startsWith(github.head_ref, 'bump/')
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - run:
-          gh release create --generate-notes {--title=CML\ ,}$(basename ${{
-          github.head_ref }})
+      - uses: actions/checkout@v3
+      - run: >
+          gh release create
+          --target ${{ github.event.pull_request.merge_commit_sha }}
+          {--title=CML\ ,}$(basename ${{ github.head_ref }})
+          --generate-notes
+          --draft
         env:
           GITHUB_TOKEN: ${{ secrets.ADMIN_GITHUB_TOKEN }}
+  package:
+    needs: release
+    secrets: inherit
+    uses: ./.github/workflows/test-deploy.yml
+    with:
+      release: true

.github/workflows/test-deploy.yml

@@ -1,11 +1,14 @@
 name: Test & Deploy
 on:
-  schedule:
-    - cron: '0 8 * * 1' # M H d m w (Mondays at 8:00)
-  release:
-    types: [created]
   pull_request_target:
   workflow_dispatch:
+  workflow_call:
+    inputs:
+      release:
+        required: true
+        type: boolean
+  schedule:
+    - cron: '0 8 * * 1' # M H d m w (Mondays at 8:00)
 jobs:
   authorize:
     environment:
@@ -22,13 +25,13 @@ jobs:
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
-      - name: check that npm lock file is on version 2
+      - name: npm lock file is v2
         run: jq --exit-status .lockfileVersion==2 < package-lock.json
   lint:
     needs: authorize
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
       - uses: actions/setup-node@v2
@@ -38,7 +41,7 @@ jobs:
     needs: authorize
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
       - uses: actions/setup-node@v2
@@ -69,7 +72,7 @@ jobs:
         system: [ubuntu, macos, windows]
     runs-on: ${{ matrix.system }}-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
       - if: matrix.system == 'windows'
@@ -91,54 +94,35 @@ jobs:
     needs: [lint, test, test-os]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
       - uses: actions/setup-node@v2
         with:
           registry-url: https://registry.npmjs.org
       - run: npm install
-      - run:
-          npm ${{ github.event_name == 'release' && 'publish' || 'publish
-          --dry-run' }}
+      - run: npm ${{ inputs.release && 'publish' || 'publish --dry-run' }}
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
-      - name: install ldid
-        run: |
+      - run: |
           sudo apt install --yes libplist-dev
-          git clone --branch v2.1.5 git://git.saurik.com/ldid.git 
+          git clone --branch v2.1.5 git://git.saurik.com/ldid.git
           sudo g++ -pipe -o /usr/bin/ldid ldid/ldid.cpp -I. -x c ldid/{lookup2.c,sha1.h} -lplist -lcrypto
-      - id: build
-        name: build
-        run: |
-          echo ::set-output name=tag::${GITHUB_REF#refs/tags/}
+      - run: |
           cp node_modules/@npcz/magic/dist/magic.mgc assets/magic.mgc
           npx --yes pkg --no-bytecode --public-packages "*" --public package.json
-          rm assets/magic.mgc
-          for cmd in '' runner publish pr; do
-            build/cml-linux-x64 $cmd --version
-          done
+          for cmd in '' runner publish pr; do build/cml-linux-x64 $cmd --version; done
           cp build/cml-linux{-x64,}
           cp build/cml-macos{-x64,}
-      - uses: softprops/action-gh-release@v1
-        if: github.event_name == 'release'
-        with:
-          files: |
-            build/cml-alpine-arm64
-            build/cml-alpine-x64
-            build/cml-linux-arm64
-            build/cml-linuxstatic-arm64
-            build/cml-linuxstatic-x64
-            build/cml-linux-x64
-            build/cml-linux
-            build/cml-macos-arm64
-            build/cml-macos-x64
-            build/cml-macos
-            build/cml-win-arm64.exe
-            build/cml-win-x64.exe
+      - if: inputs.release
+        run:
+          find build -type f | xargs gh release upload $(basename ${{
+          github.head_ref }})
         env:
           GITHUB_TOKEN: ${{ secrets.TEST_GITHUB_TOKEN }}
   images:
     needs: packages
     secrets: inherit
     uses: ./.github/workflows/images.yml
+    with:
+      release: ${{ inputs.release || false }}

.github/workflows/trigger-external.yml

@@ -7,24 +7,20 @@ on:
         type: string
         required: true
   release:
-    types:
-      - published
+    types: [published]
   pull_request:
-    branches:
-      - master
+    branches: [master]
 jobs:
   pr:
     if: ${{ github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        repos:
-          - cml-playground
+        repos: [cml-playground]
     steps:
       - name: Trigger external actions
         run: |
-          curl --silent --show-error \
-            --request POST \
+          curl --silent --show-error --request POST \
             --header "Authorization: token ${{ secrets.TEST_GITHUB_TOKEN }}" \
             --header "Accept: application/vnd.github.v3+json" \
             --url "https://api.github.com/repos/iterative/${{ matrix.repos }}/dispatches" \
@@ -36,13 +32,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        repos:
-          - cml-playground
+        repos: [cml-playground]
     steps:
       - name: Trigger external actions
         run: |
-          curl --silent --show-error \
-            --request POST \
+          curl --silent --show-error --request POST \
             --header "Authorization: token ${{ secrets.TEST_GITHUB_TOKEN }}" \
             --header "Accept: application/vnd.github.v3+json" \
             --url "https://api.github.com/repos/iterative/${{ matrix.repos }}/dispatches" \

.gitignore

@@ -9,3 +9,4 @@ terraform.*
 !terraform.test.js
 crash.log
 /build
+/coverage

bin/cml.js

@@ -157,7 +157,7 @@ const handleError = (message, error) => {
       const event = { ...telemetryEvent, error: err.message };
       await send({ event });
     }
-    winston.error({ err });
+    winston.error(err);
     process.exit(1);
   }
 })();

bin/cml/asset/publish.e2e.test.js

@@ -0,0 +1,82 @@
+const fs = require('fs');
+const { exec } = require('../../../src/utils');
+
+describe('CML e2e', () => {
+  test('cml publish assets/logo.png --md', async () => {
+    const output = await exec(`node ./bin/cml.js publish assets/logo.png --md`);
+
+    expect(output.startsWith('![](')).toBe(true);
+  });
+
+  test('cml publish assets/logo.png', async () => {
+    const output = await exec(`node ./bin/cml.js publish assets/logo.png`);
+
+    expect(output.startsWith('https://')).toBe(true);
+  });
+
+  test('cml publish assets/logo.pdf --md', async () => {
+    const title = 'this is awesome';
+    const output = await exec(
+      `node ./bin/cml.js publish assets/logo.pdf --md --title '${title}'`
+    );
+
+    expect(output.startsWith(`[${title}](`)).toBe(true);
+  });
+
+  test('cml publish assets/logo.pdf', async () => {
+    const output = await exec(`node ./bin/cml.js publish assets/logo.pdf`);
+
+    expect(output.startsWith('https://')).toBe(true);
+  });
+
+  test('cml publish assets/test.svg --md', async () => {
+    const title = 'this is awesome';
+    const output = await exec(
+      `node ./bin/cml.js publish assets/test.svg --md --title '${title}'`
+    );
+
+    expect(output.startsWith('![](') && output.endsWith(`${title}")`)).toBe(
+      true
+    );
+  });
+
+  test('cml publish assets/test.svg', async () => {
+    const output = await exec(`node ./bin/cml.js publish assets/test.svg`);
+
+    expect(output.startsWith('https://')).toBe(true);
+  });
+
+  test('cml publish assets/logo.pdf to file', async () => {
+    const file = `cml-publish-test.md`;
+
+    await exec(`node ./bin/cml.js publish assets/logo.pdf --file ${file}`);
+
+    expect(fs.existsSync(file)).toBe(true);
+    await fs.promises.unlink(file);
+  });
+
+  test('cml publish assets/vega-lite.json', async () => {
+    const output = await exec(
+      `node ./bin/cml.js publish --mime-type=application/json assets/vega-lite.json`
+    );
+
+    expect(output.startsWith('https://')).toBe(true);
+    expect(output.includes('cml=json')).toBe(true);
+  });
+
+  test('cml publish assets/test.svg in Gitlab storage', async () => {
+    const { TEST_GITLAB_REPO: repo, TEST_GITLAB_TOKEN: token } = process.env;
+
+    const output = await exec(
+      `node ./bin/cml.js publish --repo=${repo} --token=${token} --gitlab-uploads assets/test.svg`
+    );
+
+    expect(output.startsWith('https://')).toBe(true);
+  });
+
+  test('cml publish /nonexistent produces file error', async () => {
+    await expect(
+      exec('node ./bin/cml.js publish /nonexistent')
+    ).rejects.toThrowError('ENOENT');
+  });
+});