Merge pull request #1 from deep-foundation/main

Merge

Author: Konard

.github/workflows/dockerize.yaml

@@ -2,8 +2,6 @@ name: Deploy deepRunnerJs
 on:
   push:
     branches: [ main ]
-  pull_request:
-    branches: [ main ]
   workflow_dispatch:
 defaults:
   run:
@@ -16,19 +14,24 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v1
         with:
-          node-version: '14.15'
+          node-version: 18
+      - name: Install latest npm
+        run: npm install -g npm@latest
       - name: build
         run: npm ci && npm run package:build
+      - name: Display npm log
+        if: ${{ failure() }}
+        run: cat /home/runner/.npm/_logs/*-debug.log
       - name: Log in to Docker Hub
         uses: docker/login-action@v1
         with:
-          username: ${{ secrets.DOCKER_TEST_USERNAME }}
-          password: ${{ secrets.DOCKER_TEST_PASSWORD }}
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Extract metadata (tags, labels) for Docker
         id: meta
         uses: docker/metadata-action@v3
         with:
-          images: konard/deep-runner-js
+          images: deepf/js-docker-isolation-provider
       - name: build docker image and push
         id: docker_build
         uses: docker/build-push-action@v2

.github/workflows/npm-build.yml

@@ -0,0 +1,10 @@
+name: Npm Build
+
+on:
+  pull_request:
+    types: [opened, reopened, edited, synchronize]
+  workflow_dispatch:
+
+jobs:
+  main:
+    uses: deep-foundation/workflows/.github/workflows/npm-build.yml@main

.gitpod.yml

@@ -0,0 +1,2 @@
+tasks:
+  - init: npm ci

.nvmrc

@@ -0,0 +1 @@
+18

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:14.15-stretch
+FROM node:18
 
 COPY package.json .
 COPY index.js .
@@ -7,4 +7,7 @@ COPY index.ts .
 COPY node_modules ./node_modules
 COPY imports ./imports
 
+RUN apt-get update
+RUN apt-get install ffmpeg -y
+
 ENTRYPOINT ["node", "index.js"]

LICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <https://unlicense.org>

README.md

@@ -1 +1,59 @@
-# deepRunnerJs
+[![Gitpod](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/deep-foundation/js-docker-isolation-provider) 
+
+# js-docker-isolation-provider
+
+## HTTP routes
+
+- `/healthz` - GET - 200 - Health check endpoint
+  - Response:
+    - `{}`
+- `/init` - GET - 200 - Initialization endpoint
+  - Response:
+    - `{}`
+- `/call` - GET - 200 - Call executable code of handler in this isolation provider (it used in HandleInsert, HandleUpdate, HandleDelete and so on)
+  - Request:
+    - body:
+      - params:
+        - jwt: STRING - Deeplinks send this token, for create gql and deep client
+        - secret: STRING - Secret to access Hasura in unsafe mode (if package/user are allowed to use it by permissions)
+        - code: STRING - Code of handler
+        - data: {} - Data for handler execution from deeplinks
+          > If this is type handler
+          - oldLink - from deeplinks, link before transaction
+          - newLink - from deeplinks, link after transaction
+          - promiseId - from deeplinks, promise id
+  - Response:
+    - `{ resolved?: any; rejected?: any; }` - If resolved or rejected is not null, then it's result of execution
+- `/http-call` - GET - 200 - Call executable code of handler in this isolation provider to produce the http responce (it is used in HandleRoute)
+  - Request:
+    - Headers:
+      - `deep-call-options`
+        - jwt: STRING - Deeplinks send this token, for create gql and deep client
+        - secret: STRING - Secret to access Hasura in unsafe mode (if package/user are allowed to use it by permissions)
+        - code: STRING - Code of handler
+        - data: {} - Data for handler execution from deeplinks
+          > If this is type handler
+          - oldLink - from deeplinks, link before transaction
+          - newLink - from deeplinks, link after transaction
+          - promiseId - from deeplinks, promise id
+  - Responce:
+    - Http responce generated by code if execution is successful
+    - `{ rejected: any; }` - on error
+   
+## Diagnostics
+
+### Logs
+
+#### Get container logs to console:
+
+```bash
+docker logs $(docker ps -a -q --filter "ancestor=deepf/js-docker-isolation-provider:main")
+```
+
+#### Get container logs to file:
+
+Sometimes console cannot output the full logs so it might be helpful to store the entire container's logs as file. It can be done like this:
+
+```bash
+docker logs $(docker ps -a -q --filter "ancestor=deepf/js-docker-isolation-provider:main") > log.txt
+```

index.ts

@@ -1,56 +1,118 @@
 import express from 'express';
-import { generateApolloClient } from "@deep-foundation/hasura/client";
-import { DeepClient } from "@deep-foundation/deeplinks/imports/client";
-import memoize from 'lodash/memoize'
+import { generateApolloClient } from "@deep-foundation/hasura/client.js";
+import { HasuraApi } from '@deep-foundation/hasura/api.js';
+import { DeepClient, parseJwt } from "@deep-foundation/deeplinks/imports/client.js";
+import { gql } from '@apollo/client/index.js';
+import { serializeError } from 'serialize-error';
+import memoize from 'lodash/memoize.js';
+import http from 'http';
+// import { parseStream, parseFile } from 'music-metadata';
+import { createRequire } from 'node:module';
+import bodyParser from 'body-parser';
+const require = createRequire(import.meta.url);
 
 const memoEval = memoize(eval);
 
 const app = express();
-let initiated;
 
-const GQL_URN = process.env.GQL_URN || 'localhost:3006/gql';
+const GQL_URN = process.env.GQL_URN || 'host.docker.internal:3006/gql';
 const GQL_SSL = process.env.GQL_SSL || 0;
 
-const toJSON = (data) => JSON.stringify(data, Object.getOwnPropertyNames(data), 2);
+const DEEPLINKS_HASURA_PATH = process.env.DEEPLINKS_HASURA_PATH || 'host.docker.internal:8080';
+const DEEPLINKS_HASURA_SSL = !!(+process.env.DEEPLINKS_HASURA_SSL || 0);
+
+const requireWrapper = (id: string) => {
+  // if (id === 'music-metadata') {
+  //   return { parseStream, parseFile };
+  // }
+  return require(id);
+}
+
+DeepClient.resolveDependency = requireWrapper;
+
+const makeFunction = (code: string) => {
+  const fn = memoEval(code);
+  if (typeof fn !== 'function')
+  {
+    throw new Error("Executed handler's code didn't return a function.");
+  }
+  return fn;
+}
+
+const makeDeepClient = (token: string, path?: string, ssl?: boolean, secret?: string) => {
+  if (!token) throw new Error('No token provided');
+  const decoded = parseJwt(token);
+  const linkId = decoded?.userId;
+  const apolloClient = generateApolloClient({
+    path: GQL_URN,
+    ssl: !!+GQL_SSL,
+    token,
+  });
+
+  const unsafe: any = {};
+  if (secret) {
+    unsafe.hasura = new HasuraApi({
+      path,
+      ssl,
+      secret,
+    });
+  }
+
+  const deepClient = new DeepClient({ apolloClient, linkId, token, unsafe }) as any;
+  return deepClient;
+}
+
+const execute = async (args, options) => {
+  const { jwt, secret, code, data, path, ssl } = options;
+  const fn = makeFunction(code);
+  const deep = makeDeepClient(jwt, path, ssl, secret);
+  // await supports both sync and async functions the same way
+  const result = await fn(...args, { data, deep, gql, require: requireWrapper });
+  return result;
+}
+
+app.use(bodyParser.json({limit: '50mb'}));
+app.use(bodyParser.urlencoded({limit: '50mb', extended: true}));
 
-app.use(express.json());
 app.get('/healthz', (req, res) => {
   res.json({});
 });
+
 app.post('/init', (req, res) => {
   res.json({});
 });
-app.post('/call', async (req, res) => {
-  try 
-  {
-    console.log({ body: req?.body });
-    const token = req?.body?.params?.jwt;
-    if (!token) throw new Error('No token provided');
-    initiated = memoEval(req?.body?.params?.code);
-    if (typeof initiated !== 'function')
-    {
-      throw new Error("Executed handler's code didn't return a function.");
-    }
-
-    const apolloClient = generateApolloClient({
-      path: GQL_URN,
-      ssl: !!+GQL_SSL,
-      token,
-    });
 
-    const deepClient = new DeepClient({ apolloClient });
-    const result = await initiated({ deep: deepClient }); // Supports both sync and async functions the same way
+app.post('/call', async (req, res) => {
+  try {
+    const options = req?.body?.params || {};
+    console.log('call options', options);
+    const result = await execute([], options);
     console.log('call result', result);
     res.json({ resolved: result });
   }
   catch(rejected)
   {
-    const processedRejection = JSON.parse(toJSON(rejected));
-    console.log('rejected: ', processedRejection);
+    const processedRejection = serializeError(rejected);
+    console.log('rejected', processedRejection);
     res.json({ rejected: processedRejection });
   }
 });
 
-app.listen(process.env.PORT, () => {
-  console.log(`Listening ${process.env.PORT} port`);
-});
+app.use('/http-call', async (req, res, next) => {
+  try {
+    const options = JSON.parse(decodeURI(`${req.headers['deep-call-options']}`) || '{}');
+    console.log('http call options', options);
+    const result = await execute([req, res, next], options);
+    console.log('http call result', result);
+    return result;
+  }
+  catch(rejected)
+  {
+    const processedRejection = serializeError(rejected);
+    console.log('rejected', processedRejection);
+    res.json({ rejected: processedRejection }); // TODO: Do we need to send json to client? HTTP respone may not expect json output.
+  }
+});
+
+http.createServer({ maxHeaderSize: 10*1024*1024*1024 }, app).listen(process.env.PORT);
+console.log(`Listening ${process.env.PORT} port`);

key.json

@@ -0,0 +1,12 @@
+{
+  "type": "service_account",
+  "project_id": "deep-sound-handler",
+  "private_key_id": "c03fd7ed391b0e82a666f6b88089e3b145c33879",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCljHCNaHpGK/AS\n4iE9+h+eAL93H3HKLchrz3hADZjHwNetquySoh71g3zcXhKqmQWbD6o3pw4+tAsJ\nHfecgjAlxvmZ8wNbycgfnT6u2iM2fCUhwuxy2WYURHAGTqupAkkZIGksLDSBYQqH\nrqx7l142JnaJdqeYm4YjRdNO8X3ROcQcSlJflK4yBozFq2QyNJLHZMpL+1bCY88Q\nhIKEqXWbVT2iWTAtJPwZqdbRyIr/vjnk63fNuEmNy1VFW1Q6hR2wA3WyY2Re5nKG\nxMQuBpJSfrZn5mJlodPkKpPUP4Jtkn9kfT0O5Fk5nxVA5XIa8kjVHEXzVmrQYbrv\n5VEsKF71AgMBAAECggEAGxYueESm9QJNZpmPPNfHApA2w6Pmd2EkUNeCYmCXJd/c\nqjK0QI28KQmZjWjq/vTIoQVgIXj48LN+tsArenUmsZer0U+cthD+6IVO80cHyeto\nG+0LyQLfG+PSvIV0utN/6FeQPcUB1kVtdqPS3un2ZT/H11gNtf+EZBMRrvH+VxQs\nGyNDQeO9tMhrOxeOyYrZg8KBa+1K9yucBKgQUvjOFTL7jSLXq7ODCs1joUNy17ZD\nOK2H40dp+ul5b9BeA58zwDygHhh1DKgfXef0KjhfzuKQG6wXs0DUXvdG/S6QTnX/\nmZ5qP06/FFlaIVppYJIOrdG5eCtyRfitSWhSgcmPJQKBgQDYC03dSS8gS26GWGgV\nlWnyh7XXVtMx2M/i9cx6/zkXp0p1y+aZoU20CumlTXPbGbIGy5JbRSGmeGCszrZo\nTWA75Yj4lkDR0NjkcfpdvdIe3zjsWuo9N6gNJV5zCSjib9O+YZ5BbcHUoChlaLYz\nG7Uzpszp1jB3xg9AyMfpf+1IcwKBgQDEKmcwLwjcjYRsnmyiveOxNWSsR9CsN/1q\np1WS5XTs9Tv3gtTt6WweGoi/dKTOMNeVRR1pxSxYHJ2e2/hFAAwLUjFUoFvj6htO\nShfqxGtQA+RWCrAxg59mD5I8JFDdm/NGbYP0czOaOhnUbSUVDQ0dNJpjflEsqXeF\n9+bOr6So9wKBgDf2NaUsX9itsWN7Yvq0SU1ZefDdvLccwjy4Ds+NbOyDmPYKayFA\ndzA742m+NlR6w3KfKLobnivQ4rngkb9Sy6q4OlKqGQBAaO5D1+aQVO1KRSR0KFjT\nIeTF2UFTDhQZg2+9OZEZLSw2kEA0b32tn7JRcqLfqI5d73WIjMAfMwehAoGAImv+\nZ2oZd/otPpIeJgCEL9harhB8AXxhr5FlZr104w+1Uh1XF9hZ7H5jeJUTRkszyTGz\nk5fRzDRZREL9Mb5sXqAxn9Pzy+6MlBRUVhHZctT8AE0n1chu9A3Pb1ZACmMPMVCO\nUxrT90AywB/W3fbIUlOks8i6nceu/YcerS4NCskCgYBCusqLkGStC9Tw7R1xX+ey\nDkA5sGNfHIV18PQH1l0R1MNf1cXjxkBvt4aVTO5iqcLfr/LCGwYQJn/CXuBkS2Tv\njKOAdmkFVGe1DdMLXDbhHKwduHnqgAkEyWu0Jw+lPHvziTmcJy2t3m9Lcm9srgEu\nKE9c9B+zjkFrG8NTrLAhbQ==\n-----END PRIVATE KEY-----\n",
+  "client_email": "deep-sound-handler@deep-sound-handler.iam.gserviceaccount.com",
+  "client_id": "100687498969961716359",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/deep-sound-handler%40deep-sound-handler.iam.gserviceaccount.com"
+}

local/docker-compose.yml

@@ -1,7 +1,7 @@
-version: '3.6'
+version: '3.7'
 services:
-  deep-runner-js:
-    image: konard/deep-runner-js:main
+  js-docker-isolation-provider:
+    image: deepf/js-docker-isolation-provider:main
     networks:
       - deep
     ports: