From 353d2a64d9d599d2e6d82f6515884fe19e6a0385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 29 May 2024 13:01:04 +0000 Subject: [PATCH 01/11] selection, configuration and settings of CLI local development --- src/SDK/Language/CLI.php | 5 + templates/cli/index.js.twig | 2 + templates/cli/lib/commands/run.js.twig | 121 +++++++++++++++++++++++++ templates/cli/lib/parser.js.twig | 1 + templates/cli/lib/questions.js.twig | 74 ++++++++++++++- templates/cli/lib/utils.js.twig | 40 +++++++- 6 files changed, 241 insertions(+), 2 deletions(-) create mode 100644 templates/cli/lib/commands/run.js.twig diff --git a/src/SDK/Language/CLI.php b/src/SDK/Language/CLI.php index 4974c7cde..eb26e0c1f 100644 --- a/src/SDK/Language/CLI.php +++ b/src/SDK/Language/CLI.php @@ -182,6 +182,11 @@ public function getFiles(): array 'destination' => 'lib/commands/push.js', 'template' => 'cli/lib/commands/push.js.twig', ], + [ + 'scope' => 'default', + 'destination' => 'lib/commands/run.js', + 'template' => 'cli/lib/commands/run.js.twig', + ], [ 'scope' => 'service', 'destination' => '/lib/commands/{{service.name | caseDash}}.js', diff --git a/templates/cli/index.js.twig b/templates/cli/index.js.twig index 2126485c6..7b245d46a 100644 --- a/templates/cli/index.js.twig +++ b/templates/cli/index.js.twig @@ -13,6 +13,7 @@ const { client } = require("./lib/commands/generic"); {% if sdk.test != "true" %} const { login, logout, whoami } = require("./lib/commands/generic"); const { pull } = require("./lib/commands/pull"); +const { run } = require("./lib/commands/run"); const { push } = require("./lib/commands/push"); {% endif %} {% for service in spec.services %} @@ -40,6 +41,7 @@ program .addCommand(login) .addCommand(pull) .addCommand(push) + .addCommand(run) .addCommand(logout) {% endif %} {% for service in spec.services %} diff --git a/templates/cli/lib/commands/run.js.twig b/templates/cli/lib/commands/run.js.twig new file mode 100644 index 000000000..a1184c844 --- /dev/null +++ b/templates/cli/lib/commands/run.js.twig @@ -0,0 +1,121 @@ +const inquirer = require("inquirer"); +const { Command } = require("commander"); +const { localConfig, globalConfig } = require("../config"); +const { paginate } = require('../paginate'); +const { questionsRunFunctions } = require("../questions"); +const { actionRunner, success, log, error, commandDescriptions, drawTable } = require("../parser"); +const { systemHasCommand, isPortTaken } = require('../utils'); + +const systemTools = { + 'node': { + commands: [ + { command: "node", docs: "https://nodejs.org/en/download/package-manager" }, + { command: "npm", docs: "https://nodejs.org/en/download/package-manager" }, + ], + dependencyFiles: [ "package.json", "package-lock.json" ] + }, + // TODO: Add all runtime needs +}; + +const runFunction = async ({ port, engine, functionId } = {}) => { + // Selection + if(!functionId) { + const answers = await inquirer.prompt(questionsRunFunctions[0]); + functionId = answers.function; + } + + const functions = localConfig.getFunctions(); + const func = functions.find((f) => f.$id === functionId); + if (!func) { + throw new Error("Function '" + functionId + "' not found.") + } + + // Configuration: Port + if(port) { + port = +port; + } + + if(isNaN(port)) { + port = null; + } + + if(port) { + const taken = await isPortTaken(port); + + if(taken) { + log(`Port ${port} is already used.`); + port = null; + } + } + + if(!port) { + const answers = await inquirer.prompt(questionsRunFunctions[1]); + port = answers.port; + } + + // Configuration: Engine + if(engine !== "system" && engine !== "docker") { + engine = null; + } + + if(!engine) { + const answers = await inquirer.prompt(questionsRunFunctions[2]); + engine = answers.engine; + } + + if(engine === 'docker') { + log('💡 Hint: Using system is faster, but using Docker simulates the production environment precisely.'); + + if(!systemHasCommand('docker')) { + return error("Please install Docker first: https://docs.docker.com/engine/install/"); + } + } else if(engine === 'system') { + log('💡 Hint: Docker simulates the production environment precisely, but using system is faster'); + + const runtimeName = func.runtime.split('-')[0]; + const tool = systemTools[runtimeName]; + + for(const command of tool.commands) { + if(!systemHasCommand(command.command)) { + return error(`Your system is missing command "${command.command}". Please install it first: ${command.docs}`); + } + } + } + + // Settings + const settings = { + runtime: func.runtime, + entrypoint: func.entrypoint, + path: func.path, + commands: func.commands, + }; + log("Local function configuration:"); + drawTable([settings]); + log('If you wish to change local settings, update appwrite.json file and rerun the command. To deploy the function, run: appwrite push function'); + + childProcess.execSync('where ' + command, { stdio: 'pipe' }) + +} + +const run = new Command("run") + .alias("dev") + .description(commandDescriptions['run']) + .configureHelp({ + helpWidth: process.stdout.columns || 80 + }) + .action(actionRunner(async (_options, command) => { + command.help(); + })); + +run + .command("function") + .alias("functions") + .description("Run functions in the current directory.") + .option(`--functionId `, `Function ID`) + .option(`--port `, `Local port`) + .option(`--engine `, `Local engine, "system" or "docker"`) + .action(actionRunner(runFunction)); + +module.exports = { + run +} diff --git a/templates/cli/lib/parser.js.twig b/templates/cli/lib/parser.js.twig index dce756c9d..7eaf93170 100644 --- a/templates/cli/lib/parser.js.twig +++ b/templates/cli/lib/parser.js.twig @@ -157,6 +157,7 @@ const commandDescriptions = { "avatars": `The avatars command aims to help you complete everyday tasks related to your app image, icons, and avatars.`, "databases": `The databases command allows you to create structured collections of documents, query and filter lists of documents.`, "push": `The push command provides a convenient wrapper for pushing your functions, collections, buckets, teams and messaging.`, + "run": `The dev command allows you to run project locally to allow easy development and quick debugging.`, "functions": `The functions command allows you view, create and manage your Cloud Functions.`, "health": `The health command allows you to both validate and monitor your {{ spec.title|caseUcfirst }} server's health.`, "pull": `The pull command helps you pull your {{ spec.title|caseUcfirst }} project, functions, collections, buckets, teams and messaging`, diff --git a/templates/cli/lib/questions.js.twig b/templates/cli/lib/questions.js.twig index 67b5fd8ed..694e34dd5 100644 --- a/templates/cli/lib/questions.js.twig +++ b/templates/cli/lib/questions.js.twig @@ -6,6 +6,7 @@ const { accountListMfaFactors } = require("./commands/account"); const { sdkForConsole } = require("./sdks"); const { validateRequired } = require("./validations"); const { paginate } = require('./paginate'); +const { isPortTaken } = require('./utils'); const { databasesList } = require('./commands/databases'); const JSONbig = require("json-bigint")({ storeAsString: false }); @@ -515,6 +516,76 @@ const questionsMfaChallenge = [ } ]; +const questionsRunFunctions = [ + { + type: "list", + name: "function", + message: "Which function would you like to develop locally?", + validate: (value) => validateRequired('function', value), + choices: () => { + let functions = localConfig.getFunctions(); + if (functions.length === 0) { + throw new Error("No functions found in the current directory."); + } + let choices = functions.map((func, idx) => { + return { + name: `${func.name} (${func.$id})`, + value: func.$id + } + }) + return choices; + } + }, + { + type: "number", + name: "port", + message: 'Which port would you like function to listen on?', + default: async () => { + let port = 3000; + while(port < 3100) { + const taken = await isPortTaken(port); + if(!taken) { + return port; + } + + port++; + } + + return 3000; + }, + validate: function(value) { + const done = this.async(); + + (async () => { + const taken = await isPortTaken(value); + + if(taken) { + throw Error(`Port ${value} is taken. Pick another one.`); + } + })().then(() => { + done(null, true); + }).catch((err) => { + done(err.message); + }); + }, + }, + { + type: "list", + name: "engine", + message: "Which engine would you like to use?", + choices: [ + { + name: "Docker", + value: "docker", + }, + { + name: "System", + value: "system", + }, + ], + }, +]; + module.exports = { questionsPullProject, questionsLogin, @@ -528,5 +599,6 @@ module.exports = { questionsPushTeams, questionsGetEntrypoint, questionsListFactors, - questionsMfaChallenge + questionsMfaChallenge, + questionsRunFunctions }; diff --git a/templates/cli/lib/utils.js.twig b/templates/cli/lib/utils.js.twig index 289b1fa6e..48af40325 100644 --- a/templates/cli/lib/utils.js.twig +++ b/templates/cli/lib/utils.js.twig @@ -1,5 +1,7 @@ const fs = require("fs"); const path = require("path"); +const net = require("net"); +const childProcess = require('child_process'); function getAllFiles(folder) { const files = []; @@ -14,6 +16,42 @@ function getAllFiles(folder) { return files; } +async function isPortTaken(port) { + const taken = await new Promise((res, rej) => { + const tester = net.createServer() + .once('error', function (err) { + if (err.code != 'EADDRINUSE') return rej(err) + res(true) + }) + .once('listening', function() { + tester.once('close', function() { res(false) }) + .close() + }) + .listen(port); + }); + + return taken; +} + +function systemHasCommand(command) { + const isUsingWindows = process.platform == 'win32' + + try { + if(isUsingWindows) { + childProcess.execSync('where ' + command, { stdio: 'pipe' }) + } else { + childProcess.execSync(`[[ $(${command} --version) ]] || { exit 1; } && echo "OK"`, { stdio: 'pipe', shell: '/bin/bash' }); + } + } catch (error) { + console.log(error); + return false; + } + + return true; +} + module.exports = { - getAllFiles + getAllFiles, + isPortTaken, + systemHasCommand }; From 0575002daf56cafa2b18ce9b34982c557d7133b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 30 May 2024 11:05:05 +0000 Subject: [PATCH 02/11] More progress on local development --- templates/cli/base/params.twig | 2 + templates/cli/lib/commands/run.js.twig | 159 ++++++++++++++++++++++++- templates/cli/package.json.twig | 3 +- 3 files changed, 159 insertions(+), 5 deletions(-) diff --git a/templates/cli/base/params.twig b/templates/cli/base/params.twig index 87ca01341..a369232b0 100644 --- a/templates/cli/base/params.twig +++ b/templates/cli/base/params.twig @@ -16,6 +16,8 @@ const func = localConfig.getFunction(functionId); + ignore.add('.appwrite'); + if (func.ignore) { ignorer.add(func.ignore); } else if (fs.existsSync(pathLib.join({{ parameter.name | caseCamel | escapeKeyword }}, '.gitignore'))) { diff --git a/templates/cli/lib/commands/run.js.twig b/templates/cli/lib/commands/run.js.twig index a1184c844..367deafd3 100644 --- a/templates/cli/lib/commands/run.js.twig +++ b/templates/cli/lib/commands/run.js.twig @@ -1,13 +1,21 @@ +const childProcess = require('child_process'); +const chokidar = require('chokidar'); const inquirer = require("inquirer"); +const path = require("path"); const { Command } = require("commander"); const { localConfig, globalConfig } = require("../config"); const { paginate } = require('../paginate'); const { questionsRunFunctions } = require("../questions"); const { actionRunner, success, log, error, commandDescriptions, drawTable } = require("../parser"); const { systemHasCommand, isPortTaken } = require('../utils'); +const { info } = require('console'); + +const activeDockerIds = {}; const systemTools = { 'node': { + isCompiled: false, + startCommand: "node src/server.js", commands: [ { command: "node", docs: "https://nodejs.org/en/download/package-manager" }, { command: "npm", docs: "https://nodejs.org/en/download/package-manager" }, @@ -17,6 +25,101 @@ const systemTools = { // TODO: Add all runtime needs }; +async function dockerStop(id) { + delete activeDockerIds[id]; + const stopProcess = childProcess.spawn('docker', ['rm', '--force', id], { + stdio: 'pipe', + }); + + await new Promise((res) => { stopProcess.on('close', res) }); +} + +async function dockerPull(func) { + log('Pulling Docker image of function runtime ...'); + + const [ runtimeName, runtimeVersion ] = func.runtime.split('-', 2); + const imageName = `openruntimes/${runtimeName}:v3-${runtimeVersion}`; + + const pullProcess = childProcess.spawn('docker', ['pull', imageName], { + stdio: 'pipe', + pwd: path.join(process.cwd(), func.path) + }); + + pullProcess.stderr.on('data', (data) => { + process.stderr.write(`\n${data}$ `); + }); + + await new Promise((res) => { pullProcess.on('close', res) }); +} + +async function dockerBuild(func) { + log('Building function using Docker engine ...'); + + const [ runtimeName, runtimeVersion ] = func.runtime.split('-', 2); + const imageName = `openruntimes/${runtimeName}:v3-${runtimeVersion}`; + + const functionDir = path.join(process.cwd(), func.path); + + const id = `${new Date().getTime().toString(16)}${Math.round(Math.random() * 1000000000).toString(16)}`; + const params = ['run', '--rm', '--name', id, '-i', '-e', `OPEN_RUNTIMES_ENTRYPOINT=${func.entrypoint}`, '-v', `${functionDir}/:/mnt/code:rw`, imageName, 'sh', '-c', ` helpers/build.sh "${func.commands}"`]; + + const buildProcess = childProcess.spawn('docker', params, { + stdio: 'pipe', + pwd: functionDir + }); + + buildProcess.stdout.on('data', (data) => { + process.stdout.write(`\n${data}`); + }); + + buildProcess.stderr.on('data', (data) => { + process.stderr.write(`\n${data}`); + }); + + activeDockerIds[id] = true; + + await new Promise((res) => { buildProcess.on('close', res) }); + + delete activeDockerIds[id]; +} + +async function dockerStart(func, port) { + log('Starting function using Docker engine ...'); + + log("Permissions, events, CRON and timeouts dont apply when running locally."); + + log('💡 Hint: Function automatically restarts when you edit your code.'); + + success(`Visit http://localhost:${port}/ to execute your function.`); + + const [ runtimeName, runtimeVersion ] = func.runtime.split('-', 2); + const imageName = `openruntimes/${runtimeName}:v3-${runtimeVersion}`; + + const tool = systemTools[runtimeName]; + + const functionDir = path.join(process.cwd(), func.path); + + const id = `${new Date().getTime().toString(16)}${Math.round(Math.random() * 1000000000).toString(16)}`; + const params = ['run', '--rm', '--name', id, '-i', '-e', 'OPEN_RUNTIMES_SECRET=', '-p', `${port}:3000`, '-v', `${functionDir}/:/mnt/code:rw`, imageName, 'sh', '-c', ` helpers/start.sh "${tool.startCommand}"`]; + + const execProcess = childProcess.spawn('docker', params, { + stdio: 'pipe', + pwd: functionDir + }); + + // TODO: Find a way to see context.log + + execProcess.stdout.on('data', (data) => { + process.stdout.write(`\n${data}`); + }); + + execProcess.stderr.on('data', (data) => { + process.stderr.write(`\n${data}`); + }); + + activeDockerIds[id] = true; +} + const runFunction = async ({ port, engine, functionId } = {}) => { // Selection if(!functionId) { @@ -30,6 +133,9 @@ const runFunction = async ({ port, engine, functionId } = {}) => { throw new Error("Function '" + functionId + "' not found.") } + const runtimeName = func.runtime.split('-')[0]; + const tool = systemTools[runtimeName]; + // Configuration: Port if(port) { port = +port; @@ -72,9 +178,6 @@ const runFunction = async ({ port, engine, functionId } = {}) => { } else if(engine === 'system') { log('💡 Hint: Docker simulates the production environment precisely, but using system is faster'); - const runtimeName = func.runtime.split('-')[0]; - const tool = systemTools[runtimeName]; - for(const command of tool.commands) { if(!systemHasCommand(command.command)) { return error(`Your system is missing command "${command.command}". Please install it first: ${command.docs}`); @@ -93,8 +196,56 @@ const runFunction = async ({ port, engine, functionId } = {}) => { drawTable([settings]); log('If you wish to change local settings, update appwrite.json file and rerun the command. To deploy the function, run: appwrite push function'); - childProcess.execSync('where ' + command, { stdio: 'pipe' }) + process.on('SIGINT', () => { + for(const id in activeDockerIds) { + dockerStop(id); + } + + process.exit(); + }); + + if(engine === "docker") { + await dockerPull(func); + await dockerBuild(func); + await dockerStart(func, port); + + let watcherRunning = false; + + chokidar.watch('.', { + cwd: path.join(process.cwd(), func.path), + ignoreInitial: true, + ignored: [ ...(func.ignore ?? []), 'code.tar.gz', '.appwrite' ] + }).on('all', async (event, path) => { + if(watcherRunning) { + info("File change detected but ignored, because live reload is already being ran."); + return; + } + watcherRunning = true; + + log('Detected a change in ' + path); + + try { + log('Stopping the function ...'); + + for(const id in activeDockerIds) { + await dockerStop(id); + } + + if(tool.isCompiled || tool.dependencyFiles.includes(path)) { + await dockerBuild(func); + await dockerStart(func, port); + } else { + // TODO: Update code.tar.gz with latest changes + await dockerStart(func, port); + } + } catch(err) { + console.error(err); + } finally { + watcherRunning = false; + } + }); + } } const run = new Command("run") diff --git a/templates/cli/package.json.twig b/templates/cli/package.json.twig index 015e494e1..09d6ca0ba 100644 --- a/templates/cli/package.json.twig +++ b/templates/cli/package.json.twig @@ -31,7 +31,8 @@ "json-bigint": "^1.0.0", "inquirer": "^8.2.4", "tar": "^6.1.11", - "ignore": "^5.2.0" + "ignore": "^5.2.0", + "chokidar": "^3.6.0" }, "devDependencies": { "pkg": "5.8.1" From 5804bf70b1b0a43f17dbc9aa5e1c602bc6368db8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 31 May 2024 10:48:16 +0000 Subject: [PATCH 03/11] Update hot swap logic --- templates/cli/base/params.twig | 2 +- templates/cli/lib/commands/run.js.twig | 87 +++++++++++++++++++++++--- 2 files changed, 78 insertions(+), 11 deletions(-) diff --git a/templates/cli/base/params.twig b/templates/cli/base/params.twig index a369232b0..88ca44da0 100644 --- a/templates/cli/base/params.twig +++ b/templates/cli/base/params.twig @@ -16,7 +16,7 @@ const func = localConfig.getFunction(functionId); - ignore.add('.appwrite'); + ignorer.add('.appwrite'); if (func.ignore) { ignorer.add(func.ignore); diff --git a/templates/cli/lib/commands/run.js.twig b/templates/cli/lib/commands/run.js.twig index 367deafd3..ea2b4c16d 100644 --- a/templates/cli/lib/commands/run.js.twig +++ b/templates/cli/lib/commands/run.js.twig @@ -1,3 +1,6 @@ +const ignore = require("ignore"); +const tar = require("tar"); +const fs = require("fs"); const childProcess = require('child_process'); const chokidar = require('chokidar'); const inquirer = require("inquirer"); @@ -7,11 +10,12 @@ const { localConfig, globalConfig } = require("../config"); const { paginate } = require('../paginate'); const { questionsRunFunctions } = require("../questions"); const { actionRunner, success, log, error, commandDescriptions, drawTable } = require("../parser"); -const { systemHasCommand, isPortTaken } = require('../utils'); +const { systemHasCommand, isPortTaken, getAllFiles } = require('../utils'); const { info } = require('console'); const activeDockerIds = {}; +const openRuntimesVersion = 'v3'; const systemTools = { 'node': { isCompiled: false, @@ -35,10 +39,11 @@ async function dockerStop(id) { } async function dockerPull(func) { + return; // TODO: Remove log('Pulling Docker image of function runtime ...'); const [ runtimeName, runtimeVersion ] = func.runtime.split('-', 2); - const imageName = `openruntimes/${runtimeName}:v3-${runtimeVersion}`; + const imageName = `openruntimes/${runtimeName}:${openRuntimesVersion}-${runtimeVersion}`; const pullProcess = childProcess.spawn('docker', ['pull', imageName], { stdio: 'pipe', @@ -56,7 +61,7 @@ async function dockerBuild(func) { log('Building function using Docker engine ...'); const [ runtimeName, runtimeVersion ] = func.runtime.split('-', 2); - const imageName = `openruntimes/${runtimeName}:v3-${runtimeVersion}`; + const imageName = `openruntimes/${runtimeName}:${openRuntimesVersion}-${runtimeVersion}`; const functionDir = path.join(process.cwd(), func.path); @@ -93,14 +98,14 @@ async function dockerStart(func, port) { success(`Visit http://localhost:${port}/ to execute your function.`); const [ runtimeName, runtimeVersion ] = func.runtime.split('-', 2); - const imageName = `openruntimes/${runtimeName}:v3-${runtimeVersion}`; + const imageName = `openruntimes/${runtimeName}:${openRuntimesVersion}-${runtimeVersion}`; const tool = systemTools[runtimeName]; const functionDir = path.join(process.cwd(), func.path); const id = `${new Date().getTime().toString(16)}${Math.round(Math.random() * 1000000000).toString(16)}`; - const params = ['run', '--rm', '--name', id, '-i', '-e', 'OPEN_RUNTIMES_SECRET=', '-p', `${port}:3000`, '-v', `${functionDir}/:/mnt/code:rw`, imageName, 'sh', '-c', ` helpers/start.sh "${tool.startCommand}"`]; + const params = ['run', '--rm', '--name', id, '-i', '-e', 'OPEN_RUNTIMES_SECRET=', '-p', `${port}:3000`, '-v', `${functionDir}/.appwrite/logs:/mnt/logs:rw`, '-v', `${functionDir}/:/mnt/code:rw`, imageName, 'sh', '-c', ` helpers/start.sh "${tool.startCommand}"`]; const execProcess = childProcess.spawn('docker', params, { stdio: 'pipe', @@ -211,11 +216,21 @@ const runFunction = async ({ port, engine, functionId } = {}) => { let watcherRunning = false; + childProcess.execSync(`sudo mkdir -p ${path.join(process.cwd(), func.path, '.appwrite/logs')}`, { + pwd: path.join(process.cwd(), func.path) + }); + chokidar.watch('.appwrite/logs', { + cwd: path.join(process.cwd(), func.path), + ignoreInitial: true, + }).on('all', async (event, filePath) => { + console.log(fs.readFileSync(path.join(process.cwd(), func.path, filePath)).toString()); + }); + chokidar.watch('.', { cwd: path.join(process.cwd(), func.path), ignoreInitial: true, - ignored: [ ...(func.ignore ?? []), 'code.tar.gz', '.appwrite' ] - }).on('all', async (event, path) => { + ignored: [ ...(func.ignore ?? []), 'code.tar.gz', '.appwrite', '.appwrite/', '.appwrite/*', '.appwrite/**', '.appwrite/*.*', '.appwrite/**/*.*' ] + }).on('all', async (event, filePath) => { if(watcherRunning) { info("File change detected but ignored, because live reload is already being ran."); return; @@ -223,7 +238,7 @@ const runFunction = async ({ port, engine, functionId } = {}) => { watcherRunning = true; - log('Detected a change in ' + path); + log('Detected a change in ' + filePath); try { log('Stopping the function ...'); @@ -232,11 +247,63 @@ const runFunction = async ({ port, engine, functionId } = {}) => { await dockerStop(id); } - if(tool.isCompiled || tool.dependencyFiles.includes(path)) { + if(tool.isCompiled || tool.dependencyFiles.includes(filePath)) { await dockerBuild(func); await dockerStart(func, port); } else { - // TODO: Update code.tar.gz with latest changes + // TODO: Some try-catch approach, to rebuild if fails + log('Hot swapping function files ...'); + + const functionPath = path.join(process.cwd(), func.path); + const hotSwapPath = path.join(functionPath, '.appwrite/hot-swap'); + const buildPath = path.join(functionPath, 'code.tar.gz') + + // TODO: Using Node code, no sudo + childProcess.execSync(`sudo mkdir -p ${hotSwapPath} && sudo chmod 777 ${buildPath} ${hotSwapPath} && sudo tar -zxf ${buildPath} -C ${hotSwapPath}`, { + pwd: path.join(process.cwd(), func.path) + }); + + const ignorer = ignore(); + ignorer.add('.appwrite'); + + if (func.ignore) { + ignorer.add(func.ignore); + } + + // TODO: Better approach + const filesToCopy = getAllFiles(functionPath).map((file) => path.relative(functionPath, file)).filter((file) => !ignorer.ignores(file)); + + const copyCommands = []; + for(const f of filesToCopy) { + const filePath = path.join(hotSwapPath, f); + copyCommands.push(`sudo rm -rf ${filePath}`); + + const fileDir = path.dirname(filePath); + copyCommands.push(`sudo mkdir -p ${fileDir}`); + + const sourcePath = path.join(functionPath, f); + copyCommands.push(`sudo cp ${sourcePath} ${filePath}`); + } + childProcess.execSync(copyCommands.join(" && "), { + pwd: path.join(process.cwd(), func.path) + }); + + console.log(path.join(process.cwd(), func.path, '.appwrite')); + /* + childProcess.execSync("sudo chmod -R 777 .", { + pwd: path.join(process.cwd(), func.path, '.appwrite') + }); + */ + + childProcess.execSync(`sudo tar -C ${hotSwapPath} --exclude code.tar.gz -zcf ${buildPath} .`, { + pwd: hotSwapPath + }); + + // TODO: Using Node code + childProcess.execSync(`sudo rm -rf ${hotSwapPath}`, { + pwd: path.join(process.cwd(), func.path) + }); + await dockerStart(func, port); } } catch(err) { From 4ca540239de42abeacc4b68dcf18f665d7cc837a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Jun 2024 13:43:13 +0000 Subject: [PATCH 04/11] Finish run function Docker implementation --- templates/cli/lib/commands/run.js.twig | 283 ++++++++++++++++++------- templates/cli/lib/questions.js.twig | 8 +- templates/cli/package.json.twig | 3 +- 3 files changed, 215 insertions(+), 79 deletions(-) diff --git a/templates/cli/lib/commands/run.js.twig b/templates/cli/lib/commands/run.js.twig index ea2b4c16d..042b12cc3 100644 --- a/templates/cli/lib/commands/run.js.twig +++ b/templates/cli/lib/commands/run.js.twig @@ -1,3 +1,5 @@ +const Tail = require('tail').Tail; +const EventEmitter = require('node:events'); const ignore = require("ignore"); const tar = require("tar"); const fs = require("fs"); @@ -8,10 +10,10 @@ const path = require("path"); const { Command } = require("commander"); const { localConfig, globalConfig } = require("../config"); const { paginate } = require('../paginate'); +const { functionsListVariables } = require('./functions'); const { questionsRunFunctions } = require("../questions"); const { actionRunner, success, log, error, commandDescriptions, drawTable } = require("../parser"); const { systemHasCommand, isPortTaken, getAllFiles } = require('../utils'); -const { info } = require('console'); const activeDockerIds = {}; @@ -29,6 +31,33 @@ const systemTools = { // TODO: Add all runtime needs }; +// TODO: Logs dont work + +const Queue = { + files: [], + locked: false, + events: new EventEmitter(), + push(file) { + if(!this.files.includes(file)) { + this.files.push(file); + } + + if(!this.locked) { + this.events.emit('reload', { files: this.files }); + } + }, + lock() { + this.files = []; + this.locked = true; + }, + unlock() { + this.locked = false; + if(this.files.length > 0) { + this.events.emit('reload', { files: this.files }); + } + } +}; + async function dockerStop(id) { delete activeDockerIds[id]; const stopProcess = childProcess.spawn('docker', ['rm', '--force', id], { @@ -57,7 +86,7 @@ async function dockerPull(func) { await new Promise((res) => { pullProcess.on('close', res) }); } -async function dockerBuild(func) { +async function dockerBuild(func, variables) { log('Building function using Docker engine ...'); const [ runtimeName, runtimeVersion ] = func.runtime.split('-', 2); @@ -66,7 +95,20 @@ async function dockerBuild(func) { const functionDir = path.join(process.cwd(), func.path); const id = `${new Date().getTime().toString(16)}${Math.round(Math.random() * 1000000000).toString(16)}`; - const params = ['run', '--rm', '--name', id, '-i', '-e', `OPEN_RUNTIMES_ENTRYPOINT=${func.entrypoint}`, '-v', `${functionDir}/:/mnt/code:rw`, imageName, 'sh', '-c', ` helpers/build.sh "${func.commands}"`]; + + const params = [ 'run' ]; + params.push('--name', id); + params.push('-v', `${functionDir}/:/mnt/code:rw`); + params.push('-e', 'APPWRITE_ENV=development'); + params.push('-e', 'OPEN_RUNTIMES_ENV=development'); + params.push('-e', 'OPEN_RUNTIMES_SECRET='); + params.push('-e', `OPEN_RUNTIMES_ENTRYPOINT=${func.entrypoint}`); + + for(const v of variables) { + params.push('-e', `${v.key}=${v.value}`); + } + + params.push(imageName, 'sh', '-c', `helpers/build.sh "${func.commands}"`); const buildProcess = childProcess.spawn('docker', params, { stdio: 'pipe', @@ -81,14 +123,37 @@ async function dockerBuild(func) { process.stderr.write(`\n${data}`); }); - activeDockerIds[id] = true; - await new Promise((res) => { buildProcess.on('close', res) }); + const copyPath = path.join(process.cwd(), func.path, '.appwrite', 'build.tar.gz'); + const copyDir = path.dirname(copyPath); + if (!fs.existsSync(copyDir)) { + fs.mkdirSync(copyDir, { recursive: true }); + } + + const copyProcess = childProcess.spawn('docker', ['cp', `${id}:/mnt/code/code.tar.gz`, copyPath], { + stdio: 'pipe', + pwd: functionDir + }); + + await new Promise((res) => { copyProcess.on('close', res) }); + + const cleanupProcess = childProcess.spawn('docker', ['rm', '--force', id], { + stdio: 'pipe', + pwd: functionDir + }); + + await new Promise((res) => { cleanupProcess.on('close', res) }); + delete activeDockerIds[id]; + + const tempPath = path.join(process.cwd(), func.path, 'code.tar.gz'); + if (fs.existsSync(tempPath)) { + fs.rmSync(tempPath, { force: true }); + } } -async function dockerStart(func, port) { +async function dockerStart(func, variables, port) { log('Starting function using Docker engine ...'); log("Permissions, events, CRON and timeouts dont apply when running locally."); @@ -105,24 +170,51 @@ async function dockerStart(func, port) { const functionDir = path.join(process.cwd(), func.path); const id = `${new Date().getTime().toString(16)}${Math.round(Math.random() * 1000000000).toString(16)}`; - const params = ['run', '--rm', '--name', id, '-i', '-e', 'OPEN_RUNTIMES_SECRET=', '-p', `${port}:3000`, '-v', `${functionDir}/.appwrite/logs:/mnt/logs:rw`, '-v', `${functionDir}/:/mnt/code:rw`, imageName, 'sh', '-c', ` helpers/start.sh "${tool.startCommand}"`]; - const execProcess = childProcess.spawn('docker', params, { + const params = [ 'run' ]; + params.push('--rm'); + params.push('-d'); + params.push('--name', id); + params.push('-p', `${port}:3000`); + params.push('-e', 'APPWRITE_ENV=development'); + params.push('-e', 'OPEN_RUNTIMES_ENV=development'); + params.push('-e', 'OPEN_RUNTIMES_SECRET='); + + for(const v of variables) { + params.push('-e', `${v.key}=${v.value}`); + } + + params.push('-v', `${functionDir}/.appwrite/logs.txt:/mnt/logs/dev_logs.log:rw`); + params.push('-v', `${functionDir}/.appwrite/errors.txt:/mnt/logs/dev_errors.log:rw`); + params.push('-v', `${functionDir}/.appwrite/build.tar.gz:/mnt/code/code.tar.gz:ro`); + params.push(imageName, 'sh', '-c', `helpers/start.sh "${tool.startCommand}"`); + + childProcess.spawn('docker', params, { stdio: 'pipe', pwd: functionDir }); - // TODO: Find a way to see context.log + activeDockerIds[id] = true; +} - execProcess.stdout.on('data', (data) => { - process.stdout.write(`\n${data}`); - }); - - execProcess.stderr.on('data', (data) => { - process.stderr.write(`\n${data}`); - }); +async function dockerCleanup() { + const ids = Object.keys(activeDockerIds); + for await (const id of ids) { + await dockerStop(id); + } - activeDockerIds[id] = true; + const functions = localConfig.getFunctions(); + for(const func of functions) { + const appwritePath = path.join(process.cwd(), func.path, '.appwrite'); + if (fs.existsSync(appwritePath)) { + fs.rmSync(appwritePath, { recursive: true, force: true }); + } + + const tempPath = path.join(process.cwd(), func.path, 'code.tar.gz'); + if (fs.existsSync(tempPath)) { + fs.rmSync(tempPath, { force: true }); + } + } } const runFunction = async ({ port, engine, functionId } = {}) => { @@ -197,48 +289,80 @@ const runFunction = async ({ port, engine, functionId } = {}) => { path: func.path, commands: func.commands, }; + log("Local function configuration:"); drawTable([settings]); log('If you wish to change local settings, update appwrite.json file and rerun the command. To deploy the function, run: appwrite push function'); - process.on('SIGINT', () => { - for(const id in activeDockerIds) { - dockerStop(id); - } + await dockerCleanup(); + process.on('SIGINT', async () => { + log('Cleaning up ...'); + await dockerCleanup(); + success(); process.exit(); }); if(engine === "docker") { - await dockerPull(func); - await dockerBuild(func); - await dockerStart(func, port); + const logsPath = path.join(process.cwd(), func.path, '.appwrite/logs.txt'); + const errorsPath = path.join(process.cwd(), func.path, '.appwrite/errors.txt'); + + if(!fs.existsSync(path.dirname(logsPath))) { + fs.mkdirSync(path.dirname(logsPath), { recursive: true }); + } + + if (!fs.existsSync(logsPath)) { + fs.writeFileSync(logsPath, ''); + } - let watcherRunning = false; + if (!fs.existsSync(errorsPath)) { + fs.writeFileSync(errorsPath, ''); + } - childProcess.execSync(`sudo mkdir -p ${path.join(process.cwd(), func.path, '.appwrite/logs')}`, { - pwd: path.join(process.cwd(), func.path) + let variables = []; + if (globalConfig.getEndpoint() === '' || globalConfig.getCookie() === '') { + // TODO: Flag to disable + error("No user is signed in. To sign in, run: appwrite login. Function will run locally, but will not have your function's environment variables set."); + } else { + const { variables: remoteVariables } = await paginate(functionsListVariables, { + functionId: func['$id'], + parseOutput: false + }, 100, 'variables'); + + remoteVariables.forEach((v) => { + variables.push({ + key: v.key, + value: v.value + }); + }); + } + + await dockerPull(func); + await dockerBuild(func, variables); + await dockerStart(func, variables, port); + + new Tail(logsPath).on("line", function(data) { + console.log(data); }); - chokidar.watch('.appwrite/logs', { - cwd: path.join(process.cwd(), func.path), - ignoreInitial: true, - }).on('all', async (event, filePath) => { - console.log(fs.readFileSync(path.join(process.cwd(), func.path, filePath)).toString()); + new Tail(errorsPath).on("line", function(data) { + console.log(data); }); chokidar.watch('.', { cwd: path.join(process.cwd(), func.path), ignoreInitial: true, ignored: [ ...(func.ignore ?? []), 'code.tar.gz', '.appwrite', '.appwrite/', '.appwrite/*', '.appwrite/**', '.appwrite/*.*', '.appwrite/**/*.*' ] - }).on('all', async (event, filePath) => { - if(watcherRunning) { - info("File change detected but ignored, because live reload is already being ran."); - return; - } + }).on('all', async (_event, filePath) => { + Queue.push(filePath); + }); - watcherRunning = true; + Queue.events.on('reload', async ({ files }) => { + Queue.lock(); - log('Detected a change in ' + filePath); + log('Live-reloading due to file changes: '); + for(const file of files) { + log(`- ${file}`); + } try { log('Stopping the function ...'); @@ -246,70 +370,73 @@ const runFunction = async ({ port, engine, functionId } = {}) => { for(const id in activeDockerIds) { await dockerStop(id); } - - if(tool.isCompiled || tool.dependencyFiles.includes(filePath)) { - await dockerBuild(func); - await dockerStart(func, port); + + const dependencyFile = files.find((filePath) => tool.dependencyFiles.includes(filePath)); + if(tool.isCompiled || dependencyFile) { + log(`Rebuilding the function due to cange in ${dependencyFile} ...`); + await dockerBuild(func, variables); + await dockerStart(func, variables, port); } else { - // TODO: Some try-catch approach, to rebuild if fails - log('Hot swapping function files ...'); + log('Hot-swapping function files ...'); const functionPath = path.join(process.cwd(), func.path); const hotSwapPath = path.join(functionPath, '.appwrite/hot-swap'); - const buildPath = path.join(functionPath, 'code.tar.gz') + const buildPath = path.join(functionPath, '.appwrite/build.tar.gz'); + + // Prepare temp folder + if (!fs.existsSync(hotSwapPath)) { + fs.mkdirSync(hotSwapPath, { recursive: true }); + } else { + fs.rmSync(hotSwapPath, { recursive: true, force: true }); + fs.mkdirSync(hotSwapPath, { recursive: true }); + } - // TODO: Using Node code, no sudo - childProcess.execSync(`sudo mkdir -p ${hotSwapPath} && sudo chmod 777 ${buildPath} ${hotSwapPath} && sudo tar -zxf ${buildPath} -C ${hotSwapPath}`, { - pwd: path.join(process.cwd(), func.path) - }); + await tar + .extract({ + gzip: true, + sync: true, + cwd: hotSwapPath, + file: buildPath + }); const ignorer = ignore(); ignorer.add('.appwrite'); - if (func.ignore) { ignorer.add(func.ignore); } - // TODO: Better approach const filesToCopy = getAllFiles(functionPath).map((file) => path.relative(functionPath, file)).filter((file) => !ignorer.ignores(file)); - - const copyCommands = []; for(const f of filesToCopy) { const filePath = path.join(hotSwapPath, f); - copyCommands.push(`sudo rm -rf ${filePath}`); + if (fs.existsSync(filePath)) { + fs.rmSync(filePath, { force: true }); + } const fileDir = path.dirname(filePath); - copyCommands.push(`sudo mkdir -p ${fileDir}`); + if (!fs.existsSync(fileDir)) { + fs.mkdirSync(fileDir, { recursive: true }); + } const sourcePath = path.join(functionPath, f); - copyCommands.push(`sudo cp ${sourcePath} ${filePath}`); + fs.copyFileSync(sourcePath, filePath); } - childProcess.execSync(copyCommands.join(" && "), { - pwd: path.join(process.cwd(), func.path) - }); - - console.log(path.join(process.cwd(), func.path, '.appwrite')); - /* - childProcess.execSync("sudo chmod -R 777 .", { - pwd: path.join(process.cwd(), func.path, '.appwrite') - }); - */ - - childProcess.execSync(`sudo tar -C ${hotSwapPath} --exclude code.tar.gz -zcf ${buildPath} .`, { - pwd: hotSwapPath - }); + + await tar + .create({ + gzip: true, + sync: true, + cwd: hotSwapPath, + file: buildPath + }, ['.']); - // TODO: Using Node code - childProcess.execSync(`sudo rm -rf ${hotSwapPath}`, { - pwd: path.join(process.cwd(), func.path) - }); + fs.rmSync(hotSwapPath, { recursive: true, force: true }); - await dockerStart(func, port); + await dockerStart(func, variables, port); } } catch(err) { console.error(err); } finally { - watcherRunning = false; + Queue.unlock(); } }); } @@ -332,6 +459,8 @@ run .option(`--functionId `, `Function ID`) .option(`--port `, `Local port`) .option(`--engine `, `Local engine, "system" or "docker"`) + // TODO: Option to disable auto reloading + // TODO: Option to enable write operations to function folder .action(actionRunner(runFunction)); module.exports = { diff --git a/templates/cli/lib/questions.js.twig b/templates/cli/lib/questions.js.twig index 694e34dd5..17b77bcd6 100644 --- a/templates/cli/lib/questions.js.twig +++ b/templates/cli/lib/questions.js.twig @@ -537,7 +537,7 @@ const questionsRunFunctions = [ } }, { - type: "number", + type: "input", name: "port", message: 'Which port would you like function to listen on?', default: async () => { @@ -557,6 +557,12 @@ const questionsRunFunctions = [ const done = this.async(); (async () => { + if (typeof value !== 'number' && isNaN(+value)) { + throw Error(`You need to provide a number.`); + } + + value = +value; + const taken = await isPortTaken(value); if(taken) { diff --git a/templates/cli/package.json.twig b/templates/cli/package.json.twig index 09d6ca0ba..c683d0710 100644 --- a/templates/cli/package.json.twig +++ b/templates/cli/package.json.twig @@ -32,7 +32,8 @@ "inquirer": "^8.2.4", "tar": "^6.1.11", "ignore": "^5.2.0", - "chokidar": "^3.6.0" + "chokidar": "^3.6.0", + "tail": "^2.2.6" }, "devDependencies": { "pkg": "5.8.1" From 93ea80a89f21668c86279d6620927cdc65404e02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 3 Jun 2024 16:31:31 +0000 Subject: [PATCH 05/11] Add tags to dev command --- templates/cli/lib/commands/run.js.twig | 53 +++++++++++++------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/templates/cli/lib/commands/run.js.twig b/templates/cli/lib/commands/run.js.twig index 042b12cc3..68e1ee04d 100644 --- a/templates/cli/lib/commands/run.js.twig +++ b/templates/cli/lib/commands/run.js.twig @@ -31,8 +31,6 @@ const systemTools = { // TODO: Add all runtime needs }; -// TODO: Logs dont work - const Queue = { files: [], locked: false, @@ -217,7 +215,7 @@ async function dockerCleanup() { } } -const runFunction = async ({ port, engine, functionId } = {}) => { +const runFunction = async ({ port, engine, functionId, noVariables, noReload } = {}) => { // Selection if(!functionId) { const answers = await inquirer.prompt(questionsRunFunctions[0]); @@ -320,21 +318,22 @@ const runFunction = async ({ port, engine, functionId } = {}) => { } let variables = []; - if (globalConfig.getEndpoint() === '' || globalConfig.getCookie() === '') { - // TODO: Flag to disable - error("No user is signed in. To sign in, run: appwrite login. Function will run locally, but will not have your function's environment variables set."); - } else { - const { variables: remoteVariables } = await paginate(functionsListVariables, { - functionId: func['$id'], - parseOutput: false - }, 100, 'variables'); - - remoteVariables.forEach((v) => { - variables.push({ - key: v.key, - value: v.value + if(!noVariables) { + if (globalConfig.getEndpoint() === '' || globalConfig.getCookie() === '') { + error("No user is signed in. To sign in, run: appwrite login. Function will run locally, but will not have your function's environment variables set."); + } else { + const { variables: remoteVariables } = await paginate(functionsListVariables, { + functionId: func['$id'], + parseOutput: false + }, 100, 'variables'); + + remoteVariables.forEach((v) => { + variables.push({ + key: v.key, + value: v.value + }); }); - }); + } } await dockerPull(func); @@ -348,13 +347,15 @@ const runFunction = async ({ port, engine, functionId } = {}) => { console.log(data); }); - chokidar.watch('.', { - cwd: path.join(process.cwd(), func.path), - ignoreInitial: true, - ignored: [ ...(func.ignore ?? []), 'code.tar.gz', '.appwrite', '.appwrite/', '.appwrite/*', '.appwrite/**', '.appwrite/*.*', '.appwrite/**/*.*' ] - }).on('all', async (_event, filePath) => { - Queue.push(filePath); - }); + if(!noReload) { + chokidar.watch('.', { + cwd: path.join(process.cwd(), func.path), + ignoreInitial: true, + ignored: [ ...(func.ignore ?? []), 'code.tar.gz', '.appwrite', '.appwrite/', '.appwrite/*', '.appwrite/**', '.appwrite/*.*', '.appwrite/**/*.*' ] + }).on('all', async (_event, filePath) => { + Queue.push(filePath); + }); + } Queue.events.on('reload', async ({ files }) => { Queue.lock(); @@ -459,8 +460,8 @@ run .option(`--functionId `, `Function ID`) .option(`--port `, `Local port`) .option(`--engine `, `Local engine, "system" or "docker"`) - // TODO: Option to disable auto reloading - // TODO: Option to enable write operations to function folder + .option(`--noVariables`, `Prevent pulling variables from function settings`) + .option(`--noReload`, `Prevent live reloading of server when changes are made to function files`) .action(actionRunner(runFunction)); module.exports = { From 73e3c407828f9280fa6a5761a8616a75c3dc5b17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 5 Jun 2024 13:31:06 +0000 Subject: [PATCH 06/11] Add env var and headers support to local development --- templates/cli/lib/commands/run.js.twig | 131 ++++++++++++++++++++----- templates/cli/lib/config.js.twig | 3 +- templates/cli/lib/questions.js.twig | 6 +- 3 files changed, 110 insertions(+), 30 deletions(-) diff --git a/templates/cli/lib/commands/run.js.twig b/templates/cli/lib/commands/run.js.twig index 68e1ee04d..ccf609327 100644 --- a/templates/cli/lib/commands/run.js.twig +++ b/templates/cli/lib/commands/run.js.twig @@ -3,6 +3,7 @@ const EventEmitter = require('node:events'); const ignore = require("ignore"); const tar = require("tar"); const fs = require("fs"); +const ID = require("../id"); const childProcess = require('child_process'); const chokidar = require('chokidar'); const inquirer = require("inquirer"); @@ -11,6 +12,8 @@ const { Command } = require("commander"); const { localConfig, globalConfig } = require("../config"); const { paginate } = require('../paginate'); const { functionsListVariables } = require('./functions'); +const { usersGet, usersCreateJWT } = require('./users'); +const { projectsCreateJWT } = require('./projects'); const { questionsRunFunctions } = require("../questions"); const { actionRunner, success, log, error, commandDescriptions, drawTable } = require("../parser"); const { systemHasCommand, isPortTaken, getAllFiles } = require('../utils'); @@ -31,17 +34,67 @@ const systemTools = { // TODO: Add all runtime needs }; +const JwtManager = { + userJwt: null, + functionJwt: null, + + timerWarn: null, + timerError: null, + + async setup(userId = null) { + if(this.timerWarn) { + clearTimeout(this.timerWarn); + } + + if(this.timerError) { + clearTimeout(this.timerError); + } + + this.timerWarn = setTimeout(() => { + log("Warning: Authorized JWT will expire in 5 minutes. Please stop and re-run the command to refresh tokens for 1 hour."); + }, 1000 * 60 * 55); // 55 mins + + this.timerError = setTimeout(() => { + log("Warning: Authorized JWT just expired. Please stop and re-run the command to obtain new tokens with 1 hour validity."); + log("Some Appwrite API communication is not authorized now.") + }, 1000 * 60 * 60); // 60 mins + + if(userId) { + await usersGet({ + userId, + parseOutput: false + }); + const userResponse = await usersCreateJWT({ + userId, + duration: 60*60, + parseOutput: false + }); + this.userJwt = userResponse.jwt; + } + + const functionResponse = await projectsCreateJWT({ + projectId: localConfig.getProject().projectId, + // TODO: There must be better way to get the list + scopes: ["sessions.write","users.read","users.write","teams.read","teams.write","databases.read","databases.write","collections.read","collections.write","attributes.read","attributes.write","indexes.read","indexes.write","documents.read","documents.write","files.read","files.write","buckets.read","buckets.write","functions.read","functions.write","execution.read","execution.write","locale.read","avatars.read","health.read","providers.read","providers.write","messages.read","messages.write","topics.read","topics.write","subscribers.read","subscribers.write","targets.read","targets.write","rules.read","rules.write","migrations.read","migrations.write","vcs.read","vcs.write","assistant.read"], + duration: 60*60, + parseOutput: false + }); + this.functionJwt = functionResponse.jwt; + } +}; + const Queue = { files: [], locked: false, events: new EventEmitter(), + debounce: null, push(file) { if(!this.files.includes(file)) { this.files.push(file); } if(!this.locked) { - this.events.emit('reload', { files: this.files }); + this._trigger(); } }, lock() { @@ -51,13 +104,23 @@ const Queue = { unlock() { this.locked = false; if(this.files.length > 0) { - this.events.emit('reload', { files: this.files }); + this._trigger(); } + }, + _trigger() { + if(this.debounce) { + return; + } + + this.debounce = setTimeout(() => { + this.events.emit('reload', { files: this.files }); + this.debounce = null; + }, 300); } }; async function dockerStop(id) { - delete activeDockerIds[id]; + delete activeDockerIds[id]; const stopProcess = childProcess.spawn('docker', ['rm', '--force', id], { stdio: 'pipe', }); @@ -92,7 +155,7 @@ async function dockerBuild(func, variables) { const functionDir = path.join(process.cwd(), func.path); - const id = `${new Date().getTime().toString(16)}${Math.round(Math.random() * 1000000000).toString(16)}`; + const id = ID.unique(); const params = [ 'run' ]; params.push('--name', id); @@ -102,8 +165,8 @@ async function dockerBuild(func, variables) { params.push('-e', 'OPEN_RUNTIMES_SECRET='); params.push('-e', `OPEN_RUNTIMES_ENTRYPOINT=${func.entrypoint}`); - for(const v of variables) { - params.push('-e', `${v.key}=${v.value}`); + for(const k of Object.keys(variables)) { + params.push('-e', `${k}=${variables[k]}`); } params.push(imageName, 'sh', '-c', `helpers/build.sh "${func.commands}"`); @@ -167,7 +230,7 @@ async function dockerStart(func, variables, port) { const functionDir = path.join(process.cwd(), func.path); - const id = `${new Date().getTime().toString(16)}${Math.round(Math.random() * 1000000000).toString(16)}`; + const id = ID.unique(); const params = [ 'run' ]; params.push('--rm'); @@ -178,8 +241,8 @@ async function dockerStart(func, variables, port) { params.push('-e', 'OPEN_RUNTIMES_ENV=development'); params.push('-e', 'OPEN_RUNTIMES_SECRET='); - for(const v of variables) { - params.push('-e', `${v.key}=${v.value}`); + for(const k of Object.keys(variables)) { + params.push('-e', `${k}=${variables[k]}`); } params.push('-v', `${functionDir}/.appwrite/logs.txt:/mnt/logs/dev_logs.log:rw`); @@ -215,7 +278,7 @@ async function dockerCleanup() { } } -const runFunction = async ({ port, engine, functionId, noVariables, noReload } = {}) => { +const runFunction = async ({ port, engine, functionId, noVariables, noReload, userId } = {}) => { // Selection if(!functionId) { const answers = await inquirer.prompt(questionsRunFunctions[0]); @@ -265,14 +328,10 @@ const runFunction = async ({ port, engine, functionId, noVariables, noReload } = } if(engine === 'docker') { - log('💡 Hint: Using system is faster, but using Docker simulates the production environment precisely.'); - if(!systemHasCommand('docker')) { return error("Please install Docker first: https://docs.docker.com/engine/install/"); } } else if(engine === 'system') { - log('💡 Hint: Docker simulates the production environment precisely, but using system is faster'); - for(const command of tool.commands) { if(!systemHasCommand(command.command)) { return error(`Your system is missing command "${command.command}". Please install it first: ${command.docs}`); @@ -317,25 +376,45 @@ const runFunction = async ({ port, engine, functionId, noVariables, noReload } = fs.writeFileSync(errorsPath, ''); } - let variables = []; + const variables = {}; if(!noVariables) { if (globalConfig.getEndpoint() === '' || globalConfig.getCookie() === '') { error("No user is signed in. To sign in, run: appwrite login. Function will run locally, but will not have your function's environment variables set."); } else { - const { variables: remoteVariables } = await paginate(functionsListVariables, { - functionId: func['$id'], - parseOutput: false - }, 100, 'variables'); - - remoteVariables.forEach((v) => { - variables.push({ - key: v.key, - value: v.value + try { + const { variables: remoteVariables } = await paginate(functionsListVariables, { + functionId: func['$id'], + parseOutput: false + }, 100, 'variables'); + + remoteVariables.forEach((v) => { + variables[v.key] = v.value; }); - }); + } catch(err) { + error("Could not fetch remote variables: " + err.message); + error("Function will run locally, but will not have your function's environment variables set."); + } } } + variables['APPWRITE_FUNCTION_API_ENDPOINT'] = globalConfig.getFrom('endpoint'); + variables['APPWRITE_FUNCTION_ID'] = func.$id; + variables['APPWRITE_FUNCTION_NAME'] = func.name; + variables['APPWRITE_FUNCTION_DEPLOYMENT'] = ''; // TODO: Implement when relevant + variables['APPWRITE_FUNCTION_PROJECT_ID'] = localConfig.getProject().projectId; + variables['APPWRITE_FUNCTION_RUNTIME_NAME'] = ''; // TODO: Implement when relevant + variables['APPWRITE_FUNCTION_RUNTIME_VERSION'] = ''; // TODO: Implement when relevant + + await JwtManager.setup(userId); + + const headers = {}; + headers['x-appwrite-key'] = JwtManager.functionJwt ?? ''; + headers['x-appwrite-trigger'] = 'http'; + headers['x-appwrite-event'] = ''; + headers['x-appwrite-user-id'] = userId ?? ''; + headers['x-appwrite-user-jwt'] = JwtManager.userJwt ?? ''; + variables['OPEN_RUNTIMES_HEADERS'] = JSON.stringify(headers); + await dockerPull(func); await dockerBuild(func, variables); await dockerStart(func, variables, port); @@ -348,6 +427,7 @@ const runFunction = async ({ port, engine, functionId, noVariables, noReload } = }); if(!noReload) { + // TODO: Stop previous job mid-way if new deployment is ready, I think? chokidar.watch('.', { cwd: path.join(process.cwd(), func.path), ignoreInitial: true, @@ -460,6 +540,7 @@ run .option(`--functionId `, `Function ID`) .option(`--port `, `Local port`) .option(`--engine `, `Local engine, "system" or "docker"`) + .option(`--userId `, `ID of user to impersonate`) .option(`--noVariables`, `Prevent pulling variables from function settings`) .option(`--noReload`, `Prevent live reloading of server when changes are made to function files`) .action(actionRunner(runFunction)); diff --git a/templates/cli/lib/config.js.twig b/templates/cli/lib/config.js.twig index bcf48dccf..b416305eb 100644 --- a/templates/cli/lib/config.js.twig +++ b/templates/cli/lib/config.js.twig @@ -419,7 +419,7 @@ class Global extends Config { return this.get(Global.PREFERENCE_CURRENT); } - setCurrentLogin(endpoint) { + setCurrentLogin(id) { this.set(Global.PREFERENCE_CURRENT, endpoint); } @@ -516,7 +516,6 @@ class Global extends Config { this.setTo(Global.PREFERENCE_KEY, key); } - hasFrom(key) { const current = this.getCurrentLogin(); diff --git a/templates/cli/lib/questions.js.twig b/templates/cli/lib/questions.js.twig index 7e819145c..6befd56c0 100644 --- a/templates/cli/lib/questions.js.twig +++ b/templates/cli/lib/questions.js.twig @@ -820,11 +820,11 @@ const questionsRunFunctions = [ message: "Which engine would you like to use?", choices: [ { - name: "Docker", - value: "docker", + name: "Docker (recommended, simulates production precisely)", + value: "docker", }, { - name: "System", + name: "System (faster and easier to debug)", value: "system", }, ], From c024a8193960ba9f21a0484e828b29dcfc1bc0f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 12 Jun 2024 08:57:53 +0000 Subject: [PATCH 07/11] Finish local development --- templates/cli/lib/commands/run.js.twig | 393 ++++++++++++++----------- templates/cli/lib/questions.js.twig | 24 +- 2 files changed, 227 insertions(+), 190 deletions(-) diff --git a/templates/cli/lib/commands/run.js.twig b/templates/cli/lib/commands/run.js.twig index ccf609327..88539792c 100644 --- a/templates/cli/lib/commands/run.js.twig +++ b/templates/cli/lib/commands/run.js.twig @@ -21,17 +21,81 @@ const { systemHasCommand, isPortTaken, getAllFiles } = require('../utils'); const activeDockerIds = {}; const openRuntimesVersion = 'v3'; +const runtimeNames = { + 'node': 'Node.js', + 'php': 'PHP', + 'ruby': 'Ruby', + 'python': 'Python', + 'python-ml': 'Python (ML)', + 'deno': 'Deno', + 'dart': 'Dart', + 'dotnet': '.NET', + 'java': 'Java', + 'swift': 'Swift', + 'kotlin': 'Kotlin', + 'bun': 'Bun' +}; const systemTools = { 'node': { isCompiled: false, startCommand: "node src/server.js", - commands: [ - { command: "node", docs: "https://nodejs.org/en/download/package-manager" }, - { command: "npm", docs: "https://nodejs.org/en/download/package-manager" }, - ], dependencyFiles: [ "package.json", "package-lock.json" ] }, - // TODO: Add all runtime needs + 'php': { + isCompiled: false, + startCommand: "php src/server.php", + dependencyFiles: [ "composer.json", "composer.lock" ] + }, + 'ruby': { + isCompiled: false, + startCommand: "bundle exec puma -b tcp://0.0.0.0:3000 -e production", + dependencyFiles: [ "Gemfile", "Gemfile.lock" ] + }, + 'python': { + isCompiled: false, + startCommand: "python3 src/server.py", + dependencyFiles: [ "requirements.txt", "requirements.lock" ] + }, + 'python-ml': { + isCompiled: false, + startCommand: "python3 src/server.py", + dependencyFiles: [ "requirements.txt", "requirements.lock" ] + }, + 'deno': { + isCompiled: false, + startCommand: "deno start", + dependencyFiles: [ ] + }, + 'dart': { + isCompiled: true, + startCommand: "src/function/server", + dependencyFiles: [ ] + }, + 'dotnet': { + isCompiled: true, + startCommand: "dotnet src/function/DotNetRuntime.dll", + dependencyFiles: [ ] + }, + 'java': { + isCompiled: true, + startCommand: "java -jar src/function/java-runtime-1.0.0.jar", + dependencyFiles: [ ] + }, + 'swift': { + isCompiled: true, + startCommand: "src/function/Runtime serve --env production --hostname 0.0.0.0 --port 3000", + dependencyFiles: [ ] + }, + 'kotlin': { + isCompiled: true, + startCommand: "java -jar src/function/kotlin-runtime-1.0.0.jar", + dependencyFiles: [ ] + }, + 'bun': { + isCompiled: false, + startCommand: "bun src/server.ts", + dependencyFiles: [ "package.json", "package-lock.json", "bun.lockb" ] + }, }; const JwtManager = { @@ -74,7 +138,7 @@ const JwtManager = { const functionResponse = await projectsCreateJWT({ projectId: localConfig.getProject().projectId, - // TODO: There must be better way to get the list + // TODO: Once we have endpoint for this, use it scopes: ["sessions.write","users.read","users.write","teams.read","teams.write","databases.read","databases.write","collections.read","collections.write","attributes.read","attributes.write","indexes.read","indexes.write","documents.read","documents.write","files.read","files.write","buckets.read","buckets.write","functions.read","functions.write","execution.read","execution.write","locale.read","avatars.read","health.read","providers.read","providers.write","messages.read","messages.write","topics.read","topics.write","subscribers.read","subscribers.write","targets.read","targets.write","rules.read","rules.write","migrations.read","migrations.write","vcs.read","vcs.write","assistant.read"], duration: 60*60, parseOutput: false @@ -129,10 +193,11 @@ async function dockerStop(id) { } async function dockerPull(func) { - return; // TODO: Remove log('Pulling Docker image of function runtime ...'); - const [ runtimeName, runtimeVersion ] = func.runtime.split('-', 2); + const runtimeChunks = func.runtime.split("-"); + const runtimeVersion = runtimeChunks.pop(); + const runtimeName = runtimeChunks.join("-"); const imageName = `openruntimes/${runtimeName}:${openRuntimesVersion}-${runtimeVersion}`; const pullProcess = childProcess.spawn('docker', ['pull', imageName], { @@ -148,9 +213,11 @@ async function dockerPull(func) { } async function dockerBuild(func, variables) { - log('Building function using Docker engine ...'); + log('Building function using Docker ...'); - const [ runtimeName, runtimeVersion ] = func.runtime.split('-', 2); + const runtimeChunks = func.runtime.split("-"); + const runtimeVersion = runtimeChunks.pop(); + const runtimeName = runtimeChunks.join("-"); const imageName = `openruntimes/${runtimeName}:${openRuntimesVersion}-${runtimeVersion}`; const functionDir = path.join(process.cwd(), func.path); @@ -215,7 +282,7 @@ async function dockerBuild(func, variables) { } async function dockerStart(func, variables, port) { - log('Starting function using Docker engine ...'); + log('Starting function using Docker ...'); log("Permissions, events, CRON and timeouts dont apply when running locally."); @@ -223,7 +290,10 @@ async function dockerStart(func, variables, port) { success(`Visit http://localhost:${port}/ to execute your function.`); - const [ runtimeName, runtimeVersion ] = func.runtime.split('-', 2); + + const runtimeChunks = func.runtime.split("-"); + const runtimeVersion = runtimeChunks.pop(); + const runtimeName = runtimeChunks.join("-"); const imageName = `openruntimes/${runtimeName}:${openRuntimesVersion}-${runtimeVersion}`; const tool = systemTools[runtimeName]; @@ -278,7 +348,7 @@ async function dockerCleanup() { } } -const runFunction = async ({ port, engine, functionId, noVariables, noReload, userId } = {}) => { +const runFunction = async ({ port, functionId, noVariables, noReload, userId } = {}) => { // Selection if(!functionId) { const answers = await inquirer.prompt(questionsRunFunctions[0]); @@ -291,7 +361,7 @@ const runFunction = async ({ port, engine, functionId, noVariables, noReload, us throw new Error("Function '" + functionId + "' not found.") } - const runtimeName = func.runtime.split('-')[0]; + const runtimeName = func.runtime.split("-").slice(0, -1).join("-"); const tool = systemTools[runtimeName]; // Configuration: Port @@ -318,25 +388,8 @@ const runFunction = async ({ port, engine, functionId, noVariables, noReload, us } // Configuration: Engine - if(engine !== "system" && engine !== "docker") { - engine = null; - } - - if(!engine) { - const answers = await inquirer.prompt(questionsRunFunctions[2]); - engine = answers.engine; - } - - if(engine === 'docker') { - if(!systemHasCommand('docker')) { - return error("Please install Docker first: https://docs.docker.com/engine/install/"); - } - } else if(engine === 'system') { - for(const command of tool.commands) { - if(!systemHasCommand(command.command)) { - return error(`Your system is missing command "${command.command}". Please install it first: ${command.docs}`); - } - } + if(!systemHasCommand('docker')) { + return error("Please install Docker first: https://docs.docker.com/engine/install/"); } // Settings @@ -360,167 +413,164 @@ const runFunction = async ({ port, engine, functionId, noVariables, noReload, us process.exit(); }); - if(engine === "docker") { - const logsPath = path.join(process.cwd(), func.path, '.appwrite/logs.txt'); - const errorsPath = path.join(process.cwd(), func.path, '.appwrite/errors.txt'); + const logsPath = path.join(process.cwd(), func.path, '.appwrite/logs.txt'); + const errorsPath = path.join(process.cwd(), func.path, '.appwrite/errors.txt'); - if(!fs.existsSync(path.dirname(logsPath))) { - fs.mkdirSync(path.dirname(logsPath), { recursive: true }); - } + if(!fs.existsSync(path.dirname(logsPath))) { + fs.mkdirSync(path.dirname(logsPath), { recursive: true }); + } - if (!fs.existsSync(logsPath)) { - fs.writeFileSync(logsPath, ''); - } + if (!fs.existsSync(logsPath)) { + fs.writeFileSync(logsPath, ''); + } - if (!fs.existsSync(errorsPath)) { - fs.writeFileSync(errorsPath, ''); - } + if (!fs.existsSync(errorsPath)) { + fs.writeFileSync(errorsPath, ''); + } - const variables = {}; - if(!noVariables) { - if (globalConfig.getEndpoint() === '' || globalConfig.getCookie() === '') { - error("No user is signed in. To sign in, run: appwrite login. Function will run locally, but will not have your function's environment variables set."); - } else { - try { - const { variables: remoteVariables } = await paginate(functionsListVariables, { - functionId: func['$id'], - parseOutput: false - }, 100, 'variables'); - - remoteVariables.forEach((v) => { - variables[v.key] = v.value; - }); - } catch(err) { - error("Could not fetch remote variables: " + err.message); - error("Function will run locally, but will not have your function's environment variables set."); - } + const variables = {}; + if(!noVariables) { + if (globalConfig.getEndpoint() === '' || globalConfig.getCookie() === '') { + error("No user is signed in. To sign in, run: appwrite login. Function will run locally, but will not have your function's environment variables set."); + } else { + try { + const { variables: remoteVariables } = await paginate(functionsListVariables, { + functionId: func['$id'], + parseOutput: false + }, 100, 'variables'); + + remoteVariables.forEach((v) => { + variables[v.key] = v.value; + }); + } catch(err) { + error("Could not fetch remote variables: " + err.message); + error("Function will run locally, but will not have your function's environment variables set."); } } + } - variables['APPWRITE_FUNCTION_API_ENDPOINT'] = globalConfig.getFrom('endpoint'); - variables['APPWRITE_FUNCTION_ID'] = func.$id; - variables['APPWRITE_FUNCTION_NAME'] = func.name; - variables['APPWRITE_FUNCTION_DEPLOYMENT'] = ''; // TODO: Implement when relevant - variables['APPWRITE_FUNCTION_PROJECT_ID'] = localConfig.getProject().projectId; - variables['APPWRITE_FUNCTION_RUNTIME_NAME'] = ''; // TODO: Implement when relevant - variables['APPWRITE_FUNCTION_RUNTIME_VERSION'] = ''; // TODO: Implement when relevant - - await JwtManager.setup(userId); - - const headers = {}; - headers['x-appwrite-key'] = JwtManager.functionJwt ?? ''; - headers['x-appwrite-trigger'] = 'http'; - headers['x-appwrite-event'] = ''; - headers['x-appwrite-user-id'] = userId ?? ''; - headers['x-appwrite-user-jwt'] = JwtManager.userJwt ?? ''; - variables['OPEN_RUNTIMES_HEADERS'] = JSON.stringify(headers); - - await dockerPull(func); - await dockerBuild(func, variables); - await dockerStart(func, variables, port); - - new Tail(logsPath).on("line", function(data) { - console.log(data); - }); - new Tail(errorsPath).on("line", function(data) { - console.log(data); + variables['APPWRITE_FUNCTION_API_ENDPOINT'] = globalConfig.getFrom('endpoint'); + variables['APPWRITE_FUNCTION_ID'] = func.$id; + variables['APPWRITE_FUNCTION_NAME'] = func.name; + variables['APPWRITE_FUNCTION_DEPLOYMENT'] = ''; // TODO: Implement when relevant + variables['APPWRITE_FUNCTION_PROJECT_ID'] = localConfig.getProject().projectId; + variables['APPWRITE_FUNCTION_RUNTIME_NAME'] = runtimeNames[runtimeName] ?? ''; + variables['APPWRITE_FUNCTION_RUNTIME_VERSION'] = func.runtime; + + await JwtManager.setup(userId); + + const headers = {}; + headers['x-appwrite-key'] = JwtManager.functionJwt ?? ''; + headers['x-appwrite-trigger'] = 'http'; + headers['x-appwrite-event'] = ''; + headers['x-appwrite-user-id'] = userId ?? ''; + headers['x-appwrite-user-jwt'] = JwtManager.userJwt ?? ''; + variables['OPEN_RUNTIMES_HEADERS'] = JSON.stringify(headers); + + await dockerPull(func); + await dockerBuild(func, variables); + await dockerStart(func, variables, port); + + new Tail(logsPath).on("line", function(data) { + console.log(data); + }); + new Tail(errorsPath).on("line", function(data) { + console.log(data); + }); + + if(!noReload) { + chokidar.watch('.', { + cwd: path.join(process.cwd(), func.path), + ignoreInitial: true, + ignored: [ ...(func.ignore ?? []), 'code.tar.gz', '.appwrite', '.appwrite/', '.appwrite/*', '.appwrite/**', '.appwrite/*.*', '.appwrite/**/*.*' ] + }).on('all', async (_event, filePath) => { + Queue.push(filePath); }); + } - if(!noReload) { - // TODO: Stop previous job mid-way if new deployment is ready, I think? - chokidar.watch('.', { - cwd: path.join(process.cwd(), func.path), - ignoreInitial: true, - ignored: [ ...(func.ignore ?? []), 'code.tar.gz', '.appwrite', '.appwrite/', '.appwrite/*', '.appwrite/**', '.appwrite/*.*', '.appwrite/**/*.*' ] - }).on('all', async (_event, filePath) => { - Queue.push(filePath); - }); + Queue.events.on('reload', async ({ files }) => { + Queue.lock(); + + log('Live-reloading due to file changes: '); + for(const file of files) { + log(`- ${file}`); } - Queue.events.on('reload', async ({ files }) => { - Queue.lock(); + try { + log('Stopping the function ...'); - log('Live-reloading due to file changes: '); - for(const file of files) { - log(`- ${file}`); + for(const id in activeDockerIds) { + await dockerStop(id); } - try { - log('Stopping the function ...'); - - for(const id in activeDockerIds) { - await dockerStop(id); - } - - const dependencyFile = files.find((filePath) => tool.dependencyFiles.includes(filePath)); - if(tool.isCompiled || dependencyFile) { - log(`Rebuilding the function due to cange in ${dependencyFile} ...`); - await dockerBuild(func, variables); - await dockerStart(func, variables, port); + const dependencyFile = files.find((filePath) => tool.dependencyFiles.includes(filePath)); + if(tool.isCompiled || dependencyFile) { + log(`Rebuilding the function due to cange in ${dependencyFile} ...`); + await dockerBuild(func, variables); + await dockerStart(func, variables, port); + } else { + log('Hot-swapping function files ...'); + + const functionPath = path.join(process.cwd(), func.path); + const hotSwapPath = path.join(functionPath, '.appwrite/hot-swap'); + const buildPath = path.join(functionPath, '.appwrite/build.tar.gz'); + + // Prepare temp folder + if (!fs.existsSync(hotSwapPath)) { + fs.mkdirSync(hotSwapPath, { recursive: true }); } else { - log('Hot-swapping function files ...'); - - const functionPath = path.join(process.cwd(), func.path); - const hotSwapPath = path.join(functionPath, '.appwrite/hot-swap'); - const buildPath = path.join(functionPath, '.appwrite/build.tar.gz'); - - // Prepare temp folder - if (!fs.existsSync(hotSwapPath)) { - fs.mkdirSync(hotSwapPath, { recursive: true }); - } else { - fs.rmSync(hotSwapPath, { recursive: true, force: true }); - fs.mkdirSync(hotSwapPath, { recursive: true }); - } + fs.rmSync(hotSwapPath, { recursive: true, force: true }); + fs.mkdirSync(hotSwapPath, { recursive: true }); + } - await tar - .extract({ - gzip: true, - sync: true, - cwd: hotSwapPath, - file: buildPath - }); - - const ignorer = ignore(); - ignorer.add('.appwrite'); - if (func.ignore) { - ignorer.add(func.ignore); - } + await tar + .extract({ + gzip: true, + sync: true, + cwd: hotSwapPath, + file: buildPath + }); - const filesToCopy = getAllFiles(functionPath).map((file) => path.relative(functionPath, file)).filter((file) => !ignorer.ignores(file)); - for(const f of filesToCopy) { - const filePath = path.join(hotSwapPath, f); - if (fs.existsSync(filePath)) { - fs.rmSync(filePath, { force: true }); - } + const ignorer = ignore(); + ignorer.add('.appwrite'); + if (func.ignore) { + ignorer.add(func.ignore); + } - const fileDir = path.dirname(filePath); - if (!fs.existsSync(fileDir)) { - fs.mkdirSync(fileDir, { recursive: true }); - } + const filesToCopy = getAllFiles(functionPath).map((file) => path.relative(functionPath, file)).filter((file) => !ignorer.ignores(file)); + for(const f of filesToCopy) { + const filePath = path.join(hotSwapPath, f); + if (fs.existsSync(filePath)) { + fs.rmSync(filePath, { force: true }); + } - const sourcePath = path.join(functionPath, f); - fs.copyFileSync(sourcePath, filePath); + const fileDir = path.dirname(filePath); + if (!fs.existsSync(fileDir)) { + fs.mkdirSync(fileDir, { recursive: true }); } - await tar - .create({ - gzip: true, - sync: true, - cwd: hotSwapPath, - file: buildPath - }, ['.']); - - fs.rmSync(hotSwapPath, { recursive: true, force: true }); - - await dockerStart(func, variables, port); + const sourcePath = path.join(functionPath, f); + fs.copyFileSync(sourcePath, filePath); } - } catch(err) { - console.error(err); - } finally { - Queue.unlock(); + + await tar + .create({ + gzip: true, + sync: true, + cwd: hotSwapPath, + file: buildPath + }, ['.']); + + fs.rmSync(hotSwapPath, { recursive: true, force: true }); + + await dockerStart(func, variables, port); } - }); - } + } catch(err) { + console.error(err); + } finally { + Queue.unlock(); + } + }); } const run = new Command("run") @@ -539,7 +589,6 @@ run .description("Run functions in the current directory.") .option(`--functionId `, `Function ID`) .option(`--port `, `Local port`) - .option(`--engine `, `Local engine, "system" or "docker"`) .option(`--userId `, `ID of user to impersonate`) .option(`--noVariables`, `Prevent pulling variables from function settings`) .option(`--noReload`, `Prevent live reloading of server when changes are made to function files`) diff --git a/templates/cli/lib/questions.js.twig b/templates/cli/lib/questions.js.twig index 6befd56c0..dcd04e3af 100644 --- a/templates/cli/lib/questions.js.twig +++ b/templates/cli/lib/questions.js.twig @@ -16,7 +16,7 @@ const JSONbig = require("json-bigint")({ storeAsString: false }); const whenOverride = (answers) => answers.override === undefined ? true : answers.override; const getIgnores = (runtime) => { - const languge = runtime.split('-')[0]; + const languge = runtime.split("-").slice(0, -1).join("-"); switch (languge) { case 'cpp': @@ -36,6 +36,7 @@ const getIgnores = (runtime) => { case 'php': return ['vendor']; case 'python': + case 'python-ml': return ['__pypackages__']; case 'ruby': return ['vendor']; @@ -49,7 +50,7 @@ const getIgnores = (runtime) => { }; const getEntrypoint = (runtime) => { - const languge = runtime.split('-')[0]; + const languge = runtime.split("-").slice(0, -1).join("-"); switch (languge) { case 'dart': @@ -63,6 +64,7 @@ const getEntrypoint = (runtime) => { case 'php': return 'src/index.php'; case 'python': + case 'python-ml': return 'src/main.py'; case 'ruby': return 'lib/main.rb'; @@ -84,7 +86,7 @@ const getEntrypoint = (runtime) => { }; const getInstallCommand = (runtime) => { - const languge = runtime.split('-')[0]; + const languge = runtime.split("-").slice(0, -1).join("-"); switch (languge) { case 'dart': @@ -98,6 +100,7 @@ const getInstallCommand = (runtime) => { case 'php': return 'composer install'; case 'python': + case 'python-ml': return 'pip install -r requirements.txt'; case 'ruby': return 'bundle install'; @@ -814,21 +817,6 @@ const questionsRunFunctions = [ }); }, }, - { - type: "list", - name: "engine", - message: "Which engine would you like to use?", - choices: [ - { - name: "Docker (recommended, simulates production precisely)", - value: "docker", - }, - { - name: "System (faster and easier to debug)", - value: "system", - }, - ], - }, ]; module.exports = { From 7b0f655cc1abc2a52082962854dee38e4862c02a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 14 Jun 2024 14:22:29 +0000 Subject: [PATCH 08/11] PR review changes --- templates/cli/lib/parser.js.twig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/cli/lib/parser.js.twig b/templates/cli/lib/parser.js.twig index 6459f4840..532bfaa95 100644 --- a/templates/cli/lib/parser.js.twig +++ b/templates/cli/lib/parser.js.twig @@ -203,7 +203,7 @@ const commandDescriptions = { "databases": `The databases command allows you to create structured collections of documents, query and filter lists of documents.`, "init": `The init command provides a convenient wrapper for creating and initializing project, functions, collections, buckets, teams and messaging in Appwrite.`, "push": `The push command provides a convenient wrapper for pushing your functions, collections, buckets, teams and messaging.`, - "run": `The dev command allows you to run project locally to allow easy development and quick debugging.`, + "run": `The run command allows you to run project locally to allow easy development and quick debugging.`, "functions": `The functions command allows you view, create and manage your Cloud Functions.`, "health": `The health command allows you to both validate and monitor your {{ spec.title|caseUcfirst }} server's health.`, "pull": `The pull command helps you pull your {{ spec.title|caseUcfirst }} project, functions, collections, buckets, teams and messaging`, From fcde6cba9bc7ff4c00a859ad0729b97428bca786 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 17 Jun 2024 14:41:06 +0200 Subject: [PATCH 09/11] PR review changes --- templates/cli/lib/commands/run.js.twig | 342 +-------------------- templates/cli/lib/emulation/docker.js.twig | 170 ++++++++++ templates/cli/lib/emulation/utils.js.twig | 164 ++++++++++ templates/cli/lib/questions.js.twig | 8 +- 4 files changed, 346 insertions(+), 338 deletions(-) create mode 100644 templates/cli/lib/emulation/docker.js.twig create mode 100644 templates/cli/lib/emulation/utils.js.twig diff --git a/templates/cli/lib/commands/run.js.twig b/templates/cli/lib/commands/run.js.twig index 88539792c..1a100c8ed 100644 --- a/templates/cli/lib/commands/run.js.twig +++ b/templates/cli/lib/commands/run.js.twig @@ -17,336 +17,8 @@ const { projectsCreateJWT } = require('./projects'); const { questionsRunFunctions } = require("../questions"); const { actionRunner, success, log, error, commandDescriptions, drawTable } = require("../parser"); const { systemHasCommand, isPortTaken, getAllFiles } = require('../utils'); - -const activeDockerIds = {}; - -const openRuntimesVersion = 'v3'; -const runtimeNames = { - 'node': 'Node.js', - 'php': 'PHP', - 'ruby': 'Ruby', - 'python': 'Python', - 'python-ml': 'Python (ML)', - 'deno': 'Deno', - 'dart': 'Dart', - 'dotnet': '.NET', - 'java': 'Java', - 'swift': 'Swift', - 'kotlin': 'Kotlin', - 'bun': 'Bun' -}; -const systemTools = { - 'node': { - isCompiled: false, - startCommand: "node src/server.js", - dependencyFiles: [ "package.json", "package-lock.json" ] - }, - 'php': { - isCompiled: false, - startCommand: "php src/server.php", - dependencyFiles: [ "composer.json", "composer.lock" ] - }, - 'ruby': { - isCompiled: false, - startCommand: "bundle exec puma -b tcp://0.0.0.0:3000 -e production", - dependencyFiles: [ "Gemfile", "Gemfile.lock" ] - }, - 'python': { - isCompiled: false, - startCommand: "python3 src/server.py", - dependencyFiles: [ "requirements.txt", "requirements.lock" ] - }, - 'python-ml': { - isCompiled: false, - startCommand: "python3 src/server.py", - dependencyFiles: [ "requirements.txt", "requirements.lock" ] - }, - 'deno': { - isCompiled: false, - startCommand: "deno start", - dependencyFiles: [ ] - }, - 'dart': { - isCompiled: true, - startCommand: "src/function/server", - dependencyFiles: [ ] - }, - 'dotnet': { - isCompiled: true, - startCommand: "dotnet src/function/DotNetRuntime.dll", - dependencyFiles: [ ] - }, - 'java': { - isCompiled: true, - startCommand: "java -jar src/function/java-runtime-1.0.0.jar", - dependencyFiles: [ ] - }, - 'swift': { - isCompiled: true, - startCommand: "src/function/Runtime serve --env production --hostname 0.0.0.0 --port 3000", - dependencyFiles: [ ] - }, - 'kotlin': { - isCompiled: true, - startCommand: "java -jar src/function/kotlin-runtime-1.0.0.jar", - dependencyFiles: [ ] - }, - 'bun': { - isCompiled: false, - startCommand: "bun src/server.ts", - dependencyFiles: [ "package.json", "package-lock.json", "bun.lockb" ] - }, -}; - -const JwtManager = { - userJwt: null, - functionJwt: null, - - timerWarn: null, - timerError: null, - - async setup(userId = null) { - if(this.timerWarn) { - clearTimeout(this.timerWarn); - } - - if(this.timerError) { - clearTimeout(this.timerError); - } - - this.timerWarn = setTimeout(() => { - log("Warning: Authorized JWT will expire in 5 minutes. Please stop and re-run the command to refresh tokens for 1 hour."); - }, 1000 * 60 * 55); // 55 mins - - this.timerError = setTimeout(() => { - log("Warning: Authorized JWT just expired. Please stop and re-run the command to obtain new tokens with 1 hour validity."); - log("Some Appwrite API communication is not authorized now.") - }, 1000 * 60 * 60); // 60 mins - - if(userId) { - await usersGet({ - userId, - parseOutput: false - }); - const userResponse = await usersCreateJWT({ - userId, - duration: 60*60, - parseOutput: false - }); - this.userJwt = userResponse.jwt; - } - - const functionResponse = await projectsCreateJWT({ - projectId: localConfig.getProject().projectId, - // TODO: Once we have endpoint for this, use it - scopes: ["sessions.write","users.read","users.write","teams.read","teams.write","databases.read","databases.write","collections.read","collections.write","attributes.read","attributes.write","indexes.read","indexes.write","documents.read","documents.write","files.read","files.write","buckets.read","buckets.write","functions.read","functions.write","execution.read","execution.write","locale.read","avatars.read","health.read","providers.read","providers.write","messages.read","messages.write","topics.read","topics.write","subscribers.read","subscribers.write","targets.read","targets.write","rules.read","rules.write","migrations.read","migrations.write","vcs.read","vcs.write","assistant.read"], - duration: 60*60, - parseOutput: false - }); - this.functionJwt = functionResponse.jwt; - } -}; - -const Queue = { - files: [], - locked: false, - events: new EventEmitter(), - debounce: null, - push(file) { - if(!this.files.includes(file)) { - this.files.push(file); - } - - if(!this.locked) { - this._trigger(); - } - }, - lock() { - this.files = []; - this.locked = true; - }, - unlock() { - this.locked = false; - if(this.files.length > 0) { - this._trigger(); - } - }, - _trigger() { - if(this.debounce) { - return; - } - - this.debounce = setTimeout(() => { - this.events.emit('reload', { files: this.files }); - this.debounce = null; - }, 300); - } -}; - -async function dockerStop(id) { - delete activeDockerIds[id]; - const stopProcess = childProcess.spawn('docker', ['rm', '--force', id], { - stdio: 'pipe', - }); - - await new Promise((res) => { stopProcess.on('close', res) }); -} - -async function dockerPull(func) { - log('Pulling Docker image of function runtime ...'); - - const runtimeChunks = func.runtime.split("-"); - const runtimeVersion = runtimeChunks.pop(); - const runtimeName = runtimeChunks.join("-"); - const imageName = `openruntimes/${runtimeName}:${openRuntimesVersion}-${runtimeVersion}`; - - const pullProcess = childProcess.spawn('docker', ['pull', imageName], { - stdio: 'pipe', - pwd: path.join(process.cwd(), func.path) - }); - - pullProcess.stderr.on('data', (data) => { - process.stderr.write(`\n${data}$ `); - }); - - await new Promise((res) => { pullProcess.on('close', res) }); -} - -async function dockerBuild(func, variables) { - log('Building function using Docker ...'); - - const runtimeChunks = func.runtime.split("-"); - const runtimeVersion = runtimeChunks.pop(); - const runtimeName = runtimeChunks.join("-"); - const imageName = `openruntimes/${runtimeName}:${openRuntimesVersion}-${runtimeVersion}`; - - const functionDir = path.join(process.cwd(), func.path); - - const id = ID.unique(); - - const params = [ 'run' ]; - params.push('--name', id); - params.push('-v', `${functionDir}/:/mnt/code:rw`); - params.push('-e', 'APPWRITE_ENV=development'); - params.push('-e', 'OPEN_RUNTIMES_ENV=development'); - params.push('-e', 'OPEN_RUNTIMES_SECRET='); - params.push('-e', `OPEN_RUNTIMES_ENTRYPOINT=${func.entrypoint}`); - - for(const k of Object.keys(variables)) { - params.push('-e', `${k}=${variables[k]}`); - } - - params.push(imageName, 'sh', '-c', `helpers/build.sh "${func.commands}"`); - - const buildProcess = childProcess.spawn('docker', params, { - stdio: 'pipe', - pwd: functionDir - }); - - buildProcess.stdout.on('data', (data) => { - process.stdout.write(`\n${data}`); - }); - - buildProcess.stderr.on('data', (data) => { - process.stderr.write(`\n${data}`); - }); - - await new Promise((res) => { buildProcess.on('close', res) }); - - const copyPath = path.join(process.cwd(), func.path, '.appwrite', 'build.tar.gz'); - const copyDir = path.dirname(copyPath); - if (!fs.existsSync(copyDir)) { - fs.mkdirSync(copyDir, { recursive: true }); - } - - const copyProcess = childProcess.spawn('docker', ['cp', `${id}:/mnt/code/code.tar.gz`, copyPath], { - stdio: 'pipe', - pwd: functionDir - }); - - await new Promise((res) => { copyProcess.on('close', res) }); - - const cleanupProcess = childProcess.spawn('docker', ['rm', '--force', id], { - stdio: 'pipe', - pwd: functionDir - }); - - await new Promise((res) => { cleanupProcess.on('close', res) }); - - delete activeDockerIds[id]; - - const tempPath = path.join(process.cwd(), func.path, 'code.tar.gz'); - if (fs.existsSync(tempPath)) { - fs.rmSync(tempPath, { force: true }); - } -} - -async function dockerStart(func, variables, port) { - log('Starting function using Docker ...'); - - log("Permissions, events, CRON and timeouts dont apply when running locally."); - - log('💡 Hint: Function automatically restarts when you edit your code.'); - - success(`Visit http://localhost:${port}/ to execute your function.`); - - - const runtimeChunks = func.runtime.split("-"); - const runtimeVersion = runtimeChunks.pop(); - const runtimeName = runtimeChunks.join("-"); - const imageName = `openruntimes/${runtimeName}:${openRuntimesVersion}-${runtimeVersion}`; - - const tool = systemTools[runtimeName]; - - const functionDir = path.join(process.cwd(), func.path); - - const id = ID.unique(); - - const params = [ 'run' ]; - params.push('--rm'); - params.push('-d'); - params.push('--name', id); - params.push('-p', `${port}:3000`); - params.push('-e', 'APPWRITE_ENV=development'); - params.push('-e', 'OPEN_RUNTIMES_ENV=development'); - params.push('-e', 'OPEN_RUNTIMES_SECRET='); - - for(const k of Object.keys(variables)) { - params.push('-e', `${k}=${variables[k]}`); - } - - params.push('-v', `${functionDir}/.appwrite/logs.txt:/mnt/logs/dev_logs.log:rw`); - params.push('-v', `${functionDir}/.appwrite/errors.txt:/mnt/logs/dev_errors.log:rw`); - params.push('-v', `${functionDir}/.appwrite/build.tar.gz:/mnt/code/code.tar.gz:ro`); - params.push(imageName, 'sh', '-c', `helpers/start.sh "${tool.startCommand}"`); - - childProcess.spawn('docker', params, { - stdio: 'pipe', - pwd: functionDir - }); - - activeDockerIds[id] = true; -} - -async function dockerCleanup() { - const ids = Object.keys(activeDockerIds); - for await (const id of ids) { - await dockerStop(id); - } - - const functions = localConfig.getFunctions(); - for(const func of functions) { - const appwritePath = path.join(process.cwd(), func.path, '.appwrite'); - if (fs.existsSync(appwritePath)) { - fs.rmSync(appwritePath, { recursive: true, force: true }); - } - - const tempPath = path.join(process.cwd(), func.path, 'code.tar.gz'); - if (fs.existsSync(tempPath)) { - fs.rmSync(tempPath, { force: true }); - } - } -} +const { openRuntimesVersion, runtimeNames, systemTools, JwtManager, Queue } = require('../emulation/utils'); +const { dockerStop, dockerCleanup, dockerStart, dockerBuild, dockerPull, dockerStopActive } = require('../emulation/docker'); const runFunction = async ({ port, functionId, noVariables, noReload, userId } = {}) => { // Selection @@ -377,7 +49,7 @@ const runFunction = async ({ port, functionId, noVariables, noReload, userId } = const taken = await isPortTaken(port); if(taken) { - log(`Port ${port} is already used.`); + log(`Port ${port} is already in use by another process.`); port = null; } } @@ -389,7 +61,7 @@ const runFunction = async ({ port, functionId, noVariables, noReload, userId } = // Configuration: Engine if(!systemHasCommand('docker')) { - return error("Please install Docker first: https://docs.docker.com/engine/install/"); + return error("Docker Engine is required for local development. Please install Docker using: https://docs.docker.com/engine/install/"); } // Settings @@ -402,7 +74,7 @@ const runFunction = async ({ port, functionId, noVariables, noReload, userId } = log("Local function configuration:"); drawTable([settings]); - log('If you wish to change local settings, update appwrite.json file and rerun the command. To deploy the function, run: appwrite push function'); + log('If you wish to change your local settings, update the appwrite.json file and rerun the `appwrite run` command.'); await dockerCleanup(); @@ -499,9 +171,7 @@ const runFunction = async ({ port, functionId, noVariables, noReload, userId } = try { log('Stopping the function ...'); - for(const id in activeDockerIds) { - await dockerStop(id); - } + await dockerStopActive(); const dependencyFile = files.find((filePath) => tool.dependencyFiles.includes(filePath)); if(tool.isCompiled || dependencyFile) { diff --git a/templates/cli/lib/emulation/docker.js.twig b/templates/cli/lib/emulation/docker.js.twig new file mode 100644 index 000000000..fa0b942c5 --- /dev/null +++ b/templates/cli/lib/emulation/docker.js.twig @@ -0,0 +1,170 @@ +const activeDockerIds = {}; + +export async function dockerStop(id) { + delete activeDockerIds[id]; + const stopProcess = childProcess.spawn('docker', ['rm', '--force', id], { + stdio: 'pipe', + }); + + await new Promise((res) => { stopProcess.on('close', res) }); +} + +export async function dockerPull(func) { + log('Pulling Docker image of function runtime ...'); + + const runtimeChunks = func.runtime.split("-"); + const runtimeVersion = runtimeChunks.pop(); + const runtimeName = runtimeChunks.join("-"); + const imageName = `openruntimes/${runtimeName}:${openRuntimesVersion}-${runtimeVersion}`; + + const pullProcess = childProcess.spawn('docker', ['pull', imageName], { + stdio: 'pipe', + pwd: path.join(process.cwd(), func.path) + }); + + pullProcess.stderr.on('data', (data) => { + process.stderr.write(`\n${data}$ `); + }); + + await new Promise((res) => { pullProcess.on('close', res) }); +} + +export async function dockerBuild(func, variables) { + log('Building function using Docker ...'); + + const runtimeChunks = func.runtime.split("-"); + const runtimeVersion = runtimeChunks.pop(); + const runtimeName = runtimeChunks.join("-"); + const imageName = `openruntimes/${runtimeName}:${openRuntimesVersion}-${runtimeVersion}`; + + const functionDir = path.join(process.cwd(), func.path); + + const id = ID.unique(); + + const params = [ 'run' ]; + params.push('--name', id); + params.push('-v', `${functionDir}/:/mnt/code:rw`); + params.push('-e', 'APPWRITE_ENV=development'); + params.push('-e', 'OPEN_RUNTIMES_ENV=development'); + params.push('-e', 'OPEN_RUNTIMES_SECRET='); + params.push('-e', `OPEN_RUNTIMES_ENTRYPOINT=${func.entrypoint}`); + + for(const k of Object.keys(variables)) { + params.push('-e', `${k}=${variables[k]}`); + } + + params.push(imageName, 'sh', '-c', `helpers/build.sh "${func.commands}"`); + + const buildProcess = childProcess.spawn('docker', params, { + stdio: 'pipe', + pwd: functionDir + }); + + buildProcess.stdout.on('data', (data) => { + process.stdout.write(`\n${data}`); + }); + + buildProcess.stderr.on('data', (data) => { + process.stderr.write(`\n${data}`); + }); + + await new Promise((res) => { buildProcess.on('close', res) }); + + const copyPath = path.join(process.cwd(), func.path, '.appwrite', 'build.tar.gz'); + const copyDir = path.dirname(copyPath); + if (!fs.existsSync(copyDir)) { + fs.mkdirSync(copyDir, { recursive: true }); + } + + const copyProcess = childProcess.spawn('docker', ['cp', `${id}:/mnt/code/code.tar.gz`, copyPath], { + stdio: 'pipe', + pwd: functionDir + }); + + await new Promise((res) => { copyProcess.on('close', res) }); + + const cleanupProcess = childProcess.spawn('docker', ['rm', '--force', id], { + stdio: 'pipe', + pwd: functionDir + }); + + await new Promise((res) => { cleanupProcess.on('close', res) }); + + delete activeDockerIds[id]; + + const tempPath = path.join(process.cwd(), func.path, 'code.tar.gz'); + if (fs.existsSync(tempPath)) { + fs.rmSync(tempPath, { force: true }); + } +} + +export async function dockerStart(func, variables, port) { + log('Starting function using Docker ...'); + + log("Permissions, events, CRON and timeouts dont apply when running locally."); + + log('💡 Hint: Function automatically restarts when you edit your code.'); + + success(`Visit http://localhost:${port}/ to execute your function.`); + + + const runtimeChunks = func.runtime.split("-"); + const runtimeVersion = runtimeChunks.pop(); + const runtimeName = runtimeChunks.join("-"); + const imageName = `openruntimes/${runtimeName}:${openRuntimesVersion}-${runtimeVersion}`; + + const tool = systemTools[runtimeName]; + + const functionDir = path.join(process.cwd(), func.path); + + const id = ID.unique(); + + const params = [ 'run' ]; + params.push('--rm'); + params.push('-d'); + params.push('--name', id); + params.push('-p', `${port}:3000`); + params.push('-e', 'APPWRITE_ENV=development'); + params.push('-e', 'OPEN_RUNTIMES_ENV=development'); + params.push('-e', 'OPEN_RUNTIMES_SECRET='); + + for(const k of Object.keys(variables)) { + params.push('-e', `${k}=${variables[k]}`); + } + + params.push('-v', `${functionDir}/.appwrite/logs.txt:/mnt/logs/dev_logs.log:rw`); + params.push('-v', `${functionDir}/.appwrite/errors.txt:/mnt/logs/dev_errors.log:rw`); + params.push('-v', `${functionDir}/.appwrite/build.tar.gz:/mnt/code/code.tar.gz:ro`); + params.push(imageName, 'sh', '-c', `helpers/start.sh "${tool.startCommand}"`); + + childProcess.spawn('docker', params, { + stdio: 'pipe', + pwd: functionDir + }); + + activeDockerIds[id] = true; +} + +export async function dockerCleanup() { + await dockerStop(); + + const functions = localConfig.getFunctions(); + for(const func of functions) { + const appwritePath = path.join(process.cwd(), func.path, '.appwrite'); + if (fs.existsSync(appwritePath)) { + fs.rmSync(appwritePath, { recursive: true, force: true }); + } + + const tempPath = path.join(process.cwd(), func.path, 'code.tar.gz'); + if (fs.existsSync(tempPath)) { + fs.rmSync(tempPath, { force: true }); + } + } +} + +export async function dockerStopActive() { + const ids = Object.keys(activeDockerIds); + for await (const id of ids) { + await dockerStop(id); + } +} \ No newline at end of file diff --git a/templates/cli/lib/emulation/utils.js.twig b/templates/cli/lib/emulation/utils.js.twig new file mode 100644 index 000000000..a7011f3eb --- /dev/null +++ b/templates/cli/lib/emulation/utils.js.twig @@ -0,0 +1,164 @@ +export const openRuntimesVersion = 'v3'; + +export const runtimeNames = { + 'node': 'Node.js', + 'php': 'PHP', + 'ruby': 'Ruby', + 'python': 'Python', + 'python-ml': 'Python (ML)', + 'deno': 'Deno', + 'dart': 'Dart', + 'dotnet': '.NET', + 'java': 'Java', + 'swift': 'Swift', + 'kotlin': 'Kotlin', + 'bun': 'Bun' +}; + +export const systemTools = { + 'node': { + isCompiled: false, + startCommand: "node src/server.js", + dependencyFiles: [ "package.json", "package-lock.json" ] + }, + 'php': { + isCompiled: false, + startCommand: "php src/server.php", + dependencyFiles: [ "composer.json", "composer.lock" ] + }, + 'ruby': { + isCompiled: false, + startCommand: "bundle exec puma -b tcp://0.0.0.0:3000 -e production", + dependencyFiles: [ "Gemfile", "Gemfile.lock" ] + }, + 'python': { + isCompiled: false, + startCommand: "python3 src/server.py", + dependencyFiles: [ "requirements.txt", "requirements.lock" ] + }, + 'python-ml': { + isCompiled: false, + startCommand: "python3 src/server.py", + dependencyFiles: [ "requirements.txt", "requirements.lock" ] + }, + 'deno': { + isCompiled: false, + startCommand: "deno start", + dependencyFiles: [ ] + }, + 'dart': { + isCompiled: true, + startCommand: "src/function/server", + dependencyFiles: [ ] + }, + 'dotnet': { + isCompiled: true, + startCommand: "dotnet src/function/DotNetRuntime.dll", + dependencyFiles: [ ] + }, + 'java': { + isCompiled: true, + startCommand: "java -jar src/function/java-runtime-1.0.0.jar", + dependencyFiles: [ ] + }, + 'swift': { + isCompiled: true, + startCommand: "src/function/Runtime serve --env production --hostname 0.0.0.0 --port 3000", + dependencyFiles: [ ] + }, + 'kotlin': { + isCompiled: true, + startCommand: "java -jar src/function/kotlin-runtime-1.0.0.jar", + dependencyFiles: [ ] + }, + 'bun': { + isCompiled: false, + startCommand: "bun src/server.ts", + dependencyFiles: [ "package.json", "package-lock.json", "bun.lockb" ] + }, +}; + +export const JwtManager = { + userJwt: null, + functionJwt: null, + + timerWarn: null, + timerError: null, + + async setup(userId = null) { + if(this.timerWarn) { + clearTimeout(this.timerWarn); + } + + if(this.timerError) { + clearTimeout(this.timerError); + } + + this.timerWarn = setTimeout(() => { + log("Warning: Authorized JWT will expire in 5 minutes. Please stop and re-run the command to refresh tokens for 1 hour."); + }, 1000 * 60 * 55); // 55 mins + + this.timerError = setTimeout(() => { + log("Warning: Authorized JWT just expired. Please stop and re-run the command to obtain new tokens with 1 hour validity."); + log("Some Appwrite API communication is not authorized now.") + }, 1000 * 60 * 60); // 60 mins + + if(userId) { + await usersGet({ + userId, + parseOutput: false + }); + const userResponse = await usersCreateJWT({ + userId, + duration: 60*60, + parseOutput: false + }); + this.userJwt = userResponse.jwt; + } + + const functionResponse = await projectsCreateJWT({ + projectId: localConfig.getProject().projectId, + // TODO: Once we have endpoint for this, use it + scopes: ["sessions.write","users.read","users.write","teams.read","teams.write","databases.read","databases.write","collections.read","collections.write","attributes.read","attributes.write","indexes.read","indexes.write","documents.read","documents.write","files.read","files.write","buckets.read","buckets.write","functions.read","functions.write","execution.read","execution.write","locale.read","avatars.read","health.read","providers.read","providers.write","messages.read","messages.write","topics.read","topics.write","subscribers.read","subscribers.write","targets.read","targets.write","rules.read","rules.write","migrations.read","migrations.write","vcs.read","vcs.write","assistant.read"], + duration: 60*60, + parseOutput: false + }); + this.functionJwt = functionResponse.jwt; + } +}; + +export const Queue = { + files: [], + locked: false, + events: new EventEmitter(), + debounce: null, + push(file) { + if(!this.files.includes(file)) { + this.files.push(file); + } + + if(!this.locked) { + this._trigger(); + } + }, + lock() { + this.files = []; + this.locked = true; + }, + unlock() { + this.locked = false; + if(this.files.length > 0) { + this._trigger(); + } + }, + _trigger() { + if(this.debounce) { + return; + } + + this.debounce = setTimeout(() => { + this.events.emit('reload', { files: this.files }); + this.debounce = null; + }, 300); + } +}; \ No newline at end of file diff --git a/templates/cli/lib/questions.js.twig b/templates/cli/lib/questions.js.twig index 83b104011..c0843fcd7 100644 --- a/templates/cli/lib/questions.js.twig +++ b/templates/cli/lib/questions.js.twig @@ -814,15 +814,19 @@ const questionsRunFunctions = [ (async () => { if (typeof value !== 'number' && isNaN(+value)) { - throw Error(`You need to provide a number.`); + throw Error(`Port needs to be a valid integer between 1024 and 65536.`); } value = +value; + if(value < 1024 || value > 65536) { + throw Error(`Port needs to be a valid integer between 1024 and 65536.`); + } + const taken = await isPortTaken(value); if(taken) { - throw Error(`Port ${value} is taken. Pick another one.`); + throw Error(`Port ${value} is in use by another process. Pick a different one.`); } })().then(() => { done(null, true); From 797104fd36cbf72df97c9431515bd963a6dda4d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 17 Jun 2024 14:47:35 +0200 Subject: [PATCH 10/11] More PR review changes --- templates/cli/lib/commands/run.js.twig | 22 ++++++++++--- templates/cli/lib/questions.js.twig | 45 +------------------------- 2 files changed, 19 insertions(+), 48 deletions(-) diff --git a/templates/cli/lib/commands/run.js.twig b/templates/cli/lib/commands/run.js.twig index 1a100c8ed..cf0981e66 100644 --- a/templates/cli/lib/commands/run.js.twig +++ b/templates/cli/lib/commands/run.js.twig @@ -49,14 +49,28 @@ const runFunction = async ({ port, functionId, noVariables, noReload, userId } = const taken = await isPortTaken(port); if(taken) { - log(`Port ${port} is already in use by another process.`); - port = null; + error(`Port ${port} is already in use by another process.`); + return; } } if(!port) { - const answers = await inquirer.prompt(questionsRunFunctions[1]); - port = answers.port; + let portFound = fale; + port = 3000; + while(port < 3100) { + const taken = await isPortTaken(port); + if(!taken) { + portFound = true; + break; + } + + port++; + } + + if(!portFound) { + error('Could not auto-suggest an available port. Please configure port with `appwrite run --port YOUR_PORT` command.'); + return; + } } // Configuration: Engine diff --git a/templates/cli/lib/questions.js.twig b/templates/cli/lib/questions.js.twig index c0843fcd7..17417a3b6 100644 --- a/templates/cli/lib/questions.js.twig +++ b/templates/cli/lib/questions.js.twig @@ -791,50 +791,7 @@ const questionsRunFunctions = [ }) return choices; } - }, - { - type: "input", - name: "port", - message: 'Which port would you like function to listen on?', - default: async () => { - let port = 3000; - while(port < 3100) { - const taken = await isPortTaken(port); - if(!taken) { - return port; - } - - port++; - } - - return 3000; - }, - validate: function(value) { - const done = this.async(); - - (async () => { - if (typeof value !== 'number' && isNaN(+value)) { - throw Error(`Port needs to be a valid integer between 1024 and 65536.`); - } - - value = +value; - - if(value < 1024 || value > 65536) { - throw Error(`Port needs to be a valid integer between 1024 and 65536.`); - } - - const taken = await isPortTaken(value); - - if(taken) { - throw Error(`Port ${value} is in use by another process. Pick a different one.`); - } - })().then(() => { - done(null, true); - }).catch((err) => { - done(err.message); - }); - }, - }, + } ]; module.exports = { From cebd23e6a5493b2d475a58d7f2c6029ea7374ca6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 17 Jun 2024 14:59:56 +0200 Subject: [PATCH 11/11] PR review changes --- src/SDK/Language/CLI.php | 10 ++++++++++ templates/cli/lib/commands/run.js.twig | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/SDK/Language/CLI.php b/src/SDK/Language/CLI.php index 432732d87..bfab5f4f6 100644 --- a/src/SDK/Language/CLI.php +++ b/src/SDK/Language/CLI.php @@ -250,6 +250,16 @@ public function getFiles(): array 'destination' => 'lib/commands/run.js', 'template' => 'cli/lib/commands/run.js.twig', ], + [ + 'scope' => 'default', + 'destination' => 'lib/emulation/docker.js', + 'template' => 'cli/lib/emulation/docker.js.twig', + ], + [ + 'scope' => 'default', + 'destination' => 'lib/emulation/utils.js', + 'template' => 'cli/lib/emulation/utils.js.twig', + ], [ 'scope' => 'service', 'destination' => '/lib/commands/{{service.name | caseDash}}.js', diff --git a/templates/cli/lib/commands/run.js.twig b/templates/cli/lib/commands/run.js.twig index cf0981e66..593bd5a84 100644 --- a/templates/cli/lib/commands/run.js.twig +++ b/templates/cli/lib/commands/run.js.twig @@ -68,7 +68,7 @@ const runFunction = async ({ port, functionId, noVariables, noReload, userId } = } if(!portFound) { - error('Could not auto-suggest an available port. Please configure port with `appwrite run --port YOUR_PORT` command.'); + error('Could not find an available port. Please select a port with `appwrite run --port YOUR_PORT` command.'); return; } }