Merge pull request #808 from Youssef-Harby/plugin-duckdb

Plugin-duckdb

Author: FalkWolsky

server/node-service/package.json

@@ -48,6 +48,7 @@
     "@types/node-fetch": "^2.6.2",
     "axios": "^1.2.0",
     "base64-arraybuffer": "^1.0.2",
+    "duckdb-async": "^0.10.0",
     "dynamodb-data-types": "^4.0.1",
     "express": "^4.18.2",
     "express-async-errors": "^3.1.1",

server/node-service/src/plugins/duckdb/dataSourceConfig.ts

@@ -0,0 +1,27 @@
+// @server/node-service/src/plugins/duckdb/dataSourceConfig.ts
+import { ConfigToType } from "lowcoder-sdk/dataSource";
+
+const dataSourceConfig = {
+    type: "dataSource",
+    params: [
+        {
+            key: "databaseFile",
+            type: "textInput",
+            label: "Database File",
+            rules: [{ required: true, message: "Please provide a database file path 'db.duckdb' or ':memory:' for an in-memory database" }],
+            tooltip: "Please provide a database file path 'db.duckdb' or ':memory:' for an in-memory database",
+            defaultValue: ":memory:",
+        },
+        {
+            key: "options",
+            type: "textInput",
+            label: "Database Options",
+            tooltip: "Additional options to pass to the DuckDB constructor (in JSON format)",
+            defaultValue: `{"access_mode": "READ_WRITE","max_memory": "512MB","threads": "4"}`,
+        },
+    ],
+} as const;
+
+export default dataSourceConfig;
+
+export type DataSourceDataType = ConfigToType<typeof dataSourceConfig>;

server/node-service/src/plugins/duckdb/index.ts

@@ -0,0 +1,42 @@
+import { DataSourcePlugin } from "lowcoder-sdk/dataSource";
+import dataSourceConfig, { DataSourceDataType } from "./dataSourceConfig";
+import queryConfig, { ActionDataType } from "./queryConfig";
+import { Database } from "duckdb-async";
+import { ServiceError } from "../../common/error";
+
+// Helper function to handle BigInt serialization
+function serializeBigInts(row: any): any {
+    const newRow: { [key: string]: any } = {}; // Add index signature
+    for (const [key, value] of Object.entries(row)) {
+        newRow[key] = typeof value === 'bigint' ? value.toString() : value;
+    }
+    return newRow;
+}
+
+const duckdbPlugin: DataSourcePlugin<ActionDataType, DataSourceDataType> = {
+    id: "duckdb",
+    name: "DuckDB",
+    category: "database",
+    icon: "duckdb.svg",
+    dataSourceConfig,
+    queryConfig,
+    run: async function (actionData, dataSourceConfig): Promise<any> {
+        const { databaseFile, options } = dataSourceConfig;
+        const parsedOptions = JSON.parse(options);
+        const db = await Database.create(databaseFile, parsedOptions);
+
+        if (actionData.actionName === "Query") {
+            try {
+                const result = await db.all(actionData.queryString);
+                // Apply BigInt serialization to each row
+                return result.map(serializeBigInts);
+            } catch (error) {
+                throw new ServiceError((error as Error).message);
+            } finally {
+                await db.close();
+            }
+        }
+    },
+};
+
+export default duckdbPlugin;

server/node-service/src/plugins/duckdb/queryConfig.ts

@@ -0,0 +1,24 @@
+// @server/node-service/src/plugins/duckdb/queryConfig.ts
+import { ConfigToType } from "lowcoder-sdk/dataSource";
+
+const queryConfig = {
+    type: "query",
+    label: "Action",
+    actions: [
+        {
+            actionName: "Query",
+            label: "Query",
+            params: [
+                {
+                    label: "Query String",
+                    key: "queryString",
+                    type: "sqlInput",
+                },
+            ],
+        },
+    ],
+} as const;
+
+export type ActionDataType = ConfigToType<typeof queryConfig>;
+
+export default queryConfig;

server/node-service/src/plugins/index.ts

@@ -8,6 +8,7 @@ import couchdbPlugin from "./couchdb";
 import wooCommercePlugin from "./woocommerce";
 import openAiPlugin from "./openAi";
 import athenaPlugin from "./athena";
+import duckdbPlugin from "./duckdb";
 import lambdaPlugin from "./lambda";
 import googleCloudStorage from "./googleCloudStorage";
 import stripePlugin from "./stripe";
@@ -43,6 +44,7 @@ let plugins: (DataSourcePlugin | DataSourcePluginFactory)[] = [
   wooCommercePlugin,
   openAiPlugin,
   athenaPlugin,
+  duckdbPlugin,
   lambdaPlugin,
   googleCloudStorage,
   stripePlugin,
@@ -72,6 +74,6 @@ let plugins: (DataSourcePlugin | DataSourcePluginFactory)[] = [
 try {
   plugins = require("../ee/plugins").default;
   console.info("using ee plugins");
-} catch {}
+} catch { }
 
 export default plugins;