PQL: Add TS functions for automations

.github/workflows/create-build.yml

@@ -25,6 +25,8 @@ jobs:
           APP_PG_JDBC_URL: "op://Product ACT/pql-docs-bot/postgres-jdbc-url"
           APP_PG_JDBC_SCHEMAS: "op://Product ACT/pql-docs-bot/postgres-jdbc-schemas"
           APP_TS_OPENAI_API_KEY: "op://Product ACT/pql-docs-bot/openai-api-key"
+          APP_TS_PG_DATABASE_URL: "op://Product ACT/pql-docs-bot/postgres-database-url"
+          JWT_SECRET: "op://Product ACT/pql-docs-bot/jwt-secret"
 
       - name: Install DDN CLI
         run: |
@@ -48,6 +50,8 @@ jobs:
           APP_PG_JDBC_URL=$APP_PG_JDBC_URL
           APP_PG_JDBC_SCHEMAS=$APP_PG_JDBC_SCHEMAS
           APP_TS_OPENAI_API_KEY=$APP_TS_OPENAI_API_KEY
+          APP_TS_PG_DATABASE_URL=$APP_TS_PG_DATABASE_URL
+          JWT_SECRET=$JWT_SECRET
           EOF
 
       - name: Create DDN build

.github/workflows/server-ci.yml

@@ -19,6 +19,7 @@ jobs:
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
           PQL_API_KEY: "op://Product ACT/pql-docs-bot/pql-api-key"
+          JWT_SECRET: "op://Product ACT/pql-docs-bot/jwt-secret"
 
       - name: Setup Bun
         uses: oven-sh/setup-bun@v1
@@ -47,6 +48,7 @@ jobs:
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
           PQL_API_KEY: "op://Product ACT/pql-docs-bot/pql-api-key"
+          JWT_SECRET: "op://Product ACT/pql-docs-bot/jwt-secret"
 
       - name: Build Docker image
         run: |

pql/app/connector/ts/compose.yaml

@@ -12,6 +12,7 @@ services:
       OPENAI_API_KEY: $APP_TS_OPENAI_API_KEY
       OTEL_EXPORTER_OTLP_ENDPOINT: $APP_TS_OTEL_EXPORTER_OTLP_ENDPOINT
       OTEL_SERVICE_NAME: $APP_TS_OTEL_SERVICE_NAME
+      PG_DATABASE_URL: $APP_TS_PG_DATABASE_URL
     extra_hosts:
       - local.hasura.dev:host-gateway
     ports:

pql/app/connector/ts/connector.yaml

@@ -16,3 +16,5 @@ definition:
       fromEnv: APP_TS_OTEL_EXPORTER_OTLP_ENDPOINT
     OTEL_SERVICE_NAME:
       fromEnv: APP_TS_OTEL_SERVICE_NAME
+    PG_DATABASE_URL:
+      fromEnv: APP_TS_PG_DATABASE_URL

pql/app/connector/ts/functions.ts

@@ -1,18 +1,40 @@
 import OpenAI from "openai";
+import { Pool } from "pg";
 
 // Initialize OpenAI client
 const openai = new OpenAI({
-  apiKey:
-    process.env.OPENAI_API_KEY ||
-    "[REMOVED]",
+  apiKey: process.env.OPENAI_API_KEY || "[REMOVED]",
+});
+
+const pool = new Pool({
+  connectionString: process.env.PG_DATABASE_URL,
 });
 
 export type EmbeddingResult = {
   values: number[];
 };
 
+export type DocOperationResult = {
+  success: boolean;
+  docContentId?: string;
+  message: string;
+};
+
+interface Doc_Page {
+  page_url: string;
+  title: string;
+  description?: string;
+  keywords?: string[];
+  content: string;
+}
+
+interface DocChunk {
+  content: string;
+  lineStart: number;
+  lineEnd: number;
+}
+
 /**
- *
  * @readonly
  */
 export async function transformQueryIntoEmbedding(text: string): Promise<EmbeddingResult> {
@@ -31,3 +53,169 @@ export async function transformQueryIntoEmbedding(text: string): Promise<Embeddi
     throw new Error(`Failed to generate embedding: ${error instanceof Error ? error.message : "Unknown error"}`);
   }
 }
+
+/**
+ * @readonly
+ */
+export async function insertNewPage(page: Doc_Page): Promise<DocOperationResult> {
+  const client = await pool.connect();
+  try {
+    await client.query("BEGIN");
+
+    const contentResult = await client.query(
+      `INSERT INTO pql_docs.doc_content (page_url, title, description, keywords, content, is_checked)
+       VALUES ($1, $2, $3, $4, $5, true) RETURNING id`,
+      [page.page_url, page.title, page.description, page.keywords, page.content]
+    );
+
+    const docContentId = contentResult.rows[0].id;
+    await generateChunksAndEmbeddings(client, docContentId, page);
+
+    await client.query("COMMIT");
+    return {
+      success: true,
+      docContentId,
+      message: `Successfully inserted page: ${page.page_url}`,
+    };
+  } catch (error) {
+    await client.query("ROLLBACK");
+    return {
+      success: false,
+      message: `Failed to insert page: ${error instanceof Error ? error.message : "Unknown error"}`,
+    };
+  } finally {
+    client.release();
+  }
+}
+
+/**
+ * @readonly
+ */
+export async function updateExistingPage(page: Doc_Page): Promise<DocOperationResult> {
+  const client = await pool.connect();
+  try {
+    await client.query("BEGIN");
+
+    const contentResult = await client.query(
+      `UPDATE pql_docs.doc_content 
+       SET title = $2, description = $3, keywords = $4, content = $5, updated_at = NOW()
+       WHERE page_url = $1 RETURNING id`,
+      [page.page_url, page.title, page.description, page.keywords, page.content]
+    );
+
+    if (contentResult.rows.length === 0) {
+      return {
+        success: false,
+        message: `Page not found: ${page.page_url}`,
+      };
+    }
+
+    const docContentId = contentResult.rows[0].id;
+
+    await client.query("DELETE FROM pql_docs.doc_chunk WHERE doc_content_id_fk = $1", [docContentId]);
+
+    await generateChunksAndEmbeddings(client, docContentId, page);
+
+    await client.query("COMMIT");
+    return {
+      success: true,
+      docContentId,
+      message: `Successfully updated page: ${page.page_url}`,
+    };
+  } catch (error) {
+    await client.query("ROLLBACK");
+    return {
+      success: false,
+      message: `Failed to update page: ${error instanceof Error ? error.message : "Unknown error"}`,
+    };
+  } finally {
+    client.release();
+  }
+}
+
+/**
+ * @readonly
+ */
+export async function deletePage(page_url: string): Promise<DocOperationResult> {
+  const client = await pool.connect();
+  try {
+    const result = await client.query("DELETE FROM pql_docs.doc_content WHERE page_url = $1", [page_url]);
+
+    if (result.rowCount === 0) {
+      return {
+        success: false,
+        message: `Page not found: ${page_url}`,
+      };
+    }
+
+    return {
+      success: true,
+      message: `Successfully deleted page: ${page_url}`,
+    };
+  } catch (error) {
+    return {
+      success: false,
+      message: `Failed to delete page: ${error instanceof Error ? error.message : "Unknown error"}`,
+    };
+  } finally {
+    client.release();
+  }
+}
+
+async function generateChunksAndEmbeddings(client: any, docContentId: string, page: Doc_Page): Promise<void> {
+  const chunks = chunkContent(page.content, 500);
+
+  for (const chunk of chunks) {
+    const embedding = await transformQueryIntoEmbedding(chunk.content);
+
+    await client.query(
+      `INSERT INTO pql_docs.doc_chunk 
+       (doc_content_id_fk, chunk_content, page_title, page_url, page_description, 
+        chunk_line_start, chunk_line_end, embedding)
+       VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
+      [
+        docContentId,
+        chunk.content,
+        page.title,
+        page.page_url,
+        page.description,
+        chunk.lineStart,
+        chunk.lineEnd,
+        `[${embedding.values.join(",")}]`,
+      ]
+    );
+  }
+}
+
+function chunkContent(content: string, chunkSize: number): DocChunk[] {
+  const chunks: DocChunk[] = [];
+  const lines = content.split("\n");
+  let currentChunk = "";
+  let lineStart = 0;
+  let currentLine = 0;
+
+  for (const line of lines) {
+    if (currentChunk.length + line.length > chunkSize && currentChunk.length > 0) {
+      chunks.push({
+        content: currentChunk.trim(),
+        lineStart,
+        lineEnd: currentLine - 1,
+      });
+      currentChunk = line + "\n";
+      lineStart = currentLine;
+    } else {
+      currentChunk += line + "\n";
+    }
+    currentLine++;
+  }
+
+  if (currentChunk.trim()) {
+    chunks.push({
+      content: currentChunk.trim(),
+      lineStart,
+      lineEnd: currentLine - 1,
+    });
+  }
+
+  return chunks;
+}