Cyclic dependency

frontend/package.json

@@ -4,8 +4,11 @@
   "version": "0.3.0",
   "type": "module",
   "scripts": {
-    "dev": "vite",
-    "build": "tsc -b && vite build  && rm -rf ../server/dist && mv dist ../server/.",
+    "check-deps": "madge --circular --extensions ts,tsx ./src",
+    "dev": "bun run check-deps && vite",
+    "dev:unsafe": "vite",
+    "build": "bun run check-deps && tsc -b && vite build  && rm -rf ../server/dist && mv dist ../server/.",
+    "build:unsafe": "tsc -b && vite build  && rm -rf ../server/dist && mv dist ../server/.",
     "lint": "eslint .",
     "preview": "vite preview",
     "test": "vitest",
@@ -83,6 +86,7 @@
     "eslint-plugin-react-refresh": "^0.4.16",
     "globals": "^15.14.0",
     "jsdom": "^26.1.0",
+    "madge": "^8.0.0",
     "postcss": "^8.4.49",
     "tailwindcss": "^3.4.17",
     "typescript": "^5.7.2",

server/config.ts

@@ -11,7 +11,7 @@ let host = process.env.HOST || "http://localhost:3000"
 
 // Centralized database URL construction
 function getDatabaseUrl(): string {
-    return process.env.DATABASE_URL || `$`
+    return process.env.DATABASE_URL || `postgres://xyne:xyne@${postgresBaseHost}:5432/xyne`
 }
 let redirectUri = process.env.GOOGLE_REDIRECT_URI!
 let postOauthRedirect = "/"

server/package.json

@@ -22,6 +22,7 @@
     "cz-conventional-changelog": "^3.3.0",
     "drizzle-kit": "0.31.4",
     "husky": "^9.1.7",
+    "madge": "^8.0.0",
     "p-queue": "^8.0.1",
     "semantic-release": "^24.2.7"
   },
@@ -32,9 +33,13 @@
     "generate": "drizzle-kit generate",
     "migrate": "drizzle-kit migrate",
     "migrate:kb": "bun run scripts/backfillclFd.ts",
-    "dev": "bun run --watch server.ts",
-    "dev:sync": "bun run --watch sync-server.ts",
-    "build": "bunx --bun vite build",
+    "check-deps": "madge --circular --extensions ts,tsx ./",
+    "dev": "bun run check-deps && bun run --watch server.ts",
+    "dev:unsafe": "bun run --watch server.ts",
+    "dev:sync": "bun run check-deps && bun run --watch sync-server.ts",
+    "dev:sync-unsafe": "bun run --watch sync-server.ts",
+    "build": "bun run check-deps && bunx --bun vite build",
+    "build:unsafe": "bunx --bun vite build",
     "lint": "eslint .",
     "preview": "vite preview",
     "format": "bunx biome format --write ../",

server/queue/boss.ts

@@ -0,0 +1,7 @@
+import PgBoss from "pg-boss"
+import config from "@/config"
+
+export const boss = new PgBoss({
+  connectionString: config.getDatabaseUrl(),
+  monitorStateIntervalMinutes: 10, // Monitor state every 10 minutes
+})

server/queue/index.ts

@@ -31,11 +31,9 @@ import {
 const Logger = getLogger(Subsystem.Queue)
 const JobExpiryHours = config.JobExpiryHours
 
-const url = `postgres://xyne:xyne@${config.postgresBaseHost}:5432/xyne`
-export const boss = new PgBoss({
-  connectionString: url,
-  monitorStateIntervalMinutes: 10, // Monitor state every minute
-})
+import { boss } from "./boss"
+
+export { boss }
 
 // run it if we are re-doing ingestion
 // await boss.clearStorage()

server/queue/toolSync.ts

@@ -1,6 +1,5 @@
 import { getLogger } from "@/logger"
 import { Subsystem, type TxnOrClient } from "@/types"
-import { boss } from "."
 import { db, getConnectorByApp } from "@/db/connector"
 import { syncConnectorTools } from "@/db/tool"
 import { getErrorMessage } from "@/utils"

server/search/utils.ts

@@ -7,7 +7,6 @@ import {
   type VespaQueryConfig,
   type CollectionVespaIds,
 } from "@xyne/vespa-ts/types"
-import { getFolderItems } from "./vespa"
 import { db } from "@/db/connector"
 import {
   getAllFolderItems,
@@ -17,6 +16,7 @@ import {
   getCollectionFoldersItemIds,
 } from "@/db/knowledgeBase"
 import type { SelectAgent } from "@/db/agent"
+import { sharedVespaService } from "./vespaService"
 
 const Logger = getLogger(Subsystem.Vespa).child({ module: "search-utils" })
 
@@ -86,7 +86,7 @@ export async function extractDriveIds(
     if (curr) driveIds.push(curr)
     if (curr && email) {
       try {
-        const folderItem = await getFolderItems(
+        const folderItem = await sharedVespaService.getFolderItems(
           [curr],
           fileSchema,
           DriveEntity.Folder,

server/search/vespa.ts

@@ -1,65 +1,21 @@
 import { getLogger } from "@/logger"
 import { Subsystem } from "@/types"
-import {
-  createVespaService,
-  createDefaultConfig,
-  type VespaDependencies,
-} from "@xyne/vespa-ts"
-import config, { CLUSTER, NAMESPACE } from "@/config"
 import {
   Apps,
-  chatContainerSchema,
-  chatMessageSchema,
-  chatUserSchema,
   DriveEntity,
-  eventSchema,
-  fileSchema,
-  mailAttachmentSchema,
-  mailSchema,
-  userSchema,
   type Entity,
   type VespaQueryConfig,
 } from "@xyne/vespa-ts/types"
+import config from "@/config"
 import { db } from "@/db/client"
 import { getConnectorByAppAndEmailId } from "@/db/connector"
 import { AuthType, ConnectorStatus } from "@/shared/types"
 import { extractDriveIds, extractCollectionVespaIds } from "./utils"
 import { getAppSyncJobsByEmail } from "@/db/syncJob"
-// Define your Vespa endpoint and schema name
+import { sharedVespaService as vespa } from "./vespaService"
 
 const Logger = getLogger(Subsystem.Vespa).child({ module: "vespa" })
 
-const vespaConfig = createDefaultConfig({
-  vespaBaseHost: config.vespaBaseHost,
-  page: config.VespaPageSize,
-  isDebugMode: config.isDebugMode,
-  namespace: NAMESPACE,
-  cluster: CLUSTER,
-  vespaMaxRetryAttempts: config.vespaMaxRetryAttempts,
-  vespaRetryDelay: config.vespaRetryDelay,
-})
-const AllSources = [
-  fileSchema,
-  userSchema,
-  mailSchema,
-  eventSchema,
-  mailAttachmentSchema,
-  chatUserSchema,
-  chatMessageSchema,
-  chatContainerSchema,
-  // Not adding datasource or datasource_file to AllSources by default,
-  // as they are for a specific app functionality.
-  // dataSourceFileSchema and collection file schemas are intentionally excluded from search
-]
-const dependencies: VespaDependencies = {
-  logger: Logger,
-  config: vespaConfig,
-  sourceSchemas: AllSources,
-  vespaEndpoint: config.vespaEndpoint,
-}
-
-const vespa = createVespaService(dependencies)
-
 export const insert = vespa.insert.bind(vespa)
 export const GetDocument = vespa.GetDocument.bind(vespa)
 export const getDocumentOrNull = vespa.getDocumentOrNull.bind(vespa)

server/search/vespaService.ts

@@ -0,0 +1,51 @@
+import { getLogger } from "@/logger"
+import { Subsystem } from "@/types"
+import {
+  createVespaService,
+  createDefaultConfig,
+  type VespaDependencies,
+} from "@xyne/vespa-ts"
+import config, { CLUSTER, NAMESPACE } from "@/config"
+import {
+  fileSchema,
+  userSchema,
+  mailSchema,
+  eventSchema,
+  mailAttachmentSchema,
+  chatUserSchema,
+  chatMessageSchema,
+  chatContainerSchema,
+} from "@xyne/vespa-ts/types"
+
+const Logger = getLogger(Subsystem.Vespa).child({ module: "vespa-service" })
+
+const vespaConfig = createDefaultConfig({
+  vespaBaseHost: config.vespaBaseHost,
+  page: config.VespaPageSize,
+  isDebugMode: config.isDebugMode,
+  namespace: NAMESPACE,
+  cluster: CLUSTER,
+  vespaMaxRetryAttempts: config.vespaMaxRetryAttempts,
+  vespaRetryDelay: config.vespaRetryDelay,
+})
+
+const AllSources = [
+  fileSchema,
+  userSchema,
+  mailSchema,
+  eventSchema,
+  mailAttachmentSchema,
+  chatUserSchema,
+  chatMessageSchema,
+  chatContainerSchema,
+]
+
+const dependencies: VespaDependencies = {
+  logger: Logger,
+  config: vespaConfig,
+  sourceSchemas: AllSources,
+  vespaEndpoint: config.vespaEndpoint,
+}
+
+// Create a single shared vespa service instance
+export const sharedVespaService = createVespaService(dependencies)