Add crawled urls

.github/workflows/check-and-bump-versions.yaml

@@ -273,7 +273,7 @@ jobs:
                 NON_VERSION_CHANGES=$(echo "$DIFF" | grep -v '"version":' | grep '^[+-]' | wc -l)
 
                 # If there are no non-version changes, return true (1)
-                if [ "$NON_VERSION_CHANGES" -eq 0; then
+                if [ "$NON_VERSION_CHANGES" -eq 0 ]; then
                   return 0
                 fi
                 return 1

apps/db/supabase/migrations/20250210132933_new_migration.sql

@@ -124,4 +124,153 @@ create policy "Allow workspace members to have full permissions" on "public"."wo
                 (ws.id = workspace_crawlers.ws_id)
         )
     )
-);
+);
+
+create table "public"."crawled_url_next_urls" (
+    "origin_id" uuid not null default gen_random_uuid(),
+    "url" text not null,
+    "skipped" boolean not null,
+    "created_at" timestamp with time zone not null default now()
+);
+
+create table "public"."crawled_urls" (
+    "id" uuid not null default gen_random_uuid(),
+    "url" text not null,
+    "html" text,
+    "markdown" text,
+    "created_at" timestamp with time zone not null default now()
+);
+
+CREATE UNIQUE INDEX crawled_url_next_urls_pkey ON public.crawled_url_next_urls USING btree (origin_id, url);
+
+CREATE UNIQUE INDEX crawled_urls_pkey ON public.crawled_urls USING btree (id);
+
+alter table
+    "public"."crawled_url_next_urls"
+add
+    constraint "crawled_url_next_urls_pkey" PRIMARY KEY using index "crawled_url_next_urls_pkey";
+
+alter table
+    "public"."crawled_urls"
+add
+    constraint "crawled_urls_pkey" PRIMARY KEY using index "crawled_urls_pkey";
+
+grant delete on table "public"."crawled_url_next_urls" to "anon";
+
+grant
+insert
+    on table "public"."crawled_url_next_urls" to "anon";
+
+grant references on table "public"."crawled_url_next_urls" to "anon";
+
+grant
+select
+    on table "public"."crawled_url_next_urls" to "anon";
+
+grant trigger on table "public"."crawled_url_next_urls" to "anon";
+
+grant truncate on table "public"."crawled_url_next_urls" to "anon";
+
+grant
+update
+    on table "public"."crawled_url_next_urls" to "anon";
+
+grant delete on table "public"."crawled_url_next_urls" to "authenticated";
+
+grant
+insert
+    on table "public"."crawled_url_next_urls" to "authenticated";
+
+grant references on table "public"."crawled_url_next_urls" to "authenticated";
+
+grant
+select
+    on table "public"."crawled_url_next_urls" to "authenticated";
+
+grant trigger on table "public"."crawled_url_next_urls" to "authenticated";
+
+grant truncate on table "public"."crawled_url_next_urls" to "authenticated";
+
+grant
+update
+    on table "public"."crawled_url_next_urls" to "authenticated";
+
+grant delete on table "public"."crawled_url_next_urls" to "service_role";
+
+grant
+insert
+    on table "public"."crawled_url_next_urls" to "service_role";
+
+grant references on table "public"."crawled_url_next_urls" to "service_role";
+
+grant
+select
+    on table "public"."crawled_url_next_urls" to "service_role";
+
+grant trigger on table "public"."crawled_url_next_urls" to "service_role";
+
+grant truncate on table "public"."crawled_url_next_urls" to "service_role";
+
+grant
+update
+    on table "public"."crawled_url_next_urls" to "service_role";
+
+grant delete on table "public"."crawled_urls" to "anon";
+
+grant
+insert
+    on table "public"."crawled_urls" to "anon";
+
+grant references on table "public"."crawled_urls" to "anon";
+
+grant
+select
+    on table "public"."crawled_urls" to "anon";
+
+grant trigger on table "public"."crawled_urls" to "anon";
+
+grant truncate on table "public"."crawled_urls" to "anon";
+
+grant
+update
+    on table "public"."crawled_urls" to "anon";
+
+grant delete on table "public"."crawled_urls" to "authenticated";
+
+grant
+insert
+    on table "public"."crawled_urls" to "authenticated";
+
+grant references on table "public"."crawled_urls" to "authenticated";
+
+grant
+select
+    on table "public"."crawled_urls" to "authenticated";
+
+grant trigger on table "public"."crawled_urls" to "authenticated";
+
+grant truncate on table "public"."crawled_urls" to "authenticated";
+
+grant
+update
+    on table "public"."crawled_urls" to "authenticated";
+
+grant delete on table "public"."crawled_urls" to "service_role";
+
+grant
+insert
+    on table "public"."crawled_urls" to "service_role";
+
+grant references on table "public"."crawled_urls" to "service_role";
+
+grant
+select
+    on table "public"."crawled_urls" to "service_role";
+
+grant trigger on table "public"."crawled_urls" to "service_role";
+
+grant truncate on table "public"."crawled_urls" to "service_role";
+
+grant
+update
+    on table "public"."crawled_urls" to "service_role";

apps/external/package.json

@@ -10,9 +10,9 @@
   },
   "dependencies": {
     "@tutur3u/ai": "^0.0.9",
-    "@tutur3u/eslint-config": "^0.1.1",
+    "@tutur3u/eslint-config": "^0.1.2",
     "@tutur3u/supabase": "^0.0.3",
-    "@tutur3u/types": "^0.1.1",
+    "@tutur3u/types": "^0.1.3",
     "@tutur3u/typescript-config": "^0.1.0",
     "@tutur3u/ui": "^0.0.3",
     "next": "15.1.6",

apps/mira/package.json

@@ -108,7 +108,7 @@
     "framer-motion": "^12.4.2",
     "highlight.js": "^11.11.1",
     "html2canvas": "^1.4.1",
-    "jotai": "^2.11.3",
+    "jotai": "^2.12.0",
     "juice": "^11.0.0",
     "lodash": "^4.17.21",
     "lowlight": "^3.3.0",

apps/mira/src/components/providers.tsx

@@ -6,25 +6,7 @@ export function Providers({ children }: { children: ReactNode }) {
   return (
     <ThemeProvider
       attribute="class"
-      themes={[
-        'system',
-
-        'light',
-        'light-pink',
-        'light-purple',
-        'light-yellow',
-        'light-orange',
-        'light-green',
-        'light-blue',
-
-        'dark',
-        'dark-pink',
-        'dark-purple',
-        'dark-yellow',
-        'dark-orange',
-        'dark-green',
-        'dark-blue',
-      ]}
+      themes={['system', 'light', 'dark']}
       enableColorScheme={false}
       enableSystem
     >

apps/nova/package.json

@@ -108,7 +108,7 @@
     "framer-motion": "^12.4.2",
     "highlight.js": "^11.11.1",
     "html2canvas": "^1.4.1",
-    "jotai": "^2.11.3",
+    "jotai": "^2.12.0",
     "juice": "^11.0.0",
     "lodash": "^4.17.21",
     "lowlight": "^3.3.0",

apps/nova/src/app/[locale]/(dashboard)/[wsId]/layout.tsx

@@ -1,7 +1,7 @@
 import { Sidebar } from '@/components/layout/sidebar';
-import { ThemeProvider } from '@/components/theme-provider';
 import { Toaster } from '@tutur3u/ui/toaster';
 import type { Metadata } from 'next';
+import { ThemeProvider } from 'next-themes';
 import { Inter } from 'next/font/google';
 
 const inter = Inter({ subsets: ['latin'] });
@@ -18,7 +18,13 @@ export default function RootLayout({
 }) {
   return (
     <div className={`${inter.className} bg-background text-foreground`}>
-      <ThemeProvider attribute="class" defaultTheme="system" enableSystem>
+      <ThemeProvider
+        attribute="class"
+        defaultTheme="dark"
+        themes={['system', 'light', 'dark']}
+        enableColorScheme={false}
+        enableSystem
+      >
         <div className="flex h-screen overflow-hidden">
           <Sidebar />
           <main className="flex-1 overflow-y-auto">{children}</main>

apps/rewise/src/components/providers.tsx

@@ -7,25 +7,7 @@ export function Providers({ children }: { children: ReactNode }) {
     <ThemeProvider
       attribute="class"
       defaultTheme="dark"
-      themes={[
-        'system',
-
-        'light',
-        'light-pink',
-        'light-purple',
-        'light-yellow',
-        'light-orange',
-        'light-green',
-        'light-blue',
-
-        'dark',
-        'dark-pink',
-        'dark-purple',
-        'dark-yellow',
-        'dark-orange',
-        'dark-green',
-        'dark-blue',
-      ]}
+      themes={['system', 'light', 'dark']}
       enableColorScheme={false}
       enableSystem
     >

apps/web/.env.example

@@ -28,3 +28,8 @@ MODAL_TOKEN_SECRET=YOUR_MODAL_TOKEN_SECRET
 
 CF_ACCOUNT_ID=YOUR_CF_ACCOUNT_ID
 CF_API_TOKEN=YOUR_CF_API_TOKEN
+
+# Infrastructure Credentials
+SCRAPER_URL=YOUR_SCRAPER_URL
+PROXY_API_KEY=YOUR_PROXY_API_KEY
+NEXT_PUBLIC_PROXY_API_KEY=YOUR_NEXT_PUBLIC_PROXY_API_KEY

apps/web/package.json

@@ -116,7 +116,7 @@
     "framer-motion": "^12.4.2",
     "highlight.js": "^11.11.1",
     "html2canvas": "^1.4.1",
-    "jotai": "^2.11.3",
+    "jotai": "^2.12.0",
     "juice": "^11.0.0",
     "lodash": "^4.17.21",
     "lowlight": "^3.3.0",

apps/web/src/app/[locale]/(dashboard)/[wsId]/crawlers/(default)/page.tsx

@@ -2,7 +2,6 @@ import { getColumns } from '../columns';
 import ModelForm from '../form';
 import { CustomDataTable } from '@/components/custom-data-table';
 import { createClient } from '@tutur3u/supabase/next/server';
-import type { WorkspaceCrawler } from '@tutur3u/types/db';
 import FeatureSummary from '@tutur3u/ui/custom/feature-summary';
 import { Separator } from '@tutur3u/ui/separator';
 import { getTranslations } from 'next-intl/server';
@@ -31,11 +30,6 @@ export default async function WorkspaceCrawlersPage({
   const { locale, wsId } = await params;
   const { data, count } = await getData(wsId, await searchParams);
 
-  const crawlers = data.map((m) => ({
-    ...m,
-    href: `/${wsId}/crawlers/${m.id}`,
-  })) as WorkspaceCrawler[];
-
   return (
     <>
       <FeatureSummary
@@ -48,7 +42,7 @@ export default async function WorkspaceCrawlersPage({
       />
       <Separator className="my-4" />
       <CustomDataTable
-        data={crawlers}
+        data={data}
         namespace="user-data-table"
         columnGenerator={getColumns}
         extraData={{ locale, wsId }}

apps/web/src/app/[locale]/(dashboard)/[wsId]/crawlers/[crawlerId]/crawl-button.tsx

@@ -0,0 +1,48 @@
+'use client';
+
+import { Button } from '@tutur3u/ui/button';
+import { useState } from 'react';
+
+export default function CrawlButton({
+  wsId,
+  url,
+  onSuccess,
+}: {
+  wsId: string;
+  url: string;
+  onSuccess?: () => void;
+}) {
+  const [isLoading, setIsLoading] = useState(false);
+
+  return (
+    <Button
+      onClick={async () => {
+        try {
+          setIsLoading(true);
+          const res = await fetch(`/api/v1/workspaces/${wsId}/crawl`, {
+            method: 'POST',
+            headers: {
+              'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({ url }),
+          });
+
+          if (!res.ok) {
+            console.error('Failed to crawl:', res.statusText);
+            return;
+          }
+
+          await res.json();
+          onSuccess?.();
+        } catch (error) {
+          console.error('Error during crawl:', error);
+        } finally {
+          setIsLoading(false);
+        }
+      }}
+      disabled={isLoading}
+    >
+      {isLoading ? 'Crawling...' : 'Crawl'}
+    </Button>
+  );
+}