Feat/add progress (#18)

* bump fsrs-browser to 0.6.2

* update train in client-side

* update train in server-side

* update dependencies

* Feat/add Train Progress

* update own train

* Fix/timezones react-hydration-error

* Fix/not shown progress

* Feat/enabled thread pool

* set custom timezone

* update training.png

* value -> selected

* Fix/unmatched state

Author: ishiko732

images/training.png

@@ -34,5 +34,22 @@ const nextConfig = {
 
     return config;
   },
+  async headers() {
+    return [
+      {
+        source: "/(.*)",
+        headers: [
+          {
+            key: "Cross-Origin-Embedder-Policy",
+            value: "require-corp",
+          },
+          {
+            key: "Cross-Origin-Opener-Policy",
+            value: "same-origin",
+          },
+        ],
+      },
+    ];
+  }
 };
 module.exports = nextConfig;

next.config.js

@@ -21,7 +21,7 @@
     "@vercel/speed-insights": "^1.0.2",
     "bcrypt": "^5.1.1",
     "clsx": "^2.0.0",
-    "fsrs-browser": "^0.6.1",
+    "fsrs-browser": "^0.6.2",
     "next": "14.1.0",
     "next-auth": "^4.24.5",
     "next-themes": "^0.2.1",

package.json

@@ -1,17 +1,20 @@
 import { NextRequest, NextResponse } from "next/server";
 import { Readable } from "stream";
 import { getProcessW, loadCsvAndTrain } from "./train";
+import { computerMinuteOffset } from "@/lib/date";
 
 export async function GET(request: NextRequest) {
   const url = new URL(request.url);
   const searchParams = new URLSearchParams(url.search);
 
   const fileURL = searchParams.get("url") ?? `${url.origin}/revlog.csv`;
+  const timezone = searchParams.get("timezone") ?? "UTC";
+  const nextDayStart = searchParams.get("nextDayStart") ?? "4";
+  const offset = computerMinuteOffset(timezone, Number(nextDayStart));
   const stream = await getReadStream(fileURL);
 
   const wasmURL = new URL("./fsrs_browser_bg.wasm", url.origin);
-  const { w, ...others } = await loadCsvAndTrain(wasmURL, stream);
-
+  const { w, ...others } = await loadCsvAndTrain(wasmURL, stream, offset);
 
   return NextResponse.json({
     data: fileURL,
@@ -30,11 +33,11 @@ export async function POST(request: NextRequest) {
   const filename = file.name.replaceAll(" ", "_");
   const buffer = Buffer.from(await file.arrayBuffer());
   const stream = Readable.from(buffer);
-  const wasmURL = new URL(
-    "fsrs_browser_bg.wasm",
-    new URL(request.url).origin
-  );
-  const { w, ...others } = await loadCsvAndTrain(wasmURL, stream);
+  const wasmURL = new URL("fsrs_browser_bg.wasm", new URL(request.url).origin);
+  const timezone = formData.get("timezone") ?? "UTC";
+  const nextDayStart = formData.get("nextDayStart") ?? "4";
+  const offset = computerMinuteOffset(String(timezone), Number(nextDayStart));
+  const { w, ...others } = await loadCsvAndTrain(wasmURL, stream, offset);
 
   return NextResponse.json({
     fileName: filename,

pnpm-lock.yaml

@@ -10,7 +10,11 @@ Error.stackTraceLimit = 30;
 //   self.postMessage(result);
 // };
 
-export async function loadCsvAndTrain(wasmURL: URL, file: papa.LocalFile) {
+export async function loadCsvAndTrain(
+  wasmURL: URL,
+  file: papa.LocalFile,
+  minute_offset: number
+) {
   const cids: bigint[] = [];
   const eases: number[] = [];
   const ids: bigint[] = [];
@@ -28,11 +32,12 @@ export async function loadCsvAndTrain(wasmURL: URL, file: papa.LocalFile) {
         eases.push(Number(data.review_rating));
         types.push(Number(data.review_state));
       },
-      complete:async function () {
+      complete: async function () {
         const loadEndTime = performance.now();
         const trainStartTime = performance.now();
         const w = await computeParameters(
           wasmURL,
+          minute_offset,
           new BigInt64Array(cids),
           new Uint8Array(eases),
           new BigInt64Array(ids),
@@ -56,6 +61,7 @@ export async function loadCsvAndTrain(wasmURL: URL, file: papa.LocalFile) {
 
 export async function computeParameters(
   wasmURL: URL,
+  minute_offset: number,
   cids: BigInt64Array,
   eases: Uint8Array,
   ids: BigInt64Array,
@@ -65,14 +71,19 @@ export async function computeParameters(
   //   await initThreadPool(cpus().length);
   let fsrs = new Fsrs();
   console.time("full training time");
-  let parameters = fsrs.computeParametersAnki(cids, eases, ids, types);
+  let parameters = fsrs.computeParametersAnki(
+    minute_offset,
+    cids,
+    eases,
+    ids,
+    types
+  );
   console.timeEnd("full training time");
   fsrs.free();
   console.log(parameters);
   return parameters;
 }
 
-
 export function getProcessW(w: Float32Array) {
   const processed_w = [];
   for (let i = 0; i < w.length; i++) {

public/fsrs_browser_bg.wasm

@@ -12,4 +12,26 @@ interface TrainResult {
   trainTime: string;
   totalTime: string;
   count: number;
-}
+}
+
+interface ProgressStart {
+  tag: "start";
+  wasmMemoryBuffer: ArrayBuffer;
+  pointer: number;
+}
+
+interface ProgressFinish {
+  tag: "finish";
+  parameters: Float32Array;
+}
+
+interface ProgressItem {
+  itemsProcessed: number;
+  itemsTotal: number;
+}
+
+interface Progress extends ProgressItem {
+  tag: "progress";
+}
+
+type ProgressState = Progress | ProgressStart | ProgressFinish;

src/app/api/fsrs/train/route.ts

@@ -1,16 +1,20 @@
 import TrainDisplay from "@/components/train/display";
 import FileTrain from "@/components/train/file-train-button";
+import NextDayStartAt from "@/components/train/nextDayStartAt";
 import OwnTrain from "@/components/train/own-train";
-import TrainTestButton from "@/components/train/test-button";
+import TrainProgress from "@/components/train/progress";
+import TimezoneSelector from "@/components/train/timezones";
 import TrainProvider from "@/context/TrainContext";
 export default async function Page() {
   return (
     <div className="flex justify-center flex-col items-center">
       <TrainProvider>
+        <TimezoneSelector />
+        <NextDayStartAt />
         <OwnTrain />
         <FileTrain />
         <TrainDisplay />
-        <TrainTestButton/>
+        <TrainProgress />
       </TrainProvider>
     </div>
   );

src/app/api/fsrs/train/train.ts

@@ -2,20 +2,32 @@
 
 import { getProcessW } from "@/app/api/fsrs/train/train";
 import { useTrainContext } from "@/context/TrainContext";
+import { computerMinuteOffset } from "@/lib/date";
 import { useEffect, useRef } from "react";
 
 export default function FileTrain() {
   const workerRef = useRef<Worker>();
-  const { loading, setLoading, setW, setLoadTime, setTrainTime, setTotalTime } =
-    useTrainContext();
+  const timeIdRef = useRef<NodeJS.Timeout>();
+  const {
+    loading,
+    setLoading,
+    setW,
+    setLoadTime,
+    setTrainTime,
+    setTotalTime,
+    timezone,
+    nextDayStart,
+    handleProgress,
+  } = useTrainContext();
 
   const handleClick = async (e: React.ChangeEvent<HTMLInputElement>) => {
     if (!e.target.files) {
       return;
     }
     setLoading(true);
     const file = e.target.files[0];
-    workerRef.current?.postMessage(file);
+    const offset = computerMinuteOffset(timezone, nextDayStart);
+    workerRef.current?.postMessage({ file, offset });
     if (e.target.value) {
       e.target.value = "";
     }
@@ -25,13 +37,32 @@ export default function FileTrain() {
     workerRef.current = new Worker(
       new URL("./fsrs_worker.ts", import.meta.url)
     );
-    workerRef.current.onmessage = (event: MessageEvent<TrainResult>) => {
-      console.log(event.data)
-      setW(getProcessW(event.data.w));
-      setLoadTime(event.data.loadTime)
-      setTrainTime(event.data.trainTime);
-      setTotalTime(event.data.totalTime);
-      setLoading(false);
+    workerRef.current.onmessage = (
+      event: MessageEvent<TrainResult | ProgressState>
+    ) => {
+      console.log(event.data);
+      if ("tag" in event.data) {
+        // process ProgressState
+        const progressState = event.data as ProgressState;
+        if (progressState.tag === "start") {
+          const { wasmMemoryBuffer, pointer } = progressState;
+          handleProgress(wasmMemoryBuffer, pointer);
+          timeIdRef.current = setInterval(() => {
+            handleProgress(wasmMemoryBuffer, pointer);
+          }, 100);
+        } else if (progressState.tag === "finish") {
+          clearInterval(timeIdRef.current);
+          console.log("finish");
+        }
+      } else {
+        // process TrainResult
+        const trainResult = event.data as TrainResult;
+        setW(getProcessW(trainResult.w));
+        setLoadTime(trainResult.loadTime);
+        setTrainTime(trainResult.trainTime);
+        setTotalTime(trainResult.totalTime);
+        setLoading(false);
+      }
     };
     return () => {
       workerRef.current?.terminate();