Merge remote-tracking branch 'origin/master' into chat-threads-smaller

Author: haraldschilly

src/README.md

@@ -64,7 +64,6 @@ If you export the PORT environment variable, that determines what port everythin
 
 CoCalc also runs a NATS server listening on two ports on localhost, one for TCP and one for WebSocket connections.  To avoid conflicts, you can customize their ports by setting the environment variables `COCALC_NATS_PORT` (default 4222), and `COCALC_NATS_WS_PORT` (default 8443).
 
-
 **Note**: If you installed `pnpm` locally (instead of globally), simply run `npm run` in place of `pnpm` to execute
 these commands via [NPM run scripts](https://docs.npmjs.com/cli/v10/using-npm/scripts).
 
@@ -144,7 +143,7 @@ You can also just type `pnpm psql` :
 ~/cocalc/src$ pnpm psql
 ```
 
-NOTE:  As of Jan 2023, CoCalc should fully work with any version of PostgreSQL from version 10.x onward.  However, obviously at some point we will stop supporting PostgreSQL v 10.
+NOTE:  As of Jan 2023, CoCalc should fully work with any version of PostgreSQL from version 14.x onward.  However, obviously at some point we will stop supporting PostgreSQL v 14.
 
 ### 2. More about Starting the Hub
 
@@ -186,6 +185,14 @@ which installs exactly the right packages, and builds the code.
 
 See `packages/backend/data.ts` . In particular, you can set BASE_PATH, DATA, PGHOST, PGDATA, PROJECTS, SECRETS to override the defaults. Data is stored in `cocalc/src/data/` by default.
 
+For NATS when doing development, it can be useful to set these in .bashrc so NATS uses ports of your choosing that are available.
+
+```sh
+export COCALC_NATS_SERVER_NAME=localhost
+export COCALC_NATS_PORT=5007
+export COCALC_NATS_WS_PORT=5008
+```
+
 #### File System Build Caching
 
 There are two types of file system build caching. These greatly improve the time to compile typescript or start webpack between runs. However, in rare cases bugs may lead to weird broken behavior. Here's where the caches are, so you know how to clear them to check if this is the source of trouble. _As of now, I'm_ _**not**_ _aware of any bugs in file system caching._

src/package.json

@@ -9,6 +9,7 @@
     "build-dev": "./workspaces.py install && ./workspaces.py build --dev",
     "clean": "rm -rf packages/node_modules && ./workspaces.py clean && cd compute/compute && pnpm clean ",
     "hub": "cd packages/hub && npm run hub-project-dev-nobuild",
+    "hub-prod": "cd packages/hub && npm run hub-project-prod-nobuild",
     "rspack": "cd packages/static && pnpm watch",
     "psql": "export PGHOST=${PGHOST:=$INIT_CWD/data/postgres/socket}; PGUSER='smc' psql",
     "database": "cd dev/project && ./start_postgres.py",

src/packages/backend/nats/test/time.test.ts

@@ -1,7 +1,7 @@
 /*
 DEVELOPMENT:
 
-pnpm exec jest --watch --forceExit --detectOpenHandles "time.test.ts"
+pnpm test ./time.test.ts
 */
 
 // this sets client
@@ -28,3 +28,15 @@ describe("get time from nats", () => {
     expect(typeof time()).toBe("number");
   });
 });
+
+import { timeClient, createTimeService } from "@cocalc/nats/service/time";
+
+describe("start the time server and client and test that it works", () => {
+  it("starts the time server and queries it", async () => {
+    await import("@cocalc/backend/nats");
+    createTimeService();
+    const client = timeClient();
+    const t = await client.time();
+    expect(Math.abs(Date.now() - t)).toBeLessThan(200);
+  });
+});

src/packages/database/postgres-user-queries.coffee

@@ -899,7 +899,9 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext
     # This will avoid a possible subtle edge case if user is cheating and always somehow
     # crashes server...?
     _user_set_query_project_change_before: (old_val, new_val, account_id, cb) =>
-        dbg = @_dbg("_user_set_query_project_change_before #{account_id}, #{misc.to_json(old_val)} --> #{misc.to_json(new_val)}")
+        #dbg = @_dbg("_user_set_query_project_change_before #{account_id}, #{misc.to_json(old_val)} --> #{misc.to_json(new_val)}")
+        # I've seen MASSIVE OUTPUT from this, e.g., when setting avatar.
+        dbg = @_dbg("_user_set_query_project_change_before #{account_id}")
         dbg()
 
         if new_val?.name and (new_val?.name != old_val?.name)

src/packages/frontend/chat/message.tsx

@@ -41,7 +41,6 @@ import { Name } from "./name";
 import { Time } from "./time";
 import { ChatMessageTyped, Mode, SubmitMentionsFn } from "./types";
 import {
-  getReplyToRoot,
   getThreadRootDate,
   is_editing,
   message_colors,

src/packages/frontend/client/project.ts

@@ -54,6 +54,7 @@ import httpApi from "./api";
 import { WebappClient } from "./client";
 import { throttle } from "lodash";
 import { writeFile, type WriteFileOptions } from "@cocalc/nats/files/write";
+import { readFile, type ReadFileOptions } from "@cocalc/nats/files/read";
 
 export class ProjectClient {
   private client: WebappClient;
@@ -99,6 +100,18 @@ export class ProjectClient {
     return await writeFile(opts);
   };
 
+  // readFile -- read **arbitrarily large text or binary files**
+  // from a project via a readable stream.
+  // Look at the code below if you want to stream a file for memory
+  // efficiency...
+  readFile = async (opts: ReadFileOptions): Promise<Buffer> => {
+    const chunks: Uint8Array[] = [];
+    for await (const chunk of await readFile(opts)) {
+      chunks.push(chunk);
+    }
+    return Buffer.concat(chunks);
+  };
+
   public async read_text_file({
     project_id,
     path,

src/packages/frontend/components/time-elapsed.tsx

@@ -4,32 +4,32 @@
  */
 
 import React from "react";
-import * as misc from "@cocalc/util/misc";
 import { useInterval } from "react-interval-hook";
 
+import { seconds2hms, server_time } from "@cocalc/util/misc";
+
 interface Props {
   start_ts: number;
   interval_s?: number;
 }
 
-function isSame(prev, next) {
-  if (prev == null || next == null) return false;
-  return prev.start_ts != next.start_ts;
-}
-
-export const TimeElapsed: React.FC<Props> = React.memo((props: Props) => {
-  const { start_ts, interval_s = 1 } = props;
+export function TimeElapsed({ start_ts, interval_s = 1 }: Props) {
+  const [elapsed, setElapsed] = React.useState<string>(getUptimeStr());
 
-  const [elapsed, setElapsed] = React.useState("");
+  function getUptimeStr() {
+    if (start_ts == null) return "";
+    const delta_s = (server_time().getTime() - start_ts) / 1000;
+    const uptime_str = seconds2hms(delta_s, true);
+    return uptime_str;
+  }
 
   useInterval(() => {
-    if (start_ts == null) return;
-    const delta_s = (misc.server_time().getTime() - start_ts) / 1000;
-    const uptime_str = misc.seconds2hms(delta_s, true);
-    setElapsed(uptime_str);
+    const next = getUptimeStr();
+    if (!next) return;
+    setElapsed(next);
   }, interval_s * 1000);
 
   if (start_ts == null) return null;
 
   return <React.Fragment>{elapsed}</React.Fragment>;
-}, isSame);
+}

src/packages/frontend/cspell.json

@@ -35,6 +35,7 @@
     "syncdoc",
     "syncdoc",
     "syncstring",
+    "synctable",
     "synctables",
     "tidyverse",
     "timetravel",