Fix nightly, improve browser feature check and misc other improvements (#8702)

- wkorg screenshot test failed because of changed view configuration
defaults --> the wkorg test now overwrites the defaults on each run.
this should be in our best interest, anyway, because it ensures good
defaults. the downside is that we have to adapt the code if we want to
tune the default.
- flatMap was not supported by the browser used on browserstack. I fixed
this by specifying a newer osx version.
- flatMap on generators is now checked in the browser feature tests so
that users are properly alerted about a necessary browser update
- errors in the model initialization are detected in the screenshot
tests now so that there is a better error message (and an eager quit on
errors). until now, the test just waited until the timeout was exceeded.
- there were two problems with float32 datasets:
- the "auto clipping" feature did not work with infinite values (which
exist in the float32 test dataset)
- osx ventura did have shader problems with wide float32 ranges. using
sequoia fixes this.

### URL of deployed dev instance (used for testing):
- https://___.webknossos.xyz

### Steps to test:
- see this run
https://github.com/scalableminds/webknossos/actions/runs/15821624973

### Issues:
- fixes failing nightly

------
(Please delete unneeded items, merge only when none are left open)
- [X] Added changelog entry (create a `$PR_NUMBER.md` file in
`unreleased_changes` or use `./tools/create-changelog-entry.py`)

Author: philippotto

frontend/javascripts/admin/rest_api.ts

@@ -1068,10 +1068,12 @@ export function getDatasetDefaultConfiguration(datasetId: string): Promise<Datas
 export function updateDatasetDefaultConfiguration(
   datasetId: string,
   datasetConfiguration: DatasetConfiguration,
+  options?: RequestOptions,
 ): Promise<ArbitraryObject> {
   return Request.sendJSONReceiveJSON(`/api/datasetConfigurations/default/${datasetId}`, {
     method: "PUT",
     data: datasetConfiguration,
+    ...options,
   });
 }
 

frontend/javascripts/components/brain_spinner.tsx

@@ -75,7 +75,7 @@ export function BrainSpinnerWithError({
     <BrainSpinner
       message={
         <>
-          <div style={{ textAlign: "center" }}>
+          <div className="initialization-error-message" style={{ textAlign: "center" }}>
             {gotUnhandledError ? messages["tracing.unhandled_initialization_error"] : message}
           </div>
           <div className="flex-center-child" style={{ gap: 8 }}>

frontend/javascripts/libs/browser_feature_check.tsx

@@ -11,6 +11,7 @@ export default function checkBrowserFeatures() {
     new Map([[1, 2]]).values().map((v) => v);
     [].at(0);
     new Set().difference(new Set());
+    [].values().flatMap((el) => [el, el]);
   } catch (exception) {
     Toast.warning(
       <div>

frontend/javascripts/main.tsx

@@ -86,7 +86,6 @@ async function loadOrganization() {
     Store.dispatch(setActiveOrganizationAction(organization));
   }
 }
-
 document.addEventListener("DOMContentLoaded", async () => {
   ErrorHandling.initialize({
     throwAssertions: false,

frontend/javascripts/test/puppeteer/dataset_rendering.wkorg_screenshot.ts

@@ -1,6 +1,7 @@
 import path from "node:path";
 import { compareScreenshot, isPixelEquivalent } from "./screenshot_helpers";
 import {
+  getDefaultRequestOptions,
   getNewPage,
   screenshotDatasetView,
   type ScreenshotTestContext,
@@ -10,6 +11,9 @@ import {
 } from "./dataset_rendering_helpers";
 import { encodeUrlHash } from "viewer/controller/url_manager";
 import { describe, it, beforeEach, afterEach, expect } from "vitest";
+import { updateDatasetDefaultConfiguration } from "admin/rest_api";
+import type { DatasetConfiguration } from "viewer/store";
+import type { BLEND_MODES } from "viewer/constants";
 
 process.on("unhandledRejection", (err, promise) => {
   console.error("Unhandled rejection (promise: ", promise, ", reason: ", err, ").");
@@ -42,7 +46,59 @@ const viewOverrides: Record<string, string> = {
   ),
 };
 
-describe("WebKnossos.org Dataset Rendering", () => {
+// The following configuration will be set during the test.
+// The tested dataset is *the* example dataset for WK which users
+// are guided to from the landing page. We want to ensure good defaults
+// and in case they were changed accidentally, it is good that they are
+// reset automatically each night.
+// If we want different defaults, they should be set here.
+const datasetConfiguration: DatasetConfiguration = {
+  layers: {
+    color: {
+      alpha: 100,
+      gammaCorrectionValue: 1,
+      min: 0,
+      color: [255, 255, 255],
+      max: 255,
+      isInverted: false,
+      isInEditMode: false,
+      isDisabled: false,
+      intensityRange: [74, 165],
+    },
+    predictions: {
+      alpha: 50,
+      gammaCorrectionValue: 1,
+      min: 0,
+      color: [255, 255, 255],
+      max: 255,
+      isInverted: false,
+      isInEditMode: false,
+      isDisabled: true,
+      intensityRange: [103, 199],
+    },
+    segmentation: {
+      alpha: 20,
+      gammaCorrectionValue: 1,
+      mapping: null,
+      color: [255, 255, 255],
+      isInverted: false,
+      isInEditMode: false,
+      isDisabled: false,
+    },
+  },
+  fourBit: false,
+  interpolation: false,
+  segmentationPatternOpacity: 25,
+  colorLayerOrder: ["color", "predictions"],
+  position: [2924, 4474, 1770],
+  zoom: 0.9,
+  blendMode: "Additive" as BLEND_MODES,
+  nativelyRenderedLayerName: null,
+  loadingStrategy: "PROGRESSIVE_QUALITY",
+  renderMissingDataBlack: true,
+};
+
+describe("webknossos.org Dataset Rendering", () => {
   beforeEach<ScreenshotTestContext>(async (context) => {
     await setupBeforeEach(context);
   });
@@ -51,42 +107,47 @@ describe("WebKnossos.org Dataset Rendering", () => {
     await setupAfterEach(context);
   });
 
-  it.sequential<ScreenshotTestContext>(
-    `should render dataset ${demoDatasetName} correctly`,
-    async ({ browser }) => {
-      await withRetry(
-        3,
-        async () => {
-          const response = await fetch(
-            `${URL}/api/datasets/disambiguate/${owningOrganization}/${demoDatasetName}/toId`,
-          );
-          const { id: datasetId } = await response.json();
+  it<ScreenshotTestContext>(`should render dataset ${demoDatasetName} correctly`, async ({
+    browser,
+  }) => {
+    await withRetry(
+      3,
+      async () => {
+        const response = await fetch(
+          `${URL}/api/datasets/disambiguate/${owningOrganization}/${demoDatasetName}/toId`,
+        );
+        const { id: datasetId } = await response.json();
 
-          const page = await getNewPage(browser);
-          const { screenshot, width, height } = await screenshotDatasetView(
-            page,
-            URL,
-            datasetId,
-            viewOverrides[demoDatasetName],
-          );
-          const changedPixels = await compareScreenshot(
-            screenshot,
-            width,
-            height,
-            SCREENSHOTS_BASE_PATH,
-            demoDatasetName,
-          );
-          await page.close();
+        await updateDatasetDefaultConfiguration(
+          datasetId,
+          datasetConfiguration,
+          getDefaultRequestOptions(URL),
+        );
 
-          return isPixelEquivalent(changedPixels, width, height);
-        },
-        (condition) => {
-          expect(
-            condition,
-            `Dataset with name: "${demoDatasetName}" does not look the same, see ${demoDatasetName}.diff.png for the difference and ${demoDatasetName}.new.png for the new screenshot.`,
-          ).toBe(true);
-        },
-      );
-    },
-  );
+        const page = await getNewPage(browser);
+        const { screenshot, width, height } = await screenshotDatasetView(
+          page,
+          URL,
+          datasetId,
+          viewOverrides[demoDatasetName],
+        );
+        const changedPixels = await compareScreenshot(
+          screenshot,
+          width,
+          height,
+          SCREENSHOTS_BASE_PATH,
+          demoDatasetName,
+        );
+        await page.close();
+
+        return isPixelEquivalent(changedPixels, width, height);
+      },
+      (condition) => {
+        expect(
+          condition,
+          `Dataset with name: "${demoDatasetName}" does not look the same, see ${demoDatasetName}.diff.png for the difference and ${demoDatasetName}.new.png for the new screenshot.`,
+        ).toBe(true);
+      },
+    );
+  });
 });

frontend/javascripts/test/puppeteer/dataset_rendering_helpers.ts

@@ -243,7 +243,20 @@ async function waitForTracingViewLoad(page: Page) {
       console.log("Waiting suspiciously long for page to load...");
     }
     await sleep(500);
-    inputCatchers = await page.$(".inputcatcher");
+    const inputCatcherPromise = page
+      .$(".inputcatcher")
+      .then((elements) => ({ type: "inputcatchers", elements }));
+    const errorPromise = page
+      .$(".initialization-error-message")
+      .then((elements) => ({ type: "error", elements }));
+
+    const raceResult = await Promise.race([inputCatcherPromise, errorPromise]);
+
+    if (raceResult.type === "error") {
+      throw new Error("Initialization error detected");
+    }
+
+    inputCatchers = raceResult.elements;
   }
 }
 
@@ -466,7 +479,7 @@ export async function setupBeforeEach(context: ScreenshotTestContext) {
       browser: "chrome",
       browser_version: "latest",
       os: "os x",
-      os_version: "mojave",
+      os_version: "sequoia",
       name: context.task.name, // add test name to BrowserStack session
       "browserstack.username": process.env.BROWSERSTACK_USERNAME,
       "browserstack.accessKey": process.env.BROWSERSTACK_ACCESS_KEY,

frontend/javascripts/test/screenshots-wkorg/l4dense_motta_et_al_demo.png

@@ -67,9 +67,11 @@ async function getClippingValues(
 
   const localHist = new Map();
   for (let i = 0; i < dataForAllViewPorts.length; i++) {
-    if (dataForAllViewPorts[i] !== 0) {
-      const value = localHist.get(dataForAllViewPorts[i]);
-      localHist.set(dataForAllViewPorts[i], value != null ? value + 1 : 1);
+    const value = dataForAllViewPorts[i];
+    // Don't use Number.abs because value can be BigInt
+    if (value !== 0 && value !== Number.POSITIVE_INFINITY && value !== Number.NEGATIVE_INFINITY) {
+      const counter = localHist.get(dataForAllViewPorts[i]);
+      localHist.set(dataForAllViewPorts[i], counter != null ? counter + 1 : 1);
     }
   }
 

frontend/javascripts/test/screenshots/2017-05-31_mSEM_aniso-test.png

@@ -66,7 +66,9 @@ def main():
     with open(file_path, "w") as f:
         f.write(TEMPLATE)
 
-    print(f"✅ Created changelog entry: {file_path}")
+    print(
+        f"✅ Created new file for a changelog entry. Please adapt its content here: {file_path}"
+    )
 
 
 if __name__ == "__main__":

frontend/javascripts/test/screenshots/ROI2017_wkw_with_mapping_astrocyte.png

@@ -0,0 +1,3 @@
+### Fixed
+- Fixed the sandbox mode for annotations.
+- Fixed the nightly screenshot tests.

frontend/javascripts/test/screenshots/test-agglomerate-file_with_meshes_link.png

@@ -20,7 +20,7 @@ object ObjectId extends FoxImplicits {
     parseCommaSeparated(idsStrOpt)(fromString)
   private def fromBsonId(bson: BSONObjectID) = ObjectId(bson.stringify)
   def fromStringSync(input: String): Option[ObjectId] = BSONObjectID.parse(input).map(fromBsonId).toOption
-  def dummyId: ObjectId = ObjectId("dummyObjectId")
+  def dummyId: ObjectId = ObjectId("000000000000000000000000")
 
   implicit object ObjectIdFormat extends Format[ObjectId] {
     override def reads(json: JsValue): JsResult[ObjectId] =