From f46cbfdfefb22a4e81576120016328e882f11570 Mon Sep 17 00:00:00 2001
From: eyelidlessness <gnosis@gmail.com>
Date: Thu, 17 Apr 2025 07:38:58 -0700
Subject: [PATCH] Substantially narrower design for adding media to label
 interfaces

---
 .../xforms-engine/src/client/TextRange.ts     | 40 +++++++++++++++++++
 .../src/client/media/AudioMediaResource.ts    |  6 +++
 .../src/client/media/BaseMediaResource.ts     | 21 ++++++++++
 .../media/BaseStreamableMediaResource.ts      | 18 +++++++++
 .../src/client/media/ImageMediaResource.ts    | 16 ++++++++
 .../src/client/media/MediaResource.ts         | 15 +++++++
 .../src/client/media/MediaResourceURL.ts      |  5 +++
 .../src/client/media/VideoMediaResource.ts    |  6 +++
 8 files changed, 127 insertions(+)
 create mode 100644 packages/xforms-engine/src/client/media/AudioMediaResource.ts
 create mode 100644 packages/xforms-engine/src/client/media/BaseMediaResource.ts
 create mode 100644 packages/xforms-engine/src/client/media/BaseStreamableMediaResource.ts
 create mode 100644 packages/xforms-engine/src/client/media/ImageMediaResource.ts
 create mode 100644 packages/xforms-engine/src/client/media/MediaResource.ts
 create mode 100644 packages/xforms-engine/src/client/media/MediaResourceURL.ts
 create mode 100644 packages/xforms-engine/src/client/media/VideoMediaResource.ts

diff --git a/packages/xforms-engine/src/client/TextRange.ts b/packages/xforms-engine/src/client/TextRange.ts
index b490c2e7e..44b541056 100644
--- a/packages/xforms-engine/src/client/TextRange.ts
+++ b/packages/xforms-engine/src/client/TextRange.ts
@@ -1,4 +1,7 @@
 import type { ActiveLanguage } from './FormLanguage.ts';
+import type { AudioMediaResource } from './media/AudioMediaResource.ts';
+import type { ImageMediaResource } from './media/ImageMediaResource.ts';
+import type { VideoMediaResource } from './media/VideoMediaResource.ts';
 import type { RootNodeState } from './RootNode.ts';
 
 /**
@@ -57,6 +60,19 @@ import type { RootNodeState } from './RootNode.ts';
  * opaque (as in, the `jr:itext` implementation is encapsulated in the `xpath`
  * package, and the engine doesn't really deal with itext translations at the
  * node level at all).
+ *
+ * @todo Once media is parsed ({@link TextRange.audio}, {@link TextRange.image},
+ * {@link TextRange.video}), the 'translation' case will be parsed as a node
+ * rather than as a string. That will unblock output-in-itext. If a minimal
+ * first pass is to retain the current 'translation' interface, it will need to
+ * reproduce the bug preventing output-in-itext functionality by identifying the
+ * (potentially media-adjacent) `<value>` node and casting it back to a string.
+ * Future iterations could then remove this reproduction-of-bug, allowing
+ * output-in-itext to work as expected... at which point the design for the
+ * 'translation' will have increased depth (i.e. establishing {@link TextRange}
+ * as a tree rather than a list). Alternatively, the 'translation' case could be
+ * hoisted up to {@link TextRange} to preserve the current depth (until support
+ * for Markdown would, potentially, establish a tree).
  */
 // prettier-ignore
 export type TextChunkSource =
@@ -147,6 +163,26 @@ export type TextOrigin =
  * reasoned about differently by clients depending on their role (for instance,
  * a text range's role may correspond to the "short" or "guidance" `form` of a
  * {@link https://getodk.github.io/xforms-spec/#languages | translation}).
+ *
+ * @todo In following revisions, this could:
+ *
+ * - be renamed `RichText` to better represent its semantics
+ * - support cases with multiple text variants (`<label>`/`<value
+ *   form="short">`; `<hint>`: `<value form="guidance">`)
+ *
+ * Supporting the latter would imply adding one layer of depth so that the
+ * {@link TextChunk} (or whatever appropriate future naming) descendants
+ * representing a single case could represent multiple cases, e.g.
+ *
+ * ```ts
+ * get text(): {
+ *   // ... some way to access `TextChunk`s
+ * };
+ *
+ * get shortText(): {
+ *   // ... some way to access `TextChunk`s
+ * }
+ * ```
  */
 export interface TextRange<Role extends TextRole, Origin extends TextOrigin = TextOrigin> {
 	readonly origin: Origin;
@@ -154,6 +190,10 @@ export interface TextRange<Role extends TextRole, Origin extends TextOrigin = Te
 
 	[Symbol.iterator](): Iterable<TextChunk>;
 
+	get audio(): AudioMediaResource | null;
+	get image(): ImageMediaResource | null;
+	get video(): VideoMediaResource | null;
+
 	get asString(): string;
 	get formatted(): unknown;
 }
diff --git a/packages/xforms-engine/src/client/media/AudioMediaResource.ts b/packages/xforms-engine/src/client/media/AudioMediaResource.ts
new file mode 100644
index 000000000..ec35cf15b
--- /dev/null
+++ b/packages/xforms-engine/src/client/media/AudioMediaResource.ts
@@ -0,0 +1,6 @@
+import type { MediaResourceTypeAudio } from './BaseMediaResource.ts';
+import type { BaseStreamableMediaResource } from './BaseStreamableMediaResource.ts';
+
+export interface AudioMediaResource extends BaseStreamableMediaResource {
+	readonly resourceType: MediaResourceTypeAudio;
+}
diff --git a/packages/xforms-engine/src/client/media/BaseMediaResource.ts b/packages/xforms-engine/src/client/media/BaseMediaResource.ts
new file mode 100644
index 000000000..28a7ef9d6
--- /dev/null
+++ b/packages/xforms-engine/src/client/media/BaseMediaResource.ts
@@ -0,0 +1,21 @@
+import type { Awaitable } from '@getodk/common/types/helpers.js';
+
+export type MediaResourceTypeAudio = 'MEDIA_RESOURCE_TYPE_AUDIO';
+export type MediaResourceTypeVideo = 'MEDIA_RESOURCE_TYPE_VIDEO';
+export type MediaResourceTypeImage = 'MEDIA_RESOURCE_TYPE_IMAGE';
+
+export type MediaResourceType =
+	| MediaResourceTypeAudio
+	| MediaResourceTypeImage
+	| MediaResourceTypeVideo;
+
+export interface BaseMediaResourceDetail {
+	readonly fileName: string;
+
+	getResourceData(): Awaitable<Blob>;
+}
+
+export interface BaseMediaResource {
+	readonly resourceType: MediaResourceType;
+	readonly detail: BaseMediaResourceDetail;
+}
diff --git a/packages/xforms-engine/src/client/media/BaseStreamableMediaResource.ts b/packages/xforms-engine/src/client/media/BaseStreamableMediaResource.ts
new file mode 100644
index 000000000..da167bd50
--- /dev/null
+++ b/packages/xforms-engine/src/client/media/BaseStreamableMediaResource.ts
@@ -0,0 +1,18 @@
+import type { UnknownObject } from '@getodk/common/lib/type-assertions/assertUnknownObject.ts';
+import type { Awaitable } from '@getodk/common/types/helpers.d.ts';
+import type { BaseMediaResource, BaseMediaResourceDetail } from './BaseMediaResource.ts';
+import type { MediaResourceURL } from './MediaResourceURL.ts';
+
+export interface BaseStreamableMediaResourceDetail
+	extends BaseMediaResourceDetail,
+		// Note: we still have research to do into how we'll support streaming! Even
+		// this interface is fairly speculative. It exists mainly to signal intent,
+		// and as a baseline of the minimum expected surface/responsibilities.
+		UnknownObject {
+	getStream(): Awaitable<ReadableStream>;
+	getURL(): Awaitable<MediaResourceURL>;
+}
+
+export interface BaseStreamableMediaResource extends BaseMediaResource {
+	readonly detail: BaseStreamableMediaResourceDetail;
+}
diff --git a/packages/xforms-engine/src/client/media/ImageMediaResource.ts b/packages/xforms-engine/src/client/media/ImageMediaResource.ts
new file mode 100644
index 000000000..387367547
--- /dev/null
+++ b/packages/xforms-engine/src/client/media/ImageMediaResource.ts
@@ -0,0 +1,16 @@
+import type {
+	BaseMediaResource,
+	BaseMediaResourceDetail,
+	MediaResourceTypeImage,
+} from './BaseMediaResource.ts';
+import type { MediaResourceURL } from './MediaResourceURL.ts';
+
+export interface ImageMediaResourceDetail extends BaseMediaResourceDetail {
+	readonly url: MediaResourceURL;
+	readonly bigImageURL: MediaResourceURL | null;
+}
+
+export interface ImageMediaResource extends BaseMediaResource {
+	readonly resourceType: MediaResourceTypeImage;
+	readonly detail: ImageMediaResourceDetail;
+}
diff --git a/packages/xforms-engine/src/client/media/MediaResource.ts b/packages/xforms-engine/src/client/media/MediaResource.ts
new file mode 100644
index 000000000..7b99e0bd7
--- /dev/null
+++ b/packages/xforms-engine/src/client/media/MediaResource.ts
@@ -0,0 +1,15 @@
+import type { AudioMediaResource } from './AudioMediaResource.ts';
+import type { MediaResourceType } from './BaseMediaResource.ts';
+import type { ImageMediaResource } from './ImageMediaResource.ts';
+import type { VideoMediaResource } from './VideoMediaResource.ts';
+
+interface MediaResourceByTypeSuffix {
+	readonly MEDIA_RESOURCE_TYPE_AUDIO: AudioMediaResource;
+	readonly MEDIA_RESOURCE_TYPE_IMAGE: ImageMediaResource;
+	readonly MEDIA_RESOURCE_TYPE_VIDEO: VideoMediaResource;
+}
+
+// prettier-ignore
+export type MediaResource<
+	ResourceType extends MediaResourceType = MediaResourceType,
+> = MediaResourceByTypeSuffix[ResourceType];
diff --git a/packages/xforms-engine/src/client/media/MediaResourceURL.ts b/packages/xforms-engine/src/client/media/MediaResourceURL.ts
new file mode 100644
index 000000000..ea701eaa3
--- /dev/null
+++ b/packages/xforms-engine/src/client/media/MediaResourceURL.ts
@@ -0,0 +1,5 @@
+import type { PartiallyKnownString } from '@getodk/common/types/string/PartiallyKnownString.ts';
+
+export interface MediaResourceURL extends URL {
+	readonly protocol: PartiallyKnownString<'blob:' | 'data:'>;
+}
diff --git a/packages/xforms-engine/src/client/media/VideoMediaResource.ts b/packages/xforms-engine/src/client/media/VideoMediaResource.ts
new file mode 100644
index 000000000..e67d6cc73
--- /dev/null
+++ b/packages/xforms-engine/src/client/media/VideoMediaResource.ts
@@ -0,0 +1,6 @@
+import type { MediaResourceTypeVideo } from './BaseMediaResource.ts';
+import type { BaseStreamableMediaResource } from './BaseStreamableMediaResource.ts';
+
+export interface VideoMediaResource extends BaseStreamableMediaResource {
+	readonly resourceType: MediaResourceTypeVideo;
+}