Substantially narrower design for adding media to label interfaces

packages/xforms-engine/src/client/TextRange.ts

@@ -1,4 +1,7 @@
 import type { ActiveLanguage } from './FormLanguage.ts';
+import type { AudioMediaResource } from './media/AudioMediaResource.ts';
+import type { ImageMediaResource } from './media/ImageMediaResource.ts';
+import type { VideoMediaResource } from './media/VideoMediaResource.ts';
 import type { RootNodeState } from './RootNode.ts';
 
 /**
@@ -57,6 +60,19 @@ import type { RootNodeState } from './RootNode.ts';
  * opaque (as in, the `jr:itext` implementation is encapsulated in the `xpath`
  * package, and the engine doesn't really deal with itext translations at the
  * node level at all).
+ *
+ * @todo Once media is parsed ({@link TextRange.audio}, {@link TextRange.image},
+ * {@link TextRange.video}), the 'translation' case will be parsed as a node
+ * rather than as a string. That will unblock output-in-itext. If a minimal
+ * first pass is to retain the current 'translation' interface, it will need to
+ * reproduce the bug preventing output-in-itext functionality by identifying the
+ * (potentially media-adjacent) `<value>` node and casting it back to a string.
+ * Future iterations could then remove this reproduction-of-bug, allowing
+ * output-in-itext to work as expected... at which point the design for the
+ * 'translation' will have increased depth (i.e. establishing {@link TextRange}
+ * as a tree rather than a list). Alternatively, the 'translation' case could be
+ * hoisted up to {@link TextRange} to preserve the current depth (until support
+ * for Markdown would, potentially, establish a tree).
  */
 // prettier-ignore
 export type TextChunkSource =
@@ -147,13 +163,37 @@ export type TextOrigin =
  * reasoned about differently by clients depending on their role (for instance,
  * a text range's role may correspond to the "short" or "guidance" `form` of a
  * {@link https://getodk.github.io/xforms-spec/#languages | translation}).
+ *
+ * @todo In following revisions, this could:
+ *
+ * - be renamed `RichText` to better represent its semantics
+ * - support cases with multiple text variants (`<label>`/`<value
+ *   form="short">`; `<hint>`: `<value form="guidance">`)
+ *
+ * Supporting the latter would imply adding one layer of depth so that the
+ * {@link TextChunk} (or whatever appropriate future naming) descendants
+ * representing a single case could represent multiple cases, e.g.
+ *
+ * ```ts
+ * get text(): {
+ *   // ... some way to access `TextChunk`s
+ * };
+ *
+ * get shortText(): {
+ *   // ... some way to access `TextChunk`s
+ * }
+ * ```
  */
 export interface TextRange<Role extends TextRole, Origin extends TextOrigin = TextOrigin> {
 	readonly origin: Origin;
 	readonly role: Role;
 
 	[Symbol.iterator](): Iterable<TextChunk>;
 
+	get audio(): AudioMediaResource | null;
+	get image(): ImageMediaResource | null;
+	get video(): VideoMediaResource | null;
+
 	get asString(): string;
 	get formatted(): unknown;
 }

packages/xforms-engine/src/client/media/AudioMediaResource.ts

@@ -0,0 +1,6 @@
+import type { MediaResourceTypeAudio } from './BaseMediaResource.ts';
+import type { BaseStreamableMediaResource } from './BaseStreamableMediaResource.ts';
+
+export interface AudioMediaResource extends BaseStreamableMediaResource {
+	readonly resourceType: MediaResourceTypeAudio;
+}

packages/xforms-engine/src/client/media/BaseMediaResource.ts

@@ -0,0 +1,21 @@
+import type { Awaitable } from '@getodk/common/types/helpers.js';
+
+export type MediaResourceTypeAudio = 'MEDIA_RESOURCE_TYPE_AUDIO';
+export type MediaResourceTypeVideo = 'MEDIA_RESOURCE_TYPE_VIDEO';
+export type MediaResourceTypeImage = 'MEDIA_RESOURCE_TYPE_IMAGE';
+
+export type MediaResourceType =
+	| MediaResourceTypeAudio
+	| MediaResourceTypeImage
+	| MediaResourceTypeVideo;
+
+export interface BaseMediaResourceDetail {
+	readonly fileName: string;
+
+	getResourceData(): Awaitable<Blob>;
+}
+
+export interface BaseMediaResource {
+	readonly resourceType: MediaResourceType;
+	readonly detail: BaseMediaResourceDetail;
+}

packages/xforms-engine/src/client/media/BaseStreamableMediaResource.ts

@@ -0,0 +1,18 @@
+import type { UnknownObject } from '@getodk/common/lib/type-assertions/assertUnknownObject.ts';
+import type { Awaitable } from '@getodk/common/types/helpers.d.ts';
+import type { BaseMediaResource, BaseMediaResourceDetail } from './BaseMediaResource.ts';
+import type { MediaResourceURL } from './MediaResourceURL.ts';
+
+export interface BaseStreamableMediaResourceDetail
+	extends BaseMediaResourceDetail,
+		// Note: we still have research to do into how we'll support streaming! Even
+		// this interface is fairly speculative. It exists mainly to signal intent,
+		// and as a baseline of the minimum expected surface/responsibilities.
+		UnknownObject {
+	getStream(): Awaitable<ReadableStream>;
+	getURL(): Awaitable<MediaResourceURL>;
+}
+
+export interface BaseStreamableMediaResource extends BaseMediaResource {
+	readonly detail: BaseStreamableMediaResourceDetail;
+}

packages/xforms-engine/src/client/media/ImageMediaResource.ts

@@ -0,0 +1,16 @@
+import type {
+	BaseMediaResource,
+	BaseMediaResourceDetail,
+	MediaResourceTypeImage,
+} from './BaseMediaResource.ts';
+import type { MediaResourceURL } from './MediaResourceURL.ts';
+
+export interface ImageMediaResourceDetail extends BaseMediaResourceDetail {
+	readonly url: MediaResourceURL;
+	readonly bigImageURL: MediaResourceURL | null;
+}
+
+export interface ImageMediaResource extends BaseMediaResource {
+	readonly resourceType: MediaResourceTypeImage;
+	readonly detail: ImageMediaResourceDetail;
+}

packages/xforms-engine/src/client/media/MediaResource.ts

@@ -0,0 +1,15 @@
+import type { AudioMediaResource } from './AudioMediaResource.ts';
+import type { MediaResourceType } from './BaseMediaResource.ts';
+import type { ImageMediaResource } from './ImageMediaResource.ts';
+import type { VideoMediaResource } from './VideoMediaResource.ts';
+
+interface MediaResourceByTypeSuffix {
+	readonly MEDIA_RESOURCE_TYPE_AUDIO: AudioMediaResource;
+	readonly MEDIA_RESOURCE_TYPE_IMAGE: ImageMediaResource;
+	readonly MEDIA_RESOURCE_TYPE_VIDEO: VideoMediaResource;
+}
+
+// prettier-ignore
+export type MediaResource<
+	ResourceType extends MediaResourceType = MediaResourceType,
+> = MediaResourceByTypeSuffix[ResourceType];

packages/xforms-engine/src/client/media/MediaResourceURL.ts

@@ -0,0 +1,5 @@
+import type { PartiallyKnownString } from '@getodk/common/types/string/PartiallyKnownString.ts';
+
+export interface MediaResourceURL extends URL {
+	readonly protocol: PartiallyKnownString<'blob:' | 'data:'>;
+}

packages/xforms-engine/src/client/media/VideoMediaResource.ts

@@ -0,0 +1,6 @@
+import type { MediaResourceTypeVideo } from './BaseMediaResource.ts';
+import type { BaseStreamableMediaResource } from './BaseStreamableMediaResource.ts';
+
+export interface VideoMediaResource extends BaseStreamableMediaResource {
+	readonly resourceType: MediaResourceTypeVideo;
+}