Shared Tree: Persisted Schema Format v2 with persisted metadata support

.changeset/brown-dingos-switch.md

@@ -0,0 +1,8 @@
+---
+"@fluidframework/tree": minor
+"fluid-framework": minor
+"__section": tree
+---
+Add APIs for declaring "persisted" schema metadata
+
+Add alpha APIs for declaring node and field schema metadata which future versions of the Fluid Framework will provide a way to opt into persisting in the document.

packages/dds/tree/src/core/index.ts

@@ -138,11 +138,13 @@ export {
 	storedEmptyFieldSchema,
 	type StoredSchemaCollection,
 	schemaFormatV1,
+	schemaFormatV2,
 	LeafNodeStoredSchema,
 	ObjectNodeStoredSchema,
 	MapNodeStoredSchema,
 	decodeFieldSchema,
-	encodeFieldSchema,
+	encodeFieldSchemaV1,
+	encodeFieldSchemaV2,
 	storedSchemaDecodeDispatcher,
 	type SchemaAndPolicy,
 	Multiplicity,

packages/dds/tree/src/core/schema-stored/formatV2.ts

@@ -0,0 +1,78 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { type ObjectOptions, type Static, Type } from "@sinclair/typebox";
+
+import { JsonCompatibleReadOnlySchema } from "../../util/index.js";
+import {
+	FieldKindIdentifierSchema,
+	PersistedValueSchema,
+	TreeNodeSchemaIdentifierSchema,
+} from "./formatV1.js";
+import { unionOptions } from "../../codec/index.js";
+
+export const PersistedMetadataFormat = Type.Optional(JsonCompatibleReadOnlySchema);
+
+const FieldSchemaFormatBase = Type.Object({
+	kind: FieldKindIdentifierSchema,
+	types: Type.Array(TreeNodeSchemaIdentifierSchema),
+	metadata: PersistedMetadataFormat,
+});
+
+const noAdditionalProps: ObjectOptions = { additionalProperties: false };
+
+export const FieldSchemaFormat = Type.Composite([FieldSchemaFormatBase], noAdditionalProps);
+
+/**
+ * Format for the content of a {@link TreeNodeStoredSchema}.
+ *
+ * See {@link DiscriminatedUnionDispatcher} for more information on this pattern.
+ */
+export const TreeNodeSchemaUnionFormat = Type.Object(
+	{
+		/**
+		 * Object node union member.
+		 */
+		object: Type.Optional(Type.Record(Type.String(), FieldSchemaFormat)),
+		/**
+		 * Map node union member.
+		 */
+		map: Type.Optional(FieldSchemaFormat),
+		/**
+		 * Leaf node union member.
+		 */
+		leaf: Type.Optional(Type.Enum(PersistedValueSchema)),
+	},
+	unionOptions,
+);
+
+export type TreeNodeSchemaUnionFormat = Static<typeof TreeNodeSchemaUnionFormat>;
+
+/**
+ * Format for {@link TreeNodeStoredSchema}.
+ *
+ * See {@link DiscriminatedUnionDispatcher} for more information on this pattern.
+ */
+export const TreeNodeSchemaDataFormat = Type.Object(
+	{
+		/**
+		 * Node kind specific data.
+		 */
+		kind: TreeNodeSchemaUnionFormat,
+
+		// Data in common for all TreeNode schemas:
+		/**
+		 * Leaf node union member.
+		 */
+		metadata: PersistedMetadataFormat,
+	},
+	noAdditionalProps,
+);
+
+export type TreeNodeSchemaDataFormat = Static<typeof TreeNodeSchemaDataFormat>;
+
+export type FieldSchemaFormat = Static<typeof FieldSchemaFormat>;
+
+export type PersistedMetadataFormat = Static<typeof PersistedMetadataFormat>;

packages/dds/tree/src/core/schema-stored/index.ts

@@ -18,7 +18,8 @@ export {
 	ObjectNodeStoredSchema,
 	MapNodeStoredSchema,
 	decodeFieldSchema,
-	encodeFieldSchema,
+	encodeFieldSchemaV1,
+	encodeFieldSchemaV2,
 	storedSchemaDecodeDispatcher,
 	type SchemaAndPolicy,
 	type SchemaPolicy,
@@ -37,3 +38,5 @@ export type { TreeNodeSchemaIdentifier, FieldKey, FieldKindIdentifier } from "./
 
 import * as schemaFormatV1 from "./formatV1.js";
 export { schemaFormatV1 };
+import * as schemaFormatV2 from "./formatV2.js";
+export { schemaFormatV2 };

packages/dds/tree/src/core/schema-stored/schema.ts

@@ -11,20 +11,32 @@ import { type MakeNominal, brand, invertMap } from "../../util/index.js";
 import {
 	type FieldKey,
 	type FieldKindIdentifier,
-	type FieldSchemaFormat,
+	type FieldSchemaFormat as FieldSchemaFormatV1,
 	PersistedValueSchema,
-	type TreeNodeSchemaDataFormat,
+	type TreeNodeSchemaDataFormat as TreeNodeSchemaDataFormatV1,
 	type TreeNodeSchemaIdentifier,
 } from "./formatV1.js";
+import type {
+	FieldSchemaFormat as FieldSchemaFormatV2,
+	PersistedMetadataFormat,
+	TreeNodeSchemaUnionFormat,
+	TreeNodeSchemaDataFormat as TreeNodeSchemaDataFormatV2,
+} from "./formatV2.js";
 import type { Multiplicity } from "./multiplicity.js";
 
 /**
  * The format version for the schema.
  */
 export enum SchemaVersion {
 	v1 = 1,
+	/**
+	 * Adds persisted metadata to the node schema and field schema.
+	 */
+	v2 = 2,
 }
 
+type FieldSchemaFormat = FieldSchemaFormatV1 | FieldSchemaFormatV2;
+
 /**
  * Schema for what {@link TreeLeafValue} is allowed on a Leaf node.
  * @privateRemarks
@@ -128,6 +140,13 @@ export interface TreeFieldStoredSchema {
 	 * If not specified, types are unconstrained.
 	 */
 	readonly types: TreeTypeSet;
+
+	/**
+	 * Portion of the metadata which can be persisted.
+	 * @remarks
+	 * Discarded when encoding to {@link SchemaFormatVersion.V1}.
+	 */
+	readonly metadata: PersistedMetadataFormat | undefined;
 }
 
 /**
@@ -151,6 +170,7 @@ export const storedEmptyFieldSchema: TreeFieldStoredSchema = {
 	kind: brand(forbiddenFieldKindIdentifier),
 	// This type set also forces the field to be empty not not allowing any types as all.
 	types: new Set(),
+	metadata: undefined,
 };
 
 /**
@@ -164,12 +184,21 @@ export abstract class TreeNodeStoredSchema {
 	protected _typeCheck!: MakeNominal;
 
 	/**
-	 * @privateRemarks
-	 * Returns TreeNodeSchemaDataFormat.
-	 * This is uses an opaque type to avoid leaking these types out of the package,
-	 * and is runtime validated by the codec.
+	 * Constructor for a TreeNodeStoredSchema.
+	 * @param metadata - Persisted metadata for this node schema.
+	 */
+	public constructor(public readonly metadata: PersistedMetadataFormat | undefined) {}
+
+	/**
+	 * Encode in the v1 schema format.
 	 */
-	public abstract encode(): TreeNodeSchemaDataFormat;
+	public abstract encodeV1(): TreeNodeSchemaDataFormatV1;
+
+	/**
+	 * Encode in the v2 schema format.
+	 * @remarks Post-condition: if metadata was specified on the input schema, it will be present in the output.
+	 */
+	public abstract encodeV2(): TreeNodeSchemaDataFormatV2;
 
 	/**
 	 * Returns the schema for the provided field.
@@ -190,29 +219,55 @@ export class ObjectNodeStoredSchema extends TreeNodeStoredSchema {
 	 */
 	public constructor(
 		public readonly objectNodeFields: ReadonlyMap<FieldKey, TreeFieldStoredSchema>,
+		metadata?: PersistedMetadataFormat | undefined,
 	) {
-		super();
+		super(metadata);
 	}
 
-	public override encode(): TreeNodeSchemaDataFormat {
+	public override encodeV1(): TreeNodeSchemaDataFormatV1 {
 		const fieldsObject: Record<string, FieldSchemaFormat> = Object.create(null);
 		// Sort fields to ensure output is identical for for equivalent schema (since field order is not considered significant).
 		// This makes comparing schema easier, and ensures chunk reuse for schema summaries isn't needlessly broken.
 		for (const key of [...this.objectNodeFields.keys()].sort()) {
+			const value = encodeFieldSchemaV1(
+				this.objectNodeFields.get(key) ?? fail(0xae7 /* missing field */),
+			);
+
 			Object.defineProperty(fieldsObject, key, {
 				enumerable: true,
 				configurable: true,
 				writable: true,
-				value: encodeFieldSchema(
-					this.objectNodeFields.get(key) ?? fail(0xae7 /* missing field */),
-				),
+				value,
 			});
 		}
 		return {
 			object: fieldsObject,
 		};
 	}
 
+	public override encodeV2(): TreeNodeSchemaDataFormatV2 {
+		const fieldsObject: Record<string, FieldSchemaFormat> = Object.create(null);
+		// Sort fields to ensure output is identical for for equivalent schema (since field order is not considered significant).
+		// This makes comparing schema easier, and ensures chunk reuse for schema summaries isn't needlessly broken.
+		for (const key of [...this.objectNodeFields.keys()].sort()) {
+			const value = encodeFieldSchemaV2(
+				this.objectNodeFields.get(key) ?? fail(0xae7 /* missing field */),
+			);
+
+			Object.defineProperty(fieldsObject, key, {
+				enumerable: true,
+				configurable: true,
+				writable: true,
+				value,
+			});
+		}
+
+		const kind = { object: fieldsObject };
+
+		// Omit metadata from the output if it is undefined
+		return this.metadata !== undefined ? { kind, metadata: this.metadata } : { kind };
+	}
+
 	public override getFieldSchema(field: FieldKey): TreeFieldStoredSchema {
 		return this.objectNodeFields.get(field) ?? storedEmptyFieldSchema;
 	}
@@ -229,14 +284,20 @@ export class MapNodeStoredSchema extends TreeNodeStoredSchema {
 	 * since no nodes can ever be in schema if you use `FieldKind.Value` here
 	 * (that would require infinite children).
 	 */
-	public constructor(public readonly mapFields: TreeFieldStoredSchema) {
-		super();
+	public constructor(
+		public readonly mapFields: TreeFieldStoredSchema,
+		metadata?: PersistedMetadataFormat | undefined,
+	) {
+		super(metadata);
 	}
 
-	public override encode(): TreeNodeSchemaDataFormat {
-		return {
-			map: encodeFieldSchema(this.mapFields),
-		};
+	public override encodeV1(): TreeNodeSchemaDataFormatV1 {
+		return { map: encodeFieldSchemaV1(this.mapFields) };
+	}
+
+	public override encodeV2(): TreeNodeSchemaDataFormatV2 {
+		const kind = { map: encodeFieldSchemaV2(this.mapFields) };
+		return this.metadata === undefined ? { kind, metadata: this.metadata } : { kind };
 	}
 
 	public override getFieldSchema(field: FieldKey): TreeFieldStoredSchema {
@@ -260,36 +321,54 @@ export class LeafNodeStoredSchema extends TreeNodeStoredSchema {
 	 * This is simply one approach that can work for modeling them in the internal schema representation.
 	 */
 	public constructor(public readonly leafValue: ValueSchema) {
-		super();
+		// No metadata for leaf nodes.
+		super(undefined);
 	}
 
-	public override encode(): TreeNodeSchemaDataFormat {
+	public override encodeV1(): TreeNodeSchemaDataFormatV1 {
 		return {
 			leaf: encodeValueSchema(this.leafValue),
 		};
 	}
 
+	public override encodeV2(): TreeNodeSchemaDataFormatV2 {
+		return {
+			// No metadata for leaf nodes, so don't emit a metadata field.
+			kind: {
+				leaf: encodeValueSchema(this.leafValue),
+			},
+		};
+	}
+
 	public override getFieldSchema(field: FieldKey): TreeFieldStoredSchema {
 		return storedEmptyFieldSchema;
 	}
 }
 
+/**
+ * Decoder wrapper function for {@link TreeNodeStoredSchema} implementations.
+ * Curries the constructor so that the caller can inject metadata.
+ */
+type StoredSchemaDecoder = (
+	metadata: PersistedMetadataFormat | undefined,
+) => TreeNodeStoredSchema;
+
 export const storedSchemaDecodeDispatcher: DiscriminatedUnionDispatcher<
-	TreeNodeSchemaDataFormat,
+	TreeNodeSchemaUnionFormat,
 	[],
-	TreeNodeStoredSchema
+	StoredSchemaDecoder
 > = new DiscriminatedUnionDispatcher({
-	leaf: (data: PersistedValueSchema) => new LeafNodeStoredSchema(decodeValueSchema(data)),
-	object: (
-		data: Record<TreeNodeSchemaIdentifier, FieldSchemaFormat>,
-	): TreeNodeStoredSchema => {
+	leaf: (data: PersistedValueSchema) => (metadata) =>
+		new LeafNodeStoredSchema(decodeValueSchema(data)),
+	object: (data: Record<TreeNodeSchemaIdentifier, FieldSchemaFormat>) => (metadata) => {
 		const map = new Map();
 		for (const [key, value] of Object.entries(data)) {
 			map.set(key, decodeFieldSchema(value));
 		}
-		return new ObjectNodeStoredSchema(map);
+		return new ObjectNodeStoredSchema(map, metadata);
 	},
-	map: (data: FieldSchemaFormat) => new MapNodeStoredSchema(decodeFieldSchema(data)),
+	map: (data: FieldSchemaFormat) => (metadata) =>
+		new MapNodeStoredSchema(decodeFieldSchema(data), metadata),
 });
 
 const valueSchemaEncode = new Map([
@@ -310,19 +389,29 @@ function decodeValueSchema(inMemory: PersistedValueSchema): ValueSchema {
 	return valueSchemaDecode.get(inMemory) ?? fail(0xae9 /* missing ValueSchema */);
 }
 
-export function encodeFieldSchema(schema: TreeFieldStoredSchema): FieldSchemaFormat {
+export function encodeFieldSchemaV1(schema: TreeFieldStoredSchema): FieldSchemaFormatV1 {
 	return {
 		kind: schema.kind,
 		// Types are sorted by identifier to improve stability of persisted data to increase chance of schema blob reuse.
 		types: [...schema.types].sort(),
 	};
 }
 
-export function decodeFieldSchema(schema: FieldSchemaFormat): TreeFieldStoredSchema {
+export function encodeFieldSchemaV2(schema: TreeFieldStoredSchema): FieldSchemaFormatV2 {
+	const fieldSchema: FieldSchemaFormatV1 = encodeFieldSchemaV1(schema);
+
+	// Omit metadata from the output if it is undefined
+	return schema.metadata !== undefined
+		? { ...fieldSchema, metadata: schema.metadata }
+		: { ...fieldSchema };
+}
+
+export function decodeFieldSchema(schema: FieldSchemaFormatV2): TreeFieldStoredSchema {
 	const out: TreeFieldStoredSchema = {
 		// TODO: maybe provide actual FieldKind objects here, error on unrecognized kinds.
 		kind: schema.kind,
 		types: new Set(schema.types),
+		metadata: schema.metadata,
 	};
 	return out;
 }

packages/dds/tree/src/feature-libraries/modular-schema/fieldKindWithEditor.ts

@@ -79,6 +79,8 @@ export class FieldKindWithEditor<
 			isNeverField(policy, originalData, {
 				kind: this.identifier,
 				types: originalTypes,
+				// Metadata is not used for this check.
+				metadata: undefined,
 			})
 		) {
 			return true;