Shared Tree: Persisted Schema Format v2 with persisted metadata support

packages/dds/tree/api-report/tree.alpha.api.md

@@ -237,7 +237,8 @@ export enum FluidClientVersion {
     v2_0 = "v2_0",
     v2_1 = "v2_1",
     v2_2 = "v2_2",
-    v2_3 = "v2_3"
+    v2_3 = "v2_3",
+    v2_4 = "v2_4"
 }
 
 // @alpha

packages/dds/tree/src/codec/codec.ts

@@ -352,6 +352,8 @@ export enum FluidClientVersion {
 	v2_2 = "v2_2",
 	/** Fluid Framework Client 2.3 and newer. */
 	v2_3 = "v2_3",
+	/** Fluid Framework Client 2.4 and newer. */
+	v2_4 = "v2_4",
 }
 
 /**

packages/dds/tree/src/core/index.ts

@@ -138,15 +138,18 @@ export {
 	storedEmptyFieldSchema,
 	type StoredSchemaCollection,
 	schemaFormatV1,
+	schemaFormatV2,
 	LeafNodeStoredSchema,
 	ObjectNodeStoredSchema,
 	MapNodeStoredSchema,
 	decodeFieldSchema,
-	encodeFieldSchema,
+	encodeFieldSchemaV1,
+	encodeFieldSchemaV2,
 	storedSchemaDecodeDispatcher,
 	type SchemaAndPolicy,
 	Multiplicity,
 	type SchemaPolicy,
+	SchemaCodecVersion,
 } from "./schema-stored/index.js";
 
 export {

packages/dds/tree/src/core/schema-stored/formatV2.ts

@@ -0,0 +1,108 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { type ObjectOptions, type Static, Type } from "@sinclair/typebox";
+
+import { unionOptions } from "../../codec/index.js";
+import {
+	type Brand,
+	brandedStringType,
+	JsonCompatibleReadOnlySchema,
+} from "../../util/index.js";
+
+export const version = 2 as const;
+
+/**
+ * Key (aka Name or Label) for a field which is scoped to a specific TreeNodeStoredSchema.
+ *
+ * Stable identifier, used when persisting data.
+ */
+export type FieldKey = Brand<string, "tree.FieldKey">;
+
+/**
+ * TypeBox Schema for encoding {@link FieldKey} in persisted data.
+ */
+export const FieldKeySchema = brandedStringType<FieldKey>();
+
+/**
+ * Identifier for a TreeNode schema.
+ * Also known as "Definition"
+ *
+ * Stable identifier, used when persisting data.
+ */
+export type TreeNodeSchemaIdentifier<TName extends string = string> = Brand<
+	TName,
+	"tree.TreeNodeSchemaIdentifier"
+>;
+
+/**
+ * Identifier for a FieldKind.
+ * Refers to an exact stable policy (ex: specific version of a policy),
+ * for how to handle (ex: edit and merge edits to) fields marked with this kind.
+ * Persisted in documents as part of stored schema.
+ */
+export type FieldKindIdentifier = Brand<string, "tree.FieldKindIdentifier">;
+export const FieldKindIdentifierSchema = brandedStringType<FieldKindIdentifier>();
+
+/**
+ * TypeBox Schema for encoding {@link TreeNodeSchemaIdentifiers} in persisted data.
+ */
+export const TreeNodeSchemaIdentifierSchema = brandedStringType<TreeNodeSchemaIdentifier>();
+
+export const PersistedMetadataFormat = Type.Optional(JsonCompatibleReadOnlySchema);
+
+const FieldSchemaFormatBase = Type.Object({
+	kind: FieldKindIdentifierSchema,
+	types: Type.Array(TreeNodeSchemaIdentifierSchema),
+	persistedMetadata: PersistedMetadataFormat,
+});
+
+const noAdditionalProps: ObjectOptions = { additionalProperties: false };
+
+export const FieldSchemaFormat = Type.Composite([FieldSchemaFormatBase], noAdditionalProps);
+
+/**
+ * Persisted version of {@link ValueSchema}.
+ */
+export enum PersistedValueSchema {
+	Number,
+	String,
+	Boolean,
+	FluidHandle,
+	Null,
+}
+
+/**
+ * Discriminated union content of tree node schema.
+ *
+ * See {@link DiscriminatedUnionDispatcher} for more information on this pattern.
+ */
+export const TreeNodeSchemaDataFormat = Type.Object(
+	{
+		/**
+		 * Object node union member.
+		 */
+		object: Type.Optional(Type.Record(Type.String(), FieldSchemaFormat)),
+		/**
+		 * Map node union member.
+		 */
+		map: Type.Optional(FieldSchemaFormat),
+		/**
+		 * Leaf node union member.
+		 */
+		leaf: Type.Optional(Type.Enum(PersistedValueSchema)),
+		/**
+		 * Persisted metadata for this node.
+		 */
+		persistedMetadata: PersistedMetadataFormat,
+	},
+	unionOptions,
+);
+
+export type TreeNodeSchemaDataFormat = Static<typeof TreeNodeSchemaDataFormat>;
+
+export type FieldSchemaFormat = Static<typeof FieldSchemaFormat>;
+
+export type PersistedMetadataFormat = Static<typeof PersistedMetadataFormat>;

packages/dds/tree/src/core/schema-stored/index.ts

@@ -18,10 +18,12 @@ export {
 	ObjectNodeStoredSchema,
 	MapNodeStoredSchema,
 	decodeFieldSchema,
-	encodeFieldSchema,
+	encodeFieldSchemaV1,
+	encodeFieldSchemaV2,
 	storedSchemaDecodeDispatcher,
 	type SchemaAndPolicy,
 	type SchemaPolicy,
+	SchemaCodecVersion,
 } from "./schema.js";
 export {
 	type TreeStoredSchemaSubscription,
@@ -36,3 +38,5 @@ export type { TreeNodeSchemaIdentifier, FieldKey, FieldKindIdentifier } from "./
 
 import * as schemaFormatV1 from "./formatV1.js";
 export { schemaFormatV1 };
+import * as schemaFormatV2 from "./formatV2.js";
+export { schemaFormatV2 };

packages/dds/tree/src/core/schema-stored/schema.ts

@@ -9,13 +9,26 @@ import { type MakeNominal, brand, invertMap } from "../../util/index.js";
 import {
 	type FieldKey,
 	type FieldKindIdentifier,
-	type FieldSchemaFormat,
+	type FieldSchemaFormat as FieldSchemaFormatV1,
 	PersistedValueSchema,
-	type TreeNodeSchemaDataFormat,
+	type TreeNodeSchemaDataFormat as TreeNodeSchemaDataFormatV1,
 	type TreeNodeSchemaIdentifier,
 } from "./formatV1.js";
+import type {
+	FieldSchemaFormat as FieldSchemaFormatV2,
+	PersistedMetadataFormat,
+	TreeNodeSchemaDataFormat as TreeNodeSchemaDataFormatV2,
+} from "./formatV2.js";
 import type { Multiplicity } from "./multiplicity.js";
 
+export enum SchemaCodecVersion {
+	v1 = 1,
+	// Adds persisted metadata to the schema.
+	v2 = 2,
+}
+
+type FieldSchemaFormat = FieldSchemaFormatV1 | FieldSchemaFormatV2;
+
 /**
  * Schema for what {@link TreeLeafValue} is allowed on a Leaf node.
  * @privateRemarks
@@ -119,6 +132,13 @@ export interface TreeFieldStoredSchema {
 	 * If not specified, types are unconstrained.
 	 */
 	readonly types: TreeTypeSet;
+
+	/**
+	 * Portion of the metadata which can be persisted.
+	 * @remarks
+	 * Discarded when encoding to {@link SchemaFormatVersion.V1}.
+	 */
+	persistedMetadata?: PersistedMetadataFormat;
 }
 
 /**
@@ -142,6 +162,7 @@ export const storedEmptyFieldSchema: TreeFieldStoredSchema = {
 	kind: brand(forbiddenFieldKindIdentifier),
 	// This type set also forces the field to be empty not not allowing any types as all.
 	types: new Set(),
+	persistedMetadata: undefined,
 };
 
 /**
@@ -160,7 +181,9 @@ export abstract class TreeNodeStoredSchema {
 	 * This is uses an opaque type to avoid leaking these types out of the package,
 	 * and is runtime validated by the codec.
 	 */
-	public abstract encode(): TreeNodeSchemaDataFormat;
+	public abstract encode(
+		schemaWriteVersion: SchemaCodecVersion,
+	): TreeNodeSchemaDataFormatV1 | TreeNodeSchemaDataFormatV2;
 
 	/**
 	 * Returns the schema for the provided field.
@@ -185,18 +208,35 @@ export class ObjectNodeStoredSchema extends TreeNodeStoredSchema {
 		super();
 	}
 
-	public override encode(): TreeNodeSchemaDataFormat {
+	public override encode(
+		schemaWriteVersion: SchemaCodecVersion,
+	): TreeNodeSchemaDataFormatV1 | TreeNodeSchemaDataFormatV2 {
 		const fieldsObject: Record<string, FieldSchemaFormat> = Object.create(null);
 		// Sort fields to ensure output is identical for for equivalent schema (since field order is not considered significant).
 		// This makes comparing schema easier, and ensures chunk reuse for schema summaries isn't needlessly broken.
 		for (const key of [...this.objectNodeFields.keys()].sort()) {
+			let value: FieldSchemaFormatV1 | FieldSchemaFormatV2 | undefined;
+
+			switch (schemaWriteVersion) {
+				case SchemaCodecVersion.v1:
+					value = encodeFieldSchemaV1(
+						this.objectNodeFields.get(key) ?? fail(0xae6 /* missing field */),
+					);
+					break;
+				case SchemaCodecVersion.v2:
+					value = encodeFieldSchemaV2(
+						this.objectNodeFields.get(key) ?? fail(0xae7 /* missing field */),
+					);
+					break;
+				default:
+					fail(`Cannot decode schema version ${schemaWriteVersion}`);
+			}
+
 			Object.defineProperty(fieldsObject, key, {
 				enumerable: true,
 				configurable: true,
 				writable: true,
-				value: encodeFieldSchema(
-					this.objectNodeFields.get(key) ?? fail(0xae7 /* missing field */),
-				),
+				value,
 			});
 		}
 		return {
@@ -224,10 +264,23 @@ export class MapNodeStoredSchema extends TreeNodeStoredSchema {
 		super();
 	}
 
-	public override encode(): TreeNodeSchemaDataFormat {
-		return {
-			map: encodeFieldSchema(this.mapFields),
-		};
+	public override encode(
+		schemaWriteVersion: SchemaCodecVersion,
+	): TreeNodeSchemaDataFormatV1 | TreeNodeSchemaDataFormatV2 {
+		switch (schemaWriteVersion) {
+			case SchemaCodecVersion.v1: {
+				return {
+					map: encodeFieldSchemaV1(this.mapFields),
+				};
+			}
+			case SchemaCodecVersion.v2: {
+				return {
+					map: encodeFieldSchemaV2(this.mapFields),
+				};
+			}
+			default:
+				fail(`Cannot decode schema version ${schemaWriteVersion}`);
+		}
 	}
 
 	public override getFieldSchema(field: FieldKey): TreeFieldStoredSchema {
@@ -254,7 +307,9 @@ export class LeafNodeStoredSchema extends TreeNodeStoredSchema {
 		super();
 	}
 
-	public override encode(): TreeNodeSchemaDataFormat {
+	public override encode(
+		schemaWriteVersion: SchemaCodecVersion,
+	): TreeNodeSchemaDataFormatV1 | TreeNodeSchemaDataFormatV2 {
 		return {
 			leaf: encodeValueSchema(this.leafValue),
 		};
@@ -266,7 +321,7 @@ export class LeafNodeStoredSchema extends TreeNodeStoredSchema {
 }
 
 export const storedSchemaDecodeDispatcher: DiscriminatedUnionDispatcher<
-	TreeNodeSchemaDataFormat,
+	TreeNodeSchemaDataFormatV1,
 	[],
 	TreeNodeStoredSchema
 > = new DiscriminatedUnionDispatcher({
@@ -301,11 +356,20 @@ function decodeValueSchema(inMemory: PersistedValueSchema): ValueSchema {
 	return valueSchemaDecode.get(inMemory) ?? fail(0xae9 /* missing ValueSchema */);
 }
 
-export function encodeFieldSchema(schema: TreeFieldStoredSchema): FieldSchemaFormat {
+export function encodeFieldSchemaV1(schema: TreeFieldStoredSchema): FieldSchemaFormatV1 {
+	return {
+		kind: schema.kind,
+		// Types are sorted by identifier to improve stability of persisted data to increase chance of schema blob reuse.
+		types: [...schema.types].sort(),
+	};
+}
+
+export function encodeFieldSchemaV2(schema: TreeFieldStoredSchema): FieldSchemaFormatV2 {
 	return {
 		kind: schema.kind,
 		// Types are sorted by identifier to improve stability of persisted data to increase chance of schema blob reuse.
 		types: [...schema.types].sort(),
+		persistedMetadata: schema.persistedMetadata,
 	};
 }
 

packages/dds/tree/src/feature-libraries/index.ts

@@ -22,7 +22,6 @@ export {
 	encodeTreeSchema,
 	makeSchemaCodec,
 	makeSchemaCodecs,
-	SchemaCodecVersion,
 } from "./schema-index/index.js";
 export {
 	stackTreeNodeCursor,