From bd8406a454dbb6568febda40f8b2b2ccc3d64f56 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Mon, 12 May 2025 14:59:41 -0700 Subject: [PATCH 01/45] Initial change. - Added in-memory schema format v2. - Implemented the schema format v2 codec. - Updated test utils that previously assumed there was only one format. - Added snapshots for many of the v2 tests. --- .../dds/tree/api-report/tree.alpha.api.md | 3 +- packages/dds/tree/src/codec/codec.ts | 2 + packages/dds/tree/src/core/index.ts | 4 +- .../tree/src/core/schema-stored/formatV2.ts | 104 ++++++++++++++++++ .../dds/tree/src/core/schema-stored/index.ts | 5 +- .../dds/tree/src/core/schema-stored/schema.ts | 43 ++++++-- .../feature-libraries/schema-index/codec.ts | 67 +++++++++-- .../schema-index/formatV2.ts | 31 ++++++ .../dds/tree/src/shared-tree/treeAlpha.ts | 1 + .../schema-index/codecUtil.ts | 22 +++- .../encodeTreeSchema/empty - schema v2.json | 8 ++ .../simple encoded schema - schema v2.json | 56 ++++++++++ .../allTheFields-full - schema v2.json | 39 +++++++ .../allTheFields-minimal - schema v2.json | 39 +++++++ .../schema-files/empty - schema v2.json | 8 ++ .../false boolean - schema v2.json | 14 +++ .../schema-files/handle - schema v2.json | 14 +++ .../hasAllMetadata - schema v2.json | 24 ++++ .../hasAllMetadataRootField - schema v2.json | 24 ++++ .../hasAmbiguousField - schema v2.json | 28 +++++ .../hasDescriptions - schema v2.json | 24 ++++ .../hasMinimalValueField - schema v2.json | 24 ++++ .../hasNumericValueField - schema v2.json | 24 ++++ .../hasOptionalField-empty - schema v2.json | 24 ++++ .../hasPolymorphicValueField - schema v2.json | 28 +++++ .../hasRenamedField - schema v2.json | 24 ++++ .../identifier-field - schema v2.json | 14 +++ .../schema-files/minimal - schema v2.json | 14 +++ ...ode-with-identifier-field - schema v2.json | 24 ++++ .../schema-files/null - schema v2.json | 14 +++ .../schema-files/numeric - schema v2.json | 14 +++ .../numericMap-empty - schema v2.json | 22 ++++ .../numericMap-full - schema v2.json | 22 ++++ .../numericSequence - schema v2.json | 14 +++ .../recursiveType-deeper - schema v2.json | 21 ++++ .../recursiveType-empty - schema v2.json | 21 ++++ .../recursiveType-recursive - schema v2.json | 21 ++++ .../true boolean - schema v2.json | 14 +++ packages/dds/tree/src/test/utils.ts | 14 +-- .../api-report/fluid-framework.alpha.api.md | 3 +- 40 files changed, 884 insertions(+), 32 deletions(-) create mode 100644 packages/dds/tree/src/core/schema-stored/formatV2.ts create mode 100644 packages/dds/tree/src/feature-libraries/schema-index/formatV2.ts create mode 100644 packages/dds/tree/src/test/snapshots/encodeTreeSchema/empty - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/encodeTreeSchema/simple encoded schema - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/allTheFields-full - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/allTheFields-minimal - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/empty - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/false boolean - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/handle - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/hasAllMetadata - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/hasAllMetadataRootField - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/hasAmbiguousField - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/hasDescriptions - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/hasMinimalValueField - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/hasNumericValueField - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/hasOptionalField-empty - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/hasPolymorphicValueField - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/hasRenamedField - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/identifier-field - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/minimal - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/node-with-identifier-field - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/null - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/numeric - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/numericMap-empty - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/numericMap-full - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/numericSequence - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/recursiveType-deeper - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/recursiveType-empty - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/recursiveType-recursive - schema v2.json create mode 100644 packages/dds/tree/src/test/snapshots/schema-files/true boolean - schema v2.json diff --git a/packages/dds/tree/api-report/tree.alpha.api.md b/packages/dds/tree/api-report/tree.alpha.api.md index 0b64ad42ae69..da1756afe87d 100644 --- a/packages/dds/tree/api-report/tree.alpha.api.md +++ b/packages/dds/tree/api-report/tree.alpha.api.md @@ -237,7 +237,8 @@ export enum FluidClientVersion { v2_0 = "v2_0", v2_1 = "v2_1", v2_2 = "v2_2", - v2_3 = "v2_3" + v2_3 = "v2_3", + v2_4 = "v2_4" } // @alpha diff --git a/packages/dds/tree/src/codec/codec.ts b/packages/dds/tree/src/codec/codec.ts index 1d3574576be3..a7b65a554007 100644 --- a/packages/dds/tree/src/codec/codec.ts +++ b/packages/dds/tree/src/codec/codec.ts @@ -352,6 +352,8 @@ export enum FluidClientVersion { v2_2 = "v2_2", /** Fluid Framework Client 2.3 and newer. */ v2_3 = "v2_3", + /** Fluid Framework Client 2.4 and newer. */ + v2_4 = "v2_4", } /** diff --git a/packages/dds/tree/src/core/index.ts b/packages/dds/tree/src/core/index.ts index 0a58f5ef8010..573d788d51b9 100644 --- a/packages/dds/tree/src/core/index.ts +++ b/packages/dds/tree/src/core/index.ts @@ -138,11 +138,13 @@ export { storedEmptyFieldSchema, type StoredSchemaCollection, schemaFormatV1, + schemaFormatV2, LeafNodeStoredSchema, ObjectNodeStoredSchema, MapNodeStoredSchema, decodeFieldSchema, - encodeFieldSchema, + encodeFieldSchemaV1, + encodeFieldSchemaV2, storedSchemaDecodeDispatcher, type SchemaAndPolicy, Multiplicity, diff --git a/packages/dds/tree/src/core/schema-stored/formatV2.ts b/packages/dds/tree/src/core/schema-stored/formatV2.ts new file mode 100644 index 000000000000..c5d296a03f4a --- /dev/null +++ b/packages/dds/tree/src/core/schema-stored/formatV2.ts @@ -0,0 +1,104 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { type ObjectOptions, type Static, Type } from "@sinclair/typebox"; + +import { unionOptions } from "../../codec/index.js"; +import { type Brand, brandedStringType } from "../../util/index.js"; + +export const version = 2 as const; + +/** + * Key (aka Name or Label) for a field which is scoped to a specific TreeNodeStoredSchema. + * + * Stable identifier, used when persisting data. + */ +export type FieldKey = Brand; + +/** + * TypeBox Schema for encoding {@link FieldKey} in persisted data. + */ +export const FieldKeySchema = brandedStringType(); + +/** + * Identifier for a TreeNode schema. + * Also known as "Definition" + * + * Stable identifier, used when persisting data. + */ +export type TreeNodeSchemaIdentifier = Brand< + TName, + "tree.TreeNodeSchemaIdentifier" +>; + +/** + * Identifier for a FieldKind. + * Refers to an exact stable policy (ex: specific version of a policy), + * for how to handle (ex: edit and merge edits to) fields marked with this kind. + * Persisted in documents as part of stored schema. + */ +export type FieldKindIdentifier = Brand; +export const FieldKindIdentifierSchema = brandedStringType(); + +/** + * TypeBox Schema for encoding {@link TreeNodeSchemaIdentifiers} in persisted data. + */ +export const TreeNodeSchemaIdentifierSchema = brandedStringType(); + +export const PersistedMetadataFormat = Type.Optional(Type.String()); + +const FieldSchemaFormatBase = Type.Object({ + kind: FieldKindIdentifierSchema, + types: Type.Array(TreeNodeSchemaIdentifierSchema), + persistedData: PersistedMetadataFormat, +}); + +const noAdditionalProps: ObjectOptions = { additionalProperties: false }; + +export const FieldSchemaFormat = Type.Composite([FieldSchemaFormatBase], noAdditionalProps); + +/** + * Persisted version of {@link ValueSchema}. + */ +export enum PersistedValueSchema { + Number, + String, + Boolean, + FluidHandle, + Null, +} + +/** + * Discriminated union content of tree node schema. + * + * See {@link DiscriminatedUnionDispatcher} for more information on this pattern. + */ +export const TreeNodeSchemaDataFormat = Type.Object( + { + /** + * Object node union member. + */ + object: Type.Optional(Type.Record(Type.String(), FieldSchemaFormat)), + /** + * Map node union member. + */ + map: Type.Optional(FieldSchemaFormat), + /** + * Leaf node union member. + */ + leaf: Type.Optional(Type.Enum(PersistedValueSchema)), + /** + * Persisted metadata for this node. + */ + persistedData: PersistedMetadataFormat, + }, + unionOptions, +); + +export type TreeNodeSchemaDataFormat = Static; + +export type FieldSchemaFormat = Static; + +export type PersistedMetadataFormat = Static; diff --git a/packages/dds/tree/src/core/schema-stored/index.ts b/packages/dds/tree/src/core/schema-stored/index.ts index 09d8b00a2d80..7202636783da 100644 --- a/packages/dds/tree/src/core/schema-stored/index.ts +++ b/packages/dds/tree/src/core/schema-stored/index.ts @@ -18,7 +18,8 @@ export { ObjectNodeStoredSchema, MapNodeStoredSchema, decodeFieldSchema, - encodeFieldSchema, + encodeFieldSchemaV1, + encodeFieldSchemaV2, storedSchemaDecodeDispatcher, type SchemaAndPolicy, type SchemaPolicy, @@ -36,3 +37,5 @@ export type { TreeNodeSchemaIdentifier, FieldKey, FieldKindIdentifier } from "./ import * as schemaFormatV1 from "./formatV1.js"; export { schemaFormatV1 }; +import * as schemaFormatV2 from "./formatV2.js"; +export { schemaFormatV2 }; diff --git a/packages/dds/tree/src/core/schema-stored/schema.ts b/packages/dds/tree/src/core/schema-stored/schema.ts index 5f1108415183..2a7766285f1a 100644 --- a/packages/dds/tree/src/core/schema-stored/schema.ts +++ b/packages/dds/tree/src/core/schema-stored/schema.ts @@ -9,13 +9,19 @@ import { type MakeNominal, brand, invertMap } from "../../util/index.js"; import { type FieldKey, type FieldKindIdentifier, - type FieldSchemaFormat, + type FieldSchemaFormat as FieldSchemaFormatV1, PersistedValueSchema, - type TreeNodeSchemaDataFormat, + type TreeNodeSchemaDataFormat as TreeNodeSchemaDataFormatV1, type TreeNodeSchemaIdentifier, } from "./formatV1.js"; +import type { + FieldSchemaFormat as FieldSchemaFormatV2, + PersistedMetadataFormat, +} from "./formatV2.js"; import type { Multiplicity } from "./multiplicity.js"; +type FieldSchemaFormat = FieldSchemaFormatV1 | FieldSchemaFormatV2; + /** * Schema for what {@link TreeLeafValue} is allowed on a Leaf node. * @privateRemarks @@ -119,6 +125,13 @@ export interface TreeFieldStoredSchema { * If not specified, types are unconstrained. */ readonly types: TreeTypeSet; + + /** + * Portion of the metadata which can be persisted. + * @remarks + * Discarded when encoding to {@link SchemaFormatVersion.V1}. + */ + persistedData?: PersistedMetadataFormat; } /** @@ -142,6 +155,7 @@ export const storedEmptyFieldSchema: TreeFieldStoredSchema = { kind: brand(forbiddenFieldKindIdentifier), // This type set also forces the field to be empty not not allowing any types as all. types: new Set(), + persistedData: undefined, }; /** @@ -160,7 +174,7 @@ export abstract class TreeNodeStoredSchema { * This is uses an opaque type to avoid leaking these types out of the package, * and is runtime validated by the codec. */ - public abstract encode(): TreeNodeSchemaDataFormat; + public abstract encode(): TreeNodeSchemaDataFormatV1; /** * Returns the schema for the provided field. @@ -185,7 +199,7 @@ export class ObjectNodeStoredSchema extends TreeNodeStoredSchema { super(); } - public override encode(): TreeNodeSchemaDataFormat { + public override encode(): TreeNodeSchemaDataFormatV1 { const fieldsObject: Record = Object.create(null); // Sort fields to ensure output is identical for for equivalent schema (since field order is not considered significant). // This makes comparing schema easier, and ensures chunk reuse for schema summaries isn't needlessly broken. @@ -194,7 +208,7 @@ export class ObjectNodeStoredSchema extends TreeNodeStoredSchema { enumerable: true, configurable: true, writable: true, - value: encodeFieldSchema( + value: encodeFieldSchemaV1( this.objectNodeFields.get(key) ?? fail(0xae7 /* missing field */), ), }); @@ -224,9 +238,9 @@ export class MapNodeStoredSchema extends TreeNodeStoredSchema { super(); } - public override encode(): TreeNodeSchemaDataFormat { + public override encode(): TreeNodeSchemaDataFormatV1 { return { - map: encodeFieldSchema(this.mapFields), + map: encodeFieldSchemaV1(this.mapFields), }; } @@ -254,7 +268,7 @@ export class LeafNodeStoredSchema extends TreeNodeStoredSchema { super(); } - public override encode(): TreeNodeSchemaDataFormat { + public override encode(): TreeNodeSchemaDataFormatV1 { return { leaf: encodeValueSchema(this.leafValue), }; @@ -266,7 +280,7 @@ export class LeafNodeStoredSchema extends TreeNodeStoredSchema { } export const storedSchemaDecodeDispatcher: DiscriminatedUnionDispatcher< - TreeNodeSchemaDataFormat, + TreeNodeSchemaDataFormatV1, [], TreeNodeStoredSchema > = new DiscriminatedUnionDispatcher({ @@ -301,11 +315,20 @@ function decodeValueSchema(inMemory: PersistedValueSchema): ValueSchema { return valueSchemaDecode.get(inMemory) ?? fail(0xae9 /* missing ValueSchema */); } -export function encodeFieldSchema(schema: TreeFieldStoredSchema): FieldSchemaFormat { +export function encodeFieldSchemaV1(schema: TreeFieldStoredSchema): FieldSchemaFormatV1 { + return { + kind: schema.kind, + // Types are sorted by identifier to improve stability of persisted data to increase chance of schema blob reuse. + types: [...schema.types].sort(), + }; +} + +export function encodeFieldSchemaV2(schema: TreeFieldStoredSchema): FieldSchemaFormatV2 { return { kind: schema.kind, // Types are sorted by identifier to improve stability of persisted data to increase chance of schema blob reuse. types: [...schema.types].sort(), + persistedData: schema.persistedData, }; } diff --git a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts index c1db0e4ee15e..48f465be1a15 100644 --- a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts +++ b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts @@ -5,7 +5,7 @@ import { fail, unreachableCase } from "@fluidframework/core-utils/internal"; import { - type FluidClientVersion, + FluidClientVersion, type ICodecFamily, type ICodecOptions, type IJsonCodec, @@ -18,19 +18,24 @@ import { type TreeNodeStoredSchema, type TreeStoredSchema, decodeFieldSchema, - encodeFieldSchema, + encodeFieldSchemaV1, + encodeFieldSchemaV2, schemaFormatV1, + schemaFormatV2, storedSchemaDecodeDispatcher, } from "../../core/index.js"; import { brand, type JsonCompatible } from "../../util/index.js"; import { Format as FormatV1 } from "./formatV1.js"; +import { Format as FormatV2 } from "./formatV2.js"; /** * Versions for the codec that encodes an in-memory representation of a stored schema {@link TreeStoredSchema} into a persisted format (or decodes it in the opposite direction). */ export enum SchemaCodecVersion { v1 = 1, + // Adds persisted metadata to the schema. + v2 = 2, } /** @@ -41,8 +46,17 @@ export enum SchemaCodecVersion { export function clientVersionToSchemaVersion( clientVersion: FluidClientVersion, ): SchemaCodecVersion { - // Only one version of the schema codec is currently supported. - return SchemaCodecVersion.v1; + switch (clientVersion) { + case FluidClientVersion.v2_0: + case FluidClientVersion.v2_1: + case FluidClientVersion.v2_2: + case FluidClientVersion.v2_3: + return SchemaCodecVersion.v1; + case FluidClientVersion.v2_4: + return SchemaCodecVersion.v2; + default: + unreachableCase(clientVersion); + } } /** @@ -67,7 +81,10 @@ export function makeSchemaCodec( * @returns The composed codec family. */ export function makeSchemaCodecs(options: ICodecOptions): ICodecFamily { - return makeCodecFamily([[SchemaCodecVersion.v1, makeSchemaCodecV1(options)]]); + return makeCodecFamily([ + [SchemaCodecVersion.v1, makeSchemaCodecV1(options)], + [SchemaCodecVersion.v2, makeSchemaCodecV2(options)], + ]); } /** @@ -82,7 +99,9 @@ export function encodeRepo( ): JsonCompatible { switch (version) { case SchemaCodecVersion.v1: - return encodeRepoV1(repo); + return encodeRepoV1(repo) as JsonCompatible; + case SchemaCodecVersion.v2: + return encodeRepoV2(repo) as JsonCompatible; default: unreachableCase(version); } @@ -91,7 +110,7 @@ export function encodeRepo( function encodeRepoV1(repo: TreeStoredSchema): FormatV1 { const nodeSchema: Record = Object.create(null); - const rootFieldSchema = encodeFieldSchema(repo.rootFieldSchema); + const rootFieldSchema = encodeFieldSchemaV1(repo.rootFieldSchema); for (const name of [...repo.nodeSchema.keys()].sort()) { const schema = repo.nodeSchema.get(name) ?? fail(0xb28 /* missing schema */); Object.defineProperty(nodeSchema, name, { @@ -108,7 +127,27 @@ function encodeRepoV1(repo: TreeStoredSchema): FormatV1 { }; } -function decode(f: FormatV1): TreeStoredSchema { +function encodeRepoV2(repo: TreeStoredSchema): FormatV2 { + const nodeSchema: Record = + Object.create(null); + const rootFieldSchema = encodeFieldSchemaV2(repo.rootFieldSchema); + for (const name of [...repo.nodeSchema.keys()].sort()) { + const schema = repo.nodeSchema.get(name) ?? fail(0xb28 /* missing schema */); + Object.defineProperty(nodeSchema, name, { + enumerable: true, + configurable: true, + writable: true, + value: schema.encode(), + }); + } + return { + version: schemaFormatV2.version, + nodes: nodeSchema, + root: rootFieldSchema, + }; +} + +function decode(f: FormatV1 | FormatV2): TreeStoredSchema { const nodeSchema: Map = new Map(); for (const [key, schema] of Object.entries(f.nodes)) { nodeSchema.set(brand(key), storedSchemaDecodeDispatcher.dispatch(schema)); @@ -130,3 +169,15 @@ function makeSchemaCodecV1(options: ICodecOptions): IJsonCodec decode(data), }); } + +/** + * Creates a codec which performs synchronous monolithic encoding of schema content. + * @param options - Specifies common codec options, including which `validator` to use. + * @returns The codec. + */ +function makeSchemaCodecV2(options: ICodecOptions): IJsonCodec { + return makeVersionedValidatedCodec(options, new Set([schemaFormatV2.version]), FormatV2, { + encode: (data: TreeStoredSchema) => encodeRepoV2(data), + decode: (data: FormatV2) => decode(data), + }); +} diff --git a/packages/dds/tree/src/feature-libraries/schema-index/formatV2.ts b/packages/dds/tree/src/feature-libraries/schema-index/formatV2.ts new file mode 100644 index 000000000000..525c4657977b --- /dev/null +++ b/packages/dds/tree/src/feature-libraries/schema-index/formatV2.ts @@ -0,0 +1,31 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { type ObjectOptions, type Static, Type } from "@sinclair/typebox"; + +import { schemaFormatV2 } from "../../core/index.js"; + +const noAdditionalProps: ObjectOptions = { additionalProperties: false }; + +/** + * Format for encoding as json. + * + * For consistency all lists are sorted and undefined values are omitted. + * + * This chooses to use lists of named objects instead of maps: + * this choice is somewhat arbitrary, but avoids user data being used as object keys, + * which can sometimes be an issue (for example handling that for "__proto__" can require care). + * It also makes it simpler to determinately sort by keys. + */ +export const Format = Type.Object( + { + version: Type.Literal(schemaFormatV2.version), + nodes: Type.Record(Type.String(), schemaFormatV2.TreeNodeSchemaDataFormat), + root: schemaFormatV2.FieldSchemaFormat, + persistedMetadata: schemaFormatV2.PersistedMetadataFormat, + }, + noAdditionalProps, +); +export type Format = Static; diff --git a/packages/dds/tree/src/shared-tree/treeAlpha.ts b/packages/dds/tree/src/shared-tree/treeAlpha.ts index 948591bf0263..0a7eba1fa432 100644 --- a/packages/dds/tree/src/shared-tree/treeAlpha.ts +++ b/packages/dds/tree/src/shared-tree/treeAlpha.ts @@ -380,4 +380,5 @@ const versionToFormat = { v2_1: 1, v2_2: 1, v2_3: 1, + v2_4: 1, }; diff --git a/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts b/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts index b1beaac7a663..d2bf54105f0f 100644 --- a/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts +++ b/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts @@ -3,12 +3,14 @@ * Licensed under the MIT License. */ +import { FluidClientVersion } from "../../../codec/index.js"; import { makeSchemaCodecs, - type SchemaCodecVersion, + SchemaCodecVersion, // eslint-disable-next-line import/no-internal-modules } from "../../../feature-libraries/schema-index/index.js"; import { ajvValidator } from "../../codec/index.js"; +import { assert } from "@fluidframework/core-utils/internal"; /* * The list of supported schema write versions. Used in tests that cover multiple schema versions. @@ -16,3 +18,21 @@ import { ajvValidator } from "../../codec/index.js"; export const supportedSchemaFormats = Array.from( makeSchemaCodecs({ jsonValidator: ajvValidator }).getSupportedFormats(), ).filter((format) => format !== undefined) as SchemaCodecVersion[]; + +/** + * Convert a schema version to the minimum Fluid client version supporting that format. + * @param schemaFormat - The schema format version. + * @returns The Fluid client version that supports the provided schema format. + */ +export function schemaFormatToClientVersion( + schemaFormat: SchemaCodecVersion, +): FluidClientVersion { + switch (schemaFormat) { + case SchemaCodecVersion.v1: + return FluidClientVersion.v2_0; + case SchemaCodecVersion.v2: + return FluidClientVersion.v2_4; + default: + assert(false, `Unsupported schema format: ${schemaFormat}`); + } +} diff --git a/packages/dds/tree/src/test/snapshots/encodeTreeSchema/empty - schema v2.json b/packages/dds/tree/src/test/snapshots/encodeTreeSchema/empty - schema v2.json new file mode 100644 index 000000000000..359cc020d13b --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/encodeTreeSchema/empty - schema v2.json @@ -0,0 +1,8 @@ +{ + "version": 2, + "nodes": {}, + "root": { + "kind": "Forbidden", + "types": [] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/encodeTreeSchema/simple encoded schema - schema v2.json b/packages/dds/tree/src/test/snapshots/encodeTreeSchema/simple encoded schema - schema v2.json new file mode 100644 index 000000000000..3fe392cee3a9 --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/encodeTreeSchema/simple encoded schema - schema v2.json @@ -0,0 +1,56 @@ +{ + "version": 2, + "nodes": { + "com.fluidframework.json.array": { + "object": { + "": { + "kind": "Sequence", + "types": [ + "com.fluidframework.json.array", + "com.fluidframework.json.object", + "com.fluidframework.leaf.boolean", + "com.fluidframework.leaf.null", + "com.fluidframework.leaf.number", + "com.fluidframework.leaf.string" + ] + } + } + }, + "com.fluidframework.json.object": { + "map": { + "kind": "Optional", + "types": [ + "com.fluidframework.json.array", + "com.fluidframework.json.object", + "com.fluidframework.leaf.boolean", + "com.fluidframework.leaf.null", + "com.fluidframework.leaf.number", + "com.fluidframework.leaf.string" + ] + } + }, + "com.fluidframework.leaf.boolean": { + "leaf": 2 + }, + "com.fluidframework.leaf.null": { + "leaf": 4 + }, + "com.fluidframework.leaf.number": { + "leaf": 0 + }, + "com.fluidframework.leaf.string": { + "leaf": 1 + } + }, + "root": { + "kind": "Value", + "types": [ + "com.fluidframework.json.array", + "com.fluidframework.json.object", + "com.fluidframework.leaf.boolean", + "com.fluidframework.leaf.null", + "com.fluidframework.leaf.number", + "com.fluidframework.leaf.string" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/allTheFields-full - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/allTheFields-full - schema v2.json new file mode 100644 index 000000000000..f8c9da4815f1 --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/allTheFields-full - schema v2.json @@ -0,0 +1,39 @@ +{ + "version": 2, + "nodes": { + "com.fluidframework.leaf.number": { + "leaf": 0 + }, + "test.allTheFields": { + "object": { + "optional": { + "kind": "Optional", + "types": [ + "com.fluidframework.leaf.number" + ] + }, + "sequence": { + "kind": "Sequence", + "types": [ + "com.fluidframework.leaf.number" + ] + }, + "valueField": { + "kind": "Value", + "types": [ + "com.fluidframework.leaf.number" + ] + } + } + }, + "test.minimal": { + "object": {} + } + }, + "root": { + "kind": "Value", + "types": [ + "test.allTheFields" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/allTheFields-minimal - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/allTheFields-minimal - schema v2.json new file mode 100644 index 000000000000..f8c9da4815f1 --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/allTheFields-minimal - schema v2.json @@ -0,0 +1,39 @@ +{ + "version": 2, + "nodes": { + "com.fluidframework.leaf.number": { + "leaf": 0 + }, + "test.allTheFields": { + "object": { + "optional": { + "kind": "Optional", + "types": [ + "com.fluidframework.leaf.number" + ] + }, + "sequence": { + "kind": "Sequence", + "types": [ + "com.fluidframework.leaf.number" + ] + }, + "valueField": { + "kind": "Value", + "types": [ + "com.fluidframework.leaf.number" + ] + } + } + }, + "test.minimal": { + "object": {} + } + }, + "root": { + "kind": "Value", + "types": [ + "test.allTheFields" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/empty - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/empty - schema v2.json new file mode 100644 index 000000000000..6ae447caba61 --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/empty - schema v2.json @@ -0,0 +1,8 @@ +{ + "version": 2, + "nodes": {}, + "root": { + "kind": "Optional", + "types": [] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/false boolean - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/false boolean - schema v2.json new file mode 100644 index 000000000000..dbd7b1e621ba --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/false boolean - schema v2.json @@ -0,0 +1,14 @@ +{ + "version": 2, + "nodes": { + "com.fluidframework.leaf.boolean": { + "leaf": 2 + } + }, + "root": { + "kind": "Value", + "types": [ + "com.fluidframework.leaf.boolean" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/handle - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/handle - schema v2.json new file mode 100644 index 000000000000..1bc718ef657a --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/handle - schema v2.json @@ -0,0 +1,14 @@ +{ + "version": 2, + "nodes": { + "com.fluidframework.leaf.handle": { + "leaf": 3 + } + }, + "root": { + "kind": "Value", + "types": [ + "com.fluidframework.leaf.handle" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/hasAllMetadata - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/hasAllMetadata - schema v2.json new file mode 100644 index 000000000000..65c731adecea --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/hasAllMetadata - schema v2.json @@ -0,0 +1,24 @@ +{ + "version": 2, + "nodes": { + "test.hasDescriptions": { + "object": { + "stored-name": { + "kind": "Value", + "types": [ + "test.minimal" + ] + } + } + }, + "test.minimal": { + "object": {} + } + }, + "root": { + "kind": "Value", + "types": [ + "test.hasDescriptions" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/hasAllMetadataRootField - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/hasAllMetadataRootField - schema v2.json new file mode 100644 index 000000000000..a0f6eee377d2 --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/hasAllMetadataRootField - schema v2.json @@ -0,0 +1,24 @@ +{ + "version": 2, + "nodes": { + "test.hasDescriptions": { + "object": { + "stored-name": { + "kind": "Value", + "types": [ + "test.minimal" + ] + } + } + }, + "test.minimal": { + "object": {} + } + }, + "root": { + "kind": "Optional", + "types": [ + "test.hasDescriptions" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/hasAmbiguousField - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/hasAmbiguousField - schema v2.json new file mode 100644 index 000000000000..cbafc48d311b --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/hasAmbiguousField - schema v2.json @@ -0,0 +1,28 @@ +{ + "version": 2, + "nodes": { + "test.hasAmbiguousField": { + "object": { + "field": { + "kind": "Value", + "types": [ + "test.minimal", + "test.minimal2" + ] + } + } + }, + "test.minimal": { + "object": {} + }, + "test.minimal2": { + "object": {} + } + }, + "root": { + "kind": "Value", + "types": [ + "test.hasAmbiguousField" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/hasDescriptions - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/hasDescriptions - schema v2.json new file mode 100644 index 000000000000..154972d78582 --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/hasDescriptions - schema v2.json @@ -0,0 +1,24 @@ +{ + "version": 2, + "nodes": { + "test.hasDescriptions": { + "object": { + "field": { + "kind": "Value", + "types": [ + "test.minimal" + ] + } + } + }, + "test.minimal": { + "object": {} + } + }, + "root": { + "kind": "Value", + "types": [ + "test.hasDescriptions" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/hasMinimalValueField - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/hasMinimalValueField - schema v2.json new file mode 100644 index 000000000000..e875de16ed7d --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/hasMinimalValueField - schema v2.json @@ -0,0 +1,24 @@ +{ + "version": 2, + "nodes": { + "test.hasMinimalValueField": { + "object": { + "field": { + "kind": "Value", + "types": [ + "test.minimal" + ] + } + } + }, + "test.minimal": { + "object": {} + } + }, + "root": { + "kind": "Value", + "types": [ + "test.hasMinimalValueField" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/hasNumericValueField - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/hasNumericValueField - schema v2.json new file mode 100644 index 000000000000..493bfc8face7 --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/hasNumericValueField - schema v2.json @@ -0,0 +1,24 @@ +{ + "version": 2, + "nodes": { + "com.fluidframework.leaf.number": { + "leaf": 0 + }, + "test.hasNumericValueField": { + "object": { + "field": { + "kind": "Value", + "types": [ + "com.fluidframework.leaf.number" + ] + } + } + } + }, + "root": { + "kind": "Value", + "types": [ + "test.hasNumericValueField" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/hasOptionalField-empty - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/hasOptionalField-empty - schema v2.json new file mode 100644 index 000000000000..89d72e6429d6 --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/hasOptionalField-empty - schema v2.json @@ -0,0 +1,24 @@ +{ + "version": 2, + "nodes": { + "com.fluidframework.leaf.number": { + "leaf": 0 + }, + "test.hasOptionalField": { + "object": { + "field": { + "kind": "Optional", + "types": [ + "com.fluidframework.leaf.number" + ] + } + } + } + }, + "root": { + "kind": "Value", + "types": [ + "test.hasOptionalField" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/hasPolymorphicValueField - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/hasPolymorphicValueField - schema v2.json new file mode 100644 index 000000000000..a6663b20aabb --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/hasPolymorphicValueField - schema v2.json @@ -0,0 +1,28 @@ +{ + "version": 2, + "nodes": { + "com.fluidframework.leaf.number": { + "leaf": 0 + }, + "test.hasPolymorphicValueField": { + "object": { + "field": { + "kind": "Value", + "types": [ + "com.fluidframework.leaf.number", + "test.minimal" + ] + } + } + }, + "test.minimal": { + "object": {} + } + }, + "root": { + "kind": "Value", + "types": [ + "test.hasPolymorphicValueField" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/hasRenamedField - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/hasRenamedField - schema v2.json new file mode 100644 index 000000000000..cd5e07352a83 --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/hasRenamedField - schema v2.json @@ -0,0 +1,24 @@ +{ + "version": 2, + "nodes": { + "test.hasRenamedField": { + "object": { + "stored-name": { + "kind": "Value", + "types": [ + "test.minimal" + ] + } + } + }, + "test.minimal": { + "object": {} + } + }, + "root": { + "kind": "Value", + "types": [ + "test.hasRenamedField" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/identifier-field - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/identifier-field - schema v2.json new file mode 100644 index 000000000000..189a77df631c --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/identifier-field - schema v2.json @@ -0,0 +1,14 @@ +{ + "version": 2, + "nodes": { + "com.fluidframework.leaf.string": { + "leaf": 1 + } + }, + "root": { + "kind": "Identifier", + "types": [ + "com.fluidframework.leaf.string" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/minimal - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/minimal - schema v2.json new file mode 100644 index 000000000000..31f730a1651b --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/minimal - schema v2.json @@ -0,0 +1,14 @@ +{ + "version": 2, + "nodes": { + "test.minimal": { + "object": {} + } + }, + "root": { + "kind": "Value", + "types": [ + "test.minimal" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/node-with-identifier-field - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/node-with-identifier-field - schema v2.json new file mode 100644 index 000000000000..8e923d4119d7 --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/node-with-identifier-field - schema v2.json @@ -0,0 +1,24 @@ +{ + "version": 2, + "nodes": { + "com.fluidframework.leaf.string": { + "leaf": 1 + }, + "test.hasIdentifierField": { + "object": { + "field": { + "kind": "Identifier", + "types": [ + "com.fluidframework.leaf.string" + ] + } + } + } + }, + "root": { + "kind": "Value", + "types": [ + "test.hasIdentifierField" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/null - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/null - schema v2.json new file mode 100644 index 000000000000..d28d81ab2ae6 --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/null - schema v2.json @@ -0,0 +1,14 @@ +{ + "version": 2, + "nodes": { + "com.fluidframework.leaf.null": { + "leaf": 4 + } + }, + "root": { + "kind": "Value", + "types": [ + "com.fluidframework.leaf.null" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/numeric - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/numeric - schema v2.json new file mode 100644 index 000000000000..cabf326aa297 --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/numeric - schema v2.json @@ -0,0 +1,14 @@ +{ + "version": 2, + "nodes": { + "com.fluidframework.leaf.number": { + "leaf": 0 + } + }, + "root": { + "kind": "Value", + "types": [ + "com.fluidframework.leaf.number" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/numericMap-empty - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/numericMap-empty - schema v2.json new file mode 100644 index 000000000000..57fe12469f0f --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/numericMap-empty - schema v2.json @@ -0,0 +1,22 @@ +{ + "version": 2, + "nodes": { + "com.fluidframework.leaf.number": { + "leaf": 0 + }, + "test.numericMap": { + "map": { + "kind": "Optional", + "types": [ + "com.fluidframework.leaf.number" + ] + } + } + }, + "root": { + "kind": "Value", + "types": [ + "test.numericMap" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/numericMap-full - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/numericMap-full - schema v2.json new file mode 100644 index 000000000000..57fe12469f0f --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/numericMap-full - schema v2.json @@ -0,0 +1,22 @@ +{ + "version": 2, + "nodes": { + "com.fluidframework.leaf.number": { + "leaf": 0 + }, + "test.numericMap": { + "map": { + "kind": "Optional", + "types": [ + "com.fluidframework.leaf.number" + ] + } + } + }, + "root": { + "kind": "Value", + "types": [ + "test.numericMap" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/numericSequence - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/numericSequence - schema v2.json new file mode 100644 index 000000000000..8cdc31d1b205 --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/numericSequence - schema v2.json @@ -0,0 +1,14 @@ +{ + "version": 2, + "nodes": { + "com.fluidframework.leaf.number": { + "leaf": 0 + } + }, + "root": { + "kind": "Sequence", + "types": [ + "com.fluidframework.leaf.number" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-deeper - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-deeper - schema v2.json new file mode 100644 index 000000000000..ac39334292e0 --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-deeper - schema v2.json @@ -0,0 +1,21 @@ +{ + "version": 2, + "nodes": { + "test.recursiveType": { + "object": { + "field": { + "kind": "Optional", + "types": [ + "test.recursiveType" + ] + } + } + } + }, + "root": { + "kind": "Value", + "types": [ + "test.recursiveType" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-empty - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-empty - schema v2.json new file mode 100644 index 000000000000..ac39334292e0 --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-empty - schema v2.json @@ -0,0 +1,21 @@ +{ + "version": 2, + "nodes": { + "test.recursiveType": { + "object": { + "field": { + "kind": "Optional", + "types": [ + "test.recursiveType" + ] + } + } + } + }, + "root": { + "kind": "Value", + "types": [ + "test.recursiveType" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-recursive - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-recursive - schema v2.json new file mode 100644 index 000000000000..ac39334292e0 --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-recursive - schema v2.json @@ -0,0 +1,21 @@ +{ + "version": 2, + "nodes": { + "test.recursiveType": { + "object": { + "field": { + "kind": "Optional", + "types": [ + "test.recursiveType" + ] + } + } + } + }, + "root": { + "kind": "Value", + "types": [ + "test.recursiveType" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/snapshots/schema-files/true boolean - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/true boolean - schema v2.json new file mode 100644 index 000000000000..dbd7b1e621ba --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/schema-files/true boolean - schema v2.json @@ -0,0 +1,14 @@ +{ + "version": 2, + "nodes": { + "com.fluidframework.leaf.boolean": { + "leaf": 2 + } + }, + "root": { + "kind": "Value", + "types": [ + "com.fluidframework.leaf.boolean" + ] + } +} \ No newline at end of file diff --git a/packages/dds/tree/src/test/utils.ts b/packages/dds/tree/src/test/utils.ts index 6e315c8a5b9c..47a838d6225f 100644 --- a/packages/dds/tree/src/test/utils.ts +++ b/packages/dds/tree/src/test/utils.ts @@ -630,17 +630,13 @@ export function validateTree(tree: ITreeCheckout, expected: JsonableTree[]): voi assert.deepEqual(actual, expected); } +// If you are adding a new schema format, consider changing the encoding format used for this codec, given +// that equality of two schemas in tests is achieved by deep-comparing their persisted representations. +// If the newer format is a superset of the previous format, it can be safely used for comparisons. This is the +// case with schema format v2. const schemaCodec = makeSchemaCodec( { jsonValidator: typeboxValidator }, - SchemaCodecVersion.v1, -); - -// If you are adding a new schema format, consider changing the encoding format used in the above codec, given -// that equality of two schemas in tests is achieved by deep-comparing their persisted representations. -// Note we have to divide the length of the return value from `Object.keys` to get the number of enum entries. -assert( - Object.keys(SchemaCodecVersion).length / 2 === 1, - "This code only handles a single schema codec version.", + SchemaCodecVersion.v2, ); export function checkRemovedRootsAreSynchronized(trees: readonly ITreeCheckout[]): void { diff --git a/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md b/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md index e5e7c6141cc9..0051d03e2e59 100644 --- a/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md +++ b/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md @@ -281,7 +281,8 @@ export enum FluidClientVersion { v2_0 = "v2_0", v2_1 = "v2_1", v2_2 = "v2_2", - v2_3 = "v2_3" + v2_3 = "v2_3", + v2_4 = "v2_4" } // @public From 3165fe34c9b7c23594342ce6bbb83c6c800efc54 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Tue, 13 May 2025 10:32:03 -0700 Subject: [PATCH 02/45] persistedMetadata is now backed by an object instead of a string. --- examples/apps/tree-cli-app/src/utils.ts | 1 - packages/dds/tree/src/core/schema-stored/formatV2.ts | 12 ++++++++---- packages/dds/tree/src/core/schema-stored/schema.ts | 6 +++--- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/examples/apps/tree-cli-app/src/utils.ts b/examples/apps/tree-cli-app/src/utils.ts index 4618c903d7bb..24156604781b 100644 --- a/examples/apps/tree-cli-app/src/utils.ts +++ b/examples/apps/tree-cli-app/src/utils.ts @@ -157,7 +157,6 @@ export function exportContent(destination: string, tree: List): JsonCompatible { oldestCompatibleClient: FluidClientVersion.v2_3, idCompressor, }), - schema: extractPersistedSchema(config, FluidClientVersion.v2_3), idCompressor: idCompressor.serialize(true), }; diff --git a/packages/dds/tree/src/core/schema-stored/formatV2.ts b/packages/dds/tree/src/core/schema-stored/formatV2.ts index c5d296a03f4a..f9cbc68abf5a 100644 --- a/packages/dds/tree/src/core/schema-stored/formatV2.ts +++ b/packages/dds/tree/src/core/schema-stored/formatV2.ts @@ -6,7 +6,11 @@ import { type ObjectOptions, type Static, Type } from "@sinclair/typebox"; import { unionOptions } from "../../codec/index.js"; -import { type Brand, brandedStringType } from "../../util/index.js"; +import { + type Brand, + brandedStringType, + JsonCompatibleReadOnlySchema, +} from "../../util/index.js"; export const version = 2 as const; @@ -47,12 +51,12 @@ export const FieldKindIdentifierSchema = brandedStringType( */ export const TreeNodeSchemaIdentifierSchema = brandedStringType(); -export const PersistedMetadataFormat = Type.Optional(Type.String()); +export const PersistedMetadataFormat = Type.Optional(JsonCompatibleReadOnlySchema); const FieldSchemaFormatBase = Type.Object({ kind: FieldKindIdentifierSchema, types: Type.Array(TreeNodeSchemaIdentifierSchema), - persistedData: PersistedMetadataFormat, + persistedMetadata: PersistedMetadataFormat, }); const noAdditionalProps: ObjectOptions = { additionalProperties: false }; @@ -92,7 +96,7 @@ export const TreeNodeSchemaDataFormat = Type.Object( /** * Persisted metadata for this node. */ - persistedData: PersistedMetadataFormat, + persistedMetadata: PersistedMetadataFormat, }, unionOptions, ); diff --git a/packages/dds/tree/src/core/schema-stored/schema.ts b/packages/dds/tree/src/core/schema-stored/schema.ts index 2a7766285f1a..d600b545d4f4 100644 --- a/packages/dds/tree/src/core/schema-stored/schema.ts +++ b/packages/dds/tree/src/core/schema-stored/schema.ts @@ -131,7 +131,7 @@ export interface TreeFieldStoredSchema { * @remarks * Discarded when encoding to {@link SchemaFormatVersion.V1}. */ - persistedData?: PersistedMetadataFormat; + persistedMetadata?: PersistedMetadataFormat; } /** @@ -155,7 +155,7 @@ export const storedEmptyFieldSchema: TreeFieldStoredSchema = { kind: brand(forbiddenFieldKindIdentifier), // This type set also forces the field to be empty not not allowing any types as all. types: new Set(), - persistedData: undefined, + persistedMetadata: undefined, }; /** @@ -328,7 +328,7 @@ export function encodeFieldSchemaV2(schema: TreeFieldStoredSchema): FieldSchemaF kind: schema.kind, // Types are sorted by identifier to improve stability of persisted data to increase chance of schema blob reuse. types: [...schema.types].sort(), - persistedData: schema.persistedData, + persistedMetadata: schema.persistedMetadata, }; } From 9709edeea54a82ed4144ffbaa3d65f1bf048089f Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Tue, 13 May 2025 11:00:42 -0700 Subject: [PATCH 03/45] Updated codec.spec.ts to cover FormatV2. --- .../schema-index/codec.spec.ts | 63 +++++++--- .../files/SchemaIndexFormat - schema v2.json | 119 ++++++++++++++++++ 2 files changed, 167 insertions(+), 15 deletions(-) create mode 100644 packages/dds/tree/src/test/snapshots/files/SchemaIndexFormat - schema v2.json diff --git a/packages/dds/tree/src/test/feature-libraries/schema-index/codec.spec.ts b/packages/dds/tree/src/test/feature-libraries/schema-index/codec.spec.ts index a1a2c9b6ba15..46fc6d25f38f 100644 --- a/packages/dds/tree/src/test/feature-libraries/schema-index/codec.spec.ts +++ b/packages/dds/tree/src/test/feature-libraries/schema-index/codec.spec.ts @@ -17,6 +17,8 @@ import { } from "../../../feature-libraries/index.js"; /* eslint-disable-next-line import/no-internal-modules */ import { Format as FormatV1 } from "../../../feature-libraries/schema-index/formatV1.js"; +// eslint-disable-next-line import/no-internal-modules +import { Format as FormatV2 } from "../../../feature-libraries/schema-index/formatV2.js"; import { takeJsonSnapshot, useSnapshotDirectory } from "../../snapshots/index.js"; import { type EncodingTestData, makeEncodingTestSuite } from "../../utils.js"; // eslint-disable-next-line import/no-internal-modules @@ -28,6 +30,7 @@ import { makeSchemaCodecs } from "../../../feature-libraries/schema-index/index. const schemaCodecs = makeSchemaCodecs({ jsonValidator: typeboxValidator }); const codecV1 = makeSchemaCodec({ jsonValidator: typeboxValidator }, SchemaCodecVersion.v1); +const codecV2 = makeSchemaCodec({ jsonValidator: typeboxValidator }, SchemaCodecVersion.v2); const schema2 = toStoredSchema(SchemaFactory.optional(JsonAsTree.Primitive)); @@ -46,6 +49,11 @@ describe("SchemaIndex", () => { takeJsonSnapshot(FormatV1); }); + it("SchemaIndexFormat - schema v2", () => { + // Capture the json schema for the format as a snapshot, so any change to what schema is allowed shows up in this tests. + takeJsonSnapshot(FormatV2); + }); + it("accepts valid data - schema v1", () => { // TODO: should test way more cases, and check results are correct. const cases = [ @@ -60,26 +68,51 @@ describe("SchemaIndex", () => { } }); - it("rejects malformed data - schema v1", () => { - // TODO: should test way more cases - // TODO: maybe well formed but semantically invalid data should be rejected (ex: with duplicates keys)? - const badCases = [ - undefined, - null, - {}, - { version: "1.0.0" }, - { version: "1" }, - { version: "2.0.0" }, - { version: 1 }, - { version: 2 }, - { version: 1, nodeSchema: [], globalFieldSchema: [] }, - { version: 1, nodeSchema: [], extraField: 0 }, + it("accepts valid data - schema v2", () => { + // TODO: should test way more cases, and check results are correct. + const cases = [ + { + version: 2 as const, + nodes: {}, + root: { kind: "x" as FieldKindIdentifier, types: [] }, + persistedMetadata: { "ff-system": { "eDiscovery-exclude": "true" } }, + } satisfies FormatV2, ]; - for (const data of badCases) { + for (const data of cases) { + codecV2.decode(data); + } + }); + + // TODO: should test way more cases + // TODO: maybe well formed but semantically invalid data should be rejected (ex: with duplicates keys)? + /** + * A set of cases that are expected to be rejected by both the v1 and v2 codecs. + */ + const badCasesV1AndV2 = [ + undefined, + null, + {}, + { version: "1.0.0" }, + { version: "1" }, + { version: "2.0.0" }, + { version: 1 }, + { version: 2 }, + { version: 1, nodeSchema: [], globalFieldSchema: [] }, + { version: 1, nodeSchema: [], extraField: 0 }, + ]; + + it(`rejects malformed data - schema v1`, () => { + for (const data of badCasesV1AndV2) { assert.throws(() => codecV1.decode(data as unknown as FormatV1)); } }); + it(`rejects malformed data - schema v2`, () => { + for (const data of badCasesV1AndV2) { + assert.throws(() => codecV2.decode(data as unknown as FormatV2)); + } + }); + describe("codec", () => { makeEncodingTestSuite(schemaCodecs, testCases, (a, b) => { assert(allowsRepoSuperset(defaultSchemaPolicy, a, b)); diff --git a/packages/dds/tree/src/test/snapshots/files/SchemaIndexFormat - schema v2.json b/packages/dds/tree/src/test/snapshots/files/SchemaIndexFormat - schema v2.json new file mode 100644 index 000000000000..160d0d9a2d6f --- /dev/null +++ b/packages/dds/tree/src/test/snapshots/files/SchemaIndexFormat - schema v2.json @@ -0,0 +1,119 @@ +{ + "additionalProperties": false, + "type": "object", + "properties": { + "version": { + "const": 2, + "type": "number" + }, + "nodes": { + "type": "object", + "patternProperties": { + "^(.*)$": { + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "type": "object", + "properties": { + "object": { + "type": "object", + "patternProperties": { + "^(.*)$": { + "additionalProperties": false, + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "types": { + "type": "array", + "items": { + "type": "string" + } + }, + "persistedMetadata": {} + }, + "required": [ + "kind", + "types" + ] + } + } + }, + "map": { + "additionalProperties": false, + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "types": { + "type": "array", + "items": { + "type": "string" + } + }, + "persistedMetadata": {} + }, + "required": [ + "kind", + "types" + ] + }, + "leaf": { + "anyOf": [ + { + "const": 0, + "type": "number" + }, + { + "const": 1, + "type": "number" + }, + { + "const": 2, + "type": "number" + }, + { + "const": 3, + "type": "number" + }, + { + "const": 4, + "type": "number" + } + ] + }, + "persistedMetadata": {} + } + } + } + }, + "root": { + "additionalProperties": false, + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "types": { + "type": "array", + "items": { + "type": "string" + } + }, + "persistedMetadata": {} + }, + "required": [ + "kind", + "types" + ] + }, + "persistedMetadata": {} + }, + "required": [ + "version", + "nodes", + "root" + ] +} \ No newline at end of file From ba058e1131dae13cf5f3860695d7340a330ec90c Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Tue, 13 May 2025 11:05:25 -0700 Subject: [PATCH 04/45] Removed extraneous whitespace change. --- examples/apps/tree-cli-app/src/utils.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/apps/tree-cli-app/src/utils.ts b/examples/apps/tree-cli-app/src/utils.ts index 24156604781b..4618c903d7bb 100644 --- a/examples/apps/tree-cli-app/src/utils.ts +++ b/examples/apps/tree-cli-app/src/utils.ts @@ -157,6 +157,7 @@ export function exportContent(destination: string, tree: List): JsonCompatible { oldestCompatibleClient: FluidClientVersion.v2_3, idCompressor, }), + schema: extractPersistedSchema(config, FluidClientVersion.v2_3), idCompressor: idCompressor.serialize(true), }; From b819742628f3895f751ab8e5b4dbb6458046efea Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Tue, 13 May 2025 12:06:19 -0700 Subject: [PATCH 05/45] - Changed TreeNodeStoredSchema implementations to take the write version as a parameter. - Fixed naming of schema index formats. - Changed schemaChangeFormat to be schema format-agnostic. --- .../dds/tree/src/core/schema-stored/schema.ts | 62 ++++++++++++++++--- .../schema-edits/schemaChangeCodecs.ts | 6 +- .../schema-edits/schemaChangeFormat.ts | 6 +- .../feature-libraries/schema-index/codec.ts | 4 +- .../feature-libraries/schema-index/index.ts | 3 +- .../dds/tree/src/shared-tree/sharedTree.ts | 4 +- .../tree/src/simple-tree/api/storedSchema.ts | 4 +- 7 files changed, 64 insertions(+), 25 deletions(-) diff --git a/packages/dds/tree/src/core/schema-stored/schema.ts b/packages/dds/tree/src/core/schema-stored/schema.ts index d600b545d4f4..97e7f2f2689b 100644 --- a/packages/dds/tree/src/core/schema-stored/schema.ts +++ b/packages/dds/tree/src/core/schema-stored/schema.ts @@ -17,9 +17,17 @@ import { import type { FieldSchemaFormat as FieldSchemaFormatV2, PersistedMetadataFormat, + TreeNodeSchemaDataFormat as TreeNodeSchemaDataFormatV2, } from "./formatV2.js"; import type { Multiplicity } from "./multiplicity.js"; +// TODO: Remove local copy. +export enum SchemaCodecVersion { + v1 = 1, + // Adds persisted metadata to the schema. + v2 = 2, +} + type FieldSchemaFormat = FieldSchemaFormatV1 | FieldSchemaFormatV2; /** @@ -174,7 +182,9 @@ export abstract class TreeNodeStoredSchema { * This is uses an opaque type to avoid leaking these types out of the package, * and is runtime validated by the codec. */ - public abstract encode(): TreeNodeSchemaDataFormatV1; + public abstract encode( + schemaWriteVersion: SchemaCodecVersion, + ): TreeNodeSchemaDataFormatV1 | TreeNodeSchemaDataFormatV2; /** * Returns the schema for the provided field. @@ -199,18 +209,35 @@ export class ObjectNodeStoredSchema extends TreeNodeStoredSchema { super(); } - public override encode(): TreeNodeSchemaDataFormatV1 { + public override encode( + schemaWriteVersion: SchemaCodecVersion, + ): TreeNodeSchemaDataFormatV1 | TreeNodeSchemaDataFormatV2 { const fieldsObject: Record = Object.create(null); // Sort fields to ensure output is identical for for equivalent schema (since field order is not considered significant). // This makes comparing schema easier, and ensures chunk reuse for schema summaries isn't needlessly broken. for (const key of [...this.objectNodeFields.keys()].sort()) { + let value: FieldSchemaFormatV1 | FieldSchemaFormatV2 | undefined; + + switch (schemaWriteVersion) { + case SchemaCodecVersion.v1: + value = encodeFieldSchemaV1( + this.objectNodeFields.get(key) ?? fail(0xae6 /* missing field */), + ); + break; + case SchemaCodecVersion.v2: + value = encodeFieldSchemaV2( + this.objectNodeFields.get(key) ?? fail(0xae7 /* missing field */), + ); + break; + default: + fail(`Cannot decode schema version ${schemaWriteVersion}`); + } + Object.defineProperty(fieldsObject, key, { enumerable: true, configurable: true, writable: true, - value: encodeFieldSchemaV1( - this.objectNodeFields.get(key) ?? fail(0xae7 /* missing field */), - ), + value, }); } return { @@ -238,10 +265,23 @@ export class MapNodeStoredSchema extends TreeNodeStoredSchema { super(); } - public override encode(): TreeNodeSchemaDataFormatV1 { - return { - map: encodeFieldSchemaV1(this.mapFields), - }; + public override encode( + schemaWriteVersion: SchemaCodecVersion, + ): TreeNodeSchemaDataFormatV1 | TreeNodeSchemaDataFormatV2 { + switch (schemaWriteVersion) { + case SchemaCodecVersion.v1: { + return { + map: encodeFieldSchemaV1(this.mapFields), + }; + } + case SchemaCodecVersion.v2: { + return { + map: encodeFieldSchemaV2(this.mapFields), + }; + } + default: + fail(`Cannot decode schema version ${schemaWriteVersion}`); + } } public override getFieldSchema(field: FieldKey): TreeFieldStoredSchema { @@ -268,7 +308,9 @@ export class LeafNodeStoredSchema extends TreeNodeStoredSchema { super(); } - public override encode(): TreeNodeSchemaDataFormatV1 { + public override encode( + schemaWriteVersion: SchemaCodecVersion, + ): TreeNodeSchemaDataFormatV1 | TreeNodeSchemaDataFormatV2 { return { leaf: encodeValueSchema(this.leafValue), }; diff --git a/packages/dds/tree/src/feature-libraries/schema-edits/schemaChangeCodecs.ts b/packages/dds/tree/src/feature-libraries/schema-edits/schemaChangeCodecs.ts index 586f75ad471d..a0d3d1e33334 100644 --- a/packages/dds/tree/src/feature-libraries/schema-edits/schemaChangeCodecs.ts +++ b/packages/dds/tree/src/feature-libraries/schema-edits/schemaChangeCodecs.ts @@ -13,11 +13,7 @@ import { makeVersionDispatchingCodec, withSchemaValidation, } from "../../codec/index.js"; -import { - makeSchemaCodec, - SchemaCodecVersion, - type Format as FormatV1, -} from "../schema-index/index.js"; +import { makeSchemaCodec, SchemaCodecVersion, type FormatV1 } from "../schema-index/index.js"; import { EncodedSchemaChange } from "./schemaChangeFormat.js"; import type { SchemaChange } from "./schemaChangeTypes.js"; diff --git a/packages/dds/tree/src/feature-libraries/schema-edits/schemaChangeFormat.ts b/packages/dds/tree/src/feature-libraries/schema-edits/schemaChangeFormat.ts index 046aadcfe221..2b6e6c299c41 100644 --- a/packages/dds/tree/src/feature-libraries/schema-edits/schemaChangeFormat.ts +++ b/packages/dds/tree/src/feature-libraries/schema-edits/schemaChangeFormat.ts @@ -4,11 +4,11 @@ */ import { type Static, Type } from "@sinclair/typebox"; -import { Format as FormatV1 } from "../schema-index/index.js"; +import { JsonCompatibleReadOnlySchema } from "../../util/index.js"; export const EncodedSchemaChange = Type.Object({ - new: FormatV1, - old: FormatV1, + new: JsonCompatibleReadOnlySchema, + old: JsonCompatibleReadOnlySchema, }); export type EncodedSchemaChange = Static; diff --git a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts index 48f465be1a15..5aaf10537df1 100644 --- a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts +++ b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts @@ -117,7 +117,7 @@ function encodeRepoV1(repo: TreeStoredSchema): FormatV1 { enumerable: true, configurable: true, writable: true, - value: schema.encode(), + value: schema.encode(SchemaCodecVersion.v1), }); } return { @@ -137,7 +137,7 @@ function encodeRepoV2(repo: TreeStoredSchema): FormatV2 { enumerable: true, configurable: true, writable: true, - value: schema.encode(), + value: schema.encode(SchemaCodecVersion.v2), }); } return { diff --git a/packages/dds/tree/src/feature-libraries/schema-index/index.ts b/packages/dds/tree/src/feature-libraries/schema-index/index.ts index c4a0c5b24ec5..e4843abbaf0e 100644 --- a/packages/dds/tree/src/feature-libraries/schema-index/index.ts +++ b/packages/dds/tree/src/feature-libraries/schema-index/index.ts @@ -10,4 +10,5 @@ export { SchemaCodecVersion, clientVersionToSchemaVersion, } from "./codec.js"; -export { Format } from "./formatV1.js"; +export { Format as FormatV1 } from "./formatV1.js"; +export { Format as FormatV2 } from "./formatV2.js"; diff --git a/packages/dds/tree/src/shared-tree/sharedTree.ts b/packages/dds/tree/src/shared-tree/sharedTree.ts index fb74a931fa4f..19c4e08c608b 100644 --- a/packages/dds/tree/src/shared-tree/sharedTree.ts +++ b/packages/dds/tree/src/shared-tree/sharedTree.ts @@ -99,7 +99,7 @@ import { throwIfBroken, } from "../util/index.js"; // eslint-disable-next-line import/no-internal-modules -import type { Format } from "../feature-libraries/schema-index/index.js"; +import type { FormatV1 } from "../feature-libraries/schema-index/index.js"; /** * Copy of data from an {@link ITreePrivate} at some point in time. @@ -481,7 +481,7 @@ export function persistedToSimpleSchema( options: ICodecOptions, ): SimpleTreeSchema { const schemaCodec = makeSchemaCodec(options, SchemaCodecVersion.v1); - const stored = schemaCodec.decode(persisted as Format); + const stored = schemaCodec.decode(persisted as FormatV1); return exportSimpleSchema(stored); } diff --git a/packages/dds/tree/src/simple-tree/api/storedSchema.ts b/packages/dds/tree/src/simple-tree/api/storedSchema.ts index 885cc869a7d9..3268a5e55349 100644 --- a/packages/dds/tree/src/simple-tree/api/storedSchema.ts +++ b/packages/dds/tree/src/simple-tree/api/storedSchema.ts @@ -13,7 +13,7 @@ import { } from "../../feature-libraries/index.js"; import { clientVersionToSchemaVersion, - type Format, + type FormatV1, // eslint-disable-next-line import/no-internal-modules } from "../../feature-libraries/schema-index/index.js"; import type { JsonCompatible } from "../../util/index.js"; @@ -101,7 +101,7 @@ export function comparePersistedSchema( // Any version can be passed down to makeSchemaCodec here. // We only use the decode part, which always dispatches to the correct codec based on the version in the data, not the version passed to `makeSchemaCodec`. const schemaCodec = makeSchemaCodec(options, SchemaCodecVersion.v1); - const stored = schemaCodec.decode(persisted as Format); + const stored = schemaCodec.decode(persisted as FormatV1); const viewSchema = new SchemaCompatibilityTester( defaultSchemaPolicy, {}, From 0b5c84228011a812fcb9f94696bd06d4ba3a5b33 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Tue, 13 May 2025 12:46:00 -0700 Subject: [PATCH 06/45] - Changed the stored schema implementation to handle multiple schema formats. - Moved SchemaCodecVersion. --- packages/dds/tree/src/core/index.ts | 1 + .../dds/tree/src/core/schema-stored/index.ts | 1 + .../dds/tree/src/core/schema-stored/schema.ts | 1 - packages/dds/tree/src/feature-libraries/index.ts | 1 - .../schema-edits/schemaChangeCodecs.ts | 16 +++++++++++----- .../src/feature-libraries/schema-index/codec.ts | 10 +--------- .../src/feature-libraries/schema-index/index.ts | 1 - .../dds/tree/src/shared-tree/independentView.ts | 2 +- packages/dds/tree/src/shared-tree/sharedTree.ts | 2 +- .../dds/tree/src/simple-tree/api/storedSchema.ts | 3 +-- .../feature-libraries/schema-index/codec.spec.ts | 7 +++++-- .../feature-libraries/schema-index/codecUtil.ts | 2 +- packages/dds/tree/src/test/utils.ts | 2 +- 13 files changed, 24 insertions(+), 25 deletions(-) diff --git a/packages/dds/tree/src/core/index.ts b/packages/dds/tree/src/core/index.ts index 573d788d51b9..58c58dd26f0c 100644 --- a/packages/dds/tree/src/core/index.ts +++ b/packages/dds/tree/src/core/index.ts @@ -149,6 +149,7 @@ export { type SchemaAndPolicy, Multiplicity, type SchemaPolicy, + SchemaCodecVersion, } from "./schema-stored/index.js"; export { diff --git a/packages/dds/tree/src/core/schema-stored/index.ts b/packages/dds/tree/src/core/schema-stored/index.ts index 7202636783da..3a6c789af0f1 100644 --- a/packages/dds/tree/src/core/schema-stored/index.ts +++ b/packages/dds/tree/src/core/schema-stored/index.ts @@ -23,6 +23,7 @@ export { storedSchemaDecodeDispatcher, type SchemaAndPolicy, type SchemaPolicy, + SchemaCodecVersion, } from "./schema.js"; export { type TreeStoredSchemaSubscription, diff --git a/packages/dds/tree/src/core/schema-stored/schema.ts b/packages/dds/tree/src/core/schema-stored/schema.ts index 97e7f2f2689b..b06ce6949a2f 100644 --- a/packages/dds/tree/src/core/schema-stored/schema.ts +++ b/packages/dds/tree/src/core/schema-stored/schema.ts @@ -21,7 +21,6 @@ import type { } from "./formatV2.js"; import type { Multiplicity } from "./multiplicity.js"; -// TODO: Remove local copy. export enum SchemaCodecVersion { v1 = 1, // Adds persisted metadata to the schema. diff --git a/packages/dds/tree/src/feature-libraries/index.ts b/packages/dds/tree/src/feature-libraries/index.ts index 16334d694d79..330ca03874b0 100644 --- a/packages/dds/tree/src/feature-libraries/index.ts +++ b/packages/dds/tree/src/feature-libraries/index.ts @@ -22,7 +22,6 @@ export { encodeTreeSchema, makeSchemaCodec, makeSchemaCodecs, - SchemaCodecVersion, } from "./schema-index/index.js"; export { stackTreeNodeCursor, diff --git a/packages/dds/tree/src/feature-libraries/schema-edits/schemaChangeCodecs.ts b/packages/dds/tree/src/feature-libraries/schema-edits/schemaChangeCodecs.ts index a0d3d1e33334..28727acc815b 100644 --- a/packages/dds/tree/src/feature-libraries/schema-edits/schemaChangeCodecs.ts +++ b/packages/dds/tree/src/feature-libraries/schema-edits/schemaChangeCodecs.ts @@ -13,10 +13,11 @@ import { makeVersionDispatchingCodec, withSchemaValidation, } from "../../codec/index.js"; -import { makeSchemaCodec, SchemaCodecVersion, type FormatV1 } from "../schema-index/index.js"; +import { makeSchemaCodec } from "../schema-index/index.js"; import { EncodedSchemaChange } from "./schemaChangeFormat.js"; import type { SchemaChange } from "./schemaChangeTypes.js"; +import { SchemaCodecVersion } from "../../core/index.js"; /** * Create a family of schema change codecs. @@ -24,7 +25,10 @@ import type { SchemaChange } from "./schemaChangeTypes.js"; * @returns The composed codec family. */ export function makeSchemaChangeCodecs(options: ICodecOptions): ICodecFamily { - return makeCodecFamily([[SchemaCodecVersion.v1, makeSchemaChangeCodecV1(options)]]); + return makeCodecFamily([ + [SchemaCodecVersion.v1, makeSchemaChangeCodecV1(options, SchemaCodecVersion.v1)], + [SchemaCodecVersion.v2, makeSchemaChangeCodecV1(options, SchemaCodecVersion.v2)], + ]); } /** @@ -44,12 +48,14 @@ export function makeSchemaChangeCodec( /** * Compose the v1 schema change codec. * @param options - The codec options. + * @param schemaWriteVersion - The schema write version. * @returns The composed schema change codec. */ function makeSchemaChangeCodecV1( options: ICodecOptions, + schemaWriteVersion: SchemaCodecVersion, ): IJsonCodec { - const schemaCodec = makeSchemaCodec(options, SchemaCodecVersion.v1); + const schemaCodec = makeSchemaCodec(options, schemaWriteVersion); const schemaChangeCodec: IJsonCodec = { encode: (schemaChange) => { assert( @@ -57,8 +63,8 @@ function makeSchemaChangeCodecV1( 0x933 /* Inverse schema changes should never be transmitted */, ); return { - new: schemaCodec.encode(schemaChange.schema.new) as FormatV1, - old: schemaCodec.encode(schemaChange.schema.old) as FormatV1, + new: schemaCodec.encode(schemaChange.schema.new), + old: schemaCodec.encode(schemaChange.schema.old), }; }, decode: (encoded) => { diff --git a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts index 5aaf10537df1..2338d18ead74 100644 --- a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts +++ b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts @@ -28,15 +28,7 @@ import { brand, type JsonCompatible } from "../../util/index.js"; import { Format as FormatV1 } from "./formatV1.js"; import { Format as FormatV2 } from "./formatV2.js"; - -/** - * Versions for the codec that encodes an in-memory representation of a stored schema {@link TreeStoredSchema} into a persisted format (or decodes it in the opposite direction). - */ -export enum SchemaCodecVersion { - v1 = 1, - // Adds persisted metadata to the schema. - v2 = 2, -} +import { SchemaCodecVersion } from "../../core/index.js"; /** * Convert a FluidClientVersion to a SchemaCodecVersion. diff --git a/packages/dds/tree/src/feature-libraries/schema-index/index.ts b/packages/dds/tree/src/feature-libraries/schema-index/index.ts index e4843abbaf0e..aae9846c76fe 100644 --- a/packages/dds/tree/src/feature-libraries/schema-index/index.ts +++ b/packages/dds/tree/src/feature-libraries/schema-index/index.ts @@ -7,7 +7,6 @@ export { SchemaSummarizer, encodeTreeSchema } from "./schemaSummarizer.js"; export { makeSchemaCodec, makeSchemaCodecs, - SchemaCodecVersion, clientVersionToSchemaVersion, } from "./codec.js"; export { Format as FormatV1 } from "./formatV1.js"; diff --git a/packages/dds/tree/src/shared-tree/independentView.ts b/packages/dds/tree/src/shared-tree/independentView.ts index 648bb5696f86..e70648691df2 100644 --- a/packages/dds/tree/src/shared-tree/independentView.ts +++ b/packages/dds/tree/src/shared-tree/independentView.ts @@ -13,6 +13,7 @@ import type { ICodecOptions } from "../codec/index.js"; import { type RevisionTag, RevisionTagCodec, + SchemaCodecVersion, TreeStoredSchemaRepository, } from "../core/index.js"; import { @@ -23,7 +24,6 @@ import { defaultSchemaPolicy, TreeCompressionStrategy, initializeForest, - SchemaCodecVersion, } from "../feature-libraries/index.js"; // eslint-disable-next-line import/no-internal-modules import type { Format } from "../feature-libraries/schema-index/formatV1.js"; diff --git a/packages/dds/tree/src/shared-tree/sharedTree.ts b/packages/dds/tree/src/shared-tree/sharedTree.ts index 19c4e08c608b..6d04641f7470 100644 --- a/packages/dds/tree/src/shared-tree/sharedTree.ts +++ b/packages/dds/tree/src/shared-tree/sharedTree.ts @@ -31,6 +31,7 @@ import { MapNodeStoredSchema, ObjectNodeStoredSchema, RevisionTagCodec, + SchemaCodecVersion, type TaggedChange, type TreeFieldStoredSchema, type TreeNodeSchemaIdentifier, @@ -46,7 +47,6 @@ import { DetachedFieldIndexSummarizer, FieldKinds, ForestSummarizer, - SchemaCodecVersion, SchemaSummarizer, TreeCompressionStrategy, buildChunkedForest, diff --git a/packages/dds/tree/src/simple-tree/api/storedSchema.ts b/packages/dds/tree/src/simple-tree/api/storedSchema.ts index 3268a5e55349..9f236c10ae8b 100644 --- a/packages/dds/tree/src/simple-tree/api/storedSchema.ts +++ b/packages/dds/tree/src/simple-tree/api/storedSchema.ts @@ -4,12 +4,11 @@ */ import type { FluidClientVersion, ICodecOptions } from "../../codec/index.js"; -import type { TreeStoredSchema } from "../../core/index.js"; +import { SchemaCodecVersion, type TreeStoredSchema } from "../../core/index.js"; import { defaultSchemaPolicy, encodeTreeSchema, makeSchemaCodec, - SchemaCodecVersion, } from "../../feature-libraries/index.js"; import { clientVersionToSchemaVersion, diff --git a/packages/dds/tree/src/test/feature-libraries/schema-index/codec.spec.ts b/packages/dds/tree/src/test/feature-libraries/schema-index/codec.spec.ts index 46fc6d25f38f..ab53541f6896 100644 --- a/packages/dds/tree/src/test/feature-libraries/schema-index/codec.spec.ts +++ b/packages/dds/tree/src/test/feature-libraries/schema-index/codec.spec.ts @@ -7,13 +7,16 @@ import { strict as assert } from "node:assert"; // Allow importing from this specific file which is being tested: -import type { FieldKindIdentifier, TreeStoredSchema } from "../../../core/index.js"; +import { + SchemaCodecVersion, + type FieldKindIdentifier, + type TreeStoredSchema, +} from "../../../core/index.js"; import { typeboxValidator } from "../../../external-utilities/index.js"; import { allowsRepoSuperset, defaultSchemaPolicy, makeSchemaCodec, - SchemaCodecVersion, } from "../../../feature-libraries/index.js"; /* eslint-disable-next-line import/no-internal-modules */ import { Format as FormatV1 } from "../../../feature-libraries/schema-index/formatV1.js"; diff --git a/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts b/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts index d2bf54105f0f..2037dba8bb1a 100644 --- a/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts +++ b/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts @@ -4,9 +4,9 @@ */ import { FluidClientVersion } from "../../../codec/index.js"; +import { SchemaCodecVersion } from "../../../core/index.js"; import { makeSchemaCodecs, - SchemaCodecVersion, // eslint-disable-next-line import/no-internal-modules } from "../../../feature-libraries/schema-index/index.js"; import { ajvValidator } from "../../codec/index.js"; diff --git a/packages/dds/tree/src/test/utils.ts b/packages/dds/tree/src/test/utils.ts index 47a838d6225f..3dfad4c1f3fa 100644 --- a/packages/dds/tree/src/test/utils.ts +++ b/packages/dds/tree/src/test/utils.ts @@ -93,6 +93,7 @@ import { type DeltaDetachedNodeRename, type NormalizedFieldUpPath, type ExclusiveMapTree, + SchemaCodecVersion, } from "../core/index.js"; import { typeboxValidator } from "../external-utilities/index.js"; import { @@ -113,7 +114,6 @@ import { initializeForest, chunkFieldSingle, makeSchemaCodec, - SchemaCodecVersion, } from "../feature-libraries/index.js"; import { type CheckoutEvents, From 1117c2c1c8e05ef512a43ef9c49908bba0aed025 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Wed, 14 May 2025 14:47:03 -0700 Subject: [PATCH 07/45] Apply suggestions from code review Co-authored-by: Craig Macomber (Microsoft) <42876482+CraigMacomber@users.noreply.github.com> --- packages/dds/tree/src/core/schema-stored/formatV2.ts | 2 +- packages/dds/tree/src/core/schema-stored/schema.ts | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/dds/tree/src/core/schema-stored/formatV2.ts b/packages/dds/tree/src/core/schema-stored/formatV2.ts index f9cbc68abf5a..472e926f2c47 100644 --- a/packages/dds/tree/src/core/schema-stored/formatV2.ts +++ b/packages/dds/tree/src/core/schema-stored/formatV2.ts @@ -96,7 +96,7 @@ export const TreeNodeSchemaDataFormat = Type.Object( /** * Persisted metadata for this node. */ - persistedMetadata: PersistedMetadataFormat, + metadata: PersistedMetadataFormat, }, unionOptions, ); diff --git a/packages/dds/tree/src/core/schema-stored/schema.ts b/packages/dds/tree/src/core/schema-stored/schema.ts index b06ce6949a2f..2a0ff31d1bc3 100644 --- a/packages/dds/tree/src/core/schema-stored/schema.ts +++ b/packages/dds/tree/src/core/schema-stored/schema.ts @@ -138,7 +138,7 @@ export interface TreeFieldStoredSchema { * @remarks * Discarded when encoding to {@link SchemaFormatVersion.V1}. */ - persistedMetadata?: PersistedMetadataFormat; + readonly persistedMetadata: PersistedMetadataFormat | undefined; } /** @@ -366,9 +366,7 @@ export function encodeFieldSchemaV1(schema: TreeFieldStoredSchema): FieldSchemaF export function encodeFieldSchemaV2(schema: TreeFieldStoredSchema): FieldSchemaFormatV2 { return { - kind: schema.kind, - // Types are sorted by identifier to improve stability of persisted data to increase chance of schema blob reuse. - types: [...schema.types].sort(), + ...encodeFieldSchemaV1(schema), persistedMetadata: schema.persistedMetadata, }; } From 94a5c39fe040a8b1233743bc21c4d011de88eab5 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Wed, 14 May 2025 14:59:55 -0700 Subject: [PATCH 08/45] - Switched to using the SchemaCodecVersion in public APIs. Eventually we want something more intuitive (e.g., minimum client version). - Split the encode method for TreeNodeStoredSchema. --- .../dds/tree/api-report/tree.alpha.api.md | 5 +- packages/dds/tree/src/codec/codec.ts | 2 - .../dds/tree/src/core/schema-stored/schema.ts | 93 ++++++++++--------- .../feature-libraries/schema-index/codec.ts | 6 +- .../dds/tree/src/shared-tree/treeAlpha.ts | 6 +- .../tree/src/simple-tree/api/storedSchema.ts | 14 +-- .../schema-index/codecUtil.ts | 2 - .../test/simple-tree/api/storedSchema.spec.ts | 6 +- 8 files changed, 66 insertions(+), 68 deletions(-) diff --git a/packages/dds/tree/api-report/tree.alpha.api.md b/packages/dds/tree/api-report/tree.alpha.api.md index da1756afe87d..02a88bd22190 100644 --- a/packages/dds/tree/api-report/tree.alpha.api.md +++ b/packages/dds/tree/api-report/tree.alpha.api.md @@ -152,7 +152,7 @@ export function evaluateLazySchema(value: LazyItem) type ExtractItemType = Item extends () => infer Result ? Result : Item; // @alpha -export function extractPersistedSchema(schema: SimpleTreeSchema, oldestCompatibleClient: FluidClientVersion): JsonCompatible; +export function extractPersistedSchema(schema: SimpleTreeSchema, schemaWriteVersion: number): JsonCompatible; // @alpha @system export type FactoryContent = IFluidHandle | string | number | boolean | null | Iterable | readonly InsertableContent[] | FactoryContentObject; @@ -237,8 +237,7 @@ export enum FluidClientVersion { v2_0 = "v2_0", v2_1 = "v2_1", v2_2 = "v2_2", - v2_3 = "v2_3", - v2_4 = "v2_4" + v2_3 = "v2_3" } // @alpha diff --git a/packages/dds/tree/src/codec/codec.ts b/packages/dds/tree/src/codec/codec.ts index a7b65a554007..1d3574576be3 100644 --- a/packages/dds/tree/src/codec/codec.ts +++ b/packages/dds/tree/src/codec/codec.ts @@ -352,8 +352,6 @@ export enum FluidClientVersion { v2_2 = "v2_2", /** Fluid Framework Client 2.3 and newer. */ v2_3 = "v2_3", - /** Fluid Framework Client 2.4 and newer. */ - v2_4 = "v2_4", } /** diff --git a/packages/dds/tree/src/core/schema-stored/schema.ts b/packages/dds/tree/src/core/schema-stored/schema.ts index b06ce6949a2f..a16a1ee432df 100644 --- a/packages/dds/tree/src/core/schema-stored/schema.ts +++ b/packages/dds/tree/src/core/schema-stored/schema.ts @@ -181,9 +181,15 @@ export abstract class TreeNodeStoredSchema { * This is uses an opaque type to avoid leaking these types out of the package, * and is runtime validated by the codec. */ - public abstract encode( - schemaWriteVersion: SchemaCodecVersion, - ): TreeNodeSchemaDataFormatV1 | TreeNodeSchemaDataFormatV2; + public abstract encodeV1(): TreeNodeSchemaDataFormatV1; + + /** + * @privateRemarks + * Returns TreeNodeSchemaDataFormat. + * This is uses an opaque type to avoid leaking these types out of the package, + * and is runtime validated by the codec. + */ + public abstract encodeV2(): TreeNodeSchemaDataFormatV2; /** * Returns the schema for the provided field. @@ -208,29 +214,35 @@ export class ObjectNodeStoredSchema extends TreeNodeStoredSchema { super(); } - public override encode( - schemaWriteVersion: SchemaCodecVersion, - ): TreeNodeSchemaDataFormatV1 | TreeNodeSchemaDataFormatV2 { + public override encodeV1(): TreeNodeSchemaDataFormatV1 { + const fieldsObject: Record = Object.create(null); + // Sort fields to ensure output is identical for for equivalent schema (since field order is not considered significant). + // This makes comparing schema easier, and ensures chunk reuse for schema summaries isn't needlessly broken. + for (const key of [...this.objectNodeFields.keys()].sort()) { + const value = encodeFieldSchemaV1( + this.objectNodeFields.get(key) ?? fail(0xae6 /* missing field */), + ); + + Object.defineProperty(fieldsObject, key, { + enumerable: true, + configurable: true, + writable: true, + value, + }); + } + return { + object: fieldsObject, + }; + } + + public override encodeV2(): TreeNodeSchemaDataFormatV2 { const fieldsObject: Record = Object.create(null); // Sort fields to ensure output is identical for for equivalent schema (since field order is not considered significant). // This makes comparing schema easier, and ensures chunk reuse for schema summaries isn't needlessly broken. for (const key of [...this.objectNodeFields.keys()].sort()) { - let value: FieldSchemaFormatV1 | FieldSchemaFormatV2 | undefined; - - switch (schemaWriteVersion) { - case SchemaCodecVersion.v1: - value = encodeFieldSchemaV1( - this.objectNodeFields.get(key) ?? fail(0xae6 /* missing field */), - ); - break; - case SchemaCodecVersion.v2: - value = encodeFieldSchemaV2( - this.objectNodeFields.get(key) ?? fail(0xae7 /* missing field */), - ); - break; - default: - fail(`Cannot decode schema version ${schemaWriteVersion}`); - } + const value = encodeFieldSchemaV1( + this.objectNodeFields.get(key) ?? fail(0xae6 /* missing field */), + ); Object.defineProperty(fieldsObject, key, { enumerable: true, @@ -264,23 +276,16 @@ export class MapNodeStoredSchema extends TreeNodeStoredSchema { super(); } - public override encode( - schemaWriteVersion: SchemaCodecVersion, - ): TreeNodeSchemaDataFormatV1 | TreeNodeSchemaDataFormatV2 { - switch (schemaWriteVersion) { - case SchemaCodecVersion.v1: { - return { - map: encodeFieldSchemaV1(this.mapFields), - }; - } - case SchemaCodecVersion.v2: { - return { - map: encodeFieldSchemaV2(this.mapFields), - }; - } - default: - fail(`Cannot decode schema version ${schemaWriteVersion}`); - } + public override encodeV1(): TreeNodeSchemaDataFormatV1 { + return { + map: encodeFieldSchemaV1(this.mapFields), + }; + } + + public override encodeV2(): TreeNodeSchemaDataFormatV2 { + return { + map: encodeFieldSchemaV2(this.mapFields), + }; } public override getFieldSchema(field: FieldKey): TreeFieldStoredSchema { @@ -307,9 +312,13 @@ export class LeafNodeStoredSchema extends TreeNodeStoredSchema { super(); } - public override encode( - schemaWriteVersion: SchemaCodecVersion, - ): TreeNodeSchemaDataFormatV1 | TreeNodeSchemaDataFormatV2 { + public override encodeV1(): TreeNodeSchemaDataFormatV1 { + return { + leaf: encodeValueSchema(this.leafValue), + }; + } + + public override encodeV2(): TreeNodeSchemaDataFormatV1 { return { leaf: encodeValueSchema(this.leafValue), }; diff --git a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts index 2338d18ead74..5cdc66204dbe 100644 --- a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts +++ b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts @@ -44,8 +44,6 @@ export function clientVersionToSchemaVersion( case FluidClientVersion.v2_2: case FluidClientVersion.v2_3: return SchemaCodecVersion.v1; - case FluidClientVersion.v2_4: - return SchemaCodecVersion.v2; default: unreachableCase(clientVersion); } @@ -109,7 +107,7 @@ function encodeRepoV1(repo: TreeStoredSchema): FormatV1 { enumerable: true, configurable: true, writable: true, - value: schema.encode(SchemaCodecVersion.v1), + value: schema.encodeV1(), }); } return { @@ -129,7 +127,7 @@ function encodeRepoV2(repo: TreeStoredSchema): FormatV2 { enumerable: true, configurable: true, writable: true, - value: schema.encode(SchemaCodecVersion.v2), + value: schema.encodeV2(), }); } return { diff --git a/packages/dds/tree/src/shared-tree/treeAlpha.ts b/packages/dds/tree/src/shared-tree/treeAlpha.ts index 0a7eba1fa432..e59460b9b0db 100644 --- a/packages/dds/tree/src/shared-tree/treeAlpha.ts +++ b/packages/dds/tree/src/shared-tree/treeAlpha.ts @@ -39,7 +39,7 @@ import { } from "../simple-tree/index.js"; import type { JsonCompatible } from "../util/index.js"; import { noopValidator, type FluidClientVersion, type ICodecOptions } from "../codec/index.js"; -import type { ITreeCursorSynchronous } from "../core/index.js"; +import { SchemaCodecVersion, type ITreeCursorSynchronous } from "../core/index.js"; import { cursorForMapTreeField, defaultSchemaPolicy, @@ -52,7 +52,6 @@ import { } from "../feature-libraries/index.js"; import { independentInitializedView, type ViewContent } from "./independentView.js"; import { SchematizingSimpleTreeView, ViewSlot } from "./schematizingTreeView.js"; -import { currentVersion } from "../codec/index.js"; /** * Extensions to {@link (Tree:interface)} and {@link (TreeBeta:interface)} which are not yet stable. @@ -316,7 +315,8 @@ export const TreeAlpha: TreeAlpha = { ): Unhydrated> { const config = new TreeViewConfigurationAlpha({ schema }); const content: ViewContent = { - schema: extractPersistedSchema(config, currentVersion), + // TODO + schema: extractPersistedSchema(config, SchemaCodecVersion.v2), tree: compressedData, idCompressor: options.idCompressor ?? createIdCompressor(), }; diff --git a/packages/dds/tree/src/simple-tree/api/storedSchema.ts b/packages/dds/tree/src/simple-tree/api/storedSchema.ts index 9f236c10ae8b..32d52efe5d08 100644 --- a/packages/dds/tree/src/simple-tree/api/storedSchema.ts +++ b/packages/dds/tree/src/simple-tree/api/storedSchema.ts @@ -3,18 +3,15 @@ * Licensed under the MIT License. */ -import type { FluidClientVersion, ICodecOptions } from "../../codec/index.js"; +import type { ICodecOptions } from "../../codec/index.js"; import { SchemaCodecVersion, type TreeStoredSchema } from "../../core/index.js"; import { defaultSchemaPolicy, encodeTreeSchema, makeSchemaCodec, } from "../../feature-libraries/index.js"; -import { - clientVersionToSchemaVersion, - type FormatV1, - // eslint-disable-next-line import/no-internal-modules -} from "../../feature-libraries/schema-index/index.js"; +// eslint-disable-next-line import/no-internal-modules +import type { FormatV1 } from "../../feature-libraries/schema-index/index.js"; import type { JsonCompatible } from "../../util/index.js"; import { normalizeFieldSchema, type ImplicitFieldSchema } from "../schemaTypes.js"; import { simpleToStoredSchema } from "../toStoredSchema.js"; @@ -54,11 +51,10 @@ import type { SimpleTreeSchema } from "../simpleSchema.js"; */ export function extractPersistedSchema( schema: SimpleTreeSchema, - oldestCompatibleClient: FluidClientVersion, + schemaWriteVersion: number, ): JsonCompatible { const stored = simpleToStoredSchema(schema); - const writeVersion = clientVersionToSchemaVersion(oldestCompatibleClient); - return encodeTreeSchema(stored, writeVersion); + return encodeTreeSchema(stored, schemaWriteVersion); } /** diff --git a/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts b/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts index 2037dba8bb1a..02530f351c2c 100644 --- a/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts +++ b/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts @@ -30,8 +30,6 @@ export function schemaFormatToClientVersion( switch (schemaFormat) { case SchemaCodecVersion.v1: return FluidClientVersion.v2_0; - case SchemaCodecVersion.v2: - return FluidClientVersion.v2_4; default: assert(false, `Unsupported schema format: ${schemaFormat}`); } diff --git a/packages/dds/tree/src/test/simple-tree/api/storedSchema.spec.ts b/packages/dds/tree/src/test/simple-tree/api/storedSchema.spec.ts index aebf556b55f3..7d2a55f43a93 100644 --- a/packages/dds/tree/src/test/simple-tree/api/storedSchema.spec.ts +++ b/packages/dds/tree/src/test/simple-tree/api/storedSchema.spec.ts @@ -13,8 +13,8 @@ import { import { testSimpleTrees } from "../../testTrees.js"; import { takeJsonSnapshot, useSnapshotDirectory } from "../../snapshots/index.js"; import { typeboxValidator } from "../../../external-utilities/index.js"; -import { FluidClientVersion } from "../../../codec/index.js"; import { TreeViewConfigurationAlpha } from "../../../simple-tree/index.js"; +import { SchemaCodecVersion } from "../../../core/index.js"; describe("simple-tree storedSchema", () => { describe("test-schema", () => { @@ -24,7 +24,7 @@ describe("simple-tree storedSchema", () => { it(`${test.name} - schema v1`, () => { const persisted = extractPersistedSchema( new TreeViewConfigurationAlpha({ schema: test.schema }), - FluidClientVersion.v2_0, + SchemaCodecVersion.v1, ); takeJsonSnapshot(persisted); @@ -35,7 +35,7 @@ describe("simple-tree storedSchema", () => { it(`comparePersistedSchema to self ${test.name} - schema v1`, () => { const persistedA = extractPersistedSchema( new TreeViewConfigurationAlpha({ schema: test.schema }), - FluidClientVersion.v2_0, + SchemaCodecVersion.v1, ); const status = comparePersistedSchema( From 0ba34d191158a064f1bceecda390afd561ae6263 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Wed, 14 May 2025 16:22:42 -0700 Subject: [PATCH 09/45] - Removed the schema version constant and replaced it with SchemaCodecVersion. - Changed schema-stored/FormatV2 to only define the new elements. - Updated all instantiations of TreeFieldStoredSchema so that persistedMetadata is specified. - HACK: importCompressed needs to handle both schema formats. A few of the roundtrip JSON tests are failing. --- .../tree/src/core/schema-stored/formatV1.ts | 2 - .../tree/src/core/schema-stored/formatV2.ts | 74 ++----------------- .../dds/tree/src/core/schema-stored/schema.ts | 3 +- .../modular-schema/fieldKindWithEditor.ts | 2 + .../feature-libraries/schema-index/codec.ts | 12 +-- .../schema-index/formatV1.ts | 4 +- .../schema-index/formatV2.ts | 4 +- .../tree/src/shared-tree/schematizeTree.ts | 1 + .../api/schemaCompatibilityTester.ts | 2 +- .../tree/src/simple-tree/toStoredSchema.ts | 9 ++- .../codec/schemaBasedEncode.spec.ts | 7 +- .../default-schema/schemaChecker.spec.ts | 1 + .../flex-tree/lazyField.spec.ts | 1 + .../modular-schema/comparison.spec.ts | 1 + .../modular-schema/isNeverTree.spec.ts | 1 + .../dds/tree/src/test/sequenceRootUtils.ts | 1 + .../sharedTreeChangeFamily.spec.ts | 1 + .../src/test/simple-tree/toMapTree.spec.ts | 1 + .../files/SchemaIndexFormat - schema v2.json | 8 +- packages/dds/tree/src/test/testTrees.ts | 11 ++- 20 files changed, 56 insertions(+), 90 deletions(-) diff --git a/packages/dds/tree/src/core/schema-stored/formatV1.ts b/packages/dds/tree/src/core/schema-stored/formatV1.ts index b2a3d3c80343..755ead033697 100644 --- a/packages/dds/tree/src/core/schema-stored/formatV1.ts +++ b/packages/dds/tree/src/core/schema-stored/formatV1.ts @@ -8,8 +8,6 @@ import { type ObjectOptions, type Static, Type } from "@sinclair/typebox"; import { unionOptions } from "../../codec/index.js"; import { type Brand, brandedStringType } from "../../util/index.js"; -export const version = 1 as const; - /** * Key (aka Name or Label) for a field which is scoped to a specific TreeNodeStoredSchema. * diff --git a/packages/dds/tree/src/core/schema-stored/formatV2.ts b/packages/dds/tree/src/core/schema-stored/formatV2.ts index 472e926f2c47..d7962c9c3ac3 100644 --- a/packages/dds/tree/src/core/schema-stored/formatV2.ts +++ b/packages/dds/tree/src/core/schema-stored/formatV2.ts @@ -6,50 +6,12 @@ import { type ObjectOptions, type Static, Type } from "@sinclair/typebox"; import { unionOptions } from "../../codec/index.js"; +import { JsonCompatibleReadOnlySchema } from "../../util/index.js"; import { - type Brand, - brandedStringType, - JsonCompatibleReadOnlySchema, -} from "../../util/index.js"; - -export const version = 2 as const; - -/** - * Key (aka Name or Label) for a field which is scoped to a specific TreeNodeStoredSchema. - * - * Stable identifier, used when persisting data. - */ -export type FieldKey = Brand; - -/** - * TypeBox Schema for encoding {@link FieldKey} in persisted data. - */ -export const FieldKeySchema = brandedStringType(); - -/** - * Identifier for a TreeNode schema. - * Also known as "Definition" - * - * Stable identifier, used when persisting data. - */ -export type TreeNodeSchemaIdentifier = Brand< - TName, - "tree.TreeNodeSchemaIdentifier" ->; - -/** - * Identifier for a FieldKind. - * Refers to an exact stable policy (ex: specific version of a policy), - * for how to handle (ex: edit and merge edits to) fields marked with this kind. - * Persisted in documents as part of stored schema. - */ -export type FieldKindIdentifier = Brand; -export const FieldKindIdentifierSchema = brandedStringType(); - -/** - * TypeBox Schema for encoding {@link TreeNodeSchemaIdentifiers} in persisted data. - */ -export const TreeNodeSchemaIdentifierSchema = brandedStringType(); + FieldKindIdentifierSchema, + TreeNodeSchemaIdentifierSchema, + TreeNodeSchemaDataFormat as TreeNodeSchemaDataFormatV1, +} from "./formatV1.js"; export const PersistedMetadataFormat = Type.Optional(JsonCompatibleReadOnlySchema); @@ -63,17 +25,6 @@ const noAdditionalProps: ObjectOptions = { additionalProperties: false }; export const FieldSchemaFormat = Type.Composite([FieldSchemaFormatBase], noAdditionalProps); -/** - * Persisted version of {@link ValueSchema}. - */ -export enum PersistedValueSchema { - Number, - String, - Boolean, - FluidHandle, - Null, -} - /** * Discriminated union content of tree node schema. * @@ -81,20 +32,9 @@ export enum PersistedValueSchema { */ export const TreeNodeSchemaDataFormat = Type.Object( { + ...TreeNodeSchemaDataFormatV1.properties, /** - * Object node union member. - */ - object: Type.Optional(Type.Record(Type.String(), FieldSchemaFormat)), - /** - * Map node union member. - */ - map: Type.Optional(FieldSchemaFormat), - /** - * Leaf node union member. - */ - leaf: Type.Optional(Type.Enum(PersistedValueSchema)), - /** - * Persisted metadata for this node. + * Persisted metadata for the schema. */ metadata: PersistedMetadataFormat, }, diff --git a/packages/dds/tree/src/core/schema-stored/schema.ts b/packages/dds/tree/src/core/schema-stored/schema.ts index 9f4ceeff09b5..9650b466c30e 100644 --- a/packages/dds/tree/src/core/schema-stored/schema.ts +++ b/packages/dds/tree/src/core/schema-stored/schema.ts @@ -380,11 +380,12 @@ export function encodeFieldSchemaV2(schema: TreeFieldStoredSchema): FieldSchemaF }; } -export function decodeFieldSchema(schema: FieldSchemaFormat): TreeFieldStoredSchema { +export function decodeFieldSchema(schema: FieldSchemaFormatV2): TreeFieldStoredSchema { const out: TreeFieldStoredSchema = { // TODO: maybe provide actual FieldKind objects here, error on unrecognized kinds. kind: schema.kind, types: new Set(schema.types), + persistedMetadata: schema.persistedMetadata, }; return out; } diff --git a/packages/dds/tree/src/feature-libraries/modular-schema/fieldKindWithEditor.ts b/packages/dds/tree/src/feature-libraries/modular-schema/fieldKindWithEditor.ts index 6a18db241170..a59e7b2f83cb 100644 --- a/packages/dds/tree/src/feature-libraries/modular-schema/fieldKindWithEditor.ts +++ b/packages/dds/tree/src/feature-libraries/modular-schema/fieldKindWithEditor.ts @@ -79,6 +79,8 @@ export class FieldKindWithEditor< isNeverField(policy, originalData, { kind: this.identifier, types: originalTypes, + // Metadata is not used for this check. + persistedMetadata: undefined, }) ) { return true; diff --git a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts index 5cdc66204dbe..5a3e1b582ab5 100644 --- a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts +++ b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts @@ -20,8 +20,8 @@ import { decodeFieldSchema, encodeFieldSchemaV1, encodeFieldSchemaV2, - schemaFormatV1, - schemaFormatV2, + type schemaFormatV1, + type schemaFormatV2, storedSchemaDecodeDispatcher, } from "../../core/index.js"; import { brand, type JsonCompatible } from "../../util/index.js"; @@ -111,7 +111,7 @@ function encodeRepoV1(repo: TreeStoredSchema): FormatV1 { }); } return { - version: schemaFormatV1.version, + version: SchemaCodecVersion.v1, nodes: nodeSchema, root: rootFieldSchema, }; @@ -131,7 +131,7 @@ function encodeRepoV2(repo: TreeStoredSchema): FormatV2 { }); } return { - version: schemaFormatV2.version, + version: SchemaCodecVersion.v2, nodes: nodeSchema, root: rootFieldSchema, }; @@ -154,7 +154,7 @@ function decode(f: FormatV1 | FormatV2): TreeStoredSchema { * @returns The codec. */ function makeSchemaCodecV1(options: ICodecOptions): IJsonCodec { - return makeVersionedValidatedCodec(options, new Set([schemaFormatV1.version]), FormatV1, { + return makeVersionedValidatedCodec(options, new Set([SchemaCodecVersion.v1]), FormatV1, { encode: (data: TreeStoredSchema) => encodeRepoV1(data), decode: (data: FormatV1) => decode(data), }); @@ -166,7 +166,7 @@ function makeSchemaCodecV1(options: ICodecOptions): IJsonCodec { - return makeVersionedValidatedCodec(options, new Set([schemaFormatV2.version]), FormatV2, { + return makeVersionedValidatedCodec(options, new Set([SchemaCodecVersion.v2]), FormatV2, { encode: (data: TreeStoredSchema) => encodeRepoV2(data), decode: (data: FormatV2) => decode(data), }); diff --git a/packages/dds/tree/src/feature-libraries/schema-index/formatV1.ts b/packages/dds/tree/src/feature-libraries/schema-index/formatV1.ts index a0671dae2825..31456dba4551 100644 --- a/packages/dds/tree/src/feature-libraries/schema-index/formatV1.ts +++ b/packages/dds/tree/src/feature-libraries/schema-index/formatV1.ts @@ -5,7 +5,7 @@ import { type ObjectOptions, type Static, Type } from "@sinclair/typebox"; -import { schemaFormatV1 } from "../../core/index.js"; +import { SchemaCodecVersion, schemaFormatV1 } from "../../core/index.js"; const noAdditionalProps: ObjectOptions = { additionalProperties: false }; @@ -21,7 +21,7 @@ const noAdditionalProps: ObjectOptions = { additionalProperties: false }; */ export const Format = Type.Object( { - version: Type.Literal(schemaFormatV1.version), + version: Type.Literal(SchemaCodecVersion.v1), nodes: Type.Record(Type.String(), schemaFormatV1.TreeNodeSchemaDataFormat), root: schemaFormatV1.FieldSchemaFormat, }, diff --git a/packages/dds/tree/src/feature-libraries/schema-index/formatV2.ts b/packages/dds/tree/src/feature-libraries/schema-index/formatV2.ts index 525c4657977b..eefebb61bcec 100644 --- a/packages/dds/tree/src/feature-libraries/schema-index/formatV2.ts +++ b/packages/dds/tree/src/feature-libraries/schema-index/formatV2.ts @@ -5,7 +5,7 @@ import { type ObjectOptions, type Static, Type } from "@sinclair/typebox"; -import { schemaFormatV2 } from "../../core/index.js"; +import { SchemaCodecVersion, schemaFormatV2 } from "../../core/index.js"; const noAdditionalProps: ObjectOptions = { additionalProperties: false }; @@ -21,7 +21,7 @@ const noAdditionalProps: ObjectOptions = { additionalProperties: false }; */ export const Format = Type.Object( { - version: Type.Literal(schemaFormatV2.version), + version: Type.Literal(SchemaCodecVersion.v2), nodes: Type.Record(Type.String(), schemaFormatV2.TreeNodeSchemaDataFormat), root: schemaFormatV2.FieldSchemaFormat, persistedMetadata: schemaFormatV2.PersistedMetadataFormat, diff --git a/packages/dds/tree/src/shared-tree/schematizeTree.ts b/packages/dds/tree/src/shared-tree/schematizeTree.ts index 6e8f3d736947..dfc37c991ed4 100644 --- a/packages/dds/tree/src/shared-tree/schematizeTree.ts +++ b/packages/dds/tree/src/shared-tree/schematizeTree.ts @@ -67,6 +67,7 @@ export function initializeContent( rootFieldSchema: { kind: FieldKinds.optional.identifier, types: rootSchema.types, + persistedMetadata: rootSchema.persistedMetadata, }, }; } diff --git a/packages/dds/tree/src/simple-tree/api/schemaCompatibilityTester.ts b/packages/dds/tree/src/simple-tree/api/schemaCompatibilityTester.ts index 5db11af0aaae..06b5f9987db8 100644 --- a/packages/dds/tree/src/simple-tree/api/schemaCompatibilityTester.ts +++ b/packages/dds/tree/src/simple-tree/api/schemaCompatibilityTester.ts @@ -273,7 +273,7 @@ export class SchemaCompatibilityTester { } } - return { kind: original.kind, types }; + return { kind: original.kind, types, persistedMetadata: undefined }; } return original; } diff --git a/packages/dds/tree/src/simple-tree/toStoredSchema.ts b/packages/dds/tree/src/simple-tree/toStoredSchema.ts index ca972cc1c0a2..1e7319fc4f87 100644 --- a/packages/dds/tree/src/simple-tree/toStoredSchema.ts +++ b/packages/dds/tree/src/simple-tree/toStoredSchema.ts @@ -89,7 +89,7 @@ export function convertField(schema: SimpleFieldSchema): TreeFieldStoredSchema { const kind: FieldKindIdentifier = convertFieldKind.get(schema.kind)?.identifier ?? fail(0xae3 /* Invalid field kind */); const types: TreeTypeSet = schema.allowedTypesIdentifiers as TreeTypeSet; - return { kind, types }; + return { kind, types, persistedMetadata: undefined }; } const convertFieldKind = new Map([ @@ -110,13 +110,18 @@ export function getStoredSchema(schema: SimpleNodeSchema): TreeNodeStoredSchema } case NodeKind.Map: { const types = schema.allowedTypesIdentifiers as TreeTypeSet; - return new MapNodeStoredSchema({ kind: FieldKinds.optional.identifier, types }); + return new MapNodeStoredSchema({ + kind: FieldKinds.optional.identifier, + types, + persistedMetadata: undefined, + }); } case NodeKind.Array: { const types = schema.allowedTypesIdentifiers as TreeTypeSet; const field = { kind: FieldKinds.sequence.identifier, types, + persistedMetadata: undefined, }; const fields = new Map([[EmptyKey, field]]); return new ObjectNodeStoredSchema(fields); diff --git a/packages/dds/tree/src/test/feature-libraries/chunked-forest/codec/schemaBasedEncode.spec.ts b/packages/dds/tree/src/test/feature-libraries/chunked-forest/codec/schemaBasedEncode.spec.ts index f13ee5c4d143..bbf959059e08 100644 --- a/packages/dds/tree/src/test/feature-libraries/chunked-forest/codec/schemaBasedEncode.spec.ts +++ b/packages/dds/tree/src/test/feature-libraries/chunked-forest/codec/schemaBasedEncode.spec.ts @@ -152,7 +152,11 @@ describe("schemaBasedEncoding", () => { return onlyTypeShape; }, }, - { kind: FieldKinds.sequence.identifier, types: new Set([brand(Minimal.identifier)]) }, + { + kind: FieldKinds.sequence.identifier, + types: new Set([brand(Minimal.identifier)]), + persistedMetadata: undefined, + }, cache, { nodeSchema: new Map() }, ); @@ -182,6 +186,7 @@ describe("schemaBasedEncoding", () => { const storedSchema: TreeFieldStoredSchema = { kind: FieldKinds.identifier.identifier, types: new Set([brand(stringSchema.identifier)]), + persistedMetadata: undefined, }; const shape = fieldShaper( diff --git a/packages/dds/tree/src/test/feature-libraries/default-schema/schemaChecker.spec.ts b/packages/dds/tree/src/test/feature-libraries/default-schema/schemaChecker.spec.ts index 643350190f1c..51e8e8808f11 100644 --- a/packages/dds/tree/src/test/feature-libraries/default-schema/schemaChecker.spec.ts +++ b/packages/dds/tree/src/test/feature-libraries/default-schema/schemaChecker.spec.ts @@ -71,6 +71,7 @@ function getFieldSchema( return { kind: kind.identifier, types: new Set(allowedTypes), + persistedMetadata: undefined, }; } diff --git a/packages/dds/tree/src/test/feature-libraries/flex-tree/lazyField.spec.ts b/packages/dds/tree/src/test/feature-libraries/flex-tree/lazyField.spec.ts index aead36e84dbb..2f60c7b1a08b 100644 --- a/packages/dds/tree/src/test/feature-libraries/flex-tree/lazyField.spec.ts +++ b/packages/dds/tree/src/test/feature-libraries/flex-tree/lazyField.spec.ts @@ -444,6 +444,7 @@ describe("LazyField", () => { const rootSchema: TreeFieldStoredSchema = { kind: FieldKinds.sequence.identifier, types: new Set([brand(numberSchema.identifier)]), + persistedMetadata: undefined, }; const schema: TreeStoredSchema = { rootFieldSchema: rootSchema, diff --git a/packages/dds/tree/src/test/feature-libraries/modular-schema/comparison.spec.ts b/packages/dds/tree/src/test/feature-libraries/modular-schema/comparison.spec.ts index fc81bd37e139..9b19dbd19957 100644 --- a/packages/dds/tree/src/test/feature-libraries/modular-schema/comparison.spec.ts +++ b/packages/dds/tree/src/test/feature-libraries/modular-schema/comparison.spec.ts @@ -46,6 +46,7 @@ export function fieldSchema( return { kind: kind.identifier, types: new Set(types), + persistedMetadata: undefined, }; } diff --git a/packages/dds/tree/src/test/feature-libraries/modular-schema/isNeverTree.spec.ts b/packages/dds/tree/src/test/feature-libraries/modular-schema/isNeverTree.spec.ts index 86d606bf41ed..36f9de499674 100644 --- a/packages/dds/tree/src/test/feature-libraries/modular-schema/isNeverTree.spec.ts +++ b/packages/dds/tree/src/test/feature-libraries/modular-schema/isNeverTree.spec.ts @@ -42,6 +42,7 @@ function fieldSchema( return { kind: kind.identifier, types: new Set(types), + persistedMetadata: undefined, }; } diff --git a/packages/dds/tree/src/test/sequenceRootUtils.ts b/packages/dds/tree/src/test/sequenceRootUtils.ts index 69642341e9cd..f44b18b1ce5d 100644 --- a/packages/dds/tree/src/test/sequenceRootUtils.ts +++ b/packages/dds/tree/src/test/sequenceRootUtils.ts @@ -31,6 +31,7 @@ export const jsonSequenceRootSchema: TreeStoredSchema = { brand(s.identifier), ), ), + persistedMetadata: undefined, }, }; diff --git a/packages/dds/tree/src/test/shared-tree/sharedTreeChangeFamily.spec.ts b/packages/dds/tree/src/test/shared-tree/sharedTreeChangeFamily.spec.ts index 434c767c64bf..7acdb2c34b4c 100644 --- a/packages/dds/tree/src/test/shared-tree/sharedTreeChangeFamily.spec.ts +++ b/packages/dds/tree/src/test/shared-tree/sharedTreeChangeFamily.spec.ts @@ -75,6 +75,7 @@ const emptySchema: TreeStoredSchema = { rootFieldSchema: { kind: forbidden.identifier, types: new Set(), + persistedMetadata: undefined, }, }; const stSchemaChange: SharedTreeChange = { diff --git a/packages/dds/tree/src/test/simple-tree/toMapTree.spec.ts b/packages/dds/tree/src/test/simple-tree/toMapTree.spec.ts index ac2f2dc6ef53..32902007f145 100644 --- a/packages/dds/tree/src/test/simple-tree/toMapTree.spec.ts +++ b/packages/dds/tree/src/test/simple-tree/toMapTree.spec.ts @@ -73,6 +73,7 @@ function getFieldSchema( return { kind: kind.identifier, types: new Set(allowedTypes), + persistedMetadata: undefined, }; } diff --git a/packages/dds/tree/src/test/snapshots/files/SchemaIndexFormat - schema v2.json b/packages/dds/tree/src/test/snapshots/files/SchemaIndexFormat - schema v2.json index 160d0d9a2d6f..5dec6393d3d3 100644 --- a/packages/dds/tree/src/test/snapshots/files/SchemaIndexFormat - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/files/SchemaIndexFormat - schema v2.json @@ -30,8 +30,7 @@ "items": { "type": "string" } - }, - "persistedMetadata": {} + } }, "required": [ "kind", @@ -52,8 +51,7 @@ "items": { "type": "string" } - }, - "persistedMetadata": {} + } }, "required": [ "kind", @@ -84,7 +82,7 @@ } ] }, - "persistedMetadata": {} + "metadata": {} } } } diff --git a/packages/dds/tree/src/test/testTrees.ts b/packages/dds/tree/src/test/testTrees.ts index 11d137488192..3a714f8dd01d 100644 --- a/packages/dds/tree/src/test/testTrees.ts +++ b/packages/dds/tree/src/test/testTrees.ts @@ -180,6 +180,7 @@ export const allTheFields = new ObjectNodeStoredSchema( { kind: FieldKinds.optional.identifier, types: numberSet, + persistedMetadata: undefined, }, ], [ @@ -187,6 +188,7 @@ export const allTheFields = new ObjectNodeStoredSchema( { kind: FieldKinds.required.identifier, types: numberSet, + persistedMetadata: undefined, }, ], [ @@ -194,6 +196,7 @@ export const allTheFields = new ObjectNodeStoredSchema( { kind: FieldKinds.sequence.identifier, types: numberSet, + persistedMetadata: undefined, }, ], ]), @@ -270,7 +273,11 @@ export const testTrees: readonly TestTree[] = [ "numericSequence", { ...toStoredSchema(factory.number), - rootFieldSchema: { kind: FieldKinds.sequence.identifier, types: numberSet }, + rootFieldSchema: { + kind: FieldKinds.sequence.identifier, + types: numberSet, + persistedMetadata: undefined, + }, }, jsonableTreesFromFieldCursor(fieldJsonCursor([1, 2, 3])), ), @@ -302,6 +309,7 @@ export const testTrees: readonly TestTree[] = [ rootFieldSchema: { kind: FieldKinds.required.identifier, types: new Set([allTheFieldsName]), + persistedMetadata: undefined, }, }, [ @@ -318,6 +326,7 @@ export const testTrees: readonly TestTree[] = [ rootFieldSchema: { kind: FieldKinds.required.identifier, types: new Set([allTheFieldsName]), + persistedMetadata: undefined, }, }, [ From 3275ee0f5a8df9fe44792788e6ca247c6b102049 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Thu, 15 May 2025 14:51:19 -0700 Subject: [PATCH 10/45] Minor: reverted changes to FluidClientVersion utils. --- packages/dds/tree/src/codec/codec.ts | 4 +++- packages/dds/tree/src/feature-libraries/schema-index/codec.ts | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/dds/tree/src/codec/codec.ts b/packages/dds/tree/src/codec/codec.ts index 1d3574576be3..55cb2bdae5e3 100644 --- a/packages/dds/tree/src/codec/codec.ts +++ b/packages/dds/tree/src/codec/codec.ts @@ -352,9 +352,11 @@ export enum FluidClientVersion { v2_2 = "v2_2", /** Fluid Framework Client 2.3 and newer. */ v2_3 = "v2_3", + /** Fluid Framework Client 2.4 and newer. */ + v2_4 = "v2_4", } /** * The version of this code. */ -export const currentVersion: FluidClientVersion = FluidClientVersion.v2_3; +export const currentVersion: FluidClientVersion = FluidClientVersion.v2_4; diff --git a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts index 5a3e1b582ab5..2a5fc035b665 100644 --- a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts +++ b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts @@ -44,6 +44,8 @@ export function clientVersionToSchemaVersion( case FluidClientVersion.v2_2: case FluidClientVersion.v2_3: return SchemaCodecVersion.v1; + case FluidClientVersion.v2_4: + return SchemaCodecVersion.v2; default: unreachableCase(clientVersion); } From 7c48b9d6d08899a4342a72cdab044b6d317b485c Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Thu, 15 May 2025 15:44:13 -0700 Subject: [PATCH 11/45] Switched back to using min client version for importCompressed. --- packages/dds/tree/api-report/tree.alpha.api.md | 5 +++-- packages/dds/tree/src/shared-tree/treeAlpha.ts | 11 ++++++++--- .../dds/tree/src/simple-tree/api/storedSchema.ts | 12 ++++++++---- .../test/feature-libraries/schema-index/codecUtil.ts | 2 ++ .../src/test/simple-tree/api/storedSchema.spec.ts | 6 +++--- packages/dds/tree/src/test/utils.ts | 4 +++- 6 files changed, 27 insertions(+), 13 deletions(-) diff --git a/packages/dds/tree/api-report/tree.alpha.api.md b/packages/dds/tree/api-report/tree.alpha.api.md index 02a88bd22190..da1756afe87d 100644 --- a/packages/dds/tree/api-report/tree.alpha.api.md +++ b/packages/dds/tree/api-report/tree.alpha.api.md @@ -152,7 +152,7 @@ export function evaluateLazySchema(value: LazyItem) type ExtractItemType = Item extends () => infer Result ? Result : Item; // @alpha -export function extractPersistedSchema(schema: SimpleTreeSchema, schemaWriteVersion: number): JsonCompatible; +export function extractPersistedSchema(schema: SimpleTreeSchema, oldestCompatibleClient: FluidClientVersion): JsonCompatible; // @alpha @system export type FactoryContent = IFluidHandle | string | number | boolean | null | Iterable | readonly InsertableContent[] | FactoryContentObject; @@ -237,7 +237,8 @@ export enum FluidClientVersion { v2_0 = "v2_0", v2_1 = "v2_1", v2_2 = "v2_2", - v2_3 = "v2_3" + v2_3 = "v2_3", + v2_4 = "v2_4" } // @alpha diff --git a/packages/dds/tree/src/shared-tree/treeAlpha.ts b/packages/dds/tree/src/shared-tree/treeAlpha.ts index e59460b9b0db..1a623fd1538e 100644 --- a/packages/dds/tree/src/shared-tree/treeAlpha.ts +++ b/packages/dds/tree/src/shared-tree/treeAlpha.ts @@ -38,8 +38,13 @@ import { TreeViewConfigurationAlpha, } from "../simple-tree/index.js"; import type { JsonCompatible } from "../util/index.js"; -import { noopValidator, type FluidClientVersion, type ICodecOptions } from "../codec/index.js"; -import { SchemaCodecVersion, type ITreeCursorSynchronous } from "../core/index.js"; +import { + currentVersion, + noopValidator, + type FluidClientVersion, + type ICodecOptions, +} from "../codec/index.js"; +import type { ITreeCursorSynchronous } from "../core/index.js"; import { cursorForMapTreeField, defaultSchemaPolicy, @@ -316,7 +321,7 @@ export const TreeAlpha: TreeAlpha = { const config = new TreeViewConfigurationAlpha({ schema }); const content: ViewContent = { // TODO - schema: extractPersistedSchema(config, SchemaCodecVersion.v2), + schema: extractPersistedSchema(config, currentVersion), tree: compressedData, idCompressor: options.idCompressor ?? createIdCompressor(), }; diff --git a/packages/dds/tree/src/simple-tree/api/storedSchema.ts b/packages/dds/tree/src/simple-tree/api/storedSchema.ts index 32d52efe5d08..f0127b3516f9 100644 --- a/packages/dds/tree/src/simple-tree/api/storedSchema.ts +++ b/packages/dds/tree/src/simple-tree/api/storedSchema.ts @@ -3,15 +3,18 @@ * Licensed under the MIT License. */ -import type { ICodecOptions } from "../../codec/index.js"; +import type { FluidClientVersion, ICodecOptions } from "../../codec/index.js"; import { SchemaCodecVersion, type TreeStoredSchema } from "../../core/index.js"; import { defaultSchemaPolicy, encodeTreeSchema, makeSchemaCodec, } from "../../feature-libraries/index.js"; -// eslint-disable-next-line import/no-internal-modules -import type { FormatV1 } from "../../feature-libraries/schema-index/index.js"; +import { + clientVersionToSchemaVersion, + type FormatV1, + // eslint-disable-next-line import/no-internal-modules +} from "../../feature-libraries/schema-index/index.js"; import type { JsonCompatible } from "../../util/index.js"; import { normalizeFieldSchema, type ImplicitFieldSchema } from "../schemaTypes.js"; import { simpleToStoredSchema } from "../toStoredSchema.js"; @@ -51,9 +54,10 @@ import type { SimpleTreeSchema } from "../simpleSchema.js"; */ export function extractPersistedSchema( schema: SimpleTreeSchema, - schemaWriteVersion: number, + oldestCompatibleClient: FluidClientVersion, ): JsonCompatible { const stored = simpleToStoredSchema(schema); + const schemaWriteVersion = clientVersionToSchemaVersion(oldestCompatibleClient); return encodeTreeSchema(stored, schemaWriteVersion); } diff --git a/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts b/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts index 02530f351c2c..2037dba8bb1a 100644 --- a/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts +++ b/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts @@ -30,6 +30,8 @@ export function schemaFormatToClientVersion( switch (schemaFormat) { case SchemaCodecVersion.v1: return FluidClientVersion.v2_0; + case SchemaCodecVersion.v2: + return FluidClientVersion.v2_4; default: assert(false, `Unsupported schema format: ${schemaFormat}`); } diff --git a/packages/dds/tree/src/test/simple-tree/api/storedSchema.spec.ts b/packages/dds/tree/src/test/simple-tree/api/storedSchema.spec.ts index 7d2a55f43a93..8f03bb83da16 100644 --- a/packages/dds/tree/src/test/simple-tree/api/storedSchema.spec.ts +++ b/packages/dds/tree/src/test/simple-tree/api/storedSchema.spec.ts @@ -14,7 +14,7 @@ import { testSimpleTrees } from "../../testTrees.js"; import { takeJsonSnapshot, useSnapshotDirectory } from "../../snapshots/index.js"; import { typeboxValidator } from "../../../external-utilities/index.js"; import { TreeViewConfigurationAlpha } from "../../../simple-tree/index.js"; -import { SchemaCodecVersion } from "../../../core/index.js"; +import { FluidClientVersion } from "../../../codec/index.js"; describe("simple-tree storedSchema", () => { describe("test-schema", () => { @@ -24,7 +24,7 @@ describe("simple-tree storedSchema", () => { it(`${test.name} - schema v1`, () => { const persisted = extractPersistedSchema( new TreeViewConfigurationAlpha({ schema: test.schema }), - SchemaCodecVersion.v1, + FluidClientVersion.v2_0, ); takeJsonSnapshot(persisted); @@ -35,7 +35,7 @@ describe("simple-tree storedSchema", () => { it(`comparePersistedSchema to self ${test.name} - schema v1`, () => { const persistedA = extractPersistedSchema( new TreeViewConfigurationAlpha({ schema: test.schema }), - SchemaCodecVersion.v1, + FluidClientVersion.v2_0, ); const status = comparePersistedSchema( diff --git a/packages/dds/tree/src/test/utils.ts b/packages/dds/tree/src/test/utils.ts index 3dfad4c1f3fa..f492aecb3d56 100644 --- a/packages/dds/tree/src/test/utils.ts +++ b/packages/dds/tree/src/test/utils.ts @@ -176,6 +176,8 @@ import type { ISharedObjectKind, SharedObjectKind, } from "@fluidframework/shared-object-base/internal"; +// eslint-disable-next-line import/no-internal-modules +import { ajvValidator } from "./codec/ajvValidator.js"; // Testing utilities @@ -981,7 +983,7 @@ export function makeEncodingTestSuite( // pattern. const jsonCodec = codec.json.encodedSchema !== undefined - ? withSchemaValidation(codec.json.encodedSchema, codec.json, typeboxValidator) + ? withSchemaValidation(codec.json.encodedSchema, codec.json, ajvValidator) : codec.json; describe("can json roundtrip", () => { for (const includeStringification of [false, true]) { From a2f41952d6ab1477848ad7b2ece3a2ca529a2ac1 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Thu, 15 May 2025 15:48:42 -0700 Subject: [PATCH 12/45] Minor: reverted accidental change. --- packages/dds/tree/src/test/utils.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/dds/tree/src/test/utils.ts b/packages/dds/tree/src/test/utils.ts index f492aecb3d56..3dfad4c1f3fa 100644 --- a/packages/dds/tree/src/test/utils.ts +++ b/packages/dds/tree/src/test/utils.ts @@ -176,8 +176,6 @@ import type { ISharedObjectKind, SharedObjectKind, } from "@fluidframework/shared-object-base/internal"; -// eslint-disable-next-line import/no-internal-modules -import { ajvValidator } from "./codec/ajvValidator.js"; // Testing utilities @@ -983,7 +981,7 @@ export function makeEncodingTestSuite( // pattern. const jsonCodec = codec.json.encodedSchema !== undefined - ? withSchemaValidation(codec.json.encodedSchema, codec.json, ajvValidator) + ? withSchemaValidation(codec.json.encodedSchema, codec.json, typeboxValidator) : codec.json; describe("can json roundtrip", () => { for (const includeStringification of [false, true]) { From 4786dcc6fccfc1ecb919cd87a8bc37ee989a4793 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Thu, 15 May 2025 15:53:30 -0700 Subject: [PATCH 13/45] Apply suggestions from code review Co-authored-by: Alex Villarreal <716334+alexvy86@users.noreply.github.com> --- packages/dds/tree/src/core/schema-stored/schema.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/dds/tree/src/core/schema-stored/schema.ts b/packages/dds/tree/src/core/schema-stored/schema.ts index 9650b466c30e..fe3ea8f01c73 100644 --- a/packages/dds/tree/src/core/schema-stored/schema.ts +++ b/packages/dds/tree/src/core/schema-stored/schema.ts @@ -220,7 +220,7 @@ export class ObjectNodeStoredSchema extends TreeNodeStoredSchema { // This makes comparing schema easier, and ensures chunk reuse for schema summaries isn't needlessly broken. for (const key of [...this.objectNodeFields.keys()].sort()) { const value = encodeFieldSchemaV1( - this.objectNodeFields.get(key) ?? fail(0xae6 /* missing field */), + this.objectNodeFields.get(key) ?? fail(0xae7 /* missing field */), ); Object.defineProperty(fieldsObject, key, { @@ -237,11 +237,11 @@ export class ObjectNodeStoredSchema extends TreeNodeStoredSchema { public override encodeV2(): TreeNodeSchemaDataFormatV2 { const fieldsObject: Record = Object.create(null); - // Sort fields to ensure output is identical for for equivalent schema (since field order is not considered significant). + // Sort fields to ensure output is identical for equivalent schema (since field order is not considered significant). // This makes comparing schema easier, and ensures chunk reuse for schema summaries isn't needlessly broken. for (const key of [...this.objectNodeFields.keys()].sort()) { const value = encodeFieldSchemaV1( - this.objectNodeFields.get(key) ?? fail(0xae6 /* missing field */), + this.objectNodeFields.get(key) ?? fail("missing field"), ); Object.defineProperty(fieldsObject, key, { From 6fa8836e11a81c0b45b98f1fc3c53c3ecdef848c Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Thu, 15 May 2025 16:19:18 -0700 Subject: [PATCH 14/45] Made persisted metadata schema field naming consistent. --- packages/dds/tree/src/core/schema-stored/formatV2.ts | 2 +- packages/dds/tree/src/core/schema-stored/schema.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/dds/tree/src/core/schema-stored/formatV2.ts b/packages/dds/tree/src/core/schema-stored/formatV2.ts index d7962c9c3ac3..c67d0e495287 100644 --- a/packages/dds/tree/src/core/schema-stored/formatV2.ts +++ b/packages/dds/tree/src/core/schema-stored/formatV2.ts @@ -18,7 +18,7 @@ export const PersistedMetadataFormat = Type.Optional(JsonCompatibleReadOnlySchem const FieldSchemaFormatBase = Type.Object({ kind: FieldKindIdentifierSchema, types: Type.Array(TreeNodeSchemaIdentifierSchema), - persistedMetadata: PersistedMetadataFormat, + metadata: PersistedMetadataFormat, }); const noAdditionalProps: ObjectOptions = { additionalProperties: false }; diff --git a/packages/dds/tree/src/core/schema-stored/schema.ts b/packages/dds/tree/src/core/schema-stored/schema.ts index fe3ea8f01c73..6ebdaab56094 100644 --- a/packages/dds/tree/src/core/schema-stored/schema.ts +++ b/packages/dds/tree/src/core/schema-stored/schema.ts @@ -376,7 +376,7 @@ export function encodeFieldSchemaV1(schema: TreeFieldStoredSchema): FieldSchemaF export function encodeFieldSchemaV2(schema: TreeFieldStoredSchema): FieldSchemaFormatV2 { return { ...encodeFieldSchemaV1(schema), - persistedMetadata: schema.persistedMetadata, + metadata: schema.persistedMetadata, }; } @@ -385,7 +385,7 @@ export function decodeFieldSchema(schema: FieldSchemaFormatV2): TreeFieldStoredS // TODO: maybe provide actual FieldKind objects here, error on unrecognized kinds. kind: schema.kind, types: new Set(schema.types), - persistedMetadata: schema.persistedMetadata, + persistedMetadata: schema.metadata, }; return out; } From e4064c7994ba18a2b8b5306a0ca00a05595a3887 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Fri, 16 May 2025 13:45:39 -0700 Subject: [PATCH 15/45] - Created a separate storedSchemaDecodeDispatcher for v2 schemas. - Made importCompressed always use a v1 schema when encoding. --- packages/dds/tree/src/core/index.ts | 3 ++- .../dds/tree/src/core/schema-stored/index.ts | 3 ++- .../dds/tree/src/core/schema-stored/schema.ts | 22 +++++++++++++++++-- .../feature-libraries/schema-index/codec.ts | 22 ++++++++++++++----- .../dds/tree/src/shared-tree/treeAlpha.ts | 11 +++------- 5 files changed, 44 insertions(+), 17 deletions(-) diff --git a/packages/dds/tree/src/core/index.ts b/packages/dds/tree/src/core/index.ts index b837a2ec4bd5..bd2d97e49007 100644 --- a/packages/dds/tree/src/core/index.ts +++ b/packages/dds/tree/src/core/index.ts @@ -145,7 +145,8 @@ export { decodeFieldSchema, encodeFieldSchemaV1, encodeFieldSchemaV2, - storedSchemaDecodeDispatcher, + storedSchemaDecodeDispatcherV1, + storedSchemaDecodeDispatcherV2, type SchemaAndPolicy, Multiplicity, type SchemaPolicy, diff --git a/packages/dds/tree/src/core/schema-stored/index.ts b/packages/dds/tree/src/core/schema-stored/index.ts index 3a6c789af0f1..63318a3fdb37 100644 --- a/packages/dds/tree/src/core/schema-stored/index.ts +++ b/packages/dds/tree/src/core/schema-stored/index.ts @@ -20,7 +20,8 @@ export { decodeFieldSchema, encodeFieldSchemaV1, encodeFieldSchemaV2, - storedSchemaDecodeDispatcher, + storedSchemaDecodeDispatcherV1, + storedSchemaDecodeDispatcherV2, type SchemaAndPolicy, type SchemaPolicy, SchemaCodecVersion, diff --git a/packages/dds/tree/src/core/schema-stored/schema.ts b/packages/dds/tree/src/core/schema-stored/schema.ts index 6ebdaab56094..5e72a1279ddc 100644 --- a/packages/dds/tree/src/core/schema-stored/schema.ts +++ b/packages/dds/tree/src/core/schema-stored/schema.ts @@ -240,7 +240,7 @@ export class ObjectNodeStoredSchema extends TreeNodeStoredSchema { // Sort fields to ensure output is identical for equivalent schema (since field order is not considered significant). // This makes comparing schema easier, and ensures chunk reuse for schema summaries isn't needlessly broken. for (const key of [...this.objectNodeFields.keys()].sort()) { - const value = encodeFieldSchemaV1( + const value = encodeFieldSchemaV2( this.objectNodeFields.get(key) ?? fail("missing field"), ); @@ -329,7 +329,7 @@ export class LeafNodeStoredSchema extends TreeNodeStoredSchema { } } -export const storedSchemaDecodeDispatcher: DiscriminatedUnionDispatcher< +export const storedSchemaDecodeDispatcherV1: DiscriminatedUnionDispatcher< TreeNodeSchemaDataFormatV1, [], TreeNodeStoredSchema @@ -347,6 +347,24 @@ export const storedSchemaDecodeDispatcher: DiscriminatedUnionDispatcher< map: (data: FieldSchemaFormat) => new MapNodeStoredSchema(decodeFieldSchema(data)), }); +export const storedSchemaDecodeDispatcherV2: DiscriminatedUnionDispatcher< + TreeNodeSchemaDataFormatV2, + [], + TreeNodeStoredSchema +> = new DiscriminatedUnionDispatcher({ + leaf: (data: PersistedValueSchema) => new LeafNodeStoredSchema(decodeValueSchema(data)), + object: ( + data: Record, + ): TreeNodeStoredSchema => { + const map = new Map(); + for (const [key, value] of Object.entries(data)) { + map.set(key, decodeFieldSchema(value)); + } + return new ObjectNodeStoredSchema(map); + }, + map: (data: FieldSchemaFormat) => new MapNodeStoredSchema(decodeFieldSchema(data)), +}); + const valueSchemaEncode = new Map([ [ValueSchema.Number, PersistedValueSchema.Number], [ValueSchema.String, PersistedValueSchema.String], diff --git a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts index 2a5fc035b665..2a0e7ae1dbbb 100644 --- a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts +++ b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts @@ -22,7 +22,8 @@ import { encodeFieldSchemaV2, type schemaFormatV1, type schemaFormatV2, - storedSchemaDecodeDispatcher, + storedSchemaDecodeDispatcherV1, + storedSchemaDecodeDispatcherV2, } from "../../core/index.js"; import { brand, type JsonCompatible } from "../../util/index.js"; @@ -139,10 +140,21 @@ function encodeRepoV2(repo: TreeStoredSchema): FormatV2 { }; } -function decode(f: FormatV1 | FormatV2): TreeStoredSchema { +function decodeV1(f: FormatV1): TreeStoredSchema { const nodeSchema: Map = new Map(); for (const [key, schema] of Object.entries(f.nodes)) { - nodeSchema.set(brand(key), storedSchemaDecodeDispatcher.dispatch(schema)); + nodeSchema.set(brand(key), storedSchemaDecodeDispatcherV1.dispatch(schema)); + } + return { + rootFieldSchema: decodeFieldSchema(f.root), + nodeSchema, + }; +} + +function decodeV2(f: FormatV2): TreeStoredSchema { + const nodeSchema: Map = new Map(); + for (const [key, schema] of Object.entries(f.nodes)) { + nodeSchema.set(brand(key), storedSchemaDecodeDispatcherV2.dispatch(schema)); } return { rootFieldSchema: decodeFieldSchema(f.root), @@ -158,7 +170,7 @@ function decode(f: FormatV1 | FormatV2): TreeStoredSchema { function makeSchemaCodecV1(options: ICodecOptions): IJsonCodec { return makeVersionedValidatedCodec(options, new Set([SchemaCodecVersion.v1]), FormatV1, { encode: (data: TreeStoredSchema) => encodeRepoV1(data), - decode: (data: FormatV1) => decode(data), + decode: (data: FormatV1) => decodeV1(data), }); } @@ -170,6 +182,6 @@ function makeSchemaCodecV1(options: ICodecOptions): IJsonCodec { return makeVersionedValidatedCodec(options, new Set([SchemaCodecVersion.v2]), FormatV2, { encode: (data: TreeStoredSchema) => encodeRepoV2(data), - decode: (data: FormatV2) => decode(data), + decode: (data: FormatV2) => decodeV2(data), }); } diff --git a/packages/dds/tree/src/shared-tree/treeAlpha.ts b/packages/dds/tree/src/shared-tree/treeAlpha.ts index 1a623fd1538e..d944b5464024 100644 --- a/packages/dds/tree/src/shared-tree/treeAlpha.ts +++ b/packages/dds/tree/src/shared-tree/treeAlpha.ts @@ -38,12 +38,7 @@ import { TreeViewConfigurationAlpha, } from "../simple-tree/index.js"; import type { JsonCompatible } from "../util/index.js"; -import { - currentVersion, - noopValidator, - type FluidClientVersion, - type ICodecOptions, -} from "../codec/index.js"; +import { FluidClientVersion, noopValidator, type ICodecOptions } from "../codec/index.js"; import type { ITreeCursorSynchronous } from "../core/index.js"; import { cursorForMapTreeField, @@ -320,8 +315,8 @@ export const TreeAlpha: TreeAlpha = { ): Unhydrated> { const config = new TreeViewConfigurationAlpha({ schema }); const content: ViewContent = { - // TODO - schema: extractPersistedSchema(config, currentVersion), + // Always use a v1 schema codec for consistency. + schema: extractPersistedSchema(config, FluidClientVersion.v2_0), tree: compressedData, idCompressor: options.idCompressor ?? createIdCompressor(), }; From ab1b88249094cb097530b9c9924e429389158cbc Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Mon, 19 May 2025 12:02:03 -0700 Subject: [PATCH 16/45] - Refactored the node kind dispatch. - Added better test data to one of the codec tests for the v2 schema format. --- packages/dds/tree/src/core/index.ts | 3 +- .../tree/src/core/schema-stored/formatV2.ts | 18 +++++--- .../dds/tree/src/core/schema-stored/index.ts | 3 +- .../dds/tree/src/core/schema-stored/schema.ts | 46 +++---------------- .../feature-libraries/schema-index/codec.ts | 7 ++- .../schema-index/formatV2.ts | 2 +- .../schema-index/codec.spec.ts | 2 +- 7 files changed, 26 insertions(+), 55 deletions(-) diff --git a/packages/dds/tree/src/core/index.ts b/packages/dds/tree/src/core/index.ts index bd2d97e49007..b837a2ec4bd5 100644 --- a/packages/dds/tree/src/core/index.ts +++ b/packages/dds/tree/src/core/index.ts @@ -145,8 +145,7 @@ export { decodeFieldSchema, encodeFieldSchemaV1, encodeFieldSchemaV2, - storedSchemaDecodeDispatcherV1, - storedSchemaDecodeDispatcherV2, + storedSchemaDecodeDispatcher, type SchemaAndPolicy, Multiplicity, type SchemaPolicy, diff --git a/packages/dds/tree/src/core/schema-stored/formatV2.ts b/packages/dds/tree/src/core/schema-stored/formatV2.ts index c67d0e495287..2549f547ea96 100644 --- a/packages/dds/tree/src/core/schema-stored/formatV2.ts +++ b/packages/dds/tree/src/core/schema-stored/formatV2.ts @@ -5,12 +5,11 @@ import { type ObjectOptions, type Static, Type } from "@sinclair/typebox"; -import { unionOptions } from "../../codec/index.js"; import { JsonCompatibleReadOnlySchema } from "../../util/index.js"; import { FieldKindIdentifierSchema, TreeNodeSchemaIdentifierSchema, - TreeNodeSchemaDataFormat as TreeNodeSchemaDataFormatV1, + TreeNodeSchemaDataFormat as TreeNodeSchemaUnionFormat, } from "./formatV1.js"; export const PersistedMetadataFormat = Type.Optional(JsonCompatibleReadOnlySchema); @@ -26,19 +25,24 @@ const noAdditionalProps: ObjectOptions = { additionalProperties: false }; export const FieldSchemaFormat = Type.Composite([FieldSchemaFormatBase], noAdditionalProps); /** - * Discriminated union content of tree node schema. + * Format for {@link TreeNodeStoredSchema}. * * See {@link DiscriminatedUnionDispatcher} for more information on this pattern. */ export const TreeNodeSchemaDataFormat = Type.Object( { - ...TreeNodeSchemaDataFormatV1.properties, /** - * Persisted metadata for the schema. + * Node kind specific data. + */ + kind: TreeNodeSchemaUnionFormat, + + // Data in common for all TreeNode schemas: + /** + * Leaf node union member. */ metadata: PersistedMetadataFormat, }, - unionOptions, + noAdditionalProps, ); export type TreeNodeSchemaDataFormat = Static; @@ -46,3 +50,5 @@ export type TreeNodeSchemaDataFormat = Static; export type FieldSchemaFormat = Static; export type PersistedMetadataFormat = Static; + +export { TreeNodeSchemaUnionFormat }; diff --git a/packages/dds/tree/src/core/schema-stored/index.ts b/packages/dds/tree/src/core/schema-stored/index.ts index 63318a3fdb37..3a6c789af0f1 100644 --- a/packages/dds/tree/src/core/schema-stored/index.ts +++ b/packages/dds/tree/src/core/schema-stored/index.ts @@ -20,8 +20,7 @@ export { decodeFieldSchema, encodeFieldSchemaV1, encodeFieldSchemaV2, - storedSchemaDecodeDispatcherV1, - storedSchemaDecodeDispatcherV2, + storedSchemaDecodeDispatcher, type SchemaAndPolicy, type SchemaPolicy, SchemaCodecVersion, diff --git a/packages/dds/tree/src/core/schema-stored/schema.ts b/packages/dds/tree/src/core/schema-stored/schema.ts index 5e72a1279ddc..e268e71b23e6 100644 --- a/packages/dds/tree/src/core/schema-stored/schema.ts +++ b/packages/dds/tree/src/core/schema-stored/schema.ts @@ -18,6 +18,7 @@ import type { FieldSchemaFormat as FieldSchemaFormatV2, PersistedMetadataFormat, TreeNodeSchemaDataFormat as TreeNodeSchemaDataFormatV2, + TreeNodeSchemaUnionFormat, } from "./formatV2.js"; import type { Multiplicity } from "./multiplicity.js"; @@ -236,23 +237,8 @@ export class ObjectNodeStoredSchema extends TreeNodeStoredSchema { } public override encodeV2(): TreeNodeSchemaDataFormatV2 { - const fieldsObject: Record = Object.create(null); - // Sort fields to ensure output is identical for equivalent schema (since field order is not considered significant). - // This makes comparing schema easier, and ensures chunk reuse for schema summaries isn't needlessly broken. - for (const key of [...this.objectNodeFields.keys()].sort()) { - const value = encodeFieldSchemaV2( - this.objectNodeFields.get(key) ?? fail("missing field"), - ); - - Object.defineProperty(fieldsObject, key, { - enumerable: true, - configurable: true, - writable: true, - value, - }); - } return { - object: fieldsObject, + kind: this.encodeV1(), }; } @@ -284,7 +270,7 @@ export class MapNodeStoredSchema extends TreeNodeStoredSchema { public override encodeV2(): TreeNodeSchemaDataFormatV2 { return { - map: encodeFieldSchemaV2(this.mapFields), + kind: this.encodeV1(), }; } @@ -318,9 +304,9 @@ export class LeafNodeStoredSchema extends TreeNodeStoredSchema { }; } - public override encodeV2(): TreeNodeSchemaDataFormatV1 { + public override encodeV2(): TreeNodeSchemaDataFormatV2 { return { - leaf: encodeValueSchema(this.leafValue), + kind: this.encodeV1(), }; } @@ -329,26 +315,8 @@ export class LeafNodeStoredSchema extends TreeNodeStoredSchema { } } -export const storedSchemaDecodeDispatcherV1: DiscriminatedUnionDispatcher< - TreeNodeSchemaDataFormatV1, - [], - TreeNodeStoredSchema -> = new DiscriminatedUnionDispatcher({ - leaf: (data: PersistedValueSchema) => new LeafNodeStoredSchema(decodeValueSchema(data)), - object: ( - data: Record, - ): TreeNodeStoredSchema => { - const map = new Map(); - for (const [key, value] of Object.entries(data)) { - map.set(key, decodeFieldSchema(value)); - } - return new ObjectNodeStoredSchema(map); - }, - map: (data: FieldSchemaFormat) => new MapNodeStoredSchema(decodeFieldSchema(data)), -}); - -export const storedSchemaDecodeDispatcherV2: DiscriminatedUnionDispatcher< - TreeNodeSchemaDataFormatV2, +export const storedSchemaDecodeDispatcher: DiscriminatedUnionDispatcher< + TreeNodeSchemaUnionFormat, [], TreeNodeStoredSchema > = new DiscriminatedUnionDispatcher({ diff --git a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts index 2a0e7ae1dbbb..8c16c2b6428b 100644 --- a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts +++ b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts @@ -22,8 +22,7 @@ import { encodeFieldSchemaV2, type schemaFormatV1, type schemaFormatV2, - storedSchemaDecodeDispatcherV1, - storedSchemaDecodeDispatcherV2, + storedSchemaDecodeDispatcher, } from "../../core/index.js"; import { brand, type JsonCompatible } from "../../util/index.js"; @@ -143,7 +142,7 @@ function encodeRepoV2(repo: TreeStoredSchema): FormatV2 { function decodeV1(f: FormatV1): TreeStoredSchema { const nodeSchema: Map = new Map(); for (const [key, schema] of Object.entries(f.nodes)) { - nodeSchema.set(brand(key), storedSchemaDecodeDispatcherV1.dispatch(schema)); + nodeSchema.set(brand(key), storedSchemaDecodeDispatcher.dispatch(schema)); } return { rootFieldSchema: decodeFieldSchema(f.root), @@ -154,7 +153,7 @@ function decodeV1(f: FormatV1): TreeStoredSchema { function decodeV2(f: FormatV2): TreeStoredSchema { const nodeSchema: Map = new Map(); for (const [key, schema] of Object.entries(f.nodes)) { - nodeSchema.set(brand(key), storedSchemaDecodeDispatcherV2.dispatch(schema)); + nodeSchema.set(brand(key), storedSchemaDecodeDispatcher.dispatch(schema.kind)); } return { rootFieldSchema: decodeFieldSchema(f.root), diff --git a/packages/dds/tree/src/feature-libraries/schema-index/formatV2.ts b/packages/dds/tree/src/feature-libraries/schema-index/formatV2.ts index eefebb61bcec..09a47a547130 100644 --- a/packages/dds/tree/src/feature-libraries/schema-index/formatV2.ts +++ b/packages/dds/tree/src/feature-libraries/schema-index/formatV2.ts @@ -24,7 +24,7 @@ export const Format = Type.Object( version: Type.Literal(SchemaCodecVersion.v2), nodes: Type.Record(Type.String(), schemaFormatV2.TreeNodeSchemaDataFormat), root: schemaFormatV2.FieldSchemaFormat, - persistedMetadata: schemaFormatV2.PersistedMetadataFormat, + metadata: schemaFormatV2.PersistedMetadataFormat, }, noAdditionalProps, ); diff --git a/packages/dds/tree/src/test/feature-libraries/schema-index/codec.spec.ts b/packages/dds/tree/src/test/feature-libraries/schema-index/codec.spec.ts index ab53541f6896..a185b311c797 100644 --- a/packages/dds/tree/src/test/feature-libraries/schema-index/codec.spec.ts +++ b/packages/dds/tree/src/test/feature-libraries/schema-index/codec.spec.ts @@ -78,7 +78,7 @@ describe("SchemaIndex", () => { version: 2 as const, nodes: {}, root: { kind: "x" as FieldKindIdentifier, types: [] }, - persistedMetadata: { "ff-system": { "eDiscovery-exclude": "true" } }, + metadata: { "ff-system": { "eDiscovery-exclude": "true" } }, } satisfies FormatV2, ]; for (const data of cases) { From 1027daac65b097b6dc8829618cf980bb75b0a16b Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Mon, 19 May 2025 14:13:59 -0700 Subject: [PATCH 17/45] Updated snapshots. --- .../simple encoded schema - schema v2.json | 52 ++++---- .../files/SchemaIndexFormat - schema v2.json | 111 ++++++++++-------- .../allTheFields-full - schema v2.json | 46 ++++---- .../allTheFields-minimal - schema v2.json | 46 ++++---- .../false boolean - schema v2.json | 4 +- .../schema-files/handle - schema v2.json | 4 +- .../hasAllMetadata - schema v2.json | 18 +-- .../hasAllMetadataRootField - schema v2.json | 18 +-- .../hasAmbiguousField - schema v2.json | 24 ++-- .../hasDescriptions - schema v2.json | 18 +-- .../hasMinimalValueField - schema v2.json | 18 +-- .../hasNumericValueField - schema v2.json | 18 +-- .../hasOptionalField-empty - schema v2.json | 18 +-- .../hasPolymorphicValueField - schema v2.json | 24 ++-- .../hasRenamedField - schema v2.json | 18 +-- .../identifier-field - schema v2.json | 4 +- .../schema-files/minimal - schema v2.json | 4 +- ...ode-with-identifier-field - schema v2.json | 18 +-- .../schema-files/null - schema v2.json | 4 +- .../schema-files/numeric - schema v2.json | 4 +- .../numericMap-empty - schema v2.json | 16 ++- .../numericMap-full - schema v2.json | 16 ++- .../numericSequence - schema v2.json | 4 +- .../recursiveType-deeper - schema v2.json | 14 ++- .../recursiveType-empty - schema v2.json | 14 ++- .../recursiveType-recursive - schema v2.json | 14 ++- .../true boolean - schema v2.json | 4 +- 27 files changed, 330 insertions(+), 223 deletions(-) diff --git a/packages/dds/tree/src/test/snapshots/encodeTreeSchema/simple encoded schema - schema v2.json b/packages/dds/tree/src/test/snapshots/encodeTreeSchema/simple encoded schema - schema v2.json index 3fe392cee3a9..f783f848ff29 100644 --- a/packages/dds/tree/src/test/snapshots/encodeTreeSchema/simple encoded schema - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/encodeTreeSchema/simple encoded schema - schema v2.json @@ -2,9 +2,26 @@ "version": 2, "nodes": { "com.fluidframework.json.array": { - "object": { - "": { - "kind": "Sequence", + "kind": { + "object": { + "": { + "kind": "Sequence", + "types": [ + "com.fluidframework.json.array", + "com.fluidframework.json.object", + "com.fluidframework.leaf.boolean", + "com.fluidframework.leaf.null", + "com.fluidframework.leaf.number", + "com.fluidframework.leaf.string" + ] + } + } + } + }, + "com.fluidframework.json.object": { + "kind": { + "map": { + "kind": "Optional", "types": [ "com.fluidframework.json.array", "com.fluidframework.json.object", @@ -16,30 +33,25 @@ } } }, - "com.fluidframework.json.object": { - "map": { - "kind": "Optional", - "types": [ - "com.fluidframework.json.array", - "com.fluidframework.json.object", - "com.fluidframework.leaf.boolean", - "com.fluidframework.leaf.null", - "com.fluidframework.leaf.number", - "com.fluidframework.leaf.string" - ] - } - }, "com.fluidframework.leaf.boolean": { - "leaf": 2 + "kind": { + "leaf": 2 + } }, "com.fluidframework.leaf.null": { - "leaf": 4 + "kind": { + "leaf": 4 + } }, "com.fluidframework.leaf.number": { - "leaf": 0 + "kind": { + "leaf": 0 + } }, "com.fluidframework.leaf.string": { - "leaf": 1 + "kind": { + "leaf": 1 + } } }, "root": { diff --git a/packages/dds/tree/src/test/snapshots/files/SchemaIndexFormat - schema v2.json b/packages/dds/tree/src/test/snapshots/files/SchemaIndexFormat - schema v2.json index 5dec6393d3d3..3444b62bd315 100644 --- a/packages/dds/tree/src/test/snapshots/files/SchemaIndexFormat - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/files/SchemaIndexFormat - schema v2.json @@ -11,14 +11,39 @@ "patternProperties": { "^(.*)$": { "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, "type": "object", "properties": { - "object": { + "kind": { + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, "type": "object", - "patternProperties": { - "^(.*)$": { + "properties": { + "object": { + "type": "object", + "patternProperties": { + "^(.*)$": { + "additionalProperties": false, + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "types": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "kind", + "types" + ] + } + } + }, + "map": { "additionalProperties": false, "type": "object", "properties": { @@ -36,54 +61,38 @@ "kind", "types" ] - } - } - }, - "map": { - "additionalProperties": false, - "type": "object", - "properties": { - "kind": { - "type": "string" }, - "types": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "kind", - "types" - ] - }, - "leaf": { - "anyOf": [ - { - "const": 0, - "type": "number" - }, - { - "const": 1, - "type": "number" - }, - { - "const": 2, - "type": "number" - }, - { - "const": 3, - "type": "number" - }, - { - "const": 4, - "type": "number" + "leaf": { + "anyOf": [ + { + "const": 0, + "type": "number" + }, + { + "const": 1, + "type": "number" + }, + { + "const": 2, + "type": "number" + }, + { + "const": 3, + "type": "number" + }, + { + "const": 4, + "type": "number" + } + ] } - ] + } }, "metadata": {} - } + }, + "required": [ + "kind" + ] } } }, @@ -100,14 +109,14 @@ "type": "string" } }, - "persistedMetadata": {} + "metadata": {} }, "required": [ "kind", "types" ] }, - "persistedMetadata": {} + "metadata": {} }, "required": [ "version", diff --git a/packages/dds/tree/src/test/snapshots/schema-files/allTheFields-full - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/allTheFields-full - schema v2.json index f8c9da4815f1..496671aadb98 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/allTheFields-full - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/allTheFields-full - schema v2.json @@ -2,32 +2,38 @@ "version": 2, "nodes": { "com.fluidframework.leaf.number": { - "leaf": 0 + "kind": { + "leaf": 0 + } }, "test.allTheFields": { - "object": { - "optional": { - "kind": "Optional", - "types": [ - "com.fluidframework.leaf.number" - ] - }, - "sequence": { - "kind": "Sequence", - "types": [ - "com.fluidframework.leaf.number" - ] - }, - "valueField": { - "kind": "Value", - "types": [ - "com.fluidframework.leaf.number" - ] + "kind": { + "object": { + "optional": { + "kind": "Optional", + "types": [ + "com.fluidframework.leaf.number" + ] + }, + "sequence": { + "kind": "Sequence", + "types": [ + "com.fluidframework.leaf.number" + ] + }, + "valueField": { + "kind": "Value", + "types": [ + "com.fluidframework.leaf.number" + ] + } } } }, "test.minimal": { - "object": {} + "kind": { + "object": {} + } } }, "root": { diff --git a/packages/dds/tree/src/test/snapshots/schema-files/allTheFields-minimal - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/allTheFields-minimal - schema v2.json index f8c9da4815f1..496671aadb98 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/allTheFields-minimal - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/allTheFields-minimal - schema v2.json @@ -2,32 +2,38 @@ "version": 2, "nodes": { "com.fluidframework.leaf.number": { - "leaf": 0 + "kind": { + "leaf": 0 + } }, "test.allTheFields": { - "object": { - "optional": { - "kind": "Optional", - "types": [ - "com.fluidframework.leaf.number" - ] - }, - "sequence": { - "kind": "Sequence", - "types": [ - "com.fluidframework.leaf.number" - ] - }, - "valueField": { - "kind": "Value", - "types": [ - "com.fluidframework.leaf.number" - ] + "kind": { + "object": { + "optional": { + "kind": "Optional", + "types": [ + "com.fluidframework.leaf.number" + ] + }, + "sequence": { + "kind": "Sequence", + "types": [ + "com.fluidframework.leaf.number" + ] + }, + "valueField": { + "kind": "Value", + "types": [ + "com.fluidframework.leaf.number" + ] + } } } }, "test.minimal": { - "object": {} + "kind": { + "object": {} + } } }, "root": { diff --git a/packages/dds/tree/src/test/snapshots/schema-files/false boolean - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/false boolean - schema v2.json index dbd7b1e621ba..4f478d762a80 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/false boolean - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/false boolean - schema v2.json @@ -2,7 +2,9 @@ "version": 2, "nodes": { "com.fluidframework.leaf.boolean": { - "leaf": 2 + "kind": { + "leaf": 2 + } } }, "root": { diff --git a/packages/dds/tree/src/test/snapshots/schema-files/handle - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/handle - schema v2.json index 1bc718ef657a..670a2e2eb00d 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/handle - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/handle - schema v2.json @@ -2,7 +2,9 @@ "version": 2, "nodes": { "com.fluidframework.leaf.handle": { - "leaf": 3 + "kind": { + "leaf": 3 + } } }, "root": { diff --git a/packages/dds/tree/src/test/snapshots/schema-files/hasAllMetadata - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/hasAllMetadata - schema v2.json index 65c731adecea..9a7cc02e9e5a 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/hasAllMetadata - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/hasAllMetadata - schema v2.json @@ -2,17 +2,21 @@ "version": 2, "nodes": { "test.hasDescriptions": { - "object": { - "stored-name": { - "kind": "Value", - "types": [ - "test.minimal" - ] + "kind": { + "object": { + "stored-name": { + "kind": "Value", + "types": [ + "test.minimal" + ] + } } } }, "test.minimal": { - "object": {} + "kind": { + "object": {} + } } }, "root": { diff --git a/packages/dds/tree/src/test/snapshots/schema-files/hasAllMetadataRootField - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/hasAllMetadataRootField - schema v2.json index a0f6eee377d2..2f26b1f20162 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/hasAllMetadataRootField - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/hasAllMetadataRootField - schema v2.json @@ -2,17 +2,21 @@ "version": 2, "nodes": { "test.hasDescriptions": { - "object": { - "stored-name": { - "kind": "Value", - "types": [ - "test.minimal" - ] + "kind": { + "object": { + "stored-name": { + "kind": "Value", + "types": [ + "test.minimal" + ] + } } } }, "test.minimal": { - "object": {} + "kind": { + "object": {} + } } }, "root": { diff --git a/packages/dds/tree/src/test/snapshots/schema-files/hasAmbiguousField - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/hasAmbiguousField - schema v2.json index cbafc48d311b..38a1bd182866 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/hasAmbiguousField - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/hasAmbiguousField - schema v2.json @@ -2,21 +2,27 @@ "version": 2, "nodes": { "test.hasAmbiguousField": { - "object": { - "field": { - "kind": "Value", - "types": [ - "test.minimal", - "test.minimal2" - ] + "kind": { + "object": { + "field": { + "kind": "Value", + "types": [ + "test.minimal", + "test.minimal2" + ] + } } } }, "test.minimal": { - "object": {} + "kind": { + "object": {} + } }, "test.minimal2": { - "object": {} + "kind": { + "object": {} + } } }, "root": { diff --git a/packages/dds/tree/src/test/snapshots/schema-files/hasDescriptions - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/hasDescriptions - schema v2.json index 154972d78582..1a673ef5226e 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/hasDescriptions - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/hasDescriptions - schema v2.json @@ -2,17 +2,21 @@ "version": 2, "nodes": { "test.hasDescriptions": { - "object": { - "field": { - "kind": "Value", - "types": [ - "test.minimal" - ] + "kind": { + "object": { + "field": { + "kind": "Value", + "types": [ + "test.minimal" + ] + } } } }, "test.minimal": { - "object": {} + "kind": { + "object": {} + } } }, "root": { diff --git a/packages/dds/tree/src/test/snapshots/schema-files/hasMinimalValueField - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/hasMinimalValueField - schema v2.json index e875de16ed7d..dbef03ea4d01 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/hasMinimalValueField - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/hasMinimalValueField - schema v2.json @@ -2,17 +2,21 @@ "version": 2, "nodes": { "test.hasMinimalValueField": { - "object": { - "field": { - "kind": "Value", - "types": [ - "test.minimal" - ] + "kind": { + "object": { + "field": { + "kind": "Value", + "types": [ + "test.minimal" + ] + } } } }, "test.minimal": { - "object": {} + "kind": { + "object": {} + } } }, "root": { diff --git a/packages/dds/tree/src/test/snapshots/schema-files/hasNumericValueField - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/hasNumericValueField - schema v2.json index 493bfc8face7..bab5225cba21 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/hasNumericValueField - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/hasNumericValueField - schema v2.json @@ -2,15 +2,19 @@ "version": 2, "nodes": { "com.fluidframework.leaf.number": { - "leaf": 0 + "kind": { + "leaf": 0 + } }, "test.hasNumericValueField": { - "object": { - "field": { - "kind": "Value", - "types": [ - "com.fluidframework.leaf.number" - ] + "kind": { + "object": { + "field": { + "kind": "Value", + "types": [ + "com.fluidframework.leaf.number" + ] + } } } } diff --git a/packages/dds/tree/src/test/snapshots/schema-files/hasOptionalField-empty - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/hasOptionalField-empty - schema v2.json index 89d72e6429d6..ab5793c516ed 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/hasOptionalField-empty - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/hasOptionalField-empty - schema v2.json @@ -2,15 +2,19 @@ "version": 2, "nodes": { "com.fluidframework.leaf.number": { - "leaf": 0 + "kind": { + "leaf": 0 + } }, "test.hasOptionalField": { - "object": { - "field": { - "kind": "Optional", - "types": [ - "com.fluidframework.leaf.number" - ] + "kind": { + "object": { + "field": { + "kind": "Optional", + "types": [ + "com.fluidframework.leaf.number" + ] + } } } } diff --git a/packages/dds/tree/src/test/snapshots/schema-files/hasPolymorphicValueField - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/hasPolymorphicValueField - schema v2.json index a6663b20aabb..4d3de67bd772 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/hasPolymorphicValueField - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/hasPolymorphicValueField - schema v2.json @@ -2,21 +2,27 @@ "version": 2, "nodes": { "com.fluidframework.leaf.number": { - "leaf": 0 + "kind": { + "leaf": 0 + } }, "test.hasPolymorphicValueField": { - "object": { - "field": { - "kind": "Value", - "types": [ - "com.fluidframework.leaf.number", - "test.minimal" - ] + "kind": { + "object": { + "field": { + "kind": "Value", + "types": [ + "com.fluidframework.leaf.number", + "test.minimal" + ] + } } } }, "test.minimal": { - "object": {} + "kind": { + "object": {} + } } }, "root": { diff --git a/packages/dds/tree/src/test/snapshots/schema-files/hasRenamedField - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/hasRenamedField - schema v2.json index cd5e07352a83..13165b281105 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/hasRenamedField - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/hasRenamedField - schema v2.json @@ -2,17 +2,21 @@ "version": 2, "nodes": { "test.hasRenamedField": { - "object": { - "stored-name": { - "kind": "Value", - "types": [ - "test.minimal" - ] + "kind": { + "object": { + "stored-name": { + "kind": "Value", + "types": [ + "test.minimal" + ] + } } } }, "test.minimal": { - "object": {} + "kind": { + "object": {} + } } }, "root": { diff --git a/packages/dds/tree/src/test/snapshots/schema-files/identifier-field - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/identifier-field - schema v2.json index 189a77df631c..54464581a27a 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/identifier-field - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/identifier-field - schema v2.json @@ -2,7 +2,9 @@ "version": 2, "nodes": { "com.fluidframework.leaf.string": { - "leaf": 1 + "kind": { + "leaf": 1 + } } }, "root": { diff --git a/packages/dds/tree/src/test/snapshots/schema-files/minimal - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/minimal - schema v2.json index 31f730a1651b..1faedea5f391 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/minimal - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/minimal - schema v2.json @@ -2,7 +2,9 @@ "version": 2, "nodes": { "test.minimal": { - "object": {} + "kind": { + "object": {} + } } }, "root": { diff --git a/packages/dds/tree/src/test/snapshots/schema-files/node-with-identifier-field - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/node-with-identifier-field - schema v2.json index 8e923d4119d7..b2e88a500afd 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/node-with-identifier-field - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/node-with-identifier-field - schema v2.json @@ -2,15 +2,19 @@ "version": 2, "nodes": { "com.fluidframework.leaf.string": { - "leaf": 1 + "kind": { + "leaf": 1 + } }, "test.hasIdentifierField": { - "object": { - "field": { - "kind": "Identifier", - "types": [ - "com.fluidframework.leaf.string" - ] + "kind": { + "object": { + "field": { + "kind": "Identifier", + "types": [ + "com.fluidframework.leaf.string" + ] + } } } } diff --git a/packages/dds/tree/src/test/snapshots/schema-files/null - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/null - schema v2.json index d28d81ab2ae6..6e78978c22a9 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/null - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/null - schema v2.json @@ -2,7 +2,9 @@ "version": 2, "nodes": { "com.fluidframework.leaf.null": { - "leaf": 4 + "kind": { + "leaf": 4 + } } }, "root": { diff --git a/packages/dds/tree/src/test/snapshots/schema-files/numeric - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/numeric - schema v2.json index cabf326aa297..1a169e308c83 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/numeric - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/numeric - schema v2.json @@ -2,7 +2,9 @@ "version": 2, "nodes": { "com.fluidframework.leaf.number": { - "leaf": 0 + "kind": { + "leaf": 0 + } } }, "root": { diff --git a/packages/dds/tree/src/test/snapshots/schema-files/numericMap-empty - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/numericMap-empty - schema v2.json index 57fe12469f0f..d635814e4494 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/numericMap-empty - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/numericMap-empty - schema v2.json @@ -2,14 +2,18 @@ "version": 2, "nodes": { "com.fluidframework.leaf.number": { - "leaf": 0 + "kind": { + "leaf": 0 + } }, "test.numericMap": { - "map": { - "kind": "Optional", - "types": [ - "com.fluidframework.leaf.number" - ] + "kind": { + "map": { + "kind": "Optional", + "types": [ + "com.fluidframework.leaf.number" + ] + } } } }, diff --git a/packages/dds/tree/src/test/snapshots/schema-files/numericMap-full - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/numericMap-full - schema v2.json index 57fe12469f0f..d635814e4494 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/numericMap-full - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/numericMap-full - schema v2.json @@ -2,14 +2,18 @@ "version": 2, "nodes": { "com.fluidframework.leaf.number": { - "leaf": 0 + "kind": { + "leaf": 0 + } }, "test.numericMap": { - "map": { - "kind": "Optional", - "types": [ - "com.fluidframework.leaf.number" - ] + "kind": { + "map": { + "kind": "Optional", + "types": [ + "com.fluidframework.leaf.number" + ] + } } } }, diff --git a/packages/dds/tree/src/test/snapshots/schema-files/numericSequence - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/numericSequence - schema v2.json index 8cdc31d1b205..e6c2fd36deec 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/numericSequence - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/numericSequence - schema v2.json @@ -2,7 +2,9 @@ "version": 2, "nodes": { "com.fluidframework.leaf.number": { - "leaf": 0 + "kind": { + "leaf": 0 + } } }, "root": { diff --git a/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-deeper - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-deeper - schema v2.json index ac39334292e0..a2cd9bd24370 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-deeper - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-deeper - schema v2.json @@ -2,12 +2,14 @@ "version": 2, "nodes": { "test.recursiveType": { - "object": { - "field": { - "kind": "Optional", - "types": [ - "test.recursiveType" - ] + "kind": { + "object": { + "field": { + "kind": "Optional", + "types": [ + "test.recursiveType" + ] + } } } } diff --git a/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-empty - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-empty - schema v2.json index ac39334292e0..a2cd9bd24370 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-empty - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-empty - schema v2.json @@ -2,12 +2,14 @@ "version": 2, "nodes": { "test.recursiveType": { - "object": { - "field": { - "kind": "Optional", - "types": [ - "test.recursiveType" - ] + "kind": { + "object": { + "field": { + "kind": "Optional", + "types": [ + "test.recursiveType" + ] + } } } } diff --git a/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-recursive - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-recursive - schema v2.json index ac39334292e0..a2cd9bd24370 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-recursive - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/recursiveType-recursive - schema v2.json @@ -2,12 +2,14 @@ "version": 2, "nodes": { "test.recursiveType": { - "object": { - "field": { - "kind": "Optional", - "types": [ - "test.recursiveType" - ] + "kind": { + "object": { + "field": { + "kind": "Optional", + "types": [ + "test.recursiveType" + ] + } } } } diff --git a/packages/dds/tree/src/test/snapshots/schema-files/true boolean - schema v2.json b/packages/dds/tree/src/test/snapshots/schema-files/true boolean - schema v2.json index dbd7b1e621ba..4f478d762a80 100644 --- a/packages/dds/tree/src/test/snapshots/schema-files/true boolean - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/schema-files/true boolean - schema v2.json @@ -2,7 +2,9 @@ "version": 2, "nodes": { "com.fluidframework.leaf.boolean": { - "leaf": 2 + "kind": { + "leaf": 2 + } } }, "root": { From e527eba785ec0f97e2276f9c89bb2702d90186b8 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Mon, 19 May 2025 14:34:22 -0700 Subject: [PATCH 18/45] - Removed minimum client version changes. Equivalent changes will be added in another PR. - Updated a comment. --- packages/dds/tree/api-report/tree.alpha.api.md | 3 +-- packages/dds/tree/src/codec/codec.ts | 4 +--- packages/dds/tree/src/core/schema-stored/schema.ts | 4 +++- packages/dds/tree/src/feature-libraries/schema-index/codec.ts | 2 -- .../tree/src/test/feature-libraries/schema-index/codecUtil.ts | 2 -- 5 files changed, 5 insertions(+), 10 deletions(-) diff --git a/packages/dds/tree/api-report/tree.alpha.api.md b/packages/dds/tree/api-report/tree.alpha.api.md index da1756afe87d..0b64ad42ae69 100644 --- a/packages/dds/tree/api-report/tree.alpha.api.md +++ b/packages/dds/tree/api-report/tree.alpha.api.md @@ -237,8 +237,7 @@ export enum FluidClientVersion { v2_0 = "v2_0", v2_1 = "v2_1", v2_2 = "v2_2", - v2_3 = "v2_3", - v2_4 = "v2_4" + v2_3 = "v2_3" } // @alpha diff --git a/packages/dds/tree/src/codec/codec.ts b/packages/dds/tree/src/codec/codec.ts index 55cb2bdae5e3..1d3574576be3 100644 --- a/packages/dds/tree/src/codec/codec.ts +++ b/packages/dds/tree/src/codec/codec.ts @@ -352,11 +352,9 @@ export enum FluidClientVersion { v2_2 = "v2_2", /** Fluid Framework Client 2.3 and newer. */ v2_3 = "v2_3", - /** Fluid Framework Client 2.4 and newer. */ - v2_4 = "v2_4", } /** * The version of this code. */ -export const currentVersion: FluidClientVersion = FluidClientVersion.v2_4; +export const currentVersion: FluidClientVersion = FluidClientVersion.v2_3; diff --git a/packages/dds/tree/src/core/schema-stored/schema.ts b/packages/dds/tree/src/core/schema-stored/schema.ts index e268e71b23e6..4a6438435b7e 100644 --- a/packages/dds/tree/src/core/schema-stored/schema.ts +++ b/packages/dds/tree/src/core/schema-stored/schema.ts @@ -24,7 +24,9 @@ import type { Multiplicity } from "./multiplicity.js"; export enum SchemaCodecVersion { v1 = 1, - // Adds persisted metadata to the schema. + /** + * Adds persisted metadata to the node schema and field schema. + */ v2 = 2, } diff --git a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts index 8c16c2b6428b..6e04b77b6eff 100644 --- a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts +++ b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts @@ -44,8 +44,6 @@ export function clientVersionToSchemaVersion( case FluidClientVersion.v2_2: case FluidClientVersion.v2_3: return SchemaCodecVersion.v1; - case FluidClientVersion.v2_4: - return SchemaCodecVersion.v2; default: unreachableCase(clientVersion); } diff --git a/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts b/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts index 2037dba8bb1a..02530f351c2c 100644 --- a/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts +++ b/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts @@ -30,8 +30,6 @@ export function schemaFormatToClientVersion( switch (schemaFormat) { case SchemaCodecVersion.v1: return FluidClientVersion.v2_0; - case SchemaCodecVersion.v2: - return FluidClientVersion.v2_4; default: assert(false, `Unsupported schema format: ${schemaFormat}`); } From 3eae1f656a2cad733279df94e71a6f0d7b693c9b Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Mon, 19 May 2025 15:21:16 -0700 Subject: [PATCH 19/45] Removed more minimumClientVersion glue. --- .../feature-libraries/schema-index/codec.ts | 13 +++--------- .../dds/tree/src/shared-tree/treeAlpha.ts | 1 - .../schema-index/codecUtil.ts | 20 +------------------ 3 files changed, 4 insertions(+), 30 deletions(-) diff --git a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts index 6e04b77b6eff..4158abf33e52 100644 --- a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts +++ b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts @@ -5,7 +5,7 @@ import { fail, unreachableCase } from "@fluidframework/core-utils/internal"; import { - FluidClientVersion, + type FluidClientVersion, type ICodecFamily, type ICodecOptions, type IJsonCodec, @@ -38,15 +38,8 @@ import { SchemaCodecVersion } from "../../core/index.js"; export function clientVersionToSchemaVersion( clientVersion: FluidClientVersion, ): SchemaCodecVersion { - switch (clientVersion) { - case FluidClientVersion.v2_0: - case FluidClientVersion.v2_1: - case FluidClientVersion.v2_2: - case FluidClientVersion.v2_3: - return SchemaCodecVersion.v1; - default: - unreachableCase(clientVersion); - } + // Only one version of the schema codec is currently supported. + return SchemaCodecVersion.v1; } /** diff --git a/packages/dds/tree/src/shared-tree/treeAlpha.ts b/packages/dds/tree/src/shared-tree/treeAlpha.ts index d944b5464024..3bea5d65e9cd 100644 --- a/packages/dds/tree/src/shared-tree/treeAlpha.ts +++ b/packages/dds/tree/src/shared-tree/treeAlpha.ts @@ -380,5 +380,4 @@ const versionToFormat = { v2_1: 1, v2_2: 1, v2_3: 1, - v2_4: 1, }; diff --git a/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts b/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts index 02530f351c2c..c37d49f34efb 100644 --- a/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts +++ b/packages/dds/tree/src/test/feature-libraries/schema-index/codecUtil.ts @@ -3,14 +3,12 @@ * Licensed under the MIT License. */ -import { FluidClientVersion } from "../../../codec/index.js"; -import { SchemaCodecVersion } from "../../../core/index.js"; +import type { SchemaCodecVersion } from "../../../core/index.js"; import { makeSchemaCodecs, // eslint-disable-next-line import/no-internal-modules } from "../../../feature-libraries/schema-index/index.js"; import { ajvValidator } from "../../codec/index.js"; -import { assert } from "@fluidframework/core-utils/internal"; /* * The list of supported schema write versions. Used in tests that cover multiple schema versions. @@ -18,19 +16,3 @@ import { assert } from "@fluidframework/core-utils/internal"; export const supportedSchemaFormats = Array.from( makeSchemaCodecs({ jsonValidator: ajvValidator }).getSupportedFormats(), ).filter((format) => format !== undefined) as SchemaCodecVersion[]; - -/** - * Convert a schema version to the minimum Fluid client version supporting that format. - * @param schemaFormat - The schema format version. - * @returns The Fluid client version that supports the provided schema format. - */ -export function schemaFormatToClientVersion( - schemaFormat: SchemaCodecVersion, -): FluidClientVersion { - switch (schemaFormat) { - case SchemaCodecVersion.v1: - return FluidClientVersion.v2_0; - default: - assert(false, `Unsupported schema format: ${schemaFormat}`); - } -} From 98e1b7673ad12eac063201963379cfef301c88d9 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Mon, 19 May 2025 15:44:45 -0700 Subject: [PATCH 20/45] Updated API files. --- .../fluid-framework/api-report/fluid-framework.alpha.api.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md b/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md index 0051d03e2e59..e5e7c6141cc9 100644 --- a/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md +++ b/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md @@ -281,8 +281,7 @@ export enum FluidClientVersion { v2_0 = "v2_0", v2_1 = "v2_1", v2_2 = "v2_2", - v2_3 = "v2_3", - v2_4 = "v2_4" + v2_3 = "v2_3" } // @public From ecc77c9fc1e0fc7de8ce590bdf1e2683b8f737c2 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Tue, 20 May 2025 10:38:32 -0700 Subject: [PATCH 21/45] Rename: SchemaCodecVersion -> SchemaVersion. --- packages/dds/tree/src/core/index.ts | 2 +- .../dds/tree/src/core/schema-stored/index.ts | 2 +- .../dds/tree/src/core/schema-stored/schema.ts | 2 +- .../schema-edits/schemaChangeCodecs.ts | 10 +++--- .../feature-libraries/schema-index/codec.ts | 33 +++++++++---------- .../schema-index/formatV1.ts | 4 +-- .../schema-index/formatV2.ts | 4 +-- .../tree/src/shared-tree/independentView.ts | 4 +-- .../dds/tree/src/shared-tree/sharedTree.ts | 6 ++-- .../tree/src/simple-tree/api/storedSchema.ts | 4 +-- .../schema-index/codec.spec.ts | 6 ++-- .../schema-index/codecUtil.ts | 4 +-- packages/dds/tree/src/test/utils.ts | 7 ++-- 13 files changed, 41 insertions(+), 47 deletions(-) diff --git a/packages/dds/tree/src/core/index.ts b/packages/dds/tree/src/core/index.ts index b837a2ec4bd5..015901277e3a 100644 --- a/packages/dds/tree/src/core/index.ts +++ b/packages/dds/tree/src/core/index.ts @@ -149,7 +149,7 @@ export { type SchemaAndPolicy, Multiplicity, type SchemaPolicy, - SchemaCodecVersion, + SchemaVersion, } from "./schema-stored/index.js"; export { diff --git a/packages/dds/tree/src/core/schema-stored/index.ts b/packages/dds/tree/src/core/schema-stored/index.ts index 3a6c789af0f1..fa2c5575ad44 100644 --- a/packages/dds/tree/src/core/schema-stored/index.ts +++ b/packages/dds/tree/src/core/schema-stored/index.ts @@ -23,7 +23,7 @@ export { storedSchemaDecodeDispatcher, type SchemaAndPolicy, type SchemaPolicy, - SchemaCodecVersion, + SchemaVersion, } from "./schema.js"; export { type TreeStoredSchemaSubscription, diff --git a/packages/dds/tree/src/core/schema-stored/schema.ts b/packages/dds/tree/src/core/schema-stored/schema.ts index 4a6438435b7e..1f4ec4a88c20 100644 --- a/packages/dds/tree/src/core/schema-stored/schema.ts +++ b/packages/dds/tree/src/core/schema-stored/schema.ts @@ -22,7 +22,7 @@ import type { } from "./formatV2.js"; import type { Multiplicity } from "./multiplicity.js"; -export enum SchemaCodecVersion { +export enum SchemaVersion { v1 = 1, /** * Adds persisted metadata to the node schema and field schema. diff --git a/packages/dds/tree/src/feature-libraries/schema-edits/schemaChangeCodecs.ts b/packages/dds/tree/src/feature-libraries/schema-edits/schemaChangeCodecs.ts index 28727acc815b..db58e3ee8002 100644 --- a/packages/dds/tree/src/feature-libraries/schema-edits/schemaChangeCodecs.ts +++ b/packages/dds/tree/src/feature-libraries/schema-edits/schemaChangeCodecs.ts @@ -17,7 +17,7 @@ import { makeSchemaCodec } from "../schema-index/index.js"; import { EncodedSchemaChange } from "./schemaChangeFormat.js"; import type { SchemaChange } from "./schemaChangeTypes.js"; -import { SchemaCodecVersion } from "../../core/index.js"; +import { SchemaVersion } from "../../core/index.js"; /** * Create a family of schema change codecs. @@ -26,8 +26,8 @@ import { SchemaCodecVersion } from "../../core/index.js"; */ export function makeSchemaChangeCodecs(options: ICodecOptions): ICodecFamily { return makeCodecFamily([ - [SchemaCodecVersion.v1, makeSchemaChangeCodecV1(options, SchemaCodecVersion.v1)], - [SchemaCodecVersion.v2, makeSchemaChangeCodecV1(options, SchemaCodecVersion.v2)], + [SchemaVersion.v1, makeSchemaChangeCodecV1(options, SchemaVersion.v1)], + [SchemaVersion.v2, makeSchemaChangeCodecV1(options, SchemaVersion.v2)], ]); } @@ -39,7 +39,7 @@ export function makeSchemaChangeCodecs(options: ICodecOptions): ICodecFamily { const family = makeSchemaChangeCodecs(options); return makeVersionDispatchingCodec(family, { ...options, writeVersion }); @@ -53,7 +53,7 @@ export function makeSchemaChangeCodec( */ function makeSchemaChangeCodecV1( options: ICodecOptions, - schemaWriteVersion: SchemaCodecVersion, + schemaWriteVersion: SchemaVersion, ): IJsonCodec { const schemaCodec = makeSchemaCodec(options, schemaWriteVersion); const schemaChangeCodec: IJsonCodec = { diff --git a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts index 4158abf33e52..3896d7e83c1d 100644 --- a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts +++ b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts @@ -28,18 +28,18 @@ import { brand, type JsonCompatible } from "../../util/index.js"; import { Format as FormatV1 } from "./formatV1.js"; import { Format as FormatV2 } from "./formatV2.js"; -import { SchemaCodecVersion } from "../../core/index.js"; +import { SchemaVersion } from "../../core/index.js"; /** - * Convert a FluidClientVersion to a SchemaCodecVersion. + * Convert a FluidClientVersion to a SchemaVersion. * @param clientVersion - The FluidClientVersion to convert. - * @returns The SchemaCodecVersion that corresponds to the provided FluidClientVersion. + * @returns The SchemaVersion that corresponds to the provided FluidClientVersion. */ export function clientVersionToSchemaVersion( clientVersion: FluidClientVersion, -): SchemaCodecVersion { +): SchemaVersion { // Only one version of the schema codec is currently supported. - return SchemaCodecVersion.v1; + return SchemaVersion.v1; } /** @@ -52,7 +52,7 @@ export function clientVersionToSchemaVersion( */ export function makeSchemaCodec( options: ICodecOptions, - writeVersion: SchemaCodecVersion, + writeVersion: SchemaVersion, ): IJsonCodec { const family = makeSchemaCodecs(options); return makeVersionDispatchingCodec(family, { ...options, writeVersion }); @@ -65,8 +65,8 @@ export function makeSchemaCodec( */ export function makeSchemaCodecs(options: ICodecOptions): ICodecFamily { return makeCodecFamily([ - [SchemaCodecVersion.v1, makeSchemaCodecV1(options)], - [SchemaCodecVersion.v2, makeSchemaCodecV2(options)], + [SchemaVersion.v1, makeSchemaCodecV1(options)], + [SchemaVersion.v2, makeSchemaCodecV2(options)], ]); } @@ -76,14 +76,11 @@ export function makeSchemaCodecs(options: ICodecOptions): ICodecFamily { - return makeVersionedValidatedCodec(options, new Set([SchemaCodecVersion.v1]), FormatV1, { + return makeVersionedValidatedCodec(options, new Set([SchemaVersion.v1]), FormatV1, { encode: (data: TreeStoredSchema) => encodeRepoV1(data), decode: (data: FormatV1) => decodeV1(data), }); @@ -170,7 +167,7 @@ function makeSchemaCodecV1(options: ICodecOptions): IJsonCodec { - return makeVersionedValidatedCodec(options, new Set([SchemaCodecVersion.v2]), FormatV2, { + return makeVersionedValidatedCodec(options, new Set([SchemaVersion.v2]), FormatV2, { encode: (data: TreeStoredSchema) => encodeRepoV2(data), decode: (data: FormatV2) => decodeV2(data), }); diff --git a/packages/dds/tree/src/feature-libraries/schema-index/formatV1.ts b/packages/dds/tree/src/feature-libraries/schema-index/formatV1.ts index 31456dba4551..919b292c307a 100644 --- a/packages/dds/tree/src/feature-libraries/schema-index/formatV1.ts +++ b/packages/dds/tree/src/feature-libraries/schema-index/formatV1.ts @@ -5,7 +5,7 @@ import { type ObjectOptions, type Static, Type } from "@sinclair/typebox"; -import { SchemaCodecVersion, schemaFormatV1 } from "../../core/index.js"; +import { SchemaVersion, schemaFormatV1 } from "../../core/index.js"; const noAdditionalProps: ObjectOptions = { additionalProperties: false }; @@ -21,7 +21,7 @@ const noAdditionalProps: ObjectOptions = { additionalProperties: false }; */ export const Format = Type.Object( { - version: Type.Literal(SchemaCodecVersion.v1), + version: Type.Literal(SchemaVersion.v1), nodes: Type.Record(Type.String(), schemaFormatV1.TreeNodeSchemaDataFormat), root: schemaFormatV1.FieldSchemaFormat, }, diff --git a/packages/dds/tree/src/feature-libraries/schema-index/formatV2.ts b/packages/dds/tree/src/feature-libraries/schema-index/formatV2.ts index 09a47a547130..7fd7df005831 100644 --- a/packages/dds/tree/src/feature-libraries/schema-index/formatV2.ts +++ b/packages/dds/tree/src/feature-libraries/schema-index/formatV2.ts @@ -5,7 +5,7 @@ import { type ObjectOptions, type Static, Type } from "@sinclair/typebox"; -import { SchemaCodecVersion, schemaFormatV2 } from "../../core/index.js"; +import { SchemaVersion, schemaFormatV2 } from "../../core/index.js"; const noAdditionalProps: ObjectOptions = { additionalProperties: false }; @@ -21,7 +21,7 @@ const noAdditionalProps: ObjectOptions = { additionalProperties: false }; */ export const Format = Type.Object( { - version: Type.Literal(SchemaCodecVersion.v2), + version: Type.Literal(SchemaVersion.v2), nodes: Type.Record(Type.String(), schemaFormatV2.TreeNodeSchemaDataFormat), root: schemaFormatV2.FieldSchemaFormat, metadata: schemaFormatV2.PersistedMetadataFormat, diff --git a/packages/dds/tree/src/shared-tree/independentView.ts b/packages/dds/tree/src/shared-tree/independentView.ts index e70648691df2..80b293bc84ec 100644 --- a/packages/dds/tree/src/shared-tree/independentView.ts +++ b/packages/dds/tree/src/shared-tree/independentView.ts @@ -13,7 +13,7 @@ import type { ICodecOptions } from "../codec/index.js"; import { type RevisionTag, RevisionTagCodec, - SchemaCodecVersion, + SchemaVersion, TreeStoredSchemaRepository, } from "../core/index.js"; import { @@ -93,7 +93,7 @@ export function independentInitializedView format !== undefined) as SchemaCodecVersion[]; +).filter((format) => format !== undefined) as SchemaVersion[]; diff --git a/packages/dds/tree/src/test/utils.ts b/packages/dds/tree/src/test/utils.ts index 3dfad4c1f3fa..5f8a3060ac11 100644 --- a/packages/dds/tree/src/test/utils.ts +++ b/packages/dds/tree/src/test/utils.ts @@ -93,7 +93,7 @@ import { type DeltaDetachedNodeRename, type NormalizedFieldUpPath, type ExclusiveMapTree, - SchemaCodecVersion, + SchemaVersion, } from "../core/index.js"; import { typeboxValidator } from "../external-utilities/index.js"; import { @@ -634,10 +634,7 @@ export function validateTree(tree: ITreeCheckout, expected: JsonableTree[]): voi // that equality of two schemas in tests is achieved by deep-comparing their persisted representations. // If the newer format is a superset of the previous format, it can be safely used for comparisons. This is the // case with schema format v2. -const schemaCodec = makeSchemaCodec( - { jsonValidator: typeboxValidator }, - SchemaCodecVersion.v2, -); +const schemaCodec = makeSchemaCodec({ jsonValidator: typeboxValidator }, SchemaVersion.v2); export function checkRemovedRootsAreSynchronized(trees: readonly ITreeCheckout[]): void { if (trees.length > 1) { From 737633bf58b05e85613ace2236e9241096e8d9a1 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Wed, 21 May 2025 11:52:24 -0700 Subject: [PATCH 22/45] Removed top-level persisted metadata. --- .../tree/src/feature-libraries/schema-index/formatV2.ts | 1 - .../src/test/feature-libraries/schema-index/codec.spec.ts | 7 +++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/dds/tree/src/feature-libraries/schema-index/formatV2.ts b/packages/dds/tree/src/feature-libraries/schema-index/formatV2.ts index 7fd7df005831..f37c804ee063 100644 --- a/packages/dds/tree/src/feature-libraries/schema-index/formatV2.ts +++ b/packages/dds/tree/src/feature-libraries/schema-index/formatV2.ts @@ -24,7 +24,6 @@ export const Format = Type.Object( version: Type.Literal(SchemaVersion.v2), nodes: Type.Record(Type.String(), schemaFormatV2.TreeNodeSchemaDataFormat), root: schemaFormatV2.FieldSchemaFormat, - metadata: schemaFormatV2.PersistedMetadataFormat, }, noAdditionalProps, ); diff --git a/packages/dds/tree/src/test/feature-libraries/schema-index/codec.spec.ts b/packages/dds/tree/src/test/feature-libraries/schema-index/codec.spec.ts index 708a5c25a926..176e95d041dc 100644 --- a/packages/dds/tree/src/test/feature-libraries/schema-index/codec.spec.ts +++ b/packages/dds/tree/src/test/feature-libraries/schema-index/codec.spec.ts @@ -77,8 +77,11 @@ describe("SchemaIndex", () => { { version: 2 as const, nodes: {}, - root: { kind: "x" as FieldKindIdentifier, types: [] }, - metadata: { "ff-system": { "eDiscovery-exclude": "true" } }, + root: { + kind: "x" as FieldKindIdentifier, + types: [], + metadata: { "ff-system": { "eDiscovery-exclude": "true" } }, + }, } satisfies FormatV2, ]; for (const data of cases) { From 4bb136ca420e8fd1868f0084c6d02811cc06992d Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Wed, 21 May 2025 17:00:46 -0700 Subject: [PATCH 23/45] Fixed comments. --- packages/dds/tree/src/core/schema-stored/schema.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/packages/dds/tree/src/core/schema-stored/schema.ts b/packages/dds/tree/src/core/schema-stored/schema.ts index 51a4dc1fce15..abc7a3047b17 100644 --- a/packages/dds/tree/src/core/schema-stored/schema.ts +++ b/packages/dds/tree/src/core/schema-stored/schema.ts @@ -184,18 +184,12 @@ export abstract class TreeNodeStoredSchema { protected _typeCheck!: MakeNominal; /** - * @privateRemarks - * Returns TreeNodeSchemaDataFormat. - * This is uses an opaque type to avoid leaking these types out of the package, - * and is runtime validated by the codec. + * Encode in the v1 schema format. */ public abstract encodeV1(): TreeNodeSchemaDataFormatV1; /** - * @privateRemarks - * Returns TreeNodeSchemaDataFormat. - * This is uses an opaque type to avoid leaking these types out of the package, - * and is runtime validated by the codec. + * Encode in the v2 schema format. */ public abstract encodeV2(): TreeNodeSchemaDataFormatV2; From 0bcb90f9ca7e19ea1ce38a6acd0284eca1b04023 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Thu, 22 May 2025 10:23:07 -0700 Subject: [PATCH 24/45] Exposed persistedMetadata as a JsonCompatibleReadOnlyObject. --- .../dds/tree/api-report/tree.alpha.api.md | 10 +++++++++ packages/dds/tree/api-report/tree.beta.api.md | 10 +++++++++ .../tree/api-report/tree.legacy.alpha.api.md | 10 +++++++++ .../tree/api-report/tree.legacy.public.api.md | 10 +++++++++ .../dds/tree/api-report/tree.public.api.md | 10 +++++++++ packages/dds/tree/src/index.ts | 2 ++ .../dds/tree/src/simple-tree/schemaTypes.ts | 21 +++++++++++++++++-- packages/dds/tree/src/util/utils.ts | 2 ++ 8 files changed, 73 insertions(+), 2 deletions(-) diff --git a/packages/dds/tree/api-report/tree.alpha.api.md b/packages/dds/tree/api-report/tree.alpha.api.md index 4150645cbc46..74c20530f1ad 100644 --- a/packages/dds/tree/api-report/tree.alpha.api.md +++ b/packages/dds/tree/api-report/tree.alpha.api.md @@ -216,6 +216,7 @@ export interface FieldSchemaAlphaUnsafe { readonly custom?: TCustomMetadata; readonly description?: string | undefined; + readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } // @alpha @@ -443,6 +444,14 @@ export type JsonCompatibleObject = { [P in string]?: JsonCompatible; }; +// @public +export type JsonCompatibleReadOnly = string | number | boolean | null | readonly JsonCompatibleReadOnly[] | JsonCompatibleReadOnlyObject; + +// @public +export type JsonCompatibleReadOnlyObject = { + readonly [P in string]?: JsonCompatibleReadOnly; +}; + // @alpha @sealed export type JsonFieldSchema = { readonly description?: string | undefined; @@ -584,6 +593,7 @@ export enum NodeKind { export interface NodeSchemaMetadata { readonly custom?: TCustomMetadata | undefined; readonly description?: string | undefined; + readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } // @public @sealed diff --git a/packages/dds/tree/api-report/tree.beta.api.md b/packages/dds/tree/api-report/tree.beta.api.md index 7534cd977477..6b7687be09e4 100644 --- a/packages/dds/tree/api-report/tree.beta.api.md +++ b/packages/dds/tree/api-report/tree.beta.api.md @@ -77,6 +77,7 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; + readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } // @public @system @@ -176,6 +177,14 @@ export interface ITreeViewConfiguration = Item | (() => Item); @@ -227,6 +236,7 @@ export enum NodeKind { export interface NodeSchemaMetadata { readonly custom?: TCustomMetadata | undefined; readonly description?: string | undefined; + readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } // @public @sealed diff --git a/packages/dds/tree/api-report/tree.legacy.alpha.api.md b/packages/dds/tree/api-report/tree.legacy.alpha.api.md index 7aabd1819bbb..3280f9460b98 100644 --- a/packages/dds/tree/api-report/tree.legacy.alpha.api.md +++ b/packages/dds/tree/api-report/tree.legacy.alpha.api.md @@ -77,6 +77,7 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; + readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } // @public @system @@ -176,6 +177,14 @@ export interface ITreeViewConfiguration = Item | (() => Item); @@ -222,6 +231,7 @@ export enum NodeKind { export interface NodeSchemaMetadata { readonly custom?: TCustomMetadata | undefined; readonly description?: string | undefined; + readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } // @public @sealed diff --git a/packages/dds/tree/api-report/tree.legacy.public.api.md b/packages/dds/tree/api-report/tree.legacy.public.api.md index 68b1d8e7b024..b81cda67fbc6 100644 --- a/packages/dds/tree/api-report/tree.legacy.public.api.md +++ b/packages/dds/tree/api-report/tree.legacy.public.api.md @@ -77,6 +77,7 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; + readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } // @public @system @@ -176,6 +177,14 @@ export interface ITreeViewConfiguration = Item | (() => Item); @@ -222,6 +231,7 @@ export enum NodeKind { export interface NodeSchemaMetadata { readonly custom?: TCustomMetadata | undefined; readonly description?: string | undefined; + readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } // @public @sealed diff --git a/packages/dds/tree/api-report/tree.public.api.md b/packages/dds/tree/api-report/tree.public.api.md index 68b1d8e7b024..b81cda67fbc6 100644 --- a/packages/dds/tree/api-report/tree.public.api.md +++ b/packages/dds/tree/api-report/tree.public.api.md @@ -77,6 +77,7 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; + readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } // @public @system @@ -176,6 +177,14 @@ export interface ITreeViewConfiguration = Item | (() => Item); @@ -222,6 +231,7 @@ export enum NodeKind { export interface NodeSchemaMetadata { readonly custom?: TCustomMetadata | undefined; readonly description?: string | undefined; + readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } // @public @sealed diff --git a/packages/dds/tree/src/index.ts b/packages/dds/tree/src/index.ts index d1b99943f8a2..f3ecf8363061 100644 --- a/packages/dds/tree/src/index.ts +++ b/packages/dds/tree/src/index.ts @@ -285,6 +285,8 @@ export type { PopUnion, JsonCompatible, JsonCompatibleObject, + JsonCompatibleReadOnly, + JsonCompatibleReadOnlyObject, } from "./util/index.js"; export { cloneWithReplacements } from "./util/index.js"; diff --git a/packages/dds/tree/src/simple-tree/schemaTypes.ts b/packages/dds/tree/src/simple-tree/schemaTypes.ts index 6cbe171144bf..d0cde25c4ed7 100644 --- a/packages/dds/tree/src/simple-tree/schemaTypes.ts +++ b/packages/dds/tree/src/simple-tree/schemaTypes.ts @@ -19,6 +19,7 @@ import { getOrCreate, type RestrictiveStringRecord, type IsUnion, + type JsonCompatibleReadOnlyObject, } from "../util/index.js"; import type { Unhydrated, @@ -357,6 +358,11 @@ export interface FieldSchemaMetadata { * used as the `description` field. */ readonly description?: string | undefined; + + /** + * The persisted metadata for this schema element. + */ + readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } /** @@ -777,13 +783,19 @@ function areMetadataEqual( b: FieldSchemaMetadata | undefined, ): boolean { // If any new fields are added to FieldSchemaMetadata, this check will stop compiling as a reminder that this function needs to be updated. - type _keys = requireTrue>; + type _keys = requireTrue< + areOnlyKeys + >; if (a === b) { return true; } - return a?.custom === b?.custom && a?.description === b?.description; + return ( + a?.custom === b?.custom && + a?.description === b?.description && + a?.persistedMetadata === b?.persistedMetadata + ); } const cachedLazyItem = new WeakMap<() => unknown, unknown>(); @@ -1335,4 +1347,9 @@ export interface NodeSchemaMetadata { * used as the `description` property. */ readonly description?: string | undefined; + + /** + * The persisted metadata for this schema element. + */ + readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } diff --git a/packages/dds/tree/src/util/utils.ts b/packages/dds/tree/src/util/utils.ts index 97b5c89c92b2..b1f96e90e63a 100644 --- a/packages/dds/tree/src/util/utils.ts +++ b/packages/dds/tree/src/util/utils.ts @@ -310,6 +310,7 @@ export type JsonCompatibleObject = { [P in string]?: JsonCompati * @remarks * This does not robustly forbid non json comparable data via type checking, * but instead mostly restricts access to it. + * @public */ export type JsonCompatibleReadOnly = | string @@ -325,6 +326,7 @@ export type JsonCompatibleReadOnly = * @remarks * This does not robustly forbid non json comparable data via type checking, * but instead mostly restricts access to it. + * @public */ export type JsonCompatibleReadOnlyObject = { readonly [P in string]?: JsonCompatibleReadOnly }; From a675338cc9bded16e8eeedf0e098e99a8353b3f5 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Thu, 22 May 2025 10:33:49 -0700 Subject: [PATCH 25/45] Changeset. --- .changeset/brown-dingos-switch.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/brown-dingos-switch.md diff --git a/.changeset/brown-dingos-switch.md b/.changeset/brown-dingos-switch.md new file mode 100644 index 000000000000..f32abc1605c6 --- /dev/null +++ b/.changeset/brown-dingos-switch.md @@ -0,0 +1,7 @@ +--- +"@fluidframework/tree": minor +"__section": feature +--- +Added a v2 Shared Tree schema with support for persisted metadata. + +This change adds support for persisted metadata, but does not enable it by default. \ No newline at end of file From 1d1ef5f6f920d63fe73247991ffdfef3f1325065 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Thu, 22 May 2025 10:38:25 -0700 Subject: [PATCH 26/45] Wired up toStoredSchema. Still needs tests. --- packages/dds/tree/src/simple-tree/toStoredSchema.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/dds/tree/src/simple-tree/toStoredSchema.ts b/packages/dds/tree/src/simple-tree/toStoredSchema.ts index 1e7319fc4f87..66a99dbf3333 100644 --- a/packages/dds/tree/src/simple-tree/toStoredSchema.ts +++ b/packages/dds/tree/src/simple-tree/toStoredSchema.ts @@ -113,7 +113,7 @@ export function getStoredSchema(schema: SimpleNodeSchema): TreeNodeStoredSchema return new MapNodeStoredSchema({ kind: FieldKinds.optional.identifier, types, - persistedMetadata: undefined, + persistedMetadata: schema.metadata, }); } case NodeKind.Array: { @@ -121,7 +121,7 @@ export function getStoredSchema(schema: SimpleNodeSchema): TreeNodeStoredSchema const field = { kind: FieldKinds.sequence.identifier, types, - persistedMetadata: undefined, + persistedMetadata: schema.metadata, }; const fields = new Map([[EmptyKey, field]]); return new ObjectNodeStoredSchema(fields); From b9898e8756f36aecb6da944fc2728885a8cabf78 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Thu, 22 May 2025 12:01:50 -0700 Subject: [PATCH 27/45] - Updated an old snapshot that didn't include the metadata field. - Added tests calling schemaFactoryAlpha. --- .../simple-tree/api/schemaFactory.spec.ts | 37 +++++++++++++++++++ .../files/SchemaIndexFormat - schema v2.json | 3 +- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/packages/dds/tree/src/test/simple-tree/api/schemaFactory.spec.ts b/packages/dds/tree/src/test/simple-tree/api/schemaFactory.spec.ts index 8a9f4ad39ca6..779a8ecfe9e3 100644 --- a/packages/dds/tree/src/test/simple-tree/api/schemaFactory.spec.ts +++ b/packages/dds/tree/src/test/simple-tree/api/schemaFactory.spec.ts @@ -414,6 +414,43 @@ describe("schemaFactory", () => { assert.deepEqual(schema.fields.get("bar")!.metadata, barMetadata); }); + it("Node schema persisted metadata", () => { + const factory = new SchemaFactoryAlpha(""); + + const fooMetadata = { + persistedMetadata: { "a": 2 }, + }; + + class Foo extends factory.objectAlpha( + "Foo", + { bar: factory.number }, + { metadata: fooMetadata }, + ) {} + + assert.deepEqual(Foo.metadata, fooMetadata); + + // Ensure `Foo.metadata` is typed as we expect, and we can access its fields without casting. + const persistedMetadata = Foo.metadata.persistedMetadata; + const a = Foo.metadata.persistedMetadata.a; + }); + + it("Field schema persisted metadata", () => { + const schemaFactory = new SchemaFactory("com.example"); + const fooMetadata = { + persistedMetadata: { "a": 2 }, + }; + + class Foo extends schemaFactory.object("Foo", { + bar: schemaFactory.required(schemaFactory.number, { metadata: fooMetadata }), + }) {} + + const foo = hydrate(Foo, { bar: 37 }); + + const schema = Tree.schema(foo) as ObjectNodeSchema; + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + assert.deepEqual(schema.fields.get("bar")!.metadata, fooMetadata); + }); + describe("deep equality", () => { const schema = new SchemaFactory("com.example"); diff --git a/packages/dds/tree/src/test/snapshots/files/SchemaIndexFormat - schema v2.json b/packages/dds/tree/src/test/snapshots/files/SchemaIndexFormat - schema v2.json index 3444b62bd315..4d829a7f6bff 100644 --- a/packages/dds/tree/src/test/snapshots/files/SchemaIndexFormat - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/files/SchemaIndexFormat - schema v2.json @@ -115,8 +115,7 @@ "kind", "types" ] - }, - "metadata": {} + } }, "required": [ "version", From 6ca189aeb06ce497710cbd7a8fff8d3d668fa2d6 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Tue, 27 May 2025 10:56:54 -0700 Subject: [PATCH 28/45] Refactor: persistedMetadata -> metadata on persisted types. --- packages/dds/tree/src/core/schema-stored/schema.ts | 8 ++++---- .../modular-schema/fieldKindWithEditor.ts | 2 +- packages/dds/tree/src/shared-tree/schematizeTree.ts | 2 +- .../src/simple-tree/api/schemaCompatibilityTester.ts | 2 +- .../dds/tree/src/simple-tree/prepareForInsertion.ts | 2 +- packages/dds/tree/src/simple-tree/toStoredSchema.ts | 6 +++--- .../chunked-forest/codec/schemaBasedEncode.spec.ts | 4 ++-- .../default-schema/schemaChecker.spec.ts | 2 +- .../feature-libraries/flex-tree/lazyField.spec.ts | 2 +- .../modular-schema/comparison.spec.ts | 2 +- .../modular-schema/isNeverTree.spec.ts | 2 +- packages/dds/tree/src/test/sequenceRootUtils.ts | 2 +- .../test/shared-tree/sharedTreeChangeFamily.spec.ts | 2 +- .../src/test/simple-tree/prepareForInsertion.spec.ts | 2 +- .../dds/tree/src/test/simple-tree/toMapTree.spec.ts | 2 +- packages/dds/tree/src/test/testTrees.ts | 12 ++++++------ 16 files changed, 27 insertions(+), 27 deletions(-) diff --git a/packages/dds/tree/src/core/schema-stored/schema.ts b/packages/dds/tree/src/core/schema-stored/schema.ts index abc7a3047b17..98b2cd50e720 100644 --- a/packages/dds/tree/src/core/schema-stored/schema.ts +++ b/packages/dds/tree/src/core/schema-stored/schema.ts @@ -146,7 +146,7 @@ export interface TreeFieldStoredSchema { * @remarks * Discarded when encoding to {@link SchemaFormatVersion.V1}. */ - readonly persistedMetadata: PersistedMetadataFormat | undefined; + readonly metadata: PersistedMetadataFormat | undefined; } /** @@ -170,7 +170,7 @@ export const storedEmptyFieldSchema: TreeFieldStoredSchema = { kind: brand(forbiddenFieldKindIdentifier), // This type set also forces the field to be empty not not allowing any types as all. types: new Set(), - persistedMetadata: undefined, + metadata: undefined, }; /** @@ -363,7 +363,7 @@ export function encodeFieldSchemaV1(schema: TreeFieldStoredSchema): FieldSchemaF export function encodeFieldSchemaV2(schema: TreeFieldStoredSchema): FieldSchemaFormatV2 { return { ...encodeFieldSchemaV1(schema), - metadata: schema.persistedMetadata, + metadata: schema.metadata, }; } @@ -372,7 +372,7 @@ export function decodeFieldSchema(schema: FieldSchemaFormatV2): TreeFieldStoredS // TODO: maybe provide actual FieldKind objects here, error on unrecognized kinds. kind: schema.kind, types: new Set(schema.types), - persistedMetadata: schema.metadata, + metadata: schema.metadata, }; return out; } diff --git a/packages/dds/tree/src/feature-libraries/modular-schema/fieldKindWithEditor.ts b/packages/dds/tree/src/feature-libraries/modular-schema/fieldKindWithEditor.ts index a59e7b2f83cb..ace34e2bd552 100644 --- a/packages/dds/tree/src/feature-libraries/modular-schema/fieldKindWithEditor.ts +++ b/packages/dds/tree/src/feature-libraries/modular-schema/fieldKindWithEditor.ts @@ -80,7 +80,7 @@ export class FieldKindWithEditor< kind: this.identifier, types: originalTypes, // Metadata is not used for this check. - persistedMetadata: undefined, + metadata: undefined, }) ) { return true; diff --git a/packages/dds/tree/src/shared-tree/schematizeTree.ts b/packages/dds/tree/src/shared-tree/schematizeTree.ts index dfc37c991ed4..26c291968921 100644 --- a/packages/dds/tree/src/shared-tree/schematizeTree.ts +++ b/packages/dds/tree/src/shared-tree/schematizeTree.ts @@ -67,7 +67,7 @@ export function initializeContent( rootFieldSchema: { kind: FieldKinds.optional.identifier, types: rootSchema.types, - persistedMetadata: rootSchema.persistedMetadata, + metadata: rootSchema.metadata, }, }; } diff --git a/packages/dds/tree/src/simple-tree/api/schemaCompatibilityTester.ts b/packages/dds/tree/src/simple-tree/api/schemaCompatibilityTester.ts index 06b5f9987db8..7e5f63041019 100644 --- a/packages/dds/tree/src/simple-tree/api/schemaCompatibilityTester.ts +++ b/packages/dds/tree/src/simple-tree/api/schemaCompatibilityTester.ts @@ -273,7 +273,7 @@ export class SchemaCompatibilityTester { } } - return { kind: original.kind, types, persistedMetadata: undefined }; + return { kind: original.kind, types, metadata: undefined }; } return original; } diff --git a/packages/dds/tree/src/simple-tree/prepareForInsertion.ts b/packages/dds/tree/src/simple-tree/prepareForInsertion.ts index e1006ebb98f1..3dcd2d745973 100644 --- a/packages/dds/tree/src/simple-tree/prepareForInsertion.ts +++ b/packages/dds/tree/src/simple-tree/prepareForInsertion.ts @@ -93,7 +93,7 @@ export function prepareArrayContentForInsertion( { kind: FieldKinds.sequence.identifier, types: fieldSchema.types, - persistedMetadata: undefined, + metadata: undefined, }, mapTrees, ); diff --git a/packages/dds/tree/src/simple-tree/toStoredSchema.ts b/packages/dds/tree/src/simple-tree/toStoredSchema.ts index 66a99dbf3333..b52ae0ce55ab 100644 --- a/packages/dds/tree/src/simple-tree/toStoredSchema.ts +++ b/packages/dds/tree/src/simple-tree/toStoredSchema.ts @@ -89,7 +89,7 @@ export function convertField(schema: SimpleFieldSchema): TreeFieldStoredSchema { const kind: FieldKindIdentifier = convertFieldKind.get(schema.kind)?.identifier ?? fail(0xae3 /* Invalid field kind */); const types: TreeTypeSet = schema.allowedTypesIdentifiers as TreeTypeSet; - return { kind, types, persistedMetadata: undefined }; + return { kind, types, metadata: undefined }; } const convertFieldKind = new Map([ @@ -113,7 +113,7 @@ export function getStoredSchema(schema: SimpleNodeSchema): TreeNodeStoredSchema return new MapNodeStoredSchema({ kind: FieldKinds.optional.identifier, types, - persistedMetadata: schema.metadata, + metadata: schema.metadata, }); } case NodeKind.Array: { @@ -121,7 +121,7 @@ export function getStoredSchema(schema: SimpleNodeSchema): TreeNodeStoredSchema const field = { kind: FieldKinds.sequence.identifier, types, - persistedMetadata: schema.metadata, + metadata: schema.metadata, }; const fields = new Map([[EmptyKey, field]]); return new ObjectNodeStoredSchema(fields); diff --git a/packages/dds/tree/src/test/feature-libraries/chunked-forest/codec/schemaBasedEncode.spec.ts b/packages/dds/tree/src/test/feature-libraries/chunked-forest/codec/schemaBasedEncode.spec.ts index bbf959059e08..d9841c6841ab 100644 --- a/packages/dds/tree/src/test/feature-libraries/chunked-forest/codec/schemaBasedEncode.spec.ts +++ b/packages/dds/tree/src/test/feature-libraries/chunked-forest/codec/schemaBasedEncode.spec.ts @@ -155,7 +155,7 @@ describe("schemaBasedEncoding", () => { { kind: FieldKinds.sequence.identifier, types: new Set([brand(Minimal.identifier)]), - persistedMetadata: undefined, + metadata: undefined, }, cache, { nodeSchema: new Map() }, @@ -186,7 +186,7 @@ describe("schemaBasedEncoding", () => { const storedSchema: TreeFieldStoredSchema = { kind: FieldKinds.identifier.identifier, types: new Set([brand(stringSchema.identifier)]), - persistedMetadata: undefined, + metadata: undefined, }; const shape = fieldShaper( diff --git a/packages/dds/tree/src/test/feature-libraries/default-schema/schemaChecker.spec.ts b/packages/dds/tree/src/test/feature-libraries/default-schema/schemaChecker.spec.ts index e987d4f46d18..9a48f572b83b 100644 --- a/packages/dds/tree/src/test/feature-libraries/default-schema/schemaChecker.spec.ts +++ b/packages/dds/tree/src/test/feature-libraries/default-schema/schemaChecker.spec.ts @@ -71,7 +71,7 @@ function getFieldSchema( return { kind: kind.identifier, types: new Set(allowedTypes), - persistedMetadata: undefined, + metadata: undefined, }; } diff --git a/packages/dds/tree/src/test/feature-libraries/flex-tree/lazyField.spec.ts b/packages/dds/tree/src/test/feature-libraries/flex-tree/lazyField.spec.ts index 27fce830ad65..fa168f59715e 100644 --- a/packages/dds/tree/src/test/feature-libraries/flex-tree/lazyField.spec.ts +++ b/packages/dds/tree/src/test/feature-libraries/flex-tree/lazyField.spec.ts @@ -444,7 +444,7 @@ describe("LazyField", () => { const rootSchema: TreeFieldStoredSchema = { kind: FieldKinds.sequence.identifier, types: new Set([brand(numberSchema.identifier)]), - persistedMetadata: undefined, + metadata: undefined, }; const schema: TreeStoredSchema = { rootFieldSchema: rootSchema, diff --git a/packages/dds/tree/src/test/feature-libraries/modular-schema/comparison.spec.ts b/packages/dds/tree/src/test/feature-libraries/modular-schema/comparison.spec.ts index 9b19dbd19957..2aae6904eb2b 100644 --- a/packages/dds/tree/src/test/feature-libraries/modular-schema/comparison.spec.ts +++ b/packages/dds/tree/src/test/feature-libraries/modular-schema/comparison.spec.ts @@ -46,7 +46,7 @@ export function fieldSchema( return { kind: kind.identifier, types: new Set(types), - persistedMetadata: undefined, + metadata: undefined, }; } diff --git a/packages/dds/tree/src/test/feature-libraries/modular-schema/isNeverTree.spec.ts b/packages/dds/tree/src/test/feature-libraries/modular-schema/isNeverTree.spec.ts index 36f9de499674..c015027a8827 100644 --- a/packages/dds/tree/src/test/feature-libraries/modular-schema/isNeverTree.spec.ts +++ b/packages/dds/tree/src/test/feature-libraries/modular-schema/isNeverTree.spec.ts @@ -42,7 +42,7 @@ function fieldSchema( return { kind: kind.identifier, types: new Set(types), - persistedMetadata: undefined, + metadata: undefined, }; } diff --git a/packages/dds/tree/src/test/sequenceRootUtils.ts b/packages/dds/tree/src/test/sequenceRootUtils.ts index cb949a29be78..1e29a3f3449a 100644 --- a/packages/dds/tree/src/test/sequenceRootUtils.ts +++ b/packages/dds/tree/src/test/sequenceRootUtils.ts @@ -31,7 +31,7 @@ export const jsonSequenceRootSchema: TreeStoredSchema = { brand(s.identifier), ), ), - persistedMetadata: undefined, + metadata: undefined, }, }; diff --git a/packages/dds/tree/src/test/shared-tree/sharedTreeChangeFamily.spec.ts b/packages/dds/tree/src/test/shared-tree/sharedTreeChangeFamily.spec.ts index 7acdb2c34b4c..d376ac70c6e6 100644 --- a/packages/dds/tree/src/test/shared-tree/sharedTreeChangeFamily.spec.ts +++ b/packages/dds/tree/src/test/shared-tree/sharedTreeChangeFamily.spec.ts @@ -75,7 +75,7 @@ const emptySchema: TreeStoredSchema = { rootFieldSchema: { kind: forbidden.identifier, types: new Set(), - persistedMetadata: undefined, + metadata: undefined, }, }; const stSchemaChange: SharedTreeChange = { diff --git a/packages/dds/tree/src/test/simple-tree/prepareForInsertion.spec.ts b/packages/dds/tree/src/test/simple-tree/prepareForInsertion.spec.ts index f42dce92b69f..f486a8f5db54 100644 --- a/packages/dds/tree/src/test/simple-tree/prepareForInsertion.spec.ts +++ b/packages/dds/tree/src/test/simple-tree/prepareForInsertion.spec.ts @@ -135,7 +135,7 @@ describe("prepareForInsertion", () => { return { kind: kind.identifier, types: new Set(allowedTypes), - persistedMetadata: undefined, + metadata: undefined, }; } diff --git a/packages/dds/tree/src/test/simple-tree/toMapTree.spec.ts b/packages/dds/tree/src/test/simple-tree/toMapTree.spec.ts index 0560f479c38b..f2bed820eff1 100644 --- a/packages/dds/tree/src/test/simple-tree/toMapTree.spec.ts +++ b/packages/dds/tree/src/test/simple-tree/toMapTree.spec.ts @@ -63,7 +63,7 @@ function getFieldSchema( return { kind: kind.identifier, types: new Set(allowedTypes), - persistedMetadata: undefined, + metadata: undefined, }; } diff --git a/packages/dds/tree/src/test/testTrees.ts b/packages/dds/tree/src/test/testTrees.ts index 3a714f8dd01d..4ba392c1b3c4 100644 --- a/packages/dds/tree/src/test/testTrees.ts +++ b/packages/dds/tree/src/test/testTrees.ts @@ -180,7 +180,7 @@ export const allTheFields = new ObjectNodeStoredSchema( { kind: FieldKinds.optional.identifier, types: numberSet, - persistedMetadata: undefined, + metadata: undefined, }, ], [ @@ -188,7 +188,7 @@ export const allTheFields = new ObjectNodeStoredSchema( { kind: FieldKinds.required.identifier, types: numberSet, - persistedMetadata: undefined, + metadata: undefined, }, ], [ @@ -196,7 +196,7 @@ export const allTheFields = new ObjectNodeStoredSchema( { kind: FieldKinds.sequence.identifier, types: numberSet, - persistedMetadata: undefined, + metadata: undefined, }, ], ]), @@ -276,7 +276,7 @@ export const testTrees: readonly TestTree[] = [ rootFieldSchema: { kind: FieldKinds.sequence.identifier, types: numberSet, - persistedMetadata: undefined, + metadata: undefined, }, }, jsonableTreesFromFieldCursor(fieldJsonCursor([1, 2, 3])), @@ -309,7 +309,7 @@ export const testTrees: readonly TestTree[] = [ rootFieldSchema: { kind: FieldKinds.required.identifier, types: new Set([allTheFieldsName]), - persistedMetadata: undefined, + metadata: undefined, }, }, [ @@ -326,7 +326,7 @@ export const testTrees: readonly TestTree[] = [ rootFieldSchema: { kind: FieldKinds.required.identifier, types: new Set([allTheFieldsName]), - persistedMetadata: undefined, + metadata: undefined, }, }, [ From 85240dca461f3b0fc9fcf5d1975255e907edf6c2 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Tue, 27 May 2025 11:02:39 -0700 Subject: [PATCH 29/45] Apply suggestions from code review Co-authored-by: Craig Macomber (Microsoft) <42876482+CraigMacomber@users.noreply.github.com> --- .changeset/brown-dingos-switch.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.changeset/brown-dingos-switch.md b/.changeset/brown-dingos-switch.md index f32abc1605c6..5b78c39849a6 100644 --- a/.changeset/brown-dingos-switch.md +++ b/.changeset/brown-dingos-switch.md @@ -1,7 +1,8 @@ --- "@fluidframework/tree": minor -"__section": feature +"fluid-framework": minor +"__section": tree --- -Added a v2 Shared Tree schema with support for persisted metadata. +Added a v2 Shared Tree schema with support for persisted metadata This change adds support for persisted metadata, but does not enable it by default. \ No newline at end of file From 8883d450276ab42719631874e37791d2d90a5a3c Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Wed, 28 May 2025 15:28:24 -0700 Subject: [PATCH 30/45] - Updated the changeset description - Moved JsonCompatibleReadOnly and JsonCompatibleReadOnlyObject to alpha API status so they can be used in schemas - Implemented field factory methods and added test coverage - Added requiredRecursive to schemaFactoryAlpha (not sure if it should be part of this change though) --- .changeset/brown-dingos-switch.md | 4 +- .../dds/tree/api-report/tree.alpha.api.md | 51 ++++++++----- packages/dds/tree/api-report/tree.beta.api.md | 10 --- .../tree/api-report/tree.legacy.alpha.api.md | 10 --- .../tree/api-report/tree.legacy.public.api.md | 10 --- .../dds/tree/api-report/tree.public.api.md | 10 --- .../schema-edits/schemaChangeCodecs.ts | 2 +- packages/dds/tree/src/index.ts | 2 + .../tree/src/simple-tree/api/schemaFactory.ts | 11 +-- .../src/simple-tree/api/schemaFactoryAlpha.ts | 71 +++++++++++++++++-- packages/dds/tree/src/simple-tree/index.ts | 2 + .../dds/tree/src/simple-tree/schemaTypes.ts | 55 +++++++++++--- .../simple-tree/api/schemaFactory.spec.ts | 28 ++++++-- packages/dds/tree/src/util/utils.ts | 4 +- .../api-report/fluid-framework.alpha.api.md | 8 +++ 15 files changed, 193 insertions(+), 85 deletions(-) diff --git a/.changeset/brown-dingos-switch.md b/.changeset/brown-dingos-switch.md index 5b78c39849a6..a63784d878e0 100644 --- a/.changeset/brown-dingos-switch.md +++ b/.changeset/brown-dingos-switch.md @@ -3,6 +3,6 @@ "fluid-framework": minor "__section": tree --- -Added a v2 Shared Tree schema with support for persisted metadata +Add APIs for declaring "persisted" schema metadata -This change adds support for persisted metadata, but does not enable it by default. \ No newline at end of file +Add alpha APIs for declaring node and field schema metadata which future versions of the Fluid Framework will provide a way to opt into persisting in the document. diff --git a/packages/dds/tree/api-report/tree.alpha.api.md b/packages/dds/tree/api-report/tree.alpha.api.md index 74c20530f1ad..54d302d5af26 100644 --- a/packages/dds/tree/api-report/tree.alpha.api.md +++ b/packages/dds/tree/api-report/tree.alpha.api.md @@ -181,6 +181,11 @@ export interface FieldProps { readonly metadata?: FieldSchemaMetadata; } +// @alpha +export interface FieldPropsAlpha extends FieldProps { + readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; +} + // @public @sealed export class FieldSchema { protected constructor( @@ -216,7 +221,6 @@ export interface FieldSchemaAlphaUnsafe { readonly custom?: TCustomMetadata; readonly description?: string | undefined; - readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } // @alpha @@ -420,9 +424,9 @@ export namespace JsonAsTree { } const // @system _APIExtractorWorkaroundObjectBase: TreeNodeSchemaClass<"com.fluidframework.json.object", NodeKind.Map, System_Unsafe.TreeMapNodeUnsafe, LeafSchema<"number", number>, LeafSchema<"string", string>, LeafSchema<"boolean", boolean>, () => typeof JsonObject, () => typeof Array]> & WithType<"com.fluidframework.json.object", NodeKind.Map, unknown>, { - [Symbol.iterator](): Iterator<[string, string | number | JsonObject | Array | System_Unsafe.InsertableTypedNodeUnsafe, LeafSchema<"boolean", boolean>> | null], any, undefined>; + [Symbol.iterator](): Iterator<[string, string | number | System_Unsafe.InsertableTypedNodeUnsafe, LeafSchema<"boolean", boolean>> | JsonObject | Array | null], any, undefined>; } | { - readonly [x: string]: string | number | JsonObject | Array | System_Unsafe.InsertableTypedNodeUnsafe, LeafSchema<"boolean", boolean>> | null; + readonly [x: string]: string | number | System_Unsafe.InsertableTypedNodeUnsafe, LeafSchema<"boolean", boolean>> | JsonObject | Array | null; }, false, readonly [LeafSchema<"null", null>, LeafSchema<"number", number>, LeafSchema<"string", string>, LeafSchema<"boolean", boolean>, () => typeof JsonObject, () => typeof Array], undefined>; // (undocumented) export type Primitive = TreeNodeFromImplicitAllowedTypes; @@ -430,7 +434,7 @@ export namespace JsonAsTree { export type _RecursiveArrayWorkaroundJsonArray = FixRecursiveArraySchema; const // @system _APIExtractorWorkaroundArrayBase: TreeNodeSchemaClass<"com.fluidframework.json.array", NodeKind.Array, System_Unsafe.TreeArrayNodeUnsafe, LeafSchema<"number", number>, LeafSchema<"string", string>, LeafSchema<"boolean", boolean>, () => typeof JsonObject, () => typeof Array]> & WithType<"com.fluidframework.json.array", NodeKind.Array, unknown>, { - [Symbol.iterator](): Iterator, LeafSchema<"boolean", boolean>> | null, any, undefined>; + [Symbol.iterator](): Iterator, LeafSchema<"boolean", boolean>> | JsonObject | Array | null, any, undefined>; }, false, readonly [LeafSchema<"null", null>, LeafSchema<"number", number>, LeafSchema<"string", string>, LeafSchema<"boolean", boolean>, () => typeof JsonObject, () => typeof Array], undefined>; // (undocumented) export type Tree = TreeNodeFromImplicitAllowedTypes; @@ -444,10 +448,10 @@ export type JsonCompatibleObject = { [P in string]?: JsonCompatible; }; -// @public +// @alpha export type JsonCompatibleReadOnly = string | number | boolean | null | readonly JsonCompatibleReadOnly[] | JsonCompatibleReadOnlyObject; -// @public +// @alpha export type JsonCompatibleReadOnlyObject = { readonly [P in string]?: JsonCompatibleReadOnly; }; @@ -593,7 +597,6 @@ export enum NodeKind { export interface NodeSchemaMetadata { readonly custom?: TCustomMetadata | undefined; readonly description?: string | undefined; - readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } // @public @sealed @@ -601,6 +604,11 @@ export interface NodeSchemaOptions { readonly metadata?: NodeSchemaMetadata | undefined; } +// @alpha @sealed +export interface NodeSchemaOptionsAlpha extends NodeSchemaOptions { + readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; +} + // @alpha export const noopValidator: JsonValidator; @@ -768,7 +776,7 @@ export class SchemaFactory extends SchemaFactory { arrayAlpha(name: Name, allowedTypes: T, options?: NodeSchemaOptions): ArrayNodeCustomizableSchema, T, true, TCustomMetadata>; arrayRecursive(name: Name, allowedTypes: T, options?: NodeSchemaOptions): ArrayNodeCustomizableSchemaUnsafe, T, TCustomMetadata>; - static readonly identifier: (props?: Omit, "defaultProvider"> | undefined) => FieldSchemaAlpha_2 & SimpleLeafNodeSchema_2, TCustomMetadata>; + static readonly identifier: (props?: Omit, "defaultProvider"> | undefined) => FieldSchemaAlpha & SimpleLeafNodeSchema_2, TCustomMetadata>; static readonly leaves: readonly [LeafSchema_2<"string", string> & SimpleLeafNodeSchema_2, LeafSchema_2<"number", number> & SimpleLeafNodeSchema_2, LeafSchema_2<"boolean", boolean> & SimpleLeafNodeSchema_2, LeafSchema_2<"null", null> & SimpleLeafNodeSchema_2, LeafSchema_2<"handle", IFluidHandle> & SimpleLeafNodeSchema_2]; mapAlpha(name: Name, allowedTypes: T, options?: NodeSchemaOptions): MapNodeCustomizableSchema, T, true, TCustomMetadata>; mapRecursive(name: Name, allowedTypes: T, options?: NodeSchemaOptions): MapNodeCustomizableSchemaUnsafe, T, TCustomMetadata>; @@ -777,19 +785,28 @@ export class SchemaFactoryAlpha, const TCustomMetadata = unknown>(name: Name, t: T, options?: SchemaFactoryObjectOptions): TreeNodeSchemaClass, NodeKind.Object, System_Unsafe.TreeObjectNodeUnsafe>, object & System_Unsafe.InsertableObjectFromSchemaRecordUnsafe, false, T, never, TCustomMetadata> & SimpleObjectNodeSchema & Pick; static readonly optional: { - (t: T, props?: Omit, "defaultProvider"> | undefined): FieldSchemaAlpha_2; - (t: T_1, props?: Omit, "defaultProvider"> | undefined): FieldSchemaAlpha_2, TCustomMetadata_1>; + (t: T, props?: Omit, "defaultProvider"> | undefined): FieldSchemaAlpha; + (t: T_1, props?: Omit, "defaultProvider"> | undefined): FieldSchemaAlpha, TCustomMetadata_1>; }; - static readonly optionalRecursive: (t: T, props?: Omit, "defaultProvider"> | undefined) => FieldSchemaAlphaUnsafe_2; + // (undocumented) + optionalAlpha(t: T, props?: Omit, "defaultProvider">): FieldSchemaAlpha, TCustomMetadata>; + static readonly optionalRecursive: (t: T, props?: Omit, "defaultProvider"> | undefined) => FieldSchemaAlphaUnsafe; + // (undocumented) + optionalRecursiveAlpha(t: T, props?: Omit, "defaultProvider">): FieldSchemaAlphaUnsafe; static readonly required: { - (t: T, props?: Omit, "defaultProvider"> | undefined): FieldSchemaAlpha_2; - (t: T_1, props?: Omit, "defaultProvider"> | undefined): FieldSchemaAlpha_2, TCustomMetadata_1>; + (t: T, props?: Omit, "defaultProvider"> | undefined): FieldSchemaAlpha; + (t: T_1, props?: Omit, "defaultProvider"> | undefined): FieldSchemaAlpha, TCustomMetadata_1>; }; + // (undocumented) + requiredAlpha(t: T, props?: Omit, "defaultProvider">): FieldSchemaAlpha, TCustomMetadata>; + static readonly requiredRecursive: (t: T, props?: Omit, "defaultProvider"> | undefined) => FieldSchemaAlphaUnsafe; + // (undocumented) + requiredRecursiveAlpha(t: T, props?: Omit, "defaultProvider">): FieldSchemaAlphaUnsafe; scopedFactory(name: T): SchemaFactoryAlpha, TNameInner>; } // @alpha -export interface SchemaFactoryObjectOptions extends NodeSchemaOptions { +export interface SchemaFactoryObjectOptions extends NodeSchemaOptionsAlpha { allowUnknownOptionalFields?: boolean; } @@ -910,7 +927,7 @@ export namespace System_TableSchema { props: InsertableTreeFieldFromImplicitField>; }), true, { readonly props: TPropsSchema; - readonly id: FieldSchema_2, unknown>; + readonly id: FieldSchema_2, unknown>; }>; // @system export type CreateRowOptionsBase = OptionsWithSchemaFactory & OptionsWithCellSchema; @@ -924,8 +941,8 @@ export namespace System_TableSchema { props: InsertableTreeFieldFromImplicitField>; }), true, { readonly props: TPropsSchema; - readonly id: FieldSchema_2, unknown>; - readonly cells: FieldSchema_2, "Row.cells">, NodeKind.Map, TreeMapNode_2 & WithType, "Row.cells">, NodeKind.Map, unknown>, MapNodeInsertableData_2, true, TCellSchema, undefined>, unknown>; + readonly id: FieldSchema_2, unknown>; + readonly cells: FieldSchema_2, "Row.cells">, NodeKind.Map, TreeMapNode_2 & WithType, "Row.cells">, NodeKind.Map, unknown>, MapNodeInsertableData_2, true, TCellSchema, undefined>, unknown>; }>; // @system export function createTableSchema, const TRowSchema extends RowSchemaBase>(inputSchemaFactory: SchemaFactoryAlpha, _cellSchema: TCellSchema, columnSchema: TColumnSchema, rowSchema: TRowSchema): TreeNodeSchemaCore_2, "Table">, NodeKind.Object, true, { diff --git a/packages/dds/tree/api-report/tree.beta.api.md b/packages/dds/tree/api-report/tree.beta.api.md index 6b7687be09e4..7534cd977477 100644 --- a/packages/dds/tree/api-report/tree.beta.api.md +++ b/packages/dds/tree/api-report/tree.beta.api.md @@ -77,7 +77,6 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; - readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } // @public @system @@ -177,14 +176,6 @@ export interface ITreeViewConfiguration = Item | (() => Item); @@ -236,7 +227,6 @@ export enum NodeKind { export interface NodeSchemaMetadata { readonly custom?: TCustomMetadata | undefined; readonly description?: string | undefined; - readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } // @public @sealed diff --git a/packages/dds/tree/api-report/tree.legacy.alpha.api.md b/packages/dds/tree/api-report/tree.legacy.alpha.api.md index 3280f9460b98..7aabd1819bbb 100644 --- a/packages/dds/tree/api-report/tree.legacy.alpha.api.md +++ b/packages/dds/tree/api-report/tree.legacy.alpha.api.md @@ -77,7 +77,6 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; - readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } // @public @system @@ -177,14 +176,6 @@ export interface ITreeViewConfiguration = Item | (() => Item); @@ -231,7 +222,6 @@ export enum NodeKind { export interface NodeSchemaMetadata { readonly custom?: TCustomMetadata | undefined; readonly description?: string | undefined; - readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } // @public @sealed diff --git a/packages/dds/tree/api-report/tree.legacy.public.api.md b/packages/dds/tree/api-report/tree.legacy.public.api.md index b81cda67fbc6..68b1d8e7b024 100644 --- a/packages/dds/tree/api-report/tree.legacy.public.api.md +++ b/packages/dds/tree/api-report/tree.legacy.public.api.md @@ -77,7 +77,6 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; - readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } // @public @system @@ -177,14 +176,6 @@ export interface ITreeViewConfiguration = Item | (() => Item); @@ -231,7 +222,6 @@ export enum NodeKind { export interface NodeSchemaMetadata { readonly custom?: TCustomMetadata | undefined; readonly description?: string | undefined; - readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } // @public @sealed diff --git a/packages/dds/tree/api-report/tree.public.api.md b/packages/dds/tree/api-report/tree.public.api.md index b81cda67fbc6..68b1d8e7b024 100644 --- a/packages/dds/tree/api-report/tree.public.api.md +++ b/packages/dds/tree/api-report/tree.public.api.md @@ -77,7 +77,6 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; - readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } // @public @system @@ -177,14 +176,6 @@ export interface ITreeViewConfiguration = Item | (() => Item); @@ -231,7 +222,6 @@ export enum NodeKind { export interface NodeSchemaMetadata { readonly custom?: TCustomMetadata | undefined; readonly description?: string | undefined; - readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } // @public @sealed diff --git a/packages/dds/tree/src/feature-libraries/schema-edits/schemaChangeCodecs.ts b/packages/dds/tree/src/feature-libraries/schema-edits/schemaChangeCodecs.ts index db58e3ee8002..34ce95493c34 100644 --- a/packages/dds/tree/src/feature-libraries/schema-edits/schemaChangeCodecs.ts +++ b/packages/dds/tree/src/feature-libraries/schema-edits/schemaChangeCodecs.ts @@ -46,7 +46,7 @@ export function makeSchemaChangeCodec( } /** - * Compose the v1 schema change codec. + * Compose the change codec using mostly v1 logic. * @param options - The codec options. * @param schemaWriteVersion - The schema write version. * @returns The composed schema change codec. diff --git a/packages/dds/tree/src/index.ts b/packages/dds/tree/src/index.ts index f3ecf8363061..27060ac802b2 100644 --- a/packages/dds/tree/src/index.ts +++ b/packages/dds/tree/src/index.ts @@ -113,6 +113,7 @@ export { type TreeNodeSchemaClass, type SchemaCompatibilityStatus, type FieldProps, + type FieldPropsAlpha, type InternalTreeNode, type WithType, type NodeChangedData, @@ -219,6 +220,7 @@ export { type TreeBranchEvents, asTreeViewAlpha, type NodeSchemaOptions, + type NodeSchemaOptionsAlpha, type NodeSchemaMetadata, type SchemaStatics, type ITreeAlpha, diff --git a/packages/dds/tree/src/simple-tree/api/schemaFactory.ts b/packages/dds/tree/src/simple-tree/api/schemaFactory.ts index 2b912a8ce36e..5a3e6583284a 100644 --- a/packages/dds/tree/src/simple-tree/api/schemaFactory.ts +++ b/packages/dds/tree/src/simple-tree/api/schemaFactory.ts @@ -36,12 +36,12 @@ import { createFieldSchema, type DefaultProvider, getDefaultProvider, - type NodeSchemaOptions, markSchemaMostDerived, type FieldSchemaAlpha, type ImplicitAnnotatedAllowedTypes, type UnannotateImplicitAllowedTypes, type UnannotateSchemaRecord, + type NodeSchemaOptionsAlpha, } from "../schemaTypes.js"; import type { NodeKind, @@ -91,7 +91,7 @@ export function schemaFromValue(value: TreeValue): TreeNodeSchema { * @alpha */ export interface SchemaFactoryObjectOptions - extends NodeSchemaOptions { + extends NodeSchemaOptionsAlpha { /** * Allow nodes typed with this object node schema to contain optional fields that are not present in the schema declaration. * Such nodes can come into existence either via import APIs (see remarks) or by way of collaboration with another client @@ -137,8 +137,11 @@ export interface SchemaFactoryObjectOptions allowUnknownOptionalFields?: boolean; } +/** + * Omit parameters that are not relevant for common use cases. + */ export const defaultSchemaFactoryObjectOptions: Required< - Omit + Omit > = { allowUnknownOptionalFields: false, }; @@ -290,7 +293,7 @@ export interface SchemaStatics { ) => System_Unsafe.FieldSchemaUnsafe; } -const defaultOptionalProvider: DefaultProvider = getDefaultProvider(() => { +export const defaultOptionalProvider: DefaultProvider = getDefaultProvider(() => { return undefined; }); diff --git a/packages/dds/tree/src/simple-tree/api/schemaFactoryAlpha.ts b/packages/dds/tree/src/simple-tree/api/schemaFactoryAlpha.ts index 0eb6057fbd11..3973608fb8f8 100644 --- a/packages/dds/tree/src/simple-tree/api/schemaFactoryAlpha.ts +++ b/packages/dds/tree/src/simple-tree/api/schemaFactoryAlpha.ts @@ -4,24 +4,31 @@ */ import { + defaultOptionalProvider, defaultSchemaFactoryObjectOptions, SchemaFactory, schemaStatics, type SchemaFactoryObjectOptions, type ScopedSchemaName, } from "./schemaFactory.js"; -import type { - ImplicitAllowedTypes, - ImplicitAnnotatedAllowedTypes, - ImplicitAnnotatedFieldSchema, - ImplicitFieldSchema, - NodeSchemaOptions, +import { + createFieldSchema, + FieldKind, + type FieldPropsAlpha, + type FieldSchemaAlpha, + type ImplicitAllowedTypes, + type ImplicitAnnotatedAllowedTypes, + type ImplicitAnnotatedFieldSchema, + type ImplicitFieldSchema, + type NodeSchemaOptions, + type UnannotateImplicitAllowedTypes, } from "../schemaTypes.js"; import { objectSchema } from "../objectNode.js"; import type { RestrictiveStringRecord } from "../../util/index.js"; import type { NodeKind, TreeNodeSchemaClass } from "../core/index.js"; import type { ArrayNodeCustomizableSchemaUnsafe, + FieldSchemaAlphaUnsafe, MapNodeCustomizableSchemaUnsafe, System_Unsafe, } from "./typesUnsafe.js"; @@ -31,6 +38,7 @@ import type { ObjectNodeSchema } from "../objectNodeTypes.js"; import type { SimpleObjectNodeSchema } from "../simpleSchema.js"; import type { ArrayNodeCustomizableSchema } from "../arrayNodeTypes.js"; import type { MapNodeCustomizableSchema } from "../mapNodeTypes.js"; +import { createFieldSchemaUnsafe } from "./schemaFactoryRecursive.js"; /** * {@link SchemaFactory} with additional alpha APIs. @@ -155,16 +163,67 @@ export class SchemaFactoryAlpha< */ public static override readonly optional = schemaStatics.optional; + public optionalAlpha< + const T extends ImplicitAnnotatedAllowedTypes, + const TCustomMetadata = unknown, + >( + t: T, + props?: Omit, "defaultProvider">, + ): FieldSchemaAlpha, TCustomMetadata> { + return createFieldSchema(FieldKind.Optional, t, { + defaultProvider: defaultOptionalProvider, + ...props, + }); + } + /** * {@inheritDoc SchemaStatics.required} */ public static override readonly required = schemaStatics.required; + public requiredAlpha< + const T extends ImplicitAnnotatedAllowedTypes, + const TCustomMetadata = unknown, + >( + t: T, + props?: Omit, "defaultProvider">, + ): FieldSchemaAlpha, TCustomMetadata> { + return createFieldSchema(FieldKind.Required, t, props); + } + /** * {@inheritDoc SchemaStatics.optionalRecursive} */ public static override readonly optionalRecursive = schemaStatics.optionalRecursive; + public optionalRecursiveAlpha< + const T extends System_Unsafe.ImplicitAllowedTypesUnsafe, + const TCustomMetadata = unknown, + >( + t: T, + props?: Omit, "defaultProvider">, + ): FieldSchemaAlphaUnsafe { + return createFieldSchemaUnsafe(FieldKind.Optional, t, { + defaultProvider: defaultOptionalProvider, + ...props, + }); + } + + /** + * {@inheritDoc SchemaStatics.optionalRecursive} + */ + public static override readonly requiredRecursive = schemaStatics.requiredRecursive; + + public requiredRecursiveAlpha< + const T extends System_Unsafe.ImplicitAllowedTypesUnsafe, + const TCustomMetadata = unknown, + >( + t: T, + props?: Omit, "defaultProvider">, + ): FieldSchemaAlphaUnsafe { + return createFieldSchemaUnsafe(FieldKind.Required, t, props); + } + /** * Like {@link SchemaFactory.identifier} but static and a factory function that can be provided {@link FieldProps}. */ diff --git a/packages/dds/tree/src/simple-tree/index.ts b/packages/dds/tree/src/simple-tree/index.ts index efde745fc981..3b0c97f0c6ce 100644 --- a/packages/dds/tree/src/simple-tree/index.ts +++ b/packages/dds/tree/src/simple-tree/index.ts @@ -169,6 +169,7 @@ export { type NodeBuilderData, type DefaultProvider, type FieldProps, + type FieldPropsAlpha, normalizeFieldSchema, areFieldSchemaEqual, areImplicitFieldSchemaEqual, @@ -184,6 +185,7 @@ export { type ReadableField, type ReadSchema, type NodeSchemaOptions, + type NodeSchemaOptionsAlpha, type NodeSchemaMetadata, evaluateLazySchema, } from "./schemaTypes.js"; diff --git a/packages/dds/tree/src/simple-tree/schemaTypes.ts b/packages/dds/tree/src/simple-tree/schemaTypes.ts index d0cde25c4ed7..69e3f6f33c76 100644 --- a/packages/dds/tree/src/simple-tree/schemaTypes.ts +++ b/packages/dds/tree/src/simple-tree/schemaTypes.ts @@ -295,6 +295,22 @@ export interface FieldProps { readonly metadata?: FieldSchemaMetadata; } +/** + * Additional information to provide to a {@link FieldSchema}. Includes fields for alpha features. + * + * @typeParam TCustomMetadata - Custom metadata properties to associate with the field. + * See {@link FieldSchemaMetadata.custom}. + * + * @alpha + */ +export interface FieldPropsAlpha + extends FieldProps { + /** + * The persisted metadata for this schema element. + */ + readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; +} + /** * A {@link FieldProvider} which requires additional context in order to produce its content */ @@ -358,7 +374,18 @@ export interface FieldSchemaMetadata { * used as the `description` field. */ readonly description?: string | undefined; +} +/** + * Metadata associated with a {@link FieldSchema}. Includes fields used by alpha features. + * + * @remarks Specified via {@link FieldProps.metadata}. + * + * @sealed + * @alpha + */ +export interface FieldSchemaMetadataAlpha + extends FieldSchemaMetadata { /** * The persisted metadata for this schema element. */ @@ -779,12 +806,12 @@ function areFieldPropsEqual(a: FieldProps | undefined, b: FieldProps | undefined * @remarks FieldSchemaMetadata are considered equivalent if their custom data and descriptions are (respectively) reference equal. */ function areMetadataEqual( - a: FieldSchemaMetadata | undefined, - b: FieldSchemaMetadata | undefined, + a: FieldSchemaMetadataAlpha | undefined, + b: FieldSchemaMetadataAlpha | undefined, ): boolean { // If any new fields are added to FieldSchemaMetadata, this check will stop compiling as a reminder that this function needs to be updated. type _keys = requireTrue< - areOnlyKeys + areOnlyKeys >; if (a === b) { @@ -1323,6 +1350,23 @@ export interface NodeSchemaOptions { readonly metadata?: NodeSchemaMetadata | undefined; } +/** + * Additional information to provide to Node Schema creation. Includes fields for alpha features. + * + * @typeParam TCustomMetadata - Custom metadata properties to associate with the Node Schema. + * See {@link NodeSchemaMetadata.custom}. + * + * @sealed + * @alpha + */ +export interface NodeSchemaOptionsAlpha + extends NodeSchemaOptions { + /** + * The persisted metadata for this schema element. + */ + readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; +} + /** * Metadata associated with a Node Schema. * @@ -1347,9 +1391,4 @@ export interface NodeSchemaMetadata { * used as the `description` property. */ readonly description?: string | undefined; - - /** - * The persisted metadata for this schema element. - */ - readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; } diff --git a/packages/dds/tree/src/test/simple-tree/api/schemaFactory.spec.ts b/packages/dds/tree/src/test/simple-tree/api/schemaFactory.spec.ts index 779a8ecfe9e3..6a74500655ef 100644 --- a/packages/dds/tree/src/test/simple-tree/api/schemaFactory.spec.ts +++ b/packages/dds/tree/src/test/simple-tree/api/schemaFactory.spec.ts @@ -424,7 +424,7 @@ describe("schemaFactory", () => { class Foo extends factory.objectAlpha( "Foo", { bar: factory.number }, - { metadata: fooMetadata }, + { persistedMetadata: fooMetadata }, ) {} assert.deepEqual(Foo.metadata, fooMetadata); @@ -435,16 +435,34 @@ describe("schemaFactory", () => { }); it("Field schema persisted metadata", () => { - const schemaFactory = new SchemaFactory("com.example"); + const schemaFactory = new SchemaFactoryAlpha("com.example"); const fooMetadata = { persistedMetadata: { "a": 2 }, }; - class Foo extends schemaFactory.object("Foo", { - bar: schemaFactory.required(schemaFactory.number, { metadata: fooMetadata }), + class Foo extends schemaFactory.objectAlpha("Foo", { + bar: schemaFactory.requiredAlpha(schemaFactory.number, { + persistedMetadata: fooMetadata, + }), + baz: schemaFactory.optionalAlpha(schemaFactory.string, { + persistedMetadata: fooMetadata, + }), + buzz: schemaFactory.requiredRecursiveAlpha( + schemaFactory.objectAlpha("Buzz", { qux: schemaFactory.number }), + { persistedMetadata: fooMetadata }, + ), + qux: schemaFactory.optionalRecursiveAlpha( + schemaFactory.objectAlpha("Qux", { quux: schemaFactory.string }), + { persistedMetadata: fooMetadata }, + ), }) {} - const foo = hydrate(Foo, { bar: 37 }); + const foo = hydrate(Foo, { + bar: 37, + baz: "test", + buzz: { qux: 42 }, + qux: { quux: "test" }, + }); const schema = Tree.schema(foo) as ObjectNodeSchema; // eslint-disable-next-line @typescript-eslint/no-non-null-assertion diff --git a/packages/dds/tree/src/util/utils.ts b/packages/dds/tree/src/util/utils.ts index b1f96e90e63a..7cbb68bbbddf 100644 --- a/packages/dds/tree/src/util/utils.ts +++ b/packages/dds/tree/src/util/utils.ts @@ -310,7 +310,7 @@ export type JsonCompatibleObject = { [P in string]?: JsonCompati * @remarks * This does not robustly forbid non json comparable data via type checking, * but instead mostly restricts access to it. - * @public + * @alpha */ export type JsonCompatibleReadOnly = | string @@ -326,7 +326,7 @@ export type JsonCompatibleReadOnly = * @remarks * This does not robustly forbid non json comparable data via type checking, * but instead mostly restricts access to it. - * @public + * @alpha */ export type JsonCompatibleReadOnlyObject = { readonly [P in string]?: JsonCompatibleReadOnly }; diff --git a/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md b/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md index a4d703ea634a..c19b8eb1d416 100644 --- a/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md +++ b/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md @@ -782,6 +782,14 @@ export type JsonCompatibleObject = { [P in string]?: JsonCompatible; }; +// @alpha +export type JsonCompatibleReadOnly = string | number | boolean | null | readonly JsonCompatibleReadOnly[] | JsonCompatibleReadOnlyObject; + +// @alpha +export type JsonCompatibleReadOnlyObject = { + readonly [P in string]?: JsonCompatibleReadOnly; +}; + // @alpha @sealed export type JsonFieldSchema = { readonly description?: string | undefined; From 9c2c13f0e3310d342f252e1d8fad37a6fa92ae43 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Wed, 28 May 2025 15:35:38 -0700 Subject: [PATCH 31/45] Reverted unnecessary change. --- packages/dds/tree/src/test/simple-tree/api/storedSchema.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dds/tree/src/test/simple-tree/api/storedSchema.spec.ts b/packages/dds/tree/src/test/simple-tree/api/storedSchema.spec.ts index 604e8013a177..4d89f6f5af4d 100644 --- a/packages/dds/tree/src/test/simple-tree/api/storedSchema.spec.ts +++ b/packages/dds/tree/src/test/simple-tree/api/storedSchema.spec.ts @@ -13,8 +13,8 @@ import { import { testSimpleTrees } from "../../testTrees.js"; import { takeJsonSnapshot, useSnapshotDirectory } from "../../snapshots/index.js"; import { typeboxValidator } from "../../../external-utilities/index.js"; -import { TreeViewConfigurationAlpha } from "../../../simple-tree/index.js"; import { FluidClientVersion } from "../../../codec/index.js"; +import { TreeViewConfigurationAlpha } from "../../../simple-tree/index.js"; describe("simple-tree storedSchema", () => { describe("test-schema", () => { From c62f7ec0f61af676eb65490cc89b0ee8682e98c1 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Thu, 29 May 2025 13:46:04 -0700 Subject: [PATCH 32/45] Wired up node schema metadata persistence. Currently errors on tests involving FormatV2. --- .../dds/tree/src/core/schema-stored/schema.ts | 75 +++++++++++++++---- .../feature-libraries/schema-index/codec.ts | 10 ++- .../tree/src/simple-tree/toStoredSchema.ts | 17 +++-- 3 files changed, 78 insertions(+), 24 deletions(-) diff --git a/packages/dds/tree/src/core/schema-stored/schema.ts b/packages/dds/tree/src/core/schema-stored/schema.ts index 98b2cd50e720..745b0d5be4f0 100644 --- a/packages/dds/tree/src/core/schema-stored/schema.ts +++ b/packages/dds/tree/src/core/schema-stored/schema.ts @@ -19,8 +19,8 @@ import { import type { FieldSchemaFormat as FieldSchemaFormatV2, PersistedMetadataFormat, - TreeNodeSchemaDataFormat as TreeNodeSchemaDataFormatV2, TreeNodeSchemaUnionFormat, + TreeNodeSchemaDataFormat as TreeNodeSchemaDataFormatV2, } from "./formatV2.js"; import type { Multiplicity } from "./multiplicity.js"; @@ -183,6 +183,12 @@ export const identifierFieldKindIdentifier = "Identifier"; export abstract class TreeNodeStoredSchema { protected _typeCheck!: MakeNominal; + /** + * Constructor for a TreeNodeStoredSchema. + * @param metadata - Persisted metadata for this node schema. + */ + public constructor(public readonly metadata: PersistedMetadataFormat | undefined) {} + /** * Encode in the v1 schema format. */ @@ -190,6 +196,7 @@ export abstract class TreeNodeStoredSchema { /** * Encode in the v2 schema format. + * @remarks Post-condition: if metadata was specified on the input schema, it will be present in the output. */ public abstract encodeV2(): TreeNodeSchemaDataFormatV2; @@ -212,8 +219,9 @@ export class ObjectNodeStoredSchema extends TreeNodeStoredSchema { */ public constructor( public readonly objectNodeFields: ReadonlyMap, + metadata?: PersistedMetadataFormat | undefined, ) { - super(); + super(metadata); } public override encodeV1(): TreeNodeSchemaDataFormatV1 { @@ -238,8 +246,26 @@ export class ObjectNodeStoredSchema extends TreeNodeStoredSchema { } public override encodeV2(): TreeNodeSchemaDataFormatV2 { + const fieldsObject: Record = Object.create(null); + // Sort fields to ensure output is identical for for equivalent schema (since field order is not considered significant). + // This makes comparing schema easier, and ensures chunk reuse for schema summaries isn't needlessly broken. + for (const key of [...this.objectNodeFields.keys()].sort()) { + const value = encodeFieldSchemaV2( + this.objectNodeFields.get(key) ?? fail(0xae7 /* missing field */), + ); + + Object.defineProperty(fieldsObject, key, { + enumerable: true, + configurable: true, + writable: true, + value, + }); + } return { - kind: this.encodeV1(), + metadata: this.metadata, + kind: { + object: fieldsObject, + }, }; } @@ -259,8 +285,11 @@ export class MapNodeStoredSchema extends TreeNodeStoredSchema { * since no nodes can ever be in schema if you use `FieldKind.Value` here * (that would require infinite children). */ - public constructor(public readonly mapFields: TreeFieldStoredSchema) { - super(); + public constructor( + public readonly mapFields: TreeFieldStoredSchema, + metadata?: PersistedMetadataFormat | undefined, + ) { + super(metadata); } public override encodeV1(): TreeNodeSchemaDataFormatV1 { @@ -271,7 +300,10 @@ export class MapNodeStoredSchema extends TreeNodeStoredSchema { public override encodeV2(): TreeNodeSchemaDataFormatV2 { return { - kind: this.encodeV1(), + metadata: this.metadata, + kind: { + map: encodeFieldSchemaV2(this.mapFields), + }, }; } @@ -296,7 +328,8 @@ export class LeafNodeStoredSchema extends TreeNodeStoredSchema { * This is simply one approach that can work for modeling them in the internal schema representation. */ public constructor(public readonly leafValue: ValueSchema) { - super(); + // No metadata for leaf nodes. + super(undefined); } public override encodeV1(): TreeNodeSchemaDataFormatV1 { @@ -307,7 +340,11 @@ export class LeafNodeStoredSchema extends TreeNodeStoredSchema { public override encodeV2(): TreeNodeSchemaDataFormatV2 { return { - kind: this.encodeV1(), + // No metadata for leaf nodes. + metadata: undefined, + kind: { + leaf: encodeValueSchema(this.leafValue), + }, }; } @@ -316,22 +353,30 @@ export class LeafNodeStoredSchema extends TreeNodeStoredSchema { } } +/** + * Decoder wrapper function for {@link TreeNodeStoredSchema} implementations. + * Curries the constructor so that the caller can inject metadata. + */ +type StoredSchemaDecoder = ( + metadata: PersistedMetadataFormat | undefined, +) => TreeNodeStoredSchema; + export const storedSchemaDecodeDispatcher: DiscriminatedUnionDispatcher< TreeNodeSchemaUnionFormat, [], - TreeNodeStoredSchema + StoredSchemaDecoder > = new DiscriminatedUnionDispatcher({ - leaf: (data: PersistedValueSchema) => new LeafNodeStoredSchema(decodeValueSchema(data)), - object: ( - data: Record, - ): TreeNodeStoredSchema => { + leaf: (data: PersistedValueSchema) => (metadata) => + new LeafNodeStoredSchema(decodeValueSchema(data)), + object: (data: Record) => (metadata) => { const map = new Map(); for (const [key, value] of Object.entries(data)) { map.set(key, decodeFieldSchema(value)); } - return new ObjectNodeStoredSchema(map); + return new ObjectNodeStoredSchema(map, metadata); }, - map: (data: FieldSchemaFormat) => new MapNodeStoredSchema(decodeFieldSchema(data)), + map: (data: FieldSchemaFormat) => (metadata) => + new MapNodeStoredSchema(decodeFieldSchema(data), metadata), }); const valueSchemaEncode = new Map([ diff --git a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts index 21b6d10b35ec..0c4eb5dbabd7 100644 --- a/packages/dds/tree/src/feature-libraries/schema-index/codec.ts +++ b/packages/dds/tree/src/feature-libraries/schema-index/codec.ts @@ -130,7 +130,10 @@ function encodeRepoV2(repo: TreeStoredSchema): FormatV2 { function decodeV1(f: FormatV1): TreeStoredSchema { const nodeSchema: Map = new Map(); for (const [key, schema] of Object.entries(f.nodes)) { - nodeSchema.set(brand(key), storedSchemaDecodeDispatcher.dispatch(schema)); + const storedSchemaDecoder = storedSchemaDecodeDispatcher.dispatch(schema); + + // No metadata in v1, so pass undefined + nodeSchema.set(brand(key), storedSchemaDecoder(undefined)); } return { rootFieldSchema: decodeFieldSchema(f.root), @@ -141,7 +144,10 @@ function decodeV1(f: FormatV1): TreeStoredSchema { function decodeV2(f: FormatV2): TreeStoredSchema { const nodeSchema: Map = new Map(); for (const [key, schema] of Object.entries(f.nodes)) { - nodeSchema.set(brand(key), storedSchemaDecodeDispatcher.dispatch(schema.kind)); + const storedSchemaDecoder = storedSchemaDecodeDispatcher.dispatch(schema.kind); + + // Pass in the node metadata + nodeSchema.set(brand(key), storedSchemaDecoder(schema.metadata)); } return { rootFieldSchema: decodeFieldSchema(f.root), diff --git a/packages/dds/tree/src/simple-tree/toStoredSchema.ts b/packages/dds/tree/src/simple-tree/toStoredSchema.ts index b52ae0ce55ab..e1a1f3a854b7 100644 --- a/packages/dds/tree/src/simple-tree/toStoredSchema.ts +++ b/packages/dds/tree/src/simple-tree/toStoredSchema.ts @@ -110,11 +110,14 @@ export function getStoredSchema(schema: SimpleNodeSchema): TreeNodeStoredSchema } case NodeKind.Map: { const types = schema.allowedTypesIdentifiers as TreeTypeSet; - return new MapNodeStoredSchema({ - kind: FieldKinds.optional.identifier, - types, - metadata: schema.metadata, - }); + return new MapNodeStoredSchema( + { + kind: FieldKinds.optional.identifier, + types, + metadata: schema.metadata, + }, + schema.metadata, + ); } case NodeKind.Array: { const types = schema.allowedTypesIdentifiers as TreeTypeSet; @@ -124,14 +127,14 @@ export function getStoredSchema(schema: SimpleNodeSchema): TreeNodeStoredSchema metadata: schema.metadata, }; const fields = new Map([[EmptyKey, field]]); - return new ObjectNodeStoredSchema(fields); + return new ObjectNodeStoredSchema(fields, schema.metadata); } case NodeKind.Object: { const fields: Map = new Map(); for (const fieldSchema of schema.fields.values()) { fields.set(brand(fieldSchema.storedKey), convertField(fieldSchema)); } - return new ObjectNodeStoredSchema(fields); + return new ObjectNodeStoredSchema(fields, schema.metadata); } default: unreachableCase(kind); From 2f6b519b890c56b0f1086b14a28bcfb466bf2b0d Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Thu, 29 May 2025 13:58:13 -0700 Subject: [PATCH 33/45] Reverted unneeded change. --- .../src/simple-tree/api/schemaFactoryAlpha.ts | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/packages/dds/tree/src/simple-tree/api/schemaFactoryAlpha.ts b/packages/dds/tree/src/simple-tree/api/schemaFactoryAlpha.ts index 3973608fb8f8..defc50b13c11 100644 --- a/packages/dds/tree/src/simple-tree/api/schemaFactoryAlpha.ts +++ b/packages/dds/tree/src/simple-tree/api/schemaFactoryAlpha.ts @@ -208,22 +208,7 @@ export class SchemaFactoryAlpha< ...props, }); } - - /** - * {@inheritDoc SchemaStatics.optionalRecursive} - */ - public static override readonly requiredRecursive = schemaStatics.requiredRecursive; - - public requiredRecursiveAlpha< - const T extends System_Unsafe.ImplicitAllowedTypesUnsafe, - const TCustomMetadata = unknown, - >( - t: T, - props?: Omit, "defaultProvider">, - ): FieldSchemaAlphaUnsafe { - return createFieldSchemaUnsafe(FieldKind.Required, t, props); - } - + /** * Like {@link SchemaFactory.identifier} but static and a factory function that can be provided {@link FieldProps}. */ From ae1779bfccb7146ef91b4964df79ca10f2e6c7c8 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Thu, 29 May 2025 14:13:07 -0700 Subject: [PATCH 34/45] Removed tests for the alpha API I removed in a previous revision. --- packages/dds/tree/src/simple-tree/api/schemaFactoryAlpha.ts | 2 +- .../dds/tree/src/test/simple-tree/api/schemaFactory.spec.ts | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/dds/tree/src/simple-tree/api/schemaFactoryAlpha.ts b/packages/dds/tree/src/simple-tree/api/schemaFactoryAlpha.ts index defc50b13c11..5b0d0027b53a 100644 --- a/packages/dds/tree/src/simple-tree/api/schemaFactoryAlpha.ts +++ b/packages/dds/tree/src/simple-tree/api/schemaFactoryAlpha.ts @@ -208,7 +208,7 @@ export class SchemaFactoryAlpha< ...props, }); } - + /** * Like {@link SchemaFactory.identifier} but static and a factory function that can be provided {@link FieldProps}. */ diff --git a/packages/dds/tree/src/test/simple-tree/api/schemaFactory.spec.ts b/packages/dds/tree/src/test/simple-tree/api/schemaFactory.spec.ts index 6a74500655ef..ea558ccf6c6c 100644 --- a/packages/dds/tree/src/test/simple-tree/api/schemaFactory.spec.ts +++ b/packages/dds/tree/src/test/simple-tree/api/schemaFactory.spec.ts @@ -447,10 +447,6 @@ describe("schemaFactory", () => { baz: schemaFactory.optionalAlpha(schemaFactory.string, { persistedMetadata: fooMetadata, }), - buzz: schemaFactory.requiredRecursiveAlpha( - schemaFactory.objectAlpha("Buzz", { qux: schemaFactory.number }), - { persistedMetadata: fooMetadata }, - ), qux: schemaFactory.optionalRecursiveAlpha( schemaFactory.objectAlpha("Qux", { quux: schemaFactory.string }), { persistedMetadata: fooMetadata }, @@ -460,7 +456,6 @@ describe("schemaFactory", () => { const foo = hydrate(Foo, { bar: 37, baz: "test", - buzz: { qux: 42 }, qux: { quux: "test" }, }); From 7eff8abd447b7edda2db1aca9a21305203102e11 Mon Sep 17 00:00:00 2001 From: Tommy Brosman Date: Fri, 30 May 2025 15:25:02 -0700 Subject: [PATCH 35/45] This revision adds simple-tree persistence for node and field schema metadata. - Fixed a bug where FormatV2 was including FormatV1 fields. - persistedMetadata is now only written when non-undefined. - simple-tree implementations for individual schema kinds (arrayNode, mapNode, objectNode) now pass persistedMetadata down from the outer factory calls. - Introduced alpha APIs for simple-tree in places where we need to pass persistedMetadata but don't want to pollute the public API. - Updated tests to pass persistedMetadata when constructing SimpleNodeSchemas. - Wrote simple tests for node and field persisted metadata. --- .../dds/tree/api-report/tree.alpha.api.md | 32 ++++++++--------- .../tree/src/core/schema-stored/formatV2.ts | 30 ++++++++++++++-- .../dds/tree/src/core/schema-stored/schema.ts | 36 ++++++++----------- packages/dds/tree/src/index.ts | 1 + .../dds/tree/src/shared-tree/sharedTree.ts | 17 +++++++-- .../src/simple-tree/api/schemaFactoryAlpha.ts | 33 ++++++++++++++--- .../api/viewSchemaToSimpleSchema.ts | 3 ++ .../dds/tree/src/simple-tree/arrayNode.ts | 4 +++ packages/dds/tree/src/simple-tree/index.ts | 1 + .../tree/src/simple-tree/leafNodeSchema.ts | 2 ++ packages/dds/tree/src/simple-tree/mapNode.ts | 10 +++++- .../dds/tree/src/simple-tree/objectNode.ts | 9 ++++- .../dds/tree/src/simple-tree/schemaTypes.ts | 13 +++++-- .../dds/tree/src/simple-tree/simpleSchema.ts | 26 +++++++++++--- .../tree/src/simple-tree/toStoredSchema.ts | 11 +++--- .../simple-tree/api/getSimpleSchema.spec.ts | 8 +++++ .../simple-tree/api/schemaFactory.spec.ts | 12 +++---- .../api/simpleSchemaToJsonSchema.spec.ts | 18 +++++++++- .../files/SchemaIndexFormat - schema v2.json | 6 ++-- 19 files changed, 203 insertions(+), 69 deletions(-) diff --git a/packages/dds/tree/api-report/tree.alpha.api.md b/packages/dds/tree/api-report/tree.alpha.api.md index 54d302d5af26..b32bc6a40395 100644 --- a/packages/dds/tree/api-report/tree.alpha.api.md +++ b/packages/dds/tree/api-report/tree.alpha.api.md @@ -203,13 +203,14 @@ export class FieldSchema extends FieldSchema implements SimpleFieldSchema { - protected constructor(kind: Kind, types: Types, annotatedAllowedTypes: ImplicitAnnotatedAllowedTypes, props?: FieldProps); + protected constructor(kind: Kind, types: Types, annotatedAllowedTypes: ImplicitAnnotatedAllowedTypes, props?: FieldPropsAlpha); // (undocumented) get allowedTypesIdentifiers(): ReadonlySet; readonly allowedTypesMetadata: AllowedTypesMetadata; // (undocumented) readonly annotatedAllowedTypes: ImplicitAnnotatedAllowedTypes; get annotatedAllowedTypeSet(): ReadonlyMap; + get persistedMetadata(): JsonCompatibleReadOnlyObject | undefined; } // @alpha @sealed @system @@ -424,9 +425,9 @@ export namespace JsonAsTree { } const // @system _APIExtractorWorkaroundObjectBase: TreeNodeSchemaClass<"com.fluidframework.json.object", NodeKind.Map, System_Unsafe.TreeMapNodeUnsafe, LeafSchema<"number", number>, LeafSchema<"string", string>, LeafSchema<"boolean", boolean>, () => typeof JsonObject, () => typeof Array]> & WithType<"com.fluidframework.json.object", NodeKind.Map, unknown>, { - [Symbol.iterator](): Iterator<[string, string | number | System_Unsafe.InsertableTypedNodeUnsafe, LeafSchema<"boolean", boolean>> | JsonObject | Array | null], any, undefined>; + [Symbol.iterator](): Iterator<[string, string | number | JsonObject | Array | System_Unsafe.InsertableTypedNodeUnsafe, LeafSchema<"boolean", boolean>> | null], any, undefined>; } | { - readonly [x: string]: string | number | System_Unsafe.InsertableTypedNodeUnsafe, LeafSchema<"boolean", boolean>> | JsonObject | Array | null; + readonly [x: string]: string | number | JsonObject | Array | System_Unsafe.InsertableTypedNodeUnsafe, LeafSchema<"boolean", boolean>> | null; }, false, readonly [LeafSchema<"null", null>, LeafSchema<"number", number>, LeafSchema<"string", string>, LeafSchema<"boolean", boolean>, () => typeof JsonObject, () => typeof Array], undefined>; // (undocumented) export type Primitive = TreeNodeFromImplicitAllowedTypes; @@ -434,7 +435,7 @@ export namespace JsonAsTree { export type _RecursiveArrayWorkaroundJsonArray = FixRecursiveArraySchema; const // @system _APIExtractorWorkaroundArrayBase: TreeNodeSchemaClass<"com.fluidframework.json.array", NodeKind.Array, System_Unsafe.TreeArrayNodeUnsafe, LeafSchema<"number", number>, LeafSchema<"string", string>, LeafSchema<"boolean", boolean>, () => typeof JsonObject, () => typeof Array]> & WithType<"com.fluidframework.json.array", NodeKind.Array, unknown>, { - [Symbol.iterator](): Iterator, LeafSchema<"boolean", boolean>> | JsonObject | Array | null, any, undefined>; + [Symbol.iterator](): Iterator, LeafSchema<"boolean", boolean>> | null, any, undefined>; }, false, readonly [LeafSchema<"null", null>, LeafSchema<"number", number>, LeafSchema<"string", string>, LeafSchema<"boolean", boolean>, () => typeof JsonObject, () => typeof Array], undefined>; // (undocumented) export type Tree = TreeNodeFromImplicitAllowedTypes; @@ -774,11 +775,11 @@ export class SchemaFactory extends SchemaFactory { - arrayAlpha(name: Name, allowedTypes: T, options?: NodeSchemaOptions): ArrayNodeCustomizableSchema, T, true, TCustomMetadata>; + arrayAlpha(name: Name, allowedTypes: T, options?: NodeSchemaOptionsAlpha): ArrayNodeCustomizableSchema, T, true, TCustomMetadata>; arrayRecursive(name: Name, allowedTypes: T, options?: NodeSchemaOptions): ArrayNodeCustomizableSchemaUnsafe, T, TCustomMetadata>; static readonly identifier: (props?: Omit, "defaultProvider"> | undefined) => FieldSchemaAlpha & SimpleLeafNodeSchema_2, TCustomMetadata>; static readonly leaves: readonly [LeafSchema_2<"string", string> & SimpleLeafNodeSchema_2, LeafSchema_2<"number", number> & SimpleLeafNodeSchema_2, LeafSchema_2<"boolean", boolean> & SimpleLeafNodeSchema_2, LeafSchema_2<"null", null> & SimpleLeafNodeSchema_2, LeafSchema_2<"handle", IFluidHandle> & SimpleLeafNodeSchema_2]; - mapAlpha(name: Name, allowedTypes: T, options?: NodeSchemaOptions): MapNodeCustomizableSchema, T, true, TCustomMetadata>; + mapAlpha(name: Name, allowedTypes: T, options?: NodeSchemaOptionsAlpha): MapNodeCustomizableSchema, T, true, TCustomMetadata>; mapRecursive(name: Name, allowedTypes: T, options?: NodeSchemaOptions): MapNodeCustomizableSchemaUnsafe, T, TCustomMetadata>; objectAlpha, const TCustomMetadata = unknown>(name: Name, fields: T, options?: SchemaFactoryObjectOptions): ObjectNodeSchema, T, true, TCustomMetadata> & { readonly createFromInsertable: unknown; @@ -788,20 +789,14 @@ export class SchemaFactoryAlpha(t: T, props?: Omit, "defaultProvider"> | undefined): FieldSchemaAlpha; (t: T_1, props?: Omit, "defaultProvider"> | undefined): FieldSchemaAlpha, TCustomMetadata_1>; }; - // (undocumented) optionalAlpha(t: T, props?: Omit, "defaultProvider">): FieldSchemaAlpha, TCustomMetadata>; static readonly optionalRecursive: (t: T, props?: Omit, "defaultProvider"> | undefined) => FieldSchemaAlphaUnsafe; - // (undocumented) optionalRecursiveAlpha(t: T, props?: Omit, "defaultProvider">): FieldSchemaAlphaUnsafe; static readonly required: { (t: T, props?: Omit, "defaultProvider"> | undefined): FieldSchemaAlpha; (t: T_1, props?: Omit, "defaultProvider"> | undefined): FieldSchemaAlpha, TCustomMetadata_1>; }; - // (undocumented) requiredAlpha(t: T, props?: Omit, "defaultProvider">): FieldSchemaAlpha, TCustomMetadata>; - static readonly requiredRecursive: (t: T, props?: Omit, "defaultProvider"> | undefined) => FieldSchemaAlphaUnsafe; - // (undocumented) - requiredRecursiveAlpha(t: T, props?: Omit, "defaultProvider">): FieldSchemaAlphaUnsafe; scopedFactory(name: T): SchemaFactoryAlpha, TNameInner>; } @@ -858,7 +853,7 @@ export type SharedTreeFormatVersion = typeof SharedTreeFormatVersion; export type SharedTreeOptions = Partial & Partial & ForestOptions; // @alpha @sealed -export interface SimpleArrayNodeSchema extends SimpleNodeSchemaBase { +export interface SimpleArrayNodeSchema extends SimpleNodeSchemaBaseAlpha { readonly allowedTypesIdentifiers: ReadonlySet; } @@ -870,12 +865,12 @@ export interface SimpleFieldSchema { } // @alpha @sealed -export interface SimpleLeafNodeSchema extends SimpleNodeSchemaBase { +export interface SimpleLeafNodeSchema extends SimpleNodeSchemaBaseAlpha { readonly leafKind: ValueSchema; } // @alpha @sealed -export interface SimpleMapNodeSchema extends SimpleNodeSchemaBase { +export interface SimpleMapNodeSchema extends SimpleNodeSchemaBaseAlpha { readonly allowedTypesIdentifiers: ReadonlySet; } @@ -888,13 +883,18 @@ export interface SimpleNodeSchemaBase; } +// @alpha @sealed @system +export interface SimpleNodeSchemaBaseAlpha extends SimpleNodeSchemaBase { + readonly persistedMetadata: JsonCompatibleReadOnlyObject | undefined; +} + // @alpha @sealed export interface SimpleObjectFieldSchema extends SimpleFieldSchema { readonly storedKey: string; } // @alpha @sealed -export interface SimpleObjectNodeSchema extends SimpleNodeSchemaBase { +export interface SimpleObjectNodeSchema extends SimpleNodeSchemaBaseAlpha { readonly fields: ReadonlyMap; } diff --git a/packages/dds/tree/src/core/schema-stored/formatV2.ts b/packages/dds/tree/src/core/schema-stored/formatV2.ts index 2549f547ea96..fbbf5ec635d2 100644 --- a/packages/dds/tree/src/core/schema-stored/formatV2.ts +++ b/packages/dds/tree/src/core/schema-stored/formatV2.ts @@ -8,9 +8,10 @@ import { type ObjectOptions, type Static, Type } from "@sinclair/typebox"; import { JsonCompatibleReadOnlySchema } from "../../util/index.js"; import { FieldKindIdentifierSchema, + PersistedValueSchema, TreeNodeSchemaIdentifierSchema, - TreeNodeSchemaDataFormat as TreeNodeSchemaUnionFormat, } from "./formatV1.js"; +import { unionOptions } from "../../codec/index.js"; export const PersistedMetadataFormat = Type.Optional(JsonCompatibleReadOnlySchema); @@ -24,6 +25,31 @@ const noAdditionalProps: ObjectOptions = { additionalProperties: false }; export const FieldSchemaFormat = Type.Composite([FieldSchemaFormatBase], noAdditionalProps); +/** + * Format for the content of a {@link TreeNodeStoredSchema}. + * + * See {@link DiscriminatedUnionDispatcher} for more information on this pattern. + */ +export const TreeNodeSchemaUnionFormat = Type.Object( + { + /** + * Object node union member. + */ + object: Type.Optional(Type.Record(Type.String(), FieldSchemaFormat)), + /** + * Map node union member. + */ + map: Type.Optional(FieldSchemaFormat), + /** + * Leaf node union member. + */ + leaf: Type.Optional(Type.Enum(PersistedValueSchema)), + }, + unionOptions, +); + +export type TreeNodeSchemaUnionFormat = Static; + /** * Format for {@link TreeNodeStoredSchema}. * @@ -50,5 +76,3 @@ export type TreeNodeSchemaDataFormat = Static; export type FieldSchemaFormat = Static; export type PersistedMetadataFormat = Static; - -export { TreeNodeSchemaUnionFormat }; diff --git a/packages/dds/tree/src/core/schema-stored/schema.ts b/packages/dds/tree/src/core/schema-stored/schema.ts index 745b0d5be4f0..5089a376a96e 100644 --- a/packages/dds/tree/src/core/schema-stored/schema.ts +++ b/packages/dds/tree/src/core/schema-stored/schema.ts @@ -261,12 +261,11 @@ export class ObjectNodeStoredSchema extends TreeNodeStoredSchema { value, }); } - return { - metadata: this.metadata, - kind: { - object: fieldsObject, - }, - }; + + const kind = { object: fieldsObject }; + + // Omit metadata from the output if it is undefined + return this.metadata !== undefined ? { kind, metadata: this.metadata } : { kind }; } public override getFieldSchema(field: FieldKey): TreeFieldStoredSchema { @@ -293,18 +292,12 @@ export class MapNodeStoredSchema extends TreeNodeStoredSchema { } public override encodeV1(): TreeNodeSchemaDataFormatV1 { - return { - map: encodeFieldSchemaV1(this.mapFields), - }; + return { map: encodeFieldSchemaV1(this.mapFields) }; } public override encodeV2(): TreeNodeSchemaDataFormatV2 { - return { - metadata: this.metadata, - kind: { - map: encodeFieldSchemaV2(this.mapFields), - }, - }; + const kind = { map: encodeFieldSchemaV2(this.mapFields) }; + return this.metadata === undefined ? { kind, metadata: this.metadata } : { kind }; } public override getFieldSchema(field: FieldKey): TreeFieldStoredSchema { @@ -340,8 +333,7 @@ export class LeafNodeStoredSchema extends TreeNodeStoredSchema { public override encodeV2(): TreeNodeSchemaDataFormatV2 { return { - // No metadata for leaf nodes. - metadata: undefined, + // No metadata for leaf nodes, so don't emit a metadata field. kind: { leaf: encodeValueSchema(this.leafValue), }, @@ -406,10 +398,12 @@ export function encodeFieldSchemaV1(schema: TreeFieldStoredSchema): FieldSchemaF } export function encodeFieldSchemaV2(schema: TreeFieldStoredSchema): FieldSchemaFormatV2 { - return { - ...encodeFieldSchemaV1(schema), - metadata: schema.metadata, - }; + const fieldSchema: FieldSchemaFormatV1 = encodeFieldSchemaV1(schema); + + // Omit metadata from the output if it is undefined + return schema.metadata !== undefined + ? { ...fieldSchema, metadata: schema.metadata } + : { ...fieldSchema }; } export function decodeFieldSchema(schema: FieldSchemaFormatV2): TreeFieldStoredSchema { diff --git a/packages/dds/tree/src/index.ts b/packages/dds/tree/src/index.ts index 27060ac802b2..c5e6179b857c 100644 --- a/packages/dds/tree/src/index.ts +++ b/packages/dds/tree/src/index.ts @@ -203,6 +203,7 @@ export { type LazyItem, type Unenforced, type SimpleNodeSchemaBase, + type SimpleNodeSchemaBaseAlpha, type SimpleTreeSchema, type SimpleNodeSchema, type SimpleFieldSchema, diff --git a/packages/dds/tree/src/shared-tree/sharedTree.ts b/packages/dds/tree/src/shared-tree/sharedTree.ts index bd0e8ed2a363..cc09943e3868 100644 --- a/packages/dds/tree/src/shared-tree/sharedTree.ts +++ b/packages/dds/tree/src/shared-tree/sharedTree.ts @@ -743,14 +743,19 @@ function exportSimpleFieldSchemaStored(schema: TreeFieldStoredSchema): SimpleFie function exportSimpleNodeSchemaStored(schema: TreeNodeStoredSchema): SimpleNodeSchema { const arrayTypes = tryStoredSchemaAsArray(schema); if (arrayTypes !== undefined) { - return { kind: NodeKind.Array, allowedTypesIdentifiers: arrayTypes, metadata: {} }; + return { + kind: NodeKind.Array, + allowedTypesIdentifiers: arrayTypes, + metadata: {}, + persistedMetadata: undefined, + }; } if (schema instanceof ObjectNodeStoredSchema) { const fields = new Map(); for (const [storedKey, field] of schema.objectNodeFields) { fields.set(storedKey, { ...exportSimpleFieldSchemaStored(field), storedKey }); } - return { kind: NodeKind.Object, fields, metadata: {} }; + return { kind: NodeKind.Object, fields, metadata: {}, persistedMetadata: undefined }; } if (schema instanceof MapNodeStoredSchema) { assert( @@ -761,10 +766,16 @@ function exportSimpleNodeSchemaStored(schema: TreeNodeStoredSchema): SimpleNodeS kind: NodeKind.Map, allowedTypesIdentifiers: schema.mapFields.types, metadata: {}, + persistedMetadata: undefined, }; } if (schema instanceof LeafNodeStoredSchema) { - return { kind: NodeKind.Leaf, leafKind: schema.leafValue, metadata: {} }; + return { + kind: NodeKind.Leaf, + leafKind: schema.leafValue, + metadata: {}, + persistedMetadata: undefined, + }; } fail(0xacb /* invalid schema kind */); } diff --git a/packages/dds/tree/src/simple-tree/api/schemaFactoryAlpha.ts b/packages/dds/tree/src/simple-tree/api/schemaFactoryAlpha.ts index 5b0d0027b53a..9f7b16f78afb 100644 --- a/packages/dds/tree/src/simple-tree/api/schemaFactoryAlpha.ts +++ b/packages/dds/tree/src/simple-tree/api/schemaFactoryAlpha.ts @@ -21,6 +21,7 @@ import { type ImplicitAnnotatedFieldSchema, type ImplicitFieldSchema, type NodeSchemaOptions, + type NodeSchemaOptionsAlpha, type UnannotateImplicitAllowedTypes, } from "../schemaTypes.js"; import { objectSchema } from "../objectNode.js"; @@ -94,6 +95,7 @@ export class SchemaFactoryAlpha< options?.allowUnknownOptionalFields ?? defaultSchemaFactoryObjectOptions.allowUnknownOptionalFields, options?.metadata, + options?.persistedMetadata, ); } @@ -163,6 +165,9 @@ export class SchemaFactoryAlpha< */ public static override readonly optional = schemaStatics.optional; + /** + * {@inheritDoc SchemaStatics.optional} + */ public optionalAlpha< const T extends ImplicitAnnotatedAllowedTypes, const TCustomMetadata = unknown, @@ -181,6 +186,9 @@ export class SchemaFactoryAlpha< */ public static override readonly required = schemaStatics.required; + /** + * {@inheritDoc SchemaStatics.required} + */ public requiredAlpha< const T extends ImplicitAnnotatedAllowedTypes, const TCustomMetadata = unknown, @@ -196,6 +204,9 @@ export class SchemaFactoryAlpha< */ public static override readonly optionalRecursive = schemaStatics.optionalRecursive; + /** + * {@inheritDoc SchemaStatics.optionalRecursive} + */ public optionalRecursiveAlpha< const T extends System_Unsafe.ImplicitAllowedTypesUnsafe, const TCustomMetadata = unknown, @@ -235,9 +246,16 @@ export class SchemaFactoryAlpha< >( name: Name, allowedTypes: T, - options?: NodeSchemaOptions, + options?: NodeSchemaOptionsAlpha, ): MapNodeCustomizableSchema, T, true, TCustomMetadata> { - return mapSchema(this.scoped2(name), allowedTypes, true, true, options?.metadata); + return mapSchema( + this.scoped2(name), + allowedTypes, + true, + true, + options?.metadata, + options?.persistedMetadata, + ); } /** @@ -279,9 +297,16 @@ export class SchemaFactoryAlpha< >( name: Name, allowedTypes: T, - options?: NodeSchemaOptions, + options?: NodeSchemaOptionsAlpha, ): ArrayNodeCustomizableSchema, T, true, TCustomMetadata> { - return arraySchema(this.scoped2(name), allowedTypes, true, true, options?.metadata); + return arraySchema( + this.scoped2(name), + allowedTypes, + true, + true, + options?.metadata, + options?.persistedMetadata, + ); } /** diff --git a/packages/dds/tree/src/simple-tree/api/viewSchemaToSimpleSchema.ts b/packages/dds/tree/src/simple-tree/api/viewSchemaToSimpleSchema.ts index 74f53ffcff25..40b3eb4f9de1 100644 --- a/packages/dds/tree/src/simple-tree/api/viewSchemaToSimpleSchema.ts +++ b/packages/dds/tree/src/simple-tree/api/viewSchemaToSimpleSchema.ts @@ -96,6 +96,7 @@ function copySimpleLeafSchema(schema: SimpleLeafNodeSchema): SimpleLeafNodeSchem kind: NodeKind.Leaf, leafKind: schema.leafKind, metadata: schema.metadata, + persistedMetadata: schema.persistedMetadata, }; } @@ -106,6 +107,7 @@ function copySimpleMapOrArraySchema( kind: schema.kind, allowedTypesIdentifiers: schema.allowedTypesIdentifiers, metadata: schema.metadata, + persistedMetadata: schema.persistedMetadata, }; } @@ -125,5 +127,6 @@ function copySimpleObjectSchema(schema: SimpleObjectNodeSchema): SimpleObjectNod kind: NodeKind.Object, fields, metadata: schema.metadata, + persistedMetadata: schema.persistedMetadata, }; } diff --git a/packages/dds/tree/src/simple-tree/arrayNode.ts b/packages/dds/tree/src/simple-tree/arrayNode.ts index a689fb5fb912..465256abd1ef 100644 --- a/packages/dds/tree/src/simple-tree/arrayNode.ts +++ b/packages/dds/tree/src/simple-tree/arrayNode.ts @@ -52,6 +52,7 @@ import type { ArrayNodeCustomizableSchema, ArrayNodePojoEmulationSchema, } from "./arrayNodeTypes.js"; +import type { JsonCompatibleReadOnlyObject } from "../util/index.js"; /** * A covariant base type for {@link (TreeArrayNode:interface)}. @@ -1093,6 +1094,7 @@ export function arraySchema< implicitlyConstructable: ImplicitlyConstructable, customizable: boolean, metadata?: NodeSchemaMetadata, + persistedMetadata?: JsonCompatibleReadOnlyObject | undefined, ) { type Output = ArrayNodeCustomizableSchema< TName, @@ -1191,6 +1193,8 @@ export function arraySchema< return lazyChildTypes.value; } public static readonly metadata: NodeSchemaMetadata = metadata ?? {}; + public static readonly persistedMetadata: JsonCompatibleReadOnlyObject | undefined = + persistedMetadata; // eslint-disable-next-line import/no-deprecated public get [typeNameSymbol](): TName { diff --git a/packages/dds/tree/src/simple-tree/index.ts b/packages/dds/tree/src/simple-tree/index.ts index 3b0c97f0c6ce..d55a1eb2c0e5 100644 --- a/packages/dds/tree/src/simple-tree/index.ts +++ b/packages/dds/tree/src/simple-tree/index.ts @@ -136,6 +136,7 @@ export type { SimpleArrayNodeSchema, SimpleObjectNodeSchema, SimpleNodeSchemaBase, + SimpleNodeSchemaBaseAlpha, SimpleObjectFieldSchema, } from "./simpleSchema.js"; export { diff --git a/packages/dds/tree/src/simple-tree/leafNodeSchema.ts b/packages/dds/tree/src/simple-tree/leafNodeSchema.ts index 865ab6504bd1..68ce3c2d7ed2 100644 --- a/packages/dds/tree/src/simple-tree/leafNodeSchema.ts +++ b/packages/dds/tree/src/simple-tree/leafNodeSchema.ts @@ -14,6 +14,7 @@ import { import { NodeKind, type TreeNodeSchema, type TreeNodeSchemaNonClass } from "./core/index.js"; import type { NodeSchemaMetadata, TreeLeafValue } from "./schemaTypes.js"; import type { SimpleLeafNodeSchema } from "./simpleSchema.js"; +import type { JsonCompatibleReadOnlyObject } from "../util/index.js"; /** * Instances of this class are schema for leaf nodes. @@ -49,6 +50,7 @@ export class LeafNodeSchema public readonly leafKind: ValueSchema; public readonly metadata: NodeSchemaMetadata = {}; + public readonly persistedMetadata: JsonCompatibleReadOnlyObject | undefined; public constructor(name: Name, t: T) { this.identifier = name; diff --git a/packages/dds/tree/src/simple-tree/mapNode.ts b/packages/dds/tree/src/simple-tree/mapNode.ts index 2a475bcdec5b..981df9cb0869 100644 --- a/packages/dds/tree/src/simple-tree/mapNode.ts +++ b/packages/dds/tree/src/simple-tree/mapNode.ts @@ -42,7 +42,12 @@ import { type InsertableContent, } from "./toMapTree.js"; import { prepareForInsertion } from "./prepareForInsertion.js"; -import { brand, count, type RestrictiveStringRecord } from "../util/index.js"; +import { + brand, + count, + type JsonCompatibleReadOnlyObject, + type RestrictiveStringRecord, +} from "../util/index.js"; import { TreeNodeValid, type MostDerivedData } from "./treeNodeValid.js"; import type { ExclusiveMapTree } from "../core/index.js"; import { getUnhydratedContext } from "./createContext.js"; @@ -241,6 +246,7 @@ export function mapSchema< implicitlyConstructable: ImplicitlyConstructable, useMapPrototype: boolean, metadata?: NodeSchemaMetadata, + persistedMetadata?: JsonCompatibleReadOnlyObject | undefined, ) { const lazyChildTypes = new Lazy(() => normalizeAllowedTypes(unannotateImplicitAllowedTypes(info)), @@ -297,6 +303,8 @@ export function mapSchema< return lazyChildTypes.value; } public static readonly metadata: NodeSchemaMetadata = metadata ?? {}; + public static readonly persistedMetadata: JsonCompatibleReadOnlyObject | undefined = + persistedMetadata; // eslint-disable-next-line import/no-deprecated public get [typeNameSymbol](): TName { diff --git a/packages/dds/tree/src/simple-tree/objectNode.ts b/packages/dds/tree/src/simple-tree/objectNode.ts index 938f5ba2897e..d015e1b43b33 100644 --- a/packages/dds/tree/src/simple-tree/objectNode.ts +++ b/packages/dds/tree/src/simple-tree/objectNode.ts @@ -47,7 +47,11 @@ import { } from "./core/index.js"; import { mapTreeFromNodeData, type InsertableContent } from "./toMapTree.js"; import { prepareForInsertion } from "./prepareForInsertion.js"; -import type { RestrictiveStringRecord, FlattenKeys } from "../util/index.js"; +import type { + RestrictiveStringRecord, + FlattenKeys, + JsonCompatibleReadOnlyObject, +} from "../util/index.js"; import { isObjectNodeSchema, type ObjectNodeSchema, @@ -371,6 +375,7 @@ export function objectSchema< implicitlyConstructable: ImplicitlyConstructable, allowUnknownOptionalFields: boolean, metadata?: NodeSchemaMetadata, + persistedMetadata?: JsonCompatibleReadOnlyObject | undefined, ): ObjectNodeSchema & ObjectNodeSchemaInternalData { // Field set can't be modified after this since derived data is stored in maps. @@ -517,6 +522,8 @@ export function objectSchema< return lazyChildTypes.value; } public static readonly metadata: NodeSchemaMetadata = metadata ?? {}; + public static readonly persistedMetadata: JsonCompatibleReadOnlyObject | undefined = + persistedMetadata; // eslint-disable-next-line import/no-deprecated public get [typeNameSymbol](): TName { diff --git a/packages/dds/tree/src/simple-tree/schemaTypes.ts b/packages/dds/tree/src/simple-tree/schemaTypes.ts index 69e3f6f33c76..20225aa098a3 100644 --- a/packages/dds/tree/src/simple-tree/schemaTypes.ts +++ b/packages/dds/tree/src/simple-tree/schemaTypes.ts @@ -540,12 +540,20 @@ export class FieldSchemaAlpha< { private readonly lazyIdentifiers: Lazy>; private readonly lazyAnnotatedTypes: Lazy>; + private readonly propsAlpha: FieldPropsAlpha | undefined; /** * Metadata on the types of tree nodes allowed on this field. */ public readonly allowedTypesMetadata: AllowedTypesMetadata; + /** + * Persisted metadata for this field schema. + */ + public get persistedMetadata(): JsonCompatibleReadOnlyObject | undefined { + return this.propsAlpha?.persistedMetadata ?? {}; + } + static { createFieldSchemaPrivate = < Kind2 extends FieldKind, @@ -554,7 +562,7 @@ export class FieldSchemaAlpha< >( kind: Kind2, annotatedAllowedTypes: Types2, - props?: FieldProps, + props?: FieldPropsAlpha, ) => new FieldSchemaAlpha( kind, @@ -568,7 +576,7 @@ export class FieldSchemaAlpha< kind: Kind, types: Types, public readonly annotatedAllowedTypes: ImplicitAnnotatedAllowedTypes, - props?: FieldProps, + props?: FieldPropsAlpha, ) { super(kind, types, props); @@ -581,6 +589,7 @@ export class FieldSchemaAlpha< this.lazyIdentifiers = new Lazy( () => new Set([...this.allowedTypeSet].map((t) => t.identifier)), ); + this.propsAlpha = props; } public get allowedTypesIdentifiers(): ReadonlySet { diff --git a/packages/dds/tree/src/simple-tree/simpleSchema.ts b/packages/dds/tree/src/simple-tree/simpleSchema.ts index be253ce8939f..4839b9369879 100644 --- a/packages/dds/tree/src/simple-tree/simpleSchema.ts +++ b/packages/dds/tree/src/simple-tree/simpleSchema.ts @@ -4,6 +4,7 @@ */ import type { ValueSchema } from "../core/index.js"; +import type { JsonCompatibleReadOnlyObject } from "../util/index.js"; import type { NodeKind } from "./core/index.js"; import type { FieldKind, FieldSchemaMetadata, NodeSchemaMetadata } from "./schemaTypes.js"; @@ -36,6 +37,23 @@ export interface SimpleNodeSchemaBase< readonly metadata: NodeSchemaMetadata; } +/** + * A {@link SimpleNodeSchema} containing fields for alpha features. + * + * @system + * @alpha + * @sealed + */ +export interface SimpleNodeSchemaBaseAlpha< + out TNodeKind extends NodeKind, + out TCustomMetadata = unknown, +> extends SimpleNodeSchemaBase { + /** + * Persisted metadata for this node schema. + */ + readonly persistedMetadata: JsonCompatibleReadOnlyObject | undefined; +} + /** * A {@link SimpleNodeSchema} for an object node. * @@ -43,7 +61,7 @@ export interface SimpleNodeSchemaBase< * @sealed */ export interface SimpleObjectNodeSchema - extends SimpleNodeSchemaBase { + extends SimpleNodeSchemaBaseAlpha { /** * Schemas for each of the object's fields, keyed off of schema's keys. * @remarks @@ -81,7 +99,7 @@ export interface SimpleObjectFieldSchema extends SimpleFieldSchema { * @sealed */ export interface SimpleArrayNodeSchema - extends SimpleNodeSchemaBase { + extends SimpleNodeSchemaBaseAlpha { /** * The types allowed in the array. * @@ -98,7 +116,7 @@ export interface SimpleArrayNodeSchema * @sealed */ export interface SimpleMapNodeSchema - extends SimpleNodeSchemaBase { + extends SimpleNodeSchemaBaseAlpha { /** * The types allowed as values in the map. * @@ -114,7 +132,7 @@ export interface SimpleMapNodeSchema * @alpha * @sealed */ -export interface SimpleLeafNodeSchema extends SimpleNodeSchemaBase { +export interface SimpleLeafNodeSchema extends SimpleNodeSchemaBaseAlpha { /** * The kind of leaf node. */ diff --git a/packages/dds/tree/src/simple-tree/toStoredSchema.ts b/packages/dds/tree/src/simple-tree/toStoredSchema.ts index e1a1f3a854b7..dd4cd1196d50 100644 --- a/packages/dds/tree/src/simple-tree/toStoredSchema.ts +++ b/packages/dds/tree/src/simple-tree/toStoredSchema.ts @@ -114,9 +114,10 @@ export function getStoredSchema(schema: SimpleNodeSchema): TreeNodeStoredSchema { kind: FieldKinds.optional.identifier, types, - metadata: schema.metadata, + metadata: schema.persistedMetadata, }, - schema.metadata, + // TODO: Find a way to avoid injecting persistedMetadata twice in these constructor calls. + schema.persistedMetadata, ); } case NodeKind.Array: { @@ -124,17 +125,17 @@ export function getStoredSchema(schema: SimpleNodeSchema): TreeNodeStoredSchema const field = { kind: FieldKinds.sequence.identifier, types, - metadata: schema.metadata, + metadata: schema.persistedMetadata, }; const fields = new Map([[EmptyKey, field]]); - return new ObjectNodeStoredSchema(fields, schema.metadata); + return new ObjectNodeStoredSchema(fields, schema.persistedMetadata); } case NodeKind.Object: { const fields: Map = new Map(); for (const fieldSchema of schema.fields.values()) { fields.set(brand(fieldSchema.storedKey), convertField(fieldSchema)); } - return new ObjectNodeStoredSchema(fields, schema.metadata); + return new ObjectNodeStoredSchema(fields, schema.persistedMetadata); } default: unreachableCase(kind); diff --git a/packages/dds/tree/src/test/simple-tree/api/getSimpleSchema.spec.ts b/packages/dds/tree/src/test/simple-tree/api/getSimpleSchema.spec.ts index 56e41da67f64..db3ffb35cb96 100644 --- a/packages/dds/tree/src/test/simple-tree/api/getSimpleSchema.spec.ts +++ b/packages/dds/tree/src/test/simple-tree/api/getSimpleSchema.spec.ts @@ -24,12 +24,14 @@ const simpleString: SimpleLeafNodeSchema = { leafKind: ValueSchema.String, kind: NodeKind.Leaf, metadata: {}, + persistedMetadata: undefined, }; const simpleNumber: SimpleLeafNodeSchema = { leafKind: ValueSchema.Number, kind: NodeKind.Leaf, metadata: {}, + persistedMetadata: undefined, }; describe("getSimpleSchema", () => { @@ -125,6 +127,7 @@ describe("getSimpleSchema", () => { kind: NodeKind.Array, allowedTypesIdentifiers: new Set(["com.fluidframework.leaf.string"]), metadata: {}, + persistedMetadata: undefined, }, ], ["com.fluidframework.leaf.string", simpleString], @@ -150,6 +153,7 @@ describe("getSimpleSchema", () => { { kind: NodeKind.Map, metadata: {}, + persistedMetadata: undefined, allowedTypesIdentifiers: new Set(["com.fluidframework.leaf.string"]), }, ], @@ -180,6 +184,7 @@ describe("getSimpleSchema", () => { { kind: NodeKind.Object, metadata: {}, + persistedMetadata: undefined, fields: new Map([ [ "foo", @@ -229,6 +234,7 @@ describe("getSimpleSchema", () => { { kind: NodeKind.Object, metadata: {}, + persistedMetadata: undefined, fields: new Map([ [ "id", @@ -269,6 +275,7 @@ describe("getSimpleSchema", () => { { kind: NodeKind.Object, metadata: {}, + persistedMetadata: undefined, fields: new Map([ [ "foo", @@ -312,6 +319,7 @@ describe("getSimpleSchema", () => { { kind: NodeKind.Object, metadata: {}, + persistedMetadata: undefined, fields: new Map([ [ "foo", diff --git a/packages/dds/tree/src/test/simple-tree/api/schemaFactory.spec.ts b/packages/dds/tree/src/test/simple-tree/api/schemaFactory.spec.ts index ea558ccf6c6c..8f2be6beeaff 100644 --- a/packages/dds/tree/src/test/simple-tree/api/schemaFactory.spec.ts +++ b/packages/dds/tree/src/test/simple-tree/api/schemaFactory.spec.ts @@ -427,11 +427,7 @@ describe("schemaFactory", () => { { persistedMetadata: fooMetadata }, ) {} - assert.deepEqual(Foo.metadata, fooMetadata); - - // Ensure `Foo.metadata` is typed as we expect, and we can access its fields without casting. - const persistedMetadata = Foo.metadata.persistedMetadata; - const a = Foo.metadata.persistedMetadata.a; + assert.deepEqual(Foo.persistedMetadata, fooMetadata); }); it("Field schema persisted metadata", () => { @@ -461,7 +457,11 @@ describe("schemaFactory", () => { const schema = Tree.schema(foo) as ObjectNodeSchema; // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - assert.deepEqual(schema.fields.get("bar")!.metadata, fooMetadata); + assert.deepEqual(schema.fields.get("bar")!.persistedMetadata, fooMetadata); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + assert.deepEqual(schema.fields.get("baz")!.persistedMetadata, fooMetadata); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + assert.deepEqual(schema.fields.get("qux")!.persistedMetadata, fooMetadata); }); describe("deep equality", () => { diff --git a/packages/dds/tree/src/test/simple-tree/api/simpleSchemaToJsonSchema.spec.ts b/packages/dds/tree/src/test/simple-tree/api/simpleSchemaToJsonSchema.spec.ts index 85451538d4cc..8af9cdfbdeba 100644 --- a/packages/dds/tree/src/test/simple-tree/api/simpleSchemaToJsonSchema.spec.ts +++ b/packages/dds/tree/src/test/simple-tree/api/simpleSchemaToJsonSchema.spec.ts @@ -88,7 +88,12 @@ describe("simpleSchemaToJsonSchema", () => { definitions: new Map([ [ "test.handle", - { leafKind: ValueSchema.FluidHandle, metadata: {}, kind: NodeKind.Leaf }, + { + leafKind: ValueSchema.FluidHandle, + metadata: {}, + persistedMetadata: undefined, + kind: NodeKind.Leaf, + }, ], ]), }; @@ -109,6 +114,7 @@ describe("simpleSchemaToJsonSchema", () => { { kind: NodeKind.Array, metadata: {}, + persistedMetadata: undefined, allowedTypesIdentifiers: new Set([stringSchema.identifier]), }, ], @@ -161,6 +167,7 @@ describe("simpleSchemaToJsonSchema", () => { { kind: NodeKind.Map, metadata: {}, + persistedMetadata: undefined, allowedTypesIdentifiers: new Set([stringSchema.identifier]), }, ], @@ -271,6 +278,7 @@ describe("simpleSchemaToJsonSchema", () => { { kind: NodeKind.Object, metadata: {}, + persistedMetadata: undefined, fields: new Map([ [ "foo", @@ -278,6 +286,7 @@ describe("simpleSchemaToJsonSchema", () => { kind: FieldKind.Optional, allowedTypesIdentifiers: new Set([numberSchema.identifier]), metadata: { description: "A number representing the concept of Foo." }, + persistedMetadata: undefined, storedKey: "foo", }, ], @@ -287,6 +296,7 @@ describe("simpleSchemaToJsonSchema", () => { kind: FieldKind.Required, allowedTypesIdentifiers: new Set([stringSchema.identifier]), metadata: { description: "A string representing the concept of Bar." }, + persistedMetadata: undefined, storedKey: "bar", }, ], @@ -298,6 +308,7 @@ describe("simpleSchemaToJsonSchema", () => { metadata: { description: "Unique identifier for the test object.", }, + persistedMetadata: undefined, storedKey: "id", }, ], @@ -404,6 +415,7 @@ describe("simpleSchemaToJsonSchema", () => { { kind: NodeKind.Object, metadata: {}, + persistedMetadata: undefined, fields: new Map([ [ "id", @@ -459,12 +471,14 @@ describe("simpleSchemaToJsonSchema", () => { { kind: NodeKind.Object, metadata: {}, + persistedMetadata: undefined, fields: new Map([ [ "foo", { kind: FieldKind.Required, metadata: {}, + persistedMetadata: undefined, allowedTypesIdentifiers: new Set([ numberSchema.identifier, stringSchema.identifier, @@ -525,12 +539,14 @@ describe("simpleSchemaToJsonSchema", () => { { kind: NodeKind.Object, metadata: {}, + persistedMetadata: undefined, fields: new Map([ [ "foo", { kind: FieldKind.Optional, metadata: {}, + persistedMetadata: undefined, allowedTypesIdentifiers: new Set([ stringSchema.identifier, "test.recursive-object", diff --git a/packages/dds/tree/src/test/snapshots/files/SchemaIndexFormat - schema v2.json b/packages/dds/tree/src/test/snapshots/files/SchemaIndexFormat - schema v2.json index 4d829a7f6bff..740b8c921385 100644 --- a/packages/dds/tree/src/test/snapshots/files/SchemaIndexFormat - schema v2.json +++ b/packages/dds/tree/src/test/snapshots/files/SchemaIndexFormat - schema v2.json @@ -34,7 +34,8 @@ "items": { "type": "string" } - } + }, + "metadata": {} }, "required": [ "kind", @@ -55,7 +56,8 @@ "items": { "type": "string" } - } + }, + "metadata": {} }, "required": [ "kind", From 0a84a12bee133d252faeb378e83a011fe5394019 Mon Sep 17 00:00:00 2001 From: Alex Villarreal <716334+alexvy86@users.noreply.github.com> Date: Sun, 1 Jun 2025 12:46:27 -0500 Subject: [PATCH 36/45] Update example app to use SPE --- examples/apps/ai-collab/src/app/page.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/apps/ai-collab/src/app/page.tsx b/examples/apps/ai-collab/src/app/page.tsx index 7fe2e53e98fb..163968143cda 100644 --- a/examples/apps/ai-collab/src/app/page.tsx +++ b/examples/apps/ai-collab/src/app/page.tsx @@ -34,8 +34,8 @@ import { useFluidContainerNextJs } from "@/useFluidContainerNextjs"; import { useSharedTreeRerender } from "@/useSharedTreeRerender"; // Uncomment the import line that corresponds to the server you want to use -// import { createContainer, loadContainer, postAttach, containerIdFromUrl } from "./spe"; // eslint-disable-line import/order -import { createContainer, loadContainer, postAttach, containerIdFromUrl } from "./tinylicious"; // eslint-disable-line import/order +import { createContainer, loadContainer, postAttach, containerIdFromUrl } from "./spe"; // eslint-disable-line import/order +// import { createContainer, loadContainer, postAttach, containerIdFromUrl } from "./tinylicious"; // eslint-disable-line import/order export async function createAndInitializeContainer(): Promise< IFluidContainer From 2a4221fa82263609e51123925736f8dada9753aa Mon Sep 17 00:00:00 2001 From: Alex Villarreal <716334+alexvy86@users.noreply.github.com> Date: Sun, 1 Jun 2025 12:46:43 -0500 Subject: [PATCH 37/45] Add persistedMetadata to schema in example app --- .../src/types/sharedTreeAppSchema.ts | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/examples/apps/ai-collab/src/types/sharedTreeAppSchema.ts b/examples/apps/ai-collab/src/types/sharedTreeAppSchema.ts index 3b03f162dd6d..eec8ec01dbea 100644 --- a/examples/apps/ai-collab/src/types/sharedTreeAppSchema.ts +++ b/examples/apps/ai-collab/src/types/sharedTreeAppSchema.ts @@ -16,33 +16,33 @@ const sf = new SchemaFactoryAlpha("ai-collab-sample-application"); export class SharedTreeTask extends sf.objectAlpha( "Task", { - title: sf.required(sf.string, { + title: sf.requiredAlpha(sf.string, { metadata: { description: `The title of the task.`, }, }), id: sf.identifier, - description: sf.required(sf.string, { + description: sf.requiredAlpha(sf.string, { metadata: { description: `The description of the task.`, }, }), - priority: sf.required(sf.string, { + priority: sf.requiredAlpha(sf.string, { metadata: { description: `The priority of the task which can ONLY be one of three levels: "Low", "Medium", "High" (case-sensitive).`, }, }), - complexity: sf.required(sf.number, { + complexity: sf.requiredAlpha(sf.number, { metadata: { description: `The complexity of the task as a fibonacci number.`, }, }), - status: sf.required(sf.string, { + status: sf.requiredAlpha(sf.string, { metadata: { description: `The status of the task which can ONLY be one of the following values: "To Do", "In Progress", "Done" (case-sensitive).`, }, }), - assignee: sf.required(sf.string, { + assignee: sf.requiredAlpha(sf.string, { metadata: { description: `The name of the tasks assignee e.g. "Bob" or "Alice".`, }, @@ -52,6 +52,9 @@ export class SharedTreeTask extends sf.objectAlpha( metadata: { description: `A task that can be assigned to an engineer.`, }, + persistedMetadata: { + "eDiscovery-exclude": "comment", + }, }, ) {} @@ -60,21 +63,28 @@ export class SharedTreeTaskList extends sf.array("TaskList", SharedTreeTask) {} export class SharedTreeEngineer extends sf.objectAlpha( "Engineer", { - name: sf.required(sf.string, { + name: sf.requiredAlpha(sf.string, { metadata: { description: `The name of the engineer.`, }, }), id: sf.identifier, - skills: sf.required(sf.string, { + skills: sf.requiredAlpha(sf.string, { metadata: { description: `A description of the engineer's skills, which influence what types of tasks they should be assigned to.`, }, + persistedMetadata: { + "eDiscovery-exclude": "comment", + }, }), - maxCapacity: sf.required(sf.number, { + maxCapacity: sf.requiredAlpha(sf.number, { metadata: { description: `The maximum capacity of tasks this engineer can handle, measured in task complexity points.`, }, + persistedMetadata: { + "eDiscovery-exclude": "exclude", + "search-exclude": "true", + }, }), }, { @@ -89,23 +99,23 @@ export class SharedTreeEngineerList extends sf.array("EngineerList", SharedTreeE export class SharedTreeTaskGroup extends sf.objectAlpha( "TaskGroup", { - description: sf.required(sf.string, { + description: sf.requiredAlpha(sf.string, { metadata: { description: `The description of the task group.`, }, }), id: sf.identifier, - title: sf.required(sf.string, { + title: sf.requiredAlpha(sf.string, { metadata: { description: `The title of the task group.`, }, }), - tasks: sf.required(SharedTreeTaskList, { + tasks: sf.requiredAlpha(SharedTreeTaskList, { metadata: { description: `The lists of tasks within this task group.`, }, }), - engineers: sf.required(SharedTreeEngineerList, { + engineers: sf.requiredAlpha(SharedTreeEngineerList, { metadata: { description: `The lists of engineers within this task group to whom tasks may be assigned.`, }, @@ -121,7 +131,7 @@ export class SharedTreeTaskGroup extends sf.objectAlpha( export class SharedTreeTaskGroupList extends sf.array("TaskGroupList", SharedTreeTaskGroup) {} export class SharedTreeAppState extends sf.object("AppState", { - taskGroups: sf.required(SharedTreeTaskGroupList, { + taskGroups: sf.requiredAlpha(SharedTreeTaskGroupList, { metadata: { description: `The list of task groups that are being managed by this task management application.`, }, From d7ec62bf21815b6069145c5c2fe1fe7c54565f17 Mon Sep 17 00:00:00 2001 From: Alex Villarreal <716334+alexvy86@users.noreply.github.com> Date: Sun, 1 Jun 2025 12:47:17 -0500 Subject: [PATCH 38/45] Updates in SharedTree --- packages/dds/tree/src/core/schema-stored/schema.ts | 6 ++++++ .../dds/tree/src/shared-tree-core/sharedTreeCore.ts | 9 ++++++++- packages/dds/tree/src/shared-tree/sharedTree.ts | 10 ++++++++-- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/packages/dds/tree/src/core/schema-stored/schema.ts b/packages/dds/tree/src/core/schema-stored/schema.ts index 5089a376a96e..4f33bb669eff 100644 --- a/packages/dds/tree/src/core/schema-stored/schema.ts +++ b/packages/dds/tree/src/core/schema-stored/schema.ts @@ -221,10 +221,12 @@ export class ObjectNodeStoredSchema extends TreeNodeStoredSchema { public readonly objectNodeFields: ReadonlyMap, metadata?: PersistedMetadataFormat | undefined, ) { + console.log("Constructing ObjectNodeStoredSchema; metadata:", metadata !== undefined ? JSON.stringify(metadata) : "[undefined]"); super(metadata); } public override encodeV1(): TreeNodeSchemaDataFormatV1 { + console.log("In encodeV1"); const fieldsObject: Record = Object.create(null); // Sort fields to ensure output is identical for for equivalent schema (since field order is not considered significant). // This makes comparing schema easier, and ensures chunk reuse for schema summaries isn't needlessly broken. @@ -240,12 +242,14 @@ export class ObjectNodeStoredSchema extends TreeNodeStoredSchema { value, }); } + console.log("Returning from encodeV1", JSON.stringify(fieldsObject)); return { object: fieldsObject, }; } public override encodeV2(): TreeNodeSchemaDataFormatV2 { + console.log("In encodeV2"); const fieldsObject: Record = Object.create(null); // Sort fields to ensure output is identical for for equivalent schema (since field order is not considered significant). // This makes comparing schema easier, and ensures chunk reuse for schema summaries isn't needlessly broken. @@ -264,6 +268,8 @@ export class ObjectNodeStoredSchema extends TreeNodeStoredSchema { const kind = { object: fieldsObject }; + console.log("Returning from encodeV2", JSON.stringify(fieldsObject)); + console.log("Returning from encodeV2, metadata:", this.metadata !== undefined ? JSON.stringify(this.metadata) : "[undefined]"); // Omit metadata from the output if it is undefined return this.metadata !== undefined ? { kind, metadata: this.metadata } : { kind }; } diff --git a/packages/dds/tree/src/shared-tree-core/sharedTreeCore.ts b/packages/dds/tree/src/shared-tree-core/sharedTreeCore.ts index eb9857eeb755..6cef462a0d95 100644 --- a/packages/dds/tree/src/shared-tree-core/sharedTreeCore.ts +++ b/packages/dds/tree/src/shared-tree-core/sharedTreeCore.ts @@ -213,10 +213,14 @@ export class SharedTreeCore telemetryContext?: ITelemetryContext, incrementalSummaryContext?: IExperimentalIncrementalSummaryContext, ): ISummaryTreeWithStats { + console.log("In summarizeCore"); const builder = new SummaryTreeBuilder(); const summarizableBuilder = new SummaryTreeBuilder(); // Merge the summaries of all summarizables together under a single ISummaryTree for (const s of this.summarizables) { + if (s.key === "Schema") { + console.log("Adding summarizable for summary key:", s.key); + } summarizableBuilder.addWithStats( s.key, s.getAttachSummary( @@ -229,8 +233,11 @@ export class SharedTreeCore ); } + console.log("Adding summarizablesTreeKey to summary", summarizablesTreeKey); builder.addWithStats(summarizablesTreeKey, summarizableBuilder.getSummaryTree()); - return builder.getSummaryTree(); + const result = builder.getSummaryTree(); + console.log("Returning summary", JSON.stringify(result)); + return result; } public async loadCore(services: IChannelStorageService): Promise { diff --git a/packages/dds/tree/src/shared-tree/sharedTree.ts b/packages/dds/tree/src/shared-tree/sharedTree.ts index cc09943e3868..ba67ad52f78f 100644 --- a/packages/dds/tree/src/shared-tree/sharedTree.ts +++ b/packages/dds/tree/src/shared-tree/sharedTree.ts @@ -186,7 +186,7 @@ const formatVersionToTopLevelCodecVersions = new Map= 2.0.0. */ v3: 3, + + /** + * Requires \@fluidframework/tree \>= 2.0.0. + */ + v4: 4, } as const; /** @@ -697,7 +703,7 @@ export const defaultSharedTreeOptions: Required = { jsonValidator: noopValidator, forest: ForestTypeReference, treeEncodeType: TreeCompressionStrategy.Compressed, - formatVersion: SharedTreeFormatVersion.v3, + formatVersion: SharedTreeFormatVersion.v4, disposeForksAfterTransaction: true, }; From 61322c9e43906d2cadea79c21327d0afa28484c4 Mon Sep 17 00:00:00 2001 From: Alex Villarreal <716334+alexvy86@users.noreply.github.com> Date: Sun, 1 Jun 2025 13:10:43 -0500 Subject: [PATCH 39/45] Formatting --- packages/dds/tree/src/core/schema-stored/schema.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/dds/tree/src/core/schema-stored/schema.ts b/packages/dds/tree/src/core/schema-stored/schema.ts index 4f33bb669eff..6a339254c761 100644 --- a/packages/dds/tree/src/core/schema-stored/schema.ts +++ b/packages/dds/tree/src/core/schema-stored/schema.ts @@ -221,7 +221,10 @@ export class ObjectNodeStoredSchema extends TreeNodeStoredSchema { public readonly objectNodeFields: ReadonlyMap, metadata?: PersistedMetadataFormat | undefined, ) { - console.log("Constructing ObjectNodeStoredSchema; metadata:", metadata !== undefined ? JSON.stringify(metadata) : "[undefined]"); + console.log( + "Constructing ObjectNodeStoredSchema; metadata:", + metadata !== undefined ? JSON.stringify(metadata) : "[undefined]", + ); super(metadata); } @@ -269,7 +272,10 @@ export class ObjectNodeStoredSchema extends TreeNodeStoredSchema { const kind = { object: fieldsObject }; console.log("Returning from encodeV2", JSON.stringify(fieldsObject)); - console.log("Returning from encodeV2, metadata:", this.metadata !== undefined ? JSON.stringify(this.metadata) : "[undefined]"); + console.log( + "Returning from encodeV2, metadata:", + this.metadata !== undefined ? JSON.stringify(this.metadata) : "[undefined]", + ); // Omit metadata from the output if it is undefined return this.metadata !== undefined ? { kind, metadata: this.metadata } : { kind }; } From 362437939a7d16d70f897a002e2c898df34b6bb2 Mon Sep 17 00:00:00 2001 From: Alex Villarreal <716334+alexvy86@users.noreply.github.com> Date: Sun, 1 Jun 2025 13:52:07 -0500 Subject: [PATCH 40/45] Revert funky changes in API reports --- .../dds/tree/api-report/tree.alpha.api.md | 1 + .../api-report/fluid-framework.alpha.api.md | 54 +++++++++++++------ 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/packages/dds/tree/api-report/tree.alpha.api.md b/packages/dds/tree/api-report/tree.alpha.api.md index b32bc6a40395..580d98af51ad 100644 --- a/packages/dds/tree/api-report/tree.alpha.api.md +++ b/packages/dds/tree/api-report/tree.alpha.api.md @@ -844,6 +844,7 @@ export const SharedTreeFormatVersion: { readonly v1: 1; readonly v2: 2; readonly v3: 3; + readonly v4: 4; }; // @alpha diff --git a/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md b/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md index c19b8eb1d416..568217867f54 100644 --- a/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md +++ b/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md @@ -225,6 +225,11 @@ export interface FieldProps { readonly metadata?: FieldSchemaMetadata; } +// @alpha +export interface FieldPropsAlpha extends FieldProps { + readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; +} + // @public @sealed export class FieldSchema { protected constructor( @@ -242,13 +247,14 @@ export class FieldSchema extends FieldSchema implements SimpleFieldSchema { - protected constructor(kind: Kind, types: Types, annotatedAllowedTypes: ImplicitAnnotatedAllowedTypes, props?: FieldProps); + protected constructor(kind: Kind, types: Types, annotatedAllowedTypes: ImplicitAnnotatedAllowedTypes, props?: FieldPropsAlpha); // (undocumented) get allowedTypesIdentifiers(): ReadonlySet; readonly allowedTypesMetadata: AllowedTypesMetadata; // (undocumented) readonly annotatedAllowedTypes: ImplicitAnnotatedAllowedTypes; get annotatedAllowedTypeSet(): ReadonlyMap; + get persistedMetadata(): JsonCompatibleReadOnlyObject | undefined; } // @alpha @sealed @system @@ -951,6 +957,11 @@ export interface NodeSchemaOptions { readonly metadata?: NodeSchemaMetadata | undefined; } +// @alpha @sealed +export interface NodeSchemaOptionsAlpha extends NodeSchemaOptions { + readonly persistedMetadata?: JsonCompatibleReadOnlyObject | undefined; +} + // @alpha export const noopValidator: JsonValidator; @@ -1124,30 +1135,33 @@ export class SchemaFactory extends SchemaFactory { - arrayAlpha(name: Name, allowedTypes: T, options?: NodeSchemaOptions): ArrayNodeCustomizableSchema, T, true, TCustomMetadata>; + arrayAlpha(name: Name, allowedTypes: T, options?: NodeSchemaOptionsAlpha): ArrayNodeCustomizableSchema, T, true, TCustomMetadata>; arrayRecursive(name: Name, allowedTypes: T, options?: NodeSchemaOptions): ArrayNodeCustomizableSchemaUnsafe, T, TCustomMetadata>; - static readonly identifier: (props?: Omit, "defaultProvider"> | undefined) => FieldSchemaAlpha_2 & SimpleLeafNodeSchema_2, TCustomMetadata>; + static readonly identifier: (props?: Omit, "defaultProvider"> | undefined) => FieldSchemaAlpha & SimpleLeafNodeSchema_2, TCustomMetadata>; static readonly leaves: readonly [LeafSchema_2<"string", string> & SimpleLeafNodeSchema_2, LeafSchema_2<"number", number> & SimpleLeafNodeSchema_2, LeafSchema_2<"boolean", boolean> & SimpleLeafNodeSchema_2, LeafSchema_2<"null", null> & SimpleLeafNodeSchema_2, LeafSchema_2<"handle", IFluidHandle_2> & SimpleLeafNodeSchema_2]; - mapAlpha(name: Name, allowedTypes: T, options?: NodeSchemaOptions): MapNodeCustomizableSchema, T, true, TCustomMetadata>; + mapAlpha(name: Name, allowedTypes: T, options?: NodeSchemaOptionsAlpha): MapNodeCustomizableSchema, T, true, TCustomMetadata>; mapRecursive(name: Name, allowedTypes: T, options?: NodeSchemaOptions): MapNodeCustomizableSchemaUnsafe, T, TCustomMetadata>; objectAlpha, const TCustomMetadata = unknown>(name: Name, fields: T, options?: SchemaFactoryObjectOptions): ObjectNodeSchema, T, true, TCustomMetadata> & { readonly createFromInsertable: unknown; }; objectRecursive, const TCustomMetadata = unknown>(name: Name, t: T, options?: SchemaFactoryObjectOptions): TreeNodeSchemaClass, NodeKind.Object, System_Unsafe.TreeObjectNodeUnsafe>, object & System_Unsafe.InsertableObjectFromSchemaRecordUnsafe, false, T, never, TCustomMetadata> & SimpleObjectNodeSchema & Pick; static readonly optional: { - (t: T, props?: Omit, "defaultProvider"> | undefined): FieldSchemaAlpha_2; - (t: T_1, props?: Omit, "defaultProvider"> | undefined): FieldSchemaAlpha_2, TCustomMetadata_1>; + (t: T, props?: Omit, "defaultProvider"> | undefined): FieldSchemaAlpha; + (t: T_1, props?: Omit, "defaultProvider"> | undefined): FieldSchemaAlpha, TCustomMetadata_1>; }; - static readonly optionalRecursive: (t: T, props?: Omit, "defaultProvider"> | undefined) => FieldSchemaAlphaUnsafe_2; + optionalAlpha(t: T, props?: Omit, "defaultProvider">): FieldSchemaAlpha, TCustomMetadata>; + static readonly optionalRecursive: (t: T, props?: Omit, "defaultProvider"> | undefined) => FieldSchemaAlphaUnsafe; + optionalRecursiveAlpha(t: T, props?: Omit, "defaultProvider">): FieldSchemaAlphaUnsafe; static readonly required: { - (t: T, props?: Omit, "defaultProvider"> | undefined): FieldSchemaAlpha_2; - (t: T_1, props?: Omit, "defaultProvider"> | undefined): FieldSchemaAlpha_2, TCustomMetadata_1>; + (t: T, props?: Omit, "defaultProvider"> | undefined): FieldSchemaAlpha; + (t: T_1, props?: Omit, "defaultProvider"> | undefined): FieldSchemaAlpha, TCustomMetadata_1>; }; + requiredAlpha(t: T, props?: Omit, "defaultProvider">): FieldSchemaAlpha, TCustomMetadata>; scopedFactory(name: T): SchemaFactoryAlpha, TNameInner>; } // @alpha -export interface SchemaFactoryObjectOptions extends NodeSchemaOptions { +export interface SchemaFactoryObjectOptions extends NodeSchemaOptionsAlpha { allowUnknownOptionalFields?: boolean; } @@ -1198,6 +1212,7 @@ export const SharedTreeFormatVersion: { readonly v1: 1; readonly v2: 2; readonly v3: 3; + readonly v4: 4; }; // @alpha @@ -1207,7 +1222,7 @@ export type SharedTreeFormatVersion = typeof SharedTreeFormatVersion; export type SharedTreeOptions = Partial & Partial & ForestOptions; // @alpha @sealed -export interface SimpleArrayNodeSchema extends SimpleNodeSchemaBase { +export interface SimpleArrayNodeSchema extends SimpleNodeSchemaBaseAlpha { readonly allowedTypesIdentifiers: ReadonlySet; } @@ -1219,12 +1234,12 @@ export interface SimpleFieldSchema { } // @alpha @sealed -export interface SimpleLeafNodeSchema extends SimpleNodeSchemaBase { +export interface SimpleLeafNodeSchema extends SimpleNodeSchemaBaseAlpha { readonly leafKind: ValueSchema; } // @alpha @sealed -export interface SimpleMapNodeSchema extends SimpleNodeSchemaBase { +export interface SimpleMapNodeSchema extends SimpleNodeSchemaBaseAlpha { readonly allowedTypesIdentifiers: ReadonlySet; } @@ -1237,13 +1252,18 @@ export interface SimpleNodeSchemaBase; } +// @alpha @sealed @system +export interface SimpleNodeSchemaBaseAlpha extends SimpleNodeSchemaBase { + readonly persistedMetadata: JsonCompatibleReadOnlyObject | undefined; +} + // @alpha @sealed export interface SimpleObjectFieldSchema extends SimpleFieldSchema { readonly storedKey: string; } // @alpha @sealed -export interface SimpleObjectNodeSchema extends SimpleNodeSchemaBase { +export interface SimpleObjectNodeSchema extends SimpleNodeSchemaBaseAlpha { readonly fields: ReadonlyMap; } @@ -1276,7 +1296,7 @@ export namespace System_TableSchema { props: InsertableTreeFieldFromImplicitField>; }), true, { readonly props: TPropsSchema; - readonly id: FieldSchema_2, unknown>; + readonly id: FieldSchema_2, unknown>; }>; // @system export type CreateRowOptionsBase = OptionsWithSchemaFactory & OptionsWithCellSchema; @@ -1290,8 +1310,8 @@ export namespace System_TableSchema { props: InsertableTreeFieldFromImplicitField>; }), true, { readonly props: TPropsSchema; - readonly id: FieldSchema_2, unknown>; - readonly cells: FieldSchema_2, "Row.cells">, NodeKind.Map, TreeMapNode_2 & WithType, "Row.cells">, NodeKind.Map, unknown>, MapNodeInsertableData_2, true, TCellSchema, undefined>, unknown>; + readonly id: FieldSchema_2, unknown>; + readonly cells: FieldSchema_2, "Row.cells">, NodeKind.Map, TreeMapNode_2 & WithType, "Row.cells">, NodeKind.Map, unknown>, MapNodeInsertableData_2, true, TCellSchema, undefined>, unknown>; }>; // @system export function createTableSchema, const TRowSchema extends RowSchemaBase>(inputSchemaFactory: SchemaFactoryAlpha, _cellSchema: TCellSchema, columnSchema: TColumnSchema, rowSchema: TRowSchema): TreeNodeSchemaCore_2, "Table">, NodeKind.Object, true, { From e11beed36ab04d3eb7d8b6ee7122f547366f549b Mon Sep 17 00:00:00 2001 From: Alex Villarreal <716334+alexvy86@users.noreply.github.com> Date: Sun, 1 Jun 2025 14:44:58 -0500 Subject: [PATCH 41/45] Skip v4 tests --- packages/dds/tree/src/test/snapshots/summary.spec.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/dds/tree/src/test/snapshots/summary.spec.ts b/packages/dds/tree/src/test/snapshots/summary.spec.ts index 9befc24496a9..db6729ac3c96 100644 --- a/packages/dds/tree/src/test/snapshots/summary.spec.ts +++ b/packages/dds/tree/src/test/snapshots/summary.spec.ts @@ -18,6 +18,10 @@ describe("snapshot tests", () => { // Friendly description of tree encoding type const treeEncodeKey = TreeCompressionStrategy[treeEncodeType]; for (const formatVersionKey of Object.keys(SharedTreeFormatVersion)) { + // Skipping tests for v4 since we don't have the snapshots for it yet + if (formatVersionKey === "v4") { + continue; + } describe(`Using TreeCompressionStrategy.${treeEncodeKey} and SharedTreeFormatVersion.${formatVersionKey}`, () => { useSnapshotDirectory(`summary/${treeEncodeKey}/${formatVersionKey}`); const options: SharedTreeOptions = { From 3156997d253c897df0e37b0177a1d34cc9ff422c Mon Sep 17 00:00:00 2001 From: Alex Villarreal <716334+alexvy86@users.noreply.github.com> Date: Sun, 1 Jun 2025 15:27:55 -0500 Subject: [PATCH 42/45] Skip the other v4 tests --- packages/dds/tree/src/test/snapshots/opFormat.spec.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/dds/tree/src/test/snapshots/opFormat.spec.ts b/packages/dds/tree/src/test/snapshots/opFormat.spec.ts index 096d4cce8514..66d55260461a 100644 --- a/packages/dds/tree/src/test/snapshots/opFormat.spec.ts +++ b/packages/dds/tree/src/test/snapshots/opFormat.spec.ts @@ -44,6 +44,10 @@ describe("SharedTree op format snapshots", () => { let tree: ITree & IChannel; for (const versionKey of Object.keys(SharedTreeFormatVersion)) { + // Skipping tests for v4 since we don't have the snapshots for it yet + if (versionKey === "v4") { + continue; + } describe(`using SharedTreeFormatVersion.${versionKey}`, () => { useSnapshotDirectory(`op-format/${versionKey}`); beforeEach(() => { From e4c8be8e441d7e29da3de5480e81374ce1c41b6b Mon Sep 17 00:00:00 2001 From: Alex Villarreal <716334+alexvy86@users.noreply.github.com> Date: Sun, 1 Jun 2025 15:34:09 -0500 Subject: [PATCH 43/45] Formatting --- packages/dds/tree/src/test/snapshots/opFormat.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/dds/tree/src/test/snapshots/opFormat.spec.ts b/packages/dds/tree/src/test/snapshots/opFormat.spec.ts index 66d55260461a..89571f8763b3 100644 --- a/packages/dds/tree/src/test/snapshots/opFormat.spec.ts +++ b/packages/dds/tree/src/test/snapshots/opFormat.spec.ts @@ -45,9 +45,9 @@ describe("SharedTree op format snapshots", () => { for (const versionKey of Object.keys(SharedTreeFormatVersion)) { // Skipping tests for v4 since we don't have the snapshots for it yet - if (versionKey === "v4") { - continue; - } + if (versionKey === "v4") { + continue; + } describe(`using SharedTreeFormatVersion.${versionKey}`, () => { useSnapshotDirectory(`op-format/${versionKey}`); beforeEach(() => { From fe13eac01ba84ba6a0dae04879b0df25bb5d1af5 Mon Sep 17 00:00:00 2001 From: Alex Villarreal <716334+alexvy86@users.noreply.github.com> Date: Sun, 1 Jun 2025 16:03:46 -0500 Subject: [PATCH 44/45] Include metadata when exporting stored schema as simple schema --- packages/dds/tree/src/shared-tree/sharedTree.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/dds/tree/src/shared-tree/sharedTree.ts b/packages/dds/tree/src/shared-tree/sharedTree.ts index ba67ad52f78f..56c9959c19bd 100644 --- a/packages/dds/tree/src/shared-tree/sharedTree.ts +++ b/packages/dds/tree/src/shared-tree/sharedTree.ts @@ -743,7 +743,7 @@ function exportSimpleFieldSchemaStored(schema: TreeFieldStoredSchema): SimpleFie default: fail(0xaca /* invalid field kind */); } - return { kind, allowedTypesIdentifiers: schema.types, metadata: {} }; + return { kind, allowedTypesIdentifiers: schema.types, metadata: schema.metadata }; } function exportSimpleNodeSchemaStored(schema: TreeNodeStoredSchema): SimpleNodeSchema { @@ -753,7 +753,7 @@ function exportSimpleNodeSchemaStored(schema: TreeNodeStoredSchema): SimpleNodeS kind: NodeKind.Array, allowedTypesIdentifiers: arrayTypes, metadata: {}, - persistedMetadata: undefined, + persistedMetadata: schema.metadata, }; } if (schema instanceof ObjectNodeStoredSchema) { @@ -761,7 +761,7 @@ function exportSimpleNodeSchemaStored(schema: TreeNodeStoredSchema): SimpleNodeS for (const [storedKey, field] of schema.objectNodeFields) { fields.set(storedKey, { ...exportSimpleFieldSchemaStored(field), storedKey }); } - return { kind: NodeKind.Object, fields, metadata: {}, persistedMetadata: undefined }; + return { kind: NodeKind.Object, fields, metadata: {}, persistedMetadata: schema.metadata }; } if (schema instanceof MapNodeStoredSchema) { assert( @@ -772,7 +772,7 @@ function exportSimpleNodeSchemaStored(schema: TreeNodeStoredSchema): SimpleNodeS kind: NodeKind.Map, allowedTypesIdentifiers: schema.mapFields.types, metadata: {}, - persistedMetadata: undefined, + persistedMetadata: schema.metadata, }; } if (schema instanceof LeafNodeStoredSchema) { @@ -780,7 +780,7 @@ function exportSimpleNodeSchemaStored(schema: TreeNodeStoredSchema): SimpleNodeS kind: NodeKind.Leaf, leafKind: schema.leafValue, metadata: {}, - persistedMetadata: undefined, + persistedMetadata: schema.metadata, }; } fail(0xacb /* invalid schema kind */); From 5e88b30ff2f8d31bc8bbc473e0af1606d4c6521f Mon Sep 17 00:00:00 2001 From: Alex Villarreal <716334+alexvy86@users.noreply.github.com> Date: Sun, 1 Jun 2025 17:02:28 -0500 Subject: [PATCH 45/45] Hack to get persistedMetadata to get encoded (in the wrong place) for fields --- packages/dds/tree/src/core/schema-stored/schema.ts | 8 +++++++- packages/dds/tree/src/simple-tree/simpleSchema.ts | 4 ++++ packages/dds/tree/src/simple-tree/toStoredSchema.ts | 3 ++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/dds/tree/src/core/schema-stored/schema.ts b/packages/dds/tree/src/core/schema-stored/schema.ts index 6a339254c761..c2d1c76a2263 100644 --- a/packages/dds/tree/src/core/schema-stored/schema.ts +++ b/packages/dds/tree/src/core/schema-stored/schema.ts @@ -223,7 +223,8 @@ export class ObjectNodeStoredSchema extends TreeNodeStoredSchema { ) { console.log( "Constructing ObjectNodeStoredSchema; metadata:", - metadata !== undefined ? JSON.stringify(metadata) : "[undefined]", + metadata !== undefined ? JSON.stringify(metadata) : "[undefined]", "; objectNodeFields:", + JSON.stringify([...objectNodeFields.entries()]), ); super(metadata); } @@ -373,6 +374,10 @@ export const storedSchemaDecodeDispatcher: DiscriminatedUnionDispatcher< leaf: (data: PersistedValueSchema) => (metadata) => new LeafNodeStoredSchema(decodeValueSchema(data)), object: (data: Record) => (metadata) => { + console.log( + "Processing object node inside storedSchemaDecodeDispatcher; field schema data:", + JSON.stringify(data), + ); const map = new Map(); for (const [key, value] of Object.entries(data)) { map.set(key, decodeFieldSchema(value)); @@ -412,6 +417,7 @@ export function encodeFieldSchemaV1(schema: TreeFieldStoredSchema): FieldSchemaF export function encodeFieldSchemaV2(schema: TreeFieldStoredSchema): FieldSchemaFormatV2 { const fieldSchema: FieldSchemaFormatV1 = encodeFieldSchemaV1(schema); + console.log("Inside encodeFieldSchemaV2; schema:", JSON.stringify(schema)); // Omit metadata from the output if it is undefined return schema.metadata !== undefined ? { ...fieldSchema, metadata: schema.metadata } diff --git a/packages/dds/tree/src/simple-tree/simpleSchema.ts b/packages/dds/tree/src/simple-tree/simpleSchema.ts index 4839b9369879..4eb44fe1dda9 100644 --- a/packages/dds/tree/src/simple-tree/simpleSchema.ts +++ b/packages/dds/tree/src/simple-tree/simpleSchema.ts @@ -168,6 +168,10 @@ export type SimpleNodeSchema = * * @alpha * @sealed + * + * @privateRemarks + * Seems like we need an Alpha version of this one that has persistedMetadata so we can avoid the hack + * in toStoredSchema.ts line 93 */ export interface SimpleFieldSchema { /** diff --git a/packages/dds/tree/src/simple-tree/toStoredSchema.ts b/packages/dds/tree/src/simple-tree/toStoredSchema.ts index dd4cd1196d50..dece7c687746 100644 --- a/packages/dds/tree/src/simple-tree/toStoredSchema.ts +++ b/packages/dds/tree/src/simple-tree/toStoredSchema.ts @@ -89,7 +89,8 @@ export function convertField(schema: SimpleFieldSchema): TreeFieldStoredSchema { const kind: FieldKindIdentifier = convertFieldKind.get(schema.kind)?.identifier ?? fail(0xae3 /* Invalid field kind */); const types: TreeTypeSet = schema.allowedTypesIdentifiers as TreeTypeSet; - return { kind, types, metadata: undefined }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Temporary hack + return { kind, types, metadata: (schema as unknown as any)?.persistedMetadata ?? undefined }; } const convertFieldKind = new Map([