Support for model migrations

CHANGELOG.md

@@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres _loosely_ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
-## [Unreleased]
+## [0.3.0] - tbd
 
 ### Added
 * Added `LionWebVersions` _2025.1_ and _2025.1-compatible_. No content differences to _2024.1_ so far.
@@ -22,6 +22,19 @@ and this project adheres _loosely_ to [Semantic Versioning](https://semver.org/s
 * Added `M2Extensions.FindByKey()` to search for arbitrary keyed elements in a language.
 * Via `ReflectiveBaseNodeFactory.CreateEnumLiteral()` dynamically created C# enums 
   now have proper `LionCoreMetaPointer` attributes.
+* Support for model migrations:
+  * `IModelMigrator.Migrate()` takes a serialized model, runs all registered `IMigration`s, and serializes the result.
+    It works completely with `LenientNode`s and `DynamicLanguage`s (handled by `ModelMigrator`). 
+  * Implementations of `IMigration` provide the actual migrations.
+  * `MigrationBase` provides a lot of infrastructure to migrate to a new version of a language.
+  * `MigrationExtensions` provides static infrastructure methods.
+  * `ILanguageRegistry`, available to all `IMigration`s, provides access to languages present during migration.
+  * `DynamicLanguageCloner` creates a `DynamicLanguage` clone of any language.
+  * We provide two built-in migrations:
+    * `LionWebVersionMigration` migrates to a newer version of LionWeb standard.
+    * `KeysMigration` migrates changed `IKeyed.Key`s.
+  * Have a look at [plugin loading](https://learn.microsoft.com/en-us/dotnet/core/tutorials/creating-app-with-plugin-support)
+    to load several versions of the same language in parallel.
 ### Fixed
 * `LenientNode` now works properly if keys of features change.
 * Deserializer can now create instances of languages not registered beforehand.
@@ -36,7 +49,7 @@ and this project adheres _loosely_ to [Semantic Versioning](https://semver.org/s
 ### Deprecated
 ### Security
 
-## [0.2.4] - tbd
+## [0.2.4] - 2025-03-13
 
 ### Added
 * Introduced optionally compressed ids during deserialization.

src/LionWeb.Core/Core/BaseTypes.cs

@@ -46,7 +46,7 @@ public interface IReadableNode
     /// <seealso cref="IWritableNode.InsertAnnotations"/>
     /// <seealso cref="IWritableNode.RemoveAnnotations"/>
     public IReadOnlyList<IReadableNode> GetAnnotations();
-
+    
     /// The <see cref="Classifier"/> that <c>this</c> node is an instance of.
     public Classifier GetClassifier();
 

src/LionWeb.Core/Core/Migration/DynamicLanguageCloner.cs

@@ -0,0 +1,299 @@
+// Copyright 2025 TRUMPF Laser SE and other contributors
+// 
+// Licensed under the Apache License, Version 2.0 (the "License")
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// 
+// SPDX-FileCopyrightText: 2024 TRUMPF Laser SE and other contributors
+// SPDX-License-Identifier: Apache-2.0
+
+namespace LionWeb.Core.Migration;
+
+using M2;
+using M3;
+using System.Diagnostics.CodeAnalysis;
+using Utilities;
+
+/// Clones Languages as <see cref="DynamicLanguage">DynamicLanguages</see> based on <paramref name="lionWebVersion"/>.
+public class DynamicLanguageCloner(LionWebVersions lionWebVersion)
+{
+    private readonly Dictionary<IKeyed, DynamicIKeyed?> _dynamicMap = new(new KeyedIdentityComparer());
+
+    /// Provides a mapping of all cloned elements to their clones.
+    public IReadOnlyDictionary<IKeyed, DynamicIKeyed> DynamicMap =>
+        _dynamicMap
+            .Where(p => p.Value != null)
+            .ToDictionary()
+            .AsReadOnly()!;
+
+    /// Clones all of <paramref name="languages"/> as <see cref="DynamicLanguage">DynamicLanguages</see> based on <see cref="lionWebVersion"/>.
+    /// <paramref name="languages"/> MUST be self-contained, i.e. no language might refer to another language outside <paramref name="languages"/>.
+    public Dictionary<LanguageIdentity, DynamicLanguage> Clone(IEnumerable<Language> languages)
+    {
+        CreateClones(languages);
+        CloneReferencedElements();
+        ResolveReferences();
+
+        return _dynamicMap
+            .Values
+            .OfType<DynamicLanguage>()
+            .ToDictionary(LanguageIdentity.FromLanguage, l => l);
+    }
+
+    #region Cloning
+
+    private void CreateClones(IEnumerable<Language> languages)
+    {
+        foreach (var l in languages)
+        {
+            DynamicLanguage dynamicLanguage = CloneLanguage(l);
+            dynamicLanguage.AddEntities(l.Entities.Select(e => CloneEntity(e, dynamicLanguage)));
+        }
+    }
+
+    private DynamicLanguageEntity CloneEntity(LanguageEntity languageEntity, DynamicLanguage dynamicLanguage)
+    {
+        DynamicLanguageEntity entity = languageEntity switch
+        {
+            Annotation a => CloneAnnotation(a, dynamicLanguage),
+            Concept c => CloneConcept(c, dynamicLanguage),
+            Interface i => CloneInterface(i, dynamicLanguage),
+            Enumeration e => CloneEnumeration(e, dynamicLanguage),
+            PrimitiveType p => ClonePrimitiveType(p, dynamicLanguage),
+            StructuredDataType s => CloneStructuredDataType(s, dynamicLanguage),
+            _ => throw new ArgumentOutOfRangeException(languageEntity.ToString())
+        };
+        _dynamicMap[languageEntity] = entity;
+        return entity;
+    }
+
+    private DynamicLanguage CloneLanguage(Language language)
+    {
+        var result = new DynamicLanguage(language.GetId(), lionWebVersion)
+        {
+            Name = language.Name, Key = language.Key, Version = language.Version,
+        };
+        result.SetFactory(new MigrationFactory(result));
+        _dynamicMap[language] = result;
+        return result;
+    }
+
+    private DynamicAnnotation CloneAnnotation(Annotation a, DynamicLanguage language)
+    {
+        var result = new DynamicAnnotation(a.GetId(), lionWebVersion, language) { Name = a.Name, Key = a.Key };
+        result.AddFeatures(a.Features.Select(CloneFeature));
+        return result;
+    }
+
+    private DynamicConcept CloneConcept(Concept c, DynamicLanguage language)
+    {
+        var result = new DynamicConcept(c.GetId(), lionWebVersion, language)
+        {
+            Name = c.Name, Key = c.Key, Abstract = c.Abstract, Partition = c.Partition,
+        };
+        result.AddFeatures(c.Features.Select(CloneFeature));
+        return result;
+    }
+
+    private DynamicInterface CloneInterface(Interface i, DynamicLanguage language)
+    {
+        var result = new DynamicInterface(i.GetId(), lionWebVersion, language) { Name = i.Name, Key = i.Key };
+        result.AddFeatures(i.Features.Select(CloneFeature));
+        return result;
+    }
+
+    private DynamicEnumeration CloneEnumeration(Enumeration enm, DynamicLanguage language)
+    {
+        var result = new DynamicEnumeration(enm.GetId(), lionWebVersion, language) { Name = enm.Name, Key = enm.Key };
+        result.AddLiterals(enm.Literals.Select(lit =>
+        {
+            var r = new DynamicEnumerationLiteral(lit.GetId(), lionWebVersion, result)
+            {
+                Name = lit.Name, Key = lit.Key
+            };
+            _dynamicMap.Add(lit, r);
+            return r;
+        }));
+        return result;
+    }
+
+    private DynamicLanguageEntity ClonePrimitiveType(PrimitiveType p, DynamicLanguage language) =>
+        new DynamicPrimitiveType(p.Key, lionWebVersion, language) { Name = p.Name, Key = p.Key };
+
+    private DynamicStructuredDataType CloneStructuredDataType(StructuredDataType sdt, DynamicLanguage language)
+    {
+        var result = new DynamicStructuredDataType(sdt.Key, lionWebVersion, language)
+        {
+            Name = sdt.Name, Key = sdt.Key
+        };
+        result.AddFields(sdt.Fields.Select<Field, DynamicField>(f =>
+        {
+            var field = new DynamicField(f.GetId(), lionWebVersion, result) { Name = f.Name, Key = f.Key };
+            _dynamicMap.Add(f, field);
+            _dynamicMap.TryAdd(f.Type, null);
+            return field;
+        }));
+        return result;
+    }
+
+    private DynamicFeature CloneFeature(Feature f)
+    {
+        var result = (DynamicFeature)(f switch
+        {
+            Property p => new DynamicProperty(p.GetId(), lionWebVersion, null)
+            {
+                Name = p.Name, Key = p.Key, Optional = p.Optional
+            },
+            Containment c => new DynamicContainment(c.GetId(), lionWebVersion, null)
+            {
+                Name = c.Name, Key = c.Key, Optional = c.Optional, Multiple = c.Multiple
+            },
+            Reference r => new DynamicReference(r.GetId(), lionWebVersion, null)
+            {
+                Name = r.Name, Key = r.Key, Optional = r.Optional, Multiple = r.Multiple
+            },
+            _ => throw new ArgumentOutOfRangeException(f.ToString())
+        });
+        _dynamicMap.Add(f, result);
+        _dynamicMap.TryAdd(f.GetFeatureType(), null);
+        return result;
+    }
+
+    private void CloneReferencedElements()
+    {
+        List<IGrouping<Language, KeyValuePair<IKeyed, DynamicIKeyed?>>> unclonedReferencedElements = CollectUnclonedReferencedElements();
+
+        while (unclonedReferencedElements.Count != 0)
+        {
+            foreach (var grouping in unclonedReferencedElements)
+            {
+                Language inputLanguage = grouping.Key;
+                DynamicLanguage language;
+                if (TryLookup(inputLanguage, out var cloned) && cloned is DynamicLanguage l)
+                {
+                    language = l;
+                } else
+                {
+                    language = CloneLanguage(inputLanguage);
+                }
+
+                foreach ((IKeyed? keyed, _) in grouping)
+                {
+                    switch (keyed)
+                    {
+                        case LanguageEntity entity:
+                            var clonedEntity = CloneEntity(entity, language);
+                            language.AddEntities([clonedEntity]);
+                            break;
+
+                        default:
+                            throw new ArgumentException(keyed.ToString());
+                    }
+                }
+            }
+
+            unclonedReferencedElements = CollectUnclonedReferencedElements();
+        }
+
+        return;
+
+        List<IGrouping<Language, KeyValuePair<IKeyed, DynamicIKeyed?>>> CollectUnclonedReferencedElements() =>
+            _dynamicMap
+                .Where(p => p.Value == null)
+                .GroupBy(p => p.Key.GetLanguage(), new LanguageIdentityComparer())
+                .ToList();
+    }
+
+    #endregion
+
+    #region References
+
+    private void ResolveReferences()
+    {
+        foreach (var pair in _dynamicMap)
+        {
+            switch (pair)
+            {
+                case (Language i, DynamicLanguage r):
+                    r.AddDependsOn(i.DependsOn.Select(Lookup));
+                    break;
+                case (Annotation i, DynamicAnnotation r):
+                    r.Annotates = Lookup(i.Annotates);
+                    if (i.TryGetExtends(out var exA))
+                        r.Extends = Lookup(exA);
+                    r.AddImplements(i.Implements.Select(Lookup));
+                    break;
+                case (Concept i, DynamicConcept r):
+                    if (i.TryGetExtends(out var exC))
+                        r.Extends = Lookup(exC);
+                    r.AddImplements(i.Implements.Select(Lookup));
+                    break;
+                case (Interface i, DynamicInterface r):
+                    r.AddExtends(i.Extends.Select(Lookup));
+                    break;
+                case (Property i, DynamicProperty r):
+                    r.Type = Lookup(i.Type);
+                    break;
+                case (Link i, DynamicLink r):
+                    r.Type = Lookup(i.Type);
+                    break;
+                case (Field i, DynamicField r):
+                    r.Type = Lookup(i.Type);
+                    break;
+                case (Datatype, DynamicDatatype):
+                case (EnumerationLiteral, DynamicEnumerationLiteral):
+                    break;
+                default:
+                    throw new ArgumentOutOfRangeException(pair.ToString());
+            }
+        }
+    }
+
+    private T Lookup<T>(T keyed) where T : IKeyed?
+    {
+        if (keyed == null)
+            throw new UnknownLookupException(typeof(T).FullName!);
+
+#pragma warning disable CS8631 // The type cannot be used as type parameter in the generic type or method. Nullability of type argument doesn't match constraint type.
+        return TryLookup(keyed, out var result) ? result : throw new UnknownLookupException(keyed);
+#pragma warning restore CS8631
+    }
+
+    private bool TryLookup<T>(T keyed, [NotNullWhen(true)] out T? result) where T : IKeyed
+    {
+        var keyedLanguage = keyed.GetLanguage();
+        if (keyedLanguage.Key == lionWebVersion.BuiltIns.Key)
+        {
+            result = lionWebVersion.BuiltIns.FindByKey<T>(keyed.Key);
+            return true;
+        }
+
+        if (keyedLanguage.Key == lionWebVersion.LionCore.Key)
+        {
+            result = lionWebVersion.LionCore.FindByKey<T>(keyed.Key);
+            return true;
+        }
+
+        if (_dynamicMap.TryGetValue(keyed, out var value))
+        {
+            if (value is T r)
+            {
+                result = r;
+                return true;
+            }
+        }
+
+        result = default;
+        return false;
+    }
+
+    #endregion
+}

src/LionWeb.Core/Core/Migration/ILanguageRegistry.cs

@@ -0,0 +1,46 @@
+// Copyright 2025 TRUMPF Laser SE and other contributors
+// 
+// Licensed under the Apache License, Version 2.0 (the "License")
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// 
+// SPDX-FileCopyrightText: 2024 TRUMPF Laser SE and other contributors
+// SPDX-License-Identifier: Apache-2.0
+
+namespace LionWeb.Core.Migration;
+
+using M3;
+using System.Diagnostics.CodeAnalysis;
+
+/// Provides access to all languages known during the current <see cref="IModelMigrator">migration round</see>.
+public interface ILanguageRegistry
+{
+    /// Version of LionWeb standard used for migration.
+    LionWebVersions LionWebVersion { get; }
+
+    /// Enumerates all languages known in the current migration round.
+    IEnumerable<DynamicLanguage> KnownLanguages { get; }
+
+    /// Find language for <paramref name="languageIdentity"/>, if known.
+    /// <returns><c>true</c> if a language for <paramref name="languageIdentity"/> could be found; <c>false</c> otherwise.</returns>
+    bool TryGetLanguage(LanguageIdentity languageIdentity, [NotNullWhen(true)] out DynamicLanguage? language);
+
+    /// Adds a new language to the current migration round.
+    /// Uses <see cref="LanguageIdentity.FromLanguage"/> if <paramref name="languageIdentity"/> is <c>null</c>.
+    /// <returns><c>true</c> if the language has been added to the list of languages;
+    /// <c>false</c> if a language with <paramref name="languageIdentity"/> is already present.</returns>
+    bool RegisterLanguage(DynamicLanguage language, LanguageIdentity? languageIdentity = null);
+
+    /// Finds the equivalent of <paramref name="keyed"/> within the current round's languages.
+    /// <exception cref="UnknownLookupException">If no equivalent can be found for <paramref name="keyed"/>.</exception>
+    /// <exception cref="AmbiguousLanguageKeyMapping">If not exactly one language could be mapped to the same key.</exception>
+    T Lookup<T>(T keyed) where T : IKeyed;
+}