[ksqlDb.RestApi.Client]: added IgnoreInDML function to the fluent API, introduced IgnoreAttribute, and fixed the issue where IgnoreByInsertsAttribute incorrectly excluded properties from DDL statements #90

Author: tomasfabian

ksqlDb.RestApi.Client/FluentAPI/Builders/FieldTypeBuilder.cs

@@ -1,3 +1,4 @@
+using System.Linq.Expressions;
 using ksqlDb.RestApi.Client.Metadata;
 
 namespace ksqlDb.RestApi.Client.FluentAPI.Builders
@@ -9,11 +10,17 @@ namespace ksqlDb.RestApi.Client.FluentAPI.Builders
   public interface IFieldTypeBuilder<TProperty>
   {
     /// <summary>
-    /// Marks the field as ignored, excluding it from the entity's schema.
+    /// Marks the field as ignored, excluding it from the entity's schema, preventing it from being included in both DDL and DML statements. 
     /// </summary>
     /// <returns>The field type builder for chaining additional configuration.</returns>
     public IFieldTypeBuilder<TProperty> Ignore();
 
+    /// <summary>
+    /// Marks the field to be excluded from data manipulation operations, preventing it from being included in DML statements such as INSERT.
+    /// </summary>
+    /// <returns>The field type builder for chaining additional configuration.</returns>
+    public IFieldTypeBuilder<TProperty> IgnoreInDML();
+
     /// <summary>
     /// Marks the field as HEADERS.
     /// </summary>
@@ -55,9 +62,16 @@ public IFieldTypeBuilder<TProperty> Ignore()
       return this;
     }
 
+    public IFieldTypeBuilder<TProperty> IgnoreInDML()
+    {
+      fieldMetadata.IgnoreInDML = true;
+      return this;
+    }
+
     public IFieldTypeBuilder<TProperty> WithHeaders()
     {
       fieldMetadata.HasHeaders = true;
+      IgnoreInDML();
       return this;
     }
   }

ksqlDb.RestApi.Client/KSql/RestApi/Extensions/MemberInfoExtensions.cs

@@ -12,8 +12,8 @@ internal static class MemberInfoExtensions
     /// </summary>
     /// <param name="memberInfo"></param>
     /// <param name="escaping"></param>
-    /// <param name="modelBuilder"></param>
+    /// <param name="metadataProvider"></param>
     /// <returns>the <c>memberInfo.Name</c> modified based on the provided <c>format</c></returns>
-    public static string Format(this MemberInfo memberInfo, IdentifierEscaping escaping, ModelBuilder modelBuilder) => IdentifierUtil.Format(memberInfo, escaping, modelBuilder);
+    public static string Format(this MemberInfo memberInfo, IdentifierEscaping escaping, IMetadataProvider metadataProvider) => IdentifierUtil.Format(memberInfo, escaping, metadataProvider);
   }
 }

ksqlDb.RestApi.Client/KSql/RestApi/Statements/Annotations/IgnoreAttribute.cs

@@ -0,0 +1,7 @@
+namespace ksqlDB.RestApi.Client.KSql.RestApi.Statements.Annotations
+{
+  [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property, AllowMultiple = false, Inherited = true)]
+  public sealed class IgnoreAttribute : Attribute
+  {
+  }
+}

ksqlDb.RestApi.Client/KSql/RestApi/Statements/CreateInsert.cs

@@ -5,17 +5,19 @@
 using ksqlDB.RestApi.Client.KSql.RestApi.Statements.Inserts;
 using ksqlDB.RestApi.Client.KSql.RestApi.Statements.Properties;
 using ksqlDb.RestApi.Client.FluentAPI.Builders;
+using ksqlDB.RestApi.Client.KSql.RestApi.Statements.Annotations;
+using ksqlDb.RestApi.Client.Metadata;
 
 namespace ksqlDB.RestApi.Client.KSql.RestApi.Statements;
 
 internal sealed class CreateInsert : EntityInfo
 {
-  private readonly ModelBuilder modelBuilder;
+  private readonly IMetadataProvider metadataProvider;
 
-  public CreateInsert(ModelBuilder modelBuilder)
-    : base(modelBuilder)
+  public CreateInsert(ModelBuilder metadataProvider)
+    : base(metadataProvider)
   {
-    this.modelBuilder = modelBuilder;
+    this.metadataProvider = metadataProvider;
   }
 
   internal string Generate<T>(T entity, InsertProperties insertProperties)
@@ -50,11 +52,11 @@ internal string Generate<T>(InsertValues<T> insertValues, InsertProperties inser
         valuesStringBuilder.Append(", ");
       }
 
-      columnsStringBuilder.Append(memberInfo.Format(insertProperties.IdentifierEscaping, modelBuilder));
+      columnsStringBuilder.Append(memberInfo.Format(insertProperties.IdentifierEscaping, metadataProvider));
 
       var type = GetMemberType(memberInfo);
 
-      var value = GetValue(insertValues, insertProperties, memberInfo, type, mi => IdentifierUtil.Format(mi, insertProperties.IdentifierEscaping, modelBuilder));
+      var value = GetValue(insertValues, insertProperties, memberInfo, type, mi => IdentifierUtil.Format(mi, insertProperties.IdentifierEscaping, metadataProvider));
 
       valuesStringBuilder.Append(value);
     }
@@ -68,15 +70,24 @@ internal string Generate<T>(InsertValues<T> insertValues, InsertProperties inser
   private object GetValue<T>(InsertValues<T> insertValues, InsertProperties insertProperties,
     MemberInfo memberInfo, Type type, Func<MemberInfo, string> formatter)
   {
-    var hasValue = insertValues.PropertyValues.ContainsKey(memberInfo.Format(insertProperties.IdentifierEscaping, modelBuilder));
+    var hasValue = insertValues.PropertyValues.ContainsKey(memberInfo.Format(insertProperties.IdentifierEscaping, metadataProvider));
 
     object value;
 
     if (hasValue)
-      value = insertValues.PropertyValues[memberInfo.Format(insertProperties.IdentifierEscaping, modelBuilder)];
+      value = insertValues.PropertyValues[memberInfo.Format(insertProperties.IdentifierEscaping, metadataProvider)];
     else
-      value = new CreateKSqlValue(modelBuilder).ExtractValue(insertValues.Entity, insertProperties, memberInfo, type, formatter);
+      value = new CreateKSqlValue(metadataProvider).ExtractValue(insertValues.Entity, insertProperties, memberInfo, type, formatter);
 
     return value;
   }
+
+  protected override bool IncludeMemberInfo(EntityMetadata? entityMetadata, MemberInfo memberInfo)
+  {
+    var fieldMetadata = entityMetadata?.GetFieldMetadataBy(memberInfo);
+    if (fieldMetadata is {IgnoreInDML: true})
+      return false;
+
+    return base.IncludeMemberInfo(entityMetadata, memberInfo) && !memberInfo.GetCustomAttributes().OfType<IgnoreByInsertsAttribute>().Any();
+  }
 }

ksqlDb.RestApi.Client/KSql/RestApi/Statements/EntityInfo.cs

@@ -2,6 +2,7 @@
 using ksqlDb.RestApi.Client.FluentAPI.Builders;
 using ksqlDB.RestApi.Client.KSql.RestApi.Statements.Annotations;
 using ksqlDb.RestApi.Client.KSql.RestApi.Statements.Providers;
+using ksqlDb.RestApi.Client.Metadata;
 
 namespace ksqlDB.RestApi.Client.KSql.RestApi.Statements;
 
@@ -21,18 +22,19 @@ protected IEnumerable<MemberInfo> Members(Type type, bool? includeReadOnly = nul
     var properties = type
       .GetProperties(BindingFlags.Public | BindingFlags.Instance)
       .Where(c => c.CanWrite || (includeReadOnly.HasValue && includeReadOnly.Value))
-      .OfType<MemberInfo>()
-      .Concat(fields);
+      .OfType<MemberInfo>();
+
+    var members = properties.Concat(fields);
 
     var entityMetadata = metadataProvider.GetEntities().FirstOrDefault(c => c.Type == type);
-    
-    return properties.Where(c =>
-    {
-      var fieldMetadata = entityMetadata?.GetFieldMetadataBy(c);
-      if (fieldMetadata is {Ignore: true}) return false;
 
-      return !c.GetCustomAttributes().OfType<IgnoreByInsertsAttribute>().Any();
-    });
+    return members.Where(memberInfo => IncludeMemberInfo(entityMetadata, memberInfo));
+  }
+
+  protected virtual bool IncludeMemberInfo(EntityMetadata? entityMetadata, MemberInfo memberInfo)
+  {
+    var fieldMetadata = entityMetadata?.GetFieldMetadataBy(memberInfo);
+    return fieldMetadata is not {Ignore: true} && !memberInfo.GetCustomAttributes().OfType<IgnoreAttribute>().Any();
   }
 
   protected static Type GetMemberType(MemberInfo memberInfo)

ksqlDb.RestApi.Client/Metadata/BytesArrayFieldMetadata.cs

@@ -3,11 +3,8 @@ namespace ksqlDb.RestApi.Client.Metadata
   internal sealed record BytesArrayFieldMetadata : FieldMetadata
   {
     public BytesArrayFieldMetadata(FieldMetadata fieldMetadata)
+      : base(fieldMetadata)
     {
-      MemberInfo = fieldMetadata.MemberInfo;
-      Ignore = fieldMetadata.Ignore;
-      Path = fieldMetadata.Path;
-      FullPath = fieldMetadata.FullPath;
     }
 
     public string? Header { get; internal set; }

ksqlDb.RestApi.Client/Metadata/DecimalFieldMetadata.cs

@@ -3,11 +3,8 @@ namespace ksqlDb.RestApi.Client.Metadata
   internal sealed record DecimalFieldMetadata : FieldMetadata
   {
     public DecimalFieldMetadata(FieldMetadata fieldMetadata)
+     : base(fieldMetadata)
     {
-      MemberInfo = fieldMetadata.MemberInfo;
-      Ignore = fieldMetadata.Ignore;
-      Path = fieldMetadata.Path;
-      FullPath = fieldMetadata.FullPath;
     }
 
     public short Precision { get; internal set; }

ksqlDb.RestApi.Client/Metadata/FieldMetadata.cs

@@ -4,8 +4,21 @@ namespace ksqlDb.RestApi.Client.Metadata
 {
   internal record FieldMetadata
   {
+    public FieldMetadata(FieldMetadata fieldMetadata)
+    {
+      MemberInfo = fieldMetadata.MemberInfo;
+      Ignore = fieldMetadata.Ignore;
+      IgnoreInDML = fieldMetadata.IgnoreInDML;
+      HasHeaders = fieldMetadata.HasHeaders;
+      IsStruct = fieldMetadata.IsStruct;
+      Path = fieldMetadata.Path;
+      FullPath = fieldMetadata.FullPath;
+      ColumnName = fieldMetadata.ColumnName;
+    }
+
     internal MemberInfo MemberInfo { get; init; } = null!;
     public bool Ignore { get; internal set; }
+    public bool IgnoreInDML { get; internal set; }
     public bool HasHeaders { get; internal set; }
     internal bool IsStruct { get; set; }
     internal string Path { get; init; } = null!;