Merge pull request #92 from tomasfabian/90-specify-ignorebyinserts-using-the-model-builder

90 specify ignorebyinserts using the model builder

Author: tomasfabian

Tests/ksqlDB.RestApi.Client.IntegrationTests/KSql/Linq/ModelBuilderTests.cs

@@ -9,7 +9,6 @@
 using ksqlDB.RestApi.Client.KSql.RestApi.Http;
 using ksqlDB.RestApi.Client.KSql.RestApi.Statements;
 using ksqlDB.RestApi.Client.KSql.RestApi.Statements.Properties;
-using Microsoft.VisualStudio.TestTools.UnitTesting;
 using NUnit.Framework;
 
 namespace ksqlDb.RestApi.Client.IntegrationTests.KSql.Linq

Tests/ksqlDB.RestApi.Client.IntegrationTests/Models/Record.cs

@@ -1,9 +1,10 @@
-using ksqlDB.RestApi.Client.KSql.RestApi.Statements.Annotations;
+using ksqlDB.RestApi.Client.KSql.RestApi.Statements.Annotations;
 
 namespace ksqlDb.RestApi.Client.IntegrationTests.Models;
 
 public record Record
 {
-  [IgnoreByInserts]
+  [ksqlDB.RestApi.Client.KSql.RestApi.Statements.Annotations.Ignore]
+  [PseudoColumn]
   public long RowTime { get; set; }
-}
+}

Tests/ksqlDB.RestApi.Client.Tests/FluentAPI/Builders/ModelBuilderTests.cs

@@ -50,7 +50,7 @@ public void HasKey_Field()
     }
 
     [Test]
-    public void Property_IgnoreField()
+    public void Property_Ignore()
     {
       //Arrange
 
@@ -67,6 +67,23 @@ public void Property_IgnoreField()
       entityMetadata!.FieldsMetadata.First(c => c.MemberInfo.Name == nameof(Payment.Description)).Ignore.Should().BeTrue();
     }
 
+    [Test]
+    public void Property_IgnoreInDML()
+    {
+      //Arrange
+
+      //Act
+      var fieldTypeBuilder = builder.Entity<Payment>()
+        .Property(b => b.Description)
+        .IgnoreInDML();
+
+      //Assert
+      fieldTypeBuilder.Should().NotBeNull();
+      var entityMetadata = ((IMetadataProvider)builder).GetEntities().FirstOrDefault(c => c.Type == typeof(Payment));
+      entityMetadata.Should().NotBeNull();
+      entityMetadata!.FieldsMetadata.First(c => c.MemberInfo.Name == nameof(Payment.Description)).IgnoreInDML.Should().BeTrue();
+    }
+
     [Test]
     public void Property_HasColumnName()
     {
@@ -233,6 +250,7 @@ public void Headers()
       var metadata = entityMetadata!.FieldsMetadata.First(c => c.MemberInfo.Name == nameof(Payment.Header));
 
       metadata.HasHeaders.Should().BeTrue();
+      metadata.IgnoreInDML.Should().BeTrue();
     }
 
     private record KeyValuePair

Tests/ksqlDB.RestApi.Client.Tests/KSql/RestApi/Statements/CreateInsertTests.cs

@@ -149,6 +149,38 @@ public void Generate_UseModelBuilder_Ignore()
     statement.Should().Be($"INSERT INTO {insertProperties.EntityName} (Title, Id) VALUES ('Title', 1);");
   }
 
+  private class Actor
+  {
+    [Key]
+    public int Id { get; set; }
+    public string Name { get; set; } = null!;
+  }
+
+  public static IEnumerable<(IdentifierEscaping, string)> GenerateIgnoreInDMLTestCases()
+  {
+    yield return (Never, "INSERT INTO Actors (Id) VALUES (1);");
+    yield return (Keywords, "INSERT INTO Actors (Id) VALUES (1);");
+    yield return (Always, "INSERT INTO `Actors` (`Id`) VALUES (1);");
+  }
+
+  [TestCaseSource(nameof(GenerateIgnoreInDMLTestCases))]
+  public void Generate_UseModelBuilder_IgnoreInDML((IdentifierEscaping escaping, string expected) testCase)
+  {
+    //Arrange
+    modelBuilder.Entity<Actor>()
+      .Property(c => c.Name)
+      .IgnoreInDML();
+
+    var (escaping, expected) = testCase;
+    var actor = new Actor { Id = 1, Name = "E.T." };
+
+    //Act
+    var statement = new CreateInsert(modelBuilder).Generate(actor, new InsertProperties { IdentifierEscaping = escaping });
+
+    //Assert
+    statement.Should().Be(expected);
+  }
+
   [Test]
   public void Generate_ShouldNotPluralizeEntityName()
   {

Tests/ksqlDB.RestApi.Client.Tests/Models/Movies/Movie.cs

@@ -4,14 +4,14 @@ namespace ksqlDb.RestApi.Client.Tests.Models.Movies;
 
 public class Movie
 { 
-  [IgnoreByInserts]
+  [ksqlDB.RestApi.Client.KSql.RestApi.Statements.Annotations.Ignore]
   public long RowTime { get; set; }
   public string Title { get; set; } = null!;
   [Key]
   public int Id { get; set; }
   public int Release_Year { get; set; }
 
-  [IgnoreByInserts]
+  [ksqlDB.RestApi.Client.KSql.RestApi.Statements.Annotations.Ignore]
   public int IgnoreMe { get; set; }
 
   public IEnumerable<int> ReadOnly { get; } = new[] { 1, 2 };

docs/ksqldbrestapiclient.md

@@ -1311,3 +1311,22 @@ var restApiClientOptions = new KSqlDBRestApiClientOptions
 };
 servicesCollection.AddSingleton(restApiClientOptions);
 ```
+
+
+## IgnoreAttribute
+**v6.4.0**
+
+Properties and fields decorated with the `IgnoreAttribute` are excluded from both DDL and DML statements.
+
+```C#
+public class Movie
+{
+  [ksqlDB.RestApi.Client.KSql.RestApi.Statements.Annotations.Key]
+  public int Id { get; set; }
+  public string Title { get; set; }
+  public int Release_Year { get; set; }
+	
+  [ksqlDB.RestApi.Client.KSql.RestApi.Statements.Annotations.Ignore]
+  public int IgnoredProperty { get; set; }
+}
+```

docs/modelbuilder.md

@@ -34,18 +34,18 @@ public static async Task InitModelAndCreateTableAsync(CancellationToken cancella
     BaseAddress = new Uri("http://localhost:8088")
   };
   var httpClientFactory = new HttpClientFactory(httpClient);
-  var restApiProvider = new KSqlDbRestApiClient(httpClientFactory, builder);
+  var restApiClient = new KSqlDbRestApiClient(httpClientFactory, builder);
 
   var entityCreationMetadata = new EntityCreationMetadata(kafkaTopic: nameof(Payment), partitions: 3);
 
-  var responseMessage = await restApiProvider.CreateTableAsync<Payment>(entityCreationMetadata, true, cancellationToken);
+  var responseMessage = await restApiClient.CreateTableAsync<Payment>(entityCreationMetadata, true, cancellationToken);
   var content = await responseMessage.Content.ReadAsStringAsync(cancellationToken);
 
   entityCreationMetadata = new EntityCreationMetadata(kafkaTopic: nameof(Account), partitions: 1)
   {
     Replicas = 3
   };
-  responseMessage = await restApiProvider.CreateTableAsync<Account>(entityCreationMetadata, true, cancellationToken);
+  responseMessage = await restApiClient.CreateTableAsync<Account>(entityCreationMetadata, true, cancellationToken);
 }
 ```
 
@@ -189,7 +189,7 @@ var entityCreationMetadata = new EntityCreationMetadata(kafkaTopic: nameof(PocoW
 {
   Replicas = 3
 };
-var responseMessage = await restApiProvider.CreateStreamAsync<PocoWithHeader>(entityCreationMetadata, true);
+var responseMessage = await restApiClient.CreateStreamAsync<PocoWithHeader>(entityCreationMetadata, true);
 
 private record PocoWithHeader
 {
@@ -317,3 +317,53 @@ CREATE TABLE IF NOT EXISTS Records (
 	  Headers ARRAY<STRUCT<Key VARCHAR, Value BYTES>>
 ) WITH ( KAFKA_TOPIC='my_topic', VALUE_FORMAT='Json', PARTITIONS='3' );
 ```
+
+### IgnoreInDML
+**v6.4.0**
+
+The purpose of the `IgnoreInDML` function is to exclude specific fields from INSERT statements.
+
+```C#
+using System.ComponentModel.DataAnnotations;
+
+public class Actor
+{
+  [Key]
+  public int Id { get; set; }
+  public string Name { get; set; } = null!;
+}
+```
+
+```C#
+using ksqlDb.RestApi.Client.FluentAPI.Builders;
+using ksqlDB.RestApi.Client.KSql.Query.Context;
+using ksqlDB.RestApi.Client.KSql.RestApi;
+using ksqlDB.RestApi.Client.KSql.RestApi.Http;
+
+var ksqlDbUrl = "http://localhost:8088";
+
+var httpClientFactory = new HttpClientFactory(new HttpClient() { BaseAddress = new Uri(ksqlDbUrl) });
+
+var restApiClientOptions = new KSqlDBRestApiClientOptions
+{
+  ShouldPluralizeFromItemName = false,
+};
+
+var restApiClient = new KSqlDbRestApiClient(httpClientFactory, restApiClientOptions);
+
+var modelBuilder = new ModelBuilder();
+modelBuilder.Entity<Actor>()
+  .Property(c => c.Name)
+  .IgnoreInDML();
+
+var actor = new Actor { Id = 1, Name = "Matthew David McConaughey" };
+
+var ksql = restApiClient.ToInsertStatement(actor);
+```
+
+Generated KSQL:
+```SQL
+INSERT INTO Actor (Id) VALUES (1);
+```
+
+`WithHeaders` internally automatically marks the property to be ignored in DML statements.

ksqlDb.RestApi.Client/ChangeLog.md

@@ -1,5 +1,13 @@
 # ksqlDB.RestApi.Client
 
+# 6.4.0
+- added the `IgnoreInDML` function to the Fluent API to exclude fields from INSERT statements #90 (proposed by @mrt181)
+- added `IgnoreAttribute` to prevent properties or fields from being included in both DDL and DML statement
+- `Headers` property was marked as obsolete in the `Record` type
+
+## BugFix
+- `IgnoreByInsertsAttribute` no longer excludes fields or properties from DDL statements. For this purpose, a new `IgnoreAttribute` has been introduced.
+ 
 # 6.3.0
 - added `AsStruct` function to the Fluent API for marking fields as ksqldb `STRUCT` types #89 (proposed by @mrt181)
 

ksqlDb.RestApi.Client/FluentAPI/Builders/FieldTypeBuilder.cs

@@ -1,3 +1,4 @@
+using System.Linq.Expressions;
 using ksqlDb.RestApi.Client.Metadata;
 
 namespace ksqlDb.RestApi.Client.FluentAPI.Builders
@@ -9,11 +10,17 @@ namespace ksqlDb.RestApi.Client.FluentAPI.Builders
   public interface IFieldTypeBuilder<TProperty>
   {
     /// <summary>
-    /// Marks the field as ignored, excluding it from the entity's schema.
+    /// Marks the field as ignored, excluding it from the entity's schema, preventing it from being included in both DDL and DML statements. 
     /// </summary>
     /// <returns>The field type builder for chaining additional configuration.</returns>
     public IFieldTypeBuilder<TProperty> Ignore();
 
+    /// <summary>
+    /// Marks the field to be excluded from data manipulation operations, preventing it from being included in DML statements such as INSERT.
+    /// </summary>
+    /// <returns>The field type builder for chaining additional configuration.</returns>
+    public IFieldTypeBuilder<TProperty> IgnoreInDML();
+
     /// <summary>
     /// Marks the field as HEADERS.
     /// </summary>
@@ -55,9 +62,16 @@ public IFieldTypeBuilder<TProperty> Ignore()
       return this;
     }
 
+    public IFieldTypeBuilder<TProperty> IgnoreInDML()
+    {
+      fieldMetadata.IgnoreInDML = true;
+      return this;
+    }
+
     public IFieldTypeBuilder<TProperty> WithHeaders()
     {
       fieldMetadata.HasHeaders = true;
+      IgnoreInDML();
       return this;
     }
   }

ksqlDb.RestApi.Client/KSql/Query/Record.cs

@@ -10,27 +10,32 @@ public class Record
   /// <summary>
   /// Columns that are populated by the Kafka record's header.
   /// </summary>
+  [Ignore]
   [IgnoreByInserts]
   [PseudoColumn]
+  [Obsolete("This property will be removed in the future. Headers need to be defined per use case and should have the type ARRAY<STRUCT<key STRING, value BYTES>>.")]
   public string? Headers { get; set; }
 
   /// <summary>
   /// The offset of the source record.
   /// </summary>
+  [Ignore]
   [IgnoreByInserts]
   [PseudoColumn]
   public long? RowOffset { get; set; }
 
   /// <summary>
   /// The partition of the source record.
   /// </summary>
+  [Ignore]
   [IgnoreByInserts]
   [PseudoColumn]
   public short? RowPartition { get; set; }
 
   /// <summary>
   /// Row timestamp, inferred from the underlying Kafka record if not overridden.
   /// </summary>
+  [Ignore]
   [IgnoreByInserts]
   [PseudoColumn]
   public long RowTime { get; set; }

ksqlDb.RestApi.Client/KSql/RestApi/Extensions/MemberInfoExtensions.cs

@@ -12,8 +12,8 @@ internal static class MemberInfoExtensions
     /// </summary>
     /// <param name="memberInfo"></param>
     /// <param name="escaping"></param>
-    /// <param name="modelBuilder"></param>
+    /// <param name="metadataProvider"></param>
     /// <returns>the <c>memberInfo.Name</c> modified based on the provided <c>format</c></returns>
-    public static string Format(this MemberInfo memberInfo, IdentifierEscaping escaping, ModelBuilder modelBuilder) => IdentifierUtil.Format(memberInfo, escaping, modelBuilder);
+    public static string Format(this MemberInfo memberInfo, IdentifierEscaping escaping, IMetadataProvider metadataProvider) => IdentifierUtil.Format(memberInfo, escaping, metadataProvider);
   }
 }

ksqlDb.RestApi.Client/KSql/RestApi/Statements/Annotations/IgnoreAttribute.cs

@@ -0,0 +1,7 @@
+namespace ksqlDB.RestApi.Client.KSql.RestApi.Statements.Annotations
+{
+  [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property, AllowMultiple = false, Inherited = true)]
+  public sealed class IgnoreAttribute : Attribute
+  {
+  }
+}