Allow customization of SQL queries

core/esmf-aspect-model-document-generators/pom.xml

@@ -106,6 +106,11 @@
          <artifactId>assertj-core</artifactId>
          <scope>test</scope>
       </dependency>
+      <dependency>
+         <groupId>net.jqwik</groupId>
+         <artifactId>jqwik</artifactId>
+         <scope>test</scope>
+      </dependency>
       <dependency>
          <groupId>org.eclipse.esmf</groupId>
          <artifactId>esmf-test-resources</artifactId>

core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/AspectModelDatabricksDenormalizedSqlVisitor.java

@@ -18,6 +18,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
+import java.util.function.Consumer;
 import java.util.stream.Stream;
 
 import org.eclipse.esmf.aspectmodel.generator.AbstractGenerator;
@@ -92,7 +93,7 @@ public AspectModelDatabricksDenormalizedSqlVisitor( final DatabricksSqlGeneratio
             .put( XSD.xfloat.getURI(), DatabricksType.FLOAT )
             .put( XSD.date.getURI(), DatabricksType.STRING )
             .put( XSD.time.getURI(), DatabricksType.STRING )
-            .put( XSD.dateTime.getURI(), DatabricksType.STRING )
+            .put( XSD.dateTime.getURI(), DatabricksType.TIMESTAMP )
             .put( XSD.dateTimeStamp.getURI(), DatabricksType.TIMESTAMP )
             .put( XSD.gYear.getURI(), DatabricksType.STRING )
             .put( XSD.gMonth.getURI(), DatabricksType.STRING )
@@ -138,19 +139,27 @@ private String columnName( final Property property ) {
    @Override
    public String visitStructureElement( final StructureElement structureElement, final Context context ) {
       final StringBuilder result = new StringBuilder();
-      for ( final Property property : structureElement.getProperties() ) {
-         if ( property.isNotInPayload() ) {
-            continue;
-         }
-         final String propertyResult = property.accept( this, context );
-         if ( !propertyResult.isBlank() ) {
+      final Consumer<String> appendLine = line -> {
+         if ( !line.isBlank() ) {
             if ( !result.isEmpty() ) {
                result.append( ",\n" );
             }
-            if ( !propertyResult.startsWith( "  " ) ) {
+            if ( !line.startsWith( "  " ) ) {
                result.append( "  " );
             }
-            result.append( propertyResult );
+            result.append( line );
+         }
+      };
+      for ( final Property property : structureElement.getProperties() ) {
+         if ( property.isNotInPayload() ) {
+            continue;
+         }
+         final String propertyResult = property.accept( this, context );
+         appendLine.accept( propertyResult );
+      }
+      if ( structureElement instanceof Aspect ) {
+         for ( final DatabricksColumnDefinition columnDefinition : config.customColumns() ) {
+            appendLine.accept( columnDefinition.toString() );
          }
       }
       return result.toString();
@@ -271,7 +280,7 @@ private DatabricksType.DatabricksStruct entityToStruct( final ComplexType entity
                   return Stream.empty();
                }
                return Stream.of( new DatabricksType.DatabricksStructEntry( columnName( property ), databricksType,
-                     !property.isOptional(), Optional.ofNullable( property.getDescription( config.commentLanguage() ) ) ) );
+                     property.isOptional(), Optional.ofNullable( property.getDescription( config.commentLanguage() ) ) ) );
             } )
             .toList() );
    }

core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksColumnDefinition.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2024 Robert Bosch Manufacturing Solutions GmbH
+ *
+ * See the AUTHORS file(s) distributed with this work for additional
+ * information regarding authorship.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+package org.eclipse.esmf.aspectmodel.generator.sql.databricks;
+
+import java.util.Optional;
+
+import io.soabase.recordbuilder.core.RecordBuilder;
+
+/**
+ * Represents a column definition in Databricks SQL.
+ *
+ * @param name The name of the column.
+ * @param type The type of the column.
+ * @param nullable Whether the column is nullable.
+ * @param comment An optional comment for the column.
+ */
+@RecordBuilder
+public record DatabricksColumnDefinition(
+      String name,
+      DatabricksType type,
+      boolean nullable,
+      Optional<String> comment
+) {
+   @Override
+   public String toString() {
+      return "%s %s%s%s".formatted(
+            name(),
+            type(),
+            nullable ? "" : " NOT NULL",
+            comment.map( c -> " COMMENT '" + c.replaceAll( "'", "\\\\'" ) + "'" ).orElse( "" ) );
+   }
+}

core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksColumnDefinitionParser.java

@@ -0,0 +1,209 @@
+/*
+ * Copyright (c) 2024 Robert Bosch Manufacturing Solutions GmbH
+ *
+ * See the AUTHORS file(s) distributed with this work for additional
+ * information regarding authorship.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+package org.eclipse.esmf.aspectmodel.generator.sql.databricks;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Supplier;
+
+import org.eclipse.esmf.aspectmodel.generator.DocumentGenerationException;
+
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * Parses Databricks column definitions
+ */
+public class DatabricksColumnDefinitionParser implements Supplier<DatabricksColumnDefinition> {
+   private final Map<String, DatabricksType> standardTypes = ImmutableMap.<String, DatabricksType> builder()
+         .put( "BIGINT", DatabricksType.BIGINT )
+         .put( "BINARY", DatabricksType.BINARY )
+         .put( "BOOLEAN", DatabricksType.BOOLEAN )
+         .put( "DATE", DatabricksType.DATE )
+         .put( "DOUBLE", DatabricksType.DOUBLE )
+         .put( "FLOAT", DatabricksType.FLOAT )
+         .put( "INT", DatabricksType.INT )
+         .put( "SMALLINT", DatabricksType.SMALLINT )
+         .put( "STRING", DatabricksType.STRING )
+         .put( "TIMESTAMP", DatabricksType.TIMESTAMP )
+         .put( "TIMESTAMP_NTZ", DatabricksType.TIMESTAMP_NTZ )
+         .put( "TINYINT", DatabricksType.TINYINT )
+         .build();
+
+   private int index = 0;
+   private final String source;
+
+   public DatabricksColumnDefinitionParser( final String columnDefinition ) {
+      source = columnDefinition;
+   }
+
+   private void eatChars( final String chars ) {
+      while ( index < source.length() && chars.indexOf( source.charAt( index ) ) != -1 ) {
+         index++;
+      }
+   }
+
+   private void eatSpace() {
+      eatChars( " " );
+   }
+
+   private String readToken() {
+      return readToken( " >" );
+   }
+
+   private boolean consumeOptionalToken( final String token ) {
+      final int oldIndex = index;
+      eatSpace();
+      if ( readToken( " ,>" ).equals( token ) ) {
+         return true;
+      }
+      index = oldIndex;
+      return false;
+   }
+
+   private String readToken( final String delim, final boolean keepSpace ) {
+      if ( !keepSpace ) {
+         eatSpace();
+      }
+      final int startIndex = index;
+      while ( index < source.length() && delim.indexOf( source.charAt( index ) ) == -1 ) {
+         if ( source.charAt( index ) == '\\' ) {
+            index++;
+         }
+         index++;
+      }
+      return source.substring( startIndex, index );
+   }
+
+   private String readToken( final String delim ) {
+      return readToken( delim, false );
+   }
+
+   private String parseColumnName() {
+      return readToken( " :" );
+   }
+
+   private void expect( final char c ) {
+      if ( currentCharacterIs( c ) ) {
+         index++;
+         return;
+      }
+      throw new DocumentGenerationException( "Did not find expected token '" + c + "'" );
+   }
+
+   private boolean currentCharacterIs( final char c ) {
+      return index < source.length() && source.charAt( index ) == c;
+   }
+
+   private List<DatabricksType.DatabricksStructEntry> parseStructEntries() {
+      final List<DatabricksType.DatabricksStructEntry> entries = new ArrayList<>();
+      do {
+         eatChars( "," );
+         final String name = parseColumnName();
+         eatChars( " :" );
+         final DatabricksType columnType = parseType();
+         final boolean isNotNullable = parseNullable();
+         final Optional<String> comment = parseComment();
+         entries.add( new DatabricksType.DatabricksStructEntry( name, columnType, !isNotNullable, comment ) );
+      } while ( currentCharacterIs( ',' ) );
+      return entries;
+   }
+
+   private DatabricksType parseType() {
+      final String typeName = readToken( " <>()," );
+      for ( final Map.Entry<String, DatabricksType> entry : standardTypes.entrySet() ) {
+         if ( typeName.equals( entry.getKey() ) ) {
+            return entry.getValue();
+         }
+      }
+
+      if ( "ARRAY".equals( typeName ) ) {
+         expect( '<' );
+         final DatabricksType nestedType = parseType();
+         expect( '>' );
+         return new DatabricksType.DatabricksArray( nestedType );
+      } else if ( "STRUCT".equals( typeName ) ) {
+         expect( '<' );
+         final List<DatabricksType.DatabricksStructEntry> entries = parseStructEntries();
+         expect( '>' );
+         return new DatabricksType.DatabricksStruct( entries );
+      } else if ( typeName.startsWith( "DECIMAL" ) ) {
+         final Optional<Integer> precision;
+         final Optional<Integer> scale;
+         if ( currentCharacterIs( '(' ) ) {
+            expect( '(' );
+            precision = Optional.of( Integer.parseInt( readToken( ",)" ) ) );
+            if ( currentCharacterIs( ',' ) ) {
+               expect( ',' );
+               scale = Optional.of( Integer.parseInt( readToken( ")" ) ) );
+            } else {
+               scale = Optional.empty();
+            }
+            expect( ')' );
+         } else {
+            precision = Optional.empty();
+            scale = Optional.empty();
+         }
+         return new DatabricksType.DatabricksDecimal( precision, scale );
+      } else if ( "MAP".equals( typeName ) ) {
+         expect( '<' );
+         final DatabricksType keyType = parseType();
+         expect( ',' );
+         final DatabricksType valueType = parseType();
+         expect( '>' );
+         return new DatabricksType.DatabricksMap( keyType, valueType );
+      }
+      throw new DocumentGenerationException( "Could not parse databricks type" );
+   }
+
+   private boolean parseNullable() {
+      return consumeOptionalToken( "NOT" ) && consumeOptionalToken( "NULL" );
+   }
+
+   private Optional<String> parseComment() {
+      final int oldIndex = index;
+      if ( "COMMENT".equals( readToken() ) ) {
+         eatSpace();
+         expect( '\'' );
+         final String comment = readToken( "'", true ).replaceAll( "\\\\'", "'" );
+         expect( '\'' );
+         return Optional.of( comment );
+      } else {
+         index = oldIndex;
+      }
+      return Optional.empty();
+   }
+
+   @Override
+   public DatabricksColumnDefinition get() {
+      try {
+         final String columnName = parseColumnName();
+         final DatabricksType columnType = parseType();
+         final boolean isNotNull = parseNullable();
+         final Optional<String> comment = parseComment();
+         return DatabricksColumnDefinitionBuilder.builder()
+               .name( columnName )
+               .type( columnType )
+               .nullable( !isNotNull )
+               .comment( comment )
+               .build();
+      } catch ( final DocumentGenerationException exception ) {
+         throw exception;
+      } catch ( final Exception exception ) {
+         throw new DocumentGenerationException( "Could not parse column definition: " + source );
+      }
+   }
+}
+

core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksSqlGenerationConfig.java

@@ -13,6 +13,7 @@
 
 package org.eclipse.esmf.aspectmodel.generator.sql.databricks;
 
+import java.util.List;
 import java.util.Locale;
 
 import org.eclipse.esmf.aspectmodel.generator.sql.SqlGenerationConfig;
@@ -28,14 +29,16 @@
  * @param commentLanguage the language to use for comments
  * @param decimalPrecision the precision to use for decimal columns, see <a
  * href="https://docs.databricks.com/en/sql/language-manual/data-types/decimal-type.html">DECIMAL type</a> for more info.
+ * @param customColumns custom columns to add to the table
  */
 @RecordBuilder
 public record DatabricksSqlGenerationConfig(
       String createTableCommandPrefix,
       boolean includeTableComment,
       boolean includeColumnComments,
       Locale commentLanguage,
-      int decimalPrecision
+      int decimalPrecision,
+      List<DatabricksColumnDefinition> customColumns
 ) implements SqlGenerationConfig.DialectSpecificConfig {
    public static final String DEFAULT_TABLE_COMMAND_PREFIX = "CREATE TABLE IF NOT EXISTS";
    // As defined in https://docs.databricks.com/en/sql/language-manual/data-types/decimal-type.html
@@ -48,7 +51,7 @@ public record DatabricksSqlGenerationConfig(
 
    public DatabricksSqlGenerationConfig() {
       this( DEFAULT_TABLE_COMMAND_PREFIX, DEFAULT_INCLUDE_TABLE_COMMENT, DEFAULT_INCLUDE_COLUMN_COMMENTS, DEFAULT_COMMENT_LANGUAGE,
-            DECIMAL_DEFAULT_PRECISION );
+            DECIMAL_DEFAULT_PRECISION, List.of() );
    }
 
    public DatabricksSqlGenerationConfig {
@@ -64,5 +67,8 @@ public DatabricksSqlGenerationConfig() {
       if ( commentLanguage == null ) {
          commentLanguage = DEFAULT_COMMENT_LANGUAGE;
       }
+      if ( customColumns == null ) {
+         customColumns = List.of();
+      }
    }
 }

core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksType.java

@@ -76,10 +76,11 @@ public String toString() {
       }
    }
 
-   record DatabricksStructEntry( String name, DatabricksType type, boolean notNull, Optional<String> comment ) {
+   record DatabricksStructEntry( String name, DatabricksType type, boolean nullable, Optional<String> comment ) {
       @Override
       public String toString() {
-         return name + ": " + type + (notNull ? " NOT NULL" : "") + comment.map( c -> " COMMENT '" + c + "'" ).orElse( "" );
+         return name + ": " + type + (nullable ? "" : " NOT NULL")
+               + comment.map( c -> " COMMENT '%s'".formatted( c.replace( "'", "\\'" ) ) ).orElse( "" );
       }
    }
 

core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/AspectModelSqlGeneratorTest.java

@@ -50,8 +50,6 @@ void testDatabricksGeneration( final TestAspect testAspect ) {
 
          assertThat( result ).contains( "TBLPROPERTIES ('x-samm-aspect-model-urn'='" );
          assertThat( result ).doesNotContain( "ARRAY<ARRAY" );
-
-         System.out.println( sqlArtifact.getContent() );
       } ).doesNotThrowAnyException();
    }
 }