diff --git a/core/esmf-aspect-model-document-generators/pom.xml b/core/esmf-aspect-model-document-generators/pom.xml index d58efbd38..e8d4335ba 100644 --- a/core/esmf-aspect-model-document-generators/pom.xml +++ b/core/esmf-aspect-model-document-generators/pom.xml @@ -106,6 +106,11 @@ assertj-core test + + net.jqwik + jqwik + test + org.eclipse.esmf esmf-test-resources diff --git a/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/AspectModelDatabricksDenormalizedSqlVisitor.java b/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/AspectModelDatabricksDenormalizedSqlVisitor.java index b8e34d6e5..bfd1759ad 100644 --- a/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/AspectModelDatabricksDenormalizedSqlVisitor.java +++ b/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/AspectModelDatabricksDenormalizedSqlVisitor.java @@ -18,6 +18,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.function.Consumer; import java.util.stream.Stream; import org.eclipse.esmf.aspectmodel.generator.AbstractGenerator; @@ -92,7 +93,7 @@ public AspectModelDatabricksDenormalizedSqlVisitor( final DatabricksSqlGeneratio .put( XSD.xfloat.getURI(), DatabricksType.FLOAT ) .put( XSD.date.getURI(), DatabricksType.STRING ) .put( XSD.time.getURI(), DatabricksType.STRING ) - .put( XSD.dateTime.getURI(), DatabricksType.STRING ) + .put( XSD.dateTime.getURI(), DatabricksType.TIMESTAMP ) .put( XSD.dateTimeStamp.getURI(), DatabricksType.TIMESTAMP ) .put( XSD.gYear.getURI(), DatabricksType.STRING ) .put( XSD.gMonth.getURI(), DatabricksType.STRING ) @@ -138,19 +139,27 @@ private String columnName( final Property property ) { @Override public String visitStructureElement( final StructureElement structureElement, final Context context ) { final StringBuilder result = new StringBuilder(); - for ( final Property property : structureElement.getProperties() ) { - if ( property.isNotInPayload() ) { - continue; - } - final String propertyResult = property.accept( this, context ); - if ( !propertyResult.isBlank() ) { + final Consumer appendLine = line -> { + if ( !line.isBlank() ) { if ( !result.isEmpty() ) { result.append( ",\n" ); } - if ( !propertyResult.startsWith( " " ) ) { + if ( !line.startsWith( " " ) ) { result.append( " " ); } - result.append( propertyResult ); + result.append( line ); + } + }; + for ( final Property property : structureElement.getProperties() ) { + if ( property.isNotInPayload() ) { + continue; + } + final String propertyResult = property.accept( this, context ); + appendLine.accept( propertyResult ); + } + if ( structureElement instanceof Aspect ) { + for ( final DatabricksColumnDefinition columnDefinition : config.customColumns() ) { + appendLine.accept( columnDefinition.toString() ); } } return result.toString(); @@ -271,7 +280,7 @@ private DatabricksType.DatabricksStruct entityToStruct( final ComplexType entity return Stream.empty(); } return Stream.of( new DatabricksType.DatabricksStructEntry( columnName( property ), databricksType, - !property.isOptional(), Optional.ofNullable( property.getDescription( config.commentLanguage() ) ) ) ); + property.isOptional(), Optional.ofNullable( property.getDescription( config.commentLanguage() ) ) ) ); } ) .toList() ); } diff --git a/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksColumnDefinition.java b/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksColumnDefinition.java new file mode 100644 index 000000000..dde72421f --- /dev/null +++ b/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksColumnDefinition.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2024 Robert Bosch Manufacturing Solutions GmbH + * + * See the AUTHORS file(s) distributed with this work for additional + * information regarding authorship. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + * + * SPDX-License-Identifier: MPL-2.0 + */ + +package org.eclipse.esmf.aspectmodel.generator.sql.databricks; + +import java.util.Optional; + +import io.soabase.recordbuilder.core.RecordBuilder; + +/** + * Represents a column definition in Databricks SQL. + * + * @param name The name of the column. + * @param type The type of the column. + * @param nullable Whether the column is nullable. + * @param comment An optional comment for the column. + */ +@RecordBuilder +public record DatabricksColumnDefinition( + String name, + DatabricksType type, + boolean nullable, + Optional comment +) { + @Override + public String toString() { + return "%s %s%s%s".formatted( + name(), + type(), + nullable ? "" : " NOT NULL", + comment.map( c -> " COMMENT '" + c.replaceAll( "'", "\\\\'" ) + "'" ).orElse( "" ) ); + } +} diff --git a/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksColumnDefinitionParser.java b/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksColumnDefinitionParser.java new file mode 100644 index 000000000..3c6862d78 --- /dev/null +++ b/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksColumnDefinitionParser.java @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2024 Robert Bosch Manufacturing Solutions GmbH + * + * See the AUTHORS file(s) distributed with this work for additional + * information regarding authorship. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + * + * SPDX-License-Identifier: MPL-2.0 + */ + +package org.eclipse.esmf.aspectmodel.generator.sql.databricks; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; + +import org.eclipse.esmf.aspectmodel.generator.DocumentGenerationException; + +import com.google.common.collect.ImmutableMap; + +/** + * Parses Databricks column definitions + */ +public class DatabricksColumnDefinitionParser implements Supplier { + private final Map standardTypes = ImmutableMap. builder() + .put( "BIGINT", DatabricksType.BIGINT ) + .put( "BINARY", DatabricksType.BINARY ) + .put( "BOOLEAN", DatabricksType.BOOLEAN ) + .put( "DATE", DatabricksType.DATE ) + .put( "DOUBLE", DatabricksType.DOUBLE ) + .put( "FLOAT", DatabricksType.FLOAT ) + .put( "INT", DatabricksType.INT ) + .put( "SMALLINT", DatabricksType.SMALLINT ) + .put( "STRING", DatabricksType.STRING ) + .put( "TIMESTAMP", DatabricksType.TIMESTAMP ) + .put( "TIMESTAMP_NTZ", DatabricksType.TIMESTAMP_NTZ ) + .put( "TINYINT", DatabricksType.TINYINT ) + .build(); + + private int index = 0; + private final String source; + + public DatabricksColumnDefinitionParser( final String columnDefinition ) { + source = columnDefinition; + } + + private void eatChars( final String chars ) { + while ( index < source.length() && chars.indexOf( source.charAt( index ) ) != -1 ) { + index++; + } + } + + private void eatSpace() { + eatChars( " " ); + } + + private String readToken() { + return readToken( " >" ); + } + + private boolean consumeOptionalToken( final String token ) { + final int oldIndex = index; + eatSpace(); + if ( readToken( " ,>" ).equals( token ) ) { + return true; + } + index = oldIndex; + return false; + } + + private String readToken( final String delim, final boolean keepSpace ) { + if ( !keepSpace ) { + eatSpace(); + } + final int startIndex = index; + while ( index < source.length() && delim.indexOf( source.charAt( index ) ) == -1 ) { + if ( source.charAt( index ) == '\\' ) { + index++; + } + index++; + } + return source.substring( startIndex, index ); + } + + private String readToken( final String delim ) { + return readToken( delim, false ); + } + + private String parseColumnName() { + return readToken( " :" ); + } + + private void expect( final char c ) { + if ( currentCharacterIs( c ) ) { + index++; + return; + } + throw new DocumentGenerationException( "Did not find expected token '" + c + "'" ); + } + + private boolean currentCharacterIs( final char c ) { + return index < source.length() && source.charAt( index ) == c; + } + + private List parseStructEntries() { + final List entries = new ArrayList<>(); + do { + eatChars( "," ); + final String name = parseColumnName(); + eatChars( " :" ); + final DatabricksType columnType = parseType(); + final boolean isNotNullable = parseNullable(); + final Optional comment = parseComment(); + entries.add( new DatabricksType.DatabricksStructEntry( name, columnType, !isNotNullable, comment ) ); + } while ( currentCharacterIs( ',' ) ); + return entries; + } + + private DatabricksType parseType() { + final String typeName = readToken( " <>()," ); + for ( final Map.Entry entry : standardTypes.entrySet() ) { + if ( typeName.equals( entry.getKey() ) ) { + return entry.getValue(); + } + } + + if ( "ARRAY".equals( typeName ) ) { + expect( '<' ); + final DatabricksType nestedType = parseType(); + expect( '>' ); + return new DatabricksType.DatabricksArray( nestedType ); + } else if ( "STRUCT".equals( typeName ) ) { + expect( '<' ); + final List entries = parseStructEntries(); + expect( '>' ); + return new DatabricksType.DatabricksStruct( entries ); + } else if ( typeName.startsWith( "DECIMAL" ) ) { + final Optional precision; + final Optional scale; + if ( currentCharacterIs( '(' ) ) { + expect( '(' ); + precision = Optional.of( Integer.parseInt( readToken( ",)" ) ) ); + if ( currentCharacterIs( ',' ) ) { + expect( ',' ); + scale = Optional.of( Integer.parseInt( readToken( ")" ) ) ); + } else { + scale = Optional.empty(); + } + expect( ')' ); + } else { + precision = Optional.empty(); + scale = Optional.empty(); + } + return new DatabricksType.DatabricksDecimal( precision, scale ); + } else if ( "MAP".equals( typeName ) ) { + expect( '<' ); + final DatabricksType keyType = parseType(); + expect( ',' ); + final DatabricksType valueType = parseType(); + expect( '>' ); + return new DatabricksType.DatabricksMap( keyType, valueType ); + } + throw new DocumentGenerationException( "Could not parse databricks type" ); + } + + private boolean parseNullable() { + return consumeOptionalToken( "NOT" ) && consumeOptionalToken( "NULL" ); + } + + private Optional parseComment() { + final int oldIndex = index; + if ( "COMMENT".equals( readToken() ) ) { + eatSpace(); + expect( '\'' ); + final String comment = readToken( "'", true ).replaceAll( "\\\\'", "'" ); + expect( '\'' ); + return Optional.of( comment ); + } else { + index = oldIndex; + } + return Optional.empty(); + } + + @Override + public DatabricksColumnDefinition get() { + try { + final String columnName = parseColumnName(); + final DatabricksType columnType = parseType(); + final boolean isNotNull = parseNullable(); + final Optional comment = parseComment(); + return DatabricksColumnDefinitionBuilder.builder() + .name( columnName ) + .type( columnType ) + .nullable( !isNotNull ) + .comment( comment ) + .build(); + } catch ( final DocumentGenerationException exception ) { + throw exception; + } catch ( final Exception exception ) { + throw new DocumentGenerationException( "Could not parse column definition: " + source ); + } + } +} + diff --git a/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksSqlGenerationConfig.java b/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksSqlGenerationConfig.java index 3edf81340..92c1c603d 100644 --- a/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksSqlGenerationConfig.java +++ b/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksSqlGenerationConfig.java @@ -13,6 +13,7 @@ package org.eclipse.esmf.aspectmodel.generator.sql.databricks; +import java.util.List; import java.util.Locale; import org.eclipse.esmf.aspectmodel.generator.sql.SqlGenerationConfig; @@ -28,6 +29,7 @@ * @param commentLanguage the language to use for comments * @param decimalPrecision the precision to use for decimal columns, see DECIMAL type for more info. + * @param customColumns custom columns to add to the table */ @RecordBuilder public record DatabricksSqlGenerationConfig( @@ -35,7 +37,8 @@ public record DatabricksSqlGenerationConfig( boolean includeTableComment, boolean includeColumnComments, Locale commentLanguage, - int decimalPrecision + int decimalPrecision, + List customColumns ) implements SqlGenerationConfig.DialectSpecificConfig { public static final String DEFAULT_TABLE_COMMAND_PREFIX = "CREATE TABLE IF NOT EXISTS"; // As defined in https://docs.databricks.com/en/sql/language-manual/data-types/decimal-type.html @@ -48,7 +51,7 @@ public record DatabricksSqlGenerationConfig( public DatabricksSqlGenerationConfig() { this( DEFAULT_TABLE_COMMAND_PREFIX, DEFAULT_INCLUDE_TABLE_COMMENT, DEFAULT_INCLUDE_COLUMN_COMMENTS, DEFAULT_COMMENT_LANGUAGE, - DECIMAL_DEFAULT_PRECISION ); + DECIMAL_DEFAULT_PRECISION, List.of() ); } public DatabricksSqlGenerationConfig { @@ -64,5 +67,8 @@ public DatabricksSqlGenerationConfig() { if ( commentLanguage == null ) { commentLanguage = DEFAULT_COMMENT_LANGUAGE; } + if ( customColumns == null ) { + customColumns = List.of(); + } } } diff --git a/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksType.java b/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksType.java index 8b4c63fc0..71522f5ad 100644 --- a/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksType.java +++ b/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksType.java @@ -76,10 +76,11 @@ public String toString() { } } - record DatabricksStructEntry( String name, DatabricksType type, boolean notNull, Optional comment ) { + record DatabricksStructEntry( String name, DatabricksType type, boolean nullable, Optional comment ) { @Override public String toString() { - return name + ": " + type + (notNull ? " NOT NULL" : "") + comment.map( c -> " COMMENT '" + c + "'" ).orElse( "" ); + return name + ": " + type + (nullable ? "" : " NOT NULL") + + comment.map( c -> " COMMENT '%s'".formatted( c.replace( "'", "\\'" ) ) ).orElse( "" ); } } diff --git a/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/AspectModelSqlGeneratorTest.java b/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/AspectModelSqlGeneratorTest.java index 8876b8f96..528e889cf 100644 --- a/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/AspectModelSqlGeneratorTest.java +++ b/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/AspectModelSqlGeneratorTest.java @@ -50,8 +50,6 @@ void testDatabricksGeneration( final TestAspect testAspect ) { assertThat( result ).contains( "TBLPROPERTIES ('x-samm-aspect-model-urn'='" ); assertThat( result ).doesNotContain( "ARRAY new RuntimeException() ); - return aspect.accept( new AspectModelDatabricksDenormalizedSqlVisitor( config ), - AspectModelDatabricksDenormalizedSqlVisitorContextBuilder.builder().build() ); - } - - private String sql( final TestAspect testAspect ) { - final DatabricksSqlGenerationConfig config = new DatabricksSqlGenerationConfig(); - return sql( testAspect, config ); - } - +public class AspectModelDatabricksDenormalizedSqlVisitorTest extends DatabricksTestBase { @Test void testAspectWithAbstractEntity() { assertThat( sql( TestAspect.ASPECT_WITH_ABSTRACT_ENTITY ) ).isEqualTo( """ @@ -307,11 +288,11 @@ CREATE TABLE IF NOT EXISTS aspect_with_multiple_entities_and_either ( void testAspectWithMultipleEntitiesOnMultipleLevels() { assertThat( sql( TestAspect.ASPECT_WITH_MULTIPLE_ENTITIES_ON_MULTIPLE_LEVELS ) ).isEqualTo( """ CREATE TABLE IF NOT EXISTS aspect_with_multiple_entities_on_multiple_levels ( - test_entity_one__test_local_date_time STRING NOT NULL, + test_entity_one__test_local_date_time TIMESTAMP NOT NULL, test_entity_one__random_value STRING NOT NULL, test_entity_one__test_third_entity__test_string STRING NOT NULL, test_entity_one__test_third_entity__test_float FLOAT NOT NULL, - test_entity_two__test_local_date_time STRING NOT NULL, + test_entity_two__test_local_date_time TIMESTAMP NOT NULL, test_entity_two__random_value STRING NOT NULL, test_entity_two__test_third_entity__test_string STRING NOT NULL, test_entity_two__test_third_entity__test_float FLOAT NOT NULL, @@ -368,7 +349,7 @@ void testAspectWithOptionalProperties() { assertThat( sql( TestAspect.ASPECT_WITH_OPTIONAL_PROPERTIES ) ).isEqualTo( """ CREATE TABLE IF NOT EXISTS aspect_with_optional_properties ( number_property DECIMAL, - timestamp_property STRING NOT NULL + timestamp_property TIMESTAMP NOT NULL ) TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithOptionalProperties'); """ ); @@ -395,7 +376,7 @@ CREATE TABLE IF NOT EXISTS aspect_with_property_with_payload_name ( test STRING NOT NULL COMMENT 'This is a test property.' ) COMMENT 'This is a test description' - TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithPropertyWithPayloadName'); + TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithPropertyWithPayloadName'); """ ); } @@ -441,7 +422,7 @@ CREATE TABLE IF NOT EXISTS aspect_with_simple_types ( byte_property TINYINT NOT NULL, curie_property STRING NOT NULL, date_property STRING NOT NULL, - date_time_property STRING NOT NULL, + date_time_property TIMESTAMP NOT NULL, date_time_stamp_property TIMESTAMP NOT NULL, day_time_duration STRING NOT NULL, decimal_property DECIMAL(10) NOT NULL, @@ -490,7 +471,7 @@ CREATE TABLE IF NOT EXISTS aspect_with_sorted_set ( void testAspectWithTimeSeries() { assertThat( sql( TestAspect.ASPECT_WITH_TIME_SERIES ) ).isEqualTo( """ CREATE TABLE IF NOT EXISTS aspect_with_time_series ( - test_property ARRAY> NOT NULL COMMENT 'This is a test property.' + test_property ARRAY> NOT NULL COMMENT 'This is a test property.' ) COMMENT 'This is a test description' TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithTimeSeries'); @@ -507,4 +488,27 @@ CREATE TABLE IF NOT EXISTS aspect_with_complex_set ( TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithComplexSet'); """ ); } + + @Test + void testAspectWithCustomColumn() { + final DatabricksSqlGenerationConfig config = DatabricksSqlGenerationConfigBuilder.builder() + .includeTableComment( true ) + .includeColumnComments( true ) + .customColumns( List.of( + DatabricksColumnDefinitionBuilder.builder() + .name( "custom" ) + .type( new DatabricksType.DatabricksArray( DatabricksType.STRING ) ) + .nullable( false ) + .comment( Optional.of( "Custom column" ) ) + .build() + ) ).build(); + assertThat( sql( TestAspect.ASPECT_WITH_PROPERTY_WITH_PAYLOAD_NAME, config ) ).isEqualTo( """ + CREATE TABLE IF NOT EXISTS aspect_with_property_with_payload_name ( + test STRING NOT NULL COMMENT 'This is a test property.', + custom ARRAY NOT NULL COMMENT 'Custom column' + ) + COMMENT 'This is a test description' + TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithPropertyWithPayloadName'); + """ ); + } } diff --git a/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksColumnDefinitionParserTest.java b/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksColumnDefinitionParserTest.java new file mode 100644 index 000000000..5746472ef --- /dev/null +++ b/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksColumnDefinitionParserTest.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2024 Robert Bosch Manufacturing Solutions GmbH + * + * See the AUTHORS file(s) distributed with this work for additional + * information regarding authorship. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + * + * SPDX-License-Identifier: MPL-2.0 + */ + +package org.eclipse.esmf.aspectmodel.generator.sql.databricks; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.stream.Collectors; + +import org.eclipse.esmf.test.TestAspect; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +public class DatabricksColumnDefinitionParserTest extends DatabricksTestBase { + @Test + void testMinimalDefinition() { + final DatabricksColumnDefinition definition = new DatabricksColumnDefinitionParser( "abc STRING" ).get(); + assertThat( definition.name() ).isEqualTo( "abc" ); + assertThat( definition.type() ).isEqualTo( DatabricksType.STRING ); + } + + @ParameterizedTest + @EnumSource( value = TestAspect.class ) + void testParseSqlForAspectModel( final TestAspect testAspect ) { + final String sql = sql( testAspect ); + final String parsedAndSerializedSql = sql.lines() + .filter( line -> line.startsWith( " " ) ) + .map( line -> " " + new DatabricksColumnDefinitionParser( line.trim() ).get().toString() ) + .collect( Collectors.joining( "\n" ) ); + assertThat( sql.lines() + .filter( line -> line.startsWith( " " ) ) + .map( line -> line.replaceAll( ",$", "" ) ) + .collect( Collectors.joining( "\n" ) ) ) + .isEqualTo( parsedAndSerializedSql ); + } + + @Test + void testParseCommentWithEscapes() { + final String line = "column STRING COMMENT 'Test with \\' test'"; + final DatabricksColumnDefinitionParser parser = new DatabricksColumnDefinitionParser( line ); + final DatabricksColumnDefinition result = parser.get(); + assertThat( result.comment() ).contains( "Test with ' test" ); + } +} diff --git a/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksSqlPropertyTest.java b/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksSqlPropertyTest.java new file mode 100644 index 000000000..0ecc6704c --- /dev/null +++ b/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksSqlPropertyTest.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2024 Robert Bosch Manufacturing Solutions GmbH + * + * See the AUTHORS file(s) distributed with this work for additional + * information regarding authorship. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + * + * SPDX-License-Identifier: MPL-2.0 + */ + +package org.eclipse.esmf.aspectmodel.generator.sql.databricks; + +import java.util.Optional; + +import net.jqwik.api.Arbitraries; +import net.jqwik.api.Arbitrary; +import net.jqwik.api.Combinators; +import net.jqwik.api.ForAll; +import net.jqwik.api.Property; +import net.jqwik.api.Provide; + +public class DatabricksSqlPropertyTest { + @Provide + Arbitrary anyDecimal() { + final Arbitrary defaultDecimal = Arbitraries.of( new DatabricksType.DatabricksDecimal() ); + final Arbitrary decimalWithPrecision = Arbitraries.integers().between( 1, 20 ) + .map( precision -> new DatabricksType.DatabricksDecimal( Optional.of( precision ) ) ); + final Arbitrary decimalWithPrecisionAndScale = Arbitraries.integers().between( 1, 20 ) + .flatMap( precision -> Arbitraries.integers().between( 1, 20 ) + .map( scale -> new DatabricksType.DatabricksDecimal( Optional.of( precision ), Optional.of( scale ) ) ) ); + return Arbitraries.oneOf( defaultDecimal, decimalWithPrecision, decimalWithPrecisionAndScale ); + } + + @Provide + Arbitrary anyArray() { + return anyType().map( DatabricksType.DatabricksArray::new ); + } + + @Provide + Arbitrary nullableOrNot() { + return Arbitraries.of( true, false ); + } + + @Provide + Arbitrary anyComment() { + return Arbitraries.strings().ofMaxLength( 10 ); + } + + @Provide + Arbitrary anyStructEntry() { + return Combinators.combine( anyColumnName(), anyType(), nullableOrNot(), anyComment().map( Optional::of ) ) + .as( DatabricksType.DatabricksStructEntry::new ); + } + + @Provide + Arbitrary anyStruct() { + return anyStructEntry().list().ofMinSize( 1 ).ofMaxSize( 3 ).map( DatabricksType.DatabricksStruct::new ); + } + + @Provide + Arbitrary anyStandardType() { + return Arbitraries.of( DatabricksType.BIGINT, DatabricksType.BINARY, + DatabricksType.BOOLEAN, DatabricksType.DATE, DatabricksType.DOUBLE, DatabricksType.FLOAT, DatabricksType.INT, + DatabricksType.SMALLINT, DatabricksType.STRING, DatabricksType.TIMESTAMP, DatabricksType.TIMESTAMP_NTZ, + DatabricksType.TINYINT ); + } + + @Provide + Arbitrary anyMap() { + final Arbitrary anyTypeWithoutMap = Arbitraries.lazyOf( this::anyStandardType, this::anyDecimal, this::anyArray, + this::anyStruct ); + final Arbitrary anyType = Arbitraries.lazyOf( this::anyStandardType, this::anyDecimal, this::anyArray, + this::anyStruct, this::anyMap ); + return Combinators.combine( anyTypeWithoutMap, anyType ).as( DatabricksType.DatabricksMap::new ); + } + + @Provide + Arbitrary anyType() { + return Arbitraries.lazyOf( this::anyStandardType, this::anyDecimal, this::anyArray, this::anyStruct, this::anyMap ); + } + + @Provide + Arbitrary anyColumnName() { + return Arbitraries.strings().withCharRange( 'a', 'z' ).withChars( '_' ).ofMinLength( 1 ).ofMaxLength( 10 ); + } + + @Provide + Arbitrary syntheticDatabricksColumnDefinition() { + return Combinators.combine( anyColumnName(), anyType(), nullableOrNot(), anyComment().map( Optional::of ) ) + .as( DatabricksColumnDefinition::new ); + } + + @Provide + Arbitrary anyDatabricksColumnDefinition() { + return syntheticDatabricksColumnDefinition().map( DatabricksColumnDefinition::toString ); + } + + @Property + boolean isValidColumnDefinition( @ForAll( "anyDatabricksColumnDefinition" ) final String columnDefintition ) { + final DatabricksColumnDefinition column = new DatabricksColumnDefinitionParser( columnDefintition ).get(); + return column.toString().equals( columnDefintition ); + } +} diff --git a/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksTestBase.java b/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksTestBase.java new file mode 100644 index 000000000..94357b603 --- /dev/null +++ b/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksTestBase.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024 Robert Bosch Manufacturing Solutions GmbH + * + * See the AUTHORS file(s) distributed with this work for additional + * information regarding authorship. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + * + * SPDX-License-Identifier: MPL-2.0 + */ + +package org.eclipse.esmf.aspectmodel.generator.sql.databricks; + +import org.eclipse.esmf.aspectmodel.resolver.services.VersionedModel; +import org.eclipse.esmf.metamodel.Aspect; +import org.eclipse.esmf.metamodel.loader.AspectModelLoader; +import org.eclipse.esmf.samm.KnownVersion; +import org.eclipse.esmf.test.TestAspect; +import org.eclipse.esmf.test.TestResources; + +public class DatabricksTestBase { + protected String sql( final TestAspect testAspect, final DatabricksSqlGenerationConfig config ) { + final VersionedModel versionedModel = TestResources.getModel( testAspect, KnownVersion.getLatest() ).get(); + final Aspect aspect = AspectModelLoader.getSingleAspect( versionedModel ).getOrElseThrow( () -> new RuntimeException() ); + return aspect.accept( new AspectModelDatabricksDenormalizedSqlVisitor( config ), + AspectModelDatabricksDenormalizedSqlVisitorContextBuilder.builder().build() ); + } + + protected String sql( final TestAspect testAspect ) { + final DatabricksSqlGenerationConfig config = new DatabricksSqlGenerationConfig(); + return sql( testAspect, config ); + } +} diff --git a/documentation/developer-guide/modules/tooling-guide/examples/GenerateSql.java b/documentation/developer-guide/modules/tooling-guide/examples/GenerateSql.java index fbde8e896..aa4d6a176 100644 --- a/documentation/developer-guide/modules/tooling-guide/examples/GenerateSql.java +++ b/documentation/developer-guide/modules/tooling-guide/examples/GenerateSql.java @@ -16,17 +16,22 @@ // tag::imports[] import java.io.File; import java.io.IOException; +import java.util.List; import java.util.Locale; +import java.util.Optional; import org.eclipse.esmf.aspectmodel.generator.sql.AspectModelSqlGenerator; import org.eclipse.esmf.aspectmodel.generator.sql.SqlGenerationConfig; import org.eclipse.esmf.aspectmodel.generator.sql.SqlGenerationConfigBuilder; +import org.eclipse.esmf.aspectmodel.generator.sql.databricks.DatabricksColumnDefinitionBuilder; import org.eclipse.esmf.aspectmodel.generator.sql.databricks.DatabricksSqlGenerationConfig; import org.eclipse.esmf.aspectmodel.generator.sql.databricks.DatabricksSqlGenerationConfigBuilder; +import org.eclipse.esmf.aspectmodel.generator.sql.databricks.DatabricksType; import org.eclipse.esmf.aspectmodel.resolver.AspectModelResolver; import org.eclipse.esmf.metamodel.Aspect; import org.eclipse.esmf.metamodel.loader.AspectModelLoader; // end::imports[] + import org.junit.jupiter.api.Test; public class GenerateSql extends AbstractGenerator { @@ -48,6 +53,13 @@ public void generate() throws IOException { .includeTableComment( true ) // optional .includeColumnComments( true ) // optional .decimalPrecision( 10 ) // optional + .customColumns( List.of( // optional + DatabricksColumnDefinitionBuilder.builder() + .name( "custom_column" ) + .type( new DatabricksType.DatabricksArray( DatabricksType.STRING ) ) + .nullable( false ) + .comment( Optional.of( "Custom column" ) ) + .build() ) ) .build(); final SqlGenerationConfig sqlGenerationConfig = SqlGenerationConfigBuilder.builder() diff --git a/documentation/developer-guide/modules/tooling-guide/pages/maven-plugin.adoc b/documentation/developer-guide/modules/tooling-guide/pages/maven-plugin.adoc index 083d808aa..fd7c6e10a 100644 --- a/documentation/developer-guide/modules/tooling-guide/pages/maven-plugin.adoc +++ b/documentation/developer-guide/modules/tooling-guide/pages/maven-plugin.adoc @@ -377,6 +377,7 @@ Configuration Properties: | `tableCommandPrefix` | The prefix to use for Databricks table creation commands. | `String` | `CREATE TABLE IF NOT EXISTS` | {nok} | `decimalPrecision` | The precision to use for Databricks decimal columns, between 1 and 38. See also notes in the xref:java-aspect-tooling.adoc#databricks-type-mapping[Databricks type mapping]. | `Integer` | 10 | {nok} +| `customColumns` | Contains `` elements, each of which defines a custom column to add. Column defintions follow the pattern `column_name DATATYPE [NOT NULL] [COMMENT 'custom']`. | ``... | | {nok} |=== == Generate Documentation for an Aspect Model diff --git a/documentation/developer-guide/modules/tooling-guide/pages/samm-cli.adoc b/documentation/developer-guide/modules/tooling-guide/pages/samm-cli.adoc index 7292043c0..bbbc732b0 100644 --- a/documentation/developer-guide/modules/tooling-guide/pages/samm-cli.adoc +++ b/documentation/developer-guide/modules/tooling-guide/pages/samm-cli.adoc @@ -147,7 +147,7 @@ The available options and their meaning can also be seen in the help text of the | _--language, -l_ : The language from the model for which a JSON schema should be generated (default: en) | `samm aspect AspectModel.ttl to schema -l de` | _--custom-resolver_ : use an external resolver for the resolution of the model elements | -.9+| [[aspect-to-sql]] aspect to sql | Generate SQL script that sets up a table for data for this Aspect | `samm aspect AspectModel.ttl to sql` +.10+| [[aspect-to-sql]] aspect to sql | Generate SQL script that sets up a table for data for this Aspect | `samm aspect AspectModel.ttl to sql` | _--output, -o_ : output file path (default: stdout) | | _--language, -l_ : The language from the model to use for generated comments | | _--dialect, -d_ : The SQL dialect to generate for (default: `databricks`) | @@ -161,6 +161,7 @@ The available options and their meaning can also be seen in the help text of the | _--decimal-precision, -dp_ : The precision to use for Databricks decimal columns (default: 10). See also notes in the xref:java-aspect-tooling.adoc#databricks-type-mapping[Databricks type mapping]. | + | _--custom-column, -col_ : Additional custom column definition, e.g. for databricks following the pattern `column_name DATATYPE [NOT NULL] [COMMENT 'custom']`. This parameter can be repeated for multiple columns. | `samm aspect AspectModel.ttl to sql --custom-column "column_name STRING NOT NULL COMMENT 'custom'"` .5+| [[aspect-to-aas]] aspect to aas | Generate an Asset Administration Shell (AAS) submodel template from an Aspect Model | `samm aspect AspectModel.ttl to aas` | _--output, -o_ : output file path (default: stdout) | diff --git a/tools/esmf-aspect-model-maven-plugin/src/main/java/org/eclipse/esmf/aspectmodel/GenerateSql.java b/tools/esmf-aspect-model-maven-plugin/src/main/java/org/eclipse/esmf/aspectmodel/GenerateSql.java index 0090f95fe..4b2c14008 100644 --- a/tools/esmf-aspect-model-maven-plugin/src/main/java/org/eclipse/esmf/aspectmodel/GenerateSql.java +++ b/tools/esmf-aspect-model-maven-plugin/src/main/java/org/eclipse/esmf/aspectmodel/GenerateSql.java @@ -15,12 +15,15 @@ import java.io.IOException; import java.io.OutputStream; +import java.util.List; import java.util.Locale; import java.util.Set; import org.eclipse.esmf.aspectmodel.generator.sql.AspectModelSqlGenerator; import org.eclipse.esmf.aspectmodel.generator.sql.SqlArtifact; import org.eclipse.esmf.aspectmodel.generator.sql.SqlGenerationConfig; +import org.eclipse.esmf.aspectmodel.generator.sql.databricks.DatabricksColumnDefinition; +import org.eclipse.esmf.aspectmodel.generator.sql.databricks.DatabricksColumnDefinitionParser; import org.eclipse.esmf.aspectmodel.generator.sql.databricks.DatabricksSqlGenerationConfig; import org.eclipse.esmf.aspectmodel.generator.sql.databricks.DatabricksSqlGenerationConfigBuilder; import org.eclipse.esmf.metamodel.AspectContext; @@ -57,10 +60,17 @@ public class GenerateSql extends AspectModelMojo { @Parameter( defaultValue = "denormalized" ) private String strategy = SqlGenerationConfig.MappingStrategy.DENORMALIZED.toString().toLowerCase(); + @Parameter( property = "column" ) + private List customColumns = List.of(); + @Override public void execute() throws MojoExecutionException { validateParameters(); + final List customColumnDefinitions = customColumns.stream() + .map( columnDefintion -> new DatabricksColumnDefinitionParser( columnDefintion ).get() ) + .toList(); + final Set aspectModels = loadModelsOrFail(); for ( final AspectContext context : aspectModels ) { final DatabricksSqlGenerationConfig generatorConfig = @@ -70,6 +80,7 @@ public void execute() throws MojoExecutionException { .includeColumnComments( includeColumnComments ) .createTableCommandPrefix( tableCommandPrefix ) .decimalPrecision( decimalPrecision ) + .customColumns( customColumnDefinitions ) .build(); final SqlGenerationConfig sqlConfig = new SqlGenerationConfig( SqlGenerationConfig.Dialect.valueOf( dialect.toUpperCase() ), SqlGenerationConfig.MappingStrategy.valueOf( strategy.toUpperCase() ), generatorConfig ); diff --git a/tools/esmf-aspect-model-maven-plugin/src/test/java/org/eclipse/esmf/aspectmodel/GenerateSqlTest.java b/tools/esmf-aspect-model-maven-plugin/src/test/java/org/eclipse/esmf/aspectmodel/GenerateSqlTest.java index 8d35d15c9..7dbd41b1a 100644 --- a/tools/esmf-aspect-model-maven-plugin/src/test/java/org/eclipse/esmf/aspectmodel/GenerateSqlTest.java +++ b/tools/esmf-aspect-model-maven-plugin/src/test/java/org/eclipse/esmf/aspectmodel/GenerateSqlTest.java @@ -46,6 +46,7 @@ public void testGenerateSqlWithAdjustedSettings() throws Exception { final String sqlContent = new String( Files.readAllBytes( generatedFile ) ); assertThat( sqlContent ).contains( "CREATE TABLE aspect_with_simple_types" ); + assertThat( sqlContent ).contains( "custom_column ARRAY NOT NULL COMMENT 'Custom column'" ); assertThat( sqlContent ).contains( "DECIMAL(23)" ); } } diff --git a/tools/esmf-aspect-model-maven-plugin/src/test/resources/generate-sql-pom-valid-aspect-model-adjusted-settings.xml b/tools/esmf-aspect-model-maven-plugin/src/test/resources/generate-sql-pom-valid-aspect-model-adjusted-settings.xml index 2050344e4..ee7deaee0 100644 --- a/tools/esmf-aspect-model-maven-plugin/src/test/resources/generate-sql-pom-valid-aspect-model-adjusted-settings.xml +++ b/tools/esmf-aspect-model-maven-plugin/src/test/resources/generate-sql-pom-valid-aspect-model-adjusted-settings.xml @@ -33,6 +33,9 @@ CREATE TABLE 23 + + custom_column ARRAY<STRING> NOT NULL COMMENT 'Custom column' + ${basedir}/target/test-artifacts diff --git a/tools/samm-cli/src/main/java/org/eclipse/esmf/aspect/to/AspectToSqlCommand.java b/tools/samm-cli/src/main/java/org/eclipse/esmf/aspect/to/AspectToSqlCommand.java index cbc11db25..2ef9f9d87 100644 --- a/tools/samm-cli/src/main/java/org/eclipse/esmf/aspect/to/AspectToSqlCommand.java +++ b/tools/samm-cli/src/main/java/org/eclipse/esmf/aspect/to/AspectToSqlCommand.java @@ -15,6 +15,7 @@ import java.io.IOException; import java.io.OutputStream; +import java.util.List; import java.util.Locale; import org.eclipse.esmf.AbstractCommand; @@ -24,6 +25,8 @@ import org.eclipse.esmf.aspectmodel.generator.sql.AspectModelSqlGenerator; import org.eclipse.esmf.aspectmodel.generator.sql.SqlArtifact; import org.eclipse.esmf.aspectmodel.generator.sql.SqlGenerationConfig; +import org.eclipse.esmf.aspectmodel.generator.sql.databricks.DatabricksColumnDefinition; +import org.eclipse.esmf.aspectmodel.generator.sql.databricks.DatabricksColumnDefinitionParser; import org.eclipse.esmf.aspectmodel.generator.sql.databricks.DatabricksSqlGenerationConfig; import org.eclipse.esmf.aspectmodel.generator.sql.databricks.DatabricksSqlGenerationConfigBuilder; import org.eclipse.esmf.exception.CommandException; @@ -72,6 +75,12 @@ public class AspectToSqlCommand extends AbstractCommand { description = "The precision to use for Databricks decimal columns, between 1 and 38. (default: ${DEFAULT-VALUE})" ) private int decimalPrecision = DatabricksSqlGenerationConfig.DECIMAL_DEFAULT_PRECISION; + @CommandLine.Option( + names = { "--custom-column", "-col" }, + description = "Custom column to add to the table, can be repeated for multiple columns", + converter = DatabricksColumnDefinitionTypeConverter.class ) + private List customColumns; + @CommandLine.ParentCommand private AspectToCommand parentCommand; @@ -81,6 +90,13 @@ public class AspectToSqlCommand extends AbstractCommand { @CommandLine.Mixin private LoggingMixin loggingMixin; + static class DatabricksColumnDefinitionTypeConverter implements CommandLine.ITypeConverter { + @Override + public DatabricksColumnDefinition convert( final String value ) throws Exception { + return new DatabricksColumnDefinitionParser( value ).get(); + } + } + @Override public void run() { final AspectContext context = loadModelOrFail( parentCommand.parentCommand.getInput(), customResolver ); @@ -91,6 +107,7 @@ public void run() { .includeColumnComments( includeColumnComments ) .createTableCommandPrefix( tableCommandPrefix ) .decimalPrecision( decimalPrecision ) + .customColumns( customColumns ) .build(); final SqlGenerationConfig sqlConfig = new SqlGenerationConfig( dialect, strategy, generatorConfig ); final SqlArtifact result = AspectModelSqlGenerator.INSTANCE.apply( context.aspect(), sqlConfig ); diff --git a/tools/samm-cli/src/test/java/org/eclipse/esmf/SammCliTest.java b/tools/samm-cli/src/test/java/org/eclipse/esmf/SammCliTest.java index 67994fe97..ce8ed1cf9 100644 --- a/tools/samm-cli/src/test/java/org/eclipse/esmf/SammCliTest.java +++ b/tools/samm-cli/src/test/java/org/eclipse/esmf/SammCliTest.java @@ -1058,6 +1058,15 @@ void testAspectToSqlToStdout() { assertThat( result.stderr() ).isEmpty(); } + @Test + void testAspectToSqlWithCustomColumnToStdout() { + final ExecutionResult result = sammCli.runAndExpectSuccess( "--disable-color", "aspect", defaultInputFile, "to", "sql", + "--custom-column", "custom ARRAY NOT NULL COMMENT 'Custom column'" ); + assertThat( result.stdout() ).contains( "CREATE TABLE" ); + assertThat( result.stdout() ).contains( "custom ARRAY NOT NULL COMMENT 'Custom column'" ); + assertThat( result.stderr() ).isEmpty(); + } + /** * Returns the File object for a test model file */