diff --git a/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/AspectModelDatabricksDenormalizedSqlVisitor.java b/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/AspectModelDatabricksDenormalizedSqlVisitor.java index ac7449d33..85756fbb0 100644 --- a/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/AspectModelDatabricksDenormalizedSqlVisitor.java +++ b/core/esmf-aspect-model-document-generators/src/main/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/AspectModelDatabricksDenormalizedSqlVisitor.java @@ -35,6 +35,7 @@ import org.eclipse.esmf.metamodel.characteristic.Collection; import org.eclipse.esmf.metamodel.characteristic.Either; import org.eclipse.esmf.metamodel.characteristic.Trait; +import org.eclipse.esmf.metamodel.characteristic.impl.DefaultList; import org.eclipse.esmf.metamodel.vocabulary.SAMM; import org.eclipse.esmf.samm.KnownVersion; @@ -170,6 +171,7 @@ private String escapeComment( final String comment ) { @Override public String visitAspect( final Aspect aspect, final Context context ) { + final String columnDeclarations = visitStructureElement( aspect, context ); final String comment = config.includeTableComment() ? Optional.ofNullable( aspect.getDescription( config.commentLanguage() ) ).map( description -> @@ -194,7 +196,7 @@ public String visitProperty( final Property property, final Context context ) { } return property.getCharacteristic().get().accept( this, context.copy() - .prefix( ( context.prefix().isEmpty() ? "" : context.prefix() + LEVEL_DELIMITER ) + columnName( property ) ) + .prefix( (context.prefix().isEmpty() ? "" : context.prefix() + LEVEL_DELIMITER) + columnName( property ) ) .currentProperty( property ) .build() ); } @@ -256,11 +258,63 @@ public String visitCollection( final Collection collection, final Context contex ? Optional.ofNullable( Optional.ofNullable( context.forceDescriptionFromElement() ).orElse( property ) .getDescription( config.commentLanguage() ) ) : Optional.empty(); - final String typeDef = type.isComplexType() - ? entityToStruct( type.as( ComplexType.class ), false ).toString() - : type.accept( this, context ); - return column( context.prefix(), "ARRAY<" + typeDef + ">", property.isOptional() || context.forceOptional(), - comment ); + + if ( type.isComplexType() ) { + // Flattening collections of complex types + final ComplexType complexType = type.as( ComplexType.class ); + collection.as( Collection.class ).getCollectionType(); + return processComplexType( complexType, context, context.prefix(), collection.is( DefaultList.class ) ); + } else { + // Handle scalar types normally + final String typeDef = type.accept( this, context ); + return column( + context.prefix(), + "ARRAY<" + typeDef + ">", + property.isOptional() || context.forceOptional(), + comment + ); + } + } + + private String processComplexType( final ComplexType entity, final Context context, final String parentPrefix, + final boolean isDefaultList ) { + StringBuilder columns = new StringBuilder(); + final String lineDelimiter = ",\n "; + + entity.getAllProperties().forEach( property -> { + if ( property.getDataType().isEmpty() || property.isNotInPayload() ) { + return; // Skip properties with no data type or not in payload + } + + final Type type = property.getDataType().get(); + String columnPrefix = columnName( property ); + + if ( parentPrefix.contains( LEVEL_DELIMITER ) ) { + columnPrefix = parentPrefix + LEVEL_DELIMITER + columnName( property ); + + columns.append( column( columnPrefix + "_id", "BIGINT", false, Optional.empty() ) ) + .append( lineDelimiter ); + } + + if ( isDefaultList && !parentPrefix.contains( LEVEL_DELIMITER ) ) { + columnPrefix = parentPrefix + LEVEL_DELIMITER + columnName( property ); + } + + if ( type instanceof Scalar ) { + final String typeDef = type.accept( this, context ); + columns.append( column( columnPrefix, typeDef, property.isOptional(), + Optional.ofNullable( property.getDescription( config.commentLanguage() ) ) ) ) + .append( lineDelimiter ); + } else if ( type instanceof ComplexType ) { + columns.append( processComplexType( type.as( ComplexType.class ), context, columnPrefix, type.is( DefaultList.class ) ) ); + } + } ); + + if ( !columns.isEmpty() && columns.toString().endsWith( ",\n " ) ) { + columns.setLength( columns.length() - 4 ); // Remove last ",\n " + } + + return columns.toString(); } private DatabricksType.DatabricksStruct entityToStruct( final ComplexType entity, final boolean isInsideNestedType ) { diff --git a/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/AspectModelDatabricksDenormalizedSqlVisitorTest.java b/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/AspectModelDatabricksDenormalizedSqlVisitorTest.java index b38ecfa8a..2d8adff0a 100644 --- a/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/AspectModelDatabricksDenormalizedSqlVisitorTest.java +++ b/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/AspectModelDatabricksDenormalizedSqlVisitorTest.java @@ -73,7 +73,7 @@ CREATE TABLE IF NOT EXISTS aspect_with_collections_with_element_characteristic_a void testAspectWithCollectionAndElementCharacteristic() { assertThat( sql( TestAspect.ASPECT_WITH_COLLECTION_AND_ELEMENT_CHARACTERISTIC ) ).isEqualTo( """ CREATE TABLE IF NOT EXISTS aspect_with_collection_and_element_characteristic ( - items ARRAY> NOT NULL + test_property STRING NOT NULL ) TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithCollectionAndElementCharacteristic'); """ ); @@ -106,7 +106,8 @@ CREATE TABLE IF NOT EXISTS aspect_with_complex_collection_enum ( void testAspectWithComplexEntityCollectionEnum() { assertThat( sql( TestAspect.ASPECT_WITH_COMPLEX_ENTITY_COLLECTION_ENUM ) ).isEqualTo( """ CREATE TABLE IF NOT EXISTS aspect_with_complex_entity_collection_enum ( - my_property_one__entity_property_one ARRAY> NOT NULL + my_property_one__entity_property_one__entity_property_two_id BIGINT NOT NULL, + my_property_one__entity_property_one__entity_property_two STRING NOT NULL ) TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithComplexEntityCollectionEnum'); """ ); @@ -198,7 +199,11 @@ CREATE TABLE IF NOT EXISTS aspect_with_entity_instance_with_scalar_list_property void testAspectWithEntityList() { assertThat( sql( TestAspect.ASPECT_WITH_ENTITY_LIST ) ).isEqualTo( """ CREATE TABLE IF NOT EXISTS aspect_with_entity_list ( - test_list ARRAY> NOT NULL + test_list__test_string STRING NOT NULL, + test_list__test_int INT NOT NULL, + test_list__test_float FLOAT NOT NULL, + test_list__test_local_date_time TIMESTAMP NOT NULL, + test_list__random_value STRING NOT NULL ) TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithEntityList'); """ ); @@ -209,7 +214,8 @@ void testAspectWithEntityWithNestedEntityListProperty() { assertThat( sql( TestAspect.ASPECT_WITH_ENTITY_WITH_NESTED_ENTITY_LIST_PROPERTY ) ).isEqualTo( """ CREATE TABLE IF NOT EXISTS aspect_with_entity_with_nested_entity_list_property ( test_property__code SMALLINT NOT NULL, - test_property__test_list ARRAY> NOT NULL + test_property__test_list__nested_entity_property_id BIGINT NOT NULL, + test_property__test_list__nested_entity_property STRING NOT NULL ) TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithEntityWithNestedEntityListProperty'); """ ); @@ -219,7 +225,8 @@ CREATE TABLE IF NOT EXISTS aspect_with_entity_with_nested_entity_list_property ( void testAspectWithExtendedEntity() { assertThat( sql( TestAspect.ASPECT_WITH_EXTENDED_ENTITY ) ).isEqualTo( """ CREATE TABLE IF NOT EXISTS aspect_with_extended_entity ( - test_property ARRAY> NOT NULL COMMENT 'This is a test property.' + parent_string STRING NOT NULL, + parent_of_parent_string STRING NOT NULL ) TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithExtendedEntity'); """ ); @@ -308,8 +315,16 @@ CREATE TABLE IF NOT EXISTS aspect_with_multiple_entities_on_multiple_levels ( void testAspectWithMultipleEntityCollections() { assertThat( sql( TestAspect.ASPECT_WITH_MULTIPLE_ENTITY_COLLECTIONS ) ).isEqualTo( """ CREATE TABLE IF NOT EXISTS aspect_with_multiple_entity_collections ( - test_list_one ARRAY> NOT NULL, - test_list_two ARRAY> NOT NULL + test_list_one__test_string STRING NOT NULL, + test_list_one__test_int INT NOT NULL, + test_list_one__test_float FLOAT NOT NULL, + test_list_one__test_local_date_time TIMESTAMP NOT NULL, + test_list_one__random_value STRING NOT NULL, + test_list_two__test_string STRING NOT NULL, + test_list_two__test_int INT NOT NULL, + test_list_two__test_float FLOAT NOT NULL, + test_list_two__test_local_date_time TIMESTAMP NOT NULL, + test_list_two__random_value STRING NOT NULL ) TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithMultipleEntityCollections'); """ ); @@ -319,7 +334,13 @@ CREATE TABLE IF NOT EXISTS aspect_with_multiple_entity_collections ( void testAspectWithNestedEntityList() { assertThat( sql( TestAspect.ASPECT_WITH_NESTED_ENTITY_LIST ) ).isEqualTo( """ CREATE TABLE IF NOT EXISTS aspect_with_nested_entity_list ( - test_list ARRAY NOT NULL>> NOT NULL + test_list__test_string STRING NOT NULL, + test_list__test_int INT NOT NULL, + test_list__test_float FLOAT NOT NULL, + test_list__test_second_list__test_local_date_time_id BIGINT NOT NULL, + test_list__test_second_list__test_local_date_time TIMESTAMP NOT NULL, + test_list__test_second_list__random_value_id BIGINT NOT NULL, + test_list__test_second_list__random_value STRING NOT NULL ) TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithNestedEntityList'); """ ); @@ -471,7 +492,8 @@ CREATE TABLE IF NOT EXISTS aspect_with_sorted_set ( void testAspectWithTimeSeries() { assertThat( sql( TestAspect.ASPECT_WITH_TIME_SERIES ) ).isEqualTo( """ CREATE TABLE IF NOT EXISTS aspect_with_time_series ( - test_property ARRAY> NOT NULL COMMENT 'This is a test property.' + value STRING NOT NULL COMMENT 'The value that was recorded and is part of a time series.', + timestamp TIMESTAMP NOT NULL COMMENT 'The specific point in time when the corresponding value was recorded.' ) COMMENT 'This is a test description' TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithTimeSeries'); @@ -482,7 +504,7 @@ CREATE TABLE IF NOT EXISTS aspect_with_time_series ( void testAspectWithComplexSet() { assertThat( sql( TestAspect.ASPECT_WITH_COMPLEX_SET ) ).isEqualTo( """ CREATE TABLE IF NOT EXISTS aspect_with_complex_set ( - test_property ARRAY> NOT NULL COMMENT 'This is a test property.' + product_id STRING NOT NULL ) COMMENT 'This is a test description' TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithComplexSet'); diff --git a/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksColumnDefinitionParserTest.java b/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksColumnDefinitionParserTest.java index 5746472ef..68501ce67 100644 --- a/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksColumnDefinitionParserTest.java +++ b/core/esmf-aspect-model-document-generators/src/test/java/org/eclipse/esmf/aspectmodel/generator/sql/databricks/DatabricksColumnDefinitionParserTest.java @@ -23,7 +23,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; -public class DatabricksColumnDefinitionParserTest extends DatabricksTestBase { +class DatabricksColumnDefinitionParserTest extends DatabricksTestBase { @Test void testMinimalDefinition() { final DatabricksColumnDefinition definition = new DatabricksColumnDefinitionParser( "abc STRING" ).get();