Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update SQL generator for handling collections of entities #692

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
+93 −17
Open

Update SQL generator for handling collections of entities #692

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
4571c57
Update SQL generator for handling collections of entities
Yauhenikapl Dec 17, 2024
23beac4
Improve SQL generator
Yauhenikapl Dec 23, 2024
659914d
Add id column for nested Entities
Yauhenikapl Dec 25, 2024
ff5332b
Fix tests
Yauhenikapl Dec 25, 2024
58b7362
Fix logic and update tests
Yauhenikapl Jan 10, 2025
c8680e7
Fix styles
Yauhenikapl Jan 10, 2025
6d2920b
Fix styles
Yauhenikapl Jan 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 52 additions & 5 deletions ...smf/aspectmodel/generator/sql/databricks/AspectModelDatabricksDenormalizedSqlVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ private String escapeComment( final String comment ) {

@Override
public String visitAspect( final Aspect aspect, final Context context ) {

final String columnDeclarations = visitStructureElement( aspect, context );
final String comment = config.includeTableComment()
? Optional.ofNullable( aspect.getDescription( config.commentLanguage() ) ).map( description ->
Expand Down Expand Up @@ -256,11 +257,57 @@ public String visitCollection( final Collection collection, final Context contex
? Optional.ofNullable( Optional.ofNullable( context.forceDescriptionFromElement() ).orElse( property )
.getDescription( config.commentLanguage() ) )
: Optional.empty();
final String typeDef = type.isComplexType()
? entityToStruct( type.as( ComplexType.class ), false ).toString()
: type.accept( this, context );
return column( context.prefix(), "ARRAY<" + typeDef + ">", property.isOptional() || context.forceOptional(),
comment );

if ( type.isComplexType() ) {
// Flattening collections of complex types
final ComplexType complexType = type.as( ComplexType.class );
return processComplexType( complexType, context, context.prefix() );
} else {
// Handle scalar types normally
final String typeDef = type.accept( this, context );
return column(
context.prefix(),
"ARRAY<" + typeDef + ">",
property.isOptional() || context.forceOptional(),
comment
);
}
}

private String processComplexType( final ComplexType entity, final Context context, final String parentPrefix ) {
StringBuilder columns = new StringBuilder();
final String lineDelimiter = ",\n ";

entity.getAllProperties().forEach( property -> {
if ( property.getDataType().isEmpty() || property.isNotInPayload() ) {
return; // Skip properties with no data type or not in payload
}

final Type type = property.getDataType().get();
String columnPrefix = columnName( property );

if ( parentPrefix.contains( LEVEL_DELIMITER ) ) {
columnPrefix = parentPrefix + LEVEL_DELIMITER + columnName( property );

columns.append( column( columnPrefix + "_id", "BIGINT", false, Optional.empty() ) )
.append( lineDelimiter );
}

if ( type instanceof Scalar ) {
final String typeDef = type.accept( this, context );
columns.append( column( columnPrefix, typeDef, property.isOptional(),
Optional.ofNullable( property.getDescription( config.commentLanguage() ) ) ) )
.append( lineDelimiter );
} else if ( type instanceof ComplexType ) {
columns.append( processComplexType( type.as( ComplexType.class ), context, columnPrefix ) );
}
} );

if ( !columns.isEmpty() && columns.toString().endsWith( ",\n " ) ) {
columns.setLength( columns.length() - 4 ); // Remove last ",\n "
}

return columns.toString();
}

private DatabricksType.DatabricksStruct entityToStruct( final ComplexType entity, final boolean isInsideNestedType ) {
Expand Down
40 changes: 30 additions & 10 deletions ...aspectmodel/generator/sql/databricks/AspectModelDatabricksDenormalizedSqlVisitorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ CREATE TABLE IF NOT EXISTS aspect_with_collections_with_element_characteristic_a
void testAspectWithCollectionAndElementCharacteristic() {
assertThat( sql( TestAspect.ASPECT_WITH_COLLECTION_AND_ELEMENT_CHARACTERISTIC ) ).isEqualTo( """
CREATE TABLE IF NOT EXISTS aspect_with_collection_and_element_characteristic (
items ARRAY<STRUCT<test_property: STRING NOT NULL>> NOT NULL
test_property STRING NOT NULL
)
TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithCollectionAndElementCharacteristic');
""" );
Expand Down Expand Up @@ -106,7 +106,8 @@ CREATE TABLE IF NOT EXISTS aspect_with_complex_collection_enum (
void testAspectWithComplexEntityCollectionEnum() {
assertThat( sql( TestAspect.ASPECT_WITH_COMPLEX_ENTITY_COLLECTION_ENUM ) ).isEqualTo( """
CREATE TABLE IF NOT EXISTS aspect_with_complex_entity_collection_enum (
my_property_one__entity_property_one ARRAY<STRUCT<entity_property_two: STRING NOT NULL>> NOT NULL
my_property_one__entity_property_one__entity_property_two_id BIGINT NOT NULL,
my_property_one__entity_property_one__entity_property_two STRING NOT NULL
)
TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithComplexEntityCollectionEnum');
""" );
Expand Down Expand Up @@ -198,7 +199,11 @@ CREATE TABLE IF NOT EXISTS aspect_with_entity_instance_with_scalar_list_property
void testAspectWithEntityList() {
assertThat( sql( TestAspect.ASPECT_WITH_ENTITY_LIST ) ).isEqualTo( """
CREATE TABLE IF NOT EXISTS aspect_with_entity_list (
test_list ARRAY<STRUCT<test_string: STRING NOT NULL, test_int: INT NOT NULL, test_float: FLOAT NOT NULL, test_local_date_time: TIMESTAMP NOT NULL, random_value: STRING NOT NULL>> NOT NULL
test_string STRING NOT NULL,
test_int INT NOT NULL,
test_float FLOAT NOT NULL,
test_local_date_time TIMESTAMP NOT NULL,
random_value STRING NOT NULL
)
TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithEntityList');
""" );
Expand All @@ -209,7 +214,8 @@ void testAspectWithEntityWithNestedEntityListProperty() {
assertThat( sql( TestAspect.ASPECT_WITH_ENTITY_WITH_NESTED_ENTITY_LIST_PROPERTY ) ).isEqualTo( """
CREATE TABLE IF NOT EXISTS aspect_with_entity_with_nested_entity_list_property (
test_property__code SMALLINT NOT NULL,
test_property__test_list ARRAY<STRUCT<nested_entity_property: STRING NOT NULL>> NOT NULL
test_property__test_list__nested_entity_property_id BIGINT NOT NULL,
test_property__test_list__nested_entity_property STRING NOT NULL
)
TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithEntityWithNestedEntityListProperty');
""" );
Expand All @@ -219,7 +225,8 @@ CREATE TABLE IF NOT EXISTS aspect_with_entity_with_nested_entity_list_property (
void testAspectWithExtendedEntity() {
assertThat( sql( TestAspect.ASPECT_WITH_EXTENDED_ENTITY ) ).isEqualTo( """
CREATE TABLE IF NOT EXISTS aspect_with_extended_entity (
test_property ARRAY<STRUCT<parent_string: STRING NOT NULL, parent_of_parent_string: STRING NOT NULL>> NOT NULL COMMENT 'This is a test property.'
parent_string STRING NOT NULL,
parent_of_parent_string STRING NOT NULL
)
TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithExtendedEntity');
""" );
Expand Down Expand Up @@ -308,8 +315,16 @@ CREATE TABLE IF NOT EXISTS aspect_with_multiple_entities_on_multiple_levels (
void testAspectWithMultipleEntityCollections() {
assertThat( sql( TestAspect.ASPECT_WITH_MULTIPLE_ENTITY_COLLECTIONS ) ).isEqualTo( """
CREATE TABLE IF NOT EXISTS aspect_with_multiple_entity_collections (
test_list_one ARRAY<STRUCT<test_string: STRING NOT NULL, test_int: INT NOT NULL, test_float: FLOAT NOT NULL, test_local_date_time: TIMESTAMP NOT NULL, random_value: STRING NOT NULL>> NOT NULL,
test_list_two ARRAY<STRUCT<test_string: STRING NOT NULL, test_int: INT NOT NULL, test_float: FLOAT NOT NULL, test_local_date_time: TIMESTAMP NOT NULL, random_value: STRING NOT NULL>> NOT NULL
test_string STRING NOT NULL,
test_int INT NOT NULL,
test_float FLOAT NOT NULL,
test_local_date_time TIMESTAMP NOT NULL,
random_value STRING NOT NULL,
test_string STRING NOT NULL,
test_int INT NOT NULL,
test_float FLOAT NOT NULL,
test_local_date_time TIMESTAMP NOT NULL,
random_value STRING NOT NULL
Yauhenikapl marked this conversation as resolved.
Show resolved Hide resolved
)
TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithMultipleEntityCollections');
""" );
Expand All @@ -319,7 +334,11 @@ CREATE TABLE IF NOT EXISTS aspect_with_multiple_entity_collections (
void testAspectWithNestedEntityList() {
assertThat( sql( TestAspect.ASPECT_WITH_NESTED_ENTITY_LIST ) ).isEqualTo( """
CREATE TABLE IF NOT EXISTS aspect_with_nested_entity_list (
test_list ARRAY<STRUCT<test_string: STRING NOT NULL, test_int: INT NOT NULL, test_float: FLOAT NOT NULL, test_second_list: STRUCT<test_local_date_time: TIMESTAMP, random_value: STRING> NOT NULL>> NOT NULL
test_string STRING NOT NULL,
test_int INT NOT NULL,
test_float FLOAT NOT NULL,
test_local_date_time TIMESTAMP NOT NULL,
random_value STRING NOT NULL
)
TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithNestedEntityList');
""" );
Expand Down Expand Up @@ -471,7 +490,8 @@ CREATE TABLE IF NOT EXISTS aspect_with_sorted_set (
void testAspectWithTimeSeries() {
assertThat( sql( TestAspect.ASPECT_WITH_TIME_SERIES ) ).isEqualTo( """
CREATE TABLE IF NOT EXISTS aspect_with_time_series (
test_property ARRAY<STRUCT<value: STRING NOT NULL COMMENT 'The value that was recorded and is part of a time series.', timestamp: TIMESTAMP NOT NULL COMMENT 'The specific point in time when the corresponding value was recorded.'>> NOT NULL COMMENT 'This is a test property.'
value STRING NOT NULL COMMENT 'The value that was recorded and is part of a time series.',
timestamp TIMESTAMP NOT NULL COMMENT 'The specific point in time when the corresponding value was recorded.'
)
COMMENT 'This is a test description'
TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithTimeSeries');
Expand All @@ -482,7 +502,7 @@ CREATE TABLE IF NOT EXISTS aspect_with_time_series (
void testAspectWithComplexSet() {
assertThat( sql( TestAspect.ASPECT_WITH_COMPLEX_SET ) ).isEqualTo( """
CREATE TABLE IF NOT EXISTS aspect_with_complex_set (
test_property ARRAY<STRUCT<product_id: STRING NOT NULL>> NOT NULL COMMENT 'This is a test property.'
product_id STRING NOT NULL
)
COMMENT 'This is a test description'
TBLPROPERTIES ('x-samm-aspect-model-urn'='urn:samm:org.eclipse.esmf.test:1.0.0#AspectWithComplexSet');
Expand Down
2 changes: 1 addition & 1 deletion ...lipse/esmf/aspectmodel/generator/sql/databricks/DatabricksColumnDefinitionParserTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;

public class DatabricksColumnDefinitionParserTest extends DatabricksTestBase {
class DatabricksColumnDefinitionParserTest extends DatabricksTestBase {
@Test
void testMinimalDefinition() {
final DatabricksColumnDefinition definition = new DatabricksColumnDefinitionParser( "abc STRING" ).get();
Expand Down
Loading