Merge pull request #238 from AdaptiveScale/release-2.5.0

Release 2.5.0
AdaptiveScale · Aug 8, 2024 · 5e8dd1c · 5e8dd1c
2 parents fac3e69 + cebad6a
commit 5e8dd1c
Show file tree

Hide file tree

Showing 12 changed files with 283 additions and 20 deletions.
diff --git a/build.gradle b/build.gradle
@@ -9,7 +9,7 @@ repositories {
 
 allprojects {
     group = 'com.adaptivescale'
-    version = '2.4.0'
+    version = '2.5.0'
     sourceCompatibility = 11
     targetCompatibility = 11
 }

diff --git a/cli/src/main/java/com/adaptivescale/rosetta/cli/Cli.java b/cli/src/main/java/com/adaptivescale/rosetta/cli/Cli.java
@@ -60,7 +60,7 @@
 @Slf4j
 @CommandLine.Command(name = "cli",
         mixinStandardHelpOptions = true,
-        version = "2.4.0",
+        version = "2.5.0",
         description = "Declarative Database Management - DDL Transpiler"
 )
 class Cli implements Callable<Void> {

diff --git a/common/src/main/java/com/adaptivescale/rosetta/common/models/AbstractModel.java b/common/src/main/java/com/adaptivescale/rosetta/common/models/AbstractModel.java
@@ -0,0 +1,25 @@
+package com.adaptivescale.rosetta.common.models;
+
+import java.util.Map;
+
+public class AbstractModel {
+    private Map<String, Object> additionalProperties;
+
+    public Map<String, Object> getAdditionalProperties() {
+        return additionalProperties;
+    }
+
+    public void setAdditionalProperties(Map<String, Object> additionalProperties) {
+        this.additionalProperties = additionalProperties;
+    }
+
+    public Object getProperty(String name) {
+        if (this.additionalProperties != null)
+            return this.additionalProperties.get(name);
+        return null;
+    }
+
+    public String getPropertyAsString(String name) {
+        return (String) getProperty(name);
+    }
+}
diff --git a/common/src/main/java/com/adaptivescale/rosetta/common/models/ColumnProperties.java b/common/src/main/java/com/adaptivescale/rosetta/common/models/ColumnProperties.java
@@ -1,5 +1,7 @@
 package com.adaptivescale.rosetta.common.models;
 
+import java.util.Objects;
+
 public class ColumnProperties {
 
     private String name;
@@ -37,4 +39,17 @@ public String toString() {
                 ", sequenceId=" + sequenceId +
                 '}';
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ColumnProperties that = (ColumnProperties) o;
+        return Objects.equals(name, that.name) && Objects.equals(sequenceId, that.sequenceId);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, sequenceId);
+    }
 }
diff --git a/common/src/main/java/com/adaptivescale/rosetta/common/models/Table.java b/common/src/main/java/com/adaptivescale/rosetta/common/models/Table.java
@@ -4,7 +4,7 @@
 import java.util.List;
 import java.util.Objects;
 
-public class Table {
+public class Table extends AbstractModel {
 
     private String name;
     private String description;

diff --git a/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java b/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java
@@ -5,6 +5,7 @@
 import com.adaptivescale.rosetta.common.models.Database;
 import com.adaptivescale.rosetta.common.models.ForeignKey;
 import com.adaptivescale.rosetta.common.models.Table;
+import com.adaptivescale.rosetta.common.models.Index;
 import com.adaptivescale.rosetta.common.types.RosettaModuleTypes;
 import com.adaptivescale.rosetta.ddl.DDL;
 import com.adaptivescale.rosetta.ddl.change.model.ColumnChange;
@@ -13,7 +14,6 @@
 import com.adaptivescale.rosetta.ddl.utils.TemplateEngine;
 import lombok.extern.slf4j.Slf4j;
 
-import java.sql.DatabaseMetaData;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -44,6 +44,13 @@ public class KineticaDDLGenerator implements DDL {
 
     private final static String COLUMN_DROP_TEMPLATE = "kinetica/column/drop";
 
+    private final static List<String> RESERVED_SCHEMA_NAMES = List.of("ki_home");
+
+    private static final String GEOSPATIAL_INDEX_FORMAT = "GEOSPATIAL INDEX (%s)";
+    private static final String CHUNK_SKIP_INDEX_FORMAT = "CHUNK SKIP INDEX (%s)";
+    private static final String CAGRA_INDEX_FORMAT = "%s INDEX (%s) WITH OPTIONS (INDEX_OPTIONS = '%s')";
+    private static final String GENERIC_INDEX_FORMAT = "%s INDEX (%s)";
+
     private final ColumnSQLDecoratorFactory columnSQLDecoratorFactory = new KineticaColumnDecoratorFactory();
 
     @Override
@@ -59,6 +66,7 @@ public String createTable(Table table, boolean dropTableIfExists) {
 
         List<String> foreignKeysForTable = getForeignKeysColumnNames(table);
         Optional<String> primaryKeysForTable = createPrimaryKeysForTable(table, foreignKeysForTable);
+        List<String> indicesForTable = getIndicesForTable(table);
         primaryKeysForTable.ifPresent(definitions::add);
         String definitionAsString = String.join(", ", definitions);
 
@@ -67,9 +75,13 @@ public String createTable(Table table, boolean dropTableIfExists) {
             stringBuilder.append(dropTable(table));
         }
 
+        String tableType = extractTableType(table);
+
+        createParams.put("tableType", tableType);
         createParams.put("schemaName", table.getSchema());
         createParams.put("tableName", table.getName());
         createParams.put("tableCode", definitionAsString);
+        createParams.put("indices", String.join("\n", indicesForTable));
         stringBuilder.append(TemplateEngine.process(TABLE_CREATE_TEMPLATE, createParams));
 
         return stringBuilder.toString();
@@ -91,6 +103,7 @@ public String createDatabase(Database database, boolean dropTableIfExists) {
             stringBuilder.append(
                 schemas
                     .stream()
+                    .filter(s -> !RESERVED_SCHEMA_NAMES.contains(s))
                     .map(this::createSchema)
                     .collect(Collectors.joining())
             );
@@ -225,6 +238,80 @@ private Optional<String> createPrimaryKeysForTable(Table table, List<String> for
         return Optional.of("PRIMARY KEY (" + String.join(", ", primaryKeys) + ")");
     }
 
+    private List<String> getIndicesForTable(Table table) {
+        List<Index> result = table.getIndices();
+        if (result == null || result.isEmpty()) {
+            return Collections.emptyList();
+        }
+        List<String> generateIndexStatement = generateIndexStatements(result.stream().findFirst().get().getColumnNames());
+        return generateIndexStatement;
+    }
+
+    public static List<String> generateIndexStatements(List<String> values) {
+        List<String> indexStatements = new ArrayList<>();
+
+        for (String value : values) {
+            String statement = generateIndexStatement(value);
+            indexStatements.add(statement);
+        }
+
+        return indexStatements;
+    }
+
+    private static String generateIndexStatement(String value) {
+        if (value.contains("@")) {
+            String[] parts = value.split("@");
+            String type = parts[0].toUpperCase();
+            String[] ids = parts[1].split(":");
+            String joinedIds = String.join(", ", ids);
+
+            switch (type.toLowerCase()) {
+                case "geospatial":
+                    return String.format(GEOSPATIAL_INDEX_FORMAT, joinedIds);
+
+                case "chunk_skip":
+                    return String.format(CHUNK_SKIP_INDEX_FORMAT, joinedIds);
+
+                case "cagra":
+                    if (parts.length != 3) {
+                        throw new IllegalArgumentException("Invalid format. Expected 3 parts separated by '@'.");
+                    }
+
+                    String indexName = parts[1];
+                    String options = parts[2];
+
+                    // Split options by "," and reformat them
+                    String[] optionPairs = options.split(",");
+                    StringBuilder optionsBuilder = new StringBuilder();
+
+                    for (String option : optionPairs) {
+                        // Split the key-value pair by ":"
+                        String[] keyValue = option.split(":");
+                        if (keyValue.length == 2) {
+                            optionsBuilder.append(keyValue[0])
+                                    .append(": ")
+                                    .append(keyValue[1])
+                                    .append(", ");
+                        } else {
+                            throw new IllegalArgumentException("Invalid option format. Expected key:value pairs.");
+                        }
+                    }
+
+                    if (optionsBuilder.length() > 0) {
+                        optionsBuilder.setLength(optionsBuilder.length() - 2);
+                    }
+
+                    // Construct the final string using the CAGRA_INDEX_FORMAT constant
+                    return String.format(CAGRA_INDEX_FORMAT, type, indexName, optionsBuilder.toString());
+
+                default:
+                    return String.format(GENERIC_INDEX_FORMAT, type, joinedIds);
+            }
+        } else {
+            return String.format("INDEX (%s)", value);
+        }
+    }
+
     private Optional<String> foreignKeys(Table table) {
         String result = table.getColumns().stream()
             .filter(column -> column.getForeignKeys() != null && !column.getForeignKeys().isEmpty())
@@ -261,4 +348,22 @@ private String createSchema(String schema) {
         params.put("schemaName", schema);
         return TemplateEngine.process(SCHEMA_CREATE_TEMPLATE, params);
     }
+
+    private String extractTableType(Table table) {
+        List<String> tableTypes = new ArrayList<>();
+        String shardKind = table.getPropertyAsString("shard_kind");
+        String persistence = table.getPropertyAsString("persistence");
+
+        // for Kinetica when shardKind is R it means that the table is REPLICATED
+        // when persistence is T it means that the table is TEMPORARY
+        if (shardKind != null && shardKind.equals("R")) {
+            tableTypes.add("REPLICATED");
+        }
+        if (persistence != null && persistence.equals("T")) {
+            tableTypes.add("TEMP");
+        }
+        tableTypes.add("TABLE");
+
+        return String.join(" ", tableTypes);
+    }
 }
diff --git a/ddl/src/main/resources/templates/kinetica/table/create.sqlt b/ddl/src/main/resources/templates/kinetica/table/create.sqlt
@@ -1 +1,6 @@
-CREATE TABLE "[(${schemaName})]"."[(${tableName})]"([(${tableCode})]);
+[# th:if="${indices} == null or ${indices} == ''"]
+CREATE [(${tableType})] "[(${schemaName})]"."[(${tableName})]"([(${tableCode})]);
+[/]
+[# th:if="${indices} != null and ${indices} != ''"]
+CREATE [(${tableType})] "[(${schemaName})]"."[(${tableName})]"([(${tableCode})])[(${indices})];
+[/]
diff --git a/ddl/src/test/java/com/adaptivescale/rosetta/ddl/test/KineticaDDLTest.java b/ddl/src/test/java/com/adaptivescale/rosetta/ddl/test/KineticaDDLTest.java
@@ -37,25 +37,25 @@ public class KineticaDDLTest {
     @Test
     public void createDB() throws IOException {
         String ddl = generateDDL("clean_database");
-        Assertions.assertEquals("CREATE SCHEMA IF NOT EXISTS \"ROSETTA\";\r" +
+        Assertions.assertEquals(("CREATE SCHEMA IF NOT EXISTS \"ROSETTA\";\r" +
                 "CREATE TABLE \"ROSETTA\".\"PLAYER\"(\"Name\" VARCHAR(100), \"Position\" VARCHAR(100), \"Number\" NUMBER NOT NULL );\r" +
                 "\r" +
-                "CREATE TABLE \"ROSETTA\".\"USER\"(\"USER_ID\" NUMBER NOT NULL , PRIMARY KEY (\"USER_ID\"));", ddl);
+                "CREATE TABLE \"ROSETTA\".\"USER\"(\"USER_ID\" NUMBER NOT NULL , PRIMARY KEY (\"USER_ID\"));").replaceAll("(\\r|\\n|\\t)", ""), ddl.replaceAll("(\\r|\\n|\\t)", ""));
     }
 
     @Test
     public void addTable() throws IOException {
         String ddl = generateDDL("add_table");
-        Assertions.assertEquals("CREATE TABLE \"ROSETTA\".\"Position\"(\"ID\" DECIMAL(10) NOT NULL , \"DESCRIPTION\" VARCHAR," +
-                " \"Name\" VARCHAR, PRIMARY KEY (\"ID\"));", ddl);
+        Assertions.assertEquals(("CREATE TABLE \"ROSETTA\".\"Position\"(\"ID\" DECIMAL(10) NOT NULL , \"DESCRIPTION\" VARCHAR," +
+                " \"Name\" VARCHAR, PRIMARY KEY (\"ID\"));").replaceAll("(\\r|\\n|\\t)", ""), ddl.replaceAll("(\\r|\\n|\\t)", ""));
     }
 
     @Test
     public void addTable2() throws IOException {
         String ddl = generateDDL("add_table2");
-        Assertions.assertEquals("CREATE SCHEMA IF NOT EXISTS \"NEW\";\r" +
+        Assertions.assertEquals(("CREATE SCHEMA IF NOT EXISTS \"NEW\";\r" +
                 "CREATE TABLE \"NEW\".\"Position\"(\"ID\" DECIMAL(10) NOT NULL , \"DESCRIPTION\" VARCHAR," +
-                " \"Name\" VARCHAR, PRIMARY KEY (\"ID\"));", ddl);
+                " \"Name\" VARCHAR, PRIMARY KEY (\"ID\"));").replaceAll("(\\r|\\n|\\t)", ""), ddl.replaceAll("(\\r|\\n|\\t)", ""));
     }
 
     @Test

diff --git a/diff/src/main/java/com/adaptivescale/rosetta/diff/kinetica/KineticaTester.java b/diff/src/main/java/com/adaptivescale/rosetta/diff/kinetica/KineticaTester.java
@@ -7,6 +7,7 @@
 import com.adaptivescale.rosetta.common.models.Index;
 import com.adaptivescale.rosetta.common.models.Table;
 import com.adaptivescale.rosetta.common.models.View;
+import com.adaptivescale.rosetta.common.models.ColumnProperties;
 import com.adaptivescale.rosetta.common.types.RosettaModuleTypes;
 import com.adaptivescale.rosetta.diff.DefaultTester;
 
@@ -139,7 +140,7 @@ public List<String> find(Database localValue, Database targetValue) {
                     columnsChangesLogs.add(result);
                 }
 
-                if (!Objects.equals(localColumn.getColumnProperties(), targetColumn.get().getColumnProperties())) {
+                if(!areColumnPropertiesEqual(localColumn.getColumnProperties(), targetColumn.get().getColumnProperties())) {
                     String result = String.format(COLUMN_CHANGED_FORMAT, localColumn.getName(),
                             table.getName(), "Column Properties", localColumn.columnPropertiesAsString(),
                             targetColumn.get().columnPropertiesAsString());
@@ -402,4 +403,23 @@ private Optional<Table> getTable(String tableName, Database targetValue) {
     private Optional<View> getView(String viewName, Database targetValue) {
         return targetValue.getViews().stream().filter(targetView -> targetView.getName().equals(viewName)).findFirst();
     }
+
+    private static boolean areColumnPropertiesEqual(List<ColumnProperties> listLocal, List<ColumnProperties> listTarget) {
+        if (listLocal.size() != listTarget.size()) {
+            return false;
+        }
+        for (ColumnProperties prop1 : listLocal) {
+            boolean found = false;
+            for (ColumnProperties prop2 : listTarget) {
+                if (Objects.equals(prop1, prop2)){
+                    found = true;
+                    break;
+                }
+            }
+            if (!found) {
+                return false;
+            }
+        }
+        return true;
+    }
 }