From 270d79721d2f33746c6f5879cb6a878ca597a7f9 Mon Sep 17 00:00:00 2001 From: Femi3211 Date: Wed, 12 Jun 2024 11:37:30 +0200 Subject: [PATCH 1/7] modified: added table output of the data as a log when running query command --- .../java/queryhelper/service/AIService.java | 68 +++++++++++++++++-- 1 file changed, 62 insertions(+), 6 deletions(-) diff --git a/queryhelper/src/main/java/queryhelper/service/AIService.java b/queryhelper/src/main/java/queryhelper/service/AIService.java index 5fbf05a6..f0e1f412 100644 --- a/queryhelper/src/main/java/queryhelper/service/AIService.java +++ b/queryhelper/src/main/java/queryhelper/service/AIService.java @@ -16,15 +16,15 @@ import queryhelper.utils.FileUtils; import queryhelper.utils.PromptUtils; +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; import java.nio.file.Path; import java.sql.Driver; import java.sql.SQLException; import java.sql.Statement; import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Properties; +import java.util.*; public class AIService { @@ -57,10 +57,15 @@ public static GenericResponse generateQuery(String userQueryRequest, String apiK QueryDataResponse queryDataResponse = (QueryDataResponse) response.getData(); String csvFile = createCSVFile(queryDataResponse, queryRequest.getQuery(), dataDirectory, outputFileName); + String table = generateTablePreview(csvFile, 15); + response.setMessage( query + "\n" + - "Total rows: " + data.getRecords().size() + "\n" + - "Your response is saved to a CSV file named '" + csvFile + "'!" + "Your response is saved to a CSV file named '" + csvFile + "'!" + "\n" + + "Table Output:" +"\n" + + table + + "..." + "\n" + + "Total rows: " + data.getRecords().size() ); return response; @@ -146,5 +151,56 @@ public static String generateAIOutput(String apiKey, String aiModel, QueryReques return query; } + private static String generateTablePreview(String csvFile, int rowLimit) { + List rows = new ArrayList<>(); + try (BufferedReader reader = new BufferedReader(new FileReader(csvFile))) { + String line; + int rowCount = 0; + while ((line = reader.readLine()) != null && rowCount < rowLimit) { + String[] columns = line.split(","); + rows.add(columns); + rowCount++; + } + } catch (IOException e) { + throw new RuntimeException("Error reading CSV file", e); + } + + if (rows.isEmpty()) { + return "No data available to display."; + } + int maxColumns = rows.stream().mapToInt(row -> row.length).max().orElse(0); + int[] columnWidths = new int[maxColumns]; + for (String[] row : rows) { + for (int i = 0; i < row.length; i++) { + columnWidths[i] = Math.max(columnWidths[i], row[i].length()); + } + } + StringBuilder table = new StringBuilder(); + String rowSeparator = buildRowSeparator(columnWidths); + + table.append(rowSeparator); + for (String[] row : rows) { + table.append("|"); + for (int i = 0; i < maxColumns; i++) { + String cell = (i < row.length) ? row[i] : ""; + table.append(" ").append(String.format("%-" + columnWidths[i] + "s", cell)).append(" |"); + } + table.append("\n").append(rowSeparator); + } + + return table.toString(); + } + + private static String buildRowSeparator(int[] columnWidths) { + StringBuilder separator = new StringBuilder("+"); + for (int width : columnWidths) { + for (int i = 0; i < width + 2; i++) { + separator.append("-"); + } + separator.append("+"); + } + separator.append("\n"); + return separator.toString(); + } } \ No newline at end of file From 100304bfa697595a13f96096d936979123850098 Mon Sep 17 00:00:00 2001 From: jetroni Date: Wed, 7 Aug 2024 15:58:44 +0200 Subject: [PATCH 2/7] added: support for temporary, replicated tables for Kinetica --- .../rosetta/common/models/BaseModel.java | 28 +++++++++++++++ .../rosetta/common/models/Table.java | 2 +- .../kinetica/KineticaDDLGenerator.java | 18 ++++++++++ .../templates/kinetica/table/create.sqlt | 7 +++- .../table/KineticaTablesExtractor.java | 34 ++++++++++++++++++- 5 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 common/src/main/java/com/adaptivescale/rosetta/common/models/BaseModel.java diff --git a/common/src/main/java/com/adaptivescale/rosetta/common/models/BaseModel.java b/common/src/main/java/com/adaptivescale/rosetta/common/models/BaseModel.java new file mode 100644 index 00000000..7eb95bb5 --- /dev/null +++ b/common/src/main/java/com/adaptivescale/rosetta/common/models/BaseModel.java @@ -0,0 +1,28 @@ +package com.adaptivescale.rosetta.common.models; + +import java.util.HashMap; +import java.util.Map; + +public class BaseModel { + private Map additionalProperties = new HashMap<>(); + + public Map getAdditionalProperties() { + return additionalProperties; + } + + public void setAdditionalProperties(Map additionalProperties) { + this.additionalProperties = additionalProperties; + } + + public void addProperty(String name, Object value) { + this.additionalProperties.put(name, value); + } + + public Object getProperty(String name) { + return this.additionalProperties.get(name); + } + + public String getPropertyAsString(String name) { + return (String) getProperty(name); + } +} diff --git a/common/src/main/java/com/adaptivescale/rosetta/common/models/Table.java b/common/src/main/java/com/adaptivescale/rosetta/common/models/Table.java index 2969088f..379adaf5 100644 --- a/common/src/main/java/com/adaptivescale/rosetta/common/models/Table.java +++ b/common/src/main/java/com/adaptivescale/rosetta/common/models/Table.java @@ -4,7 +4,7 @@ import java.util.List; import java.util.Objects; -public class Table { +public class Table extends BaseModel { private String name; private String description; diff --git a/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java b/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java index 957f7809..39762b46 100644 --- a/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java +++ b/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java @@ -67,6 +67,9 @@ public String createTable(Table table, boolean dropTableIfExists) { stringBuilder.append(dropTable(table)); } + String tableType = extractTableType(table); + + createParams.put("tableType", tableType); createParams.put("schemaName", table.getSchema()); createParams.put("tableName", table.getName()); createParams.put("tableCode", definitionAsString); @@ -261,4 +264,19 @@ private String createSchema(String schema) { params.put("schemaName", schema); return TemplateEngine.process(SCHEMA_CREATE_TEMPLATE, params); } + + private String extractTableType(Table table) { + List tableTypes = new ArrayList<>(); + String shardKind = table.getPropertyAsString("shard_kind"); + String persistence = table.getPropertyAsString("persistence"); + + if (shardKind != null && shardKind.equals("R")) { + tableTypes.add("REPLICATED"); + } + if (persistence != null && persistence.equals("T")) { + tableTypes.add("TEMP"); + } + + return String.join(" ", tableTypes); + } } diff --git a/ddl/src/main/resources/templates/kinetica/table/create.sqlt b/ddl/src/main/resources/templates/kinetica/table/create.sqlt index 2d5414c2..d1b2f9b9 100644 --- a/ddl/src/main/resources/templates/kinetica/table/create.sqlt +++ b/ddl/src/main/resources/templates/kinetica/table/create.sqlt @@ -1 +1,6 @@ -CREATE TABLE "[(${schemaName})]"."[(${tableName})]"([(${tableCode})]); \ No newline at end of file +[# th:if="${tableType} == null or ${tableType} == ''"] +CREATE TABLE "[(${schemaName})]"."[(${tableName})]"([(${tableCode})]); +[/] +[# th:if="${tableType} != null and ${tableType} != ''"] +CREATE [(${tableType})] TABLE "[(${schemaName})]"."[(${tableName})]"([(${tableCode})]); +[/] \ No newline at end of file diff --git a/source/src/main/java/com/adataptivescale/rosetta/source/core/extractors/table/KineticaTablesExtractor.java b/source/src/main/java/com/adataptivescale/rosetta/source/core/extractors/table/KineticaTablesExtractor.java index b7c3da0c..04268012 100644 --- a/source/src/main/java/com/adataptivescale/rosetta/source/core/extractors/table/KineticaTablesExtractor.java +++ b/source/src/main/java/com/adataptivescale/rosetta/source/core/extractors/table/KineticaTablesExtractor.java @@ -1,11 +1,43 @@ package com.adataptivescale.rosetta.source.core.extractors.table; import com.adaptivescale.rosetta.common.annotations.RosettaModule; +import com.adaptivescale.rosetta.common.models.Table; +import com.adaptivescale.rosetta.common.models.input.Connection; import com.adaptivescale.rosetta.common.types.RosettaModuleTypes; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Collection; +import java.util.Optional; + @RosettaModule( name = "kinetica", type = RosettaModuleTypes.TABLE_EXTRACTOR ) -public class KineticaTablesExtractor extends DefaultTablesExtractor{ +public class KineticaTablesExtractor extends DefaultTablesExtractor { + @Override + public Collection extract(Connection target, java.sql.Connection connection) throws SQLException { + Collection
tables = super.extract(target, connection); + + return attachTableType(tables, connection); + } + + private Collection
attachTableType(Collection
tables, java.sql.Connection connection) throws SQLException { + ResultSet resultSet = connection.createStatement().executeQuery("SELECT object_name, schema_name, shard_kind, persistence FROM ki_catalog.ki_objects;"); + + while (resultSet.next()) { + String object_schema = resultSet.getString("schema_name"); + String object_name = resultSet.getString("object_name"); + Optional
found_table = tables.stream().filter(table -> table.getSchema().equals(object_schema) && table.getName().equals(object_name)).findFirst(); + if (found_table.isPresent()) { + found_table.get().addProperty("shard_kind", resultSet.getString("shard_kind")); + found_table.get().addProperty("persistence", resultSet.getString("persistence")); + } + } + + if (!resultSet.isClosed()) { + resultSet.close(); + } + return tables; + } } From d9a909af73188736b521de8f1adc72808e718898 Mon Sep 17 00:00:00 2001 From: Femi3211 Date: Wed, 7 Aug 2024 16:02:22 +0200 Subject: [PATCH 3/7] fixed: diff functionality for when comparing column properties of kinetica added: support for generating the DDL for kinetica indices --- .../common/models/ColumnProperties.java | 15 +++++++ .../kinetica/KineticaDDLGenerator.java | 45 ++++++++++++++++--- .../templates/kinetica/table/create.sqlt | 7 ++- .../rosetta/diff/kinetica/KineticaTester.java | 28 +++++++++--- 4 files changed, 82 insertions(+), 13 deletions(-) diff --git a/common/src/main/java/com/adaptivescale/rosetta/common/models/ColumnProperties.java b/common/src/main/java/com/adaptivescale/rosetta/common/models/ColumnProperties.java index 4a0a1019..abe35034 100644 --- a/common/src/main/java/com/adaptivescale/rosetta/common/models/ColumnProperties.java +++ b/common/src/main/java/com/adaptivescale/rosetta/common/models/ColumnProperties.java @@ -1,5 +1,7 @@ package com.adaptivescale.rosetta.common.models; +import java.util.Objects; + public class ColumnProperties { private String name; @@ -37,4 +39,17 @@ public String toString() { ", sequenceId=" + sequenceId + '}'; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ColumnProperties that = (ColumnProperties) o; + return Objects.equals(name, that.name) && Objects.equals(sequenceId, that.sequenceId); + } + + @Override + public int hashCode() { + return Objects.hash(name, sequenceId); + } } diff --git a/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java b/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java index 957f7809..63e71df0 100644 --- a/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java +++ b/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java @@ -1,10 +1,7 @@ package com.adaptivescale.rosetta.ddl.targets.kinetica; import com.adaptivescale.rosetta.common.annotations.RosettaModule; -import com.adaptivescale.rosetta.common.models.Column; -import com.adaptivescale.rosetta.common.models.Database; -import com.adaptivescale.rosetta.common.models.ForeignKey; -import com.adaptivescale.rosetta.common.models.Table; +import com.adaptivescale.rosetta.common.models.*; import com.adaptivescale.rosetta.common.types.RosettaModuleTypes; import com.adaptivescale.rosetta.ddl.DDL; import com.adaptivescale.rosetta.ddl.change.model.ColumnChange; @@ -13,7 +10,6 @@ import com.adaptivescale.rosetta.ddl.utils.TemplateEngine; import lombok.extern.slf4j.Slf4j; -import java.sql.DatabaseMetaData; import java.util.*; import java.util.stream.Collectors; @@ -59,6 +55,7 @@ public String createTable(Table table, boolean dropTableIfExists) { List foreignKeysForTable = getForeignKeysColumnNames(table); Optional primaryKeysForTable = createPrimaryKeysForTable(table, foreignKeysForTable); + List indicesForTable = getIndicesForTable(table); primaryKeysForTable.ifPresent(definitions::add); String definitionAsString = String.join(", ", definitions); @@ -70,6 +67,7 @@ public String createTable(Table table, boolean dropTableIfExists) { createParams.put("schemaName", table.getSchema()); createParams.put("tableName", table.getName()); createParams.put("tableCode", definitionAsString); + createParams.put("indices", String.join("\n", indicesForTable)); stringBuilder.append(TemplateEngine.process(TABLE_CREATE_TEMPLATE, createParams)); return stringBuilder.toString(); @@ -225,6 +223,43 @@ private Optional createPrimaryKeysForTable(Table table, List for return Optional.of("PRIMARY KEY (" + String.join(", ", primaryKeys) + ")"); } + private List getIndicesForTable(Table table) { + List result = table.getIndices(); + List indices = new ArrayList<>(); + List generateIndexStatement = generateIndexStatements(result.stream().findFirst().get().getColumnNames()); + return generateIndexStatement; + } + + public static List generateIndexStatements(List values) { + List indexStatements = new ArrayList<>(); + + for (String value : values) { + String statement = generateIndexStatement(value); + indexStatements.add(statement); + } + + return indexStatements; + } + + private static String generateIndexStatement(String value) { + if (value.contains("@")) { + String[] parts = value.split("@"); + String type = parts[0].toUpperCase(); + String[] ids = parts[1].split(":"); + String joinedIds = String.join(", ", ids); + switch (type.toLowerCase()) { + case "geospatial": + return String.format("GEOSPATIAL INDEX (%s)", joinedIds); + case "chunk_skip": + return String.format("CHUNK SKIP INDEX (%s)", joinedIds); + default: + return String.format("%s INDEX (%s)", type, joinedIds); + } + } else { + return String.format("INDEX (%s)", value); + } + } + private Optional foreignKeys(Table table) { String result = table.getColumns().stream() .filter(column -> column.getForeignKeys() != null && !column.getForeignKeys().isEmpty()) diff --git a/ddl/src/main/resources/templates/kinetica/table/create.sqlt b/ddl/src/main/resources/templates/kinetica/table/create.sqlt index 2d5414c2..14bcbdc0 100644 --- a/ddl/src/main/resources/templates/kinetica/table/create.sqlt +++ b/ddl/src/main/resources/templates/kinetica/table/create.sqlt @@ -1 +1,6 @@ -CREATE TABLE "[(${schemaName})]"."[(${tableName})]"([(${tableCode})]); \ No newline at end of file +[# th:if="${indices} == null or ${indices} == ''"] +CREATE TABLE "[(${schemaName})]"."[(${tableName})]"([(${tableCode})]); +[/] +[# th:if="${indices} != null and ${indices} != ''"] +CREATE TABLE "[(${schemaName})]"."[(${tableName})]"([(${tableCode})])[(${indices})]; +[/] \ No newline at end of file diff --git a/diff/src/main/java/com/adaptivescale/rosetta/diff/kinetica/KineticaTester.java b/diff/src/main/java/com/adaptivescale/rosetta/diff/kinetica/KineticaTester.java index ef594395..83ebfb98 100644 --- a/diff/src/main/java/com/adaptivescale/rosetta/diff/kinetica/KineticaTester.java +++ b/diff/src/main/java/com/adaptivescale/rosetta/diff/kinetica/KineticaTester.java @@ -1,12 +1,7 @@ package com.adaptivescale.rosetta.diff.kinetica; import com.adaptivescale.rosetta.common.annotations.RosettaModule; -import com.adaptivescale.rosetta.common.models.Column; -import com.adaptivescale.rosetta.common.models.Database; -import com.adaptivescale.rosetta.common.models.ForeignKey; -import com.adaptivescale.rosetta.common.models.Index; -import com.adaptivescale.rosetta.common.models.Table; -import com.adaptivescale.rosetta.common.models.View; +import com.adaptivescale.rosetta.common.models.*; import com.adaptivescale.rosetta.common.types.RosettaModuleTypes; import com.adaptivescale.rosetta.diff.DefaultTester; @@ -139,7 +134,7 @@ public List find(Database localValue, Database targetValue) { columnsChangesLogs.add(result); } - if (!Objects.equals(localColumn.getColumnProperties(), targetColumn.get().getColumnProperties())) { + if(!areColumnPropertiesEqual(localColumn.getColumnProperties(), targetColumn.get().getColumnProperties())) { String result = String.format(COLUMN_CHANGED_FORMAT, localColumn.getName(), table.getName(), "Column Properties", localColumn.columnPropertiesAsString(), targetColumn.get().columnPropertiesAsString()); @@ -402,4 +397,23 @@ private Optional
getTable(String tableName, Database targetValue) { private Optional getView(String viewName, Database targetValue) { return targetValue.getViews().stream().filter(targetView -> targetView.getName().equals(viewName)).findFirst(); } + + private static boolean areColumnPropertiesEqual(List list1, List list2) { + if (list1.size() != list2.size()) { + return false; + } + for (ColumnProperties prop1 : list1) { + boolean found = false; + for (ColumnProperties prop2 : list2) { + if (Objects.equals(prop1, prop2)){ + found = true; + break; + } + } + if (!found) { + return false; + } + } + return true; + } } From 93593a707b1594a24bd5879b0d41c652e4775ae6 Mon Sep 17 00:00:00 2001 From: jetroni Date: Thu, 8 Aug 2024 11:46:08 +0200 Subject: [PATCH 4/7] renamed: BaseModel to AbstractModel modified: additionalProperties to be null in beginning modified: template for creating table in Kinetica modified: attaching part for table types in Kinetica --- .../models/{BaseModel.java => AbstractModel.java} | 13 +++++-------- .../adaptivescale/rosetta/common/models/Table.java | 2 +- .../ddl/targets/kinetica/KineticaDDLGenerator.java | 3 +++ .../resources/templates/kinetica/table/create.sqlt | 7 +------ .../extractors/table/KineticaTablesExtractor.java | 8 ++++++-- 5 files changed, 16 insertions(+), 17 deletions(-) rename common/src/main/java/com/adaptivescale/rosetta/common/models/{BaseModel.java => AbstractModel.java} (62%) diff --git a/common/src/main/java/com/adaptivescale/rosetta/common/models/BaseModel.java b/common/src/main/java/com/adaptivescale/rosetta/common/models/AbstractModel.java similarity index 62% rename from common/src/main/java/com/adaptivescale/rosetta/common/models/BaseModel.java rename to common/src/main/java/com/adaptivescale/rosetta/common/models/AbstractModel.java index 7eb95bb5..e4562d43 100644 --- a/common/src/main/java/com/adaptivescale/rosetta/common/models/BaseModel.java +++ b/common/src/main/java/com/adaptivescale/rosetta/common/models/AbstractModel.java @@ -1,10 +1,9 @@ package com.adaptivescale.rosetta.common.models; -import java.util.HashMap; import java.util.Map; -public class BaseModel { - private Map additionalProperties = new HashMap<>(); +public class AbstractModel { + private Map additionalProperties; public Map getAdditionalProperties() { return additionalProperties; @@ -14,12 +13,10 @@ public void setAdditionalProperties(Map additionalProperties) { this.additionalProperties = additionalProperties; } - public void addProperty(String name, Object value) { - this.additionalProperties.put(name, value); - } - public Object getProperty(String name) { - return this.additionalProperties.get(name); + if (this.additionalProperties != null) + return this.additionalProperties.get(name); + return null; } public String getPropertyAsString(String name) { diff --git a/common/src/main/java/com/adaptivescale/rosetta/common/models/Table.java b/common/src/main/java/com/adaptivescale/rosetta/common/models/Table.java index 379adaf5..d59f6d28 100644 --- a/common/src/main/java/com/adaptivescale/rosetta/common/models/Table.java +++ b/common/src/main/java/com/adaptivescale/rosetta/common/models/Table.java @@ -4,7 +4,7 @@ import java.util.List; import java.util.Objects; -public class Table extends BaseModel { +public class Table extends AbstractModel { private String name; private String description; diff --git a/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java b/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java index 39762b46..acae2b4f 100644 --- a/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java +++ b/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java @@ -270,12 +270,15 @@ private String extractTableType(Table table) { String shardKind = table.getPropertyAsString("shard_kind"); String persistence = table.getPropertyAsString("persistence"); + // for Kinetica when shardKind is R it means that the table is REPLICATED + // when persistence is T it means that the table is TEMPORARY if (shardKind != null && shardKind.equals("R")) { tableTypes.add("REPLICATED"); } if (persistence != null && persistence.equals("T")) { tableTypes.add("TEMP"); } + tableTypes.add("TABLE"); return String.join(" ", tableTypes); } diff --git a/ddl/src/main/resources/templates/kinetica/table/create.sqlt b/ddl/src/main/resources/templates/kinetica/table/create.sqlt index d1b2f9b9..bbc968ce 100644 --- a/ddl/src/main/resources/templates/kinetica/table/create.sqlt +++ b/ddl/src/main/resources/templates/kinetica/table/create.sqlt @@ -1,6 +1 @@ -[# th:if="${tableType} == null or ${tableType} == ''"] -CREATE TABLE "[(${schemaName})]"."[(${tableName})]"([(${tableCode})]); -[/] -[# th:if="${tableType} != null and ${tableType} != ''"] -CREATE [(${tableType})] TABLE "[(${schemaName})]"."[(${tableName})]"([(${tableCode})]); -[/] \ No newline at end of file +CREATE [(${tableType})] "[(${schemaName})]"."[(${tableName})]"([(${tableCode})]); \ No newline at end of file diff --git a/source/src/main/java/com/adataptivescale/rosetta/source/core/extractors/table/KineticaTablesExtractor.java b/source/src/main/java/com/adataptivescale/rosetta/source/core/extractors/table/KineticaTablesExtractor.java index 04268012..a4a206ee 100644 --- a/source/src/main/java/com/adataptivescale/rosetta/source/core/extractors/table/KineticaTablesExtractor.java +++ b/source/src/main/java/com/adataptivescale/rosetta/source/core/extractors/table/KineticaTablesExtractor.java @@ -8,6 +8,8 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.Collection; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; @RosettaModule( @@ -30,8 +32,10 @@ private Collection
attachTableType(Collection
tables, java.sql.Con String object_name = resultSet.getString("object_name"); Optional
found_table = tables.stream().filter(table -> table.getSchema().equals(object_schema) && table.getName().equals(object_name)).findFirst(); if (found_table.isPresent()) { - found_table.get().addProperty("shard_kind", resultSet.getString("shard_kind")); - found_table.get().addProperty("persistence", resultSet.getString("persistence")); + Map additionalProps = new HashMap<>(); + additionalProps.put("shard_kind", resultSet.getString("shard_kind")); + additionalProps.put("persistence", resultSet.getString("persistence")); + found_table.get().setAdditionalProperties(additionalProps); } } From 5c4197cc71c47f3b4f2c6bbb48c44af5e200004f Mon Sep 17 00:00:00 2001 From: Femi3211 Date: Thu, 8 Aug 2024 12:21:24 +0200 Subject: [PATCH 5/7] applied Nushi recommandatios, added support for Cagra Index --- .../kinetica/KineticaDDLGenerator.java | 56 +++++++++++++++++-- .../rosetta/ddl/test/KineticaDDLTest.java | 12 ++-- .../rosetta/diff/kinetica/KineticaTester.java | 16 ++++-- 3 files changed, 68 insertions(+), 16 deletions(-) diff --git a/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java b/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java index 63e71df0..89e3b646 100644 --- a/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java +++ b/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java @@ -1,7 +1,11 @@ package com.adaptivescale.rosetta.ddl.targets.kinetica; import com.adaptivescale.rosetta.common.annotations.RosettaModule; -import com.adaptivescale.rosetta.common.models.*; +import com.adaptivescale.rosetta.common.models.Column; +import com.adaptivescale.rosetta.common.models.Database; +import com.adaptivescale.rosetta.common.models.ForeignKey; +import com.adaptivescale.rosetta.common.models.Table; +import com.adaptivescale.rosetta.common.models.Index; import com.adaptivescale.rosetta.common.types.RosettaModuleTypes; import com.adaptivescale.rosetta.ddl.DDL; import com.adaptivescale.rosetta.ddl.change.model.ColumnChange; @@ -40,6 +44,11 @@ public class KineticaDDLGenerator implements DDL { private final static String COLUMN_DROP_TEMPLATE = "kinetica/column/drop"; + private static final String GEOSPATIAL_INDEX_FORMAT = "GEOSPATIAL INDEX (%s)"; + private static final String CHUNK_SKIP_INDEX_FORMAT = "CHUNK SKIP INDEX (%s)"; + private static final String CAGRA_INDEX_FORMAT = "%s INDEX (%s) WITH OPTIONS (INDEX_OPTIONS = '%s')"; + private static final String GENERIC_INDEX_FORMAT = "%s INDEX (%s)"; + private final ColumnSQLDecoratorFactory columnSQLDecoratorFactory = new KineticaColumnDecoratorFactory(); @Override @@ -225,7 +234,9 @@ private Optional createPrimaryKeysForTable(Table table, List for private List getIndicesForTable(Table table) { List result = table.getIndices(); - List indices = new ArrayList<>(); + if (result == null){ + return Collections.emptyList(); + } List generateIndexStatement = generateIndexStatements(result.stream().findFirst().get().getColumnNames()); return generateIndexStatement; } @@ -247,13 +258,48 @@ private static String generateIndexStatement(String value) { String type = parts[0].toUpperCase(); String[] ids = parts[1].split(":"); String joinedIds = String.join(", ", ids); + switch (type.toLowerCase()) { case "geospatial": - return String.format("GEOSPATIAL INDEX (%s)", joinedIds); + return String.format(GEOSPATIAL_INDEX_FORMAT, joinedIds); + case "chunk_skip": - return String.format("CHUNK SKIP INDEX (%s)", joinedIds); + return String.format(CHUNK_SKIP_INDEX_FORMAT, joinedIds); + + case "cagra": + if (parts.length != 3) { + throw new IllegalArgumentException("Invalid format. Expected 3 parts separated by '@'."); + } + + String indexName = parts[1]; + String options = parts[2]; + + // Split options by "," and reformat them + String[] optionPairs = options.split(","); + StringBuilder optionsBuilder = new StringBuilder(); + + for (String option : optionPairs) { + // Split the key-value pair by ":" + String[] keyValue = option.split(":"); + if (keyValue.length == 2) { + optionsBuilder.append(keyValue[0]) + .append(": ") + .append(keyValue[1]) + .append(", "); + } else { + throw new IllegalArgumentException("Invalid option format. Expected key:value pairs."); + } + } + + if (optionsBuilder.length() > 0) { + optionsBuilder.setLength(optionsBuilder.length() - 2); + } + + // Construct the final string using the CAGRA_INDEX_FORMAT constant + return String.format(CAGRA_INDEX_FORMAT, type, indexName, optionsBuilder.toString()); + default: - return String.format("%s INDEX (%s)", type, joinedIds); + return String.format(GENERIC_INDEX_FORMAT, type, joinedIds); } } else { return String.format("INDEX (%s)", value); diff --git a/ddl/src/test/java/com/adaptivescale/rosetta/ddl/test/KineticaDDLTest.java b/ddl/src/test/java/com/adaptivescale/rosetta/ddl/test/KineticaDDLTest.java index 79224f5e..dd12ae6d 100644 --- a/ddl/src/test/java/com/adaptivescale/rosetta/ddl/test/KineticaDDLTest.java +++ b/ddl/src/test/java/com/adaptivescale/rosetta/ddl/test/KineticaDDLTest.java @@ -37,25 +37,25 @@ public class KineticaDDLTest { @Test public void createDB() throws IOException { String ddl = generateDDL("clean_database"); - Assertions.assertEquals("CREATE SCHEMA IF NOT EXISTS \"ROSETTA\";\r" + + Assertions.assertEquals(("CREATE SCHEMA IF NOT EXISTS \"ROSETTA\";\r" + "CREATE TABLE \"ROSETTA\".\"PLAYER\"(\"Name\" VARCHAR(100), \"Position\" VARCHAR(100), \"Number\" NUMBER NOT NULL );\r" + "\r" + - "CREATE TABLE \"ROSETTA\".\"USER\"(\"USER_ID\" NUMBER NOT NULL , PRIMARY KEY (\"USER_ID\"));", ddl); + "CREATE TABLE \"ROSETTA\".\"USER\"(\"USER_ID\" NUMBER NOT NULL , PRIMARY KEY (\"USER_ID\"));").replaceAll("(\\r|\\n|\\t)", ""), ddl.replaceAll("(\\r|\\n|\\t)", "")); } @Test public void addTable() throws IOException { String ddl = generateDDL("add_table"); - Assertions.assertEquals("CREATE TABLE \"ROSETTA\".\"Position\"(\"ID\" DECIMAL(10) NOT NULL , \"DESCRIPTION\" VARCHAR," + - " \"Name\" VARCHAR, PRIMARY KEY (\"ID\"));", ddl); + Assertions.assertEquals(("CREATE TABLE \"ROSETTA\".\"Position\"(\"ID\" DECIMAL(10) NOT NULL , \"DESCRIPTION\" VARCHAR," + + " \"Name\" VARCHAR, PRIMARY KEY (\"ID\"));").replaceAll("(\\r|\\n|\\t)", ""), ddl.replaceAll("(\\r|\\n|\\t)", "")); } @Test public void addTable2() throws IOException { String ddl = generateDDL("add_table2"); - Assertions.assertEquals("CREATE SCHEMA IF NOT EXISTS \"NEW\";\r" + + Assertions.assertEquals(("CREATE SCHEMA IF NOT EXISTS \"NEW\";\r" + "CREATE TABLE \"NEW\".\"Position\"(\"ID\" DECIMAL(10) NOT NULL , \"DESCRIPTION\" VARCHAR," + - " \"Name\" VARCHAR, PRIMARY KEY (\"ID\"));", ddl); + " \"Name\" VARCHAR, PRIMARY KEY (\"ID\"));").replaceAll("(\\r|\\n|\\t)", ""), ddl.replaceAll("(\\r|\\n|\\t)", "")); } @Test diff --git a/diff/src/main/java/com/adaptivescale/rosetta/diff/kinetica/KineticaTester.java b/diff/src/main/java/com/adaptivescale/rosetta/diff/kinetica/KineticaTester.java index 83ebfb98..2246ff4e 100644 --- a/diff/src/main/java/com/adaptivescale/rosetta/diff/kinetica/KineticaTester.java +++ b/diff/src/main/java/com/adaptivescale/rosetta/diff/kinetica/KineticaTester.java @@ -1,7 +1,13 @@ package com.adaptivescale.rosetta.diff.kinetica; import com.adaptivescale.rosetta.common.annotations.RosettaModule; -import com.adaptivescale.rosetta.common.models.*; +import com.adaptivescale.rosetta.common.models.Column; +import com.adaptivescale.rosetta.common.models.Database; +import com.adaptivescale.rosetta.common.models.ForeignKey; +import com.adaptivescale.rosetta.common.models.Index; +import com.adaptivescale.rosetta.common.models.Table; +import com.adaptivescale.rosetta.common.models.View; +import com.adaptivescale.rosetta.common.models.ColumnProperties; import com.adaptivescale.rosetta.common.types.RosettaModuleTypes; import com.adaptivescale.rosetta.diff.DefaultTester; @@ -398,13 +404,13 @@ private Optional getView(String viewName, Database targetValue) { return targetValue.getViews().stream().filter(targetView -> targetView.getName().equals(viewName)).findFirst(); } - private static boolean areColumnPropertiesEqual(List list1, List list2) { - if (list1.size() != list2.size()) { + private static boolean areColumnPropertiesEqual(List listLocal, List listTarget) { + if (listLocal.size() != listTarget.size()) { return false; } - for (ColumnProperties prop1 : list1) { + for (ColumnProperties prop1 : listLocal) { boolean found = false; - for (ColumnProperties prop2 : list2) { + for (ColumnProperties prop2 : listTarget) { if (Objects.equals(prop1, prop2)){ found = true; break; From 19ea8e8195af16cf58726952207466c0966af56c Mon Sep 17 00:00:00 2001 From: Femi3211 Date: Thu, 8 Aug 2024 13:42:21 +0200 Subject: [PATCH 6/7] added missing translation for kinetica BIGINT UNSIGNED, added functionality to bypass ki_home when generating Create Table DDL, resolved conflicts --- .../rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java | 5 ++++- .../src/main/resources/translation_matrix/translation.csv | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java b/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java index 557885dc..0b899845 100644 --- a/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java +++ b/ddl/src/main/java/com/adaptivescale/rosetta/ddl/targets/kinetica/KineticaDDLGenerator.java @@ -44,6 +44,8 @@ public class KineticaDDLGenerator implements DDL { private final static String COLUMN_DROP_TEMPLATE = "kinetica/column/drop"; + private final static List RESERVED_SCHEMA_NAMES = List.of("ki_home"); + private static final String GEOSPATIAL_INDEX_FORMAT = "GEOSPATIAL INDEX (%s)"; private static final String CHUNK_SKIP_INDEX_FORMAT = "CHUNK SKIP INDEX (%s)"; private static final String CAGRA_INDEX_FORMAT = "%s INDEX (%s) WITH OPTIONS (INDEX_OPTIONS = '%s')"; @@ -101,6 +103,7 @@ public String createDatabase(Database database, boolean dropTableIfExists) { stringBuilder.append( schemas .stream() + .filter(s -> !RESERVED_SCHEMA_NAMES.contains(s)) .map(this::createSchema) .collect(Collectors.joining()) ); @@ -237,7 +240,7 @@ private Optional createPrimaryKeysForTable(Table table, List for private List getIndicesForTable(Table table) { List result = table.getIndices(); - if (result == null){ + if (result == null || result.isEmpty()) { return Collections.emptyList(); } List generateIndexStatement = generateIndexStatements(result.stream().findFirst().get().getColumnNames()); diff --git a/translator/src/main/resources/translation_matrix/translation.csv b/translator/src/main/resources/translation_matrix/translation.csv index f62482cb..3d95dc5a 100644 --- a/translator/src/main/resources/translation_matrix/translation.csv +++ b/translator/src/main/resources/translation_matrix/translation.csv @@ -2358,4 +2358,5 @@ 2543;;kinetica;;DOUBLE;;kinetica;;double 2544;;kinetica;;DATE;;kinetica;;date 2545;;kinetica;;BYTES;;kinetica;;bytes -2546;;postgres;;_text;;oracle;;varchar2 \ No newline at end of file +2546;;postgres;;_text;;oracle;;varchar2 +2547;;kinetica;;UNSIGNED_BIGINT;;kinetica;;UNSIGNED BIGINT \ No newline at end of file From cebad6a842ca75640966b3295a3c31e50373afb9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 8 Aug 2024 12:45:53 +0000 Subject: [PATCH 7/7] Version bump: to 2.5.0 --- build.gradle | 2 +- cli/src/main/java/com/adaptivescale/rosetta/cli/Cli.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 1d7dae00..9e9f1276 100644 --- a/build.gradle +++ b/build.gradle @@ -9,7 +9,7 @@ repositories { allprojects { group = 'com.adaptivescale' - version = '2.4.0' + version = '2.5.0' sourceCompatibility = 11 targetCompatibility = 11 } diff --git a/cli/src/main/java/com/adaptivescale/rosetta/cli/Cli.java b/cli/src/main/java/com/adaptivescale/rosetta/cli/Cli.java index dc3b6a90..eea3f5a2 100644 --- a/cli/src/main/java/com/adaptivescale/rosetta/cli/Cli.java +++ b/cli/src/main/java/com/adaptivescale/rosetta/cli/Cli.java @@ -60,7 +60,7 @@ @Slf4j @CommandLine.Command(name = "cli", mixinStandardHelpOptions = true, - version = "2.4.0", + version = "2.5.0", description = "Declarative Database Management - DDL Transpiler" ) class Cli implements Callable {