From 0ce3613cfc63665851aedb046dd053a0fa37d109 Mon Sep 17 00:00:00 2001 From: Padraig O'Sullivan Date: Wed, 8 Nov 2023 16:07:20 -0500 Subject: [PATCH] Add support for Oracle as backend database --- docs/resource-groups-api.md | 4 +- gateway-ha/pom.xml | 6 ++ .../ha/persistence/dao/QueryHistory.java | 12 ++-- .../ha/persistence/dao/ResourceGroups.java | 1 - .../ha/router/HaQueryHistoryManager.java | 8 +-- .../gateway-ha-persistence-oracle.sql | 70 +++++++++++++++++++ .../main/resources/gateway-ha-persistence.sql | 4 +- .../ha/router/TestResourceGroupsManager.java | 24 +++---- .../TestSpecificDbResourceGroupsManager.java | 1 - pom.xml | 1 + 10 files changed, 101 insertions(+), 30 deletions(-) create mode 100644 gateway-ha/src/main/resources/gateway-ha-persistence-oracle.sql diff --git a/docs/resource-groups-api.md b/docs/resource-groups-api.md index 39737684a..12a9142c0 100644 --- a/docs/resource-groups-api.md +++ b/docs/resource-groups-api.md @@ -96,12 +96,12 @@ curl -X POST http://localhost:8080/trino/resourcegroup/delete/{INSERT_ID_HERE} ## Add a selector -To add a single selector, specify all relevant fields in the body. Resource -group id should not be specified since the database should autoincrement it. +To add a single selector, specify all relevant fields in the body. ```$xslt curl -X POST http://localhost:8080/trino/selector/create \ -d '{ + "resourceGroupId": 1, \ "priority": 1, \ "userRegex": "selector1", \ "sourceRegex": "resourcegroup1", \ diff --git a/gateway-ha/pom.xml b/gateway-ha/pom.xml index 776854839..df84315c5 100644 --- a/gateway-ha/pom.xml +++ b/gateway-ha/pom.xml @@ -60,6 +60,12 @@ ${dep.h2.version} + + com.oracle.database.jdbc + ojdbc11 + ${dep.oracle.version} + + com.squareup.okhttp3 okhttp diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/QueryHistory.java b/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/QueryHistory.java index 3d45bb470..a93123941 100644 --- a/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/QueryHistory.java +++ b/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/QueryHistory.java @@ -28,12 +28,12 @@ public class QueryHistory extends Model { - private static final String queryId = "query_id"; - private static final String queryText = "query_text"; - private static final String backendUrl = "backend_url"; - private static final String userName = "user_name"; - private static final String source = "source"; - private static final String created = "created"; + public static final String queryId = "query_id"; + public static final String queryText = "query_text"; + public static final String backendUrl = "backend_url"; + public static final String userName = "user_name"; + public static final String source = "source"; + public static final String created = "created"; public static List upcast(List queryHistoryList) { diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/ResourceGroups.java b/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/ResourceGroups.java index c27ad439b..7809d1b2b 100644 --- a/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/ResourceGroups.java +++ b/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/ResourceGroups.java @@ -90,7 +90,6 @@ public static List upcast(List resourceGro */ public static void create(ResourceGroups model, ResourceGroupsDetail resourceGroupDetail) { - model.set(resourceGroupId, resourceGroupDetail.getResourceGroupId()); model.set(name, resourceGroupDetail.getName()); model.set(parent, resourceGroupDetail.getParent()); diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/router/HaQueryHistoryManager.java b/gateway-ha/src/main/java/io/trino/gateway/ha/router/HaQueryHistoryManager.java index 31cd316a6..4a36de43a 100644 --- a/gateway-ha/src/main/java/io/trino/gateway/ha/router/HaQueryHistoryManager.java +++ b/gateway-ha/src/main/java/io/trino/gateway/ha/router/HaQueryHistoryManager.java @@ -47,14 +47,10 @@ public List fetchQueryHistory(Optional user) { try { connectionManager.open(); - String sql = "select * from query_history"; if (user.isPresent()) { - sql += " where user_name = '" + user.orElseThrow() + "'"; + return QueryHistory.upcast(QueryHistory.where(QueryHistory.userName + " = '" + user.orElseThrow() + "'").limit(2000).orderBy(QueryHistory.created + " desc")); } - return QueryHistory.upcast(QueryHistory.findBySQL(String.join(" ", - sql, - "order by created desc", - "limit 2000"))); + return QueryHistory.upcast(QueryHistory.findAll().limit(2000).orderBy("created desc")); } finally { connectionManager.close(); diff --git a/gateway-ha/src/main/resources/gateway-ha-persistence-oracle.sql b/gateway-ha/src/main/resources/gateway-ha-persistence-oracle.sql new file mode 100644 index 000000000..af642c2b0 --- /dev/null +++ b/gateway-ha/src/main/resources/gateway-ha-persistence-oracle.sql @@ -0,0 +1,70 @@ +CREATE TABLE gateway_backend ( + name VARCHAR(256) PRIMARY KEY, + routing_group VARCHAR (256), + backend_url VARCHAR (256), + external_url VARCHAR (256), + active NUMBER(1) +); + +CREATE TABLE query_history ( + query_id VARCHAR(256) PRIMARY KEY, + query_text VARCHAR (256), + created NUMBER, + backend_url VARCHAR (256), + user_name VARCHAR(256), + source VARCHAR(256) +); +CREATE INDEX query_history_created_idx ON query_history(created); + +CREATE TABLE resource_groups ( + resource_group_id NUMBER GENERATED ALWAYS as IDENTITY(START with 1 INCREMENT by 1), + name VARCHAR(250) NOT NULL, + -- OPTIONAL POLICY CONTROLS + parent NUMBER, + jmx_export CHAR(1), + scheduling_policy VARCHAR(128), + scheduling_weight NUMBER, + -- REQUIRED QUOTAS + soft_memory_limit VARCHAR(128) NOT NULL, + max_queued INT NOT NULL, + hard_concurrency_limit NUMBER NOT NULL, + -- OPTIONAL QUOTAS + soft_concurrency_limit NUMBER, + soft_cpu_limit VARCHAR(128), + hard_cpu_limit VARCHAR(128), + environment VARCHAR(128), + PRIMARY KEY(resource_group_id), + FOREIGN KEY (parent) REFERENCES resource_groups (resource_group_id) ON DELETE CASCADE +); + +CREATE TABLE selectors ( + resource_group_id NUMBER NOT NULL, + priority NUMBER NOT NULL, + -- Regex fields -- these will be used as a regular expression pattern to + -- match against the field of the same name on queries + user_regex VARCHAR(512), + source_regex VARCHAR(512), + -- Selector fields -- these must match exactly. + query_type VARCHAR(512), + client_tags VARCHAR(512), + selector_resource_estimate VARCHAR(1024), + FOREIGN KEY (resource_group_id) REFERENCES resource_groups (resource_group_id) ON DELETE CASCADE +); + +CREATE TABLE resource_groups_global_properties ( + name VARCHAR(128) NOT NULL PRIMARY KEY, + value VARCHAR(512) NULL, + CHECK (name in ('cpu_quota_period')) +); + +CREATE TABLE exact_match_source_selectors( + environment VARCHAR(256), + update_time TIMESTAMP NOT NULL, + -- Selector fields which must exactly match a query + source VARCHAR(512) NOT NULL, + query_type VARCHAR(512), + resource_group_id VARCHAR(256) NOT NULL, + PRIMARY KEY (environment, source, resource_group_id), + UNIQUE (source, environment, query_type, resource_group_id) +); + diff --git a/gateway-ha/src/main/resources/gateway-ha-persistence.sql b/gateway-ha/src/main/resources/gateway-ha-persistence.sql index 6e1c09a47..ef818b71e 100644 --- a/gateway-ha/src/main/resources/gateway-ha-persistence.sql +++ b/gateway-ha/src/main/resources/gateway-ha-persistence.sql @@ -18,7 +18,7 @@ CREATE INDEX query_history_created_idx ON query_history(created); CREATE TABLE IF NOT EXISTS resource_groups ( resource_group_id BIGINT NOT NULL AUTO_INCREMENT, - name VARCHAR(250) NOT NULL UNIQUE, + name VARCHAR(250) NOT NULL, -- OPTIONAL POLICY CONTROLS parent BIGINT NULL, @@ -75,4 +75,4 @@ CREATE TABLE IF NOT EXISTS exact_match_source_selectors ( PRIMARY KEY (environment, source, query_type), UNIQUE (source, environment, query_type, resource_group_id) -); \ No newline at end of file +); diff --git a/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestResourceGroupsManager.java b/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestResourceGroupsManager.java index 35353474e..4fe9d189b 100644 --- a/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestResourceGroupsManager.java +++ b/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestResourceGroupsManager.java @@ -72,7 +72,7 @@ public void testReadResourceGroup() List resourceGroups = resourceGroupManager.readAllResourceGroups(null); assertEquals(resourceGroups.size(), 1); - assertEquals(0L, resourceGroups.get(0).getResourceGroupId()); + assertEquals(1L, resourceGroups.get(0).getResourceGroupId()); assertEquals("admin", resourceGroups.get(0).getName()); assertEquals(20, resourceGroups.get(0).getHardConcurrencyLimit()); assertEquals(200, resourceGroups.get(0).getMaxQueued()); @@ -85,7 +85,7 @@ public void testReadResourceGroup() public void testUpdateResourceGroup() { ResourceGroupsDetail resourceGroup = new ResourceGroupsDetail(); - resourceGroup.setResourceGroupId(0L); + resourceGroup.setResourceGroupId(1L); resourceGroup.setName("admin"); resourceGroup.setHardConcurrencyLimit(50); resourceGroup.setMaxQueued(50); @@ -99,7 +99,7 @@ public void testUpdateResourceGroup() /* Update resourceGroups that do not exist yet. * In this case, new resourceGroups should be created. */ - resourceGroup.setResourceGroupId(1L); + resourceGroup.setResourceGroupId(2L); resourceGroup.setName("localization-eng"); resourceGroup.setHardConcurrencyLimit(50); resourceGroup.setMaxQueued(70); @@ -121,14 +121,14 @@ public void testUpdateResourceGroup() assertEquals(3, resourceGroups.size()); // updated 2 non-existing groups, so count should be 3 - assertEquals(0L, resourceGroups.get(0).getResourceGroupId()); + assertEquals(1L, resourceGroups.get(0).getResourceGroupId()); assertEquals("admin", resourceGroups.get(0).getName()); assertEquals(50, resourceGroups.get(0).getHardConcurrencyLimit()); assertEquals(50, resourceGroups.get(0).getMaxQueued()); assertEquals(Boolean.FALSE, resourceGroups.get(0).getJmxExport()); assertEquals("20%", resourceGroups.get(0).getSoftMemoryLimit()); - assertEquals(1L, resourceGroups.get(1).getResourceGroupId()); + assertEquals(2L, resourceGroups.get(1).getResourceGroupId()); assertEquals("localization-eng", resourceGroups.get(1).getName()); assertEquals(50, resourceGroups.get(1).getHardConcurrencyLimit()); assertEquals(70, resourceGroups.get(1).getMaxQueued()); @@ -144,15 +144,15 @@ public void testDeleteResourceGroup() List resourceGroups = resourceGroupManager.readAllResourceGroups(null); assertEquals(3, resourceGroups.size()); - assertEquals(0L, resourceGroups.get(0).getResourceGroupId()); - assertEquals(1L, resourceGroups.get(1).getResourceGroupId()); + assertEquals(1L, resourceGroups.get(0).getResourceGroupId()); + assertEquals(2L, resourceGroups.get(1).getResourceGroupId()); assertEquals(3L, resourceGroups.get(2).getResourceGroupId()); resourceGroupManager.deleteResourceGroup(resourceGroups.get(1).getResourceGroupId(), null); resourceGroups = resourceGroupManager.readAllResourceGroups(null); assertEquals(2, resourceGroups.size()); - assertEquals(0L, resourceGroups.get(0).getResourceGroupId()); + assertEquals(1L, resourceGroups.get(0).getResourceGroupId()); assertEquals(3L, resourceGroups.get(1).getResourceGroupId()); } @@ -161,7 +161,7 @@ public void testDeleteResourceGroup() public void testCreateSelector() { SelectorsDetail selector = new SelectorsDetail(); - selector.setResourceGroupId(0L); + selector.setResourceGroupId(1L); selector.setPriority(0L); selector.setUserRegex("data-platform-admin"); selector.setSourceRegex("admin"); @@ -181,7 +181,7 @@ public void testReadSelector() List selectors = resourceGroupManager.readAllSelectors(null); assertEquals(1, selectors.size()); - assertEquals(0L, selectors.get(0).getResourceGroupId()); + assertEquals(1L, selectors.get(0).getResourceGroupId()); assertEquals(0L, selectors.get(0).getPriority()); assertEquals("data-platform-admin", selectors.get(0).getUserRegex()); assertEquals("admin", selectors.get(0).getSourceRegex()); @@ -196,7 +196,7 @@ public void testUpdateSelector() { SelectorsDetail selector = new SelectorsDetail(); - selector.setResourceGroupId(0L); + selector.setResourceGroupId(1L); selector.setPriority(0L); selector.setUserRegex("data-platform-admin_updated"); selector.setSourceRegex("admin_updated"); @@ -250,7 +250,7 @@ public void testDeleteSelector() { List selectors = resourceGroupManager.readAllSelectors(null); assertEquals(3, selectors.size()); - assertEquals(0L, selectors.get(0).getResourceGroupId()); + assertEquals(1L, selectors.get(0).getResourceGroupId()); resourceGroupManager.deleteSelector(selectors.get(0), null); selectors = resourceGroupManager.readAllSelectors(null); diff --git a/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestSpecificDbResourceGroupsManager.java b/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestSpecificDbResourceGroupsManager.java index 91d6c58bd..2493457f7 100644 --- a/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestSpecificDbResourceGroupsManager.java +++ b/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestSpecificDbResourceGroupsManager.java @@ -56,7 +56,6 @@ private void createResourceGroup() { ResourceGroupsDetail resourceGroup = new ResourceGroupsDetail(); - resourceGroup.setResourceGroupId(1L); resourceGroup.setName("admin2"); resourceGroup.setHardConcurrencyLimit(20); resourceGroup.setMaxQueued(200); diff --git a/pom.xml b/pom.xml index 531bec407..724d25b59 100644 --- a/pom.xml +++ b/pom.xml @@ -62,6 +62,7 @@ 5.8.0 8.0.17 3.9.0 + 21.9.0.0 2.3 3.0.0 42.6.0