diff --git a/docs/resource-groups-api.md b/docs/resource-groups-api.md index 7fa591996..3a3efa901 100644 --- a/docs/resource-groups-api.md +++ b/docs/resource-groups-api.md @@ -95,12 +95,12 @@ curl -X POST http://localhost:8080/trino/resourcegroup/delete/{INSERT_ID_HERE} ## Add a selector -To add a single selector, specify all relevant fields in the body. Resource -group id should not be specified since the database should autoincrement it. +To add a single selector, specify all relevant fields in the body. ```$xslt curl -X POST http://localhost:8080/trino/selector/create \ -d '{ + "resourceGroupId": 1, \ "priority": 1, \ "userRegex": "selector1", \ "sourceRegex": "resourcegroup1", \ diff --git a/gateway-ha/pom.xml b/gateway-ha/pom.xml index 27b1fcf62..826cc0e2d 100644 --- a/gateway-ha/pom.xml +++ b/gateway-ha/pom.xml @@ -146,6 +146,11 @@ postgresql ${dep.postgresql.version} + + com.oracle.database.jdbc + ojdbc11 + ${dep.oracle.version} + org.junit.jupiter diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/QueryHistory.java b/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/QueryHistory.java index f4274df7d..40a3a97aa 100644 --- a/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/QueryHistory.java +++ b/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/QueryHistory.java @@ -12,12 +12,12 @@ @Table("query_history") @Cached public class QueryHistory extends Model { - private static final String queryId = "query_id"; - private static final String queryText = "query_text"; - private static final String backendUrl = "backend_url"; - private static final String userName = "user_name"; - private static final String source = "source"; - private static final String created = "created"; + public static final String queryId = "query_id"; + public static final String queryText = "query_text"; + public static final String backendUrl = "backend_url"; + public static final String userName = "user_name"; + public static final String source = "source"; + public static final String created = "created"; public static List upcast(List queryHistoryList) { List queryDetails = new ArrayList<>(); diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/ResourceGroups.java b/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/ResourceGroups.java index 515b26475..c826e4d2d 100644 --- a/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/ResourceGroups.java +++ b/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/ResourceGroups.java @@ -77,7 +77,6 @@ public static List upcast(List resourceGro * @param resourceGroupDetail */ public static void create(ResourceGroups model, ResourceGroupsDetail resourceGroupDetail) { - model.set(resourceGroupId, resourceGroupDetail.getResourceGroupId()); model.set(name, resourceGroupDetail.getName()); model.set(parent, resourceGroupDetail.getParent()); diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/router/HaQueryHistoryManager.java b/gateway-ha/src/main/java/io/trino/gateway/ha/router/HaQueryHistoryManager.java index cc42c940e..5f1b4f9d1 100644 --- a/gateway-ha/src/main/java/io/trino/gateway/ha/router/HaQueryHistoryManager.java +++ b/gateway-ha/src/main/java/io/trino/gateway/ha/router/HaQueryHistoryManager.java @@ -29,14 +29,10 @@ public void submitQueryDetail(QueryDetail queryDetail) { public List fetchQueryHistory(Optional user) { try { connectionManager.open(); - String sql = "select * from query_history"; if (user.isPresent()) { - sql += " where user_name = '" + user.get() + "'"; + return QueryHistory.upcast(QueryHistory.where(QueryHistory.userName + " = '" + user.get() + "'").limit(2000).orderBy(QueryHistory.created + " desc")); } - return QueryHistory.upcast(QueryHistory.findBySQL(String.join(" ", - sql, - "order by created desc", - "limit 2000"))); + return QueryHistory.upcast(QueryHistory.findAll().limit(2000).orderBy("created desc")); } finally { connectionManager.close(); } diff --git a/gateway-ha/src/main/resources/gateway-ha-persistence-oracle.sql b/gateway-ha/src/main/resources/gateway-ha-persistence-oracle.sql new file mode 100644 index 000000000..af642c2b0 --- /dev/null +++ b/gateway-ha/src/main/resources/gateway-ha-persistence-oracle.sql @@ -0,0 +1,70 @@ +CREATE TABLE gateway_backend ( + name VARCHAR(256) PRIMARY KEY, + routing_group VARCHAR (256), + backend_url VARCHAR (256), + external_url VARCHAR (256), + active NUMBER(1) +); + +CREATE TABLE query_history ( + query_id VARCHAR(256) PRIMARY KEY, + query_text VARCHAR (256), + created NUMBER, + backend_url VARCHAR (256), + user_name VARCHAR(256), + source VARCHAR(256) +); +CREATE INDEX query_history_created_idx ON query_history(created); + +CREATE TABLE resource_groups ( + resource_group_id NUMBER GENERATED ALWAYS as IDENTITY(START with 1 INCREMENT by 1), + name VARCHAR(250) NOT NULL, + -- OPTIONAL POLICY CONTROLS + parent NUMBER, + jmx_export CHAR(1), + scheduling_policy VARCHAR(128), + scheduling_weight NUMBER, + -- REQUIRED QUOTAS + soft_memory_limit VARCHAR(128) NOT NULL, + max_queued INT NOT NULL, + hard_concurrency_limit NUMBER NOT NULL, + -- OPTIONAL QUOTAS + soft_concurrency_limit NUMBER, + soft_cpu_limit VARCHAR(128), + hard_cpu_limit VARCHAR(128), + environment VARCHAR(128), + PRIMARY KEY(resource_group_id), + FOREIGN KEY (parent) REFERENCES resource_groups (resource_group_id) ON DELETE CASCADE +); + +CREATE TABLE selectors ( + resource_group_id NUMBER NOT NULL, + priority NUMBER NOT NULL, + -- Regex fields -- these will be used as a regular expression pattern to + -- match against the field of the same name on queries + user_regex VARCHAR(512), + source_regex VARCHAR(512), + -- Selector fields -- these must match exactly. + query_type VARCHAR(512), + client_tags VARCHAR(512), + selector_resource_estimate VARCHAR(1024), + FOREIGN KEY (resource_group_id) REFERENCES resource_groups (resource_group_id) ON DELETE CASCADE +); + +CREATE TABLE resource_groups_global_properties ( + name VARCHAR(128) NOT NULL PRIMARY KEY, + value VARCHAR(512) NULL, + CHECK (name in ('cpu_quota_period')) +); + +CREATE TABLE exact_match_source_selectors( + environment VARCHAR(256), + update_time TIMESTAMP NOT NULL, + -- Selector fields which must exactly match a query + source VARCHAR(512) NOT NULL, + query_type VARCHAR(512), + resource_group_id VARCHAR(256) NOT NULL, + PRIMARY KEY (environment, source, resource_group_id), + UNIQUE (source, environment, query_type, resource_group_id) +); + diff --git a/gateway-ha/src/main/resources/gateway-ha-persistence.sql b/gateway-ha/src/main/resources/gateway-ha-persistence.sql index 6e1c09a47..ef818b71e 100644 --- a/gateway-ha/src/main/resources/gateway-ha-persistence.sql +++ b/gateway-ha/src/main/resources/gateway-ha-persistence.sql @@ -18,7 +18,7 @@ CREATE INDEX query_history_created_idx ON query_history(created); CREATE TABLE IF NOT EXISTS resource_groups ( resource_group_id BIGINT NOT NULL AUTO_INCREMENT, - name VARCHAR(250) NOT NULL UNIQUE, + name VARCHAR(250) NOT NULL, -- OPTIONAL POLICY CONTROLS parent BIGINT NULL, @@ -75,4 +75,4 @@ CREATE TABLE IF NOT EXISTS exact_match_source_selectors ( PRIMARY KEY (environment, source, query_type), UNIQUE (source, environment, query_type, resource_group_id) -); \ No newline at end of file +); diff --git a/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestResourceGroupsManager.java b/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestResourceGroupsManager.java index d78be5c22..25de59024 100644 --- a/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestResourceGroupsManager.java +++ b/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestResourceGroupsManager.java @@ -65,7 +65,7 @@ public void testReadResourceGroup() { List resourceGroups = resourceGroupManager.readAllResourceGroups(null); assertEquals(resourceGroups.size(), 1); - assertEquals(0L, resourceGroups.get(0).getResourceGroupId()); + assertEquals(1L, resourceGroups.get(0).getResourceGroupId()); assertEquals("admin", resourceGroups.get(0).getName()); assertEquals(20, resourceGroups.get(0).getHardConcurrencyLimit()); assertEquals(200, resourceGroups.get(0).getMaxQueued()); @@ -77,7 +77,7 @@ public void testReadResourceGroup() { @Order(3) public void testUpdateResourceGroup() { ResourceGroupsDetail resourceGroup = new ResourceGroupsDetail(); - resourceGroup.setResourceGroupId(0L); + resourceGroup.setResourceGroupId(1L); resourceGroup.setName("admin"); resourceGroup.setHardConcurrencyLimit(50); resourceGroup.setMaxQueued(50); @@ -91,7 +91,7 @@ public void testUpdateResourceGroup() { /* Update resourceGroups that do not exist yet. * In this case, new resourceGroups should be created. */ - resourceGroup.setResourceGroupId(1L); + resourceGroup.setResourceGroupId(2L); resourceGroup.setName("localization-eng"); resourceGroup.setHardConcurrencyLimit(50); resourceGroup.setMaxQueued(70); @@ -113,14 +113,14 @@ public void testUpdateResourceGroup() { assertEquals(3, resourceGroups.size()); // updated 2 non-existing groups, so count should be 3 - assertEquals(0L, resourceGroups.get(0).getResourceGroupId()); + assertEquals(1L, resourceGroups.get(0).getResourceGroupId()); assertEquals("admin", resourceGroups.get(0).getName()); assertEquals(50, resourceGroups.get(0).getHardConcurrencyLimit()); assertEquals(50, resourceGroups.get(0).getMaxQueued()); assertEquals(Boolean.FALSE, resourceGroups.get(0).getJmxExport()); assertEquals("20%", resourceGroups.get(0).getSoftMemoryLimit()); - assertEquals(1L, resourceGroups.get(1).getResourceGroupId()); + assertEquals(2L, resourceGroups.get(1).getResourceGroupId()); assertEquals("localization-eng", resourceGroups.get(1).getName()); assertEquals(50, resourceGroups.get(1).getHardConcurrencyLimit()); assertEquals(70, resourceGroups.get(1).getMaxQueued()); @@ -135,15 +135,15 @@ public void testDeleteResourceGroup() { List resourceGroups = resourceGroupManager.readAllResourceGroups(null); assertEquals(3, resourceGroups.size()); - assertEquals(0L, resourceGroups.get(0).getResourceGroupId()); - assertEquals(1L, resourceGroups.get(1).getResourceGroupId()); + assertEquals(1L, resourceGroups.get(0).getResourceGroupId()); + assertEquals(2L, resourceGroups.get(1).getResourceGroupId()); assertEquals(3L, resourceGroups.get(2).getResourceGroupId()); resourceGroupManager.deleteResourceGroup(resourceGroups.get(1).getResourceGroupId(), null); resourceGroups = resourceGroupManager.readAllResourceGroups(null); assertEquals(2, resourceGroups.size()); - assertEquals(0L, resourceGroups.get(0).getResourceGroupId()); + assertEquals(1L, resourceGroups.get(0).getResourceGroupId()); assertEquals(3L, resourceGroups.get(1).getResourceGroupId()); } @@ -151,7 +151,7 @@ public void testDeleteResourceGroup() { @Order(5) public void testCreateSelector() { SelectorsDetail selector = new SelectorsDetail(); - selector.setResourceGroupId(0L); + selector.setResourceGroupId(1L); selector.setPriority(0L); selector.setUserRegex("data-platform-admin"); selector.setSourceRegex("admin"); @@ -170,7 +170,7 @@ public void testReadSelector() { List selectors = resourceGroupManager.readAllSelectors(null); assertEquals(1, selectors.size()); - assertEquals(0L, selectors.get(0).getResourceGroupId()); + assertEquals(1L, selectors.get(0).getResourceGroupId()); assertEquals(0L, selectors.get(0).getPriority()); assertEquals("data-platform-admin", selectors.get(0).getUserRegex()); assertEquals("admin", selectors.get(0).getSourceRegex()); @@ -184,7 +184,7 @@ public void testReadSelector() { public void testUpdateSelector() { SelectorsDetail selector = new SelectorsDetail(); - selector.setResourceGroupId(0L); + selector.setResourceGroupId(1L); selector.setPriority(0L); selector.setUserRegex("data-platform-admin_updated"); selector.setSourceRegex("admin_updated"); @@ -237,7 +237,7 @@ public void testUpdateSelector() { public void testDeleteSelector() { List selectors = resourceGroupManager.readAllSelectors(null); assertEquals(3, selectors.size()); - assertEquals(0L, selectors.get(0).getResourceGroupId()); + assertEquals(1L, selectors.get(0).getResourceGroupId()); resourceGroupManager.deleteSelector(selectors.get(0), null); selectors = resourceGroupManager.readAllSelectors(null); diff --git a/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestSpecificDbResourceGroupsManager.java b/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestSpecificDbResourceGroupsManager.java index c81bd1fe6..2905a60ff 100644 --- a/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestSpecificDbResourceGroupsManager.java +++ b/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestSpecificDbResourceGroupsManager.java @@ -41,7 +41,6 @@ public void setUp() { private void createResourceGroup() { ResourceGroupsDetail resourceGroup = new ResourceGroupsDetail(); - resourceGroup.setResourceGroupId(1L); resourceGroup.setName("admin2"); resourceGroup.setHardConcurrencyLimit(20); resourceGroup.setMaxQueued(200); diff --git a/pom.xml b/pom.xml index 164695e5f..83f045398 100644 --- a/pom.xml +++ b/pom.xml @@ -59,6 +59,7 @@ 11.0.15 1.18.22 3.9.0 + 21.9.0.0 2.3 3.0.0 1.2.1