Add support for Oracle as backend database

trinodb · Jan 10, 2024 · 0ce3613 · 0ce3613
1 parent 0f0a10f
commit 0ce3613
Show file tree

Hide file tree

Showing 10 changed files with 101 additions and 30 deletions.
diff --git a/docs/resource-groups-api.md b/docs/resource-groups-api.md
@@ -96,12 +96,12 @@ curl -X POST http://localhost:8080/trino/resourcegroup/delete/{INSERT_ID_HERE}
 
 ## Add a selector
 
-To add a single selector, specify all relevant fields in the body. Resource
-group id should not be specified since the database should autoincrement it.
+To add a single selector, specify all relevant fields in the body.
 
 ```$xslt
 curl -X POST http://localhost:8080/trino/selector/create \
  -d '{
+        "resourceGroupId": 1, \
         "priority": 1, \
         "userRegex": "selector1", \
         "sourceRegex": "resourcegroup1", \

diff --git a/gateway-ha/pom.xml b/gateway-ha/pom.xml
@@ -60,6 +60,12 @@
             <version>${dep.h2.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.oracle.database.jdbc</groupId>
+            <artifactId>ojdbc11</artifactId>
+            <version>${dep.oracle.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>com.squareup.okhttp3</groupId>
             <artifactId>okhttp</artifactId>

diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/QueryHistory.java b/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/QueryHistory.java
@@ -28,12 +28,12 @@
 public class QueryHistory
         extends Model
 {
-    private static final String queryId = "query_id";
-    private static final String queryText = "query_text";
-    private static final String backendUrl = "backend_url";
-    private static final String userName = "user_name";
-    private static final String source = "source";
-    private static final String created = "created";
+    public static final String queryId = "query_id";
+    public static final String queryText = "query_text";
+    public static final String backendUrl = "backend_url";
+    public static final String userName = "user_name";
+    public static final String source = "source";
+    public static final String created = "created";
 
     public static List<QueryHistoryManager.QueryDetail> upcast(List<QueryHistory> queryHistoryList)
     {

diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/ResourceGroups.java b/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/ResourceGroups.java
@@ -90,7 +90,6 @@ public static List<ResourceGroupsDetail> upcast(List<ResourceGroups> resourceGro
      */
     public static void create(ResourceGroups model, ResourceGroupsDetail resourceGroupDetail)
     {
-        model.set(resourceGroupId, resourceGroupDetail.getResourceGroupId());
         model.set(name, resourceGroupDetail.getName());
 
         model.set(parent, resourceGroupDetail.getParent());

diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/router/HaQueryHistoryManager.java b/gateway-ha/src/main/java/io/trino/gateway/ha/router/HaQueryHistoryManager.java
@@ -47,14 +47,10 @@ public List<QueryDetail> fetchQueryHistory(Optional<String> user)
     {
         try {
             connectionManager.open();
-            String sql = "select * from query_history";
             if (user.isPresent()) {
-                sql += " where user_name = '" + user.orElseThrow() + "'";
+                return QueryHistory.upcast(QueryHistory.where(QueryHistory.userName + " = '" + user.orElseThrow() + "'").limit(2000).orderBy(QueryHistory.created + " desc"));
             }
-            return QueryHistory.upcast(QueryHistory.findBySQL(String.join(" ",
-                    sql,
-                    "order by created desc",
-                    "limit 2000")));
+            return QueryHistory.upcast(QueryHistory.findAll().limit(2000).orderBy("created desc"));
         }
         finally {
             connectionManager.close();

diff --git a/gateway-ha/src/main/resources/gateway-ha-persistence-oracle.sql b/gateway-ha/src/main/resources/gateway-ha-persistence-oracle.sql
@@ -0,0 +1,70 @@
+CREATE TABLE gateway_backend (
+    name VARCHAR(256) PRIMARY KEY,
+    routing_group VARCHAR (256),
+    backend_url VARCHAR (256),
+    external_url VARCHAR (256),
+    active NUMBER(1)
+);
+
+CREATE TABLE query_history (
+    query_id VARCHAR(256) PRIMARY KEY,
+    query_text VARCHAR (256),
+    created NUMBER,
+    backend_url VARCHAR (256),
+    user_name VARCHAR(256),
+    source VARCHAR(256)
+);
+CREATE INDEX query_history_created_idx ON query_history(created);
+
+CREATE TABLE resource_groups (
+    resource_group_id NUMBER GENERATED ALWAYS as IDENTITY(START with 1 INCREMENT by 1),
+    name VARCHAR(250) NOT NULL,
+    -- OPTIONAL POLICY CONTROLS
+    parent NUMBER,
+    jmx_export CHAR(1),
+    scheduling_policy VARCHAR(128),
+    scheduling_weight NUMBER,
+    -- REQUIRED QUOTAS
+    soft_memory_limit VARCHAR(128) NOT NULL,
+    max_queued INT NOT NULL,
+    hard_concurrency_limit NUMBER NOT NULL,
+    -- OPTIONAL QUOTAS
+    soft_concurrency_limit NUMBER,
+    soft_cpu_limit VARCHAR(128),
+    hard_cpu_limit VARCHAR(128),
+    environment VARCHAR(128),
+    PRIMARY KEY(resource_group_id),
+    FOREIGN KEY (parent) REFERENCES resource_groups (resource_group_id) ON DELETE CASCADE
+);
+
+CREATE TABLE selectors (
+    resource_group_id NUMBER NOT NULL,
+    priority NUMBER NOT NULL,
+    -- Regex fields -- these will be used as a regular expression pattern to
+    --                 match against the field of the same name on queries
+    user_regex VARCHAR(512),
+    source_regex VARCHAR(512),
+    -- Selector fields -- these must match exactly.
+    query_type VARCHAR(512),
+    client_tags VARCHAR(512),
+    selector_resource_estimate VARCHAR(1024),
+    FOREIGN KEY (resource_group_id) REFERENCES resource_groups (resource_group_id) ON DELETE CASCADE
+);
+
+CREATE TABLE resource_groups_global_properties (
+    name VARCHAR(128) NOT NULL PRIMARY KEY,
+    value VARCHAR(512) NULL,
+    CHECK (name in ('cpu_quota_period'))
+);
+
+CREATE TABLE exact_match_source_selectors(
+    environment VARCHAR(256),
+    update_time TIMESTAMP NOT NULL,
+    -- Selector fields which must exactly match a query
+    source VARCHAR(512) NOT NULL,
+    query_type VARCHAR(512),
+    resource_group_id VARCHAR(256) NOT NULL,
+    PRIMARY KEY (environment, source, resource_group_id),
+    UNIQUE (source, environment, query_type, resource_group_id)
+);
+
diff --git a/gateway-ha/src/main/resources/gateway-ha-persistence.sql b/gateway-ha/src/main/resources/gateway-ha-persistence.sql
@@ -18,7 +18,7 @@ CREATE INDEX query_history_created_idx ON query_history(created);
 
 CREATE TABLE IF NOT EXISTS resource_groups (
     resource_group_id BIGINT NOT NULL AUTO_INCREMENT,
-    name VARCHAR(250) NOT NULL UNIQUE,
+    name VARCHAR(250) NOT NULL,
 
     -- OPTIONAL POLICY CONTROLS
     parent BIGINT NULL,
@@ -75,4 +75,4 @@ CREATE TABLE IF NOT EXISTS exact_match_source_selectors (
 
     PRIMARY KEY (environment, source, query_type),
     UNIQUE (source, environment, query_type, resource_group_id)
-);
+);
diff --git a/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestResourceGroupsManager.java b/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestResourceGroupsManager.java
@@ -72,7 +72,7 @@ public void testReadResourceGroup()
         List<ResourceGroupsDetail> resourceGroups = resourceGroupManager.readAllResourceGroups(null);
         assertEquals(resourceGroups.size(), 1);
 
-        assertEquals(0L, resourceGroups.get(0).getResourceGroupId());
+        assertEquals(1L, resourceGroups.get(0).getResourceGroupId());
         assertEquals("admin", resourceGroups.get(0).getName());
         assertEquals(20, resourceGroups.get(0).getHardConcurrencyLimit());
         assertEquals(200, resourceGroups.get(0).getMaxQueued());
@@ -85,7 +85,7 @@ public void testReadResourceGroup()
     public void testUpdateResourceGroup()
     {
         ResourceGroupsDetail resourceGroup = new ResourceGroupsDetail();
-        resourceGroup.setResourceGroupId(0L);
+        resourceGroup.setResourceGroupId(1L);
         resourceGroup.setName("admin");
         resourceGroup.setHardConcurrencyLimit(50);
         resourceGroup.setMaxQueued(50);
@@ -99,7 +99,7 @@ public void testUpdateResourceGroup()
 
         /* Update resourceGroups that do not exist yet.
          *  In this case, new resourceGroups should be created. */
-        resourceGroup.setResourceGroupId(1L);
+        resourceGroup.setResourceGroupId(2L);
         resourceGroup.setName("localization-eng");
         resourceGroup.setHardConcurrencyLimit(50);
         resourceGroup.setMaxQueued(70);
@@ -121,14 +121,14 @@ public void testUpdateResourceGroup()
 
         assertEquals(3, resourceGroups.size()); // updated 2 non-existing groups, so count should be 3
 
-        assertEquals(0L, resourceGroups.get(0).getResourceGroupId());
+        assertEquals(1L, resourceGroups.get(0).getResourceGroupId());
         assertEquals("admin", resourceGroups.get(0).getName());
         assertEquals(50, resourceGroups.get(0).getHardConcurrencyLimit());
         assertEquals(50, resourceGroups.get(0).getMaxQueued());
         assertEquals(Boolean.FALSE, resourceGroups.get(0).getJmxExport());
         assertEquals("20%", resourceGroups.get(0).getSoftMemoryLimit());
 
-        assertEquals(1L, resourceGroups.get(1).getResourceGroupId());
+        assertEquals(2L, resourceGroups.get(1).getResourceGroupId());
         assertEquals("localization-eng", resourceGroups.get(1).getName());
         assertEquals(50, resourceGroups.get(1).getHardConcurrencyLimit());
         assertEquals(70, resourceGroups.get(1).getMaxQueued());
@@ -144,15 +144,15 @@ public void testDeleteResourceGroup()
         List<ResourceGroupsDetail> resourceGroups = resourceGroupManager.readAllResourceGroups(null);
         assertEquals(3, resourceGroups.size());
 
-        assertEquals(0L, resourceGroups.get(0).getResourceGroupId());
-        assertEquals(1L, resourceGroups.get(1).getResourceGroupId());
+        assertEquals(1L, resourceGroups.get(0).getResourceGroupId());
+        assertEquals(2L, resourceGroups.get(1).getResourceGroupId());
         assertEquals(3L, resourceGroups.get(2).getResourceGroupId());
 
         resourceGroupManager.deleteResourceGroup(resourceGroups.get(1).getResourceGroupId(), null);
         resourceGroups = resourceGroupManager.readAllResourceGroups(null);
 
         assertEquals(2, resourceGroups.size());
-        assertEquals(0L, resourceGroups.get(0).getResourceGroupId());
+        assertEquals(1L, resourceGroups.get(0).getResourceGroupId());
         assertEquals(3L, resourceGroups.get(1).getResourceGroupId());
     }
 
@@ -161,7 +161,7 @@ public void testDeleteResourceGroup()
     public void testCreateSelector()
     {
         SelectorsDetail selector = new SelectorsDetail();
-        selector.setResourceGroupId(0L);
+        selector.setResourceGroupId(1L);
         selector.setPriority(0L);
         selector.setUserRegex("data-platform-admin");
         selector.setSourceRegex("admin");
@@ -181,7 +181,7 @@ public void testReadSelector()
         List<SelectorsDetail> selectors = resourceGroupManager.readAllSelectors(null);
 
         assertEquals(1, selectors.size());
-        assertEquals(0L, selectors.get(0).getResourceGroupId());
+        assertEquals(1L, selectors.get(0).getResourceGroupId());
         assertEquals(0L, selectors.get(0).getPriority());
         assertEquals("data-platform-admin", selectors.get(0).getUserRegex());
         assertEquals("admin", selectors.get(0).getSourceRegex());
@@ -196,7 +196,7 @@ public void testUpdateSelector()
     {
         SelectorsDetail selector = new SelectorsDetail();
 
-        selector.setResourceGroupId(0L);
+        selector.setResourceGroupId(1L);
         selector.setPriority(0L);
         selector.setUserRegex("data-platform-admin_updated");
         selector.setSourceRegex("admin_updated");
@@ -250,7 +250,7 @@ public void testDeleteSelector()
     {
         List<SelectorsDetail> selectors = resourceGroupManager.readAllSelectors(null);
         assertEquals(3, selectors.size());
-        assertEquals(0L, selectors.get(0).getResourceGroupId());
+        assertEquals(1L, selectors.get(0).getResourceGroupId());
         resourceGroupManager.deleteSelector(selectors.get(0), null);
         selectors = resourceGroupManager.readAllSelectors(null);
 

diff --git a/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestSpecificDbResourceGroupsManager.java b/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestSpecificDbResourceGroupsManager.java
@@ -56,7 +56,6 @@ private void createResourceGroup()
     {
         ResourceGroupsDetail resourceGroup = new ResourceGroupsDetail();
 
-        resourceGroup.setResourceGroupId(1L);
         resourceGroup.setName("admin2");
         resourceGroup.setHardConcurrencyLimit(20);
         resourceGroup.setMaxQueued(200);

diff --git a/pom.xml b/pom.xml
@@ -62,6 +62,7 @@
         <dep.mockito.version>5.8.0</dep.mockito.version>
         <dep.mysqlconnector.version>8.0.17</dep.mysqlconnector.version>
         <dep.okhttp.version>3.9.0</dep.okhttp.version>
+        <dep.oracle.version>21.9.0.0</dep.oracle.version>
         <dep.plugin.maven.shade>2.3</dep.plugin.maven.shade>
         <dep.plugin.maven.source>3.0.0</dep.plugin.maven.source>
         <dep.postgresql.version>42.6.0</dep.postgresql.version>