Add support for Oracle as backend database

trinodb · Nov 15, 2023 · 6469a13 · 6469a13
1 parent e155380
commit 6469a13
Show file tree

Hide file tree

Showing 10 changed files with 100 additions and 30 deletions.
diff --git a/docs/resource-groups-api.md b/docs/resource-groups-api.md
@@ -95,12 +95,12 @@ curl -X POST http://localhost:8080/trino/resourcegroup/delete/{INSERT_ID_HERE}
 
 ## Add a selector
 
-To add a single selector, specify all relevant fields in the body. Resource
-group id should not be specified since the database should autoincrement it.
+To add a single selector, specify all relevant fields in the body.
 
 ```$xslt
 curl -X POST http://localhost:8080/trino/selector/create \
  -d '{
+        "resourceGroupId": 1, \
         "priority": 1, \
         "userRegex": "selector1", \
         "sourceRegex": "resourcegroup1", \

diff --git a/gateway-ha/pom.xml b/gateway-ha/pom.xml
@@ -146,6 +146,11 @@
             <artifactId>postgresql</artifactId>
             <version>${dep.postgresql.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.oracle.database.jdbc</groupId>
+            <artifactId>ojdbc11</artifactId>
+            <version>${dep.oracle.version}</version>
+        </dependency>
         <!-- Test deps -->
         <dependency>
             <groupId>org.junit.jupiter</groupId>

diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/QueryHistory.java b/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/QueryHistory.java
@@ -12,12 +12,12 @@
 @Table("query_history")
 @Cached
 public class QueryHistory extends Model {
-  private static final String queryId = "query_id";
-  private static final String queryText = "query_text";
-  private static final String backendUrl = "backend_url";
-  private static final String userName = "user_name";
-  private static final String source = "source";
-  private static final String created = "created";
+  public static final String queryId = "query_id";
+  public static final String queryText = "query_text";
+  public static final String backendUrl = "backend_url";
+  public static final String userName = "user_name";
+  public static final String source = "source";
+  public static final String created = "created";
 
   public static List<QueryHistoryManager.QueryDetail> upcast(List<QueryHistory> queryHistoryList) {
     List<QueryHistoryManager.QueryDetail> queryDetails = new ArrayList<>();

diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/ResourceGroups.java b/gateway-ha/src/main/java/io/trino/gateway/ha/persistence/dao/ResourceGroups.java
@@ -77,7 +77,6 @@ public static List<ResourceGroupsDetail> upcast(List<ResourceGroups> resourceGro
    * @param resourceGroupDetail
    */
   public static void create(ResourceGroups model, ResourceGroupsDetail resourceGroupDetail) {
-    model.set(resourceGroupId, resourceGroupDetail.getResourceGroupId());
     model.set(name, resourceGroupDetail.getName());
 
     model.set(parent, resourceGroupDetail.getParent());

diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/router/HaQueryHistoryManager.java b/gateway-ha/src/main/java/io/trino/gateway/ha/router/HaQueryHistoryManager.java
@@ -29,14 +29,10 @@ public void submitQueryDetail(QueryDetail queryDetail) {
   public List<QueryDetail> fetchQueryHistory(Optional<String> user) {
     try {
       connectionManager.open();
-      String sql = "select * from query_history";
       if (user.isPresent()) {
-        sql += " where user_name = '" + user.get() + "'";
+        return QueryHistory.upcast(QueryHistory.where(QueryHistory.userName + " = '" + user.get() + "'").limit(2000).orderBy(QueryHistory.created + " desc"));
       }
-      return QueryHistory.upcast(QueryHistory.findBySQL(String.join(" ",
-          sql,
-          "order by created desc",
-          "limit 2000")));
+      return QueryHistory.upcast(QueryHistory.findAll().limit(2000).orderBy("created desc"));
     } finally {
       connectionManager.close();
     }

diff --git a/gateway-ha/src/main/resources/gateway-ha-persistence-oracle.sql b/gateway-ha/src/main/resources/gateway-ha-persistence-oracle.sql
@@ -0,0 +1,70 @@
+CREATE TABLE gateway_backend (
+    name VARCHAR(256) PRIMARY KEY,
+    routing_group VARCHAR (256),
+    backend_url VARCHAR (256),
+    external_url VARCHAR (256),
+    active NUMBER(1)
+);
+
+CREATE TABLE query_history (
+    query_id VARCHAR(256) PRIMARY KEY,
+    query_text VARCHAR (256),
+    created NUMBER,
+    backend_url VARCHAR (256),
+    user_name VARCHAR(256),
+    source VARCHAR(256)
+);
+CREATE INDEX query_history_created_idx ON query_history(created);
+
+CREATE TABLE resource_groups (
+    resource_group_id NUMBER GENERATED ALWAYS as IDENTITY(START with 1 INCREMENT by 1),
+    name VARCHAR(250) NOT NULL,
+    -- OPTIONAL POLICY CONTROLS
+    parent NUMBER,
+    jmx_export CHAR(1),
+    scheduling_policy VARCHAR(128),
+    scheduling_weight NUMBER,
+    -- REQUIRED QUOTAS
+    soft_memory_limit VARCHAR(128) NOT NULL,
+    max_queued INT NOT NULL,
+    hard_concurrency_limit NUMBER NOT NULL,
+    -- OPTIONAL QUOTAS
+    soft_concurrency_limit NUMBER,
+    soft_cpu_limit VARCHAR(128),
+    hard_cpu_limit VARCHAR(128),
+    environment VARCHAR(128),
+    PRIMARY KEY(resource_group_id),
+    FOREIGN KEY (parent) REFERENCES resource_groups (resource_group_id) ON DELETE CASCADE
+);
+
+CREATE TABLE selectors (
+    resource_group_id NUMBER NOT NULL,
+    priority NUMBER NOT NULL,
+    -- Regex fields -- these will be used as a regular expression pattern to
+    --                 match against the field of the same name on queries
+    user_regex VARCHAR(512),
+    source_regex VARCHAR(512),
+    -- Selector fields -- these must match exactly.
+    query_type VARCHAR(512),
+    client_tags VARCHAR(512),
+    selector_resource_estimate VARCHAR(1024),
+    FOREIGN KEY (resource_group_id) REFERENCES resource_groups (resource_group_id) ON DELETE CASCADE
+);
+
+CREATE TABLE resource_groups_global_properties (
+    name VARCHAR(128) NOT NULL PRIMARY KEY,
+    value VARCHAR(512) NULL,
+    CHECK (name in ('cpu_quota_period'))
+);
+
+CREATE TABLE exact_match_source_selectors(
+    environment VARCHAR(256),
+    update_time TIMESTAMP NOT NULL,
+    -- Selector fields which must exactly match a query
+    source VARCHAR(512) NOT NULL,
+    query_type VARCHAR(512),
+    resource_group_id VARCHAR(256) NOT NULL,
+    PRIMARY KEY (environment, source, resource_group_id),
+    UNIQUE (source, environment, query_type, resource_group_id)
+);
+
diff --git a/gateway-ha/src/main/resources/gateway-ha-persistence.sql b/gateway-ha/src/main/resources/gateway-ha-persistence.sql
@@ -18,7 +18,7 @@ CREATE INDEX query_history_created_idx ON query_history(created);
 
 CREATE TABLE IF NOT EXISTS resource_groups (
     resource_group_id BIGINT NOT NULL AUTO_INCREMENT,
-    name VARCHAR(250) NOT NULL UNIQUE,
+    name VARCHAR(250) NOT NULL,
 
     -- OPTIONAL POLICY CONTROLS
     parent BIGINT NULL,
@@ -75,4 +75,4 @@ CREATE TABLE IF NOT EXISTS exact_match_source_selectors (
 
     PRIMARY KEY (environment, source, query_type),
     UNIQUE (source, environment, query_type, resource_group_id)
-);
+);
diff --git a/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestResourceGroupsManager.java b/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestResourceGroupsManager.java
@@ -65,7 +65,7 @@ public void testReadResourceGroup() {
     List<ResourceGroupsDetail> resourceGroups = resourceGroupManager.readAllResourceGroups(null);
     assertEquals(resourceGroups.size(), 1);
 
-    assertEquals(0L, resourceGroups.get(0).getResourceGroupId());
+    assertEquals(1L, resourceGroups.get(0).getResourceGroupId());
     assertEquals("admin", resourceGroups.get(0).getName());
     assertEquals(20, resourceGroups.get(0).getHardConcurrencyLimit());
     assertEquals(200, resourceGroups.get(0).getMaxQueued());
@@ -77,7 +77,7 @@ public void testReadResourceGroup() {
   @Order(3)
   public void testUpdateResourceGroup() {
     ResourceGroupsDetail resourceGroup = new ResourceGroupsDetail();
-    resourceGroup.setResourceGroupId(0L);
+    resourceGroup.setResourceGroupId(1L);
     resourceGroup.setName("admin");
     resourceGroup.setHardConcurrencyLimit(50);
     resourceGroup.setMaxQueued(50);
@@ -91,7 +91,7 @@ public void testUpdateResourceGroup() {
 
     /* Update resourceGroups that do not exist yet.
      *  In this case, new resourceGroups should be created. */
-    resourceGroup.setResourceGroupId(1L);
+    resourceGroup.setResourceGroupId(2L);
     resourceGroup.setName("localization-eng");
     resourceGroup.setHardConcurrencyLimit(50);
     resourceGroup.setMaxQueued(70);
@@ -113,14 +113,14 @@ public void testUpdateResourceGroup() {
 
     assertEquals(3, resourceGroups.size()); // updated 2 non-existing groups, so count should be 3
 
-    assertEquals(0L, resourceGroups.get(0).getResourceGroupId());
+    assertEquals(1L, resourceGroups.get(0).getResourceGroupId());
     assertEquals("admin", resourceGroups.get(0).getName());
     assertEquals(50, resourceGroups.get(0).getHardConcurrencyLimit());
     assertEquals(50, resourceGroups.get(0).getMaxQueued());
     assertEquals(Boolean.FALSE, resourceGroups.get(0).getJmxExport());
     assertEquals("20%", resourceGroups.get(0).getSoftMemoryLimit());
 
-    assertEquals(1L, resourceGroups.get(1).getResourceGroupId());
+    assertEquals(2L, resourceGroups.get(1).getResourceGroupId());
     assertEquals("localization-eng", resourceGroups.get(1).getName());
     assertEquals(50, resourceGroups.get(1).getHardConcurrencyLimit());
     assertEquals(70, resourceGroups.get(1).getMaxQueued());
@@ -135,23 +135,23 @@ public void testDeleteResourceGroup() {
     List<ResourceGroupsDetail> resourceGroups = resourceGroupManager.readAllResourceGroups(null);
     assertEquals(3, resourceGroups.size());
 
-    assertEquals(0L, resourceGroups.get(0).getResourceGroupId());
-    assertEquals(1L, resourceGroups.get(1).getResourceGroupId());
+    assertEquals(1L, resourceGroups.get(0).getResourceGroupId());
+    assertEquals(2L, resourceGroups.get(1).getResourceGroupId());
     assertEquals(3L, resourceGroups.get(2).getResourceGroupId());
 
     resourceGroupManager.deleteResourceGroup(resourceGroups.get(1).getResourceGroupId(), null);
     resourceGroups = resourceGroupManager.readAllResourceGroups(null);
 
     assertEquals(2, resourceGroups.size());
-    assertEquals(0L, resourceGroups.get(0).getResourceGroupId());
+    assertEquals(1L, resourceGroups.get(0).getResourceGroupId());
     assertEquals(3L, resourceGroups.get(1).getResourceGroupId());
   }
 
   @Test
   @Order(5)
   public void testCreateSelector() {
     SelectorsDetail selector = new SelectorsDetail();
-    selector.setResourceGroupId(0L);
+    selector.setResourceGroupId(1L);
     selector.setPriority(0L);
     selector.setUserRegex("data-platform-admin");
     selector.setSourceRegex("admin");
@@ -170,7 +170,7 @@ public void testReadSelector() {
     List<SelectorsDetail> selectors = resourceGroupManager.readAllSelectors(null);
 
     assertEquals(1, selectors.size());
-    assertEquals(0L, selectors.get(0).getResourceGroupId());
+    assertEquals(1L, selectors.get(0).getResourceGroupId());
     assertEquals(0L, selectors.get(0).getPriority());
     assertEquals("data-platform-admin", selectors.get(0).getUserRegex());
     assertEquals("admin", selectors.get(0).getSourceRegex());
@@ -184,7 +184,7 @@ public void testReadSelector() {
   public void testUpdateSelector() {
     SelectorsDetail selector = new SelectorsDetail();
 
-    selector.setResourceGroupId(0L);
+    selector.setResourceGroupId(1L);
     selector.setPriority(0L);
     selector.setUserRegex("data-platform-admin_updated");
     selector.setSourceRegex("admin_updated");
@@ -237,7 +237,7 @@ public void testUpdateSelector() {
   public void testDeleteSelector() {
     List<SelectorsDetail> selectors = resourceGroupManager.readAllSelectors(null);
     assertEquals(3, selectors.size());
-    assertEquals(0L, selectors.get(0).getResourceGroupId());
+    assertEquals(1L, selectors.get(0).getResourceGroupId());
     resourceGroupManager.deleteSelector(selectors.get(0), null);
     selectors = resourceGroupManager.readAllSelectors(null);
 

diff --git a/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestSpecificDbResourceGroupsManager.java b/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestSpecificDbResourceGroupsManager.java
@@ -41,7 +41,6 @@ public void setUp() {
   private void createResourceGroup() {
     ResourceGroupsDetail resourceGroup = new ResourceGroupsDetail();
 
-    resourceGroup.setResourceGroupId(1L);
     resourceGroup.setName("admin2");
     resourceGroup.setHardConcurrencyLimit(20);
     resourceGroup.setMaxQueued(200);

diff --git a/pom.xml b/pom.xml
@@ -59,6 +59,7 @@
         <dep.jetty.version>11.0.15</dep.jetty.version>
         <dep.lombok.version>1.18.22</dep.lombok.version>
         <dep.okhttp.version>3.9.0</dep.okhttp.version>
+        <dep.oracle.version>21.9.0.0</dep.oracle.version>
         <dep.plugin.maven.shade>2.3</dep.plugin.maven.shade>
         <dep.plugin.maven.source>3.0.0</dep.plugin.maven.source>
         <dep.mockwebserver.version>1.2.1</dep.mockwebserver.version>