ENG-50193: Document store using credentials (#100)

hypertrace · Oct 25, 2024 · f4d0dd8 · f4d0dd8
1 parent 492e076
commit f4d0dd8
Show file tree

Hide file tree

Showing 10 changed files with 126 additions and 85 deletions.
diff --git a/.gitignore b/.gitignore
@@ -21,5 +21,6 @@ test-output
 .idea/dictionaries/
 .idea/codeStyles/
 .idea/.name
+.idea/
 # Local config to handle using Java 8 vs java 11.
-.java-version
+.java-version
diff --git a/config-bootstrapper/build.gradle.kts b/config-bootstrapper/build.gradle.kts
@@ -128,7 +128,7 @@ tasks.test {
 dependencies {
   implementation("org.hypertrace.entity.service:entity-service-client:0.6.4")
   implementation("org.hypertrace.entity.service:entity-service-api:0.6.4")
-  implementation("org.hypertrace.core.documentstore:document-store:0.7.23")
+  implementation("org.hypertrace.core.documentstore:document-store:0.8.4")
   implementation("org.hypertrace.core.attribute.service:attribute-service-client:0.14.5")
   implementation("org.hypertrace.core.grpcutils:grpc-context-utils:0.11.2")
   implementation("org.hypertrace.core.grpcutils:grpc-client-utils:0.11.2")

diff --git a/...egrationTest/java/org/hypertrace/core/bootstrapper/ConfigBootstrapperIntegrationTest.java b/...egrationTest/java/org/hypertrace/core/bootstrapper/ConfigBootstrapperIntegrationTest.java
@@ -19,6 +19,8 @@
 import org.hypertrace.core.documentstore.Datastore;
 import org.hypertrace.core.documentstore.DatastoreProvider;
 import org.hypertrace.core.documentstore.DocumentStoreConfig;
+import org.hypertrace.core.documentstore.model.config.DatastoreConfig;
+import org.hypertrace.core.documentstore.model.config.TypesafeConfigDatastoreConfigExtractor;
 import org.hypertrace.core.grpcutils.context.RequestContext;
 import org.hypertrace.core.grpcutils.context.RequestContextConstants;
 import org.hypertrace.entity.service.client.config.EntityServiceClientConfig;
@@ -30,6 +32,7 @@
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
+import static org.hypertrace.core.bootstrapper.ConfigBootstrapper.DATASTORE_TYPE_CONFIG_KEY;
 import static org.hypertrace.core.bootstrapper.dao.ConfigBootstrapStatusDao.CONFIG_BOOTSTRAPPER_COLLECTION;
 
 public class ConfigBootstrapperIntegrationTest {
@@ -62,9 +65,11 @@ public static void setUp() {
             new TenantIdClientInterceptor(TENANT_ID));
     attributesServiceClient = new AttributeServiceClient(channel);
 
-    String dataStoreType = config.getString(DocumentStoreConfig.DATASTORE_TYPE_CONFIG_KEY);
+    Config documentStoreConfig = config.getConfig("document.store");
+    DatastoreConfig datastoreConfig = TypesafeConfigDatastoreConfigExtractor.from(documentStoreConfig, DATASTORE_TYPE_CONFIG_KEY)
+            .extract();
     Datastore datastore =
-        DatastoreProvider.getDatastore(dataStoreType, config.getConfig(dataStoreType));
+            DatastoreProvider.getDatastore(datastoreConfig);
     datastore.getCollection(CONFIG_BOOTSTRAPPER_COLLECTION).drop();
   }
 

diff --git a/config-bootstrapper/src/integrationTest/resources/config-bootstrapper-test/application.conf b/config-bootstrapper/src/integrationTest/resources/config-bootstrapper-test/application.conf
@@ -6,8 +6,19 @@ entity.service.config = {
     host=localhost
     port=50061
 }
-dataStoreType = mongo
-mongo = {
-  host=localhost
-  port=37017
+
+document.store {
+  dataStoreType = "mongo"
+  appName = "config-bootstrapper-integration-test"
+  maxPoolSize = 10
+  mongo = {
+    endpoints = [
+      {
+        host: "localhost"
+        port: 37017
+      }
+    ]
+    database = default_db
+    authDb = admin
+  }
 }
diff --git a/config-bootstrapper/src/main/java/org/hypertrace/core/bootstrapper/ConfigBootstrapper.java b/config-bootstrapper/src/main/java/org/hypertrace/core/bootstrapper/ConfigBootstrapper.java
@@ -10,12 +10,14 @@
 import org.hypertrace.core.bootstrapper.dao.ConfigBootstrapStatusDao;
 import org.hypertrace.core.documentstore.Datastore;
 import org.hypertrace.core.documentstore.DatastoreProvider;
-import org.hypertrace.core.documentstore.DocumentStoreConfig;
+import org.hypertrace.core.documentstore.model.config.DatastoreConfig;
+import org.hypertrace.core.documentstore.model.config.TypesafeConfigDatastoreConfigExtractor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class ConfigBootstrapper {
   private static final Logger LOGGER = LoggerFactory.getLogger(ConfigBootstrapper.class);
+  public static final String DATASTORE_TYPE_CONFIG_KEY = "dataStoreType";
 
   public static void main(String[] args) {
     updateRuntime();
@@ -29,9 +31,14 @@ public static void main(String[] args) {
    * */
   public static BootstrapRunner bootstrapper(BootstrapArgs bootstrapArgs) {
     Config config = ConfigFactory.parseFile(new File(bootstrapArgs.getConfigFile())).resolve();
-    String dataStoreType = config.getString(DocumentStoreConfig.DATASTORE_TYPE_CONFIG_KEY);
+    Config documentStoreConfig = config.getConfig("document.store");
+    DatastoreConfig datastoreConfig = TypesafeConfigDatastoreConfigExtractor.from(documentStoreConfig, DATASTORE_TYPE_CONFIG_KEY)
+            .extract();
     Datastore datastore =
-        DatastoreProvider.getDatastore(dataStoreType, config.getConfig(dataStoreType));
+        DatastoreProvider.getDatastore(datastoreConfig);
+
+    Runtime.getRuntime().addShutdownHook(new Thread(datastore::close));
+
     return new BootstrapRunner(new ConfigBootstrapStatusDao(datastore));
   }
 

diff --git a/...strapper/src/main/java/org/hypertrace/core/bootstrapper/dao/ConfigBootstrapStatusDao.java b/...strapper/src/main/java/org/hypertrace/core/bootstrapper/dao/ConfigBootstrapStatusDao.java
@@ -9,6 +9,7 @@
 import java.util.stream.StreamSupport;
 import org.hypertrace.core.bootstrapper.model.ConfigBootstrapStatus;
 import org.hypertrace.core.bootstrapper.model.ConfigBootstrapStatusKey;
+import org.hypertrace.core.documentstore.CloseableIterator;
 import org.hypertrace.core.documentstore.Datastore;
 import org.hypertrace.core.documentstore.Document;
 import org.hypertrace.core.documentstore.Filter;
@@ -33,15 +34,20 @@ public ConfigBootstrapStatusDao(Datastore datastore) {
   public ConfigBootstrapStatus getConfigBootstrapStatus(ConfigBootstrapStatusKey key) {
     Query query = new Query();
     query.setFilter(new Filter(Filter.Op.EQ, ID, key.toString()));
-    Iterator<Document> docs = configBootstrapStatusCollection.search(query);
-    if (docs.hasNext()) {
-      Document doc = docs.next();
-      try {
-        return ConfigBootstrapStatus.fromJson(doc.toJson());
-      } catch (IOException ex) {
-        LOGGER.error(String.format("Error creating ConfigBootstrapStatus from doc:%s", doc), ex);
-        return null;
+    try (final CloseableIterator<Document> docsIterator =
+                 configBootstrapStatusCollection.search(query)){
+      if (docsIterator.hasNext()) {
+        Document doc = docsIterator.next();
+        try {
+          return ConfigBootstrapStatus.fromJson(doc.toJson());
+        } catch (IOException ex) {
+          LOGGER.error(String.format("Error creating ConfigBootstrapStatus from doc:%s", doc), ex);
+          return null;
+        }
       }
+    } catch (IOException e) {
+      LOGGER.error("Error in query: {}", query);
+      return null;
     }
     return null;
   }
@@ -65,22 +71,26 @@ public boolean deleteConfigBootstrapStatus(ConfigBootstrapStatusKey key) {
   public List<ConfigBootstrapStatus> getConfigBootstrapStatusGreaterThan(int version) {
     Query query = new Query();
     query.setFilter(new Filter(Filter.Op.GT, VERSION, version));
-    Iterator<Document> docs = configBootstrapStatusCollection.search(query);
-    return StreamSupport.stream(Spliterators.spliteratorUnknownSize(docs, 0), false)
-        .map(
-            document -> {
-              try {
-                return ConfigBootstrapStatus.fromJson(document.toJson());
-              } catch (IOException ex) {
-                LOGGER.error("Error creating ConfigBootstrapStatus from doc:{}", document);
-                return null;
-              }
-            })
-        .filter(Objects::nonNull)
-        // sort from higher version to lower version
-        .sorted(
-            (c1, c2) ->
-                ConfigBootstrapStatusKey.from(c2).compareTo(ConfigBootstrapStatusKey.from(c1)))
-        .collect(Collectors.toList());
+    try(final CloseableIterator<Document> docs = configBootstrapStatusCollection.search(query)){
+      return StreamSupport.stream(Spliterators.spliteratorUnknownSize(docs, 0), false)
+              .map(
+                      document -> {
+                        try {
+                          return ConfigBootstrapStatus.fromJson(document.toJson());
+                        } catch (IOException ex) {
+                          LOGGER.error("Error creating ConfigBootstrapStatus from doc:{}", document);
+                          return null;
+                        }
+                      })
+              .filter(Objects::nonNull)
+              // sort from higher version to lower version
+              .sorted(
+                      (c1, c2) ->
+                              ConfigBootstrapStatusKey.from(c2).compareTo(ConfigBootstrapStatusKey.from(c1)))
+              .collect(Collectors.toList());
+    } catch (IOException e) {
+      LOGGER.error("Error in query: {}", query);
+      return null;
+    }
   }
 }
diff --git a/config-bootstrapper/src/main/resources/configs/config-bootstrapper/application.conf b/config-bootstrapper/src/main/resources/configs/config-bootstrapper/application.conf
@@ -10,9 +10,22 @@ entity.service.config = {
   port=50061
   port=${?ENTITY_SERVICE_PORT_CONFIG}
 }
-dataStoreType = mongo
-mongo = {
-  host=localhost
-  host=${?MONGO_HOST}
-  port=27017
+
+document.store {
+  dataStoreType = mongo
+  appName = config-bootstrapper-local
+  maxPoolSize = 10
+  mongo {
+    database = default_db
+    user = ${?MONGO_SERVICE_USERNAME}
+    password = ${?MONGO_SERVICE_USER_PASSWORD}
+    endpoints = [
+      {
+        host = localhost
+        host = ${?MONGO_HOST} # provides a way to override the mongo_host via an environment variable
+        port = 27017
+        port = ${?MONGO_PORT}
+      }
+    ]
+  }
 }
diff --git a/helm/templates/config-bootstrapper-config.yaml b/helm/templates/config-bootstrapper-config.yaml
@@ -14,32 +14,28 @@ data:
         host={{ .Values.entityServiceConfig.data.host }}
         port={{ .Values.entityServiceConfig.data.port }}
     }
-    dataStoreType = {{ .Values.dataStoreType }}
-    {{- if eq .Values.dataStoreType "mongo" }}
-    mongo = {
-          {{- if .Values.mongoConfig.data.url }}
-          url = {{ .Values.mongoConfig.data.url | quote }}
+    
+    document.store {
+      {{- if .Values.database }}
+      {{- $dst := .Values.database.type }}
+      dataStoreType = {{ $dst }}
+      appName = {{ .Values.name }}
+      {{- range $key, $value := (index .Values "database") }}
+      {{- if ne $key $dst }}
+      {{ $key }} = {{- toJson $value }}
+      {{- end }}
+      {{- end }}
+  
+      {{ $dst }} {
+        {{- range $key, $value := (index .Values "database" (printf "%s" $dst)) }}
+        {{- if $value }}
+          {{- if hasPrefix "${?" (printf "%s" $value) }}
+        {{ $key }} = {{ $value }}
           {{- else }}
-          host={{ .Values.mongoConfig.data.host }}
-          port={{ .Values.mongoConfig.data.port }}
+        {{ $key }} = {{- toJson $value }}
           {{- end }}
-        }
-    {{ else if eq .Values.dataStoreType "postgres" }}
-    postgres = {
-          {{- if .Values.postgresConfig.data.url }}
-          url={{ .Values.postgresConfig.data.url | quote }}
-          {{- else }}
-          host={{ .Values.postgresConfig.data.host }}
-          port={{ .Values.postgresConfig.data.port }}
-          {{- end }}
-          {{- if .Values.postgresConfig.data.database }}
-          database= {{ .Values.postgresConfig.data.database }}
-          {{- end }}
-          {{- if .Values.postgresConfig.data.user }}
-          user={{ .Values.postgresConfig.data.user }}
-          {{- end }}
-          {{- if .Values.postgresConfig.data.password }}
-          password={{ .Values.postgresConfig.data.password }}
-          {{- end }}
-        }
-    {{- end }}
+        {{- end }}
+        {{- end }}
+      }
+      {{- end }}
+    }
diff --git a/helm/templates/job.yaml b/helm/templates/job.yaml
@@ -103,6 +103,20 @@ spec:
               value: "/var/{{ .Chart.Name }}/log/log4j2.properties"
             - name: JAVA_TOOL_OPTIONS
               value: {{ .Values.javaOpts | quote }}
+            {{- if and .Values.database .Values.database.mongo.authEnabled }}
+            {{- with .Values.database.mongo.credentials }}
+            - name: MONGO_SERVICE_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .secretName }}
+                  key: {{ .secretUsernameKey }}
+            - name: MONGO_SERVICE_USER_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .secretName }}
+                  key: {{ .secretPasswordKey }}
+            {{- end }}
+            {{- end }}
             {{- if and (eq .Values.dataStoreType "postgres") .Values.postgresConfig.data.passwordSecretName .Values.postgresConfig.data.passwordSecretKey }}
             - name: POSTGRESQL_PASSWORD
               valueFrom:

diff --git a/helm/values.yaml b/helm/values.yaml
@@ -13,6 +13,7 @@
 ###########
 # Job
 ###########
+name: config-bootstrapper
 
 image:
   repository: hypertrace/config-bootstrapper
@@ -86,23 +87,6 @@ attributeServiceConfig:
     host: attribute-service
     port: 9012
 
-dataStoreType: "mongo"
-mongoConfig:
-  name: mongo-config
-  data:
-    host: mongo
-    port: 27017
-    url: ""
-
-postgresConfig:
-  name: postgres-config
-  data:
-    host: postgres
-    port: 5432
-    url: ""
-    passwordSecretName: ""
-    passwordSecretKey: ""
-
 logConfig:
   name: config-bootstrapper-log-appender-config
   rootLogger: