Merge pull request #162 from neicnordic/broker_prefetch

Add option to set Broker prefetch count
neicnordic · May 30, 2023 · 5a06089 · 5a06089
2 parents 903279b + 09c69a7
commit 5a06089
Show file tree

Hide file tree

Showing 12 changed files with 28 additions and 17 deletions.
diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v2
 name: sda-db
-version: "0.5.1"
+version: "0.5.2"
 description: Database component for Sensitive Data Archive (SDA) installation
 home: https://neic-sda.readthedocs.io
 icon: https://neic.no/assets/images/logo.png

diff --git a/charts/sda-db/values.yaml b/charts/sda-db/values.yaml
@@ -26,7 +26,7 @@ extraSecurityContext: {}
 
 image:
   repository: ghcr.io/neicnordic/sda-db
-  tag: v2.0.7
+  tag: v2.1.10
   pullPolicy: IfNotPresent
 
 # utilize network isolation

diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v2
 name: sda-mq
-version: "0.4.5"
+version: "0.4.6"
 description: RabbitMQ component for Sensitive Data Archive (SDA) installation
 home: https://neic-sda.readthedocs.io
 icon: https://neic.no/assets/images/logo.png

diff --git a/charts/sda-mq/values.yaml b/charts/sda-mq/values.yaml
@@ -35,7 +35,7 @@ extraSecurityContext: {}
 
 image:
   repository: ghcr.io/neicnordic/sda-mq
-  tag: v1.4.20
+  tag: v1.4.38
   pullPolicy: Always
 
 # utilize network isolation

diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v2
 name: sda-svc
-version: "0.18.7"
+version: "0.18.8"
 kubeVersion: ">= 1.19.0-0"
 description: Components for Sensitive Data Archive (SDA) installation
 home: https://neic-sda.readthedocs.io

diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md
@@ -87,6 +87,7 @@ Parameter | Description | Default
 `global.broker.password` | Shared password to the message broker. |`/`
 `global.broker.username` | Shared user to the message broker. |`/`
 `global.broker.backupRoutingKey` | routing key used to send messages to backup service |`""`
+`global.broker.prefetchCount` | Number of messages to retrieve from the broker at the time, setting this to `1` will create a round-robin behavior between consumers |`2`
 `global.cega.host` | Full URI to the EGA user authentication service. |`""`
 `global.cega.user` | Username for the EGA user authentication service. |`""`
 `global.cega.password` | Password for the EGA user authentication service. |`""`
@@ -120,7 +121,6 @@ Parameter | Description | Default
 `global.download.trusted.iss` | Array of trusted OIDC endpoints | ``
 `global.download.trusted.iss[iss]` | URI to the OIDC service | `https://login.elixir-czech.org/oidc/`
 `global.download.trusted.iss[jku]` | The URI to the OIDCs jwk endpoint | `https://login.elixir-czech.org/oidc/jwk`
-
 `global.elixir.oidcdHost` | URL to the OIDc service. | `"https://login.elixir-czech.org/oidc/"`
 `global.elixir.jwkPath` | Public key path on the OIDC host. | `jwk`
 `global.inbox.servicePort` | The port that the inbox is accessible via. | `2222`

diff --git a/charts/sda-svc/templates/backup-deploy.yaml b/charts/sda-svc/templates/backup-deploy.yaml
@@ -133,6 +133,8 @@ spec:
           value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }}
         - name: BROKER_PORT
           value: {{ .Values.global.broker.port | quote }}
+        - name: BROKER_PREFETCHCOUNT
+          value: {{ .Values.global.broker.prefetchCount | quote }}
         - name: BROKER_ROUTINGERROR
           value: {{ .Values.global.broker.routingError }}
         - name: BROKER_ROUTINGKEY

diff --git a/charts/sda-svc/templates/finalize-deploy.yaml b/charts/sda-svc/templates/finalize-deploy.yaml
@@ -76,6 +76,8 @@ spec:
           value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }}
         - name: BROKER_PORT
           value: {{ .Values.global.broker.port | quote }}
+        - name: BROKER_PREFETCHCOUNT
+          value: {{ .Values.global.broker.prefetchCount | quote }}
         - name: BROKER_ROUTINGERROR
           value: {{ .Values.global.broker.routingError }}
         - name: BROKER_ROUTINGKEY

diff --git a/charts/sda-svc/templates/ingest-deploy.yaml b/charts/sda-svc/templates/ingest-deploy.yaml
@@ -115,6 +115,8 @@ spec:
           value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }}
         - name: BROKER_PORT
           value: {{ .Values.global.broker.port | quote }}
+        - name: BROKER_PREFETCHCOUNT
+          value: {{ .Values.global.broker.prefetchCount | quote }}
         - name: BROKER_ROUTINGERROR
           value: {{ .Values.global.broker.routingError }}
         - name: BROKER_ROUTINGKEY

diff --git a/charts/sda-svc/templates/mapper-deploy.yaml b/charts/sda-svc/templates/mapper-deploy.yaml
@@ -76,6 +76,8 @@ spec:
           value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }}
         - name: BROKER_PORT
           value: {{ .Values.global.broker.port | quote }}
+        - name: BROKER_PREFETCHCOUNT
+          value: {{ .Values.global.broker.prefetchCount | quote }}
         - name: BROKER_ROUTINGERROR
           value: {{ .Values.global.broker.routingError }}
         - name: BROKER_VHOST

diff --git a/charts/sda-svc/templates/verify-deploy.yaml b/charts/sda-svc/templates/verify-deploy.yaml
@@ -135,6 +135,8 @@ spec:
           value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }}
         - name: BROKER_PORT
           value: {{ .Values.global.broker.port | quote }}
+        - name: BROKER_PREFETCHCOUNT
+          value: {{ .Values.global.broker.prefetchCount | quote }}
         - name: BROKER_QUEUE
           value: "archived"
         - name: BROKER_ROUTINGERROR

diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml
@@ -168,6 +168,7 @@ global:
     username: ""
     verifyPeer: true
     vhost: "/"
+    prefetchCount: 2
 
   cega:
     ## @param host, URI to CEGA NSS server users endpoint
@@ -316,7 +317,7 @@ auth:
   name: auth
   replicaCount: 2
   repository: ghcr.io/neicnordic/sda-auth
-  imageTag: v0.6.54
+  imageTag: v0.7.6
   imagePullPolicy: IfNotPresent
   resources:
     requests:
@@ -337,7 +338,7 @@ backup:
   deploy: false
   replicaCount: 1
   repository: ghcr.io/neicnordic/sda-pipeline
-  imageTag: v0.3.94
+  imageTag: v0.4.27
   imagePullPolicy: IfNotPresent
   resources:
     requests:
@@ -356,7 +357,7 @@ backup:
 doa:
   name: doa
   repository: ghcr.io/neicnordic/sda-doa
-  imageTag: v1.6.46
+  imageTag: v1.6.62
   imagePullPolicy: IfNotPresent
   replicaCount: 2
   resources:
@@ -380,7 +381,7 @@ download:
   name: download
   replicaCount: 2
   repository: ghcr.io/neicnordic/sda-download
-  imageTag: v1.6.117
+  imageTag: v1.9.14
   imagePullPolicy: IfNotPresent
   resources:
     requests:
@@ -400,7 +401,7 @@ finalize:
   name: finalize
   replicaCount: 1
   repository: ghcr.io/neicnordic/sda-pipeline
-  imageTag: v0.3.94
+  imageTag: v0.4.27
   imagePullPolicy: IfNotPresent
   resources:
     requests:
@@ -420,7 +421,7 @@ ingest:
   name: ingest
   replicaCount: 1
   repository: ghcr.io/neicnordic/sda-pipeline
-  imageTag: v0.3.94
+  imageTag: v0.4.27
   imagePullPolicy: IfNotPresent
   resources:
     requests:
@@ -441,7 +442,7 @@ intercept:
   name: ingest
   replicaCount: 1
   repository: ghcr.io/neicnordic/sda-pipeline
-  imageTag: v0.3.94
+  imageTag: v0.4.27
   imagePullPolicy: IfNotPresent
   resources:
     requests:
@@ -460,7 +461,7 @@ intercept:
 mapper:
   replicaCount: 1
   repository: ghcr.io/neicnordic/sda-pipeline
-  imageTag: v0.3.94
+  imageTag: v0.4.27
   imagePullPolicy: IfNotPresent
   resources:
     requests:
@@ -479,7 +480,7 @@ mapper:
 s3Inbox:
   name: s3Inbox
   repository: ghcr.io/neicnordic/sda-s3proxy
-  imageTag: v0.1.153
+  imageTag: v0.2.38
   imagePullPolicy: IfNotPresent
   replicaCount: 2
   resources:
@@ -500,7 +501,7 @@ s3Inbox:
 sftpInbox:
   name: sftpInbox
   repository: ghcr.io/neicnordic/sda-inbox-sftp
-  imageTag: v1.10.4
+  imageTag: v1.12.16
   imagePullPolicy: IfNotPresent
   replicaCount: 2
   resources:
@@ -533,7 +534,7 @@ releasetest:
 verify:
   replicaCount: 1
   repository: ghcr.io/neicnordic/sda-pipeline
-  imageTag: v0.3.94
+  imageTag: v0.4.27
   imagePullPolicy: IfNotPresent
   resources:
     requests: