From c5b6999f2f1a63657fa40cd5bc7619750c71a4ef Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Thu, 8 Feb 2024 16:06:00 -0800
Subject: [PATCH 01/18] add auxiliary worker to run different queues

Add a second worker that runs all GoodJob queues. The "base" worker will
run all queues except the :auxiliary queue.

This, combined with the high priority assigned to all jobs in the
:auxiliary queue, effectively builds a fast lane; all jobs run in the
:auxiliary queue will be executed before all other jobs by the auxiliary
worker.

The fact that the base worker does not run any jobs in the :auxiliary
queue means we can have this fast lane without putting every other job
on hold for, potentially, long periods of time.

P.S. The reason we use `before_enqueue` in ApplicationJob is because
GoodJob does not support setting a default priority per queue.
---
 .env                        |  1 +
 app/jobs/application_job.rb |  9 ++++++
 docker-compose.yml          | 64 +++++++++++++++++++++----------------
 3 files changed, 46 insertions(+), 28 deletions(-)

diff --git a/.env b/.env
index 17dc7b76..9df84cac 100644
--- a/.env
+++ b/.env
@@ -15,6 +15,7 @@ FITS_SERVLET_URL=http://fits:8080/fits
 GOOD_JOB_CLEANUP_DISCARDED_JOBS=false
 GOOD_JOB_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO=604800
 GOOD_JOB_CLEANUP_INTERVAL_SECONDS=86400
+GOOD_JOB_QUEUES=-auxiliary
 INITIAL_ADMIN_EMAIL=admin@example.com
 INITIAL_ADMIN_PASSWORD=testing123
 IN_DOCKER=true
diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb
index 70d71689..3589e201 100644
--- a/app/jobs/application_job.rb
+++ b/app/jobs/application_job.rb
@@ -1,8 +1,17 @@
 # frozen_string_literal: true
 
 class ApplicationJob < ActiveJob::Base
+  before_enqueue :set_auxiliary_queue_priority
+
   # limit to 5 attempts
   retry_on StandardError, wait: :exponentially_longer, attempts: 5 do |_job, _exception|
     # Log error, do nothing, etc.
   end
+
+  private
+
+    def set_auxiliary_queue_priority
+      # TODO: make ENV variable
+      self.priority = 100 if queue_name.to_sym == :auxiliary
+    end
 end
diff --git a/docker-compose.yml b/docker-compose.yml
index b7a6d1d5..533f21fa 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -23,6 +23,36 @@ x-app: &app
   networks:
     internal:
 
+x-app-worker: &app-worker
+  <<: *app
+  build:
+    context: .
+    target: hyku-worker
+    args:
+      - SETTINGS__BULKRAX__ENABLED=true
+    cache_from:
+      - ghcr.io/scientist-softserv/adventist-dl/base:${TAG:-latest}
+      - ghcr.io/scientist-softserv/adventist-dl:${TAG:-latest}
+      - ghcr.io/scientist-softserv/adventist-dl/worker:${TAG:-latest}
+  image: ghcr.io/scientist-softserv/adventist-dl/worker:${TAG:-latest}
+  depends_on:
+    db:
+      condition: service_started
+    solr:
+      condition: service_started
+    fcrepo:
+      condition: service_started
+    redis:
+      condition: service_started
+    fits:
+      condition: service_started
+    zoo:
+      condition: service_started
+    check_volumes:
+      condition: service_completed_successfully
+    initialize_app:
+      condition: service_completed_successfully
+
 volumes:
   assets:
   cache:
@@ -186,7 +216,7 @@ services:
       - 3000
 
   worker:
-    <<: *app
+    <<: *app-worker
     ################################################################################
     ## Note on commands: by default the commands don't run bundle.  That is to
     ## reduce boot times.  However, when the application is in active
@@ -203,33 +233,11 @@ services:
     ## in your code, bash into the worker container, and interact with the
     ## breakpoints.
     # command: sh -l -c "bundle && tail -f /dev/null"
-    build:
-      context: .
-      target: hyku-worker
-      args:
-        - SETTINGS__BULKRAX__ENABLED=true
-      cache_from:
-        - ghcr.io/scientist-softserv/adventist-dl/base:${TAG:-latest}
-        - ghcr.io/scientist-softserv/adventist-dl:${TAG:-latest}
-        - ghcr.io/scientist-softserv/adventist-dl/worker:${TAG:-latest}
-    image: ghcr.io/scientist-softserv/adventist-dl/worker:${TAG:-latest}
-    depends_on:
-      db:
-        condition: service_started
-      solr:
-        condition: service_started
-      fcrepo:
-        condition: service_started
-      redis:
-        condition: service_started
-      fits:
-        condition: service_started
-      zoo:
-        condition: service_started
-      check_volumes:
-        condition: service_completed_successfully
-      initialize_app:
-        condition: service_completed_successfully
+
+  worker_auxiliary:
+    <<: *app-worker
+    environment:
+      - GOOD_JOB_QUEUES=*
 
   # Do not recurse through all of tmp. derivitives will make booting
   # very slow and eventually just time out as data grows

From 9cf9a7fcca3848d90b038bdc552dad773bc24ca4 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Thu, 8 Feb 2024 18:08:03 -0800
Subject: [PATCH 02/18] add workerAuxiliary as extra deploy (dev)

---
 ops/dev-deploy.tmpl.yaml | 152 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 152 insertions(+)

diff --git a/ops/dev-deploy.tmpl.yaml b/ops/dev-deploy.tmpl.yaml
index 1f17a2d2..f811197a 100644
--- a/ops/dev-deploy.tmpl.yaml
+++ b/ops/dev-deploy.tmpl.yaml
@@ -94,6 +94,8 @@ extraEnvVars: &envVars
     value: /rest
   - name: FEDORA_URL
     value: http://fcrepo.default.svc.cluster.local:8080/rest
+  - name: GOOD_JOB_QUEUES
+    value: -auxiliary
   - name: IN_DOCKER
     value: "true"
   - name: LD_LIBRARY_PATH
@@ -228,6 +230,156 @@ worker:
   extraVolumeMounts: *volMounts
   extraEnvVars: *envVars
 
+# When adding/removing key-value pairs to this block, ensure the
+# corresponding changes are made in the `extraDeploy` block below.
+workerAuxiliary:
+  extraEnvVars:
+    - name: GOOD_JOB_QUEUES
+      value: "*"
+
+extraDeploy:
+  - |-
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: {{ include "hyrax.fullname" . }}-auxiliary-worker
+        labels:
+          {{- include "hyrax.labels" . | nindent 4 }}
+      spec:
+        replicas: {{ .Values.worker.replicaCount }}
+        selector:
+          matchLabels:
+            {{- include "hyrax.workerSelectorLabels" . | nindent 6 }}
+        template:
+          metadata:
+          {{- with .Values.podAnnotations }}
+            annotations:
+              {{- toYaml . | nindent 8 }}
+          {{- end }}
+            labels:
+              {{- include "hyrax.workerSelectorLabels" . | nindent 8 }}
+          spec:
+            initContainers:
+              - name: db-wait
+                image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}"
+                imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
+                envFrom:
+                  - configMapRef:
+                      name: {{ include "hyrax.fullname" . }}-env
+                  - secretRef:
+                      name: {{ template "hyrax.fullname" . }}
+                env:
+                  {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
+                  {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
+                command:
+                  - sh
+                  - -c
+                  - db-wait.sh "$REDIS_HOST:6379"
+              {{- if .Values.worker.extraInitContainers }}
+              {{- toYaml .Values.worker.extraInitContainers | nindent 8 }}
+              {{- end }}
+            {{- with .Values.imagePullSecrets }}
+            imagePullSecrets:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            serviceAccountName: {{ include "hyrax.serviceAccountName" . }}
+            securityContext:
+              {{- toYaml .Values.worker.podSecurityContext | nindent 8 }}
+            containers:
+              - name: {{ .Chart.Name }}-worker
+                securityContext:
+                  {{- toYaml .Values.worker.securityContext | nindent 12 }}
+                image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}"
+                imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
+                envFrom:
+                  - configMapRef:
+                      name: {{ include "hyrax.fullname" . }}-env
+                  - secretRef:
+                      name: {{ template "hyrax.fullname" . }}
+                  {{- if .Values.solrExistingSecret }}
+                  - secretRef:
+                      name: {{ .Values.solrExistingSecret }}
+                  {{- end }}
+                  {{- with .Values.worker.extraEnvFrom }}
+                    {{- toYaml . | nindent 12 }}
+                  {{- end }}
+                env:
+                  {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
+                  {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
+                {{- if .Values.worker.readinessProbe.enabled }}
+                readinessProbe:
+                  exec:
+                    command:
+                      {{- toYaml .Values.worker.readinessProbe.command | nindent 16 }}
+                  failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }}
+                  initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }}
+                  periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }}
+                  successThreshold: {{ .Values.worker.readinessProbe.successThreshold }}
+                  timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }}
+                {{- end }}
+                volumeMounts:
+                  - name: derivatives
+                    mountPath: /app/samvera/derivatives
+                  - name: uploads
+                    subPath: file_cache
+                    mountPath: /app/samvera/file_cache
+                  - name: uploads
+                    subPath: uploads
+                    mountPath: /app/samvera/uploads
+                  {{- if .Values.applicationExistingClaim }}
+                  - name: application
+                    mountPath: /app/samvera/hyrax-webapp
+                  {{- end }}
+                  {{- with .Values.worker.extraVolumeMounts }}
+                  {{- toYaml . | nindent 12 }}
+                  {{- end }}
+                resources:
+                  {{- toYaml .Values.worker.resources | nindent 12 }}
+                {{- with .Values.extraContainerConfiguration }}
+                {{- toYaml . | nindent 10 }}
+                {{- end }}
+            volumes:
+              - name: "derivatives"
+                {{- if and .Values.derivativesVolume.enabled .Values.derivativesVolume.existingClaim }}
+                persistentVolumeClaim:
+                  claimName: {{ .Values.derivativesVolume.existingClaim }}
+                {{- else if .Values.derivativesVolume.enabled }}
+                persistentVolumeClaim:
+                  claimName: {{ template "hyrax.fullname" . }}-derivatives
+                {{ else }}
+                emptyDir: {}
+                {{ end }}
+              - name: "uploads"
+                {{- if and .Values.uploadsVolume.enabled .Values.uploadsVolume.existingClaim }}
+                persistentVolumeClaim:
+                  claimName: {{ .Values.uploadsVolume.existingClaim }}
+                {{- else if .Values.uploadsVolume.enabled }}
+                persistentVolumeClaim:
+                  claimName: {{ template "hyrax.fullname" . }}-uploads
+                {{ else }}
+                emptyDir: {}
+                {{ end }}
+              {{- if .Values.applicationExistingClaim }}
+              - name: "application"
+                persistentVolumeClaim:
+                  claimName: {{ .Values.applicationExistingClaim }}
+              {{- end }}
+              {{- with .Values.worker.extraVolumes }}
+              {{- toYaml . | nindent 8 }}
+              {{- end }}
+            {{- with .Values.worker.nodeSelector }}
+            nodeSelector:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            {{- with .Values.worker.affinity }}
+            affinity:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            {{- with .Values.worker.tolerations }}
+            tolerations:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+
 podSecurityContext:
   runAsUser: 1001
   runAsGroup: 101

From 5988af285ce252297bccb3bd54d4886d7f166637 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Thu, 8 Feb 2024 18:12:39 -0800
Subject: [PATCH 03/18] replace JSON syntax with YAML syntax

---
 ops/dev-deploy.tmpl.yaml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/ops/dev-deploy.tmpl.yaml b/ops/dev-deploy.tmpl.yaml
index f811197a..ddba14dc 100644
--- a/ops/dev-deploy.tmpl.yaml
+++ b/ops/dev-deploy.tmpl.yaml
@@ -46,11 +46,10 @@ ingress:
     - host: "*.s2.adventistdigitallibrary.org"
       paths:
         - path: /
-  annotations: {
-    kubernetes.io/ingress.class: "nginx",
-    nginx.ingress.kubernetes.io/proxy-body-size: "0",
-    cert-manager.io/cluster-issuer: wildcard-issuer
-  }
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    cert-manager.io/cluster-issuer: "wildcard-issuer"
   tls:
     - hosts:
         - '*.s2.adventistdigitallibrary.org'

From e9cc8165268b3c10c7bd600c7ad8419bd9ba72f6 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Fri, 9 Feb 2024 11:21:39 -0800
Subject: [PATCH 04/18] use ENV var for :auxiliary queue priority

---
 .env                        | 1 +
 app/jobs/application_job.rb | 5 +++--
 docker-compose.yml          | 2 ++
 ops/dev-deploy.tmpl.yaml    | 2 ++
 4 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/.env b/.env
index 9df84cac..d190bec1 100644
--- a/.env
+++ b/.env
@@ -1,3 +1,4 @@
+AUXILIARY_QUEUE_PRIORITY=100
 CHROME_HOSTNAME=chrome
 DATABASE_ADAPTER=postgresql
 DATABASE_HOST=db
diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb
index 3589e201..459f4f20 100644
--- a/app/jobs/application_job.rb
+++ b/app/jobs/application_job.rb
@@ -11,7 +11,8 @@ class ApplicationJob < ActiveJob::Base
   private
 
     def set_auxiliary_queue_priority
-      # TODO: make ENV variable
-      self.priority = 100 if queue_name.to_sym == :auxiliary
+      return unless queue_name.to_sym == :auxiliary
+
+      self.priority = ENV.fetch('AUXILIARY_QUEUE_PRIORITY', 100).to_i
     end
 end
diff --git a/docker-compose.yml b/docker-compose.yml
index 533f21fa..28dea7ab 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -210,6 +210,8 @@ services:
         condition: service_started
       worker:
         condition: service_started
+      worker_auxiliary:
+        condition: service_started
       initialize_app:
         condition: service_completed_successfully
     expose:
diff --git a/ops/dev-deploy.tmpl.yaml b/ops/dev-deploy.tmpl.yaml
index ddba14dc..d7f76e52 100644
--- a/ops/dev-deploy.tmpl.yaml
+++ b/ops/dev-deploy.tmpl.yaml
@@ -95,6 +95,8 @@ extraEnvVars: &envVars
     value: http://fcrepo.default.svc.cluster.local:8080/rest
   - name: GOOD_JOB_QUEUES
     value: -auxiliary
+  - name: AUXILIARY_QUEUE_PRIORITY
+    value: "100"
   - name: IN_DOCKER
     value: "true"
   - name: LD_LIBRARY_PATH

From 5f55100cb9b05d8344d36a74c327dde58a9cf69b Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Fri, 9 Feb 2024 12:14:36 -0800
Subject: [PATCH 05/18] fix redis url in dev helm deploy templ

---
 ops/dev-deploy.tmpl.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ops/dev-deploy.tmpl.yaml b/ops/dev-deploy.tmpl.yaml
index d7f76e52..9ea84ec3 100644
--- a/ops/dev-deploy.tmpl.yaml
+++ b/ops/dev-deploy.tmpl.yaml
@@ -275,7 +275,7 @@ extraDeploy:
                 command:
                   - sh
                   - -c
-                  - db-wait.sh "$REDIS_HOST:6379"
+                  - "db-wait.sh {{ include "hyrax.redis.host" . }}:6379"
               {{- if .Values.worker.extraInitContainers }}
               {{- toYaml .Values.worker.extraInitContainers | nindent 8 }}
               {{- end }}

From f1e57ffe213e463934a80a6910dfd2851281c051 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Fri, 9 Feb 2024 12:20:04 -0800
Subject: [PATCH 06/18] set GOOD_JOB_QUEUES on each worker specifically

This resolves the "duplicate name" warning when deploying. However, it
may remove all other ENV vars from the base worker, so this change may
not stick around
---
 ops/dev-deploy.tmpl.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ops/dev-deploy.tmpl.yaml b/ops/dev-deploy.tmpl.yaml
index 9ea84ec3..84fe2812 100644
--- a/ops/dev-deploy.tmpl.yaml
+++ b/ops/dev-deploy.tmpl.yaml
@@ -93,8 +93,6 @@ extraEnvVars: &envVars
     value: /rest
   - name: FEDORA_URL
     value: http://fcrepo.default.svc.cluster.local:8080/rest
-  - name: GOOD_JOB_QUEUES
-    value: -auxiliary
   - name: AUXILIARY_QUEUE_PRIORITY
     value: "100"
   - name: IN_DOCKER
@@ -229,7 +227,9 @@ worker:
     fsGroup: 101
     fsGroupChangePolicy: "OnRootMismatch"
   extraVolumeMounts: *volMounts
-  extraEnvVars: *envVars
+  extraEnvVars:
+    - name: GOOD_JOB_QUEUES
+      value: -auxiliary
 
 # When adding/removing key-value pairs to this block, ensure the
 # corresponding changes are made in the `extraDeploy` block below.

From 4258d94d53d100a89fe9e64c670b7d577501ccad Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Fri, 9 Feb 2024 15:57:36 -0800
Subject: [PATCH 07/18] pull extraEnvVars into auxiliary worker directly

---
 ops/dev-deploy.tmpl.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ops/dev-deploy.tmpl.yaml b/ops/dev-deploy.tmpl.yaml
index 84fe2812..d90f43b0 100644
--- a/ops/dev-deploy.tmpl.yaml
+++ b/ops/dev-deploy.tmpl.yaml
@@ -93,6 +93,8 @@ extraEnvVars: &envVars
     value: /rest
   - name: FEDORA_URL
     value: http://fcrepo.default.svc.cluster.local:8080/rest
+  - name: GOOD_JOB_QUEUES
+    value: -auxiliary
   - name: AUXILIARY_QUEUE_PRIORITY
     value: "100"
   - name: IN_DOCKER
@@ -227,9 +229,7 @@ worker:
     fsGroup: 101
     fsGroupChangePolicy: "OnRootMismatch"
   extraVolumeMounts: *volMounts
-  extraEnvVars:
-    - name: GOOD_JOB_QUEUES
-      value: -auxiliary
+  extraEnvVars: *envVars
 
 # When adding/removing key-value pairs to this block, ensure the
 # corresponding changes are made in the `extraDeploy` block below.
@@ -271,7 +271,7 @@ extraDeploy:
                       name: {{ template "hyrax.fullname" . }}
                 env:
                   {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
-                  {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
+                  {{- toYaml .Values.extraEnvVars | nindent 12 }}
                 command:
                   - sh
                   - -c

From 16e1bc392641ad455faf867b609715830a2ef55c Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Fri, 9 Feb 2024 16:34:44 -0800
Subject: [PATCH 08/18] helm is picky about order of env variables

---
 ops/dev-deploy.tmpl.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ops/dev-deploy.tmpl.yaml b/ops/dev-deploy.tmpl.yaml
index d90f43b0..522c85bb 100644
--- a/ops/dev-deploy.tmpl.yaml
+++ b/ops/dev-deploy.tmpl.yaml
@@ -93,8 +93,6 @@ extraEnvVars: &envVars
     value: /rest
   - name: FEDORA_URL
     value: http://fcrepo.default.svc.cluster.local:8080/rest
-  - name: GOOD_JOB_QUEUES
-    value: -auxiliary
   - name: AUXILIARY_QUEUE_PRIORITY
     value: "100"
   - name: IN_DOCKER
@@ -213,6 +211,8 @@ extraEnvVars: &envVars
     value: sqs://us-east-1.amazonaws.com/031107666127/space-stone-production-thumbnail/
   - name: SERVERLESS_WORD_DLQ
     value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-word-coordinates-dlq
+  - name: GOOD_JOB_QUEUES
+    value: -auxiliary
 
 worker:
   replicaCount: 1
@@ -270,8 +270,8 @@ extraDeploy:
                   - secretRef:
                       name: {{ template "hyrax.fullname" . }}
                 env:
-                  {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
                   {{- toYaml .Values.extraEnvVars | nindent 12 }}
+                  {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
                 command:
                   - sh
                   - -c

From 98cf7c1d7177947c47d530aaa2f6b27f42035aa0 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Fri, 9 Feb 2024 17:14:06 -0800
Subject: [PATCH 09/18] bring container config inline with initContainers

---
 ops/dev-deploy.tmpl.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ops/dev-deploy.tmpl.yaml b/ops/dev-deploy.tmpl.yaml
index 522c85bb..07091f7d 100644
--- a/ops/dev-deploy.tmpl.yaml
+++ b/ops/dev-deploy.tmpl.yaml
@@ -305,8 +305,8 @@ extraDeploy:
                     {{- toYaml . | nindent 12 }}
                   {{- end }}
                 env:
+                  {{- toYaml .Values.extraEnvVars | nindent 12 }}
                   {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
-                  {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
                 {{- if .Values.worker.readinessProbe.enabled }}
                 readinessProbe:
                   exec:

From 4afc1dfacd22a65c40b344faeb36ee1b7e4fe6e3 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Mon, 12 Feb 2024 09:48:39 -0800
Subject: [PATCH 10/18] explicitly duplicate extraEnvVars

This is an attempt to resolve this helm upgrade error:
> The order in patch list: ... doesn't match $setElementOrder list: ...
---
 ops/dev-deploy.tmpl.yaml | 314 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 311 insertions(+), 3 deletions(-)

diff --git a/ops/dev-deploy.tmpl.yaml b/ops/dev-deploy.tmpl.yaml
index 07091f7d..191c0c5d 100644
--- a/ops/dev-deploy.tmpl.yaml
+++ b/ops/dev-deploy.tmpl.yaml
@@ -229,12 +229,322 @@ worker:
     fsGroup: 101
     fsGroupChangePolicy: "OnRootMismatch"
   extraVolumeMounts: *volMounts
-  extraEnvVars: *envVars
+  extraEnvVars:
+    - name: SETTINGS__MULTITENANCY__ADMIN_HOST
+      value: s2.adventistdigitallibrary.org
+    - name: SETTINGS__MULTITENANCY__ROOT_HOST
+      value: s2.adventistdigitallibrary.org
+    - name: SETTINGS__MULTITENANCY__DEFAULT_HOST
+      value: "%{tenant}.s2.adventistdigitallibrary.org"
+    - name: CONFDIR
+      value: "/app/samvera/hyrax-webapp/solr/config"
+    - name: INITIAL_ADMIN_EMAIL
+      value: support@notch8.com
+    - name: INITIAL_ADMIN_PASSWORD
+      value: testing123
+    - name: DATABASE_ADAPTER
+      value: postgresql
+    - name: DB_ADAPTER
+      value: postgresql
+    - name: DATABASE_HOST
+      value: acid-postgres-cluster-bravo.postgres.svc.cluster.local
+    - name: DB_HOST
+      value: acid-postgres-cluster-bravo.postgres.svc.cluster.local
+    - name: DATABASE_NAME
+      value: hyku-dev-hyrax
+    - name: DB_PORT
+      value: "5432"
+    - name: DATABASE_USER
+      value: hyku-dev-hyrax
+    - name: EXTERNAL_IIIF_URL
+      value: https://d3pg70bdc74ala.cloudfront.net/iiif/2
+    - name: FCREPO_BASE_PATH
+      value: /adventist-dev
+    - name: FCREPO_HOST
+      value: fcrepo.default.svc.cluster.local:8080
+    - name: FCREPO_PATH
+      value: /rest
+    - name: FEDORA_URL
+      value: http://fcrepo.default.svc.cluster.local:8080/rest
+    - name: AUXILIARY_QUEUE_PRIORITY
+      value: "100"
+    - name: IN_DOCKER
+      value: "true"
+    - name: LD_LIBRARY_PATH
+      value: /app/fits/tools/mediainfo/linux
+    - name: PASSENGER_APP_ENV
+      value: production
+    - name: RAILS_CACHE_STORE_URL
+      value: redis://:$REDIS_PASSWORD@hyku-dev-redis-master:6379/adventist
+    - name: RAILS_ENV
+      value: production
+    - name: RAILS_LOG_TO_STDOUT
+      value: "true"
+    - name: RAILS_MAX_THREADS
+      value: "5"
+    - name: RAILS_SERVE_STATIC_FILES
+      value: "true"
+    - name: REDIS_HOST
+      value: hyku-dev-redis-master
+    - name: REDIS_URL
+      value: redis://:$REDIS_PASSWORD@hyku-dev-redis-master:6379/adventist
+    - name: SETTINGS__ACTIVE_JOB__QUEUE_ADAPTER
+      value: good_job
+    - name: SETTINGS__BULKRAX__ENABLED
+      value: "true"
+    - name: SETTINGS__CONTACT_EMAIL
+      value: support@notch8.com
+    - name: SETTINGS__DEVISE__INVITATION_FROM_EMAIL
+      value: support@notch8.com
+    - name: SETTINGS__FITS_PATH
+      value: /app/fits/fits.sh
+    - name: SETTINGS__MULTITENANCY__ADMIN_ONLY_TENANT_CREATION
+      value: "true"
+    - name: SETTINGS__MULTITENANCY__DEFAULT_HOST
+      value: "%{tenant}.s2.adventistdigitallibrary.org"
+    - name: SETTINGS__MULTITENANCY__ENABLED
+      value: "true"
+    - name: SETTINGS__SOLR__COLLECTION_OPTIONS__COLLECTION__CONFIG_NAME
+      value: hyku-adventist-dev
+    - name: SETTINGS__SOLR__CONFIGSET
+      value: hyku-adventist-dev
+    - name: SETTINGS__ZOOKEEPER__CONNECTION_STR
+      value: solr-zookeeper-headless.default:2181/configs
+    - name: SOLR_ADMIN_USER
+      value: admin
+    - name: SOLR_COLLECTION_NAME
+      value: hyku-adventist-dev
+    - name: SOLR_CONFIGSET_NAME
+      value: hyku-adventist-dev
+    - name: SOLR_HOST
+      value: solr.default
+    - name: SOLR_PORT
+      value: "8983"
+    - name: SOLR_URL
+      value: http://admin:$SOLR_ADMIN_PASSWORD@solr.default:8983/solr/
+    - name: SETTINGS__SOLR__URL
+      value: http://admin:$SOLR_ADMIN_PASSWORD@solr.default:8983/solr/
+    - name: SMTP_ENABLED
+      value: "true"
+    - name: SMTP_USER_NAME
+      value: fc604e3b82d260
+    - name: SMTP_PASSWORD
+      value: $SMTP_PASSWORD
+    - name: SMTP_ADDRESS
+      value: smtp.mailtrap.io
+    - name: SMTP_DOMAIN
+      value: smtp.mailtrap.io
+    - name: SMTP_PORT
+      value: "2525"
+    - name: SMTP_TYPE
+      value: cram_md5
+    - name: SENTRY_DSN
+      value: https://5ee54fb1e8cd448a9e709c846635ec49@o1008683.ingest.sentry.io/6745020
+    - name: SENTRY_ENVIRONMENT
+      value: "staging"
+    - name: OMP_THREAD_LIMIT
+      value: "1"
+    - name: SECRET_KEY_BASE
+      value: $SECRET_KEY_BASE
+    - name: AWS_ACCESS_KEY_ID
+      value: $AWS_ACCESS_KEY_ID
+    - name: AWS_REGION
+      value: us-east-1
+    - name: AWS_S3_BUCKET
+      value: space-stone-production-preprocessedbucketf21466dd-15sun4xy658nh
+    - name: AWS_SECRET_ACCESS_KEY
+      value: $AWS_SECRET_ACCESS_KEY
+    - name: SERVERLESS_ALTO_DLQ
+      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-alto-xml-dlq
+    - name: SERVERLESS_BATCH_SIZE
+      value: "10"
+    - name: SERVERLESS_COPY_DLQ
+      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-copy-dlq
+    - name: SERVERLESS_COPY_SQS_URL
+      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-copy
+    - name: SERVERLESS_COPY_URL
+      value: https://3oqe2l9evk.execute-api.us-east-1.amazonaws.com/copy
+    - name: SERVERLESS_OCR_DLQ
+      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-ocr-dlq
+    - name: SERVERLESS_OCR_SQS_URL
+      value: sqs://us-east-1.amazonaws.com/031107666127/space-stone-production-ocr/
+    - name: SERVERLESS_OCR_THUMB_DLQ
+      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-split-ocr-thumbnail-dlq
+    - name: SERVERLESS_PLAIN_TEXT_DLQ
+      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-plain-text-dlq
+    - name: SERVERLESS_S3_URL
+      value: s3://space-stone-production-preprocessedbucketf21466dd-15sun4xy658nh.s3.us-east-1.amazonaws.com/
+    - name: SERVERLESS_SPLIT_SQS_URL
+      value: sqs://us-east-1.amazonaws.com/031107666127/space-stone-production-split-ocr-thumbnail/
+    - name: SERVERLESS_TEMPLATE
+      value: "{{dir_parts[-1..-1]}}/{{ basename }}{{ extension }}"
+    - name: SERVERLESS_THUMBNAIL_DLQ
+      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-thumbnail-dlq
+    - name: SERVERLESS_THUMBNAIL_SQS_URL
+      value: sqs://us-east-1.amazonaws.com/031107666127/space-stone-production-thumbnail/
+    - name: SERVERLESS_WORD_DLQ
+      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-word-coordinates-dlq
+    - name: GOOD_JOB_QUEUES
+      value: -auxiliary
 
 # When adding/removing key-value pairs to this block, ensure the
 # corresponding changes are made in the `extraDeploy` block below.
 workerAuxiliary:
   extraEnvVars:
+    - name: SETTINGS__MULTITENANCY__ADMIN_HOST
+      value: s2.adventistdigitallibrary.org
+    - name: SETTINGS__MULTITENANCY__ROOT_HOST
+      value: s2.adventistdigitallibrary.org
+    - name: SETTINGS__MULTITENANCY__DEFAULT_HOST
+      value: "%{tenant}.s2.adventistdigitallibrary.org"
+    - name: CONFDIR
+      value: "/app/samvera/hyrax-webapp/solr/config"
+    - name: INITIAL_ADMIN_EMAIL
+      value: support@notch8.com
+    - name: INITIAL_ADMIN_PASSWORD
+      value: testing123
+    - name: DATABASE_ADAPTER
+      value: postgresql
+    - name: DB_ADAPTER
+      value: postgresql
+    - name: DATABASE_HOST
+      value: acid-postgres-cluster-bravo.postgres.svc.cluster.local
+    - name: DB_HOST
+      value: acid-postgres-cluster-bravo.postgres.svc.cluster.local
+    - name: DATABASE_NAME
+      value: hyku-dev-hyrax
+    - name: DB_PORT
+      value: "5432"
+    - name: DATABASE_USER
+      value: hyku-dev-hyrax
+    - name: EXTERNAL_IIIF_URL
+      value: https://d3pg70bdc74ala.cloudfront.net/iiif/2
+    - name: FCREPO_BASE_PATH
+      value: /adventist-dev
+    - name: FCREPO_HOST
+      value: fcrepo.default.svc.cluster.local:8080
+    - name: FCREPO_PATH
+      value: /rest
+    - name: FEDORA_URL
+      value: http://fcrepo.default.svc.cluster.local:8080/rest
+    - name: AUXILIARY_QUEUE_PRIORITY
+      value: "100"
+    - name: IN_DOCKER
+      value: "true"
+    - name: LD_LIBRARY_PATH
+      value: /app/fits/tools/mediainfo/linux
+    - name: PASSENGER_APP_ENV
+      value: production
+    - name: RAILS_CACHE_STORE_URL
+      value: redis://:$REDIS_PASSWORD@hyku-dev-redis-master:6379/adventist
+    - name: RAILS_ENV
+      value: production
+    - name: RAILS_LOG_TO_STDOUT
+      value: "true"
+    - name: RAILS_MAX_THREADS
+      value: "5"
+    - name: RAILS_SERVE_STATIC_FILES
+      value: "true"
+    - name: REDIS_HOST
+      value: hyku-dev-redis-master
+    - name: REDIS_URL
+      value: redis://:$REDIS_PASSWORD@hyku-dev-redis-master:6379/adventist
+    - name: SETTINGS__ACTIVE_JOB__QUEUE_ADAPTER
+      value: good_job
+    - name: SETTINGS__BULKRAX__ENABLED
+      value: "true"
+    - name: SETTINGS__CONTACT_EMAIL
+      value: support@notch8.com
+    - name: SETTINGS__DEVISE__INVITATION_FROM_EMAIL
+      value: support@notch8.com
+    - name: SETTINGS__FITS_PATH
+      value: /app/fits/fits.sh
+    - name: SETTINGS__MULTITENANCY__ADMIN_ONLY_TENANT_CREATION
+      value: "true"
+    - name: SETTINGS__MULTITENANCY__DEFAULT_HOST
+      value: "%{tenant}.s2.adventistdigitallibrary.org"
+    - name: SETTINGS__MULTITENANCY__ENABLED
+      value: "true"
+    - name: SETTINGS__SOLR__COLLECTION_OPTIONS__COLLECTION__CONFIG_NAME
+      value: hyku-adventist-dev
+    - name: SETTINGS__SOLR__CONFIGSET
+      value: hyku-adventist-dev
+    - name: SETTINGS__ZOOKEEPER__CONNECTION_STR
+      value: solr-zookeeper-headless.default:2181/configs
+    - name: SOLR_ADMIN_USER
+      value: admin
+    - name: SOLR_COLLECTION_NAME
+      value: hyku-adventist-dev
+    - name: SOLR_CONFIGSET_NAME
+      value: hyku-adventist-dev
+    - name: SOLR_HOST
+      value: solr.default
+    - name: SOLR_PORT
+      value: "8983"
+    - name: SOLR_URL
+      value: http://admin:$SOLR_ADMIN_PASSWORD@solr.default:8983/solr/
+    - name: SETTINGS__SOLR__URL
+      value: http://admin:$SOLR_ADMIN_PASSWORD@solr.default:8983/solr/
+    - name: SMTP_ENABLED
+      value: "true"
+    - name: SMTP_USER_NAME
+      value: fc604e3b82d260
+    - name: SMTP_PASSWORD
+      value: $SMTP_PASSWORD
+    - name: SMTP_ADDRESS
+      value: smtp.mailtrap.io
+    - name: SMTP_DOMAIN
+      value: smtp.mailtrap.io
+    - name: SMTP_PORT
+      value: "2525"
+    - name: SMTP_TYPE
+      value: cram_md5
+    - name: SENTRY_DSN
+      value: https://5ee54fb1e8cd448a9e709c846635ec49@o1008683.ingest.sentry.io/6745020
+    - name: SENTRY_ENVIRONMENT
+      value: "staging"
+    - name: OMP_THREAD_LIMIT
+      value: "1"
+    - name: SECRET_KEY_BASE
+      value: $SECRET_KEY_BASE
+    - name: AWS_ACCESS_KEY_ID
+      value: $AWS_ACCESS_KEY_ID
+    - name: AWS_REGION
+      value: us-east-1
+    - name: AWS_S3_BUCKET
+      value: space-stone-production-preprocessedbucketf21466dd-15sun4xy658nh
+    - name: AWS_SECRET_ACCESS_KEY
+      value: $AWS_SECRET_ACCESS_KEY
+    - name: SERVERLESS_ALTO_DLQ
+      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-alto-xml-dlq
+    - name: SERVERLESS_BATCH_SIZE
+      value: "10"
+    - name: SERVERLESS_COPY_DLQ
+      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-copy-dlq
+    - name: SERVERLESS_COPY_SQS_URL
+      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-copy
+    - name: SERVERLESS_COPY_URL
+      value: https://3oqe2l9evk.execute-api.us-east-1.amazonaws.com/copy
+    - name: SERVERLESS_OCR_DLQ
+      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-ocr-dlq
+    - name: SERVERLESS_OCR_SQS_URL
+      value: sqs://us-east-1.amazonaws.com/031107666127/space-stone-production-ocr/
+    - name: SERVERLESS_OCR_THUMB_DLQ
+      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-split-ocr-thumbnail-dlq
+    - name: SERVERLESS_PLAIN_TEXT_DLQ
+      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-plain-text-dlq
+    - name: SERVERLESS_S3_URL
+      value: s3://space-stone-production-preprocessedbucketf21466dd-15sun4xy658nh.s3.us-east-1.amazonaws.com/
+    - name: SERVERLESS_SPLIT_SQS_URL
+      value: sqs://us-east-1.amazonaws.com/031107666127/space-stone-production-split-ocr-thumbnail/
+    - name: SERVERLESS_TEMPLATE
+      value: "{{dir_parts[-1..-1]}}/{{ basename }}{{ extension }}"
+    - name: SERVERLESS_THUMBNAIL_DLQ
+      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-thumbnail-dlq
+    - name: SERVERLESS_THUMBNAIL_SQS_URL
+      value: sqs://us-east-1.amazonaws.com/031107666127/space-stone-production-thumbnail/
+    - name: SERVERLESS_WORD_DLQ
+      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-word-coordinates-dlq
     - name: GOOD_JOB_QUEUES
       value: "*"
 
@@ -270,7 +580,6 @@ extraDeploy:
                   - secretRef:
                       name: {{ template "hyrax.fullname" . }}
                 env:
-                  {{- toYaml .Values.extraEnvVars | nindent 12 }}
                   {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
                 command:
                   - sh
@@ -305,7 +614,6 @@ extraDeploy:
                     {{- toYaml . | nindent 12 }}
                   {{- end }}
                 env:
-                  {{- toYaml .Values.extraEnvVars | nindent 12 }}
                   {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
                 {{- if .Values.worker.readinessProbe.enabled }}
                 readinessProbe:

From f1aa344b5342210ab104566f0364531e92ae39f1 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Wed, 14 Feb 2024 13:45:15 -0800
Subject: [PATCH 11/18] attempt to merge all vars separately

---
 ops/dev-deploy.tmpl.yaml | 312 +--------------------------------------
 1 file changed, 2 insertions(+), 310 deletions(-)

diff --git a/ops/dev-deploy.tmpl.yaml b/ops/dev-deploy.tmpl.yaml
index 191c0c5d..a27d83ea 100644
--- a/ops/dev-deploy.tmpl.yaml
+++ b/ops/dev-deploy.tmpl.yaml
@@ -211,8 +211,6 @@ extraEnvVars: &envVars
     value: sqs://us-east-1.amazonaws.com/031107666127/space-stone-production-thumbnail/
   - name: SERVERLESS_WORD_DLQ
     value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-word-coordinates-dlq
-  - name: GOOD_JOB_QUEUES
-    value: -auxiliary
 
 worker:
   replicaCount: 1
@@ -230,160 +228,7 @@ worker:
     fsGroupChangePolicy: "OnRootMismatch"
   extraVolumeMounts: *volMounts
   extraEnvVars:
-    - name: SETTINGS__MULTITENANCY__ADMIN_HOST
-      value: s2.adventistdigitallibrary.org
-    - name: SETTINGS__MULTITENANCY__ROOT_HOST
-      value: s2.adventistdigitallibrary.org
-    - name: SETTINGS__MULTITENANCY__DEFAULT_HOST
-      value: "%{tenant}.s2.adventistdigitallibrary.org"
-    - name: CONFDIR
-      value: "/app/samvera/hyrax-webapp/solr/config"
-    - name: INITIAL_ADMIN_EMAIL
-      value: support@notch8.com
-    - name: INITIAL_ADMIN_PASSWORD
-      value: testing123
-    - name: DATABASE_ADAPTER
-      value: postgresql
-    - name: DB_ADAPTER
-      value: postgresql
-    - name: DATABASE_HOST
-      value: acid-postgres-cluster-bravo.postgres.svc.cluster.local
-    - name: DB_HOST
-      value: acid-postgres-cluster-bravo.postgres.svc.cluster.local
-    - name: DATABASE_NAME
-      value: hyku-dev-hyrax
-    - name: DB_PORT
-      value: "5432"
-    - name: DATABASE_USER
-      value: hyku-dev-hyrax
-    - name: EXTERNAL_IIIF_URL
-      value: https://d3pg70bdc74ala.cloudfront.net/iiif/2
-    - name: FCREPO_BASE_PATH
-      value: /adventist-dev
-    - name: FCREPO_HOST
-      value: fcrepo.default.svc.cluster.local:8080
-    - name: FCREPO_PATH
-      value: /rest
-    - name: FEDORA_URL
-      value: http://fcrepo.default.svc.cluster.local:8080/rest
-    - name: AUXILIARY_QUEUE_PRIORITY
-      value: "100"
-    - name: IN_DOCKER
-      value: "true"
-    - name: LD_LIBRARY_PATH
-      value: /app/fits/tools/mediainfo/linux
-    - name: PASSENGER_APP_ENV
-      value: production
-    - name: RAILS_CACHE_STORE_URL
-      value: redis://:$REDIS_PASSWORD@hyku-dev-redis-master:6379/adventist
-    - name: RAILS_ENV
-      value: production
-    - name: RAILS_LOG_TO_STDOUT
-      value: "true"
-    - name: RAILS_MAX_THREADS
-      value: "5"
-    - name: RAILS_SERVE_STATIC_FILES
-      value: "true"
-    - name: REDIS_HOST
-      value: hyku-dev-redis-master
-    - name: REDIS_URL
-      value: redis://:$REDIS_PASSWORD@hyku-dev-redis-master:6379/adventist
-    - name: SETTINGS__ACTIVE_JOB__QUEUE_ADAPTER
-      value: good_job
-    - name: SETTINGS__BULKRAX__ENABLED
-      value: "true"
-    - name: SETTINGS__CONTACT_EMAIL
-      value: support@notch8.com
-    - name: SETTINGS__DEVISE__INVITATION_FROM_EMAIL
-      value: support@notch8.com
-    - name: SETTINGS__FITS_PATH
-      value: /app/fits/fits.sh
-    - name: SETTINGS__MULTITENANCY__ADMIN_ONLY_TENANT_CREATION
-      value: "true"
-    - name: SETTINGS__MULTITENANCY__DEFAULT_HOST
-      value: "%{tenant}.s2.adventistdigitallibrary.org"
-    - name: SETTINGS__MULTITENANCY__ENABLED
-      value: "true"
-    - name: SETTINGS__SOLR__COLLECTION_OPTIONS__COLLECTION__CONFIG_NAME
-      value: hyku-adventist-dev
-    - name: SETTINGS__SOLR__CONFIGSET
-      value: hyku-adventist-dev
-    - name: SETTINGS__ZOOKEEPER__CONNECTION_STR
-      value: solr-zookeeper-headless.default:2181/configs
-    - name: SOLR_ADMIN_USER
-      value: admin
-    - name: SOLR_COLLECTION_NAME
-      value: hyku-adventist-dev
-    - name: SOLR_CONFIGSET_NAME
-      value: hyku-adventist-dev
-    - name: SOLR_HOST
-      value: solr.default
-    - name: SOLR_PORT
-      value: "8983"
-    - name: SOLR_URL
-      value: http://admin:$SOLR_ADMIN_PASSWORD@solr.default:8983/solr/
-    - name: SETTINGS__SOLR__URL
-      value: http://admin:$SOLR_ADMIN_PASSWORD@solr.default:8983/solr/
-    - name: SMTP_ENABLED
-      value: "true"
-    - name: SMTP_USER_NAME
-      value: fc604e3b82d260
-    - name: SMTP_PASSWORD
-      value: $SMTP_PASSWORD
-    - name: SMTP_ADDRESS
-      value: smtp.mailtrap.io
-    - name: SMTP_DOMAIN
-      value: smtp.mailtrap.io
-    - name: SMTP_PORT
-      value: "2525"
-    - name: SMTP_TYPE
-      value: cram_md5
-    - name: SENTRY_DSN
-      value: https://5ee54fb1e8cd448a9e709c846635ec49@o1008683.ingest.sentry.io/6745020
-    - name: SENTRY_ENVIRONMENT
-      value: "staging"
-    - name: OMP_THREAD_LIMIT
-      value: "1"
-    - name: SECRET_KEY_BASE
-      value: $SECRET_KEY_BASE
-    - name: AWS_ACCESS_KEY_ID
-      value: $AWS_ACCESS_KEY_ID
-    - name: AWS_REGION
-      value: us-east-1
-    - name: AWS_S3_BUCKET
-      value: space-stone-production-preprocessedbucketf21466dd-15sun4xy658nh
-    - name: AWS_SECRET_ACCESS_KEY
-      value: $AWS_SECRET_ACCESS_KEY
-    - name: SERVERLESS_ALTO_DLQ
-      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-alto-xml-dlq
-    - name: SERVERLESS_BATCH_SIZE
-      value: "10"
-    - name: SERVERLESS_COPY_DLQ
-      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-copy-dlq
-    - name: SERVERLESS_COPY_SQS_URL
-      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-copy
-    - name: SERVERLESS_COPY_URL
-      value: https://3oqe2l9evk.execute-api.us-east-1.amazonaws.com/copy
-    - name: SERVERLESS_OCR_DLQ
-      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-ocr-dlq
-    - name: SERVERLESS_OCR_SQS_URL
-      value: sqs://us-east-1.amazonaws.com/031107666127/space-stone-production-ocr/
-    - name: SERVERLESS_OCR_THUMB_DLQ
-      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-split-ocr-thumbnail-dlq
-    - name: SERVERLESS_PLAIN_TEXT_DLQ
-      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-plain-text-dlq
-    - name: SERVERLESS_S3_URL
-      value: s3://space-stone-production-preprocessedbucketf21466dd-15sun4xy658nh.s3.us-east-1.amazonaws.com/
-    - name: SERVERLESS_SPLIT_SQS_URL
-      value: sqs://us-east-1.amazonaws.com/031107666127/space-stone-production-split-ocr-thumbnail/
-    - name: SERVERLESS_TEMPLATE
-      value: "{{dir_parts[-1..-1]}}/{{ basename }}{{ extension }}"
-    - name: SERVERLESS_THUMBNAIL_DLQ
-      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-thumbnail-dlq
-    - name: SERVERLESS_THUMBNAIL_SQS_URL
-      value: sqs://us-east-1.amazonaws.com/031107666127/space-stone-production-thumbnail/
-    - name: SERVERLESS_WORD_DLQ
-      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-word-coordinates-dlq
+    - <<: *envVars
     - name: GOOD_JOB_QUEUES
       value: -auxiliary
 
@@ -391,160 +236,7 @@ worker:
 # corresponding changes are made in the `extraDeploy` block below.
 workerAuxiliary:
   extraEnvVars:
-    - name: SETTINGS__MULTITENANCY__ADMIN_HOST
-      value: s2.adventistdigitallibrary.org
-    - name: SETTINGS__MULTITENANCY__ROOT_HOST
-      value: s2.adventistdigitallibrary.org
-    - name: SETTINGS__MULTITENANCY__DEFAULT_HOST
-      value: "%{tenant}.s2.adventistdigitallibrary.org"
-    - name: CONFDIR
-      value: "/app/samvera/hyrax-webapp/solr/config"
-    - name: INITIAL_ADMIN_EMAIL
-      value: support@notch8.com
-    - name: INITIAL_ADMIN_PASSWORD
-      value: testing123
-    - name: DATABASE_ADAPTER
-      value: postgresql
-    - name: DB_ADAPTER
-      value: postgresql
-    - name: DATABASE_HOST
-      value: acid-postgres-cluster-bravo.postgres.svc.cluster.local
-    - name: DB_HOST
-      value: acid-postgres-cluster-bravo.postgres.svc.cluster.local
-    - name: DATABASE_NAME
-      value: hyku-dev-hyrax
-    - name: DB_PORT
-      value: "5432"
-    - name: DATABASE_USER
-      value: hyku-dev-hyrax
-    - name: EXTERNAL_IIIF_URL
-      value: https://d3pg70bdc74ala.cloudfront.net/iiif/2
-    - name: FCREPO_BASE_PATH
-      value: /adventist-dev
-    - name: FCREPO_HOST
-      value: fcrepo.default.svc.cluster.local:8080
-    - name: FCREPO_PATH
-      value: /rest
-    - name: FEDORA_URL
-      value: http://fcrepo.default.svc.cluster.local:8080/rest
-    - name: AUXILIARY_QUEUE_PRIORITY
-      value: "100"
-    - name: IN_DOCKER
-      value: "true"
-    - name: LD_LIBRARY_PATH
-      value: /app/fits/tools/mediainfo/linux
-    - name: PASSENGER_APP_ENV
-      value: production
-    - name: RAILS_CACHE_STORE_URL
-      value: redis://:$REDIS_PASSWORD@hyku-dev-redis-master:6379/adventist
-    - name: RAILS_ENV
-      value: production
-    - name: RAILS_LOG_TO_STDOUT
-      value: "true"
-    - name: RAILS_MAX_THREADS
-      value: "5"
-    - name: RAILS_SERVE_STATIC_FILES
-      value: "true"
-    - name: REDIS_HOST
-      value: hyku-dev-redis-master
-    - name: REDIS_URL
-      value: redis://:$REDIS_PASSWORD@hyku-dev-redis-master:6379/adventist
-    - name: SETTINGS__ACTIVE_JOB__QUEUE_ADAPTER
-      value: good_job
-    - name: SETTINGS__BULKRAX__ENABLED
-      value: "true"
-    - name: SETTINGS__CONTACT_EMAIL
-      value: support@notch8.com
-    - name: SETTINGS__DEVISE__INVITATION_FROM_EMAIL
-      value: support@notch8.com
-    - name: SETTINGS__FITS_PATH
-      value: /app/fits/fits.sh
-    - name: SETTINGS__MULTITENANCY__ADMIN_ONLY_TENANT_CREATION
-      value: "true"
-    - name: SETTINGS__MULTITENANCY__DEFAULT_HOST
-      value: "%{tenant}.s2.adventistdigitallibrary.org"
-    - name: SETTINGS__MULTITENANCY__ENABLED
-      value: "true"
-    - name: SETTINGS__SOLR__COLLECTION_OPTIONS__COLLECTION__CONFIG_NAME
-      value: hyku-adventist-dev
-    - name: SETTINGS__SOLR__CONFIGSET
-      value: hyku-adventist-dev
-    - name: SETTINGS__ZOOKEEPER__CONNECTION_STR
-      value: solr-zookeeper-headless.default:2181/configs
-    - name: SOLR_ADMIN_USER
-      value: admin
-    - name: SOLR_COLLECTION_NAME
-      value: hyku-adventist-dev
-    - name: SOLR_CONFIGSET_NAME
-      value: hyku-adventist-dev
-    - name: SOLR_HOST
-      value: solr.default
-    - name: SOLR_PORT
-      value: "8983"
-    - name: SOLR_URL
-      value: http://admin:$SOLR_ADMIN_PASSWORD@solr.default:8983/solr/
-    - name: SETTINGS__SOLR__URL
-      value: http://admin:$SOLR_ADMIN_PASSWORD@solr.default:8983/solr/
-    - name: SMTP_ENABLED
-      value: "true"
-    - name: SMTP_USER_NAME
-      value: fc604e3b82d260
-    - name: SMTP_PASSWORD
-      value: $SMTP_PASSWORD
-    - name: SMTP_ADDRESS
-      value: smtp.mailtrap.io
-    - name: SMTP_DOMAIN
-      value: smtp.mailtrap.io
-    - name: SMTP_PORT
-      value: "2525"
-    - name: SMTP_TYPE
-      value: cram_md5
-    - name: SENTRY_DSN
-      value: https://5ee54fb1e8cd448a9e709c846635ec49@o1008683.ingest.sentry.io/6745020
-    - name: SENTRY_ENVIRONMENT
-      value: "staging"
-    - name: OMP_THREAD_LIMIT
-      value: "1"
-    - name: SECRET_KEY_BASE
-      value: $SECRET_KEY_BASE
-    - name: AWS_ACCESS_KEY_ID
-      value: $AWS_ACCESS_KEY_ID
-    - name: AWS_REGION
-      value: us-east-1
-    - name: AWS_S3_BUCKET
-      value: space-stone-production-preprocessedbucketf21466dd-15sun4xy658nh
-    - name: AWS_SECRET_ACCESS_KEY
-      value: $AWS_SECRET_ACCESS_KEY
-    - name: SERVERLESS_ALTO_DLQ
-      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-alto-xml-dlq
-    - name: SERVERLESS_BATCH_SIZE
-      value: "10"
-    - name: SERVERLESS_COPY_DLQ
-      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-copy-dlq
-    - name: SERVERLESS_COPY_SQS_URL
-      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-copy
-    - name: SERVERLESS_COPY_URL
-      value: https://3oqe2l9evk.execute-api.us-east-1.amazonaws.com/copy
-    - name: SERVERLESS_OCR_DLQ
-      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-ocr-dlq
-    - name: SERVERLESS_OCR_SQS_URL
-      value: sqs://us-east-1.amazonaws.com/031107666127/space-stone-production-ocr/
-    - name: SERVERLESS_OCR_THUMB_DLQ
-      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-split-ocr-thumbnail-dlq
-    - name: SERVERLESS_PLAIN_TEXT_DLQ
-      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-plain-text-dlq
-    - name: SERVERLESS_S3_URL
-      value: s3://space-stone-production-preprocessedbucketf21466dd-15sun4xy658nh.s3.us-east-1.amazonaws.com/
-    - name: SERVERLESS_SPLIT_SQS_URL
-      value: sqs://us-east-1.amazonaws.com/031107666127/space-stone-production-split-ocr-thumbnail/
-    - name: SERVERLESS_TEMPLATE
-      value: "{{dir_parts[-1..-1]}}/{{ basename }}{{ extension }}"
-    - name: SERVERLESS_THUMBNAIL_DLQ
-      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-thumbnail-dlq
-    - name: SERVERLESS_THUMBNAIL_SQS_URL
-      value: sqs://us-east-1.amazonaws.com/031107666127/space-stone-production-thumbnail/
-    - name: SERVERLESS_WORD_DLQ
-      value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-word-coordinates-dlq
+    - <<: *envVars
     - name: GOOD_JOB_QUEUES
       value: "*"
 

From e769ef2312fa447784d32a86e4704fc018764eac Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Thu, 15 Feb 2024 15:17:15 -0800
Subject: [PATCH 12/18] attempt using templ to merge ENV var lists

---
 ops/dev-deploy.tmpl.yaml | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/ops/dev-deploy.tmpl.yaml b/ops/dev-deploy.tmpl.yaml
index a27d83ea..44174ee9 100644
--- a/ops/dev-deploy.tmpl.yaml
+++ b/ops/dev-deploy.tmpl.yaml
@@ -212,6 +212,18 @@ extraEnvVars: &envVars
   - name: SERVERLESS_WORD_DLQ
     value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-word-coordinates-dlq
 
+{{- define "adventist.workerEnvVars" -}}
+{{- toYaml .Values.extraEnvVars -}}
+- name: GOOD_JOB_QUEUES
+  value: -auxiliary
+{{- end -}}
+
+{{- define "adventist.workerAuxiliaryEnvVars" -}}
+{{- toYaml .Values.extraEnvVars -}}
+- name: GOOD_JOB_QUEUES
+  value: "*"
+{{- end -}}
+
 worker:
   replicaCount: 1
   resources:
@@ -228,17 +240,13 @@ worker:
     fsGroupChangePolicy: "OnRootMismatch"
   extraVolumeMounts: *volMounts
   extraEnvVars:
-    - <<: *envVars
-    - name: GOOD_JOB_QUEUES
-      value: -auxiliary
+    {{ include "adventist.workerEnvVars" . | nindent 4 }}
 
 # When adding/removing key-value pairs to this block, ensure the
 # corresponding changes are made in the `extraDeploy` block below.
 workerAuxiliary:
   extraEnvVars:
-    - <<: *envVars
-    - name: GOOD_JOB_QUEUES
-      value: "*"
+    {{ include "adventist.workerAuxiliaryEnvVars" . | nindent 4 }}
 
 extraDeploy:
   - |-

From a1fb7759dbb32007690aca1e5f6b0e64c07a8fb4 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Tue, 20 Feb 2024 11:51:46 -0800
Subject: [PATCH 13/18] use unique var for auxiliary worker

Attempting to merge/override the same ENV variable in different
deployments ended up being very difficult. This is perhaps less elegant,
but simpler
---
 bin/worker               |  6 +++++-
 ops/dev-deploy.tmpl.yaml | 20 +++++---------------
 2 files changed, 10 insertions(+), 16 deletions(-)

diff --git a/bin/worker b/bin/worker
index edfb2576..a81d5cca 100755
--- a/bin/worker
+++ b/bin/worker
@@ -9,4 +9,8 @@ else
   puts 'DATABASE_URL not set, no pool change needed'
 end
 
-exec "echo $DATABASE_URL && bundle exec good_job start"
\ No newline at end of file
+if ENV['AUX_WORKER']
+  exec "echo $DATABASE_URL && bundle exec good_job start --queues=*"
+else
+  exec "echo $DATABASE_URL && bundle exec good_job start --queues=-auxiliary"
+end
diff --git a/ops/dev-deploy.tmpl.yaml b/ops/dev-deploy.tmpl.yaml
index 44174ee9..508b1a73 100644
--- a/ops/dev-deploy.tmpl.yaml
+++ b/ops/dev-deploy.tmpl.yaml
@@ -212,18 +212,6 @@ extraEnvVars: &envVars
   - name: SERVERLESS_WORD_DLQ
     value: https://sqs.us-east-1.amazonaws.com/031107666127/space-stone-production-word-coordinates-dlq
 
-{{- define "adventist.workerEnvVars" -}}
-{{- toYaml .Values.extraEnvVars -}}
-- name: GOOD_JOB_QUEUES
-  value: -auxiliary
-{{- end -}}
-
-{{- define "adventist.workerAuxiliaryEnvVars" -}}
-{{- toYaml .Values.extraEnvVars -}}
-- name: GOOD_JOB_QUEUES
-  value: "*"
-{{- end -}}
-
 worker:
   replicaCount: 1
   resources:
@@ -239,14 +227,14 @@ worker:
     fsGroup: 101
     fsGroupChangePolicy: "OnRootMismatch"
   extraVolumeMounts: *volMounts
-  extraEnvVars:
-    {{ include "adventist.workerEnvVars" . | nindent 4 }}
+  extraEnvVars: *envVars
 
 # When adding/removing key-value pairs to this block, ensure the
 # corresponding changes are made in the `extraDeploy` block below.
 workerAuxiliary:
   extraEnvVars:
-    {{ include "adventist.workerAuxiliaryEnvVars" . | nindent 4 }}
+    - name: AUX_WORKER
+      value: "true"
 
 extraDeploy:
   - |-
@@ -280,6 +268,7 @@ extraDeploy:
                   - secretRef:
                       name: {{ template "hyrax.fullname" . }}
                 env:
+                  {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
                   {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
                 command:
                   - sh
@@ -314,6 +303,7 @@ extraDeploy:
                     {{- toYaml . | nindent 12 }}
                   {{- end }}
                 env:
+                  {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
                   {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
                 {{- if .Values.worker.readinessProbe.enabled }}
                 readinessProbe:

From d984f6daecd038479c5de859e21eb98b227bad8a Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Tue, 20 Feb 2024 14:53:56 -0800
Subject: [PATCH 14/18] fix good_job command in workers

---
 bin/worker | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/worker b/bin/worker
index a81d5cca..43bfebbe 100755
--- a/bin/worker
+++ b/bin/worker
@@ -10,7 +10,7 @@ else
 end
 
 if ENV['AUX_WORKER']
-  exec "echo $DATABASE_URL && bundle exec good_job start --queues=*"
+  exec "echo $DATABASE_URL && bundle exec good_job --queues='*'"
 else
-  exec "echo $DATABASE_URL && bundle exec good_job start --queues=-auxiliary"
+  exec "echo $DATABASE_URL && bundle exec good_job --queues='-auxiliary'"
 end

From c8fbed6e16a2d1c9bb42ac6a685bebe61f12991d Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Tue, 20 Feb 2024 16:57:55 -0800
Subject: [PATCH 15/18] use ENV variables to configure worker queues

Attempting to use the --queues flag didn't work, but using the ENV
variable does
---
 bin/worker               | 8 ++++++--
 ops/dev-deploy.tmpl.yaml | 3 ++-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/bin/worker b/bin/worker
index 43bfebbe..5c95ab54 100755
--- a/bin/worker
+++ b/bin/worker
@@ -10,7 +10,11 @@ else
 end
 
 if ENV['AUX_WORKER']
-  exec "echo $DATABASE_URL && bundle exec good_job --queues='*'"
+  # Run all queues (including :auxiliary)
+  exec "export GOOD_JOB_QUEUES='*'"
+  exec "echo $DATABASE_URL && bundle exec good_job"
 else
-  exec "echo $DATABASE_URL && bundle exec good_job --queues='-auxiliary'"
+  # Run all queues (excluding :auxiliary)
+  exec "export GOOD_JOB_QUEUES='-auxiliary'"
+  exec "echo $DATABASE_URL && bundle exec good_job"
 end
diff --git a/ops/dev-deploy.tmpl.yaml b/ops/dev-deploy.tmpl.yaml
index 508b1a73..64dcc8c2 100644
--- a/ops/dev-deploy.tmpl.yaml
+++ b/ops/dev-deploy.tmpl.yaml
@@ -227,12 +227,13 @@ worker:
     fsGroup: 101
     fsGroupChangePolicy: "OnRootMismatch"
   extraVolumeMounts: *volMounts
-  extraEnvVars: *envVars
+  extraEnvVars: *envVars # GOOD_JOB_QUEUES gets set in bin/worker
 
 # When adding/removing key-value pairs to this block, ensure the
 # corresponding changes are made in the `extraDeploy` block below.
 workerAuxiliary:
   extraEnvVars:
+    # GOOD_JOB_QUEUES gets set in bin/worker
     - name: AUX_WORKER
       value: "true"
 

From 9ffd497e65f8722d3e88399b586985bbae32fd29 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Wed, 21 Feb 2024 15:56:24 -0800
Subject: [PATCH 16/18] two exec calls doesn't work

---
 bin/worker | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/worker b/bin/worker
index 5c95ab54..d57cf13d 100755
--- a/bin/worker
+++ b/bin/worker
@@ -11,10 +11,10 @@ end
 
 if ENV['AUX_WORKER']
   # Run all queues (including :auxiliary)
-  exec "export GOOD_JOB_QUEUES='*'"
+  %x{export GOOD_JOB_QUEUES='*'}
   exec "echo $DATABASE_URL && bundle exec good_job"
 else
   # Run all queues (excluding :auxiliary)
-  exec "export GOOD_JOB_QUEUES='-auxiliary'"
+  %x{export GOOD_JOB_QUEUES='-auxiliary'}
   exec "echo $DATABASE_URL && bundle exec good_job"
 end

From c3e302863a6e6e22aa42448a02ccaeb416ca0e9d Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Wed, 21 Feb 2024 17:24:27 -0800
Subject: [PATCH 17/18] export didn't reliably set ENV var

---
 bin/worker | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/bin/worker b/bin/worker
index d57cf13d..e605849a 100755
--- a/bin/worker
+++ b/bin/worker
@@ -11,10 +11,8 @@ end
 
 if ENV['AUX_WORKER']
   # Run all queues (including :auxiliary)
-  %x{export GOOD_JOB_QUEUES='*'}
-  exec "echo $DATABASE_URL && bundle exec good_job"
+  exec "echo $DATABASE_URL && GOOD_JOB_QUEUES='*' bundle exec good_job"
 else
   # Run all queues (excluding :auxiliary)
-  %x{export GOOD_JOB_QUEUES='-auxiliary'}
-  exec "echo $DATABASE_URL && bundle exec good_job"
+  exec "echo $DATABASE_URL && GOOD_JOB_QUEUES='-auxiliary' bundle exec good_job"
 end

From 0d2340638fe7ba74d0439cfa27b5a908f109a4a8 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Thu, 22 Feb 2024 10:32:21 -0800
Subject: [PATCH 18/18] auxiliary is hard to spell

---
 .env                     | 1 -
 docker-compose.yml       | 6 +++---
 ops/dev-deploy.tmpl.yaml | 8 ++++----
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/.env b/.env
index d190bec1..3b25d820 100644
--- a/.env
+++ b/.env
@@ -16,7 +16,6 @@ FITS_SERVLET_URL=http://fits:8080/fits
 GOOD_JOB_CLEANUP_DISCARDED_JOBS=false
 GOOD_JOB_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO=604800
 GOOD_JOB_CLEANUP_INTERVAL_SECONDS=86400
-GOOD_JOB_QUEUES=-auxiliary
 INITIAL_ADMIN_EMAIL=admin@example.com
 INITIAL_ADMIN_PASSWORD=testing123
 IN_DOCKER=true
diff --git a/docker-compose.yml b/docker-compose.yml
index 28dea7ab..641f3254 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -210,7 +210,7 @@ services:
         condition: service_started
       worker:
         condition: service_started
-      worker_auxiliary:
+      worker_aux:
         condition: service_started
       initialize_app:
         condition: service_completed_successfully
@@ -236,10 +236,10 @@ services:
     ## breakpoints.
     # command: sh -l -c "bundle && tail -f /dev/null"
 
-  worker_auxiliary:
+  worker_aux:
     <<: *app-worker
     environment:
-      - GOOD_JOB_QUEUES=*
+      - AUX_WORKER="true"
 
   # Do not recurse through all of tmp. derivitives will make booting
   # very slow and eventually just time out as data grows
diff --git a/ops/dev-deploy.tmpl.yaml b/ops/dev-deploy.tmpl.yaml
index 64dcc8c2..c07906ff 100644
--- a/ops/dev-deploy.tmpl.yaml
+++ b/ops/dev-deploy.tmpl.yaml
@@ -231,7 +231,7 @@ worker:
 
 # When adding/removing key-value pairs to this block, ensure the
 # corresponding changes are made in the `extraDeploy` block below.
-workerAuxiliary:
+workerAux:
   extraEnvVars:
     # GOOD_JOB_QUEUES gets set in bin/worker
     - name: AUX_WORKER
@@ -242,7 +242,7 @@ extraDeploy:
       apiVersion: apps/v1
       kind: Deployment
       metadata:
-        name: {{ include "hyrax.fullname" . }}-auxiliary-worker
+        name: {{ include "hyrax.fullname" . }}-worker-aux
         labels:
           {{- include "hyrax.labels" . | nindent 4 }}
       spec:
@@ -270,7 +270,7 @@ extraDeploy:
                       name: {{ template "hyrax.fullname" . }}
                 env:
                   {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
-                  {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
+                  {{- toYaml .Values.workerAux.extraEnvVars | nindent 12 }}
                 command:
                   - sh
                   - -c
@@ -305,7 +305,7 @@ extraDeploy:
                   {{- end }}
                 env:
                   {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
-                  {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
+                  {{- toYaml .Values.workerAux.extraEnvVars | nindent 12 }}
                 {{- if .Values.worker.readinessProbe.enabled }}
                 readinessProbe:
                   exec: