diff --git a/.env b/.env index 17dc7b76..3b25d820 100644 --- a/.env +++ b/.env @@ -1,3 +1,4 @@ +AUXILIARY_QUEUE_PRIORITY=100 CHROME_HOSTNAME=chrome DATABASE_ADAPTER=postgresql DATABASE_HOST=db diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb index 70d71689..459f4f20 100644 --- a/app/jobs/application_job.rb +++ b/app/jobs/application_job.rb @@ -1,8 +1,18 @@ # frozen_string_literal: true class ApplicationJob < ActiveJob::Base + before_enqueue :set_auxiliary_queue_priority + # limit to 5 attempts retry_on StandardError, wait: :exponentially_longer, attempts: 5 do |_job, _exception| # Log error, do nothing, etc. end + + private + + def set_auxiliary_queue_priority + return unless queue_name.to_sym == :auxiliary + + self.priority = ENV.fetch('AUXILIARY_QUEUE_PRIORITY', 100).to_i + end end diff --git a/bin/worker b/bin/worker index edfb2576..e605849a 100755 --- a/bin/worker +++ b/bin/worker @@ -9,4 +9,10 @@ else puts 'DATABASE_URL not set, no pool change needed' end -exec "echo $DATABASE_URL && bundle exec good_job start" \ No newline at end of file +if ENV['AUX_WORKER'] + # Run all queues (including :auxiliary) + exec "echo $DATABASE_URL && GOOD_JOB_QUEUES='*' bundle exec good_job" +else + # Run all queues (excluding :auxiliary) + exec "echo $DATABASE_URL && GOOD_JOB_QUEUES='-auxiliary' bundle exec good_job" +end diff --git a/docker-compose.yml b/docker-compose.yml index b7a6d1d5..641f3254 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,36 @@ x-app: &app networks: internal: +x-app-worker: &app-worker + <<: *app + build: + context: . + target: hyku-worker + args: + - SETTINGS__BULKRAX__ENABLED=true + cache_from: + - ghcr.io/scientist-softserv/adventist-dl/base:${TAG:-latest} + - ghcr.io/scientist-softserv/adventist-dl:${TAG:-latest} + - ghcr.io/scientist-softserv/adventist-dl/worker:${TAG:-latest} + image: ghcr.io/scientist-softserv/adventist-dl/worker:${TAG:-latest} + depends_on: + db: + condition: service_started + solr: + condition: service_started + fcrepo: + condition: service_started + redis: + condition: service_started + fits: + condition: service_started + zoo: + condition: service_started + check_volumes: + condition: service_completed_successfully + initialize_app: + condition: service_completed_successfully + volumes: assets: cache: @@ -180,13 +210,15 @@ services: condition: service_started worker: condition: service_started + worker_aux: + condition: service_started initialize_app: condition: service_completed_successfully expose: - 3000 worker: - <<: *app + <<: *app-worker ################################################################################ ## Note on commands: by default the commands don't run bundle. That is to ## reduce boot times. However, when the application is in active @@ -203,33 +235,11 @@ services: ## in your code, bash into the worker container, and interact with the ## breakpoints. # command: sh -l -c "bundle && tail -f /dev/null" - build: - context: . - target: hyku-worker - args: - - SETTINGS__BULKRAX__ENABLED=true - cache_from: - - ghcr.io/scientist-softserv/adventist-dl/base:${TAG:-latest} - - ghcr.io/scientist-softserv/adventist-dl:${TAG:-latest} - - ghcr.io/scientist-softserv/adventist-dl/worker:${TAG:-latest} - image: ghcr.io/scientist-softserv/adventist-dl/worker:${TAG:-latest} - depends_on: - db: - condition: service_started - solr: - condition: service_started - fcrepo: - condition: service_started - redis: - condition: service_started - fits: - condition: service_started - zoo: - condition: service_started - check_volumes: - condition: service_completed_successfully - initialize_app: - condition: service_completed_successfully + + worker_aux: + <<: *app-worker + environment: + - AUX_WORKER="true" # Do not recurse through all of tmp. derivitives will make booting # very slow and eventually just time out as data grows diff --git a/ops/dev-deploy.tmpl.yaml b/ops/dev-deploy.tmpl.yaml index 1f17a2d2..c07906ff 100644 --- a/ops/dev-deploy.tmpl.yaml +++ b/ops/dev-deploy.tmpl.yaml @@ -46,11 +46,10 @@ ingress: - host: "*.s2.adventistdigitallibrary.org" paths: - path: / - annotations: { - kubernetes.io/ingress.class: "nginx", - nginx.ingress.kubernetes.io/proxy-body-size: "0", - cert-manager.io/cluster-issuer: wildcard-issuer - } + annotations: + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/proxy-body-size: "0" + cert-manager.io/cluster-issuer: "wildcard-issuer" tls: - hosts: - '*.s2.adventistdigitallibrary.org' @@ -94,6 +93,8 @@ extraEnvVars: &envVars value: /rest - name: FEDORA_URL value: http://fcrepo.default.svc.cluster.local:8080/rest + - name: AUXILIARY_QUEUE_PRIORITY + value: "100" - name: IN_DOCKER value: "true" - name: LD_LIBRARY_PATH @@ -226,7 +227,158 @@ worker: fsGroup: 101 fsGroupChangePolicy: "OnRootMismatch" extraVolumeMounts: *volMounts - extraEnvVars: *envVars + extraEnvVars: *envVars # GOOD_JOB_QUEUES gets set in bin/worker + +# When adding/removing key-value pairs to this block, ensure the +# corresponding changes are made in the `extraDeploy` block below. +workerAux: + extraEnvVars: + # GOOD_JOB_QUEUES gets set in bin/worker + - name: AUX_WORKER + value: "true" + +extraDeploy: + - |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: {{ include "hyrax.fullname" . }}-worker-aux + labels: + {{- include "hyrax.labels" . | nindent 4 }} + spec: + replicas: {{ .Values.worker.replicaCount }} + selector: + matchLabels: + {{- include "hyrax.workerSelectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "hyrax.workerSelectorLabels" . | nindent 8 }} + spec: + initContainers: + - name: db-wait + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + envFrom: + - configMapRef: + name: {{ include "hyrax.fullname" . }}-env + - secretRef: + name: {{ template "hyrax.fullname" . }} + env: + {{- toYaml .Values.worker.extraEnvVars | nindent 12 }} + {{- toYaml .Values.workerAux.extraEnvVars | nindent 12 }} + command: + - sh + - -c + - "db-wait.sh {{ include "hyrax.redis.host" . }}:6379" + {{- if .Values.worker.extraInitContainers }} + {{- toYaml .Values.worker.extraInitContainers | nindent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "hyrax.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.worker.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }}-worker + securityContext: + {{- toYaml .Values.worker.securityContext | nindent 12 }} + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + envFrom: + - configMapRef: + name: {{ include "hyrax.fullname" . }}-env + - secretRef: + name: {{ template "hyrax.fullname" . }} + {{- if .Values.solrExistingSecret }} + - secretRef: + name: {{ .Values.solrExistingSecret }} + {{- end }} + {{- with .Values.worker.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- toYaml .Values.worker.extraEnvVars | nindent 12 }} + {{- toYaml .Values.workerAux.extraEnvVars | nindent 12 }} + {{- if .Values.worker.readinessProbe.enabled }} + readinessProbe: + exec: + command: + {{- toYaml .Values.worker.readinessProbe.command | nindent 16 }} + failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }} + initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.worker.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }} + {{- end }} + volumeMounts: + - name: derivatives + mountPath: /app/samvera/derivatives + - name: uploads + subPath: file_cache + mountPath: /app/samvera/file_cache + - name: uploads + subPath: uploads + mountPath: /app/samvera/uploads + {{- if .Values.applicationExistingClaim }} + - name: application + mountPath: /app/samvera/hyrax-webapp + {{- end }} + {{- with .Values.worker.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.worker.resources | nindent 12 }} + {{- with .Values.extraContainerConfiguration }} + {{- toYaml . | nindent 10 }} + {{- end }} + volumes: + - name: "derivatives" + {{- if and .Values.derivativesVolume.enabled .Values.derivativesVolume.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.derivativesVolume.existingClaim }} + {{- else if .Values.derivativesVolume.enabled }} + persistentVolumeClaim: + claimName: {{ template "hyrax.fullname" . }}-derivatives + {{ else }} + emptyDir: {} + {{ end }} + - name: "uploads" + {{- if and .Values.uploadsVolume.enabled .Values.uploadsVolume.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.uploadsVolume.existingClaim }} + {{- else if .Values.uploadsVolume.enabled }} + persistentVolumeClaim: + claimName: {{ template "hyrax.fullname" . }}-uploads + {{ else }} + emptyDir: {} + {{ end }} + {{- if .Values.applicationExistingClaim }} + - name: "application" + persistentVolumeClaim: + claimName: {{ .Values.applicationExistingClaim }} + {{- end }} + {{- with .Values.worker.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} podSecurityContext: runAsUser: 1001