From c06e10632eb3cde8a56e37414ed210396a38d834 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Thu, 19 Sep 2024 22:15:41 -0700 Subject: [PATCH] bump hyku/hyrax and add draft demo deploy --- hyrax-webapp | 2 +- ops/demo-deploy.tmpl.yaml | 552 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 553 insertions(+), 1 deletion(-) create mode 100644 ops/demo-deploy.tmpl.yaml diff --git a/hyrax-webapp b/hyrax-webapp index ff02adb..84c3daa 160000 --- a/hyrax-webapp +++ b/hyrax-webapp @@ -1 +1 @@ -Subproject commit ff02adb6066e57fb51dd8aa43e42f9243c2ac89e +Subproject commit 84c3daa36d2aad03dad6a92af06ff99078e465e3 diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml new file mode 100644 index 0000000..fea51e3 --- /dev/null +++ b/ops/demo-deploy.tmpl.yaml @@ -0,0 +1,552 @@ +replicaCount: 2 + +resources: + limits: + memory: "4Gi" + cpu: "600m" + requests: + memory: "2Gi" + cpu: "300m" + +livenessProbe: + enabled: false +readinessProbe: + enabled: true + path: "/healthz" + periodSeconds: 30 + timeoutSeconds: 10 + +brandingVolume: + storageClass: efs-sc +derivativesVolume: + storageClass: efs-sc +uploadsVolume: + storageClass: efs-sc + size: 200Gi + +imagePullSecrets: + - name: github + +extraVolumeMounts: &volMounts + - name: uploads + mountPath: /app/samvera/hyrax-webapp/tmp/imports + subPath: imports + - name: uploads + mountPath: /app/samvera/hyrax-webapp/tmp/exports + subPath: exports + - name: uploads + mountPath: /app/samvera/hyrax-webapp/public/system + subPath: public-system + - name: uploads + mountPath: /app/samvera/hyrax-webapp/public/uploads + subPath: public-uploads + - name: uploads + mountPath: /app/samvera/hyrax-webapp/tmp/network_files + subPath: network-files + - name: uploads + mountPath: /app/samvera/hyrax-webapp/storage/files + subPath: storage-files + +ingress: + enabled: true + hosts: + - host: commons-archive.org + paths: + - path: / + pathType: ImplementationSpecific + - host: "*.commons-archive.org" + paths: + - path: / + pathType: ImplementationSpecific + annotations: { + kubernetes.io/ingress.class: "nginx", + nginx.ingress.kubernetes.io/proxy-body-size: "0", + cert-manager.io/cluster-issuer: letsencrypt-production-dns + } + tls: + - hosts: + - commons-archive.org + - "*.commons-archive.org" + secretName: demo-palni-palci-tls + +extraEnvVars: &envVars + - name: BUNDLE_DISABLE_LOCAL_BRANCH_CHECK + value: "true" + - name: BUNDLE_LOCAL__HYKU_KNAPSACK + value: /app/samvera + - name: CLIENT_ADMIN_USER_EMAIL + value: $CLIENT_ADMIN_USER_EMAIL + - name: CLIENT_ADMIN_USER_PASSWORD + value: $CLIENT_ADMIN_USER_PASSWORD + - name: CONFDIR + value: "/app/samvera/hyrax-webapp/solr/conf" + - name: DB_ADAPTER + value: postgresql + - name: DB_HOST + value: acid-postgres-cluster-delta.postgres.svc.cluster.local + - name: DB_NAME + value: pals_demo + - name: DB_USER + value: postgres + - name: FCREPO_BASE_PATH + value: /demo-palni-palci + - name: FCREPO_HOST + value: fcrepo.fcrepo.svc.cluster.local + - name: FCREPO_PORT + value: "8080" + - name: FCREPO_REST_PATH + value: rest + - name: GOOGLE_ACCOUNT_JSON + value: $GOOGLE_ACCOUNT_JSON + - name: GOOGLE_ANALYTICS_ID + value: $GOOGLE_ANALYTICS_ID + - name: GOOGLE_ANALYTICS_PROPERTY_ID + value: "$GOOGLE_ANALYTICS_PROPERTY_ID" + - name: GOOGLE_FONTS_KEY + value: $GOOGLE_FONTS_KEY + - name: HYKU_ADMIN_HOST + value: commons-archive.org + - name: HYKU_ADMIN_ONLY_TENANT_CREATION + value: "false" + - name: HYKU_ALLOW_SIGNUP + value: "false" + - name: HYKU_BLOCK_VALKYRIE_REDIRECT + value: "false" + - name: HYKU_BULKRAX_ENABLED + value: "true" + - name: HYKU_CONTACT_EMAIL + value: support@notch8.com + - name: HYKU_CROSS_TENANT_SEARCH_HOST + value: search.commons-archive.org + - name: HYKU_DEFAULT_HOST + value: "%{tenant}.commons-archive.org" + - name: HYKU_FILE_ACL + value: "false" + - name: HYKU_MULTITENANT + value: "true" + - name: HYKU_RESTRICT_CREATE_AND_DESTROY_PERMISSIONS + value: "true" + - name: HYKU_ROOT_HOST + value: commons-archive.org + - name: HYKU_USER_DEFAULT_PASSWORD + value: password + - name: HYRAX_ACTIVE_JOB_QUEUE + value: sidekiq + - name: HYRAX_ANALYTICS + value: "true" + - name: HYRAX_ANALYTICS_PROVIDER + value: 'ga4' + - name: HYRAX_FITS_PATH + value: /app/fits/fits.sh + - name: HYRAX_VALKYRIE + value: "true" + - name: INITIAL_ADMIN_EMAIL + value: admin@example.com + - name: INITIAL_ADMIN_PASSWORD + value: testing123 + - name: IN_DOCKER + value: "true" + - name: LD_LIBRARY_PATH + value: /app/fits/tools/mediainfo/linux + - name: NEGATIVE_CAPTCHA_SECRET + value: $NEGATIVE_CAPTCHA_SECRET + - name: RAILS_CACHE_STORE_URL + value: redis://:$REDIS_PASSWORD@palni-palci-knapsack-demo-redis-master:6379/0 + - name: RAILS_ENV + value: production + - name: RAILS_LOG_TO_STDOUT + value: "true" + - name: RAILS_MAX_THREADS + value: "5" + - name: RAILS_SERVE_STATIC_FILES + value: "true" + - name: REDIS_HOST + value: palni-palci-knapsack-demo-redis-master + - name: REDIS_URL + value: redis://:$REDIS_PASSWORD@palni-palci-knapsack-demo-redis-master:6379/0 + - name: REPOSITORY_S3_STORAGE + value: true + - name: REPOSITORY_S3_BUCKET + value: palni-palci-demo + - name: REPOSITORY_S3_REGION + value: us-west-2 + - name: REPOSITORY_S3_ACCESS_KEY + value: $AWS_ACCESS_KEY_ID + - name: REPOSITORY_S3_SECRET_KEY + value: $AWS_SECRET_ACCESS_KEY + - name: SENTRY_DSN + value: $SENTRY_DSN + - name: SENTRY_ENVIRONMENT + value: $SENTRY_ENVIRONMENT + - name: SMTP_ADDRESS + value: "maildev-smtp.maildev.svc.cluster.local" + - name: SMTP_DOMAIN + value: "maildev-smtp.maildev.svc.cluster.local" + - name: SMTP_ENABLED + value: "true" + - name: SMTP_PASSWORD + value: $SMTP_PASSWORD + - name: SMTP_PORT + value: "1025" + - name: SMTP_STARTTLS + value: "false" + - name: SMTP_TYPE + value: "plain" + - name: SMTP_USER_NAME + value: "admin" + - name: SOLR_ADMIN_PASSWORD + value: $SOLR_ADMIN_PASSWORD + - name: SOLR_ADMIN_USER + value: admin + - name: SOLR_COLLECTION_NAME + value: demo-palni-palci + - name: SOLR_CONFIGSET_NAME + value: demo-palni-palci + - name: SOLR_HOST + value: solr-headless.solr + - name: SOLR_PORT + value: "8983" + - name: SOLR_URL + value: http://admin:$SOLR_ADMIN_PASSWORD@solr-headless.solr:8983/solr/ + - name: TEST_USER_EMAIL + value: user@example.com + - name: TEST_USER_PASSWORD + value: testing123 + - name: VALKYRIE_ID_TYPE + value: string + - name: VALKYRIE_TRANSITION + value: "true" + +worker: + replicaCount: 1 + resources: + limits: + memory: "4Gi" + cpu: "300m" + requests: + memory: "2Gi" + cpu: "150m" + extraVolumeMounts: *volMounts + extraEnvVars: *envVars + podSecurityContext: + runAsUser: 1001 + runAsGroup: 101 + fsGroup: 101 + fsGroupChangePolicy: "OnRootMismatch" +podSecurityContext: + runAsUser: 1001 + runAsGroup: 101 + fsGroup: 101 + fsGroupChangePolicy: "OnRootMismatch" + +# When adding/removing key-value pairs to this block, ensure the +# corresponding changes are made in the `extraDeploy` block below. +workerAuxiliary: + replicaCount: 1 + resources: + limits: + memory: "16Gi" + cpu: "4" + requests: + memory: "8Gi" + cpu: "2" + extraEnvVars: + - name: SIDEKIQ_CONFIG + value: "config/sidekiq_resource_intensive.yml" + +extraDeploy: + - |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: {{ include "hyrax.fullname" . }}-auxiliary-worker + labels: + {{- include "hyrax.labels" . | nindent 4 }} + spec: + replicas: {{ .Values.workerAuxiliary.replicaCount }} + selector: + matchLabels: + {{- include "hyrax.workerSelectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "hyrax.workerSelectorLabels" . | nindent 8 }} + spec: + initContainers: + - name: db-wait + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + envFrom: + - configMapRef: + name: {{ include "hyrax.fullname" . }}-env + - secretRef: + name: {{ template "hyrax.fullname" . }} + env: + {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }} + {{- toYaml .Values.worker.extraEnvVars | nindent 12 }} + command: + - sh + - -c + - "db-wait.sh {{ include "hyrax.redis.host" . }}:6379" + {{- if .Values.worker.extraInitContainers }} + {{- toYaml .Values.worker.extraInitContainers | nindent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "hyrax.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.worker.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }}-worker + securityContext: + {{- toYaml .Values.worker.securityContext | nindent 12 }} + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + envFrom: + - configMapRef: + name: {{ include "hyrax.fullname" . }}-env + - secretRef: + name: {{ template "hyrax.fullname" . }} + {{- if .Values.solrExistingSecret }} + - secretRef: + name: {{ .Values.solrExistingSecret }} + {{- end }} + {{- with .Values.worker.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }} + {{- toYaml .Values.worker.extraEnvVars | nindent 12 }} + {{- if .Values.worker.readinessProbe.enabled }} + readinessProbe: + exec: + command: + {{- toYaml .Values.worker.readinessProbe.command | nindent 16 }} + failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }} + initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.worker.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }} + {{- end }} + volumeMounts: + - name: derivatives + mountPath: /app/samvera/derivatives + - name: uploads + subPath: file_cache + mountPath: /app/samvera/file_cache + - name: uploads + subPath: uploads + mountPath: /app/samvera/uploads + {{- if .Values.applicationExistingClaim }} + - name: application + mountPath: /app/samvera/hyrax-webapp + {{- end }} + {{- with .Values.worker.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.workerAuxiliary.resources | nindent 12 }} + {{- with .Values.extraContainerConfiguration }} + {{- toYaml . | nindent 10 }} + {{- end }} + volumes: + - name: "derivatives" + {{- if and .Values.derivativesVolume.enabled .Values.derivativesVolume.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.derivativesVolume.existingClaim }} + {{- else if .Values.derivativesVolume.enabled }} + persistentVolumeClaim: + claimName: {{ template "hyrax.fullname" . }}-derivatives + {{ else }} + emptyDir: {} + {{ end }} + - name: "uploads" + {{- if and .Values.uploadsVolume.enabled .Values.uploadsVolume.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.uploadsVolume.existingClaim }} + {{- else if .Values.uploadsVolume.enabled }} + persistentVolumeClaim: + claimName: {{ template "hyrax.fullname" . }}-uploads + {{ else }} + emptyDir: {} + {{ end }} + {{- if .Values.applicationExistingClaim }} + - name: "application" + persistentVolumeClaim: + claimName: {{ .Values.applicationExistingClaim }} + {{- end }} + {{- with .Values.worker.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + +embargoRelease: + enabled: false +leaseRelease: + enabled: false + +fcrepo: + enabled: false +postgresql: + enabled: false +redis: + enabled: true + cluster: + enabled: false + architecture: standalone + password: $REDIS_PASSWORD + auth: + password: $REDIS_PASSWORD +solr: + enabled: false +fits: + enabled: true + servicePort: 8080 + subPath: /fits + +externalPostgresql: + host: acid-postgres-cluster-delta.postgres.svc.cluster.local + username: pals_demo + database: pals_demo + password: $DB_PASSWORD + +externalSolrHost: solr-headless.solr +externalSolrUser: admin +externalSolrCollection: demo-palni-palci +externalSolrPassword: $SOLR_ADMIN_PASSWORD + +global: + hyraxHostName: palni-palci-demo-pals + +nginx: + enabled: true + service: + type: ClusterIP + image: + registry: registry.gitlab.com + repository: notch8/scripts/bitnami-nginx + tag: 1.21.5-debian-10-r4 + serverBlock: |- + upstream rails_app { + server {{ .Values.global.hyraxHostName }};} + + map status loggable { + ~^444 0; + default 1; + } + + log_format loki 'host=host ip=http_x_forwarded_for remote_user=remote_user [time_local] ' + 'request="request" status=status bytes=body_bytes_sent ' + 'referer="http_referer" agent="http_user_agent" request_time=request_time upstream_response_time=upstream_response_time upstream_response_length=upstream_response_length'; + + error_log /opt/bitnami/nginx/logs/error.log warn; + #tcp_nopush on; + + # Cloudflare ips see for refresh + # https://support.cloudflare.com/hc/en-us/articles/200170786-Restoring-original-visitor-IPs-logging-visitor-IP-addresses + # update list https://www.cloudflare.com/ips/ + set_real_ip_from 103.21.244.0/22; + set_real_ip_from 103.22.200.0/22; + set_real_ip_from 103.31.4.0/22; + set_real_ip_from 104.16.0.0/13; + set_real_ip_from 104.24.0.0/14; + set_real_ip_from 108.162.192.0/18; + set_real_ip_from 131.0.72.0/22; + set_real_ip_from 141.101.64.0/18; + set_real_ip_from 162.158.0.0/15; + set_real_ip_from 172.64.0.0/13; + set_real_ip_from 173.245.48.0/20; + set_real_ip_from 188.114.96.0/20; + set_real_ip_from 190.93.240.0/20; + set_real_ip_from 197.234.240.0/22; + set_real_ip_from 198.41.128.0/17; + set_real_ip_from 2400:cb00::/32; + set_real_ip_from 2606:4700::/32; + set_real_ip_from 2803:f800::/32; + set_real_ip_from 2405:b500::/32; + set_real_ip_from 2405:8100::/32; + set_real_ip_from 2a06:98c0::/29; + set_real_ip_from 2c0f:f248::/32; + + real_ip_header X-Forwarded-For; + real_ip_recursive on; + include /opt/bitnami/nginx/conf/conf.d/*.conf; + server { + listen 8080; + server_name _; + root /app/samvera/hyrax-webapp/public; + index index.html; + + client_body_in_file_only clean; + client_body_buffer_size 32K; + client_max_body_size 0; + access_log /opt/bitnami/nginx/logs/access.log loki; + # if=loggable; + + sendfile on; + send_timeout 300s; + + include /opt/bitnami/nginx/conf/bots.d/ddos.conf; + include /opt/bitnami/nginx/conf/bots.d/blockbots.conf; + + location ~ (\.php|\.aspx|\.asp) { + return 404; + } + + # deny requests for files that should never be accessed + location ~ /\. { + deny all; + } + + location ~* ^.+\.(rb|log) { + deny all; + } + + # serve static (compiled) assets directly if they exist (for rails production) + location ~ ^/(assets|packs|fonts|images|javascripts|stylesheets|swfs|system)/ { + try_files uri @rails; + + # access_log off; + gzip_static on; # to serve pre-gzipped version + + expires max; + add_header Cache-Control public; + + # Some browsers still send conditional-GET requests if there's a + # Last-Modified header or an ETag header even if they haven't + # reached the expiry date sent in the Expires header. + add_header Last-Modified ""; + add_header ETag ""; + break; + } + + # send non-static file requests to the app server + location / { + try_files uri @rails; + } + + location @rails { + proxy_set_header X-Real-IP remote_addr; + proxy_set_header X-Forwarded-For proxy_add_x_forwarded_for; + proxy_set_header Host http_host; + proxy_redirect off; + proxy_pass http://rails_app; + } + }