From d359f4f80858465ed86eca6e706fa218e54950b1 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 7 Dec 2024 07:52:24 -0500 Subject: [PATCH 1/7] [minor] Add HLS streaming derivative service --- examples/hls/Dockerfile | 10 +++++++++ examples/hls/README.md | 5 +++++ examples/hls/cmd.sh | 42 ++++++++++++++++++++++++++++++++++++ examples/hls/scyllaridae.yml | 9 ++++++++ 4 files changed, 66 insertions(+) create mode 100644 examples/hls/Dockerfile create mode 100644 examples/hls/README.md create mode 100755 examples/hls/cmd.sh create mode 100644 examples/hls/scyllaridae.yml diff --git a/examples/hls/Dockerfile b/examples/hls/Dockerfile new file mode 100644 index 0000000..e2bc41a --- /dev/null +++ b/examples/hls/Dockerfile @@ -0,0 +1,10 @@ +ARG TAG=main +ARG DOCKER_REPOSITORY=lehighlts +FROM ${DOCKER_REPOSITORY}/scyllaridae:${TAG} + +RUN apk update && \ + apk add --no-cache \ + ffmpeg==6.1.1-r8 \ + jq==1.7.1-r0 + +COPY scyllaridae.yml /app/scyllaridae.yml diff --git a/examples/hls/README.md b/examples/hls/README.md new file mode 100644 index 0000000..77a6925 --- /dev/null +++ b/examples/hls/README.md @@ -0,0 +1,5 @@ +# Streaming Audio and Video Derivatives + +Create an HLS streaming derivative for audio and video files + +Requires `islandora_hls` module running on your Islandora site to render the streaming derivative. diff --git a/examples/hls/cmd.sh b/examples/hls/cmd.sh new file mode 100755 index 0000000..821c17f --- /dev/null +++ b/examples/hls/cmd.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -eou pipefail + +SOURCE_MIMETYPE="$1" +NODE_URL="$2" +NID=$(basename "$NODE_URL") +TMP_DIR=$(mktemp -d) + +ffmpeg \ + -f "$SOURCE_MIMETYPE" \ + -i - \ + -profile:v \ + baseline \ + -level 3.0 \ + -s 640x360 \ + -start_number 0 \ + -hls_time 10 \ + -hls_list_size 0 \ + -f hls \ + -b:v 800k \ + -maxrate 800k \ + -bufsize 1200k \ + -b:a 96k "$TMP_DIR/$NID.m3u8" + +if [ ! -f "$TMP_DIR/$NID.m3u8" ]; then + exit 1 +fi + +tar -czf "$TMP_DIR/hls.tar.gz" -C "$TMP_DIR" "$NID.m3u8" ./*.ts + +BASE_URL=$(dirname "$NODE_URL" | xargs dirname) +TID=$(curl "$BASE_URL/term_from_term_name?vocab=islandora_media_use&name=Service+File&_format=json" | jq '.[0].tid[0].value') + +curl \ + -H "Authorization: $SCYLLARIDAE_AUTH" \ + -H "Content-Type: application/gzip" \ + -H "Content-Location: private://derivatives/hls/$NID/hls.tar.gz" \ + -T "$TMP_DIR/hls.tar.gz" \ + "$NODE_URL/media/file/$TID" + +rm -rf "$TMP_DIR" diff --git a/examples/hls/scyllaridae.yml b/examples/hls/scyllaridae.yml new file mode 100644 index 0000000..30c0c2b --- /dev/null +++ b/examples/hls/scyllaridae.yml @@ -0,0 +1,9 @@ +allowedMimeTypes: + - "audio/*" + - "video/*" +cmdByMimeType: + default: + cmd: /app/cmd.sh + args: + - "%source-mime-ext" + - "%canonical" From efbdbbc6d95c02692457ab2fb0623879830cc412 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 7 Dec 2024 07:55:18 -0500 Subject: [PATCH 2/7] With canonical, HLS requires the full event message --- examples/parry/scyllaridae.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/parry/scyllaridae.yml b/examples/parry/scyllaridae.yml index 4b1c553..d43bd87 100644 --- a/examples/parry/scyllaridae.yml +++ b/examples/parry/scyllaridae.yml @@ -9,3 +9,6 @@ queueMiddlewares: - queueName: islandora-merge-pdf url: http://mergepdf:8080 consumers: 3 + - queueName: islandora-hls + url: http://hls:8080 + consumers: 3 From 4a1d071ae641db89ab70d6099913e0f13f34d64f Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 7 Dec 2024 09:02:13 -0500 Subject: [PATCH 3/7] A little more port switching to help with host ports --- ci/k8s/coverpage.yaml | 4 ++-- ci/k8s/crayfits.yaml | 5 +++-- ci/k8s/fits.yaml | 4 +++- ci/k8s/homarus.yaml | 6 ++++-- ci/k8s/htr.yaml | 4 ++-- ci/k8s/mergepdf.yaml | 4 ++-- ci/k8s/ocrpdf.yaml | 4 ++-- examples/hls/Dockerfile | 1 + examples/hls/cmd.sh | 12 +++++++----- examples/hls/scyllaridae.yml | 1 + 10 files changed, 27 insertions(+), 18 deletions(-) diff --git a/ci/k8s/coverpage.yaml b/ci/k8s/coverpage.yaml index 67fe516..4559e3d 100644 --- a/ci/k8s/coverpage.yaml +++ b/ci/k8s/coverpage.yaml @@ -8,7 +8,7 @@ spec: app: islandora-coverpage ports: - protocol: TCP - port: 8888 + port: 8082 targetPort: 8080 --- apiVersion: apps/v1 @@ -37,7 +37,7 @@ spec: memory: "1Gi" ports: - containerPort: 8080 - hostPort: 8888 + hostPort: 8082 readinessProbe: httpGet: path: /healthcheck diff --git a/ci/k8s/crayfits.yaml b/ci/k8s/crayfits.yaml index 0f8af92..e9f0a50 100644 --- a/ci/k8s/crayfits.yaml +++ b/ci/k8s/crayfits.yaml @@ -8,7 +8,8 @@ spec: app: islandora-crayfits ports: - protocol: TCP - port: 8080 + port: 8083 + targetPort: 8080 --- apiVersion: apps/v1 kind: Deployment @@ -36,7 +37,7 @@ spec: memory: "128Mi" ports: - containerPort: 8080 - hostPort: 8080 + hostPort: 8083 readinessProbe: httpGet: path: /healthcheck diff --git a/ci/k8s/fits.yaml b/ci/k8s/fits.yaml index 50dd273..136ef1d 100644 --- a/ci/k8s/fits.yaml +++ b/ci/k8s/fits.yaml @@ -8,7 +8,8 @@ spec: app: fits ports: - protocol: TCP - port: 8080 + port: 8084 + targetPort: 8080 --- apiVersion: apps/v1 kind: Deployment @@ -41,6 +42,7 @@ spec: memory: "2Gi" ports: - containerPort: 8080 + hostPort: 8084 readinessProbe: httpGet: path: /fits/version diff --git a/ci/k8s/homarus.yaml b/ci/k8s/homarus.yaml index fc86e2b..36ab336 100644 --- a/ci/k8s/homarus.yaml +++ b/ci/k8s/homarus.yaml @@ -8,7 +8,8 @@ spec: app: islandora-ffmpeg ports: - protocol: TCP - port: 8080 + port: 8086 + targetPort: 8080 --- apiVersion: apps/v1 kind: Deployment @@ -35,7 +36,8 @@ spec: limits: memory: "4Gi" ports: - - containerPort: 8080 + - hostPort: 8086 + containerPort: 8080 readinessProbe: httpGet: path: /healthcheck diff --git a/ci/k8s/htr.yaml b/ci/k8s/htr.yaml index de51390..20f8b1e 100644 --- a/ci/k8s/htr.yaml +++ b/ci/k8s/htr.yaml @@ -8,7 +8,7 @@ spec: app: islandora-openai-htr ports: - protocol: TCP - port: 8001 + port: 8087 targetPort: 8080 --- apiVersion: apps/v1 @@ -42,7 +42,7 @@ spec: limits: memory: "1Gi" ports: - - hostPort: 8001 + - hostPort: 8087 containerPort: 8080 readinessProbe: httpGet: diff --git a/ci/k8s/mergepdf.yaml b/ci/k8s/mergepdf.yaml index 800167b..af65cd0 100644 --- a/ci/k8s/mergepdf.yaml +++ b/ci/k8s/mergepdf.yaml @@ -8,7 +8,7 @@ spec: app: islandora-mergepdf ports: - protocol: TCP - port: 8887 + port: 8088 targetPort: 8080 --- apiVersion: apps/v1 @@ -40,7 +40,7 @@ spec: memory: "4Gi" ports: - containerPort: 8080 - hostPort: 8887 + hostPort: 8088 readinessProbe: httpGet: path: /healthcheck diff --git a/ci/k8s/ocrpdf.yaml b/ci/k8s/ocrpdf.yaml index 1ebd9a4..2d0d1dd 100644 --- a/ci/k8s/ocrpdf.yaml +++ b/ci/k8s/ocrpdf.yaml @@ -8,7 +8,7 @@ spec: app: islandora-ocrpdf ports: - protocol: TCP - port: 8886 + port: 8089 targetPort: 8080 --- apiVersion: apps/v1 @@ -37,7 +37,7 @@ spec: memory: "2Gi" ports: - containerPort: 8080 - hostPort: 8886 + hostPort: 8089 readinessProbe: httpGet: path: /healthcheck diff --git a/examples/hls/Dockerfile b/examples/hls/Dockerfile index e2bc41a..ec7be9e 100644 --- a/examples/hls/Dockerfile +++ b/examples/hls/Dockerfile @@ -8,3 +8,4 @@ RUN apk update && \ jq==1.7.1-r0 COPY scyllaridae.yml /app/scyllaridae.yml +COPY cmd.sh /app/cmd.sh diff --git a/examples/hls/cmd.sh b/examples/hls/cmd.sh index 821c17f..562c6f0 100755 --- a/examples/hls/cmd.sh +++ b/examples/hls/cmd.sh @@ -21,22 +21,24 @@ ffmpeg \ -b:v 800k \ -maxrate 800k \ -bufsize 1200k \ - -b:a 96k "$TMP_DIR/$NID.m3u8" + -b:a 96k "$TMP_DIR/$NID.m3u8" > /dev/null 2>&1 if [ ! -f "$TMP_DIR/$NID.m3u8" ]; then exit 1 fi -tar -czf "$TMP_DIR/hls.tar.gz" -C "$TMP_DIR" "$NID.m3u8" ./*.ts +# shellcheck disable=SC2046 +tar -czf "$TMP_DIR/hls.tar.gz" -C "$TMP_DIR" "./$NID.m3u8" $(cd "$TMP_DIR" ; echo ./*.ts) BASE_URL=$(dirname "$NODE_URL" | xargs dirname) -TID=$(curl "$BASE_URL/term_from_term_name?vocab=islandora_media_use&name=Service+File&_format=json" | jq '.[0].tid[0].value') +TID=$(curl -s "$BASE_URL/term_from_term_name?vocab=islandora_media_use&name=Service+File&_format=json" | jq '.[0].tid[0].value') -curl \ +curl -s \ + -X PUT \ -H "Authorization: $SCYLLARIDAE_AUTH" \ -H "Content-Type: application/gzip" \ -H "Content-Location: private://derivatives/hls/$NID/hls.tar.gz" \ - -T "$TMP_DIR/hls.tar.gz" \ + --data-binary "@$TMP_DIR/hls.tar.gz" \ "$NODE_URL/media/file/$TID" rm -rf "$TMP_DIR" diff --git a/examples/hls/scyllaridae.yml b/examples/hls/scyllaridae.yml index 30c0c2b..3da5afa 100644 --- a/examples/hls/scyllaridae.yml +++ b/examples/hls/scyllaridae.yml @@ -1,3 +1,4 @@ +forwardAuth: true allowedMimeTypes: - "audio/*" - "video/*" From 065ea9a56371d0fcf361543de5e6c025bfe4014c Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 7 Dec 2024 09:03:08 -0500 Subject: [PATCH 4/7] ports --- ci/k8s/htr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/k8s/htr.yaml b/ci/k8s/htr.yaml index 20f8b1e..282b1d2 100644 --- a/ci/k8s/htr.yaml +++ b/ci/k8s/htr.yaml @@ -26,7 +26,7 @@ spec: app: islandora-openai-htr spec: containers: - - name: scyllaridae-tesseract + - name: scyllaridae-htr image: lehighlts/scyllaridae-openai-htr:main imagePullPolicy: IfNotPresent env: From 6d3859d9707784a1f88e0db20e8bff76c5d50d67 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 7 Dec 2024 09:03:15 -0500 Subject: [PATCH 5/7] Add hls --- ci/k8s/hls.yaml | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 ci/k8s/hls.yaml diff --git a/ci/k8s/hls.yaml b/ci/k8s/hls.yaml new file mode 100644 index 0000000..498974a --- /dev/null +++ b/ci/k8s/hls.yaml @@ -0,0 +1,46 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: islandora-hls +spec: + selector: + app: islandora-hls + ports: + - protocol: TCP + port: 8085 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: islandora-hls +spec: + replicas: 3 + selector: + matchLabels: + app: islandora-hls + template: + metadata: + labels: + app: islandora-hls + spec: + containers: + - name: scyllaridae-hls + image: lehighlts/scyllaridae-hls:main + imagePullPolicy: IfNotPresent + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "4Gi" + ports: + - hostPort: 8085 + containerPort: 8080 + readinessProbe: + httpGet: + path: /healthcheck + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 10 From fef30b86076881fa1ee6cfd5b639e00bfd853fb0 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 7 Dec 2024 09:03:52 -0500 Subject: [PATCH 6/7] forwardAuth --- examples/parry/scyllaridae.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/parry/scyllaridae.yml b/examples/parry/scyllaridae.yml index d43bd87..24ef8dc 100644 --- a/examples/parry/scyllaridae.yml +++ b/examples/parry/scyllaridae.yml @@ -2,6 +2,7 @@ queueMiddlewares: - queueName: islandora-pdf-coverpage url: http://coverpage:8080 consumers: 3 + forwardAuth: true - queueName: islandora-cache-warmer url: http://cache-warmer:8080 consumers: 3 @@ -9,6 +10,8 @@ queueMiddlewares: - queueName: islandora-merge-pdf url: http://mergepdf:8080 consumers: 3 + forwardAuth: true - queueName: islandora-hls url: http://hls:8080 consumers: 3 + forwardAuth: true From 54377cec6e927d9ed4ca55f92636b476dd772a5f Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Sat, 7 Dec 2024 10:32:48 -0500 Subject: [PATCH 7/7] fixup --- examples/hls/cmd.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/hls/cmd.sh b/examples/hls/cmd.sh index 562c6f0..7c5c571 100755 --- a/examples/hls/cmd.sh +++ b/examples/hls/cmd.sh @@ -34,11 +34,10 @@ BASE_URL=$(dirname "$NODE_URL" | xargs dirname) TID=$(curl -s "$BASE_URL/term_from_term_name?vocab=islandora_media_use&name=Service+File&_format=json" | jq '.[0].tid[0].value') curl -s \ - -X PUT \ -H "Authorization: $SCYLLARIDAE_AUTH" \ -H "Content-Type: application/gzip" \ -H "Content-Location: private://derivatives/hls/$NID/hls.tar.gz" \ - --data-binary "@$TMP_DIR/hls.tar.gz" \ + -T "$TMP_DIR/hls.tar.gz" \ "$NODE_URL/media/file/$TID" rm -rf "$TMP_DIR"