diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..32ca3fe --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +docker-images +scyllaridae +.git +.github + + diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml new file mode 100644 index 0000000..c82f82a --- /dev/null +++ b/.github/workflows/build-push.yml @@ -0,0 +1,76 @@ +name: build-push-ar +on: + workflow_call: + inputs: + dockerFile: + required: true + type: string +jobs: + build-push-ar: + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + id-token: write + steps: + - uses: 'actions/checkout@v4' + + - name: Extract branch name as docker tag + shell: bash + run: |- + BRANCH=$(echo "${GITHUB_REF#refs/heads/}" | sed 's/[^a-zA-Z0-9._-]//g' | awk '{print substr($0, length($0)-120)}') + echo "branch=$BRANCH" >> $GITHUB_OUTPUT + id: extract_branch + + - name: Extract tag name + shell: bash + run: |- + t=$(echo ${GITHUB_SHA} | cut -c1-7) + echo "tag=$t" >> $GITHUB_OUTPUT + id: extract_tag + + - name: Setup docker build + shell: bash + run: |- + # aka base build + if [ ${{ inputs.dockerFile }} == "Dockerfile" ]; then + echo "image=scyllaridae" >> $GITHUB_OUTPUT + exit 0 + fi + + # put the YML file in place so it's copied into the Docker container + DIR=$(dirname "${{ inputs.dockerFile }}") + cp ./$DIR/scyllaridae.yml . + # name the docker image after the folder name prefixed by scyllaridae + # e.g. scyllaridae-curl + echo "image=scyllaridae-$(basename $DIR)" >> $GITHUB_OUTPUT + id: setup + + - id: 'auth' + name: 'Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v1' + with: + workload_identity_provider: ${{ secrets.GCLOUD_OIDC_POOL }} + create_credentials_file: true + service_account: ${{ secrets.GSA }} + token_format: 'access_token' + + - uses: 'docker/login-action@v3' + name: 'Docker login' + with: + registry: 'us-docker.pkg.dev' + username: 'oauth2accesstoken' + password: '${{ steps.auth.outputs.access_token }}' + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + file: ${{ inputs.dockerFile }} + build-args: | + TAG=${{steps.extract_branch.outputs.branch}} + DOCKER_REPOSITORY=us-docker.pkg.dev/${{ secrets.GCLOUD_PROJECT }}/public + push: true + tags: | + us-docker.pkg.dev/${{ secrets.GCLOUD_PROJECT }}/public/${{steps.setup.outputs.image}}:${{steps.extract_branch.outputs.branch}}-${{steps.extract_tag.outputs.tag}} + us-docker.pkg.dev/${{ secrets.GCLOUD_PROJECT }}/public/${{steps.setup.outputs.image}}:${{steps.extract_branch.outputs.branch}} diff --git a/.github/workflows/lint-test-build.yml b/.github/workflows/lint-test-build.yml new file mode 100644 index 0000000..b951dd5 --- /dev/null +++ b/.github/workflows/lint-test-build.yml @@ -0,0 +1,71 @@ +name: lint-test +on: + push: + +permissions: + contents: read + +jobs: + + lint-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v4 + + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + version: v1.54 + + - name: Install dependencies + run: go get . + + - name: Build + run: go build -v ./... + + - name: Put fixture in place + run: cp scyllaridae.example.yml scyllaridae.yml + + - name: Test with the Go CLI + run: go test -v ./... + + build-push-base: + needs: [lint-test] + uses: ./.github/workflows/build-push.yml + with: + dockerFile: Dockerfile + permissions: + contents: read + id-token: write + secrets: inherit + + find-images: + needs: [build-push-base] + name: Find docker images needing built + runs-on: ubuntu-latest + outputs: + dockerFiles: ${{ steps.images.outputs.dockerFiles }} + steps: + - uses: actions/checkout@v4 + - name: Find docker files + id: images + run: | + dockerFiles=$(find docker-images -name Dockerfile | jq -c --raw-input --slurp 'split("\n")| .[0:-1]') + echo "dockerFiles=$dockerFiles" >> $GITHUB_OUTPUT + env: + GITHUB_REF: ${{ github.ref }} + + build-push: + needs: [find-images] + strategy: + matrix: + dockerFile: ${{ fromJson(needs.find-images.outputs.dockerFiles )}} + uses: ./.github/workflows/build-push.yml + with: + dockerFile: ${{ matrix.dockerFile }} + permissions: + contents: read + id-token: write + secrets: inherit diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml deleted file mode 100644 index dc85d7a..0000000 --- a/.github/workflows/lint-test.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: lint-test -on: [push] -permissions: - contents: read - -jobs: - lint-test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 - - name: golangci-lint - uses: golangci/golangci-lint-action@v3 - with: - version: v1.54 - - name: Install dependencies - run: go get . - - name: Build - run: go build -v ./... - - name: Put fixture in place - run: cp scyllaridae.example.yml scyllaridae.yml - - name: Test with the Go CLI - run: go test -v ./... diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c1363b6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM golang:1.21-alpine + +WORKDIR /app + +RUN apk update && \ + apk add openssl && \ + openssl s_client -connect helloworld.letsencrypt.org:443 -showcerts </dev/null 2>/dev/null | sed -e '/-----BEGIN/,/-----END/!d' | tee "/usr/local/share/ca-certificates/ca.crt" >/dev/null && \ + update-ca-certificates + +COPY . ./ +RUN go mod download && \ + go build -o /app/scyllaridae && \ + go clean -cache -modcache + +ENTRYPOINT ["/app/scyllaridae"] diff --git a/docker-images/curl/Dockerfile b/docker-images/curl/Dockerfile new file mode 100644 index 0000000..cf2638d --- /dev/null +++ b/docker-images/curl/Dockerfile @@ -0,0 +1,8 @@ +ARG TAG=main +ARG DOCKER_REPOSITORY=local +FROM ${DOCKER_REPOSITORY}/scyllaridae:${TAG} + +RUN apk update && \ + apk add --no-cache curl + +COPY scyllaridae.yml /app/scyllaridae.yml diff --git a/docker-images/curl/scyllaridae.yml b/docker-images/curl/scyllaridae.yml new file mode 100644 index 0000000..4d8446d --- /dev/null +++ b/docker-images/curl/scyllaridae.yml @@ -0,0 +1,8 @@ +destinationHttpMethod: GET +forwardAuth: false +allowedMimeTypes: [ + "text/html" +] +cmdByMimeType: + default: + cmd: "curl" diff --git a/docker-images/ffmpeg/Dockerfile b/docker-images/ffmpeg/Dockerfile new file mode 100644 index 0000000..1d1a106 --- /dev/null +++ b/docker-images/ffmpeg/Dockerfile @@ -0,0 +1,8 @@ +ARG TAG=main +ARG DOCKER_REPOSITORY=local +FROM ${DOCKER_REPOSITORY}/scyllaridae:${TAG} + +RUN apk update && \ + apk add --no-cache ffmpeg + +COPY scyllaridae.yml /app/scyllaridae.yml diff --git a/docker-images/ffmpeg/scyllaridae.yml b/docker-images/ffmpeg/scyllaridae.yml new file mode 100644 index 0000000..8197a0c --- /dev/null +++ b/docker-images/ffmpeg/scyllaridae.yml @@ -0,0 +1,98 @@ +destinationHttpMethod: PUT +allowedMimeTypes: [ + "audio/*", + "video/*", + "image/jpeg", + "image/png" +] +cmdByMimeType: + "video/x-msvideo" + cmd: "ffmpeg" + args: [ + "-i", + "-", + "%s", + "-f", + "avi" + ] + "video/ogg" + cmd: "ffmpeg" + args: [ + "-i", + "-", + "%s", + "-f", + "ogg" + ] + "audio/x-wav" + cmd: "ffmpeg" + args: [ + "-i", + "-", + "%s", + "-f", + "wav" + ] + "audio/mpeg" + cmd: "ffmpeg" + args: [ + "-i", + "-", + "%s", + "-f", + "mp3" + ] + "audio/aac" + cmd: "ffmpeg" + args: [ + "-i", + "-", + "%s", + "-f", + "m4a" + ] + "image/jpeg" + cmd: "ffmpeg" + args: [ + "-i", + "-", + "%s", + "-f", + "image2pipe" + ] + "image/png" + cmd: "ffmpeg" + args: [ + "-i", + "-", + "%s", + "-f", + "image2pipe" + ] + "video/mp4": + cmd: "ffmpeg" + args: [ + "-i", + "-", + "%s", + "-vcodec", + "libx264", + "-preset", + "medium", + "-acodec", + "aac", + "-strict", + "-2", + "-ab", + "128k", + "-ac", + "2", + "-async", + "1", + "-movflags", + "faststart", + "-y", + "-f", + "mp4", + "-" + ] diff --git a/docker-images/fits/Dockerfile b/docker-images/fits/Dockerfile new file mode 100644 index 0000000..cf2638d --- /dev/null +++ b/docker-images/fits/Dockerfile @@ -0,0 +1,8 @@ +ARG TAG=main +ARG DOCKER_REPOSITORY=local +FROM ${DOCKER_REPOSITORY}/scyllaridae:${TAG} + +RUN apk update && \ + apk add --no-cache curl + +COPY scyllaridae.yml /app/scyllaridae.yml diff --git a/docker-images/fits/scyllaridae.yml b/docker-images/fits/scyllaridae.yml new file mode 100644 index 0000000..c2ca446 --- /dev/null +++ b/docker-images/fits/scyllaridae.yml @@ -0,0 +1,15 @@ +destinationHttpMethod: GET +forwardAuth: false +allowedMimeTypes: [ + "*" +] +cmdByMimeType: + default: + cmd: "curl" + args: [ + "http://fits:8080/fits/examine", + "-X", + "POST", + "-F", + "datafile=@-" + ] diff --git a/docker-images/imagemagick/Dockerfile b/docker-images/imagemagick/Dockerfile new file mode 100644 index 0000000..5c6736f --- /dev/null +++ b/docker-images/imagemagick/Dockerfile @@ -0,0 +1,8 @@ +ARG TAG=main +ARG DOCKER_REPOSITORY=local +FROM ${DOCKER_REPOSITORY}/scyllaridae:${TAG} + +RUN apk update && \ + apk add --no-cache imagemagick + +COPY scyllaridae.yml /app/scyllaridae.yml diff --git a/docker-images/imagemagick/scyllaridae.yml b/docker-images/imagemagick/scyllaridae.yml new file mode 100644 index 0000000..5244f57 --- /dev/null +++ b/docker-images/imagemagick/scyllaridae.yml @@ -0,0 +1,7 @@ +destinationHttpMethod: PUT +allowedMimeTypes: [ + "text/html" +] +cmdByMimeType: + default: + cmd: "convert" diff --git a/docker-images/tesseract/Dockerfile b/docker-images/tesseract/Dockerfile new file mode 100644 index 0000000..1baa5b3 --- /dev/null +++ b/docker-images/tesseract/Dockerfile @@ -0,0 +1,19 @@ +ARG TAG=main +ARG DOCKER_REPOSITORY=local +FROM ${DOCKER_REPOSITORY}/scyllaridae:${TAG} + +RUN apk update && \ + apk add --no-cache leptonica-dev \ + tesseract-ocr \ + tesseract-ocr-data-eng \ + tesseract-ocr-data-fra \ + tesseract-ocr-data-spa \ + tesseract-ocr-data-ita \ + tesseract-ocr-data-por \ + tesseract-ocr-data-hin \ + tesseract-ocr-data-deu \ + tesseract-ocr-data-jpn \ + tesseract-ocr-data-rus \ + poppler-utils + +COPY scyllaridae.yml /app/scyllaridae.yml diff --git a/docker-images/tesseract/scyllaridae.yml b/docker-images/tesseract/scyllaridae.yml new file mode 100644 index 0000000..9465f94 --- /dev/null +++ b/docker-images/tesseract/scyllaridae.yml @@ -0,0 +1,20 @@ +destinationHttpMethod: PUT +allowedMimeTypes: [ + "application/pdf", + "image/*" +] +cmdByMimeType: + "application/pdf": + cmd: pdftotext + args: [ + "%s", + "-", + "-" + ] + default: + cmd: tesseract + args: [ + "stdin", + "stdout", + "%s" + ] diff --git a/internal/config/server.go b/internal/config/server.go index e93ba41..e3f7cf3 100644 --- a/internal/config/server.go +++ b/internal/config/server.go @@ -69,6 +69,9 @@ func IsAllowedMimeType(mimetype string, allowedFormats []string) bool { if format == mimetype { return true } + if format == "*" { + return true + } if strings.HasSuffix(format, "/*") { // Check wildcard MIME type prefix := strings.TrimSuffix(format, "*")