From 45fc3082ae58353463c8397bcf06fcd60708d2dd Mon Sep 17 00:00:00 2001 From: Eliot Jordan Date: Mon, 30 Sep 2024 12:39:47 -0500 Subject: [PATCH 1/6] Update GDAL command to use built-in COG output format --- .../geo_derivatives/processors/gdal.rb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/app/derivative_services/geo_derivatives/processors/gdal.rb b/app/derivative_services/geo_derivatives/processors/gdal.rb index 4d31450aca..008b53b4df 100644 --- a/app/derivative_services/geo_derivatives/processors/gdal.rb +++ b/app/derivative_services/geo_derivatives/processors/gdal.rb @@ -26,7 +26,7 @@ def self.translate(in_path, out_path, _options) # @param options [Hash] creation options def self.warp(in_path, out_path, options) execute "gdalwarp -q -t_srs #{options[:output_srid]} "\ - "\"#{in_path}\" #{out_path} -co TILED=YES -co COMPRESS=NONE" + "\"#{in_path}\" #{out_path} -co COMPRESS=NONE" end # Executes a gdal_translate command. Used to compress @@ -46,13 +46,12 @@ def self.compress(in_path, out_path, options) # @param out_path [String] processor output file path # @param options [Hash] creation options def self.cloud_optimized_geotiff(in_path, out_path, _options) - system("gdaladdo -q -r average \"#{in_path}\" 2 4 8 16 32") - execute("gdal_translate -q -expand rgb \"#{in_path}\" #{out_path} -ot Byte -co TILED=YES "\ - "-a_nodata 256 -co COMPRESS=LZW -co COPY_SRC_OVERVIEWS=YES") + execute("gdal_translate -q -expand rgb \"#{in_path}\" #{out_path} -ot Byte -of COG "\ + "-a_nodata 256 -co COMPRESS=LZW -co TILING_SCHEME=GoogleMapsCompatible") rescue StandardError # Try without expanding rgb - execute("gdal_translate -q \"#{in_path}\" #{out_path} -ot Byte -co TILED=YES "\ - "-a_nodata 256 -co COMPRESS=LZW -co COPY_SRC_OVERVIEWS=YES") + execute("gdal_translate -q \"#{in_path}\" #{out_path} -ot Byte -of COG "\ + "-a_nodata 256 -co COMPRESS=LZW -co TILING_SCHEME=GoogleMapsCompatible") end # Executes a gdal_rasterize command. Used to rasterize vector From 301ef6d2a7c53322409e921736bef645b01d3ba4 Mon Sep 17 00:00:00 2001 From: Eliot Jordan Date: Mon, 30 Sep 2024 14:49:34 -0500 Subject: [PATCH 2/6] Build CI container in github actions --- .circleci/Dockerfile | 2 +- .circleci/config.yml | 2 +- .github/workflows/build-ci-docker.yml | 59 +++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/build-ci-docker.yml diff --git a/.circleci/Dockerfile b/.circleci/Dockerfile index 3641b0b289..2fbf183f9c 100644 --- a/.circleci/Dockerfile +++ b/.circleci/Dockerfile @@ -1,4 +1,4 @@ -FROM cimg/ruby:3.1.0-browsers +FROM cimg/ruby:3.1.6-browsers RUN sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' \ && wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /etc/apt/trusted.gpg.d/pgdg.asc &>/dev/null diff --git a/.circleci/config.yml b/.circleci/config.yml index a7c1f1191b..d78188c036 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,7 +60,7 @@ jobs: test: working_directory: ~/figgy docker: - - image: pulibrary/ci-figgy:1.6 + - image: ghcr.io/pulibrary/figgy:latest environment: RAILS_ENV: test FIGGY_DB_HOST: localhost diff --git a/.github/workflows/build-ci-docker.yml b/.github/workflows/build-ci-docker.yml new file mode 100644 index 0000000000..f0990b80bc --- /dev/null +++ b/.github/workflows/build-ci-docker.yml @@ -0,0 +1,59 @@ +name: Create and publish a Docker CI image + +on: + push: + paths: + - .circleci/Dockerfile + - bin/** + branches: + - main + pull_request: + paths: + - .circleci/Dockerfile + - bin/** + branches: + - main + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=sha + env: + DOCKER_METADATA_PR_HEAD_SHA: true + + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: "${{ github.workspace }}" + file: "${{ github.workspace }}/.circleci/Dockerfile" + push: true + tags: ghcr.io/pulibrary/figgy:ci + labels: ${{ steps.meta.outputs.labels }} From b4c7f87ebb1f02c6a9ce9767498efdd8f816bf16 Mon Sep 17 00:00:00 2001 From: Eliot Jordan Date: Mon, 30 Sep 2024 16:15:58 -0500 Subject: [PATCH 3/6] Use github package in CI --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d78188c036..9abd6d83d5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ jobs: build: working_directory: ~/figgy docker: - - image: pulibrary/ci-figgy:1.6 + - image: ghcr.io/pulibrary/figgy:ci environment: RAILS_ENV: test FIGGY_DB_HOST: localhost @@ -60,7 +60,7 @@ jobs: test: working_directory: ~/figgy docker: - - image: ghcr.io/pulibrary/figgy:latest + - image: ghcr.io/pulibrary/figgy:ci environment: RAILS_ENV: test FIGGY_DB_HOST: localhost @@ -137,7 +137,7 @@ jobs: coverage_report: working_directory: ~/figgy docker: - - image: pulibrary/ci-figgy:1.6 + - image: ghcr.io/pulibrary/figgy:ci resource_class: large steps: - attach_workspace: @@ -154,7 +154,7 @@ jobs: rubocop: working_directory: ~/figgy docker: - - image: pulibrary/ci-figgy:1.6 + - image: ghcr.io/pulibrary/figgy:ci environment: RAILS_ENV: test FIGGY_DB_HOST: localhost From f2a26130aa91fdc24473dc7b678034129e63ce8d Mon Sep 17 00:00:00 2001 From: Eliot Jordan Date: Mon, 30 Sep 2024 17:02:08 -0500 Subject: [PATCH 4/6] Tag images depending on trigger --- .github/workflows/build-ci-docker.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-ci-docker.yml b/.github/workflows/build-ci-docker.yml index f0990b80bc..3b4bef68a5 100644 --- a/.github/workflows/build-ci-docker.yml +++ b/.github/workflows/build-ci-docker.yml @@ -22,7 +22,6 @@ env: jobs: build-and-push-image: runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. permissions: contents: read packages: write @@ -50,6 +49,7 @@ jobs: DOCKER_METADATA_PR_HEAD_SHA: true - name: Build and push Docker image + if: (github.event_name == 'workflow_dispatch' || github.event_name == 'push') && github.ref == 'refs/heads/main' uses: docker/build-push-action@v6 with: context: "${{ github.workspace }}" @@ -57,3 +57,13 @@ jobs: push: true tags: ghcr.io/pulibrary/figgy:ci labels: ${{ steps.meta.outputs.labels }} + + - name: Build and push Docker image + if: (github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request') && github.ref != 'refs/heads/main' + uses: docker/build-push-action@v6 + with: + context: "${{ github.workspace }}" + file: "${{ github.workspace }}/.circleci/Dockerfile" + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} From 774c577f13e9d02a86ac21319746b399e6dcf853 Mon Sep 17 00:00:00 2001 From: Eliot Jordan Date: Mon, 30 Sep 2024 17:02:24 -0500 Subject: [PATCH 5/6] Add job to prune containers --- .github/workflows/prune-containers.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/prune-containers.yml diff --git a/.github/workflows/prune-containers.yml b/.github/workflows/prune-containers.yml new file mode 100644 index 0000000000..2d0d71175d --- /dev/null +++ b/.github/workflows/prune-containers.yml @@ -0,0 +1,19 @@ +name: Prune containers + +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * SUN" # every sunday at midnight + +jobs: + clean: + runs-on: ubuntu-latest + name: Delete old images + steps: + - uses: snok/container-retention-policy@v3.0.0 + with: + account: pulibrary + image-names: figgy + token: ${{ secrets.GITHUB_TOKEN }} + image-tags: "!ci !main" + cut-off: 2w From 2f9a0ccba58f50f715c896a8bc0d360a32d39d5d Mon Sep 17 00:00:00 2001 From: Eliot Jordan Date: Tue, 1 Oct 2024 09:45:24 -0500 Subject: [PATCH 6/6] Update gdal spec with new options --- .../geo_derivatives/processors/base_geo_processor_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/derivative_services/geo_derivatives/processors/base_geo_processor_spec.rb b/spec/derivative_services/geo_derivatives/processors/base_geo_processor_spec.rb index db672ae898..889080b059 100644 --- a/spec/derivative_services/geo_derivatives/processors/base_geo_processor_spec.rb +++ b/spec/derivative_services/geo_derivatives/processors/base_geo_processor_spec.rb @@ -73,7 +73,7 @@ def source_path; end describe "#warp" do it "executes a reproject command" do - command = "gdalwarp -q -t_srs EPSG:4326 \"files/geo.tif\" output/geo.png -co TILED=YES -co COMPRESS=NONE" + command = "gdalwarp -q -t_srs EPSG:4326 \"files/geo.tif\" output/geo.png -co COMPRESS=NONE" processor.class.warp(file_name, output_file, options) expect(processor.class).to have_received(:execute).with command end