Merge pull request #12 from mlibrary/add-hathifiles

Add hathifiles
mlibrary · Jan 17, 2024 · 8a1f312 · 8a1f312
2 parents 6d70fbb + 8f7ce64
commit 8a1f312
Show file tree

Hide file tree

Showing 24 changed files with 530 additions and 206 deletions.
diff --git a/.github/workflows/build-deploy-on-release.yaml b/.github/workflows/build-deploy-on-release.yaml
@@ -14,7 +14,7 @@ jobs:
       dockerfile: Dockerfile
     secrets: inherit
 
-  deploy-production:
+  deploy-production-default:
     needs: build-production
     name: Deploy to production
     uses: mlibrary/platform-engineering-workflows/.github/workflows/deploy.yml@v1
@@ -25,3 +25,15 @@ jobs:
       CONFIG_REPO_RW_INSTALL_ID: ${{ vars.CONFIG_REPO_RW_INSTALL_ID }}
       CONFIG_REPO_FULL_NAME: ${{ vars.CONFIG_REPO_FULL_NAME }}
     secrets: inherit
+
+  deploy-production-hathifiles:
+    needs: build-production
+    name: Deploy to production
+    uses: mlibrary/platform-engineering-workflows/.github/workflows/deploy.yml@v1
+    with:
+      image: ghcr.io/mlibrary/${{ vars.IMAGE_NAME }}:${{ github.event.release.tag_name }}
+      file: environments/hathifiles/production/app-image.txt
+      CONFIG_REPO_RW_APP_ID: ${{ vars.CONFIG_REPO_RW_APP_ID }}
+      CONFIG_REPO_RW_INSTALL_ID: ${{ vars.CONFIG_REPO_RW_INSTALL_ID }}
+      CONFIG_REPO_FULL_NAME: ${{ vars.CONFIG_REPO_FULL_NAME }}
+    secrets: inherit
diff --git a/.github/workflows/build-main.yml b/.github/workflows/build-main.yml
@@ -16,9 +16,9 @@ jobs:
       dockerfile: Dockerfile
     secrets: inherit
 
-  deploy-workshop:
+  deploy-workshop-default:
     needs: build-unstable
-    name: Deploy to workshop
+    name: Deploy to workshop - default
     uses: mlibrary/platform-engineering-workflows/.github/workflows/deploy.yml@v1
     with:
       image: ${{ needs.build-unstable.outputs.image }}
@@ -27,3 +27,15 @@ jobs:
       CONFIG_REPO_RW_INSTALL_ID: ${{ vars.CONFIG_REPO_RW_INSTALL_ID }}
       CONFIG_REPO_FULL_NAME: ${{ vars.CONFIG_REPO_FULL_NAME }}
     secrets: inherit
+
+  deploy-workshop-hathifiles:
+    needs: build-unstable
+    name: Deploy to workshop - hathifiles
+    uses: mlibrary/platform-engineering-workflows/.github/workflows/deploy.yml@v1
+    with:
+      image: ${{ needs.build-unstable.outputs.image }}
+      file: environments/hathifiles/workshop/app-image.txt
+      CONFIG_REPO_RW_APP_ID: ${{ vars.CONFIG_REPO_RW_APP_ID }}
+      CONFIG_REPO_RW_INSTALL_ID: ${{ vars.CONFIG_REPO_RW_INSTALL_ID }}
+      CONFIG_REPO_FULL_NAME: ${{ vars.CONFIG_REPO_FULL_NAME }}
+    secrets: inherit
diff --git a/.github/workflows/manual-deploy-production.yaml b/.github/workflows/manual-deploy-production.yaml
@@ -17,7 +17,7 @@ jobs:
       dockerfile: Dockerfile
     secrets: inherit
 
-  deploy-production:
+  deploy-production-default:
     needs: build-production
     name: Deploy to production
     uses: mlibrary/platform-engineering-workflows/.github/workflows/deploy.yml@v1
@@ -28,3 +28,15 @@ jobs:
       CONFIG_REPO_RW_INSTALL_ID: ${{ vars.CONFIG_REPO_RW_INSTALL_ID }}
       CONFIG_REPO_FULL_NAME: ${{ vars.CONFIG_REPO_FULL_NAME }}
     secrets: inherit
+
+  deploy-production-hathifiles:
+    needs: build-production
+    name: Deploy to production
+    uses: mlibrary/platform-engineering-workflows/.github/workflows/deploy.yml@v1
+    with:
+      image: ghcr.io/mlibrary/${{ vars.IMAGE_NAME }}:${{ github.event.inputs.tag }}
+      file: environments/hathifiles/production/app-image.txt
+      CONFIG_REPO_RW_APP_ID: ${{ vars.CONFIG_REPO_RW_APP_ID }}
+      CONFIG_REPO_RW_INSTALL_ID: ${{ vars.CONFIG_REPO_RW_INSTALL_ID }}
+      CONFIG_REPO_FULL_NAME: ${{ vars.CONFIG_REPO_FULL_NAME }}
+    secrets: inherit
diff --git a/.github/workflows/manual-deploy-workshop.yaml b/.github/workflows/manual-deploy-workshop.yaml
@@ -16,7 +16,7 @@ jobs:
       dockerfile: Dockerfile
     secrets: inherit
 
-  deploy:
+  deploy-workshop-default:
     needs: build-unstable
     name: Deploy to workshop
     uses: mlibrary/platform-engineering-workflows/.github/workflows/deploy.yml@v1
@@ -27,3 +27,15 @@ jobs:
       CONFIG_REPO_RW_INSTALL_ID: ${{ vars.CONFIG_REPO_RW_INSTALL_ID }}
       CONFIG_REPO_FULL_NAME: ${{ vars.CONFIG_REPO_FULL_NAME }}
     secrets: inherit
+
+  deploy-workshop-hathifiles:
+    needs: build-unstable
+    name: Deploy to workshop
+    uses: mlibrary/platform-engineering-workflows/.github/workflows/deploy.yml@v1
+    with:
+      image: ${{ needs.build-unstable.outputs.image }}
+      file: environments/hathifiles/workshop/app-image.txt
+      CONFIG_REPO_RW_APP_ID: ${{ vars.CONFIG_REPO_RW_APP_ID }}
+      CONFIG_REPO_RW_INSTALL_ID: ${{ vars.CONFIG_REPO_RW_INSTALL_ID }}
+      CONFIG_REPO_FULL_NAME: ${{ vars.CONFIG_REPO_FULL_NAME }}
+    secrets: inherit
diff --git a/Gemfile b/Gemfile
@@ -5,11 +5,17 @@ source "https://rubygems.pkg.github.com/mlibrary" do
   gem "sftp"
 end
 
+gem "hathifiles_database",
+  git: "https://github.com/hathitrust/hathifiles_database",
+  branch: "main"
+
 gem "thor"
 gem "twilio-ruby"
 gem "telephone_number"
 gem "yabeda"
 gem "yabeda-prometheus"
+gem "canister"
+gem "semantic_logger"
 
 group :development, :test do
   gem "standard"
@@ -18,4 +24,5 @@ group :development, :test do
   gem "pry-byebug"
   gem "webmock"
   gem "simplecov"
+  gem "climate_control"
 end
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,121 +1,199 @@
+GIT
+  remote: https://github.com/hathitrust/hathifiles_database
+  revision: a56b4177f7988dbbb0d5f7311c73cd7ff01b681f
+  branch: main
+  specs:
+    hathifiles_database (0.1.0)
+      date_named_file
+      dotenv
+      ettin
+      hanami-cli
+      library_stdnums
+      mysql2
+      sequel
+      sqlite3
+      tty-prompt
+
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.0.4.3)
+    activesupport (7.1.2)
+      base64
+      bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
+      mutex_m
       tzinfo (~> 2.0)
-    addressable (2.8.3)
+    addressable (2.8.6)
       public_suffix (>= 2.0.2, < 6.0)
-    anyway_config (2.4.2)
-      ruby-next-core (>= 0.14.0)
+    anyway_config (2.6.1)
+      ruby-next-core (~> 1.0)
     ast (2.4.2)
+    base64 (0.2.0)
+    bigdecimal (3.1.5)
     byebug (11.1.3)
+    canister (0.9.2)
+    climate_control (1.2.0)
     coderay (1.1.3)
     concurrent-ruby (1.2.2)
+    connection_pool (2.4.1)
     crack (0.4.5)
       rexml
+    date_named_file (0.1.1)
+    deep_merge (1.2.2)
     diff-lcs (1.5.0)
     docile (1.4.0)
+    dotenv (2.8.1)
+    drb (2.2.0)
+      ruby2_keywords
+    dry-cli (1.0.0)
+    dry-files (1.1.0)
+    dry-inflector (1.0.0)
     dry-initializer (3.1.1)
-    faraday (2.7.4)
+    ettin (1.3.0)
+      deep_merge
+    faraday (2.8.1)
+      base64
       faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
     faraday-net_http (3.0.2)
-    faraday-retry (2.1.0)
+    faraday-retry (2.2.0)
       faraday (~> 2.0)
-    hashdiff (1.0.1)
-    http-2-next (0.5.1)
-    httpx (0.22.5)
-      http-2-next (>= 0.4.1)
-    i18n (1.12.0)
+    hanami-cli (2.0.3)
+      bundler (~> 2.1)
+      dry-cli (~> 1.0, < 2)
+      dry-files (~> 1.0, >= 1.0.1, < 2)
+      dry-inflector (~> 1.0, < 2)
+      rake (~> 13.0)
+      zeitwerk (~> 2.6)
+    hashdiff (1.1.0)
+    http-2-next (1.0.2)
+    httpx (1.2.0)
+      http-2-next (>= 1.0.1)
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    json (2.6.3)
+    json (2.7.1)
     jwt (2.7.1)
     language_server-protocol (3.17.0.3)
+    library_stdnums (1.6.0)
+    lint_roller (1.1.0)
     method_source (1.0.0)
-    minitest (5.18.0)
-    nokogiri (1.15.3-x86_64-linux)
+    minitest (5.20.0)
+    mutex_m (0.2.0)
+    mysql2 (0.5.5)
+    nokogiri (1.16.0-x86_64-linux)
       racc (~> 1.4)
-    parallel (1.22.1)
-    parser (3.2.2.0)
+    parallel (1.24.0)
+    parser (3.3.0.0)
       ast (~> 2.4.1)
-    prometheus-client (2.1.0)
+      racc
+    pastel (0.8.0)
+      tty-color (~> 0.5)
+    prometheus-client (4.2.2)
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
     pry-byebug (3.10.1)
       byebug (~> 11.0)
       pry (>= 0.13, < 0.15)
-    public_suffix (5.0.1)
-    racc (1.7.1)
+    public_suffix (5.0.4)
+    racc (1.7.3)
     rack (3.0.8)
     rainbow (3.1.1)
-    regexp_parser (2.7.0)
-    rexml (3.2.5)
+    rake (13.1.0)
+    regexp_parser (2.8.3)
+    rexml (3.2.6)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
       rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.1)
+    rspec-core (3.12.2)
       rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.2)
+    rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.5)
+    rspec-mocks (3.12.6)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-support (3.12.0)
-    rubocop (1.48.1)
+    rspec-support (3.12.1)
+    rubocop (1.59.0)
       json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.2.0.0)
+      parser (>= 3.2.2.4)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.26.0, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.28.0)
+    rubocop-ast (1.30.0)
       parser (>= 3.2.1.0)
-    rubocop-performance (1.16.0)
-      rubocop (>= 1.7.0, < 2.0)
-      rubocop-ast (>= 0.4.0)
-    ruby-next-core (0.15.3)
+    rubocop-performance (1.20.1)
+      rubocop (>= 1.48.1, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
+    ruby-next-core (1.0.0)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
+    semantic_logger (4.15.0)
+      concurrent-ruby (~> 1.0)
+    sequel (5.76.0)
+      bigdecimal
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    standard (1.26.0)
+    sqlite3 (1.7.0-x86_64-linux)
+    standard (1.33.0)
       language_server-protocol (~> 3.17.0.2)
-      rubocop (~> 1.48.1)
-      rubocop-performance (~> 1.16.0)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.59.0)
+      standard-custom (~> 1.0.0)
+      standard-performance (~> 1.3)
+    standard-custom (1.0.2)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.50)
+    standard-performance (1.3.0)
+      lint_roller (~> 1.1)
+      rubocop-performance (~> 1.20.1)
     telephone_number (1.4.20)
-    thor (1.2.2)
-    twilio-ruby (6.3.0)
+    thor (1.3.0)
+    tty-color (0.6.0)
+    tty-cursor (0.7.1)
+    tty-prompt (0.23.1)
+      pastel (~> 0.8)
+      tty-reader (~> 0.8)
+    tty-reader (0.9.0)
+      tty-cursor (~> 0.7)
+      tty-screen (~> 0.8)
+      wisper (~> 2.0)
+    tty-screen (0.8.2)
+    twilio-ruby (6.9.0)
       faraday (>= 0.9, < 3.0)
       jwt (>= 1.5, < 3.0)
       nokogiri (>= 1.6, < 2.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.4.2)
-    webmock (3.18.1)
+    unicode-display_width (2.5.0)
+    webmock (3.19.1)
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    yabeda (0.11.0)
+    wisper (2.0.1)
+    yabeda (0.12.0)
       anyway_config (>= 1.0, < 3)
       concurrent-ruby
       dry-initializer
-    yabeda-prometheus (0.8.0)
-      prometheus-client (>= 0.10, < 3.0)
+    yabeda-prometheus (0.9.1)
+      prometheus-client (>= 3.0, < 5.0)
       rack
       yabeda (~> 0.10)
+    zeitwerk (2.6.12)
 
 GEM
   remote: https://rubygems.pkg.github.com/mlibrary/
@@ -133,9 +211,13 @@ PLATFORMS
 
 DEPENDENCIES
   alma_rest_client (~> 2.0)!
+  canister
+  climate_control
+  hathifiles_database!
   pry
   pry-byebug
   rspec
+  semantic_logger
   sftp!
   simplecov
   standard