Merge pull request #2 from demml/presigned

Storage Integration

.github/ISSUE_TEMPLATE/bug_report_python.yml

@@ -0,0 +1,57 @@
+name: "🐞 Bug report - Python"
+description: Report an issue with Python opsml-core.
+labels: [bug, python]
+
+body:
+  - type: checkboxes
+    id: checks
+    attributes:
+      label: Checks
+      options:
+        - label: >
+            I have checked that this issue has not already been reported.
+          required: true
+        - label: >
+            I have confirmed this bug exists on the
+            latest version.
+          required: true
+
+  - type: textarea
+    id: example
+    attributes:
+      label: Reproducible example
+      description: >
+        Please follow [this guide](https://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports) on how to
+        provide a minimal, copy-pastable example. Include the (wrong) output if applicable.
+      value: |
+        ```python
+
+        ```
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Log output
+      description: >
+        Provide any relevant log output or stack trace.
+      render: shell
+
+  - type: textarea
+    id: problem
+    attributes:
+      label: Issue description
+      description: >
+        Provide any additional information you think might be relevant.
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected-behavior
+    attributes:
+      label: Expected behavior
+      description: >
+        Describe or show a code example of the expected behavior.
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,2 @@
+# Ref: https://help.github.com/en/github/building-a-strong-community/configuring-issue-templates-for-your-repository#configuring-the-template-chooser
+blank_issues_enabled: true

.github/ISSUE_TEMPLATE/documentation.yml

@@ -0,0 +1,22 @@
+name: '📖 Documentation improvement'
+description: Report an issue with the documentation.
+labels: [documentation]
+
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: >
+        Describe the issue with the documentation and how it can be fixed or improved.
+    validations:
+      required: true
+
+  - type: input
+    id: link
+    attributes:
+      label: Link
+      description: >
+        Provide a link to the existing documentation, if applicable.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,14 @@
+name: "✨ Feature request"
+description: Suggest a new feature or enhancement for Opsml.
+labels: [enhancement]
+
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: >
+        Describe the feature or enhancement and explain why it should be implemented.
+        Include a code example if applicable.
+    validations:
+      required: true

.github/reviewer_config.yml

@@ -0,0 +1,13 @@
+# Set to true to add reviewers to pull requests
+addReviewers: true
+
+# Set to true to add assignees to pull requests
+addAssignees: false
+
+# A list of reviewers to be added to pull requests (GitHub user name)
+reviewers:
+  - thorrester
+
+# A number of reviewers added to the pull request
+# Set 0 to add all the reviewers (default: 0)
+numberOfReviewers: 1

.github/workflows/auto-assign-py.yml

@@ -0,0 +1,12 @@
+name: "Auto Assign PR"
+on:
+  pull_request:
+    types: [opened, ready_for_review]
+
+jobs:
+  add-reviewers:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: kentaro-m/[email protected]
+        with:
+          configuration-path: ".github/reviewer_config.yml"

.github/workflows/dependency-review.yml

@@ -0,0 +1,23 @@
+name: Dependency Review
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+permissions:
+  contents: read
+  # this is required to comment on PRs
+  pull-requests: write
+
+jobs:
+  Dependency-Review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
+      - name: "Dependency Review"
+        uses: actions/dependency-review-action@v3
+        with:
+          fail-on-severity: critical
+          comment-summary-in-pr: always

.github/workflows/storage-tests-client.yml

@@ -0,0 +1,223 @@
+name: Client Storage Tests
+
+on:
+  push:
+    branches:
+      - main
+      - master
+  pull_request:
+
+env:
+  RUSTFLAGS: -C debuginfo=0 # Do not produce debug symbols to keep memory usage down
+  OPSML_SERVER_PORT: 3000
+
+  # aws args
+  AWS_REGION: "${{ secrets.AWS_REGION }}"
+  PYTHON_VERSION: "3.11"
+  AWS_CLOUD_BUCKET_NAME: "${{ secrets.AWS_CLOUD_BUCKET_NAME }}"
+
+  # gcs args
+  GOOGLE_ACCOUNT_JSON_BASE64: ${{ secrets.GOOGLE_ACCOUNT_JSON_BASE64 }}
+  GCS_CLOUD_BUCKET_NAME: "${{ secrets.GCS_CLOUD_BUCKET_NAME }}"
+
+  #azure args
+  AZURE_STORAGE_ACCOUNT: ${{ secrets.AZURE_STORAGE_ACCOUNT }}
+  AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+  AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+  AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
+  AZURE_CLOUD_BUCKET_NAME: "${{ secrets.AZURE_CLOUD_BUCKET_NAME }}"
+
+jobs:
+  aws-lints-client-test:
+    permissions:
+      id-token: write
+      contents: read
+    env:
+      OPSML_STORAGE_URI: s3://${{ secrets.AWS_CLOUD_BUCKET_NAME }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: configure aws credentials
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Set up Rust
+        run: rustup override set stable && rustup update
+
+      - name: Install clippy
+        run: rustup component add clippy
+
+      - name: Cache Rust
+        uses: Swatinem/rust-cache@v2
+
+      - name: Run cargo clippy
+        working-directory: ./crates
+        run: cargo clippy --workspace --all-targets -- -D warnings
+
+      - name: Build opsml_server
+        run: cargo build -p opsml-server
+
+      - name: Upload opsml_server binary
+        uses: actions/upload-artifact@v4
+        with:
+          name: opsml_server
+          path: ./target/debug/opsml-server
+
+      - name: Run opsml_server in the background
+        run: ./target/debug/opsml-server &
+        env:
+          OPSML_STORAGE_URI: ${{ env.OPSML_STORAGE_URI }}
+          OPSML_SERVER_PORT: ${{ env.OPSML_SERVER_PORT }}
+
+      - name: Tests
+        working-directory: ./crates
+        run: |
+          make test.storage.aws.client
+        env:
+          RUST_BACKTRACE: 1
+          OPSML_TRACKING_URI: http://0.0.0.0:3000
+
+      - name: Shutdown background processes
+        run: |
+          lsof -ti tcp:3000 | xargs kill -9
+
+  gcs-client-test:
+    permissions:
+      id-token: write
+      contents: read
+    env:
+      OPSML_STORAGE_URI: gs://${{ secrets.GCS_CLOUD_BUCKET_NAME }}
+      GOOGLE_ACCOUNT_JSON_BASE64: ${{ secrets.GOOGLE_ACCOUNT_JSON_BASE64 }}
+    runs-on: ubuntu-latest
+    needs: aws-lints-client-test
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Rust
+        run: rustup override set stable && rustup update
+
+      - name: Cache Rust
+        uses: Swatinem/rust-cache@v2
+
+      - name: Download opsml_server binary
+        uses: actions/download-artifact@v4
+        with:
+          name: opsml_server
+          path: target/debug/
+
+      - name: Make opsml_server executable
+        run: chmod +x target/debug/opsml-server
+
+      - name: Run opsml_server in the background
+        run: target/debug/opsml-server &
+        env:
+          OPSML_STORAGE_URI: ${{ env.OPSML_STORAGE_URI }}
+          OPSML_SERVER_PORT: ${{ env.OPSML_SERVER_PORT }}
+
+      - name: Tests
+        working-directory: ./crates
+        run: |
+          make test.storage.gcs.client
+        env:
+          RUST_BACKTRACE: 1
+          OPSML_TRACKING_URI: http://0.0.0.0:3000
+
+      - name: Shutdown background processes
+        run: |
+          lsof -ti tcp:3000 | xargs kill -9
+
+  azure-client-test:
+    needs: gcs-client-test
+    permissions:
+      id-token: write
+      contents: read
+    env:
+      OPSML_STORAGE_URI: az://${{ secrets.AZURE_CLOUD_BUCKET_NAME }}
+      AZURE_STORAGE_ACCOUNT: ${{ secrets.AZURE_STORAGE_ACCOUNT }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: "Az CLI login"
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+      - name: Set up Rust
+        run: rustup override set stable && rustup update
+
+      - name: Cache Rust
+        uses: Swatinem/rust-cache@v2
+
+      - name: Download opsml_server binary
+        uses: actions/download-artifact@v4
+        with:
+          name: opsml_server
+          path: target/debug/
+
+      - name: Make opsml_server executable
+        run: chmod +x target/debug/opsml-server
+
+      - name: Run opsml_server in the background
+        run: target/debug/opsml-server &
+        env:
+          OPSML_STORAGE_URI: ${{ env.OPSML_STORAGE_URI }}
+          OPSML_SERVER_PORT: ${{ env.OPSML_SERVER_PORT }}
+
+      - name: Tests
+        working-directory: ./crates
+        run: |
+          make test.storage.azure.client
+        env:
+          RUST_BACKTRACE: 1
+          OPSML_TRACKING_URI: http://0.0.0.0:3000
+
+      - name: Shutdown background processes
+        run: |
+          lsof -ti tcp:3000 | xargs kill -9
+
+  local-client-test:
+    needs: azure-client-test
+    permissions:
+      id-token: write
+      contents: read
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Rust
+        run: rustup override set stable && rustup update
+
+      - name: Cache Rust
+        uses: Swatinem/rust-cache@v2
+
+      - name: Download opsml_server binary
+        uses: actions/download-artifact@v4
+        with:
+          name: opsml_server
+          path: target/debug/
+
+      - name: Make opsml_server executable
+        run: chmod +x target/debug/opsml-server
+
+      - name: Run opsml_server in the background
+        run: target/debug/opsml-server &
+        env:
+          OPSML_SERVER_PORT: ${{ env.OPSML_SERVER_PORT }}
+
+      - name: Tests
+        working-directory: ./crates
+        run: |
+          make test.storage.local.client
+        env:
+          RUST_BACKTRACE: 1
+          OPSML_TRACKING_URI: http://0.0.0.0:3000
+
+      - name: Shutdown background processes
+        run: |
+          lsof -ti tcp:3000 | xargs kill -9