add server tests

demml · Nov 27, 2024 · c3e117d · c3e117d
1 parent e1bf7b9
commit c3e117d
Show file tree

Hide file tree

Showing 10 changed files with 241 additions and 114 deletions.
diff --git a/.github/workflows/storage-tests-client.yml b/.github/workflows/storage-tests-client.yml
@@ -75,7 +75,7 @@ jobs:
       - name: Tests
         working-directory: ./crates
         run: |
-          make test.storage.aws.http
+          make test.storage.aws.client
         env:
           RUST_BACKTRACE: 1
           OPSML_TRACKING_URI: http://0.0.0.0:3000
@@ -99,9 +99,6 @@ jobs:
       - name: Set up Rust
         run: rustup override set stable && rustup update
 
-      - name: Install clippy
-        run: rustup component add clippy
-
       - name: Cache Rust
         uses: Swatinem/rust-cache@v2
 
@@ -123,7 +120,7 @@ jobs:
       - name: Tests
         working-directory: ./crates
         run: |
-          make test.storage.gcs.http
+          make test.storage.gcs.client
         env:
           RUST_BACKTRACE: 1
           OPSML_TRACKING_URI: http://0.0.0.0:3000
@@ -154,9 +151,6 @@ jobs:
       - name: Set up Rust
         run: rustup override set stable && rustup update
 
-      - name: Install clippy
-        run: rustup component add clippy
-
       - name: Cache Rust
         uses: Swatinem/rust-cache@v2
 
@@ -178,7 +172,48 @@ jobs:
       - name: Tests
         working-directory: ./crates
         run: |
-          make test.storage.azure.http
+          make test.storage.azure.client
+        env:
+          RUST_BACKTRACE: 1
+          OPSML_TRACKING_URI: http://0.0.0.0:3000
+
+      - name: Shutdown background processes
+        run: |
+          lsof -ti tcp:3000 | xargs kill -9
+
+  local-client-test:
+    needs: azure-client-test
+    permissions:
+      id-token: write
+      contents: read
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Rust
+        run: rustup override set stable && rustup update
+
+      - name: Cache Rust
+        uses: Swatinem/rust-cache@v2
+
+      - name: Download opsml_server binary
+        uses: actions/download-artifact@v4
+        with:
+          name: opsml_server
+          path: target/debug/
+
+      - name: Make opsml_server executable
+        run: chmod +x target/debug/opsml-server
+
+      - name: Run opsml_server in the background
+        run: target/debug/opsml-server &
+        env:
+          OPSML_SERVER_PORT: ${{ env.OPSML_SERVER_PORT }}
+
+      - name: Tests
+        working-directory: ./crates
+        run: |
+          make test.storage.local.client
         env:
           RUST_BACKTRACE: 1
           OPSML_TRACKING_URI: http://0.0.0.0:3000

diff --git a/.github/workflows/storage-tests-server.yml b/.github/workflows/storage-tests-server.yml
@@ -0,0 +1,143 @@
+name: Server Storage Tests
+
+on:
+  push:
+    branches:
+      - main
+      - master
+  pull_request:
+
+env:
+  RUSTFLAGS: -C debuginfo=0 # Do not produce debug symbols to keep memory usage down
+
+  # aws args
+  AWS_REGION: "${{ secrets.AWS_REGION }}"
+  PYTHON_VERSION: "3.11"
+  AWS_CLOUD_BUCKET_NAME: "${{ secrets.AWS_CLOUD_BUCKET_NAME }}"
+
+  # gcs args
+  GOOGLE_ACCOUNT_JSON_BASE64: ${{ secrets.GOOGLE_ACCOUNT_JSON_BASE64 }}
+  GCS_CLOUD_BUCKET_NAME: "${{ secrets.GCS_CLOUD_BUCKET_NAME }}"
+
+  #azure args
+  AZURE_STORAGE_ACCOUNT: ${{ secrets.AZURE_STORAGE_ACCOUNT }}
+  AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+  AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+  AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
+  AZURE_CLOUD_BUCKET_NAME: "${{ secrets.AZURE_CLOUD_BUCKET_NAME }}"
+
+jobs:
+  aws-lints-client-test:
+    permissions:
+      id-token: write
+      contents: read
+    env:
+      OPSML_STORAGE_URI: s3://${{ secrets.AWS_CLOUD_BUCKET_NAME }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: configure aws credentials
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Set up Rust
+        run: rustup override set stable && rustup update
+
+      - name: Install clippy
+        run: rustup component add clippy
+
+      - name: Cache Rust
+        uses: Swatinem/rust-cache@v2
+
+      - name: Run cargo clippy
+        working-directory: ./crates
+        run: cargo clippy --workspace --all-targets -- -D warnings
+
+      - name: Tests
+        working-directory: ./crates
+        run: |
+          make test.storage.aws.server
+        env:
+          RUST_BACKTRACE: 1
+
+  gcs-client-test:
+    permissions:
+      id-token: write
+      contents: read
+    env:
+      OPSML_STORAGE_URI: gs://${{ secrets.GCS_CLOUD_BUCKET_NAME }}
+      GOOGLE_ACCOUNT_JSON_BASE64: ${{ secrets.GOOGLE_ACCOUNT_JSON_BASE64 }}
+    runs-on: ubuntu-latest
+    needs: aws-lints-client-test
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Cache Rust
+        uses: Swatinem/rust-cache@v2
+
+      - name: Set up Rust
+        run: rustup override set stable && rustup update
+
+      - name: Tests
+        working-directory: ./crates
+        run: |
+          make test.storage.gcs.server
+        env:
+          RUST_BACKTRACE: 1
+
+  azure-client-test:
+    needs: gcs-client-test
+    permissions:
+      id-token: write
+      contents: read
+    env:
+      OPSML_STORAGE_URI: az://${{ secrets.AZURE_CLOUD_BUCKET_NAME }}
+      AZURE_STORAGE_ACCOUNT: ${{ secrets.AZURE_STORAGE_ACCOUNT }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Cache Rust
+        uses: Swatinem/rust-cache@v2
+
+      - name: "Az CLI login"
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+      - name: Set up Rust
+        run: rustup override set stable && rustup update
+
+      - name: Tests
+        working-directory: ./crates
+        run: |
+          make test.storage.azure.server
+        env:
+          RUST_BACKTRACE: 1
+
+  local-client-test:
+    needs: azure-client-test
+    permissions:
+      id-token: write
+      contents: read
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Rust
+        run: rustup override set stable && rustup update
+
+      - name: Cache Rust
+        uses: Swatinem/rust-cache@v2
+
+      - name: Tests
+        working-directory: ./crates
+        run: |
+          make test.storage.local.server
+        env:
+          RUST_BACKTRACE: 1
diff --git a/crates/makefile b/crates/makefile
@@ -1,19 +1,3 @@
-.PHONY: test.storage.gcs.http
-test.storage.gcs.http:
-	cargo test -p opsml-storage test_filesystemstorage_with_http_google -- --nocapture
-
-.PHONY: test.storage.aws.http
-test.storage.aws.http:
-	cargo test -p opsml-storage test_filesystemstorage_with_http_aws -- --nocapture
-
-.PHONY: test.storage.local.http
-test.storage.local.http:
-	cargo test -p opsml-storage test_filesystemstorage_with_http_local -- --nocapture
-
-.PHONY: test.storage.azure.http
-test.storage.azure.http:
-	cargo test -p opsml-storage test_filesystemstorage_with_http_azure -- --nocapture
-
 .PHONY: test.storage.gcs.client
 test.storage.gcs.client:
 	cargo test -p opsml-storage test_gcs_storage_client -- --nocapture
@@ -22,13 +6,29 @@ test.storage.gcs.client:
 test.storage.aws.client:
 	cargo test -p opsml-storage test_aws_storage_client -- --nocapture
 
-.PHONY: test.storage.az.client
-test.storage.az.client:
-	cargo test -p opsml-storage test_azure_storage_client -- --nocapture
+.PHONY: test.storage.azure.client
+test.storage.azure.client:
+	cargo test -p opsml-storage test_azure_storage_client  -- --nocapture
 
 .PHONY: test.storage.local.client
 test.storage.local.client:
-	cargo test -p opsml-storage test_local_storage_client -- --nocapture --test-threads 1
+	cargo test -p opsml-storage test_local_storage_client -- --nocapture
+
+.PHONY: test.storage.gcs.server
+test.storage.gcs.server:
+	cargo test -p opsml-storage test_gcs_storage_server -- --nocapture
+
+.PHONY: test.storage.aws.server
+test.storage.aws.server:
+	cargo test -p opsml-storage test_aws_storage_server -- --nocapture
+
+.PHONY: test.storage.azure.server
+test.storage.azure.server:
+	cargo test -p opsml-storage test_azure_storage_server -- --nocapture
+
+.PHONY: test.storage.local.server
+test.storage.local.server:
+	cargo test -p opsml-storage test_local_storage_server -- --nocapture --test-threads 1
 
 .PHONY: lints
 lints:

diff --git a/crates/opsml_storage/src/storage/aws/client.rs b/crates/opsml_storage/src/storage/aws/client.rs
@@ -943,7 +943,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_aws_storage_client() -> Result<(), StorageError> {
+    async fn test_aws_storage_server() -> Result<(), StorageError> {
         let rand_name = uuid::Uuid::new_v4().to_string();
         let filename = format!("file-{}.txt", rand_name);
 
@@ -1016,7 +1016,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_aws_storage_client_trees() -> Result<(), StorageError> {
+    async fn test_aws_storage_server_trees() -> Result<(), StorageError> {
         let rand_name = uuid::Uuid::new_v4().to_string();
 
         let tmp_dir = TempDir::new().unwrap();

diff --git a/crates/opsml_storage/src/storage/azure/client.rs b/crates/opsml_storage/src/storage/azure/client.rs
@@ -705,7 +705,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_azure_storage_client() -> Result<(), StorageError> {
+    async fn test_azure_storage_server() -> Result<(), StorageError> {
         let rand_name = uuid::Uuid::new_v4().to_string();
         let filename = format!("file-{}.txt", rand_name);
 
@@ -778,7 +778,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_azure_storage_client_trees() -> Result<(), StorageError> {
+    async fn test_azure_storage_server_trees() -> Result<(), StorageError> {
         let rand_name = uuid::Uuid::new_v4().to_string();
 
         let tmp_dir = TempDir::new().unwrap();