Auth with gcp to use Native GCS client

Eventual-Inc · Sep 21, 2023 · 438d965 · 438d965
1 parent ce9fa8a
commit 438d965
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 13 deletions.
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -258,6 +258,11 @@ jobs:
         aws-region: us-west-2
         role-to-assume: ${{ secrets.ACTIONS_AWS_ROLE_ARN }}
         role-session-name: DaftPythonPackageGitHubWorkflow
+    - name: Assume GitHub Actions GCloud Credentials
+      uses: google-github-actions/auth@v1
+      with:
+        workload_identity_provider: projects/511003941844/locations/global/workloadIdentityPools/github/providers/github-ci
+        service_account: github-integration-tests@compact-harbor-353622.iam.gserviceaccount.com
     - name: Spin up IO services
       uses: isbang/[email protected]
       with:

diff --git a/tests/integration/io/test_list_files_gcs.py b/tests/integration/io/test_list_files_gcs.py
@@ -68,16 +68,16 @@ def test_gs_notfound():
     assert daft_ls_result == []
 
 
-# @pytest.mark.integration()
-# @pytest.mark.parametrize(
-#     "path",
-#     [
-#         f"gs://{BUCKET}/test_ls",
-#         f"gs://{BUCKET}/test_ls/",
-#     ],
-# )
-# def test_gs_flat_directory_listing_recursive(path):
-#     fs = gcsfs.GCSFileSystem()
-#     daft_ls_result = io_list(path, recursive=True)
-#     fsspec_result = list(fs.glob(path.rstrip("/") + "/**", detail=True).values())
-#     compare_gcs_result(daft_ls_result, fsspec_result)
+@pytest.mark.integration()
+@pytest.mark.parametrize(
+    "path",
+    [
+        f"gs://{BUCKET}/test_ls",
+        f"gs://{BUCKET}/test_ls/",
+    ],
+)
+def test_gs_flat_directory_listing_recursive(path):
+    fs = gcsfs.GCSFileSystem()
+    daft_ls_result = io_list(path, recursive=True)
+    fsspec_result = list(fs.glob(path.rstrip("/") + "/**", detail=True).values())
+    compare_gcs_result(daft_ls_result, fsspec_result)