Setup CI to authenticate to AWS if in base repo.

* Add S3FSStore aws integration tests and minio tests.
* Allow skipping of aws integration tests when in fork or in local setup.

.github/workflows/ci.yml

@@ -51,6 +51,17 @@ jobs:
           workload_identity_provider: "projects/498651197656/locations/global/workloadIdentityPools/qc-minimalkv-gh-actions-pool/providers/github-actions-provider"
           service_account: "[email protected]"
           token_format: "access_token"
+      - name: Authenticate to AWS
+        if: steps.check-id-token.outcome == 'success'
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::211125346859:role/github-poweruser
+          aws-region: eu-central-1
+        # We set an env variable according to the result of the check
+        # to only allow skipping of aws integration test when in fork.
+        # When being run in the base repo, the aws integration test should always be executed.
+      - name: Check whether the workflow runs in a fork
+        run: echo "CI_IN_FORK=${{ github.event.pull_request && github.repository != github.event.pull_request.head.repo.full_name }}" >> $GITHUB_ENV
       - name: Run the unittests
         shell: bash -x -l {0}
         run: |

docs/changes.rst

@@ -1,6 +1,12 @@
 Changelog
 *********
 
+1.9.1
+=====
+* Add a real AWS integration test for S3FSStore
+* Add minio test for S3FSStore
+* `verify` url param that can be passed to url when creating a `[h]s3://` store now really controls SSL verifaction
+
 1.9.0
 =====
 * Add `session_token` url param that can be set when creating a `[h]s3://` store

tests/test_s3fs_aws.py

@@ -0,0 +1,112 @@
+import os
+from random import randint
+from typing import Tuple, Union
+from urllib.parse import quote_plus
+
+import pytest
+from boto3 import Session
+
+from minimalkv import get_store_from_url
+
+
+@pytest.fixture()
+def aws_credentials() -> Tuple[str, str, Union[str, None]]:
+    env_var_name = "AWS_PROFILE"
+    profile_name = os.environ.get(env_var_name, None)
+
+    access_key: Union[str, None] = None
+    secret_key: Union[str, None] = None
+    session_token: Union[str, None] = None
+
+    if profile_name:
+        session = Session(profile_name=profile_name)
+        aws_credentials = session.get_credentials()
+        assert aws_credentials is not None
+        access_key = aws_credentials.access_key
+        secret_key = aws_credentials.secret_key
+        session_token = aws_credentials.token
+    else:
+        access_key = os.environ.get("AWS_ACCESS_KEY_ID", None)
+        secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY", None)
+        session_token = os.environ.get("AWS_SESSION_TOKEN", None)
+
+    if not (access_key and secret_key):
+        msg = (
+            f"No s3 credentials available. Set '{env_var_name}' env variable to "
+            "provide a valid AWS profile or set 'AWS_ACCESS_KEY_ID' and "
+            "'AWS_SECRET_ACCESS_KEY' and optional 'AWS_SESSION_TOKEN'."
+        )
+
+        if "CI_IN_FORK" not in os.environ or os.environ["CI_IN_FORK"].lower() == "true":
+            # We skip if the variable is not set at all (local development)
+            # or if it is explicitely set to "true".q
+            pytest.skip(reason=msg)
+
+        # If in CI of base repo and credentials couldn't be acquired, fail test
+
+    assert access_key and secret_key
+    return (
+        access_key,
+        secret_key,
+        session_token,
+    )
+
+
+@pytest.fixture()
+def ci_bucket_name() -> str:
+    return "minimalkv-test-ci-bucket"
+
+
+@pytest.fixture()
+def ci_s3_point() -> str:
+    return "s3.eu-north-1.amazonaws.com"
+
+
+def get_s3_url(
+    access_key: str,
+    secret_key: str,
+    session_token: Union[str, None],
+    bucket_name: str,
+    s3_point: str,
+) -> str:
+    access_key = quote_plus(access_key)
+    secret_key = quote_plus(secret_key)
+    return f"hs3://{access_key}:{secret_key}@{s3_point}/{bucket_name}?force_bucket_suffix=false&create_if_missing=false&session_token={session_token}"
+
+
+@pytest.fixture()
+def test_id() -> str:
+    return f"test-id-{randint(1, 1000)}"
+
+
+def test_s3fs_aws_integration(
+    test_id,
+    aws_credentials: Tuple[str, str, Union[str, None]],
+    ci_bucket_name,
+    ci_s3_point,
+):
+    """Authenticates with AWS S3 bucket via short-lived credentials and tests basic operation of S3FSStore.
+
+    Test the basic interface:
+    - keys()
+    - put()
+    - get()
+    - delete()
+    """
+    access_key, secret_key, session_token = aws_credentials
+
+    bucket = get_store_from_url(
+        get_s3_url(access_key, secret_key, session_token, ci_bucket_name, ci_s3_point)
+    )
+
+    new_filename = f"{test_id}-folder/file"  # Testing the h of hs3
+    new_content = b"content"
+
+    assert new_filename not in bucket.keys(), "Test prerequisites not fulfilled."
+
+    bucket.put(new_filename, new_content)
+    assert new_filename in bucket.keys()
+    assert bucket.get(new_filename) == new_content
+
+    bucket.delete(new_filename)
+    assert new_filename not in bucket.keys()