From 215eb1bfb23fe2ed4a88dd18dfaf2de8e97bc020 Mon Sep 17 00:00:00 2001 From: Timo Strunk Date: Tue, 5 Mar 2024 12:39:19 +0100 Subject: [PATCH] Enabled support for colon character in HAzureBlockBlobStore Issue: #119 --- docs/changes.rst | 1 + minimalkv/_constants.py | 8 ++++++++ minimalkv/_hstores.py | 4 ++-- minimalkv/_mixins.py | 38 ++++++++++++++++++++++++++++++++--- minimalkv/contrib/__init__.py | 2 ++ tests/test_azure_store.py | 19 +++++++++++++----- 6 files changed, 62 insertions(+), 10 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 18694a2a..1c99d59d 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -5,6 +5,7 @@ Changelog ===== * Add `session_token` url param that can be set when creating a `[h]s3://` store via `get_store_from_url`. +* Allow colons in filenames for `HAzureBlockBlobStore` 1.8.6 ===== diff --git a/minimalkv/_constants.py b/minimalkv/_constants.py index d91bea55..de10931a 100644 --- a/minimalkv/_constants.py +++ b/minimalkv/_constants.py @@ -17,3 +17,11 @@ Allowed are all alphanumeric characters, as well as ``!"`#$%&'()+,-.<=>?@[]^_{}~/``. and spaces""" VALID_KEY_RE_EXTENDED = re.compile(VALID_KEY_REGEXP_EXTENDED) """A compiled version of :data:`~minimalkv._constants.VALID_KEY_REGEXP_EXTENDED`.""" + +VALID_KEY_REGEXP_COLON_EXTENDED = "^[%s0-9a-zA-Z:]+$" % re.escape( + VALID_NON_NUM_EXTENDED +) +"""This regular expression is the same as :data:`~minimalkv._constants.VALID_KEY_REGEXP_EXTENDED` +but also allows a colon.""" +VALID_KEY_RE_COLON_EXTENDED = re.compile(VALID_KEY_REGEXP_COLON_EXTENDED) +"""A compiled version of :data:`~minimalkv._constants.VALID_KEY_REGEXP_COLON_EXTENDED`.""" diff --git a/minimalkv/_hstores.py b/minimalkv/_hstores.py index 88589f0f..e5a1a5b8 100644 --- a/minimalkv/_hstores.py +++ b/minimalkv/_hstores.py @@ -1,6 +1,6 @@ import os -from minimalkv._mixins import ExtendedKeyspaceMixin +from minimalkv._mixins import ExtendedKeyspaceMixin, ExtendedKeyspaceMixinColon from minimalkv.fs import FilesystemStore from minimalkv.memory import DictStore from minimalkv.memory.redisstore import RedisStore @@ -19,7 +19,7 @@ class HRedisStore(ExtendedKeyspaceMixin, RedisStore): # noqa D pass -class HAzureBlockBlobStore(ExtendedKeyspaceMixin, AzureBlockBlobStore): # noqa D +class HAzureBlockBlobStore(ExtendedKeyspaceMixinColon, AzureBlockBlobStore): # noqa D pass diff --git a/minimalkv/_mixins.py b/minimalkv/_mixins.py index 71bee5f3..2d29abf5 100644 --- a/minimalkv/_mixins.py +++ b/minimalkv/_mixins.py @@ -1,7 +1,14 @@ +import abc +import re from io import BytesIO from typing import BinaryIO, Callable, Optional, Union -from minimalkv._constants import FOREVER, NOT_SET, VALID_KEY_RE_EXTENDED +from minimalkv._constants import ( + FOREVER, + NOT_SET, + VALID_KEY_RE_COLON_EXTENDED, + VALID_KEY_RE_EXTENDED, +) class UrlMixin: @@ -387,7 +394,7 @@ def _move(self, source: str, dest: str) -> str: return dest -class ExtendedKeyspaceMixin: +class ExtendedKeyspaceMixinBase: """A mixin to extend the keyspace to allow slashes and spaces in keynames. Attention: A single / is NOT allowed. @@ -397,6 +404,18 @@ class ExtendedKeyspaceMixin: """ + @property + @abc.abstractmethod + def VALID_KEY_RE(self) -> re.Pattern: + """Method returning a compiled regular expression to validate the key. + + Returns + ------- + re.Pattern: + Expression to validate against + """ + raise NotImplementedError("Implement this property in child classes") + def _check_valid_key(self, key: Optional[str]) -> None: """Check if a key is valid and raises a ValueError if its not. @@ -412,5 +431,18 @@ def _check_valid_key(self, key: Optional[str]) -> None: if key is not None: if not isinstance(key, str): raise ValueError("%r is not a valid key type" % key) - elif not VALID_KEY_RE_EXTENDED.match(key) or key == "/": + elif not self.VALID_KEY_RE.match(key) or key == "/": + breakpoint() raise ValueError("%r contains illegal characters" % key) + + +class ExtendedKeyspaceMixin(ExtendedKeyspaceMixinBase): + @property + def VALID_KEY_RE(self) -> re.Pattern: + return VALID_KEY_RE_EXTENDED + + +class ExtendedKeyspaceMixinColon(ExtendedKeyspaceMixinBase): + @property + def VALID_KEY_RE(self) -> re.Pattern: + return VALID_KEY_RE_COLON_EXTENDED diff --git a/minimalkv/contrib/__init__.py b/minimalkv/contrib/__init__.py index 41ca029e..13b13e28 100644 --- a/minimalkv/contrib/__init__.py +++ b/minimalkv/contrib/__init__.py @@ -1,4 +1,5 @@ from minimalkv._constants import ( + VALID_KEY_RE_COLON_EXTENDED, VALID_KEY_RE_EXTENDED, VALID_KEY_REGEXP_EXTENDED, VALID_NON_NUM_EXTENDED, @@ -9,5 +10,6 @@ "VALID_NON_NUM_EXTENDED", "VALID_KEY_REGEXP_EXTENDED", "VALID_KEY_RE_EXTENDED", + "VALID_KEY_RE_COLON_EXTENDED", "ExtendedKeyspaceMixin", ] diff --git a/tests/test_azure_store.py b/tests/test_azure_store.py index 0dee10f9..b4cb687e 100644 --- a/tests/test_azure_store.py +++ b/tests/test_azure_store.py @@ -7,7 +7,7 @@ from basic_store import BasicStore, OpenSeekTellStore from conftest import ExtendedKeyspaceTests -from minimalkv._mixins import ExtendedKeyspaceMixin +from minimalkv._hstores import HAzureBlockBlobStore from minimalkv.net.azurestore import AzureBlockBlobStore asb = pytest.importorskip("azure.storage.blob") @@ -86,10 +86,7 @@ def store(self): pytest.skip("Compatibility issues with azurite and azure-storage-blob<12") container = str(uuid()) - class ExtendedKeysStore(ExtendedKeyspaceMixin, AzureBlockBlobStore): - pass - - with ExtendedKeysStore( + with HAzureBlockBlobStore( conn_string=conn_string, container=container, public=False ) as store: yield store @@ -139,6 +136,18 @@ def test_azure_dangling_port_explicit_close_multi(): store.close() +@pytest.mark.filterwarnings("error") +def test_azure_colon_compatibility(): + container = str(uuid()) + conn_string = get_azure_conn_string() + with HAzureBlockBlobStore(conn_string=conn_string, container=container) as store: + if not hasattr(store, "blob_container_client"): + # This test only runs for azurestore_new + return + store.put("Test:file", b"Test data") + assert store.get("Test:file") == b"Test data" + + def test_azure_setgetstate(): container = str(uuid()) conn_string = get_azure_conn_string()