Skip to content

Commit

Permalink
Improve type checking fot generic datastore
Browse files Browse the repository at this point in the history
  • Loading branch information
andy-slac committed Sep 13, 2023
1 parent d3e68c3 commit 4c9f3eb
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 12 deletions.
2 changes: 1 addition & 1 deletion python/lsst/daf/butler/datastores/fileDatastore.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ class DatastoreFileGetInformation:
"""The `StorageClass` of the dataset being read."""


class FileDatastore(GenericBaseDatastore):
class FileDatastore(GenericBaseDatastore[StoredFileInfo]):
"""Generic Datastore for file-based implementations.
Should always be sub-classed since key abstract methods are missing.
Expand Down
27 changes: 17 additions & 10 deletions python/lsst/daf/butler/datastores/genericDatastore.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,21 +33,23 @@

import logging
from abc import abstractmethod
from collections.abc import Iterable, Mapping, Sequence
from typing import TYPE_CHECKING, Any
from collections.abc import Iterable, Mapping
from typing import TYPE_CHECKING, Any, Generic, TypeVar

from lsst.daf.butler import DatasetTypeNotSupportedError, Datastore
from lsst.daf.butler import DatasetTypeNotSupportedError, Datastore, StoredDatastoreItemInfo
from lsst.daf.butler.registry.interfaces import DatastoreRegistryBridge

from ..registry.interfaces import DatabaseInsertMode

if TYPE_CHECKING:
from lsst.daf.butler import DatasetRef, StorageClass, StoredDatastoreItemInfo
from lsst.daf.butler import DatasetRef, StorageClass

log = logging.getLogger(__name__)

_InfoType = TypeVar("_InfoType", bound=StoredDatastoreItemInfo)

class GenericBaseDatastore(Datastore):

class GenericBaseDatastore(Datastore, Generic[_InfoType]):
"""Methods useful for most implementations of a `Datastore`.
Should always be sub-classed since key abstract methods are missing.
Expand Down Expand Up @@ -86,7 +88,7 @@ def addStoredItemInfo(
raise NotImplementedError()

@abstractmethod
def getStoredItemsInfo(self, ref: DatasetRef) -> Sequence[Any]:
def getStoredItemsInfo(self, ref: DatasetRef) -> Iterable[_InfoType]:
"""Retrieve information associated with files stored in this
`Datastore` associated with this dataset ref.
Expand All @@ -97,7 +99,7 @@ def getStoredItemsInfo(self, ref: DatasetRef) -> Sequence[Any]:
Returns
-------
items : `list` [`StoredDatastoreItemInfo`]
items : `~collections.abc.Iterable` [`StoredDatastoreItemInfo`]
Stored information about the files and associated formatters
associated with this dataset. Only one file will be returned
if the dataset has not been disassembled. Can return an empty
Expand Down Expand Up @@ -156,11 +158,11 @@ def _register_datasets(

def _post_process_get(
self,
inMemoryDataset: Any,
inMemoryDataset: object,
readStorageClass: StorageClass,
assemblerParams: Mapping[str, Any] | None = None,
isComponent: bool = False,
) -> Any:
) -> object:
"""Given the Python object read from the datastore, manipulate
it based on the supplied parameters and ensure the Python
type is correct.
Expand All @@ -176,6 +178,11 @@ def _post_process_get(
Parameters to pass to the assembler. Can be `None`.
isComponent : `bool`, optional
If this is a component, allow the inMemoryDataset to be `None`.
Returns
-------
dataset : `object`
In-memory dataset, potentially converted to expected type.
"""
# Process any left over parameters
if assemblerParams:
Expand All @@ -197,7 +204,7 @@ def _post_process_get(

return inMemoryDataset

def _validate_put_parameters(self, inMemoryDataset: Any, ref: DatasetRef) -> None:
def _validate_put_parameters(self, inMemoryDataset: object, ref: DatasetRef) -> None:
"""Validate the supplied arguments for put.
Parameters
Expand Down
2 changes: 1 addition & 1 deletion python/lsst/daf/butler/datastores/inMemoryDatastore.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ class StoredMemoryItemInfo(StoredDatastoreItemInfo):
"""


class InMemoryDatastore(GenericBaseDatastore):
class InMemoryDatastore(GenericBaseDatastore[StoredMemoryItemInfo]):
"""Basic Datastore for writing to an in memory cache.
This datastore is ephemeral in that the contents of the datastore
Expand Down

0 comments on commit 4c9f3eb

Please sign in to comment.