diff --git a/CHANGES.rst b/CHANGES.rst index 96d20a31..5af9e970 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -30,6 +30,10 @@ Version 3.X.X (2019-09-XX) * `~kartothek.serialization.ConjunctionType` * `~kartothek.serialization.LiteralType` +Internal changes +^^^^^^^^^^^^^^^^ +- Move the docs module from `io_components` to `core` + Version 3.4.0 (2019-09-17) ========================== diff --git a/kartothek/core/dataset.py b/kartothek/core/dataset.py index 703b216f..f223d7ba 100644 --- a/kartothek/core/dataset.py +++ b/kartothek/core/dataset.py @@ -15,6 +15,7 @@ from kartothek.core._compat import load_json from kartothek.core._mixins import CopyMixin from kartothek.core.common_metadata import SchemaWrapper, read_schema_metadata +from kartothek.core.docs import default_docs from kartothek.core.index import ( ExplicitSecondaryIndex, IndexBase, @@ -349,6 +350,7 @@ def load_partition_indices(self) -> "DatasetMetadataBase": combined_indices.update(indices) return self.copy(indices=combined_indices) + @default_docs def get_indices_as_dataframe( self, columns: Optional[List[str]] = None, @@ -370,11 +372,6 @@ def get_indices_as_dataframe( Parameters ---------- - columns: list of str - If provided, the dataframe will only be constructed for the provided columns/indices. - If `None` is given, all indices are included. - date_as_object: bool, optional - Cast dates to objects. """ if columns is None: columns = sorted(self.indices.keys()) diff --git a/kartothek/io_components/docs.py b/kartothek/core/docs.py similarity index 100% rename from kartothek/io_components/docs.py rename to kartothek/core/docs.py diff --git a/kartothek/core/index.py b/kartothek/core/index.py index 5280a957..127d836f 100644 --- a/kartothek/core/index.py +++ b/kartothek/core/index.py @@ -15,6 +15,7 @@ from kartothek.core._compat import ARROW_LARGER_EQ_0150 from kartothek.core._mixins import CopyMixin from kartothek.core.common_metadata import normalize_type +from kartothek.core.docs import default_docs from kartothek.core.urlencode import quote from kartothek.serialization import ( PredicatesType, @@ -427,6 +428,7 @@ def __eq__(self, other) -> bool: def __ne__(self, other) -> bool: return not (self == other) + @default_docs def as_flat_series( self, compact: bool = False, diff --git a/kartothek/io/dask/bag.py b/kartothek/io/dask/bag.py index 6ab86d89..bed2cc90 100644 --- a/kartothek/io/dask/bag.py +++ b/kartothek/io/dask/bag.py @@ -5,6 +5,7 @@ import dask.bag as db from kartothek.core import naming +from kartothek.core.docs import default_docs from kartothek.core.factory import _ensure_factory from kartothek.core.utils import _check_callable from kartothek.core.uuid import gen_uuid @@ -14,7 +15,6 @@ _identity, _maybe_get_categoricals_from_index, ) -from kartothek.io_components.docs import default_docs from kartothek.io_components.index import update_indices_from_partitions from kartothek.io_components.metapartition import ( MetaPartition, diff --git a/kartothek/io/dask/dataframe.py b/kartothek/io/dask/dataframe.py index 71cf265c..bfc8c30e 100644 --- a/kartothek/io/dask/dataframe.py +++ b/kartothek/io/dask/dataframe.py @@ -6,9 +6,9 @@ import numpy as np from kartothek.core.common_metadata import empty_dataframe_from_schema +from kartothek.core.docs import default_docs from kartothek.core.factory import _ensure_factory from kartothek.core.naming import DEFAULT_METADATA_VERSION -from kartothek.io_components.docs import default_docs from kartothek.io_components.metapartition import parse_input_to_metapartition from kartothek.io_components.update import update_dataset_from_partitions from kartothek.io_components.utils import ( diff --git a/kartothek/io/dask/delayed.py b/kartothek/io/dask/delayed.py index 065d7717..eb684752 100644 --- a/kartothek/io/dask/delayed.py +++ b/kartothek/io/dask/delayed.py @@ -8,6 +8,7 @@ from dask import delayed from kartothek.core import naming +from kartothek.core.docs import default_docs from kartothek.core.factory import _ensure_factory from kartothek.core.naming import DEFAULT_METADATA_VERSION from kartothek.core.utils import _check_callable @@ -17,7 +18,6 @@ delete_indices, delete_top_level_metadata, ) -from kartothek.io_components.docs import default_docs from kartothek.io_components.gc import delete_files, dispatch_files_to_gc from kartothek.io_components.merge import align_datasets from kartothek.io_components.metapartition import ( diff --git a/kartothek/io/eager.py b/kartothek/io/eager.py index 08755c27..37f3e076 100644 --- a/kartothek/io/eager.py +++ b/kartothek/io/eager.py @@ -11,6 +11,7 @@ store_schema_metadata, ) from kartothek.core.dataset import DatasetMetadataBuilder +from kartothek.core.docs import default_docs from kartothek.core.factory import _ensure_factory from kartothek.core.naming import ( DEFAULT_METADATA_STORAGE_FORMAT, @@ -24,7 +25,6 @@ delete_indices, delete_top_level_metadata, ) -from kartothek.io_components.docs import default_docs from kartothek.io_components.gc import delete_files, dispatch_files_to_gc from kartothek.io_components.index import update_indices_from_partitions from kartothek.io_components.metapartition import ( diff --git a/kartothek/io/iter.py b/kartothek/io/iter.py index cbee7280..b5890111 100644 --- a/kartothek/io/iter.py +++ b/kartothek/io/iter.py @@ -2,13 +2,13 @@ from functools import partial from typing import cast +from kartothek.core.docs import default_docs from kartothek.core.factory import _ensure_factory from kartothek.core.naming import ( DEFAULT_METADATA_STORAGE_FORMAT, DEFAULT_METADATA_VERSION, ) from kartothek.core.uuid import gen_uuid -from kartothek.io_components.docs import default_docs from kartothek.io_components.metapartition import ( MetaPartition, parse_input_to_metapartition, diff --git a/kartothek/io_components/metapartition.py b/kartothek/io_components/metapartition.py index 8516ee38..52db167d 100644 --- a/kartothek/io_components/metapartition.py +++ b/kartothek/io_components/metapartition.py @@ -25,6 +25,7 @@ validate_compatible, validate_shared_columns, ) +from kartothek.core.docs import default_docs from kartothek.core.index import ExplicitSecondaryIndex, IndexBase from kartothek.core.index import merge_indices as merge_indices_algo from kartothek.core.naming import get_partition_file_prefix @@ -32,7 +33,6 @@ from kartothek.core.urlencode import decode_key, quote_indices from kartothek.core.utils import ensure_string_type, verify_metadata_version from kartothek.core.uuid import gen_uuid -from kartothek.io_components.docs import default_docs from kartothek.io_components.utils import _instantiate_store, combine_metadata from kartothek.serialization import ( DataFrameSerializer, diff --git a/tests/io_components/test_docs.py b/tests/core/test_docs.py similarity index 97% rename from tests/io_components/test_docs.py rename to tests/core/test_docs.py index 08ab62cf..bf329af2 100644 --- a/tests/io_components/test_docs.py +++ b/tests/core/test_docs.py @@ -2,6 +2,7 @@ import pytest +from kartothek.core.docs import _PARAMETER_MAPPING from kartothek.io.dask.bag import ( build_dataset_indices__bag, read_dataset_as_dataframe_bag, @@ -37,7 +38,6 @@ store_dataframes_as_dataset__iter, update_dataset_from_dataframes__iter, ) -from kartothek.io_components.docs import _PARAMETER_MAPPING @pytest.mark.parametrize(