From 8688acd1731aa04b041090c7b1c049bfba1717ed Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Mon, 15 Jan 2024 14:59:14 -0500 Subject: [PATCH] feat: Add python bytes to array type conversion support proto (#3874) * feat: add redis sentinel support format lint Signed-off-by: snowron Signed-off-by: Rob Howley * chore: Bump pyarrow Bumps [pyarrow](https://github.com/apache/arrow) from 6.0.0 to 14.0.1. - [Commits](https://github.com/apache/arrow/compare/go/arrow/v6.0.0...apache-arrow-14.0.1) --- updated-dependencies: - dependency-name: pyarrow dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: Rob Howley * fix: upgrade the pyarrow to latest v14.0.1 for CVE-2023-47248. Signed-off-by: Shuchu Han Signed-off-by: Rob Howley * feat: add bytes to array type conversion in python -> proto Signed-off-by: Rob Howley * ignore type like in other proto val assignments Signed-off-by: Rob Howley * run black Signed-off-by: Rob Howley * floats can also appear as ints Signed-off-by: Rob Howley --------- Signed-off-by: snowron Signed-off-by: Rob Howley Signed-off-by: dependabot[bot] Signed-off-by: Shuchu Han Co-authored-by: snowron Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Shuchu Han Co-authored-by: Rob Howley --- sdk/python/feast/type_map.py | 17 +++++++++++++- sdk/python/tests/unit/test_type_map.py | 32 ++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index cdb65f886e..9dbbb5a64c 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json from collections import defaultdict from datetime import datetime, timezone from typing import ( @@ -297,7 +298,7 @@ def _type_err(item, dtype): None, ), ValueType.FLOAT: ("float_val", lambda x: float(x), None), - ValueType.DOUBLE: ("double_val", lambda x: x, {float, np.float64}), + ValueType.DOUBLE: ("double_val", lambda x: x, {float, np.float64, int, np.int_}), ValueType.STRING: ("string_val", lambda x: str(x), None), ValueType.BYTES: ("bytes_val", lambda x: x, {bytes}), ValueType.BOOL: ("bool_val", lambda x: x, {bool, np.bool_, int, np.int_}), @@ -353,6 +354,19 @@ def _python_value_to_proto_value( feast_value_type ] + # Bytes to array type conversion + if isinstance(sample, (bytes, bytearray)): + # Bytes of an array containing elements of bytes not supported + if feast_value_type == ValueType.BYTES_LIST: + raise _type_err(sample, ValueType.BYTES_LIST) + + json_value = json.loads(sample) + if isinstance(json_value, list): + if feast_value_type == ValueType.BOOL_LIST: + json_value = [bool(item) for item in json_value] + return [ProtoValue(**{field_name: proto_type(val=json_value)})] # type: ignore + raise _type_err(sample, valid_types[0]) + if sample is not None and not all( type(item) in valid_types for item in sample ): @@ -631,6 +645,7 @@ def redshift_to_feast_value_type(redshift_type_as_str: str) -> ValueType: "varchar": ValueType.STRING, "timestamp": ValueType.UNIX_TIMESTAMP, "timestamptz": ValueType.UNIX_TIMESTAMP, + "super": ValueType.BYTES, # skip date, geometry, hllsketch, time, timetz } diff --git a/sdk/python/tests/unit/test_type_map.py b/sdk/python/tests/unit/test_type_map.py index 78ff15fe93..9b21900e6d 100644 --- a/sdk/python/tests/unit/test_type_map.py +++ b/sdk/python/tests/unit/test_type_map.py @@ -48,3 +48,35 @@ def test_python_values_to_proto_values_bool(values): converted = feast_value_type_to_python_type(protos[0]) assert converted is bool(values[0]) + + +@pytest.mark.parametrize( + "values, value_type, expected", + ( + (np.array([b"[1,2,3]"]), ValueType.INT64_LIST, [1, 2, 3]), + (np.array([b"[1,2,3]"]), ValueType.INT32_LIST, [1, 2, 3]), + (np.array([b"[1.5,2.5,3.5]"]), ValueType.FLOAT_LIST, [1.5, 2.5, 3.5]), + (np.array([b"[1.5,2.5,3.5]"]), ValueType.DOUBLE_LIST, [1.5, 2.5, 3.5]), + (np.array([b'["a","b","c"]']), ValueType.STRING_LIST, ["a", "b", "c"]), + (np.array([b"[true,false]"]), ValueType.BOOL_LIST, [True, False]), + (np.array([b"[1,0]"]), ValueType.BOOL_LIST, [True, False]), + (np.array([None]), ValueType.STRING_LIST, None), + ([b"[1,2,3]"], ValueType.INT64_LIST, [1, 2, 3]), + ([b"[1,2,3]"], ValueType.INT32_LIST, [1, 2, 3]), + ([b"[1.5,2.5,3.5]"], ValueType.FLOAT_LIST, [1.5, 2.5, 3.5]), + ([b"[1.5,2.5,3.5]"], ValueType.DOUBLE_LIST, [1.5, 2.5, 3.5]), + ([b'["a","b","c"]'], ValueType.STRING_LIST, ["a", "b", "c"]), + ([b"[true,false]"], ValueType.BOOL_LIST, [True, False]), + ([b"[1,0]"], ValueType.BOOL_LIST, [True, False]), + ([None], ValueType.STRING_LIST, None), + ), +) +def test_python_values_to_proto_values_bytes_to_list(values, value_type, expected): + protos = python_values_to_proto_values(values, value_type) + converted = feast_value_type_to_python_type(protos[0]) + assert converted == expected + + +def test_python_values_to_proto_values_bytes_to_list_not_supported(): + with pytest.raises(TypeError): + _ = python_values_to_proto_values([b"[]"], ValueType.BYTES_LIST)