Skip to content

Commit

Permalink
placeholder
Browse files Browse the repository at this point in the history
  • Loading branch information
colin-ho committed Feb 2, 2024
1 parent 4734862 commit f4cdad8
Show file tree
Hide file tree
Showing 2 changed files with 37 additions and 1 deletion.
8 changes: 7 additions & 1 deletion src/daft-json/src/inference.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,13 @@ fn infer_object(inner: &IndexMap<String, Value>) -> Result<DataType> {
Ok(Field::new(key, dt, true))
})
.collect::<Result<Vec<_>>>()?;
Ok(DataType::Struct(fields))
if fields.is_empty() {
// Converts empty Structs to structs with a single field named "" and with a NullType
// This is because Arrow2 MutableStructArray cannot handle empty Structs
Ok(DataType::Struct(vec![Field::new("", DataType::Null, true)]))
} else {
Ok(DataType::Struct(fields))
}
}

fn infer_array(values: &[Value]) -> Result<DataType> {
Expand Down
30 changes: 30 additions & 0 deletions tests/dataframe/test_creation.py
Original file line number Diff line number Diff line change
Expand Up @@ -797,6 +797,36 @@ def test_create_dataframe_json_schema_hints_ignore_random_hint(valid_data: list[
assert len(pd_df) == len(valid_data)


@pytest.mark.parametrize(
"input,expected",
[
pytest.param(['{"foo": {}}', '{"foo": {}}'], {"foo": [{"": None}, {"": None}]}, id="AllEmptyObjects"),
pytest.param(
['{"foo": {"bar":"baz"}}', '{"foo": {}}'],
{"foo": [{"": None, "bar": "baz"}, {"": None, "bar": None}]},
id="FirstObjectNonEmpty",
),
pytest.param(
['{"foo": {}}', '{"foo": {"bar":"baz"}}'],
{"foo": [{"": None, "bar": None}, {"": None, "bar": "baz"}]},
id="FirstObjectEmpty",
),
],
)
def test_create_dataframe_json_schema_hints_empty_objects(input, expected) -> None:
with create_temp_filename() as fname:
with open(fname, "w") as f:
for data in input:
f.write(data)
f.write("\n")
f.flush()

df = daft.read_json(fname)
pydict = df.to_pydict()

assert pydict == expected


###
# Parquet tests
###
Expand Down

0 comments on commit f4cdad8

Please sign in to comment.