Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix reduce type constraints on JSON materializer #774

Merged
merged 2 commits into from
Mar 21, 2024
Merged

fix reduce type constraints on JSON materializer #774

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8623910
reduced type constraints on JSON materializer
Mar 21, 2024
95e0336
added test cases for lists
Mar 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions hamilton/io/default_data_loaders.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import os
import pathlib
import pickle
from typing import Any, Collection, Dict, List, Tuple, Type, Union
from typing import Any, Collection, Dict, Tuple, Type, Union

from hamilton.io.data_adapters import DataLoader, DataSaver
from hamilton.io.utils import get_file_metadata
Expand All @@ -16,7 +16,7 @@ class JSONDataLoader(DataLoader):

@classmethod
def applicable_types(cls) -> Collection[Type]:
return [dict, List[dict]]
return [dict, list]

def load_data(self, type_: Type) -> Tuple[dict, Dict[str, Any]]:
with open(self.path, "r") as f:
Expand All @@ -33,7 +33,7 @@ class JSONDataSaver(DataSaver):

@classmethod
def applicable_types(cls) -> Collection[Type]:
return [dict, List[dict]]
return [dict, list]

@classmethod
def name(cls) -> str:
Expand Down
9 changes: 4 additions & 5 deletions tests/io/test_default_adapters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import io
import json
import pathlib
from typing import List

import pytest

Expand Down Expand Up @@ -32,15 +31,15 @@ def test_raw_file_adapter(data, tmp_path: pathlib.Path) -> None:
@pytest.mark.parametrize("data", [{"key": "value"}, [{"key": "value1"}, {"key": "value2"}]])
def test_json_save_object_and_array(data, tmp_path: pathlib.Path):
"""Test that `from_.json` and `to.json` can handle JSON objects where
the top-level is an object `{ }` -> dict or an array `[ ]` -> list[dict]
the top-level is an object `{ }` -> dict or an array `[ ]` -> list
"""
data_path = tmp_path / "data.json"
saver = JSONDataSaver(path=data_path)

metadata = saver.save_data(data)
loaded_data = json.loads(data_path.read_text())

assert JSONDataSaver.applicable_types() == [dict, List[dict]]
assert JSONDataSaver.applicable_types() == [dict, list]
assert data_path.exists()
assert metadata[FILE_METADATA]["path"] == str(data_path)
assert data == loaded_data
Expand All @@ -49,13 +48,13 @@ def test_json_save_object_and_array(data, tmp_path: pathlib.Path):
@pytest.mark.parametrize("data", [{"key": "value"}, [{"key": "value1"}, {"key": "value2"}]])
def test_json_load_object_and_array(data, tmp_path: pathlib.Path):
"""Test that `from_.json` and `to.json` can handle JSON objects where
the top-level is an object `{ }` -> dict or an array `[ ]` -> list[dict]
the top-level is an object `{ }` -> dict or an array `[ ]` -> list
"""
data_path = tmp_path / "data.json"
loader = JSONDataLoader(path=data_path)

json.dump(data, data_path.open("w"))
loaded_data, metadata = loader.load_data(type(data))

assert JSONDataLoader.applicable_types() == [dict, List[dict]]
assert JSONDataLoader.applicable_types() == [dict, list]
assert data == loaded_data