Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Allow setting metadata for ByteStream when created from file or from string #6857

Merged
merged 7 commits into from
Feb 1, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 18 additions & 5 deletions haystack/dataclasses/byte_stream.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from typing import Optional, Dict, Any


@dataclass(frozen=True)
@dataclass
class ByteStream:
"""
Base data class representing a binary object in the Haystack API.
Expand All @@ -14,25 +14,38 @@ class ByteStream:
mime_type: Optional[str] = field(default=None)

def to_file(self, destination_path: Path):
"""
Write the ByteStream to a file. Note: the metadata will be lost.

:param destination_path: The path to write the ByteStream to.
"""
with open(destination_path, "wb") as fd:
fd.write(self.data)

@classmethod
def from_file_path(cls, filepath: Path, mime_type: Optional[str] = None) -> "ByteStream":
def from_file_path(
cls, filepath: Path, mime_type: Optional[str] = None, meta: Optional[Dict[str, Any]] = None
) -> "ByteStream":
"""
Create a ByteStream from the contents read from a file.

:param filepath: A valid path to a file.
:param mime_type: The mime type of the file.
:param meta: Additional metadata to be stored with the ByteStream.
"""
with open(filepath, "rb") as fd:
return cls(data=fd.read(), mime_type=mime_type)
return cls(data=fd.read(), mime_type=mime_type, meta=meta or {})

@classmethod
def from_string(cls, text: str, encoding: str = "utf-8", mime_type: Optional[str] = None) -> "ByteStream":
def from_string(
cls, text: str, encoding: str = "utf-8", mime_type: Optional[str] = None, meta: Optional[Dict[str, Any]] = None
) -> "ByteStream":
"""
Create a ByteStream encoding a string.

:param text: The string to encode
:param encoding: The encoding used to convert the string into bytes
:param mime_type: The mime type of the file.
:param meta: Additional metadata to be stored with the ByteStream.
"""
return cls(data=text.encode(encoding), mime_type=mime_type)
return cls(data=text.encode(encoding), mime_type=mime_type, meta=meta or {})
3 changes: 3 additions & 0 deletions releasenotes/notes/meta-in-bytestream-a29816c919c0be5a.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
---
enhancements:
- Add meta parameter to `ByteStream.from_file_path()` and `ByteStream.from_string()`.
12 changes: 8 additions & 4 deletions test/dataclasses/test_byte_stream.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
import io

from haystack.dataclasses import ByteStream

import pytest


def test_from_file_path(tmp_path, request):
test_bytes = "Hello, world!\n".encode()
Expand All @@ -19,6 +15,10 @@ def test_from_file_path(tmp_path, request):
assert b.data == test_bytes
assert b.mime_type == "text/plain"

b = ByteStream.from_file_path(test_path, meta={"foo": "bar"})
assert b.data == test_bytes
assert b.meta == {"foo": "bar"}


def test_from_string():
test_string = "Hello, world!"
Expand All @@ -30,6 +30,10 @@ def test_from_string():
assert b.data.decode() == test_string
assert b.mime_type == "text/plain"

b = ByteStream.from_string(test_string, meta={"foo": "bar"})
assert b.data.decode() == test_string
assert b.meta == {"foo": "bar"}


def test_to_file(tmp_path, request):
test_str = "Hello, world!\n"
Expand Down