diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe5f1ee..215615b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,11 @@ jobs: PYTHONIOENCODING: utf-8 PYTHONUTF8: 1 run: pytest --cov agate + - name: Read from stdin + if: matrix.os != 'windows-latest' + run: python -c 'import sys; import agate; agate.Table.from_csv(sys.stdin, sniff_limit=1)' < examples/test.csv + - name: Read from pipe + run: printf 'a,b,c\n1,2,3' | python -c 'import sys; import agate; agate.Table.from_csv(sys.stdin, sniff_limit=1)' - run: python charts.py - env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 14e28e8..42aab85 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,8 @@ +Unreleased +---------- + +- fix: Version 1.10.0 errors on piped data. + 1.10.1 - April 28, 2024 ----------------------- diff --git a/agate/table/from_csv.py b/agate/table/from_csv.py index 860f473..b20f2aa 100644 --- a/agate/table/from_csv.py +++ b/agate/table/from_csv.py @@ -1,5 +1,6 @@ +import io import itertools -from io import StringIO +import sys @classmethod @@ -63,14 +64,23 @@ def from_csv(cls, path, column_names=None, column_types=None, row_names=None, sk if sniff_limit is None: # Reads to the end of the tile, but avoid reading the file twice. - handle = StringIO(f.read()) - kwargs['dialect'] = csv.Sniffer().sniff(handle.getvalue()) + handle = io.StringIO(f.read()) + sample = handle.getvalue() elif sniff_limit > 0: - offset = f.tell() + if f == sys.stdin: + # "At most one single read on the raw stream is done to satisfy the call. The number of bytes returned + # may be less or more than requested." In other words, it reads the buffer_size, which might be less or + # more than the sniff_limit. On my machine, the buffer_size of sys.stdin.buffer is the length of the + # input, up to 65536. This assumes that users don't sniff more than 64 KiB. + # https://docs.python.org/3/library/io.html#io.BufferedReader.peek + sample = sys.stdin.buffer.peek(sniff_limit).decode(encoding, 'ignore')[:sniff_limit] # reads *bytes* + else: + offset = f.tell() + sample = f.read(sniff_limit) # reads *characters* + f.seek(offset) # can't do f.seek(-sniff_limit, os.SEEK_CUR) on file opened in text mode - # Reads only the start of the file. - kwargs['dialect'] = csv.Sniffer().sniff(f.read(sniff_limit)) - f.seek(offset) + if sniff_limit is None or sniff_limit > 0: + kwargs['dialect'] = csv.Sniffer().sniff(sample) reader = csv.reader(handle, header=header, **kwargs)