Fix workflows (#5)

* Fix workflows * [squash] run yapf * [squash] * [squash] fix markdown lint * [squash] the md lint is quite strict * [squash] fix spelling * [squash] fix yapf * [squash] fix spelling once again * [squash] install -e instead of requirements * [squash] add copyright to autogenerated files * [squash] add copyright for setup.py * [squash] most of pylint * [squash] * [squash] * [squash] * [squash] * [squash] * [squash] * [squash] * [squash] * [squash] * [squash] * [squash] * [squash] * [squash] * [squash] this will trigger because no hashes * [squash] remove 3.9 * [squash] path for mypy_stubs * [squash] cache venv (in mypy) * [squash] fix mypy wf * [squash] * [squash] why do we have tabs * [squash] fine * [squash] do you know what is the difference between theory and practice? Theoretically there is no difference. * [squash] bump cache version * [squash] let's see * [squash] cleanup mypy * [squash] * [squash] try with pytest * [squash] * [squash] omit no-depts * [squash] force saving the pipenv * [squash] split test_requirements * [squash] * [squash] cache venv for pytest * [squash] * [squash] save cache only if that was not a hit * [squash] ignore coverage problems * [squash] add coveralls badge * [squash] * [squash] mypy bash env * [squash] add lz4 * [squash] pip install . * [squash] * [squash] "install" sedpack * [squash] toml 1, Karel 0 * [squash] * [squash] fix base tooling * [squash] fix not throwing StopIteration * [squash] fixme is just warning * [squash] remove TODOs * [squash] no need for Python in licence checks * [squash] spell
google · Sep 11, 2024 · 465b268 · 465b268
1 parent 39b3016
commit 465b268
Show file tree

Hide file tree

Showing 51 changed files with 1,734 additions and 367 deletions.
diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml
@@ -16,12 +16,39 @@ jobs:
         with:
           python-version: '3.10'
           cache: 'pip'
-      - name: Installing package
+      - name: Get pip cache directory
+        id: pip-cache
+        shell: bash
         run: |
-          pip install --require-hashes --no-deps -r requirements.txt
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+      - name: Use cached venv or create it
+        uses: actions/cache/restore@v4
+        id: cache
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          # The cache key depends on requirements.txt
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      # Build a virtualenv, but only if it doesn't already exist
+      - name: Populate pip cache
+        run: pip install --require-hashes --no-deps -r requirements.txt
+      - name: Save cache
+        id: cache-save
+        uses: actions/cache/save@v4
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ steps.cache.outputs.cache-primary-key }}
+        if: steps.cache.outputs.cache-hit != 'true'
       - name: Register matcher
         run: echo ::add-matcher::./.github/python_matcher.json
       - name: Running mypy
+        env:
+          MYPYPATH: mypy_stubs/
+        # Start by "installing" sedpack to be sure all dependencies are listed
         run: |
+          echo "PYTHONPATH=./src:$PYTHONPATH" >> $GITHUB_ENV
+          pip install -r test_requirements.txt
           mkdir -p .mypy_cache
+          stubgen -o $MYPYPATH -p flatbuffers -p lz4
           mypy --no-color-output --install-types --non-interactive src
diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml
@@ -11,19 +11,14 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-          cache: 'pip'
       - name: Check licence headers
         run: ./tools/check_copyright.sh
 
   pylint:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: ['3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.10', '3.11', '3.12']
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
@@ -34,6 +29,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip setuptools wheel
+          pip install --require-hashes --no-deps -r requirements.txt
           pip install --upgrade pylint
       - name: Register matcher
         run: echo ::add-matcher::./.github/python_matcher.json

diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -20,11 +20,38 @@ jobs:
         with:
           python-version: '3.10'
           cache: 'pip'
-      - name: Installing package
+      - name: Get pip cache directory
+        id: pip-cache
+        shell: bash
         run: |
-          pip install --require-hashes --no-deps -r requirements.txt
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+      - name: Use cached venv or create it
+        uses: actions/cache/restore@v4
+        id: cache
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          # The cache key depends on requirements.txt
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      # Build a virtualenv, but only if it doesn't already exist
+      - name: Populate pip cache
+        run: python -m pip install --require-hashes --no-deps -r requirements.txt
+      - name: Save cache
+        id: cache-save
+        uses: actions/cache/save@v4
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ steps.cache.outputs.cache-primary-key }}
+        if: steps.cache.outputs.cache-hit != 'true'
+      - name: Installing test requirements and sedpack
+        # Start by "installing" sedpack to be sure all dependencies are listed
+        run: |
+          pip install -r test_requirements.txt
+          echo "PYTHONPATH=./src:$PYTHONPATH" >> $GITHUB_ENV
       - name: Running unit tests
-        run: python -m pytest
+        run: |
+          python -m pytest
 
   coverage:
     runs-on: ubuntu-22.04
@@ -35,17 +62,44 @@ jobs:
         with:
           python-version: '3.10'
           cache: 'pip'
-      - name: Installing package
+      - name: Get pip cache directory
+        id: pip-cache
+        shell: bash
+        run: |
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+      - name: Use cached venv or create it
+        uses: actions/cache/restore@v4
+        id: cache
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          # The cache key depends on requirements.txt
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      # Build a virtualenv, but only if it doesn't already exist
+      - name: Populate pip cache
+        run: python -m pip install --require-hashes --no-deps -r requirements.txt
+      - name: Save cache
+        id: cache-save
+        uses: actions/cache/save@v4
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ steps.cache.outputs.cache-primary-key }}
+        if: steps.cache.outputs.cache-hit != 'true'
+      - name: Installing test requirements and sedpack
+        # Start by "installing" sedpack to be sure all dependencies are listed
         run: |
-          pip install --require-hashes --no-deps -r requirements.txt
+          pip install -r test_requirements.txt
+          echo "PYTHONPATH=./src:$PYTHONPATH" >> $GITHUB_ENV
       - name: Install workflow dependencies
         run: pip install --upgrade pytest coverage
       - name: Running unit tests with coverage
         env:
           DISABLE_AUTOGRAPH: 1
+        # TODO remove the -i (ignore errors)
         run: |
           coverage run -m pytest
-          coverage xml
+          coverage xml -i
       - name: Upload results
         uses: coverallsapp/github-action@v2
         with:

diff --git a/.markdownlint.json b/.markdownlint.json
@@ -0,0 +1,37 @@
+{
+    "default": true,
+    "heading-style": {
+        "style": "atx"
+    },
+    "no-trailing-spaces": {
+        "br_spaces": 0,
+        "strict": true
+    },
+    "ul-indent": {
+        "indent": 4
+    },
+    "line-length": {
+        "line_length": 80,
+        "heading_line_length": 120,
+        "tables": false,
+        "code_blocks": false
+    },
+    "list-marker-space": {
+        "ol_single": 2,
+        "ol_multi": 2,
+        "ul_single": 3,
+        "ul_multi": 3
+    },
+    "no-inline-html": {
+        "allowed_elements": [
+            "img"
+        ]
+    },
+    "fenced-code-language": true,
+    "code-block-style": {
+        "style": "fenced"
+    },
+    "code-fence-style": {
+        "style": "backtick"
+    }
+}
diff --git a/README.md b/README.md
@@ -1,24 +1,28 @@
 # Sedpack - Scalable and efficient data packing
 
+[![Coverage Status](https://coveralls.io/repos/github/google/sedpack/badge.svg?branch=main)](https://coveralls.io/github/google/sedpack?branch=main)
+
 Mainly refactored from the [SCAAML](https://github.com/google/scaaml) project.
 
 ## Available components
 
-- TODO
+-   TODO
 
 ## Install
 
 ### Dependencies
 
-To use this library you need to have a working version of [TensorFlow 2.x](https://www.tensorflow.org/install).
+To use this library you need to have a working version of [TensorFlow
+2.x](https://www.tensorflow.org/install).
 
 ### Dataset install
 
 #### Development install
 
-1. Clone the repository: `git clone https://github.com/google/sedpack`
-2. Install dependencies: `python3 -m pip install --require-hashes -r requirements.txt`
-3. Install the package in development mode: `python3 -m pip install --editable .` (short `pip install -e .` or legacy `python setup.py develop`)
+1.  Clone the repository: `git clone https://github.com/google/sedpack`
+2.  Install dependencies: `python3 -m pip install --require-hashes -r requirements.txt`
+3.  Install the package in development mode: `python3 -m pip install --editable
+    .` (short `pip install -e .` or legacy `python setup.py develop`)
 
 ### Update dependencies
 
@@ -36,7 +40,8 @@ Update: `pip-compile requirements.in --generate-hashes --upgrade` and commit req
 ### Tutorial
 
 Tutorials available in the docs/tutorials/ directory.  For a "hello world" see
-[docs/tutorials/quick_start/mnist_save.py](https://github.com/google/sedpack/blob/main/docs/tutorials/quick_start/mnist_save.py) and
+[docs/tutorials/quick_start/mnist_save.py](https://github.com/google/sedpack/blob/main/docs/tutorials/quick_start/mnist_save.py)
+and
 [docs/tutorials/quick_start/mnist_save.py](https://github.com/google/sedpack/blob/main/docs/tutorials/quick_start/mnist_read.py).
 
 ## Disclaimer

diff --git a/base-tooling-requirements.txt b/base-tooling-requirements.txt
@@ -1,12 +1,12 @@
-build==1.0.0     --hash=sha256:49a60f212df4d9925727c2118e1cbe3abf30b393eff7d0e7287d2170eb36844d     --hash=sha256:f4c7b45e70e2c345e673902253d435a9a7729ff09ab574924420cf120c60bcc9
+build==1.2.1     --hash=sha256:526263f4870c26f26c433545579475377b2b7588b6f1eac76a001e873ae3e19d     --hash=sha256:75e10f767a433d9a86e50d83f418e83efc18ede923ee5ff7df93b6cb0306c5d4
 click==8.1.5     --hash=sha256:4be4b1af8d665c6d942909916d31a213a106800c47d0eeba73d34da3cbc11367     --hash=sha256:e576aa487d679441d7d30abb87e1b43d24fc53bffb8758443b1a9e1cee504548
-importlib-metadata==7.0.0     --hash=sha256:7fc841f8b8332803464e5dc1c63a2e59121f46ca186c0e2e182e80bf8c1319f7     --hash=sha256:d97503976bb81f40a193d41ee6570868479c69d5068651eb039c40d850c59d67
-packaging==23.2     --hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5     --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7
-pip-tools==7.3.0     --hash=sha256:8717693288720a8c6ebd07149c93ab0be1fced0b5191df9e9decd3263e20d85e     --hash=sha256:8e9c99127fe024c025b46a0b2d15c7bd47f18f33226cf7330d35493663fc1d1d
-pyproject-hooks==1.0.0     --hash=sha256:283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8     --hash=sha256:f271b298b97f5955d53fb12b72c1fb1948c22c1a6b70b315c54cedaca0264ef5
+importlib-metadata==8.4.0     --hash=sha256:66f342cc6ac9818fc6ff340576acd24d65ba0b3efabb2b4ac08b598965a4a2f1     --hash=sha256:9a547d3bc3608b025f93d403fdd1aae741c24fbb8314df4b155675742ce303c5
+packaging==24.1     --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002     --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124
+pip-tools==7.4.0     --hash=sha256:a92a6ddfa86ff389fe6ace381d463bc436e2c705bd71d52117c25af5ce867bb7     --hash=sha256:b67432fd0759ed834c5367f9e0ce8c95441acecfec9c8e24b41aca166757adf0
+pyproject-hooks==1.1.0     --hash=sha256:4b37730834edbd6bd37f26ece6b44802fb1c1ee2ece0e54ddff8bfc06db86965     --hash=sha256:7ceeefe9aec63a1064c18d939bdc3adf2d8aa1988a510afec15151578b232aa2
 tomli==2.0.1     --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc     --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f
-typing-extensions==4.9.0     --hash=sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783     --hash=sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd
-wheel==0.42.0     --hash=sha256:177f9c9b0d45c47873b619f5b650346d632cdc35fb5e4d25058e09c9e581433d     --hash=sha256:c45be39f7882c9d34243236f2d63cbd58039e360f85d0913425fbd7ceea617a8
-zipp==3.19.1     --hash=sha256:2828e64edb5386ea6a52e7ba7cdb17bb30a73a858f5eb6eb93d8d36f5ea26091     --hash=sha256:35427f6d5594f4acf82d25541438348c26736fa9b3afa2754bcd63cdb99d8e8f
-pip==24.0     --hash=sha256:ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc     --hash=sha256:ea9bd1a847e8c5774a5777bb398c19e80bcd4e2aa16a4b301b718fe6f593aba2
-setuptools==70.0.0     --hash=sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4     --hash=sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0
+typing-extensions==4.12.0     --hash=sha256:8cbcdc8606ebcb0d95453ad7dc5065e6237b6aa230a31e81d0f440c30fed5fd8     --hash=sha256:b349c66bea9016ac22978d800cfff206d5f9816951f12a7d0ec5578b0a819594
+wheel==0.44.0     --hash=sha256:2376a90c98cc337d18623527a97c31797bd02bad0033d41547043a1cbfbe448f     --hash=sha256:a29c3f2817e95ab89aa4660681ad547c0e9547f20e75b0562fe7723c9a2a9d49
+zipp==3.20.0     --hash=sha256:0145e43d89664cfe1a2e533adc75adafed82fe2da404b4bbb6b026c0157bdb31     --hash=sha256:58da6168be89f0be59beb194da1250516fdaa062ccebd30127ac65d30045e10d
+pip==24.2     --hash=sha256:2cd581cf58ab7fcfca4ce8efa6dcacd0de5bf8d0a3eb9ec927e07405f4d9e2a2     --hash=sha256:5b5e490b5e9cb275c879595064adce9ebd31b854e3e803740b72f9ccf34a45b8
+setuptools==74.1.2     --hash=sha256:5f4c08aa4d3ebcb57a50c33b1b07e94315d7fc7230f7115e47fc99776c8ce308     --hash=sha256:95b40ed940a1c67eb70fc099094bd6e99c6ee7c23aa2306f4d2697ba7916f9c6
diff --git a/docs/code-of-conduct.md b/docs/code-of-conduct.md
@@ -75,7 +75,7 @@ receive and address reported violations of the code of conduct. They will then
 work with a committee consisting of representatives from the Open Source
 Programs Office and the Google Open Source Strategy team. If for any reason you
 are uncomfortable reaching out to the Project Steward, please email
-opensource@google.com.
+[opensource@google.com](opensource@google.com).
 
 We will investigate every complaint, but you may not receive a direct response.
 We will use our discretion in determining when and how to follow up on reported
@@ -90,4 +90,4 @@ harassment or threats to anyone's safety, we may take action without notice.
 
 This Code of Conduct is adapted from the Contributor Covenant, version 1.4,
 available at
-https://www.contributor-covenant.org/version/1/4/code-of-conduct/
+[https://www.contributor-covenant.org/version/1/4/code-of-conduct/](https://www.contributor-covenant.org/version/1/4/code-of-conduct/)
diff --git a/docs/tutorials/quick_start/mnist_read.py b/docs/tutorials/quick_start/mnist_read.py
@@ -108,7 +108,7 @@ def process_record(rec: ExampleT) -> Tuple[Any, Any]:
         validation_steps=steps_per_epoch // 10,
     )
 
-    # Evaluate the model on hodlout.
+    # Evaluate the model on holdout.
     holdout_data = dataset.as_tfdataset(
         "holdout",
         batch_size=batch_size,
@@ -142,8 +142,8 @@ def process_record(rec: ExampleT) -> Tuple[Any, Any]:
         print(f"Predicted: {predicted_class} (should be {correct_class}) for")
         # Turn into ASCII art
         for row in image:
-            print("".join(
-                ascii_shades[int(pixel * len(ascii_shades))] for pixel in row))
+            print("".join(ascii_shades[int(pixel * len(ascii_shades))]
+                          for pixel in row))
 
 
 if __name__ == "__main__":

diff --git a/project-words.txt b/project-words.txt
@@ -1 +1,26 @@
+Elems
+MNIST
+SCAAML
+asyncstdlib
+byteswap
+compresslevel
+crossentropy
+diutils
+fbapi
+flatbuffer
+frombuffer
+itemsize
+kwarguments
+ndarray
+newbyteorder
+pickleable
+savez
 sedpack
+setaf
+shardfile
+tfdata
+tfdataset
+tfrec
+tfrecord
+tobytes
+uoffset
diff --git a/requirements.in b/requirements.in
@@ -0,0 +1,15 @@
+aiofiles
+asyncstdlib
+flatbuffers
+lz4
+numpy
+perfcounters
+pydantic
+pytest
+pytest-asyncio
+schema
+semver
+tabulate
+tensorflow
+termcolor
+tqdm