Merge pull request #99 from kafonek/wasm_release

Attach emscripten / wasm wheels to Release as assets

.github/workflows/test.yml

@@ -23,25 +23,26 @@ jobs:
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v2
+
       - name: Install Python
         uses: conda-incubator/setup-miniconda@v2
         with:
           mamba-version: "*"
           channels: conda-forge
           python-version: ${{ matrix.python-version }}
+
       - name: Install Rust
         uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
           profile: minimal
           default: true
-      - name: Install Hatch
-        run: |
-          pip install hatch
+
+      - name: Install Dependencies
+        run: pip install pytest maturin
+
       - name: Build Ypy
-        run: |
-          unset CONDA_PREFIX 
-          hatch run maturin develop
+        run: maturin develop
+
       - name: Run Tests
-        run: |
-          hatch run pytest
+        run: pytest

.github/workflows/wheels.yml

@@ -99,11 +99,15 @@ jobs:
 
   wasm:
     runs-on: ubuntu-latest
+    # Pyodide began supporting `micropip.install` from emscripten-compiled binary wheels
+    # in Pyodide 0.21.0 (Aug 2022), so no need to build wheels for versions before then.
+    # As of Nov 2022, the matrix for emscripten/python versions since then is simple. 
+    # Update this matrix when new Pyodide versions come out that bump the Python interpreter
+    # or emscripten version. Ref: https://pyodide.org/en/stable/project/changelog.html
     strategy:
       matrix:
-        # Match Python version, which is 3.10.2 since Pyodide 0.20.0
-        # https://pyodide.org/en/stable/project/changelog.html#version-0-20-0
         python-version: ["3.10.2"]
+        emscripten-version: ["3.1.14"]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
@@ -124,22 +128,20 @@ jobs:
       - name: Setup emsdk
         uses: mymindstorm/setup-emsdk@v11
         with: 
-          # Match emscripten version, which is 3.1.14 since Pyodide 0.21.0
-          # https://pyodide.org/en/stable/project/changelog.html#version-0-21-0
-          version: 3.1.14 
+          version: ${{ matrix.emscripten-version }}
       - name: Build wheels
         uses: messense/maturin-action@v1
         with:
           target: wasm32-unknown-emscripten
-          args: --release --out dist --find-interpreter
+          args: --release --out wasm_wheel --find-interpreter
       - name: Upload wheels
         uses: actions/upload-artifact@v2
         with:
-          name: wheels
-          path: dist
+          name: wasm_wheel
+          path: wasm_wheel
 
-  release:
-    name: Release
+  pypi-release:
+    name: Publish to Pypi on Release
     runs-on: ubuntu-latest
     needs:
       - macos
@@ -158,3 +160,27 @@ jobs:
         run: |
           pip install --upgrade twine
           twine upload --skip-existing *
+
+  # Can't upload emscripten wheels to Pypi, see https://github.com/pypi/warehouse/issues/10416.
+  # For now, this will attach the binary wheels to the Release page in Github. Users can 
+  # download those into a pyodide environment and micropip.install from there.
+  wasm-release:
+    name: Attach wasm wheel to Release
+    runs-on: ubuntu-latest
+    needs:
+      - wasm
+    if: startsWith(github.ref, 'refs/tags/')
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: wasm_wheel
+
+      - name: Attach assets to Release
+        uses: softprops/action-gh-release@v1
+        with:
+          files: '*.whl'
+          # 'name' (release name) defaults to the tag ref in this action.
+          # If release naming pattern changes, will need to configure 'name' here.
+
+
+

README.md

@@ -47,17 +47,81 @@ assert value == "hello world!"
 
 ## Tests
 
-All tests are located in `/tests`. If you are using `hatch`, there is a `test` environment matrix defined in `pyproject.toml` that will run `pytest` against `py37` through `py311`. To run the tests, install `pytest` and run the command line tool from the project root:
+All tests are located in `/tests`. To run the tests, install `pytest` and run the command line tool from the project root:
 
 ```
 pip install pytest
 pytest
 ```
 
-## Build Ypy :
+## Using Hatch
+
+If you are using `hatch`, there is a `test` environment matrix defined in `pyproject.toml` that will run commands in virtual environments for `py37` through `py311`.
+
+```
+hatch run test:maturin develop
+hatch run test:pytest
+```
+
+## Build Ypy 
 
 Build the library as a wheel and store them in `target/wheels`:
 
 ```
 maturin build
 ```
+
+## Ypy in WASM (Pyodide)
+
+As a Rust-based library, Ypy cannot build "pure Python" wheels. CI processes build and upload a number of wheels to PyPI, but PyPI does not support hosting `emscripten` / `wasm32` wheels necessary to import in Pyodide (see https://github.com/pypi/warehouse/issues/10416 for more info and updates). For now, Ypy will build `emscripten` wheels and attach the binaries as assets in the appropriate [Releases](https://github.com/y-crdt/ypy/releases) entry. Unfortunately, trying to install directly from the Github download link will result in a CORS error, so you'll need to use a proxy to pull in the binary and write / install from emscripten file system or host the binary somewhere that is CORS accessible for your application.
+
+You can try out Ypy in Pyodide using the [terminal emulator at pyodide.org](https://pyodide.org/en/stable/console.html):
+
+```
+Welcome to the Pyodide terminal emulator 🐍
+Python 3.10.2 (main, Sep 15 2022 23:28:12) on WebAssembly/Emscripten
+Type "help", "copyright", "credits" or "license" for more information.
+>>> wheel_url = 'https://github.com/y-crdt/ypy/releases/download/v0.5.5/y_py-0.5.5-cp310-cp310-emscripten_3_1_14_wasm32.whl'
+>>> wheel_name = wheel_url.split('/')[-1]
+>>> wheel_name
+'y_py-0.5.5-cp310-cp310-emscripten_3_1_14_wasm32.whl'
+>>> 
+>>> proxy_url = f'https://api.allorigins.win/raw?url={wheel_url}'
+>>> proxy_url
+'https://api.allorigins.win/raw?url=https://github.com/y-crdt/ypy/releases/download/v0.5.5/y_py-0.5.5-cp310-cp310-emscripten_3_1_14_wasm32.whl'
+>>> 
+>>> import pyodide
+>>> resp = await pyodide.http.pyfetch(proxy_url)
+>>> resp.status
+200
+>>> 
+>>> content = await resp.bytes()
+>>> len(content)
+360133
+>>> content[:50]
+b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xae\xb2}U\x92l\xa7E\xe6\x04\x00\x00u\t\x00\x00\x1d\x00\x00\x00y_py-0.5.5.dist-info'
+>>>
+>>> with open(wheel_name, 'wb') as f:
+...   f.write(content)
+... 
+360133
+>>> 
+>>> import micropip
+>>> await micropip.install(f'emfs:./{wheel_name}')
+>>> 
+>>> import y_py as Y
+>>> Y
+<module 'y_py' from '/lib/python3.10/site-packages/y_py/__init__.py'>
+>>> 
+>>> d1 = Y.YDoc()
+>>> text = d1.get_text('test')
+>>> with d1.begin_transaction() as txn:
+    text.extend(txn, "hello world!")
+... 
+>>> d2 = Y.YDoc()
+>>> state_vector = Y.encode_state_vector(d2)
+>>> diff = Y.encode_state_as_update(d1, state_vector)
+>>> Y.apply_update(d2, diff)
+>>> d2.get_text('test')
+YText(hello world!)
+```

pyproject.toml

@@ -14,6 +14,8 @@ authors = [
 readme = "README.md"
 homepage = "https://github.com/y-crdt/ypy"
 repository = "https://github.com/y-crdt/ypy"
+
+[tool.hatch.envs.test]
 dependencies = ["pytest", "maturin"]
 
 [[tool.hatch.envs.test.matrix]]