Skip to content

Commit

Permalink
Add code to generate a Python baseline snapshot
Browse files Browse the repository at this point in the history
  • Loading branch information
hoodmane committed Sep 27, 2024
1 parent 246c248 commit 20f32c0
Show file tree
Hide file tree
Showing 3 changed files with 75 additions and 3 deletions.
70 changes: 70 additions & 0 deletions .github/workflows/make-python-baseline-snapshot.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
name: Make Python Baseline Snapshot

permissions:
id-token: write # This is required for requesting the JWT
contents: read # This is required for actions/checkout

on:
workflow_dispatch:
inputs:
tag:
description: The release tag/version string
dry-run:
description: Actually upload or just test build step?
default: false
type: boolean
push:
branches:
- 'hoodmane/pyodide-load-from-file'

jobs:
version:
outputs:
version: ${{ steps.echo.outputs.version }}
# version job uses ubuntu 24.04, this way we don't have to install the updated clang while
# the build job uses 20.04 for libc compatibility.
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4
- id: echo
run: |
echo "::set-output name=version::1.$(cat src/workerd/io/supported-compatibility-date.txt | tr -d '-').0"
build:
runs-on: ubuntu-20.04
name: build Python runtime
steps:
- uses: actions/checkout@v4
with:
show-progress: false
- name: Setup Linux
if: runner.os == 'Linux'
run: |
export DEBIAN_FRONTEND=noninteractive
wget https://apt.llvm.org/llvm.sh
sed -i '/apt-get install/d' llvm.sh
chmod +x llvm.sh
sudo ./llvm.sh 15
sudo apt-get install -y --no-install-recommends clang-15 lld-15 libunwind-15 libc++abi1-15 libc++1-15 libc++-15-dev
echo "build:linux --action_env=CC=/usr/lib/llvm-15/bin/clang --action_env=CXX=/usr/lib/llvm-15/bin/clang++" >> .bazelrc
echo "build:linux --host_action_env=CC=/usr/lib/llvm-15/bin/clang --host_action_env=CXX=/usr/lib/llvm-15/bin/clang++" >> .bazelrc
- name: Configure download mirrors
shell: bash
run: |
if [ ! -z "${{ secrets.WORKERS_MIRROR_URL }}" ] ; then
# Strip comment in front of WORKERS_MIRROR_URL, then substitute secret to use it.
sed -e '/WORKERS_MIRROR_URL/ { s@# *@@; s@WORKERS_MIRROR_URL@${{ secrets.WORKERS_MIRROR_URL }}@; }' -i.bak WORKSPACE
fi
- name: Bazel build
# Strip debug info here – we don't generate debug info but some is pulled in from external
# static libraries, for example the Rust STL. This is equivalent to the -Wl,-S linker
# option, symbols will not be removed.
run: |
bazelisk build --config=release_linux @workerd//src/workerd/server:workerd
cd samples/pyodide
../../bazel-bin/src/workerd/server/workerd test config.capnp \
--python-save-baseline-snapshot \
--pyodide-package-disk-cache-dir=save_dir \
| tee log.txt
cat log.txt | grep inputs.tag
cp save_dir/snapshot.bin pyodide_${{ inputs.tag }}.capnp.bin
2 changes: 1 addition & 1 deletion src/workerd/io/compatibility-date.capnp
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,7 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef {
pythonWorkers @43 :Bool
$compatEnableFlag("python_workers")
$pythonSnapshotRelease(pyodide = "0.26.0a2", pyodideRevision = "2024-03-01",
packages = "2024-03-01", backport = 0)
packages = "2024-03-01", backport = 2)
$impliedByAfterDate(name = "pythonWorkersDevPyodide", date = "2000-01-01");
# Enables Python Workers. Access to this flag is not restricted, instead bundles containing
# Python modules are restricted in EWC.
Expand Down
6 changes: 4 additions & 2 deletions src/workerd/server/workerd-api.c++
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,7 @@ void writePyodideBundleFileToDisk(const kj::Maybe<kj::Own<const kj::Directory>>&

kj::Maybe<jsg::Bundle::Reader> fetchPyodideBundle(
const api::pyodide::PythonConfig& pyConfig, kj::StringPtr version) {
KJ_DBG(version);
KJ_IF_SOME(version, pyConfig.pyodideBundleManager.getPyodideBundle(version)) {
return version;
}
Expand Down Expand Up @@ -477,12 +478,13 @@ kj::Maybe<jsg::Bundle::Reader> fetchPyodideBundle(
kj::HttpHeaders headers(table);

kj::String url =
kj::str("https://pyodide.runtime-playground.workers.dev/pyodide-capnp-bin/pyodide_",
version, ".capnp.bin");
kj::str("https://www.googleapis.com/storage/v1/b/cloudflare-edgeworker-python-packages/o/python-runtime-capnp-bin%2Fpyodide_",
version, ".capnp.bin?alt=media");

auto req = client->request(kj::HttpMethod::GET, kj::StringPtr(url), headers);

auto res = req.response.wait(io.waitScope);
KJ_DBG(url, res.statusCode);
auto body = res.body->readAllBytes().wait(io.waitScope);

writePyodideBundleFileToDisk(pyConfig.pyodideDiskCacheRoot, version, body);
Expand Down

0 comments on commit 20f32c0

Please sign in to comment.