From 817452cea888c2957c075b39371f297feb40ac5c Mon Sep 17 00:00:00 2001 From: Taylor Brown Date: Sun, 29 Aug 2021 20:34:57 +0000 Subject: [PATCH 1/4] Run `npm ci` automatically after the devcontainer is created. This reduces the chance that another dev will forget to do this. --- .devcontainer/devcontainer.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f40ff7e8..e590542e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -18,7 +18,9 @@ // Use 'forwardPorts' to make a list of ports inside the container available locally. // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. - // "postCreateCommand": "yarn install", + // We use `npm ci` instead of `npm install` because we want to respect the lockfile and ONLY the lockfile. + // That way, our devcontainer is more reproducible. --Taytay + "postCreateCommand": "npm ci", // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. "remoteUser": "node" } \ No newline at end of file From 5333ce04fbf14112136094e942bf7c69de17b52d Mon Sep 17 00:00:00 2001 From: Taylor Brown Date: Sun, 29 Aug 2021 20:05:42 +0000 Subject: [PATCH 2/4] Upgrade to Emscripten 2.0.29 Changes required: * Defined EM_NODE_JS environment variable to get rid of a warning that appears if the NODE environment variable is set, but EM_NODE_JS is not. * EXTRA_EXPORTED_RUNTIME_METHODS is now EXPORTED_RUNTIME_METHODS * No longer pass the `-s LINKABLE=1` option to emcc when compiling. (This is a linktime setting and emcc warns about not using a linktime setting when compiling now). --- .devcontainer/Dockerfile | 11 +++++++++-- Makefile | 4 ++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index f3d381d0..f76103f6 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -24,7 +24,7 @@ FROM mcr.microsoft.com/vscode/devcontainers/typescript-node:0-${VARIANT} # Install EMSDK to /emsdk just like the EMSDK Dockerfile: https://github.com/emscripten-core/emsdk/blob/master/docker/Dockerfile ENV EMSDK /emsdk # We pin EMSDK to 2.0.15 rather than 'latest' so that everyone is using the same compiler version -ENV EMSCRIPTEN_VERSION 2.0.15 +ENV EMSCRIPTEN_VERSION 2.0.29 RUN git clone https://github.com/emscripten-core/emsdk.git $EMSDK @@ -36,7 +36,7 @@ RUN echo "## Install Emscripten" \ # Copied directly from https://github.com/emscripten-core/emsdk/blob/master/docker/Dockerfile RUN cd ${EMSDK} \ && echo "## Generate standard configuration" \ - && ./emsdk activate $EMSCRIPTEN_VERSION \ + && ./emsdk activate ${EMSCRIPTEN_VERSION} \ && chmod 777 ${EMSDK}/upstream/emscripten \ && chmod -R 777 ${EMSDK}/upstream/emscripten/cache \ && echo "int main() { return 0; }" > hello.c \ @@ -47,6 +47,7 @@ RUN cd ${EMSDK} \ ENV PATH $EMSDK:$EMSDK/upstream/emscripten/:$PATH # Cleanup Emscripten installation and strip some symbols +# Copied directly from https://github.com/emscripten-core/emsdk/blob/master/docker/Dockerfile RUN echo "## Aggressive optimization: Remove debug symbols" \ && cd ${EMSDK} && . ./emsdk_env.sh \ # Remove debugging symbols from embedded node (extra 7MB) @@ -60,6 +61,12 @@ RUN echo "## Aggressive optimization: Remove debug symbols" \ && echo "## Done" RUN echo ". /emsdk/emsdk_env.sh" >> /etc/bash.bashrc +# We must set the EM_NODE_JS environment variable for a somewhat silly reason +# We run our build scripts with `npm run`, which sets the NODE environment variable as it runs. +# The EMSDK picks up on that environment variable and gives a deprecation warning: warning: honoring legacy environment variable `NODE`. Please switch to using `EM_NODE_JS` instead` +# So, we are going to put this environment variable here explicitly to avoid the deprecation warning. +RUN echo 'export EM_NODE_JS="$EMSDK_NODE"' >> /etc/bash.bashrc + # END EMSDK # -------------------------------------------------------------------- diff --git a/Makefile b/Makefile index 02eb68f4..dfdb10f4 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ EMFLAGS = \ -s RESERVED_FUNCTION_POINTERS=64 \ -s ALLOW_TABLE_GROWTH=1 \ -s EXPORTED_FUNCTIONS=@src/exported_functions.json \ - -s EXTRA_EXPORTED_RUNTIME_METHODS=@src/exported_runtime_methods.json \ + -s EXPORTED_RUNTIME_METHODS=@src/exported_runtime_methods.json \ -s SINGLE_FILE=0 \ -s NODEJS_CATCH_EXIT=0 \ -s NODEJS_CATCH_REJECTION=0 @@ -154,7 +154,7 @@ out/sqlite3.bc: sqlite-src/$(SQLITE_AMALGAMATION) out/extension-functions.bc: sqlite-src/$(SQLITE_AMALGAMATION) mkdir -p out # Generate llvm bitcode - $(EMCC) $(CFLAGS) -s LINKABLE=1 -c sqlite-src/$(SQLITE_AMALGAMATION)/extension-functions.c -o $@ + $(EMCC) $(CFLAGS) -c sqlite-src/$(SQLITE_AMALGAMATION)/extension-functions.c -o $@ # TODO: This target appears to be unused. If we re-instatate it, we'll need to add more files inside of the JS folder # module.tar.gz: test package.json AUTHORS README.md dist/sql-asm.js From a3bca11563fe2e8d93ab0d7d13af2748850b98f0 Mon Sep 17 00:00:00 2001 From: Taylor Brown Date: Sun, 29 Aug 2021 21:43:15 +0000 Subject: [PATCH 3/4] Upgrade to Sqlite 3.36.0 Sqlite now publishes the hash as a SHA3 instead of SHA1, necessitating the installation of the sha3sum command. --- .devcontainer/Dockerfile | 3 +++ Makefile | 10 +++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index f76103f6..3ed8a281 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -124,6 +124,9 @@ RUN apt-get update \ wget \ xdg-utils +# Installs the command "sha3sum", which is used check the download integrity of sqlite source. +RUN apt-get install -y libdigest-sha3-perl + # We set this env variable (RUN_WORKER_TEST_WITHOUT_PUPPETEER_SANDBOX=1) this to tell our sql.js test harness to run Puppeteer without the sandbox. # Otherwise, when we instantiate Puppeteer, we get this error: # Puppeteer can't start due to a sandbox error. (Details follow.) diff --git a/Makefile b/Makefile index dfdb10f4..a4ce5ebc 100644 --- a/Makefile +++ b/Makefile @@ -6,9 +6,9 @@ # I got this handy makefile syntax from : https://github.com/mandel59/sqlite-wasm (MIT License) Credited in LICENSE # To use another version of Sqlite, visit https://www.sqlite.org/download.html and copy the appropriate values here: -SQLITE_AMALGAMATION = sqlite-amalgamation-3350000 -SQLITE_AMALGAMATION_ZIP_URL = https://www.sqlite.org/2021/sqlite-amalgamation-3350000.zip -SQLITE_AMALGAMATION_ZIP_SHA1 = ba64bad885c9f51df765a9624700747e7bf21b79 +SQLITE_AMALGAMATION = sqlite-amalgamation-3360000 +SQLITE_AMALGAMATION_ZIP_URL = https://www.sqlite.org/2021/sqlite-amalgamation-3360000.zip +SQLITE_AMALGAMATION_ZIP_SHA3 = d25609210ec93b3c8c7da66a03cf82e2c9868cfbd2d7d866982861855e96f972 # Note that extension-functions.c hasn't been updated since 2010-02-06, so likely doesn't need to be updated EXTENSION_FUNCTIONS = extension-functions.c @@ -175,8 +175,8 @@ sqlite-src: sqlite-src/$(SQLITE_AMALGAMATION) sqlite-src/$(SQLITE_AMALGAMATION)/ sqlite-src/$(SQLITE_AMALGAMATION): cache/$(SQLITE_AMALGAMATION).zip sqlite-src/$(SQLITE_AMALGAMATION)/$(EXTENSION_FUNCTIONS) mkdir -p sqlite-src/$(SQLITE_AMALGAMATION) - echo '$(SQLITE_AMALGAMATION_ZIP_SHA1) ./cache/$(SQLITE_AMALGAMATION).zip' > cache/check.txt - sha1sum -c cache/check.txt + echo '$(SQLITE_AMALGAMATION_ZIP_SHA3) ./cache/$(SQLITE_AMALGAMATION).zip' > cache/check.txt + sha3sum -c cache/check.txt # We don't delete the sqlite_amalgamation folder. That's a job for clean # Also, the extension functions get copied here, and if we get the order of these steps wrong, # this step could remove the extension functions, and that's not what we want From 991747d210f3ca44092c49498705f0b39845f56b Mon Sep 17 00:00:00 2001 From: Taylor Brown Date: Sun, 29 Aug 2021 22:19:21 +0000 Subject: [PATCH 4/4] Use -Oz optimizations instead of -O2 This reduces most compilation output by around 50% --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index a4ce5ebc..bd592f26 100644 --- a/Makefile +++ b/Makefile @@ -17,8 +17,8 @@ EXTENSION_FUNCTIONS_SHA1 = c68fa706d6d9ff98608044c00212473f9c14892f EMCC=emcc -CFLAGS = \ - -O2 \ +SQLITE_COMPILATION_FLAGS = \ + -Oz \ -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_DISABLE_LFS \ -DSQLITE_ENABLE_FTS3 \ @@ -53,7 +53,7 @@ EMFLAGS_WASM = \ EMFLAGS_OPTIMIZED= \ -s INLINING_LIMIT=50 \ - -O3 \ + -Oz \ -flto \ --closure 1 @@ -148,13 +148,13 @@ dist/worker.sql-wasm-debug.js: dist/sql-wasm-debug.js src/worker.js out/sqlite3.bc: sqlite-src/$(SQLITE_AMALGAMATION) mkdir -p out # Generate llvm bitcode - $(EMCC) $(CFLAGS) -c sqlite-src/$(SQLITE_AMALGAMATION)/sqlite3.c -o $@ + $(EMCC) $(SQLITE_COMPILATION_FLAGS) -c sqlite-src/$(SQLITE_AMALGAMATION)/sqlite3.c -o $@ # Since the extension-functions.c includes other headers in the sqlite_amalgamation, we declare that this depends on more than just extension-functions.c out/extension-functions.bc: sqlite-src/$(SQLITE_AMALGAMATION) mkdir -p out # Generate llvm bitcode - $(EMCC) $(CFLAGS) -c sqlite-src/$(SQLITE_AMALGAMATION)/extension-functions.c -o $@ + $(EMCC) $(SQLITE_COMPILATION_FLAGS) -c sqlite-src/$(SQLITE_AMALGAMATION)/extension-functions.c -o $@ # TODO: This target appears to be unused. If we re-instatate it, we'll need to add more files inside of the JS folder # module.tar.gz: test package.json AUTHORS README.md dist/sql-asm.js