From f2c4b754d045b209609cf3cd073e1c45138833c5 Mon Sep 17 00:00:00 2001 From: Matt <1009003+tantaman@users.noreply.github.com> Date: Thu, 30 Nov 2023 17:17:39 -0500 Subject: [PATCH 1/3] use zip so we can keep extensions and differing filenames from the container --- .github/workflows/publish.yaml | 46 ++++---- core/nodejs-install-helper.js | 5 +- core/package.json | 5 +- core/pnpm-lock.yaml | 209 +++++++++++++++++++++++++++++++++ 4 files changed, 235 insertions(+), 30 deletions(-) create mode 100644 core/pnpm-lock.yaml diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 712cb8ad1..17ad18e86 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -16,12 +16,12 @@ jobs: include: - os: ubuntu-latest library_name: crsqlite.so - artifact_name: crsqlite.gz - asset_name: crsqlite-linux-x86_64.gz + artifact_name: crsqlite.zip + asset_name: crsqlite-linux-x86_64.zip - os: macos-latest library_name: crsqlite.dylib - artifact_name: crsqlite.gz - asset_name: crsqlite-darwin-x86_64.gz + artifact_name: crsqlite.zip + asset_name: crsqlite-darwin-x86_64.zip steps: - uses: actions/checkout@v3 @@ -37,8 +37,7 @@ jobs: run: | cd core make loadable - gzip dist/${{ matrix.library_name }} - mv dist/${{ matrix.library_name }}.gz dist/crsqlite.gz + zip dist/crsqlite.zip dist/${{ matrix.library_name }} - name: Upload binaries to release uses: svenstaro/upload-release-action@v2 @@ -57,8 +56,8 @@ jobs: include: - os: macos-latest library_name: crsqlite.dylib - artifact_name: crsqlite.gz - asset_name: crsqlite-darwin-aarch64.gz + artifact_name: crsqlite.zip + asset_name: crsqlite-darwin-aarch64.zip steps: - uses: actions/checkout@v3 @@ -74,8 +73,7 @@ jobs: run: | cd core export CI_MAYBE_TARGET="aarch64-apple-darwin" && make loadable - gzip dist/${{ matrix.library_name }} - mv dist/${{ matrix.library_name }}.gz dist/crsqlite.gz + zip dist/crsqlite.zip dist/${{ matrix.library_name }} - name: Upload macos dylib to release uses: svenstaro/upload-release-action@v2 @@ -106,8 +104,8 @@ jobs: include: - os: ubuntu-latest library_name: crsqlite.so - artifact_name: crsqlite.gz - asset_name: crsqlite-linux-aarch64.gz + artifact_name: crsqlite.zip + asset_name: crsqlite-linux-aarch64.zip steps: - name: Install toolchain @@ -128,8 +126,7 @@ jobs: run: | cd core export CI_MAYBE_TARGET="aarch64-unknown-linux-gnu" && export CI_GCC="aarch64-linux-gnu-gcc" && make loadable - gzip dist/${{ matrix.library_name }} - mv dist/${{ matrix.library_name }}.gz dist/crsqlite.gz + zip dist/crsqlite.zip dist/${{ matrix.library_name }} - name: Upload binaries to release uses: svenstaro/upload-release-action@v2 @@ -147,8 +144,8 @@ jobs: include: - os: ubuntu-latest library_name: crsqlite.dll - artifact_name: crsqlite.gz - asset_name: crsqlite-win-x86_64.gz + artifact_name: crsqlite.zip + asset_name: crsqlite-win-x86_64.zip steps: - uses: actions/checkout@v3 @@ -169,8 +166,7 @@ jobs: run: | cd core export CI_MAYBE_TARGET="x86_64-pc-windows-gnu" && export CI_GCC="x86_64-w64-mingw32-gcc" && make loadable - gzip dist/${{ matrix.library_name }} - mv dist/${{ matrix.library_name }}.gz dist/crsqlite.gz + zip dist/crsqlite.zip dist/${{ matrix.library_name }} - name: Upload binaries to release uses: svenstaro/upload-release-action@v2 @@ -188,8 +184,8 @@ jobs: include: - os: ubuntu-latest library_name: crsqlite.dll - artifact_name: crsqlite.gz - asset_name: crsqlite-win-i686.gz + artifact_name: crsqlite.zip + asset_name: crsqlite-win-i686.zip steps: - uses: actions/checkout@v3 @@ -210,8 +206,7 @@ jobs: run: | cd core export CI_MAYBE_TARGET="i686-pc-windows-gnu" && export CI_GCC="i686-w64-mingw32-gcc" && make loadable - gzip dist/${{ matrix.library_name }} - mv dist/${{ matrix.library_name }}.gz dist/crsqlite.gz + zip dist/crsqlite.zip dist/${{ matrix.library_name }} - name: Upload binaries to release uses: svenstaro/upload-release-action@v2 @@ -229,8 +224,8 @@ jobs: include: - os: ubuntu-latest library_name: crsqlite.so - artifact_name: crsqlite.gz - asset_name: crsqlite-aarch64-linux-android.gz + artifact_name: crsqlite.zip + asset_name: crsqlite-aarch64-linux-android.zip steps: - uses: actions/checkout@v3 @@ -259,8 +254,7 @@ jobs: run: | cd core export ANDROID_TARGET=aarch64-linux-android; make loadable - gzip dist/${{ matrix.library_name }} - mv dist/${{ matrix.library_name }}.gz dist/crsqlite.gz + zip dist/crsqlite.zip dist/${{ matrix.library_name }} - name: Upload binaries to release uses: svenstaro/upload-release-action@v2 diff --git a/core/nodejs-install-helper.js b/core/nodejs-install-helper.js index d728dc3b4..ca7d82452 100644 --- a/core/nodejs-install-helper.js +++ b/core/nodejs-install-helper.js @@ -8,8 +8,7 @@ import fs from "fs"; import https from "https"; import pkg from "./package.json" assert { type: "json" }; import { exec } from "child_process"; -import zlip from "zlib"; -const unzip = zlip.createUnzip(); +import unzipper from "unzipper"; let { version } = pkg; let arch = process.arch; @@ -91,7 +90,7 @@ if (process.env.CRSQLITE_NOPREBUILD) { } const file = fs.createWriteStream(distPath); - res.pipe(unzip).pipe(file); + res.pip(unzipper.Extract({ path: join(".", "dist") })); file.on("finish", () => { file.close(); console.log("Prebuilt binary downloaded"); diff --git a/core/package.json b/core/package.json index 57a65236e..8b53e87ef 100644 --- a/core/package.json +++ b/core/package.json @@ -32,5 +32,8 @@ "sqlite", "sqlite3", "crdt" - ] + ], + "dependencies": { + "unzipper": "^0.10.14" + } } diff --git a/core/pnpm-lock.yaml b/core/pnpm-lock.yaml new file mode 100644 index 000000000..8daaac1b4 --- /dev/null +++ b/core/pnpm-lock.yaml @@ -0,0 +1,209 @@ +lockfileVersion: '6.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +dependencies: + unzipper: + specifier: ^0.10.14 + version: 0.10.14 + +packages: + + /balanced-match@1.0.2: + resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + dev: false + + /big-integer@1.6.52: + resolution: {integrity: sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==} + engines: {node: '>=0.6'} + dev: false + + /binary@0.3.0: + resolution: {integrity: sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==} + dependencies: + buffers: 0.1.1 + chainsaw: 0.1.0 + dev: false + + /bluebird@3.4.7: + resolution: {integrity: sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==} + dev: false + + /brace-expansion@1.1.11: + resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} + dependencies: + balanced-match: 1.0.2 + concat-map: 0.0.1 + dev: false + + /buffer-indexof-polyfill@1.0.2: + resolution: {integrity: sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==} + engines: {node: '>=0.10'} + dev: false + + /buffers@0.1.1: + resolution: {integrity: sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==} + engines: {node: '>=0.2.0'} + dev: false + + /chainsaw@0.1.0: + resolution: {integrity: sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==} + dependencies: + traverse: 0.3.9 + dev: false + + /concat-map@0.0.1: + resolution: {integrity: sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=} + dev: false + + /core-util-is@1.0.3: + resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} + dev: false + + /duplexer2@0.1.4: + resolution: {integrity: sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==} + dependencies: + readable-stream: 2.3.8 + dev: false + + /fs.realpath@1.0.0: + resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} + dev: false + + /fstream@1.0.12: + resolution: {integrity: sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==} + engines: {node: '>=0.6'} + dependencies: + graceful-fs: 4.2.11 + inherits: 2.0.4 + mkdirp: 0.5.6 + rimraf: 2.7.1 + dev: false + + /glob@7.2.3: + resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} + dependencies: + fs.realpath: 1.0.0 + inflight: 1.0.6 + inherits: 2.0.4 + minimatch: 3.1.2 + once: 1.4.0 + path-is-absolute: 1.0.1 + dev: false + + /graceful-fs@4.2.11: + resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + dev: false + + /inflight@1.0.6: + resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} + dependencies: + once: 1.4.0 + wrappy: 1.0.2 + dev: false + + /inherits@2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + dev: false + + /isarray@1.0.0: + resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} + dev: false + + /listenercount@1.0.1: + resolution: {integrity: sha512-3mk/Zag0+IJxeDrxSgaDPy4zZ3w05PRZeJNnlWhzFz5OkX49J4krc+A8X2d2M69vGMBEX0uyl8M+W+8gH+kBqQ==} + dev: false + + /minimatch@3.1.2: + resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} + dependencies: + brace-expansion: 1.1.11 + dev: false + + /minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + dev: false + + /mkdirp@0.5.6: + resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==} + hasBin: true + dependencies: + minimist: 1.2.8 + dev: false + + /once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + dependencies: + wrappy: 1.0.2 + dev: false + + /path-is-absolute@1.0.1: + resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} + engines: {node: '>=0.10.0'} + dev: false + + /process-nextick-args@2.0.1: + resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + dev: false + + /readable-stream@2.3.8: + resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} + dependencies: + core-util-is: 1.0.3 + inherits: 2.0.4 + isarray: 1.0.0 + process-nextick-args: 2.0.1 + safe-buffer: 5.1.2 + string_decoder: 1.1.1 + util-deprecate: 1.0.2 + dev: false + + /rimraf@2.7.1: + resolution: {integrity: sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==} + hasBin: true + dependencies: + glob: 7.2.3 + dev: false + + /safe-buffer@5.1.2: + resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} + dev: false + + /setimmediate@1.0.5: + resolution: {integrity: sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==} + dev: false + + /string_decoder@1.1.1: + resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} + dependencies: + safe-buffer: 5.1.2 + dev: false + + /traverse@0.3.9: + resolution: {integrity: sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==} + dev: false + + /unzipper@0.10.14: + resolution: {integrity: sha512-ti4wZj+0bQTiX2KmKWuwj7lhV+2n//uXEotUmGuQqrbVZSEGFMbI68+c6JCQ8aAmUWYvtHEz2A8K6wXvueR/6g==} + dependencies: + big-integer: 1.6.52 + binary: 0.3.0 + bluebird: 3.4.7 + buffer-indexof-polyfill: 1.0.2 + duplexer2: 0.1.4 + fstream: 1.0.12 + graceful-fs: 4.2.11 + listenercount: 1.0.1 + readable-stream: 2.3.8 + setimmediate: 1.0.5 + dev: false + + /util-deprecate@1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + dev: false + + /wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + dev: false From 3cc1018aa862fbe3ddce5f8378f3734a89ade78d Mon Sep 17 00:00:00 2001 From: Matt <1009003+tantaman@users.noreply.github.com> Date: Thu, 30 Nov 2023 21:06:40 -0500 Subject: [PATCH 2/3] deal with zip files instead of gz --- core/nodejs-install-helper.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/core/nodejs-install-helper.js b/core/nodejs-install-helper.js index ca7d82452..f079a475a 100644 --- a/core/nodejs-install-helper.js +++ b/core/nodejs-install-helper.js @@ -51,7 +51,7 @@ if (process.env.CRSQLITE_NOPREBUILD) { break; } - const binaryUrl = `https://github.com/vlcn-io/cr-sqlite/releases/download/${version}/crsqlite-${os}-${arch}.gz`; + const binaryUrl = `https://github.com/vlcn-io/cr-sqlite/releases/download/${version}/crsqlite-${os}-${arch}.zip`; console.log(`Look for prebuilt binary from ${binaryUrl}`); const distPath = join("dist", `crsqlite.${ext}`); @@ -89,10 +89,7 @@ if (process.env.CRSQLITE_NOPREBUILD) { return; } - const file = fs.createWriteStream(distPath); - res.pip(unzipper.Extract({ path: join(".", "dist") })); - file.on("finish", () => { - file.close(); + res.pipe(unzipper.Extract({ path: join(".", "dist") })).on("close", () => { console.log("Prebuilt binary downloaded"); process.exit(0); }); From a205eb1279358ec39ba326f857db7e98fa7b488f Mon Sep 17 00:00:00 2001 From: Matt <1009003+tantaman@users.noreply.github.com> Date: Thu, 30 Nov 2023 21:18:31 -0500 Subject: [PATCH 3/3] fix zip packaging --- .github/workflows/publish.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 17ad18e86..eeb943f3d 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -37,7 +37,7 @@ jobs: run: | cd core make loadable - zip dist/crsqlite.zip dist/${{ matrix.library_name }} + cd dist; zip crsqlite.zip ${{ matrix.library_name }} - name: Upload binaries to release uses: svenstaro/upload-release-action@v2 @@ -73,7 +73,7 @@ jobs: run: | cd core export CI_MAYBE_TARGET="aarch64-apple-darwin" && make loadable - zip dist/crsqlite.zip dist/${{ matrix.library_name }} + cd dist; zip crsqlite.zip ${{ matrix.library_name }} - name: Upload macos dylib to release uses: svenstaro/upload-release-action@v2 @@ -126,7 +126,7 @@ jobs: run: | cd core export CI_MAYBE_TARGET="aarch64-unknown-linux-gnu" && export CI_GCC="aarch64-linux-gnu-gcc" && make loadable - zip dist/crsqlite.zip dist/${{ matrix.library_name }} + cd dist; zip crsqlite.zip ${{ matrix.library_name }} - name: Upload binaries to release uses: svenstaro/upload-release-action@v2 @@ -166,7 +166,7 @@ jobs: run: | cd core export CI_MAYBE_TARGET="x86_64-pc-windows-gnu" && export CI_GCC="x86_64-w64-mingw32-gcc" && make loadable - zip dist/crsqlite.zip dist/${{ matrix.library_name }} + cd dist; zip crsqlite.zip ${{ matrix.library_name }} - name: Upload binaries to release uses: svenstaro/upload-release-action@v2 @@ -206,7 +206,7 @@ jobs: run: | cd core export CI_MAYBE_TARGET="i686-pc-windows-gnu" && export CI_GCC="i686-w64-mingw32-gcc" && make loadable - zip dist/crsqlite.zip dist/${{ matrix.library_name }} + cd dist; zip crsqlite.zip ${{ matrix.library_name }} - name: Upload binaries to release uses: svenstaro/upload-release-action@v2 @@ -254,7 +254,7 @@ jobs: run: | cd core export ANDROID_TARGET=aarch64-linux-android; make loadable - zip dist/crsqlite.zip dist/${{ matrix.library_name }} + cd dist; zip crsqlite.zip ${{ matrix.library_name }} - name: Upload binaries to release uses: svenstaro/upload-release-action@v2