From eb465831e9bdf53086bd6b54940f5d492be57bff Mon Sep 17 00:00:00 2001 From: Stephen Haberman Date: Sat, 27 Nov 2021 11:55:21 -0600 Subject: [PATCH] chore: Run codegen with N=5, fix update-bins (#418) * chore: Go back to N=5 for codegen. * Fix `yarn bin2ts simple`. * Remove timestamp change. * Fix update-bins. --- .nvmrc | 1 + docker-compose.yml | 6 ------ integration/codegen.sh | 9 ++++++--- .../grpc-js/google/protobuf/wrappers.bin | Bin 4316 -> 4316 bytes .../meta-typings/google/protobuf/wrappers.bin | Bin 4316 -> 4316 bytes integration/meta-typings/google/type/date.bin | Bin 2086 -> 2086 bytes integration/meta-typings/import_dir/thing.bin | Bin 6408 -> 6652 bytes .../google/protobuf/wrappers.bin | Bin 4316 -> 4316 bytes .../simple-long/google/protobuf/wrappers.bin | Bin 4316 -> 4316 bytes .../google/protobuf/wrappers.bin | Bin 4316 -> 4316 bytes .../simple-optionals/import_dir/thing.bin | Bin 6408 -> 6652 bytes .../simple-snake/google/protobuf/wrappers.bin | Bin 4316 -> 4316 bytes integration/simple-snake/import_dir/thing.bin | Bin 6408 -> 6652 bytes .../google/protobuf/wrappers.bin | Bin 4316 -> 4316 bytes .../import_dir/thing.bin | Bin 6408 -> 6652 bytes .../simple/google/protobuf/wrappers.bin | Bin 4316 -> 4316 bytes integration/simple/google/type/date.bin | Bin 2086 -> 2086 bytes integration/simple/import_dir/thing.bin | Bin 6408 -> 6652 bytes integration/type-registry/bar/bar.bin | Bin 6719 -> 6963 bytes .../google/protobuf/timestamp.ts | 11 ++++++++++- integration/update-bins.sh | 6 ++++-- package.json | 8 ++++---- protoc.Dockerfile | 9 +++------ 23 files changed, 28 insertions(+), 22 deletions(-) create mode 100644 .nvmrc diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 000000000..29b7b9ad9 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +16.4.0 diff --git a/docker-compose.yml b/docker-compose.yml index 7a7b6acc7..6726a4e3e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,11 +4,5 @@ services: build: context: . dockerfile: "protoc.Dockerfile" - args: - BASE: "node:17-alpine3.14" - BIN: "/ts-proto" - PROTOC_VERSION: "3.19.1" - working_dir: "/host" volumes: - - "${PWD:-.}:/host" - ".:/ts-proto" diff --git a/integration/codegen.sh b/integration/codegen.sh index b204f1e9c..ab3048998 100755 --- a/integration/codegen.sh +++ b/integration/codegen.sh @@ -18,8 +18,10 @@ INTEGRATION_DIR=$(realpath $(dirname "$BASH_SOURCE")) -# Run the code generator in parallel, with one process per core. -N=$(nproc) +# Run the code generator in parallel. Note this is purposefully pinned to 5 because +# CI only has 2 cores, but we can go faster than that, and for me locally using all +# 16 cores is overly taxes the machine/kicks on fans/etc. 5 is a good balance. +N=5 echo "Generating typescript code for integration tests using ${N} cores..." @@ -28,6 +30,7 @@ if [ -n "${1}" ]; then dir="${@}" fi +cd $INTEGRATION_DIR list=$(find $dir -name "*.bin" -type f | grep -v dump-response.bin) for file in $list; do @@ -42,7 +45,7 @@ for file in $list; do fi ((i=i%N)); ((i++==0)) && wait - "${INTEGRATION_DIR}/../node_modules/.bin/ts-node" "${INTEGRATION_DIR}/codegen.ts" "${dir}" "${file}" "${params}" & + "../node_modules/.bin/ts-node" "./codegen.ts" "${dir}" "${file}" "${params}" & done wait diff --git a/integration/grpc-js/google/protobuf/wrappers.bin b/integration/grpc-js/google/protobuf/wrappers.bin index 5b4b18672f3f9cde1e43dc5f0ab67cd1ca9ff126..7f34d6ce0cd3a5ebf8b18b31567e0fd536c12f1b 100644 GIT binary patch delta 21 ccmcbkct>%9Du=KHqY^{adc_F}8-4o)0804=od5s; delta 21 ccmcbkct>%9DhHnggAzm4dc_F}8-4o)07|C@mjD0& diff --git a/integration/meta-typings/google/protobuf/wrappers.bin b/integration/meta-typings/google/protobuf/wrappers.bin index ba0182a949a0445f1e7e064194b0a8fe2a7ed22a..7f34d6ce0cd3a5ebf8b18b31567e0fd536c12f1b 100644 GIT binary patch delta 21 ccmcbkct>%9Du=KHqY^{adc_F}8-4o)0804=od5s; delta 21 ccmcbkct>%9DhH1QixNZCdc_F}8-4o)07|ZRxBB<7{-6cU%q!#OE=jE5@>6c{YrWThZ<`xtrO?;)zc~qZ^U5HhVVbW$f#y&P4 zPcBX_CJt^PW)%j{$vfCX1@FmnF|jZRv9&WYOk!k|Ixo9fmSZNnsaIlIqC$da9?+z` z5C8TsWoA!>liCM)nO2LeU)(o;)Z3-XgQf>V?8 z^HMZ4HQ`FY#_1KOmiQ&+j0ZE04ZvKpQ6Q^tFXs2hEWRxc9CFkerr|0J+=B4Ww z6y=xXCzYn@7nD>Mq!#O!Wag$8mn7yEO#GzHc~h5*U5HhVp>wkuV;>uj6Bj2J69=~t zvkHUL%9Du=KHqY^{adc_F}8-4o)0804=od5s; delta 21 ccmcbkct>%9DhH1QixNZCdc_F}8-4o)07|%9Du=KHqY^{adc_F}8-4o)0804=od5s; delta 21 ccmcbkct>%9DhICwgAzm4dc_F}8-4o)07{t#mH+?% diff --git a/integration/simple-optionals/google/protobuf/wrappers.bin b/integration/simple-optionals/google/protobuf/wrappers.bin index ba0182a949a0445f1e7e064194b0a8fe2a7ed22a..7f34d6ce0cd3a5ebf8b18b31567e0fd536c12f1b 100644 GIT binary patch delta 21 ccmcbkct>%9Du=KHqY^{adc_F}8-4o)0804=od5s; delta 21 ccmcbkct>%9DhH1QixNZCdc_F}8-4o)07|ZRxBB<7{-6cU%q!#OE=jE5@>6c{YrWThZ<`xtrO?;)zc~qZ^U5HhVVbW$f#y&P4 zPcBX_CJt^PW)%j{$vfCX1@FmnF|jZRv9&WYOk!k|Ixo9fmSZNnsaIlIqC$da9?+z` z5C8TsWoA!>liCM)nO2LeU)(o;)Z3-XgQf>V?8 z^HMZ4HQ`FY#_1KOmiQ&+j0ZE04ZvKpQ6Q^tFXs2hEWRxc9CFkerr|0J+=B4Ww z6y=xXCzYn@7nD>Mq!#O!Wag$8mn7yEO#GzHc~h5*U5HhVp>wkuV;>uj6Bj2J69=~t zvkHUL%9Du=KHqY^{adc_F}8-4o)0804=od5s; delta 21 ccmcbkct>%9DhH1QixNZCdc_F}8-4o)07|ZRxBB<7{-6cU%q!#OE=jE5@>6c{YrWThZ<`xtrO?;)zc~qZ^U5HhVVbW$f#y&P4 zPcBX_CJt^PW)%j{$vfCX1@FmnF|jZRv9&WYOk!k|Ixo9fmSZNnsaIlIqC$da9?+z` z5C8TsWoA!>liCM)nO2LeU)(o;)Z3-XgQf>V?8 z^HMZ4HQ`FY#_1KOmiQ&+j0ZE04ZvKpQ6Q^tFXs2hEWRxc9CFkerr|0J+=B4Ww z6y=xXCzYn@7nD>Mq!#O!Wag$8mn7yEO#GzHc~h5*U5HhVp>wkuV;>uj6Bj2J69=~t zvkHUL%9Du=KHqY^{adc_F}8-4o)0804=od5s; delta 21 ccmcbkct>%9DhH1QixNZCdc_F}8-4o)07|ZRxBB<7{-6cU%q!#OE=jE5@>6c{YrWThZ<`xtrO?;)zc~qZ^U5HhVVbW$f#y&P4 zPcBX_CJt^PW)%j{$vfCX1@FmnF|jZRv9&WYOk!k|Ixo9fmSZNnsaIlIqC$da9?+z` z5C8TsWoA!>liCM)nO2LeU)(o;)Z3-XgQf>V?8 z^HMZ4HQ`FY#_1KOmiQ&+j0ZE04ZvKpQ6Q^tFXs2hEWRxc9CFkerr|0J+=B4Ww z6y=xXCzYn@7nD>Mq!#O!Wag$8mn7yEO#GzHc~h5*U5HhVp>wkuV;>uj6Bj2J69=~t zvkHUL%9Du=KHqY^{adc_F}8-4o)0804=od5s; delta 21 ccmcbkct>%9DhH1QixNZCdc_F}8-4o)07|ZRxBB<7{-6cU%q!#OE=jE5@>6c{YrWThZ<`xtrO?;)zc~qZ^U5HhVVbW$f#y&P4 zPcBX_CJt^PW)%j{$vfCX1@FmnF|jZRv9&WYOk!k|Ixo9fmSZNnsaIlIqC$da9?+z` z5C8TsWoA!>liCM)nO2LeU)(o;)Z3-XgQf>V?8 z^HMZ4HQ`FY#_1KOmiQ&+j0ZE04ZvKpQ6Q^tFXs2hEWRxc9CFkerr|0J+=B4Ww z6y=xXCzYn@7nD>Mq!#O!Wag$8mn7yEO#GzHc~h5*U5HhVp>wkuV;>uj6Bj2J69=~t zvkHULm)$JFF_YcGE3qt5A;B{bXi{E@9?)P7 z%>+FzE(H*PiYfp_6>MQVh#Kq3j@(k*3bqi1K(&*_`IQ5K3iZ-cOI!=`lQV)-lk@XZ zG&D8gO2B676{nW?CFbQ9Yd{r)I6zf;Nu`-NK;_n4K+9b#fZobURWO@8flqStT7F+P z32rWC76wKkmUc!CL9V%sTr4aMObkLS-IE2y=IeDcGO=^)3}7iQf6 delta 246 zcmdmNw%=re2nUY@ixNZCW&MdV1&nnQ`!sa4(=$slN|W@G^KEDS=dbqt-1j8adfH_LL&WZ%3%z?V&&i;J0sfl-LLjzf@ZE+ZEU3j-5_ z5KGf!R`L0IO^i(JT)bSYATdUuAR`AqjLQt-3JG&Pg=hiFc1*q`{#?ETp@k77$jHG9 P)xrqW(y{r5L^KlsMrSo5 diff --git a/integration/type-registry/google/protobuf/timestamp.ts b/integration/type-registry/google/protobuf/timestamp.ts index 3369aef3b..2f723335c 100644 --- a/integration/type-registry/google/protobuf/timestamp.ts +++ b/integration/type-registry/google/protobuf/timestamp.ts @@ -57,7 +57,16 @@ export const protobufPackage = 'google.protobuf'; * .setNanos((int) ((millis % 1000) * 1000000)).build(); * * - * Example 5: Compute Timestamp from current time in Python. + * Example 5: Compute Timestamp from Java `Instant.now()`. + * + * Instant now = Instant.now(); + * + * Timestamp timestamp = + * Timestamp.newBuilder().setSeconds(now.getEpochSecond()) + * .setNanos(now.getNano()).build(); + * + * + * Example 6: Compute Timestamp from current time in Python. * * timestamp = Timestamp() * timestamp.GetCurrentTime() diff --git a/integration/update-bins.sh b/integration/update-bins.sh index ae2be93a5..07750f884 100755 --- a/integration/update-bins.sh +++ b/integration/update-bins.sh @@ -12,11 +12,13 @@ else PLUGIN_PATH="protoc-gen-dump" fi -PROTO_FILES=$(find . -maxdepth 2 -name "*.proto" -type f) +PROTO_FILES=$(find . -name "*.proto" -type f) for FILE in $PROTO_FILES; do echo "${FILE}" - INPUT_DIR="$(dirname "$FILE")" + # Strip the longest suffix starting at the 1st slash + INPUT_DIR="${FILE##./}" + INPUT_DIR="${INPUT_DIR%%/*}" OUTPUT_FILE="${FILE%proto}bin" protoc --experimental_allow_proto3_optional "--plugin=$PLUGIN_PATH" --dump_out=. "${FILE}" "-I${INPUT_DIR}" mv file.bin "${OUTPUT_FILE}" diff --git a/package.json b/package.json index 0691194c8..9e1e029ac 100644 --- a/package.json +++ b/package.json @@ -11,10 +11,10 @@ "build": "yarn tsc", "build:test": "yarn proto2bin && yarn proto2pbjs && yarn bin2ts", "build:test:local": "yarn proto2bin:local && yarn proto2pbjs:local && yarn bin2ts:local", - "proto2bin": "docker-compose run --rm --entrypoint bash -w /ts-proto/integration protoc update-bins.sh", - "proto2bin-node": "docker-compose run --rm --entrypoint bash -w /ts-proto/integration node update-bins.sh", - "proto2pbjs": "docker-compose run --rm --entrypoint bash -w /ts-proto/integration protoc pbjs.sh", - "bin2ts": "docker-compose run --rm --entrypoint bash -w /ts-proto/integration protoc codegen.sh", + "proto2bin": "docker-compose run --rm protoc update-bins.sh", + "proto2bin-node": "docker-compose run --rm node update-bins.sh", + "proto2pbjs": "docker-compose run --rm protoc pbjs.sh", + "bin2ts": "docker-compose run --rm protoc codegen.sh", "proto2bin:local": "integration/update-bins.sh", "proto2pbjs:local": "integration/pbjs.sh", "bin2ts:local": "integration/codegen.sh", diff --git a/protoc.Dockerfile b/protoc.Dockerfile index f8a127b02..ef3b50c6f 100644 --- a/protoc.Dockerfile +++ b/protoc.Dockerfile @@ -1,8 +1,6 @@ # Docker image for protoc -ARG BASE="alpine:3.14.3" -FROM $BASE +FROM node:17-alpine3.14 ARG PROTOC_VERSION="3.19.1" -ARG BIN="" RUN apk add bash RUN apk add gcompat @@ -11,7 +9,6 @@ RUN mkdir /usr/local/lib/protoc && unzip protoc.zip -d /usr/local/lib/protoc && RUN ln -s /usr/local/lib/protoc/bin/protoc /usr/local/bin/protoc RUN protoc --version +WORKDIR /ts-proto/integration -ENV PATH "$BIN:$PATH" - -ENTRYPOINT ["/usr/local/bin/protoc"] +ENTRYPOINT ["/bin/bash"]