chore: Run codegen with N=5, fix update-bins (#418)

* chore: Go back to N=5 for codegen. * Fix `yarn bin2ts simple`. * Remove timestamp change. * Fix update-bins.
stephenh · Nov 27, 2021 · eb46583 · eb46583
1 parent 93546c6
commit eb46583
Show file tree

Hide file tree

Showing 23 changed files with 28 additions and 22 deletions.
diff --git a/.nvmrc b/.nvmrc
@@ -0,0 +1 @@
+16.4.0
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -4,11 +4,5 @@ services:
     build:
       context: .
       dockerfile: "protoc.Dockerfile"
-      args:
-        BASE: "node:17-alpine3.14"
-        BIN: "/ts-proto"
-        PROTOC_VERSION: "3.19.1"
-    working_dir: "/host"
     volumes:
-      - "${PWD:-.}:/host"
       - ".:/ts-proto"
diff --git a/integration/codegen.sh b/integration/codegen.sh
@@ -18,8 +18,10 @@
 
 INTEGRATION_DIR=$(realpath $(dirname "$BASH_SOURCE"))
 
-# Run the code generator in parallel, with one process per core.
-N=$(nproc)
+# Run the code generator in parallel. Note this is purposefully pinned to 5 because
+# CI only has 2 cores, but we can go faster than that, and for me locally using all
+# 16 cores is overly taxes the machine/kicks on fans/etc. 5 is a good balance.
+N=5
 
 echo "Generating typescript code for integration tests using ${N} cores..."
 
@@ -28,6 +30,7 @@ if [ -n "${1}" ]; then
   dir="${@}"
 fi
 
+cd $INTEGRATION_DIR
 list=$(find $dir -name "*.bin" -type f | grep -v dump-response.bin)
 
 for file in $list; do
@@ -42,7 +45,7 @@ for file in $list; do
   fi
 
   ((i=i%N)); ((i++==0)) && wait
-  "${INTEGRATION_DIR}/../node_modules/.bin/ts-node" "${INTEGRATION_DIR}/codegen.ts" "${dir}" "${file}" "${params}" &
+  "../node_modules/.bin/ts-node" "./codegen.ts" "${dir}" "${file}" "${params}" &
 done
 
 wait
diff --git a/integration/grpc-js/google/protobuf/wrappers.bin b/integration/grpc-js/google/protobuf/wrappers.bin
diff --git a/integration/meta-typings/google/protobuf/wrappers.bin b/integration/meta-typings/google/protobuf/wrappers.bin
diff --git a/integration/meta-typings/google/type/date.bin b/integration/meta-typings/google/type/date.bin
diff --git a/integration/meta-typings/import_dir/thing.bin b/integration/meta-typings/import_dir/thing.bin
diff --git a/integration/simple-long-string/google/protobuf/wrappers.bin b/integration/simple-long-string/google/protobuf/wrappers.bin
diff --git a/integration/simple-long/google/protobuf/wrappers.bin b/integration/simple-long/google/protobuf/wrappers.bin
diff --git a/integration/simple-optionals/google/protobuf/wrappers.bin b/integration/simple-optionals/google/protobuf/wrappers.bin
diff --git a/integration/simple-optionals/import_dir/thing.bin b/integration/simple-optionals/import_dir/thing.bin
diff --git a/integration/simple-snake/google/protobuf/wrappers.bin b/integration/simple-snake/google/protobuf/wrappers.bin
diff --git a/integration/simple-snake/import_dir/thing.bin b/integration/simple-snake/import_dir/thing.bin
diff --git a/integration/simple-unrecognized-enum/google/protobuf/wrappers.bin b/integration/simple-unrecognized-enum/google/protobuf/wrappers.bin
diff --git a/integration/simple-unrecognized-enum/import_dir/thing.bin b/integration/simple-unrecognized-enum/import_dir/thing.bin
diff --git a/integration/simple/google/protobuf/wrappers.bin b/integration/simple/google/protobuf/wrappers.bin
diff --git a/integration/simple/google/type/date.bin b/integration/simple/google/type/date.bin
diff --git a/integration/simple/import_dir/thing.bin b/integration/simple/import_dir/thing.bin
diff --git a/integration/type-registry/bar/bar.bin b/integration/type-registry/bar/bar.bin
diff --git a/integration/type-registry/google/protobuf/timestamp.ts b/integration/type-registry/google/protobuf/timestamp.ts
@@ -57,7 +57,16 @@ export const protobufPackage = 'google.protobuf';
  *         .setNanos((int) ((millis % 1000) * 1000000)).build();
  *
  *
- * Example 5: Compute Timestamp from current time in Python.
+ * Example 5: Compute Timestamp from Java `Instant.now()`.
+ *
+ *     Instant now = Instant.now();
+ *
+ *     Timestamp timestamp =
+ *         Timestamp.newBuilder().setSeconds(now.getEpochSecond())
+ *             .setNanos(now.getNano()).build();
+ *
+ *
+ * Example 6: Compute Timestamp from current time in Python.
  *
  *     timestamp = Timestamp()
  *     timestamp.GetCurrentTime()

diff --git a/integration/update-bins.sh b/integration/update-bins.sh
@@ -12,11 +12,13 @@ else
   PLUGIN_PATH="protoc-gen-dump"
 fi
 
-PROTO_FILES=$(find . -maxdepth 2 -name "*.proto" -type f)
+PROTO_FILES=$(find . -name "*.proto" -type f)
 
 for FILE in $PROTO_FILES; do
   echo "${FILE}"
-  INPUT_DIR="$(dirname "$FILE")"
+  # Strip the longest suffix starting at the 1st slash
+  INPUT_DIR="${FILE##./}"
+  INPUT_DIR="${INPUT_DIR%%/*}"
   OUTPUT_FILE="${FILE%proto}bin"
   protoc --experimental_allow_proto3_optional "--plugin=$PLUGIN_PATH" --dump_out=. "${FILE}" "-I${INPUT_DIR}"
   mv file.bin "${OUTPUT_FILE}"

diff --git a/package.json b/package.json
@@ -11,10 +11,10 @@
     "build": "yarn tsc",
     "build:test": "yarn proto2bin && yarn proto2pbjs && yarn bin2ts",
     "build:test:local": "yarn proto2bin:local && yarn proto2pbjs:local && yarn bin2ts:local",
-    "proto2bin": "docker-compose run --rm --entrypoint bash -w /ts-proto/integration protoc update-bins.sh",
-    "proto2bin-node": "docker-compose run --rm --entrypoint bash -w /ts-proto/integration node update-bins.sh",
-    "proto2pbjs": "docker-compose run --rm --entrypoint bash -w /ts-proto/integration protoc pbjs.sh",
-    "bin2ts": "docker-compose run --rm --entrypoint bash -w /ts-proto/integration protoc codegen.sh",
+    "proto2bin": "docker-compose run --rm protoc update-bins.sh",
+    "proto2bin-node": "docker-compose run --rm node update-bins.sh",
+    "proto2pbjs": "docker-compose run --rm protoc pbjs.sh",
+    "bin2ts": "docker-compose run --rm protoc codegen.sh",
     "proto2bin:local": "integration/update-bins.sh",
     "proto2pbjs:local": "integration/pbjs.sh",
     "bin2ts:local": "integration/codegen.sh",

diff --git a/protoc.Dockerfile b/protoc.Dockerfile
@@ -1,8 +1,6 @@
 # Docker image for protoc
-ARG BASE="alpine:3.14.3"
-FROM $BASE
+FROM node:17-alpine3.14
 ARG PROTOC_VERSION="3.19.1"
-ARG BIN=""
 
 RUN apk add bash
 RUN apk add gcompat
@@ -11,7 +9,6 @@ RUN mkdir /usr/local/lib/protoc && unzip protoc.zip -d /usr/local/lib/protoc &&
 RUN ln -s /usr/local/lib/protoc/bin/protoc /usr/local/bin/protoc
 
 RUN protoc --version
+WORKDIR /ts-proto/integration
 
-ENV PATH "$BIN:$PATH"
-
-ENTRYPOINT ["/usr/local/bin/protoc"]
+ENTRYPOINT ["/bin/bash"]