feat: Include dockerized protoc

README.markdown

@@ -18,7 +18,7 @@
   - [Supported options](#supported-options)
   - [Only Types](#only-types)
   - [NestJS Support](NESTJS.markdown)
-- [Building](#building)
+- [Development](#development)
 - [Assumptions](#assumptions)
 - [Todo](#todo)
 - [OneOf Handling](#oneof-handling)
@@ -362,15 +362,45 @@ Kudos to our sponsors:
 
 If you need ts-proto customizations or priority support for your company, you can ping me at [via email](mailto:stephen.haberman@gmail.com).
 
-# Building
+# Development
 
-After running `yarn install`, run `./integration/pbjs.sh` to create the integration test types. These pbjs-generated files are not currently checked in.
+**Requirements**
 
-After this, the tests should pass.
+- [Docker](https://www.docker.com) or [protoc](https://github.com/protocolbuffers/protobuf/releases) v3.19.1
+- `yarn` — `npm install -g yarn`
 
-After making changes to `ts-proto`, you can run `cd integration` and `./codegen.sh` to re-generate the test case `*.ts` output files that are in each `integration/<test-case>/` directory.
+**Setup**
 
-The test suite's proto files (i.e. `simple.proto`, `batching.proto`, etc.) currently have serialized/`.bin` copies checked into git (i.e. `simple.bin`, `batching.bin`, etc.), so that the test suite can run without having to invoke the `protoc` build chain. I.e. if you change the `simple.proto`/etc. files, you'll need to run `./integration/update-bins.sh`, which does require having the `protoc` executable available.
+The commands below assume you have **Docker** installed. To use a **local** copy of `protoc` without docker, use commands suffixed with `:local`
+
+- Check out the [repository]() for the latest code.
+- Run `yarn install` to install the dependencies.
+- Run `yarn build:test` or `yarn build:test:local` to generate the test files.
+  > _This runs the following commands:_
+  >  - `proto2bin` — Converts integration test `.proto` files to `.bin`.
+  >  - `bin2ts` — Runs `ts-proto` on the `.bin` files to generate  `.ts`  files.
+  >  - `bin2pbjs` — Generates a reference implementation using `pbjs` for testing compatibility.
+- Run `yarn test`
+
+**Workflow**
+
+- Modifying the plugin implementation:
+  - Run `yarn bin2ts` or `yarn bin2ts:local`.  
+    _Since the proto files were not changed, you only need to regenerate the typescript files._
+  - Run `yarn test` to verify the typescript files are compatible with the reference implementation, and pass other tests.
+- Updating or adding `.proto` files in the integration directory:
+  - Run `yarn build:test` to regenerate the integration test files.
+  - Run `yarn test` to retest.
+
+**Contributing**
+
+- Run `yarn build:test` and `yarn test` to make sure everything works.
+- Run `yarn prettier` to format the typescript files.
+- Commit the changes:
+  - Also include the generated `.bin` files for the tests where you added or modified `.proto` files.  
+    > These are checked into git so that the test suite can run without having to invoke the `protoc` build chain.
+  - Also include the generated `.ts` files.
+- Create a pull request
 
 # Assumptions
 

aliases.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+PROJECT_ROOT=$(realpath $(dirname "$BASH_SOURCE"))
+PROJECT_ROOT_DOCKER="//ts-proto" # double slash to support git bash on windows
+
+# Alias docker-compose to make it usable from anywhere
+function _docker-compose() { docker-compose -f $PROJECT_ROOT/docker-compose.yml "$@"; }
+
+function protoc() { _docker-compose run --rm protoc "$@"; }
+function protoc-sh() { _docker-compose run --rm --entrypoint sh -- protoc "$@"; }
+function protoc-build() { _docker-compose build protoc; }
+function ts-protoc { protoc --plugin=$PROJECT_ROOT_DOCKER/protoc-gen-ts_proto "$@"; }

docker-compose.yml

@@ -0,0 +1,14 @@
+version: "3"
+services:
+  protoc:
+    build:
+      context: .
+      dockerfile: "protoc.Dockerfile"
+      args:
+        BASE: "node:17-alpine3.14"
+        BIN: "/ts-proto"
+        PROTOC_VERSION: "3.19.1"
+    working_dir: "/host"
+    volumes:
+      - "${PWD:-.}:/host"
+      - ".:/ts-proto"

integration/codegen.sh

@@ -9,23 +9,26 @@
 #
 #    Updates generated output for all integration tests.
 #
-# ./codegen.sh simple
+# ./codegen.sh simple value
 #
-#    Updates generated output only for the 'simple' integration test.
+#    Updates generated output only for the 'simple' and 'value' integration test.
 #
 # Each integration test can optionally have a `parameters.txt` file that will
 # be used as the ts-proto_opt... args for generating that test's code.
 
-dir=${1:-*}
+INTEGRATION_DIR=$(realpath $(dirname "$BASH_SOURCE"))
 
-N=6
+# Run the code generator in parallel, with one process per core.
+N=$(nproc)
+
+echo "Generating typescript code for integration tests using ${N} cores..."
 
 dir=.
 if [ -n "${1}" ]; then
-  dir=$1
+  dir="${@}"
 fi
 
-list=$(find "$dir" -name "*.bin" -type f | grep -v dump-response.bin)
+list=$(find $dir -name "*.bin" -type f | grep -v dump-response.bin)
 
 for file in $list; do
   echo "${file}"
@@ -39,7 +42,7 @@ for file in $list; do
   fi
 
   ((i=i%N)); ((i++==0)) && wait
-  ../node_modules/.bin/ts-node ./codegen.ts "${dir}" "${file}" "${params}" &
+  "${INTEGRATION_DIR}/../node_modules/.bin/ts-node" "${INTEGRATION_DIR}/codegen.ts" "${dir}" "${file}" "${params}" &
 done
 
 wait

integration/protoc-gen-dump.bat

@@ -0,0 +1 @@
+@bash protoc-gen-dump

integration/update-bins.sh

@@ -1,17 +1,23 @@
 #!/usr/bin/env bash
 
 # To avoid running the protoc pipeline all the time, we capture the incoming Plugin
-# proto requests into .bin files that then unit tests can pull in directly as needed.
-list=$(find . -name "*.proto" -type f)
+# proto requests into .bin PROTO_FILES that then unit tests can pull in directly as needed.
 
-for file in $list; do
-  echo "${file}"
-#  # Strip the longest suffix starting at the 1st slash
-  dir="${file##./}"
-  dir="${dir%%/*}"
+INTEGRATION_DIR=$(realpath $(dirname "$BASH_SOURCE"))
+cd $INTEGRATION_DIR;
 
-  # Strip the proto suffix and add bin
-  dest="${file%proto}bin"
-  protoc --experimental_allow_proto3_optional "--plugin=$(pwd)/protoc-gen-dump" --dump_out=. "${file}" "-I${dir}"
-  mv file.bin "${dest}"
+if [[ "$OSTYPE" == "msys" ]]; then
+  PLUGIN_PATH="protoc-gen-dump.bat"
+else
+  PLUGIN_PATH="protoc-gen-dump"
+fi
+
+PROTO_FILES=$(find . -maxdepth 2 -name "*.proto" -type f)
+
+for FILE in $PROTO_FILES; do
+  echo "${FILE}"
+  INPUT_DIR="$(dirname "$FILE")"
+  OUTPUT_FILE="${FILE%proto}bin"
+  protoc --experimental_allow_proto3_optional "--plugin=$PLUGIN_PATH" --dump_out=. "${FILE}" "-I${INPUT_DIR}"\
+   && mv FILE.bin "${OUTPUT_FILE}"
 done

package.json

@@ -9,10 +9,19 @@
   },
   "scripts": {
     "build": "yarn tsc",
-    "setup": "cd ./integration && ./pbjs.sh && ./update-bins.sh && ./codegen.sh",
+    "build:test": "yarn proto2bin && yarn bin2pbjs && yarn bin2ts",
+    "build:test:local": "yarn proto2bin:local && yarn bin2pbjs:local && yarn bin2ts:local",
+    "proto2bin": "docker-compose run --rm --entrypoint bash -w /ts-proto/integration protoc update-bins.sh",
+    "proto2bin-node": "docker-compose run --rm --entrypoint bash -w /ts-proto/integration node update-bins.sh",
+    "bin2pbjs": "docker-compose run --rm --entrypoint bash -w /ts-proto/integration protoc pbjs.sh",
+    "bin2ts": "docker-compose run --rm --entrypoint bash -w /ts-proto/integration protoc codegen.sh",
+    "proto2bin:local": "integration/update-bins.sh",
+    "bin2pbjs:local": "integration/pbjs.sh",
+    "bin2ts:local": "integration/codegen.sh",
     "test": "yarn jest -c jest.config.js --maxWorkers=2",
     "prettier": "prettier --write {src,tests}/**/*.ts",
-    "prettier:check": "prettier --list-different {src,tests}/**/*.ts"
+    "prettier:check": "prettier --list-different {src,tests}/**/*.ts",
+    "setup:docker": "docker-compose build"
   },
   "files": [
     "build"

protoc.Dockerfile

@@ -0,0 +1,17 @@
+# Docker image for protoc
+ARG BASE="alpine:3.14.3"
+FROM $BASE
+ARG PROTOC_VERSION="3.19.1"
+ARG BIN=""
+
+RUN apk add bash
+RUN apk add gcompat
+ADD "https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/protoc-$PROTOC_VERSION-linux-x86_64.zip" protoc.zip
+RUN mkdir /usr/local/lib/protoc && unzip protoc.zip -d /usr/local/lib/protoc && rm protoc.zip
+RUN ln -s /usr/local/lib/protoc/bin/protoc /usr/local/bin/protoc
+
+RUN protoc --version
+
+ENV PATH "$BIN:$PATH"
+
+ENTRYPOINT ["/usr/local/bin/protoc"]