From 0f4e525e4b10a2de5455f61307e42579ed85b90b Mon Sep 17 00:00:00 2001 From: Levent Demir Date: Wed, 26 Jul 2023 18:30:04 +0200 Subject: [PATCH 1/4] Improve build time Let go package manager handling go-ethereum and ethermint for build with docker. The difference is that go-ethereum and ethermint are now public so no need to clone them on-premise for docker build. Another needed update is the first line of go.mod for both repositories, i.e. replace ethereum by zama-ai in module path. Another improvement is to use the docker image of fhevm-tfhe-cli for key generation instead of build from source. Still the user can choose to build the binary from sources by setting the FHEVM_TFHE_CLI_PATH env variable. --- Makefile | 18 +++++------------- README.md | 16 ++++++++-------- docker/Dockerfile.evmos-node.local | 6 ++---- go.mod | 4 ++-- go.sum | 8 ++++---- scripts/prepare_volumes_from_fhe_tool.sh | 5 ----- 6 files changed, 21 insertions(+), 36 deletions(-) diff --git a/Makefile b/Makefile index d2e882c6..1a192c62 100755 --- a/Makefile +++ b/Makefile @@ -38,9 +38,10 @@ TFHE_RS_EXISTS := $(shell test -d $(TFHE_RS_PATH)/.git && echo "true" || echo "f TFHE_RS_VERSION ?= 0.3.0-beta.0 +# If defined, the local version will be used, docker image otherwise FHEVM_TFHE_CLI_PATH ?= $(WORKDIR)/fhevm-tfhe-cli FHEVM_TFHE_CLI_PATH_EXISTS := $(shell test -d $(FHEVM_TFHE_CLI_PATH)/.git && echo "true" || echo "false") -FHEVM_TFHE_CLI_VERSION ?= v0.1.1-renaming +FHEVM_TFHE_CLI_VERSION ?= v0.1.2 FHEVM_REQUIRES_DB_PATH ?= $(WORKDIR)/fhevm-requires-db FHEVM_REQUIRES_DB_PATH_EXISTS := $(shell test -d $(FHEVM_REQUIRES_DB_PATH)/.git && echo "true" || echo "false") @@ -48,7 +49,7 @@ FHEVM_REQUIRES_DB_VERSION ?= v0.1.0 FHEVM_SOLIDITY_PATH ?= $(WORKDIR)/fhevm-solidity FHEVM_SOLIDITY_PATH_EXISTS := $(shell test -d $(FHEVM_SOLIDITY_PATH)/.git && echo "true" || echo "false") -FHEVM_SOLIDITY_VERSION ?= ci/use-fhevm-tfhe-cli-multiarch +FHEVM_SOLIDITY_VERSION ?= main ETHERMINT_VERSION := $(shell ./scripts/get_module_version.sh go.mod zama.ai/ethermint) GO_ETHEREUM_VERSION := $(shell ./scripts/get_module_version.sh go.mod zama.ai/go-ethereum) @@ -321,7 +322,7 @@ $(WORKDIR)/: $(info WORKDIR) mkdir -p $(WORKDIR) -check-all-test-repo: check-fhevm-tfhe-cli check-fhevm-solidity +check-all-test-repo: check-fhevm-solidity update-go-mod: @cp go.mod $(UPDATE_GO_MOD) @@ -339,15 +340,6 @@ build-base-image: build-local-docker: -ifeq ($(GITHUB_ACTIONS),true) - $(info Running in a GitHub Actions workflow) -else - $(info Not running in a GitHub Actions workflow) - @$(MAKE) build-base-image - @$(MAKE) clone_go_ethereum - @$(MAKE) clone_ethermint -endif - $(MAKE) update-go-mod $(MAKE) check-tfhe-rs @docker compose -f docker-compose/docker-compose.local.yml build evmosnodelocal @@ -370,7 +362,7 @@ build-from-registry: generate_fhe_keys: @echo 'generate_fhe_keys' # Generate fhe global keys and copy into volumes - @bash ./scripts/prepare_volumes_from_fhe_tool.sh $(FHEVM_TFHE_CLI_PATH)/target/release + @bash ./scripts/prepare_volumes_from_fhe_tool_docker.sh $(FHEVM_TFHE_CLI_VERSION) run_evmos: diff --git a/README.md b/README.md index a548a0fd..c389d714 100644 --- a/README.md +++ b/README.md @@ -141,8 +141,8 @@ To execute the e2e test, here are the dependencies: | Name | Type | Variable name | where it is defined | | ------------- | ---------- | --------------------- | ------------------- | | evmos | repository | LOCAL_BUILD | .env | -| zbc-solidity | repository | ZBC_SOLIDITY_VERSION | Makefile/.env | -| zbc-fhe-tool | repository | ZBC_FHE_TOOL_VERSION | Makefile/.env | +| fhevm-solidity| repository | ZBC_SOLIDITY_VERSION | Makefile/.env | +| fhevm-tfhe-cli | repository | ZBC_FHE_TOOL_VERSION | Makefile/.env | | zbc-oracle-db | repository | ZBC_ORACLE_DB_VERSION | Makefile/.env | @@ -164,11 +164,11 @@ make stop_evmos - check you have all the needed repositories - - zbc-fhe-tool - - zbc-solidity + - fhevm-tfhe-cli + - fhevm-solidity - zbc-oracledb - init evmos node by calling /config/setup.sh file -- generate fhe keys using zbc-fhe-tool based on scripts/prepare_volumes_from_fhe_tool.sh script +- generate fhe keys using fhevm-tfhe-cli based on scripts/prepare_volumes_from_fhe_tool.sh script - copy them at the right folder using scripts/prepare_demo_local.sh script - start evmosnodelocal0 and oracledb (local build) using docker-compose/docker-compose.local.yml file - run the e2e test @@ -222,8 +222,8 @@ make stop_evmos | evmos | evmos | LOCAL_BUILD | .env | | ghcr.io/zama-ai/evmos-node | docker image name | hard-coded | docker-compose.validator.yml | | oracle-db-service | docker image name | hard-coded | docker-compose.validator.yml | -| zbc-solidity | repository | ZBC_SOLIDITY_VERSION | Makefile/.env | -| zbc-fhe-tool | repository | ZBC_FHE_TOOL_VERSION | Makefile/.env | +| fhevm-solidity | repository | ZBC_SOLIDITY_VERSION | Makefile/.env | +| fhevm-tfhe-cli | repository | ZBC_FHE_TOOL_VERSION | Makefile/.env | | zbc-oracle-db | repository | ZBC_ORACLE_DB_VERSION | Makefile/.env | @@ -236,7 +236,7 @@ Note:
Troubleshoot ghcr.io -Here is a tutorial on [how to manage ghcr.io access](https://github.com/zama-ai/zbc-fhe-tool#using-the-published-image-easiest-way). +Here is a tutorial on [how to manage ghcr.io access](https://github.com/zama-ai/fhevm-tfhe-cli#using-the-published-image-easiest-way). If you get trouble to pull image from ghcri.io, one can build it locally with ```bash diff --git a/docker/Dockerfile.evmos-node.local b/docker/Dockerfile.evmos-node.local index aa39303f..ef3c8193 100644 --- a/docker/Dockerfile.evmos-node.local +++ b/docker/Dockerfile.evmos-node.local @@ -1,4 +1,4 @@ -FROM ghcr.io/zama-ai/zama-zbc-build:latest AS build-env +FROM ghcr.io/zama-ai/zama-zbc-build:v0.1.3 AS build-env ENV LD_LIBRARY_PATH=/usr/lib/tfhe ADD . /src/evmos @@ -7,8 +7,6 @@ WORKDIR /src/evmos RUN mkdir -p /usr/include -RUN cp go.mod.updated /src/evmos/go.mod - RUN tail /src/evmos/go.mod RUN make build @@ -16,7 +14,7 @@ RUN ls /src/evmos RUN ls /src/evmos/build RUN mkdir -p /src/evmos/build -FROM ghcr.io/zama-ai/zama-zbc-build:latest +FROM ghcr.io/zama-ai/zama-zbc-build:v0.1.3 RUN apt-get update -y RUN apt-get install ca-certificates jq -y diff --git a/go.mod b/go.mod index 9857ff6a..747b3061 100644 --- a/go.mod +++ b/go.mod @@ -173,6 +173,6 @@ replace ( replace github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0 // TODO: Remove once our forks are public. -replace github.com/ethereum/go-ethereum v1.10.19 => github.com/zama-ai/go-ethereum v0.1.7 +replace github.com/ethereum/go-ethereum v1.10.19 => github.com/zama-ai/go-ethereum v0.1.8-beta -replace github.com/evmos/ethermint v0.19.3 => github.com/zama-ai/ethermint v0.1.2 +replace github.com/evmos/ethermint v0.19.3 => github.com/zama-ai/ethermint v0.1.3-beta diff --git a/go.sum b/go.sum index ace83802..59b1af99 100644 --- a/go.sum +++ b/go.sum @@ -862,10 +862,10 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/zama-ai/ethermint v1.0.0-test h1:x00Rxncw8qhzvV75S9lPbcmYBumGZ8ZZ8lGWgDawjHY= -github.com/zama-ai/ethermint v1.0.0-test/go.mod h1:JEqkvu6ZC6GV0nq7txpzmaRQdfDzwFay1SRumKsEkg8= -github.com/zama-ai/go-ethereum v1.0.1-test h1:WpTU7WP2W9fyZZYD+n0t26S1Njk6lzb3szKVgvVZ/SE= -github.com/zama-ai/go-ethereum v1.0.1-test/go.mod h1:IJBNMtzKcNHPtllYihy6BL2IgK1u+32JriaTbdt4v+w= +github.com/zama-ai/ethermint v0.1.2 h1:Zs7X5o6cb2hiZmWIrZICPqv5sK8QtackTaA2tK0djI0= +github.com/zama-ai/ethermint v0.1.2/go.mod h1:JEqkvu6ZC6GV0nq7txpzmaRQdfDzwFay1SRumKsEkg8= +github.com/zama-ai/go-ethereum v0.1.7 h1:krnwfFJajqn7lROjkt8wbCcthl2XTpGQCzkANjEnUig= +github.com/zama-ai/go-ethereum v0.1.7/go.mod h1:IJBNMtzKcNHPtllYihy6BL2IgK1u+32JriaTbdt4v+w= github.com/zondax/hid v0.9.0 h1:eiT3P6vNxAEVxXMw66eZUAAnU2zD33JBkfG/EnfAKl8= github.com/zondax/hid v0.9.0/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= diff --git a/scripts/prepare_volumes_from_fhe_tool.sh b/scripts/prepare_volumes_from_fhe_tool.sh index 4def966e..23d596b3 100755 --- a/scripts/prepare_volumes_from_fhe_tool.sh +++ b/scripts/prepare_volumes_from_fhe_tool.sh @@ -74,9 +74,4 @@ echo "Copying $key to $NETWORK_KEYS_PRIVATE_PATH, please wait ..." cp $KEYS_FULL_PATH/$key $NETWORK_KEYS_PRIVATE_PATH/cks -echo "###########################################################" -echo "If you want to execute a local test using the same keys," -echo "please run the following command in zbc-solidity:" -echo "prepare_fhe_keys_from_fhe_tool.sh $KEYS_FULL_PATH" -echo "###########################################################" From 699db367e7f8bcae1ef62bb6592ec3c77ed1a6b8 Mon Sep 17 00:00:00 2001 From: Levent Demir Date: Wed, 26 Jul 2023 18:42:38 +0200 Subject: [PATCH 2/4] build: add script file to use docker image for keygen --- Makefile | 15 ++++- .../prepare_volumes_from_fhe_tool_docker.sh | 67 +++++++++++++++++++ 2 files changed, 79 insertions(+), 3 deletions(-) create mode 100755 scripts/prepare_volumes_from_fhe_tool_docker.sh diff --git a/Makefile b/Makefile index 1a192c62..28f8f553 100755 --- a/Makefile +++ b/Makefile @@ -38,8 +38,9 @@ TFHE_RS_EXISTS := $(shell test -d $(TFHE_RS_PATH)/.git && echo "true" || echo "f TFHE_RS_VERSION ?= 0.3.0-beta.0 -# If defined, the local version will be used, docker image otherwise -FHEVM_TFHE_CLI_PATH ?= $(WORKDIR)/fhevm-tfhe-cli +# If defined here or in .env, the FHEVM_TFHE_CLI_VERSION version will be cloned, +# built and used, docker image otherwise +# FHEVM_TFHE_CLI_PATH FHEVM_TFHE_CLI_PATH_EXISTS := $(shell test -d $(FHEVM_TFHE_CLI_PATH)/.git && echo "true" || echo "false") FHEVM_TFHE_CLI_VERSION ?= v0.1.2 @@ -351,6 +352,7 @@ ifeq ($(LOCAL_BUILD),true) else $(info LOCAL_BUILD is not set, use docker registry for docker images) @$(MAKE) build-from-registry + endif @@ -362,7 +364,14 @@ build-from-registry: generate_fhe_keys: @echo 'generate_fhe_keys' # Generate fhe global keys and copy into volumes - @bash ./scripts/prepare_volumes_from_fhe_tool_docker.sh $(FHEVM_TFHE_CLI_VERSION) + ifeq ($(FHEVM_TFHE_CLI_PATH),) + @echo 'use of docker image to generate fhe keys ' + @bash ./scripts/prepare_volumes_from_fhe_tool_docker.sh $(FHEVM_TFHE_CLI_VERSION) + else + @$(MAKE) check-fhevm-tfhe-cli + @bash ./scripts/prepare_volumes_from_fhe_tool.sh $(FHEVM_TFHE_CLI_PATH)/target/release + endif + run_evmos: diff --git a/scripts/prepare_volumes_from_fhe_tool_docker.sh b/scripts/prepare_volumes_from_fhe_tool_docker.sh new file mode 100755 index 00000000..b10ccce6 --- /dev/null +++ b/scripts/prepare_volumes_from_fhe_tool_docker.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# This bash script creates global fhe keys +# and copy them to the right folder in volumes directory. +# It accepts the versin of fhevm-tfhe-cli as parameter. + +set -Eeuo pipefail + +if [ "$#" -ne 1 ]; then + echo "Please give the version of fhevm-tfhe-cli" + echo "Example: $(basename $0) v0.1.2" + exit +fi + +FHEVM_TFHE_CLI_VERSION=$1 +BINARY_NAME="fhevm-tfhe-cli" +DOCKER_IMAGE=ghcr.io/zama-ai/fhevm-tfhe-cli:$FHEVM_TFHE_CLI_VERSION +CURRENT_FOLDER=$PWD + + +KEYS_FULL_PATH=$CURRENT_FOLDER/res/keys +mkdir -p $KEYS_FULL_PATH + +docker run -v $PWD:/usr/local/app $DOCKER_IMAGE $BINARY_NAME generate-keys -d res/keys + + +echo "###########################################################" +echo "Keys creation is done, they are stored in $KEYS_FULL_PATH" +echo "###########################################################" + + +NETWORK_KEYS_PUBLIC_PATH=./volumes/network-public-fhe-keys +NETWORK_KEYS_PRIVATE_PATH=./volumes/network-private-fhe-keys + +MANDATORY_KEYS_LIST=('sks' 'cks' 'pks') + +for key in "${MANDATORY_KEYS_LIST[@]}" + do + if [ ! -f "$KEYS_FULL_PATH/$key" ]; then + echo "#####ATTENTION######" + echo "$key does not exist in $KEYS_FULL_PATH!" + echo "####################" + exit + fi +done + +echo "###########################################################" +echo "All the required keys exist in $KEYS_FULL_PATH" +echo "###########################################################" + +mkdir -p $NETWORK_KEYS_PUBLIC_PATH +mkdir -p $NETWORK_KEYS_PRIVATE_PATH + +key="sks" +echo "Copying $key to $NETWORK_KEYS_PUBLIC_PATH, please wait ..." +cp $KEYS_FULL_PATH/$key $NETWORK_KEYS_PUBLIC_PATH/sks + +key="pks" +echo "Copying $key to $NETWORK_KEYS_PUBLIC_PATH, please wait ..." +cp $KEYS_FULL_PATH/$key $NETWORK_KEYS_PUBLIC_PATH/pks + +key="cks" +echo "Copying $key to $NETWORK_KEYS_PRIVATE_PATH, please wait ..." +cp $KEYS_FULL_PATH/$key $NETWORK_KEYS_PRIVATE_PATH/cks + + + From f66549f9230042342161a35c351722c5e2982e96 Mon Sep 17 00:00:00 2001 From: Levent Demir Date: Thu, 27 Jul 2023 08:30:04 +0200 Subject: [PATCH 3/4] build: add script file to use docker image for keygen --fixes --- .env | 5 ++++- Makefile | 24 +++++++++--------------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/.env b/.env index dd9870f7..edbe10b6 100644 --- a/.env +++ b/.env @@ -5,4 +5,7 @@ # ZBC_FHE_TOOL_PATH=../zbc-fhe-tool #LOCAL_BUILD=false LOCAL_BUILD=true -GOPRIVATE=github.com/zama-ai/* + +# If false, FHEVM_TFHE_CLI_VERSION version will be cloned, +# built and used, docker image otherwise +USE_DOCKER_FOR_FHE_KEYS=true diff --git a/Makefile b/Makefile index 28f8f553..e58a11a6 100755 --- a/Makefile +++ b/Makefile @@ -37,10 +37,7 @@ TFHE_RS_PATH ?= $(WORKDIR)/tfhe-rs TFHE_RS_EXISTS := $(shell test -d $(TFHE_RS_PATH)/.git && echo "true" || echo "false") TFHE_RS_VERSION ?= 0.3.0-beta.0 - -# If defined here or in .env, the FHEVM_TFHE_CLI_VERSION version will be cloned, -# built and used, docker image otherwise -# FHEVM_TFHE_CLI_PATH +FHEVM_TFHE_CLI_PATH ?= $(WORKDIR)/fhevm-tfhe-cli FHEVM_TFHE_CLI_PATH_EXISTS := $(shell test -d $(FHEVM_TFHE_CLI_PATH)/.git && echo "true" || echo "false") FHEVM_TFHE_CLI_VERSION ?= v0.1.2 @@ -352,7 +349,6 @@ ifeq ($(LOCAL_BUILD),true) else $(info LOCAL_BUILD is not set, use docker registry for docker images) @$(MAKE) build-from-registry - endif @@ -362,16 +358,14 @@ build-from-registry: generate_fhe_keys: - @echo 'generate_fhe_keys' - # Generate fhe global keys and copy into volumes - ifeq ($(FHEVM_TFHE_CLI_PATH),) - @echo 'use of docker image to generate fhe keys ' - @bash ./scripts/prepare_volumes_from_fhe_tool_docker.sh $(FHEVM_TFHE_CLI_VERSION) - else - @$(MAKE) check-fhevm-tfhe-cli - @bash ./scripts/prepare_volumes_from_fhe_tool.sh $(FHEVM_TFHE_CLI_PATH)/target/release - endif - +ifeq ($(USE_DOCKER_FOR_FHE_KEYS),true) + $(info USE_DOCKER_FOR_FHE_KEYS is set, use docker) + @bash ./scripts/prepare_volumes_from_fhe_tool_docker.sh $(FHEVM_TFHE_CLI_VERSION) +else + $(info USE_DOCKER_FOR_FHE_KEYS is not set, build from sources) + @$(MAKE) check-fhevm-tfhe-cli + @bash ./scripts/prepare_volumes_from_fhe_tool.sh $(FHEVM_TFHE_CLI_PATH)/target/release +endif run_evmos: From 3a349f544f735c3c63a0115550d67ecef5fc95cd Mon Sep 17 00:00:00 2001 From: Levent Demir Date: Thu, 27 Jul 2023 09:13:33 +0200 Subject: [PATCH 4/4] build: by default use docker image for fhe keys gen --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index e58a11a6..7fb27e75 100755 --- a/Makefile +++ b/Makefile @@ -37,6 +37,7 @@ TFHE_RS_PATH ?= $(WORKDIR)/tfhe-rs TFHE_RS_EXISTS := $(shell test -d $(TFHE_RS_PATH)/.git && echo "true" || echo "false") TFHE_RS_VERSION ?= 0.3.0-beta.0 +USE_DOCKER_FOR_FHE_KEYS ?= true FHEVM_TFHE_CLI_PATH ?= $(WORKDIR)/fhevm-tfhe-cli FHEVM_TFHE_CLI_PATH_EXISTS := $(shell test -d $(FHEVM_TFHE_CLI_PATH)/.git && echo "true" || echo "false") FHEVM_TFHE_CLI_VERSION ?= v0.1.2