Refactor docker scripts and make WasmEdge default (#1902)

* Refactor Docker scripts * Make WasmEdge default * Add libgmp to kagome docker images * Add libgmp to kagome-dev * Change kagome-dev image
qdrvm · Jan 17, 2024 · 3efdad9 · 3efdad9
1 parent 7e43d5e
commit 3efdad9
Show file tree

Hide file tree

Showing 11 changed files with 40 additions and 26 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -59,7 +59,7 @@ jobs:
 
     name: "${{ matrix.options.name }}"
     runs-on: ubuntu-latest
-    container: qdrvm/kagome-dev@sha256:14d5aa92c971073e82ba9bbac9b615701e99c71f64e58bdd45e5b3dbc09944bd
+    container: qdrvm/kagome-dev@sha256:2d70246c32418a3dd45c246d3f5c2dd99bdafde145b903271849affe476c4cfc
     steps:
       - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6
       - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed
@@ -97,7 +97,7 @@ jobs:
 
     name: "${{ matrix.options.name }}"
     runs-on: [ self-hosted ]
-    container: qdrvm/kagome-dev@sha256:14d5aa92c971073e82ba9bbac9b615701e99c71f64e58bdd45e5b3dbc09944bd
+    container: qdrvm/kagome-dev@sha256:2d70246c32418a3dd45c246d3f5c2dd99bdafde145b903271849affe476c4cfc
     steps:
       - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6
       - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed
@@ -135,7 +135,7 @@ jobs:
 
     name: "${{ matrix.options.name }}"
     runs-on: [ self-hosted ]
-    container: qdrvm/kagome-dev@sha256:14d5aa92c971073e82ba9bbac9b615701e99c71f64e58bdd45e5b3dbc09944bd
+    container: qdrvm/kagome-dev@sha256:2d70246c32418a3dd45c246d3f5c2dd99bdafde145b903271849affe476c4cfc
     steps:
       - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6
       - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed
@@ -159,7 +159,7 @@ jobs:
   clang-tidy:
     name: "Linux: clang-tidy"
     runs-on: ubuntu-latest
-    container: qdrvm/kagome-dev@sha256:14d5aa92c971073e82ba9bbac9b615701e99c71f64e58bdd45e5b3dbc09944bd
+    container: qdrvm/kagome-dev@sha256:2d70246c32418a3dd45c246d3f5c2dd99bdafde145b903271849affe476c4cfc
     steps:
       - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6
         with:
@@ -180,7 +180,7 @@ jobs:
     if: ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/') || contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted')  }}
     name: "Self-hosted: Linux: gcc-12 coverage/sonar"
     runs-on: [ self-hosted ]
-    container: qdrvm/kagome-dev@sha256:14d5aa92c971073e82ba9bbac9b615701e99c71f64e58bdd45e5b3dbc09944bd
+    container: qdrvm/kagome-dev@sha256:2d70246c32418a3dd45c246d3f5c2dd99bdafde145b903271849affe476c4cfc
     steps:
       - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6
       - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed
@@ -228,7 +228,7 @@ jobs:
             build-type: "Release"
     name: "${{ matrix.options.name }}"
     runs-on: ubuntu-latest
-    container: qdrvm/kagome-dev@sha256:14d5aa92c971073e82ba9bbac9b615701e99c71f64e58bdd45e5b3dbc09944bd
+    container: qdrvm/kagome-dev@sha256:2d70246c32418a3dd45c246d3f5c2dd99bdafde145b903271849affe476c4cfc
     steps:
       - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6
       - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed
@@ -252,7 +252,7 @@ jobs:
             build-type: "Release"
     name: "${{ matrix.options.name }}"
     runs-on: [ self-hosted ]
-    container: qdrvm/kagome-dev@sha256:14d5aa92c971073e82ba9bbac9b615701e99c71f64e58bdd45e5b3dbc09944bd
+    container: qdrvm/kagome-dev@sha256:2d70246c32418a3dd45c246d3f5c2dd99bdafde145b903271849affe476c4cfc
     steps:
       - uses: actions/checkout@dc323e67f16fb5f7663d20ff7941f27f5809e9b6
       - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -63,7 +63,7 @@ if(NOT($ENV{CI}) OR NOT($ENV{GITHUB_ACTIONS}))
 endif()
 option(EXTERNAL_PROJECT "Build external project" ${_EXTERNAL_PROJECT_DEFAULT})
 
-set(WASM_COMPILER WAVM CACHE STRING "WebAssembly compiler built into Kagome: one of [WAVM, WasmEdge]")
+set(WASM_COMPILER WasmEdge CACHE STRING "WebAssembly compiler built into Kagome: one of [WAVM, WasmEdge]")
 
 if (NOT ${WASM_COMPILER} MATCHES "^(WAVM|WasmEdge)$")
   fatal_error("WASM_COMPILER is set to ${WASM_COMPILER} but should be one of [WAVM, WasmEdge]")

diff --git a/Makefile b/Makefile
@@ -23,8 +23,7 @@ debug_docker:
 	VERSION=0.0.1 BUILD_DIR=build BUILD_TYPE=Debug ./housekeeping/docker/kagome/build_and_push.sh
 
 custom_docker:
-	VERSION=0.0.1 BUILD_TYPE=Custom BUILD_DIR=build ./housekeeping/docker/kagome/build_and_push.sh
+	VERSION=0.0.1 BUILD_TYPE=Custom ./housekeeping/docker/kagome/build_and_push.sh
 
 clear:
 	rm -rf build
-
diff --git a/housekeeping/docker/kagome-dev/build_and_push.sh b/housekeeping/docker/kagome-dev/build_and_push.sh
@@ -7,7 +7,7 @@
 
 cd "$(dirname "$0")"
 
-VERSION=${VERSION:-4}
+VERSION=${VERSION:-5}
 TAG=qdrvm/kagome-dev:$VERSION
 
 docker build -t ${TAG}-minideb -f minideb.Dockerfile .

diff --git a/housekeeping/docker/kagome-dev/make.sh b/housekeeping/docker/kagome-dev/make.sh
@@ -27,6 +27,9 @@ else # CI
   BUILD_THREADS="${BUILD_THREADS:-$(( $(nproc 2>/dev/null || sysctl -n hw.ncpu) ))}"
   # Configure CI git security
   git config --global --add safe.directory /__w/kagome/kagome
+fi
+
+if [[ "${KAGOME_IN_DOCKER}" = 1 ]]; then
   source /venv/bin/activate
 fi
 

diff --git a/housekeeping/docker/kagome-dev/minideb.Dockerfile b/housekeeping/docker/kagome-dev/minideb.Dockerfile
@@ -2,6 +2,9 @@ FROM bitnami/minideb@sha256:c84aa349081c182fbaa92434eb6f6a0e14e69fc70aa1a5af2c7a
 MAINTAINER Vladimir Shcherba <abrehchs@gmail.com>
 
 SHELL ["/bin/bash", "-c"]
+
+ENV KAGOME_IN_DOCKER=1
+
 # add some required tools
 RUN apt-get update && \
     apt-get install --no-install-recommends -y \
@@ -71,6 +74,7 @@ RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /
         ccache \
         lcov \
         zlib1g-dev \
+        libgmp10 \
         unzip && \
     rm -rf /var/lib/apt/lists/*
 
@@ -114,32 +118,32 @@ ENV CXX=g++-12
 
 RUN update-alternatives --install /usr/bin/python       python       /venv/bin/python3              90 && \
     update-alternatives --install /usr/bin/python       python       /usr/bin/python3               80 && \
-
+    \
     update-alternatives --install /usr/bin/clang-tidy   clang-tidy   /usr/bin/clang-tidy-11         90 && \
     update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-11       90 && \
     update-alternatives --install /usr/bin/clang        clang        /usr/lib/llvm-11/bin/clang-11  90 && \
     update-alternatives --install /usr/bin/clang++      clang++      /usr/bin/clang++-11            90 && \
-
+    \
     update-alternatives --install /usr/bin/clang-tidy   clang-tidy   /usr/bin/clang-tidy-12         80 && \
     update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-12       80 && \
     update-alternatives --install /usr/bin/clang        clang        /usr/lib/llvm-12/bin/clang-12  80 && \
     update-alternatives --install /usr/bin/clang++      clang++      /usr/bin/clang++-12            80 && \
-
+    \
     update-alternatives --install /usr/bin/clang-tidy   clang-tidy   /usr/bin/clang-tidy-13         70 && \
     update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-13       70 && \
     update-alternatives --install /usr/bin/clang        clang        /usr/lib/llvm-13/bin/clang-13  70 && \
     update-alternatives --install /usr/bin/clang++      clang++      /usr/bin/clang++-13            70 && \
-
+    \
     update-alternatives --install /usr/bin/clang-tidy   clang-tidy   /usr/bin/clang-tidy-14         60 && \
     update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-14       60 && \
     update-alternatives --install /usr/bin/clang        clang        /usr/lib/llvm-14/bin/clang-14  60 && \
     update-alternatives --install /usr/bin/clang++      clang++      /usr/bin/clang++-14            60 && \
-
+    \
     update-alternatives --install /usr/bin/clang-tidy   clang-tidy   /usr/bin/clang-tidy-15         50 && \
     update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-15       50 && \
     update-alternatives --install /usr/bin/clang        clang        /usr/lib/llvm-15/bin/clang-15  50 && \
     update-alternatives --install /usr/bin/clang++      clang++      /usr/bin/clang++-15            50 && \
-
+    \
     update-alternatives --install /usr/bin/gcc          gcc          /usr/bin/gcc-12                90 && \
     update-alternatives --install /usr/bin/g++          g++          /usr/bin/g++-12                90 && \
     update-alternatives --install /usr/bin/gcov         gcov         /usr/bin/gcov-12               90
diff --git a/housekeeping/docker/kagome/build_and_push.sh b/housekeeping/docker/kagome/build_and_push.sh
@@ -5,8 +5,11 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-if [[ "${CI}" ]]; then # CI
+if [[ "${CI}" ]]; then
   git config --global --add safe.directory /__w/kagome/kagome
+fi
+
+if [[ "${KAGOME_IN_DOCKER}" = 1 ]]; then
   source /venv/bin/activate
 fi
 
@@ -25,7 +28,7 @@ if [ "$BUILD_TYPE" != "Debug" ] && [ "$BUILD_TYPE" != "Release" ] && [ "$BUILD_T
 fi
 
 VERSION="${VERSION:?VERSION variable is not defined}"
-VERSION_COMMIT="${VERSION}"
+
 # For github action we need remove ref prefix
 if [ "$VERSION" = "refs/heads/master" ]; then
   VERSION=latest

diff --git a/housekeeping/docker/kagome/minideb-debug.Dockerfile b/housekeeping/docker/kagome/minideb-debug.Dockerfile
@@ -8,10 +8,10 @@ RUN apt-get update && \
     curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \
     echo \
       "deb http://deb.debian.org/debian/ experimental main" | tee -a /etc/apt/sources.list.d/docker.list > /dev/null && \
-    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \  
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
     add-apt-repository -y "deb http://deb.debian.org/debian/ testing main" && \
     apt-get update && \
-    apt-get install --no-install-recommends -y libstdc++6 gdb gdbserver vim libc6 libnsl2 libatomic1 && \
+    apt-get install --no-install-recommends -y libgmp10 libstdc++6 gdb gdbserver vim libc6 libnsl2 libatomic1 && \
     rm -rf /var/lib/apt/lists/*
 
 COPY kagome /usr/local/bin/
diff --git a/housekeeping/docker/kagome/minideb-release.Dockerfile b/housekeeping/docker/kagome/minideb-release.Dockerfile
@@ -8,11 +8,10 @@ RUN apt-get update && \
     curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \
     echo \
       "deb http://deb.debian.org/debian/ experimental main" | tee -a /etc/apt/sources.list.d/docker.list > /dev/null && \
-    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \  
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
     add-apt-repository -y "deb http://deb.debian.org/debian/ testing main" && \
     apt-get update && \
-    apt-get install --no-install-recommends -y libstdc++6 libc6 libnsl2 libatomic1 gdb gdbserver && \
+    apt-get install --no-install-recommends -y libgmp10 libstdc++6 libc6 libnsl2 libatomic1 gdb gdbserver && \
     rm -rf /var/lib/apt/lists/*
 
-
 COPY kagome /usr/local/bin/
diff --git a/housekeeping/make_build.sh b/housekeeping/make_build.sh
@@ -10,9 +10,12 @@ BUILD_FINAL_TARGET="${BUILD_FINAL_TARGET:-test}"
 
 if [[ -z "${CI}" ]]; then
   BUILD_THREADS="${BUILD_THREADS:-$(( $(nproc 2>/dev/null || sysctl -n hw.ncpu) / 2 + 1 ))}"
-else # CI
+else
   BUILD_THREADS="${BUILD_THREADS:-$(( $(nproc 2>/dev/null || sysctl -n hw.ncpu) ))}"
   git config --global --add safe.directory /__w/kagome/kagome
+fi
+
+if [[ "${KAGOME_IN_DOCKER}" = 1 ]]; then
   source /venv/bin/activate
 fi
 

diff --git a/housekeeping/make_external_build.sh b/housekeeping/make_external_build.sh
@@ -33,9 +33,12 @@ mkdir -p "$EXTERNAL_PROJECT_BINARY_DIR"
 
 if [[ -z "${CI}" ]]; then
   BUILD_THREADS="${BUILD_THREADS:-$(( $(nproc 2>/dev/null || sysctl -n hw.ncpu) / 2 + 1 ))}"
-else # CI
+else
   BUILD_THREADS="${BUILD_THREADS:-$(( $(nproc 2>/dev/null || sysctl -n hw.ncpu) ))}"
   git config --global --add safe.directory /__w/kagome/kagome
+fi
+
+if [[ "${KAGOME_IN_DOCKER}" = 1 ]]; then
   source /venv/bin/activate
 fi