[WIP] Sovrin and Node GHA CD

system/docker/client/Dockerfile

@@ -12,10 +12,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     && rm -rf /var/lib/apt/lists/*
 
 # python
-ARG PYTHON3_VERSION
-ENV PYTHON3_VERSION=${PYTHON3_VERSION:-3.5}
 RUN apt-get update && apt-get install -y \
-        python${PYTHON3_VERSION} \
+        python3.5 \
         python3-pip \
         python-setuptools \
     && rm -rf /var/lib/apt/lists/*
@@ -25,37 +23,44 @@ RUN apt-get update && apt-get install -y \
 # https://github.com/pypa/pip/issues/5240
 # https://github.com/pypa/pip/issues/5221
 RUN python3 -m pip install -U \
-        pip \
+        pip==10.0.0 \
         setuptools \
         virtualenv==16.0.0 \
         pipenv==2018.11.26 \
     && pip3 list
 
-# libindy
-ARG LIBINDY_REPO_COMPONENT
-ARG LIBINDY_VERSION
-ENV LIBINDY_REPO_COMPONENT=${LIBINDY_REPO_COMPONENT:-master}
-ENV LIBINDY_VERSION=${LIBINDY_VERSION:-1.10.0~1171}
+# sovrin repo init
+ARG CLIENT_REPO_COMPONENT
+RUN : ${CLIENT_REPO_COMPONENT:?"CLIENT_REPO_COMPONENT must be provided"}
+ENV CLIENT_REPO_COMPONENT=${CLIENT_REPO_COMPONENT}
+
 RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CE7709D068DB5E88 \
-    && add-apt-repository "deb https://repo.sovrin.org/sdk/deb xenial ${LIBINDY_REPO_COMPONENT}" \
-    && apt-get update && apt-get install -y \
-        libindy=${LIBINDY_VERSION} \
-    && rm -rf /var/lib/apt/lists/*
+    && add-apt-repository "deb https://repo.sovrin.org/sdk/deb xenial ${CLIENT_REPO_COMPONENT}" \
+    && apt-get update
 
-# libsovtoken
-# FIXME
-#   - for now it's tricky to set proper version for master artifacts
-#   - defaults are for stable
-ARG LIBSOVTOKEN_INSTALL
-ARG LIBSOVTOKEN_VERSION
-ENV LIBSOVTOKEN_INSTALL=$LIBSOVTOKEN_INSTALL
-ENV LIBSOVTOKEN_VERSION=${LIBSOVTOKEN_VERSION:-0.10.0~74}
-RUN if [ "$LIBSOVTOKEN_INSTALL" = "yes" ]; then \
-        set -ex; \
-        apt-get update && apt-get install -y \
-            libsovtoken=${LIBSOVTOKEN_VERSION}; \
-        rm -rf /var/lib/apt/lists/*; \
-    fi
+# indy dependencies
+ARG LIBINDY_VERSION
+RUN : ${LIBINDY_VERSION:?"LIBINDY_VERSION must be provided"}
+ENV LIBINDY_VERSION=${LIBINDY_VERSION}
+
+RUN apt-get install -y \
+        libindy=${LIBINDY_VERSION}
+
+# TODO: Move to Sovrin tests
+# sovrin dependencies
+#ARG LIBSOVTOKEN_INSTALL
+#ARG LIBSOVTOKEN_VERSION
+#ENV LIBSOVTOKEN_INSTALL=$LIBSOVTOKEN_INSTALL
+#ENV LIBSOVTOKEN_VERSION=${LIBSOVTOKEN_VERSION:-0.10.0~74}
+#RUN if [ "$LIBSOVTOKEN_INSTALL" = "yes" ]; then \
+#        apt-get update && apt-get install -y \
+#            libsovtoken=${LIBSOVTOKEN_VERSION}; \
+#        rm -rf /var/lib/apt/lists/*; \
+#    fi
+
+# repo cleanup
+RUN add-apt-repository --remove "deb https://repo.sovrin.org/sdk/deb xenial ${CLIENT_REPO_COMPONENT}" \
+    && rm -rf /var/lib/apt/lists/*
 
 # user
 ARG u_id=1000

system/docker/docker-compose.yml

@@ -7,7 +7,7 @@ services:
       context: ./client
       args:
           - PYTHON3_VERSION
-          - LIBINDY_REPO_COMPONENT
+          - CLIENT_REPO_COMPONENT
           - LIBINDY_VERSION
           - LIBSOVTOKEN_INSTALL
           - LIBSOVTOKEN_VERSION
@@ -18,9 +18,10 @@ services:
     build:
       context: ./node
       args:
-          - INDY_NODE_REPO_COMPONENT
+          - NODE_REPO_COMPONENT
           - LIBINDY_CRYPTO_VERSION
           - PYTHON3_LIBINDY_CRYPTO_VERSION
+          - PYTHON3_PYZMQ_VERSION
           - INDY_PLENUM_VERSION
           - INDY_NODE_VERSION
           - TOKEN_PLUGINS_INSTALL

system/docker/node/Dockerfile

@@ -16,61 +16,75 @@ RUN apt-get update -y && apt-get install -y \
         stress \
 	&& rm -rf /var/lib/apt/lists/*
 
+# pip<10.0.0 is important. pip==10.0.0 doesn't work.
 RUN pip3 install -U \
         'pip<10.0.0' \
         setuptools
 
-ARG INDY_NODE_REPO_COMPONENT
+# sovrin repo init
+ARG NODE_REPO_COMPONENT
+RUN : ${NODE_REPO_COMPONENT:?"NODE_REPO_COMPONENT must be provided"}
+ENV NODE_REPO_COMPONENT=${NODE_REPO_COMPONENT}
+
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CE7709D068DB5E88 \
+    && echo "deb https://repo.sovrin.org/deb xenial ${NODE_REPO_COMPONENT}" >> /etc/apt/sources.list \
+    && apt-get update -y
+
+# indy depedencies
 ARG LIBINDY_CRYPTO_VERSION
+RUN : ${LIBINDY_CRYPTO_VERSION:?"LIBINDY_CRYPTO_VERSION must be provided"}
+ENV LIBINDY_CRYPTO_VERSION=${LIBINDY_CRYPTO_VERSION}
+
 ARG PYTHON3_LIBINDY_CRYPTO_VERSION
+RUN : ${PYTHON3_LIBINDY_CRYPTO_VERSION:?"PYTHON3_LIBINDY_CRYPTO_VERSION must be provided"}
+ENV PYTHON3_LIBINDY_CRYPTO_VERSION=${PYTHON3_LIBINDY_CRYPTO_VERSION}
+
+ARG PYTHON3_PYZMQ_VERSION
+RUN : ${PYTHON3_PYZMQ_VERSION:?"PYTHON3_PYZMQ_VERSION must be provided"}
+ENV PYTHON3_PYZMQ_VERSION=${PYTHON3_PYZMQ_VERSION}
+
 ARG INDY_PLENUM_VERSION
+RUN : ${INDY_PLENUM_VERSION:?"INDY_PLENUM_VERSION must be provided"}
+ENV INDY_PLENUM_VERSION=${INDY_PLENUM_VERSION}
+
 ARG INDY_NODE_VERSION
-ENV INDY_NODE_REPO_COMPONENT=${INDY_NODE_REPO_COMPONENT:-rc}
-ENV LIBINDY_CRYPTO_VERSION=${LIBINDY_CRYPTO_VERSION:-0.4.5}
-ENV PYTHON3_LIBINDY_CRYPTO_VERSION=${PYTHON3_LIBINDY_CRYPTO_VERSION:-0.4.5}
-ENV PYTHON3_PYZMQ_VERSION=${PYTHON3_PYZMQ_VERSION:-18.1.0}
-ENV INDY_PLENUM_VERSION=${INDY_PLENUM_VERSION:-1.12.2}
-ENV INDY_NODE_VERSION=${INDY_NODE_VERSION:-1.12.2~rc1}
-# ENV INDY_NODE_REPO_COMPONENT=${INDY_NODE_REPO_COMPONENT:-master}
-# ENV LIBINDY_CRYPTO_VERSION=${LIBINDY_CRYPTO_VERSION:-0.4.5}
-# ENV PYTHON3_LIBINDY_CRYPTO_VERSION=${PYTHON3_LIBINDY_CRYPTO_VERSION:-0.4.5}
-# ENV PYTHON3_PYZMQ_VERSION=${PYTHON3_PYZMQ_VERSION:-18.1.0}
-# ENV INDY_PLENUM_VERSION=${INDY_PLENUM_VERSION:-1.13.0~dev1012}
-# ENV INDY_NODE_VERSION=${INDY_NODE_VERSION:-1.13.0~dev1199}
-RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CE7709D068DB5E88 \
-    && echo "deb https://repo.sovrin.org/deb xenial ${INDY_NODE_REPO_COMPONENT}" >> /etc/apt/sources.list \
-    && apt-get update -y && apt-get install -y \
+RUN : ${INDY_NODE_VERSION:?"INDY_NODE_VERSION must be provided"}
+ENV INDY_NODE_VERSION=${INDY_NODE_VERSION}
+
+RUN apt-get install -y \
         libindy-crypto=${LIBINDY_CRYPTO_VERSION} \
         python3-indy-crypto=${PYTHON3_LIBINDY_CRYPTO_VERSION} \
         python3-pyzmq=${PYTHON3_PYZMQ_VERSION} \
         indy-plenum=${INDY_PLENUM_VERSION} \
-        indy-node=${INDY_NODE_VERSION} \
-	&& rm -rf /var/lib/apt/lists/*
+        indy-node=${INDY_NODE_VERSION}
 
+# TODO: Move to Sovrin tests
 # sovrin & token plugins
 # FIXME
 #   - for now it's tricky to set proper versions for master artifacts
 #   - defaults are for stable
-ARG TOKEN_PLUGINS_INSTALL
-ARG SOVRIN_VERSION
-ARG SOVTOKEN_VERSION
-ARG SOVTOKENFEES_VERSION
-ENV TOKEN_PLUGINS_INSTALL=$TOKEN_PLUGINS_INSTALL
-ENV SOVRIN_VERSION=${SOVRIN_VERSION:-1.1.69}
-ENV SOVTOKEN_VERSION=${SOVTOKEN_VERSION:-1.0.7~rc32}
-# ENV SOVRIN_VERSION=${SOVRIN_VERSION:-1.1.207}
-# ENV SOVTOKEN_VERSION=${SOVTOKEN_VERSION:-1.0.8~dev150}
-ENV SOVTOKENFEES_VERSION=${SOVTOKENFEES_VERSION:-${SOVTOKEN_VERSION}}
+#ARG TOKEN_PLUGINS_INSTALL
+#ARG SOVRIN_VERSION
+#ARG SOVTOKEN_VERSION
+#ARG SOVTOKENFEES_VERSION
+#ENV TOKEN_PLUGINS_INSTALL=$TOKEN_PLUGINS_INSTALL
+#ENV SOVRIN_VERSION=${SOVRIN_VERSION:-1.1.69}
+#ENV SOVTOKEN_VERSION=${SOVTOKEN_VERSION:-1.0.7~rc32}
+#ENV SOVTOKENFEES_VERSION=${SOVTOKENFEES_VERSION:-${SOVTOKEN_VERSION}}
 #RUN if [ "$TOKEN_PLUGINS_INSTALL" = "yes" ]; then \
-RUN \
-        set -ex; \
-        apt-get update && apt-get install -y \
-            sovrin=${SOVRIN_VERSION} \
-            sovtoken=${SOVTOKEN_VERSION} \
-            sovtokenfees=${SOVTOKENFEES_VERSION}; \
-        rm -rf /var/lib/apt/lists/*; \
-    fi
+#        set -ex; \
+#        apt-get update && apt-get install -y \
+#            sovrin=${SOVRIN_VERSION} \
+#            sovtoken=${SOVTOKEN_VERSION} \
+#            sovtokenfees=${SOVTOKENFEES_VERSION}; \
+#        rm -rf /var/lib/apt/lists/*; \
+#    fi
+
+# repo cleanup
+RUN sed '/deb https:\/\/repo.sovrin.org\/sdk\/deb xenial ${NODE_REPO_COMPONENT}/d' /etc/apt/sources.list \
+    && rm -rf /var/lib/apt/lists/*
 
+# network config
 RUN echo " " >> /etc/indy/indy_config.py \
     && echo "NETWORK_NAME = 'sandbox'" >> /etc/indy/indy_config.py \
     && echo " " >> /etc/indy/indy_config.py \
@@ -79,7 +93,3 @@ RUN echo " " >> /etc/indy/indy_config.py \
 
 RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
 COPY test_key.pub /root/.ssh/authorized_keys
-## hotfix indy-node-control files here
-#COPY node_control_utils_1_1_35.py /usr/local/lib/python3.5/dist-packages/indy_node/utils/node_control_utils.py
-#RUN chmod 644 /usr/local/lib/python3.5/dist-packages/indy_node/utils/node_control_utils.py \
-#    && chown root:root /usr/local/lib/python3.5/dist-packages/indy_node/utils/node_control_utils.py

system/docker/prepare.sh

@@ -1,5 +1,10 @@
 #!/bin/bash
 
+set -o errexit
+set -o pipefail
+#set -o nounset
+set -o xtrace
+
 DEF_TEST_NETWORK_NAME="indy-test-automation-network"
 # TODO limit default subnet range to reduce risk of overlapping with system resources
 DEF_TEST_NETWORK_SUBNET="10.0.0.0/24"
@@ -18,8 +23,6 @@ if [ "$1" = "--help" ] ; then
     exit 0
 fi
 
-set -ex
-
 test_network_name="${1:-$DEF_TEST_NETWORK_NAME}"
 test_network_subnet="${2:-$DEF_TEST_NETWORK_SUBNET}"
 
@@ -34,14 +37,13 @@ image_repository="hyperledger/indy-test-automation"
 docker_compose_image_name="${image_repository}:docker-compose"
 
 node_env_variables=" \
-    PYTHON3_VERSION \
-    LIBINDY_REPO_COMPONENT \
+    NODE_REPO_COMPONENT \
     LIBINDY_VERSION \
     LIBSOVTOKEN_INSTALL \
     LIBSOVTOKEN_VERSION \
 "
 client_env_variables=" \
-    INDY_NODE_REPO_COMPONENT \
+    CLIENT_REPO_COMPONENT \
     LIBINDY_CRYPTO_VERSION \
     PYTHON3_LIBINDY_CRYPTO_VERSION \
     INDY_PLENUM_VERSION \
@@ -72,8 +74,7 @@ docker run -t --rm \
     -u "$user_id" \
     -e "IMAGE_REPOSITORY=$image_repository" \
     -e u_id="$user_id" \
-    -e PYTHON3_VERSION \
-    -e LIBINDY_REPO_COMPONENT \
+    -e CLIENT_REPO_COMPONENT \
     -e LIBINDY_VERSION \
     -e LIBSOVTOKEN_INSTALL \
     -e LIBSOVTOKEN_VERSION \
@@ -88,9 +89,10 @@ docker run -t --rm \
     -u "$user_id" \
     -e "IMAGE_REPOSITORY=$image_repository" \
     -e u_id="$user_id" \
-    -e INDY_NODE_REPO_COMPONENT \
+    -e NODE_REPO_COMPONENT \
     -e LIBINDY_CRYPTO_VERSION \
     -e PYTHON3_LIBINDY_CRYPTO_VERSION \
+    -e PYTHON3_PYZMQ_VERSION \
     -e INDY_PLENUM_VERSION \
     -e INDY_NODE_VERSION \
     -e TOKEN_PLUGINS_INSTALL \
@@ -101,7 +103,7 @@ docker run -t --rm \
 
 docker images "$image_repository"
 
-# 4. clean existing envronment
+# 4. clean existing environment
 $docker_routine_path/clean.sh "$test_network_name"
 
 # 5. remove test network if exists