Dev/hshami/udate wiki (#527)

* update documentation

.devcontainer/Dockerfile

@@ -1,68 +1,75 @@
 # Dependencies versions
-ARG PYTHON_VERSION="3.9.10"
+ARG PYTHON_VERSION="3.9"
 
-# see https://hub.docker.com/_/python/?tab=tag
-FROM python:${PYTHON_VERSION}-buster
+FROM mcr.microsoft.com/vscode/devcontainers/python:0-${PYTHON_VERSION}
 
 # Dependencies versions
-ARG AZURE_CLI_VERSION="2.24.0-1~buster"
-ARG NODE_VERSION="15.x"
+ARG AZURE_CLI_VERSION="2.34.1-1~bullseye"
+ARG NODE_VERSION="17.x"
 ARG DOTNET_VERSION="5.0"
-ARG JDK_VERSION="2:1.11-71"
-ARG MAVEN_VERSION="3.6.0-1"
+ARG JDK_VERSION="2:1.11-72"
+ARG MAVEN_VERSION="3.6.3-5"
 
-RUN apt-get update && \
-    apt-get upgrade -y && \
-    apt-get install -y --no-install-recommends build-essential && \
-    apt-get install -y libffi-dev libssl-dev vim
+# Bring in utility scripts
+COPY .devcontainer/library-scripts/*.sh /tmp/library-scripts/
 
-# Install python development dependencies
-ADD requirements_dev.txt requirements.txt /tmp/
-RUN python -m pip install --upgrade pip
-RUN pip install -r /tmp/requirements_dev.txt
+RUN \
+    # Install some basics
+    apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends build-essential apt-utils && \
+    apt-get install -y libffi-dev libssl-dev vim sudo && \
+    apt-get upgrade -y
 
 RUN \
-    # Install the Azure CLI and IoT extension
+    # Upgrade to latest pip
+    python -m pip install --upgrade pip && \
+    # Install node, npm
+    curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION} | bash - && \
+    apt-get install -y nodejs && \
+    npm install npm@latest -g && \
+    # Install azure-cli
     apt-get install -y ca-certificates curl apt-transport-https lsb-release gnupg && \
     curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/microsoft.gpg > /dev/null && \
     echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/azure-cli.list && \
     apt-get update && apt-get install -y azure-cli=${AZURE_CLI_VERSION} && \
     az extension add --name azure-iot --system && \
     az extension update --name azure-iot && \
-    # Install Docker CE CLI
-    apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common lsb-release && \
-    curl -fsSL https://download.docker.com/linux/$(lsb_release -is | tr '[:upper:]' '[:lower:]')/gpg | apt-key add - 2>/dev/null && \
-    add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/$(lsb_release -is | tr '[:upper:]' '[:lower:]') $(lsb_release -cs) stable" && \
-    apt-get update && \
-    apt-get install -y docker-ce-cli && \
-    # Install node, npm, Yoeman, node.js modules
-    curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION} | bash - && \
-    apt-get install -y nodejs && \
+    # Use Docker script from script library to set things up - enable non-root docker, user vscode, using moby
+    /bin/bash /tmp/library-scripts/docker-in-docker-debian.sh "true" "vscode" "true" \
+    # Install Yoeman, node.js modules
     npm install -g yo && \
     npm i -g yo generator-azure-iot-edge-module && \
-    cd ~ && \ 
     # Install dotnet SDK
-    wget https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb && \
+    cd ~ && \
+    wget https://packages.microsoft.com/config/debian/11/packages-microsoft-prod.deb -O packages-microsoft-prod.deb && \
     dpkg -i packages-microsoft-prod.deb && \
     rm packages-microsoft-prod.deb && \ 
     apt-get update && apt-get install -y apt-transport-https && \
     apt-get update && apt-get install -y dotnet-sdk-${DOTNET_VERSION} && \
     # Install Java JDK, Maven
     apt-get install -y default-jdk=${JDK_VERSION} && \
     apt-get install -y maven=${MAVEN_VERSION} && \
-    #
+    # Install bash completion
+    apt-get update && apt-get -y install bash-completion && \
     # Clean up
     apt-get autoremove -y && \
     apt-get clean -y && \
     rm -rf /tmp/* && \
     rm -rf /var/lib/apt/lists/*
 
-# Customize bash
+# customize vscode environmnet
+USER vscode
 RUN \
-    # Git command prompt
-    git clone https://github.com/magicmonty/bash-git-prompt.git ~/.bash-git-prompt --depth=1 && \
-    echo "if [ -f \"$HOME/.bash-git-prompt/gitprompt.sh\" ]; then GIT_PROMPT_ONLY_IN_REPO=1 && source $HOME/.bash-git-prompt/gitprompt.sh; fi" >> "/root/.bashrc" && \
-    # Install bash completion
-    apt-get update && \
-    apt-get -y install bash-completion && \
-    echo "source /usr/share/bash-completion/bash_completion" >> ~/.bashrc 
+    git clone https://github.com/magicmonty/bash-git-prompt.git $HOME/.bash-git-prompt --depth=1 && \
+    echo "\n# setup GIT prompt and completion\nif [ -f \"$HOME/.bash-git-prompt/gitprompt.sh\" ]; then\n  GIT_PROMPT_ONLY_IN_REPO=1 && source $HOME/.bash-git-prompt/gitprompt.sh;\nfi" >> $HOME/.bashrc && \
+    echo "source /usr/share/bash-completion/bash_completion" >> $HOME/.bashrc && \
+    echo "\n# add local python programs to PATH\nexport PATH=$PATH:$HOME/.local/bin\n" >> $HOME/.bashrc && \
+    # enable some useful aliases
+    sed -i -e "s/#alias/alias/" $HOME/.bashrc && \
+    # add the cookiecutter
+    pip install cookiecutter
+
+# launch docker-ce
+ENTRYPOINT [ "/usr/local/share/docker-init.sh" ]
+CMD [ "sleep", "infinity" ]

.devcontainer/devcontainer.json

@@ -4,25 +4,23 @@
     "name": "iotedgedev",
     "build": {
         "dockerfile": "Dockerfile",
-        "context": "..",
+        "context": ".."
     },
 
-    "runArgs": [
-        "--network",
-        "host"
-    ],
-
-    "postCreateCommand": "pip install -e .",
+    "runArgs": ["--init", "--privileged"],
     "mounts": [
         // Keep command history 
         "source=ostf-bashhistory,target=/commandhistory,type=volume",
-        // Use hosts docker socket
-        "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind",
+        // Use docker-in-docker socket
+        "source=dind-var-lib-docker,target=/var/lib/docker,type=volume"
     ],
 
+    "overrideCommand": false,
+    "postCreateCommand": "docker image prune -a -f && sudo chown vscode:users -R /home/vscode/Dev && pip install -r requirements_dev.txt && pip install -e .",
+
     // Set *default* container specific settings.json values on container create.
     "settings": {
-        "#terminal.integrated.defaultProfile.linux#": "/bin/bash",
+        "#terminal.integrated.defaultProfile.linux#": "/bin/bash"
     },
 
     // Add the IDs of extensions you want installed when the container is created.
@@ -34,5 +32,9 @@
         "streetsidesoftware.code-spell-checker",
         "yzhang.markdown-all-in-one",
         "davidanson.vscode-markdownlint"
-    ]
+    ],
+
+    "workspaceMount": "source=${localWorkspaceFolder},target=/home/vscode/Dev,type=bind,consistency=cached",
+    "workspaceFolder": "/home/vscode/Dev",
+    "remoteUser": "vscode"
 }

.devcontainer/library-scripts/docker-in-docker-debian.sh

@@ -0,0 +1,236 @@
+#!/usr/bin/env bash
+#-------------------------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information.
+#-------------------------------------------------------------------------------------------------------------
+#
+# Docs: https://github.com/microsoft/vscode-dev-containers/blob/main/script-library/docs/docker-in-docker.md
+# Maintainer: The VS Code and Codespaces Teams
+#
+# Syntax: ./docker-in-docker-debian.sh [enable non-root docker access flag] [non-root user] [use moby]
+
+ENABLE_NONROOT_DOCKER=${1:-"true"}
+USERNAME=${2:-"automatic"}
+USE_MOBY=${3:-"true"}
+MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc"
+
+set -e
+
+if [ "$(id -u)" -ne 0 ]; then
+    echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.'
+    exit 1
+fi
+
+# Determine the appropriate non-root user
+if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then
+    USERNAME=""
+    POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)")
+    for CURRENT_USER in ${POSSIBLE_USERS[@]}; do
+        if id -u ${CURRENT_USER} > /dev/null 2>&1; then
+            USERNAME=${CURRENT_USER}
+            break
+        fi
+    done
+    if [ "${USERNAME}" = "" ]; then
+        USERNAME=root
+    fi
+elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then
+    USERNAME=root
+fi
+
+# Get central common setting
+get_common_setting() {
+    if [ "${common_settings_file_loaded}" != "true" ]; then
+        curl -sfL "https://aka.ms/vscode-dev-containers/script-library/settings.env" 2>/dev/null -o /tmp/vsdc-settings.env || echo "Could not download settings file. Skipping."
+        common_settings_file_loaded=true
+    fi
+    if [ -f "/tmp/vsdc-settings.env" ]; then
+        local multi_line=""
+        if [ "$2" = "true" ]; then multi_line="-z"; fi
+        local result="$(grep ${multi_line} -oP "$1=\"?\K[^\"]+" /tmp/vsdc-settings.env | tr -d '\0')"
+        if [ ! -z "${result}" ]; then declare -g $1="${result}"; fi
+    fi
+    echo "$1=${!1}"
+}
+
+# Function to run apt-get if needed
+apt_get_update_if_needed()
+{
+    if [ ! -d "/var/lib/apt/lists" ] || [ "$(ls /var/lib/apt/lists/ | wc -l)" = "0" ]; then
+        echo "Running apt-get update..."
+        apt-get update
+    else
+        echo "Skipping apt-get update."
+    fi
+}
+
+# Checks if packages are installed and installs them if not
+check_packages() {
+    if ! dpkg -s "$@" > /dev/null 2>&1; then
+        apt_get_update_if_needed
+        apt-get -y install --no-install-recommends "$@"
+    fi
+}
+
+# Ensure apt is in non-interactive to avoid prompts
+export DEBIAN_FRONTEND=noninteractive
+
+# Install dependencies
+check_packages apt-transport-https curl ca-certificates lxc pigz iptables gnupg2 dirmngr
+
+# Swap to legacy iptables for compatibility
+if type iptables-legacy > /dev/null 2>&1; then
+    update-alternatives --set iptables /usr/sbin/iptables-legacy
+    update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
+fi
+
+# Install Docker / Moby CLI if not already installed
+architecture="$(dpkg --print-architecture)"
+if type docker > /dev/null 2>&1 && type dockerd > /dev/null 2>&1; then
+    echo "Docker / Moby CLI and Engine already installed."
+else
+    # Source /etc/os-release to get OS info
+    . /etc/os-release
+    if [ "${USE_MOBY}" = "true" ]; then
+        # Import key safely (new 'signed-by' method rather than deprecated apt-key approach) and install
+        get_common_setting MICROSOFT_GPG_KEYS_URI
+        curl -sSL ${MICROSOFT_GPG_KEYS_URI} | gpg --dearmor > /usr/share/keyrings/microsoft-archive-keyring.gpg
+        echo "deb [arch=${architecture} signed-by=/usr/share/keyrings/microsoft-archive-keyring.gpg] https://packages.microsoft.com/repos/microsoft-${ID}-${VERSION_CODENAME}-prod ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/microsoft.list
+        apt-get update
+        apt-get -y install --no-install-recommends moby-cli moby-buildx moby-engine
+        apt-get -y install --no-install-recommends moby-compose || echo "(*) Package moby-compose (Docker Compose v2) not available for ${VERSION_CODENAME} ${architecture}. Skipping."
+    else
+        # Import key safely (new 'signed-by' method rather than deprecated apt-key approach) and install
+        curl -fsSL https://download.docker.com/linux/${ID}/gpg | gpg --dearmor > /usr/share/keyrings/docker-archive-keyring.gpg
+        echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/${ID} ${VERSION_CODENAME} stable" > /etc/apt/sources.list.d/docker.list
+        apt-get update
+        apt-get -y install --no-install-recommends docker-ce-cli docker-ce
+    fi
+fi
+
+echo "Finished installing docker / moby"
+
+# Install Docker Compose if not already installed  and is on a supported architecture
+if type docker-compose > /dev/null 2>&1; then
+    echo "Docker Compose already installed."
+else
+    target_compose_arch="${architecture}"
+    if [ "${target_compose_arch}" != "x86_64" ]; then
+        # Use pip to get a version that runns on this architecture
+        if ! dpkg -s python3-minimal python3-pip libffi-dev python3-venv > /dev/null 2>&1; then
+            apt_get_update_if_needed
+            apt-get -y install python3-minimal python3-pip libffi-dev python3-venv
+        fi
+        export PIPX_HOME=/usr/local/pipx
+        mkdir -p ${PIPX_HOME}
+        export PIPX_BIN_DIR=/usr/local/bin
+        export PYTHONUSERBASE=/tmp/pip-tmp
+        export PIP_CACHE_DIR=/tmp/pip-tmp/cache
+        pipx_bin=pipx
+        if ! type pipx > /dev/null 2>&1; then
+            pip3 install --disable-pip-version-check --no-warn-script-location  --no-cache-dir --user pipx
+            pipx_bin=/tmp/pip-tmp/bin/pipx
+        fi
+        ${pipx_bin} install --system-site-packages --pip-args '--no-cache-dir --force-reinstall' docker-compose
+        rm -rf /tmp/pip-tmp
+    else
+        latest_compose_version=$(basename "$(curl -fsSL -o /dev/null -w "%{url_effective}" https://github.com/docker/compose/releases/latest)")
+        curl -fsSL "https://github.com/docker/compose/releases/download/${latest_compose_version}/docker-compose-$(uname -s)-${target_compose_arch}" -o /usr/local/bin/docker-compose
+        chmod +x /usr/local/bin/docker-compose
+    fi
+fi
+
+# If init file already exists, exit
+if [ -f "/usr/local/share/docker-init.sh" ]; then
+    echo "/usr/local/share/docker-init.sh already exists, so exiting."
+    exit 0
+fi
+echo "docker-init doesnt exist..."
+
+# Add user to the docker group
+if [ "${ENABLE_NONROOT_DOCKER}" = "true" ]; then
+    if ! getent group docker > /dev/null 2>&1; then
+        groupadd docker
+    fi
+
+    usermod -aG docker ${USERNAME}
+fi
+
+tee /usr/local/share/docker-init.sh > /dev/null \
+<< 'EOF'
+#!/usr/bin/env bash
+#-------------------------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information.
+#-------------------------------------------------------------------------------------------------------------
+
+sudoIf()
+{
+    if [ "$(id -u)" -ne 0 ]; then
+        sudo "$@"
+    else
+        "$@"
+    fi
+}
+
+# explicitly remove dockerd and containerd PID file to ensure that it can start properly if it was stopped uncleanly
+# ie: docker kill <ID>
+sudoIf find /run /var/run -iname 'docker*.pid' -delete || :
+sudoIf find /run /var/run -iname 'container*.pid' -delete || :
+
+set -e
+
+## Dind wrapper script from docker team
+# Maintained: https://github.com/moby/moby/blob/master/hack/dind
+
+export container=docker
+
+if [ -d /sys/kernel/security ] && ! sudoIf mountpoint -q /sys/kernel/security; then
+	sudoIf mount -t securityfs none /sys/kernel/security || {
+		echo >&2 'Could not mount /sys/kernel/security.'
+		echo >&2 'AppArmor detection and --privileged mode might break.'
+	}
+fi
+
+# Mount /tmp (conditionally)
+if ! sudoIf mountpoint -q /tmp; then
+	sudoIf mount -t tmpfs none /tmp
+fi
+
+# cgroup v2: enable nesting
+if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
+	# move the init process (PID 1) from the root group to the /init group,
+	# otherwise writing subtree_control fails with EBUSY.
+	sudoIf mkdir -p /sys/fs/cgroup/init
+	sudoIf echo 1 > /sys/fs/cgroup/init/cgroup.procs
+	# enable controllers
+	sudoIf sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \
+		> /sys/fs/cgroup/cgroup.subtree_control
+fi
+## Dind wrapper over.
+
+# Handle DNS
+set +e
+cat /etc/resolv.conf | grep -i 'internal.cloudapp.net'
+if [ $? -eq 0 ]
+then
+  echo "Setting dockerd Azure DNS."
+  CUSTOMDNS="--dns 168.63.129.16"
+else
+  echo "Not setting dockerd DNS manually."
+  CUSTOMDNS=""
+fi
+set -e
+
+# Start docker/moby engine
+( sudoIf dockerd $CUSTOMDNS > /tmp/dockerd.log 2>&1 ) &
+
+set +e
+
+# Execute whatever commands were passed in (if any). This allows us
+# to set this script to ENTRYPOINT while still executing the default CMD.
+exec "$@"
+EOF
+
+chmod +x /usr/local/share/docker-init.sh
+chown ${USERNAME}:root /usr/local/share/docker-init.sh