[WIP] Add IPv6 only KIND deployment to CI matrix.

Signed-off-by: Billy McFall <22157057+Billy99@users.noreply.github.com>

.github/workflows/test.yml

@@ -11,6 +11,7 @@ env:
   K8S_VERSION: v1.18.2
   KIND_CLUSTER_NAME: ovn
   KIND_INSTALL_INGRESS: true
+  KIND_UPDATE_SYSTEM: true
 
 jobs:
   build:
@@ -127,10 +128,27 @@ jobs:
            name: "HA"
          - enabled: "false"
            name: "noHA"
+        ipv4:
+         - enabled: "true"
+           name: "IPv4"
+         - enabled: "false"
+           name: "noIPv4"
+        ipv6:
+         - enabled: "true"
+           name: "IPv6"
+         - enabled: "false"
+           name: "noIPv6"
+        exclude:
+         - {"ipv4": {"enabled": "false"}, "ipv6": {"enabled": "false"}}
+         - {"ipv4": {"enabled": "true"}, "ipv6": {"enabled": "true"}}
+         - {"ipv4": {"enabled": "true"}, "ipv6": {"enabled": "false"}}
+         - {"ipv4": {"enabled": "false"}, "ipv6": {"enabled": "true"}, "ha": {"enabled": "true"}}
     needs: k8s
     env:
-      JOB_NAME: "${{ matrix.target }}-${{ matrix.ha.name }}"
+      JOB_NAME: "${{ matrix.target }}-${{ matrix.ha.name }}-${{ matrix.ipv4.name }}${{ matrix.ipv6.name }}"
       KIND_HA: "${{ matrix.ha.enabled }}"
+      KIND_IPV4_SUPPORT: "${{ matrix.ipv4.enabled }}"
+      KIND_IPV6_SUPPORT: "${{ matrix.ipv6.enabled }}"
     steps:
 
     - name: Free up disk space
@@ -151,6 +169,11 @@ jobs:
         echo "::set-env name=GOPATH::$GOPATH"
         export PATH=$GOPATH/bin:$PATH
         echo "::add-path::$GOPATH/bin"
+  
+    - name: IPv6 Disable firewall
+      if: matrix.ipv6.enabled == 'true' && matrix.ipv4.enabled == 'false'
+      run: |
+        sudo ufw disable
 
     - name: Restore Kubernetes from cache
       id: cache-k8s

contrib/kind.sh

@@ -94,6 +94,7 @@ print_params()
      echo "KIND_IPV4_SUPPORT = $KIND_IPV4_SUPPORT"
      echo "KIND_IPV6_SUPPORT = $KIND_IPV6_SUPPORT"
      echo "KIND_NUM_WORKER = $KIND_NUM_WORKER"
+     echo "KIND_UPDATE_SYSTEM = $KIND_UPDATE_SYSTEM"
      echo ""
 }
 
@@ -108,6 +109,7 @@ KIND_CONFIG=${KIND_CONFIG:-./kind.yaml.j2}
 KIND_REMOVE_TAINT=${KIND_REMOVE_TAINT:-true}
 KIND_IPV4_SUPPORT=${KIND_IPV4_SUPPORT:-true}
 KIND_IPV6_SUPPORT=${KIND_IPV6_SUPPORT:-false}
+KIND_UPDATE_SYSTEM=${KIND_UPDATE_SYSTEM:-false}
 
 # Input not currently validated. Modify outside script at your own risk.
 # These are the same values defaulted to in KIND code (kind/default.go).
@@ -147,6 +149,125 @@ fi
 
 API_IPV6=""
 if [ "$KIND_IPV6_SUPPORT" == true ]; then
+  # Collect additional IPv6 data on test environment
+  # TEST-BEGIN
+  ERROR_FOUND=false
+  RESTART_DOCKER=false
+  TMPVAR=`sysctl net.ipv6.conf.all.forwarding | awk '{print $3}'`
+  echo "net.ipv6.conf.all.forwarding is equal to $TMPVAR"
+  if [ "$TMPVAR" != 1 ]; then
+    if [ "$KIND_UPDATE_SYSTEM" == true ]; then
+      sudo sysctl -w net.ipv6.conf.all.forwarding=1
+    else
+      echo "RUN: 'sudo sysctl -w net.ipv6.conf.all.forwarding=1' to use IPv6."
+      ERROR_FOUND=true
+    fi
+  fi
+  TMPVAR=`sysctl net.ipv6.conf.all.disable_ipv6 | awk '{print $3}'`
+  echo "net.ipv6.conf.all.disable_ipv6 is equal to $TMPVAR"
+  if [ "$TMPVAR" != 0 ]; then
+    if [ "$KIND_UPDATE_SYSTEM" == true ]; then
+      sudo sysctl -w net.ipv6.conf.all.disable_ipv6=0
+    else
+      echo "RUN: 'sudo sysctl -w net.ipv6.conf.all.disable_ipv6=0' to use IPv6."
+      ERROR_FOUND=true
+    fi
+  fi
+
+  DOCKER_DAEMON_FILE=/etc/docker/daemon.json
+  if [ -f $DOCKER_DAEMON_FILE ]; then
+    DOCKER_APPEND_STR=""
+    if ! grep -q "\"ipv6\": true" $DOCKER_DAEMON_FILE; then
+      echo "'\"ipv6\": true' NOT found!"
+      DOCKER_APPEND_STR="${DOCKER_APPEND_STR} \"ipv6\": true,"
+    fi
+    if ! grep -q "\"fixed-cidr-v6\":" $DOCKER_DAEMON_FILE ; then
+      echo "'\"fixed-cidr-v6\":' NOT found!"
+      DOCKER_APPEND_STR="${DOCKER_APPEND_STR} \"fixed-cidr-v6\": \"2001:db8:1::/64\","
+    fi
+
+    if [ "$DOCKER_APPEND_STR" != "" ]; then
+      if [ "$KIND_UPDATE_SYSTEM" == true ]; then
+        echo "Updating '$DOCKER_DAEMON_FILE' with DOCKER_APPEND_STR=$DOCKER_APPEND_STR"
+        RESTART_DOCKER=true
+        # The awk command:
+        #   Matches on '{' at the beginning of a line via: /^{/
+        #   On a match: 
+        #     Copy the first field (the '{') to awk variable out via: out=$1;
+        #     Then copy the generated string which is in awk variable DCKSTR to awk variable out via: out=out DCKSTR;
+        #     Then copy remaining fields to awk variable out via: for(i=2;i<=NF;i++) out=out" "$i;
+        #     Then print the awk variable out: print out;
+        #     Then move to the next input string
+        #   On NO match, perform default action of print line via: }1
+        # Write output to a temp file and move that file back to original.
+        sudo awk -v DCKSTR="$DOCKER_APPEND_STR" \
+           '/^{/{out=$1; out=out DCKSTR; for(i=2;i<=NF;i++) out=out" "$i; print out; next}1' $DOCKER_DAEMON_FILE > kind.tmp && \
+           sudo mv kind.tmp $DOCKER_DAEMON_FILE
+        echo "Dumping edited $DOCKER_DAEMON_FILE"
+        sudo cat $DOCKER_DAEMON_FILE
+      else
+        ERROR_FOUND=true
+        echo "Add: '$DOCKER_APPEND_STR' to '$DOCKER_DAEMON_FILE' to use IPv6."
+      fi
+    else
+      echo "'$DOCKER_DAEMON_FILE' NOT updated."
+     fi
+
+  else
+    if [ "$KIND_UPDATE_SYSTEM" == true ]; then
+      RESTART_DOCKER=true
+      cat << EOF | sudo tee -a "${DOCKER_DAEMON_FILE}" >/dev/null
+{
+  "ipv6": true,
+  "fixed-cidr-v6": "2001:db8:1::/64"
+}
+EOF
+    else
+      ERROR_FOUND=true
+      echo "Create '$DOCKER_DAEMON_FILE' with '{ \"ipv6\": true, \"fixed-cidr-v6\": \"2001:db8:1::/64\" }' to use IPv6."
+    fi
+  fi
+  if [ "$ERROR_FOUND" == true ]; then
+    exit 1
+  fi
+  if [ "$RESTART_DOCKER" == true ]; then
+    echo "Restarting Docker"
+    sudo systemctl restart docker
+    LOOPCNT=5
+    while true
+    do
+      echo "$LOOPCNT"
+      sudo systemctl status docker | grep -q "Active: active (running)"
+      if [ $? -eq 0 ] ; then
+         break
+      fi
+      let "LOOPCNT--"
+      if [ $LOOPCNT == 0 ] ; then
+        echo "Docker NOT restarting. Exiting ..."
+        exit 1
+      fi
+      sleep 1
+    done
+  fi
+
+  # REMOVE: Verify docker has ipv6 enabled
+  docker run --rm busybox ip a
+
+
+  echo ""
+  echo "Running: ip a"
+  ip a
+
+  echo ""
+  echo "Running: Check for /proc/net/if_inet6"
+  [ -f /proc/net/if_inet6 ] && echo '  IPv6 ready system!' || echo '  No IPv6 support found! Compile the kernel!!'
+
+  echo ""
+  echo "Running: lsmod"
+  lsmod | grep -qw ipv6 && echo "IPv6 kernel driver loaded and configured." || echo "IPv6 not configured and/or driver loaded on the system."
+
+  # TEST-END
+
   # ip -6 addr -> Run ip command for IPv6
   # grep "inet6" -> Use only the lines with the IPv6 Address
   # sed 's@/.*@@g' -> Strip off the trailing subnet mask, /xx
@@ -156,8 +277,15 @@ if [ "$KIND_IPV6_SUPPORT" == true ]; then
   API_IPV6=$(ip -6 addr  | grep "inet6" | awk -F' ' '{print $2}' | \
              sed 's@/.*@@g' | grep -v "^::1$" | sed '/^fe80:/ d' | head -n 1)
   if [ -z "$API_IPV6" ]; then
-    echo "Error detecting machine IPv6 to use as API server"
-    exit 1
+    # No IPv6 global addresses, Repeat but allow local address
+    API_IPV6=$(ip -6 addr  | grep "inet6" | awk -F' ' '{print $2}' | \
+               sed 's@/.*@@g' | grep -v "^::1$" | head -n 1)
+    if [ -z "$API_IPV6" ]; then
+      echo "Error detecting machine IPv6 to use as API server"
+      exit 1
+    else
+      echo "Using IPv6 LOCAL for API_IPV6"
+    fi
   fi
 fi
 
@@ -197,7 +325,7 @@ ovn_apiServerAddress=${API_IP} \
   j2 ${KIND_CONFIG} -o ${KIND_CONFIG_LCL}
 
 # Create KIND cluster. For additional debug, add '--verbosity <int>': 0 None .. 3 Debug
-kind create cluster --name ${KIND_CLUSTER_NAME} --kubeconfig ${HOME}/admin.conf --image kindest/node:${K8S_VERSION} --config=${KIND_CONFIG_LCL}
+kind create cluster --name ${KIND_CLUSTER_NAME} --verbosity 5 --kubeconfig ${HOME}/admin.conf --image kindest/node:${K8S_VERSION} --config=${KIND_CONFIG_LCL}
 export KUBECONFIG=${HOME}/admin.conf
 cat ${KUBECONFIG}
 mkdir -p /tmp/kind