Merge pull request #717 from ccamacho/skopeo

fix: mirror correctly images from docker.io to quay

.github/workflows/linters.yml

@@ -34,12 +34,11 @@ jobs:
           sudo apt-get install jq libyaml-dev git build-essential findutils -y
           sudo apt-get remove ansible -y
           sudo python3 -m pip uninstall ansible ansible-base ansible-core -y
-          sudo python3 -m pip install ansible==5.1.0
+          sudo python3 -m pip install -r test-requirements.txt
           sudo python3 -m pip install --upgrade --ignore-installed PyYAML
           sudo python3 -m pip install --upgrade pip
           sudo python3 -m pip install --upgrade virtualenv
           sudo python3 -m pip install --upgrade setuptools
-          sudo python3 -m pip install --upgrade -r ./test-requirements.txt
       - name: Run Linters
         run: |
           tox -e linters

.github/workflows/molecule.yml

@@ -30,7 +30,6 @@ jobs:
           sudo apt install jq -y
           sudo apt-get remove ansible -y
           sudo python3 -m pip uninstall ansible ansible-base ansible-core -y
-          sudo python3 -m pip install ansible==5.1.0
           sudo locale-gen en_US.UTF-8
           sudo dpkg-reconfigure locales
           sudo apt install build-essential findutils -y

.github/workflows/quay_mirror.yml

@@ -0,0 +1,33 @@
+---
+# This job will mirror some container
+# images from docker.io to quay.io
+name: quay_mirror
+on:
+  # Run the mirror every week
+  schedule:
+    - cron: '0 0 * * 0'
+jobs:
+  build:
+    if: github.repository_owner == 'Kubeinit'
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: [3.9]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          sudo apt update -y
+          sudo apt install jq -y
+          sudo apt-get install skopeo
+      - name: Mirror to quay
+        run: |
+          chmod +x ./ci/container_sync.sh
+          sudo QUAY_BOT_USER=${{ secrets.QUAY_BOT_USER }} \
+               QUAY_BOT_KEY=${{ secrets.QUAY_BOT_KEY }} \
+               ./ci/container_sync.sh

.github/workflows/release.yml

@@ -28,17 +28,16 @@ jobs:
           sudo apt update -y
           sudo apt install jq -y
           sudo apt-get remove ansible -y
-          sudo python3 -m pip uninstall ansible ansible-base ansible-core -y
-          sudo python3 -m pip install ansible==5.1.0
           sudo locale-gen en_US.UTF-8
           sudo dpkg-reconfigure locales
           sudo apt install build-essential findutils -y
           sudo apt-get --purge autoremove python3-pip
           sudo apt install python3-pip
+          sudo python3 -m pip uninstall ansible ansible-base ansible-core -y
+          sudo python3 -m pip install -r test-requirements.txt
           sudo python3 -m pip install --upgrade pip
           sudo python3 -m pip install --upgrade virtualenv
           sudo python3 -m pip install --upgrade setuptools
-          sudo python3 -m pip install -r ./test-requirements.txt
           sudo python3 -m pip install tox shyaml
           sudo python3 -m pip install twine
       - name: Mirror to GitLab

.github/workflows/units.yml

@@ -29,17 +29,17 @@ jobs:
           sudo apt update -y
           sudo apt install jq -y
           sudo apt-get remove ansible -y
-          sudo python3 -m pip uninstall ansible ansible-base ansible-core -y
-          sudo python3 -m pip install ansible==5.1.0
           sudo locale-gen en_US.UTF-8
           sudo dpkg-reconfigure locales
           sudo apt install build-essential findutils -y
           sudo apt-get --purge autoremove python3-pip
           sudo apt install python3-pip
+          sudo python3 -m pip uninstall ansible ansible-base ansible-core -y
+          sudo python3 -m pip install -r test-requirements.txt
+          sudo python3 -m pip install --upgrade --ignore-installed PyYAML
           sudo python3 -m pip install --upgrade pip
           sudo python3 -m pip install --upgrade virtualenv
           sudo python3 -m pip install --upgrade setuptools
-          sudo python3 -m pip install -r ./test-requirements.txt
       - name: Run unit tests
         run: |
           tox -e units

_skeleton_role_/molecule/default/molecule.yml

@@ -5,7 +5,7 @@ driver:
   name: docker
 platforms:
   - name: instance
-    image: docker.io/pycontribs/centos:8
+    image: quay.io/centos/centos:stream8
     pre_build_image: true
 provisioner:
   name: ansible

ci/ansible/roles/kubeinit_ci/tasks/main.yml

@@ -215,7 +215,7 @@
       changed_when: "_result.rc == 0"
 
     - name: Create a new working container image
-      ansible.builtin.command: buildah from --name kubeinit-ara docker.io/recordsansible/ara-api:latest
+      ansible.builtin.command: buildah from --name kubeinit-ara quay.io/recordsansible/ara-api:latest
       register: _result
       changed_when: "_result.rc == 0"
 
@@ -262,7 +262,7 @@
       changed_when: "_result.rc == 0"
 
     - name: Create a new working container image
-      ansible.builtin.command: buildah from --name kubeinit-output-data docker.io/httpd:2.4
+      ansible.builtin.command: buildah from --name kubeinit-output-data quay.io/kubeinit/httpd:2.4
       register: _result
       changed_when: "_result.rc == 0"
 

ci/container_sync.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+set -e
+
+#############################################################################
+#                                                                           #
+# Copyright kubeinit contributors.                                          #
+#                                                                           #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may   #
+# not use this file except in compliance with the License. You may obtain   #
+# a copy of the License at:                                                 #
+#                                                                           #
+# http://www.apache.org/licenses/LICENSE-2.0                                #
+#                                                                           #
+# Unless required by applicable law or agreed to in writing, software       #
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT #
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the  #
+# License for the specific language governing permissions and limitations   #
+# under the License.                                                        #
+#                                                                           #
+#############################################################################
+
+# NOTE:
+# If there is the need of adding more container images
+# to quay.io/kubeinit, then:
+# - Make sure the repository is already created
+#   and the visibility is public, for example that
+#   https://quay.io/repository/kubeinit/haproxy exists.
+# - Make sure the bot (kubeinit+sync) have write access
+#   to that repository.
+
+if [ -z "$QUAY_BOT_USER" ]; then
+    echo "QUAY_BOT_USER is not set";
+    exit 1;
+fi
+
+if [ -z "$QUAY_BOT_KEY" ]; then
+    echo "QUAY_BOT_KEY is not set";
+    exit 1;
+fi
+
+user=$QUAY_BOT_USER
+token=$QUAY_BOT_KEY
+
+skopeo login quay.io -u$user -p$token
+
+# Add here all the images to be syncronized to the main
+# quay organization in Kubeinit
+declare -a container_images=(
+    "library haproxy 2.3"
+    "library registry 2"
+    "library httpd 2.4"
+    "library nginx latest"
+    "library debian 11"
+    "library ubuntu focal"
+    "library ubuntu jammy"
+    "internetsystemsconsortium bind9 9.18"
+    "sonatype nexus3 3.30.0"
+)
+
+retry() {
+    local -r -i max_attempts="$1"; shift
+    local -r cmd="$@"
+    local -i attempt_num=1
+    until $cmd; do
+        if ((attempt_num==max_attempts))
+        then
+            echo "Attempt $attempt_num failed and there are no more attempts left!"
+            return 1
+        else
+            echo "Attempt $attempt_num failed! Trying again in $attempt_num seconds..."
+            sleep $((attempt_num++))
+        fi
+    done
+}
+
+for image in "${container_images[@]}"; do
+    read -a strarr <<< "$image"
+    namespace=${strarr[0]}
+    container=${strarr[1]}
+    tag=${strarr[2]}
+
+    copy="skopeo copy docker://docker.io/$namespace/$container:$tag docker://quay.io/kubeinit/$container:$tag"
+    retry 5 $copy
+done

ci/launch_e2e.sh

@@ -127,7 +127,7 @@ podman run --name api-server \
             --detach --tty \
             --volume ~/.ara/server:/opt/ara:z \
             --volume ~/.ara/output_data:/opt/output_data:z \
-            docker.io/recordsansible/ara-api:latest
+            quay.io/recordsansible/ara-api:latest
 
 #
 # Any change in the way the logs from Ansible are

kubeinit/README.md

@@ -20,6 +20,7 @@
     <a href="https://github.com/Kubeinit/kubeinit/actions?workflow=container_image"><img height="20px" src="https://github.com/Kubeinit/kubeinit/workflows/container_image/badge.svg?event=schedule"/> </a>
     <a href="https://github.com/Kubeinit/kubeinit/actions?workflow=distro_test"><img height="20px" src="https://github.com/Kubeinit/kubeinit/workflows/distro_test/badge.svg?event=schedule"/> </a>
     <a href="https://github.com/Kubeinit/kubeinit/actions?workflow=check_package_build"><img height="20px" src="https://github.com/Kubeinit/kubeinit/workflows/check_package_build/badge.svg?event=schedule"/> </a>
+    <a href="https://github.com/Kubeinit/kubeinit/actions?workflow=quay_mirror"><img height="20px" src="https://github.com/Kubeinit/kubeinit/workflows/quay_mirror/badge.svg?event=schedule"/> </a>
     <a href="https://kubernetes.slack.com/archives/C01FKK19T0B"><img height="20px" src="https://img.shields.io/badge/chat-on%20slack-blue.svg?logo=slack&longCache=true&style=flat"/> </a>
 </p>
 

kubeinit/roles/kubeinit_apache/tasks/main.yml

@@ -20,7 +20,7 @@
     name: "buildah"
 
 - name: Create a new working container image
-  ansible.builtin.command: buildah from --name {{ kubeinit_cluster_name }}-apache quay.io/jlarriba/httpd:2.4
+  ansible.builtin.command: buildah from --name {{ kubeinit_cluster_name }}-apache quay.io/kubeinit/httpd:2.4
   register: _result
   changed_when: "_result.rc == 0"
 

kubeinit/roles/kubeinit_apps/templates/sampleapp.yml.j2

@@ -20,7 +20,7 @@ spec:
         # We try to avoid pulling from docker.io because
         # of the limit rate, this cause multiple installs
         # to fail that should be successful deployments
-        image: quay.io/packit/nginx-unprivileged
+        image: quay.io/kubeinit/nginx:latest
         ports:
         - containerPort: 8080
       nodeSelector:

kubeinit/roles/kubeinit_bind/tasks/main.yml

@@ -44,7 +44,7 @@
   changed_when: "_result.rc == 0"
 
 - name: Create a new working container image
-  ansible.builtin.command: buildah from --name {{ kubeinit_cluster_name }}-bind --volume {{ kubeinit_bind_directory }}:/bind-config:Z docker.io/internetsystemsconsortium/bind9:9.18
+  ansible.builtin.command: buildah from --name {{ kubeinit_cluster_name }}-bind --volume {{ kubeinit_bind_directory }}:/bind-config:Z quay.io/kubeinit/bind9:9.18
   register: _result
   changed_when: "_result.rc == 0"
 

kubeinit/roles/kubeinit_bind/templates/create-external-ingress.sh.j2

@@ -24,7 +24,7 @@ fi
 rm -rf /var/kubeinit/bind
 mkdir -p /var/kubeinit/bind
 if [ "$(buildah ls --filter 'name={{ kubeinit_cluster_name }}-external' --format {% raw %}'{{ .ContainerName }}'{% endraw %})" != "" ]; then buildah rm {{ kubeinit_cluster_name }}-external; fi
-buildah from --name {{ kubeinit_cluster_name }}-external -v /var/kubeinit/bind:/var/kubeinit/bind:Z docker.io/internetsystemsconsortium/bind9:9.18
+buildah from --name {{ kubeinit_cluster_name }}-external -v /var/kubeinit/bind:/var/kubeinit/bind:Z quay.io/kubeinit/bind9:9.18
 buildah run {{ kubeinit_cluster_name }}-external -- apt-get update -y
 buildah run {{ kubeinit_cluster_name }}-external -- apt-get install -y openssh-client
 buildah run {{ kubeinit_cluster_name }}-external -- cp -pr /etc/bind/. /var/kubeinit/bind/.

kubeinit/roles/kubeinit_haproxy/tasks/main.yml

@@ -44,7 +44,7 @@
   changed_when: "_result.rc == 0"
 
 - name: Create a new working container image
-  ansible.builtin.command: buildah from --name {{ kubeinit_cluster_name }}-haproxy quay.io/jlarriba/haproxy:2.3
+  ansible.builtin.command: buildah from --name {{ kubeinit_cluster_name }}-haproxy quay.io/kubeinit/haproxy:2.3
   register: _result
   changed_when: "_result.rc == 0"
 

kubeinit/roles/kubeinit_nexus/tasks/main.yml

@@ -60,7 +60,7 @@
   changed_when: "_result.rc == 0"
 
 - name: Create a new working container image
-  ansible.builtin.command: buildah from --name {{ kubeinit_cluster_name }}-nexus --volume "{{ kubeinit_nexus_directory_data }}:/nexus-data" docker.io/sonatype/nexus3:3.30.0
+  ansible.builtin.command: buildah from --name {{ kubeinit_cluster_name }}-nexus --volume "{{ kubeinit_nexus_directory_data }}:/nexus-data" quay.io/kubeinit/nexus3:3.30.0
   register: _result
   changed_when: "_result.rc == 0"
 

kubeinit/roles/kubeinit_nodered/molecule/default/molecule.yml

@@ -5,7 +5,7 @@ driver:
   name: docker
 platforms:
   - name: instance
-    image: docker.io/pycontribs/centos:8
+    image: quay.io/centos/centos:stream8
     pre_build_image: true
 provisioner:
   name: ansible

kubeinit/roles/kubeinit_ooonextgen/molecule/default/molecule.yml

@@ -5,7 +5,7 @@ driver:
   name: docker
 platforms:
   - name: instance
-    image: docker.io/pycontribs/centos:8
+    image: quay.io/centos/centos:stream8
     pre_build_image: true
 provisioner:
   name: ansible

kubeinit/roles/kubeinit_openstack/molecule/default/molecule.yml

@@ -5,7 +5,7 @@ driver:
   name: docker
 platforms:
   - name: instance
-    image: docker.io/pycontribs/centos:8
+    image: quay.io/centos/centos:stream8
     pre_build_image: true
 provisioner:
   name: ansible

kubeinit/roles/kubeinit_openwhisk/molecule/default/molecule.yml

@@ -5,7 +5,7 @@ driver:
   name: docker
 platforms:
   - name: instance
-    image: docker.io/pycontribs/centos:8
+    image: quay.io/centos/centos:stream8
     pre_build_image: true
 provisioner:
   name: ansible

kubeinit/roles/kubeinit_registry/tasks/main.yml

@@ -20,7 +20,7 @@
     name: "buildah"
 
 - name: Create a new working container image
-  ansible.builtin.command: buildah from --name {{ kubeinit_cluster_name }}-registry quay.io/jlarriba/registry:2
+  ansible.builtin.command: buildah from --name {{ kubeinit_cluster_name }}-registry quay.io/kubeinit/registry:2
   register: _result
   changed_when: "_result.rc == 0"
 

kubeinit/roles/kubeinit_services/tasks/create_provision_container.yml

@@ -53,7 +53,7 @@
     - name: Setup Debian container image
       block:
         - name: Create a new working container image
-          ansible.builtin.command: buildah from --name {{ kubeinit_cluster_name }}-provision docker.io/debian:{{ kubeinit_libvirt_debian_release }}
+          ansible.builtin.command: buildah from --name {{ kubeinit_cluster_name }}-provision quay.io/kubeinit/debian:{{ kubeinit_libvirt_debian_release }}
           register: _result
           changed_when: "_result.rc == 0"
         - name: Update the container
@@ -73,7 +73,7 @@
     - name: Setup Ubuntu container image
       block:
         - name: Create a new working container image
-          ansible.builtin.command: buildah from --name {{ kubeinit_cluster_name }}-provision docker.io/ubuntu:{{ kubeinit_libvirt_ubuntu_release }}
+          ansible.builtin.command: buildah from --name {{ kubeinit_cluster_name }}-provision quay.io/kubeinit/ubuntu:{{ kubeinit_libvirt_ubuntu_release }}
           register: _result
           changed_when: "_result.rc == 0"
         - name: Update the container