refactor updater with some improvements

.gitignore

@@ -10,4 +10,4 @@ official-eclipse-temurin
 *hotspot.txt
 library/
 .vscode/
- __pycache__/
+__pycache__/

README.md

@@ -1,6 +1,6 @@
 # Containers
-[![Docker Stars](https://img.shields.io/docker/stars/_/eclipse-temurin?style=flat-square)](https://hub.docker.com/r/_/eclipse-temurin)
 [![DockerPulls](https://img.shields.io/docker/pulls/_/eclipse-temurin?label=Docker%20Pulls)](https://hub.docker.com/_/eclipse-temurin)
+[![Docker Stars](https://img.shields.io/docker/stars/_/eclipse-temurin?label=Docker%20Stars)](https://hub.docker.com/r/_/eclipse-temurin)
 
 This repository contains the Dockerfiles for the official [Adoptium](https://adoptium.net) images of the Eclipse Temurin distribution (OpenJDK). These images are made available in Docker Hub.
 
@@ -28,7 +28,7 @@ A [Updater GitHub Action](.github/workflows/updater.yml) runs every 30 mins whic
 
 #### generate_dockerfiles.py
 
-[`./update_all.sh`](./update_all.sh) is a wrapper script to control what is passed into [`./update_multiarch.sh`](./update_multiarch.sh).
+[`./generate_dockerfiles.py`](./generate_dockerfiles.py) is a python script which uses the jinja templates defined in [docker_templates](./docker_templates/) to generate the docker updates. It uses the Adoptium API to fetch the lastest artefacts for each release.
 
 ### Manual Release
 

config/hotspot.yml

@@ -17,13 +17,14 @@ supported_distributions:
   Versions: [8, 11, 17, 21]
 
 configurations:
+  linux:
   - directory: ubuntu/jammy
     image: ubuntu:22.04
-    architectures: [aarch64, armv7l, ppc64le, s390x, x64]
+    architectures: [aarch64, arm, ppc64le, s390x, x64]
     os: ubuntu
 
   - directory: ubuntu/focal
-    architectures: [aarch64, armv7l, ppc64le, s390x, x64]
+    architectures: [aarch64, arm, ppc64le, s390x, x64]
     image: ubuntu:20.04
     deprecated: 20
     os: ubuntu
@@ -38,12 +39,14 @@ configurations:
     architectures: [aarch64, ppc64le, s390x, x64]
     image: redhat/ubi9-minimal
     os: ubi9-minimal
-
+
+  alpine-linux:
   - directory: alpine
     architectures: [aarch64, x64]
     image: alpine:3.18
     os: alpine-linux
-
+
+  windows:
   - directory: windows/windowsservercore-1809
     architectures: [x64]
     image: mcr.microsoft.com/windows/servercore:1809

docker_templates/partials/version-check.j2

@@ -1,7 +1,10 @@
-RUN echo Verifying install ... \
-    {% if version|int >= 11 -%}
+RUN {% if version|int >= 11 -%}
     {% if os_family != "windows" -%}
+    set -eux; \
+    && echo Verifying install ... \
     && fileEncoding="$(echo 'System.out.println(System.getProperty("file.encoding"))' | jshell -s -)"; [ "$fileEncoding" = 'UTF-8' ]; rm -rf ~/.java \
+    {% else -%}
+    echo Verifying install ... \
     {% endif -%}
     {% if image_type == "jdk" -%}
     && echo javac --version && javac --version \

dockerhub_doc_config_update.sh

@@ -29,6 +29,7 @@ else
 fi
 
 supported_versions="8 11 17 21"
+latest_version="21"
 all_jvms="hotspot"
 all_packages="jdk jre"
 

generate_dockerfiles.py

@@ -24,6 +24,7 @@
     "User-Agent": "Adoptium Dockerfile Updater",
 }
 
+
 def archHelper(arch, os_family):
     if arch == "aarch64":
         return "aarch64|arm64"
@@ -42,118 +43,112 @@ def archHelper(arch, os_family):
         return arch
 
 
-def osFamilyHelper(os):
-    if os == "ubuntu":
-        return "linux"
-    elif os == "centos":
-        return "linux"
-    elif os == "ubi9-minimal":
-        return "linux"
-    elif os == "nanoserver":
-        return "windows"
-    elif os == "servercore":
-        return "windows"
-    else:
-        return os
-
-
 # Load the YAML configuration
 with open("config/hotspot.yml", "r") as file:
     config = yaml.safe_load(file)
 
-# Iterate through configurations
-for configuration in config["configurations"]:
-    directory = configuration["directory"]
-    architectures = configuration["architectures"]
-    os_name = configuration["os"]
-    os_family = osFamilyHelper(os_name)
-    base_image = configuration["image"]
-    deprecated = configuration.get("deprecated", None)
-    versions = configuration.get(
-        "versions", config["supported_distributions"]["Versions"]
-    )
-
-    # Define the path for the template based on OS
-    template_name = f"{os_name}.Dockerfile.j2"
-    template = env.get_template(template_name)
-
-    # Create output directories if they don't exist
-    for version in versions:
-        # if deprecated is set and version is greater than or equal to deprecated, skip
-        if deprecated and version >= deprecated:
-            continue
-        for image_type in ["jdk", "jre"]:
-            output_directory = os.path.join(str(version), image_type, directory)
-            os.makedirs(output_directory, exist_ok=True)
-
-            # Fetch latest release for version from Adoptium API
-            url = f"https://api.adoptium.net/v3/assets/feature_releases/{version}/ga?page=0&image_type={image_type}&page_size=1&vendor=eclipse"
-            response = requests.get(url, headers=headers)
-            response.raise_for_status()
-            data = response.json()
-
-            release = response.json()[0]
-
-            # Extract the version number from the release name
-            openjdk_version = release["release_name"]
-
-            # Generate the data for each architecture
-            arch_data = {}
-
-            for binary in release["binaries"]:
-                if (
-                    binary["architecture"] in architectures
-                    and binary["os"] == os_family
-                ):
-                    if os_family == "windows":
-                        # Windows only has x64 binaries
-                        copy_from = openjdk_version.replace(
-                            "jdk", ""
-                        )  # jdk8u292-b10 -> 8u292-b10
-                        if version != 8:
-                            copy_from = copy_from.replace("-", "").replace(
-                                "+", "_"
-                            )  # 11.0.11+9 -> 11.0.11_9
-                        copy_from = f"{copy_from}-{image_type}-windowsservercore-{base_image.split(':')[1]}"
-                        arch_data = {
-                            "download_url": binary["installer"]["link"],
-                            "checksum": binary["installer"]["checksum"],
-                            "copy_from": copy_from,
-                        }
+# Iterate through OS families and then configurations
+for os_family, configurations in config["configurations"].items():
+    for configuration in configurations:
+        directory = configuration["directory"]
+        architectures = configuration["architectures"]
+        os_name = configuration["os"]
+        base_image = configuration["image"]
+        deprecated = configuration.get("deprecated", None)
+        versions = configuration.get(
+            "versions", config["supported_distributions"]["Versions"]
+        )
+
+        # Define the path for the template based on OS
+        template_name = f"{os_name}.Dockerfile.j2"
+        template = env.get_template(template_name)
+
+        # Create output directories if they don't exist
+        for version in versions:
+            # if deprecated is set and version is greater than or equal to deprecated, skip
+            if deprecated and version >= deprecated:
+                continue
+            print("Generating Dockerfiles for", base_image, "-", version)
+            for image_type in ["jdk", "jre"]:
+                output_directory = os.path.join(str(version), image_type, directory)
+                os.makedirs(output_directory, exist_ok=True)
+
+                # Fetch latest release for version from Adoptium API
+                url = f"https://api.adoptium.net/v3/assets/feature_releases/{version}/ga?page=0&image_type={image_type}&page_size=1&vendor=eclipse"
+                response = requests.get(url, headers=headers)
+                response.raise_for_status()
+                data = response.json()
+
+                release = response.json()[0]
+
+                # Extract the version number from the release name
+                openjdk_version = release["release_name"]
+
+                # Generate the data for each architecture
+                arch_data = {}
+
+                for binary in release["binaries"]:
+                    if (
+                        binary["architecture"] in architectures
+                        and binary["os"] == os_family
+                    ):
+                        if os_family == "windows":
+                            # Windows only has x64 binaries
+                            copy_from = openjdk_version.replace(
+                                "jdk", ""
+                            )  # jdk8u292-b10 -> 8u292-b10
+                            if version != 8:
+                                copy_from = copy_from.replace("-", "").replace(
+                                    "+", "_"
+                                )  # 11.0.11+9 -> 11.0.11_9
+                            copy_from = f"{copy_from}-{image_type}-windowsservercore-{base_image.split(':')[1]}"
+                            arch_data = {
+                                "download_url": binary["installer"]["link"],
+                                "checksum": binary["installer"]["checksum"],
+                                "copy_from": copy_from,
+                            }
+                        else:
+                            arch_data[archHelper(binary["architecture"], os_family)] = {
+                                "download_url": binary["package"]["link"],
+                                "checksum": binary["package"]["checksum"],
+                            }
+
                     else:
-                        arch_data[archHelper(binary["architecture"], os_family)] = {
-                            "download_url": binary["package"]["link"],
-                            "checksum": binary["package"]["checksum"],
-                        }
-
-                    # Generate Dockerfile for each architecture
-                    rendered_dockerfile = template.render(
-                        base_image=base_image,
-                        image_type=image_type,
-                        java_version=openjdk_version,
-                        version=version,
-                        arch_data=arch_data,
-                        os_family=os_family,
-                        os=os_name,
-                    )
+                        continue
 
-                    # Save the rendered Dockerfile
-                    with open(
-                        os.path.join(output_directory, "Dockerfile"), "w"
-                    ) as out_file:
-                        out_file.write(rendered_dockerfile)
+                # If arch_data is empty, skip updating the dockerfile
+                if arch_data.__len__() == 0:
+                    continue
 
-                    # Copy entrypoint.sh to output directory
-                    entrypoint_path = os.path.join(
-                        "docker_templates", "scripts", f"entrypoint.{os_name}.sh"
+                # Sort arch_data by key
+                arch_data = dict(sorted(arch_data.items()))
+
+                # Generate Dockerfile for each architecture
+                rendered_dockerfile = template.render(
+                    base_image=base_image,
+                    image_type=image_type,
+                    java_version=openjdk_version,
+                    version=version,
+                    arch_data=arch_data,
+                    os_family=os_family,
+                    os=os_name,
+                )
+
+                print("Writing Dockerfile to", output_directory)
+                # Save the rendered Dockerfile
+                with open(
+                    os.path.join(output_directory, "Dockerfile"), "w"
+                ) as out_file:
+                    out_file.write(rendered_dockerfile)
+
+                # Copy entrypoint.sh to output directory
+                entrypoint_path = os.path.join(
+                    "docker_templates", "scripts", f"entrypoint.{os_name}.sh"
+                )
+
+                if os.path.exists(entrypoint_path):
+                    os.system(
+                        f"cp {entrypoint_path} {os.path.join(output_directory, 'entrypoint.sh')}"
                     )
 
-                    if os.path.exists(entrypoint_path):
-                        os.system(
-                            f"cp {entrypoint_path} {os.path.join(output_directory, 'entrypoint.sh')}"
-                        )
-
-                else:
-                    continue
-
 print("Dockerfiles generated successfully!")

test_generate_dockerfiles.py

@@ -16,59 +16,6 @@
 
 from jinja2 import Environment, FileSystemLoader
 
-import generate_dockerfiles
-
-
-class TestHelperFunctions(unittest.TestCase):
-    def test_archHelper(self):
-        test_data = [
-            ("aarch64", "some-os", "aarch64|arm64"),
-            ("ppc64le", "some-os", "ppc64el|powerpc:common64"),
-            ("s390x", "some-os", "s390x|s390:64-bit"),
-            ("arm", "some-os", "armhf|arm"),
-            ("x64", "alpine-linux", "amd64|x86_64"),
-            ("x64", "ubuntu", "amd64|i386:x86-64"),
-            ("random-arch", "some-os", "random-arch"),
-        ]
-
-        for arch, os_family, expected in test_data:
-            self.assertEqual(generate_dockerfiles.archHelper(arch, os_family), expected)
-
-    def test_osFamilyHelper(self):
-        test_data = [
-            ("ubuntu", "linux"),
-            ("centos", "linux"),
-            ("ubi9-minimal", "linux"),
-            ("nanoserver", "windows"),
-            ("servercore", "windows"),
-            ("random-os", "random-os"),
-        ]
-
-        for os_name, expected in test_data:
-            self.assertEqual(generate_dockerfiles.osFamilyHelper(os_name), expected)
-
-    @patch("requests.get")
-    def test_fetch_latest_release(self, mock_get):
-        # Mocking the request.get call
-        mock_response = Mock()
-        mock_response.raise_for_status.return_value = None
-        mock_response.json.return_value = [{"key": "value"}]
-        mock_get.return_value = mock_response
-
-        url = "https://api.adoptium.net/v3/assets/feature_releases/some_version/ga?page=0&image_type=some_type&page_size=1&vendor=eclipse"
-        response = generate_dockerfiles.requests.get(
-            url, headers=generate_dockerfiles.headers
-        )
-        data = response.json()
-        self.assertIn("key", data[0])
-        self.assertEqual(data[0]["key"], "value")
-
-    @patch("builtins.open", new_callable=mock_open, read_data="configurations: []")
-    def test_load_config(self, mock_file):
-        with open("config/hotspot.yml", "r") as file:
-            config = generate_dockerfiles.yaml.safe_load(file)
-            self.assertIn("configurations", config)
-
 
 class TestJinjaRendering(unittest.TestCase):
     def setUp(self):