Auto download images by base board, make python code a python package #…

…214

pyproject.toml

@@ -0,0 +1,25 @@
+[tool.poetry]
+name = "custompios"
+version = "2.0.0"
+description = "A Raspberry Pi and other ARM devices distribution builder. CustomPiOS opens an already existing image, modifies it and repackages the image ready to ship."
+authors = ["Guy Sheffer <guysoft@gmail.com>"]
+license = "GPLv3"
+readme = "README.rst"
+packages = [
+#     { include = "src/*" },
+    { include = "custompios_core", from = "src" }
+    ]
+
+[tool.poetry.dependencies]
+python = "^3.11"
+GitPython = "^3.1.41"
+
+[tool.poetry.group.dev.dependencies]
+types-PyYAML = "^6.0.12.12"
+
+[tool.poetry.scripts]
+custompios_build = 'custompios_core.multi_build:main'
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

src/base_image_downloader_wrapper.sh

@@ -17,5 +17,5 @@ fi
 # source "${DIST_PATH}/config"
 source "${CUSTOM_PI_OS_PATH}/config" "${WORKSPACE_SUFFIX}"
 
-python3 ${CUSTOM_PI_OS_PATH}/base_image_downloader.py "${WORKSPACE_SUFFIX}"
+python3 ${CUSTOM_PI_OS_PATH}/custompios_core/base_image_downloader.py "${WORKSPACE_SUFFIX}"
 

src/build

@@ -10,12 +10,16 @@ define(){ IFS='\n' read -r -d '' ${1} || true; }
 
 define SCRIPT <<'EOF'
 BUILD_SCRIPT_PATH=$(dirname $(realpath -s $BASH_SOURCE))
+export EXTRA_BOARD_CONFIG=$(mktemp)
+${BUILD_SCRIPT_PATH}/custompios_core/generate_board_config.py "${EXTRA_BOARD_CONFIG}"
+echo "Temp source file: ${EXTRA_BOARD_CONFIG}"
+
 source ${BUILD_SCRIPT_PATH}/common.sh
 install_cleanup_trap
 
 CUSTOM_OS_PATH=$(dirname $(realpath -s $0))
 
-source ${CUSTOM_PI_OS_PATH}/config ${@}
+source ${CUSTOM_PI_OS_PATH}/config "${1}" "${EXTRA_BOARD_CONFIG}" ${@}
 ${CUSTOM_PI_OS_PATH}/config_sanity
 
 [ "$CONFIG_ONLY" == "yes" ] || source ${CUSTOM_OS_PATH}/custompios ${@}

src/common.sh

@@ -150,10 +150,13 @@ function unpack() {
 }
 
 function detach_all_loopback(){
+  image_name=$1
   # Cleans up mounted loopback devices from the image name
   # NOTE: it might need a better way to grep for the image name, its might clash with other builds
   for img in $(losetup  | grep $1 | awk '{ print $1 }' );  do
-    if [ -f "${img}" ] || [ -b "${img}" ]; then
+    # test if the image name is a substring
+    if [ "${img}" != "$(printf '%s' "${img}" | sed 's/'"${image_name}"'//g')" ] && ([ -f "${img}" ] || [ -b "${img}" ]); then
+      echo "freeing up $img"
     	losetup -d $img
     fi
   done

src/config

@@ -7,6 +7,8 @@ export BUILD_VARIANT=""
 BUILD_VARIANT="$1"
 : ${BUILD_VARIANT:=default}
 
+EXTRA_BAORD_CONFIG=$2
+
 export BUILD_FLAVOR=""
 # Disable flavor system
 #BUILD_FLAVOR="$1"
@@ -86,6 +88,13 @@ MODULES_LIST="${TMP//)/,}"
 # [ -n "$BASE_CHROOT_SCRIPT_PATH" ] || BASE_CHROOT_SCRIPT_PATH=$BASE_SCRIPT_PATH/chroot_script
 [ -n "$BASE_MOUNT_PATH" ] || BASE_MOUNT_PATH=$BASE_WORKSPACE/mount
 
+# Import remote and submodules config
+if [ -f "${EXTRA_BAORD_CONFIG}" ]; then
+  source "${EXTRA_BAORD_CONFIG}"
+else
+  echo "Note: Not sourceing board config"
+fi
+
 export REMOTE_AND_META_CONFIG="$BASE_WORKSPACE"/remote_and_meta_config
 # Remote modules and meta modulese go in first if they want to change standard behaviour
 if [ -f "${REMOTE_AND_META_CONFIG}" ]; then

src/custompios

@@ -108,6 +108,11 @@ fi
 mkdir -p $BASE_WORKSPACE
 mkdir -p $BASE_MOUNT_PATH
 
+# This is already genrated at "build" sourced in "config", but copying here mostly for debug
+if [ -f "${EXTRA_BAORD_CONFIG}" ]; then
+  mv -v "${EXTRA_BAORD_CONFIG}" "${BASE_WORKSPACE}"/extra_board_config
+fi
+
 # Clean exported artifacts from other builds
 rm -rf "${BASE_WORKSPACE}"/*.tar.gz
 
@@ -121,6 +126,7 @@ pushd $BASE_WORKSPACE
   fi
   if [ ! -f "$BASE_ZIP_IMG" ] || [ "$BASE_ZIP_IMG" == "" ]; then
     echo "Error: could not find image: $BASE_ZIP_IMG"
+    echo "On CustomPiOS v2 you can provide -d to download the latest image of your board automatically"
     exit 1
   fi
 
@@ -181,7 +187,7 @@ pushd $BASE_WORKSPACE
     CHROOT_SCRIPT=${BASE_WORKSPACE}/chroot_script
     MODULES_AFTER_PATH=${BASE_WORKSPACE}/modules_after
     MODULES_BEFORE="${MODULES}"
-    ${CUSTOM_PI_OS_PATH}/execution_order.py "${MODULES}" "${CHROOT_SCRIPT}" "${MODULES_AFTER_PATH}" "${REMOTE_AND_META_CONFIG}"
+    ${CUSTOM_PI_OS_PATH}/custompios_core/execution_order.py "${MODULES}" "${CHROOT_SCRIPT}" "${MODULES_AFTER_PATH}" "${REMOTE_AND_META_CONFIG}"
     if [ -f "${REMOTE_AND_META_CONFIG}" ]; then
       echo "Sourcing remote and submodules config"
       source "${REMOTE_AND_META_CONFIG}" ${@}

src/base_image_downloader.py → src/custompios_core/base_image_downloader.py

@@ -7,12 +7,13 @@
 import hashlib
 import shutil
 import re
+from typing import Dict, Any, Optional, cast, Tuple
+from common import get_image_config, read_images
 PRECENT_PROGRESS_SIZE = 5
 
 class ChecksumFailException(Exception):
     pass
 
-IMAGES_CONFIG = os.path.join(os.path.dirname(__file__), "images.yml")
 RETRY = 3
 
 def ensure_dir(d, chmod=0o777):
@@ -26,13 +27,31 @@ def ensure_dir(d, chmod=0o777):
         return False
     return True
 
-def read_images():
-    if not os.path.isfile(IMAGES_CONFIG):
-        raise Exception(f"Error: Remotes config file not found: {IMAGES_CONFIG}")
-    with open(IMAGES_CONFIG,'r') as f:
-        output = yaml.safe_load(f)
-        return output
 
+def download_webpage(url: str) -> Optional[str]:
+    try:
+        with urllib.request.urlopen(url) as response:
+            # Decode the response to a string
+            webpage = response.read().decode('utf-8')
+            return webpage
+    except Exception as e:
+        print(str(e))
+        return None
+
+def get_location_header(url: str) -> str:
+    try:
+        with urllib.request.urlopen(url) as response:
+            response_url = response.url
+
+            if response_url is None:
+                raise Exception("Location header is None, can't determine latest rpi image")
+            return response_url
+    except Exception as e:
+        print(str(e))
+        print("Error: Failed to determine latest rpi image")
+        raise e
+
+
 class DownloadProgress:
     last_precent: float = 0
     def show_progress(self, block_num, block_size, total_size):
@@ -41,8 +60,10 @@ def show_progress(self, block_num, block_size, total_size):
             print(f"{new_precent}%", end="\r")
             self.last_precent = new_precent
 
-def get_file_name(headers):
-    return re.findall("filename=(\S+)", headers["Content-Disposition"])[0]
+def get_file_name(headers, url):
+    if "Content-Disposition" in headers.keys():
+        return re.findall("filename=(\S+)", headers["Content-Disposition"])[0]
+    return url.split('/')[-1]
 
 def get_sha256(filename):
     sha256_hash = hashlib.sha256()
@@ -53,10 +74,12 @@ def get_sha256(filename):
         return file_checksum
     return
 
-def download_image_http(board: str, dest_folder: str, redownload: bool = False):
+def download_image_http(board: Dict[str, Any], dest_folder: str, redownload: bool = False):
     url = board["url"]
     checksum = board["checksum"]
+    download_http(url, checksum)
 
+def download_http(url: str, checksum_url: str, dest_folder: str, redownload: bool = False):
     with tempfile.TemporaryDirectory() as tmpdirname:
         print('created temporary directory', tmpdirname)
         temp_file_name = os.path.join(tmpdirname, "image.xz")
@@ -66,8 +89,8 @@ def download_image_http(board: str, dest_folder: str, redownload: bool = False):
             try:
                 # Get sha and confirm its the right image
                 download_progress = DownloadProgress()
-                _, headers_checksum = urllib.request.urlretrieve(checksum, temp_file_checksum, download_progress.show_progress)
-                file_name_checksum = get_file_name(headers_checksum)
+                _, headers_checksum = urllib.request.urlretrieve(checksum_url, temp_file_checksum, download_progress.show_progress)
+                file_name_checksum = get_file_name(headers_checksum, checksum_url)
 
                 checksum_data = None
                 with open(temp_file_checksum, 'r') as f:
@@ -82,13 +105,13 @@ def download_image_http(board: str, dest_folder: str, redownload: bool = False):
                 if os.path.isfile(dest_file_name):
                     file_checksum = get_sha256(dest_file_name)
                     if file_checksum == online_checksum:
-                        # We got file and checksum is right
+                        print("We got base image file and checksum is right")
                         return
                 # Get the file
                 download_progress = DownloadProgress()
                 _, headers = urllib.request.urlretrieve(url, temp_file_name, download_progress.show_progress)
 
-                file_name = get_file_name(headers)                        
+                file_name = get_file_name(headers, url)
                 file_checksum = get_sha256(temp_file_name)
                 if file_checksum != online_checksum:
                     print(f'Failed. Attempt # {r + 1}, checksum missmatch: {file_checksum} expected: {online_checksum}')
@@ -102,11 +125,31 @@ def download_image_http(board: str, dest_folder: str, redownload: bool = False):
                 else:
                     print('Error encoutered at {RETRY} attempt')
                     print(e)
+                    exit(1)
             else:
                 print(f"Success: {temp_file_name}")
                 break
     return
 
+
+def download_image_rpi(board: Dict[str, Any], dest_folder: str):
+    port = board.get("port", "lite_armhf")
+    os_name = f"raspios"
+    distribution = board.get("distribution", "bookworm")
+    version_file = board.get("version_file", "latest")
+    version_folder = board.get("version_folder", "latest")
+
+    latest_url = f"https://downloads.raspberrypi.org/{os_name}_{port}_latest"
+
+    download_url = f"https://downloads.raspberrypi.org/{os_name}_{port}/images/{os_name}_{port}-{version_folder}/{version_file}-{os_name}-{distribution}-{port}.img.xz"
+    if version_file == "latest" or version_folder == "latest":
+        download_url = get_location_header(latest_url)
+
+    checksum_url = f"{download_url}.sha256"
+    download_http(download_url, checksum_url, dest_folder)
+    return
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(add_help=True, description='Download images based on BASE_BOARD and BASE_O')
     parser.add_argument('WORKSPACE_SUFFIX', nargs='?', default="default", help="The workspace folder suffix used folder")
@@ -118,14 +161,28 @@ def download_image_http(board: str, dest_folder: str, redownload: bool = False):
     base_board = os.environ.get("BASE_BOARD", None)
     base_image_path = os.environ.get("BASE_IMAGE_PATH", None)
 
-    if base_board is not None and base_board in images["images"]:
-        if images["images"][base_board]["type"] == "http":
-            download_image_http(images["images"][base_board], base_image_path)
-        elif images["images"][base_board]["type"] == "torrent":
+    if base_image_path is None:
+        print(f'Error: did not find image config file')
+        exit(1)
+    cast(str, base_image_path)
+
+    image_config = get_image_config()
+    if image_config is not None:
+        if image_config["type"] == "http":
+            print(f"Downloading image for {base_board}")
+            download_image_http(image_config, base_image_path)
+        elif image_config["type"] == "rpi":
+            print(f"Downloading Raspberry Pi image for {base_board}")
+            download_image_rpi(image_config, base_image_path)
+        elif image_config["type"] == "torrent":
             print("Error: Torrent not implemented")
             exit(1)
         else:
-            print("Error: Unsupported image download type")
+            print(f'Error: Unsupported image download type: {image_config["type"]}')
             exit(1)
+    else:
+        print(f"Error: Image config not found for: {base_board}")
+        exit(1)
+
 
-    print("Done")
+    print("Done")

src/custompios_core/common.py

@@ -0,0 +1,32 @@
+""" Common functions between CustomPiOS python scripts"""
+from typing import Dict, Any, Optional, cast
+import yaml
+import os
+from pathlib import Path
+
+def get_custompios_folder():
+    custompios_path = os.environ.get("CUSTOM_PI_OS_PATH", None)
+    if custompios_path is not None:
+        return Path(custompios_path)
+    return Path(__file__).parent.parent
+
+
+IMAGES_CONFIG = os.path.join(get_custompios_folder(), "images.yml")
+
+
+def read_images() -> Dict[str, Dict[str,str]]:
+    if not os.path.isfile(IMAGES_CONFIG):
+        raise Exception(f"Error: Remotes config file not found: {IMAGES_CONFIG}")
+    with open(IMAGES_CONFIG,'r') as f:
+        output = yaml.safe_load(f)
+        return output
+
+def get_image_config() -> Optional[Dict["str", Any]]:
+    images = read_images()
+
+    base_board = os.environ.get("BASE_BOARD", None)
+    base_image_path = os.environ.get("BASE_IMAGE_PATH", None)
+
+    if base_board is not None and base_board in images["images"]:
+        return images["images"][base_board]
+    return None

src/custompios_core/generate_board_config.py

@@ -0,0 +1,28 @@
+#!/usr/bin/python3
+import os
+import yaml
+from pathlib import Path
+from typing import Tuple, Optional, Dict, Any, cast
+import git
+from git import RemoteProgress
+from common import get_image_config
+import argparse
+import sys
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(add_help=True, description='Create an export shell script to use the yaml-configured variables')
+    parser.add_argument('output_script', type=str, help='path to output the chroot script master')
+    args = parser.parse_args()
+    image_config = get_image_config()
+    if image_config is None:
+        print("Error: Could not get image config")
+        sys.exit(1)
+    cast(Dict[str,Any], image_config)
+    if not "env" in image_config.keys():
+        print("Warning: no env in image config")
+        exit()
+    env = image_config["env"]
+    with open(args.output_script, "w+") as w:
+        for key in env.keys():
+            w.write(f'export {key}="{env[key]}"\n')
+

src/get_remote_modules.py → src/custompios_core/get_remote_modules.py

@@ -4,10 +4,11 @@
 from typing import Tuple, Optional
 import git
 from git import RemoteProgress
+from common import get_custompios_folder
 
 # TODO add env var to set this
-REMOTES_DIR = os.path.join(os.path.dirname(__file__), "remotes")
-REMOTE_CONFIG = os.path.join(os.path.dirname(__file__), "modules_remote.yml")
+REMOTES_DIR = os.path.join(get_custompios_folder(), "remotes")
+REMOTE_CONFIG = os.path.join(get_custompios_folder(), "modules_remote.yml")
 
 
 class CloneProgress(RemoteProgress):

src/custompios_core/list_boards.py

@@ -0,0 +1,11 @@
+#!/usr/bin/python3
+from common import read_images
+
+if __name__ == "__main__":
+    images = read_images()["images"]
+    print("Available board targest for --board are:")
+    for key in sorted(images):
+        if "description" in images[key].keys():
+            print(f'{key} - {images[key]["description"]}')
+        else:
+            print(key)

src/custompios_core/multi_build.py

@@ -0,0 +1,15 @@
+import argparse
+
+def get_choices():
+    return ['rock', 'paper', 'scissors']
+
+def main():
+    parser = argparse.ArgumentParser(add_help=True, description='Build mulitple images for multiple devices')
+    parser.add_argument('--list', "-l", choices=get_choices(), type=str, nargs='+')
+    args = parser.parse_args()
+    print(args.list)
+    print("Done")
+    return
+
+if __name__ == "__main__":
+    main()