diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 47cf3a1b..e9df3ab4 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -6,6 +6,9 @@ on: - "master" - "devel" - "docker-github-actions" + - "feature/remote-module-support" + - "feature/meta-modules" + - "beta" jobs: docker: diff --git a/README.rst b/README.rst index bde1ae99..5a8b97ee 100644 --- a/README.rst +++ b/README.rst @@ -63,6 +63,7 @@ Requirements #. sudo (the script itself calls it, running as root without sudo won't work) #. p7zip-full #. Python 3.2+ +#. GitPython Known to work building configurations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..9a637d54 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,25 @@ +[tool.poetry] +name = "custompios" +version = "2.0.0" +description = "A Raspberry Pi and other ARM devices distribution builder. CustomPiOS opens an already existing image, modifies it and repackages the image ready to ship." +authors = ["Guy Sheffer "] +license = "GPLv3" +readme = "README.rst" +packages = [ +# { include = "src/*" }, + { include = "custompios_core", from = "src" } + ] + +[tool.poetry.dependencies] +python = "^3.11" +GitPython = "^3.1.41" + +[tool.poetry.group.dev.dependencies] +types-PyYAML = "^6.0.12.12" + +[tool.poetry.scripts] +custompios_build = 'custompios_core.multi_build:main' + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/src/Dockerfile b/src/Dockerfile index aaadf870..ec1f380d 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -13,6 +13,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ wget \ p7zip-full \ python3 \ + python3-distutils \ + python3-dev \ + python3-git \ + python3-yaml \ binfmt-support \ qemu-system \ qemu-user \ diff --git a/src/base_image_downloader_wrapper.sh b/src/base_image_downloader_wrapper.sh new file mode 100755 index 00000000..c2676ad3 --- /dev/null +++ b/src/base_image_downloader_wrapper.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set +x +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +source "$DIR/argparse.bash" || exit 1 +argparse "$@" < Optional[str]: + try: + with urllib.request.urlopen(url) as response: + # Decode the response to a string + webpage = response.read().decode('utf-8') + return webpage + except Exception as e: + print(str(e)) + return None + +def get_location_header(url: str) -> str: + try: + with urllib.request.urlopen(url) as response: + response_url = response.url + + if response_url is None: + raise Exception("Location header is None, can't determine latest rpi image") + return response_url + except Exception as e: + print(str(e)) + print("Error: Failed to determine latest rpi image") + raise e + + +class DownloadProgress: + last_precent: float = 0 + def show_progress(self, block_num, block_size, total_size): + new_precent = round(block_num * block_size / total_size * 100, 1) + if new_precent > self.last_precent + PRECENT_PROGRESS_SIZE: + print(f"{new_precent}%", end="\r") + self.last_precent = new_precent + +def get_file_name(headers, url): + if "Content-Disposition" in headers.keys(): + return re.findall("filename=(\S+)", headers["Content-Disposition"])[0] + return url.split('/')[-1] + +def get_sha256(filename): + sha256_hash = hashlib.sha256() + with open(filename,"rb") as f: + for byte_block in iter(lambda: f.read(4096),b""): + sha256_hash.update(byte_block) + file_checksum = sha256_hash.hexdigest() + return file_checksum + return + +def download_image_http(board: Dict[str, Any], dest_folder: str, redownload: bool = False): + url = board["url"] + checksum = board["checksum"] + download_http(url, checksum) + +def download_http(url: str, checksum_url: str, dest_folder: str, redownload: bool = False): + with tempfile.TemporaryDirectory() as tmpdirname: + print('created temporary directory', tmpdirname) + temp_file_name = os.path.join(tmpdirname, "image.xz") + temp_file_checksum = os.path.join(tmpdirname, "checksum.sha256") + + for r in range(RETRY): + try: + # Get sha and confirm its the right image + download_progress = DownloadProgress() + _, headers_checksum = urllib.request.urlretrieve(checksum_url, temp_file_checksum, download_progress.show_progress) + file_name_checksum = get_file_name(headers_checksum, checksum_url) + + checksum_data = None + with open(temp_file_checksum, 'r') as f: + checksum_data = f.read() + + checksum_data_parsed = [x.strip() for x in checksum_data.split()] + online_checksum = checksum_data_parsed[0] + file_name_from_checksum = checksum_data_parsed[1] + dest_file_name = os.path.join(dest_folder, file_name_from_checksum) + print(f"Downloading {dest_file_name}") + + if os.path.isfile(dest_file_name): + file_checksum = get_sha256(dest_file_name) + if file_checksum == online_checksum: + print("We got base image file and checksum is right") + return + # Get the file + download_progress = DownloadProgress() + _, headers = urllib.request.urlretrieve(url, temp_file_name, download_progress.show_progress) + + file_name = get_file_name(headers, url) + file_checksum = get_sha256(temp_file_name) + if file_checksum != online_checksum: + print(f'Failed. Attempt # {r + 1}, checksum missmatch: {file_checksum} expected: {online_checksum}') + continue + ensure_dir(os.path.dirname(dest_file_name)) + shutil.move(temp_file_name, dest_file_name) + + except Exception as e: + if r < 2: + print(f'Failed. Attempt # {r + 1}, got: {e}') + else: + print('Error encoutered at {RETRY} attempt') + print(e) + exit(1) + else: + print(f"Success: {temp_file_name}") + break + return + + +def download_image_rpi(board: Dict[str, Any], dest_folder: str): + port = board.get("port", "lite_armhf") + os_name = f"raspios" + distribution = board.get("distribution", "bookworm") + version_file = board.get("version_file", "latest") + version_folder = board.get("version_folder", "latest") + + latest_url = f"https://downloads.raspberrypi.org/{os_name}_{port}_latest" + + download_url = f"https://downloads.raspberrypi.org/{os_name}_{port}/images/{os_name}_{port}-{version_folder}/{version_file}-{os_name}-{distribution}-{port}.img.xz" + if version_file == "latest" or version_folder == "latest": + download_url = get_location_header(latest_url) + + checksum_url = f"{download_url}.sha256" + download_http(download_url, checksum_url, dest_folder) + return + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(add_help=True, description='Download images based on BASE_BOARD and BASE_O') + parser.add_argument('WORKSPACE_SUFFIX', nargs='?', default="default", help="The workspace folder suffix used folder") + parser.add_argument('-s', '--sha256', action='store_true', help='Create a sha256 hash for the .img file in .sha256') + args = parser.parse_args() + + images = read_images() + + base_board = os.environ.get("BASE_BOARD", None) + base_image_path = os.environ.get("BASE_IMAGE_PATH", None) + + if base_image_path is None: + print(f'Error: did not find image config file') + exit(1) + cast(str, base_image_path) + + image_config = get_image_config() + if image_config is not None: + if image_config["type"] == "http": + print(f"Downloading image for {base_board}") + download_image_http(image_config, base_image_path) + elif image_config["type"] == "rpi": + print(f"Downloading Raspberry Pi image for {base_board}") + download_image_rpi(image_config, base_image_path) + elif image_config["type"] == "torrent": + print("Error: Torrent not implemented") + exit(1) + else: + print(f'Error: Unsupported image download type: {image_config["type"]}') + exit(1) + else: + print(f"Error: Image config not found for: {base_board}") + exit(1) + + + print("Done") diff --git a/src/custompios_core/common.py b/src/custompios_core/common.py new file mode 100644 index 00000000..eef3bbd2 --- /dev/null +++ b/src/custompios_core/common.py @@ -0,0 +1,32 @@ +""" Common functions between CustomPiOS python scripts""" +from typing import Dict, Any, Optional, cast +import yaml +import os +from pathlib import Path + +def get_custompios_folder(): + custompios_path = os.environ.get("CUSTOM_PI_OS_PATH", None) + if custompios_path is not None: + return Path(custompios_path) + return Path(__file__).parent.parent + + +IMAGES_CONFIG = os.path.join(get_custompios_folder(), "images.yml") + + +def read_images() -> Dict[str, Dict[str,str]]: + if not os.path.isfile(IMAGES_CONFIG): + raise Exception(f"Error: Remotes config file not found: {IMAGES_CONFIG}") + with open(IMAGES_CONFIG,'r') as f: + output = yaml.safe_load(f) + return output + +def get_image_config() -> Optional[Dict["str", Any]]: + images = read_images() + + base_board = os.environ.get("BASE_BOARD", None) + base_image_path = os.environ.get("BASE_IMAGE_PATH", None) + + if base_board is not None and base_board in images["images"]: + return images["images"][base_board] + return None diff --git a/src/custompios_core/execution_order.py b/src/custompios_core/execution_order.py new file mode 100755 index 00000000..ee4b8942 --- /dev/null +++ b/src/custompios_core/execution_order.py @@ -0,0 +1,187 @@ +#!/usr/bin/python3 +#a='base(octopi,a(b,c(a2)),mm)' +import argparse +import os +import subprocess +from get_remote_modules import get_remote_module +from typing import TextIO, List, Tuple, Dict, Any, cast, Union + + +def run_command(command: List[str], **kwargs: Dict[str, Any]): + is_timeout = False + p = subprocess.Popen(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) # type: ignore + try: + stdout, stderr = p.communicate(timeout=5) + except subprocess.TimeoutExpired as e: + p.kill() + stdout,stderr = p.communicate() + is_timeout = True + try: + stdout = stdout.decode("utf-8") + except UnicodeDecodeError as e: + print("Error: can't decode stdout") + print(e) + print(stdout) + stdout = "" + + try: + stderr = stderr.decode("utf-8") + except UnicodeDecodeError as e: + print("Error: can't decode stderr") + print(stderr) + print(e) + stderr = "" + + return stdout, stderr, is_timeout + + +def write_modules_scripts(module: str, state: str, module_folder: str, out: TextIO): + out.write("# " + state + "_" + module + "\n") + script = os.path.join(module_folder, state + "_chroot_script") + if os.path.isfile(script): + out.write("execute_chroot_script '" + module_folder + "' '" + script + "'\n") + else: + print("WARNING: No file at - " + script) + + return + +def parse(a: str) -> List[Tuple[str,str]]: + stack=[] + return_value = [] + token = "" + + for char in a: + if char == "(": + stack.append(token) + if token != "": + return_value.append((token, "start")) + token = "" + elif char == ")": + parent = stack.pop() + if token != "": + return_value.append((token, "start")) + return_value.append((token, "end")) + token = "" + if parent != "": + return_value.append((parent, "end")) + elif char == ",": + if token != "": + return_value.append((token, "start")) + return_value.append((token, "end")) + token = "" + else: + token += char + + if token != "": + return_value.append((token, "start")) + return_value.append((token, "end")) + if len(stack) > 0: + raise Exception(str(stack)) + return return_value + +def handle_meta_modules(modules: List[Tuple[str,str]]) -> Tuple[List[Tuple[str,str]],Dict[str,str]]: + return_value = [] + modules_to_modules_folder = {} + for module, state in modules: + module_folders = [ + os.path.join(os.environ['DIST_PATH'], "modules", module), + os.path.join(os.environ['CUSTOM_PI_OS_PATH'], "modules", module) + ] + # In case this is a meta module, order counts + if state == "start": + return_value.append((module, state)) + found_local = False + found_remote = False + for module_folder in module_folders: + if os.path.isdir(module_folder): + found_local = True + modules_to_modules_folder[module] = module_folder + break + + if not found_local and module: + # TODO: Handle update + found_remote, module_folder_remote = get_remote_module(module) + if module_folder_remote is not None: + module_folder = module_folder_remote + + modules_to_modules_folder[module] = module_folder + + + if not found_local and not found_remote: + print(f"Error: Module {module} does not exist and is not in remote modules list") + exit(1) + + meta_module_path = os.path.join(module_folder, "meta") + if os.path.isfile(meta_module_path): + # Meta module detected + print(f"Running: {meta_module_path}") + print(f"ENV: {os.environ['BASE_BOARD']}") + submodules, meta_module_errors, is_timeout = run_command([meta_module_path]) + submodules = submodules.strip() + print(f"Adding in modules: {submodules}") + if meta_module_errors != "" or is_timeout: + print(meta_module_errors) + print(f"Got error processing meta module at: {meta_module_path}") + exit(1) + if submodules != "": + print(f"Got sub modules: {submodules}") + + for sub_module in submodules.split(","): + sub_module = sub_module.strip() + return_value_sub, modules_to_modules_folder_sub = handle_meta_modules([(sub_module, state)]) + return_value += return_value_sub + modules_to_modules_folder.update(modules_to_modules_folder_sub) + # In case this is a meta module, order counts + if state == "end": + return_value.append((module, state)) + + return return_value, modules_to_modules_folder + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(add_help=True, description='Parse and run CustomPiOS chroot modules') + parser.add_argument('modules', type=str, help='A string showing how the modules should be called') + parser.add_argument('output_script', type=str, help='path to output the chroot script master') + parser.add_argument('modules_after_path', nargs='?', default=None, type=str, help='path to output the chroot script master') + parser.add_argument('remote_and_meta_config_path', nargs='?', default=None, type=str, help='path to output the config script of remote modules and submodules') + args = parser.parse_args() + + if os.path.isfile(args.output_script): + os.remove(args.output_script) + + with open(args.output_script, "w+") as f: + f.write("#!/usr/bin/env bash\n") + f.write("set -x\n") + f.write("set -e\n") + initial_execution_order = parse(args.modules.replace(" ", "")) + f.write(f"# Defined execution order: {initial_execution_order}\n") + modules_execution_order, modules_to_modules_folder = handle_meta_modules(initial_execution_order) + f.write(f"# With meta modules order: {modules_execution_order}\n") + + for module, state in modules_execution_order: + module_folder = modules_to_modules_folder[module] + write_modules_scripts(module, state, module_folder, f) + + # List all new modules add them in, then remove existing ones + list_new_modules = [] + for module, state in modules_execution_order: + if module not in list_new_modules: + list_new_modules.append(module) + for module, state in initial_execution_order: + if module in list_new_modules: + list_new_modules.remove(module) + + # TODO2: load configs from yaml + if args.modules_after_path is not None: + with open(args.modules_after_path, "w") as w: + w.write(",".join(list_new_modules)) + + with open(args.remote_and_meta_config_path, "w") as f: + for module in list_new_modules: + module_folder = modules_to_modules_folder[module] + module_config_path = os.path.join(module_folder, "config") + if os.path.isfile(module_config_path): + f.write(f"source {module_config_path}\n") + + os.chmod(args.output_script, 0o755) + diff --git a/src/custompios_core/generate_board_config.py b/src/custompios_core/generate_board_config.py new file mode 100755 index 00000000..73ffc4b8 --- /dev/null +++ b/src/custompios_core/generate_board_config.py @@ -0,0 +1,28 @@ +#!/usr/bin/python3 +import os +import yaml +from pathlib import Path +from typing import Tuple, Optional, Dict, Any, cast +import git +from git import RemoteProgress +from common import get_image_config +import argparse +import sys + +if __name__ == "__main__": + parser = argparse.ArgumentParser(add_help=True, description='Create an export shell script to use the yaml-configured variables') + parser.add_argument('output_script', type=str, help='path to output the chroot script master') + args = parser.parse_args() + image_config = get_image_config() + if image_config is None: + print("Error: Could not get image config") + sys.exit(1) + cast(Dict[str,Any], image_config) + if not "env" in image_config.keys(): + print("Warning: no env in image config") + exit() + env = image_config["env"] + with open(args.output_script, "w+") as w: + for key in env.keys(): + w.write(f'export {key}="{env[key]}"\n') + diff --git a/src/custompios_core/get_remote_modules.py b/src/custompios_core/get_remote_modules.py new file mode 100644 index 00000000..c723e0e8 --- /dev/null +++ b/src/custompios_core/get_remote_modules.py @@ -0,0 +1,91 @@ +import os +import yaml +from pathlib import Path +from typing import Tuple, Optional +import git +from git import RemoteProgress +from common import get_custompios_folder + +# TODO add env var to set this +REMOTES_DIR = os.path.join(get_custompios_folder(), "remotes") +REMOTE_CONFIG = os.path.join(get_custompios_folder(), "modules_remote.yml") + + +class CloneProgress(RemoteProgress): + def update(self, op_code, cur_count, max_count=None, message=''): + if message: + print(message) + + +def ensure_dir(d, chmod=0o777): + """ + Ensures a folder exists. + Returns True if the folder already exists + """ + if not os.path.exists(d): + os.makedirs(d, chmod) + os.chmod(d, chmod) + return False + return True + + +def read_remotes(): + if not os.path.isfile(REMOTE_CONFIG): + raise Exception(f"Error: Remotes config file not found: {REMOTE_CONFIG}") + with open(REMOTE_CONFIG,'r') as f: + output = yaml.safe_load(f) + return output + +def get_remote_module(module: str) -> Tuple[bool, Optional[str]]: + """ Gets the remote module and saves it to cache. Returns True if found, else false""" + print(f'INFO: Module "{module}", looking for remote module and downloading') + modules_remotes = read_remotes() + print(modules_remotes.keys()) + + if "modules" not in modules_remotes.keys() and module not in modules_remotes["modules"].keys(): + return False, None + + ensure_dir(REMOTES_DIR) + + if "remotes" not in modules_remotes.keys() or module not in modules_remotes["modules"].keys(): + return False, None + + module_config = modules_remotes["modules"][module] + + remote_for_module = module_config["remote"] + remote_config = modules_remotes["remotes"][remote_for_module] + + if remote_config.get("type", "git") == "git": + if "repo" not in remote_config.keys(): + print(f'Error: repo field not set for remote: "{remote_for_module}" used by remote module "{module}"') + return False, None + + if "tag" not in remote_config.keys(): + print(f'Error: repo tag field not set for remote: "{remote_for_module}" used by remote module "{module}"') + return False, None + + repo_url = remote_config["repo"] + branch = remote_config["tag"] + + # credentials = base64.b64encode(f"{GHE_TOKEN}:".encode("latin-1")).decode("latin-1") + # TODO: Handle update of remote + remote_to_path = os.path.join(REMOTES_DIR, remote_for_module) + if not os.path.exists(remote_to_path): + git.Repo.clone_from( + url=repo_url, + single_branch=True, + depth=1, + to_path=f"{remote_to_path}", + branch=branch, + ) + + if "path" not in module_config.keys(): + print(f"Error: repo tag field not set for remote: {remote_for_module} used by remote module {module}") + return False, None + module_path = os.path.join(remote_to_path, module_config["path"]) + return True, module_path + + else: + print(f"Error: unsupported type {modules_remotes[module]['type']} for module {module}") + return False, None + return False, None diff --git a/src/custompios_core/list_boards.py b/src/custompios_core/list_boards.py new file mode 100755 index 00000000..4a770b4c --- /dev/null +++ b/src/custompios_core/list_boards.py @@ -0,0 +1,11 @@ +#!/usr/bin/python3 +from common import read_images + +if __name__ == "__main__": + images = read_images()["images"] + print("Available board targest for --board are:") + for key in sorted(images): + if "description" in images[key].keys(): + print(f'{key} - {images[key]["description"]}') + else: + print(key) diff --git a/src/make_rpi-imager-snipplet.py b/src/custompios_core/make_rpi-imager-snipplet.py similarity index 82% rename from src/make_rpi-imager-snipplet.py rename to src/custompios_core/make_rpi-imager-snipplet.py index bfee8e3f..76e4bd6e 100755 --- a/src/make_rpi-imager-snipplet.py +++ b/src/custompios_core/make_rpi-imager-snipplet.py @@ -6,6 +6,7 @@ import argparse from datetime import date import glob +from typing import Optional, Dict, Union @@ -44,12 +45,13 @@ def handle_arg(key, optional=False): output_path = os.path.join(workspace_path, "rpi-imager-snipplet.json") - json_out = {"name": name, - "description": description, - "url": url, - "icon": icon, - "release_date": release_date, - } + json_out: Dict[str, Optional[Union[str, int]]] = { + "name": name, + "description": description, + "url": url, + "icon": icon, + "release_date": release_date, + } if website is not None: json_out["website"] = website @@ -60,14 +62,14 @@ def handle_arg(key, optional=False): json_out["extract_sha256"] = f.read().split()[0] json_out["extract_size"] = None - with zipfile.ZipFile(zip_local) as zipfile: - json_out["extract_size"] = zipfile.filelist[0].file_size + with zipfile.ZipFile(zip_local) as zip_file: + json_out["extract_size"] = zip_file.filelist[0].file_size json_out["image_download_size"] = os.stat(zip_local).st_size json_out["image_download_sha256"] = None - with open(zip_local,"rb") as f: - json_out["image_download_sha256"] = hashlib.sha256(f.read()).hexdigest() + with open(zip_local,"rb") as fh: + json_out["image_download_sha256"] = hashlib.sha256(fh.read()).hexdigest() with open(output_path, "w") as w: json.dump(json_out, w, indent=2) diff --git a/src/multi-arch-manifest.yaml b/src/custompios_core/multi-arch-manifest.yaml similarity index 100% rename from src/multi-arch-manifest.yaml rename to src/custompios_core/multi-arch-manifest.yaml diff --git a/src/custompios_core/multi_build.py b/src/custompios_core/multi_build.py new file mode 100644 index 00000000..34e4f90a --- /dev/null +++ b/src/custompios_core/multi_build.py @@ -0,0 +1,15 @@ +import argparse + +def get_choices(): + return ['rock', 'paper', 'scissors'] + +def main(): + parser = argparse.ArgumentParser(add_help=True, description='Build mulitple images for multiple devices') + parser.add_argument('--list', "-l", choices=get_choices(), type=str, nargs='+') + args = parser.parse_args() + print(args.list) + print("Done") + return + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/execution_order.py b/src/execution_order.py deleted file mode 100755 index da22e04c..00000000 --- a/src/execution_order.py +++ /dev/null @@ -1,71 +0,0 @@ -#a='base(octopi,a(b,c(a2)),mm)' -import argparse -import os - -def handle(module, state, out): - out.write("# " + state + "_" + module + "\n") - - module_folders = [os.path.join(os.environ['DIST_PATH'], "modules", module), - os.path.join(os.environ['CUSTOM_PI_OS_PATH'], "modules", module) - ] - - for module_folder in module_folders: - if os.path.isdir(module_folder): - script = os.path.join(module_folder, state + "_chroot_script") - if os.path.isfile(script): - out.write("execute_chroot_script '" + module_folder + "' '" + script + "'\n") - else: - print("WARNING: No file at - " + script) - break - return - -def parse(a, callback): - stack=[] - token = "" - - for char in a: - if char == "(": - stack.append(token) - if token != "": - callback(token, "start") - token = "" - elif char == ")": - parent = stack.pop() - if token != "": - callback(token, "start") - callback(token, "end") - token = "" - if parent != "": - callback(parent, "end") - elif char == ",": - if token != "": - callback(token, "start") - callback(token, "end") - token = "" - else: - token += char - - if token != "": - callback(token, "start") - callback(token, "end") - if len(stack) > 0: - raise Exception(str(stack)) - return - -if __name__ == "__main__": - parser = argparse.ArgumentParser(add_help=True, description='Parse and run CustomPiOS chroot modules') - parser.add_argument('modules', type=str, help='A string showing how the modules should be called') - parser.add_argument('output_script', type=str, help='path to output the chroot script master') - args = parser.parse_args() - - if os.path.isfile(args.output_script): - os.remove(args.output_script) - - with open(args.output_script, "w+") as f: - f.write("#!/usr/bin/env bash\n") - f.write("set -x\n") - f.write("set -e\n") - parse(args.modules.replace(" ", ""), lambda module, state: handle(module, state, f)) - - os.chmod(args.output_script, 0o755) - diff --git a/src/images.yml b/src/images.yml new file mode 100644 index 00000000..78c96ee2 --- /dev/null +++ b/src/images.yml @@ -0,0 +1,66 @@ +images: + raspberrypiarmhf: + description: "Official raspberrypi lite 32bit image" + type: rpi + env: + BASE_ARCH: armhf + raspberrypiarm64: + description: "Official raspberrypi lite 64bit image" + type: rpi + port: lite_arm64 + env: + BASE_ARCH: arm64 + raspberrypiarmhf_full: + description: "Official raspberrypi full 32bit image" + type: rpi + port: full_armhf + env: + BASE_ARCH: armhf + raspberrypiarm64: + description: "Official raspberrypi full 64bit image" + type: rpi + port: full_arm64 + env: + BASE_ARCH: arm64 + orangepi_orangepi_zero2: + description: "Orange Pi Zero2" + url: "https://github.com/mainsail-crew/armbian-builds/releases/latest/download/orangepi-orangepi_zero2_bullseye.img.xz" + checksum: "https://github.com/mainsail-crew/armbian-builds/releases/latest/download/orangepi-orangepi_zero2_bullseye.img.xz.sha256" + type: http + env: + BASE_ARCH: arm64 + armbian_bananapim2zero: + description: "Banana Pi BPI-M2 ZERO" + url: "https://github.com/mainsail-crew/armbian-builds/releases/latest/download/armbian-bananapi_m2_zero_bullseye.img.xz" + checksum: "https://github.com/mainsail-crew/armbian-builds/releases/latest/download/armbian-bananapi_m2_zero_bullseye.img.xz.sha256" + type: http + env: + BASE_ARCH: arm64 + armbian_orangepi3lts: + description: "Orange Pi 3 LTS" + url: "https://github.com/mainsail-crew/armbian-builds/releases/latest/download/armbian-orangepi3_lts_bullseye.img.xz" + checksum: "https://github.com/mainsail-crew/armbian-builds/releases/latest/download/armbian-orangepi3_lts_bullseye.img.xz.sha256" + type: http + env: + BASE_ARCH: arm64 + armbian_orangepi4lts: + description: "Orange Pi 4 LTS" + url: "https://github.com/mainsail-crew/armbian-builds/releases/latest/download/armbian-orangepi4_lts_bullseye.img.xz" + checksum: "https://github.com/mainsail-crew/armbian-builds/releases/latest/download/armbian-orangepi4_lts_bullseye.img.xz.sha256" + type: http + env: + BASE_ARCH: arm64 + BASE_DISTRO: armbian + BASE_IMAGE_RASPBIAN: "no" + raspbian_lepotato: + description: "Le Potato AML-S905X-CC Raspbian image" + url: "https://distro.libre.computer/ci/raspbian/11/2023-05-03-raspbian-bullseye-arm64-lite%2Baml-s905x-cc.img.xz" + checksum: "https://distro.libre.computer/ci/raspbian/11/SHA256SUMS" + type: http + env: + BASE_ARCH: arm64 + BASE_DISTRO: raspbian + BASE_IMAGE_RASPBIAN: "yes" + BASE_ADD_USER: "yes" + BASE_USER: "pi" + BASE_USER_PASSWORD: "lepotato" diff --git a/src/modules/base/config b/src/modules/base/config index da791a6a..f4c9c461 100644 --- a/src/modules/base/config +++ b/src/modules/base/config @@ -8,8 +8,19 @@ BASE_VERSION=1.5.0 [ -n "$BASE_PRESCRIPT" ] || BASE_PRESCRIPT= [ -n "$BASE_POSTSCRIPT" ] || BASE_POSTSCRIPT= +# Board and OS +[ -n "$BASE_BOARD" ] || BASE_BOARD="raspberrypi" +[ -n "$BASE_OS" ] || BASE_OS="debian_bookworm" + +# TODO: UNIFY AND MAKE A SINGLE IMAGE FLOW, ATM THERE IS THE OLD AND NEW FLOW TOGETHER, UBUTU CAN GO IN ITS OWN + #[ -n "$BASE_SCRIPT_PATH" ] || BASE_SCRIPT_PATH=$CONFIG_DIR -[ -n "$BASE_IMAGE_PATH" ] || BASE_IMAGE_PATH=${DIST_PATH}/image +if [ "${BASE_BOARD}" == "raspberrypiarmhf" ] && [ "${BASE_BOARD}" == "raspberrypiarm64" ]; then + [ -n "$BASE_IMAGE_PATH" ] || BASE_IMAGE_PATH=${DIST_PATH}/image +else + [ -n "$BASE_IMAGE_PATH" ] || BASE_IMAGE_PATH=${DIST_PATH}/image-"${BASE_BOARD}" + BASE_ZIP_IMG=`ls -t $BASE_IMAGE_PATH/*.{zip,7z,xz} | head -n 1` +fi [ -n "$BASE_IMAGE_RASPBIAN" ] || BASE_IMAGE_RASPBIAN=yes # Distro @@ -28,7 +39,7 @@ if [ "${BASE_DISTRO}" = "ubuntu" ]; then [ -n "$BASE_USER_PASSWORD" ] || BASE_USER_PASSWORD=ubuntu else # Default image raspbian - if [ "${BASE_DISTRO}" = "raspios64" ]; then + if [ "${BASE_DISTRO}" == "raspios64" ] && [ "${BASE_BOARD}" == "raspberrypiarmhf" ] && [ "${BASE_BOARD}" == "raspberrypiarm64" ]; then [ -n "$BASE_ZIP_IMG" ] || BASE_ZIP_IMG=$(ls -t $BASE_IMAGE_PATH/*-{raspbian,raspios}-*-arm64-*.{zip,7z,xz} | head -n 1) else [ -n "$BASE_ZIP_IMG" ] || BASE_ZIP_IMG=$(ls -t $BASE_IMAGE_PATH/*-{raspbian,raspios}*.{zip,7z,xz} | head -n 1) diff --git a/src/modules/base/meta b/src/modules/base/meta new file mode 100755 index 00000000..73c65266 --- /dev/null +++ b/src/modules/base/meta @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Base script +# Basic and manditory settings for the base of a CustomPiOS build +# Written by Guy Sheffer +# GPL V3 +######## +set -e + +export LC_ALL=C + +FINAL_MODULES=() + +if [[ "${BASE_BOARD}" = armbian* ]]; then + FINAL_MODULES+=("armbian") +elif [[ "${BASE_BOARD}" = orange* ]]; then + FINAL_MODULES+=("orange") +fi + +printf '%s\n' "$(IFS=,; printf '%s' "${FINAL_MODULES[*]}")" diff --git a/src/modules/network/config b/src/modules/network/config index 1504d394..9b8923fd 100755 --- a/src/modules/network/config +++ b/src/modules/network/config @@ -10,4 +10,10 @@ # on reboots # udev - creates a udev rules that should affect all wifi devices. -[ -n "$NETWORK_PWRSAVE_TYPE" ] || NETWORK_PWRSAVE_TYPE=udev \ No newline at end of file +[ -n "$NETWORK_PWRSAVE_TYPE" ] || NETWORK_PWRSAVE_TYPE=udev + +# Enable WPA-Supplicant boot folder support (pre rpios bookworm) +[ -n "$NETWORK_WPA_SUPPLICANT" ] || NETWORK_WPA_SUPPLICANT=no + +# Enable Network Manager boot folder support (bookworm) +[ -n "$NETWORK_NETWORK_MANAGER" ] || NETWORK_NETWORK_MANAGER=yes \ No newline at end of file diff --git a/src/modules/network/filesystem/network-manager/boot/wifi.nmconnection b/src/modules/network/filesystem/network-manager/boot/wifi.nmconnection new file mode 100644 index 00000000..afca98b3 --- /dev/null +++ b/src/modules/network/filesystem/network-manager/boot/wifi.nmconnection @@ -0,0 +1,24 @@ +# Uncomment everything below this line and set your ssid and password +# [connection] +# id=wifi +# uuid=593819b8-135a-4a3e-9611-c36cdeadbeef +# type=wifi +# interface-name=wlan0 + +# [wifi] +# mode=infrastructure +# ssid=set your wifi ssid here + +# [wifi-security] +# auth-alg=open +# key-mgmt=wpa-psk +# psk=set your password here + +# [ipv4] +# method=auto + +# [ipv6] +# addr-gen-mode=default +# method=auto + +# [proxy] diff --git a/src/modules/network/filesystem/network-manager/root/etc/systemd/system/copy-network-manager-config@.service b/src/modules/network/filesystem/network-manager/root/etc/systemd/system/copy-network-manager-config@.service new file mode 100644 index 00000000..d5c52923 --- /dev/null +++ b/src/modules/network/filesystem/network-manager/root/etc/systemd/system/copy-network-manager-config@.service @@ -0,0 +1,21 @@ +#### Enable network manager configuration from boot +#### +#### Written by Guy Sheffer +#### Copyright 2024 +#### https://github.com/guysoft/CustomPiOS +#### +#### This File is distributed under GPLv3 + +[Unit] +Description=persistent setup on %I +Wants=network-pre.target +Before=network-pre.target +BindsTo=sys-subsystem-net-devices-%i.device +After=sys-subsystem-net-devices-%i.device + +[Service] +ExecStart=/opt/custompios/copy-network-manager-config %I +Type=oneshot + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/src/modules/network/filesystem/network-manager/root/opt/custompios/copy-network-manager-config b/src/modules/network/filesystem/network-manager/root/opt/custompios/copy-network-manager-config new file mode 100755 index 00000000..3b2ec777 --- /dev/null +++ b/src/modules/network/filesystem/network-manager/root/opt/custompios/copy-network-manager-config @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +( + echo "# DO NOT EDIT THIS FILE" + echo "# Edit /boot/firmware/wifi.nmconnection and it will be copied here" + cat /boot/firmware/wifi.nmconnection +) > /etc/NetworkManager/system-connections/wifi.nmconnection +chmod 600 /etc/NetworkManager/system-connections/wifi.nmconnection +echo Done \ No newline at end of file diff --git a/src/modules/network/filesystem/boot/custompios-wpa-supplicant.txt b/src/modules/network/filesystem/wpa-supplicant/boot/custompios-wpa-supplicant.txt similarity index 100% rename from src/modules/network/filesystem/boot/custompios-wpa-supplicant.txt rename to src/modules/network/filesystem/wpa-supplicant/boot/custompios-wpa-supplicant.txt diff --git a/src/modules/network/meta b/src/modules/network/meta new file mode 100755 index 00000000..d56198b9 --- /dev/null +++ b/src/modules/network/meta @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Net script +# Basic and manditory settings for the network of a CustomPiOS build +# Written by Guy Sheffer +# GPL V3 +######## +set -e + +export LC_ALL=C + +FINAL_MODULES=() + +if [[ "${BASE_BOARD}" = armbian* ]]; then + FINAL_MODULES+=("armbian_net") +elif [[ "${BASE_BOARD}" = orange* ]]; then + FINAL_MODULES+=("orange_net") +fi + +printf '%s\n' "$(IFS=,; printf '%s' "${FINAL_MODULES[*]}")" diff --git a/src/modules/network/start_chroot_script b/src/modules/network/start_chroot_script index c2be6a0c..2b086a08 100755 --- a/src/modules/network/start_chroot_script +++ b/src/modules/network/start_chroot_script @@ -12,29 +12,39 @@ export LC_ALL=C source /common.sh install_cleanup_trap -unpack /filesystem/boot /"${BASE_BOOT_MOUNT_PATH}" +if [ "${NETWORK_WPA_SUPPLICANT}" == "yes" ]; then + unpack /filesystem/wpa-supplicant/boot /"${BASE_BOOT_MOUNT_PATH}" + DIST_NETWORK_FILE=/"${BASE_BOOT_MOUNT_PATH}"/${DIST_NAME,,}-wpa-supplicant.txt -DIST_NETWORK_FILE=/"${BASE_BOOT_MOUNT_PATH}"/${DIST_NAME,,}-wpa-supplicant.txt + # allow configuring multiple wifi networks via /boot/DIST_NAME-wpa-supplicant.txt + mv /"${BASE_BOOT_MOUNT_PATH}"/custompios-wpa-supplicant.txt ${DIST_NETWORK_FILE} -# allow configuring multiple wifi networks via /boot/DIST_NAME-wpa-supplicant.txt -mv /"${BASE_BOOT_MOUNT_PATH}"/custompios-wpa-supplicant.txt ${DIST_NETWORK_FILE} + if [ "${BASE_DISTRO}" == "ubuntu" ] || [ "${BASE_DISTRO}" == "armbian" ]; then + echo "ctrl_interface=DIR=/var/run/wpa_supplicant GROUP=netdev" > /etc/wpa_supplicant/wpa_supplicant.conf + echo "update_config=1" >> /etc/wpa_supplicant/wpa_supplicant.conf + fi -if [ "${BASE_DISTRO}" == "ubuntu" ] || [ "${BASE_DISTRO}" == "armbian" ]; then - echo "ctrl_interface=DIR=/var/run/wpa_supplicant GROUP=netdev" > /etc/wpa_supplicant/wpa_supplicant.conf - echo "update_config=1" >> /etc/wpa_supplicant/wpa_supplicant.conf + cat /etc/wpa_supplicant/wpa_supplicant.conf >> ${DIST_NETWORK_FILE} # append distributed conf to our own + rm -f /etc/wpa_supplicant/wpa_supplicant.conf # remove distributed conf + + # create symlink + ln -s "${DIST_NETWORK_FILE}" /etc/wpa_supplicant/wpa_supplicant.conf fi -cat /etc/wpa_supplicant/wpa_supplicant.conf >> ${DIST_NETWORK_FILE} # append distributed conf to our own -rm -f /etc/wpa_supplicant/wpa_supplicant.conf # remove distributed conf +if [ "${NETWORK_NETWORK_MANAGER}" == "yes" ]; then + unpack filesystem/network-manager/root / root + unpack filesystem/network-manager/boot /"${BASE_BOOT_MOUNT_PATH}" + + # mv /"${BASE_BOOT_MOUNT_PATH}"/wifi.nmconnection ${DIST_NETWORK_FILE} + + systemctl_if_exists enable copy-network-manager-config@wlan0.service +fi if [ "${BASE_DISTRO}" == "raspbian" ]; then # Workaround rfkill not unblocking on boot rm /var/lib/systemd/rfkill/* fi -# create symlink -ln -s "${DIST_NETWORK_FILE}" /etc/wpa_supplicant/wpa_supplicant.conf - # copy /etc/wpa_supplicant/ifupdown.sh to /etc/ifplugd/action.d/ifupdown - for wlan auto reconnect [ -f /etc/ifplugd/action.d/ifupdown ] && mv /etc/ifplugd/action.d/ifupdown /etc/ifplugd/action.d/ifupdown.original [ -f /etc/wpa_supplicant/ifupdown.sh ] && ln -s /etc/wpa_supplicant/ifupdown.sh /etc/ifplugd/action.d/ifupdown diff --git a/src/modules_remote.yml b/src/modules_remote.yml new file mode 100644 index 00000000..603aa4b3 --- /dev/null +++ b/src/modules_remote.yml @@ -0,0 +1,67 @@ +modules: + armbian: + remote: MainsailOS + path: src/modules/armbian + + armbian_net: + remote: MainsailOS + path: src/modules/armbian_net + + crowsnest: + remote: MainsailOS + path: src/modules/crowsnest + + is_req_preinstall: + remote: MainsailOS + path: src/modules/is_req_preinstall + + klipper: + remote: MainsailOS + path: src/modules/klipper + + mainsail: + remote: MainsailOS + path: src/modules/mainsail + + mainsailos: + remote: MainsailOS + path: src/modules/mainsailos + + moonraker: + remote: MainsailOS + path: src/modules/moonraker + + orangepi: + remote: MainsailOS + path: src/modules/orangepi + + orangepi_net: + remote: MainsailOS + path: src/modules/orangepi_net + + piconfig: + remote: MainsailOS + path: src/modules/piconfig + + postrename: + remote: MainsailOS + path: src/modules/postrename + + sonar: + remote: MainsailOS + path: src/modules/sonar + + timelapse: + remote: MainsailOS + path: src/modules/timelapse + + udev_fix: + remote: MainsailOS + path: src/modules/udev_fix + +remotes: + MainsailOS: + repo: https://github.com/mainsail-crew/MainsailOS.git + type: git + tag: develop + auto_update: true diff --git a/src/nightly_build_scripts/custompios_nightly_build b/src/nightly_build_scripts/custompios_nightly_build index 51b76443..bf78c4d3 100755 --- a/src/nightly_build_scripts/custompios_nightly_build +++ b/src/nightly_build_scripts/custompios_nightly_build @@ -1,5 +1,4 @@ #!/bin/bash -set -x set -e export DIST_PATH=/distro @@ -11,8 +10,15 @@ argparse "$@" < "${IMG_FILENAME}".sha256 fi +if [ ! -z "${BOARD}" ]; then + export BASE_BOARD="${BOARD}" +fi +if [ "${BASE_BOARD}" == "" ]; then + echo "No BASE_BOARD defined, defaulting to raspberrypiarmhf" + export BASE_BOARD="raspberrypiarmhf" +fi + for i in `lsof "${DIST_PATH}/workspace${WORKSPACE_POSTFIX}/mount" | awk '{print $2}'`; do kill -9 $i; done || true rm ${DIST_PATH}/workspace${WORKSPACE_POSTFIX}/*.img || true @@ -40,6 +54,13 @@ pushd "${DIST_PATH}" #export OCTOPI_OCTOPRINT_REPO_BUILD='http://localhost/git/OctoPrint.git/' #export OCTOPI_MJPGSTREAMER_REPO_BUILD='http://localhost/git/mjpg-streamer.git/' #export OCTOPI_WIRINGPI_REPO_BUILD='http://localhost/git/wiringPi.git/' + + # Download latest image of correct board type if possible + source "${CUSTOM_PI_OS_PATH}/config" "${WORKSPACE_SUFFIX}" + if [ "${DOWNLOAD}" == "yes" ]; then + ${CUSTOM_PI_OS_PATH}/custompios_core/base_image_downloader.py "${WORKSPACE_SUFFIX}" + fi + ${CUSTOM_PI_OS_PATH}/build_custom_os "${WORKSPACE_SUFFIX}" exit_code=$? if [ $exit_code -ne 0 ]; then @@ -59,8 +80,7 @@ pushd "${DIST_PATH}" echo "Error: you must have \${CUSTOM_PI_OS_PATH} to generate a rpi-imager snipplet" exit 1 fi - source "${CUSTOM_PI_OS_PATH}/config" "${WORKSPACE_SUFFIX}" - "${CUSTOM_PI_OS_PATH}/make_rpi-imager-snipplet.py" --rpi_imager_url "${RPI_IMAGER_URL}" -- "${WORKSPACE_SUFFIX}" + "${CUSTOM_PI_OS_PATH}/custompios_core/make_rpi-imager-snipplet.py" --rpi_imager_url "${RPI_IMAGER_URL}" -- "${WORKSPACE_SUFFIX}" fi chmod 777 ${DIST_PATH}/workspace${WORKSPACE_POSTFIX}/* diff --git a/src/requirements-devel.txt b/src/requirements-devel.txt new file mode 100644 index 00000000..eef8302b --- /dev/null +++ b/src/requirements-devel.txt @@ -0,0 +1 @@ +types-PyYAML \ No newline at end of file diff --git a/src/requirements.txt b/src/requirements.txt new file mode 100644 index 00000000..64b1adae --- /dev/null +++ b/src/requirements.txt @@ -0,0 +1 @@ +GitPython diff --git a/src/variants/armbian/config b/src/variants/armbian/config index 826ad78c..e83c7ecd 100755 --- a/src/variants/armbian/config +++ b/src/variants/armbian/config @@ -1,6 +1,7 @@ VARIANT_CONFIG_DIR=$(realpath -s $(dirname $(realpath -s $BASH_SOURCE))/../..) BASE_ZIP_IMG=`ls -t ${DIST_PATH}/image-armbian/*.{zip,7z,xz} | head -n 1` BASE_APT_CACHE=no +BASE_BOARD=armbian OCTOPI_INCLUDE_WIRINGPI=no export BASE_DISTRO=armbian # The root partition of the image filesystem, 2 for raspbian, 1 for armbian