From 50b3055275acd3597c421d3e3338f25f2f6010d2 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 13 Feb 2024 18:23:05 +0530 Subject: [PATCH 01/33] feat: vscode debugger integration --- frappe_manager/__init__.py | 16 ++++--- frappe_manager/commands.py | 9 ++-- frappe_manager/site_manager/SiteManager.py | 50 ++++++++++++++++++---- frappe_manager/site_manager/__init__.py | 29 +++++++++++++ 4 files changed, 84 insertions(+), 20 deletions(-) diff --git a/frappe_manager/__init__.py b/frappe_manager/__init__.py index 14d37fac..21d54d95 100644 --- a/frappe_manager/__init__.py +++ b/frappe_manager/__init__.py @@ -2,24 +2,26 @@ from enum import Enum # TODO configure this using config -#sites_dir = Path().home() / __name__.split(".")[0] -CLI_DIR = Path.home() / 'frappe' -CLI_METADATA_PATH = CLI_DIR / '.fm.toml' -CLI_SITES_ARCHIVE = CLI_DIR / 'archived' +# sites_dir = Path().home() / __name__.split(".")[0] +CLI_DIR = Path.home() / "frappe" +CLI_METADATA_PATH = CLI_DIR / ".fm.toml" +CLI_SITES_ARCHIVE = CLI_DIR / "archived" default_extension = [ "dbaeumer.vscode-eslint", "esbenp.prettier-vscode", "ms-python.python", - "ms-python.black-formatter", + "ms-python.debugpy", "ms-python.flake8", + "ms-python.black-formatter", "visualstudioexptteam.vscodeintellicode", - "VisualStudioExptTeam.intellicode-api-usage-examples" + "VisualStudioExptTeam.intellicode-api-usage-examples", ] + class SiteServicesEnum(str, Enum): - frappe= "frappe" + frappe = "frappe" nginx = "nginx" mailhog = "mailhog" adminer = "adminer" diff --git a/frappe_manager/commands.py b/frappe_manager/commands.py index 130ba8ca..764725c8 100644 --- a/frappe_manager/commands.py +++ b/frappe_manager/commands.py @@ -241,7 +241,7 @@ def stop(sitename: Annotated[str, typer.Argument(help="Name of the site")]): sites.stop_site() -def code_command_callback(extensions: List[str]) -> List[str]: +def code_command_extensions_callback(extensions: List[str]) -> List[str]: extx = extensions + default_extension unique_ext: Set = set(extx) unique_ext_list: List[str] = [x for x in unique_ext] @@ -258,16 +258,17 @@ def code( "--extension", "-e", help="List of extensions to install in vscode at startup.Provide extension id eg: ms-python.python", - callback=code_command_callback, + callback=code_command_extensions_callback, ), ] = default_extension, - force_start: Annotated[bool , typer.Option('--force-start','-f',help="Force start the site before attaching to container.")] = False + force_start: Annotated[bool , typer.Option('--force-start','-f',help="Force start the site before attaching to container.")] = False, + debugger: Annotated[bool , typer.Option('--debugger','-d',help="Sync vscode debugger configuration.")] = False ): """Open site in vscode. """ sites.init(sitename) if force_start: sites.start_site() - sites.attach_to_site(user, extensions) + sites.attach_to_site(user, extensions, debugger) @app.command(no_args_is_help=True) diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index 24e88557..d87a4249 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -7,13 +7,14 @@ from typing import List, Optional from pathlib import Path +from datetime import datetime +from frappe_manager.site_manager import VSCODE_LAUNCH_JSON, VSCODE_TASKS_JSON from frappe_manager.site_manager.site import Site from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.docker_wrapper import DockerClient, DockerException from frappe_manager import CLI_DIR from rich.table import Table -from frappe_manager.utils.helpers import check_and_display_port_status from frappe_manager.utils.site import generate_services_table @@ -238,7 +239,7 @@ def start_site(self): self.site.frappe_logs_till_start(status_msg="Starting Site") self.site.sync_workers_compose() - def attach_to_site(self, user: str, extensions: List[str]): + def attach_to_site(self, user: str, extensions: List[str], debugger: bool = False): """ Attaches to a running site's container using Visual Studio Code Remote Containers extension. @@ -255,7 +256,7 @@ def attach_to_site(self, user: str, extensions: List[str]): if not vscode_path: richprint.exit( - "Visual Studio Code excutable 'code' nott accessible via cli." + "Visual Studio Code binary i.e 'code' is not accessible via cli." ) container_hex = self.site.get_frappe_container_hex() @@ -271,7 +272,15 @@ def attach_to_site(self, user: str, extensions: List[str]): vscode_config_json = [ { "remoteUser": user, - "customizations": {"vscode": {"extensions": extensions}}, + "remoteEnv": { + "SHELL": "/bin/zsh" + }, + "customizations": {"vscode": { + "settings": { + "python.pythonPath": "/workspace/frappe-bench/env/bin/python" + }, + "extensions": extensions + }}, } ] @@ -299,6 +308,34 @@ def attach_to_site(self, user: str, extensions: List[str]): self.site.start() richprint.print(f"Recreating Containers : Done") + # sync debugger files + if debugger: + richprint.change_head("Sync vscode debugger configuration") + dot_vscode_dir = self.site.path / 'workspace' / '.vscode' + tasks_json_path = dot_vscode_dir / 'tasks' + launch_json_path = dot_vscode_dir / 'launch' + + dot_vscode_config = { + tasks_json_path : VSCODE_TASKS_JSON, + launch_json_path : VSCODE_LAUNCH_JSON, + } + + if not dot_vscode_dir.exists(): + dot_vscode_dir.mkdir(exist_ok=True,parents=True) + + for file_path in [launch_json_path,tasks_json_path]: + file_name = f'{file_path.name}.json' + real_file_path = file_path.parent / file_name + if real_file_path.exists(): + backup_tasks_path = file_path.parent / f"{file_path.name}.{datetime.now().strftime('%d-%b-%y--%H-%M-%S')}.json" + shutil.copy2(real_file_path, backup_tasks_path) + richprint.print(f"Backup previous '{file_name}' : {backup_tasks_path}") + + with open(real_file_path, 'w+') as f: + f.write(json.dumps(dot_vscode_config[file_path])) + + richprint.print("Sync vscode debugger configuration: Done") + richprint.change_head("Attaching to Container") output = subprocess.run(vscode_cmd, shell=True) @@ -424,7 +461,6 @@ def info(self): site_info_table.add_row("Bench Apps", bench_apps_list_table) running_site_services = self.site.get_services_running_status() - running_site_workers = self.site.workers.get_services_running_status() if running_site_services: @@ -436,10 +472,6 @@ def info(self): site_info_table.add_row("Site Workers", site_workers_table) richprint.stdout.print(site_info_table) - # richprint.print( - # f":green_square: -> Active :red_square: -> Inactive", - # emoji_code=":information: ", - # ) def migrate_site(self): """ diff --git a/frappe_manager/site_manager/__init__.py b/frappe_manager/site_manager/__init__.py index e69de29b..c01dea6f 100644 --- a/frappe_manager/site_manager/__init__.py +++ b/frappe_manager/site_manager/__init__.py @@ -0,0 +1,29 @@ +VSCODE_LAUNCH_JSON = { + "version": "0.2.0", + "configurations": [ + { + "name": "fm-frappe-debug", + "type": "debugpy", + "request": "launch", + "program": "/workspace/frappe-bench/apps/frappe/frappe/utils/bench_helper.py", + "args": ["frappe", "serve", "--port", "80", "--noreload", "--nothreading"], + "cwd": "/workspace/frappe-bench/sites", + "env": {"DEV_SERVER": "1"}, + "preLaunchTask": "fm-kill-port-80", + } + ], +} + +VSCODE_TASKS_JSON = { + "version": "2.0.0", + "tasks": [ + { + "label": "fm-kill-port-80", + "type": "shell", + "command": "/bin/bash", + "args": ["-c", "fuser -k 80/tcp || true"], + "presentation": {"reveal": "never", "panel": "dedicated"}, + "options": {"ignoreExitCode": True}, + } + ], +} From a1bc55965a14c1fe0555f6cadb65a8f1bb6dc597 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 13 Feb 2024 18:41:06 +0530 Subject: [PATCH 02/33] cleanup code and import --- frappe_manager/migration_manager/migration_executor.py | 2 -- .../migration_manager/migrations/migrate_0_10_0.py | 8 -------- 2 files changed, 10 deletions(-) diff --git a/frappe_manager/migration_manager/migration_executor.py b/frappe_manager/migration_manager/migration_executor.py index 953f35e8..84cc4358 100644 --- a/frappe_manager/migration_manager/migration_executor.py +++ b/frappe_manager/migration_manager/migration_executor.py @@ -6,8 +6,6 @@ import pkgutil from pathlib import Path -import rich - from rich import inspect from frappe_manager import CLI_DIR , CLI_SITES_ARCHIVE from frappe_manager.compose_manager.ComposeFile import ComposeFile diff --git a/frappe_manager/migration_manager/migrations/migrate_0_10_0.py b/frappe_manager/migration_manager/migrations/migrate_0_10_0.py index 60e52110..e74cc47d 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_10_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_10_0.py @@ -1,4 +1,3 @@ -from sys import exception from typing import Optional from dataclasses import dataclass import shutil @@ -26,13 +25,6 @@ from frappe_manager.migration_manager.version import Version from pathlib import Path -# from frappe_manager.display_manager.DisplayManager import richprint - -# @dataclass -# class MigrateSite: -# site_name : str -# exception: Optional[Exception] = None - class MigrationV0100(MigrationBase): version = Version("0.10.0") From 27b197c5cc67eef1bbd445d32049bc6e8b6e8d02 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Wed, 14 Feb 2024 17:23:58 +0530 Subject: [PATCH 03/33] bench restart kill child process at exit --- Docker/frappe/Dockerfile | 3 ++- Docker/frappe/bench-dev-watch.sh | 3 +++ Docker/frappe/frappe-dev.conf | 4 +++- 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 Docker/frappe/bench-dev-watch.sh diff --git a/Docker/frappe/Dockerfile b/Docker/frappe/Dockerfile index ad6d3d7a..f1ef6d9a 100644 --- a/Docker/frappe/Dockerfile +++ b/Docker/frappe/Dockerfile @@ -145,6 +145,7 @@ RUN echo 'export PATH="/opt/user/.bin:$PATH"' >> "$USERZSHRC" COPY ./supervisord.conf /opt/user/ COPY ./bench-dev-server /opt/user/ +COPY ./bench-dev-watch.sh /opt/user/ COPY ./frappe-dev.conf /opt/user/ COPY ./bench-wrapper.sh /opt/user/.bin/bench @@ -155,6 +156,6 @@ COPY ./divide-supervisor-conf.py /scripts/ RUN mkdir -p /scripts -RUN sudo chmod +x /entrypoint.sh /scripts/user-script.sh /scripts/launch.sh /scripts/divide-supervisor-conf.py /opt/user/bench-dev-server /opt/user/.bin/bench +RUN sudo chmod +x /entrypoint.sh /scripts/user-script.sh /scripts/launch.sh /scripts/divide-supervisor-conf.py /opt/user/bench-dev-server /opt/user/bench-dev-watch.sh /opt/user/.bin/bench ENTRYPOINT ["/bin/bash","/entrypoint.sh"] diff --git a/Docker/frappe/bench-dev-watch.sh b/Docker/frappe/bench-dev-watch.sh new file mode 100644 index 00000000..a08efe67 --- /dev/null +++ b/Docker/frappe/bench-dev-watch.sh @@ -0,0 +1,3 @@ +#!/bin/bash +trap "kill -- -$$" EXIT +bench watch diff --git a/Docker/frappe/frappe-dev.conf b/Docker/frappe/frappe-dev.conf index 7bcc92f6..d362301f 100644 --- a/Docker/frappe/frappe-dev.conf +++ b/Docker/frappe/frappe-dev.conf @@ -9,7 +9,7 @@ user=frappe directory=/workspace/frappe-bench [program:frappe-bench-frappe-watch] -command=bench watch +command=/opt/user/bench-dev-watch.sh priority=4 autostart=true autorestart=false @@ -17,6 +17,8 @@ stdout_logfile=/workspace/frappe-bench/logs/watch.dev.log redirect_stderr=true user=frappe directory=/workspace/frappe-bench +stopasgroup=true +stopsignal=QUIT [group:frappe-bench-dev] programs=frappe-bench-frappe-dev,frappe-bench-frappe-watch From bda416a6881cdf543a17b21d849cd68727e6588a Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 23 Feb 2024 14:29:00 +0530 Subject: [PATCH 04/33] feat: prebaked bench in frappe image --- Docker/frappe/Dockerfile | 99 ++++++++++----- Docker/frappe/entrypoint.sh | 15 +-- Docker/frappe/helper-function.sh | 189 ++++++++++++++++++++++++++++ Docker/frappe/prebake.sh | 44 +++++++ Docker/frappe/user-script.sh | 143 ++++++++++----------- frappe_manager/site_manager/site.py | 14 ++- 6 files changed, 385 insertions(+), 119 deletions(-) create mode 100755 Docker/frappe/helper-function.sh create mode 100644 Docker/frappe/prebake.sh diff --git a/Docker/frappe/Dockerfile b/Docker/frappe/Dockerfile index f1ef6d9a..73714347 100644 --- a/Docker/frappe/Dockerfile +++ b/Docker/frappe/Dockerfile @@ -2,7 +2,10 @@ FROM ubuntu:22.04 as bench LABEL author=rtCamp LABEL org.opencontainers.image.source=https://github.com/rtcamp/Frappe-Manager + ARG PYTHON_VERSION=3.11.0 +ARG PREBAKE_APPS='erpnext:version-15,hrms:version-15' +ARG PREBAKE_FRAPPE_BRANCH='version-15' RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ # For frappe framework @@ -15,7 +18,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install --no-instal xfonts-75dpi \ xfonts-base \ libssl-dev \ - fonts-cantarell \ + fonts-cantarell \ libpangocairo-1.0-0 \ # to work inside the container locales \ @@ -75,35 +78,47 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install --no-instal RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \ && dpkg-reconfigure --frontend=noninteractive locales -# Detect arch and install wkhtmltopdf -ENV WKHTMLTOPDF_VERSION 0.12.6.1-3 -RUN if [ "$(uname -m)" = "aarch64" ]; then export ARCH=arm64; fi \ - && if [ "$(uname -m)" = "x86_64" ]; then export ARCH=amd64; fi \ - && downloaded_file=wkhtmltox_$WKHTMLTOPDF_VERSION.jammy_${ARCH}.deb \ - && wget -q https://github.com/wkhtmltopdf/packaging/releases/download/$WKHTMLTOPDF_VERSION/$downloaded_file \ - && dpkg -i $downloaded_file \ - && rm $downloaded_file +# setup user +RUN export NAME='frappe' && \ + groupadd -g 1000 $NAME && \ + useradd --no-log-init -r -m -u 1000 -g 1000 -G sudo -s /usr/bin/zsh -d /workspace "$NAME" && \ + usermod -a -G tty "$NAME" && \ + echo "$NAME ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers +RUN echo PREBAKE_APPS="$PREBAKE_APPS" >> /prebake_info && echo PREBAKE_FRAPPE_BRANCH="$PREBAKE_FRAPPE_BRANCH" >> /prebake_info -# Install Python via pyenv ENV PYENV_ROOT /opt/.pyenv ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH + +# for bench wrapper +ENV PATH /opt/user/.bin:${PATH} + +# for nvm +ENV NODE_VERSION=18.17.0 +ENV NVM_DIR /opt/.nvm +ENV PATH ${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH} + +From bench as prebake + +RUN chown -R frappe:frappe /opt + +USER frappe + ENV USERZSHRC /opt/user/.zshrc ENV USERPROFILE /opt/user/.profile -# From https://github.com/pyenv/pyenv#basic-github-checkout WORKDIR /opt -RUN mkdir -p /opt/user && touch /opt/user/.profile +RUN mkdir -p /opt/user && touch /opt/user/.profile && ls -lah ENV ZSH /opt/user/.oh-my-zsh ENV DISABLE_UPDATE_PROMPT true +# install ohmyzsh RUN sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" "" --unattended - RUN unset ZSH -COPY ./zshrc /opt/user/.zshrc +COPY --chown=frappe:frappe ./zshrc /opt/user/.zshrc RUN git clone --depth 1 https://github.com/pyenv/pyenv.git .pyenv \ && pyenv install $PYTHON_VERSION \ @@ -115,12 +130,6 @@ RUN git clone --depth 1 https://github.com/pyenv/pyenv.git .pyenv \ RUN pip install frappe-bench -# Install Node via nvm -ENV NODE_VERSION=18.17.0 - -ENV NVM_DIR /opt/.nvm -ENV PATH ${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH} - RUN mkdir -p /opt/.nvm \ && wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash \ && . ${NVM_DIR}/nvm.sh \ @@ -133,29 +142,55 @@ RUN mkdir -p /opt/.nvm \ && echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm' >> "$USERZSHRC" \ && echo '[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion' >> "$USERZSHRC" - RUN mkdir -p /workspace WORKDIR /workspace RUN mkdir -p /opt/user/.bin -ENV PATH /opt/user/.bin:${PATH} RUN echo 'export PATH="/opt/user/.bin:$PATH"' >> "$USERZSHRC" -COPY ./supervisord.conf /opt/user/ -COPY ./bench-dev-server /opt/user/ -COPY ./bench-dev-watch.sh /opt/user/ -COPY ./frappe-dev.conf /opt/user/ -COPY ./bench-wrapper.sh /opt/user/.bin/bench +COPY --chown=frappe:frappe ./supervisord.conf /opt/user/ +COPY --chown=frappe:frappe ./frappe-dev.conf /opt/user/ +COPY --chown=frappe:frappe --chmod=0755 ./bench-dev-watch.sh /opt/user/ +COPY --chown=frappe:frappe --chmod=0755 ./bench-dev-server /opt/user/ +COPY --chown=frappe:frappe --chmod=0755 ./bench-wrapper.sh /opt/user/.bin/bench -COPY ./entrypoint.sh / -COPY ./user-script.sh /scripts/ -COPY ./launch.sh /scripts/ -COPY ./divide-supervisor-conf.py /scripts/ +COPY --chmod=0755 ./prebake.sh /scripts/ +COPY --chmod=0755 ./helper-function.sh /scripts/ +COPY --chmod=0755 ./divide-supervisor-conf.py /scripts/ + +RUN ls -lah /workspace && /scripts/prebake.sh && mv /workspace/frappe-bench/apps/* /workspace/ + +FROM bench as fm_image + +# Detect arch and install wkhtmltopdf +ENV WKHTMLTOPDF_VERSION 0.12.6.1-3 +RUN if [ "$(uname -m)" = "aarch64" ]; then export ARCH=arm64; fi \ + && if [ "$(uname -m)" = "x86_64" ]; then export ARCH=amd64; fi \ + && downloaded_file=wkhtmltox_$WKHTMLTOPDF_VERSION.jammy_${ARCH}.deb \ + && wget -q https://github.com/wkhtmltopdf/packaging/releases/download/$WKHTMLTOPDF_VERSION/$downloaded_file \ + && dpkg -i $downloaded_file \ + && rm $downloaded_file RUN mkdir -p /scripts -RUN sudo chmod +x /entrypoint.sh /scripts/user-script.sh /scripts/launch.sh /scripts/divide-supervisor-conf.py /opt/user/bench-dev-server /opt/user/bench-dev-watch.sh /opt/user/.bin/bench +COPY --chmod=0755 ./entrypoint.sh / +COPY --chmod=0755 ./user-script.sh /scripts/ +COPY --chmod=0755 ./launch.sh /scripts/ +COPY --chmod=0755 ./divide-supervisor-conf.py /scripts/ +COPY --chmod=0755 ./helper-function.sh /scripts/ + +RUN rm -rf /opt && mkdir -p /workspace /opt + +WORKDIR /workspace + +COPY --from=prebake --chown=frappe:frappe /workspace/frappe-bench /workspace/frappe-bench +COPY --from=prebake --chown=frappe:frappe /workspace/frappe /workspace/frappe-bench/apps/frappe +COPY --from=prebake --chown=frappe:frappe /workspace/erpnext /workspace/frappe-bench/apps/erpnext +COPY --from=prebake --chown=frappe:frappe /workspace/hrms /workspace/frappe-bench/apps/hrms +COPY --from=prebake --chown=frappe:frappe /opt/.pyenv /opt/.pyenv +COPY --from=prebake --chown=frappe:frappe /opt/.nvm /opt/.nvm +COPY --from=prebake --chown=frappe:frappe /opt/user /opt/user ENTRYPOINT ["/bin/bash","/entrypoint.sh"] diff --git a/Docker/frappe/entrypoint.sh b/Docker/frappe/entrypoint.sh index 14760e62..12926dd6 100755 --- a/Docker/frappe/entrypoint.sh +++ b/Docker/frappe/entrypoint.sh @@ -1,5 +1,7 @@ #!/bin/bash +source /scripts/helper-function.sh + emer() { echo "$1" exit 1 @@ -10,11 +12,7 @@ emer() { echo "Setting up user" -NAME='frappe' -groupadd -g "$USERGROUP" $NAME -useradd --no-log-init -r -m -u "$USERID" -g "$USERGROUP" -G sudo -s /usr/bin/zsh -d /workspace "$NAME" -usermod -a -G tty "$NAME" -echo "$NAME ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers +update_uid_gid "${USERID}" "${USERGROUP}" "frappe" "frappe" mkdir -p /opt/user/conf.d @@ -33,12 +31,7 @@ if [[ ! -f "/workspace/.profile" ]]; then fi -if [[ ! -d '/workspace/frappe-bench' ]]; then - chown -R "$USERID":"$USERGROUP" /workspace - # find /workspace -type d -print0 | xargs -0 -n 200 -P "$(nproc)" chown "$USERID":"$USERGROUP" - # find /workspace -type f -print0 | xargs -0 -n 200 -P "$(nproc)" chown "$USERID":"$USERGROUP" -fi - +chown -R "$USERID":"$USERGROUP" /workspace if [ "$#" -gt 0 ]; then gosu "$USERID":"$USERGROUP" "/scripts/$@" diff --git a/Docker/frappe/helper-function.sh b/Docker/frappe/helper-function.sh new file mode 100755 index 00000000..9278467f --- /dev/null +++ b/Docker/frappe/helper-function.sh @@ -0,0 +1,189 @@ +#!/usr/bin/bash + +# Function: update_common_site_config +# Description: Updates the common site config file with the provided key-value pair. +# Parameters: +# - key: The key to be updated in the config file. +# - value: The value to be assigned to the key in the config file. +# - is_value_json: Optional parameter. Set to true if the value provided is in JSON format. +update_common_site_config() { + local key="$1" + local value="$2" + local is_value_json="$3" # set to true if any value provided + local config_file="/workspace/frappe-bench/sites/common_site_config.json" + + # Check if the config file exists + if [ ! -f "$config_file" ]; then + echo "Error: Common site config file not found." + return 1 + fi + + # Update the config file using jq + if [[ "${is_value_json:-}" ]]; then + jq -r --arg key "$key" --argjson value "$value" '.[$key] = $value' "$config_file" > "$config_file.tmp" && mv "$config_file.tmp" "$config_file" + else + jq -r --arg key "$key" --arg value "$value" '.[$key] = $value' "$config_file" > "$config_file.tmp" && mv "$config_file.tmp" "$config_file" + fi + + echo "Updated $key => $value" +} + +# Function to retrieve a value from the common site config file based on a given key +# Arguments: +# - key: The key to search for in the common site config file +# Returns: +# - The value corresponding to the given key, or "null" if the key does not exist +get_common_site_config() { + local key="$1" + local config_file="/workspace/frappe-bench/sites/common_site_config.json" + + # Check if the config file exists + if [ ! -f "$config_file" ]; then + echo "Error: Common site config file not found." + return 1 + fi + + # Use jq to extract the value corresponding to the given key + value=$(jq -r --arg key "$key" '.[$key]' "$config_file") + + # Check if the key exists + if [ "$value" = "null" ]; then + echo "null" + else + echo "$value" + fi +} + +# Function to install apps +# Parameters: +# - apps_lists: comma-separated list of apps to install +# - already_installed_apps: comma-separated list of apps already installed +install_apps() { + local apps_lists + local already_installed_apps + local remove_apps + + apps_lists="$1" + already_installed_apps="$2" + + keep_prebaked_apps=$(mktemp) + + # get apps_json if not available then default to empty list + apps_json='[]' + + if [[ "${apps_lists:-}" ]]; then + apps=$(awk -F ',' '{for (i=1; i<=NF; i++) {print $i}}' <<<"$apps_lists") + for app in $apps; do + + app_contains_http=0 + + if [[ $app == http* ]]; then + app_contains_http=1 + app="${app//http:/https:}" + app="${app//https:/https;}" + fi + + app_name=$(echo "$app" | awk 'BEGIN {FS=":"}; {print $1}') + branch_name=$(echo "$app" | awk 'BEGIN {FS=":"}; {print $2}') + + if [[ "${app_contains_http}" -gt 0 ]]; then + app_name="${app_name//https\;/https:}" + fi + + # check if app prebaked + ALREADY_PREBAKED=$(grep -cw "$app" <<<"$already_installed_apps" || exit 0) + + if [[ "${ALREADY_PREBAKED}" -gt 0 ]]; then + echo "${app} already prebaked and installed." + echo "${app}" >> "$keep_prebaked_apps" + # apps_json=$(echo "$apps_json" | jq -rc --arg app_name "${app_name}" '.+ [$app_name]') + continue + fi + + if [[ "${branch_name:-}" ]]; then + echo "Installing app $app_name -> $branch_name" + $BENCH_COMMAND get-app --resolve-deps --overwrite --skip-assets --branch "${branch_name}" "${app_name}" + else + echo "Installing app $app_name" + $BENCH_COMMAND get-app --resolve-deps --overwrite --skip-assets "${app_name}" + fi + done + else + echo "No app provided to install." + fi + + remove_apps=$(awk -F ',' '{for (i=1; i<=NF; i++) {print $i}}' <<<"$already_installed_apps") + + for app in $remove_apps; do + app_name=$(echo "$app" | awk 'BEGIN {FS=":"}; {print $1}') + + is_app_installed=$(cat "$keep_prebaked_apps" | grep -cw "$app_name" || exit 0) + + if [[ ! "$is_app_installed" -gt 0 ]]; then + echo "remove app ${app_name}" + (bench rm --no-backup --force "${app_name}" || exit 0) + apps_json=$(echo "$apps_json" | jq -rc --arg app_name "${app_name}" 'del(.[] | select(. == $app_name))') + fi + done + + # create apps_json + for app_name in $(ls -1 apps | grep -v 'frappe' || exit 0); do + apps_json=$(echo "$apps_json" | jq -rc --arg app_name "${app_name}" '.+ [$app_name]') + done + update_common_site_config install_apps "$apps_json" 'true' +} + + +# Function: update_uid_gid +# Description: Updates the UID (User ID) and GID (Group ID) of a user and group in the system. +# Parameters: +# - uid: The new UID to be assigned to the user. +# - gid: The new GID to be assigned to the group. +# - username: The username of the user. +# - groupname: The name of the group. +# Returns: +# - 0: If the UID and GID are updated successfully. +# - 1: If the function is called with incorrect number of parameters or if the UID or GID are not numeric values. +# Notes: +# - If a user or group with the same UID or GID already exists, it will be deleted and updated with the provided username or groupname. +update_uid_gid() { + if [ "$#" -ne 4 ]; then + echo "Usage: update_uid_gid " + return 1 + fi + + uid="$1" + gid="$2" + username="$3" + groupname="$4" + + # Validate numeric fields + if [[ ! "$uid" =~ ^[0-9]+$ || ! "$gid" =~ ^[0-9]+$ ]]; then + echo "Error: UID and GID must be numeric values." + return 1 + fi + + # Check if UID and GID already exist + existing_uid_user="$(getent passwd "$uid" | cut -d: -f1)" + existing_gid_group="$(getent group "$gid" | cut -d: -f1)" + + if [[ ! -z "$existing_uid_user" && "$existing_uid_user" != "$username" ]]; then + # User already registered, but with different username + # Delete the user with existing UID and update with provided username + userdel "$existing_uid_user" + echo "User $existing_uid_user deleted." + fi + + if [[ ! -z "$existing_gid_group" && "$existing_gid_group" != "$groupname" ]]; then + # Group already registered, but with different groupname + # Delete the group with existing GID and update with provided groupname + groupdel "$existing_gid_group" + echo "Group $existing_gid_group deleted." + fi + + # Update UID and GID + usermod -u "$uid" "$username" + groupmod -g "$gid" "$groupname" + + echo "UID and GID updated successfully." +} diff --git a/Docker/frappe/prebake.sh b/Docker/frappe/prebake.sh new file mode 100644 index 00000000..f2c41432 --- /dev/null +++ b/Docker/frappe/prebake.sh @@ -0,0 +1,44 @@ +#!/bin/bash +source /scripts/helper-function.sh + +set -e + +WEB_PORT=80 +REDIS_SOCKETIO_PORT=80 + +emer() { + echo "$@" + exit 1 +} + +BENCH_COMMAND='/opt/.pyenv/shims/bench' + +configure_common_site_config(){ + $BENCH_COMMAND config dns_multitenant on + update_common_site_config webserver_port "$WEB_PORT" + update_common_site_config socketio_port "$REDIS_SOCKETIO_PORT" +} + +# create bench +$BENCH_COMMAND init --skip-assets --skip-redis-config-generation --frappe-branch "$PREBAKE_FRAPPE_BRANCH" frappe-bench + +cd frappe-bench + +configure_common_site_config + +# install apps +install_apps "$PREBAKE_APPS" + +# Addresses the introduction of the --host flag in bench serve command for compatibility with Frappe version updates. +bench_serve_help_output=$($BENCH_COMMAND serve --help) +host_changed=$(echo "$bench_serve_help_output" | grep -c 'host' || true) + +if [[ "$host_changed" -ge 1 ]]; then + awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--host 0.0.0.0 --port "a); print}' /opt/user/bench-dev-server > file.tmp && mv file.tmp /opt/user/bench-dev-server.sh +else + awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--port "a); print}' /opt/user/bench-dev-server > file.tmp && mv file.tmp /opt/user/bench-dev-server.sh +fi + +chmod +x /opt/user/bench-dev-server.sh + +$BENCH_COMMAND build diff --git a/Docker/frappe/user-script.sh b/Docker/frappe/user-script.sh index f233c5c1..c6cfc7dd 100755 --- a/Docker/frappe/user-script.sh +++ b/Docker/frappe/user-script.sh @@ -1,28 +1,32 @@ #!/bin/bash -# This script creates bench and executes it. + +source /scripts/helper-function.sh +source /prebake_info + set -e + emer() { echo "$@" exit 1 } -if [[ -d 'logs' ]]; then +# handle previous logs location symlink +if [[ -d 'logs' ]]; then if [[ -f 'logs/bench-start.log' ]]; then mv logs/bench-start.log logs/bench-start.log.bak fi ln -sfn ../frappe-bench/logs/web.dev.log logs/bench-start.log fi - REDIS_SOCKETIO_PORT=80 WEB_PORT=80 if [[ ! "${MARIADB_HOST:-}" ]]; then - MARIADB_HOST='global-db' + MARIADB_HOST='global-db' fi if [[ ! "${MARIADB_ROOT_PASS:-}" ]]; then - MARIADB_ROOT_PASS='root' + MARIADB_ROOT_PASS='root' fi BENCH_COMMAND='/opt/.pyenv/shims/bench' @@ -31,8 +35,29 @@ BENCH_COMMAND='/opt/.pyenv/shims/bench' [[ "${WEB_PORT:-}" ]] || emer "[ERROR] WEB_PORT env not found. Please provide WEB_PORT env." [[ "${SITENAME:-}" ]] || emer "[ERROR] SITENAME env not found. Please provide SITENAME env." -# if the bench doesn't exists -if [[ ! -d "frappe-bench" ]]; then +configure_common_site_config() { + # start_time=$(date +%s.%N) + + update_common_site_config db_host "$MARIADB_HOST" + update_common_site_config db_port 3307 + update_common_site_config redis_cache "redis://${CONTAINER_NAME_PREFIX}-redis-cache:6379" + update_common_site_config redis_queue "redis://${CONTAINER_NAME_PREFIX}-redis-queue:6379" + update_common_site_config redis_socketio "redis://${CONTAINER_NAME_PREFIX}-redis-socketio:6379" + update_common_site_config mail_port 1025 + update_common_site_config mail_server 'mailhog' + update_common_site_config disable_mail_smtp_authentication 1 + update_common_site_config webserver_port "$WEB_PORT" + update_common_site_config developer_mode "$DEVELOPER_MODE" + update_common_site_config socketio_port "$REDIS_SOCKETIO_PORT" + update_common_site_config restart_supervisor_on_update 0 + + # end_time=$(date +%s.%N) + # execution_time=$(awk "BEGIN {print $end_time - $start_time}") + # echo "Execution time for set-config : $execution_time seconds" +} + +# check if the site is created +if [[ ! -d "/workspace/frappe-bench/sites/$SITENAME" ]]; then [[ "${REDIS_SOCKETIO_PORT:-}" ]] || emer "[ERROR] REDIS_SOCKETIO_PORT env not found. Please provide REDIS_SOCKETIO_PORT env." [[ "${DEVELOPER_MODE:-}" ]] || emer "[ERROR] DEVELOPER_MODE env not found. Please provide DEVELOPER_MODE env." @@ -42,78 +67,45 @@ if [[ ! -d "frappe-bench" ]]; then [[ "${DB_NAME:-}" ]] || emer "[ERROR] DB_NAME env not found. Please provide DB_NAME env." [[ "${CONTAINER_NAME_PREFIX:-}" ]] || emer "[ERROR] CONTAINER_NAME_PREFIX env not found. Please provide CONTAINER_NAME_PREFIX env." - # create the bench - $BENCH_COMMAND init --skip-assets --skip-redis-config-generation --frappe-branch "$FRAPPE_BRANCH" frappe-bench - # setting configuration - wait-for-it -t 120 "$MARIADB_HOST":3306; - wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-cache":6379; - wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-queue":6379; - wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-socketio":6379; + wait-for-it -t 120 "$MARIADB_HOST":3306 + wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-cache":6379 + wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-queue":6379 + wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-socketio":6379 cd frappe-bench - $BENCH_COMMAND config dns_multitenant on - $BENCH_COMMAND set-config -g db_host "$MARIADB_HOST" - $BENCH_COMMAND set-config -g db_port 3306 - $BENCH_COMMAND set-config -g redis_cache "redis://${CONTAINER_NAME_PREFIX}-redis-cache:6379" - $BENCH_COMMAND set-config -g redis_queue "redis://${CONTAINER_NAME_PREFIX}-redis-queue:6379" - $BENCH_COMMAND set-config -g redis_socketio "redis://${CONTAINER_NAME_PREFIX}-redis-socketio:6379" - $BENCH_COMMAND set-config -g mail_port 1025 - $BENCH_COMMAND set-config -g mail_server 'mailhog' - $BENCH_COMMAND set-config -g disable_mail_smtp_authentication 1 - $BENCH_COMMAND set-config -g webserver_port "$WEB_PORT" - $BENCH_COMMAND set-config -g developer_mode "$DEVELOPER_MODE" - $BENCH_COMMAND set-config -g socketio_port "$REDIS_SOCKETIO_PORT" - - # HANDLE APPS - # apps are taken as follows - # appsname:branch - # no branch if you want to do default installation of the app - - apps_json='[]' - if [[ "${APPS_LIST:-}" ]]; then - apps=$(awk -F ',' '{for (i=1; i<=NF; i++) {print $i}}' <<<"$APPS_LIST") - for app in $apps; do - app_name=$(echo "$app" | awk 'BEGIN {FS=":"}; {print $1}') - branch_name=$(echo "$app" | awk 'BEGIN {FS=":"}; {print $2}') - if [[ "${branch_name:-}" ]]; then - echo "Installing app $app_name -> $branch_name" - $BENCH_COMMAND get-app --skip-assets --branch "${branch_name}" "${app_name}" - else - echo "Installing app $app_name" - $BENCH_COMMAND get-app --skip-assets "${app_name}" - fi - apps_json=$(echo "$apps_json" | jq --arg app_name "${app_name}" '.+ [$app_name]') - done + configure_common_site_config + + # HANDLE Frappe + if [[ ! "${FRAPPE_BRANCH}" = "${PREBAKE_FRAPPE_BRANCH}" ]]; then + bench get-app --overwrite --branch "${FRAPPE_BRANCH}" frappe fi - apps_json=$(echo "$apps_json" | jq -rc '.') - # add install apps config to common site config - $BENCH_COMMAND set-config -g install_apps "$apps_json" --parse + install_apps "$APPS_LIST" "$PREBAKE_APPS" - # change the procfile port 8000 to 80 - # this will chaange the web serving port from 8000 to 80 + rm -rf archived - bench_serve_help_output=$($BENCH_COMMAND serve --help) - - host_changed=$(echo "$bench_serve_help_output" | grep -c 'host' || true) - - # SUPERVIOSRCONFIG_STATUS=$(bench setup supervisor --skip-redis --skip-supervisord --yes --user "$USER") $BENCH_COMMAND setup supervisor --skip-redis --skip-supervisord --yes --user "$USER" - /scripts/divide-supervisor-conf.py config/supervisor.conf + echo "Environment: ${ENVIRONMENT}" + echo "Configuring frappe dev server" + bench_serve_help_output=$($BENCH_COMMAND serve --help) + host_changed=$(echo "$bench_serve_help_output" | grep -c 'host' || true) + + # Addresses the introduction of the --host flag in bench serve command for compatibility with Frappe version updates. if [[ "$host_changed" -ge 1 ]]; then - awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--host 0.0.0.0 --port "a); print}' /opt/user/bench-dev-server > file.tmp && mv file.tmp /opt/user/bench-dev-server.sh + awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--host 0.0.0.0 --port "a); print}' /opt/user/bench-dev-server >file.tmp && mv file.tmp /opt/user/bench-dev-server.sh else - awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--port "a); print}' /opt/user/bench-dev-server > file.tmp && mv file.tmp /opt/user/bench-dev-server.sh + awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--port "a); print}' /opt/user/bench-dev-server >file.tmp && mv file.tmp /opt/user/bench-dev-server.sh fi chmod +x /opt/user/bench-dev-server.sh - $BENCH_COMMAND build - $BENCH_COMMAND new-site --db-root-password $(cat $MARIADB_ROOT_PASS) --db-name "$DB_NAME" --db-host "$MARIADB_HOST" --admin-password "$ADMIN_PASS" --no-mariadb-socket "$SITENAME" + $BENCH_COMMAND build & + $BENCH_COMMAND new-site --db-root-password $(cat $MARIADB_ROOT_PASS) --db-name "$DB_NAME" --db-host "$MARIADB_HOST" --admin-password "$ADMIN_PASS" --db-port 3306 --verbose --no-mariadb-socket "$SITENAME" + $BENCH_COMMAND use "$SITENAME" $BENCH_COMMAND --site "$SITENAME" scheduler enable wait @@ -130,48 +122,49 @@ if [[ ! -d "frappe-bench" ]]; then supervisord -c /opt/user/supervisord.conf fi - else - set -x - wait-for-it -t 120 "$MARIADB_HOST":3306; - wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-cache":6379; - wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-queue":6379; - wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-socketio":6379; + wait-for-it -t 120 "$MARIADB_HOST":3306 + wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-cache":6379 + wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-queue":6379 + wait-for-it -t 120 "${CONTAINER_NAME_PREFIX}-redis-socketio":6379 cd frappe-bench + echo "Environment: ${ENVIRONMENT}" + echo "Configuring frappe dev server" + # Addresses the introduction of the --host flag in bench serve command for compatibility with Frappe version updates. bench_serve_help_output=$($BENCH_COMMAND serve --help) host_changed=$(echo "$bench_serve_help_output" | grep -c 'host' || true) $BENCH_COMMAND setup supervisor --skip-redis --skip-supervisord --yes --user "$USER" - /scripts/divide-supervisor-conf.py config/supervisor.conf if [[ "$host_changed" -ge 1 ]]; then - awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--host 0.0.0.0 --port "a); print}' /opt/user/bench-dev-server > file.tmp && mv file.tmp /opt/user/bench-dev-server.sh + awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--host 0.0.0.0 --port "a); print}' /opt/user/bench-dev-server >file.tmp && mv file.tmp /opt/user/bench-dev-server.sh else - awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--port "a); print}' /opt/user/bench-dev-server > file.tmp && mv file.tmp /opt/user/bench-dev-server.sh + awk -v a="$WEB_PORT" '{sub(/--port [[:digit:]]+/,"--port "a); print}' /opt/user/bench-dev-server >file.tmp && mv file.tmp /opt/user/bench-dev-server.sh fi chmod +x /opt/user/bench-dev-server.sh + # Addresses the introduction of the --host flag in bench serve command for compatibility with Frappe version updates. if [[ "${ENVIRONMENT}" = "dev" ]]; then - cp /opt/user/frappe-dev.conf /opt/user/conf.d/frappe-dev.conf else if [[ -f '/opt/user/conf.d/frappe-bench-frappe-web.fm.supervisor.conf' ]]; then - ln -sfn /workspace/frappe-bench/config/frappe-bench-frappe-web.fm.supervisor.conf /opt/user/conf.d/frappe-bench-frappe-web.fm.supervisor.conf + + ln -sfn /workspace/frappe-bench/config/frappe-bench-frappe-web.fm.supervisor.conf /opt/user/conf.d/frappe-bench-frappe-web.fm.supervisor.conf else - emer 'Not able to start the server. /opt/user/conf.d/frappe-bench-frappe-web.fm.supervisor.conf not available.' + emer 'Not able to start the server. /opt/user/conf.d/frappe-bench-frappe-web.fm.supervisor.conf not found.' fi fi if [[ -n "$BENCH_START_OFF" ]]; then tail -f /dev/null else + echo "Starting supervisor.." supervisord -c /opt/user/supervisord.conf fi fi - diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index f03ce605..c3a6095e 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -258,8 +258,20 @@ def create_compose_dirs(self) -> bool: richprint.change_head("Creating Compose directories") # create compose bind dirs -> workspace + # create compose bind dirs -> workspace + # create workspace from frappe image + + frappe_image = self.composefile.yml["services"]["frappe"]["image"] + workspace_path = self.path / "workspace" - workspace_path.mkdir(parents=True, exist_ok=True) + workspace_path_abs = str(workspace_path.absolute()) + + host_run_cp( + frappe_image, + source="/workspace", + destination=workspace_path_abs, + docker=self.docker, + ) configs_path = self.path / "configs" configs_path.mkdir(parents=True, exist_ok=True) From b77456fd0eaf36293f9ae67ae27ce88e113d356b Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 23 Feb 2024 14:39:13 +0530 Subject: [PATCH 05/33] support public github apps in fm create --- frappe_manager/utils/callbacks.py | 18 +++++++++++++++++- frappe_manager/utils/helpers.py | 21 ++++++++++++++------- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/frappe_manager/utils/callbacks.py b/frappe_manager/utils/callbacks.py index 0d9e4216..6ea38bed 100644 --- a/frappe_manager/utils/callbacks.py +++ b/frappe_manager/utils/callbacks.py @@ -17,13 +17,26 @@ def apps_list_validation_callback(value: List[str] | None): Returns: List[str] | None: The validated list of apps. """ + apps_list = [] + if value: for app in value: appx = app.split(":") + if appx == "frappe": raise typer.BadParameter("Frappe should not be included here.") + + if 'https:' in app or 'http:' in app: + temp_appx = appx + appx = [":".join(appx[:2])] + + if len(temp_appx) == 3: + appx.append(temp_appx[2]) + if len(appx) == 1: + exists = check_frappe_app_exists(appx[0]) + if not exists["app"]: raise typer.BadParameter(f"{app} is not a valid FrappeVerse app!") if len(appx) == 2: @@ -38,7 +51,10 @@ def apps_list_validation_callback(value: List[str] | None): raise typer.BadParameter( "App should be specified in format : or " ) - return value + + appx = ":".join(appx) + apps_list.append(appx) + return apps_list def frappe_branch_validation_callback(value: str): diff --git a/frappe_manager/utils/helpers.py b/frappe_manager/utils/helpers.py index 98f4c1e1..34903498 100644 --- a/frappe_manager/utils/helpers.py +++ b/frappe_manager/utils/helpers.py @@ -1,5 +1,6 @@ import importlib import sys +from typing import Optional import requests import json import subprocess @@ -201,8 +202,7 @@ def get_current_fm_version(): """ return importlib.metadata.version("frappe-manager") - -def check_frappe_app_exists(appname: str, branchname: str | None = None): +def check_repo_exists(app_url:str, branch_name: str | None = None): """ Check if a Frappe app exists on GitHub. @@ -214,19 +214,26 @@ def check_frappe_app_exists(appname: str, branchname: str | None = None): dict: A dictionary containing the existence status of the app and branch (if provided). """ try: - app_url = f"https://github.com/frappe/{appname}" app = requests.get(app_url).status_code - if branchname: - branch_url = f"https://github.com/frappe/{appname}/tree/{branchname}" + if branch_name: + branch_url = f"{app_url}/tree/{branch_name}" branch = requests.get(branch_url).status_code return { "app": True if app == 200 else False, "branch": True if branch == 200 else False, } return {"app": True if app == 200 else False} - except Exception: - richprint.exit("Not able to connect to github.com.") + + except Exception as e: + raise Exception("Not able to connect to github.com.") + +def check_frappe_app_exists(app: str, branch_name: Optional[str] = None): + + if 'github.com' not in app: + app= f"https://github.com/frappe/{app}" + + return check_repo_exists(app_url=app,branch_name=branch_name) def represent_null_empty(string_null): From bb3e2140ccfe0c6edf902cbeae9245ab6c82436e Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 23 Feb 2024 14:43:06 +0530 Subject: [PATCH 06/33] default apps stable app-branch mapping --- frappe_manager/__init__.py | 6 ++++++ frappe_manager/utils/callbacks.py | 11 ++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/frappe_manager/__init__.py b/frappe_manager/__init__.py index 21d54d95..e52a03bb 100644 --- a/frappe_manager/__init__.py +++ b/frappe_manager/__init__.py @@ -31,3 +31,9 @@ class SiteServicesEnum(str, Enum): redis_socketio = "redis-socketio" schedule = "schedule" socketio = "socketio" + + +STABLE_APP_BRANCH_MAPPING_LIST = { + "erpnext" :'version-15', + "hrms" :'version-15', +} diff --git a/frappe_manager/utils/callbacks.py b/frappe_manager/utils/callbacks.py index 6ea38bed..37e6ac85 100644 --- a/frappe_manager/utils/callbacks.py +++ b/frappe_manager/utils/callbacks.py @@ -2,6 +2,7 @@ from typing import List, Optional from frappe_manager.utils.helpers import check_frappe_app_exists, get_current_fm_version from frappe_manager.display_manager.DisplayManager import richprint +from frappe_manager import STABLE_APP_BRANCH_MAPPING_LIST def apps_list_validation_callback(value: List[str] | None): @@ -39,14 +40,22 @@ def apps_list_validation_callback(value: List[str] | None): if not exists["app"]: raise typer.BadParameter(f"{app} is not a valid FrappeVerse app!") + + if appx[0] in STABLE_APP_BRANCH_MAPPING_LIST: + appx.append(STABLE_APP_BRANCH_MAPPING_LIST[appx[0]]) + if len(appx) == 2: + exists = check_frappe_app_exists(appx[0], appx[1]) + if not exists["app"]: raise typer.BadParameter(f"{app} is not a valid FrappeVerse app!") + if not exists["branch"]: raise typer.BadParameter( f"{appx[1]} is not a valid branch of {appx[0]}!" ) + if len(appx) > 2: raise typer.BadParameter( "App should be specified in format : or " @@ -87,4 +96,4 @@ def version_callback(version: Optional[bool] = None): if version: fm_version = get_current_fm_version() richprint.print(fm_version, emoji_code='') - raise typer.Exit() \ No newline at end of file + raise typer.Exit() From 3c08735dd2ab07c437a2d2ab72d35df9b5f7fa3c Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 23 Feb 2024 15:07:36 +0530 Subject: [PATCH 07/33] Modified error msg in apps validation --- frappe_manager/utils/callbacks.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/frappe_manager/utils/callbacks.py b/frappe_manager/utils/callbacks.py index 37e6ac85..b770ad51 100644 --- a/frappe_manager/utils/callbacks.py +++ b/frappe_manager/utils/callbacks.py @@ -25,21 +25,35 @@ def apps_list_validation_callback(value: List[str] | None): appx = app.split(":") if appx == "frappe": - raise typer.BadParameter("Frappe should not be included here.") + raise typer.BadParameter("'frappe' should not be included here.") if 'https:' in app or 'http:' in app: temp_appx = appx appx = [":".join(appx[:2])] + if len(temp_appx) == 3: appx.append(temp_appx[2]) + elif len(temp_appx) > 3: + appx.append(temp_appx[2]) + appx.append(temp_appx[2]) + + if len(appx) > 2: + richprint.stop() + msg = ( + "Specify the app in the format : or ." + "\n can be a URL or, if it's a FrappeVerse app, simply provide it as 'erpnext' or 'hrms:develop'." + ) + raise typer.BadParameter(msg) + if len(appx) == 1: exists = check_frappe_app_exists(appx[0]) if not exists["app"]: - raise typer.BadParameter(f"{app} is not a valid FrappeVerse app!") + richprint.stop() + raise typer.BadParameter(f"Invalid app '{appx[0]}'.") if appx[0] in STABLE_APP_BRANCH_MAPPING_LIST: appx.append(STABLE_APP_BRANCH_MAPPING_LIST[appx[0]]) @@ -49,17 +63,15 @@ def apps_list_validation_callback(value: List[str] | None): exists = check_frappe_app_exists(appx[0], appx[1]) if not exists["app"]: - raise typer.BadParameter(f"{app} is not a valid FrappeVerse app!") + richprint.stop() + raise typer.BadParameter(f"Invalid app '{appx[0]}'.") if not exists["branch"]: + richprint.stop() raise typer.BadParameter( - f"{appx[1]} is not a valid branch of {appx[0]}!" + f"Invaid branch '{appx[1]}' for '{appx[0]}'." ) - if len(appx) > 2: - raise typer.BadParameter( - "App should be specified in format : or " - ) appx = ":".join(appx) apps_list.append(appx) From 8fcf2c7131fa1e9ae574203c285d3dd402a5cd48 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 26 Feb 2024 14:39:11 +0530 Subject: [PATCH 08/33] Optmize image pull, migration v0.11.0 --- frappe_manager/compose_manager/ComposeFile.py | 80 +++++--- frappe_manager/docker_wrapper/DockerClient.py | 25 +++ .../migration_manager/migration_base.py | 4 + .../migration_manager/migration_exections.py | 1 - .../migration_manager/migration_executor.py | 165 ++++++++++------- .../migrations/migrate_0_10_0.py | 171 +++++++++--------- .../migrations/migrate_0_11_0.py | 143 +++++++++++++++ frappe_manager/migration_manager/version.py | 3 + frappe_manager/site_manager/SiteManager.py | 2 - frappe_manager/site_manager/site.py | 7 +- .../templates/docker-compose.migration.tmpl | 165 +++++++++++++++++ frappe_manager/templates/docker-compose.tmpl | 2 +- 12 files changed, 587 insertions(+), 181 deletions(-) create mode 100644 frappe_manager/migration_manager/migrations/migrate_0_11_0.py create mode 100644 frappe_manager/templates/docker-compose.migration.tmpl diff --git a/frappe_manager/compose_manager/ComposeFile.py b/frappe_manager/compose_manager/ComposeFile.py index 8cc8d071..e0a9602d 100644 --- a/frappe_manager/compose_manager/ComposeFile.py +++ b/frappe_manager/compose_manager/ComposeFile.py @@ -2,11 +2,15 @@ from rich import inspect from ruamel.yaml import YAML import platform -from ruamel.yaml.comments import CommentedMap as OrderedDict, CommentedSeq as OrderedList +from ruamel.yaml.comments import ( + CommentedMap as OrderedDict, + CommentedSeq as OrderedList, +) from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.utils.site import parse_docker_volume from frappe_manager.utils.helpers import represent_null_empty import importlib.resources as pkg_resources +from frappe_manager.migration_manager.version import Version yaml = YAML(typ="rt") yaml.representer.ignore_aliases = lambda *args: True @@ -50,9 +54,7 @@ def get_compose_path(self): """ return self.compose_path - def get_template( - self, file_name: str - ): + def get_template(self, file_name: str): """ Get the file path of a template. @@ -66,9 +68,11 @@ def get_template( try: template_path = f"templates/{file_name}" - return Path(str(pkg_resources.files('frappe_manager').joinpath(template_path))) + return Path( + str(pkg_resources.files("frappe_manager").joinpath(template_path)) + ) except FileNotFoundError as e: - richprint.exit(f"{file_name} template not found.",error_msg=e) + richprint.exit(f"{file_name} template not found.", error_msg=e) def load_template(self): """ @@ -233,9 +237,9 @@ def get_version(self): """ try: compose_version = self.yml["x-version"] - return compose_version + return Version(compose_version) except KeyError: - return '0.0.0' + return Version("0.0.0") def set_version(self, version): """ @@ -475,7 +479,7 @@ def get_all_volumes(self): Get all the root volumes. """ - volumes = self.yml['volumes'] + volumes = self.yml["volumes"] return volumes @@ -489,7 +493,7 @@ def get_all_services_volumes(self): for service in services: try: - volumes_list = self.yml["services"][service]['volumes'] + volumes_list = self.yml["services"][service]["volumes"] for volume in volumes_list: volumes_set.add(volume) except KeyError as e: @@ -502,34 +506,62 @@ def get_all_services_volumes(self): return volumes_list - def set_secret_file_path(self,secret_name,file_path): + def set_secret_file_path(self, secret_name, file_path): try: - self.yml['secrets'][secret_name]['file'] = file_path + self.yml["secrets"][secret_name]["file"] = file_path except KeyError: - richprint.warning("Not able to set secrets in compose.") + richprint.warning("Not able to set secrets in compose.") - def get_secret_file_path(self,secret_name): + def get_secret_file_path(self, secret_name): try: - file_path = self.yml['secrets'][secret_name]['file'] + file_path = self.yml["secrets"][secret_name]["file"] return file_path except KeyError: - richprint.warning("Not able to set secrets in compose.") + richprint.warning("Not able to set secrets in compose.") - def remove_secrets_from_container(self,container): + def remove_secrets_from_container(self, container): try: - del self.yml['services'][container]['secrets'] + del self.yml["services"][container]["secrets"] except KeyError: - richprint.warning(f"Not able to remove secrets from {container}.") + richprint.warning(f"Not able to remove secrets from {container}.") def remove_root_secrets_compose(self): try: - del self.yml['secrets'] + del self.yml["secrets"] except KeyError: - richprint.warning(f"root level secrets not present.") - + richprint.warning(f"root level secrets not present.") def remove_container_user(self, container): try: - del self.yml['services'][container]['user'] + del self.yml["services"][container]["user"] except KeyError: - richprint.warning(f"user not present.") + richprint.warning(f"user not present.") + + def get_all_images(self): + """ + Retrieves all the images for each service in the Compose file. + + Returns: + dict: A dictionary containing the service names as keys and their respective image names and tags as values. + """ + images = {} + for service in self.yml["services"].keys(): + try: + image = self.yml["services"][service]["image"] + name, tag = image.split(":") if ":" in image else (image, "latest") + images[service] = {"name": name, "tag": tag} + except KeyError: + pass + return images + + def set_all_images(self, images: dict): + """ + Sets the image for all services in the ComposeFile. + + Args: + images (dict): A dictionary containing the service names as keys and the image names and tags as values. + """ + for service, image_info in images.items(): + image = f'{image_info["name"]}:{image_info["tag"]}' + if service in self.yml["services"]: + self.yml["services"][service]["image"] = image diff --git a/frappe_manager/docker_wrapper/DockerClient.py b/frappe_manager/docker_wrapper/DockerClient.py index 8eb7914c..87aad302 100644 --- a/frappe_manager/docker_wrapper/DockerClient.py +++ b/frappe_manager/docker_wrapper/DockerClient.py @@ -191,3 +191,28 @@ def run( self.docker_cmd + run_cmd, quiet=stream_only_exit_code, stream=stream ) return iterator + + def pull( + self, + container_name: str, + all_tags: bool = False, + platform: Optional[str] = None, + quiet: bool = False, + stream: bool = False, + stream_only_exit_code: bool = False, + ): + parameters: dict = locals() + + pull_cmd: list[str] = ["pull"] + + remove_parameters = ["stream", "stream_only_exit_code","container_name"] + + pull_cmd += parameters_to_options(parameters, exclude=remove_parameters) + pull_cmd += [container_name] + + iterator = run_command_with_exit_code( + self.docker_cmd + pull_cmd, + quiet=stream_only_exit_code, + stream=stream, + ) + return iterator diff --git a/frappe_manager/migration_manager/migration_base.py b/frappe_manager/migration_manager/migration_base.py index 416792d8..0bc95993 100644 --- a/frappe_manager/migration_manager/migration_base.py +++ b/frappe_manager/migration_manager/migration_base.py @@ -17,6 +17,10 @@ def init(self): self.backup_manager = BackupManager(str(self.version)) # Assign the value to backup_manager self.logger = log.get_logger() + def get_rollback_version(self): + + return self.version + def up(self): pass diff --git a/frappe_manager/migration_manager/migration_exections.py b/frappe_manager/migration_manager/migration_exections.py index 19c1fad1..d2d7d903 100644 --- a/frappe_manager/migration_manager/migration_exections.py +++ b/frappe_manager/migration_manager/migration_exections.py @@ -1,4 +1,3 @@ - class MigrationExceptionInSite(Exception): def __init__( self, diff --git a/frappe_manager/migration_manager/migration_executor.py b/frappe_manager/migration_manager/migration_executor.py index c0605ebf..671e6536 100644 --- a/frappe_manager/migration_manager/migration_executor.py +++ b/frappe_manager/migration_manager/migration_executor.py @@ -1,17 +1,21 @@ import shutil -import typer +from typing import Optional import importlib import pkgutil from pathlib import Path -from frappe_manager import CLI_SITES_ARCHIVE +from rich.prompt import Prompt +from frappe_manager import CLI_DIR, CLI_SITES_ARCHIVE from frappe_manager.metadata_manager import MetadataManager -from frappe_manager.migration_manager.migration_exections import MigrationExceptionInSite +from frappe_manager.migration_manager.migration_exections import ( + MigrationExceptionInSite, +) from frappe_manager.utils.helpers import downgrade_package, get_current_fm_version from frappe_manager.logger import log from frappe_manager.migration_manager.version import Version from frappe_manager.display_manager.DisplayManager import richprint -class MigrationExecutor(): + +class MigrationExecutor: """ Migration executor class. @@ -21,8 +25,9 @@ class MigrationExecutor(): def __init__(self): self.metadata_manager = MetadataManager() self.prev_version = self.metadata_manager.get_version() + self.rollback_version = self.metadata_manager.get_version() self.current_version = Version(get_current_fm_version()) - self.migrations_path = Path(__file__).parent / 'migrations' + self.migrations_path = Path(__file__).parent / "migrations" self.logger = log.get_logger() self.migrations = [] self.undo_stack = [] @@ -41,21 +46,28 @@ def execute(self): current_migration = None # Dynamically import all modules in the 'migrations' subfolder - for (_, name, _) in pkgutil.iter_modules([str(self.migrations_path)]): + for _, name, _ in pkgutil.iter_modules([str(self.migrations_path)]): try: - module = importlib.import_module(f'.migrations.{name}', __package__) + module = importlib.import_module(f".migrations.{name}", __package__) for attr_name in dir(module): attr = getattr(module, attr_name) - if isinstance(attr, type) and hasattr(attr, 'up') and hasattr(attr, 'down') and hasattr(attr, 'set_migration_executor'): + if ( + isinstance(attr, type) + and hasattr(attr, "up") + and hasattr(attr, "down") + and hasattr(attr, "set_migration_executor") + ): migration = attr() - migration.set_migration_executor(migration_executor = self) + migration.set_migration_executor(migration_executor=self) current_migration = migration - if migration.version > self.prev_version and migration.version <= self.current_version: + if ( + migration.version > self.prev_version + and migration.version <= self.current_version + ): # if not migration.skip: self.migrations.append(migration) # else: - except Exception as e: print(f"Failed to register migration {name}: {e}") @@ -67,28 +79,30 @@ def execute(self): for migration in self.migrations: richprint.print(f"[bold]MIGRATION:[/bold] v{migration.version}") - richprint.print( - "This may take some time.", emoji_code=":light_bulb:" - ) + richprint.print("This may take some time.", emoji_code=":light_bulb:") richprint.print( - "Manual migration guide can be found here -> https://github.com/rtCamp/Frappe-Manager/wiki/Migrations#manual-migration-procedure", emoji_code=":light_bulb:" + "Manual migration guide -> https://github.com/rtCamp/Frappe-Manager/wiki/Migrations#manual-migration-procedure", + emoji_code=":light_bulb:", ) - migrate_msg =( - "\nIF [y]: Start Migration." - "\nIF [N]: Don't migrate and revert to previous fm version." - "\nDo you want to migrate ?" + migrate_msg = ( + "\n[blue]yes[/blue] : Start Migration." + "\n[blue]no[/blue] : Don't migrate and revert to previous fm version." + "\nDo you want to migrate ?" ) # prompt richprint.stop() - continue_migration = typer.confirm(migrate_msg) + continue_migration = Prompt.ask(migrate_msg, choices=["yes", "no"]) - if not continue_migration: - downgrade_package('frappe-manager',str(self.prev_version.version)) - richprint.exit(f'Successfully installed [bold][blue]Frappe-Manager[/blue][/bold] version: v{str(self.prev_version.version)}',emoji_code=':white_check_mark:') + if continue_migration == "no": + downgrade_package("frappe-manager", str(self.prev_version.version)) + richprint.exit( + f"Successfully installed [bold][blue]Frappe-Manager[/blue][/bold] version: v{str(self.prev_version.version)}", + emoji_code=":white_check_mark:", + ) - richprint.start('Working') + richprint.start("Working") rollback = False archive = False @@ -96,12 +110,23 @@ def execute(self): try: # run all the migrations for migration in self.migrations: - richprint.change_head(f"Running migration introduced in v{migration.version}") + richprint.change_head( + f"Running migration introduced in v{migration.version}" + ) self.logger.info(f"[{migration.version}] : Migration starting") try: self.undo_stack.append(migration) + migration.up() - self.prev_version = migration.version + + if not self.rollback_version > migration.version: + self.rollback_version = migration.get_rollback_version() + + except MigrationExceptionInSite as e: + self.logger.error(f"[{migration.version}] : Migration Failed\n{e}") + if migration.version < self.migrations[-1].version: + continue + raise e except Exception as e: self.logger.error(f"[{migration.version}] : Migration Failed\n{e}") @@ -110,71 +135,87 @@ def execute(self): except MigrationExceptionInSite as e: richprint.stop() if self.migrate_sites: - richprint.print("[green]Migration was successfull on these sites.[/green]") + richprint.print( + "[green]Migration was successfull on these sites.[/green]" + ) - for site, exception in self.migrate_sites.items(): - if not exception: - richprint.print(f"[bold][green]SITE:[/green][/bold] {site.name}") + for site, site_status in self.migrate_sites.items(): + if not site_status["exception"]: + richprint.print(f"[bold][green]SITE:[/green][/bold] {site}") richprint.print("[red]Migration failed on these sites[/red]") - for site, exception in self.migrate_sites.items(): - if exception: - richprint.print(f"[bold][red]SITE[/red]:[/bold] {site.name}") - richprint.print(f"[bold][red]EXCEPTION[/red]:[/bold] {exception}") - - richprint.print(f"More details about the error can be found in the log -> `~/frappe/logs/fm.log`") + for site, site_status in self.migrate_sites.items(): + if site_status["exception"]: + richprint.print(f"[bold][red]SITE[/red]:[/bold] {site}") + richprint.print( + f"[bold][red]FAILED MIGRATION VERSION[/red]:[/bold] {site_status['last_migration_version']}" + ) + richprint.print( + f"[bold][red]EXCEPTION[/red]:[/bold] {site_status['exception']}" + ) + + richprint.print( + f"More error details can be found in the log -> '{CLI_DIR}/logs/fm.log'" + ) - archive_msg =( - f"\nIF [y]: Sites that have failed will be rolled back and stored in {CLI_SITES_ARCHIVE}." - "\nIF [N]: Revert the entire migration to the previous fm version." - "\nDo you wish to archive all sites that failed during migration?" + archive_msg = ( + f"\n[blue]yes[/blue] : Sites that have failed will be rolled back and stored in '{CLI_SITES_ARCHIVE}'." + "\n[blue]no[/blue] : Revert the entire migration to the previous fm version." + "\nDo you wish to archive all sites that failed during migration?" ) - from rich.text import Text - archive = typer.confirm(archive_msg) + archive = Prompt.ask(archive_msg, choices=["yes", "no"]) - if not archive: + if archive == "no": rollback = True except Exception as e: richprint.print(f"Migration failed: {e}") rollback = True - - if archive: + if archive == "yes": self.prev_version = self.undo_stack[-1].version - for site, exception in self.migrate_sites.items(): - if exception: - archive_site_path = CLI_SITES_ARCHIVE / site.name + for site, site_info in self.migrate_sites.items(): + if site_info["exception"]: + archive_site_path = CLI_SITES_ARCHIVE / site CLI_SITES_ARCHIVE.mkdir(exist_ok=True, parents=True) - shutil.move(site.path,archive_site_path ) - richprint.print(f"[bold]Archived site:[/bold] {site.name}") + shutil.move(site_info["object"].path, archive_site_path) + richprint.print(f"[bold]Archived site:[/bold] {site}") if rollback: - richprint.start('Rollback') + richprint.start("Rollback") self.rollback() richprint.stop() - self.metadata_manager.set_version(self.prev_version) + self.metadata_manager.set_version(self.rollback_version) self.metadata_manager.save() - richprint.print(f"Installing [bold][blue]Frappe-Manager[/blue][/bold] version: v{str(self.prev_version.version)}") - downgrade_package('frappe-manager',str(self.prev_version.version)) + richprint.print( + f"Installing [bold][blue]Frappe-Manager[/blue][/bold] version: v{str(self.rollback_version.version)}" + ) + downgrade_package("frappe-manager", str(self.rollback_version.version)) richprint.exit("Rollback complete.") self.metadata_manager.set_version(self.current_version) self.metadata_manager.save() - return True - def set_site_data(self,site,data = None): - self.migrate_sites[site] = data + def set_site_data( + self, site, exception=None, migration_version: Optional[Version] = None + ): + self.migrate_sites[site.name] = { + "object": site, + "exception": exception, + "last_migration_version": migration_version, + } - def get_site_data(self,site): + def get_site_data(self, site_name): try: - data = self.migrate_sites[site] + data = self.migrate_sites[site_name] except KeyError as e: return None + return data + def rollback(self): """ Rollback the migration. @@ -184,8 +225,10 @@ def rollback(self): # run all the migrations for migration in reversed(self.undo_stack): - if migration.version > self.prev_version: - richprint.change_head(f"Rolling back migration introduced in v{migration.version}") + if migration.version > self.rollback_version: + richprint.change_head( + f"Rolling back migration introduced in v{migration.version}" + ) self.logger.info(f"[{migration.version}] : Rollback starting") try: migration.down() diff --git a/frappe_manager/migration_manager/migrations/migrate_0_10_0.py b/frappe_manager/migration_manager/migrations/migrate_0_10_0.py index e74cc47d..2c635946 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_10_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_10_0.py @@ -1,15 +1,13 @@ -from typing import Optional -from dataclasses import dataclass import shutil - from copy import deepcopy - -import importlib +from pathlib import Path from frappe_manager.docker_wrapper import DockerException from frappe_manager.migration_manager.backup_manager import BackupData from frappe_manager.migration_manager.migration_base import MigrationBase -from frappe_manager import CLI_DIR, services_manager -from frappe_manager.migration_manager.migration_exections import MigrationExceptionInSite +from frappe_manager import CLI_DIR +from frappe_manager.migration_manager.migration_exections import ( + MigrationExceptionInSite, +) from frappe_manager.migration_manager.migration_executor import MigrationExecutor from frappe_manager.services_manager.services import ServicesManager from frappe_manager.site_manager.site_exceptions import ( @@ -23,7 +21,8 @@ from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.utils.helpers import get_container_name_prefix from frappe_manager.migration_manager.version import Version -from pathlib import Path +from datetime import datetime + class MigrationV0100(MigrationBase): version = Version("0.10.0") @@ -32,12 +31,16 @@ def __init__(self): super().init() self.sites_dir = CLI_DIR / "sites" self.services_manager = ServicesManager(verbose=False) + self.string_timestamp = datetime.now().strftime("%d-%b-%y--%H-%M-%S") + + def get_rollback_version(self): + # this was without any migrations + return Version("0.10.1") def set_migration_executor(self, migration_executor: MigrationExecutor): self.migration_executor = migration_executor def up(self): - # richprint.print(f"Started",prefix=f"[ Migration v{str(self.version)} ] : ") richprint.print(f"Started", prefix=f"[bold]v{str(self.version)}:[/bold] ") self.logger.info("-" * 40) @@ -56,30 +59,32 @@ def up(self): main_error = False for site_name, site_path in sites.items(): - site = Site(name=site_name, path=site_path.parent) - self.migration_executor.set_site_data(site) + self.migration_executor.set_site_data(site, migration_version=self.version) try: self.migrate_site(site) except Exception as e: import traceback + traceback_str = traceback.format_exc() self.logger.error(f"[ EXCEPTION TRACEBACK ]:\n {traceback_str}") richprint.update_live() main_error = True - self.migration_executor.set_site_data(site,e) + self.migration_executor.set_site_data(site, e, self.version) self.undo_site_migrate(site) - site.down(volumes=False,timeout=5) + site.down(volumes=False, timeout=5) if main_error: - raise MigrationExceptionInSite('') + raise MigrationExceptionInSite("") # new bind mount is introudced so create it richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)}:[/bold] ") self.logger.info("-" * 40) - def migrate_site(self,site): - richprint.print(f"Migrating site {site.name}", prefix=f"[bold]v{str(self.version)}:[/bold] ") + def migrate_site(self, site): + richprint.print( + f"Migrating site {site.name}", prefix=f"[bold]v{str(self.version)}:[/bold] " + ) # backup docker compose.yml self.backup_manager.backup( @@ -100,34 +105,16 @@ def migrate_site(self,site): self.migrate_site_compose(site) - #recreate_env_cmd = 'python -m venv env && env/bin/python -m pip install --quiet --upgrade pip && env/bin/python -m pip install --quiet wheel' - # recreat env - # output = site.docker.compose.run( - # 'frappe', - # entrypoint=recreate_env_cmd, - # stream=True, - # stream_only_exit_code=True, - # ) - site.down(volumes=False) site_db_info = site.get_site_db_info() - site_db_name = site_db_info['name'] - site_db_user = site_db_info['user'] - site_db_pass = site_db_info['password'] - - self.services_manager.add_user(site_db_name,site_db_user,site_db_pass) - - # service = 'mariadb' - # services_list = [] - # services_list.append(service) - # output = site.docker.compose.up(services=services_list,stream=True) - # richprint.live_lines(output, padding=(0, 0, 0, 2)) + site_db_name = site_db_info["name"] + site_db_user = site_db_info["user"] + site_db_pass = site_db_info["password"] - # site.add_user('mariadb','root','root',force=True) + self.services_manager.add_user(site_db_name, site_db_user, site_db_pass) def down(self): - # richprint.print(f"Started",prefix=f"[ Migration v{str(self.version)} ][ROLLBACK] : ") richprint.print( f"Started", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] " ) @@ -142,7 +129,7 @@ def down(self): sites = sites_manager.get_all_sites() # undo each site - for site, exception in self.migration_executor.migrate_sites.items(): + for site, exception in self.migration_executor.migrate_sites.items(): if not exception: self.undo_site_migrate(site) @@ -154,13 +141,12 @@ def down(self): ) self.logger.info("-" * 40) - def undo_site_migrate(self,site): - + def undo_site_migrate(self, site): for backup in self.backup_manager.backups: if backup.site == site.name: self.backup_manager.restore(backup, force=True) - configs_backup = site.path / "configs.bak" + configs_backup = site.path / f"configs-{self.string_timestamp}.bak" configs_path = site.path / "configs" @@ -169,6 +155,8 @@ def undo_site_migrate(self,site): if configs_backup.exists(): shutil.copytree(configs_backup, configs_path) + shutil.rmtree(configs_backup) + self.logger.info(f"Removed : {configs_backup}") service = "mariadb" services_list = [] @@ -187,17 +175,15 @@ def undo_site_migrate(self,site): except DockerException as e: pass - self.logger.info(f'Undo successfull for site: {site.name}') + self.logger.info(f"Undo successfull for site: {site.name}") def migrate_site_compose(self, site: Site): - - richprint.change_head('Migrating database') + richprint.change_head("Migrating database") compose_version = site.composefile.get_version() - fm_version = importlib.metadata.version("frappe-manager") if not site.composefile.exists(): richprint.print( - f"{status_msg} {compose_version} -> {fm_version}: Failed " + f"{status_msg} {compose_version} -> {self.version.version}: Failed " ) return @@ -205,11 +191,13 @@ def migrate_site_compose(self, site: Site): db_backup_file = self.db_migration_export(site) # backup site_db - db_backup = self.backup_manager.backup(db_backup_file, site_name=site.name,allow_restore=False) + db_backup = self.backup_manager.backup( + db_backup_file, site_name=site.name, allow_restore=False + ) self.db_migration_import(site=site, db_backup_file=db_backup) - status_msg = 'Migrating site compose' + status_msg = "Migrating site compose" richprint.change_head(status_msg) # get all the payloads @@ -226,7 +214,7 @@ def migrate_site_compose(self, site: Site): envs["nginx"]["VIRTUAL_HOST"] = site.name - envs["adminer"] ={"ADMINER_DEFAULT_SERVER":"global-db"} + envs["adminer"] = {"ADMINER_DEFAULT_SERVER": "global-db"} import os @@ -243,6 +231,8 @@ def migrate_site_compose(self, site: Site): self.create_compose_dirs(site) + site.composefile.template_name = "docker-compose.migration.tmpl" + # load template site.composefile.yml = site.composefile.load_template() @@ -258,19 +248,17 @@ def migrate_site_compose(self, site: Site): site.composefile.set_network_alias("nginx", "site-network", [site.name]) site.composefile.set_container_names(get_container_name_prefix(site.name)) - fm_version = importlib.metadata.version("frappe-manager") - - site.composefile.set_version(fm_version) + site.composefile.set_version(str(self.version)) site.composefile.set_top_networks_name( "site-network", get_container_name_prefix(site.name) ) site.composefile.write_to_file() # change the node socketio port - site.common_site_config_set({"socketio_port":"80"}) + site.common_site_config_set({"socketio_port": "80"}) richprint.print( - f"{status_msg} {compose_version} -> {fm_version}: Done" + f"{status_msg} {compose_version} -> {self.version.version}: Done" ) return db_backup @@ -282,7 +270,8 @@ def create_compose_dirs(self, site): # custom config directory found moving it # check if config directory exits if exists then move it if configs_path.exists(): - shutil.move(configs_path, configs_path.parent / f"{configs_path.name}.bak") + backup_path = f"{configs_path.absolute()}.{self.string_timestamp}.bak" + shutil.move(configs_path, backup_path) configs_path.mkdir(parents=True, exist_ok=True) @@ -305,16 +294,25 @@ def create_compose_dirs(self, site): docker=site.docker, ) - # raise Exception("Migration not implemented") - nginx_subdirs = ["logs", "cache", "run"] for directory in nginx_subdirs: new_dir = nginx_dir / directory new_dir.mkdir(parents=True, exist_ok=True) - def is_database_started(self, docker_object, db_user='root', db_password='root',db_host='127.0.0.1', service="mariadb", interval=5, timeout=30): + def is_database_started( + self, + site_name, + docker_object, + db_user="root", + db_password="root", + db_host="127.0.0.1", + service="mariadb", + interval=5, + timeout=30, + ): import time + i = 0 check_connection_command = f"/usr/bin/mariadb -h{db_host} -u{db_user} -p'{db_password}' -e 'SHOW DATABASES;'" @@ -339,7 +337,7 @@ def is_database_started(self, docker_object, db_user='root', db_password='root', i += 1 if not connected: - raise SiteDatabaseStartTimeout(f"Not able to start db: {error}") + raise SiteDatabaseStartTimeout(site_name, f"Not able to start db: {error}") def db_migration_export(self, site) -> Path: self.logger.debug("[db export] site: %s", site.name) @@ -351,11 +349,12 @@ def db_migration_export(self, site) -> Path: output = site.docker.compose.up( services=["mariadb", "frappe"], detach=True, pull="missing", stream=True ) + richprint.live_lines(output, padding=(0, 0, 0, 2)) self.logger.debug("[db export] checking if mariadb started") - self.is_database_started(site.docker) + self.is_database_started(site.name, site.docker) # create dir to store migration db_migration_dir_path = site.path / "workspace" / "migrations" @@ -372,11 +371,8 @@ def db_migration_export(self, site) -> Path: site_db_info = site.get_site_db_info() site_db_name = site_db_info["name"] - # site_db_user = site_db_info['user'] - # site_db_pass = site_db_info['password'] db_backup_command = f"mysqldump -uroot -proot -h'mariadb' -P3306 {site_db_name} --result-file={db_migration_file_path}" # db_backup_command = f"mysqldump -uroot -proot -h'mariadb' -p3306 {site_db_name} {db_migration_file_path}" - # db_backup_command = f"/opt/.pyenv/shims/bench --site {site.name} backup --backup-path-db {db_migration_file_path}" # backup the db output_backup_db = site.docker.compose.exec( @@ -388,17 +384,6 @@ def db_migration_export(self, site) -> Path: stream_only_exit_code=True, ) - # gunzip_command = f"gunzip {db_migration_file_path}" - - # output_gunzip = site.docker.compose.exec( - # "frappe", - # command=gunzip_command, - # stream=True, - # workdir='/workspace/migrations', - # user='frappe', - # stream_only_exit_code=True - # ) - output_stop = site.docker.compose.stop(timeout=10, stream=True) site_db_migration_file_path = Path(site.path / db_migration_file_path[1:]) @@ -409,8 +394,9 @@ def db_migration_export(self, site) -> Path: raise SiteDatabaseExport(site.name, f"Error while exporting db: {e}") def db_migration_import(self, site: Site, db_backup_file: BackupData): - - self.logger.info(f"[database import: global-db] {site.name} -> {db_backup_file}") + self.logger.info( + f"[database import: global-db] {site.name} -> {db_backup_file}" + ) # cp into the global contianer self.services_manager.docker.compose.cp( @@ -433,9 +419,17 @@ def db_migration_import(self, site: Site, db_backup_file: BackupData): mariadb_command = f"/usr/bin/mariadb -u{services_db_user} -p'{services_db_pass}' -h'{services_db_host}' -P3306 -e " mariadb = f"/usr/bin/mariadb -u{services_db_user} -p'{services_db_pass}' -h'{services_db_host}' -P3306" - self.is_database_started(self.services_manager.docker, service='global-db',db_user=services_db_user,db_password=services_db_pass) + self.is_database_started( + site.name, + self.services_manager.docker, + service="global-db", + db_user=services_db_user, + db_password=services_db_pass, + ) - db_add_database = mariadb_command + f"'CREATE DATABASE IF NOT EXISTS `{site_db_name}`';" + db_add_database = ( + mariadb_command + f"'CREATE DATABASE IF NOT EXISTS `{site_db_name}`';" + ) output_add_db = self.services_manager.docker.compose.exec( "global-db", @@ -444,7 +438,6 @@ def db_migration_import(self, site: Site, db_backup_file: BackupData): stream_only_exit_code=True, ) - db_remove_user = mariadb_command + f"'DROP USER `{site_db_user}`@`%`;'" error = None @@ -464,8 +457,10 @@ def db_migration_import(self, site: Site, db_backup_file: BackupData): removed = True if removed: - - db_add_user = mariadb_command + f"'CREATE USER `{site_db_user}`@`%` IDENTIFIED BY \"{site_db_pass}\";'" + db_add_user = ( + mariadb_command + + f"'CREATE USER `{site_db_user}`@`%` IDENTIFIED BY \"{site_db_pass}\";'" + ) output_add_user_db = self.services_manager.docker.compose.exec( "global-db", @@ -474,7 +469,10 @@ def db_migration_import(self, site: Site, db_backup_file: BackupData): stream_only_exit_code=True, ) - db_grant_user = mariadb_command + f"'GRANT ALL PRIVILEGES ON `{site_db_name}`.* TO `{site_db_user}`@`%`;'" + db_grant_user = ( + mariadb_command + + f"'GRANT ALL PRIVILEGES ON `{site_db_name}`.* TO `{site_db_user}`@`%`;'" + ) output_grant_user_db = self.services_manager.docker.compose.exec( "global-db", @@ -483,7 +481,9 @@ def db_migration_import(self, site: Site, db_backup_file: BackupData): stream_only_exit_code=True, ) - db_import_command = mariadb + f" {site_db_name} -e 'source /tmp/{db_backup_file.src.name}'" + db_import_command = ( + mariadb + f" {site_db_name} -e 'source /tmp/{db_backup_file.src.name}'" + ) output_import_db = self.services_manager.docker.compose.exec( "global-db", @@ -492,7 +492,7 @@ def db_migration_import(self, site: Site, db_backup_file: BackupData): stream_only_exit_code=True, ) - check_connection_command = mariadb_command + f"'SHOW DATABASES;'" + check_connection_command = mariadb_command + f"'SHOW DATABASES;'" output_check_db = self.services_manager.docker.compose.exec( "global-db", @@ -504,4 +504,3 @@ def db_migration_import(self, site: Site, db_backup_file: BackupData): raise SiteDatabaseAddUserException( site.name, f"Database user creation failed: {error}" ) - diff --git a/frappe_manager/migration_manager/migrations/migrate_0_11_0.py b/frappe_manager/migration_manager/migrations/migrate_0_11_0.py new file mode 100644 index 00000000..56bcb9c2 --- /dev/null +++ b/frappe_manager/migration_manager/migrations/migrate_0_11_0.py @@ -0,0 +1,143 @@ +import importlib +from frappe_manager.migration_manager.migration_base import MigrationBase +from frappe_manager.migration_manager.migration_exections import MigrationExceptionInSite +from frappe_manager.migration_manager.migration_executor import MigrationExecutor +from frappe_manager.services_manager.services import ServicesManager +from frappe_manager.site_manager.SiteManager import Site +from frappe_manager.site_manager.SiteManager import SiteManager +from frappe_manager.display_manager.DisplayManager import richprint +from frappe_manager.migration_manager.version import Version +from frappe_manager import CLI_DIR + +class MigrationV0110(MigrationBase): + version = Version("0.11.0") + + def __init__(self): + super().init() + self.sites_dir = CLI_DIR / "sites" + self.services_manager = ServicesManager(verbose=False) + + def set_migration_executor(self, migration_executor: MigrationExecutor): + self.migration_executor = migration_executor + + def up(self): + richprint.print(f"Started", prefix=f"[bold]v{str(self.version)}:[/bold] ") + self.logger.info("-" * 40) + + # take backup of each of the site docker compose + sites_manager = SiteManager(self.sites_dir) + sites_manager.stop_sites() + sites = sites_manager.get_all_sites() + + # migrate each site + main_error = False + + for site_name, site_path in sites.items(): + site = Site(name=site_name, path=site_path.parent) + if site.name in self.migration_executor.migrate_sites.keys(): + site_info = self.migration_executor.migrate_sites[site.name] + if site_info['exception']: + richprint.print(f"Skipping migration for failed site {site.name}.") + main_error = True + continue + + self.migration_executor.set_site_data(site,migration_version=self.version) + try: + self.migrate_site(site) + except Exception as e: + import traceback + traceback_str = traceback.format_exc() + self.logger.error(f"{site.name} [ EXCEPTION TRACEBACK ]:\n {traceback_str}") + richprint.update_live() + main_error = True + self.migration_executor.set_site_data(site, e, self.version) + self.undo_site_migrate(site) + site.down(volumes=False,timeout=5) + + if main_error: + raise MigrationExceptionInSite('') + + richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)}:[/bold] ") + self.logger.info("-" * 40) + + def migrate_site(self,site): + richprint.print(f"Migrating site {site.name}", prefix=f"[bold]v{str(self.version)}:[/bold] ") + + # backup docker compose.yml + self.backup_manager.backup( + site.path / "docker-compose.yml", site_name=site.name + ) + + # backup common_site_config.json + self.backup_manager.backup( + site.path + / "workspace" + / "frappe-bench" + / "sites" + / "common_site_config.json", + site_name=site.name, + ) + + site.down(volumes=False) + self.migrate_site_compose(site) + + def down(self): + # richprint.print(f"Started",prefix=f"[ Migration v{str(self.version)} ][ROLLBACK] : ") + richprint.print( + f"Started", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] " + ) + self.logger.info("-" * 40) + + # undo each site + for site, exception in self.migration_executor.migrate_sites.items(): + if not exception: + self.undo_site_migrate(site) + + for backup in self.backup_manager.backups: + self.backup_manager.restore(backup, force=True) + + richprint.print( + f"Successfull", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] " + ) + self.logger.info("-" * 40) + + def undo_site_migrate(self,site): + + for backup in self.backup_manager.backups: + if backup.site == site.name: + self.backup_manager.restore(backup, force=True) + + self.logger.info(f'Undo successfull for site: {site.name}') + + def migrate_site_compose(self, site: Site): + + status_msg = 'Migrating site compose' + richprint.change_head(status_msg) + + compose_version = site.composefile.get_version() + fm_version = importlib.metadata.version("frappe-manager") + + if not site.composefile.exists(): + richprint.print(f"{status_msg} {compose_version} -> {fm_version}: Failed ") + raise MigrationExceptionInSite(f"{site.composefile.compose_path} not found.") + + # change image tag to the latest + # in this migration only tag of frappe container is changed + images_info = site.composefile.get_all_images() + image_info = images_info['frappe'] + + # get v0.11.0 frappe image + image_info['tag'] = self.version.version_string() + image_info['name'] = 'ghcr.io/rtcamp/frappe-manager-frappe' + + output = site.docker.pull(container_name=f"{image_info['name']}:{image_info['tag']}", stream=True) + richprint.live_lines(output, padding=(0, 0, 0, 2)) + + site.composefile.set_all_images(images_info) + + site.composefile.set_version(str(self.version)) + site.composefile.write_to_file() + + richprint.print( + f"{status_msg} {compose_version} -> {fm_version}: Done" + ) diff --git a/frappe_manager/migration_manager/version.py b/frappe_manager/migration_manager/version.py index b848713e..d0fb5a9c 100644 --- a/frappe_manager/migration_manager/version.py +++ b/frappe_manager/migration_manager/version.py @@ -26,3 +26,6 @@ def __gt__(self, other): def __str__(self): return self.version + + def version_string(self): + return f"v{self.version}" diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index d87a4249..d9d5d5c3 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -125,7 +125,6 @@ def create_site(self, template_inputs: dict,template_site: bool = False): richprint.change_head(f"Generating Compose") self.site.generate_compose(template_inputs) self.site.create_compose_dirs() - self.site.pull() if template_site: self.site.remove_secrets() @@ -233,7 +232,6 @@ def start_site(self): Starts the site. """ #self.migrate_site() - self.site.pull() self.site.sync_site_common_site_config() self.site.start() self.site.frappe_logs_till_start(status_msg="Starting Site") diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index c3a6095e..40dd32f5 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -3,13 +3,8 @@ import shutil import re import json -from typing import List, Type from pathlib import Path -from rich import inspect - -from rich.table import Table from frappe_manager.docker_wrapper import DockerClient, DockerException - from frappe_manager.compose_manager.ComposeFile import ComposeFile from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.site_manager.site_exceptions import ( @@ -35,9 +30,9 @@ def init(self): The function checks if the Docker daemon is running and exits with an error message if it is not. """ self.composefile = ComposeFile(self.path / "docker-compose.yml") + self.docker = DockerClient(compose_file_path=self.composefile.compose_path) self.workers = SiteWorkers(self.path, self.name, self.quiet) - # remove this from init if self.workers.exists(): if not self.workers.running(): diff --git a/frappe_manager/templates/docker-compose.migration.tmpl b/frappe_manager/templates/docker-compose.migration.tmpl new file mode 100644 index 00000000..03e16bcf --- /dev/null +++ b/frappe_manager/templates/docker-compose.migration.tmpl @@ -0,0 +1,165 @@ +version: "3.9" +services: + frappe: + image: ghcr.io/rtcamp/frappe-manager-frappe:v0.10.0 + container_name: REPLACE_ME_WITH_CONTAINER_NAME + restart: always + environment: + ADMIN_PASS: REPLACE_me_with_frappe_web_admin_pass + # apps are defined as :, if branch name not given then default github branch will be used. + APPS_LIST: REPLACE_ME_APPS_LIST + DB_NAME: REPLACE_ME_WITH_DB_NAME_TO_CREATE + # DEVERLOPER_MODE bool -> true/false + DEVELOPER_MODE: REPLACE_ME_WITH_DEVELOPER_MODE_TOGGLE + FRAPPE_BRANCH: REPLACE_ME_WITH_BRANCH_OF_FRAPPE + MARIADB_ROOT_PASS: REPLACE_ME_WITH_DB_ROOT_PASSWORD + SITENAME: REPLACE_ME_WITH_THE_SITE_NAME + USERGROUP: REPLACE_ME_WITH_CURRENT_USER_GROUP + USERID: REPLACE_ME_WITH_CURRENT_USER + MARIADB_HOST: REPLACE_ME_WITH_DB_HOST + volumes: + - ./workspace:/workspace:cached + expose: + - 80 + labels: + devcontainer.metadata: '[{ "remoteUser": "frappe"}]' + networks: + site-network: + global-backend-network: + secrets: + - db_root_password + + nginx: + image: ghcr.io/rtcamp/frappe-manager-nginx:v0.10.0 + container_name: REPLACE_ME_WITH_CONTAINER_NAME + user: REPLACE_ME_WITH_CURRENT_USER:REPLACE_ME_WITH_CURRENT_USER_GROUP + restart: always + environment: + # not implemented as of now + ENABLE_SSL: REPLACE_ME_WITH_TOGGLE_ENABLE_SSL + SITENAME: REPLACE_ME_WITH_THE_SITE_NAME + # for nginx-proxy + VIRTUAL_HOST: REPLACE_ME_WITH_SITE_NAME + VIRTUAL_PORT: 80 + volumes: + - ./workspace:/workspace:cached + - ./configs/nginx/conf:/etc/nginx + - ./configs/nginx/logs:/var/log/nginx + - ./configs/nginx/cache:/var/cache/nginx + - ./configs/nginx/run:/var/run + expose: + - 80 + networks: + site-network: + global-frontend-network: + + mailhog: + image: ghcr.io/rtcamp/frappe-manager-mailhog:v0.8.3 + container_name: REPLACE_ME_WITH_CONTAINER_NAME + restart: always + expose: + - 1025 + - 8025 + networks: + site-network: + + adminer: + image: adminer:latest + container_name: REPLACE_ME_WITH_CONTAINER_NAME + restart: always + environment: + ADMINER_DEFAULT_SERVER: global-db + expose: + - 8080 + networks: + site-network: + global-backend-network: + + socketio: + image: ghcr.io/rtcamp/frappe-manager-frappe:v0.10.0 + container_name: REPLACE_ME_WITH_CONTAINER_NAME + restart: always + environment: + TIMEOUT: 60000 + CHANGE_DIR: /workspace/frappe-bench/logs + WAIT_FOR: '/workspace/frappe-bench/config/frappe-bench-node-socketio.fm.supervisor.conf' + COMMAND: | + ln -sfn /workspace/frappe-bench/config/frappe-bench-node-socketio.fm.supervisor.conf /opt/user/conf.d/frappe-bench-node-socketio.fm.supervisor.conf + supervisord -c /opt/user/supervisord.conf + expose: + - 80 + command: launch.sh + volumes: + - ./workspace:/workspace:cached + networks: + site-network: + + schedule: + image: ghcr.io/rtcamp/frappe-manager-frappe:v0.10.0 + container_name: REPLACE_ME_WITH_CONTAINER_NAME + restart: always + environment: + TIMEOUT: 60000 + CHANGE_DIR: /workspace/frappe-bench + WAIT_FOR: '/workspace/frappe-bench/config/frappe-bench-frappe-schedule.fm.supervisor.conf' + COMMAND: | + ln -sfn /workspace/frappe-bench/config/frappe-bench-frappe-schedule.fm.supervisor.conf /opt/user/conf.d/frappe-bench-frappe-schedule.fm.supervisor.conf + supervisord -c /opt/user/supervisord.conf + command: launch.sh + volumes: + - ./workspace:/workspace:cached + networks: + site-network: + global-backend-network: + + redis-cache: + image: redis:alpine + container_name: REPLACE_ME_WITH_CONTAINER_NAME + restart: always + volumes: + - redis-cache-data:/data + expose: + - 6379 + networks: + site-network: + + redis-queue: + image: redis:alpine + container_name: REPLACE_ME_WITH_CONTAINER_NAME + restart: always + volumes: + - redis-queue-data:/data + expose: + - 6379 + networks: + site-network: + + redis-socketio: + image: redis:alpine + container_name: REPLACE_ME_WITH_CONTAINER_NAME + restart: always + volumes: + - redis-socketio-data:/data + expose: + - 6379 + networks: + site-network: + +volumes: + redis-socketio-data: + redis-queue-data: + redis-cache-data: + +networks: + site-network: + name: REPLACE_ME_WITH_SITE_NAME_NETWORK + global-frontend-network: + name: fm-global-frontend-network + external: true + global-backend-network: + name: fm-global-backend-network + external: true + +secrets: + db_root_password: + file: REPLACE_ME_WITH_SECERETS_PATH diff --git a/frappe_manager/templates/docker-compose.tmpl b/frappe_manager/templates/docker-compose.tmpl index 03e16bcf..cc13ed81 100644 --- a/frappe_manager/templates/docker-compose.tmpl +++ b/frappe_manager/templates/docker-compose.tmpl @@ -1,7 +1,7 @@ version: "3.9" services: frappe: - image: ghcr.io/rtcamp/frappe-manager-frappe:v0.10.0 + image: ghcr.io/rtcamp/frappe-manager-frappe:v0.11.0 container_name: REPLACE_ME_WITH_CONTAINER_NAME restart: always environment: From e41ee207871a14f2045c162a359d878b50229556 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 26 Feb 2024 18:10:40 +0530 Subject: [PATCH 09/33] feat: `fm self` command group --- frappe_manager/commands.py | 2 + .../display_manager/DisplayManager.py | 7 +- .../migration_manager/migration_executor.py | 6 +- frappe_manager/sub_commands/__init__.py | 0 frappe_manager/sub_commands/self_commands.py | 5 ++ frappe_manager/sub_commands/update_command.py | 68 +++++++++++++++++++ frappe_manager/utils/helpers.py | 2 +- 7 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 frappe_manager/sub_commands/__init__.py create mode 100644 frappe_manager/sub_commands/self_commands.py create mode 100644 frappe_manager/sub_commands/update_command.py diff --git a/frappe_manager/commands.py b/frappe_manager/commands.py index 764725c8..a978d061 100644 --- a/frappe_manager/commands.py +++ b/frappe_manager/commands.py @@ -18,9 +18,11 @@ from frappe_manager.utils.callbacks import apps_list_validation_callback, frappe_branch_validation_callback, version_callback from frappe_manager.utils.helpers import get_container_name_prefix, is_cli_help_called, get_current_fm_version from frappe_manager.services_manager.commands import services_app +from frappe_manager.sub_commands.self_commands import self_app app = typer.Typer(no_args_is_help=True,rich_markup_mode='rich') app.add_typer(services_app, name="services", help="Handle global services.") +app.add_typer(self_app, name="self", help="Perform operations related to the [bold][blue]fm[/bold][/blue] itself.") # this will be initiated later in the app_callback sites: Optional[SiteManager] = None diff --git a/frappe_manager/display_manager/DisplayManager.py b/frappe_manager/display_manager/DisplayManager.py index 1d7b8bda..d40e2193 100644 --- a/frappe_manager/display_manager/DisplayManager.py +++ b/frappe_manager/display_manager/DisplayManager.py @@ -117,7 +117,7 @@ def update_head(self, text: str): text=Text(self.current_head, style="blue bold"), style="bold blue" ) - def change_head(self, text: str): + def change_head(self, text: str,style: Optional[str] = 'blue bold'): """ Change the head text and update the spinner and live display. @@ -129,7 +129,10 @@ def change_head(self, text: str): """ self.previous_head = self.current_head self.current_head = text - self.spinner.update(text=Text(self.current_head, style="blue bold")) + if style: + self.spinner.update(text=Text(self.current_head, style="blue bold")) + else: + self.spinner.update(text=self.current_head) self.live.refresh() def update_live(self, renderable=None, padding: tuple = (0, 0, 0, 0)): diff --git a/frappe_manager/migration_manager/migration_executor.py b/frappe_manager/migration_manager/migration_executor.py index 671e6536..43741621 100644 --- a/frappe_manager/migration_manager/migration_executor.py +++ b/frappe_manager/migration_manager/migration_executor.py @@ -9,7 +9,7 @@ from frappe_manager.migration_manager.migration_exections import ( MigrationExceptionInSite, ) -from frappe_manager.utils.helpers import downgrade_package, get_current_fm_version +from frappe_manager.utils.helpers import install_package, get_current_fm_version from frappe_manager.logger import log from frappe_manager.migration_manager.version import Version from frappe_manager.display_manager.DisplayManager import richprint @@ -96,7 +96,7 @@ def execute(self): continue_migration = Prompt.ask(migrate_msg, choices=["yes", "no"]) if continue_migration == "no": - downgrade_package("frappe-manager", str(self.prev_version.version)) + install_package("frappe-manager", str(self.prev_version.version)) richprint.exit( f"Successfully installed [bold][blue]Frappe-Manager[/blue][/bold] version: v{str(self.prev_version.version)}", emoji_code=":white_check_mark:", @@ -192,7 +192,7 @@ def execute(self): richprint.print( f"Installing [bold][blue]Frappe-Manager[/blue][/bold] version: v{str(self.rollback_version.version)}" ) - downgrade_package("frappe-manager", str(self.rollback_version.version)) + install_package("frappe-manager", str(self.rollback_version.version)) richprint.exit("Rollback complete.") self.metadata_manager.set_version(self.current_version) diff --git a/frappe_manager/sub_commands/__init__.py b/frappe_manager/sub_commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/frappe_manager/sub_commands/self_commands.py b/frappe_manager/sub_commands/self_commands.py new file mode 100644 index 00000000..cd2b8e89 --- /dev/null +++ b/frappe_manager/sub_commands/self_commands.py @@ -0,0 +1,5 @@ +import typer +from frappe_manager.sub_commands.update_command import update_app + +self_app = typer.Typer(no_args_is_help=True, rich_markup_mode="rich") +self_app.add_typer(update_app, name="update", help="Check for updates, Update images etc.") diff --git a/frappe_manager/sub_commands/update_command.py b/frappe_manager/sub_commands/update_command.py new file mode 100644 index 00000000..12e8bde7 --- /dev/null +++ b/frappe_manager/sub_commands/update_command.py @@ -0,0 +1,68 @@ +import typer +import requests +import importlib +import json +from rich.prompt import Prompt +from pathlib import Path +from frappe_manager.compose_manager.ComposeFile import ComposeFile +from frappe_manager.docker_wrapper import DockerClient, DockerException +from frappe_manager.display_manager.DisplayManager import richprint +from frappe_manager.utils.helpers import install_package + +update_app = typer.Typer(rich_markup_mode="rich") + +@update_app.callback(invoke_without_command=True) +def update_callback( + ctx: typer.Context, + ): + if ctx.invoked_subcommand == None: + url = "https://pypi.org/pypi/frappe-manager/json" + try: + update_info = requests.get(url, timeout=0.1) + update_info = json.loads(update_info.text) + fm_version = importlib.metadata.version("frappe-manager") + latest_version = update_info["info"]["version"] + if not fm_version == latest_version: + update_msg = ( + f":arrows_counterclockwise: New update available [blue][bold]v{latest_version}[/bold][/blue]" + "\nDo you want to update ?" + ) + richprint.stop() + continue_update= Prompt.ask(update_msg, choices=["yes", "no"]) + + if continue_update == 'yes': + install_package("frappe-manager", latest_version) + except Exception as e: + richprint.exit(f"Error occured while updating the app : {e}") + + +@update_app.command() +def images( + ctx: typer.Context, + ): + services = ctx.obj['services'] + composefile = ComposeFile(loadfile=Path('docker-compose.yml')) + images_list = [] + docker = DockerClient() + if composefile.is_template_loaded: + images = composefile.get_all_images() + images.update(services.composefile.get_all_images()) + + for service ,image_info in images.items(): + image = f"{image_info['name']}:{image_info['tag']}" + images_list.append(image) + + # remove duplicates + images_list = list(dict.fromkeys(images_list)) + + for image in images_list: + status = f"[blue]Pulling image[/blue] [bold][yellow]{image}[/yellow][/bold]" + richprint.change_head(status,style=None) + try: + output = docker.pull(container_name=image , stream=True) + richprint.live_lines(output, padding=(0, 0, 0, 2)) + richprint.print(f"{status} : Done") + except DockerException as e: + richprint.error(f"{status} : Failed") + richprint.error(f"[red][bold]Error :[/bold][/red] {e}") + continue diff --git a/frappe_manager/utils/helpers.py b/frappe_manager/utils/helpers.py index 34903498..304acedf 100644 --- a/frappe_manager/utils/helpers.py +++ b/frappe_manager/utils/helpers.py @@ -315,5 +315,5 @@ def get_unix_groups(): groups[group_name] = group_entry.gr_gid return groups -def downgrade_package(package_name, version): +def install_package(package_name, version): subprocess.check_call([sys.executable, '-m', 'pip', 'install', f'{package_name}=={version}']) From 2ab650090e3d172ed2e8575bc3af36dbc7ef28c0 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 26 Feb 2024 18:26:51 +0530 Subject: [PATCH 10/33] bump: v0.11.0 --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e85757f6..0064652b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "frappe-manager" -version = "0.10.1" +version = "0.11.0" license = "MIT" repository = "https://github.com/rtcamp/frappe-manager" description = "A CLI tool based on Docker Compose to easily manage Frappe based projects. As of now, only suitable for development in local machines running on Mac and Linux based OS." @@ -9,6 +9,10 @@ maintainers = ["Alok Singh "] documentation = "https://github.com/rtcamp/frappe-manager/wiki" readme = "README.md" +[tool.black] +line-length = 79 +inline-quotes = "'" + [tool.poetry.urls] "Bug Tracker" = "https://github.com/rtcamp/frappe-manager/issues" From 14792a438c4ad0d1d2e017278d0c835cc853b7ee Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 26 Feb 2024 19:06:19 +0530 Subject: [PATCH 11/33] feat: fqdn support --- frappe_manager/site_manager/SiteManager.py | 3 +- frappe_manager/site_manager/site.py | 10 +++---- frappe_manager/utils/site.py | 32 +++++++++++++++++++++- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index d9d5d5c3..ecd0e5b6 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -14,6 +14,7 @@ from frappe_manager.docker_wrapper import DockerClient, DockerException from frappe_manager import CLI_DIR from rich.table import Table +from frappe_manager.utils.site import generate_services_table, domain_level from frappe_manager.utils.site import generate_services_table @@ -35,7 +36,7 @@ def init(self, sitename: str | None = None): sitename (str | None): The name of the site. If None, the default site will be used. """ if sitename: - if not ".localhost" in sitename: + if domain_level(sitename) == 0: sitename = sitename + ".localhost" sitepath: Path = self.sitesdir / sitename diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 40dd32f5..1923e899 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -1,7 +1,6 @@ from copy import deepcopy import importlib import shutil -import re import json from pathlib import Path from frappe_manager.docker_wrapper import DockerClient, DockerException @@ -9,11 +8,11 @@ from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.site_manager.site_exceptions import ( SiteDatabaseAddUserException, - SiteException, ) from frappe_manager.site_manager.workers_manager.SiteWorker import SiteWorkers from frappe_manager.utils.helpers import log_file, get_container_name_prefix from frappe_manager.utils.docker import host_run_cp +from frappe_manager.utils.site import is_fqdn class Site: @@ -54,12 +53,11 @@ def validate_sitename(self) -> bool: it returns False. """ sitename = self.name - match = re.search( - r"^[A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])?.localhost$", sitename - ) + match = is_fqdn(sitename) + if not match: richprint.exit( - "The site name must follow a single-level subdomain Fully Qualified Domain Name (FQDN) format of localhost, such as 'subdomain.localhost'." + f"The {sitename} must follow a single-level subdomain Fully Qualified Domain Name (FQDN) format of localhost, such as 'subdomain.localhost'." ) def get_frappe_container_hex(self) -> None | str: diff --git a/frappe_manager/utils/site.py b/frappe_manager/utils/site.py index 99f8d7c7..1350213a 100644 --- a/frappe_manager/utils/site.py +++ b/frappe_manager/utils/site.py @@ -1,5 +1,5 @@ from rich.table import Table - +import re def generate_services_table(services_status: dict): # running site services status @@ -75,3 +75,33 @@ def parse_docker_volume(volume_string): volume['type'] = 'volume' return volume + +def is_fqdn(hostname: str) -> bool: + """ + https://en.m.wikipedia.org/wiki/Fully_qualified_domain_name + """ + if not 1 < len(hostname) < 253: + return False + + # Remove trailing dot + if hostname[-1] == '.': + hostname = hostname[0:-1] + + # Split hostname into list of DNS labels + labels = hostname.split('.') + + # Define pattern of DNS label + # Can begin and end with a number or letter only + # Can contain hyphens, a-z, A-Z, 0-9 + # 1 - 63 chars allowed + fqdn = re.compile(r'^[a-z0-9]([a-z-0-9-]{0,61}[a-z0-9])?$', re.IGNORECASE) + + # Check that all labels match that pattern. + return all(fqdn.match(label) for label in labels) + +def domain_level(domain): + # Split the domain name into individual parts + parts = domain.split('.') + + # Return the number of parts minus 1 (excluding the TLD) + return len(parts) - 1 From 87e6afc37c5111469f265e7aee4f966f668bad09 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 26 Feb 2024 19:27:52 +0530 Subject: [PATCH 12/33] fix: bench get-app in prebake --- Docker/frappe/helper-function.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Docker/frappe/helper-function.sh b/Docker/frappe/helper-function.sh index 9278467f..c3122170 100755 --- a/Docker/frappe/helper-function.sh +++ b/Docker/frappe/helper-function.sh @@ -102,10 +102,10 @@ install_apps() { if [[ "${branch_name:-}" ]]; then echo "Installing app $app_name -> $branch_name" - $BENCH_COMMAND get-app --resolve-deps --overwrite --skip-assets --branch "${branch_name}" "${app_name}" + $BENCH_COMMAND get-app --overwrite --skip-assets --branch "${branch_name}" "${app_name}" else echo "Installing app $app_name" - $BENCH_COMMAND get-app --resolve-deps --overwrite --skip-assets "${app_name}" + $BENCH_COMMAND get-app --overwrite --skip-assets "${app_name}" fi done else From d94b4da6a6fafdb80e7d40de252db05c25ceb2dd Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 27 Feb 2024 04:16:19 +0530 Subject: [PATCH 13/33] fix: apps.txt and install_apps --- Docker/frappe/helper-function.sh | 34 ++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/Docker/frappe/helper-function.sh b/Docker/frappe/helper-function.sh index c3122170..93c727a2 100755 --- a/Docker/frappe/helper-function.sh +++ b/Docker/frappe/helper-function.sh @@ -126,9 +126,23 @@ install_apps() { fi done + # create apps_txt + local apps_list + + apps_txt=$(mktemp) + + apps_list=$(ls -1 apps|| exit 0) + + for app_name in $(echo "$apps_list"); do + get_app_name "$app_name" + echo "$APP_NAME" >> "$apps_txt" + done + + cat "$apps_txt" > sites/apps.txt + # create apps_json - for app_name in $(ls -1 apps | grep -v 'frappe' || exit 0); do - apps_json=$(echo "$apps_json" | jq -rc --arg app_name "${app_name}" '.+ [$app_name]') + for app_name in $(cat "$apps_txt" | grep -v 'frappe' || exit 0); do + apps_json=$(echo "$apps_json" | jq -rc --arg app_name "${APP_NAME}" '.+ [$app_name]') done update_common_site_config install_apps "$apps_json" 'true' } @@ -187,3 +201,19 @@ update_uid_gid() { echo "UID and GID updated successfully." } + +# this return the list of apps +# input +# $1 -> app_name respective to apps dir +get_app_name(){ + local app="$1" + hooks_py_path=f"/workspce/frappe-bench/apps/$app" + + # Extract the app name from the hooks.py file + APP_NAME=$(awk -F'"' '/app_name/{print $2}' "$hooks_py_path" || exit 0) + + if ! [[ "${APP_NAME:-}" ]]; then + # If the app name is not found, use app name from basename of the app dir + APP_NAME=${app##*/} + fi +} From c3b44433a6c3a7d7f6a3e87c80d2d939e0377d46 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 27 Feb 2024 04:18:14 +0530 Subject: [PATCH 14/33] override command 'bench status' for supervisor status --- Docker/frappe/bench-wrapper.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Docker/frappe/bench-wrapper.sh b/Docker/frappe/bench-wrapper.sh index c4cdbff2..e8bfe4e1 100644 --- a/Docker/frappe/bench-wrapper.sh +++ b/Docker/frappe/bench-wrapper.sh @@ -1,9 +1,15 @@ #!/bin/bash -after_command() { - supervisorctl -c /opt/user/supervisord.conf restart frappe-bench-dev: +restart_command() { + supervisorctl -c /opt/user/supervisord.conf restart frappe-bench-dev: } +status_command() { + supervisorctl -c /opt/user/supervisord.conf status frappe-bench-dev: +} + if [[ "$@" =~ ^restart[[:space:]]* ]]; then - after_command + restart_command +elif [[ "$@" =~ ^status[[:space:]]* ]]; then + status_command else - /opt/.pyenv/shims/bench "$@" + /opt/.pyenv/shims/bench "$@" fi From 638e8c3aa0de277391dee5fab5381182c796de8c Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 27 Feb 2024 04:20:06 +0530 Subject: [PATCH 15/33] fix: frappe dev server not killed after supervisorctl stop --- Docker/frappe/bench-dev-server | 1 + 1 file changed, 1 insertion(+) diff --git a/Docker/frappe/bench-dev-server b/Docker/frappe/bench-dev-server index 6c7904e7..b2f97cc5 100644 --- a/Docker/frappe/bench-dev-server +++ b/Docker/frappe/bench-dev-server @@ -1,3 +1,4 @@ #!/bin/bash +trap "kill -- -$$" EXIT fuser -k 80/tcp bench serve --port 80 From 535359a1406a9ed15740756df4ca43f6ba120094 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 27 Feb 2024 04:20:40 +0530 Subject: [PATCH 16/33] adjust chown --- Docker/frappe/entrypoint.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Docker/frappe/entrypoint.sh b/Docker/frappe/entrypoint.sh index 12926dd6..e83b942a 100755 --- a/Docker/frappe/entrypoint.sh +++ b/Docker/frappe/entrypoint.sh @@ -30,8 +30,10 @@ if [[ ! -f "/workspace/.profile" ]]; then cat /opt/user/.profile > /workspace/.profile fi +chown "$USERID":"$USERGROUP" /workspace /workspace/frappe-bench + +ls -p /workspace | grep -v 'frappe-bench/' | xargs -I{} chown -R "$USERID":"$USERGROUP" /workspace{} -chown -R "$USERID":"$USERGROUP" /workspace if [ "$#" -gt 0 ]; then gosu "$USERID":"$USERGROUP" "/scripts/$@" From 1ac8a6119ea3b9f17a15d8a572fb32a6382b83cc Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 27 Feb 2024 04:22:07 +0530 Subject: [PATCH 17/33] fix: docker images pull on first install --- frappe_manager/commands.py | 68 +++++++++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 9 deletions(-) diff --git a/frappe_manager/commands.py b/frappe_manager/commands.py index a978d061..89d092b0 100644 --- a/frappe_manager/commands.py +++ b/frappe_manager/commands.py @@ -1,17 +1,22 @@ +from copy import deepcopy +from re import template from ruamel.yaml import serialize +from pathlib import Path import typer import os import requests import sys import shutil +import importlib +import json from typing import Annotated, List, Optional, Set from frappe_manager.services_manager.services_exceptions import ServicesNotCreated from frappe_manager.site_manager.SiteManager import SiteManager from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager import CLI_DIR, default_extension, SiteServicesEnum, services_manager +from frappe_manager.docker_wrapper import DockerClient, DockerException from frappe_manager.logger import log -from frappe_manager.docker_wrapper import DockerClient from frappe_manager.services_manager.services import ServicesManager from frappe_manager.migration_manager.migration_executor import MigrationExecutor from frappe_manager.site_manager.site_exceptions import SiteException @@ -19,6 +24,9 @@ from frappe_manager.utils.helpers import get_container_name_prefix, is_cli_help_called, get_current_fm_version from frappe_manager.services_manager.commands import services_app from frappe_manager.sub_commands.self_commands import self_app +from frappe_manager.metadata_manager import MetadataManager +from frappe_manager.migration_manager.version import Version +from frappe_manager.compose_manager.ComposeFile import ComposeFile app = typer.Typer(no_args_is_help=True,rich_markup_mode='rich') app.add_typer(services_app, name="services", help="Handle global services.") @@ -57,7 +65,6 @@ def app_callback( CLI_DIR.mkdir(parents=True, exist_ok=True) sitesdir.mkdir(parents=True, exist_ok=True) richprint.print(f"fm directory doesn't exists! Created at -> {str(CLI_DIR)}") - first_time_install = True else: if not CLI_DIR.is_dir(): richprint.exit("Sites directory is not a directory! Aborting!") @@ -78,13 +85,56 @@ def app_callback( if not DockerClient().server_running(): richprint.exit("Docker daemon not running. Please start docker service.") - if first_time_install : - from frappe_manager.metadata_manager import MetadataManager - from frappe_manager.migration_manager.version import Version - metadata_manager = MetadataManager() - current_version = Version(get_current_fm_version()) - metadata_manager.set_version(current_version) - metadata_manager.save() + metadata_manager = MetadataManager() + + # docker pull + if not metadata_manager.toml_file.exists(): + richprint.print("🔍 It seems like the first installation. Pulling images... 🖼️") + site_composefile = ComposeFile(loadfile=Path('docker-compose.yml')) + services_composefile = ComposeFile(loadfile=Path('docker-compose.services.yml',template='docker-compose.services.tmpl')) + images_list = [] + docker = DockerClient() + + if site_composefile.is_template_loaded: + images = site_composefile.get_all_images() + images.update(services_composefile.get_all_images()) + + for service ,image_info in images.items(): + image = f"{image_info['name']}:{image_info['tag']}" + images_list.append(image) + + # remove duplicates + images_dict = dict.fromkeys(images_list) + images_list = deepcopy(images_dict).keys() + error = False + + for image in images_list: + status = f"[blue]Pulling image[/blue] [bold][yellow]{image}[/yellow][/bold]" + richprint.change_head(status,style=None) + try: + output = docker.pull(container_name=image , stream=True) + richprint.live_lines(output, padding=(0, 0, 0, 2)) + richprint.print(f"{status} : Done") + except DockerException as e: + error = True + images_dict[image] = e + continue + + # richprint.error(f"[red][bold]Error :[/bold][/red] {e}") + + if error: + print('') + richprint.error(f"[bold][red]Pulling images failed for these images[/bold][/red]") + for image,exception in images_dict.items(): + if exception: + richprint.error(f'[bold][red]Image [/bold][/red]: {image}') + richprint.error(f'[bold][red]Error [/bold][/red]: {exception}') + shutil.rmtree(CLI_DIR) + richprint.exit("Aborting. [bold][blue]fm[/blue][/bold] will not be able to work without images. 🖼️") + + current_version = Version(get_current_fm_version()) + metadata_manager.set_version(current_version) + metadata_manager.save() migrations = MigrationExecutor() migration_status = migrations.execute() From 2b7a64d574125315454ba97f9f275cc16579a866 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 27 Feb 2024 04:22:38 +0530 Subject: [PATCH 18/33] enhance exception logging --- frappe_manager/main.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/frappe_manager/main.py b/frappe_manager/main.py index 3e812c95..533dedfa 100644 --- a/frappe_manager/main.py +++ b/frappe_manager/main.py @@ -1,7 +1,7 @@ import atexit from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.logger import log -from frappe_manager.utils.helpers import check_update, remove_zombie_subprocess_process +from frappe_manager.utils.helpers import check_update, remove_zombie_subprocess_process from frappe_manager.utils.docker import process_opened from frappe_manager.commands import app @@ -11,8 +11,12 @@ def cli_entrypoint(): app() except Exception as e: logger = log.get_logger() - logger.exception(f"Exception: : {e}") - raise e + richprint.stop() + with richprint.stdout.capture() as capture: + richprint.stdout.print_exception(show_locals=True) + excep = capture.get() + logger.error(f"Exception Occured: : \n{excep}") + finally: atexit.register(exit_cleanup) @@ -21,5 +25,5 @@ def exit_cleanup(): This function is used to perform cleanup at the exit. """ remove_zombie_subprocess_process(process_opened) - #check_update() + check_update() richprint.stop() From 4f3439d5507ea896732f8e0e4321323855a39da9 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 27 Feb 2024 04:23:03 +0530 Subject: [PATCH 19/33] fix site check logic after creation --- frappe_manager/site_manager/site.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 1923e899..7564a10d 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -328,6 +328,7 @@ def pull(self): richprint.print(f"{status_text}: Done") except DockerException as e: richprint.warning(f"{status_text}: Failed") + raise e def logs(self, service: str, follow: bool = False): """ @@ -530,22 +531,23 @@ def bench_dev_server_logs(self, follow=False): richprint.error(f"Log file not found: {bench_start_log_path}") def is_site_created(self, retry=60, interval=1) -> bool: - import requests from time import sleep - i = 0 - while i < retry: + for _ in range(retry): try: - host_header = {"Host": f"{self.name}"} - response = requests.get(url=f"http://127.0.0.1", headers=host_header) - if response.status_code == 200: - return True - else: - raise Exception("Site not working.") + # Execute curl command on frappe service + result = self.docker.compose.exec( + service="frappe", + command=f"curl -I --max-time {retry} --connect-timeout {retry} http://localhost", + stream = True + ) + + # Check if the site is working + for source , line in result: + if "HTTP/1.1 200 OK" in line.decode(): + return True except Exception as e: sleep(interval) - i += 1 - continue return False From c83019f7825574b4fdf9219ae715448c54c72d21 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 27 Feb 2024 17:02:05 +0530 Subject: [PATCH 20/33] fix: get_app_name --- Docker/frappe/helper-function.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Docker/frappe/helper-function.sh b/Docker/frappe/helper-function.sh index 93c727a2..645413be 100755 --- a/Docker/frappe/helper-function.sh +++ b/Docker/frappe/helper-function.sh @@ -144,6 +144,7 @@ install_apps() { for app_name in $(cat "$apps_txt" | grep -v 'frappe' || exit 0); do apps_json=$(echo "$apps_json" | jq -rc --arg app_name "${APP_NAME}" '.+ [$app_name]') done + update_common_site_config install_apps "$apps_json" 'true' } @@ -207,7 +208,7 @@ update_uid_gid() { # $1 -> app_name respective to apps dir get_app_name(){ local app="$1" - hooks_py_path=f"/workspce/frappe-bench/apps/$app" + hooks_py_path="/workspce/frappe-bench/apps/$app" # Extract the app name from the hooks.py file APP_NAME=$(awk -F'"' '/app_name/{print $2}' "$hooks_py_path" || exit 0) From 7507d92ed2c1aae00743365beaa7f7b59a100f9d Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 27 Feb 2024 17:03:29 +0530 Subject: [PATCH 21/33] fix: migrations execution --- frappe_manager/commands.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe_manager/commands.py b/frappe_manager/commands.py index 89d092b0..fd05f051 100644 --- a/frappe_manager/commands.py +++ b/frappe_manager/commands.py @@ -132,9 +132,9 @@ def app_callback( shutil.rmtree(CLI_DIR) richprint.exit("Aborting. [bold][blue]fm[/blue][/bold] will not be able to work without images. 🖼️") - current_version = Version(get_current_fm_version()) - metadata_manager.set_version(current_version) - metadata_manager.save() + # current_version = Version(get_current_fm_version()) + # metadata_manager.set_version(current_version) + # metadata_manager.save() migrations = MigrationExecutor() migration_status = migrations.execute() From 6bcd41f6ddab01776a7ef73f4108340386146a06 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 27 Feb 2024 17:03:51 +0530 Subject: [PATCH 22/33] update check_update msg --- frappe_manager/utils/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe_manager/utils/helpers.py b/frappe_manager/utils/helpers.py index 304acedf..073d55f9 100644 --- a/frappe_manager/utils/helpers.py +++ b/frappe_manager/utils/helpers.py @@ -56,7 +56,7 @@ def check_update(): latest_version = update_info["info"]["version"] if not fm_version == latest_version: richprint.warning( - f'Ready for an update? Run "pip install --upgrade frappe-manager" to update to the latest version {latest_version}.', + f'[dim]Update available v{latest_version}.[/dim]', emoji_code=":arrows_counterclockwise:️", ) except Exception as e: From 5f7ebb9d630a4128c833831ce7906c3d83c62b2a Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 27 Feb 2024 17:04:41 +0530 Subject: [PATCH 23/33] fix permission and prompt before delete --- frappe_manager/docker_wrapper/DockerClient.py | 3 ++- frappe_manager/site_manager/SiteManager.py | 10 +++++++--- frappe_manager/site_manager/site.py | 15 ++++++++++++--- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/frappe_manager/docker_wrapper/DockerClient.py b/frappe_manager/docker_wrapper/DockerClient.py index 87aad302..3c7300b6 100644 --- a/frappe_manager/docker_wrapper/DockerClient.py +++ b/frappe_manager/docker_wrapper/DockerClient.py @@ -162,9 +162,10 @@ def rm( def run( self, - command: str, image: str, + command: Optional[str] = None, name: Optional[str] = None, + volume: Optional[str] = None, detach: bool = False, entrypoint: Optional[str] = None, pull: Literal["missing", "never", "always"] = "missing", diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index ecd0e5b6..58900639 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -2,6 +2,7 @@ import json import shlex from ruamel.yaml import serialize +from rich.prompt import Prompt import typer import shutil @@ -174,9 +175,12 @@ def remove_site(self): """ Removes the site. """ - richprint.change_head(f"Removing Site") - self.site.remove_database_and_user() - self.site.remove() + richprint.stop() + continue_remove= Prompt.ask(f"🤔 Do you want to remove [bold][green]'{self.site.name}'[/bold][/green]", choices=["yes", "no"],default='no') + if continue_remove == 'yes': + richprint.start('Removing Site') + self.site.remove_database_and_user() + self.site.remove() def list_sites(self): """ diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index 7564a10d..ca0645e1 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -450,11 +450,20 @@ def remove(self) -> bool: except DockerException as e: richprint.exit(f"{status_text}: Failed") richprint.change_head(f"Removing Dirs") + try: shutil.rmtree(self.path) - except Exception as e: - richprint.error(e) - richprint.exit(f"Please remove {self.path} manually") + except PermissionError as e: + images = self.composefile.get_all_images() + if 'frappe' in images: + try: + frappe_image = images['frappe'] + frappe_image = f"{frappe_image['name']}:{frappe_image['tag']}" + output = self.docker.run(image=frappe_image,entrypoint="/bin/sh",command="-c 'chown -R frappe:frappe .'",volume=f'{self.path}/workspace:/workspace',stream=True, stream_only_exit_code=True) + shutil.rmtree(self.path) + except Exception: + richprint.error(e) + richprint.exit(f"Please remove {self.path} manually") richprint.change_head(f"Removing Dirs: Done") def shell(self, container: str, user: str | None = None): From 0bea0bcad69bae4c08446af8baae48c793fc60db Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 27 Feb 2024 17:05:33 +0530 Subject: [PATCH 24/33] reduce .pyenv size in frappe container --- Docker/frappe/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Docker/frappe/Dockerfile b/Docker/frappe/Dockerfile index 73714347..6dac0231 100644 --- a/Docker/frappe/Dockerfile +++ b/Docker/frappe/Dockerfile @@ -126,7 +126,10 @@ RUN git clone --depth 1 https://github.com/pyenv/pyenv.git .pyenv \ && pyenv global $PYTHON_VERSION \ && echo 'export PYENV_ROOT="/opt/.pyenv"' >> "$USERZSHRC" \ && echo 'command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"' >> "$USERZSHRC" \ - && echo 'eval "$(pyenv init --path)"' >>"$USERZSHRC" + && echo 'eval "$(pyenv init --path)"' >>"$USERZSHRC" \ + # remove *.pyc and *.pyo as used here in official docker image to reduce size + # https://github.com/docker-library/python/blob/789d789e4a8db71d3d393667971c49b845ffdc3f/3.11/alpine3.19/Dockerfile#L106-L111 + && find /opt/.pyenv/versions -depth \( \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \) \) \) -exec rm -rf '{}' + ; RUN pip install frappe-bench From 4354046b33c4cf398a400e1f18841f7b4fbc277c Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 27 Feb 2024 18:42:22 +0530 Subject: [PATCH 25/33] fix: docker images pull on first install --- frappe_manager/commands.py | 96 +++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 47 deletions(-) diff --git a/frappe_manager/commands.py b/frappe_manager/commands.py index fd05f051..a9a1261b 100644 --- a/frappe_manager/commands.py +++ b/frappe_manager/commands.py @@ -65,6 +65,7 @@ def app_callback( CLI_DIR.mkdir(parents=True, exist_ok=True) sitesdir.mkdir(parents=True, exist_ok=True) richprint.print(f"fm directory doesn't exists! Created at -> {str(CLI_DIR)}") + first_time_install = True else: if not CLI_DIR.is_dir(): richprint.exit("Sites directory is not a directory! Aborting!") @@ -88,53 +89,54 @@ def app_callback( metadata_manager = MetadataManager() # docker pull - if not metadata_manager.toml_file.exists(): - richprint.print("🔍 It seems like the first installation. Pulling images... 🖼️") - site_composefile = ComposeFile(loadfile=Path('docker-compose.yml')) - services_composefile = ComposeFile(loadfile=Path('docker-compose.services.yml',template='docker-compose.services.tmpl')) - images_list = [] - docker = DockerClient() - - if site_composefile.is_template_loaded: - images = site_composefile.get_all_images() - images.update(services_composefile.get_all_images()) - - for service ,image_info in images.items(): - image = f"{image_info['name']}:{image_info['tag']}" - images_list.append(image) - - # remove duplicates - images_dict = dict.fromkeys(images_list) - images_list = deepcopy(images_dict).keys() - error = False - - for image in images_list: - status = f"[blue]Pulling image[/blue] [bold][yellow]{image}[/yellow][/bold]" - richprint.change_head(status,style=None) - try: - output = docker.pull(container_name=image , stream=True) - richprint.live_lines(output, padding=(0, 0, 0, 2)) - richprint.print(f"{status} : Done") - except DockerException as e: - error = True - images_dict[image] = e - continue - - # richprint.error(f"[red][bold]Error :[/bold][/red] {e}") - - if error: - print('') - richprint.error(f"[bold][red]Pulling images failed for these images[/bold][/red]") - for image,exception in images_dict.items(): - if exception: - richprint.error(f'[bold][red]Image [/bold][/red]: {image}') - richprint.error(f'[bold][red]Error [/bold][/red]: {exception}') - shutil.rmtree(CLI_DIR) - richprint.exit("Aborting. [bold][blue]fm[/blue][/bold] will not be able to work without images. 🖼️") - - # current_version = Version(get_current_fm_version()) - # metadata_manager.set_version(current_version) - # metadata_manager.save() + if first_time_install: + if not metadata_manager.toml_file.exists(): + richprint.print("🔍 It seems like the first installation. Pulling images... 🖼️") + site_composefile = ComposeFile(loadfile=Path('docker-compose.yml')) + services_composefile = ComposeFile(loadfile=Path('docker-compose.services.yml',template='docker-compose.services.tmpl')) + images_list = [] + docker = DockerClient() + + if site_composefile.is_template_loaded: + images = site_composefile.get_all_images() + images.update(services_composefile.get_all_images()) + + for service ,image_info in images.items(): + image = f"{image_info['name']}:{image_info['tag']}" + images_list.append(image) + + # remove duplicates + images_dict = dict.fromkeys(images_list) + images_list = deepcopy(images_dict).keys() + error = False + + for image in images_list: + status = f"[blue]Pulling image[/blue] [bold][yellow]{image}[/yellow][/bold]" + richprint.change_head(status,style=None) + try: + output = docker.pull(container_name=image , stream=True) + richprint.live_lines(output, padding=(0, 0, 0, 2)) + richprint.print(f"{status} : Done") + except DockerException as e: + error = True + images_dict[image] = e + continue + + # richprint.error(f"[red][bold]Error :[/bold][/red] {e}") + + if error: + print('') + richprint.error(f"[bold][red]Pulling images failed for these images[/bold][/red]") + for image,exception in images_dict.items(): + if exception: + richprint.error(f'[bold][red]Image [/bold][/red]: {image}') + richprint.error(f'[bold][red]Error [/bold][/red]: {exception}') + shutil.rmtree(CLI_DIR) + richprint.exit("Aborting. [bold][blue]fm[/blue][/bold] will not be able to work without images. 🖼️") + + current_version = Version(get_current_fm_version()) + metadata_manager.set_version(current_version) + metadata_manager.save() migrations = MigrationExecutor() migration_status = migrations.execute() From 7f83e38e3ccf1d12b9419123fc9da12553980c26 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 27 Feb 2024 18:43:20 +0530 Subject: [PATCH 26/33] disable pypi update check --- frappe_manager/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe_manager/main.py b/frappe_manager/main.py index 533dedfa..198d3dcb 100644 --- a/frappe_manager/main.py +++ b/frappe_manager/main.py @@ -25,5 +25,5 @@ def exit_cleanup(): This function is used to perform cleanup at the exit. """ remove_zombie_subprocess_process(process_opened) - check_update() + #check_update() richprint.stop() From cc8a769f0abd3fbdfe9af1fb2ea1f9febb6d0bc4 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 27 Feb 2024 18:51:55 +0530 Subject: [PATCH 27/33] alert user for adding site in hosts file --- frappe_manager/site_manager/SiteManager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index 58900639..00db0333 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -148,8 +148,9 @@ def create_site(self, template_inputs: dict,template_site: bool = False): f"SITE_STATUS {self.site.name}: WORKING" ) richprint.print(f"Started site") - self.info() + if not '.localhost' in self.site.name: + richprint.print(f"Please note that You will have to add a host entry to your system's hosts file to access the site locally.") else: self.typer_context.obj["logger"].error(f"{self.site.name}: NOT WORKING") From 97d27ca6cb024d595a386bc06dbd52a1468b7923 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 27 Feb 2024 23:39:35 +0530 Subject: [PATCH 28/33] fix: app installation path --- Docker/frappe/helper-function.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Docker/frappe/helper-function.sh b/Docker/frappe/helper-function.sh index 645413be..59b17eae 100755 --- a/Docker/frappe/helper-function.sh +++ b/Docker/frappe/helper-function.sh @@ -208,7 +208,9 @@ update_uid_gid() { # $1 -> app_name respective to apps dir get_app_name(){ local app="$1" - hooks_py_path="/workspce/frappe-bench/apps/$app" + local app_dir + app_dir="/workspace/frappe-bench/apps/${app}" + hooks_py_path=$(find "$app_dir" -maxdepth 2 -type f -name hooks.py) # Extract the app name from the hooks.py file APP_NAME=$(awk -F'"' '/app_name/{print $2}' "$hooks_py_path" || exit 0) From db6f829beb0fe66bfd610caf3a7de28a384bbd52 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 27 Feb 2024 23:40:50 +0530 Subject: [PATCH 29/33] Update image tag in template --- frappe_manager/templates/docker-compose.tmpl | 4 ++-- frappe_manager/templates/docker-compose.workers.tmpl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe_manager/templates/docker-compose.tmpl b/frappe_manager/templates/docker-compose.tmpl index cc13ed81..a44006ee 100644 --- a/frappe_manager/templates/docker-compose.tmpl +++ b/frappe_manager/templates/docker-compose.tmpl @@ -76,7 +76,7 @@ services: global-backend-network: socketio: - image: ghcr.io/rtcamp/frappe-manager-frappe:v0.10.0 + image: ghcr.io/rtcamp/frappe-manager-frappe:v0.11.0 container_name: REPLACE_ME_WITH_CONTAINER_NAME restart: always environment: @@ -95,7 +95,7 @@ services: site-network: schedule: - image: ghcr.io/rtcamp/frappe-manager-frappe:v0.10.0 + image: ghcr.io/rtcamp/frappe-manager-frappe:v0.11.0 container_name: REPLACE_ME_WITH_CONTAINER_NAME restart: always environment: diff --git a/frappe_manager/templates/docker-compose.workers.tmpl b/frappe_manager/templates/docker-compose.workers.tmpl index 4cdeaf16..98a6accd 100644 --- a/frappe_manager/templates/docker-compose.workers.tmpl +++ b/frappe_manager/templates/docker-compose.workers.tmpl @@ -1,6 +1,6 @@ services: worker-name: - image: ghcr.io/rtcamp/frappe-manager-frappe:v0.10.0 + image: ghcr.io/rtcamp/frappe-manager-frappe:v0.11.0 restart: always environment: TIMEOUT: 6000 From a8c7888c7e59952919d3529f14278257b9c51be9 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 27 Feb 2024 23:42:41 +0530 Subject: [PATCH 30/33] remove double prompt in fm create, format code --- frappe_manager/site_manager/SiteManager.py | 137 +++++++++++++-------- 1 file changed, 86 insertions(+), 51 deletions(-) diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index 00db0333..179f2a87 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -21,7 +21,7 @@ class SiteManager: - def __init__(self, sitesdir: Path, services = None): + def __init__(self, sitesdir: Path, services=None): self.sitesdir = sitesdir self.site = None self.sitepath = None @@ -57,7 +57,12 @@ def init(self, sitename: str | None = None): f"The site '{sitename}' does not exist. Aborting operation." ) - self.site: Site = Site(sitepath, sitename, verbose=self.verbose,services=self.services) + self.site: Site = Site( + sitepath, + sitename, + verbose=self.verbose, + services=self.services, + ) def set_verbose(self): """ @@ -102,14 +107,16 @@ def stop_sites(self): for site_compose_path in site_compose: docker = DockerClient(compose_file_path=site_compose_path) try: - output = docker.compose.stop(timeout=10, stream=not self.verbose) + output = docker.compose.stop( + timeout=10, stream=not self.verbose + ) if not self.verbose: richprint.live_lines(output, padding=(0, 0, 0, 2)) except DockerException as e: richprint.exit(f"{status_text}: Failed") richprint.print(f"{status_text}: Done") - def create_site(self, template_inputs: dict,template_site: bool = False): + def create_site(self, template_inputs: dict, template_site: bool = False): """ Creates a new site using the provided template inputs. @@ -130,7 +137,10 @@ def create_site(self, template_inputs: dict,template_site: bool = False): if template_site: self.site.remove_secrets() - richprint.exit(f"Created template site: {self.site.name}",emoji_code=":white_check_mark:") + richprint.exit( + f"Created template site: {self.site.name}", + emoji_code=":white_check_mark:", + ) richprint.change_head(f"Starting Site") self.site.start() @@ -149,10 +159,14 @@ def create_site(self, template_inputs: dict,template_site: bool = False): ) richprint.print(f"Started site") self.info() - if not '.localhost' in self.site.name: - richprint.print(f"Please note that You will have to add a host entry to your system's hosts file to access the site locally.") + if not ".localhost" in self.site.name: + richprint.print( + f"Please note that You will have to add a host entry to your system's hosts file to access the site locally." + ) else: - self.typer_context.obj["logger"].error(f"{self.site.name}: NOT WORKING") + self.typer_context.obj["logger"].error( + f"{self.site.name}: NOT WORKING" + ) richprint.stop() @@ -165,23 +179,27 @@ def create_site(self, template_inputs: dict,template_site: bool = False): richprint.error(error_message.format(log_path)) - # prompt if site not working to delete the site - if typer.confirm(f"Do you want to delete this site {self.site.name}?"): - richprint.start("Removing Site") - self.remove_site() - else: + remove_status = self.remove_site() + if not remove_status: self.info() - def remove_site(self): + def remove_site(self) -> bool: """ Removes the site. """ richprint.stop() - continue_remove= Prompt.ask(f"🤔 Do you want to remove [bold][green]'{self.site.name}'[/bold][/green]", choices=["yes", "no"],default='no') - if continue_remove == 'yes': - richprint.start('Removing Site') - self.site.remove_database_and_user() - self.site.remove() + continue_remove = Prompt.ask( + f"🤔 Do you want to remove [bold][green]'{self.site.name}'[/bold][/green]", + choices=["yes", "no"], + default="no", + ) + if continue_remove == "no": + return False + + richprint.start("Removing Site") + self.site.remove_database_and_user() + self.site.remove() + return True def list_sites(self): """ @@ -206,7 +224,9 @@ def list_sites(self): temp_site = Site(site_path, site_name) row_data = f"[link=http://{temp_site.name}]{temp_site.name}[/link]" - path_data = f"[link=file://{temp_site.path}]{temp_site.path}[/link]" + path_data = ( + f"[link=file://{temp_site.path}]{temp_site.path}[/link]" + ) status_color = "white" status_msg = "Inactive" @@ -237,13 +257,15 @@ def start_site(self): """ Starts the site. """ - #self.migrate_site() + # self.migrate_site() self.site.sync_site_common_site_config() self.site.start() self.site.frappe_logs_till_start(status_msg="Starting Site") self.site.sync_workers_compose() - def attach_to_site(self, user: str, extensions: List[str], debugger: bool = False): + def attach_to_site( + self, user: str, extensions: List[str], debugger: bool = False + ): """ Attaches to a running site's container using Visual Studio Code Remote Containers extension. @@ -276,15 +298,15 @@ def attach_to_site(self, user: str, extensions: List[str], debugger: bool = Fals vscode_config_json = [ { "remoteUser": user, - "remoteEnv": { - "SHELL": "/bin/zsh" + "remoteEnv": {"SHELL": "/bin/zsh"}, + "customizations": { + "vscode": { + "settings": { + "python.pythonPath": "/workspace/frappe-bench/env/bin/python" + }, + "extensions": extensions, + } }, - "customizations": {"vscode": { - "settings": { - "python.pythonPath": "/workspace/frappe-bench/env/bin/python" - }, - "extensions": extensions - }}, } ] @@ -295,10 +317,12 @@ def attach_to_site(self, user: str, extensions: List[str], debugger: bool = Fals # check if the extension are the same if they are different then only update # check if customizations key available try: - extensions_previous = json.loads(labels_previous["devcontainer.metadata"]) - extensions_previous = extensions_previous[0]["customizations"]["vscode"][ - "extensions" - ] + extensions_previous = json.loads( + labels_previous["devcontainer.metadata"] + ) + extensions_previous = extensions_previous[0]["customizations"][ + "vscode" + ]["extensions"] except KeyError: extensions_previous = [] @@ -315,27 +339,32 @@ def attach_to_site(self, user: str, extensions: List[str], debugger: bool = Fals # sync debugger files if debugger: richprint.change_head("Sync vscode debugger configuration") - dot_vscode_dir = self.site.path / 'workspace' / '.vscode' - tasks_json_path = dot_vscode_dir / 'tasks' - launch_json_path = dot_vscode_dir / 'launch' + dot_vscode_dir = self.site.path / "workspace" / ".vscode" + tasks_json_path = dot_vscode_dir / "tasks" + launch_json_path = dot_vscode_dir / "launch" dot_vscode_config = { - tasks_json_path : VSCODE_TASKS_JSON, - launch_json_path : VSCODE_LAUNCH_JSON, + tasks_json_path: VSCODE_TASKS_JSON, + launch_json_path: VSCODE_LAUNCH_JSON, } if not dot_vscode_dir.exists(): - dot_vscode_dir.mkdir(exist_ok=True,parents=True) + dot_vscode_dir.mkdir(exist_ok=True, parents=True) - for file_path in [launch_json_path,tasks_json_path]: - file_name = f'{file_path.name}.json' + for file_path in [launch_json_path, tasks_json_path]: + file_name = f"{file_path.name}.json" real_file_path = file_path.parent / file_name if real_file_path.exists(): - backup_tasks_path = file_path.parent / f"{file_path.name}.{datetime.now().strftime('%d-%b-%y--%H-%M-%S')}.json" + backup_tasks_path = ( + file_path.parent + / f"{file_path.name}.{datetime.now().strftime('%d-%b-%y--%H-%M-%S')}.json" + ) shutil.copy2(real_file_path, backup_tasks_path) - richprint.print(f"Backup previous '{file_name}' : {backup_tasks_path}") + richprint.print( + f"Backup previous '{file_name}' : {backup_tasks_path}" + ) - with open(real_file_path, 'w+') as f: + with open(real_file_path, "w+") as f: f.write(json.dumps(dot_vscode_config[file_path])) richprint.print("Sync vscode debugger configuration: Done") @@ -419,13 +448,17 @@ def info(self): db_user = site_config["db_name"] db_pass = site_config["db_password"] - frappe_password = self.site.composefile.get_envs("frappe")["ADMIN_PASS"] + frappe_password = self.site.composefile.get_envs("frappe")[ + "ADMIN_PASS" + ] services_db_info = self.services.get_database_info() - root_db_password = services_db_info['password'] - root_db_host = services_db_info['host'] - root_db_user = services_db_info['user'] + root_db_password = services_db_info["password"] + root_db_host = services_db_info["host"] + root_db_user = services_db_info["user"] - site_info_table = Table(show_lines=True, show_header=False, highlight=True) + site_info_table = Table( + show_lines=True, show_header=False, highlight=True + ) data = { "Site Url": f"http://{self.site.name}", @@ -468,7 +501,9 @@ def info(self): running_site_workers = self.site.workers.get_services_running_status() if running_site_services: - site_services_table = generate_services_table(running_site_services) + site_services_table = generate_services_table( + running_site_services + ) site_info_table.add_row("Site Services", site_services_table) if running_site_workers: From 24a4ecc742bc14e1b44f46fe57c9f8ef24e00431 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Thu, 29 Feb 2024 15:43:17 +0530 Subject: [PATCH 31/33] feat: action workflow for baking images --- .github/workflows/bake-images.yml | 62 +++++++++++++++++++++++++++++++ .github/workflows/docker-push.yml | 37 ------------------ 2 files changed, 62 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/bake-images.yml delete mode 100644 .github/workflows/docker-push.yml diff --git a/.github/workflows/bake-images.yml b/.github/workflows/bake-images.yml new file mode 100644 index 00000000..e7372f9f --- /dev/null +++ b/.github/workflows/bake-images.yml @@ -0,0 +1,62 @@ +name: Publish image + +on: + schedule: + - cron: '0 2 * * *' + push: + tags: + - v* + +jobs: + build-and-push: + name: Build and push image + strategy: + matrix: + os: [self-hosted-arm64,ubuntu-latest] + platform: [linux/amd64, linux/arm64] + service_name: [frappe, mailhog, nginx] + exclude: + - os: ubuntu-latest + platform: linux/arm64 + - os: self-hosted-arm64 + platform: linux/amd64 + + runs-on: ${{ matrix.os }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up tag + id: set-tag + run: | + latest_tag=$(git describe --abbrev=0 --tags) + if [[ "${{ github.ref == 'refs/tags/v*' }}" == 'true' ]]; then + latest-tag=${GITHUB_REF/refs\/tags\//} + fi + owner=$( echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]' ) + echo "Checkout: $latest_tag" + git checkout "$latest_tag" + tag=$( cat "$GITHUB_WORKSPACE/Docker/images-tag.info" | jq -rc .${{ matrix.service_name }}) + echo "image_name=ghcr.io/${owner}/frappe-manager-${{ matrix.service_name }}:${tag}" >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + platforms: ${{ matrix.platform }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ secrets.GHCR_USER }} + password: ${{ secrets.GHCR_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v4 + with: + context: Docker/${{ matrix.service_name }}/. + push: true + platforms: ${{ matrix.platform }} + tags: ${{ env.image_name }} diff --git a/.github/workflows/docker-push.yml b/.github/workflows/docker-push.yml deleted file mode 100644 index c3423dd2..00000000 --- a/.github/workflows/docker-push.yml +++ /dev/null @@ -1,37 +0,0 @@ -on: - workflow_dispatch: - inputs: - logLevel: - description: 'Log level' - required: true - default: 'warning' - type: choice - options: - - info - - warning - - debug - -name: build and push docker images -jobs: - docker: - runs-on: self-hosted - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push - uses: docker/build-push-action@v4 - with: - context: "{{defaultContext}}:docker-files/docker-images/frappe-docker" - platforms: linux/amd64,linux/arm64 - push: true - tags: ghcr.io/xieyt/fm-frappe:latest From 8b9d7d6d9106bcebb4f2590efa53b8638fadf0ad Mon Sep 17 00:00:00 2001 From: Xieyt Date: Thu, 29 Feb 2024 15:49:11 +0530 Subject: [PATCH 32/33] add images tag info --- Docker/images-tag.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 Docker/images-tag.json diff --git a/Docker/images-tag.json b/Docker/images-tag.json new file mode 100644 index 00000000..bbe7a28c --- /dev/null +++ b/Docker/images-tag.json @@ -0,0 +1,5 @@ +{ + "frappe": "v0.11.0", + "nginx": "v0.10.0", + "mailhog": "v0.8.3" +} From 47f9723132ab0a455279362bd92ee4348f8064b5 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Thu, 29 Feb 2024 15:50:22 +0530 Subject: [PATCH 33/33] Update workflow --- .github/workflows/bake-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bake-images.yml b/.github/workflows/bake-images.yml index e7372f9f..82a2ccff 100644 --- a/.github/workflows/bake-images.yml +++ b/.github/workflows/bake-images.yml @@ -38,7 +38,7 @@ jobs: owner=$( echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]' ) echo "Checkout: $latest_tag" git checkout "$latest_tag" - tag=$( cat "$GITHUB_WORKSPACE/Docker/images-tag.info" | jq -rc .${{ matrix.service_name }}) + tag=$( cat "$GITHUB_WORKSPACE/Docker/images-tag.json" | jq -rc .${{ matrix.service_name }}) echo "image_name=ghcr.io/${owner}/frappe-manager-${{ matrix.service_name }}:${tag}" >> $GITHUB_ENV - name: Set up Docker Buildx