-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #24 from aisingapore/0.3.0-pytorch-abstract
Abstracting Pytorch example, refactoring documentation (Check #24 for more details)
- Loading branch information
Showing
91 changed files
with
4,137 additions
and
1,683 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| import shutil | ||
| import os | ||
|
|
||
|
|
||
| def populate_problem(problem_domain: str) -> None: | ||
|
|
||
| SUB_DIRS = ["src", "conf", "notebooks", "aisg-context"] | ||
|
|
||
| for subdir in SUB_DIRS: | ||
|
|
||
| working_dir = os.getcwd() | ||
| src_dir = os.path.join(working_dir, "problem-templates", problem_domain) | ||
| shutil.copytree( | ||
| os.path.join(src_dir, subdir), | ||
| os.path.join(working_dir, subdir), | ||
| dirs_exist_ok=True | ||
| ) | ||
| shutil.copy2( | ||
| os.path.join(src_dir, "{{cookiecutter.repo_name}}-conda-env.yaml"), | ||
| os.path.join(working_dir, "{{cookiecutter.repo_name}}-conda-env.yaml") | ||
| ) | ||
|
|
||
|
|
||
| def generate_template_scripts() -> None: | ||
|
|
||
| PROBLEM_TEMPLATE = "{{cookiecutter.problem_template}}" | ||
|
|
||
| match PROBLEM_TEMPLATE: | ||
| case "base": | ||
| pass | ||
| case "cv": | ||
| populate_problem("cv") | ||
| case "nlp": | ||
| populate_problem("nlp") | ||
| case "tabular": | ||
| populate_problem("tabular") | ||
| case _: | ||
| raise ValueError(f"{PROBLEM_TEMPLATE} is not a valid problem template.") | ||
| shutil.rmtree(os.path.join(os.getcwd(), "problem-templates")) | ||
|
|
||
|
|
||
| def remove_redundant_files() -> None: | ||
|
|
||
| PLATFORM = "{{cookiecutter.platform}}" | ||
| ORCH = "{{cookiecutter.orchestrator}}" | ||
|
|
||
| BANNER_PATH = os.path.join( | ||
| os.getcwd(), "aisg-context", "guide-site", "docs" | ||
| ) | ||
| BANNERS = [ | ||
| 'kapitan-hull-eptg-gcp-runai-banner.png', | ||
| 'kapitan-hull-eptg-onprem-runai-banner.png' | ||
| ] | ||
| WORKFLOW_HTML_PATH = os.path.join( | ||
| BANNER_PATH, 'guide-for-user', 'assets' | ||
| ) | ||
| WORKFLOWS_HTML = [ | ||
| 'aisg-e2e-mlops-gcp-runai-workflow-components_jul2023.html', | ||
| 'aisg-e2e-mlops-onprem-runai-workflow-components_jul2023.html', | ||
| ] | ||
| WORKFLOW_PNG_PATH = os.path.join( | ||
| WORKFLOW_HTML_PATH, 'images' | ||
| ) | ||
| WORKFLOWS_PNG = [ | ||
| 'aisg-e2e-mlops-gcp-runai-workflow-components_jul2023.png', | ||
| 'aisg-e2e-mlops-onprem-runai-workflow-components_jul2023.png' | ||
| ] | ||
|
|
||
| # Remove all banners that doesn't have both PLATFORM and ORCH | ||
| for path, item in zip( | ||
| [BANNER_PATH, WORKFLOW_HTML_PATH, WORKFLOW_PNG_PATH], | ||
| [BANNERS, WORKFLOWS_HTML, WORKFLOWS_PNG] | ||
| ): | ||
| [os.remove( | ||
| os.path.join(path, x)) for x in item | ||
| if not (PLATFORM in x and ORCH in x | ||
| )] | ||
|
|
||
| # Remove runai yaml files if ORCH != runai | ||
| RUNAI_YAML_PATH = os.path.join( | ||
| os.getcwd(), "aisg-context", "runai" | ||
| ) | ||
| if ORCH != 'runai': | ||
| shutil.rmtree(RUNAI_YAML_PATH) | ||
|
|
||
| if __name__ == "__main__": | ||
| generate_template_scripts() | ||
| remove_redundant_files() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
55 changes: 55 additions & 0 deletions
55
rocm/docker/{{cookiecutter.repo_name}}-gpu-rocm.Dockerfile
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| FROM rocm/dev-ubuntu-20.04:5.7.1 | ||
|
|
||
| ARG DEBIAN_FRONTEND="noninteractive" | ||
|
|
||
| ARG NON_ROOT_USER="aisg" | ||
| ARG NON_ROOT_UID="2222" | ||
| ARG NON_ROOT_GID="2222" | ||
| ARG HOME_DIR="/home/${NON_ROOT_USER}" | ||
|
|
||
| ARG REPO_DIR="." | ||
| ARG CONDA_ENV_FILE="{{cookiecutter.repo_name}}-rocm-conda-env.yaml" | ||
| ARG CONDA_ENV_NAME="{{cookiecutter.repo_name}}" | ||
|
|
||
| # Miniconda arguments | ||
| ARG CONDA_HOME="/miniconda3" | ||
| ARG CONDA_BIN="${CONDA_HOME}/bin/conda" | ||
| ARG CONDA_VER="py310_23.5.2-0" | ||
| ARG CONDA_ARCH="Linux-x86_64" | ||
| ARG MINICONDA_SH="Miniconda3-${CONDA_VER}-${CONDA_ARCH}.sh" | ||
|
|
||
| RUN useradd -l -m -s /bin/bash -u ${NON_ROOT_UID} ${NON_ROOT_USER} && \ | ||
| mkdir -p ${CONDA_HOME} && \ | ||
| chown -R ${NON_ROOT_USER}:${NON_ROOT_GID} ${CONDA_HOME} | ||
|
|
||
| RUN apt-get update && \ | ||
| apt-get -y install bzip2 curl wget gcc rsync git vim locales && \ | ||
| sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ | ||
| locale-gen && \ | ||
| dpkg-reconfigure --frontend=noninteractive locales && \ | ||
| update-locale LANG=en_US.UTF-8 && \ | ||
| apt-get clean | ||
|
|
||
| ENV PYTHONIOENCODING utf8 | ||
| ENV LANG "C.UTF-8" | ||
| ENV LC_ALL "C.UTF-8" | ||
| ENV HIP_VISIBLE_DEVICES 0 | ||
| ENV LD_LIBRARY_PATH /opt/rocm/lib | ||
|
|
||
| USER ${NON_ROOT_USER} | ||
| WORKDIR ${HOME_DIR} | ||
|
|
||
| COPY --chown=${NON_ROOT_USER}:${NON_ROOT_GID} ${REPO_DIR} {{cookiecutter.repo_name}} | ||
|
|
||
| # Install Miniconda | ||
| RUN curl -O https://repo.anaconda.com/miniconda/${MINICONDA_SH} && \ | ||
| chmod +x ${MINICONDA_SH} && \ | ||
| ./${MINICONDA_SH} -u -b -p ${CONDA_HOME} && \ | ||
| rm ${MINICONDA_SH} | ||
| ENV PATH ${CONDA_HOME}/bin:${HOME_DIR}/.local/bin:$PATH | ||
|
|
||
| # Install conda environment | ||
| RUN ${CONDA_BIN} env create -f {{cookiecutter.repo_name}}/${CONDA_ENV_FILE} && \ | ||
| ${CONDA_BIN} init bash && \ | ||
| ${CONDA_BIN} clean -a -y && \ | ||
| echo "source activate ${CONDA_ENV_NAME}" >> "${HOME_DIR}/.bashrc" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| name: {{cookiecutter.repo_name}}-rocm | ||
| channels: | ||
| - defaults | ||
| - pytorch | ||
| - conda-forge | ||
| dependencies: | ||
| - python=3.11.7 | ||
| - pip=23.3.2 | ||
| - pip: | ||
| - mlflow-skinny==2.9.2 | ||
| - hydra-core==1.3.2 | ||
| - hydra-optuna-sweeper==1.2.0 | ||
| - python-json-logger==2.0.7 | ||
| - fastapi==0.109.0 | ||
| - uvicorn[standard]==0.25.0 | ||
| - python-multipart==0.0.6 | ||
| - jsonlines==4.0.0 | ||
| - pandas==2.1.4 | ||
| - gunicorn==21.2.0 | ||
| - pydantic==2.5.3 | ||
| - pydantic-settings==2.1.0 | ||
| - ipykernel==6.25.0 | ||
| - https://download.pytorch.org/whl/rocm5.7/torch-2.2.1%2Brocm5.7-cp311-cp311-linux_x86_64.whl | ||
| - https://download.pytorch.org/whl/rocm5.7/torchvision-0.17.1%2Brocm5.7-cp311-cp311-linux_x86_64.whl | ||
| - https://download.pytorch.org/whl/pytorch_triton_rocm-2.2.0-cp311-cp311-linux_x86_64.whl |
Oops, something went wrong.