diff --git a/.circleci/config.yml b/.circleci/config.yml index 532c900..41ffb2c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,21 +2,16 @@ version: 2.1 jobs: build: docker: - - image: circleci/python:3.6.15 + - image: cimg/python:3.9 working_directory: ~/skelebot steps: - checkout - run: - name: pip env - command: | - sudo pip install pipenv==2022.4.8 - pipenv install - - run: - name: install dependencies - command: pipenv run pip install -r ./requirements.txt + name: install package + command: pipenv run pip install .[dev] - run: name: run tests - command: pipenv run coverage run --source=skelebot setup.py test && pipenv run coverage xml -o codecov.xml + command: pipenv run coverage run --source=skelebot -m pytest && pipenv run coverage xml -o codecov.xml - run: name: upload coverage command: bash <(curl -s https://codecov.io/bash) -f codecov.xml diff --git a/.dockerignore b/.dockerignore index 829613b..1fa8c31 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,7 +3,6 @@ # Editing this file manually is not advised as all changes will be overwritten by Skelebot **/*.zip -**/*.RData **/*.pkl **/*.csv **/*.model diff --git a/.github/workflows/pypi-upload.yaml b/.github/workflows/pypi-upload.yaml index 890b2eb..1a15290 100644 --- a/.github/workflows/pypi-upload.yaml +++ b/.github/workflows/pypi-upload.yaml @@ -9,19 +9,19 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Install dependencies run: | - python -m pip install --upgrade pip - python -m pip install setuptools wheel + python3 -m pip install --upgrade pip + python3 -m pip install build - name: Build distribution - run: python setup.py sdist bdist_wheel + run: python3 -m build - name: Publish to PYPI - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.PYPI_PASSWORD }} diff --git a/CHANGELOG.md b/CHANGELOG.md index a2d5b75..c06a184 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,11 +3,28 @@ Documenting All Changes to the Skelebot Project --- -## v1.37.0 +## v2.0.0 #### Changed +- **Dependencies** | All project dependencies, regardless of source (individual packages, local or remote files, requirements files, pyproject scripts) are now installed together in a single `pip install ...` command. +- **Build system** | Refactored build system to use a single `pyproject.toml` script. Switched build backend from `setuptools` to `hatchling`. + +#### Removed +- **Config** | Removed now unnecessary `language` config variable. Existing projects may keep that variable and it will be ignored going forward. +- **Artifactory** | Removed old deprecated Artifactory component. +- **Python versions** | Support for python versions 3.6, 3.7, and 3.8, including base docker images, is **removed**. +- **R support** | All support for `R` and `R+Python` projects, including base docker images, is **removed**. + +--- + +## v1.37.0 +#### Merged: 2023-05-29 +#### Released: 2024-05-29 +#### Deprecated - **Python versions** | Support for python versions 3.6, 3.7, and 3.8, including base docker images, is **deprecated** and will be removed in skelebot 2.0 - **R support** | All support for `R` and `R+Python` projects, including base docker images, is **deprecated** and will be removed in skelebot 2.0 -- **Python 3.11** | Adding support for Python 3.11 + +#### Added +- **Python 3.11** | Adding support for Python 3.11. --- diff --git a/Dockerfile b/Dockerfile index 7b7cf54..899483e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,12 +2,11 @@ # This Dockerfile was generated by Skelebot # Editing this file manually is not advised as all changes will be overwritten by Skelebot -FROM skelebot/python-base:3.6 +FROM skelebot/python-base:3.9 MAINTAINER Sean Shookman WORKDIR /app ENV TZ=America/Chicago RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -COPY requirements.txt requirements.txt -RUN ["pip", "install", "-r", "/app/requirements.txt"] +RUN ["pip", "install", "PyYAML>=5.1.2", "dohq-artifactory>=0.1.17", "schema>=0.7.0", "colorama>=0.4.1", "boto3>=1.10", "tomli>=1.1.0 ; python_version < '3.11'", "pytest~=8.2", "coverage~=7.5"] COPY . /app ENTRYPOINT ["bash", "jobs/test.sh"] diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 9f6239b..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,11 +0,0 @@ -include skelebot/systems/scaffolding/templates/python/template.yaml - -include skelebot/systems/scaffolding/templates/python_dash/template.yaml -include skelebot/systems/scaffolding/templates/python_dash/files/app_py -include skelebot/systems/scaffolding/templates/python_dash/files/config_py -include skelebot/systems/scaffolding/templates/python_dash/files/server_py -include skelebot/systems/scaffolding/templates/python_dash/files/style_css - -include skelebot/systems/scaffolding/templates/r/template.yaml - -include skelebot/systems/scaffolding/templates/r_python/template.yaml diff --git a/README.md b/README.md index 22e9cb4..d2a8676 100644 --- a/README.md +++ b/README.md @@ -62,8 +62,8 @@ Skelebot is a command-line tool for developing machine learning projects and exe ``` [/code/my-iris-model] > skelebot -h -usage: skelebot [-h] [-e ENV] [-s] [-n] - {loadData,train,score,push,pull,jupyter,plugin,bump,prime,exec} +usage: skelebot [-h] [-v] [-e ENV] [-d HOST] [-s] [-n] [-c] [-V] + {loadData,train,score,push,pull,jupyter,plugin,bump,prime,exec,publish,envs} ... Iris Example @@ -71,29 +71,34 @@ Example Skelebot Project ----------------------------------- Version: 1.1.0 Environment: None -Skelebot Version: 1.8.5 +Skelebot Version: 2.0.0 ----------------------------------- positional arguments: - {loadData,train,score,push,pull,jupyter,plugin,bump,prime,exec} + {loadData,train,score,push,pull,jupyter,plugin,bump,prime,exec,publish,envs} loadData Load the Iris Dataset and save it into the data folder for the train job to access (src/loadData.py) train Use the data loaded in the loadData job to train the iris model (src/train.py) score Use the model that was built in the train job to score new data against the iris model (src/score.py) - push Push an artifact to artifactory - pull Pull an artifact from artifactory + push Push an artifact to Artifactory + pull Pull an artifact from Artifactory jupyter Spin up Jupyter in a Docker Container (port=8888, folder=.) plugin Install a plugin for skelebot from a local zip file bump Bump the skelebot.yaml project version prime Generate Dockerfile and .dockerignore and build the docker image exec Exec into the running Docker container + publish Publish your versioned Docker Image to the registry + envs Display the available environments for the project optional arguments: -h, --help show this help message and exit -v, --version Display the version number of Skelebot -e ENV, --env ENV Specify the runtime environment configurations + -d HOST, --docker-host HOST + Set the Docker Host on which the command will be executed -s, --skip-build Skip the build process and attempt to use previous docker build -n, --native Run natively instead of through Docker -c, --contact Display the contact email of the Skelebot project + -V, --verbose Print all job commands to the screen just before execution ``` ## Install diff --git a/VERSION b/VERSION index e4dce85..359a5b9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.37.0 \ No newline at end of file +2.0.0 \ No newline at end of file diff --git a/base-images/python-base/3.6/Dockerfile b/base-images/python-base/3.6/Dockerfile deleted file mode 100644 index 0a57df9..0000000 --- a/base-images/python-base/3.6/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM python:3.6.5-slim -MAINTAINER Sean Shookman - -# Install basic compilers and libraries commonly needed for downstream packages -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive \ - apt-get install -y -q build-essential libgomp1 && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN pip --no-cache-dir install -U pip -RUN pip --no-cache-dir install jupyter -RUN pip --no-cache-dir install jupyterlab diff --git a/base-images/python-base/3.7/Dockerfile b/base-images/python-base/3.7/Dockerfile deleted file mode 100644 index 827f66e..0000000 --- a/base-images/python-base/3.7/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM python:3.7-slim -MAINTAINER Sean Shookman - -# Install basic compilers and libraries commonly needed for downstream packages -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive \ - apt-get install -y -q build-essential libgomp1 && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN pip --no-cache-dir install -U pip -RUN pip --no-cache-dir install jupyter -RUN pip --no-cache-dir install jupyterlab diff --git a/base-images/python-base/3.8/Dockerfile b/base-images/python-base/3.8/Dockerfile deleted file mode 100644 index 5141b08..0000000 --- a/base-images/python-base/3.8/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM python:3.8-slim -MAINTAINER Sean Shookman - -# Install basic compilers and libraries commonly needed for downstream packages -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive \ - apt-get install -y -q build-essential libgomp1 && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN pip --no-cache-dir install -U pip -RUN pip --no-cache-dir install jupyter -RUN pip --no-cache-dir install jupyterlab diff --git a/base-images/r-aws/Dockerfile b/base-images/r-aws/Dockerfile deleted file mode 100644 index e7395ee..0000000 --- a/base-images/r-aws/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM skelebot/r-base -MAINTAINER Sean Shookman - -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive \ - apt-get install -y -q build-essential krb5-user libsasl2-dev libsasl2-modules-gssapi-mit && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN apt-get update -RUN apt-get install -y openjdk-11-jdk -RUN apt-get install -y libmariadb-dev-compat libmariadb-dev -RUN apt-get install -y curl -RUN apt-get install -y python3 -RUN apt-get install -y python3-pip - -RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" -RUN unzip awscliv2.zip -RUN ./aws/install - -RUN R CMD javareconf -RUN ["Rscript", "-e", "install.packages('rJava',repo='https://cloud.r-project.org');library(rJava)"] -RUN ["Rscript", "-e", "install.packages('RJDBC',repo='https://cloud.r-project.org');library(RJDBC)"] - -RUN ["python3", "-m", "pip", "install", "--no-use-pep51", "pyarrow"] -RUN ["pip3", "install", "s3fs==0.2.1"] -RUN ["pip3", "install", "pandas"] -RUN ["pip3", "install", "pyyaml==5.1.2"] diff --git a/base-images/r-base/Dockerfile b/base-images/r-base/Dockerfile deleted file mode 100644 index c4d6602..0000000 --- a/base-images/r-base/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM r-base:3.6.3 -MAINTAINER Sean Shookman - -RUN apt-get update -RUN apt-get install -y jupyter - -RUN ["Rscript", "-e", "install.packages('IRkernel', repo='https://cloud.r-project.org'); IRkernel::installspec()"] - -RUN apt-get install -y libcurl4-openssl-dev -RUN apt-get install -y libssl-dev -RUN apt-get install -y libxml2-dev -RUN apt-get install -y gfortran -RUN ["Rscript", "-e", "install.packages('glue', repo='https://cloud.r-project.org'); library(glue)"] -RUN ["Rscript", "-e", "install.packages('devtools', repo='https://cloud.r-project.org'); library(devtools)"] -RUN apt-get install -y python3-pip -RUN pip --no-cache-dir install jupyterlab diff --git a/base-images/r-krb/Dockerfile b/base-images/r-krb/Dockerfile deleted file mode 100644 index 3bbf21f..0000000 --- a/base-images/r-krb/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM skelebot/r-base -MAINTAINER Sean Shookman - -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive \ - apt-get install -y -q build-essential krb5-user libsasl2-dev libsasl2-modules-gssapi-mit && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN apt-get update -RUN apt-get install -y openjdk-8-jdk -RUN apt-get install -y libmariadbclient-dev -RUN apt-get install -y curl -RUN apt-get install -y libpq-dev -RUN R CMD javareconf -RUN ["Rscript", "-e", "install.packages('rJava',repo='https://cloud.r-project.org');library(rJava)"] -RUN ["Rscript", "-e", "install.packages('RJDBC',repo='https://cloud.r-project.org');library(RJDBC)"] -RUN ["Rscript", "-e", "install.packages('implyr',repo='https://cloud.r-project.org');library(implyr)"] -RUN ["Rscript", "-e", "install.packages('data.table',repo='https://cloud.r-project.org');library(data.table)"] - -RUN curl http://archive.cloudera.com/cdh5/cdh/5/hadoop-2.6.0-cdh5.13.3.tar.gz -o /etc/hadoop.tar.gz -RUN tar -xvzf /etc/hadoop.tar.gz - -COPY jars.zip /etc/CDH/ -WORKDIR /etc/CDH/ -RUN unzip jars.zip -WORKDIR / - -COPY init.sh /krb/ -RUN chmod +x /krb/init.sh diff --git a/base-images/r-krb/init.sh b/base-images/r-krb/init.sh deleted file mode 100644 index beff976..0000000 --- a/base-images/r-krb/init.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -user=$1 -kinit -k -t /krb/auth.keytab $user diff --git a/base-images/r-redshift/Dockerfile b/base-images/r-redshift/Dockerfile deleted file mode 100644 index 398f551..0000000 --- a/base-images/r-redshift/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM skelebot/r-base -MAINTAINER Joao Moreira - -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive \ - apt-get install -y --no-install-recommends openjdk-8-jdk git curl && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Install the Redshift driver -RUN R CMD javareconf -RUN ["Rscript", "-e", "install.packages('RJDBC',repo='https://cloud.r-project.org');library(RJDBC)"] -RUN mkdir -p /usr/lib/redshift/lib && \ - cd /usr/lib/redshift/lib && \ - curl -O http://s3.amazonaws.com/redshift-downloads/drivers/RedshiftJDBC41-1.1.9.1009.jar - -# Install the AWS CLI -RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ - unzip awscliv2.zip && \ - ./aws/install && \ - rm -rf aws awscliv2.zip diff --git a/docs/api.md b/docs/api.md index f2bb644..bd3b1f4 100644 --- a/docs/api.md +++ b/docs/api.md @@ -3,7 +3,7 @@ ---

Skelebot API

-
Version 1
+
Version 2
--- @@ -12,12 +12,12 @@ Skelebot exposes a number of functions that can be called inside of a plugin in order to execute and leverage important parts of the Skelebot System. -This document specifies the API contract for v1 of Skelebot. +This document specifies the API contract for v2 of Skelebot. There are packages and classes outside of this API that are technically accessible via Skelebot, -but since they are not included in this specification, they are not part of the v1 contract for +but since they are not included in this specification, they are not part of the v2 contract for Skelebot. As such it is not advised to use any function or Class that is not specified in this -document because it may be subject to change throughout iterations of v1. +document because it may be subject to change throughout iterations of v2. --- @@ -38,15 +38,4 @@ document because it may be subject to change throughout iterations of v1. --- -### Updates - -_v1.1.0 Additions_ - -- SkeleYaml - - schema (attribute) - - loadList (method) - - validate (method) - ---- -
<< Timezone | Home >>
diff --git a/docs/api/common.md b/docs/api/common.md index e446099..b4829bc 100644 --- a/docs/api/common.md +++ b/docs/api/common.md @@ -3,7 +3,7 @@ ---

Skelebot API

-
Version 1
+
Version 2
--- diff --git a/docs/api/component.md b/docs/api/component.md index a1e628b..a9e7272 100644 --- a/docs/api/component.md +++ b/docs/api/component.md @@ -3,7 +3,7 @@ ---

Skelebot API

-
Version 1
+
Version 2
--- diff --git a/docs/api/docker.md b/docs/api/docker.md index 226aa54..d685fe1 100644 --- a/docs/api/docker.md +++ b/docs/api/docker.md @@ -3,7 +3,7 @@ ---

Skelebot API

-
Version 1
+
Version 2
--- diff --git a/docs/api/dockerfile.md b/docs/api/dockerfile.md index e9b8336..d718a30 100644 --- a/docs/api/dockerfile.md +++ b/docs/api/dockerfile.md @@ -3,7 +3,7 @@ ---

Skelebot API

-
Version 1
+
Version 2
--- diff --git a/docs/api/dockerignore.md b/docs/api/dockerignore.md index f6b42f0..df358ce 100644 --- a/docs/api/dockerignore.md +++ b/docs/api/dockerignore.md @@ -3,7 +3,7 @@ ---

Skelebot API

-
Version 1
+
Version 2
--- diff --git a/docs/api/prompt.md b/docs/api/prompt.md index 923bcee..4afe0b6 100644 --- a/docs/api/prompt.md +++ b/docs/api/prompt.md @@ -3,7 +3,7 @@ ---

Skelebot API

-
Version 1
+
Version 2
--- @@ -50,5 +50,5 @@ This function can be used during a component's scaffolding process in order to g user to populate the attributes of a component. By default, the function will allow for any String of text to be entered as a response. The options parameter allows for the prompt to lock the user's response to a list of choices. The boolean parameter, if set to True, will ensure that the user's -response is either True or False. +response is either True or False. ``` diff --git a/docs/api/skeleyaml.md b/docs/api/skeleyaml.md index 53ea073..34c67d3 100644 --- a/docs/api/skeleyaml.md +++ b/docs/api/skeleyaml.md @@ -3,7 +3,7 @@ ---

Skelebot API

-
Version 1
+
Version 2
--- diff --git a/docs/api/yaml.md b/docs/api/yaml.md index 4bf862d..af5d58f 100644 --- a/docs/api/yaml.md +++ b/docs/api/yaml.md @@ -3,7 +3,7 @@ ---

Skelebot API

-
Version 1
+
Version 2
--- diff --git a/docs/artifacts.md b/docs/artifacts.md index 247e5f0..3231530 100644 --- a/docs/artifacts.md +++ b/docs/artifacts.md @@ -2,29 +2,6 @@ --- -# Artifactory - DEPRECATED v1.11 - -**The Artifactory component has been deprecated as of v1.11 and should no longer be used. The same functionality (and more) is provided with the Repository component as detailed below. The manner in which the components operate is identical (push and pull commands), they merely utilize a different config structure that allows the Repository component to handle more than just Artifactory repositories.** - ---- - -~~Skelebot currently only supports deploying artifacts to Artifactory. This can be setup by editing the skelebot.yaml file and adding the following `artifactory` section to the components section of the config.~~ - -``` -components: - artifactory: - url: http://my-host:5000/artifactory - repo: my-repo - path: path/to/artifact/folder - artifacts: - name: artifact-name - file: path/to/artifact.ext -``` - -~~The artifacts field accepts a list of artifacts names and path to the actual artifact object file. The url, repo, and path fields specify where the artifact will end up when it is pushed (or from where it will be pulled).~~ - ---- - # Repository Skelebot supports the management of artifacts in either Artifactory or S3 with the Repository component. This can be setup by editing the skelebot.yaml file and adding the following `repository` section to the components section of the config. diff --git a/docs/base-images.md b/docs/base-images.md index 4edcae4..9eb221f 100644 --- a/docs/base-images.md +++ b/docs/base-images.md @@ -4,19 +4,24 @@ # Base Images -By default Skelebot will select and use a base image based on the language you have selected for the project (Python or R). +By default Skelebot will select and use a base image based on python 3.9. This can be customized by adding the `pythonVersion` config field to the top level of the skelebot.yaml: -### Without Language +``` +pythonVersion: '3.11' +``` + +The desired Python version should be specified as a string. + +As of Skelebot v2.0.0, the available versions are 3.9, 3.10, 3.11. -If no language is specified, the image will default to ubuntu:18.04 and certain features, such as dependency management, may not work. ### Custom Base Image ``` -baseImage: ubuntu:16.04 +baseImage: ubuntu:22.04 ``` -A custom base image can be specified by adding the `baseImage` config field to the top level of the skelebot.yaml. This custom image can be anything, but will need to have the right language based dependencies installed in order to function properly. The custom base image provides flexibility, but the behavior of Skelebot cannot be garunteed when using a custom base image. +A custom base image can be specified by adding the `baseImage` config field to the top level of the skelebot.yaml. This custom image can be anything, but will need to have the right language based dependencies installed in order to function properly. The custom base image provides flexibility, but the behavior of Skelebot cannot be guaranteed when using a custom base image. --- diff --git a/docs/dependencies.md b/docs/dependencies.md index aebec60..1df418a 100644 --- a/docs/dependencies.md +++ b/docs/dependencies.md @@ -4,52 +4,39 @@ # Dependencies -For both R and Python projects, Skelebot offers the ability to list your dependencies directly in the skelebot.yaml file and have these dependencies installed into your Docker image automatically. +Skelebot offers the ability to list your Python dependencies directly in the skelebot.yaml file and have these dependencies installed into your Docker image automatically. ``` ... dependencies: -- data.table=1.11.2 -- stringr +- numpy=1.11.2 +- pandas - ... ``` -By default R dependencies are installed using install.packages from CRAN. +By default dependencies are installed using pip install. -By default Python dependencies are installed using pip install. +Versions for packages can be specified by appending `={version}`, `=={version}`, `>={version}`, etc. to the end of the dependency name. Skelebot supports most of the standard python [version specifiers](https://packaging.python.org/en/latest/specifications/version-specifiers/#id5). -Versions for packages in R can be specified by appending `={version}` to the end of the dependency name. - -Versions for packages in Python can be specified by appending `={version}` or `=={version}` to the end of the dependency name. - -R and Python also both support dependencies to be installed from the local file system as well as from GitHub using the following structure. - -Python also allows for installs using a text file via `req:requirements.txt` syntax or using a [`pyproject.toml`](https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html) via `proj:pyproject.toml` syntax. When using the later, all specified required and optional set(s) of dependencies will be installed. +Skelebot also supports dependencies to be installed from the local file system, GitHub repo, requirements file, or [`pyproject.toml`](https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html) file: ``` -language: R dependencies: -- {type}:{source}:{name} -- file:libs/myPackage.tgz:mypack -- github:myGitHub/fakeRepo:fakeRepo -``` - -``` -language: Python -dependencies: -- {type}:{source} +- req:requirement.txt +- proj:pyproject.toml - file:libs/myPackage.tgz - req:requirements.txt - github:myGitHub/fakeRepo - proj:pyproject.toml ``` +When using a pyproject file, *all* specified required and optional set(s) of dependencies will be installed. + ### CodeArtifact Python Packages Skelebot also supports pulling Python packages that are stored in AWS CodeArtifact. This requires a good deal of information in order to authenticate and pull the correct asset for the package. ``` -language: Python dependencies: - ca_file:{domain}:{owner}:{repo}:{pkg}:{version}:{profile} ``` diff --git a/docs/docker-ignores.md b/docs/docker-ignores.md index 3548eb4..980e2f7 100644 --- a/docs/docker-ignores.md +++ b/docs/docker-ignores.md @@ -9,7 +9,7 @@ When a docker build is run, any file contained within the root folder of the pro ``` ... ignores: -- '**/*.RData' +- '**/*.pyc' - '**/*.pkl' - '**/*.csv' - '**/*.model' diff --git a/docs/help-info.md b/docs/help-info.md index f435641..7189360 100644 --- a/docs/help-info.md +++ b/docs/help-info.md @@ -13,17 +13,18 @@ The most important command in Skelebot is the help command. This command will pr If this command is executed from inside a folder that is not a Skelebot project, you will be met with a simple message stating that your only options from this particular directory are to scaffold a new project, or install a plugin. ``` -usage: skelebot [-h] {plugin,scaffold} ... +usage: skelebot [-h] [-v] {scaffold,plugin} ... -Skelebot Version: 1.0.0 +Skelebot Version: 2.0.0 positional arguments: - {plugin,scaffold} + {scaffold,plugin} + scaffold Scaffold a new or existing project with Skelebot plugin Install a plugin for skelebot from a local zip file - scaffold Scaffold a new skelebot project from scratch optional arguments: -h, --help show this help message and exit + -v, --version Display the version number of Skelebot ``` If the help command is run from inside a Skelebot project, the output looks quite different. The scaffold option is no longer available, as it is not needed, and the rest of the Standard Tasks are now present. More details for each task can be obtained by running the desired command with `-h` appended. @@ -33,37 +34,41 @@ There are also more optional arguments available for these tasks, which allows y NOTE: The Artifactory tasks (push and pull) will only be present if artifacts are configuring in the skelebot.yaml file of the project. ``` -usage: skelebot [-h] [-e ENV] [-s] [-d] - {plugin,bump,prime,exec,jupyter,push,pull,loadData,train,score} - ... +usage: skelebot [-h] [-v] [-e ENV] [-d HOST] [-s] [-n] [-c] [-V] {loadData,train,score,push,pull,jupyter,plugin,bump,prime,exec,publish,envs} ... Iris Example Example Skelebot Project ----------------------------------- -Version: 0.1.0 +Version: 1.1.0 Environment: None -Skelebot Version: 1.8.1 +Skelebot Version: 2.0.0 ----------------------------------- positional arguments: - {plugin,bump,prime,exec,jupyter,push,pull,loadData,train,score} - plugin Install a plugin for skelebot from a local zip file - bump Increment the version of the project - prime Prime skelebot with latest config - exec Start the Docker container and access it via bash - jupyter Start a Jupyter notebook inside of Docker - push Push an artifact to artifactory - pull Pull an artifact from artifactory + {loadData,train,score,push,pull,jupyter,plugin,bump,prime,exec,publish,envs} loadData Load the Iris Dataset and save it into the data folder for the train job to access (src/loadData.py) train Use the data loaded in the loadData job to train the iris model (src/train.py) score Use the model that was built in the train job to score new data against the iris model (src/score.py) + push Push an artifact to Artifactory + pull Pull an artifact from Artifactory + jupyter Spin up Jupyter in a Docker Container (port=8888, folder=.) + plugin Install a plugin for skelebot from a local zip file + bump Bump the skelebot.yaml project version + prime Generate Dockerfile and .dockerignore and build the docker image + exec Exec into the running Docker container + publish Publish your versioned Docker Image to the registry + envs Display the available environments for the project optional arguments: -h, --help show this help message and exit + -v, --version Display the version number of Skelebot -e ENV, --env ENV Specify the runtime environment configurations + -d HOST, --docker-host HOST + Set the Docker Host on which the command will be executed -s, --skip-build Skip the build process and attempt to use previous docker build -n, --native Run natively instead of through Docker - -v, --version Display the version number of Skelebot + -c, --contact Display the contact email of the Skelebot project + -V, --verbose Print all job commands to the screen just before execution ``` ### Version Parameter @@ -71,7 +76,7 @@ The version of Skelebot is printed in the help output, but sometimes that is the ``` > skelebot --version -Skelebot v.1.8.1 +Skelebot v.2.0.0 ``` ### Contact Parameter diff --git a/docs/index.md b/docs/index.md index be36c1b..e707a6d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,9 +4,9 @@ Machine learning projects are too often just a loose collection of unorganized s ### Purpose -Skelebot is a command-line tool for developing machine learning projects and executing them in Docker. The purpose of Skelebot is to simply make the life of a Data Scientist easier by doing a lot of the legwork for mundane tasks automatically through a unified, consistent interface. +Skelebot is a command-line tool for developing Python machine learning projects and executing them in Docker. The purpose of Skelebot is to simply make the life of a Data Scientist easier by doing a lot of the legwork for mundane tasks automatically through a unified, consistent interface. -By allowing jobs to be executed in Docker, it removes the need for the developer to install specific R and Python packages on their own machine, and even removes the need to have R installed at all. Configured jobs in Skelebot also come with the added benefit of built-in help documentation in order to assist others in understanding what jobs your project has, and what those jobs do. +By allowing jobs to be executed in Docker, it removes the need for the developer to install specific Python packages on their own machine. Configured jobs in Skelebot also come with the added benefit of built-in help documentation in order to assist others in understanding what jobs your project has, and what those jobs do. Skelebot also saves developer time by integrating with HDFS through Kerberos automatically. By building on top of a library of pre-built Docker images tailored specifically for Skelebot’s purposes, the process of building the Docker image for a project is greatly reduced. Skelebot also encourages a specific folder structure through it’s scaffolding process thereby introducing consistency across projects and developers. By providing a uniform interface on which to discover the project's jobs it greatly helps to reduce the barrier to entry for newcomers to the project. diff --git a/docs/installing.md b/docs/installing.md index b5a2052..c6a6509 100644 --- a/docs/installing.md +++ b/docs/installing.md @@ -4,7 +4,7 @@ # Installing -Skelebot requires Python version 3.6 or later to run. +Skelebot requires Python version 3.9 or later to run. ### Pip Install @@ -17,17 +17,26 @@ Skelebot can be installed via pip. ### Install From Source To install from source, copy or clone the repository onto your machine. Then, navigate -to the root of the project and execute the install.sh script. +to the root of the project and execute this command: ``` -> ./jobs/install.sh +> pip install . ``` -If you do not have proper access on the machine you are using, Skelebot can be installed locally with the following command. +If you do not have proper access on the machine you are using, Skelebot can be installed locally with the following command: ``` -> python setup.py clean --all install --user +> pip install --user . +``` + +### Development Install + +Skelebot developers should first install the package with the additional `dev` dependencies and then test their installation: + +``` +> pip install .[dev] +> pytest . ``` # Executing diff --git a/docs/jobs.md b/docs/jobs.md index e25cd1f..2becfbe 100644 --- a/docs/jobs.md +++ b/docs/jobs.md @@ -36,10 +36,10 @@ jobs: ... ``` -A job must contain three things in order to work. It must have a name, so you can call it from the command line, a source file to execute, and a help message for users to understand it. Python projects must utilize the `.py` extension while R projects must utilize the `.R` extension, but Bash scripts `.sh` are supported for both Python and R projects. Jobs can contain several additional fields: +A job must contain three things in order to work. It must have a name, so you can call it from the command line, a source file to execute, and a help message for users to understand it. Skelebot supports calling both Python `.py` scripts and also Bash scripts `.sh`. Jobs can contain several additional fields: - **name** - The name that is used to execute the job from the command line - - **source** - Command to be executed(ex: echo 'Hello') or the path to the script (R, Python, or Bash) that will be executed + - **source** - Command to be executed(ex: echo 'Hello') or the path to the script (Python, or Bash) that will be executed - **help** - Text that will be displayed when the -h (--help) parameter is passed - **mode** - The mode in which to execute the docker image [i: interactive(default), d: detached] - **native** - Optional parameter to specify if the job should 'always' be run natively or 'never' be run natively [always, never, optional(default)] diff --git a/docs/jupyter.md b/docs/jupyter.md index 7e220e5..3264e0c 100644 --- a/docs/jupyter.md +++ b/docs/jupyter.md @@ -21,7 +21,7 @@ All of these values are optional. By default the port will be set to 8888, the f components: jupyter: port: 1127 - folder: my-notebooks/r-notebooks + folder: my-notebooks/python-notebooks lab: False ... ``` diff --git a/docs/plugins.md b/docs/plugins.md index 145161b..367bb74 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -56,8 +56,7 @@ class Jupyter(Component): status = docker.build(config) if (status == 0): - root = " --allow-root" if config.language == "R" else "" - command = "jupyter notebook --ip=0.0.0.0 --port=8888{root} --notebook-dir={folder}".format(root=root, folder=self.folder) + command = "jupyter notebook --ip=0.0.0.0 --port=8888 --allow-root --notebook-dir={folder}".format(folder=self.folder) ports = ["{port}:8888".format(port=self.port)] return docker.run(config, command, "i", ports, ".", "jupyter") diff --git a/docs/scaffolding.md b/docs/scaffolding.md index 89fa753..5ada9c3 100644 --- a/docs/scaffolding.md +++ b/docs/scaffolding.md @@ -35,7 +35,6 @@ Enter a PROJECT NAME (no spaces): my-ml-project Enter a PROJECT DESCRIPTION: A Machine Learning Project Enter a MAINTAINER NAME: Firstname Lastname Enter a CONTACT EMAIL: flastname@gmail.com -Select a LANGUAGE [Python, R]: Python Select a TEMPLATE [Default, Dash, Git]: Dash -:---:---:---:---:---:---:---:---:---:-- SKELEBOT --:---:---:---:---:---:---:---:---:---:- Setting up the my-ml-project Skelebot project in the current directory diff --git a/docs/template.md b/docs/template.md index 7b60516..446b65a 100644 --- a/docs/template.md +++ b/docs/template.md @@ -32,12 +32,11 @@ files: template: files/trainer.py config: - language: {language} dependencies: - pandas~=1.1 - numpy~=1.19 commands: - - "python setup.py install" + - "pip install ." primaryJob: score jobs: - name: train @@ -73,13 +72,12 @@ The default scaffolding process will prompt for and store a number of variables - **description** - The PROJECT DESCRIPTION - **maintainer** - The MAINTAINER NAME - **contact** - The CONTACT EMAIL -- **language** - The PROJECT LANGUAGE -These variables, as well as any variables setup by the template's `prompts`, can be used within the template.yaml and any file templates as well by using the `{varable_name}` syntax. +These variables, as well as any variables setup by the template's `prompts`, can be used within the template.yaml and any file templates as well by using the `{variable_name}` syntax. ``` config: - language: {language} + name: {name} ``` ### Scaffolding Directories @@ -117,12 +115,11 @@ The skelebot.yaml config file can also be scaffolded with the template as well. ``` config: - language: {language} dependencies: - pandas~=1.1 - numpy~=1.19 commands: - - "python setup.py install" + - "pip install ." primaryJob: score jobs: - name: train diff --git a/docs/versioning.md b/docs/versioning.md index bc3a128..423e9cd 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -6,7 +6,7 @@ Skelebot offers a way to manage the version of your project by keeping it in a standalone `VERSION` file in the project folder. -The reason this file stands alone is to allow for the version to be read more easily from a variety of different sources (GitHub README, python setup.py script, etc.). +The reason this file stands alone is to allow for the version to be read more easily from a variety of different sources (GitHub README, python pyproject file, etc.). By default the project will start with version 0.1.0 (initial pre-release version) but this can be changed manually or updated through the bump command. It is recommended to use the bump command since it will follow the standards of semantic versioning for you. diff --git a/example/.dockerignore b/example/.dockerignore index 6e04315..38a9ae0 100644 --- a/example/.dockerignore +++ b/example/.dockerignore @@ -1,2 +1,4 @@ + # This dockerignore was generated by Skelebot -# Editing this file manually is not advised as all changes will be overwritten during Skelebot execution +# Editing this file manually is not advised as all changes will be overwritten by Skelebot + diff --git a/example/.gitignore b/example/.gitignore index 1d66a6c..2ad08ac 100644 --- a/example/.gitignore +++ b/example/.gitignore @@ -23,3 +23,4 @@ vignettes/*.pdf *.utf8.md *.knit.md *.pkl +*.txt diff --git a/example/Dockerfile b/example/Dockerfile index 25fd2c2..7aeaf73 100644 --- a/example/Dockerfile +++ b/example/Dockerfile @@ -1,10 +1,9 @@ + # This Dockerfile was generated by Skelebot -# Editing this file manually is not advised as all changes will be overwritten during Skelebot execution -FROM skelebot/python-base +# Editing this file manually is not advised as all changes will be overwritten by Skelebot + +FROM skelebot/python-base:3.9 MAINTAINER Sean Shookman WORKDIR /app -RUN ["pip", "install", "numpy"] -RUN ["pip", "install", "pandas"] -RUN ["pip", "install", "scipy"] -RUN ["pip", "install", "scikit-learn"] +RUN ["pip", "install", "numpy", "pandas", "scipy", "scikit-learn"] COPY . /app diff --git a/example/skelebot.yaml b/example/skelebot.yaml index 691ff29..cc56745 100644 --- a/example/skelebot.yaml +++ b/example/skelebot.yaml @@ -1,11 +1,12 @@ components: - artifactory: + repository: + artifactory: + path: models + repo: iris-example + url: http://my-artifactory-url.com artifacts: - file: models/model.pkl name: model - path: models - repo: iris-example - url: http://my-artifactory-url.com jupyter: folder: . port: 8888 @@ -34,7 +35,7 @@ jobs: help: the algorithm to use for modeling (only glm currently supported) help: Use the data loaded in the loadData job to train the iris model mappings: - - /Users/seanshookman/Code/open-source/skelebot/example/models/:/app/models/ + - models/ mode: i name: train params: @@ -50,7 +51,7 @@ jobs: - help: Use the model that was built in the train job to score new data against the iris model mappings: - - ~/Code/open-source/skelebot/example/scored/:/app/scored/ + - scored mode: i name: score params: @@ -63,6 +64,5 @@ jobs: name: output help: the name of the output file to be written source: src/score.py -language: Python maintainer: Sean Shookman name: iris-example diff --git a/example/src/score.py b/example/src/score.py index f1e42bc..cf34efa 100644 --- a/example/src/score.py +++ b/example/src/score.py @@ -8,10 +8,11 @@ args = parser.parse_args() filename = "./models/{name}.pkl".format(name=args.name) -outfile = "./scored/{output}.pkl".format(output=args.output) +outfile = "./scored/{output}.txt".format(output=args.output) print("Loading The Model ({})".format(filename)) -model = pickle.load(open(filename, "rb" ) ) +with open(filename, "rb") as fobj: + model = pickle.load(fobj) print("Scoring") Xnew = [[0.79415228, 2.10495117, 0.79415228, 2.10495117]] @@ -19,6 +20,5 @@ results = "X={val}, Predicted={pred}\n".format(val=Xnew[0], pred=ynew[0]) print("Saving Results ({})".format(outfile)) -text_file = open(outfile, "w") -text_file.write(results) -text_file.close() +with open(outfile, "w", encoding="utf-8") as text_file: + text_file.write(results) diff --git a/example/src/train.py b/example/src/train.py index e2fbb91..9d376fd 100644 --- a/example/src/train.py +++ b/example/src/train.py @@ -24,6 +24,7 @@ model.fit(df[['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']], df['target']) print("Saving Model") - pickle.dump(model, open(filename, 'wb')) + with open(filename, 'wb') as fobj: + pickle.dump(model, fobj) else: print("ALGORITHM NOT YET SUPPORTED") diff --git a/jobs/build.sh b/jobs/build.sh index 22896f2..b1a4ebc 100755 --- a/jobs/build.sh +++ b/jobs/build.sh @@ -1,20 +1,5 @@ -# Build r-base -docker build -t skelebot/r-base base-images/r-base/ - -# Build r-krb -docker build -t skelebot/r-krb base-images/r-krb/ - -# Build r-aws -docker build -t skelebot/r-aws base-images/r-aws/ - -# Build r-redshift -docker build -t skelebot/r-redshift base-images/r-redshift/ - # Build python-base -docker build -t skelebot/python-base:3.6 -t skelebot/python-base:latest base-images/python-base/3.6/ -docker build -t skelebot/python-base:3.7 base-images/python-base/3.7/ -docker build -t skelebot/python-base:3.8 base-images/python-base/3.8/ -docker build -t skelebot/python-base:3.9 base-images/python-base/3.9/ +docker build -t skelebot/python-base:3.9 -t skelebot/python-base:latest base-images/python-base/3.9/ docker build -t skelebot/python-base:3.10 base-images/python-base/3.10/ docker build -t skelebot/python-base:3.11 base-images/python-base/3.11/ diff --git a/jobs/install.sh b/jobs/install.sh index 75dc49e..438c949 100755 --- a/jobs/install.sh +++ b/jobs/install.sh @@ -1 +1 @@ -python setup.py clean --all install test +pip install .[dev] diff --git a/jobs/publish.sh b/jobs/publish.sh index 9d9e51e..f6f0002 100755 --- a/jobs/publish.sh +++ b/jobs/publish.sh @@ -1,31 +1,12 @@ # Login to Docker Hub with Skelebot user docker login -u skelebot -# Build and Publish r-base -docker build -t skelebot/r-base base-images/r-base/ -docker push skelebot/r-base:latest - -# Build and Publish r-krb -docker build -t skelebot/r-krb base-images/r-krb/ -docker push skelebot/r-krb:latest - -# Build and Publish r-aws -docker build -t skelebot/r-aws base-images/r-aws/ -docker push skelebot/r-aws:latest - -# Build and Publish r-redshift -docker build -t skelebot/r-redshift base-images/r-redshift/ -docker push skelebot/r-redshift:latest - # Build and Publish python-base -docker build -t skelebot/python-base:3.6 -t skelebot/python-base:latest base-images/python-base/3.6/ -docker build -t skelebot/python-base:3.7 base-images/python-base/3.7/ -docker build -t skelebot/python-base:3.8 base-images/python-base/3.8/ -docker build -t skelebot/python-base:3.9 base-images/python-base/3.9/ +docker build -t skelebot/python-base:3.9 -t skelebot/python-base:latest base-images/python-base/3.9/ docker build -t skelebot/python-base:3.10 base-images/python-base/3.10/ docker build -t skelebot/python-base:3.11 base-images/python-base/3.11/ # Newer Docker clients will push latest by default and will need to do `docker push skelebot/python-base --all-tags` -docker push skelebot/python-base +docker push skelebot/python-base --all-tags # Build and Publish python-krb docker build -t skelebot/python-krb base-images/python-krb/ diff --git a/jobs/test.sh b/jobs/test.sh index 1f645c2..4d690f0 100755 --- a/jobs/test.sh +++ b/jobs/test.sh @@ -1,6 +1,7 @@ +pip install .[dev] if [ "--coverage" = "$1" ] then - coverage run --source=skelebot setup.py test && coverage report -m + coverage run --source=skelebot -m pytest && coverage report -m else - python setup.py test + pytest . fi diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..1b97325 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,54 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "skelebot" +description = "ML Build Tool" +readme = "README.md" +license = {file = "LICENSE"} +authors = [ + {name = "Sean Shookman", email = "sshookman@cars.com"}, + {name = "Joao Moreira", email = "jmoreira@cars.com"}, +] +requires-python = ">=3.9" +dependencies = [ + "PyYAML>=5.1.2", + "dohq-artifactory>=0.1.17", + "schema>=0.7.0", + "colorama>=0.4.1", + "boto3>=1.10", + "tomli>=1.1.0 ; python_version < '3.11'", +] +dynamic = ["version"] + +[project.optional-dependencies] +# Development-only dependencies +dev = [ + "pytest~=8.2", + "coverage~=7.5", +] + +[project.urls] +Homepage = "https://github.com/carsdotcom/skelebot" +Documentation = "https://carsdotcom.github.io/skelebot/" +Issues = "https://github.com/carsdotcom/skelebot/issues" +Changelog = "https://github.com/carsdotcom/skelebot/blob/master/CHANGELOG.md" + +[project.scripts] +skelebot = "skelebot:main" + +[tool.hatch.version] +path = "VERSION" +pattern = "(?P[^']+)" + +[tool.hatch.build.targets.sdist] +include = [ + "skelebot/**/*.py", + "skelebot/systems/scaffolding/templates/*", + "/test", + "/VERSION", +] + +[tool.hatch.build.targets.wheel] +packages = ["skelebot"] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index fda7039..0000000 --- a/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -PyYAML>=5.1.2 -dohq-artifactory>=0.1.17 -requests>=2.22.0 -schema~=0.7.0 -colorama~=0.4.1 -coverage~=4.5.4 -pytest~=5.1 -boto3~=1.10 -tomli >= 1.1.0 ; python_version < "3.11" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index b7e4789..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[aliases] -test=pytest diff --git a/setup.py b/setup.py deleted file mode 100644 index fa7bf55..0000000 --- a/setup.py +++ /dev/null @@ -1,27 +0,0 @@ -from setuptools import setup, find_packages - -VERSION = '0.0.0' -with open('VERSION', 'r') as version: - VERSION = version.read().replace("\n", "") - -with open('requirements.txt') as f: - requirements = f.read().splitlines() - -setup( - name="skelebot", - version=VERSION, - description="ML Build Tool", - author="Sean Shookman", - author_email="sshookman@cars.com", - packages=find_packages(), - include_package_data=True, - zip_safe=False, - setup_requires=["pytest-runner"], - tests_require=requirements, - install_requires=requirements, - entry_points={ - 'console_scripts': [ - 'skelebot = skelebot:main', - ], - } -) diff --git a/skelebot-exp.yaml b/skelebot-exp.yaml index 87c64e4..25566b2 100644 --- a/skelebot-exp.yaml +++ b/skelebot-exp.yaml @@ -17,7 +17,6 @@ dependencies: - tomli>=1.1.0 ignores: - '**/*.zip' -- '**/*.RData' - '**/*.pkl' - '**/*.csv' - '**/*.model' diff --git a/skelebot.yaml b/skelebot.yaml index 3a5038b..72edb2b 100644 --- a/skelebot.yaml +++ b/skelebot.yaml @@ -5,13 +5,11 @@ contact: sshookman@cars.com ephemeral: False timezone: America/Chicago -language: Python dependencies: -- req:requirements.txt + - proj:pyproject.toml ignores: - '**/*.zip' -- '**/*.RData' - '**/*.pkl' - '**/*.csv' - '**/*.model' diff --git a/skelebot/common.py b/skelebot/common.py index f0b17fb..205ead3 100644 --- a/skelebot/common.py +++ b/skelebot/common.py @@ -1,10 +1,10 @@ """Common Global Variables""" -import pkg_resources +from importlib import metadata from colorama import Fore, Style -VERSION = pkg_resources.get_distribution("skelebot").version +VERSION = metadata.version("skelebot") DESCRIPTION = Style.BRIGHT + "{project}" + Style.RESET_ALL + """ {desc} ----------------------------------- @@ -17,51 +17,19 @@ PLUGINS_QUARANTINE = "{home}/plugins-quarantine".format(home=SKELEBOT_HOME) LANGUAGE_IMAGE = { - "NA": { - "base": "ubuntu:18.04", - "krb": "ubuntu:18.04" - }, - "Python": { - "base": "skelebot/python-base:{pythonVersion}", - "krb": "skelebot/python-krb" - }, - "R": { - "base": "skelebot/r-base", - "krb": "skelebot/r-krb" - }, - "R+Python": { - "base": "skelebot/r-base", - "krb": "skelebot/r-krb" - } + "base": "skelebot/python-base:{pythonVersion}", + "krb": "skelebot/python-krb" } -_all_deps: dict = {"Python":["numpy", "pandas", "scipy", "scikit-learn"],"R":["data.table", "here", "stringr", "readr", "testthat", "yaml"]} -LANGUAGE_DEPENDENCIES = { - "Python": _all_deps["Python"], - "R": _all_deps["R"], - "R+Python": _all_deps, -} -DEPRECATED_LANGUAGES = ["R", "R+Python"] -PYTHON_VERSIONS = ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11'] -DEPRECATED_VERSIONS = ['3.6', '3.7', '3.8'] +PYTHON_VERSIONS = ['3.9', '3.10', '3.11'] +DEPRECATED_VERSIONS = [] TEMPLATE_PATH = "templates/{name}" -TEMPLATES = { - "Python": { - "Default": "python", - "Dash": "python_dash", - "Git": "git" - }, - "R": { - "Default": "r", - "Git": "git" - }, - "R+Python": { - "Default": "r_python", - "Git": "git" - } +TEMPLATES = { + "Default": "python", + "Dash": "python_dash", + "Git": "git" } -GITHUB_RAW = "https://raw.githubusercontent.com/{owner}/{repo}/{branch}/{filepath}" EXT_COMMAND = {"py":"python -u ", "R":"Rscript ", "sh":"bash ", "None":""} diff --git a/skelebot/components/__init__.py b/skelebot/components/__init__.py index 34c242f..802515a 100644 --- a/skelebot/components/__init__.py +++ b/skelebot/components/__init__.py @@ -1,3 +1,3 @@ """Built-In Components of Skelebot and the Component Factory to Access Them""" -from . import artifactory, bump, componentFactory, dexec, jupyter, kerberos, plugin, prime, registry +from . import bump, componentFactory, dexec, jupyter, kerberos, plugin, prime, registry diff --git a/skelebot/components/artifactory.py b/skelebot/components/artifactory.py deleted file mode 100644 index 6ed9a70..0000000 --- a/skelebot/components/artifactory.py +++ /dev/null @@ -1,223 +0,0 @@ -"""Artifactory Component""" - -import os -import shutil -import artifactory -from requests.exceptions import MissingSchema -from schema import Schema, And, Optional -from ..common import DEPRECATION_WARNING -from ..objects.component import Activation, Component -from ..objects.skeleYaml import SkeleYaml -from ..objects.semver import Semver - -ERROR_NOT_COMPATIBLE = "No Compatible Version Found" -ERROR_ALREADY_PUSHED = "This artifact version already exists. Please bump the version or use the force parameter (-f) to overwrite the artifact." - -def pushArtifact(artifactFile, user, token, file, url, force): - """Pushes the given file to the url with the provided user/token auth""" - - # Error and exit if artifact already exists and we are not forcing an override - try: - if (not force) and (artifactory.ArtifactoryPath(url, auth=(user, token)).exists()): - raise RuntimeError(ERROR_ALREADY_PUSHED) - except MissingSchema: - pass - - # Rename artifact, deploy the renamed artifact, and then rename it back to original name - print("Deploying {file} to {url}".format(file=file, url=url)) - path = artifactory.ArtifactoryPath(url, auth=(user, token)) - shutil.copyfile(artifactFile, file) - try: - path.deploy_file(file) - os.remove(file) - except: - os.remove(file) - raise - -def pullArtifact(user, token, file, url, override, original): - """Pulls the given file from the url with the provided user/token auth""" - - if (artifactory.ArtifactoryPath(url, auth=(user, token)).exists()): - print("Pulling {file} from {url}".format(file=file, url=url)) - path = artifactory.ArtifactoryPath(url, auth=(user, token)) - with path.open() as fd: - dest = original if (override) else file - with open(dest, "wb") as out: - out.write(fd.read()) - else: - print("Artifact Not Found: {url}".format(url=url)) - -def findCompatibleArtifact(user, token, listUrl, currentVersion, filename, ext): - """Searches the artifact folder to find the latest compatible artifact version""" - - print("Searching for Latest Compatible Artifact") - compatibleSemver = None - currentSemver = Semver(currentVersion) - path = artifactory.ArtifactoryPath(listUrl, auth=(user, token)) - - # Iterate over all artifacts in the ArtifactoryPath (because path.glob was throwing exceptions on Linux systems) - if (path.exists()): - for artifact in path: # Only look at artifacts with the same filename and major version - modelPrefix = "{filename}_v{major}".format(filename=filename, major=currentSemver.major) - if modelPrefix in str(artifact): - artifactSemver = Semver(str(artifact).split("_v")[1].split(ext)[0]) - if (currentSemver.isBackwardCompatible(artifactSemver)) and ((compatibleSemver is None) or (compatibleSemver < artifactSemver)): - compatibleSemver = artifactSemver # Identify the latest compatible version - - # Raise an error if no compatible version is found - if (compatibleSemver is None): - raise RuntimeError(ERROR_NOT_COMPATIBLE) - - return "{filename}_v{version}.{ext}".format(filename=filename, version=compatibleSemver, ext=ext) - -class Artifact(SkeleYaml): - """ - Artifact Class - - Object contained within a list of the artifactory configuration in order to specify the - artifact files and artifact names - """ - - schema = Schema({ - 'name': And(str, error='Artifact \'name\' must be a String'), - 'file': And(str, error='Artifact \'file\' must be a String'), - }, ignore_extra_keys=True) - - name = None - file = None - - def __init__(self, name, file): - self.name = name - self.file = file - -class Artifactory(Component): - """ - *** DEPRECATED v1.11.0 *** - - Artifactory Component Class - - Provides the ability to push and pull artifacts that are defined in the skelebot config - file to and from Artifactory based on the project version number - """ - - activation = Activation.CONFIG - commands = ["push", "pull"] - - schema = Schema({ - Optional('artifacts'): And(list, error='Artifactory \'artifacts\' must be a List'), - 'url': And(str, error='Artifactory \'url\' must be a String'), - 'repo': And(str, error='Artifactory \'repo\' must be a String'), - 'path': And(str, error='Artifactory \'path\' must be a String'), - }, ignore_extra_keys=True) - - artifacts = None - url = None - repo = None - path = None - - @classmethod - def load(cls, config): - """Instantiate the Artifact Class Object based on a config dict""" - - cls.validate(config) - - artifactDicts = config["artifacts"] - artifacts = [] - for artifact in artifactDicts: - newArtifact = Artifact.load(artifact) - artifacts.append(newArtifact) - - return cls(artifacts, config["url"], config["repo"], config["path"]) - - def __init__(self, artifacts=None, url=None, repo=None, path=None): - """Instantiate the Artifact Class Object based on the provided parameters""" - - self.artifacts = artifacts if artifacts is not None else [] - self.url = url - self.repo = repo - self.path = path - - def addParsers(self, subparsers): - """ - SkeleParser Hook - - Adds the parsers for push and pull commands to the SkeleParser to push artifacts - with the project version and pull artifacts with the provided version number - """ - - artifactNames = [] - for artifact in self.artifacts: - artifactNames.append(artifact.name) - - parser = subparsers.add_parser("push", help="Push an artifact to artifactory") - parser.add_argument("artifact", choices=artifactNames) - parser.add_argument("-u", "--user", help="Auth user for Artifactory") - parser.add_argument("-t", "--token", help="Auth token for Artifactory") - parser.add_argument("-f", "--force", action='store_true', help="Force the push") - - parser = subparsers.add_parser("pull", help="Pull an artifact from artifactory") - parser.add_argument("artifact", choices=artifactNames) - parser.add_argument("version", help="The version of the artifact to pull") - parser.add_argument("-u", "--user", help="Auth user for Artifactory") - parser.add_argument("-t", "--token", help="Auth token for Artifactory") - parser.add_argument("-o", "--override", action='store_true', help="Override the model in the existing directory") - - return subparsers - - def execute(self, config, args, host=None): - """ - Execution Hook - - Executes either the push or pull command depending on which one was provided to push - rename the artifact and push it to Artifactory or pull down the given artifact version - from Artifactory - """ - - print(DEPRECATION_WARNING.format(code="Artifactory Component", version="1.11.0", - msg="Please use the Repository Component instead.")) - - artifactory.global_config = { - self.url: { - 'username': None, - 'verify': True, - 'cert': None, - 'password': None - } - } - - # Get User and Token if not provided in args - user = args.user - token = args.token - if (user is None): - user = input("Please provide a valid Artifactory user: ") - - if (token is None): - token = input("Please provide a valid Artifactory token: ") - - # Obtain the artifact that matches the provided name - selectedArtifact = None - for artifact in self.artifacts: - if (artifact.name == args.artifact): - selectedArtifact = artifact - - # Generate the local artifact file and the final Artifactory url path - ext = selectedArtifact.file.split(".")[1] - version = config.version if (args.job == "push") else args.version - - file = None - if (version == "LATEST"): - listUrl = "{url}/{repo}/{path}/" - listUrl = listUrl.format(url=self.url, repo=self.repo, path=self.path) - file = findCompatibleArtifact(user, token, listUrl, config.version, selectedArtifact.name, ext) - else: - file = "{filename}_v{version}.{ext}" - file = file.format(filename=selectedArtifact.name, version=version, ext=ext) - - url = "{url}/{repo}/{path}/{file}" - url = url.format(url=self.url, repo=self.repo, path=self.path, file=file) - - # Push the artifact with the config version, or pull with the arg version - if (args.job == "push"): - pushArtifact(selectedArtifact.file, user, token, file, url, args.force) - elif (args.job == "pull"): - pullArtifact(user, token, file, url, args.override, selectedArtifact.file) diff --git a/skelebot/components/componentFactory.py b/skelebot/components/componentFactory.py index 0114877..d6baaaf 100644 --- a/skelebot/components/componentFactory.py +++ b/skelebot/components/componentFactory.py @@ -13,7 +13,6 @@ from .bump import Bump from .prime import Prime from .dexec import Dexec -from .artifactory import Artifactory from .registry import Registry from .repository.repository import Repository from .environments import Environments @@ -41,7 +40,6 @@ def __init__(self): Bump.__name__.lower(): Bump, Prime.__name__.lower(): Prime, Dexec.__name__.lower(): Dexec, - Artifactory.__name__.lower(): Artifactory, Registry.__name__.lower(): Registry, Repository.__name__.lower(): Repository, Environments.__name__.lower(): Environments diff --git a/skelebot/components/environments.py b/skelebot/components/environments.py index dc488a9..c21b932 100644 --- a/skelebot/components/environments.py +++ b/skelebot/components/environments.py @@ -2,9 +2,8 @@ import os import re -from schema import Schema, And, Optional + from ..objects.component import Activation, Component -from ..systems.execution import docker HELP_MESSAGE = "Display the available environments for the project" @@ -22,7 +21,7 @@ def addParsers(self, subparsers): """ SkeleParser Hook - Adds a parser for the envs command to list the available skelebot environments. + Adds a parser for the envs command to list the available skelebot environments. """ subparsers.add_parser("envs", help=HELP_MESSAGE) diff --git a/skelebot/components/plugin.py b/skelebot/components/plugin.py index 5e7d70d..f35a5bf 100644 --- a/skelebot/components/plugin.py +++ b/skelebot/components/plugin.py @@ -46,6 +46,5 @@ def execute(self, config, args, host=None): os.makedirs(pluginsHome, exist_ok=True) # Unzip the plugin into the plugins folder - zip_ref = zipfile.ZipFile(args.plugin, 'r') - zip_ref.extractall(pluginsHome) - zip_ref.close() + with zipfile.ZipFile(args.plugin, "r") as zip_ref: + zip_ref.extractall(pluginsHome) diff --git a/skelebot/components/repository/artifactRepo.py b/skelebot/components/repository/artifactRepo.py index 9ab0d4d..80b64cb 100644 --- a/skelebot/components/repository/artifactRepo.py +++ b/skelebot/components/repository/artifactRepo.py @@ -17,4 +17,3 @@ def push(self, artifact, version, force=False, user=None, token=None, prefix=Non def pull(self, artifact, version, currentVersion=None, override=False, user=None, token=None): """ Function to be defined for pulling an artifact from an artifact repository """ pass - diff --git a/skelebot/objects/config.py b/skelebot/objects/config.py index b6c5cc3..a56e92c 100644 --- a/skelebot/objects/config.py +++ b/skelebot/objects/config.py @@ -23,7 +23,6 @@ class Config(SkeleYaml): Optional('maintainer'): And(str, error='\'maintainer\' must be a String'), Optional('contact'): And(str, error='\'contact\' must be a String'), Optional('host'): And(str, error='\'host\' must be a String'), - 'language': And(str, error='\'language\' must be a String'), Optional('baseImage'): And(str, error='\'baseImage\' must be a String'), Optional('timezone'): And(str, error='\'timezone\' must be a String'), Optional('primaryJob'): And(str, error='\'primaryJob\' must be a String'), @@ -47,7 +46,6 @@ class Config(SkeleYaml): maintainer = None contact = None host = None - language = None baseImage = None timezone = None primaryJob = None @@ -60,13 +58,13 @@ class Config(SkeleYaml): components = None params = None commands = None - pythonVersion = '3.6' + pythonVersion = '3.9' gpu = False def __init__(self, name=None, env=None, description=None, version=None, maintainer=None, - contact=None, host=None, language=None, baseImage=None, timezone=None, + contact=None, host=None, baseImage=None, timezone=None, primaryJob=None, primaryExe=None, ephemeral=None, dependencies=None, ignores=None, - jobs=None, ports=None, components=None, params=None, commands=None, pythonVersion = '3.6', + jobs=None, ports=None, components=None, params=None, commands=None, pythonVersion = '3.9', gpu = False): """Initialize the config object with all provided optional attributes""" @@ -77,7 +75,6 @@ def __init__(self, name=None, env=None, description=None, version=None, maintain self.maintainer = maintainer self.contact = contact self.host = host - self.language = language self.baseImage = baseImage self.timezone = timezone self.primaryJob = primaryJob @@ -118,24 +115,19 @@ def toDict(self): def getBaseImage(self): """ - Returns the proper base image based on the values for language and kerberos, + Returns the proper base image based on the values for python version and kerberos, or returns the user defined base image if provided in the config yaml """ if self.baseImage: image = self.baseImage else: - language = self.language if self.language is not None else "NA" - image = LANGUAGE_IMAGE[language] - if language == 'Python': - image['base'] = image['base'].format(pythonVersion = self.pythonVersion) + image = LANGUAGE_IMAGE["base"].format(pythonVersion=self.pythonVersion) - variant = "base" for component in self.components: if component.__class__.__name__.lower() == "kerberos": - variant = "krb" - - image = image[variant] + image = LANGUAGE_IMAGE["krb"] + break return image diff --git a/skelebot/systems/execution/dockerCommand.py b/skelebot/systems/execution/dockerCommand.py index 6ab0715..e760286 100644 --- a/skelebot/systems/execution/dockerCommand.py +++ b/skelebot/systems/execution/dockerCommand.py @@ -55,7 +55,7 @@ def set_entrypoint(self, parameters): self.entrypoint = True self.cmd += " --entrypoint" return self - + def set_gpu(self): self.cmd += " --gpus all --ipc=host" return self diff --git a/skelebot/systems/generators/dockerfile.py b/skelebot/systems/generators/dockerfile.py index 75018c1..fd81471 100644 --- a/skelebot/systems/generators/dockerfile.py +++ b/skelebot/systems/generators/dockerfile.py @@ -14,23 +14,8 @@ FILE_PATH = "{path}/Dockerfile" PY_DOWNLOAD_CA = "aws codeartifact get-package-version-asset --domain {domain} --domain-owner {owner} --repository {repo} --package {pkg} --package-version {version}{profile} --format pypi --asset {asset} libs/{asset}" -PY_INSTALL = "RUN [\"pip\", \"install\", \"{dep}\"]\n" -PY_INSTALL_VERSION = "RUN [\"pip\", \"install\", \"{depName}=={version}\"]\n" -PY_INSTALL_GITHUB = "RUN [\"pip\", \"install\", \"git+{depPath}\"]\n" -PY_INSTALL_FILE = "COPY {depPath} {depPath}\n" -PY_INSTALL_FILE += "RUN [\"pip\", \"install\", \"/app/{depPath}\"]\n" -PY_INSTALL_REQ = "COPY {depPath} {depPath}\n" -PY_INSTALL_REQ += "RUN [\"pip\", \"install\", \"-r\", \"/app/{depPath}\"]\n" -PY_R_INSTALL = "RUN [\"pip3\", \"install\", \"{dep}\"]\n" -PY_R_INSTALL_VERSION = "RUN [\"pip3\", \"install\", \"{depName}=={version}\"]\n" -PY_R_INSTALL_GITHUB = "RUN [\"pip3\", \"install\", \"git+{depPath}\"]\n" -PY_R_INSTALL_FILE = "COPY {depPath} {depPath}\n" -PY_R_INSTALL_FILE += "RUN [\"pip3\", \"install\", \"/app/{depPath}\"]\n" -R_INSTALL = "RUN [\"Rscript\", \"-e\", \"install.packages('{dep}', repo='https://cloud.r-project.org'); library({dep})\"]\n" -R_INSTALL_VERSION = "RUN [\"Rscript\", \"-e\", \"library(devtools); install_version('{depName}', version='{version}', repos='http://cran.us.r-project.org'); library({depName})\"]\n" -R_INSTALL_GITHUB = "RUN [\"Rscript\", \"-e\", \"library(devtools); install_github('{depPath}'); library({depName})\"]\n" -R_INSTALL_FILE = "COPY {depPath} {depPath}\n" -R_INSTALL_FILE += "RUN [\"Rscript\", \"-e\", \"install.packages('/app/{depPath}', repos=NULL, type='source'); library({depName})\"]\n" +COPY_FILE = "COPY {depPath} {depPath}\n" +PY_INSTALL = 'RUN ["pip", "install", "{deps}"]\n' TIMEZONE = "ENV TZ={timezone}\nRUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone\n" DOCKERFILE = """ @@ -52,7 +37,7 @@ def parse_pyproj(pyproject_file): # break the Dockerfile deps = [d.replace('"', "'") for d in deps] - return '", "'.join(deps) + return deps def buildDockerfile(config): @@ -67,92 +52,46 @@ def buildDockerfile(config): if (config.timezone is not None): docker += TIMEZONE.format(timezone=config.timezone) - # Add language dependencies - if (config.language == "Python"): - for dep in config.dependencies: - depSplit = dep.split(":") - if (dep.startswith("github:")): - docker += PY_INSTALL_GITHUB.format(depPath=dep.split(":", maxsplit=1)[1]) - elif (dep.startswith("file:")): - docker += PY_INSTALL_FILE.format(depPath=depSplit[1]) - elif (dep.startswith("req:")): - docker += PY_INSTALL_REQ.format(depPath=depSplit[1]) - elif (dep.startswith("ca_file:")): - domain = depSplit[1] - owner = depSplit[2] - repo = depSplit[3] - pkg = depSplit[4] - version = depSplit[5] - asset = f"{pkg.replace('-', '_')}-{version}-py3-none-any.whl" - profile = f" --profile {depSplit[6]}" if (len(depSplit) > 6) else "" - cmd = PY_DOWNLOAD_CA.format(domain=domain, owner=owner, repo=repo, pkg=pkg, version=version, asset=asset, profile=profile) - os.makedirs(os.path.join(os.getcwd(), 'libs'), exist_ok=True) - status = call(cmd, shell=True) - if (status != 0): - raise Exception("Failed to Obtain CodeArtifact Package") - - docker += PY_INSTALL_FILE.format(depPath=f"libs/{asset}") - elif (dep.startswith("proj:")): - deps = parse_pyproj(depSplit[1]) - docker += PY_INSTALL.format(dep=deps) - # if using PIP version specifiers, will be handled as a standard case - elif dep.count("=") == 1 and not re.search(r"[!<>~]", dep): - verSplit = dep.split("=") - docker += PY_INSTALL_VERSION.format(depName=verSplit[0], version=verSplit[1]) - else: - docker += PY_INSTALL.format(dep=dep) - if (config.language == "R"): - for dep in config.dependencies: - depSplit = dep.split(":") - if (dep.startswith("github:")): - docker += R_INSTALL_GITHUB.format(depPath=depSplit[1], depName=depSplit[2]) - elif (dep.startswith("file:")): - docker += R_INSTALL_FILE.format(depPath=depSplit[1], depName=depSplit[2]) - elif ("=" in dep): - verSplit = dep.split("=") - docker += R_INSTALL_VERSION.format(depName=verSplit[0], version=verSplit[1]) - else: - docker += R_INSTALL.format(dep=dep) - - if (config.language == "R+Python"): - for dep in config.dependencies["Python"]: - depSplit = dep.split(":") - if (dep.startswith("github:")): - docker += PY_R_INSTALL_GITHUB.format(depPath=dep.split(":", maxsplit=1)[1]) - elif (dep.startswith("file:")): - docker += PY_R_INSTALL_FILE.format(depPath=depSplit[1]) - elif (dep.startswith("ca_file:")): - domain = depSplit[1] - owner = depSplit[2] - repo = depSplit[3] - pkg = depSplit[4] - version = depSplit[5] - asset = f"{pkg.replace('-', '_')}-{version}-py3-none-any.whl" - profile = f" --profile {depSplit[6]}" if (len(depSplit) > 6) else "" - cmd = PY_DOWNLOAD_CA.format(domain=domain, owner=owner, repo=repo, pkg=pkg, version=version, asset=asset, profile=profile) - os.makedirs(os.path.join(os.getcwd(), 'libs'), exist_ok=True) - status = call(cmd, shell=True) - if (status != 0): - raise Exception("Failed to Obtain CodeArtifact Package") - - docker += PY_R_INSTALL_FILE.format(depPath=f"libs/{asset}") - # if using PIP version specifiers, will be handled as a standard case - elif dep.count("=") == 1 and not re.search(r"[!<>~]", dep): - verSplit = dep.split("=") - docker += PY_R_INSTALL_VERSION.format(depName=verSplit[0], version=verSplit[1]) - else: - docker += PY_R_INSTALL.format(dep=dep) - for dep in config.dependencies["R"]: - depSplit = dep.split(":") - if (dep.startswith("github:")): - docker += R_INSTALL_GITHUB.format(depPath=depSplit[1], depName=depSplit[2]) - elif (dep.startswith("file:")): - docker += R_INSTALL_FILE.format(depPath=depSplit[1], depName=depSplit[2]) - elif ("=" in dep): - verSplit = dep.split("=") - docker += R_INSTALL_VERSION.format(depName=verSplit[0], version=verSplit[1]) - else: - docker += R_INSTALL.format(dep=dep) + # Copy any needed files first then install all dependencies together + docker_deps = [] + for dep in config.dependencies: + depSplit = dep.split(":") + if (dep.startswith("github:")): + docker_deps.append("git+" + dep.split(":", maxsplit=1)[1]) + elif (dep.startswith("file:")): + docker += COPY_FILE.format(depPath=depSplit[1]) + docker_deps.append(f"/app/{depSplit[1]}") + elif (dep.startswith("req:")): + docker += COPY_FILE.format(depPath=depSplit[1]) + docker_deps += ["-r", f"/app/{depSplit[1]}"] + elif (dep.startswith("ca_file:")): + domain = depSplit[1] + owner = depSplit[2] + repo = depSplit[3] + pkg = depSplit[4] + version = depSplit[5] + asset = f"{pkg.replace('-', '_')}-{version}-py3-none-any.whl" + profile = f" --profile {depSplit[6]}" if (len(depSplit) > 6) else "" + cmd = PY_DOWNLOAD_CA.format(domain=domain, owner=owner, repo=repo, pkg=pkg, version=version, asset=asset, profile=profile) + os.makedirs(os.path.join(os.getcwd(), 'libs'), exist_ok=True) + status = call(cmd, shell=True) + if (status != 0): + raise Exception("Failed to Obtain CodeArtifact Package") + + docker += COPY_FILE.format(depPath=f"libs/{asset}") + docker_deps.append(f"/app/libs/{asset}") + # For pyprojects the source code does not exist at this point inside the docker + # image so we install only the dependencies + elif (dep.startswith("proj:")): + docker_deps += parse_pyproj(depSplit[1]) + # if using PIP version specifiers, will be handled as a standard case + elif dep.count("=") == 1 and not re.search(r"[!<>~]", dep): + verSplit = dep.split("=") + docker_deps.append(f"{verSplit[0]}=={verSplit[1]}") + else: + docker_deps.append(f"{dep}") + + docker += PY_INSTALL.format(deps='", "'.join(docker_deps)) # Copy the project into the /app folder of the Docker Image # Ignores anything in the .dockerignore file of the project @@ -189,6 +128,5 @@ def buildDockerfile(config): break - dockerfile = open(FILE_PATH.format(path=os.getcwd()), "w") - dockerfile.write(docker) - dockerfile.close() + with open(FILE_PATH.format(path=os.getcwd()), "w", encoding="utf-8") as dockerfile: + dockerfile.write(docker) diff --git a/skelebot/systems/generators/dockerignore.py b/skelebot/systems/generators/dockerignore.py index d921e44..db83f9f 100644 --- a/skelebot/systems/generators/dockerignore.py +++ b/skelebot/systems/generators/dockerignore.py @@ -23,6 +23,6 @@ def buildDockerignore(config): for component in config.components: docker += component.appendDockerignore() - dockerignore = open(FILE_PATH.format(path=os.getcwd()), "w") + dockerignore = open(FILE_PATH.format(path=os.getcwd()), "w", encoding="utf-8") dockerignore.write(docker) dockerignore.close() diff --git a/skelebot/systems/generators/readme.py b/skelebot/systems/generators/readme.py index b705ff2..481b4f4 100644 --- a/skelebot/systems/generators/readme.py +++ b/skelebot/systems/generators/readme.py @@ -27,6 +27,6 @@ def buildREADME(config): readme = README.format(name=name, description=desc, version=version, maintainer=maintainer, contact=contact) - readmeFile = open(FILE_PATH.format(path=os.getcwd()), "w") + readmeFile = open(FILE_PATH.format(path=os.getcwd()), "w", encoding="utf-8") readmeFile.write(readme) readmeFile.close() diff --git a/skelebot/systems/generators/yaml.py b/skelebot/systems/generators/yaml.py index be113d2..1d29b1e 100644 --- a/skelebot/systems/generators/yaml.py +++ b/skelebot/systems/generators/yaml.py @@ -27,7 +27,7 @@ def saveConfig(config): config.version = None yml = yaml.dump(config.toDict(), default_flow_style=False) - with open(FILE_PATH.format(path=os.getcwd()), "w") as file: + with open(FILE_PATH.format(path=os.getcwd()), "w", encoding="utf-8") as file: file.write(yml) def loadVersion(): @@ -36,14 +36,14 @@ def loadVersion(): version = None versionFile = VERSION_PATH.format(path=os.getcwd()) if os.path.isfile(versionFile): - with open(versionFile, 'r') as file: + with open(versionFile, "r", encoding="utf-8") as file: version = file.read().replace("\n", "") return version def saveVersion(version): """Overwrite the version number in the VERSION file with a new version""" - with open(VERSION_PATH.format(path=os.getcwd()), 'w') as file: + with open(VERSION_PATH.format(path=os.getcwd()), "w", encoding="utf-8") as file: file.write(version) def readYaml(env=None): @@ -53,12 +53,12 @@ def readYaml(env=None): cwd = os.getcwd() cfgFile = FILE_PATH.format(path=cwd) if os.path.isfile(cfgFile): - with open(cfgFile, 'r') as stream: + with open(cfgFile, "r", encoding="utf-8") as stream: yamlData = yaml.load(stream, Loader=yaml.FullLoader) if (env is not None): envFile = ENV_FILE_PATH.format(path=cwd, env=env) if os.path.isfile(envFile): - with open(envFile, 'r') as stream: + with open(envFile, "r", encoding="utf-8") as stream: overrideYaml = yaml.load(stream, Loader=yaml.FullLoader) yamlData = override(yamlData, overrideYaml) else: diff --git a/skelebot/systems/scaffolding/scaffolder.py b/skelebot/systems/scaffolding/scaffolder.py index 4d6ac27..331fd88 100644 --- a/skelebot/systems/scaffolding/scaffolder.py +++ b/skelebot/systems/scaffolding/scaffolder.py @@ -2,15 +2,14 @@ import os import re -import json -import requests -import yaml as pyyaml from subprocess import call + +import yaml as pyyaml + from ...objects.config import Config from ...components.componentFactory import ComponentFactory from ...systems.generators import dockerfile, dockerignore, readme, yaml -from ...common import (LANGUAGE_DEPENDENCIES, TEMPLATES, TEMPLATE_PATH, GITHUB_RAW, - DEPRECATED_LANGUAGES, DEPRECATION_WARNING) +from ...common import TEMPLATES, TEMPLATE_PATH from .prompt import promptUser class Scaffolder: @@ -50,7 +49,7 @@ def __load_template(self, name): path = TEMPLATE_PATH.format(name=name) path = os.path.join(os.path.dirname(__file__), path) - with open(f"{path}/template.yaml", "r") as yaml_file: + with open(f"{path}/template.yaml", "r", encoding="utf-8") as yaml_file: yaml_text = self.__format_variables(yaml_file.read()) template = pyyaml.load(yaml_text, Loader=pyyaml.FullLoader) @@ -75,16 +74,6 @@ def scaffold(self): description = promptUser("Enter a PROJECT DESCRIPTION") maintainer = promptUser("Enter a MAINTAINER NAME") contact = promptUser("Enter a CONTACT EMAIL") - language = promptUser("Select a LANGUAGE", - options=list(LANGUAGE_DEPENDENCIES.keys()), - deprecated_options=DEPRECATED_LANGUAGES) - - if language in DEPRECATED_LANGUAGES: - print(DEPRECATION_WARNING.format( - code=f"support for {language} language", - version="1.37.0", - msg="Please use a different language." - )) # Configure Template Variables self.variables["name"] = name @@ -92,12 +81,11 @@ def scaffold(self): self.variables["description"] = description self.variables["maintainer"] = maintainer self.variables["contact"] = contact - self.variables["language"] = language # Load the Template Config - options = list(TEMPLATES[language].keys()) + options = list(TEMPLATES.keys()) template = promptUser("Select a TEMPLATE", options=options) - template_name = TEMPLATES[language][template.capitalize()] + template_name = TEMPLATES[template.capitalize()] if (template_name == "git"): url = promptUser("Enter Git Repo URL") template = self.__load_git_template(url) @@ -133,8 +121,8 @@ def scaffold(self): # Setting up the file templates for the project print("Attaching fiber optic ligaments...") for file_dict in template.get("files", []): - with open(file_dict["template"], "r") as tmp_file: - with open(file_dict["name"], "w") as dst_file: + with open(file_dict["template"], "r", encoding="utf-8") as tmp_file: + with open(file_dict["name"], "w", encoding="utf-8") as dst_file: dst_file.write(self.__format_variables(tmp_file.read())) print("Soldering the micro-computer to the skele-skull...") diff --git a/skelebot/systems/scaffolding/templates/python/template.yaml b/skelebot/systems/scaffolding/templates/python/template.yaml index eae11aa..411c33d 100644 --- a/skelebot/systems/scaffolding/templates/python/template.yaml +++ b/skelebot/systems/scaffolding/templates/python/template.yaml @@ -10,7 +10,6 @@ dirs: - "src/jobs/" config: - language: Python dependencies: - numpy - pandas diff --git a/skelebot/systems/scaffolding/templates/python_dash/template.yaml b/skelebot/systems/scaffolding/templates/python_dash/template.yaml index 6633314..9feef71 100644 --- a/skelebot/systems/scaffolding/templates/python_dash/template.yaml +++ b/skelebot/systems/scaffolding/templates/python_dash/template.yaml @@ -12,7 +12,6 @@ files: template: files/style_css config: - language: Python dependencies: - dash~=2.0 ports: diff --git a/skelebot/systems/scaffolding/templates/r/template.yaml b/skelebot/systems/scaffolding/templates/r/template.yaml deleted file mode 100644 index 93bceb2..0000000 --- a/skelebot/systems/scaffolding/templates/r/template.yaml +++ /dev/null @@ -1,20 +0,0 @@ -name: default - -dirs: -- "config/" -- "data/" -- "models/" -- "notebooks/" -- "output" -- "queries" -- "src/jobs/" - -config: - language: R - dependencies: - - data.table - - here - - stringr - - readr - - testthat - - yaml diff --git a/skelebot/systems/scaffolding/templates/r_python/template.yaml b/skelebot/systems/scaffolding/templates/r_python/template.yaml deleted file mode 100644 index f3fabe5..0000000 --- a/skelebot/systems/scaffolding/templates/r_python/template.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: default - -dirs: -- "config/" -- "data/" -- "models/" -- "notebooks/" -- "output" -- "queries" -- "src/jobs/" - -config: - language: R+Python - dependencies: - Python: - - numpy - - pandas - - scipy - - scikit-learn - R: - - data.table - - here - - stringr - - readr - - testthat - - yaml diff --git a/test/files/.dockerignore b/test/files/.dockerignore index 829613b..1fa8c31 100644 --- a/test/files/.dockerignore +++ b/test/files/.dockerignore @@ -3,7 +3,6 @@ # Editing this file manually is not advised as all changes will be overwritten by Skelebot **/*.zip -**/*.RData **/*.pkl **/*.csv **/*.model diff --git a/test/files/Dockerfile b/test/files/Dockerfile index 9c2644f..c16cbe2 100644 --- a/test/files/Dockerfile +++ b/test/files/Dockerfile @@ -2,20 +2,13 @@ # This Dockerfile was generated by Skelebot # Editing this file manually is not advised as all changes will be overwritten by Skelebot -FROM skelebot/r-base +FROM skelebot/python-base:3.9 MAINTAINER Mega Man WORKDIR /app ENV TZ=America/Chicago RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN ["Rscript", "-e", "install.packages('pyyaml', repo='https://cloud.r-project.org'); library(pyyaml)"] -RUN ["Rscript", "-e", "install.packages('artifactory', repo='https://cloud.r-project.org'); library(artifactory)"] -RUN ["Rscript", "-e", "install.packages('argparse', repo='https://cloud.r-project.org'); library(argparse)"] -RUN ["Rscript", "-e", "install.packages('coverage', repo='https://cloud.r-project.org'); library(coverage)"] -RUN ["Rscript", "-e", "install.packages('pytest', repo='https://cloud.r-project.org'); library(pytest)"] -RUN ["Rscript", "-e", "library(devtools); install_github('github.com/repo'); library(cool-lib)"] COPY libs/proj libs/proj -RUN ["Rscript", "-e", "install.packages('/app/libs/proj', repos=NULL, type='source'); library(cool-proj)"] -RUN ["Rscript", "-e", "library(devtools); install_version('dtable', version='9.0', repos='http://cran.us.r-project.org'); library(dtable)"] +RUN ["pip", "install", "pyyaml", "artifactory", "argparse", "coverage", "pytest", "git+github.com/repo", "/app/libs/proj", "dtable==9.0"] COPY . /app RUN rm -rf build/ RUN rm -rf dist/ diff --git a/test/files/skelebot-test.yaml b/test/files/skelebot-test.yaml index c0884d1..a45f6f9 100644 --- a/test/files/skelebot-test.yaml +++ b/test/files/skelebot-test.yaml @@ -19,13 +19,3 @@ components: artifacts: - file: test name: test - artifactory: - artifacts: - - file: test - name: test - - file: test2 - name: test2 - singular: True - path: test - repo: cars-ml-core - url: https://repository.cars.com/artifactory diff --git a/test/files/skelebot.yaml b/test/files/skelebot.yaml index e5bc555..da2a607 100644 --- a/test/files/skelebot.yaml +++ b/test/files/skelebot.yaml @@ -2,13 +2,6 @@ commands: - rm -rf build/ - rm -rf dist/ components: - artifactory: - artifacts: - - file: test - name: test - path: test - repo: cars-ml-core - url: https://repository.cars.com/artifactory jupyter: folder: notebooks/ lab: false @@ -26,7 +19,6 @@ description: test cases gpu: false ignores: - '**/*.zip' -- '**/*.RData' - '**/*.pkl' - '**/*.csv' - '**/*.model' @@ -70,7 +62,6 @@ jobs: default: csv name: f source: test_short.sh -language: Python maintainer: Mega Man name: test params: @@ -84,4 +75,4 @@ params: name: log primaryExe: CMD primaryJob: build -pythonVersion: '3.6' +pythonVersion: '3.9' diff --git a/test/test_components_artifactory.py b/test/test_components_artifactory.py deleted file mode 100644 index 13a0e36..0000000 --- a/test/test_components_artifactory.py +++ /dev/null @@ -1,235 +0,0 @@ -import argparse -import copy -import unittest -from unittest import mock -from requests.exceptions import MissingSchema -from schema import SchemaError - -import skelebot as sb - -class TestArtifactory(unittest.TestCase): - artifcatory = None - - artifactoryDict = { - "url": "test", - "repo": "test", - "path": "path", - "artifacts": [1, 2] - } - - artifactDict = { - "name": "test", - "file": "test" - } - - def setUp(self): - artifact = sb.components.artifactory.Artifact("test", "test.pkl") - self.artifactory = sb.components.artifactory.Artifactory([artifact], "artifactory.test.com", "ml", "test") - - def test_addParsers(self): - parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) - subparsers = parser.add_subparsers(dest="job") - subparsers = self.artifactory.addParsers(subparsers) - - self.assertNotEqual(subparsers.choices["push"], None) - self.assertNotEqual(subparsers.choices["pull"], None) - - @mock.patch('os.rename') - @mock.patch('artifactory.ArtifactoryPath') - def test_execute_push_conflict(self, mock_artifactory, mock_rename): - config = sb.objects.config.Config(version="1.0.0") - args = argparse.Namespace(job="push", force=False, artifact='test', user='sean', token='abc123') - - expectedException = "This artifact version already exists. Please bump the version or use the force parameter (-f) to overwrite the artifact." - - try: - self.artifactory.execute(config, args) - self.fail("Exception Not Thrown") - except Exception as exc: - self.assertEqual(str(exc), expectedException) - mock_artifactory.assert_called_with("artifactory.test.com/ml/test/test_v1.0.0.pkl", auth=('sean', 'abc123')) - - @mock.patch('shutil.copyfile') - @mock.patch('os.remove') - @mock.patch('artifactory.ArtifactoryPath') - def test_execute_push_error(self, mock_artifactory, mock_remove, mock_copy): - mock_path = mock.MagicMock() - mock_path.deploy_file = mock.MagicMock(side_effect=KeyError('foo')) - mock_artifactory.return_value = mock_path - - config = sb.objects.config.Config(version="1.0.0") - args = argparse.Namespace(job="push", force=True, artifact='test', user='sean', token='abc123') - - with self.assertRaises(KeyError): - self.artifactory.execute(config, args) - - mock_artifactory.assert_called_with("artifactory.test.com/ml/test/test_v1.0.0.pkl", auth=('sean', 'abc123')) - mock_copy.assert_called_with("test.pkl", "test_v1.0.0.pkl") - mock_remove.assert_called_with("test_v1.0.0.pkl") - - @mock.patch('shutil.copyfile') - @mock.patch('os.remove') - @mock.patch('artifactory.ArtifactoryPath') - def test_execute_push_missing_schema(self, mock_artifactory, mock_remove, mock_copy): - mock_path = mock.MagicMock() - mock_path.exists = mock.MagicMock(side_effect=MissingSchema()) - mock_artifactory.return_value = mock_path - - config = sb.objects.config.Config(version="1.0.0") - args = argparse.Namespace(job="push", force=False, artifact='test', user='sean', token='abc123') - - self.artifactory.execute(config, args) - - mock_artifactory.assert_called_with("artifactory.test.com/ml/test/test_v1.0.0.pkl", auth=('sean', 'abc123')) - mock_copy.assert_called_with("test.pkl", "test_v1.0.0.pkl") - mock_remove.assert_called_with("test_v1.0.0.pkl") - - @mock.patch('shutil.copyfile') - @mock.patch('os.remove') - @mock.patch('artifactory.ArtifactoryPath') - def test_execute_push(self, mock_artifactory, mock_remove, mock_copy): - config = sb.objects.config.Config(version="1.0.0") - args = argparse.Namespace(job="push", force=True, artifact='test', user='sean', token='abc123') - - self.artifactory.execute(config, args) - - mock_artifactory.assert_called_with("artifactory.test.com/ml/test/test_v1.0.0.pkl", auth=('sean', 'abc123')) - mock_copy.assert_called_with("test.pkl", "test_v1.0.0.pkl") - mock_remove.assert_called_with("test_v1.0.0.pkl") - - @mock.patch('skelebot.components.artifactory.input') - @mock.patch('builtins.open') - @mock.patch('artifactory.ArtifactoryPath') - def test_execute_pull(self, mock_artifactory, mock_open, mock_input): - mock_input.return_value = "abc" - - config = sb.objects.config.Config(version="1.0.0") - args = argparse.Namespace(job="pull", version='0.1.0', artifact='test', user=None, token=None, override=False) - - self.artifactory.execute(config, args) - - mock_artifactory.assert_called_with("artifactory.test.com/ml/test/test_v0.1.0.pkl", auth=("abc", "abc")) - mock_open.assert_called_with("test_v0.1.0.pkl", "wb") - - @mock.patch('skelebot.components.artifactory.input') - @mock.patch('builtins.open') - @mock.patch('artifactory.ArtifactoryPath') - def test_execute_pull_lcv(self, mock_artifactory, mock_open, mock_input): - mock_apath = mock_artifactory.return_value - mock_input.return_value = "abc" - mock_apath.__iter__.return_value = ["test_v1.1.0", "test_v0.2.4", "test_v1.0.0", "test_v2.0.1"] - - config = sb.objects.config.Config(version="1.0.9") - args = argparse.Namespace(job="pull", version='LATEST', artifact='test', user=None, token=None, override=False) - - self.artifactory.execute(config, args) - - mock_artifactory.assert_called_with("artifactory.test.com/ml/test/test_v1.0.0.pkl", auth=("abc", "abc")) - mock_open.assert_called_with("test_v1.0.0.pkl", "wb") - - @mock.patch('skelebot.components.artifactory.input') - @mock.patch('builtins.open') - @mock.patch('artifactory.ArtifactoryPath') - def test_execute_pull_lcv_not_found(self, mock_artifactory, mock_open, mock_input): - mock_apath = mock_artifactory.return_value - mock_input.return_value = "abc" - mock_apath.__iter__.return_value = ["test_v1.1.0", "test_v0.2.4", "test_v1.0.0", "test_v2.0.1"] - - config = sb.objects.config.Config(version="3.0.9") - args = argparse.Namespace(job="pull", version='LATEST', artifact='test', user=None, token=None, override=False) - - try: - self.artifactory.execute(config, args) - self.fail("Exception Not Thrown") - except RuntimeError as err: - self.assertEqual(str(err), "No Compatible Version Found") - - @mock.patch('skelebot.components.artifactory.input') - @mock.patch('builtins.open') - @mock.patch('artifactory.ArtifactoryPath') - def test_execute_pull_override_and_lcv(self, mock_artifactory, mock_open, mock_input): - mock_apath = mock_artifactory.return_value - mock_input.return_value = "abc" - mock_apath.__iter__.return_value = ["test_v1.1.0", "test_v0.2.4", "test_v1.0.0", "test_v2.0.1"] - - config = sb.objects.config.Config(version="0.6.9") - args = argparse.Namespace(job="pull", version='LATEST', artifact='test', user=None, token=None, override=True) - - self.artifactory.execute(config, args) - - mock_artifactory.assert_called_with("artifactory.test.com/ml/test/test_v0.2.4.pkl", auth=("abc", "abc")) - mock_open.assert_called_with("test.pkl", "wb") - - @mock.patch('skelebot.components.artifactory.input') - @mock.patch('artifactory.ArtifactoryPath') - def test_execute_pull_not_found(self, mock_artifactory, mock_input): - mock_input.return_value = "abc" - path = mock_artifactory.return_value - path.exists.return_value = False - - config = sb.objects.config.Config(version="1.0.0") - args = argparse.Namespace(job="pull", version='0.1.0', artifact='test', user=None, token=None, override=False) - - self.artifactory.execute(config, args) - - mock_artifactory.assert_called_with("artifactory.test.com/ml/test/test_v0.1.0.pkl", auth=("abc", "abc")) - - def test_validate_valid(self): - try: - sb.components.artifactory.Artifactory.validate(self.artifactoryDict) - except: - self.fail("Validation Raised Exception Unexpectedly") - - try: - sb.components.artifactory.Artifact.validate(self.artifactDict) - except: - self.fail("Validation Raised Exception Unexpectedly") - - def test_validate_mising(self): - artifactoryDict = copy.deepcopy(self.artifactoryDict) - del artifactoryDict['url'] - del artifactoryDict['repo'] - del artifactoryDict['path'] - - try: - sb.components.artifactory.Artifactory.validate(artifactoryDict) - except SchemaError as error: - self.assertEqual(error.code, "Missing keys: 'path', 'repo', 'url'") - - artifactDict = copy.deepcopy(self.artifactDict) - del artifactDict['name'] - del artifactDict['file'] - - try: - sb.components.artifactory.Artifact.validate(artifactDict) - except SchemaError as error: - self.assertEqual(error.code, "Missing keys: 'file', 'name'") - - def validate_error_artifactory(self, attr, reset, expected): - artifactoryDict = copy.deepcopy(self.artifactoryDict) - artifactoryDict[attr] = reset - - try: - sb.components.artifactory.Artifactory.validate(artifactoryDict) - except SchemaError as error: - self.assertEqual(error.code, "Artifactory '{attr}' must be a {expected}".format(attr=attr, expected=expected)) - - def validate_error_artifact(self, attr, reset, expected): - artifactDict = copy.deepcopy(self.artifactDict) - artifactDict[attr] = reset - - try: - sb.components.artifactory.Artifact.validate(artifactDict) - except SchemaError as error: - self.assertEqual(error.code, "Artifact '{attr}' must be a {expected}".format(attr=attr, expected=expected)) - - def test_invalid(self): - self.validate_error_artifactory('url', 123, 'String') - self.validate_error_artifactory('repo', 123, 'String') - self.validate_error_artifactory('path', 123, 'String') - self.validate_error_artifactory('artifacts', 123, 'List') - self.validate_error_artifact('name', 123, 'String') - self.validate_error_artifact('file', 123, 'String') - -if __name__ == '__main__': - unittest.main() diff --git a/test/test_components_jupyter.py b/test/test_components_jupyter.py index a7a7ef8..a57ddf8 100644 --- a/test/test_components_jupyter.py +++ b/test/test_components_jupyter.py @@ -24,25 +24,9 @@ def test_addParsers(self): self.assertNotEqual(subparsers.choices["jupyter"], None) @mock.patch('skelebot.components.jupyter.docker') - def test_execute_R(self, mock_docker): + def test_execute(self, mock_docker): mock_docker.build.return_value = 0 - config = sb.objects.config.Config(language="R") - args = argparse.Namespace(verbose_global=False) - - jupyter = sb.components.jupyter.Jupyter(port=1127, folder="notebooks/") - jupyter.execute(config, args) - - expectedCommand = "jupyter notebook --ip=0.0.0.0 --port=8888 --allow-root --notebook-dir=notebooks/" - - mock_docker.build.assert_called_with(config, host=None, verbose=False) - mock_docker.run.assert_called_with( - config, expectedCommand, "it", ["1127:8888"], ["."], "jupyter", host=None, verbose=False - ) - - @mock.patch('skelebot.components.jupyter.docker') - def test_execute_Python(self, mock_docker): - mock_docker.build.return_value = 0 - config = sb.objects.config.Config(language="Python") + config = sb.objects.config.Config() args = argparse.Namespace(verbose_global=False) jupyter = sb.components.jupyter.Jupyter(port=1127, folder="notebooks/", mappings = ["other_project/data"]) @@ -59,7 +43,7 @@ def test_execute_Python(self, mock_docker): @mock.patch('skelebot.components.jupyter.docker') def test_execute_host(self, mock_docker): mock_docker.build.return_value = 0 - config = sb.objects.config.Config(language="Python") + config = sb.objects.config.Config() args = argparse.Namespace(verbose_global=False) jupyter = sb.components.jupyter.Jupyter(port=1127, folder="notebooks/") @@ -75,7 +59,7 @@ def test_execute_host(self, mock_docker): @mock.patch('skelebot.components.jupyter.docker') def test_execute_lab(self, mock_docker): mock_docker.build.return_value = 0 - config = sb.objects.config.Config(language="Python") + config = sb.objects.config.Config() args = argparse.Namespace(verbose_global=False) jupyter = sb.components.jupyter.Jupyter(port=1127, folder="notebooks/", mappings=["other_project/data"], lab=True) diff --git a/test/test_components_plugin.py b/test/test_components_plugin.py index 82694f5..a818619 100644 --- a/test/test_components_plugin.py +++ b/test/test_components_plugin.py @@ -18,22 +18,21 @@ def test_addParsers(self): @mock.patch('os.path.expanduser') @mock.patch('os.path.exists') @mock.patch('os.makedirs') - @mock.patch('skelebot.components.plugin.zipfile') + @mock.patch('skelebot.components.plugin.zipfile.ZipFile') def test_execute(self, mock_zipfile, mock_makedirs, mock_exists, mock_expanduser): mock_expanduser.return_value = "test/dummy" mock_exists.return_value = False - mock_zip = mock.MagicMock() - mock_zipfile.ZipFile.return_value = mock_zip + mock_zip_ref = mock.MagicMock() + mock_zipfile.return_value.__enter__.return_value = mock_zip_ref config = sb.objects.config.Config() args = argparse.Namespace(plugin="test.zip") plugin = sb.components.plugin.Plugin() plugin.execute(config, args) - mock_zipfile.ZipFile.assert_called_with("test.zip", "r") - mock_zip.extractall.assert_called() - mock_zip.close.assert_called() + mock_zipfile.assert_called_once_with("test.zip", "r") + mock_zip_ref.extractall.assert_called_once_with("test/dummy") mock_makedirs.assert_any_call("test/dummy", exist_ok=True) diff --git a/test/test_components_registry.py b/test/test_components_registry.py index c2182c7..4df502f 100644 --- a/test/test_components_registry.py +++ b/test/test_components_registry.py @@ -32,7 +32,7 @@ def test_addParsers(self): def test_execute(self, mock_docker): mock_docker.build.return_value = 0 - config = sb.objects.config.Config(language="R") + config = sb.objects.config.Config() args = argparse.Namespace(tags=None, skip_build_global=False, verbose_global=False, omit_version=False, omit_latest=False) @@ -50,7 +50,7 @@ def test_execute(self, mock_docker): def test_execute_omit_latest(self, mock_docker): mock_docker.build.return_value = 0 - config = sb.objects.config.Config(language="R") + config = sb.objects.config.Config() args = argparse.Namespace(tags=None, skip_build_global=False, verbose_global=False, omit_version=False, omit_latest=True) @@ -68,7 +68,7 @@ def test_execute_omit_latest(self, mock_docker): def test_execute_omit_version(self, mock_docker): mock_docker.build.return_value = 0 - config = sb.objects.config.Config(language="R") + config = sb.objects.config.Config() args = argparse.Namespace(tags=None, skip_build_global=False, verbose_global=False, omit_version=True, omit_latest=False) @@ -86,7 +86,7 @@ def test_execute_omit_version(self, mock_docker): def test_execute_skip_build(self, mock_docker): mock_docker.build.return_value = 0 - config = sb.objects.config.Config(language="R") + config = sb.objects.config.Config() args = argparse.Namespace(tags=None, skip_build_global=True, verbose_global=False, omit_version=False, omit_latest=False) @@ -103,7 +103,7 @@ def test_execute_skip_build(self, mock_docker): def test_execute_host(self, mock_docker): mock_docker.build.return_value = 0 - config = sb.objects.config.Config(language="R") + config = sb.objects.config.Config() args = argparse.Namespace(tags=None, skip_build_global=False, verbose_global=False, omit_version=False, omit_latest=False) @@ -121,7 +121,7 @@ def test_execute_host(self, mock_docker): def test_execute_aws(self, mock_docker): mock_docker.build.return_value = 0 - config = sb.objects.config.Config(language="R") + config = sb.objects.config.Config() args = argparse.Namespace(tags=None, skip_build_global=False, verbose_global=False, omit_version=False, omit_latest=False) @@ -142,7 +142,7 @@ def test_execute_aws(self, mock_docker): def test_execute_aws_host(self, mock_docker): mock_docker.build.return_value = 0 - config = sb.objects.config.Config(language="R") + config = sb.objects.config.Config() args = argparse.Namespace(tags=None, skip_build_global=False, verbose_global=False, omit_version=False, omit_latest=False) @@ -163,7 +163,7 @@ def test_execute_aws_host(self, mock_docker): def test_execute_tags(self, mock_docker): mock_docker.build.return_value = 0 - config = sb.objects.config.Config(language="R") + config = sb.objects.config.Config() args = argparse.Namespace(tags=["test", "dev", "stage"], skip_build_global=False, verbose_global=False, omit_version=False, omit_latest=False) diff --git a/test/test_objects_config_validate.py b/test/test_objects_config_validate.py index 7795ad2..5d15b20 100644 --- a/test/test_objects_config_validate.py +++ b/test/test_objects_config_validate.py @@ -16,7 +16,6 @@ class TestConfigValidate(unittest.TestCase): 'description': 'test', 'maintainer': 'test', 'contact': 'test', - 'language': 'test', 'baseImage': 'test', 'primaryJob': 'test', 'host': 'test', @@ -28,7 +27,7 @@ class TestConfigValidate(unittest.TestCase): 'components': {}, 'params': [1, 2], 'commands': [], - 'pythonVersion': '3.8', + 'pythonVersion': '3.9', 'gpu': True } @@ -48,6 +47,7 @@ def test_valid(self): except: self.fail("Validation Raised Exception Unexpectedly") + @mock.patch.object(sb.objects.config, 'DEPRECATED_VERSIONS', ['3.6']) @mock.patch('skelebot.objects.config.print') def test_valid_deprecated(self, mock_print): _ = sb.objects.config.Config(pythonVersion='3.6') @@ -60,12 +60,11 @@ def test_valid_deprecated(self, mock_print): def test_invalid_mising(self): config = copy.deepcopy(self.config) del config['name'] - del config['language'] try: sb.objects.config.Config.validate(config) except SchemaError as error: - self.assertEqual(error.code, "Missing keys: 'language', 'name'") + self.assertEqual(error.code, "Missing key: 'name'") def test_invalid(self): self.validate_error('name', 123, 'a String') @@ -73,7 +72,6 @@ def test_invalid(self): self.validate_error('description', 123, 'a String') self.validate_error('maintainer', 123, 'a String') self.validate_error('contact', 123, 'a String') - self.validate_error('language', 123, 'a String') self.validate_error('baseImage', 123, 'a String') self.validate_error('primaryJob', 123, 'a String') self.validate_error('primaryExe', 123, 'CMD or ENTRYPOINT') diff --git a/test/test_systems_generators_dockerfile.py b/test/test_systems_generators_dockerfile.py index 807aceb..3a18b5f 100644 --- a/test/test_systems_generators_dockerfile.py +++ b/test/test_systems_generators_dockerfile.py @@ -25,79 +25,6 @@ def setUp(self): self.path = os.getcwd() self.maxDiff = None - @mock.patch('os.path.expanduser') - @mock.patch('os.getcwd') - def test_buildDockerfile_no_language(self, mock_getcwd, mock_expanduser): - folderPath = "{path}/test/files".format(path=self.path) - filePath = "{folder}/Dockerfile".format(folder=folderPath) - mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path) - mock_getcwd.return_value = folderPath - config = sb.systems.generators.yaml.loadConfig() - config.language = None - - expectedDockerfile = """ -# This Dockerfile was generated by Skelebot -# Editing this file manually is not advised as all changes will be overwritten by Skelebot - -FROM ubuntu:18.04 -MAINTAINER Mega Man -WORKDIR /app -COPY . /app -RUN rm -rf build/ -RUN rm -rf dist/ -CMD /bin/bash -c \"bash build.sh --env local --log info\"\n""" - - sb.systems.generators.dockerfile.buildDockerfile(config) - - data = None - with open(filePath, "r") as file: - data = file.read() - self.assertTrue(data is not None) - self.assertEqual(data, expectedDockerfile) - - @mock.patch('os.path.expanduser') - @mock.patch('os.getcwd') - def test_buildDockerfile_base(self, mock_getcwd, mock_expanduser): - folderPath = "{path}/test/files".format(path=self.path) - filePath = "{folder}/Dockerfile".format(folder=folderPath) - - mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path) - mock_getcwd.return_value = folderPath - config = sb.systems.generators.yaml.loadConfig() - config.language = "R" - config.dependencies.append("github:github.com/repo:cool-lib") - config.dependencies.append("file:libs/proj:cool-proj") - config.dependencies.append("dtable=9.0") - - expectedDockerfile = """ -# This Dockerfile was generated by Skelebot -# Editing this file manually is not advised as all changes will be overwritten by Skelebot - -FROM skelebot/r-base -MAINTAINER Mega Man -WORKDIR /app -RUN ["Rscript", "-e", "install.packages('pyyaml', repo='https://cloud.r-project.org'); library(pyyaml)"] -RUN ["Rscript", "-e", "install.packages('artifactory', repo='https://cloud.r-project.org'); library(artifactory)"] -RUN ["Rscript", "-e", "install.packages('argparse', repo='https://cloud.r-project.org'); library(argparse)"] -RUN ["Rscript", "-e", "install.packages('coverage', repo='https://cloud.r-project.org'); library(coverage)"] -RUN ["Rscript", "-e", "install.packages('pytest', repo='https://cloud.r-project.org'); library(pytest)"] -RUN ["Rscript", "-e", "library(devtools); install_github('github.com/repo'); library(cool-lib)"] -COPY libs/proj libs/proj -RUN ["Rscript", "-e", "install.packages('/app/libs/proj', repos=NULL, type='source'); library(cool-proj)"] -RUN ["Rscript", "-e", "library(devtools); install_version('dtable', version='9.0', repos='http://cran.us.r-project.org'); library(dtable)"] -COPY . /app -RUN rm -rf build/ -RUN rm -rf dist/ -CMD /bin/bash -c \"bash build.sh --env local --log info\"\n""" - - sb.systems.generators.dockerfile.buildDockerfile(config) - - data = None - with open(filePath, "r") as file: - data = file.read() - self.assertTrue(data is not None) - self.assertEqual(data, expectedDockerfile) - @mock.patch('os.path.expanduser') @mock.patch('os.getcwd') def test_buildDockerfile_entrypoint_exec(self, mock_getcwd, mock_expanduser): @@ -113,14 +40,10 @@ def test_buildDockerfile_entrypoint_exec(self, mock_getcwd, mock_expanduser): # This Dockerfile was generated by Skelebot # Editing this file manually is not advised as all changes will be overwritten by Skelebot -FROM skelebot/python-base:3.6 +FROM skelebot/python-base:3.9 MAINTAINER Mega Man WORKDIR /app -RUN ["pip", "install", "pyyaml"] -RUN ["pip", "install", "artifactory"] -RUN ["pip", "install", "argparse"] -RUN ["pip", "install", "coverage"] -RUN ["pip", "install", "pytest"] +RUN ["pip", "install", "pyyaml", "artifactory", "argparse", "coverage", "pytest"] COPY . /app RUN rm -rf build/ RUN rm -rf dist/ @@ -129,7 +52,7 @@ def test_buildDockerfile_entrypoint_exec(self, mock_getcwd, mock_expanduser): sb.systems.generators.dockerfile.buildDockerfile(config) data = None - with open(filePath, "r") as file: + with open(filePath, "r", encoding="utf-8") as file: data = file.read() self.assertTrue(data is not None) self.assertEqual(data, expectedDockerfile) @@ -153,14 +76,10 @@ def test_buildDockerfile_entrypoint_path(self, mock_getcwd, mock_expanduser): # This Dockerfile was generated by Skelebot # Editing this file manually is not advised as all changes will be overwritten by Skelebot -FROM skelebot/python-base:3.6 +FROM skelebot/python-base:3.9 MAINTAINER Mega Man WORKDIR /app -RUN ["pip", "install", "pyyaml"] -RUN ["pip", "install", "artifactory"] -RUN ["pip", "install", "argparse"] -RUN ["pip", "install", "coverage"] -RUN ["pip", "install", "pytest"] +RUN ["pip", "install", "pyyaml", "artifactory", "argparse", "coverage", "pytest"] COPY . /app RUN rm -rf build/ RUN rm -rf dist/ @@ -169,7 +88,7 @@ def test_buildDockerfile_entrypoint_path(self, mock_getcwd, mock_expanduser): sb.systems.generators.dockerfile.buildDockerfile(config) data = None - with open(filePath, "r") as file: + with open(filePath, "r", encoding="utf-8") as file: data = file.read() self.assertTrue(data is not None) self.assertEqual(data, expectedDockerfile) @@ -193,14 +112,10 @@ def test_buildDockerfile_cmd_path(self, mock_getcwd, mock_expanduser): # This Dockerfile was generated by Skelebot # Editing this file manually is not advised as all changes will be overwritten by Skelebot -FROM skelebot/python-base:3.6 +FROM skelebot/python-base:3.9 MAINTAINER Mega Man WORKDIR /app -RUN ["pip", "install", "pyyaml"] -RUN ["pip", "install", "artifactory"] -RUN ["pip", "install", "argparse"] -RUN ["pip", "install", "coverage"] -RUN ["pip", "install", "pytest"] +RUN ["pip", "install", "pyyaml", "artifactory", "argparse", "coverage", "pytest"] COPY . /app RUN rm -rf build/ RUN rm -rf dist/ @@ -209,7 +124,7 @@ def test_buildDockerfile_cmd_path(self, mock_getcwd, mock_expanduser): sb.systems.generators.dockerfile.buildDockerfile(config) data = None - with open(filePath, "r") as file: + with open(filePath, "r", encoding="utf-8") as file: data = file.read() self.assertTrue(data is not None) self.assertEqual(data, expectedDockerfile) @@ -226,7 +141,6 @@ def test_buildDockerfile_py_ca_file_error(self, mock_getcwd, mock_mkdir, mock_ex mock_mkdir.return_value = 1 mock_call.return_value = 1 config = sb.systems.generators.yaml.loadConfig() - config.language = "Python" config.dependencies.append("ca_file:cars:12345:python-pkg:ml-lib:0.1.0:prod") try: @@ -239,7 +153,7 @@ def test_buildDockerfile_py_ca_file_error(self, mock_getcwd, mock_mkdir, mock_ex @mock.patch('os.path.expanduser') @mock.patch('os.makedirs') @mock.patch('os.getcwd') - def test_buildDockerfile_base_py(self, mock_getcwd, mock_mkdir, mock_expanduser, mock_call): + def test_buildDockerfile_base(self, mock_getcwd, mock_mkdir, mock_expanduser, mock_call): folderPath = "{path}/test/files".format(path=self.path) filePath = "{folder}/Dockerfile".format(folder=folderPath) @@ -248,7 +162,6 @@ def test_buildDockerfile_base_py(self, mock_getcwd, mock_mkdir, mock_expanduser, mock_mkdir.return_value = 1 mock_call.return_value = 0 config = sb.systems.generators.yaml.loadConfig() - config.language = "Python" config.dependencies.append("github:github.com/repo") config.dependencies.append("github:https://github.com/securerepo") config.dependencies.append("file:libs/proj") @@ -261,24 +174,13 @@ def test_buildDockerfile_base_py(self, mock_getcwd, mock_mkdir, mock_expanduser, # This Dockerfile was generated by Skelebot # Editing this file manually is not advised as all changes will be overwritten by Skelebot -FROM skelebot/python-base:3.6 +FROM skelebot/python-base:3.9 MAINTAINER Mega Man WORKDIR /app -RUN ["pip", "install", "pyyaml"] -RUN ["pip", "install", "artifactory"] -RUN ["pip", "install", "argparse"] -RUN ["pip", "install", "coverage"] -RUN ["pip", "install", "pytest"] -RUN ["pip", "install", "git+github.com/repo"] -RUN ["pip", "install", "git+https://github.com/securerepo"] COPY libs/proj libs/proj -RUN ["pip", "install", "/app/libs/proj"] COPY libs/ml_lib-0.1.0-py3-none-any.whl libs/ml_lib-0.1.0-py3-none-any.whl -RUN ["pip", "install", "/app/libs/ml_lib-0.1.0-py3-none-any.whl"] COPY requirements.txt requirements.txt -RUN ["pip", "install", "-r", "/app/requirements.txt"] -RUN ["pip", "install", "requests", "numpy==1.22.0", "pandas~=1.1", "scikit-learn<=2.0.0 ; python_version<=\'3.6\'", "pytest ~= 6.2", "pytest-cov ~= 3.0", "fake-package == 1.2.3", "not-real"] -RUN ["pip", "install", "dtable==9.0"] +RUN ["pip", "install", "pyyaml", "artifactory", "argparse", "coverage", "pytest", "git+github.com/repo", "git+https://github.com/securerepo", "/app/libs/proj", "/app/libs/ml_lib-0.1.0-py3-none-any.whl", "-r", "/app/requirements.txt", "requests", "numpy==1.22.0", "pandas~=1.1", "scikit-learn<=2.0.0 ; python_version<=\'3.6\'", "pytest ~= 6.2", "pytest-cov ~= 3.0", "fake-package == 1.2.3", "not-real", "dtable==9.0"] COPY . /app RUN rm -rf build/ RUN rm -rf dist/ @@ -289,7 +191,7 @@ def test_buildDockerfile_base_py(self, mock_getcwd, mock_mkdir, mock_expanduser, sb.systems.generators.dockerfile.buildDockerfile(config) data = None - with open(filePath, "r") as file: + with open(filePath, "r", encoding="utf-8") as file: data = file.read() mock_mkdir.assert_called_with("{folder}/libs".format(folder=folderPath), exist_ok=True) mock_call.assert_called_with(expectedCMD, shell=True) @@ -298,14 +200,13 @@ def test_buildDockerfile_base_py(self, mock_getcwd, mock_mkdir, mock_expanduser, @mock.patch('os.path.expanduser') @mock.patch('os.getcwd') - def test_buildDockerfile_base_py_versions(self, mock_getcwd, mock_expanduser): + def test_buildDockerfile_base_versions(self, mock_getcwd, mock_expanduser): folderPath = "{path}/test/files".format(path=self.path) filePath = "{folder}/Dockerfile".format(folder=folderPath) mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path) mock_getcwd.return_value = folderPath config = sb.systems.generators.yaml.loadConfig() - config.language = "Python" config.dependencies.append("dtable=9.0") config.dependencies.append("pandas==0.25") config.dependencies.append("numpy~=1.17") @@ -316,19 +217,10 @@ def test_buildDockerfile_base_py_versions(self, mock_getcwd, mock_expanduser): # This Dockerfile was generated by Skelebot # Editing this file manually is not advised as all changes will be overwritten by Skelebot -FROM skelebot/python-base:3.6 +FROM skelebot/python-base:3.9 MAINTAINER Mega Man WORKDIR /app -RUN ["pip", "install", "pyyaml"] -RUN ["pip", "install", "artifactory"] -RUN ["pip", "install", "argparse"] -RUN ["pip", "install", "coverage"] -RUN ["pip", "install", "pytest"] -RUN ["pip", "install", "dtable==9.0"] -RUN ["pip", "install", "pandas==0.25"] -RUN ["pip", "install", "numpy~=1.17"] -RUN ["pip", "install", "requests>= 2.2, == 2.*"] -RUN ["pip", "install", "scipy!= 1.3.*"] +RUN ["pip", "install", "pyyaml", "artifactory", "argparse", "coverage", "pytest", "dtable==9.0", "pandas==0.25", "numpy~=1.17", "requests>= 2.2, == 2.*", "scipy!= 1.3.*"] COPY . /app RUN rm -rf build/ RUN rm -rf dist/ @@ -337,7 +229,7 @@ def test_buildDockerfile_base_py_versions(self, mock_getcwd, mock_expanduser): sb.systems.generators.dockerfile.buildDockerfile(config) data = None - with open(filePath, "r") as file: + with open(filePath, "r", encoding="utf-8") as file: data = file.read() self.assertTrue(data is not None) self.assertEqual(data, expectedDockerfile) @@ -351,9 +243,8 @@ def test_buildDockerfile_krb(self, mock_getcwd, mock_expanduser): mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path) mock_getcwd.return_value = folderPath config = sb.systems.generators.yaml.loadConfig() - config.language = "R" - config.dependencies.append("github:github.com/repo:cool-lib") - config.dependencies.append("file:libs/proj:cool-proj") + config.dependencies.append("github:github.com/repo") + config.dependencies.append("file:libs/proj") config.dependencies.append("dtable=9.0") config.components.append(sb.components.kerberos.Kerberos("conf", "tab", "user")) @@ -361,18 +252,11 @@ def test_buildDockerfile_krb(self, mock_getcwd, mock_expanduser): # This Dockerfile was generated by Skelebot # Editing this file manually is not advised as all changes will be overwritten by Skelebot -FROM skelebot/r-krb +FROM skelebot/python-krb MAINTAINER Mega Man WORKDIR /app -RUN ["Rscript", "-e", "install.packages('pyyaml', repo='https://cloud.r-project.org'); library(pyyaml)"] -RUN ["Rscript", "-e", "install.packages('artifactory', repo='https://cloud.r-project.org'); library(artifactory)"] -RUN ["Rscript", "-e", "install.packages('argparse', repo='https://cloud.r-project.org'); library(argparse)"] -RUN ["Rscript", "-e", "install.packages('coverage', repo='https://cloud.r-project.org'); library(coverage)"] -RUN ["Rscript", "-e", "install.packages('pytest', repo='https://cloud.r-project.org'); library(pytest)"] -RUN ["Rscript", "-e", "library(devtools); install_github('github.com/repo'); library(cool-lib)"] COPY libs/proj libs/proj -RUN ["Rscript", "-e", "install.packages('/app/libs/proj', repos=NULL, type='source'); library(cool-proj)"] -RUN ["Rscript", "-e", "library(devtools); install_version('dtable', version='9.0', repos='http://cran.us.r-project.org'); library(dtable)"] +RUN ["pip", "install", "pyyaml", "artifactory", "argparse", "coverage", "pytest", "git+github.com/repo", "/app/libs/proj", "dtable==9.0"] COPY . /app RUN rm -rf build/ RUN rm -rf dist/ @@ -383,7 +267,7 @@ def test_buildDockerfile_krb(self, mock_getcwd, mock_expanduser): sb.systems.generators.dockerfile.buildDockerfile(config) data = None - with open(filePath, "r") as file: + with open(filePath, "r", encoding="utf-8") as file: data = file.read() self.assertTrue(data is not None) self.assertEqual(data, expectedDockerfile) @@ -399,22 +283,22 @@ def test_buildDockerfile_no_command(self, mock_getcwd, mock_expanduser): config = sb.systems.generators.yaml.loadConfig() # No custom docker run command config.commands = [] - config.language = None expectedDockerfile = """ # This Dockerfile was generated by Skelebot # Editing this file manually is not advised as all changes will be overwritten by Skelebot -FROM ubuntu:18.04 +FROM skelebot/python-base:3.9 MAINTAINER Mega Man WORKDIR /app +RUN ["pip", "install", "pyyaml", "artifactory", "argparse", "coverage", "pytest"] COPY . /app CMD /bin/bash -c \"bash build.sh --env local --log info\"\n""" sb.systems.generators.dockerfile.buildDockerfile(config) data = None - with open(filePath, "r") as file: + with open(filePath, "r", encoding="utf-8") as file: data = file.read() self.assertTrue(data is not None) self.assertEqual(data, expectedDockerfile) @@ -435,9 +319,10 @@ def test_buildDockerfile_append_command(self, mock_getcwd, mock_expanduser): # This Dockerfile was generated by Skelebot # Editing this file manually is not advised as all changes will be overwritten by Skelebot -FROM ubuntu:18.04 +FROM skelebot/python-base:3.9 MAINTAINER Mega Man WORKDIR /app +RUN ["pip", "install", "pyyaml", "artifactory", "argparse", "coverage", "pytest"] COPY . /app RUN rm -rf build/ RUN rm -rf dist/ @@ -446,7 +331,7 @@ def test_buildDockerfile_append_command(self, mock_getcwd, mock_expanduser): sb.systems.generators.dockerfile.buildDockerfile(config) data = None - with open(filePath, "r") as file: + with open(filePath, "r", encoding="utf-8") as file: data = file.read() self.assertTrue(data is not None) self.assertEqual(data, expectedDockerfile) @@ -469,11 +354,7 @@ def test_buildDockerfile_custom(self, mock_getcwd, mock_expanduser): FROM whatever:uwant MAINTAINER Mega Man WORKDIR /app -RUN ["pip", "install", "pyyaml"] -RUN ["pip", "install", "artifactory"] -RUN ["pip", "install", "argparse"] -RUN ["pip", "install", "coverage"] -RUN ["pip", "install", "pytest"] +RUN ["pip", "install", "pyyaml", "artifactory", "argparse", "coverage", "pytest"] COPY . /app RUN rm -rf build/ RUN rm -rf dist/ @@ -482,118 +363,11 @@ def test_buildDockerfile_custom(self, mock_getcwd, mock_expanduser): sb.systems.generators.dockerfile.buildDockerfile(config) data = None - with open(filePath, "r") as file: + with open(filePath, "r", encoding="utf-8") as file: data = file.read() self.assertTrue(data is not None) self.assertEqual(data, expectedDockerfile) - @mock.patch('skelebot.systems.generators.dockerfile.call') - @mock.patch('os.path.expanduser') - @mock.patch('os.makedirs') - @mock.patch('os.getcwd') - def test_buildDockerfile_R_py_ca_file_error(self, mock_getcwd, mock_mkdir, mock_expanduser, mock_call): - folderPath = "{path}/test/files".format(path=self.path) - - mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path) - mock_getcwd.return_value = folderPath - mock_mkdir.return_value = 1 - mock_call.return_value = -1 - config = sb.systems.generators.yaml.loadConfig() - config.language = "R+Python" - config.dependencies = { - "Python":[ - "numpy", "pandas", - "github:github.com/repo", "github:https://github.com/securerepo", - "file:libs/proj", - "ca_file:cars:12345:python-pkg:ml-lib:0.1.0:prod", - "dtable>=9.0", "dtable=9.0" - ], - "R":[ - "data.table", "here", - "github:github.com/repo:cool-lib", - "file:libs/proj:cool-proj", - "dtable=9.0" - ] - } - - try: - sb.systems.generators.dockerfile.buildDockerfile(config) - self.fail("Exception Not Thrown") - except Exception as exc: - self.assertEqual(str(exc), "Failed to Obtain CodeArtifact Package") - - @mock.patch('skelebot.systems.generators.dockerfile.call') - @mock.patch('os.path.expanduser') - @mock.patch('os.makedirs') - @mock.patch('os.getcwd') - def test_buildDockerfile_R_plus_Python(self, mock_getcwd, mock_mkdir, mock_expanduser, mock_call): - folderPath = "{path}/test/files".format(path=self.path) - filePath = "{folder}/Dockerfile".format(folder=folderPath) - - mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path) - mock_getcwd.return_value = folderPath - mock_mkdir.return_value = 1 - mock_call.return_value = 0 - config = sb.systems.generators.yaml.loadConfig() - config.language = "R+Python" - config.dependencies = { - "Python":[ - "numpy", "pandas", - "github:github.com/repo", "github:https://github.com/securerepo", - "file:libs/proj", - "ca_file:cars:12345:python-pkg:ml-lib:0.1.0:prod", - "dtable>=9.0", "dtable=9.0" - ], - "R":[ - "data.table", "here", - "github:github.com/repo:cool-lib", - "file:libs/proj:cool-proj", - "dtable=9.0" - ] - } - config.components.append(sb.components.kerberos.Kerberos("conf", "tab", "user")) - - expectedDockerfile = """ -# This Dockerfile was generated by Skelebot -# Editing this file manually is not advised as all changes will be overwritten by Skelebot - -FROM skelebot/r-krb -MAINTAINER Mega Man -WORKDIR /app -RUN ["pip3", "install", "numpy"] -RUN ["pip3", "install", "pandas"] -RUN ["pip3", "install", "git+github.com/repo"] -RUN ["pip3", "install", "git+https://github.com/securerepo"] -COPY libs/proj libs/proj -RUN ["pip3", "install", "/app/libs/proj"] -COPY libs/ml_lib-0.1.0-py3-none-any.whl libs/ml_lib-0.1.0-py3-none-any.whl -RUN ["pip3", "install", "/app/libs/ml_lib-0.1.0-py3-none-any.whl"] -RUN ["pip3", "install", "dtable>=9.0"] -RUN ["pip3", "install", "dtable==9.0"] -RUN ["Rscript", "-e", "install.packages('data.table', repo='https://cloud.r-project.org'); library(data.table)"] -RUN ["Rscript", "-e", "install.packages('here', repo='https://cloud.r-project.org'); library(here)"] -RUN ["Rscript", "-e", "library(devtools); install_github('github.com/repo'); library(cool-lib)"] -COPY libs/proj libs/proj -RUN ["Rscript", "-e", "install.packages('/app/libs/proj', repos=NULL, type='source'); library(cool-proj)"] -RUN ["Rscript", "-e", "library(devtools); install_version('dtable', version='9.0', repos='http://cran.us.r-project.org'); library(dtable)"] -COPY . /app -RUN rm -rf build/ -RUN rm -rf dist/ -COPY conf /etc/krb5.conf -COPY tab /krb/auth.keytab -CMD /bin/bash -c "/./krb/init.sh user && bash build.sh --env local --log info\"\n""" - sb.systems.generators.dockerfile.buildDockerfile(config) - - expectedCMD = "aws codeartifact get-package-version-asset --domain cars --domain-owner 12345 --repository python-pkg --package ml-lib --package-version 0.1.0 --profile prod --format pypi --asset ml_lib-0.1.0-py3-none-any.whl libs/ml_lib-0.1.0-py3-none-any.whl" - - data = None - with open(filePath, "r") as file: - data = file.read() - self.assertTrue(data is not None) - self.assertEqual(data, expectedDockerfile) - mock_mkdir.assert_called_with("{folder}/libs".format(folder=folderPath), exist_ok=True) - mock_call.assert_called_with(expectedCMD, shell=True) - @mock.patch('os.path.expanduser') @mock.patch('os.getcwd') def test_buildDockerfile_timezone(self, mock_getcwd, mock_expanduser): @@ -603,30 +377,22 @@ def test_buildDockerfile_timezone(self, mock_getcwd, mock_expanduser): mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path) mock_getcwd.return_value = folderPath config = sb.systems.generators.yaml.loadConfig() - config.language = "R" config.timezone = "America/Chicago" - config.dependencies.append("github:github.com/repo:cool-lib") - config.dependencies.append("file:libs/proj:cool-proj") + config.dependencies.append("github:github.com/repo") + config.dependencies.append("file:libs/proj") config.dependencies.append("dtable=9.0") expectedDockerfile = """ # This Dockerfile was generated by Skelebot # Editing this file manually is not advised as all changes will be overwritten by Skelebot -FROM skelebot/r-base +FROM skelebot/python-base:3.9 MAINTAINER Mega Man WORKDIR /app ENV TZ=America/Chicago RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN ["Rscript", "-e", "install.packages('pyyaml', repo='https://cloud.r-project.org'); library(pyyaml)"] -RUN ["Rscript", "-e", "install.packages('artifactory', repo='https://cloud.r-project.org'); library(artifactory)"] -RUN ["Rscript", "-e", "install.packages('argparse', repo='https://cloud.r-project.org'); library(argparse)"] -RUN ["Rscript", "-e", "install.packages('coverage', repo='https://cloud.r-project.org'); library(coverage)"] -RUN ["Rscript", "-e", "install.packages('pytest', repo='https://cloud.r-project.org'); library(pytest)"] -RUN ["Rscript", "-e", "library(devtools); install_github('github.com/repo'); library(cool-lib)"] COPY libs/proj libs/proj -RUN ["Rscript", "-e", "install.packages('/app/libs/proj', repos=NULL, type='source'); library(cool-proj)"] -RUN ["Rscript", "-e", "library(devtools); install_version('dtable', version='9.0', repos='http://cran.us.r-project.org'); library(dtable)"] +RUN ["pip", "install", "pyyaml", "artifactory", "argparse", "coverage", "pytest", "git+github.com/repo", "/app/libs/proj", "dtable==9.0"] COPY . /app RUN rm -rf build/ RUN rm -rf dist/ @@ -635,7 +401,7 @@ def test_buildDockerfile_timezone(self, mock_getcwd, mock_expanduser): sb.systems.generators.dockerfile.buildDockerfile(config) data = None - with open(filePath, "r") as file: + with open(filePath, "r", encoding="utf-8") as file: data = file.read() self.assertTrue(data is not None) self.assertEqual(data, expectedDockerfile) diff --git a/test/test_systems_generators_dockerignore.py b/test/test_systems_generators_dockerignore.py index d505540..fe5d644 100644 --- a/test/test_systems_generators_dockerignore.py +++ b/test/test_systems_generators_dockerignore.py @@ -26,7 +26,6 @@ def test_buildDockerignore(self, mock_getcwd, mock_expanduser): # Editing this file manually is not advised as all changes will be overwritten by Skelebot **/*.zip -**/*.RData **/*.pkl **/*.csv **/*.model @@ -36,7 +35,7 @@ def test_buildDockerignore(self, mock_getcwd, mock_expanduser): sb.systems.generators.dockerignore.buildDockerignore(self.config) data = None - with open(filePath, "r") as file: + with open(filePath, "r", encoding="utf-8") as file: data = file.read() self.assertTrue(data is not None) self.assertEqual(data, expected) diff --git a/test/test_systems_generators_yaml.py b/test/test_systems_generators_yaml.py index bc454d0..5c75c6c 100644 --- a/test/test_systems_generators_yaml.py +++ b/test/test_systems_generators_yaml.py @@ -20,14 +20,13 @@ def validateYaml(self, config, isTestEnv=False): self.assertEqual(config.version, "6.6.6") self.assertEqual(config.maintainer, "Mega Man") self.assertEqual(config.contact, "megaman@cars.com") - self.assertEqual(config.language, "Python") self.assertEqual(config.primaryJob, "build") self.assertEqual(config.ephemeral, True if isTestEnv else None) self.assertEqual(config.dependencies, ["pyyaml", "artifactory", "argparse", "coverage", "pytest"]) if isTestEnv: self.assertEqual(config.ignores, ['data/', 'libs/']) else: - self.assertEqual(config.ignores, ['**/*.zip', '**/*.RData', '**/*.pkl', '**/*.csv', '**/*.model', '**/*.pyc']) + self.assertEqual(config.ignores, ['**/*.zip', '**/*.pkl', '**/*.csv', '**/*.model', '**/*.pyc']) self.assertEqual(config.commands, ["rm -rf build/", "rm -rf dist/"]) self.assertEqual(config.jobs[0].name, "build") self.assertEqual(config.jobs[0].source, "build.sh") @@ -51,9 +50,7 @@ def validateYaml(self, config, isTestEnv=False): self.assertEqual(component.folder, "notebooks/") expectedComponents = ["Registry", "Plugin", "Jupyter", "Bump", "Prime", "Environments", "Dexec", "AddNumbers"] - if not isTestEnv: - expectedComponents.append("Artifactory") - else: + if isTestEnv: expectedComponents.append("Repository") self.assertTrue(all(elem in components for elem in expectedComponents)) self.assertTrue(all(elem in expectedComponents for elem in components)) @@ -121,7 +118,6 @@ def test_loadConfig_without_yaml(self, mock_getcwd, mock_expanduser): self.assertEqual(config.version, None) self.assertEqual(config.maintainer, None) self.assertEqual(config.contact, None) - self.assertEqual(config.language, None) self.assertEqual(config.primaryJob, None) self.assertEqual(config.ephemeral, None) self.assertEqual(config.dependencies, []) diff --git a/test/test_systems_scaffolding_scaffolder.py b/test/test_systems_scaffolding_scaffolder.py index 9acc86d..9b501c8 100644 --- a/test/test_systems_scaffolding_scaffolder.py +++ b/test/test_systems_scaffolding_scaffolder.py @@ -3,8 +3,6 @@ import unittest from unittest import mock -from colorama import Fore, Style - import skelebot as sb from skelebot.components.bump import Bump @@ -32,7 +30,7 @@ } ], "config": { - "language": "{language}", + "name": "{name}", "dependencies": ["dash~=2.0"], "ports": ["5000:5000"], "primaryJob": "run", @@ -52,7 +50,7 @@ class TestScaffolder(unittest.TestCase): @mock.patch('skelebot.systems.scaffolding.scaffolder.promptUser') def test_execute_scaffold_abort(self, mock_prompt, mock_cFactory, mock_getcwd, mock_expanduser): mock_expanduser.return_value = "test/plugins" - mock_prompt.side_effect = ["test", "test proj", "sean", "email", "Python", "Default", False] + mock_prompt.side_effect = ["test", "test proj", "sean", "email", "Default", False] try: scaffolder = sb.systems.scaffolding.scaffolder.Scaffolder(existing=False) scaffolder.scaffold() @@ -64,41 +62,9 @@ def test_execute_scaffold_abort(self, mock_prompt, mock_cFactory, mock_getcwd, m mock_prompt.assert_any_call("Enter a PROJECT DESCRIPTION") mock_prompt.assert_any_call("Enter a MAINTAINER NAME") mock_prompt.assert_any_call("Enter a CONTACT EMAIL") - mock_prompt.assert_any_call("Select a LANGUAGE", - options=["Python", "R", "R+Python"], - deprecated_options=["R", "R+Python"]) mock_prompt.assert_any_call("Select a TEMPLATE", options=["Default", "Dash", "Git"]) mock_prompt.assert_any_call("Confirm Skelebot Setup", boolean=True) - @mock.patch('os.path.expanduser') - @mock.patch('os.getcwd') - @mock.patch('skelebot.systems.scaffolding.scaffolder.ComponentFactory') - @mock.patch('skelebot.systems.scaffolding.scaffolder.promptUser') - @mock.patch('skelebot.systems.scaffolding.scaffolder.print') - def test_execute_scaffold_deprecated(self, mock_print, mock_prompt, mock_cFactory, - mock_getcwd, mock_expanduser): - mock_expanduser.return_value = "test/plugins" - mock_prompt.side_effect = ["test", "test proj", "sean", "email", "R", "Default", False] - try: - scaffolder = sb.systems.scaffolding.scaffolder.Scaffolder(existing=False) - scaffolder.scaffold() - self.fail("Exception Expected") - except Exception as exc: - self.assertEqual(str(exc), "Aborting Scaffolding Process") - - mock_prompt.assert_any_call("Enter a PROJECT NAME") - mock_prompt.assert_any_call("Enter a PROJECT DESCRIPTION") - mock_prompt.assert_any_call("Enter a MAINTAINER NAME") - mock_prompt.assert_any_call("Enter a CONTACT EMAIL") - mock_prompt.assert_any_call("Select a LANGUAGE", - options=["Python", "R", "R+Python"], - deprecated_options=["R", "R+Python"]) - mock_prompt.assert_any_call("Select a TEMPLATE", options=["Default", "Git"]) - mock_prompt.assert_any_call("Confirm Skelebot Setup", boolean=True) - - mock_print.assert_any_call(Fore.YELLOW + "WARN" + Style.RESET_ALL - + " | The support for R language has been deprecated as of v1.37.0." - + " Please use a different language.") @mock.patch('os.path.expanduser') @mock.patch('os.getcwd') @@ -110,7 +76,7 @@ def test_execute_scaffold_existing_init(self, mock_prompt, mock_yaml, #mock_cFac mock_makedirs, mock_getcwd, mock_expanduser): mock_expanduser.return_value = "test/plugins" mock_prompt.side_effect = ["test", "test proj", "sean", "email", - "Python", "Dash", True] + "Dash", True] scaffolder = sb.systems.scaffolding.scaffolder.Scaffolder(existing=True) scaffolder.scaffold() @@ -119,9 +85,6 @@ def test_execute_scaffold_existing_init(self, mock_prompt, mock_yaml, #mock_cFac mock_prompt.assert_any_call("Enter a PROJECT DESCRIPTION") mock_prompt.assert_any_call("Enter a MAINTAINER NAME") mock_prompt.assert_any_call("Enter a CONTACT EMAIL") - mock_prompt.assert_any_call("Select a LANGUAGE", - options=["Python", "R", "R+Python"], - deprecated_options=["R", "R+Python"]) mock_prompt.assert_any_call("Select a TEMPLATE", options=["Default", "Dash", "Git"]) mock_prompt.assert_any_call("Confirm Skelebot Setup", boolean=True) @@ -151,7 +114,7 @@ def test_execute_scaffold_git_pull(self, mock_prompt, mock_yaml, mock_readme, mo mock_expanduser.return_value = "test/plugins" mock_prompt.side_effect = ["test", "test proj", "sean", "email", - "Python", "Git", "git@repo", "data-prod", True] + "Git", "git@repo", "data-prod", True] exp_cfg = copy.deepcopy(TEMPLATE) mock_pyyaml.load.return_value = exp_cfg @@ -173,28 +136,24 @@ def test_execute_scaffold_git_pull(self, mock_prompt, mock_yaml, mock_readme, mo mock_prompt.assert_any_call("Enter a PROJECT DESCRIPTION") mock_prompt.assert_any_call("Enter a MAINTAINER NAME") mock_prompt.assert_any_call("Enter a CONTACT EMAIL") - mock_prompt.assert_any_call("Select a LANGUAGE", - options=["Python", "R", "R+Python"], - deprecated_options=["R", "R+Python"]) mock_prompt.assert_any_call("Select a TEMPLATE", options=["Default", "Dash", "Git"]) mock_prompt.assert_any_call("Enter AWS-PROD PROFILE") mock_prompt.assert_any_call("Confirm Skelebot Setup", boolean=True) - exp_cfg["config"]["language"] = "Python" exp_cfg["config"]["name"] = "test" exp_cfg["config"]["components"] = {"magicmock": {"fake_attr": "aaaa"}} mock_config.load.assert_called_once_with(exp_cfg["config"]) mock_makedirs.assert_any_call("src/assets/", exist_ok=True) - dirname = os.path.dirname(os.path.dirname(__file__)) - mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/git_repo/files/app_py"), "r") - mock_open.assert_any_call("src/app.py", "w") - mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/git_repo/files/server_py"), "r") - mock_open.assert_any_call("src/server.py", "w") - mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/git_repo/files/config_py"), "r") - mock_open.assert_any_call("src/config.py", "w") - mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/git_repo/files/style_css"), "r") - mock_open.assert_any_call("src/assets/style.css", "w") + dirname = os.path.dirname(os.path.dirname(sb.__file__)) + mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/git_repo/files/app_py"), "r", encoding="utf-8") + mock_open.assert_any_call("src/app.py", "w", encoding="utf-8") + mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/git_repo/files/server_py"), "r", encoding="utf-8") + mock_open.assert_any_call("src/server.py", "w", encoding="utf-8") + mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/git_repo/files/config_py"), "r", encoding="utf-8") + mock_open.assert_any_call("src/config.py", "w", encoding="utf-8") + mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/git_repo/files/style_css"), "r", encoding="utf-8") + mock_open.assert_any_call("src/assets/style.css", "w", encoding="utf-8") mock_dockerfile.buildDockerfile.assert_called_once() mock_dignore.buildDockerignore.assert_called_once() @@ -227,7 +186,7 @@ def test_execute_scaffold_git_clone(self, mock_prompt, mock_yaml, mock_readme, m mock_expanduser.return_value = "test/plugins" mock_prompt.side_effect = ["test", "test proj", "sean", "email", - "Python", "Git", "git@repo", "data-prod", True] + "Git", "git@repo", "data-prod", True] mock_pyyaml.load.return_value = copy.deepcopy(TEMPLATE) @@ -245,9 +204,6 @@ def test_execute_scaffold_git_clone(self, mock_prompt, mock_yaml, mock_readme, m mock_prompt.assert_any_call("Enter a PROJECT DESCRIPTION") mock_prompt.assert_any_call("Enter a MAINTAINER NAME") mock_prompt.assert_any_call("Enter a CONTACT EMAIL") - mock_prompt.assert_any_call("Select a LANGUAGE", - options=["Python", "R", "R+Python"], - deprecated_options=["R", "R+Python"]) mock_prompt.assert_any_call("Select a TEMPLATE", options=["Default", "Dash", "Git"]) mock_prompt.assert_any_call("Enter AWS-PROD PROFILE") mock_prompt.assert_any_call("Confirm Skelebot Setup", boolean=True) @@ -255,15 +211,15 @@ def test_execute_scaffold_git_clone(self, mock_prompt, mock_yaml, mock_readme, m mock_config.load.assert_called_once() mock_makedirs.assert_any_call("src/assets/", exist_ok=True) - dirname = os.path.dirname(os.path.dirname(__file__)) - mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/git_repo/files/app_py"), "r") - mock_open.assert_any_call("src/app.py", "w") - mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/git_repo/files/server_py"), "r") - mock_open.assert_any_call("src/server.py", "w") - mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/git_repo/files/config_py"), "r") - mock_open.assert_any_call("src/config.py", "w") - mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/git_repo/files/style_css"), "r") - mock_open.assert_any_call("src/assets/style.css", "w") + dirname = os.path.dirname(os.path.dirname(sb.__file__)) + mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/git_repo/files/app_py"), "r", encoding="utf-8") + mock_open.assert_any_call("src/app.py", "w", encoding="utf-8") + mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/git_repo/files/server_py"), "r", encoding="utf-8") + mock_open.assert_any_call("src/server.py", "w", encoding="utf-8") + mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/git_repo/files/config_py"), "r", encoding="utf-8") + mock_open.assert_any_call("src/config.py", "w", encoding="utf-8") + mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/git_repo/files/style_css"), "r", encoding="utf-8") + mock_open.assert_any_call("src/assets/style.css", "w", encoding="utf-8") mock_dockerfile.buildDockerfile.assert_called_once() mock_dignore.buildDockerignore.assert_called_once() @@ -289,7 +245,7 @@ def test_execute_scaffold_dash(self, mock_prompt, mock_yaml, mock_readme, mock_p mock_makedirs, mock_getcwd, mock_expanduser): mock_expanduser.return_value = "test/plugins" mock_prompt.side_effect = ["test", "test proj", "sean", "email", - "Python", "Dash", "data-prod", True] + "Dash", "data-prod", True] mock_pyyaml.load.return_value = copy.deepcopy(TEMPLATE) @@ -307,9 +263,6 @@ def test_execute_scaffold_dash(self, mock_prompt, mock_yaml, mock_readme, mock_p mock_prompt.assert_any_call("Enter a PROJECT DESCRIPTION") mock_prompt.assert_any_call("Enter a MAINTAINER NAME") mock_prompt.assert_any_call("Enter a CONTACT EMAIL") - mock_prompt.assert_any_call("Select a LANGUAGE", - options=["Python", "R", "R+Python"], - deprecated_options=["R", "R+Python"]) mock_prompt.assert_any_call("Select a TEMPLATE", options=["Default", "Dash", "Git"]) mock_prompt.assert_any_call("Enter AWS-PROD PROFILE") mock_prompt.assert_any_call("Confirm Skelebot Setup", boolean=True) @@ -317,15 +270,15 @@ def test_execute_scaffold_dash(self, mock_prompt, mock_yaml, mock_readme, mock_p mock_config.load.assert_called_once() mock_makedirs.assert_any_call("src/assets/", exist_ok=True) - dirname = os.path.dirname(os.path.dirname(__file__)) - mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/python_dash/files/app_py"), "r") - mock_open.assert_any_call("src/app.py", "w") - mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/python_dash/files/server_py"), "r") - mock_open.assert_any_call("src/server.py", "w") - mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/python_dash/files/config_py"), "r") - mock_open.assert_any_call("src/config.py", "w") - mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/python_dash/files/style_css"), "r") - mock_open.assert_any_call("src/assets/style.css", "w") + dirname = os.path.dirname(os.path.dirname(sb.__file__)) + mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/python_dash/files/app_py"), "r", encoding="utf-8") + mock_open.assert_any_call("src/app.py", "w", encoding="utf-8") + mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/python_dash/files/server_py"), "r", encoding="utf-8") + mock_open.assert_any_call("src/server.py", "w", encoding="utf-8") + mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/python_dash/files/config_py"), "r", encoding="utf-8") + mock_open.assert_any_call("src/config.py", "w", encoding="utf-8") + mock_open.assert_any_call(os.path.join(dirname, "skelebot/systems/scaffolding/templates/python_dash/files/style_css"), "r", encoding="utf-8") + mock_open.assert_any_call("src/assets/style.css", "w", encoding="utf-8") mock_dockerfile.buildDockerfile.assert_called_once() mock_dignore.buildDockerignore.assert_called_once()