ci/cd #59
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # .github/workflows/test-code.yaml | |
| # Name of the workflow | |
| name: Validate model | |
| # Trigger when? | |
| on: | |
| push: # this will trigger the workflow/pipeline only if there is push on `main` branch | |
| branches: | |
| - main | |
| - dev | |
| paths: # the push should be specifically to the folders `src` or `scripts` to trigger this workflow, otherwise, the workflow will not be triggered | |
| - 'src/**' | |
| - 'scripts/**' | |
| - 'services/airflow/dags/**' | |
| - 'pipelines/**' | |
| - 'tests/**' | |
| - 'configs/**' | |
| - '.github/**' | |
| # Allows to only read the contents of the repository | |
| # `contents: read` permits an action to list the commits | |
| # `contents: write` allows the action to create a release | |
| permissions: | |
| contents: read | |
| # Declare environment variables to be used in this workflow file | |
| env: | |
| message: "Validating model!" | |
| # Tasks | |
| jobs: | |
| # Task name | |
| validate-model: | |
| # OS to run the task | |
| runs-on: ubuntu-latest # The ubuntu-latest label currently uses the Ubuntu 22.04 runner image | |
| defaults: # Set working directory of the job | |
| run: | |
| shell: bash # Set the default shell | |
| working-directory: . | |
| # The steps of the task/job | |
| steps: | |
| - name: Checking out our code | |
| uses: actions/checkout@v4 | |
| - name: setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11.0' | |
| cache: 'pip' # caching pip dependencies | |
| - name: install python packages | |
| run: | | |
| pip3.11 install -r requirements.txt | |
| - name: create-json | |
| id: create-json | |
| uses: jsdaniell/create-json@1.1.2 | |
| with: | |
| name: "gdrive-credentials.json" | |
| json: ${{ secrets.GDRIVE_CREDENTIALS_DATA }} | |
| - name: Configure DVC remote with service account | |
| run: | | |
| dvc remote modify storage gdrive_use_service_account true | |
| dvc remote modify storage --local gdrive_service_account_json_file_path gdrive-credentials.json | |
| - name: Pull DVC data | |
| run: | | |
| dvc fetch | |
| dvc pull | |
| - name: Test git config | |
| run: | | |
| git status | |
| git pull | |
| git status | |
| - name: Fetch tags | |
| run: | | |
| git fetch --tags --force | |
| # - name: Git checkout | |
| # run: | | |
| # git checkout main | |
| - name: Test tags version | |
| run: | | |
| git tag | |
| - name: Test git checkout | |
| run: | | |
| git checkout v1.0 data/samples.dvc | |
| dvc pull | |
| dvc checkout data/samples.dvc | |
| git checkout v2.0 data/samples.dvc | |
| dvc pull | |
| dvc checkout data/samples.dvc | |
| git checkout v3.0 data/samples.dvc | |
| dvc pull | |
| dvc checkout data/samples.dvc | |
| git checkout v4.0 data/samples.dvc | |
| dvc pull | |
| dvc checkout data/samples.dvc | |
| git checkout v5.0 data/samples.dvc | |
| dvc pull | |
| dvc checkout data/samples.dvc | |
| - name: Test dvc checkout | |
| run: | | |
| dvc checkout data/samples.dvc | |
| - name: Test checkout | |
| run: | | |
| git checkout HEAD data/samples.dvc | |
| dvc checkout data/samples.dvc | |
| - name: Export variables | |
| run: | | |
| echo $PWD | |
| export ZENML_CONFIG_PATH=$PWD/services/zenml | |
| export PROJECTPATH=$PWD | |
| export AIRFLOW_HOME=$PWD/services/airflow | |
| export PYTHONPATH=$PWD/src | |
| - name: Run ZenML server | |
| run: zenml down && zenml up | |
| - name: Generate test artifacts | |
| run: | | |
| export ZENML_CONFIG_PATH=/home/runner/work/Sum2024MLOps/services/zenml | |
| export PROJECTPATH=/home/runner/work/Sum2024MLOps | |
| export AIRFLOW_HOME=/home/runner/work/Sum2024MLOps/services/airflow | |
| export PYTHONPATH=/home/runner/work/Sum2024MLOps | |
| python3.11 src/utils.py 1 | |
| python3.11 pipelines/data_prepare.py | |
| python3.11 src/utils.py 2 | |
| python3.11 pipelines/data_prepare.py | |
| - name: Run MLflow server | |
| run: mlflow ui & | |
| - name: Run experiments | |
| run: | | |
| mlflow run . --env-manager=local | |
| - name: Run validation for champion model | |
| run: python3.11 src/validate.py | |
| # Another job | |
| print_info: | |
| runs-on: ubuntu-latest | |
| needs: validate-model | |
| steps: | |
| - name: print my password | |
| run: | | |
| echo My password is ${{ secrets.PASSWORD }} | |
| echo My name is '${{ vars.NAME }}' | |
| - name: print message | |
| run: echo $message |