diff --git a/.github/workflows/test-code.yaml b/.github/workflows/test-code.yaml index f4d0c32..e9d2c0a 100644 --- a/.github/workflows/test-code.yaml +++ b/.github/workflows/test-code.yaml @@ -54,6 +54,23 @@ jobs: run: | python3.11 -m pip install --upgrade pip pip3.11 install -r requirements.txt + + - name: create-json + id: create-json + uses: jsdaniell/create-json@1.1.2 + with: + name: "gdrive-credentials.json" + json: ${{ secrets.GDRIVE_CREDENTIALS_DATA }} + + - name: Configure DVC remote with service account + run: | + dvc remote modify storage gdrive_use_service_account true + dvc remote modify storage --local gdrive_service_account_json_file_path gdrive-credentials.json + + - name: Pull DVC data + run: | + dvc fetch + dvc pull - name: Run your app run: python3.11 src/app.py & diff --git a/.github/workflows/validate-model.yaml b/.github/workflows/validate-model.yaml new file mode 100644 index 0000000..e69de29 diff --git a/.gitignore b/.gitignore index 839b567..c3ccd44 100644 Binary files a/.gitignore and b/.gitignore differ diff --git a/src/app.py b/src/app.py index feb7ea5..b15430b 100644 --- a/src/app.py +++ b/src/app.py @@ -116,10 +116,10 @@ def predict( gr.Text(label="province_name"), gr.Number(label="latitude"), gr.Number(label="longitude"), - gr.Number(label="baths"), + gr.Number(label="baths"), # slider gr.Text(label="area"), # Marla or Kanal + size gr.Dropdown(label="purpose", choices=["For Sale", "For Rent"]), - gr.Number(label="bedrooms"), + gr.Number(label="bedrooms"), #slider gr.Textbox(label="date_added"), # TODO: How to add datetime? gr.Text(label="agency"), gr.Text(label="agent"), diff --git a/src/data.py b/src/data.py index c1d4e63..8d33da0 100644 --- a/src/data.py +++ b/src/data.py @@ -91,16 +91,13 @@ def format_area(value): print(f"Refactored data saved to {data_path}") -def read_datastore(): - with initialize(config_path="../configs", version_base=None): - cfg = compose(config_name="main") - +@hydra.main(config_path="../configs", config_name="main", version_base=None) +def read_datastore(cfg: DictConfig): version = cfg.test_data_version if cfg.test else cfg.index try: - subprocess.run(["dvc", "fetch"]) + subprocess.run(["dvc", "fetch", f"{cfg.dvc_file_path}"]) subprocess.run(["dvc", "pull"], check=True) - subprocess.run( - ["git", "checkout", f"v{version}.0", f"{cfg.dvc_file_path}"], check=True) + subprocess.run(["git", "checkout", f"v{version}.0", f"{cfg.dvc_file_path}"], check=True) subprocess.run(["dvc", "checkout", f"{cfg.dvc_file_path}"], check=True) sample_path = cfg.sample_path @@ -110,8 +107,7 @@ def read_datastore(): return sample finally: # Return to the HEAD state - subprocess.run(["git", "checkout", "HEAD", - f"{cfg.dvc_file_path}"], check=True) + subprocess.run(["git", "checkout", "HEAD", f"{cfg.dvc_file_path}"], check=True) subprocess.run(["dvc", "checkout", f"{cfg.dvc_file_path}"], check=True) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..e328ee0 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,33 @@ +from typing import Tuple + +import pandas as pd +import pytest +from hydra import initialize, compose +from omegaconf import DictConfig + +from src.data import ( + read_datastore, + preprocess_data, +) + + +@pytest.fixture +def cfg() -> DictConfig: + """ + Load the test_config.yaml configuration file + """ + with initialize(config_path="../configs", version_base=None): + cfg = compose(config_name="test_config") + return cfg + + +@pytest.fixture +def raw_sample(cfg) -> pd.DataFrame: + df = read_datastore(cfg) + return df + + +@pytest.fixture +def preprocessed_sample(raw_sample) -> Tuple[pd.DataFrame, pd.Series]: + X, y = preprocess_data(raw_sample) + return X, y diff --git a/tests/test_data.py b/tests/test_data.py index fc2e69f..9a204f0 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -12,28 +12,6 @@ read_datastore, preprocess_data, load_features -@pytest.fixture -def cfg() -> DictConfig: - """ - Load the test_config.yaml configuration file - """ - with initialize(config_path="../configs", version_base=None): - cfg = compose(config_name="test_config") - return cfg - - -@pytest.fixture -def raw_sample(cfg) -> pd.DataFrame: - df = read_datastore(cfg) - return df - - -@pytest.fixture -def preprocessed_sample(raw_sample) -> Tuple[pd.DataFrame, pd.Series]: - X, y = preprocess_data(raw_sample) - return X, y - - def sample_data_stage(cfg: DictConfig, index: int, sample_file: str): """ Helper function to sample data for a specific project stage diff --git a/tests/test_data_expectations.py b/tests/test_data_expectations.py index 436e5f1..08ecee0 100644 --- a/tests/test_data_expectations.py +++ b/tests/test_data_expectations.py @@ -1,35 +1,6 @@ -import pandas as pd -import pytest -from hydra import compose, initialize -from omegaconf import DictConfig -from typing import Tuple - - -from src.data import read_datastore, preprocess_data from src.data_expectations import validate_features, validate_initial_data -@pytest.fixture -def cfg() -> DictConfig: - """ - Load the test_config.yaml configuration file - """ - with initialize(config_path="../configs", version_base=None): - cfg = compose(config_name="test_config") - return cfg - - -@pytest.fixture -def raw_sample(cfg) -> pd.DataFrame: - df = read_datastore(cfg) - return df - -@pytest.fixture -def preprocessed_sample(raw_sample) -> Tuple[pd.DataFrame, pd.Series]: - X, y = preprocess_data(raw_sample) - return X, y - - def test_validate_initial_data(): try: validate_initial_data()