From df788bbd67744986a2bb9d8ff269a599e7aed2fb Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Sun, 11 Feb 2024 16:57:51 +0200 Subject: [PATCH 1/6] use simple file load for template CLI command and add a load config template utility --- .../anyscale.yaml | 0 .../{config => config_templates}/azure.yaml | 0 .../{config => config_templates}/cohere.yaml | 0 .../{config => config_templates}/default.yaml | 0 src/canopy/utils/config.py | 12 ++++++ src/canopy_cli/cli.py | 42 +++++++------------ tests/system/utils/test_config.py | 6 +-- 7 files changed, 30 insertions(+), 30 deletions(-) rename src/canopy/{config => config_templates}/anyscale.yaml (100%) rename src/canopy/{config => config_templates}/azure.yaml (100%) rename src/canopy/{config => config_templates}/cohere.yaml (100%) rename src/canopy/{config => config_templates}/default.yaml (100%) diff --git a/src/canopy/config/anyscale.yaml b/src/canopy/config_templates/anyscale.yaml similarity index 100% rename from src/canopy/config/anyscale.yaml rename to src/canopy/config_templates/anyscale.yaml diff --git a/src/canopy/config/azure.yaml b/src/canopy/config_templates/azure.yaml similarity index 100% rename from src/canopy/config/azure.yaml rename to src/canopy/config_templates/azure.yaml diff --git a/src/canopy/config/cohere.yaml b/src/canopy/config_templates/cohere.yaml similarity index 100% rename from src/canopy/config/cohere.yaml rename to src/canopy/config_templates/cohere.yaml diff --git a/src/canopy/config/default.yaml b/src/canopy/config_templates/default.yaml similarity index 100% rename from src/canopy/config/default.yaml rename to src/canopy/config_templates/default.yaml diff --git a/src/canopy/utils/config.py b/src/canopy/utils/config.py index 1528771b..0601e953 100644 --- a/src/canopy/utils/config.py +++ b/src/canopy/utils/config.py @@ -1,6 +1,18 @@ +import os +import yaml from typing import Dict, Any +def load_config_template(name: str) -> Dict[str, Any]: + if not name.endswith(".yaml"): + name += ".yaml" + + config_template_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), + "config_templates", name) + with open(config_template_path, "r") as f: + return yaml.safe_load(f) + + class ConfigurableMixin: _DEFAULT_COMPONENTS: Dict[str, type] = {} diff --git a/src/canopy_cli/cli.py b/src/canopy_cli/cli.py index 5e1bbd6b..fbdecc27 100644 --- a/src/canopy_cli/cli.py +++ b/src/canopy_cli/cli.py @@ -1,8 +1,8 @@ import os +import shutil from typing import Dict, Any, Optional, List, Iterable import click -import importlib.resources as pkg_resources from prompt_toolkit import prompt import time @@ -211,38 +211,28 @@ def health(url): return -@cli.command(help="Writes a config template YAML file to the specified path.") -@click.argument("path", type=click.Path(), required=True) -@click.option("--template", default="default", help="The name of the template to use.") -def create_config(path, template): +@cli.command(help="Writes the config templates to a directory.") +@click.argument("out_path", type=click.Path(), required=True) +def create_config(out_path): - if not template.endswith('.yaml'): - template += '.yaml' + if os.path.isfile(out_path): + raise CLIError(f"Path expected to be a directory," + f"but found a file at {out_path}") - if os.path.exists(path): - click.confirm(click.style(f"File {path} already exists. Overwrite?", fg="red"), + if os.path.exists(out_path) and os.path.isdir(out_path) and os.listdir(out_path): + click.confirm(click.style(f"Path {out_path} is not empty. Overwrite?", + fg="red"), abort=True) + project_root = os.path.dirname(os.path.dirname(__file__)) + config_templates_path = os.path.join(project_root, "canopy", "config_templates") + try: - with pkg_resources.open_text('canopy.config', template) as f: - config = yaml.safe_load(f) - except FileNotFoundError: - files = pkg_resources.files('canopy.config') - template_names = [f.name for f in files.iterdir()] - click.echo(f"Template '{template}' not found." - f"Available templates: {template_names}", - err=True) - return + shutil.copytree(config_templates_path, out_path, dirs_exist_ok=True) except Exception as e: - click.echo(f"Failed to load template '{template}'. Reason: {e}" - f"Make sure you pip installed `canopy-sdk`.", - err=True) - return - - with open(path, 'w') as dst_file: - yaml.dump(config, dst_file) + raise CLIError(f"Failed to write config template to {out_path}. Reason:\n{e}") - click.echo(f"Config template '{template}' written to {path}") + click.echo(click.style(f"Config templates written to {out_path}", fg="green")) @cli.command( diff --git a/tests/system/utils/test_config.py b/tests/system/utils/test_config.py index 942b9a2f..b1fce57f 100644 --- a/tests/system/utils/test_config.py +++ b/tests/system/utils/test_config.py @@ -1,13 +1,12 @@ import os import pytest -import yaml from canopy.chat_engine import ChatEngine from canopy.context_engine import ContextEngine from canopy.knowledge_base import KnowledgeBase +from canopy.utils.config import load_config_template -DEFAULT_CONFIG_PATH = 'src/canopy/config/default.yaml' @pytest.fixture(scope='module') @@ -24,8 +23,7 @@ def temp_index_name(): def test_default_config_matches_code_defaults(temp_index_name): - with open(DEFAULT_CONFIG_PATH) as f: - default_config = yaml.safe_load(f) + default_config = load_config_template("default.yaml") chat_engine_config = default_config['chat_engine'] loaded_chat_engine = ChatEngine.from_config(chat_engine_config) From fe31f598475ffa0497ab355751db965f63fb3ce2 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Sun, 11 Feb 2024 17:19:18 +0200 Subject: [PATCH 2/6] lint --- tests/system/utils/test_config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/system/utils/test_config.py b/tests/system/utils/test_config.py index b1fce57f..03a21ee9 100644 --- a/tests/system/utils/test_config.py +++ b/tests/system/utils/test_config.py @@ -8,7 +8,6 @@ from canopy.utils.config import load_config_template - @pytest.fixture(scope='module') def temp_index_name(): index_name_before = os.getenv("INDEX_NAME", None) From 147be02e6e3b184eb602d03b06aa2d412f6d8e01 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Mon, 12 Feb 2024 14:39:53 +0200 Subject: [PATCH 3/6] Create a Directory class to hold the lib paths --- src/canopy/utils/config.py | 12 ------------ src/canopy/utils/directory.py | 8 ++++++++ src/canopy_cli/cli.py | 14 +++++++------- 3 files changed, 15 insertions(+), 19 deletions(-) create mode 100644 src/canopy/utils/directory.py diff --git a/src/canopy/utils/config.py b/src/canopy/utils/config.py index 0601e953..1528771b 100644 --- a/src/canopy/utils/config.py +++ b/src/canopy/utils/config.py @@ -1,18 +1,6 @@ -import os -import yaml from typing import Dict, Any -def load_config_template(name: str) -> Dict[str, Any]: - if not name.endswith(".yaml"): - name += ".yaml" - - config_template_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), - "config_templates", name) - with open(config_template_path, "r") as f: - return yaml.safe_load(f) - - class ConfigurableMixin: _DEFAULT_COMPONENTS: Dict[str, type] = {} diff --git a/src/canopy/utils/directory.py b/src/canopy/utils/directory.py new file mode 100644 index 00000000..64a9a6ea --- /dev/null +++ b/src/canopy/utils/directory.py @@ -0,0 +1,8 @@ +from pathlib import Path + + +class Directory: + """Stores the directory paths for Canopy library""" + + ROOT = Path(__file__).parent.parent + CONFIG_TEMPLATES = ROOT.joinpath("config_templates") diff --git a/src/canopy_cli/cli.py b/src/canopy_cli/cli.py index fbdecc27..458a8ec2 100644 --- a/src/canopy_cli/cli.py +++ b/src/canopy_cli/cli.py @@ -5,7 +5,7 @@ import click from prompt_toolkit import prompt import time - +from pathlib import Path import requests import yaml from dotenv import load_dotenv @@ -22,6 +22,7 @@ from canopy.chat_engine import ChatEngine from canopy.models.data_models import Document, UserMessage from canopy.tokenizer import Tokenizer +from canopy.utils.directory import Directory from canopy_cli.data_loader import ( load_from_path, IDsNotUniqueError, @@ -215,20 +216,19 @@ def health(url): @click.argument("out_path", type=click.Path(), required=True) def create_config(out_path): - if os.path.isfile(out_path): + out_path = Path(out_path) + + if out_path.is_file(): raise CLIError(f"Path expected to be a directory," f"but found a file at {out_path}") - if os.path.exists(out_path) and os.path.isdir(out_path) and os.listdir(out_path): + if out_path.exists() and any(out_path.iterdir()): click.confirm(click.style(f"Path {out_path} is not empty. Overwrite?", fg="red"), abort=True) - project_root = os.path.dirname(os.path.dirname(__file__)) - config_templates_path = os.path.join(project_root, "canopy", "config_templates") - try: - shutil.copytree(config_templates_path, out_path, dirs_exist_ok=True) + shutil.copytree(Directory.CONFIG_TEMPLATES, out_path, dirs_exist_ok=True) except Exception as e: raise CLIError(f"Failed to write config template to {out_path}. Reason:\n{e}") From 29c1d077fa1f6ef7d7e4d9dd9b5a3a67f946b737 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Mon, 12 Feb 2024 14:42:21 +0200 Subject: [PATCH 4/6] test fix --- tests/system/utils/test_config.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/system/utils/test_config.py b/tests/system/utils/test_config.py index 03a21ee9..0c758963 100644 --- a/tests/system/utils/test_config.py +++ b/tests/system/utils/test_config.py @@ -6,6 +6,7 @@ from canopy.context_engine import ContextEngine from canopy.knowledge_base import KnowledgeBase from canopy.utils.config import load_config_template +from canopy.utils.directory import Directory @pytest.fixture(scope='module') @@ -22,7 +23,10 @@ def temp_index_name(): def test_default_config_matches_code_defaults(temp_index_name): - default_config = load_config_template("default.yaml") + + with open(Directory.CONFIG_TEMPLATES.joinpath("default.yaml")) as file: + default_config = load_config_template(file) + chat_engine_config = default_config['chat_engine'] loaded_chat_engine = ChatEngine.from_config(chat_engine_config) From 4c740076770a9074b36d2149c8c5d9b9f36a2aa8 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Tue, 13 Feb 2024 14:00:32 +0200 Subject: [PATCH 5/6] fix config test --- tests/system/utils/test_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system/utils/test_config.py b/tests/system/utils/test_config.py index 0c758963..d9ee8789 100644 --- a/tests/system/utils/test_config.py +++ b/tests/system/utils/test_config.py @@ -1,11 +1,11 @@ import os import pytest +import yaml from canopy.chat_engine import ChatEngine from canopy.context_engine import ContextEngine from canopy.knowledge_base import KnowledgeBase -from canopy.utils.config import load_config_template from canopy.utils.directory import Directory @@ -25,7 +25,7 @@ def temp_index_name(): def test_default_config_matches_code_defaults(temp_index_name): with open(Directory.CONFIG_TEMPLATES.joinpath("default.yaml")) as file: - default_config = load_config_template(file) + default_config = yaml.safe_load(file) chat_engine_config = default_config['chat_engine'] From 4fc3004711be59f8a3825122e57a7647249f9c05 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Tue, 13 Feb 2024 14:43:55 +0200 Subject: [PATCH 6/6] dummy change to toml --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5636c238..46e0ff4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,8 +47,8 @@ grpc = ["pinecone-client"] [tool.poetry.group.dev.dependencies] -jupyter = "^1.0.0" pytest = "^7.3.2" +jupyter = "^1.0.0" mypy = "^1.4.1" flake8 = "^6.1.0" pytest-html = "^4.1.0" @@ -104,4 +104,4 @@ skip-checking-raises = true canopy = "canopy_cli.cli:cli" [tool.pytest.ini_options] -log_cli = true \ No newline at end of file +log_cli = true