From e8a8382c6cab7f0b383ffff76c295574fe0f8529 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 23 May 2017 17:42:39 -0400 Subject: [PATCH 1/7] wip --- dbt/config.py | 57 ++++++++++++++++++++++++++++++++++++++++++++------ dbt/main.py | 24 ++++++++++++++------- dbt/project.py | 21 ++++++++++++++----- 3 files changed, 83 insertions(+), 19 deletions(-) diff --git a/dbt/config.py b/dbt/config.py index 32f769e02a9..950c4ab3933 100644 --- a/dbt/config.py +++ b/dbt/config.py @@ -3,22 +3,67 @@ import yaml.scanner import dbt.exceptions +import dbt.compat from dbt.logger import GLOBAL_LOGGER as logger +INVALID_PROFILE_MESSAGE = """ +dbt encountered an error while trying to read your profiles.yml file: + +{profiles_file} + +Error: +{error_string} + +{guess} +""" + + +def guess_yaml_error(raw_contents, mark): + line, col = getattr(mark, 'line'), getattr(mark, 'column') + if line is None or col is None: + return '' + + line = int(line) + col = int(col) + + lines = raw_contents.split('\n') + + context_up = "\n".join(lines[line-3:line]) + errant_line = lines[line] + " <---- There's yer problem" + context_down = "\n".join(lines[line+1:line+3]) + + output = [ + "-"*20, + context_up, + errant_line, + context_down, + "-"*20 + ] + + return "\n".join(output) + def read_profile(profiles_dir): - # TODO: validate profiles_dir path = os.path.join(profiles_dir, 'profiles.yml') + contents = None if os.path.isfile(path): try: with open(path, 'r') as f: - return yaml.safe_load(f) - except (yaml.scanner.ScannerError, - yaml.YAMLError) as e: - raise dbt.exceptions.ValidationException( - ' Could not read {}\n\n{}'.format(path, str(e))) + contents = f.read() + return yaml.safe_load(contents) + except (yaml.scanner.ScannerError, yaml.YAMLError) as e: + if e.problem_mark is None: + guess = '' + else: + guess = guess_yaml_error(contents, e.problem_mark) + + msg = INVALID_PROFILE_MESSAGE.format( + profiles_file=path, + error_string=dbt.compat.to_string(e), + guess=guess).strip() + raise dbt.exceptions.ValidationException(msg) return {} diff --git a/dbt/main.py b/dbt/main.py index 3238638461b..4d792c65385 100644 --- a/dbt/main.py +++ b/dbt/main.py @@ -20,7 +20,13 @@ import dbt.config as config import dbt.adapters.cache as adapter_cache import dbt.ui.printer +import dbt.compat +PROFILES_HELP_MESSAGE = """ +For more information on configuring profiles, please consult the dbt docs: + +https://dbt.readme.io/docs/configure-your-profile +""" def main(args=None): if args is None: @@ -139,17 +145,19 @@ def invoke_dbt(parsed): proj.validate() except project.DbtProjectError as e: logger.info("Encountered an error while reading the project:") - logger.info(" ERROR {}".format(str(e))) - logger.info( - "Did you set the correct --profile? Using: {}" - .format(parsed.profile)) - - logger.info("Valid profiles:") + logger.info(dbt.compat.to_string(e)) all_profiles = project.read_profiles(parsed.profiles_dir).keys() - for profile in all_profiles: - logger.info(" - {}".format(profile)) + if len(all_profiles) > 0: + logger.info("Defined profiles:") + for profile in all_profiles: + logger.info(" - {}".format(profile)) + else: + logger.info("There are no profiles defined in your " + "profiles.yml file") + + logger.info(PROFILES_HELP_MESSAGE) dbt.tracking.track_invalid_invocation( project=proj, diff --git a/dbt/project.py b/dbt/project.py index 77a81a0b16a..95abcc61389 100644 --- a/dbt/project.py +++ b/dbt/project.py @@ -2,13 +2,13 @@ import yaml import pprint import copy -import sys import hashlib import re -from voluptuous import Schema, Required, Invalid +from voluptuous import Required, Invalid import dbt.deprecations import dbt.contracts.connection +import dbt.ui.printer from dbt.logger import GLOBAL_LOGGER as logger default_project_cfg = { @@ -30,6 +30,19 @@ default_profiles_dir = os.path.join(os.path.expanduser('~'), '.dbt') +NO_SUPPLIED_PROFILE_ERROR = """\ +dbt cannot run because no profile was specified for this dbt project. +To specify a profile for this project, add a line like the this to +your dbt_project.yml file: + +profile: [profile name] + +Here, [profile name] should be replaced with a profile name +defined in your profiles.yml file. You can find profiles.yml here: + +{profiles_file}/profiles.yml +""".format(profiles_file=default_profiles_dir) + class DbtProjectError(Exception): def __init__(self, message, project): @@ -60,9 +73,7 @@ def __init__(self, cfg, profiles, profiles_dir, profile_to_load=None, self.profile_to_load = self.cfg['profile'] if self.profile_to_load is None: - raise DbtProjectError( - "No profile was supplied in the dbt_project.yml file, or the " - "command line", self) + raise DbtProjectError(NO_SUPPLIED_PROFILE_ERROR, self) if self.profile_to_load in self.profiles: self.cfg.update(self.profiles[self.profile_to_load]) From 2704e21b38ca6e798878d80d0fe022b93a9c5caa Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 23 May 2017 21:41:03 -0400 Subject: [PATCH 2/7] overhauled yaml validation error messages --- dbt/clients/yaml.py | 54 +++++++++++++++++++++++++++++++++++++++++++++ dbt/config.py | 53 ++++++-------------------------------------- dbt/parser.py | 13 ++++++++--- 3 files changed, 71 insertions(+), 49 deletions(-) create mode 100644 dbt/clients/yaml.py diff --git a/dbt/clients/yaml.py b/dbt/clients/yaml.py new file mode 100644 index 00000000000..4cc741e58a7 --- /dev/null +++ b/dbt/clients/yaml.py @@ -0,0 +1,54 @@ +import dbt.compat +import dbt.exceptions + +import yaml + + +def line_no(i, line, width=3): + line_number = dbt.compat.to_string(i).ljust(width) + return "{}| {}".format(line_number, line) + + +def prefix_with_line_numbers(line_list, starting_number): + numbers = range(starting_number, starting_number + len(line_list)) + + lines = [line_no(i, line) for (i, line) in zip(numbers, line_list)] + return "\n".join(lines) + + +def contextualized_yaml_erro(raw_contents, error): + mark = error.problem_mark + + line = mark.line + human_line = line + 1 + + line_list = raw_contents.split('\n') + + min_line = max(line - 3, 0) + max_line = line + 3 + + relevant_lines = line_list[min_line:max_line] + lines = prefix_with_line_numbers(relevant_lines, min_line + 1) + + output = [ + "Syntax error near line {}".format(human_line), + "-" * 30, + lines, + "\nRaw Error:", + "-" * 30, + dbt.compat.to_string(error) + ] + + return "\n".join(output) + + +def load_yaml_text(contents): + try: + return yaml.safe_load(contents) + except (yaml.scanner.ScannerError, yaml.YAMLError) as e: + if hasattr(e, 'problem_mark'): + error = contextualized_yaml_erro(contents, e) + else: + error = dbt.compat.to_string(e) + + raise dbt.exceptions.ValidationException(error) diff --git a/dbt/config.py b/dbt/config.py index 950c4ab3933..10af24e4b21 100644 --- a/dbt/config.py +++ b/dbt/config.py @@ -1,68 +1,29 @@ import os.path -import yaml -import yaml.scanner import dbt.exceptions -import dbt.compat +import dbt.clients.yaml +import dbt.clients.system from dbt.logger import GLOBAL_LOGGER as logger INVALID_PROFILE_MESSAGE = """ -dbt encountered an error while trying to read your profiles.yml file: +dbt encountered an error while trying to read your profiles.yml file. -{profiles_file} - -Error: {error_string} - -{guess} """ -def guess_yaml_error(raw_contents, mark): - line, col = getattr(mark, 'line'), getattr(mark, 'column') - if line is None or col is None: - return '' - - line = int(line) - col = int(col) - - lines = raw_contents.split('\n') - - context_up = "\n".join(lines[line-3:line]) - errant_line = lines[line] + " <---- There's yer problem" - context_down = "\n".join(lines[line+1:line+3]) - - output = [ - "-"*20, - context_up, - errant_line, - context_down, - "-"*20 - ] - - return "\n".join(output) - def read_profile(profiles_dir): path = os.path.join(profiles_dir, 'profiles.yml') contents = None if os.path.isfile(path): try: - with open(path, 'r') as f: - contents = f.read() - return yaml.safe_load(contents) - except (yaml.scanner.ScannerError, yaml.YAMLError) as e: - if e.problem_mark is None: - guess = '' - else: - guess = guess_yaml_error(contents, e.problem_mark) - - msg = INVALID_PROFILE_MESSAGE.format( - profiles_file=path, - error_string=dbt.compat.to_string(e), - guess=guess).strip() + contents = dbt.clients.system.load_file_contents(path, strip=False) + return dbt.clients.yaml.load_yaml_text(contents) + except dbt.exceptions.ValidationException as e: + msg = INVALID_PROFILE_MESSAGE.format(error_string=e) raise dbt.exceptions.ValidationException(msg) return {} diff --git a/dbt/parser.py b/dbt/parser.py index f1ba1c21e3b..0fa8290d051 100644 --- a/dbt/parser.py +++ b/dbt/parser.py @@ -1,6 +1,5 @@ import copy import os -import yaml import re import dbt.flags @@ -9,6 +8,7 @@ import jinja2.runtime import dbt.clients.jinja +import dbt.clients.yaml import dbt.contracts.graph.parsed import dbt.contracts.graph.unparsed @@ -426,7 +426,14 @@ def parse_schema_tests(tests, root_project, projects): to_return = {} for test in tests: - test_yml = yaml.safe_load(test.get('raw_yml')) + raw_yml = test.get('raw_yml') + test_name = "{}:{}".format(test.get('package_name'), test.get('path')) + + try: + test_yml = dbt.clients.yaml.load_yaml_text(raw_yml) + except dbt.exceptions.ValidationException as e: + test_yml = None + logger.info("Error reading {} - Skipping\n{}".format(test_name, e)) if test_yml is None: continue @@ -551,7 +558,7 @@ def load_and_parse_yml(package_name, root_project, all_projects, root_dir, for file_match in file_matches: file_contents = dbt.clients.system.load_file_contents( - file_match.get('absolute_path')) + file_match.get('absolute_path'), strip=False) parts = dbt.utils.split_path(file_match.get('relative_path', '')) name, _ = os.path.splitext(parts[-1]) From 79542a931ab9002710af3c8106372e0aee1ccbdf Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 23 May 2017 21:47:14 -0400 Subject: [PATCH 3/7] use yaml client (mostly) everywhere --- dbt/project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/project.py b/dbt/project.py index 95abcc61389..4c7ec368019 100644 --- a/dbt/project.py +++ b/dbt/project.py @@ -1,5 +1,4 @@ import os.path -import yaml import pprint import copy import hashlib @@ -9,6 +8,7 @@ import dbt.deprecations import dbt.contracts.connection import dbt.ui.printer +import dbt.clients.yaml from dbt.logger import GLOBAL_LOGGER as logger default_project_cfg = { @@ -198,7 +198,7 @@ def read_project(filename, profiles_dir=None, validate=True, project_file_contents = dbt.clients.system.load_file_contents(filename) - project_cfg = yaml.safe_load(project_file_contents) + project_cfg = dbt.clients.yaml.load_yaml_text(project_file_contents) project_cfg['project-root'] = os.path.dirname( os.path.abspath(filename)) profiles = read_profiles(profiles_dir) From 2bf7697541213d110d67c0e073418f86c7063efe Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 23 May 2017 21:59:30 -0400 Subject: [PATCH 4/7] fix imports --- dbt/clients/yaml.py | 1 + dbt/project.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/clients/yaml.py b/dbt/clients/yaml.py index 4cc741e58a7..42270feea07 100644 --- a/dbt/clients/yaml.py +++ b/dbt/clients/yaml.py @@ -2,6 +2,7 @@ import dbt.exceptions import yaml +import yaml.scanner def line_no(i, line, width=3): diff --git a/dbt/project.py b/dbt/project.py index 4c7ec368019..3963ee56dd2 100644 --- a/dbt/project.py +++ b/dbt/project.py @@ -7,7 +7,6 @@ import dbt.deprecations import dbt.contracts.connection -import dbt.ui.printer import dbt.clients.yaml from dbt.logger import GLOBAL_LOGGER as logger From 0bfec3bb314cce46dbf337b8874bcc3985a56322 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Tue, 23 May 2017 23:32:10 -0400 Subject: [PATCH 5/7] fix yaml client namespace for python2 --- dbt/clients/{yaml.py => yaml_helper.py} | 0 dbt/config.py | 4 ++-- dbt/parser.py | 4 ++-- dbt/project.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) rename dbt/clients/{yaml.py => yaml_helper.py} (100%) diff --git a/dbt/clients/yaml.py b/dbt/clients/yaml_helper.py similarity index 100% rename from dbt/clients/yaml.py rename to dbt/clients/yaml_helper.py diff --git a/dbt/config.py b/dbt/config.py index 10af24e4b21..639cb161caf 100644 --- a/dbt/config.py +++ b/dbt/config.py @@ -1,7 +1,7 @@ import os.path import dbt.exceptions -import dbt.clients.yaml +import dbt.clients.yaml_helper import dbt.clients.system from dbt.logger import GLOBAL_LOGGER as logger @@ -21,7 +21,7 @@ def read_profile(profiles_dir): if os.path.isfile(path): try: contents = dbt.clients.system.load_file_contents(path, strip=False) - return dbt.clients.yaml.load_yaml_text(contents) + return dbt.clients.yaml_helper.load_yaml_text(contents) except dbt.exceptions.ValidationException as e: msg = INVALID_PROFILE_MESSAGE.format(error_string=e) raise dbt.exceptions.ValidationException(msg) diff --git a/dbt/parser.py b/dbt/parser.py index 0fa8290d051..8451ff0c6f2 100644 --- a/dbt/parser.py +++ b/dbt/parser.py @@ -8,7 +8,7 @@ import jinja2.runtime import dbt.clients.jinja -import dbt.clients.yaml +import dbt.clients.yaml_helper import dbt.contracts.graph.parsed import dbt.contracts.graph.unparsed @@ -430,7 +430,7 @@ def parse_schema_tests(tests, root_project, projects): test_name = "{}:{}".format(test.get('package_name'), test.get('path')) try: - test_yml = dbt.clients.yaml.load_yaml_text(raw_yml) + test_yml = dbt.clients.yaml_helper.load_yaml_text(raw_yml) except dbt.exceptions.ValidationException as e: test_yml = None logger.info("Error reading {} - Skipping\n{}".format(test_name, e)) diff --git a/dbt/project.py b/dbt/project.py index 3963ee56dd2..8d66a0ee249 100644 --- a/dbt/project.py +++ b/dbt/project.py @@ -7,7 +7,7 @@ import dbt.deprecations import dbt.contracts.connection -import dbt.clients.yaml +import dbt.clients.yaml_helper from dbt.logger import GLOBAL_LOGGER as logger default_project_cfg = { @@ -197,7 +197,7 @@ def read_project(filename, profiles_dir=None, validate=True, project_file_contents = dbt.clients.system.load_file_contents(filename) - project_cfg = dbt.clients.yaml.load_yaml_text(project_file_contents) + project_cfg = dbt.clients.yaml_helper.load_yaml_text(project_file_contents) project_cfg['project-root'] = os.path.dirname( os.path.abspath(filename)) profiles = read_profiles(profiles_dir) From db32304738fa0dd09c9c84936723550bb324ecfb Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Wed, 24 May 2017 00:49:23 -0400 Subject: [PATCH 6/7] pep8 --- dbt/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt/main.py b/dbt/main.py index 4d792c65385..116e40d860d 100644 --- a/dbt/main.py +++ b/dbt/main.py @@ -28,6 +28,7 @@ https://dbt.readme.io/docs/configure-your-profile """ + def main(args=None): if args is None: args = sys.argv[1:] From fb37e44665fa967d29da4e684030668f05e9e0aa Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Wed, 24 May 2017 11:32:57 -0400 Subject: [PATCH 7/7] code cleanup + typos --- dbt/clients/yaml_helper.py | 52 ++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/dbt/clients/yaml_helper.py b/dbt/clients/yaml_helper.py index 42270feea07..3e4c2ac2bbf 100644 --- a/dbt/clients/yaml_helper.py +++ b/dbt/clients/yaml_helper.py @@ -5,42 +5,44 @@ import yaml.scanner +YAML_ERROR_MESSAGE = """ +Syntax error near line {line_number} +------------------------------ +{nice_error} + +Raw Error: +------------------------------ +{raw_error} +""".strip() + + def line_no(i, line, width=3): line_number = dbt.compat.to_string(i).ljust(width) return "{}| {}".format(line_number, line) -def prefix_with_line_numbers(line_list, starting_number): - numbers = range(starting_number, starting_number + len(line_list)) +def prefix_with_line_numbers(string, no_start, no_end): + line_list = string.split('\n') - lines = [line_no(i, line) for (i, line) in zip(numbers, line_list)] - return "\n".join(lines) + numbers = range(no_start, no_end) + relevant_lines = line_list[no_start:no_end] + return "\n".join([ + line_no(i + 1, line) for (i, line) in zip(numbers, relevant_lines) + ]) -def contextualized_yaml_erro(raw_contents, error): - mark = error.problem_mark - line = mark.line - human_line = line + 1 - - line_list = raw_contents.split('\n') - - min_line = max(line - 3, 0) - max_line = line + 3 +def contextualized_yaml_error(raw_contents, error): + mark = error.problem_mark - relevant_lines = line_list[min_line:max_line] - lines = prefix_with_line_numbers(relevant_lines, min_line + 1) + min_line = max(mark.line - 3, 0) + max_line = mark.line + 4 - output = [ - "Syntax error near line {}".format(human_line), - "-" * 30, - lines, - "\nRaw Error:", - "-" * 30, - dbt.compat.to_string(error) - ] + nice_error = prefix_with_line_numbers(raw_contents, min_line, max_line) - return "\n".join(output) + return YAML_ERROR_MESSAGE.format(line_number=mark.line + 1, + nice_error=nice_error, + raw_error=error) def load_yaml_text(contents): @@ -48,7 +50,7 @@ def load_yaml_text(contents): return yaml.safe_load(contents) except (yaml.scanner.ScannerError, yaml.YAMLError) as e: if hasattr(e, 'problem_mark'): - error = contextualized_yaml_erro(contents, e) + error = contextualized_yaml_error(contents, e) else: error = dbt.compat.to_string(e)