dbt init Interactive profile creation

core/dbt/main.py

@@ -256,7 +256,6 @@ def run_from_args(parsed):
 
     with track_run(task):
         results = task.run()
-
     return task, results
 
 
@@ -361,7 +360,7 @@ def _build_init_subparser(subparsers, base_subparser):
         '''
     )
     sub.add_argument(
-        'project_name',
+        '--project_name',
         type=str,
         help='''
         Name of the new project

core/dbt/task/init.py

@@ -1,5 +1,9 @@
+import copy
 import os
 import shutil
+import yaml
+
+import click
 
 import dbt.config
 import dbt.clients.system
@@ -10,7 +14,7 @@
 
 from dbt.include.starter_project import PACKAGE_PATH as starter_project_directory
 
-from dbt.task.base import BaseTask
+from dbt.task.base import BaseTask, move_to_nearest_project_dir
 
 DOCS_URL = 'https://docs.getdbt.com/docs/configure-your-profile'
 SLACK_URL = 'https://community.getdbt.com/'
@@ -54,19 +58,18 @@ def create_profiles_dir(self, profiles_dir):
             return True
         return False
 
-    def create_profiles_file(self, profiles_file, sample_adapter):
+    def create_sample_profiles_file(self, profiles_file, adapter):
         # Line below raises an exception if the specified adapter is not found
-        load_plugin(sample_adapter)
-        adapter_path = get_include_paths(sample_adapter)[0]
+        load_plugin(adapter)
+        adapter_path = get_include_paths(adapter)[0]
         sample_profiles_path = adapter_path / 'sample_profiles.yml'
 
         if not sample_profiles_path.exists():
-            logger.debug(f"No sample profile found for {sample_adapter}, skipping")
+            logger.debug(f"No sample profile found for {adapter}, skipping")
             return False
 
         if not os.path.exists(profiles_file):
-            msg = "With sample profiles.yml for {}"
-            logger.info(msg.format(sample_adapter))
+            logger.info(f"With sample profiles.yml for {adapter}")
             shutil.copyfile(sample_profiles_path, profiles_file)
             return True
 
@@ -83,29 +86,117 @@ def get_addendum(self, project_name, profiles_path):
             slack_url=SLACK_URL
         )
 
-    def run(self):
-        project_dir = self.args.project_name
-        sample_adapter = self.args.adapter
-        if not sample_adapter:
-            try:
-                # pick first one available, often postgres
-                sample_adapter = next(_get_adapter_plugin_names())
-            except StopIteration:
-                logger.debug("No adapters installed, skipping")
+    def generate_target_from_input(self, target_options, target={}):
+        target_options_local = copy.deepcopy(target_options)
+        # value = click.prompt('Please enter a valid integer', type=int)
+        for key, value in target_options_local.items():
+            if not key.startswith("_"):
+                if isinstance(value, str) and (value[0] + value[-1] == "[]"):
+                    hide_input = key == "password"
+                    target[key] = click.prompt(
+                        f"{key} ({value[1:-1]})", hide_input=hide_input
+                    )
+                else:
+                    target[key] = target_options_local[key]
+            if key.startswith("_choose"):
+                choice_type = key[8:]
+                option_list = list(value.keys())
+                options_msg = "\n".join([
+                    f"[{n+1}] {v}" for n, v in enumerate(option_list)
+                ])
+                click.echo(options_msg)
+                numeric_choice = click.prompt(
+                    f"desired {choice_type} option (enter a number)", type=int
+                )
+                choice = option_list[numeric_choice - 1]
+                target = self.generate_target_from_input(
+                    target_options_local[key][choice], target
+                )
+        return target
+
+    def get_profile_name_from_current_project(self):
+        with open("dbt_project.yml") as f:
+            dbt_project = yaml.load(f)
+        return dbt_project["profile"]
+
+    def write_profile(self, profiles_file, profile, profile_name=None):
+        if not profile_name:
+            profile_name = self.get_profile_name_from_current_project()
+        if os.path.exists(profiles_file):
+            with open(profiles_file, "r+") as f:
+                profiles = yaml.load(f) or {}
+                profiles[profile_name] = profile
+                f.seek(0)
+                yaml.dump(profiles, f)
+        else:
+            profiles = {profile_name: profile}
+            with open(profiles_file, "w") as f:
+                yaml.dump(profiles, f)
+
+    def configure_profile_from_scratch(self, selected_adapter):
+        # Line below raises an exception if the specified adapter is not found
+        load_plugin(selected_adapter)
+        adapter_path = get_include_paths(selected_adapter)[0]
+        target_options_path = adapter_path / 'target_options.yml'
+        profiles_file = os.path.join(dbt.config.PROFILES_DIR, 'profiles.yml')
+
+        if not target_options_path.exists():
+            logger.info(f"No options found for {selected_adapter}, using " +
+                        "sample profiles instead. Make sure to update it at" +
+                        "{profiles_file}.")
+            self.create_sample_profiles_file(profiles_file, selected_adapter)
+        else:
+            logger.info(f"Using {selected_adapter} profile options.")
+            with open(target_options_path) as f:
+                target_options = yaml.load(f)
+            target = self.generate_target_from_input(target_options)
+            profile = {
+                "outputs": {
+                    "dev": target
+                },
+                "target": "dev"
+            }
+            self.write_profile(profiles_file, profile)
+
+    def configure_profile_using_defaults(self, selected_adapter):
+        raise(NotImplementedError())
 
+    def run(self):
+        selected_adapter = self.args.adapter
         profiles_dir = dbt.config.PROFILES_DIR
-        profiles_file = os.path.join(profiles_dir, 'profiles.yml')
-
         self.create_profiles_dir(profiles_dir)
-        if sample_adapter:
-            self.create_profiles_file(profiles_file, sample_adapter)
-
-        if os.path.exists(project_dir):
-            raise RuntimeError("directory {} already exists!".format(
-                project_dir
-            ))
-
-        self.copy_starter_repo(project_dir)
 
-        addendum = self.get_addendum(project_dir, profiles_dir)
-        logger.info(addendum)
+        # Determine whether we're initializing a new project or configuring a
+        # profile for an existing one
+        if self.args.project_name:
+            project_dir = self.args.project_name
+            if os.path.exists(project_dir):
+                raise RuntimeError("directory {} already exists!".format(
+                    project_dir
+                ))
+
+            self.copy_starter_repo(project_dir)
+
+            addendum = self.get_addendum(project_dir, profiles_dir)
+            logger.info(addendum)
+            if not selected_adapter:
+                try:
+                    # pick first one available, often postgres
+                    selected_adapter = next(_get_adapter_plugin_names())
+                except StopIteration:
+                    logger.debug("No adapters installed, skipping")
+            self.configure_profile_from_scratch(
+                selected_adapter
+            )
+        else:
+            logger.info("Setting up your profile.")
+            move_to_nearest_project_dir(self.args)
+            if os.path.exists("target_defaults.yml"):
+                self.configure_profile_using_defaults()
+            else:
+                if not selected_adapter:
+                    raise RuntimeError("No adapter specified.")
+                logger.info("Configuring from scratch.")
+                self.configure_profile_from_scratch(
+                    selected_adapter
+                )

core/setup.py

@@ -52,6 +52,7 @@ def read(fname):
         'Jinja2==2.11.3',
         'PyYAML>=3.11',
         'agate>=1.6,<1.6.2',
+        'click>=8,<9',
         'colorama>=0.3.9,<0.4.5',
         'dataclasses>=0.6,<0.9;python_version<"3.7"',
         'hologram==0.0.14',

plugins/bigquery/dbt/include/bigquery/target_options.yml

@@ -0,0 +1,14 @@
+type: bigquery
+_choose_method:
+  _oauth:
+    method: oauth
+  _service_account:
+    method: service-account
+    keyfile: [/path/to/bigquery/keyfile.json]
+project: [GCP project id]
+dataset: [the name of your dbt dataset]
+threads: [1 or more]
+timeout_seconds: 300
+location: [one of US or EU]
+priority: interactive
+retries: 1

plugins/postgres/dbt/include/postgres/target_options.yml

@@ -0,0 +1,8 @@
+type: postgres
+threads: [1 or more]
+host: [host]
+port: [port]
+user: [dev_username]
+pass: [dev_password]
+dbname: [dbname]
+schema: [dev_schema]

plugins/redshift/dbt/include/redshift/target_options.yml

@@ -0,0 +1,8 @@
+type: redshift
+threads: [1 or more]
+host: [host]
+port: [port]
+user: [dev_username]
+pass: [dev_password]
+dbname: [dbname]
+schema: [dev_schema]

plugins/snowflake/dbt/adapters/snowflake/connections.py

@@ -31,9 +31,9 @@
 class SnowflakeCredentials(Credentials):
     account: str
     user: str
+    password: str
     warehouse: Optional[str] = None
     role: Optional[str] = None
-    password: Optional[str] = None
     authenticator: Optional[str] = None
     private_key_path: Optional[str] = None
     private_key_passphrase: Optional[str] = None

plugins/snowflake/dbt/include/snowflake/target_options.yml

@@ -0,0 +1,15 @@
+type: snowflake
+account: '[account id + region (if applicable)]'
+user: '[username]'
+_choose_authentication_type:
+  password:
+    password: '[password]'
+  keypair:
+    private_key_path: '[path/to/private.key]'
+    private_key_passphrase: '[passphrase for the private key, if key is encrypted]'
+role: '[user role]'
+database: '[database name]'
+warehouse: '[warehouse name]'
+schema: '[dbt schema]'
+threads: '[1 or more]'
+client_session_keep_alive: False