Merge pull request #71 from databrickslabs/address-dbx-changes

prepare new version

CHANGELOG.md

@@ -43,4 +43,8 @@ Fixed multiple issues with run status checks:
 - Add installation of `dbx` to the `unit-requirements.txt` file
 - Switch generated top-level folder name to project_name variable, not project_slug.
 - Add `.coveragerc` to the generated project
-- Explicitly include support for `dbutils` in the `common.py` file
+- Explicitly include support for `dbutils` in the `common.py` file
+
+## Release 1.0.10
+
+- Remove `init_adapter` logic, now all parameters are directly passed from the `deployment.json` file.

Makefile

@@ -1,4 +1,4 @@
-VERSION = 1.0.9
+VERSION = 1.0.10
 
 release:
 	git add .

{{cookiecutter.project_name}}/{{cookiecutter.project_slug}}/common.py

@@ -5,18 +5,11 @@
 from typing import Dict, Any
 
 from pyspark.sql import SparkSession
+import sys
 
 
 # abstract class for jobs
 class Job(ABC):
-    @abstractmethod
-    def init_adapter(self):
-        """
-        Init adapter is an abstract method to perform some particular settings in the Job subclass.
-        Method is called after creation of the SparkSession.
-        :return:
-        """
-        pass
 
     def __init__(self, spark=None, init_conf=None):
         self.spark = self._prepare_spark(spark)
@@ -26,7 +19,6 @@ def __init__(self, spark=None, init_conf=None):
             self.conf = init_conf
         else:
             self.conf = self._provide_config()
-        self.init_adapter()
         self._log_conf()
 
     @staticmethod
@@ -77,7 +69,7 @@ def _provide_config(self):
     def _get_conf_file():
         p = ArgumentParser()
         p.add_argument("--conf-file", required=False, type=str)
-        namespace = p.parse_known_args()[0]
+        namespace = p.parse_known_args(sys.argv[1:])[0]
         return namespace.conf_file
 
     def _read_config(self, conf_file) -> Dict[str, Any]:

{{cookiecutter.project_name}}/{{cookiecutter.project_slug}}/jobs/sample/entrypoint.py

@@ -2,17 +2,6 @@
 
 
 class SampleJob(Job):
-    def init_adapter(self):
-        if not self.conf:
-            self.logger.info(
-                "Init configuration was not provided, using configuration from default_init method"
-            )
-            self.conf = {
-                "output_format": "delta",
-                "output_path": "dbfs:/dbx/tmp/test/interactive/{{cookiecutter.project_slug}}",
-            }
-        else:
-            self.logger.info("Init configuration is already provided")
 
     def launch(self):
         self.logger.info("Launching sample job")