⬆️ update security-related dependencies and apply isort (#798)

* apply isort and fix deps

* update requests dep

* apply formatting

* fix pydantic version

* fix versions

* move to 11.3 LTS in the deployment file

* add changelog

CHANGELOG.md

@@ -16,6 +16,16 @@ This may lead to instability when using dbx API methods directly.
 
 ### Fixed
 
+- Several dependency issues related to both security and library versions
+
+### Changed
+
+- Switched to 11.3 LTS in the main template
+
+## [0.8.15] - 2023-06-02
+
+### Fixed
+
 - Allow provisioning of `source` property for `spark_python_task`
 
 ## [0.8.14] - 2023-05-11

dbx/__init__.py

@@ -1 +1 @@
-__version__ = "0.8.15"
+__version__ = "0.8.16"

dbx/api/adjuster/adjuster.py

@@ -1,4 +1,4 @@
-from typing import Any, Optional, Union, List
+from typing import Any, List, Optional, Union
 
 from databricks_cli.sdk import ApiClient
 from pydantic import BaseModel

dbx/api/adjuster/mixins/existing_cluster.py

@@ -1,5 +1,5 @@
 import functools
-from typing import Optional, List, Any
+from typing import Any, List, Optional
 
 from databricks_cli.sdk import ClusterService
 

dbx/api/adjuster/mixins/instance_profile.py

@@ -1,5 +1,5 @@
 import functools
-from typing import Optional, Any, List
+from typing import Any, List, Optional
 
 from dbx.api.adjuster.mixins.base import ApiClientMixin, ElementSetterMixin
 from dbx.models.workflow.common.flexible import FlexibleModel

dbx/api/auth.py

@@ -3,23 +3,22 @@
 from typing import List, Optional, Protocol, Union
 
 from databricks_cli.configure.provider import (
+    DEFAULT_SECTION,
+    HOST,
+    INSECURE,
+    JOBS_API_VERSION,
+    PASSWORD,
+    REFRESH_TOKEN,
+    TOKEN,
+    USERNAME,
     DatabricksConfig,
     DatabricksConfigProvider,
     _fetch_from_fs,
     _get_option_if_exists,
-    HOST,
-    USERNAME,
-    PASSWORD,
-    TOKEN,
-    REFRESH_TOKEN,
-    INSECURE,
-    JOBS_API_VERSION,
-    DEFAULT_SECTION,
 )
 
 from dbx.utils import dbx_echo
 
-
 AZURE_SERVICE_PRINCIPAL_TOKEN = "azure_service_principal_token"
 WORKSPACE_ID = "workspace_id"
 ORG_ID = "org_id"

dbx/api/build.py

@@ -2,7 +2,7 @@
 import subprocess
 import sys
 from pathlib import Path
-from typing import Union, List, Optional
+from typing import List, Optional, Union
 
 from dbx.utils import dbx_echo
 

dbx/api/client_provider.py

@@ -1,9 +1,9 @@
 import copy
-from typing import Dict, Any, Optional
+from typing import Any, Dict, Optional
 
 import requests
 from databricks_cli.sdk import ApiClient
-from tenacity import retry, wait_exponential, stop_after_attempt
+from tenacity import retry, stop_after_attempt, wait_exponential
 
 from dbx.api.auth import AuthConfigProvider
 

dbx/api/config_reader.py

@@ -2,7 +2,7 @@
 import os
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import List, Optional, Dict, Any
+from typing import Any, Dict, List, Optional
 
 import jinja2
 import yaml

dbx/api/context.py

@@ -1,7 +1,7 @@
 import time
 from base64 import b64encode
 from pathlib import Path
-from typing import Optional, List, Any
+from typing import Any, List, Optional
 
 import typer
 from databricks_cli.sdk import ApiClient

dbx/api/deployment.py

@@ -4,7 +4,7 @@
 from dbx.api.services.jobs import NamedJobsService
 from dbx.api.services.permissions import PermissionsService
 from dbx.api.services.pipelines import NamedPipelinesService
-from dbx.models.deployment import WorkflowList, AnyWorkflow
+from dbx.models.deployment import AnyWorkflow, WorkflowList
 from dbx.models.workflow.common.workflow_types import WorkflowType
 from dbx.utils import dbx_echo
 

dbx/api/destroyer.py

@@ -14,7 +14,7 @@
 
 from dbx.api.services.jobs import NamedJobsService
 from dbx.api.services.pipelines import NamedPipelinesService
-from dbx.models.cli.destroyer import DestroyerConfig, DeletionMode
+from dbx.models.cli.destroyer import DeletionMode, DestroyerConfig
 from dbx.models.deployment import AnyWorkflow
 from dbx.models.files.project import EnvironmentInfo
 from dbx.models.workflow.common.workflow_types import WorkflowType

dbx/api/execute.py

@@ -1,17 +1,17 @@
 from pathlib import Path
-from typing import Optional, List, Union, Dict
+from typing import Dict, List, Optional, Union
 
 import mlflow
 from rich.console import Console
 
-from dbx.api.adjuster.adjuster import Adjuster, AdditionalLibrariesProvider
+from dbx.api.adjuster.adjuster import AdditionalLibrariesProvider, Adjuster
 from dbx.api.context import RichExecutionContextClient
 from dbx.models.workflow.common.libraries import Library
 from dbx.models.workflow.common.task_type import TaskType
 from dbx.models.workflow.v2dot1.task import PythonWheelTask
 from dbx.types import ExecuteTask
 from dbx.utils import dbx_echo
-from dbx.utils.file_uploader import MlflowFileUploader, ContextBasedUploader
+from dbx.utils.file_uploader import ContextBasedUploader, MlflowFileUploader
 
 
 class ExecutionController:

dbx/api/launch/functions.py

@@ -1,5 +1,5 @@
 import time
-from typing import Dict, Any, List
+from typing import Any, Dict, List
 
 import mlflow
 from databricks_cli.sdk import ApiClient, JobsService

dbx/api/launch/pipeline_models.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import Optional, List
+from typing import List, Optional
 
 from pydantic import BaseModel
 

dbx/api/launch/runners/asset_based.py

@@ -1,6 +1,6 @@
 import inspect
 import json
-from typing import Optional, Union, Tuple, Dict, Any
+from typing import Any, Dict, Optional, Tuple, Union
 
 from databricks_cli.sdk import ApiClient, JobsService
 

dbx/api/launch/runners/pipeline.py

@@ -1,7 +1,7 @@
 import json
 import time
 from functools import partial
-from typing import Optional, List, Tuple
+from typing import List, Optional, Tuple
 
 from databricks_cli.sdk import ApiClient
 from pydantic import BaseModel

dbx/api/launch/runners/standard.py

@@ -1,9 +1,9 @@
 import json
-from typing import Optional, Union, Tuple
+from typing import Optional, Tuple, Union
 
 from databricks_cli.sdk import ApiClient, JobsService
 
-from dbx.api.launch.functions import wait_run, cancel_run
+from dbx.api.launch.functions import cancel_run, wait_run
 from dbx.api.launch.runners.base import RunData
 from dbx.api.services.jobs import NamedJobsService
 from dbx.models.cli.options import ExistingRunsOption

dbx/api/launch/tracer.py

@@ -3,9 +3,9 @@
 from databricks_cli.sdk import ApiClient
 from rich.console import Console
 
-from dbx.api.launch.functions import trace_run, cancel_run
+from dbx.api.launch.functions import cancel_run, trace_run
 from dbx.api.launch.pipeline_models import PipelineUpdateState, PipelineUpdateStatus
-from dbx.api.launch.runners.base import RunData, PipelineUpdateResponse
+from dbx.api.launch.runners.base import PipelineUpdateResponse, RunData
 from dbx.utils import dbx_echo
 
 

dbx/api/output_provider.py

@@ -1,4 +1,4 @@
-from typing import Dict, Any, List
+from typing import Any, Dict, List
 
 from databricks_cli.sdk import JobsService
 

dbx/api/services/_base.py

@@ -1,5 +1,5 @@
 from abc import abstractmethod
-from typing import Optional, Any
+from typing import Any, Optional
 
 from dbx.api.adjuster.mixins.base import ApiClientMixin
 from dbx.models.deployment import AnyWorkflow

dbx/api/services/pipelines.py

@@ -1,4 +1,4 @@
-from typing import Optional, List
+from typing import List, Optional
 
 from requests import HTTPError
 from rich.markup import escape

dbx/api/storage/io.py

@@ -2,7 +2,7 @@
 import shutil
 import tempfile
 from pathlib import Path
-from typing import Dict, Any
+from typing import Any, Dict
 
 import mlflow
 from mlflow.tracking import MlflowClient

dbx/commands/configure.py

@@ -1,9 +1,9 @@
 import typer
 
-from dbx.api.configure import ProjectConfigurationManager, EnvironmentInfo
+from dbx.api.configure import EnvironmentInfo, ProjectConfigurationManager
 from dbx.models.files.project import MlflowStorageProperties, StorageType
 from dbx.options import ENVIRONMENT_OPTION, PROFILE_OPTION
-from dbx.utils import dbx_echo, current_folder_name
+from dbx.utils import current_folder_name, dbx_echo
 
 
 def configure(

dbx/commands/deploy.py

@@ -26,11 +26,7 @@
     WORKFLOW_ARGUMENT,
 )
 from dbx.utils import dbx_echo
-from dbx.utils.common import (
-    get_current_branch_name,
-    parse_multiple,
-    prepare_environment,
-)
+from dbx.utils.common import get_current_branch_name, parse_multiple, prepare_environment
 from dbx.utils.file_uploader import MlflowFileUploader
 
 

dbx/commands/destroy.py

@@ -9,13 +9,13 @@
 
 from dbx.api.config_reader import ConfigReader
 from dbx.api.destroyer import Destroyer
-from dbx.models.cli.destroyer import DestroyerConfig, DeletionMode
+from dbx.models.cli.destroyer import DeletionMode, DestroyerConfig
 from dbx.options import (
-    HEADERS_OPTION,
-    WORKFLOW_ARGUMENT,
     DEPLOYMENT_FILE_OPTION,
     ENVIRONMENT_OPTION,
+    HEADERS_OPTION,
     JINJA_VARIABLES_FILE_OPTION,
+    WORKFLOW_ARGUMENT,
 )
 from dbx.utils import dbx_echo
 from dbx.utils.common import parse_multiple, prepare_environment

dbx/commands/execute.py

@@ -4,24 +4,24 @@
 import typer
 
 from dbx.api.cluster import ClusterController
-from dbx.api.config_reader import ConfigReader, BuildProperties
+from dbx.api.config_reader import BuildProperties, ConfigReader
 from dbx.api.configure import ProjectConfigurationManager
 from dbx.api.context import RichExecutionContextClient
 from dbx.api.dependency.core_package import CorePackageManager
 from dbx.api.execute import ExecutionController
 from dbx.models.cli.execute import ExecuteParametersPayload
 from dbx.models.workflow.common.workflow_types import WorkflowType
 from dbx.options import (
+    DEBUG_OPTION,
     DEPLOYMENT_FILE_OPTION,
     ENVIRONMENT_OPTION,
+    EXECUTE_PARAMETERS_OPTION,
     HEADERS_OPTION,
-    REQUIREMENTS_FILE_OPTION,
-    NO_REBUILD_OPTION,
-    NO_PACKAGE_OPTION,
     JINJA_VARIABLES_FILE_OPTION,
-    DEBUG_OPTION,
+    NO_PACKAGE_OPTION,
+    NO_REBUILD_OPTION,
+    REQUIREMENTS_FILE_OPTION,
     WORKFLOW_ARGUMENT,
-    EXECUTE_PARAMETERS_OPTION,
 )
 from dbx.types import ExecuteTask
 from dbx.utils import dbx_echo

dbx/commands/launch.py

@@ -1,37 +1,31 @@
-from typing import List, Dict, Any
-from typing import Optional
+from typing import Any, Dict, List, Optional
 
 import mlflow
 import typer
 from databricks_cli.jobs.api import JobsService
 from rich.markup import escape
 
-from dbx.api.launch.contexts import EmptyContext, AssetBasedLaunchContext
+from dbx.api.launch.contexts import AssetBasedLaunchContext, EmptyContext
 from dbx.api.launch.functions import find_deployment_run
 from dbx.api.launch.pipeline_models import PipelineUpdateState
 from dbx.api.launch.runners.asset_based import AssetBasedLauncher
 from dbx.api.launch.runners.base import RunData
 from dbx.api.launch.runners.pipeline import PipelineLauncher
 from dbx.api.launch.runners.standard import StandardLauncher
-from dbx.api.launch.tracer import RunTracer, PipelineTracer
+from dbx.api.launch.tracer import PipelineTracer, RunTracer
 from dbx.api.output_provider import OutputProvider
 from dbx.models.cli.options import ExistingRunsOption, IncludeOutputOption
 from dbx.options import (
+    BRANCH_NAME_OPTION,
+    DEBUG_OPTION,
     ENVIRONMENT_OPTION,
     HEADERS_OPTION,
+    LAUNCH_PARAMETERS_OPTION,
     TAGS_OPTION,
-    BRANCH_NAME_OPTION,
-    DEBUG_OPTION,
     WORKFLOW_ARGUMENT,
-    LAUNCH_PARAMETERS_OPTION,
 )
 from dbx.utils import dbx_echo
-from dbx.utils.common import (
-    generate_filter_string,
-    prepare_environment,
-    parse_multiple,
-    get_current_branch_name,
-)
+from dbx.utils.common import generate_filter_string, get_current_branch_name, parse_multiple, prepare_environment
 
 
 def launch(

dbx/commands/sync/functions.py

@@ -7,7 +7,7 @@
 from databricks_cli.configure.provider import DatabricksConfig
 
 from dbx.constants import DBX_SYNC_DEFAULT_IGNORES
-from dbx.sync import DeleteUnmatchedOption, PathMatcher, BaseClient, RemoteSyncer
+from dbx.sync import BaseClient, DeleteUnmatchedOption, PathMatcher, RemoteSyncer
 from dbx.sync.clients import get_user
 from dbx.sync.event_handler import file_watcher
 from dbx.utils import dbx_echo

dbx/commands/sync/sync.py

@@ -10,26 +10,26 @@
 from dbx.api.configure import ProjectConfigurationManager
 from dbx.commands.sync.functions import (
     create_path_matcher,
-    main_loop,
-    handle_source,
-    get_user_name,
     get_source_base_name,
+    get_user_name,
+    handle_source,
+    main_loop,
 )
 from dbx.commands.sync.options import (
-    SOURCE_OPTION,
-    FULL_SYNC_OPTION,
     DRY_RUN_OPTION,
-    INCLUDE_DIRS_OPTION,
-    FORCE_INCLUDE_DIRS_OPTION,
     EXCLUDE_DIRS_OPTION,
-    WATCH_OPTION,
-    POLLING_INTERVAL_OPTION,
-    INCLUDE_PATTERNS_OPTION,
-    FORCE_INCLUDE_PATTERNS_OPTION,
     EXCLUDE_PATTERNS_OPTION,
-    USE_GITIGNORE_OPTION,
-    UNMATCHED_BEHAVIOUR_OPTION,
+    FORCE_INCLUDE_DIRS_OPTION,
+    FORCE_INCLUDE_PATTERNS_OPTION,
+    FULL_SYNC_OPTION,
+    INCLUDE_DIRS_OPTION,
+    INCLUDE_PATTERNS_OPTION,
+    POLLING_INTERVAL_OPTION,
+    SOURCE_OPTION,
     SYNC_ENVIRONMENT_OPTION,
+    UNMATCHED_BEHAVIOUR_OPTION,
+    USE_GITIGNORE_OPTION,
+    WATCH_OPTION,
 )
 from dbx.options import PROFILE_OPTION
 from dbx.sync import DeleteUnmatchedOption