From aa1ff16288bf69a3090b52fe1c12182da5643840 Mon Sep 17 00:00:00 2001 From: danieldiamond Date: Fri, 27 May 2022 09:20:54 +1000 Subject: [PATCH 01/16] init --- octavia-cli/octavia_cli/entrypoint.py | 19 ++++-- octavia-cli/octavia_cli/get/__init__.py | 3 + octavia-cli/octavia_cli/get/commands.py | 41 +++++++++++++ octavia-cli/octavia_cli/get/resources.py | 76 ++++++++++++++++++++++++ 4 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 octavia-cli/octavia_cli/get/__init__.py create mode 100644 octavia-cli/octavia_cli/get/commands.py create mode 100644 octavia-cli/octavia_cli/get/resources.py diff --git a/octavia-cli/octavia_cli/entrypoint.py b/octavia-cli/octavia_cli/entrypoint.py index 461a033df6b7..e0208021ff37 100644 --- a/octavia-cli/octavia_cli/entrypoint.py +++ b/octavia-cli/octavia_cli/entrypoint.py @@ -15,9 +15,11 @@ from .generate import commands as generate_commands from .init import commands as init_commands from .list import commands as list_commands +from .get import commands as get_commands from .telemetry import TelemetryClient, build_user_agent -AVAILABLE_COMMANDS: List[click.Command] = [list_commands._list, init_commands.init, generate_commands.generate, apply_commands.apply] +AVAILABLE_COMMANDS: List[click.Command] = [list_commands._list, get_commands.get, + init_commands.init, generate_commands.generate, apply_commands.apply] def set_context_object(ctx: click.Context, airbyte_url: str, workspace_id: str, enable_telemetry: bool) -> click.Context: @@ -39,11 +41,13 @@ def set_context_object(ctx: click.Context, airbyte_url: str, workspace_id: str, telemetry_client = TelemetryClient(enable_telemetry) try: ctx.ensure_object(dict) - ctx.obj["OCTAVIA_VERSION"] = pkg_resources.require("octavia-cli")[0].version + ctx.obj["OCTAVIA_VERSION"] = pkg_resources.require( + "octavia-cli")[0].version ctx.obj["TELEMETRY_CLIENT"] = telemetry_client api_client = get_api_client(airbyte_url) ctx.obj["WORKSPACE_ID"] = get_workspace_id(api_client, workspace_id) - ctx.obj["ANONYMOUS_DATA_COLLECTION"] = get_anonymous_data_collection(api_client, ctx.obj["WORKSPACE_ID"]) + ctx.obj["ANONYMOUS_DATA_COLLECTION"] = get_anonymous_data_collection( + api_client, ctx.obj["WORKSPACE_ID"]) api_client.user_agent = build_user_agent(ctx.obj["OCTAVIA_VERSION"]) ctx.obj["API_CLIENT"] = api_client ctx.obj["PROJECT_IS_INITIALIZED"] = check_is_initialized() @@ -76,11 +80,13 @@ def octavia(ctx: click.Context, airbyte_url: str, workspace_id: str, enable_tele ) ) if not ctx.obj["PROJECT_IS_INITIALIZED"]: - click.echo(click.style("🐙 - Project is not yet initialized.", fg="red", bold=True)) + click.echo(click.style( + "🐙 - Project is not yet initialized.", fg="red", bold=True)) def get_api_client(airbyte_url): - client_configuration = airbyte_api_client.Configuration(host=f"{airbyte_url}/api") + client_configuration = airbyte_api_client.Configuration( + host=f"{airbyte_url}/api") api_client = airbyte_api_client.ApiClient(client_configuration) check_api_health(api_client) return api_client @@ -98,7 +104,8 @@ def get_workspace_id(api_client, user_defined_workspace_id): def get_anonymous_data_collection(api_client, workspace_id): api_instance = workspace_api.WorkspaceApi(api_client) - api_response = api_instance.get_workspace(WorkspaceIdRequestBody(workspace_id), _check_return_type=False) + api_response = api_instance.get_workspace( + WorkspaceIdRequestBody(workspace_id), _check_return_type=False) return api_response.anonymous_data_collection diff --git a/octavia-cli/octavia_cli/get/__init__.py b/octavia-cli/octavia_cli/get/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/octavia-cli/octavia_cli/get/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/octavia-cli/octavia_cli/get/commands.py b/octavia-cli/octavia_cli/get/commands.py new file mode 100644 index 000000000000..e85223e21ff9 --- /dev/null +++ b/octavia-cli/octavia_cli/get/commands.py @@ -0,0 +1,41 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import click +from octavia_cli.base_commands import OctaviaCommand + +from .resources import Source, Destination, Connection + + +@click.group("get", help="Get a YAML spec for a source, destination or a connection.") +@click.pass_context +def get(ctx: click.Context): + pass + + +@get.command(cls=OctaviaCommand, name="source", help="Get YAML for a source") +@click.argument("resource_id", type=click.STRING) +@click.pass_context +def source(ctx: click.Context, resource_id: str): + resource = Source(ctx.obj["API_CLIENT"], + ctx.obj["WORKSPACE_ID"], resource_id) + click.echo(resource) + + +@get.command(cls=OctaviaCommand, name="destination", help="Get YAML for a destination") +@click.argument("resource_id", type=click.STRING) +@click.pass_context +def destination(ctx: click.Context, resource_id: str): + resource = Destination(ctx.obj["API_CLIENT"], + ctx.obj["WORKSPACE_ID"], resource_id) + click.echo(resource) + + +@get.command(cls=OctaviaCommand, name="connection", help="Get YAML for a connection") +@click.argument("resource_id", type=click.STRING) +@click.pass_context +def connection(ctx: click.Context, resource_id: str): + resource = Connection(ctx.obj["API_CLIENT"], + ctx.obj["WORKSPACE_ID"], resource_id) + click.echo(resource) diff --git a/octavia-cli/octavia_cli/get/resources.py b/octavia-cli/octavia_cli/get/resources.py new file mode 100644 index 000000000000..aeaefeaa3f13 --- /dev/null +++ b/octavia-cli/octavia_cli/get/resources.py @@ -0,0 +1,76 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import abc +from typing import Dict + +import airbyte_api_client +from airbyte_api_client.api import source_api, destination_api, connection_api +from airbyte_api_client.model.source_id_request_body import SourceIdRequestBody +from airbyte_api_client.model.destination_id_request_body import DestinationIdRequestBody +from airbyte_api_client.model.connection_id_request_body import ConnectionIdRequestBody + + +class BaseResource(abc.ABC): + @property + @abc.abstractmethod + def api( + self, + ): # pragma: no cover + pass + + @property + @abc.abstractmethod + def get_function_name( + self, + ) -> str: # pragma: no cover + pass + + @property + def _get_fn(self): + return getattr(self.api, self.get_function_name) + + @property + def get_function_kwargs(self) -> dict: + return {} + + def __init__(self, api_client: airbyte_api_client.ApiClient, workspace_id: str, resource_id: str): + self.api_instance = self.api(api_client) + self.workspace_id = workspace_id + self.resource_id = resource_id + + def get_resource(self) -> Dict: + api_response = self._get_fn( + self.api_instance, **self.get_function_kwargs) + return api_response + + def __repr__(self): + return str(self.get_resource()) + + +class Source(BaseResource): + api = source_api.SourceApi + get_function_name = "get_source" + + @ property + def get_function_kwargs(self) -> dict: + return {"source_id_request_body": SourceIdRequestBody(source_id=self.resource_id)} + + +class Destination(BaseResource): + api = destination_api.DestinationApi + get_function_name = "get_destination" + + @ property + def get_function_kwargs(self) -> dict: + return {"destination_id_request_body": DestinationIdRequestBody(destination_id=self.resource_id)} + + +class Connection(BaseResource): + api = connection_api.ConnectionApi + get_function_name = "get_connection" + + @ property + def get_function_kwargs(self) -> dict: + return {"connection_id_request_body": ConnectionIdRequestBody(connection_id=self.resource_id)} From 19a52a2cedc0ab7741150bd9477b64e8e5d8c48c Mon Sep 17 00:00:00 2001 From: danieldiamond Date: Fri, 27 May 2022 13:38:07 +1000 Subject: [PATCH 02/16] Update docs and add tests --- octavia-cli/README.md | 126 +++++++++++++++++- octavia-cli/octavia_cli/entrypoint.py | 26 ++-- octavia-cli/octavia_cli/get/commands.py | 30 +++-- octavia-cli/octavia_cli/get/resources.py | 25 ++-- octavia-cli/unit_tests/test_entrypoint.py | 1 + octavia-cli/unit_tests/test_get/__init__.py | 3 + .../unit_tests/test_get/test_commands.py | 75 +++++++++++ .../unit_tests/test_get/test_resources.py | 59 ++++++++ 8 files changed, 309 insertions(+), 36 deletions(-) create mode 100644 octavia-cli/unit_tests/test_get/__init__.py create mode 100644 octavia-cli/unit_tests/test_get/test_commands.py create mode 100644 octavia-cli/unit_tests/test_get/test_resources.py diff --git a/octavia-cli/README.md b/octavia-cli/README.md index 2f1ddf5c7499..7db104c0a1cd 100644 --- a/octavia-cli/README.md +++ b/octavia-cli/README.md @@ -105,7 +105,7 @@ This script: ```bash touch ~/.octavia # Create a file to store env variables that will be mapped the octavia-cli container mkdir my_octavia_project_directory # Create your octavia project directory where YAML configurations will be stored. -docker run --name octavia-cli -i --rm -v my_octavia_project_directory:/home/octavia-project --network host --user $(id -u):$(id -g) --env-file ~/.octavia airbyte/octavia-cli:0.39.1-alpha +docker run --name octavia-cli -i --rm -v ./my_repo:/home/octavia-project --network host --user $(id -u):$(id -g) --env-file ~/.octavia airbyte/octavia-cli:0.39.1-alpha ``` ### Using `docker-compose` @@ -154,6 +154,9 @@ docker-compose run octavia-cli ` | **`octavia list workspace sources`** | List existing sources in current the Airbyte workspace. | | **`octavia list workspace destinations`** | List existing destinations in the current Airbyte workspace. | | **`octavia list workspace connections`** | List existing connections in the current Airbyte workspace. | +| **`octavia get workspace source`** | Get an existing source in current the Airbyte workspace. | +| **`octavia get workspace destination`** | Get an existing destination in the current Airbyte workspace. | +| **`octavia get workspace connection`** | Get an existing connection in the current Airbyte workspace. | | **`octavia generate source`** | Generate a local YAML configuration for a new source. | | **`octavia generate destination`** | Generate a local YAML configuration for a new destination. | | **`octavia generate connection`** | Generate a local YAML configuration for a new connection. | @@ -242,6 +245,127 @@ NAME CONNECTION ID STATUS SOURCE ID weather_to_pg a4491317-153e-436f-b646-0b39338f9aab active c4aa8550-2122-4a33-9a21-adbfaa638544 c0c977c2-48e7-46fe-9f57-576285c26d42 ``` +#### `octavia get source ` + +Get an existing source in current the Airbyte workspace + +| **Argument** | **Description** | +|-----------------|-----------------------------------------------------------------------------------------------| +| `SOURCE_ID` | The source connector id. Can be retrieved using `octavia list workspace sources`. | + +**Example**: + +```bash +$ octavia get source c0c977c2-48e7-46fe-9f57-576285c26d42 +{'connection_configuration': {'key': '**********', + 'start_date': '2010-01-01T00:00:00.000Z', + 'token': '**********'}, + 'name': 'Pokemon', + 'source_definition_id': 'b08e4776-d1de-4e80-ab5c-1e51dad934a2', + 'source_id': 'c0c977c2-48e7-46fe-9f57-576285c26d42', + 'source_name': 'My Poke', + 'workspace_id': 'c4aa8550-2122-4a33-9a21-adbfaa638544'} +``` + +#### `octavia get destination ` + +Get an existing destination in current the Airbyte workspace + +| **Argument** | **Description** | +|-----------------|-----------------------------------------------------------------------------------------------| +| `DESTINATION_ID` | The destination connector id. Can be retrieved using `octavia list workspace destinations`. | + +**Example**: + +```bash +$ octavia get destination c0c977c2-48e7-46fe-9f57-576285c26d42 +{ + "sourceDefinitionId": "18102e7c-5160-4000-821f-4d7cfdf87201", + "sourceId": "18102e7c-5160-4000-841b-15e8ec48c301", + "workspaceId": "18102e7c-5160-4000-883a-30bc7cd65601", + "connectionConfiguration": { + "user": "charles" + }, + "name": "string", + "sourceName": "string" +} +``` + +#### `octavia get connection ` + +Get an existing connection in current the Airbyte workspace + +| **Argument** | **Description** | +|-----------------|-----------------------------------------------------------------------------------------------| +| `CONNECTION_ID` | The connection connector id. Can be retrieved using `octavia list workspace connections`. | + +**Example**: + +```bash +$ octavia get connection c0c977c2-48e7-46fe-9f57-576285c26d42 +{ + "connectionId": "18102e7c-5340-4000-8656-c433ed782601", + "name": "string", + "namespaceDefinition": "source", + "namespaceFormat": "${SOURCE_NAMESPACE}", + "prefix": "string", + "sourceId": "18102e7c-5340-4000-8eaa-4a86f844b101", + "destinationId": "18102e7c-5340-4000-8e58-6bed49c24b01", + "operationIds": [ + "18102e7c-5340-4000-8ef0-f35c05a49a01" + ], + "syncCatalog": { + "streams": [ + { + "stream": { + "name": "string", + "jsonSchema": {}, + "supportedSyncModes": [ + "full_refresh" + ], + "sourceDefinedCursor": false, + "defaultCursorField": [ + "string" + ], + "sourceDefinedPrimaryKey": [ + [ + "string" + ] + ], + "namespace": "string" + }, + "config": { + "syncMode": "full_refresh", + "cursorField": [ + "string" + ], + "destinationSyncMode": "append", + "primaryKey": [ + [ + "string" + ] + ], + "aliasName": "string", + "selected": false + } + } + ] + }, + "schedule": { + "units": 0, + "timeUnit": "minutes" + }, + "status": "active", + "resourceRequirements": { + "cpu_request": "string", + "cpu_limit": "string", + "memory_request": "string", + "memory_limit": "string" + }, + "sourceCatalogId": "18102e7c-5340-4000-85f3-204ab7715801" +} +``` + #### `octavia generate source ` Generate a YAML configuration for a source. diff --git a/octavia-cli/octavia_cli/entrypoint.py b/octavia-cli/octavia_cli/entrypoint.py index e0208021ff37..fe4e5b230976 100644 --- a/octavia-cli/octavia_cli/entrypoint.py +++ b/octavia-cli/octavia_cli/entrypoint.py @@ -13,13 +13,18 @@ from .apply import commands as apply_commands from .check_context import check_api_health, check_is_initialized, check_workspace_exists from .generate import commands as generate_commands +from .get import commands as get_commands from .init import commands as init_commands from .list import commands as list_commands -from .get import commands as get_commands from .telemetry import TelemetryClient, build_user_agent -AVAILABLE_COMMANDS: List[click.Command] = [list_commands._list, get_commands.get, - init_commands.init, generate_commands.generate, apply_commands.apply] +AVAILABLE_COMMANDS: List[click.Command] = [ + list_commands._list, + get_commands.get, + init_commands.init, + generate_commands.generate, + apply_commands.apply, +] def set_context_object(ctx: click.Context, airbyte_url: str, workspace_id: str, enable_telemetry: bool) -> click.Context: @@ -41,13 +46,11 @@ def set_context_object(ctx: click.Context, airbyte_url: str, workspace_id: str, telemetry_client = TelemetryClient(enable_telemetry) try: ctx.ensure_object(dict) - ctx.obj["OCTAVIA_VERSION"] = pkg_resources.require( - "octavia-cli")[0].version + ctx.obj["OCTAVIA_VERSION"] = pkg_resources.require("octavia-cli")[0].version ctx.obj["TELEMETRY_CLIENT"] = telemetry_client api_client = get_api_client(airbyte_url) ctx.obj["WORKSPACE_ID"] = get_workspace_id(api_client, workspace_id) - ctx.obj["ANONYMOUS_DATA_COLLECTION"] = get_anonymous_data_collection( - api_client, ctx.obj["WORKSPACE_ID"]) + ctx.obj["ANONYMOUS_DATA_COLLECTION"] = get_anonymous_data_collection(api_client, ctx.obj["WORKSPACE_ID"]) api_client.user_agent = build_user_agent(ctx.obj["OCTAVIA_VERSION"]) ctx.obj["API_CLIENT"] = api_client ctx.obj["PROJECT_IS_INITIALIZED"] = check_is_initialized() @@ -80,13 +83,11 @@ def octavia(ctx: click.Context, airbyte_url: str, workspace_id: str, enable_tele ) ) if not ctx.obj["PROJECT_IS_INITIALIZED"]: - click.echo(click.style( - "🐙 - Project is not yet initialized.", fg="red", bold=True)) + click.echo(click.style("🐙 - Project is not yet initialized.", fg="red", bold=True)) def get_api_client(airbyte_url): - client_configuration = airbyte_api_client.Configuration( - host=f"{airbyte_url}/api") + client_configuration = airbyte_api_client.Configuration(host=f"{airbyte_url}/api") api_client = airbyte_api_client.ApiClient(client_configuration) check_api_health(api_client) return api_client @@ -104,8 +105,7 @@ def get_workspace_id(api_client, user_defined_workspace_id): def get_anonymous_data_collection(api_client, workspace_id): api_instance = workspace_api.WorkspaceApi(api_client) - api_response = api_instance.get_workspace( - WorkspaceIdRequestBody(workspace_id), _check_return_type=False) + api_response = api_instance.get_workspace(WorkspaceIdRequestBody(workspace_id), _check_return_type=False) return api_response.anonymous_data_collection diff --git a/octavia-cli/octavia_cli/get/commands.py b/octavia-cli/octavia_cli/get/commands.py index e85223e21ff9..75bacaeec843 100644 --- a/octavia-cli/octavia_cli/get/commands.py +++ b/octavia-cli/octavia_cli/get/commands.py @@ -2,10 +2,12 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # +from typing import List + import click from octavia_cli.base_commands import OctaviaCommand -from .resources import Source, Destination, Connection +from .resources import Connection, Destination, Source @click.group("get", help="Get a YAML spec for a source, destination or a connection.") @@ -18,24 +20,32 @@ def get(ctx: click.Context): @click.argument("resource_id", type=click.STRING) @click.pass_context def source(ctx: click.Context, resource_id: str): - resource = Source(ctx.obj["API_CLIENT"], - ctx.obj["WORKSPACE_ID"], resource_id) - click.echo(resource) + resource = Source(ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) + click.echo(resource.get_config()) @get.command(cls=OctaviaCommand, name="destination", help="Get YAML for a destination") @click.argument("resource_id", type=click.STRING) @click.pass_context def destination(ctx: click.Context, resource_id: str): - resource = Destination(ctx.obj["API_CLIENT"], - ctx.obj["WORKSPACE_ID"], resource_id) - click.echo(resource) + resource = Destination(ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) + click.echo(resource.get_config()) @get.command(cls=OctaviaCommand, name="connection", help="Get YAML for a connection") @click.argument("resource_id", type=click.STRING) @click.pass_context def connection(ctx: click.Context, resource_id: str): - resource = Connection(ctx.obj["API_CLIENT"], - ctx.obj["WORKSPACE_ID"], resource_id) - click.echo(resource) + resource = Connection(ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) + click.echo(resource.get_config()) + + +AVAILABLE_COMMANDS: List[click.Command] = [source, destination, connection] + + +def add_commands_to_list(): + for command in AVAILABLE_COMMANDS: + get.add_command(command) + + +add_commands_to_list() diff --git a/octavia-cli/octavia_cli/get/resources.py b/octavia-cli/octavia_cli/get/resources.py index aeaefeaa3f13..4a52ef373299 100644 --- a/octavia-cli/octavia_cli/get/resources.py +++ b/octavia-cli/octavia_cli/get/resources.py @@ -6,10 +6,15 @@ from typing import Dict import airbyte_api_client -from airbyte_api_client.api import source_api, destination_api, connection_api -from airbyte_api_client.model.source_id_request_body import SourceIdRequestBody -from airbyte_api_client.model.destination_id_request_body import DestinationIdRequestBody +import click +from airbyte_api_client.api import connection_api, destination_api, source_api from airbyte_api_client.model.connection_id_request_body import ConnectionIdRequestBody +from airbyte_api_client.model.destination_id_request_body import DestinationIdRequestBody +from airbyte_api_client.model.source_id_request_body import SourceIdRequestBody + + +class DefinitionNotFoundError(click.ClickException): + pass class BaseResource(abc.ABC): @@ -40,20 +45,16 @@ def __init__(self, api_client: airbyte_api_client.ApiClient, workspace_id: str, self.workspace_id = workspace_id self.resource_id = resource_id - def get_resource(self) -> Dict: - api_response = self._get_fn( - self.api_instance, **self.get_function_kwargs) + def get_config(self) -> Dict: + api_response = self._get_fn(self.api_instance, **self.get_function_kwargs) return api_response - def __repr__(self): - return str(self.get_resource()) - class Source(BaseResource): api = source_api.SourceApi get_function_name = "get_source" - @ property + @property def get_function_kwargs(self) -> dict: return {"source_id_request_body": SourceIdRequestBody(source_id=self.resource_id)} @@ -62,7 +63,7 @@ class Destination(BaseResource): api = destination_api.DestinationApi get_function_name = "get_destination" - @ property + @property def get_function_kwargs(self) -> dict: return {"destination_id_request_body": DestinationIdRequestBody(destination_id=self.resource_id)} @@ -71,6 +72,6 @@ class Connection(BaseResource): api = connection_api.ConnectionApi get_function_name = "get_connection" - @ property + @property def get_function_kwargs(self) -> dict: return {"connection_id_request_body": ConnectionIdRequestBody(connection_id=self.resource_id)} diff --git a/octavia-cli/unit_tests/test_entrypoint.py b/octavia-cli/unit_tests/test_entrypoint.py index cc4e866c5753..5afce0dd7ddd 100644 --- a/octavia-cli/unit_tests/test_entrypoint.py +++ b/octavia-cli/unit_tests/test_entrypoint.py @@ -142,6 +142,7 @@ def test_not_implemented_commands(command): def test_available_commands(): assert entrypoint.AVAILABLE_COMMANDS == [ entrypoint.list_commands._list, + entrypoint.get_commands.get, entrypoint.init_commands.init, entrypoint.generate_commands.generate, entrypoint.apply_commands.apply, diff --git a/octavia-cli/unit_tests/test_get/__init__.py b/octavia-cli/unit_tests/test_get/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/octavia-cli/unit_tests/test_get/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/octavia-cli/unit_tests/test_get/test_commands.py b/octavia-cli/unit_tests/test_get/test_commands.py new file mode 100644 index 000000000000..fb4ef61c71a6 --- /dev/null +++ b/octavia-cli/unit_tests/test_get/test_commands.py @@ -0,0 +1,75 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import pytest +from click.testing import CliRunner +from octavia_cli.get import commands + + +def test_commands_in_get_group(): + get_commands = commands.get.commands.values() + for command in commands.AVAILABLE_COMMANDS: + assert command in get_commands + + +@pytest.fixture +def context_object(mock_api_client, mock_telemetry_client): + return { + "API_CLIENT": mock_api_client, + "WORKSPACE_ID": "my_workspace_id", + "resource_id": "my_resource_id", + "TELEMETRY_CLIENT": mock_telemetry_client, + } + + +def test_available_commands(): + assert commands.AVAILABLE_COMMANDS == [commands.source, commands.destination, commands.connection] + + +@pytest.mark.parametrize( + "command,resource_id", + [ + (commands.source, "my_resource_id"), + ], +) +def test_source(mocker, context_object, command, resource_id): + runner = CliRunner() + mocker.patch.object(commands, "Source", mocker.Mock()) + mock_renderer = commands.Source.return_value + mock_renderer.get_config.return_value = '{"hello": "world"}' + result = runner.invoke(command, [resource_id], obj=context_object) + assert result.exit_code == 0 + commands.Source.assert_called_with(context_object["API_CLIENT"], context_object["WORKSPACE_ID"], resource_id) + + +@pytest.mark.parametrize( + "command,resource_id", + [ + (commands.destination, "my_resource_id"), + ], +) +def test_destination(mocker, context_object, command, resource_id): + runner = CliRunner() + mocker.patch.object(commands, "Destination", mocker.Mock()) + mock_renderer = commands.Destination.return_value + mock_renderer.get_config.return_value = '{"hello": "world"}' + result = runner.invoke(command, [resource_id], obj=context_object) + assert result.exit_code == 0 + commands.Destination.assert_called_with(context_object["API_CLIENT"], context_object["WORKSPACE_ID"], resource_id) + + +@pytest.mark.parametrize( + "command,resource_id", + [ + (commands.connection, "my_resource_id"), + ], +) +def test_connection(mocker, context_object, command, resource_id): + runner = CliRunner() + mocker.patch.object(commands, "Connection", mocker.Mock()) + mock_renderer = commands.Connection.return_value + mock_renderer.get_config.return_value = '{"hello": "world"}' + result = runner.invoke(command, [resource_id], obj=context_object) + assert result.exit_code == 0 + commands.Connection.assert_called_with(context_object["API_CLIENT"], context_object["WORKSPACE_ID"], resource_id) diff --git a/octavia-cli/unit_tests/test_get/test_resources.py b/octavia-cli/unit_tests/test_get/test_resources.py new file mode 100644 index 000000000000..84560de41e63 --- /dev/null +++ b/octavia-cli/unit_tests/test_get/test_resources.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import pytest +from airbyte_api_client.api import connection_api, destination_api, source_api +from airbyte_api_client.model.connection_id_request_body import ConnectionIdRequestBody +from airbyte_api_client.model.destination_id_request_body import DestinationIdRequestBody +from airbyte_api_client.model.source_id_request_body import SourceIdRequestBody +from octavia_cli.get.resources import BaseResource, Connection, Destination, Source + + +class TestBaseResource: + @pytest.fixture + def patch_base_class(self, mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(BaseResource, "__abstractmethods__", set()) + mocker.patch.object(BaseResource, "api", mocker.Mock()) + mocker.patch.object(BaseResource, "get_function_name", "foo") + + def test_init(self, patch_base_class, mock_api_client, mocker): + base_definition = BaseResource(mock_api_client, "workspace_id", "resource_id") + assert base_definition.api_instance == base_definition.api.return_value + base_definition.api.assert_called_with(mock_api_client) + assert base_definition.get_function_kwargs == {} + assert base_definition._get_fn == getattr(base_definition.api, base_definition.get_function_name) + + def test_get_config(self, patch_base_class, mock_api_client, mocker): + assert Source.__base__ == BaseResource + + base_definition = BaseResource(mock_api_client, "workspace_id", "resource_id") + assert base_definition._get_fn == getattr(base_definition.api, base_definition.get_function_name) + + +class TestSource: + def test_init(self, mock_api_client): + assert Source.__base__ == BaseResource + source = Source(mock_api_client, "workspace_id", "resource_id") + assert source.api == source_api.SourceApi + assert source.get_function_name == "get_source" + assert source.get_function_kwargs == {"source_id_request_body": SourceIdRequestBody("resource_id")} + + +class TestDestination: + def test_init(self, mock_api_client): + assert Destination.__base__ == BaseResource + destination = Destination(mock_api_client, "workspace_id", "resource_id") + assert destination.api == destination_api.DestinationApi + assert destination.get_function_name == "get_destination" + assert destination.get_function_kwargs == {"destination_id_request_body": DestinationIdRequestBody("resource_id")} + + +class TestConnection: + def test_init(self, mock_api_client): + assert Connection.__base__ == BaseResource + connection = Connection(mock_api_client, "workspace_id", "resource_id") + assert connection.api == connection_api.ConnectionApi + assert connection.get_function_name == "get_connection" + assert connection.get_function_kwargs == {"connection_id_request_body": ConnectionIdRequestBody("resource_id")} From 0150ae9dd693e2b9ef1a38eb7000f3686fe1b458 Mon Sep 17 00:00:00 2001 From: danieldiamond Date: Fri, 27 May 2022 13:44:01 +1000 Subject: [PATCH 03/16] Revert docker run docs --- octavia-cli/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octavia-cli/README.md b/octavia-cli/README.md index 7db104c0a1cd..aaf6475897f0 100644 --- a/octavia-cli/README.md +++ b/octavia-cli/README.md @@ -105,7 +105,7 @@ This script: ```bash touch ~/.octavia # Create a file to store env variables that will be mapped the octavia-cli container mkdir my_octavia_project_directory # Create your octavia project directory where YAML configurations will be stored. -docker run --name octavia-cli -i --rm -v ./my_repo:/home/octavia-project --network host --user $(id -u):$(id -g) --env-file ~/.octavia airbyte/octavia-cli:0.39.1-alpha +docker run --name octavia-cli -i --rm -v my_octavia_project_directory:/home/octavia-project --network host --user $(id -u):$(id -g) --env-file ~/.octavia airbyte/octavia-cli:0.39.1-alpha ``` ### Using `docker-compose` From 7748d63d85f1c29ad22494ab1f6b433cc1910d6d Mon Sep 17 00:00:00 2001 From: danieldiamond Date: Sun, 29 May 2022 21:00:17 +1000 Subject: [PATCH 04/16] Add "update" commands --- octavia-cli/README.md | 48 +++++++++++++ octavia-cli/octavia_cli/entrypoint.py | 2 + octavia-cli/octavia_cli/generate/renderers.py | 67 +++++++++++++++++++ octavia-cli/octavia_cli/get/resources.py | 3 + octavia-cli/octavia_cli/update/__init__.py | 3 + octavia-cli/octavia_cli/update/commands.py | 64 ++++++++++++++++++ octavia-cli/unit_tests/test_entrypoint.py | 1 + 7 files changed, 188 insertions(+) create mode 100644 octavia-cli/octavia_cli/update/__init__.py create mode 100644 octavia-cli/octavia_cli/update/commands.py diff --git a/octavia-cli/README.md b/octavia-cli/README.md index aaf6475897f0..1c125eae5b70 100644 --- a/octavia-cli/README.md +++ b/octavia-cli/README.md @@ -157,6 +157,9 @@ docker-compose run octavia-cli ` | **`octavia get workspace source`** | Get an existing source in current the Airbyte workspace. | | **`octavia get workspace destination`** | Get an existing destination in the current Airbyte workspace. | | **`octavia get workspace connection`** | Get an existing connection in the current Airbyte workspace. | +| **`octavia update workspace source`** | Update a local YAML configuration of an existing source in current the Airbyte workspace. | +| **`octavia update workspace destination`** | Update a local YAML configuration of an existing destination in the current Airbyte workspace. | +| **`octavia update workspace connection`** | Update a local YAML configuration of an existing connection in the current Airbyte workspace. | | **`octavia generate source`** | Generate a local YAML configuration for a new source. | | **`octavia generate destination`** | Generate a local YAML configuration for a new destination. | | **`octavia generate connection`** | Generate a local YAML configuration for a new connection. | @@ -366,6 +369,51 @@ $ octavia get connection c0c977c2-48e7-46fe-9f57-576285c26d42 } ``` +#### `octavia update source ` + +Update a local YAML configuration of an existing source in current the Airbyte workspace. + +| **Argument** | **Description** | +|-----------------|-----------------------------------------------------------------------------------------------| +| `SOURCE_ID` | The source connector id. Can be retrieved using `octavia list workspace sources`. | + +**Example**: + +```bash +$ octavia update source c0c977c2-48e7-46fe-9f57-576285c26d42 +✅ - Updated the source template for weather_to_pg in ./source/weather_to_pg/configuration.yaml. +``` + +#### `octavia update destination ` + +Update a local YAML configuration of an existing destination in current the Airbyte workspace. + +| **Argument** | **Description** | +|-----------------|-----------------------------------------------------------------------------------------------| +| `DESTINATION_ID` | The destination connector id. Can be retrieved using `octavia list workspace destinations`. | + +**Example**: + +```bash +$ octavia update destination c0c977c2-48e7-46fe-9f57-576285c26d42 +✅ - Updated the destination template for weather_to_pg in ./destination/weather_to_pg/configuration.yaml. +``` + +#### `octavia update conection ` + +Update a local YAML configuration of an existing connection in current the Airbyte workspace. + +| **Argument** | **Description** | +|-----------------|-----------------------------------------------------------------------------------------------| +| `CONNECTION_ID` | The connection connector id. Can be retrieved using `octavia list workspace connections`. | + +**Example**: + +```bash +$ octavia update connection c0c977c2-48e7-46fe-9f57-576285c26d42 +✅ - Updated the connection template for weather_to_pg in ./connection/weather_to_pg/configuration.yaml. +``` + #### `octavia generate source ` Generate a YAML configuration for a source. diff --git a/octavia-cli/octavia_cli/entrypoint.py b/octavia-cli/octavia_cli/entrypoint.py index fe4e5b230976..eaed9c290b6c 100644 --- a/octavia-cli/octavia_cli/entrypoint.py +++ b/octavia-cli/octavia_cli/entrypoint.py @@ -17,10 +17,12 @@ from .init import commands as init_commands from .list import commands as list_commands from .telemetry import TelemetryClient, build_user_agent +from .update import commands as update_commands AVAILABLE_COMMANDS: List[click.Command] = [ list_commands._list, get_commands.get, + update_commands.update, init_commands.init, generate_commands.generate, apply_commands.apply, diff --git a/octavia-cli/octavia_cli/generate/renderers.py b/octavia-cli/octavia_cli/generate/renderers.py index cd5371399974..e0f279c203d4 100644 --- a/octavia-cli/octavia_cli/generate/renderers.py +++ b/octavia-cli/octavia_cli/generate/renderers.py @@ -3,6 +3,7 @@ # import abc +import collections.abc import os from typing import Any, Callable, List @@ -17,6 +18,15 @@ JINJA_ENV = Environment(loader=PackageLoader(__package__), autoescape=select_autoescape(), trim_blocks=False, lstrip_blocks=True) +def update_nested_dict(d, u): + for k, v in u.items(): + if isinstance(v, collections.abc.Mapping): + d[k] = update_nested_dict(d.get(k, {}), v) + else: + d[k] = v + return d + + class FieldToRender: def __init__(self, name: str, required: bool, field_metadata: dict) -> None: """Initialize a FieldToRender instance @@ -187,6 +197,29 @@ def write_yaml(self, project_path: str) -> str: f.write(rendered) return output_path + def update_configuration(self, project_path: str, configuration: dict) -> str: + """Update configuration from the YAML file in local project path. + Resource must be generated prior to updating. + + Args: + project_path (str): Path to directory hosting the octavia project. + configuration (dict): Configuraton of the resource. + + Returns: + str: Path to the rendered specification. + """ + output_path = self._get_output_path(project_path, self.definition.type) + + with open(output_path) as f: + data = yaml.safe_load(f) + + data["configuration"] = update_nested_dict(data["configuration"], configuration) + + with open(output_path, "wb") as f: + yaml.safe_dump(data, f, default_flow_style=False, sort_keys=False, allow_unicode=True, encoding="utf-8") + + return output_path + class ConnectorSpecificationRenderer(BaseRenderer): TEMPLATE = JINJA_ENV.get_template("source_or_destination.yaml.j2") @@ -224,6 +257,40 @@ def _render(self) -> str: ) +class ConnectorRenderer(BaseRenderer): + TEMPLATE = JINJA_ENV.get_template("source_or_destination.yaml.j2") + + def __init__(self, resource_id: str, definition: BaseDefinition) -> None: + """Connector specification renderer constructor. + + Args: + resource_id (str): ID of the existing source or destination. + definition (BaseDefinition): The definition related to a source or a destination. + """ + super().__init__(resource_id) + self.definition = definition + + def _parse_connection_specification(self, schema: dict) -> List[List["FieldToRender"]]: + """Create a renderable structure from the specification schema + + Returns: + List[List["FieldToRender"]]: List of list of fields to render. + """ + if schema.get("oneOf"): + roots = [] + for one_of_value in schema.get("oneOf"): + required_fields = one_of_value.get("required", []) + roots.append(parse_fields(required_fields, one_of_value["properties"])) + return roots + else: + required_fields = schema.get("required", []) + return [parse_fields(required_fields, schema["properties"])] + + def _render(self) -> str: + parsed_schema = self._parse_connection_specification(self.definition.specification.connection_specification) + return self.TEMPLATE.render({"resource_id": self.resource_id, "definition": self.definition, "configuration_fields": parsed_schema}) + + class ConnectionRenderer(BaseRenderer): TEMPLATE = JINJA_ENV.get_template("connection.yaml.j2") diff --git a/octavia-cli/octavia_cli/get/resources.py b/octavia-cli/octavia_cli/get/resources.py index 4a52ef373299..30f0150c8881 100644 --- a/octavia-cli/octavia_cli/get/resources.py +++ b/octavia-cli/octavia_cli/get/resources.py @@ -51,6 +51,7 @@ def get_config(self) -> Dict: class Source(BaseResource): + name = "source" api = source_api.SourceApi get_function_name = "get_source" @@ -60,6 +61,7 @@ def get_function_kwargs(self) -> dict: class Destination(BaseResource): + name = "destination" api = destination_api.DestinationApi get_function_name = "get_destination" @@ -69,6 +71,7 @@ def get_function_kwargs(self) -> dict: class Connection(BaseResource): + name = "connection" api = connection_api.ConnectionApi get_function_name = "get_connection" diff --git a/octavia-cli/octavia_cli/update/__init__.py b/octavia-cli/octavia_cli/update/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/octavia-cli/octavia_cli/update/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/octavia-cli/octavia_cli/update/commands.py b/octavia-cli/octavia_cli/update/commands.py new file mode 100644 index 000000000000..763c0f5b364f --- /dev/null +++ b/octavia-cli/octavia_cli/update/commands.py @@ -0,0 +1,64 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from typing import List + +import click +from octavia_cli.base_commands import OctaviaCommand +from octavia_cli.generate import definitions +from octavia_cli.generate.renderers import ConnectorSpecificationRenderer +from octavia_cli.get.resources import Connection, Destination, Source + + +@click.group("update", help="Update configurations in a YAML spec for a source, destination or a connection.") +@click.pass_context +def update(ctx: click.Context): + pass + + +def update_resource(resource_type, api_client, workspace_id, resource_id): + resource = resource_type(api_client, workspace_id, resource_id) + definition_type = resource_type.name + config = resource.get_config() + definition_id = config[f"{resource_type.name}_definition_id"] + resource_name = config[f"{resource_type.name}_name"] + + definition = definitions.factory(definition_type, api_client, workspace_id, definition_id) + renderer = ConnectorSpecificationRenderer(resource_name, definition) + + output_path = renderer.update_configuration(project_path=".", configuration=config["connection_configuration"]) + message = f"✅ - Updated the {resource_type.name} template for {resource_name} in {output_path}." + click.echo(click.style(message, fg="green")) + + +@update.command(cls=OctaviaCommand, name="source", help="Get YAML for a source") +@click.argument("resource_id", type=click.STRING) +@click.pass_context +def source(ctx: click.Context, resource_id: str): + update_resource(Source, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) + + +@update.command(cls=OctaviaCommand, name="destination", help="Get YAML for a destination") +@click.argument("resource_id", type=click.STRING) +@click.pass_context +def destination(ctx: click.Context, resource_id: str): + update_resource(Destination, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) + + +@update.command(cls=OctaviaCommand, name="connection", help="Get YAML for a connection") +@click.argument("resource_id", type=click.STRING) +@click.pass_context +def connection(ctx: click.Context, resource_id: str): + update_resource(Connection, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) + + +AVAILABLE_COMMANDS: List[click.Command] = [source, destination, connection] + + +def add_commands_to_list(): + for command in AVAILABLE_COMMANDS: + update.add_command(command) + + +add_commands_to_list() diff --git a/octavia-cli/unit_tests/test_entrypoint.py b/octavia-cli/unit_tests/test_entrypoint.py index 5afce0dd7ddd..b330a9aef82d 100644 --- a/octavia-cli/unit_tests/test_entrypoint.py +++ b/octavia-cli/unit_tests/test_entrypoint.py @@ -143,6 +143,7 @@ def test_available_commands(): assert entrypoint.AVAILABLE_COMMANDS == [ entrypoint.list_commands._list, entrypoint.get_commands.get, + entrypoint.update_commands.update, entrypoint.init_commands.init, entrypoint.generate_commands.generate, entrypoint.apply_commands.apply, From 58303ea625fe6279856ed7da4d9917731b9f21fd Mon Sep 17 00:00:00 2001 From: danieldiamond Date: Sun, 29 May 2022 22:12:05 +1000 Subject: [PATCH 05/16] Add "update sources" and "update destinations" to create yaml files for all existing sources and destinations --- .../templates/source_or_destination.yaml.j2 | 14 +++--- octavia-cli/octavia_cli/update/commands.py | 46 ++++++++++++++++++- 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/octavia-cli/octavia_cli/generate/templates/source_or_destination.yaml.j2 b/octavia-cli/octavia_cli/generate/templates/source_or_destination.yaml.j2 index 3ea86d4902a1..c8a33f708577 100644 --- a/octavia-cli/octavia_cli/generate/templates/source_or_destination.yaml.j2 +++ b/octavia-cli/octavia_cli/generate/templates/source_or_destination.yaml.j2 @@ -1,6 +1,6 @@ # Configuration for {{ definition.docker_repository }} # Documentation about this connector can be found at {{ definition.documentation_url }} -resource_name: {{ resource_name}} +resource_name: "{{ resource_name}}" definition_type: {{ definition.type}} definition_id: {{ definition.id }} definition_image: {{ definition.docker_repository }} @@ -32,7 +32,7 @@ definition_version: {{ definition.docker_image_tag }} {%- macro render_one_of(field) %} -{{ field.name }}: +{{ field.name }}: {%- for one_of_value in field.one_of_values %} {%- if loop.first %} ## -------- Pick one valid structure among the examples below: -------- @@ -41,17 +41,17 @@ definition_version: {{ definition.docker_image_tag }} ## -------- Another valid structure for {{ field.name }}: -------- {{- render_sub_fields(one_of_value, True)|indent(2, False) }} {%- endif %} -{%- endfor %} +{%- endfor %} {%- endmacro %} {%- macro render_object_field(field) %} -{{ field.name }}: - {{- render_sub_fields(field.object_properties, is_commented=False)|indent(2, False)}} +{{ field.name }}: + {{- render_sub_fields(field.object_properties, is_commented=False)|indent(2, False)}} {%- endmacro %} {%- macro render_array_of_objects(field) %} -{{ field.name }}: - {{- render_array_sub_fields(field.array_items, is_commented=False)|indent(2, False)}} +{{ field.name }}: + {{- render_array_sub_fields(field.array_items, is_commented=False)|indent(2, False)}} {%- endmacro %} {%- macro render_root(root, is_commented) %} diff --git a/octavia-cli/octavia_cli/update/commands.py b/octavia-cli/octavia_cli/update/commands.py index 763c0f5b364f..9dfec9e4f2bb 100644 --- a/octavia-cli/octavia_cli/update/commands.py +++ b/octavia-cli/octavia_cli/update/commands.py @@ -7,8 +7,10 @@ import click from octavia_cli.base_commands import OctaviaCommand from octavia_cli.generate import definitions +from octavia_cli.generate.commands import generate_source_or_destination from octavia_cli.generate.renderers import ConnectorSpecificationRenderer from octavia_cli.get.resources import Connection, Destination, Source +from octavia_cli.list.listings import Destinations, Sources @click.group("update", help="Update configurations in a YAML spec for a source, destination or a connection.") @@ -17,12 +19,24 @@ def update(ctx: click.Context): pass +def list_resource(resource_type, api_client, workspace_id): + resources = resource_type(api_client, workspace_id) + return {resource_id: resource_name for resource_name, _, resource_id in resources.get_listing()} + + +def get_resource_definition_id(resource_type, api_client, workspace_id, resource_id): + resource = resource_type(api_client, workspace_id, resource_id) + config = resource.get_config() + return config[f"{resource_type.name}_definition_id"] + + def update_resource(resource_type, api_client, workspace_id, resource_id): resource = resource_type(api_client, workspace_id, resource_id) definition_type = resource_type.name config = resource.get_config() definition_id = config[f"{resource_type.name}_definition_id"] - resource_name = config[f"{resource_type.name}_name"] + # Schema varies across resources, e.g. pull "name" else default to "source_name" + resource_name = config.get("name") or config[f"{resource_type.name}_name"] definition = definitions.factory(definition_type, api_client, workspace_id, definition_id) renderer = ConnectorSpecificationRenderer(resource_name, definition) @@ -39,6 +53,36 @@ def source(ctx: click.Context, resource_id: str): update_resource(Source, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) +@update.command(cls=OctaviaCommand, name="sources", help="Get YAML for a source") +@click.pass_context +def sources(ctx: click.Context): + api_client = ctx.obj["API_CLIENT"] + workspace_id = ctx.obj["WORKSPACE_ID"] + sources = list_resource(Sources, api_client, workspace_id) + + for source_id, source_name in sources.items(): + definition_id = get_resource_definition_id(Source, api_client, workspace_id, source_id) + + generate_source_or_destination("source", api_client, workspace_id, definition_id, source_name) + + update_resource(Source, api_client, workspace_id, source_id) + + +@update.command(cls=OctaviaCommand, name="destinations", help="Get YAML for a destination") +@click.pass_context +def destinations(ctx: click.Context): + api_client = ctx.obj["API_CLIENT"] + workspace_id = ctx.obj["WORKSPACE_ID"] + destinations = list_resource(Destinations, api_client, workspace_id) + + for destination_id, destination_name in destinations.items(): + definition_id = get_resource_definition_id(Destination, api_client, workspace_id, destination_id) + + generate_source_or_destination("destination", api_client, workspace_id, definition_id, destination_name) + + update_resource(Destination, api_client, workspace_id, destination_id) + + @update.command(cls=OctaviaCommand, name="destination", help="Get YAML for a destination") @click.argument("resource_id", type=click.STRING) @click.pass_context From 48676c13b7d670873860edae26bb9fd48ebae1ce Mon Sep 17 00:00:00 2001 From: danieldiamond Date: Fri, 24 Jun 2022 16:01:15 +1000 Subject: [PATCH 06/16] WIP --- octavia-cli/octavia_cli/entrypoint.py | 4 +-- octavia-cli/octavia_cli/get/commands.py | 2 +- octavia-cli/octavia_cli/get/resources.py | 2 +- .../octavia_cli/import_commands/__init__.py | 3 ++ .../{update => import_commands}/commands.py | 32 +++++++++---------- octavia-cli/octavia_cli/update/__init__.py | 3 -- octavia-cli/unit_tests/test_entrypoint.py | 2 +- .../unit_tests/test_get/test_commands.py | 2 +- .../unit_tests/test_get/test_resources.py | 2 +- 9 files changed, 26 insertions(+), 26 deletions(-) create mode 100644 octavia-cli/octavia_cli/import_commands/__init__.py rename octavia-cli/octavia_cli/{update => import_commands}/commands.py (73%) delete mode 100644 octavia-cli/octavia_cli/update/__init__.py diff --git a/octavia-cli/octavia_cli/entrypoint.py b/octavia-cli/octavia_cli/entrypoint.py index b25241bf1b11..7d9537f28255 100644 --- a/octavia-cli/octavia_cli/entrypoint.py +++ b/octavia-cli/octavia_cli/entrypoint.py @@ -15,15 +15,15 @@ from .check_context import check_api_health, check_is_initialized, check_workspace_exists from .generate import commands as generate_commands from .get import commands as get_commands +from .import_commands import commands as import_commands from .init import commands as init_commands from .list import commands as list_commands from .telemetry import TelemetryClient, build_user_agent -from .update import commands as update_commands AVAILABLE_COMMANDS: List[click.Command] = [ list_commands._list, get_commands.get, - update_commands.update, + import_commands.import_commands, init_commands.init, generate_commands.generate, apply_commands.apply, diff --git a/octavia-cli/octavia_cli/get/commands.py b/octavia-cli/octavia_cli/get/commands.py index 75bacaeec843..e384c97204ed 100644 --- a/octavia-cli/octavia_cli/get/commands.py +++ b/octavia-cli/octavia_cli/get/commands.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # from typing import List diff --git a/octavia-cli/octavia_cli/get/resources.py b/octavia-cli/octavia_cli/get/resources.py index 30f0150c8881..1b2828a8104c 100644 --- a/octavia-cli/octavia_cli/get/resources.py +++ b/octavia-cli/octavia_cli/get/resources.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # import abc diff --git a/octavia-cli/octavia_cli/import_commands/__init__.py b/octavia-cli/octavia_cli/import_commands/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/octavia-cli/octavia_cli/import_commands/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/octavia-cli/octavia_cli/update/commands.py b/octavia-cli/octavia_cli/import_commands/commands.py similarity index 73% rename from octavia-cli/octavia_cli/update/commands.py rename to octavia-cli/octavia_cli/import_commands/commands.py index 9dfec9e4f2bb..d25afa51dfe2 100644 --- a/octavia-cli/octavia_cli/update/commands.py +++ b/octavia-cli/octavia_cli/import_commands/commands.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # from typing import List @@ -13,9 +13,9 @@ from octavia_cli.list.listings import Destinations, Sources -@click.group("update", help="Update configurations in a YAML spec for a source, destination or a connection.") +@click.group("import", help="Import configurations in a YAML spec for a source, destination or a connection.") @click.pass_context -def update(ctx: click.Context): +def import_commands(ctx: click.Context): pass @@ -30,7 +30,7 @@ def get_resource_definition_id(resource_type, api_client, workspace_id, resource return config[f"{resource_type.name}_definition_id"] -def update_resource(resource_type, api_client, workspace_id, resource_id): +def import_resource(resource_type, api_client, workspace_id, resource_id): resource = resource_type(api_client, workspace_id, resource_id) definition_type = resource_type.name config = resource.get_config() @@ -42,18 +42,18 @@ def update_resource(resource_type, api_client, workspace_id, resource_id): renderer = ConnectorSpecificationRenderer(resource_name, definition) output_path = renderer.update_configuration(project_path=".", configuration=config["connection_configuration"]) - message = f"✅ - Updated the {resource_type.name} template for {resource_name} in {output_path}." + message = f"✅ - Imported the {resource_type.name} template for {resource_name} in {output_path}." click.echo(click.style(message, fg="green")) -@update.command(cls=OctaviaCommand, name="source", help="Get YAML for a source") +@import_commands.command(cls=OctaviaCommand, name="source", help="Get YAML for a source") @click.argument("resource_id", type=click.STRING) @click.pass_context def source(ctx: click.Context, resource_id: str): - update_resource(Source, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) + import_resource(Source, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) -@update.command(cls=OctaviaCommand, name="sources", help="Get YAML for a source") +@import_commands.command(cls=OctaviaCommand, name="sources", help="Get YAML for a source") @click.pass_context def sources(ctx: click.Context): api_client = ctx.obj["API_CLIENT"] @@ -65,10 +65,10 @@ def sources(ctx: click.Context): generate_source_or_destination("source", api_client, workspace_id, definition_id, source_name) - update_resource(Source, api_client, workspace_id, source_id) + import_resource(Source, api_client, workspace_id, source_id) -@update.command(cls=OctaviaCommand, name="destinations", help="Get YAML for a destination") +@import_commands.command(cls=OctaviaCommand, name="destinations", help="Get YAML for a destination") @click.pass_context def destinations(ctx: click.Context): api_client = ctx.obj["API_CLIENT"] @@ -80,21 +80,21 @@ def destinations(ctx: click.Context): generate_source_or_destination("destination", api_client, workspace_id, definition_id, destination_name) - update_resource(Destination, api_client, workspace_id, destination_id) + import_resource(Destination, api_client, workspace_id, destination_id) -@update.command(cls=OctaviaCommand, name="destination", help="Get YAML for a destination") +@import_commands.command(cls=OctaviaCommand, name="destination", help="Get YAML for a destination") @click.argument("resource_id", type=click.STRING) @click.pass_context def destination(ctx: click.Context, resource_id: str): - update_resource(Destination, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) + import_resource(Destination, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) -@update.command(cls=OctaviaCommand, name="connection", help="Get YAML for a connection") +@import_commands.command(cls=OctaviaCommand, name="connection", help="Get YAML for a connection") @click.argument("resource_id", type=click.STRING) @click.pass_context def connection(ctx: click.Context, resource_id: str): - update_resource(Connection, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) + import_resource(Connection, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) AVAILABLE_COMMANDS: List[click.Command] = [source, destination, connection] @@ -102,7 +102,7 @@ def connection(ctx: click.Context, resource_id: str): def add_commands_to_list(): for command in AVAILABLE_COMMANDS: - update.add_command(command) + import_commands.add_command(command) add_commands_to_list() diff --git a/octavia-cli/octavia_cli/update/__init__.py b/octavia-cli/octavia_cli/update/__init__.py deleted file mode 100644 index 46b7376756ec..000000000000 --- a/octavia-cli/octavia_cli/update/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/octavia-cli/unit_tests/test_entrypoint.py b/octavia-cli/unit_tests/test_entrypoint.py index fea493e3a431..bd5c020e2da0 100644 --- a/octavia-cli/unit_tests/test_entrypoint.py +++ b/octavia-cli/unit_tests/test_entrypoint.py @@ -217,7 +217,7 @@ def test_available_commands(): assert entrypoint.AVAILABLE_COMMANDS == [ entrypoint.list_commands._list, entrypoint.get_commands.get, - entrypoint.update_commands.update, + entrypoint.import_commands._import, entrypoint.init_commands.init, entrypoint.generate_commands.generate, entrypoint.apply_commands.apply, diff --git a/octavia-cli/unit_tests/test_get/test_commands.py b/octavia-cli/unit_tests/test_get/test_commands.py index fb4ef61c71a6..21d6280bf042 100644 --- a/octavia-cli/unit_tests/test_get/test_commands.py +++ b/octavia-cli/unit_tests/test_get/test_commands.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # import pytest diff --git a/octavia-cli/unit_tests/test_get/test_resources.py b/octavia-cli/unit_tests/test_get/test_resources.py index 84560de41e63..332e8a61214a 100644 --- a/octavia-cli/unit_tests/test_get/test_resources.py +++ b/octavia-cli/unit_tests/test_get/test_resources.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # import pytest From 89549dcce10c69db688122d8770152f478c8a6fd Mon Sep 17 00:00:00 2001 From: danieldiamond Date: Fri, 24 Jun 2022 16:39:31 +1000 Subject: [PATCH 07/16] Revert "WIP" This reverts commit 48676c13b7d670873860edae26bb9fd48ebae1ce. --- octavia-cli/octavia_cli/entrypoint.py | 4 +-- octavia-cli/octavia_cli/get/commands.py | 2 +- octavia-cli/octavia_cli/get/resources.py | 2 +- .../octavia_cli/import_commands/__init__.py | 3 -- octavia-cli/octavia_cli/update/__init__.py | 3 ++ .../{import_commands => update}/commands.py | 32 +++++++++---------- octavia-cli/unit_tests/test_entrypoint.py | 2 +- .../unit_tests/test_get/test_commands.py | 2 +- .../unit_tests/test_get/test_resources.py | 2 +- 9 files changed, 26 insertions(+), 26 deletions(-) delete mode 100644 octavia-cli/octavia_cli/import_commands/__init__.py create mode 100644 octavia-cli/octavia_cli/update/__init__.py rename octavia-cli/octavia_cli/{import_commands => update}/commands.py (73%) diff --git a/octavia-cli/octavia_cli/entrypoint.py b/octavia-cli/octavia_cli/entrypoint.py index 7d9537f28255..b25241bf1b11 100644 --- a/octavia-cli/octavia_cli/entrypoint.py +++ b/octavia-cli/octavia_cli/entrypoint.py @@ -15,15 +15,15 @@ from .check_context import check_api_health, check_is_initialized, check_workspace_exists from .generate import commands as generate_commands from .get import commands as get_commands -from .import_commands import commands as import_commands from .init import commands as init_commands from .list import commands as list_commands from .telemetry import TelemetryClient, build_user_agent +from .update import commands as update_commands AVAILABLE_COMMANDS: List[click.Command] = [ list_commands._list, get_commands.get, - import_commands.import_commands, + update_commands.update, init_commands.init, generate_commands.generate, apply_commands.apply, diff --git a/octavia-cli/octavia_cli/get/commands.py b/octavia-cli/octavia_cli/get/commands.py index e384c97204ed..75bacaeec843 100644 --- a/octavia-cli/octavia_cli/get/commands.py +++ b/octavia-cli/octavia_cli/get/commands.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. # from typing import List diff --git a/octavia-cli/octavia_cli/get/resources.py b/octavia-cli/octavia_cli/get/resources.py index 1b2828a8104c..30f0150c8881 100644 --- a/octavia-cli/octavia_cli/get/resources.py +++ b/octavia-cli/octavia_cli/get/resources.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. # import abc diff --git a/octavia-cli/octavia_cli/import_commands/__init__.py b/octavia-cli/octavia_cli/import_commands/__init__.py deleted file mode 100644 index 1100c1c58cf5..000000000000 --- a/octavia-cli/octavia_cli/import_commands/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# diff --git a/octavia-cli/octavia_cli/update/__init__.py b/octavia-cli/octavia_cli/update/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/octavia-cli/octavia_cli/update/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/octavia-cli/octavia_cli/import_commands/commands.py b/octavia-cli/octavia_cli/update/commands.py similarity index 73% rename from octavia-cli/octavia_cli/import_commands/commands.py rename to octavia-cli/octavia_cli/update/commands.py index d25afa51dfe2..9dfec9e4f2bb 100644 --- a/octavia-cli/octavia_cli/import_commands/commands.py +++ b/octavia-cli/octavia_cli/update/commands.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. # from typing import List @@ -13,9 +13,9 @@ from octavia_cli.list.listings import Destinations, Sources -@click.group("import", help="Import configurations in a YAML spec for a source, destination or a connection.") +@click.group("update", help="Update configurations in a YAML spec for a source, destination or a connection.") @click.pass_context -def import_commands(ctx: click.Context): +def update(ctx: click.Context): pass @@ -30,7 +30,7 @@ def get_resource_definition_id(resource_type, api_client, workspace_id, resource return config[f"{resource_type.name}_definition_id"] -def import_resource(resource_type, api_client, workspace_id, resource_id): +def update_resource(resource_type, api_client, workspace_id, resource_id): resource = resource_type(api_client, workspace_id, resource_id) definition_type = resource_type.name config = resource.get_config() @@ -42,18 +42,18 @@ def import_resource(resource_type, api_client, workspace_id, resource_id): renderer = ConnectorSpecificationRenderer(resource_name, definition) output_path = renderer.update_configuration(project_path=".", configuration=config["connection_configuration"]) - message = f"✅ - Imported the {resource_type.name} template for {resource_name} in {output_path}." + message = f"✅ - Updated the {resource_type.name} template for {resource_name} in {output_path}." click.echo(click.style(message, fg="green")) -@import_commands.command(cls=OctaviaCommand, name="source", help="Get YAML for a source") +@update.command(cls=OctaviaCommand, name="source", help="Get YAML for a source") @click.argument("resource_id", type=click.STRING) @click.pass_context def source(ctx: click.Context, resource_id: str): - import_resource(Source, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) + update_resource(Source, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) -@import_commands.command(cls=OctaviaCommand, name="sources", help="Get YAML for a source") +@update.command(cls=OctaviaCommand, name="sources", help="Get YAML for a source") @click.pass_context def sources(ctx: click.Context): api_client = ctx.obj["API_CLIENT"] @@ -65,10 +65,10 @@ def sources(ctx: click.Context): generate_source_or_destination("source", api_client, workspace_id, definition_id, source_name) - import_resource(Source, api_client, workspace_id, source_id) + update_resource(Source, api_client, workspace_id, source_id) -@import_commands.command(cls=OctaviaCommand, name="destinations", help="Get YAML for a destination") +@update.command(cls=OctaviaCommand, name="destinations", help="Get YAML for a destination") @click.pass_context def destinations(ctx: click.Context): api_client = ctx.obj["API_CLIENT"] @@ -80,21 +80,21 @@ def destinations(ctx: click.Context): generate_source_or_destination("destination", api_client, workspace_id, definition_id, destination_name) - import_resource(Destination, api_client, workspace_id, destination_id) + update_resource(Destination, api_client, workspace_id, destination_id) -@import_commands.command(cls=OctaviaCommand, name="destination", help="Get YAML for a destination") +@update.command(cls=OctaviaCommand, name="destination", help="Get YAML for a destination") @click.argument("resource_id", type=click.STRING) @click.pass_context def destination(ctx: click.Context, resource_id: str): - import_resource(Destination, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) + update_resource(Destination, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) -@import_commands.command(cls=OctaviaCommand, name="connection", help="Get YAML for a connection") +@update.command(cls=OctaviaCommand, name="connection", help="Get YAML for a connection") @click.argument("resource_id", type=click.STRING) @click.pass_context def connection(ctx: click.Context, resource_id: str): - import_resource(Connection, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) + update_resource(Connection, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) AVAILABLE_COMMANDS: List[click.Command] = [source, destination, connection] @@ -102,7 +102,7 @@ def connection(ctx: click.Context, resource_id: str): def add_commands_to_list(): for command in AVAILABLE_COMMANDS: - import_commands.add_command(command) + update.add_command(command) add_commands_to_list() diff --git a/octavia-cli/unit_tests/test_entrypoint.py b/octavia-cli/unit_tests/test_entrypoint.py index bd5c020e2da0..fea493e3a431 100644 --- a/octavia-cli/unit_tests/test_entrypoint.py +++ b/octavia-cli/unit_tests/test_entrypoint.py @@ -217,7 +217,7 @@ def test_available_commands(): assert entrypoint.AVAILABLE_COMMANDS == [ entrypoint.list_commands._list, entrypoint.get_commands.get, - entrypoint.import_commands._import, + entrypoint.update_commands.update, entrypoint.init_commands.init, entrypoint.generate_commands.generate, entrypoint.apply_commands.apply, diff --git a/octavia-cli/unit_tests/test_get/test_commands.py b/octavia-cli/unit_tests/test_get/test_commands.py index 21d6280bf042..fb4ef61c71a6 100644 --- a/octavia-cli/unit_tests/test_get/test_commands.py +++ b/octavia-cli/unit_tests/test_get/test_commands.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. # import pytest diff --git a/octavia-cli/unit_tests/test_get/test_resources.py b/octavia-cli/unit_tests/test_get/test_resources.py index 332e8a61214a..84560de41e63 100644 --- a/octavia-cli/unit_tests/test_get/test_resources.py +++ b/octavia-cli/unit_tests/test_get/test_resources.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. # import pytest From 87f6b48b43445f420d0c21de74ffaa707bdcb24c Mon Sep 17 00:00:00 2001 From: danieldiamond Date: Fri, 24 Jun 2022 16:41:38 +1000 Subject: [PATCH 08/16] Remove update method --- octavia-cli/README.md | 150 ++++++++------------- octavia-cli/octavia_cli/entrypoint.py | 2 - octavia-cli/octavia_cli/update/__init__.py | 3 - octavia-cli/octavia_cli/update/commands.py | 108 --------------- 4 files changed, 54 insertions(+), 209 deletions(-) delete mode 100644 octavia-cli/octavia_cli/update/__init__.py delete mode 100644 octavia-cli/octavia_cli/update/commands.py diff --git a/octavia-cli/README.md b/octavia-cli/README.md index 22d848028a8b..c848d9428191 100644 --- a/octavia-cli/README.md +++ b/octavia-cli/README.md @@ -5,7 +5,6 @@ The project is in **alpha** version. Readers can refer to our [opened GitHub issues](https://github.com/airbytehq/airbyte/issues?q=is%3Aopen+is%3Aissue+label%3Aarea%2Foctavia-cli) to check the ongoing work on this project. - ## What is `octavia` CLI? Octavia CLI is a tool to manage Airbyte configurations in YAML. @@ -44,7 +43,7 @@ Feel free to share your use cases with the community in [#octavia-cli](https://a ### 1. Generate local YAML files for sources or destinations -1. Retrieve the *definition id* of the connector you want to use using `octavia list command`. +1. Retrieve the _definition id_ of the connector you want to use using `octavia list command`. 2. Generate YAML configuration running `octavia generate source ` or `octavia generate destination `. ### 2. Edit your local YAML configurations @@ -67,7 +66,7 @@ Feel free to share your use cases with the community in [#octavia-cli](https://a ### 6. Update your configurations -Changes in your local configurations can be propagated to your Airbyte instance using `octavia apply`. You will be prompted for validation of changes. You can bypass the validation step using the `--force` flag. +Changes in your local configurations can be propagated to your Airbyte instance using `octavia apply`. You will be prompted for validation of changes. You can bypass the validation step using the `--force` flag. ## Secret management @@ -79,7 +78,7 @@ configuration: password: ${MY_PASSWORD} ``` -If you have set a `MY_PASSWORD` environment variable, `octavia apply` will load its value into the `password` field. +If you have set a `MY_PASSWORD` environment variable, `octavia apply` will load its value into the `password` field. ## Install @@ -138,27 +137,32 @@ docker-compose run octavia-cli ` ### `octavia` command flags -| **Flag** | **Description** | **Env Variable** | **Default** | -|--------------------------------------------|-----------------------------------------------------------------------------------|------------------------------|--------------------------------------------------------| -| `--airbyte-url` | Airbyte instance URL. | `AIRBYTE_URL` | `http://localhost:8000` | -| `--workspace-id` | Airbyte workspace id. | `AIRBYTE_WORKSPACE_ID` | The first workspace id found on your Airbyte instance. | -| `--enable-telemetry/--disable-telemetry` | Enable or disable the sending of telemetry data. | `OCTAVIA_ENABLE_TELEMETRY` | True | -| `--api-http-header` | HTTP Header value pairs passed while calling Airbyte's API | not supported. | None | None | -| `--api-http-headers-file-path` | Path to the YAML file that contains custom HTTP Headers to send to Airbyte's API. | None | None | +| **Flag** | **Description** | **Env Variable** | **Default** | +| ---------------------------------------- | --------------------------------------------------------------------------------- | -------------------------- | ------------------------------------------------------ | ---- | +| `--airbyte-url` | Airbyte instance URL. | `AIRBYTE_URL` | `http://localhost:8000` | +| `--workspace-id` | Airbyte workspace id. | `AIRBYTE_WORKSPACE_ID` | The first workspace id found on your Airbyte instance. | +| `--enable-telemetry/--disable-telemetry` | Enable or disable the sending of telemetry data. | `OCTAVIA_ENABLE_TELEMETRY` | True | +| `--api-http-header` | HTTP Header value pairs passed while calling Airbyte's API | not supported. | None | None | +| `--api-http-headers-file-path` | Path to the YAML file that contains custom HTTP Headers to send to Airbyte's API. | None | None | #### Using custom HTTP headers -You can set custom HTTP headers to send to Airbyte's API with options: + +You can set custom HTTP headers to send to Airbyte's API with options: + ```bash octavia --api-http-header Header-Name Header-Value --api-http-header Header-Name-2 Header-Value-2 list connectors sources ``` You can also use a custom YAML file (one is already created on init in `api_http_headers.yaml`) to declare the HTTP headers to send to the API: + ```yaml headers: Authorization: Basic foobar== User-Agent: octavia-cli/0.0.0 ``` + Environment variable expansion is available in this Yaml file + ```yaml headers: Authorization: Bearer ${MY_API_TOKEN} @@ -168,20 +172,17 @@ headers: ### `octavia` subcommands -| **Command** | **Usage** | -|-----------------------------------------|-------------------------------------------------------------------------------------| +| **Command** | **Usage** | +| ----------------------------------------- | --------------------------------------------------------------------------------- | | **`octavia init`** | Initialize required directories for the project. | | **`octavia list connectors sources`** | List all sources connectors available on the remote Airbyte instance. | | **`octavia list connectors destination`** | List all destinations connectors available on the remote Airbyte instance. | | **`octavia list workspace sources`** | List existing sources in current the Airbyte workspace. | | **`octavia list workspace destinations`** | List existing destinations in the current Airbyte workspace. | | **`octavia list workspace connections`** | List existing connections in the current Airbyte workspace. | -| **`octavia get workspace source`** | Get an existing source in current the Airbyte workspace. | -| **`octavia get workspace destination`** | Get an existing destination in the current Airbyte workspace. | -| **`octavia get workspace connection`** | Get an existing connection in the current Airbyte workspace. | -| **`octavia update workspace source`** | Update a local YAML configuration of an existing source in current the Airbyte workspace. | -| **`octavia update workspace destination`** | Update a local YAML configuration of an existing destination in the current Airbyte workspace. | -| **`octavia update workspace connection`** | Update a local YAML configuration of an existing connection in the current Airbyte workspace. | +| **`octavia get workspace source`** | Get an existing source in current the Airbyte workspace. | +| **`octavia get workspace destination`** | Get an existing destination in the current Airbyte workspace. | +| **`octavia get workspace connection`** | Get an existing connection in the current Airbyte workspace. | | **`octavia generate source`** | Generate a local YAML configuration for a new source. | | **`octavia generate destination`** | Generate a local YAML configuration for a new destination. | | **`octavia generate connection`** | Generate a local YAML configuration for a new connection. | @@ -274,9 +275,9 @@ weather_to_pg a4491317-153e-436f-b646-0b39338f9aab active c4aa8550-2122-4a33- Get an existing source in current the Airbyte workspace -| **Argument** | **Description** | -|-----------------|-----------------------------------------------------------------------------------------------| -| `SOURCE_ID` | The source connector id. Can be retrieved using `octavia list workspace sources`. | +| **Argument** | **Description** | +| ------------ | --------------------------------------------------------------------------------- | +| `SOURCE_ID` | The source connector id. Can be retrieved using `octavia list workspace sources`. | **Example**: @@ -296,8 +297,8 @@ $ octavia get source c0c977c2-48e7-46fe-9f57-576285c26d42 Get an existing destination in current the Airbyte workspace -| **Argument** | **Description** | -|-----------------|-----------------------------------------------------------------------------------------------| +| **Argument** | **Description** | +| ---------------- | ------------------------------------------------------------------------------------------- | | `DESTINATION_ID` | The destination connector id. Can be retrieved using `octavia list workspace destinations`. | **Example**: @@ -320,8 +321,8 @@ $ octavia get destination c0c977c2-48e7-46fe-9f57-576285c26d42 Get an existing connection in current the Airbyte workspace -| **Argument** | **Description** | -|-----------------|-----------------------------------------------------------------------------------------------| +| **Argument** | **Description** | +| --------------- | ----------------------------------------------------------------------------------------- | | `CONNECTION_ID` | The connection connector id. Can be retrieved using `octavia list workspace connections`. | **Example**: @@ -391,58 +392,13 @@ $ octavia get connection c0c977c2-48e7-46fe-9f57-576285c26d42 } ``` -#### `octavia update source ` - -Update a local YAML configuration of an existing source in current the Airbyte workspace. - -| **Argument** | **Description** | -|-----------------|-----------------------------------------------------------------------------------------------| -| `SOURCE_ID` | The source connector id. Can be retrieved using `octavia list workspace sources`. | - -**Example**: - -```bash -$ octavia update source c0c977c2-48e7-46fe-9f57-576285c26d42 -✅ - Updated the source template for weather_to_pg in ./source/weather_to_pg/configuration.yaml. -``` - -#### `octavia update destination ` - -Update a local YAML configuration of an existing destination in current the Airbyte workspace. - -| **Argument** | **Description** | -|-----------------|-----------------------------------------------------------------------------------------------| -| `DESTINATION_ID` | The destination connector id. Can be retrieved using `octavia list workspace destinations`. | - -**Example**: - -```bash -$ octavia update destination c0c977c2-48e7-46fe-9f57-576285c26d42 -✅ - Updated the destination template for weather_to_pg in ./destination/weather_to_pg/configuration.yaml. -``` - -#### `octavia update conection ` - -Update a local YAML configuration of an existing connection in current the Airbyte workspace. - -| **Argument** | **Description** | -|-----------------|-----------------------------------------------------------------------------------------------| -| `CONNECTION_ID` | The connection connector id. Can be retrieved using `octavia list workspace connections`. | - -**Example**: - -```bash -$ octavia update connection c0c977c2-48e7-46fe-9f57-576285c26d42 -✅ - Updated the connection template for weather_to_pg in ./connection/weather_to_pg/configuration.yaml. -``` - #### `octavia generate source ` Generate a YAML configuration for a source. The YAML file will be stored at `./sources//configuration.yaml`. -| **Argument** | **Description** | -|-----------------|-----------------------------------------------------------------------------------------------| +| **Argument** | **Description** | +| --------------- | --------------------------------------------------------------------------------------------- | | `DEFINITION_ID` | The source connector definition id. Can be retrieved using `octavia list connectors sources`. | | `SOURCE_NAME` | The name you want to give to this source in Airbyte. | @@ -459,7 +415,7 @@ Generate a YAML configuration for a destination. The YAML file will be stored at `./destinations//configuration.yaml`. | **Argument** | **Description** | -|--------------------|---------------------------------------------------------------------------------------------------------| +| ------------------ | ------------------------------------------------------------------------------------------------------- | | `DEFINITION_ID` | The destination connector definition id. Can be retrieved using `octavia list connectors destinations`. | | `DESTINATION_NAME` | The name you want to give to this destination in Airbyte. | @@ -475,13 +431,13 @@ $ octavia generate destination 25c5221d-dce2-4163-ade9-739ef790f503 my_db Generate a YAML configuration for a connection. The YAML file will be stored at `./connections//configuration.yaml`. -| **Option** | **Required** | **Description** | -|-------------------|--------------|--------------------------------------------------------------------------------------------| -| `--source` | Yes | Path to the YAML configuration file of the source you want to create a connection from. | -| `--destination` | Yes | Path to the YAML configuration file of the destination you want to create a connection to. | +| **Option** | **Required** | **Description** | +| --------------- | ------------ | ------------------------------------------------------------------------------------------ | +| `--source` | Yes | Path to the YAML configuration file of the source you want to create a connection from. | +| `--destination` | Yes | Path to the YAML configuration file of the destination you want to create a connection to. | | **Argument** | **Description** | -|-------------------|----------------------------------------------------------| +| ----------------- | -------------------------------------------------------- | | `CONNECTION_NAME` | The name you want to give to this connection in Airbyte. | **Example**: @@ -498,10 +454,10 @@ If the resource was not found on your Airbyte instance, **apply** will **create* If the resource was found on your Airbyte instance, **apply** will prompt you for validation of the changes and will run an **update** of your resource. Please note that if a secret field was updated on your configuration, **apply** will run this change without prompt. -| **Option** | **Required** | **Description** | -|-----------------|--------------|--------------------------------------------------------------------------------------------| -| `--file` | No | Path to the YAML configuration files you want to create or update. | -| `--force` | No | Run update without prompting for changes validation. | +| **Option** | **Required** | **Description** | +| ---------- | ------------ | ------------------------------------------------------------------ | +| `--file` | No | Path to the YAML configuration files you want to create or update. | +| `--force` | No | Run update without prompting for changes validation. | **Example**: @@ -545,23 +501,25 @@ $ octavia apply 7. Make sure the build passes (step 0) before opening a PR. ## Telemetry + This CLI has some telemetry tooling to send Airbyte some data about the usage of this tool. We will use this data to improve the CLI and measure its adoption. The telemetry sends data about: -* Which command was run (not the arguments or options used). -* Success or failure of the command run and the error type (not the error payload). -* The current Airbyte workspace id if the user has not set the *anonymous data collection* on their Airbyte instance. + +- Which command was run (not the arguments or options used). +- Success or failure of the command run and the error type (not the error payload). +- The current Airbyte workspace id if the user has not set the _anonymous data collection_ on their Airbyte instance. You can disable telemetry by setting the `OCTAVIA_ENABLE_TELEMETRY` environment variable to `False` or using the `--disable-telemetry` flag. ## Changelog -| Version | Date | Description | PR | -|----------|------------|----------------------------------------------------|----------------------------------------------------------| -| 0.39.19 | 2022-06-16 | Allow connection management on multiple workspaces | [#12727](https://github.com/airbytehq/airbyte/pull/12727)| -| 0.39.19 | 2022-06-15 | Allow users to set custom HTTP headers | [#12893](https://github.com/airbytehq/airbyte/pull/12893) | -| 0.39.14 | 2022-05-12 | Enable normalization on connection | [#12727](https://github.com/airbytehq/airbyte/pull/12727)| -| 0.37.0 | 2022-05-05 | Use snake case in connection fields | [#12133](https://github.com/airbytehq/airbyte/pull/12133)| -| 0.35.68 | 2022-04-15 | Improve telemetry | [#12072](https://github.com/airbytehq/airbyte/issues/11896)| -| 0.35.68 | 2022-04-12 | Add telemetry | [#11896](https://github.com/airbytehq/airbyte/issues/11896)| -| 0.35.61 | 2022-04-07 | Alpha release | [EPIC](https://github.com/airbytehq/airbyte/issues/10704)| +| Version | Date | Description | PR | +| ------- | ---------- | -------------------------------------------------- | ----------------------------------------------------------- | +| 0.39.19 | 2022-06-16 | Allow connection management on multiple workspaces | [#12727](https://github.com/airbytehq/airbyte/pull/12727) | +| 0.39.19 | 2022-06-15 | Allow users to set custom HTTP headers | [#12893](https://github.com/airbytehq/airbyte/pull/12893) | +| 0.39.14 | 2022-05-12 | Enable normalization on connection | [#12727](https://github.com/airbytehq/airbyte/pull/12727) | +| 0.37.0 | 2022-05-05 | Use snake case in connection fields | [#12133](https://github.com/airbytehq/airbyte/pull/12133) | +| 0.35.68 | 2022-04-15 | Improve telemetry | [#12072](https://github.com/airbytehq/airbyte/issues/11896) | +| 0.35.68 | 2022-04-12 | Add telemetry | [#11896](https://github.com/airbytehq/airbyte/issues/11896) | +| 0.35.61 | 2022-04-07 | Alpha release | [EPIC](https://github.com/airbytehq/airbyte/issues/10704) | diff --git a/octavia-cli/octavia_cli/entrypoint.py b/octavia-cli/octavia_cli/entrypoint.py index b25241bf1b11..e42846cb243b 100644 --- a/octavia-cli/octavia_cli/entrypoint.py +++ b/octavia-cli/octavia_cli/entrypoint.py @@ -18,12 +18,10 @@ from .init import commands as init_commands from .list import commands as list_commands from .telemetry import TelemetryClient, build_user_agent -from .update import commands as update_commands AVAILABLE_COMMANDS: List[click.Command] = [ list_commands._list, get_commands.get, - update_commands.update, init_commands.init, generate_commands.generate, apply_commands.apply, diff --git a/octavia-cli/octavia_cli/update/__init__.py b/octavia-cli/octavia_cli/update/__init__.py deleted file mode 100644 index 46b7376756ec..000000000000 --- a/octavia-cli/octavia_cli/update/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/octavia-cli/octavia_cli/update/commands.py b/octavia-cli/octavia_cli/update/commands.py deleted file mode 100644 index 9dfec9e4f2bb..000000000000 --- a/octavia-cli/octavia_cli/update/commands.py +++ /dev/null @@ -1,108 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# - -from typing import List - -import click -from octavia_cli.base_commands import OctaviaCommand -from octavia_cli.generate import definitions -from octavia_cli.generate.commands import generate_source_or_destination -from octavia_cli.generate.renderers import ConnectorSpecificationRenderer -from octavia_cli.get.resources import Connection, Destination, Source -from octavia_cli.list.listings import Destinations, Sources - - -@click.group("update", help="Update configurations in a YAML spec for a source, destination or a connection.") -@click.pass_context -def update(ctx: click.Context): - pass - - -def list_resource(resource_type, api_client, workspace_id): - resources = resource_type(api_client, workspace_id) - return {resource_id: resource_name for resource_name, _, resource_id in resources.get_listing()} - - -def get_resource_definition_id(resource_type, api_client, workspace_id, resource_id): - resource = resource_type(api_client, workspace_id, resource_id) - config = resource.get_config() - return config[f"{resource_type.name}_definition_id"] - - -def update_resource(resource_type, api_client, workspace_id, resource_id): - resource = resource_type(api_client, workspace_id, resource_id) - definition_type = resource_type.name - config = resource.get_config() - definition_id = config[f"{resource_type.name}_definition_id"] - # Schema varies across resources, e.g. pull "name" else default to "source_name" - resource_name = config.get("name") or config[f"{resource_type.name}_name"] - - definition = definitions.factory(definition_type, api_client, workspace_id, definition_id) - renderer = ConnectorSpecificationRenderer(resource_name, definition) - - output_path = renderer.update_configuration(project_path=".", configuration=config["connection_configuration"]) - message = f"✅ - Updated the {resource_type.name} template for {resource_name} in {output_path}." - click.echo(click.style(message, fg="green")) - - -@update.command(cls=OctaviaCommand, name="source", help="Get YAML for a source") -@click.argument("resource_id", type=click.STRING) -@click.pass_context -def source(ctx: click.Context, resource_id: str): - update_resource(Source, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) - - -@update.command(cls=OctaviaCommand, name="sources", help="Get YAML for a source") -@click.pass_context -def sources(ctx: click.Context): - api_client = ctx.obj["API_CLIENT"] - workspace_id = ctx.obj["WORKSPACE_ID"] - sources = list_resource(Sources, api_client, workspace_id) - - for source_id, source_name in sources.items(): - definition_id = get_resource_definition_id(Source, api_client, workspace_id, source_id) - - generate_source_or_destination("source", api_client, workspace_id, definition_id, source_name) - - update_resource(Source, api_client, workspace_id, source_id) - - -@update.command(cls=OctaviaCommand, name="destinations", help="Get YAML for a destination") -@click.pass_context -def destinations(ctx: click.Context): - api_client = ctx.obj["API_CLIENT"] - workspace_id = ctx.obj["WORKSPACE_ID"] - destinations = list_resource(Destinations, api_client, workspace_id) - - for destination_id, destination_name in destinations.items(): - definition_id = get_resource_definition_id(Destination, api_client, workspace_id, destination_id) - - generate_source_or_destination("destination", api_client, workspace_id, definition_id, destination_name) - - update_resource(Destination, api_client, workspace_id, destination_id) - - -@update.command(cls=OctaviaCommand, name="destination", help="Get YAML for a destination") -@click.argument("resource_id", type=click.STRING) -@click.pass_context -def destination(ctx: click.Context, resource_id: str): - update_resource(Destination, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) - - -@update.command(cls=OctaviaCommand, name="connection", help="Get YAML for a connection") -@click.argument("resource_id", type=click.STRING) -@click.pass_context -def connection(ctx: click.Context, resource_id: str): - update_resource(Connection, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) - - -AVAILABLE_COMMANDS: List[click.Command] = [source, destination, connection] - - -def add_commands_to_list(): - for command in AVAILABLE_COMMANDS: - update.add_command(command) - - -add_commands_to_list() From 29c3a64225b0f542a0b90229e886f5c421dee375 Mon Sep 17 00:00:00 2001 From: danieldiamond Date: Fri, 24 Jun 2022 16:44:15 +1000 Subject: [PATCH 09/16] Remove update from tests --- octavia-cli/unit_tests/test_entrypoint.py | 1 - 1 file changed, 1 deletion(-) diff --git a/octavia-cli/unit_tests/test_entrypoint.py b/octavia-cli/unit_tests/test_entrypoint.py index fea493e3a431..0e4a9b7cea33 100644 --- a/octavia-cli/unit_tests/test_entrypoint.py +++ b/octavia-cli/unit_tests/test_entrypoint.py @@ -217,7 +217,6 @@ def test_available_commands(): assert entrypoint.AVAILABLE_COMMANDS == [ entrypoint.list_commands._list, entrypoint.get_commands.get, - entrypoint.update_commands.update, entrypoint.init_commands.init, entrypoint.generate_commands.generate, entrypoint.apply_commands.apply, From 35d9cdd71bbe0081e10d154938d57c670c4fb7a0 Mon Sep 17 00:00:00 2001 From: danieldiamond Date: Fri, 24 Jun 2022 16:47:29 +1000 Subject: [PATCH 10/16] Remove update related methods in generate_renderers --- octavia-cli/octavia_cli/generate/renderers.py | 33 ------------------- 1 file changed, 33 deletions(-) diff --git a/octavia-cli/octavia_cli/generate/renderers.py b/octavia-cli/octavia_cli/generate/renderers.py index 7e51c26ce339..52e31ab05c96 100644 --- a/octavia-cli/octavia_cli/generate/renderers.py +++ b/octavia-cli/octavia_cli/generate/renderers.py @@ -3,7 +3,6 @@ # import abc -import collections.abc import os from typing import Any, Callable, List @@ -18,15 +17,6 @@ JINJA_ENV = Environment(loader=PackageLoader(__package__), autoescape=select_autoescape(), trim_blocks=False, lstrip_blocks=True) -def update_nested_dict(d, u): - for k, v in u.items(): - if isinstance(v, collections.abc.Mapping): - d[k] = update_nested_dict(d.get(k, {}), v) - else: - d[k] = v - return d - - class FieldToRender: def __init__(self, name: str, required: bool, field_metadata: dict) -> None: """Initialize a FieldToRender instance @@ -197,29 +187,6 @@ def write_yaml(self, project_path: str) -> str: f.write(rendered) return output_path - def update_configuration(self, project_path: str, configuration: dict) -> str: - """Update configuration from the YAML file in local project path. - Resource must be generated prior to updating. - - Args: - project_path (str): Path to directory hosting the octavia project. - configuration (dict): Configuraton of the resource. - - Returns: - str: Path to the rendered specification. - """ - output_path = self._get_output_path(project_path, self.definition.type) - - with open(output_path) as f: - data = yaml.safe_load(f) - - data["configuration"] = update_nested_dict(data["configuration"], configuration) - - with open(output_path, "wb") as f: - yaml.safe_dump(data, f, default_flow_style=False, sort_keys=False, allow_unicode=True, encoding="utf-8") - - return output_path - class ConnectorSpecificationRenderer(BaseRenderer): TEMPLATE = JINJA_ENV.get_template("source_or_destination.yaml.j2") From 528afc164db6a65790999cb22bf77f787e17253a Mon Sep 17 00:00:00 2001 From: danieldiamond Date: Sun, 26 Jun 2022 13:23:34 +1000 Subject: [PATCH 11/16] Reformat get commands --- octavia-cli/octavia_cli/get/__init__.py | 2 +- octavia-cli/octavia_cli/get/commands.py | 95 +++++++++++--- octavia-cli/octavia_cli/get/resources.py | 153 ++++++++++++++++++++--- 3 files changed, 210 insertions(+), 40 deletions(-) diff --git a/octavia-cli/octavia_cli/get/__init__.py b/octavia-cli/octavia_cli/get/__init__.py index 46b7376756ec..1100c1c58cf5 100644 --- a/octavia-cli/octavia_cli/get/__init__.py +++ b/octavia-cli/octavia_cli/get/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # diff --git a/octavia-cli/octavia_cli/get/commands.py b/octavia-cli/octavia_cli/get/commands.py index 75bacaeec843..fea5c6d96377 100644 --- a/octavia-cli/octavia_cli/get/commands.py +++ b/octavia-cli/octavia_cli/get/commands.py @@ -1,43 +1,100 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from typing import List +import uuid +from typing import List, Optional, Tuple, Type, Union +import airbyte_api_client import click from octavia_cli.base_commands import OctaviaCommand from .resources import Connection, Destination, Source +COMMON_HELP_MESSAGE_PREFIX = "Get a JSON representation of a remote" -@click.group("get", help="Get a YAML spec for a source, destination or a connection.") + +def build_help_message(resource_type: str) -> str: + """Helper function to build help message consistently for all the commands in this module. + + Args: + resource_type (str): source, destination or connection + + Returns: + str: The generated help message. + """ + return f"Get a JSON representation of a remote {resource_type}." + + +def get_resource_id_or_name(resource: str) -> Tuple[Optional[str], Optional[str]]: + """Helper function to detect if the resource argument passed to the CLI is a resource ID or name. + + Args: + resource (str): the resource ID or name passed as an argument to the CLI. + + Returns: + Tuple[Optional[str], Optional[str]]: the resource_id and resource_name, the not detected kind is set to None. + """ + resource_id, resource_name = None, None + try: + uuid.UUID(resource) + resource_id = resource + except ValueError: + resource_name = resource + return resource_id, resource_name + + +def get_json_representation( + api_client: airbyte_api_client.ApiClient, + workspace_id: str, + ResourceCls: Type[Union[Source, Destination, Connection]], + resource_to_get: str, +) -> str: + """Helper function to retrieve a resource json representation and avoid repeating the same logic for Source/Destination and connection. + + + Args: + api_client (airbyte_api_client.ApiClient): The Airbyte API client. + workspace_id (str): Current workspace id. + ResourceCls (Type[Union[Source, Destination, Connection]]): Resource class to use + resource_to_get (str): resource name or id to get JSON representation for. + + Returns: + str: The resource's JSON representation. + """ + resource_id, resource_name = get_resource_id_or_name(resource_to_get) + resource = ResourceCls(api_client, workspace_id, resource_id=resource_id, resource_name=resource_name) + return resource.to_json() + + +@click.group( + "get", + help=f'{build_help_message("source, destination or connection")} ID or name can be used as argument. Example: \'octavia get source "My Pokemon source"\' or \'octavia get source cb5413b2-4159-46a2-910a-dc282a439d2d\'', +) @click.pass_context -def get(ctx: click.Context): +def get(ctx: click.Context): # pragma: no cover pass -@get.command(cls=OctaviaCommand, name="source", help="Get YAML for a source") -@click.argument("resource_id", type=click.STRING) +@get.command(cls=OctaviaCommand, name="source", help=build_help_message("source")) +@click.argument("resource", type=click.STRING) @click.pass_context -def source(ctx: click.Context, resource_id: str): - resource = Source(ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) - click.echo(resource.get_config()) +def source(ctx: click.Context, resource: str): + click.echo(get_json_representation(ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], Source, resource)) -@get.command(cls=OctaviaCommand, name="destination", help="Get YAML for a destination") -@click.argument("resource_id", type=click.STRING) +@get.command(cls=OctaviaCommand, name="destination", help=build_help_message("destination")) +@click.argument("resource", type=click.STRING) @click.pass_context -def destination(ctx: click.Context, resource_id: str): - resource = Destination(ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) - click.echo(resource.get_config()) +def destination(ctx: click.Context, resource: str): + click.echo(get_json_representation(ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], Destination, resource)) -@get.command(cls=OctaviaCommand, name="connection", help="Get YAML for a connection") -@click.argument("resource_id", type=click.STRING) +@get.command(cls=OctaviaCommand, name="connection", help=build_help_message("connection")) +@click.argument("resource", type=click.STRING) @click.pass_context -def connection(ctx: click.Context, resource_id: str): - resource = Connection(ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], resource_id) - click.echo(resource.get_config()) +def connection(ctx: click.Context, resource: str): + click.echo(get_json_representation(ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], Connection, resource)) AVAILABLE_COMMANDS: List[click.Command] = [source, destination, connection] diff --git a/octavia-cli/octavia_cli/get/resources.py b/octavia-cli/octavia_cli/get/resources.py index 30f0150c8881..aef89cda00c0 100644 --- a/octavia-cli/octavia_cli/get/resources.py +++ b/octavia-cli/octavia_cli/get/resources.py @@ -1,19 +1,28 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # import abc -from typing import Dict +import json +from typing import Optional, Union import airbyte_api_client import click -from airbyte_api_client.api import connection_api, destination_api, source_api -from airbyte_api_client.model.connection_id_request_body import ConnectionIdRequestBody +from airbyte_api_client.api import destination_api, source_api, web_backend_api from airbyte_api_client.model.destination_id_request_body import DestinationIdRequestBody +from airbyte_api_client.model.destination_read import DestinationRead from airbyte_api_client.model.source_id_request_body import SourceIdRequestBody +from airbyte_api_client.model.source_read import SourceRead +from airbyte_api_client.model.web_backend_connection_read import WebBackendConnectionRead +from airbyte_api_client.model.web_backend_connection_request_body import WebBackendConnectionRequestBody +from airbyte_api_client.model.workspace_id_request_body import WorkspaceIdRequestBody -class DefinitionNotFoundError(click.ClickException): +class DuplicateResourceError(click.ClickException): + pass + + +class ResourceNotFoundError(click.ClickException): pass @@ -25,6 +34,13 @@ def api( ): # pragma: no cover pass + @property + @abc.abstractmethod + def name( + self, + ) -> str: # pragma: no cover + pass + @property @abc.abstractmethod def get_function_name( @@ -37,44 +53,141 @@ def _get_fn(self): return getattr(self.api, self.get_function_name) @property - def get_function_kwargs(self) -> dict: - return {} + @abc.abstractmethod + def get_payload( + self, + ): # pragma: no cover + pass + + @property + @abc.abstractmethod + def list_for_workspace_function_name( + self, + ) -> str: # pragma: no cover + pass + + @property + def _list_for_workspace_fn(self): + return getattr(self.api, self.list_for_workspace_function_name) - def __init__(self, api_client: airbyte_api_client.ApiClient, workspace_id: str, resource_id: str): + @property + def list_for_workspace_payload( + self, + ): + return WorkspaceIdRequestBody(workspace_id=self.workspace_id) + + def __init__( + self, + api_client: airbyte_api_client.ApiClient, + workspace_id: str, + resource_id: Optional[str] = None, + resource_name: Optional[str] = None, + ): + if resource_id is None and resource_name is None: + raise ValueError("resource_id and resource_name keyword arguments can't be both None.") + if resource_id is not None and resource_name is not None: + raise ValueError("resource_id and resource_name keyword arguments can't be both set.") + self.resource_id = resource_id + self.resource_name = resource_name self.api_instance = self.api(api_client) self.workspace_id = workspace_id - self.resource_id = resource_id - def get_config(self) -> Dict: - api_response = self._get_fn(self.api_instance, **self.get_function_kwargs) - return api_response + def _find_by_resource_name( + self, + ) -> Union[WebBackendConnectionRead, SourceRead, DestinationRead]: + """Retrieve a remote resource from its name by listing the available resources on the Airbyte instance. + + Raises: + ResourceNotFoundError: Raised if no resource was found with the current resource_name. + DuplicateResourceError: Raised if multiple resources were found with the current resource_name. + + Returns: + Union[WebBackendConnectionRead, SourceRead, DestinationRead]: The remote resource model instance. + """ + + api_response = self._list_for_workspace_fn(self.api_instance, self.list_for_workspace_payload) + matching_resources = [] + for resource in getattr(api_response, f"{self.name}s"): + if resource.name == self.resource_name: + matching_resources.append(resource) + if not matching_resources: + raise ResourceNotFoundError(f"The {self.name} {self.resource_name} was not found in your current Airbyte workspace.") + if len(matching_resources) > 1: + raise DuplicateResourceError( + f"{len(matching_resources)} {self.name}s with the name {self.resource_name} were found in your current Airbyte workspace." + ) + return matching_resources[0] + + def _find_by_resource_id( + self, + ) -> Union[WebBackendConnectionRead, SourceRead, DestinationRead]: + """Retrieve a remote resource from its id by calling the get endpoint of the resource type. + + Returns: + Union[WebBackendConnectionRead, SourceRead, DestinationRead]: The remote resource model instance. + """ + return self._get_fn(self.api_instance, self.get_payload) + + def get_remote_resource(self) -> Union[WebBackendConnectionRead, SourceRead, DestinationRead]: + """Retrieve a remote resource with a resource_name or a resource_id + + Returns: + Union[WebBackendConnectionRead, SourceRead, DestinationRead]: The remote resource model instance. + """ + if self.resource_id is not None: + return self._find_by_resource_id() + else: + return self._find_by_resource_name() + + def to_json(self) -> str: + """Get the JSON representation of the remote resource model instance. + + Returns: + str: The JSON representation of the remote resource model instance. + """ + return json.dumps(self.get_remote_resource().to_dict()) class Source(BaseResource): name = "source" api = source_api.SourceApi get_function_name = "get_source" + list_for_workspace_function_name = "list_sources_for_workspace" @property - def get_function_kwargs(self) -> dict: - return {"source_id_request_body": SourceIdRequestBody(source_id=self.resource_id)} + def get_payload(self) -> Optional[SourceIdRequestBody]: + """Defines the payload to retrieve the remote source according to its resource_id. + Returns: + SourceIdRequestBody: The SourceIdRequestBody payload. + """ + return SourceIdRequestBody(self.resource_id) class Destination(BaseResource): name = "destination" api = destination_api.DestinationApi get_function_name = "get_destination" + list_for_workspace_function_name = "list_destinations_for_workspace" @property - def get_function_kwargs(self) -> dict: - return {"destination_id_request_body": DestinationIdRequestBody(destination_id=self.resource_id)} + def get_payload(self) -> Optional[DestinationIdRequestBody]: + """Defines the payload to retrieve the remote destination according to its resource_id. + Returns: + DestinationIdRequestBody: The DestinationIdRequestBody payload. + """ + return DestinationIdRequestBody(self.resource_id) class Connection(BaseResource): name = "connection" - api = connection_api.ConnectionApi - get_function_name = "get_connection" + api = web_backend_api.WebBackendApi + get_function_name = "web_backend_get_connection" + list_for_workspace_function_name = "web_backend_list_connections_for_workspace" @property - def get_function_kwargs(self) -> dict: - return {"connection_id_request_body": ConnectionIdRequestBody(connection_id=self.resource_id)} + def get_payload(self) -> Optional[WebBackendConnectionRequestBody]: + """Defines the payload to retrieve the remote connection according to its resource_id. + Returns: + WebBackendConnectionRequestBody: The WebBackendConnectionRequestBody payload. + """ + return WebBackendConnectionRequestBody(with_refreshed_catalog=False, connection_id=self.resource_id) From e376f4e05c0fab49e2fb7933c8600ceff7858344 Mon Sep 17 00:00:00 2001 From: danieldiamond Date: Sun, 26 Jun 2022 13:24:04 +1000 Subject: [PATCH 12/16] Remove DuplicateResourceError from apply --- octavia-cli/octavia_cli/apply/resources.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/octavia-cli/octavia_cli/apply/resources.py b/octavia-cli/octavia_cli/apply/resources.py index 2124f9e760a1..b846fa675a98 100644 --- a/octavia-cli/octavia_cli/apply/resources.py +++ b/octavia-cli/octavia_cli/apply/resources.py @@ -57,10 +57,6 @@ from .yaml_loaders import EnvVarLoader -class DuplicateResourceError(click.ClickException): - pass - - class NonExistingResourceError(click.ClickException): pass From 3d96529400792fbd7483ded8e5304f76715dd850 Mon Sep 17 00:00:00 2001 From: danieldiamond Date: Sun, 26 Jun 2022 13:24:28 +1000 Subject: [PATCH 13/16] Remove ConnectorRenderer from generate --- octavia-cli/octavia_cli/generate/renderers.py | 34 ------------------- .../templates/source_or_destination.yaml.j2 | 2 +- 2 files changed, 1 insertion(+), 35 deletions(-) diff --git a/octavia-cli/octavia_cli/generate/renderers.py b/octavia-cli/octavia_cli/generate/renderers.py index 52e31ab05c96..f4e3ebc1c9fb 100644 --- a/octavia-cli/octavia_cli/generate/renderers.py +++ b/octavia-cli/octavia_cli/generate/renderers.py @@ -224,40 +224,6 @@ def _render(self) -> str: ) -class ConnectorRenderer(BaseRenderer): - TEMPLATE = JINJA_ENV.get_template("source_or_destination.yaml.j2") - - def __init__(self, resource_id: str, definition: BaseDefinition) -> None: - """Connector specification renderer constructor. - - Args: - resource_id (str): ID of the existing source or destination. - definition (BaseDefinition): The definition related to a source or a destination. - """ - super().__init__(resource_id) - self.definition = definition - - def _parse_connection_specification(self, schema: dict) -> List[List["FieldToRender"]]: - """Create a renderable structure from the specification schema - - Returns: - List[List["FieldToRender"]]: List of list of fields to render. - """ - if schema.get("oneOf"): - roots = [] - for one_of_value in schema.get("oneOf"): - required_fields = one_of_value.get("required", []) - roots.append(parse_fields(required_fields, one_of_value["properties"])) - return roots - else: - required_fields = schema.get("required", []) - return [parse_fields(required_fields, schema["properties"])] - - def _render(self) -> str: - parsed_schema = self._parse_connection_specification(self.definition.specification.connection_specification) - return self.TEMPLATE.render({"resource_id": self.resource_id, "definition": self.definition, "configuration_fields": parsed_schema}) - - class ConnectionRenderer(BaseRenderer): TEMPLATE = JINJA_ENV.get_template("connection.yaml.j2") diff --git a/octavia-cli/octavia_cli/generate/templates/source_or_destination.yaml.j2 b/octavia-cli/octavia_cli/generate/templates/source_or_destination.yaml.j2 index c8a33f708577..9f5131e1789b 100644 --- a/octavia-cli/octavia_cli/generate/templates/source_or_destination.yaml.j2 +++ b/octavia-cli/octavia_cli/generate/templates/source_or_destination.yaml.j2 @@ -1,6 +1,6 @@ # Configuration for {{ definition.docker_repository }} # Documentation about this connector can be found at {{ definition.documentation_url }} -resource_name: "{{ resource_name}}" +resource_name: {{ resource_name}} definition_type: {{ definition.type}} definition_id: {{ definition.id }} definition_image: {{ definition.docker_repository }} From 6f3944d8bf4ddce4eaf67d50ebd95262edc55de9 Mon Sep 17 00:00:00 2001 From: danieldiamond Date: Sun, 26 Jun 2022 13:24:53 +1000 Subject: [PATCH 14/16] Update unit tests for commands and resources --- octavia-cli/unit_tests/test_get/__init__.py | 2 +- .../unit_tests/test_get/test_commands.py | 101 +++++++++------ .../unit_tests/test_get/test_resources.py | 118 +++++++++++++++--- 3 files changed, 163 insertions(+), 58 deletions(-) diff --git a/octavia-cli/unit_tests/test_get/__init__.py b/octavia-cli/unit_tests/test_get/__init__.py index 46b7376756ec..1100c1c58cf5 100644 --- a/octavia-cli/unit_tests/test_get/__init__.py +++ b/octavia-cli/unit_tests/test_get/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # diff --git a/octavia-cli/unit_tests/test_get/test_commands.py b/octavia-cli/unit_tests/test_get/test_commands.py index fb4ef61c71a6..a380c290689f 100644 --- a/octavia-cli/unit_tests/test_get/test_commands.py +++ b/octavia-cli/unit_tests/test_get/test_commands.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # import pytest @@ -27,49 +27,76 @@ def test_available_commands(): assert commands.AVAILABLE_COMMANDS == [commands.source, commands.destination, commands.connection] -@pytest.mark.parametrize( - "command,resource_id", - [ - (commands.source, "my_resource_id"), - ], -) -def test_source(mocker, context_object, command, resource_id): - runner = CliRunner() - mocker.patch.object(commands, "Source", mocker.Mock()) - mock_renderer = commands.Source.return_value - mock_renderer.get_config.return_value = '{"hello": "world"}' - result = runner.invoke(command, [resource_id], obj=context_object) - assert result.exit_code == 0 - commands.Source.assert_called_with(context_object["API_CLIENT"], context_object["WORKSPACE_ID"], resource_id) +def test_build_help_message(): + assert commands.build_help_message("fake_resource_type") == "Get a JSON representation of a remote fake_resource_type." -@pytest.mark.parametrize( - "command,resource_id", - [ - (commands.destination, "my_resource_id"), - ], -) -def test_destination(mocker, context_object, command, resource_id): - runner = CliRunner() - mocker.patch.object(commands, "Destination", mocker.Mock()) - mock_renderer = commands.Destination.return_value - mock_renderer.get_config.return_value = '{"hello": "world"}' - result = runner.invoke(command, [resource_id], obj=context_object) - assert result.exit_code == 0 - commands.Destination.assert_called_with(context_object["API_CLIENT"], context_object["WORKSPACE_ID"], resource_id) +def test_get_resource_id_or_name(): + resource_id, resource_name = commands.get_resource_id_or_name("resource_name") + assert resource_id is None and resource_name == "resource_name" + resource_id, resource_name = commands.get_resource_id_or_name("8c2e8369-3b81-471a-9945-32a3c67c31b7") + assert resource_id == "8c2e8369-3b81-471a-9945-32a3c67c31b7" and resource_name is None + + +def test_get_json_representation(mocker, context_object): + mock_cls = mocker.Mock() + mocker.patch.object(commands.click, "echo") + mock_resource_id = mocker.Mock() + mock_resource_name = mocker.Mock() + mocker.patch.object(commands, "get_resource_id_or_name", mocker.Mock(return_value=(mock_resource_id, mock_resource_name))) + json_repr = commands.get_json_representation(context_object["API_CLIENT"], context_object["WORKSPACE_ID"], mock_cls, "resource_to_get") + commands.get_resource_id_or_name.assert_called_with("resource_to_get") + mock_cls.assert_called_with( + context_object["API_CLIENT"], context_object["WORKSPACE_ID"], resource_id=mock_resource_id, resource_name=mock_resource_name + ) + assert json_repr == mock_cls.return_value.to_json.return_value @pytest.mark.parametrize( - "command,resource_id", + "command, resource_cls, resource", [ - (commands.connection, "my_resource_id"), + (commands.source, commands.Source, "my_resource_id"), + (commands.destination, commands.Destination, "my_resource_id"), + (commands.connection, commands.Connection, "my_resource_id"), ], ) -def test_connection(mocker, context_object, command, resource_id): +def test_commands(context_object, mocker, command, resource_cls, resource): + mocker.patch.object(commands, "get_json_representation", mocker.Mock(return_value='{"foo": "bar"}')) runner = CliRunner() - mocker.patch.object(commands, "Connection", mocker.Mock()) - mock_renderer = commands.Connection.return_value - mock_renderer.get_config.return_value = '{"hello": "world"}' - result = runner.invoke(command, [resource_id], obj=context_object) + result = runner.invoke(command, [resource], obj=context_object) + commands.get_json_representation.assert_called_once_with( + context_object["API_CLIENT"], context_object["WORKSPACE_ID"], resource_cls, resource + ) assert result.exit_code == 0 - commands.Connection.assert_called_with(context_object["API_CLIENT"], context_object["WORKSPACE_ID"], resource_id) + + +# @pytest.mark.parametrize( +# "command,resource_id", +# [ +# (commands.destination, "my_resource_id"), +# ], +# ) +# def test_destination(mocker, context_object, command, resource_id): +# runner = CliRunner() +# mocker.patch.object(commands, "Destination", mocker.Mock()) +# mock_renderer = commands.Destination.return_value +# mock_renderer.get_remote_resource.return_value = '{"hello": "world"}' +# result = runner.invoke(command, [resource_id], obj=context_object) +# assert result.exit_code == 0 +# commands.Destination.assert_called_with(context_object["API_CLIENT"], context_object["WORKSPACE_ID"], resource_id) + + +# @pytest.mark.parametrize( +# "command,resource_id", +# [ +# (commands.connection, "my_resource_id"), +# ], +# ) +# def test_connection(mocker, context_object, command, resource_id): +# runner = CliRunner() +# mocker.patch.object(commands, "Connection", mocker.Mock()) +# mock_renderer = commands.Connection.return_value +# mock_renderer.get_remote_resource.return_value = '{"hello": "world"}' +# result = runner.invoke(command, [resource_id], obj=context_object) +# assert result.exit_code == 0 +# commands.Connection.assert_called_with(context_object["API_CLIENT"], context_object["WORKSPACE_ID"], resource_id) diff --git a/octavia-cli/unit_tests/test_get/test_resources.py b/octavia-cli/unit_tests/test_get/test_resources.py index 84560de41e63..3ac680c6a239 100644 --- a/octavia-cli/unit_tests/test_get/test_resources.py +++ b/octavia-cli/unit_tests/test_get/test_resources.py @@ -1,13 +1,13 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. # import pytest -from airbyte_api_client.api import connection_api, destination_api, source_api -from airbyte_api_client.model.connection_id_request_body import ConnectionIdRequestBody +from airbyte_api_client.api import destination_api, source_api, web_backend_api from airbyte_api_client.model.destination_id_request_body import DestinationIdRequestBody from airbyte_api_client.model.source_id_request_body import SourceIdRequestBody -from octavia_cli.get.resources import BaseResource, Connection, Destination, Source +from airbyte_api_client.model.web_backend_connection_request_body import WebBackendConnectionRequestBody +from octavia_cli.get.resources import BaseResource, Connection, Destination, DuplicateResourceError, ResourceNotFoundError, Source class TestBaseResource: @@ -16,20 +16,95 @@ def patch_base_class(self, mocker): # Mock abstract methods to enable instantiating abstract class mocker.patch.object(BaseResource, "__abstractmethods__", set()) mocker.patch.object(BaseResource, "api", mocker.Mock()) - mocker.patch.object(BaseResource, "get_function_name", "foo") + mocker.patch.object(BaseResource, "get_function_name", "get_function_name") + mocker.patch.object(BaseResource, "get_payload", "get_payload") + mocker.patch.object(BaseResource, "list_for_workspace_function_name", "list_for_workspace_function_name") + mocker.patch.object(BaseResource, "name", "fake_resource") - def test_init(self, patch_base_class, mock_api_client, mocker): - base_definition = BaseResource(mock_api_client, "workspace_id", "resource_id") - assert base_definition.api_instance == base_definition.api.return_value - base_definition.api.assert_called_with(mock_api_client) - assert base_definition.get_function_kwargs == {} - assert base_definition._get_fn == getattr(base_definition.api, base_definition.get_function_name) + @pytest.mark.parametrize( + "resource_id, resource_name, expected_error, expected_error_message", + [ + ("my_resource_id", None, None, None), + (None, "my_resource_name", None, None), + (None, None, ValueError, "resource_id and resource_name keyword arguments can't be both None."), + ("my_resource_id", "my_resource_name", ValueError, "resource_id and resource_name keyword arguments can't be both set."), + ], + ) + def test_init(self, patch_base_class, mock_api_client, resource_id, resource_name, expected_error, expected_error_message): + if expected_error: + with pytest.raises(expected_error, match=expected_error_message): + base_resource = BaseResource(mock_api_client, "workspace_id", resource_id=resource_id, resource_name=resource_name) + else: + base_resource = BaseResource(mock_api_client, "workspace_id", resource_id=resource_id, resource_name=resource_name) + base_resource.api.assert_called_with(mock_api_client) + assert base_resource.api_instance == base_resource.api.return_value + assert base_resource.workspace_id == "workspace_id" + assert base_resource._get_fn == getattr(base_resource.api, base_resource.get_function_name) + assert base_resource._list_for_workspace_fn == getattr(base_resource.api, base_resource.list_for_workspace_function_name) + assert base_resource.resource_id == resource_id + assert base_resource.resource_name == resource_name - def test_get_config(self, patch_base_class, mock_api_client, mocker): - assert Source.__base__ == BaseResource + @pytest.mark.parametrize( + "resource_name, api_response_resources_names, expected_error, expected_error_message", + [ + ("foo", ["foo", "bar"], None, None), + ("foo", ["bar", "fooo"], ResourceNotFoundError, "The fake_resource foo was not found in your current Airbyte workspace."), + ( + "foo", + ["foo", "foo"], + DuplicateResourceError, + "2 fake_resources with the name foo were found in your current Airbyte workspace.", + ), + ], + ) + def test__find_by_resource_name( + self, mocker, patch_base_class, mock_api_client, resource_name, api_response_resources_names, expected_error, expected_error_message + ): + mock_api_response_records = [] + for fake_resource_name in api_response_resources_names: + mock_api_response_record = mocker.Mock() # We can't set the mock name on creation as it's a reserved attribute + mock_api_response_record.name = fake_resource_name + mock_api_response_records.append(mock_api_response_record) + + mocker.patch.object( + BaseResource, "_list_for_workspace_fn", mocker.Mock(return_value=mocker.Mock(fake_resources=mock_api_response_records)) + ) + base_resource = BaseResource(mock_api_client, "workspace_id", resource_id=None, resource_name=resource_name) + if not expected_error: + found_resource = base_resource._find_by_resource_name() + assert found_resource.name == resource_name + if expected_error: + with pytest.raises(expected_error, match=expected_error_message): + base_resource._find_by_resource_name() + + def test__find_by_id(self, mocker, patch_base_class, mock_api_client): + mocker.patch.object(BaseResource, "_get_fn") + base_resource = BaseResource(mock_api_client, "workspace_id", resource_id="my_resource_id") + base_resource._find_by_resource_id() + base_resource._get_fn.assert_called_with(base_resource.api_instance, base_resource.get_payload) + + @pytest.mark.parametrize("resource_id, resource_name", [("my_resource_id", None), (None, "my_resource_name")]) + def test_get_remote_resource(self, mocker, patch_base_class, mock_api_client, resource_id, resource_name): + mocker.patch.object(BaseResource, "_find_by_resource_id") + mocker.patch.object(BaseResource, "_find_by_resource_name") + base_resource = BaseResource(mock_api_client, "workspace_id", resource_id=resource_id, resource_name=resource_name) + remote_resource = base_resource.get_remote_resource() + if resource_id is not None: + base_resource._find_by_resource_id.assert_called_once() + base_resource._find_by_resource_name.assert_not_called() + assert remote_resource == base_resource._find_by_resource_id.return_value + if resource_name is not None: + base_resource._find_by_resource_id.assert_not_called() + base_resource._find_by_resource_name.assert_called_once() + assert remote_resource == base_resource._find_by_resource_name.return_value - base_definition = BaseResource(mock_api_client, "workspace_id", "resource_id") - assert base_definition._get_fn == getattr(base_definition.api, base_definition.get_function_name) + def test_to_json(self, mocker, patch_base_class, mock_api_client): + mocker.patch.object( + BaseResource, "get_remote_resource", mocker.Mock(return_value=mocker.Mock(to_dict=mocker.Mock(return_value={"foo": "bar"}))) + ) + base_resource = BaseResource(mock_api_client, "workspace_id", resource_id="my_resource_id") + json_repr = base_resource.to_json() + assert json_repr == '{"foo": "bar"}' class TestSource: @@ -38,7 +113,8 @@ def test_init(self, mock_api_client): source = Source(mock_api_client, "workspace_id", "resource_id") assert source.api == source_api.SourceApi assert source.get_function_name == "get_source" - assert source.get_function_kwargs == {"source_id_request_body": SourceIdRequestBody("resource_id")} + assert source.list_for_workspace_function_name == "list_sources_for_workspace" + assert source.get_payload == SourceIdRequestBody("resource_id") class TestDestination: @@ -47,13 +123,15 @@ def test_init(self, mock_api_client): destination = Destination(mock_api_client, "workspace_id", "resource_id") assert destination.api == destination_api.DestinationApi assert destination.get_function_name == "get_destination" - assert destination.get_function_kwargs == {"destination_id_request_body": DestinationIdRequestBody("resource_id")} + assert destination.list_for_workspace_function_name == "list_destinations_for_workspace" + assert destination.get_payload == DestinationIdRequestBody("resource_id") class TestConnection: def test_init(self, mock_api_client): assert Connection.__base__ == BaseResource connection = Connection(mock_api_client, "workspace_id", "resource_id") - assert connection.api == connection_api.ConnectionApi - assert connection.get_function_name == "get_connection" - assert connection.get_function_kwargs == {"connection_id_request_body": ConnectionIdRequestBody("resource_id")} + assert connection.api == web_backend_api.WebBackendApi + assert connection.get_function_name == "web_backend_get_connection" + assert connection.list_for_workspace_function_name == "web_backend_list_connections_for_workspace" + assert connection.get_payload == WebBackendConnectionRequestBody(with_refreshed_catalog=False, connection_id=connection.resource_id) From c44903ea27743ac272d5f5f75eb90cedcfb31494 Mon Sep 17 00:00:00 2001 From: danieldiamond Date: Sun, 26 Jun 2022 13:25:16 +1000 Subject: [PATCH 15/16] Update docs and setup --- octavia-cli/Dockerfile | 2 +- octavia-cli/README.md | 188 ++++++++++++++++++++++++++++++----------- octavia-cli/install.sh | 2 +- octavia-cli/setup.py | 2 +- 4 files changed, 144 insertions(+), 50 deletions(-) diff --git a/octavia-cli/Dockerfile b/octavia-cli/Dockerfile index 1f64525b74b4..eef020a8012e 100644 --- a/octavia-cli/Dockerfile +++ b/octavia-cli/Dockerfile @@ -14,5 +14,5 @@ USER octavia-cli WORKDIR /home/octavia-project ENTRYPOINT ["octavia"] -LABEL io.airbyte.version=0.39.25-alpha +LABEL io.airbyte.version=0.39.24-alpha LABEL io.airbyte.name=airbyte/octavia-cli diff --git a/octavia-cli/README.md b/octavia-cli/README.md index 325e4bcbb535..f210aca345bb 100644 --- a/octavia-cli/README.md +++ b/octavia-cli/README.md @@ -104,7 +104,7 @@ This script: ```bash touch ~/.octavia # Create a file to store env variables that will be mapped the octavia-cli container mkdir my_octavia_project_directory # Create your octavia project directory where YAML configurations will be stored. -docker run --name octavia-cli -i --rm -v my_octavia_project_directory:/home/octavia-project --network host --user $(id -u):$(id -g) --env-file ~/.octavia airbyte/octavia-cli:0.39.25-alpha +docker run --name octavia-cli -i --rm -v my_octavia_project_directory:/home/octavia-project --network host --user $(id -u):$(id -g) --env-file ~/.octavia airbyte/octavia-cli:0.39.24-alpha ``` ### Using `docker-compose` @@ -172,21 +172,21 @@ headers: ### `octavia` subcommands -| **Command** | **Usage** | -| ----------------------------------------- | --------------------------------------------------------------------------------- | -| **`octavia init`** | Initialize required directories for the project. | -| **`octavia list connectors sources`** | List all sources connectors available on the remote Airbyte instance. | -| **`octavia list connectors destination`** | List all destinations connectors available on the remote Airbyte instance. | -| **`octavia list workspace sources`** | List existing sources in current the Airbyte workspace. | -| **`octavia list workspace destinations`** | List existing destinations in the current Airbyte workspace. | -| **`octavia list workspace connections`** | List existing connections in the current Airbyte workspace. | -| **`octavia get workspace source`** | Get an existing source in current the Airbyte workspace. | -| **`octavia get workspace destination`** | Get an existing destination in the current Airbyte workspace. | -| **`octavia get workspace connection`** | Get an existing connection in the current Airbyte workspace. | -| **`octavia generate source`** | Generate a local YAML configuration for a new source. | -| **`octavia generate destination`** | Generate a local YAML configuration for a new destination. | -| **`octavia generate connection`** | Generate a local YAML configuration for a new connection. | -| **`octavia apply`** | Create or update Airbyte remote resources according to local YAML configurations. | +| **Command** | **Usage** | +| ----------------------------------------- | ---------------------------------------------------------------------------------------- | +| **`octavia init`** | Initialize required directories for the project. | +| **`octavia list connectors sources`** | List all sources connectors available on the remote Airbyte instance. | +| **`octavia list connectors destination`** | List all destinations connectors available on the remote Airbyte instance. | +| **`octavia list workspace sources`** | List existing sources in current the Airbyte workspace. | +| **`octavia list workspace destinations`** | List existing destinations in the current Airbyte workspace. | +| **`octavia list workspace connections`** | List existing connections in the current Airbyte workspace. | +| **`octavia get source`** | Get the JSON representation of an existing source in current the Airbyte workspace. | +| **`octavia get destination`** | Get the JSON representation of an existing destination in the current Airbyte workspace. | +| **`octavia get connection`** | Get the JSON representation of an existing connection in the current Airbyte workspace. | +| **`octavia generate source`** | Generate a local YAML configuration for a new source. | +| **`octavia generate destination`** | Generate a local YAML configuration for a new destination. | +| **`octavia generate connection`** | Generate a local YAML configuration for a new connection. | +| **`octavia apply`** | Create or update Airbyte remote resources according to local YAML configurations. | #### `octavia init` @@ -271,15 +271,16 @@ NAME CONNECTION ID STATUS SOURCE ID weather_to_pg a4491317-153e-436f-b646-0b39338f9aab active c4aa8550-2122-4a33-9a21-adbfaa638544 c0c977c2-48e7-46fe-9f57-576285c26d42 ``` -#### `octavia get source ` +#### `octavia get source or ` -Get an existing source in current the Airbyte workspace +Get an existing source in current the Airbyte workspace. You can use a source ID or name. -| **Argument** | **Description** | -| ------------ | --------------------------------------------------------------------------------- | -| `SOURCE_ID` | The source connector id. Can be retrieved using `octavia list workspace sources`. | +| **Argument** | **Description** | +| --------------| -----------------| +| `SOURCE_ID` | The source id. | +| `SOURCE_NAME` | The source name. | -**Example**: +**Examples**: ```bash $ octavia get source c0c977c2-48e7-46fe-9f57-576285c26d42 @@ -293,45 +294,73 @@ $ octavia get source c0c977c2-48e7-46fe-9f57-576285c26d42 'workspace_id': 'c4aa8550-2122-4a33-9a21-adbfaa638544'} ``` -#### `octavia get destination ` +```bash +$ octavia get source "My Poke" +{'connection_configuration': {'key': '**********', + 'start_date': '2010-01-01T00:00:00.000Z', + 'token': '**********'}, + 'name': 'Pokemon', + 'source_definition_id': 'b08e4776-d1de-4e80-ab5c-1e51dad934a2', + 'source_id': 'c0c977c2-48e7-46fe-9f57-576285c26d42', + 'source_name': 'My Poke', + 'workspace_id': 'c4aa8550-2122-4a33-9a21-adbfaa638544'} +``` -Get an existing destination in current the Airbyte workspace +#### `octavia get destination or ` -| **Argument** | **Description** | -| ---------------- | ------------------------------------------------------------------------------------------- | -| `DESTINATION_ID` | The destination connector id. Can be retrieved using `octavia list workspace destinations`. | +Get an existing destination in current the Airbyte workspace. You can use a destination ID or name. -**Example**: +| **Argument** | **Description** | +| ------------------ | ----------------------| +| `DESTINATION_ID` | The destination id. | +| `DESTINATION_NAME` | The destination name. | + +**Examples**: ```bash $ octavia get destination c0c977c2-48e7-46fe-9f57-576285c26d42 { - "sourceDefinitionId": "18102e7c-5160-4000-821f-4d7cfdf87201", - "sourceId": "18102e7c-5160-4000-841b-15e8ec48c301", + "destinationDefinitionId": "c0c977c2-48e7-46fe-9f57-576285c26d42", + "destinationId": "18102e7c-5160-4000-841b-15e8ec48c301", + "workspaceId": "18102e7c-5160-4000-883a-30bc7cd65601", + "connectionConfiguration": { + "user": "charles" + }, + "name": "pg", + "destinationName": "Postgres" +} +``` + +```bash +$ octavia get destination pg +{ + "destinationDefinitionId": "18102e7c-5160-4000-821f-4d7cfdf87201", + "destinationId": "18102e7c-5160-4000-841b-15e8ec48c301", "workspaceId": "18102e7c-5160-4000-883a-30bc7cd65601", "connectionConfiguration": { "user": "charles" }, "name": "string", - "sourceName": "string" + "destinationName": "string" } ``` -#### `octavia get connection ` +#### `octavia get connection or ` -Get an existing connection in current the Airbyte workspace +Get an existing connection in current the Airbyte workspace. You can use a connection ID or name. -| **Argument** | **Description** | -| --------------- | ----------------------------------------------------------------------------------------- | -| `CONNECTION_ID` | The connection connector id. Can be retrieved using `octavia list workspace connections`. | +| **Argument** | **Description** | +| ------------------ | ----------------------| +| `CONNECTION_ID` | The connection id. | +| `CONNECTION_NAME` | The connection name. | **Example**: ```bash $ octavia get connection c0c977c2-48e7-46fe-9f57-576285c26d42 { - "connectionId": "18102e7c-5340-4000-8656-c433ed782601", - "name": "string", + "connectionId": "c0c977c2-48e7-46fe-9f57-576285c26d42", + "name": "Poke To PG", "namespaceDefinition": "source", "namespaceFormat": "${SOURCE_NAMESPACE}", "prefix": "string", @@ -392,6 +421,70 @@ $ octavia get connection c0c977c2-48e7-46fe-9f57-576285c26d42 } ``` +```bash +$ octavia get connection "Poke To PG" +{ + "connectionId": "c0c977c2-48e7-46fe-9f57-576285c26d42", + "name": "Poke To PG", + "namespaceDefinition": "source", + "namespaceFormat": "${SOURCE_NAMESPACE}", + "prefix": "string", + "sourceId": "18102e7c-5340-4000-8eaa-4a86f844b101", + "destinationId": "18102e7c-5340-4000-8e58-6bed49c24b01", + "operationIds": [ + "18102e7c-5340-4000-8ef0-f35c05a49a01" + ], + "syncCatalog": { + "streams": [ + { + "stream": { + "name": "string", + "jsonSchema": {}, + "supportedSyncModes": [ + "full_refresh" + ], + "sourceDefinedCursor": false, + "defaultCursorField": [ + "string" + ], + "sourceDefinedPrimaryKey": [ + [ + "string" + ] + ], + "namespace": "string" + }, + "config": { + "syncMode": "full_refresh", + "cursorField": [ + "string" + ], + "destinationSyncMode": "append", + "primaryKey": [ + [ + "string" + ] + ], + "aliasName": "string", + "selected": false + } + } + ] + }, + "schedule": { + "units": 0, + "timeUnit": "minutes" + }, + "status": "active", + "resourceRequirements": { + "cpu_request": "string", + "cpu_limit": "string", + "memory_request": "string", + "memory_limit": "string" + }, + "sourceCatalogId": "18102e7c-5340-4000-85f3-204ab7715801" +} +``` #### `octavia generate source ` Generate a YAML configuration for a source. @@ -514,12 +607,13 @@ You can disable telemetry by setting the `OCTAVIA_ENABLE_TELEMETRY` environment ## Changelog -| Version | Date | Description | PR | -| ------- | ---------- | -------------------------------------------------- | ----------------------------------------------------------- | -| 0.39.19 | 2022-06-16 | Allow connection management on multiple workspaces | [#12727](https://github.com/airbytehq/airbyte/pull/12727) | -| 0.39.19 | 2022-06-15 | Allow users to set custom HTTP headers | [#12893](https://github.com/airbytehq/airbyte/pull/12893) | -| 0.39.14 | 2022-05-12 | Enable normalization on connection | [#12727](https://github.com/airbytehq/airbyte/pull/12727) | -| 0.37.0 | 2022-05-05 | Use snake case in connection fields | [#12133](https://github.com/airbytehq/airbyte/pull/12133) | -| 0.35.68 | 2022-04-15 | Improve telemetry | [#12072](https://github.com/airbytehq/airbyte/issues/11896) | -| 0.35.68 | 2022-04-12 | Add telemetry | [#11896](https://github.com/airbytehq/airbyte/issues/11896) | -| 0.35.61 | 2022-04-07 | Alpha release | [EPIC](https://github.com/airbytehq/airbyte/issues/10704) | +| Version | Date | Description | PR | +| ------- | ---------- | ------------------------------------------------------------ | ----------------------------------------------------------- | +| 0.39.25 | 2022-06-24 | Create get command to retrieve resources JSON representation | [#13254](https://github.com/airbytehq/airbyte/pull/13254) | +| 0.39.19 | 2022-06-16 | Allow connection management on multiple workspaces | [#13070](https://github.com/airbytehq/airbyte/pull/12727) | +| 0.39.19 | 2022-06-15 | Allow users to set custom HTTP headers | [#12893](https://github.com/airbytehq/airbyte/pull/12893) | +| 0.39.14 | 2022-05-12 | Enable normalization on connection | [#12727](https://github.com/airbytehq/airbyte/pull/12727) | +| 0.37.0 | 2022-05-05 | Use snake case in connection fields | [#12133](https://github.com/airbytehq/airbyte/pull/12133) | +| 0.35.68 | 2022-04-15 | Improve telemetry | [#12072](https://github.com/airbytehq/airbyte/issues/11896) | +| 0.35.68 | 2022-04-12 | Add telemetry | [#11896](https://github.com/airbytehq/airbyte/issues/11896) | +| 0.35.61 | 2022-04-07 | Alpha release | [EPIC](https://github.com/airbytehq/airbyte/issues/10704) | diff --git a/octavia-cli/install.sh b/octavia-cli/install.sh index dcd80a849500..49124181ad92 100755 --- a/octavia-cli/install.sh +++ b/octavia-cli/install.sh @@ -3,7 +3,7 @@ # This install scripts currently only works for ZSH and Bash profiles. # It creates an octavia alias in your profile bound to a docker run command and your current user. -VERSION=0.39.25-alpha +VERSION=0.39.24-alpha OCTAVIA_ENV_FILE=${HOME}/.octavia detect_profile() { diff --git a/octavia-cli/setup.py b/octavia-cli/setup.py index 54df7e9ba7f1..294665aed20b 100644 --- a/octavia-cli/setup.py +++ b/octavia-cli/setup.py @@ -15,7 +15,7 @@ setup( name="octavia-cli", - version="0.39.25", + version="0.39.24", description="A command line interface to manage Airbyte configurations", long_description=README, author="Airbyte", From d67eaefdfc8b88e67485617cdc8aaf86d8d144c5 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 27 Jun 2022 08:45:09 +0200 Subject: [PATCH 16/16] bump version in changelog --- octavia-cli/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octavia-cli/README.md b/octavia-cli/README.md index 28e949b59dad..bc9fd98e571c 100644 --- a/octavia-cli/README.md +++ b/octavia-cli/README.md @@ -609,7 +609,7 @@ You can disable telemetry by setting the `OCTAVIA_ENABLE_TELEMETRY` environment | Version | Date | Description | PR | | ------- | ---------- | ------------------------------------------------------------ | ----------------------------------------------------------- | -| 0.39.25 | 2022-06-24 | Create get command to retrieve resources JSON representation | [#13254](https://github.com/airbytehq/airbyte/pull/13254) | +| 0.39.27 | 2022-06-24 | Create get command to retrieve resources JSON representation | [#13254](https://github.com/airbytehq/airbyte/pull/13254) | | 0.39.19 | 2022-06-16 | Allow connection management on multiple workspaces | [#13070](https://github.com/airbytehq/airbyte/pull/12727) | | 0.39.19 | 2022-06-15 | Allow users to set custom HTTP headers | [#12893](https://github.com/airbytehq/airbyte/pull/12893) | | 0.39.14 | 2022-05-12 | Enable normalization on connection | [#12727](https://github.com/airbytehq/airbyte/pull/12727) |