From 8e78454de8f06c646cda81160ab61042ab16390a Mon Sep 17 00:00:00 2001 From: Simo Tumelius Date: Wed, 18 May 2022 10:37:39 +0300 Subject: [PATCH] Add 'dbt-cloud audit-log get' command (#58) * Add 'dbt-cloud audit-log get' command * Add unit test * Update README * Add integration test Co-authored-by: Simo Tumelius --- .circleci/config.yml | 8 ++ README.md | 76 +++++++++++++- dbt_cloud/cli.py | 13 +++ dbt_cloud/command/__init__.py | 1 + dbt_cloud/command/audit_log/__init__.py | 1 + dbt_cloud/command/audit_log/get.py | 40 ++++++++ dbt_cloud/command/run/list.py | 4 +- pytest.ini | 3 +- tests/conftest.py | 12 +++ tests/data/audit_log_get_response.json | 127 ++++++++++++++++++++++++ 10 files changed, 278 insertions(+), 7 deletions(-) create mode 100644 dbt_cloud/command/audit_log/__init__.py create mode 100644 dbt_cloud/command/audit_log/get.py create mode 100644 tests/data/audit_log_get_response.json diff --git a/.circleci/config.yml b/.circleci/config.yml index 88a6c55..5bb9873 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -155,6 +155,14 @@ jobs: account_count=$(cat accounts.json | jq '.data | length') [[ $account_count > 0 ]] && exit 0 || exit 1 + - run: + name: Test 'dbt-cloud audit-log get' + command: | + dbt-cloud audit-log get > audit_logs.json + cat audit_logs.json | jq '.data[] | {id: .id}' + log_count=$(cat audit_logs.json | jq '.data | length') + [[ $log_count > 0 ]] && exit 0 || exit 1 + - run: name: Test 'dbt-cloud metadata query' command: | diff --git a/README.md b/README.md index f8f3eff..0cdc71f 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ Group | API endpoint | Command | Description | | --- | --- | --- | --- | | Accounts | [https://cloud.getdbt.com/api/v2/accounts/](https://docs.getdbt.com/dbt-cloud/api-v2#operation/listAccounts) | [dbt-cloud account list](#dbt-cloud-account-list) | Retrieves all available accounts | | Accounts | [https://cloud.getdbt.com/api/v2/accounts/{accountId}/](https://docs.getdbt.com/dbt-cloud/api-v2#operation/getAccountById) | `dbt-cloud account get` | Not implemented yet | +| Audit Logs | https://cloud.getdbt.com/api/v3/accounts/{accountId}/audit-logs/ | [dbt-cloud audit-log get](#dbt-cloud-audit-log-get) | Retrieves audit logs for the dbt Cloud account | | Projects | https://cloud.getdbt.com/api/v2/accounts/{accountId}/projects/ | [dbt-cloud project list](#dbt-cloud-project-list) | Returns a list of projects in the account | | Projects | [https://cloud.getdbt.com/api/v2/accounts/{accountId}/projects/{projectId}](https://docs.getdbt.com/dbt-cloud/api-v2#operation/getProjectById) | `dbt-cloud project get` | Not implemented yet | | Environments | https://cloud.getdbt.com/api/v3/accounts/{accountId}/projects/{projectId}/environments | [dbt-cloud environment list](#dbt-cloud-environment-list) | Retrieves environments for a given project | @@ -59,6 +60,7 @@ Group | API endpoint | Command | Description | # Commands * [dbt-cloud account list](#dbt-cloud-account-list) +* [dbt-cloud audit-log get](#dbt-cloud-audit-log-get) * [dbt-cloud project list](#dbt-cloud-project-list) * [dbt-cloud environment list](#dbt-cloud-environment-list) * [dbt-cloud job run](#dbt-cloud-job-run) @@ -143,6 +145,72 @@ This command retrieves all available dbt Cloud accounts. For more information on ``` +## dbt-cloud audit-log get + +❗ **This command is available for Enterprise accounts only.** + +This command retrieves audit logs for the dbt Cloud account. For more information on the command, run `dbt-cloud audit-log get --help`. This command uses the API v3 which has no official documentation yet. + +
+ Usage + +```bash +>> dbt-cloud audit-log get --logged-at-start 2022-05-01 --logged-at-end 2022-05-07 --limit 1 +{ + "status": { + "code": 200, + "is_success": true, + "user_message": "Success!", + "developer_message": "" + }, + "data": [ + { + "account_id": 123456, + "service": "SERVICE_DBT_CLOUD", + "source": "SOURCE_CLOUD_UI", + "routing_key": "v1.events.auth.credentialsloginsucceeded", + "actor_type": "ACTOR_USER", + "actor_name": "REDACTED", + "actor_id": 123454, + "logged_at": "2022-05-05 06:51:10+00:00", + "uuid": "8868c439-8928-4e8c-924b-77558d65db0b", + "actor_ip": "REDACTED", + "metadata": { + "auth_credentials": { + "user": { + "id": "REDACTED", + "email": "REDACTED" + } + } + }, + "internal": false, + "id": 1809583, + "state": 1, + "created_at": "2022-05-05 06:51:12.454677+00:00", + "updated_at": "2022-05-05 06:51:12.454677+00:00" + } + ], + "extra": { + "filters": { + "account_id": 123456, + "limit": 1, + "offset": 0, + "logged_at__range": [ + "2022-05-01 00:00:00Z", + "2022-05-07 00:00:00Z" + ], + "internal": false + }, + "order_by": "-logged_at", + "pagination": { + "count": 1, + "total_count": 4 + } + } +} +``` +
+ ## dbt-cloud project list This command returns a list of projects in the account. For more information on the API endpoint arguments and response, run `dbt-cloud project list --help` and check out the [dbt Cloud API docs](https://docs.getdbt.com/dbt-cloud/api-v2#operation/listProjects). @@ -717,7 +785,7 @@ This command deletes a job in a dbt Cloud project. Note that this command uses a ## dbt-cloud job delete-all -**This command is a composition of one or more base commands.** +💡 **This command is a composition of one or more base commands.** This command fetches all jobs on the account, deletes them one-by-one after user confirmation via prompt and prints out the job delete responses. For more information on the command and its arguments, run `dbt-cloud job delete-all --help`. @@ -848,7 +916,7 @@ Job 54659 was deleted. ## dbt-cloud job export -**This command is a composition of one or more base commands.** +💡 **This command is a composition of one or more base commands.** This command exports a dbt Cloud job as JSON to a file and can be used in conjunction with [dbt-cloud job import](#dbt-cloud-job-import) to copy jobs between dbt Cloud projects. @@ -908,7 +976,7 @@ This command exports a dbt Cloud job as JSON to a file and can be used in conjun ## dbt-cloud job import -**This command is a composition of one or more base commands.** +💡 **This command is a composition of one or more base commands.** This command imports a dbt Cloud job from exported JSON. You can use JSON manipulation tools (e.g., [jq](https://stedolan.github.io/jq/)) to modify the job definition before importing it. @@ -1215,7 +1283,7 @@ This command cancels a dbt Cloud run. For more information on the API endpoint a ## dbt-cloud run cancel-all -**This command is a composition of one or more base commands.** +💡 **This command is a composition of one or more base commands.** This command fetches all runs on the account, cancels them one-by-one after user confirmation via prompt and prints out the run cancellation responses. For more information on the command and its arguments, run `dbt-cloud run cancel-all --help`. diff --git a/dbt_cloud/cli.py b/dbt_cloud/cli.py index b8a01c5..b2d000c 100644 --- a/dbt_cloud/cli.py +++ b/dbt_cloud/cli.py @@ -19,6 +19,7 @@ DbtCloudProjectListCommand, DbtCloudEnvironmentListCommand, DbtCloudAccountListCommand, + DbtCloudAuditLogGetCommand, ) from dbt_cloud.demo import data_catalog from dbt_cloud.serde import json_to_dict, dict_to_json @@ -71,6 +72,11 @@ def account(): pass +@dbt_cloud.group(help="Interact with dbt Cloud audit logs (Enterprise only).") +def audit_log(): + pass + + @dbt_cloud.group(help="Interact with the dbt Cloud Metadata API.") def metadata(): pass @@ -332,6 +338,13 @@ def list(**kwargs): response = execute_and_print(command) +@audit_log.command(help=DbtCloudAuditLogGetCommand.get_description()) +@DbtCloudAuditLogGetCommand.click_options +def get(**kwargs): + command = DbtCloudAuditLogGetCommand.from_click_options(**kwargs) + response = execute_and_print(command) + + @metadata.command(help=DbtCloudMetadataQueryCommand.get_description()) @click.option( "-f", diff --git a/dbt_cloud/command/__init__.py b/dbt_cloud/command/__init__.py index ed34e4b..2326898 100644 --- a/dbt_cloud/command/__init__.py +++ b/dbt_cloud/command/__init__.py @@ -16,5 +16,6 @@ from .project import DbtCloudProjectListCommand from .environment import DbtCloudEnvironmentListCommand from .account import DbtCloudAccountListCommand +from .audit_log import DbtCloudAuditLogGetCommand from .metadata import DbtCloudMetadataQueryCommand from .command import DbtCloudAccountCommand diff --git a/dbt_cloud/command/audit_log/__init__.py b/dbt_cloud/command/audit_log/__init__.py new file mode 100644 index 0000000..10873ce --- /dev/null +++ b/dbt_cloud/command/audit_log/__init__.py @@ -0,0 +1 @@ +from .get import DbtCloudAuditLogGetCommand diff --git a/dbt_cloud/command/audit_log/get.py b/dbt_cloud/command/audit_log/get.py new file mode 100644 index 0000000..82f701a --- /dev/null +++ b/dbt_cloud/command/audit_log/get.py @@ -0,0 +1,40 @@ +import requests +from typing import Optional +from pydantic import Field, PrivateAttr +from dbt_cloud.command.command import DbtCloudAccountCommand + + +class DbtCloudAuditLogGetCommand(DbtCloudAccountCommand): + """Retrieves audit logs for the dbt Cloud account.""" + + logged_at_start: Optional[str] = Field( + description="Start date (YYYY-MM-DD) for the returned logs." + ) + logged_at_end: Optional[str] = Field( + description="End date (YYYY-MM-DD) for the returned logs." + ) + offset: Optional[int] = Field( + 0, + ge=0, + description="Offset for the returned logs. Must be a positive integer.", + ) + limit: Optional[int] = Field( + 100, + ge=0, + description="A limit on the number of logs to be returned. Must be a positive integer.", + ) + _api_version: str = PrivateAttr("v3") + + @property + def api_url(self) -> str: + return f"{super().api_url}/audit-logs/" + + def execute(self) -> requests.Response: + response = requests.get( + url=self.api_url, + headers=self.request_headers, + params=self.get_payload( + exclude=["api_token", "dbt_cloud_host", "account_id"] + ), + ) + return response diff --git a/dbt_cloud/command/run/list.py b/dbt_cloud/command/run/list.py index 73f3310..1ab5d9c 100644 --- a/dbt_cloud/command/run/list.py +++ b/dbt_cloud/command/run/list.py @@ -20,8 +20,8 @@ class DbtCloudRunListCommand(DbtCloudAccountCommand): limit: Optional[int] = Field( 100, - gte=1, - lte=100, + ge=1, + le=100, description="A limit on the number of objects to be returned, between 1 and 100.", ) environment_id: Optional[str] = Field(description="Filter runs by environment ID.") diff --git a/pytest.ini b/pytest.ini index 8bdf6cb..88341f6 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,4 +6,5 @@ markers = run: tests related to dbt Cloud runs project: tests related to dbt Cloud projects environment: tests related to dbt Cloud environments - account: tests related to dbt Cloud accounts \ No newline at end of file + account: tests related to dbt Cloud accounts + audit_log: tests related to dbt Cloud audit logs \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 6e9b813..de43f1a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,6 +15,7 @@ DbtCloudRunListCommand, DbtCloudEnvironmentListCommand, DbtCloudAccountListCommand, + DbtCloudAuditLogGetCommand, ) @@ -154,6 +155,17 @@ def load_response(response_name): "get", marks=pytest.mark.account, ), + pytest.param( + "audit_log_get", + DbtCloudAuditLogGetCommand( + api_token=API_TOKEN, + logged_at_start="2022-05-01", + logged_at_end="2022-05-07", + ), + load_response("audit_log_get_response"), + "get", + marks=pytest.mark.audit_log, + ), ] diff --git a/tests/data/audit_log_get_response.json b/tests/data/audit_log_get_response.json new file mode 100644 index 0000000..121df23 --- /dev/null +++ b/tests/data/audit_log_get_response.json @@ -0,0 +1,127 @@ +{ + "status": { + "code": 200, + "is_success": true, + "user_message": "Success!", + "developer_message": "" + }, + "data": [ + { + "account_id": 123456, + "service": "SERVICE_DBT_CLOUD", + "source": "SOURCE_CLOUD_UI", + "routing_key": "v1.events.auth.credentialsloginsucceeded", + "actor_type": "ACTOR_USER", + "actor_name": "REDACTED", + "actor_id": 123454, + "logged_at": "2022-05-05 06:51:10+00:00", + "uuid": "8868c439-8928-4e8c-924b-77558d65db0b", + "actor_ip": "REDACTED", + "metadata": { + "auth_credentials": { + "user": { + "id": "REDACTED", + "email": "REDACTED" + } + } + }, + "internal": false, + "id": 1809583, + "state": 1, + "created_at": "2022-05-05 06:51:12.454677+00:00", + "updated_at": "2022-05-05 06:51:12.454677+00:00" + }, + { + "account_id": 123456, + "service": "SERVICE_DBT_CLOUD", + "source": "SOURCE_CLOUD_UI", + "routing_key": "v1.events.auth.credentialsloginsucceeded", + "actor_type": "ACTOR_USER", + "actor_name": "REDACTED", + "actor_id": 123454, + "logged_at": "2022-05-05 05:42:26+00:00", + "uuid": "029625d0-8dcf-4fa7-9567-26f8f4648122", + "actor_ip": "REDACTED", + "metadata": { + "auth_credentials": { + "user": { + "id": "REDACTED", + "email": "REDACTED" + } + } + }, + "internal": false, + "id": 1809076, + "state": 1, + "created_at": "2022-05-05 05:42:30.635635+00:00", + "updated_at": "2022-05-05 05:42:30.635635+00:00" + }, + { + "account_id": 123456, + "service": "SERVICE_DBT_CLOUD", + "source": "SOURCE_CLOUD_UI", + "routing_key": "v1.events.auth.credentialsloginsucceeded", + "actor_type": "ACTOR_USER", + "actor_name": "REDACTED", + "actor_id": 123454, + "logged_at": "2022-05-04 16:13:33+00:00", + "uuid": "ba0c717d-6ea7-4e10-b4ba-48e956163938", + "actor_ip": "REDACTED", + "metadata": { + "auth_credentials": { + "user": { + "id": "REDACTED", + "email": "REDACTED" + } + } + }, + "internal": false, + "id": 1802985, + "state": 1, + "created_at": "2022-05-04 16:13:34.295300+00:00", + "updated_at": "2022-05-04 16:13:34.295300+00:00" + }, + { + "account_id": 123456, + "service": "SERVICE_DBT_CLOUD", + "source": "SOURCE_CLOUD_UI", + "routing_key": "v1.events.auth.credentialsloginsucceeded", + "actor_type": "ACTOR_USER", + "actor_name": "REDACTED", + "actor_id": 123454, + "logged_at": "2022-05-02 06:33:48+00:00", + "uuid": "fe0dfd93-8654-4568-8c13-ecf61eaf9cdb", + "actor_ip": "REDACTED", + "metadata": { + "auth_credentials": { + "user": { + "id": "REDACTED", + "email": "REDACTED" + } + } + }, + "internal": false, + "id": 1770400, + "state": 1, + "created_at": "2022-05-02 06:33:49.777343+00:00", + "updated_at": "2022-05-02 06:33:49.777343+00:00" + } + ], + "extra": { + "filters": { + "account_id": 123456, + "limit": 10, + "offset": 0, + "logged_at__range": [ + "2022-05-01 00:00:00Z", + "2022-05-07 00:00:00Z" + ], + "internal": false + }, + "order_by": "-logged_at", + "pagination": { + "count": 4, + "total_count": 4 + } + } +} \ No newline at end of file