From 37685fcaac823a37c567d5c0f7ba5b59be926883 Mon Sep 17 00:00:00 2001 From: reesercollins <10563996+reesercollins@users.noreply.github.com> Date: Thu, 2 Jun 2022 14:01:55 -0400 Subject: [PATCH 01/17] Inital duplicate functionality --- .../views/CRUD/data/dataset/DatasetList.tsx | 86 ++++++++++++++++++- .../data/dataset/DuplicateDatasetModal.tsx | 63 ++++++++++++++ 2 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 superset-frontend/src/views/CRUD/data/dataset/DuplicateDatasetModal.tsx diff --git a/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx b/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx index a56a69b346c11..9faed6e9c23a1 100644 --- a/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx +++ b/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx @@ -65,6 +65,7 @@ import { PASSWORDS_NEEDED_MESSAGE, CONFIRM_OVERWRITE_MESSAGE, } from './constants'; +import DuplicateDatasetModal from './DuplicateDatasetModal'; const FlexRowContainer = styled.div` align-items: center; @@ -115,6 +116,11 @@ type Dataset = { table_name: string; }; +interface VirtualDataset extends Dataset { + extra: any, + sql: string +} + interface DatasetListProps { addDangerToast: (msg: string) => void; addSuccessToast: (msg: string) => void; @@ -153,6 +159,8 @@ const DatasetList: FunctionComponent = ({ const [datasetCurrentlyEditing, setDatasetCurrentlyEditing] = useState(null); + const [datasetCurrentlyDuplicating, setDatasetCurrentlyDuplicating] = useState(null); + const [importingDataset, showImportModal] = useState(false); const [passwordFields, setPasswordFields] = useState([]); const [preparingExport, setPreparingExport] = useState(false); @@ -175,6 +183,7 @@ const DatasetList: FunctionComponent = ({ const canDelete = hasPerm('can_write'); const canCreate = hasPerm('can_write'); const canExport = hasPerm('can_export'); + const canDuplicate = hasPerm('can_read'); const initialSort = SORT_BY; @@ -229,6 +238,10 @@ const DatasetList: FunctionComponent = ({ ), ), ); + + const openDatasetDuplicateModal = (dataset: VirtualDataset) => { + setDatasetCurrentlyDuplicating(dataset); + } const columns = useMemo( () => [ @@ -374,6 +387,7 @@ const DatasetList: FunctionComponent = ({ const handleEdit = () => openDatasetEditModal(original); const handleDelete = () => openDatasetDeleteModal(original); const handleExport = () => handleBulkDatasetExport([original]); + const handleDuplicate = () => openDatasetDuplicateModal(original); if (!canEdit && !canDelete && !canExport) { return null; } @@ -433,16 +447,32 @@ const DatasetList: FunctionComponent = ({ )} + {canDuplicate && original.kind === 'virtual' && ( + + + + + + )} ); }, Header: t('Actions'), id: 'actions', - hidden: !canEdit && !canDelete, + hidden: !canEdit && !canDelete && !canDuplicate, disableSortBy: true, }, ], - [canEdit, canDelete, canExport, openDatasetEditModal], + [canEdit, canDelete, canExport, openDatasetEditModal, canDuplicate], ); const filterTypes: Filters = useMemo( @@ -582,6 +612,10 @@ const DatasetList: FunctionComponent = ({ setDatasetCurrentlyEditing(null); }; + const closeDatasetDuplicateModal = () => { + setDatasetCurrentlyDuplicating(null); + } + const handleDatasetDelete = ({ id, table_name: tableName }: Dataset) => { SupersetClient.delete({ endpoint: `/api/v1/dataset/${id}`, @@ -625,6 +659,49 @@ const DatasetList: FunctionComponent = ({ setPreparingExport(true); }; + const handleDatasetDuplicate = (newDatasetName: string) => { + if (datasetCurrentlyDuplicating === null) { + addDangerToast(t('There was an issue duplicating the dataset.')); + } + + const { id, schema, sql } = datasetCurrentlyDuplicating as VirtualDataset; + + SupersetClient.get({ + endpoint: `/api/v1/dataset/${id}`, + }).then( + ({ json = {} }) => { + const data = { + schema, + sql, + dbId: datasetCurrentlyDuplicating?.database.id, + templateParams: '', + datasourceName: newDatasetName, + // This is done because the api expects 'name' instead of 'column_name' + columns: json.result.columns.map((e: Record) => { + return { name: e.column_name, ...e } + }), + } + SupersetClient.post({ + endpoint: '/superset/sqllab_viz/', + postPayload: { data }, + }).then(() => { + setDatasetCurrentlyDuplicating(null); + refreshData(); + }, + createErrorHandler(errMsg => + addDangerToast( + t('There was an issue duplicating the selected datasets during POST: %s', errMsg), + ), + )); + }, + createErrorHandler(errMsg => + addDangerToast( + t('There was an issue duplicating the selected datasets during GET: %s', errMsg), + ), + ), + ); + } + return ( <> @@ -659,6 +736,11 @@ const DatasetList: FunctionComponent = ({ show /> )} + void, + onDuplicate: (newDatasetName: string) => void +} + +const DuplicateDatasetModal: FunctionComponent = ({ + dataset, + onHide, + onDuplicate, +}) => { + + const [show, setShow] = useState(false); + const [disableSave, setDisableSave] = useState(false); + const [newDuplicateDatasetName, setNewDuplicateDatasetName] = useState(""); + + const onChange = (event: React.ChangeEvent) => { + const targetValue = event.target.value ?? ''; + setNewDuplicateDatasetName(targetValue); + setDisableSave(targetValue === ''); + }; + + const duplicateDataset = () => { + onDuplicate(newDuplicateDatasetName); + } + + useEffect(() => { + setNewDuplicateDatasetName(""); + setShow(dataset !== null); + }, [dataset]); + + return ( + + + {t('New dataset name')} + + + + ); +} + +export default DuplicateDatasetModal; \ No newline at end of file From 9de47875e3dae9458cbf74feaf3e27dcc53ad739 Mon Sep 17 00:00:00 2001 From: reesercollins <10563996+reesercollins@users.noreply.github.com> Date: Tue, 7 Jun 2022 09:36:44 -0400 Subject: [PATCH 02/17] Fix formatting --- .../views/CRUD/data/dataset/DatasetList.tsx | 52 +++++++++++-------- .../data/dataset/DuplicateDatasetModal.tsx | 34 ++++++------ 2 files changed, 47 insertions(+), 39 deletions(-) diff --git a/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx b/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx index 9faed6e9c23a1..63ea0335b0fa9 100644 --- a/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx +++ b/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx @@ -117,9 +117,9 @@ type Dataset = { }; interface VirtualDataset extends Dataset { - extra: any, - sql: string -} + extra: any; + sql: string; +} interface DatasetListProps { addDangerToast: (msg: string) => void; @@ -159,7 +159,8 @@ const DatasetList: FunctionComponent = ({ const [datasetCurrentlyEditing, setDatasetCurrentlyEditing] = useState(null); - const [datasetCurrentlyDuplicating, setDatasetCurrentlyDuplicating] = useState(null); + const [datasetCurrentlyDuplicating, setDatasetCurrentlyDuplicating] = + useState(null); const [importingDataset, showImportModal] = useState(false); const [passwordFields, setPasswordFields] = useState([]); @@ -238,10 +239,10 @@ const DatasetList: FunctionComponent = ({ ), ), ); - + const openDatasetDuplicateModal = (dataset: VirtualDataset) => { setDatasetCurrentlyDuplicating(dataset); - } + }; const columns = useMemo( () => [ @@ -614,7 +615,7 @@ const DatasetList: FunctionComponent = ({ const closeDatasetDuplicateModal = () => { setDatasetCurrentlyDuplicating(null); - } + }; const handleDatasetDelete = ({ id, table_name: tableName }: Dataset) => { SupersetClient.delete({ @@ -677,30 +678,39 @@ const DatasetList: FunctionComponent = ({ templateParams: '', datasourceName: newDatasetName, // This is done because the api expects 'name' instead of 'column_name' - columns: json.result.columns.map((e: Record) => { - return { name: e.column_name, ...e } - }), - } + columns: json.result.columns.map((e: Record) => ({ + name: e.column_name, + ...e, + })), + }; SupersetClient.post({ endpoint: '/superset/sqllab_viz/', postPayload: { data }, - }).then(() => { - setDatasetCurrentlyDuplicating(null); - refreshData(); - }, - createErrorHandler(errMsg => - addDangerToast( - t('There was an issue duplicating the selected datasets during POST: %s', errMsg), + }).then( + () => { + setDatasetCurrentlyDuplicating(null); + refreshData(); + }, + createErrorHandler(errMsg => + addDangerToast( + t( + 'There was an issue duplicating the selected datasets during POST: %s', + errMsg, + ), + ), ), - )); + ); }, createErrorHandler(errMsg => addDangerToast( - t('There was an issue duplicating the selected datasets during GET: %s', errMsg), + t( + 'There was an issue duplicating the selected datasets during GET: %s', + errMsg, + ), ), ), ); - } + }; return ( <> diff --git a/superset-frontend/src/views/CRUD/data/dataset/DuplicateDatasetModal.tsx b/superset-frontend/src/views/CRUD/data/dataset/DuplicateDatasetModal.tsx index 8e9090286ef96..0885bf03155f0 100644 --- a/superset-frontend/src/views/CRUD/data/dataset/DuplicateDatasetModal.tsx +++ b/superset-frontend/src/views/CRUD/data/dataset/DuplicateDatasetModal.tsx @@ -1,14 +1,14 @@ -import { t } from "@superset-ui/core"; -import { FunctionComponent, useEffect, useState } from "react"; -import { FormLabel } from "src/components/Form"; -import { Input } from "src/components/Input"; -import Modal from "src/components/Modal"; -import Dataset from "src/types/Dataset"; +import { t } from '@superset-ui/core'; +import React, { FunctionComponent, useEffect, useState } from 'react'; +import { FormLabel } from 'src/components/Form'; +import { Input } from 'src/components/Input'; +import Modal from 'src/components/Modal'; +import Dataset from 'src/types/Dataset'; interface DuplicateDatasetModalProps { - dataset: Dataset | null, - onHide: () => void, - onDuplicate: (newDatasetName: string) => void + dataset: Dataset | null; + onHide: () => void; + onDuplicate: (newDatasetName: string) => void; } const DuplicateDatasetModal: FunctionComponent = ({ @@ -16,10 +16,10 @@ const DuplicateDatasetModal: FunctionComponent = ({ onHide, onDuplicate, }) => { - const [show, setShow] = useState(false); const [disableSave, setDisableSave] = useState(false); - const [newDuplicateDatasetName, setNewDuplicateDatasetName] = useState(""); + const [newDuplicateDatasetName, setNewDuplicateDatasetName] = + useState(''); const onChange = (event: React.ChangeEvent) => { const targetValue = event.target.value ?? ''; @@ -29,10 +29,10 @@ const DuplicateDatasetModal: FunctionComponent = ({ const duplicateDataset = () => { onDuplicate(newDuplicateDatasetName); - } + }; useEffect(() => { - setNewDuplicateDatasetName(""); + setNewDuplicateDatasetName(''); setShow(dataset !== null); }, [dataset]); @@ -44,9 +44,7 @@ const DuplicateDatasetModal: FunctionComponent = ({ disablePrimaryButton={disableSave} onHandledPrimaryAction={duplicateDataset} > - - {t('New dataset name')} - + {t('New dataset name')} = ({ /> ); -} +}; -export default DuplicateDatasetModal; \ No newline at end of file +export default DuplicateDatasetModal; From 47b3167a115a8c92ed267b8f16da69fdb9a51319 Mon Sep 17 00:00:00 2001 From: reesercollins <10563996+reesercollins@users.noreply.github.com> Date: Tue, 21 Jun 2022 13:10:26 -0400 Subject: [PATCH 03/17] Create dedicated duplicate API --- docs/static/resources/openapi.json | 1277 ++++++++++++++++++---- superset/dao/base.py | 1 + superset/datasets/api.py | 75 +- superset/datasets/commands/duplicate.py | 115 ++ superset/datasets/commands/exceptions.py | 4 + superset/datasets/schemas.py | 5 + 6 files changed, 1266 insertions(+), 211 deletions(-) create mode 100644 superset/datasets/commands/duplicate.py diff --git a/docs/static/resources/openapi.json b/docs/static/resources/openapi.json index 4eb0cc7c48660..6e6470e50d7a8 100644 --- a/docs/static/resources/openapi.json +++ b/docs/static/resources/openapi.json @@ -93,6 +93,31 @@ } }, "schemas": { + "AdvancedDataTypeSchema": { + "properties": { + "display_value": { + "description": "The string representation of the parsed values", + "type": "string" + }, + "error_message": { + "type": "string" + }, + "valid_filter_operators": { + "items": { + "type": "string" + }, + "type": "array" + }, + "values": { + "items": { + "description": "parsed value (can be any value)", + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, "AnnotationLayer": { "properties": { "annotationType": { @@ -232,7 +257,7 @@ "AnnotationLayerRestApi.get_list": { "properties": { "changed_by": { - "$ref": "#/components/schemas/AnnotationLayerRestApi.get_list.User" + "$ref": "#/components/schemas/AnnotationLayerRestApi.get_list.User1" }, "changed_on": { "format": "date-time", @@ -243,7 +268,7 @@ "readOnly": true }, "created_by": { - "$ref": "#/components/schemas/AnnotationLayerRestApi.get_list.User1" + "$ref": "#/components/schemas/AnnotationLayerRestApi.get_list.User" }, "created_on": { "format": "date-time", @@ -389,13 +414,13 @@ "AnnotationRestApi.get_list": { "properties": { "changed_by": { - "$ref": "#/components/schemas/AnnotationRestApi.get_list.User" + "$ref": "#/components/schemas/AnnotationRestApi.get_list.User1" }, "changed_on_delta_humanized": { "readOnly": true }, "created_by": { - "$ref": "#/components/schemas/AnnotationRestApi.get_list.User1" + "$ref": "#/components/schemas/AnnotationRestApi.get_list.User" }, "end_dttm": { "format": "date-time", @@ -780,8 +805,12 @@ "type": { "description": "Datasource type", "enum": [ - "druid", - "table" + "sl_table", + "table", + "dataset", + "query", + "saved_query", + "view" ], "type": "string" } @@ -1031,22 +1060,24 @@ "operation": { "description": "Post processing operation type", "enum": [ + "_flatten_column_after_pivot", "aggregate", "boxplot", + "compare", "contribution", "cum", + "diff", + "flatten", "geodetic_parse", "geohash_decode", "geohash_encode", "pivot", "prophet", + "rename", + "resample", "rolling", "select", - "sort", - "diff", - "compare", - "resample", - "flatten" + "sort" ], "example": "aggregate", "type": "string" @@ -1533,6 +1564,9 @@ "nullable": true, "type": "string" }, + "is_managed_externally": { + "type": "boolean" + }, "owners": { "$ref": "#/components/schemas/ChartDataRestApi.get.User" }, @@ -1617,7 +1651,7 @@ "type": "string" }, "changed_by": { - "$ref": "#/components/schemas/ChartDataRestApi.get_list.User1" + "$ref": "#/components/schemas/ChartDataRestApi.get_list.User" }, "changed_by_name": { "readOnly": true @@ -1632,7 +1666,7 @@ "readOnly": true }, "created_by": { - "$ref": "#/components/schemas/ChartDataRestApi.get_list.User2" + "$ref": "#/components/schemas/ChartDataRestApi.get_list.User1" }, "datasource_id": { "format": "int32", @@ -1664,13 +1698,16 @@ "format": "int32", "type": "integer" }, + "is_managed_externally": { + "type": "boolean" + }, "last_saved_at": { "format": "date-time", "nullable": true, "type": "string" }, "last_saved_by": { - "$ref": "#/components/schemas/ChartDataRestApi.get_list.User" + "$ref": "#/components/schemas/ChartDataRestApi.get_list.User2" }, "owners": { "$ref": "#/components/schemas/ChartDataRestApi.get_list.User3" @@ -1723,10 +1760,6 @@ "maxLength": 64, "type": "string" }, - "id": { - "format": "int32", - "type": "integer" - }, "last_name": { "maxLength": 64, "type": "string" @@ -1744,6 +1777,10 @@ "maxLength": 64, "type": "string" }, + "id": { + "format": "int32", + "type": "integer" + }, "last_name": { "maxLength": 64, "type": "string" @@ -1841,8 +1878,11 @@ "datasource_type": { "description": "The type of dataset/datasource identified on `datasource_id`.", "enum": [ - "druid", + "sl_table", "table", + "dataset", + "query", + "saved_query", "view" ], "type": "string" @@ -1944,8 +1984,11 @@ "datasource_type": { "description": "The type of dataset/datasource identified on `datasource_id`.", "enum": [ - "druid", + "sl_table", "table", + "dataset", + "query", + "saved_query", "view" ], "nullable": true, @@ -2188,9 +2231,6 @@ "description": "Form data from the Explore controls used to form the chart's data query.", "type": "object" }, - "modified": { - "type": "string" - }, "slice_id": { "format": "int32", "type": "integer" @@ -2282,6 +2322,9 @@ "nullable": true, "type": "string" }, + "is_managed_externally": { + "type": "boolean" + }, "owners": { "$ref": "#/components/schemas/ChartRestApi.get.User" }, @@ -2366,7 +2409,7 @@ "type": "string" }, "changed_by": { - "$ref": "#/components/schemas/ChartRestApi.get_list.User1" + "$ref": "#/components/schemas/ChartRestApi.get_list.User" }, "changed_by_name": { "readOnly": true @@ -2381,7 +2424,7 @@ "readOnly": true }, "created_by": { - "$ref": "#/components/schemas/ChartRestApi.get_list.User2" + "$ref": "#/components/schemas/ChartRestApi.get_list.User1" }, "datasource_id": { "format": "int32", @@ -2413,13 +2456,16 @@ "format": "int32", "type": "integer" }, + "is_managed_externally": { + "type": "boolean" + }, "last_saved_at": { "format": "date-time", "nullable": true, "type": "string" }, "last_saved_by": { - "$ref": "#/components/schemas/ChartRestApi.get_list.User" + "$ref": "#/components/schemas/ChartRestApi.get_list.User2" }, "owners": { "$ref": "#/components/schemas/ChartRestApi.get_list.User3" @@ -2472,10 +2518,6 @@ "maxLength": 64, "type": "string" }, - "id": { - "format": "int32", - "type": "integer" - }, "last_name": { "maxLength": 64, "type": "string" @@ -2493,6 +2535,10 @@ "maxLength": 64, "type": "string" }, + "id": { + "format": "int32", + "type": "integer" + }, "last_name": { "maxLength": 64, "type": "string" @@ -2590,8 +2636,11 @@ "datasource_type": { "description": "The type of dataset/datasource identified on `datasource_id`.", "enum": [ - "druid", + "sl_table", "table", + "dataset", + "query", + "saved_query", "view" ], "type": "string" @@ -2693,8 +2742,11 @@ "datasource_type": { "description": "The type of dataset/datasource identified on `datasource_id`.", "enum": [ - "druid", + "sl_table", "table", + "dataset", + "query", + "saved_query", "view" ], "nullable": true, @@ -2804,13 +2856,13 @@ "CssTemplateRestApi.get_list": { "properties": { "changed_by": { - "$ref": "#/components/schemas/CssTemplateRestApi.get_list.User" + "$ref": "#/components/schemas/CssTemplateRestApi.get_list.User1" }, "changed_on_delta_humanized": { "readOnly": true }, "created_by": { - "$ref": "#/components/schemas/CssTemplateRestApi.get_list.User1" + "$ref": "#/components/schemas/CssTemplateRestApi.get_list.User" }, "created_on": { "format": "date-time", @@ -3087,6 +3139,10 @@ "format": "int32", "type": "integer" }, + "is_managed_externally": { + "nullable": true, + "type": "boolean" + }, "json_metadata": { "description": "This JSON object is generated dynamically when clicking the save or overwrite button in the dashboard view. It is exposed here for reference and for power users who may want to alter specific parameters.", "type": "string" @@ -3126,6 +3182,7 @@ "properties": { "filterState": { "description": "Native filter state", + "nullable": true, "type": "object" }, "hash": { @@ -3143,9 +3200,6 @@ "type": "array" } }, - "required": [ - "filterState" - ], "type": "object" }, "DashboardRestApi.get": { @@ -3185,6 +3239,9 @@ "created_by": { "$ref": "#/components/schemas/DashboardRestApi.get_list.User1" }, + "created_on_delta_humanized": { + "readOnly": true + }, "css": { "nullable": true, "type": "string" @@ -3198,6 +3255,9 @@ "format": "int32", "type": "integer" }, + "is_managed_externally": { + "type": "boolean" + }, "json_metadata": { "nullable": true, "type": "string" @@ -3205,12 +3265,6 @@ "owners": { "$ref": "#/components/schemas/DashboardRestApi.get_list.User2" }, - "advanced_data_type": { - "maxLength": 255, - "minLength": 1, - "nullable": true, - "type": "string" - }, "position_json": { "nullable": true, "type": "string" @@ -3510,6 +3564,14 @@ }, "type": "object" }, + "Database1": { + "properties": { + "database_name": { + "type": "string" + } + }, + "type": "object" + }, "DatabaseFunctionNamesResponse": { "properties": { "function_names": { @@ -3667,6 +3729,9 @@ "nullable": true, "type": "boolean" }, + "is_managed_externally": { + "type": "boolean" + }, "parameters": { "readOnly": true }, @@ -4087,6 +4152,12 @@ }, "DatasetColumnsPut": { "properties": { + "advanced_data_type": { + "maxLength": 255, + "minLength": 1, + "nullable": true, + "type": "string" + }, "column_name": { "maxLength": 255, "minLength": 1, @@ -4181,6 +4252,24 @@ }, "type": "object" }, + "DatasetDuplicateSchema": { + "properties": { + "base_model_id": { + "format": "int32", + "type": "integer" + }, + "table_name": { + "maxLength": 250, + "minLength": 1, + "type": "string" + } + }, + "required": [ + "base_model_id", + "table_name" + ], + "type": "object" + }, "DatasetMetricRestApi.get": { "properties": { "id": { @@ -4251,12 +4340,6 @@ "nullable": true, "type": "string" }, - "advanced_data_type": { - "maxLength": 255, - "minLength": 1, - "nullable": true, - "type": "string" - }, "uuid": { "format": "uuid", "nullable": true, @@ -4395,6 +4478,9 @@ "format": "int32", "type": "integer" }, + "is_managed_externally": { + "type": "boolean" + }, "is_sqllab_view": { "nullable": true, "type": "boolean" @@ -4527,6 +4613,11 @@ }, "DatasetRestApi.get.TableColumn": { "properties": { + "advanced_data_type": { + "maxLength": 255, + "nullable": true, + "type": "string" + }, "changed_on": { "format": "date-time", "nullable": true, @@ -4630,7 +4721,7 @@ "DatasetRestApi.get_list": { "properties": { "changed_by": { - "$ref": "#/components/schemas/DatasetRestApi.get_list.User" + "$ref": "#/components/schemas/DatasetRestApi.get_list.User1" }, "changed_by_name": { "readOnly": true @@ -4673,7 +4764,7 @@ "readOnly": true }, "owners": { - "$ref": "#/components/schemas/DatasetRestApi.get_list.User1" + "$ref": "#/components/schemas/DatasetRestApi.get_list.User" }, "schema": { "maxLength": 255, @@ -4717,6 +4808,14 @@ "maxLength": 64, "type": "string" }, + "id": { + "format": "int32", + "type": "integer" + }, + "last_name": { + "maxLength": 64, + "type": "string" + }, "username": { "maxLength": 64, "type": "string" @@ -4724,6 +4823,7 @@ }, "required": [ "first_name", + "last_name", "username" ], "type": "object" @@ -4734,14 +4834,6 @@ "maxLength": 64, "type": "string" }, - "id": { - "format": "int32", - "type": "integer" - }, - "last_name": { - "maxLength": 64, - "type": "string" - }, "username": { "maxLength": 64, "type": "string" @@ -4749,7 +4841,6 @@ }, "required": [ "first_name", - "last_name", "username" ], "type": "object" @@ -4901,8 +4992,11 @@ "datasource_type": { "description": "The type of dataset/datasource identified on `datasource_id`.", "enum": [ - "druid", + "sl_table", "table", + "dataset", + "query", + "saved_query", "view" ], "type": "string" @@ -4942,6 +5036,80 @@ }, "type": "object" }, + "EmbeddedDashboardConfig": { + "properties": { + "allowed_domains": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "allowed_domains" + ], + "type": "object" + }, + "EmbeddedDashboardResponseSchema": { + "properties": { + "allowed_domains": { + "items": { + "type": "string" + }, + "type": "array" + }, + "changed_by": { + "$ref": "#/components/schemas/User" + }, + "changed_on": { + "format": "date-time", + "type": "string" + }, + "dashboard_id": { + "type": "string" + }, + "uuid": { + "type": "string" + } + }, + "type": "object" + }, + "EmbeddedDashboardRestApi.get": { + "properties": { + "uuid": { + "format": "uuid", + "type": "string" + } + }, + "type": "object" + }, + "EmbeddedDashboardRestApi.get_list": { + "properties": { + "uuid": { + "format": "uuid", + "type": "string" + } + }, + "type": "object" + }, + "EmbeddedDashboardRestApi.post": { + "properties": { + "uuid": { + "format": "uuid", + "type": "string" + } + }, + "type": "object" + }, + "EmbeddedDashboardRestApi.put": { + "properties": { + "uuid": { + "format": "uuid", + "type": "string" + } + }, + "type": "object" + }, "ExplorePermalinkPostSchema": { "properties": { "formData": { @@ -5125,18 +5293,31 @@ "format": "int32", "type": "integer" }, - "dataset_id": { - "description": "The dataset ID", + "datasource_id": { + "description": "The datasource ID", "format": "int32", "type": "integer" }, + "datasource_type": { + "description": "The datasource type", + "enum": [ + "sl_table", + "table", + "dataset", + "query", + "saved_query", + "view" + ], + "type": "string" + }, "form_data": { "description": "Any type of JSON supported text.", "type": "string" } }, "required": [ - "dataset_id", + "datasource_id", + "datasource_type", "form_data" ], "type": "object" @@ -5148,18 +5329,31 @@ "format": "int32", "type": "integer" }, - "dataset_id": { - "description": "The dataset ID", + "datasource_id": { + "description": "The datasource ID", "format": "int32", "type": "integer" }, + "datasource_type": { + "description": "The datasource type", + "enum": [ + "sl_table", + "table", + "dataset", + "query", + "saved_query", + "view" + ], + "type": "string" + }, "form_data": { "description": "Any type of JSON supported text.", "type": "string" } }, "required": [ - "dataset_id", + "datasource_id", + "datasource_type", "form_data" ], "type": "object" @@ -5481,18 +5675,16 @@ "properties": { "changed_on": { "format": "date-time", - "nullable": true, "type": "string" }, "database": { - "$ref": "#/components/schemas/QueryRestApi.get_list.Database" + "$ref": "#/components/schemas/Database1" }, "end_time": { - "nullable": true, + "format": "float", "type": "number" }, "executed_sql": { - "nullable": true, "type": "string" }, "id": { @@ -5501,89 +5693,37 @@ }, "rows": { "format": "int32", - "nullable": true, "type": "integer" }, "schema": { - "maxLength": 256, - "nullable": true, "type": "string" }, "sql": { - "nullable": true, "type": "string" }, "sql_tables": { "readOnly": true }, "start_time": { - "nullable": true, + "format": "float", "type": "number" }, "status": { - "maxLength": 16, - "nullable": true, "type": "string" }, "tab_name": { - "maxLength": 256, - "nullable": true, "type": "string" }, "tmp_table_name": { - "maxLength": 256, - "nullable": true, "type": "string" }, "tracking_url": { - "nullable": true, "type": "string" }, "user": { - "$ref": "#/components/schemas/QueryRestApi.get_list.User" - } - }, - "required": [ - "database" - ], - "type": "object" - }, - "QueryRestApi.get_list.Database": { - "properties": { - "database_name": { - "maxLength": 250, - "type": "string" - } - }, - "required": [ - "database_name" - ], - "type": "object" - }, - "QueryRestApi.get_list.User": { - "properties": { - "first_name": { - "maxLength": 64, - "type": "string" - }, - "id": { - "format": "int32", - "type": "integer" - }, - "last_name": { - "maxLength": 64, - "type": "string" - }, - "username": { - "maxLength": 64, - "type": "string" + "$ref": "#/components/schemas/User" } }, - "required": [ - "first_name", - "last_name", - "username" - ], "type": "object" }, "QueryRestApi.post": { @@ -6003,6 +6143,11 @@ "changed_on_delta_humanized": { "readOnly": true }, + "chart_id": { + "format": "int32", + "nullable": true, + "type": "integer" + }, "created_by": { "$ref": "#/components/schemas/ReportScheduleRestApi.get_list.User1" }, @@ -6023,6 +6168,11 @@ "crontab_humanized": { "readOnly": true }, + "dashboard_id": { + "format": "int32", + "nullable": true, + "type": "integer" + }, "description": { "nullable": true, "type": "string" @@ -6787,6 +6937,7 @@ "Pacific/Guam", "Pacific/Honolulu", "Pacific/Johnston", + "Pacific/Kanton", "Pacific/Kiritimati", "Pacific/Kosrae", "Pacific/Kwajalein", @@ -7511,6 +7662,7 @@ "Pacific/Guam", "Pacific/Honolulu", "Pacific/Johnston", + "Pacific/Kanton", "Pacific/Kiritimati", "Pacific/Kosrae", "Pacific/Kwajalein", @@ -7882,6 +8034,20 @@ }, "type": "object" }, + "TableExtraMetadataResponseSchema": { + "properties": { + "clustering": { + "type": "object" + }, + "metadata": { + "type": "object" + }, + "partitions": { + "type": "object" + } + }, + "type": "object" + }, "TableMetadataColumnsResponse": { "properties": { "duplicates_constraint": { @@ -8107,6 +8273,46 @@ }, "type": "object" }, + "ValidateSQLRequest": { + "properties": { + "schema": { + "nullable": true, + "type": "string" + }, + "sql": { + "description": "SQL statement to validate", + "type": "string" + }, + "template_params": { + "nullable": true, + "type": "object" + } + }, + "required": [ + "sql" + ], + "type": "object" + }, + "ValidateSQLResponse": { + "properties": { + "end_column": { + "format": "int32", + "type": "integer" + }, + "line_number": { + "format": "int32", + "type": "integer" + }, + "message": { + "type": "string" + }, + "start_column": { + "format": "int32", + "type": "integer" + } + }, + "type": "object" + }, "ValidatorConfigJSON": { "properties": { "op": { @@ -8128,6 +8334,26 @@ }, "type": "object" }, + "advanced_data_type_convert_schema": { + "properties": { + "type": { + "default": "port", + "type": "string" + }, + "values": { + "items": { + "default": "http" + }, + "minItems": 1, + "type": "array" + } + }, + "required": [ + "type", + "values" + ], + "type": "object" + }, "database_schemas_query_schema": { "properties": { "force": { @@ -8371,6 +8597,98 @@ }, "openapi": "3.0.2", "paths": { + "/api/v1/advanced_data_type/convert": { + "get": { + "parameters": [ + { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/advanced_data_type_convert_schema" + } + } + }, + "in": "query", + "name": "q" + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AdvancedDataTypeSchema" + } + } + }, + "description": "AdvancedDataTypeResponse object has been returned." + }, + "400": { + "$ref": "#/components/responses/400" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "404": { + "$ref": "#/components/responses/404" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "summary": "Returns a AdvancedDataTypeResponse object populated with the passed in args.", + "tags": [ + "Advanced Data Type" + ] + } + }, + "/api/v1/advanced_data_type/types": { + "get": { + "description": "Returns a list of available advanced data types.", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "result": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + } + } + }, + "description": "a successful return of the available advanced data types has taken place." + }, + "401": { + "$ref": "#/components/responses/401" + }, + "404": { + "$ref": "#/components/responses/404" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Advanced Data Type" + ] + } + }, "/api/v1/annotation_layer/": { "delete": { "description": "Deletes multiple annotation layers in a bulk operation.", @@ -9377,9 +9695,6 @@ }, "description": "ZIP file" }, - "400": { - "$ref": "#/components/responses/400" - }, "401": { "$ref": "#/components/responses/401" }, @@ -9437,7 +9752,7 @@ } } }, - "description": "Dashboard import result" + "description": "Assets import result" }, "400": { "$ref": "#/components/responses/400" @@ -10457,7 +10772,7 @@ } ], "responses": { - "200": { + "202": { "content": { "application/json": { "schema": { @@ -10467,9 +10782,6 @@ }, "description": "Chart async result" }, - "302": { - "description": "Redirects to the current digest" - }, "400": { "$ref": "#/components/responses/400" }, @@ -10597,9 +10909,6 @@ }, "description": "Chart thumbnail image" }, - "302": { - "description": "Redirects to the current digest" - }, "400": { "$ref": "#/components/responses/400" }, @@ -11450,9 +11759,6 @@ }, "description": "Dashboard added" }, - "302": { - "description": "Redirects to the current digest" - }, "400": { "$ref": "#/components/responses/400" }, @@ -12160,9 +12466,6 @@ }, "description": "Dashboard" }, - "302": { - "description": "Redirects to the current digest" - }, "400": { "$ref": "#/components/responses/400" }, @@ -12218,9 +12521,6 @@ }, "description": "Dashboard chart definitions" }, - "302": { - "description": "Redirects to the current digest" - }, "400": { "$ref": "#/components/responses/400" }, @@ -12277,9 +12577,6 @@ }, "description": "Dashboard dataset definitions" }, - "302": { - "description": "Redirects to the current digest" - }, "400": { "$ref": "#/components/responses/400" }, @@ -12303,16 +12600,17 @@ ] } }, - "/api/v1/dashboard/{pk}": { + "/api/v1/dashboard/{id_or_slug}/embedded": { "delete": { - "description": "Deletes a Dashboard.", + "description": "Removes a dashboard's embedded configuration.", "parameters": [ { + "description": "The dashboard id or slug", "in": "path", - "name": "pk", + "name": "id_or_slug", "required": true, "schema": { - "type": "integer" + "type": "string" } } ], @@ -12330,20 +12628,11 @@ } } }, - "description": "Dashboard deleted" + "description": "Successfully removed the configuration" }, "401": { "$ref": "#/components/responses/401" }, - "403": { - "$ref": "#/components/responses/403" - }, - "404": { - "$ref": "#/components/responses/404" - }, - "422": { - "$ref": "#/components/responses/422" - }, "500": { "$ref": "#/components/responses/500" } @@ -12357,25 +12646,237 @@ "Dashboards" ] }, - "put": { - "description": "Changes a Dashboard.", + "get": { + "description": "Returns the dashboard's embedded configuration", "parameters": [ { + "description": "The dashboard id or slug", "in": "path", - "name": "pk", + "name": "id_or_slug", "required": true, "schema": { - "type": "integer" + "type": "string" } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DashboardRestApi.put" - } - } + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "result": { + "$ref": "#/components/schemas/EmbeddedDashboardResponseSchema" + } + }, + "type": "object" + } + } + }, + "description": "Result contains the embedded dashboard config" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Dashboards" + ] + }, + "post": { + "description": "Sets a dashboard's embedded configuration.", + "parameters": [ + { + "description": "The dashboard id or slug", + "in": "path", + "name": "id_or_slug", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EmbeddedDashboardConfig" + } + } + }, + "description": "The embedded configuration to set", + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "result": { + "$ref": "#/components/schemas/EmbeddedDashboardResponseSchema" + } + }, + "type": "object" + } + } + }, + "description": "Successfully set the configuration" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Dashboards" + ] + }, + "put": { + "description": "Sets a dashboard's embedded configuration.", + "parameters": [ + { + "description": "The dashboard id or slug", + "in": "path", + "name": "id_or_slug", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EmbeddedDashboardConfig" + } + } + }, + "description": "The embedded configuration to set", + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "result": { + "$ref": "#/components/schemas/EmbeddedDashboardResponseSchema" + } + }, + "type": "object" + } + } + }, + "description": "Successfully set the configuration" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Dashboards" + ] + } + }, + "/api/v1/dashboard/{pk}": { + "delete": { + "description": "Deletes a Dashboard.", + "parameters": [ + { + "in": "path", + "name": "pk", + "required": true, + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "message": { + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Dashboard deleted" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "403": { + "$ref": "#/components/responses/403" + }, + "404": { + "$ref": "#/components/responses/404" + }, + "422": { + "$ref": "#/components/responses/422" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Dashboards" + ] + }, + "put": { + "description": "Changes a Dashboard.", + "parameters": [ + { + "in": "path", + "name": "pk", + "required": true, + "schema": { + "type": "integer" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DashboardRestApi.put" + } + } }, "description": "Dashboard schema", "required": true @@ -12834,6 +13335,9 @@ }, "description": "Thumbnail does not exist on cache, fired async to compute" }, + "302": { + "description": "Redirects to the current digest" + }, "401": { "$ref": "#/components/responses/401" }, @@ -12997,9 +13501,6 @@ }, "description": "Database added" }, - "302": { - "description": "Redirects to the current digest" - }, "400": { "$ref": "#/components/responses/400" }, @@ -13764,7 +14265,145 @@ }, "/api/v1/database/{pk}/select_star/{table_name}/": { "get": { - "description": "Get database select star for table", + "description": "Get database select star for table", + "parameters": [ + { + "description": "The database id", + "in": "path", + "name": "pk", + "required": true, + "schema": { + "type": "integer" + } + }, + { + "description": "Table name", + "in": "path", + "name": "table_name", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "Table schema", + "in": "path", + "name": "schema_name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SelectStarResponseSchema" + } + } + }, + "description": "SQL statement for a select star for table" + }, + "400": { + "$ref": "#/components/responses/400" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "404": { + "$ref": "#/components/responses/404" + }, + "422": { + "$ref": "#/components/responses/422" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Database" + ] + } + }, + "/api/v1/database/{pk}/select_star/{table_name}/{schema_name}/": { + "get": { + "description": "Get database select star for table", + "parameters": [ + { + "description": "The database id", + "in": "path", + "name": "pk", + "required": true, + "schema": { + "type": "integer" + } + }, + { + "description": "Table name", + "in": "path", + "name": "table_name", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "Table schema", + "in": "path", + "name": "schema_name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SelectStarResponseSchema" + } + } + }, + "description": "SQL statement for a select star for table" + }, + "400": { + "$ref": "#/components/responses/400" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "404": { + "$ref": "#/components/responses/404" + }, + "422": { + "$ref": "#/components/responses/422" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Database" + ] + } + }, + "/api/v1/database/{pk}/table/{table_name}/{schema_name}/": { + "get": { + "description": "Get database table metadata", "parameters": [ { "description": "The database id", @@ -13799,11 +14438,11 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SelectStarResponseSchema" + "$ref": "#/components/schemas/TableMetadataResponseSchema" } } }, - "description": "SQL statement for a select star for table" + "description": "Table metadata information" }, "400": { "$ref": "#/components/responses/400" @@ -13831,9 +14470,9 @@ ] } }, - "/api/v1/database/{pk}/select_star/{table_name}/{schema_name}/": { + "/api/v1/database/{pk}/table_extra/{table_name}/{schema_name}/": { "get": { - "description": "Get database select star for table", + "description": "Response depends on each DB engine spec normally focused on partitions", "parameters": [ { "description": "The database id", @@ -13868,11 +14507,11 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SelectStarResponseSchema" + "$ref": "#/components/schemas/TableExtraMetadataResponseSchema" } } }, - "description": "SQL statement for a select star for table" + "description": "Table extra metadata information" }, "400": { "$ref": "#/components/responses/400" @@ -13895,53 +14534,55 @@ "jwt": [] } ], + "summary": "Get table extra metadata", "tags": [ "Database" ] } }, - "/api/v1/database/{pk}/table/{table_name}/{schema_name}/": { - "get": { - "description": "Get database table metadata", + "/api/v1/database/{pk}/validate_sql": { + "post": { + "description": "Validates arbitrary SQL.", "parameters": [ { - "description": "The database id", "in": "path", "name": "pk", "required": true, "schema": { "type": "integer" } - }, - { - "description": "Table name", - "in": "path", - "name": "table_name", - "required": true, - "schema": { - "type": "string" - } - }, - { - "description": "Table schema", - "in": "path", - "name": "schema_name", - "required": true, - "schema": { - "type": "string" - } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidateSQLRequest" + } + } + }, + "description": "Validate SQL request", + "required": true + }, "responses": { "200": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/TableMetadataResponseSchema" + "properties": { + "result": { + "description": "A List of SQL errors found on the statement", + "items": { + "$ref": "#/components/schemas/ValidateSQLResponse" + }, + "type": "array" + } + }, + "type": "object" } } }, - "description": "Table metadata information" + "description": "Validation result" }, "400": { "$ref": "#/components/responses/400" @@ -13952,9 +14593,6 @@ "404": { "$ref": "#/components/responses/404" }, - "422": { - "$ref": "#/components/responses/422" - }, "500": { "$ref": "#/components/responses/500" } @@ -13964,6 +14602,7 @@ "jwt": [] } ], + "summary": "Validates that arbitrary sql is acceptable for the given database", "tags": [ "Database" ] @@ -14334,6 +14973,75 @@ ] } }, + "/api/v1/dataset/duplicate": { + "post": { + "description": "Duplicates a Dataset", + "parameters": [ + { + "in": "path", + "name": "pk", + "required": true, + "schema": { + "type": "integer" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DatasetDuplicateSchema" + } + } + }, + "description": "Dataset schema", + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "message": { + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Dataset duplicate" + }, + "400": { + "$ref": "#/components/responses/400" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "403": { + "$ref": "#/components/responses/403" + }, + "404": { + "$ref": "#/components/responses/404" + }, + "422": { + "$ref": "#/components/responses/422" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Datasets" + ] + } + }, "/api/v1/dataset/export/": { "get": { "description": "Exports multiple datasets and downloads them as YAML files", @@ -14968,6 +15676,118 @@ ] } }, + "/api/v1/dataset/{pk}/samples": { + "get": { + "description": "get samples from a Dataset", + "parameters": [ + { + "in": "path", + "name": "pk", + "required": true, + "schema": { + "type": "integer" + } + }, + { + "in": "query", + "name": "force", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "result": { + "$ref": "#/components/schemas/ChartDataResponseResult" + } + }, + "type": "object" + } + } + }, + "description": "Dataset samples" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "403": { + "$ref": "#/components/responses/403" + }, + "404": { + "$ref": "#/components/responses/404" + }, + "422": { + "$ref": "#/components/responses/422" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Datasets" + ] + } + }, + "/api/v1/embedded_dashboard/{uuid}": { + "get": { + "description": "Get a report schedule log", + "parameters": [ + { + "description": "The embedded configuration uuid", + "in": "path", + "name": "uuid", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "result": { + "$ref": "#/components/schemas/EmbeddedDashboardResponseSchema" + } + }, + "type": "object" + } + } + }, + "description": "Result contains the embedded dashboard configuration" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "404": { + "$ref": "#/components/responses/404" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Embedded Dashboard" + ] + } + }, "/api/v1/explore/form_data": { "post": { "description": "Stores a new form_data.", @@ -15617,6 +16437,34 @@ ] } }, + "/api/v1/me/roles/": { + "get": { + "description": "Returns the user roles corresponding to the agent making the request, or returns a 401 error if the user is unauthenticated.", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "result": { + "$ref": "#/components/schemas/UserResponseSchema" + } + }, + "type": "object" + } + } + }, + "description": "The current user" + }, + "401": { + "$ref": "#/components/responses/401" + } + }, + "tags": [ + "Current User" + ] + } + }, "/api/v1/menu/": { "get": { "description": "Get the menu data structure. Returns a forest like structure with the menu the user has access to", @@ -16208,6 +17056,9 @@ "404": { "$ref": "#/components/responses/404" }, + "422": { + "$ref": "#/components/responses/422" + }, "500": { "$ref": "#/components/responses/500" } @@ -16572,6 +17423,9 @@ "404": { "$ref": "#/components/responses/404" }, + "422": { + "$ref": "#/components/responses/422" + }, "500": { "$ref": "#/components/responses/500" } @@ -17557,6 +18411,9 @@ }, "description": "Result contains the guest token" }, + "400": { + "$ref": "#/components/responses/400" + }, "401": { "$ref": "#/components/responses/401" }, diff --git a/superset/dao/base.py b/superset/dao/base.py index 0090c4e535e23..ea01a93842281 100644 --- a/superset/dao/base.py +++ b/superset/dao/base.py @@ -121,6 +121,7 @@ def create(cls, properties: Dict[str, Any], commit: bool = True) -> Model: raise DAOConfigError() model = cls.model_cls() # pylint: disable=not-callable for key, value in properties.items(): + print(key, value) setattr(model, key, value) try: db.session.add(model) diff --git a/superset/datasets/api.py b/superset/datasets/api.py index 17e99959e9675..9c9f06c1e89e1 100644 --- a/superset/datasets/api.py +++ b/superset/datasets/api.py @@ -20,8 +20,10 @@ from io import BytesIO from typing import Any from zipfile import is_zipfile, ZipFile +from numpy import save import simplejson +from superset.datasets.commands.duplicate import DuplicateDatasetCommand import yaml from flask import g, make_response, request, Response, send_file from flask_appbuilder.api import expose, protect, rison, safe @@ -57,6 +59,7 @@ from superset.datasets.dao import DatasetDAO from superset.datasets.filters import DatasetCertifiedFilter, DatasetIsNullOrEmptyFilter from superset.datasets.schemas import ( + DatasetDuplicateSchema, DatasetPostSchema, DatasetPutSchema, DatasetRelatedObjectsResponse, @@ -93,6 +96,7 @@ class DatasetRestApi(BaseSupersetModelRestApi): "bulk_delete", "refresh", "related_objects", + "duplicate", "samples", } list_columns = [ @@ -175,6 +179,7 @@ class DatasetRestApi(BaseSupersetModelRestApi): ] add_model_schema = DatasetPostSchema() edit_model_schema = DatasetPutSchema() + duplicate_model_schema = DatasetDuplicateSchema() add_columns = ["database", "schema", "table_name", "owners"] edit_columns = [ "table_name", @@ -211,7 +216,10 @@ class DatasetRestApi(BaseSupersetModelRestApi): apispec_parameter_schemas = { "get_export_ids_schema": get_export_ids_schema, } - openapi_spec_component_schemas = (DatasetRelatedObjectsResponse,) + openapi_spec_component_schemas = ( + DatasetRelatedObjectsResponse, + DatasetDuplicateSchema, + ) @expose("/", methods=["POST"]) @protect() @@ -505,6 +513,71 @@ def export(self, **kwargs: Any) -> Response: mimetype="application/text", ) + @expose("/duplicate", methods=["POST"]) + @protect() + @safe + @statsd_metrics + @event_logger.log_this_with_context( + action=lambda self, *args, **kwargs: f"{self.__class__.__name__}" f".duplicate", + log_to_statsd=False, + ) + @requires_json + def duplicate(self, **kwargs: Any) -> Response: + """Duplicates a Dataset + --- + post: + description: >- + Duplicates a Dataset + requestBody: + description: Dataset schema + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/DatasetDuplicateSchema' + responses: + 200: + description: Dataset duplicate + content: + application/json: + schema: + type: object + properties: + message: + type: string + 400: + $ref: '#/components/responses/400' + 401: + $ref: '#/components/responses/401' + 403: + $ref: '#/components/responses/403' + 404: + $ref: '#/components/responses/404' + 422: + $ref: '#/components/responses/422' + 500: + $ref: '#/components/responses/500' + """ + try: + item = self.duplicate_model_schema.load(request.json) + # This validates custom Schema with custom validations + except ValidationError as error: + return self.response_400(message=error.messages) + + try: + new_model = DuplicateDatasetCommand(g.user, item).run() + return self.response(201, id=new_model.id, result=item) + except DatasetInvalidError as ex: + return self.response_422(message=ex.normalized_messages()) + except DatasetCreateFailedError as ex: + logger.error( + "Error creating model %s: %s", + self.__class__.__name__, + str(ex), + exc_info=True, + ) + return self.response_422(message=str(ex)) + @expose("//refresh", methods=["PUT"]) @protect() @safe diff --git a/superset/datasets/commands/duplicate.py b/superset/datasets/commands/duplicate.py new file mode 100644 index 0000000000000..9a9e07dc6c902 --- /dev/null +++ b/superset/datasets/commands/duplicate.py @@ -0,0 +1,115 @@ +import logging +from typing import Any, Dict, List + +from flask_appbuilder.models.sqla import Model +from flask_appbuilder.security.sqla.models import User +from flask_babel import gettext as __ +from marshmallow import ValidationError +from sqlalchemy.exc import SQLAlchemyError + +from superset.commands.base import BaseCommand, CreateMixin +from superset.commands.exceptions import DatasourceTypeInvalidError +from superset.connectors.sqla.models import SqlMetric, SqlaTable, TableColumn +from superset.dao.exceptions import DAOCreateFailedError +from superset.datasets.commands.exceptions import ( + DatasetDuplicateFailedError, + DatasetExistsValidationError, + DatasetInvalidError, + DatasetNotFoundError, +) +from superset.datasets.dao import DatasetDAO +from superset.errors import ErrorLevel, SupersetError, SupersetErrorType +from superset.exceptions import SupersetErrorException +from superset.extensions import db, security_manager +from superset.models.core import Database +from superset.sql_parse import ParsedQuery + +logger = logging.getLogger(__name__) + + +class DuplicateDatasetCommand(CreateMixin, BaseCommand): + def __init__(self, user: User, data: Dict[str, Any]): + self._actor = user + self._properties = data.copy() + + def run(self) -> Model: + self.validate() + try: + database_id = self._base_model.database_id + table_name = self._properties["table_name"] + owners = self._properties["owners"] + database = db.session.query(Database).get(database_id) + if not database: + raise SupersetErrorException( + SupersetError( + message=__("The database was not found."), + error_type=SupersetErrorType.DATABASE_NOT_FOUND_ERROR, + level=ErrorLevel.ERROR, + ), + status=404, + ) + table = SqlaTable(table_name=table_name, owners=owners) + table.database = database + table.schema = self._base_model.schema + table.template_params = self._base_model.template_params + table.is_sqllab_view = True + table.sql = ParsedQuery(self._base_model.sql).stripped() + db.session.add(table) + cols = [] + for config_ in self._base_model.columns: + column_name = config_.column_name + col = TableColumn( + column_name=column_name, + verbose_name=config_.verbose_name, + filterable=True, + groupby=True, + is_dttm=config_.is_dttm, + type=config_.type, + ) + cols.append(col) + table.columns = cols + mets = [] + for config_ in self._base_model.metrics: + metric_name = config_.metric_name + met = SqlMetric( + metric_name=metric_name, + verbose_name=config_.verbose_name, + expression=config_.expression, + metric_type=config_.metric_type, + description=config_.description, + ) + mets.append(met) + table.metrics = mets + db.session.commit() + except (SQLAlchemyError, DAOCreateFailedError) as ex: + logger.warning(ex, exc_info=True) + db.session.rollback() + raise DatasetDuplicateFailedError() from ex + return table + + def validate(self) -> None: + exceptions: List[ValidationError] = [] + base_model_id = self._properties["base_model_id"] + duplicate_name = self._properties["table_name"] + + self._base_model = DatasetDAO.find_by_id(base_model_id) + if not self._base_model: + exceptions.append(DatasetNotFoundError()) + + if self._base_model.kind != "virtual": + exceptions.append(DatasourceTypeInvalidError()) + + if DatasetDAO.find_one_or_none(table_name=duplicate_name): + exceptions.append(DatasetExistsValidationError(table_name=duplicate_name)) + + try: + owners = self.populate_owners(self._actor) + self._properties["owners"] = owners + except ValidationError as ex: + exceptions.append(ex) + + if exceptions: + exception = DatasetInvalidError() + exception.add_list(exceptions) + print(exception) + raise exception diff --git a/superset/datasets/commands/exceptions.py b/superset/datasets/commands/exceptions.py index b743a4355ea06..c76b7b3ad53dc 100644 --- a/superset/datasets/commands/exceptions.py +++ b/superset/datasets/commands/exceptions.py @@ -187,3 +187,7 @@ class DatasetImportError(ImportFailedError): class DatasetAccessDeniedError(ForbiddenError): message = _("You don't have access to this dataset.") + + +class DatasetDuplicateFailedError(CreateFailedError): + message = _("Dataset could not be duplicated.") diff --git a/superset/datasets/schemas.py b/superset/datasets/schemas.py index 8a44da458f564..a2647e3e0b3fc 100644 --- a/superset/datasets/schemas.py +++ b/superset/datasets/schemas.py @@ -107,6 +107,11 @@ class DatasetPutSchema(Schema): external_url = fields.String(allow_none=True) +class DatasetDuplicateSchema(Schema): + base_model_id = fields.Integer(required=True) + table_name = fields.String(required=True, allow_none=False, validate=Length(1, 250)) + + class DatasetRelatedChart(Schema): id = fields.Integer() slice_name = fields.String() From 261b19b4371e8c70d7c7b43d1fbd70df043f20af Mon Sep 17 00:00:00 2001 From: reesercollins <10563996+reesercollins@users.noreply.github.com> Date: Tue, 21 Jun 2022 13:25:05 -0400 Subject: [PATCH 04/17] Make use of new API --- .../views/CRUD/data/dataset/DatasetList.tsx | 48 ++++--------------- 1 file changed, 10 insertions(+), 38 deletions(-) diff --git a/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx b/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx index 63ea0335b0fa9..63490e6ad2111 100644 --- a/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx +++ b/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx @@ -665,48 +665,20 @@ const DatasetList: FunctionComponent = ({ addDangerToast(t('There was an issue duplicating the dataset.')); } - const { id, schema, sql } = datasetCurrentlyDuplicating as VirtualDataset; - - SupersetClient.get({ - endpoint: `/api/v1/dataset/${id}`, + SupersetClient.post({ + endpoint: `/api/v1/dataset/duplicate`, + postPayload: { + base_model_id: datasetCurrentlyDuplicating?.id, + table_name: newDatasetName, + }, }).then( - ({ json = {} }) => { - const data = { - schema, - sql, - dbId: datasetCurrentlyDuplicating?.database.id, - templateParams: '', - datasourceName: newDatasetName, - // This is done because the api expects 'name' instead of 'column_name' - columns: json.result.columns.map((e: Record) => ({ - name: e.column_name, - ...e, - })), - }; - SupersetClient.post({ - endpoint: '/superset/sqllab_viz/', - postPayload: { data }, - }).then( - () => { - setDatasetCurrentlyDuplicating(null); - refreshData(); - }, - createErrorHandler(errMsg => - addDangerToast( - t( - 'There was an issue duplicating the selected datasets during POST: %s', - errMsg, - ), - ), - ), - ); + () => { + setDatasetCurrentlyDuplicating(null); + refreshData(); }, createErrorHandler(errMsg => addDangerToast( - t( - 'There was an issue duplicating the selected datasets during GET: %s', - errMsg, - ), + t('There was an issue duplicating the selected datasets: %s', errMsg), ), ), ); From 9ce8da30710c8178cf544360502ac23309265947 Mon Sep 17 00:00:00 2001 From: reesercollins <10563996+reesercollins@users.noreply.github.com> Date: Wed, 22 Jun 2022 12:32:12 -0400 Subject: [PATCH 05/17] Make use of new api permissions --- superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx b/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx index 63490e6ad2111..9fb2c02af25e6 100644 --- a/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx +++ b/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx @@ -184,7 +184,7 @@ const DatasetList: FunctionComponent = ({ const canDelete = hasPerm('can_write'); const canCreate = hasPerm('can_write'); const canExport = hasPerm('can_export'); - const canDuplicate = hasPerm('can_read'); + const canDuplicate = hasPerm('can_duplicate'); const initialSort = SORT_BY; @@ -389,7 +389,7 @@ const DatasetList: FunctionComponent = ({ const handleDelete = () => openDatasetDeleteModal(original); const handleExport = () => handleBulkDatasetExport([original]); const handleDuplicate = () => openDatasetDuplicateModal(original); - if (!canEdit && !canDelete && !canExport) { + if (!canEdit && !canDelete && !canExport && !canDuplicate) { return null; } return ( From c89cc2a1a6f1dd55266b6cbe2e8428d28a7585b3 Mon Sep 17 00:00:00 2001 From: reesercollins <10563996+reesercollins@users.noreply.github.com> Date: Wed, 22 Jun 2022 13:58:58 -0400 Subject: [PATCH 06/17] Add integration tests for duplicating datasets --- superset/datasets/api.py | 6 +- superset/datasets/commands/duplicate.py | 3 +- tests/integration_tests/datasets/api_tests.py | 80 ++++++++++++++++++- 3 files changed, 85 insertions(+), 4 deletions(-) diff --git a/superset/datasets/api.py b/superset/datasets/api.py index 9c9f06c1e89e1..d08f0abdb0269 100644 --- a/superset/datasets/api.py +++ b/superset/datasets/api.py @@ -568,7 +568,11 @@ def duplicate(self, **kwargs: Any) -> Response: new_model = DuplicateDatasetCommand(g.user, item).run() return self.response(201, id=new_model.id, result=item) except DatasetInvalidError as ex: - return self.response_422(message=ex.normalized_messages()) + return self.response_422( + message=ex.normalized_messages() + if isinstance(ex, ValidationError) + else str(ex) + ) except DatasetCreateFailedError as ex: logger.error( "Error creating model %s: %s", diff --git a/superset/datasets/commands/duplicate.py b/superset/datasets/commands/duplicate.py index 9a9e07dc6c902..1c2f1d747d2cb 100644 --- a/superset/datasets/commands/duplicate.py +++ b/superset/datasets/commands/duplicate.py @@ -96,7 +96,7 @@ def validate(self) -> None: if not self._base_model: exceptions.append(DatasetNotFoundError()) - if self._base_model.kind != "virtual": + if self._base_model and self._base_model.kind != "virtual": exceptions.append(DatasourceTypeInvalidError()) if DatasetDAO.find_one_or_none(table_name=duplicate_name): @@ -111,5 +111,4 @@ def validate(self) -> None: if exceptions: exception = DatasetInvalidError() exception.add_list(exceptions) - print(exception) raise exception diff --git a/tests/integration_tests/datasets/api_tests.py b/tests/integration_tests/datasets/api_tests.py index 28bb617c17c19..bfbeef1bfbe7a 100644 --- a/tests/integration_tests/datasets/api_tests.py +++ b/tests/integration_tests/datasets/api_tests.py @@ -100,6 +100,13 @@ def get_fixture_datasets(self) -> List[SqlaTable]: .all() ) + def get_fixture_virtual_datasets(self) -> List[SqlaTable]: + return ( + db.session.query(SqlaTable) + .filter(SqlaTable.table_name.in_(self.fixture_virtual_table_names)) + .all() + ) + @pytest.fixture() def create_virtual_datasets(self): with self.create_app().app_context(): @@ -406,7 +413,12 @@ def test_info_security_dataset(self): rv = self.get_assert_metric(uri, "info") data = json.loads(rv.data.decode("utf-8")) assert rv.status_code == 200 - assert set(data["permissions"]) == {"can_read", "can_write", "can_export"} + assert set(data["permissions"]) == { + "can_read", + "can_write", + "can_export", + "can_duplicate", + } def test_create_dataset_item(self): """ @@ -1955,3 +1967,69 @@ def test_get_dataset_samples_on_virtual_dataset(self): db.session.delete(virtual_dataset) db.session.commit() + + @pytest.mark.usefixtures("create_virtual_datasets") + def test_duplicate_virtual_dataset(self): + """ + Dataset API: Test duplicate virtual dataset + """ + dataset = self.get_fixture_virtual_datasets()[0] + + self.login(username="admin") + uri = f"api/v1/dataset/duplicate" + table_data = {"base_model_id": dataset.id, "table_name": "Dupe1"} + rv = self.post_assert_metric(uri, table_data, "duplicate") + assert rv.status_code == 201 + rv_data = json.loads(rv.data) + new_dataset: SqlaTable = ( + db.session.query(SqlaTable).filter_by(id=rv_data["id"]).one_or_none() + ) + assert new_dataset is not None + assert new_dataset.id != dataset.id + assert new_dataset.table_name == "Dupe1" + assert len(new_dataset.columns) == 2 + assert new_dataset.columns[0].column_name == "id" + assert new_dataset.columns[1].column_name == "name" + + @pytest.mark.usefixtures("create_datasets") + def test_duplicate_physical_dataset(self): + """ + Dataset API: Test duplicate physical dataset + """ + dataset = self.get_fixture_datasets()[0] + + self.login(username="admin") + uri = f"api/v1/dataset/duplicate" + table_data = {"base_model_id": dataset.id, "table_name": "Dupe2"} + rv = self.post_assert_metric(uri, table_data, "duplicate") + assert rv.status_code == 422 + + @pytest.mark.usefixtures("create_virtual_datasets") + def test_duplicate_existing_dataset(self): + """ + Dataset API: Test duplicate dataset with existing name + """ + dataset = self.get_fixture_virtual_datasets()[0] + + self.login(username="admin") + uri = f"api/v1/dataset/duplicate" + table_data = { + "base_model_id": dataset.id, + "table_name": "sql_virtual_dataset_2", + } + rv = self.post_assert_metric(uri, table_data, "duplicate") + assert rv.status_code == 422 + + def test_duplicate_invalid_dataset(self): + """ + Dataset API: Test duplicate invalid dataset + """ + + self.login(username="admin") + uri = f"api/v1/dataset/duplicate" + table_data = { + "base_model_id": -1, + "table_name": "Dupe3", + } + rv = self.post_assert_metric(uri, table_data, "duplicate") + assert rv.status_code == 422 From 2f8f3a47b8b3c276d3b0158725e98dec28f829b9 Mon Sep 17 00:00:00 2001 From: reesercollins <10563996+reesercollins@users.noreply.github.com> Date: Wed, 22 Jun 2022 14:05:23 -0400 Subject: [PATCH 07/17] Add licenses --- .../data/dataset/DuplicateDatasetModal.tsx | 18 ++++++++++++++++++ superset/datasets/commands/duplicate.py | 16 ++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/superset-frontend/src/views/CRUD/data/dataset/DuplicateDatasetModal.tsx b/superset-frontend/src/views/CRUD/data/dataset/DuplicateDatasetModal.tsx index 0885bf03155f0..bac4c8b4d415a 100644 --- a/superset-frontend/src/views/CRUD/data/dataset/DuplicateDatasetModal.tsx +++ b/superset-frontend/src/views/CRUD/data/dataset/DuplicateDatasetModal.tsx @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ import { t } from '@superset-ui/core'; import React, { FunctionComponent, useEffect, useState } from 'react'; import { FormLabel } from 'src/components/Form'; diff --git a/superset/datasets/commands/duplicate.py b/superset/datasets/commands/duplicate.py index 1c2f1d747d2cb..df868618d46d4 100644 --- a/superset/datasets/commands/duplicate.py +++ b/superset/datasets/commands/duplicate.py @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. import logging from typing import Any, Dict, List From 984ffe8dedfef42fd6b7e3e731ad1d5836117482 Mon Sep 17 00:00:00 2001 From: reesercollins <10563996+reesercollins@users.noreply.github.com> Date: Wed, 22 Jun 2022 14:17:07 -0400 Subject: [PATCH 08/17] Fix linting errors --- .../src/views/CRUD/data/dataset/DatasetList.tsx | 2 +- superset/datasets/api.py | 5 ++--- superset/datasets/commands/duplicate.py | 11 +++++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx b/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx index 7659a2fca9634..4e950e0098b7f 100644 --- a/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx +++ b/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx @@ -246,7 +246,7 @@ const DatasetList: FunctionComponent = ({ const openDatasetDuplicateModal = (dataset: VirtualDataset) => { setDatasetCurrentlyDuplicating(dataset); }; - + const handleBulkDatasetExport = (datasetsToExport: Dataset[]) => { const ids = datasetsToExport.map(({ id }) => id); handleResourceExport('dataset', ids, () => { diff --git a/superset/datasets/api.py b/superset/datasets/api.py index ac2f6590d9552..58134e9f3799d 100644 --- a/superset/datasets/api.py +++ b/superset/datasets/api.py @@ -20,10 +20,8 @@ from io import BytesIO from typing import Any from zipfile import is_zipfile, ZipFile -from numpy import save import simplejson -from superset.datasets.commands.duplicate import DuplicateDatasetCommand import yaml from flask import g, make_response, request, Response, send_file from flask_appbuilder.api import expose, protect, rison, safe @@ -40,6 +38,7 @@ from superset.datasets.commands.bulk_delete import BulkDeleteDatasetCommand from superset.datasets.commands.create import CreateDatasetCommand from superset.datasets.commands.delete import DeleteDatasetCommand +from superset.datasets.commands.duplicate import DuplicateDatasetCommand from superset.datasets.commands.exceptions import ( DatasetBulkDeleteFailedError, DatasetCreateFailedError, @@ -523,7 +522,7 @@ def export(self, **kwargs: Any) -> Response: log_to_statsd=False, ) @requires_json - def duplicate(self, **kwargs: Any) -> Response: + def duplicate(self) -> Response: """Duplicates a Dataset --- post: diff --git a/superset/datasets/commands/duplicate.py b/superset/datasets/commands/duplicate.py index df868618d46d4..3ee538230b68c 100644 --- a/superset/datasets/commands/duplicate.py +++ b/superset/datasets/commands/duplicate.py @@ -25,7 +25,7 @@ from superset.commands.base import BaseCommand, CreateMixin from superset.commands.exceptions import DatasourceTypeInvalidError -from superset.connectors.sqla.models import SqlMetric, SqlaTable, TableColumn +from superset.connectors.sqla.models import SqlaTable, SqlMetric, TableColumn from superset.dao.exceptions import DAOCreateFailedError from superset.datasets.commands.exceptions import ( DatasetDuplicateFailedError, @@ -36,7 +36,7 @@ from superset.datasets.dao import DatasetDAO from superset.errors import ErrorLevel, SupersetError, SupersetErrorType from superset.exceptions import SupersetErrorException -from superset.extensions import db, security_manager +from superset.extensions import db from superset.models.core import Database from superset.sql_parse import ParsedQuery @@ -46,6 +46,7 @@ class DuplicateDatasetCommand(CreateMixin, BaseCommand): def __init__(self, user: User, data: Dict[str, Any]): self._actor = user + self._base_model: SqlaTable = SqlaTable() self._properties = data.copy() def run(self) -> Model: @@ -108,9 +109,11 @@ def validate(self) -> None: base_model_id = self._properties["base_model_id"] duplicate_name = self._properties["table_name"] - self._base_model = DatasetDAO.find_by_id(base_model_id) - if not self._base_model: + base_model = DatasetDAO.find_by_id(base_model_id) + if not base_model: exceptions.append(DatasetNotFoundError()) + else: + self._base_model = base_model if self._base_model and self._base_model.kind != "virtual": exceptions.append(DatasourceTypeInvalidError()) From 7e3a462d52ed1803dce860035303e023b91d9702 Mon Sep 17 00:00:00 2001 From: reesercollins <10563996+reesercollins@users.noreply.github.com> Date: Thu, 30 Jun 2022 09:04:39 -0400 Subject: [PATCH 09/17] Change confirm button to 'Duplicate' --- .../src/views/CRUD/data/dataset/DuplicateDatasetModal.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/superset-frontend/src/views/CRUD/data/dataset/DuplicateDatasetModal.tsx b/superset-frontend/src/views/CRUD/data/dataset/DuplicateDatasetModal.tsx index bac4c8b4d415a..6766adf74eab5 100644 --- a/superset-frontend/src/views/CRUD/data/dataset/DuplicateDatasetModal.tsx +++ b/superset-frontend/src/views/CRUD/data/dataset/DuplicateDatasetModal.tsx @@ -61,6 +61,7 @@ const DuplicateDatasetModal: FunctionComponent = ({ title={t('Duplicate dataset')} disablePrimaryButton={disableSave} onHandledPrimaryAction={duplicateDataset} + primaryButtonName={t('Duplicate')} > {t('New dataset name')} Date: Thu, 30 Jun 2022 10:20:24 -0400 Subject: [PATCH 10/17] Fix HTTP status code and response --- superset/datasets/api.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/superset/datasets/api.py b/superset/datasets/api.py index 58134e9f3799d..ca94a67fb69a8 100644 --- a/superset/datasets/api.py +++ b/superset/datasets/api.py @@ -536,15 +536,17 @@ def duplicate(self) -> Response: schema: $ref: '#/components/schemas/DatasetDuplicateSchema' responses: - 200: - description: Dataset duplicate + 201: + description: Dataset duplicated content: application/json: schema: type: object properties: - message: - type: string + id: + type: number + result: + $ref: '#/components/schemas/DatasetDuplicateSchema' 400: $ref: '#/components/responses/400' 401: From 0c60262f159c4e1f87b3c2756544a7b3a8b253ee Mon Sep 17 00:00:00 2001 From: reesercollins <10563996+reesercollins@users.noreply.github.com> Date: Thu, 21 Jul 2022 13:47:19 -0400 Subject: [PATCH 11/17] Add missing import --- superset/datasets/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/datasets/api.py b/superset/datasets/api.py index bc6df293dd950..8b5ead87b3bb8 100644 --- a/superset/datasets/api.py +++ b/superset/datasets/api.py @@ -23,7 +23,7 @@ import simplejson import yaml -from flask import make_response, request, Response, send_file +from flask import g, make_response, request, Response, send_file from flask_appbuilder.api import expose, protect, rison, safe from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_babel import ngettext From d42fa72b09f2b15a8e224c024aa900ddba3a9b8d Mon Sep 17 00:00:00 2001 From: reesercollins <10563996+reesercollins@users.noreply.github.com> Date: Fri, 22 Jul 2022 08:58:51 -0400 Subject: [PATCH 12/17] Use user id instead of user object --- superset/datasets/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/datasets/api.py b/superset/datasets/api.py index 8b5ead87b3bb8..a95fad31cc3b3 100644 --- a/superset/datasets/api.py +++ b/superset/datasets/api.py @@ -577,7 +577,7 @@ def duplicate(self) -> Response: return self.response_400(message=error.messages) try: - new_model = DuplicateDatasetCommand(g.user, item).run() + new_model = DuplicateDatasetCommand([g.user.id], item).run() return self.response(201, id=new_model.id, result=item) except DatasetInvalidError as ex: return self.response_422( From 087d67f4cc0d7b31f30b215555e0f8d47d327174 Mon Sep 17 00:00:00 2001 From: Reese <10563996+reesercollins@users.noreply.github.com> Date: Mon, 25 Jul 2022 08:27:39 -0400 Subject: [PATCH 13/17] Remove stray debug print --- superset/dao/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/superset/dao/base.py b/superset/dao/base.py index ea01a93842281..0090c4e535e23 100644 --- a/superset/dao/base.py +++ b/superset/dao/base.py @@ -121,7 +121,6 @@ def create(cls, properties: Dict[str, Any], commit: bool = True) -> Model: raise DAOConfigError() model = cls.model_cls() # pylint: disable=not-callable for key, value in properties.items(): - print(key, value) setattr(model, key, value) try: db.session.add(model) From bd845af93d3d34b92d91d4cc2d00d1dc6423463a Mon Sep 17 00:00:00 2001 From: reesercollins <10563996+reesercollins@users.noreply.github.com> Date: Mon, 25 Jul 2022 09:49:21 -0400 Subject: [PATCH 14/17] Fix sqlite tests --- tests/integration_tests/datasets/api_tests.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/integration_tests/datasets/api_tests.py b/tests/integration_tests/datasets/api_tests.py index 93ffba62027d1..6eaf0815f1a5d 100644 --- a/tests/integration_tests/datasets/api_tests.py +++ b/tests/integration_tests/datasets/api_tests.py @@ -2101,6 +2101,9 @@ def test_duplicate_virtual_dataset(self): """ Dataset API: Test duplicate virtual dataset """ + if backend() == "sqlite": + return + dataset = self.get_fixture_virtual_datasets()[0] self.login(username="admin") @@ -2124,6 +2127,9 @@ def test_duplicate_physical_dataset(self): """ Dataset API: Test duplicate physical dataset """ + if backend() == "sqlite": + return + dataset = self.get_fixture_datasets()[0] self.login(username="admin") @@ -2137,6 +2143,9 @@ def test_duplicate_existing_dataset(self): """ Dataset API: Test duplicate dataset with existing name """ + if backend() == "sqlite": + return + dataset = self.get_fixture_virtual_datasets()[0] self.login(username="admin") From 5d46e54f9286c03526b984ce687aff8fd4cac7a2 Mon Sep 17 00:00:00 2001 From: reesercollins <10563996+reesercollins@users.noreply.github.com> Date: Tue, 2 Aug 2022 08:54:18 -0400 Subject: [PATCH 15/17] Specify type of extra --- superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx b/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx index 27013d0f78797..9a788ac953da3 100644 --- a/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx +++ b/superset-frontend/src/views/CRUD/data/dataset/DatasetList.tsx @@ -120,7 +120,7 @@ type Dataset = { }; interface VirtualDataset extends Dataset { - extra: any; + extra: Record; sql: string; } From 456fe89943856271ddfa72df352d5b2d4d83a92a Mon Sep 17 00:00:00 2001 From: reesercollins <10563996+reesercollins@users.noreply.github.com> Date: Tue, 2 Aug 2022 09:34:25 -0400 Subject: [PATCH 16/17] Add frontend tests --- .../CRUD/data/dataset/DatasetList.test.jsx | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/superset-frontend/src/views/CRUD/data/dataset/DatasetList.test.jsx b/superset-frontend/src/views/CRUD/data/dataset/DatasetList.test.jsx index 2f23f45573311..e506970cdcfdd 100644 --- a/superset-frontend/src/views/CRUD/data/dataset/DatasetList.test.jsx +++ b/superset-frontend/src/views/CRUD/data/dataset/DatasetList.test.jsx @@ -41,6 +41,7 @@ const store = mockStore({}); const datasetsInfoEndpoint = 'glob:*/api/v1/dataset/_info*'; const datasetsOwnersEndpoint = 'glob:*/api/v1/dataset/related/owners*'; const datasetsSchemaEndpoint = 'glob:*/api/v1/dataset/distinct/schema*'; +const datasetsDuplicateEndpoint = 'glob:*/api/v1/dataset/duplicate*'; const databaseEndpoint = 'glob:*/api/v1/dataset/related/database*'; const datasetsEndpoint = 'glob:*/api/v1/dataset/?*'; @@ -63,7 +64,7 @@ const mockUser = { }; fetchMock.get(datasetsInfoEndpoint, { - permissions: ['can_read', 'can_write'], + permissions: ['can_read', 'can_write', 'can_duplicate'], }); fetchMock.get(datasetsOwnersEndpoint, { result: [], @@ -71,6 +72,9 @@ fetchMock.get(datasetsOwnersEndpoint, { fetchMock.get(datasetsSchemaEndpoint, { result: [], }); +fetchMock.post(datasetsDuplicateEndpoint, { + result: [], +}); fetchMock.get(datasetsEndpoint, { result: mockdatasets, dataset_count: 3, @@ -181,6 +185,42 @@ describe('DatasetList', () => { wrapper.find('[data-test="bulk-select-copy"]').text(), ).toMatchInlineSnapshot(`"3 Selected (2 Physical, 1 Virtual)"`); }); + + it('shows duplicate modal when duplicate action is clicked', async () => { + await waitForComponentToPaint(wrapper); + expect( + wrapper.find('[data-test="duplicate-modal-input"]').exists(), + ).toBeFalsy(); + act(() => { + wrapper + .find('#duplicate-action-tooltop') + .at(0) + .find('.action-button') + .props() + .onClick(); + }); + await waitForComponentToPaint(wrapper); + expect(wrapper.find('[data-test="duplicate-modal-input"]').exists()); + }); + + it('calls the duplicate endpoint', async () => { + await waitForComponentToPaint(wrapper); + await act(async () => { + wrapper + .find('#duplicate-action-tooltop') + .at(0) + .find('.action-button') + .props() + .onClick(); + await waitForComponentToPaint(wrapper); + wrapper + .find('[data-test="duplicate-modal-input"]') + .at(0) + .props() + .onPressEnter(); + }); + expect(fetchMock.calls(/dataset\/duplicate/)).toHaveLength(1); + }); }); jest.mock('react-router-dom', () => ({ From 2ceee595a12bbf3d4515fef436b96862843604fe Mon Sep 17 00:00:00 2001 From: reesercollins <10563996+reesercollins@users.noreply.github.com> Date: Tue, 2 Aug 2022 09:56:25 -0400 Subject: [PATCH 17/17] Add match statement to test --- .../src/views/CRUD/data/dataset/DatasetList.test.jsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/superset-frontend/src/views/CRUD/data/dataset/DatasetList.test.jsx b/superset-frontend/src/views/CRUD/data/dataset/DatasetList.test.jsx index e506970cdcfdd..a2c2ab6954778 100644 --- a/superset-frontend/src/views/CRUD/data/dataset/DatasetList.test.jsx +++ b/superset-frontend/src/views/CRUD/data/dataset/DatasetList.test.jsx @@ -200,7 +200,9 @@ describe('DatasetList', () => { .onClick(); }); await waitForComponentToPaint(wrapper); - expect(wrapper.find('[data-test="duplicate-modal-input"]').exists()); + expect( + wrapper.find('[data-test="duplicate-modal-input"]').exists(), + ).toBeTruthy(); }); it('calls the duplicate endpoint', async () => {