From 8db46965ee2ff2bd428935fb3c25d4c96afa8294 Mon Sep 17 00:00:00 2001 From: Diego Medina Date: Thu, 5 Jan 2023 19:29:40 -0300 Subject: [PATCH 1/2] chore: Migrate /superset/queries/ to API v1 --- docs/static/resources/openapi.json | 1249 +++++++++++------ .../components/QueryAutoRefresh/index.tsx | 20 +- superset/constants.py | 1 + superset/queries/api.py | 76 +- superset/queries/dao.py | 14 +- superset/queries/schemas.py | 8 + superset/views/core.py | 1 + tests/integration_tests/queries/api_tests.py | 58 +- tests/unit_tests/dao/queries_test.py | 56 + 9 files changed, 1058 insertions(+), 425 deletions(-) diff --git a/docs/static/resources/openapi.json b/docs/static/resources/openapi.json index 62153bac51cb8..22f38abbc7824 100644 --- a/docs/static/resources/openapi.json +++ b/docs/static/resources/openapi.json @@ -547,6 +547,17 @@ }, "type": "object" }, + "AvailableDomainsSchema": { + "properties": { + "domains": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, "CacheInvalidationRequestSchema": { "properties": { "datasource_uids": { @@ -822,17 +833,12 @@ }, "ChartDataExtras": { "properties": { - "druid_time_origin": { - "description": "Starting point for time grain counting on legacy Druid datasources. Used to change e.g. Monday/Sunday first-day-of-week.", - "nullable": true, - "type": "string" - }, "having": { "description": "HAVING clause to be added to aggregate queries using AND operator.", "type": "string" }, "having_druid": { - "description": "HAVING filters to be added to legacy Druid datasource queries.", + "description": "HAVING filters to be added to legacy Druid datasource queries. This field is deprecated", "items": { "$ref": "#/components/schemas/ChartDataFilter" }, @@ -920,18 +926,20 @@ "NOT IN", "REGEX", "IS TRUE", - "IS FALSE" + "IS FALSE", + "TEMPORAL_RANGE" ], "example": "IN", "type": "string" }, "val": { - "description": "The value or values to compare against. Can be a string, integer, decimal or list, depending on the operator.", + "description": "The value or values to compare against. Can be a string, integer, decimal, None or list, depending on the operator.", "example": [ "China", "France", "Japan" - ] + ], + "nullable": true } }, "required": [ @@ -1060,13 +1068,13 @@ "operation": { "description": "Post processing operation type", "enum": [ - "_flatten_column_after_pivot", "aggregate", "boxplot", "compare", "contribution", "cum", "diff", + "escape_separator", "flatten", "geodetic_parse", "geohash_decode", @@ -1077,7 +1085,8 @@ "resample", "rolling", "select", - "sort" + "sort", + "unescape_separator" ], "example": "aggregate", "type": "string" @@ -1174,11 +1183,18 @@ }, "ChartDataQueryContextSchema": { "properties": { + "custom_cache_timeout": { + "description": "Override the default cache timeout", + "format": "int32", + "nullable": true, + "type": "integer" + }, "datasource": { "$ref": "#/components/schemas/ChartDataDatasource" }, "force": { "description": "Should the queries be forced to load from the source. Default: `false`", + "nullable": true, "type": "boolean" }, "form_data": { @@ -1436,7 +1452,7 @@ "type": "string" }, "cache_timeout": { - "description": "Cache timeout in following order: custom timeout, datasource timeout, default config timeout.", + "description": "Cache timeout in following order: custom timeout, datasource timeout, cache default timeout, config default cache timeout.", "format": "int32", "nullable": true, "type": "integer" @@ -1557,6 +1573,9 @@ "nullable": true, "type": "string" }, + "changed_on_delta_humanized": { + "readOnly": true + }, "dashboards": { "$ref": "#/components/schemas/ChartDataRestApi.get.Dashboard" }, @@ -1564,6 +1583,10 @@ "nullable": true, "type": "string" }, + "id": { + "format": "int32", + "type": "integer" + }, "is_managed_externally": { "type": "boolean" }, @@ -1583,6 +1606,12 @@ "nullable": true, "type": "string" }, + "thumbnail_url": { + "readOnly": true + }, + "url": { + "readOnly": true + }, "viz_type": { "maxLength": 250, "nullable": true, @@ -1651,7 +1680,7 @@ "type": "string" }, "changed_by": { - "$ref": "#/components/schemas/ChartDataRestApi.get_list.User" + "$ref": "#/components/schemas/ChartDataRestApi.get_list.User1" }, "changed_by_name": { "readOnly": true @@ -1666,7 +1695,13 @@ "readOnly": true }, "created_by": { - "$ref": "#/components/schemas/ChartDataRestApi.get_list.User1" + "$ref": "#/components/schemas/ChartDataRestApi.get_list.User2" + }, + "created_on_delta_humanized": { + "readOnly": true + }, + "dashboards": { + "$ref": "#/components/schemas/ChartDataRestApi.get_list.Dashboard" }, "datasource_id": { "format": "int32", @@ -1707,10 +1742,10 @@ "type": "string" }, "last_saved_by": { - "$ref": "#/components/schemas/ChartDataRestApi.get_list.User2" + "$ref": "#/components/schemas/ChartDataRestApi.get_list.User3" }, "owners": { - "$ref": "#/components/schemas/ChartDataRestApi.get_list.User1" + "$ref": "#/components/schemas/ChartDataRestApi.get_list.User" }, "params": { "nullable": true, @@ -1738,6 +1773,20 @@ }, "type": "object" }, + "ChartDataRestApi.get_list.Dashboard": { + "properties": { + "dashboard_title": { + "maxLength": 500, + "nullable": true, + "type": "string" + }, + "id": { + "format": "int32", + "type": "integer" + } + }, + "type": "object" + }, "ChartDataRestApi.get_list.SqlaTable": { "properties": { "default_endpoint": { @@ -1760,14 +1809,23 @@ "maxLength": 64, "type": "string" }, + "id": { + "format": "int32", + "type": "integer" + }, "last_name": { "maxLength": 64, "type": "string" + }, + "username": { + "maxLength": 64, + "type": "string" } }, "required": [ "first_name", - "last_name" + "last_name", + "username" ], "type": "object" }, @@ -1777,23 +1835,14 @@ "maxLength": 64, "type": "string" }, - "id": { - "format": "int32", - "type": "integer" - }, "last_name": { "maxLength": 64, "type": "string" - }, - "username": { - "maxLength": 64, - "type": "string" } }, "required": [ "first_name", - "last_name", - "username" + "last_name" ], "type": "object" }, @@ -2231,7 +2280,8 @@ "description": "Form data from the Explore controls used to form the chart's data query.", "type": "object" }, - "slice_id": { + "id": { + "description": "The id of the chart.", "format": "int32", "type": "integer" }, @@ -2315,6 +2365,9 @@ "nullable": true, "type": "string" }, + "changed_on_delta_humanized": { + "readOnly": true + }, "dashboards": { "$ref": "#/components/schemas/ChartRestApi.get.Dashboard" }, @@ -2322,6 +2375,10 @@ "nullable": true, "type": "string" }, + "id": { + "format": "int32", + "type": "integer" + }, "is_managed_externally": { "type": "boolean" }, @@ -2341,6 +2398,12 @@ "nullable": true, "type": "string" }, + "thumbnail_url": { + "readOnly": true + }, + "url": { + "readOnly": true + }, "viz_type": { "maxLength": 250, "nullable": true, @@ -2409,7 +2472,7 @@ "type": "string" }, "changed_by": { - "$ref": "#/components/schemas/ChartRestApi.get_list.User" + "$ref": "#/components/schemas/ChartRestApi.get_list.User1" }, "changed_by_name": { "readOnly": true @@ -2424,7 +2487,13 @@ "readOnly": true }, "created_by": { - "$ref": "#/components/schemas/ChartRestApi.get_list.User1" + "$ref": "#/components/schemas/ChartRestApi.get_list.User2" + }, + "created_on_delta_humanized": { + "readOnly": true + }, + "dashboards": { + "$ref": "#/components/schemas/ChartRestApi.get_list.Dashboard" }, "datasource_id": { "format": "int32", @@ -2465,10 +2534,10 @@ "type": "string" }, "last_saved_by": { - "$ref": "#/components/schemas/ChartRestApi.get_list.User2" + "$ref": "#/components/schemas/ChartRestApi.get_list.User3" }, "owners": { - "$ref": "#/components/schemas/ChartRestApi.get_list.User1" + "$ref": "#/components/schemas/ChartRestApi.get_list.User" }, "params": { "nullable": true, @@ -2496,6 +2565,20 @@ }, "type": "object" }, + "ChartRestApi.get_list.Dashboard": { + "properties": { + "dashboard_title": { + "maxLength": 500, + "nullable": true, + "type": "string" + }, + "id": { + "format": "int32", + "type": "integer" + } + }, + "type": "object" + }, "ChartRestApi.get_list.SqlaTable": { "properties": { "default_endpoint": { @@ -2518,14 +2601,23 @@ "maxLength": 64, "type": "string" }, + "id": { + "format": "int32", + "type": "integer" + }, "last_name": { "maxLength": 64, "type": "string" + }, + "username": { + "maxLength": 64, + "type": "string" } }, "required": [ "first_name", - "last_name" + "last_name", + "username" ], "type": "object" }, @@ -2535,23 +2627,14 @@ "maxLength": 64, "type": "string" }, - "id": { - "format": "int32", - "type": "integer" - }, "last_name": { "maxLength": 64, "type": "string" - }, - "username": { - "maxLength": 64, - "type": "string" } }, "required": [ "first_name", - "last_name", - "username" + "last_name" ], "type": "object" }, @@ -3044,8 +3127,7 @@ }, "owners": { "items": { - "format": "int32", - "type": "integer" + "type": "object" }, "type": "array" }, @@ -3180,16 +3262,24 @@ }, "DashboardPermalinkPostSchema": { "properties": { - "filterState": { - "description": "Native filter state", + "activeTabs": { + "description": "Current active dashboard tabs", + "items": { + "type": "string" + }, "nullable": true, - "type": "object" + "type": "array" }, - "hash": { - "description": "Optional anchor link", + "anchor": { + "description": "Optional anchor link added to url hash", "nullable": true, "type": "string" }, + "dataMask": { + "description": "Data mask used for native filter state", + "nullable": true, + "type": "object" + }, "urlParams": { "description": "URL Parameters", "items": { @@ -3222,7 +3312,7 @@ "type": "string" }, "changed_by": { - "$ref": "#/components/schemas/DashboardRestApi.get_list.User" + "$ref": "#/components/schemas/DashboardRestApi.get_list.User1" }, "changed_by_name": { "readOnly": true @@ -3242,9 +3332,6 @@ "created_on_delta_humanized": { "readOnly": true }, - "created_on_delta_humanized": { - "readOnly": true - }, "css": { "nullable": true, "type": "string" @@ -3266,7 +3353,7 @@ "type": "string" }, "owners": { - "$ref": "#/components/schemas/DashboardRestApi.get_list.User2" + "$ref": "#/components/schemas/DashboardRestApi.get_list.User" }, "position_json": { "nullable": true, @@ -3314,6 +3401,10 @@ }, "DashboardRestApi.get_list.User": { "properties": { + "email": { + "maxLength": 64, + "type": "string" + }, "first_name": { "maxLength": 64, "type": "string" @@ -3332,6 +3423,7 @@ } }, "required": [ + "email", "first_name", "last_name", "username" @@ -3340,10 +3432,6 @@ }, "DashboardRestApi.get_list.User1": { "properties": { - "email": { - "maxLength": 64, - "type": "string" - }, "first_name": { "maxLength": 64, "type": "string" @@ -3362,7 +3450,6 @@ } }, "required": [ - "email", "first_name", "last_name", "username" @@ -3560,17 +3647,6 @@ }, "name": { "type": "string" - }, - "engine_information": { - "type": "object" - } - }, - "type": "object" - }, - "Database1": { - "properties": { - "database_name": { - "type": "string" } }, "type": "object" @@ -3711,9 +3787,11 @@ "maxLength": 250, "type": "string" }, - "encrypted_extra": { - "nullable": true, - "type": "string" + "driver": { + "readOnly": true + }, + "engine_information": { + "readOnly": true }, "expose_in_sqllab": { "nullable": true, @@ -3739,6 +3817,9 @@ "is_managed_externally": { "type": "boolean" }, + "masked_encrypted_extra": { + "readOnly": true + }, "parameters": { "readOnly": true }, @@ -3753,8 +3834,10 @@ "maxLength": 1024, "type": "string" }, - "engine_information": { - "readOnly": true + "uuid": { + "format": "uuid", + "nullable": true, + "type": "string" } }, "required": [ @@ -3815,6 +3898,9 @@ "disable_data_preview": { "readOnly": true }, + "engine_information": { + "readOnly": true + }, "explore_database_id": { "readOnly": true }, @@ -3835,8 +3921,10 @@ "format": "int32", "type": "integer" }, - "engine_information": { - "readOnly": true + "uuid": { + "format": "uuid", + "nullable": true, + "type": "string" } }, "required": [ @@ -3899,8 +3987,8 @@ "minLength": 1, "type": "string" }, - "encrypted_extra": { - "description": "

JSON string containing additional connection configuration.
This is used to provide connection information for systems like Hive, Presto, and BigQuery, which do not conform to the username:password syntax normally used by SQLAlchemy.

", + "driver": { + "description": "SQLAlchemy driver to use", "nullable": true, "type": "string" }, @@ -3936,6 +4024,11 @@ "nullable": true, "type": "boolean" }, + "masked_encrypted_extra": { + "description": "

JSON string containing additional connection configuration.
This is used to provide connection information for systems like Hive, Presto, and BigQuery, which do not conform to the username:password syntax normally used by SQLAlchemy.

", + "nullable": true, + "type": "string" + }, "parameters": { "additionalProperties": {}, "description": "DB-specific parameters for configuration", @@ -3951,6 +4044,17 @@ "maxLength": 1024, "minLength": 1, "type": "string" + }, + "ssh_tunnel": { + "allOf": [ + { + "$ref": "#/components/schemas/DatabaseSSHTunnel" + } + ], + "nullable": true + }, + "uuid": { + "type": "string" } }, "required": [ @@ -3997,8 +4101,8 @@ "nullable": true, "type": "string" }, - "encrypted_extra": { - "description": "

JSON string containing additional connection configuration.
This is used to provide connection information for systems like Hive, Presto, and BigQuery, which do not conform to the username:password syntax normally used by SQLAlchemy.

", + "driver": { + "description": "SQLAlchemy driver to use", "nullable": true, "type": "string" }, @@ -4034,6 +4138,11 @@ "nullable": true, "type": "boolean" }, + "masked_encrypted_extra": { + "description": "

JSON string containing additional connection configuration.
This is used to provide connection information for systems like Hive, Presto, and BigQuery, which do not conform to the username:password syntax normally used by SQLAlchemy.

", + "nullable": true, + "type": "string" + }, "parameters": { "additionalProperties": {}, "description": "DB-specific parameters for configuration", @@ -4049,6 +4158,38 @@ "maxLength": 1024, "minLength": 0, "type": "string" + }, + "ssh_tunnel": { + "allOf": [ + { + "$ref": "#/components/schemas/DatabaseSSHTunnel" + } + ], + "nullable": true + } + }, + "type": "object" + }, + "DatabaseSSHTunnel": { + "properties": { + "password": { + "type": "string" + }, + "private_key": { + "type": "string" + }, + "private_key_password": { + "type": "string" + }, + "server_address": { + "type": "string" + }, + "server_port": { + "format": "int32", + "type": "integer" + }, + "username": { + "type": "string" } }, "type": "object" @@ -4066,8 +4207,8 @@ "nullable": true, "type": "string" }, - "encrypted_extra": { - "description": "

JSON string containing additional connection configuration.
This is used to provide connection information for systems like Hive, Presto, and BigQuery, which do not conform to the username:password syntax normally used by SQLAlchemy.

", + "driver": { + "description": "SQLAlchemy driver to use", "nullable": true, "type": "string" }, @@ -4084,6 +4225,11 @@ "description": "If Presto, all the queries in SQL Lab are going to be executed as the currently logged on user who must have permission to run them.
If Hive and hive.server2.enable.doAs is enabled, will run the queries as service account, but impersonate the currently logged on user via hive.server2.proxy.user property.", "type": "boolean" }, + "masked_encrypted_extra": { + "description": "

JSON string containing additional connection configuration.
This is used to provide connection information for systems like Hive, Presto, and BigQuery, which do not conform to the username:password syntax normally used by SQLAlchemy.

", + "nullable": true, + "type": "string" + }, "parameters": { "additionalProperties": {}, "description": "DB-specific parameters for configuration", @@ -4099,12 +4245,27 @@ "maxLength": 1024, "minLength": 1, "type": "string" - } - }, - "type": "object" - }, + }, + "ssh_tunnel": { + "allOf": [ + { + "$ref": "#/components/schemas/DatabaseSSHTunnel" + } + ], + "nullable": true + } + }, + "type": "object" + }, "DatabaseValidateParametersSchema": { "properties": { + "catalog": { + "additionalProperties": { + "nullable": true + }, + "description": "Gsheets specific column for managing label to sheet urls", + "type": "object" + }, "configuration_method": { "description": "Configuration_method is used on the frontend to inform the backend whether to explode parameters or to provide only a sqlalchemy_uri." }, @@ -4115,8 +4276,8 @@ "nullable": true, "type": "string" }, - "encrypted_extra": { - "description": "

JSON string containing additional connection configuration.
This is used to provide connection information for systems like Hive, Presto, and BigQuery, which do not conform to the username:password syntax normally used by SQLAlchemy.

", + "driver": { + "description": "SQLAlchemy driver to use", "nullable": true, "type": "string" }, @@ -4128,10 +4289,21 @@ "description": "

JSON string containing extra configuration elements.
1. The engine_params object gets unpacked into the sqlalchemy.create_engine call, while the metadata_params gets unpacked into the sqlalchemy.MetaData call.
2. The metadata_cache_timeout is a cache timeout setting in seconds for metadata fetch of this database. Specify it as \"metadata_cache_timeout\": {\"schema_cache_timeout\": 600, \"table_cache_timeout\": 600}. If unset, cache will not be enabled for the functionality. A timeout of 0 indicates that the cache never expires.
3. The schemas_allowed_for_file_upload is a comma separated list of schemas that CSVs are allowed to upload to. Specify it as \"schemas_allowed_for_file_upload\": [\"public\", \"csv_upload\"]. If database flavor does not support schema or any schema is allowed to be accessed, just leave the list empty
4. The version field is a string specifying the this db's version. This should be used with Presto DBs so that the syntax is correct
5. The allows_virtual_table_explore field is a boolean specifying whether or not the Explore button in SQL Lab results is shown.
6. The disable_data_preview field is a boolean specifying whether or not data preview queries will be run when fetching table metadata in SQL Lab.

", "type": "string" }, + "id": { + "description": "Database ID (for updates)", + "format": "int32", + "nullable": true, + "type": "integer" + }, "impersonate_user": { "description": "If Presto, all the queries in SQL Lab are going to be executed as the currently logged on user who must have permission to run them.
If Hive and hive.server2.enable.doAs is enabled, will run the queries as service account, but impersonate the currently logged on user via hive.server2.proxy.user property.", "type": "boolean" }, + "masked_encrypted_extra": { + "description": "

JSON string containing additional connection configuration.
This is used to provide connection information for systems like Hive, Presto, and BigQuery, which do not conform to the username:password syntax normally used by SQLAlchemy.

", + "nullable": true, + "type": "string" + }, "parameters": { "additionalProperties": { "nullable": true @@ -4151,6 +4323,174 @@ ], "type": "object" }, + "Dataset": { + "properties": { + "cache_timeout": { + "description": "Duration (in seconds) of the caching timeout for this dataset.", + "format": "int32", + "type": "integer" + }, + "column_formats": { + "description": "Column formats.", + "type": "object" + }, + "columns": { + "description": "Columns metadata.", + "items": { + "type": "object" + }, + "type": "array" + }, + "database": { + "description": "Database associated with the dataset.", + "type": "object" + }, + "datasource_name": { + "description": "Dataset name.", + "type": "string" + }, + "default_endpoint": { + "description": "Default endpoint for the dataset.", + "type": "string" + }, + "description": { + "description": "Dataset description.", + "type": "string" + }, + "edit_url": { + "description": "The URL for editing the dataset.", + "type": "string" + }, + "extra": { + "description": "JSON string containing extra configuration elements.", + "type": "object" + }, + "fetch_values_predicate": { + "description": "Predicate used when fetching values from the dataset.", + "type": "string" + }, + "filter_select": { + "description": "SELECT filter applied to the dataset.", + "type": "boolean" + }, + "filter_select_enabled": { + "description": "If the SELECT filter is enabled.", + "type": "boolean" + }, + "granularity_sqla": { + "description": "Name of temporal column used for time filtering for SQL datasources. This field is deprecated, use `granularity` instead.", + "items": { + "items": { + "type": "object" + }, + "type": "array" + }, + "type": "array" + }, + "health_check_message": { + "description": "Health check message.", + "type": "string" + }, + "id": { + "description": "Dataset ID.", + "format": "int32", + "type": "integer" + }, + "is_sqllab_view": { + "description": "If the dataset is a SQL Lab view.", + "type": "boolean" + }, + "main_dttm_col": { + "description": "The main temporal column.", + "type": "string" + }, + "metrics": { + "description": "Dataset metrics.", + "items": { + "type": "object" + }, + "type": "array" + }, + "name": { + "description": "Dataset name.", + "type": "string" + }, + "offset": { + "description": "Dataset offset.", + "format": "int32", + "type": "integer" + }, + "order_by_choices": { + "description": "List of order by columns.", + "items": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": "array" + }, + "owners": { + "description": "List of owners identifiers", + "items": { + "format": "int32", + "type": "integer" + }, + "type": "array" + }, + "params": { + "description": "Extra params for the dataset.", + "type": "object" + }, + "perm": { + "description": "Permission expression.", + "type": "string" + }, + "schema": { + "description": "Dataset schema.", + "type": "string" + }, + "select_star": { + "description": "Select all clause.", + "type": "string" + }, + "sql": { + "description": "A SQL statement that defines the dataset.", + "type": "string" + }, + "table_name": { + "description": "The name of the table associated with the dataset.", + "type": "string" + }, + "template_params": { + "description": "Table template params.", + "type": "object" + }, + "time_grain_sqla": { + "description": "List of temporal granularities supported by the dataset.", + "items": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": "array" + }, + "type": { + "description": "Dataset type.", + "type": "string" + }, + "uid": { + "description": "Dataset unique identifier.", + "type": "string" + }, + "verbose_map": { + "description": "Mapping from raw name to verbose name.", + "type": "object" + } + }, + "type": "object" + }, "DatasetColumnsPut": { "properties": { "advanced_data_type": { @@ -4446,9 +4786,31 @@ "nullable": true, "type": "integer" }, + "changed_by": { + "$ref": "#/components/schemas/DatasetRestApi.get.User1" + }, + "changed_on": { + "format": "date-time", + "nullable": true, + "type": "string" + }, + "changed_on_humanized": { + "readOnly": true + }, "columns": { "$ref": "#/components/schemas/DatasetRestApi.get.TableColumn" }, + "created_by": { + "$ref": "#/components/schemas/DatasetRestApi.get.User2" + }, + "created_on": { + "format": "date-time", + "nullable": true, + "type": "string" + }, + "created_on_humanized": { + "readOnly": true + }, "database": { "$ref": "#/components/schemas/DatasetRestApi.get.Database" }, @@ -4510,6 +4872,9 @@ "nullable": true, "type": "string" }, + "select_star": { + "readOnly": true + }, "sql": { "nullable": true, "type": "string" @@ -4594,11 +4959,6 @@ "nullable": true, "type": "string" }, - "uuid": { - "format": "uuid", - "nullable": true, - "type": "string" - }, "verbose_name": { "maxLength": 1024, "nullable": true, @@ -4722,6 +5082,40 @@ ], "type": "object" }, + "DatasetRestApi.get.User1": { + "properties": { + "first_name": { + "maxLength": 64, + "type": "string" + }, + "last_name": { + "maxLength": 64, + "type": "string" + } + }, + "required": [ + "first_name", + "last_name" + ], + "type": "object" + }, + "DatasetRestApi.get.User2": { + "properties": { + "first_name": { + "maxLength": 64, + "type": "string" + }, + "last_name": { + "maxLength": 64, + "type": "string" + } + }, + "required": [ + "first_name", + "last_name" + ], + "type": "object" + }, "DatasetRestApi.get_list": { "properties": { "changed_by": { @@ -4875,6 +5269,10 @@ "minLength": 0, "type": "string" }, + "sql": { + "nullable": true, + "type": "string" + }, "table_name": { "maxLength": 250, "minLength": 1, @@ -5026,95 +5424,21 @@ "items": { "$ref": "#/components/schemas/DistinctResultResponse" }, - "type": "array" - } - }, - "type": "object" - }, - "DistinctResultResponse": { - "properties": { - "text": { - "description": "The distinct item", - "type": "string" - } - }, - "type": "object" - }, - "EmbeddedDashboardConfig": { - "properties": { - "allowed_domains": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "required": [ - "allowed_domains" - ], - "type": "object" - }, - "EmbeddedDashboardResponseSchema": { - "properties": { - "allowed_domains": { - "items": { - "type": "string" - }, - "type": "array" - }, - "changed_by": { - "$ref": "#/components/schemas/User" - }, - "changed_on": { - "format": "date-time", - "type": "string" - }, - "dashboard_id": { - "type": "string" - }, - "uuid": { - "type": "string" - } - }, - "type": "object" - }, - "EmbeddedDashboardRestApi.get": { - "properties": { - "uuid": { - "format": "uuid", - "type": "string" - } - }, - "type": "object" - }, - "EmbeddedDashboardRestApi.get_list": { - "properties": { - "uuid": { - "format": "uuid", - "type": "string" - } - }, - "type": "object" - }, - "EmbeddedDashboardRestApi.post": { - "properties": { - "uuid": { - "format": "uuid", - "type": "string" + "type": "array" } }, "type": "object" }, - "EmbeddedDashboardRestApi.put": { + "DistinctResultResponse": { "properties": { - "uuid": { - "format": "uuid", + "text": { + "description": "The distinct item", "type": "string" } }, "type": "object" }, - "ExplorePermalinkPostSchema": { + "EmbeddedDashboardConfig": { "properties": { "allowed_domains": { "items": { @@ -5188,6 +5512,25 @@ }, "type": "object" }, + "ExploreContextSchema": { + "properties": { + "dataset": { + "$ref": "#/components/schemas/Dataset" + }, + "form_data": { + "description": "Form data from the Explore controls used to form the chart's data query.", + "type": "object" + }, + "message": { + "description": "Any message related to the processed request.", + "type": "string" + }, + "slice": { + "$ref": "#/components/schemas/Slice" + } + }, + "type": "object" + }, "ExplorePermalinkPostSchema": { "properties": { "formData": { @@ -5730,8 +6073,7 @@ "type": "string" }, "tracking_url": { - "nullable": true, - "type": "string" + "readOnly": true } }, "required": [ @@ -5840,6 +6182,10 @@ }, "RelatedResultResponse": { "properties": { + "extra": { + "description": "The extra metadata for related item", + "type": "object" + }, "text": { "description": "The related item string representation", "type": "string" @@ -6027,6 +6373,9 @@ "nullable": true, "type": "string" }, + "extra": { + "readOnly": true + }, "force_screenshot": { "nullable": true, "type": "boolean" @@ -6211,7 +6560,7 @@ "type": "boolean" }, "changed_by": { - "$ref": "#/components/schemas/ReportScheduleRestApi.get_list.User" + "$ref": "#/components/schemas/ReportScheduleRestApi.get_list.User1" }, "changed_on": { "format": "date-time", @@ -6255,6 +6604,9 @@ "nullable": true, "type": "string" }, + "extra": { + "readOnly": true + }, "id": { "format": "int32", "type": "integer" @@ -6274,7 +6626,7 @@ "type": "string" }, "owners": { - "$ref": "#/components/schemas/ReportScheduleRestApi.get_list.User1" + "$ref": "#/components/schemas/ReportScheduleRestApi.get_list.User" }, "recipients": { "$ref": "#/components/schemas/ReportScheduleRestApi.get_list.ReportRecipients" @@ -6318,6 +6670,10 @@ "maxLength": 64, "type": "string" }, + "id": { + "format": "int32", + "type": "integer" + }, "last_name": { "maxLength": 64, "type": "string" @@ -6335,10 +6691,6 @@ "maxLength": 64, "type": "string" }, - "id": { - "format": "int32", - "type": "integer" - }, "last_name": { "maxLength": 64, "type": "string" @@ -7143,6 +7495,9 @@ "nullable": true, "type": "string" }, + "extra": { + "type": "object" + }, "force_screenshot": { "type": "boolean" }, @@ -7867,6 +8222,9 @@ }, "SavedQueryRestApi.get": { "properties": { + "changed_on_delta_humanized": { + "readOnly": true + }, "created_by": { "$ref": "#/components/schemas/SavedQueryRestApi.get.User" }, @@ -7897,6 +8255,10 @@ }, "sql_tables": { "readOnly": true + }, + "template_parameters": { + "nullable": true, + "type": "string" } }, "type": "object" @@ -8059,6 +8421,10 @@ "sql": { "nullable": true, "type": "string" + }, + "template_parameters": { + "nullable": true, + "type": "string" } }, "type": "object" @@ -8087,6 +8453,10 @@ "sql": { "nullable": true, "type": "string" + }, + "template_parameters": { + "nullable": true, + "type": "string" } }, "type": "object" @@ -8112,6 +8482,85 @@ }, "type": "object" }, + "Slice": { + "properties": { + "cache_timeout": { + "description": "Duration (in seconds) of the caching timeout for this chart.", + "format": "int32", + "type": "integer" + }, + "certification_details": { + "description": "Details of the certification.", + "type": "string" + }, + "certified_by": { + "description": "Person or group that has certified this dashboard.", + "type": "string" + }, + "changed_on": { + "description": "Timestamp of the last modification.", + "type": "string" + }, + "changed_on_humanized": { + "description": "Timestamp of the last modification in human readable form.", + "type": "string" + }, + "datasource": { + "description": "Datasource identifier.", + "type": "string" + }, + "description": { + "description": "Slice description.", + "type": "string" + }, + "description_markeddown": { + "description": "Sanitized HTML version of the chart description.", + "type": "string" + }, + "edit_url": { + "description": "The URL for editing the slice.", + "type": "string" + }, + "form_data": { + "description": "Form data associated with the slice.", + "type": "object" + }, + "is_managed_externally": { + "description": "If the chart is managed outside externally.", + "type": "boolean" + }, + "modified": { + "description": "Last modification in human readable form.", + "type": "string" + }, + "owners": { + "description": "Owners identifiers.", + "items": { + "format": "int32", + "type": "integer" + }, + "type": "array" + }, + "query_context": { + "description": "The context associated with the query.", + "type": "object" + }, + "slice_id": { + "description": "The slice ID.", + "format": "int32", + "type": "integer" + }, + "slice_name": { + "description": "The slice name.", + "type": "string" + }, + "slice_url": { + "description": "The slice URL.", + "type": "string" + } + }, + "type": "object" + }, "TableExtraMetadataResponseSchema": { "properties": { "clustering": { @@ -8626,6 +9075,17 @@ }, "type": "object" }, + "queries_get_updated_since_schema": { + "properties": { + "last_updated_ms": { + "type": "number" + } + }, + "required": [ + "last_updated_ms" + ], + "type": "object" + }, "screenshot_query_schema": { "properties": { "force": { @@ -8767,99 +9227,6 @@ ] } }, - "/api/v1/annotation_layer/": { - "delete": { - "description": "Deletes multiple annotation layers in a bulk operation.", - "parameters": [ - { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/advanced_data_type_convert_schema" - } - } - }, - "in": "query", - "name": "q" - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AdvancedDataTypeSchema" - } - } - }, - "description": "AdvancedDataTypeResponse object has been returned." - }, - "400": { - "$ref": "#/components/responses/400" - }, - "401": { - "$ref": "#/components/responses/401" - }, - "404": { - "$ref": "#/components/responses/404" - }, - "500": { - "$ref": "#/components/responses/500" - } - }, - "security": [ - { - "jwt": [] - } - ], - "summary": "Returns a AdvancedDataTypeResponse object populated with the passed in args.", - "tags": [ - "Advanced Data Type" - ] - } - }, - "/api/v1/advanced_data_type/types": { - "get": { - "description": "Returns a list of available advanced data types.", - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "properties": { - "result": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - } - } - }, - "description": "a successful return of the available advanced data types has taken place." - }, - "401": { - "$ref": "#/components/responses/401" - }, - "404": { - "$ref": "#/components/responses/404" - }, - "500": { - "$ref": "#/components/responses/500" - } - }, - "security": [ - { - "jwt": [] - } - ], - "tags": [ - "Advanced Data Type" - ] - } - }, "/api/v1/annotation_layer/": { "delete": { "description": "Deletes multiple annotation layers in a bulk operation.", @@ -10004,13 +10371,49 @@ } } }, - "description": "Async event results" + "description": "Async event results" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "AsyncEventsRestApi" + ] + } + }, + "/api/v1/available_domains/": { + "get": { + "description": "Get all available domains", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "result": { + "$ref": "#/components/schemas/AvailableDomainsSchema" + } + }, + "type": "object" + } + } + }, + "description": "a list of available domains" }, "401": { "$ref": "#/components/responses/401" }, - "500": { - "$ref": "#/components/responses/500" + "403": { + "$ref": "#/components/responses/403" } }, "security": [ @@ -10019,7 +10422,7 @@ } ], "tags": [ - "AsyncEventsRestApi" + "Available Domains" ] } }, @@ -11004,6 +11407,14 @@ "schema": { "type": "string" } + }, + { + "description": "Should the queries be forced to load from the source", + "in": "query", + "name": "force", + "schema": { + "type": "boolean" + } } ], "responses": { @@ -13806,6 +14217,16 @@ "description": "Name of the SQLAlchemy engine", "type": "string" }, + "engine_information": { + "description": "Dict with public properties form the DB Engine", + "properties": { + "supports_file_upload": { + "description": "Whether the engine supports file uploads", + "type": "boolean" + } + }, + "type": "object" + }, "name": { "description": "Name of the database", "type": "string" @@ -13821,10 +14242,6 @@ "sqlalchemy_uri_placeholder": { "description": "Example placeholder for the SQLAlchemy URI", "type": "string" - }, - "engine_information": { - "description": "Object with properties we want to expose from our DB engine", - "type": "object" } }, "type": "object" @@ -14121,26 +14538,16 @@ ] }, "get": { - "description": "Get an item model", + "description": "Get a database", "parameters": [ { + "description": "The database id", "in": "path", "name": "pk", "required": true, "schema": { "type": "integer" } - }, - { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/get_item_schema" - } - } - }, - "in": "query", - "name": "q" } ], "responses": { @@ -14148,52 +14555,11 @@ "content": { "application/json": { "schema": { - "properties": { - "description_columns": { - "properties": { - "column_name": { - "description": "The description for the column name. Will be translated by babel", - "example": "A Nice description for the column", - "type": "string" - } - }, - "type": "object" - }, - "id": { - "description": "The item id", - "type": "string" - }, - "label_columns": { - "properties": { - "column_name": { - "description": "The label for the column name. Will be translated by babel", - "example": "A Nice label for the column", - "type": "string" - } - }, - "type": "object" - }, - "result": { - "$ref": "#/components/schemas/DatabaseRestApi.get" - }, - "show_columns": { - "description": "A list of columns", - "items": { - "type": "string" - }, - "type": "array" - }, - "show_title": { - "description": "A title to render. Will be translated by babel", - "example": "Show Item Details", - "type": "string" - } - }, "type": "object" } } }, - "description": "Item from Model" + "description": "Database" }, "400": { "$ref": "#/components/responses/400" @@ -14201,9 +14567,6 @@ "401": { "$ref": "#/components/responses/401" }, - "404": { - "$ref": "#/components/responses/404" - }, "422": { "$ref": "#/components/responses/422" }, @@ -14576,36 +14939,17 @@ ] } }, - "/api/v1/database/{pk}/select_star/{table_name}/{schema_name}/": { - "get": { - "description": "Get database select star for table", + "/api/v1/database/{pk}/ssh_tunnel/": { + "delete": { + "description": "Deletes a SSH Tunnel.", "parameters": [ { - "description": "The database id", "in": "path", "name": "pk", "required": true, "schema": { "type": "integer" } - }, - { - "description": "Table name", - "in": "path", - "name": "table_name", - "required": true, - "schema": { - "type": "string" - } - }, - { - "description": "Table schema", - "in": "path", - "name": "schema_name", - "required": true, - "schema": { - "type": "string" - } } ], "responses": { @@ -14613,18 +14957,23 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SelectStarResponseSchema" + "properties": { + "message": { + "type": "string" + } + }, + "type": "object" } } }, - "description": "SQL statement for a select star for table" - }, - "400": { - "$ref": "#/components/responses/400" + "description": "SSH Tunnel deleted" }, "401": { "$ref": "#/components/responses/401" }, + "403": { + "$ref": "#/components/responses/403" + }, "404": { "$ref": "#/components/responses/404" }, @@ -14784,7 +15133,7 @@ ] } }, - "/api/v1/database/{pk}/validate_sql": { + "/api/v1/database/{pk}/validate_sql/": { "post": { "description": "Validates arbitrary SQL.", "parameters": [ @@ -15220,16 +15569,6 @@ "/api/v1/dataset/duplicate": { "post": { "description": "Duplicates a Dataset", - "parameters": [ - { - "in": "path", - "name": "pk", - "required": true, - "schema": { - "type": "integer" - } - } - ], "requestBody": { "content": { "application/json": { @@ -15242,20 +15581,23 @@ "required": true }, "responses": { - "200": { + "201": { "content": { "application/json": { "schema": { "properties": { - "message": { - "type": "string" + "id": { + "type": "number" + }, + "result": { + "$ref": "#/components/schemas/DatasetDuplicateSchema" } }, "type": "object" } } }, - "description": "Dataset duplicate" + "description": "Dataset duplicated" }, "400": { "$ref": "#/components/responses/400" @@ -15928,23 +16270,17 @@ ] } }, - "/api/v1/dataset/{pk}/samples": { + "/api/v1/embedded_dashboard/{uuid}": { "get": { - "description": "get samples from a Dataset", + "description": "Get a report schedule log", "parameters": [ { + "description": "The embedded configuration uuid", "in": "path", - "name": "pk", + "name": "uuid", "required": true, "schema": { - "type": "integer" - } - }, - { - "in": "query", - "name": "force", - "schema": { - "type": "boolean" + "type": "string" } } ], @@ -15955,27 +16291,21 @@ "schema": { "properties": { "result": { - "$ref": "#/components/schemas/ChartDataResponseResult" + "$ref": "#/components/schemas/EmbeddedDashboardResponseSchema" } }, "type": "object" } } }, - "description": "Dataset samples" + "description": "Result contains the embedded dashboard configuration" }, "401": { "$ref": "#/components/responses/401" }, - "403": { - "$ref": "#/components/responses/403" - }, "404": { "$ref": "#/components/responses/404" }, - "422": { - "$ref": "#/components/responses/422" - }, "500": { "$ref": "#/components/responses/500" } @@ -15986,19 +16316,45 @@ } ], "tags": [ - "Datasets" + "Embedded Dashboard" ] } }, - "/api/v1/embedded_dashboard/{uuid}": { + "/api/v1/explore/": { "get": { - "description": "Get a report schedule log", + "description": "Assembles Explore related information (form_data, slice, dataset)\\n in a single endpoint.

\\nThe information can be assembled from:
- The cache using a form_data_key
- The metadata database using a permalink_key
- Build from scratch using dataset or slice identifiers.", "parameters": [ { - "description": "The embedded configuration uuid", - "in": "path", - "name": "uuid", - "required": true, + "in": "query", + "name": "form_data_key", + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "permalink_key", + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "slice_id", + "schema": { + "type": "integer" + } + }, + { + "in": "query", + "name": "datasource_id", + "schema": { + "type": "integer" + } + }, + { + "in": "query", + "name": "datasource_type", "schema": { "type": "string" } @@ -16009,16 +16365,14 @@ "content": { "application/json": { "schema": { - "properties": { - "result": { - "$ref": "#/components/schemas/EmbeddedDashboardResponseSchema" - } - }, - "type": "object" + "$ref": "#/components/schemas/ExploreContextSchema" } } }, - "description": "Result contains the embedded dashboard configuration" + "description": "Returns the initial context." + }, + "400": { + "$ref": "#/components/responses/400" }, "401": { "$ref": "#/components/responses/401" @@ -16026,6 +16380,9 @@ "404": { "$ref": "#/components/responses/404" }, + "422": { + "$ref": "#/components/responses/422" + }, "500": { "$ref": "#/components/responses/500" } @@ -16035,8 +16392,9 @@ "jwt": [] } ], + "summary": "Assembles Explore related information (form_data, slice, dataset)\\n in a single endpoint.", "tags": [ - "Embedded Dashboard" + "Explore" ] } }, @@ -17001,6 +17359,65 @@ ] } }, + "/api/v1/query/updated_since": { + "get": { + "parameters": [ + { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/queries_get_updated_since_schema" + } + } + }, + "in": "query", + "name": "q" + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "result": { + "description": "A List of queries that changed after last_updated_ms", + "items": { + "$ref": "#/components/schemas/QueryRestApi.get" + }, + "type": "array" + } + }, + "type": "object" + } + } + }, + "description": "Queries list" + }, + "400": { + "$ref": "#/components/responses/400" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "404": { + "$ref": "#/components/responses/404" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "summary": "Get a list of queries that changed after last_updated_ms", + "tags": [ + "Queries" + ] + } + }, "/api/v1/query/{pk}": { "get": { "description": "Get query detail information.", diff --git a/superset-frontend/src/SqlLab/components/QueryAutoRefresh/index.tsx b/superset-frontend/src/SqlLab/components/QueryAutoRefresh/index.tsx index eb3e6f4c38f87..2d01e724e2479 100644 --- a/superset-frontend/src/SqlLab/components/QueryAutoRefresh/index.tsx +++ b/superset-frontend/src/SqlLab/components/QueryAutoRefresh/index.tsx @@ -18,10 +18,12 @@ */ import { useState } from 'react'; import { isObject } from 'lodash'; +import rison from 'rison'; import { SupersetClient, Query, runningQueryStateList, + QueryResponse, } from '@superset-ui/core'; import { QueryDictionary } from 'src/SqlLab/types'; import useInterval from 'src/SqlLab/utils/useInterval'; @@ -62,22 +64,30 @@ function QueryAutoRefresh({ refreshQueries, queriesLastUpdate, }: QueryAutoRefreshProps) { - // We do not want to spam requests in the case of slow connections and potentially recieve responses out of order + // We do not want to spam requests in the case of slow connections and potentially receive responses out of order // pendingRequest check ensures we only have one active http call to check for query statuses const [pendingRequest, setPendingRequest] = useState(false); const checkForRefresh = () => { if (!pendingRequest && shouldCheckForQueries(queries)) { + const params = rison.encode({ + last_updated_ms: queriesLastUpdate - QUERY_UPDATE_BUFFER_MS, + }); + setPendingRequest(true); SupersetClient.get({ - endpoint: `/superset/queries/${ - queriesLastUpdate - QUERY_UPDATE_BUFFER_MS - }`, + endpoint: `/api/v1/query/updated_since?q=${params}`, timeout: QUERY_TIMEOUT_LIMIT, }) .then(({ json }) => { if (json) { - refreshQueries?.(json); + const jsonPayload = json as { result?: QueryResponse[] }; + const queries = + jsonPayload?.result?.reduce((acc, current) => { + acc[current.id] = current; + return acc; + }, {}) ?? {}; + refreshQueries?.(queries); } }) .catch(() => {}) diff --git a/superset/constants.py b/superset/constants.py index ea7920ff2fd7a..27aac03618281 100644 --- a/superset/constants.py +++ b/superset/constants.py @@ -140,6 +140,7 @@ class RouteMethod: # pylint: disable=too-few-public-methods "get_data": "read", "samples": "read", "delete_ssh_tunnel": "write", + "get_updated_since": "read", } EXTRA_FORM_DATA_APPEND_KEYS = { diff --git a/superset/queries/api.py b/superset/queries/api.py index 1fb342f067d3d..6daadaa47ee22 100644 --- a/superset/queries/api.py +++ b/superset/queries/api.py @@ -15,15 +15,29 @@ # specific language governing permissions and limitations # under the License. import logging +from typing import Any +from flask_appbuilder.api import expose, protect, rison, safe from flask_appbuilder.models.sqla.interface import SQLAInterface +from superset import event_logger from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP, RouteMethod from superset.databases.filters import DatabaseFilter +from superset.exceptions import SupersetException from superset.models.sql_lab import Query +from superset.queries.dao import QueryDAO from superset.queries.filters import QueryFilter -from superset.queries.schemas import openapi_spec_methods_override, QuerySchema -from superset.views.base_api import BaseSupersetModelRestApi, RelatedFieldFilter +from superset.queries.schemas import ( + openapi_spec_methods_override, + queries_get_updated_since_schema, + QuerySchema, +) +from superset.superset_typing import FlaskResponse +from superset.views.base_api import ( + BaseSupersetModelRestApi, + RelatedFieldFilter, + statsd_metrics, +) from superset.views.filters import BaseFilterRelatedUsers, FilterRelatedOwners logger = logging.getLogger(__name__) @@ -43,6 +57,11 @@ class QueryRestApi(BaseSupersetModelRestApi): RouteMethod.GET_LIST, RouteMethod.RELATED, RouteMethod.DISTINCT, + "get_updated_since", + } + + apispec_parameter_schemas = { + "queries_get_updated_since_schema": queries_get_updated_since_schema, } list_columns = [ @@ -123,3 +142,56 @@ class QueryRestApi(BaseSupersetModelRestApi): base_related_field_filters = {"database": [["id", DatabaseFilter, lambda: []]]} allowed_rel_fields = {"database", "user"} allowed_distinct_fields = {"status"} + + @expose("/updated_since") + @protect() + @safe + @rison(queries_get_updated_since_schema) + @statsd_metrics + @event_logger.log_this_with_context( + action=lambda self, *args, **kwargs: f"{self.__class__.__name__}" + f".get_updated_since", + log_to_statsd=False, + ) + def get_updated_since(self, **kwargs: Any) -> FlaskResponse: + """Get a list of queries that changed after last_updated_ms + --- + get: + summary: Get a list of queries that changed after last_updated_ms + parameters: + - in: query + name: q + content: + application/json: + schema: + $ref: '#/components/schemas/queries_get_updated_since_schema' + responses: + 200: + description: Queries list + content: + application/json: + schema: + type: object + properties: + result: + description: >- + A List of queries that changed after last_updated_ms + type: array + items: + $ref: '#/components/schemas/{{self.__class__.__name__}}.get' + 400: + $ref: '#/components/responses/400' + 401: + $ref: '#/components/responses/401' + 404: + $ref: '#/components/responses/404' + 500: + $ref: '#/components/responses/500' + """ + try: + last_updated_ms = kwargs["rison"].get("last_updated_ms", 0) + queries = QueryDAO.get_queries_changed_after(last_updated_ms) + payload = [q.to_dict() for q in queries] + return self.response(200, result=payload) + except SupersetException as ex: + return self.response(ex.status, message=ex.message) diff --git a/superset/queries/dao.py b/superset/queries/dao.py index c7d59343e8587..45adf6c2b3ea5 100644 --- a/superset/queries/dao.py +++ b/superset/queries/dao.py @@ -16,12 +16,13 @@ # under the License. import logging from datetime import datetime -from typing import Any, Dict +from typing import Any, Dict, List, Union from superset.dao.base import BaseDAO from superset.extensions import db from superset.models.sql_lab import Query, SavedQuery from superset.queries.filters import QueryFilter +from superset.utils.core import get_user_id logger = logging.getLogger(__name__) @@ -56,3 +57,14 @@ def save_metadata(query: Query, payload: Dict[str, Any]) -> None: columns = payload.get("columns", {}) db.session.add(query) query.set_extra_json_key("columns", columns) + + @staticmethod + def get_queries_changed_after(last_updated_ms: Union[float, int]) -> List[Query]: + # UTC date time, same that is stored in the DB. + last_updated_dt = datetime.utcfromtimestamp(last_updated_ms / 1000) + + return ( + db.session.query(Query) + .filter(Query.user_id == get_user_id(), Query.changed_on >= last_updated_dt) + .all() + ) diff --git a/superset/queries/schemas.py b/superset/queries/schemas.py index f11cf37127756..027f9da8cfe9e 100644 --- a/superset/queries/schemas.py +++ b/superset/queries/schemas.py @@ -33,6 +33,14 @@ }, } +queries_get_updated_since_schema = { + "type": "object", + "properties": { + "last_updated_ms": {"type": "number"}, + }, + "required": ["last_updated_ms"], +} + class DatabaseSchema(Schema): database_name = fields.String() diff --git a/superset/views/core.py b/superset/views/core.py index 534f8f667d707..e1fc1ea4026e7 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -2567,6 +2567,7 @@ def fetch_datasource_metadata(self) -> FlaskResponse: # pylint: disable=no-self @event_logger.log_this @expose("/queries/") @expose("/queries/") + @deprecated() def queries(self, last_updated_ms: Union[float, int]) -> FlaskResponse: """ Get the updated queries. diff --git a/tests/integration_tests/queries/api_tests.py b/tests/integration_tests/queries/api_tests.py index eaf4e00576573..694d30e88a6ba 100644 --- a/tests/integration_tests/queries/api_tests.py +++ b/tests/integration_tests/queries/api_tests.py @@ -51,6 +51,7 @@ def insert_query( rows: int = 100, tab_name: str = "", status: str = "success", + changed_on: datetime = datetime(2020, 1, 1), ) -> Query: database = db.session.query(Database).get(database_id) user = db.session.query(security_manager.user_model).get(user_id) @@ -66,7 +67,7 @@ def insert_query( rows=rows, tab_name=tab_name, status=status, - changed_on=datetime(2020, 1, 1), + changed_on=changed_on, ) db.session.add(query) db.session.commit() @@ -392,3 +393,58 @@ def test_get_list_query_no_data_access(self): # rollback changes db.session.delete(query) db.session.commit() + + def test_get_updated_since(self): + """ + Query API: Test get queries updated since timestamp + """ + now = datetime.utcnow() + client_id = self.get_random_string() + + admin = self.get_user("admin") + example_db = get_example_database() + + old_query = self.insert_query( + example_db.id, + admin.id, + self.get_random_string(), + sql="SELECT col1, col2 from table1", + select_sql="SELECT col1, col2 from table1", + executed_sql="SELECT col1, col2 from table1 LIMIT 100", + changed_on=now - timedelta(days=3), + ) + updated_query = self.insert_query( + example_db.id, + admin.id, + client_id, + sql="SELECT col1, col2 from table1", + select_sql="SELECT col1, col2 from table1", + executed_sql="SELECT col1, col2 from table1 LIMIT 100", + changed_on=now - timedelta(days=1), + ) + + self.login(username="admin") + timestamp = datetime.timestamp(now - timedelta(days=2)) * 1000 + uri = f"api/v1/query/updated_since?q={prison.dumps({'last_updated_ms': timestamp})}" + rv = self.client.get(uri) + self.assertEqual(rv.status_code, 200) + + expected_result = updated_query.to_dict() + data = json.loads(rv.data.decode("utf-8")) + self.assertEqual(len(data["result"]), 1) + for key, value in data["result"][0].items(): + # We can't assert timestamp + print(key) + if key not in ( + "changedOn", + "changed_on", + "end_time", + "start_running_time", + "start_time", + "id", + ): + self.assertEqual(value, expected_result[key]) + # rollback changes + db.session.delete(old_query) + db.session.delete(updated_query) + db.session.commit() diff --git a/tests/unit_tests/dao/queries_test.py b/tests/unit_tests/dao/queries_test.py index 8e2a458434cd9..d99f93c1d95dc 100644 --- a/tests/unit_tests/dao/queries_test.py +++ b/tests/unit_tests/dao/queries_test.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. import json +from datetime import datetime, timedelta from typing import Iterator import pytest @@ -53,3 +54,58 @@ def test_query_dao_save_metadata(session: Session) -> None: query = session.query(Query).one() QueryDAO.save_metadata(query=query, payload={"columns": []}) assert query.extra.get("columns", None) == [] + + +def test_query_dao_get_queries_changed_after(session: Session) -> None: + from superset.models.core import Database + from superset.models.sql_lab import Query + + engine = session.get_bind() + Query.metadata.create_all(engine) # pylint: disable=no-member + + db = Database(database_name="my_database", sqlalchemy_uri="sqlite://") + + now = datetime.utcnow() + + old_query_obj = Query( + client_id="foo", + database=db, + tab_name="test_tab", + sql_editor_id="test_editor_id", + sql="select * from bar", + select_sql="select * from bar", + executed_sql="select * from bar", + limit=100, + select_as_cta=False, + rows=100, + error_message="none", + results_key="abc", + changed_on=now - timedelta(days=3), + ) + + updated_query_obj = Query( + client_id="updated_foo", + database=db, + tab_name="test_tab", + sql_editor_id="test_editor_id", + sql="select * from foo", + select_sql="select * from foo", + executed_sql="select * from foo", + limit=100, + select_as_cta=False, + rows=100, + error_message="none", + results_key="abc", + changed_on=now - timedelta(days=1), + ) + + session.add(db) + session.add(old_query_obj) + session.add(updated_query_obj) + + from superset.queries.dao import QueryDAO + + timestamp = datetime.timestamp(now - timedelta(days=2)) * 1000 + result = QueryDAO.get_queries_changed_after(timestamp) + assert len(result) == 1 + assert result[0].client_id == "updated_foo" From cfd2135fc92bf0b3d05e24b90de2d049c519cec2 Mon Sep 17 00:00:00 2001 From: Diego Medina Date: Fri, 6 Jan 2023 17:09:13 -0300 Subject: [PATCH 2/2] cleanup --- tests/integration_tests/queries/api_tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration_tests/queries/api_tests.py b/tests/integration_tests/queries/api_tests.py index 694d30e88a6ba..43c7107dc3db1 100644 --- a/tests/integration_tests/queries/api_tests.py +++ b/tests/integration_tests/queries/api_tests.py @@ -434,7 +434,6 @@ def test_get_updated_since(self): self.assertEqual(len(data["result"]), 1) for key, value in data["result"][0].items(): # We can't assert timestamp - print(key) if key not in ( "changedOn", "changed_on",