refactor: Repeated boilerplate code between upload to database forms (#…

…16756)

* abstract boilerplate code into class and rename csv to file

* add db migration

* fix some stuff

* more renaming of csv to file

* rename in translations

* update down revision

* update down revision

* bump chart version

* switch to alter column name approach in db migration

* fix db migration for MySQL

* db migration conflict

docs/src/pages/docs/installation/kubernetes.mdx

@@ -185,12 +185,12 @@ Data source definitions can be automatically declared by providing key/value yam
 extraConfigs:
   datasources-init.yaml: |
     databases:
-    - allow_csv_upload: true
+    - allow_file_upload: true
       allow_ctas: true
       allow_cvas: true
       database_name: example-db
       extra: "{\r\n    \"metadata_params\": {},\r\n    \"engine_params\": {},\r\n    \"\
-        metadata_cache_timeout\": {},\r\n    \"schemas_allowed_for_csv_upload\": []\r\n\
+        metadata_cache_timeout\": {},\r\n    \"schemas_allowed_for_file_upload\": []\r\n\
         }"
       sqlalchemy_uri: example://example-db.local
       tables: []

helm/superset/Chart.yaml

@@ -22,7 +22,7 @@ maintainers:
   - name: craig-rueda
     email: craig@craigrueda.com
     url: https://github.com/craig-rueda
-version: 0.3.10
+version: 0.3.11
 dependencies:
 - name: postgresql
   version: 10.2.0

helm/superset/values.yaml

@@ -80,12 +80,12 @@ extraSecretEnv: {}
 extraConfigs: {}
   # datasources-init.yaml: |
   #     databases:
-  #     - allow_csv_upload: true
+  #     - allow_file_upload: true
   #       allow_ctas: true
   #       allow_cvas: true
   #       database_name: example-db
   #       extra: "{\r\n    \"metadata_params\": {},\r\n    \"engine_params\": {},\r\n    \"\
-  #         metadata_cache_timeout\": {},\r\n    \"schemas_allowed_for_csv_upload\": []\r\n\
+  #         metadata_cache_timeout\": {},\r\n    \"schemas_allowed_for_file_upload\": []\r\n\
   #         }"
   #       sqlalchemy_uri: example://example-db.local
   #       tables: []

superset-frontend/src/components/DatabaseSelector/DatabaseSelector.test.tsx

@@ -66,7 +66,7 @@ beforeEach(() => {
           description_columns: {},
           ids: [1, 2],
           label_columns: {
-            allow_csv_upload: 'Allow Csv Upload',
+            allow_file_upload: 'Allow Csv Upload',
             allow_ctas: 'Allow Ctas',
             allow_cvas: 'Allow Cvas',
             allow_dml: 'Allow Dml',
@@ -88,7 +88,7 @@ beforeEach(() => {
             id: 'Id',
           },
           list_columns: [
-            'allow_csv_upload',
+            'allow_file_upload',
             'allow_ctas',
             'allow_cvas',
             'allow_dml',
@@ -110,7 +110,7 @@ beforeEach(() => {
           ],
           list_title: 'List Database',
           order_columns: [
-            'allow_csv_upload',
+            'allow_file_upload',
             'allow_dml',
             'allow_run_async',
             'changed_on',
@@ -121,7 +121,7 @@ beforeEach(() => {
           ],
           result: [
             {
-              allow_csv_upload: false,
+              allow_file_upload: false,
               allow_ctas: false,
               allow_cvas: false,
               allow_dml: false,

superset-frontend/src/views/CRUD/data/database/DatabaseList.test.jsx

@@ -54,7 +54,7 @@ const mockdatabases = [...new Array(3)].map((_, i) => ({
   backend: 'postgresql',
   allow_run_async: true,
   allow_dml: false,
-  allow_csv_upload: true,
+  allow_file_upload: true,
   expose_in_sqllab: false,
   changed_on_delta_humanized: `${i} day(s) ago`,
   changed_on: new Date().toISOString,

superset-frontend/src/views/CRUD/data/database/DatabaseList.tsx

@@ -267,13 +267,13 @@ function DatabaseList({ addDangerToast, addSuccessToast }: DatabaseListProps) {
         size: 'sm',
       },
       {
-        accessor: 'allow_csv_upload',
+        accessor: 'allow_file_upload',
         Header: t('CSV upload'),
         Cell: ({
           row: {
-            original: { allow_csv_upload: allowCSVUpload },
+            original: { allow_file_upload: allowFileUpload },
           },
-        }: any) => <BooleanDisplay value={allowCSVUpload} />,
+        }: any) => <BooleanDisplay value={allowFileUpload} />,
         size: 'md',
       },
       {

superset-frontend/src/views/CRUD/data/database/DatabaseModal/ExtraOptions.tsx

@@ -371,9 +371,9 @@ const ExtraOptions = ({
           <div className="input-container">
             <input
               type="text"
-              name="schemas_allowed_for_csv_upload"
+              name="schemas_allowed_for_file_upload"
               value={(
-                db?.extra_json?.schemas_allowed_for_csv_upload || []
+                db?.extra_json?.schemas_allowed_for_file_upload || []
               ).join(',')}
               placeholder="schema1,schema2"
               onChange={onExtraInputChange}
@@ -410,9 +410,9 @@ const ExtraOptions = ({
         <StyledInputContainer css={{ ...no_margin_bottom }}>
           <div className="input-container">
             <IndeterminateCheckbox
-              id="allow_csv_upload"
+              id="allow_file_upload"
               indeterminate={false}
-              checked={!!db?.allow_csv_upload}
+              checked={!!db?.allow_file_upload}
               onChange={onInputChange}
               labelText={t('Allow data upload')}
             />

superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx

@@ -237,12 +237,12 @@ function dbReducer(
           },
         };
       }
-      if (action.payload.name === 'schemas_allowed_for_csv_upload') {
+      if (action.payload.name === 'schemas_allowed_for_file_upload') {
         return {
           ...trimmedState,
           extra_json: {
             ...trimmedState.extra_json,
-            schemas_allowed_for_csv_upload: (action.payload.value || '').split(
+            schemas_allowed_for_file_upload: (action.payload.value || '').split(
               ',',
             ),
           },
@@ -346,8 +346,8 @@ function dbReducer(
           ...JSON.parse(action.payload.extra || ''),
           metadata_params: JSON.stringify(extra_json?.metadata_params),
           engine_params: JSON.stringify(extra_json?.engine_params),
-          schemas_allowed_for_csv_upload:
-            extra_json?.schemas_allowed_for_csv_upload,
+          schemas_allowed_for_file_upload:
+            extra_json?.schemas_allowed_for_file_upload,
         };
       }
 
@@ -412,8 +412,8 @@ const serializeExtra = (extraJson: DatabaseObject['extra_json']) =>
     engine_params: JSON.parse(
       ((extraJson?.engine_params as unknown) as string) || '{}',
     ),
-    schemas_allowed_for_csv_upload: (
-      extraJson?.schemas_allowed_for_csv_upload || []
+    schemas_allowed_for_file_upload: (
+      extraJson?.schemas_allowed_for_file_upload || []
     ).filter(schema => schema !== ''),
   });
 

superset-frontend/src/views/CRUD/data/database/types.ts

@@ -72,7 +72,7 @@ export type DatabaseObject = {
   // Security
   encrypted_extra?: string;
   server_cert?: string;
-  allow_csv_upload?: boolean;
+  allow_file_upload?: boolean;
   impersonate_user?: boolean;
   parameters_schema?: Record<string, any>;
 
@@ -87,7 +87,7 @@ export type DatabaseObject = {
       table_cache_timeout?: number; // in Performance
     }; // No field, holds schema and table timeout
     allows_virtual_table_explore?: boolean; // in SQL Lab
-    schemas_allowed_for_csv_upload?: string[]; // in Security
+    schemas_allowed_for_file_upload?: string[]; // in Security
     cancel_query_on_windows_unload?: boolean; // in Performance
 
     version?: string;

superset/config.py

@@ -863,7 +863,7 @@ def CSV_TO_HIVE_UPLOAD_DIRECTORY_FUNC(  # pylint: disable=invalid-name
 UPLOADED_CSV_HIVE_NAMESPACE: Optional[str] = None
 
 # Function that computes the allowed schemas for the CSV uploads.
-# Allowed schemas will be a union of schemas_allowed_for_csv_upload
+# Allowed schemas will be a union of schemas_allowed_for_file_upload
 # db configuration and a result of this function.
 
 # mypy doesn't catch that if case ensures list content being always str

superset/databases/api.py

@@ -104,7 +104,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
         "cache_timeout",
         "expose_in_sqllab",
         "allow_run_async",
-        "allow_csv_upload",
+        "allow_file_upload",
         "configuration_method",
         "allow_ctas",
         "allow_cvas",
@@ -121,7 +121,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
         "sqlalchemy_uri",
     ]
     list_columns = [
-        "allow_csv_upload",
+        "allow_file_upload",
         "allow_ctas",
         "allow_cvas",
         "allow_dml",
@@ -148,7 +148,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
         "cache_timeout",
         "expose_in_sqllab",
         "allow_run_async",
-        "allow_csv_upload",
+        "allow_file_upload",
         "allow_ctas",
         "allow_cvas",
         "allow_dml",
@@ -164,7 +164,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
 
     list_select_columns = list_columns + ["extra", "sqlalchemy_uri", "password"]
     order_columns = [
-        "allow_csv_upload",
+        "allow_file_upload",
         "allow_dml",
         "allow_run_async",
         "changed_on",

superset/databases/commands/export.py

@@ -39,11 +39,11 @@ def parse_extra(extra_payload: str) -> Dict[str, Any]:
         logger.info("Unable to decode `extra` field: %s", extra_payload)
         return {}
 
-    # Fix for DBs saved with an invalid ``schemas_allowed_for_csv_upload``
-    schemas_allowed_for_csv_upload = extra.get("schemas_allowed_for_csv_upload")
-    if isinstance(schemas_allowed_for_csv_upload, str):
-        extra["schemas_allowed_for_csv_upload"] = json.loads(
-            schemas_allowed_for_csv_upload
+    # Fix for DBs saved with an invalid ``schemas_allowed_for_file_upload``
+    schemas_allowed_for_file_upload = extra.get("schemas_allowed_for_file_upload")
+    if isinstance(schemas_allowed_for_file_upload, str):
+        extra["schemas_allowed_for_file_upload"] = json.loads(
+            schemas_allowed_for_file_upload
         )
 
     return extra

superset/databases/schemas.py

@@ -54,7 +54,7 @@
     "as a results backend. Refer to the installation docs "
     "for more information."
 )
-allow_csv_upload_description = (
+allow_file_upload_description = (
     "Allow to upload CSV file data into this database"
     "If selected, please set the schemas allowed for csv upload in Extra."
 )
@@ -108,9 +108,9 @@
     '"table_cache_timeout": 600}**. '
     "If unset, cache will not be enabled for the functionality. "
     "A timeout of 0 indicates that the cache never expires.<br/>"
-    "3. The ``schemas_allowed_for_csv_upload`` is a comma separated list "
+    "3. The ``schemas_allowed_for_file_upload`` is a comma separated list "
     "of schemas that CSVs are allowed to upload to. "
-    'Specify it as **"schemas_allowed_for_csv_upload": '
+    'Specify it as **"schemas_allowed_for_file_upload": '
     '["public", "csv_upload"]**. '
     "If database flavor does not support schema or any schema is allowed "
     "to be accessed, just leave the list empty<br/>"
@@ -355,7 +355,7 @@ class Meta:  # pylint: disable=too-few-public-methods
     )
     expose_in_sqllab = fields.Boolean(description=expose_in_sqllab_description)
     allow_run_async = fields.Boolean(description=allow_run_async_description)
-    allow_csv_upload = fields.Boolean(description=allow_csv_upload_description)
+    allow_file_upload = fields.Boolean(description=allow_file_upload_description)
     allow_ctas = fields.Boolean(description=allow_ctas_description)
     allow_cvas = fields.Boolean(description=allow_cvas_description)
     allow_dml = fields.Boolean(description=allow_dml_description)
@@ -397,7 +397,7 @@ class Meta:  # pylint: disable=too-few-public-methods
     )
     expose_in_sqllab = fields.Boolean(description=expose_in_sqllab_description)
     allow_run_async = fields.Boolean(description=allow_run_async_description)
-    allow_csv_upload = fields.Boolean(description=allow_csv_upload_description)
+    allow_file_upload = fields.Boolean(description=allow_file_upload_description)
     allow_ctas = fields.Boolean(description=allow_ctas_description)
     allow_cvas = fields.Boolean(description=allow_cvas_description)
     allow_dml = fields.Boolean(description=allow_dml_description)
@@ -558,23 +558,23 @@ def fix_schemas_allowed_for_csv_upload(
         self, data: Dict[str, Any], **kwargs: Any
     ) -> Dict[str, Any]:
         """
-        Fix ``schemas_allowed_for_csv_upload`` being a string.
+        Fix ``schemas_allowed_for_file_upload`` being a string.
 
         Due to a bug in the database modal, some databases might have been
-        saved and exported with a string for ``schemas_allowed_for_csv_upload``.
+        saved and exported with a string for ``schemas_allowed_for_file_upload``.
         """
-        schemas_allowed_for_csv_upload = data.get("schemas_allowed_for_csv_upload")
-        if isinstance(schemas_allowed_for_csv_upload, str):
-            data["schemas_allowed_for_csv_upload"] = json.loads(
-                schemas_allowed_for_csv_upload
+        schemas_allowed_for_file_upload = data.get("schemas_allowed_for_file_upload")
+        if isinstance(schemas_allowed_for_file_upload, str):
+            data["schemas_allowed_for_file_upload"] = json.loads(
+                schemas_allowed_for_file_upload
             )
 
         return data
 
     metadata_params = fields.Dict(keys=fields.Str(), values=fields.Raw())
     engine_params = fields.Dict(keys=fields.Str(), values=fields.Raw())
     metadata_cache_timeout = fields.Dict(keys=fields.Str(), values=fields.Integer())
-    schemas_allowed_for_csv_upload = fields.List(fields.String())
+    schemas_allowed_for_file_upload = fields.List(fields.String())
     cost_estimate_enabled = fields.Boolean()
 
 
@@ -587,7 +587,7 @@ class ImportV1DatabaseSchema(Schema):
     allow_run_async = fields.Boolean()
     allow_ctas = fields.Boolean()
     allow_cvas = fields.Boolean()
-    allow_csv_upload = fields.Boolean()
+    allow_file_upload = fields.Boolean()
     extra = fields.Nested(ImportV1DatabaseExtraSchema)
     uuid = fields.UUID(required=True)
     version = fields.String(required=True)

superset/migrations/versions/b92d69a6643c_rename_csv_to_file.py

@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""rename_csv_to_file
+
+Revision ID: b92d69a6643c
+Revises: 32646df09c64
+Create Date: 2021-09-19 14:42:20.130368
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "b92d69a6643c"
+down_revision = "32646df09c64"
+
+import sqlalchemy as sa
+from alembic import op
+
+
+def upgrade():
+    with op.batch_alter_table("dbs") as batch_op:
+        batch_op.alter_column(
+            "allow_csv_upload",
+            new_column_name="allow_file_upload",
+            existing_type=sa.Boolean(),
+        )
+
+
+def downgrade():
+    with op.batch_alter_table("dbs") as batch_op:
+        batch_op.alter_column(
+            "allow_file_upload",
+            new_column_name="allow_csv_upload",
+            existing_type=sa.Boolean(),
+        )