fix: SqlaColumn.type overflow on mysql

superset/connectors/sqla/models.py

@@ -900,7 +900,7 @@ def fetch_metadata(self):
 
         for col in table.columns:
             try:
-                datatype = col.type.compile(dialect=db_dialect).upper()
+                datatype = db_engine_spec.column_datatype_to_string(col.type, db_dialect)
             except Exception as e:
                 datatype = 'UNKNOWN'
                 logging.error(

superset/db_engine_specs.py

@@ -529,6 +529,10 @@ def truncate_label(cls, label):
             label = label[:cls.max_column_name_length]
         return label
 
+    @classmethod
+    def column_datatype_to_string(cls, sqla_column_type, dialect):
+        return sqla_column_type.compile(dialect=dialect).upper()
+
 
 class PostgresBaseEngineSpec(BaseEngineSpec):
     """ Abstract class for Postgres 'like' databases """
@@ -864,6 +868,17 @@ def extract_error_message(cls, e):
             pass
         return message
 
+    @classmethod
+    def column_datatype_to_string(cls, sqla_column_type, dialect):
+        datatype = super().column_datatype_to_string(sqla_column_type, dialect)
+        # MySQL dialect started returning long overflowing datatype
+        # as in 'VARCHAR(255) COLLATE UTF8MB4_GENERAL_CI'
+        # and we don't need the verbose collation type
+        str_cutoff = ' COLLATE '
+        if str_cutoff in datatype:
+            datatype = datatype.split(str_cutoff)[0]
+        return datatype
+
 
 class PrestoEngineSpec(BaseEngineSpec):
     engine = 'presto'

tests/base_tests.py

@@ -174,12 +174,15 @@ def revoke_public_access_to_table(self, table):
                     perm.view_menu and table.perm in perm.view_menu.name):
                 security_manager.del_permission_role(public_role, perm)
 
+    def get_main_database(self):
+        return get_main_database(db.session)
+
     def run_sql(self, sql, client_id=None, user_name=None, raise_on_error=False,
                 query_limit=None):
         if user_name:
             self.logout()
             self.login(username=(user_name if user_name else 'admin'))
-        dbid = get_main_database(db.session).id
+        dbid = self.get_main_database().id
         resp = self.get_json_resp(
             '/superset/sql_json/',
             raise_on_error=False,
@@ -195,7 +198,7 @@ def validate_sql(self, sql, client_id=None, user_name=None,
         if user_name:
             self.logout()
             self.login(username=(user_name if user_name else 'admin'))
-        dbid = get_main_database(db.session).id
+        dbid = self.get_main_database().id
         resp = self.get_json_resp(
             '/superset/validate_sql_json/',
             raise_on_error=False,

tests/db_engine_specs_test.py

@@ -814,3 +814,17 @@ def test_pinot_time_expression_sec_1m_grain(self):
         expr = PinotEngineSpec.get_timestamp_expr(col, 'epoch_s', 'P1M')
         result = str(expr.compile())
         self.assertEqual(result, 'DATETIMECONVERT(tstamp, "1:SECONDS:EPOCH", "1:SECONDS:EPOCH", "1:MONTHS")')  # noqa
+
+    def test_column_datatype_to_string(self):
+        main_db = self.get_main_database()
+        sqla_table = main_db.get_table('energy_usage')
+        dialect = main_db.get_dialect()
+        col_names = [
+            main_db.db_engine_spec.column_datatype_to_string(c.type, dialect)
+            for c in sqla_table.columns
+        ]
+        if main_db.backend == 'postgresql':
+            expected = ['VARCHAR(255)', 'VARCHAR(255)', 'DOUBLE PRECISION']
+        else:
+            expected = ['VARCHAR(255)', 'VARCHAR(255)', 'FLOAT']
+        self.assertEquals(col_names, expected)