Merge branch 'master' into upgrade

querybook/migrations/versions/27ed76f75106_extend_datatablecolumn_type_length.py

@@ -0,0 +1,40 @@
+"""Extend DataTableColumn type length
+
+Revision ID: 27ed76f75106
+Revises: 63bde0162416
+Create Date: 2023-01-05 09:43:06.639449
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = "27ed76f75106"
+down_revision = "63bde0162416"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "data_table_column",
+        "type",
+        nullable=True,
+        existing_type=sa.String(length=255),
+        type_=sa.String(length=4096),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "data_table_column",
+        "type",
+        nullable=True,
+        existing_type=sa.String(length=4096),
+        type_=sa.String(length=255),
+    )
+    # ### end Alembic commands ###

querybook/server/const/db.py

@@ -3,6 +3,7 @@
 now = datetime.datetime.utcnow
 utf8mb4_name_length = 191
 name_length = 255
+type_length = 4096
 password_length = 255
 description_length = 5000
 text_length = 65535

querybook/server/models/metastore.py

@@ -9,6 +9,7 @@
     description_length,
     url_length,
     mediumtext_length,
+    type_length,
 )
 from const.metastore import DataTableWarningSeverity
 from lib.sqlalchemy import CRUDMixin, TruncateString
@@ -283,7 +284,7 @@ class DataTableColumn(TruncateString("name", "type", "comment"), Base):
     updated_at = sql.Column(sql.DateTime, default=now)
 
     name = sql.Column(sql.String(length=name_length), index=True)
-    type = sql.Column(sql.String(length=name_length))
+    type = sql.Column(sql.String(length=type_length))
 
     comment = sql.Column(sql.String(length=description_length))
     description = sql.Column(sql.Text(length=mediumtext_length))

querybook/webapp/__tests__/lib/utils/complex-types.test.ts

@@ -0,0 +1,258 @@
+import { parseType, prettyPrintType } from 'lib/utils/complex-types';
+
+test('simple type', () => {
+    expect(parseType('column', 'string')).toEqual({
+        key: 'column',
+        type: 'string',
+    });
+});
+
+test('truncated type', () => {
+    expect(
+        parseType(
+            'column',
+            'struct<date:struct<year:int,month:int,day:int>,hour:int,minute:int,second:int,'
+        )
+    ).toEqual({
+        key: 'column',
+        type: 'struct<date:struct<year:int,month:int,day:int>,hour:int,minute:int,second:int,',
+    });
+
+    // Truncated, but coincidentally matches the regex
+    expect(parseType('column', 'struct<date:struct<hour:int>')).toEqual({
+        key: 'column',
+        type: 'struct<date:struct<hour:int>',
+    });
+});
+
+test('malformed struct type', () => {
+    expect(parseType('column', 'STRUCT <id:string>')).toEqual({
+        key: 'column',
+        type: 'STRUCT <id:string>',
+    });
+});
+
+test('complex type', () => {
+    expect(parseType('column', 'STRUCT<id:string>')).toEqual({
+        key: 'column',
+        type: 'STRUCT<id:string>',
+        children: [
+            {
+                key: 'id',
+                type: 'string',
+            },
+        ],
+    });
+
+    expect(
+        parseType(
+            'column',
+            'struct<date:struct<year:int,month:int,day:int>,hour:int,minute:int,second:int,timeZoneId:string>'
+        )
+    ).toEqual({
+        key: 'column',
+        type: 'struct<date:struct<year:int,month:int,day:int>,hour:int,minute:int,second:int,timeZoneId:string>',
+        children: [
+            {
+                key: 'date',
+                type: 'struct<year:int,month:int,day:int>',
+                children: [
+                    { key: 'year', type: 'int' },
+                    { key: 'month', type: 'int' },
+                    { key: 'day', type: 'int' },
+                ],
+            },
+            { key: 'hour', type: 'int' },
+            { key: 'minute', type: 'int' },
+            { key: 'second', type: 'int' },
+            { key: 'timeZoneId', type: 'string' },
+        ],
+    });
+
+    expect(
+        parseType(
+            'column',
+            'array<struct<size:struct<width:int,height:int,isAspectRatio:boolean>>>'
+        )
+    ).toEqual({
+        key: 'column',
+        type: 'array<struct<size:struct<width:int,height:int,isAspectRatio:boolean>>>',
+        children: [
+            {
+                key: '<element>',
+                type: 'struct<size:struct<width:int,height:int,isAspectRatio:boolean>>',
+                children: [
+                    {
+                        key: 'size',
+                        type: 'struct<width:int,height:int,isAspectRatio:boolean>',
+                        children: [
+                            { key: 'width', type: 'int' },
+                            { key: 'height', type: 'int' },
+                            { key: 'isAspectRatio', type: 'boolean' },
+                        ],
+                    },
+                ],
+            },
+        ],
+    });
+
+    expect(
+        parseType(
+            'column',
+            'struct<purchasePath:string,resultToken:string,sessionId:string,site:struct<eapid:bigint,tpid:bigint>,tests:array<struct<bucketValue:string,experimentId:string,instanceId:string>>,user:struct<guid:string,tuid:string>>'
+        )
+    ).toEqual({
+        key: 'column',
+        type: 'struct<purchasePath:string,resultToken:string,sessionId:string,site:struct<eapid:bigint,tpid:bigint>,tests:array<struct<bucketValue:string,experimentId:string,instanceId:string>>,user:struct<guid:string,tuid:string>>',
+        children: [
+            { key: 'purchasePath', type: 'string' },
+            { key: 'resultToken', type: 'string' },
+            { key: 'sessionId', type: 'string' },
+            {
+                key: 'site',
+                type: 'struct<eapid:bigint,tpid:bigint>',
+                children: [
+                    { key: 'eapid', type: 'bigint' },
+                    { key: 'tpid', type: 'bigint' },
+                ],
+            },
+            {
+                key: 'tests',
+                type: 'array<struct<bucketValue:string,experimentId:string,instanceId:string>>',
+                children: [
+                    {
+                        key: '<element>',
+                        type: 'struct<bucketValue:string,experimentId:string,instanceId:string>',
+                        children: [
+                            { key: 'bucketValue', type: 'string' },
+                            { key: 'experimentId', type: 'string' },
+                            { key: 'instanceId', type: 'string' },
+                        ],
+                    },
+                ],
+            },
+            {
+                key: 'user',
+                type: 'struct<guid:string,tuid:string>',
+                children: [
+                    { key: 'guid', type: 'string' },
+                    { key: 'tuid', type: 'string' },
+                ],
+            },
+        ],
+    });
+
+    expect(parseType('column', 'map<string,float>')).toEqual({
+        key: 'column',
+        type: 'map<string,float>',
+        children: [
+            {
+                key: '<key>',
+                type: 'string',
+            },
+            {
+                key: '<value>',
+                type: 'float',
+            },
+        ],
+    });
+
+    expect(
+        parseType(
+            'column',
+            'map<string,uniontype<string,int,bigint,float,double,struct<year:int,month:int,day:int>>>'
+        )
+    ).toEqual({
+        key: 'column',
+        type: 'map<string,uniontype<string,int,bigint,float,double,struct<year:int,month:int,day:int>>>',
+        children: [
+            {
+                key: '<key>',
+                type: 'string',
+            },
+            {
+                key: '<value>',
+                type: 'uniontype<string,int,bigint,float,double,struct<year:int,month:int,day:int>>',
+                children: [
+                    { key: '<element>', type: 'string' },
+                    { key: '<element>', type: 'int' },
+                    { key: '<element>', type: 'bigint' },
+                    { key: '<element>', type: 'float' },
+                    { key: '<element>', type: 'double' },
+                    {
+                        key: '<element>',
+                        type: 'struct<year:int,month:int,day:int>',
+                        children: [
+                            { key: 'year', type: 'int' },
+                            { key: 'month', type: 'int' },
+                            { key: 'day', type: 'int' },
+                        ],
+                    },
+                ],
+            },
+        ],
+    });
+});
+
+test('prettyPrintType', () => {
+    expect(prettyPrintType('map<string,string>')).toEqual(`map<
+  string,
+  string
+>`);
+
+    expect(
+        prettyPrintType(
+            'struct<ids:array<string>,data:uniontype<int,float,string>>'
+        )
+    ).toEqual(`struct<
+  ids: array<
+    string
+  >,
+  data: uniontype<
+    int,
+    float,
+    string
+  >
+>`);
+
+    expect(
+        prettyPrintType(
+            'struct<ids:array<string>,comment:string,data:map<int,int>>'
+        )
+    ).toEqual(`struct<
+  ids: array<
+    string
+  >,
+  comment: string,
+  data: map<
+    int,
+    int
+  >
+>`);
+
+    expect(
+        prettyPrintType(
+            'map<struct<ids:array<string>,comment:string,data:map<int,int>>,struct<data:array<string>,event:map<int,int>>>'
+        )
+    ).toEqual(`map<
+  struct<
+    ids: array<
+      string
+    >,
+    comment: string,
+    data: map<
+      int,
+      int
+    >
+  >,
+  struct<
+    data: array<
+      string
+    >,
+    event: map<
+      int,
+      int
+    >
+  >
+>`);
+});

querybook/webapp/components/DataDocScheduleList/DataDocScheduleItem.tsx

@@ -38,7 +38,12 @@ export const DataDocScheduleItem: React.FC<IDataDocScheduleItemProps> = ({
                         Runs <HumanReadableCronSchedule cron={schedule.cron} />
                     </StyledText>
                     <StyledText color="light" className="mt4">
-                        Next Run: <NextRun cron={schedule.cron} />
+                        Next Run:{' '}
+                        {schedule.enabled ? (
+                            <NextRun cron={schedule.cron} />
+                        ) : (
+                            'Disabled'
+                        )}
                     </StyledText>
                 </div>
                 {lastRecord && (
@@ -99,7 +104,11 @@ export const DataDocScheduleItem: React.FC<IDataDocScheduleItemProps> = ({
                 <div className="flex-row">
                     <Link to={getWithinEnvUrl(`/datadoc/${doc.id}/`)}>
                         {doc.title ? (
-                            <AccentText weight="bold" size="med">
+                            <AccentText
+                                weight="bold"
+                                size="med"
+                                color={schedule.enabled ? 'text' : 'lightest'}
+                            >
                                 {doc.title}
                             </AccentText>
                         ) : (

querybook/webapp/components/DataTableViewColumn/DataTableColumnCard.scss

@@ -29,3 +29,20 @@
         }
     }
 }
+
+.DataTableColumnCardNestedType {
+
+    .column-type {
+        min-width: 80px;
+        word-break: break-all;
+    }
+
+    .nested-indent {
+        margin-left: 32px;
+    }
+
+    .expand-icon {
+        margin-left: -32px;
+        padding: 0 8px;
+    }
+}