Merge branch 'main' into L028_autofix

sqlfluff · Mar 4, 2022 · 9b9565f · 9b9565f
2 parents 0f2019b + 78bb1bd
commit 9b9565f
Show file tree

Hide file tree

Showing 3 changed files with 250 additions and 2 deletions.
diff --git a/src/sqlfluff/dialects/dialect_spark3.py b/src/sqlfluff/dialects/dialect_spark3.py
@@ -554,8 +554,21 @@ class DatatypeSegment(PrimitiveTypeSegment):
         Sequence(
             "STRUCT",
             Bracketed(
-                Delimited(
+                # Manually rebuild Delimited.
+                # Delimited breaks futher nesting (MAP, STRUCT, ARRAY)
+                # of complex datatypes (Comma splits angle bracket blocks)
+                #
+                # CommentGrammar here is valid Spark SQL
+                # even though its not stored in Sparks Catalog
+                Sequence(
+                    Ref("NakedIdentifierSegment"),
+                    Ref("ColonSegment"),
+                    Ref("DatatypeSegment"),
+                    Ref("CommentGrammar", optional=True),
+                ),
+                AnyNumberOf(
                     Sequence(
+                        Ref("CommaSegment"),
                         Ref("NakedIdentifierSegment"),
                         Ref("ColonSegment"),
                         Ref("DatatypeSegment"),
@@ -791,8 +804,16 @@ class CreateTableStatementSegment(BaseSegment):
             # Columns and comment syntax:
             Sequence(
                 Bracketed(
-                    Delimited(
+                    # Manually rebuild Delimited.
+                    # Delimited breaks complex (MAP, STRUCT) datatypes
+                    # (Comma splits angle bracket blocks)
+                    Sequence(
+                        Ref("ColumnDefinitionSegment"),
+                        Ref("CommentGrammar", optional=True),
+                    ),
+                    AnyNumberOf(
                         Sequence(
+                            Ref("CommaSegment"),
                             Ref("ColumnDefinitionSegment"),
                             Ref("CommentGrammar", optional=True),
                         ),

diff --git a/test/fixtures/dialects/spark3/create_table_complex_datatypes.sql b/test/fixtures/dialects/spark3/create_table_complex_datatypes.sql
@@ -0,0 +1,11 @@
+--Create Table with complex datatypes
+CREATE TABLE table_identifier
+( a STRUCT<b: STRING, c: BOOLEAN>, d MAP<STRING, BOOLEAN>, e ARRAY<STRING>);
+
+--Create Table with complex datatypes and comments
+CREATE TABLE table_identifier
+( a STRUCT<b: STRING COMMENT 'struct_comment', c: BOOLEAN> COMMENT 'col_comment', d MAP<STRING, BOOLEAN> COMMENT 'col_comment', e ARRAY<STRING> COMMENT 'col_comment');
+
+--Create Table with nested complex datatypes
+CREATE TABLE table_identifier
+( a STRUCT<b: STRING, c: MAP<STRING, BOOLEAN>>, d MAP<STRING, STRUCT<e: STRING, f: MAP<STRING, BOOLEAN>>>, g ARRAY<STRUCT<h: STRING, i: MAP<STRING, BOOLEAN>>>);
diff --git a/test/fixtures/dialects/spark3/create_table_complex_datatypes.yml b/test/fixtures/dialects/spark3/create_table_complex_datatypes.yml
@@ -0,0 +1,216 @@
+# YML test files are auto-generated from SQL files and should not be edited by
+# hand. To help enforce this, the "hash" field in the file must match a hash
+# computed by SQLFluff when running the tests. Please run
+# `python test/generate_parse_fixture_yml.py`  to generate them after adding or
+# altering SQL files.
+_hash: 7e2cde585ebd2449087447234f57d697481af4f5ded4d07695f8641650b997fa
+file:
+- statement:
+    create_table_statement:
+    - keyword: CREATE
+    - keyword: TABLE
+    - table_reference:
+        identifier: table_identifier
+    - bracketed:
+      - start_bracket: (
+      - column_definition:
+          identifier: a
+          data_type:
+          - keyword: STRUCT
+          - start_angle_bracket: <
+          - identifier: b
+          - colon: ':'
+          - data_type:
+              primitive_type:
+                keyword: STRING
+          - comma: ','
+          - identifier: c
+          - colon: ':'
+          - data_type:
+              primitive_type:
+                keyword: BOOLEAN
+          - end_angle_bracket: '>'
+      - comma: ','
+      - column_definition:
+          identifier: d
+          data_type:
+            keyword: MAP
+            start_angle_bracket: <
+            primitive_type:
+              keyword: STRING
+            comma: ','
+            data_type:
+              primitive_type:
+                keyword: BOOLEAN
+            end_angle_bracket: '>'
+      - comma: ','
+      - column_definition:
+          identifier: e
+          data_type:
+            keyword: ARRAY
+            start_angle_bracket: <
+            data_type:
+              primitive_type:
+                keyword: STRING
+            end_angle_bracket: '>'
+      - end_bracket: )
+- statement_terminator: ;
+- statement:
+    create_table_statement:
+    - keyword: CREATE
+    - keyword: TABLE
+    - table_reference:
+        identifier: table_identifier
+    - bracketed:
+      - start_bracket: (
+      - column_definition:
+          identifier: a
+          data_type:
+          - keyword: STRUCT
+          - start_angle_bracket: <
+          - identifier: b
+          - colon: ':'
+          - data_type:
+              primitive_type:
+                keyword: STRING
+          - keyword: COMMENT
+          - literal: "'struct_comment'"
+          - comma: ','
+          - identifier: c
+          - colon: ':'
+          - data_type:
+              primitive_type:
+                keyword: BOOLEAN
+          - end_angle_bracket: '>'
+          column_constraint_segment:
+            comment_clause:
+              keyword: COMMENT
+              literal: "'col_comment'"
+      - comma: ','
+      - column_definition:
+          identifier: d
+          data_type:
+            keyword: MAP
+            start_angle_bracket: <
+            primitive_type:
+              keyword: STRING
+            comma: ','
+            data_type:
+              primitive_type:
+                keyword: BOOLEAN
+            end_angle_bracket: '>'
+          column_constraint_segment:
+            comment_clause:
+              keyword: COMMENT
+              literal: "'col_comment'"
+      - comma: ','
+      - column_definition:
+          identifier: e
+          data_type:
+            keyword: ARRAY
+            start_angle_bracket: <
+            data_type:
+              primitive_type:
+                keyword: STRING
+            end_angle_bracket: '>'
+          column_constraint_segment:
+            comment_clause:
+              keyword: COMMENT
+              literal: "'col_comment'"
+      - end_bracket: )
+- statement_terminator: ;
+- statement:
+    create_table_statement:
+    - keyword: CREATE
+    - keyword: TABLE
+    - table_reference:
+        identifier: table_identifier
+    - bracketed:
+      - start_bracket: (
+      - column_definition:
+          identifier: a
+          data_type:
+          - keyword: STRUCT
+          - start_angle_bracket: <
+          - identifier: b
+          - colon: ':'
+          - data_type:
+              primitive_type:
+                keyword: STRING
+          - comma: ','
+          - identifier: c
+          - colon: ':'
+          - data_type:
+              keyword: MAP
+              start_angle_bracket: <
+              primitive_type:
+                keyword: STRING
+              comma: ','
+              data_type:
+                primitive_type:
+                  keyword: BOOLEAN
+              end_angle_bracket: '>'
+          - end_angle_bracket: '>'
+      - comma: ','
+      - column_definition:
+          identifier: d
+          data_type:
+            keyword: MAP
+            start_angle_bracket: <
+            primitive_type:
+              keyword: STRING
+            comma: ','
+            data_type:
+            - keyword: STRUCT
+            - start_angle_bracket: <
+            - identifier: e
+            - colon: ':'
+            - data_type:
+                primitive_type:
+                  keyword: STRING
+            - comma: ','
+            - identifier: f
+            - colon: ':'
+            - data_type:
+                keyword: MAP
+                start_angle_bracket: <
+                primitive_type:
+                  keyword: STRING
+                comma: ','
+                data_type:
+                  primitive_type:
+                    keyword: BOOLEAN
+                end_angle_bracket: '>'
+            - end_angle_bracket: '>'
+            end_angle_bracket: '>'
+      - comma: ','
+      - column_definition:
+          identifier: g
+          data_type:
+            keyword: ARRAY
+            start_angle_bracket: <
+            data_type:
+            - keyword: STRUCT
+            - start_angle_bracket: <
+            - identifier: h
+            - colon: ':'
+            - data_type:
+                primitive_type:
+                  keyword: STRING
+            - comma: ','
+            - identifier: i
+            - colon: ':'
+            - data_type:
+                keyword: MAP
+                start_angle_bracket: <
+                primitive_type:
+                  keyword: STRING
+                comma: ','
+                data_type:
+                  primitive_type:
+                    keyword: BOOLEAN
+                end_angle_bracket: '>'
+            - end_angle_bracket: '>'
+            end_angle_bracket: '>'
+      - end_bracket: )
+- statement_terminator: ;