diff --git a/libs/parser-sql/src/postgresAst.ts b/libs/parser-sql/src/postgresAst.ts index 3f3b02b53..6958c2449 100644 --- a/libs/parser-sql/src/postgresAst.ts +++ b/libs/parser-sql/src/postgresAst.ts @@ -114,7 +114,7 @@ export type FunctionReturnsAst = { kind: 'Type', token: TokenInfo, setOf?: Token // basic parts export type AliasAst = { token?: TokenInfo, name: IdentifierAst } export type ObjectNameAst = { schema?: IdentifierAst, name: IdentifierAst } -export type ExpressionAst = (LiteralAst | ParameterAst | ColumnAst | WildcardAst | FunctionAst | GroupAst | OperationAst | OperationLeftAst | OperationRightAst | ListAst) & { cast?: { token: TokenInfo, type: ColumnTypeAst } } +export type ExpressionAst = (LiteralAst | ParameterAst | ColumnAst | WildcardAst | FunctionAst | GroupAst | OperationAst | OperationLeftAst | OperationRightAst | ArrayAst | ListAst) & { cast?: { token: TokenInfo, type: ColumnTypeAst } } export type LiteralAst = StringAst | IntegerAst | DecimalAst | BooleanAst | NullAst export type ColumnAst = { kind: 'Column', schema?: IdentifierAst, table?: IdentifierAst, column: IdentifierAst, json?: ColumnJsonAst[] } export type ColumnJsonAst = { kind: JsonOp, token: TokenInfo, field: StringAst | ParameterAst } @@ -127,6 +127,7 @@ export type OperationLeftAst = { kind: 'OperationLeft', op: OperatorLeftAst, rig export type OperatorLeftAst = { kind: OperatorLeft, token: TokenInfo } export type OperationRightAst = { kind: 'OperationRight', left: ExpressionAst, op: OperatorRightAst } export type OperatorRightAst = { kind: OperatorRight, token: TokenInfo } +export type ArrayAst = { kind: 'Array', token: TokenInfo, items: ExpressionAst[] } export type ListAst = { kind: 'List', items: LiteralAst[] } export type SortOrderAst = { kind: SortOrder, token: TokenInfo } export type SortNullsAst = { kind: SortNulls, token: TokenInfo } diff --git a/libs/parser-sql/src/postgresBuilder.ts b/libs/parser-sql/src/postgresBuilder.ts index 655aa0a03..409194c3d 100644 --- a/libs/parser-sql/src/postgresBuilder.ts +++ b/libs/parser-sql/src/postgresBuilder.ts @@ -415,6 +415,7 @@ function expressionToString(e: ExpressionAst): string { if (e.kind === 'Operation') return expressionToString(e.left) + ' ' + operatorToString(e.op.kind) + ' ' + expressionToString(e.right) if (e.kind === 'OperationLeft') return operatorLeftToString(e.op.kind) + ' ' + expressionToString(e.right) if (e.kind === 'OperationRight') return expressionToString(e.left) + ' ' + operatorRightToString(e.op.kind) + if (e.kind === 'Array') return '[' + e.items.map(expressionToString).join(', ') + ']' if (e.kind === 'List') return '(' + e.items.map(expressionToString).join(', ') + ')' return isNever(e) } @@ -433,6 +434,7 @@ function expressionToValue(e: ExpressionAst): AttributeValue { if (e.kind === 'Operation') return '`' + expressionToString(e) + '`' if (e.kind === 'OperationLeft') return '`' + expressionToString(e) + '`' if (e.kind === 'OperationRight') return '`' + expressionToString(e) + '`' + if (e.kind === 'Array') return e.items.map(expressionToValue) if (e.kind === 'List') return e.items.map(expressionToValue) return isNever(e) } @@ -488,6 +490,7 @@ function expressionAttrs(e: ExpressionAst): AttributePath[] { if (e.kind === 'Operation') return distinctBy(expressionAttrs(e.left).concat(expressionAttrs(e.right)), p => p.join('.')) if (e.kind === 'OperationLeft') return expressionAttrs(e.right) if (e.kind === 'OperationRight') return expressionAttrs(e.left) + if (e.kind === 'Array') return [] if (e.kind === 'List') return [] return isNever(e) } @@ -615,6 +618,8 @@ function expressionToken(e: ExpressionAst): TokenPosition { return mergePositions([e.op.token, expressionToken(e.right)]) } else if (e.kind === 'OperationRight') { return mergePositions([expressionToken(e.left), e.op.token]) + } else if (e.kind === 'Array') { + return mergePositions([e.token, ...e.items.map(expressionToken)]) } else if (e.kind === 'List') { return mergePositions(e.items.map(expressionToken)) } else { diff --git a/libs/parser-sql/src/postgresParser.test.ts b/libs/parser-sql/src/postgresParser.test.ts index ea83c9095..784efd36c 100644 --- a/libs/parser-sql/src/postgresParser.test.ts +++ b/libs/parser-sql/src/postgresParser.test.ts @@ -72,12 +72,6 @@ describe('postgresParser', () => { const sql = fs.readFileSync('./resources/plausible.sql', 'utf8') const parsed = parsePostgresAst(sql, {strict: true}) expect(parsed.errors || []).toEqual([]) - // TODO:246 `features character varying(255)[] DEFAULT ARRAY['props'::character varying, 'stats_api'::character varying] NOT NULL,` - // TODO:542 `recipients public.citext[] DEFAULT ARRAY[]::public.citext[] NOT NULL,` - // TODO:575 `errors jsonb[] DEFAULT ARRAY[]::jsonb[] NOT NULL,` - // TODO:585 `tags character varying(255)[] DEFAULT ARRAY[]::character varying[],` - // TODO:1357 `recipients public.citext[] DEFAULT ARRAY[]::public.citext[] NOT NULL,` - // TODO:1357 `recipients public.citext[] DEFAULT ARRAY[]::public.citext[] NOT NULL` // TODO:2246 `CREATE TRIGGER ...` }) test.skip('other structures', async () => { @@ -87,10 +81,10 @@ describe('postgresParser', () => { 'https://raw.githubusercontent.com/gocardless/draupnir/refs/heads/master/structure.sql', 'https://raw.githubusercontent.com/LuisMDeveloper/travis-core/refs/heads/master/db/structure.sql', 'https://raw.githubusercontent.com/gustavodiel/monet-backend/refs/heads/main/db/structure.sql', - // 'https://raw.githubusercontent.com/plausible/analytics/refs/heads/master/priv/repo/structure.sql', // fail#246: `DEFAULT ARRAY['props'::character varying, 'stats_api'::character varying]` - // 'https://raw.githubusercontent.com/inaturalist/inaturalist/refs/heads/main/db/structure.sql', // fail#44: column type: `numeric[]` + // 'https://raw.githubusercontent.com/plausible/analytics/refs/heads/master/priv/repo/structure.sql', // fail#2246: `CREATE TRIGGER` + // 'https://raw.githubusercontent.com/inaturalist/inaturalist/refs/heads/main/db/structure.sql', // fail#44: `CREATE FUNCTION` with unnamed parameter // 'https://raw.githubusercontent.com/cardstack/cardstack/refs/heads/main/packages/hub/config/structure.sql', // fail#52: `ALTER TYPE` & 1986: `COPY` - // 'https://raw.githubusercontent.com/drenther/Empirical-Core/refs/heads/develop/db/structure.sql', // fail#688: `ARRAY[]` + // 'https://raw.githubusercontent.com/drenther/Empirical-Core/refs/heads/develop/db/structure.sql', // fail#2894: `CREATE INDEX email_idx ON users USING gin (email gin_trgm_ops);` // 'https://raw.githubusercontent.com/spuddybike/archivist/refs/heads/develop/db/structure.sql', // fail#849: parenthesis in FROM clause // 'https://raw.githubusercontent.com/sathreddy/bety/refs/heads/master/db/structure.sql', // fail#274: LexingError: unexpected character: ->\\<- // 'https://raw.githubusercontent.com/dhbtk/achabus/refs/heads/master/db/structure.sql', // fail#293: column type: `geography(Point,4326)` @@ -98,7 +92,7 @@ describe('postgresParser', () => { // 'https://raw.githubusercontent.com/yazilimcilarinmolayeri/pixels-clone/refs/heads/master/Structure.sql', // fail#68: column `GENERATED ALWAYS AS IDENTITY` TODO // 'https://raw.githubusercontent.com/henry2992/aprendemy/refs/heads/master/db/structure.sql', // fail#1961: `CREATE TRIGGER` // 'https://raw.githubusercontent.com/ppawel/openstreetmap-watch-list/refs/heads/master/db/structure.sql', // fail#92: `CREATE TYPE change AS (geom geometry(Geometry,4326))` - // 'https://raw.githubusercontent.com/OPG-813/electronic-queue-server/refs/heads/master/src/db/structure.sql', // fail#2: `CREATE OR REPLACE FUNCTION` TODO + // 'https://raw.githubusercontent.com/OPG-813/electronic-queue-server/refs/heads/master/src/db/structure.sql', // fail#110: `DEFAULT CURRENT_DATE AT TIME ZONE( SYSTEM_TIMEZONE() )` // 'https://raw.githubusercontent.com/Rotabot-io/rotabot/refs/heads/main/assets/structure.sql', // fail#57: `ALTER FUNCTION public.generate_uid(size integer) OWNER TO rotabot;` TODO // 'https://raw.githubusercontent.com/bocoup/devstats/refs/heads/master/structure.sql', // fail#57: `ALTER FUNCTION current_state.label_prefix(some_label text) OWNER TO devstats_team;` TODO // 'https://raw.githubusercontent.com/TechnoDann/PPC-board-2.0/refs/heads/master/db/structure.sql', // fail#350: `CREATE INDEX index_posts_on_ancestry ON public.posts USING btree (ancestry text_pattern_ops NULLS FIRST);` @@ -1202,7 +1196,7 @@ describe('postgresParser', () => { expect(parseRule(p => p.identifierRule(), '"an id with \\""')).toEqual({result: {...identifier('an id with "', 0, 14), quoted: true}}) }) test('not empty', () => { - const specials = ['Add', 'Commit', 'Data', 'Database', 'Deferrable', 'Domain', 'Increment', 'Index', 'Input', 'Nulls', 'Rows', 'Schema', 'Start', 'Temporary', 'Type', 'Version'] + const specials = ['Add', 'Commit', 'Data', 'Database', 'Deferrable', 'Domain', 'Increment', 'Index', 'Input', 'Nulls', 'Rows', 'Schema', 'Session', 'Start', 'Temporary', 'Type', 'Version'] expect(parseRule(p => p.identifierRule(), '""')).toEqual({errors: [ {kind: 'LexingError', level: 'error', message: 'unexpected character: ->"<- at offset: 0, skipped 2 characters.', ...token(0, 2)}, {kind: 'NoViableAltException', level: 'error', message: `Expecting: one of these possible Token sequences:\n 1. [Identifier]${specials.map((n, i) => `\n ${i + 2}. [${n}]`).join('')}\nbut found: ''`, offset: {start: -1, end: -1}, position: {start: {line: -1, column: -1}, end: {line: -1, column: -1}}} diff --git a/libs/parser-sql/src/postgresParser.ts b/libs/parser-sql/src/postgresParser.ts index c942c03f5..08c637c14 100644 --- a/libs/parser-sql/src/postgresParser.ts +++ b/libs/parser-sql/src/postgresParser.ts @@ -124,6 +124,7 @@ const Add = createToken({name: 'Add', pattern: /\bADD\b/i, longer_alt: Identifie const All = createToken({name: 'All', pattern: /\bALL\b/i, longer_alt: Identifier}) const Alter = createToken({name: 'Alter', pattern: /\bALTER\b/i, longer_alt: Identifier}) const And = createToken({name: 'And', pattern: /\bAND\b/i, longer_alt: Identifier}) +const Array = createToken({name: 'Array', pattern: /\bARRAY\b/i, longer_alt: Identifier}) const As = createToken({name: 'As', pattern: /\bAS\b/i, longer_alt: Identifier}) const Asc = createToken({name: 'Asc', pattern: /\bASC\b/i, longer_alt: Identifier}) const Authorization = createToken({name: 'Authorization', pattern: /\bAUTHORIZATION\b/i, longer_alt: Identifier}) @@ -270,7 +271,7 @@ const Window = createToken({name: 'Window', pattern: /\bWINDOW\b/i, longer_alt: const With = createToken({name: 'With', pattern: /\bWITH\b/i, longer_alt: Identifier}) const Work = createToken({name: 'Work', pattern: /\bWORK\b/i, longer_alt: Identifier}) const keywordTokens: TokenType[] = [ - Add, All, Alter, And, As, Asc, Authorization, Begin, By, Cache, Called, Cascade, Chain, Check, Collate, Column, + Add, All, Alter, And, Array, As, Asc, Authorization, Begin, By, Cache, Called, Cascade, Chain, Check, Collate, Column, Comment, Commit, Concurrently, Conflict, Constraint, Create, Cross, CurrentRole, CurrentUser, Cycle, Data, Database, Default, Deferrable, Delete, Desc, Distinct, Do, Domain, Drop, Enum, Except, Exists, Extension, False, Fetch, Filter, First, ForeignKey, From, Full, Function, Global, GroupBy, Having, If, Immutable, In, Include, Increment, @@ -1437,6 +1438,14 @@ class PostgresParser extends EmbeddedActionsParser { {ALT: () => $.SUBRULE(groupRule)}, {ALT: () => $.SUBRULE($.literalRule)}, {ALT: () => ({kind: 'Wildcard', token: tokenInfo($.CONSUME(Asterisk))})}, + {ALT: () => { + const token = tokenInfo($.CONSUME(Array)) + $.CONSUME(BracketLeft) + const items: ExpressionAst[] = [] + $.MANY_SEP({SEP: Comma, DEF: () => items.push($.SUBRULE($.expressionRule))}) + $.CONSUME(BracketRight) + return {kind: 'Array' as const, token, items} + }}, {ALT: () => { const first = $.SUBRULE($.identifierRule) const nest = $.OPTION(() => $.OR2([ @@ -1658,6 +1667,7 @@ class PostgresParser extends EmbeddedActionsParser { {ALT: () => toIdentifier($.CONSUME(Nulls))}, {ALT: () => toIdentifier($.CONSUME(Rows))}, {ALT: () => toIdentifier($.CONSUME(Schema))}, + {ALT: () => toIdentifier($.CONSUME(Session))}, {ALT: () => toIdentifier($.CONSUME(Start))}, {ALT: () => toIdentifier($.CONSUME(Temporary))}, {ALT: () => toIdentifier($.CONSUME(Type))},