diff --git a/libs/parser-sql/src/postgresAst.ts b/libs/parser-sql/src/postgresAst.ts index ee312d8e7..c8cc478a0 100644 --- a/libs/parser-sql/src/postgresAst.ts +++ b/libs/parser-sql/src/postgresAst.ts @@ -25,7 +25,7 @@ export type BeginStatementAst = { kind: 'Begin', token: TokenInfo, object?: {kin export type CommentOnStatementAst = { kind: 'CommentOn', token: TokenInfo, object: { token: TokenInfo, kind: CommentObject }, schema?: IdentifierAst, parent?: IdentifierAst, entity: IdentifierAst, comment: StringAst | NullAst } export type CommitStatementAst = { kind: 'Commit', token: TokenInfo, object?: { kind: 'Work' | 'Transaction', token: TokenInfo }, chain?: { token: TokenInfo, no?: TokenInfo } } export type CreateExtensionStatementAst = { kind: 'CreateExtension', token: TokenInfo, ifNotExists?: TokenInfo, name: IdentifierAst, with?: TokenInfo, schema?: { token: TokenInfo, name: IdentifierAst }, version?: { token: TokenInfo, number: StringAst | IdentifierAst }, cascade?: TokenInfo } -export type CreateFunctionStatementAst = { kind: 'CreateFunction', token: TokenInfo } // TODO +export type CreateFunctionStatementAst = { kind: 'CreateFunction', token: TokenInfo, replace?: TokenInfo, schema?: IdentifierAst, name: IdentifierAst, args: FunctionArgumentAst[], returns?: FunctionReturnsAst, language?: { token: TokenInfo, name: IdentifierAst }, behavior?: { kind: 'Immutable' | 'Stable' | 'Volatile', token: TokenInfo }, nullBehavior?: { kind: 'Called' | 'ReturnsNull' | 'Strict', token: TokenInfo }, definition?: { token: TokenInfo, value: StringAst }, return?: { token: TokenInfo, expression: ExpressionAst } } export type CreateIndexStatementAst = { kind: 'CreateIndex', token: TokenInfo, unique?: TokenInfo, concurrently?: TokenInfo, ifNotExists?: TokenInfo, name?: IdentifierAst, only?: TokenInfo, schema?: IdentifierAst, table: IdentifierAst, using?: { token: TokenInfo, method: IdentifierAst }, columns: IndexColumnAst[], include?: { token: TokenInfo, columns: IdentifierAst[] }, where?: WhereClauseAst } export type CreateMaterializedViewStatementAst = { kind: 'CreateMaterializedView', token: TokenInfo, ifNotExists?: TokenInfo, schema?: IdentifierAst, name: IdentifierAst, columns?: IdentifierAst[], query: SelectStatementInnerAst, withData?: { token: TokenInfo, no?: TokenInfo } } export type CreateSchemaStatementAst = { kind: 'CreateSchema', token: TokenInfo, ifNotExists?: TokenInfo, schema?: IdentifierAst, authorization?: { token: TokenInfo, role: SchemaRoleAst } } @@ -104,6 +104,8 @@ export type SequenceTypeAst = { token: TokenInfo, type: IdentifierAst } export type SequenceParamAst = { token: TokenInfo, value: IntegerAst } export type SequenceParamOptAst = { token: TokenInfo, value?: IntegerAst } export type SequenceOwnedByAst = { token: TokenInfo, owner: { kind: 'None', token: TokenInfo } | { kind: 'Column', schema?: IdentifierAst, table: IdentifierAst, column: IdentifierAst } } +export type FunctionArgumentAst = { mode?: { kind: 'In' | 'Out' | 'InOut' | 'Variadic', token: TokenInfo }, name?: IdentifierAst, type: ColumnTypeAst } +export type FunctionReturnsAst = { kind: 'Type', token: TokenInfo, setOf?: TokenInfo, type: ColumnTypeAst } | { kind: 'Table', token: TokenInfo, columns: { name: IdentifierAst, type: ColumnTypeAst }[] } // basic parts export type AliasAst = { token?: TokenInfo, name: IdentifierAst } @@ -128,7 +130,7 @@ export type SortNullsAst = { kind: SortNulls, token: TokenInfo } // elements export type ParameterAst = { kind: 'Parameter', token: TokenInfo, value: string, index?: number } export type IdentifierAst = { kind: 'Identifier', token: TokenInfo, value: string, quoted?: boolean } -export type StringAst = { kind: 'String', token: TokenInfo, value: string, escaped?: boolean } +export type StringAst = { kind: 'String', token: TokenInfo, value: string, escaped?: boolean, dollar?: string } export type DecimalAst = { kind: 'Decimal', token: TokenInfo, value: number } export type IntegerAst = { kind: 'Integer', token: TokenInfo, value: number } export type BooleanAst = { kind: 'Boolean', token: TokenInfo, value: boolean } diff --git a/libs/parser-sql/src/postgresParser.test.ts b/libs/parser-sql/src/postgresParser.test.ts index 5f14647e0..2099ce506 100644 --- a/libs/parser-sql/src/postgresParser.test.ts +++ b/libs/parser-sql/src/postgresParser.test.ts @@ -32,7 +32,6 @@ import { import {parsePostgresAst, parseRule} from "./postgresParser"; describe('postgresParser', () => { - // TODO: CREATE FUNCTION // TODO: ALTER TYPE test('empty', () => { expect(parsePostgresAst('')).toEqual({result: {statements: []}}) @@ -82,26 +81,26 @@ describe('postgresParser', () => { // cf https://github.com/search?q=path%3A**%2Fstructure.sql&type=code const structures = [ 'https://raw.githubusercontent.com/ChanAlex2357/gestion_analytique_S5/refs/heads/main/Structure.sql', - // 'https://raw.githubusercontent.com/plausible/analytics/refs/heads/master/priv/repo/structure.sql', // fail#73: CREATE FUNCTION - // 'https://raw.githubusercontent.com/inaturalist/inaturalist/refs/heads/main/db/structure.sql', // fail#44: CREATE FUNCTION + // 'https://raw.githubusercontent.com/plausible/analytics/refs/heads/master/priv/repo/structure.sql', // fail#246: `DEFAULT ARRAY['props'::character varying, 'stats_api'::character varying]` + // 'https://raw.githubusercontent.com/inaturalist/inaturalist/refs/heads/main/db/structure.sql', // fail#44: column type: `numeric[]` // 'https://raw.githubusercontent.com/cardstack/cardstack/refs/heads/main/packages/hub/config/structure.sql', // fail#52: `ALTER TYPE` & 1986: `COPY` - // 'https://raw.githubusercontent.com/gocardless/draupnir/refs/heads/master/structure.sql', // fail#118: ALTER TABLE ... ALTER COLUMN id SET DEFAULT ...; - // 'https://raw.githubusercontent.com/drenther/Empirical-Core/refs/heads/develop/db/structure.sql', // fail#79: CREATE FUNCTION - // 'https://raw.githubusercontent.com/spuddybike/archivist/refs/heads/develop/db/structure.sql', // fail#44: CREATE FUNCTION + // 'https://raw.githubusercontent.com/gocardless/draupnir/refs/heads/master/structure.sql', // fail#118: `ALTER TABLE ... ALTER COLUMN id SET DEFAULT ...` TODO + // 'https://raw.githubusercontent.com/drenther/Empirical-Core/refs/heads/develop/db/structure.sql', // fail#688: `ARRAY[]` + // 'https://raw.githubusercontent.com/spuddybike/archivist/refs/heads/develop/db/structure.sql', // fail#849: parenthesis in FROM clause // 'https://raw.githubusercontent.com/sathreddy/bety/refs/heads/master/db/structure.sql', // fail#274: LexingError: unexpected character: ->\\<- - // 'https://raw.githubusercontent.com/dhbtk/achabus/refs/heads/master/db/structure.sql', // fail#71: CREATE FUNCTION + // 'https://raw.githubusercontent.com/dhbtk/achabus/refs/heads/master/db/structure.sql', // fail#293: column type: `geography(Point,4326)` // 'https://raw.githubusercontent.com/mveytsman/community/refs/heads/master/db/structure.sql', // fail#299: JOIN (`SELECT * FROM users, events;`) - // 'https://raw.githubusercontent.com/yazilimcilarinmolayeri/pixels-clone/refs/heads/master/Structure.sql', // fail#27: CREATE SEQUENCE param order (START at the end) - // 'https://raw.githubusercontent.com/henry2992/aprendemy/refs/heads/master/db/structure.sql', // fail#58: CREATE FUNCTION - // 'https://raw.githubusercontent.com/LuisMDeveloper/travis-core/refs/heads/master/db/structure.sql', // fail#32: CREATE FUNCTION + // 'https://raw.githubusercontent.com/yazilimcilarinmolayeri/pixels-clone/refs/heads/master/Structure.sql', // fail#27: CREATE SEQUENCE param order (START at the end) TODO + // 'https://raw.githubusercontent.com/henry2992/aprendemy/refs/heads/master/db/structure.sql', // fail#1291: `ALTER TABLE ... ALTER COLUMN id SET DEFAULT ...` TODO + // 'https://raw.githubusercontent.com/LuisMDeveloper/travis-core/refs/heads/master/db/structure.sql', // fail#832: `ALTER TABLE ... ALTER COLUMN id SET DEFAULT ...` TODO // 'https://raw.githubusercontent.com/ppawel/openstreetmap-watch-list/refs/heads/master/db/structure.sql', // fail#92: `CREATE TYPE change AS (geom geometry(Geometry,4326))` // 'https://raw.githubusercontent.com/OPG-813/electronic-queue-server/refs/heads/master/src/db/structure.sql', // fail#2: `CREATE OR REPLACE FUNCTION` - // 'https://raw.githubusercontent.com/Rotabot-io/rotabot/refs/heads/main/assets/structure.sql', // fail#31: CREATE FUNCTION - // 'https://raw.githubusercontent.com/TechnoDann/PPC-board-2.0/refs/heads/master/db/structure.sql', // fail#16: CREATE FUNCTION - // 'https://raw.githubusercontent.com/bocoup/devstats/refs/heads/master/structure.sql', // fail#46: CREATE FUNCTION - // 'https://raw.githubusercontent.com/gustavodiel/monet-backend/refs/heads/main/db/structure.sql', // fail#23: CREATE FUNCTION - // 'https://raw.githubusercontent.com/Leonardo-Zappani/bd2/refs/heads/main/db/structure.sql', // fail#16: CREATE FUNCTION - // 'https://raw.githubusercontent.com/Style12341/UTN-gestor-aulas-backend/refs/heads/main/db/structure.sql', // fail#16: CREATE FUNCTION + // 'https://raw.githubusercontent.com/Rotabot-io/rotabot/refs/heads/main/assets/structure.sql', // fail#57: `ALTER FUNCTION public.generate_uid(size integer) OWNER TO rotabot;` + // 'https://raw.githubusercontent.com/TechnoDann/PPC-board-2.0/refs/heads/master/db/structure.sql', // fail#254: `ALTER TABLE ... ALTER COLUMN id SET DEFAULT ...` TODO + // 'https://raw.githubusercontent.com/bocoup/devstats/refs/heads/master/structure.sql', // fail#57: `ALTER FUNCTION current_state.label_prefix(some_label text) OWNER TO devstats_team;` + // 'https://raw.githubusercontent.com/gustavodiel/monet-backend/refs/heads/main/db/structure.sql', // fail#199: `ALTER TABLE ... ALTER COLUMN id SET DEFAULT ...` TODO + // 'https://raw.githubusercontent.com/Leonardo-Zappani/bd2/refs/heads/main/db/structure.sql', // fail#250: `ALTER TABLE ... ALTER COLUMN id SET DEFAULT ...` TODO + // 'https://raw.githubusercontent.com/Style12341/UTN-gestor-aulas-backend/refs/heads/main/db/structure.sql', // fail#25: `CREATE TYPE public.timerange AS RANGE` ] await Promise.all(structures.map(async url => { const sql = await fetch(url).then(res => res.text()) @@ -278,26 +277,63 @@ describe('postgresParser', () => { }]}}) }) }) - describe.skip('createFunction', () => { - test('using SQL query', () => { - expect(parsePostgresAst("CREATE FUNCTION add(integer, integer) RETURNS integer AS 'select $1 + $2;' LANGUAGE SQL IMMUTABLE RETURNS NULL ON NULL INPUT;")).toEqual({result: {statements: [{ - ...stmt('CreateFunction', 0, 11, 28), - table: identifier('users', 16), - columns: [column('name', 23)], + describe('createFunction', () => { + test('sample', () => { + expect(parsePostgresAst("CREATE FUNCTION add(IN a integer, IN b integer) RETURNS integer LANGUAGE SQL IMMUTABLE AS $$ BEGIN RETURN a + b; END; $$ RETURNS NULL ON NULL INPUT;")).toEqual({result: {statements: [{ + ...stmt('CreateFunction', 0, 14, 147), + name: identifier('add', 16), + args: [ + {mode: kind('In', 20), name: identifier('a', 23), type: {name: {token: token(25, 31), value: 'integer'}, token: token(25, 31)}}, + {mode: kind('In', 34), name: identifier('b', 37), type: {name: {token: token(39, 45), value: 'integer'}, token: token(39, 45)}}, + ], + returns: {kind: 'Type', token: token(48, 54), type: {name: {token: token(56, 62), value: 'integer'}, token: token(56, 62)}}, + language: {token: token(64, 71), name: identifier('SQL', 73)}, + behavior: kind('Immutable', 77), + definition: {token: token(87, 88), value: {...string('BEGIN RETURN a + b; END;', 90, 119), dollar: '$$'}}, + nullBehavior: kind('ReturnsNull', 121, 146), }]}}) }) test('using SQL', () => { expect(parsePostgresAst('CREATE FUNCTION add(a integer, b integer) RETURNS integer LANGUAGE SQL IMMUTABLE RETURNS NULL ON NULL INPUT RETURN a + b;')).toEqual({result: {statements: [{ - ...stmt('CreateFunction', 0, 11, 28), - table: identifier('users', 16), - columns: [column('name', 23)], + ...stmt('CreateFunction', 0, 14, 120), + name: identifier('add', 16), + args: [ + {name: identifier('a', 20), type: {name: {token: token(22, 28), value: 'integer'}, token: token(22, 28)}}, + {name: identifier('b', 31), type: {name: {token: token(33, 39), value: 'integer'}, token: token(33, 39)}}, + ], + returns: {kind: 'Type', token: token(42, 48), type: {name: {token: token(50, 56), value: 'integer'}, token: token(50, 56)}}, + language: {token: token(58, 65), name: identifier('SQL', 67)}, + behavior: kind('Immutable', 71), + nullBehavior: kind('ReturnsNull', 81, 106), + return: {token: token(108, 113), expression: operation(column('a', 115), op('+', 117), column('b', 119))} }]}}) }) test('using plSQL', () => { - expect(parsePostgresAst('CREATE OR REPLACE FUNCTION increment(i integer) RETURNS integer AS $$ BEGIN RETURN i + 1; END; $$ LANGUAGE plpgsql;')).toEqual({result: {statements: [{ - ...stmt('CreateFunction', 0, 11, 28), - table: identifier('users', 16), - columns: [column('name', 23)], + expect(parsePostgresAst('CREATE FUNCTION increment(i integer) RETURNS integer AS $$ BEGIN RETURN i + 1; END; $$ LANGUAGE plpgsql;')).toEqual({result: {statements: [{ + ...stmt('CreateFunction', 0, 14, 103), + name: identifier('increment', 16), + args: [{name: identifier('i', 26), type: {name: {token: token(28, 34), value: 'integer'}, token: token(28, 34)}}], + returns: {kind: 'Type', token: token(37, 43), type: {name: {token: token(45, 51), value: 'integer'}, token: token(45, 51)}}, + definition: {token: token(53, 54), value: {...string('BEGIN RETURN i + 1; END;', 56, 85), dollar: '$$'}}, + language: {token: token(87, 94), name: identifier('plpgsql', 96)}, + }]}}) + }) + test.skip('using SQL query', () => { + // FIXME: ambiguous `CREATE OR REPLACE` + // FIXME: bad argument parsing when no name + expect(parsePostgresAst("CREATE OR REPLACE FUNCTION public.add(integer, integer) RETURNS integer AS 'select $1 + $2;' LANGUAGE SQL IMMUTABLE RETURNS NULL ON NULL INPUT;")).toEqual({result: {statements: [{ + ...stmt('CreateFunction', 0, 14, 28), + schema: identifier('public', 16), + name: identifier('add', 23), + args: [ + {type: {name: {token: token(25, 31), value: 'integer'}, token: token(25, 31)}}, + {type: {name: {token: token(39, 45), value: 'integer'}, token: token(39, 45)}}, + ], + returns: {kind: 'Type', token: token(48, 54), type: {name: {token: token(56, 62), value: 'integer'}, token: token(56, 62)}}, + definition: {token: token(87, 88), value: string('select $1 + $2;', 90, 119)}, + language: {token: token(64, 71), name: identifier('SQL', 73)}, + behavior: kind('Immutable', 77), + nullBehavior: kind('ReturnsNull', 121, 146), }]}}) }) }) @@ -1131,7 +1167,7 @@ describe('postgresParser', () => { expect(parseRule(p => p.identifierRule(), '"an id with \\""')).toEqual({result: {...identifier('an id with "', 0, 14), quoted: true}}) }) test('not empty', () => { - const specials = ['Data', 'Database', 'Deferrable', 'Index', 'Nulls', 'Rows', 'Schema', 'Type', 'Version'] + const specials = ['Add', 'Commit', 'Data', 'Database', 'Deferrable', 'Increment', 'Index', 'Input', 'Nulls', 'Rows', 'Schema', 'Start', 'Temporary', 'Type', 'Version'] expect(parseRule(p => p.identifierRule(), '""')).toEqual({errors: [ {kind: 'LexingError', level: 'error', message: 'unexpected character: ->"<- at offset: 0, skipped 2 characters.', ...token(0, 2)}, {kind: 'NoViableAltException', level: 'error', message: `Expecting: one of these possible Token sequences:\n 1. [Identifier]${specials.map((n, i) => `\n ${i + 2}. [${n}]`).join('')}\nbut found: ''`, offset: {start: -1, end: -1}, position: {start: {line: -1, column: -1}, end: {line: -1, column: -1}}} @@ -1166,6 +1202,9 @@ describe('postgresParser', () => { test('escaped', () => { expect(parseRule(p => p.stringRule(), "E'value\\nmulti\\nline'")).toEqual({result: {...string('value\\nmulti\\nline', 0, 20), escaped: true}}) }) + test('dollar', () => { + expect(parseRule(p => p.stringRule(), "$tag$\nmulti\nline\nvalue\n$tag$")).toEqual({result: {value: 'multi\nline\nvalue', dollar: '$tag$', kind: 'String', token: {offset: {start: 0, end: 27}, position: {start: {line: 1, column: 1}, end: {line: 5, column: 5}}}}}) + }) }) describe('integerRule', () => { test('0', () => { diff --git a/libs/parser-sql/src/postgresParser.ts b/libs/parser-sql/src/postgresParser.ts index 926e452a7..7fb3842c6 100644 --- a/libs/parser-sql/src/postgresParser.ts +++ b/libs/parser-sql/src/postgresParser.ts @@ -42,6 +42,8 @@ import { FromJoinAst, FromQueryAst, FromTableAst, + FunctionArgumentAst, + FunctionReturnsAst, GroupAst, GroupByClauseAst, HavingClauseAst, @@ -112,9 +114,10 @@ const WhiteSpace = createToken({name: 'WhiteSpace', pattern: /\s+/, group: Lexer const Identifier = createToken({name: 'Identifier', pattern: /\b[a-zA-Z_]\w*\b|"([^\\"]|\\\\|\\")+"/}) const String = createToken({name: 'String', pattern: /E?'([^']|'')*'/i}) +const StringDollar = createToken({name: 'StringDollar', pattern: /\$(\w*)\$[\s\S]*?\$\1\$/i}) const Decimal = createToken({name: 'Decimal', pattern: /\d+\.\d+/}) const Integer = createToken({name: 'Integer', pattern: /0|[1-9]\d*/, longer_alt: Decimal}) -const valueTokens: TokenType[] = [Integer, Decimal, String, Identifier, LineComment, BlockComment] +const valueTokens: TokenType[] = [Integer, Decimal, String, StringDollar, Identifier, LineComment, BlockComment] const Add = createToken({name: 'Add', pattern: /\bADD\b/i, longer_alt: Identifier}) const All = createToken({name: 'All', pattern: /\bALL\b/i, longer_alt: Identifier}) @@ -126,6 +129,7 @@ const Authorization = createToken({name: 'Authorization', pattern: /\bAUTHORIZAT const Begin = createToken({name: 'Begin', pattern: /\bBEGIN\b/i, longer_alt: Identifier}) const By = createToken({name: 'By', pattern: /\bBY\b/i, longer_alt: Identifier}) const Cache = createToken({name: 'Cache', pattern: /\bCACHE\b/i, longer_alt: Identifier}) +const Called = createToken({name: 'Called', pattern: /\bCALLED\b/i, longer_alt: Identifier}) const Cascade = createToken({name: 'Cascade', pattern: /\bCASCADE\b/i, longer_alt: Identifier}) const Chain = createToken({name: 'Chain', pattern: /\bCHAIN\b/i, longer_alt: Identifier}) const Check = createToken({name: 'Check', pattern: /\bCHECK\b/i, longer_alt: Identifier}) @@ -167,11 +171,14 @@ const Global = createToken({name: 'Global', pattern: /\bGLOBAL\b/i, longer_alt: const GroupBy = createToken({name: 'GroupBy', pattern: /\bGROUP\s+BY\b/i}) const Having = createToken({name: 'Having', pattern: /\bHAVING\b/i, longer_alt: Identifier}) const If = createToken({name: 'If', pattern: /\bIF\b/i, longer_alt: Identifier}) +const Immutable = createToken({name: 'Immutable', pattern: /\bIMMUTABLE\b/i, longer_alt: Identifier}) const In = createToken({name: 'In', pattern: /\bIN\b/i, longer_alt: Identifier}) const Include = createToken({name: 'Include', pattern: /\bINCLUDE\b/i, longer_alt: Identifier}) const Increment = createToken({name: 'Increment', pattern: /\bINCREMENT\b/i, longer_alt: Identifier}) const Index = createToken({name: 'Index', pattern: /\bINDEX\b/i, longer_alt: Identifier}) const Inner = createToken({name: 'Inner', pattern: /\bINNER\b/i, longer_alt: Identifier}) +const InOut = createToken({name: 'InOut', pattern: /\bINOUT\b/i, longer_alt: Identifier}) +const Input = createToken({name: 'Input', pattern: /\bINPUT\b/i, longer_alt: Identifier}) const InsertInto = createToken({name: 'InsertInto', pattern: /\bINSERT\s+INTO\b/i}) const Intersect = createToken({name: 'Intersect', pattern: /\bINTERSECT\b/i, longer_alt: Identifier}) const Interval = createToken({name: 'Interval', pattern: /\bINTERVAL\b/i, longer_alt: Identifier}) @@ -179,6 +186,7 @@ const Is = createToken({name: 'Is', pattern: /\bIS\b/i, longer_alt: Identifier}) const IsNull = createToken({name: 'IsNull', pattern: /\bISNULL\b/i, longer_alt: Identifier}) const IsolationLevel = createToken({name: 'IsolationLevel', pattern: /\bISOLATION\s+LEVEL\b/i}) const Join = createToken({name: 'Join', pattern: /\bJOIN\b/i, longer_alt: Identifier}) +const Language = createToken({name: 'Language', pattern: /\bLANGUAGE\b/i, longer_alt: Identifier}) const Last = createToken({name: 'Last', pattern: /\bLAST\b/i, longer_alt: Identifier}) const Left = createToken({name: 'Left', pattern: /\bLEFT\b/i, longer_alt: Identifier}) const Like = createToken({name: 'Like', pattern: /\bLIKE\b/i, longer_alt: Identifier}) @@ -202,6 +210,7 @@ const On = createToken({name: 'On', pattern: /\bON\b/i, longer_alt: Identifier}) const Only = createToken({name: 'Only', pattern: /\bONLY\b/i, longer_alt: Identifier}) const Or = createToken({name: 'Or', pattern: /\bOR\b/i, longer_alt: Identifier}) const OrderBy = createToken({name: 'OrderBy', pattern: /\bORDER\s+BY\b/i}) +const Out = createToken({name: 'Out', pattern: /\bOUT\b/i, longer_alt: Identifier}) const Outer = createToken({name: 'Outer', pattern: /\bOUTER\b/i, longer_alt: Identifier}) const Over = createToken({name: 'Over', pattern: /\bOVER\b/i, longer_alt: Identifier}) const OwnedBy = createToken({name: 'OwnedBy', pattern: /\bOWNED\s+BY\b/i}) @@ -218,7 +227,9 @@ const RenameTo = createToken({name: 'RenameTo', pattern: /\bRENAME\s+TO\b/i}) const RepeatableRead = createToken({name: 'RepeatableRead', pattern: /\bREPEATABLE\s+READ\b/i}) const Replace = createToken({name: 'Replace', pattern: /\bREPLACE\b/i, longer_alt: Identifier}) const Restrict = createToken({name: 'Restrict', pattern: /\bRESTRICT\b/i, longer_alt: Identifier}) +const Return = createToken({name: 'Return', pattern: /\bRETURN\b/i, longer_alt: Identifier}) const Returning = createToken({name: 'Returning', pattern: /\bRETURNING\b/i, longer_alt: Identifier}) +const Returns = createToken({name: 'Returns', pattern: /\bRETURNS\b/i, longer_alt: Identifier}) const Right = createToken({name: 'Right', pattern: /\bRIGHT\b/i, longer_alt: Identifier}) const Row = createToken({name: 'Row', pattern: /\bROW\b/i, longer_alt: Identifier}) const Rows = createToken({name: 'Rows', pattern: /\bROWS\b/i, longer_alt: Identifier}) @@ -231,8 +242,11 @@ const SessionUser = createToken({name: 'SessionUser', pattern: /\bSESSION_USER\b const SetDefault = createToken({name: 'SetDefault', pattern: /\bSET\s+DEFAULT\b/i}) const SetNull = createToken({name: 'SetNull', pattern: /\bSET\s+NULL\b/i}) const Set = createToken({name: 'Set', pattern: /\bSET\b/i, longer_alt: Identifier}) +const SetOf = createToken({name: 'SetOf', pattern: /\bSETOF\b/i, longer_alt: Identifier}) const Show = createToken({name: 'Show', pattern: /\bSHOW\b/i, longer_alt: Identifier}) +const Stable = createToken({name: 'Stable', pattern: /\bSTABLE\b/i, longer_alt: Identifier}) const Start = createToken({name: 'Start', pattern: /\bSTART\b/i, longer_alt: Identifier}) +const Strict = createToken({name: 'Strict', pattern: /\bSTRICT\b/i, longer_alt: Identifier}) const Table = createToken({name: 'Table', pattern: /\bTABLE\b/i, longer_alt: Identifier}) const Temp = createToken({name: 'Temp', pattern: /\bTEMP\b/i, longer_alt: Identifier}) const Temporary = createToken({name: 'Temporary', pattern: /\bTEMPORARY\b/i, longer_alt: Identifier}) @@ -247,22 +261,27 @@ const Unlogged = createToken({name: 'Unlogged', pattern: /\bUNLOGGED\b/i, longer const Update = createToken({name: 'Update', pattern: /\bUPDATE\b/i, longer_alt: Identifier}) const Using = createToken({name: 'Using', pattern: /\bUSING\b/i, longer_alt: Identifier}) const Values = createToken({name: 'Values', pattern: /\bVALUES\b/i, longer_alt: Identifier}) +const Variadic = createToken({name: 'Variadic', pattern: /\bVARIADIC\b/i, longer_alt: Identifier}) const Version = createToken({name: 'Version', pattern: /\bVERSION\b/i, longer_alt: Identifier}) const View = createToken({name: 'View', pattern: /\bVIEW\b/i, longer_alt: Identifier}) +const Volatile = createToken({name: 'Volatile', pattern: /\bVOLATILE\b/i, longer_alt: Identifier}) const Where = createToken({name: 'Where', pattern: /\bWHERE\b/i, longer_alt: Identifier}) const Window = createToken({name: 'Window', pattern: /\bWINDOW\b/i, longer_alt: Identifier}) const With = createToken({name: 'With', pattern: /\bWITH\b/i, longer_alt: Identifier}) const Work = createToken({name: 'Work', pattern: /\bWORK\b/i, longer_alt: Identifier}) const keywordTokens: TokenType[] = [ - Add, All, Alter, And, As, Asc, Authorization, Begin, By, Cache, Cascade, Chain, Check, Collate, Column, Comment, Commit, Concurrently, - Conflict, Constraint, Create, Cross, CurrentRole, CurrentUser, Cycle, Data, Database, Default, Deferrable, Delete, Desc, Distinct, Do, Domain, Drop, Enum, Except, - Exists, Extension, False, Fetch, Filter, First, ForeignKey, From, Full, Function, Global, GroupBy, Having, If, In, Include, Increment, Index, - Inner, InsertInto, Intersect, Interval, Is, IsNull, IsolationLevel, Join, Last, Left, Like, Limit, Local, MaterializedView, Maxvalue, Minvalue, - Natural, Next, No, None, NoAction, Not, Nothing, NotNull, Null, Nulls, Offset, On, Only, Or, OrderBy, Outer, Over, OwnedBy, OwnerTo, PartitionBy, PrimaryKey, - ReadCommitted, ReadOnly, ReadUncommitted, ReadWrite, Recursive, References, RenameTo, RepeatableRead, Replace, Restrict, - Returning, Right, Row, Rows, Schema, Select, Sequence, Serializable, Session, SessionUser, SetDefault, SetNull, Set, Show, Table, Start, Temp, - Temporary, Ties, To, Transaction, True, Type, Union, Unique, Unlogged, Update, Using, Values, Version, View, Where, - Window, With, Work + Add, All, Alter, And, As, Asc, Authorization, Begin, By, Cache, Called, Cascade, Chain, Check, Collate, Column, + Comment, Commit, Concurrently, Conflict, Constraint, Create, Cross, CurrentRole, CurrentUser, Cycle, Data, Database, + Default, Deferrable, Delete, Desc, Distinct, Do, Domain, Drop, Enum, Except, Exists, Extension, False, Fetch, + Filter, First, ForeignKey, From, Full, Function, Global, GroupBy, Having, If, Immutable, In, Include, Increment, + Index, Inner, InOut, Input, InsertInto, Intersect, Interval, Is, IsNull, IsolationLevel, Join, Language, Last, Left, + Like, Limit, Local, MaterializedView, Maxvalue, Minvalue, Natural, Next, No, None, NoAction, Not, Nothing, NotNull, + Null, Nulls, Offset, On, Only, Or, OrderBy, Out, Outer, Over, OwnedBy, OwnerTo, PartitionBy, PrimaryKey, + ReadCommitted, ReadOnly, ReadUncommitted, ReadWrite, Recursive, References, RenameTo, RepeatableRead, Replace, + Restrict, Return, Returning, Returns, Right, Row, Rows, Schema, Select, Sequence, Serializable, Session, + SessionUser, SetDefault, SetNull, Set, SetOf, Show, Table, Stable, Start, Strict, Temp, Temporary, Ties, To, + Transaction, True, Type, Union, Unique, Unlogged, Update, Using, Values, Version, View, Volatile, Where, Window, + With, Work ] const Amp = createToken({name: 'Amp', pattern: /&/}) @@ -275,7 +294,7 @@ const Comma = createToken({name: 'Comma', pattern: /,/}) const CurlyLeft = createToken({name: 'CurlyLeft', pattern: /\{/}) const CurlyRight = createToken({name: 'CurlyRight', pattern: /}/}) const Dash = createToken({name: 'Dash', pattern: /-/, longer_alt: LineComment}) -const Dollar = createToken({name: 'Dollar', pattern: /\$/}) +const Dollar = createToken({name: 'Dollar', pattern: /\$/, longer_alt: StringDollar}) const Dot = createToken({name: 'Dot', pattern: /\./}) const Equal = createToken({name: 'Equal', pattern: /=/}) const Exclamation = createToken({name: 'Exclamation', pattern: /!/}) @@ -543,10 +562,60 @@ class PostgresParser extends EmbeddedActionsParser { this.createFunctionStatementRule = $.RULE<() => CreateFunctionStatementAst>('createFunctionStatementRule', () => { // https://www.postgresql.org/docs/current/sql-createfunction.html const start = $.CONSUME(Create) + const replace = undefined // TODO: $.OPTION(() => tokenInfo2($.CONSUME(Or), $.CONSUME(Replace))) const token = tokenInfo2(start, $.CONSUME(Function)) - // TODO + const object = $.SUBRULE($.objectNameRule) + const args = $.SUBRULE(functionArgumentsRule) + const statement: Pick = {} + $.MANY({DEF: () => $.OR([ + {ALT: () => statement.returns = $.SUBRULE(functionReturnsRule)}, + {ALT: () => statement.language = {token: tokenInfo($.CONSUME(Language)), name: $.SUBRULE3($.identifierRule)}}, + {ALT: () => statement.behavior = $.OR2([ + {ALT: () => ({kind: 'Immutable' as const, token: tokenInfo($.CONSUME(Immutable))})}, + {ALT: () => ({kind: 'Stable' as const, token: tokenInfo($.CONSUME(Stable))})}, + {ALT: () => ({kind: 'Volatile' as const, token: tokenInfo($.CONSUME(Volatile))})}, + ])}, + {ALT: () => statement.definition = {token: tokenInfo($.CONSUME(As)), value: $.SUBRULE(this.stringRule)}}, + {ALT: () => statement.nullBehavior = $.OR3([ + {ALT: () => ({kind: 'Called' as const, token: tokenInfoN([$.CONSUME(Called), $.CONSUME(On), $.CONSUME(Null), $.CONSUME(Input)])})}, + {ALT: () => ({kind: 'ReturnsNull' as const, token: tokenInfoN([$.CONSUME(Returns), $.CONSUME2(Null), $.CONSUME2(On), $.CONSUME3(Null), $.CONSUME2(Input)])})}, + {ALT: () => ({kind: 'Strict' as const, token: tokenInfo($.CONSUME(Strict))})}, + ])}, + {ALT: () => statement.return = {token: tokenInfo($.CONSUME(Return)), expression: $.SUBRULE(this.expressionRule)}}, + ])}) const end = $.CONSUME(Semicolon) - return removeUndefined({kind: 'CreateFunction' as const, meta: tokenInfo2(start, end), token}) + return removeUndefined({kind: 'CreateFunction' as const, meta: tokenInfo2(start, end), token, replace, ...object, args, ...statement}) + }) + const functionArgumentsRule = $.RULE<() => FunctionArgumentAst[]>('functionArgumentsRule', () => { + const args: FunctionArgumentAst[] = [] + $.CONSUME(ParenLeft) + $.MANY_SEP({SEP: Comma, DEF: () => { + const mode = $.OPTION(() => $.OR([ + {ALT: () => ({kind: 'In' as const, token: tokenInfo($.CONSUME(In))})}, + {ALT: () => ({kind: 'Out' as const, token: tokenInfo($.CONSUME(Out))})}, + {ALT: () => ({kind: 'InOut' as const, token: tokenInfo($.CONSUME(InOut))})}, + {ALT: () => ({kind: 'Variadic' as const, token: tokenInfo($.CONSUME(Variadic))})}, + ])) + const name = $.OPTION2(() => $.SUBRULE($.identifierRule)) + const type = $.SUBRULE($.columnTypeRule) + args.push(removeUndefined({mode, name, type})) + }}) + $.CONSUME(ParenRight) + return args + }) + const functionReturnsRule = $.RULE<() => FunctionReturnsAst>('functionReturnsRule', () => { + const ret = $.CONSUME(Returns) + return $.OR([ + {ALT: () => removeUndefined({kind: 'Type' as const, token: tokenInfo(ret), setOf: $.OPTION(() => tokenInfo($.CONSUME(SetOf))), type: $.SUBRULE($.columnTypeRule)})}, + {ALT: () => { + const token = tokenInfo2(ret, $.CONSUME(Table)) + $.CONSUME(ParenLeft) + const columns: {name: IdentifierAst, type: ColumnTypeAst}[] = [] + $.MANY_SEP({SEP: Comma, DEF: () => columns.push({name: $.SUBRULE($.identifierRule), type: $.SUBRULE2($.columnTypeRule)})}) + $.CONSUME(ParenRight) + return {kind: 'Table' as const, token, columns} + }}, + ]) }) this.createIndexStatementRule = $.RULE<() => CreateIndexStatementAst>('createIndexStatementRule', () => { @@ -1570,26 +1639,41 @@ class PostgresParser extends EmbeddedActionsParser { } }}, // tokens allowed as identifiers: + {ALT: () => toIdentifier($.CONSUME(Add))}, + {ALT: () => toIdentifier($.CONSUME(Commit))}, {ALT: () => toIdentifier($.CONSUME(Data))}, {ALT: () => toIdentifier($.CONSUME(Database))}, {ALT: () => toIdentifier($.CONSUME(Deferrable))}, + {ALT: () => toIdentifier($.CONSUME(Increment))}, {ALT: () => toIdentifier($.CONSUME(Index))}, + {ALT: () => toIdentifier($.CONSUME(Input))}, {ALT: () => toIdentifier($.CONSUME(Nulls))}, {ALT: () => toIdentifier($.CONSUME(Rows))}, {ALT: () => toIdentifier($.CONSUME(Schema))}, + {ALT: () => toIdentifier($.CONSUME(Start))}, + {ALT: () => toIdentifier($.CONSUME(Temporary))}, {ALT: () => toIdentifier($.CONSUME(Type))}, {ALT: () => toIdentifier($.CONSUME(Version))}, ])) - this.stringRule = $.RULE<() => StringAst>('stringRule', () => { - const token = $.CONSUME(String) - if (token.image.match(/^E/i)) { - // https://www.postgresql.org/docs/current/sql-syntax-lexical.html - return {kind: 'String', token: tokenInfo(token), value: token.image.slice(2, -1).replaceAll(/''/g, "'"), escaped: true} - } else { - return {kind: 'String', token: tokenInfo(token), value: token.image.slice(1, -1).replaceAll(/''/g, "'")} - } - }) + this.stringRule = $.RULE<() => StringAst>('stringRule', () => $.OR([ + {ALT: () => { + const token = $.CONSUME(String) + if (token.image.match(/^E/i)) { + // https://www.postgresql.org/docs/current/sql-syntax-lexical.html + return {kind: 'String', token: tokenInfo(token), value: token.image.slice(2, -1).replaceAll(/''/g, "'"), escaped: true} + } else { + return {kind: 'String', token: tokenInfo(token), value: token.image.slice(1, -1).replaceAll(/''/g, "'")} + } + }}, + {ALT: () => { + // https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-DOLLAR-QUOTING + const token = $.CONSUME(StringDollar) + const [, dollar] = token.image.match(/^(\$[^$]*\$)/) || [] + const prefix = dollar?.length || 0 + return {kind: 'String', token: tokenInfo(token), value: token.image.slice(prefix, -prefix).trim(), dollar} + }}, + ])) this.integerRule = $.RULE<() => IntegerAst>('integerRule', () => { const neg = $.OPTION(() => $.CONSUME(Dash)) @@ -1682,6 +1766,10 @@ function tokenInfo3(start: IToken | undefined, middle: IToken | undefined, end: return removeEmpty({...mergePositions([start, middle, end].map(t => t ? tokenPosition(t) : undefined)), issues}) } +function tokenInfoN(tokens: (IToken | undefined)[], issues?: TokenIssue[]): TokenInfo { + return removeEmpty({...mergePositions(tokens.map(t => t ? tokenPosition(t) : undefined)), issues}) +} + function tokenPosition(token: IToken): TokenPosition { return { offset: {start: pos(token.startOffset), end: pos(token.endOffset)},