Skip to content

Commit

Permalink
Add ARRAY in expressions
Browse files Browse the repository at this point in the history
  • Loading branch information
@loicknuchel
loicknuchel committed Nov 8, 2024
1 parent 05927bb commit 9dc8e7e
Show file tree
Hide file tree
Showing 4 changed files with 23 additions and 13 deletions.
3 changes: 2 additions & 1 deletion libs/parser-sql/src/postgresAst.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ export type FunctionReturnsAst = { kind: 'Type', token: TokenInfo, setOf?: Token
// basic parts
export type AliasAst = { token?: TokenInfo, name: IdentifierAst }
export type ObjectNameAst = { schema?: IdentifierAst, name: IdentifierAst }
export type ExpressionAst = (LiteralAst | ParameterAst | ColumnAst | WildcardAst | FunctionAst | GroupAst | OperationAst | OperationLeftAst | OperationRightAst | ListAst) & { cast?: { token: TokenInfo, type: ColumnTypeAst } }
export type ExpressionAst = (LiteralAst | ParameterAst | ColumnAst | WildcardAst | FunctionAst | GroupAst | OperationAst | OperationLeftAst | OperationRightAst | ArrayAst | ListAst) & { cast?: { token: TokenInfo, type: ColumnTypeAst } }
export type LiteralAst = StringAst | IntegerAst | DecimalAst | BooleanAst | NullAst
export type ColumnAst = { kind: 'Column', schema?: IdentifierAst, table?: IdentifierAst, column: IdentifierAst, json?: ColumnJsonAst[] }
export type ColumnJsonAst = { kind: JsonOp, token: TokenInfo, field: StringAst | ParameterAst }
Expand All @@ -127,6 +127,7 @@ export type OperationLeftAst = { kind: 'OperationLeft', op: OperatorLeftAst, rig
export type OperatorLeftAst = { kind: OperatorLeft, token: TokenInfo }
export type OperationRightAst = { kind: 'OperationRight', left: ExpressionAst, op: OperatorRightAst }
export type OperatorRightAst = { kind: OperatorRight, token: TokenInfo }
export type ArrayAst = { kind: 'Array', token: TokenInfo, items: ExpressionAst[] }
export type ListAst = { kind: 'List', items: LiteralAst[] }
export type SortOrderAst = { kind: SortOrder, token: TokenInfo }
export type SortNullsAst = { kind: SortNulls, token: TokenInfo }
Expand Down
5 changes: 5 additions & 0 deletions libs/parser-sql/src/postgresBuilder.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -415,6 +415,7 @@ function expressionToString(e: ExpressionAst): string {
if (e.kind === 'Operation') return expressionToString(e.left) + ' ' + operatorToString(e.op.kind) + ' ' + expressionToString(e.right)
if (e.kind === 'OperationLeft') return operatorLeftToString(e.op.kind) + ' ' + expressionToString(e.right)
if (e.kind === 'OperationRight') return expressionToString(e.left) + ' ' + operatorRightToString(e.op.kind)
if (e.kind === 'Array') return '[' + e.items.map(expressionToString).join(', ') + ']'
if (e.kind === 'List') return '(' + e.items.map(expressionToString).join(', ') + ')'
return isNever(e)
}
Expand All @@ -433,6 +434,7 @@ function expressionToValue(e: ExpressionAst): AttributeValue {
if (e.kind === 'Operation') return '`' + expressionToString(e) + '`'
if (e.kind === 'OperationLeft') return '`' + expressionToString(e) + '`'
if (e.kind === 'OperationRight') return '`' + expressionToString(e) + '`'
if (e.kind === 'Array') return e.items.map(expressionToValue)
if (e.kind === 'List') return e.items.map(expressionToValue)
return isNever(e)
}
Expand Down Expand Up @@ -488,6 +490,7 @@ function expressionAttrs(e: ExpressionAst): AttributePath[] {
if (e.kind === 'Operation') return distinctBy(expressionAttrs(e.left).concat(expressionAttrs(e.right)), p => p.join('.'))
if (e.kind === 'OperationLeft') return expressionAttrs(e.right)
if (e.kind === 'OperationRight') return expressionAttrs(e.left)
if (e.kind === 'Array') return []
if (e.kind === 'List') return []
return isNever(e)
}
Expand Down Expand Up @@ -615,6 +618,8 @@ function expressionToken(e: ExpressionAst): TokenPosition {
return mergePositions([e.op.token, expressionToken(e.right)])
} else if (e.kind === 'OperationRight') {
return mergePositions([expressionToken(e.left), e.op.token])
} else if (e.kind === 'Array') {
return mergePositions([e.token, ...e.items.map(expressionToken)])
} else if (e.kind === 'List') {
return mergePositions(e.items.map(expressionToken))
} else {
Expand Down
16 changes: 5 additions & 11 deletions libs/parser-sql/src/postgresParser.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,12 +72,6 @@ describe('postgresParser', () => {
const sql = fs.readFileSync('./resources/plausible.sql', 'utf8')
const parsed = parsePostgresAst(sql, {strict: true})
expect(parsed.errors || []).toEqual([])
// TODO:246 `features character varying(255)[] DEFAULT ARRAY['props'::character varying, 'stats_api'::character varying] NOT NULL,`
// TODO:542 `recipients public.citext[] DEFAULT ARRAY[]::public.citext[] NOT NULL,`
// TODO:575 `errors jsonb[] DEFAULT ARRAY[]::jsonb[] NOT NULL,`
// TODO:585 `tags character varying(255)[] DEFAULT ARRAY[]::character varying[],`
// TODO:1357 `recipients public.citext[] DEFAULT ARRAY[]::public.citext[] NOT NULL,`
// TODO:1357 `recipients public.citext[] DEFAULT ARRAY[]::public.citext[] NOT NULL`
// TODO:2246 `CREATE TRIGGER ...`
})
test.skip('other structures', async () => {
Expand All @@ -87,18 +81,18 @@ describe('postgresParser', () => {
'https://raw.githubusercontent.com/gocardless/draupnir/refs/heads/master/structure.sql',
'https://raw.githubusercontent.com/LuisMDeveloper/travis-core/refs/heads/master/db/structure.sql',
'https://raw.githubusercontent.com/gustavodiel/monet-backend/refs/heads/main/db/structure.sql',
// 'https://raw.githubusercontent.com/plausible/analytics/refs/heads/master/priv/repo/structure.sql', // fail#246: `DEFAULT ARRAY['props'::character varying, 'stats_api'::character varying]`
// 'https://raw.githubusercontent.com/inaturalist/inaturalist/refs/heads/main/db/structure.sql', // fail#44: column type: `numeric[]`
// 'https://raw.githubusercontent.com/plausible/analytics/refs/heads/master/priv/repo/structure.sql', // fail#2246: `CREATE TRIGGER`
// 'https://raw.githubusercontent.com/inaturalist/inaturalist/refs/heads/main/db/structure.sql', // fail#44: `CREATE FUNCTION` with unnamed parameter
// 'https://raw.githubusercontent.com/cardstack/cardstack/refs/heads/main/packages/hub/config/structure.sql', // fail#52: `ALTER TYPE` & 1986: `COPY`
// 'https://raw.githubusercontent.com/drenther/Empirical-Core/refs/heads/develop/db/structure.sql', // fail#688: `ARRAY[]`
// 'https://raw.githubusercontent.com/drenther/Empirical-Core/refs/heads/develop/db/structure.sql', // fail#2894: `CREATE INDEX email_idx ON users USING gin (email gin_trgm_ops);`
// 'https://raw.githubusercontent.com/spuddybike/archivist/refs/heads/develop/db/structure.sql', // fail#849: parenthesis in FROM clause
// 'https://raw.githubusercontent.com/sathreddy/bety/refs/heads/master/db/structure.sql', // fail#274: LexingError: unexpected character: ->\\<-
// 'https://raw.githubusercontent.com/dhbtk/achabus/refs/heads/master/db/structure.sql', // fail#293: column type: `geography(Point,4326)`
// 'https://raw.githubusercontent.com/mveytsman/community/refs/heads/master/db/structure.sql', // fail#299: JOIN (`SELECT * FROM users, events;`)
// 'https://raw.githubusercontent.com/yazilimcilarinmolayeri/pixels-clone/refs/heads/master/Structure.sql', // fail#68: column `GENERATED ALWAYS AS IDENTITY` TODO
// 'https://raw.githubusercontent.com/henry2992/aprendemy/refs/heads/master/db/structure.sql', // fail#1961: `CREATE TRIGGER`
// 'https://raw.githubusercontent.com/ppawel/openstreetmap-watch-list/refs/heads/master/db/structure.sql', // fail#92: `CREATE TYPE change AS (geom geometry(Geometry,4326))`
// 'https://raw.githubusercontent.com/OPG-813/electronic-queue-server/refs/heads/master/src/db/structure.sql', // fail#2: `CREATE OR REPLACE FUNCTION` TODO
// 'https://raw.githubusercontent.com/OPG-813/electronic-queue-server/refs/heads/master/src/db/structure.sql', // fail#110: `DEFAULT CURRENT_DATE AT TIME ZONE( SYSTEM_TIMEZONE() )`
// 'https://raw.githubusercontent.com/Rotabot-io/rotabot/refs/heads/main/assets/structure.sql', // fail#57: `ALTER FUNCTION public.generate_uid(size integer) OWNER TO rotabot;` TODO
// 'https://raw.githubusercontent.com/bocoup/devstats/refs/heads/master/structure.sql', // fail#57: `ALTER FUNCTION current_state.label_prefix(some_label text) OWNER TO devstats_team;` TODO
// 'https://raw.githubusercontent.com/TechnoDann/PPC-board-2.0/refs/heads/master/db/structure.sql', // fail#350: `CREATE INDEX index_posts_on_ancestry ON public.posts USING btree (ancestry text_pattern_ops NULLS FIRST);`
Expand Down Expand Up @@ -1202,7 +1196,7 @@ describe('postgresParser', () => {
expect(parseRule(p => p.identifierRule(), '"an id with \\""')).toEqual({result: {...identifier('an id with "', 0, 14), quoted: true}})
})
test('not empty', () => {
const specials = ['Add', 'Commit', 'Data', 'Database', 'Deferrable', 'Domain', 'Increment', 'Index', 'Input', 'Nulls', 'Rows', 'Schema', 'Start', 'Temporary', 'Type', 'Version']
const specials = ['Add', 'Commit', 'Data', 'Database', 'Deferrable', 'Domain', 'Increment', 'Index', 'Input', 'Nulls', 'Rows', 'Schema', 'Session', 'Start', 'Temporary', 'Type', 'Version']
expect(parseRule(p => p.identifierRule(), '""')).toEqual({errors: [
{kind: 'LexingError', level: 'error', message: 'unexpected character: ->"<- at offset: 0, skipped 2 characters.', ...token(0, 2)},
{kind: 'NoViableAltException', level: 'error', message: `Expecting: one of these possible Token sequences:\n 1. [Identifier]${specials.map((n, i) => `\n ${i + 2}. [${n}]`).join('')}\nbut found: ''`, offset: {start: -1, end: -1}, position: {start: {line: -1, column: -1}, end: {line: -1, column: -1}}}
Expand Down
12 changes: 11 additions & 1 deletion libs/parser-sql/src/postgresParser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ const Add = createToken({name: 'Add', pattern: /\bADD\b/i, longer_alt: Identifie
const All = createToken({name: 'All', pattern: /\bALL\b/i, longer_alt: Identifier})
const Alter = createToken({name: 'Alter', pattern: /\bALTER\b/i, longer_alt: Identifier})
const And = createToken({name: 'And', pattern: /\bAND\b/i, longer_alt: Identifier})
const Array = createToken({name: 'Array', pattern: /\bARRAY\b/i, longer_alt: Identifier})
const As = createToken({name: 'As', pattern: /\bAS\b/i, longer_alt: Identifier})
const Asc = createToken({name: 'Asc', pattern: /\bASC\b/i, longer_alt: Identifier})
const Authorization = createToken({name: 'Authorization', pattern: /\bAUTHORIZATION\b/i, longer_alt: Identifier})
Expand Down Expand Up @@ -270,7 +271,7 @@ const Window = createToken({name: 'Window', pattern: /\bWINDOW\b/i, longer_alt:
const With = createToken({name: 'With', pattern: /\bWITH\b/i, longer_alt: Identifier})
const Work = createToken({name: 'Work', pattern: /\bWORK\b/i, longer_alt: Identifier})
const keywordTokens: TokenType[] = [
Add, All, Alter, And, As, Asc, Authorization, Begin, By, Cache, Called, Cascade, Chain, Check, Collate, Column,
Add, All, Alter, And, Array, As, Asc, Authorization, Begin, By, Cache, Called, Cascade, Chain, Check, Collate, Column,
Comment, Commit, Concurrently, Conflict, Constraint, Create, Cross, CurrentRole, CurrentUser, Cycle, Data, Database,
Default, Deferrable, Delete, Desc, Distinct, Do, Domain, Drop, Enum, Except, Exists, Extension, False, Fetch,
Filter, First, ForeignKey, From, Full, Function, Global, GroupBy, Having, If, Immutable, In, Include, Increment,
Expand Down Expand Up @@ -1437,6 +1438,14 @@ class PostgresParser extends EmbeddedActionsParser {
{ALT: () => $.SUBRULE(groupRule)},
{ALT: () => $.SUBRULE($.literalRule)},
{ALT: () => ({kind: 'Wildcard', token: tokenInfo($.CONSUME(Asterisk))})},
{ALT: () => {
const token = tokenInfo($.CONSUME(Array))
$.CONSUME(BracketLeft)
const items: ExpressionAst[] = []
$.MANY_SEP({SEP: Comma, DEF: () => items.push($.SUBRULE($.expressionRule))})
$.CONSUME(BracketRight)
return {kind: 'Array' as const, token, items}
}},
{ALT: () => {
const first = $.SUBRULE($.identifierRule)
const nest = $.OPTION(() => $.OR2([
Expand Down Expand Up @@ -1658,6 +1667,7 @@ class PostgresParser extends EmbeddedActionsParser {
{ALT: () => toIdentifier($.CONSUME(Nulls))},
{ALT: () => toIdentifier($.CONSUME(Rows))},
{ALT: () => toIdentifier($.CONSUME(Schema))},
{ALT: () => toIdentifier($.CONSUME(Session))},
{ALT: () => toIdentifier($.CONSUME(Start))},
{ALT: () => toIdentifier($.CONSUME(Temporary))},
{ALT: () => toIdentifier($.CONSUME(Type))},
Expand Down

0 comments on commit 9dc8e7e

Please sign in to comment.