Skip to content

Commit

Permalink
Merge pull request #90 from MDeiml/code-span-parse-ahead
Browse files Browse the repository at this point in the history 
Avoid conflicts for code spans by parsing ahead
  • Loading branch information
@MDeiml
MDeiml committed Mar 5, 2023
2 parents 10b9b5a + 5fb1552 commit cadb630
Show file tree
Hide file tree
Showing 5 changed files with 23,612 additions and 23,707 deletions.
16 changes: 16 additions & 0 deletions tree-sitter-markdown-inline/corpus/issues.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,3 +88,19 @@ foo `<!--comment-->`
(code_span
(code_span_delimiter)
(code_span_delimiter)))

================================================================================
#75 - code spans with `[x][]`
================================================================================
`some code`
normal text (or even nothing) `[index][]`

--------------------------------------------------------------------------------

(inline
(code_span
(code_span_delimiter)
(code_span_delimiter))
(code_span
(code_span_delimiter)
(code_span_delimiter)))
24 changes: 10 additions & 14 deletions tree-sitter-markdown-inline/grammar.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,6 @@ const common = require('../common/grammar.js');
const PRECEDENCE_LEVEL_EMPHASIS = 1;
const PRECEDENCE_LEVEL_LINK = 10;
const PRECEDENCE_LEVEL_HTML = 100;
const PRECEDENCE_LEVEL_CODE_SPAN = 100;
const PRECEDENCE_LEVEL_LATEX = 100;

// Punctuation characters as specified in
// https://github.github.com/gfm/#ascii-punctuation-character
Expand Down Expand Up @@ -70,6 +68,10 @@ module.exports = grammar(add_inline_rules({
// An opening token does not mean the text after has to be latex if there is no closing token
$._latex_span_start,
$._latex_span_close,

// Token emmited when encountering opening delimiters for a leaf span
// e.g. a code span, that does not have a matching closing span
$._unclosed_span
],
precedences: $ => [
// [$._strong_emphasis_star, $._inline_element_no_star],
Expand All @@ -81,8 +83,6 @@ module.exports = grammar(add_inline_rules({
],
// More conflicts are defined in `add_inline_rules`
conflicts: $ => [
[$.code_span, $._inline_base],
[$.latex_block, $._inline_base],

[$._closing_tag, $._text_base],
[$._open_tag, $._text_base],
Expand Down Expand Up @@ -121,25 +121,23 @@ module.exports = grammar(add_inline_rules({
// A lot of inlines are defined in `add_inline_rules`, including:
//
// * collections of inlines
// * code spans
// * latex spans
// * emphasis
// * textual content
//
// This is done to reduce code duplication, as some inlines need to be parsed differently
// depending on the context. For example inlines in ATX headings may not contain newlines.

code_span: $ => prec.dynamic(PRECEDENCE_LEVEL_CODE_SPAN, seq(
code_span: $ => seq(
alias($._code_span_start, $.code_span_delimiter),
repeat(choice($._text_base, '[', ']', $._soft_line_break, $._html_tag)),
alias($._code_span_close, $.code_span_delimiter)
)),
),

latex_block: $ => prec.dynamic(PRECEDENCE_LEVEL_LATEX, seq(
latex_block: $ => seq(
alias($._latex_span_start, $.latex_span_delimiter),
repeat(choice($._text_base, '[', ']', $._soft_line_break, $._html_tag)),
alias($._latex_span_close, $.latex_span_delimiter),
)),
),

// Different kinds of links:
// * inline links (https://github.github.com/gfm/#inline-link)
Expand Down Expand Up @@ -346,9 +344,8 @@ module.exports = grammar(add_inline_rules({
$.code_span,
alias($._html_tag, $.html_tag),
$._text_base,
$._code_span_start,
common.EXTENSION_TAGS ? $.tag : choice(),
(common.EXTENSION_LATEX ? $._latex_span_start : choice()),
$._unclosed_span,
))),
_text_base: $ => choice(
$._word,
Expand All @@ -361,10 +358,9 @@ module.exports = grammar(add_inline_rules({
),
_text_inline_no_link: $ => choice(
$._text_base,
$._code_span_start,
$._latex_span_start,
$._emphasis_open_star,
$._emphasis_open_underscore,
$._unclosed_span,
),

...(common.EXTENSION_TAGS ? {
Expand Down
224 changes: 102 additions & 122 deletions tree-sitter-markdown-inline/src/grammar.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1458,114 +1458,106 @@
"value": "\\n|\\r\\n?"
},
"code_span": {
"type": "PREC_DYNAMIC",
"value": 100,
"content": {
"type": "SEQ",
"members": [
{
"type": "ALIAS",
"content": {
"type": "SYMBOL",
"name": "_code_span_start"
},
"named": true,
"value": "code_span_delimiter"
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_text_base"
},
{
"type": "STRING",
"value": "["
},
{
"type": "STRING",
"value": "]"
},
{
"type": "SYMBOL",
"name": "_soft_line_break"
},
{
"type": "SYMBOL",
"name": "_html_tag"
}
]
}
"type": "SEQ",
"members": [
{
"type": "ALIAS",
"content": {
"type": "SYMBOL",
"name": "_code_span_start"
},
{
"type": "ALIAS",
"content": {
"type": "SYMBOL",
"name": "_code_span_close"
},
"named": true,
"value": "code_span_delimiter"
"named": true,
"value": "code_span_delimiter"
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_text_base"
},
{
"type": "STRING",
"value": "["
},
{
"type": "STRING",
"value": "]"
},
{
"type": "SYMBOL",
"name": "_soft_line_break"
},
{
"type": "SYMBOL",
"name": "_html_tag"
}
]
}
]
}
},
{
"type": "ALIAS",
"content": {
"type": "SYMBOL",
"name": "_code_span_close"
},
"named": true,
"value": "code_span_delimiter"
}
]
},
"latex_block": {
"type": "PREC_DYNAMIC",
"value": 100,
"content": {
"type": "SEQ",
"members": [
{
"type": "ALIAS",
"content": {
"type": "SYMBOL",
"name": "_latex_span_start"
},
"named": true,
"value": "latex_span_delimiter"
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_text_base"
},
{
"type": "STRING",
"value": "["
},
{
"type": "STRING",
"value": "]"
},
{
"type": "SYMBOL",
"name": "_soft_line_break"
},
{
"type": "SYMBOL",
"name": "_html_tag"
}
]
}
"type": "SEQ",
"members": [
{
"type": "ALIAS",
"content": {
"type": "SYMBOL",
"name": "_latex_span_start"
},
{
"type": "ALIAS",
"content": {
"type": "SYMBOL",
"name": "_latex_span_close"
},
"named": true,
"value": "latex_span_delimiter"
"named": true,
"value": "latex_span_delimiter"
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_text_base"
},
{
"type": "STRING",
"value": "["
},
{
"type": "STRING",
"value": "]"
},
{
"type": "SYMBOL",
"name": "_soft_line_break"
},
{
"type": "SYMBOL",
"name": "_html_tag"
}
]
}
]
}
},
{
"type": "ALIAS",
"content": {
"type": "SYMBOL",
"name": "_latex_span_close"
},
"named": true,
"value": "latex_span_delimiter"
}
]
},
"_link_text": {
"type": "PREC_DYNAMIC",
Expand Down Expand Up @@ -4215,17 +4207,13 @@
"type": "SYMBOL",
"name": "_text_base"
},
{
"type": "SYMBOL",
"name": "_code_span_start"
},
{
"type": "CHOICE",
"members": []
},
{
"type": "SYMBOL",
"name": "_latex_span_start"
"name": "_unclosed_span"
}
]
}
Expand Down Expand Up @@ -4411,19 +4399,15 @@
},
{
"type": "SYMBOL",
"name": "_code_span_start"
},
{
"type": "SYMBOL",
"name": "_latex_span_start"
"name": "_emphasis_open_star"
},
{
"type": "SYMBOL",
"name": "_emphasis_open_star"
"name": "_emphasis_open_underscore"
},
{
"type": "SYMBOL",
"name": "_emphasis_open_underscore"
"name": "_unclosed_span"
}
]
},
Expand Down Expand Up @@ -5556,14 +5540,6 @@
},
"extras": [],
"conflicts": [
[
"code_span",
"_inline_base"
],
[
"latex_block",
"_inline_base"
],
[
"_closing_tag",
"_text_base"
Expand Down Expand Up @@ -5888,6 +5864,10 @@
{
"type": "SYMBOL",
"name": "_latex_span_close"
},
{
"type": "SYMBOL",
"name": "_unclosed_span"
}
],
"inline": [],
Expand Down
Loading

0 comments on commit cadb630

Please sign in to comment.