diff --git a/CHANGELOG.md b/CHANGELOG.md index 7153d969..80ef7c6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,9 @@ # Changelog -## 9.1.1 (upcoming) +## 9.2.0 (upcoming) +- Add support for Pandoc-style inline math (`$...$`) and display math (`$$...$$` with the `$$` being at the beginning/end of a Markdown block) to Markdown parser (fixes [vscode-ltex#210](https://github.com/valentjn/vscode-ltex/issues/210)) - Fix false positives for words added by `Add to dictionary` for Slovak rule IDs `MUZSKY_ROD_NEZIV_A`, `ZENSKY_ROD_A`, and `STREDNY_ROD_A` (fixes [vscode-ltex#221](https://github.com/valentjn/vscode-ltex/issues/221)) - Fix BibTEX field `seealso` not ignored, ignore `category` and `parent` (see [vscode-ltex#211](https://github.com/valentjn/vscode-ltex/issues/211)) - Disable `UPPERCASE_SENTENCE_START` in BibTEX files (see [vscode-ltex#211](https://github.com/valentjn/vscode-ltex/issues/211)) diff --git a/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/LtexMarkdownDisplayMath.java b/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/LtexMarkdownDisplayMath.java new file mode 100644 index 00000000..16409044 --- /dev/null +++ b/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/LtexMarkdownDisplayMath.java @@ -0,0 +1,98 @@ +/* Copyright (C) 2020 Julian Valentin, LTeX Development Community + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package org.bsplines.ltexls.parsing.markdown; + +import com.vladsch.flexmark.ast.Paragraph; +import com.vladsch.flexmark.ast.ParagraphContainer; +import com.vladsch.flexmark.util.ast.Block; +import com.vladsch.flexmark.util.ast.BlockContent; +import com.vladsch.flexmark.util.sequence.BasedSequence; +import java.util.List; +import org.jetbrains.annotations.NotNull; + +public class LtexMarkdownDisplayMath extends Block implements ParagraphContainer { + private BasedSequence openingMarker = BasedSequence.NULL; + private BasedSequence openingTrailing = BasedSequence.NULL; + private BasedSequence closingMarker = BasedSequence.NULL; + private BasedSequence closingTrailing = BasedSequence.NULL; + + @Override + public void getAstExtra(@NotNull StringBuilder out) { + segmentSpanChars(out, this.openingMarker, "open"); + segmentSpanChars(out, this.openingTrailing, "openTrail"); + segmentSpanChars(out, this.closingMarker, "close"); + segmentSpanChars(out, this.closingTrailing, "closeTrail"); + } + + @Override + public @NotNull BasedSequence[] getSegments() { + return new BasedSequence[] { + this.openingMarker, + this.openingTrailing, + this.closingMarker, + this.closingTrailing + }; + } + + @Override + public boolean isParagraphEndWrappingDisabled(Paragraph node) { + return ((node == getLastChild()) || (node.getNext() instanceof LtexMarkdownDisplayMath)); + } + + @Override + public boolean isParagraphStartWrappingDisabled(Paragraph node) { + return ((node == getFirstChild()) || (node.getPrevious() instanceof LtexMarkdownDisplayMath)); + } + + public LtexMarkdownDisplayMath() { + } + + public LtexMarkdownDisplayMath(BasedSequence chars) { + super(chars); + } + + public LtexMarkdownDisplayMath(BasedSequence chars, List segments) { + super(chars, segments); + } + + public LtexMarkdownDisplayMath(BlockContent blockContent) { + super(blockContent); + } + + public BasedSequence getOpeningMarker() { + return this.openingMarker; + } + + public void setOpeningMarker(BasedSequence openingMarker) { + this.openingMarker = openingMarker; + } + + public BasedSequence getClosingMarker() { + return this.closingMarker; + } + + public void setClosingMarker(BasedSequence closingMarker) { + this.closingMarker = closingMarker; + } + + public BasedSequence getOpeningTrailing() { + return this.openingTrailing; + } + + public void setOpeningTrailing(BasedSequence openingTrailing) { + this.openingTrailing = openingTrailing; + } + + public BasedSequence getClosingTrailing() { + return this.closingTrailing; + } + + public void setClosingTrailing(BasedSequence closingTrailing) { + this.closingTrailing = closingTrailing; + } +} diff --git a/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/LtexMarkdownDisplayMathParser.java b/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/LtexMarkdownDisplayMathParser.java new file mode 100644 index 00000000..589f6a99 --- /dev/null +++ b/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/LtexMarkdownDisplayMathParser.java @@ -0,0 +1,165 @@ +/* Copyright (C) 2020 Julian Valentin, LTeX Development Community + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package org.bsplines.ltexls.parsing.markdown; + +import com.vladsch.flexmark.parser.InlineParser; +import com.vladsch.flexmark.parser.block.AbstractBlockParser; +import com.vladsch.flexmark.parser.block.AbstractBlockParserFactory; +import com.vladsch.flexmark.parser.block.BlockContinue; +import com.vladsch.flexmark.parser.block.BlockParser; +import com.vladsch.flexmark.parser.block.BlockParserFactory; +import com.vladsch.flexmark.parser.block.BlockStart; +import com.vladsch.flexmark.parser.block.CustomBlockParserFactory; +import com.vladsch.flexmark.parser.block.MatchedBlockParser; +import com.vladsch.flexmark.parser.block.ParserState; +import com.vladsch.flexmark.util.ast.Block; +import com.vladsch.flexmark.util.ast.BlockContent; +import com.vladsch.flexmark.util.ast.Node; +import com.vladsch.flexmark.util.data.DataHolder; +import com.vladsch.flexmark.util.sequence.BasedSequence; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class LtexMarkdownDisplayMathParser extends AbstractBlockParser { + private static final Pattern DISPLAY_MATH_START_PATTERN = Pattern.compile("\\$\\$(\\s*$)"); + private static final Pattern DISPLAY_MATH_END_PATTERN = Pattern.compile("\\$\\$(\\s*$)"); + + private LtexMarkdownDisplayMath block = new LtexMarkdownDisplayMath(); + private @Nullable BlockContent content = new BlockContent(); + private boolean hadClose = false; + + LtexMarkdownDisplayMathParser(DataHolder options, BasedSequence openMarker, + BasedSequence openTrailing) { + this.block.setOpeningMarker(openMarker); + this.block.setOpeningTrailing(openTrailing); + } + + @Override + public Block getBlock() { + return this.block; + } + + @Override + public BlockContinue tryContinue(ParserState state) { + if (this.hadClose) return BlockContinue.none(); + + int index = state.getIndex(); + BasedSequence line = state.getLineWithEOL(); + Matcher matcher = DISPLAY_MATH_END_PATTERN.matcher(line.subSequence(index)); + + if (!matcher.matches()) { + return BlockContinue.atIndex(index); + } else { + @Nullable Node lastChild = this.block.getLastChild(); + + if ((lastChild != null) && (lastChild instanceof LtexMarkdownDisplayMath)) { + BlockParser parser = state.getActiveBlockParser((Block)lastChild); + + if ((parser instanceof LtexMarkdownDisplayMathParser) + && !((LtexMarkdownDisplayMathParser)parser).hadClose) { + return BlockContinue.atIndex(index); + } + } + + this.hadClose = true; + this.block.setClosingMarker(state.getLine().subSequence(index, index + 2)); + this.block.setClosingTrailing( + state.getLineWithEOL().subSequence(matcher.start(1), matcher.end(1))); + + return BlockContinue.atIndex(state.getLineEndIndex()); + } + } + + @Override + public void addLine(ParserState state, BasedSequence line) { + if (this.content == null) return; + this.content.add(line, state.getIndent()); + } + + @Override + public void closeBlock(ParserState state) { + if (this.content == null) return; + this.block.setContent(this.content); + this.block.setCharsFromContent(); + this.content = null; + } + + @Override + public boolean isContainer() { + return false; + } + + @Override + public boolean canContain(ParserState state, BlockParser blockParser, Block block) { + return false; + } + + @Override + public void parseInlines(InlineParser inlineParser) { + } + + public static class Factory implements CustomBlockParserFactory { + @Override + public @Nullable Set> getAfterDependents() { + return null; + } + + @Override + public @Nullable Set> getBeforeDependents() { + return null; + } + + @Override + public boolean affectsGlobalScope() { + return false; + } + + @Override + public @NotNull BlockParserFactory apply(@NotNull DataHolder options) { + return new BlockFactory(options); + } + } + + private static class BlockFactory extends AbstractBlockParserFactory { + BlockFactory(DataHolder options) { + super(options); + } + + private static boolean haveDisplayMathParser(ParserState state) { + List parsers = state.getActiveBlockParsers(); + + for (int i = parsers.size() - 1; i >= 0; i--) { + if (parsers.get(i) instanceof LtexMarkdownDisplayMathParser) return true; + } + + return false; + } + + @Override + public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockParser) { + if (!haveDisplayMathParser(state)) { + BasedSequence line = state.getLineWithEOL(); + Matcher matcher = DISPLAY_MATH_START_PATTERN.matcher(line); + + if (matcher.matches()) { + LtexMarkdownDisplayMathParser parser = new LtexMarkdownDisplayMathParser( + state.getProperties(), + line.subSequence(0, 2), + line.subSequence(matcher.start(1), matcher.end(1))); + return BlockStart.of(parser).atIndex(state.getLineEndIndex()); + } + } + + return BlockStart.none(); + } + } +} \ No newline at end of file diff --git a/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/LtexMarkdownExtension.java b/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/LtexMarkdownExtension.java new file mode 100644 index 00000000..66fd63c6 --- /dev/null +++ b/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/LtexMarkdownExtension.java @@ -0,0 +1,43 @@ +/* Copyright (C) 2020 Julian Valentin, LTeX Development Community + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package org.bsplines.ltexls.parsing.markdown; + +import com.vladsch.flexmark.parser.Parser; +import com.vladsch.flexmark.util.data.DataKey; +import com.vladsch.flexmark.util.data.MutableDataHolder; + +public class LtexMarkdownExtension implements Parser.ParserExtension { + public static final DataKey DISPLAY_MATH_PARSER = + new DataKey<>("DISPLAY_MATH_PARSER", true); + public static final DataKey INLINE_MATH_PARSER = + new DataKey<>("INLINE_MATH_PARSER", true); + + private LtexMarkdownExtension() { + } + + public static LtexMarkdownExtension create() { + return new LtexMarkdownExtension(); + } + + @Override + public void parserOptions(MutableDataHolder options) { + } + + @Override + public void extend(Parser.Builder parserBuilder) { + LtexMarkdownOptions options = new LtexMarkdownOptions(parserBuilder); + + if (options.displayMathParser) { + parserBuilder.customBlockParserFactory(new LtexMarkdownDisplayMathParser.Factory()); + } + + if (options.inlineMathParser) { + parserBuilder.customInlineParserExtensionFactory(new LtexMarkdownInlineMathParser.Factory()); + } + } +} diff --git a/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/LtexMarkdownInlineMath.java b/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/LtexMarkdownInlineMath.java new file mode 100644 index 00000000..68b59241 --- /dev/null +++ b/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/LtexMarkdownInlineMath.java @@ -0,0 +1,69 @@ +/* Copyright (C) 2020 Julian Valentin, LTeX Development Community + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package org.bsplines.ltexls.parsing.markdown; + +import com.vladsch.flexmark.util.ast.DelimitedNode; +import com.vladsch.flexmark.util.ast.Node; +import com.vladsch.flexmark.util.sequence.BasedSequence; +import org.jetbrains.annotations.NotNull; + +public class LtexMarkdownInlineMath extends Node implements DelimitedNode { + protected BasedSequence openingMarker = BasedSequence.NULL; + protected BasedSequence text = BasedSequence.NULL; + protected BasedSequence closingMarker = BasedSequence.NULL; + + @Override + public @NotNull BasedSequence[] getSegments() { + return new BasedSequence[] { this.openingMarker, this.text, this.closingMarker }; + } + + @Override + public void getAstExtra(@NotNull StringBuilder out) { + delimitedSegmentSpanChars(out, this.openingMarker, this.text, this.closingMarker, "text"); + } + + public LtexMarkdownInlineMath() { + } + + public LtexMarkdownInlineMath(BasedSequence chars) { + super(chars); + } + + public LtexMarkdownInlineMath(BasedSequence openingMarker, BasedSequence text, + BasedSequence closingMarker) { + super(openingMarker.baseSubSequence( + openingMarker.getStartOffset(), closingMarker.getEndOffset())); + this.openingMarker = openingMarker; + this.text = text; + this.closingMarker = closingMarker; + } + + public BasedSequence getOpeningMarker() { + return this.openingMarker; + } + + public void setOpeningMarker(BasedSequence openingMarker) { + this.openingMarker = openingMarker; + } + + public BasedSequence getText() { + return this.text; + } + + public void setText(BasedSequence text) { + this.text = text; + } + + public BasedSequence getClosingMarker() { + return this.closingMarker; + } + + public void setClosingMarker(BasedSequence closingMarker) { + this.closingMarker = closingMarker; + } +} diff --git a/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/LtexMarkdownInlineMathParser.java b/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/LtexMarkdownInlineMathParser.java new file mode 100644 index 00000000..ccae1658 --- /dev/null +++ b/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/LtexMarkdownInlineMathParser.java @@ -0,0 +1,87 @@ +/* Copyright (C) 2020 Julian Valentin, LTeX Development Community + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package org.bsplines.ltexls.parsing.markdown; + +import com.vladsch.flexmark.parser.InlineParser; +import com.vladsch.flexmark.parser.InlineParserExtension; +import com.vladsch.flexmark.parser.InlineParserExtensionFactory; +import com.vladsch.flexmark.parser.LightInlineParser; +import com.vladsch.flexmark.util.sequence.BasedSequence; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class LtexMarkdownInlineMathParser implements InlineParserExtension { + private static final Pattern MATH_PATTERN = Pattern.compile( + "\\$([^ ](?:.|\n)*?[^ ])(?> getAfterDependents() { + return null; + } + + @Override + public @NotNull CharSequence getCharacters() { + return "$"; + } + + @Override + public @Nullable Set> getBeforeDependents() { + return null; + } + + @Override + public @NotNull InlineParserExtension apply(@NotNull LightInlineParser lightInlineParser) { + return new LtexMarkdownInlineMathParser(lightInlineParser); + } + + @Override + public boolean affectsGlobalScope() { + return false; + } + } +} diff --git a/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/LtexMarkdownOptions.java b/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/LtexMarkdownOptions.java new file mode 100644 index 00000000..5ce5d570 --- /dev/null +++ b/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/LtexMarkdownOptions.java @@ -0,0 +1,31 @@ +/* Copyright (C) 2020 Julian Valentin, LTeX Development Community + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package org.bsplines.ltexls.parsing.markdown; + +import com.vladsch.flexmark.util.data.DataHolder; +import com.vladsch.flexmark.util.data.MutableDataHolder; +import com.vladsch.flexmark.util.data.MutableDataSetter; +import org.jetbrains.annotations.NotNull; + +public class LtexMarkdownOptions implements MutableDataSetter { + public boolean displayMathParser; + public boolean inlineMathParser; + + public LtexMarkdownOptions(DataHolder options) { + this.displayMathParser = LtexMarkdownExtension.DISPLAY_MATH_PARSER.get(options); + this.inlineMathParser = LtexMarkdownExtension.INLINE_MATH_PARSER.get(options); + } + + @Override + public @NotNull MutableDataHolder setIn(@NotNull MutableDataHolder dataHolder) { + dataHolder.set(LtexMarkdownExtension.DISPLAY_MATH_PARSER, this.displayMathParser); + dataHolder.set(LtexMarkdownExtension.INLINE_MATH_PARSER, this.inlineMathParser); + + return dataHolder; + } +} diff --git a/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/MarkdownAnnotatedTextBuilder.java b/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/MarkdownAnnotatedTextBuilder.java index 270ee604..113a7803 100644 --- a/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/MarkdownAnnotatedTextBuilder.java +++ b/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/MarkdownAnnotatedTextBuilder.java @@ -34,7 +34,8 @@ public class MarkdownAnnotatedTextBuilder extends CodeAnnotatedTextBuilder { Arrays.asList( GitLabExtension.create(), TablesExtension.create(), - YamlFrontMatterExtension.create() + YamlFrontMatterExtension.create(), + LtexMarkdownExtension.create() )); private Parser parser = Parser.builder(parserOptions).build(); diff --git a/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/MarkdownAnnotatedTextBuilderDefaults.java b/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/MarkdownAnnotatedTextBuilderDefaults.java index cf931fc3..c2039dd1 100644 --- a/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/MarkdownAnnotatedTextBuilderDefaults.java +++ b/ltexls-core/src/main/java/org/bsplines/ltexls/parsing/markdown/MarkdownAnnotatedTextBuilderDefaults.java @@ -24,6 +24,9 @@ private static List createDefaultMarkdownNodeSignatures() list.add(new MarkdownNodeSignature("FencedCodeBlock")); list.add(new MarkdownNodeSignature("GitLabInlineMath", MarkdownNodeSignature.Action.DUMMY)); list.add(new MarkdownNodeSignature("IndentedCodeBlock")); + list.add(new MarkdownNodeSignature("LtexMarkdownDisplayMath")); + list.add(new MarkdownNodeSignature("LtexMarkdownInlineMath", + MarkdownNodeSignature.Action.DUMMY)); list.add(new MarkdownNodeSignature("TableSeparator")); return list; diff --git a/ltexls-core/src/test/java/org/bsplines/ltexls/parsing/markdown/MarkdownAnnotatedTextBuilderTest.java b/ltexls-core/src/test/java/org/bsplines/ltexls/parsing/markdown/MarkdownAnnotatedTextBuilderTest.java index 0c99380e..8065b288 100644 --- a/ltexls-core/src/test/java/org/bsplines/ltexls/parsing/markdown/MarkdownAnnotatedTextBuilderTest.java +++ b/ltexls-core/src/test/java/org/bsplines/ltexls/parsing/markdown/MarkdownAnnotatedTextBuilderTest.java @@ -55,6 +55,21 @@ public void test() throws IOException { assertPlainText( "This is a test: $`E = mc^2`$.\n\n```math\na^2 + b^2 = c^2\n```\n\nThis is another test.\n", "This is a test: Dummy0.\n\n\n\n\n\nThis is another test.\n"); + assertPlainText( + "This is a test: $E = mc^2\n" + + "$.\n" + + "The book is $3, not $5.\n" + + "\n" + + "Interesting: $1 \\$2 3$.\n" + + "\n" + + "$$\n" + + "a^2 + b^2 = c^2\n" + + "\n" + + "$$\n" + + "\n" + + "This is another test.\n", + "This is a test: Dummy0. The book is $3, not $5.\n\nInteresting: Dummy1.\n\n\n\n\n\n\n" + + "This is another test.\n"); assertPlainText( "This is a test.\n" + "\n"