diff --git a/server/engine/src/main/java/org/eclipse/lsp/cobol/core/visitor/CobolProcedureDivisionVisitor.java b/server/engine/src/main/java/org/eclipse/lsp/cobol/core/visitor/CobolProcedureDivisionVisitor.java index d201724532..4311b062ef 100644 --- a/server/engine/src/main/java/org/eclipse/lsp/cobol/core/visitor/CobolProcedureDivisionVisitor.java +++ b/server/engine/src/main/java/org/eclipse/lsp/cobol/core/visitor/CobolProcedureDivisionVisitor.java @@ -909,9 +909,7 @@ private void areaAWarning(Token token) { private void areaBWarning(ParserRuleContext ctx) { final int start = ctx.getStart().getTokenIndex(); final int stop = ctx.getStop().getTokenIndex(); - - areaBWarning( - start < stop ? tokenStream.getTokens(start, stop) : ImmutableList.of(ctx.getStart())); + areaBWarning(start < stop ? tokenStream.getTokens(start, stop) : ImmutableList.of(ctx.getStart())); } private void areaBWarning(@NonNull List tokenList) { diff --git a/server/engine/src/main/java/org/eclipse/lsp/cobol/core/visitor/CobolVisitor.java b/server/engine/src/main/java/org/eclipse/lsp/cobol/core/visitor/CobolVisitor.java index 29bb992b98..b732d26af9 100644 --- a/server/engine/src/main/java/org/eclipse/lsp/cobol/core/visitor/CobolVisitor.java +++ b/server/engine/src/main/java/org/eclipse/lsp/cobol/core/visitor/CobolVisitor.java @@ -25,10 +25,12 @@ import static org.eclipse.lsp.cobol.core.visitor.VisitorHelper.*; import com.google.common.collect.ImmutableList; + import java.util.*; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; + import lombok.Getter; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -296,12 +298,12 @@ public List visitXmlParseStatement(XmlParseStatementContext ctx) { .orElse(null); ProcedureName procName = parseProcedureName(Optional.ofNullable(ctx.xmlProcessinProcedure()) - .map(XmlProcessinProcedureContext::procedureName) - .orElse(null)); + .map(XmlProcessinProcedureContext::procedureName) + .orElse(null)); ProcedureName thru = parseProcedureName(Optional.ofNullable(ctx.through()) - .map(ThroughContext::procedureName) - .orElse(null)); + .map(ThroughContext::procedureName) + .orElse(null)); return addTreeNode( ctx, @@ -315,7 +317,7 @@ public List visitXmlParseStatement(XmlParseStatementContext ctx) { xmlNationalContext, procName, thru) - ); + ); } @Override @@ -331,10 +333,10 @@ public List visitNotOnExceptionClause(CobolParser.NotOnExceptionClauseCont @Override public List visitUseStatement(CobolParser.UseStatementContext ctx) { return ctx.useDebugClause() != null - ? - addTreeNode(ctx, l -> new UseForDebuggingNode(l, null)) - : - addTreeNode(ctx, UseNode::new); + ? + addTreeNode(ctx, l -> new UseForDebuggingNode(l, null)) + : + addTreeNode(ctx, UseNode::new); } /** @@ -430,7 +432,7 @@ public List visitXmlGenerate(XmlGenerateContext ctx) { identifier6, identifier7, identifier8 - )); + )); } @@ -757,8 +759,8 @@ public List visitPerformInlineStatement(PerformInlineStatementContext ctx) public List visitPerformProcedureStatement(PerformProcedureStatementContext ctx) { final ProcedureName targetName = parseProcedureName(ctx.procedureName()); final ProcedureName thruName = parseProcedureName(Optional.ofNullable(ctx.through()) - .map(ThroughContext::procedureName) - .orElse(null)); + .map(ThroughContext::procedureName) + .orElse(null)); return addTreeNode(ctx, locality -> new PerformNode(locality, targetName, thruName)); } @@ -1055,8 +1057,8 @@ public List visitProcedureDivisionBody(ProcedureDivisionBodyContext ctx) { List children = visitChildren(ctx); return retrieveLocality(context) - .map(constructNode(ProcedureDivisionBodyNode::new, children)) - .orElse(children); + .map(constructNode(ProcedureDivisionBodyNode::new, children)) + .orElse(children); } @Override @@ -1161,17 +1163,17 @@ public List visitSortStatement(CobolParser.SortStatementContext ctx) { String key; if (ctx.sortOnKeyClause() != null) { ascending = ctx.sortOnKeyClause().stream() - .map(s -> s.ASCENDING() != null) - .filter(b -> b) - .findAny() - .orElse(false); + .map(s -> s.ASCENDING() != null) + .filter(b -> b) + .findAny() + .orElse(false); key = ctx.sortOnKeyClause().stream() - .map(SortOnKeyClauseContext::KEY) - .filter(Objects::nonNull) - .map(ParseTree::getText) - .findFirst() - .orElse(""); + .map(SortOnKeyClauseContext::KEY) + .filter(Objects::nonNull) + .map(ParseTree::getText) + .findFirst() + .orElse(""); } else { ascending = false; key = ""; @@ -1186,17 +1188,17 @@ public List visitMergeStatement(CobolParser.MergeStatementContext ctx) { String key; if (ctx.mergeOnKeyClause() != null) { ascending = ctx.mergeOnKeyClause().stream() - .map(s -> s.ASCENDING() != null) - .filter(b -> b) - .findAny() - .orElse(false); + .map(s -> s.ASCENDING() != null) + .filter(b -> b) + .findAny() + .orElse(false); key = ctx.mergeOnKeyClause().stream() - .map(MergeOnKeyClauseContext::KEY) - .filter(Objects::nonNull) - .map(ParseTree::getText) - .findFirst() - .orElse(""); + .map(MergeOnKeyClauseContext::KEY) + .filter(Objects::nonNull) + .map(ParseTree::getText) + .findFirst() + .orElse(""); } else { ascending = false; key = ""; @@ -1209,8 +1211,8 @@ public List visitMergeStatement(CobolParser.MergeStatementContext ctx) { public List visitInputProcedurePhrase(CobolParser.InputProcedurePhraseContext ctx) { final ProcedureName procName = parseProcedureName(ctx.procedureName()); final ProcedureName thru = parseProcedureName(Optional.ofNullable(ctx.through()) - .map(ThroughContext::procedureName) - .orElse(null)); + .map(ThroughContext::procedureName) + .orElse(null)); Locality locality = retrieveLocality(ctx).orElse(null); InputNode node = new InputNode(locality); @@ -1225,8 +1227,8 @@ public List visitInputProcedurePhrase(CobolParser.InputProcedurePhraseCont public List visitOutputProcedurePhrase(CobolParser.OutputProcedurePhraseContext ctx) { final ProcedureName procName = parseProcedureName(ctx.procedureName()); final ProcedureName thru = parseProcedureName(Optional.ofNullable(ctx.through()) - .map(ThroughContext::procedureName) - .orElse(null)); + .map(ThroughContext::procedureName) + .orElse(null)); Locality locality = retrieveLocality(ctx).orElse(null); OutputNode node = new OutputNode(locality); @@ -1237,7 +1239,8 @@ public List visitOutputProcedurePhrase(CobolParser.OutputProcedurePhraseCo return ImmutableList.of(node); } - @Override public List visitAlterStatement(CobolParser.AlterStatementContext ctx) { + @Override + public List visitAlterStatement(CobolParser.AlterStatementContext ctx) { if (ctx.alterProceedTo() != null && ctx.alterProceedTo().size() > 0) { CobolParser.AlterProceedToContext alter = ctx.alterProceedTo().get(0); if (alter.procedureName() != null && alter.procedureName().size() == 2) { @@ -1246,14 +1249,14 @@ public List visitOutputProcedurePhrase(CobolParser.OutputProcedurePhraseCo String name = Optional.ofNullable(from.paragraphName()).map(RuleContext::getText).orElse(null); String inSection = Optional.ofNullable(from.inSection()) - .map(InSectionContext::sectionName) - .map(RuleContext::getText).orElse(null); + .map(InSectionContext::sectionName) + .map(RuleContext::getText).orElse(null); ProcedureName alterFrom = new ProcedureName(name, inSection); name = Optional.ofNullable(to.paragraphName()).map(RuleContext::getText).orElse(null); inSection = Optional.ofNullable(to.inSection()) - .map(InSectionContext::sectionName) - .map(RuleContext::getText).orElse(null); + .map(InSectionContext::sectionName) + .map(RuleContext::getText).orElse(null); ProcedureName alterTo = new ProcedureName(name, inSection); Locality locality = retrieveLocality(ctx).orElse(null); @@ -1356,8 +1359,7 @@ protected void areaBWarning(ParserRuleContext ctx) { final int start = ctx.getStart().getTokenIndex(); final int stop = ctx.getStop().getTokenIndex(); - areaBWarning( - start < stop ? tokenStream.getTokens(start, stop) : ImmutableList.of(ctx.getStart())); + areaBWarning(start < stop ? tokenStream.getTokens(start, stop) : ImmutableList.of(ctx.getStart())); } private void areaBWarning(@NonNull List tokenList) { @@ -1379,11 +1381,11 @@ private Predicate startsInAreaA(Token token) { int charPosition = it.getRange().getStart().getCharacter(); int areaBStartIndex = programLayout.getSequenceLength() - + programLayout.getIndicatorLength() - + programLayout.getAreaALength(); + + programLayout.getIndicatorLength() + + programLayout.getAreaALength(); return charPosition > programLayout.getSequenceLength() - && charPosition < areaBStartIndex - && token.getChannel() != HIDDEN; + && charPosition < areaBStartIndex + && token.getChannel() != HIDDEN; }; } diff --git a/server/engine/src/test/java/org/eclipse/lsp/cobol/positive/PositiveTest.java b/server/engine/src/test/java/org/eclipse/lsp/cobol/positive/PositiveTest.java index 42ea17387d..41c1dbcedf 100644 --- a/server/engine/src/test/java/org/eclipse/lsp/cobol/positive/PositiveTest.java +++ b/server/engine/src/test/java/org/eclipse/lsp/cobol/positive/PositiveTest.java @@ -14,21 +14,42 @@ */ package org.eclipse.lsp.cobol.positive; +import static java.lang.System.getProperty; +import static java.util.Collections.emptyList; +import static java.util.Optional.ofNullable; +import static java.util.stream.Collectors.toList; import static org.eclipse.lsp.cobol.common.copybook.CopybookProcessingMode.ENABLED; +import static org.eclipse.lsp.cobol.positive.CobolTextRegistry.DEFAULT_LISTING_PATH; +import static org.eclipse.lsp.cobol.positive.CobolTextRegistry.PATH_TO_LISTING_SNAP; +import static org.junit.jupiter.api.Assertions.assertEquals; -import java.util.List; -import java.util.Map; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; import lombok.extern.slf4j.Slf4j; +import org.eclipse.lsp.cobol.ConfigurableTest; import org.eclipse.lsp.cobol.common.AnalysisConfig; import org.eclipse.lsp.cobol.common.AnalysisResult; +import org.eclipse.lsp.cobol.common.copybook.SQLBackend; import org.eclipse.lsp.cobol.test.CobolText; import org.eclipse.lsp.cobol.test.engine.UseCase; import org.eclipse.lsp.cobol.test.engine.UseCaseUtils; +import org.eclipse.lsp.cobol.usecases.DialectConfigs; +import org.eclipse.lsp4j.Diagnostic; +import org.eclipse.lsp4j.DiagnosticSeverity; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.params.provider.NullSource; /** * This class provides capability to run the server for actual cobol files that are provided using @@ -36,44 +57,203 @@ * regressions. The complete error description with the file name logged. */ @Slf4j -class PositiveTest extends FileBasedTest { +class PositiveTest extends ConfigurableTest { private CobolTextRegistry cobolTextRegistry; - @ParameterizedTest - @MethodSource("getSourceFolder") + @ParameterizedTest(name = "[{index}] run {1} in {2}") + @MethodSource("getTestFiles") @DisplayName("Parameterized - positive tests") - @NullSource - void test(String testFolder) { - LOG.info("-- {} under test --", testFolder); - cobolTextRegistry = retrieveTextsRegistry(testFolder); - List textsToTest = getTextsToTest(cobolTextRegistry); - for (CobolText text : textsToTest) { - if (text == null) { - return; - } - String fileName = text.getFileName(); - Map> dataNameRefs = - getDataNameRefs(fileName, cobolTextRegistry); - LOG.debug("Processing: " + fileName); - AnalysisConfig analysisConfig = getAnalysisConfiguration(cobolTextRegistry.getDialect()); - UseCase useCase = - UseCase.builder() - .documentUri(fileName) - .text(text.getFullText()) - .copybooks(getFileSpecificCopybooks(cobolTextRegistry, fileName)) - .copybookProcessingMode(ENABLED) - .dialects(analysisConfig.getDialects()) - .dialectsSettings(analysisConfig.getDialectsSettings()) - .build(); - AnalysisResult analyze = UseCaseUtils.analyze(useCase); - PositiveTestUtility.assetDefinitionsNReferencesFromSnap(analyze.getSymbolTableMap(), dataNameRefs, analyze.getRootNode(), fileName); - assertNoError(fileName, analyze); - } + void test(CobolTextRegistry cobolTextRegistry, CobolText text, String folderName) { + LOG.info("-- {} under test --", text.getFileName()); + this.cobolTextRegistry = cobolTextRegistry; + String fileName = text.getFileName(); + Map> dataNameRefs = + getDataNameRefs(fileName, cobolTextRegistry); + LOG.debug("Processing: " + fileName); + AnalysisConfig analysisConfig = getAnalysisConfiguration(cobolTextRegistry.getDialect()); + UseCase useCase = + UseCase.builder() + .documentUri(fileName) + .text(text.getFullText()) + .copybooks(getTestFileSpecificCopybooks(cobolTextRegistry, fileName)) + .copybookProcessingMode(ENABLED) + .dialects(analysisConfig.getDialects()) + .dialectsSettings(analysisConfig.getDialectsSettings()) + .build(); + AnalysisResult analyze = UseCaseUtils.analyze(useCase); + PositiveTestUtility.assetDefinitionsNReferencesFromSnap(analyze.getSymbolTableMap(), dataNameRefs, analyze.getRootNode(), fileName); + assertNoError(fileName, analyze); } @AfterEach void check() { updateSnaps(cobolTextRegistry); } + + static Stream getTestFiles() { + String path = ofNullable(getProperty(PATH_TO_TEST_RESOURCES)).orElse("../../tests/test_files"); + return searchFolderToTest(ImmutableList.of(Paths.get(path).toFile())).stream().map(p -> p.toAbsolutePath().toString()) + .flatMap(folder -> { + CobolTextRegistry cobolTextRegistry = retrieveTextsRegistry(folder); + List textsToTest = getTextsToTest(cobolTextRegistry); + return textsToTest.stream().map(file -> Arguments.of(cobolTextRegistry, file, new File(folder).getName())); + }); + } + + private static List searchFolderToTest(List files) { + List paths = new ArrayList<>(); + for (File file : files) { + if (file.isHidden()) continue; + if (file.isDirectory()) { + if (isValidTestFolder(file)) { + paths.add(file.toPath()); + } + paths.addAll(searchFolderToTest(listFiles(file))); + } + } + return paths; + } + + private static List listFiles(File file) { + return Optional.ofNullable(file.listFiles()) + .map(Arrays::stream) + .orElse(Stream.empty()) + .collect(Collectors.toList()); + } + + private static boolean isValidTestFolder(File file) { + return Optional.ofNullable(file.listFiles()) + .map(Arrays::stream) + .orElse(Stream.empty()) + .anyMatch(f -> f.getName().equals("positive")); + } + + + /** + * Get the files to be analyzed by Language Server from {@link CobolTextRegistry} using file-based + * implementation. + * + * @return the list of objects that would be passed to the constructor one by one + */ + protected static List getTextsToTest(CobolTextRegistry textRegistry) { + return textRegistry.getPositives(); + } + + /** + * Get the copybooks to be passed to the Language Server while analyzing from {@link + * CobolTextRegistry} using file-based implementation. + * + * @return the list of all defined copybooks + */ + protected static List getCopybooks(CobolTextRegistry textRegistry) { + return textRegistry.getCopybooks(); + } + + /** + * Get the copybooks to be passed to the Language Server while analyzing from {@link + * CobolTextRegistry} using file-based implementation. + * + * @return the list of all defined copybooks + */ + protected static List getCopybooks(CobolTextRegistry textRegistry, String filename) { + return textRegistry.getCopybooks(filename); + } + + protected static Map> getDataNameRefs( + String filename, CobolTextRegistry textRegistry) { + return textRegistry.getSnapForFile(filename); + } + + /** + * Check that there were no errors found. + * + * @param diagnostics list of diagnostic from the analysis result + * @param fileName name of the file that has been tested + */ + protected void assertNoSyntaxErrorsFound(List diagnostics, String fileName) { + assertEquals(0, diagnostics.size(), createErrorMessage(diagnostics, fileName)); + } + + private String createErrorMessage(List diagnostics, String fileName) { + StringBuilder result = new StringBuilder(fileName); + result.append(" contains syntax errors:\r\n"); + diagnostics.forEach( + it -> { + result.append(it.getRange().getStart().getLine() + 1); + result.append(":"); + result.append(it.getRange().getStart().getCharacter()); + result.append(" - "); + result.append(it.getRange().getEnd().getLine() + 1); + result.append(":"); + result.append(it.getRange().getEnd().getCharacter()); + result.append(" : "); + result.append(it.getMessage()); + result.append("\r\n"); + }); + return result.toString(); + } + + void updateSnaps(CobolTextRegistry cobolTextRegistry) { + String listingSnap = ofNullable(getProperty(PATH_TO_LISTING_SNAP)).orElse(DEFAULT_LISTING_PATH); + String updateFlag = ofNullable(getProperty("UpdateSnapListing")).orElse("false"); + if (Files.exists(Paths.get(listingSnap)) && !updateFlag.equals("false")) { + FolderTextRegistry textRegistry = (FolderTextRegistry) cobolTextRegistry; + textRegistry.createListingSnap(textRegistry.getSnaps(), textRegistry.getTestResourcePath()); + } + } + + void assertNoError(String fileName, AnalysisResult analyze) { + List diagnostic = + ofNullable(analyze.getDiagnostics().get(fileName)) + .map( + diagnostics -> + diagnostics.stream() + .filter(it -> it.getSeverity() == DiagnosticSeverity.Error) + .collect(toList())) + .orElse(emptyList()); + assertNoSyntaxErrorsFound(diagnostic, fileName); + } + + /** + * Get the copybooks specific to a cobol file, to be passed to the Language Server while analyzing + * from {@link CobolTextRegistry} using file-based implementation. + * + * @param cobolTextRegistry {@link CobolTextRegistry} + * @param fileName file to be analysed + * @return List of copybooks + */ + protected List getTestFileSpecificCopybooks( + CobolTextRegistry cobolTextRegistry, String fileName) { + Stream cobolTextStream = + getCopybooks(cobolTextRegistry).stream() + .filter( + book -> + getCopybooks(cobolTextRegistry, fileName.split("\\.")[0]).stream() + .noneMatch(b1 -> b1.getFileName().equals(book.getFileName()))); + return Stream.concat( + cobolTextStream, getCopybooks(cobolTextRegistry, fileName.split("\\.")[0]).stream()) + .collect(toList()); + } + + /** + * Returns a {@link AnalysisConfig} based on passed system variables + * + * @return {@link AnalysisConfig} + */ + protected AnalysisConfig getAnalysisConfiguration(String testDialectsLists) { + if (testDialectsLists.contains("DaCo")) { + return DialectConfigs.getDaCoAnalysisConfig(); + } + + if (testDialectsLists.contains("IDMS")) { + return new AnalysisConfig( + ENABLED, + ImmutableList.of("IDMS"), + true, + ImmutableList.of(), + ImmutableMap.of("target-sql-backend", new Gson().toJsonTree(SQLBackend.DB2_SERVER))); + } + return AnalysisConfig.defaultConfig(ENABLED); + } } diff --git a/server/engine/src/test/java/org/eclipse/lsp/cobol/positive/PositiveTestOld.java b/server/engine/src/test/java/org/eclipse/lsp/cobol/positive/PositiveTestOld.java new file mode 100644 index 0000000000..25ed3fd9d3 --- /dev/null +++ b/server/engine/src/test/java/org/eclipse/lsp/cobol/positive/PositiveTestOld.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2024 Broadcom. + * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Broadcom, Inc. - initial API and implementation + * + */ +package org.eclipse.lsp.cobol.positive; + +import lombok.extern.slf4j.Slf4j; +import org.eclipse.lsp.cobol.common.AnalysisConfig; +import org.eclipse.lsp.cobol.common.AnalysisResult; +import org.eclipse.lsp.cobol.test.CobolText; +import org.eclipse.lsp.cobol.test.engine.UseCase; +import org.eclipse.lsp.cobol.test.engine.UseCaseUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.NullSource; + +import java.util.List; +import java.util.Map; + +import static org.eclipse.lsp.cobol.common.copybook.CopybookProcessingMode.ENABLED; + +/** + * This class provides capability to run the server for actual cobol files that are provided using + * {@link CobolTextRegistry}. The positive test should always pass. If not, then there are some + * regressions. The complete error description with the file name logged. + */ +@Slf4j +@Disabled("To be removed if PositiveTest are good enough.") +class PositiveTestOld extends FileBasedTest { + + private CobolTextRegistry cobolTextRegistry; + + @ParameterizedTest + @MethodSource("getSourceFolder") + @DisplayName("Parameterized - positive tests") + @NullSource + void test(String testFolder) { + LOG.info("-- {} under test --", testFolder); + cobolTextRegistry = retrieveTextsRegistry(testFolder); + List textsToTest = getTextsToTest(cobolTextRegistry); + for (CobolText text : textsToTest) { + if (text == null) { + return; + } + String fileName = text.getFileName(); + Map> dataNameRefs = + getDataNameRefs(fileName, cobolTextRegistry); + LOG.debug("Processing: " + fileName); + AnalysisConfig analysisConfig = getAnalysisConfiguration(cobolTextRegistry.getDialect()); + UseCase useCase = + UseCase.builder() + .documentUri(fileName) + .text(text.getFullText()) + .copybooks(getFileSpecificCopybooks(cobolTextRegistry, fileName)) + .copybookProcessingMode(ENABLED) + .dialects(analysisConfig.getDialects()) + .dialectsSettings(analysisConfig.getDialectsSettings()) + .build(); + AnalysisResult analyze = UseCaseUtils.analyze(useCase); + PositiveTestUtility.assetDefinitionsNReferencesFromSnap(analyze.getSymbolTableMap(), dataNameRefs, analyze.getRootNode(), fileName); + assertNoError(fileName, analyze); + } + } + + @AfterEach + void check() { + updateSnaps(cobolTextRegistry); + } +} diff --git a/server/parser/src/main/antlr4/org/eclipse/lsp/cobol/core/parser/CobolProcedureDivisionLexer.g4 b/server/parser/src/main/antlr4/org/eclipse/lsp/cobol/core/parser/CobolProcedureDivisionLexer.g4 index f1f13984f2..976f5c8b9d 100644 --- a/server/parser/src/main/antlr4/org/eclipse/lsp/cobol/core/parser/CobolProcedureDivisionLexer.g4 +++ b/server/parser/src/main/antlr4/org/eclipse/lsp/cobol/core/parser/CobolProcedureDivisionLexer.g4 @@ -392,8 +392,8 @@ COMMENTLINE : COMMENTTAG ~('\n' | '\r')* -> channel(COMMENTS); WS : [ \t\f]+ -> channel(HIDDEN); COMPILERLINE : DOUBLEMORETHANCHAR ~('\n' | '\r')* -> channel(HIDDEN); // period full stopPosition -DOT : '.'; DOT_FS : '.' EOF?; +DOT : '.'; LEVEL_NUMBER : ([1-9])|([0][1-9])|([1234][0-9]); LEVEL_NUMBER_66 : '66'; diff --git a/server/parser/src/main/java/org/eclipse/lsp/cobol/parser/hw/CobolLexer.java b/server/parser/src/main/java/org/eclipse/lsp/cobol/parser/hw/CobolLexer.java index 362929d91a..90ad9d9450 100644 --- a/server/parser/src/main/java/org/eclipse/lsp/cobol/parser/hw/CobolLexer.java +++ b/server/parser/src/main/java/org/eclipse/lsp/cobol/parser/hw/CobolLexer.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Predicate; /** @@ -37,6 +38,7 @@ public CobolLexer(String source) { /** * Check if we have more tokens. + * * @return true if there are more tokens. */ public boolean hasMore() { @@ -45,6 +47,7 @@ public boolean hasMore() { /** * Return one token and move forward. + * * @param rule current syntax rule. * @return list of possible tokens (at least one). */ @@ -59,6 +62,7 @@ public List forward(GrammarRule rule) { /** * Return one token but don't move forward. + * * @param rule current syntax rule. * @return list of possible tokens (at least one). */ @@ -80,23 +84,66 @@ private TokenType detectType(GrammarRule rule, String lexeme) { private String scan(GrammarRule rule, boolean look) { if (isWhitespace(source.charAt(position.getIndex()))) { - return consumeUntil(c -> !isWhitespace(c), look); + return consumeUntil(c -> !isWhitespace(source.charAt(c.getIndex())), look); } if (isNewLine(source.charAt(position.getIndex()))) { - return consumeUntil(c -> !isNewLine(c), look); + return consumeUntil(c -> !isNewLine(source.charAt(c.getIndex())), look); } if (isSeparator(source.charAt(position.getIndex()))) { - return consumeUntil(c -> !isSeparator(c), look); + return consumeUntil(c -> !isSeparator(source.charAt(c.getIndex())), look); } - return consumeUntil(c -> isSeparator(c) || isWhitespace(c) || isNewLine(c), look); + if (isLiteral(source.charAt(position.getIndex()))) { + return processLiteral(source, look); + } + return consumeUntil(c -> isSeparator(source.charAt(c.getIndex())) + || isWhitespace(source.charAt(c.getIndex())) + || isNewLine(source.charAt(c.getIndex())), look); + } + + private String processLiteral(String source, boolean look) { + // https://www.ibm.com/docs/en/cobol-zos/6.4?topic=literals-basic-alphanumeric + char quoteSymbol = source.charAt(position.getIndex()); + + // Those flags are to include quoteSymbol into the lexeme. + AtomicBoolean isFirst = new AtomicBoolean(true); + AtomicBoolean isLast = new AtomicBoolean(false); + + AtomicBoolean wasEscaped = new AtomicBoolean(false); + + return consumeUntil(pos -> { + if (isFirst.get()) { + isFirst.set(false); + return false; + } + if (pos.getIndex() + 1 < source.length() && source.charAt(pos.getIndex()) == quoteSymbol && source.charAt(pos.getIndex() + 1) == quoteSymbol) { + wasEscaped.set(true); + return false; + } + if (wasEscaped.get()) { + wasEscaped.set(false); + return false; + } + if (isLast.get()) { + return true; + } + if (source.charAt(pos.getIndex()) != quoteSymbol) { + return false; + } + + if (source.charAt(pos.getIndex()) == quoteSymbol) { + isLast.set(true); + return false; + } + return true; + }, look); } - private String consumeUntil(Predicate predicate, boolean look) { + private String consumeUntil(Predicate predicate, boolean look) { StringBuilder sb = new StringBuilder(); Position localPos = new Position(position); while (localPos.getIndex() < source.length()) { char charAt = source.charAt(localPos.getIndex()); - if (predicate.test(charAt)) { + if (predicate.test(localPos)) { break; } sb.append(charAt); @@ -113,6 +160,10 @@ private boolean isSeparator(char c) { return c == '.'; } + private boolean isLiteral(char c) { + return c == '"' || c == '\''; + } + private boolean isWhitespace(char c) { return c == ' ' || c == '\t'; } @@ -123,9 +174,10 @@ private boolean isNewLine(char c) { /** * Find a sequence of tokens ignoring one that passes skip predicate. - * @param rule current syntax rule. + * + * @param rule current syntax rule. * @param count how many tokens to take. - * @param skip predicate to ignore tokens. + * @param skip predicate to ignore tokens. * @return a list of tokens. */ // TODO: it should return a list of lists @@ -158,8 +210,10 @@ class Position { line = position.line; character = position.character; } + Position() { } + void updateLinePosition(char charAt) { index++; if (charAt == '\n') { diff --git a/server/parser/src/main/java/org/eclipse/lsp/cobol/parser/hw/antlradapter/AntlrAdapter.java b/server/parser/src/main/java/org/eclipse/lsp/cobol/parser/hw/antlradapter/AntlrAdapter.java index 9884a72df1..29c6ec11b5 100644 --- a/server/parser/src/main/java/org/eclipse/lsp/cobol/parser/hw/antlradapter/AntlrAdapter.java +++ b/server/parser/src/main/java/org/eclipse/lsp/cobol/parser/hw/antlradapter/AntlrAdapter.java @@ -98,20 +98,6 @@ private CobolParser.StartRuleContext convertSourceUnit(SourceUnit cstNode) { return start; } - private AntlrAdapted findAntlrNode(CstNode cstNode) { - for (CstNode node : cstNode.getChildren()) { - if (node instanceof AntlrAdapted) { - return (AntlrAdapted) node; - } else { - AntlrAdapted a = findAntlrNode(node); - if (a != null) { - return a; - } - } - } - return null; - } - private CobolParser.ProgramUnitContext convertProgramNode(CstNode programUnit) { CobolParser.ProgramUnitContext program = new CobolParser.ProgramUnitContext(null, 0); Utils.initNode(programUnit, program, charStream); @@ -138,17 +124,25 @@ private ParserRuleContext convertNode(CstNode cstNode) { // All programs should be adapted to this point return ((AntlrAdapted) cstNode.getChildren().get(0)).getRuleContext(); } else if (cstNode instanceof DataDivision) { - return antlrDataDivisionParser(cstNode).dataDivision(); + CobolDataDivisionParser.DataDivisionContext dataDivisionContext = antlrDataDivisionParser(cstNode).dataDivision(); + Utils.removeEofNode(dataDivisionContext); + return dataDivisionContext; } else if (cstNode instanceof IdentificationDivision) { - return antlrIdDivisionParser(cstNode).identificationDivision(); + CobolIdentificationDivisionParser.IdentificationDivisionContext identificationDivisionContext = antlrIdDivisionParser(cstNode).identificationDivision(); + Utils.removeEofNode(identificationDivisionContext); + return identificationDivisionContext; } else if (cstNode instanceof EnvironmentDivision) { - return antlrParser(cstNode).environmentDivision(); + CobolParser.EnvironmentDivisionContext environmentDivisionContext = antlrParser(cstNode).environmentDivision(); + Utils.removeEofNode(environmentDivisionContext); + return environmentDivisionContext; } else if (cstNode instanceof ProcedureDivision) { ProcedureDivisionAntlrAdapter adapter = new ProcedureDivisionAntlrAdapter(charStream, errorListener, errorStrategy, treeListener); - return adapter.processProcedureDivisionContext((ProcedureDivision) cstNode); + CobolProcedureDivisionParser.ProcedureDivisionContext procedureDivisionContext = adapter.processProcedureDivisionContext((ProcedureDivision) cstNode); + Utils.removeEofNode(procedureDivisionContext); + return procedureDivisionContext; } else { return null; } @@ -175,7 +169,6 @@ private CobolIdentificationDivisionParser antlrIdDivisionParser(CstNode node) { CobolIdentificationDivisionParser antlrParser = new CobolIdentificationDivisionParser(tokens); antlrParser.removeErrorListeners(); antlrParser.addErrorListener(errorListener); - antlrParser.setErrorHandler(errorStrategy); antlrParser.addParseListener(treeListener); return antlrParser; } @@ -220,7 +213,6 @@ public CommonTokenStream adaptTokens(SourceUnit su) { List tokens = new ArrayList<>(); collectTokens(su, tokens); CommonTokenStream commonTokenStream = new CommonTokenStream(new ListTokenSource(tokens.stream() - .filter(t -> ((org.eclipse.lsp.cobol.parser.hw.Token) t).getType() != TokenType.WHITESPACE) .map(org.eclipse.lsp.cobol.parser.hw.Token.class::cast) .map(token -> Utils.toAntlrToken(token, charStream)).collect(Collectors.toList()))); commonTokenStream.fill(); diff --git a/server/parser/src/main/java/org/eclipse/lsp/cobol/parser/hw/antlradapter/ProcedureDivisionAntlrAdapter.java b/server/parser/src/main/java/org/eclipse/lsp/cobol/parser/hw/antlradapter/ProcedureDivisionAntlrAdapter.java index 86d780b2bc..19406881c7 100644 --- a/server/parser/src/main/java/org/eclipse/lsp/cobol/parser/hw/antlradapter/ProcedureDivisionAntlrAdapter.java +++ b/server/parser/src/main/java/org/eclipse/lsp/cobol/parser/hw/antlradapter/ProcedureDivisionAntlrAdapter.java @@ -57,12 +57,6 @@ public ProcedureDivisionAntlrAdapter( ProcedureDivisionContext processProcedureDivisionContext(ProcedureDivision cstNode) { ProcedureDivisionContext pdCtx = parseProcedureDivision(cstNode); createProcedureDivisionBodyContext(pdCtx, cstNode); - // Remove EOF token if eny - ParseTree lastChild = pdCtx.children.get(pdCtx.children.size() - 1); - if (lastChild instanceof TerminalNodeImpl - && ((TerminalNodeImpl) lastChild).getSymbol().getType() == org.antlr.v4.runtime.Token.EOF) { - pdCtx.removeLastChild(); - } return pdCtx; } @@ -82,7 +76,7 @@ private ProcedureDivisionBodyContext createProcedureDivisionBodyContext(Procedur LinkedList genStack = new LinkedList<>(); genStack.push(pdbCtx); - // make a free flat + // make a tree flat List nodes = cstNode.getChildren().stream() .flatMap(m -> (m instanceof Section) ? Stream.concat(Stream.of(m), m.getChildren().stream()) : Stream.of(m)) .flatMap(m -> (m instanceof Paragraph) ? Stream.concat(Stream.of(m), m.getChildren().stream()) : Stream.of(m)) @@ -143,7 +137,9 @@ private void handleDeclaratives(LinkedList genStack, Declarat antlrParser.addErrorListener(errorListener); antlrParser.setErrorHandler(errorStrategy); antlrParser.addParseListener(treeListener); - genStack.peek().addChild(antlrParser.procedureDeclaratives()); + ProcedureDeclarativesContext declaratives = antlrParser.procedureDeclaratives(); + Utils.removeEofNode(declaratives); + genStack.peek().addChild(declaratives); } private void assureParagraphsCtx(LinkedList genStack, ProcedureDivisionBodyContext pdbCtx, CstNode node) { @@ -225,7 +221,9 @@ private ParserRuleContext parseSentence(Statement node) { antlrParser.addErrorListener(errorListener); antlrParser.setErrorHandler(errorStrategy); antlrParser.addParseListener(treeListener); - return antlrParser.sentence(); + SentenceContext sentence = antlrParser.sentence(); + Utils.removeEofNode(sentence); + return sentence; } private ProcedureDivisionContext parseProcedureDivision(ProcedureDivision node) { @@ -245,7 +243,6 @@ private ProcedureDivisionContext parseProcedureDivision(ProcedureDivision node) CobolProcedureDivisionParser antlrParser = new CobolProcedureDivisionParser(tokens); antlrParser.removeErrorListeners(); antlrParser.addErrorListener(errorListener); - antlrParser.setErrorHandler(errorStrategy); antlrParser.addParseListener(treeListener); return antlrParser.procedureDivision(); } diff --git a/server/parser/src/main/java/org/eclipse/lsp/cobol/parser/hw/antlradapter/Utils.java b/server/parser/src/main/java/org/eclipse/lsp/cobol/parser/hw/antlradapter/Utils.java index 2bec254621..539504e70b 100644 --- a/server/parser/src/main/java/org/eclipse/lsp/cobol/parser/hw/antlradapter/Utils.java +++ b/server/parser/src/main/java/org/eclipse/lsp/cobol/parser/hw/antlradapter/Utils.java @@ -22,6 +22,8 @@ import org.antlr.v4.runtime.TokenSource; import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.misc.Pair; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.TerminalNodeImpl; import org.eclipse.lsp.cobol.cst.base.CstNode; import org.eclipse.lsp.cobol.parser.hw.Token; import org.eclipse.lsp.cobol.parser.hw.TokenType; @@ -32,28 +34,30 @@ import static org.antlr.v4.runtime.Token.HIDDEN_CHANNEL; -/** Token related utilities. */ +/** + * Token related utilities. + */ public class Utils { /** * Transform cst token into antlr one. * - * @param token cst token node. + * @param token cst token node. * @param charStream the stream of program characters. * @return antlr token. */ public static CommonToken toAntlrToken(Token token, CharStream charStream) { Pair source = new Pair<>(null, charStream); int channel = - token.getType() == TokenType.WHITESPACE || token.getType() == TokenType.NEW_LINE - ? HIDDEN_CHANNEL - : 0; + token.getType() == TokenType.WHITESPACE || token.getType() == TokenType.NEW_LINE + ? HIDDEN_CHANNEL + : 0; CommonToken commonToken = - new CommonToken( - source, - 0, - channel, - token.getIndex(), - token.getIndex() + token.getLexeme().length() - 1); + new CommonToken( + source, + 0, + channel, + token.getIndex(), + token.getIndex() + token.getLexeme().length() - 1); commonToken.setLine(token.getLine() + 1); commonToken.setCharPositionInLine(token.getStartPositionInLine()); commonToken.setText(token.toText()); @@ -63,7 +67,7 @@ public static CommonToken toAntlrToken(Token token, CharStream charStream) { /** * Find the first token in CST. * - * @param cstNode a CST node. + * @param cstNode a CST node. * @param ignoreWhitespaces set to true if you want to ignore whitespace tokens * @return Optionally token. */ @@ -90,17 +94,17 @@ public static Optional findStartToken(CstNode cstNode) { /** * Initialize antlr node by CST node. * - * @param cstNode the CST node. - * @param node tha ANTLR node to set up. + * @param cstNode the CST node. + * @param node tha ANTLR node to set up. * @param charStream the stream of program characters. - * @param the type of ANTLR node + * @param the type of ANTLR node * @return node reference. */ public static T initNode( - CstNode cstNode, T node, CharStream charStream) { + CstNode cstNode, T node, CharStream charStream) { node.start = findStartToken(cstNode).map(token -> toAntlrToken(token, charStream)).orElse(null); node.stop = - findStopToken(cstNode, true).map(token -> toAntlrToken(token, charStream)).orElse(null); + findStopToken(cstNode, true).map(token -> toAntlrToken(token, charStream)).orElse(null); node.children = new ArrayList<>(); return node; } @@ -121,7 +125,7 @@ public static Optional findStopToken(CstNode cstNode) { /** * Find the last token in CST. * - * @param cstNodes a CST node list. + * @param cstNodes a CST node list. * @param ignoreWhitespaces set to true if you want to ignore whitespace tokens * @return Optionally token. */ @@ -145,7 +149,7 @@ public static Optional findStopToken(List cstNodes, boolean igno /** * Find the last token in CST. * - * @param cstNode a CST node. + * @param cstNode a CST node. * @param ignoreWhitespaces set to true if you want to ignore whitespace tokens * @return Optionally token. */ @@ -175,7 +179,7 @@ private static Optional firstToken(List cstNodes, boolean ignore /** * Find the last token in CST. * - * @param cstNodes a list of nodes to check (ignore whitespaces) + * @param cstNodes a list of nodes to check (ignore whitespaces) * @param ignoreWhitespaces set to true if you want to ignore whitespace tokens * @return possible the end node. */ @@ -210,4 +214,20 @@ static String generatePrefix(CharStream charStream, Token startToken) { } return new String(chars); } + + /** + * Remove EOF token if eny + * + * @param parent a node to find a EOF token. + */ + static void removeEofNode(ParserRuleContext parent) { + if (parent.children == null) { + return; + } + ParseTree lastChild = parent.children.get(parent.children.size() - 1); + if (lastChild instanceof TerminalNodeImpl + && ((TerminalNodeImpl) lastChild).getSymbol().getType() == org.antlr.v4.runtime.Token.EOF) { + parent.removeLastChild(); + } + } } diff --git a/server/parser/src/main/java/org/eclipse/lsp/cobol/rules/procedure/DeclarativesRule.java b/server/parser/src/main/java/org/eclipse/lsp/cobol/rules/procedure/DeclarativesRule.java index c7406cf571..44bc07d3c0 100644 --- a/server/parser/src/main/java/org/eclipse/lsp/cobol/rules/procedure/DeclarativesRule.java +++ b/server/parser/src/main/java/org/eclipse/lsp/cobol/rules/procedure/DeclarativesRule.java @@ -37,6 +37,12 @@ public void parse(ParsingContext ctx, CobolLanguage language) { ctx.consume(); ctx.spaces(); } + ctx.spaces(); + ctx.consume("END"); + ctx.spaces(); + ctx.consume("DECLARATIVES"); + ctx.spaces(); + ctx.consume("."); } finally { ctx.popAndAttach(); } diff --git a/server/parser/src/main/java/org/eclipse/lsp/cobol/rules/procedure/ParagraphRule.java b/server/parser/src/main/java/org/eclipse/lsp/cobol/rules/procedure/ParagraphRule.java index d3ac96fd23..2a02cf2274 100644 --- a/server/parser/src/main/java/org/eclipse/lsp/cobol/rules/procedure/ParagraphRule.java +++ b/server/parser/src/main/java/org/eclipse/lsp/cobol/rules/procedure/ParagraphRule.java @@ -32,8 +32,8 @@ public void parse(ParsingContext ctx, CobolLanguage language) { ((Paragraph) ctx.peek()).setName(ctx.consume().get(0).getLexeme()); ctx.spaces(); ctx.consume("."); - ctx.spaces(); try { + ctx.spaces(); while (!CobolLanguageUtils.isNextDivisionEofOrEop(ctx) && !language.tryMatchRule(ParagraphRule.class, ctx) && !language.tryMatchRule(SectionRule.class, ctx) diff --git a/server/parser/src/main/java/org/eclipse/lsp/cobol/rules/procedure/ProcedureDivisionRule.java b/server/parser/src/main/java/org/eclipse/lsp/cobol/rules/procedure/ProcedureDivisionRule.java index 702d4e9674..1d23bd1f61 100644 --- a/server/parser/src/main/java/org/eclipse/lsp/cobol/rules/procedure/ProcedureDivisionRule.java +++ b/server/parser/src/main/java/org/eclipse/lsp/cobol/rules/procedure/ProcedureDivisionRule.java @@ -24,44 +24,46 @@ * COBOL language grammar rule class. */ public class ProcedureDivisionRule implements LanguageRule { - @Override - public void parse(ParsingContext ctx, CobolLanguage language) { + @Override + public void parse(ParsingContext ctx, CobolLanguage language) { + ctx.spaces(); + ctx.push(new ProcedureDivision()); + procedureDivisionHeader(ctx); + ((ProcedureDivision) ctx.peek()).setBodyStartToken(ctx.getLexer().peek(null).get(0)); + try { + if (language.tryParseRule(DeclarativesRule.class, ctx).isPresent()) { ctx.spaces(); - ctx.push(new ProcedureDivision()); - procedureDivisionHeader(ctx); - ((ProcedureDivision) ctx.peek()).setBodyStartToken(ctx.getLexer().peek(null).get(0)); - try { - while (!CobolLanguageUtils.isNextDivisionEofOrEop(ctx) && !CobolLanguageUtils.isEndOfProgram(ctx)) { - language.tryParseRule(DeclarativesRule.class, ctx); - if (language.tryMatchRule(ParagraphRule.class, ctx)) { - language.parseRule(ParagraphRule.class, ctx); - } else if (language.tryMatchRule(SectionRule.class, ctx)) { - language.tryParseRule(SectionRule.class, ctx); - } else { - language.tryParseRule(StatementRule.class, ctx); - } - ctx.spaces(); - } - } finally { - ctx.popAndAttach(); + } + while (!CobolLanguageUtils.isNextDivisionEofOrEop(ctx) && !CobolLanguageUtils.isEndOfProgram(ctx)) { + if (language.tryMatchRule(ParagraphRule.class, ctx)) { + language.parseRule(ParagraphRule.class, ctx); + } else if (language.tryMatchRule(SectionRule.class, ctx)) { + language.tryParseRule(SectionRule.class, ctx); + } else { + language.tryParseRule(StatementRule.class, ctx); } + ctx.spaces(); + } + } finally { + ctx.popAndAttach(); } + } - private void procedureDivisionHeader(ParsingContext ctx) { - ctx.consume("PROCEDURE"); - ctx.spaces(); - ctx.consume("DIVISION"); - ctx.spaces(); - while (!ctx.match(".")) { - ctx.consume(); - } - ctx.consume("."); - ctx.spaces(); + private void procedureDivisionHeader(ParsingContext ctx) { + ctx.consume("PROCEDURE"); + ctx.spaces(); + ctx.consume("DIVISION"); + ctx.spaces(); + while (!ctx.match(".")) { + ctx.consume(); } + ctx.consume("."); + ctx.spaces(); + } - @Override - public boolean tryMatch(ParsingContext ctx, CobolLanguage language) { - return ctx.matchSeq("PROCEDURE", "DIVISION"); - } + @Override + public boolean tryMatch(ParsingContext ctx, CobolLanguage language) { + return ctx.matchSeq("PROCEDURE", "DIVISION"); + } } diff --git a/server/parser/src/main/java/org/eclipse/lsp/cobol/rules/procedure/StatementRule.java b/server/parser/src/main/java/org/eclipse/lsp/cobol/rules/procedure/StatementRule.java index ca75e98115..3e9217b1c1 100644 --- a/server/parser/src/main/java/org/eclipse/lsp/cobol/rules/procedure/StatementRule.java +++ b/server/parser/src/main/java/org/eclipse/lsp/cobol/rules/procedure/StatementRule.java @@ -25,27 +25,27 @@ */ public class StatementRule implements LanguageRule { - @Override - public void parse(ParsingContext ctx, CobolLanguage language) { - ctx.push(new Statement()); - try { - while (!ctx.match(".") - && !CobolLanguageUtils.isNextDivisionEofOrEop(ctx) - && !language.tryMatchRule(ParagraphRule.class, ctx) - && !language.tryMatchRule(SectionRule.class, ctx) - && !CobolLanguageUtils.isEndOfProgram(ctx)) { - ctx.consume(); - } - ctx.optional("."); - } finally { - ctx.popAndAttach(); - } - + @Override + public void parse(ParsingContext ctx, CobolLanguage language) { + ctx.push(new Statement()); + try { + while (!ctx.match(".") + && !CobolLanguageUtils.isNextDivisionEofOrEop(ctx) + && !language.tryMatchRule(ParagraphRule.class, ctx) + && !language.tryMatchRule(SectionRule.class, ctx) + && !CobolLanguageUtils.isEndOfProgram(ctx)) { + ctx.consume(); + } + ctx.optional("."); + } finally { + ctx.popAndAttach(); } - @Override - public boolean tryMatch(ParsingContext ctx, CobolLanguage language) { - // For now, it can be anything. - return true; - } + } + + @Override + public boolean tryMatch(ParsingContext ctx, CobolLanguage language) { + // For now, it can be anything. + return true; + } } diff --git a/server/parser/src/test/java/org/eclipse/lsp/cobol/core/HwCobolLexerTest.java b/server/parser/src/test/java/org/eclipse/lsp/cobol/core/HwCobolLexerTest.java index f2c52089cc..d7d05ff438 100644 --- a/server/parser/src/test/java/org/eclipse/lsp/cobol/core/HwCobolLexerTest.java +++ b/server/parser/src/test/java/org/eclipse/lsp/cobol/core/HwCobolLexerTest.java @@ -56,6 +56,42 @@ void newLine() { assertFalse(lexer.hasMore()); } + @Test + void quotes() { + CobolLexer lexer = new CobolLexer("A '\n B'"); + assertToken(lexer.forward(GrammarRule.ProgramUnit).get(0), "A", 0, 0, 0); + assertToken(lexer.forward(GrammarRule.ProgramUnit).get(0), " ", 0, 1, 1); + assertToken(lexer.forward(GrammarRule.ProgramUnit).get(0), "'\n B'", 0, 2, 2); + assertFalse(lexer.hasMore()); + } + + @Test + void quotesEscape() { + CobolLexer lexer = new CobolLexer("A '''\n B'"); + assertToken(lexer.forward(GrammarRule.ProgramUnit).get(0), "A", 0, 0, 0); + assertToken(lexer.forward(GrammarRule.ProgramUnit).get(0), " ", 0, 1, 1); + assertToken(lexer.forward(GrammarRule.ProgramUnit).get(0), "'''\n B'", 0, 2, 2); + assertFalse(lexer.hasMore()); + } + + @Test + void doubleQuotes() { + CobolLexer lexer = new CobolLexer("A \"\n B\""); + assertToken(lexer.forward(GrammarRule.ProgramUnit).get(0), "A", 0, 0, 0); + assertToken(lexer.forward(GrammarRule.ProgramUnit).get(0), " ", 0, 1, 1); + assertToken(lexer.forward(GrammarRule.ProgramUnit).get(0), "\"\n B\"", 0, 2, 2); + assertFalse(lexer.hasMore()); + } + + @Test + void doubleQuotesEscape() { + CobolLexer lexer = new CobolLexer("A \"\"\"\n B\""); + assertToken(lexer.forward(GrammarRule.ProgramUnit).get(0), "A", 0, 0, 0); + assertToken(lexer.forward(GrammarRule.ProgramUnit).get(0), " ", 0, 1, 1); + assertToken(lexer.forward(GrammarRule.ProgramUnit).get(0), "\"\"\"\n B\"", 0, 2, 2); + assertFalse(lexer.hasMore()); + } + @Test void peekTest() { CobolLexer lexer = new CobolLexer("Aa\n B");