chore: Code refactor due to preparation for different COBOL dialects

server/common/src/main/java/org/eclipse/lsp/cobol/common/copybook/CopybookService.java

@@ -15,7 +15,12 @@
 package org.eclipse.lsp.cobol.common.copybook;
 
 import java.util.Collection;
+import java.util.Collections;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
 import lombok.NonNull;
+import org.eclipse.lsp.cobol.common.CleanerPreprocessor;
 import org.eclipse.lsp.cobol.common.ResultWithErrors;
 
 /**
@@ -27,6 +32,8 @@ public interface CopybookService {
   /** Remove all the stored copybook. */
   void invalidateCache();
 
+  void invalidateCache(CopybookId copybookId);
+
   /**
    * Retrieve and return the copybook by its name.
    * Returns a CopybookModel and preprocessed errors for the resolved copybook wrapped inside {@link ResultWithErrors}.
@@ -35,8 +42,7 @@ public interface CopybookService {
    * @param copybookName       - the name of the copybook to be retrieved
    * @param programDocumentUri - the currently processing program document
    * @param documentUri        - the currently processing document that contains the copy statement
-   * @param preprocess         - indicates if copybook needs to be preprocessed after resolving
-   * @param languageId
+   * @param preprocessor       - Cleanup preprocessor that will be used for new copybooks or null
    * @return a CopybookModel wrapped inside {@link ResultWithErrors} which contains copybook name, its URI and the content.
    * Wrapped errors are preprocessed errors for the returned CopybookModel.
    */
@@ -45,7 +51,7 @@ ResultWithErrors<CopybookModel> resolve(
           @NonNull CopybookName copybookName,
           @NonNull String programDocumentUri,
           @NonNull String documentUri,
-          boolean preprocess, String languageId);
+          CleanerPreprocessor preprocessor);
 
   /**
    * Store the copybookModel in cache. Copybook depends on a document from where it is imported.
@@ -58,10 +64,9 @@ ResultWithErrors<CopybookModel> resolve(
    * Store the copybookModel in cache. Copybook depends on a document from where it is imported.
    *
    * @param copybookModel the copybook model
-   * @param languageId languageId of the source code
-   * @param doCleanUp is copybook clean up required before storing copybookModel.
+   * @param preprocessor       - Cleanup preprocessor that will be used for new copybooks or null
    */
-  void store(CopybookModel copybookModel, String languageId, boolean doCleanUp);
+  void store(CopybookModel copybookModel, CleanerPreprocessor preprocessor);
 
   /**
    * Send downloading requests to the Client for copybooks not presented locally, if any.
@@ -71,4 +76,13 @@ ResultWithErrors<CopybookModel> resolve(
    * @param processingMode copybook processing mode.
    */
   void sendCopybookDownloadRequest(String documentUri, Collection<String> copybookUris, CopybookProcessingMode processingMode);
+
+  /**
+   * Get the list of copybook used by a document
+   *
+   * @param documentUri  current document uri.
+   * @return Set of all the {@link CopybookModel} used by the passed document
+   */
+  Set<CopybookModel> getCopybookUsage(String documentUri);
+
 }

server/common/src/main/java/org/eclipse/lsp/cobol/common/dialects/DialectProcessingContext.java

@@ -19,6 +19,7 @@
 import lombok.Builder;
 import lombok.Value;
 import org.eclipse.lsp.cobol.common.AnalysisConfig;
+import org.eclipse.lsp.cobol.common.CleanerPreprocessor;
 import org.eclipse.lsp.cobol.common.mapping.ExtendedDocument;
 import org.eclipse.lsp.cobol.common.model.tree.Node;
 
@@ -29,6 +30,7 @@ public class DialectProcessingContext {
   AnalysisConfig config;
   String programDocumentUri;
   ExtendedDocument extendedDocument;
+  CleanerPreprocessor preprocessor;
   @Builder.Default List<Node> dialectNodes = new ArrayList<>();
   @Builder.Default String languageId = "cobol";
 }

server/dialect-daco/src/main/java/org/eclipse/lsp/cobol/dialects/daco/DaCoMaidProcessor.java

@@ -211,7 +211,7 @@ private CopyNode createMaidCopybookNode(
             copybookName,
             context.getExtendedDocument().getUri(),
             context.getExtendedDocument().getUri(),
-            true, context.getLanguageId());
+            context.getPreprocessor());
     CopybookModel copybookModel = resolvedCopybook.getResult();
 
     DaCoCopyNode cbNode =

server/dialect-daco/src/test/java/org/eclipse/lsp/cobol/dialects/daco/utils/UseCaseInitializerService.java

@@ -35,8 +35,8 @@
 import org.eclipse.lsp.cobol.common.file.FileSystemService;
 import org.eclipse.lsp.cobol.common.file.WorkspaceFileService;
 import org.eclipse.lsp.cobol.core.engine.dialects.DialectDiscoveryService;
-import org.eclipse.lsp.cobol.core.preprocessor.TextPreprocessor;
-import org.eclipse.lsp.cobol.core.preprocessor.TextPreprocessorImpl;
+import org.eclipse.lsp.cobol.dialects.hp.HpTextPreprocessor;
+import org.eclipse.lsp.cobol.dialects.ibm.IbmTextPreprocessor;
 import org.eclipse.lsp.cobol.domain.modules.DatabusModule;
 import org.eclipse.lsp.cobol.domain.modules.EngineModule;
 import org.eclipse.lsp.cobol.lsp.DisposableLSPStateService;
@@ -81,8 +81,11 @@ protected void configure() {
                 bind(FileSystemService.class).toInstance(new WorkspaceFileService());
                 bind(CobolLanguageClient.class).toInstance(languageClient);
                 bind(SubroutineService.class).to(SubroutineServiceImpl.class);
-                bind(TextPreprocessor.class).to(TextPreprocessorImpl.class);
-                bind(CleanerPreprocessor.class).to(TextPreprocessorImpl.class);
+
+                bind(CleanerPreprocessor.class).annotatedWith(Names.named("cobol")).to(IbmTextPreprocessor.class);
+                bind(CleanerPreprocessor.class).annotatedWith(Names.named("hpcobol")).to(HpTextPreprocessor.class);
+
+                bind(CleanerPreprocessor.class).to(IbmTextPreprocessor.class);
                 bind(WatcherService.class).to(WatcherServiceImpl.class);
                 bind(DialectDiscoveryService.class).to(ExplicitDialectDiscoveryService.class);
                 bind(CopybookIdentificationService.class)

server/dialect-idms/src/main/java/org/eclipse/lsp/cobol/dialects/idms/IdmsCopybookService.java

@@ -24,6 +24,7 @@
 import org.antlr.v4.runtime.CommonTokenStream;
 import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.tree.ParseTreeListener;
+import org.eclipse.lsp.cobol.common.CleanerPreprocessor;
 import org.eclipse.lsp.cobol.common.ResultWithErrors;
 import org.eclipse.lsp.cobol.common.copybook.CopybookModel;
 import org.eclipse.lsp.cobol.common.copybook.CopybookName;
@@ -44,6 +45,7 @@ class IdmsCopybookService {
 
   private final String programDocumentUri;
   private final CopybookService copybookService;
+  private final CleanerPreprocessor preprocessor;
   private final CopybookProcessingMode copybookProcessingMode;
   private final ParseTreeListener treeListener;
   private final MessageService messageService;
@@ -95,7 +97,7 @@ private ResultWithErrors<List<Node>> processNodes(CopybookModel copybookModel, i
     parser.setErrorHandler(new CobolErrorStrategy(messageService));
     parser.addParseListener(treeListener);
 
-    IdmsCopybookVisitor visitor = new IdmsCopybookVisitor(copybookService, copybookProcessingMode, treeListener, messageService,
+    IdmsCopybookVisitor visitor = new IdmsCopybookVisitor(copybookService, preprocessor, copybookProcessingMode, treeListener, messageService,
         programDocumentUri, copybookModel.getUri(), parentLevel, processedCopybooks);
 
     ParserRuleContext node = parser.startRule();

server/dialect-idms/src/main/java/org/eclipse/lsp/cobol/dialects/idms/IdmsCopybookVisitor.java

@@ -29,6 +29,7 @@
 import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.ParseTreeListener;
 import org.antlr.v4.runtime.tree.TerminalNode;
+import org.eclipse.lsp.cobol.common.CleanerPreprocessor;
 import org.eclipse.lsp.cobol.common.ResultWithErrors;
 import org.eclipse.lsp.cobol.common.copybook.*;
 import org.eclipse.lsp.cobol.common.error.SyntaxError;
@@ -51,17 +52,17 @@
 @RequiredArgsConstructor
 class IdmsCopybookVisitor extends IdmsCopyParserBaseVisitor<List<Node>> {
   private final CopybookService copybookService;
-  private final CopybookProcessingMode copybookProcessingMode;
+  private final CleanerPreprocessor preprocessor;
   private final IdmsCopybookService idmsCopybookService;
   private final String programDocumentUri;
   private final String documentUri;
   private final int parentLevel;
-  private final Set<CopybookName> processedCopybooks;
   @Getter private final List<SyntaxError> errors = new LinkedList<>();
 
   private int firstCopybookLevel = 0;
 
   IdmsCopybookVisitor(CopybookService copybookService,
+                      CleanerPreprocessor preprocessor,
                       CopybookProcessingMode copybookProcessingMode,
                       ParseTreeListener treeListener,
                       MessageService messageService,
@@ -70,12 +71,11 @@ class IdmsCopybookVisitor extends IdmsCopyParserBaseVisitor<List<Node>> {
                       int parentLevel,
                       Set<CopybookName> processedCopybooks) {
     this.copybookService = copybookService;
-    this.copybookProcessingMode = copybookProcessingMode;
+    this.preprocessor = preprocessor;
     this.programDocumentUri = programDocumentUri;
     this.documentUri = documentUri;
     this.parentLevel = parentLevel;
-    this.processedCopybooks = processedCopybooks;
-    idmsCopybookService = new IdmsCopybookService(programDocumentUri, copybookService,
+    idmsCopybookService = new IdmsCopybookService(programDocumentUri, copybookService, preprocessor,
             copybookProcessingMode, treeListener, messageService, processedCopybooks);
   }
 
@@ -90,7 +90,7 @@ public List<Node> visitCopyIdmsStatement(IdmsCopyParser.CopyIdmsStatementContext
             copybookName,
             programDocumentUri,
             documentUri,
-            true, "cobol");
+            preprocessor);
     CopybookModel copybookModel = resolvedCopybook.getResult();
     Locality locality = IdmsParserHelper.buildNameRangeLocality(optionsContext, copybookName.getDisplayName(), documentUri);
     errors.addAll(resolvedCopybook.getErrors());

server/dialect-idms/src/main/java/org/eclipse/lsp/cobol/dialects/idms/IdmsDialect.java

@@ -79,11 +79,11 @@ private void insertIdmsCopybook(DialectProcessingContext ctx, ExtendedDocument e
                                   Deque<String> copybookStack) {
     CopybookName copybookName = new CopybookName(cb.getName(), IdmsDialect.NAME);
     ResultWithErrors<CopybookModel> resolvedCopybook = copybookService.resolve(
-            copybookName.toCopybookId(programDocumentUri),
-            copybookName,
-            programDocumentUri,
-            currentUri,
-            true, "cobol");
+        copybookName.toCopybookId(programDocumentUri),
+        copybookName,
+        programDocumentUri,
+        currentUri,
+        ctx.getPreprocessor());
     CopybookModel copybookModel = resolvedCopybook.getResult();
 
     if (copybookModel.getUri() == null || copybookModel.getContent() == null) {

server/dialect-idms/src/test/java/org/eclipse/lsp/cobol/dialects/idms/IdmsCopybookServiceTest.java

@@ -22,6 +22,7 @@
 import java.util.List;
 import java.util.Set;
 import org.antlr.v4.runtime.tree.ParseTreeListener;
+import org.eclipse.lsp.cobol.common.CleanerPreprocessor;
 import org.eclipse.lsp.cobol.common.ResultWithErrors;
 import org.eclipse.lsp.cobol.common.copybook.*;
 import org.eclipse.lsp.cobol.common.error.SyntaxError;
@@ -46,14 +47,17 @@ class IdmsCopybookServiceTest {
   private ParseTreeListener treeListener;
   @Mock
   private MessageService messageService;
+  @Mock
+  private CleanerPreprocessor preprocessor;
 
   private Set<CopybookName> processedCopybooks;
   private IdmsCopybookService service;
 
   @BeforeEach
   void init() {
     processedCopybooks = new HashSet<>();
-    service = new IdmsCopybookService("uri", copybookService, CopybookProcessingMode.ENABLED,
+    service = new IdmsCopybookService("uri", copybookService, preprocessor,
+        CopybookProcessingMode.ENABLED,
         treeListener, messageService, processedCopybooks);
   }
 

server/dialect-idms/src/test/java/org/eclipse/lsp/cobol/dialects/idms/IdmsCopybookVisitorTest.java

@@ -17,21 +17,19 @@
 import static org.eclipse.lsp.cobol.dialects.idms.IdmsCopyParser.*;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 import com.google.common.collect.ImmutableList;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.tree.TerminalNode;
+import org.eclipse.lsp.cobol.common.CleanerPreprocessor;
 import org.eclipse.lsp.cobol.common.ResultWithErrors;
 import org.eclipse.lsp.cobol.common.copybook.*;
 import org.eclipse.lsp.cobol.common.model.NodeType;
@@ -55,14 +53,13 @@ class IdmsCopybookVisitorTest {
   private CopybookService copybookService;
   @Mock
   private IdmsCopybookService idmsCopybookService;
+  @Mock
+  private CleanerPreprocessor preprocessor;
   private IdmsCopybookVisitor visitor;
 
   @BeforeEach
   void init() {
-    Set<CopybookName> processedCopybooks = new HashSet<>();
-
-    visitor = new IdmsCopybookVisitor(copybookService, CopybookProcessingMode.ENABLED,
-        idmsCopybookService, PROGRAM_DOCUMENT_URI, "documentUri", 1, processedCopybooks);
+      visitor = new IdmsCopybookVisitor(copybookService, preprocessor, idmsCopybookService, PROGRAM_DOCUMENT_URI, "documentUri", 1);
   }
 
   @Test
@@ -72,7 +69,7 @@ void testVisitCopyIdmsStatement() {
     IdmsCopyParser.CopyIdmsSourceContext sourceContext = mock(IdmsCopyParser.CopyIdmsSourceContext.class);
     CopybookName expectedCopybookName = new CopybookName("copybook", "IDMS");
     when(copybookService.resolve(any(CopybookId.class), any(CopybookName.class), anyString(), anyString(),
-            anyBoolean(), anyString()))
+            any()))
             .thenReturn(
                     new ResultWithErrors<>(
                             new CopybookModel(expectedCopybookName.toCopybookId(PROGRAM_DOCUMENT_URI), expectedCopybookName, null, null),

server/dialect-idms/src/test/java/org/eclipse/lsp/cobol/dialects/idms/utils/UseCaseInitializerService.java

@@ -35,8 +35,7 @@
 import org.eclipse.lsp.cobol.common.file.FileSystemService;
 import org.eclipse.lsp.cobol.common.file.WorkspaceFileService;
 import org.eclipse.lsp.cobol.core.engine.dialects.DialectDiscoveryService;
-import org.eclipse.lsp.cobol.core.preprocessor.TextPreprocessor;
-import org.eclipse.lsp.cobol.core.preprocessor.TextPreprocessorImpl;
+import org.eclipse.lsp.cobol.dialects.ibm.IbmTextPreprocessor;
 import org.eclipse.lsp.cobol.domain.modules.DatabusModule;
 import org.eclipse.lsp.cobol.domain.modules.EngineModule;
 import org.eclipse.lsp.cobol.lsp.DisposableLSPStateService;
@@ -82,8 +81,7 @@ protected void configure() {
                 bind(FileSystemService.class).toInstance(new WorkspaceFileService());
                 bind(CobolLanguageClient.class).toInstance(languageClient);
                 bind(SubroutineService.class).to(SubroutineServiceImpl.class);
-                bind(TextPreprocessor.class).to(TextPreprocessorImpl.class);
-                bind(CleanerPreprocessor.class).to(TextPreprocessorImpl.class);
+                bind(CleanerPreprocessor.class).to(IbmTextPreprocessor.class);
                 bind(WatcherService.class).to(WatcherServiceImpl.class);
                 bind(DialectDiscoveryService.class).to(ExplicitDialectDiscoveryService.class);
                 bind(CopybookIdentificationService.class)

server/engine/src/main/java/org/eclipse/lsp/cobol/cli/Cli.java

@@ -27,6 +27,7 @@
 import org.eclipse.lsp.cobol.cli.di.CliModule;
 import org.eclipse.lsp.cobol.cli.modules.CliClientProvider;
 import org.eclipse.lsp.cobol.common.AnalysisConfig;
+import org.eclipse.lsp.cobol.common.CleanerPreprocessor;
 import org.eclipse.lsp.cobol.common.ResultWithErrors;
 import org.eclipse.lsp.cobol.common.SubroutineService;
 import org.eclipse.lsp.cobol.common.benchmark.BenchmarkService;
@@ -45,9 +46,9 @@
 import org.eclipse.lsp.cobol.core.engine.pipeline.stages.*;
 import org.eclipse.lsp.cobol.core.engine.processor.AstProcessor;
 import org.eclipse.lsp.cobol.core.engine.symbols.SymbolsRepository;
-import org.eclipse.lsp.cobol.core.preprocessor.TextPreprocessor;
 import org.eclipse.lsp.cobol.core.preprocessor.delegates.GrammarPreprocessor;
 import org.eclipse.lsp.cobol.core.semantics.CopybooksRepository;
+import org.eclipse.lsp.cobol.dialects.ibm.IbmTextPreprocessor;
 import org.eclipse.lsp.cobol.lsp.CobolLanguageId;
 import org.eclipse.lsp.cobol.service.settings.CachingConfigurationService;
 import org.eclipse.lsp.cobol.service.settings.layout.CodeLayoutStore;
@@ -102,7 +103,7 @@ public Integer call() throws Exception {
     cliClientProvider.setCpyExt(Arrays.asList(cpyExt));
 
     // Cleaning up
-    TextPreprocessor preprocessor = diCtx.getInstance(TextPreprocessor.class);
+    CleanerPreprocessor preprocessor = diCtx.getInstance(IbmTextPreprocessor.class);
     BenchmarkService benchmarkService = diCtx.getInstance(BenchmarkService.class);
 
     if (src == null) {
@@ -111,7 +112,7 @@ public Integer call() throws Exception {
     }
     String documentUri = src.toURI().toString();
     String text = new String(Files.readAllBytes(src.toPath()));
-    ResultWithErrors<ExtendedText> resultWithErrors = preprocessor.cleanUpCode(documentUri, text, CobolLanguageId.COBOL);
+    ResultWithErrors<ExtendedText> resultWithErrors = preprocessor.cleanUpCode(documentUri, text);
     AnalysisContext ctx =
         new AnalysisContext(
             new ExtendedDocument(resultWithErrors.getResult(), text),
@@ -193,6 +194,7 @@ private static Pipeline setupPipeline(Injector diCtx, Action action) {
     DialectService dialectService = diCtx.getInstance(DialectService.class);
     MessageService messageService = diCtx.getInstance(MessageService.class);
     GrammarPreprocessor grammarPreprocessor = diCtx.getInstance(GrammarPreprocessor.class);
+    CleanerPreprocessor preprocessor = diCtx.getInstance(IbmTextPreprocessor.class);
     ParseTreeListener parseTreeListener = diCtx.getInstance(ParseTreeListener.class);
     SymbolsRepository symbolsRepository = diCtx.getInstance(SymbolsRepository.class);
     SubroutineService subroutineService = diCtx.getInstance(SubroutineService.class);
@@ -203,8 +205,8 @@ private static Pipeline setupPipeline(Injector diCtx, Action action) {
 
     Pipeline pipeline = new Pipeline();
     pipeline.add(new CompilerDirectivesStage(messageService));
-    pipeline.add(new DialectProcessingStage(dialectService));
-    pipeline.add(new PreprocessorStage(grammarPreprocessor));
+    pipeline.add(new DialectProcessingStage(dialectService, preprocessor));
+    pipeline.add(new PreprocessorStage(grammarPreprocessor, preprocessor));
     if (action == Action.analysis) {
       pipeline.add(new ImplicitDialectProcessingStage(dialectService));
       pipeline.add(new ParserStage(messageService, parseTreeListener));

server/engine/src/main/java/org/eclipse/lsp/cobol/cli/di/CliModule.java

@@ -26,6 +26,7 @@
 import org.eclipse.lsp.cobol.cfg.CFASTBuilder;
 import org.eclipse.lsp.cobol.cfg.CFASTBuilderImpl;
 import org.eclipse.lsp.cobol.cli.modules.CliClientProvider;
+import org.eclipse.lsp.cobol.common.CleanerPreprocessor;
 import org.eclipse.lsp.cobol.common.LanguageEngineFacade;
 import org.eclipse.lsp.cobol.common.SubroutineService;
 import org.eclipse.lsp.cobol.common.action.CodeActionProvider;
@@ -40,8 +41,8 @@
 import org.eclipse.lsp.cobol.core.engine.dialects.DialectDiscoveryService;
 import org.eclipse.lsp.cobol.core.messages.LocaleStoreImpl;
 import org.eclipse.lsp.cobol.core.messages.PropertiesMessageService;
-import org.eclipse.lsp.cobol.core.preprocessor.TextPreprocessor;
-import org.eclipse.lsp.cobol.core.preprocessor.TextPreprocessorImpl;
+import org.eclipse.lsp.cobol.dialects.hp.HpTextPreprocessor;
+import org.eclipse.lsp.cobol.dialects.ibm.IbmTextPreprocessor;
 import org.eclipse.lsp.cobol.core.preprocessor.delegates.GrammarPreprocessor;
 import org.eclipse.lsp.cobol.core.preprocessor.delegates.GrammarPreprocessorImpl;
 import org.eclipse.lsp.cobol.core.preprocessor.delegates.copybooks.GrammarPreprocessorListenerFactory;
@@ -90,7 +91,8 @@
 public class CliModule extends AbstractModule {
   @Override
   protected void configure() {
-    bind(TextPreprocessor.class).to(TextPreprocessorImpl.class);
+    bind(CleanerPreprocessor.class).annotatedWith(Names.named("cobol")).to(IbmTextPreprocessor.class);
+    bind(CleanerPreprocessor.class).annotatedWith(Names.named("hpcobol")).to(HpTextPreprocessor.class);
     bind(CobolLineReader.class).annotatedWith(Names.named("hpcobol")).to(HPCobolLineReaderImpl.class);
     bind(CobolLineReader.class).annotatedWith(Names.named("cobol")).to(CobolLineReaderImpl.class);