fix: lg parse template body/comments issue (#2373)

* upgrade lg package

* update api methods

* update api

* upgrade package

* upgrade package

* update test

* Update Composer/packages/client/package.json

Co-Authored-By: Andy Brown <asbrown002@gmail.com>

* Update Composer/packages/lib/indexers/package.json

Co-Authored-By: Andy Brown <asbrown002@gmail.com>

* Update Composer/packages/tools/language-servers/language-generation/package.json

Co-Authored-By: Andy Brown <asbrown002@gmail.com>

* Update package.json

* Update package.json

* Update package.json

Co-authored-by: Chris Whitten <christopher.whitten@microsoft.com>
Co-authored-by: Andy Brown <asbrown002@gmail.com>

Composer/packages/client/package.json

@@ -37,7 +37,7 @@
     "@uifabric/styling": "^7.10.4",
     "adaptive-expressions": "^4.8.0-preview-110700",
     "axios": "^0.18.0",
-    "botbuilder-lg": "^4.8.0-preview-109324",
+    "botbuilder-lg": "^4.9.0-preview-117748",
     "format-message": "^6.2.3",
     "immer": "^5.2.0",
     "jwt-decode": "^2.2.0",

Composer/packages/client/src/store/index.tsx

@@ -3,7 +3,7 @@
 
 import React, { useReducer, useRef } from 'react';
 import once from 'lodash/once';
-import { ImportResolverDelegate, LGParser } from 'botbuilder-lg';
+import { ImportResolverDelegate, TemplatesParser } from 'botbuilder-lg';
 import { LgFile, LuFile, importResolverGenerator } from '@bfc/shared';
 
 import { prepareAxios } from '../utils/auth';
@@ -25,7 +25,7 @@ import { undoActionsMiddleware } from './middlewares/undo';
 import { ActionType } from './action/types';
 import filePersistence from './persistence/FilePersistence';
 
-const { defaultFileResolver } = LGParser;
+const { defaultFileResolver } = TemplatesParser;
 
 const initialState: State = {
   dialogs: [],

Composer/packages/extensions/visual-designer/src/hooks/useLgTemplate.ts

@@ -2,17 +2,17 @@
 // Licensed under the MIT License.
 
 import { useContext } from 'react';
-import { LGTemplate } from 'botbuilder-lg';
+import { Template } from 'botbuilder-lg';
 import { LgTemplateRef } from '@bfc/shared';
 import get from 'lodash/get';
 
 import { NodeRendererContext } from '../store/NodeRendererContext';
 import { normalizeLgTemplate } from '../utils/normalizeLgTemplate';
 
-export const queryLgTemplateFromFiles = (lgTemplateName: string, lgFiles: any): LGTemplate | undefined => {
+export const queryLgTemplateFromFiles = (lgTemplateName: string, lgFiles: any): Template | undefined => {
   if (!Array.isArray(lgFiles)) return;
 
-  const allTemplates: LGTemplate[] = [];
+  const allTemplates: Template[] = [];
   for (const file of lgFiles) {
     const templates = get(file, 'templates');
     if (Array.isArray(templates)) {

Composer/packages/extensions/visual-designer/src/store/NodeRendererContext.ts

@@ -2,7 +2,7 @@
 // Licensed under the MIT License.
 
 import React from 'react';
-import { LGTemplate } from 'botbuilder-lg';
+import { Template } from 'botbuilder-lg';
 import { ShellApi, DialogFactory } from '@bfc/shared';
 
 type ShellApiFuncs =
@@ -18,7 +18,7 @@ export interface NodeRendererContextValue extends Pick<ShellApi, ShellApiFuncs>
   focusedEvent?: string;
   focusedTab?: string;
   clipboardActions: any[];
-  getLgTemplateSync: (lgTemplateName: string) => LGTemplate | undefined;
+  getLgTemplateSync: (lgTemplateName: string) => Template | undefined;
   dialogFactory: DialogFactory;
 }
 

Composer/packages/extensions/visual-designer/src/utils/normalizeLgTemplate.ts

@@ -1,9 +1,9 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-import { LGTemplate } from 'botbuilder-lg';
+import { Template } from 'botbuilder-lg';
 
-export function normalizeLgTemplate(template: LGTemplate): string {
+export function normalizeLgTemplate(template: Template): string {
   const templateTexts = template.body.split('\n').map(line => (line.startsWith('-') ? line.substring(1) : line));
   let showText = '';
 

Composer/packages/lib/indexers/__tests__/lgUtil.test.ts

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-import { LGParser } from 'botbuilder-lg';
+import { Templates } from 'botbuilder-lg';
 
 import { updateTemplate, addTemplate, removeTemplate } from '../src/utils/lgUtil';
 
@@ -15,7 +15,7 @@ describe('update lg template', () => {
 
     const template = { name: 'Exit', parameters: [], body: '-Bye' };
     const newContent = updateTemplate(content, 'Exit', template);
-    const { templates }: any = LGParser.parseText(newContent);
+    const templates = Templates.parseText(newContent).toArray();
     expect(templates.length).toEqual(2);
     expect(templates[0].name).toEqual('Exit');
     expect(templates[0].body).toEqual('-Bye');
@@ -31,7 +31,7 @@ describe('add lg template', () => {
 -What's up bro`;
     const template = { name: 'Hi', parameters: [], body: '-hello' };
     const newContent = addTemplate(content, template);
-    const { templates }: any = LGParser.parseText(newContent);
+    const templates = Templates.parseText(newContent).toArray();
     expect(templates.length).toEqual(3);
     expect(templates[0].name).toEqual('Exit');
     expect(templates[1].name).toEqual('Greeting');
@@ -47,7 +47,7 @@ describe('add lg template', () => {
 # Greeting
 -What's up bro`;
     const newContent = removeTemplate(content, 'Greeting');
-    const { templates }: any = LGParser.parseText(newContent);
+    const templates = Templates.parseText(newContent).toArray();
     expect(templates.length).toEqual(1);
     expect(templates[0].name).toEqual('Exit');
   });

Composer/packages/lib/indexers/package.json

@@ -31,7 +31,7 @@
     "@bfc/shared": "*",
     "@microsoft/bf-lu": "^4.9.0-preview.115707",
     "adaptive-expressions": "^4.8.0-preview-110700",
-    "botbuilder-lg": "^4.8.0-preview-109324",
+    "botbuilder-lg": "^4.9.0-preview-117748",
     "lodash": "^4.17.15"
   }
 }

Composer/packages/lib/indexers/src/lgIndexer.ts

@@ -1,13 +1,13 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-import { LGParser, Diagnostic as LGDiagnostic, ImportResolverDelegate } from 'botbuilder-lg';
+import { Templates, TemplatesParser, Diagnostic as LGDiagnostic, ImportResolverDelegate } from 'botbuilder-lg';
 import get from 'lodash/get';
 import { LgTemplate, LgFile, FileInfo, Diagnostic, Position, Range } from '@bfc/shared';
 
 import { getBaseName } from './utils/help';
 
-const { defaultFileResolver } = LGParser;
+const { defaultFileResolver } = TemplatesParser;
 
 // NOTE: LGDiagnostic is defined in PascalCase which should be corrected
 function convertLGDiagnostic(d: LGDiagnostic, source: string): Diagnostic {
@@ -25,8 +25,8 @@ function parse(
   id = '',
   importResolver: ImportResolverDelegate = defaultFileResolver
 ): { templates: LgTemplate[]; diagnostics: Diagnostic[] } {
-  const lgFile = LGParser.parseText(content, id, importResolver);
-  const templates = lgFile.templates.map(t => {
+  const lgFile = Templates.parseText(content, id, importResolver);
+  const templates = lgFile.toArray().map(t => {
     return {
       name: t.name,
       body: t.body,

Composer/packages/lib/indexers/src/utils/lgUtil.ts

@@ -7,7 +7,7 @@
  *
  */
 
-import { LGParser } from 'botbuilder-lg';
+import { Templates } from 'botbuilder-lg';
 import { LgTemplate } from '@bfc/shared';
 
 export interface Template {
@@ -34,9 +34,9 @@ export function updateTemplate(
   templateName: string,
   { name, parameters = [], body }: LgTemplate
 ): string {
-  const resource = LGParser.parseText(content);
+  const resource = Templates.parseText(content);
   // add if not exist
-  if (resource.templates.findIndex(t => t.name === templateName) === -1) {
+  if (resource.toArray().findIndex(t => t.name === templateName) === -1) {
     return resource.addTemplate(name, parameters, body).toString();
   } else {
     return resource.updateTemplate(templateName, name, parameters, body).toString();
@@ -45,7 +45,7 @@ export function updateTemplate(
 
 // if name exist, throw error.
 export function addTemplate(content: string, { name, parameters = [], body }: LgTemplate): string {
-  const resource = LGParser.parseText(content);
+  const resource = Templates.parseText(content);
   return resource.addTemplate(name, parameters, body).toString();
 }
 
@@ -54,16 +54,16 @@ export function addTemplateAnyway(
   content: string,
   { name = 'TemplateName', parameters = [], body = '-TemplateBody' }: LgTemplate
 ): string {
-  const resource = LGParser.parseText(content);
-  const newName = increaseNameUtilNotExist(resource.templates, name);
+  const resource = Templates.parseText(content);
+  const newName = increaseNameUtilNotExist(resource.toArray(), name);
 
   return resource.addTemplate(newName, parameters, body).toString();
 }
 
 // if toTemplateName exist, throw error.
 export function copyTemplate(content: string, fromTemplateName: string, toTemplateName: string): string {
-  const resource = LGParser.parseText(content);
-  const fromTemplate = resource.templates.find(t => t.name === fromTemplateName);
+  const resource = Templates.parseText(content);
+  const fromTemplate = resource.toArray().find(t => t.name === fromTemplateName);
   if (!fromTemplate) {
     throw new Error('fromTemplateName no exist');
   }
@@ -73,28 +73,28 @@ export function copyTemplate(content: string, fromTemplateName: string, toTempla
 
 // if toTemplateName exist, add it anyway, with name like `${toTemplateName}1` `${toTemplateName}2`
 export function copyTemplateAnyway(content: string, fromTemplateName: string, toTemplateName?: string): string {
-  const resource = LGParser.parseText(content);
-  const fromTemplate = resource.templates.find(t => t.name === fromTemplateName);
+  const resource = Templates.parseText(content);
+  const fromTemplate = resource.toArray().find(t => t.name === fromTemplateName);
   if (!fromTemplate) {
     return resource.toString();
   }
 
   let newName = toTemplateName;
   if (!newName) {
     const copyName = `${fromTemplate.name}_Copy`;
-    newName = increaseNameUtilNotExist(resource.templates, copyName);
+    newName = increaseNameUtilNotExist(resource.toArray(), copyName);
   }
   const { parameters, body } = fromTemplate;
   return resource.addTemplate(newName, parameters, body).toString();
 }
 
 export function removeTemplate(content: string, templateName: string): string {
-  const resource = LGParser.parseText(content);
+  const resource = Templates.parseText(content);
   return resource.deleteTemplate(templateName).toString();
 }
 
 export function removeTemplates(content: string, templateNames: string[]): string {
-  let resource = LGParser.parseText(content);
+  let resource = Templates.parseText(content);
   templateNames.forEach(templateName => {
     resource = resource.deleteTemplate(templateName);
   });
@@ -127,7 +127,7 @@ export function textFromTemplates(templates: LgTemplate[]): string {
 export function checkSingleLgTemplate(template: LgTemplate) {
   const content = textFromTemplates([template]);
 
-  if (LGParser.parseText(content).templates.length !== 1) {
+  if (Templates.parseText(content).toArray().length !== 1) {
     throw new Error('Not a single template');
   }
 }

Composer/packages/tools/language-servers/language-generation/__tests__/app.test.ts

@@ -112,7 +112,7 @@ describe('lg lsp server', () => {
       [
         response => {
           expect(response.id).toEqual(2);
-          expect(response.result.contents[0]).toEqual('-Good evening');
+          expect(response.result.contents[0]).toContain('-Good evening');
         },
       ]
     );

Composer/packages/tools/language-servers/language-generation/package.json

@@ -16,8 +16,8 @@
   },
   "dependencies": {
     "@bfc/indexers": "*",
-    "botbuilder-lg": "^4.8.0-preview-109324",
     "adaptive-expressions": "^4.8.0-preview-110700",
+    "botbuilder-lg": "^4.9.0-preview-117748",
     "request-light": "^0.2.2",
     "vscode-languageserver": "^5.3.0-next"
   },

Composer/packages/tools/language-servers/language-generation/src/LGServer.ts

@@ -21,7 +21,7 @@ import { TextDocumentPositionParams, DocumentOnTypeFormattingParams } from 'vsco
 import get from 'lodash/get';
 import { filterTemplateDiagnostics, isValid } from '@bfc/indexers';
 import { MemoryResolver } from '@bfc/shared';
-import { ImportResolverDelegate, LGParser } from 'botbuilder-lg';
+import { ImportResolverDelegate, Templates } from 'botbuilder-lg';
 
 import { buildInfunctionsMap } from './builtinFunctionsMap';
 import {
@@ -227,7 +227,7 @@ export class LGServer {
       }
 
       const id = fileId || uri;
-      const { allTemplates, diagnostics } = LGParser.parseText(content, id, importResolver);
+      const { allTemplates, diagnostics } = Templates.parseText(content, id, importResolver);
 
       return { templates: allTemplates, diagnostics };
     };
@@ -731,7 +731,7 @@ export class LGServer {
       this.sendDiagnostics(document, lspDiagnostics);
       return;
     }
-    const lgDiagnostics = LGParser.parseText(text, fileId || uri, this.getImportResolver(document)).diagnostics;
+    const lgDiagnostics = Templates.parseText(text, fileId || uri, this.getImportResolver(document)).diagnostics;
     const lspDiagnostics = convertDiagnostics(lgDiagnostics, document);
     this.sendDiagnostics(document, lspDiagnostics);
   }

Composer/packages/tools/language-servers/language-generation/src/utils.ts

@@ -2,7 +2,7 @@
 // Licensed under the MIT License.
 
 import { TextDocument, Range, Position, DiagnosticSeverity, Diagnostic } from 'vscode-languageserver-types';
-import { DiagnosticSeverity as LGDiagnosticSeverity, Diagnostic as LGDiagnostic, LGParser } from 'botbuilder-lg';
+import { DiagnosticSeverity as LGDiagnosticSeverity, Diagnostic as LGDiagnostic, Templates } from 'botbuilder-lg';
 import { LgTemplate, Diagnostic as BFDiagnostic, LgFile, LgParsed } from '@bfc/shared';
 import { offsetRange } from '@bfc/indexers';
 
@@ -118,15 +118,15 @@ export function textFromTemplate(template: Template): string {
 
 export function checkTemplate(template: Template): LGDiagnostic[] {
   const text = textFromTemplate(template);
-  return LGParser.parseText(text, '').diagnostics.filter(diagnostic => {
+  return Templates.parseText(text, '').diagnostics.filter(diagnostic => {
     // ignore non-exist references in template body.
     return diagnostic.message.includes('does not have an evaluator') === false;
   });
 }
 
 export function updateTemplate(content: string, name: string, body: string): string {
-  const lgFile = LGParser.parseText(content);
-  const template = lgFile.templates.find(t => t.name === name);
+  const lgFile = Templates.parseText(content);
+  const template = lgFile.toArray().find(t => t.name === name);
   // add if not exist
   if (!template) {
     return lgFile.addTemplate(name, [], body).toString();

Composer/yarn.lock

@@ -4011,10 +4011,10 @@ acorn@^7.1.0:
   resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.1.0.tgz#949d36f2c292535da602283586c2477c57eb2d6c"
   integrity sha512-kL5CuoXA/dgxlBbVrflsflzQ3PAas7RYZB52NOm/6839iVYJgKMJ3cQJD+t2i5+qFa8h3MDpEOJiS64E8JLnSQ==
 
-adaptive-expressions@4.8.0-preview-109324:
-  version "4.8.0-preview-109324"
-  resolved "https://registry.yarnpkg.com/adaptive-expressions/-/adaptive-expressions-4.8.0-preview-109324.tgz#daf8abac37a25f519f73019343bc3a1f797e3048"
-  integrity sha512-TNEZUISxfJ8jj7K0mLmkAKgiakhLhyUglFIZhb4m+oSPCN7BtkE7cUkhTSYXyvlO5n3zYW9vimlC1C4BfnpJug==
+adaptive-expressions@4.9.0-preview-117748:
+  version "4.9.0-preview-117748"
+  resolved "https://botbuilder.myget.org/F/botbuilder-v4-js-daily/npm/adaptive-expressions/-/adaptive-expressions-4.9.0-preview-117748.tgz#19373501962f88e23dbbaa5c8affa901a0a91b3a"
+  integrity sha1-GTc1AZYviOI9u6pciv+pAaCpGzo=
   dependencies:
     "@microsoft/recognizers-text-data-types-timex-expression" "^1.1.4"
     "@types/lru-cache" "^5.1.0"
@@ -4412,14 +4412,6 @@ assert@^1.1.1:
   dependencies:
     util "0.10.3"
 
-assert@^1.4.1:
-  version "1.5.0"
-  resolved "https://botbuilder.myget.org/F/botbuilder-v4-js-daily/npm/assert/-/assert-1.5.0.tgz#55c109aaf6e0aefdb3dc4b71240c70bf574b18eb"
-  integrity sha1-VcEJqvbgrv2z3EtxJAxwv1dLGOs=
-  dependencies:
-    object-assign "^4.1.1"
-    util "0.10.3"
-
 assign-symbols@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/assign-symbols/-/assign-symbols-1.0.0.tgz#59667f41fadd4f20ccbc2bb96b8d4f7f78ec0367"
@@ -5009,31 +5001,17 @@ boolbase@^1.0.0, boolbase@~1.0.0:
   resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e"
   integrity sha1-aN/1++YMUes3cl6p4+0xDcwed24=
 
-botbuilder-core@4.8.0-preview-109324:
-  version "4.8.0-preview-109324"
-  resolved "https://registry.yarnpkg.com/botbuilder-core/-/botbuilder-core-4.8.0-preview-109324.tgz#b4c6080bfc85520f923bafb8ce2ca42563402464"
-  integrity sha512-P5VpfDWnb1dAnapOfNm4tSO94ApE3EE/F7dARBEjrMpPfhaqY8uDhDOh+BUy9wYYXqNSBOpIdnVo5TAAmfi9Sg==
-  dependencies:
-    assert "^1.4.1"
-    botframework-schema "4.8.0-preview-109324"
-
-botbuilder-lg@^4.8.0-preview-109324:
-  version "4.8.0-preview-109324"
-  resolved "https://registry.yarnpkg.com/botbuilder-lg/-/botbuilder-lg-4.8.0-preview-109324.tgz#7ee7e0e97e7c272a81266b1dbda6dfe0858b5f17"
-  integrity sha512-Ej4ywr7wkUhj5FT0Yx5vOY3TXJrQqVCo+W/xWxEGsrp2NJeWmAiz2kc4LI7QC6XN5fPypVV10iaerGDgEU1Wxg==
+botbuilder-lg@^4.9.0-preview-117748:
+  version "4.9.0-preview-117748"
+  resolved "https://botbuilder.myget.org/F/botbuilder-v4-js-daily/npm/botbuilder-lg/-/botbuilder-lg-4.9.0-preview-117748.tgz#818ebe001584d09c9d1021b57c48348f2d4c41f8"
+  integrity sha1-gY6+ABWE0JydECG1fEg0jy1MQfg=
   dependencies:
-    adaptive-expressions "4.8.0-preview-109324"
+    adaptive-expressions "4.9.0-preview-117748"
     antlr4ts "0.5.0-alpha.1"
-    botbuilder-core "4.8.0-preview-109324"
     lodash "^4.17.11"
     path "^0.12.7"
     uuid "^3.3.3"
 
-botframework-schema@4.8.0-preview-109324:
-  version "4.8.0-preview-109324"
-  resolved "https://registry.yarnpkg.com/botframework-schema/-/botframework-schema-4.8.0-preview-109324.tgz#e65ce6219b050e7f2e992a5cfbebf45f17d7dd04"
-  integrity sha512-ee4SejGhDWZSWP0SM6WnPA2o9tR3H8pvk0m2E4sWqy3nqmxfFsu7+5UDv364UIaPZxvcI99bA4cm288EE9+DRA==
-
 boxen@^1.2.1:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/boxen/-/boxen-1.3.0.tgz#55c6c39a8ba58d9c61ad22cd877532deb665a20b"