From e4e562045bf0356c7d2c59eb75ed00239c4c2ec3 Mon Sep 17 00:00:00 2001 From: Emilio Munoz Date: Tue, 18 Feb 2020 20:59:32 -0800 Subject: [PATCH] Deleting commands from lu api package (#584) * Deleting commands from lu api package * Commenting out missing translate test * Removing oclif dependencies on LU api package * Adding file reader to LU * Fixing variable name error and broken tests --- packages/lu/package.json | 4 - packages/lu/src/commands/luis/build.ts | 130 ------------ packages/lu/src/commands/luis/convert.ts | 99 --------- packages/lu/src/commands/luis/generate/cs.ts | 74 ------- packages/lu/src/commands/luis/generate/ts.ts | 66 ------ packages/lu/src/commands/luis/translate.ts | 99 --------- packages/lu/src/commands/qnamaker/convert.ts | 107 ---------- .../lu/src/commands/qnamaker/translate.ts | 99 --------- packages/lu/src/parser/luis/luisGenBuilder.js | 5 +- packages/lu/src/utils/filehelper.ts | 34 +-- packages/lu/src/utils/textfilereader.ts | 51 +++++ .../lu/test/commands/luis/translate.test.ts | 2 +- packages/lu/test/utils/filehelper.test.js | 12 +- packages/luis/src/commands/luis/build.ts | 2 +- packages/luis/src/commands/luis/convert.ts | 2 +- .../luis/src/commands/luis/generate/cs.ts | 74 ++++--- .../luis/src/commands/luis/generate/ts.ts | 61 +++--- packages/luis/src/commands/luis/translate.ts | 2 +- packages/luis/src/utils/filehelper.ts | 195 ------------------ .../qnamaker/src/commands/qnamaker/convert.ts | 3 +- .../src/commands/qnamaker/translate.ts | 2 +- packages/qnamaker/src/utils/filehelper.ts | 195 ------------------ 22 files changed, 156 insertions(+), 1162 deletions(-) delete mode 100644 packages/lu/src/commands/luis/build.ts delete mode 100644 packages/lu/src/commands/luis/convert.ts delete mode 100644 packages/lu/src/commands/luis/generate/cs.ts delete mode 100644 packages/lu/src/commands/luis/generate/ts.ts delete mode 100644 packages/lu/src/commands/luis/translate.ts delete mode 100644 packages/lu/src/commands/qnamaker/convert.ts delete mode 100644 packages/lu/src/commands/qnamaker/translate.ts create mode 100644 packages/lu/src/utils/textfilereader.ts delete mode 100644 packages/luis/src/utils/filehelper.ts delete mode 100644 packages/qnamaker/src/utils/filehelper.ts diff --git a/packages/lu/package.json b/packages/lu/package.json index 0088d59d7..4e33e7ead 100644 --- a/packages/lu/package.json +++ b/packages/lu/package.json @@ -46,10 +46,6 @@ "dependencies": { "@azure/cognitiveservices-luis-authoring": "3.0.1", "@azure/ms-rest-azure-js": "2.0.1", - "@microsoft/bf-cli-command": "1.0.0", - "@oclif/command": "~1.5.19", - "@oclif/config": "~1.13.3", - "@oclif/errors": "~1.2.2", "antlr4": "^4.7.2", "chalk": "2.4.1", "console-stream": "^0.1.1", diff --git a/packages/lu/src/commands/luis/build.ts b/packages/lu/src/commands/luis/build.ts deleted file mode 100644 index 3e9d44998..000000000 --- a/packages/lu/src/commands/luis/build.ts +++ /dev/null @@ -1,130 +0,0 @@ -/*! - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. - */ - -import {Command, flags} from '@microsoft/bf-cli-command' -import {Builder} from './../../parser/lubuild/builder' -import {Settings} from './../../parser/lubuild/settings' -import {MultiLanguageRecognizer} from './../../parser/lubuild/multi-language-recognizer' -import {Recognizer} from './../../parser/lubuild/recognizer' -const path = require('path') -const fs = require('fs-extra') -const fileHelper = require('./../../utils/filehelper') -const fileExtEnum = require('./../../parser/utils/helpers').FileExtTypeEnum -const Content = require('./../../parser/lu/lu') - -export default class LuisBuild extends Command { - static description = 'Build lu files to train and publish luis applications' - - static examples = [` - $ bf luis:build --in {INPUT_FILE_OR_FOLDER} --authoringKey {AUTHORING_KEY} --botName {BOT_NAME} --dialog {true} - `] - - static flags: any = { - help: flags.help({char: 'h'}), - in: flags.string({char: 'i', description: 'Lu file or folder'}), - authoringKey: flags.string({description: 'LUIS authoring key', required: true}), - botName: flags.string({description: 'Bot name'}), - out: flags.string({description: 'Output location'}), - defaultCulture: flags.string({description: 'Culture code for the content. Infer from .lu if available. Defaults to en-us'}), - region: flags.string({description: 'LUIS authoring region'}), - suffix: flags.string({description: 'Environment name as a suffix identifier to include in LUIS app name'}), - force: flags.boolean({char: 'f', description: 'Force write dialog and settings files', default: false}), - dialog: flags.boolean({description: 'Write out .dialog files', default: false}), - fallbackLocale: flags.string({description: 'Locale to be used at the fallback if no locale specific recognizer is found. Only valid if --dialog is set'}), - luConfig: flags.string({description: 'Path to config for lu build'}), - log: flags.boolean({description: 'write out log messages to console', default: false}) - } - - async run() { - const {flags} = this.parse(LuisBuild) - - flags.stdin = await this.readStdin() - - let files: string[] = [] - if (flags.luConfig) { - const configFilePath = path.resolve(flags.luConfig) - if (fs.existsSync(configFilePath)) { - const configObj = JSON.parse(await fileHelper.getContentFromFile(configFilePath)) - if (configObj.name && configObj.name !== '') flags.botName = configObj.name - if (configObj.defaultLanguage && configObj.defaultLanguage !== '') flags.defaultCulture = configObj.defaultLanguage - if (configObj.deleteOldVersion) flags.deleteOldVersion = true - if (configObj.out && configObj.out !== '') flags.out = configObj.out - if (configObj.writeDialogFiles) flags.dialog = true - if (configObj.models && configObj.models.length > 0) { - files = configObj.models.map((m: string) => path.resolve(m)) - } - } - } - - if (!flags.stdin && !flags.in && files.length === 0) { - throw new Error('Missing input. Please use stdin or pass a file or folder location with --in flag') - } - - if (!flags.botName) { - throw new Error('Missing bot name. Please pass bot name with --botName flag') - } - - flags.defaultCulture = flags.defaultCulture && flags.defaultCulture !== '' ? flags.defaultCulture : 'en-us' - flags.region = flags.region && flags.region !== '' ? flags.region : 'westus' - flags.suffix = flags.suffix && flags.suffix !== '' ? flags.suffix : 'development' - flags.fallbackLocale = flags.fallbackLocale && flags.fallbackLocale !== '' ? flags.fallbackLocale : 'en-us' - - // create builder class - const builder = new Builder((input: string) => { - if (flags.log) this.log(input) - }) - - let luContents: any[] = [] - let recognizers = new Map() - let multiRecognizers = new Map() - let settings = new Map() - - if (flags.stdin && flags.stdin !== '') { - // load lu content from stdin and create default recognizer, multiRecognier and settings - this.log('Load lu content from stdin') - const content = new Content(flags.stdin, 'stdin', true, flags.defaultCulture, path.join(process.cwd(), 'stdin')) - luContents.push(content) - multiRecognizers.set('stdin', new MultiLanguageRecognizer(path.join(process.cwd(), 'stdin.lu.dialog'), {})) - settings.set('stdin', new Settings(path.join(process.cwd(), `luis.settings.${flags.suffix}.${flags.region}.json`), {})) - let recognizer = Recognizer.load(content.path, content.name, path.join(process.cwd(), `${content.name}.dialog`), settings.get('stdin') as Settings, {}) - recognizers.set(content.name, recognizer) - } else { - this.log('Start to load lu files') - - // get lu files from flags.in - if (flags.in && flags.in !== '') { - let luFiles = await fileHelper.getLuFiles(flags.in, true, fileExtEnum.LUFile) - files.push(...luFiles) - } - - // load lu contents from lu files - // load existing recognizers, multiRecogniers and settings or create default ones - const loadedResources = await builder.loadContents(files, flags.defaultCulture, flags.suffix, flags.region) - luContents = loadedResources.luContents - recognizers = loadedResources.recognizers - multiRecognizers = loadedResources.multiRecognizers - settings = loadedResources.settings - } - - // update or create and then train and publish luis applications based on loaded resources - this.log('Start to handle applications') - const dialogContents = await builder.build(luContents, recognizers, flags.authoringKey, flags.region, flags.botName, flags.suffix, flags.fallbackLocale, flags.deleteOldVersion, multiRecognizers, settings) - - // write dialog assets based on config - if (flags.dialog) { - const writeDone = await builder.writeDialogAssets(dialogContents, flags.force, flags.out) - const dialogFilePath = (flags.stdin || !flags.in) ? process.cwd() : flags.in.endsWith(fileExtEnum.LUFile) ? path.dirname(path.resolve(flags.in)) : path.resolve(flags.in) - const outputFolder = flags.out ? path.resolve(flags.out) : dialogFilePath - if (writeDone) { - this.log(`Successfully wrote .dialog files to ${outputFolder}`) - } else { - this.log(`No changes to the .dialog files in ${outputFolder}`) - } - } else { - this.log('The published application ids:') - this.log(JSON.parse(dialogContents[dialogContents.length - 1].content).luis) - } - } -} diff --git a/packages/lu/src/commands/luis/convert.ts b/packages/lu/src/commands/luis/convert.ts deleted file mode 100644 index 1de50f044..000000000 --- a/packages/lu/src/commands/luis/convert.ts +++ /dev/null @@ -1,99 +0,0 @@ -/*! - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. - */ - -import {CLIError, Command, flags, utils} from '@microsoft/bf-cli-command' -const exception = require('./../../parser/utils/exception') -const fs = require('fs-extra') -const file = require('./../../utils/filehelper') -const fileExtEnum = require('./../../parser/utils/helpers').FileExtTypeEnum -const Luis = require('./../../parser/luis/luis') -const LuisBuilder = require('./../../parser/luis/luisBuilder') - -export default class LuisConvert extends Command { - static description = 'Convert .lu file(s) to a LUIS application JSON model or vice versa' - - static flags: flags.Input = { - in: flags.string({char: 'i', description: 'Source .lu file(s) or LUIS application JSON model'}), - recurse: flags.boolean({char: 'r', description: 'Indicates if sub-folders need to be considered to file .lu file(s)', default: false}), - log: flags.boolean({description: 'Enables log messages', default: false}), - sort: flags.boolean({description: 'When set, intent, utterances, entities are alphabetically sorted in .lu files', default: false}), - out: flags.string({char: 'o', description: 'Output file or folder name. If not specified stdout will be used as output'}), - name: flags.string({description: 'Name of the LUIS application'}), - description: flags.string({description: 'Text describing the LUIS applicaion'}), - culture: flags.string({description: 'Lang code for the LUIS application'}), - versionid: flags.string({description: 'Version ID of the LUIS application'}), - schemaversion: flags.string({description: 'Schema version of the LUIS application'}), - force: flags.boolean({char: 'f', description: 'If --out flag is provided with the path to an existing file, overwrites that file', default: false}), - help: flags.help({char: 'h', description: 'luis:convert help'}) - } - - async run() { - try { - const {flags} = this.parse(LuisConvert) - // Check if data piped in stdin - const stdin = await this.readStdin() - - //Check if file or folder - //if folder, only lu to luis is supported - const isLu = await file.detectLuContent(stdin, flags.in) - - // Parse the object depending on the input - let result: any - if (isLu) { - const luFiles = await file.getLuObjects(stdin, flags.in, flags.recurse, fileExtEnum.LUFile) - result = await LuisBuilder.build(luFiles, flags.log, flags.culture) - if (!result.hasContent()) { - throw new CLIError('No LU or Luis content parsed!') - } - } else { - const luisContent = stdin ? stdin : await file.getContentFromFile(flags.in) - const luisObject = new Luis(file.parseJSON(luisContent, 'Luis')) - if (flags.sort) { - luisObject.sort() - } - - result = luisObject.parseToLuContent() - if (!result) { - throw new CLIError('No LU or Luis content parsed!') - } - } - - // Add headers to Luis Json - if (isLu) { - result.luis_schema_version = flags.schemaversion || result.luis_schema_version || '3.2.0' - result.versionId = flags.versionid || result.versionId || '0.1' - result.name = flags.name || result.name || '' - result.desc = flags.description || result.desc || '' - result.culture = flags.culture || result.culture || 'en-us' - result.culture = result.culture.toLowerCase() - result = JSON.stringify(result, null, 2) - } - - // Print or write the parsed object - if (flags.out) { - await this.writeOutput(result, flags, isLu) - } else { - this.log(result) - } - } catch (err) { - if (err instanceof exception) { - throw new CLIError(err.text) - } - throw err - } - } - - private async writeOutput(convertedObject: any, flags: any, isLu: boolean) { - let filePath = await file.generateNewFilePath(flags.out, flags.in, isLu) - const validatedPath = utils.validatePath(filePath, '', flags.force) - // write out the final file - try { - await fs.writeFile(validatedPath, convertedObject, 'utf-8') - } catch (err) { - throw new CLIError('Unable to write file - ' + validatedPath + ' Error: ' + err.message) - } - this.log('Successfully wrote LUIS model to ' + validatedPath) - } -} diff --git a/packages/lu/src/commands/luis/generate/cs.ts b/packages/lu/src/commands/luis/generate/cs.ts deleted file mode 100644 index 0f45a761a..000000000 --- a/packages/lu/src/commands/luis/generate/cs.ts +++ /dev/null @@ -1,74 +0,0 @@ -/*! - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. - */ - -import {CLIError, Command, flags} from '@microsoft/bf-cli-command' -import {camelCase, upperFirst} from 'lodash' -import * as path from 'path' - -const LuisToCsConverter = require('./../../../parser/converters/luistocsconverter') -const file = require('./../../../utils/filehelper') -const fs = require('fs-extra') - -export default class LuisGenerateCs extends Command { - static description = 'Generate:cs generates a strongly typed C# source code from an exported (json) LUIS model.' - - static flags: flags.Input = { - in: flags.string({char: 'i', description: 'Path to the file containing the LUIS application JSON model'}), - out: flags.string({char: 'o', description: 'Output file or folder name. If not specified stdout will be used as output', default: ''}), - className: flags.string({description: 'Name of the autogenerated class (can include namespace)'}), - force: flags.boolean({char: 'f', description: 'If --out flag is provided with the path to an existing file, overwrites that file', default: false}), - help: flags.help({char: 'h', description: 'luis:generate:cs help'}) - - } - - reorderEntities(app: any, name: string): void { - if (app[name] !== null && app[name] !== undefined) { - app[name].sort((a: any, b: any) => (a.name > b.name ? 1 : -1)) - } - } - - async run() { - const {flags} = this.parse(LuisGenerateCs) - let space = 'Luis' - let stdInput = await this.readStdin() - - if (!flags.in && !stdInput) { - throw new CLIError('Missing input. Please use stdin or pass a file location with --in flag') - } - - const pathPrefix = flags.in && path.isAbsolute(flags.in) ? '' : process.cwd() - let app: any - try { - app = stdInput ? JSON.parse(stdInput as string) : await fs.readJSON(path.join(pathPrefix, flags.in)) - } catch (err) { - throw new CLIError(err) - } - - flags.className = flags.className || app.name - - const dot_index = flags.className ? flags.className.lastIndexOf('.') : -1 - if (dot_index !== -1) { - space = flags.className.substr(0, dot_index) - flags.className = flags.className.substr(dot_index + 1) - } else { - flags.className = upperFirst(camelCase(flags.className)) - } - - this.reorderEntities(app, 'entities') - this.reorderEntities(app, 'prebuiltEntities') - this.reorderEntities(app, 'closedLists') - this.reorderEntities(app, 'regex_entities') - this.reorderEntities(app, 'patternAnyEntities') - this.reorderEntities(app, 'composites') - - const outputPath = flags.out ? file.validatePath(flags.out, flags.className + '.cs', flags.force) : flags.out - - this.log( - `Generating file at ${outputPath || 'stdout'} that contains class ${space}.${flags.className}.` - ) - - await LuisToCsConverter.writeFromLuisJson(app, flags.className, space, outputPath) - } -} diff --git a/packages/lu/src/commands/luis/generate/ts.ts b/packages/lu/src/commands/luis/generate/ts.ts deleted file mode 100644 index 6e99f1a2f..000000000 --- a/packages/lu/src/commands/luis/generate/ts.ts +++ /dev/null @@ -1,66 +0,0 @@ -/*! - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. - */ - -import {CLIError, Command, flags} from '@microsoft/bf-cli-command' -import {camelCase, kebabCase, upperFirst} from 'lodash' -import * as path from 'path' - -const LuisToTsConverter = require('./../../../parser/converters/luistotsconverter') -const file = require('./../../../utils/filehelper') -const fs = require('fs-extra') - -export default class LuisGenerateTs extends Command { - static description = 'Generate:ts generates a strongly typed typescript source code from an exported (json) LUIS model.' - - static flags: flags.Input = { - in: flags.string({char: 'i', description: 'Path to the file containing the LUIS application JSON model'}), - out: flags.string({char: 'o', description: 'Output file or folder name. If not specified stdout will be used as output', default: ''}), - className: flags.string({description: 'Name of the autogenerated class'}), - force: flags.boolean({char: 'f', description: 'If --out flag is provided with the path to an existing file, overwrites that file', default: false}), - help: flags.help({char: 'h', description: 'luis:generate:ts help'}) - - } - - reorderEntities(app: any, name: string): void { - if (app[name] !== null && app[name] !== undefined) { - app[name].sort((a: any, b: any) => (a.name > b.name ? 1 : -1)) - } - } - - async run() { - const {flags} = this.parse(LuisGenerateTs) - let stdInput = await this.readStdin() - - if (!flags.in && !stdInput) { - throw new CLIError('Missing input. Please use stdin or pass a file location with --in flag') - } - - const pathPrefix = flags.in && path.isAbsolute(flags.in) ? '' : process.cwd() - let app: any - try { - app = stdInput ? JSON.parse(stdInput as string) : await fs.readJSON(path.join(pathPrefix, flags.in)) - } catch (err) { - throw new CLIError(err) - } - - flags.className = flags.className || app.name - flags.className = upperFirst(camelCase(flags.className)) - - this.reorderEntities(app, 'entities') - this.reorderEntities(app, 'prebuiltEntities') - this.reorderEntities(app, 'closedLists') - this.reorderEntities(app, 'regex_entities') - this.reorderEntities(app, 'patternAnyEntities') - this.reorderEntities(app, 'composites') - - const outputPath = flags.out ? file.validatePath(flags.out, kebabCase(flags.className) + '.ts', flags.force) : flags.out - - this.log( - `Generating file at ${outputPath || 'stdout'} that contains class ${flags.className}.` - ) - - await LuisToTsConverter.writeFromLuisJson(app, flags.className, outputPath) - } -} diff --git a/packages/lu/src/commands/luis/translate.ts b/packages/lu/src/commands/luis/translate.ts deleted file mode 100644 index baf5ef110..000000000 --- a/packages/lu/src/commands/luis/translate.ts +++ /dev/null @@ -1,99 +0,0 @@ -/*! - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. - */ - -import {CLIError, Command, flags, utils} from '@microsoft/bf-cli-command' -const fs = require('fs-extra') -const path = require('path') -const fileHelper = require('./../../utils/filehelper') -const exception = require('./../../parser/utils/exception') -const luTranslator = require('./../../parser/translator/lutranslate') -const fileExtEnum = require('./../../parser/utils/helpers').FileExtTypeEnum -const Lu = require('./../../parser/lu/lu') -const Luis = require('./../../parser/luis/luis') - -export default class LuisTranslate extends Command { - static description = ' Translate given LUIS application JSON model or lu file(s)' - - static flags: flags.Input = { - in: flags.string({char: 'i', description: 'Source .lu file(s) or LUIS application JSON model'}), - recurse: flags.boolean({char: 'r', description: 'Indicates if sub-folders need to be considered to file .lu file(s)'}), - out: flags.string({char: 'o', description: 'Output folder name. If not specified stdout will be used as output'}), - srclang: flags.string({description: 'Source lang code. Auto detect if missing.'}), - tgtlang: flags.string({description: 'Comma separated list of target languages.', required: true}), - translatekey: flags.string({description: 'Machine translation endpoint key.', required: true}), - translate_comments: flags.boolean({description: 'When set, machine translate comments found in .lu file'}), - translate_link_text: flags.boolean({description: 'When set, machine translate link description in .lu file'}), - force: flags.boolean({char: 'f', description: 'If --out flag is provided with the path to an existing file, overwrites that file', default: false}), - help: flags.help({char: 'h', description: 'luis:translate help'}) - } - - /* tslint:disable:forin no-for-in*/ - async run() { - try { - const {flags} = this.parse(LuisTranslate) - // Check if data piped in stdin - let stdin = await this.readStdin() - let isLu = await fileHelper.detectLuContent(stdin, flags.in) - let result: any = {} - - if (isLu) { - let luFiles = await fileHelper.getLuObjects(stdin, flags.in, flags.recurse, fileExtEnum.LUFile) - let translatedLuFiles = await luTranslator.translateLuList(luFiles, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text) - luFiles.forEach((lu: any) => { - if (!result[lu.id]) { - result[lu.id] = {} - } - translatedLuFiles[lu.id].forEach((t: any) => { - result[t.id][t.language] = t.content - }) - }) - } else { - let json = stdin ? stdin : await fileHelper.getContentFromFile(flags.in) - let luisObject = new Luis(fileHelper.parseJSON(json, 'Luis')) - let key = stdin ? 'stdin' : path.basename(flags.in) - let translation = new Lu(luisObject.parseToLuContent(), key) - let translatedLuis = await luTranslator.translateLu(translation, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text) - result = { - [key] : {} - } - for (let lu of translatedLuis) { - result[key][lu.language] = await lu.parseToLuis() - } - } - - if (flags.out) { - await this.writeOutput(result, flags.out, isLu, flags.force) - } else { - if (isLu) { - this.log(result) - } else { - this.log(JSON.stringify(result, null, 2)) - } - } - - } catch (err) { - if (err instanceof exception) { - throw new CLIError(err.text) - } - throw err - } - } - - private async writeOutput(translatedObject: any, out: string, isLu: boolean, force: boolean) { - let filePath = '' - try { - for (let file in translatedObject) { - for (let lng in translatedObject[file]) { - filePath = await fileHelper.generateNewTranslatedFilePath(file, lng, out) - let content = isLu ? translatedObject[file][lng] : JSON.stringify(translatedObject[file][lng], null, 2) - const validatedPath = utils.validatePath(filePath, '', force) - await fs.writeFile(validatedPath, content, 'utf-8') - } - } - } catch (err) { - throw new CLIError('Unable to write file - ' + filePath + ' Error: ' + err.message) - } - } -} diff --git a/packages/lu/src/commands/qnamaker/convert.ts b/packages/lu/src/commands/qnamaker/convert.ts deleted file mode 100644 index 432c67867..000000000 --- a/packages/lu/src/commands/qnamaker/convert.ts +++ /dev/null @@ -1,107 +0,0 @@ -/*! - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. - */ - -import {CLIError, Command, flags, utils} from '@microsoft/bf-cli-command' -const exception = require('./../../parser/utils/exception') -const fs = require('fs-extra') -const file = require('./../../utils/filehelper') -const fileExtEnum = require('./../../parser/utils/helpers').FileExtTypeEnum - -const QnAMaker = require('./../../parser/qna/qnamaker/qnamaker') -const Alterations = require('./../../parser/qna/alterations/alterations') -const QnAMakerBuilder = require('./../../parser/qna/qnamaker/qnaMakerBuilder') -const alterationsBuilder = require('./../../parser/qna/alterations/alterationsBuilder') - -export default class QnamakerConvert extends Command { - static description = 'Converts .qna file(s) to QnA application JSON models or vice versa.' - - static flags: flags.Input = { - in: flags.string({char: 'i', description: 'Source .qna file(s) or QnA KB JSON file'}), - alterations: flags.boolean({description: 'Indicates if files is QnA Alterations'}), - log: flags.boolean({description: 'Enables log messages', default: false}), - sort: flags.boolean({description: 'When set, questions collections are alphabetically sorted are alphabetically sorted in .qna files', default: false}), - recurse: flags.boolean({char: 'r', description: 'Indicates if sub-folders need to be considered to file .qna file(s)'}), - out: flags.string({char: 'o', description: 'Output file or folder name. If not specified stdout will be used as output'}), - name: flags.string({description: 'Name of the QnA KB'}), - force: flags.boolean({char: 'f', description: 'If --out flag is provided with the path to an existing file, overwrites that file', default: false}), - help: flags.help({char: 'h', description: 'qnamaker:convert help'}) - } - - async run() { - try { - const {flags} = this.parse(QnamakerConvert) - - // Check if data piped in stdin - const stdin = await this.readStdin() - - //Check if file or folder - //if folder, only lu to luis is supported - const isQnA = await file.detectLuContent(stdin, flags.in) - - // Parse the object depending on the input - let result: any - if (isQnA) { - const luFiles = await file.getLuObjects(stdin, flags.in, flags.recurse, fileExtEnum.QnAFile) - result = {} - result.finalQnAJSON = await QnAMakerBuilder.build([...luFiles], false, flags.luis_culture) - result.finalQnAAlterations = await alterationsBuilder.build([...luFiles], false, flags.luis_culture) - } else { - const qnaContent = stdin ? stdin : await file.getContentFromFile(flags.in) - const QnA = flags.alterations ? new Alterations(file.parseJSON(qnaContent, 'QnA Alterations')) : new QnAMaker(file.parseJSON(qnaContent, 'QnA')) - if (flags.sort) { - QnA.sort() - } - result = QnA.parseToLuContent() - } - - // If result is null or undefined return - if (!result) { - throw new CLIError('No LU or QnA content parsed!') - } - - // Add headers to QnAJson - if (isQnA) { - result.finalQnAJSON.name = flags.name || result.name || '' - } - - // Print or write the parsed object - if (flags.out) { - await this.writeOutput(result, flags, isQnA) - } else { - if (isQnA) { - this.log(JSON.stringify(result.finalQnAJSON, null, 2)) - this.log(JSON.stringify(result.finalQnAAlterations, null, 2)) - } else { - this.log(result) - } - } - } catch (err) { - if (err instanceof exception) { - throw new CLIError(err.text) - } - throw err - } - } - - private async writeOutput(convertedObject: any, flags: any, isQnA: boolean) { - let filePath = await file.generateNewFilePath(flags.out, flags.in, isQnA, '', fileExtEnum.QnAFile) - const validatedPath = utils.validatePath(filePath, '', flags.force) - try { - if (isQnA) { - await fs.writeFile(validatedPath, JSON.stringify(convertedObject.finalQnAJSON, null, 2), 'utf-8') - if (convertedObject.finalQnAAlterations) { - let filePathAlterations = await file.generateNewFilePath(flags.out, flags.in, isQnA, 'alterations_', fileExtEnum.QnAFile) - const validatedPathAlter = utils.validatePath(filePathAlterations, '', flags.force) - await fs.writeFile(validatedPathAlter, JSON.stringify(convertedObject.finalQnAAlterations, null, 2), 'utf-8') - } - } else { - await fs.writeFile(validatedPath, convertedObject, 'utf-8') - } - } catch (err) { - throw new CLIError('Unable to write file - ' + validatedPath + ' Error: ' + err.message) - } - this.log('Successfully wrote QnA model to ' + validatedPath) - } -} diff --git a/packages/lu/src/commands/qnamaker/translate.ts b/packages/lu/src/commands/qnamaker/translate.ts deleted file mode 100644 index ba7c771f3..000000000 --- a/packages/lu/src/commands/qnamaker/translate.ts +++ /dev/null @@ -1,99 +0,0 @@ -/*! - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. - */ - -import {CLIError, Command, flags, utils} from '@microsoft/bf-cli-command' -const fs = require('fs-extra') -const path = require('path') -const fileHelper = require('./../../utils/filehelper') -const exception = require('./../../parser/utils/exception') -const luTranslator = require('./../../parser/translator/lutranslate') -const qnaMaker = require('./../../parser/qna/qnamaker/qnamaker') -const QnA = require('./../../parser/lu/qna') -const fileExtEnum = require('./../../parser/utils/helpers').FileExtTypeEnum - -export default class QnamakerTranslate extends Command { - static description = 'Translate given QnA maker application JSON model or qna file(s)' - - static flags: flags.Input = { - in: flags.string({char: 'i', description: 'Source .qna file(s) or QnA maker application JSON model'}), - recurse: flags.boolean({char: 'r', description: 'Indicates if sub-folders need to be considered to find .qna file(s)'}), - out: flags.string({char: 'o', description: 'Output folder name. If not specified stdout will be used as output'}), - srclang: flags.string({description: 'Source lang code. Auto detect if missing.'}), - tgtlang: flags.string({description: 'Comma separated list of target languages.', required: true}), - translatekey: flags.string({description: 'Machine translation endpoint key.', required: true}), - translate_comments: flags.boolean({description: 'When set, machine translate comments found in .qna file'}), - translate_link_text: flags.boolean({description: 'When set, machine translate link description in .qna file'}), - force: flags.boolean({char: 'f', description: 'If --out flag is provided with the path to an existing file, overwrites that file', default: false}), - help: flags.help({char: 'h', description: 'qnamaker:translate help'}) - } - - /* tslint:disable:forin no-for-in*/ - async run() { - try { - const {flags} = this.parse(QnamakerTranslate) - // Check if data piped in stdin - let stdin = await this.readStdin() - - let isLu = await fileHelper.detectLuContent(stdin, flags.in) - let result: any = {} - if (isLu) { - let luFiles = await fileHelper.getLuObjects(stdin, flags.in, flags.recurse, fileExtEnum.QnAFile) - let translatedLuFiles = await luTranslator.translateQnAList(luFiles, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text) - luFiles.forEach((lu: any) => { - if (!result[lu.id]) { - result[lu.id] = {} - } - translatedLuFiles[lu.id].forEach((t: any) => { - result[t.id][t.language] = t.content - }) - }) - } else { - let json = stdin ? stdin : await fileHelper.getContentFromFile(flags.in) - let qnaM = new qnaMaker(fileHelper.parseJSON(json, 'QnA')) - let qna = new QnA(qnaM.parseToLuContent()) - let qnaTranslation = await luTranslator.translateQnA(qna, flags.translatekey, flags.tgtlang, flags.srclang, flags.translate_comments, flags.translate_link_text) - let key = stdin ? 'stdin' : path.basename(flags.in) - result = { - [key] : {} - } - for (let q of qnaTranslation) { - result[key][q.language] = await q.parseToQna() - } - } - - if (flags.out) { - await this.writeOutput(result, flags.out, isLu, flags.force) - } else { - if (isLu) { - this.log(result) - } else { - this.log(JSON.stringify(result, null, 2)) - } - } - - } catch (err) { - if (err instanceof exception) { - throw new CLIError(err.text) - } - throw err - } - } - - private async writeOutput(translatedObject: any, out: string, isLu: boolean, force: boolean) { - let filePath = '' - try { - for (let file in translatedObject) { - for (let lng in translatedObject[file]) { - filePath = await fileHelper.generateNewTranslatedFilePath(file, lng, out) - let content = isLu ? translatedObject[file][lng] : JSON.stringify(translatedObject[file][lng], null, 2) - const validatedPath = utils.validatePath(filePath, '', force) - await fs.writeFile(validatedPath, content, 'utf-8') - } - } - } catch (err) { - throw new CLIError('Unable to write file - ' + filePath + ' Error: ' + err.message) - } - } -} diff --git a/packages/lu/src/parser/luis/luisGenBuilder.js b/packages/lu/src/parser/luis/luisGenBuilder.js index e2b5621f6..9532ba13f 100644 --- a/packages/lu/src/parser/luis/luisGenBuilder.js +++ b/packages/lu/src/parser/luis/luisGenBuilder.js @@ -1,6 +1,7 @@ const LuisGen = require('./luisGen') const propertyHelper = require('./propertyHelper') -const {CLIError} = require('@microsoft/bf-cli-command') +const error = require('./../utils/exception') +const retCode = require('./../utils/enums/CLI-errors') class LuisGenBuilder { static build(luisApp) { @@ -14,7 +15,7 @@ class LuisGenBuilder { result.patternAnyEntities = extractEntities(luisApp.patternAnyEntities); result.composites = extractComposites(luisApp.composites); } catch (err) { - throw new CLIError("Invalid LUIS JSON file content.") + throw (new error(retCode.errorCode.INVALID_INPUT_FILE, "Invalid LUIS JSON file content.")) } return result } diff --git a/packages/lu/src/utils/filehelper.ts b/packages/lu/src/utils/filehelper.ts index 43cb72988..fc9d1c73a 100644 --- a/packages/lu/src/utils/filehelper.ts +++ b/packages/lu/src/utils/filehelper.ts @@ -3,7 +3,9 @@ * Licensed under the MIT License. */ -import {CLIError, utils} from '@microsoft/bf-cli-command' +import {readTextFile} from './textfilereader' +const exception = require('./../parser/utils/exception') +const retCode = require('./../parser/utils/enums/CLI-errors') const fs = require('fs-extra') const path = require('path') const helpers = require('./../parser/utils/helpers') @@ -35,13 +37,13 @@ export async function getLuFiles(input: string | undefined, recurse = false, ext } if (!fileStat.isDirectory()) { - throw new CLIError('Sorry, ' + input + ' is not a folder or does not exist') + throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, 'Sorry, ' + input + ' is not a folder or does not exist')) } filesToParse = helpers.findLUFiles(input, recurse, extType) if (filesToParse.length === 0) { - throw new CLIError(`Sorry, no ${extType} files found in the specified folder.`) + throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, `Sorry, no ${extType} files found in the specified folder.`)) } return filesToParse } @@ -49,16 +51,16 @@ export async function getLuFiles(input: string | undefined, recurse = false, ext export async function getContentFromFile(file: string) { // catch if input file is a folder if (fs.lstatSync(file).isDirectory()) { - throw new CLIError('Sorry, "' + file + '" is a directory! Unable to read as a file') + throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, 'Sorry, "' + file + '" is a directory! Unable to read as a file')) } if (!fs.existsSync(path.resolve(file))) { - throw new CLIError('Sorry [' + file + '] does not exist') + throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, 'Sorry [' + file + '] does not exist')) } let fileContent try { - fileContent = await utils.readTextFile(file) + fileContent = await readTextFile(file) } catch (err) { - throw new CLIError('Sorry, error reading file: ' + file) + throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, 'Sorry, error reading file: ' + file)) } return fileContent } @@ -67,7 +69,7 @@ export async function generateNewFilePath(outFileName: string, inputfile: string let base = path.resolve(outFileName) let root = path.dirname(base) if (!fs.existsSync(root)) { - throw new CLIError('Path not found: ' + root) + throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, 'Path not found: ' + root)) } let extension = path.extname(base) @@ -90,11 +92,11 @@ export async function generateNewTranslatedFilePath(fileName: string, translated let extension = path.extname(newPath) if (extension) { - throw new CLIError('Output can only be writen to a folder') + throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, 'Output can only be writen to a folder')) } if (!fs.existsSync(newPath)) { - throw new CLIError('Path not found: ' + newPath) + throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, 'Path not found: ' + newPath)) } newPath = path.join(output, translatedLanguage) @@ -107,7 +109,7 @@ export function validatePath(outputPath: string, defaultFileName: string, forceW const containingDir = path.dirname(completePath) // If the cointaining folder doesnt exist - if (!fs.existsSync(containingDir)) throw new CLIError(`Containing directory path doesn't exist: ${containingDir}`) + if (!fs.existsSync(containingDir)) throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, `Containing directory path doesn't exist: ${containingDir}`)) const baseElement = path.basename(completePath) const pathAlreadyExist = fs.existsSync(completePath) @@ -118,7 +120,7 @@ export function validatePath(outputPath: string, defaultFileName: string, forceW } // If the last element in the path is a folder - if (!pathAlreadyExist) throw new CLIError(`Target directory path doesn't exist: ${completePath}`) + if (!pathAlreadyExist) throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, `Target directory path doesn't exist: ${completePath}`)) completePath = path.join(completePath, defaultFileName) return fs.existsSync(completePath) && !forceWrite ? enumerateFileName(completePath) : completePath } @@ -127,7 +129,7 @@ function enumerateFileName(filePath: string): string { const fileName = path.basename(filePath) const containingDir = path.dirname(filePath) - if (!fs.existsSync(containingDir)) throw new CLIError(`Containing directory path doesn't exist: ${containingDir}`) + if (!fs.existsSync(containingDir)) throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, `Containing directory path doesn't exist: ${containingDir}`)) const extension = path.extname(fileName) const baseName = path.basename(fileName, extension) @@ -143,12 +145,12 @@ function enumerateFileName(filePath: string): string { export async function detectLuContent(stdin: string, input: string) { if (!stdin && !input) { - throw new CLIError('Missing input. Please use stdin or pass a file location with --in flag') + throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, 'Missing input. Please use stdin or pass a file location with --in flag')) } if (!stdin) { if (!fs.existsSync(path.resolve(input))) { - throw new CLIError(`Sorry unable to open [${input}]`) + throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, `Sorry unable to open [${input}]`)) } let inputStat = await fs.stat(input) @@ -167,7 +169,7 @@ export function parseJSON(input: string, appType: string) { try { return JSON.parse(input) } catch (error) { - throw new CLIError(`Sorry, error parsing content as ${appType} JSON`) + throw (new exception(retCode.errorCode.INVALID_INPUT_FILE, `Sorry, error parsing content as ${appType} JSON`)) } } diff --git a/packages/lu/src/utils/textfilereader.ts b/packages/lu/src/utils/textfilereader.ts new file mode 100644 index 000000000..ee6d9de47 --- /dev/null +++ b/packages/lu/src/utils/textfilereader.ts @@ -0,0 +1,51 @@ +/*! + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. + */ + +const fs = require('fs-extra') +const error = require('./../parser/utils/exception') +const retCode = require('./../parser/utils/enums/CLI-errors') + +export async function readTextFile(file: any): Promise { + return new Promise(async (resolve, reject) => { + try { + if (!fs.existsSync(file)) { + return reject('ENOENT: no such file or directory, ' + file) + } + let fileBuffer = await fs.readFile(file) + if (fileBuffer) { + // If the data starts with BOM, we know it is UTF + if (fileBuffer[0] === 0xEF && fileBuffer[1] === 0xBB && fileBuffer[2] === 0xBF) { + // EF BB BF UTF-8 with BOM + fileBuffer = fileBuffer.slice(3) + } else if (fileBuffer[0] === 0xFF && fileBuffer[1] === 0xFE && fileBuffer[2] === 0x00 && fileBuffer[3] === 0x00) { + // FF FE 00 00 UTF-32, little-endian BOM + fileBuffer = fileBuffer.slice(4) + } else if (fileBuffer[0] === 0x00 && fileBuffer[1] === 0x00 && fileBuffer[2] === 0xFE && fileBuffer[3] === 0xFF) { + // 00 00 FE FF UTF-32, big-endian BOM + fileBuffer = fileBuffer.slice(4) + } else if (fileBuffer[0] === 0xFE && fileBuffer[1] === 0xFF && fileBuffer[2] === 0x00 && fileBuffer[3] === 0x00) { + // FE FF 00 00 UCS-4, unusual octet order BOM (3412) + fileBuffer = fileBuffer.slice(4) + } else if (fileBuffer[0] === 0x00 && fileBuffer[1] === 0x00 && fileBuffer[2] === 0xFF && fileBuffer[3] === 0xFE) { + // 00 00 FF FE UCS-4, unusual octet order BOM (2143) + fileBuffer = fileBuffer.slice(4) + } else if (fileBuffer[0] === 0xFF && fileBuffer[1] === 0xFE) { + // FF FE UTF-16, little endian BOM + fileBuffer = fileBuffer.slice(2) + } else if (fileBuffer[0] === 0xFE && fileBuffer[1] === 0xFF) { + // FE FF UTF-16, big endian BOM + fileBuffer = fileBuffer.slice(2) + } + } + return resolve(fileBuffer.toString('utf8').replace(/\0/g, '')) + } catch (err) { + if (err.message.match(/ENOENT: no such file or directory/)) { + return reject(new error(retCode.errorCode.INVALID_INPUT_FILE, err.message)) + } + + return reject(`Invalid Input. Sorry, unable to parse file: ${err}`) + } + }) +} diff --git a/packages/lu/test/commands/luis/translate.test.ts b/packages/lu/test/commands/luis/translate.test.ts index f25237c40..3f84eeed9 100644 --- a/packages/lu/test/commands/luis/translate.test.ts +++ b/packages/lu/test/commands/luis/translate.test.ts @@ -59,7 +59,7 @@ xdescribe('luis:translate luis json', async () => { }) }) -describe('luis:translate Phrase list entity references are translated correctly', async () => { +xdescribe('luis:translate Phrase list entity references are translated correctly', async () => { const response = require('./../../fixtures/translation/serviceresponses/phraseList.json') after(async function(){ await fs.remove(path.join(__dirname, './../../../de/')) diff --git a/packages/lu/test/utils/filehelper.test.js b/packages/lu/test/utils/filehelper.test.js index 90fe3f7b7..1eaec3fde 100644 --- a/packages/lu/test/utils/filehelper.test.js +++ b/packages/lu/test/utils/filehelper.test.js @@ -1,4 +1,4 @@ -const utils = require('@microsoft/bf-cli-command').utils +import {readTextFile} from './../../src/utils/textfilereader' const expect = require('chai').expect; const fileHelper = require('./../../src/utils/filehelper') const luObject = require('./../../src/parser/lu/lu') @@ -6,20 +6,15 @@ const path = require('path') describe('utils/filehelper test', () => { it('File helper correctly builds a luObject list from a file', async function(){ - try{ let expected = [] let pathToFile = path.resolve(path.join(__dirname, './../fixtures/file.lu')) - let content = await utils.readTextFile(pathToFile) + let content = await readTextFile(pathToFile) expected.push(new luObject(content, pathToFile)) let luObjArray = await fileHelper.getLuObjects('', pathToFile) expect(luObjArray).to.deep.equal(expected) - }catch(err){ - console.log(err) - } }) it('File helper correctly builds a luObject list from stdin', async function(){ - try{ let content = `> Definition for greeting intent # Greeting - Hi @@ -31,8 +26,5 @@ describe('utils/filehelper test', () => { let expected = [] expected.push(new luObject(content, 'stdin')) expect(luObjArray).to.deep.equal(expected) - }catch(err){ - console.log(err) - } }) }) \ No newline at end of file diff --git a/packages/luis/src/commands/luis/build.ts b/packages/luis/src/commands/luis/build.ts index 4bd2fb584..a7685663f 100644 --- a/packages/luis/src/commands/luis/build.ts +++ b/packages/luis/src/commands/luis/build.ts @@ -6,7 +6,7 @@ import {Command, flags} from '@microsoft/bf-cli-command' const path = require('path') const fs = require('fs-extra') -import * as file from './../../utils/filehelper' +const file = require('./../../../node_modules/@microsoft/bf-lu/lib/utils/filehelper') const fileExtEnum = require('./../../../node_modules/@microsoft/bf-lu/lib/parser/utils/helpers').FileExtTypeEnum const Content = require('./../../../node_modules/@microsoft/bf-lu/lib/parser/lu/lu') const Settings = require('./../../../node_modules/@microsoft/bf-lu/lib/parser/lubuild/settings') diff --git a/packages/luis/src/commands/luis/convert.ts b/packages/luis/src/commands/luis/convert.ts index 3183a13ca..f45053098 100644 --- a/packages/luis/src/commands/luis/convert.ts +++ b/packages/luis/src/commands/luis/convert.ts @@ -5,7 +5,7 @@ import {CLIError, Command, flags, utils} from '@microsoft/bf-cli-command' const fs = require('fs-extra') -import * as file from './../../utils/filehelper' +const file = require('./../../../node_modules/@microsoft/bf-lu/lib/utils/filehelper') const exception = require('./../../../node_modules/@microsoft/bf-lu/lib/parser/utils/exception') const fileExtEnum = require('./../../../node_modules/@microsoft/bf-lu/lib/parser/utils/helpers').FileExtTypeEnum const Luis = require('./../../../node_modules/@microsoft/bf-lu/lib/parser/luis/luis') diff --git a/packages/luis/src/commands/luis/generate/cs.ts b/packages/luis/src/commands/luis/generate/cs.ts index 580015ca6..67de1f386 100644 --- a/packages/luis/src/commands/luis/generate/cs.ts +++ b/packages/luis/src/commands/luis/generate/cs.ts @@ -7,7 +7,8 @@ import {CLIError, Command, flags} from '@microsoft/bf-cli-command' import {camelCase, upperFirst} from 'lodash' import * as path from 'path' const fs = require('fs-extra') -import * as file from './../../../utils/filehelper' +const file = require('./../../../../node_modules/@microsoft/bf-lu/lib/utils/filehelper') +const exception = require('./../../../../node_modules/@microsoft/bf-lu/lib/parser/utils/exception') const LuisToCsConverter = require('./../../../../node_modules/@microsoft/bf-lu/lib/parser/converters/luistocsconverter') export default class LuisGenerateCs extends Command { @@ -29,45 +30,52 @@ export default class LuisGenerateCs extends Command { } async run() { - const {flags} = this.parse(LuisGenerateCs) - let space = 'Luis' - const stdInput = await this.readStdin() + try { + const {flags} = this.parse(LuisGenerateCs) + let space = 'Luis' + const stdInput = await this.readStdin() - if (!flags.in && !stdInput) { - throw new CLIError('Missing input. Please use stdin or pass a file location with --in flag') - } + if (!flags.in && !stdInput) { + throw new CLIError('Missing input. Please use stdin or pass a file location with --in flag') + } - const pathPrefix = flags.in && path.isAbsolute(flags.in) ? '' : process.cwd() - let app: any - try { - app = stdInput ? JSON.parse(stdInput as string) : await fs.readJSON(path.join(pathPrefix, flags.in)) - } catch (error) { - throw new CLIError(error) - } + const pathPrefix = flags.in && path.isAbsolute(flags.in) ? '' : process.cwd() + let app: any + try { + app = stdInput ? JSON.parse(stdInput as string) : await fs.readJSON(path.join(pathPrefix, flags.in)) + } catch (error) { + throw new CLIError(error) + } - flags.className = flags.className || app.name + flags.className = flags.className || app.name - const dot_index = flags.className ? flags.className.lastIndexOf('.') : -1 - if (dot_index !== -1) { - space = flags.className.substr(0, dot_index) - flags.className = flags.className.substr(dot_index + 1) - } else { - flags.className = upperFirst(camelCase(flags.className)) - } + const dot_index = flags.className ? flags.className.lastIndexOf('.') : -1 + if (dot_index !== -1) { + space = flags.className.substr(0, dot_index) + flags.className = flags.className.substr(dot_index + 1) + } else { + flags.className = upperFirst(camelCase(flags.className)) + } - this.reorderEntities(app, 'entities') - this.reorderEntities(app, 'prebuiltEntities') - this.reorderEntities(app, 'closedLists') - this.reorderEntities(app, 'regex_entities') - this.reorderEntities(app, 'patternAnyEntities') - this.reorderEntities(app, 'composites') + this.reorderEntities(app, 'entities') + this.reorderEntities(app, 'prebuiltEntities') + this.reorderEntities(app, 'closedLists') + this.reorderEntities(app, 'regex_entities') + this.reorderEntities(app, 'patternAnyEntities') + this.reorderEntities(app, 'composites') - const outputPath = flags.out ? file.validatePath(flags.out, flags.className + '.cs', flags.force) : flags.out + const outputPath = flags.out ? file.validatePath(flags.out, flags.className + '.cs', flags.force) : flags.out - this.log( - `Generating file at ${outputPath || 'stdout'} that contains class ${space}.${flags.className}.` - ) + this.log( + `Generating file at ${outputPath || 'stdout'} that contains class ${space}.${flags.className}.` + ) - await LuisToCsConverter.writeFromLuisJson(app, flags.className, space, outputPath) + await LuisToCsConverter.writeFromLuisJson(app, flags.className, space, outputPath) + } catch (error) { + if (error instanceof exception) { + throw new CLIError(error.text) + } + throw error + } } } diff --git a/packages/luis/src/commands/luis/generate/ts.ts b/packages/luis/src/commands/luis/generate/ts.ts index 615f52ee4..0fa209951 100644 --- a/packages/luis/src/commands/luis/generate/ts.ts +++ b/packages/luis/src/commands/luis/generate/ts.ts @@ -7,7 +7,8 @@ import {CLIError, Command, flags} from '@microsoft/bf-cli-command' import {camelCase, kebabCase, upperFirst} from 'lodash' import * as path from 'path' const fs = require('fs-extra') -import * as file from './../../../utils/filehelper' +const exception = require('./../../../../node_modules/@microsoft/bf-lu/lib/parser/utils/exception') +const file = require('./../../../../node_modules/@microsoft/bf-lu/lib/utils/filehelper') const LuisToTsConverter = require('./../../../../node_modules/@microsoft/bf-lu/lib/parser/converters/luistotsconverter') export default class LuisGenerateTs extends Command { @@ -19,7 +20,6 @@ export default class LuisGenerateTs extends Command { className: flags.string({description: 'Name of the autogenerated class'}), force: flags.boolean({char: 'f', description: 'If --out flag is provided with the path to an existing file, overwrites that file', default: false}), help: flags.help({char: 'h', description: 'luis:generate:ts help'}), - } reorderEntities(app: any, name: string): void { @@ -29,37 +29,44 @@ export default class LuisGenerateTs extends Command { } async run() { - const {flags} = this.parse(LuisGenerateTs) - const stdInput = await this.readStdin() + try { + const {flags} = this.parse(LuisGenerateTs) + const stdInput = await this.readStdin() - if (!flags.in && !stdInput) { - throw new CLIError('Missing input. Please use stdin or pass a file location with --in flag') - } + if (!flags.in && !stdInput) { + throw new CLIError('Missing input. Please use stdin or pass a file location with --in flag') + } - const pathPrefix = flags.in && path.isAbsolute(flags.in) ? '' : process.cwd() - let app: any - try { - app = stdInput ? JSON.parse(stdInput as string) : await fs.readJSON(path.join(pathPrefix, flags.in)) - } catch (error) { - throw new CLIError(error) - } + const pathPrefix = flags.in && path.isAbsolute(flags.in) ? '' : process.cwd() + let app: any + try { + app = stdInput ? JSON.parse(stdInput as string) : await fs.readJSON(path.join(pathPrefix, flags.in)) + } catch (error) { + throw new CLIError(error) + } - flags.className = flags.className || app.name - flags.className = upperFirst(camelCase(flags.className)) + flags.className = flags.className || app.name + flags.className = upperFirst(camelCase(flags.className)) - this.reorderEntities(app, 'entities') - this.reorderEntities(app, 'prebuiltEntities') - this.reorderEntities(app, 'closedLists') - this.reorderEntities(app, 'regex_entities') - this.reorderEntities(app, 'patternAnyEntities') - this.reorderEntities(app, 'composites') + this.reorderEntities(app, 'entities') + this.reorderEntities(app, 'prebuiltEntities') + this.reorderEntities(app, 'closedLists') + this.reorderEntities(app, 'regex_entities') + this.reorderEntities(app, 'patternAnyEntities') + this.reorderEntities(app, 'composites') - const outputPath = flags.out ? file.validatePath(flags.out, kebabCase(flags.className) + '.ts', flags.force) : flags.out + const outputPath = flags.out ? file.validatePath(flags.out, kebabCase(flags.className) + '.ts', flags.force) : flags.out - this.log( - `Generating file at ${outputPath || 'stdout'} that contains class ${flags.className}.` - ) + this.log( + `Generating file at ${outputPath || 'stdout'} that contains class ${flags.className}.` + ) - await LuisToTsConverter.writeFromLuisJson(app, flags.className, outputPath) + await LuisToTsConverter.writeFromLuisJson(app, flags.className, outputPath) + } catch (error) { + if (error instanceof exception) { + throw new CLIError(error.text) + } + throw error + } } } diff --git a/packages/luis/src/commands/luis/translate.ts b/packages/luis/src/commands/luis/translate.ts index 7bbe20a1c..fb20bf8bf 100644 --- a/packages/luis/src/commands/luis/translate.ts +++ b/packages/luis/src/commands/luis/translate.ts @@ -6,7 +6,7 @@ import {CLIError, Command, flags, utils} from '@microsoft/bf-cli-command' const fs = require('fs-extra') const path = require('path') -import * as fileHelper from './../../utils/filehelper' +const fileHelper = require('./../../../node_modules/@microsoft/bf-lu/lib/utils/filehelper') const exception = require('./../../../node_modules/@microsoft/bf-lu/lib/parser/utils/exception') const luTranslator = require('./../../../node_modules/@microsoft/bf-lu/lib/parser/translator/lutranslate') const fileExtEnum = require('./../../../node_modules/@microsoft/bf-lu/lib/parser/utils/helpers').FileExtTypeEnum diff --git a/packages/luis/src/utils/filehelper.ts b/packages/luis/src/utils/filehelper.ts deleted file mode 100644 index feef11d86..000000000 --- a/packages/luis/src/utils/filehelper.ts +++ /dev/null @@ -1,195 +0,0 @@ -/*! - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. - */ - -import {CLIError, utils} from '@microsoft/bf-cli-command' -const fs = require('fs-extra') -const path = require('path') -const helpers = require('./../../node_modules/@microsoft/bf-lu/lib/parser/utils/helpers') -const LuObject = require('./../../node_modules/@microsoft/bf-lu/lib/parser/lu/lu') - -/* tslint:disable:prefer-for-of no-unused */ - -export async function getLuFiles(input: string | undefined, recurse = false, extType: string | undefined): Promise> { - let filesToParse: any[] = [] - const fileStat = await fs.stat(input) - if (fileStat.isFile()) { - filesToParse.push(path.resolve(input)) - return filesToParse - } - - if (!fileStat.isDirectory()) { - throw new CLIError('Sorry, ' + input + ' is not a folder or does not exist') - } - - filesToParse = helpers.findLUFiles(input, recurse, extType) - - if (filesToParse.length === 0) { - throw new CLIError(`Sorry, no ${extType} files found in the specified folder.`) - } - return filesToParse -} - -export async function getContentFromFile(file: string) { - // catch if input file is a folder - if (fs.lstatSync(file).isDirectory()) { - throw new CLIError('Sorry, "' + file + '" is a directory! Unable to read as a file') - } - if (!fs.existsSync(path.resolve(file))) { - throw new CLIError('Sorry [' + file + '] does not exist') - } - let fileContent - try { - fileContent = await utils.readTextFile(file) - } catch (error) { - throw new CLIError('Sorry, error reading file: ' + file) - } - return fileContent -} - -export async function getLuObjects(stdin: string, input: string | undefined, recurse = false, extType: string | undefined) { - const luObjects: any = [] - if (stdin) { - luObjects.push(new LuObject(stdin, 'stdin')) - } else { - const luFiles = await getLuFiles(input, recurse, extType) - for (let i = 0; i < luFiles.length; i++) { - const luContent = await getContentFromFile(luFiles[i]) - luObjects.push(new LuObject(luContent, path.resolve(luFiles[i]))) - } - } - - return luObjects -} - -export async function generateNewFilePath(outFileName: string, inputfile: string, isLu: boolean, prefix = '', extType: string = helpers.FileExtTypeEnum.LUFile): Promise { - const base = path.resolve(outFileName) - const root = path.dirname(base) - if (!fs.existsSync(root)) { - throw new CLIError('Path not found: ' + root) - } - - const extension = path.extname(base) - if (extension) { - return path.join(root, prefix + path.basename(base)) - } - - let name = '' - const inputStat = await fs.stat(inputfile) - if (inputStat.isFile()) { - name += path.basename(inputfile, path.extname(inputfile)) + (isLu ? '.json' : extType) - } else { - name += isLu ? 'converted.json' : `converted.${extType}` - } - return path.join(base, prefix + name) -} - -export async function generateNewTranslatedFilePath(fileName: string, translatedLanguage: string, output: string): Promise { - let newPath = path.resolve(output) - - const extension = path.extname(newPath) - if (extension) { - throw new CLIError('Output can only be writen to a folder') - } - - if (!fs.existsSync(newPath)) { - throw new CLIError('Path not found: ' + newPath) - } - - newPath = path.join(output, translatedLanguage) - await fs.mkdirp(newPath) - return path.join(newPath, path.basename(fileName)) -} - -function enumerateFileName(filePath: string): string { - const fileName = path.basename(filePath) - const containingDir = path.dirname(filePath) - - if (!fs.existsSync(containingDir)) throw new CLIError(`Containing directory path doesn't exist: ${containingDir}`) - - const extension = path.extname(fileName) - const baseName = path.basename(fileName, extension) - let nextNumber = 0 - let newPath = '' - - do { - newPath = path.join(containingDir, baseName + `(${++nextNumber})` + extension) - } while (fs.existsSync(newPath)) - - return newPath -} - -export function validatePath(outputPath: string, defaultFileName: string, forceWrite = false): string { - let completePath = path.resolve(outputPath) - const containingDir = path.dirname(completePath) - - // If the cointaining folder doesnt exist - if (!fs.existsSync(containingDir)) throw new CLIError(`Containing directory path doesn't exist: ${containingDir}`) - - const baseElement = path.basename(completePath) - const pathAlreadyExist = fs.existsSync(completePath) - - // If the last element in the path is a file - if (baseElement.includes('.')) { - return pathAlreadyExist && !forceWrite ? enumerateFileName(completePath) : completePath - } - - // If the last element in the path is a folder - if (!pathAlreadyExist) throw new CLIError(`Target directory path doesn't exist: ${completePath}`) - completePath = path.join(completePath, defaultFileName) - return fs.existsSync(completePath) && !forceWrite ? enumerateFileName(completePath) : completePath -} - -export async function detectLuContent(stdin: string, input: string) { - if (!stdin && !input) { - throw new CLIError('Missing input. Please use stdin or pass a file location with --in flag') - } - - if (!stdin) { - if (!fs.existsSync(path.resolve(input))) { - throw new CLIError(`Sorry unable to open [${input}]`) - } - - const inputStat = await fs.stat(input) - return !inputStat.isFile() ? true : (path.extname(input) === '.lu' || path.extname(input) === '.qna') - } - - try { - await JSON.parse(stdin) - } catch (error) { - return true - } - return false -} - -export function parseJSON(input: string, appType: string) { - try { - return JSON.parse(input) - } catch (error) { - throw new CLIError(`Sorry, error parsing content as ${appType} JSON`) - } -} - -export function getCultureFromPath(file: string): string | null { - const fn = path.basename(file, path.extname(file)) - const lang = path.extname(fn).substring(1) - switch (lang.toLowerCase()) { - case 'en-us': - case 'zh-cn': - case 'nl-nl': - case 'fr-fr': - case 'fr-ca': - case 'de-de': - case 'it-it': - case 'ja-jp': - case 'ko-kr': - case 'pt-br': - case 'es-es': - case 'es-mx': - case 'tr-tr': - return lang - default: - return null - } -} diff --git a/packages/qnamaker/src/commands/qnamaker/convert.ts b/packages/qnamaker/src/commands/qnamaker/convert.ts index e3eed2ee9..f1ee066e6 100644 --- a/packages/qnamaker/src/commands/qnamaker/convert.ts +++ b/packages/qnamaker/src/commands/qnamaker/convert.ts @@ -4,7 +4,7 @@ */ import {CLIError, Command, flags, utils} from '@microsoft/bf-cli-command' -import * as file from './../../utils/filehelper' +const file = require('./../../../node_modules/@microsoft/bf-lu/lib/utils/filehelper') const exception = require('./../../../node_modules/@microsoft/bf-lu/lib/parser/utils/exception') const fs = require('fs-extra') const fileExtEnum = require('./../../../node_modules/@microsoft/bf-lu/lib/parser/utils/helpers').FileExtTypeEnum @@ -78,6 +78,7 @@ export default class QnamakerConvert extends Command { this.log(result) } } catch (error) { + console.log(error) if (error instanceof exception) { throw new CLIError(error.text) } diff --git a/packages/qnamaker/src/commands/qnamaker/translate.ts b/packages/qnamaker/src/commands/qnamaker/translate.ts index cb56a0674..abb3f4e4c 100644 --- a/packages/qnamaker/src/commands/qnamaker/translate.ts +++ b/packages/qnamaker/src/commands/qnamaker/translate.ts @@ -4,7 +4,7 @@ */ import {CLIError, Command, flags, utils} from '@microsoft/bf-cli-command' -import * as fileHelper from './../../utils/filehelper' +const fileHelper = require('./../../../node_modules/@microsoft/bf-lu/lib/utils/filehelper') const fs = require('fs-extra') const path = require('path') const exception = require('./../../../node_modules/@microsoft/bf-lu/lib/parser/utils/exception') diff --git a/packages/qnamaker/src/utils/filehelper.ts b/packages/qnamaker/src/utils/filehelper.ts deleted file mode 100644 index feef11d86..000000000 --- a/packages/qnamaker/src/utils/filehelper.ts +++ /dev/null @@ -1,195 +0,0 @@ -/*! - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. - */ - -import {CLIError, utils} from '@microsoft/bf-cli-command' -const fs = require('fs-extra') -const path = require('path') -const helpers = require('./../../node_modules/@microsoft/bf-lu/lib/parser/utils/helpers') -const LuObject = require('./../../node_modules/@microsoft/bf-lu/lib/parser/lu/lu') - -/* tslint:disable:prefer-for-of no-unused */ - -export async function getLuFiles(input: string | undefined, recurse = false, extType: string | undefined): Promise> { - let filesToParse: any[] = [] - const fileStat = await fs.stat(input) - if (fileStat.isFile()) { - filesToParse.push(path.resolve(input)) - return filesToParse - } - - if (!fileStat.isDirectory()) { - throw new CLIError('Sorry, ' + input + ' is not a folder or does not exist') - } - - filesToParse = helpers.findLUFiles(input, recurse, extType) - - if (filesToParse.length === 0) { - throw new CLIError(`Sorry, no ${extType} files found in the specified folder.`) - } - return filesToParse -} - -export async function getContentFromFile(file: string) { - // catch if input file is a folder - if (fs.lstatSync(file).isDirectory()) { - throw new CLIError('Sorry, "' + file + '" is a directory! Unable to read as a file') - } - if (!fs.existsSync(path.resolve(file))) { - throw new CLIError('Sorry [' + file + '] does not exist') - } - let fileContent - try { - fileContent = await utils.readTextFile(file) - } catch (error) { - throw new CLIError('Sorry, error reading file: ' + file) - } - return fileContent -} - -export async function getLuObjects(stdin: string, input: string | undefined, recurse = false, extType: string | undefined) { - const luObjects: any = [] - if (stdin) { - luObjects.push(new LuObject(stdin, 'stdin')) - } else { - const luFiles = await getLuFiles(input, recurse, extType) - for (let i = 0; i < luFiles.length; i++) { - const luContent = await getContentFromFile(luFiles[i]) - luObjects.push(new LuObject(luContent, path.resolve(luFiles[i]))) - } - } - - return luObjects -} - -export async function generateNewFilePath(outFileName: string, inputfile: string, isLu: boolean, prefix = '', extType: string = helpers.FileExtTypeEnum.LUFile): Promise { - const base = path.resolve(outFileName) - const root = path.dirname(base) - if (!fs.existsSync(root)) { - throw new CLIError('Path not found: ' + root) - } - - const extension = path.extname(base) - if (extension) { - return path.join(root, prefix + path.basename(base)) - } - - let name = '' - const inputStat = await fs.stat(inputfile) - if (inputStat.isFile()) { - name += path.basename(inputfile, path.extname(inputfile)) + (isLu ? '.json' : extType) - } else { - name += isLu ? 'converted.json' : `converted.${extType}` - } - return path.join(base, prefix + name) -} - -export async function generateNewTranslatedFilePath(fileName: string, translatedLanguage: string, output: string): Promise { - let newPath = path.resolve(output) - - const extension = path.extname(newPath) - if (extension) { - throw new CLIError('Output can only be writen to a folder') - } - - if (!fs.existsSync(newPath)) { - throw new CLIError('Path not found: ' + newPath) - } - - newPath = path.join(output, translatedLanguage) - await fs.mkdirp(newPath) - return path.join(newPath, path.basename(fileName)) -} - -function enumerateFileName(filePath: string): string { - const fileName = path.basename(filePath) - const containingDir = path.dirname(filePath) - - if (!fs.existsSync(containingDir)) throw new CLIError(`Containing directory path doesn't exist: ${containingDir}`) - - const extension = path.extname(fileName) - const baseName = path.basename(fileName, extension) - let nextNumber = 0 - let newPath = '' - - do { - newPath = path.join(containingDir, baseName + `(${++nextNumber})` + extension) - } while (fs.existsSync(newPath)) - - return newPath -} - -export function validatePath(outputPath: string, defaultFileName: string, forceWrite = false): string { - let completePath = path.resolve(outputPath) - const containingDir = path.dirname(completePath) - - // If the cointaining folder doesnt exist - if (!fs.existsSync(containingDir)) throw new CLIError(`Containing directory path doesn't exist: ${containingDir}`) - - const baseElement = path.basename(completePath) - const pathAlreadyExist = fs.existsSync(completePath) - - // If the last element in the path is a file - if (baseElement.includes('.')) { - return pathAlreadyExist && !forceWrite ? enumerateFileName(completePath) : completePath - } - - // If the last element in the path is a folder - if (!pathAlreadyExist) throw new CLIError(`Target directory path doesn't exist: ${completePath}`) - completePath = path.join(completePath, defaultFileName) - return fs.existsSync(completePath) && !forceWrite ? enumerateFileName(completePath) : completePath -} - -export async function detectLuContent(stdin: string, input: string) { - if (!stdin && !input) { - throw new CLIError('Missing input. Please use stdin or pass a file location with --in flag') - } - - if (!stdin) { - if (!fs.existsSync(path.resolve(input))) { - throw new CLIError(`Sorry unable to open [${input}]`) - } - - const inputStat = await fs.stat(input) - return !inputStat.isFile() ? true : (path.extname(input) === '.lu' || path.extname(input) === '.qna') - } - - try { - await JSON.parse(stdin) - } catch (error) { - return true - } - return false -} - -export function parseJSON(input: string, appType: string) { - try { - return JSON.parse(input) - } catch (error) { - throw new CLIError(`Sorry, error parsing content as ${appType} JSON`) - } -} - -export function getCultureFromPath(file: string): string | null { - const fn = path.basename(file, path.extname(file)) - const lang = path.extname(fn).substring(1) - switch (lang.toLowerCase()) { - case 'en-us': - case 'zh-cn': - case 'nl-nl': - case 'fr-fr': - case 'fr-ca': - case 'de-de': - case 'it-it': - case 'ja-jp': - case 'ko-kr': - case 'pt-br': - case 'es-es': - case 'es-mx': - case 'tr-tr': - return lang - default: - return null - } -}