From 69ae22670975a0676ced27bd65129bae74975c11 Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Thu, 28 Nov 2019 16:54:33 +0800 Subject: [PATCH 01/25] start runtime from samplpebot csharp runtime --- BotProject/CSharp/Program.cs | 3 + .../server/src/controllers/connector.ts | 4 +- .../src/models/connector/build_runtime.ps1 | 25 +++ .../models/connector/csharpBotConnector.ts | 173 ++++++++++++++---- .../models/environment/defaultEnvironment.ts | 2 +- 5 files changed, 166 insertions(+), 41 deletions(-) create mode 100644 Composer/packages/server/src/models/connector/build_runtime.ps1 diff --git a/BotProject/CSharp/Program.cs b/BotProject/CSharp/Program.cs index d2fa81364c..f530e1f958 100644 --- a/BotProject/CSharp/Program.cs +++ b/BotProject/CSharp/Program.cs @@ -22,9 +22,12 @@ public static IWebHost BuildWebHost(string[] args) => { var env = hostingContext.HostingEnvironment; var luisAuthoringRegion = Environment.GetEnvironmentVariable("LUIS_AUTHORING_REGION") ?? "westus"; + var luisSettingFiles = Directory.GetFiles($"ComposerDialogs\\generated", "luis.settings.*.json"); config .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) .AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true, reloadOnChange: true) + .AddJsonFile($"ComposerDialogs/settings/appsettings.json", optional: true, reloadOnChange: true) + .AddJsonFile(luisSettingFiles.Length > 0 ? luisSettingFiles[0] : string.Empty, optional: true, reloadOnChange: true) .AddJsonFile($"luis.settings.{env.EnvironmentName}.{luisAuthoringRegion}.json", optional: true, reloadOnChange: true) .AddJsonFile($"luis.settings.{Environment.UserName}.{luisAuthoringRegion}.json", optional: true, reloadOnChange: true); diff --git a/Composer/packages/server/src/controllers/connector.ts b/Composer/packages/server/src/controllers/connector.ts index ee721da2b4..fb746665a8 100644 --- a/Composer/packages/server/src/controllers/connector.ts +++ b/Composer/packages/server/src/controllers/connector.ts @@ -1,5 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +import merge from 'lodash/merge'; import { BotEnvironments } from '../models/connector'; import { EnvironmentProvider } from '../models/environment'; @@ -33,7 +34,8 @@ async function getPublishHistory(req: any, res: any) { async function sync(req: any, res: any) { try { const environment = EnvironmentProvider.getCurrent(); - await environment.getBotConnector().sync({ ...req.body, user: req.user }); + const settingsInDisk = await environment.getSettingsManager().get('', false); + await environment.getBotConnector().sync(merge(settingsInDisk, req.body, { user: req.user })); res.send('OK'); } catch (error) { res.status(400).json({ diff --git a/Composer/packages/server/src/models/connector/build_runtime.ps1 b/Composer/packages/server/src/models/connector/build_runtime.ps1 new file mode 100644 index 0000000000..a2d36baf28 --- /dev/null +++ b/Composer/packages/server/src/models/connector/build_runtime.ps1 @@ -0,0 +1,25 @@ +Param( + [object] $config, + [string] $customSettingFolder, + [string] $luisAuthroingKey, + [SecureString] $appPassword, + [string] $projFolder = $(Join-Path $(Get-Location) BotProject CSharp) +) + +if ($PSVersionTable.PSVersion.Major -lt 6){ + Write-Host "! Powershell 6 is required, current version is $($PSVersionTable.PSVersion.Major), please refer following documents for help." + Write-Host "For Windows - https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-windows?view=powershell-6" + Write-Host "For Mac - https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-macos?view=powershell-6" + Break +} + +if ((dotnet --version) -lt 3) { + Write-Host "! dotnet core 3.0 is required, please refer following documents for help." + Write-Host "https://dotnet.microsoft.com/download/dotnet-core/3.0" + Break +} + +# Init user secret id +dotnet user-secrets init + +dotnet build \ No newline at end of file diff --git a/Composer/packages/server/src/models/connector/csharpBotConnector.ts b/Composer/packages/server/src/models/connector/csharpBotConnector.ts index 8a17c3eeee..a31aa030fe 100644 --- a/Composer/packages/server/src/models/connector/csharpBotConnector.ts +++ b/Composer/packages/server/src/models/connector/csharpBotConnector.ts @@ -2,74 +2,169 @@ // Licensed under the MIT License. import fs from 'fs'; -import Path from 'path'; +import { ChildProcess, spawn } from 'child_process'; -import axios from 'axios'; import archiver from 'archiver'; -import FormData from 'form-data'; import { BotProjectService } from '../../services/project'; import { DialogSetting } from '../bot/interface'; +import { Path } from '../../utility/path'; import { BotConfig, BotEnvironments, BotStatus, IBotConnector, IPublishHistory } from './interface'; export class CSharpBotConnector implements IBotConnector { - private adminEndpoint: string; + public status: BotStatus = BotStatus.NotConnected; private endpoint: string; - constructor(adminEndpoint: string, endpoint: string) { - this.adminEndpoint = adminEndpoint; + private runtime: ChildProcess | null = null; + constructor(endpoint: string) { this.endpoint = endpoint; + this.addProcessListeners(); } - public status: BotStatus = BotStatus.NotConnected; + private addProcessListeners = () => { + process.on('SIGINT', () => { + console.log('[SIGINT] start graceful shutdown'); + this.stop(); + process.exit(1); + }); + process.on('SIGTERM', () => { + console.log('[SIGTERM] start graceful shutdown'); + this.stop(); + process.exit(1); + }); + process.on('SIGQUIT', () => { + console.log('[SIGQUIT] start graceful shutdown'); + this.stop(); + process.exit(1); + }); + }; - connect = async (_: BotEnvironments, __: string) => { - // confirm bot runtime is listening here - try { - await axios.get(this.adminEndpoint + '/api/admin'); - } catch (err) { - throw new Error(err); + private stop = () => { + if (this.runtime) { + console.log(`kill this bot with process PID: ${this.runtime.pid}`); + this.runtime.kill('SIGKILL'); + this.runtime = null; } + }; - this.status = BotStatus.NotConnected; + private buildProcess = async (dir: string): Promise => { + return new Promise((resolve, reject) => { + const startScript = Path.resolve(__dirname, './build_runtime.ps1'); + console.log(startScript); + const build = spawn(`pwsh ${startScript}`, { + cwd: dir, + detached: true, + shell: true, + stdio: ['ignore', 'ignore', 'inherit'], + }); + console.log(`build pid : ${build.pid}`); - return `${this.endpoint}/api/messages`; + build.stderr && + build.stderr.on('data', function(err) { + reject(err.toString()); + }); + + build.on('exit', function(code) { + resolve(code); + }); + }); }; - sync = async (config: DialogSetting) => { - // archive the project - // send to bot runtime service - const currentProject = BotProjectService.getCurrentBotProject(); - if (currentProject === undefined) { - throw new Error('no project is opened, nothing to sync'); + private getConnectorConfig = (config: DialogSetting) => { + const configList: string[] = []; + if (config.MicrosoftAppPassword) { + configList.push('--MicrosoftAppPassword'); + configList.push(config.MicrosoftAppPassword); + } + if (config.luis) { + if (config.luis.authoringKey) { + configList.push('--luis:endpointKey'); + configList.push(config.luis.authoringKey); + } + if (config.luis.authoringRegion) { + configList.push('--luis:endpoint'); + configList.push(`https://${config.luis.authoringRegion}.api.cognitive.microsoft.com`); + } } - const dir = Path.join(currentProject.dataDir); - const luisConfig = currentProject.luPublisher.getLuisConfig(); - await this.archiveDirectory(dir, './tmp.zip'); - const content = fs.readFileSync('./tmp.zip'); - const form = new FormData(); - form.append('file', content, 'bot.zip'); + return configList; + }; + private addListeners = (child: ChildProcess, handler: Function) => { + if (child.stdout !== null) { + child.stdout.on('data', (data: any) => { + console.log(`stdout: ${data}`); + }); + } - if (luisConfig && luisConfig.authoringKey !== null && !currentProject.checkLuisPublished()) { - throw new Error('Please publish your Luis models'); + if (child.stderr !== null) { + child.stderr.on('data', (data: any) => { + console.log(`stderr: ${data}`); + }); } - if (luisConfig) { - form.append('endpointKey', luisConfig.endpointKey || luisConfig.authoringKey || ''); + child.on('close', code => { + console.log(`close ${code}`); + handler(); + }); + + child.on('error', (err: any) => { + console.log(`stderr: ${err}`); + }); + + child.on('exit', code => { + console.log(`exit: ${code}`); + handler(); + }); + + child.on('message', msg => { + console.log(msg); + }); + + child.on('disconnect', code => { + console.log(`disconnect: ${code}`); + handler(); + }); + }; + + private getBotPath = () => { + const currentProject = BotProjectService.getCurrentBotProject(); + if (currentProject === undefined) { + throw new Error('no project is opened, nothing to sync'); } + return Path.join(currentProject.dir); + }; - config = { - ...(await currentProject.settingManager.get(currentProject.environment.getDefaultSlot(), false)), - ...config, - }; - if (config.MicrosoftAppPassword) { - form.append('microsoftAppPassword', config.MicrosoftAppPassword); + private start = async (dir: string, config: DialogSetting) => { + this.runtime = spawn( + 'dotnet', + ['bin/Debug/netcoreapp2.1/BotProject.dll', `--urls`, this.endpoint, ...this.getConnectorConfig(config)], + { + detached: true, + cwd: dir, + stdio: ['ignore', 'ignore', 'inherit'], + } + ); + this.addListeners(this.runtime, this.stop); + }; + + connect = async (_: BotEnvironments, __: string) => { + // confirm bot runtime is listening here + try { + const dir = this.getBotPath(); + await this.buildProcess(dir); + return Promise.resolve(`${this.endpoint}/api/messages`); + } catch (err) { + throw new Error(err); } + }; + + sync = async (config: DialogSetting) => { try { - await axios.post(this.adminEndpoint + '/api/admin', form, { headers: form.getHeaders() }); + const dir = this.getBotPath(); + await this.start(dir, config); } catch (err) { - throw new Error('Unable to sync content to bot runtime'); + this.stop(); + throw new Error(err); } }; diff --git a/Composer/packages/server/src/models/environment/defaultEnvironment.ts b/Composer/packages/server/src/models/environment/defaultEnvironment.ts index d7b2a8e7ba..c18d8a9290 100644 --- a/Composer/packages/server/src/models/environment/defaultEnvironment.ts +++ b/Composer/packages/server/src/models/environment/defaultEnvironment.ts @@ -18,7 +18,7 @@ export class DefaultEnvironment implements IEnvironment { public constructor(config: IEnvironmentConfig) { this.config = config; this.settingManager = new DefaultSettingManager(this.config.basePath); - this.botConnector = new CSharpBotConnector(this.config.adminEndpoint, this.config.endpoint); + this.botConnector = new CSharpBotConnector(this.config.endpoint); } public getEnvironmentName(_: string): string | undefined { From 1f3b631d655e4983928bc07f017cf5a6e438ad4e Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Thu, 28 Nov 2019 17:13:56 +0800 Subject: [PATCH 02/25] fix --- BotProject/CSharp/Program.cs | 3 --- BotProject/Templates/CSharp/Program.cs | 5 ++++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/BotProject/CSharp/Program.cs b/BotProject/CSharp/Program.cs index f530e1f958..d2fa81364c 100644 --- a/BotProject/CSharp/Program.cs +++ b/BotProject/CSharp/Program.cs @@ -22,12 +22,9 @@ public static IWebHost BuildWebHost(string[] args) => { var env = hostingContext.HostingEnvironment; var luisAuthoringRegion = Environment.GetEnvironmentVariable("LUIS_AUTHORING_REGION") ?? "westus"; - var luisSettingFiles = Directory.GetFiles($"ComposerDialogs\\generated", "luis.settings.*.json"); config .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) .AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true, reloadOnChange: true) - .AddJsonFile($"ComposerDialogs/settings/appsettings.json", optional: true, reloadOnChange: true) - .AddJsonFile(luisSettingFiles.Length > 0 ? luisSettingFiles[0] : string.Empty, optional: true, reloadOnChange: true) .AddJsonFile($"luis.settings.{env.EnvironmentName}.{luisAuthoringRegion}.json", optional: true, reloadOnChange: true) .AddJsonFile($"luis.settings.{Environment.UserName}.{luisAuthoringRegion}.json", optional: true, reloadOnChange: true); diff --git a/BotProject/Templates/CSharp/Program.cs b/BotProject/Templates/CSharp/Program.cs index d1a3176b0f..bfe89422d5 100644 --- a/BotProject/Templates/CSharp/Program.cs +++ b/BotProject/Templates/CSharp/Program.cs @@ -14,16 +14,18 @@ public static void Main(string[] args) { BuildWebHost(args).Run(); } - public static IWebHost BuildWebHost(string[] args) => WebHost.CreateDefaultBuilder(args) .ConfigureAppConfiguration((hostingContext, config) => { var env = hostingContext.HostingEnvironment; var luisAuthoringRegion = Environment.GetEnvironmentVariable("LUIS_AUTHORING_REGION") ?? "westus"; + var luisSettingFiles = Directory.GetFiles($"ComposerDialogs\\generated", "luis.settings.*.json"); config .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) .AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true, reloadOnChange: true) + .AddJsonFile($"ComposerDialogs/settings/appsettings.json", optional: true, reloadOnChange: true) + .AddJsonFile(luisSettingFiles.Length > 0 ? luisSettingFiles[0] : string.Empty, optional: true, reloadOnChange: true) .AddJsonFile($"luis.settings.{env.EnvironmentName}.{luisAuthoringRegion}.json", optional: true, reloadOnChange: true) .AddJsonFile($"luis.settings.{Environment.UserName}.{luisAuthoringRegion}.json", optional: true, reloadOnChange: true); @@ -37,5 +39,6 @@ public static IWebHost BuildWebHost(string[] args) => .AddCommandLine(args); }).UseStartup() .Build(); + } } From b8670c9c34d1da54d21fe674122fcb7030f8ca81 Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Thu, 28 Nov 2019 17:21:06 +0800 Subject: [PATCH 03/25] fix --- BotProject/Templates/CSharp/Program.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/BotProject/Templates/CSharp/Program.cs b/BotProject/Templates/CSharp/Program.cs index bfe89422d5..d386d076f7 100644 --- a/BotProject/Templates/CSharp/Program.cs +++ b/BotProject/Templates/CSharp/Program.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using System.IO; using Microsoft.AspNetCore; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Configuration; @@ -14,6 +15,7 @@ public static void Main(string[] args) { BuildWebHost(args).Run(); } + public static IWebHost BuildWebHost(string[] args) => WebHost.CreateDefaultBuilder(args) .ConfigureAppConfiguration((hostingContext, config) => From 6b04384f6fb29e85ae2bc94257a50e60dd711f15 Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Thu, 28 Nov 2019 20:38:22 +0800 Subject: [PATCH 04/25] fix --- BotProject/Templates/CSharp/Program.cs | 15 ++++++++++++--- .../Templates/CSharp/Scripts}/build_runtime.ps1 | 0 .../src/models/connector/csharpBotConnector.ts | 13 +++++-------- .../src/models/environment/defaultEnvironment.ts | 1 + 4 files changed, 18 insertions(+), 11 deletions(-) rename {Composer/packages/server/src/models/connector => BotProject/Templates/CSharp/Scripts}/build_runtime.ps1 (100%) diff --git a/BotProject/Templates/CSharp/Program.cs b/BotProject/Templates/CSharp/Program.cs index d386d076f7..fd4a5e14f0 100644 --- a/BotProject/Templates/CSharp/Program.cs +++ b/BotProject/Templates/CSharp/Program.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using System.Diagnostics; using System.IO; using Microsoft.AspNetCore; using Microsoft.AspNetCore.Hosting; @@ -22,14 +23,23 @@ public static IWebHost BuildWebHost(string[] args) => { var env = hostingContext.HostingEnvironment; var luisAuthoringRegion = Environment.GetEnvironmentVariable("LUIS_AUTHORING_REGION") ?? "westus"; - var luisSettingFiles = Directory.GetFiles($"ComposerDialogs\\generated", "luis.settings.*.json"); config .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) .AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true, reloadOnChange: true) .AddJsonFile($"ComposerDialogs/settings/appsettings.json", optional: true, reloadOnChange: true) - .AddJsonFile(luisSettingFiles.Length > 0 ? luisSettingFiles[0] : string.Empty, optional: true, reloadOnChange: true) .AddJsonFile($"luis.settings.{env.EnvironmentName}.{luisAuthoringRegion}.json", optional: true, reloadOnChange: true) .AddJsonFile($"luis.settings.{Environment.UserName}.{luisAuthoringRegion}.json", optional: true, reloadOnChange: true); + try + { + foreach (string filePath in Directory.GetFiles($"ComposerDialogs", "generated/luis.settings.*.json")) + { + config.AddJsonFile(filePath, optional: true, reloadOnChange: true); + } + } + catch (Exception ex) + { + Trace.WriteLine(ex.Message); + } if (env.IsDevelopment()) { @@ -41,6 +51,5 @@ public static IWebHost BuildWebHost(string[] args) => .AddCommandLine(args); }).UseStartup() .Build(); - } } diff --git a/Composer/packages/server/src/models/connector/build_runtime.ps1 b/BotProject/Templates/CSharp/Scripts/build_runtime.ps1 similarity index 100% rename from Composer/packages/server/src/models/connector/build_runtime.ps1 rename to BotProject/Templates/CSharp/Scripts/build_runtime.ps1 diff --git a/Composer/packages/server/src/models/connector/csharpBotConnector.ts b/Composer/packages/server/src/models/connector/csharpBotConnector.ts index a31aa030fe..44a6a6a0d0 100644 --- a/Composer/packages/server/src/models/connector/csharpBotConnector.ts +++ b/Composer/packages/server/src/models/connector/csharpBotConnector.ts @@ -49,7 +49,7 @@ export class CSharpBotConnector implements IBotConnector { private buildProcess = async (dir: string): Promise => { return new Promise((resolve, reject) => { - const startScript = Path.resolve(__dirname, './build_runtime.ps1'); + const startScript = Path.resolve(dir, './Scripts/build_runtime.ps1'); console.log(startScript); const build = spawn(`pwsh ${startScript}`, { cwd: dir, @@ -144,23 +144,20 @@ export class CSharpBotConnector implements IBotConnector { stdio: ['ignore', 'ignore', 'inherit'], } ); + console.log(`start runtime at ${this.runtime.pid}`); this.addListeners(this.runtime, this.stop); }; connect = async (_: BotEnvironments, __: string) => { // confirm bot runtime is listening here - try { - const dir = this.getBotPath(); - await this.buildProcess(dir); - return Promise.resolve(`${this.endpoint}/api/messages`); - } catch (err) { - throw new Error(err); - } + return Promise.resolve(`${this.endpoint}/api/messages`); }; sync = async (config: DialogSetting) => { try { + this.stop(); const dir = this.getBotPath(); + await this.buildProcess(dir); await this.start(dir, config); } catch (err) { this.stop(); diff --git a/Composer/packages/server/src/models/environment/defaultEnvironment.ts b/Composer/packages/server/src/models/environment/defaultEnvironment.ts index c18d8a9290..21a3b00f4d 100644 --- a/Composer/packages/server/src/models/environment/defaultEnvironment.ts +++ b/Composer/packages/server/src/models/environment/defaultEnvironment.ts @@ -18,6 +18,7 @@ export class DefaultEnvironment implements IEnvironment { public constructor(config: IEnvironmentConfig) { this.config = config; this.settingManager = new DefaultSettingManager(this.config.basePath); + console.log(this.config.basePath); this.botConnector = new CSharpBotConnector(this.config.endpoint); } From 1e0ca12dfc0f86bdf439fa301dd1f82a8f72b26d Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Thu, 28 Nov 2019 22:16:16 +0800 Subject: [PATCH 05/25] use logger instead of console log --- .../models/connector/csharpBotConnector.ts | 30 ++++++++++--------- .../models/environment/defaultEnvironment.ts | 1 - 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/Composer/packages/server/src/models/connector/csharpBotConnector.ts b/Composer/packages/server/src/models/connector/csharpBotConnector.ts index 44a6a6a0d0..4540297e9a 100644 --- a/Composer/packages/server/src/models/connector/csharpBotConnector.ts +++ b/Composer/packages/server/src/models/connector/csharpBotConnector.ts @@ -9,6 +9,7 @@ import archiver from 'archiver'; import { BotProjectService } from '../../services/project'; import { DialogSetting } from '../bot/interface'; import { Path } from '../../utility/path'; +import log from '../../logger'; import { BotConfig, BotEnvironments, BotStatus, IBotConnector, IPublishHistory } from './interface'; @@ -23,17 +24,17 @@ export class CSharpBotConnector implements IBotConnector { private addProcessListeners = () => { process.on('SIGINT', () => { - console.log('[SIGINT] start graceful shutdown'); + log('[SIGINT] start graceful shutdown'); this.stop(); process.exit(1); }); process.on('SIGTERM', () => { - console.log('[SIGTERM] start graceful shutdown'); + log('[SIGTERM] start graceful shutdown'); this.stop(); process.exit(1); }); process.on('SIGQUIT', () => { - console.log('[SIGQUIT] start graceful shutdown'); + log('[SIGQUIT] start graceful shutdown'); this.stop(); process.exit(1); }); @@ -41,23 +42,23 @@ export class CSharpBotConnector implements IBotConnector { private stop = () => { if (this.runtime) { - console.log(`kill this bot with process PID: ${this.runtime.pid}`); + log(`kill this bot with process PID: ${this.runtime.pid}`); this.runtime.kill('SIGKILL'); this.runtime = null; } + this.status = BotStatus.NotConnected; }; private buildProcess = async (dir: string): Promise => { return new Promise((resolve, reject) => { const startScript = Path.resolve(dir, './Scripts/build_runtime.ps1'); - console.log(startScript); const build = spawn(`pwsh ${startScript}`, { cwd: dir, detached: true, shell: true, stdio: ['ignore', 'ignore', 'inherit'], }); - console.log(`build pid : ${build.pid}`); + log(`build pid : ${build.pid}`); build.stderr && build.stderr.on('data', function(err) { @@ -92,36 +93,36 @@ export class CSharpBotConnector implements IBotConnector { private addListeners = (child: ChildProcess, handler: Function) => { if (child.stdout !== null) { child.stdout.on('data', (data: any) => { - console.log(`stdout: ${data}`); + log(`stdout: ${data}`); }); } if (child.stderr !== null) { child.stderr.on('data', (data: any) => { - console.log(`stderr: ${data}`); + log(`stderr: ${data}`); }); } child.on('close', code => { - console.log(`close ${code}`); + log(`close ${code}`); handler(); }); child.on('error', (err: any) => { - console.log(`stderr: ${err}`); + log(`stderr: ${err}`); }); child.on('exit', code => { - console.log(`exit: ${code}`); + log(`exit: ${code}`); handler(); }); child.on('message', msg => { - console.log(msg); + log(msg); }); child.on('disconnect', code => { - console.log(`disconnect: ${code}`); + log(`disconnect: ${code}`); handler(); }); }; @@ -144,8 +145,9 @@ export class CSharpBotConnector implements IBotConnector { stdio: ['ignore', 'ignore', 'inherit'], } ); - console.log(`start runtime at ${this.runtime.pid}`); + log(`start runtime at ${this.runtime.pid}`); this.addListeners(this.runtime, this.stop); + this.status = BotStatus.Connected; }; connect = async (_: BotEnvironments, __: string) => { diff --git a/Composer/packages/server/src/models/environment/defaultEnvironment.ts b/Composer/packages/server/src/models/environment/defaultEnvironment.ts index 21a3b00f4d..c18d8a9290 100644 --- a/Composer/packages/server/src/models/environment/defaultEnvironment.ts +++ b/Composer/packages/server/src/models/environment/defaultEnvironment.ts @@ -18,7 +18,6 @@ export class DefaultEnvironment implements IEnvironment { public constructor(config: IEnvironmentConfig) { this.config = config; this.settingManager = new DefaultSettingManager(this.config.basePath); - console.log(this.config.basePath); this.botConnector = new CSharpBotConnector(this.config.endpoint); } From 3964fd7208a2f94b8512a5d428189c2f170245db Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Mon, 2 Dec 2019 23:39:39 +0800 Subject: [PATCH 06/25] fix comment --- .../client/src/store/action/setting.ts | 5 +- .../client/src/store/reducer/index.ts | 5 +- .../models/connector/csharpBotConnector.ts | 205 ------------------ .../models/settings/defaultSettingManager.ts | 3 +- 4 files changed, 10 insertions(+), 208 deletions(-) delete mode 100644 Composer/packages/server/src/models/connector/csharpBotConnector.ts diff --git a/Composer/packages/client/src/store/action/setting.ts b/Composer/packages/client/src/store/action/setting.ts index 8dd79ac515..706f7e714f 100644 --- a/Composer/packages/client/src/store/action/setting.ts +++ b/Composer/packages/client/src/store/action/setting.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. import get from 'lodash/get'; -import { SensitiveProperties } from '@bfc/shared'; +import { SensitiveProperties as defaultSensitiveProperties } from '@bfc/shared'; import { ActionCreator, DialogSetting } from '../types'; import settingsStorage from '../../utils/dialogSettingStorage'; @@ -26,6 +26,9 @@ export const setSettings: ActionCreator = async ( }, }); // set value in local storage + const SensitiveProperties = settings.SensitiveProperties + ? settings.SensitiveProperties + : defaultSensitiveProperties; for (const property of SensitiveProperties) { const propertyValue = get(settings, property); settingsStorage.setField(botName, property, propertyValue ? propertyValue : ''); diff --git a/Composer/packages/client/src/store/reducer/index.ts b/Composer/packages/client/src/store/reducer/index.ts index 40f7fd0890..f89b8eaf71 100644 --- a/Composer/packages/client/src/store/reducer/index.ts +++ b/Composer/packages/client/src/store/reducer/index.ts @@ -4,7 +4,7 @@ import get from 'lodash/get'; import set from 'lodash/set'; import { dialogIndexer } from '@bfc/indexers/lib/dialogIndexer'; -import { SensitiveProperties } from '@bfc/shared'; +import { SensitiveProperties as defaultSensitiveProperties } from '@bfc/shared'; import { ActionTypes, FileTypes } from '../../constants'; import { DialogSetting, ReducerFunc } from '../types'; @@ -18,6 +18,8 @@ const projectFiles = ['bot', 'botproj']; // if user set value in terminal or appsetting.json, it should update the value in localStorage const refreshLocalStorage = (botName: string, settings: DialogSetting) => { + const SensitiveProperties = settings.SensitiveProperties ? settings.SensitiveProperties : defaultSensitiveProperties; + console.log(SensitiveProperties); for (const property of SensitiveProperties) { const value = get(settings, property); if (value) { @@ -29,6 +31,7 @@ const refreshLocalStorage = (botName: string, settings: DialogSetting) => { // merge sensitive values in localStorage const mergeLocalStorage = (botName: string, settings: DialogSetting) => { const localSetting = settingStorage.get(botName); + const SensitiveProperties = settings.SensitiveProperties ? settings.SensitiveProperties : defaultSensitiveProperties; if (localSetting) { for (const property of SensitiveProperties) { const value = get(localSetting, property); diff --git a/Composer/packages/server/src/models/connector/csharpBotConnector.ts b/Composer/packages/server/src/models/connector/csharpBotConnector.ts deleted file mode 100644 index 4540297e9a..0000000000 --- a/Composer/packages/server/src/models/connector/csharpBotConnector.ts +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import fs from 'fs'; -import { ChildProcess, spawn } from 'child_process'; - -import archiver from 'archiver'; - -import { BotProjectService } from '../../services/project'; -import { DialogSetting } from '../bot/interface'; -import { Path } from '../../utility/path'; -import log from '../../logger'; - -import { BotConfig, BotEnvironments, BotStatus, IBotConnector, IPublishHistory } from './interface'; - -export class CSharpBotConnector implements IBotConnector { - public status: BotStatus = BotStatus.NotConnected; - private endpoint: string; - private runtime: ChildProcess | null = null; - constructor(endpoint: string) { - this.endpoint = endpoint; - this.addProcessListeners(); - } - - private addProcessListeners = () => { - process.on('SIGINT', () => { - log('[SIGINT] start graceful shutdown'); - this.stop(); - process.exit(1); - }); - process.on('SIGTERM', () => { - log('[SIGTERM] start graceful shutdown'); - this.stop(); - process.exit(1); - }); - process.on('SIGQUIT', () => { - log('[SIGQUIT] start graceful shutdown'); - this.stop(); - process.exit(1); - }); - }; - - private stop = () => { - if (this.runtime) { - log(`kill this bot with process PID: ${this.runtime.pid}`); - this.runtime.kill('SIGKILL'); - this.runtime = null; - } - this.status = BotStatus.NotConnected; - }; - - private buildProcess = async (dir: string): Promise => { - return new Promise((resolve, reject) => { - const startScript = Path.resolve(dir, './Scripts/build_runtime.ps1'); - const build = spawn(`pwsh ${startScript}`, { - cwd: dir, - detached: true, - shell: true, - stdio: ['ignore', 'ignore', 'inherit'], - }); - log(`build pid : ${build.pid}`); - - build.stderr && - build.stderr.on('data', function(err) { - reject(err.toString()); - }); - - build.on('exit', function(code) { - resolve(code); - }); - }); - }; - - private getConnectorConfig = (config: DialogSetting) => { - const configList: string[] = []; - if (config.MicrosoftAppPassword) { - configList.push('--MicrosoftAppPassword'); - configList.push(config.MicrosoftAppPassword); - } - if (config.luis) { - if (config.luis.authoringKey) { - configList.push('--luis:endpointKey'); - configList.push(config.luis.authoringKey); - } - if (config.luis.authoringRegion) { - configList.push('--luis:endpoint'); - configList.push(`https://${config.luis.authoringRegion}.api.cognitive.microsoft.com`); - } - } - - return configList; - }; - private addListeners = (child: ChildProcess, handler: Function) => { - if (child.stdout !== null) { - child.stdout.on('data', (data: any) => { - log(`stdout: ${data}`); - }); - } - - if (child.stderr !== null) { - child.stderr.on('data', (data: any) => { - log(`stderr: ${data}`); - }); - } - - child.on('close', code => { - log(`close ${code}`); - handler(); - }); - - child.on('error', (err: any) => { - log(`stderr: ${err}`); - }); - - child.on('exit', code => { - log(`exit: ${code}`); - handler(); - }); - - child.on('message', msg => { - log(msg); - }); - - child.on('disconnect', code => { - log(`disconnect: ${code}`); - handler(); - }); - }; - - private getBotPath = () => { - const currentProject = BotProjectService.getCurrentBotProject(); - if (currentProject === undefined) { - throw new Error('no project is opened, nothing to sync'); - } - return Path.join(currentProject.dir); - }; - - private start = async (dir: string, config: DialogSetting) => { - this.runtime = spawn( - 'dotnet', - ['bin/Debug/netcoreapp2.1/BotProject.dll', `--urls`, this.endpoint, ...this.getConnectorConfig(config)], - { - detached: true, - cwd: dir, - stdio: ['ignore', 'ignore', 'inherit'], - } - ); - log(`start runtime at ${this.runtime.pid}`); - this.addListeners(this.runtime, this.stop); - this.status = BotStatus.Connected; - }; - - connect = async (_: BotEnvironments, __: string) => { - // confirm bot runtime is listening here - return Promise.resolve(`${this.endpoint}/api/messages`); - }; - - sync = async (config: DialogSetting) => { - try { - this.stop(); - const dir = this.getBotPath(); - await this.buildProcess(dir); - await this.start(dir, config); - } catch (err) { - this.stop(); - throw new Error(err); - } - }; - - archiveDirectory = (src: string, dest: string) => { - return new Promise((resolve, reject) => { - const archive = archiver('zip'); - const output = fs.createWriteStream(dest); - - archive.pipe(output); - archive.directory(src, false); - archive.finalize(); - - output.on('close', () => resolve(archive)); - archive.on('error', err => reject(err)); - }); - }; - - getEditingStatus = (): Promise => { - return new Promise(resolve => { - resolve(true); - }); - }; - - getPublishHistory = (): Promise => { - return new Promise(resolve => { - resolve({ - production: undefined, - previousProduction: undefined, - integration: undefined, - }); - }); - }; - - publish = (_: BotConfig, __: string): Promise => { - return new Promise(resolve => { - resolve(); - }); - }; -} diff --git a/Composer/packages/server/src/models/settings/defaultSettingManager.ts b/Composer/packages/server/src/models/settings/defaultSettingManager.ts index 3df934368e..b322b2f0fe 100644 --- a/Composer/packages/server/src/models/settings/defaultSettingManager.ts +++ b/Composer/packages/server/src/models/settings/defaultSettingManager.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. import omit from 'lodash/omit'; -import { SensitiveProperties } from '@bfc/shared'; +import { SensitiveProperties as defaultSensitiveProperties } from '@bfc/shared'; import { Path } from '../../utility/path'; @@ -34,6 +34,7 @@ export class DefaultSettingManager extends FileSettingManager { }; private filterOutSensitiveValue = (obj: any) => { + const SensitiveProperties = obj.SensitiveProperties ? obj.SensitiveProperties : defaultSensitiveProperties; if (obj && typeof obj === 'object') { return omit(obj, SensitiveProperties); } From c0f1411b0cff2223cdcb0f969cb24227db997a2f Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Mon, 2 Dec 2019 23:50:54 +0800 Subject: [PATCH 07/25] fix comment --- .../models/connector/csharpBotConnector.ts | 202 ++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 Composer/packages/server/src/models/connector/csharpBotConnector.ts diff --git a/Composer/packages/server/src/models/connector/csharpBotConnector.ts b/Composer/packages/server/src/models/connector/csharpBotConnector.ts new file mode 100644 index 0000000000..cfbe1208f0 --- /dev/null +++ b/Composer/packages/server/src/models/connector/csharpBotConnector.ts @@ -0,0 +1,202 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import fs from 'fs'; +import { ChildProcess, spawn } from 'child_process'; + +import archiver from 'archiver'; + +import { BotProjectService } from '../../services/project'; +import { DialogSetting } from '../bot/interface'; +import { Path } from '../../utility/path'; + +import { BotConfig, BotEnvironments, BotStatus, IBotConnector, IPublishHistory } from './interface'; +let runtime: ChildProcess | null = null; +export class CSharpBotConnector implements IBotConnector { + public status: BotStatus = BotStatus.NotConnected; + private endpoint: string; + constructor(endpoint: string) { + this.endpoint = endpoint; + } + + private stop = () => { + shutdown(); + this.status = BotStatus.NotConnected; + }; + + private buildProcess = async (dir: string): Promise => { + return new Promise((resolve, reject) => { + const build = spawn(`pwsh ./Scripts/build_runtime.ps1`, { + cwd: dir, + detached: true, + shell: true, + stdio: ['ignore', 'ignore', 'inherit'], + }); + console.log(`build pid : ${build.pid}`); + + build.stderr && + build.stderr.on('data', function(err) { + reject(err.toString()); + }); + + build.on('exit', function(code) { + resolve(code); + }); + }); + }; + + private getConnectorConfig = (config: DialogSetting) => { + const configList: string[] = []; + if (config.MicrosoftAppPassword) { + configList.push('--MicrosoftAppPassword'); + configList.push(config.MicrosoftAppPassword); + } + if (config.luis) { + if (config.luis.authoringKey) { + configList.push('--luis:endpointKey'); + configList.push(config.luis.authoringKey); + } + if (config.luis.authoringRegion) { + configList.push('--luis:endpoint'); + configList.push(`https://${config.luis.authoringRegion}.api.cognitive.microsoft.com`); + } + } + + return configList; + }; + private addListeners = (child: ChildProcess, handler: Function) => { + if (child.stdout !== null) { + child.stdout.on('data', (data: any) => { + console.log(`stdout: ${data}`); + }); + } + + if (child.stderr !== null) { + child.stderr.on('data', (data: any) => { + console.log(`stderr: ${data}`); + }); + } + + child.on('close', code => { + console.log(`close ${code}`); + handler(); + }); + + child.on('error', (err: any) => { + console.log(`stderr: ${err}`); + }); + + child.on('exit', code => { + console.log(`exit: ${code}`); + handler(); + }); + + child.on('message', msg => { + console.log(msg); + }); + + child.on('disconnect', code => { + console.log(`disconnect: ${code}`); + handler(); + }); + }; + + private getBotPath = () => { + const currentProject = BotProjectService.getCurrentBotProject(); + if (currentProject === undefined) { + throw new Error('no project is opened, nothing to sync'); + } + return Path.join(currentProject.dir); + }; + + private start = async (dir: string, config: DialogSetting) => { + runtime = spawn( + 'dotnet', + ['bin/Debug/netcoreapp2.1/BotProject.dll', `--urls`, this.endpoint, ...this.getConnectorConfig(config)], + { + detached: true, + cwd: dir, + stdio: ['ignore', 'ignore', 'inherit'], + } + ); + console.log(`start runtime at ${runtime.pid}`); + this.addListeners(runtime, this.stop); + this.status = BotStatus.Connected; + }; + + connect = async (_: BotEnvironments, __: string) => { + // confirm bot runtime can listening here + return Promise.resolve(`${this.endpoint}/api/messages`); + }; + + sync = async (config: DialogSetting) => { + try { + this.stop(); + const dir = this.getBotPath(); + await this.buildProcess(dir); + await this.start(dir, config); + } catch (err) { + this.stop(); + throw new Error(err); + } + }; + + archiveDirectory = (src: string, dest: string) => { + return new Promise((resolve, reject) => { + const archive = archiver('zip'); + const output = fs.createWriteStream(dest); + + archive.pipe(output); + archive.directory(src, false); + archive.finalize(); + + output.on('close', () => resolve(archive)); + archive.on('error', err => reject(err)); + }); + }; + + getEditingStatus = (): Promise => { + return new Promise(resolve => { + resolve(true); + }); + }; + + getPublishHistory = (): Promise => { + return new Promise(resolve => { + resolve({ + production: undefined, + previousProduction: undefined, + integration: undefined, + }); + }); + }; + + publish = (_: BotConfig, __: string): Promise => { + return new Promise(resolve => { + resolve(); + }); + }; +} + +process.on('SIGINT', () => { + console.log('[SIGINT] start graceful shutdown'); + shutdown(); + process.exit(1); +}); +process.on('SIGTERM', () => { + console.log('[SIGTERM] start graceful shutdown'); + shutdown(); + process.exit(1); +}); +process.on('SIGQUIT', () => { + console.log('[SIGQUIT] start graceful shutdown'); + shutdown(); + process.exit(1); +}); +function shutdown() { + if (runtime) { + console.log(`kill runtime before exit at ${runtime.pid}`); + runtime.kill('SIGKILL'); + runtime = null; + } +} From 306a0d2eadeeb1af5899572c4bb0eb9a9daae931 Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Tue, 3 Dec 2019 17:14:20 +0800 Subject: [PATCH 08/25] add port usable validation and use log instead console log --- .../client/src/store/action/setting.ts | 11 +-- .../client/src/store/reducer/index.ts | 5 +- .../models/connector/csharpBotConnector.ts | 85 +++++++++++++------ .../models/settings/defaultSettingManager.ts | 2 +- 4 files changed, 66 insertions(+), 37 deletions(-) diff --git a/Composer/packages/client/src/store/action/setting.ts b/Composer/packages/client/src/store/action/setting.ts index 706f7e714f..02ddbb15ac 100644 --- a/Composer/packages/client/src/store/action/setting.ts +++ b/Composer/packages/client/src/store/action/setting.ts @@ -2,6 +2,7 @@ // Licensed under the MIT License. import get from 'lodash/get'; +import has from 'lodash/has'; import { SensitiveProperties as defaultSensitiveProperties } from '@bfc/shared'; import { ActionCreator, DialogSetting } from '../types'; @@ -26,12 +27,12 @@ export const setSettings: ActionCreator = async ( }, }); // set value in local storage - const SensitiveProperties = settings.SensitiveProperties - ? settings.SensitiveProperties - : defaultSensitiveProperties; + const SensitiveProperties = settings.SensitiveProperties || defaultSensitiveProperties; for (const property of SensitiveProperties) { - const propertyValue = get(settings, property); - settingsStorage.setField(botName, property, propertyValue ? propertyValue : ''); + if (has(settings, property)) { + const propertyValue = get(settings, property); + settingsStorage.setField(botName, property, propertyValue ? propertyValue : ''); + } } // set value to server const suffix = slot ? `/${slot}` : ''; diff --git a/Composer/packages/client/src/store/reducer/index.ts b/Composer/packages/client/src/store/reducer/index.ts index f89b8eaf71..08a157e2f3 100644 --- a/Composer/packages/client/src/store/reducer/index.ts +++ b/Composer/packages/client/src/store/reducer/index.ts @@ -18,8 +18,7 @@ const projectFiles = ['bot', 'botproj']; // if user set value in terminal or appsetting.json, it should update the value in localStorage const refreshLocalStorage = (botName: string, settings: DialogSetting) => { - const SensitiveProperties = settings.SensitiveProperties ? settings.SensitiveProperties : defaultSensitiveProperties; - console.log(SensitiveProperties); + const SensitiveProperties = settings.SensitiveProperties || defaultSensitiveProperties; for (const property of SensitiveProperties) { const value = get(settings, property); if (value) { @@ -31,7 +30,7 @@ const refreshLocalStorage = (botName: string, settings: DialogSetting) => { // merge sensitive values in localStorage const mergeLocalStorage = (botName: string, settings: DialogSetting) => { const localSetting = settingStorage.get(botName); - const SensitiveProperties = settings.SensitiveProperties ? settings.SensitiveProperties : defaultSensitiveProperties; + const SensitiveProperties = settings.SensitiveProperties || defaultSensitiveProperties; if (localSetting) { for (const property of SensitiveProperties) { const value = get(localSetting, property); diff --git a/Composer/packages/server/src/models/connector/csharpBotConnector.ts b/Composer/packages/server/src/models/connector/csharpBotConnector.ts index cfbe1208f0..68d78dbf8e 100644 --- a/Composer/packages/server/src/models/connector/csharpBotConnector.ts +++ b/Composer/packages/server/src/models/connector/csharpBotConnector.ts @@ -2,6 +2,7 @@ // Licensed under the MIT License. import fs from 'fs'; +import { parse } from 'url'; import { ChildProcess, spawn } from 'child_process'; import archiver from 'archiver'; @@ -9,18 +10,28 @@ import archiver from 'archiver'; import { BotProjectService } from '../../services/project'; import { DialogSetting } from '../bot/interface'; import { Path } from '../../utility/path'; +import log from '../../logger'; import { BotConfig, BotEnvironments, BotStatus, IBotConnector, IPublishHistory } from './interface'; -let runtime: ChildProcess | null = null; + export class CSharpBotConnector implements IBotConnector { public status: BotStatus = BotStatus.NotConnected; private endpoint: string; + static botRuntimes: { [key: string]: ChildProcess } = {}; constructor(endpoint: string) { this.endpoint = endpoint; } + static stopAll = (signal: string) => { + for (const pid in CSharpBotConnector.botRuntimes) { + const runtime = CSharpBotConnector.botRuntimes[pid]; + log(`kill runtime before exit at ${runtime.pid}`); + runtime.kill(signal); + delete CSharpBotConnector.botRuntimes[pid]; + } + }; private stop = () => { - shutdown(); + CSharpBotConnector.stopAll('SIGKILL'); this.status = BotStatus.NotConnected; }; @@ -32,7 +43,7 @@ export class CSharpBotConnector implements IBotConnector { shell: true, stdio: ['ignore', 'ignore', 'inherit'], }); - console.log(`build pid : ${build.pid}`); + log(`build pid : ${build.pid}`); build.stderr && build.stderr.on('data', function(err) { @@ -67,36 +78,36 @@ export class CSharpBotConnector implements IBotConnector { private addListeners = (child: ChildProcess, handler: Function) => { if (child.stdout !== null) { child.stdout.on('data', (data: any) => { - console.log(`stdout: ${data}`); + log(`stdout: ${data}`); }); } if (child.stderr !== null) { child.stderr.on('data', (data: any) => { - console.log(`stderr: ${data}`); + log(`stderr: ${data}`); }); } child.on('close', code => { - console.log(`close ${code}`); + log(`close ${code}`); handler(); }); child.on('error', (err: any) => { - console.log(`stderr: ${err}`); + log(`stderr: ${err}`); }); child.on('exit', code => { - console.log(`exit: ${code}`); + log(`exit: ${code}`); handler(); }); child.on('message', msg => { - console.log(msg); + log(msg); }); child.on('disconnect', code => { - console.log(`disconnect: ${code}`); + log(`disconnect: ${code}`); handler(); }); }; @@ -110,7 +121,7 @@ export class CSharpBotConnector implements IBotConnector { }; private start = async (dir: string, config: DialogSetting) => { - runtime = spawn( + const runtime = spawn( 'dotnet', ['bin/Debug/netcoreapp2.1/BotProject.dll', `--urls`, this.endpoint, ...this.getConnectorConfig(config)], { @@ -119,14 +130,39 @@ export class CSharpBotConnector implements IBotConnector { stdio: ['ignore', 'ignore', 'inherit'], } ); - console.log(`start runtime at ${runtime.pid}`); + log(`runtime started. pid: ${runtime.pid}`); this.addListeners(runtime, this.stop); + CSharpBotConnector.botRuntimes[runtime.pid] = runtime; this.status = BotStatus.Connected; }; + private checkPortUsable = async (port: string | number): Promise => { + return new Promise((resolve, reject) => { + const netstat = spawn('netstat', ['-ano']); + const findstr = spawn('findstr', [`${port}`], { + stdio: [netstat.stdout, 'pipe', process.stderr], + }); + let output = ''; + findstr.stderr && + findstr.stderr.on('data', function(err) { + reject(err.toString()); + }); + + findstr.stdout && findstr.stdout.on('data', data => (output += data)); + findstr.on('exit', function() { + resolve(output); + }); + }); + }; + connect = async (_: BotEnvironments, __: string) => { - // confirm bot runtime can listening here - return Promise.resolve(`${this.endpoint}/api/messages`); + const port = parse(this.endpoint).port || '3979'; + const portStatus = await this.checkPortUsable(port); + if (portStatus.trim() === '') { + return Promise.resolve(`${this.endpoint}/api/messages`); + } else { + throw new Error('Port already been used'); + } }; sync = async (config: DialogSetting) => { @@ -137,7 +173,7 @@ export class CSharpBotConnector implements IBotConnector { await this.start(dir, config); } catch (err) { this.stop(); - throw new Error(err); + throw err; } }; @@ -179,24 +215,17 @@ export class CSharpBotConnector implements IBotConnector { } process.on('SIGINT', () => { - console.log('[SIGINT] start graceful shutdown'); - shutdown(); + log('[SIGINT] start graceful shutdown'); + CSharpBotConnector.stopAll('SIGINT'); process.exit(1); }); process.on('SIGTERM', () => { - console.log('[SIGTERM] start graceful shutdown'); - shutdown(); + log('[SIGTERM] start graceful shutdown'); + CSharpBotConnector.stopAll('SIGTERM'); process.exit(1); }); process.on('SIGQUIT', () => { - console.log('[SIGQUIT] start graceful shutdown'); - shutdown(); + log('[SIGQUIT] start graceful shutdown'); + CSharpBotConnector.stopAll('SIGQUIT'); process.exit(1); }); -function shutdown() { - if (runtime) { - console.log(`kill runtime before exit at ${runtime.pid}`); - runtime.kill('SIGKILL'); - runtime = null; - } -} diff --git a/Composer/packages/server/src/models/settings/defaultSettingManager.ts b/Composer/packages/server/src/models/settings/defaultSettingManager.ts index b322b2f0fe..416c1d0abd 100644 --- a/Composer/packages/server/src/models/settings/defaultSettingManager.ts +++ b/Composer/packages/server/src/models/settings/defaultSettingManager.ts @@ -34,7 +34,7 @@ export class DefaultSettingManager extends FileSettingManager { }; private filterOutSensitiveValue = (obj: any) => { - const SensitiveProperties = obj.SensitiveProperties ? obj.SensitiveProperties : defaultSensitiveProperties; + const SensitiveProperties = obj.SensitiveProperties || defaultSensitiveProperties; if (obj && typeof obj === 'object') { return omit(obj, SensitiveProperties); } From 14c4a21f8f8bba6a928641362098fd74f12d4672 Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Wed, 4 Dec 2019 17:50:24 +0800 Subject: [PATCH 09/25] fix comments --- .../client/src/store/action/setting.ts | 7 +- .../client/src/store/reducer/index.ts | 4 +- .../server/src/controllers/connector.ts | 4 +- .../models/connector/csharpBotConnector.ts | 78 +++++++++++-------- .../models/settings/defaultSettingManager.ts | 3 +- .../src/models/settings/fileSettingManager.ts | 2 +- .../server/src/models/settings/interface.ts | 2 +- 7 files changed, 56 insertions(+), 44 deletions(-) diff --git a/Composer/packages/client/src/store/action/setting.ts b/Composer/packages/client/src/store/action/setting.ts index 02ddbb15ac..dda0bbef98 100644 --- a/Composer/packages/client/src/store/action/setting.ts +++ b/Composer/packages/client/src/store/action/setting.ts @@ -3,7 +3,7 @@ import get from 'lodash/get'; import has from 'lodash/has'; -import { SensitiveProperties as defaultSensitiveProperties } from '@bfc/shared'; +import { SensitiveProperties } from '@bfc/shared'; import { ActionCreator, DialogSetting } from '../types'; import settingsStorage from '../../utils/dialogSettingStorage'; @@ -27,11 +27,10 @@ export const setSettings: ActionCreator = async ( }, }); // set value in local storage - const SensitiveProperties = settings.SensitiveProperties || defaultSensitiveProperties; for (const property of SensitiveProperties) { if (has(settings, property)) { - const propertyValue = get(settings, property); - settingsStorage.setField(botName, property, propertyValue ? propertyValue : ''); + const propertyValue = get(settings, property, ''); + settingsStorage.setField(botName, property, propertyValue); } } // set value to server diff --git a/Composer/packages/client/src/store/reducer/index.ts b/Composer/packages/client/src/store/reducer/index.ts index 08a157e2f3..40f7fd0890 100644 --- a/Composer/packages/client/src/store/reducer/index.ts +++ b/Composer/packages/client/src/store/reducer/index.ts @@ -4,7 +4,7 @@ import get from 'lodash/get'; import set from 'lodash/set'; import { dialogIndexer } from '@bfc/indexers/lib/dialogIndexer'; -import { SensitiveProperties as defaultSensitiveProperties } from '@bfc/shared'; +import { SensitiveProperties } from '@bfc/shared'; import { ActionTypes, FileTypes } from '../../constants'; import { DialogSetting, ReducerFunc } from '../types'; @@ -18,7 +18,6 @@ const projectFiles = ['bot', 'botproj']; // if user set value in terminal or appsetting.json, it should update the value in localStorage const refreshLocalStorage = (botName: string, settings: DialogSetting) => { - const SensitiveProperties = settings.SensitiveProperties || defaultSensitiveProperties; for (const property of SensitiveProperties) { const value = get(settings, property); if (value) { @@ -30,7 +29,6 @@ const refreshLocalStorage = (botName: string, settings: DialogSetting) => { // merge sensitive values in localStorage const mergeLocalStorage = (botName: string, settings: DialogSetting) => { const localSetting = settingStorage.get(botName); - const SensitiveProperties = settings.SensitiveProperties || defaultSensitiveProperties; if (localSetting) { for (const property of SensitiveProperties) { const value = get(localSetting, property); diff --git a/Composer/packages/server/src/controllers/connector.ts b/Composer/packages/server/src/controllers/connector.ts index fb746665a8..d2d503a032 100644 --- a/Composer/packages/server/src/controllers/connector.ts +++ b/Composer/packages/server/src/controllers/connector.ts @@ -14,7 +14,7 @@ async function connect(req: any, res: any) { res.send({ botEndpoint }); } catch (error) { res.status(400).json({ - message: 'cannot connect to a bot runtime, make sure you start the bot runtime', + message: error.message || 'cannot connect to a bot runtime, make sure you start the bot runtime', }); } } @@ -34,7 +34,7 @@ async function getPublishHistory(req: any, res: any) { async function sync(req: any, res: any) { try { const environment = EnvironmentProvider.getCurrent(); - const settingsInDisk = await environment.getSettingsManager().get('', false); + const settingsInDisk = await environment.getSettingsManager().get(''); await environment.getBotConnector().sync(merge(settingsInDisk, req.body, { user: req.user })); res.send('OK'); } catch (error) { diff --git a/Composer/packages/server/src/models/connector/csharpBotConnector.ts b/Composer/packages/server/src/models/connector/csharpBotConnector.ts index 68d78dbf8e..f9e3cb56f5 100644 --- a/Composer/packages/server/src/models/connector/csharpBotConnector.ts +++ b/Composer/packages/server/src/models/connector/csharpBotConnector.ts @@ -14,6 +14,8 @@ import log from '../../logger'; import { BotConfig, BotEnvironments, BotStatus, IBotConnector, IPublishHistory } from './interface'; +const buildDebug = log.extend('build bot runtime'); +const runtimeDebugs: { [key: string]: debug.Debugger } = {}; export class CSharpBotConnector implements IBotConnector { public status: BotStatus = BotStatus.NotConnected; private endpoint: string; @@ -24,14 +26,14 @@ export class CSharpBotConnector implements IBotConnector { static stopAll = (signal: string) => { for (const pid in CSharpBotConnector.botRuntimes) { const runtime = CSharpBotConnector.botRuntimes[pid]; - log(`kill runtime before exit at ${runtime.pid}`); + runtimeDebugs[pid]('successfully stopped bot runtime: %d', pid); runtime.kill(signal); delete CSharpBotConnector.botRuntimes[pid]; } }; private stop = () => { - CSharpBotConnector.stopAll('SIGKILL'); + CSharpBotConnector.stopAll('SIGINT'); this.status = BotStatus.NotConnected; }; @@ -43,7 +45,7 @@ export class CSharpBotConnector implements IBotConnector { shell: true, stdio: ['ignore', 'ignore', 'inherit'], }); - log(`build pid : ${build.pid}`); + buildDebug('building bot runtime: %d', build.pid); build.stderr && build.stderr.on('data', function(err) { @@ -76,38 +78,39 @@ export class CSharpBotConnector implements IBotConnector { return configList; }; private addListeners = (child: ChildProcess, handler: Function) => { + const currentDebugger = runtimeDebugs[child.pid]; if (child.stdout !== null) { child.stdout.on('data', (data: any) => { - log(`stdout: ${data}`); + currentDebugger('bot runtime (%d): %s', child.pid, data); }); } if (child.stderr !== null) { child.stderr.on('data', (data: any) => { - log(`stderr: ${data}`); + currentDebugger('bot runtime (%d): Error %s', child.pid, data); }); } child.on('close', code => { - log(`close ${code}`); + currentDebugger('close %d', code); handler(); }); child.on('error', (err: any) => { - log(`stderr: ${err}`); + currentDebugger('stderr: %s', err); }); child.on('exit', code => { - log(`exit: ${code}`); + currentDebugger('exit %d', code); handler(); }); child.on('message', msg => { - log(msg); + currentDebugger('bot runtime received: %s', msg); }); child.on('disconnect', code => { - log(`disconnect: ${code}`); + currentDebugger('disconnect %d', code); handler(); }); }; @@ -130,29 +133,45 @@ export class CSharpBotConnector implements IBotConnector { stdio: ['ignore', 'ignore', 'inherit'], } ); - log(`runtime started. pid: ${runtime.pid}`); + runtimeDebugs[runtime.pid] = log.extend(`port ${runtime.pid}`); + runtimeDebugs[runtime.pid]('bot runtime started. pid: %d', runtime.pid); this.addListeners(runtime, this.stop); CSharpBotConnector.botRuntimes[runtime.pid] = runtime; this.status = BotStatus.Connected; }; private checkPortUsable = async (port: string | number): Promise => { - return new Promise((resolve, reject) => { - const netstat = spawn('netstat', ['-ano']); - const findstr = spawn('findstr', [`${port}`], { - stdio: [netstat.stdout, 'pipe', process.stderr], + if (process.platform === 'win32') { + return new Promise((resolve, reject) => { + const netstat = spawn('netstat', ['-ano']); + const findstr = spawn('findstr', [`${port}`], { + stdio: [netstat.stdout, 'pipe', process.stderr], + }); + let output = ''; + findstr.stderr && + findstr.stderr.on('data', function(err) { + reject(err.toString()); + }); + findstr.stdout && findstr.stdout.on('data', data => (output += data)); + findstr.on('exit', function() { + resolve(output); + }); }); - let output = ''; - findstr.stderr && - findstr.stderr.on('data', function(err) { - reject(err.toString()); + } else { + return new Promise((resolve, reject) => { + const lsof = spawn('sudo', ['lsof', '-i', `:${port}`]); + let output = ''; + lsof.stderr && lsof.stderr.on('data', err => reject(err.toString())); + lsof.stdout && + lsof.stdout.on('data', function(chunk) { + output += chunk.toString(); + }); + + lsof.on('exit', function() { + resolve(output); }); - - findstr.stdout && findstr.stdout.on('data', data => (output += data)); - findstr.on('exit', function() { - resolve(output); }); - }); + } }; connect = async (_: BotEnvironments, __: string) => { @@ -161,7 +180,7 @@ export class CSharpBotConnector implements IBotConnector { if (portStatus.trim() === '') { return Promise.resolve(`${this.endpoint}/api/messages`); } else { - throw new Error('Port already been used'); + throw new Error(`Port ${port} already in use`); } }; @@ -215,17 +234,14 @@ export class CSharpBotConnector implements IBotConnector { } process.on('SIGINT', () => { - log('[SIGINT] start graceful shutdown'); CSharpBotConnector.stopAll('SIGINT'); - process.exit(1); + process.exit(0); }); process.on('SIGTERM', () => { - log('[SIGTERM] start graceful shutdown'); CSharpBotConnector.stopAll('SIGTERM'); - process.exit(1); + process.exit(0); }); process.on('SIGQUIT', () => { - log('[SIGQUIT] start graceful shutdown'); CSharpBotConnector.stopAll('SIGQUIT'); - process.exit(1); + process.exit(0); }); diff --git a/Composer/packages/server/src/models/settings/defaultSettingManager.ts b/Composer/packages/server/src/models/settings/defaultSettingManager.ts index 416c1d0abd..3df934368e 100644 --- a/Composer/packages/server/src/models/settings/defaultSettingManager.ts +++ b/Composer/packages/server/src/models/settings/defaultSettingManager.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. import omit from 'lodash/omit'; -import { SensitiveProperties as defaultSensitiveProperties } from '@bfc/shared'; +import { SensitiveProperties } from '@bfc/shared'; import { Path } from '../../utility/path'; @@ -34,7 +34,6 @@ export class DefaultSettingManager extends FileSettingManager { }; private filterOutSensitiveValue = (obj: any) => { - const SensitiveProperties = obj.SensitiveProperties || defaultSensitiveProperties; if (obj && typeof obj === 'object') { return omit(obj, SensitiveProperties); } diff --git a/Composer/packages/server/src/models/settings/fileSettingManager.ts b/Composer/packages/server/src/models/settings/fileSettingManager.ts index c8ea14c67e..3a43a77a22 100644 --- a/Composer/packages/server/src/models/settings/fileSettingManager.ts +++ b/Composer/packages/server/src/models/settings/fileSettingManager.ts @@ -18,7 +18,7 @@ export class FileSettingManager implements ISettingManager { this.storage = new LocalDiskStorage(); } - public get = async (slot: string, obfuscate: boolean): Promise => { + public get = async (slot: string, obfuscate = false): Promise => { this.validateSlot(slot); const path = this.getPath(slot); diff --git a/Composer/packages/server/src/models/settings/interface.ts b/Composer/packages/server/src/models/settings/interface.ts index 3514aae953..725f0712d4 100644 --- a/Composer/packages/server/src/models/settings/interface.ts +++ b/Composer/packages/server/src/models/settings/interface.ts @@ -4,6 +4,6 @@ export const OBFUSCATED_VALUE = '*****'; export interface ISettingManager { - get(slot: string, obfuscate: boolean): Promise; + get(slot: string, obfuscate?: boolean): Promise; set(slot: string, settings: any): Promise; } From 543785bb54780839861f121d7020841d19ef40a3 Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Wed, 4 Dec 2019 23:11:58 +0800 Subject: [PATCH 10/25] move build script from template to server --- Composer/package.json | 2 +- .../src/models/connector}/build_runtime.ps1 | 0 .../models/connector/csharpBotConnector.ts | 19 +++++++++++++++---- 3 files changed, 16 insertions(+), 5 deletions(-) rename {BotProject/Templates/CSharp/Scripts => Composer/packages/server/src/models/connector}/build_runtime.ps1 (100%) diff --git a/Composer/package.json b/Composer/package.json index 58cde06e03..1f1081b0db 100644 --- a/Composer/package.json +++ b/Composer/package.json @@ -29,7 +29,7 @@ "build:client": "yarn workspace @bfc/client build", "build:tools": "yarn workspace @bfc/tools build:all", "start": "cross-env NODE_ENV=production PORT=3000 yarn start:server", - "startall": "node scripts/update.js && concurrently --kill-others-on-fail \"npm:runtime\" \"npm:start\"", + "startall": "node scripts/update.js && yarn start", "start:dev": "concurrently \"npm:start:client\" \"npm:start:server:dev\"", "start:client": "yarn workspace @bfc/client start", "start:server": "yarn workspace @bfc/server start", diff --git a/BotProject/Templates/CSharp/Scripts/build_runtime.ps1 b/Composer/packages/server/src/models/connector/build_runtime.ps1 similarity index 100% rename from BotProject/Templates/CSharp/Scripts/build_runtime.ps1 rename to Composer/packages/server/src/models/connector/build_runtime.ps1 diff --git a/Composer/packages/server/src/models/connector/csharpBotConnector.ts b/Composer/packages/server/src/models/connector/csharpBotConnector.ts index f9e3cb56f5..8c7f892670 100644 --- a/Composer/packages/server/src/models/connector/csharpBotConnector.ts +++ b/Composer/packages/server/src/models/connector/csharpBotConnector.ts @@ -23,6 +23,7 @@ export class CSharpBotConnector implements IBotConnector { constructor(endpoint: string) { this.endpoint = endpoint; } + static stopAll = (signal: string) => { for (const pid in CSharpBotConnector.botRuntimes) { const runtime = CSharpBotConnector.botRuntimes[pid]; @@ -38,20 +39,29 @@ export class CSharpBotConnector implements IBotConnector { }; private buildProcess = async (dir: string): Promise => { + // check build file exist + const buildScript = Path.resolve(__dirname, './build_runtime.ps1'); + const fileExisted = fs.existsSync(buildScript); + if (!fileExisted) { + return Promise.reject(new Error('build script not existed')); + } + // build bot runtime return new Promise((resolve, reject) => { - const build = spawn(`pwsh ./Scripts/build_runtime.ps1`, { + const build = spawn('pwsh', [buildScript], { cwd: dir, detached: true, - shell: true, + windowsHide: true, stdio: ['ignore', 'ignore', 'inherit'], }); buildDebug('building bot runtime: %d', build.pid); - + build.stdout && + build.stdout.on('data', function(str) { + buildDebug('%s', str); + }); build.stderr && build.stderr.on('data', function(err) { reject(err.toString()); }); - build.on('exit', function(code) { resolve(code); }); @@ -77,6 +87,7 @@ export class CSharpBotConnector implements IBotConnector { return configList; }; + private addListeners = (child: ChildProcess, handler: Function) => { const currentDebugger = runtimeDebugs[child.pid]; if (child.stdout !== null) { From 1cc7e70602b3b8f9e78d57551a6c4ca18be8cbc6 Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Thu, 5 Dec 2019 22:38:07 +0800 Subject: [PATCH 11/25] polish output error message --- BotProject/Templates/CSharp/Program.cs | 5 - .../packages/client/src/constants/index.ts | 1 + .../packages/client/src/store/action/bot.ts | 10 +- .../client/src/store/reducer/index.ts | 1 + .../src/models/connector/build_runtime.ps1 | 25 ---- .../models/connector/csharpBotConnector.ts | 109 +++++++----------- 6 files changed, 54 insertions(+), 97 deletions(-) delete mode 100644 Composer/packages/server/src/models/connector/build_runtime.ps1 diff --git a/BotProject/Templates/CSharp/Program.cs b/BotProject/Templates/CSharp/Program.cs index fd4a5e14f0..75c13a9741 100644 --- a/BotProject/Templates/CSharp/Program.cs +++ b/BotProject/Templates/CSharp/Program.cs @@ -41,11 +41,6 @@ public static IWebHost BuildWebHost(string[] args) => Trace.WriteLine(ex.Message); } - if (env.IsDevelopment()) - { - config.AddUserSecrets(); - } - config .AddEnvironmentVariables() .AddCommandLine(args); diff --git a/Composer/packages/client/src/constants/index.ts b/Composer/packages/client/src/constants/index.ts index d3d416d0ce..1f7d922ba1 100644 --- a/Composer/packages/client/src/constants/index.ts +++ b/Composer/packages/client/src/constants/index.ts @@ -67,6 +67,7 @@ export enum ActionTypes { SET_CREATION_FLOW_STATUS = 'SET_CREATION_FLOW_STATUS', SET_DESIGN_PAGE_LOCATION = 'SET_DESIGN_PAGE_LOCATION', CONNECT_BOT_SUCCESS = 'CONNECT_BOT_SUCCESS', + CONNECT_BOT_FAILURE = 'CONNECT_BOT_FAILURE', RELOAD_BOT_SUCCESS = 'RELOAD_BOT_SUCCESS', SYNC_ENV_SETTING = 'SYNC_ENV_SETTING', GET_ENV_SETTING = 'GET_ENV_SETTING', diff --git a/Composer/packages/client/src/store/action/bot.ts b/Composer/packages/client/src/store/action/bot.ts index 976d9cd4d9..6082ec784d 100644 --- a/Composer/packages/client/src/store/action/bot.ts +++ b/Composer/packages/client/src/store/action/bot.ts @@ -13,6 +13,7 @@ export const connectBot: ActionCreator = async (store, settings) => { try { const res = await httpClient.get(path); + await reloadBot(store, settings); store.dispatch({ type: ActionTypes.CONNECT_BOT_SUCCESS, payload: { @@ -21,10 +22,15 @@ export const connectBot: ActionCreator = async (store, settings) => { }, }); } catch (err) { + store.dispatch({ + type: ActionTypes.CONNECT_BOT_FAILURE, + payload: { + status: 'unConnected', + }, + }); + console.error(err.response.data.message); throw new Error(err.response.data.message); } - - await reloadBot(store, settings); }; // return only the connect URL -- do not reload diff --git a/Composer/packages/client/src/store/reducer/index.ts b/Composer/packages/client/src/store/reducer/index.ts index 7de2dbf2b3..fad2a80052 100644 --- a/Composer/packages/client/src/store/reducer/index.ts +++ b/Composer/packages/client/src/store/reducer/index.ts @@ -313,6 +313,7 @@ export const reducer = createReducer({ [ActionTypes.REMOVE_LU_FAILURE]: noOp, [ActionTypes.PUBLISH_LU_SUCCCESS]: updateLuTemplate, [ActionTypes.CONNECT_BOT_SUCCESS]: setBotStatus, + [ActionTypes.CONNECT_BOT_FAILURE]: setBotStatus, [ActionTypes.RELOAD_BOT_SUCCESS]: setBotLoadErrorMsg, // [ActionTypes.RELOAD_BOT_FAILURE]: setBotLoadErrorMsg, [ActionTypes.SET_ERROR]: setError, diff --git a/Composer/packages/server/src/models/connector/build_runtime.ps1 b/Composer/packages/server/src/models/connector/build_runtime.ps1 deleted file mode 100644 index a2d36baf28..0000000000 --- a/Composer/packages/server/src/models/connector/build_runtime.ps1 +++ /dev/null @@ -1,25 +0,0 @@ -Param( - [object] $config, - [string] $customSettingFolder, - [string] $luisAuthroingKey, - [SecureString] $appPassword, - [string] $projFolder = $(Join-Path $(Get-Location) BotProject CSharp) -) - -if ($PSVersionTable.PSVersion.Major -lt 6){ - Write-Host "! Powershell 6 is required, current version is $($PSVersionTable.PSVersion.Major), please refer following documents for help." - Write-Host "For Windows - https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-windows?view=powershell-6" - Write-Host "For Mac - https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-macos?view=powershell-6" - Break -} - -if ((dotnet --version) -lt 3) { - Write-Host "! dotnet core 3.0 is required, please refer following documents for help." - Write-Host "https://dotnet.microsoft.com/download/dotnet-core/3.0" - Break -} - -# Init user secret id -dotnet user-secrets init - -dotnet build \ No newline at end of file diff --git a/Composer/packages/server/src/models/connector/csharpBotConnector.ts b/Composer/packages/server/src/models/connector/csharpBotConnector.ts index 8c7f892670..9243e3460a 100644 --- a/Composer/packages/server/src/models/connector/csharpBotConnector.ts +++ b/Composer/packages/server/src/models/connector/csharpBotConnector.ts @@ -1,12 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import fs from 'fs'; import { parse } from 'url'; import { ChildProcess, spawn } from 'child_process'; -import archiver from 'archiver'; - import { BotProjectService } from '../../services/project'; import { DialogSetting } from '../bot/interface'; import { Path } from '../../utility/path'; @@ -39,19 +36,11 @@ export class CSharpBotConnector implements IBotConnector { }; private buildProcess = async (dir: string): Promise => { - // check build file exist - const buildScript = Path.resolve(__dirname, './build_runtime.ps1'); - const fileExisted = fs.existsSync(buildScript); - if (!fileExisted) { - return Promise.reject(new Error('build script not existed')); - } // build bot runtime return new Promise((resolve, reject) => { - const build = spawn('pwsh', [buildScript], { + const build = spawn('dotnet', ['build'], { cwd: dir, - detached: true, - windowsHide: true, - stdio: ['ignore', 'ignore', 'inherit'], + stdio: ['ignore', 'ignore', 'pipe'], }); buildDebug('building bot runtime: %d', build.pid); build.stdout && @@ -88,42 +77,32 @@ export class CSharpBotConnector implements IBotConnector { return configList; }; - private addListeners = (child: ChildProcess, handler: Function) => { + private addListeners = (child: ChildProcess, handler: Function, resolve: Function, reject: Function) => { const currentDebugger = runtimeDebugs[child.pid]; - if (child.stdout !== null) { + let erroutput = ''; + child.stdout && child.stdout.on('data', (data: any) => { currentDebugger('bot runtime (%d): %s', child.pid, data); + resolve(child.pid); }); - } - if (child.stderr !== null) { - child.stderr.on('data', (data: any) => { - currentDebugger('bot runtime (%d): Error %s', child.pid, data); + child.stderr && + child.stderr.on('data', (err: any) => { + erroutput += err.toString(); }); - } - - child.on('close', code => { - currentDebugger('close %d', code); - handler(); - }); - - child.on('error', (err: any) => { - currentDebugger('stderr: %s', err); - }); child.on('exit', code => { currentDebugger('exit %d', code); handler(); + if (code !== 0) { + currentDebugger('exit %d: %s', code, erroutput); + reject(erroutput); + } }); child.on('message', msg => { currentDebugger('bot runtime received: %s', msg); }); - - child.on('disconnect', code => { - currentDebugger('disconnect %d', code); - handler(); - }); }; private getBotPath = () => { @@ -134,21 +113,30 @@ export class CSharpBotConnector implements IBotConnector { return Path.join(currentProject.dir); }; - private start = async (dir: string, config: DialogSetting) => { - const runtime = spawn( - 'dotnet', - ['bin/Debug/netcoreapp2.1/BotProject.dll', `--urls`, this.endpoint, ...this.getConnectorConfig(config)], - { - detached: true, - cwd: dir, - stdio: ['ignore', 'ignore', 'inherit'], - } - ); - runtimeDebugs[runtime.pid] = log.extend(`port ${runtime.pid}`); - runtimeDebugs[runtime.pid]('bot runtime started. pid: %d', runtime.pid); - this.addListeners(runtime, this.stop); - CSharpBotConnector.botRuntimes[runtime.pid] = runtime; - this.status = BotStatus.Connected; + private start = async (dir: string, config: DialogSetting): Promise => { + return new Promise((resolve, reject) => { + const runtime = spawn( + 'dotnet', + [ + 'bin/Debug/netcoreapp2.1/BotProject.dll', + `--urls`, + this.endpoint, + ...this.getConnectorConfig(config), + '--environment', + 'development', + ], + { + detached: true, + cwd: dir, + stdio: ['ignore', 'pipe', 'pipe'], + } + ); + // extend runtime debugger + runtimeDebugs[runtime.pid] = log.extend(`port ${runtime.pid}`); + runtimeDebugs[runtime.pid]('bot runtime started. pid: %d', runtime.pid); + CSharpBotConnector.botRuntimes[runtime.pid] = runtime; + this.addListeners(runtime, this.stop, resolve, reject); + }); }; private checkPortUsable = async (port: string | number): Promise => { @@ -186,7 +174,10 @@ export class CSharpBotConnector implements IBotConnector { }; connect = async (_: BotEnvironments, __: string) => { - const port = parse(this.endpoint).port || '3979'; + const port = parse(this.endpoint).port; + if (!port) { + return Promise.resolve(''); + } const portStatus = await this.checkPortUsable(port); if (portStatus.trim() === '') { return Promise.resolve(`${this.endpoint}/api/messages`); @@ -201,26 +192,14 @@ export class CSharpBotConnector implements IBotConnector { const dir = this.getBotPath(); await this.buildProcess(dir); await this.start(dir, config); + this.status = BotStatus.Connected; } catch (err) { this.stop(); - throw err; + this.status = BotStatus.NotConnected; + throw new Error('Runtime Exception'); } }; - archiveDirectory = (src: string, dest: string) => { - return new Promise((resolve, reject) => { - const archive = archiver('zip'); - const output = fs.createWriteStream(dest); - - archive.pipe(output); - archive.directory(src, false); - archive.finalize(); - - output.on('close', () => resolve(archive)); - archive.on('error', err => reject(err)); - }); - }; - getEditingStatus = (): Promise => { return new Promise(resolve => { resolve(true); From 4336354ec568f834cf4d9528cb3e08af079f5702 Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Fri, 6 Dec 2019 10:43:54 +0800 Subject: [PATCH 12/25] remove environment --- .../server/src/models/connector/csharpBotConnector.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Composer/packages/server/src/models/connector/csharpBotConnector.ts b/Composer/packages/server/src/models/connector/csharpBotConnector.ts index 9243e3460a..d1b205902f 100644 --- a/Composer/packages/server/src/models/connector/csharpBotConnector.ts +++ b/Composer/packages/server/src/models/connector/csharpBotConnector.ts @@ -117,14 +117,7 @@ export class CSharpBotConnector implements IBotConnector { return new Promise((resolve, reject) => { const runtime = spawn( 'dotnet', - [ - 'bin/Debug/netcoreapp2.1/BotProject.dll', - `--urls`, - this.endpoint, - ...this.getConnectorConfig(config), - '--environment', - 'development', - ], + ['bin/Debug/netcoreapp2.1/BotProject.dll', `--urls`, this.endpoint, ...this.getConnectorConfig(config)], { detached: true, cwd: dir, From ffb617e7f85c6484a86fd38f06cc5e8bcf931dc2 Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Fri, 6 Dec 2019 17:56:31 +0800 Subject: [PATCH 13/25] fix comments and add back the build script into template runtime --- BotProject/Templates/CSharp/Program.cs | 5 ++ .../CSharp/Scripts/build_runtime.ps1 | 25 +++++++ .../packages/client/src/store/action/bot.ts | 1 - Composer/packages/server/package.json | 1 + .../server/src/controllers/connector.ts | 2 +- .../models/connector/csharpBotConnector.ts | 70 ++++++------------- .../src/models/settings/fileSettingManager.ts | 2 +- .../server/src/models/settings/interface.ts | 2 +- 8 files changed, 54 insertions(+), 54 deletions(-) create mode 100644 BotProject/Templates/CSharp/Scripts/build_runtime.ps1 diff --git a/BotProject/Templates/CSharp/Program.cs b/BotProject/Templates/CSharp/Program.cs index 75c13a9741..fd4a5e14f0 100644 --- a/BotProject/Templates/CSharp/Program.cs +++ b/BotProject/Templates/CSharp/Program.cs @@ -41,6 +41,11 @@ public static IWebHost BuildWebHost(string[] args) => Trace.WriteLine(ex.Message); } + if (env.IsDevelopment()) + { + config.AddUserSecrets(); + } + config .AddEnvironmentVariables() .AddCommandLine(args); diff --git a/BotProject/Templates/CSharp/Scripts/build_runtime.ps1 b/BotProject/Templates/CSharp/Scripts/build_runtime.ps1 new file mode 100644 index 0000000000..f1fd97e738 --- /dev/null +++ b/BotProject/Templates/CSharp/Scripts/build_runtime.ps1 @@ -0,0 +1,25 @@ +Param( + [object] $config, + [string] $customSettingFolder, + [string] $luisAuthroingKey, + [SecureString] $appPassword, + [string] $projFolder = $(Join-Path $(Get-Location) BotProject CSharp) +) + +if ($PSVersionTable.PSVersion.Major -lt 6){ + Write-Host "! Powershell 6 is required, current version is $($PSVersionTable.PSVersion.Major), please refer following documents for help." + Write-Host "For Windows - https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-windows?view=powershell-6" + Write-Host "For Mac - https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-macos?view=powershell-6" + Break +} + +if ((dotnet --version) -lt 3) { + Write-Host "! dotnet core 3.0 is required, please refer following documents for help." + Write-Host "https://dotnet.microsoft.com/download/dotnet-core/3.0" + Break +} + +# Init user secret id +dotnet user-secrets init + +dotnet build \ No newline at end of file diff --git a/Composer/packages/client/src/store/action/bot.ts b/Composer/packages/client/src/store/action/bot.ts index 6082ec784d..e35dd7f6e5 100644 --- a/Composer/packages/client/src/store/action/bot.ts +++ b/Composer/packages/client/src/store/action/bot.ts @@ -28,7 +28,6 @@ export const connectBot: ActionCreator = async (store, settings) => { status: 'unConnected', }, }); - console.error(err.response.data.message); throw new Error(err.response.data.message); } }; diff --git a/Composer/packages/server/package.json b/Composer/packages/server/package.json index a87658097c..ccbbc4d5a1 100644 --- a/Composer/packages/server/package.json +++ b/Composer/packages/server/package.json @@ -71,6 +71,7 @@ "express": "^4.16.4", "form-data": "^2.3.3", "format-message": "^6.2.1", + "get-port": "^5.0.0", "globby": "^9.1.0", "http-errors": "^1.7.2", "immer": "^2.1.4", diff --git a/Composer/packages/server/src/controllers/connector.ts b/Composer/packages/server/src/controllers/connector.ts index d2d503a032..28e438d969 100644 --- a/Composer/packages/server/src/controllers/connector.ts +++ b/Composer/packages/server/src/controllers/connector.ts @@ -35,7 +35,7 @@ async function sync(req: any, res: any) { try { const environment = EnvironmentProvider.getCurrent(); const settingsInDisk = await environment.getSettingsManager().get(''); - await environment.getBotConnector().sync(merge(settingsInDisk, req.body, { user: req.user })); + await environment.getBotConnector().sync(merge({}, settingsInDisk, req.body, { user: req.user })); res.send('OK'); } catch (error) { res.status(400).json({ diff --git a/Composer/packages/server/src/models/connector/csharpBotConnector.ts b/Composer/packages/server/src/models/connector/csharpBotConnector.ts index d1b205902f..7e2073683c 100644 --- a/Composer/packages/server/src/models/connector/csharpBotConnector.ts +++ b/Composer/packages/server/src/models/connector/csharpBotConnector.ts @@ -1,8 +1,11 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { parse } from 'url'; +import { parse as urlParse } from 'url'; import { ChildProcess, spawn } from 'child_process'; +import fs from 'fs'; + +import getPort from 'get-port'; import { BotProjectService } from '../../services/project'; import { DialogSetting } from '../bot/interface'; @@ -24,8 +27,8 @@ export class CSharpBotConnector implements IBotConnector { static stopAll = (signal: string) => { for (const pid in CSharpBotConnector.botRuntimes) { const runtime = CSharpBotConnector.botRuntimes[pid]; - runtimeDebugs[pid]('successfully stopped bot runtime: %d', pid); runtime.kill(signal); + runtimeDebugs[pid]('successfully stopped bot runtime: %d', pid); delete CSharpBotConnector.botRuntimes[pid]; } }; @@ -36,9 +39,13 @@ export class CSharpBotConnector implements IBotConnector { }; private buildProcess = async (dir: string): Promise => { + // check the build script existed + if (!fs.existsSync(Path.resolve(dir, './Scripts/build_runtime.ps1'))) { + throw new Error('build script not existed'); + } // build bot runtime return new Promise((resolve, reject) => { - const build = spawn('dotnet', ['build'], { + const build = spawn('pwsh', ['./Scripts/build_runtime.ps1'], { cwd: dir, stdio: ['ignore', 'ignore', 'pipe'], }); @@ -119,64 +126,25 @@ export class CSharpBotConnector implements IBotConnector { 'dotnet', ['bin/Debug/netcoreapp2.1/BotProject.dll', `--urls`, this.endpoint, ...this.getConnectorConfig(config)], { - detached: true, cwd: dir, stdio: ['ignore', 'pipe', 'pipe'], } ); // extend runtime debugger - runtimeDebugs[runtime.pid] = log.extend(`port ${runtime.pid}`); + runtimeDebugs[runtime.pid] = log.extend(`process ${runtime.pid}`); runtimeDebugs[runtime.pid]('bot runtime started. pid: %d', runtime.pid); CSharpBotConnector.botRuntimes[runtime.pid] = runtime; this.addListeners(runtime, this.stop, resolve, reject); }); }; - private checkPortUsable = async (port: string | number): Promise => { - if (process.platform === 'win32') { - return new Promise((resolve, reject) => { - const netstat = spawn('netstat', ['-ano']); - const findstr = spawn('findstr', [`${port}`], { - stdio: [netstat.stdout, 'pipe', process.stderr], - }); - let output = ''; - findstr.stderr && - findstr.stderr.on('data', function(err) { - reject(err.toString()); - }); - findstr.stdout && findstr.stdout.on('data', data => (output += data)); - findstr.on('exit', function() { - resolve(output); - }); - }); - } else { - return new Promise((resolve, reject) => { - const lsof = spawn('sudo', ['lsof', '-i', `:${port}`]); - let output = ''; - lsof.stderr && lsof.stderr.on('data', err => reject(err.toString())); - lsof.stdout && - lsof.stdout.on('data', function(chunk) { - output += chunk.toString(); - }); - - lsof.on('exit', function() { - resolve(output); - }); - }); - } - }; - connect = async (_: BotEnvironments, __: string) => { - const port = parse(this.endpoint).port; - if (!port) { - return Promise.resolve(''); - } - const portStatus = await this.checkPortUsable(port); - if (portStatus.trim() === '') { - return Promise.resolve(`${this.endpoint}/api/messages`); - } else { - throw new Error(`Port ${port} already in use`); - } + const originPort = urlParse(this.endpoint).port; + const protocol = urlParse(this.endpoint).protocol; + const hostName = urlParse(this.endpoint).hostname; + const port = await getPort({ host: hostName, port: parseInt(originPort || '3979') }); + this.endpoint = `${protocol}//${hostName}:${port}`; + return Promise.resolve(`${this.endpoint}/api/messages`); }; sync = async (config: DialogSetting) => { @@ -215,7 +183,9 @@ export class CSharpBotConnector implements IBotConnector { }); }; } - +process.on('uncaughtException', err => { + log(err); +}); process.on('SIGINT', () => { CSharpBotConnector.stopAll('SIGINT'); process.exit(0); diff --git a/Composer/packages/server/src/models/settings/fileSettingManager.ts b/Composer/packages/server/src/models/settings/fileSettingManager.ts index 3a43a77a22..df9887d532 100644 --- a/Composer/packages/server/src/models/settings/fileSettingManager.ts +++ b/Composer/packages/server/src/models/settings/fileSettingManager.ts @@ -18,7 +18,7 @@ export class FileSettingManager implements ISettingManager { this.storage = new LocalDiskStorage(); } - public get = async (slot: string, obfuscate = false): Promise => { + public get = async (slot = '', obfuscate = false): Promise => { this.validateSlot(slot); const path = this.getPath(slot); diff --git a/Composer/packages/server/src/models/settings/interface.ts b/Composer/packages/server/src/models/settings/interface.ts index 725f0712d4..2a839af129 100644 --- a/Composer/packages/server/src/models/settings/interface.ts +++ b/Composer/packages/server/src/models/settings/interface.ts @@ -4,6 +4,6 @@ export const OBFUSCATED_VALUE = '*****'; export interface ISettingManager { - get(slot: string, obfuscate?: boolean): Promise; + get(slot?: string, obfuscate?: boolean): Promise; set(slot: string, settings: any): Promise; } From 3d67936d64dcb90c3bd1cec4a65c3fd7143801c2 Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Mon, 9 Dec 2019 16:49:51 +0800 Subject: [PATCH 14/25] add migrate old bot feature --- .../server/src/models/asset/assetManager.ts | 20 +++++++----- .../models/connector/csharpBotConnector.ts | 31 +++++++++++++------ 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/Composer/packages/server/src/models/asset/assetManager.ts b/Composer/packages/server/src/models/asset/assetManager.ts index 440b5b4ace..cc53f73401 100644 --- a/Composer/packages/server/src/models/asset/assetManager.ts +++ b/Composer/packages/server/src/models/asset/assetManager.ts @@ -10,6 +10,7 @@ import { LocationRef } from '../bot/interface'; import { Path } from '../../utility/path'; import { copyDir } from '../../utility/storage'; import StorageService from '../../services/storage'; +import { IFileStorage } from '../storage/interface'; interface TemplateData { [key: string]: { @@ -157,17 +158,25 @@ export class AssetManager { return output; } - public async copyProjectTemplateTo(templateId: string, ref: LocationRef): Promise { + public async copyDataFilesTo(templateId: string, dstDir: string, dstStorage: IFileStorage) { const template = find(this.projectTemplates, { id: templateId }); if (template === undefined || template.path === undefined) { throw new Error(`no such template with id ${templateId}`); } + // copy Composer data files + await copyDir(template.path, this.templateStorage, dstDir, dstStorage); + } + public async copyRuntimeTo(dstDir: string, dstStorage: IFileStorage) { const runtime = find(this.runtimeTemplates, { id: DEFAULT_RUNTIME }); if (runtime === undefined || runtime.path === undefined) { throw new Error(`no such runtime with id ${DEFAULT_RUNTIME}`); } + // copy runtime code files + await copyDir(runtime.path, this.templateStorage, dstDir, dstStorage); + } + public async copyProjectTemplateTo(templateId: string, ref: LocationRef): Promise { // user storage maybe diff from template storage const dstStorage = StorageService.getStorageClient(ref.storageId); const dstDir = Path.resolve(ref.path); @@ -175,13 +184,8 @@ export class AssetManager { log('Failed copying template to %s', dstDir); throw new Error('already have this folder, please give another name'); } - - // copy Composer data files - await copyDir(template.path, this.templateStorage, dstDir, dstStorage); - - // copy runtime code files - await copyDir(runtime.path, this.templateStorage, dstDir, dstStorage); - + await this.copyDataFilesTo(templateId, dstDir, dstStorage); + await this.copyRuntimeTo(dstDir, dstStorage); return ref; } } diff --git a/Composer/packages/server/src/models/connector/csharpBotConnector.ts b/Composer/packages/server/src/models/connector/csharpBotConnector.ts index 7e2073683c..30a1c496b5 100644 --- a/Composer/packages/server/src/models/connector/csharpBotConnector.ts +++ b/Composer/packages/server/src/models/connector/csharpBotConnector.ts @@ -11,6 +11,8 @@ import { BotProjectService } from '../../services/project'; import { DialogSetting } from '../bot/interface'; import { Path } from '../../utility/path'; import log from '../../logger'; +import AssectService from '../../services/asset'; +import { IFileStorage } from '../storage/interface'; import { BotConfig, BotEnvironments, BotStatus, IBotConnector, IPublishHistory } from './interface'; @@ -20,6 +22,7 @@ export class CSharpBotConnector implements IBotConnector { public status: BotStatus = BotStatus.NotConnected; private endpoint: string; static botRuntimes: { [key: string]: ChildProcess } = {}; + static readonly DEFAULT_RUNTIME = 'CSharp'; constructor(endpoint: string) { this.endpoint = endpoint; } @@ -38,11 +41,19 @@ export class CSharpBotConnector implements IBotConnector { this.status = BotStatus.NotConnected; }; - private buildProcess = async (dir: string): Promise => { - // check the build script existed - if (!fs.existsSync(Path.resolve(dir, './Scripts/build_runtime.ps1'))) { - throw new Error('build script not existed'); + private isOldBot = (dir: string): boolean => { + // check bot the bot version through build script existence. + return !fs.existsSync(Path.resolve(dir, './Scripts/build_runtime.ps1')); + }; + + private migrateBot = async (dir: string, storage: IFileStorage) => { + if (this.isOldBot(dir)) { + // cover the old bot runtime with new runtime template + await AssectService.manager.copyRuntimeTo(dir, storage); } + }; + + private buildProcess = async (dir: string): Promise => { // build bot runtime return new Promise((resolve, reject) => { const build = spawn('pwsh', ['./Scripts/build_runtime.ps1'], { @@ -112,12 +123,12 @@ export class CSharpBotConnector implements IBotConnector { }); }; - private getBotPath = () => { + private getBotPathAndStorage = () => { const currentProject = BotProjectService.getCurrentBotProject(); if (currentProject === undefined) { throw new Error('no project is opened, nothing to sync'); } - return Path.join(currentProject.dir); + return { dir: Path.join(currentProject.dir), storage: currentProject.fileStorage }; }; private start = async (dir: string, config: DialogSetting): Promise => { @@ -150,7 +161,9 @@ export class CSharpBotConnector implements IBotConnector { sync = async (config: DialogSetting) => { try { this.stop(); - const dir = this.getBotPath(); + const { dir, storage } = this.getBotPathAndStorage(); + await this.migrateBot(dir, storage); + await this.buildProcess(dir); await this.start(dir, config); this.status = BotStatus.Connected; @@ -183,9 +196,7 @@ export class CSharpBotConnector implements IBotConnector { }); }; } -process.on('uncaughtException', err => { - log(err); -}); + process.on('SIGINT', () => { CSharpBotConnector.stopAll('SIGINT'); process.exit(0); From 5aebd4834db018534bbdf95b7b344b48a5dfde6d Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Tue, 10 Dec 2019 23:13:45 +0800 Subject: [PATCH 15/25] change docker file --- Composer/Dockerfile | 228 +++++++++++++++++++++++++++++++++++++++++++- docker-compose.yml | 8 +- 2 files changed, 226 insertions(+), 10 deletions(-) diff --git a/Composer/Dockerfile b/Composer/Dockerfile index 351572a63c..3d2670eac3 100644 --- a/Composer/Dockerfile +++ b/Composer/Dockerfile @@ -6,7 +6,6 @@ # before doing yarn install due to yarn workspace symlinking. # ################ - FROM node:12-alpine as build WORKDIR /src/Composer @@ -18,8 +17,229 @@ ENV NODE_ENV "production" RUN yarn build:prod -# use a multi-stage build to reduce the final image size -FROM node:12-alpine +# Define arg(s) needed for the From statement +FROM alpine:3.8 as installer-env + +# Define Args for the needed to add the package +ARG PS_VERSION=6.2.0 +ARG PS_PACKAGE=powershell-${PS_VERSION}-linux-alpine-x64.tar.gz +ARG PS_PACKAGE_URL=https://github.com/PowerShell/PowerShell/releases/download/v${PS_VERSION}/${PS_PACKAGE} +ARG PS_INSTALL_VERSION=6 + +# Download the Linux tar.gz and save it +ADD ${PS_PACKAGE_URL} /tmp/linux.tar.gz + +# define the folder we will be installing PowerShell to +ENV PS_INSTALL_FOLDER=/opt/microsoft/powershell/$PS_INSTALL_VERSION + +# Create the install folder +RUN mkdir -p ${PS_INSTALL_FOLDER} + +# Unzip the Linux tar.gz +RUN tar zxf /tmp/linux.tar.gz -C ${PS_INSTALL_FOLDER} -v + + + + + +FROM alpine:3.10 +# Copy only the files we need from the previous stage +COPY --from=installer-env ["/opt/microsoft/powershell", "/opt/microsoft/powershell"] + +# Define Args and Env needed to create links +ARG PS_INSTALL_VERSION=6 +ENV PS_INSTALL_FOLDER=/opt/microsoft/powershell/$PS_INSTALL_VERSION \ + \ + # Define ENVs for Localization/Globalization + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=false \ + LC_ALL=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + # set a fixed location for the Module analysis cache + PSModuleAnalysisCachePath=/var/cache/microsoft/powershell/PSModuleAnalysisCache/ModuleAnalysisCache + +# Install dotnet dependencies and ca-certificates +RUN apk add --no-cache \ + ca-certificates \ + less \ + \ + # PSReadline/console dependencies + ncurses-terminfo-base \ + \ + # .NET Core dependencies + krb5-libs \ + libgcc \ + libintl \ + libssl1.1 \ + libstdc++ \ + tzdata \ + userspace-rcu \ + zlib \ + icu-libs \ + && apk -X https://dl-cdn.alpinelinux.org/alpine/edge/main add --no-cache \ + lttng-ust \ + \ + # Create the pwsh symbolic link that points to powershell + && ln -s ${PS_INSTALL_FOLDER}/pwsh /usr/bin/pwsh \ + # Give all user execute permissions and remove write permissions for others + && chmod a+x,o-w ${PS_INSTALL_FOLDER}/pwsh \ + # intialize powershell module cache + && pwsh \ + -NoLogo \ + -NoProfile \ + -Command " \ + \$ErrorActionPreference = 'Stop' ; \ + \$ProgressPreference = 'SilentlyContinue' ; \ + while(!(Test-Path -Path \$env:PSModuleAnalysisCachePath)) { \ + Write-Host "'Waiting for $env:PSModuleAnalysisCachePath'" ; \ + Start-Sleep -Seconds 6 ; \ + }" + +# Define args needed only for the labels +ARG PS_VERSION=6.2.0 +ARG IMAGE_NAME=mcr.microsoft.com/powershell:alpine-3.8 +ARG VCS_REF="none" + +# Add label last as it's just metadata and uses a lot of parameters +LABEL maintainer="PowerShell Team " \ + readme.md="https://github.com/PowerShell/PowerShell/blob/master/docker/README.md" \ + description="This Dockerfile will install the latest release of PowerShell." \ + org.label-schema.usage="https://github.com/PowerShell/PowerShell/tree/master/docker#run-the-docker-image-you-built" \ + org.label-schema.url="https://github.com/PowerShell/PowerShell/blob/master/docker/README.md" \ + org.label-schema.vcs-url="https://github.com/PowerShell/PowerShell-Docker" \ + org.label-schema.name="powershell" \ + org.label-schema.vendor="PowerShell" \ + org.label-schema.vcs-ref=${VCS_REF} \ + org.label-schema.version=${PS_VERSION} \ + org.label-schema.schema-version="1.0" \ + org.label-schema.docker.cmd="docker run ${IMAGE_NAME} pwsh -c '$psversiontable'" \ + org.label-schema.docker.cmd.devel="docker run ${IMAGE_NAME}" \ + org.label-schema.docker.cmd.test="docker run ${IMAGE_NAME} pwsh -c Invoke-Pester" \ + org.label-schema.docker.cmd.help="docker run ${IMAGE_NAME} pwsh -c Get-Help" + +# Install .Net Core SDK +ENV \ + # Unset the value from the base image + ASPNETCORE_URLS= \ + # Disable the invariant mode (set in base image) + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=false \ + # Enable correct mode for dotnet watch (only mode supported in a container) + DOTNET_USE_POLLING_FILE_WATCHER=true \ + LC_ALL=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + # Skip extraction of XML docs - generally not useful within an image/container - helps performance + NUGET_XMLDOC_MODE=skip \ + # PowerShell telemetry for docker image usage + POWERSHELL_DISTRIBUTION_CHANNEL=PSDocker-DotnetCoreSDK-Alpine-3.10 + +# Add dependencies for disabling invariant mode (set in base image) +RUN apk add --no-cache icu-libs + +# Install .NET Core SDK and .NET Core Runtime +RUN dotnet_sdk_version=3.1.100 \ + && dotnet_version=3.1.0 \ + && wget -O dotnet.tar.gz https://dotnetcli.azureedge.net/dotnet/Sdk/$dotnet_sdk_version/dotnet-sdk-$dotnet_sdk_version-linux-musl-x64.tar.gz \ + && dotnet_sha512='517c1dadbc9081e112f75589eb7160ef70183eb3d93fd55800e145b21f4dd6f5fbe19397ee7476aa16493e112ef95b311ff61bb08d9231b30a7ea609806d85ee' \ + && echo "$dotnet_sha512 dotnet.tar.gz" | sha512sum -c - \ + && mkdir -p /usr/share/dotnet \ + && tar -C /usr/share/dotnet -oxzf dotnet.tar.gz \ + && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet \ + && rm dotnet.tar.gz \ + # Trigger first run experience by running arbitrary cmd + && dotnet help + +# Enable detection of running in a container +ENV DOTNET_RUNNING_IN_CONTAINER=true \ + # Set the invariant mode since icu_libs isn't included (see https://github.com/dotnet/announcements/issues/20) + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=true + +# Install nodejs +ENV NODE_VERSION 12.13.1 +RUN addgroup -g 1000 node \ + && adduser -u 1000 -G node -s /bin/sh -D node \ + && apk add --no-cache \ + libstdc++ \ + && apk add --no-cache --virtual .build-deps \ + curl \ + && ARCH= && alpineArch="$(apk --print-arch)" \ + && case "${alpineArch##*-}" in \ + x86_64) \ + ARCH='x64' \ + CHECKSUM="cf493d306a6367fb7bcff5608731e1dd44b9ad8d64e7df7706916d8be0f497a1" \ + ;; \ + *) ;; \ + esac \ + && if [ -n "${CHECKSUM}" ]; then \ + set -eu; \ + curl -fsSLO --compressed "https://unofficial-builds.nodejs.org/download/release/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz"; \ + echo "$CHECKSUM node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz" | sha256sum -c - \ + && tar -xJf "node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ + && ln -s /usr/local/bin/node /usr/local/bin/nodejs; \ + else \ + echo "Building from source" \ + # backup build + && apk add --no-cache --virtual .build-deps-full \ + binutils-gold \ + g++ \ + gcc \ + gnupg \ + libgcc \ + linux-headers \ + make \ + python \ + # gpg keys listed at https://github.com/nodejs/node#release-keys + && for key in \ + 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \ + FD3A5288F042B6850C66B31F09FE44734EB7990E \ + 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \ + DD8F2338BAE7501E3DD5AC78C273792F7D83545D \ + C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \ + B9AE9905FFD7803F25714661B63B535A4C206CA9 \ + 77984A986EBC2AA786BC0F66B01FBB92821C587A \ + 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \ + 4ED778F539E3634C779C87C6D7062848A1AB005C \ + A48C2BEE680E841632CD4E44F07496B3EB3C1762 \ + B9E2F5981AA6E0CD28160D9FF13993A75599653C \ + ; do \ + gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ + gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ + done \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION.tar.xz" \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ + && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ + && grep " node-v$NODE_VERSION.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ + && tar -xf "node-v$NODE_VERSION.tar.xz" \ + && cd "node-v$NODE_VERSION" \ + && ./configure \ + && make -j$(getconf _NPROCESSORS_ONLN) V= \ + && make install \ + && apk del .build-deps-full \ + && cd .. \ + && rm -Rf "node-v$NODE_VERSION" \ + && rm "node-v$NODE_VERSION.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt; \ + fi \ + && rm -f "node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz" \ + && apk del .build-deps + +ENV YARN_VERSION 1.19.1 + +RUN apk add --no-cache --virtual .build-deps-yarn curl gnupg tar \ + && for key in \ + 6A010C5166006599AA17F08146C2130DFD2497F5 \ + ; do \ + gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ + gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ + done \ + && curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz" \ + && curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz.asc" \ + && gpg --batch --verify yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \ + && mkdir -p /opt \ + && tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ \ + && ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn \ + && ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg \ + && rm yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \ + && apk del .build-deps-yarn WORKDIR /app/Composer COPY --from=build /src/Composer/yarn.lock . @@ -29,6 +249,8 @@ COPY --from=build /src/Composer/packages/lib ./packages/lib COPY --from=build /src/Composer/packages/tools ./packages/tools ENV NODE_ENV "production" +ENV DEBUG "composer*" RUN yarn --production --frozen-lockfile --force && yarn cache clean WORKDIR /app/Composer CMD ["yarn", "start:server"] +# CMD ["dotnet", "--version" ] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index d5f6c88df3..6bbead3b95 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,11 +12,5 @@ services: COMPOSER_BOTS_FOLDER: /Bots COMPOSER_RUNTIME_FOLDER: /BotProject/Templates COMPOSER_APP_DATA: /appdata/data.json - BOT_ENDPOINT: http://botruntime:80 + BOT_ENDPOINT: http://composer:3979 PORT: 3000 - botruntime: - build: BotProject/CSharp - ports: - - "3979:80" - volumes: - - ~/Documents/Composer:/Bots From af79c2c1135e4a0a89bc38f8a09e9c05006d3a89 Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Tue, 10 Dec 2019 23:17:12 +0800 Subject: [PATCH 16/25] remove annotation --- Composer/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Composer/Dockerfile b/Composer/Dockerfile index 3d2670eac3..f4b180e0b6 100644 --- a/Composer/Dockerfile +++ b/Composer/Dockerfile @@ -252,5 +252,4 @@ ENV NODE_ENV "production" ENV DEBUG "composer*" RUN yarn --production --frozen-lockfile --force && yarn cache clean WORKDIR /app/Composer -CMD ["yarn", "start:server"] -# CMD ["dotnet", "--version" ] \ No newline at end of file +CMD ["yarn", "start:server"] \ No newline at end of file From 2985023b0519689e2922e23a846a3e21121d7bff Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Wed, 11 Dec 2019 23:09:50 +0800 Subject: [PATCH 17/25] fix docker --- Composer/Dockerfile | 26 ++++++++++++------- .../models/connector/csharpBotConnector.ts | 3 +-- docker-compose.yml | 1 + 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/Composer/Dockerfile b/Composer/Dockerfile index f4b180e0b6..a5f6e71c83 100644 --- a/Composer/Dockerfile +++ b/Composer/Dockerfile @@ -6,6 +6,7 @@ # before doing yarn install due to yarn workspace symlinking. # ################ + FROM node:12-alpine as build WORKDIR /src/Composer @@ -134,18 +135,25 @@ ENV \ # Add dependencies for disabling invariant mode (set in base image) RUN apk add --no-cache icu-libs -# Install .NET Core SDK and .NET Core Runtime -RUN dotnet_sdk_version=3.1.100 \ - && dotnet_version=3.1.0 \ - && wget -O dotnet.tar.gz https://dotnetcli.azureedge.net/dotnet/Sdk/$dotnet_sdk_version/dotnet-sdk-$dotnet_sdk_version-linux-musl-x64.tar.gz \ - && dotnet_sha512='517c1dadbc9081e112f75589eb7160ef70183eb3d93fd55800e145b21f4dd6f5fbe19397ee7476aa16493e112ef95b311ff61bb08d9231b30a7ea609806d85ee' \ +# Install .NET Core 2.1 +ENV DOTNET_SDK_VERSION 2.1.607 + +RUN wget -O dotnet.tar.gz https://dotnetcli.azureedge.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-musl-x64.tar.gz \ + && dotnet_sha512='61caf6602b8a2aa89769b3e91ddaec963d8ab9f802cd7f6c6da4f02426358712bc2bb0930e7ee3a81d75c7607039543b554cb8ed50e45610655f9e91ed0f2f17' \ && echo "$dotnet_sha512 dotnet.tar.gz" | sha512sum -c - \ && mkdir -p /usr/share/dotnet \ - && tar -C /usr/share/dotnet -oxzf dotnet.tar.gz \ + && tar -C /usr/share/dotnet -xzf dotnet.tar.gz \ && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet \ - && rm dotnet.tar.gz \ - # Trigger first run experience by running arbitrary cmd - && dotnet help + && rm dotnet.tar.gz + +# Install .NET Core SDK 3.0 +ENV DOTNET_SDK_VERSION 3.0.101 + +RUN wget -O dotnet.tar.gz https://dotnetcli.azureedge.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-musl-x64.tar.gz \ + && dotnet_sha512='98cc98f58187d208bd388f8c71862ea75e50ca25666e265f40a4e7c28082c2784738172e8ae4af7815057f7c57072cbe4fc03301d01738fc1ed5bb5e4d30a363' \ + && echo "$dotnet_sha512 dotnet.tar.gz" | sha512sum -c - \ + && tar -C /usr/share/dotnet -xzf dotnet.tar.gz \ + && rm dotnet.tar.gz # Enable detection of running in a container ENV DOTNET_RUNNING_IN_CONTAINER=true \ diff --git a/Composer/packages/server/src/models/connector/csharpBotConnector.ts b/Composer/packages/server/src/models/connector/csharpBotConnector.ts index 30a1c496b5..2c0a3f9a32 100644 --- a/Composer/packages/server/src/models/connector/csharpBotConnector.ts +++ b/Composer/packages/server/src/models/connector/csharpBotConnector.ts @@ -22,7 +22,6 @@ export class CSharpBotConnector implements IBotConnector { public status: BotStatus = BotStatus.NotConnected; private endpoint: string; static botRuntimes: { [key: string]: ChildProcess } = {}; - static readonly DEFAULT_RUNTIME = 'CSharp'; constructor(endpoint: string) { this.endpoint = endpoint; } @@ -58,7 +57,7 @@ export class CSharpBotConnector implements IBotConnector { return new Promise((resolve, reject) => { const build = spawn('pwsh', ['./Scripts/build_runtime.ps1'], { cwd: dir, - stdio: ['ignore', 'ignore', 'pipe'], + stdio: ['pipe', 'pipe', 'pipe'], }); buildDebug('building bot runtime: %d', build.pid); build.stdout && diff --git a/docker-compose.yml b/docker-compose.yml index 6bbead3b95..88aa7fb8a2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,7 @@ services: build: Composer ports: - "3000:3000" + - "3979:3979" volumes: - ~/Documents/Composer:/Bots - ./BotProject:/BotProject From 019f62314166dd944fbd679a4cc9a255cb8bf9c6 Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Thu, 12 Dec 2019 22:29:11 +0800 Subject: [PATCH 18/25] simpler dockerfile --- .../CSharp/Scripts/build_runtime.ps1 | 2 +- Composer/Dockerfile | 194 ++---------------- .../models/connector/csharpBotConnector.ts | 7 +- .../packages/server/src/settings/index.ts | 2 +- docker-compose.yml | 4 +- 5 files changed, 24 insertions(+), 185 deletions(-) diff --git a/BotProject/Templates/CSharp/Scripts/build_runtime.ps1 b/BotProject/Templates/CSharp/Scripts/build_runtime.ps1 index f1fd97e738..dc6cd87d9a 100644 --- a/BotProject/Templates/CSharp/Scripts/build_runtime.ps1 +++ b/BotProject/Templates/CSharp/Scripts/build_runtime.ps1 @@ -19,7 +19,7 @@ if ((dotnet --version) -lt 3) { Break } -# Init user secret id +# This command need dotnet core more than 3.0 dotnet user-secrets init dotnet build \ No newline at end of file diff --git a/Composer/Dockerfile b/Composer/Dockerfile index a5f6e71c83..b9aa3bd93e 100644 --- a/Composer/Dockerfile +++ b/Composer/Dockerfile @@ -18,53 +18,11 @@ ENV NODE_ENV "production" RUN yarn build:prod -# Define arg(s) needed for the From statement -FROM alpine:3.8 as installer-env -# Define Args for the needed to add the package -ARG PS_VERSION=6.2.0 -ARG PS_PACKAGE=powershell-${PS_VERSION}-linux-alpine-x64.tar.gz -ARG PS_PACKAGE_URL=https://github.com/PowerShell/PowerShell/releases/download/v${PS_VERSION}/${PS_PACKAGE} -ARG PS_INSTALL_VERSION=6 +FROM node:12-alpine -# Download the Linux tar.gz and save it -ADD ${PS_PACKAGE_URL} /tmp/linux.tar.gz - -# define the folder we will be installing PowerShell to -ENV PS_INSTALL_FOLDER=/opt/microsoft/powershell/$PS_INSTALL_VERSION - -# Create the install folder -RUN mkdir -p ${PS_INSTALL_FOLDER} - -# Unzip the Linux tar.gz -RUN tar zxf /tmp/linux.tar.gz -C ${PS_INSTALL_FOLDER} -v - - - - - -FROM alpine:3.10 -# Copy only the files we need from the previous stage -COPY --from=installer-env ["/opt/microsoft/powershell", "/opt/microsoft/powershell"] - -# Define Args and Env needed to create links -ARG PS_INSTALL_VERSION=6 -ENV PS_INSTALL_FOLDER=/opt/microsoft/powershell/$PS_INSTALL_VERSION \ - \ - # Define ENVs for Localization/Globalization - DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=false \ - LC_ALL=en_US.UTF-8 \ - LANG=en_US.UTF-8 \ - # set a fixed location for the Module analysis cache - PSModuleAnalysisCachePath=/var/cache/microsoft/powershell/PSModuleAnalysisCache/ModuleAnalysisCache - -# Install dotnet dependencies and ca-certificates RUN apk add --no-cache \ ca-certificates \ - less \ - \ - # PSReadline/console dependencies - ncurses-terminfo-base \ \ # .NET Core dependencies krb5-libs \ @@ -72,50 +30,7 @@ RUN apk add --no-cache \ libintl \ libssl1.1 \ libstdc++ \ - tzdata \ - userspace-rcu \ - zlib \ - icu-libs \ - && apk -X https://dl-cdn.alpinelinux.org/alpine/edge/main add --no-cache \ - lttng-ust \ - \ - # Create the pwsh symbolic link that points to powershell - && ln -s ${PS_INSTALL_FOLDER}/pwsh /usr/bin/pwsh \ - # Give all user execute permissions and remove write permissions for others - && chmod a+x,o-w ${PS_INSTALL_FOLDER}/pwsh \ - # intialize powershell module cache - && pwsh \ - -NoLogo \ - -NoProfile \ - -Command " \ - \$ErrorActionPreference = 'Stop' ; \ - \$ProgressPreference = 'SilentlyContinue' ; \ - while(!(Test-Path -Path \$env:PSModuleAnalysisCachePath)) { \ - Write-Host "'Waiting for $env:PSModuleAnalysisCachePath'" ; \ - Start-Sleep -Seconds 6 ; \ - }" - -# Define args needed only for the labels -ARG PS_VERSION=6.2.0 -ARG IMAGE_NAME=mcr.microsoft.com/powershell:alpine-3.8 -ARG VCS_REF="none" - -# Add label last as it's just metadata and uses a lot of parameters -LABEL maintainer="PowerShell Team " \ - readme.md="https://github.com/PowerShell/PowerShell/blob/master/docker/README.md" \ - description="This Dockerfile will install the latest release of PowerShell." \ - org.label-schema.usage="https://github.com/PowerShell/PowerShell/tree/master/docker#run-the-docker-image-you-built" \ - org.label-schema.url="https://github.com/PowerShell/PowerShell/blob/master/docker/README.md" \ - org.label-schema.vcs-url="https://github.com/PowerShell/PowerShell-Docker" \ - org.label-schema.name="powershell" \ - org.label-schema.vendor="PowerShell" \ - org.label-schema.vcs-ref=${VCS_REF} \ - org.label-schema.version=${PS_VERSION} \ - org.label-schema.schema-version="1.0" \ - org.label-schema.docker.cmd="docker run ${IMAGE_NAME} pwsh -c '$psversiontable'" \ - org.label-schema.docker.cmd.devel="docker run ${IMAGE_NAME}" \ - org.label-schema.docker.cmd.test="docker run ${IMAGE_NAME} pwsh -c Invoke-Pester" \ - org.label-schema.docker.cmd.help="docker run ${IMAGE_NAME} pwsh -c Get-Help" + zlib # Install .Net Core SDK ENV \ @@ -160,104 +75,31 @@ ENV DOTNET_RUNNING_IN_CONTAINER=true \ # Set the invariant mode since icu_libs isn't included (see https://github.com/dotnet/announcements/issues/20) DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=true -# Install nodejs -ENV NODE_VERSION 12.13.1 -RUN addgroup -g 1000 node \ - && adduser -u 1000 -G node -s /bin/sh -D node \ - && apk add --no-cache \ - libstdc++ \ - && apk add --no-cache --virtual .build-deps \ - curl \ - && ARCH= && alpineArch="$(apk --print-arch)" \ - && case "${alpineArch##*-}" in \ - x86_64) \ - ARCH='x64' \ - CHECKSUM="cf493d306a6367fb7bcff5608731e1dd44b9ad8d64e7df7706916d8be0f497a1" \ - ;; \ - *) ;; \ - esac \ - && if [ -n "${CHECKSUM}" ]; then \ - set -eu; \ - curl -fsSLO --compressed "https://unofficial-builds.nodejs.org/download/release/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz"; \ - echo "$CHECKSUM node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz" | sha256sum -c - \ - && tar -xJf "node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ - && ln -s /usr/local/bin/node /usr/local/bin/nodejs; \ - else \ - echo "Building from source" \ - # backup build - && apk add --no-cache --virtual .build-deps-full \ - binutils-gold \ - g++ \ - gcc \ - gnupg \ - libgcc \ - linux-headers \ - make \ - python \ - # gpg keys listed at https://github.com/nodejs/node#release-keys - && for key in \ - 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \ - FD3A5288F042B6850C66B31F09FE44734EB7990E \ - 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \ - DD8F2338BAE7501E3DD5AC78C273792F7D83545D \ - C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \ - B9AE9905FFD7803F25714661B63B535A4C206CA9 \ - 77984A986EBC2AA786BC0F66B01FBB92821C587A \ - 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \ - 4ED778F539E3634C779C87C6D7062848A1AB005C \ - A48C2BEE680E841632CD4E44F07496B3EB3C1762 \ - B9E2F5981AA6E0CD28160D9FF13993A75599653C \ - ; do \ - gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ - gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ - gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ - done \ - && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION.tar.xz" \ - && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ - && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ - && grep " node-v$NODE_VERSION.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ - && tar -xf "node-v$NODE_VERSION.tar.xz" \ - && cd "node-v$NODE_VERSION" \ - && ./configure \ - && make -j$(getconf _NPROCESSORS_ONLN) V= \ - && make install \ - && apk del .build-deps-full \ - && cd .. \ - && rm -Rf "node-v$NODE_VERSION" \ - && rm "node-v$NODE_VERSION.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt; \ - fi \ - && rm -f "node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz" \ - && apk del .build-deps - -ENV YARN_VERSION 1.19.1 +# Install PowerShell global tool +ENV POWERSHELL_VERSION=7.0.0-preview.4 \ + POWERSHELL_DISTRIBUTION_CHANNEL=PSDocker-DotnetCoreSDK-Alpine-3.10 -RUN apk add --no-cache --virtual .build-deps-yarn curl gnupg tar \ - && for key in \ - 6A010C5166006599AA17F08146C2130DFD2497F5 \ - ; do \ - gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ - gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ - gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ - done \ - && curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz" \ - && curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz.asc" \ - && gpg --batch --verify yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \ - && mkdir -p /opt \ - && tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ \ - && ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn \ - && ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg \ - && rm yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \ - && apk del .build-deps-yarn +RUN wget -O PowerShell.Linux.Alpine.$POWERSHELL_VERSION.nupkg https://pwshtool.blob.core.windows.net/tool/$POWERSHELL_VERSION/PowerShell.Linux.Alpine.$POWERSHELL_VERSION.nupkg \ + && powershell_sha512='25dc93d87e4a0ae94fd161ff3b2d26b0e0eb5ae69a913500f7816debd705d1b84fcebb147b8a02d5fc53cf2718ec3d65dc0f2b73e0e4b917599b6562cf140c39' \ + && echo "$powershell_sha512 PowerShell.Linux.Alpine.$POWERSHELL_VERSION.nupkg" | sha512sum -c - \ + && mkdir -p /usr/share/powershell \ + && dotnet tool install --add-source / --tool-path /usr/share/powershell --version $POWERSHELL_VERSION PowerShell.Linux.Alpine \ + && rm PowerShell.Linux.Alpine.$POWERSHELL_VERSION.nupkg \ + && ln -s /usr/share/powershell/pwsh /usr/bin/pwsh \ + && chmod 755 /usr/share/powershell/pwsh \ + # To reduce image size, remove the copy nupkg that nuget keeps. + && find /usr/share/powershell -print | grep -i '.*[.]nupkg$' | xargs rm \ + # Add ncurses-terminfo-base to resolve psreadline dependency + && apk add --no-cache ncurses-terminfo-base WORKDIR /app/Composer -COPY --from=build /src/Composer/yarn.lock . +COPY --from=build /src/Composer/yarn.lock .dotnet COPY --from=build /src/Composer/package.json . COPY --from=build /src/Composer/packages/server ./packages/server COPY --from=build /src/Composer/packages/lib ./packages/lib COPY --from=build /src/Composer/packages/tools ./packages/tools ENV NODE_ENV "production" -ENV DEBUG "composer*" RUN yarn --production --frozen-lockfile --force && yarn cache clean WORKDIR /app/Composer CMD ["yarn", "start:server"] \ No newline at end of file diff --git a/Composer/packages/server/src/models/connector/csharpBotConnector.ts b/Composer/packages/server/src/models/connector/csharpBotConnector.ts index 2c0a3f9a32..657cd9c6bc 100644 --- a/Composer/packages/server/src/models/connector/csharpBotConnector.ts +++ b/Composer/packages/server/src/models/connector/csharpBotConnector.ts @@ -150,11 +150,8 @@ export class CSharpBotConnector implements IBotConnector { connect = async (_: BotEnvironments, __: string) => { const originPort = urlParse(this.endpoint).port; - const protocol = urlParse(this.endpoint).protocol; - const hostName = urlParse(this.endpoint).hostname; - const port = await getPort({ host: hostName, port: parseInt(originPort || '3979') }); - this.endpoint = `${protocol}//${hostName}:${port}`; - return Promise.resolve(`${this.endpoint}/api/messages`); + const port = await getPort({ host: 'localhost', port: parseInt(originPort || '3979') }); + return Promise.resolve(`http://localhost:${port}/api/messages`); }; sync = async (config: DialogSetting) => { diff --git a/Composer/packages/server/src/settings/index.ts b/Composer/packages/server/src/settings/index.ts index e9d5231635..27294068ed 100644 --- a/Composer/packages/server/src/settings/index.ts +++ b/Composer/packages/server/src/settings/index.ts @@ -22,7 +22,7 @@ interface Settings { const envSettings: { [env: string]: Settings } = { development: { botAdminEndpoint: botEndpoint, - botEndpoint: 'http://localhost:3979', //botEndpoint, + botEndpoint: botEndpoint, assetsLibray: Path.resolve('./assets'), botsFolder: botsFolder || Path.join(os.homedir(), 'Documents', 'Composer'), runtimeFolder, diff --git a/docker-compose.yml b/docker-compose.yml index 88aa7fb8a2..845abdbda2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: build: Composer ports: - "3000:3000" - - "3979:3979" + - "3979:80" volumes: - ~/Documents/Composer:/Bots - ./BotProject:/BotProject @@ -13,5 +13,5 @@ services: COMPOSER_BOTS_FOLDER: /Bots COMPOSER_RUNTIME_FOLDER: /BotProject/Templates COMPOSER_APP_DATA: /appdata/data.json - BOT_ENDPOINT: http://composer:3979 + BOT_ENDPOINT: http://0.0.0.0:80 PORT: 3000 From c05ec0dfd7c7689acf44e7f0ced7f8d3b9b4db11 Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Fri, 13 Dec 2019 20:57:21 +0800 Subject: [PATCH 19/25] fix dockerfile bug and port mapping --- BotProject/CSharp/Dockerfile | 20 -------------------- Composer/Dockerfile | 2 +- docker-compose.yml | 4 ++-- 3 files changed, 3 insertions(+), 23 deletions(-) delete mode 100644 BotProject/CSharp/Dockerfile diff --git a/BotProject/CSharp/Dockerfile b/BotProject/CSharp/Dockerfile deleted file mode 100644 index 9893fa62bf..0000000000 --- a/BotProject/CSharp/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM mcr.microsoft.com/dotnet/core/sdk:2.1-alpine AS build - -WORKDIR /src/botproject/csharp - -COPY *.sln . -COPY *.csproj . -COPY Tests/*.csproj Tests/ -COPY NuGet.Config . - -# run restore to create a distinct layer -RUN dotnet restore - -COPY . . -RUN dotnet publish -o out - -FROM mcr.microsoft.com/dotnet/core/aspnet:2.1-alpine AS runtime -WORKDIR /app/botproject/csharp -COPY --from=build /src/botproject/csharp/ComposerDialogs ./ComposerDialogs -COPY --from=build /src/botproject/csharp/out . -CMD ["dotnet", "BotProject.dll"] diff --git a/Composer/Dockerfile b/Composer/Dockerfile index b9aa3bd93e..93f1f6fcc7 100644 --- a/Composer/Dockerfile +++ b/Composer/Dockerfile @@ -93,7 +93,7 @@ RUN wget -O PowerShell.Linux.Alpine.$POWERSHELL_VERSION.nupkg https://pwshtool.b && apk add --no-cache ncurses-terminfo-base WORKDIR /app/Composer -COPY --from=build /src/Composer/yarn.lock .dotnet +COPY --from=build /src/Composer/yarn.lock . COPY --from=build /src/Composer/package.json . COPY --from=build /src/Composer/packages/server ./packages/server COPY --from=build /src/Composer/packages/lib ./packages/lib diff --git a/docker-compose.yml b/docker-compose.yml index 845abdbda2..1b6ef5fd72 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: build: Composer ports: - "3000:3000" - - "3979:80" + - "3979:3979" volumes: - ~/Documents/Composer:/Bots - ./BotProject:/BotProject @@ -13,5 +13,5 @@ services: COMPOSER_BOTS_FOLDER: /Bots COMPOSER_RUNTIME_FOLDER: /BotProject/Templates COMPOSER_APP_DATA: /appdata/data.json - BOT_ENDPOINT: http://0.0.0.0:80 + BOT_ENDPOINT: http://0.0.0.0:3979 PORT: 3000 From 7a7a2cba6cc4c4d5aeac126b13a344aaa76149df Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Tue, 17 Dec 2019 10:34:55 +0800 Subject: [PATCH 20/25] add bash script for unix, and remove pwsh from docker --- .../Templates/CSharp/Scripts/build_runtime.sh | 11 +++++++++++ Composer/Dockerfile | 19 +------------------ .../models/connector/csharpBotConnector.ts | 14 ++++++++++++-- 3 files changed, 24 insertions(+), 20 deletions(-) create mode 100644 BotProject/Templates/CSharp/Scripts/build_runtime.sh diff --git a/BotProject/Templates/CSharp/Scripts/build_runtime.sh b/BotProject/Templates/CSharp/Scripts/build_runtime.sh new file mode 100644 index 0000000000..3a87dea815 --- /dev/null +++ b/BotProject/Templates/CSharp/Scripts/build_runtime.sh @@ -0,0 +1,11 @@ +versionString=`dotnet --version` +versionNum=`echo $versionString | cut -d . -f 1` +if [[ $versionNum -lt 3 ]] +then + echo "! dotnet core 3.0 is required, please refer following documents for help. +https://dotnet.microsoft.com/download/dotnet-core/3.0" + exit 1 +else + dotnet user-secrets init + dotnet build +fi diff --git a/Composer/Dockerfile b/Composer/Dockerfile index 93f1f6fcc7..67e923e64b 100644 --- a/Composer/Dockerfile +++ b/Composer/Dockerfile @@ -75,23 +75,6 @@ ENV DOTNET_RUNNING_IN_CONTAINER=true \ # Set the invariant mode since icu_libs isn't included (see https://github.com/dotnet/announcements/issues/20) DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=true -# Install PowerShell global tool -ENV POWERSHELL_VERSION=7.0.0-preview.4 \ - POWERSHELL_DISTRIBUTION_CHANNEL=PSDocker-DotnetCoreSDK-Alpine-3.10 - -RUN wget -O PowerShell.Linux.Alpine.$POWERSHELL_VERSION.nupkg https://pwshtool.blob.core.windows.net/tool/$POWERSHELL_VERSION/PowerShell.Linux.Alpine.$POWERSHELL_VERSION.nupkg \ - && powershell_sha512='25dc93d87e4a0ae94fd161ff3b2d26b0e0eb5ae69a913500f7816debd705d1b84fcebb147b8a02d5fc53cf2718ec3d65dc0f2b73e0e4b917599b6562cf140c39' \ - && echo "$powershell_sha512 PowerShell.Linux.Alpine.$POWERSHELL_VERSION.nupkg" | sha512sum -c - \ - && mkdir -p /usr/share/powershell \ - && dotnet tool install --add-source / --tool-path /usr/share/powershell --version $POWERSHELL_VERSION PowerShell.Linux.Alpine \ - && rm PowerShell.Linux.Alpine.$POWERSHELL_VERSION.nupkg \ - && ln -s /usr/share/powershell/pwsh /usr/bin/pwsh \ - && chmod 755 /usr/share/powershell/pwsh \ - # To reduce image size, remove the copy nupkg that nuget keeps. - && find /usr/share/powershell -print | grep -i '.*[.]nupkg$' | xargs rm \ - # Add ncurses-terminfo-base to resolve psreadline dependency - && apk add --no-cache ncurses-terminfo-base - WORKDIR /app/Composer COPY --from=build /src/Composer/yarn.lock . COPY --from=build /src/Composer/package.json . @@ -102,4 +85,4 @@ COPY --from=build /src/Composer/packages/tools ./packages/tools ENV NODE_ENV "production" RUN yarn --production --frozen-lockfile --force && yarn cache clean WORKDIR /app/Composer -CMD ["yarn", "start:server"] \ No newline at end of file +CMD ["yarn", "start:server"] diff --git a/Composer/packages/server/src/models/connector/csharpBotConnector.ts b/Composer/packages/server/src/models/connector/csharpBotConnector.ts index 657cd9c6bc..1ade737f74 100644 --- a/Composer/packages/server/src/models/connector/csharpBotConnector.ts +++ b/Composer/packages/server/src/models/connector/csharpBotConnector.ts @@ -42,7 +42,11 @@ export class CSharpBotConnector implements IBotConnector { private isOldBot = (dir: string): boolean => { // check bot the bot version through build script existence. - return !fs.existsSync(Path.resolve(dir, './Scripts/build_runtime.ps1')); + if (process.platform === 'win32') { + return !fs.existsSync(Path.resolve(dir, './Scripts/build_runtime.ps1')); + } else { + return !fs.existsSync(Path.resolve(dir, './Scripts/build_runtime.sh')); + } }; private migrateBot = async (dir: string, storage: IFileStorage) => { @@ -55,7 +59,13 @@ export class CSharpBotConnector implements IBotConnector { private buildProcess = async (dir: string): Promise => { // build bot runtime return new Promise((resolve, reject) => { - const build = spawn('pwsh', ['./Scripts/build_runtime.ps1'], { + let shell = 'sh'; + let script = './Scripts/build_runtime.sh'; + if (process.platform === 'win32') { + shell = 'pwsh'; + script = './Scripts/build_runtime.ps1'; + } + const build = spawn(`${shell}`, [`${script}`], { cwd: dir, stdio: ['pipe', 'pipe', 'pipe'], }); From 1090b932f2460ae4d3987ccdd47884819f52009d Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Thu, 19 Dec 2019 21:21:18 +0800 Subject: [PATCH 21/25] split dockerfile into two --- Composer/Dockerfile | 56 +------------------------------------------- Dockerfile | 57 +++++++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 6 ++++- 3 files changed, 63 insertions(+), 56 deletions(-) create mode 100644 Dockerfile diff --git a/Composer/Dockerfile b/Composer/Dockerfile index 67e923e64b..3b561c7daf 100644 --- a/Composer/Dockerfile +++ b/Composer/Dockerfile @@ -21,60 +21,6 @@ RUN yarn build:prod FROM node:12-alpine -RUN apk add --no-cache \ - ca-certificates \ - \ - # .NET Core dependencies - krb5-libs \ - libgcc \ - libintl \ - libssl1.1 \ - libstdc++ \ - zlib - -# Install .Net Core SDK -ENV \ - # Unset the value from the base image - ASPNETCORE_URLS= \ - # Disable the invariant mode (set in base image) - DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=false \ - # Enable correct mode for dotnet watch (only mode supported in a container) - DOTNET_USE_POLLING_FILE_WATCHER=true \ - LC_ALL=en_US.UTF-8 \ - LANG=en_US.UTF-8 \ - # Skip extraction of XML docs - generally not useful within an image/container - helps performance - NUGET_XMLDOC_MODE=skip \ - # PowerShell telemetry for docker image usage - POWERSHELL_DISTRIBUTION_CHANNEL=PSDocker-DotnetCoreSDK-Alpine-3.10 - -# Add dependencies for disabling invariant mode (set in base image) -RUN apk add --no-cache icu-libs - -# Install .NET Core 2.1 -ENV DOTNET_SDK_VERSION 2.1.607 - -RUN wget -O dotnet.tar.gz https://dotnetcli.azureedge.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-musl-x64.tar.gz \ - && dotnet_sha512='61caf6602b8a2aa89769b3e91ddaec963d8ab9f802cd7f6c6da4f02426358712bc2bb0930e7ee3a81d75c7607039543b554cb8ed50e45610655f9e91ed0f2f17' \ - && echo "$dotnet_sha512 dotnet.tar.gz" | sha512sum -c - \ - && mkdir -p /usr/share/dotnet \ - && tar -C /usr/share/dotnet -xzf dotnet.tar.gz \ - && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet \ - && rm dotnet.tar.gz - -# Install .NET Core SDK 3.0 -ENV DOTNET_SDK_VERSION 3.0.101 - -RUN wget -O dotnet.tar.gz https://dotnetcli.azureedge.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-musl-x64.tar.gz \ - && dotnet_sha512='98cc98f58187d208bd388f8c71862ea75e50ca25666e265f40a4e7c28082c2784738172e8ae4af7815057f7c57072cbe4fc03301d01738fc1ed5bb5e4d30a363' \ - && echo "$dotnet_sha512 dotnet.tar.gz" | sha512sum -c - \ - && tar -C /usr/share/dotnet -xzf dotnet.tar.gz \ - && rm dotnet.tar.gz - -# Enable detection of running in a container -ENV DOTNET_RUNNING_IN_CONTAINER=true \ - # Set the invariant mode since icu_libs isn't included (see https://github.com/dotnet/announcements/issues/20) - DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=true - WORKDIR /app/Composer COPY --from=build /src/Composer/yarn.lock . COPY --from=build /src/Composer/package.json . @@ -85,4 +31,4 @@ COPY --from=build /src/Composer/packages/tools ./packages/tools ENV NODE_ENV "production" RUN yarn --production --frozen-lockfile --force && yarn cache clean WORKDIR /app/Composer -CMD ["yarn", "start:server"] + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..0e4c38bfc9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,57 @@ +FROM composerbasic + +RUN apk add --no-cache \ + ca-certificates \ + \ + # .NET Core dependencies + krb5-libs \ + libgcc \ + libintl \ + libssl1.1 \ + libstdc++ \ + zlib + +# Install .Net Core SDK +ENV \ + # Unset the value from the base image + ASPNETCORE_URLS= \ + # Disable the invariant mode (set in base image) + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=false \ + # Enable correct mode for dotnet watch (only mode supported in a container) + DOTNET_USE_POLLING_FILE_WATCHER=true \ + LC_ALL=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + # Skip extraction of XML docs - generally not useful within an image/container - helps performance + NUGET_XMLDOC_MODE=skip \ + # PowerShell telemetry for docker image usage + POWERSHELL_DISTRIBUTION_CHANNEL=PSDocker-DotnetCoreSDK-Alpine-3.10 + +# Add dependencies for disabling invariant mode (set in base image) +RUN apk add --no-cache icu-libs + +# Install .NET Core 2.1 +ENV DOTNET_SDK_VERSION 2.1.607 + +RUN wget -O dotnet.tar.gz https://dotnetcli.azureedge.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-musl-x64.tar.gz \ + && dotnet_sha512='61caf6602b8a2aa89769b3e91ddaec963d8ab9f802cd7f6c6da4f02426358712bc2bb0930e7ee3a81d75c7607039543b554cb8ed50e45610655f9e91ed0f2f17' \ + && echo "$dotnet_sha512 dotnet.tar.gz" | sha512sum -c - \ + && mkdir -p /usr/share/dotnet \ + && tar -C /usr/share/dotnet -xzf dotnet.tar.gz \ + && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet \ + && rm dotnet.tar.gz + +# Install .NET Core SDK 3.0 +ENV DOTNET_SDK_VERSION 3.0.101 + +RUN wget -O dotnet.tar.gz https://dotnetcli.azureedge.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-musl-x64.tar.gz \ + && dotnet_sha512='98cc98f58187d208bd388f8c71862ea75e50ca25666e265f40a4e7c28082c2784738172e8ae4af7815057f7c57072cbe4fc03301d01738fc1ed5bb5e4d30a363' \ + && echo "$dotnet_sha512 dotnet.tar.gz" | sha512sum -c - \ + && tar -C /usr/share/dotnet -xzf dotnet.tar.gz \ + && rm dotnet.tar.gz + +# Enable detection of running in a container +ENV DOTNET_RUNNING_IN_CONTAINER=true \ + # Set the invariant mode since icu_libs isn't included (see https://github.com/dotnet/announcements/issues/20) + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=true + +CMD ["yarn","start:server"] diff --git a/docker-compose.yml b/docker-compose.yml index 1b6ef5fd72..93034c1e3e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,10 @@ version: "3" services: composer: build: Composer + image: composerbasic + container_name: composer + runtime: + build: . ports: - "3000:3000" - "3979:3979" @@ -13,5 +17,5 @@ services: COMPOSER_BOTS_FOLDER: /Bots COMPOSER_RUNTIME_FOLDER: /BotProject/Templates COMPOSER_APP_DATA: /appdata/data.json - BOT_ENDPOINT: http://0.0.0.0:3979 PORT: 3000 + BOT_ENDPOINT: http://0.0.0.0:3979 From a791621d91d3240ae3dbefad07f947678d82955b Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Fri, 20 Dec 2019 23:38:14 +0800 Subject: [PATCH 22/25] use nultistage instead of two dockerfile --- Composer/Dockerfile | 60 ++++++++++++++++++++++++++++++++++++++++++++- Dockerfile | 57 ------------------------------------------ docker-compose.yml | 4 --- 3 files changed, 59 insertions(+), 62 deletions(-) delete mode 100644 Dockerfile diff --git a/Composer/Dockerfile b/Composer/Dockerfile index 3b561c7daf..b8c9fdfda3 100644 --- a/Composer/Dockerfile +++ b/Composer/Dockerfile @@ -19,7 +19,7 @@ RUN yarn build:prod -FROM node:12-alpine +FROM node:12-alpine as composerbasic WORKDIR /app/Composer COPY --from=build /src/Composer/yarn.lock . @@ -32,3 +32,61 @@ ENV NODE_ENV "production" RUN yarn --production --frozen-lockfile --force && yarn cache clean WORKDIR /app/Composer + +FROM composerbasic + +RUN apk add --no-cache \ + ca-certificates \ + \ + # .NET Core dependencies + krb5-libs \ + libgcc \ + libintl \ + libssl1.1 \ + libstdc++ \ + zlib + +# Install .Net Core SDK +ENV \ + # Unset the value from the base image + ASPNETCORE_URLS= \ + # Disable the invariant mode (set in base image) + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=false \ + # Enable correct mode for dotnet watch (only mode supported in a container) + DOTNET_USE_POLLING_FILE_WATCHER=true \ + LC_ALL=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + # Skip extraction of XML docs - generally not useful within an image/container - helps performance + NUGET_XMLDOC_MODE=skip \ + # PowerShell telemetry for docker image usage + POWERSHELL_DISTRIBUTION_CHANNEL=PSDocker-DotnetCoreSDK-Alpine-3.10 + +# Add dependencies for disabling invariant mode (set in base image) +RUN apk add --no-cache icu-libs + +# Install .NET Core 2.1 +ENV DOTNET_SDK_VERSION 2.1.607 + +RUN wget -O dotnet.tar.gz https://dotnetcli.azureedge.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-musl-x64.tar.gz \ + && dotnet_sha512='61caf6602b8a2aa89769b3e91ddaec963d8ab9f802cd7f6c6da4f02426358712bc2bb0930e7ee3a81d75c7607039543b554cb8ed50e45610655f9e91ed0f2f17' \ + && echo "$dotnet_sha512 dotnet.tar.gz" | sha512sum -c - \ + && mkdir -p /usr/share/dotnet \ + && tar -C /usr/share/dotnet -xzf dotnet.tar.gz \ + && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet \ + && rm dotnet.tar.gz + +# Install .NET Core SDK 3.0 +ENV DOTNET_SDK_VERSION 3.0.101 + +RUN wget -O dotnet.tar.gz https://dotnetcli.azureedge.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-musl-x64.tar.gz \ + && dotnet_sha512='98cc98f58187d208bd388f8c71862ea75e50ca25666e265f40a4e7c28082c2784738172e8ae4af7815057f7c57072cbe4fc03301d01738fc1ed5bb5e4d30a363' \ + && echo "$dotnet_sha512 dotnet.tar.gz" | sha512sum -c - \ + && tar -C /usr/share/dotnet -xzf dotnet.tar.gz \ + && rm dotnet.tar.gz + +# Enable detection of running in a container +ENV DOTNET_RUNNING_IN_CONTAINER=true \ + # Set the invariant mode since icu_libs isn't included (see https://github.com/dotnet/announcements/issues/20) + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=true + +CMD ["yarn","start:server"] diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 0e4c38bfc9..0000000000 --- a/Dockerfile +++ /dev/null @@ -1,57 +0,0 @@ -FROM composerbasic - -RUN apk add --no-cache \ - ca-certificates \ - \ - # .NET Core dependencies - krb5-libs \ - libgcc \ - libintl \ - libssl1.1 \ - libstdc++ \ - zlib - -# Install .Net Core SDK -ENV \ - # Unset the value from the base image - ASPNETCORE_URLS= \ - # Disable the invariant mode (set in base image) - DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=false \ - # Enable correct mode for dotnet watch (only mode supported in a container) - DOTNET_USE_POLLING_FILE_WATCHER=true \ - LC_ALL=en_US.UTF-8 \ - LANG=en_US.UTF-8 \ - # Skip extraction of XML docs - generally not useful within an image/container - helps performance - NUGET_XMLDOC_MODE=skip \ - # PowerShell telemetry for docker image usage - POWERSHELL_DISTRIBUTION_CHANNEL=PSDocker-DotnetCoreSDK-Alpine-3.10 - -# Add dependencies for disabling invariant mode (set in base image) -RUN apk add --no-cache icu-libs - -# Install .NET Core 2.1 -ENV DOTNET_SDK_VERSION 2.1.607 - -RUN wget -O dotnet.tar.gz https://dotnetcli.azureedge.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-musl-x64.tar.gz \ - && dotnet_sha512='61caf6602b8a2aa89769b3e91ddaec963d8ab9f802cd7f6c6da4f02426358712bc2bb0930e7ee3a81d75c7607039543b554cb8ed50e45610655f9e91ed0f2f17' \ - && echo "$dotnet_sha512 dotnet.tar.gz" | sha512sum -c - \ - && mkdir -p /usr/share/dotnet \ - && tar -C /usr/share/dotnet -xzf dotnet.tar.gz \ - && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet \ - && rm dotnet.tar.gz - -# Install .NET Core SDK 3.0 -ENV DOTNET_SDK_VERSION 3.0.101 - -RUN wget -O dotnet.tar.gz https://dotnetcli.azureedge.net/dotnet/Sdk/$DOTNET_SDK_VERSION/dotnet-sdk-$DOTNET_SDK_VERSION-linux-musl-x64.tar.gz \ - && dotnet_sha512='98cc98f58187d208bd388f8c71862ea75e50ca25666e265f40a4e7c28082c2784738172e8ae4af7815057f7c57072cbe4fc03301d01738fc1ed5bb5e4d30a363' \ - && echo "$dotnet_sha512 dotnet.tar.gz" | sha512sum -c - \ - && tar -C /usr/share/dotnet -xzf dotnet.tar.gz \ - && rm dotnet.tar.gz - -# Enable detection of running in a container -ENV DOTNET_RUNNING_IN_CONTAINER=true \ - # Set the invariant mode since icu_libs isn't included (see https://github.com/dotnet/announcements/issues/20) - DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=true - -CMD ["yarn","start:server"] diff --git a/docker-compose.yml b/docker-compose.yml index 93034c1e3e..380431c1af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,10 +2,6 @@ version: "3" services: composer: build: Composer - image: composerbasic - container_name: composer - runtime: - build: . ports: - "3000:3000" - "3979:3979" From 803fe7a4e7f45a79f8997f430d30349a200dd4ac Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Tue, 7 Jan 2020 20:43:00 +0800 Subject: [PATCH 23/25] update yarn lock --- Composer/packages/server/package.json | 2 +- Composer/yarn.lock | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Composer/packages/server/package.json b/Composer/packages/server/package.json index 1b4fb77a10..01d4c84bcb 100644 --- a/Composer/packages/server/package.json +++ b/Composer/packages/server/package.json @@ -69,7 +69,7 @@ "express": "^4.16.4", "form-data": "^2.3.3", "format-message": "^6.2.1", - "get-port": "^5.0.0", + "get-port": "^5.1.0", "globby": "^9.1.0", "http-errors": "^1.7.2", "immer": "^2.1.4", diff --git a/Composer/yarn.lock b/Composer/yarn.lock index fa75a114bf..4dbe07aed3 100644 --- a/Composer/yarn.lock +++ b/Composer/yarn.lock @@ -8718,6 +8718,11 @@ get-own-enumerable-property-symbols@^3.0.0: resolved "https://registry.yarnpkg.com/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.0.tgz#b877b49a5c16aefac3655f2ed2ea5b684df8d203" integrity sha512-CIJYJC4GGF06TakLg8z4GQKvDsx9EMspVxOYih7LerEL/WosUnFIww45CGfxfeKHqlg3twgUrYRT1O3WQqjGCg== +get-port@^5.1.0: + version "5.1.0" + resolved "https://registry.npmjs.org/get-port/-/get-port-5.1.0.tgz#a8f6510d0000f07d5c65653a4b0ae648fe832683" + integrity sha512-bjioH1E9bTQUvgaB6VycVy1QVbTZI41yTnF9qkZz6ixgy/uhCH6D63bKeZ6Code/07JYA61MeI94jSdHss8PNA== + get-stdin@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-6.0.0.tgz#9e09bf712b360ab9225e812048f71fde9c89657b" From 6ce7143b01cad533236ca6f4d973288931a3003e Mon Sep 17 00:00:00 2001 From: Andy Brown Date: Wed, 8 Jan 2020 12:59:31 -0800 Subject: [PATCH 24/25] add more debug logs --- .../packages/server/src/models/bot/botProject.ts | 4 ++++ .../src/models/connector/csharpBotConnector.ts | 13 +++++++------ .../src/models/settings/defaultSettingManager.ts | 4 ++++ .../src/models/settings/fileSettingManager.ts | 4 ++++ 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/Composer/packages/server/src/models/bot/botProject.ts b/Composer/packages/server/src/models/bot/botProject.ts index f4cf30299c..876b234db5 100644 --- a/Composer/packages/server/src/models/bot/botProject.ts +++ b/Composer/packages/server/src/models/bot/botProject.ts @@ -20,12 +20,14 @@ import { copyDir } from '../../utility/storage'; import StorageService from '../../services/storage'; import { IEnvironment, EnvironmentProvider } from '../environment'; import { ISettingManager, OBFUSCATED_VALUE } from '../settings'; +import log from '../../logger'; import { IFileStorage } from './../storage/interface'; import { LocationRef, LuisStatus, FileUpdateType } from './interface'; import { LuPublisher } from './luPublisher'; import { DialogSetting } from './interface'; +const debug = log.extend('bot-project'); const DIALOGFOLDER = 'ComposerDialogs'; const oauthInput = () => ({ @@ -385,6 +387,7 @@ export class BotProject { private _createFile = async (relativePath: string, content: string) => { const absolutePath = Path.resolve(this.dir, relativePath); await this.ensureDirExists(Path.dirname(absolutePath)); + debug('Creating file: %s', absolutePath); await this.fileStorage.writeFile(absolutePath, content); // update this.files which is memory cache of all files @@ -457,6 +460,7 @@ export class BotProject { return; } if (!(await this.fileStorage.exists(dir))) { + debug('Creating directory: %s', dir); await this.fileStorage.mkDir(dir, { recursive: true }); } }; diff --git a/Composer/packages/server/src/models/connector/csharpBotConnector.ts b/Composer/packages/server/src/models/connector/csharpBotConnector.ts index a837d7885c..d6d825e605 100644 --- a/Composer/packages/server/src/models/connector/csharpBotConnector.ts +++ b/Composer/packages/server/src/models/connector/csharpBotConnector.ts @@ -16,7 +16,8 @@ import { IFileStorage } from '../storage/interface'; import { BotConfig, BotEnvironments, BotStatus, IBotConnector, IPublishHistory } from './interface'; -const buildDebug = log.extend('build bot runtime'); +const debug = log.extend('bot-runtime'); +const buildDebug = debug.extend('build'); const runtimeDebugs: { [key: string]: debug.Debugger } = {}; export class CSharpBotConnector implements IBotConnector { public status: BotStatus = BotStatus.NotConnected; @@ -30,7 +31,7 @@ export class CSharpBotConnector implements IBotConnector { for (const pid in CSharpBotConnector.botRuntimes) { const runtime = CSharpBotConnector.botRuntimes[pid]; runtime.kill(signal); - runtimeDebugs[pid]('successfully stopped bot runtime: %d', pid); + runtimeDebugs[pid]('successfully stopped bot runtime'); delete CSharpBotConnector.botRuntimes[pid]; } }; @@ -109,7 +110,7 @@ export class CSharpBotConnector implements IBotConnector { let erroutput = ''; child.stdout && child.stdout.on('data', (data: any) => { - currentDebugger('bot runtime (%d): %s', child.pid, data); + currentDebugger('%s', data); resolve(child.pid); }); @@ -128,7 +129,7 @@ export class CSharpBotConnector implements IBotConnector { }); child.on('message', msg => { - currentDebugger('bot runtime received: %s', msg); + currentDebugger('%s', msg); }); }; @@ -151,8 +152,8 @@ export class CSharpBotConnector implements IBotConnector { } ); // extend runtime debugger - runtimeDebugs[runtime.pid] = log.extend(`process ${runtime.pid}`); - runtimeDebugs[runtime.pid]('bot runtime started. pid: %d', runtime.pid); + runtimeDebugs[runtime.pid] = debug.extend(`process (${runtime.pid})`); + runtimeDebugs[runtime.pid]('bot runtime started'); CSharpBotConnector.botRuntimes[runtime.pid] = runtime; this.addListeners(runtime, this.stop, resolve, reject); }); diff --git a/Composer/packages/server/src/models/settings/defaultSettingManager.ts b/Composer/packages/server/src/models/settings/defaultSettingManager.ts index 3df934368e..a985f80b7d 100644 --- a/Composer/packages/server/src/models/settings/defaultSettingManager.ts +++ b/Composer/packages/server/src/models/settings/defaultSettingManager.ts @@ -5,9 +5,12 @@ import omit from 'lodash/omit'; import { SensitiveProperties } from '@bfc/shared'; import { Path } from '../../utility/path'; +import log from '../../logger'; import { FileSettingManager } from './fileSettingManager'; +const debug = log.extend('default-settings-manager'); + export class DefaultSettingManager extends FileSettingManager { constructor(basePath: string) { super(basePath); @@ -45,6 +48,7 @@ export class DefaultSettingManager extends FileSettingManager { const path = this.getPath(slot); const dir = Path.dirname(path); if (!(await this.storage.exists(dir))) { + debug('Storage path does not exist. Creating directory now: %s', dir); await this.storage.mkDir(dir, { recursive: true }); } // remove sensitive values before saving to disk diff --git a/Composer/packages/server/src/models/settings/fileSettingManager.ts b/Composer/packages/server/src/models/settings/fileSettingManager.ts index df9887d532..c4923f985c 100644 --- a/Composer/packages/server/src/models/settings/fileSettingManager.ts +++ b/Composer/packages/server/src/models/settings/fileSettingManager.ts @@ -3,9 +3,12 @@ import { Path } from '../../utility/path'; import { LocalDiskStorage } from '../storage/localDiskStorage'; +import log from '../../logger'; import { ISettingManager, OBFUSCATED_VALUE } from '.'; +const debug = log.extend('file-settings-manager'); + // TODO: this causes tests to fail const subPath = 'ComposerDialogs/settings/appsettings.json'; @@ -50,6 +53,7 @@ export class FileSettingManager implements ISettingManager { const dir = Path.dirname(path); if (!(await this.storage.exists(dir))) { + debug('Storage path does not exist. Creating directory now: %s', dir); await this.storage.mkDir(dir, { recursive: true }); } await this.storage.writeFile(path, JSON.stringify(settings, null, 2)); From 429886e26dc98d3b1de73ceba675bb80ee05e736 Mon Sep 17 00:00:00 2001 From: Wenyi Luo Date: Thu, 9 Jan 2020 17:17:42 +0800 Subject: [PATCH 25/25] fix settings file mis-created in server folder --- .../server/src/controllers/connector.ts | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Composer/packages/server/src/controllers/connector.ts b/Composer/packages/server/src/controllers/connector.ts index 28e438d969..14ea9cc88c 100644 --- a/Composer/packages/server/src/controllers/connector.ts +++ b/Composer/packages/server/src/controllers/connector.ts @@ -3,14 +3,14 @@ import merge from 'lodash/merge'; import { BotEnvironments } from '../models/connector'; -import { EnvironmentProvider } from '../models/environment'; +import { BotProjectService } from '../services/project'; async function connect(req: any, res: any) { try { const hostName = req.hostname; const env: BotEnvironments = req.query && req.query.botEnvironment ? req.query.botEnvironment : 'production'; - const environment = EnvironmentProvider.getCurrent(); - const botEndpoint = await environment.getBotConnector().connect(env || 'production', hostName); + const environment = BotProjectService.getCurrentBotProject()?.environment; + const botEndpoint = await environment?.getBotConnector().connect(env || 'production', hostName); res.send({ botEndpoint }); } catch (error) { res.status(400).json({ @@ -21,8 +21,8 @@ async function connect(req: any, res: any) { async function getPublishHistory(req: any, res: any) { try { - const environment = EnvironmentProvider.getCurrent(); - const history = await environment.getBotConnector().getPublishHistory(); + const environment = BotProjectService.getCurrentBotProject()?.environment; + const history = await environment?.getBotConnector().getPublishHistory(); res.send(history); } catch (error) { res.status(400).json({ @@ -33,9 +33,9 @@ async function getPublishHistory(req: any, res: any) { async function sync(req: any, res: any) { try { - const environment = EnvironmentProvider.getCurrent(); - const settingsInDisk = await environment.getSettingsManager().get(''); - await environment.getBotConnector().sync(merge({}, settingsInDisk, req.body, { user: req.user })); + const environment = BotProjectService.getCurrentBotProject()?.environment; + const settingsInDisk = await environment?.getSettingsManager().get(); + await environment?.getBotConnector().sync(merge({}, settingsInDisk, req.body, { user: req.user })); res.send('OK'); } catch (error) { res.status(400).json({ @@ -47,8 +47,8 @@ async function sync(req: any, res: any) { async function publish(req: any, res: any) { try { const label = req.params ? req.params.label : undefined; - const environment = EnvironmentProvider.getCurrent(); - await environment.getBotConnector().publish({ ...req.body, user: req.user }, label); + const environment = BotProjectService.getCurrentBotProject()?.environment; + await environment?.getBotConnector().publish({ ...req.body, user: req.user }, label); res.send('OK'); } catch (error) { res.status(400).json({ @@ -58,8 +58,8 @@ async function publish(req: any, res: any) { } function status(req: any, res: any) { - const environment = EnvironmentProvider.getCurrent(); - res.send(environment.getBotConnector().status); + const environment = BotProjectService.getCurrentBotProject()?.environment; + res.send(environment?.getBotConnector().status); } export const BotConnectorController = {