feat: Runtime refactor, new directory structure and Azure Function in…

…troduction phase 1 (#2855)

* Runtime: new folder structure, refactor common c# code into core, create function runtime

* Remove deprecated Bot Project

* Runtime: Fix tests

* Update runtime code owners

* Runtime: Part 1 of updating composer server to honor new runtime paths

* Runtime: tweaks for local publish to run prior to merge

* Runtime: fixes post merge

* Update azure publish to use the azure web app template.

* Azure publish: update bot project deploy to new directory structure

* Fix codeowners alias mistake

* Revert "Merge branch 'master' of https://github.com/microsoft/BotFramework-Composer into wenyluo/azure"

This reverts commit 76251ec, reversing
changes made to 7d83253.

* WebApp + new runtime: deployment and local runtime all working

* Runtime: move nuget.config to dotnet runtime root

* Asset manager: add mock folder to reflect new runtime structure

* Local publish: remove unnecessary space

* Runtime: Add copyright header to all missing files

* Fix bad merge

* Runtime: rename js -> node and update readmes

.github/CODEOWNERS

@@ -2,7 +2,7 @@
 
 * @cwhitten @boydc2014 @a-b-r-o-w-n
 
-/BotProject/ @boydc2014 @luhan2017
+/runtime/ @boydc2014 @luhan2017 @carlosscastro @benbrown
 
 /Composer/ @cwhitten @boydc2014 @a-b-r-o-w-n @corinagum @beyackle @srinaath @tonyanziano
 

.github/workflows/main.yml

@@ -63,10 +63,10 @@ jobs:
           dotnet-version: "3.1.102" # SDK Version to use.
       - name: dotnet build
         run: dotnet build
-        working-directory: BotProject/Templates/CSharp
+        working-directory: runtime/dotnet
       - name: dotnet test
         run: dotnet test
-        working-directory: BotProject/Templates/CSharp
+        working-directory: runtime/dotnet/tests
 
   docker-build:
     name: Docker Build

Composer/package.json

@@ -40,7 +40,7 @@
     "start:client": "yarn workspace @bfc/client start",
     "start:server": "yarn workspace @bfc/server start",
     "start:server:dev": "yarn workspace @bfc/server start:dev",
-    "runtime": "cd ../BotProject/CSharp/ && dotnet build && dotnet run",
+    "runtime": "cd ../runtime/dotnet/azurewebapp && dotnet build && dotnet run",
     "test": "yarn typecheck && jest",
     "test:watch": "yarn typecheck && jest --watch",
     "test:coverage": "yarn test --coverage --no-cache --reporters=default",

Composer/packages/electron-server/scripts/copy-templates.js

@@ -4,7 +4,7 @@
 const fs = require('fs-extra');
 const { resolve } = require('path');
 
-const source = resolve(__dirname, '../../../../BotProject/Templates');
+const source = resolve(__dirname, '../../../../runtime');
 const destination = resolve(__dirname, '../build/templates');
 console.log(`[copy-templates.js] Copying templates from ${source} to ${destination}`);
 

Composer/packages/extensions/adaptive-form/src/utils/resolveRef.ts

@@ -8,7 +8,7 @@ export function resolveRef(
 ): JSONSchema7 {
   if (typeof schema?.$ref === 'string') {
     const defName = schema.$ref.replace('#/definitions/', '');
-    const defSchema = typeof definitions[defName] === 'object' ? (definitions[defName] as JSONSchema7) : {};
+    const defSchema = typeof definitions?.[defName] === 'object' ? (definitions?.[defName] as JSONSchema7) : {};
 
     const resolvedSchema = {
       ...defSchema,

Composer/packages/extensions/plugin-loader/src/composerPluginRegistration.ts

@@ -78,9 +78,9 @@ export class ComposerPluginRegistration {
    * to communicate with the Bot Framework Emulator.
    * ```ts
    * await composer.addRuntimeTemplate({
-   *   key: 'csharp',
+   *   key: 'azurewebapp',
    *   name: 'C#',
-   *   path: __dirname + '/../../../../BotProject/Templates/CSharp',
+   *   path: __dirname + '/../../../../runtime/dotnet/azurewebapp',
    *   startCommand: 'dotnet run',
    * });
    * ```

Composer/packages/lib/bot-deploy/README.md

@@ -35,7 +35,7 @@
      // The ARM template file path, default is 'DeploymentTemplates/template-with-preexisting-rg.json'
      templatePath?: string
 
-     // Dotnet project path, default is 'BotProject.csproj'
+     // Dotnet project path, default is 'Microsoft.BotFramework.Composer.WebApp.csproj'
      dotnetProjectPath?: string
 
      // Lubuild generated folder path, default is 'generated'

Composer/packages/lib/bot-deploy/src/botProjectDeploy.ts

@@ -74,7 +74,8 @@ export class BotProjectDeploy {
       config.templatePath ?? path.join(this.projPath, 'DeploymentTemplates', 'template-with-preexisting-rg.json');
 
     // path to the dotnet project file
-    this.dotnetProjectPath = config.dotnetProjectPath ?? path.join(this.projPath, 'BotProject.csproj');
+    this.dotnetProjectPath =
+      config.dotnetProjectPath ?? path.join(this.projPath, 'Microsoft.BotFramework.Composer.WebApp.csproj');
 
     // path to the built, ready to deploy declarative assets
     this.remoteBotPath = config.remoteBotPath ?? path.join(this.publishFolder, 'ComposerDialogs');
@@ -270,7 +271,7 @@ export class BotProjectDeploy {
 
   private async botPrepareDeploy(pathToDeploymentFile: string) {
     return new Promise((resolve, reject) => {
-      const data = `[config]\nproject = BotProject.csproj`;
+      const data = `[config]\nproject = Microsoft.BotFramework.Composer.WebApp.csproj`;
       fs.writeFile(pathToDeploymentFile, data, err => {
         if (err) {
           reject(err);

Composer/packages/lib/bot-deploy/src/botProjectDeployConfig.ts

@@ -35,7 +35,7 @@ export interface BotProjectDeployConfig {
   // The ARM template file path, default is 'DeploymentTemplates/template-with-preexisting-rg.json'
   templatePath?: string;
 
-  // Dotnet project path, default is 'BotProject.csproj'
+  // Dotnet project path, default is 'Microsoft.BotFramework.Composer.WebApp.csproj'
   dotnetProjectPath?: string;
 
   // Lubuild generated folder path, default is 'generated'

Composer/packages/lib/shared/src/types/schema.ts

@@ -6,7 +6,7 @@ import { JSONSchema6 } from 'json-schema';
 // All of the known SDK types. Update this list when we take a schema update.
 // To get this list copy the output of the following commands in a node repl from the project root:
 
-// const schema = JSON.parse(fs.readFileSync('./BotProject/CSharp/Schemas/sdk.schema', 'utf-8'));
+// const schema = JSON.parse(fs.readFileSync('./runtime/dotnet/azurewebapp/Schemas/sdk.schema', 'utf-8'));
 // const types = schema.oneOf.map(t => t.title);
 // let uType = 'export enum SDKKinds {\n';
 // uType += types.map(t => `  ${t.replace('Microsoft.', '')} = '${t}',`).join('\n');

Composer/packages/server/__tests__/mocks/runtimes/dotnet/readme.md

@@ -0,0 +1 @@
+this is a sample (empty) runtime folder for tests.

Composer/packages/server/src/controllers/publisher.ts

@@ -14,7 +14,7 @@ const defaultPublishConfig = {
   type: 'localpublish',
   configuration: JSON.stringify({}),
 };
-const DEFAULT_RUNTIME = 'CSharp';
+const DEFAULT_RUNTIME = 'dotnet';
 export const PublishController = {
   getTypes: async (req, res) => {
     res.json(

Composer/packages/server/src/models/asset/assetManager.ts

@@ -22,6 +22,14 @@ interface TemplateData {
   };
 }
 
+interface ProjectTemplateCollection {
+  [key: string]: {
+    name: string;
+    description: string;
+    path: string;
+  };
+}
+
 const templates: TemplateData = {
   EchoBot: {
     name: 'Echo Bot',
@@ -78,16 +86,17 @@ const templates: TemplateData = {
   },
 };
 
-const runtimes: TemplateData = {
-  CSharp: {
-    name: 'CSharp Runtime',
-    description: 'A Bot Framework runtime using the CSharp/dotnet version of the SDK',
+const runtimes: ProjectTemplateCollection = {
+  dotnet: {
+    name: 'dotnet webapp runtime',
+    description: 'A Bot Framework runtime using the dotnet version of the SDK, hosted in azure web app',
+    path: 'dotnet',
   },
 };
 
 // set a default runtime template.
 // when we have multiple runtimes this will be a parameter.
-const DEFAULT_RUNTIME = 'CSharp';
+const DEFAULT_RUNTIME = 'dotnet';
 
 export class AssetManager {
   public templateStorage: LocalDiskStorage;
@@ -101,7 +110,7 @@ export class AssetManager {
     this.runtimesPath = runtimesPath;
     this.templateStorage = new LocalDiskStorage();
 
-    // initialize the list of project tempaltes
+    // initialize the list of project templates
     this.getProjectTemplates();
 
     // initialize the list of runtimes.
@@ -145,12 +154,15 @@ export class AssetManager {
     const output: ProjectTemplate[] = [];
 
     if (await this.templateStorage.exists(path)) {
-      const folders = await this.templateStorage.readDir(path);
       this.runtimeTemplates = [];
-      for (const name of folders) {
-        const absPath = Path.join(path, name);
+      for (const runtimeKey in runtimes) {
+        const absPath = Path.join(path, runtimes[runtimeKey].path);
         if ((await this.templateStorage.stat(absPath)).isDir) {
-          const base = { id: name, name: runtimes[name].name, description: runtimes[name].description };
+          const base = {
+            id: runtimeKey,
+            name: runtimes[runtimeKey].name,
+            description: runtimes[runtimeKey].description,
+          };
           this.runtimeTemplates.push({ ...base, path: absPath });
           output.push(base);
         }

Composer/packages/server/src/settings/env.ts

@@ -37,6 +37,5 @@ export const environment = process.env.NODE_ENV || 'development';
 export const botsFolder = folder;
 export const botEndpoint = process.env.BOT_ENDPOINT || 'http://localhost:3979';
 export const appDataPath = process.env.COMPOSER_APP_DATA || Path.resolve(__dirname, '../../data.json');
-export const runtimeFolder =
-  process.env.COMPOSER_RUNTIME_FOLDER || Path.resolve(__dirname, '../../../../../BotProject/Templates');
+export const runtimeFolder = process.env.COMPOSER_RUNTIME_FOLDER || Path.resolve(__dirname, '../../../../../runtime');
 export const runtimeFrameworkVersion = process.env.COMPOSER_RUNTIME_VERSION || 'netcoreapp3.1';

Composer/plugins/README.md

@@ -251,9 +251,9 @@ to communicate with the Bot Framework Emulator.
 
 ```ts
 await composer.addRuntimeTemplate({
-  key: 'csharp',
+  key: 'azurewebapp',
   name: 'C#',
-  path: __dirname + '/../../../../BotProject/Templates/CSharp',
+  path: __dirname + '/../../../../runtime/dotnet/azurewebapp',
   startCommand: 'dotnet run',
 });
 ```

Composer/plugins/azurePublish/src/index.ts

@@ -14,6 +14,7 @@ import schema from './schema';
 // set to TRUE for history to be saved to disk
 // set to FALSE for history to be cached in memory only
 const PERSIST_HISTORY = false;
+const DEFAULT_RUNTIME = 'azurewebapp';
 
 interface CreateAndDeployResources {
   publishName: string;
@@ -45,20 +46,23 @@ class AzurePublisher {
     this.publishingBots = {};
     this.logMessages = [];
   }
-  private getProjectFolder = (key: string) => path.resolve(__dirname, `../publishBots/${key}`);
-  private getBotFolder = (key: string) => path.resolve(this.getProjectFolder(key), 'ComposerDialogs');
-  private getSettingsPath = (key: string) => path.resolve(this.getBotFolder(key), 'settings/appsettings.json');
+
+  private getRuntimeFolder = (key: string) => path.resolve(__dirname, `../publishBots/${key}`);
+  private getProjectFolder = (key: string, template: string) => path.resolve(__dirname, `../publishBots/${key}/${template}`);
+  private getBotFolder = (key: string, template: string) => path.resolve(this.getProjectFolder(key, template), 'ComposerDialogs');
+  private getSettingsPath = (key: string, template: string) => path.resolve(this.getBotFolder(key, template), 'settings/appsettings.json');
 
   private init = async (botFiles: any, settings: any, srcTemplate: string, resourcekey: string) => {
-    const projExist = await pathExists(this.getProjectFolder(resourcekey));
-    const botExist = await pathExists(this.getBotFolder(resourcekey));
-    const botFolder = this.getBotFolder(resourcekey);
-    const projFolder = this.getProjectFolder(resourcekey);
-    const settingsPath = this.getSettingsPath(resourcekey);
+    const runtimeExist = await pathExists(this.getRuntimeFolder(resourcekey));
+    const botExist = await pathExists(this.getBotFolder(resourcekey, DEFAULT_RUNTIME));
+    const botFolder = this.getBotFolder(resourcekey, DEFAULT_RUNTIME);
+    const runtimeFolder = this.getRuntimeFolder(resourcekey);
+    const settingsPath = this.getSettingsPath(resourcekey, DEFAULT_RUNTIME);
+
     // deploy resource exist
-    await emptyDir(projFolder);
-    if (!projExist) {
-      mkdirSync(projFolder, { recursive: true });
+    await emptyDir(runtimeFolder);
+    if (!runtimeExist) {
+      mkdirSync(runtimeFolder, { recursive: true });
     }
     if (!botExist) {
       mkdirSync(botFolder, { recursive: true });
@@ -78,11 +82,11 @@ class AzurePublisher {
     }
     await writeJson(settingsPath, settings, { spaces: 4 });
     // copy bot and runtime into projFolder
-    await copy(srcTemplate, projFolder);
+    await copy(srcTemplate, runtimeFolder);
   };
 
   private async cleanup(resourcekey: string) {
-    const projFolder = this.getProjectFolder(resourcekey);
+    const projFolder = this.getRuntimeFolder(resourcekey);
     await emptyDir(projFolder);
     await rmdir(projFolder);
   }
@@ -187,7 +191,7 @@ class AzurePublisher {
    * plugin methods
    *************************************************************************************************/
   publish = async (config: PublishConfig, project, metadata, user) => {
-    // templatePath point to the CSharp code
+    // templatePath point to the dotnet code
     const {
       settings,
       templatePath,
@@ -255,7 +259,9 @@ class AzurePublisher {
       };
 
       // append provision resource into file
-      const resourcePath = path.resolve(this.getProjectFolder(resourcekey), 'appsettings.deployment.json');
+      // TODO: here is where we configure the template for the runtime, and should be parameterized when we 
+      // implement interchangeable runtimes
+      const resourcePath = path.resolve(this.getProjectFolder(resourcekey, DEFAULT_RUNTIME), 'appsettings.deployment.json');
       const appSettings = await readJson(resourcePath);
       await writeJson(
         resourcePath,
@@ -272,7 +278,7 @@ class AzurePublisher {
           this.logMessages.push(JSON.stringify(msg, null, 2));
         },
         accessToken: accessToken,
-        projPath: this.getProjectFolder(resourcekey),
+        projPath: this.getProjectFolder(resourcekey, 'azurewebapp'),
       });
 
       this.logMessages = ['Publish starting...'];