feature: Supporting core+package templates stored on NuGet (#4588)

* - added hosted template to template list
- updated template type to account for hosted templates

* initial scaffolding of hosted template grab

* external nuget template working e2e as template in creation flow

* Removing redundent types

* adding support for relative runtime paths.
Removing unused dependencies.

* adding comments  and removing autogenerated yan lock file from root dir

* Adding version to template acquasition
Generating build files
Fixing broken unit test

* updating template version to latest stable

* hiding remote template feature flag until NuGet packages are public

* Making PR changes

* simplifying featureflag logic
changing feature flag default values

* removing bad null check

* update naming

* Fixing broken test
updating locale files with latest added strings

* Removing 'del' dependency in favor of rimraf for directory deletions

* Reverting feature flag bug fix in favor for bug fix implemented in seperate PR

Co-authored-by: Patrick Volum <pavolum@microsoft.com>
Co-authored-by: Chris Whitten <christopher.whitten@microsoft.com>

Composer/packages/client/__tests__/components/CreationFlow/index.test.tsx

@@ -5,9 +5,15 @@ import * as React from 'react';
 import { render, fireEvent, act } from '@botframework-composer/test-utils';
 import { createHistory, createMemorySource, LocationProvider } from '@reach/router';
 import { RecoilRoot } from 'recoil';
+import { getDefaultFeatureFlags } from '@bfc/shared';
 
 import CreationFlow from '../../../src/components/CreationFlow/CreationFlow';
-import { focusedStorageFolderState, creationFlowStatusState, dispatcherState } from '../../../src/recoilModel';
+import {
+  focusedStorageFolderState,
+  creationFlowStatusState,
+  dispatcherState,
+  featureFlagsState,
+} from '../../../src/recoilModel';
 import { CreationFlowStatus } from '../../../src/constants';
 
 describe('<CreationFlow/>', () => {
@@ -26,7 +32,7 @@ describe('<CreationFlow/>', () => {
       saveTemplateId: jest.fn(),
     });
     set(creationFlowStatusState, CreationFlowStatus.NEW_FROM_TEMPLATE);
-
+    set(featureFlagsState, getDefaultFeatureFlags());
     set(focusedStorageFolderState, {
       name: 'Desktop',
       parent: '/test-folder',

Composer/packages/client/__tests__/components/home.test.tsx

@@ -3,7 +3,7 @@
 
 import * as React from 'react';
 import { fireEvent, render } from '@botframework-composer/test-utils';
-import { ProjectTemplate } from '@bfc/shared';
+import { BotTemplate } from '@bfc/shared';
 
 import { RecentBotList } from '../../src/pages/home/RecentBotList';
 import { ExampleList } from '../../src/pages/home/ExampleList';
@@ -32,7 +32,7 @@ describe('<Home/>', () => {
     const templates = [
       { description: 'echo bot', id: 'EchoBot', name: 'Echo Bot' },
       { description: 'empty bot', id: 'EmptyBot', name: 'Empty Bot' },
-    ] as ProjectTemplate[];
+    ] as BotTemplate[];
     const onClickTemplate = jest.fn((item) => item);
     const { container, getByText } = render(<ExampleList examples={templates} onClick={onClickTemplate} />);
     expect(container).toHaveTextContent('Echo Bot');

Composer/packages/client/src/components/CreationFlow/CreateOptions.tsx

@@ -20,7 +20,7 @@ import {
   DetailsRow,
 } from 'office-ui-fabric-react/lib/DetailsList';
 import { Sticky, StickyPositionType } from 'office-ui-fabric-react/lib/Sticky';
-import { ProjectTemplate } from '@bfc/shared';
+import { BotTemplate } from '@bfc/shared';
 import { DialogWrapper, DialogTypes } from '@bfc/ui-shared';
 import { NeutralColors } from '@uifabric/fluent-theme';
 import { RouteComponentProps } from '@reach/router';
@@ -105,7 +105,7 @@ const optionKeys = {
 
 // -------------------- CreateOptions -------------------- //
 type CreateOptionsProps = {
-  templates: ProjectTemplate[];
+  templates: BotTemplate[];
   onDismiss: () => void;
   onNext: (data: string) => void;
 } & RouteComponentProps<{}>;
@@ -119,7 +119,7 @@ export function CreateOptions(props: CreateOptionsProps) {
   const selection = useMemo(() => {
     return new Selection({
       onSelectionChanged: () => {
-        const t = selection.getSelection()[0] as ProjectTemplate;
+        const t = selection.getSelection()[0] as BotTemplate;
         if (t) {
           setCurrentTemplate(t.id);
         }

Composer/packages/client/src/pages/home/ExampleList.tsx

@@ -6,7 +6,7 @@ import { jsx } from '@emotion/core';
 import React from 'react';
 import { ScrollablePane, ScrollbarVisibility } from 'office-ui-fabric-react/lib/ScrollablePane';
 import { List } from 'office-ui-fabric-react/lib/List';
-import { ProjectTemplate } from '@bfc/shared';
+import { BotTemplate } from '@bfc/shared';
 
 import * as exampleIcons from '../../images/samples';
 
@@ -20,7 +20,7 @@ import {
 } from './styles';
 
 interface ExampleListProps {
-  examples: ProjectTemplate[];
+  examples: BotTemplate[];
   onClick: (templateId: string) => void;
 }
 
@@ -35,7 +35,7 @@ const resolveIcon = (exampleId: string): string => {
 export const ExampleList: React.FC<ExampleListProps> = (props) => {
   const { onClick, examples } = props;
 
-  function onRenderCell(item?: ProjectTemplate): React.ReactNode {
+  function onRenderCell(item?: BotTemplate): React.ReactNode {
     if (!item) {
       return;
     }

Composer/packages/client/src/recoilModel/atoms/appState.ts

@@ -2,7 +2,7 @@
 // Licensed under the MIT License.
 
 import { atom, atomFamily } from 'recoil';
-import { FormDialogSchemaTemplate, FeatureFlagMap, ProjectTemplate, UserSettings } from '@bfc/shared';
+import { FormDialogSchemaTemplate, FeatureFlagMap, BotTemplate, UserSettings } from '@bfc/shared';
 import { ExtensionMetadata } from '@bfc/extension-client';
 
 import {
@@ -52,7 +52,7 @@ export const recentProjectsState = atom<any[]>({
   default: [],
 });
 
-export const templateProjectsState = atom<ProjectTemplate[]>({
+export const templateProjectsState = atom<BotTemplate[]>({
   key: getFullyQualifiedKey('templateProjects'),
   default: [],
 });

Composer/packages/client/src/recoilModel/selectors/projectTemplates.ts

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+import { BotTemplate } from '@bfc/shared';
 import { selector } from 'recoil';
 
 import { featureFlagsState, templateProjectsState } from '../atoms/appState';
@@ -11,13 +12,20 @@ export const filteredTemplatesSelector = selector({
     const templates = get(templateProjectsState);
     const featureFlags = get(featureFlagsState);
 
-    const filteredTemplates = [...templates];
+    let filteredTemplates = [...templates];
     if (!featureFlags?.VA_CREATION?.enabled) {
       const vaTemplateIndex = filteredTemplates.findIndex((template) => template.id === 'va-core');
       if (vaTemplateIndex !== -1) {
         filteredTemplates.splice(vaTemplateIndex, 1);
       }
     }
+    if (!featureFlags.REMOTE_TEMPLATE_CREATION_EXPERIENCE.enabled) {
+      filteredTemplates = filteredTemplates.filter((template: BotTemplate) => {
+        if (template.path) {
+          return template;
+        }
+      });
+    }
     return filteredTemplates;
   },
 });

Composer/packages/lib/shared/src/featureFlagUtils/index.ts

@@ -14,15 +14,15 @@ export type FeatureFlag = {
   enabled: boolean;
 };
 
-export type FeatureFlagKey = 'VA_CREATION' | 'FORM_DIALOG';
+export type FeatureFlagKey = 'VA_CREATION' | 'FORM_DIALOG' | 'REMOTE_TEMPLATE_CREATION_EXPERIENCE';
 
 export type FeatureFlagMap = Record<FeatureFlagKey, FeatureFlag>;
 
 export const getDefaultFeatureFlags = (): FeatureFlagMap => ({
   VA_CREATION: {
     displayName: formatMessage('VA Creation'),
     description: formatMessage('VA template made available in new bot flow.'),
-    isHidden: false,
+    isHidden: true,
     enabled: false,
   },
   FORM_DIALOG: {
@@ -31,4 +31,12 @@ export const getDefaultFeatureFlags = (): FeatureFlagMap => ({
     isHidden: true,
     enabled: false,
   },
+  REMOTE_TEMPLATE_CREATION_EXPERIENCE: {
+    displayName: formatMessage('Remote templates'),
+    description: formatMessage(
+      'If turned on then externally stored templates will be selectable in the new bot flow template list'
+    ),
+    isHidden: true,
+    enabled: false,
+  },
 });

Composer/packages/server/package.json

@@ -62,6 +62,7 @@
     "@bfc/lg-languageserver": "*",
     "@bfc/lu-languageserver": "*",
     "@bfc/shared": "*",
+    "@microsoft/bf-dialog": "4.11.0-dev.20201025.69cf2b9",
     "@microsoft/bf-dispatcher": "^4.11.0-beta.20201016.393c6b2",
     "@microsoft/bf-generate-library": "^4.10.0-daily.20201026.178799",
     "@microsoft/bf-lu": "^4.11.0-rc.20201030.a9f9b96",

Composer/packages/server/src/controllers/__tests__/eject.test.ts

@@ -51,6 +51,7 @@ beforeAll(async () => {
     name: 'C#',
     startCommand: 'dotnet run --project azurewebapp',
     path: './',
+    identifyManifest: jest.fn(),
     eject: jest.fn(),
     build: jest.fn(),
     run: jest.fn(),

Composer/packages/server/src/controllers/project.ts

@@ -6,6 +6,7 @@ import * as fs from 'fs';
 import { Request, Response } from 'express';
 import { Archiver } from 'archiver';
 import { ExtensionContext } from '@bfc/extension';
+import { SchemaMerger } from '@microsoft/bf-dialog/lib/library/schemaMerger';
 
 import log from '../logger';
 import { BotProjectService } from '../services/project';
@@ -54,6 +55,33 @@ async function createProject(req: Request, res: Response) {
     await AssetService.manager.copyBoilerplate(currentProject.dataDir, currentProject.fileStorage);
 
     if (currentProject !== undefined) {
+      if (currentProject.settings?.runtime?.customRuntime === true) {
+        const runtime = ExtensionContext.getRuntimeByProject(currentProject);
+        const runtimePath = currentProject.settings.runtime.path;
+
+        if (!fs.existsSync(runtimePath)) {
+          await runtime.eject(currentProject, currentProject.fileStorage);
+        }
+
+        // install all dependencies and build the app
+        await runtime.build(runtimePath, currentProject);
+
+        const manifestFile = runtime.identifyManifest(runtimePath);
+
+        // run the merge command to merge all package dependencies from the template to the bot project
+        const realMerge = new SchemaMerger(
+          [manifestFile],
+          Path.join(currentProject.dataDir, 'schemas/sdk'),
+          Path.join(currentProject.dataDir, 'dialogs/imported'),
+          false,
+          false,
+          console.log,
+          console.warn,
+          console.error
+        );
+
+        await realMerge.merge();
+      }
       await currentProject.updateBotInfo(name, description, preserveRoot);
       if (schemaUrl) {
         await currentProject.saveSchemaToProject(schemaUrl, locationRef.path);
@@ -367,7 +395,7 @@ async function checkBoilerplateVersion(req: Request, res: Response) {
 
   const currentProject = await BotProjectService.getProjectById(projectId, user);
   if (currentProject !== undefined) {
-    const latestVersion = AssetService.manager.getBoilerplateCurrentVersion();
+    const latestVersion = await AssetService.manager.getBoilerplateCurrentVersion();
     const currentVersion = await AssetService.manager.getBoilerplateVersionFromProject(currentProject);
     const updateRequired =
       (latestVersion && currentVersion && latestVersion > currentVersion) || // versions are present in both locations, latest is newer

Composer/packages/server/src/locales/en-US.json

@@ -41,6 +41,9 @@
   "a_valid_url_should_start_with_http_or_https_327b1a30": {
     "message": "A valid URL should start with http:// or https://"
   },
+  "a_valid_url_should_start_with_http_or_https_d24b3591": {
+    "message": "A valid url should start with http:// or https://"
+  },
   "about_70c18bba": {
     "message": "About"
   },
@@ -128,6 +131,9 @@
   "add_a_welcome_message_9e1480b2": {
     "message": "Add a welcome message"
   },
+  "add_additional_url_bdfac25d": {
+    "message": "Add additional URL"
+  },
   "add_alternative_phrasing_17e0304c": {
     "message": "+ Add alternative phrasing"
   },
@@ -227,6 +233,9 @@
   "any_string_f22dc2e1": {
     "message": "any string"
   },
+  "append_choices_35c45a2d": {
+    "message": "Append choices"
+  },
   "application_language_87691b6": {
     "message": "Application Language"
   },
@@ -611,6 +620,9 @@
   "could_not_connect_to_storage_50411de0": {
     "message": "Could not connect to storage."
   },
+  "could_not_init_plugin_1f1c29cd": {
+    "message": "Could not init plugin"
+  },
   "couldn_t_complete_the_update_a337a359": {
     "message": "Couldn''t complete the update:"
   },
@@ -683,6 +695,9 @@
   "create_kb_from_url_or_file_49ad6671": {
     "message": "Create KB from URL or file"
   },
+  "create_knowledge_base_db6d66c4": {
+    "message": "Create knowledge base"
+  },
   "create_knowledge_base_from_scratch_afe4d2a2": {
     "message": "Create knowledge base from scratch"
   },
@@ -1268,6 +1283,9 @@
   "if_this_problem_persists_please_file_an_issue_on_6fbc8e2b": {
     "message": "If this problem persists, please file an issue on"
   },
+  "if_turned_on_then_externally_stored_templates_will_3f487651": {
+    "message": "If turned on then externally stored templates will be selectable in the new bot flow template list"
+  },
   "if_you_already_have_a_luis_account_provide_the_inf_bede07a4": {
     "message": "If you already have a LUIS account, provide the information below. If you do not have an account yet, create a (free) account first."
   },
@@ -1403,6 +1421,9 @@
   "learn_more_about_knowledge_base_sources_24369b09": {
     "message": "Learn more about knowledge base sources. "
   },
+  "learn_more_about_qna_maker_skus_998c567": {
+    "message": "Learn more about QnA Maker SKUs."
+  },
   "learn_more_about_title_d1d3edbe": {
     "message": "Learn more about { title }"
   },
@@ -1799,6 +1820,9 @@
   "open_notification_panel_5796edb3": {
     "message": "Open notification panel"
   },
+  "open_skills_page_for_configuration_details_a2a484ea": {
+    "message": "Open Skills page for configuration details"
+  },
   "optional_221bcc9d": {
     "message": "Optional"
   },
@@ -1889,6 +1913,9 @@
   "please_select_an_activity_type_92f4a8a1": {
     "message": "Please select an activity type"
   },
+  "populate_your_knowledge_base_bb2d3605": {
+    "message": "Populate your Knowledge Base"
+  },
   "press_enter_to_add_this_item_or_tab_to_move_to_the_6beb8a14": {
     "message": "press Enter to add this item or Tab to move to the next interactive element"
   },
@@ -2069,6 +2096,9 @@
   "regex_intent_is_already_defined_df095c1f": {
     "message": "RegEx { intent } is already defined"
   },
+  "regular_expression_855557bf": {
+    "message": "Regular Expression"
+  },
   "regular_expression_recognizer_44664557": {
     "message": "Regular expression recognizer"
   },
@@ -2078,6 +2108,9 @@
   "reloading_49d2f661": {
     "message": "Reloading"
   },
+  "remote_templates_a23c242d": {
+    "message": "Remote templates"
+  },
   "remove_f47dc62a": {
     "message": "Remove"
   },
@@ -2315,6 +2348,9 @@
   "skills_49cccd6a": {
     "message": "Skills"
   },
+  "skip_this_step_to_add_questions_and_answers_manual_ed1b9f80": {
+    "message": "Skip this step to add questions and answers manually after creation. The number of sources and file size you can add depends on the QnA service SKU you choose. "
+  },
   "sorry_something_went_wrong_with_connecting_bot_run_7d6785e3": {
     "message": "Sorry, something went wrong with connecting bot runtime"
   },
@@ -2510,6 +2546,9 @@
   "this_trigger_type_is_not_supported_by_the_regex_re_dc3eefa2": {
     "message": "This trigger type is not supported by the RegEx recognizer. To ensure this trigger is fired, change the recognizer type."
   },
+  "this_url_is_duplicated_a0768f44": {
+    "message": "This url is duplicated"
+  },
   "this_version_of_the_content_is_out_of_date_and_you_5e878f29": {
     "message": "This version of the content is out of date, and your last change was rejected. The content will be automatically refreshed."
   },
@@ -2621,6 +2660,9 @@
   "typing_activity_6b634ae": {
     "message": "Typing activity"
   },
+  "unable_to_determine_recognizer_type_from_data_valu_2960f526": {
+    "message": "Unable to determine recognizer type from data: { value }"
+  },
   "undo_a7be8fef": {
     "message": "Undo"
   },
@@ -2672,6 +2714,9 @@
   "updating_scripts_e17a5722": {
     "message": "Updating scripts... "
   },
+  "url_22a5f3b8": {
+    "message": "URL"
+  },
   "url_8c4ff7d2": {
     "message": "Url"
   },