Support scaffolding scenario in vscode (microsoft#5294)

Support scaffolding scenario in vscode whose user experience is like
"tsp init". Vscode would collect information from end user and work with
compiler to create new TypeSpec project underneath.

fixes: microsoft#4859

.chronus/changes/vscode-scaffolding-2024-11-7-12-44-28.md

@@ -0,0 +1,20 @@
+---
+changeKind: feature
+packages:
+  - typespec-vscode
+---
+
+Support "Create TypeSpec Project" in vscode command and EXPLORER when no folder opened
+Add Setting "typespec.initTemplatesUrls" where user can configure additional template to use to create TypeSpec project
+example:
+```
+{
+  "typespec.initTemplatesUrls": [
+    {
+      "name": "displayName",
+      "url": "https://urlToTheFileContainsTemplates"
+    }],
+}
+```
+Support "Install TypeSpec Compiler/CLI globally" in vscode command to install TypeSpec compiler globally easily
+

.chronus/changes/vscode-scaffolding-2024-11-7-12-46-5.md

@@ -0,0 +1,7 @@
+---
+changeKind: feature
+packages:
+  - "@typespec/compiler"
+---
+
+Add capacities in TypeSpec Language Server to support "Scaffolding new TypeSpec project" in IDE

packages/compiler/src/core/node-host.browser.ts

@@ -1 +1,2 @@
 export const NodeHost = undefined;
+export const CompilerPackageRoot = undefined;

packages/compiler/src/init/core-templates.ts

@@ -1,12 +1,22 @@
-import { readFile } from "fs/promises";
 import { CompilerPackageRoot } from "../core/node-host.js";
 import { resolvePath } from "../core/path-utils.js";
+import { CompilerHost } from "../index.js";
 
 export const templatesDir = resolvePath(CompilerPackageRoot, "templates");
+export interface LoadedCoreTemplates {
+  readonly baseUri: string;
+  readonly templates: Record<string, any>;
+}
 
-const content = JSON.parse(await readFile(resolvePath(templatesDir, "scaffolding.json"), "utf-8"));
-
-export const TypeSpecCoreTemplates = {
-  baseUri: templatesDir,
-  templates: content,
-};
+let typeSpecCoreTemplates: LoadedCoreTemplates | undefined;
+export async function getTypeSpecCoreTemplates(host: CompilerHost): Promise<LoadedCoreTemplates> {
+  if (typeSpecCoreTemplates === undefined) {
+    const file = await host.readFile(resolvePath(templatesDir, "scaffolding.json"));
+    const content = JSON.parse(file.text);
+    typeSpecCoreTemplates = {
+      baseUri: templatesDir,
+      templates: content,
+    };
+  }
+  return typeSpecCoreTemplates;
+}

packages/compiler/src/init/init-template-validate.ts

@@ -0,0 +1,20 @@
+import { createJSONSchemaValidator } from "../core/schema-validator.js";
+import { Diagnostic, NoTarget, SourceFile } from "../index.js";
+import { InitTemplateSchema } from "./init-template.js";
+
+export type ValidationResult = {
+  valid: boolean;
+  diagnostics: readonly Diagnostic[];
+};
+
+export function validateTemplateDefinitions(
+  template: unknown,
+  templateName: SourceFile | typeof NoTarget,
+  strictValidation: boolean,
+): ValidationResult {
+  const validator = createJSONSchemaValidator(InitTemplateSchema, {
+    strict: strictValidation,
+  });
+  const diagnostics = validator.validate(template, templateName);
+  return { valid: diagnostics.length === 0, diagnostics };
+}

packages/compiler/src/init/init.ts

@@ -4,12 +4,12 @@ import prompts from "prompts";
 import * as semver from "semver";
 import { createDiagnostic } from "../core/messages.js";
 import { getBaseFileName, getDirectoryPath } from "../core/path-utils.js";
-import { createJSONSchemaValidator } from "../core/schema-validator.js";
 import { CompilerHost, Diagnostic, NoTarget, SourceFile } from "../core/types.js";
 import { MANIFEST } from "../manifest.js";
 import { readUrlOrPath } from "../utils/misc.js";
-import { TypeSpecCoreTemplates } from "./core-templates.js";
-import { InitTemplate, InitTemplateLibrarySpec, InitTemplateSchema } from "./init-template.js";
+import { getTypeSpecCoreTemplates } from "./core-templates.js";
+import { validateTemplateDefinitions, ValidationResult } from "./init-template-validate.js";
+import { InitTemplate, InitTemplateLibrarySpec } from "./init-template.js";
 import { makeScaffoldingConfig, normalizeLibrary, scaffoldNewProject } from "./scaffold.js";
 
 export interface InitTypeSpecProjectOptions {
@@ -30,15 +30,16 @@ export async function initTypeSpecProject(
 
   // Download template configuration and prompt user to select a template
   // No validation is done until one has been selected
+  const typeSpecCoreTemplates = await getTypeSpecCoreTemplates(host);
   const result =
     options.templatesUrl === undefined
-      ? (TypeSpecCoreTemplates as LoadedTemplate)
+      ? (typeSpecCoreTemplates as LoadedTemplate)
       : await downloadTemplates(host, options.templatesUrl);
   const templateName = options.template ?? (await promptTemplateSelection(result.templates));
 
   // Validate minimum compiler version for non built-in templates
   if (
-    result !== TypeSpecCoreTemplates &&
+    result !== typeSpecCoreTemplates &&
     !(await validateTemplate(result.templates[templateName], result))
   ) {
     return;
@@ -193,11 +194,6 @@ async function promptTemplateSelection(templates: Record<string, any>): Promise<
   return templateName;
 }
 
-type ValidationResult = {
-  valid: boolean;
-  diagnostics: readonly Diagnostic[];
-};
-
 async function validateTemplate(template: any, loaded: LoadedTemplate): Promise<boolean> {
   // After selection, validate the template definition
   const currentCompilerVersion = MANIFEST.version;
@@ -278,18 +274,6 @@ export class InitTemplateError extends Error {
   }
 }
 
-function validateTemplateDefinitions(
-  template: unknown,
-  templateName: SourceFile,
-  strictValidation: boolean,
-): ValidationResult {
-  const validator = createJSONSchemaValidator(InitTemplateSchema, {
-    strict: strictValidation,
-  });
-  const diagnostics = validator.validate(template, templateName);
-  return { valid: diagnostics.length === 0, diagnostics };
-}
-
 function logDiagnostics(diagnostics: readonly Diagnostic[]): void {
   diagnostics.forEach((diagnostic) => {
     // eslint-disable-next-line no-console

packages/compiler/src/server/server.ts

@@ -15,7 +15,7 @@ import {
 import { NodeHost } from "../core/node-host.js";
 import { typespecVersion } from "../utils/misc.js";
 import { createServer } from "./serverlib.js";
-import { Server, ServerHost, ServerLog } from "./types.js";
+import { CustomRequestName, Server, ServerHost, ServerLog } from "./types.js";
 
 let server: Server | undefined = undefined;
 
@@ -129,6 +129,13 @@ function main() {
   connection.onExecuteCommand(profile(s.executeCommand));
   connection.languages.semanticTokens.on(profile(s.buildSemanticTokens));
 
+  const validateInitProjectTemplate: CustomRequestName = "typespec/validateInitProjectTemplate";
+  connection.onRequest(validateInitProjectTemplate, profile(s.validateInitProjectTemplate));
+  const getInitProjectContextRequestName: CustomRequestName = "typespec/getInitProjectContext";
+  connection.onRequest(getInitProjectContextRequestName, profile(s.getInitProjectContext));
+  const initProjectRequestName: CustomRequestName = "typespec/initProject";
+  connection.onRequest(initProjectRequestName, profile(s.initProject));
+
   documents.onDidChangeContent(profile(s.checkChange));
   documents.onDidClose(profile(s.documentClosed));
 

packages/compiler/src/server/serverlib.ts

@@ -50,11 +50,22 @@ import { resolveCodeFix } from "../core/code-fixes.js";
 import { compilerAssert, getSourceLocation } from "../core/diagnostics.js";
 import { formatTypeSpec } from "../core/formatter.js";
 import { getEntityName, getTypeName } from "../core/helpers/type-name-utils.js";
-import { ProcessedLog, resolveModule, ResolveModuleHost } from "../core/index.js";
+import {
+  NoTarget,
+  ProcessedLog,
+  resolveModule,
+  ResolveModuleHost,
+  typespecVersion,
+} from "../core/index.js";
 import { formatLog } from "../core/logger/index.js";
 import { getPositionBeforeTrivia } from "../core/parser-utils.js";
 import { getNodeAtPosition, getNodeAtPositionDetail, visitChildren } from "../core/parser.js";
-import { ensureTrailingDirectorySeparator, getDirectoryPath } from "../core/path-utils.js";
+import {
+  ensureTrailingDirectorySeparator,
+  getDirectoryPath,
+  joinPaths,
+  normalizePath,
+} from "../core/path-utils.js";
 import type { Program } from "../core/program.js";
 import { skipTrivia, skipWhiteSpace } from "../core/scanner.js";
 import { createSourceFile, getSourceFileKindFromExt } from "../core/source-file.js";
@@ -75,6 +86,10 @@ import {
   TypeReferenceNode,
   TypeSpecScriptNode,
 } from "../core/types.js";
+import { getTypeSpecCoreTemplates } from "../init/core-templates.js";
+import { validateTemplateDefinitions } from "../init/init-template-validate.js";
+import { InitTemplate } from "../init/init-template.js";
+import { scaffoldNewProject } from "../init/scaffold.js";
 import { getNormalizedRealPath, resolveTspMain } from "../utils/misc.js";
 import { getSemanticTokens } from "./classify.js";
 import { createCompileService } from "./compile-service.js";
@@ -94,9 +109,13 @@ import {
 } from "./type-details.js";
 import {
   CompileResult,
+  InitProjectConfig,
+  InitProjectContext,
   SemanticTokenKind,
   Server,
+  ServerCustomCapacities,
   ServerHost,
+  ServerInitializeResult,
   ServerLog,
   ServerSourceFile,
   ServerWorkspaceFolder,
@@ -162,6 +181,10 @@ export function createServer(host: ServerHost): Server {
     getCodeActions,
     executeCommand,
     log,
+
+    getInitProjectContext,
+    validateInitProjectTemplate,
+    initProject,
   };
 
   async function initialize(params: InitializeParams): Promise<InitializeResult> {
@@ -246,14 +269,76 @@ export function createServer(host: ServerHost): Server {
     }
 
     log({ level: "info", message: `Workspace Folders`, detail: workspaceFolders });
-    return { capabilities };
+    const customCapacities: ServerCustomCapacities = {
+      getInitProjectContext: true,
+      initProject: true,
+      validateInitProjectTemplate: true,
+    };
+    // the file path is expected to be .../@typespec/compiler/dist/src/server/serverlib.js
+    const curFile = normalizePath(compilerHost.fileURLToPath(import.meta.url));
+    const SERVERLIB_PATH_ENDWITH = "/dist/src/server/serverlib.js";
+    let compilerRootFolder = undefined;
+    if (!curFile.endsWith(SERVERLIB_PATH_ENDWITH)) {
+      log({ level: "warning", message: `Unexpected path for serverlib found: ${curFile}` });
+    } else {
+      compilerRootFolder = curFile.slice(0, curFile.length - SERVERLIB_PATH_ENDWITH.length);
+    }
+    const result: ServerInitializeResult = {
+      serverInfo: {
+        name: "TypeSpec Language Server",
+        version: typespecVersion,
+      },
+      capabilities,
+      customCapacities,
+      compilerRootFolder,
+      compilerCliJsPath: compilerRootFolder
+        ? joinPaths(compilerRootFolder, "cmd", "tsp.js")
+        : undefined,
+    };
+    return result;
   }
 
   function initialized(params: InitializedParams): void {
     isInitialized = true;
     log({ level: "info", message: "Initialization complete." });
   }
 
+  async function getInitProjectContext(): Promise<InitProjectContext> {
+    return {
+      coreInitTemplates: await getTypeSpecCoreTemplates(host.compilerHost),
+    };
+  }
+
+  async function validateInitProjectTemplate(param: { template: InitTemplate }): Promise<boolean> {
+    const { template } = param;
+    // even when the strict validation fails, we still try to proceed with relaxed validation
+    // so just do relaxed validation directly here
+    const validationResult = validateTemplateDefinitions(template, NoTarget, false);
+    if (!validationResult.valid) {
+      for (const diag of validationResult.diagnostics) {
+        log({
+          level: diag.severity,
+          message: diag.message,
+          detail: {
+            code: diag.code,
+            url: diag.url,
+          },
+        });
+      }
+    }
+    return validationResult.valid;
+  }
+
+  async function initProject(param: { config: InitProjectConfig }): Promise<boolean> {
+    try {
+      await scaffoldNewProject(compilerHost, param.config);
+      return true;
+    } catch (e) {
+      log({ level: "error", message: "Unexpected error when initializing project", detail: e });
+      return false;
+    }
+  }
+
   async function workspaceFoldersChanged(e: WorkspaceFoldersChangeEvent) {
     log({ level: "info", message: "Workspace Folders Changed", detail: e });
     const map = new Map(workspaceFolders.map((f) => [f.uri, f]));

packages/compiler/src/server/types.ts

@@ -38,6 +38,9 @@ import {
 } from "vscode-languageserver";
 import { TextDocument, TextEdit } from "vscode-languageserver-textdocument";
 import type { CompilerHost, Program, SourceFile, TypeSpecScriptNode } from "../core/index.js";
+import { LoadedCoreTemplates } from "../init/core-templates.js";
+import { InitTemplate, InitTemplateLibrarySpec } from "../init/init-template.js";
+import { ScaffoldingConfig } from "../init/scaffold.js";
 
 export type ServerLogLevel = "trace" | "debug" | "info" | "warning" | "error";
 export interface ServerLog {
@@ -89,6 +92,15 @@ export interface Server {
   getCodeActions(params: CodeActionParams): Promise<CodeAction[]>;
   executeCommand(params: ExecuteCommandParams): Promise<void>;
   log(log: ServerLog): void;
+
+  // Following custom capacities are added for supporting tsp init project from IDE (vscode for now) so that IDE can trigger compiler
+  // to do the real job while collecting the necessary information accordingly from the user.
+  // We can't do the tsp init experience by simple cli interface because the experience needs to talk
+  // with the compiler for multiple times in different steps (i.e. get core templates, validate the selected template, scaffold the project)
+  // and it's not a good idea to expose these capacity in cli interface and call cli again and again.
+  getInitProjectContext(): Promise<InitProjectContext>;
+  validateInitProjectTemplate(param: { template: InitTemplate }): Promise<boolean>;
+  initProject(param: { config: InitProjectConfig }): Promise<boolean>;
 }
 
 export interface ServerSourceFile extends SourceFile {
@@ -135,3 +147,28 @@ export interface SemanticToken {
   pos: number;
   end: number;
 }
+
+export type CustomRequestName =
+  | "typespec/getInitProjectContext"
+  | "typespec/initProject"
+  | "typespec/validateInitProjectTemplate";
+export interface ServerCustomCapacities {
+  getInitProjectContext?: boolean;
+  validateInitProjectTemplate?: boolean;
+  initProject?: boolean;
+}
+
+export interface ServerInitializeResult extends InitializeResult {
+  customCapacities?: ServerCustomCapacities;
+  compilerRootFolder?: string;
+  compilerCliJsPath?: string;
+}
+
+export interface InitProjectContext {
+  /** provide the default templates current compiler/cli supports */
+  coreInitTemplates: LoadedCoreTemplates;
+}
+
+export type InitProjectConfig = ScaffoldingConfig;
+export type InitProjectTemplate = InitTemplate;
+export type InitProjectTemplateLibrarySpec = InitTemplateLibrarySpec;

packages/compiler/test/e2e/init-templates.e2e.ts

@@ -6,7 +6,7 @@ import { resolve } from "path/posix";
 import { fileURLToPath } from "url";
 import { beforeAll, describe, it } from "vitest";
 import { NodeHost } from "../../src/index.js";
-import { TypeSpecCoreTemplates } from "../../src/init/core-templates.js";
+import { getTypeSpecCoreTemplates } from "../../src/init/core-templates.js";
 import { makeScaffoldingConfig, scaffoldNewProject } from "../../src/init/scaffold.js";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -64,15 +64,16 @@ describe("Init templates e2e tests", () => {
   });
 
   async function scaffoldTemplateTo(name: string, targetFolder: string) {
-    const template = TypeSpecCoreTemplates.templates[name];
+    const typeSpecCoreTemplates = await getTypeSpecCoreTemplates(NodeHost);
+    const template = typeSpecCoreTemplates.templates[name];
     ok(template, `Template '${name}' not found`);
     await scaffoldNewProject(
       NodeHost,
       makeScaffoldingConfig(template, {
         name,
         folderName: name,
         directory: targetFolder,
-        baseUri: TypeSpecCoreTemplates.baseUri,
+        baseUri: typeSpecCoreTemplates.baseUri,
       }),
     );
   }