Speed up integration tests

[ManasJayanth]

Integration tests seem very slow. I presumed it was because of editor instance being spawn once each for each project type - opam, esy and bucklescript. Tried opening the editor instance once by having a unified entry for mocha

// runner.js
const path = require("path");

const { runTests } = require("vscode-test");

async function main() {
  try {
    // The folder containing the Extension Manifest package.json
    // Passed to `--extensionDevelopmentPath`
    const extensionDevelopmentPath = path.resolve(__dirname, "../");

    // The path to the extension test script
    // Passed to --extensionTestsPath
    const extensionTestsPath = path.resolve(__dirname, "./mocha-entry.js");

    // Download VS Code, unzip it and run the integration test
    await runTests({
      extensionDevelopmentPath,
      extensionTestsPath,
      launchArgs: ["--disable-extensions"]
    });
  } catch (err) {
    console.error("Failed to run tests");
    process.exit(1);
  }
}

main();

const path = require("path");
const Mocha = require("mocha");
const glob = require("glob");

function run() {
  // Create the mocha test
  const mocha = new Mocha({
    ui: "tdd"
  });
  // Use any mocha API
  mocha.useColors(true);

  const testsRoot = path.resolve(__dirname);

  return new Promise((c, e) => {
    // Add files to the test suite
    mocha.addFile(
      path.resolve(
        testsRoot,
        path.resolve(testsRoot, "suite", "integration.test.js")
      )
    );

    try {
      // Run the mocha test
      mocha.run(failures => {
        if (failures > 0) {
          e(new Error(`${failures} tests failed.`));
        } else {
          c();
        }
      });
    } catch (err) {
      e(err);
    }
  });
}

module.exports = {
  run
};

And clubbed the suites, but the test runner would loop indefinitely. Clubbed the test() together in one suite - still indefinite running. Clubbing everything into one test didn't help either

const assert = require("assert");
const path = require("path");
const vscode = require("vscode");
const os = require("os");
const cp = require("child_process");
const fs = require("fs-extra");
const { Uri } = vscode;

let root = path.dirname(path.dirname(__dirname));
let fixtureSrcDir = path.join(root, "fixtures");

suite("Running e2e tests", () => {
  test("Esy", async () => {
    let sampleEsySrc = path.join(fixtureSrcDir, "sample-esy");
    let projectPath = path.join(os.tmpdir(), "sample-esy");
    let projectUri = Uri.file(projectPath);

    fs.copySync(sampleEsySrc, projectPath);
    cp.execSync("esy", { cwd: projectPath });

    await vscode.commands.executeCommand("vscode.openFolder", projectUri);
    let reasonDocument = await vscode.workspace.openTextDocument(
      Uri.file(path.join(projectPath, "bin", "SampleEsyApp.re"))
    );

    let ocamlDocument = await vscode.workspace.openTextDocument(
      Uri.file(path.join(projectPath, "bin", "CamlUtil.ml"))
    );

    assert.equal(
      reasonDocument.languageId,
      "reason",
      "Must be identified as a Reason document"
    );

    assert.equal(
      ocamlDocument.languageId,
      "ocaml",
      "Must be identified as an OCaml document"
    );

    function delay(timeout) {
      return new Promise(resolve => {
        setTimeout(resolve, timeout);
      });
    }

    await delay(500);
    let diagnostics = await vscode.languages.getDiagnostics(
      Uri.file(path.join(projectPath, "bin", "SampleEsyApp.re"))
    );
    if (process.platform != "win32" && process.platform != "win64") {
      assert.equal(
        diagnostics.length,
        1,
        "There should only be one diagnostic"
      );
      assert.equal(diagnostics[0].message, "Warning 26: unused variable foo.");
      assert.equal(
        diagnostics[0].severity,
        1,
        "Severity of this diagnostic should be 1 (Warning). It was " +
          diagnostics[0].severity
      );
      assert.equal(diagnostics[0].range.start.line, 3);
      assert.equal(diagnostics[0].range.start.character, 6);
      assert.equal(diagnostics[0].range.end.line, 3);
      assert.equal(diagnostics[0].range.end.character, 9);
    }

    // TODO: the plugin could support build related tasks
    // const expected = [
    //   { subcommand: "build", group: vscode.TaskGroup.Build },
    //   { subcommand: "run", group: undefined }
    // ];
    // const tasks = await vscode.tasks.fetchTasks();

    console.log("Cleaning up (esy)...");
    try {
      fs.removeSync(projectPath);
    } catch (e) {}
    if (process.platform == "win32" || process.platform == "win64") {
      return;
    }

    let sampleOpamSrc = path.join(fixtureSrcDir, "sample-opam");
    projectPath = path.join(os.tmpdir(), "sample-opam");
    let opamRoot = path.join(os.tmpdir(), "opam-root");
    fs.copySync(sampleOpamSrc, projectPath);
    cp.execSync(`mkdir -p ${opamRoot}`);
    let env = cp
      .execSync(
        `sh -c 'opam install . --deps-only --yes > /dev/null; opam env'`,
        { cwd: projectPath }
      )
      .toString();
    let regexpMatch = env.match(/PATH=[^;]+;/g);
    if (regexpMatch.length >= 1) {
      process.env["PATH"] = Array.prototype.reduce.call(
        regexpMatch,
        function(acc, pathString) {
          return (
            acc +
            pathString
              .replace(/[;'"]/g, "")
              .split("=")[1]
              .split(":")
              .concat(process.env["PATH"].split("=")[1])
              .join(":")
          );
        },
        ""
      );
    }
    projectUri = Uri.file(projectPath);
    await vscode.commands.executeCommand("vscode.openFolder", projectUri);
    reasonDocument = await vscode.workspace.openTextDocument(
      Uri.file(path.join(projectPath, "foo.re"))
    );

    ocamlDocument = await vscode.workspace.openTextDocument(
      Uri.file(path.join(projectPath, "main.ml"))
    );

    assert.equal(
      reasonDocument.languageId,
      "reason",
      "Must be identified as a Reason document"
    );

    assert.equal(
      ocamlDocument.languageId,
      "ocaml",
      "Must be identified as an OCaml document"
    );
    console.log("Cleaning up (opam)...");
    try {
      console.log("  Removing switch");
      console.log(cp.execSync("opam switch remove e2e --yes").toString());
      console.log("  Removing test project");
      fs.removeSync(projectPath);
    } catch (e) {}
  });
  // test("Bsb", async () => {
  //   let sampleEsySrc = path.join(fixtureSrcDir, "sample-bsb");
  //   let projectPath = path.join(os.tmpdir(), "sample-bsb");
  //   let projectUri = Uri.file(projectPath);

  //   fs.copySync(sampleEsySrc, projectPath);
  //   cp.execSync("npm i", { cwd: projectPath });

  //   await vscode.commands.executeCommand("vscode.openFolder", projectUri);
  //   let reasonDocument = await vscode.workspace.openTextDocument(
  //     Uri.file(path.join(projectPath, "src", "Lib.re"))
  //   );

  //   let ocamlDocument = await vscode.workspace.openTextDocument(
  //     Uri.file(path.join(projectPath, "src", "main.ml"))
  //   );

  //   assert.equal(
  //     reasonDocument.languageId,
  //     "reason",
  //     "Must be identified as a Reason document"
  //   );

  //   assert.equal(
  //     ocamlDocument.languageId,
  //     "ocaml",
  //     "Must be identified as an OCaml document"
  //   );

  //   function delay(timeout) {
  //     return new Promise(resolve => {
  //       setTimeout(resolve, timeout);
  //     });
  //   }

  //   await delay(500);
  //   let diagnostics = await vscode.languages.getDiagnostics(
  //     Uri.file(path.join(projectPath, "lib", "Lib.re"))
  //   );

  //   assert.equal(diagnostics.length, 0, "There should only be one diagnostic");

  //   // TODO: the plugin could support build related tasks
  //   // const expected = [
  //   //   { subcommand: "build", group: vscode.TaskGroup.Build },
  //   //   { subcommand: "run", group: undefined }
  //   // ];
  //   // const tasks = await vscode.tasks.fetchTasks();

  //   console.log("Cleaning up...");
  //   try {
  //     fs.removeSync(projectPath);
  //   } catch (e) {}
  // });
});

[ManasJayanth]

Moving on to other issues. This is up for grabs