Functional test coverage (#15754)

* Functional test coverage

* Enable functional test coverage for pr, insiders, and release builds

.github/workflows/insiders.yml

@@ -308,8 +308,18 @@ jobs:
           working-directory: ${{env.special-working-directory}}
         if: matrix.test-suite == 'debugger'
 
-      - name: Run functional tests
-        run: npm run test:functional
+      # Run TypeScript functional tests
+      - name: Run TypeScript functional tests
+        run: npm run test:functional:cover
+        if: matrix.test-suite == 'functional'
+
+      # Upload functional test coverage reports for later use in the "reports" job.
+      - name: Upload functional test coverage reports
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{runner.os}}-${{env.COVERAGE_REPORTS}}
+          path: ${{ env.special-working-directory }}/.nyc_output
+          retention-days: 1
         if: matrix.test-suite == 'functional'
 
   smoke-tests:

.github/workflows/pr-check.yml

@@ -311,8 +311,18 @@ jobs:
           working-directory: ${{env.special-working-directory}}
         if: matrix.test-suite == 'debugger'
 
-      - name: Run functional tests
-        run: npm run test:functional
+      # Run TypeScript functional tests
+      - name: Run TypeScript functional tests
+        run: npm run test:functional:cover
+        if: matrix.test-suite == 'functional'
+
+      # Upload functional test coverage reports for later use in the "reports" job.
+      - name: Upload functional test coverage reports
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{runner.os}}-${{env.COVERAGE_REPORTS}}
+          path: ${{ env.special-working-directory }}/.nyc_output
+          retention-days: 1
         if: matrix.test-suite == 'functional'
 
   smoke-tests:

.github/workflows/release.yml

@@ -292,8 +292,18 @@ jobs:
           working-directory: ${{env.special-working-directory}}
         if: matrix.test-suite == 'debugger'
 
-      - name: Run functional tests
-        run: npm run test:functional
+      # Run TypeScript functional tests
+      - name: Run TypeScript functional tests
+        run: npm run test:functional:cover
+        if: matrix.test-suite == 'functional'
+
+      # Upload functional test coverage reports for later use in the "reports" job.
+      - name: Upload functional test coverage reports
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{runner.os}}-${{env.COVERAGE_REPORTS}}
+          path: ${{ env.special-working-directory }}/.nyc_output
+          retention-days: 1
         if: matrix.test-suite == 'functional'
 
   smoke-tests:

build/.mocha.functional.json

@@ -1,9 +1,11 @@
 {
     "spec": "./out/test/**/*.functional.test.js",
-    "require": ["out/test/unittests.js"],
+    "require": [
+        "out/test/unittests.js"
+    ],
     "exclude": "out/**/*.jsx",
     "reporter": "mocha-multi-reporters",
-    "reporter-option": "configFile=build/.mocha-multi-reporters.config",
+    "reporter-option": "configFile=./build/.mocha-multi-reporters.config",
     "ui": "tdd",
     "recursive": true,
     "colors": true,

build/.mocha.unittests.json

@@ -1,9 +1,13 @@
 {
     "spec": "./out/test/**/*.unit.test.js",
-    "require": ["out/test/unittests.js"],
+    "require": [
+        "out/test/unittests.js"
+    ],
+    "exclude": "out/**/*.jsx",
     "reporter": "mocha-multi-reporters",
-    "reporter-option": "configFile=build/.mocha-multi-reporters.config",
+    "reporter-option": "configFile=./build/.mocha-multi-reporters.config",
     "ui": "tdd",
     "recursive": true,
-    "colors": true
+    "colors": true,
+    "timeout": 180000
 }

build/.nycrc

@@ -2,8 +2,9 @@
     "extends": "@istanbuljs/nyc-config-typescript",
     "all": true,
     "include": [
-        "src/client/**/*.ts", "src/test/**/*.js",
-        "src/startPage-ui/**/*.ts", "src/startPage-ui/**/*.js"
+        "src/client/**/*.ts", "out/client/**/*.js",
+        "src/startPage-ui/**/*.ts", "out/startPage-ui/**/*.js"
     ],
-    "exclude": ["src/test/**/*.ts", "src/test/**/*.js"]
+    "exclude": ["src/test/**/*.ts", "out/test/**/*.js"],
+    "exclude-node-modules": true
 }

package.json

@@ -1968,13 +1968,13 @@
         "kill-compile-webviews-watchd": "deemon --kill npm run compile-webviews-watch",
         "checkDependencies": "gulp checkDependencies",
         "test": "node ./out/test/standardTest.js && node ./out/test/multiRootTest.js",
-        "test:unittests": "mocha --config ./build/.mocha.unittests.js.json",
-        "test:unittests:cover": "nyc --no-clean --nycrc-path build/.nycrc mocha --config ./build/.mocha.unittests.ts.json",
+        "test:unittests": "mocha --config ./build/.mocha.unittests.json",
+        "test:unittests:cover": "nyc --no-clean --nycrc-path ./build/.nycrc mocha --config ./build/.mocha.unittests.json",
         "test:functional": "mocha --require source-map-support/register --config ./build/.mocha.functional.json",
         "test:functional:perf": "node --inspect-brk ./node_modules/mocha/bin/_mocha --require source-map-support/register --config ./build/.mocha.functional.perf.json",
         "test:functional:memleak": "node --inspect-brk ./node_modules/mocha/bin/_mocha --require source-map-support/register --config ./build/.mocha.functional.json",
-        "test:functional:cover": "npm run test:functional",
-        "test:cover:report": "nyc --nycrc-path build/.nycrc  report --reporter=text --reporter=html --reporter=text-summary --reporter=cobertura",
+        "test:functional:cover": "nyc --no-clean --nycrc-path ./build/.nycrc mocha --require source-map-support/register --config ./build/.mocha.functional.json",
+        "test:cover:report": "nyc --nycrc-path ./build/.nycrc  report --reporter=text --reporter=html --reporter=text-summary --reporter=cobertura",
         "testDebugger": "node ./out/test/testBootstrap.js ./out/test/debuggerTest.js",
         "testSingleWorkspace": "node ./out/test/testBootstrap.js ./out/test/standardTest.js",
         "preTestJediLSP": "node ./out/test/languageServers/jedi/lspSetup.js",