(chore) fix flaky test (hopefully) (#2064)

add a warmup phase to all the tests and add a timeout determination function for the benchmark
sveltejs · Jun 22, 2023 · 48ae8ce · 48ae8ce
1 parent 71da8d3
commit 48ae8ce
Show file tree

Hide file tree

Showing 23 changed files with 254 additions and 69 deletions.
diff --git a/docs/internal/overview.md b/docs/internal/overview.md
@@ -83,8 +83,11 @@ function render() {
     let world = 'name';
     // -- the transformed template
     async () => {
-      { svelteHTML.createElement("h1", {}); world; }
-    }
+        {
+            svelteHTML.createElement('h1', {});
+            world;
+        }
+    };
     // -- obtained props, slots and events,
     //    to get intellisense for this component in other components
     return { props: { world }, slots: {}, events: {} };

diff --git a/packages/language-server/test/plugins/typescript/TypescriptPlugin.test.ts b/packages/language-server/test/plugins/typescript/TypescriptPlugin.test.ts
@@ -16,8 +16,13 @@ import { INITIAL_VERSION } from '../../../src/plugins/typescript/DocumentSnapsho
 import { __resetCache } from '../../../src/plugins/typescript/service';
 import { ignoredBuildDirectories } from '../../../src/plugins/typescript/SnapshotManager';
 import { pathToUrl } from '../../../src/utils';
+import { serviceWarmup } from './test-utils';
+
+const testDir = path.join(__dirname, 'testfiles');
+
+describe('TypescriptPlugin', function () {
+    serviceWarmup(this, testDir);
 
-describe('TypescriptPlugin', () => {
     function getUri(filename: string) {
         const filePath = path.join(__dirname, 'testfiles', filename);
         return pathToUrl(filePath);
@@ -33,7 +38,6 @@ describe('TypescriptPlugin', () => {
                 ? new Document(args.uri, harmonizeNewLines(args.text))
                 : document
         );
-        const testDir = path.join(__dirname, 'testfiles');
         const filePath = path.join(testDir, filename);
         const document = new Document(pathToUrl(filePath), ts.sys.readFile(filePath) || '');
         const lsConfigManager = new LSConfigManager();

diff --git a/packages/language-server/test/plugins/typescript/features/CallHierarchyProvider.test.ts b/packages/language-server/test/plugins/typescript/features/CallHierarchyProvider.test.ts
@@ -13,11 +13,13 @@ import { CallHierarchyProviderImpl } from '../../../../src/plugins/typescript/fe
 import { LSAndTSDocResolver } from '../../../../src/plugins/typescript/LSAndTSDocResolver';
 import { __resetCache } from '../../../../src/plugins/typescript/service';
 import { pathToUrl } from '../../../../src/utils';
+import { serviceWarmup } from '../test-utils';
 
 const testDir = path.join(__dirname, '..');
 
-describe('CallHierarchyProvider', () => {
+describe('CallHierarchyProvider', function () {
     const callHierarchyTestDirRelative = path.join('testfiles', 'call-hierarchy');
+    serviceWarmup(this, path.join(testDir, callHierarchyTestDirRelative), pathToUrl(testDir));
 
     function getFullPath(filename: string) {
         return path.join(testDir, 'testfiles', 'call-hierarchy', filename);

diff --git a/packages/language-server/test/plugins/typescript/features/CodeActionsProvider.test.ts b/packages/language-server/test/plugins/typescript/features/CodeActionsProvider.test.ts
@@ -19,11 +19,18 @@ import { CompletionsProviderImpl } from '../../../../src/plugins/typescript/feat
 import { LSAndTSDocResolver } from '../../../../src/plugins/typescript/LSAndTSDocResolver';
 import { __resetCache } from '../../../../src/plugins/typescript/service';
 import { pathToUrl } from '../../../../src/utils';
+import { recursiveServiceWarmup } from '../test-utils';
 
 const testDir = path.join(__dirname, '..');
 const indent = ' '.repeat(4);
 
-describe('CodeActionsProvider', () => {
+describe('CodeActionsProvider', function () {
+    recursiveServiceWarmup(
+        this,
+        path.join(testDir, 'testfiles', 'code-actions'),
+        pathToUrl(testDir)
+    );
+
     function getFullPath(filename: string) {
         return path.join(testDir, 'testfiles', 'code-actions', filename);
     }

diff --git a/packages/language-server/test/plugins/typescript/features/CompletionProvider.test.ts b/packages/language-server/test/plugins/typescript/features/CompletionProvider.test.ts
@@ -23,7 +23,7 @@ import { LSAndTSDocResolver } from '../../../../src/plugins/typescript/LSAndTSDo
 import { sortBy } from 'lodash';
 import { LSConfigManager } from '../../../../src/ls-config';
 import { __resetCache } from '../../../../src/plugins/typescript/service';
-import { getRandomVirtualDirPath, setupVirtualEnvironment } from '../test-utils';
+import { getRandomVirtualDirPath, serviceWarmup, setupVirtualEnvironment } from '../test-utils';
 
 const testDir = join(__dirname, '..');
 const testFilesDir = join(testDir, 'testfiles', 'completions');
@@ -39,7 +39,9 @@ function harmonizeNewLines(input?: string) {
 }
 
 // describe('CompletionProviderImpl (old transformation)', test(false));
-describe('CompletionProviderImpl', () => {
+describe('CompletionProviderImpl', function () {
+    serviceWarmup(this, testFilesDir, pathToUrl(testDir));
+
     function setup(filename: string) {
         const docManager = new DocumentManager(
             (textDocument) => new Document(textDocument.uri, textDocument.text)

diff --git a/packages/language-server/test/plugins/typescript/features/DiagnosticsProvider.test.ts b/packages/language-server/test/plugins/typescript/features/DiagnosticsProvider.test.ts
@@ -8,10 +8,13 @@ import { DiagnosticsProviderImpl } from '../../../../src/plugins/typescript/feat
 import { LSAndTSDocResolver } from '../../../../src/plugins/typescript/LSAndTSDocResolver';
 import { __resetCache } from '../../../../src/plugins/typescript/service';
 import { normalizePath, pathToUrl } from '../../../../src/utils';
+import { serviceWarmup } from '../test-utils';
 
 const testDir = path.join(__dirname, '..', 'testfiles', 'diagnostics');
 
-describe('DiagnosticsProvider', () => {
+describe('DiagnosticsProvider', function () {
+    serviceWarmup(this, testDir);
+
     function setup(filename: string) {
         const docManager = new DocumentManager(
             (textDocument) => new Document(textDocument.uri, textDocument.text)

diff --git a/.../language-server/test/plugins/typescript/features/FindComponentReferencesProvider.test.ts b/.../language-server/test/plugins/typescript/features/FindComponentReferencesProvider.test.ts
@@ -6,10 +6,13 @@ import { LSConfigManager } from '../../../../src/ls-config';
 import { FindComponentReferencesProviderImpl } from '../../../../src/plugins/typescript/features/FindComponentReferencesProvider';
 import { LSAndTSDocResolver } from '../../../../src/plugins/typescript/LSAndTSDocResolver';
 import { pathToUrl } from '../../../../src/utils';
+import { serviceWarmup } from '../test-utils';
 
 const testDir = path.join(__dirname, '..');
 
-describe('FindComponentReferencesProvider', () => {
+describe('FindComponentReferencesProvider', function () {
+    serviceWarmup(this, testDir);
+
     function getFullPath(filename: string) {
         return path.join(testDir, 'testfiles', filename);
     }
@@ -24,7 +27,11 @@ describe('FindComponentReferencesProvider', () => {
         );
         const lsConfigManager = new LSConfigManager();
 
-        const lsAndTsDocResolver = new LSAndTSDocResolver(docManager, [testDir], lsConfigManager);
+        const lsAndTsDocResolver = new LSAndTSDocResolver(
+            docManager,
+            [pathToUrl(testDir)],
+            lsConfigManager
+        );
         const provider = new FindComponentReferencesProviderImpl(lsAndTsDocResolver);
         const document = openDoc(filename);
         return { provider, document, openDoc, lsConfigManager };

diff --git a/packages/language-server/test/plugins/typescript/features/FindFileReferencesProvider.test.ts b/packages/language-server/test/plugins/typescript/features/FindFileReferencesProvider.test.ts
@@ -7,10 +7,13 @@ import { LSConfigManager } from '../../../../src/ls-config';
 import { FindFileReferencesProviderImpl } from '../../../../src/plugins/typescript/features/FindFileReferencesProvider';
 import { LSAndTSDocResolver } from '../../../../src/plugins/typescript/LSAndTSDocResolver';
 import { pathToUrl } from '../../../../src/utils';
+import { serviceWarmup } from '../test-utils';
 
 const testDir = path.join(__dirname, '..');
 
-describe('FindFileReferencesProvider', () => {
+describe('FindFileReferencesProvider', function () {
+    serviceWarmup(this, testDir);
+
     function getFullPath(filename: string) {
         return path.join(testDir, 'testfiles', filename);
     }
@@ -24,7 +27,11 @@ describe('FindFileReferencesProvider', () => {
             (textDocument) => new Document(textDocument.uri, textDocument.text)
         );
         const lsConfigManager = new LSConfigManager();
-        const lsAndTsDocResolver = new LSAndTSDocResolver(docManager, [testDir], lsConfigManager);
+        const lsAndTsDocResolver = new LSAndTSDocResolver(
+            docManager,
+            [pathToUrl(testDir)],
+            lsConfigManager
+        );
         const provider = new FindFileReferencesProviderImpl(lsAndTsDocResolver);
         const document = openDoc(filename);
         return { provider, document, openDoc };
@@ -39,7 +46,7 @@ describe('FindFileReferencesProvider', () => {
         }
     }
 
-    it('finds file references', async () => {
+    it('finds file references', async function () {
         const { provider, document, openDoc } = setup('find-file-references-child.svelte');
         //Make known all the associated files
         openDoc('find-file-references-parent.svelte');

diff --git a/packages/language-server/test/plugins/typescript/features/FindReferencesProvider.test.ts b/packages/language-server/test/plugins/typescript/features/FindReferencesProvider.test.ts
@@ -8,10 +8,13 @@ import { FindReferencesProviderImpl } from '../../../../src/plugins/typescript/f
 import { LSAndTSDocResolver } from '../../../../src/plugins/typescript/LSAndTSDocResolver';
 import { __resetCache } from '../../../../src/plugins/typescript/service';
 import { pathToUrl } from '../../../../src/utils';
+import { serviceWarmup } from '../test-utils';
 
 const testDir = path.join(__dirname, '..');
 
-describe('FindReferencesProvider', () => {
+describe('FindReferencesProvider', function () {
+    serviceWarmup(this, testDir);
+
     function getFullPath(filename: string) {
         return path.join(testDir, 'testfiles', filename);
     }

diff --git a/packages/language-server/test/plugins/typescript/features/HoverProvider.test.ts b/packages/language-server/test/plugins/typescript/features/HoverProvider.test.ts
@@ -8,12 +8,16 @@ import { HoverProviderImpl } from '../../../../src/plugins/typescript/features/H
 import { LSAndTSDocResolver } from '../../../../src/plugins/typescript/LSAndTSDocResolver';
 import { __resetCache } from '../../../../src/plugins/typescript/service';
 import { pathToUrl } from '../../../../src/utils';
+import { serviceWarmup } from '../test-utils';
 
 const testDir = path.join(__dirname, '..');
+const hoverTestDir = path.join(testDir, 'testfiles', 'hover');
+
+describe('HoverProvider', function () {
+    serviceWarmup(this, hoverTestDir, pathToUrl(testDir));
 
-describe('HoverProvider', () => {
     function getFullPath(filename: string) {
-        return path.join(testDir, 'testfiles', 'hover', filename);
+        return path.join(hoverTestDir, filename);
     }
 
     function setup(filename: string) {

diff --git a/packages/language-server/test/plugins/typescript/features/ImplemenationProvider.test.ts b/packages/language-server/test/plugins/typescript/features/ImplemenationProvider.test.ts
@@ -7,10 +7,14 @@ import { LSAndTSDocResolver } from '../../../../src/plugins';
 import { ImplementationProviderImpl } from '../../../../src/plugins/typescript/features/ImplementationProvider';
 import { pathToUrl } from '../../../../src/utils';
 import { Location } from 'vscode-languageserver-protocol';
+import { serviceWarmup } from '../test-utils';
 
 const testDir = path.join(__dirname, '..');
+const implementationTestDir = path.join(testDir, 'testfiles', 'implementation');
+
+describe('ImplementationProvider', function () {
+    serviceWarmup(this, implementationTestDir, pathToUrl(testDir));
 
-describe('ImplementationProvider', () => {
     function getFullPath(filename: string) {
         return path.join(testDir, 'testfiles', 'implementation', filename);
     }

diff --git a/packages/language-server/test/plugins/typescript/features/RenameProvider.test.ts b/packages/language-server/test/plugins/typescript/features/RenameProvider.test.ts
@@ -8,12 +8,16 @@ import { RenameProviderImpl } from '../../../../src/plugins/typescript/features/
 import { LSAndTSDocResolver } from '../../../../src/plugins/typescript/LSAndTSDocResolver';
 import { __resetCache } from '../../../../src/plugins/typescript/service';
 import { pathToUrl } from '../../../../src/utils';
+import { serviceWarmup } from '../test-utils';
 
 const testDir = path.join(__dirname, '..');
+const renameTestDir = path.join(testDir, 'testfiles', 'rename');
+
+describe('RenameProvider', function () {
+    serviceWarmup(this, renameTestDir, pathToUrl(testDir));
 
-describe('RenameProvider', () => {
     function getFullPath(filename: string) {
-        return path.join(testDir, 'testfiles', 'rename', filename);
+        return path.join(renameTestDir, filename);
     }
 
     function getUri(filename: string) {

diff --git a/packages/language-server/test/plugins/typescript/features/SelectionRangeProvider.test.ts b/packages/language-server/test/plugins/typescript/features/SelectionRangeProvider.test.ts
@@ -7,10 +7,14 @@ import { SelectionRangeProviderImpl } from '../../../../src/plugins/typescript/f
 import { LSAndTSDocResolver } from '../../../../src/plugins/typescript/LSAndTSDocResolver';
 import { pathToUrl } from '../../../../src/utils';
 import { LSConfigManager } from '../../../../src/ls-config';
+import { serviceWarmup } from '../test-utils';
 
 const testDir = path.join(__dirname, '..');
+const selectionRangeTestDir = path.join(testDir, 'testfiles', 'selection-range');
+
+describe('SelectionRangeProvider', function () {
+    serviceWarmup(this, selectionRangeTestDir, pathToUrl(testDir));
 
-describe('SelectionRangeProvider', () => {
     function setup() {
         const docManager = new DocumentManager(
             (textDocument) => new Document(textDocument.uri, textDocument.text)

diff --git a/packages/language-server/test/plugins/typescript/features/SemanticTokensProvider.test.ts b/packages/language-server/test/plugins/typescript/features/SemanticTokensProvider.test.ts
@@ -13,17 +13,20 @@ import { LSConfigManager } from '../../../../src/ls-config';
 import { SemanticTokensProviderImpl } from '../../../../src/plugins/typescript/features/SemanticTokensProvider';
 import { LSAndTSDocResolver } from '../../../../src/plugins/typescript/LSAndTSDocResolver';
 import { pathToUrl } from '../../../../src/utils';
+import { serviceWarmup } from '../test-utils';
 
 const testDir = path.join(__dirname, '..');
+const semanticTokenTestDir = path.join(testDir, 'testfiles', 'semantic-tokens');
 
-describe('SemanticTokensProvider', () => {
+describe('SemanticTokensProvider', function () {
     const tsFile = 'tokens.svelte';
+    serviceWarmup(this, semanticTokenTestDir, pathToUrl(testDir));
 
     function setup(filename: string) {
         const docManager = new DocumentManager(
             (textDocument) => new Document(textDocument.uri, textDocument.text)
         );
-        const filePath = path.join(testDir, 'testfiles', 'semantic-tokens', filename);
+        const filePath = path.join(semanticTokenTestDir, filename);
         const lsAndTsDocResolver = new LSAndTSDocResolver(
             docManager,
             [pathToUrl(testDir)],
@@ -38,7 +41,7 @@ describe('SemanticTokensProvider', () => {
     }
 
     // TODO reenable with updated tokens for new transformation
-    it.skip('provides semantic token', async () => {
+    it('provides semantic token', async () => {
         const { provider, document } = setup(tsFile);
 
         const { data } = (await provider.getSemanticTokens(document)) ?? {
@@ -184,8 +187,15 @@ describe('SemanticTokensProvider', () => {
                 line: 11,
                 character: 25,
                 length: 'text'.length,
-                type: TokenType.parameter,
-                modifiers: [TokenModifier.declaration]
+                type: TokenType.variable,
+                modifiers: [TokenModifier.declaration, TokenModifier.local, TokenModifier.readonly]
+            },
+            {
+                line: 12,
+                character: 5,
+                length: 'Imported'.length,
+                type: TokenType.class,
+                modifiers: []
             },
             {
                 line: 12,
@@ -198,8 +208,8 @@ describe('SemanticTokensProvider', () => {
                 line: 12,
                 character: 43,
                 length: 'text'.length,
-                type: TokenType.parameter,
-                modifiers: []
+                type: TokenType.variable,
+                modifiers: [TokenModifier.local, TokenModifier.readonly]
             },
             {
                 line: 12,
@@ -212,15 +222,15 @@ describe('SemanticTokensProvider', () => {
                 line: 14,
                 character: 16,
                 length: 1,
-                type: TokenType.parameter,
-                modifiers: [TokenModifier.declaration]
+                type: TokenType.variable,
+                modifiers: [TokenModifier.declaration, TokenModifier.local]
             },
             {
                 line: 15,
                 character: 5,
                 length: 1,
-                type: TokenType.parameter,
-                modifiers: []
+                type: TokenType.variable,
+                modifiers: [TokenModifier.local]
             }
         ];
 

diff --git a/packages/language-server/test/plugins/typescript/features/SignatureHelpProvider.test.ts b/packages/language-server/test/plugins/typescript/features/SignatureHelpProvider.test.ts
@@ -12,15 +12,19 @@ import { SignatureHelpProviderImpl } from '../../../../src/plugins/typescript/fe
 import { LSAndTSDocResolver } from '../../../../src/plugins/typescript/LSAndTSDocResolver';
 import { pathToUrl } from '../../../../src/utils';
 import { LSConfigManager } from '../../../../src/ls-config';
+import { serviceWarmup } from '../test-utils';
 
 const testDir = path.join(__dirname, '..');
+const signatureHelpTestDir = path.join(testDir, 'testfiles', 'signature-help');
+
+describe('SignatureHelpProvider', function () {
+    serviceWarmup(this, signatureHelpTestDir, pathToUrl(testDir));
 
-describe('SignatureHelpProvider', () => {
     function setup() {
         const docManager = new DocumentManager(
             (textDocument) => new Document(textDocument.uri, textDocument.text)
         );
-        const filePath = path.join(testDir, 'testfiles', 'signature-help', 'signature-help.svelte');
+        const filePath = path.join(signatureHelpTestDir, 'signature-help.svelte');
         const lsAndTsDocResolver = new LSAndTSDocResolver(
             docManager,
             [pathToUrl(testDir)],

diff --git a/packages/language-server/test/plugins/typescript/features/TypeDefinitionProvider.test.ts b/packages/language-server/test/plugins/typescript/features/TypeDefinitionProvider.test.ts
@@ -7,12 +7,16 @@ import { LSConfigManager } from '../../../../src/ls-config';
 import { LSAndTSDocResolver } from '../../../../src/plugins';
 import { TypeDefinitionProviderImpl } from '../../../../src/plugins/typescript/features/TypeDefinitionProvider';
 import { pathToUrl } from '../../../../src/utils';
+import { serviceWarmup } from '../test-utils';
 
 const testDir = path.join(__dirname, '..');
+const typeDefinitionTestDir = path.join(testDir, 'testfiles', 'typedefinition');
+
+describe('TypeDefinitionProvider', function () {
+    serviceWarmup(this, typeDefinitionTestDir, pathToUrl(testDir));
 
-describe('TypeDefinitionProvider', () => {
     function getFullPath(filename: string) {
-        return path.join(testDir, 'testfiles', 'typedefinition', filename);
+        return path.join(typeDefinitionTestDir, filename);
     }
 
     function getUri(filename: string) {