test: by default, don't generate code for runner integration (#905)

Closes #900

### Summary of Changes

* We now have only a few targeted generator tests for the runner
integration, instead of having this scattered across almost all tests.
* Use vitest's `toMatchFileSnapshot` to easily be able to update the
expected output after changes to the code generator. Run `vitest
--update` to do so.

.gitignore

@@ -12,6 +12,7 @@ coverage/
 dist/
 dist-ssr/
 generated/
+!**/tests/resources/generation/**/generated/
 lib/
 out/
 *.tsbuildinfo

docs/development/generation-testing.md

@@ -6,16 +6,14 @@ generation test.
 ## Adding a generation test
 
 1. Create a new **folder** (not just a file!) in the `tests/resources/generation` directory or any subdirectory. Give
-   the folder a descriptive name, since the folder name becomes part of the test name.
+   the folder a descriptive name, since the folder name becomes part of the test name. By default, the runner
+   integration is disabled. If you want to test the runner integration, place the folder in
+   `tests/resources/generation/runner integration` or any subdirectory instead.
 
     !!! tip "Skipping a test"
 
         If you want to skip a test, add the prefix `skip-` to the folder name.
 
-    !!! tip "Tests without runner integration"
-
-        If you want to create a test without runner integration (memoization and placeholder saving), put it in the `eject` folder or use `eject` as a prefix for a new top level folder.
-
 2. Add files with the extension `.sdstest`, `.sdspipe`, or `.sdsstub` **directly inside the folder**. All files in a
    folder will be loaded into the same workspace, so they can reference each other. Files in different folders are
    loaded into different workspaces, so they cannot reference each other. Generation will be triggered for all files in
@@ -27,7 +25,11 @@ generation test.
     ```ts
     // $TEST$ run_until
     ```
-5. Add another folder called `output` inside the folder that you created in step 1. Place folders and Python files
-   inside the `output` folder to specify the expected output of the program. The relative paths to the Python files and
-   the contents of the Python files will be compared to the actual generation output.
+5. Add another folder called `generated` inside the folder that you created in step 1. Place folders and Python files
+   inside the `generated` folder to specify the expected output of the program. The relative paths to the Python files
+   and the contents of the Python files will be compared to the actual generation output.
 6. Run the tests. The test runner will automatically pick up the new test.
+
+## Updating the expected output
+
+To quickly update the expected output after changes to the code generator, run `vitest` with the `--update` flag.

packages/safe-ds-lang/src/language/generation/safe-ds-python-generator.ts

@@ -1106,9 +1106,14 @@ export class SafeDsPythonGenerator {
     }
 
     private getExternalReferenceNeededImport(
-        expression: SdsExpression,
-        declaration: SdsDeclaration,
+        expression: SdsExpression | undefined,
+        declaration: SdsDeclaration | undefined,
     ): ImportData | undefined {
+        if (!expression || !declaration) {
+            /* c8 ignore next 2 */
+            return undefined;
+        }
+
         // Root Node is always a module.
         const currentModule = <SdsModule>findRootNode(expression);
         const targetModule = <SdsModule>findRootNode(declaration);

packages/safe-ds-lang/tests/language/generation/creator.ts

@@ -18,6 +18,7 @@ import { URI } from 'langium';
 const services = createSafeDsServices(NodeFileSystem).SafeDs;
 await services.shared.workspace.WorkspaceManager.initializeWorkspace([]);
 const rootResourceName = 'generation';
+const runnerIntegration = 'runner integration';
 
 export const createGenerationTests = async (): Promise<GenerationTest[]> => {
     const filesGroupedByParentDirectory = listTestSafeDsFilesGroupedByParentDirectory(rootResourceName);
@@ -27,9 +28,8 @@ export const createGenerationTests = async (): Promise<GenerationTest[]> => {
 };
 
 const createGenerationTest = async (parentDirectory: URI, inputUris: URI[]): Promise<GenerationTest> => {
-    const expectedOutputRoot = URI.file(path.join(parentDirectory.fsPath, 'output'));
-    const actualOutputRoot = URI.file(path.join(parentDirectory.fsPath, 'generated'));
-    const expectedOutputFiles = readExpectedOutputFiles(expectedOutputRoot, actualOutputRoot);
+    const outputRoot = URI.file(path.join(parentDirectory.fsPath, 'generated'));
+    const expectedOutputFiles = readExpectedOutputFiles(outputRoot);
     let runUntil: Location | undefined;
 
     // Load all files, so they get linked
@@ -60,7 +60,7 @@ const createGenerationTest = async (parentDirectory: URI, inputUris: URI[]): Pro
         if (checksResult.value.length === 1) {
             const check = checksResult.value[0]!;
 
-            // Expected unresolved reference
+            // Partial execution
             if (check.comment !== 'run_until') {
                 return invalidTest('FILE', new InvalidCommentError(check.comment, uri));
             }
@@ -79,23 +79,22 @@ const createGenerationTest = async (parentDirectory: URI, inputUris: URI[]): Pro
     return {
         testName: `[${shortenedResourceName}]`,
         inputUris,
-        actualOutputRoot,
+        outputRoot,
         expectedOutputFiles,
         runUntil,
-        disableRunnerIntegration: shortenedResourceName.startsWith('eject'), // Tests in the "eject" top level folder are tested with disabled runner integration
+        disableRunnerIntegration: !shortenedResourceName.startsWith(runnerIntegration),
     };
 };
 
 /**
  * Read all expected output files.
  *
- * @param expectedOutputRoot Where the expected output files are located.
- * @param actualOutputRoot Where the actual output files supposed to be located.
+ * @param outputRoot Where the actual output files are supposed to be located.
  */
-const readExpectedOutputFiles = (expectedOutputRoot: URI, actualOutputRoot: URI): ExpectedOutputFile[] => {
-    return listTestFilesWithExtensions(uriToShortenedTestResourceName(expectedOutputRoot), ['py', 'map'])
+const readExpectedOutputFiles = (outputRoot: URI): ExpectedOutputFile[] => {
+    return listTestFilesWithExtensions(uriToShortenedTestResourceName(outputRoot), ['py', 'map'])
         .sort((a, b) => {
-            // List .py files first
+            // List .py files first, so they get compared first
             if (a.fsPath.endsWith('.map') && b.fsPath.endsWith('.py')) {
                 return 1;
             }
@@ -104,7 +103,7 @@ const readExpectedOutputFiles = (expectedOutputRoot: URI, actualOutputRoot: URI)
         })
         .map((uri) => {
             return {
-                uri: URI.file(path.join(actualOutputRoot.fsPath, path.relative(expectedOutputRoot.fsPath, uri.fsPath))),
+                uri,
                 code: fs.readFileSync(uri.fsPath).toString(),
             };
         });
@@ -122,7 +121,7 @@ const invalidTest = (level: 'FILE' | 'SUITE', error: TestDescriptionError): Gene
     return {
         testName,
         inputUris: [],
-        actualOutputRoot: URI.file(''),
+        outputRoot: URI.file(''),
         expectedOutputFiles: [],
         error,
         disableRunnerIntegration: false,
@@ -139,9 +138,9 @@ interface GenerationTest extends TestDescription {
     inputUris: URI[];
 
     /**
-     * The directory, where actual output files should be temporarily stored.
+     * The directory, where output files are located.
      */
-    actualOutputRoot: URI;
+    outputRoot: URI;
 
     /**
      * The expected generated code.

packages/safe-ds-lang/tests/language/generation/safe-ds-python-generator.test.ts

@@ -21,7 +21,6 @@ describe('generation', async () => {
 
         // Get target placeholder name for "run until"
         let runUntilPlaceholderName: string | undefined = undefined;
-
         if (test.runUntil) {
             const document = services.shared.workspace.LangiumDocuments.getOrCreateDocument(
                 URI.parse(test.runUntil.uri),
@@ -30,30 +29,30 @@ describe('generation', async () => {
         }
 
         // Generate code for all documents
-        const actualOutputs = stream(documents)
+        const actualOutputs: Map<string, string> = stream(documents)
             .flatMap((document) =>
                 pythonGenerator.generate(document, {
-                    destination: test.actualOutputRoot,
+                    destination: test.outputRoot,
                     createSourceMaps: true,
                     targetPlaceholder: runUntilPlaceholderName,
                     disableRunnerIntegration: test.disableRunnerIntegration,
                 }),
             )
             .map((textDocument) => [textDocument.uri, textDocument.getText()])
             .toMap(
-                (entry) => entry[0],
-                (entry) => entry[1],
+                (entry) => entry[0]!,
+                (entry) => entry[1]!,
             );
 
+        // File contents must match
+        for (const [uriString, code] of actualOutputs) {
+            const fsPath = URI.parse(uriString).fsPath;
+            await expect(code).toMatchFileSnapshot(fsPath);
+        }
+
         // File paths must match
         const actualOutputPaths = Array.from(actualOutputs.keys()).sort();
         const expectedOutputPaths = test.expectedOutputFiles.map((file) => file.uri.toString()).sort();
         expect(actualOutputPaths).toStrictEqual(expectedOutputPaths);
-
-        // File contents must match
-        for (const expectedOutputFile of test.expectedOutputFiles) {
-            const actualCode = actualOutputs.get(expectedOutputFile.uri.toString());
-            expect(actualCode).toBe(expectedOutputFile.code);
-        }
     });
 });

packages/safe-ds-lang/tests/resources/generation/expressions/block lambda result/generated/tests/generator/blockLambdaResult/gen_input.py

@@ -0,0 +1,21 @@
+# Segments ---------------------------------------------------------------------
+
+def f1(l):
+    h(l(1, 2))
+
+def f2(l):
+    h(l(1, 2)[1])
+    h(l(1, 2)[0])
+
+# Pipelines --------------------------------------------------------------------
+
+def test():
+    def __gen_block_lambda_0(a, b):
+        __gen_block_lambda_result_d = g()
+        return __gen_block_lambda_result_d
+    f1(__gen_block_lambda_0)
+    def __gen_block_lambda_1(a, b):
+        __gen_block_lambda_result_d = g()
+        __gen_block_lambda_result_e = g()
+        return __gen_block_lambda_result_d, __gen_block_lambda_result_e
+    f2(__gen_block_lambda_1)

packages/safe-ds-lang/tests/resources/generation/expressions/block lambda/generated/tests/generator/blockLambda/gen_input.py

@@ -0,0 +1,23 @@
+# Pipelines --------------------------------------------------------------------
+
+def test():
+    def __gen_block_lambda_0(a, b=2):
+        __gen_block_lambda_result_d = g()
+        return __gen_block_lambda_result_d
+    f1(__gen_block_lambda_0)
+    def __gen_block_lambda_1(a, b):
+        __gen_block_lambda_result_d = g()
+        return __gen_block_lambda_result_d
+    f1(__gen_block_lambda_1)
+    def __gen_block_lambda_2():
+        pass
+    f2(__gen_block_lambda_2)
+    def __gen_block_lambda_3(a, b=2):
+        __gen_block_lambda_result_d = g()
+        return __gen_block_lambda_result_d
+    g2(f3(__gen_block_lambda_3))
+    def __gen_block_lambda_4(a, b=2):
+        __gen_block_lambda_result_d = g()
+        return __gen_block_lambda_result_d
+    c = f3(__gen_block_lambda_4)
+    g2(c)

packages/safe-ds-lang/tests/resources/generation/expressions/call/generated/tests/generator/call/gen_input.py

@@ -0,0 +1,32 @@
+# Imports ----------------------------------------------------------------------
+
+from typing import Any, Callable, TypeVar
+
+# Type variables ---------------------------------------------------------------
+
+__gen_T = TypeVar("__gen_T")
+
+# Utils ------------------------------------------------------------------------
+
+def __gen_null_safe_call(receiver: Any, callable: Callable[[], __gen_T]) -> __gen_T | None:
+    return callable() if receiver is not None else None
+
+# Pipelines --------------------------------------------------------------------
+
+def test():
+    f(g(1, param2=2))
+    f(g(2, param2=1))
+    f(h(1, param_2=2))
+    f(h(2, param_2=1))
+    f(h(2))
+    'abc'.i()
+    'abc'.j(123)
+    k(456, 1.23)
+    __gen_null_safe_call(f, lambda: f(g(1, param2=2)))
+    __gen_null_safe_call(f, lambda: f(g(2, param2=1)))
+    __gen_null_safe_call(f, lambda: f(h(1, param_2=2)))
+    __gen_null_safe_call(f, lambda: f(h(2, param_2=1)))
+    __gen_null_safe_call(f, lambda: f(h(2)))
+    __gen_null_safe_call(i, lambda: 'abc'.i())
+    __gen_null_safe_call(j, lambda: 'abc'.j(123))
+    __gen_null_safe_call(k, lambda: k(456, 1.23))