testing: Add Theia Playwright framework (#10337)

Co-authored-by: Nina Doschek <ndoschek@eclipsesource.com>
Co-authored-by: Johannes Faltermeier <jfaltermeier@eclipsesource.com>
Co-authored-by: Martin Fleck <mfleck@eclipsesource.com>
Co-authored-by: Simon Graband <sgraband@eclipsesource.com>
Co-authored-by: Tobias Ortmayr <tortmayr@eclipsesource.com>
Co-authored-by: Vincent Fugnitto <vincent.fugnitto@ericsson.com>

Fixes #10337

.vscode/launch.json

@@ -176,6 +176,21 @@
       "outFiles": [
         "${workspaceFolder}/../.js"
       ]
+    },
+    {
+      "name": "Run Playwright Test",
+      "type": "node",
+      "request": "launch",
+      "program": "${workspaceFolder}/examples/playwright/node_modules/.bin/playwright",
+      "args": [
+        "test",
+        "--debug",
+        "--config=./configs/playwright.debug.config.ts",
+        "${fileBasenameNoExtension}"
+      ],
+      "cwd": "${workspaceFolder}/examples/playwright",
+      "console": "integratedTerminal",
+      "internalConsoleOptions": "neverOpen"
     }
   ],
   "compounds": [

dev-packages/application-manager/package.json

@@ -50,6 +50,7 @@
     "node-abi": "*",
     "semver": "^7.3.5",
     "setimmediate": "^1.0.5",
+    "source-map": "^0.6.1",
     "source-map-loader": "^2.0.1",
     "source-map-support": "^0.5.19",
     "style-loader": "^2.0.0",

examples/playwright/.eslintrc.js

@@ -0,0 +1,12 @@
+/** @type {import('eslint').Linter.Config} */
+module.exports = {
+    extends: [
+        '../../configs/build.eslintrc.json',
+        './configs/ui-tests.eslintrc.json',
+        './configs/ui-tests.playwright.eslintrc.json'
+    ],
+    parserOptions: {
+        tsconfigRootDir: __dirname,
+        project: 'tsconfig.json'
+    }
+};

examples/playwright/.gitignore

@@ -0,0 +1,2 @@
+allure-results
+test-results

examples/playwright/README.md

@@ -0,0 +1,53 @@
+<div align='center'>
+
+<br />
+
+<img src='https://raw.githubusercontent.com/eclipse-theia/theia/master/logo/theia.svg?sanitize=true' alt='theia-ext-logo' width='100px' />
+
+<h2>ECLIPSE THEIA - PLAYWRIGHT</h2>
+
+<hr />
+
+</div>
+
+## Description
+
+Theia 🎭 Playwright is a [page object](https://martinfowler.com/bliki/PageObject.html) framework based on [Playwright](https://github.com/microsoft/playwright) for developing system tests of [Theia](https://github.com/eclipse-theia/theia)-based applications. See it in action below.
+
+<div style='margin:0 auto;width:70%;'>
+
+![Theia System Testing in Action](./docs/images/teaser.gif)
+
+</div>
+
+The Theia 🎭 Playwright page objects introduce abstraction over Theia's user interfaces, encapsulating the details of the user interface interactions, wait conditions, etc., to help keeping your tests more concise, maintainable, and stable.
+Ready for an [example](./docs/GETTING_STARTED.md)?
+
+The actual interaction with the Theia application is implemented with 🎭 Playwright in Typescript. Thus, we can take advantage of [Playwright's benefits](https://playwright.dev/docs/why-playwright/) and run or debug tests headless or headful across all modern browsers.
+Check out [Playwright's documentation](https://playwright.dev/docs/intro) for more information.
+
+This page object framework not only covers Theia's generic capabilities, such as handling views, the quick command palette, file explorer etc.
+It is [extensible](./docs/EXTENSIBILITY.md) so you can add dedicated page objects for custom Theia components, such as custom views, editors, menus, etc.
+
+## Documentation
+
+- [Getting Started](./docs/GETTING_STARTED.md)
+- [Extensibility](./docs/EXTENSIBILITY.md)
+- [Building and Developing Theia 🎭 Playwright](./docs/DEVELOPING.md)
+
+## Additional Information
+
+- [Theia - GitHub](https://github.com/eclipse-theia/theia)
+- [Theia - Website](https://theia-ide.org/)
+- [Playwright - GitHub](https://github.com/microsoft/playwright)
+- [Playwright - Website](https://playwright.dev)
+
+## License
+
+- [Eclipse Public License 2.0](http://www.eclipse.org/legal/epl-2.0/)
+- [一 (Secondary) GNU General Public License, version 2 with the GNU Classpath Exception](https://projects.eclipse.org/license/secondary-gpl-2.0-cp)
+
+## Trademark
+
+"Theia" is a trademark of the Eclipse Foundation
+https://www.eclipse.org/theia

examples/playwright/configs/playwright.config.ts

@@ -0,0 +1,42 @@
+/********************************************************************************
+ * Copyright (C) 2021 logi.cals GmbH, EclipseSource and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/
+
+import { PlaywrightTestConfig } from '@playwright/test';
+
+const config: PlaywrightTestConfig = {
+    testDir: '../lib/tests',
+    testMatch: ['**/*.js'],
+    workers: 2,
+    // Timeout for each test in milliseconds.
+    timeout: 60 * 1000,
+    use: {
+        baseURL: 'http://localhost:3000',
+        browserName: 'chromium',
+        screenshot: 'only-on-failure',
+        viewport: { width: 1920, height: 1080 }
+    },
+    snapshotDir: '../tests/snapshots',
+    expect: {
+        toMatchSnapshot: { threshold: 0.15 }
+    },
+    preserveOutput: 'failures-only',
+    reporter: [
+        ['list'],
+        ['allure-playwright']
+    ]
+};
+
+export default config;

examples/playwright/configs/playwright.debug.config.ts

@@ -0,0 +1,27 @@
+/********************************************************************************
+ * Copyright (C) 2021 logi.cals GmbH, EclipseSource and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/
+
+import { PlaywrightTestConfig } from '@playwright/test';
+
+import baseConfig from './playwright.config';
+
+const debugConfig: PlaywrightTestConfig = {
+    ...baseConfig,
+    workers: 1,
+    timeout: 15000000
+};
+
+export default debugConfig;

examples/playwright/configs/playwright.headful.config.ts

@@ -0,0 +1,30 @@
+/********************************************************************************
+ * Copyright (C) 2021 logi.cals GmbH, EclipseSource and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/
+
+import { PlaywrightTestConfig } from '@playwright/test';
+
+import baseConfig from './playwright.config';
+
+const headfulConfig: PlaywrightTestConfig = {
+    ...baseConfig,
+    workers: 1,
+    use: {
+        ...baseConfig.use,
+        headless: false
+    }
+};
+
+export default headfulConfig;

examples/playwright/configs/ui-tests.eslintrc.json

@@ -0,0 +1,7 @@
+{
+  // override existing rules for ui-tests package
+  "rules": {
+    "no-undef": "off", // disabled due to 'browser', '$', '$$'
+    "no-unused-expressions": "off"
+  }
+}

examples/playwright/configs/ui-tests.playwright.eslintrc.json

@@ -0,0 +1,6 @@
+{
+  // override existing rules for ui-tests playwright package
+  "rules": {
+    "no-null/no-null": "off"
+  }
+}

examples/playwright/docs/DEVELOPING.md

@@ -0,0 +1,26 @@
+# Building and developing Theia 🎭 Playwright
+
+## Building
+
+Run `yarn` in the root directory of the repository.
+
+## Executing the tests
+
+### Prerequisites
+
+To work with the tests the Theia Application under test needs to be running.
+
+Run `yarn browser start` to start the browser-app located in this repository.
+
+You may also use the `Launch Browser Backend` launch configuration in VS Code.
+
+### Running the tests headless
+
+To start the tests run `yarn ui-tests` in the root of this package. This will start the tests located in `tests` in a headless mode.
+
+To only run a single test file, the path of a test file can be set with `yarn ui-tests <path-to-file>` or `yarn ui-tests -g "<partial test file name>"`.
+See the [Playwright Test command line documentation](https://playwright.dev/docs/intro#command-line).
+
+### Debugging the tests
+
+Please refer to the section [debugging tests](./GETTING_STARTED.md#debugging-the-tests).

examples/playwright/docs/EXTENSIBILITY.md

@@ -0,0 +1,103 @@
+# Extensibility
+
+Theia is an extensible tool platform for building custom tools with custom user interface elements, such as views, editors, commands, etc.
+Correspondingly, Theia 🎭 Playwright supports adding dedicated page objects for your custom user interface elements.
+Depending on the nature of your custom components, you can extend the generic base objects, such as for views or editors, or add your own from scratch.
+
+## Custom commands or menu items
+
+Commands and menu items are handled by their label, so no further customization of the page object framework is required.
+Simply interact with them via the menu or quick commands.
+
+```typescript
+const app = await TheiaApp.load(page);
+const menuBar = app.menuBar;
+
+const yourMenu = await menuBar.openMenu("Your Menu");
+const yourItem = await mainMenu.menuItemByName("Your Item");
+
+expect(await yourItem?.hasSubmenu()).toBe(true);
+```
+
+## Custom Theia applications
+
+The main entry point of the page object model is `TheiaApp`.
+To add further capabilities to it, for instance a custom toolbar, extend the `TheiaApp` class and add an accessor for a custom toolbar page object.
+
+```typescript
+export class MyTheiaApp extends TheiaApp {
+  readonly toolbar = new MyToolbar(this);
+}
+
+export class MyToolbar extends TheiaPageObject {
+  selector = "div#myToolbar";
+  async clickItem1(): Promise<void> {
+    await this.page.click(`${this.selector} .item1`);
+  }
+}
+
+const ws = new TheiaWorkspace(["tests/resources/sample-files1"]);
+const app = await MyTheiaApp.load(page, ws);
+await app.toolbar.clickItem1();
+```
+
+## Custom views and status indicators
+
+Many custom Theia applications add dedicated views, editors, or status indicators.
+To support these custom user interface elements in the testing framework, you can add dedicated page objects for them.
+Typically these dedicated page objects for your custom user interface elements are subclasses of the generic classes, `TheiaView`, `TheiaEditor`, etc.
+Consequently, they inherit the generic behavior of views or editors, such as activating or closing them, querying the title, check whether editors are dirty, etc.
+
+Let's take a custom view as an example. This custom view has a button that we want to be able to click.
+
+```typescript
+export class MyView extends TheiaView {
+  constructor(public app: TheiaApp) {
+    super(
+      {
+        tabSelector: "#shell-tab-my-view", // the id of the tab
+        viewSelector: "#my-view-container", // the id of the view container
+        viewName: "My View", // the user visible view name
+      },
+      app
+    );
+  }
+
+  async clickMyButton(): Promise<void> {
+    await this.activate();
+    const viewElement = await this.viewElement();
+    const button = await viewElement?.waitForSelector("#idOfMyButton");
+    await button?.click();
+  }
+}
+```
+
+So first, we create a new class that inherits all generic view capabilities from `TheiaView`.
+We have to specify the selectors for the tab and for the view container element that we specify in the view implementation.
+Optionally we can specify a view name, which corresponds to the label in Theia's view menu.
+This information is enough to open, close, find and interact with the view.
+
+Additionally we can add further custom methods for the specific actions and queries we want to use for our custom view.
+As an example, `MyView` above introduces a method that allows to click a button.
+
+To use this custom page object in a test, we pass our custom page object as a parameter when opening the view with `app.openView`.
+
+```typescript
+const app = await TheiaApp.load(page, ws);
+const myView = await app.openView(MyView);
+await myView.clickMyButton();
+```
+
+A similar approach is used for custom editors. The only difference is that we extend `TheiaEditor` instead and pass our custom page object as an argument to `app.openEditor`.
+As a reference for custom views and editors, please refer to the existing page objects, such as `TheiaPreferenceView`, `TheiaTextEditor`, etc.
+
+Custom status indicators are supported with the same mechanism. They are accessed via `TheiaApp.statusBar`.
+
+```typescript
+const app = await TheiaApp.load(page);
+const problemIndicator = await app.statusBar.statusIndicator(
+  TheiaProblemIndicator
+);
+const numberOfProblems = await problemIndicator.numberOfProblems();
+expect(numberOfProblems).to.be(2);
+```