From 7866be21056b6e3ee843d0421c372d847be8f7ff Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Thu, 27 Jun 2019 11:02:47 -0700 Subject: [PATCH 01/17] WIP: Adding libs for webgl --- x-pack/build_chromium/build.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/build_chromium/build.py b/x-pack/build_chromium/build.py index 190a8dfc92d4a7..c0af409c6e28c7 100644 --- a/x-pack/build_chromium/build.py +++ b/x-pack/build_chromium/build.py @@ -73,12 +73,16 @@ def archive_file(name): # must be bundled with the Chromium executable. if platform.system() == 'Linux': archive_file('headless_shell') + archive_file('libEGL.so') + archive_file('libGLESv2.so') elif platform.system() == 'Windows': archive_file('headless_shell.exe') archive_file('dbghelp.dll') archive_file('icudtl.dat') elif platform.system() == 'Darwin': archive_file('headless_shell') + archive_file('libswiftshader_libEGL.dylib') + archive_file('libswiftshader_libGLESv2.dylib') archive_file('Helpers/chrome_crashpad_handler') archive.close() From 83a9d4df69899883e850fbf4f98de640f84c9781 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Fri, 28 Jun 2019 12:06:13 -0700 Subject: [PATCH 02/17] WIP Adding swiftshader libs to chromium --- x-pack/build_chromium/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/build_chromium/README.md b/x-pack/build_chromium/README.md index 0c147d003c92eb..1446aeb15e607d 100644 --- a/x-pack/build_chromium/README.md +++ b/x-pack/build_chromium/README.md @@ -90,9 +90,9 @@ Find the Chromium revision (modify the following command to be wherever you have - `cat ~/dev/elastic/kibana/x-pack/node_modules/puppeteer-core/package.json | grep chromium_revision` - Take the revision number from that, and tack it to the end of this URL: https://crrev.com - - (For example: https://crrev.com/637110) + - (For example: https://crrev.com/672088) - Grab the SHA from there - - (For example, rev 637110 has sha 2fac04abf6133ab2da2846a8fbd0e97690722699) + - (For example, rev 672088 has sha 21a0edd1ded0244cd04af082b5c7945655759ae0) Note: In Linux, you should run the build command in tmux so that if your ssh session disconnects, the build can keep going. To do this, just type `tmux` into your terminal to hop into a tmux session. If you get disconnected, you can hop back in like so: @@ -102,9 +102,9 @@ Note: In Linux, you should run the build command in tmux so that if your ssh ses To run the build, replace the sha in the following commands with the sha that you wish to build: -- Mac: `python ~/chromium/build_chromium/build.py 2fac04abf6133ab2da2846a8fbd0e97690722699` -- Linux: `python ~/chromium/build_chromium/build.py 2fac04abf6133ab2da2846a8fbd0e97690722699` -- Windows: `python c:\chromium\build_chromium\build.py 2fac04abf6133ab2da2846a8fbd0e97690722699` +- Mac: `python ~/chromium/build_chromium/build.py 21a0edd1ded0244cd04af082b5c7945655759ae0` +- Linux: `python ~/chromium/build_chromium/build.py 21a0edd1ded0244cd04af082b5c7945655759ae0` +- Windows: `python c:\chromium\build_chromium\build.py 21a0edd1ded0244cd04af082b5c7945655759ae0` ## Artifacts From 3e68905275863ff8d17d01076f807e1e71171032 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Mon, 1 Jul 2019 08:26:27 -0700 Subject: [PATCH 03/17] WIP: Adding missing binaries for webgl in chromium --- x-pack/build_chromium/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/build_chromium/build.py b/x-pack/build_chromium/build.py index c0af409c6e28c7..880a2ede7d13b2 100644 --- a/x-pack/build_chromium/build.py +++ b/x-pack/build_chromium/build.py @@ -73,8 +73,8 @@ def archive_file(name): # must be bundled with the Chromium executable. if platform.system() == 'Linux': archive_file('headless_shell') - archive_file('libEGL.so') - archive_file('libGLESv2.so') + archive_file('swiftshader/libEGL.so') + archive_file('swiftshader/libGLESv2.so') elif platform.system() == 'Windows': archive_file('headless_shell.exe') archive_file('dbghelp.dll') From 87d15282f2d3e60aa8df708e13515d9b9817300c Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Fri, 26 Jul 2019 15:18:19 -0700 Subject: [PATCH 04/17] Use pipes for communication with chrome to avoid networking snafus --- .../reporting/server/browsers/chromium/driver_factory/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts index a08b775d742a34..792d4a4e84c009 100644 --- a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts +++ b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts @@ -102,6 +102,7 @@ export class HeadlessChromiumDriverFactory { let page: puppeteer.Page; try { browser = await puppeteer.launch({ + pipe: true, userDataDir, executablePath: this.binaryPath, ignoreHTTPSErrors: true, From 33f2a0b169e0c3acb8b2b6a7f59411e0781183cc Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Tue, 6 Aug 2019 10:17:38 -0700 Subject: [PATCH 05/17] Bumps puppeteer in prep for new chromium build + types and better @types package --- .../browsers/chromium/driver_factory/index.ts | 41 +++++++++---------- .../chromium/driver_factory/puppeteer.ts | 16 ++++++++ x-pack/package.json | 6 +-- yarn.lock | 17 +++++--- 4 files changed, 50 insertions(+), 30 deletions(-) create mode 100644 x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/puppeteer.ts diff --git a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts index 49efb077dfc58b..acc0003978293e 100644 --- a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts +++ b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts @@ -6,13 +6,12 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; -// @ts-ignore -import puppeteer from 'puppeteer-core'; import rimraf from 'rimraf'; import * as Rx from 'rxjs'; import { map, share, mergeMap, filter, partition } from 'rxjs/operators'; import { InnerSubscriber } from 'rxjs/internal/InnerSubscriber'; +import { launch, Browser, Page } from './puppeteer'; import { HeadlessChromiumDriver } from '../driver'; import { args, IArgOptions } from './args'; import { safeChildProcess } from '../../safe_child_process'; @@ -62,23 +61,21 @@ export class HeadlessChromiumDriverFactory { proxyConfig: this.browserConfig.proxy, }); - return puppeteer - .launch({ - userDataDir, - executablePath: this.binaryPath, - ignoreHTTPSErrors: true, - args: chromiumArgs, - env: { - TZ: browserTimezone, - }, - }) - .catch((error: Error) => { - logger.warning( - `The Reporting plugin encountered issues launching Chromium in a self-test. You may have trouble generating reports: [${error}]` - ); - logger.warning(`See Chromium's log output at "${getChromeLogLocation(this.binaryPath)}"`); - return null; - }); + return launch({ + userDataDir, + executablePath: this.binaryPath, + ignoreHTTPSErrors: true, + args: chromiumArgs, + env: { + TZ: browserTimezone, + }, + }).catch((error: Error) => { + logger.warning( + `The Reporting plugin encountered issues launching Chromium in a self-test. You may have trouble generating reports: [${error}]` + ); + logger.warning(`See Chromium's log output at "${getChromeLogLocation(this.binaryPath)}"`); + return null; + }); } create({ @@ -103,10 +100,10 @@ export class HeadlessChromiumDriverFactory { proxyConfig: this.browserConfig.proxy, }); - let browser: puppeteer.Browser; - let page: puppeteer.Page; + let browser: Browser; + let page: Page; try { - browser = await puppeteer.launch({ + browser = await launch({ pipe: true, userDataDir, executablePath: this.binaryPath, diff --git a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/puppeteer.ts b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/puppeteer.ts new file mode 100644 index 00000000000000..47b9cb1ff7aced --- /dev/null +++ b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/puppeteer.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import puppeteer from 'puppeteer'; +// @ts-ignore lacking typedefs +import puppeteerCore from 'puppeteer-core'; + +// We export a set of types and other methods since we use puppeteer-core, which has an outdated @types package. +// However, @types/puppeteer _is_ up-to-date, and this module merges them together. +export const launch: (opts?: puppeteer.LaunchOptions) => Promise = + puppeteerCore.launch; +export type Browser = puppeteer.Browser; +export type Page = puppeteer.Page; diff --git a/x-pack/package.json b/x-pack/package.json index ebd35d96a91d98..8d9ca9f8792509 100644 --- a/x-pack/package.json +++ b/x-pack/package.json @@ -58,6 +58,7 @@ "@types/git-url-parse": "^9.0.0", "@types/glob": "^7.1.1", "@types/graphql": "^0.13.1", + "@types/hapi__wreck": "^15.0.1", "@types/history": "^4.6.2", "@types/jest": "^24.0.9", "@types/joi": "^13.4.2", @@ -78,7 +79,7 @@ "@types/pngjs": "^3.3.1", "@types/prop-types": "^15.5.3", "@types/proper-lockfile": "^3.0.0", - "@types/puppeteer": "^1.12.4", + "@types/puppeteer": "^1.19.0", "@types/react": "^16.8.0", "@types/react-dom": "^16.8.0", "@types/react-redux": "^6.0.6", @@ -99,7 +100,6 @@ "@types/tar-fs": "^1.16.1", "@types/tinycolor2": "^1.4.1", "@types/uuid": "^3.4.4", - "@types/hapi__wreck": "^15.0.1", "abab": "^1.0.4", "ansi-colors": "^3.0.5", "ansicolors": "0.3.2", @@ -297,7 +297,7 @@ "prop-types": "^15.6.0", "proper-lockfile": "^3.0.2", "puid": "1.0.7", - "puppeteer-core": "^1.13.0", + "puppeteer-core": "^1.19.0", "raw-loader": "3.0.0", "react": "^16.8.0", "react-apollo": "^2.1.4", diff --git a/yarn.lock b/yarn.lock index 49052c3d47a4b3..c8a1550d6b875d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4266,7 +4266,14 @@ resolved "https://registry.yarnpkg.com/@types/proper-lockfile/-/proper-lockfile-3.0.0.tgz#dcc7cc3714857a4ae6583331d2687e89dc5c94d2" integrity sha512-+tfnsA3KNPDm7Sj9x5omRgvS6ALc+edjTZXYeR3kVEm+qmsrF+59yJUWZDreV/O0+EQ6t0YSWlzxfdV58UOEVg== -"@types/puppeteer@^1.12.4", "@types/puppeteer@^1.6.0": +"@types/puppeteer@^1.19.0": + version "1.19.0" + resolved "https://registry.yarnpkg.com/@types/puppeteer/-/puppeteer-1.19.0.tgz#59f0050bae019cee7c3af2bb840a25892a3078b6" + integrity sha512-Db9LWOuTm2bR/qgPE7PQCmnsCQ6flHdULuIDWTks8YdQ/SGHKg5WGWG54gl0734NDKCTF5MbqAp2qWuvBiyQ3Q== + dependencies: + "@types/node" "*" + +"@types/puppeteer@^1.6.0": version "1.12.4" resolved "https://registry.yarnpkg.com/@types/puppeteer/-/puppeteer-1.12.4.tgz#8388efdb0b30a54a7e7c4831ca0d709191d77ff1" integrity sha512-aaGbJaJ9TuF9vZfTeoh876sBa+rYJWPwtsmHmYr28pGr42ewJnkDTq2aeSKEmS39SqUdkwLj73y/d7rBSp7mDQ== @@ -22377,10 +22384,10 @@ punycode@^1.2.4, punycode@^1.4.1: resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" integrity sha1-wNWmOycYgArY4esPpSachN1BhF4= -puppeteer-core@^1.13.0: - version "1.13.0" - resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-1.13.0.tgz#f8001851e924e6e9ef6e9fae1778c3ab87c3f307" - integrity sha512-8MypjWVHu2EEdtN2HxhCsTtIYdJgiCcbGpHoosv265fzanfOICC2/DadLZq6/Qc/OKsovQmjkO+2vKMrV3BRfA== +puppeteer-core@^1.19.0: + version "1.19.0" + resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-1.19.0.tgz#3c3f98edb5862583e3a9c19cbc0da57ccc63ba5c" + integrity sha512-ZPbbjUymorIJomHBvdZX5+2gciUmQtAdepCrkweHH6rMJr96xd/dXzHgmYEOBMatH44SmJrcMtWkgsLHJqT89g== dependencies: debug "^4.1.0" extract-zip "^1.6.6" From ac9088adc4c9a3b4031756655df7329b74f0cc52 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Tue, 6 Aug 2019 10:18:33 -0700 Subject: [PATCH 06/17] Remove ignore --- .../reporting/server/browsers/chromium/driver_factory/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts index acc0003978293e..e24c0e679efbf1 100644 --- a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts +++ b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts @@ -120,7 +120,6 @@ export class HeadlessChromiumDriverFactory { // which can cause the job to fail even if we bump timeouts in // the config. Help alleviate errors like // "TimeoutError: waiting for selector ".application" failed: timeout 30000ms exceeded" - // @ts-ignore outdated typedefs for puppteer page.setDefaultTimeout(this.queueTimeout); } catch (err) { observer.error(new Error(`Error spawning Chromium browser: [${err}]`)); From 26deffabf89810967d4e36b9fd8f3e305b8c61ff Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Tue, 6 Aug 2019 10:23:31 -0700 Subject: [PATCH 07/17] Removing of final @ts-ignore now that we have types --- .../server/browsers/chromium/driver/chromium_driver.ts | 8 ++++---- .../server/browsers/chromium/driver_factory/index.ts | 2 +- .../browsers/chromium/{driver_factory => }/puppeteer.ts | 2 +- .../reporting/server/lib/validate/validate_browser.ts | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) rename x-pack/legacy/plugins/reporting/server/browsers/chromium/{driver_factory => }/puppeteer.ts (92%) diff --git a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts index 5c65e500b244c1..d60f32cca586e6 100644 --- a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts +++ b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts @@ -5,9 +5,8 @@ */ import open from 'opn'; -// @ts-ignore -import * as puppeteer from 'puppeteer-core'; import { parse as parseUrl } from 'url'; + import { ConditionalHeaders, ConditionalHeadersConditions, @@ -18,6 +17,7 @@ import { Logger, ViewZoomWidthHeight, } from '../../../../types'; +import { Page } from '../puppeteer'; export interface ChromiumDriverOptions { logger: Logger; @@ -31,11 +31,11 @@ interface WaitForSelectorOpts { const WAIT_FOR_DELAY_MS: number = 100; export class HeadlessChromiumDriver { - private readonly page: puppeteer.Page; + private readonly page: Page; private readonly logger: Logger; private readonly inspect: boolean; - constructor(page: puppeteer.Page, { logger, inspect }: ChromiumDriverOptions) { + constructor(page: Page, { logger, inspect }: ChromiumDriverOptions) { this.page = page; // @ts-ignore https://github.com/elastic/kibana/issues/32140 this.logger = logger.clone(['headless-chromium-driver']); diff --git a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts index e24c0e679efbf1..fac59199dd0968 100644 --- a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts +++ b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts @@ -11,7 +11,7 @@ import * as Rx from 'rxjs'; import { map, share, mergeMap, filter, partition } from 'rxjs/operators'; import { InnerSubscriber } from 'rxjs/internal/InnerSubscriber'; -import { launch, Browser, Page } from './puppeteer'; +import { launch, Browser, Page } from '../puppeteer'; import { HeadlessChromiumDriver } from '../driver'; import { args, IArgOptions } from './args'; import { safeChildProcess } from '../../safe_child_process'; diff --git a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/puppeteer.ts b/x-pack/legacy/plugins/reporting/server/browsers/chromium/puppeteer.ts similarity index 92% rename from x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/puppeteer.ts rename to x-pack/legacy/plugins/reporting/server/browsers/chromium/puppeteer.ts index 47b9cb1ff7aced..099844db75a739 100644 --- a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/puppeteer.ts +++ b/x-pack/legacy/plugins/reporting/server/browsers/chromium/puppeteer.ts @@ -5,7 +5,7 @@ */ import puppeteer from 'puppeteer'; -// @ts-ignore lacking typedefs +// @ts-ignore lacking typedefs which this module fixes import puppeteerCore from 'puppeteer-core'; // We export a set of types and other methods since we use puppeteer-core, which has an outdated @types package. diff --git a/x-pack/legacy/plugins/reporting/server/lib/validate/validate_browser.ts b/x-pack/legacy/plugins/reporting/server/lib/validate/validate_browser.ts index 823aa44e9e12f2..39b98361f142f1 100644 --- a/x-pack/legacy/plugins/reporting/server/lib/validate/validate_browser.ts +++ b/x-pack/legacy/plugins/reporting/server/lib/validate/validate_browser.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ // @ts-ignore no module definition -import * as puppeteer from 'puppeteer-core'; +import { Browser } from '../../browsers/chromium/puppeteer'; import { KbnServer, Logger } from '../../../types'; import { CHROMIUM } from '../../browsers/browser_types'; @@ -21,7 +21,7 @@ export const validateBrowser = async (server: KbnServer, browserFactory: any, lo }, logger ) - .then((browser: puppeteer.Browser | null) => { + .then((browser: Browser | null) => { if (browser && browser.close) { browser.close(); } else { From 48c3134d1102e16b6bac2c65d8fc72e1da2a5f18 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Tue, 6 Aug 2019 13:25:30 -0700 Subject: [PATCH 08/17] README updates --- x-pack/build_chromium/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/build_chromium/README.md b/x-pack/build_chromium/README.md index 0c147d003c92eb..21ccfbd2c351da 100644 --- a/x-pack/build_chromium/README.md +++ b/x-pack/build_chromium/README.md @@ -86,13 +86,13 @@ In windows, at least, you will need to do a number of extra steps: Find the sha of the Chromium commit you wish to build. Most likely, you want to build the Chromium revision that is tied to the version of puppeteer that we're using. -Find the Chromium revision (modify the following command to be wherever you have the kibana source installed): +Find the Chromium revision (run in kibana's working directory): -- `cat ~/dev/elastic/kibana/x-pack/node_modules/puppeteer-core/package.json | grep chromium_revision` +- `cat node_modules/puppeteer-core/package.json | grep chromium_revision` - Take the revision number from that, and tack it to the end of this URL: https://crrev.com - (For example: https://crrev.com/637110) - Grab the SHA from there - - (For example, rev 637110 has sha 2fac04abf6133ab2da2846a8fbd0e97690722699) + - (For example, rev 637110 has sha 312d84c8ce62810976feda0d3457108a6dfff9e6) Note: In Linux, you should run the build command in tmux so that if your ssh session disconnects, the build can keep going. To do this, just type `tmux` into your terminal to hop into a tmux session. If you get disconnected, you can hop back in like so: @@ -102,9 +102,9 @@ Note: In Linux, you should run the build command in tmux so that if your ssh ses To run the build, replace the sha in the following commands with the sha that you wish to build: -- Mac: `python ~/chromium/build_chromium/build.py 2fac04abf6133ab2da2846a8fbd0e97690722699` -- Linux: `python ~/chromium/build_chromium/build.py 2fac04abf6133ab2da2846a8fbd0e97690722699` -- Windows: `python c:\chromium\build_chromium\build.py 2fac04abf6133ab2da2846a8fbd0e97690722699` +- Mac: `python ~/chromium/build_chromium/build.py 312d84c8ce62810976feda0d3457108a6dfff9e6` +- Linux: `python ~/chromium/build_chromium/build.py 312d84c8ce62810976feda0d3457108a6dfff9e6` +- Windows: `python c:\chromium\build_chromium\build.py 312d84c8ce62810976feda0d3457108a6dfff9e6` ## Artifacts From 91458dbd6664dbd0010b07c98076f5e40a57af64 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Fri, 9 Aug 2019 14:36:43 -0700 Subject: [PATCH 09/17] Fixing binding issues --- .../plugins/reporting/server/browsers/chromium/puppeteer.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/reporting/server/browsers/chromium/puppeteer.ts b/x-pack/legacy/plugins/reporting/server/browsers/chromium/puppeteer.ts index 099844db75a739..0612bc59f23181 100644 --- a/x-pack/legacy/plugins/reporting/server/browsers/chromium/puppeteer.ts +++ b/x-pack/legacy/plugins/reporting/server/browsers/chromium/puppeteer.ts @@ -10,7 +10,9 @@ import puppeteerCore from 'puppeteer-core'; // We export a set of types and other methods since we use puppeteer-core, which has an outdated @types package. // However, @types/puppeteer _is_ up-to-date, and this module merges them together. -export const launch: (opts?: puppeteer.LaunchOptions) => Promise = - puppeteerCore.launch; export type Browser = puppeteer.Browser; export type Page = puppeteer.Page; + +export const launch: ( + opts?: puppeteer.LaunchOptions +) => Promise = puppeteerCore.launch.bind(puppeteerCore); From 776148691a386de859256d3ce98c3fc55f44fb47 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Tue, 13 Aug 2019 08:44:26 -0700 Subject: [PATCH 10/17] Fixing maps integration wrt reporting + conditional pipes for puppeteer --- .../connected_components/gis_map/view.js | 20 ++++++++++++++++++- .../browsers/chromium/driver_factory/index.ts | 6 +++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/connected_components/gis_map/view.js b/x-pack/legacy/plugins/maps/public/connected_components/gis_map/view.js index 66010e2b2407bb..c119782e9e3e12 100644 --- a/x-pack/legacy/plugins/maps/public/connected_components/gis_map/view.js +++ b/x-pack/legacy/plugins/maps/public/connected_components/gis_map/view.js @@ -13,11 +13,15 @@ import { AddLayerPanel } from '../layer_addpanel/index'; import { EuiFlexGroup, EuiFlexItem, EuiCallOut } from '@elastic/eui'; import { ExitFullScreenButton } from 'ui/exit_full_screen'; import { i18n } from '@kbn/i18n'; +import uuid from 'uuid/v4'; + +const RENDER_COMPLETE_EVENT = 'renderComplete'; export class GisMap extends Component { state = { isInitialLoadRenderTimeoutComplete: false, + domId: uuid(), } componentDidMount() { @@ -39,6 +43,16 @@ export class GisMap extends Component { this._clearRefreshTimer(); } + // Need to get the right element until https://github.com/elastic/eui/issues/2220 + // is fixed, otherwise we can't use ref's to emit the custom event for reporting + _onInitialLoadRenderComplete = () => { + const el = document.querySelector(`[data-dom-id="${this.state.domId}"]`); + + if (el) { + el.dispatchEvent(new CustomEvent(RENDER_COMPLETE_EVENT, { bubbles: true })); + } + } + _setRefreshTimer = () => { const { isPaused, interval } = this.props.refreshConfig; @@ -75,9 +89,10 @@ export class GisMap extends Component { () => { if (this._isMounted) { this.setState({ isInitialLoadRenderTimeoutComplete: true }); + this._onInitialLoadRenderComplete(); } }, - 1000 + 5000 ); } @@ -92,6 +107,8 @@ export class GisMap extends Component { mapInitError, } = this.props; + const { domId } = this.state; + if (mapInitError) { return (
@@ -136,6 +153,7 @@ export class GisMap extends Component { diff --git a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts index a06c32adb04198..364477d3494ba0 100644 --- a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts +++ b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts @@ -102,9 +102,13 @@ export class HeadlessChromiumDriverFactory { let browser: Browser; let page: Page; + + // The inspector works over websockets, + // so we need to conditionally interact + // with chrome via pipes depending on this flag try { browser = await launch({ - pipe: true, + pipe: !this.browserConfig.inspect, userDataDir, executablePath: this.binaryPath, ignoreHTTPSErrors: true, From 8ffbbfca09f1a4fb5235821148d4ea31f56de12d Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Mon, 26 Aug 2019 07:59:43 -0700 Subject: [PATCH 11/17] Adding new deps to the windows build --- x-pack/build_chromium/build.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/x-pack/build_chromium/build.py b/x-pack/build_chromium/build.py index 880a2ede7d13b2..82b0561fdcfe1c 100644 --- a/x-pack/build_chromium/build.py +++ b/x-pack/build_chromium/build.py @@ -73,17 +73,23 @@ def archive_file(name): # must be bundled with the Chromium executable. if platform.system() == 'Linux': archive_file('headless_shell') - archive_file('swiftshader/libEGL.so') - archive_file('swiftshader/libGLESv2.so') + archive_file(os.path.join('swiftshader', 'libEGL.so')) + archive_file(os.path.join('swiftshader', 'libGLESv2.so')) + elif platform.system() == 'Windows': archive_file('headless_shell.exe') archive_file('dbghelp.dll') archive_file('icudtl.dat') + archive_file(os.path.join('swiftshader', 'libEGL.dll')) + archive_file(os.path.join('swiftshader', 'libEGL.dll.lib')) + archive_file(os.path.join('swiftshader', 'libGLESv2.dll')) + archive_file(os.path.join('swiftshader', 'libGLESv2.dll.lib')) + elif platform.system() == 'Darwin': archive_file('headless_shell') archive_file('libswiftshader_libEGL.dylib') archive_file('libswiftshader_libGLESv2.dylib') - archive_file('Helpers/chrome_crashpad_handler') + archive_file(path.join('Helpers', 'chrome_crashpad_handler')) archive.close() From 199b5ccdf252c9d449151c2bccd35545ad7c2828 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Mon, 26 Aug 2019 08:19:02 -0700 Subject: [PATCH 12/17] New s3 builds --- .../server/browsers/chromium/paths.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/legacy/plugins/reporting/server/browsers/chromium/paths.ts b/x-pack/legacy/plugins/reporting/server/browsers/chromium/paths.ts index 49e5228a96394d..cdd828faf326d6 100644 --- a/x-pack/legacy/plugins/reporting/server/browsers/chromium/paths.ts +++ b/x-pack/legacy/plugins/reporting/server/browsers/chromium/paths.ts @@ -12,23 +12,23 @@ export const paths = { packages: [ { platforms: ['darwin', 'freebsd', 'openbsd'], - archiveFilename: 'chromium-2fac04a-darwin.zip', - archiveChecksum: '36814b1629457aa178b4ecdf6cc1bc5f', - rawChecksum: '9b40e2efa7f4f1870835ee4cdaf1dd51', + archiveFilename: 'chromium-312d84c-darwin.zip', + archiveChecksum: '020303e829745fd332ae9b39442ce570', + rawChecksum: '101dfea297c5818a7a3f3317a99dde02', binaryRelativePath: 'headless_shell-darwin/headless_shell', }, { platforms: ['linux'], - archiveFilename: 'chromium-2fac04a-linux.zip', - archiveChecksum: '5cd6b898a35f9dc0ba6f49d821b8a2a3', - rawChecksum: 'b3fd218d3c3446c388da4e6c8a82754c', + archiveFilename: 'chromium-312d84c-linux.zip', + archiveChecksum: '1bc18b0e35701f473157799b2cd16152', + rawChecksum: 'fea8f6f3eb9e9be28be8022e3b0f04cd', binaryRelativePath: 'headless_shell-linux/headless_shell', }, { platforms: ['win32'], - archiveFilename: 'chromium-2fac04a-windows.zip', - archiveChecksum: '1499a4d5847792d59b9c1a8ab7dc8b94', - rawChecksum: '08b48d2f3d23c4bc8b58779ca4a7b627', + archiveFilename: 'chromium-312d84c-windows.zip', + archiveChecksum: '3e36adfb755dacacc226ed5fd6b43105', + rawChecksum: 'ec7aa6cfecb172129474b447311275ec', binaryRelativePath: 'headless_shell-windows\\headless_shell.exe', }, ], From bbf0e8f9f4f07e725227d9a41b33ed3b9af7e0d6 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Mon, 26 Aug 2019 14:06:36 -0700 Subject: [PATCH 13/17] Checksums for updated linux build --- .../plugins/reporting/server/browsers/chromium/paths.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/reporting/server/browsers/chromium/paths.ts b/x-pack/legacy/plugins/reporting/server/browsers/chromium/paths.ts index cdd828faf326d6..fee621d293c730 100644 --- a/x-pack/legacy/plugins/reporting/server/browsers/chromium/paths.ts +++ b/x-pack/legacy/plugins/reporting/server/browsers/chromium/paths.ts @@ -20,8 +20,8 @@ export const paths = { { platforms: ['linux'], archiveFilename: 'chromium-312d84c-linux.zip', - archiveChecksum: '1bc18b0e35701f473157799b2cd16152', - rawChecksum: 'fea8f6f3eb9e9be28be8022e3b0f04cd', + archiveChecksum: '15ba9166a42f93ee92e42217b737018d', + rawChecksum: '3455db62ea4bd2d6e891e9155313305a', binaryRelativePath: 'headless_shell-linux/headless_shell', }, { From c929c5947465be40b43d16b35acd9cb928089b65 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Tue, 27 Aug 2019 07:32:22 -0700 Subject: [PATCH 14/17] Moving types out of puppeteer file and into core puppeteer module --- .../server/browsers/chromium/driver/chromium_driver.ts | 2 +- .../server/browsers/chromium/driver_factory/index.ts | 3 ++- .../reporting/server/browsers/chromium/puppeteer.ts | 8 -------- .../reporting/server/lib/validate/validate_browser.ts | 2 +- 4 files changed, 4 insertions(+), 11 deletions(-) diff --git a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts index 081e6bf273d911..8b19f34ea3dd4a 100644 --- a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts +++ b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts @@ -6,7 +6,7 @@ import open from 'opn'; import { parse as parseUrl } from 'url'; -import { Page, SerializableOrJSHandle, EvaluateFn } from '../puppeteer'; +import { Page, SerializableOrJSHandle, EvaluateFn } from 'puppeteer'; import { ViewZoomWidthHeight } from '../../../../export_types/common/layouts/layout'; import { LevelLogger } from '../../../../server/lib'; import { diff --git a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts index fb4adb53cd5ca3..9fc1f00383e739 100644 --- a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts +++ b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts @@ -6,12 +6,13 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; +import { Browser, Page, LaunchOptions } from 'puppeteer'; import rimraf from 'rimraf'; import * as Rx from 'rxjs'; import { map, share, mergeMap, filter, partition, ignoreElements, tap } from 'rxjs/operators'; import { InnerSubscriber } from 'rxjs/internal/InnerSubscriber'; -import { Browser, Page, launch, LaunchOptions } from '../puppeteer'; +import { launch } from '../puppeteer'; import { LevelLogger as Logger } from '../../../lib/level_logger'; import { HeadlessChromiumDriver } from '../driver'; import { args, IArgOptions } from './args'; diff --git a/x-pack/legacy/plugins/reporting/server/browsers/chromium/puppeteer.ts b/x-pack/legacy/plugins/reporting/server/browsers/chromium/puppeteer.ts index 7aa88f2f901be4..93292f170c32ef 100644 --- a/x-pack/legacy/plugins/reporting/server/browsers/chromium/puppeteer.ts +++ b/x-pack/legacy/plugins/reporting/server/browsers/chromium/puppeteer.ts @@ -8,14 +8,6 @@ import puppeteer from 'puppeteer'; // @ts-ignore lacking typedefs which this module fixes import puppeteerCore from 'puppeteer-core'; -// We export a set of types and other methods since we use puppeteer-core, which has an outdated @types package. -// However, @types/puppeteer _is_ up-to-date, and this module merges them together. -export type Browser = puppeteer.Browser; -export type Page = puppeteer.Page; -export type LaunchOptions = puppeteer.LaunchOptions; -export type SerializableOrJSHandle = puppeteer.SerializableOrJSHandle; -export type EvaluateFn = puppeteer.EvaluateFn; - export const launch: ( opts?: puppeteer.LaunchOptions ) => Promise = puppeteerCore.launch.bind(puppeteerCore); diff --git a/x-pack/legacy/plugins/reporting/server/lib/validate/validate_browser.ts b/x-pack/legacy/plugins/reporting/server/lib/validate/validate_browser.ts index 9681221799fcb9..cbe60174830b34 100644 --- a/x-pack/legacy/plugins/reporting/server/lib/validate/validate_browser.ts +++ b/x-pack/legacy/plugins/reporting/server/lib/validate/validate_browser.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { Browser } from '../../browsers/chromium/puppeteer'; +import { Browser } from 'puppeteer'; import { KbnServer, Logger } from '../../../types'; import { CHROMIUM } from '../../browsers/browser_types'; From aefc12faf693c0a0e7b69b9b43454811483c274e Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Tue, 27 Aug 2019 09:33:26 -0700 Subject: [PATCH 15/17] launch => puppeteerLaunch --- .../server/browsers/chromium/driver_factory/index.ts | 6 +++--- .../plugins/reporting/server/browsers/chromium/puppeteer.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts index 9fc1f00383e739..6df8b37dd1d305 100644 --- a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts +++ b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver_factory/index.ts @@ -12,7 +12,7 @@ import * as Rx from 'rxjs'; import { map, share, mergeMap, filter, partition, ignoreElements, tap } from 'rxjs/operators'; import { InnerSubscriber } from 'rxjs/internal/InnerSubscriber'; -import { launch } from '../puppeteer'; +import { puppeteerLaunch } from '../puppeteer'; import { LevelLogger as Logger } from '../../../lib/level_logger'; import { HeadlessChromiumDriver } from '../driver'; import { args, IArgOptions } from './args'; @@ -62,7 +62,7 @@ export class HeadlessChromiumDriverFactory { proxyConfig: this.browserConfig.proxy, }); - return launch({ + return puppeteerLaunch({ userDataDir, executablePath: this.binaryPath, ignoreHTTPSErrors: true, @@ -106,7 +106,7 @@ export class HeadlessChromiumDriverFactory { let browser: Browser; let page: Page; try { - browser = await launch({ + browser = await puppeteerLaunch({ pipe: !this.browserConfig.inspect, userDataDir, executablePath: this.binaryPath, diff --git a/x-pack/legacy/plugins/reporting/server/browsers/chromium/puppeteer.ts b/x-pack/legacy/plugins/reporting/server/browsers/chromium/puppeteer.ts index 93292f170c32ef..caa25aab06287e 100644 --- a/x-pack/legacy/plugins/reporting/server/browsers/chromium/puppeteer.ts +++ b/x-pack/legacy/plugins/reporting/server/browsers/chromium/puppeteer.ts @@ -8,6 +8,6 @@ import puppeteer from 'puppeteer'; // @ts-ignore lacking typedefs which this module fixes import puppeteerCore from 'puppeteer-core'; -export const launch: ( +export const puppeteerLaunch: ( opts?: puppeteer.LaunchOptions ) => Promise = puppeteerCore.launch.bind(puppeteerCore); From a0a13b3e58218b8a3d1c5dbd6462d2697be085c6 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Tue, 27 Aug 2019 09:49:53 -0700 Subject: [PATCH 16/17] Maps comment about render loading in reporting --- .../maps/public/connected_components/gis_map/view.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/maps/public/connected_components/gis_map/view.js b/x-pack/legacy/plugins/maps/public/connected_components/gis_map/view.js index 74fd9b263035be..fd1368501e1bd4 100644 --- a/x-pack/legacy/plugins/maps/public/connected_components/gis_map/view.js +++ b/x-pack/legacy/plugins/maps/public/connected_components/gis_map/view.js @@ -52,7 +52,12 @@ export class GisMap extends Component { } // Need to get the right element until https://github.com/elastic/eui/issues/2220 - // is fixed, otherwise we can't use ref's to emit the custom event for reporting + // is fixed, otherwise we can't use ref's to emit the custom event for reporting. + // Reporting uses both the 'data-render-complete' attribute, as well as this event, + // to determine if the item is done loading. The former is there to indicate on + // pageload if the visualization is "done" -- and if not then it adds it to group + // of listeners to determine that it's done. If the attribute is missing it'll timeout. See: + // x-pack/legacy/plugins/reporting/export_types/common/lib/screenshots/wait_for_render.ts _onInitialLoadRenderComplete = () => { const el = document.querySelector(`[data-dom-id="${this.state.domId}"]`); From 59fd444ef4b7c08c69fcd68a299dd52073a5e8a5 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Tue, 27 Aug 2019 10:03:51 -0700 Subject: [PATCH 17/17] Clarify how reporting uses hooks and events for viz --- .../public/connected_components/gis_map/view.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/connected_components/gis_map/view.js b/x-pack/legacy/plugins/maps/public/connected_components/gis_map/view.js index fd1368501e1bd4..081f764978d78f 100644 --- a/x-pack/legacy/plugins/maps/public/connected_components/gis_map/view.js +++ b/x-pack/legacy/plugins/maps/public/connected_components/gis_map/view.js @@ -51,13 +51,13 @@ export class GisMap extends Component { this._clearRefreshTimer(); } - // Need to get the right element until https://github.com/elastic/eui/issues/2220 - // is fixed, otherwise we can't use ref's to emit the custom event for reporting. - // Reporting uses both the 'data-render-complete' attribute, as well as this event, - // to determine if the item is done loading. The former is there to indicate on - // pageload if the visualization is "done" -- and if not then it adds it to group - // of listeners to determine that it's done. If the attribute is missing it'll timeout. See: - // x-pack/legacy/plugins/reporting/export_types/common/lib/screenshots/wait_for_render.ts + // Reporting uses both a `data-render-complete` attribute and a DOM event listener to determine + // if a visualization is done loading. The process roughly is: + // - See if the `data-render-complete` attribute is "true". If so we're done! + // - If it's not, then reporting injects a listener into the browser for a custom "renderComplete" event. + // - When that event is fired, we snapshot the viz and move on. + // Failure to not have the dom attribute, or custom event, will timeout the job. + // See x-pack/legacy/plugins/reporting/export_types/common/lib/screenshots/wait_for_render.ts for more. _onInitialLoadRenderComplete = () => { const el = document.querySelector(`[data-dom-id="${this.state.domId}"]`);