From 9296369bfd8cc3325460fd3e761e2a9648436bdb Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Fri, 17 Jul 2020 11:04:14 +0100 Subject: [PATCH] chore(NA): teardown dynamic dll plugin (#72096) * chore(NA): teardown dynamic dll plugin * chore(NA): remove missing ts-ignore * chore(NA): remove last mentions to the DLL machinery * chore(NA): update notice file * prevent duplication and searching target/public * remove changes to es-ui code to unblock pr * add node internals override for legacy tests bundle Co-authored-by: spalger Co-authored-by: Elastic Machine --- NOTICE.txt | 3 - package.json | 1 - packages/kbn-babel-code-parser/.babelrc | 3 - packages/kbn-babel-code-parser/README.md | 19 - packages/kbn-babel-code-parser/package.json | 26 -- .../kbn-babel-code-parser/src/can_require.js | 36 -- .../kbn-babel-code-parser/src/code_parser.js | 105 ----- packages/kbn-babel-code-parser/src/index.js | 22 -- .../kbn-babel-code-parser/src/strategies.js | 103 ----- .../src/strategies.test.js | 108 ------ .../kbn-babel-code-parser/src/visitors.js | 124 ------ .../src/visitors.test.js | 68 ---- packages/kbn-babel-code-parser/yarn.lock | 1 - src/dev/build/README.md | 11 - src/dev/build/build_distributables.js | 2 - src/dev/build/tasks/index.js | 1 - .../clean_client_modules_on_dll_task.js | 126 ------ .../tasks/nodejs_modules/get_dependencies.js | 27 -- src/dev/build/tasks/nodejs_modules/index.js | 20 - .../build/tasks/nodejs_modules/webpack_dll.js | 131 ------- .../tasks/nodejs_modules/webpack_dll.test.js | 123 ------ src/dev/build/tasks/optimize_task.js | 1 - src/dev/notice/generate_notice_from_source.ts | 9 +- src/legacy/core_plugins/tests_bundle/index.js | 7 + .../server/logging/log_format_string.js | 2 - src/legacy/ui/ui_render/ui_render_mixin.js | 11 - src/optimize/base_optimizer.js | 8 - .../bundles_route/__tests__/bundles_route.js | 12 - src/optimize/bundles_route/bundles_route.ts | 19 +- .../bundles_route/proxy_bundles_route.ts | 1 - .../dynamic_dll_plugin/dll_allowed_modules.js | 36 -- .../dynamic_dll_plugin/dll_compiler.js | 366 ------------------ .../dynamic_dll_plugin/dll_config_model.js | 278 ------------- .../dynamic_dll_plugin/dll_entry_template.js | 37 -- .../dynamic_dll_plugin/dynamic_dll_plugin.js | 354 ----------------- src/optimize/dynamic_dll_plugin/index.js | 21 - src/optimize/optimize_mixin.ts | 5 +- src/optimize/watch/optmzr_role.js | 10 - src/optimize/watch/watch.js | 2 +- src/optimize/watch/watch_cache.ts | 189 --------- src/optimize/watch/watch_optimizer.js | 6 - tasks/config/karma.js | 12 - 42 files changed, 15 insertions(+), 2431 deletions(-) delete mode 100644 packages/kbn-babel-code-parser/.babelrc delete mode 100755 packages/kbn-babel-code-parser/README.md delete mode 100755 packages/kbn-babel-code-parser/package.json delete mode 100644 packages/kbn-babel-code-parser/src/can_require.js delete mode 100644 packages/kbn-babel-code-parser/src/code_parser.js delete mode 100644 packages/kbn-babel-code-parser/src/index.js delete mode 100644 packages/kbn-babel-code-parser/src/strategies.js delete mode 100644 packages/kbn-babel-code-parser/src/strategies.test.js delete mode 100644 packages/kbn-babel-code-parser/src/visitors.js delete mode 100644 packages/kbn-babel-code-parser/src/visitors.test.js delete mode 120000 packages/kbn-babel-code-parser/yarn.lock delete mode 100644 src/dev/build/tasks/nodejs_modules/clean_client_modules_on_dll_task.js delete mode 100644 src/dev/build/tasks/nodejs_modules/get_dependencies.js delete mode 100644 src/dev/build/tasks/nodejs_modules/index.js delete mode 100644 src/dev/build/tasks/nodejs_modules/webpack_dll.js delete mode 100644 src/dev/build/tasks/nodejs_modules/webpack_dll.test.js delete mode 100644 src/optimize/dynamic_dll_plugin/dll_allowed_modules.js delete mode 100644 src/optimize/dynamic_dll_plugin/dll_compiler.js delete mode 100644 src/optimize/dynamic_dll_plugin/dll_config_model.js delete mode 100644 src/optimize/dynamic_dll_plugin/dll_entry_template.js delete mode 100644 src/optimize/dynamic_dll_plugin/dynamic_dll_plugin.js delete mode 100644 src/optimize/dynamic_dll_plugin/index.js delete mode 100644 src/optimize/watch/watch_cache.ts diff --git a/NOTICE.txt b/NOTICE.txt index 56280e6e3883e..e1552852d0349 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -26,9 +26,6 @@ This module was heavily inspired by the externals plugin that ships with webpack MIT License http://www.opensource.org/licenses/mit-license.php Author Tobias Koppers @sokra ---- -This product has relied on ASTExplorer that is licensed under MIT. - --- This product includes code that is based on Ace editor, which was available under a "BSD" license. diff --git a/package.json b/package.json index 541fcc46a3d6e..593e8f82f0541 100644 --- a/package.json +++ b/package.json @@ -137,7 +137,6 @@ "@hapi/good-squeeze": "5.2.1", "@hapi/wreck": "^15.0.2", "@kbn/analytics": "1.0.0", - "@kbn/babel-code-parser": "1.0.0", "@kbn/babel-preset": "1.0.0", "@kbn/config-schema": "1.0.0", "@kbn/i18n": "1.0.0", diff --git a/packages/kbn-babel-code-parser/.babelrc b/packages/kbn-babel-code-parser/.babelrc deleted file mode 100644 index 7da72d1779128..0000000000000 --- a/packages/kbn-babel-code-parser/.babelrc +++ /dev/null @@ -1,3 +0,0 @@ -{ - "presets": ["@kbn/babel-preset/node_preset"] -} diff --git a/packages/kbn-babel-code-parser/README.md b/packages/kbn-babel-code-parser/README.md deleted file mode 100755 index f99d01541ee22..0000000000000 --- a/packages/kbn-babel-code-parser/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# @kbn/babel-code-parser - -Simple abstraction over the `@babel/parser` and the `@babel/traverse` in order -to build a code parser on top. - -We have two main functions `parseSingleFile` (sync and sync version) and the -`parseEntries` (only async version). The first one just parse one entry file -and the second one parses recursively all the files from a list of -start entry points. - -Then we have `visitors` and `strategies`. The first ones are basically the -`visitors` to use into the ast from the `@babel/traverse`. They are the only -way to collect info when using the `parseSingleFile`. The `strategies` are -meant to be used with the `parseEntries` and configures the info we want -to collect from our parsed code. After each loop, one per entry file, the -`parseEntries` method will call the given `strategy` expecting that -`strategy` would call the desired visitors, assemble the important -information to collect and adds them to the final results. - diff --git a/packages/kbn-babel-code-parser/package.json b/packages/kbn-babel-code-parser/package.json deleted file mode 100755 index 6f42c086ecaab..0000000000000 --- a/packages/kbn-babel-code-parser/package.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "name": "@kbn/babel-code-parser", - "description": "babel code parser for Kibana", - "private": true, - "version": "1.0.0", - "main": "./target/index.js", - "license": "Apache-2.0", - "repository": { - "type": "git", - "url": "https://github.com/elastic/kibana/tree/master/packages/kbn-babel-code-parser" - }, - "scripts": { - "build": "babel src --out-dir target", - "kbn:bootstrap": "yarn build --quiet", - "kbn:watch": "yarn build --watch" - }, - "devDependencies": { - "@babel/cli": "^7.10.1" - }, - "dependencies": { - "@kbn/babel-preset": "1.0.0", - "@babel/parser": "^7.10.2", - "@babel/traverse": "^7.10.1", - "lodash": "^4.17.15" - } -} diff --git a/packages/kbn-babel-code-parser/src/can_require.js b/packages/kbn-babel-code-parser/src/can_require.js deleted file mode 100644 index b410611237766..0000000000000 --- a/packages/kbn-babel-code-parser/src/can_require.js +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -export function canRequire(entry, cwd = require.resolve.paths(entry) || []) { - try { - // We will try to test if we can resolve - // this entry through the require.resolve - // setting as the start looking path the - // given cwd. That cwd variable could be - // a path or an array of paths - // from where Require.resolve will keep - // looking recursively as normal starting - // from those locations. - return require.resolve(entry, { - paths: [].concat(cwd), - }); - } catch (e) { - return false; - } -} diff --git a/packages/kbn-babel-code-parser/src/code_parser.js b/packages/kbn-babel-code-parser/src/code_parser.js deleted file mode 100644 index 91927780363ac..0000000000000 --- a/packages/kbn-babel-code-parser/src/code_parser.js +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { canRequire } from './can_require'; -import { readFile, readFileSync } from 'fs'; -import { extname } from 'path'; -import { promisify } from 'util'; -import * as parser from '@babel/parser'; -import traverse from '@babel/traverse'; -import * as babelParserOptions from '@kbn/babel-preset/common_babel_parser_options'; - -const read = promisify(readFile); - -function _cannotParseFile(filePath) { - return extname(filePath) !== '.js'; -} - -function _parseAndTraverseFileContent(fileContent, visitorsGenerator) { - const results = []; - - // Parse and get the code AST - // All the babel parser plugins - // were enabled - const ast = parser.parse(fileContent, babelParserOptions); - - // Loop through the code AST with - // the defined visitors - traverse(ast, visitorsGenerator(results)); - - return results; -} - -export async function parseSingleFile(filePath, visitorsGenerator) { - // Don't parse any other files than .js ones - if (_cannotParseFile(filePath)) { - return []; - } - - // Read the file - const content = await read(filePath, { encoding: 'utf8' }); - - // return the results found on parse and traverse - // the file content with the given visitors - return _parseAndTraverseFileContent(content, visitorsGenerator); -} - -export function parseSingleFileSync(filePath, visitorsGenerator) { - // Don't parse any other files than .js ones - if (_cannotParseFile(filePath)) { - return []; - } - - // Read the file - const content = readFileSync(filePath, { encoding: 'utf8' }); - - // return the results found on parse and traverse - // the file content with the given visitors - return _parseAndTraverseFileContent(content, visitorsGenerator); -} - -export async function parseEntries(cwd, entries, strategy, results, wasParsed = {}) { - // Assure that we always have a cwd - const sanitizedCwd = cwd || process.cwd(); - - // Test each entry against canRequire function - const entriesQueue = entries.map((entry) => canRequire(entry)); - - while (entriesQueue.length) { - // Get the first element in the queue as - // select it as our current entry to parse - const mainEntry = entriesQueue.shift(); - - // Avoid parse the current entry if it is not valid - // or it was already parsed - if (typeof mainEntry !== 'string' || wasParsed[mainEntry]) { - continue; - } - - // Find new entries and adds them to the end of the queue - entriesQueue.push( - ...(await strategy(sanitizedCwd, parseSingleFile, mainEntry, wasParsed, results)) - ); - - // Mark the current main entry as already parsed - wasParsed[mainEntry] = true; - } - - return results; -} diff --git a/packages/kbn-babel-code-parser/src/index.js b/packages/kbn-babel-code-parser/src/index.js deleted file mode 100644 index 70e4eb18068b0..0000000000000 --- a/packages/kbn-babel-code-parser/src/index.js +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -export { dependenciesParseStrategy } from './strategies'; -export { dependenciesVisitorsGenerator } from './visitors'; -export { parseSingleFile, parseSingleFileSync, parseEntries } from './code_parser'; diff --git a/packages/kbn-babel-code-parser/src/strategies.js b/packages/kbn-babel-code-parser/src/strategies.js deleted file mode 100644 index 2369692ad434b..0000000000000 --- a/packages/kbn-babel-code-parser/src/strategies.js +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { canRequire } from './can_require'; -import { dependenciesVisitorsGenerator } from './visitors'; -import { dirname, isAbsolute, resolve } from 'path'; -import { builtinModules } from 'module'; - -export function _calculateTopLevelDependency(inputDep, outputDep = '') { - // The path separator will be always the forward slash - // as at this point we only have the found entries into - // the provided source code entries where we just use it - const pathSeparator = '/'; - const depSplitPaths = inputDep.split(pathSeparator); - const firstPart = depSplitPaths.shift(); - const outputDepFirstArgAppend = outputDep ? pathSeparator : ''; - - outputDep += `${outputDepFirstArgAppend}${firstPart}`; - - // In case our dependency isn't started by @ - // we are already done and we can return the - // dependency value we already have - if (firstPart.charAt(0) !== '@') { - return outputDep; - } - - // Otherwise we need to keep constructing the dependency - // value because dependencies starting with @ points to - // folders of dependencies. For example, in case we found - // dependencies values with '@the-deps/a' and '@the-deps/a/b' - // we don't want to map it to '@the-deps' but also to @'the-deps/a' - // because inside '@the-deps' we can also have '@the-dep/b' - return _calculateTopLevelDependency(depSplitPaths.join(pathSeparator), outputDep); -} - -export async function dependenciesParseStrategy( - cwd, - parseSingleFile, - mainEntry, - wasParsed, - results -) { - // Get dependencies from a single file and filter - // out node native modules from the result - const dependencies = (await parseSingleFile(mainEntry, dependenciesVisitorsGenerator)).filter( - (dep) => !builtinModules.includes(dep) - ); - - // Return the list of all the new entries found into - // the current mainEntry that we could use to look for - // new dependencies - return dependencies.reduce((filteredEntries, entry) => { - const absEntryPath = resolve(cwd, dirname(mainEntry), entry); - - // NOTE: cwd for following canRequires is absEntryPath - // because we should start looking from there - const requiredPath = canRequire(absEntryPath, absEntryPath); - const requiredRelativePath = canRequire(entry, absEntryPath); - - const isRelativeFile = !isAbsolute(entry); - const isNodeModuleDep = isRelativeFile && !requiredPath && requiredRelativePath; - const isNewEntry = isRelativeFile && requiredPath; - - // If it is a node_module add it to the results and also - // add the resolved path for the node_module main file - // as an entry point to look for dependencies it was - // not already parsed - if (isNodeModuleDep) { - // Save the result as the top level dependency - results[_calculateTopLevelDependency(entry)] = true; - - if (!wasParsed[requiredRelativePath]) { - filteredEntries.push(requiredRelativePath); - } - } - - // If a new, not yet parsed, relative entry were found - // add it to the list of entries to be parsed - if (isNewEntry && !wasParsed[requiredPath]) { - if (!wasParsed[requiredPath]) { - filteredEntries.push(requiredPath); - } - } - - return filteredEntries; - }, []); -} diff --git a/packages/kbn-babel-code-parser/src/strategies.test.js b/packages/kbn-babel-code-parser/src/strategies.test.js deleted file mode 100644 index e61c784cdcd54..0000000000000 --- a/packages/kbn-babel-code-parser/src/strategies.test.js +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { readFile } from 'fs'; -import { canRequire } from './can_require'; -import { parseSingleFile } from './code_parser'; -import { _calculateTopLevelDependency, dependenciesParseStrategy } from './strategies'; - -jest.mock('./can_require', () => ({ - canRequire: jest.fn(), -})); - -jest.mock('fs', () => ({ - readFile: jest.fn(), -})); - -const mockCwd = '/tmp/project/dir/'; - -describe('Code Parser Strategies', () => { - it('should calculate the top level dependencies correctly', () => { - const plainDep = 'dep1/file'; - const foldedDep = '@kbn/es/file'; - const otherFoldedDep = '@kbn/es'; - - expect(_calculateTopLevelDependency(plainDep)).toEqual('dep1'); - expect(_calculateTopLevelDependency(foldedDep)).toEqual('@kbn/es'); - expect(_calculateTopLevelDependency(otherFoldedDep)).toEqual('@kbn/es'); - }); - - it('should exclude native modules', async () => { - readFile.mockImplementationOnce((path, options, cb) => { - cb(null, `require('fs')`); - }); - - const results = []; - await dependenciesParseStrategy(mockCwd, parseSingleFile, 'dep1/file.js', {}, results); - - expect(results.length).toBe(0); - }); - - it('should return a dep from_modules', async () => { - readFile.mockImplementationOnce((path, options, cb) => { - cb(null, `require('dep_from_node_modules')`); - }); - - canRequire.mockImplementation((entry, cwd) => { - if (entry === `${cwd}dep1/dep_from_node_modules`) { - return false; - } - - if (entry === 'dep_from_node_modules') { - return `${mockCwd}node_modules/dep_from_node_modules/index.js`; - } - }); - - const results = await dependenciesParseStrategy( - mockCwd, - parseSingleFile, - 'dep1/file.js', - {}, - {} - ); - expect(results[0]).toBe(`${mockCwd}node_modules/dep_from_node_modules/index.js`); - }); - - it('should return a relative dep file', async () => { - readFile.mockImplementationOnce((path, options, cb) => { - cb(null, `require('./relative_dep')`); - }); - - canRequire.mockImplementation((entry) => { - if (entry === `${mockCwd}dep1/relative_dep`) { - return `${entry}/index.js`; - } - - return false; - }); - - const results = await dependenciesParseStrategy( - mockCwd, - parseSingleFile, - 'dep1/file.js', - {}, - {} - ); - expect(results[0]).toBe(`${mockCwd}dep1/relative_dep/index.js`); - }); - - afterAll(() => { - jest.clearAllMocks(); - }); -}); diff --git a/packages/kbn-babel-code-parser/src/visitors.js b/packages/kbn-babel-code-parser/src/visitors.js deleted file mode 100644 index b159848d424fa..0000000000000 --- a/packages/kbn-babel-code-parser/src/visitors.js +++ /dev/null @@ -1,124 +0,0 @@ -/* eslint-disable @kbn/eslint/require-license-header */ - -import { matches } from 'lodash'; - -/* @notice - * - * This product has relied on ASTExplorer that is licensed under MIT. - */ -export function dependenciesVisitorsGenerator(dependenciesAcc) { - return (() => { - // This was built with help on an ast explorer and some ESTree docs - // like the babel parser ast spec and the main docs for the Esprima - // which is a complete and useful docs for the ESTree spec. - // - // https://astexplorer.net - // https://github.com/babel/babel/blob/master/packages/babel-parser/ast/spec.md - // https://esprima.readthedocs.io/en/latest/syntax-tree-format.html - // https://github.com/estree/estree - return { - // Visitors to traverse and found dependencies - // raw values on require + require.resolve - CallExpression: ({ node }) => { - // AST check for require expressions - const isRequire = (node) => { - return matches({ - callee: { - type: 'Identifier', - name: 'require', - }, - })(node); - }; - - // AST check for require.resolve expressions - const isRequireResolve = (node) => { - return matches({ - callee: { - type: 'MemberExpression', - object: { - type: 'Identifier', - name: 'require', - }, - property: { - type: 'Identifier', - name: 'resolve', - }, - }, - })(node); - }; - - // Get string values inside the expressions - // whether they are require or require.resolve - if (isRequire(node) || isRequireResolve(node)) { - const nodeArguments = node.arguments; - const reqArg = Array.isArray(nodeArguments) ? nodeArguments.shift() : null; - - if (!reqArg) { - return; - } - - if (reqArg.type === 'StringLiteral') { - dependenciesAcc.push(reqArg.value); - } - } - }, - // Visitors to traverse and found dependencies - // raw values on import - ImportDeclaration: ({ node }) => { - // AST check for supported import expressions - const isImport = (node) => { - return matches({ - type: 'ImportDeclaration', - source: { - type: 'StringLiteral', - }, - })(node); - }; - - // Get string values from import expressions - if (isImport(node)) { - const importSource = node.source; - dependenciesAcc.push(importSource.value); - } - }, - // Visitors to traverse and found dependencies - // raw values on export from - ExportNamedDeclaration: ({ node }) => { - // AST check for supported export from expressions - const isExportFrom = (node) => { - return matches({ - type: 'ExportNamedDeclaration', - source: { - type: 'StringLiteral', - }, - })(node); - }; - - // Get string values from export from expressions - if (isExportFrom(node)) { - const exportFromSource = node.source; - dependenciesAcc.push(exportFromSource.value); - } - }, - // Visitors to traverse and found dependencies - // raw values on export * from - ExportAllDeclaration: ({ node }) => { - // AST check for supported export * from expressions - const isExportAllFrom = (node) => { - return matches({ - type: 'ExportAllDeclaration', - source: { - type: 'StringLiteral', - }, - })(node); - }; - - // Get string values from export * from expressions - if (isExportAllFrom(node)) { - const exportAllFromSource = node.source; - dependenciesAcc.push(exportAllFromSource.value); - } - }, - }; - })(); -} diff --git a/packages/kbn-babel-code-parser/src/visitors.test.js b/packages/kbn-babel-code-parser/src/visitors.test.js deleted file mode 100644 index d2704fa9dfb72..0000000000000 --- a/packages/kbn-babel-code-parser/src/visitors.test.js +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import * as parser from '@babel/parser'; -import traverse from '@babel/traverse'; -import { dependenciesVisitorsGenerator } from './visitors'; - -const visitorsApplier = (code) => { - const result = []; - traverse( - parser.parse(code, { - sourceType: 'unambiguous', - plugins: ['exportDefaultFrom'], - }), - dependenciesVisitorsGenerator(result) - ); - return result; -}; - -describe('Code Parser Visitors', () => { - it('should get values from require', () => { - const rawCode = `/*foo*/require('dep1'); const bar = 1;`; - const foundDeps = visitorsApplier(rawCode); - expect(foundDeps[0] === 'dep1'); - }); - - it('should get values from require.resolve', () => { - const rawCode = `/*foo*/require.resolve('dep2'); const bar = 1;`; - const foundDeps = visitorsApplier(rawCode); - expect(foundDeps[0] === 'dep2'); - }); - - it('should get values from import', () => { - const rawCode = `/*foo*/import dep1 from 'dep1'; import dep2 from 'dep2';const bar = 1;`; - const foundDeps = visitorsApplier(rawCode); - expect(foundDeps[0] === 'dep1'); - expect(foundDeps[1] === 'dep2'); - }); - - it('should get values from export from', () => { - const rawCode = `/*foo*/export dep1 from 'dep1'; import dep2 from 'dep2';const bar = 1;`; - const foundDeps = visitorsApplier(rawCode); - expect(foundDeps[0] === 'dep1'); - }); - - it('should get values from export * from', () => { - const rawCode = `/*foo*/export * from 'dep1'; export dep2 from 'dep2';const bar = 1;`; - const foundDeps = visitorsApplier(rawCode); - expect(foundDeps[0] === 'dep1'); - expect(foundDeps[1] === 'dep2'); - }); -}); diff --git a/packages/kbn-babel-code-parser/yarn.lock b/packages/kbn-babel-code-parser/yarn.lock deleted file mode 120000 index 3f82ebc9cdbae..0000000000000 --- a/packages/kbn-babel-code-parser/yarn.lock +++ /dev/null @@ -1 +0,0 @@ -../../yarn.lock \ No newline at end of file diff --git a/src/dev/build/README.md b/src/dev/build/README.md index ed8750f6fee56..460ab01794334 100644 --- a/src/dev/build/README.md +++ b/src/dev/build/README.md @@ -39,14 +39,3 @@ The majority of this logic is extracted from the grunt build that has existed fo [lib/build.js]: ./lib/build.js [build_distributables.js]: ./build_distributables.js [../tooling_log/tooling_log.js]: ../tooling_log/tooling_log.js - -# Client Node Modules Cleaning - -We have introduced in our bundle a webpack dll for the client vendor modules in order to improve -the optimization time both in dev and in production. As for those modules we already have the -code into the vendors_${chunk_number}.bundle.dll.js we have decided to delete those bundled modules from the -distributable node_modules folder. However, in order to accomplish this, we need to exclude -every node_module used in the server side code. This logic is performed -under `nodejs_modules/clean_client_modules_on_dll_task.js`. In case we need to add any new cli -or any other piece of server code other than `x-pack` or `core_plugins` we'll need -to update the globs present on `clean_client_modules_on_dll_task.js` accordingly. diff --git a/src/dev/build/build_distributables.js b/src/dev/build/build_distributables.js index 22a348b78dc0a..39a32fff891c2 100644 --- a/src/dev/build/build_distributables.js +++ b/src/dev/build/build_distributables.js @@ -22,7 +22,6 @@ import { getConfig, createRunner } from './lib'; import { BuildKibanaPlatformPluginsTask, BuildPackagesTask, - CleanClientModulesOnDLLTask, CleanEmptyFoldersTask, CleanExtraBinScriptsTask, CleanExtraFilesFromModulesTask, @@ -127,7 +126,6 @@ export async function buildDistributables(options) { await run(TranspileScssTask); await run(BuildKibanaPlatformPluginsTask); await run(OptimizeBuildTask); - await run(CleanClientModulesOnDLLTask); await run(CleanTypescriptTask); await run(CleanExtraFilesFromModulesTask); await run(CleanEmptyFoldersTask); diff --git a/src/dev/build/tasks/index.js b/src/dev/build/tasks/index.js index d96e745c10776..0a3a67313d6a4 100644 --- a/src/dev/build/tasks/index.js +++ b/src/dev/build/tasks/index.js @@ -30,7 +30,6 @@ export * from './create_readme_task'; export * from './install_chromium'; export * from './install_dependencies_task'; export * from './license_file_task'; -export * from './nodejs_modules'; export * from './nodejs'; export * from './notice_file_task'; export * from './optimize_task'; diff --git a/src/dev/build/tasks/nodejs_modules/clean_client_modules_on_dll_task.js b/src/dev/build/tasks/nodejs_modules/clean_client_modules_on_dll_task.js deleted file mode 100644 index 05bfd3ca31a65..0000000000000 --- a/src/dev/build/tasks/nodejs_modules/clean_client_modules_on_dll_task.js +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { - getDllEntries, - cleanDllModuleFromEntryPath, - writeEmptyFileForDllEntry, -} from './webpack_dll'; -import { getDependencies } from './get_dependencies'; -import globby from 'globby'; -import normalizePosixPath from 'normalize-path'; - -export const CleanClientModulesOnDLLTask = { - description: 'Cleaning client node_modules bundled into the DLL', - - async run(config, log, build) { - const baseDir = normalizePosixPath(build.resolvePath('.')); - const kbnPkg = config.getKibanaPkg(); - const kbnPkgDependencies = (kbnPkg && kbnPkg.dependencies) || {}; - const kbnWebpackLoaders = Object.keys(kbnPkgDependencies).filter( - (dep) => !!dep.includes('-loader') - ); - - // Define the entry points for the server code in order to - // start here later looking for the server side dependencies - const mainCodeEntries = [ - `${baseDir}/src/cli`, - `${baseDir}/src/cli_keystore`, - `${baseDir}/src/cli_plugin`, - `${baseDir}/x-pack`, - ...kbnWebpackLoaders.map((loader) => `${baseDir}/node_modules/${loader}`), - ]; - const discoveredLegacyCorePluginEntries = await globby([ - `${baseDir}/src/legacy/core_plugins/*/index.js`, - `!${baseDir}/src/legacy/core_plugins/**/public`, - ]); - const discoveredPluginEntries = await globby([ - `${baseDir}/src/plugins/*/server/index.js`, - // Small exception to load dynamically discovered functions for timelion plugin - `${baseDir}/src/plugins/vis_type_timelion/server/*_functions/**/*.js`, - `!${baseDir}/src/plugins/**/public`, - ]); - const discoveredNewPlatformXpackPlugins = await globby([ - `${baseDir}/x-pack/plugins/*/server/index.js`, - `!${baseDir}/x-pack/plugins/**/public`, - ]); - - // Compose all the needed entries - const serverEntries = [ - ...mainCodeEntries, - ...discoveredLegacyCorePluginEntries, - ...discoveredPluginEntries, - ...discoveredNewPlatformXpackPlugins, - ]; - - // Get the dependencies found searching through the server - // side code entries that were provided - const serverDependencies = await getDependencies(baseDir, serverEntries); - - // This fulfill a particular exceptional case where - // we need to keep loading a file from a node_module - // only used in the front-end like we do when using the file-loader - // in https://github.com/elastic/kibana/blob/master/x-pack/legacy/plugins/maps/public/connected_components/map/mb/view.js - // - // manual list of exception modules - const manualExceptionModules = ['mapbox-gl']; - - // consider the top modules as exceptions as the entry points - // to look for other exceptions dependent on that one - const manualExceptionEntries = [ - ...manualExceptionModules.map((module) => `${baseDir}/node_modules/${module}`), - ]; - - // dependencies for declared exception modules - const manualExceptionModulesDependencies = await getDependencies(baseDir, [ - ...manualExceptionEntries, - ]); - - // final list of manual exceptions to add - const manualExceptions = [...manualExceptionModules, ...manualExceptionModulesDependencies]; - - // Consider this as our whiteList for the modules we can't delete - const whiteListedModules = [...serverDependencies, ...kbnWebpackLoaders, ...manualExceptions]; - - // Resolve the client vendors dll manifest paths - // excluding the runtime one - const dllManifestPaths = await globby([ - `${baseDir}/built_assets/dlls/vendors_*.manifest.dll.json`, - `!${baseDir}/built_assets/dlls/vendors_runtime.manifest.dll.json`, - ]); - - // Get dll entries filtering out the ones - // from any whitelisted module - const dllEntries = await getDllEntries(dllManifestPaths, whiteListedModules, baseDir); - - for (const relativeEntryPath of dllEntries) { - const entryPath = `${baseDir}/${relativeEntryPath}`; - - if (entryPath.endsWith('package.json')) { - continue; - } - - // Clean a module included into the dll - // and then write a blank file for each - // entry file present into the dll - await cleanDllModuleFromEntryPath(log, entryPath); - await writeEmptyFileForDllEntry(entryPath); - } - }, -}; diff --git a/src/dev/build/tasks/nodejs_modules/get_dependencies.js b/src/dev/build/tasks/nodejs_modules/get_dependencies.js deleted file mode 100644 index 5a39448f57d14..0000000000000 --- a/src/dev/build/tasks/nodejs_modules/get_dependencies.js +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { parseEntries, dependenciesParseStrategy } from '@kbn/babel-code-parser'; - -export async function getDependencies(cwd, entries) { - // Return the dependencies retrieve from the - // provided code entries (sanitized) and - // parseStrategy (dependencies one) - return Object.keys(await parseEntries(cwd, entries, dependenciesParseStrategy, {})); -} diff --git a/src/dev/build/tasks/nodejs_modules/index.js b/src/dev/build/tasks/nodejs_modules/index.js deleted file mode 100644 index 1c5392662ad13..0000000000000 --- a/src/dev/build/tasks/nodejs_modules/index.js +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -export { CleanClientModulesOnDLLTask } from './clean_client_modules_on_dll_task'; diff --git a/src/dev/build/tasks/nodejs_modules/webpack_dll.js b/src/dev/build/tasks/nodejs_modules/webpack_dll.js deleted file mode 100644 index 8de5e582c3d36..0000000000000 --- a/src/dev/build/tasks/nodejs_modules/webpack_dll.js +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { deleteAll, isFileAccessible, read, write } from '../../lib'; -import { dirname, relative, resolve } from 'path'; -import pkgUp from 'pkg-up'; -import globby from 'globby'; -import normalizePosixPath from 'normalize-path'; - -function checkDllEntryAccess(entry, baseDir = '') { - const resolvedPath = baseDir ? resolve(baseDir, entry) : entry; - return isFileAccessible(resolvedPath); -} - -export async function getDllEntries(manifestPaths, whiteListedModules, baseDir = '') { - // Read and parse all manifests - const manifests = await Promise.all( - manifestPaths.map(async (manifestPath) => JSON.parse(await read(manifestPath))) - ); - - // Process and group modules from all manifests - const manifestsModules = manifests.flatMap((manifest, idx) => { - if (!manifest || !manifest.content) { - // It should fails because if we don't have the manifest file - // or it is malformed something wrong is happening and we - // should stop - throw new Error(`The following dll manifest doesn't exists: ${manifestPaths[idx]}`); - } - - const modules = Object.keys(manifest.content); - if (!modules.length) { - // It should fails because if we don't have any - // module inside the client vendors dll something - // wrong is happening and we should stop too - throw new Error( - `The following dll manifest is reporting an empty dll: ${manifestPaths[idx]}` - ); - } - - return modules; - }); - - // Only includes modules who are not in the white list of modules - // and that are node_modules - return manifestsModules.filter((entry) => { - const isWhiteListed = whiteListedModules.some((nonEntry) => - normalizePosixPath(entry).includes(`node_modules/${nonEntry}`) - ); - const isNodeModule = entry.includes('node_modules'); - - // NOTE: when using dynamic imports on webpack the entry paths could be created - // with special context module (ex: lazy recursive) values over directories that are not real files - // and only exists in runtime, so we need to check if the entry is a real file. - // We found that problem through the issue https://github.com/elastic/kibana/issues/38481 - // - // More info: - // https://github.com/webpack/webpack/blob/master/examples/code-splitting-harmony/README.md - // https://webpack.js.org/guides/dependency-management/#require-with-expression - const isAccessible = checkDllEntryAccess(entry, baseDir); - - return !isWhiteListed && isNodeModule && isAccessible; - }); -} - -export async function cleanDllModuleFromEntryPath(logger, entryPath) { - const modulePkgPath = await pkgUp(entryPath); - const modulePkg = JSON.parse(await read(modulePkgPath)); - const moduleDir = dirname(modulePkgPath); - const normalizedModuleDir = normalizePosixPath(moduleDir); - - // Cancel the cleanup for this module as it - // was already done. - if (modulePkg.cleaned) { - return; - } - - // Clear dependencies from dll module package.json - if (modulePkg.dependencies) { - modulePkg.dependencies = {}; - } - - // Clear devDependencies from dll module package.json - if (modulePkg.devDependencies) { - modulePkg.devDependencies = {}; - } - - // Delete module contents. It will delete everything - // excepts package.json, images and css - // - // NOTE: We can't use cwd option with globby - // until the following issue gets closed - // https://github.com/sindresorhus/globby/issues/87 - const filesToDelete = await globby([ - `${normalizedModuleDir}/**`, - `!${normalizedModuleDir}/**/*.+(css)`, - `!${normalizedModuleDir}/**/*.+(gif|ico|jpeg|jpg|tiff|tif|svg|png|webp)`, - ]); - - await deleteAll( - filesToDelete.filter((path) => { - const relativePath = relative(moduleDir, path); - return !relativePath.endsWith('package.json') || relativePath.includes('node_modules'); - }) - ); - - // Mark this module as cleaned - modulePkg.cleaned = true; - - // Rewrite modified package.json - await write(modulePkgPath, JSON.stringify(modulePkg, null, 2)); -} - -export async function writeEmptyFileForDllEntry(entryPath) { - await write(entryPath, ''); -} diff --git a/src/dev/build/tasks/nodejs_modules/webpack_dll.test.js b/src/dev/build/tasks/nodejs_modules/webpack_dll.test.js deleted file mode 100644 index ce305169a777b..0000000000000 --- a/src/dev/build/tasks/nodejs_modules/webpack_dll.test.js +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { isFileAccessible, read } from '../../lib'; -import { getDllEntries } from './webpack_dll'; - -jest.mock('../../lib', () => ({ - read: jest.fn(), - isFileAccessible: jest.fn(), -})); - -const manifestContentMock = JSON.stringify({ - name: 'vendors', - content: { - '/mock/node_modules/dep1': {}, - '/mock/node_modules/dep2': {}, - '/mock/node_modules/dep3': {}, - '/mock/tmp/dep2': {}, - }, -}); - -const emptyManifestContentMock = JSON.stringify({ - name: 'vendors', - content: {}, -}); - -const noManifestMock = JSON.stringify(null); - -const noContentFieldManifestMock = JSON.stringify({ - name: 'vendors', -}); - -describe('Webpack DLL Build Tasks Utils', () => { - it('should get dll entries correctly', async () => { - read.mockImplementationOnce(async () => manifestContentMock); - - isFileAccessible.mockImplementation(() => true); - - const mockManifestPath = ['/mock/mock_dll_manifest.json']; - const mockModulesWhitelist = ['dep1']; - const dllEntries = await getDllEntries(mockManifestPath, mockModulesWhitelist); - - expect(dllEntries).toEqual( - expect.arrayContaining(['/mock/node_modules/dep2', '/mock/node_modules/dep3']) - ); - }); - - it('should only include accessible files', async () => { - read.mockImplementationOnce(async () => manifestContentMock); - - isFileAccessible.mockImplementation(() => false); - - const mockManifestPath = ['/mock/mock_dll_manifest.json']; - const mockModulesWhitelist = ['dep1']; - const dllEntries = await getDllEntries(mockManifestPath, mockModulesWhitelist); - - isFileAccessible.mockRestore(); - - expect(dllEntries.length).toEqual(0); - }); - - it('should throw an error for no manifest file', async () => { - read.mockImplementationOnce(async () => noManifestMock); - - const mockManifestPath = ['/mock/mock_dll_manifest.json']; - - try { - await getDllEntries(mockManifestPath, []); - } catch (error) { - expect(error.message).toEqual( - `The following dll manifest doesn't exists: /mock/mock_dll_manifest.json` - ); - } - }); - - it('should throw an error for no manifest content field', async () => { - read.mockImplementation(async () => noContentFieldManifestMock); - - const mockManifestPath = ['/mock/mock_dll_manifest.json']; - - try { - await getDllEntries(mockManifestPath, []); - } catch (error) { - expect(error.message).toEqual( - `The following dll manifest doesn't exists: /mock/mock_dll_manifest.json` - ); - } - }); - - it('should throw an error for manifest file without any content', async () => { - read.mockImplementation(async () => emptyManifestContentMock); - - const mockManifestPath = ['/mock/mock_dll_manifest.json']; - - try { - await getDllEntries(mockManifestPath, []); - } catch (error) { - expect(error.message).toEqual( - `The following dll manifest is reporting an empty dll: /mock/mock_dll_manifest.json` - ); - } - }); - - afterAll(() => { - jest.clearAllMocks(); - }); -}); diff --git a/src/dev/build/tasks/optimize_task.js b/src/dev/build/tasks/optimize_task.js index 4034092273239..16a7537b8ac9e 100644 --- a/src/dev/build/tasks/optimize_task.js +++ b/src/dev/build/tasks/optimize_task.js @@ -41,7 +41,6 @@ export const OptimizeBuildTask = { await exec(log, kibanaScript, kibanaArgs, { cwd: build.resolvePath('.'), env: { - FORCE_DLL_CREATION: 'true', KBN_CACHE_LOADER_WRITABLE: 'true', NODE_OPTIONS: '--max-old-space-size=4096', }, diff --git a/src/dev/notice/generate_notice_from_source.ts b/src/dev/notice/generate_notice_from_source.ts index fb74bed0f26f4..37bbcce72e497 100644 --- a/src/dev/notice/generate_notice_from_source.ts +++ b/src/dev/notice/generate_notice_from_source.ts @@ -47,10 +47,11 @@ export async function generateNoticeFromSource({ productName, directory, log }: cwd: directory, nodir: true, ignore: [ - '{node_modules,build,target,dist,data,built_assets}/**', - 'packages/*/{node_modules,build,target,dist}/**', - 'x-pack/{node_modules,build,target,dist,data}/**', - 'x-pack/packages/*/{node_modules,build,target,dist}/**', + '{node_modules,build,dist,data,built_assets}/**', + 'packages/*/{node_modules,build,dist}/**', + 'x-pack/{node_modules,build,dist,data}/**', + 'x-pack/packages/*/{node_modules,build,dist}/**', + '**/target/**', ], }; diff --git a/src/legacy/core_plugins/tests_bundle/index.js b/src/legacy/core_plugins/tests_bundle/index.js index 431c161585fe0..da7c1c4f4527e 100644 --- a/src/legacy/core_plugins/tests_bundle/index.js +++ b/src/legacy/core_plugins/tests_bundle/index.js @@ -108,6 +108,13 @@ export default (kibana) => { resolve: { extensions: ['.karma_mock.js', '.karma_mock.tsx', '.karma_mock.ts'], }, + node: { + fs: 'empty', + child_process: 'empty', + dns: 'empty', + net: 'empty', + tls: 'empty', + }, }, webpackConfig ); diff --git a/src/legacy/server/logging/log_format_string.js b/src/legacy/server/logging/log_format_string.js index 3c18aab2e3d09..cbbf71dd894ac 100644 --- a/src/legacy/server/logging/log_format_string.js +++ b/src/legacy/server/logging/log_format_string.js @@ -41,8 +41,6 @@ const typeColors = { optmzr: 'white', manager: 'green', optimize: 'magentaBright', - 'optimize:dynamic_dll_plugin': 'magentaBright', - 'optimize:watch_cache': 'magentaBright', listening: 'magentaBright', scss: 'magentaBright', }; diff --git a/src/legacy/ui/ui_render/ui_render_mixin.js b/src/legacy/ui/ui_render/ui_render_mixin.js index 168dddf0253d9..7788aeaee72e5 100644 --- a/src/legacy/ui/ui_render/ui_render_mixin.js +++ b/src/legacy/ui/ui_render/ui_render_mixin.js @@ -24,7 +24,6 @@ import { i18n } from '@kbn/i18n'; import * as UiSharedDeps from '@kbn/ui-shared-deps'; import { AppBootstrap } from './bootstrap'; import { getApmConfig } from '../apm'; -import { DllCompiler } from '../../../optimize/dynamic_dll_plugin'; /** * @typedef {import('../../server/kbn_server').default} KbnServer @@ -106,17 +105,8 @@ export function uiRenderMixin(kbnServer, server, config) { const basePath = config.get('server.basePath'); const regularBundlePath = `${basePath}/${buildHash}/bundles`; - const dllBundlePath = `${basePath}/${buildHash}/built_assets/dlls`; - - const dllStyleChunks = DllCompiler.getRawDllConfig().chunks.map( - (chunk) => `${dllBundlePath}/vendors${chunk}.style.dll.css` - ); - const dllJsChunks = DllCompiler.getRawDllConfig().chunks.map( - (chunk) => `${dllBundlePath}/vendors${chunk}.bundle.dll.js` - ); const styleSheetPaths = [ - ...(isCore ? [] : dllStyleChunks), `${regularBundlePath}/kbn-ui-shared-deps/${UiSharedDeps.baseCssDistFilename}`, ...(darkMode ? [ @@ -173,7 +163,6 @@ export function uiRenderMixin(kbnServer, server, config) { (filename) => `${regularBundlePath}/kbn-ui-shared-deps/${filename}` ), `${regularBundlePath}/kbn-ui-shared-deps/${UiSharedDeps.jsFilename}`, - ...(isCore ? [] : [`${dllBundlePath}/vendors_runtime.bundle.dll.js`, ...dllJsChunks]), `${regularBundlePath}/core/core.entry.js`, ...kpPluginBundlePaths, diff --git a/src/optimize/base_optimizer.js b/src/optimize/base_optimizer.js index 55752db55e28a..41628a2264193 100644 --- a/src/optimize/base_optimizer.js +++ b/src/optimize/base_optimizer.js @@ -30,8 +30,6 @@ import webpackMerge from 'webpack-merge'; import WrapperPlugin from 'wrapper-webpack-plugin'; import * as UiSharedDeps from '@kbn/ui-shared-deps'; -import { DynamicDllPlugin } from './dynamic_dll_plugin'; - import { IS_KIBANA_DISTRIBUTABLE } from '../legacy/utils'; import { fromRoot } from '../core/server/utils'; import { PUBLIC_PATH_PLACEHOLDER } from './public_path_placeholder'; @@ -286,12 +284,6 @@ export default class BaseOptimizer { }, plugins: [ - new DynamicDllPlugin({ - uiBundles: this.uiBundles, - threadLoaderPoolConfig: this.getThreadLoaderPoolConfig(), - logWithMetadata: this.logWithMetadata, - }), - new MiniCssExtractPlugin({ filename: '[name].style.css', }), diff --git a/src/optimize/bundles_route/__tests__/bundles_route.js b/src/optimize/bundles_route/__tests__/bundles_route.js index 902fa59b20569..5b42b658300fe 100644 --- a/src/optimize/bundles_route/__tests__/bundles_route.js +++ b/src/optimize/bundles_route/__tests__/bundles_route.js @@ -56,7 +56,6 @@ describe('optimizer/bundle route', () => { function createServer(options = {}) { const { regularBundlesPath = outputFixture, - dllBundlesPath = outputFixture, basePublicPath = '', builtCssPath = outputFixture, npUiPluginPublicDirs = [], @@ -70,7 +69,6 @@ describe('optimizer/bundle route', () => { server.route( createBundlesRoute({ regularBundlesPath, - dllBundlesPath, basePublicPath, builtCssPath, npUiPluginPublicDirs, @@ -89,28 +87,24 @@ describe('optimizer/bundle route', () => { expect(() => { createBundlesRoute({ regularBundlesPath: null, - dllBundlesPath: '/absolute/path', basePublicPath: '', }); }).to.throwError(/absolute path/); expect(() => { createBundlesRoute({ regularBundlesPath: './relative', - dllBundlesPath: '/absolute/path', basePublicPath: '', }); }).to.throwError(/absolute path/); expect(() => { createBundlesRoute({ regularBundlesPath: 1234, - dllBundlesPath: '/absolute/path', basePublicPath: '', }); }).to.throwError(/absolute path/); expect(() => { createBundlesRoute({ regularBundlesPath: '/absolute/path', - dllBundlesPath: '/absolute/path', basePublicPath: '', }); }).to.not.throwError(); @@ -119,42 +113,36 @@ describe('optimizer/bundle route', () => { expect(() => { createBundlesRoute({ regularBundlesPath: '/bundles', - dllBundlesPath: '/absolute/path', basePublicPath: 123, }); }).to.throwError(/string/); expect(() => { createBundlesRoute({ regularBundlesPath: '/bundles', - dllBundlesPath: '/absolute/path', basePublicPath: {}, }); }).to.throwError(/string/); expect(() => { createBundlesRoute({ regularBundlesPath: '/bundles', - dllBundlesPath: '/absolute/path', basePublicPath: '/a/', }); }).to.throwError(/start and not end with a \//); expect(() => { createBundlesRoute({ regularBundlesPath: '/bundles', - dllBundlesPath: '/absolute/path', basePublicPath: 'a/', }); }).to.throwError(/start and not end with a \//); expect(() => { createBundlesRoute({ regularBundlesPath: '/bundles', - dllBundlesPath: '/absolute/path', basePublicPath: '/a', }); }).to.not.throwError(); expect(() => { createBundlesRoute({ regularBundlesPath: '/bundles', - dllBundlesPath: '/absolute/path', basePublicPath: '', }); }).to.not.throwError(); diff --git a/src/optimize/bundles_route/bundles_route.ts b/src/optimize/bundles_route/bundles_route.ts index e9cfba0130d95..8c1092efed252 100644 --- a/src/optimize/bundles_route/bundles_route.ts +++ b/src/optimize/bundles_route/bundles_route.ts @@ -28,22 +28,19 @@ import { assertIsNpUiPluginPublicDirs, NpUiPluginPublicDirs } from '../np_ui_plu import { fromRoot } from '../../core/server/utils'; /** - * Creates the routes that serves files from `bundlesPath` or from - * `dllBundlesPath` (if they are dll bundle's related files). If the + * Creates the routes that serves files from `bundlesPath`. If the * file is js or css then it is searched for instances of * PUBLIC_PATH_PLACEHOLDER and replaces them with `publicPath`. * * @param {Object} options * @property {Array<{id,path}>} options.npUiPluginPublicDirs array of ids and paths that should be served for new platform plugins * @property {string} options.regularBundlesPath - * @property {string} options.dllBundlesPath * @property {string} options.basePublicPath * * @return Array.of({Hapi.Route}) */ export function createBundlesRoute({ regularBundlesPath, - dllBundlesPath, basePublicPath, builtCssPath, npUiPluginPublicDirs = [], @@ -51,7 +48,6 @@ export function createBundlesRoute({ isDist = false, }: { regularBundlesPath: string; - dllBundlesPath: string; basePublicPath: string; builtCssPath: string; npUiPluginPublicDirs?: NpUiPluginPublicDirs; @@ -70,12 +66,6 @@ export function createBundlesRoute({ ); } - if (typeof dllBundlesPath !== 'string' || !isAbsolute(dllBundlesPath)) { - throw new TypeError( - 'dllBundlesPath must be an absolute path to the directory containing the dll bundles' - ); - } - if (typeof basePublicPath !== 'string') { throw new TypeError('basePublicPath must be a string'); } @@ -118,13 +108,6 @@ export function createBundlesRoute({ fileHashCache, isDist, }), - buildRouteForBundles({ - publicPath: `${basePublicPath}/${buildHash}/built_assets/dlls/`, - routePath: `/${buildHash}/built_assets/dlls/`, - bundlesPath: dllBundlesPath, - fileHashCache, - isDist, - }), buildRouteForBundles({ publicPath: `${basePublicPath}/`, routePath: `/${buildHash}/built_assets/css/`, diff --git a/src/optimize/bundles_route/proxy_bundles_route.ts b/src/optimize/bundles_route/proxy_bundles_route.ts index 1d189054324a1..108d253d45202 100644 --- a/src/optimize/bundles_route/proxy_bundles_route.ts +++ b/src/optimize/bundles_route/proxy_bundles_route.ts @@ -28,7 +28,6 @@ export function createProxyBundlesRoute({ }) { return [ buildProxyRouteForBundles(`/${buildHash}/bundles/`, host, port), - buildProxyRouteForBundles(`/${buildHash}/built_assets/dlls/`, host, port), buildProxyRouteForBundles(`/${buildHash}/built_assets/css/`, host, port), ]; } diff --git a/src/optimize/dynamic_dll_plugin/dll_allowed_modules.js b/src/optimize/dynamic_dll_plugin/dll_allowed_modules.js deleted file mode 100644 index 4314cebd3b7e9..0000000000000 --- a/src/optimize/dynamic_dll_plugin/dll_allowed_modules.js +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import path from 'path'; - -export function notInNodeModules(checkPath) { - return !checkPath.includes(`${path.sep}node_modules${path.sep}`); -} - -export function notInNodeModulesOrWebpackShims(checkPath) { - return notInNodeModules(checkPath) && !checkPath.includes(`${path.sep}webpackShims${path.sep}`); -} - -export function inPluginNodeModules(checkPath) { - return checkPath.match(/[\/\\]plugins.*[\/\\]node_modules/); -} - -export function inDllPluginPublic(checkPath) { - return checkPath.includes(`${path.sep}dynamic_dll_plugin${path.sep}public${path.sep}`); -} diff --git a/src/optimize/dynamic_dll_plugin/dll_compiler.js b/src/optimize/dynamic_dll_plugin/dll_compiler.js deleted file mode 100644 index 9ab21ee0e9076..0000000000000 --- a/src/optimize/dynamic_dll_plugin/dll_compiler.js +++ /dev/null @@ -1,366 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { configModel } from './dll_config_model'; -import { - notInNodeModulesOrWebpackShims, - notInNodeModules, - inDllPluginPublic, -} from './dll_allowed_modules'; -import { - dllEntryFileContentArrayToString, - dllEntryFileContentStringToArray, - dllMergeAllEntryFilesContent, -} from './dll_entry_template'; -import { fromRoot } from '../../core/server/utils'; -import { PUBLIC_PATH_PLACEHOLDER } from '../public_path_placeholder'; -import fs from 'fs'; -import webpack from 'webpack'; -import { promisify } from 'util'; -import path from 'path'; -import del from 'del'; -import { chunk } from 'lodash'; -import seedrandom from 'seedrandom'; - -const readFileAsync = promisify(fs.readFile); -const mkdirAsync = promisify(fs.mkdir); -const accessAsync = promisify(fs.access); -const writeFileAsync = promisify(fs.writeFile); - -export class DllCompiler { - static getRawDllConfig( - uiBundles = {}, - babelLoaderCacheDir = '', - threadLoaderPoolConfig = {}, - chunks = Array.from(Array(4).keys()).map((chunkN) => `_${chunkN}`) - ) { - return { - uiBundles, - babelLoaderCacheDir, - threadLoaderPoolConfig, - chunks, - context: fromRoot('.'), - entryName: 'vendors', - dllName: '[name]', - manifestName: '[name]', - styleName: '[name]', - entryExt: '.entry.dll.js', - dllExt: '.bundle.dll.js', - manifestExt: '.manifest.dll.json', - styleExt: '.style.dll.css', - outputPath: fromRoot('built_assets/dlls'), - publicPath: PUBLIC_PATH_PLACEHOLDER, - }; - } - - constructor(uiBundles, threadLoaderPoolConfig, logWithMetadata) { - this.rawDllConfig = DllCompiler.getRawDllConfig( - uiBundles, - uiBundles.getCacheDirectory('babel'), - threadLoaderPoolConfig - ); - this.logWithMetadata = logWithMetadata || (() => null); - } - - async init() { - await this.ensureEntryFilesExists(); - await this.ensureManifestFilesExists(); - await this.ensureOutputPathExists(); - } - - seededShuffle(array) { - // Implementation based on https://github.com/TimothyGu/knuth-shuffle-seeded/blob/gh-pages/index.js#L46 - let currentIndex; - let temporaryValue; - let randomIndex; - const rand = seedrandom('predictable', { global: false }); - - if (array.constructor !== Array) throw new Error('Input is not an array'); - currentIndex = array.length; - - // While there remain elements to shuffle... - while (0 !== currentIndex) { - // Pick a remaining element... - randomIndex = Math.floor(rand() * currentIndex--); - - // And swap it with the current element. - temporaryValue = array[currentIndex]; - array[currentIndex] = array[randomIndex]; - array[randomIndex] = temporaryValue; - } - - return array; - } - - async upsertEntryFiles(content) { - const arrayContent = this.seededShuffle(dllEntryFileContentStringToArray(content)); - const chunks = chunk( - arrayContent, - Math.ceil(arrayContent.length / this.rawDllConfig.chunks.length) - ); - const entryPaths = this.getEntryPaths(); - - await Promise.all( - entryPaths.map( - async (entryPath, idx) => - await this.upsertFile(entryPath, dllEntryFileContentArrayToString(chunks[idx])) - ) - ); - } - - async upsertFile(filePath, content = '') { - await this.ensurePathExists(filePath); - await writeFileAsync(filePath, content, 'utf8'); - } - - getDllPaths() { - return this.rawDllConfig.chunks.map((chunk) => - this.resolvePath(`${this.rawDllConfig.entryName}${chunk}${this.rawDllConfig.dllExt}`) - ); - } - - getEntryPaths() { - return this.rawDllConfig.chunks.map((chunk) => - this.resolvePath(`${this.rawDllConfig.entryName}${chunk}${this.rawDllConfig.entryExt}`) - ); - } - - getManifestPaths() { - return this.rawDllConfig.chunks.map((chunk) => - this.resolvePath(`${this.rawDllConfig.entryName}${chunk}${this.rawDllConfig.manifestExt}`) - ); - } - - getStylePaths() { - return this.rawDllConfig.chunks.map((chunk) => - this.resolvePath(`${this.rawDllConfig.entryName}${chunk}${this.rawDllConfig.styleExt}`) - ); - } - - async ensureEntryFilesExists() { - const entryPaths = this.getEntryPaths(); - - await Promise.all(entryPaths.map(async (entryPath) => await this.ensureFileExists(entryPath))); - } - - async ensureManifestFilesExists() { - const manifestPaths = this.getManifestPaths(); - - await Promise.all( - manifestPaths.map( - async (manifestPath, idx) => - await this.ensureFileExists( - manifestPath, - JSON.stringify({ - name: `${this.rawDllConfig.entryName}${this.rawDllConfig.chunks[idx]}`, - content: {}, - }) - ) - ) - ); - } - - async ensureStyleFileExists() { - const stylePaths = this.getStylePaths(); - - await Promise.all(stylePaths.map(async (stylePath) => await this.ensureFileExists(stylePath))); - } - - async ensureFileExists(filePath, content) { - const exists = await this.ensurePathExists(filePath); - - if (!exists) { - await this.upsertFile(filePath, content); - } - } - - async ensurePathExists(filePath) { - try { - await accessAsync(filePath); - } catch (e) { - await mkdirAsync(path.dirname(filePath), { recursive: true }); - return false; - } - - return true; - } - - async ensureOutputPathExists() { - await this.ensurePathExists(this.rawDllConfig.outputPath); - } - - dllsExistsSync() { - const dllPaths = this.getDllPaths(); - - return dllPaths.every((dllPath) => this.existsSync(dllPath)); - } - - existsSync(filePath) { - return fs.existsSync(filePath); - } - - resolvePath() { - return path.resolve(this.rawDllConfig.outputPath, ...arguments); - } - - async readEntryFiles() { - const entryPaths = this.getEntryPaths(); - - const entryFilesContent = await Promise.all( - entryPaths.map(async (entryPath) => await this.readFile(entryPath)) - ); - - // merge all the module contents from entry files again into - // sorted single one - return dllMergeAllEntryFilesContent(entryFilesContent); - } - - async readFile(filePath, content) { - await this.ensureFileExists(filePath, content); - return await readFileAsync(filePath, 'utf8'); - } - - async run(dllEntries) { - const dllConfig = this.dllConfigGenerator(this.rawDllConfig); - await this.upsertEntryFiles(dllEntries); - - try { - this.logWithMetadata( - ['info', 'optimize:dynamic_dll_plugin'], - 'Client vendors dll compilation started' - ); - - await this.runWebpack(dllConfig()); - - this.logWithMetadata( - ['info', 'optimize:dynamic_dll_plugin'], - `Client vendors dll compilation finished with success` - ); - } catch (e) { - this.logWithMetadata( - ['fatal', 'optimize:dynamic_dll_plugin'], - `Client vendors dll compilation failed` - ); - - // Still throw the original error has here we just want - // log the fail message - throw e; - } - - // Style dll file isn't always created but we are - // expecting it to exist always as we are referencing - // it from the bootstrap template - // - // NOTE: We should review the way we deal with the css extraction - // in ours webpack builds. The industry standard is about to - // only extract css for production but we are extracting it - // in every single compilation. - await this.ensureStyleFileExists(); - } - - dllConfigGenerator(dllConfig) { - return configModel.bind(this, dllConfig); - } - - async runWebpack(config) { - return new Promise((resolve, reject) => { - webpack(config, async (err, stats) => { - // If a critical error occurs or we have - // errors in the stats compilation, - // reject the promise and logs the errors - const webpackErrors = - err || - (stats.hasErrors() && - stats.toString({ - all: false, - colors: true, - errors: true, - errorDetails: true, - moduleTrace: true, - })); - - if (webpackErrors) { - // Reject with webpack fatal errors - return reject(webpackErrors); - } - - // Identify if we have not allowed modules - // bundled inside the dll bundle - const notAllowedModules = []; - - stats.compilation.modules.forEach((module) => { - // ignore if no module or userRequest are defined - if (!module || !module.resource) { - return; - } - - // ignore if this module represents the - // dll entry file - if (this.getEntryPaths().includes(module.resource)) { - return; - } - - // ignore if this module is part of the - // files inside dynamic dll plugin public folder - if (inDllPluginPublic(module.resource)) { - return; - } - - // A module is not allowed if it's not a node_module, a webpackShim - // or the reasons from being bundled into the dll are not node_modules - if (notInNodeModulesOrWebpackShims(module.resource)) { - const reasons = module.reasons || []; - - reasons.forEach((reason) => { - // Skip if we can't read the reason info - if (!reason || !reason.module || !reason.module.resource) { - return; - } - - // Is the reason for this module being bundle a - // node_module or no? - if (notInNodeModules(reason.module.resource)) { - notAllowedModules.push(module.resource); - } - }); - } - }); - - if (notAllowedModules.length) { - // Delete the built dll, as it contains invalid modules, and reject listing - // all the not allowed modules - try { - await del(this.rawDllConfig.outputPath); - } catch (e) { - return reject(e); - } - - return reject( - `The following modules are not allowed to be bundled into the dll: \n${notAllowedModules.join( - '\n' - )}` - ); - } - - // Otherwise it has succeed - return resolve(stats); - }); - }); - } -} diff --git a/src/optimize/dynamic_dll_plugin/dll_config_model.js b/src/optimize/dynamic_dll_plugin/dll_config_model.js deleted file mode 100644 index eec369b194fef..0000000000000 --- a/src/optimize/dynamic_dll_plugin/dll_config_model.js +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { IS_KIBANA_DISTRIBUTABLE } from '../../legacy/utils'; -import { fromRoot } from '../../core/server/utils'; -import webpack from 'webpack'; -import webpackMerge from 'webpack-merge'; -import MiniCssExtractPlugin from 'mini-css-extract-plugin'; -import TerserPlugin from 'terser-webpack-plugin'; -import * as UiSharedDeps from '@kbn/ui-shared-deps'; - -function generateDLL(config) { - const { - dllAlias, - dllValidateSyntax, - dllNoParseRules, - dllContext, - dllEntry, - dllOutputPath, - dllPublicPath, - dllBundleName, - dllBundleFilename, - dllStyleFilename, - dllManifestPath, - babelLoaderCacheDir, - threadLoaderPoolConfig, - } = config; - - const BABEL_PRESET_PATH = require.resolve('@kbn/babel-preset/webpack_preset'); - const BABEL_EXCLUDE_RE = [/[\/\\](webpackShims|node_modules|bower_components)[\/\\]/]; - - /** - * Wrap plugin loading in a function so that we can require - * `@kbn/optimizer` only when absolutely necessary since we - * don't ship this package in the distributable but this code - * is still shipped, though it's not used. - */ - const getValidateSyntaxPlugins = () => { - if (!dllValidateSyntax) { - return []; - } - - // only require @kbn/optimizer - const { DisallowedSyntaxPlugin } = require('@kbn/optimizer'); - return [new DisallowedSyntaxPlugin()]; - }; - - return { - entry: dllEntry, - context: dllContext, - output: { - futureEmitAssets: true, // TODO: remove on webpack 5 - filename: dllBundleFilename, - path: dllOutputPath, - publicPath: dllPublicPath, - library: dllBundleName, - }, - node: { fs: 'empty', child_process: 'empty', dns: 'empty', net: 'empty', tls: 'empty' }, - resolve: { - extensions: ['.js', '.json'], - mainFields: ['browser', 'browserify', 'main'], - alias: dllAlias, - modules: ['webpackShims', fromRoot('webpackShims'), 'node_modules', fromRoot('node_modules')], - }, - module: { - rules: [ - { - resource: [ - { - test: /\.js$/, - exclude: BABEL_EXCLUDE_RE.concat(dllNoParseRules), - }, - { - test: /\.js$/, - include: /[\/\\]node_modules[\/\\]x-pack[\/\\]/, - exclude: /[\/\\]node_modules[\/\\]x-pack[\/\\](.+?[\/\\])*node_modules[\/\\]/, - }, - // TODO: remove when we drop support for IE11 - // We need because normalize-url is distributed without - // any kind of transpilation - // More info: https://github.com/elastic/kibana/pull/35804 - { - test: /\.js$/, - include: /[\/\\]node_modules[\/\\]normalize-url[\/\\]/, - exclude: /[\/\\]node_modules[\/\\]normalize-url[\/\\](.+?[\/\\])*node_modules[\/\\]/, - }, - ], - // Self calling function with the equivalent logic - // from maybeAddCacheLoader one from base optimizer - use: ((babelLoaderCacheDirPath, loaders) => { - return [ - { - loader: 'cache-loader', - options: { - cacheContext: fromRoot('.'), - cacheDirectory: babelLoaderCacheDirPath, - readOnly: process.env.KBN_CACHE_LOADER_WRITABLE ? false : IS_KIBANA_DISTRIBUTABLE, - }, - }, - ...loaders, - ]; - })(babelLoaderCacheDir, [ - { - loader: 'thread-loader', - options: threadLoaderPoolConfig, - }, - { - loader: 'babel-loader', - options: { - babelrc: false, - presets: [BABEL_PRESET_PATH], - }, - }, - ]), - }, - { - test: /\.(html|tmpl)$/, - loader: 'raw-loader', - }, - { - test: /\.css$/, - use: [MiniCssExtractPlugin.loader, 'css-loader'], - }, - { - test: /\.png$/, - loader: 'url-loader', - }, - { - test: /\.(woff|woff2|ttf|eot|svg|ico)(\?|$)/, - loader: 'file-loader', - }, - ], - noParse: dllNoParseRules, - }, - plugins: [ - new webpack.DllPlugin({ - context: dllContext, - name: dllBundleName, - path: dllManifestPath, - }), - new MiniCssExtractPlugin({ - filename: dllStyleFilename, - }), - ...getValidateSyntaxPlugins(), - ], - // Single runtime for the dll bundles which assures that common transient dependencies won't be evaluated twice. - // The module cache will be shared, even when module code may be duplicated across chunks. - optimization: { - runtimeChunk: { - name: 'vendors_runtime', - }, - }, - performance: { - // NOTE: we are disabling this as those hints - // are more tailored for the final bundles result - // and not for the webpack compilations performance itself - hints: false, - }, - externals: { - ...UiSharedDeps.externals, - }, - }; -} - -function extendRawConfig(rawConfig) { - // Build all extended configs from raw config - const dllAlias = rawConfig.uiBundles.getAliases(); - const dllValidateSyntax = rawConfig.uiBundles.shouldValidateSyntaxOfNodeModules(); - const dllNoParseRules = rawConfig.uiBundles.getWebpackNoParseRules(); - const dllDevMode = rawConfig.uiBundles.isDevMode(); - const dllContext = rawConfig.context; - const dllChunks = rawConfig.chunks; - const dllEntry = {}; - const dllEntryName = rawConfig.entryName; - const dllBundleName = rawConfig.dllName; - const dllManifestName = rawConfig.dllName; - const dllStyleName = rawConfig.styleName; - const dllEntryExt = rawConfig.entryExt; - const dllBundleExt = rawConfig.dllExt; - const dllManifestExt = rawConfig.manifestExt; - const dllStyleExt = rawConfig.styleExt; - const dllOutputPath = rawConfig.outputPath; - const dllPublicPath = rawConfig.publicPath; - const dllBundleFilename = `${dllBundleName}${dllBundleExt}`; - const dllManifestPath = `${dllOutputPath}/${dllManifestName}${dllManifestExt}`; - const dllStyleFilename = `${dllStyleName}${dllStyleExt}`; - const babelLoaderCacheDir = rawConfig.babelLoaderCacheDir; - const threadLoaderPoolConfig = rawConfig.threadLoaderPoolConfig; - - // Create webpack entry object key with the provided dllEntryName - dllChunks.reduce((dllEntryObj, chunk) => { - dllEntryObj[`${dllEntryName}${chunk}`] = [ - `${dllOutputPath}/${dllEntryName}${chunk}${dllEntryExt}`, - ]; - return dllEntryObj; - }, dllEntry); - - // Export dll config map - return { - dllAlias, - dllValidateSyntax, - dllNoParseRules, - dllDevMode, - dllContext, - dllEntry, - dllOutputPath, - dllPublicPath, - dllBundleName, - dllBundleFilename, - dllStyleFilename, - dllManifestPath, - babelLoaderCacheDir, - threadLoaderPoolConfig, - }; -} - -function common(config) { - return webpackMerge(generateDLL(config)); -} - -function optimized() { - return webpackMerge({ - mode: 'production', - optimization: { - minimizer: [ - new TerserPlugin({ - // NOTE: we should not enable that option for now - // Since 2.0.0 terser-webpack-plugin is using jest-worker - // to run tasks in a pool of workers. Currently it looks like - // is requiring too much memory and break on large entry points - // compilations (like this) one. Also the gain we have enabling - // that option was barely noticed. - // https://github.com/webpack-contrib/terser-webpack-plugin/issues/143 - parallel: false, - sourceMap: false, - cache: false, - extractComments: false, - terserOptions: { - compress: false, - mangle: false, - }, - }), - ], - }, - }); -} - -function unoptimized() { - return webpackMerge({ - mode: 'development', - }); -} - -export function configModel(rawConfig = {}) { - const config = extendRawConfig(rawConfig); - - if (config.dllDevMode) { - return webpackMerge(common(config), unoptimized()); - } - - return webpackMerge(common(config), optimized()); -} diff --git a/src/optimize/dynamic_dll_plugin/dll_entry_template.js b/src/optimize/dynamic_dll_plugin/dll_entry_template.js deleted file mode 100644 index 351bed4e591ba..0000000000000 --- a/src/optimize/dynamic_dll_plugin/dll_entry_template.js +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -export function dllEntryTemplate(requirePaths = []) { - return requirePaths - .map((path) => `require('${path}');`) - .sort() - .join('\n'); -} - -export function dllEntryFileContentStringToArray(content = '') { - return content.split('\n'); -} - -export function dllEntryFileContentArrayToString(content = []) { - return content.join('\n'); -} - -export function dllMergeAllEntryFilesContent(content = []) { - return content.join('\n').split('\n').sort().join('\n'); -} diff --git a/src/optimize/dynamic_dll_plugin/dynamic_dll_plugin.js b/src/optimize/dynamic_dll_plugin/dynamic_dll_plugin.js deleted file mode 100644 index fb6f6e097b5cd..0000000000000 --- a/src/optimize/dynamic_dll_plugin/dynamic_dll_plugin.js +++ /dev/null @@ -1,354 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { DllCompiler } from './dll_compiler'; -import { notInNodeModulesOrWebpackShims, inPluginNodeModules } from './dll_allowed_modules'; -import { IS_KIBANA_DISTRIBUTABLE } from '../../legacy/utils'; -import { dllEntryTemplate } from './dll_entry_template'; -import RawModule from 'webpack/lib/RawModule'; -import webpack from 'webpack'; -import path from 'path'; -import normalizePosixPath from 'normalize-path'; -import fs from 'fs'; -import { promisify } from 'util'; - -const realPathAsync = promisify(fs.realpath); -const DLL_ENTRY_STUB_MODULE_TYPE = 'javascript/dll-entry-stub'; - -export class DynamicDllPlugin { - constructor({ uiBundles, threadLoaderPoolConfig, logWithMetadata, maxCompilations = 1 }) { - this.logWithMetadata = logWithMetadata || (() => null); - this.dllCompiler = new DllCompiler(uiBundles, threadLoaderPoolConfig, logWithMetadata); - this.entryPaths = dllEntryTemplate(); - this.afterCompilationEntryPaths = dllEntryTemplate(); - this.maxCompilations = maxCompilations; - this.performedCompilations = 0; - this.forceDLLCreationFlag = !!(process && process.env && process.env.FORCE_DLL_CREATION); - } - - async init() { - await this.dllCompiler.init(); - this.entryPaths = await this.dllCompiler.readEntryFiles(); - } - - apply(compiler) { - // Just register the init basic hooks - // in order to run the init function - this.registerInitBasicHooks(compiler); - // The dll reference should always be bind to the - // main webpack config. - this.bindDllReferencePlugin(compiler); - - // Verify if we must init and run the dynamic dll plugin tasks. - // We must run it every time we are not under a distributable env - if (!this.mustRunDynamicDllPluginTasks()) { - return; - } - - // This call init all the DynamicDllPlugin tasks - // as it attaches the plugin to the main webpack - // lifecycle hooks needed to perform the logic - this.registerTasksHooks(compiler); - } - - bindDllReferencePlugin(compiler) { - const rawDllConfig = this.dllCompiler.rawDllConfig; - const dllContext = rawDllConfig.context; - const dllManifestPaths = this.dllCompiler.getManifestPaths(); - - dllManifestPaths.forEach((dllChunkManifestPath) => { - new webpack.DllReferencePlugin({ - context: dllContext, - manifest: dllChunkManifestPath, - }).apply(compiler); - }); - } - - registerInitBasicHooks(compiler) { - this.registerRunHook(compiler); - this.registerWatchRunHook(compiler); - } - - registerTasksHooks(compiler) { - this.logWithMetadata( - ['info', 'optimize:dynamic_dll_plugin'], - 'Started dynamic dll plugin tasks' - ); - this.registerBeforeCompileHook(compiler); - this.registerCompilationHook(compiler); - this.registerDoneHook(compiler); - } - - registerRunHook(compiler) { - compiler.hooks.run.tapPromise('DynamicDllPlugin', async () => { - await this.init(); - }); - } - - registerWatchRunHook(compiler) { - compiler.hooks.watchRun.tapPromise('DynamicDllPlugin', async () => { - await this.init(); - }); - } - - registerBeforeCompileHook(compiler) { - compiler.hooks.beforeCompile.tapPromise('DynamicDllPlugin', async ({ normalModuleFactory }) => { - normalModuleFactory.hooks.factory.tap('DynamicDllPlugin', (actualFactory) => (params, cb) => { - // This is used in order to avoid the cache for DLL modules - // resolved from other dependencies - normalModuleFactory.cachePredicate = (module) => - !(module.stubType === DLL_ENTRY_STUB_MODULE_TYPE); - - // Overrides the normalModuleFactory module creation behaviour - // in order to understand the modules we need to add to the DLL - actualFactory(params, (error, module) => { - if (error || !module) { - cb(error, module); - } else { - this.mapNormalModule(module).then( - (m = module) => cb(undefined, m), - (error) => cb(error) - ); - } - }); - }); - }); - } - - registerCompilationHook(compiler) { - compiler.hooks.compilation.tap('DynamicDllPlugin', (compilation) => { - compilation.hooks.needAdditionalPass.tap('DynamicDllPlugin', () => { - // Run the procedures in order to execute our dll compilation - // The process is very straightforward in it's conception: - // - // * 1 - loop through every compilation module in order to start building - // the dll entry paths arrays and assume it is the new entry paths - // * 1.1 - start from adding the modules already included into the dll, if any. - // * 1.2 - adding the new discovered stub modules - // * 1.3 - check if the module added to the entry path is from node_modules or - // webpackShims, otherwise throw an error. - // * 1.3.1 - for the entry path modules coming from webpackShims search for every - // require statements inside of them - // * 1.3.2 - discard the ones that are not js dependencies - // * 1.3.3 - add those new discovered dependencies inside the webpackShims to the - // entry paths array - // * 2 - compare the built entry paths and compares it to the old one (if any) - // * 3 - runs a new dll compilation in case there is none old entry paths or if the - // new built one differs from the old one. - // - const rawDllConfig = this.dllCompiler.rawDllConfig; - const dllContext = rawDllConfig.context; - const dllOutputPath = rawDllConfig.outputPath; - const requiresMap = {}; - - for (const module of compilation.modules) { - // re-include requires for modules already handled by the dll - if (module.delegateData) { - const absoluteResource = path.resolve(dllContext, module.userRequest); - if ( - absoluteResource.includes('node_modules') || - absoluteResource.includes('webpackShims') - ) { - // NOTE: normalizePosixPath is been used as we only want to have posix - // paths inside our final dll entry file - requiresMap[ - normalizePosixPath(path.relative(dllOutputPath, absoluteResource)) - ] = true; - } - } - - // include requires for modules that need to be added to the dll - if (module.stubType === DLL_ENTRY_STUB_MODULE_TYPE) { - requiresMap[ - normalizePosixPath(path.relative(dllOutputPath, module.stubResource)) - ] = true; - } - } - - // Sort and join all the discovered require deps - // in order to create a consistent entry file - this.afterCompilationEntryPaths = dllEntryTemplate(Object.keys(requiresMap)); - // The dll compilation will run if on of the following conditions return true: - // 1 - the new generated entry paths are different from the - // old ones - // 2 - if no dll bundle is yet created - // 3 - if this.forceDLLCreationFlag were set from the node env var FORCE_DLL_CREATION and - // we are not running over the distributable. If we are running under the watch optimizer, - // this.forceDLLCreationFlag will only be applied in the very first execution, - // then will be set to false - compilation.needsDLLCompilation = - this.afterCompilationEntryPaths !== this.entryPaths || - !this.dllCompiler.dllsExistsSync() || - (this.isToForceDLLCreation() && this.performedCompilations === 0); - this.entryPaths = this.afterCompilationEntryPaths; - - // Only run this info log in the first performed dll compilation - // per each execution run - if (this.performedCompilations === 0) { - this.logWithMetadata( - ['info', 'optimize:dynamic_dll_plugin'], - compilation.needsDLLCompilation - ? 'Need to compile the client vendors dll' - : 'No need to compile client vendors dll' - ); - } - - return compilation.needsDLLCompilation; - }); - }); - } - - registerDoneHook(compiler) { - compiler.hooks.done.tapPromise('DynamicDllPlugin', async (stats) => { - if (stats.compilation.needsDLLCompilation) { - // Run the dlls compiler and increment - // the performed compilations - // - // NOTE: check the need for this extra try/catch after upgrading - // past webpack v4.29.3. For now it is needed so we can log the error - // otherwise the error log we'll get will be something like: [fatal] [object Object] - try { - await this.runDLLCompiler(compiler); - } catch (error) { - this.logWithMetadata(['error', 'optimize:dynamic_dll_plugin'], error.message); - throw error; - } - - return; - } - - this.performedCompilations = 0; - // reset this flag var set from the node env FORCE_DLL_CREATION on init, - // has the force_dll_creation is only valid for the very first run - if (this.forceDLLCreationFlag) { - this.forceDLLCreationFlag = false; - } - this.logWithMetadata( - ['info', 'optimize:dynamic_dll_plugin'], - 'Finished all dynamic dll plugin tasks' - ); - }); - } - - isToForceDLLCreation() { - return this.forceDLLCreationFlag; - } - - mustRunDynamicDllPluginTasks() { - return !IS_KIBANA_DISTRIBUTABLE || this.isToForceDLLCreation(); - } - - async mapNormalModule(module) { - // ignore anything that doesn't have a resource (ignored) or is already delegating to the DLL - if (!module.resource || module.delegateData) { - return; - } - - // ignore anything that needs special loaders or config - if (module.request.includes('!') || module.request.includes('?')) { - return; - } - - // ignore files that are not in node_modules - if (notInNodeModulesOrWebpackShims(module.resource)) { - return; - } - - // also ignore files that are symlinked into node_modules, but only - // do the `realpath` call after checking the plain resource path - if (notInNodeModulesOrWebpackShims(await realPathAsync(module.resource))) { - return; - } - - const dirs = module.resource.split(path.sep); - const nodeModuleName = dirs[dirs.lastIndexOf('node_modules') + 1]; - - // ignore webpack loader modules - if (nodeModuleName.endsWith('-loader')) { - return; - } - - // ignore modules from plugins - if (inPluginNodeModules(module.resource)) { - return; - } - - // also ignore files that are symlinked into plugins node_modules, but only - // do the `realpath` call after checking the plain resource path - if (inPluginNodeModules(await realPathAsync(module.resource))) { - return; - } - - // This is a StubModule (as a RawModule) in order - // to mimic the missing modules from the DLL and - // also hold useful metadata - const stubModule = new RawModule( - `/* pending dll entry */`, - `dll pending:${module.resource}`, - module.resource - ); - stubModule.stubType = DLL_ENTRY_STUB_MODULE_TYPE; - stubModule.stubResource = module.resource; - stubModule.stubOriginalModule = module; - - return stubModule; - } - - async assertMaxCompilations() { - // Logic to run the max compilation requirements. - // Only enable this for CI builds in order to ensure - // we have an healthy dll ecosystem. - if (this.performedCompilations === this.maxCompilations) { - throw new Error( - 'All the allowed dll compilations were already performed and one more is needed which is not possible' - ); - } - } - - async runDLLCompiler(mainCompiler) { - const runCompilerErrors = []; - - try { - await this.dllCompiler.run(this.entryPaths); - } catch (e) { - runCompilerErrors.push(e); - } - - try { - await this.assertMaxCompilations(); - } catch (e) { - runCompilerErrors.push(e); - } - - // We need to purge the cache into the inputFileSystem - // for every single built in previous compilation - // that we rely in next ones. - this.dllCompiler - .getManifestPaths() - .forEach((chunkDllManifestPath) => mainCompiler.inputFileSystem.purge(chunkDllManifestPath)); - - this.performedCompilations++; - - if (!runCompilerErrors.length) { - return; - } - - throw new Error(runCompilerErrors.join('\n-')); - } -} diff --git a/src/optimize/dynamic_dll_plugin/index.js b/src/optimize/dynamic_dll_plugin/index.js deleted file mode 100644 index 347811b5356b7..0000000000000 --- a/src/optimize/dynamic_dll_plugin/index.js +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -export { DynamicDllPlugin } from './dynamic_dll_plugin'; -export { DllCompiler } from './dll_compiler'; diff --git a/src/optimize/optimize_mixin.ts b/src/optimize/optimize_mixin.ts index 9a3f08e2f667e..947c918a567f5 100644 --- a/src/optimize/optimize_mixin.ts +++ b/src/optimize/optimize_mixin.ts @@ -22,8 +22,6 @@ import Hapi from 'hapi'; // @ts-ignore not TS yet import FsOptimizer from './fs_optimizer'; import { createBundlesRoute } from './bundles_route'; -// @ts-ignore not TS yet -import { DllCompiler } from './dynamic_dll_plugin'; import { fromRoot } from '../core/server/utils'; import { getNpUiPluginPublicDirs } from './np_ui_plugin_public_dirs'; import KbnServer, { KibanaConfig } from '../legacy/server/kbn_server'; @@ -40,7 +38,7 @@ export const optimizeMixin = async ( // bundles in a "middleware" style. // // the server listening on 5601 may be restarted a number of times, depending - // on the watch setup managed by the cli. It proxies all bundles/* and built_assets/dlls/* + // on the watch setup managed by the cli. It proxies all bundles/* // requests to the other server. The server on 5602 is long running, in order // to prevent complete rebuilds of the optimize content. const watch = config.get('optimize.watch'); @@ -53,7 +51,6 @@ export const optimizeMixin = async ( server.route( createBundlesRoute({ regularBundlesPath: uiBundles.getWorkingDir(), - dllBundlesPath: DllCompiler.getRawDllConfig().outputPath, basePublicPath: config.get('server.basePath'), builtCssPath: fromRoot('built_assets/css'), npUiPluginPublicDirs: getNpUiPluginPublicDirs(kbnServer), diff --git a/src/optimize/watch/optmzr_role.js b/src/optimize/watch/optmzr_role.js index ba8007e1065b4..0057c04219ec6 100644 --- a/src/optimize/watch/optmzr_role.js +++ b/src/optimize/watch/optmzr_role.js @@ -17,12 +17,8 @@ * under the License. */ -import { resolve } from 'path'; - import WatchServer from './watch_server'; import WatchOptimizer, { STATUS } from './watch_optimizer'; -import { DllCompiler } from '../dynamic_dll_plugin'; -import { WatchCache } from './watch_cache'; import { getNpUiPluginPublicDirs } from '../np_ui_plugin_public_dirs'; export default async (kbnServer, kibanaHapiServer, config) => { @@ -36,12 +32,6 @@ export default async (kbnServer, kibanaHapiServer, config) => { sourceMaps: config.get('optimize.sourceMaps'), workers: config.get('optimize.workers'), prebuild: config.get('optimize.watchPrebuild'), - watchCache: new WatchCache({ - logWithMetadata, - outputPath: config.get('path.data'), - dllsPath: DllCompiler.getRawDllConfig().outputPath, - cachePath: resolve(kbnServer.uiBundles.getCacheDirectory(), '../'), - }), }); const server = new WatchServer( diff --git a/src/optimize/watch/watch.js b/src/optimize/watch/watch.js index a284da11f294f..7774577fb7677 100644 --- a/src/optimize/watch/watch.js +++ b/src/optimize/watch/watch.js @@ -32,7 +32,7 @@ export default async (kbnServer) => { * while the optimizer is running * * server: this process runs the entire kibana server and proxies - * all requests for /bundles/* or /built_assets/dlls/* to the optmzr process + * all requests for /bundles/* to the optmzr process * * @param {string} process.env.kbnWorkerType */ diff --git a/src/optimize/watch/watch_cache.ts b/src/optimize/watch/watch_cache.ts deleted file mode 100644 index 40bd1d6075f47..0000000000000 --- a/src/optimize/watch/watch_cache.ts +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { createHash } from 'crypto'; -import { readFile, writeFile, readdir, unlink, rmdir } from 'fs'; -import { resolve } from 'path'; -import { promisify } from 'util'; -import path from 'path'; -import del from 'del'; -import normalizePosixPath from 'normalize-path'; - -const readAsync = promisify(readFile); -const writeAsync = promisify(writeFile); -const readdirAsync = promisify(readdir); -const unlinkAsync = promisify(unlink); -const rmdirAsync = promisify(rmdir); - -interface Params { - logWithMetadata: (tags: string[], message: string, metadata?: { [key: string]: any }) => void; - outputPath: string; - dllsPath: string; - cachePath: string; -} - -interface WatchCacheStateContent { - optimizerConfigSha?: string; - yarnLockSha?: string; -} - -export class WatchCache { - private readonly logWithMetadata: Params['logWithMetadata']; - private readonly outputPath: Params['outputPath']; - private readonly dllsPath: Params['dllsPath']; - private readonly cachePath: Params['cachePath']; - private readonly cacheState: WatchCacheStateContent; - private statePath: string; - private diskCacheState: WatchCacheStateContent; - private isInitialized: boolean; - - constructor(params: Params) { - this.logWithMetadata = params.logWithMetadata; - this.outputPath = params.outputPath; - this.dllsPath = params.dllsPath; - this.cachePath = params.cachePath; - - this.isInitialized = false; - this.statePath = ''; - this.cacheState = {}; - this.diskCacheState = {}; - this.cacheState.yarnLockSha = ''; - this.cacheState.optimizerConfigSha = ''; - } - - public async tryInit() { - if (!this.isInitialized) { - this.statePath = resolve(this.outputPath, 'watch_optimizer_cache_state.json'); - this.diskCacheState = await this.read(); - this.cacheState.yarnLockSha = await this.buildYarnLockSha(); - this.cacheState.optimizerConfigSha = await this.buildOptimizerConfigSha(); - this.isInitialized = true; - } - } - - public async tryReset() { - await this.tryInit(); - - if (!this.isResetNeeded()) { - return; - } - - await this.reset(); - } - - public async reset() { - this.logWithMetadata(['info', 'optimize:watch_cache'], 'The optimizer watch cache will reset'); - - // start by deleting the state file to lower the - // amount of time that another process might be able to - // successfully read it once we decide to delete it - await del(this.statePath, { force: true }); - - // delete everything in optimize/.cache directory - await recursiveDelete(normalizePosixPath(this.cachePath)); - - // delete dlls - await del(this.dllsPath); - - // re-write new cache state file - await this.write(); - - this.logWithMetadata(['info', 'optimize:watch_cache'], 'The optimizer watch cache has reset'); - } - - private async buildShaWithMultipleFiles(filePaths: string[]) { - const shaHash = createHash('sha1'); - - for (const filePath of filePaths) { - try { - shaHash.update(await readAsync(filePath, 'utf8'), 'utf8'); - } catch (e) { - /* no-op */ - } - } - - return shaHash.digest('hex'); - } - - private async buildYarnLockSha() { - const kibanaYarnLock = resolve(__dirname, '../../../yarn.lock'); - - return await this.buildShaWithMultipleFiles([kibanaYarnLock]); - } - - private async buildOptimizerConfigSha() { - const baseOptimizer = resolve(__dirname, '../base_optimizer.js'); - const dynamicDllConfigModel = resolve(__dirname, '../dynamic_dll_plugin/dll_config_model.js'); - const dynamicDllPlugin = resolve(__dirname, '../dynamic_dll_plugin/dynamic_dll_plugin.js'); - - return await this.buildShaWithMultipleFiles([ - baseOptimizer, - dynamicDllConfigModel, - dynamicDllPlugin, - ]); - } - - private isResetNeeded() { - return this.hasYarnLockChanged() || this.hasOptimizerConfigChanged(); - } - - private hasYarnLockChanged() { - return this.cacheState.yarnLockSha !== this.diskCacheState.yarnLockSha; - } - - private hasOptimizerConfigChanged() { - return this.cacheState.optimizerConfigSha !== this.diskCacheState.optimizerConfigSha; - } - - private async write() { - await writeAsync(this.statePath, JSON.stringify(this.cacheState, null, 2), 'utf8'); - this.diskCacheState = this.cacheState; - } - - private async read(): Promise { - try { - return JSON.parse(await readAsync(this.statePath, 'utf8')); - } catch (error) { - return {}; - } - } -} - -/** - * Recursively deletes a folder. This is a workaround for a bug in `del` where - * very large folders (with 84K+ files) cause a stack overflow. - */ -async function recursiveDelete(directory: string) { - try { - const entries = await readdirAsync(directory, { withFileTypes: true }); - - await Promise.all( - entries.map((entry) => { - const absolutePath = path.join(directory, entry.name); - return entry.isDirectory() ? recursiveDelete(absolutePath) : unlinkAsync(absolutePath); - }) - ); - - return rmdirAsync(directory); - } catch (error) { - if (error.code !== 'ENOENT') { - throw error; - } - } -} diff --git a/src/optimize/watch/watch_optimizer.js b/src/optimize/watch/watch_optimizer.js index 816185e544ab5..000c03ffb34fe 100644 --- a/src/optimize/watch/watch_optimizer.js +++ b/src/optimize/watch/watch_optimizer.js @@ -19,7 +19,6 @@ import BaseOptimizer from '../base_optimizer'; import { createBundlesRoute } from '../bundles_route'; -import { DllCompiler } from '../dynamic_dll_plugin'; import { fromRoot } from '../../core/server/utils'; import * as Rx from 'rxjs'; import { mergeMap, take } from 'rxjs/operators'; @@ -35,7 +34,6 @@ export default class WatchOptimizer extends BaseOptimizer { constructor(opts) { super(opts); this.prebuild = opts.prebuild || false; - this.watchCache = opts.watchCache; this.status$ = new Rx.ReplaySubject(1); } @@ -43,9 +41,6 @@ export default class WatchOptimizer extends BaseOptimizer { this.initializing = true; this.initialBuildComplete = false; - // try reset the watch optimizer cache - await this.watchCache.tryReset(); - // log status changes this.status$.subscribe(this.onStatusChangeHandler); await this.uiBundles.resetBundleDir(); @@ -120,7 +115,6 @@ export default class WatchOptimizer extends BaseOptimizer { npUiPluginPublicDirs: npUiPluginPublicDirs, buildHash, regularBundlesPath: this.compiler.outputPath, - dllBundlesPath: DllCompiler.getRawDllConfig().outputPath, basePublicPath: basePath, builtCssPath: fromRoot('built_assets/css'), }) diff --git a/tasks/config/karma.js b/tasks/config/karma.js index fa4bdc8ed2266..114e09876406c 100644 --- a/tasks/config/karma.js +++ b/tasks/config/karma.js @@ -21,7 +21,6 @@ import { dirname } from 'path'; import { times } from 'lodash'; import { makeJunitReportPath } from '@kbn/test'; import * as UiSharedDeps from '@kbn/ui-shared-deps'; -import { DllCompiler } from '../../src/optimize/dynamic_dll_plugin'; const TOTAL_CI_SHARDS = 4; const ROOT = dirname(require.resolve('../../package.json')); @@ -63,12 +62,6 @@ module.exports = function (grunt) { ), `http://localhost:5610/${buildHash}/bundles/kbn-ui-shared-deps/${UiSharedDeps.jsFilename}`, - `http://localhost:5610/${buildHash}/built_assets/dlls/vendors_runtime.bundle.dll.js`, - ...DllCompiler.getRawDllConfig().chunks.map( - (chunk) => - `http://localhost:5610/${buildHash}/built_assets/dlls/vendors${chunk}.bundle.dll.js` - ), - shardNum === undefined ? `http://localhost:5610/${buildHash}/bundles/tests.bundle.js` : `http://localhost:5610/${buildHash}/bundles/tests.bundle.js?shards=${TOTAL_CI_SHARDS}&shard_num=${shardNum}`, @@ -77,10 +70,6 @@ module.exports = function (grunt) { // this causes tilemap tests to fail, probably because the eui styles haven't been // included in the karma harness a long some time, if ever // `http://localhost:5610/bundles/kbn-ui-shared-deps/${UiSharedDeps.lightCssDistFilename}`, - ...DllCompiler.getRawDllConfig().chunks.map( - (chunk) => - `http://localhost:5610/${buildHash}/built_assets/dlls/vendors${chunk}.style.dll.css` - ), `http://localhost:5610/${buildHash}/bundles/tests.style.css`, ]; } @@ -133,7 +122,6 @@ module.exports = function (grunt) { '/tests/': 'http://localhost:5610/tests/', '/test_bundle/': 'http://localhost:5610/test_bundle/', [`/${buildHash}/bundles/`]: `http://localhost:5610/${buildHash}/bundles/`, - [`/${buildHash}/built_assets/dlls/`]: `http://localhost:5610/${buildHash}/built_assets/dlls/`, }, client: {