diff --git a/package.json b/package.json index 348eedeef30a9..916302aa06eae 100644 --- a/package.json +++ b/package.json @@ -720,7 +720,7 @@ "less": "npm:@elastic/less@2.7.3-kibana", "license-checker": "^16.0.0", "listr": "^0.14.1", - "lmdb-store": "^0.6.10", + "lmdb-store": "^0.8.15", "load-grunt-config": "^3.0.1", "loader-utils": "^1.2.3", "log-symbols": "^2.2.0", diff --git a/packages/kbn-optimizer/src/node/cache.ts b/packages/kbn-optimizer/src/node/cache.ts index e918bae86c835..417e38d5fb7ab 100644 --- a/packages/kbn-optimizer/src/node/cache.ts +++ b/packages/kbn-optimizer/src/node/cache.ts @@ -17,91 +17,67 @@ * under the License. */ -import Path from 'path'; -import Fs from 'fs'; +import { Writable } from 'stream'; -// @ts-expect-error no types available +import chalk from 'chalk'; import * as LmdbStore from 'lmdb-store'; -import { REPO_ROOT, UPSTREAM_BRANCH } from '@kbn/dev-utils'; - -const LMDB_PKG = JSON.parse( - Fs.readFileSync(Path.resolve(REPO_ROOT, 'node_modules/lmdb-store/package.json'), 'utf8') -); -const CACHE_DIR = Path.resolve( - REPO_ROOT, - `data/node_auto_transpilation_cache/lmdb-${LMDB_PKG.version}/${UPSTREAM_BRANCH}` -); - -const reportError = () => { - // right now I'm not sure we need to worry about errors, the cache isn't actually - // necessary, and if the cache is broken it should just rebuild on the next restart - // of the process. We don't know how often errors occur though and what types of - // things might fail on different machines so we probably want some way to signal - // to users that something is wrong -}; const GLOBAL_ATIME = `${Date.now()}`; const MINUTE = 1000 * 60; const HOUR = MINUTE * 60; const DAY = HOUR * 24; -interface Lmdb { - name: string; - get(key: string): T | undefined; - put(key: string, value: T, version?: number, ifVersion?: number): Promise; - remove(key: string, ifVersion?: number): Promise; - removeSync(key: string): void; - openDB(options: { - name: string; - encoding: 'msgpack' | 'string' | 'json' | 'binary'; - }): Lmdb; - getRange(options?: { - start?: T; - end?: T; - reverse?: boolean; - limit?: number; - versions?: boolean; - }): Iterable<{ key: string; value: T }>; -} +const dbName = (db: LmdbStore.Database) => + // @ts-expect-error db.name is not a documented/typed property + db.name; export class Cache { - private readonly codes: Lmdb; - private readonly atimes: Lmdb; - private readonly mtimes: Lmdb; - private readonly sourceMaps: Lmdb; + private readonly codes: LmdbStore.RootDatabase; + private readonly atimes: LmdbStore.Database; + private readonly mtimes: LmdbStore.Database; + private readonly sourceMaps: LmdbStore.Database; private readonly prefix: string; + private readonly log?: Writable; + private readonly timer: NodeJS.Timer; - constructor(config: { prefix: string }) { + constructor(config: { dir: string; prefix: string; log?: Writable }) { this.prefix = config.prefix; + this.log = config.log; - this.codes = LmdbStore.open({ + this.codes = LmdbStore.open(config.dir, { name: 'codes', - path: CACHE_DIR, + encoding: 'string', maxReaders: 500, }); - this.atimes = this.codes.openDB({ + // TODO: redundant 'name' syntax is necessary because of a bug that I have yet to fix + this.atimes = this.codes.openDB('atimes', { name: 'atimes', encoding: 'string', }); - this.mtimes = this.codes.openDB({ + this.mtimes = this.codes.openDB('mtimes', { name: 'mtimes', encoding: 'string', }); - this.sourceMaps = this.codes.openDB({ + this.sourceMaps = this.codes.openDB('sourceMaps', { name: 'sourceMaps', - encoding: 'msgpack', + encoding: 'string', }); // after the process has been running for 30 minutes prune the // keys which haven't been used in 30 days. We use `unref()` to // make sure this timer doesn't hold other processes open // unexpectedly - setTimeout(() => { + this.timer = setTimeout(() => { this.pruneOldKeys(); - }, 30 * MINUTE).unref(); + }, 30 * MINUTE); + + // timer.unref is not defined in jest which emulates the dom by default + if (typeof this.timer.unref === 'function') { + this.timer.unref(); + } } getMtime(path: string) { @@ -110,45 +86,78 @@ export class Cache { getCode(path: string) { const key = this.getKey(path); + const code = this.safeGet(this.codes, key); - // when we use a file from the cache set the "atime" of that cache entry - // so that we know which cache items we use and which haven't been - // touched in a long time (currently 30 days) - this.atimes.put(key, GLOBAL_ATIME).catch(reportError); + if (code !== undefined) { + // when we use a file from the cache set the "atime" of that cache entry + // so that we know which cache items we use and which haven't been + // touched in a long time (currently 30 days) + this.safePut(this.atimes, key, GLOBAL_ATIME); + } - return this.safeGet(this.codes, key); + return code; } getSourceMap(path: string) { - return this.safeGet(this.sourceMaps, this.getKey(path)); + const map = this.safeGet(this.sourceMaps, this.getKey(path)); + if (typeof map === 'string') { + return JSON.parse(map); + } } - update(path: string, file: { mtime: string; code: string; map: any }) { + async update(path: string, file: { mtime: string; code: string; map: any }) { const key = this.getKey(path); - Promise.all([ - this.atimes.put(key, GLOBAL_ATIME), - this.mtimes.put(key, file.mtime), - this.codes.put(key, file.code), - this.sourceMaps.put(key, file.map), - ]).catch(reportError); + await Promise.all([ + this.safePut(this.atimes, key, GLOBAL_ATIME), + this.safePut(this.mtimes, key, file.mtime), + this.safePut(this.codes, key, file.code), + this.safePut(this.sourceMaps, key, JSON.stringify(file.map)), + ]); + } + + close() { + clearTimeout(this.timer); } private getKey(path: string) { return `${this.prefix}${path}`; } - private safeGet(db: Lmdb, key: string) { + private safeGet(db: LmdbStore.Database, key: string) { try { - return db.get(key); + const value = db.get(key); + this.debug(value === undefined ? 'MISS' : 'HIT', db, key); + return value; } catch (error) { - process.stderr.write( - `failed to read node transpilation [${db.name}] cache for [${key}]: ${error.stack}\n` - ); - db.removeSync(key); + this.logError('GET', db, key, error); } } + private async safePut(db: LmdbStore.Database, key: string, value: V) { + try { + await db.put(key, value); + this.debug('PUT', db, key); + } catch (error) { + this.logError('PUT', db, key, error); + } + } + + private debug(type: string, db: LmdbStore.Database, key: LmdbStore.Key) { + if (this.log) { + this.log.write(`${type} [${dbName(db)}] ${String(key)}\n`); + } + } + + private logError(type: 'GET' | 'PUT', db: LmdbStore.Database, key: LmdbStore.Key, error: Error) { + this.debug(`ERROR/${type}`, db, `${String(key)}: ${error.stack}`); + process.stderr.write( + chalk.red( + `[@kbn/optimizer/node] ${type} error [${dbName(db)}/${String(key)}]: ${error.stack}\n` + ) + ); + } + private async pruneOldKeys() { try { const ATIME_LIMIT = Date.now() - 30 * DAY; @@ -157,9 +166,10 @@ export class Cache { const validKeys: string[] = []; const invalidKeys: string[] = []; + // @ts-expect-error See https://github.com/DoctorEvidence/lmdb-store/pull/18 for (const { key, value } of this.atimes.getRange()) { - const atime = parseInt(value, 10); - if (atime < ATIME_LIMIT) { + const atime = parseInt(`${value}`, 10); + if (Number.isNaN(atime) || atime < ATIME_LIMIT) { invalidKeys.push(key); } else { validKeys.push(key); diff --git a/packages/kbn-optimizer/src/node/integration_tests/cache.test.ts b/packages/kbn-optimizer/src/node/integration_tests/cache.test.ts new file mode 100644 index 0000000000000..c860164d4306a --- /dev/null +++ b/packages/kbn-optimizer/src/node/integration_tests/cache.test.ts @@ -0,0 +1,97 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import Path from 'path'; +import { Writable } from 'stream'; + +import del from 'del'; + +import { Cache } from '../cache'; + +const DIR = Path.resolve(__dirname, '../__tmp__/cache'); + +const makeTestLog = () => { + const log = Object.assign( + new Writable({ + write(chunk, enc, cb) { + log.output += chunk; + cb(); + }, + }), + { + output: '', + } + ); + + return log; +}; + +const instances: Cache[] = []; +const makeCache = (...options: ConstructorParameters) => { + const instance = new Cache(...options); + instances.push(instance); + return instance; +}; + +beforeEach(async () => await del(DIR)); +afterEach(async () => { + await del(DIR); + for (const instance of instances) { + instance.close(); + } + instances.length = 0; +}); + +it('returns undefined until values are set', async () => { + const path = '/foo/bar.js'; + const mtime = new Date().toJSON(); + const log = makeTestLog(); + const cache = makeCache({ + dir: DIR, + prefix: 'foo', + log, + }); + + expect(cache.getMtime(path)).toBe(undefined); + expect(cache.getCode(path)).toBe(undefined); + expect(cache.getSourceMap(path)).toBe(undefined); + + await cache.update(path, { + mtime, + code: 'var x = 1', + map: { foo: 'bar' }, + }); + + expect(cache.getMtime(path)).toBe(mtime); + expect(cache.getCode(path)).toBe('var x = 1'); + expect(cache.getSourceMap(path)).toEqual({ foo: 'bar' }); + expect(log.output).toMatchInlineSnapshot(` + "MISS [mtimes] foo/foo/bar.js + MISS [codes] foo/foo/bar.js + MISS [sourceMaps] foo/foo/bar.js + PUT [atimes] foo/foo/bar.js + PUT [mtimes] foo/foo/bar.js + PUT [codes] foo/foo/bar.js + PUT [sourceMaps] foo/foo/bar.js + HIT [mtimes] foo/foo/bar.js + HIT [codes] foo/foo/bar.js + HIT [sourceMaps] foo/foo/bar.js + " + `); +}); diff --git a/packages/kbn-optimizer/src/node/node_auto_tranpilation.ts b/packages/kbn-optimizer/src/node/node_auto_tranpilation.ts index ff6ab1c68da53..d85413c516aee 100644 --- a/packages/kbn-optimizer/src/node/node_auto_tranpilation.ts +++ b/packages/kbn-optimizer/src/node/node_auto_tranpilation.ts @@ -39,7 +39,7 @@ import Crypto from 'crypto'; import * as babel from '@babel/core'; import { addHook } from 'pirates'; -import { REPO_ROOT } from '@kbn/dev-utils'; +import { REPO_ROOT, UPSTREAM_BRANCH } from '@kbn/dev-utils'; import sourceMapSupport from 'source-map-support'; import { Cache } from './cache'; @@ -134,7 +134,13 @@ export function registerNodeAutoTranspilation() { installed = true; const cache = new Cache({ + dir: Path.resolve(REPO_ROOT, 'data/node_auto_transpilation_cache', UPSTREAM_BRANCH), prefix: determineCachePrefix(), + log: process.env.DEBUG_NODE_TRANSPILER_CACHE + ? Fs.createWriteStream(Path.resolve(REPO_ROOT, 'node_auto_transpilation_cache.log'), { + flags: 'a', + }) + : undefined, }); sourceMapSupport.install({ diff --git a/yarn.lock b/yarn.lock index 1155c0c296827..97809cdead633 100644 --- a/yarn.lock +++ b/yarn.lock @@ -18945,16 +18945,28 @@ livereload-js@^2.3.0: resolved "https://registry.yarnpkg.com/livereload-js/-/livereload-js-2.4.0.tgz#447c31cf1ea9ab52fc20db615c5ddf678f78009c" integrity sha512-XPQH8Z2GDP/Hwz2PCDrh2mth4yFejwA1OZ/81Ti3LgKyhDcEjsSsqFWZojHG0va/duGd+WyosY7eXLDoOyqcPw== -lmdb-store@^0.6.10: - version "0.6.10" - resolved "https://registry.yarnpkg.com/lmdb-store/-/lmdb-store-0.6.10.tgz#db8efde6e052aabd17ebc63c8a913e1f31694129" - integrity sha512-ZLvp3qbBQ5VlBmaWa4EUAPyYEZ8qdUHsW69HmxkDi84pFQ37WMxYhFaF/7PQkdtxS/vyiKkZigd9TFgHjek1Nw== +lmdb-store-0.9@0.7.3: + version "0.7.3" + resolved "https://registry.yarnpkg.com/lmdb-store-0.9/-/lmdb-store-0.9-0.7.3.tgz#c2cb27dfa916ab966cceed692c67e4236813104a" + integrity sha512-t8iCnN6T3NZPFelPmjYIjCg+nhGbOdc0xcHCG40v01AWRTN49OINSt2k/u+16/2/HrI+b6Ssb8WByXUhbyHz6w== + dependencies: + fs-extra "^9.0.1" + msgpackr "^0.5.3" + nan "^2.14.1" + node-gyp-build "^4.2.3" + weak-lru-cache "^0.3.9" + +lmdb-store@^0.8.15: + version "0.8.15" + resolved "https://registry.yarnpkg.com/lmdb-store/-/lmdb-store-0.8.15.tgz#4efb0341c2df505dd6f3a7f26f834f0a142a80a2" + integrity sha512-4Q0WZh2FmcJC6esZRUWMfkCmNiz0WU9cOgrxt97ZMTnVfHyOdZhtrt0oOF5EQPfetxxJf/BorKY28aX92R6G6g== dependencies: fs-extra "^9.0.1" - msgpackr "^0.5.0" + lmdb-store-0.9 "0.7.3" + msgpackr "^0.5.4" nan "^2.14.1" node-gyp-build "^4.2.3" - weak-lru-cache "^0.2.0" + weak-lru-cache "^0.3.9" load-bmfont@^1.3.1, load-bmfont@^1.4.0: version "1.4.0" @@ -20477,20 +20489,20 @@ ms@2.1.1, ms@^2.0.0, ms@^2.1.1: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a" integrity sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg== -msgpackr-extract@^0.3.4: - version "0.3.4" - resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-0.3.4.tgz#8ee5e73d1135340e564c498e8c593134365eb060" - integrity sha512-d3+qwTJzgqqsq2L2sQuH0SoO4StvpUhMqMAKy6tMimn7XdBaRtDlquFzRJsp0iMGt2hnU4UOqD8Tz9mb0KglTA== +msgpackr-extract@^0.3.5: + version "0.3.5" + resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-0.3.5.tgz#0f206da058bd3dad0f8605d324de001a8f4de967" + integrity sha512-zHhstybu+m/j3H6CVBMcILVIzATK6dWRGtlePJjsnSAj8kLT5joMa9i0v21Uc80BPNDcwFsnG/dz2318tfI81w== dependencies: nan "^2.14.1" node-gyp-build "^4.2.3" -msgpackr@^0.5.0: - version "0.5.1" - resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-0.5.1.tgz#7eecbf342645b7718dd2e3386894368d06732b3f" - integrity sha512-nK2uJl67Q5KU3MWkYBUlYynqKS1UUzJ5M1h6TQejuJtJzD3hW2Suv2T1pf01E9lUEr93xaLokf/xC+jwBShMPQ== +msgpackr@^0.5.3, msgpackr@^0.5.4: + version "0.5.4" + resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-0.5.4.tgz#c21c03d5e132d2e54d0b9ced02a75b1f48413380" + integrity sha512-ILEWtIWwd5ESWHKoVjJ4GP7JWkpuAUJ20qi2j2qEC6twecBmK4E6YG3QW847OpmvdAhMJGq2LoDJRn/kNERTeQ== optionalDependencies: - msgpackr-extract "^0.3.4" + msgpackr-extract "^0.3.5" multicast-dns-service-types@^1.1.0: version "1.1.0" @@ -29326,10 +29338,10 @@ wcwidth@^1.0.1: dependencies: defaults "^1.0.3" -weak-lru-cache@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-0.2.0.tgz#447379ccff6dfda1b7a9566c9ef168260be859d1" - integrity sha512-M1l5CzKvM7maa7tCbtL0NW6sOnp8gqup853+9Aq7GL0XNWKNnFOkeE3v3Z5X2IeMzedPwQyPbi4RlFvD6rxs7A== +weak-lru-cache@^0.3.9: + version "0.3.9" + resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-0.3.9.tgz#9e56920d4115e8542625d8ef8cc278cbd97f7624" + integrity sha512-WqAu3wzbHQvjSi/vgYhidZkf2p7L3Z8iDEIHnqvE31EQQa7Vh7PDOphrRJ1oxlW8JIjgr2HvMcRe9Q1GhW2NPw== web-namespaces@^1.0.0: version "1.1.4"