Skip to content

Commit

Permalink
[7.x] [kbn/optimizer/node] properly separate lmdb databases, log bett…
Browse files Browse the repository at this point in the history 
…er (#83849) (#83941)

Co-authored-by: spalger <spalger@users.noreply.github.com>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: spalger <spalger@users.noreply.github.com>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
  • Loading branch information
@spalger @kibanamachine
3 people authored Nov 20, 2020
1 parent 447722e commit 79d4646
Show file tree
Hide file tree
Showing 5 changed files with 218 additions and 93 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -720,7 +720,7 @@
"less": "npm:@elastic/less@2.7.3-kibana",
"license-checker": "^16.0.0",
"listr": "^0.14.1",
"lmdb-store": "^0.6.10",
"lmdb-store": "^0.8.15",
"load-grunt-config": "^3.0.1",
"loader-utils": "^1.2.3",
"log-symbols": "^2.2.0",
Expand Down
154 changes: 82 additions & 72 deletions packages/kbn-optimizer/src/node/cache.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,91 +17,67 @@
* under the License.
*/

import Path from 'path';
import Fs from 'fs';
import { Writable } from 'stream';

// @ts-expect-error no types available
import chalk from 'chalk';
import * as LmdbStore from 'lmdb-store';
import { REPO_ROOT, UPSTREAM_BRANCH } from '@kbn/dev-utils';

const LMDB_PKG = JSON.parse(
Fs.readFileSync(Path.resolve(REPO_ROOT, 'node_modules/lmdb-store/package.json'), 'utf8')
);
const CACHE_DIR = Path.resolve(
REPO_ROOT,
`data/node_auto_transpilation_cache/lmdb-${LMDB_PKG.version}/${UPSTREAM_BRANCH}`
);

const reportError = () => {
// right now I'm not sure we need to worry about errors, the cache isn't actually
// necessary, and if the cache is broken it should just rebuild on the next restart
// of the process. We don't know how often errors occur though and what types of
// things might fail on different machines so we probably want some way to signal
// to users that something is wrong
};

const GLOBAL_ATIME = `${Date.now()}`;
const MINUTE = 1000 * 60;
const HOUR = MINUTE * 60;
const DAY = HOUR * 24;

interface Lmdb<T> {
name: string;
get(key: string): T | undefined;
put(key: string, value: T, version?: number, ifVersion?: number): Promise<boolean>;
remove(key: string, ifVersion?: number): Promise<boolean>;
removeSync(key: string): void;
openDB<T2>(options: {
name: string;
encoding: 'msgpack' | 'string' | 'json' | 'binary';
}): Lmdb<T2>;
getRange(options?: {
start?: T;
end?: T;
reverse?: boolean;
limit?: number;
versions?: boolean;
}): Iterable<{ key: string; value: T }>;
}
const dbName = (db: LmdbStore.Database) =>
// @ts-expect-error db.name is not a documented/typed property
db.name;

export class Cache {
private readonly codes: Lmdb<string>;
private readonly atimes: Lmdb<string>;
private readonly mtimes: Lmdb<string>;
private readonly sourceMaps: Lmdb<any>;
private readonly codes: LmdbStore.RootDatabase<string, string>;
private readonly atimes: LmdbStore.Database<string, string>;
private readonly mtimes: LmdbStore.Database<string, string>;
private readonly sourceMaps: LmdbStore.Database<string, string>;
private readonly prefix: string;
private readonly log?: Writable;
private readonly timer: NodeJS.Timer;

constructor(config: { prefix: string }) {
constructor(config: { dir: string; prefix: string; log?: Writable }) {
this.prefix = config.prefix;
this.log = config.log;

this.codes = LmdbStore.open({
this.codes = LmdbStore.open(config.dir, {
name: 'codes',
path: CACHE_DIR,
encoding: 'string',
maxReaders: 500,
});

this.atimes = this.codes.openDB({
// TODO: redundant 'name' syntax is necessary because of a bug that I have yet to fix
this.atimes = this.codes.openDB('atimes', {
name: 'atimes',
encoding: 'string',
});

this.mtimes = this.codes.openDB({
this.mtimes = this.codes.openDB('mtimes', {
name: 'mtimes',
encoding: 'string',
});

this.sourceMaps = this.codes.openDB({
this.sourceMaps = this.codes.openDB('sourceMaps', {
name: 'sourceMaps',
encoding: 'msgpack',
encoding: 'string',
});

// after the process has been running for 30 minutes prune the
// keys which haven't been used in 30 days. We use `unref()` to
// make sure this timer doesn't hold other processes open
// unexpectedly
setTimeout(() => {
this.timer = setTimeout(() => {
this.pruneOldKeys();
}, 30 * MINUTE).unref();
}, 30 * MINUTE);

// timer.unref is not defined in jest which emulates the dom by default
if (typeof this.timer.unref === 'function') {
this.timer.unref();
}
}

getMtime(path: string) {
Expand All @@ -110,45 +86,78 @@ export class Cache {

getCode(path: string) {
const key = this.getKey(path);
const code = this.safeGet(this.codes, key);

// when we use a file from the cache set the "atime" of that cache entry
// so that we know which cache items we use and which haven't been
// touched in a long time (currently 30 days)
this.atimes.put(key, GLOBAL_ATIME).catch(reportError);
if (code !== undefined) {
// when we use a file from the cache set the "atime" of that cache entry
// so that we know which cache items we use and which haven't been
// touched in a long time (currently 30 days)
this.safePut(this.atimes, key, GLOBAL_ATIME);
}

return this.safeGet(this.codes, key);
return code;
}

getSourceMap(path: string) {
return this.safeGet(this.sourceMaps, this.getKey(path));
const map = this.safeGet(this.sourceMaps, this.getKey(path));
if (typeof map === 'string') {
return JSON.parse(map);
}
}

update(path: string, file: { mtime: string; code: string; map: any }) {
async update(path: string, file: { mtime: string; code: string; map: any }) {
const key = this.getKey(path);

Promise.all([
this.atimes.put(key, GLOBAL_ATIME),
this.mtimes.put(key, file.mtime),
this.codes.put(key, file.code),
this.sourceMaps.put(key, file.map),
]).catch(reportError);
await Promise.all([
this.safePut(this.atimes, key, GLOBAL_ATIME),
this.safePut(this.mtimes, key, file.mtime),
this.safePut(this.codes, key, file.code),
this.safePut(this.sourceMaps, key, JSON.stringify(file.map)),
]);
}

close() {
clearTimeout(this.timer);
}

private getKey(path: string) {
return `${this.prefix}${path}`;
}

private safeGet<V>(db: Lmdb<V>, key: string) {
private safeGet<V>(db: LmdbStore.Database<V, string>, key: string) {
try {
return db.get(key);
const value = db.get(key);
this.debug(value === undefined ? 'MISS' : 'HIT', db, key);
return value;
} catch (error) {
process.stderr.write(
`failed to read node transpilation [${db.name}] cache for [${key}]: ${error.stack}\n`
);
db.removeSync(key);
this.logError('GET', db, key, error);
}
}

private async safePut<V>(db: LmdbStore.Database<V, string>, key: string, value: V) {
try {
await db.put(key, value);
this.debug('PUT', db, key);
} catch (error) {
this.logError('PUT', db, key, error);
}
}

private debug(type: string, db: LmdbStore.Database, key: LmdbStore.Key) {
if (this.log) {
this.log.write(`${type} [${dbName(db)}] ${String(key)}\n`);
}
}

private logError(type: 'GET' | 'PUT', db: LmdbStore.Database, key: LmdbStore.Key, error: Error) {
this.debug(`ERROR/${type}`, db, `${String(key)}: ${error.stack}`);
process.stderr.write(
chalk.red(
`[@kbn/optimizer/node] ${type} error [${dbName(db)}/${String(key)}]: ${error.stack}\n`
)
);
}

private async pruneOldKeys() {
try {
const ATIME_LIMIT = Date.now() - 30 * DAY;
Expand All @@ -157,9 +166,10 @@ export class Cache {
const validKeys: string[] = [];
const invalidKeys: string[] = [];

// @ts-expect-error See https://github.com/DoctorEvidence/lmdb-store/pull/18
for (const { key, value } of this.atimes.getRange()) {
const atime = parseInt(value, 10);
if (atime < ATIME_LIMIT) {
const atime = parseInt(`${value}`, 10);
if (Number.isNaN(atime) || atime < ATIME_LIMIT) {
invalidKeys.push(key);
} else {
validKeys.push(key);
Expand Down
97 changes: 97 additions & 0 deletions packages/kbn-optimizer/src/node/integration_tests/cache.test.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import Path from 'path';
import { Writable } from 'stream';

import del from 'del';

import { Cache } from '../cache';

const DIR = Path.resolve(__dirname, '../__tmp__/cache');

const makeTestLog = () => {
const log = Object.assign(
new Writable({
write(chunk, enc, cb) {
log.output += chunk;
cb();
},
}),
{
output: '',
}
);

return log;
};

const instances: Cache[] = [];
const makeCache = (...options: ConstructorParameters<typeof Cache>) => {
const instance = new Cache(...options);
instances.push(instance);
return instance;
};

beforeEach(async () => await del(DIR));
afterEach(async () => {
await del(DIR);
for (const instance of instances) {
instance.close();
}
instances.length = 0;
});

it('returns undefined until values are set', async () => {
const path = '/foo/bar.js';
const mtime = new Date().toJSON();
const log = makeTestLog();
const cache = makeCache({
dir: DIR,
prefix: 'foo',
log,
});

expect(cache.getMtime(path)).toBe(undefined);
expect(cache.getCode(path)).toBe(undefined);
expect(cache.getSourceMap(path)).toBe(undefined);

await cache.update(path, {
mtime,
code: 'var x = 1',
map: { foo: 'bar' },
});

expect(cache.getMtime(path)).toBe(mtime);
expect(cache.getCode(path)).toBe('var x = 1');
expect(cache.getSourceMap(path)).toEqual({ foo: 'bar' });
expect(log.output).toMatchInlineSnapshot(`
"MISS [mtimes] foo/foo/bar.js
MISS [codes] foo/foo/bar.js
MISS [sourceMaps] foo/foo/bar.js
PUT [atimes] foo/foo/bar.js
PUT [mtimes] foo/foo/bar.js
PUT [codes] foo/foo/bar.js
PUT [sourceMaps] foo/foo/bar.js
HIT [mtimes] foo/foo/bar.js
HIT [codes] foo/foo/bar.js
HIT [sourceMaps] foo/foo/bar.js
"
`);
});
8 changes: 7 additions & 1 deletion packages/kbn-optimizer/src/node/node_auto_tranpilation.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ import Crypto from 'crypto';

import * as babel from '@babel/core';
import { addHook } from 'pirates';
import { REPO_ROOT } from '@kbn/dev-utils';
import { REPO_ROOT, UPSTREAM_BRANCH } from '@kbn/dev-utils';
import sourceMapSupport from 'source-map-support';

import { Cache } from './cache';
Expand Down Expand Up @@ -134,7 +134,13 @@ export function registerNodeAutoTranspilation() {
installed = true;

const cache = new Cache({
dir: Path.resolve(REPO_ROOT, 'data/node_auto_transpilation_cache', UPSTREAM_BRANCH),
prefix: determineCachePrefix(),
log: process.env.DEBUG_NODE_TRANSPILER_CACHE
? Fs.createWriteStream(Path.resolve(REPO_ROOT, 'node_auto_transpilation_cache.log'), {
flags: 'a',
})
: undefined,
});

sourceMapSupport.install({
Expand Down
Loading

0 comments on commit 79d4646

Please sign in to comment.