Skip to content

Commit

Permalink
refactor: works towards dynamic flatpack (#6264)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Jason3S
Jason3S committed Sep 22, 2024
1 parent 5a09a97 commit 1bd64e9
Show file tree
Hide file tree
Showing 5 changed files with 555 additions and 10 deletions.
81 changes: 76 additions & 5 deletions packages/flatpack-json/src/Flatpack.mts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import assert from 'node:assert';

import { FlatpackedWrapper } from './flatpackUtil.mjs';
import { proxyDate, proxyObject, proxySet } from './proxy.mjs';
import {
ArrayRefElement,
BigIntRefElement,
Expand All @@ -8,6 +11,7 @@ import {
ObjectRefElement,
ObjectWrapperRefElement,
PrimitiveRefElement,
PrimitiveRefElementBase,
RefElements,
RegExpRefElement,
SetRefElement,
Expand Down Expand Up @@ -76,6 +80,8 @@ export class FlatpackStore {
private cachedSets = new Map<ArrayRefElement | undefined, SetRefElement>();
private cachedMaps = new Map<ArrayRefElement | undefined, Map<ArrayRefElement | undefined, MapRefElement>>();

private cachedProxies = new WeakMap<RefElements, Serializable>();

/**
* Cache of strings that have been deduped and stored in the data array.
*/
Expand Down Expand Up @@ -275,7 +281,7 @@ export class FlatpackStore {

private dedupeSetRefs(value: Set<Serializable>, element: SetRefElement): SetRefElement {
if (!this.dedupe) return element;
const values = element.values();
const values = element.valueRefs();
const found = this.cachedSets.get(values);
if (!found) {
this.cachedSets.set(values, element);
Expand All @@ -287,6 +293,10 @@ export class FlatpackStore {
return found;
}

private proxySetRef(ref: SetRefElement): Set<Serializable> {
return proxySet(new Set(this.#toValue(ref.valueRefs()) as Serializable[]), () => {});
}

private createUniqueKeys(keys: Serializable[]): ArrayRefElement {
const cacheValue = false;
let k = this.arrToRef(keys, cacheValue);
Expand Down Expand Up @@ -320,8 +330,8 @@ export class FlatpackStore {

private dedupeMapRefs(value: Map<Serializable, Serializable>, element: MapRefElement): MapRefElement {
if (!this.dedupe) return element;
const keys = element.keys();
const values = element.values();
const keys = element.keyRefs();
const values = element.valueRefs();
let found = this.cachedMaps.get(keys);
if (!found) {
found = new Map();
Expand All @@ -340,6 +350,10 @@ export class FlatpackStore {
return element;
}

private proxyMapRef(_ref: MapRefElement): Map<Serializable, Serializable> {
return new Map();
}

private cvtRegExpToRef(value: RegExp): RegExpRefElement {
const found = this.cache.get(value);
if (found !== undefined) {
Expand All @@ -361,6 +375,10 @@ export class FlatpackStore {
return this.addValueAndElement(value, new DateRefElement(value.getTime()));
}

private proxyDateRef(ref: DateRefElement): Date {
return proxyDate(ref.value, (date) => ref.setTime(date.getTime()));
}

private cvtBigintToRef(value: bigint): BigIntRefElement {
const found = this.cache.get(value);
if (found !== undefined) {
Expand Down Expand Up @@ -415,8 +433,8 @@ export class FlatpackStore {

private dedupeObject(value: PrimitiveObject | ObjectWrapper, element: ObjectRefElement): ObjectRefElement {
if (!this.dedupe) return element;
const keys = element.keys();
const values = element.values();
const keys = element.keyRefs();
const values = element.valueRefs();
let found = this.cachedObjects.get(keys);
if (!found) {
found = new Map();
Expand All @@ -435,6 +453,18 @@ export class FlatpackStore {
return element;
}

private proxyObjectRef(ref: ObjectRefElement): PrimitiveObject {
const keys = this.#toValue(ref.keyRefs()) as string[] | undefined;
const values = this.#toValue(ref.valueRefs()) as Serializable[] | undefined;
const obj = keys && values ? Object.fromEntries(keys.map((key, i) => [key, values[i]])) : {};
return proxyObject(obj, (_value) => {});
}

private proxyObjectWrapperRef(ref: ObjectWrapperRefElement): PrimitiveObject {
const value = Object(this.#toValue(ref.valueRef())) as PrimitiveObject;
return proxyObject(value, (_value) => {});
}

/**
*
* @param value - The array converted to an ArrayRefElement.
Expand Down Expand Up @@ -482,6 +512,11 @@ export class FlatpackStore {
return this.dedupeArray(value, element, cacheValue);
}

private proxyArrayRef(ref: ArrayRefElement): PrimitiveArray {
const arr = ref.valueRefs().map((v) => this.#toValue(v));
return proxyObject(arr, (_value) => {});
}

private valueToRef(value: Serializable): RefElements {
if (value === null) {
return this.primitiveToRef(value);
Expand Down Expand Up @@ -619,6 +654,26 @@ export class FlatpackStore {
}
}

#resolveToValueProxy(ref: RefElements | undefined): Unpacked {
if (!ref) return undefined;
if (ref instanceof ArrayRefElement) return this.proxyArrayRef(ref);
if (ref instanceof ObjectRefElement) return this.proxyObjectRef(ref);
if (ref instanceof PrimitiveRefElementBase) return ref.value;
if (isStringRefElements(ref)) return ref.value;
if (ref instanceof MapRefElement) return this.proxyMapRef(ref);
if (ref instanceof SetRefElement) return this.proxySetRef(ref);
if (ref instanceof BigIntRefElement) return ref.value;
if (ref instanceof RegExpRefElement) return ref.value;
if (ref instanceof DateRefElement) return this.proxyDateRef(ref);
if (ref instanceof ObjectWrapperRefElement) return this.proxyObjectWrapperRef(ref);
assert(false, 'Unknown ref type');
}

#toValue(ref: RefElements | undefined): Unpacked {
if (!ref) return undefined;
return getOrResolve(this.cachedProxies, ref, (ref) => this.#resolveToValueProxy(ref));
}

toJSON(): Flatpacked {
const data = [dataHeader] as Flatpacked;
const idxLookup = this.assignedElements;
Expand Down Expand Up @@ -654,6 +709,10 @@ export class FlatpackStore {
toValue(): Unpacked {
return fromJSON(this.toJSON());
}

_toValueProxy(): Unpacked {
return this.#toValue(this.root);
}
}

type TrieData = StringRefElements;
Expand Down Expand Up @@ -686,3 +745,15 @@ export function toJSON<V extends Serializable>(json: V, options?: FlatpackOption
export function stringify(data: Unpacked, pretty = true): string {
return pretty ? stringifyFlatpacked(toJSON(data)) : JSON.stringify(toJSON(data));
}

type WeakOrNever<K, V> = K extends WeakKey ? WeakMap<K, V> : never;
type SupportedMap<K, V> = Map<K, V> | WeakOrNever<K, V>;

function getOrResolve<K, V>(map: SupportedMap<K, V>, key: K, resolver: (key: K) => V): V {
let value = map.get(key);
if (value === undefined && !map.has(key)) {
value = resolver(key);
map.set(key, value);
}
return value as V;
}
49 changes: 49 additions & 0 deletions packages/flatpack-json/src/Flatpack.test.mts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { describe, expect, test } from 'vitest';
import { FlatpackStore, stringify, toJSON } from './Flatpack.mjs';
import { stringifyFlatpacked } from './stringify.mjs';
import { fromJSON } from './unpack.mjs';
import { deepEqual } from './proxy.mts';

const urlFileList = new URL('../fixtures/fileList.txt', import.meta.url);
const baseFilename = new URL(import.meta.url).pathname.split('/').slice(-1).join('').split('.').slice(0, -2).join('.');
Expand Down Expand Up @@ -189,6 +190,23 @@ describe('Flatpack', async () => {
expect(fromJSON(fp.toJSON())).toEqual(data);
expect(fp.toJSON()).not.toEqual(v);
});

test.each`
data
${undefined}
${'string'}
${1}
${1.1}
${null}
${true}
${false}
${new Date()}
${/[a-z]+/}
`('toValue $data', ({ data }) => {
const fp = new FlatpackStore(data);
expect(fp.toValue()).toEqual(data);
expect(fp._toValueProxy()).toEqual(data);
});
});

async function sampleFileList() {
Expand Down Expand Up @@ -249,3 +267,34 @@ function sampleNestedData() {
cValues,
};
}

describe('Flatpack value proxy', () => {
test.each`
value
${undefined}
${'string'}
${1}
${1.1}
${null}
${true}
${false}
${[]}
${[1, 2]}
${['a', 'b', 'a', 'b']}
${{}}
${{ a: 1 }}
${{ a: { b: 1 } }}
${{ a: { a: 'a', b: 42 } }}
${{ a: [1] }}
${new Set(['apple', 'banana', 'pineapple'])}
${new Map([['apple', 1], ['banana', 2], ['pineapple', 3]])}
${/[\p{L}\p{M}]+/gu}
${new Date('2024-01-01')}
`('identity $value', ({ value }) => {
const fp = new FlatpackStore(value);
const proxy = fp._toValueProxy();
expect(deepEqual(proxy, value)).toBe(true);
!(proxy instanceof Map || proxy instanceof Set) && expect(proxy).toEqual(value);
expect(fp._toValueProxy()).toBe(proxy);
});
});
30 changes: 25 additions & 5 deletions packages/flatpack-json/src/RefElements.mts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,11 +124,11 @@ export class ObjectRefElement extends BaseRefElement implements RefElement<Objec
this.#v = values;
}

keys(): ArrayRefElement | undefined {
keyRefs(): ArrayRefElement | undefined {
return this.#k;
}

values(): ArrayRefElement | undefined {
valueRefs(): ArrayRefElement | undefined {
return this.#v;
}

Expand Down Expand Up @@ -174,6 +174,10 @@ export class ObjectWrapperRefElement extends BaseRefElement implements RefElemen
this.#v = value;
}

valueRef(): RefElements | undefined {
return this.#v;
}

toElement(lookup: FnIndexLookup): ObjectWrapperElement {
return [ElementType.Object, 0, lookup(this.#v)];
}
Expand Down Expand Up @@ -216,7 +220,7 @@ export class SetRefElement extends BaseRefElement implements RefElement<SetEleme
return this.#v ? [this.#v] : undefined;
}

values(): ArrayRefElement | undefined {
valueRefs(): ArrayRefElement | undefined {
return this.#v;
}

Expand Down Expand Up @@ -255,11 +259,11 @@ export class MapRefElement extends BaseRefElement implements RefElement<MapEleme
return [this.#k, this.#v].filter((r) => !!r);
}

keys(): ArrayRefElement | undefined {
keyRefs(): ArrayRefElement | undefined {
return this.#k;
}

values(): ArrayRefElement | undefined {
valueRefs(): ArrayRefElement | undefined {
return this.#v;
}

Expand All @@ -286,6 +290,10 @@ export class RegExpRefElement extends BaseRefElement implements RefElement<RegEx
return [ElementType.RegExp, lookup(this.#p), lookup(this.#f)];
}

get value(): RegExp {
return new RegExp(this.#p.value, this.#f.value);
}

clone(): RegExpRefElement {
return new RegExpRefElement(this.#p, this.#f);
}
Expand Down Expand Up @@ -315,6 +323,14 @@ export class DateRefElement extends BaseRefElement implements RefElement<DateEle
return [ElementType.Date, this.#v];
}

get value(): Date {
return new Date(this.#v);
}

setTime(time: number): void {
this.#v = time;
}

clone(): DateRefElement {
return new DateRefElement(this.#v);
}
Expand All @@ -336,6 +352,10 @@ export class BigIntRefElement extends BaseRefElement implements RefElement<BigIn
this.#v = value;
}

get value(): bigint {
return BigInt(this.#v.value);
}

toElement(lookup: FnIndexLookup): BigIntElement {
return [ElementType.BigInt, lookup(this.#v)];
}
Expand Down
Loading

0 comments on commit 1bd64e9

Please sign in to comment.