From 375c3e968cc3de0769aaf6ad9f4d2ed3954b0aa0 Mon Sep 17 00:00:00 2001 From: kumavis Date: Sun, 3 Mar 2024 12:37:04 -1000 Subject: [PATCH 01/20] feat(daemon): NameHubs expose listIdentifiers method --- packages/daemon/src/directory.js | 16 ++++++++++++++++ packages/daemon/src/guest.js | 2 ++ packages/daemon/src/host.js | 2 ++ packages/daemon/src/types.d.ts | 1 + 4 files changed, 21 insertions(+) diff --git a/packages/daemon/src/directory.js b/packages/daemon/src/directory.js index f519acdc46..0041f52b8c 100644 --- a/packages/daemon/src/directory.js +++ b/packages/daemon/src/directory.js @@ -95,6 +95,21 @@ export const makeDirectoryMaker = ({ return hub.list(); }; + /** @type {import('./types.js').EndoDirectory['listIdentifiers']} */ + const listIdentifiers = async (...petNamePath) => { + const names = await list(...petNamePath); + const identities = new Set(); + await Promise.all( + names.map(async name => { + const formulaIdentifier = await identify(...petNamePath, name); + if (formulaIdentifier !== undefined) { + identities.add(formulaIdentifier); + } + }), + ); + return Array.from(identities); + }; + /** @type {import('./types.js').EndoDirectory['followChanges']} */ const followChanges = async function* followChanges(...petNamePath) { if (petNamePath.length === 0) { @@ -179,6 +194,7 @@ export const makeDirectoryMaker = ({ has, identify, list, + listIdentifiers, followChanges, lookup, reverseLookup, diff --git a/packages/daemon/src/guest.js b/packages/daemon/src/guest.js index 4964352925..77b31156f9 100644 --- a/packages/daemon/src/guest.js +++ b/packages/daemon/src/guest.js @@ -69,6 +69,7 @@ export const makeGuestMaker = ({ has, identify, list, + listIdentifiers, followChanges, lookup, reverseLookup, @@ -97,6 +98,7 @@ export const makeGuestMaker = ({ has, identify, list, + listIdentifiers, followChanges, lookup, reverseLookup, diff --git a/packages/daemon/src/host.js b/packages/daemon/src/host.js index 05bcc1a789..aa58deebc2 100644 --- a/packages/daemon/src/host.js +++ b/packages/daemon/src/host.js @@ -518,6 +518,7 @@ export const makeHostMaker = ({ has, identify, list, + listIdentifiers, followChanges, lookup, reverseLookup, @@ -546,6 +547,7 @@ export const makeHostMaker = ({ has, identify, list, + listIdentifiers, followChanges, lookup, reverseLookup, diff --git a/packages/daemon/src/types.d.ts b/packages/daemon/src/types.d.ts index 4f3c53f4aa..1607d01aa9 100644 --- a/packages/daemon/src/types.d.ts +++ b/packages/daemon/src/types.d.ts @@ -331,6 +331,7 @@ export interface NameHub { has(...petNamePath: string[]): Promise; identify(...petNamePath: string[]): Promise; list(...petNamePath: string[]): Promise>; + listIdentifiers(...petNamePath: string[]): Promise>; followChanges( ...petNamePath: string[] ): AsyncGenerator; From b9c88720dab64694a1e51b4de271e4c80790d5bf Mon Sep 17 00:00:00 2001 From: kumavis Date: Sun, 3 Mar 2024 12:43:59 -1000 Subject: [PATCH 02/20] feat(daemon): add empty networks dir + peer formula + invite/accept --- packages/daemon/src/daemon.js | 151 ++++++++++++++++++++++++++++++- packages/daemon/src/host.js | 46 +++++++++- packages/daemon/src/pet-store.js | 2 +- packages/daemon/src/types.d.ts | 51 ++++++++++- 4 files changed, 244 insertions(+), 6 deletions(-) diff --git a/packages/daemon/src/daemon.js b/packages/daemon/src/daemon.js index c4435d38c5..8bd72a74a5 100644 --- a/packages/daemon/src/daemon.js +++ b/packages/daemon/src/daemon.js @@ -402,10 +402,11 @@ const makeDaemonCore = async ( // eslint-disable-next-line no-use-before-define return makeIdentifiedHost( formulaIdentifier, - formula.endo, formula.petStore, formula.inspector, formula.worker, + formula.endo, + formula.networks, formula.leastAuthority, context, ); @@ -489,6 +490,23 @@ const makeDaemonCore = async ( const bundle = await E(bundleBlob).json(); await E(webPageP).makeBundle(bundle, endowedPowers); }, + reviveNetworks: async () => { + const networksDirectory = + /** @type {import('./types.js').EndoDirectory} */ ( + // Behold, recursion: + // eslint-disable-next-line no-use-before-define + await provideValueForFormulaIdentifier(formula.networks) + ); + const networkFormulaIdentifiers = + await networksDirectory.listIdentifiers(); + await Promise.allSettled( + networkFormulaIdentifiers.map( + // Behold, recursion: + // eslint-disable-next-line no-use-before-define + provideValueForFormulaIdentifier, + ), + ); + }, }); return { external: endoBootstrap, @@ -543,6 +561,15 @@ const makeDaemonCore = async ( petStoreFormulaIdentifier: formula.petStore, context, }); + } else if (formula.type === 'peer') { + // Behold, forward reference: + // eslint-disable-next-line no-use-before-define + return makePeer( + formula.networks, + formula.powers, + formula.addresses, + context, + ); } else { throw new TypeError(`Invalid formula: ${q(formula)}`); } @@ -570,6 +597,7 @@ const makeDaemonCore = async ( 'host', 'guest', 'least-authority', + 'peer', 'web-bundle', 'web-page-js', 'handle', @@ -828,6 +856,7 @@ const makeDaemonCore = async ( /** @type {import('./types.js').DaemonCore['incarnateHost']} */ const incarnateHost = async ( endoFormulaIdentifier, + networksDirectoryFormulaIdentifier, leastAuthorityFormulaIdentifier, specifiedWorkerFormulaIdentifier, ) => { @@ -849,6 +878,7 @@ const makeDaemonCore = async ( inspector: inspectorFormulaIdentifier, worker: workerFormulaIdentifier, endo: endoFormulaIdentifier, + networks: networksDirectoryFormulaIdentifier, leastAuthority: leastAuthorityFormulaIdentifier, }; return /** @type {import('./types').IncarnateResult} */ ( @@ -1057,6 +1087,27 @@ const makeDaemonCore = async ( ); }; + /** @type {import('./types.js').DaemonCore['incarnatePeer']} */ + const incarnatePeer = async ( + networksDirectoryFormulaIdentifier, + remotePowersFormulaIdentifier, + addresses, + ) => { + const formulaNumber = await randomHex512(); + // TODO: validate addresses + // TODO: mutable state like addresses should not be stored in formula + /** @type {import('./types.js').PeerFormula} */ + const formula = { + type: 'peer', + networks: networksDirectoryFormulaIdentifier, + powers: remotePowersFormulaIdentifier, + addresses, + }; + return /** @type {import('./types').IncarnateResult} */ ( + provideValueForNumberedFormula(formula.type, formulaNumber, formula) + ); + }; + /** @type {import('./types.js').DaemonCore['incarnateEndoBootstrap']} */ const incarnateEndoBootstrap = async specifiedFormulaNumber => { const formulaNumber = await (specifiedFormulaNumber ?? randomHex512()); @@ -1064,6 +1115,8 @@ const makeDaemonCore = async ( const { formulaIdentifier: defaultHostWorkerFormulaIdentifier } = await incarnateWorker(); + const { formulaIdentifier: networksDirectoryFormulaIdentifier } = + await incarnateDirectory(); const { formulaIdentifier: leastAuthorityFormulaIdentifier } = await incarnateLeastAuthority(); @@ -1071,6 +1124,7 @@ const makeDaemonCore = async ( const { formulaIdentifier: defaultHostFormulaIdentifier } = await incarnateHost( endoFormulaIdentifier, + networksDirectoryFormulaIdentifier, leastAuthorityFormulaIdentifier, defaultHostWorkerFormulaIdentifier, ); @@ -1087,6 +1141,7 @@ const makeDaemonCore = async ( /** @type {import('./types.js').EndoFormula} */ const formula = { type: 'endo', + networks: networksDirectoryFormulaIdentifier, host: defaultHostFormulaIdentifier, leastAuthority: leastAuthorityFormulaIdentifier, webPageJs: webPageJsFormulaIdentifier, @@ -1096,6 +1151,85 @@ const makeDaemonCore = async ( ); }; + /** + * @param {string} networksDirectoryFormulaIdentifier + * @returns {Promise} + */ + const getAllNetworks = async networksDirectoryFormulaIdentifier => { + const networksDirectory = /** @type {import('./types').EndoDirectory} */ ( + // eslint-disable-next-line no-use-before-define + await provideValueForFormulaIdentifier(networksDirectoryFormulaIdentifier) + ); + const networkFormulaIdentifiers = await networksDirectory.listIdentifiers(); + const networks = /** @type {import('./types').EndoNetwork[]} */ ( + await Promise.all( + networkFormulaIdentifiers.map(provideValueForFormulaIdentifier), + ) + ); + return networks; + }; + + /** @type {import('./types.js').DaemonCore['getAllNetworkAddresses']} */ + const getAllNetworkAddresses = async networksDirectoryFormulaIdentifier => { + const networks = await getAllNetworks(networksDirectoryFormulaIdentifier); + const addresses = ( + await Promise.all( + networks.map(async network => { + return E(network).addresses(); + }), + ) + ).flat(); + return addresses; + }; + + /** + * @param {string} networksDirectoryFormulaIdentifier + * @param {string} remotePowersFormulaIdentifier + * @param {string[]} addresses + * @param {import('./types.js').Context} context + * @returns {Promise} + */ + const makePeer = async ( + networksDirectoryFormulaIdentifier, + remotePowersFormulaIdentifier, + addresses, + context, + ) => { + // TODO race networks that support protocol for connection + // TODO retry, exponential back-off, with full jitter + // TODO (in connect implementations) allow for the possibility of + // connection loss and invalidate the connection formula and its transitive + // dependees when this occurs. + const networks = await getAllNetworks(networksDirectoryFormulaIdentifier); + // Connect on first support address. + for (const address of addresses) { + const { protocol } = new URL(address); + for (const network of networks) { + // eslint-disable-next-line no-await-in-loop + if (await E(network).supports(protocol)) { + const remoteGateway = E(network).connect( + address, + makeFarContext(context), + ); + const external = + /** @type {Promise} */ ( + E(remoteGateway).provideValueForFormulaIdentifier( + remotePowersFormulaIdentifier, + ) + ); + const internal = Promise.resolve(undefined); + // const internal = { + // receive, // TODO + // respond, // TODO + // lookupPath, // TODO + // }; + return harden({ internal, external }); + } + } + } + throw new Error('Cannot connect to peer: no supported addresses'); + }; + const makeContext = makeContextMaker({ controllerForFormulaIdentifier, provideControllerForFormulaIdentifier, @@ -1131,9 +1265,11 @@ const makeDaemonCore = async ( incarnateBundle, incarnateWebBundle, incarnateHandle, + incarnatePeer, storeReaderRef, makeMailbox, makeDirectoryNode, + getAllNetworkAddresses, }); /** @@ -1241,6 +1377,15 @@ const makeDaemonCore = async ( powers: provideValueForFormulaIdentifier(formula.powers), }), ); + } else if (formula.type === 'peer') { + return makeInspector( + formula.type, + formulaNumber, + harden({ + POWERS: provideValueForFormulaIdentifier(formula.powers), + ADDRESSES: formula.addresses, + }), + ); } return makeInspector(formula.type, formulaNumber, harden({})); }; @@ -1263,6 +1408,7 @@ const makeDaemonCore = async ( provideValueForFormulaIdentifier, provideValueForNumberedFormula, getFormulaIdentifierForRef, + getAllNetworkAddresses, cancelValue, storeReaderRef, makeMailbox, @@ -1275,6 +1421,7 @@ const makeDaemonCore = async ( incarnateWorker, incarnateHost, incarnateGuest, + incarnatePeer, incarnateEval, incarnateUnconfined, incarnateReadableBlob, @@ -1363,5 +1510,7 @@ export const makeDaemon = async (powers, daemonLabel, cancel, cancelled) => { }, ); + await E(endoBootstrap).reviveNetworks(); + return { endoBootstrap, cancelGracePeriod, assignWebletPort }; }; diff --git a/packages/daemon/src/host.js b/packages/daemon/src/host.js index aa58deebc2..d42936cbc3 100644 --- a/packages/daemon/src/host.js +++ b/packages/daemon/src/host.js @@ -20,7 +20,9 @@ const { quote: q } = assert; * @param {import('./types.js').DaemonCore['incarnateBundle']} args.incarnateBundle * @param {import('./types.js').DaemonCore['incarnateWebBundle']} args.incarnateWebBundle * @param {import('./types.js').DaemonCore['incarnateHandle']} args.incarnateHandle + * @param {import('./types.js').DaemonCore['incarnatePeer']} args.incarnatePeer * @param {import('./types.js').DaemonCore['storeReaderRef']} args.storeReaderRef + * @param {import('./types.js').DaemonCore['getAllNetworkAddresses']} args.getAllNetworkAddresses * @param {import('./types.js').MakeMailbox} args.makeMailbox * @param {import('./types.js').MakeDirectoryNode} args.makeDirectoryNode */ @@ -36,25 +38,29 @@ export const makeHostMaker = ({ incarnateBundle, incarnateWebBundle, incarnateHandle, + incarnatePeer, storeReaderRef, + getAllNetworkAddresses, makeMailbox, makeDirectoryNode, }) => { /** * @param {string} hostFormulaIdentifier - * @param {string} endoFormulaIdentifier * @param {string} storeFormulaIdentifier * @param {string} inspectorFormulaIdentifier * @param {string} mainWorkerFormulaIdentifier + * @param {string} endoFormulaIdentifier + * @param {string} networksDirectoryFormulaIdentifier * @param {string} leastAuthorityFormulaIdentifier * @param {import('./types.js').Context} context */ const makeIdentifiedHost = async ( hostFormulaIdentifier, - endoFormulaIdentifier, storeFormulaIdentifier, inspectorFormulaIdentifier, mainWorkerFormulaIdentifier, + endoFormulaIdentifier, + networksDirectoryFormulaIdentifier, leastAuthorityFormulaIdentifier, context, ) => { @@ -69,6 +75,7 @@ export const makeHostMaker = ({ const specialStore = makePetSitter(basePetStore, { SELF: hostFormulaIdentifier, ENDO: endoFormulaIdentifier, + NETS: networksDirectoryFormulaIdentifier, INFO: inspectorFormulaIdentifier, NONE: leastAuthorityFormulaIdentifier, }); @@ -438,6 +445,7 @@ export const makeHostMaker = ({ const { formulaIdentifier: newFormulaIdentifier, value } = await incarnateHost( endoFormulaIdentifier, + networksDirectoryFormulaIdentifier, leastAuthorityFormulaIdentifier, ); if (petName !== undefined) { @@ -514,6 +522,38 @@ export const makeHostMaker = ({ return cancelValue(formulaIdentifier, reason); }; + // TODO expand guestName to guestPath + /** @type {import('./types.js').EndoHost['invite']} */ + const invite = async guestName => { + assertPetName(guestName); + const { formulaIdentifier: guestFormulaIdentifier } = await makeGuest( + guestName, + ); + if (guestFormulaIdentifier === undefined) { + throw new Error(`Unknown pet name: ${guestName}`); + } + const addresses = await getAllNetworkAddresses( + networksDirectoryFormulaIdentifier, + ); + return harden({ + powers: guestFormulaIdentifier, + addresses, + }); + }; + + /** @type {import('./types.js').EndoHost['accept']} */ + const accept = async (invitation, ...resultPath) => { + // TODO validate invitation + const { powers, addresses } = invitation; + const { formulaIdentifier, value: endoPeer } = await incarnatePeer( + networksDirectoryFormulaIdentifier, + powers, + addresses, + ); + await directory.write(resultPath, formulaIdentifier); + return endoPeer; + }; + const { has, identify, @@ -576,6 +616,8 @@ export const makeHostMaker = ({ makeBundle, provideWebPage, cancel, + invite, + accept, }; const external = Far('EndoHost', { diff --git a/packages/daemon/src/pet-store.js b/packages/daemon/src/pet-store.js index 5b462051ef..7102138092 100644 --- a/packages/daemon/src/pet-store.js +++ b/packages/daemon/src/pet-store.js @@ -7,7 +7,7 @@ const { quote: q } = assert; const validIdPattern = /^[0-9a-f]{128}$/; const validFormulaPattern = - /^(?:(?:readable-blob|worker|pet-store|pet-inspector|eval|lookup|make-unconfined|make-bundle|host|guest|handle):[0-9a-f]{128}|web-bundle:[0-9a-f]{32})$/; + /^(?:(?:readable-blob|worker|pet-store|pet-inspector|eval|lookup|make-unconfined|make-bundle|host|guest|handle|peer):[0-9a-f]{128}|web-bundle:[0-9a-f]{32})$/; /** * @param {import('./types.js').FilePowers} filePowers diff --git a/packages/daemon/src/types.d.ts b/packages/daemon/src/types.d.ts index 1607d01aa9..7c59a1dcab 100644 --- a/packages/daemon/src/types.d.ts +++ b/packages/daemon/src/types.d.ts @@ -59,6 +59,7 @@ type FormulaIdentifierRecord = { type EndoFormula = { type: 'endo'; + networks: string; host: string; leastAuthority: string; webPageJs?: string; @@ -74,6 +75,7 @@ type HostFormula = { inspector: string; petStore: string; endo: string; + networks: string; leastAuthority: string; }; @@ -141,6 +143,13 @@ type MakeBundleFormula = { // TODO formula slots }; +type PeerFormula = { + type: 'peer'; + networks: string; + powers: string; + addresses: Array; +}; + type WebBundleFormula = { type: 'web-bundle'; bundle: string; @@ -181,7 +190,8 @@ export type Formula = | HandleFormula | PetInspectorFormula | PetStoreFormula - | DirectoryFormula; + | DirectoryFormula + | PeerFormula; export type Label = { number: number; @@ -213,6 +223,11 @@ export type InternalPayload = InternalRequest | InternalPackage; export type Message = Label & Payload; export type InternalMessage = InternalLabel & InternalPayload; +export type Invitation = { + powers: string; + addresses: Array; +}; + export interface Topic< TRead, TWrite = undefined, @@ -282,9 +297,13 @@ export interface InternalExternal { internal: Internal; } -export interface Controller { +export interface ControllerPartial { external: Promise; internal: Promise; +} + +export interface Controller + extends ControllerPartial { context: Context; } @@ -427,6 +446,22 @@ export type MakeHostOrGuestOptions = { introducedNames?: Record; }; +export interface EndoPeer {} +export type EndoPeerControllerPartial = ControllerPartial; +export type EndoPeerController = Controller; + +export interface EndoGateway { + provideValueForFormulaIdentifier: ( + formulaIdentifier: string, + ) => Promise; +} + +export interface EndoNetwork { + supports: (network: string) => boolean; + addresses: () => Array; + connect: (address: string, farContext: FarContext) => EndoGateway; +} + export interface EndoGuest extends EndoDirectory { listMessages: Mail['listMessages']; followMessages: Mail['followMessages']; @@ -487,6 +522,8 @@ export interface EndoHost extends EndoDirectory { powersName: string, ): Promise; cancel(petName: string, reason: Error): Promise; + invite(guestName: string): Promise; + accept(invitation: Invitation): Promise; } export interface InternalEndoHost { @@ -526,6 +563,7 @@ export type FarEndoBootstrap = FarRef<{ leastAuthority: () => Promise; webPageJs: () => Promise; importAndEndowInWebPage: () => Promise; + reviveNetworks: () => Promise; }>; export type CryptoPowers = { @@ -669,6 +707,9 @@ export interface DaemonCore { formula: Formula, ) => Promise<{ formulaIdentifier: string; value: unknown }>; getFormulaIdentifierForRef: (ref: unknown) => string | undefined; + getAllNetworkAddresses: ( + networksDirectoryFormulaIdentifier: string, + ) => Promise; incarnateEndoBootstrap: ( specifiedFormulaNumber: string, ) => IncarnateResult; @@ -680,6 +721,7 @@ export interface DaemonCore { ) => IncarnateResult; incarnateHost: ( endoFormulaIdentifier: string, + networksDirectoryFormulaIdentifier: string, leastAuthorityFormulaIdentifier: string, specifiedWorkerFormulaIdentifier?: string | undefined, ) => IncarnateResult; @@ -718,6 +760,11 @@ export interface DaemonCore { incarnateHandle: ( targetFormulaIdentifier: string, ) => IncarnateResult; + incarnatePeer: ( + networksFormulaIdentifier: string, + powersFormulaIdentifier: string, + addresses: Array, + ) => IncarnateResult; incarnateLeastAuthority: () => IncarnateResult; cancelValue: (formulaIdentifier: string, reason: Error) => Promise; storeReaderRef: ( From 53d41206291bc3ddd1216a4117e8591bb6da944c Mon Sep 17 00:00:00 2001 From: kumavis Date: Mon, 4 Mar 2024 09:23:22 -1000 Subject: [PATCH 03/20] feat(daemon): add loopback network --- packages/daemon/src/daemon.js | 42 ++++++++++++++++++++++-- packages/daemon/src/networks/loopback.js | 25 ++++++++++++++ packages/daemon/src/pet-store.js | 2 +- packages/daemon/src/types.d.ts | 7 ++++ 4 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 packages/daemon/src/networks/loopback.js diff --git a/packages/daemon/src/daemon.js b/packages/daemon/src/daemon.js index 8bd72a74a5..98aa66b07a 100644 --- a/packages/daemon/src/daemon.js +++ b/packages/daemon/src/daemon.js @@ -16,6 +16,7 @@ import { makeContextMaker } from './context.js'; import { parseFormulaIdentifier } from './formula-identifier.js'; import { makeMutex } from './mutex.js'; import { makeWeakMultimap } from './weak-multimap.js'; +import { makeLoopbackNetwork } from './networks/loopback.js'; const delay = async (ms, cancelled) => { // Do not attempt to set up a timer if already cancelled. @@ -512,6 +513,16 @@ const makeDaemonCore = async ( external: endoBootstrap, internal: undefined, }; + } else if (formula.type === 'loopback-network') { + // Behold, forward-reference: + const loopbackNetwork = makeLoopbackNetwork({ + // eslint-disable-next-line no-use-before-define + provideValueForFormulaIdentifier, + }); + return { + external: loopbackNetwork, + internal: undefined, + }; } else if (formula.type === 'least-authority') { const disallowedFn = async () => { throw new Error('not allowed'); @@ -597,6 +608,7 @@ const makeDaemonCore = async ( 'host', 'guest', 'least-authority', + 'loopback-network', 'peer', 'web-bundle', 'web-page-js', @@ -907,7 +919,7 @@ const makeDaemonCore = async ( /** @type {import('./types.js').DaemonCore['incarnateEval']} */ const incarnateEval = async ( - hostFormulaIdentifier, + nameHubFormulaIdentifier, source, codeNames, endowmentFormulaIdsOrPaths, @@ -938,7 +950,7 @@ const makeDaemonCore = async ( ( await incarnateNumberedLookup( await randomHex512(), - hostFormulaIdentifier, + nameHubFormulaIdentifier, formulaIdOrPath, ) ).formulaIdentifier @@ -1108,6 +1120,28 @@ const makeDaemonCore = async ( ); }; + /** @type {import('./types.js').DaemonCore['incarnateLoopbackNetwork']} */ + const incarnateLoopbackNetwork = async () => { + const formulaNumber = await randomHex512(); + /** @type {import('./types').LoopbackNetworkFormula} */ + const formula = { + type: 'loopback-network', + }; + return /** @type {import('./types').IncarnateResult} */ ( + provideValueForNumberedFormula(formula.type, formulaNumber, formula) + ); + }; + + /** @type {import('./types.js').DaemonCore['incarnateNetworksDirectory']} */ + const incarnateNetworksDirectory = async () => { + const { formulaIdentifier, value } = await incarnateDirectory(); + // Make default networks. + const { formulaIdentifier: loopbackNetworkFormulaIdentifier } = + await incarnateLoopbackNetwork(); + await E(value).write(['loop'], loopbackNetworkFormulaIdentifier); + return { formulaIdentifier, value }; + }; + /** @type {import('./types.js').DaemonCore['incarnateEndoBootstrap']} */ const incarnateEndoBootstrap = async specifiedFormulaNumber => { const formulaNumber = await (specifiedFormulaNumber ?? randomHex512()); @@ -1116,7 +1150,7 @@ const makeDaemonCore = async ( const { formulaIdentifier: defaultHostWorkerFormulaIdentifier } = await incarnateWorker(); const { formulaIdentifier: networksDirectoryFormulaIdentifier } = - await incarnateDirectory(); + await incarnateNetworksDirectory(); const { formulaIdentifier: leastAuthorityFormulaIdentifier } = await incarnateLeastAuthority(); @@ -1415,6 +1449,8 @@ const makeDaemonCore = async ( makeDirectoryNode, incarnateEndoBootstrap, incarnateLeastAuthority, + incarnateNetworksDirectory, + incarnateLoopbackNetwork, incarnateHandle, incarnatePetStore, incarnateDirectory, diff --git a/packages/daemon/src/networks/loopback.js b/packages/daemon/src/networks/loopback.js new file mode 100644 index 0000000000..d144d43145 --- /dev/null +++ b/packages/daemon/src/networks/loopback.js @@ -0,0 +1,25 @@ +// @ts-check +import { Far } from '@endo/far'; + +/** + * @param {object} args + * @param {import('../types.js').DaemonCore['provideValueForFormulaIdentifier']} args.provideValueForFormulaIdentifier + * @returns {import('@endo/far').FarRef} + */ +export const makeLoopbackNetwork = ({ provideValueForFormulaIdentifier }) => { + return Far( + 'Loopback Network', + /** @type {import('../types.js').EndoNetwork} */ ({ + addresses: () => ['loop:'], + supports: address => new URL(address).protocol === 'loop:', + connect: address => { + if (address !== 'loop:') { + throw new Error( + 'Failed invariant: loopback only supports loop: address', + ); + } + return { provideValueForFormulaIdentifier }; + }, + }), + ); +}; diff --git a/packages/daemon/src/pet-store.js b/packages/daemon/src/pet-store.js index 7102138092..6aa199d4f0 100644 --- a/packages/daemon/src/pet-store.js +++ b/packages/daemon/src/pet-store.js @@ -7,7 +7,7 @@ const { quote: q } = assert; const validIdPattern = /^[0-9a-f]{128}$/; const validFormulaPattern = - /^(?:(?:readable-blob|worker|pet-store|pet-inspector|eval|lookup|make-unconfined|make-bundle|host|guest|handle|peer):[0-9a-f]{128}|web-bundle:[0-9a-f]{32})$/; + /^(?:(?:readable-blob|worker|pet-store|pet-inspector|eval|lookup|make-unconfined|make-bundle|host|guest|handle|peer|loopback-network):[0-9a-f]{128}|web-bundle:[0-9a-f]{32})$/; /** * @param {import('./types.js').FilePowers} filePowers diff --git a/packages/daemon/src/types.d.ts b/packages/daemon/src/types.d.ts index 7c59a1dcab..00a2dae6a4 100644 --- a/packages/daemon/src/types.d.ts +++ b/packages/daemon/src/types.d.ts @@ -65,6 +65,10 @@ type EndoFormula = { webPageJs?: string; }; +type LoopbackNetworkFormula = { + type: 'loopback-network'; +}; + type WorkerFormula = { type: 'worker'; }; @@ -177,6 +181,7 @@ type DirectoryFormula = { export type Formula = | EndoFormula + | LoopbackNetworkFormula | WorkerFormula | HostFormula | GuestFormula @@ -765,6 +770,8 @@ export interface DaemonCore { powersFormulaIdentifier: string, addresses: Array, ) => IncarnateResult; + incarnateNetworksDirectory: () => IncarnateResult; + incarnateLoopbackNetwork: () => IncarnateResult; incarnateLeastAuthority: () => IncarnateResult; cancelValue: (formulaIdentifier: string, reason: Error) => Promise; storeReaderRef: ( From f7ecd4eac578af20276d6ed7ffb9a3074a55c4f6 Mon Sep 17 00:00:00 2001 From: kumavis Date: Mon, 4 Mar 2024 09:48:25 -1000 Subject: [PATCH 04/20] test(daemon): test loopback network --- packages/daemon/test/test-endo.js | 46 +++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/packages/daemon/test/test-endo.js b/packages/daemon/test/test-endo.js index af8a9ca2e6..d38565c810 100644 --- a/packages/daemon/test/test-endo.js +++ b/packages/daemon/test/test-endo.js @@ -1215,3 +1215,49 @@ test('guest cannot access host methods', async t => { const revealedTarget = await E.get(guestsHost).targetFormulaIdentifier; t.is(revealedTarget, undefined); }); + +test('loopback network', async t => { + const { promise: cancelled, reject: cancel } = makePromiseKit(); + t.teardown(() => cancel(Error('teardown'))); + const locator = makeLocator('tmp', 'loopback'); + + await stop(locator).catch(() => {}); + await purge(locator); + await start(locator); + + const { getBootstrap } = await makeEndoClient( + 'client', + locator.sockPath, + cancelled, + ); + const bootstrap = getBootstrap(); + const host = E(bootstrap).host(); + + // Creates a guest "guest" from the inviter side. + const { powers } = await E(host).invite('guest'); + // We manually create the invitation to specify the network. + const invitation = { + addresses: ['loop:'], + powers, + }; + // Stores the received value as "peer". + const peerFromAccept = await E(host).accept(invitation, 'peer'); + const peerByName = await E(host).lookup('peer'); + t.is(peerFromAccept, peerByName); + + const guestId = await E(host).identify('guest'); + const peerId = await E(host).identify('peer'); + t.not(guestId, peerId); + + const guest = await E(host).lookup('guest'); + const peer = peerFromAccept; + await E(guest).copy(['SELF'], ['a']); + await E(peer).copy(['SELF'], ['b']); + const guestNames = await E(guest).list(); + const peerNames = await E(peer).list(); + t.deepEqual(guestNames, peerNames); + t.assert(guestNames.includes('a')); + t.assert(guestNames.includes('b')); + + await stop(locator); +}); From a1abc3610cd3facccaa91223e2c5583965cc4d90 Mon Sep 17 00:00:00 2001 From: kumavis Date: Mon, 4 Mar 2024 13:04:30 -1000 Subject: [PATCH 05/20] feat(daemon): separate peer and remote formulas --- packages/daemon/src/daemon.js | 80 +++++++++++++++++++++++++------- packages/daemon/src/host.js | 14 ++++-- packages/daemon/src/pet-store.js | 2 +- packages/daemon/src/types.d.ts | 19 ++++++-- 4 files changed, 89 insertions(+), 26 deletions(-) diff --git a/packages/daemon/src/daemon.js b/packages/daemon/src/daemon.js index 98aa66b07a..874e1ecfd3 100644 --- a/packages/daemon/src/daemon.js +++ b/packages/daemon/src/daemon.js @@ -575,12 +575,11 @@ const makeDaemonCore = async ( } else if (formula.type === 'peer') { // Behold, forward reference: // eslint-disable-next-line no-use-before-define - return makePeer( - formula.networks, - formula.powers, - formula.addresses, - context, - ); + return makePeer(formula.networks, formula.addresses, context); + } else if (formula.type === 'remote') { + // Behold, forward reference: + // eslint-disable-next-line no-use-before-define + return makeRemote(formula.peer, formula.value, context); } else { throw new TypeError(`Invalid formula: ${q(formula)}`); } @@ -1102,7 +1101,6 @@ const makeDaemonCore = async ( /** @type {import('./types.js').DaemonCore['incarnatePeer']} */ const incarnatePeer = async ( networksDirectoryFormulaIdentifier, - remotePowersFormulaIdentifier, addresses, ) => { const formulaNumber = await randomHex512(); @@ -1112,7 +1110,6 @@ const makeDaemonCore = async ( const formula = { type: 'peer', networks: networksDirectoryFormulaIdentifier, - powers: remotePowersFormulaIdentifier, addresses, }; return /** @type {import('./types').IncarnateResult} */ ( @@ -1120,6 +1117,23 @@ const makeDaemonCore = async ( ); }; + /** @type {import('./types.js').DaemonCore['incarnateRemote']} */ + const incarnateRemote = async ( + peerFormulaIdentifier, + remoteValueFormulaIdentifier, + ) => { + const formulaNumber = await randomHex512(); + /** @type {import('./types.js').RemoteFormula} */ + const formula = { + type: 'remote', + peer: peerFormulaIdentifier, + value: remoteValueFormulaIdentifier, + }; + return /** @type {import('./types').IncarnateResult} */ ( + provideValueForNumberedFormula(formula.type, formulaNumber, formula) + ); + }; + /** @type {import('./types.js').DaemonCore['incarnateLoopbackNetwork']} */ const incarnateLoopbackNetwork = async () => { const formulaNumber = await randomHex512(); @@ -1218,14 +1232,12 @@ const makeDaemonCore = async ( /** * @param {string} networksDirectoryFormulaIdentifier - * @param {string} remotePowersFormulaIdentifier * @param {string[]} addresses * @param {import('./types.js').Context} context * @returns {Promise} */ const makePeer = async ( networksDirectoryFormulaIdentifier, - remotePowersFormulaIdentifier, addresses, context, ) => { @@ -1245,12 +1257,15 @@ const makeDaemonCore = async ( address, makeFarContext(context), ); - const external = - /** @type {Promise} */ ( - E(remoteGateway).provideValueForFormulaIdentifier( - remotePowersFormulaIdentifier, - ) - ); + const external = Promise.resolve({ + provideValueForFormulaIdentifier: remoteFormulaIdentifier => { + return /** @type {Promise} */ ( + E(remoteGateway).provideValueForFormulaIdentifier( + remoteFormulaIdentifier, + ) + ); + }, + }); const internal = Promise.resolve(undefined); // const internal = { // receive, // TODO @@ -1264,6 +1279,28 @@ const makeDaemonCore = async ( throw new Error('Cannot connect to peer: no supported addresses'); }; + /** + * @param {string} peerFormulaIdentifier + * @param {string} remoteFormulaIdentifier + * @param {import('./types.js').Context} context + * @returns {Promise>} + */ + const makeRemote = async ( + peerFormulaIdentifier, + remoteFormulaIdentifier, + context, + ) => { + const peer = /** @type {import('./types.js').EndoPeer} */ ( + await provideValueForFormulaIdentifier(peerFormulaIdentifier) + ); + const remoteValueP = peer.provideValueForFormulaIdentifier( + remoteFormulaIdentifier, + ); + const external = remoteValueP; + const internal = Promise.resolve(undefined); + return harden({ internal, external }); + }; + const makeContext = makeContextMaker({ controllerForFormulaIdentifier, provideControllerForFormulaIdentifier, @@ -1300,6 +1337,7 @@ const makeDaemonCore = async ( incarnateWebBundle, incarnateHandle, incarnatePeer, + incarnateRemote, storeReaderRef, makeMailbox, makeDirectoryNode, @@ -1416,10 +1454,17 @@ const makeDaemonCore = async ( formula.type, formulaNumber, harden({ - POWERS: provideValueForFormulaIdentifier(formula.powers), ADDRESSES: formula.addresses, }), ); + } else if (formula.type === 'remote') { + return makeInspector( + formula.type, + formulaNumber, + harden({ + PEER: provideValueForFormulaIdentifier(formula.peer), + }), + ); } return makeInspector(formula.type, formulaNumber, harden({})); }; @@ -1458,6 +1503,7 @@ const makeDaemonCore = async ( incarnateHost, incarnateGuest, incarnatePeer, + incarnateRemote, incarnateEval, incarnateUnconfined, incarnateReadableBlob, diff --git a/packages/daemon/src/host.js b/packages/daemon/src/host.js index d42936cbc3..de39777085 100644 --- a/packages/daemon/src/host.js +++ b/packages/daemon/src/host.js @@ -21,6 +21,7 @@ const { quote: q } = assert; * @param {import('./types.js').DaemonCore['incarnateWebBundle']} args.incarnateWebBundle * @param {import('./types.js').DaemonCore['incarnateHandle']} args.incarnateHandle * @param {import('./types.js').DaemonCore['incarnatePeer']} args.incarnatePeer + * @param {import('./types.js').DaemonCore['incarnateRemote']} args.incarnateRemote * @param {import('./types.js').DaemonCore['storeReaderRef']} args.storeReaderRef * @param {import('./types.js').DaemonCore['getAllNetworkAddresses']} args.getAllNetworkAddresses * @param {import('./types.js').MakeMailbox} args.makeMailbox @@ -39,6 +40,7 @@ export const makeHostMaker = ({ incarnateWebBundle, incarnateHandle, incarnatePeer, + incarnateRemote, storeReaderRef, getAllNetworkAddresses, makeMailbox, @@ -544,14 +546,18 @@ export const makeHostMaker = ({ /** @type {import('./types.js').EndoHost['accept']} */ const accept = async (invitation, ...resultPath) => { // TODO validate invitation - const { powers, addresses } = invitation; - const { formulaIdentifier, value: endoPeer } = await incarnatePeer( + const { powers: remoteValueFormulaId, addresses } = invitation; + // TODO: Search for existing peer + const { formulaIdentifier: peerFormulaIdentifier } = await incarnatePeer( networksDirectoryFormulaIdentifier, - powers, addresses, ); + const { formulaIdentifier, value: remoteValue } = await incarnateRemote( + peerFormulaIdentifier, + remoteValueFormulaId, + ); await directory.write(resultPath, formulaIdentifier); - return endoPeer; + return /** @type {import('./types.js').FarEndoGuest} */ (remoteValue); }; const { diff --git a/packages/daemon/src/pet-store.js b/packages/daemon/src/pet-store.js index 6aa199d4f0..b261c72272 100644 --- a/packages/daemon/src/pet-store.js +++ b/packages/daemon/src/pet-store.js @@ -7,7 +7,7 @@ const { quote: q } = assert; const validIdPattern = /^[0-9a-f]{128}$/; const validFormulaPattern = - /^(?:(?:readable-blob|worker|pet-store|pet-inspector|eval|lookup|make-unconfined|make-bundle|host|guest|handle|peer|loopback-network):[0-9a-f]{128}|web-bundle:[0-9a-f]{32})$/; + /^(?:(?:readable-blob|worker|pet-store|pet-inspector|eval|lookup|make-unconfined|make-bundle|host|guest|handle|peer|remote|loopback-network):[0-9a-f]{128}|web-bundle:[0-9a-f]{32})$/; /** * @param {import('./types.js').FilePowers} filePowers diff --git a/packages/daemon/src/types.d.ts b/packages/daemon/src/types.d.ts index 00a2dae6a4..e7cc5e460e 100644 --- a/packages/daemon/src/types.d.ts +++ b/packages/daemon/src/types.d.ts @@ -150,10 +150,15 @@ type MakeBundleFormula = { type PeerFormula = { type: 'peer'; networks: string; - powers: string; addresses: Array; }; +type RemoteFormula = { + type: 'remote'; + peer: string; + value: string; +}; + type WebBundleFormula = { type: 'web-bundle'; bundle: string; @@ -196,6 +201,7 @@ export type Formula = | PetInspectorFormula | PetStoreFormula | DirectoryFormula + | RemoteFormula | PeerFormula; export type Label = { @@ -451,7 +457,11 @@ export type MakeHostOrGuestOptions = { introducedNames?: Record; }; -export interface EndoPeer {} +export interface EndoPeer { + provideValueForFormulaIdentifier: ( + formulaIdentifier: string, + ) => Promise; +} export type EndoPeerControllerPartial = ControllerPartial; export type EndoPeerController = Controller; @@ -477,6 +487,7 @@ export interface EndoGuest extends EndoDirectory { request: Mail['request']; send: Mail['send']; } +export type FarEndoGuest = FarRef; export interface EndoHost extends EndoDirectory { listMessages: Mail['listMessages']; @@ -528,7 +539,7 @@ export interface EndoHost extends EndoDirectory { ): Promise; cancel(petName: string, reason: Error): Promise; invite(guestName: string): Promise; - accept(invitation: Invitation): Promise; + accept(invitation: Invitation): Promise; } export interface InternalEndoHost { @@ -767,9 +778,9 @@ export interface DaemonCore { ) => IncarnateResult; incarnatePeer: ( networksFormulaIdentifier: string, - powersFormulaIdentifier: string, addresses: Array, ) => IncarnateResult; + incarnateRemote: (peer: string, value: string) => IncarnateResult; incarnateNetworksDirectory: () => IncarnateResult; incarnateLoopbackNetwork: () => IncarnateResult; incarnateLeastAuthority: () => IncarnateResult; From cb55c2de043930517e03e0dc638bc01f6bb294c9 Mon Sep 17 00:00:00 2001 From: kumavis Date: Mon, 4 Mar 2024 13:05:28 -1000 Subject: [PATCH 06/20] fix(daemon): loopback network should not advertise its address --- packages/daemon/src/networks/loopback.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/daemon/src/networks/loopback.js b/packages/daemon/src/networks/loopback.js index d144d43145..480f268b21 100644 --- a/packages/daemon/src/networks/loopback.js +++ b/packages/daemon/src/networks/loopback.js @@ -10,7 +10,7 @@ export const makeLoopbackNetwork = ({ provideValueForFormulaIdentifier }) => { return Far( 'Loopback Network', /** @type {import('../types.js').EndoNetwork} */ ({ - addresses: () => ['loop:'], + addresses: () => [], supports: address => new URL(address).protocol === 'loop:', connect: address => { if (address !== 'loop:') { From 7fd707fa5cd36eac9991359f12e183025844e145 Mon Sep 17 00:00:00 2001 From: kumavis Date: Tue, 5 Mar 2024 09:54:28 -1000 Subject: [PATCH 07/20] refactor(daemon): breakout assertValidFormulaIdentifier utility --- packages/daemon/src/pet-store.js | 49 ++++++++++++++------------------ 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/packages/daemon/src/pet-store.js b/packages/daemon/src/pet-store.js index b261c72272..f7e8644f97 100644 --- a/packages/daemon/src/pet-store.js +++ b/packages/daemon/src/pet-store.js @@ -7,7 +7,21 @@ const { quote: q } = assert; const validIdPattern = /^[0-9a-f]{128}$/; const validFormulaPattern = - /^(?:(?:readable-blob|worker|pet-store|pet-inspector|eval|lookup|make-unconfined|make-bundle|host|guest|handle|peer|remote|loopback-network):[0-9a-f]{128}|web-bundle:[0-9a-f]{32})$/; + +/** + * @param {string} formulaIdentifier + * @param {string} [petName] + * @returns {void} + */ +export const assertValidFormulaIdentifier = (formulaIdentifier, petName) => { + if (!validFormulaPattern.test(formulaIdentifier)) { + let message = `Invalid formula identifier ${q(formulaIdentifier)}`; + if (petName !== undefined) { + message += ` for pet name ${q(petName)}`; + } + throw new Error(message); + } +}; /** * @param {import('./types.js').FilePowers} filePowers @@ -32,13 +46,7 @@ export const makePetStoreMaker = (filePowers, locator) => { const petNamePath = filePowers.joinPath(petNameDirectoryPath, petName); const petNameText = await filePowers.readFileText(petNamePath); const formulaIdentifier = petNameText.trim(); - if (!validFormulaPattern.test(formulaIdentifier)) { - throw new Error( - `Invalid formula identifier ${q(formulaIdentifier)} for pet name ${q( - petName, - )}`, - ); - } + assertValidFormulaIdentifier(formulaIdentifier, petName); return formulaIdentifier; }; @@ -74,9 +82,7 @@ export const makePetStoreMaker = (filePowers, locator) => { /** @type {import('./types.js').PetStore['write']} */ const write = async (petName, formulaIdentifier) => { assertValidName(petName); - if (!validFormulaPattern.test(formulaIdentifier)) { - throw new Error(`Invalid formula identifier ${q(formulaIdentifier)}`); - } + assertValidFormulaIdentifier(formulaIdentifier, petName); if (petNames.has(petName)) { // Perform cleanup on the overwritten pet name. @@ -144,9 +150,7 @@ export const makePetStoreMaker = (filePowers, locator) => { `Formula does not exist for pet name ${JSON.stringify(petName)}`, ); } - if (!validFormulaPattern.test(formulaIdentifier)) { - throw new Error(`Invalid formula identifier ${q(formulaIdentifier)}`); - } + assertValidFormulaIdentifier(formulaIdentifier, petName); const petNamePath = filePowers.joinPath(petNameDirectoryPath, petName); await filePowers.removePath(petNamePath); @@ -174,16 +178,9 @@ export const makePetStoreMaker = (filePowers, locator) => { `Formula does not exist for pet name ${JSON.stringify(fromName)}`, ); } - if (!validFormulaPattern.test(formulaIdentifier)) { - throw new Error(`Invalid formula identifier ${q(formulaIdentifier)}`); - } - if ( - overwrittenFormulaIdentifier !== undefined && - !validFormulaPattern.test(overwrittenFormulaIdentifier) - ) { - throw new Error( - `Invalid formula identifier ${q(overwrittenFormulaIdentifier)}`, - ); + assertValidFormulaIdentifier(formulaIdentifier, fromName); + if (overwrittenFormulaIdentifier !== undefined) { + assertValidFormulaIdentifier(overwrittenFormulaIdentifier, toName); } const fromPath = filePowers.joinPath(petNameDirectoryPath, fromName); @@ -220,9 +217,7 @@ export const makePetStoreMaker = (filePowers, locator) => { /** @type {import('./types.js').PetStore['reverseIdentify']} */ const reverseIdentify = formulaIdentifier => { - if (!validFormulaPattern.test(formulaIdentifier)) { - throw new Error(`Invalid formula identifier ${q(formulaIdentifier)}`); - } + assertValidFormulaIdentifier(formulaIdentifier); const formulaPetNames = formulaIdentifiers.get(formulaIdentifier); if (formulaPetNames === undefined) { return harden([]); From cb3e2c54ea52086facd5b00ed079c4f27dd0adde Mon Sep 17 00:00:00 2001 From: kumavis Date: Tue, 5 Mar 2024 09:56:11 -1000 Subject: [PATCH 08/20] feat(daemon): include locationId in formulaId --- packages/daemon/src/daemon.js | 88 +++++++++++++++++------ packages/daemon/src/formula-identifier.js | 12 +++- packages/daemon/src/host.js | 2 +- packages/daemon/src/pet-store.js | 1 + packages/daemon/src/types.d.ts | 3 +- 5 files changed, 78 insertions(+), 28 deletions(-) diff --git a/packages/daemon/src/daemon.js b/packages/daemon/src/daemon.js index 874e1ecfd3..5a7355963e 100644 --- a/packages/daemon/src/daemon.js +++ b/packages/daemon/src/daemon.js @@ -13,7 +13,10 @@ import { makeGuestMaker } from './guest.js'; import { makeHostMaker } from './host.js'; import { assertPetName } from './pet-name.js'; import { makeContextMaker } from './context.js'; -import { parseFormulaIdentifier } from './formula-identifier.js'; +import { + parseFormulaIdentifier, + serializeFormulaIdentifier, +} from './formula-identifier.js'; import { makeMutex } from './mutex.js'; import { makeWeakMultimap } from './weak-multimap.js'; import { makeLoopbackNetwork } from './networks/loopback.js'; @@ -64,6 +67,7 @@ const makeFarContext = context => /** * @param {import('./types.js').DaemonicPowers} powers * @param {Promise} webletPortP + * @param {string} locationId * @param {object} args * @param {(error: Error) => void} args.cancel * @param {number} args.gracePeriodMs @@ -72,6 +76,7 @@ const makeFarContext = context => const makeDaemonCore = async ( powers, webletPortP, + locationId, { cancel, gracePeriodMs, gracePeriodElapsed }, ) => { const { @@ -586,16 +591,23 @@ const makeDaemonCore = async ( }; /** - * @param {string} formulaType - * @param {string} formulaNumber + * @param {string} formulaIdentifier * @param {import('./types.js').Context} context */ const makeControllerForFormulaIdentifier = async ( - formulaType, - formulaNumber, + formulaIdentifier, context, ) => { - const formulaIdentifier = `${formulaType}:${formulaNumber}`; + const { + type: formulaType, + number: formulaNumber, + location: formulaLocation, + } = parseFormulaIdentifier(formulaIdentifier); + if (formulaLocation !== locationId) { + throw new Error( + `Invalid formula identifier, not local: ${q(formulaIdentifier)}`, + ); + } if ( [ 'endo', @@ -642,7 +654,11 @@ const makeDaemonCore = async ( formulaNumber, formula, ) => { - const formulaIdentifier = `${formulaType}:${formulaNumber}`; + const formulaIdentifier = serializeFormulaIdentifier({ + type: formulaType, + number: formulaNumber, + location: locationId, + }); // Memoize for lookup. console.log(`Making ${formulaIdentifier}`); @@ -693,9 +709,6 @@ const makeDaemonCore = async ( /** @type {import('./types.js').DaemonCore['provideControllerForFormulaIdentifier']} */ const provideControllerForFormulaIdentifier = formulaIdentifier => { - const { type: formulaType, number: formulaNumber } = - parseFormulaIdentifier(formulaIdentifier); - let controller = controllerForFormulaIdentifier.get(formulaIdentifier); if (controller !== undefined) { return controller; @@ -718,9 +731,7 @@ const makeDaemonCore = async ( }); controllerForFormulaIdentifier.set(formulaIdentifier, controller); - resolve( - makeControllerForFormulaIdentifier(formulaType, formulaNumber, context), - ); + resolve(makeControllerForFormulaIdentifier(formulaIdentifier, context)); return controller; }; @@ -928,9 +939,14 @@ const makeDaemonCore = async ( const { workerFormulaIdentifier, endowmentFormulaIdentifiers, - evalFormulaNumber, + evalFormulaIdentifier, } = await formulaGraphMutex.enqueue(async () => { const ownFormulaNumber = await randomHex512(); + const ownFormulaIdentifier = serializeFormulaIdentifier({ + type: 'eval', + number: ownFormulaNumber, + location: locationId, + }); const workerFormulaNumber = await (specifiedWorkerFormulaIdentifier ? parseFormulaIdentifier(specifiedWorkerFormulaIdentifier).number : randomHex512()); @@ -957,13 +973,16 @@ const makeDaemonCore = async ( ); }), ), - evalFormulaNumber: ownFormulaNumber, + evalFormulaIdentifier: ownFormulaIdentifier, }); await Promise.all(hooks.map(hook => hook(identifiers))); return identifiers; }); + const { number: evalFormulaNumber } = parseFormulaIdentifier( + evalFormulaIdentifier, + ); /** @type {import('./types.js').EvalFormula} */ const formula = { type: 'eval', @@ -1159,7 +1178,11 @@ const makeDaemonCore = async ( /** @type {import('./types.js').DaemonCore['incarnateEndoBootstrap']} */ const incarnateEndoBootstrap = async specifiedFormulaNumber => { const formulaNumber = await (specifiedFormulaNumber ?? randomHex512()); - const endoFormulaIdentifier = `endo:${formulaNumber}`; + const endoFormulaIdentifier = serializeFormulaIdentifier({ + type: 'endo', + number: formulaNumber, + location: locationId, + }); const { formulaIdentifier: defaultHostWorkerFormulaIdentifier } = await incarnateWorker(); @@ -1515,6 +1538,19 @@ const makeDaemonCore = async ( return daemonCore; }; +/** + * + * @param {string} rootNonce + * @param {import('./types.js').Sha512} digester + * @returns {string} + */ +const getLocationIdFromRootNonce = (rootNonce, digester) => { + digester.updateText(rootNonce); + digester.updateText('location'); + const locationId = digester.digestHex(); + return locationId; +}; + /** * @param {import('./types.js').DaemonicPowers} powers * @param {Promise} webletPortP @@ -1529,19 +1565,25 @@ const provideEndoBootstrap = async ( webletPortP, { cancel, gracePeriodMs, gracePeriodElapsed }, ) => { - const { persistence: persistencePowers } = powers; - - const daemonCore = await makeDaemonCore(powers, webletPortP, { + const { persistence: persistencePowers, crypto: cryptoPowers } = powers; + const { rootNonce: endoFormulaNumber, isNewlyCreated } = + await persistencePowers.provideRootNonce(); + const locationId = getLocationIdFromRootNonce( + endoFormulaNumber, + cryptoPowers.makeSha512(), + ); + const daemonCore = await makeDaemonCore(powers, webletPortP, locationId, { cancel, gracePeriodMs, gracePeriodElapsed, }); - - const { rootNonce: endoFormulaNumber, isNewlyCreated } = - await persistencePowers.provideRootNonce(); const isInitialized = !isNewlyCreated; if (isInitialized) { - const endoFormulaIdentifier = `endo:${endoFormulaNumber}`; + const endoFormulaIdentifier = serializeFormulaIdentifier({ + type: 'endo', + number: endoFormulaNumber, + location: locationId, + }); return /** @type {Promise} */ ( daemonCore.provideValueForFormulaIdentifier(endoFormulaIdentifier) ); diff --git a/packages/daemon/src/formula-identifier.js b/packages/daemon/src/formula-identifier.js index a93e86f014..d1d1191421 100644 --- a/packages/daemon/src/formula-identifier.js +++ b/packages/daemon/src/formula-identifier.js @@ -12,7 +12,13 @@ export const parseFormulaIdentifier = formulaIdentifier => { ); } - const type = formulaIdentifier.slice(0, delimiterIndex); - const number = formulaIdentifier.slice(delimiterIndex + 1); - return { type, number }; + const [type, number, location] = formulaIdentifier.split(':'); + return { type, number, location }; }; + +/** + * @param {import("./types").FormulaIdentifierRecord} formulaRecord + * @returns {string} + */ +export const serializeFormulaIdentifier = ({ type, number, location }) => + `${type}:${number}:${location}`; diff --git a/packages/daemon/src/host.js b/packages/daemon/src/host.js index de39777085..3033454693 100644 --- a/packages/daemon/src/host.js +++ b/packages/daemon/src/host.js @@ -335,7 +335,7 @@ export const makeHostMaker = ({ if (resultName !== undefined) { addHook(identifiers => - petStore.write(resultName, `eval:${identifiers.evalFormulaNumber}`), + petStore.write(resultName, identifiers.evalFormulaIdentifier), ); } diff --git a/packages/daemon/src/pet-store.js b/packages/daemon/src/pet-store.js index f7e8644f97..a4ee80a337 100644 --- a/packages/daemon/src/pet-store.js +++ b/packages/daemon/src/pet-store.js @@ -7,6 +7,7 @@ const { quote: q } = assert; const validIdPattern = /^[0-9a-f]{128}$/; const validFormulaPattern = + /^(?:(?:readable-blob|worker|pet-store|pet-inspector|eval|lookup|make-unconfined|make-bundle|host|guest|handle|peer|remote|loopback-network):[0-9a-f]{128}:[0-9a-f]{128}|web-bundle:[0-9a-f]{32}:[0-9a-f]{128})$/; /** * @param {string} formulaIdentifier diff --git a/packages/daemon/src/types.d.ts b/packages/daemon/src/types.d.ts index e7cc5e460e..8f8e93abdb 100644 --- a/packages/daemon/src/types.d.ts +++ b/packages/daemon/src/types.d.ts @@ -55,6 +55,7 @@ export type MignonicPowers = { type FormulaIdentifierRecord = { type: string; number: string; + location: string; }; type EndoFormula = { @@ -106,7 +107,7 @@ type EvalFormula = { export type EvalFormulaHook = ( identifiers: Readonly<{ endowmentFormulaIdentifiers: string[]; - evalFormulaNumber: string; + evalFormulaIdentifier: string; workerFormulaIdentifier: string; }>, ) => Promise; From b344a93726de48fc79fc698cdf23b6569309e665 Mon Sep 17 00:00:00 2001 From: kumavis Date: Tue, 5 Mar 2024 10:12:14 -1000 Subject: [PATCH 09/20] test(daemon): test failure of lookup of formulaIdentifier with unknown location --- packages/daemon/test/test-endo.js | 40 +++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/packages/daemon/test/test-endo.js b/packages/daemon/test/test-endo.js index d38565c810..dfb433a4d9 100644 --- a/packages/daemon/test/test-endo.js +++ b/packages/daemon/test/test-endo.js @@ -9,6 +9,7 @@ import '@endo/lockdown/commit-debug.js'; import test from 'ava'; import url from 'url'; import path from 'path'; +import crypto from 'crypto'; import { E } from '@endo/far'; import { makePromiseKit } from '@endo/promise-kit'; import bundleSource from '@endo/bundle-source'; @@ -21,6 +22,10 @@ import { makeEndoClient, makeReaderRef, } from '../index.js'; +import { makeCryptoPowers } from '../src/daemon-node-powers.js'; +import { serializeFormulaIdentifier } from '../src/formula-identifier.js'; + +const cryptoPowers = makeCryptoPowers(crypto); const { raw } = String; @@ -1261,3 +1266,38 @@ test('loopback network', async t => { await stop(locator); }); + +test('read unknown location', async t => { + const { promise: cancelled, reject: cancel } = makePromiseKit(); + t.teardown(() => cancel(Error('teardown'))); + const locator = makeLocator('tmp', 'read unknown location'); + + await stop(locator).catch(() => {}); + await purge(locator); + await start(locator); + + const { getBootstrap } = await makeEndoClient( + 'client', + locator.sockPath, + cancelled, + ); + const bootstrap = getBootstrap(); + const host = E(bootstrap).host(); + + // write a bogus value for a bogus location + const location = await cryptoPowers.randomHex512(); + const number = await cryptoPowers.randomHex512(); + const type = 'eval'; + const formulaIdentifier = serializeFormulaIdentifier({ + location, + number, + type, + }); + await E(host).write(['abc'], formulaIdentifier); + // observe reification failure + t.throwsAsync(() => E(host).lookup('abc'), { + message: /Invalid formula identifier, not local: /u, + }); + + await stop(locator); +}); From b850b6c864ee4c3db6a4fd35a726d9a1e018dd39 Mon Sep 17 00:00:00 2001 From: kumavis Date: Thu, 7 Mar 2024 08:55:10 -1000 Subject: [PATCH 10/20] feat(daemon): peer info exchange methods + remote lookup --- packages/daemon/src/daemon.js | 118 +++++++++--- packages/daemon/src/host.js | 36 +++- packages/daemon/src/networks/tcp-netstring.js | 177 ++++++++++++++++++ packages/daemon/src/types.d.ts | 15 +- packages/daemon/test/test-endo.js | 99 +++++++++- 5 files changed, 418 insertions(+), 27 deletions(-) create mode 100644 packages/daemon/src/networks/tcp-netstring.js diff --git a/packages/daemon/src/daemon.js b/packages/daemon/src/daemon.js index 5a7355963e..a707a08c8d 100644 --- a/packages/daemon/src/daemon.js +++ b/packages/daemon/src/daemon.js @@ -67,7 +67,7 @@ const makeFarContext = context => /** * @param {import('./types.js').DaemonicPowers} powers * @param {Promise} webletPortP - * @param {string} locationId + * @param {string} rootEntropy * @param {object} args * @param {(error: Error) => void} args.cancel * @param {number} args.gracePeriodMs @@ -76,7 +76,7 @@ const makeFarContext = context => const makeDaemonCore = async ( powers, webletPortP, - locationId, + rootEntropy, { cancel, gracePeriodMs, gracePeriodElapsed }, ) => { const { @@ -88,6 +88,23 @@ const makeDaemonCore = async ( const { randomHex512 } = cryptoPowers; const contentStore = persistencePowers.makeContentSha512Store(); const formulaGraphMutex = makeMutex(); + // eslint-disable-next-line no-use-before-define + const ownLocation = getDerivedId( + 'location', + rootEntropy, + cryptoPowers.makeSha512(), + ); + // eslint-disable-next-line no-use-before-define + const peersFormulaNumber = getDerivedId( + 'peers', + rootEntropy, + cryptoPowers.makeSha512(), + ); + const peersFormulaIdentifier = serializeFormulaIdentifier({ + type: 'pet-store', + number: peersFormulaNumber, + location: ownLocation, + }); /** * The two functions "provideValueForNumberedFormula" and "provideValueForFormulaIdentifier" @@ -513,6 +530,25 @@ const makeDaemonCore = async ( ), ); }, + addPeerInfo: async peerInfo => { + const peerPetstore = + /** @type {import('./types.js').PetStore} */ + // Behold, recursion: + // eslint-disable-next-line no-use-before-define + (await provideValueForFormulaIdentifier(formula.peers)); + const { location, addresses } = peerInfo; + // eslint-disable-next-line no-use-before-define + const locationName = petStoreNameForLocation(location); + if (peerPetstore.has(locationName)) { + // We already have this peer. + // TODO: merge connection info + return; + } + const { formulaIdentifier: peerFormulaIdentifier } = + // eslint-disable-next-line no-use-before-define + await incarnatePeer(formula.networks, addresses); + await peerPetstore.write(locationName, peerFormulaIdentifier); + }, }); return { external: endoBootstrap, @@ -603,10 +639,15 @@ const makeDaemonCore = async ( number: formulaNumber, location: formulaLocation, } = parseFormulaIdentifier(formulaIdentifier); - if (formulaLocation !== locationId) { - throw new Error( - `Invalid formula identifier, not local: ${q(formulaIdentifier)}`, + const isRemote = formulaLocation !== ownLocation; + if (isRemote) { + // eslint-disable-next-line no-use-before-define + const peerIdentifier = await getPeerFormulaIdentifierForLocation( + formulaLocation, ); + // Behold, forward reference: + // eslint-disable-next-line no-use-before-define + return makeRemote(peerIdentifier, formulaIdentifier, context); } if ( [ @@ -657,7 +698,7 @@ const makeDaemonCore = async ( const formulaIdentifier = serializeFormulaIdentifier({ type: formulaType, number: formulaNumber, - location: locationId, + location: ownLocation, }); // Memoize for lookup. @@ -736,6 +777,25 @@ const makeDaemonCore = async ( return controller; }; + // TODO: sorry, forcing location into a petstore name + const petStoreNameForLocation = location => { + const locationName = `p${location.slice(0, 126)}`; + return locationName; + }; + + const getPeerFormulaIdentifierForLocation = async location => { + const peerStore = /** @type {import('./types.js').PetStore} */ ( + // eslint-disable-next-line no-use-before-define + await provideValueForFormulaIdentifier(peersFormulaIdentifier) + ); + const locationName = petStoreNameForLocation(location); + const peerFormulaIdentifier = peerStore.identifyLocal(locationName); + if (peerFormulaIdentifier === undefined) { + throw new Error(`No peer found for location ${q(location)}.`); + } + return peerFormulaIdentifier; + }; + /** @type {import('./types.js').DaemonCore['cancelValue']} */ const cancelValue = async (formulaIdentifier, reason) => { await formulaGraphMutex.enqueue(); @@ -816,8 +876,8 @@ const makeDaemonCore = async ( /** * @type {import('./types.js').DaemonCore['incarnatePetStore']} */ - const incarnatePetStore = async () => { - const formulaNumber = await randomHex512(); + const incarnatePetStore = async specifiedFormulaNumber => { + const formulaNumber = specifiedFormulaNumber ?? (await randomHex512()); /** @type {import('./types.js').PetStoreFormula} */ const formula = { type: 'pet-store', @@ -945,7 +1005,7 @@ const makeDaemonCore = async ( const ownFormulaIdentifier = serializeFormulaIdentifier({ type: 'eval', number: ownFormulaNumber, - location: locationId, + location: ownLocation, }); const workerFormulaNumber = await (specifiedWorkerFormulaIdentifier ? parseFormulaIdentifier(specifiedWorkerFormulaIdentifier).number @@ -1181,7 +1241,7 @@ const makeDaemonCore = async ( const endoFormulaIdentifier = serializeFormulaIdentifier({ type: 'endo', number: formulaNumber, - location: locationId, + location: ownLocation, }); const { formulaIdentifier: defaultHostWorkerFormulaIdentifier } = @@ -1190,6 +1250,13 @@ const makeDaemonCore = async ( await incarnateNetworksDirectory(); const { formulaIdentifier: leastAuthorityFormulaIdentifier } = await incarnateLeastAuthority(); + const { formulaIdentifier: newPeersFormulaIdentifier } = + await incarnatePetStore(peersFormulaNumber); + if (newPeersFormulaIdentifier !== peersFormulaIdentifier) { + throw new Error( + `Peers PetStore formula identifier did not match expected value.`, + ); + } // Ensure the default host is incarnated and persisted. const { formulaIdentifier: defaultHostFormulaIdentifier } = @@ -1213,6 +1280,7 @@ const makeDaemonCore = async ( const formula = { type: 'endo', networks: networksDirectoryFormulaIdentifier, + peers: peersFormulaIdentifier, host: defaultHostFormulaIdentifier, leastAuthority: leastAuthorityFormulaIdentifier, webPageJs: webPageJsFormulaIdentifier, @@ -1365,6 +1433,7 @@ const makeDaemonCore = async ( makeMailbox, makeDirectoryNode, getAllNetworkAddresses, + ownLocation, }); /** @@ -1505,6 +1574,7 @@ const makeDaemonCore = async ( /** @type {import('./types.js').DaemonCore} */ const daemonCore = { + ownLocation, provideControllerForFormulaIdentifier, provideControllerForFormulaIdentifierAndResolveHandle, provideValueForFormulaIdentifier, @@ -1540,15 +1610,16 @@ const makeDaemonCore = async ( /** * + * @param {string} path * @param {string} rootNonce * @param {import('./types.js').Sha512} digester * @returns {string} */ -const getLocationIdFromRootNonce = (rootNonce, digester) => { +const getDerivedId = (path, rootNonce, digester) => { digester.updateText(rootNonce); - digester.updateText('location'); - const locationId = digester.digestHex(); - return locationId; + digester.updateText(path); + const nonce = digester.digestHex(); + return nonce; }; /** @@ -1565,24 +1636,25 @@ const provideEndoBootstrap = async ( webletPortP, { cancel, gracePeriodMs, gracePeriodElapsed }, ) => { - const { persistence: persistencePowers, crypto: cryptoPowers } = powers; + const { persistence: persistencePowers } = powers; const { rootNonce: endoFormulaNumber, isNewlyCreated } = await persistencePowers.provideRootNonce(); - const locationId = getLocationIdFromRootNonce( + const daemonCore = await makeDaemonCore( + powers, + webletPortP, endoFormulaNumber, - cryptoPowers.makeSha512(), + { + cancel, + gracePeriodMs, + gracePeriodElapsed, + }, ); - const daemonCore = await makeDaemonCore(powers, webletPortP, locationId, { - cancel, - gracePeriodMs, - gracePeriodElapsed, - }); const isInitialized = !isNewlyCreated; if (isInitialized) { const endoFormulaIdentifier = serializeFormulaIdentifier({ type: 'endo', number: endoFormulaNumber, - location: locationId, + location: daemonCore.ownLocation, }); return /** @type {Promise} */ ( daemonCore.provideValueForFormulaIdentifier(endoFormulaIdentifier) diff --git a/packages/daemon/src/host.js b/packages/daemon/src/host.js index 3033454693..44b999b988 100644 --- a/packages/daemon/src/host.js +++ b/packages/daemon/src/host.js @@ -1,6 +1,6 @@ // @ts-check -import { Far } from '@endo/far'; +import { E, Far } from '@endo/far'; import { makeIteratorRef } from './reader-ref.js'; import { assertPetName, petNamePathFrom } from './pet-name.js'; import { makePetSitter } from './pet-sitter.js'; @@ -26,6 +26,7 @@ const { quote: q } = assert; * @param {import('./types.js').DaemonCore['getAllNetworkAddresses']} args.getAllNetworkAddresses * @param {import('./types.js').MakeMailbox} args.makeMailbox * @param {import('./types.js').MakeDirectoryNode} args.makeDirectoryNode + * @param {string} args.ownLocation */ export const makeHostMaker = ({ provideValueForFormulaIdentifier, @@ -45,6 +46,7 @@ export const makeHostMaker = ({ getAllNetworkAddresses, makeMailbox, makeDirectoryNode, + ownLocation, }) => { /** * @param {string} hostFormulaIdentifier @@ -89,6 +91,7 @@ export const makeHostMaker = ({ }); const { petStore } = mailbox; const directory = makeDirectoryNode(petStore); + const { lookup } = directory; /** * @returns {Promise<{ formulaIdentifier: string, value: import('./types').ExternalHandle }>} @@ -560,13 +563,39 @@ export const makeHostMaker = ({ return /** @type {import('./types.js').FarEndoGuest} */ (remoteValue); }; + /** @type {import('./types.js').EndoHost['gateway']} */ + const gateway = async () => { + // TODO: we should only materialize local objects this way + return Far('Gateway', { provideValueForFormulaIdentifier }); + }; + + /** @type {import('./types.js').EndoHost['addPeerInfo']} */ + const addPeerInfo = async peerInfo => { + const endoBootstrap = + /** @type {import('./types.js').FarEndoBootstrap} */ ( + await lookup('ENDO') + ); + await E(endoBootstrap).addPeerInfo(peerInfo); + }; + + /** @type {import('./types.js').EndoHost['getPeerInfo']} */ + const getPeerInfo = async () => { + const addresses = await getAllNetworkAddresses( + networksDirectoryFormulaIdentifier, + ); + const peerInfo = { + location: ownLocation, + addresses, + }; + return peerInfo; + }; + const { has, identify, list, listIdentifiers, followChanges, - lookup, reverseLookup, write, remove, @@ -624,6 +653,9 @@ export const makeHostMaker = ({ cancel, invite, accept, + gateway, + getPeerInfo, + addPeerInfo, }; const external = Far('EndoHost', { diff --git a/packages/daemon/src/networks/tcp-netstring.js b/packages/daemon/src/networks/tcp-netstring.js new file mode 100644 index 0000000000..72d14bad9e --- /dev/null +++ b/packages/daemon/src/networks/tcp-netstring.js @@ -0,0 +1,177 @@ +// @ts-check +import net from 'net'; + +import { E, Far } from '@endo/far'; +import { mapWriter, mapReader } from '@endo/stream'; +import { makeNetstringReader, makeNetstringWriter } from '@endo/netstring'; +import { + makeMessageCapTP, + bytesToMessage, + messageToBytes, +} from '../connection.js'; +import { makeSocketPowers } from '../daemon-node-powers.js'; + +const protocol = 'tcp+netstring+json+captp0'; + +export const make = async (powers, context) => { + const { servePort, connectPort } = makeSocketPowers({ net }); + + const cancelled = E(context).whenCancelled(); + const cancel = error => E(context).cancel(error); + + /** @type {Array} */ + const addresses = []; + + const gateway = E(powers).gateway(); + + // TODO + // const port = await E(powers).request('port to listen for public web socket connections', 'port', { + // default: '8080', + // }); + + const connectionNumbers = (function* generateNumbers() { + let n = 0; + for (;;) { + yield n; + n += 1; + } + })(); + + /** @type {Set>} */ + const connectionClosedPromises = new Set(); + + const started = (async () => { + // TODO make responding to this less of a pain. + // defaults, type hints, anything and everything. + // TODO more validation + const hostPort = await E(powers).request( + 'SELF', + 'Please select a host:port like 127.0.0.1:8920', + 'tcp-netstring-json-captp0-host-port', + ); + const { hostname: host, port: portname } = new URL( + `protocol://${hostPort}`, + ); + const port = Number(portname); + const { port: assignedPort, connections } = await servePort({ + port, + host, + cancelled, + }); + + // TODO log assigned port + console.log(`Endo daemon started local ${protocol} network device`); + addresses.push(`${protocol}://${host}:${assignedPort}`); + + return connections; + })(); + + const stopped = (async () => { + const connections = await started; + for await (const { + reader: bytesReader, + writer: bytesWriter, + closed: connectionClosed, + } of connections) { + (async () => { + const { value: connectionNumber } = connectionNumbers.next(); + + // TODO listen and connect addresses should be logged + console.log( + `Endo daemon accepted connection ${connectionNumber} over ${protocol} at ${new Date().toISOString()}`, + ); + + const messageWriter = mapWriter( + makeNetstringWriter(bytesWriter, { chunked: true }), + messageToBytes, + ); + const messageReader = mapReader( + makeNetstringReader(bytesReader), + bytesToMessage, + ); + + const { closed: capTpClosed } = makeMessageCapTP( + 'Endo', + messageWriter, + messageReader, + cancelled, + gateway, + ); + + const closed = Promise.race([connectionClosed, capTpClosed]); + connectionClosedPromises.add(closed); + closed.finally(() => { + connectionClosedPromises.delete(closed); + console.log( + `Endo daemon closed connection ${connectionNumber} over ${protocol} at ${new Date().toISOString()}`, + ); + }); + })().catch(cancel); + } + + await Promise.all(Array.from(connectionClosedPromises)); + })(); + + E.sendOnly(context).addDisposalHook(() => stopped); + + const connect = async (address, connectionContext) => { + const { value: connectionNumber } = connectionNumbers.next(); + + const { port: portname, hostname: host } = new URL(address); + const port = Number(portname); + + const connectionCancelled = E(connectionContext).whenCancelled(); + + const { + reader: bytesReader, + writer: bytesWriter, + closed: connectionClosed, + } = await connectPort({ + port, + host, + cancelled: connectionCancelled, + }); + + // TODO listen and connect addresses should be logged + console.log( + `Endo daemon connected ${connectionNumber} over ${protocol} at ${new Date().toISOString()}`, + ); + + const messageWriter = mapWriter( + makeNetstringWriter(bytesWriter, { chunked: true }), + messageToBytes, + ); + const messageReader = mapReader( + makeNetstringReader(bytesReader), + bytesToMessage, + ); + + const { closed: capTpClosed, getBootstrap } = makeMessageCapTP( + 'Endo', + messageWriter, + messageReader, + cancelled, + gateway, + ); + + const closed = Promise.race([connectionClosed, capTpClosed]); + connectionClosedPromises.add(closed); + closed.finally(() => { + E(context).cancel(); + connectionClosedPromises.delete(closed); + console.log( + `Endo daemon closed connection ${connectionNumber} over ${protocol}at ${new Date().toISOString()}`, + ); + }); + + return getBootstrap(); + }; + + await started; + + return Far('TcpNetstringService', { + addresses: () => harden(addresses), + supports: address => new URL(address).protocol === `${protocol}:`, + connect, + }); +}; diff --git a/packages/daemon/src/types.d.ts b/packages/daemon/src/types.d.ts index 8f8e93abdb..b5dc1461e8 100644 --- a/packages/daemon/src/types.d.ts +++ b/packages/daemon/src/types.d.ts @@ -61,6 +61,7 @@ type FormulaIdentifierRecord = { type EndoFormula = { type: 'endo'; networks: string; + peers: string; host: string; leastAuthority: string; webPageJs?: string; @@ -472,6 +473,11 @@ export interface EndoGateway { ) => Promise; } +export interface PeerInfo { + location: string; + addresses: string[]; +} + export interface EndoNetwork { supports: (network: string) => boolean; addresses: () => Array; @@ -541,6 +547,9 @@ export interface EndoHost extends EndoDirectory { cancel(petName: string, reason: Error): Promise; invite(guestName: string): Promise; accept(invitation: Invitation): Promise; + gateway(): Promise; + getPeerInfo(): Promise; + addPeerInfo(peerInfo: PeerInfo): Promise; } export interface InternalEndoHost { @@ -581,6 +590,7 @@ export type FarEndoBootstrap = FarRef<{ webPageJs: () => Promise; importAndEndowInWebPage: () => Promise; reviveNetworks: () => Promise; + addPeerInfo: (peerInfo: PeerInfo) => Promise; }>; export type CryptoPowers = { @@ -709,6 +719,7 @@ export type DaemonicPowers = { type IncarnateResult = Promise<{ formulaIdentifier: string; value: T }>; export interface DaemonCore { + ownLocation: string; provideValueForFormulaIdentifier: ( formulaIdentifier: string, ) => Promise; @@ -731,7 +742,9 @@ export interface DaemonCore { specifiedFormulaNumber: string, ) => IncarnateResult; incarnateWorker: () => IncarnateResult; - incarnatePetStore: () => IncarnateResult; + incarnatePetStore: ( + specifiedFormulaNumber?: string, + ) => IncarnateResult; incarnateDirectory: () => IncarnateResult; incarnatePetInspector: ( petStoreFormulaIdentifier: string, diff --git a/packages/daemon/test/test-endo.js b/packages/daemon/test/test-endo.js index dfb433a4d9..6f14b2c26e 100644 --- a/packages/daemon/test/test-endo.js +++ b/packages/daemon/test/test-endo.js @@ -1296,8 +1296,105 @@ test('read unknown location', async t => { await E(host).write(['abc'], formulaIdentifier); // observe reification failure t.throwsAsync(() => E(host).lookup('abc'), { - message: /Invalid formula identifier, not local: /u, + message: /No peer found for location /u, }); await stop(locator); }); + +test('read remote location', async t => { + const { promise: cancelled, reject: cancel } = makePromiseKit(); + t.teardown(() => cancel(Error('teardown'))); + const locatorA = makeLocator('tmp', 'read remote location A'); + const locatorB = makeLocator('tmp', 'read remote location B'); + let hostA; + { + await stop(locatorA).catch(() => {}); + await purge(locatorA); + await start(locatorA); + const { getBootstrap } = await makeEndoClient( + 'client', + locatorA.sockPath, + cancelled, + ); + const bootstrap = getBootstrap(); + hostA = E(bootstrap).host(); + // Install test network + const servicePath = path.join( + dirname, + 'src', + 'networks', + 'tcp-netstring.js', + ); + const serviceLocation = url.pathToFileURL(servicePath).href; + const networkA = E(hostA).makeUnconfined( + 'MAIN', + serviceLocation, + 'SELF', + 'test-network', + ); + // set address via request + const iteratorRef = E(hostA).followMessages(); + const { value: message } = await E(iteratorRef).next(); + const { number } = E.get(message); + await E(hostA).evaluate('MAIN', '`127.0.0.1:8920`', [], [], 'netport'); + await E(hostA).resolve(await number, 'netport'); + // move test network to network dir + await networkA; + await E(hostA).move(['test-network'], ['NETS', 'tcp']); + } + + let hostB; + { + await stop(locatorB).catch(() => {}); + await purge(locatorB); + await start(locatorB); + const { getBootstrap } = await makeEndoClient( + 'client', + locatorB.sockPath, + cancelled, + ); + const bootstrap = getBootstrap(); + hostB = E(bootstrap).host(); + // Install test network + const servicePath = path.join( + dirname, + 'src', + 'networks', + 'tcp-netstring.js', + ); + const serviceLocation = url.pathToFileURL(servicePath).href; + const networkB = E(hostB).makeUnconfined( + 'MAIN', + serviceLocation, + 'SELF', + 'test-network', + ); + // set address via requestcd + const iteratorRef = E(hostB).followMessages(); + const { value: message } = await E(iteratorRef).next(); + const { number } = E.get(message); + await E(hostB).evaluate('MAIN', '`127.0.0.1:8921`', [], [], 'netport'); + await E(hostB).resolve(await number, 'netport'); + // move test network to network dir + await networkB; + await E(hostB).move(['test-network'], ['NETS', 'tcp']); + } + + // introduce nodes to each other + await E(hostA).addPeerInfo(await E(hostB).getPeerInfo()); + await E(hostB).addPeerInfo(await E(hostA).getPeerInfo()); + + // create value to share + await E(hostB).evaluate('MAIN', '`haay wuurl`', [], [], 'salutations'); + const hostBValueIdentifier = await E(hostB).identify('salutations'); + + // insert in hostA out of band + await E(hostA).write(['greetings'], hostBValueIdentifier); + const hostAValue = await E(hostA).lookup('greetings'); + + t.is(hostAValue, 'haay wuurl'); + + await stop(locatorA); + await stop(locatorB); +}); From 31737240e51ffce8a1dbe94b7c31446763ae6b45 Mon Sep 17 00:00:00 2001 From: kumavis Date: Thu, 7 Mar 2024 10:18:55 -1000 Subject: [PATCH 11/20] fix(daemon): remove invite/accept and RemoteFormula --- packages/daemon/src/daemon.js | 32 --------------------- packages/daemon/src/host.js | 42 ---------------------------- packages/daemon/src/types.d.ts | 10 ------- packages/daemon/test/test-endo.js | 46 ------------------------------- 4 files changed, 130 deletions(-) diff --git a/packages/daemon/src/daemon.js b/packages/daemon/src/daemon.js index a707a08c8d..6d886806e0 100644 --- a/packages/daemon/src/daemon.js +++ b/packages/daemon/src/daemon.js @@ -617,10 +617,6 @@ const makeDaemonCore = async ( // Behold, forward reference: // eslint-disable-next-line no-use-before-define return makePeer(formula.networks, formula.addresses, context); - } else if (formula.type === 'remote') { - // Behold, forward reference: - // eslint-disable-next-line no-use-before-define - return makeRemote(formula.peer, formula.value, context); } else { throw new TypeError(`Invalid formula: ${q(formula)}`); } @@ -1196,23 +1192,6 @@ const makeDaemonCore = async ( ); }; - /** @type {import('./types.js').DaemonCore['incarnateRemote']} */ - const incarnateRemote = async ( - peerFormulaIdentifier, - remoteValueFormulaIdentifier, - ) => { - const formulaNumber = await randomHex512(); - /** @type {import('./types.js').RemoteFormula} */ - const formula = { - type: 'remote', - peer: peerFormulaIdentifier, - value: remoteValueFormulaIdentifier, - }; - return /** @type {import('./types').IncarnateResult} */ ( - provideValueForNumberedFormula(formula.type, formulaNumber, formula) - ); - }; - /** @type {import('./types.js').DaemonCore['incarnateLoopbackNetwork']} */ const incarnateLoopbackNetwork = async () => { const formulaNumber = await randomHex512(); @@ -1427,8 +1406,6 @@ const makeDaemonCore = async ( incarnateBundle, incarnateWebBundle, incarnateHandle, - incarnatePeer, - incarnateRemote, storeReaderRef, makeMailbox, makeDirectoryNode, @@ -1549,14 +1526,6 @@ const makeDaemonCore = async ( ADDRESSES: formula.addresses, }), ); - } else if (formula.type === 'remote') { - return makeInspector( - formula.type, - formulaNumber, - harden({ - PEER: provideValueForFormulaIdentifier(formula.peer), - }), - ); } return makeInspector(formula.type, formulaNumber, harden({})); }; @@ -1596,7 +1565,6 @@ const makeDaemonCore = async ( incarnateHost, incarnateGuest, incarnatePeer, - incarnateRemote, incarnateEval, incarnateUnconfined, incarnateReadableBlob, diff --git a/packages/daemon/src/host.js b/packages/daemon/src/host.js index 44b999b988..50a12fbcb0 100644 --- a/packages/daemon/src/host.js +++ b/packages/daemon/src/host.js @@ -20,8 +20,6 @@ const { quote: q } = assert; * @param {import('./types.js').DaemonCore['incarnateBundle']} args.incarnateBundle * @param {import('./types.js').DaemonCore['incarnateWebBundle']} args.incarnateWebBundle * @param {import('./types.js').DaemonCore['incarnateHandle']} args.incarnateHandle - * @param {import('./types.js').DaemonCore['incarnatePeer']} args.incarnatePeer - * @param {import('./types.js').DaemonCore['incarnateRemote']} args.incarnateRemote * @param {import('./types.js').DaemonCore['storeReaderRef']} args.storeReaderRef * @param {import('./types.js').DaemonCore['getAllNetworkAddresses']} args.getAllNetworkAddresses * @param {import('./types.js').MakeMailbox} args.makeMailbox @@ -40,8 +38,6 @@ export const makeHostMaker = ({ incarnateBundle, incarnateWebBundle, incarnateHandle, - incarnatePeer, - incarnateRemote, storeReaderRef, getAllNetworkAddresses, makeMailbox, @@ -527,42 +523,6 @@ export const makeHostMaker = ({ return cancelValue(formulaIdentifier, reason); }; - // TODO expand guestName to guestPath - /** @type {import('./types.js').EndoHost['invite']} */ - const invite = async guestName => { - assertPetName(guestName); - const { formulaIdentifier: guestFormulaIdentifier } = await makeGuest( - guestName, - ); - if (guestFormulaIdentifier === undefined) { - throw new Error(`Unknown pet name: ${guestName}`); - } - const addresses = await getAllNetworkAddresses( - networksDirectoryFormulaIdentifier, - ); - return harden({ - powers: guestFormulaIdentifier, - addresses, - }); - }; - - /** @type {import('./types.js').EndoHost['accept']} */ - const accept = async (invitation, ...resultPath) => { - // TODO validate invitation - const { powers: remoteValueFormulaId, addresses } = invitation; - // TODO: Search for existing peer - const { formulaIdentifier: peerFormulaIdentifier } = await incarnatePeer( - networksDirectoryFormulaIdentifier, - addresses, - ); - const { formulaIdentifier, value: remoteValue } = await incarnateRemote( - peerFormulaIdentifier, - remoteValueFormulaId, - ); - await directory.write(resultPath, formulaIdentifier); - return /** @type {import('./types.js').FarEndoGuest} */ (remoteValue); - }; - /** @type {import('./types.js').EndoHost['gateway']} */ const gateway = async () => { // TODO: we should only materialize local objects this way @@ -651,8 +611,6 @@ export const makeHostMaker = ({ makeBundle, provideWebPage, cancel, - invite, - accept, gateway, getPeerInfo, addPeerInfo, diff --git a/packages/daemon/src/types.d.ts b/packages/daemon/src/types.d.ts index b5dc1461e8..829c9994c0 100644 --- a/packages/daemon/src/types.d.ts +++ b/packages/daemon/src/types.d.ts @@ -155,12 +155,6 @@ type PeerFormula = { addresses: Array; }; -type RemoteFormula = { - type: 'remote'; - peer: string; - value: string; -}; - type WebBundleFormula = { type: 'web-bundle'; bundle: string; @@ -203,7 +197,6 @@ export type Formula = | PetInspectorFormula | PetStoreFormula | DirectoryFormula - | RemoteFormula | PeerFormula; export type Label = { @@ -545,8 +538,6 @@ export interface EndoHost extends EndoDirectory { powersName: string, ): Promise; cancel(petName: string, reason: Error): Promise; - invite(guestName: string): Promise; - accept(invitation: Invitation): Promise; gateway(): Promise; getPeerInfo(): Promise; addPeerInfo(peerInfo: PeerInfo): Promise; @@ -794,7 +785,6 @@ export interface DaemonCore { networksFormulaIdentifier: string, addresses: Array, ) => IncarnateResult; - incarnateRemote: (peer: string, value: string) => IncarnateResult; incarnateNetworksDirectory: () => IncarnateResult; incarnateLoopbackNetwork: () => IncarnateResult; incarnateLeastAuthority: () => IncarnateResult; diff --git a/packages/daemon/test/test-endo.js b/packages/daemon/test/test-endo.js index 6f14b2c26e..d0b9cfbb05 100644 --- a/packages/daemon/test/test-endo.js +++ b/packages/daemon/test/test-endo.js @@ -1221,52 +1221,6 @@ test('guest cannot access host methods', async t => { t.is(revealedTarget, undefined); }); -test('loopback network', async t => { - const { promise: cancelled, reject: cancel } = makePromiseKit(); - t.teardown(() => cancel(Error('teardown'))); - const locator = makeLocator('tmp', 'loopback'); - - await stop(locator).catch(() => {}); - await purge(locator); - await start(locator); - - const { getBootstrap } = await makeEndoClient( - 'client', - locator.sockPath, - cancelled, - ); - const bootstrap = getBootstrap(); - const host = E(bootstrap).host(); - - // Creates a guest "guest" from the inviter side. - const { powers } = await E(host).invite('guest'); - // We manually create the invitation to specify the network. - const invitation = { - addresses: ['loop:'], - powers, - }; - // Stores the received value as "peer". - const peerFromAccept = await E(host).accept(invitation, 'peer'); - const peerByName = await E(host).lookup('peer'); - t.is(peerFromAccept, peerByName); - - const guestId = await E(host).identify('guest'); - const peerId = await E(host).identify('peer'); - t.not(guestId, peerId); - - const guest = await E(host).lookup('guest'); - const peer = peerFromAccept; - await E(guest).copy(['SELF'], ['a']); - await E(peer).copy(['SELF'], ['b']); - const guestNames = await E(guest).list(); - const peerNames = await E(peer).list(); - t.deepEqual(guestNames, peerNames); - t.assert(guestNames.includes('a')); - t.assert(guestNames.includes('b')); - - await stop(locator); -}); - test('read unknown location', async t => { const { promise: cancelled, reject: cancel } = makePromiseKit(); t.teardown(() => cancel(Error('teardown'))); From f7e7f21cc435bd95504fe4384bf1e7741cc8874c Mon Sep 17 00:00:00 2001 From: kumavis Date: Thu, 7 Mar 2024 10:19:33 -1000 Subject: [PATCH 12/20] test(daemon): launch tcp on a system specified port --- packages/daemon/test/test-endo.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/daemon/test/test-endo.js b/packages/daemon/test/test-endo.js index d0b9cfbb05..8d6d7924b1 100644 --- a/packages/daemon/test/test-endo.js +++ b/packages/daemon/test/test-endo.js @@ -1291,7 +1291,7 @@ test('read remote location', async t => { const iteratorRef = E(hostA).followMessages(); const { value: message } = await E(iteratorRef).next(); const { number } = E.get(message); - await E(hostA).evaluate('MAIN', '`127.0.0.1:8920`', [], [], 'netport'); + await E(hostA).evaluate('MAIN', '`127.0.0.1:0`', [], [], 'netport'); await E(hostA).resolve(await number, 'netport'); // move test network to network dir await networkA; @@ -1328,7 +1328,7 @@ test('read remote location', async t => { const iteratorRef = E(hostB).followMessages(); const { value: message } = await E(iteratorRef).next(); const { number } = E.get(message); - await E(hostB).evaluate('MAIN', '`127.0.0.1:8921`', [], [], 'netport'); + await E(hostB).evaluate('MAIN', '`127.0.0.1:0`', [], [], 'netport'); await E(hostB).resolve(await number, 'netport'); // move test network to network dir await networkB; From 9ad0e3cbc5efa3f89cb67703b96fad35062b098d Mon Sep 17 00:00:00 2001 From: kumavis Date: Thu, 7 Mar 2024 10:25:22 -1000 Subject: [PATCH 13/20] fix(daemon): rename Gateway and Peer method to "provide" --- packages/daemon/src/daemon.js | 10 +++------- packages/daemon/src/host.js | 2 +- packages/daemon/src/networks/loopback.js | 2 +- packages/daemon/src/types.d.ts | 8 ++------ 4 files changed, 7 insertions(+), 15 deletions(-) diff --git a/packages/daemon/src/daemon.js b/packages/daemon/src/daemon.js index 6d886806e0..40995e4010 100644 --- a/packages/daemon/src/daemon.js +++ b/packages/daemon/src/daemon.js @@ -1328,11 +1328,9 @@ const makeDaemonCore = async ( makeFarContext(context), ); const external = Promise.resolve({ - provideValueForFormulaIdentifier: remoteFormulaIdentifier => { + provide: remoteFormulaIdentifier => { return /** @type {Promise} */ ( - E(remoteGateway).provideValueForFormulaIdentifier( - remoteFormulaIdentifier, - ) + E(remoteGateway).provide(remoteFormulaIdentifier) ); }, }); @@ -1363,9 +1361,7 @@ const makeDaemonCore = async ( const peer = /** @type {import('./types.js').EndoPeer} */ ( await provideValueForFormulaIdentifier(peerFormulaIdentifier) ); - const remoteValueP = peer.provideValueForFormulaIdentifier( - remoteFormulaIdentifier, - ); + const remoteValueP = peer.provide(remoteFormulaIdentifier); const external = remoteValueP; const internal = Promise.resolve(undefined); return harden({ internal, external }); diff --git a/packages/daemon/src/host.js b/packages/daemon/src/host.js index 50a12fbcb0..7042cce377 100644 --- a/packages/daemon/src/host.js +++ b/packages/daemon/src/host.js @@ -526,7 +526,7 @@ export const makeHostMaker = ({ /** @type {import('./types.js').EndoHost['gateway']} */ const gateway = async () => { // TODO: we should only materialize local objects this way - return Far('Gateway', { provideValueForFormulaIdentifier }); + return Far('Gateway', { provide: provideValueForFormulaIdentifier }); }; /** @type {import('./types.js').EndoHost['addPeerInfo']} */ diff --git a/packages/daemon/src/networks/loopback.js b/packages/daemon/src/networks/loopback.js index 480f268b21..9c4abdf0f5 100644 --- a/packages/daemon/src/networks/loopback.js +++ b/packages/daemon/src/networks/loopback.js @@ -18,7 +18,7 @@ export const makeLoopbackNetwork = ({ provideValueForFormulaIdentifier }) => { 'Failed invariant: loopback only supports loop: address', ); } - return { provideValueForFormulaIdentifier }; + return { provide: provideValueForFormulaIdentifier }; }, }), ); diff --git a/packages/daemon/src/types.d.ts b/packages/daemon/src/types.d.ts index 829c9994c0..946c202510 100644 --- a/packages/daemon/src/types.d.ts +++ b/packages/daemon/src/types.d.ts @@ -453,17 +453,13 @@ export type MakeHostOrGuestOptions = { }; export interface EndoPeer { - provideValueForFormulaIdentifier: ( - formulaIdentifier: string, - ) => Promise; + provide: (formulaIdentifier: string) => Promise; } export type EndoPeerControllerPartial = ControllerPartial; export type EndoPeerController = Controller; export interface EndoGateway { - provideValueForFormulaIdentifier: ( - formulaIdentifier: string, - ) => Promise; + provide: (formulaIdentifier: string) => Promise; } export interface PeerInfo { From dabf46e4658592b1f5c5d9ece55d2b791fa4980c Mon Sep 17 00:00:00 2001 From: kumavis Date: Thu, 7 Mar 2024 11:29:44 -1000 Subject: [PATCH 14/20] refactor(daemon): only construct the gateway once --- packages/daemon/src/daemon.js | 10 ++++++++++ packages/daemon/src/host.js | 17 +++++++++++------ packages/daemon/src/types.d.ts | 1 + 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/packages/daemon/src/daemon.js b/packages/daemon/src/daemon.js index 40995e4010..1618b60949 100644 --- a/packages/daemon/src/daemon.js +++ b/packages/daemon/src/daemon.js @@ -470,6 +470,13 @@ const makeDaemonCore = async ( }, }; } else if (formula.type === 'endo') { + // Gateway is equivalent to E's "nonce locator". It provides a value for + // a formula identifier to a remote client. + // TODO: we should only materialize local objects this way + const gateway = Far('Gateway', { + // eslint-disable-next-line no-use-before-define + provide: provideValueForFormulaIdentifier, + }); /** @type {import('./types.js').FarEndoBootstrap} */ const endoBootstrap = Far('Endo private facet', { // TODO for user named @@ -513,6 +520,9 @@ const makeDaemonCore = async ( const bundle = await E(bundleBlob).json(); await E(webPageP).makeBundle(bundle, endowedPowers); }, + gateway: async () => { + return gateway; + }, reviveNetworks: async () => { const networksDirectory = /** @type {import('./types.js').EndoDirectory} */ ( diff --git a/packages/daemon/src/host.js b/packages/daemon/src/host.js index 7042cce377..9512639f99 100644 --- a/packages/daemon/src/host.js +++ b/packages/daemon/src/host.js @@ -89,6 +89,14 @@ export const makeHostMaker = ({ const directory = makeDirectoryNode(petStore); const { lookup } = directory; + const getEndoBootstrap = async () => { + const endoBootstrap = + /** @type {import('./types.js').FarEndoBootstrap} */ ( + await provideValueForFormulaIdentifier(endoFormulaIdentifier) + ); + return endoBootstrap; + }; + /** * @returns {Promise<{ formulaIdentifier: string, value: import('./types').ExternalHandle }>} */ @@ -525,16 +533,13 @@ export const makeHostMaker = ({ /** @type {import('./types.js').EndoHost['gateway']} */ const gateway = async () => { - // TODO: we should only materialize local objects this way - return Far('Gateway', { provide: provideValueForFormulaIdentifier }); + const endoBootstrap = getEndoBootstrap(); + return E(endoBootstrap).gateway(); }; /** @type {import('./types.js').EndoHost['addPeerInfo']} */ const addPeerInfo = async peerInfo => { - const endoBootstrap = - /** @type {import('./types.js').FarEndoBootstrap} */ ( - await lookup('ENDO') - ); + const endoBootstrap = getEndoBootstrap(); await E(endoBootstrap).addPeerInfo(peerInfo); }; diff --git a/packages/daemon/src/types.d.ts b/packages/daemon/src/types.d.ts index 946c202510..9303768610 100644 --- a/packages/daemon/src/types.d.ts +++ b/packages/daemon/src/types.d.ts @@ -576,6 +576,7 @@ export type FarEndoBootstrap = FarRef<{ leastAuthority: () => Promise; webPageJs: () => Promise; importAndEndowInWebPage: () => Promise; + gateway: () => Promise; reviveNetworks: () => Promise; addPeerInfo: (peerInfo: PeerInfo) => Promise; }>; From 622da83c7254ee4890160d806c8368bf35be645c Mon Sep 17 00:00:00 2001 From: kumavis Date: Thu, 7 Mar 2024 11:33:52 -1000 Subject: [PATCH 15/20] feat(daemon): only provide local values over gateway --- packages/daemon/src/daemon.js | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/daemon/src/daemon.js b/packages/daemon/src/daemon.js index 1618b60949..de29747297 100644 --- a/packages/daemon/src/daemon.js +++ b/packages/daemon/src/daemon.js @@ -472,10 +472,21 @@ const makeDaemonCore = async ( } else if (formula.type === 'endo') { // Gateway is equivalent to E's "nonce locator". It provides a value for // a formula identifier to a remote client. - // TODO: we should only materialize local objects this way const gateway = Far('Gateway', { - // eslint-disable-next-line no-use-before-define - provide: provideValueForFormulaIdentifier, + provide: async requestedFormulaIdentifier => { + const { location } = parseFormulaIdentifier( + requestedFormulaIdentifier, + ); + if (location !== ownLocation) { + throw new Error( + `Gateway can only provide local values. Got location ${q( + location, + )}`, + ); + } + // eslint-disable-next-line no-use-before-define + return provideValueForFormulaIdentifier(requestedFormulaIdentifier); + }, }); /** @type {import('./types.js').FarEndoBootstrap} */ const endoBootstrap = Far('Endo private facet', { From d4e47012a0d70a06eec40fec4f2908ed7563b549 Mon Sep 17 00:00:00 2001 From: kumavis Date: Thu, 7 Mar 2024 11:35:30 -1000 Subject: [PATCH 16/20] fix(daemon): directory.listIdentifiers is sorted --- packages/daemon/src/directory.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/daemon/src/directory.js b/packages/daemon/src/directory.js index 0041f52b8c..46511dc3d4 100644 --- a/packages/daemon/src/directory.js +++ b/packages/daemon/src/directory.js @@ -107,7 +107,7 @@ export const makeDirectoryMaker = ({ } }), ); - return Array.from(identities); + return harden(Array.from(identities).sort()); }; /** @type {import('./types.js').EndoDirectory['followChanges']} */ From 793adefb2eeed5f49443fa841df96239f097dbc3 Mon Sep 17 00:00:00 2001 From: kumavis Date: Thu, 7 Mar 2024 11:52:36 -1000 Subject: [PATCH 17/20] feat(daemon): rename "location" to "node" --- packages/daemon/src/daemon.js | 100 ++++++++++++---------- packages/daemon/src/formula-identifier.js | 8 +- packages/daemon/src/host.js | 6 +- packages/daemon/src/types.d.ts | 6 +- packages/daemon/test/test-endo.js | 18 ++-- 5 files changed, 72 insertions(+), 66 deletions(-) diff --git a/packages/daemon/src/daemon.js b/packages/daemon/src/daemon.js index de29747297..160f679ac8 100644 --- a/packages/daemon/src/daemon.js +++ b/packages/daemon/src/daemon.js @@ -64,6 +64,20 @@ const makeFarContext = context => addDisposalHook: context.onCancel, }); +/** + * + * @param {string} path + * @param {string} rootNonce + * @param {import('./types.js').Sha512} digester + * @returns {string} + */ +const getDerivedId = (path, rootNonce, digester) => { + digester.updateText(rootNonce); + digester.updateText(path); + const nonce = digester.digestHex(); + return nonce; +}; + /** * @param {import('./types.js').DaemonicPowers} powers * @param {Promise} webletPortP @@ -88,13 +102,13 @@ const makeDaemonCore = async ( const { randomHex512 } = cryptoPowers; const contentStore = persistencePowers.makeContentSha512Store(); const formulaGraphMutex = makeMutex(); - // eslint-disable-next-line no-use-before-define - const ownLocation = getDerivedId( - 'location', + // This is the id of the node that is hosting the values. + // This will likely get replaced with a public key in the future. + const ownNodeIdentifier = getDerivedId( + 'nodeId', rootEntropy, cryptoPowers.makeSha512(), ); - // eslint-disable-next-line no-use-before-define const peersFormulaNumber = getDerivedId( 'peers', rootEntropy, @@ -103,7 +117,7 @@ const makeDaemonCore = async ( const peersFormulaIdentifier = serializeFormulaIdentifier({ type: 'pet-store', number: peersFormulaNumber, - location: ownLocation, + node: ownNodeIdentifier, }); /** @@ -474,13 +488,11 @@ const makeDaemonCore = async ( // a formula identifier to a remote client. const gateway = Far('Gateway', { provide: async requestedFormulaIdentifier => { - const { location } = parseFormulaIdentifier( - requestedFormulaIdentifier, - ); - if (location !== ownLocation) { + const { node } = parseFormulaIdentifier(requestedFormulaIdentifier); + if (node !== ownNodeIdentifier) { throw new Error( - `Gateway can only provide local values. Got location ${q( - location, + `Gateway can only provide local values. Got request for node ${q( + node, )}`, ); } @@ -557,10 +569,10 @@ const makeDaemonCore = async ( // Behold, recursion: // eslint-disable-next-line no-use-before-define (await provideValueForFormulaIdentifier(formula.peers)); - const { location, addresses } = peerInfo; + const { node, addresses } = peerInfo; // eslint-disable-next-line no-use-before-define - const locationName = petStoreNameForLocation(location); - if (peerPetstore.has(locationName)) { + const nodeName = petStoreNameForNodeIdentifier(node); + if (peerPetstore.has(nodeName)) { // We already have this peer. // TODO: merge connection info return; @@ -568,7 +580,7 @@ const makeDaemonCore = async ( const { formulaIdentifier: peerFormulaIdentifier } = // eslint-disable-next-line no-use-before-define await incarnatePeer(formula.networks, addresses); - await peerPetstore.write(locationName, peerFormulaIdentifier); + await peerPetstore.write(nodeName, peerFormulaIdentifier); }, }); return { @@ -654,13 +666,13 @@ const makeDaemonCore = async ( const { type: formulaType, number: formulaNumber, - location: formulaLocation, + node: formulaNode, } = parseFormulaIdentifier(formulaIdentifier); - const isRemote = formulaLocation !== ownLocation; + const isRemote = formulaNode !== ownNodeIdentifier; if (isRemote) { // eslint-disable-next-line no-use-before-define - const peerIdentifier = await getPeerFormulaIdentifierForLocation( - formulaLocation, + const peerIdentifier = await getPeerFormulaIdentifierForNodeIdentifier( + formulaNode, ); // Behold, forward reference: // eslint-disable-next-line no-use-before-define @@ -715,7 +727,7 @@ const makeDaemonCore = async ( const formulaIdentifier = serializeFormulaIdentifier({ type: formulaType, number: formulaNumber, - location: ownLocation, + node: ownNodeIdentifier, }); // Memoize for lookup. @@ -794,21 +806,29 @@ const makeDaemonCore = async ( return controller; }; - // TODO: sorry, forcing location into a petstore name - const petStoreNameForLocation = location => { - const locationName = `p${location.slice(0, 126)}`; - return locationName; + // TODO: sorry, forcing nodeId into a petstore name + const petStoreNameForNodeIdentifier = nodeIdentifier => { + return `p${nodeIdentifier.slice(0, 126)}`; }; - const getPeerFormulaIdentifierForLocation = async location => { + /** + * @param {string} nodeIdentifier + * @returns {Promise} + */ + const getPeerFormulaIdentifierForNodeIdentifier = async nodeIdentifier => { + if (nodeIdentifier === ownNodeIdentifier) { + throw new Error(`Cannot get peer formula identifier for self`); + } const peerStore = /** @type {import('./types.js').PetStore} */ ( // eslint-disable-next-line no-use-before-define await provideValueForFormulaIdentifier(peersFormulaIdentifier) ); - const locationName = petStoreNameForLocation(location); - const peerFormulaIdentifier = peerStore.identifyLocal(locationName); + const nodeName = petStoreNameForNodeIdentifier(nodeIdentifier); + const peerFormulaIdentifier = peerStore.identifyLocal(nodeName); if (peerFormulaIdentifier === undefined) { - throw new Error(`No peer found for location ${q(location)}.`); + throw new Error( + `No peer found for node identifier ${q(nodeIdentifier)}.`, + ); } return peerFormulaIdentifier; }; @@ -1022,7 +1042,7 @@ const makeDaemonCore = async ( const ownFormulaIdentifier = serializeFormulaIdentifier({ type: 'eval', number: ownFormulaNumber, - location: ownLocation, + node: ownNodeIdentifier, }); const workerFormulaNumber = await (specifiedWorkerFormulaIdentifier ? parseFormulaIdentifier(specifiedWorkerFormulaIdentifier).number @@ -1241,7 +1261,7 @@ const makeDaemonCore = async ( const endoFormulaIdentifier = serializeFormulaIdentifier({ type: 'endo', number: formulaNumber, - location: ownLocation, + node: ownNodeIdentifier, }); const { formulaIdentifier: defaultHostWorkerFormulaIdentifier } = @@ -1427,7 +1447,7 @@ const makeDaemonCore = async ( makeMailbox, makeDirectoryNode, getAllNetworkAddresses, - ownLocation, + ownNodeIdentifier, }); /** @@ -1560,7 +1580,7 @@ const makeDaemonCore = async ( /** @type {import('./types.js').DaemonCore} */ const daemonCore = { - ownLocation, + nodeIdentifier: ownNodeIdentifier, provideControllerForFormulaIdentifier, provideControllerForFormulaIdentifierAndResolveHandle, provideValueForFormulaIdentifier, @@ -1593,20 +1613,6 @@ const makeDaemonCore = async ( return daemonCore; }; -/** - * - * @param {string} path - * @param {string} rootNonce - * @param {import('./types.js').Sha512} digester - * @returns {string} - */ -const getDerivedId = (path, rootNonce, digester) => { - digester.updateText(rootNonce); - digester.updateText(path); - const nonce = digester.digestHex(); - return nonce; -}; - /** * @param {import('./types.js').DaemonicPowers} powers * @param {Promise} webletPortP @@ -1639,7 +1645,7 @@ const provideEndoBootstrap = async ( const endoFormulaIdentifier = serializeFormulaIdentifier({ type: 'endo', number: endoFormulaNumber, - location: daemonCore.ownLocation, + node: daemonCore.nodeIdentifier, }); return /** @type {Promise} */ ( daemonCore.provideValueForFormulaIdentifier(endoFormulaIdentifier) diff --git a/packages/daemon/src/formula-identifier.js b/packages/daemon/src/formula-identifier.js index d1d1191421..c50d89a07a 100644 --- a/packages/daemon/src/formula-identifier.js +++ b/packages/daemon/src/formula-identifier.js @@ -12,13 +12,13 @@ export const parseFormulaIdentifier = formulaIdentifier => { ); } - const [type, number, location] = formulaIdentifier.split(':'); - return { type, number, location }; + const [type, number, node] = formulaIdentifier.split(':'); + return { type, number, node }; }; /** * @param {import("./types").FormulaIdentifierRecord} formulaRecord * @returns {string} */ -export const serializeFormulaIdentifier = ({ type, number, location }) => - `${type}:${number}:${location}`; +export const serializeFormulaIdentifier = ({ type, number, node }) => + `${type}:${number}:${node}`; diff --git a/packages/daemon/src/host.js b/packages/daemon/src/host.js index 9512639f99..5c11d93440 100644 --- a/packages/daemon/src/host.js +++ b/packages/daemon/src/host.js @@ -24,7 +24,7 @@ const { quote: q } = assert; * @param {import('./types.js').DaemonCore['getAllNetworkAddresses']} args.getAllNetworkAddresses * @param {import('./types.js').MakeMailbox} args.makeMailbox * @param {import('./types.js').MakeDirectoryNode} args.makeDirectoryNode - * @param {string} args.ownLocation + * @param {string} args.ownNodeIdentifier */ export const makeHostMaker = ({ provideValueForFormulaIdentifier, @@ -42,7 +42,7 @@ export const makeHostMaker = ({ getAllNetworkAddresses, makeMailbox, makeDirectoryNode, - ownLocation, + ownNodeIdentifier, }) => { /** * @param {string} hostFormulaIdentifier @@ -549,7 +549,7 @@ export const makeHostMaker = ({ networksDirectoryFormulaIdentifier, ); const peerInfo = { - location: ownLocation, + node: ownNodeIdentifier, addresses, }; return peerInfo; diff --git a/packages/daemon/src/types.d.ts b/packages/daemon/src/types.d.ts index 9303768610..feeb13595b 100644 --- a/packages/daemon/src/types.d.ts +++ b/packages/daemon/src/types.d.ts @@ -55,7 +55,7 @@ export type MignonicPowers = { type FormulaIdentifierRecord = { type: string; number: string; - location: string; + node: string; }; type EndoFormula = { @@ -463,7 +463,7 @@ export interface EndoGateway { } export interface PeerInfo { - location: string; + node: string; addresses: string[]; } @@ -707,7 +707,7 @@ export type DaemonicPowers = { type IncarnateResult = Promise<{ formulaIdentifier: string; value: T }>; export interface DaemonCore { - ownLocation: string; + nodeIdentifier: string; provideValueForFormulaIdentifier: ( formulaIdentifier: string, ) => Promise; diff --git a/packages/daemon/test/test-endo.js b/packages/daemon/test/test-endo.js index 8d6d7924b1..05e00f7f2f 100644 --- a/packages/daemon/test/test-endo.js +++ b/packages/daemon/test/test-endo.js @@ -1221,10 +1221,10 @@ test('guest cannot access host methods', async t => { t.is(revealedTarget, undefined); }); -test('read unknown location', async t => { +test('read unknown nodeId', async t => { const { promise: cancelled, reject: cancel } = makePromiseKit(); t.teardown(() => cancel(Error('teardown'))); - const locator = makeLocator('tmp', 'read unknown location'); + const locator = makeLocator('tmp', 'read unknown nodeId'); await stop(locator).catch(() => {}); await purge(locator); @@ -1238,29 +1238,29 @@ test('read unknown location', async t => { const bootstrap = getBootstrap(); const host = E(bootstrap).host(); - // write a bogus value for a bogus location - const location = await cryptoPowers.randomHex512(); + // write a bogus value for a bogus nodeId + const node = await cryptoPowers.randomHex512(); const number = await cryptoPowers.randomHex512(); const type = 'eval'; const formulaIdentifier = serializeFormulaIdentifier({ - location, + node, number, type, }); await E(host).write(['abc'], formulaIdentifier); // observe reification failure t.throwsAsync(() => E(host).lookup('abc'), { - message: /No peer found for location /u, + message: /No peer found for node identifier /u, }); await stop(locator); }); -test('read remote location', async t => { +test('read remote value', async t => { const { promise: cancelled, reject: cancel } = makePromiseKit(); t.teardown(() => cancel(Error('teardown'))); - const locatorA = makeLocator('tmp', 'read remote location A'); - const locatorB = makeLocator('tmp', 'read remote location B'); + const locatorA = makeLocator('tmp', 'read remote value A'); + const locatorB = makeLocator('tmp', 'read remote value B'); let hostA; { await stop(locatorA).catch(() => {}); From e08f03ef11b27d435daaac1e4c5e683e0c3aac37 Mon Sep 17 00:00:00 2001 From: kumavis Date: Thu, 7 Mar 2024 11:54:17 -1000 Subject: [PATCH 18/20] fix(daemon): remove orphaned remote formula type check --- packages/daemon/src/pet-store.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/daemon/src/pet-store.js b/packages/daemon/src/pet-store.js index a4ee80a337..8bfcf31fef 100644 --- a/packages/daemon/src/pet-store.js +++ b/packages/daemon/src/pet-store.js @@ -7,7 +7,7 @@ const { quote: q } = assert; const validIdPattern = /^[0-9a-f]{128}$/; const validFormulaPattern = - /^(?:(?:readable-blob|worker|pet-store|pet-inspector|eval|lookup|make-unconfined|make-bundle|host|guest|handle|peer|remote|loopback-network):[0-9a-f]{128}:[0-9a-f]{128}|web-bundle:[0-9a-f]{32}:[0-9a-f]{128})$/; + /^(?:(?:readable-blob|worker|pet-store|pet-inspector|eval|lookup|make-unconfined|make-bundle|host|guest|handle|peer|loopback-network):[0-9a-f]{128}:[0-9a-f]{128}|web-bundle:[0-9a-f]{32}:[0-9a-f]{128})$/; /** * @param {string} formulaIdentifier From 01812a282ad1820bbe985ef63845ee1a28dfe6ad Mon Sep 17 00:00:00 2001 From: kumavis Date: Thu, 7 Mar 2024 11:56:08 -1000 Subject: [PATCH 19/20] fix(daemon): fix loopback network error message format --- packages/daemon/src/networks/loopback.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/daemon/src/networks/loopback.js b/packages/daemon/src/networks/loopback.js index 9c4abdf0f5..4d1af0a1a2 100644 --- a/packages/daemon/src/networks/loopback.js +++ b/packages/daemon/src/networks/loopback.js @@ -15,7 +15,7 @@ export const makeLoopbackNetwork = ({ provideValueForFormulaIdentifier }) => { connect: address => { if (address !== 'loop:') { throw new Error( - 'Failed invariant: loopback only supports loop: address', + 'Failed invariant: loopback only supports "loop:" address', ); } return { provide: provideValueForFormulaIdentifier }; From f0451bd3722c74956cfad93c8891a8eb7ec88a7e Mon Sep 17 00:00:00 2001 From: kumavis Date: Thu, 7 Mar 2024 12:01:07 -1000 Subject: [PATCH 20/20] fix(daemon): rename "makeRemote" to "provideRemoteValue" --- packages/daemon/src/daemon.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/daemon/src/daemon.js b/packages/daemon/src/daemon.js index 160f679ac8..f469d8542b 100644 --- a/packages/daemon/src/daemon.js +++ b/packages/daemon/src/daemon.js @@ -676,7 +676,7 @@ const makeDaemonCore = async ( ); // Behold, forward reference: // eslint-disable-next-line no-use-before-define - return makeRemote(peerIdentifier, formulaIdentifier, context); + return provideRemoteValue(peerIdentifier, formulaIdentifier, context); } if ( [ @@ -1389,20 +1389,22 @@ const makeDaemonCore = async ( }; /** + * This is used to provide a value for a formula identifier that is known to + * originate from the specified peer. * @param {string} peerFormulaIdentifier - * @param {string} remoteFormulaIdentifier + * @param {string} remoteValueFormulaIdentifier * @param {import('./types.js').Context} context * @returns {Promise>} */ - const makeRemote = async ( + const provideRemoteValue = async ( peerFormulaIdentifier, - remoteFormulaIdentifier, + remoteValueFormulaIdentifier, context, ) => { const peer = /** @type {import('./types.js').EndoPeer} */ ( await provideValueForFormulaIdentifier(peerFormulaIdentifier) ); - const remoteValueP = peer.provide(remoteFormulaIdentifier); + const remoteValueP = peer.provide(remoteValueFormulaIdentifier); const external = remoteValueP; const internal = Promise.resolve(undefined); return harden({ internal, external });