diff --git a/packages/daemon/src/daemon.js b/packages/daemon/src/daemon.js index c4435d38c5..f469d8542b 100644 --- a/packages/daemon/src/daemon.js +++ b/packages/daemon/src/daemon.js @@ -13,9 +13,13 @@ import { makeGuestMaker } from './guest.js'; import { makeHostMaker } from './host.js'; import { assertPetName } from './pet-name.js'; import { makeContextMaker } from './context.js'; -import { parseFormulaIdentifier } from './formula-identifier.js'; +import { + parseFormulaIdentifier, + serializeFormulaIdentifier, +} from './formula-identifier.js'; import { makeMutex } from './mutex.js'; import { makeWeakMultimap } from './weak-multimap.js'; +import { makeLoopbackNetwork } from './networks/loopback.js'; const delay = async (ms, cancelled) => { // Do not attempt to set up a timer if already cancelled. @@ -60,9 +64,24 @@ const makeFarContext = context => addDisposalHook: context.onCancel, }); +/** + * + * @param {string} path + * @param {string} rootNonce + * @param {import('./types.js').Sha512} digester + * @returns {string} + */ +const getDerivedId = (path, rootNonce, digester) => { + digester.updateText(rootNonce); + digester.updateText(path); + const nonce = digester.digestHex(); + return nonce; +}; + /** * @param {import('./types.js').DaemonicPowers} powers * @param {Promise} webletPortP + * @param {string} rootEntropy * @param {object} args * @param {(error: Error) => void} args.cancel * @param {number} args.gracePeriodMs @@ -71,6 +90,7 @@ const makeFarContext = context => const makeDaemonCore = async ( powers, webletPortP, + rootEntropy, { cancel, gracePeriodMs, gracePeriodElapsed }, ) => { const { @@ -82,6 +102,23 @@ const makeDaemonCore = async ( const { randomHex512 } = cryptoPowers; const contentStore = persistencePowers.makeContentSha512Store(); const formulaGraphMutex = makeMutex(); + // This is the id of the node that is hosting the values. + // This will likely get replaced with a public key in the future. + const ownNodeIdentifier = getDerivedId( + 'nodeId', + rootEntropy, + cryptoPowers.makeSha512(), + ); + const peersFormulaNumber = getDerivedId( + 'peers', + rootEntropy, + cryptoPowers.makeSha512(), + ); + const peersFormulaIdentifier = serializeFormulaIdentifier({ + type: 'pet-store', + number: peersFormulaNumber, + node: ownNodeIdentifier, + }); /** * The two functions "provideValueForNumberedFormula" and "provideValueForFormulaIdentifier" @@ -402,10 +439,11 @@ const makeDaemonCore = async ( // eslint-disable-next-line no-use-before-define return makeIdentifiedHost( formulaIdentifier, - formula.endo, formula.petStore, formula.inspector, formula.worker, + formula.endo, + formula.networks, formula.leastAuthority, context, ); @@ -446,6 +484,22 @@ const makeDaemonCore = async ( }, }; } else if (formula.type === 'endo') { + // Gateway is equivalent to E's "nonce locator". It provides a value for + // a formula identifier to a remote client. + const gateway = Far('Gateway', { + provide: async requestedFormulaIdentifier => { + const { node } = parseFormulaIdentifier(requestedFormulaIdentifier); + if (node !== ownNodeIdentifier) { + throw new Error( + `Gateway can only provide local values. Got request for node ${q( + node, + )}`, + ); + } + // eslint-disable-next-line no-use-before-define + return provideValueForFormulaIdentifier(requestedFormulaIdentifier); + }, + }); /** @type {import('./types.js').FarEndoBootstrap} */ const endoBootstrap = Far('Endo private facet', { // TODO for user named @@ -489,11 +543,60 @@ const makeDaemonCore = async ( const bundle = await E(bundleBlob).json(); await E(webPageP).makeBundle(bundle, endowedPowers); }, + gateway: async () => { + return gateway; + }, + reviveNetworks: async () => { + const networksDirectory = + /** @type {import('./types.js').EndoDirectory} */ ( + // Behold, recursion: + // eslint-disable-next-line no-use-before-define + await provideValueForFormulaIdentifier(formula.networks) + ); + const networkFormulaIdentifiers = + await networksDirectory.listIdentifiers(); + await Promise.allSettled( + networkFormulaIdentifiers.map( + // Behold, recursion: + // eslint-disable-next-line no-use-before-define + provideValueForFormulaIdentifier, + ), + ); + }, + addPeerInfo: async peerInfo => { + const peerPetstore = + /** @type {import('./types.js').PetStore} */ + // Behold, recursion: + // eslint-disable-next-line no-use-before-define + (await provideValueForFormulaIdentifier(formula.peers)); + const { node, addresses } = peerInfo; + // eslint-disable-next-line no-use-before-define + const nodeName = petStoreNameForNodeIdentifier(node); + if (peerPetstore.has(nodeName)) { + // We already have this peer. + // TODO: merge connection info + return; + } + const { formulaIdentifier: peerFormulaIdentifier } = + // eslint-disable-next-line no-use-before-define + await incarnatePeer(formula.networks, addresses); + await peerPetstore.write(nodeName, peerFormulaIdentifier); + }, }); return { external: endoBootstrap, internal: undefined, }; + } else if (formula.type === 'loopback-network') { + // Behold, forward-reference: + const loopbackNetwork = makeLoopbackNetwork({ + // eslint-disable-next-line no-use-before-define + provideValueForFormulaIdentifier, + }); + return { + external: loopbackNetwork, + internal: undefined, + }; } else if (formula.type === 'least-authority') { const disallowedFn = async () => { throw new Error('not allowed'); @@ -543,22 +646,38 @@ const makeDaemonCore = async ( petStoreFormulaIdentifier: formula.petStore, context, }); + } else if (formula.type === 'peer') { + // Behold, forward reference: + // eslint-disable-next-line no-use-before-define + return makePeer(formula.networks, formula.addresses, context); } else { throw new TypeError(`Invalid formula: ${q(formula)}`); } }; /** - * @param {string} formulaType - * @param {string} formulaNumber + * @param {string} formulaIdentifier * @param {import('./types.js').Context} context */ const makeControllerForFormulaIdentifier = async ( - formulaType, - formulaNumber, + formulaIdentifier, context, ) => { - const formulaIdentifier = `${formulaType}:${formulaNumber}`; + const { + type: formulaType, + number: formulaNumber, + node: formulaNode, + } = parseFormulaIdentifier(formulaIdentifier); + const isRemote = formulaNode !== ownNodeIdentifier; + if (isRemote) { + // eslint-disable-next-line no-use-before-define + const peerIdentifier = await getPeerFormulaIdentifierForNodeIdentifier( + formulaNode, + ); + // Behold, forward reference: + // eslint-disable-next-line no-use-before-define + return provideRemoteValue(peerIdentifier, formulaIdentifier, context); + } if ( [ 'endo', @@ -570,6 +689,8 @@ const makeDaemonCore = async ( 'host', 'guest', 'least-authority', + 'loopback-network', + 'peer', 'web-bundle', 'web-page-js', 'handle', @@ -603,7 +724,11 @@ const makeDaemonCore = async ( formulaNumber, formula, ) => { - const formulaIdentifier = `${formulaType}:${formulaNumber}`; + const formulaIdentifier = serializeFormulaIdentifier({ + type: formulaType, + number: formulaNumber, + node: ownNodeIdentifier, + }); // Memoize for lookup. console.log(`Making ${formulaIdentifier}`); @@ -654,9 +779,6 @@ const makeDaemonCore = async ( /** @type {import('./types.js').DaemonCore['provideControllerForFormulaIdentifier']} */ const provideControllerForFormulaIdentifier = formulaIdentifier => { - const { type: formulaType, number: formulaNumber } = - parseFormulaIdentifier(formulaIdentifier); - let controller = controllerForFormulaIdentifier.get(formulaIdentifier); if (controller !== undefined) { return controller; @@ -679,13 +801,38 @@ const makeDaemonCore = async ( }); controllerForFormulaIdentifier.set(formulaIdentifier, controller); - resolve( - makeControllerForFormulaIdentifier(formulaType, formulaNumber, context), - ); + resolve(makeControllerForFormulaIdentifier(formulaIdentifier, context)); return controller; }; + // TODO: sorry, forcing nodeId into a petstore name + const petStoreNameForNodeIdentifier = nodeIdentifier => { + return `p${nodeIdentifier.slice(0, 126)}`; + }; + + /** + * @param {string} nodeIdentifier + * @returns {Promise} + */ + const getPeerFormulaIdentifierForNodeIdentifier = async nodeIdentifier => { + if (nodeIdentifier === ownNodeIdentifier) { + throw new Error(`Cannot get peer formula identifier for self`); + } + const peerStore = /** @type {import('./types.js').PetStore} */ ( + // eslint-disable-next-line no-use-before-define + await provideValueForFormulaIdentifier(peersFormulaIdentifier) + ); + const nodeName = petStoreNameForNodeIdentifier(nodeIdentifier); + const peerFormulaIdentifier = peerStore.identifyLocal(nodeName); + if (peerFormulaIdentifier === undefined) { + throw new Error( + `No peer found for node identifier ${q(nodeIdentifier)}.`, + ); + } + return peerFormulaIdentifier; + }; + /** @type {import('./types.js').DaemonCore['cancelValue']} */ const cancelValue = async (formulaIdentifier, reason) => { await formulaGraphMutex.enqueue(); @@ -766,8 +913,8 @@ const makeDaemonCore = async ( /** * @type {import('./types.js').DaemonCore['incarnatePetStore']} */ - const incarnatePetStore = async () => { - const formulaNumber = await randomHex512(); + const incarnatePetStore = async specifiedFormulaNumber => { + const formulaNumber = specifiedFormulaNumber ?? (await randomHex512()); /** @type {import('./types.js').PetStoreFormula} */ const formula = { type: 'pet-store', @@ -828,6 +975,7 @@ const makeDaemonCore = async ( /** @type {import('./types.js').DaemonCore['incarnateHost']} */ const incarnateHost = async ( endoFormulaIdentifier, + networksDirectoryFormulaIdentifier, leastAuthorityFormulaIdentifier, specifiedWorkerFormulaIdentifier, ) => { @@ -849,6 +997,7 @@ const makeDaemonCore = async ( inspector: inspectorFormulaIdentifier, worker: workerFormulaIdentifier, endo: endoFormulaIdentifier, + networks: networksDirectoryFormulaIdentifier, leastAuthority: leastAuthorityFormulaIdentifier, }; return /** @type {import('./types').IncarnateResult} */ ( @@ -877,7 +1026,7 @@ const makeDaemonCore = async ( /** @type {import('./types.js').DaemonCore['incarnateEval']} */ const incarnateEval = async ( - hostFormulaIdentifier, + nameHubFormulaIdentifier, source, codeNames, endowmentFormulaIdsOrPaths, @@ -887,9 +1036,14 @@ const makeDaemonCore = async ( const { workerFormulaIdentifier, endowmentFormulaIdentifiers, - evalFormulaNumber, + evalFormulaIdentifier, } = await formulaGraphMutex.enqueue(async () => { const ownFormulaNumber = await randomHex512(); + const ownFormulaIdentifier = serializeFormulaIdentifier({ + type: 'eval', + number: ownFormulaNumber, + node: ownNodeIdentifier, + }); const workerFormulaNumber = await (specifiedWorkerFormulaIdentifier ? parseFormulaIdentifier(specifiedWorkerFormulaIdentifier).number : randomHex512()); @@ -908,7 +1062,7 @@ const makeDaemonCore = async ( ( await incarnateNumberedLookup( await randomHex512(), - hostFormulaIdentifier, + nameHubFormulaIdentifier, formulaIdOrPath, ) ).formulaIdentifier @@ -916,13 +1070,16 @@ const makeDaemonCore = async ( ); }), ), - evalFormulaNumber: ownFormulaNumber, + evalFormulaIdentifier: ownFormulaIdentifier, }); await Promise.all(hooks.map(hook => hook(identifiers))); return identifiers; }); + const { number: evalFormulaNumber } = parseFormulaIdentifier( + evalFormulaIdentifier, + ); /** @type {import('./types.js').EvalFormula} */ const formula = { type: 'eval', @@ -1057,20 +1214,75 @@ const makeDaemonCore = async ( ); }; + /** @type {import('./types.js').DaemonCore['incarnatePeer']} */ + const incarnatePeer = async ( + networksDirectoryFormulaIdentifier, + addresses, + ) => { + const formulaNumber = await randomHex512(); + // TODO: validate addresses + // TODO: mutable state like addresses should not be stored in formula + /** @type {import('./types.js').PeerFormula} */ + const formula = { + type: 'peer', + networks: networksDirectoryFormulaIdentifier, + addresses, + }; + return /** @type {import('./types').IncarnateResult} */ ( + provideValueForNumberedFormula(formula.type, formulaNumber, formula) + ); + }; + + /** @type {import('./types.js').DaemonCore['incarnateLoopbackNetwork']} */ + const incarnateLoopbackNetwork = async () => { + const formulaNumber = await randomHex512(); + /** @type {import('./types').LoopbackNetworkFormula} */ + const formula = { + type: 'loopback-network', + }; + return /** @type {import('./types').IncarnateResult} */ ( + provideValueForNumberedFormula(formula.type, formulaNumber, formula) + ); + }; + + /** @type {import('./types.js').DaemonCore['incarnateNetworksDirectory']} */ + const incarnateNetworksDirectory = async () => { + const { formulaIdentifier, value } = await incarnateDirectory(); + // Make default networks. + const { formulaIdentifier: loopbackNetworkFormulaIdentifier } = + await incarnateLoopbackNetwork(); + await E(value).write(['loop'], loopbackNetworkFormulaIdentifier); + return { formulaIdentifier, value }; + }; + /** @type {import('./types.js').DaemonCore['incarnateEndoBootstrap']} */ const incarnateEndoBootstrap = async specifiedFormulaNumber => { const formulaNumber = await (specifiedFormulaNumber ?? randomHex512()); - const endoFormulaIdentifier = `endo:${formulaNumber}`; + const endoFormulaIdentifier = serializeFormulaIdentifier({ + type: 'endo', + number: formulaNumber, + node: ownNodeIdentifier, + }); const { formulaIdentifier: defaultHostWorkerFormulaIdentifier } = await incarnateWorker(); + const { formulaIdentifier: networksDirectoryFormulaIdentifier } = + await incarnateNetworksDirectory(); const { formulaIdentifier: leastAuthorityFormulaIdentifier } = await incarnateLeastAuthority(); + const { formulaIdentifier: newPeersFormulaIdentifier } = + await incarnatePetStore(peersFormulaNumber); + if (newPeersFormulaIdentifier !== peersFormulaIdentifier) { + throw new Error( + `Peers PetStore formula identifier did not match expected value.`, + ); + } // Ensure the default host is incarnated and persisted. const { formulaIdentifier: defaultHostFormulaIdentifier } = await incarnateHost( endoFormulaIdentifier, + networksDirectoryFormulaIdentifier, leastAuthorityFormulaIdentifier, defaultHostWorkerFormulaIdentifier, ); @@ -1087,6 +1299,8 @@ const makeDaemonCore = async ( /** @type {import('./types.js').EndoFormula} */ const formula = { type: 'endo', + networks: networksDirectoryFormulaIdentifier, + peers: peersFormulaIdentifier, host: defaultHostFormulaIdentifier, leastAuthority: leastAuthorityFormulaIdentifier, webPageJs: webPageJsFormulaIdentifier, @@ -1096,6 +1310,106 @@ const makeDaemonCore = async ( ); }; + /** + * @param {string} networksDirectoryFormulaIdentifier + * @returns {Promise} + */ + const getAllNetworks = async networksDirectoryFormulaIdentifier => { + const networksDirectory = /** @type {import('./types').EndoDirectory} */ ( + // eslint-disable-next-line no-use-before-define + await provideValueForFormulaIdentifier(networksDirectoryFormulaIdentifier) + ); + const networkFormulaIdentifiers = await networksDirectory.listIdentifiers(); + const networks = /** @type {import('./types').EndoNetwork[]} */ ( + await Promise.all( + networkFormulaIdentifiers.map(provideValueForFormulaIdentifier), + ) + ); + return networks; + }; + + /** @type {import('./types.js').DaemonCore['getAllNetworkAddresses']} */ + const getAllNetworkAddresses = async networksDirectoryFormulaIdentifier => { + const networks = await getAllNetworks(networksDirectoryFormulaIdentifier); + const addresses = ( + await Promise.all( + networks.map(async network => { + return E(network).addresses(); + }), + ) + ).flat(); + return addresses; + }; + + /** + * @param {string} networksDirectoryFormulaIdentifier + * @param {string[]} addresses + * @param {import('./types.js').Context} context + * @returns {Promise} + */ + const makePeer = async ( + networksDirectoryFormulaIdentifier, + addresses, + context, + ) => { + // TODO race networks that support protocol for connection + // TODO retry, exponential back-off, with full jitter + // TODO (in connect implementations) allow for the possibility of + // connection loss and invalidate the connection formula and its transitive + // dependees when this occurs. + const networks = await getAllNetworks(networksDirectoryFormulaIdentifier); + // Connect on first support address. + for (const address of addresses) { + const { protocol } = new URL(address); + for (const network of networks) { + // eslint-disable-next-line no-await-in-loop + if (await E(network).supports(protocol)) { + const remoteGateway = E(network).connect( + address, + makeFarContext(context), + ); + const external = Promise.resolve({ + provide: remoteFormulaIdentifier => { + return /** @type {Promise} */ ( + E(remoteGateway).provide(remoteFormulaIdentifier) + ); + }, + }); + const internal = Promise.resolve(undefined); + // const internal = { + // receive, // TODO + // respond, // TODO + // lookupPath, // TODO + // }; + return harden({ internal, external }); + } + } + } + throw new Error('Cannot connect to peer: no supported addresses'); + }; + + /** + * This is used to provide a value for a formula identifier that is known to + * originate from the specified peer. + * @param {string} peerFormulaIdentifier + * @param {string} remoteValueFormulaIdentifier + * @param {import('./types.js').Context} context + * @returns {Promise>} + */ + const provideRemoteValue = async ( + peerFormulaIdentifier, + remoteValueFormulaIdentifier, + context, + ) => { + const peer = /** @type {import('./types.js').EndoPeer} */ ( + await provideValueForFormulaIdentifier(peerFormulaIdentifier) + ); + const remoteValueP = peer.provide(remoteValueFormulaIdentifier); + const external = remoteValueP; + const internal = Promise.resolve(undefined); + return harden({ internal, external }); + }; + const makeContext = makeContextMaker({ controllerForFormulaIdentifier, provideControllerForFormulaIdentifier, @@ -1134,6 +1448,8 @@ const makeDaemonCore = async ( storeReaderRef, makeMailbox, makeDirectoryNode, + getAllNetworkAddresses, + ownNodeIdentifier, }); /** @@ -1241,6 +1557,14 @@ const makeDaemonCore = async ( powers: provideValueForFormulaIdentifier(formula.powers), }), ); + } else if (formula.type === 'peer') { + return makeInspector( + formula.type, + formulaNumber, + harden({ + ADDRESSES: formula.addresses, + }), + ); } return makeInspector(formula.type, formulaNumber, harden({})); }; @@ -1258,23 +1582,28 @@ const makeDaemonCore = async ( /** @type {import('./types.js').DaemonCore} */ const daemonCore = { + nodeIdentifier: ownNodeIdentifier, provideControllerForFormulaIdentifier, provideControllerForFormulaIdentifierAndResolveHandle, provideValueForFormulaIdentifier, provideValueForNumberedFormula, getFormulaIdentifierForRef, + getAllNetworkAddresses, cancelValue, storeReaderRef, makeMailbox, makeDirectoryNode, incarnateEndoBootstrap, incarnateLeastAuthority, + incarnateNetworksDirectory, + incarnateLoopbackNetwork, incarnateHandle, incarnatePetStore, incarnateDirectory, incarnateWorker, incarnateHost, incarnateGuest, + incarnatePeer, incarnateEval, incarnateUnconfined, incarnateReadableBlob, @@ -1301,18 +1630,25 @@ const provideEndoBootstrap = async ( { cancel, gracePeriodMs, gracePeriodElapsed }, ) => { const { persistence: persistencePowers } = powers; - - const daemonCore = await makeDaemonCore(powers, webletPortP, { - cancel, - gracePeriodMs, - gracePeriodElapsed, - }); - const { rootNonce: endoFormulaNumber, isNewlyCreated } = await persistencePowers.provideRootNonce(); + const daemonCore = await makeDaemonCore( + powers, + webletPortP, + endoFormulaNumber, + { + cancel, + gracePeriodMs, + gracePeriodElapsed, + }, + ); const isInitialized = !isNewlyCreated; if (isInitialized) { - const endoFormulaIdentifier = `endo:${endoFormulaNumber}`; + const endoFormulaIdentifier = serializeFormulaIdentifier({ + type: 'endo', + number: endoFormulaNumber, + node: daemonCore.nodeIdentifier, + }); return /** @type {Promise} */ ( daemonCore.provideValueForFormulaIdentifier(endoFormulaIdentifier) ); @@ -1363,5 +1699,7 @@ export const makeDaemon = async (powers, daemonLabel, cancel, cancelled) => { }, ); + await E(endoBootstrap).reviveNetworks(); + return { endoBootstrap, cancelGracePeriod, assignWebletPort }; }; diff --git a/packages/daemon/src/directory.js b/packages/daemon/src/directory.js index f519acdc46..46511dc3d4 100644 --- a/packages/daemon/src/directory.js +++ b/packages/daemon/src/directory.js @@ -95,6 +95,21 @@ export const makeDirectoryMaker = ({ return hub.list(); }; + /** @type {import('./types.js').EndoDirectory['listIdentifiers']} */ + const listIdentifiers = async (...petNamePath) => { + const names = await list(...petNamePath); + const identities = new Set(); + await Promise.all( + names.map(async name => { + const formulaIdentifier = await identify(...petNamePath, name); + if (formulaIdentifier !== undefined) { + identities.add(formulaIdentifier); + } + }), + ); + return harden(Array.from(identities).sort()); + }; + /** @type {import('./types.js').EndoDirectory['followChanges']} */ const followChanges = async function* followChanges(...petNamePath) { if (petNamePath.length === 0) { @@ -179,6 +194,7 @@ export const makeDirectoryMaker = ({ has, identify, list, + listIdentifiers, followChanges, lookup, reverseLookup, diff --git a/packages/daemon/src/formula-identifier.js b/packages/daemon/src/formula-identifier.js index a93e86f014..c50d89a07a 100644 --- a/packages/daemon/src/formula-identifier.js +++ b/packages/daemon/src/formula-identifier.js @@ -12,7 +12,13 @@ export const parseFormulaIdentifier = formulaIdentifier => { ); } - const type = formulaIdentifier.slice(0, delimiterIndex); - const number = formulaIdentifier.slice(delimiterIndex + 1); - return { type, number }; + const [type, number, node] = formulaIdentifier.split(':'); + return { type, number, node }; }; + +/** + * @param {import("./types").FormulaIdentifierRecord} formulaRecord + * @returns {string} + */ +export const serializeFormulaIdentifier = ({ type, number, node }) => + `${type}:${number}:${node}`; diff --git a/packages/daemon/src/guest.js b/packages/daemon/src/guest.js index 4964352925..77b31156f9 100644 --- a/packages/daemon/src/guest.js +++ b/packages/daemon/src/guest.js @@ -69,6 +69,7 @@ export const makeGuestMaker = ({ has, identify, list, + listIdentifiers, followChanges, lookup, reverseLookup, @@ -97,6 +98,7 @@ export const makeGuestMaker = ({ has, identify, list, + listIdentifiers, followChanges, lookup, reverseLookup, diff --git a/packages/daemon/src/host.js b/packages/daemon/src/host.js index 05bcc1a789..5c11d93440 100644 --- a/packages/daemon/src/host.js +++ b/packages/daemon/src/host.js @@ -1,6 +1,6 @@ // @ts-check -import { Far } from '@endo/far'; +import { E, Far } from '@endo/far'; import { makeIteratorRef } from './reader-ref.js'; import { assertPetName, petNamePathFrom } from './pet-name.js'; import { makePetSitter } from './pet-sitter.js'; @@ -21,8 +21,10 @@ const { quote: q } = assert; * @param {import('./types.js').DaemonCore['incarnateWebBundle']} args.incarnateWebBundle * @param {import('./types.js').DaemonCore['incarnateHandle']} args.incarnateHandle * @param {import('./types.js').DaemonCore['storeReaderRef']} args.storeReaderRef + * @param {import('./types.js').DaemonCore['getAllNetworkAddresses']} args.getAllNetworkAddresses * @param {import('./types.js').MakeMailbox} args.makeMailbox * @param {import('./types.js').MakeDirectoryNode} args.makeDirectoryNode + * @param {string} args.ownNodeIdentifier */ export const makeHostMaker = ({ provideValueForFormulaIdentifier, @@ -37,24 +39,28 @@ export const makeHostMaker = ({ incarnateWebBundle, incarnateHandle, storeReaderRef, + getAllNetworkAddresses, makeMailbox, makeDirectoryNode, + ownNodeIdentifier, }) => { /** * @param {string} hostFormulaIdentifier - * @param {string} endoFormulaIdentifier * @param {string} storeFormulaIdentifier * @param {string} inspectorFormulaIdentifier * @param {string} mainWorkerFormulaIdentifier + * @param {string} endoFormulaIdentifier + * @param {string} networksDirectoryFormulaIdentifier * @param {string} leastAuthorityFormulaIdentifier * @param {import('./types.js').Context} context */ const makeIdentifiedHost = async ( hostFormulaIdentifier, - endoFormulaIdentifier, storeFormulaIdentifier, inspectorFormulaIdentifier, mainWorkerFormulaIdentifier, + endoFormulaIdentifier, + networksDirectoryFormulaIdentifier, leastAuthorityFormulaIdentifier, context, ) => { @@ -69,6 +75,7 @@ export const makeHostMaker = ({ const specialStore = makePetSitter(basePetStore, { SELF: hostFormulaIdentifier, ENDO: endoFormulaIdentifier, + NETS: networksDirectoryFormulaIdentifier, INFO: inspectorFormulaIdentifier, NONE: leastAuthorityFormulaIdentifier, }); @@ -80,6 +87,15 @@ export const makeHostMaker = ({ }); const { petStore } = mailbox; const directory = makeDirectoryNode(petStore); + const { lookup } = directory; + + const getEndoBootstrap = async () => { + const endoBootstrap = + /** @type {import('./types.js').FarEndoBootstrap} */ ( + await provideValueForFormulaIdentifier(endoFormulaIdentifier) + ); + return endoBootstrap; + }; /** * @returns {Promise<{ formulaIdentifier: string, value: import('./types').ExternalHandle }>} @@ -326,7 +342,7 @@ export const makeHostMaker = ({ if (resultName !== undefined) { addHook(identifiers => - petStore.write(resultName, `eval:${identifiers.evalFormulaNumber}`), + petStore.write(resultName, identifiers.evalFormulaIdentifier), ); } @@ -438,6 +454,7 @@ export const makeHostMaker = ({ const { formulaIdentifier: newFormulaIdentifier, value } = await incarnateHost( endoFormulaIdentifier, + networksDirectoryFormulaIdentifier, leastAuthorityFormulaIdentifier, ); if (petName !== undefined) { @@ -514,12 +531,36 @@ export const makeHostMaker = ({ return cancelValue(formulaIdentifier, reason); }; + /** @type {import('./types.js').EndoHost['gateway']} */ + const gateway = async () => { + const endoBootstrap = getEndoBootstrap(); + return E(endoBootstrap).gateway(); + }; + + /** @type {import('./types.js').EndoHost['addPeerInfo']} */ + const addPeerInfo = async peerInfo => { + const endoBootstrap = getEndoBootstrap(); + await E(endoBootstrap).addPeerInfo(peerInfo); + }; + + /** @type {import('./types.js').EndoHost['getPeerInfo']} */ + const getPeerInfo = async () => { + const addresses = await getAllNetworkAddresses( + networksDirectoryFormulaIdentifier, + ); + const peerInfo = { + node: ownNodeIdentifier, + addresses, + }; + return peerInfo; + }; + const { has, identify, list, + listIdentifiers, followChanges, - lookup, reverseLookup, write, remove, @@ -546,6 +587,7 @@ export const makeHostMaker = ({ has, identify, list, + listIdentifiers, followChanges, lookup, reverseLookup, @@ -574,6 +616,9 @@ export const makeHostMaker = ({ makeBundle, provideWebPage, cancel, + gateway, + getPeerInfo, + addPeerInfo, }; const external = Far('EndoHost', { diff --git a/packages/daemon/src/networks/loopback.js b/packages/daemon/src/networks/loopback.js new file mode 100644 index 0000000000..4d1af0a1a2 --- /dev/null +++ b/packages/daemon/src/networks/loopback.js @@ -0,0 +1,25 @@ +// @ts-check +import { Far } from '@endo/far'; + +/** + * @param {object} args + * @param {import('../types.js').DaemonCore['provideValueForFormulaIdentifier']} args.provideValueForFormulaIdentifier + * @returns {import('@endo/far').FarRef} + */ +export const makeLoopbackNetwork = ({ provideValueForFormulaIdentifier }) => { + return Far( + 'Loopback Network', + /** @type {import('../types.js').EndoNetwork} */ ({ + addresses: () => [], + supports: address => new URL(address).protocol === 'loop:', + connect: address => { + if (address !== 'loop:') { + throw new Error( + 'Failed invariant: loopback only supports "loop:" address', + ); + } + return { provide: provideValueForFormulaIdentifier }; + }, + }), + ); +}; diff --git a/packages/daemon/src/networks/tcp-netstring.js b/packages/daemon/src/networks/tcp-netstring.js new file mode 100644 index 0000000000..72d14bad9e --- /dev/null +++ b/packages/daemon/src/networks/tcp-netstring.js @@ -0,0 +1,177 @@ +// @ts-check +import net from 'net'; + +import { E, Far } from '@endo/far'; +import { mapWriter, mapReader } from '@endo/stream'; +import { makeNetstringReader, makeNetstringWriter } from '@endo/netstring'; +import { + makeMessageCapTP, + bytesToMessage, + messageToBytes, +} from '../connection.js'; +import { makeSocketPowers } from '../daemon-node-powers.js'; + +const protocol = 'tcp+netstring+json+captp0'; + +export const make = async (powers, context) => { + const { servePort, connectPort } = makeSocketPowers({ net }); + + const cancelled = E(context).whenCancelled(); + const cancel = error => E(context).cancel(error); + + /** @type {Array} */ + const addresses = []; + + const gateway = E(powers).gateway(); + + // TODO + // const port = await E(powers).request('port to listen for public web socket connections', 'port', { + // default: '8080', + // }); + + const connectionNumbers = (function* generateNumbers() { + let n = 0; + for (;;) { + yield n; + n += 1; + } + })(); + + /** @type {Set>} */ + const connectionClosedPromises = new Set(); + + const started = (async () => { + // TODO make responding to this less of a pain. + // defaults, type hints, anything and everything. + // TODO more validation + const hostPort = await E(powers).request( + 'SELF', + 'Please select a host:port like 127.0.0.1:8920', + 'tcp-netstring-json-captp0-host-port', + ); + const { hostname: host, port: portname } = new URL( + `protocol://${hostPort}`, + ); + const port = Number(portname); + const { port: assignedPort, connections } = await servePort({ + port, + host, + cancelled, + }); + + // TODO log assigned port + console.log(`Endo daemon started local ${protocol} network device`); + addresses.push(`${protocol}://${host}:${assignedPort}`); + + return connections; + })(); + + const stopped = (async () => { + const connections = await started; + for await (const { + reader: bytesReader, + writer: bytesWriter, + closed: connectionClosed, + } of connections) { + (async () => { + const { value: connectionNumber } = connectionNumbers.next(); + + // TODO listen and connect addresses should be logged + console.log( + `Endo daemon accepted connection ${connectionNumber} over ${protocol} at ${new Date().toISOString()}`, + ); + + const messageWriter = mapWriter( + makeNetstringWriter(bytesWriter, { chunked: true }), + messageToBytes, + ); + const messageReader = mapReader( + makeNetstringReader(bytesReader), + bytesToMessage, + ); + + const { closed: capTpClosed } = makeMessageCapTP( + 'Endo', + messageWriter, + messageReader, + cancelled, + gateway, + ); + + const closed = Promise.race([connectionClosed, capTpClosed]); + connectionClosedPromises.add(closed); + closed.finally(() => { + connectionClosedPromises.delete(closed); + console.log( + `Endo daemon closed connection ${connectionNumber} over ${protocol} at ${new Date().toISOString()}`, + ); + }); + })().catch(cancel); + } + + await Promise.all(Array.from(connectionClosedPromises)); + })(); + + E.sendOnly(context).addDisposalHook(() => stopped); + + const connect = async (address, connectionContext) => { + const { value: connectionNumber } = connectionNumbers.next(); + + const { port: portname, hostname: host } = new URL(address); + const port = Number(portname); + + const connectionCancelled = E(connectionContext).whenCancelled(); + + const { + reader: bytesReader, + writer: bytesWriter, + closed: connectionClosed, + } = await connectPort({ + port, + host, + cancelled: connectionCancelled, + }); + + // TODO listen and connect addresses should be logged + console.log( + `Endo daemon connected ${connectionNumber} over ${protocol} at ${new Date().toISOString()}`, + ); + + const messageWriter = mapWriter( + makeNetstringWriter(bytesWriter, { chunked: true }), + messageToBytes, + ); + const messageReader = mapReader( + makeNetstringReader(bytesReader), + bytesToMessage, + ); + + const { closed: capTpClosed, getBootstrap } = makeMessageCapTP( + 'Endo', + messageWriter, + messageReader, + cancelled, + gateway, + ); + + const closed = Promise.race([connectionClosed, capTpClosed]); + connectionClosedPromises.add(closed); + closed.finally(() => { + E(context).cancel(); + connectionClosedPromises.delete(closed); + console.log( + `Endo daemon closed connection ${connectionNumber} over ${protocol}at ${new Date().toISOString()}`, + ); + }); + + return getBootstrap(); + }; + + await started; + + return Far('TcpNetstringService', { + addresses: () => harden(addresses), + supports: address => new URL(address).protocol === `${protocol}:`, + connect, + }); +}; diff --git a/packages/daemon/src/pet-store.js b/packages/daemon/src/pet-store.js index 5b462051ef..8bfcf31fef 100644 --- a/packages/daemon/src/pet-store.js +++ b/packages/daemon/src/pet-store.js @@ -7,7 +7,22 @@ const { quote: q } = assert; const validIdPattern = /^[0-9a-f]{128}$/; const validFormulaPattern = - /^(?:(?:readable-blob|worker|pet-store|pet-inspector|eval|lookup|make-unconfined|make-bundle|host|guest|handle):[0-9a-f]{128}|web-bundle:[0-9a-f]{32})$/; + /^(?:(?:readable-blob|worker|pet-store|pet-inspector|eval|lookup|make-unconfined|make-bundle|host|guest|handle|peer|loopback-network):[0-9a-f]{128}:[0-9a-f]{128}|web-bundle:[0-9a-f]{32}:[0-9a-f]{128})$/; + +/** + * @param {string} formulaIdentifier + * @param {string} [petName] + * @returns {void} + */ +export const assertValidFormulaIdentifier = (formulaIdentifier, petName) => { + if (!validFormulaPattern.test(formulaIdentifier)) { + let message = `Invalid formula identifier ${q(formulaIdentifier)}`; + if (petName !== undefined) { + message += ` for pet name ${q(petName)}`; + } + throw new Error(message); + } +}; /** * @param {import('./types.js').FilePowers} filePowers @@ -32,13 +47,7 @@ export const makePetStoreMaker = (filePowers, locator) => { const petNamePath = filePowers.joinPath(petNameDirectoryPath, petName); const petNameText = await filePowers.readFileText(petNamePath); const formulaIdentifier = petNameText.trim(); - if (!validFormulaPattern.test(formulaIdentifier)) { - throw new Error( - `Invalid formula identifier ${q(formulaIdentifier)} for pet name ${q( - petName, - )}`, - ); - } + assertValidFormulaIdentifier(formulaIdentifier, petName); return formulaIdentifier; }; @@ -74,9 +83,7 @@ export const makePetStoreMaker = (filePowers, locator) => { /** @type {import('./types.js').PetStore['write']} */ const write = async (petName, formulaIdentifier) => { assertValidName(petName); - if (!validFormulaPattern.test(formulaIdentifier)) { - throw new Error(`Invalid formula identifier ${q(formulaIdentifier)}`); - } + assertValidFormulaIdentifier(formulaIdentifier, petName); if (petNames.has(petName)) { // Perform cleanup on the overwritten pet name. @@ -144,9 +151,7 @@ export const makePetStoreMaker = (filePowers, locator) => { `Formula does not exist for pet name ${JSON.stringify(petName)}`, ); } - if (!validFormulaPattern.test(formulaIdentifier)) { - throw new Error(`Invalid formula identifier ${q(formulaIdentifier)}`); - } + assertValidFormulaIdentifier(formulaIdentifier, petName); const petNamePath = filePowers.joinPath(petNameDirectoryPath, petName); await filePowers.removePath(petNamePath); @@ -174,16 +179,9 @@ export const makePetStoreMaker = (filePowers, locator) => { `Formula does not exist for pet name ${JSON.stringify(fromName)}`, ); } - if (!validFormulaPattern.test(formulaIdentifier)) { - throw new Error(`Invalid formula identifier ${q(formulaIdentifier)}`); - } - if ( - overwrittenFormulaIdentifier !== undefined && - !validFormulaPattern.test(overwrittenFormulaIdentifier) - ) { - throw new Error( - `Invalid formula identifier ${q(overwrittenFormulaIdentifier)}`, - ); + assertValidFormulaIdentifier(formulaIdentifier, fromName); + if (overwrittenFormulaIdentifier !== undefined) { + assertValidFormulaIdentifier(overwrittenFormulaIdentifier, toName); } const fromPath = filePowers.joinPath(petNameDirectoryPath, fromName); @@ -220,9 +218,7 @@ export const makePetStoreMaker = (filePowers, locator) => { /** @type {import('./types.js').PetStore['reverseIdentify']} */ const reverseIdentify = formulaIdentifier => { - if (!validFormulaPattern.test(formulaIdentifier)) { - throw new Error(`Invalid formula identifier ${q(formulaIdentifier)}`); - } + assertValidFormulaIdentifier(formulaIdentifier); const formulaPetNames = formulaIdentifiers.get(formulaIdentifier); if (formulaPetNames === undefined) { return harden([]); diff --git a/packages/daemon/src/types.d.ts b/packages/daemon/src/types.d.ts index 4f3c53f4aa..feeb13595b 100644 --- a/packages/daemon/src/types.d.ts +++ b/packages/daemon/src/types.d.ts @@ -55,15 +55,22 @@ export type MignonicPowers = { type FormulaIdentifierRecord = { type: string; number: string; + node: string; }; type EndoFormula = { type: 'endo'; + networks: string; + peers: string; host: string; leastAuthority: string; webPageJs?: string; }; +type LoopbackNetworkFormula = { + type: 'loopback-network'; +}; + type WorkerFormula = { type: 'worker'; }; @@ -74,6 +81,7 @@ type HostFormula = { inspector: string; petStore: string; endo: string; + networks: string; leastAuthority: string; }; @@ -100,7 +108,7 @@ type EvalFormula = { export type EvalFormulaHook = ( identifiers: Readonly<{ endowmentFormulaIdentifiers: string[]; - evalFormulaNumber: string; + evalFormulaIdentifier: string; workerFormulaIdentifier: string; }>, ) => Promise; @@ -141,6 +149,12 @@ type MakeBundleFormula = { // TODO formula slots }; +type PeerFormula = { + type: 'peer'; + networks: string; + addresses: Array; +}; + type WebBundleFormula = { type: 'web-bundle'; bundle: string; @@ -168,6 +182,7 @@ type DirectoryFormula = { export type Formula = | EndoFormula + | LoopbackNetworkFormula | WorkerFormula | HostFormula | GuestFormula @@ -181,7 +196,8 @@ export type Formula = | HandleFormula | PetInspectorFormula | PetStoreFormula - | DirectoryFormula; + | DirectoryFormula + | PeerFormula; export type Label = { number: number; @@ -213,6 +229,11 @@ export type InternalPayload = InternalRequest | InternalPackage; export type Message = Label & Payload; export type InternalMessage = InternalLabel & InternalPayload; +export type Invitation = { + powers: string; + addresses: Array; +}; + export interface Topic< TRead, TWrite = undefined, @@ -282,9 +303,13 @@ export interface InternalExternal { internal: Internal; } -export interface Controller { +export interface ControllerPartial { external: Promise; internal: Promise; +} + +export interface Controller + extends ControllerPartial { context: Context; } @@ -331,6 +356,7 @@ export interface NameHub { has(...petNamePath: string[]): Promise; identify(...petNamePath: string[]): Promise; list(...petNamePath: string[]): Promise>; + listIdentifiers(...petNamePath: string[]): Promise>; followChanges( ...petNamePath: string[] ): AsyncGenerator; @@ -426,6 +452,27 @@ export type MakeHostOrGuestOptions = { introducedNames?: Record; }; +export interface EndoPeer { + provide: (formulaIdentifier: string) => Promise; +} +export type EndoPeerControllerPartial = ControllerPartial; +export type EndoPeerController = Controller; + +export interface EndoGateway { + provide: (formulaIdentifier: string) => Promise; +} + +export interface PeerInfo { + node: string; + addresses: string[]; +} + +export interface EndoNetwork { + supports: (network: string) => boolean; + addresses: () => Array; + connect: (address: string, farContext: FarContext) => EndoGateway; +} + export interface EndoGuest extends EndoDirectory { listMessages: Mail['listMessages']; followMessages: Mail['followMessages']; @@ -436,6 +483,7 @@ export interface EndoGuest extends EndoDirectory { request: Mail['request']; send: Mail['send']; } +export type FarEndoGuest = FarRef; export interface EndoHost extends EndoDirectory { listMessages: Mail['listMessages']; @@ -486,6 +534,9 @@ export interface EndoHost extends EndoDirectory { powersName: string, ): Promise; cancel(petName: string, reason: Error): Promise; + gateway(): Promise; + getPeerInfo(): Promise; + addPeerInfo(peerInfo: PeerInfo): Promise; } export interface InternalEndoHost { @@ -525,6 +576,9 @@ export type FarEndoBootstrap = FarRef<{ leastAuthority: () => Promise; webPageJs: () => Promise; importAndEndowInWebPage: () => Promise; + gateway: () => Promise; + reviveNetworks: () => Promise; + addPeerInfo: (peerInfo: PeerInfo) => Promise; }>; export type CryptoPowers = { @@ -653,6 +707,7 @@ export type DaemonicPowers = { type IncarnateResult = Promise<{ formulaIdentifier: string; value: T }>; export interface DaemonCore { + nodeIdentifier: string; provideValueForFormulaIdentifier: ( formulaIdentifier: string, ) => Promise; @@ -668,17 +723,23 @@ export interface DaemonCore { formula: Formula, ) => Promise<{ formulaIdentifier: string; value: unknown }>; getFormulaIdentifierForRef: (ref: unknown) => string | undefined; + getAllNetworkAddresses: ( + networksDirectoryFormulaIdentifier: string, + ) => Promise; incarnateEndoBootstrap: ( specifiedFormulaNumber: string, ) => IncarnateResult; incarnateWorker: () => IncarnateResult; - incarnatePetStore: () => IncarnateResult; + incarnatePetStore: ( + specifiedFormulaNumber?: string, + ) => IncarnateResult; incarnateDirectory: () => IncarnateResult; incarnatePetInspector: ( petStoreFormulaIdentifier: string, ) => IncarnateResult; incarnateHost: ( endoFormulaIdentifier: string, + networksDirectoryFormulaIdentifier: string, leastAuthorityFormulaIdentifier: string, specifiedWorkerFormulaIdentifier?: string | undefined, ) => IncarnateResult; @@ -717,6 +778,12 @@ export interface DaemonCore { incarnateHandle: ( targetFormulaIdentifier: string, ) => IncarnateResult; + incarnatePeer: ( + networksFormulaIdentifier: string, + addresses: Array, + ) => IncarnateResult; + incarnateNetworksDirectory: () => IncarnateResult; + incarnateLoopbackNetwork: () => IncarnateResult; incarnateLeastAuthority: () => IncarnateResult; cancelValue: (formulaIdentifier: string, reason: Error) => Promise; storeReaderRef: ( diff --git a/packages/daemon/test/test-endo.js b/packages/daemon/test/test-endo.js index af8a9ca2e6..05e00f7f2f 100644 --- a/packages/daemon/test/test-endo.js +++ b/packages/daemon/test/test-endo.js @@ -9,6 +9,7 @@ import '@endo/lockdown/commit-debug.js'; import test from 'ava'; import url from 'url'; import path from 'path'; +import crypto from 'crypto'; import { E } from '@endo/far'; import { makePromiseKit } from '@endo/promise-kit'; import bundleSource from '@endo/bundle-source'; @@ -21,6 +22,10 @@ import { makeEndoClient, makeReaderRef, } from '../index.js'; +import { makeCryptoPowers } from '../src/daemon-node-powers.js'; +import { serializeFormulaIdentifier } from '../src/formula-identifier.js'; + +const cryptoPowers = makeCryptoPowers(crypto); const { raw } = String; @@ -1215,3 +1220,135 @@ test('guest cannot access host methods', async t => { const revealedTarget = await E.get(guestsHost).targetFormulaIdentifier; t.is(revealedTarget, undefined); }); + +test('read unknown nodeId', async t => { + const { promise: cancelled, reject: cancel } = makePromiseKit(); + t.teardown(() => cancel(Error('teardown'))); + const locator = makeLocator('tmp', 'read unknown nodeId'); + + await stop(locator).catch(() => {}); + await purge(locator); + await start(locator); + + const { getBootstrap } = await makeEndoClient( + 'client', + locator.sockPath, + cancelled, + ); + const bootstrap = getBootstrap(); + const host = E(bootstrap).host(); + + // write a bogus value for a bogus nodeId + const node = await cryptoPowers.randomHex512(); + const number = await cryptoPowers.randomHex512(); + const type = 'eval'; + const formulaIdentifier = serializeFormulaIdentifier({ + node, + number, + type, + }); + await E(host).write(['abc'], formulaIdentifier); + // observe reification failure + t.throwsAsync(() => E(host).lookup('abc'), { + message: /No peer found for node identifier /u, + }); + + await stop(locator); +}); + +test('read remote value', async t => { + const { promise: cancelled, reject: cancel } = makePromiseKit(); + t.teardown(() => cancel(Error('teardown'))); + const locatorA = makeLocator('tmp', 'read remote value A'); + const locatorB = makeLocator('tmp', 'read remote value B'); + let hostA; + { + await stop(locatorA).catch(() => {}); + await purge(locatorA); + await start(locatorA); + const { getBootstrap } = await makeEndoClient( + 'client', + locatorA.sockPath, + cancelled, + ); + const bootstrap = getBootstrap(); + hostA = E(bootstrap).host(); + // Install test network + const servicePath = path.join( + dirname, + 'src', + 'networks', + 'tcp-netstring.js', + ); + const serviceLocation = url.pathToFileURL(servicePath).href; + const networkA = E(hostA).makeUnconfined( + 'MAIN', + serviceLocation, + 'SELF', + 'test-network', + ); + // set address via request + const iteratorRef = E(hostA).followMessages(); + const { value: message } = await E(iteratorRef).next(); + const { number } = E.get(message); + await E(hostA).evaluate('MAIN', '`127.0.0.1:0`', [], [], 'netport'); + await E(hostA).resolve(await number, 'netport'); + // move test network to network dir + await networkA; + await E(hostA).move(['test-network'], ['NETS', 'tcp']); + } + + let hostB; + { + await stop(locatorB).catch(() => {}); + await purge(locatorB); + await start(locatorB); + const { getBootstrap } = await makeEndoClient( + 'client', + locatorB.sockPath, + cancelled, + ); + const bootstrap = getBootstrap(); + hostB = E(bootstrap).host(); + // Install test network + const servicePath = path.join( + dirname, + 'src', + 'networks', + 'tcp-netstring.js', + ); + const serviceLocation = url.pathToFileURL(servicePath).href; + const networkB = E(hostB).makeUnconfined( + 'MAIN', + serviceLocation, + 'SELF', + 'test-network', + ); + // set address via requestcd + const iteratorRef = E(hostB).followMessages(); + const { value: message } = await E(iteratorRef).next(); + const { number } = E.get(message); + await E(hostB).evaluate('MAIN', '`127.0.0.1:0`', [], [], 'netport'); + await E(hostB).resolve(await number, 'netport'); + // move test network to network dir + await networkB; + await E(hostB).move(['test-network'], ['NETS', 'tcp']); + } + + // introduce nodes to each other + await E(hostA).addPeerInfo(await E(hostB).getPeerInfo()); + await E(hostB).addPeerInfo(await E(hostA).getPeerInfo()); + + // create value to share + await E(hostB).evaluate('MAIN', '`haay wuurl`', [], [], 'salutations'); + const hostBValueIdentifier = await E(hostB).identify('salutations'); + + // insert in hostA out of band + await E(hostA).write(['greetings'], hostBValueIdentifier); + const hostAValue = await E(hostA).lookup('greetings'); + + t.is(hostAValue, 'haay wuurl'); + + await stop(locatorA); + await stop(locatorB); +});