Skip to content

Commit

Permalink
feat: use blockstore sessions (#50)
Browse files Browse the repository at this point in the history 
* feat: use blockstore sessions

Adds a configurable session cache that creates sessions based on
the base URL of the requested resource.

E.g. `https://Qmfoo.ipfs.gateway.com/foo.txt` and`https://Qmfoo.ipfs.gateway.com/bar.txt`
will be loaded from the same session.

Defaults to 100 sessions maximum with a TTL of one minute. These
are arbitrary numbers that will require some tweaking.

* chore: update helia version

* chore: change cache key format

* chore: linting

* chore: remove pre-releases

* chore: update versions

* test: remove cleanup fn call

---------

Co-authored-by: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
  • Loading branch information
@achingbrain @SgtPooki
achingbrain and SgtPooki committed May 9, 2024
1 parent f7ac2e7 commit 541dd64
Show file tree
Hide file tree
Showing 15 changed files with 409 additions and 90 deletions.
14 changes: 2 additions & 12 deletions packages/interop/src/fixtures/load-fixtures.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,8 @@ import { path as kuboPath } from 'kubo'
*/
export async function loadFixtures (IPFS_PATH = undefined): Promise<void> {
const kuboBinary = process.env.KUBO_BINARY ?? kuboPath()
/**
* fast-glob does not like windows paths, see https://github.com/mrmlnc/fast-glob/issues/237
* fast-glob performs search from process.cwd() by default, which will be:
* 1. the root of the monorepo when running tests in CI
* 2. the package root when running tests in the package directory
*/
let globRoot = process.cwd().replace(/\\/g, '/')
if (!globRoot.includes('packages/interop')) {
// we only want car files from the interop package
globRoot = [...globRoot.split('/'), 'packages/interop'].join('/')
}
for (const carFile of await fg.glob('src/fixtures/data/*.car', { cwd: globRoot })) {

for (const carFile of await fg.glob('**/fixtures/data/*.car')) {
await $({ env: { IPFS_PATH } })`${kuboBinary} dag import --pin-roots=false --offline ${carFile}`
}
}
9 changes: 7 additions & 2 deletions packages/interop/src/json.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@ describe('@helia/verified-fetch - json', () => {
// child2: QmWNBJX6fZyNTLWNYBHxAHpBctCP43R2zeqV2G8uavqFZn // partial JSON
verifiedFetch = await createVerifiedFetch({
gateways: ['http://127.0.0.1:8180'],
routers: []
routers: ['http://127.0.0.1:8180'],
allowInsecure: true,
allowLocal: true
})
})

Expand All @@ -23,7 +25,10 @@ describe('@helia/verified-fetch - json', () => {
})

it('handles UnixFS-chunked JSON file', async () => {
const resp = await verifiedFetch(CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'))
const resp = await verifiedFetch(CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), {
allowLocal: true,
allowInsecure: true
})
expect(resp).to.be.ok()
const jsonObj = await resp.json()
expect(jsonObj).to.be.ok()
Expand Down
40 changes: 31 additions & 9 deletions packages/interop/src/unixfs-dir.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@ describe('@helia/verified-fetch - unixfs directory', () => {
before(async () => {
verifiedFetch = await createVerifiedFetch({
gateways: ['http://127.0.0.1:8180'],
routers: []
routers: [],
allowInsecure: true,
allowLocal: true
})
verifiedFetch = await createVerifiedFetch()
})

after(async () => {
Expand All @@ -26,7 +27,11 @@ describe('@helia/verified-fetch - unixfs directory', () => {
'http://example.com/ipfs/bafybeifq2rzpqnqrsdupncmkmhs3ckxxjhuvdcbvydkgvch3ms24k5lo7q'
].forEach((url: string) => {
it(`request to unixfs directory with ${url} should return a 301 with a trailing slash`, async () => {
const response = await verifiedFetch(url, { redirect: 'manual' })
const response = await verifiedFetch(url, {
redirect: 'manual',
allowLocal: true,
allowInsecure: true
})
expect(response).to.be.ok()
expect(response.status).to.equal(301)
expect(response.headers.get('location')).to.equal(`${url}/`)
Expand All @@ -38,20 +43,29 @@ describe('@helia/verified-fetch - unixfs directory', () => {
describe('XKCD Barrel Part 1', () => {
it('fails to load when passed the root', async () => {
// The spec says we should generate HTML with directory listings, but we don't do that yet, so expect a failure
const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR')
const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR', {
allowLocal: true,
allowInsecure: true
})
expect(resp).to.be.ok()
expect(resp.status).to.equal(501) // TODO: we should do a directory listing instead
})

it('can return a string for unixfs pathed data', async () => {
const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1 - alt.txt')
const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1 - alt.txt', {
allowLocal: true,
allowInsecure: true
})
expect(resp).to.be.ok()
const text = await resp.text()
expect(text).to.equal('Don\'t we all.')
})

it('can return an image for unixfs pathed data', async () => {
const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1.png')
const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1.png', {
allowLocal: true,
allowInsecure: true
})
expect(resp).to.be.ok()
const imgData = await resp.blob()
expect(imgData).to.be.ok()
Expand All @@ -64,7 +78,9 @@ describe('@helia/verified-fetch - unixfs directory', () => {
await verifiedFetch.stop()
verifiedFetch = await createVerifiedFetch({
gateways: ['http://127.0.0.1:8180'],
routers: []
routers: ['http://127.0.0.1:8180'],
allowInsecure: true,
allowLocal: true
}, {
contentTypeParser: (bytes) => {
return filetypemime(bytes)?.[0]
Expand All @@ -73,7 +89,10 @@ describe('@helia/verified-fetch - unixfs directory', () => {
})

it('can return an image content-type for unixfs pathed data', async () => {
const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1.png')
const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1.png', {
allowLocal: true,
allowInsecure: true
})
// tediously this is actually a jpeg file with a .png extension
expect(resp.headers.get('content-type')).to.equal('image/jpeg')
})
Expand All @@ -82,7 +101,10 @@ describe('@helia/verified-fetch - unixfs directory', () => {
// from https://github.com/ipfs/gateway-conformance/blob/193833b91f2e9b17daf45c84afaeeae61d9d7c7e/fixtures/trustless_gateway_car/single-layer-hamt-with-multi-block-files.car
describe('HAMT-sharded directory', () => {
it('loads path /ipfs/bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i/685.txt', async () => {
const resp = await verifiedFetch('ipfs://bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i/685.txt')
const resp = await verifiedFetch('ipfs://bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i/685.txt', {
allowLocal: true,
allowInsecure: true
})
expect(resp).to.be.ok()
const text = await resp.text()
// npx kubo@0.25.0 cat '/ipfs/bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i/685.txt'
Expand Down
23 changes: 18 additions & 5 deletions packages/interop/src/websites.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ describe('@helia/verified-fetch - websites', () => {
// 2024-01-22 CID for _dnslink.helia-identify.on.fleek.co
verifiedFetch = await createVerifiedFetch({
gateways: ['http://127.0.0.1:8180'],
routers: []
routers: ['http://127.0.0.1:8180'],
allowInsecure: true,
allowLocal: true
})
})

Expand All @@ -19,15 +21,21 @@ describe('@helia/verified-fetch - websites', () => {
})

it('loads index.html when passed helia-identify.on.fleek.co root CID', async () => {
const resp = await verifiedFetch('ipfs://QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv')
const resp = await verifiedFetch('ipfs://QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv', {
allowLocal: true,
allowInsecure: true
})
expect(resp).to.be.ok()
const html = await resp.text()
expect(html).to.be.ok()
expect(html).to.include('<title>Run Identify on a remote node with Helia</title>')
})

it('loads helia-identify.on.fleek.co index.html directly ', async () => {
const resp = await verifiedFetch('ipfs://QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv/index.html')
const resp = await verifiedFetch('ipfs://QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv/index.html', {
allowLocal: true,
allowInsecure: true
})
expect(resp).to.be.ok()
const html = await resp.text()
expect(html).to.be.ok()
Expand All @@ -52,7 +60,9 @@ describe('@helia/verified-fetch - websites', () => {
before(async () => {
verifiedFetch = await createVerifiedFetch({
gateways: ['http://127.0.0.1:8180'],
routers: []
routers: ['http://127.0.0.1:8180'],
allowInsecure: true,
allowLocal: true
})
})

Expand All @@ -61,7 +71,10 @@ describe('@helia/verified-fetch - websites', () => {
})

it('loads index.html when passed fake-blog.libp2p.io root CID', async () => {
const resp = await verifiedFetch('ipfs://QmeiDMLtPUS3RT2xAcUwsNyZz169wPke2q7im9vZpVLSYw')
const resp = await verifiedFetch('ipfs://QmeiDMLtPUS3RT2xAcUwsNyZz169wPke2q7im9vZpVLSYw', {
allowLocal: true,
allowInsecure: true
})
expect(resp).to.be.ok()
const html = await resp.text()
expect(html).to.be.ok()
Expand Down
11 changes: 7 additions & 4 deletions packages/verified-fetch/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,17 +122,20 @@ You can see variations of Helia and js-libp2p configuration options at <https://
```typescript
import { trustlessGateway } from '@helia/block-brokers'
import { createHeliaHTTP } from '@helia/http'
import { delegatedHTTPRouting } from '@helia/routers'
import { delegatedHTTPRouting, httpGatewayRouting } from '@helia/routers'
import { createVerifiedFetch } from '@helia/verified-fetch'

const fetch = await createVerifiedFetch(
await createHeliaHTTP({
blockBrokers: [
trustlessGateway({
trustlessGateway()
],
routers: [
delegatedHTTPRouting('http://delegated-ipfs.dev'),
httpGatewayRouting({
gateways: ['https://mygateway.example.net', 'https://trustless-gateway.link']
})
],
routers: ['http://delegated-ipfs.dev'].map((routerUrl) => delegatedHTTPRouting(routerUrl))
]
})
)

Expand Down
26 changes: 13 additions & 13 deletions packages/verified-fetch/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,12 @@
"release": "aegir release"
},
"dependencies": {
"@helia/block-brokers": "^2.1.0",
"@helia/car": "^3.1.3",
"@helia/http": "^1.0.4",
"@helia/interface": "^4.2.0",
"@helia/ipns": "^7.2.1",
"@helia/routers": "^1.0.3",
"@helia/block-brokers": "^3.0.0",
"@helia/car": "^3.1.5",
"@helia/http": "^1.0.7",
"@helia/interface": "^4.3.0",
"@helia/ipns": "^7.2.2",
"@helia/routers": "^1.1.0",
"@ipld/dag-cbor": "^9.2.0",
"@ipld/dag-json": "^10.2.0",
"@ipld/dag-pb": "^4.1.0",
Expand All @@ -79,17 +79,17 @@
"it-pipe": "^3.0.1",
"it-tar": "^6.0.5",
"it-to-browser-readablestream": "^2.0.6",
"lru-cache": "^10.2.0",
"multiformats": "^13.1.0",
"progress-events": "^1.0.0",
"uint8arrays": "^5.0.3"
},
"devDependencies": {
"@helia/car": "^3.1.3",
"@helia/dag-cbor": "^3.0.3",
"@helia/dag-json": "^3.0.3",
"@helia/json": "^3.0.3",
"@helia/unixfs": "^3.0.4",
"@helia/utils": "^0.2.0",
"@helia/dag-cbor": "^3.0.4",
"@helia/dag-json": "^3.0.4",
"@helia/json": "^3.0.4",
"@helia/unixfs": "^3.0.6",
"@helia/utils": "^0.3.0",
"@ipld/car": "^5.3.0",
"@libp2p/interface-compliance-tests": "^5.3.4",
"@libp2p/logger": "^4.0.9",
Expand All @@ -100,7 +100,7 @@
"blockstore-core": "^4.4.1",
"browser-readablestream-to-it": "^2.0.5",
"datastore-core": "^9.2.9",
"helia": "^4.1.1",
"helia": "^4.2.1",
"ipfs-unixfs-importer": "^15.2.5",
"ipns": "^9.1.0",
"it-all": "^3.0.4",
Expand Down
Loading

0 comments on commit 541dd64

Please sign in to comment.