Delete and fixes (ipfs-shipyard#4)

* wip: prep for fleek publishing

* fix: no raw leaves

We want to reuse pinning provided by Fleek, but atm it uses CIDv0,
which does not support raw leaves.

By disabling raw leaves in CIDv1 add, we get the same underlying dag-pb,
and the same multihash inside of both CIDv1 used in tests and CIDv0
produced by Fleek.

* Add DELETE tests, add DAG tests, clean up README, fix empty dir test with fleek, disable Worker tests

Co-authored-by: Marcin Rataj <lidel@lidel.org>

README.md

@@ -30,9 +30,30 @@ Progress: https://github.com/ipfs/community/discussions/573
   - [x] Generate directory listing for folders
   - [x] Render empty directories
   - [x] Resolve `index.html` in a path
-  - [x] Resolve `NAME.html` for `/name/`
-  - [ ] JS `fetch('ipfs://', {method: 'POST'})`
-  - [ ] JS `fetch('ipns://', {method: 'POST'})`
+- `mutable.html` Tests (experimental, some things subject to change)
+  - [x] JS `fetch('ipfs://CID/example.txt', {method: 'POST'})`
+  - [x] JS `fetch('ipns://CID/', {method: 'POST'})`
+  - [x] JS `fetch('ipns://CID/example.txt', {method: 'POST'})`
+  - [x] JS `fetch('ipfs://CID/', {method: 'POST', body: new FormData})`
+  - [x] JS `fetch('ipfs://CID/example.txt', {method: 'DELETE'})`
+  - [x] JS `fetch('ipns://CID/example.txt', {method: 'DELETE'})`
+- `dag.html` Tests (experimental, some things subject to change)
+	- [x] GET `ipfs://CID/?format=CAR`
+	- [x] GET `ipfs://CID/?format=block`
+	- [x] GET `ipfs://CID/?format=dag-json`
+	- [x] GET `ipfs://CID/?format=dag-cbor`
+	- [x] POST `Content-Type: application/json` `?format=dag-cbor` & GET `?format=dag-cbor`
+	- [x] POST over existing CID to add to graph
+
+## Screenshots:
+
+Brave Browser (as of 2021-08-13)
+
+![Screenshot of Brave Browser with 34 tests passing, 7 failing, 1 timeouts, and 8 not run](screenshots/brave.png)
+
+Agregore Browser (as of 2021-08-13)
+
+![Screenshot of Agregore Browser with 45 tests passing, and 5 failing](screenshots/agregore.png)
 
 ## Publishing:
 

constants.js

@@ -1,7 +1,7 @@
 // Constants for test files, generated with publish-files.js
 
 // Raw hashes
-export const URL_IPFS_MEDIA = 'ipfs://bafybeievhgvy6xq64x32fufpkwkhly32au7pkdmczdgyypg435w6ftmet4/'
+export const URL_IPFS_MEDIA = 'ipfs://bafybeic7n7s52drjp67yjklbiyxyzh3ddw35qr4wcuxkra6qmu37czxrmi/'
 export const URL_IPFS_TEXT_FILE_RAW = 'ipfs://bafybeiduiecxoeiqs3gyc6r7v3lymmhserldnpw62qjnhmqsulqjxjmtzi/?filename=files/example.txt'
 export const URL_IPFS_IMAGE_FILE_RAW = 'ipfs://bafybeihc4hti5ix4ds2tefhy35qd4c7n5as5cazdmksrxj7ipvcxm64h54/?filename=files/ipfs-logo.svg'
 export const URL_IPFS_VIDEO_FILE_RAW = 'ipfs://bafybeifmi5ykqzw4xnyarcx4wltdqid3ivacz3dr5qobskrajprs53uvj4/?filename=files/IPFS.mp4'

dag-tests.js

@@ -0,0 +1,134 @@
+/* global
+promise_test,
+assert_true,
+assert_equals,
+fetch,
+*/
+
+import {
+  URL_IPFS_TEXT_FILE_RAW
+} from './constants.js'
+
+// TODO: These tests currently don't pass in any browsers.
+// Based on https://github.com/ipfs/go-ipfs/issues/8234
+// Work on this is pending on gateway support / API availability
+
+promise_test(async (t) => {
+  const response = await fetch(URL_IPFS_TEXT_FILE_RAW + '?format=car')
+
+  assert_true(response.ok, 'Able to get CID')
+
+  assert_true(response.headers.get('Content-Type').includes('application/octet-stream'), 'Got binary content type')
+
+  const content = await response.arrayBuffer()
+
+  assert_true(content.byteLength > 0, 'Content is non-empty')
+  // TODO: Check the content format
+}, 'GET CID as a CAR file')
+promise_test(async (t) => {
+  const response = await fetch(URL_IPFS_TEXT_FILE_RAW + '?format=block')
+
+  assert_true(response.ok, 'Able to get CID')
+
+  assert_true(response.headers.get('Content-Type').includes('application/octet-stream'), 'Got binary content type')
+
+  const content = await response.arrayBuffer()
+
+  assert_true(content.byteLength > 0, 'Content is non-empty')
+  // TODO: Check the content format
+}, 'GET CID as a block')
+promise_test(async (t) => {
+  const response = await fetch(URL_IPFS_TEXT_FILE_RAW + '?format=dag-json')
+
+  assert_true(response.ok, 'Able to get CID')
+
+  assert_true(response.headers.get('Content-Type').includes('application/json'), 'Got binary content type')
+
+  const content = await response.json()
+
+  assert_true(content && Object.keys(content).length !== 0, 'Content is non-empty')
+  // TODO: Check the content format
+}, 'GET CID as a dag-json file')
+promise_test(async (t) => {
+  const response = await fetch(URL_IPFS_TEXT_FILE_RAW + '?format=dag-cbor')
+
+  assert_true(response.ok, 'Able to get CID')
+
+  assert_true(response.headers.get('Content-Type').includes('application/cbor'), 'Got binary content type')
+
+  const content = await response.arrayBuffer()
+
+  assert_true(content.byteLength > 0, 'Content is non-empty')
+  // TODO: Check the content format
+}, 'GET CID as a dag-cbor file')
+
+promise_test(async (t) => {
+  // Data will be saved as dag-cbor
+  const postResponse = await fetch('ipfs://?format=dag-cbor', {
+    method: 'POST',
+    headers: {
+      // Data is represented as JSON before it's converted
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({
+      hello: 'World'
+    })
+  })
+
+  assert_true(postResponse.ok, 'Able to POST JSON')
+
+  const url = await postResponse.text()
+
+  assert_true(url.startsWith('ipfs://'), 'Returned IPFS URL')
+
+  const getResponse = await fetch(url + '?format=dag-cbor')
+
+  assert_true(getResponse.ok, 'Able to get CID')
+
+  assert_true(getResponse.headers.get('Content-Type').includes('application/cbor'), 'Got binary content type')
+
+  const content = await getResponse.arrayBuffer()
+
+  assert_true(content.byteLength > 0, 'Content is non-empty')
+}, 'POST JSON, get CBOR from CID')
+promise_test(async (t) => {
+  const postResponse1 = await fetch('ipfs://?format=dag-cbor', {
+    method: 'POST',
+    headers: {
+      // Data is represented as JSON before it's converted
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({
+      hello: 'World'
+    })
+  })
+
+  assert_true(postResponse1.ok, 'Able to POST JSON')
+
+  const url1 = await postResponse1.text()
+
+  const postResponse2 = await fetch(url1 + '/newField?format=dag-cbor', {
+    method: 'POST',
+    headers: {
+      // Data is represented as JSON before it's converted
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({
+      goodbye: 'World'
+    })
+  })
+
+  assert_true(postResponse2.ok, 'Able to POST over CID')
+
+  const url2 = await postResponse2.text()
+
+  assert_true(url2.startsWith('ipfs://'), 'Returned IPFS URL')
+
+  const getResponse = await fetch(url2 + '?format=dag-json')
+
+  assert_true(getResponse.ok, 'Able to GET new CID')
+
+  const content = await getResponse.json()
+
+  assert_equals(content?.newKey?.goodbye, 'world', 'Subkey got added to content')
+}, 'POST JSON (as cbor), POST on top of CID, get JSON')

dag.html

@@ -0,0 +1,7 @@
+<!DOCTYPE html>
+<title>IPFS Protocol Compliance Suite - DAG</title>
+
+<script src="testharness/testharness.js"></script>
+<script src="testharness/testharnessreport.js"></script>
+
+<script type="module" src="dag-tests.js"></script>

mutable-tests.js

@@ -9,6 +9,7 @@ const TEST_FILE_CONTENT = 'Hello World'
 const TEST_FILE_NAME = 'example.txt'
 const OTHER_TEST_FILE_CONTENT = 'Goodby World'
 const OTHER_TEST_FILE_NAME = 'example2.txt'
+const EMPTY_DIR_URL = 'ipfs://bafybeiczsscdsbs7ffqz55asqdf3smv6klcw3gofszvwlyarci47bgf354/'
 
 promise_test(async (t) => {
   const postResponse = await fetch(`ipfs://${TEST_FILE_NAME}`, {
@@ -86,7 +87,56 @@ promise_test(async (t) => {
 
   const base = new URL('./', url).href
 
-  const ipnsResponse = await fetch('ipns://compliance-suite-example1/', {
+  const secondFileResponse = await fetch(new URL(`./${OTHER_TEST_FILE_NAME}`, base).href, {
+    method: 'POST',
+    body: OTHER_TEST_FILE_CONTENT
+  })
+
+  assert_true(secondFileResponse.ok, 'Able to post second file')
+
+  const secondUrl = await secondFileResponse.text()
+
+  const secondBase = new URL('./', secondUrl).href
+
+  const listingRequest = await fetch(secondBase)
+
+  const listing = await listingRequest.text()
+
+  assert_true(listing.includes(TEST_FILE_NAME), 'First file shows up in parent folder')
+  assert_true(listing.includes(OTHER_TEST_FILE_NAME), 'Second file shows up in parent folder')
+
+  const deleteResponse = await fetch(secondUrl, {
+    method: 'DELETE'
+  })
+
+  assert_true(deleteResponse.ok, 'Able to DELETE file URL')
+
+  const finalUrl = await deleteResponse.text()
+
+  assert_true(finalUrl.startsWith('ipfs://'), 'Got an IPFS url for the content')
+  assert_true(finalUrl.endsWith('/'), 'URL ends with a / to signify a directory')
+
+  const finalListingRequest = await fetch(finalUrl)
+
+  const finalListing = await finalListingRequest.text()
+
+  assert_true(finalListing.includes(TEST_FILE_NAME), 'First file still shows up in folder')
+  assert_true(!finalListing.includes(OTHER_TEST_FILE_NAME), 'Second file no longer shows up in folder')
+}, 'DELETE file from an infohash')
+
+promise_test(async (t) => {
+  const firstFileResponse = await fetch(`ipfs://${TEST_FILE_NAME}`, {
+    method: 'POST',
+    body: TEST_FILE_CONTENT
+  })
+
+  assert_true(firstFileResponse.ok, 'Able to post first file')
+
+  const url = await firstFileResponse.text()
+
+  const base = new URL('./', url).href
+
+  const ipnsResponse = await fetch('ipns://compliance-suite-example/', {
     method: 'POST',
     body: base
   })
@@ -113,7 +163,7 @@ promise_test(async (t) => {
   formData.append('file', new Blob([TEST_FILE_CONTENT]), TEST_FILE_NAME)
   formData.append('file', new Blob([OTHER_TEST_FILE_CONTENT]), OTHER_TEST_FILE_NAME)
 
-  const postResponse = await fetch('ipfs:///', {
+  const postResponse = await fetch(EMPTY_DIR_URL, {
     method: 'POST',
     body: formData
   })
@@ -153,6 +203,8 @@ promise_test(async (t) => {
 
   const base = new URL('./', url).href
 
+  // Note that the key pet name is a temporary measure
+  // This should be replaced with public keys once a key generation standard is figured out
   const ipnsResponse = await fetch('ipns://compliance-suite-example2/', {
     method: 'POST',
     body: base
@@ -162,7 +214,9 @@ promise_test(async (t) => {
 
   const ipnsUrl = await ipnsResponse.text()
 
-  const secondFileResponse = await fetch(`ipns://compliance-suite-example2/${OTHER_TEST_FILE_NAME}`, {
+  const otherFileURL = new URL(`/${OTHER_TEST_FILE_NAME}`, ipnsUrl)
+
+  const secondFileResponse = await fetch(otherFileURL, {
     method: 'POST',
     body: OTHER_TEST_FILE_CONTENT
   })
@@ -176,5 +230,5 @@ promise_test(async (t) => {
   const listing = await listingRequest.text()
 
   assert_true(listing.includes(TEST_FILE_NAME), 'First file shows up in IPNS folder')
-  assert_true(listing.includes(OTHER_TEST_FILE_NAME), 'Secondt file shows up in IPNS folder')
+  assert_true(listing.includes(OTHER_TEST_FILE_NAME), 'Second file shows up in IPNS folder')
 }, 'POST text file to IPNS with existing data')

publish-files.js

@@ -56,6 +56,12 @@ run().catch((e) => {
 })
 
 async function run () {
+  // This is necessary because git cannot create empty directories
+  console.log('Ensure empty dir exists')
+  const emptyDir = require('path').join(__dirname, 'files', 'empty')
+
+  await fs.mkdir(emptyDir, { recursive: true })
+
   console.log('Uploading to IPFS')
   const { stdout: output } = await exec('ipfs add ./files/ --cid-version=1 --raw-leaves=false -r')
 

tests.js

@@ -5,8 +5,8 @@ assert_equals,
 assert_throws_dom,
 fetch,
 XMLHttpRequest,
-document,
-Worker */
+document
+*/
 
 import * as CONSTANTS from './constants.js'
 
@@ -219,6 +219,10 @@ promise_test(async (t) => {
   assert_true(true, 'Loaded content')
 }, 'IPFS Script `import()`')
 
+/*
+// These tests are currently turned off due to CORS errors.
+// We need a way to specify CORS headers reliabily,
+// or run the tests only on the current IPFS domain.
 for (const urlKey of makePermutations('JS_FILE')) {
   promise_test(async (t) => {
     const worker = new Worker(CONSTANTS[urlKey])
@@ -233,6 +237,7 @@ for (const urlKey of makePermutations('JS_FILE')) {
     assert_true(message.example, 'Got example message from worker')
   }, `IPFS Script 'Worker()' - ${urlKey}`)
 }
+*/
 
 for (const urlKey of makePermutations('JS_FILE')) {
   promise_test(async (t) => {