feat: remote pinning settings page (#1713)

* chore: WIP in settings page
* feat: add remote pinning services to the settings page
* chore: memoise pinning service color
* chore: set the same colors for pinning services manager items
* chore: update src/components/pinning-manager/PinningManager.js
* chore: remove console.log
* chore: make pinning work in go-ipfs 0.7 & add docs link
* chore: move pinning constants to another file
* feat: detect if remote services are available in the settings page
* chore: switch to production Pinata
https://pinata.cloud/documentation#PinningServicesAPI

* feat: remote pins on files page (#1721)
* feat: upload progress ui (#1655)

This PR adds visual feedback for when big files are imported:
progress bar + % status.

There is also new http-client and a bunch of required fixes.

* fix: broken file list stories
* feat: upload progress ui
* chore: integrate external changes
* fix: integrate new http client
* fix: jsdom problem
* fix(cid) switch circleci to new images
* fix(e2e): test/e2e/remote-api.test.js
  This makes E2E tests for remote API less flaky:
  - leverage expect-puppeteer where possible
  - tweak navigation so it is not impacted by connection-error state
  - force refresh of status page to avoid wait for manual refresh
  * fix(ci): E2E_IPFSD_TYPE=js npm run test:e2e
  js-ipfs changed CLI path at some point recently

Co-authored-by: Marcin Rataj <lidel@lidel.org>

* chore: rollback connect-deps

* chore: go-ipfs 0.8.0-rc1

* feat: integrate remote pinning in the files page

* chore: update deps

* chore: linting fixes

* feat: add pinning services to the pinning modal

* chore: update conflicts

Co-authored-by: Irakli Gozalishvili <contact@gozala.io>
Co-authored-by: Marcin Rataj <lidel@lidel.org>

* chore: disable js-ipfs tests for the time being

* feat: add autoUpload toggle to the pinning manager

* chore: change autoupload labels

* refactor: simplify size calculations

Using pre-existing stat helper makes webui immune to unexpected
exceptions.

Pin size calculation was removed, because it was already hidden in GUI
of Settings screen.

* fix: pin.remote.ls status check

This fixes a racy bug: when a single service was offline,
any remaining services were not checked due to error thrown.

This usually did not happen in Chromium, but failed pretty often in
Firefox, which seem to execute looped for awaits bit differently.

* fix: meaningful labels and variable names

* fix: auto upload labels

- basic explainer whatpolicy change means
- enable/disable
- manual/all files

* fix: disable pinning to offline services

- filled missing labels with basic explainer
  (can be improved in separate PRs)
- disabled pinning to services that are offline + added visual
  indication when there is a problem with service

* fix: pinning service templates

- sets icon and docs url when name includes template name

* style: auto upload modal title

Co-authored-by: Jessica Schilling <jessica@protocol.ai>

* style: auto upload modal description

Co-authored-by: Jessica Schilling <jessica@protocol.ai>

* style: auto upload menu item

Co-authored-by: Jessica Schilling <jessica@protocol.ai>

* fix: no online check for local pinning

* style: separate title for remote pin glyph

* chore: cleanup

Co-authored-by: Marcin Rataj <lidel@lidel.org>
Co-authored-by: Irakli Gozalishvili <contact@gozala.io>
Co-authored-by: Jessica Schilling <jessica@protocol.ai>

.circleci/config.yml

@@ -19,8 +19,8 @@ jobs:
           command: npm run build
       - run:
           command: E2E_IPFSD_TYPE=go npm run test:e2e
-      - run:
-          command: E2E_IPFSD_TYPE=js npm run test:e2e
+      # - run:      - TODO: uncomment after pinning services are coded in js-ipfs 
+          # command: E2E_IPFSD_TYPE=js npm run test:e2e
       - run:
           command: npm run bundlesize
       - persist_to_workspace:

.gitignore

@@ -23,3 +23,4 @@ yarn-error.log*
 .vscode
 .idea
 .eslintcache
+tsconfig.tsbuildinfo

package.json

@@ -6,7 +6,7 @@
     "start": "run-script-os",
     "start:win32": "@powershell -Command $env:REACT_APP_GIT_REV=(git rev-parse --short HEAD); react-scripts start",
     "start:darwin:linux": "cross-env REACT_APP_GIT_REV=`git rev-parse --short HEAD` react-scripts start",
-    "lint": "eslint src/ test/",
+    "lint": "eslint src/ test/ && tsc --noEmit",
     "prebuild": "lol public/locales > src/lib/languages.json",
     "build": "run-script-os",
     "build:win32": "@powershell -Command $env:REACT_APP_GIT_REV=(git rev-parse --short HEAD); react-scripts build",
@@ -127,13 +127,13 @@
     "go-ipfs": "0.8.0",
     "http-proxy": "^1.18.1",
     "http-server": "^0.12.3",
-    "ipfs": "^0.54.2",
+    "ipfs": "^0.54.4",
     "ipfsd-ctl": "^7.2.0",
     "is-pull-stream": "0.0.0",
     "jest-puppeteer": "^4.4.0",
     "multihashing-async": "^1.0.0",
     "npm-run-all": "^4.1.5",
-    "puppeteer": "^7.1.0",
+    "puppeteer": "^8.0.0",
     "run-script-os": "^1.1.1",
     "shx": "^0.3.2",
     "typescript": "^4.1.3",

public/locales/en/files.json

@@ -100,6 +100,7 @@
   "hashUnavailable": "hash unavailable",
   "checkboxLabel": "View more options for {name}",
   "pinned": "Pinned",
+  "pinnedRemotely": "Pinned remotely",
   "blocks": "Blocks",
   "allBlocks": "All blocks",
   "allBlocksDescription": "Total size of blocks on your entire IPFS node; this includes everything in Files, plus all locally pinned items and any temporary cached data",

public/locales/en/settings.json

@@ -3,9 +3,13 @@
   "save": "Save",
   "saving": "Saving…",
   "reset": "Reset",
+  "learnMoreLink": "Learn more.",
   "pinningServices": {
     "title": "Pinning Services",
-    "description": "Support for third-party remote pinning services will be implemented in a future release, and you'll be able to add and configure them here. In the meantime, use local pinning when you want to ensure an item on your node is never garbage-collected, even if you remove it from Files."
+    "description": "Use local pinning when you want to ensure an item on your node is never garbage-collected, even if you remove it from Files. You can also link your accounts with other remote pinning services to automatically or selectively persist files with those providers, enabling you to keep backup copies of your files and/or make them available to others when your local node is offline.",
+    "noPinRemoteDescription": "Support for third-party remote pinning services is missing. Update your IPFS node to go-ipfs 0.8.0 or later, and you'll be able to add and configure them here.",
+    "addAutoUpload": "Enable auto upload",
+    "removeAutoUpload": "Disable Auto Upload"
   },
   "language": "Language",
   "analytics": "Analytics",
@@ -37,17 +41,26 @@
     "secretApiKey": "Secret API key",
     "autoUpload": "Auto upload"
   },
+  "autoUploadModal": {
+    "title": "Change upload policy for {name}",
+    "description": "When enabled, auto upload will periodically pin the root of your Files directory, ensuring everything is pinned remotely and available to other peers even when your local IPFS node goes offline. If you disable it, you can still pin each file manually using the \"Set pinning\" option on the Files screen."
+  },
   "errors": {
     "nickname": "Nickname is required",
+    "apiError": "API error",
     "apiEndpoint": "Must be a valid URL",
-    "secretApiKey": "Secret key is required"
+    "secretApiKey": "Secret key is required",
+    "failedToFetch": "Failed to fetch",
+    "failedToFetchTitle": "Unable to fetch pin count from this remote service. Make sure it is online and that you entered correct credentials. If this is a newly added service, try removing and adding it again."
   },
   "actions": {
     "addService": "Add Service",
     "edit": "Change",
     "close": "Close",
     "save": "Save",
-    "cancel": "Cancel"
+    "cancel": "Cancel",
+    "enable": "Enable",
+    "disable": "Disable"
   },
   "edit": "Edit",
   "visitService": "Visit service",
@@ -57,10 +70,9 @@
   "size": "Size",
   "pins": "Pins",
   "autoUpload": "Auto Upload",
-  "autoUploadKeys": {
-    "ALL_FILES": "All files",
-    "DISABLED": "Disabled",
-    "PINS_ONLY": "Pins only"
+  "autoUploadPolicy": {
+    "true": "All files",
+    "false": "Off"
   },
   "fetchingSettings": "Fetching settings...",
   "configApiNotAvailable": "The IPFS config API is not available. Please disable the \"IPFS Companion\" Web Extension and try again.",

src/bundles/files/actions.js

@@ -50,6 +50,16 @@ const fileFromStats = ({ cumulativeSize, type, size, cid, name, path, pinned, is
   isParent: isParent
 })
 
+/**
+ * @param {IPFSService} ipfs
+ * @param {string|CID} cidOrPath
+ * @returns {Promise<number>}
+ */
+const cumulativeSize = async (ipfs, cidOrPath) => {
+  const { cumulativeSize } = await stat(ipfs, cidOrPath)
+  return cumulativeSize || 0
+}
+
 /**
  * @param {string} path
  * @returns {string}
@@ -68,6 +78,7 @@ export const realMfsPath = (path) => {
  * @property {string} path
  * @property {'file'|'directory'|'unknown'} type
  * @property {CID} cid
+ * @property {number} cumulativeSize
  * @property {number} size
  *
  * @param {IPFSService} ipfs
@@ -81,6 +92,7 @@ const stat = async (ipfs, cidOrPath) => {
     : `/ipfs/${hashOrPath}`
 
   try {
+    // TODO: memoize/cache result per CID
     const stats = await ipfs.files.stat(path)
     return { path, ...stats }
   } catch (e) {
@@ -93,6 +105,7 @@ const stat = async (ipfs, cidOrPath) => {
       path: hashOrPath,
       cid: new CID(cid),
       type: 'unknown',
+      cumulativeSize: 0,
       size: 0
     }
   }
@@ -511,14 +524,11 @@ const actions = () => ({
    * Gets total size of the local pins. On successful completion `state.mfsSize` will get
    * updated.
    */
-  doPinsSizeGet: () => perform(ACTIONS.PINS_SIZE_GET, async (ipfs) => {
-    const allPinsCids = await ipfs.pin.ls({ type: 'recursive' })
-
-    let pinsSize = 0
+  doPinsStatsGet: () => perform(ACTIONS.PINS_SIZE_GET, async (ipfs) => {
+    const pinsSize = -1 // TODO: right now calculating size of all pins is too expensive (requires ipfs.files.stat per CID)
     let numberOfPins = 0
 
-    for await (const { cid } of allPinsCids) {
-      pinsSize += (await ipfs.files.stat(`/ipfs/${cid.toString()}`)).cumulativeSize
+    for await (const _ of ipfs.pin.ls({ type: 'recursive' })) { // eslint-disable-line  no-unused-vars
       numberOfPins++
     }
 
@@ -530,8 +540,7 @@ const actions = () => ({
    * updated.
    */
   doFilesSizeGet: () => perform(ACTIONS.SIZE_GET, async (ipfs) => {
-    const stat = await ipfs.files.stat('/')
-    return { size: stat.cumulativeSize }
+    return cumulativeSize(ipfs, '/')
   }),
 
   /**
@@ -544,8 +553,7 @@ const actions = () => ({
     */
     async (store) => {
       const ipfs = store.getIpfs()
-      const stat = await ipfs.files.stat(`/ipfs/${cid}`)
-      return stat.cumulativeSize
+      return cumulativeSize(ipfs, cid)
     }
 })
 
@@ -615,7 +623,7 @@ const dirStats = async (ipfs, cid, { path, isRoot, sorting }) => {
       }
 
       parent = fileFromStats({
-        ...await ipfs.files.stat(parentInfo.realPath),
+        ...await stat(ipfs, parentInfo.realPath),
         path: parentInfo.path,
         name: '..',
         isParent: true

src/bundles/peer-bandwidth.js

@@ -2,7 +2,7 @@ import { createSelector } from 'redux-bundler'
 import last from 'it-last'
 
 // Depends on ipfsBundle, peersBundle, routesBundle
-export default function (opts) {
+const bundle = function (opts) {
   opts = opts || {}
   // Max number of peers to update at once
   opts.peerUpdateConcurrency = opts.peerUpdateConcurrency || 5
@@ -180,3 +180,5 @@ export default function (opts) {
     )
   }
 }
+
+export default bundle