From 26e34acfe70cac099acfa6dc8c2cf156c46fdae0 Mon Sep 17 00:00:00 2001 From: Christopher Joel <240083+cdata@users.noreply.github.com> Date: Mon, 14 Nov 2022 14:12:33 -0800 Subject: [PATCH] feat: Add `SphereFS` read/write to FFI (#141) This change also adds automated Swift testing to our Github Actions workflow. BREAKING CHANGE: Some FFI interfaces now have simplified interfaces. --- .github/workflows/noosphere_apple_build.yaml | 94 +++++-- .github/workflows/release.yaml | 28 ++ .github/workflows/run_test_suite.yaml | 32 ++- Cargo.lock | 8 +- README.md | 8 +- design/explainer.md | 77 +++--- design/{ => images}/Memo_1.png | Bin design/{ => images}/Noosphere_1.png | Bin design/images/noosphere-dark.svg | 1 + design/images/noosphere-light.svg | 1 + rust/noosphere-fs/src/file.rs | 21 +- rust/noosphere-fs/src/fs.rs | 1 + rust/noosphere/Cargo.toml | 2 +- rust/noosphere/src/ffi/fs.rs | 243 ++++++++++++++++++ rust/noosphere/src/ffi/headers.rs | 37 +++ rust/noosphere/src/ffi/key.rs | 5 +- rust/noosphere/src/ffi/mod.rs | 9 +- rust/noosphere/src/ffi/noosphere.rs | 29 ++- rust/noosphere/src/ffi/sphere.rs | 44 ++-- rust/noosphere/src/sphere/context.rs | 2 +- .../SwiftNoosphereTests/NoosphereTests.swift | 63 ++++- 21 files changed, 603 insertions(+), 102 deletions(-) rename design/{ => images}/Memo_1.png (100%) rename design/{ => images}/Noosphere_1.png (100%) create mode 100644 design/images/noosphere-dark.svg create mode 100644 design/images/noosphere-light.svg create mode 100644 rust/noosphere/src/ffi/fs.rs create mode 100644 rust/noosphere/src/ffi/headers.rs diff --git a/.github/workflows/noosphere_apple_build.yaml b/.github/workflows/noosphere_apple_build.yaml index fe1133833..3b2aebebb 100644 --- a/.github/workflows/noosphere_apple_build.yaml +++ b/.github/workflows/noosphere_apple_build.yaml @@ -1,26 +1,63 @@ on: - - workflow_call - - workflow_dispatch + workflow_call: + inputs: + for-test: + type: boolean + default: false + workflow_dispatch: + inputs: + for-test: + description: 'MacOS x86_64 support only' + type: boolean + default: false name: 'Build Noosphere artifacts (Apple)' jobs: + determine-build-matrix: + name: 'Determine build matrix' + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - id: set-matrix + env: + FOR_TEST: ${{ inputs.for-test }} + run: | + if [[ $FOR_TEST == true ]]; then + targets=("x86_64-apple-darwin") + else + targets=( + "aarch64-apple-ios" + "x84_64-apple-ios" + "aarch64-apple-ios-sim" + "x86_64-apple-darwin" + "aarch64-apple-darwin" + ) + fi + + echo -n 'matrix={"include":[' >> $GITHUB_OUTPUT + + target_out="" + + for target in "${targets[@]}"; do + target_json="{\"target\":\"$target\"}" + if [ -z "$target_out" ]; then + target_out="$target_json" + else + target_out="$target_out,$target_json" + fi + done + + echo -n "$target_out ]}" >> $GITHUB_OUTPUT + # Build Noosphere out of the noosphere crate for Apple targets noosphere-apple-build: name: 'Build Noosphere libraries (Apple)' + needs: ['determine-build-matrix'] strategy: - fail-fast: false - matrix: - include: - # iOS - - target: aarch64-apple-ios - # iOS Simulator - - target: x86_64-apple-ios - - target: aarch64-apple-ios-sim - # Mac OS - - target: x86_64-apple-darwin - - target: aarch64-apple-darwin - + fail-fast: true + matrix: ${{ fromJSON(needs.determine-build-matrix.outputs.matrix) }} runs-on: macos-12 steps: - uses: actions/checkout@v3 @@ -92,7 +129,9 @@ jobs: runs-on: macos-12 steps: - uses: actions/download-artifact@v3 + if: ${{ !inputs.for-test }} - name: 'Make a universal library' + if: ${{ !inputs.for-test }} run: | lipo -create \ ./lib_${{ matrix.legacy_target }}/libnoosphere.a \ @@ -100,6 +139,7 @@ jobs: -output ./libnoosphere.a - uses: actions/upload-artifact@v3 + if: ${{ !inputs.for-test }} with: name: lib_${{ matrix.platform }}-universal path: libnoosphere.a @@ -113,15 +153,25 @@ jobs: steps: - uses: actions/download-artifact@v3 - name: 'Generate XCFramework' + env: + FOR_TEST: ${{ inputs.for-test }} run: | - xcodebuild -create-xcframework \ - -library ./lib_macos-universal/libnoosphere.a \ - -headers ./include/ \ - -library ./lib_ios-simulator-universal/libnoosphere.a \ - -headers ./include/ \ - -library ./lib_aarch64-apple-ios/libnoosphere.a \ - -headers ./include/ \ - -output ./LibNoosphere.xcframework + if [ "$FOR_TEST" = true ]; then + xcodebuild -create-xcframework \ + -library ./lib_x86_64-apple-darwin/libnoosphere.a \ + -headers ./include/ \ + -output ./LibNoosphere.xcframework + else + xcodebuild -create-xcframework \ + -library ./lib_macos-universal/libnoosphere.a \ + -headers ./include/ \ + -library ./lib_ios-simulator-universal/libnoosphere.a \ + -headers ./include/ \ + -library ./lib_aarch64-apple-ios/libnoosphere.a \ + -headers ./include/ \ + -output ./LibNoosphere.xcframework + fi + zip -r ./libnoosphere-apple-xcframework.zip ./LibNoosphere.xcframework - uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6059ba5a8..159778633 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -109,6 +109,34 @@ jobs: libnoosphere-apple-xcframework.zip.sha256 tag_name: ${{ needs['release-please'].outputs.noosphere_release_tag_name }} + update-swift-noosphere-binary-target: + name: 'Update SwiftNoosphere binary target' + needs: ['release-please', 'noosphere-release-artifacts'] + runs-on: ubuntu-latest + steps: + - name: 'Download XCode Framework artifact' + uses: actions/download-artifact@v3 + with: + name: libnoosphere_apple_framework + + - name: 'Generate checksum' + id: generate-checksum + run: | + CHECKSUM=`openssl dgst -r -sha256 ./libnoosphere-apple-xcframework.zip | cut -d " " -f 1` + echo "checksum=\"$CHECKSUM\"" >> $GITHUB_OUTPUT + - name: 'Modify Package.swift' + run: | + URL="https://github.com/subconsciousnetwork/noosphere/releases/download/${{ needs.release-please.outputs.noosphere_release_tag_name }}/libnoosphere-apple-xcframework.zip" + + sed -i '' -e "s#url: \"[^\"]*\",#url: \"$URL\",#" ./Package.swift + sed -i '' -e "s#checksum: \"[^\"]*\"),#checksum: \"${{ steps.generate-checksum.checksum }}\"),#" ./Package.swift + # TODO(#135): Evaluate if we want to automatically check this change in + # and push it to the project repository + - uses: actions/upload-artifact@v3 + with: + name: swift-package-manifest + path: ./Package.swift + # Publishes crates to crates.io in dependency order. This command is # idempotent and won't re-publish crates that are already published, so it's # safe for us to run it indiscriminately diff --git a/.github/workflows/run_test_suite.yaml b/.github/workflows/run_test_suite.yaml index ebb1e430a..2bc110c4b 100644 --- a/.github/workflows/run_test_suite.yaml +++ b/.github/workflows/run_test_suite.yaml @@ -1,9 +1,37 @@ -on: [push, pull_request] +on: + push: + branches: [main] + pull_request: name: Run test suite jobs: - run-test-suite: + build-noosphere-apple-artifacts: + name: 'Build Noosphere artifacts (Apple)' + uses: ./.github/workflows/noosphere_apple_build.yaml + with: + for-test: true + + run-test-suite-mac-os: + runs-on: macos-12 + needs: ['build-noosphere-apple-artifacts'] + steps: + - uses: actions/checkout@v2 + - name: 'Download XCode Framework artifact' + uses: actions/download-artifact@v3 + with: + name: libnoosphere_apple_framework + + - name: 'Run Swift tests' + run: | + unzip ./libnoosphere-apple-xcframework.zip + + sed -i '' -e "s#url: \"[^\"]*\",#path: \"./LibNoosphere.xcframework\"),#" ./Package.swift + sed -i '' -e "s#checksum: \"[^\"]*\"),##" ./Package.swift + + swift build + swift test + run-test-suite-linux: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 diff --git a/Cargo.lock b/Cargo.lock index 0a42e22dc..b34310124 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2271,9 +2271,9 @@ dependencies = [ "noosphere-core", "noosphere-fs", "noosphere-storage", - "pollster", "rexie", "safer-ffi", + "subtext", "temp-dir", "thiserror", "tokio", @@ -2849,12 +2849,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "pollster" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5da3b0203fd7ee5720aa0b5e790b591aa5d3f41c3ed2c34a3a393382198af2f7" - [[package]] name = "poly1305" version = "0.7.2" diff --git a/README.md b/README.md index a52108bfd..90cd62d68 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,14 @@ ![License](https://img.shields.io/badge/license-MIT%2FApache--2.0-blue) [![Discord](https://img.shields.io/discord/1003419732516552724.svg?logo=discord&colorB=7289DA)](https://discord.gg/HmHypb6DCj) -# Noosphere + + + + Noosphere + > Noosphere (noun): +> > 1. Planetary consciousness. A hypothetical new evolutionary phenomena rising out of the biosphere. > 2. A protocol for thought. @@ -38,4 +43,3 @@ Apache-2.0: https://www.apache.org/licenses/license-2.0 [design]: https://github.com/subconsciousnetwork/noosphere/tree/main/design/ [roadmap]: https://github.com/orgs/subconsciousnetwork/projects/1/views/4 [noosphere-kanban]: https://github.com/orgs/subconsciousnetwork/projects/1/views/8 - diff --git a/design/explainer.md b/design/explainer.md index 916841701..25a5c7434 100644 --- a/design/explainer.md +++ b/design/explainer.md @@ -1,21 +1,21 @@ # Noosphere Explainer -*Last updated: July 2nd, 2022* +_Last updated: July 2nd, 2022_ Noosphere is a massively-multiplayer knowledge graph. The technical pillars that Noosphere builds upon are: -* [Public key infrastructure](https://en.wikipedia.org/wiki/Public_key_infrastructure) -* [Content addressing](https://en.wikipedia.org/wiki/Content-addressable_storage) -* [Immutable data](https://en.wikipedia.org/wiki/Immutable_object) -* [P2P routing and discovery](https://en.wikipedia.org/wiki/Peer-to-peer) +- [Public key infrastructure](https://en.wikipedia.org/wiki/Public_key_infrastructure) +- [Content addressing](https://en.wikipedia.org/wiki/Content-addressable_storage) +- [Immutable data](https://en.wikipedia.org/wiki/Immutable_object) +- [P2P routing and discovery](https://en.wikipedia.org/wiki/Peer-to-peer) Above this substructure, Noosphere gives users: -* Entry to a zero-trust, **decentralized network** of self-sovereign nodes -* **Human-readable names** for peers and their public content -* **Local-first authoring** and offline-available content with conflict-free synchronization -* A complete, space-efficient **revision history** for any content -* Coherence and **compatibility with the hypertext web** +- Entry to a zero-trust, **decentralized network** of self-sovereign nodes +- **Human-readable names** for peers and their public content +- **Local-first authoring** and offline-available content with conflict-free synchronization +- A complete, space-efficient **revision history** for any content +- Coherence and **compatibility with the hypertext web** You can think of it like a world-wide Wiki. @@ -23,21 +23,21 @@ You can think of it like a world-wide Wiki. Subconscious user Bob authors a note about cats, formatted as Subtext: -| *Bob's notebook · /cat-thoughts* | -| -------------------------------------- | +| _Bob's notebook · /cat-thoughts_ | +| ------------------------------------------- | | I love cats.
I love every kind of cat. | -Notes in Bob's Subconscious notebook have a corresponding slug (e.g., *cat-thoughts*), that can be used as anchors for linking across notes. Later on when Bob authors another note on a related topic, they include a link to their first note: +Notes in Bob's Subconscious notebook have a corresponding slug (e.g., _cat-thoughts_), that can be used as anchors for linking across notes. Later on when Bob authors another note on a related topic, they include a link to their first note: -| *Bob's notebook · /animal-notes* | -| -------------------------------------- | +| _Bob's notebook · /animal-notes_ | +| -------------------------------------------------------------- | | I have strongly felt **/cat-thoughts**
Dogs are just okay | By following the link, Bob or another reader can navigate from one note to another note within Bob's local notebook. One day Bob meets Alice and they get to talking about their shared interest in animals. Bob discovers that Alice also uses Subconscious. They exchange contact details, and soon Alice is able view an index of Bob's public notes: -| *Bob's notebook · Links* | +| _Bob's notebook · Links_ | | ------------------------ | | /animal-notes | | /cat-thoughts | @@ -45,9 +45,9 @@ One day Bob meets Alice and they get to talking about their shared interest in a After reading through Bob's cat-thoughts, Alice decides to reference it for later review. Alice opens up a local note about animals and links to Bob's note from there: -| *Alice's notebook · /awesome-animal-links* | -| ------------------------------------------ | -| Here are some cool **/zebra-facts**
I love that **/skateboarding-dogs** exist
Bob sure has some **@bob/cat–thoughts**| +| _Alice's notebook · /awesome-animal-links_ | +| ------------------------------------------------------------------------------------------------------------------------------- | +| Here are some cool **/zebra-facts**
I love that **/skateboarding-dogs** exist
Bob sure has some **@bob/cat–thoughts** | By following the link **@bob/cat-thoughts**, Alice can navigate from a note in her local notebook to a note in Bob's notebook. @@ -59,20 +59,20 @@ Let's break down how a link like **@bob/cat–thoughts** works. Please note that ### Public key infrastructure -When Alice and Bob exchanged contact details in the story above, the exchanged data included [public keys][public key] (encoded as [DIDs][DID] that represent their respective notebooks. Bob's public key was then recorded against a [pet name][petname] in Alice's notebook: +When Alice and Bob exchanged contact details in the story above, the exchanged data included [public keys][public key] (encoded as [DIDs][did] that represent their respective notebooks. Bob's public key was then recorded against a [pet name][petname] in Alice's notebook: -| *Alice's notebook · Names* | +| _Alice's notebook · Names_ | | --------------------------------------------------------------------- | | mallory => `did:key:z6MktafZTREjJkvV5mfJxcLpNBoVPwDLhTuMg9ng7dY4zMAL` | | bob => `did:key:z6MkffDZCkCTWreg8868fG1FGFogcJj5X6PY93pPcWDn9bob` | ### Content addressing -As Alice updated her notebook to include Bob's name, a snapshot of her entire notebook at its latest state was recorded and condensed down to a short, unique ID: [a content ID, or CID][CID]. Such a snapshot and corresponding CID is recorded for every update to every user's notebook (including Bob's). +As Alice updated her notebook to include Bob's name, a snapshot of her entire notebook at its latest state was recorded and condensed down to a short, unique ID: [a content ID, or CID][cid]. Such a snapshot and corresponding CID is recorded for every update to every user's notebook (including Bob's). -Similarly, when Bob updated their *animal-notes* to include a link to *cat-thoughts*, a new snapshot of the note was recorded and a CID was computed for it. When Alice viewed the index of links in Bob's notebook, what she actually viewed was a mapping of note slugs to their CIDs: +Similarly, when Bob updated their _animal-notes_ to include a link to _cat-thoughts_, a new snapshot of the note was recorded and a CID was computed for it. When Alice viewed the index of links in Bob's notebook, what she actually viewed was a mapping of note slugs to their CIDs: -| *Bob's notebook · Links* | +| _Bob's notebook · Links_ | | ----------------------------------------- | | /animal-notes => Cid(bafy2bza...3xcyge7s) | | /cat-thoughts => Cid(bafy2bza...thqn3hrm) | @@ -80,11 +80,11 @@ Similarly, when Bob updated their *animal-notes* to include a link to *cat-thoug ### Immutable data -Noosphere data is formatted in terms of [IPLD][IPLD] and encoded in a low-overhead binary format called [DAG-CBOR][DAG-CBOR]. Even though a full snapshot is recorded with every revision, new storage space is only allocated for the delta between any two revisions. This strategy is similar to how Git efficiently stores the delta between any two sequential commits. +Noosphere data is formatted in terms of [IPLD][ipld] and encoded in a low-overhead binary format called [DAG-CBOR][dag-cbor]. Even though a full snapshot is recorded with every revision, new storage space is only allocated for the delta between any two revisions. This strategy is similar to how Git efficiently stores the delta between any two sequential commits. -A data structure that we call a *Memo* is used to pair open-ended header fields with a retained historical record of revisions to notebooks and their contents: +A data structure that we call a _Memo_ is used to pair open-ended header fields with a retained historical record of revisions to notebooks and their contents: -![Memo](Memo_1.png "Memo") +![Memo](images/Memo_1.png 'Memo') The properties of immutable data structures allow content to be edited offline for an indefinite period of time and safely copied to replicas without the risk of conflicts at the convenience of the client. @@ -92,13 +92,13 @@ The properties of immutable data structures allow content to be edited offline f Every user who publishes to the network does so via a gateway server. The gateway represents the boundary edge of user sovereignty, and also gives the user a reliably available foothold in the Noosphere network. The owner of a notebook is also the owner of the gateway, and third-parties neither have or need direct access to it (even in managed hosting scenarios). -The owner of a notebook enables the gateway to publish the notebook to the network using a [UCAN][UCAN]-based authorization scheme. UCANs establish a cryptographic chain of proof that allows anyone to verify in place that the gateway was authorized to perform actions such as signing content on the user's behalf (and without asking the user to share cryptographic keys with the gateway). +The owner of a notebook enables the gateway to publish the notebook to the network using a [UCAN][ucan]-based authorization scheme. UCANs establish a cryptographic chain of proof that allows anyone to verify in place that the gateway was authorized to perform actions such as signing content on the user's behalf (and without asking the user to share cryptographic keys with the gateway). -When the user updates their notebook, they replicate the revision deltas to the gateway over HTTP (as network availability allows), and also tell the gateway which CID represents the latest version of the notebook. The revision deltas are syndicated to the public network via [IPFS][IPFS]. Then, the gateway publishes the latest revision CID to a [DHT][DHT], mapping it to the notebook's public key and pairing it with the UCAN proof chain. +When the user updates their notebook, they replicate the revision deltas to the gateway over HTTP (as network availability allows), and also tell the gateway which CID represents the latest version of the notebook. The revision deltas are syndicated to the public network via [IPFS][ipfs]. Then, the gateway publishes the latest revision CID to a [DHT][dht] (see: [Noosphere Name System][noosphere-ns]) mapping it to the notebook's public key and pairing it with the UCAN proof chain. ### Putting it all together -After the gateway publishes an update to the DHT, it becomes possible for anyone who knows the public key of a notebook to discover the latest published revision of that notebook as a CID. +After the gateway publishes an update to the DHT, it becomes possible for anyone who knows the public key of a notebook to discover the latest published revision of that notebook as a CID. When that CID is known, it becomes possible for the entire notebook (or discrete parts of it) to be downloaded from IPFS and replicated to the local client of the reader. @@ -106,14 +106,15 @@ Once the notebook is downloaded, any slug can be resolved to content using the s Here is Alice's notebook, visualized as simplified data structures; the content referenced by **@bob/cat-thoughts** is able to be resolved with a combination of metadata in Alice's notebook, and details published by Bob to the DHT: -![Noosphere](Noosphere_1.png "Noosphere") +![Noosphere](images/Noosphere_1.png 'Noosphere') [public key]: https://en.wikipedia.org/wiki/Public-key_cryptography [petname]: http://www.skyhunter.com/marcs/petnames/IntroPetNames.html -[DID]: https://www.w3.org/TR/did-core/ -[CID]: https://docs.ipfs.io/concepts/content-addressing/ -[IPLD]: https://ipld.io/ -[IPFS]: https://ipfs.io/ -[DAG-CBOR]: https://ipld.io/docs/codecs/known/dag-cbor/ -[UCAN]: https://ucan.xyz/ -[DHT]: https://en.wikipedia.org/wiki/Distributed_hash_table +[did]: https://www.w3.org/TR/did-core/ +[cid]: https://docs.ipfs.io/concepts/content-addressing/ +[ipld]: https://ipld.io/ +[ipfs]: https://ipfs.io/ +[dag-cbor]: https://ipld.io/docs/codecs/known/dag-cbor/ +[ucan]: https://ucan.xyz/ +[dht]: https://en.wikipedia.org/wiki/Distributed_hash_table +[noosphere-ns]: name-system.md diff --git a/design/Memo_1.png b/design/images/Memo_1.png similarity index 100% rename from design/Memo_1.png rename to design/images/Memo_1.png diff --git a/design/Noosphere_1.png b/design/images/Noosphere_1.png similarity index 100% rename from design/Noosphere_1.png rename to design/images/Noosphere_1.png diff --git a/design/images/noosphere-dark.svg b/design/images/noosphere-dark.svg new file mode 100644 index 000000000..bffc5a4d5 --- /dev/null +++ b/design/images/noosphere-dark.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/design/images/noosphere-light.svg b/design/images/noosphere-light.svg new file mode 100644 index 000000000..cd7b6f3cd --- /dev/null +++ b/design/images/noosphere-light.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/rust/noosphere-fs/src/file.rs b/rust/noosphere-fs/src/file.rs index f6d8175e0..d36b2947d 100644 --- a/rust/noosphere-fs/src/file.rs +++ b/rust/noosphere-fs/src/file.rs @@ -1,10 +1,29 @@ +use std::pin::Pin; + use cid::Cid; -use noosphere_core::data::MemoIpld; +use noosphere_core::data::{Did, MemoIpld}; +use tokio::io::AsyncRead; /// A descriptor for contents that is stored in a sphere. pub struct SphereFile { + pub sphere_identity: Did, pub sphere_revision: Cid, pub memo_revision: Cid, pub memo: MemoIpld, pub contents: C, } + +impl SphereFile +where + C: AsyncRead + 'static, +{ + pub fn boxed(self) -> SphereFile>> { + SphereFile { + sphere_identity: self.sphere_identity, + sphere_revision: self.sphere_revision, + memo_revision: self.memo_revision, + memo: self.memo, + contents: Box::pin(self.contents), + } + } +} diff --git a/rust/noosphere-fs/src/fs.rs b/rust/noosphere-fs/src/fs.rs index d3a3d4d01..28d21b569 100644 --- a/rust/noosphere-fs/src/fs.rs +++ b/rust/noosphere-fs/src/fs.rs @@ -99,6 +99,7 @@ where }; Ok(SphereFile { + sphere_identity: self.sphere_identity.clone(), sphere_revision: self.sphere_revision, memo_revision: *memo_revision, memo, diff --git a/rust/noosphere/Cargo.toml b/rust/noosphere/Cargo.toml index 7e16c0c85..d500bc519 100644 --- a/rust/noosphere/Cargo.toml +++ b/rust/noosphere/Cargo.toml @@ -30,6 +30,7 @@ cid = "~0.8" async-trait = "~0.1" tracing = "~0.1" url = "^2" +subtext = { version = "~0.3" } noosphere-core = { version = "0.1.0", path = "../noosphere-core" } noosphere-fs = { version = "0.1.1-alpha.1", path = "../noosphere-fs" } @@ -56,7 +57,6 @@ features = [ [target.'cfg(not(target_arch = "wasm32"))'.dependencies] safer-ffi = { version = "~0.0.10", features = ["proc_macros"] } -pollster = "~0.2" tokio = { version = "^1", features = ["full"] } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] diff --git a/rust/noosphere/src/ffi/fs.rs b/rust/noosphere/src/ffi/fs.rs new file mode 100644 index 000000000..b13ef540d --- /dev/null +++ b/rust/noosphere/src/ffi/fs.rs @@ -0,0 +1,243 @@ +use anyhow::anyhow; +use noosphere_core::data::Did; +use noosphere_fs::{SphereFile, SphereFs}; +use safer_ffi::prelude::*; +use std::{pin::Pin, str::FromStr}; +use subtext::{Peer, Slashlink}; +use tokio::io::{AsyncRead, AsyncReadExt}; + +use crate::{ + ffi::{NsHeaders, NsNoosphereContext}, + platform::{PlatformKeyMaterial, PlatformStore}, +}; + +ReprC! { + #[ReprC::opaque] + pub struct NsSphereFs { + inner: SphereFs + } +} + +impl NsSphereFs { + pub fn inner(&self) -> &SphereFs { + &self.inner + } + + pub fn inner_mut(&mut self) -> &mut SphereFs { + &mut self.inner + } +} + +ReprC! { + #[ReprC::opaque] + pub struct NsSphereFile { + inner: SphereFile>> + } +} + +impl NsSphereFile { + pub fn inner(&self) -> &SphereFile>> { + &self.inner + } + + pub fn inner_mut(&mut self) -> &mut SphereFile>> { + &mut self.inner + } +} + +#[ffi_export] +/// Initialize an instance of a [NsSphereFs] that is a read/write view into +/// the contents (as addressed by the slug namespace) of the identitifed sphere +/// This will fail if it is not possible to initialize a sphere with the given +/// identity (which implies that no such sphere was ever created or joined on +/// this device). +pub fn ns_sphere_fs_open( + noosphere: &NsNoosphereContext, + sphere_identity: char_p::Ref<'_>, +) -> repr_c::Box { + noosphere + .async_runtime() + .block_on(async { + let sphere_context = noosphere + .inner() + .get_sphere_context(&Did(sphere_identity.to_str().into())) + .await?; + + let sphere_context = sphere_context.lock().await; + + Ok(repr_c::Box::new(NsSphereFs { + inner: sphere_context.fs().await?, + })) as Result<_, anyhow::Error> + }) + .unwrap() +} + +#[ffi_export] +/// De-allocate an [NsSphereFs] instance +pub fn ns_sphere_fs_free(sphere_fs: repr_c::Box) { + drop(sphere_fs) +} + +#[ffi_export] +/// Read a [NsSphereFile] from a [NsSphereFs] instance by slashlink. Note that +/// although this function will eventually support slashlinks that include the +/// pet name of a peer, at this time only slashlinks with slugs referencing the +/// slug namespace of the local sphere are allowed. +/// +/// This function will return a null pointer if the slug does not have a file +/// associated with it at the revision of the sphere that is referred to by the +/// [NsSphereFs] being read from. +pub fn ns_sphere_fs_read( + noosphere: &NsNoosphereContext, + sphere_fs: &NsSphereFs, + slashlink: char_p::Ref<'_>, +) -> Option> { + noosphere + .async_runtime() + .block_on(async { + let slashlink = match Slashlink::from_str(slashlink.to_str()) { + Ok(slashlink) => slashlink, + _ => return Ok(None), + }; + + if Peer::None != slashlink.peer { + return Err(anyhow!("Peer in slashlink not yet supported")); + } + + let slug = match slashlink.slug { + Some(slug) => slug, + None => return Err(anyhow!("No slug specified in slashlink!")), + }; + + println!( + "Reading sphere {} slug {}...", + sphere_fs.inner().identity(), + slug + ); + + let file = sphere_fs.inner().read(&slug).await?; + + Ok(file.map(|sphere_file| { + repr_c::Box::new(NsSphereFile { + inner: sphere_file.boxed(), + }) + })) + }) + .unwrap() +} + +#[ffi_export] +/// Write a byte buffer to a slug in the given [NsSphereFs] instance, assigning +/// its content-type header to the specified value. If additional headers are +/// specified, they will be appended to the list of headers in the memo that is +/// created for the content. If some content already exists at the specified +/// slug, it will be assigned to be the previous historical revision of the new +/// content. +/// +/// Note that you must invoke [ns_sphere_fs_save] to commit one or more writes +/// to the sphere. +pub fn ns_sphere_fs_write( + noosphere: &NsNoosphereContext, + sphere_fs: &mut NsSphereFs, + slug: char_p::Ref<'_>, + content_type: char_p::Ref<'_>, + bytes: c_slice::Ref<'_, u8>, + additional_headers: Option<&NsHeaders>, +) { + noosphere.async_runtime().block_on(async { + let slug = slug.to_str(); + + println!( + "Writing sphere {} slug {}...", + sphere_fs.inner().identity(), + slug + ); + + match sphere_fs + .inner_mut() + .write( + slug, + content_type.to_str().try_into().unwrap(), + bytes.as_ref(), + additional_headers.map(|headers| headers.inner().clone()), + ) + .await + { + Ok(_) => println!("Updated {:?}...", slug), + Err(error) => println!("Sphere write failed: {}", error), + } + }) +} + +#[ffi_export] +/// Save any writes performed on the [NsSphereFs] instance. If additional +/// headers are specified, they will be appended to the headers in the memo that +/// is created to wrap the latest sphere revision. +/// +/// This will fail if both no writes have been performed and no additional +/// headers were specified (in other words: no actual changes were made). +pub fn ns_sphere_fs_save( + noosphere: &NsNoosphereContext, + sphere_fs: &mut NsSphereFs, + additional_headers: Option<&NsHeaders>, +) { + match noosphere.async_runtime().block_on( + sphere_fs + .inner_mut() + .save(additional_headers.map(|headers| headers.inner().clone())), + ) { + Ok(cid) => println!( + "Saved sphere {}; new revision is {}", + sphere_fs.inner().identity(), + cid + ), + Err(error) => println!("Sphere save failed: {}", error), + } +} + +#[ffi_export] +/// De-allocate an [NsSphereFile] instance +pub fn ns_sphere_file_free(sphere_file: repr_c::Box) { + drop(sphere_file) +} + +#[ffi_export] +/// Read the contents of an [NsSphereFile] as a byte array. Note that the +/// [NsSphereFile] is lazy and stateful: it doesn't read any bytes from disk +/// until this function is invoked, and once the bytes have been read from the +/// file you must create a new [NsSphereFile] instance to read them again. +pub fn ns_sphere_file_contents_read( + noosphere: &NsNoosphereContext, + sphere_file: &mut NsSphereFile, +) -> c_slice::Box { + noosphere.async_runtime().block_on(async { + let mut buffer = Vec::new(); + + sphere_file + .inner_mut() + .contents + .read_to_end(&mut buffer) + .await + .unwrap(); + + buffer.into_boxed_slice().into() + }) +} + +#[ffi_export] +/// Read all header values for a file that correspond to a given name, returning +/// them as an array of strings +pub fn ns_sphere_file_header_values_read( + sphere_file: &NsSphereFile, + name: char_p::Ref<'_>, +) -> c_slice::Box { + sphere_file + .inner + .memo + .get_header(name.to_str()) + .into_iter() + .map(|header| header.try_into().unwrap()) + .collect::>() + .into_boxed_slice() + .into() +} diff --git a/rust/noosphere/src/ffi/headers.rs b/rust/noosphere/src/ffi/headers.rs new file mode 100644 index 000000000..d49457232 --- /dev/null +++ b/rust/noosphere/src/ffi/headers.rs @@ -0,0 +1,37 @@ +use safer_ffi::prelude::*; + +ReprC! { + #[ReprC::opaque] + pub struct NsHeaders { + inner: Vec<(String, String)> + } +} + +impl NsHeaders { + pub fn inner(&self) -> &Vec<(String, String)> { + &self.inner + } + + pub fn inner_mut(&mut self) -> &mut Vec<(String, String)> { + &mut self.inner + } +} + +#[ffi_export] +/// Create a [NsHeaders] buffer for the purpose of building up a set of headers +/// intended to be added to a memo before it is written to a sphere +pub fn ns_headers_create() -> repr_c::Box { + repr_c::Box::new(NsHeaders { inner: Vec::new() }) +} + +#[ffi_export] +/// Add a name/value pair to an [NsHeaders] buffer +pub fn ns_headers_add(headers: &mut NsHeaders, name: char_p::Ref<'_>, value: char_p::Ref<'_>) { + headers.inner.push((name.to_string(), value.to_string())) +} + +#[ffi_export] +/// De-allocate an [NsHeaders] buffer +pub fn ns_headers_free(headers: repr_c::Box) { + drop(headers) +} diff --git a/rust/noosphere/src/ffi/key.rs b/rust/noosphere/src/ffi/key.rs index 0c57a1458..cbb2a114c 100644 --- a/rust/noosphere/src/ffi/key.rs +++ b/rust/noosphere/src/ffi/key.rs @@ -6,5 +6,8 @@ use crate::ffi::NsNoosphereContext; /// Create a key with the given name in the current platform's support key /// storage mechanism. pub fn ns_key_create(noosphere: &NsNoosphereContext, name: char_p::Ref<'_>) { - pollster::block_on(noosphere.inner().create_key(name.to_str())).unwrap(); + noosphere + .async_runtime() + .block_on(noosphere.inner().create_key(name.to_str())) + .unwrap(); } diff --git a/rust/noosphere/src/ffi/mod.rs b/rust/noosphere/src/ffi/mod.rs index d4c828d66..b09d29a77 100644 --- a/rust/noosphere/src/ffi/mod.rs +++ b/rust/noosphere/src/ffi/mod.rs @@ -1,13 +1,18 @@ -///! This module contains FFI implementation for all C ABI-speaking language -///! integrations. +mod fs; +mod headers; mod key; mod noosphere; mod sphere; pub use crate::ffi::noosphere::*; +pub use fs::*; +pub use headers::*; pub use key::*; pub use sphere::*; +///! This module contains FFI implementation for all C ABI-speaking language +///! integrations. + #[cfg(feature = "headers")] pub fn generate_headers() -> std::io::Result<()> { safer_ffi::headers::builder() diff --git a/rust/noosphere/src/ffi/noosphere.rs b/rust/noosphere/src/ffi/noosphere.rs index 9a41c8f7b..cb18d5276 100644 --- a/rust/noosphere/src/ffi/noosphere.rs +++ b/rust/noosphere/src/ffi/noosphere.rs @@ -1,5 +1,8 @@ +use std::sync::Arc; + use anyhow::Result; use safer_ffi::prelude::*; +use tokio::runtime::Runtime as TokioRuntime; use url::Url; use crate::noosphere::{NoosphereContext, NoosphereContextConfiguration}; @@ -7,7 +10,8 @@ use crate::noosphere::{NoosphereContext, NoosphereContextConfiguration}; ReprC! { #[ReprC::opaque] pub struct NsNoosphereContext { - inner: NoosphereContext + inner: NoosphereContext, + async_runtime: Arc } } @@ -23,9 +27,14 @@ impl NsNoosphereContext { sphere_storage_path: sphere_storage_path.into(), gateway_url: gateway_url.cloned(), })?, + async_runtime: Arc::new(TokioRuntime::new()?), }) } + pub fn async_runtime(&self) -> Arc { + self.async_runtime.clone() + } + pub fn inner(&self) -> &NoosphereContext { &self.inner } @@ -72,3 +81,21 @@ pub fn ns_initialize( pub fn ns_free(noosphere: repr_c::Box) { drop(noosphere) } + +#[ffi_export] +/// De-allocate a Noosphere-allocated byte array +pub fn ns_bytes_free(bytes: c_slice::Box) { + drop(bytes) +} + +#[ffi_export] +/// De-allocate a Noosphere-allocated string +pub fn ns_string_free(string: char_p::Box) { + drop(string) +} + +#[ffi_export] +/// De-allocate a Noosphere-allocated array of strings +pub fn ns_string_array_free(string_array: c_slice::Box) { + drop(string_array) +} diff --git a/rust/noosphere/src/ffi/sphere.rs b/rust/noosphere/src/ffi/sphere.rs index 67c49b449..72e221a46 100644 --- a/rust/noosphere/src/ffi/sphere.rs +++ b/rust/noosphere/src/ffi/sphere.rs @@ -22,18 +22,24 @@ impl From for NsSphereReceipt { #[ffi_export] /// Read the sphere identity (a DID encoded as a UTF-8 string) from a /// [SphereReceipt] -pub fn ns_sphere_receipt_identity<'a>( - sphere_receipt: &'a repr_c::Box, -) -> char_p::Ref<'a> { - char_p::Ref::try_from(sphere_receipt.inner.identity.as_str()).unwrap() +pub fn ns_sphere_receipt_identity<'a>(sphere_receipt: &'a NsSphereReceipt) -> char_p::Box { + sphere_receipt + .inner + .identity + .to_string() + .try_into() + .unwrap() } #[ffi_export] /// Read the mnemonic from a [SphereReceipt] -pub fn ns_sphere_receipt_mnemonic<'a>( - sphere_receipt: &'a repr_c::Box, -) -> char_p::Ref<'a> { - char_p::Ref::try_from(sphere_receipt.inner.mnemonic.as_str()).unwrap() +pub fn ns_sphere_receipt_mnemonic<'a>(sphere_receipt: &'a NsSphereReceipt) -> char_p::Box { + sphere_receipt + .inner + .mnemonic + .to_string() + .try_into() + .unwrap() } #[ffi_export] @@ -48,11 +54,13 @@ pub fn ns_sphere_receipt_free(sphere_receipt: repr_c::Box) { /// and a human-readable mnemonic that can be used to rotate the key authorized /// to administer the sphere. pub fn ns_sphere_create( - noosphere: &mut repr_c::Box, + noosphere: &mut NsNoosphereContext, owner_key_name: char_p::Ref<'_>, ) -> repr_c::Box { repr_c::Box::new( - pollster::block_on(noosphere.inner_mut().create_sphere(owner_key_name.to_str())) + noosphere + .async_runtime() + .block_on(noosphere.inner_mut().create_sphere(owner_key_name.to_str())) .unwrap() .into(), ) @@ -63,16 +71,18 @@ pub fn ns_sphere_create( /// key and authorization. The authorization should be provided in the form of /// a base64-encoded CID v1 string. pub fn ns_sphere_join( - noosphere: &mut repr_c::Box, + noosphere: &mut NsNoosphereContext, sphere_identity: char_p::Ref<'_>, local_key_name: char_p::Ref<'_>, authorization: char_p::Ref<'_>, ) { let authorization = Authorization::Cid(Cid::try_from(authorization.to_str()).unwrap()); - pollster::block_on(noosphere.inner_mut().join_sphere( - &Did::from(sphere_identity.to_str()), - local_key_name.to_str(), - &authorization, - )) - .unwrap(); + noosphere + .async_runtime() + .block_on(noosphere.inner_mut().join_sphere( + &Did::from(sphere_identity.to_str()), + local_key_name.to_str(), + &authorization, + )) + .unwrap(); } diff --git a/rust/noosphere/src/sphere/context.rs b/rust/noosphere/src/sphere/context.rs index 10d3b82c8..f76936061 100644 --- a/rust/noosphere/src/sphere/context.rs +++ b/rust/noosphere/src/sphere/context.rs @@ -58,7 +58,7 @@ where } } - /// The DID identity of the sphere + /// The identity of the sphere pub fn identity(&self) -> &Did { &self.sphere_identity } diff --git a/swift/Tests/SwiftNoosphereTests/NoosphereTests.swift b/swift/Tests/SwiftNoosphereTests/NoosphereTests.swift index 4c098985a..60c767cad 100644 --- a/swift/Tests/SwiftNoosphereTests/NoosphereTests.swift +++ b/swift/Tests/SwiftNoosphereTests/NoosphereTests.swift @@ -2,11 +2,60 @@ import XCTest @testable import SwiftNoosphere final class NoosphereTests: XCTestCase { - func testExample() throws { - // This is an example of a functional test case. - // Use XCTAssert and related functions to verify your tests produce the correct - // results. - // TODO - noosphere_key_create("foobar") + func testInitializeNoosphereThenWriteAFileThenSaveThenReadItBack() throws { + // This is a basic integration test to ensure that file writing and + // reading from swift works as intended + + let noosphere = ns_initialize("/tmp/foo", "/tmp/bar", nil) + + ns_key_create(noosphere, "bob") + + let sphere_receipt = ns_sphere_create(noosphere, "bob") + + let sphere_identity_ptr = ns_sphere_receipt_identity(sphere_receipt) + let sphere_mnemonic_ptr = ns_sphere_receipt_mnemonic(sphere_receipt) + + let sphere_identity = String.init(cString: sphere_identity_ptr!) + let sphere_mnemonic = String.init(cString: sphere_mnemonic_ptr!) + + print("Sphere identity:", sphere_identity) + print("Recovery code:", sphere_mnemonic) + + let sphere_fs = ns_sphere_fs_open(noosphere, sphere_identity_ptr) + + let file_bytes = "Hello, Subconscious".data(using: .utf8)! + + file_bytes.withUnsafeBytes({ (ptr: UnsafePointer) in + let contents = slice_ref_uint8(ptr: ptr, len: file_bytes.count) + ns_sphere_fs_write(noosphere, sphere_fs, "hello", "text/subtext", contents, nil) + + print("File write done") + }) + + ns_sphere_fs_save(noosphere, sphere_fs, nil) + + let file = ns_sphere_fs_read(noosphere, sphere_fs, "/hello") + + let content_type_values = ns_sphere_file_header_values_read(file, "Content-Type") + let content_type = String.init(cString: content_type_values.ptr.pointee!) + + print("Content-Type:", content_type) + + let contents = ns_sphere_file_contents_read(noosphere, file) + let data: Data = .init(bytes: contents.ptr, count: contents.len) + let subtext = String.init(decoding: data, as: UTF8.self) + + print("Contents:", subtext) + + ns_string_array_free(content_type_values) + ns_bytes_free(contents) + ns_sphere_file_free(file) + ns_sphere_fs_free(sphere_fs) + ns_string_free(sphere_identity_ptr) + ns_string_free(sphere_mnemonic_ptr) + ns_sphere_receipt_free(sphere_receipt) + ns_free(noosphere) + + print("fin!") } -} \ No newline at end of file +}