Feat/database data improvement (#531)

* feat: add doc store * feat: support readonly client * fix: fix the comment * feat: add database counter * feat: update the typescript sdk * fix: fix test case * fix: fix the test case * feat: fix the project bug * test: add more query test * fix: fix query test case * fix: fix test case
dbpunk-labs · Jul 1, 2023 · 5df94d2 · 5df94d2
1 parent 4976fc7
commit 5df94d2
Show file tree

Hide file tree

Showing 22 changed files with 639 additions and 293 deletions.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -41,7 +41,7 @@ cd sdk && make && yarn && yarn build
 ### Start Local Test Environment
 
 ```
-cd tools && bash start_localtestnet.sh
+cd tools && bash start_localnet.sh
 ```
 
 ### Run test cases
@@ -50,22 +50,3 @@ cd tools && bash start_localtestnet.sh
 cargo test
 ```
 
-## Update Documents
-
-if you want update db3 documents , you can follow the steps
-
-### Install Mkdocs
-
-```shell
-pip install mkdocs
-```
-### Document Template
-
-db3 uses https://squidfunk.github.io/mkdocs-material/ as its document framework and you can get started from [here](https://squidfunk.github.io/mkdocs-material/getting-started/)
-
-### Serve the docs
-
-```shell
-git clone https://github.com/dbpunk-labs/db3.git
-mkdocs serve
-```
diff --git a/Cargo.toml b/Cargo.toml
@@ -20,5 +20,5 @@ ethers = {git="https://github.com/imotai/ethers-rs", rev="d526191b7972e8cf4412fe
 tonic = {git="https://github.com/hyperium/tonic", rev="ae7580160431cd25c1eecda4c85014ef6ce8d93f"}
 tonic-web = {git="https://github.com/hyperium/tonic", rev="ae7580160431cd25c1eecda4c85014ef6ce8d93f"}
 arweave-rs = {git="https://github.com/imotai/arweave-rs", rev="4bdb736c565dbce0f3ff698f0265e7e781cca1d6"}
-ejdb2rs = {git="https://github.com/imotai/ejdb2rs", rev="c937a085eda043bd854a0ef4d3e7bb63650610d1"}
+ejdb2rs = {git="https://github.com/imotai/ejdb2rs", rev="dfe9a00c66986c072b8375ce68086b9c98ded4fe"}
 serde_json= "1.0"
diff --git a/README.md b/README.md
@@ -26,9 +26,10 @@ DB3 Network is a Lightweight, *Permanent* JSON document database for Web3. It is
 ## Set up self-hosted Node
 
 **1. Start docker container**
+
 ```shell
 sudo docker run -p 26639:26639 -p 26619:26619 -p 26629:26629 \
-                -e ADMIN_ADDR=0xF78c...29679 \ 
+                -e ADMIN_ADDR=0xF78c...29679 \ # use your own wallet address
                 -it ghcr.io/dbpunk-labs/db3:latest 
 ```
 you should see the following output

diff --git a/sdk/package.json b/sdk/package.json
@@ -1,6 +1,6 @@
 {
     "name": "db3.js",
-    "version": "0.3.15",
+    "version": "0.3.16",
     "description": "DB3 Network Javascript API",
     "author": "dbpunk labs",
     "keywords": [

diff --git a/sdk/src/client/client_v2.ts b/sdk/src/client/client_v2.ts
@@ -26,13 +26,39 @@ import {
 } from '../proto/db3_mutation_v2'
 import type { DocumentData, DocumentEntry } from './base'
 import type { DB3Account } from '../account/types'
-import type { Client } from './types'
+import type { Client, ReadClient } from './types'
 import { Index } from '../proto/db3_database_v2'
 import { StorageProviderV2 } from '../provider/storage_provider_v2'
 import { IndexerProvider } from '../provider/indexer_provider'
 import { fromHEX } from '../crypto/crypto_utils'
 import { BSON } from 'db3-bson'
 
+/**
+ *
+ * Create a readonly client of db3 network
+ *
+ * ```ts
+ * const client = createReadonlyClient("http://127.0.0.1:26619",
+ *                                     "http://127.0.0.1:26620")
+ * ```
+ *
+ * @param rollup_node_url  - the url of db3 rollup node
+ * @param index_node_url   - the url of db3 index node
+ * @returns the client instance
+ *
+ **/
+
+export function createReadonlyClient(
+    rollupNodeUrl: string,
+    indexNodeUrl: string
+) {
+    const provider = new StorageProviderV2(rollupNodeUrl)
+    const indexer = new IndexerProvider(indexNodeUrl)
+    return {
+        provider,
+        indexer,
+    } as ReadClient
+}
 /**
  *
  * Create a client of db3 network

diff --git a/sdk/src/client/types.ts b/sdk/src/client/types.ts
@@ -26,3 +26,8 @@ export type Client = {
     account: DB3Account
     nonce: number
 }
+
+export type ReadClient = {
+    provider: StorageProviderV2
+    indexer: IndexerProvider
+}
diff --git a/sdk/src/index.ts b/sdk/src/index.ts
@@ -21,10 +21,11 @@ export {
     createFromExternal,
 } from './account/db3_account'
 export type { DB3Account } from './account/types'
-export type { Client } from './client/types'
+export type { Client, ReadClient } from './client/types'
 export { addDoc, updateDoc, deleteDoc, queryDoc } from './store/document_v2'
 export {
     createClient,
+    createReadonlyClient,
     syncAccountNonce,
     getMutationHeader,
     getMutationBody,
@@ -35,7 +36,7 @@ export {
     getIndexNodeStatus,
     setupStorageNode,
     getContractSyncStatus,
-    getMutationState
+    getMutationState,
 } from './client/client_v2'
 
 export {
@@ -49,6 +50,10 @@ export {
 } from './store/database_v2'
 
 export { Index, IndexType } from './proto/db3_database_v2'
-export { MutationAction, MutationHeader, MutationBody } from './proto/db3_mutation_v2'
+export {
+    MutationAction,
+    MutationHeader,
+    MutationBody,
+} from './proto/db3_mutation_v2'
 export { MutationStateView } from './proto/db3_storage'
 export { db3MetaStoreContractConfig } from './abi/metastore_abi'
diff --git a/sdk/src/provider/storage_provider_v2.ts b/sdk/src/provider/storage_provider_v2.ts
@@ -49,7 +49,7 @@ export class StorageProviderV2 {
     /**
      * new a storage provider with db3 storage grpc url
      */
-    constructor(url: string, account: DB3Account | undefined) {
+    constructor(url: string, account?: DB3Account) {
         const goptions: GrpcWebOptions = {
             baseUrl: url,
             // simple example for how to add auth headers to each request
@@ -91,10 +91,10 @@ export class StorageProviderV2 {
     }
 
     async getNonce() {
-        const request: GetNonceRequest = {
-            address: this.account.address,
-        }
         try {
+            const request: GetNonceRequest = {
+                address: this.account.address,
+            }
             const { response } = await this.client.getNonce(request)
             return response.nonce
         } catch (e) {
@@ -106,9 +106,8 @@ export class StorageProviderV2 {
      * send mutation to db3 network
      */
     async sendMutation(mutation: Uint8Array, nonce: string) {
-        const request = await this.wrapTypedRequest(mutation, nonce)
-
         try {
+            const request = await this.wrapTypedRequest(mutation, nonce)
             const { response } = await this.client.sendMutation(request)
             return response
         } catch (e) {

diff --git a/sdk/src/store/database_v2.ts b/sdk/src/store/database_v2.ts
@@ -203,18 +203,11 @@ export async function getDatabase(addr: string, client: Client) {
     if (!db) {
         throw new Error('db with addr ' + addr + ' does not exist')
     }
-    if (db.database.oneofKind === 'docDb') {
-        return {
-            addr,
-            client,
-            internal: db,
-        }
-    } else {
-        return {
-            addr,
-            client,
-            internal: db,
-        }
+    return {
+        addr,
+        client,
+        internal: db,
+        state: response.state,
     }
 }
 
@@ -234,25 +227,28 @@ export async function showDatabase(owner: string, client: Client) {
     const response = await client.provider.getDatabaseOfOwner(owner)
     return response.databases
         .filter((item) => item.database.oneofKind != undefined)
-        .map((db) => {
+        .map((db, index) => {
             if (db.database.oneofKind === 'docDb') {
                 return {
                     addr: '0x' + toHEX(db.database.docDb.address),
                     client,
                     internal: db,
+                    state: response.states[index],
                 }
             } else if (db.database.oneofKind === 'eventDb') {
                 return {
                     addr: '0x' + toHEX(db.database.eventDb.address),
                     client,
                     internal: db,
+                    state: response.states[index],
                 }
             } else {
                 //will not go here
                 return {
                     addr: '',
                     client,
                     internal: undefined,
+                    state: response.states[index],
                 }
             }
         })
@@ -280,10 +276,10 @@ export async function showDatabase(owner: string, client: Client) {
 export async function createCollection(
     db: Database,
     name: string,
-    indexFields: Index[]
+    indexFields?: Index[]
 ) {
     const collection: CollectionMutation = {
-        indexFields,
+        indexFields: indexFields ? indexFields : [],
         collectionName: name,
     }
     const body: Mutation_BodyWrapper = {
@@ -310,6 +306,7 @@ export async function createCollection(
             db,
             indexFields,
             internal: undefined,
+            state: undefined,
         }
 
         const result: MutationResult = {
@@ -341,12 +338,13 @@ export async function createCollection(
  **/
 export async function showCollection(db: Database) {
     const response = await db.client.provider.getCollectionOfDatabase(db.addr)
-    const collectionList = response.collections.map((c) => {
+    const collectionList = response.collections.map((c, index) => {
         return {
             name: c.name,
             db,
             indexFields: c.indexFields,
             internal: c,
+            state: response.states[index],
         } as Collection
     })
     return collectionList

diff --git a/sdk/src/store/document_v2.ts b/sdk/src/store/document_v2.ts
@@ -43,17 +43,52 @@ async function runQueryInternal<T>(col: Collection, query: Query) {
     return {
         docs: entries,
         collection: col,
+        count: response.count,
     } as QueryResult<T>
 }
 
 /**
  *
  * Query document with a query language
  *
+ * The usage of query language
+ * for example we have a document looks like the following
+ * ```json
+ *     {
+ *     "firstName": "John",
+ *     "lastName": "Doe",
+ *     "age": 28,
+ *     "pets": [
+ *       {"name": "Rexy rex", "kind": "dog", "likes": ["bones", "jumping", "toys"]},
+ *       {"name": "Grenny", "kind": "parrot", "likes": ["green color", "night", "toys"]}
+ *     ]
+ *    }
+ * ```
+ * 1. Query one document from collection
  * ```ts
  * const queryStr = '/* | limit 1'
  * const resultSet = await queryDoc<Profile>(collection, queryStr)
  * ```
+ * 2. Query documents with filter
+ * ```ts
+ * const queryByName = '/[firstname="John"]'
+ * const resultSet = await queryDoc<Profile>(collection, queryByName)
+ * const queryByFirstAndLast = '/[firstName="John"] and [lastName="Doe"]'
+ * const resultSet = await queryDoc<Profile>(collection, queryByName)
+ * ```
+ * 3. Query documents with filter and projection
+ * ```ts
+ * // only query the firstName and lastName
+ * const queryByName = '/[firstname="John"] | / {firstName, lastName}'
+ * const resultSet = await queryDoc<Profile>(collection, queryByName)
+ * ```
+ * 4. Query documents with filter and aggregate count
+ * ```ts
+ * // only query the firstName and lastName
+ * const queryByNameAndCount = '/[firstname="John"] | count'
+ * const resultSet = await queryDoc<Profile>(collection, queryByNameAndCount)
+ * ```
+ *
  * @param col        - the instance of collection
  * @param queryStr   - a document query string
  * @param parameters - an optional query parameters
@@ -159,6 +194,7 @@ export async function updateDoc(
         throw new Error('fail to update doc')
     }
 }
+
 export async function addDoc(col: Collection, doc: DocumentData) {
     const documentMutation: DocumentMutation = {
         collectionName: col.name,

diff --git a/sdk/src/store/types.ts b/sdk/src/store/types.ts
@@ -21,6 +21,8 @@ import {
     DatabaseMessage as InternalDatabase,
     Index,
     Collection as InternalCollection,
+    DatabaseState,
+    CollectionState,
 } from '../proto/db3_database_v2'
 
 export type CreateDBResult = {
@@ -37,6 +39,7 @@ export type Database = {
     addr: string
     client: Client
     internal: InternalDatabase | undefined
+    state: DatabaseState | undefined
 }
 
 export type MutationResult = {
@@ -50,11 +53,13 @@ export type Collection = {
     db: Database
     indexFields: Index[]
     internal: InternalCollection | undefined
+    state: CollectionState | undefined
 }
 
 export type QueryResult<T = DocumentData> = {
     docs: Array<DocumentEntry<T>>
     collection: Collection
+    count: string
 }
 
 export type EventDatabaseOption = {