ci: merge staging to master

.clang-format

@@ -0,0 +1,2 @@
+BasedOnStyle: Google
+SortIncludes: false

.eslintrc

@@ -50,6 +50,11 @@
         "ignoreConsecutiveComments": true
       }
     ],
+    "curly": [
+      "error",
+      "multi-line",
+      "consistent"
+    ],
     "import/order": [
       "error",
       {
@@ -137,6 +142,10 @@
         "format": ["PascalCase"],
         "trailingUnderscore": "allowSingleOrDouble"
       },
+      {
+        "selector": "enumMember",
+        "format": ["PascalCase", "UPPER_CASE"]
+      },
       {
         "selector": "objectLiteralProperty",
         "format": null

.gitlab-ci.yml

@@ -5,33 +5,41 @@ workflow:
       when: never
     - when: always
 
-default:
-  interruptible: true
-
 variables:
+  GIT_SUBMODULE_STRATEGY: recursive
   GH_PROJECT_PATH: "MatrixAI/${CI_PROJECT_NAME}"
   GH_PROJECT_URL: "https://${GITHUB_TOKEN}@github.com/${GH_PROJECT_PATH}.git"
-  GIT_SUBMODULE_STRATEGY: recursive
   # Cache .npm
-  NPM_CONFIG_CACHE: "${CI_PROJECT_DIR}/tmp/npm"
+  npm_config_cache: "${CI_PROJECT_DIR}/tmp/npm"
   # Prefer offline node module installation
-  NPM_CONFIG_PREFER_OFFLINE: "true"
+  npm_config_prefer_offline: "true"
   # `ts-node` has its own cache
   TS_CACHED_TRANSPILE_CACHE: "${CI_PROJECT_DIR}/tmp/ts-node-cache"
   TS_CACHED_TRANSPILE_PORTABLE: "true"
   # Homebrew cache only used by macos runner
   HOMEBREW_CACHE: "${CI_PROJECT_DIR}/tmp/Homebrew"
 
+default:
+  interruptible: true
+  before_script:
+    # Replace this in windows runners that use powershell
+    # with `mkdir -Force "$CI_PROJECT_DIR/tmp"`
+    - mkdir -p "$CI_PROJECT_DIR/tmp"
+
 # Cached directories shared between jobs & pipelines per-branch per-runner
 cache:
   key: $CI_COMMIT_REF_SLUG
+  # Preserve cache even if job fails
+  when: 'always'
   paths:
     - ./tmp/npm/
     - ./tmp/ts-node-cache/
     # Homebrew cache is only used by the macos runner
     - ./tmp/Homebrew
     # `jest` cache is configured in jest.config.js
     - ./tmp/jest/
+    # `node-gyp` cache used by windows and macos
+    - ./tmp/node-gyp
 
 stages:
   - check       # Linting, unit tests
@@ -48,6 +56,7 @@ check:lint:
     - >
         nix-shell --run '
         npm run lint;
+        npm run lint-native;
         '
   rules:
     # Runs on feature and staging commits and ignores version commits
@@ -65,13 +74,17 @@ check:test:
     - >
         nix-shell --run '
         npm run build --verbose;
-        npm test -- --ci;
+        npm test -- --ci --coverage;
         '
   artifacts:
     when: always
     reports:
       junit:
-        - ./tmp/junit.xml
+        - ./tmp/junit/junit.xml
+      coverage_report:
+        coverage_format: cobertura
+        path: ./tmp/coverage/cobertura-coverage.xml
+  coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
   rules:
     # Runs on staging commits and ignores version commits
     - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
@@ -110,21 +123,28 @@ build:merge:
 build:linux:
   stage: build
   needs: []
+  variables:
+    # Only x64 architecture is needed
+    npm_config_arch: "x64"
   script:
     - >
         nix-shell --run '
         npm run build --verbose;
-        npm test -- --ci;
+        npm test -- --ci --coverage;
         '
   artifacts:
     when: always
     reports:
       junit:
-        - ./tmp/junit.xml
+        - ./tmp/junit/junit.xml
+      coverage_report:
+        coverage_format: cobertura
+        path: ./tmp/coverage/cobertura-coverage.xml
     paths:
       - ./prebuilds/
       # Only the build:linux preserves the dist
       - ./dist
+  coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
   rules:
     # Runs on staging commits and ignores version commits
     - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
@@ -136,12 +156,17 @@ build:windows:
   needs: []
   tags:
     - windows
+  variables:
+    # Location node-gyp installed headers and static libraries
+    npm_config_devdir: "${CI_PROJECT_DIR}/tmp/node-gyp"
+    # Only x64 architecture is needed
+    npm_config_arch: "x64"
   before_script:
-    - choco install nodejs --version=16.14.2 -y
+    - mkdir -Force "$CI_PROJECT_DIR/tmp"
+    - choco install nodejs --version="16.15.1" -y
     - choco install python --version=3.9.12 -y
     - refreshenv
   script:
-    - npm config set msvs_version 2019
     - npm install --ignore-scripts
     - $env:Path = "$(npm bin);" + $env:Path
     - npm run build --verbose
@@ -150,7 +175,7 @@ build:windows:
     when: always
     reports:
       junit:
-        - ./tmp/junit.xml
+        - ./tmp/junit/junit.xml
     paths:
       - ./prebuilds/
   rules:
@@ -168,7 +193,12 @@ build:macos:
   variables:
     HOMEBREW_NO_INSTALL_UPGRADE: "true"
     HOMEBREW_NO_INSTALL_CLEANUP: "true"
+    # Location node-gyp installed headers and static libraries
+    npm_config_devdir: "${CI_PROJECT_DIR}/tmp/node-gyp"
+    # Produce universal binary
+    npm_config_arch: 'x64+arm64'
   before_script:
+    - eval "$(brew shellenv)"
     - brew install node@16
     - brew link --overwrite node@16
     - brew install python@3.9
@@ -177,14 +207,13 @@ build:macos:
   script:
     - npm install --ignore-scripts
     - export PATH="$(npm bin):$PATH"
-    - export PREBUILD_ARCH=x64+arm64
     - npm run build --verbose
     - npm test -- --ci
   artifacts:
     when: always
     reports:
       junit:
-        - ./tmp/junit.xml
+        - ./tmp/junit/junit.xml
     paths:
       - ./prebuilds/
   rules:
@@ -209,6 +238,26 @@ build:prerelease:
         nix-shell --run '
         npm publish --tag prerelease --access public;
         '
+    - >
+      for d in prebuilds/*; do
+        tar \
+          --create \
+          --verbose \
+          --file="prebuilds/$(basename $d).tar" \
+          --directory=prebuilds \
+          "$(basename $d)";
+      done
+    - >
+        nix-shell -I nixpkgs=./pkgs.nix --packages gitAndTools.gh --run '
+        gh release \
+          create "$CI_COMMIT_TAG" \
+          prebuilds/*.tar \
+          --title "${CI_COMMIT_TAG}-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+          --notes "" \
+          --prerelease \
+          --target staging \
+          --repo "$GH_PROJECT_PATH";
+        '
   after_script:
     - rm -f ./.npmrc
   rules:
@@ -272,6 +321,25 @@ release:distribution:
         nix-shell --run '
         npm publish --access public;
         '
+    - >
+      for d in prebuilds/*; do
+        tar \
+          --create \
+          --verbose \
+          --file="prebuilds/$(basename $d).tar" \
+          --directory=prebuilds \
+          "$(basename $d)";
+      done
+    - >
+        nix-shell -I nixpkgs=./pkgs.nix --packages gitAndTools.gh --run '
+        gh release \
+          create "$CI_COMMIT_TAG" \
+          prebuilds/*.tar \
+          --title "${CI_COMMIT_TAG}-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+          --notes "" \
+          --target master \
+          --repo "$GH_PROJECT_PATH";
+        '
   after_script:
     - rm -f ./.npmrc
   rules:

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "deps/snappy/snappy"]
+	path = deps/snappy/snappy
+	url = https://github.com/google/snappy.git
+[submodule "deps/rocksdb/rocksdb"]
+	path = deps/rocksdb/rocksdb
+	url = https://github.com/facebook/rocksdb.git

.npmrc

@@ -0,0 +1,2 @@
+# Enables npm link
+prefix=~/.npm

README.md

@@ -5,12 +5,76 @@ master: [![pipeline status](https://gitlab.com/MatrixAI/open-source/js-db/badges
 
 DB is library managing key value state for MatrixAI's JavaScript/TypeScript applications.
 
+This forks classic-level's C++ binding code around LevelDB 1.20. Differences from classic-level:
+
+* Uses TypeScript from ground-up
+* Supports Snapshot-Isolation based transactions via `DBTransaction`
+* API supports "key paths" which can be used to manipulate "levels" of nested keys
+* Value encryption (key-encryption is not supported yet) - requires additional work with block-encryption
+
 ## Installation
 
 ```sh
 npm install --save @matrixai/db
 ```
 
+## Usage
+
+
+```ts
+import { DB } from '@matrixai/db';
+
+async function main () {
+
+  const key = Buffer.from([
+    0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03,
+    0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x02, 0x03,
+  ]);
+
+  const encrypt = async (
+    key: ArrayBuffer,
+    plainText: ArrayBuffer
+  ): Promise<ArrayBuffer> {
+    return plainText;
+  };
+
+  const decrypt = async (
+    key: ArrayBuffer,
+    cipherText: ArrayBuffer
+  ): Promise<ArrayBuffer | undefined> {
+    return cipherText;
+  }
+
+  const db = await DB.createDB({
+    dbPath: './tmp/db',
+    crypto: {
+      key,
+      ops: { encrypt, decrypt },
+    },
+    fresh: true,
+  });
+
+  await db.put(['level', Buffer.from([0x30, 0x30]), 'a'], 'value');
+  await db.put(['level', Buffer.from([0x30, 0x31]), 'b'], 'value');
+  await db.put(['level', Buffer.from([0x30, 0x32]), 'c'], 'value');
+  await db.put(['level', Buffer.from([0x30, 0x33]), 'c'], 'value');
+
+  console.log(await db.get(['level', Buffer.from([0x30, 0x32]), 'c']));
+
+  await db.del(['level', Buffer.from([0x30, 0x32]), 'c']);
+
+  for await (const [kP, v] of db.iterator({
+    lt: [Buffer.from([0x30, 0x32]), ''],
+  }, ['level'])) {
+    console.log(kP, v);
+  }
+
+  await db.stop();
+}
+
+main();
+```
+
 ## Development
 
 Run `nix-shell`, and once you're inside, you can use:

benches/DB1KiB.ts

@@ -58,9 +58,7 @@ async function main() {
 }
 
 if (require.main === module) {
-  (async () => {
-    await main();
-  })();
+  void main();
 }
 
 export default main;

benches/DB1MiB.ts

@@ -20,13 +20,13 @@ async function main() {
   const summary = await b.suite(
     'DB1MiB',
     b.add('get 1 MiB of data', async () => {
-      await db.put('1kib', data1MiB, true);
+      await db.put('1mib', data1MiB, true);
       return async () => {
-        await db.get('1kib', true);
+        await db.get('1mib', true);
       };
     }),
     b.add('put 1 MiB of data', async () => {
-      await db.put('1kib', data1MiB, true);
+      await db.put('1mib', data1MiB, true);
     }),
     b.add('put zero data', async () => {
       await db.put('0', data0, true);
@@ -58,9 +58,7 @@ async function main() {
 }
 
 if (require.main === module) {
-  (async () => {
-    await main();
-  })();
+  void main();
 }
 
 export default main;

benches/index.ts

@@ -20,9 +20,7 @@ async function main(): Promise<void> {
 }
 
 if (require.main === module) {
-  (async () => {
-    await main();
-  })();
+  void main();
 }
 
 export default main;