feat: add caching layer for children blocks

migrations/20230122163241_add-children-cache.js

@@ -0,0 +1,23 @@
+/**
+ * @param { import("knex").Knex } knex
+ * @returns {Knex.SchemaBuilder}
+ */
+exports.up = function (knex) {
+  return knex.schema.createTable('blocks', function (table) {
+    table.increments('id').unique().primary();
+    table.string('owner', 255).notNullable();
+    table.string('object_id', 255).unique().notNullable();
+    table.json('payload').notNullable();
+    table.integer('fetch').notNullable().defaultTo(0);
+    table.timestamp('created_at').notNullable();
+    table.timestamp('last_edited_time').notNullable();
+  });
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns {Knex.SchemaBuilder}
+ */
+exports.down = function (knex) {
+  return knex.schema.dropTable('blocks');
+};

package.json

@@ -41,6 +41,7 @@
     "cheerio": "^1.0.0-rc.12",
     "cookie-parser": "^1.4.6",
     "crypto-js": "^4.1.1",
+    "date-fns": "^2.29.3",
     "dotenv": "^16.0.1",
     "express": "^4.18.1",
     "fflate": "^0.7.4",

src/lib/notion/BlockHandler/BlockHandler.test.ts

@@ -14,7 +14,7 @@ import MockNotionAPI from '../_mock/MockNotionAPI';
 import { getToggleBlocks } from '../helpers/getToggleBlocks';
 
 dotenv.config({ path: 'test/.env' });
-const api = new MockNotionAPI(process.env.NOTION_KEY!);
+const api = new MockNotionAPI(process.env.NOTION_KEY!, '3');
 
 type Options = { [key: string]: string };
 
@@ -64,10 +64,12 @@ describe('BlockHandler', () => {
 
   test('Get Blocks', async () => {
     // This should be mocked
-    const blocks = await api.getBlocks(
-      '07a7b319183642b9afecdcc4c456f73d',
-      true
-    );
+    const blocks = await api.getBlocks({
+      createdAt: '',
+      lastEditedAt: '',
+      id: '07a7b319183642b9afecdcc4c456f73d',
+      all: true,
+    });
     const topLevelToggles = getToggleBlocks(blocks.results);
     expect(topLevelToggles.length).toEqual(14);
   });

src/lib/notion/BlockHandler/BlockHandler.ts

@@ -6,6 +6,7 @@ import {
   GetBlockResponse,
   ImageBlockObjectResponse,
   ListBlockChildrenResponse,
+  PageObjectResponse,
 } from '@notionhq/client/build/src/api-endpoints';
 import axios from 'axios';
 import NotionAPIWrapper from '../NotionAPIWrapper';
@@ -117,12 +118,17 @@ class BlockHandler {
    * @returns
    */
   async getBackSide(
-    block: GetBlockResponse,
+    block: BlockObjectResponse,
     handleChildren?: boolean
   ): Promise<string | null> {
     let response2: ListBlockChildrenResponse | null;
     try {
-      response2 = await this.api.getBlocks(block.id, this.useAll);
+      response2 = await this.api.getBlocks({
+        createdAt: block.created_time,
+        lastEditedAt: block.last_edited_time,
+        id: block.id,
+        all: this.useAll,
+      });
       const requestChildren = response2.results;
       return await renderBack(this, requestChildren, response2, handleChildren);
     } catch (e: unknown) {
@@ -165,7 +171,7 @@ class BlockHandler {
           );
         }
       } else {
-        back = await this.getBackSide(block);
+        back = await this.getBackSide(block as BlockObjectResponse);
       }
       if (!name) {
         console.debug('name is not valid for front, skipping', name, back);
@@ -248,7 +254,7 @@ class BlockHandler {
     } else if (parentType === 'database') {
       const dbResult = await this.api.queryDatabase(topLevelId);
       const database = await this.api.getDatabase(topLevelId);
-      const dbName = this.api.getDatabaseTitle(database, this.settings);
+      const dbName = await this.api.getDatabaseTitle(database, this.settings);
       let dbDecks = [];
       for (const entry of dbResult.results) {
         dbDecks = await this.findFlashcardsFromPage({
@@ -276,12 +282,17 @@ class BlockHandler {
     const { topLevelId, rules, parentName, parentType } = locator;
     let { decks } = locator;
 
+    const page = await this.api.getPage(topLevelId);
     const tags = await this.api.getTopLevelTags(topLevelId, rules);
-    const response = await this.api.getBlocks(topLevelId, rules.UNLIMITED);
+    const response = await this.api.getBlocks({
+      createdAt: (page as PageObjectResponse).created_time,
+      lastEditedAt: (page as PageObjectResponse).last_edited_time,
+      id: topLevelId,
+      all: rules.UNLIMITED,
+    });
     const blocks = response.results;
     const flashCardTypes = rules.flaschardTypeNames();
 
-    const page = await this.api.getPage(topLevelId);
     const title = await this.api.getPageTitle(page, this.settings);
     if (!this.firstPageTitle) {
       this.firstPageTitle = title;
@@ -332,7 +343,12 @@ class BlockHandler {
         if (isFullBlock(sd)) {
           const subDeckType = sd.type;
           console.log('sd.type', subDeckType);
-          const res = await this.api.getBlocks(sd.id, rules.UNLIMITED);
+          const res = await this.api.getBlocks({
+            createdAt: sd.created_time,
+            lastEditedAt: sd.last_edited_time,
+            id: sd.id,
+            all: rules.UNLIMITED,
+          });
           const cBlocks = res.results.filter((b: GetBlockResponse) =>
             flashCardTypes.includes((b as BlockObjectResponse).type)
           );

src/lib/notion/NotionAPIWrapper.ts

@@ -17,27 +17,45 @@ import getBlockIcon, { WithIcon } from './blocks/getBlockIcon';
 import { isHeading } from './helpers/isHeading';
 import { getHeadingText } from './helpers/getHeadingText';
 import getObjectTitle from './helpers/getObjectTitle';
+import DB from '../storage/db';
+import { getBlockCache } from './helpers/getBlockCache';
 
 const DEFAULT_PAGE_SIZE_LIMIT = 100 * 2;
+export interface GetBlockParams {
+  createdAt: string;
+  lastEditedAt: string;
+  id: string;
+  all?: boolean;
+}
 
 class NotionAPIWrapper {
   private notion: Client;
 
   page?: GetPageResponse;
 
-  constructor(key: string) {
+  private owner: string;
+
+  constructor(key: string, owner: string) {
     this.notion = new Client({ auth: key });
+    this.owner = owner;
   }
 
   async getPage(id: string): Promise<GetPageResponse | null> {
     return this.notion.pages.retrieve({ page_id: id });
   }
 
-  async getBlocks(
-    id: string,
-    all?: boolean
-  ): Promise<ListBlockChildrenResponse> {
+  async getBlocks({
+    createdAt,
+    lastEditedAt,
+    id,
+    all,
+  }: GetBlockParams): Promise<ListBlockChildrenResponse> {
     console.log('getBlocks', id, all);
+    const cachedPayload = await getBlockCache(id, this.owner, lastEditedAt);
+    if (cachedPayload) {
+      console.log('using payload cache');
+      return cachedPayload;
+    }
     const response = await this.notion.blocks.children.list({
       block_id: id,
       page_size: DEFAULT_PAGE_SIZE_LIMIT,
@@ -61,6 +79,18 @@ class NotionAPIWrapper {
         }
       }
     }
+    if (!createdAt || !lastEditedAt) {
+      console.log('not enough input block cache');
+    } else {
+      await DB('blocks').insert({
+        owner: this.owner,
+        object_id: id,
+        payload: JSON.stringify(response),
+        fetch: 1,
+        created_at: createdAt,
+        last_edited_time: lastEditedAt,
+      });
+    }
     return response;
   }
 
@@ -157,8 +187,14 @@ class NotionAPIWrapper {
   }
 
   async getTopLevelTags(pageId: string, rules: ParserRules) {
+    console.info('[NO_CACHE] - getTopLevelTags');
     const useHeadings = rules.TAGS === 'heading';
-    const response = await this.getBlocks(pageId, rules.UNLIMITED);
+    const response = await this.getBlocks({
+      createdAt: '',
+      lastEditedAt: '',
+      id: pageId,
+      all: rules.UNLIMITED,
+    });
     const globalTags = [];
     if (useHeadings) {
       const headings = response.results.filter((block) => isHeading(block));
@@ -195,6 +231,17 @@ class NotionAPIWrapper {
     return sanitizeTags(globalTags);
   }
 
+  async getBlockTitle(icon: string | null, title: string, settings: Settings) {
+    if (!icon) {
+      return title;
+    }
+
+    // the order here matters due to icon not being set and last not being default
+    return settings.pageEmoji !== 'last_emoji'
+      ? `${icon}${title}`
+      : `${title}${icon}`;
+  }
+
   async getPageTitle(
     page: GetPageResponse | null,
     settings: Settings
@@ -204,24 +251,20 @@ class NotionAPIWrapper {
     }
     let title = getObjectTitle(page) ?? `Untitled: ${new Date()}`;
     let icon = renderIcon(getBlockIcon(page as WithIcon, settings.pageEmoji));
-
-    // the order here matters due to icon not being set and last not being default
-    return settings.pageEmoji !== 'last_emoji'
-      ? `${icon}${title}`
-      : `${title}${icon}`;
+    return this.getBlockTitle(icon, title, settings);
   }
 
-  getDatabaseTitle(database: GetDatabaseResponse, settings: Settings): string {
+  getDatabaseTitle(
+    database: GetDatabaseResponse,
+    settings: Settings
+  ): Promise<string> {
     let icon = renderIcon(
       getBlockIcon(database as WithIcon, settings.pageEmoji)
     );
     let title = isFullDatabase(database)
       ? database.title.map((t) => t.plain_text).join('')
       : '';
-
-    return settings.pageEmoji !== 'last_emoji'
-      ? `${icon}${title}`
-      : `${title}${icon}`;
+    return this.getBlockTitle(icon, title, settings);
   }
 }
 

src/lib/notion/_mock/MockNotionAPI.ts

@@ -4,23 +4,28 @@ import {
   ListBlockChildrenResponse,
   QueryDatabaseResponse,
 } from '@notionhq/client/build/src/api-endpoints';
-import NotionAPIWrapper from '../NotionAPIWrapper';
+import NotionAPIWrapper, { GetBlockParams } from '../NotionAPIWrapper';
 import dataMockPath from './helpers/dataMockPath';
 import { mockDataExists } from './helpers/mockDataExists';
 import getPayload from './helpers/getPayload';
 import savePayload from './helpers/savePayload';
 
 export default class MockNotionAPI extends NotionAPIWrapper {
-  async getBlocks(
-    id: string,
-    all?: boolean
-  ): Promise<ListBlockChildrenResponse> {
+  async getBlocks({
+    id,
+    all,
+  }: GetBlockParams): Promise<ListBlockChildrenResponse> {
     if (mockDataExists('ListBlockChildrenResponse', id)) {
       return getPayload(
         dataMockPath('ListBlockChildrenResponse', id)
       ) as ListBlockChildrenResponse;
     }
-    const blocks = await super.getBlocks(id, all);
+    const blocks = await super.getBlocks({
+      createdAt: '',
+      lastEditedAt: '',
+      id,
+      all,
+    });
     savePayload(dataMockPath('ListBlockChildrenResponse', id), blocks);
     return blocks;
   }

src/lib/notion/blocks/getBlockIcon.ts

@@ -2,6 +2,7 @@ export type WithIcon = {
   icon:
     | { emoji: string; type?: 'emoji' }
     | { external: { url: string }; type?: 'external' }
+    | { file: { url: string }; type?: 'file' }
     | null;
 };
 
@@ -15,6 +16,8 @@ export default function getBlockIcon(p?: WithIcon, emoji?: string): string {
       return p.icon.emoji;
     case 'external':
       return p.icon.external.url;
+    case 'file':
+      return p.icon.file.url;
     default:
       return '';
   }

src/lib/notion/helpers/getBlockCache.ts

@@ -0,0 +1,25 @@
+import Blocks from '../../../schemas/public/Blocks';
+import DB from '../../storage/db';
+import isAfter from 'date-fns/isAfter';
+import { ListBlockChildrenResponse } from '@notionhq/client/build/src/api-endpoints';
+
+export async function getBlockCache(
+  id: string,
+  owner: string,
+  lastEditedAt: string | Date
+): Promise<ListBlockChildrenResponse | undefined> {
+  const cache: Blocks = await DB('blocks')
+    .where({ object_id: id, owner })
+    .first();
+  // We did not find a cache entry or the user has made changes
+  if (!cache || isAfter(new Date(lastEditedAt), cache.last_edited_time)) {
+    return undefined;
+  }
+  // Found cache and update the fetch request (used for performance analysis)
+  DB('blocks')
+    .where({ object_id: id })
+    .update({
+      fetch: cache.fetch + 1,
+    });
+  return cache.payload as ListBlockChildrenResponse;
+}