From f97a14efb213bbe1754cd67db815480b42efc17a Mon Sep 17 00:00:00 2001 From: Alexander Alemayhu Date: Sun, 22 Jan 2023 19:58:24 +0100 Subject: [PATCH] feat: add caching layer for children blocks --- .../20230122163241_add-children-cache.js | 23 ++++++ package-lock.json | 18 +++++ package.json | 1 + .../notion/BlockHandler/BlockHandler.test.ts | 12 +-- src/lib/notion/BlockHandler/BlockHandler.ts | 30 ++++++-- src/lib/notion/NotionAPIWrapper.ts | 75 +++++++++++++++---- src/lib/notion/_mock/MockNotionAPI.ts | 17 +++-- src/lib/notion/blocks/getBlockIcon.ts | 3 + src/lib/notion/helpers/getBlockCache.ts | 25 +++++++ src/lib/notion/helpers/getColumn.ts | 7 +- src/lib/notion/helpers/getNotionAPI.ts | 2 +- .../jobs/helpers/notifyUserIfNecessary.ts | 2 +- src/routes/notion/getBlocks.ts | 8 +- src/routes/notion/renderBlock.ts | 2 +- src/routes/upload/deleteUpload.ts | 5 +- src/routes/upload/purgeBlockCache.ts | 4 + src/routes/users/index.ts | 24 +++--- src/schemas/public/Blocks.ts | 58 ++++++++++++++ src/server.ts | 11 +-- 19 files changed, 272 insertions(+), 55 deletions(-) create mode 100644 migrations/20230122163241_add-children-cache.js create mode 100644 src/lib/notion/helpers/getBlockCache.ts create mode 100644 src/routes/upload/purgeBlockCache.ts create mode 100644 src/schemas/public/Blocks.ts diff --git a/migrations/20230122163241_add-children-cache.js b/migrations/20230122163241_add-children-cache.js new file mode 100644 index 000000000..034739891 --- /dev/null +++ b/migrations/20230122163241_add-children-cache.js @@ -0,0 +1,23 @@ +/** + * @param { import("knex").Knex } knex + * @returns {Knex.SchemaBuilder} + */ +exports.up = function (knex) { + return knex.schema.createTable('blocks', function (table) { + table.increments('id').unique().primary(); + table.string('owner', 255).notNullable(); + table.string('object_id', 255).unique().notNullable(); + table.json('payload').notNullable(); + table.integer('fetch').notNullable().defaultTo(0); + table.timestamp('created_at').notNullable(); + table.timestamp('last_edited_time').notNullable(); + }); +}; + +/** + * @param { import("knex").Knex } knex + * @returns {Knex.SchemaBuilder} + */ +exports.down = function (knex) { + return knex.schema.dropTable('blocks'); +}; diff --git a/package-lock.json b/package-lock.json index d727ff16f..fc0db3cc3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "cheerio": "^1.0.0-rc.12", "cookie-parser": "^1.4.6", "crypto-js": "^4.1.1", + "date-fns": "^2.29.3", "dotenv": "^16.0.1", "express": "^4.18.1", "fflate": "^0.7.4", @@ -3601,6 +3602,18 @@ "node": ">=12" } }, + "node_modules/date-fns": { + "version": "2.29.3", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.29.3.tgz", + "integrity": "sha512-dDCnyH2WnnKusqvZZ6+jA1O51Ibt8ZMRNkDZdyAyK4YfbDwa/cEmuztzG5pk6hqlp9aSBPYcjOlktquahGwGeA==", + "engines": { + "node": ">=0.11" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/date-fns" + } + }, "node_modules/dayjs": { "version": "1.11.5", "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.5.tgz", @@ -13192,6 +13205,11 @@ } } }, + "date-fns": { + "version": "2.29.3", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.29.3.tgz", + "integrity": "sha512-dDCnyH2WnnKusqvZZ6+jA1O51Ibt8ZMRNkDZdyAyK4YfbDwa/cEmuztzG5pk6hqlp9aSBPYcjOlktquahGwGeA==" + }, "dayjs": { "version": "1.11.5", "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.5.tgz", diff --git a/package.json b/package.json index e520ef3f0..1ccd3c43a 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "cheerio": "^1.0.0-rc.12", "cookie-parser": "^1.4.6", "crypto-js": "^4.1.1", + "date-fns": "^2.29.3", "dotenv": "^16.0.1", "express": "^4.18.1", "fflate": "^0.7.4", diff --git a/src/lib/notion/BlockHandler/BlockHandler.test.ts b/src/lib/notion/BlockHandler/BlockHandler.test.ts index 0a15aa2a6..ac1e291ac 100644 --- a/src/lib/notion/BlockHandler/BlockHandler.test.ts +++ b/src/lib/notion/BlockHandler/BlockHandler.test.ts @@ -14,7 +14,7 @@ import MockNotionAPI from '../_mock/MockNotionAPI'; import { getToggleBlocks } from '../helpers/getToggleBlocks'; dotenv.config({ path: 'test/.env' }); -const api = new MockNotionAPI(process.env.NOTION_KEY!); +const api = new MockNotionAPI(process.env.NOTION_KEY!, '3'); type Options = { [key: string]: string }; @@ -64,10 +64,12 @@ describe('BlockHandler', () => { test('Get Blocks', async () => { // This should be mocked - const blocks = await api.getBlocks( - '07a7b319183642b9afecdcc4c456f73d', - true - ); + const blocks = await api.getBlocks({ + createdAt: '', + lastEditedAt: '', + id: '07a7b319183642b9afecdcc4c456f73d', + all: true, + }); const topLevelToggles = getToggleBlocks(blocks.results); expect(topLevelToggles.length).toEqual(14); }); diff --git a/src/lib/notion/BlockHandler/BlockHandler.ts b/src/lib/notion/BlockHandler/BlockHandler.ts index c717925e5..dc0a1ee1c 100644 --- a/src/lib/notion/BlockHandler/BlockHandler.ts +++ b/src/lib/notion/BlockHandler/BlockHandler.ts @@ -6,6 +6,7 @@ import { GetBlockResponse, ImageBlockObjectResponse, ListBlockChildrenResponse, + PageObjectResponse, } from '@notionhq/client/build/src/api-endpoints'; import axios from 'axios'; import NotionAPIWrapper from '../NotionAPIWrapper'; @@ -117,12 +118,17 @@ class BlockHandler { * @returns */ async getBackSide( - block: GetBlockResponse, + block: BlockObjectResponse, handleChildren?: boolean ): Promise { let response2: ListBlockChildrenResponse | null; try { - response2 = await this.api.getBlocks(block.id, this.useAll); + response2 = await this.api.getBlocks({ + createdAt: block.created_time, + lastEditedAt: block.last_edited_time, + id: block.id, + all: this.useAll, + }); const requestChildren = response2.results; return await renderBack(this, requestChildren, response2, handleChildren); } catch (e: unknown) { @@ -165,7 +171,7 @@ class BlockHandler { ); } } else { - back = await this.getBackSide(block); + back = await this.getBackSide(block as BlockObjectResponse); } if (!name) { console.debug('name is not valid for front, skipping', name, back); @@ -248,7 +254,7 @@ class BlockHandler { } else if (parentType === 'database') { const dbResult = await this.api.queryDatabase(topLevelId); const database = await this.api.getDatabase(topLevelId); - const dbName = this.api.getDatabaseTitle(database, this.settings); + const dbName = await this.api.getDatabaseTitle(database, this.settings); let dbDecks = []; for (const entry of dbResult.results) { dbDecks = await this.findFlashcardsFromPage({ @@ -276,12 +282,17 @@ class BlockHandler { const { topLevelId, rules, parentName, parentType } = locator; let { decks } = locator; + const page = await this.api.getPage(topLevelId); const tags = await this.api.getTopLevelTags(topLevelId, rules); - const response = await this.api.getBlocks(topLevelId, rules.UNLIMITED); + const response = await this.api.getBlocks({ + createdAt: (page as PageObjectResponse).created_time, + lastEditedAt: (page as PageObjectResponse).last_edited_time, + id: topLevelId, + all: rules.UNLIMITED, + }); const blocks = response.results; const flashCardTypes = rules.flaschardTypeNames(); - const page = await this.api.getPage(topLevelId); const title = await this.api.getPageTitle(page, this.settings); if (!this.firstPageTitle) { this.firstPageTitle = title; @@ -332,7 +343,12 @@ class BlockHandler { if (isFullBlock(sd)) { const subDeckType = sd.type; console.log('sd.type', subDeckType); - const res = await this.api.getBlocks(sd.id, rules.UNLIMITED); + const res = await this.api.getBlocks({ + createdAt: sd.created_time, + lastEditedAt: sd.last_edited_time, + id: sd.id, + all: rules.UNLIMITED, + }); const cBlocks = res.results.filter((b: GetBlockResponse) => flashCardTypes.includes((b as BlockObjectResponse).type) ); diff --git a/src/lib/notion/NotionAPIWrapper.ts b/src/lib/notion/NotionAPIWrapper.ts index b541bcb2a..9491bd7ac 100644 --- a/src/lib/notion/NotionAPIWrapper.ts +++ b/src/lib/notion/NotionAPIWrapper.ts @@ -17,27 +17,45 @@ import getBlockIcon, { WithIcon } from './blocks/getBlockIcon'; import { isHeading } from './helpers/isHeading'; import { getHeadingText } from './helpers/getHeadingText'; import getObjectTitle from './helpers/getObjectTitle'; +import DB from '../storage/db'; +import { getBlockCache } from './helpers/getBlockCache'; const DEFAULT_PAGE_SIZE_LIMIT = 100 * 2; +export interface GetBlockParams { + createdAt: string; + lastEditedAt: string; + id: string; + all?: boolean; +} class NotionAPIWrapper { private notion: Client; page?: GetPageResponse; - constructor(key: string) { + private owner: string; + + constructor(key: string, owner: string) { this.notion = new Client({ auth: key }); + this.owner = owner; } async getPage(id: string): Promise { return this.notion.pages.retrieve({ page_id: id }); } - async getBlocks( - id: string, - all?: boolean - ): Promise { + async getBlocks({ + createdAt, + lastEditedAt, + id, + all, + }: GetBlockParams): Promise { console.log('getBlocks', id, all); + const cachedPayload = await getBlockCache(id, this.owner, lastEditedAt); + if (cachedPayload) { + console.log('using payload cache'); + return cachedPayload; + } const response = await this.notion.blocks.children.list({ block_id: id, page_size: DEFAULT_PAGE_SIZE_LIMIT, @@ -61,6 +79,18 @@ class NotionAPIWrapper { } } } + if (!createdAt || !lastEditedAt) { + console.log('not enough input block cache'); + } else { + await DB('blocks').insert({ + owner: this.owner, + object_id: id, + payload: JSON.stringify(response), + fetch: 1, + created_at: createdAt, + last_edited_time: lastEditedAt, + }); + } return response; } @@ -157,8 +187,14 @@ class NotionAPIWrapper { } async getTopLevelTags(pageId: string, rules: ParserRules) { + console.info('[NO_CACHE] - getTopLevelTags'); const useHeadings = rules.TAGS === 'heading'; - const response = await this.getBlocks(pageId, rules.UNLIMITED); + const response = await this.getBlocks({ + createdAt: '', + lastEditedAt: '', + id: pageId, + all: rules.UNLIMITED, + }); const globalTags = []; if (useHeadings) { const headings = response.results.filter((block) => isHeading(block)); @@ -195,6 +231,17 @@ class NotionAPIWrapper { return sanitizeTags(globalTags); } + async getBlockTitle(icon: string | null, title: string, settings: Settings) { + if (!icon) { + return title; + } + + // the order here matters due to icon not being set and last not being default + return settings.pageEmoji !== 'last_emoji' + ? `${icon}${title}` + : `${title}${icon}`; + } + async getPageTitle( page: GetPageResponse | null, settings: Settings @@ -204,24 +251,20 @@ class NotionAPIWrapper { } let title = getObjectTitle(page) ?? `Untitled: ${new Date()}`; let icon = renderIcon(getBlockIcon(page as WithIcon, settings.pageEmoji)); - - // the order here matters due to icon not being set and last not being default - return settings.pageEmoji !== 'last_emoji' - ? `${icon}${title}` - : `${title}${icon}`; + return this.getBlockTitle(icon, title, settings); } - getDatabaseTitle(database: GetDatabaseResponse, settings: Settings): string { + getDatabaseTitle( + database: GetDatabaseResponse, + settings: Settings + ): Promise { let icon = renderIcon( getBlockIcon(database as WithIcon, settings.pageEmoji) ); let title = isFullDatabase(database) ? database.title.map((t) => t.plain_text).join('') : ''; - - return settings.pageEmoji !== 'last_emoji' - ? `${icon}${title}` - : `${title}${icon}`; + return this.getBlockTitle(icon, title, settings); } } diff --git a/src/lib/notion/_mock/MockNotionAPI.ts b/src/lib/notion/_mock/MockNotionAPI.ts index 38dc5117f..16d1a3a87 100644 --- a/src/lib/notion/_mock/MockNotionAPI.ts +++ b/src/lib/notion/_mock/MockNotionAPI.ts @@ -4,23 +4,28 @@ import { ListBlockChildrenResponse, QueryDatabaseResponse, } from '@notionhq/client/build/src/api-endpoints'; -import NotionAPIWrapper from '../NotionAPIWrapper'; +import NotionAPIWrapper, { GetBlockParams } from '../NotionAPIWrapper'; import dataMockPath from './helpers/dataMockPath'; import { mockDataExists } from './helpers/mockDataExists'; import getPayload from './helpers/getPayload'; import savePayload from './helpers/savePayload'; export default class MockNotionAPI extends NotionAPIWrapper { - async getBlocks( - id: string, - all?: boolean - ): Promise { + async getBlocks({ + id, + all, + }: GetBlockParams): Promise { if (mockDataExists('ListBlockChildrenResponse', id)) { return getPayload( dataMockPath('ListBlockChildrenResponse', id) ) as ListBlockChildrenResponse; } - const blocks = await super.getBlocks(id, all); + const blocks = await super.getBlocks({ + createdAt: '', + lastEditedAt: '', + id, + all, + }); savePayload(dataMockPath('ListBlockChildrenResponse', id), blocks); return blocks; } diff --git a/src/lib/notion/blocks/getBlockIcon.ts b/src/lib/notion/blocks/getBlockIcon.ts index d99a53195..a097a4b0a 100644 --- a/src/lib/notion/blocks/getBlockIcon.ts +++ b/src/lib/notion/blocks/getBlockIcon.ts @@ -2,6 +2,7 @@ export type WithIcon = { icon: | { emoji: string; type?: 'emoji' } | { external: { url: string }; type?: 'external' } + | { file: { url: string }; type?: 'file' } | null; }; @@ -15,6 +16,8 @@ export default function getBlockIcon(p?: WithIcon, emoji?: string): string { return p.icon.emoji; case 'external': return p.icon.external.url; + case 'file': + return p.icon.file.url; default: return ''; } diff --git a/src/lib/notion/helpers/getBlockCache.ts b/src/lib/notion/helpers/getBlockCache.ts new file mode 100644 index 000000000..aea34ec9e --- /dev/null +++ b/src/lib/notion/helpers/getBlockCache.ts @@ -0,0 +1,25 @@ +import Blocks from '../../../schemas/public/Blocks'; +import DB from '../../storage/db'; +import isAfter from 'date-fns/isAfter'; +import { ListBlockChildrenResponse } from '@notionhq/client/build/src/api-endpoints'; + +export async function getBlockCache( + id: string, + owner: string, + lastEditedAt: string | Date +): Promise { + const cache: Blocks = await DB('blocks') + .where({ object_id: id, owner }) + .first(); + // We did not find a cache entry or the user has made changes + if (!cache || isAfter(new Date(lastEditedAt), cache.last_edited_time)) { + return undefined; + } + // Found cache and update the fetch request (used for performance analysis) + DB('blocks') + .where({ object_id: id }) + .update({ + fetch: cache.fetch + 1, + }); + return cache.payload as ListBlockChildrenResponse; +} diff --git a/src/lib/notion/helpers/getColumn.ts b/src/lib/notion/helpers/getColumn.ts index 0bfe55ffe..814a12baf 100644 --- a/src/lib/notion/helpers/getColumn.ts +++ b/src/lib/notion/helpers/getColumn.ts @@ -6,7 +6,12 @@ export default async function getColumn( handler: BlockHandler, index: number ): Promise { - const getBlocks = await handler.api.getBlocks(parentId); + console.info('[NO_CACHE] - getColumn'); + const getBlocks = await handler.api.getBlocks({ + createdAt: '', + lastEditedAt: '', + id: parentId, + }); const blocks = getBlocks?.results; if (blocks?.length > 0 && blocks?.length >= index + 1) { return blocks[index]; diff --git a/src/lib/notion/helpers/getNotionAPI.ts b/src/lib/notion/helpers/getNotionAPI.ts index df4c1b8c8..67c1a8973 100644 --- a/src/lib/notion/helpers/getNotionAPI.ts +++ b/src/lib/notion/helpers/getNotionAPI.ts @@ -8,5 +8,5 @@ export const getNotionAPI = async ( ): Promise => { console.debug(`Configuring Notion API for ${req.originalUrl}`); const token = await TokenHandler.GetNotionToken(res.locals.owner); - return new NotionAPIWrapper(token!); + return new NotionAPIWrapper(token!, res.locals.owner); }; diff --git a/src/lib/storage/jobs/helpers/notifyUserIfNecessary.ts b/src/lib/storage/jobs/helpers/notifyUserIfNecessary.ts index 044c450c9..2eb0dcf22 100644 --- a/src/lib/storage/jobs/helpers/notifyUserIfNecessary.ts +++ b/src/lib/storage/jobs/helpers/notifyUserIfNecessary.ts @@ -21,7 +21,7 @@ export const notifyUserIfNecessary = async ({ key, apkg, }: JobInfo) => { - console.log('rules.email', rules.EMAIL_NOTIFICATION); + console.debug('rules.email', rules.EMAIL_NOTIFICATION); const email = await getEmailFromOwner(db, owner); if (size > 24) { const link = `${process.env.DOMAIN}/download/u/${key}`; diff --git a/src/routes/notion/getBlocks.ts b/src/routes/notion/getBlocks.ts index b8852ec06..bca308f40 100644 --- a/src/routes/notion/getBlocks.ts +++ b/src/routes/notion/getBlocks.ts @@ -6,10 +6,16 @@ export default async function getBlocks( req: express.Request, res: express.Response ) { + console.info('[NO_CACHE] - getBlocks'); const { id } = req.params; if (!id) { return res.status(400).send(); } - const blocks = await api.getBlocks(id); + const blocks = await api.getBlocks({ + all: true, + createdAt: '', + lastEditedAt: '', + id, + }); res.json(blocks); } diff --git a/src/routes/notion/renderBlock.ts b/src/routes/notion/renderBlock.ts index e53b6cfe7..a8be55880 100644 --- a/src/routes/notion/renderBlock.ts +++ b/src/routes/notion/renderBlock.ts @@ -23,7 +23,7 @@ export default async function renderBlock( api, settings ); - await handler.getBackSide(block, false); + await handler.getBackSide(block as BlockObjectResponse, false); const frontSide = await blockToStaticMarkup( handler, block as BlockObjectResponse diff --git a/src/routes/upload/deleteUpload.ts b/src/routes/upload/deleteUpload.ts index f5ed8a34d..2d04e3b53 100644 --- a/src/routes/upload/deleteUpload.ts +++ b/src/routes/upload/deleteUpload.ts @@ -3,6 +3,7 @@ import { Request, Response } from 'express'; import StorageHandler from '../../lib/storage/StorageHandler'; import DB from '../../lib/storage/db'; import { sendError } from '../../lib/error/sendError'; +import { purgeBlockCache } from './purgeBlockCache'; export default async function deleteUpload(req: Request, res: Response) { const { key } = req.params; @@ -11,7 +12,9 @@ export default async function deleteUpload(req: Request, res: Response) { return res.status(400).send(); } try { - await DB('uploads').del().where({ owner: res.locals.owner, key }); + const owner = res.locals.owner; + await DB('uploads').del().where({ owner, key }); + await purgeBlockCache(owner); const s = new StorageHandler(); await s.deleteWith(key); console.log('done deleting', key); diff --git a/src/routes/upload/purgeBlockCache.ts b/src/routes/upload/purgeBlockCache.ts new file mode 100644 index 000000000..e5397fef1 --- /dev/null +++ b/src/routes/upload/purgeBlockCache.ts @@ -0,0 +1,4 @@ +import DB from '../../lib/storage/db'; + +export const purgeBlockCache = (owner: string) => + DB('blocks').del().where({ owner }); diff --git a/src/routes/users/index.ts b/src/routes/users/index.ts index 8d4ede896..221b0202f 100644 --- a/src/routes/users/index.ts +++ b/src/routes/users/index.ts @@ -195,16 +195,20 @@ router.post('/delete-account', RequireAuthentication, async (req, res) => { if (!owner && req.body.confirmed === true) { return res.status(400).json({}); } - - await DB('access_tokens').where({ owner }).del(); - await DB('favorites').where({ owner }).del(); - await DB('jobs').where({ owner }).del(); - await DB('notion_tokens').where({ owner }).del(); - await DB('parser_rules').where({ owner }).del(); - await DB('patreon_tokens').where({ owner }).del(); - await DB('settings').where({ owner }).del(); - await DB('templates').where({ owner }).del(); - await DB('uploads').where({ owner }).del(); + const ownerTables = [ + 'access_tokens', + 'favorites', + 'jobs', + 'notion_tokens', + 'patreon_tokens', + 'settings', + 'templates', + 'uploads', + 'blocks', + ]; + await Promise.all( + ownerTables.map((tableName) => DB(tableName).where({ owner }).del()) + ); await DB('users').where({ id: owner }).del(); res.status(200).json({}); }); diff --git a/src/schemas/public/Blocks.ts b/src/schemas/public/Blocks.ts new file mode 100644 index 000000000..a405b396b --- /dev/null +++ b/src/schemas/public/Blocks.ts @@ -0,0 +1,58 @@ +// @generated +// This file is automatically generated by Kanel. Do not modify manually. + +/** Identifier type for public.blocks */ +export type BlocksId = number & { __brand: 'BlocksId' }; + +/** Represents the table public.blocks */ +export default interface Blocks { + id: BlocksId; + + owner: string; + + object_id: string; + + payload: unknown; + + fetch: number; + + created_at: Date; + + last_edited_time: Date; +} + +/** Represents the initializer for the table public.blocks */ +export interface BlocksInitializer { + /** Default value: nextval('blocks_id_seq'::regclass) */ + id?: BlocksId; + + owner: string; + + object_id: string; + + payload: unknown; + + /** Default value: 0 */ + fetch?: number; + + created_at: Date; + + last_edited_time: Date; +} + +/** Represents the mutator for the table public.blocks */ +export interface BlocksMutator { + id?: BlocksId; + + owner?: string; + + object_id?: string; + + payload?: unknown; + + fetch?: number; + + created_at?: Date; + + last_edited_time?: Date; +} diff --git a/src/server.ts b/src/server.ts index 17bd85da5..cd5c261e9 100644 --- a/src/server.ts +++ b/src/server.ts @@ -36,7 +36,7 @@ if (existsSync(localEnvFile)) { import MigratorConfig = Knex.MigratorConfig; -function serve() { +async function serve() { const templateDir = path.join(__dirname, 'templates'); const app = express(); @@ -119,15 +119,16 @@ function serve() { ); process.on('uncaughtException', sendError); - DB.raw('SELECT 1').then(() => { - console.info('DB is ready'); - }); + console.info('DB is ready'); + DB.raw('SELECT 1').then(() => {}); const cwd = process.cwd(); if (process.env.MIGRATIONS_DIR) { process.chdir(path.join(process.env.MIGRATIONS_DIR, '..')); } ScheduleCleanup(DB); - DB.migrate.latest(KnexConfig as MigratorConfig).then(() => { + DB.migrate.latest(KnexConfig as MigratorConfig).then(async () => { + // Completed jobs become uploads. Any left during startup means they failed. + await DB.raw("UPDATE jobs SET status = 'failed';"); process.chdir(cwd); process.env.SECRET ||= 'victory'; const port = process.env.PORT || 2020;