PB data migration (#116)

* feat(data-migration): start working on PB data migration Signed-off-by: Robert Goniszewski <robertgoniszewski@outlook.com> * feat(metadata): enhance performance logging and URL display Signed-off-by: Robert Goniszewski <robertgoniszewski@outlook.com> * refactor(utils): enhance performance logging format Signed-off-by: Robert Goniszewski <robertgoniszewski@outlook.com> * chore(deps): update development and production dependencies Signed-off-by: Robert Goniszewski <robertgoniszewski@outlook.com> * feat(migration): implement data migration UI and functionality Signed-off-by: Robert Goniszewski <robertgoniszewski@outlook.com> * feat(database): update foreign key constraints for cascading deletes Signed-off-by: Robert Goniszewski <robertgoniszewski@outlook.com> * feat(bookmark): optimize image storage and improve first-time user experience Signed-off-by: Robert Goniszewski <robertgoniszewski@outlook.com> * feat(various): finish migration tool preview, improve file handling Signed-off-by: Robert Goniszewski <robertgoniszewski@outlook.com> * feat(setup): simplify installation process and update dependencies Signed-off-by: Robert Goniszewski <robertgoniszewski@outlook.com> * chore: release v0.4.0-pre.4 --------- Signed-off-by: Robert Goniszewski <robertgoniszewski@outlook.com>
goniszewski · Sep 16, 2024 · d95f4d2 · d95f4d2
1 parent 7612593
commit d95f4d2
Show file tree

Hide file tree

Showing 23 changed files with 1,261 additions and 100 deletions.
diff --git a/.gitignore b/.gitignore
@@ -11,4 +11,5 @@ vite.config.ts.timestamp-*
 coverage
 data/db*
 data/user-uploads/*
+data/temp/*
 !data/user-uploads/.gitkeep
diff --git a/README.md b/README.md
@@ -10,10 +10,7 @@
 <br>
 
 > [!IMPORTANT]
-> This application is undergoing a comprehensive overhaul to simplify its architecture, deliver an enhanced user experience, and have a more efficient development cycle.
-> 
-> Read the [Refactoring and streamlining effort](https://github.com/users/goniszewski/projects/1/views/1?pane=issue&itemId=65225221) for more information.
-
+> Version `0.4` introduces a new approach for data storage and user authorization. If you are upgrading from version `0.3.X` you may want to utilize the built-in [**migration tool** (read more)]().
 
 Glimpse into the magical book of _your_ forbidden knowledge - **Grimoire!** 📖💫
 
@@ -58,21 +55,41 @@ More screenshots can be found in the [screenshots directory](screenshots).
 
 ### Steps
 
-```bash
-# Clone the repository
-git clone https://github.com/goniszewski/grimoire
-
-# Rename the `.env.example` file to `.env`
-# "mv .env.example .env" on Linux/MacOS, "ren .env.example .env" on Windows
-
-# Build and run the containers
-docker compose up
+1. Create a `docker-compose.yml` file with the following content:
+
+```yml
+services:
+  grimoire:
+    image: goniszewski/grimoire:develop
+    container_name: grimoire
+    restart: unless-stopped
+    environment:
+      - PORT=5173
+      - PUBLIC_HTTPS_ONLY=false
+      - PUBLIC_SIGNUP_DISABLED=false
+    volumes:
+      - grimoire_data:/app/data/
+    build:
+      context: .
+      dockerfile: Dockerfile
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:$PORT/api/health || exit 1
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    ports:
+      - '${PORT:-5173}:${PORT:-5173}'
+volumes:
+  grimoire_data:
 ```
 
+2. [Optional] Update the environment variables to match your needs.
+3. Run the app using `docker compose up -d` command.
+
 </details>
 
 > [!NOTE]
-> For the recommended setup, only the `docker-compose.yml` and `.env.example` files are needed.
+> For the recommended setup, only the `docker-compose.yml` file is required.
 
 <details>
   <summary><strong>Run app using Node</strong></summary>
@@ -138,11 +155,14 @@ Special thanks to: [@extractus/article-extractor](https://github.com/extractus/a
 [DaisyUI](https://github.com/saadeghi/daisyui),
 [Drizzle](https://github.com/drizzle-team/drizzle-orm),
 [Fuse.js](https://github.com/krisk/fuse),
+[Lucia](https://github.com/pilcrowOnPaper/lucia),
 [MetaScraper](https://github.com/microlinkhq/metascraper),
 [PocketBase](https://github.com/pocketbase/pocketbase),
 [sanitize-html](https://github.com/apostrophecms/sanitize-html),
 [SvelteKit](https://github.com/sveltejs/kit),
 [Svelte Select](https://github.com/rob-balfre/svelte-select),
 [Svelte French Toast](https://github.com/kbrgl/svelte-french-toast),
+[Swagger UI](https://github.com/swagger-api/swagger-ui),
+[Tabler Icons](https://github.com/tabler/tabler-icons),
 [Tailwind CSS](https://tailwindcss.com),
-[url-metadata](https://github.com/laurengarcia/url-metadata),
+[url-metadata](https://github.com/laurengarcia/url-metadata)
diff --git a/bun.lockb b/bun.lockb
diff --git a/migrations/0000_init.sql b/migrations/0000_init.sql
@@ -36,8 +36,8 @@ CREATE TABLE `bookmarks_to_tags` (
 	`bookmark_id` integer NOT NULL,
 	`tag_id` integer NOT NULL,
 	PRIMARY KEY(`bookmark_id`, `tag_id`),
-	FOREIGN KEY (`bookmark_id`) REFERENCES `bookmark`(`id`) ON UPDATE no action ON DELETE no action,
-	FOREIGN KEY (`tag_id`) REFERENCES `tag`(`id`) ON UPDATE no action ON DELETE no action
+	FOREIGN KEY (`bookmark_id`) REFERENCES `bookmark`(`id`) ON UPDATE no action ON DELETE cascade,
+	FOREIGN KEY (`tag_id`) REFERENCES `tag`(`id`) ON UPDATE no action ON DELETE cascade
 );
 --> statement-breakpoint
 CREATE TABLE `category` (
@@ -55,7 +55,7 @@ CREATE TABLE `category` (
 	`created` integer DEFAULT (unixepoch()) NOT NULL,
 	`updated` integer DEFAULT (unixepoch()) NOT NULL,
 	FOREIGN KEY (`owner_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade,
-	FOREIGN KEY (`parent_id`) REFERENCES `category`(`id`) ON UPDATE no action ON DELETE no action
+	FOREIGN KEY (`parent_id`) REFERENCES `category`(`id`) ON UPDATE no action ON DELETE set null
 );
 --> statement-breakpoint
 CREATE TABLE `file` (

diff --git a/migrations/meta/0000_snapshot.json b/migrations/meta/0000_snapshot.json
@@ -1,7 +1,7 @@
 {
   "version": "6",
   "dialect": "sqlite",
-  "id": "eaa80552-21c4-4c1e-aede-e67b9ff3a247",
+  "id": "cd87d2c5-7ead-4fac-ba76-cfa0df80697f",
   "prevId": "00000000-0000-0000-0000-000000000000",
   "tables": {
     "bookmark": {
@@ -335,7 +335,7 @@
           "columnsTo": [
             "id"
           ],
-          "onDelete": "no action",
+          "onDelete": "cascade",
           "onUpdate": "no action"
         },
         "bookmarks_to_tags_tag_id_tag_id_fk": {
@@ -348,7 +348,7 @@
           "columnsTo": [
             "id"
           ],
-          "onDelete": "no action",
+          "onDelete": "cascade",
           "onUpdate": "no action"
         }
       },
@@ -503,7 +503,7 @@
           "columnsTo": [
             "id"
           ],
-          "onDelete": "no action",
+          "onDelete": "set null",
           "onUpdate": "no action"
         }
       },

diff --git a/migrations/meta/_journal.json b/migrations/meta/_journal.json
@@ -5,7 +5,7 @@
     {
       "idx": 0,
       "version": "6",
-      "when": 1723048264894,
+      "when": 1726497020846,
       "tag": "0000_init",
       "breakpoints": true
     }

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "grimoire",
-	"version": "0.4.0-pre.3",
+	"version": "0.4.0-pre.4",
 	"description": "Bookmark manager for the wizards 🧙",
 	"author": "Robert Goniszewski <robert@goniszewski.com>",
 	"main": "./build/index.js",
@@ -39,55 +39,57 @@
 	},
 	"type": "module",
 	"devDependencies": {
-		"@libsql/client": "^0.10.0",
+		"@libsql/client": "^0.11.0",
 		"@sveltejs/adapter-auto": "^3.2.4",
-		"@sveltejs/kit": "^2.5.24",
+		"@sveltejs/kit": "^2.5.27",
 		"@tailwindcss/typography": "^0.5.14",
-		"@types/bun": "^1.1.8",
+		"@types/adm-zip": "^0.5.5",
+		"@types/bun": "^1.1.9",
 		"@types/express": "^4.17.21",
 		"@types/express-http-proxy": "^1.6.6",
 		"@types/swagger-ui": "^3.52.4",
-		"@typescript-eslint/eslint-plugin": "^8.3.0",
-		"@typescript-eslint/parser": "^8.3.0",
-		"@vitest/coverage-v8": "^2.0.5",
+		"@typescript-eslint/eslint-plugin": "^8.5.0",
+		"@typescript-eslint/parser": "^8.5.0",
+		"@vitest/coverage-v8": "^2.1.1",
 		"autoprefixer": "^10.4.20",
 		"drizzle-kit": "^0.24.2",
-		"eslint": "^9.9.1",
+		"eslint": "^9.10.0",
 		"eslint-config-prettier": "^9.1.0",
-		"eslint-plugin-perfectionist": "^3.3.0",
-		"eslint-plugin-svelte": "^2.43.0",
-		"postcss": "^8.4.41",
+		"eslint-plugin-perfectionist": "^3.6.0",
+		"eslint-plugin-svelte": "^2.44.0",
+		"postcss": "^8.4.47",
 		"prettier": "^3.3.3",
 		"prettier-plugin-svelte": "^3.2.6",
 		"prettier-plugin-tailwindcss": "^0.6.6",
 		"release-it": "^17.6.0",
 		"svelte": "^4.2.19",
 		"svelte-adapter-bun": "^0.5.2",
-		"svelte-check": "^3.8.6",
-		"tailwindcss": "^3.4.10",
+		"svelte-check": "^4.0.2",
+		"tailwindcss": "^3.4.11",
 		"tslib": "^2.7.0",
-		"typescript": "^5.5.4",
-		"vite": "^5.4.2",
-		"vitest": "^2.0.5"
+		"typescript": "^5.6.2",
+		"vite": "^5.4.5",
+		"vitest": "^2.1.1"
 	},
 	"dependencies": {
 		"@extractus/article-extractor": "^8.0.10",
 		"@lucia-auth/adapter-drizzle": "^1.1.0",
 		"@sveltejs/vite-plugin-svelte": "^3.1.2",
-		"@tabler/icons": "^3.14.0",
-		"@tabler/icons-svelte": "^3.14.0",
+		"@tabler/icons": "^3.16.0",
+		"@tabler/icons-svelte": "^3.16.0",
 		"@tailwindcss/line-clamp": "^0.4.4",
 		"@types/html-to-text": "^9.0.4",
 		"@types/lodash": "^4.17.7",
 		"@types/sanitize-html": "^2.13.0",
+		"adm-zip": "^0.5.16",
+		"chalk": "^5.3.0",
 		"daisyui": "^4.12.10",
 		"dotenv": "^16.4.5",
 		"drizzle-orm": "^0.33.0",
 		"eslint-plugin-drizzle": "^0.2.3",
-		"express": "^4.19.2",
+		"express": "^4.21.0",
 		"fuse.js": "^7.0.0",
 		"html-to-text": "^9.0.5",
-		"http-proxy-middleware": "^3.0.0",
 		"joi": "^17.13.3",
 		"lodash": "^4.17.21",
 		"lucia": "^3.2.0",

diff --git a/src/lib/database/schema.ts b/src/lib/database/schema.ts
@@ -5,6 +5,7 @@ import { FileSourceEnum, FileStorageTypeEnum } from '../enums/files';
 
 import type { AnySQLiteColumn } from 'drizzle-orm/sqlite-core';
 import type { UserSettings } from '$lib/types/UserSettings.type';
+
 export const userSchema = sqliteTable(
 	'user',
 	{
@@ -73,7 +74,9 @@ export const categorySchema = sqliteTable(
 		ownerId: integer('owner_id')
 			.notNull()
 			.references(() => userSchema.id, { onDelete: 'cascade' }),
-		parentId: integer('parent_id').references((): AnySQLiteColumn => categorySchema.id),
+		parentId: integer('parent_id').references((): AnySQLiteColumn => categorySchema.id, {
+			onDelete: 'set null'
+		}),
 		archived: integer('archived', { mode: 'timestamp' }),
 		public: integer('public', { mode: 'timestamp' }),
 		icon: text('icon'),
@@ -177,10 +180,14 @@ export const bookmarksToTagsSchema = sqliteTable(
 	{
 		bookmarkId: integer('bookmark_id')
 			.notNull()
-			.references(() => bookmarkSchema.id),
+			.references(() => bookmarkSchema.id, {
+				onDelete: 'cascade'
+			}),
 		tagId: integer('tag_id')
 			.notNull()
-			.references(() => tagSchema.id)
+			.references(() => tagSchema.id, {
+				onDelete: 'cascade'
+			})
 	},
 	(table) => ({
 		pk: primaryKey({ columns: [table.bookmarkId, table.tagId] }),

diff --git a/src/lib/storage/storage.ts b/src/lib/storage/storage.ts
@@ -72,7 +72,7 @@ export class Storage {
 		return file;
 	}
 
-	async storeImage(url: string, title: string, ownerId: number) {
+	async storeImage(url: string, title: string, ownerId: number, relatedEntityId?: number) {
 		const storage = new Storage();
 
 		if (!url || url.length === 0) {
@@ -87,7 +87,8 @@ export class Storage {
 		const fileName = `${createSlug(title)}.${url.split('.').pop()?.split('?')[0]}`;
 
 		return await storage.storeFile(blob as Blob, ownerId, {
-			fileName
+			fileName,
+			relatedEntityId
 		});
 	}
 }