Merge pull request #7 from TechTheAwesome/feature/content-hash

ADD content hashing to eliminate duplicate uploads
ttax00 · Jun 11, 2023 · 7a8e324 · 7a8e324
2 parents e356177 + df82854
commit 7a8e324
Show file tree

Hide file tree

Showing 7 changed files with 1,181 additions and 1,387 deletions.
diff --git a/main.ts b/main.ts
@@ -1,10 +1,10 @@
-import { Editor, MarkdownView, Notice, Plugin } from 'obsidian';
+import { Editor, MarkdownFileInfo, MarkdownView, Notice, Plugin } from 'obsidian';
 import { DEFAULT_SETTINGS, IObsidianSetting, S3ClientSettings, SettingsTab } from 'src/settings';
 import { S3Server } from 'src/httpServer';
 import { mimeType } from 'src/settings';
 import { S3Client } from 'src/s3Client';
 import prettyBytes from 'pretty-bytes';
-import { generateResourceName, getS3Path, getS3URLs } from 'src/helper';
+import { buf2hex, generateResourceName, getS3Path, getS3URLs } from 'src/helper';
 
 
 function allFilesAreValidUploads(files: FileList) {
@@ -155,17 +155,17 @@ export default class ObsidianS3 extends Plugin {
 		await this.saveData(settings);
 	}
 
-	private async pasteEventHandler(e: ClipboardEvent, _: Editor, markdownView: MarkdownView) {
+	private pasteEventHandler(e: ClipboardEvent, _: Editor, markdownView: MarkdownView | MarkdownFileInfo) {
 		if (!this.s3) return this.credentialsError();
 		if (!e.clipboardData) return;
 		const files = e.clipboardData.files;
 
 		if (!allFilesAreValidUploads(files)) return;
 		e.preventDefault();
 
-		await this.uploadFiles(files);
+		this.uploadFiles(files);
 	}
-	private async dropEventHandler(e: DragEvent, _: Editor, markdownView: MarkdownView) {
+	private dropEventHandler(e: DragEvent, _: Editor, markdownView: MarkdownView | MarkdownFileInfo) {
 		if (!this.s3) return this.credentialsError();
 		if (!e.dataTransfer) return;
 		if (!e.dataTransfer.types.length || !e.dataTransfer.types.includes("Files")) return;
@@ -175,7 +175,7 @@ export default class ObsidianS3 extends Plugin {
 		if (!allFilesAreValidUploads(files)) return;
 
 		e.preventDefault();
-		await this.uploadFiles(files);
+		this.uploadFiles(files);
 	}
 
 	private setupHandlers() {
@@ -210,39 +210,50 @@ export default class ObsidianS3 extends Plugin {
 		editor.setCursor(cursor);
 	}
 
-	private async uploadFiles(files: FileList) {
+	private uploadFiles(files: FileList) {
 		for (let i = 0; i < files.length; i += 1) {
 			const file = files[i];
-			const fileName = generateResourceName(file.name, this.app.workspace.getActiveFile()?.basename);
-
-			new Notice(`Uploading: ${fileName} ${prettyBytes(file.size)} ...`);
-			try {
-				const s3 = this.s3;
-				let progress = 0;
-				const handle = window.setInterval(() => new Notice(`Uploading: ${fileName} ${progress}%`), 5000);
-				this.registerInterval(handle);
-				await s3.upload(file, fileName,
-					(prog) => progress = prog,
-					() => window.clearInterval(handle));
-
-				const url = s3.createObjURL(server.url, fileName);
-
-				let linkTxt = `![S3 File](${url})`;
-				const method = mimeType.getMethod(file.type);
-				if (method === 'iframe') {
-					linkTxt = `<iframe src="${url}" alt="${fileName}" style="overflow:hidden;height:400;width:100%" allowfullscreen></iframe>`;
-				} else if (method === 'link') {
-					linkTxt = `${url}`;
-				}
+			const baseName = this.app.workspace.getActiveFile()?.basename;
+			const reader = new FileReader();
+			const callback = (file: File, fileName: string) => { void this.uploadFile(file, fileName); };
+			reader.readAsArrayBuffer(file);
+			reader.onloadend = async function (ev) {
+				const hashBuf = await crypto.subtle.digest("SHA-1", reader.result as ArrayBuffer);
+				const hash = buf2hex(hashBuf);
+				const fileName = generateResourceName(file.name, baseName, hash);
+				callback(file, fileName);
+			};
+		}
+	}
 
-				this.writeLine(linkTxt);
-			}
-			catch (e) {
-				new Notice(`Error: Unable to upload ${fileName}. Make sure your S3 credentials are correct.`);
-				return console.log(e);
+	private async uploadFile(file: File, fileName: string) {
+		new Notice(`Uploading: ${fileName} ${prettyBytes(file.size)} ...`);
+		try {
+			const s3 = this.s3;
+			let progress = 0;
+			const handle = window.setInterval(() => new Notice(`Uploading: ${fileName} ${progress}%`), 5000);
+			this.registerInterval(handle);
+			await s3.upload(file, fileName,
+				(prog) => progress = prog,
+				() => window.clearInterval(handle));
+
+			const url = s3.createObjURL(server.url, fileName);
+
+			let linkTxt = `![S3 File](${url})`;
+			const method = mimeType.getMethod(file.type);
+			if (method === 'iframe') {
+				linkTxt = `<iframe src="${url}" alt="${fileName}" style="overflow:hidden;height:400;width:100%" allowfullscreen></iframe>`;
+			} else if (method === 'link') {
+				linkTxt = `${url}`;
 			}
+
+			this.writeLine(linkTxt);
+		}
+		catch (e) {
+			new Notice(`Error: Unable to upload ${fileName}. Make sure your S3 credentials are correct.`);
+			return console.log(e);
 		}
-	}
 
+	}
 }
 
diff --git a/manifest.json b/manifest.json
@@ -1,7 +1,7 @@
 {
 	"id": "s3-attachments-storage",
 	"name": "S3 attachments storage",
-	"version": "0.1.1",
+	"version": "0.1.2",
 	"minAppVersion": "0.15.0",
 	"description": "An Obsidian plugin for storage and retrieval of media attachments on S3 compatible services.",
 	"author": "TechTheAwesome",

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "s3-attachments-storage",
-	"version": "0.1.1",
+	"version": "0.1.2",
 	"description": "An Obsidian plugin for storage and retrieval of media attachments on S3 compatible services.",
 	"main": "main.js",
 	"packageManager": "pnpm@7.17.0",
@@ -17,24 +17,23 @@
 	"author": "",
 	"license": "MIT",
 	"devDependencies": {
-		"@types/jest": "^29.2.3",
-		"@types/node": "^16.18.3",
+		"@types/jest": "^29.5.2",
+		"@types/node": "^16.18.35",
 		"@typescript-eslint/eslint-plugin": "5.29.0",
 		"@typescript-eslint/parser": "5.29.0",
 		"builtin-modules": "3.3.0",
 		"esbuild": "0.14.47",
-		"eslint": "^8.28.0",
-		"jest": "^29.3.1",
+		"eslint": "^8.42.0",
+		"jest": "^29.5.0",
 		"obsidian": "latest",
-		"ts-jest": "^29.0.3",
+		"ts-jest": "^29.1.0",
 		"tslib": "2.4.0",
 		"typescript": "4.7.4"
 	},
 	"dependencies": {
-		"@types/minio": "^7.0.15",
-		"blob-to-it": "^2.0.0",
-		"minio": "^7.0.32",
-		"pretty-bytes": "^6.0.0",
+		"blob-to-it": "^2.0.3",
+		"minio": "^7.1.1",
+		"pretty-bytes": "^6.1.0",
 		"rimraf": "^3.0.2"
 	}
 }