fix: Properly handle image files

Signed-off-by: Julius Knorr <jus@bitgrid.net>

src/collaboration/Portal.ts

@@ -7,7 +7,7 @@
 import type { ExcalidrawElement } from '@excalidraw/excalidraw/types/element/types'
 import { io, type Socket } from 'socket.io-client'
 import type { Collab } from './collab'
-import type { AppState, Gesture } from '@excalidraw/excalidraw/types/types'
+import type { AppState, BinaryFiles, Gesture } from '@excalidraw/excalidraw/types/types'
 import axios from '@nextcloud/axios'
 import { loadState } from '@nextcloud/initial-state'
 
@@ -136,6 +136,9 @@ export class Portal {
 			this.collab.handleRemoteSceneUpdate(reconciledElements)
 			this.collab.scrollToContent()
 		})
+		this.socket?.on('image-data', (file) => {
+			this.collab.addFile(file)
+		})
 	}
 
 	handleClientBroadcast(data: ArrayBuffer) {
@@ -236,4 +239,11 @@ export class Portal {
 		await this._broadcastSocketData(data, true)
 	}
 
+	async sendImageFiles(files: BinaryFiles) {
+		Object.values(files).forEach(file => {
+			this.collab.addFile(file)
+			this.socket?.emit('image-add', this.roomId, file.id, file)
+		})
+	}
+
 }

src/collaboration/collab.ts

@@ -4,7 +4,7 @@
  */
 
 import type { ExcalidrawElement } from '@excalidraw/excalidraw/types/element/types'
-import type { AppState, Collaborator, ExcalidrawImperativeAPI, Gesture } from '@excalidraw/excalidraw/types/types'
+import type { AppState, BinaryFileData, BinaryFiles, Collaborator, ExcalidrawImperativeAPI, Gesture } from '@excalidraw/excalidraw/types/types'
 import { Portal } from './Portal'
 import { restoreElements } from '@excalidraw/excalidraw'
 import { throttle } from 'lodash'
@@ -18,6 +18,7 @@ export class Collab {
 	publicSharingToken: string | null
 	lastBroadcastedOrReceivedSceneVersion: number = -1
 	private collaborators = new Map<string, Collaborator>()
+	private files = new Map<string, BinaryFileData>()
 
 	constructor(excalidrawAPI: ExcalidrawImperativeAPI, fileId: number, publicSharingToken: string | null) {
 		this.excalidrawAPI = excalidrawAPI
@@ -59,12 +60,23 @@ export class Collab {
 	}
 
 	// eslint-disable-next-line @typescript-eslint/no-unused-vars
-	private onChange = (elements: readonly ExcalidrawElement[], _state: AppState) => {
+	private onChange = (elements: readonly ExcalidrawElement[], _state: AppState, files: BinaryFiles) => {
 		if (hashElementsVersion(elements)
 			> this.getLastBroadcastedOrReceivedSceneVersion()
 		) {
 			this.lastBroadcastedOrReceivedSceneVersion = hashElementsVersion(elements)
-			throttle(() => this.portal.broadcastScene('SCENE_INIT', elements))()
+			throttle(() => {
+				this.portal.broadcastScene('SCENE_INIT', elements)
+
+				const syncedFiles = Array.from(this.files.keys())
+				const newFiles = Object.keys(files).filter((id) => !syncedFiles.includes(id)).reduce((acc, id) => {
+					acc[id] = files[id]
+					return acc
+				}, {} as BinaryFiles)
+				if (Object.keys(newFiles).length > 0) {
+					this.portal.sendImageFiles(newFiles)
+				}
+			})()
 		}
 	}
 
@@ -137,4 +149,9 @@ export class Collab {
 		})
 	}
 
+	addFile = (file: BinaryFileData) => {
+		this.files.set(file.id, file)
+		this.excalidrawAPI.addFiles([file])
+	}
+
 }

websocket_server/ApiService.js

@@ -55,13 +55,25 @@ export default class ApiService {
 		return this.fetchData(url, options)
 	}
 
-	async saveRoomDataToServer(roomID, roomData, lastEditedUser) {
-		console.log('Saving room data to file')
+	async saveRoomDataToServer(roomID, roomData, lastEditedUser, files) {
+		console.log(`[${roomID}] Saving room data to server: ${roomData.length} elements, ${Object.keys(files).length} files`)
 
 		const url = `${this.NEXTCLOUD_URL}/index.php/apps/whiteboard/${roomID}`
-		const body = { data: { elements: roomData } }
+		const body = { data: { elements: roomData, files: this.cleanupFiles(roomData, files) } }
 		const options = this.fetchOptions('PUT', null, body, roomID, lastEditedUser)
 		return this.fetchData(url, options)
 	}
 
+	cleanupFiles(elements, files) {
+		const elementFileIds = elements.filter(e => e?.fileId && e?.isDeleted !== true).map((e) => e.fileId)
+		const fileIds = Object.keys(files)
+
+		const fileIdsToStore = fileIds.filter((fileId) => elementFileIds.includes(fileId))
+		const filesToStore = fileIdsToStore.reduce((acc, fileId) => {
+			acc[fileId] = files[fileId]
+			return acc
+		}, {})
+		return filesToStore
+	}
+
 }

websocket_server/LRUCacheStrategy.js

@@ -18,13 +18,14 @@ export default class LRUCacheStrategy extends StorageStrategy {
 			ttl: 30 * 60 * 1000,
 			ttlAutopurge: true,
 			dispose: async (value, key) => {
-				console.log('Disposing room', key)
+				console.log(`[${key}] Disposing room`)
 				if (value?.data && value?.lastEditedUser) {
 					try {
 						await this.apiService.saveRoomDataToServer(
 							key,
 							value.data,
 							value.lastEditedUser,
+							value.files,
 						)
 					} catch (error) {
 						console.error(`Failed to save room ${key} data:`, error)

websocket_server/RedisStrategy.js

@@ -72,6 +72,7 @@ export default class RedisStrategy extends StorageStrategy {
 					key,
 					room.data,
 					room.lastEditedUser,
+					room.files,
 				)
 			}
 			await this.client.del(key)

websocket_server/Room.js

@@ -5,11 +5,12 @@
 
 export default class Room {
 
-	constructor(id, data = null, users = new Set(), lastEditedUser = null) {
+	constructor(id, data = null, users = new Set(), lastEditedUser = null, files = {}) {
 		this.id = id
 		this.data = data
 		this.users = new Set(users)
 		this.lastEditedUser = lastEditedUser
+		this.files = files
 	}
 
 	setUsers(users) {
@@ -24,6 +25,26 @@ export default class Room {
 		this.data = data
 	}
 
+	setFiles(files) {
+		this.files = files
+	}
+
+	getFiles() {
+		return this.files
+	}
+
+	addFile(id, file) {
+		this.files[id] = file
+	}
+
+	removeFile(id) {
+		delete this.files[id]
+	}
+
+	getFile(id) {
+		return this.files[id] ?? undefined
+	}
+
 	isEmpty() {
 		return this.users.size === 0
 	}
@@ -34,6 +55,7 @@ export default class Room {
 			data: this.data,
 			users: Array.from(this.users),
 			lastEditedUser: this.lastEditedUser,
+			files: this.files,
 		}
 	}
 
@@ -43,6 +65,7 @@ export default class Room {
 			json.data,
 			new Set(json.users),
 			json.lastEditedUser,
+			json.files,
 		)
 	}
 

websocket_server/RoomDataManager.js

@@ -28,13 +28,16 @@ export default class RoomDataManager {
 			data = await this.fetchRoomDataFromServer(roomId, jwtToken)
 		}
 
-		if (data) room.setData(data)
+		const files = data?.files
+		const elements = data?.elements ?? data
+		if (elements) room.setData(elements)
 		if (lastEditedUser) room.updateLastEditedUser(lastEditedUser)
 		if (users) room.setUsers(users)
+		if (files) room.setFiles(files)
 
 		await this.storageManager.set(roomId, room)
 
-		console.log(`[${roomId}] Room data synced. Users: ${room.users.size}, Last edited by: ${room.lastEditedUser}`)
+		console.log(`[${roomId}] Room data synced. Users: ${room.users.size}, Last edited by: ${room.lastEditedUser}, files: ${Object.keys(room.files).length}`)
 
 		if (room.isEmpty()) {
 			await this.storageManager.delete(roomId)
@@ -49,10 +52,11 @@ export default class RoomDataManager {
 		console.log(`[${roomId}] No data provided or existing, fetching from server...`)
 		try {
 			const result = await this.apiService.getRoomDataFromServer(roomId, jwtToken)
-			return result?.data?.elements || { elements: [] }
+			console.log(`[${roomId}] Fetched data from server: \n`, result)
+			return result?.data || { elements: [], files: {} }
 		} catch (error) {
 			console.error(`[${roomId}] Failed to fetch data from server:`, error)
-			return { elements: [] }
+			return { elements: [], files: {} }
 		}
 	}
 

websocket_server/SocketManager.js

@@ -95,6 +95,9 @@ export default class SocketManager {
 		socket.on('server-volatile-broadcast', (roomID, encryptedData) =>
 			this.serverVolatileBroadcastHandler(socket, roomID, encryptedData),
 		)
+		socket.on('image-add', (roomID, id, data) => this.imageAddHandler(socket, roomID, id, data))
+		socket.on('image-remove', (roomID, id, data) => this.imageRemoveHandler(socket, roomID, id, data))
+		socket.on('image-get', (roomID, id, data) => this.imageGetHandler(socket, roomID, id, data))
 		socket.on('disconnecting', () => {
 			const rooms = Array.from(socket.rooms).filter((room) => room !== socket.id)
 			this.disconnectingHandler(socket, rooms)
@@ -170,6 +173,10 @@ export default class SocketManager {
 				[],
 			)
 
+			Object.values(room.getFiles()).forEach((file) => {
+				socket.emit('image-data', file)
+			})
+
 			this.io.to(roomID).emit('room-user-change', userSocketsAndIds)
 		} else {
 			socket.emit('room-not-found')
@@ -184,6 +191,7 @@ export default class SocketManager {
 
 		const decryptedData = JSON.parse(Utils.convertArrayBufferToString(encryptedData))
 		const socketData = await this.socketDataManager.getSocketData(socket.id)
+		if (!socketData) return
 		const userSocketsAndIds = await this.getUserSocketsAndIds(roomID)
 
 		await this.roomDataManager.syncRoomData(
@@ -203,6 +211,7 @@ export default class SocketManager {
 			const socketData = await this.socketDataManager.getSocketData(
 				socket.id,
 			)
+			if (!socketData) return
 			const eventData = {
 				type: 'MOUSE_LOCATION',
 				payload: {
@@ -220,8 +229,45 @@ export default class SocketManager {
 		}
 	}
 
+	async imageAddHandler(socket, roomID, id, data) {
+		const isReadOnly = await this.isSocketReadOnly(socket.id)
+		if (!socket.rooms.has(roomID) || isReadOnly) return
+
+		socket.broadcast.to(roomID).emit('image-data', data)
+		const room = await this.storageManager.get(roomID)
+
+		console.log(`[${roomID}] ${socket.id} added image ${id}`)
+		room.addFile(id, data)
+	}
+
+	async imageRemoveHandler(socket, roomID, id) {
+		const isReadOnly = await this.isSocketReadOnly(socket.id)
+		if (!socket.rooms.has(roomID) || isReadOnly) return
+
+		socket.broadcast.to(roomID).emit('image-remove', id)
+		const room = await this.storageManager.get(roomID)
+		room.removeFile(id)
+	}
+
+	async imageGetHandler(socket, roomId, id) {
+		const isReadOnly = await this.isSocketReadOnly(socket.id)
+		if (!socket.rooms.has(roomId) || isReadOnly) return
+
+		console.log(`[${roomId}] ${socket.id} requested image ${id}`)
+		const room = await this.storageManager.get(roomId)
+		const file = room.getFile(id)
+
+		if (file) {
+			socket.emit('image-data', file)
+			console.log(`[${roomId}] ${socket.id} sent image data ${id}`)
+		} else {
+			console.warn(`[${roomId}] Image ${id} not found`)
+		}
+	}
+
 	async disconnectingHandler(socket, rooms) {
 		const socketData = await this.socketDataManager.getSocketData(socket.id)
+		if (!socketData) return
 		console.log(`[${socketData.fileId}] ${socketData.user.name} has disconnected`)
 		console.log('socket rooms', rooms)