From 393359e1105edce58e3434debe0f7ce10b472118 Mon Sep 17 00:00:00 2001 From: Alun Turner Date: Wed, 7 Jun 2023 11:21:59 +0100 Subject: [PATCH 01/15] get rough funcitonality working --- .../components/PlainTextComposer.tsx | 3 ++- .../hooks/usePlainTextListeners.ts | 25 ++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx b/src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx index efc4971657d..ed53a3f61ea 100644 --- a/src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx +++ b/src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx @@ -50,6 +50,7 @@ export function PlainTextComposer({ initialContent, leftComponent, rightComponent, + eventRelation, }: PlainTextComposerProps): JSX.Element { const { ref: editorRef, @@ -63,7 +64,7 @@ export function PlainTextComposer({ onSelect, handleCommand, handleMention, - } = usePlainTextListeners(initialContent, onChange, onSend); + } = usePlainTextListeners(initialContent, onChange, onSend, eventRelation); const composerFunctions = useComposerFunctions(editorRef, setContent); usePlainTextInitialization(initialContent, editorRef); diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts b/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts index 2bccfc444ae..cf5e554846c 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts @@ -16,6 +16,7 @@ limitations under the License. import { KeyboardEvent, RefObject, SyntheticEvent, useCallback, useRef, useState } from "react"; import { Attributes, MappedSuggestion } from "@matrix-org/matrix-wysiwyg"; +import { IEventRelation } from "matrix-js-sdk/src/matrix"; import { useSettingValue } from "../../../../../hooks/useSettings"; import { IS_MAC, Key } from "../../../../../Keyboard"; @@ -23,6 +24,9 @@ import Autocomplete from "../../Autocomplete"; import { handleEventWithAutocomplete } from "./utils"; import { useSuggestion } from "./useSuggestion"; import { isNotNull, isNotUndefined } from "../../../../../Typeguards"; +import { handleClipboardEvent } from "./useInputEventProcessor"; +import { useRoomContext } from "../../../../../contexts/RoomContext"; +import { useMatrixClientContext } from "../../../../../contexts/MatrixClientContext"; function isDivElement(target: EventTarget): target is HTMLDivElement { return target instanceof HTMLDivElement; @@ -59,6 +63,7 @@ export function usePlainTextListeners( initialContent?: string, onChange?: (content: string) => void, onSend?: () => void, + eventRelation?: IEventRelation, ): { ref: RefObject; autocompleteRef: React.RefObject; @@ -72,6 +77,9 @@ export function usePlainTextListeners( onSelect: (event: SyntheticEvent) => void; suggestion: MappedSuggestion | null; } { + const roomContext = useRoomContext(); + const mxClient = useMatrixClientContext(); + const ref = useRef(null); const autocompleteRef = useRef(null); const [content, setContent] = useState(initialContent); @@ -106,7 +114,10 @@ export function usePlainTextListeners( const enterShouldSend = !useSettingValue("MessageComposerInput.ctrlEnterToSend"); const onInput = useCallback( (event: SyntheticEvent) => { + console.log("<<< handling ", event); if (isDivElement(event.target)) { + console.log("<<< setting ", event.target.innerHTML); + // if enterShouldSend, we do not need to amend the html before setting text const newInnerHTML = enterShouldSend ? event.target.innerHTML : amendInnerHtml(event.target.innerHTML); setText(newInnerHTML); @@ -115,6 +126,18 @@ export function usePlainTextListeners( [setText, enterShouldSend], ); + const onPaste = useCallback( + (event: SyntheticEvent) => { + const handled = handleClipboardEvent(event, roomContext, mxClient, eventRelation); + if (handled) { + event.preventDefault(); // we only handle image pasting manually + } else { + onInput(event); + } + }, + [eventRelation, mxClient, onInput, roomContext], + ); + const onKeyDown = useCallback( (event: KeyboardEvent) => { // we need autocomplete to take priority when it is open for using enter to select @@ -150,7 +173,7 @@ export function usePlainTextListeners( ref, autocompleteRef, onInput, - onPaste: onInput, + onPaste, onKeyDown, content, setContent: setText, From 18542a0d79fcda9e4e49f179e523a57082720e0b Mon Sep 17 00:00:00 2001 From: Alun Turner Date: Wed, 7 Jun 2023 11:24:35 +0100 Subject: [PATCH 02/15] try to tidy up types --- .../wysiwyg_composer/hooks/useInputEventProcessor.ts | 5 ++--- .../wysiwyg_composer/hooks/usePlainTextListeners.ts | 10 +++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts b/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts index 27f40880140..863938da7d9 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts @@ -15,7 +15,7 @@ limitations under the License. */ import { Wysiwyg, WysiwygEvent } from "@matrix-org/matrix-wysiwyg"; -import { useCallback } from "react"; +import { useCallback, ClipboardEvent as ReactClipboardEvent } from "react"; import { IEventRelation, MatrixClient } from "matrix-js-sdk/src/matrix"; import { useSettingValue } from "../../../../../hooks/useSettings"; @@ -257,8 +257,7 @@ function handleInputEvent(event: InputEvent, send: Send, isCtrlEnterToSend: bool * @returns - boolean to show if the event was handled or not */ export function handleClipboardEvent( - event: ClipboardEvent | InputEvent, - data: DataTransfer | null, + clipboardEvent: ClipboardEvent | ReactClipboardEvent | InputEvent, roomContext: IRoomState, mxClient: MatrixClient, eventRelation?: IEventRelation, diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts b/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts index cf5e554846c..cf662fefa0a 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { KeyboardEvent, RefObject, SyntheticEvent, useCallback, useRef, useState } from "react"; +import { KeyboardEvent, RefObject, SyntheticEvent, ClipboardEvent, useCallback, useRef, useState } from "react"; import { Attributes, MappedSuggestion } from "@matrix-org/matrix-wysiwyg"; import { IEventRelation } from "matrix-js-sdk/src/matrix"; @@ -68,8 +68,8 @@ export function usePlainTextListeners( ref: RefObject; autocompleteRef: React.RefObject; content?: string; - onInput(event: SyntheticEvent): void; - onPaste(event: SyntheticEvent): void; + onInput(event: SyntheticEvent | ClipboardEvent): void; + onPaste(event: ClipboardEvent): void; onKeyDown(event: KeyboardEvent): void; setContent(text?: string): void; handleMention: (link: string, text: string, attributes: Attributes) => void; @@ -113,7 +113,7 @@ export function usePlainTextListeners( const enterShouldSend = !useSettingValue("MessageComposerInput.ctrlEnterToSend"); const onInput = useCallback( - (event: SyntheticEvent) => { + (event: SyntheticEvent | ClipboardEvent) => { console.log("<<< handling ", event); if (isDivElement(event.target)) { console.log("<<< setting ", event.target.innerHTML); @@ -127,7 +127,7 @@ export function usePlainTextListeners( ); const onPaste = useCallback( - (event: SyntheticEvent) => { + (event: ClipboardEvent) => { const handled = handleClipboardEvent(event, roomContext, mxClient, eventRelation); if (handled) { event.preventDefault(); // we only handle image pasting manually From 7c1746d71593b0282fd34c73e5afad6a3bf5e659 Mon Sep 17 00:00:00 2001 From: Alun Turner Date: Fri, 9 Jun 2023 09:43:55 +0100 Subject: [PATCH 03/15] fix merge error --- .../views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts b/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts index 863938da7d9..a6e4a4fc1a6 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts @@ -258,6 +258,7 @@ function handleInputEvent(event: InputEvent, send: Send, isCtrlEnterToSend: bool */ export function handleClipboardEvent( clipboardEvent: ClipboardEvent | ReactClipboardEvent | InputEvent, + data: DataTransfer, roomContext: IRoomState, mxClient: MatrixClient, eventRelation?: IEventRelation, From 4fac2c934d91489117af95357c26bb4e0e7d71ce Mon Sep 17 00:00:00 2001 From: Alun Turner Date: Fri, 9 Jun 2023 09:47:54 +0100 Subject: [PATCH 04/15] fix signature change error --- .../views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts b/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts index cf662fefa0a..f15a82ec2c7 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts @@ -128,7 +128,7 @@ export function usePlainTextListeners( const onPaste = useCallback( (event: ClipboardEvent) => { - const handled = handleClipboardEvent(event, roomContext, mxClient, eventRelation); + const handled = handleClipboardEvent(event, event.clipboardData, roomContext, mxClient, eventRelation); if (handled) { event.preventDefault(); // we only handle image pasting manually } else { From 880dbcd8494120dcd441c1ea058f894b58a0c934 Mon Sep 17 00:00:00 2001 From: Alun Turner Date: Fri, 9 Jun 2023 13:50:23 +0100 Subject: [PATCH 05/15] type wrangling --- .../hooks/useInputEventProcessor.ts | 4 ++-- .../hooks/usePlainTextListeners.ts | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts b/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts index a6e4a4fc1a6..847ce8ee3b5 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts @@ -15,7 +15,7 @@ limitations under the License. */ import { Wysiwyg, WysiwygEvent } from "@matrix-org/matrix-wysiwyg"; -import { useCallback, ClipboardEvent as ReactClipboardEvent } from "react"; +import { useCallback } from "react"; import { IEventRelation, MatrixClient } from "matrix-js-sdk/src/matrix"; import { useSettingValue } from "../../../../../hooks/useSettings"; @@ -257,7 +257,7 @@ function handleInputEvent(event: InputEvent, send: Send, isCtrlEnterToSend: bool * @returns - boolean to show if the event was handled or not */ export function handleClipboardEvent( - clipboardEvent: ClipboardEvent | ReactClipboardEvent | InputEvent, + event: ClipboardEvent | InputEvent, data: DataTransfer, roomContext: IRoomState, mxClient: MatrixClient, diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts b/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts index f15a82ec2c7..cb59176ea1b 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { KeyboardEvent, RefObject, SyntheticEvent, ClipboardEvent, useCallback, useRef, useState } from "react"; +import { KeyboardEvent, RefObject, SyntheticEvent, useCallback, useRef, useState } from "react"; import { Attributes, MappedSuggestion } from "@matrix-org/matrix-wysiwyg"; import { IEventRelation } from "matrix-js-sdk/src/matrix"; @@ -68,8 +68,8 @@ export function usePlainTextListeners( ref: RefObject; autocompleteRef: React.RefObject; content?: string; - onInput(event: SyntheticEvent | ClipboardEvent): void; - onPaste(event: ClipboardEvent): void; + onInput(event: SyntheticEvent): void; + onPaste(event: SyntheticEvent): void; onKeyDown(event: KeyboardEvent): void; setContent(text?: string): void; handleMention: (link: string, text: string, attributes: Attributes) => void; @@ -113,7 +113,7 @@ export function usePlainTextListeners( const enterShouldSend = !useSettingValue("MessageComposerInput.ctrlEnterToSend"); const onInput = useCallback( - (event: SyntheticEvent | ClipboardEvent) => { + (event: SyntheticEvent) => { console.log("<<< handling ", event); if (isDivElement(event.target)) { console.log("<<< setting ", event.target.innerHTML); @@ -127,8 +127,14 @@ export function usePlainTextListeners( ); const onPaste = useCallback( - (event: ClipboardEvent) => { - const handled = handleClipboardEvent(event, event.clipboardData, roomContext, mxClient, eventRelation); + (event: SyntheticEvent) => { + const handled = handleClipboardEvent( + event.nativeEvent, + event.nativeEvent.clipboardData, + roomContext, + mxClient, + eventRelation, + ); if (handled) { event.preventDefault(); // we only handle image pasting manually } else { From cadc518b50331726e7fb21d9f94be18a7b280d28 Mon Sep 17 00:00:00 2001 From: Alun Turner Date: Fri, 9 Jun 2023 14:13:36 +0100 Subject: [PATCH 06/15] use onBeforeInput listener --- .../rooms/wysiwyg_composer/components/PlainTextComposer.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx b/src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx index ed53a3f61ea..cf54fa7bef2 100644 --- a/src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx +++ b/src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx @@ -55,6 +55,7 @@ export function PlainTextComposer({ const { ref: editorRef, autocompleteRef, + onBeforeInput, onInput, onPaste, onKeyDown, @@ -78,6 +79,7 @@ export function PlainTextComposer({ className={classNames(className, { [`${className}-focused`]: isFocused })} onFocus={onFocus} onBlur={onFocus} + onBeforeInput={onBeforeInput} onInput={onInput} onPaste={onPaste} onKeyDown={onKeyDown} From da53c266cfb9e29b72ac5c608ceb1341ece5ce1d Mon Sep 17 00:00:00 2001 From: Alun Turner Date: Fri, 9 Jun 2023 14:17:15 +0100 Subject: [PATCH 07/15] add onBeforeInput handler, add logic to onPaste --- .../hooks/usePlainTextListeners.ts | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts b/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts index cb59176ea1b..8b878c9864f 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts @@ -68,8 +68,9 @@ export function usePlainTextListeners( ref: RefObject; autocompleteRef: React.RefObject; content?: string; + onBeforeInput(event: SyntheticEvent): void; onInput(event: SyntheticEvent): void; - onPaste(event: SyntheticEvent): void; + onPaste(event: SyntheticEvent): void; onKeyDown(event: KeyboardEvent): void; setContent(text?: string): void; handleMention: (link: string, text: string, attributes: Attributes) => void; @@ -114,10 +115,7 @@ export function usePlainTextListeners( const enterShouldSend = !useSettingValue("MessageComposerInput.ctrlEnterToSend"); const onInput = useCallback( (event: SyntheticEvent) => { - console.log("<<< handling ", event); if (isDivElement(event.target)) { - console.log("<<< setting ", event.target.innerHTML); - // if enterShouldSend, we do not need to amend the html before setting text const newInnerHTML = enterShouldSend ? event.target.innerHTML : amendInnerHtml(event.target.innerHTML); setText(newInnerHTML); @@ -127,16 +125,32 @@ export function usePlainTextListeners( ); const onPaste = useCallback( - (event: SyntheticEvent) => { - const handled = handleClipboardEvent( - event.nativeEvent, - event.nativeEvent.clipboardData, - roomContext, - mxClient, - eventRelation, - ); - if (handled) { - event.preventDefault(); // we only handle image pasting manually + (event: SyntheticEvent) => { + // this is required to handle edge case image pasting in Safari, see + // https://github.com/vector-im/element-web/issues/25327 and it is caught by the + // `beforeinput` listener attached to the composer + const { nativeEvent } = event; + + // attempt to manually handle image paste events occurring from the clipboard or the special + // case detailed in the above issue + const isClipboardEvent = nativeEvent instanceof ClipboardEvent; + const isSpecialCaseInputEvent = + nativeEvent instanceof InputEvent && + nativeEvent.inputType === "insertFromPaste" && + isNotNull(nativeEvent.dataTransfer); + + const isEventToHandle = isClipboardEvent || isSpecialCaseInputEvent; + + let imagePasteWasHandled = false; + + if (isEventToHandle) { + const data = isClipboardEvent ? nativeEvent.clipboardData : nativeEvent.dataTransfer; + imagePasteWasHandled = handleClipboardEvent(nativeEvent, data, roomContext, mxClient, eventRelation); + } + + // only prevent default behaviour if the image paste event was handled + if (imagePasteWasHandled) { + event.preventDefault(); } else { onInput(event); } @@ -178,6 +192,7 @@ export function usePlainTextListeners( return { ref, autocompleteRef, + onBeforeInput: onPaste, onInput, onPaste, onKeyDown, From efb9bfd3b71fda94ba9c9d9b4aa7204a175c0507 Mon Sep 17 00:00:00 2001 From: Alun Turner Date: Fri, 9 Jun 2023 14:23:06 +0100 Subject: [PATCH 08/15] fix type error --- .../rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts b/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts index 847ce8ee3b5..27f40880140 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts @@ -258,7 +258,7 @@ function handleInputEvent(event: InputEvent, send: Send, isCtrlEnterToSend: bool */ export function handleClipboardEvent( event: ClipboardEvent | InputEvent, - data: DataTransfer, + data: DataTransfer | null, roomContext: IRoomState, mxClient: MatrixClient, eventRelation?: IEventRelation, From 1e55f9effa297b647fcaa1dbc1d282864e4c6892 Mon Sep 17 00:00:00 2001 From: Alun Turner Date: Fri, 9 Jun 2023 16:12:37 +0100 Subject: [PATCH 09/15] bring plain text listeners in line with useInputEventProcessor --- .../rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts b/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts index 8b878c9864f..15671e74d28 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts @@ -134,16 +134,16 @@ export function usePlainTextListeners( // attempt to manually handle image paste events occurring from the clipboard or the special // case detailed in the above issue const isClipboardEvent = nativeEvent instanceof ClipboardEvent; - const isSpecialCaseInputEvent = + const isInputEventForClipboard = nativeEvent instanceof InputEvent && nativeEvent.inputType === "insertFromPaste" && isNotNull(nativeEvent.dataTransfer); - const isEventToHandle = isClipboardEvent || isSpecialCaseInputEvent; + const shouldHandleAsClipboardEvent = isClipboardEvent || isInputEventForClipboard; let imagePasteWasHandled = false; - if (isEventToHandle) { + if (shouldHandleAsClipboardEvent) { const data = isClipboardEvent ? nativeEvent.clipboardData : nativeEvent.dataTransfer; imagePasteWasHandled = handleClipboardEvent(nativeEvent, data, roomContext, mxClient, eventRelation); } From 5fcb69d930d885aa679b74b1cbc74b1428a1f042 Mon Sep 17 00:00:00 2001 From: Alun Turner Date: Fri, 9 Jun 2023 16:36:27 +0100 Subject: [PATCH 10/15] extract common function to util file, move tests --- .../hooks/useInputEventProcessor.ts | 103 +--------------- .../hooks/usePlainTextListeners.ts | 18 +-- .../rooms/wysiwyg_composer/hooks/utils.ts | 110 ++++++++++++++++++ ...EventProcessor-test.tsx => utils-test.tsx} | 2 +- 4 files changed, 118 insertions(+), 115 deletions(-) rename test/components/views/rooms/wysiwyg_composer/hooks/{useInputEventProcessor-test.tsx => utils-test.tsx} (99%) diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts b/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts index 27f40880140..a9cfa2966e1 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts @@ -33,10 +33,7 @@ import { isCaretAtEnd, isCaretAtStart } from "../utils/selection"; import { getEventsFromEditorStateTransfer, getEventsFromRoom } from "../utils/event"; import { endEditing } from "../utils/editing"; import Autocomplete from "../../Autocomplete"; -import { handleEventWithAutocomplete } from "./utils"; -import ContentMessages from "../../../../../ContentMessages"; -import { getBlobSafeMimeType } from "../../../../../utils/blobs"; -import { isNotNull } from "../../../../../Typeguards"; +import { handleClipboardEvent, handleEventWithAutocomplete, isEventToHandleAsClipboardEvent } from "./utils"; export function useInputEventProcessor( onSend: () => void, @@ -61,17 +58,8 @@ export function useInputEventProcessor( onSend(); }; - // this is required to handle edge case image pasting in Safari, see - // https://github.com/vector-im/element-web/issues/25327 and it is caught by the - // `beforeinput` listener attached to the composer - const isInputEventForClipboard = - event instanceof InputEvent && event.inputType === "insertFromPaste" && isNotNull(event.dataTransfer); - const isClipboardEvent = event instanceof ClipboardEvent; - - const shouldHandleAsClipboardEvent = isClipboardEvent || isInputEventForClipboard; - - if (shouldHandleAsClipboardEvent) { - const data = isClipboardEvent ? event.clipboardData : event.dataTransfer; + if (isEventToHandleAsClipboardEvent(event)) { + const data = event instanceof ClipboardEvent ? event.clipboardData : event.dataTransfer; const handled = handleClipboardEvent(event, data, roomContext, mxClient, eventRelation); return handled ? null : event; } @@ -244,88 +232,3 @@ function handleInputEvent(event: InputEvent, send: Send, isCtrlEnterToSend: bool return event; } - -/** - * Takes an event and handles image pasting. Returns a boolean to indicate if it has handled - * the event or not. Must accept either clipboard or input events in order to prevent issue: - * https://github.com/vector-im/element-web/issues/25327 - * - * @param event - event to process - * @param roomContext - room in which the event occurs - * @param mxClient - current matrix client - * @param eventRelation - used to send the event to the correct place eg timeline vs thread - * @returns - boolean to show if the event was handled or not - */ -export function handleClipboardEvent( - event: ClipboardEvent | InputEvent, - data: DataTransfer | null, - roomContext: IRoomState, - mxClient: MatrixClient, - eventRelation?: IEventRelation, -): boolean { - // Logic in this function follows that of `SendMessageComposer.onPaste` - const { room, timelineRenderingType, replyToEvent } = roomContext; - - function handleError(error: unknown): void { - if (error instanceof Error) { - console.log(error.message); - } else if (typeof error === "string") { - console.log(error); - } - } - - if (event.type !== "paste" || data === null || room === undefined) { - return false; - } - - // Prioritize text on the clipboard over files if RTF is present as Office on macOS puts a bitmap - // in the clipboard as well as the content being copied. Modern versions of Office seem to not do this anymore. - // We check text/rtf instead of text/plain as when copy+pasting a file from Finder or Gnome Image Viewer - // it puts the filename in as text/plain which we want to ignore. - if (data.files.length && !data.types.includes("text/rtf")) { - ContentMessages.sharedInstance() - .sendContentListToRoom(Array.from(data.files), room.roomId, eventRelation, mxClient, timelineRenderingType) - .catch(handleError); - return true; - } - - // Safari `Insert from iPhone or iPad` - // data.getData("text/html") returns a string like: - if (data.types.includes("text/html")) { - const imgElementStr = data.getData("text/html"); - const parser = new DOMParser(); - const imgDoc = parser.parseFromString(imgElementStr, "text/html"); - - if ( - imgDoc.getElementsByTagName("img").length !== 1 || - !imgDoc.querySelector("img")?.src.startsWith("blob:") || - imgDoc.childNodes.length !== 1 - ) { - handleError("Failed to handle pasted content as Safari inserted content"); - return false; - } - const imgSrc = imgDoc.querySelector("img")!.src; - - fetch(imgSrc) - .then((response) => { - response - .blob() - .then((imgBlob) => { - const type = imgBlob.type; - const safetype = getBlobSafeMimeType(type); - const ext = type.split("/")[1]; - const parts = response.url.split("/"); - const filename = parts[parts.length - 1]; - const file = new File([imgBlob], filename + "." + ext, { type: safetype }); - ContentMessages.sharedInstance() - .sendContentToRoom(file, room.roomId, eventRelation, mxClient, replyToEvent) - .catch(handleError); - }) - .catch(handleError); - }) - .catch(handleError); - return true; - } - - return false; -} diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts b/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts index 15671e74d28..ff4c39f4232 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts @@ -21,10 +21,9 @@ import { IEventRelation } from "matrix-js-sdk/src/matrix"; import { useSettingValue } from "../../../../../hooks/useSettings"; import { IS_MAC, Key } from "../../../../../Keyboard"; import Autocomplete from "../../Autocomplete"; -import { handleEventWithAutocomplete } from "./utils"; +import { handleClipboardEvent, handleEventWithAutocomplete, isEventToHandleAsClipboardEvent } from "./utils"; import { useSuggestion } from "./useSuggestion"; import { isNotNull, isNotUndefined } from "../../../../../Typeguards"; -import { handleClipboardEvent } from "./useInputEventProcessor"; import { useRoomContext } from "../../../../../contexts/RoomContext"; import { useMatrixClientContext } from "../../../../../contexts/MatrixClientContext"; @@ -131,20 +130,11 @@ export function usePlainTextListeners( // `beforeinput` listener attached to the composer const { nativeEvent } = event; - // attempt to manually handle image paste events occurring from the clipboard or the special - // case detailed in the above issue - const isClipboardEvent = nativeEvent instanceof ClipboardEvent; - const isInputEventForClipboard = - nativeEvent instanceof InputEvent && - nativeEvent.inputType === "insertFromPaste" && - isNotNull(nativeEvent.dataTransfer); - - const shouldHandleAsClipboardEvent = isClipboardEvent || isInputEventForClipboard; - let imagePasteWasHandled = false; - if (shouldHandleAsClipboardEvent) { - const data = isClipboardEvent ? nativeEvent.clipboardData : nativeEvent.dataTransfer; + if (isEventToHandleAsClipboardEvent(nativeEvent)) { + const data = + nativeEvent instanceof ClipboardEvent ? nativeEvent.clipboardData : nativeEvent.dataTransfer; imagePasteWasHandled = handleClipboardEvent(nativeEvent, data, roomContext, mxClient, eventRelation); } diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/utils.ts b/src/components/views/rooms/wysiwyg_composer/hooks/utils.ts index 636b5d2bf2a..2c18502cab6 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/utils.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/utils.ts @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { IEventRelation, MatrixClient } from "matrix-js-sdk/src/matrix"; import { MutableRefObject, RefObject } from "react"; import { TimelineRenderingType } from "../../../../../contexts/RoomContext"; @@ -21,6 +22,10 @@ import { IRoomState } from "../../../../structures/RoomView"; import Autocomplete from "../../Autocomplete"; import { getKeyBindingsManager } from "../../../../../KeyBindingsManager"; import { KeyBindingAction } from "../../../../../accessibility/KeyboardShortcuts"; +import { getBlobSafeMimeType } from "../../../../../utils/blobs"; +import ContentMessages from "../../../../../ContentMessages"; +import { isNotNull } from "../../../../../Typeguards"; +import { WysiwygEvent, WysiwygInputEvent } from "@matrix-org/matrix-wysiwyg"; export function focusComposer( composerElement: MutableRefObject, @@ -110,3 +115,108 @@ export function handleEventWithAutocomplete( return handled; } + +/** + * Takes an event and handles image pasting. Returns a boolean to indicate if it has handled + * the event or not. Must accept either clipboard or input events in order to prevent issue: + * https://github.com/vector-im/element-web/issues/25327 + * + * @param event - event to process + * @param roomContext - room in which the event occurs + * @param mxClient - current matrix client + * @param eventRelation - used to send the event to the correct place eg timeline vs thread + * @returns - boolean to show if the event was handled or not + */ +export function handleClipboardEvent( + event: ClipboardEvent | InputEvent, + data: DataTransfer | null, + roomContext: IRoomState, + mxClient: MatrixClient, + eventRelation?: IEventRelation, +): boolean { + // Logic in this function follows that of `SendMessageComposer.onPaste` + const { room, timelineRenderingType, replyToEvent } = roomContext; + + function handleError(error: unknown): void { + if (error instanceof Error) { + console.log(error.message); + } else if (typeof error === "string") { + console.log(error); + } + } + + if (event.type !== "paste" || data === null || room === undefined) { + return false; + } + + // Prioritize text on the clipboard over files if RTF is present as Office on macOS puts a bitmap + // in the clipboard as well as the content being copied. Modern versions of Office seem to not do this anymore. + // We check text/rtf instead of text/plain as when copy+pasting a file from Finder or Gnome Image Viewer + // it puts the filename in as text/plain which we want to ignore. + if (data.files.length && !data.types.includes("text/rtf")) { + ContentMessages.sharedInstance() + .sendContentListToRoom(Array.from(data.files), room.roomId, eventRelation, mxClient, timelineRenderingType) + .catch(handleError); + return true; + } + + // Safari `Insert from iPhone or iPad` + // data.getData("text/html") returns a string like: + if (data.types.includes("text/html")) { + const imgElementStr = data.getData("text/html"); + const parser = new DOMParser(); + const imgDoc = parser.parseFromString(imgElementStr, "text/html"); + + if ( + imgDoc.getElementsByTagName("img").length !== 1 || + !imgDoc.querySelector("img")?.src.startsWith("blob:") || + imgDoc.childNodes.length !== 1 + ) { + handleError("Failed to handle pasted content as Safari inserted content"); + return false; + } + const imgSrc = imgDoc.querySelector("img")!.src; + + fetch(imgSrc) + .then((response) => { + response + .blob() + .then((imgBlob) => { + const type = imgBlob.type; + const safetype = getBlobSafeMimeType(type); + const ext = type.split("/")[1]; + const parts = response.url.split("/"); + const filename = parts[parts.length - 1]; + const file = new File([imgBlob], filename + "." + ext, { type: safetype }); + ContentMessages.sharedInstance() + .sendContentToRoom(file, room.roomId, eventRelation, mxClient, replyToEvent) + .catch(handleError); + }) + .catch(handleError); + }) + .catch(handleError); + return true; + } + + return false; +} + +/** + * Util to determine if an input event or clipboard event must be handled as a clipboard event. + * Required due to https://github.com/vector-im/element-web/issues/25327, certain paste events + * must be detected with an onBeforeInput handler and will be input events + * @param event - the event to test + * @returns - true if event should be handled as a clipboard event + */ +export function isEventToHandleAsClipboardEvent( + event: WysiwygEvent | InputEvent | ClipboardEvent, +): event is InputEvent | ClipboardEvent { + // this is required to handle edge case image pasting in Safari, see + // https://github.com/vector-im/element-web/issues/25327 and it is caught by the + // `beforeinput` listener attached to the composer + const isInputEventForClipboard = + event instanceof InputEvent && event.inputType === "insertFromPaste" && isNotNull(event.dataTransfer); + const isClipboardEvent = event instanceof ClipboardEvent; + + return isClipboardEvent || isInputEventForClipboard; +} diff --git a/test/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor-test.tsx b/test/components/views/rooms/wysiwyg_composer/hooks/utils-test.tsx similarity index 99% rename from test/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor-test.tsx rename to test/components/views/rooms/wysiwyg_composer/hooks/utils-test.tsx index 8d6f9d19cc4..783971b9708 100644 --- a/test/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor-test.tsx +++ b/test/components/views/rooms/wysiwyg_composer/hooks/utils-test.tsx @@ -16,11 +16,11 @@ limitations under the License. import { IEventRelation, MatrixEvent } from "matrix-js-sdk/src/matrix"; import { waitFor } from "@testing-library/react"; -import { handleClipboardEvent } from "../../../../../../src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor"; import { TimelineRenderingType } from "../../../../../../src/contexts/RoomContext"; import { mkStubRoom, stubClient } from "../../../../../test-utils"; import ContentMessages from "../../../../../../src/ContentMessages"; import { IRoomState } from "../../../../../../src/components/structures/RoomView"; +import { handleClipboardEvent } from "../../../../../../src/components/views/rooms/wysiwyg_composer/hooks/utils"; const mockClient = stubClient(); const mockRoom = mkStubRoom("mock room", "mock room", mockClient); From a693a22dbd2542c048ed8215b9c1e084b273b197 Mon Sep 17 00:00:00 2001 From: Alun Turner Date: Fri, 9 Jun 2023 16:50:34 +0100 Subject: [PATCH 11/15] tidy comment --- .../rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts b/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts index ff4c39f4232..21b43126bb3 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts @@ -125,11 +125,7 @@ export function usePlainTextListeners( const onPaste = useCallback( (event: SyntheticEvent) => { - // this is required to handle edge case image pasting in Safari, see - // https://github.com/vector-im/element-web/issues/25327 and it is caught by the - // `beforeinput` listener attached to the composer const { nativeEvent } = event; - let imagePasteWasHandled = false; if (isEventToHandleAsClipboardEvent(nativeEvent)) { @@ -138,7 +134,7 @@ export function usePlainTextListeners( imagePasteWasHandled = handleClipboardEvent(nativeEvent, data, roomContext, mxClient, eventRelation); } - // only prevent default behaviour if the image paste event was handled + // prevent default behaviour and skip call to onInput if the image paste event was handled if (imagePasteWasHandled) { event.preventDefault(); } else { From f74a2879894911c91b881ebb77ed7a2263b5707a Mon Sep 17 00:00:00 2001 From: Alun Turner Date: Fri, 9 Jun 2023 16:53:46 +0100 Subject: [PATCH 12/15] tidy comments --- .../views/rooms/wysiwyg_composer/hooks/utils.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/utils.ts b/src/components/views/rooms/wysiwyg_composer/hooks/utils.ts index 2c18502cab6..55f88af84b5 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/utils.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/utils.ts @@ -122,6 +122,7 @@ export function handleEventWithAutocomplete( * https://github.com/vector-im/element-web/issues/25327 * * @param event - event to process + * @param data - data from the event to process * @param roomContext - room in which the event occurs * @param mxClient - current matrix client * @param eventRelation - used to send the event to the correct place eg timeline vs thread @@ -203,17 +204,16 @@ export function handleClipboardEvent( /** * Util to determine if an input event or clipboard event must be handled as a clipboard event. - * Required due to https://github.com/vector-im/element-web/issues/25327, certain paste events - * must be detected with an onBeforeInput handler and will be input events - * @param event - the event to test + * Due to https://github.com/vector-im/element-web/issues/25327, certain paste events + * must be listenened for with an onBeforeInput handler and so will be caught as input events. + * + * @param event - the event to test, can be a WysiwygEvent if it comes from the rich text editor, or + * input of clipboard events if from the plain text editor * @returns - true if event should be handled as a clipboard event */ export function isEventToHandleAsClipboardEvent( event: WysiwygEvent | InputEvent | ClipboardEvent, ): event is InputEvent | ClipboardEvent { - // this is required to handle edge case image pasting in Safari, see - // https://github.com/vector-im/element-web/issues/25327 and it is caught by the - // `beforeinput` listener attached to the composer const isInputEventForClipboard = event instanceof InputEvent && event.inputType === "insertFromPaste" && isNotNull(event.dataTransfer); const isClipboardEvent = event instanceof ClipboardEvent; From 25c365a1742708ab4070f5d55a0a26e3db0bdc6c Mon Sep 17 00:00:00 2001 From: Alun Turner Date: Fri, 9 Jun 2023 16:57:02 +0100 Subject: [PATCH 13/15] fix typo --- src/components/views/rooms/wysiwyg_composer/hooks/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/utils.ts b/src/components/views/rooms/wysiwyg_composer/hooks/utils.ts index 55f88af84b5..f22870824c3 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/utils.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/utils.ts @@ -208,7 +208,7 @@ export function handleClipboardEvent( * must be listenened for with an onBeforeInput handler and so will be caught as input events. * * @param event - the event to test, can be a WysiwygEvent if it comes from the rich text editor, or - * input of clipboard events if from the plain text editor + * input or clipboard events if from the plain text editor * @returns - true if event should be handled as a clipboard event */ export function isEventToHandleAsClipboardEvent( From d9c73e816abc64017876b4c3da4b42ae0b68a630 Mon Sep 17 00:00:00 2001 From: Alun Turner Date: Fri, 9 Jun 2023 17:13:28 +0100 Subject: [PATCH 14/15] add util tests --- .../rooms/wysiwyg_composer/hooks/utils.ts | 4 +-- .../wysiwyg_composer/hooks/utils-test.tsx | 28 ++++++++++++++++++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/utils.ts b/src/components/views/rooms/wysiwyg_composer/hooks/utils.ts index f22870824c3..f95405c3bfd 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/utils.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/utils.ts @@ -14,8 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { IEventRelation, MatrixClient } from "matrix-js-sdk/src/matrix"; import { MutableRefObject, RefObject } from "react"; +import { IEventRelation, MatrixClient } from "matrix-js-sdk/src/matrix"; +import { WysiwygEvent } from "@matrix-org/matrix-wysiwyg"; import { TimelineRenderingType } from "../../../../../contexts/RoomContext"; import { IRoomState } from "../../../../structures/RoomView"; @@ -25,7 +26,6 @@ import { KeyBindingAction } from "../../../../../accessibility/KeyboardShortcuts import { getBlobSafeMimeType } from "../../../../../utils/blobs"; import ContentMessages from "../../../../../ContentMessages"; import { isNotNull } from "../../../../../Typeguards"; -import { WysiwygEvent, WysiwygInputEvent } from "@matrix-org/matrix-wysiwyg"; export function focusComposer( composerElement: MutableRefObject, diff --git a/test/components/views/rooms/wysiwyg_composer/hooks/utils-test.tsx b/test/components/views/rooms/wysiwyg_composer/hooks/utils-test.tsx index 783971b9708..81489e3beb4 100644 --- a/test/components/views/rooms/wysiwyg_composer/hooks/utils-test.tsx +++ b/test/components/views/rooms/wysiwyg_composer/hooks/utils-test.tsx @@ -20,7 +20,10 @@ import { TimelineRenderingType } from "../../../../../../src/contexts/RoomContex import { mkStubRoom, stubClient } from "../../../../../test-utils"; import ContentMessages from "../../../../../../src/ContentMessages"; import { IRoomState } from "../../../../../../src/components/structures/RoomView"; -import { handleClipboardEvent } from "../../../../../../src/components/views/rooms/wysiwyg_composer/hooks/utils"; +import { + handleClipboardEvent, + isEventToHandleAsClipboardEvent, +} from "../../../../../../src/components/views/rooms/wysiwyg_composer/hooks/utils"; const mockClient = stubClient(); const mockRoom = mkStubRoom("mock room", "mock room", mockClient); @@ -285,3 +288,26 @@ describe("handleClipboardEvent", () => { expect(output).toBe(true); }); }); + +describe("isEventToHandleAsClipboardEvent", () => { + it("returns true for ClipboardEvent", () => { + const input = new ClipboardEvent("clipboard"); + expect(isEventToHandleAsClipboardEvent(input)).toBe(true); + }); + + it("returns true for special case input", () => { + const input = new InputEvent("insertFromPaste", { inputType: "insertFromPaste" }); + Object.assign(input, { dataTransfer: "not null" }); + expect(isEventToHandleAsClipboardEvent(input)).toBe(true); + }); + + it("returns false for regular InputEvent", () => { + const input = new InputEvent("input"); + expect(isEventToHandleAsClipboardEvent(input)).toBe(false); + }); + + it("returns false for other input", () => { + const input = new KeyboardEvent("keyboard"); + expect(isEventToHandleAsClipboardEvent(input)).toBe(false); + }); +}); From a5642e2cd20cd5c377ddcc99273ac0ea28218716 Mon Sep 17 00:00:00 2001 From: Alun Turner Date: Mon, 12 Jun 2023 10:14:19 +0100 Subject: [PATCH 15/15] add text paste test --- .../components/PlainTextComposer-test.tsx | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/components/views/rooms/wysiwyg_composer/components/PlainTextComposer-test.tsx b/test/components/views/rooms/wysiwyg_composer/components/PlainTextComposer-test.tsx index 9277ecb16c7..cb7104f8e4c 100644 --- a/test/components/views/rooms/wysiwyg_composer/components/PlainTextComposer-test.tsx +++ b/test/components/views/rooms/wysiwyg_composer/components/PlainTextComposer-test.tsx @@ -290,4 +290,16 @@ describe("PlainTextComposer", () => { expect(screen.getByTestId("autocomplete-wrapper")).toBeInTheDocument(); }); + + it("Should allow pasting of text values", async () => { + customRender(); + + const textBox = screen.getByRole("textbox"); + + await userEvent.click(textBox); + await userEvent.type(textBox, "hello"); + await userEvent.paste(" world"); + + expect(textBox).toHaveTextContent("hello world"); + }); });