Skip to content
This repository has been archived by the owner on Sep 11, 2024. It is now read-only.

Commit

Permalink
Allow image pasting in plain mode in RTE (#11056)
Browse files Browse the repository at this point in the history 
* get rough funcitonality working

* try to tidy up types

* fix merge error

* fix signature change error

* type wrangling

* use onBeforeInput listener

* add onBeforeInput handler, add logic to onPaste

* fix type error

* bring plain text listeners in line with useInputEventProcessor

* extract common function to util file, move tests

* tidy comment

* tidy comments

* fix typo

* add util tests

* add text paste test
  • Loading branch information
@artcodespace
artcodespace authored Jun 12, 2023
1 parent 47ab99f commit e32823e
Show file tree
Hide file tree
Showing 6 changed files with 188 additions and 104 deletions.
5 changes: 4 additions & 1 deletion src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,12 @@ export function PlainTextComposer({
initialContent,
leftComponent,
rightComponent,
eventRelation,
}: PlainTextComposerProps): JSX.Element {
const {
ref: editorRef,
autocompleteRef,
onBeforeInput,
onInput,
onPaste,
onKeyDown,
Expand All @@ -63,7 +65,7 @@ export function PlainTextComposer({
onSelect,
handleCommand,
handleMention,
} = usePlainTextListeners(initialContent, onChange, onSend);
} = usePlainTextListeners(initialContent, onChange, onSend, eventRelation);

const composerFunctions = useComposerFunctions(editorRef, setContent);
usePlainTextInitialization(initialContent, editorRef);
Expand All @@ -77,6 +79,7 @@ export function PlainTextComposer({
className={classNames(className, { [`${className}-focused`]: isFocused })}
onFocus={onFocus}
onBlur={onFocus}
onBeforeInput={onBeforeInput}
onInput={onInput}
onPaste={onPaste}
onKeyDown={onKeyDown}
Expand Down
103 changes: 3 additions & 100 deletions src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,7 @@ import { isCaretAtEnd, isCaretAtStart } from "../utils/selection";
import { getEventsFromEditorStateTransfer, getEventsFromRoom } from "../utils/event";
import { endEditing } from "../utils/editing";
import Autocomplete from "../../Autocomplete";
import { handleEventWithAutocomplete } from "./utils";
import ContentMessages from "../../../../../ContentMessages";
import { getBlobSafeMimeType } from "../../../../../utils/blobs";
import { isNotNull } from "../../../../../Typeguards";
import { handleClipboardEvent, handleEventWithAutocomplete, isEventToHandleAsClipboardEvent } from "./utils";

export function useInputEventProcessor(
onSend: () => void,
Expand All @@ -61,17 +58,8 @@ export function useInputEventProcessor(
onSend();
};

// this is required to handle edge case image pasting in Safari, see
// https://github.com/vector-im/element-web/issues/25327 and it is caught by the
// `beforeinput` listener attached to the composer
const isInputEventForClipboard =
event instanceof InputEvent && event.inputType === "insertFromPaste" && isNotNull(event.dataTransfer);
const isClipboardEvent = event instanceof ClipboardEvent;

const shouldHandleAsClipboardEvent = isClipboardEvent || isInputEventForClipboard;

if (shouldHandleAsClipboardEvent) {
const data = isClipboardEvent ? event.clipboardData : event.dataTransfer;
if (isEventToHandleAsClipboardEvent(event)) {
const data = event instanceof ClipboardEvent ? event.clipboardData : event.dataTransfer;
const handled = handleClipboardEvent(event, data, roomContext, mxClient, eventRelation);
return handled ? null : event;
}
Expand Down Expand Up @@ -244,88 +232,3 @@ function handleInputEvent(event: InputEvent, send: Send, isCtrlEnterToSend: bool

return event;
}

/**
* Takes an event and handles image pasting. Returns a boolean to indicate if it has handled
* the event or not. Must accept either clipboard or input events in order to prevent issue:
* https://github.com/vector-im/element-web/issues/25327
*
* @param event - event to process
* @param roomContext - room in which the event occurs
* @param mxClient - current matrix client
* @param eventRelation - used to send the event to the correct place eg timeline vs thread
* @returns - boolean to show if the event was handled or not
*/
export function handleClipboardEvent(
event: ClipboardEvent | InputEvent,
data: DataTransfer | null,
roomContext: IRoomState,
mxClient: MatrixClient,
eventRelation?: IEventRelation,
): boolean {
// Logic in this function follows that of `SendMessageComposer.onPaste`
const { room, timelineRenderingType, replyToEvent } = roomContext;

function handleError(error: unknown): void {
if (error instanceof Error) {
console.log(error.message);
} else if (typeof error === "string") {
console.log(error);
}
}

if (event.type !== "paste" || data === null || room === undefined) {
return false;
}

// Prioritize text on the clipboard over files if RTF is present as Office on macOS puts a bitmap
// in the clipboard as well as the content being copied. Modern versions of Office seem to not do this anymore.
// We check text/rtf instead of text/plain as when copy+pasting a file from Finder or Gnome Image Viewer
// it puts the filename in as text/plain which we want to ignore.
if (data.files.length && !data.types.includes("text/rtf")) {
ContentMessages.sharedInstance()
.sendContentListToRoom(Array.from(data.files), room.roomId, eventRelation, mxClient, timelineRenderingType)
.catch(handleError);
return true;
}

// Safari `Insert from iPhone or iPad`
// data.getData("text/html") returns a string like: <img src="blob:https://...">
if (data.types.includes("text/html")) {
const imgElementStr = data.getData("text/html");
const parser = new DOMParser();
const imgDoc = parser.parseFromString(imgElementStr, "text/html");

if (
imgDoc.getElementsByTagName("img").length !== 1 ||
!imgDoc.querySelector("img")?.src.startsWith("blob:") ||
imgDoc.childNodes.length !== 1
) {
handleError("Failed to handle pasted content as Safari inserted content");
return false;
}
const imgSrc = imgDoc.querySelector("img")!.src;

fetch(imgSrc)
.then((response) => {
response
.blob()
.then((imgBlob) => {
const type = imgBlob.type;
const safetype = getBlobSafeMimeType(type);
const ext = type.split("/")[1];
const parts = response.url.split("/");
const filename = parts[parts.length - 1];
const file = new File([imgBlob], filename + "." + ext, { type: safetype });
ContentMessages.sharedInstance()
.sendContentToRoom(file, room.roomId, eventRelation, mxClient, replyToEvent)
.catch(handleError);
})
.catch(handleError);
})
.catch(handleError);
return true;
}

return false;
}
34 changes: 32 additions & 2 deletions src/components/views/rooms/wysiwyg_composer/hooks/usePlainTextListeners.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,16 @@ limitations under the License.

import { KeyboardEvent, RefObject, SyntheticEvent, useCallback, useRef, useState } from "react";
import { Attributes, MappedSuggestion } from "@matrix-org/matrix-wysiwyg";
import { IEventRelation } from "matrix-js-sdk/src/matrix";

import { useSettingValue } from "../../../../../hooks/useSettings";
import { IS_MAC, Key } from "../../../../../Keyboard";
import Autocomplete from "../../Autocomplete";
import { handleEventWithAutocomplete } from "./utils";
import { handleClipboardEvent, handleEventWithAutocomplete, isEventToHandleAsClipboardEvent } from "./utils";
import { useSuggestion } from "./useSuggestion";
import { isNotNull, isNotUndefined } from "../../../../../Typeguards";
import { useRoomContext } from "../../../../../contexts/RoomContext";
import { useMatrixClientContext } from "../../../../../contexts/MatrixClientContext";

function isDivElement(target: EventTarget): target is HTMLDivElement {
return target instanceof HTMLDivElement;
Expand Down Expand Up @@ -59,10 +62,12 @@ export function usePlainTextListeners(
initialContent?: string,
onChange?: (content: string) => void,
onSend?: () => void,
eventRelation?: IEventRelation,
): {
ref: RefObject<HTMLDivElement>;
autocompleteRef: React.RefObject<Autocomplete>;
content?: string;
onBeforeInput(event: SyntheticEvent<HTMLDivElement, InputEvent | ClipboardEvent>): void;
onInput(event: SyntheticEvent<HTMLDivElement, InputEvent | ClipboardEvent>): void;
onPaste(event: SyntheticEvent<HTMLDivElement, InputEvent | ClipboardEvent>): void;
onKeyDown(event: KeyboardEvent<HTMLDivElement>): void;
Expand All @@ -72,6 +77,9 @@ export function usePlainTextListeners(
onSelect: (event: SyntheticEvent<HTMLDivElement>) => void;
suggestion: MappedSuggestion | null;
} {
const roomContext = useRoomContext();
const mxClient = useMatrixClientContext();

const ref = useRef<HTMLDivElement | null>(null);
const autocompleteRef = useRef<Autocomplete | null>(null);
const [content, setContent] = useState<string | undefined>(initialContent);
Expand Down Expand Up @@ -115,6 +123,27 @@ export function usePlainTextListeners(
[setText, enterShouldSend],
);

const onPaste = useCallback(
(event: SyntheticEvent<HTMLDivElement, InputEvent | ClipboardEvent>) => {
const { nativeEvent } = event;
let imagePasteWasHandled = false;

if (isEventToHandleAsClipboardEvent(nativeEvent)) {
const data =
nativeEvent instanceof ClipboardEvent ? nativeEvent.clipboardData : nativeEvent.dataTransfer;
imagePasteWasHandled = handleClipboardEvent(nativeEvent, data, roomContext, mxClient, eventRelation);
}

// prevent default behaviour and skip call to onInput if the image paste event was handled
if (imagePasteWasHandled) {
event.preventDefault();
} else {
onInput(event);
}
},
[eventRelation, mxClient, onInput, roomContext],
);

const onKeyDown = useCallback(
(event: KeyboardEvent<HTMLDivElement>) => {
// we need autocomplete to take priority when it is open for using enter to select
Expand Down Expand Up @@ -149,8 +178,9 @@ export function usePlainTextListeners(
return {
ref,
autocompleteRef,
onBeforeInput: onPaste,
onInput,
onPaste: onInput,
onPaste,
onKeyDown,
content,
setContent: setText,
Expand Down
110 changes: 110 additions & 0 deletions src/components/views/rooms/wysiwyg_composer/hooks/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,17 @@ limitations under the License.
*/

import { MutableRefObject, RefObject } from "react";
import { IEventRelation, MatrixClient } from "matrix-js-sdk/src/matrix";
import { WysiwygEvent } from "@matrix-org/matrix-wysiwyg";

import { TimelineRenderingType } from "../../../../../contexts/RoomContext";
import { IRoomState } from "../../../../structures/RoomView";
import Autocomplete from "../../Autocomplete";
import { getKeyBindingsManager } from "../../../../../KeyBindingsManager";
import { KeyBindingAction } from "../../../../../accessibility/KeyboardShortcuts";
import { getBlobSafeMimeType } from "../../../../../utils/blobs";
import ContentMessages from "../../../../../ContentMessages";
import { isNotNull } from "../../../../../Typeguards";

export function focusComposer(
composerElement: MutableRefObject<HTMLElement | null>,
Expand Down Expand Up @@ -110,3 +115,108 @@ export function handleEventWithAutocomplete(

return handled;
}

/**
* Takes an event and handles image pasting. Returns a boolean to indicate if it has handled
* the event or not. Must accept either clipboard or input events in order to prevent issue:
* https://github.com/vector-im/element-web/issues/25327
*
* @param event - event to process
* @param data - data from the event to process
* @param roomContext - room in which the event occurs
* @param mxClient - current matrix client
* @param eventRelation - used to send the event to the correct place eg timeline vs thread
* @returns - boolean to show if the event was handled or not
*/
export function handleClipboardEvent(
event: ClipboardEvent | InputEvent,
data: DataTransfer | null,
roomContext: IRoomState,
mxClient: MatrixClient,
eventRelation?: IEventRelation,
): boolean {
// Logic in this function follows that of `SendMessageComposer.onPaste`
const { room, timelineRenderingType, replyToEvent } = roomContext;

function handleError(error: unknown): void {
if (error instanceof Error) {
console.log(error.message);
} else if (typeof error === "string") {
console.log(error);
}
}

if (event.type !== "paste" || data === null || room === undefined) {
return false;
}

// Prioritize text on the clipboard over files if RTF is present as Office on macOS puts a bitmap
// in the clipboard as well as the content being copied. Modern versions of Office seem to not do this anymore.
// We check text/rtf instead of text/plain as when copy+pasting a file from Finder or Gnome Image Viewer
// it puts the filename in as text/plain which we want to ignore.
if (data.files.length && !data.types.includes("text/rtf")) {
ContentMessages.sharedInstance()
.sendContentListToRoom(Array.from(data.files), room.roomId, eventRelation, mxClient, timelineRenderingType)
.catch(handleError);
return true;
}

// Safari `Insert from iPhone or iPad`
// data.getData("text/html") returns a string like: <img src="blob:https://...">
if (data.types.includes("text/html")) {
const imgElementStr = data.getData("text/html");
const parser = new DOMParser();
const imgDoc = parser.parseFromString(imgElementStr, "text/html");

if (
imgDoc.getElementsByTagName("img").length !== 1 ||
!imgDoc.querySelector("img")?.src.startsWith("blob:") ||
imgDoc.childNodes.length !== 1
) {
handleError("Failed to handle pasted content as Safari inserted content");
return false;
}
const imgSrc = imgDoc.querySelector("img")!.src;

fetch(imgSrc)
.then((response) => {
response
.blob()
.then((imgBlob) => {
const type = imgBlob.type;
const safetype = getBlobSafeMimeType(type);
const ext = type.split("/")[1];
const parts = response.url.split("/");
const filename = parts[parts.length - 1];
const file = new File([imgBlob], filename + "." + ext, { type: safetype });
ContentMessages.sharedInstance()
.sendContentToRoom(file, room.roomId, eventRelation, mxClient, replyToEvent)
.catch(handleError);
})
.catch(handleError);
})
.catch(handleError);
return true;
}

return false;
}

/**
* Util to determine if an input event or clipboard event must be handled as a clipboard event.
* Due to https://github.com/vector-im/element-web/issues/25327, certain paste events
* must be listenened for with an onBeforeInput handler and so will be caught as input events.
*
* @param event - the event to test, can be a WysiwygEvent if it comes from the rich text editor, or
* input or clipboard events if from the plain text editor
* @returns - true if event should be handled as a clipboard event
*/
export function isEventToHandleAsClipboardEvent(
event: WysiwygEvent | InputEvent | ClipboardEvent,
): event is InputEvent | ClipboardEvent {
const isInputEventForClipboard =
event instanceof InputEvent && event.inputType === "insertFromPaste" && isNotNull(event.dataTransfer);
const isClipboardEvent = event instanceof ClipboardEvent;

return isClipboardEvent || isInputEventForClipboard;
}
12 changes: 12 additions & 0 deletions test/components/views/rooms/wysiwyg_composer/components/PlainTextComposer-test.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -290,4 +290,16 @@ describe("PlainTextComposer", () => {

expect(screen.getByTestId("autocomplete-wrapper")).toBeInTheDocument();
});

it("Should allow pasting of text values", async () => {
customRender();

const textBox = screen.getByRole("textbox");

await userEvent.click(textBox);
await userEvent.type(textBox, "hello");
await userEvent.paste(" world");

expect(textBox).toHaveTextContent("hello world");
});
});
Loading

0 comments on commit e32823e

Please sign in to comment.