Fix thumbnail creation

README.md

@@ -103,7 +103,7 @@ Connect to others in complex network situations, or over the Internet.
 * [zip.js](https://gildas-lormeau.github.io/zip.js/) library
 * [cyrb53](https://github.com/bryc/code/blob/master/jshash/experimental/cyrb53.js) super-fast hash function
 * [NoSleep](https://github.com/richtr/NoSleep.js) display sleep, add wake lock ([MIT](licenses/MIT-NoSleep))
-* [heic2any](https://github.com/alexcorvi/heic2any) HEIC/HEIF to PNG/GIF/JPEG ([MIT](licenses/MIT-heic2any))
+* [libheif](https://github.com/strukturag/libheif) library to handle HEIC/HEIF files (GPLv3)
 * [Weblate](https://weblate.org/) web-based localization tool
 
 [FAQ](docs/faq.md)

public/scripts/heif-convert.js

@@ -0,0 +1,36 @@
+function HeifConvert(libheif) {
+    this.libheif = libheif;
+    this.decoder = new libheif.HeifDecoder();
+}
+
+
+HeifConvert.prototype.convert = async function (buffer) {
+    const decodeResult = this.decoder.decode(buffer);
+    const image = decodeResult[0];
+
+    let w = image.get_width();
+    let h = image.get_height();
+    const canvas = document.createElement("canvas");
+    canvas.width = w;
+    canvas.height = h;
+
+    const ctx = canvas.getContext("2d");
+    const imageData = ctx.createImageData(w, h);
+
+    await copyData(imageData, image);
+
+    ctx.putImageData(imageData, 0, 0);
+    image.free();
+    return canvas;
+};
+
+function copyData(dataContainer, image) {
+    return new Promise((resolve, reject) => {
+        image.display(
+            dataContainer,
+            function () {
+                resolve()
+            }
+        );
+    })
+}

public/scripts/main.js

@@ -17,7 +17,8 @@ class PairDrop {
             "scripts/qr-code.min.js",
             "scripts/zip.min.js",
             "scripts/no-sleep.min.js",
-            "scripts/heic2any.min.js"
+            "scripts/heif-convert.js",
+            "scripts/libheif.js"
         ];
 
         this.registerServiceWorker();

public/scripts/network.js

@@ -332,6 +332,8 @@ class Peer {
         this._filesQueue = [];
         this._busy = false;
 
+        this.maxMessageSize = 65536; // 64 KB
+
         // evaluate auto accept
         this._evaluateAutoAccept();
     }
@@ -450,14 +452,7 @@ class Peer {
 
         Events.fire('set-progress', {peerId: this._peerId, progress: 0.8, status: 'prepare'})
 
-        let dataUrl = '';
-        if (files[0].type.split('/')[0] === 'image') {
-            try {
-                dataUrl = await getThumbnailAsDataUrl(files[0], 400, null, 0.9);
-            } catch (e) {
-                console.error(e);
-            }
-        }
+        let dataUrl = await this.getFileTransferThumbnail(files[0]);
 
         Events.fire('set-progress', {peerId: this._peerId, progress: 1, status: 'prepare'})
 
@@ -472,6 +467,30 @@ class Peer {
         Events.fire('set-progress', {peerId: this._peerId, progress: 0, status: 'wait'})
     }
 
+    async getFileTransferThumbnail(image) {
+        if (image.type.split('/')[0] !== 'image') {
+            // file is not of type image -> abort!
+            return '';
+        }
+
+        let dataUrl = '';
+        try {
+            // Iteratively lower thumbnail quality until its size is less than maxMessageSize - 2 kB
+            let quality = 1;
+            do {
+                quality -= 0.1;
+                if (quality <= 0) {
+                    console.error("Could not create thumbnail that fits into one message.");
+                    return '';
+                }
+                dataUrl = await getThumbnailAsDataUrl(image, 450, 450, quality);
+            } while (new Blob([dataUrl]).size + 2_000 > this.maxMessageSize);
+        } catch (e) {
+            console.error(e);
+        }
+        return dataUrl;
+    }
+
     async sendFiles() {
         for (let i=0; i<this._filesRequested.length; i++) {
             this._filesQueue.push(this._filesRequested[i]);
@@ -725,7 +744,8 @@ class RTCPeer extends Peer {
         super(serverConnection, isCaller, peerId, roomType, roomId);
 
         this.rtcSupported = true;
-        this.rtcConfig = rtcConfig
+        this.rtcConfig = rtcConfig;
+        this.maxMessageSize = 262144; // 256 KB
 
         if (!this._isCaller) return; // we will listen for a caller
         this._connect();
@@ -811,6 +831,13 @@ class RTCPeer extends Peer {
         Events.on('beforeunload', e => this._onBeforeUnload(e));
         Events.on('pagehide', _ => this._onPageHide());
         Events.fire('peer-connected', {peerId: this._peerId, connectionHash: this.getConnectionHash()});
+        this._setMaxMessageSize();
+    }
+
+    _setMaxMessageSize() {
+        this.maxMessageSize = this._conn && this._conn.sctp
+            ? Math.min(this._conn.sctp.maxMessageSize, 1048576) // 1 MB max
+            : 262144; // 256 KB
     }
 
     _onMessage(message) {

public/scripts/ui.js

@@ -281,10 +281,8 @@ class PeersUI {
 
             if (files[0].type.split('/')[0] === 'image') {
                 try {
-                    let imageUrl = await getThumbnailAsDataUrl(files[0], 80, null, 0.9);
-
+                    let imageUrl = await getThumbnailAsDataUrl(files[0], 80, 80, 0.9);
                     this.$shareModeImageThumb.style.backgroundImage = `url(${imageUrl})`;
-
                     this.$shareModeImageThumb.removeAttribute('hidden');
                 } catch (e) {
                     console.error(e);

public/scripts/util.js

@@ -478,12 +478,7 @@ function getThumbnailAsDataUrl(file, width = undefined, height = undefined, qual
         try {
             if (file.type === "image/heif" || file.type === "image/heic") {
                 // browsers can't show heic files --> convert to jpeg before creating thumbnail
-                let blob = await fileToBlob(file);
-                file = await heic2any({
-                    blob,
-                    toType: "image/jpeg",
-                    quality: quality
-                });
+                file = await heicToJpeg(file, 0.5);
             }
 
             let imageUrl = URL.createObjectURL(file);
@@ -493,26 +488,40 @@ function getThumbnailAsDataUrl(file, width = undefined, height = undefined, qual
 
             await waitUntilImageIsLoaded(imageUrl);
 
-            let imageWidth = image.width;
-            let imageHeight = image.height;
             let canvas = document.createElement('canvas');
+            let heightForSpecifiedWidth;
+            let widthForSpecifiedHeight;
 
-            // resize the canvas and draw the image data into it
+            if (width) {
+                heightForSpecifiedWidth = Math.floor(image.height * width / image.width);
+            }
+            if (height) {
+                widthForSpecifiedHeight = Math.floor(image.width * height / image.height);
+            }
+
+            // resize the canvas and draw the image on it
             if (width && height) {
-                canvas.width = width;
-                canvas.height = height;
+                // mode "contain": preserve aspect ratio and use arguments as boundaries
+                if (height > heightForSpecifiedWidth) {
+                    canvas.width = width;
+                    canvas.height = heightForSpecifiedWidth;
+                }
+                else {
+                    canvas.width = widthForSpecifiedHeight;
+                    canvas.height = height;
+                }
             }
             else if (width) {
                 canvas.width = width;
-                canvas.height = Math.floor(imageHeight * width / imageWidth)
+                canvas.height = heightForSpecifiedWidth;
             }
             else if (height) {
-                canvas.width = Math.floor(imageWidth * height / imageHeight);
+                canvas.width = widthForSpecifiedHeight;
                 canvas.height = height;
             }
             else {
-                canvas.width = imageWidth;
-                canvas.height = imageHeight
+                canvas.width = image.width;
+                canvas.height = image.height
             }
 
             let ctx = canvas.getContext("2d");
@@ -527,6 +536,32 @@ function getThumbnailAsDataUrl(file, width = undefined, height = undefined, qual
     })
 }
 
+function initHeicConverter() {
+    return new Promise((resolve, reject) => {
+        fetch("libheif.wasm")
+            .then((res) => res.arrayBuffer())
+            .then(async (wasmBinary) => {
+                resolve(new HeifConvert(libheif({ wasmBinary: wasmBinary })));
+            })
+            .catch(reject);
+    });
+}
+
+async function heicToJpeg(file, quality) {
+    const heicConverter = await initHeicConverter();
+    console.log("Using libheif", heicConverter.libheif.heif_get_version());
+
+    const buffer = await file.arrayBuffer();
+    const canvas = await heicConverter.convert(buffer);
+
+    return new Promise(resolve => {
+        canvas.toBlob(blob => resolve(blob),
+            'image/jpeg',
+            quality
+        );
+    });
+}
+
 // Resolves returned promise when image is loaded and throws error if image cannot be shown
 function waitUntilImageIsLoaded(imageUrl, timeout = 10000) {
     return new Promise((resolve, reject) => {

public/service-worker.js

@@ -7,6 +7,9 @@ const relativePathsToCache = [
     'manifest.json',
     'styles/styles-main.css',
     'styles/styles-deferred.css',
+    'scripts/heif-convert.js',
+    'scripts/libheif.js',
+    'scripts/libheif.wasm',
     'scripts/localization.js',
     'scripts/main.js',
     'scripts/network.js',