From 6de79a9b2dbc8d69eb172947fb499032ecf086eb Mon Sep 17 00:00:00 2001
From: Stefan Weil <sw@weilnetz.de>
Date: Sat, 20 Aug 2016 20:00:00 +0200
Subject: [PATCH] Prevent overflow by large arguments of opj_calloc

calloc allocates (num * size) bytes, possibly without handling an
overflow when doing the multiplication. This is fatal because the
allocated memory would be too small in that case.

Signed-off-by: Stefan Weil <sw@weilnetz.de>
---
 src/lib/openjp2/opj_malloc.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/lib/openjp2/opj_malloc.c b/src/lib/openjp2/opj_malloc.c
index dca91bfcb..aba8d277b 100644
--- a/src/lib/openjp2/opj_malloc.c
+++ b/src/lib/openjp2/opj_malloc.c
@@ -32,6 +32,8 @@
 #define OPJ_SKIP_POISON
 #include "opj_includes.h"
 
+#include <errno.h>
+
 #if defined(OPJ_HAVE_MALLOC_H) && defined(OPJ_HAVE_MEMALIGN)
 # include <malloc.h>
 #endif
@@ -201,6 +203,11 @@ void * opj_calloc(size_t num, size_t size)
         /* prevent implementation defined behavior of realloc */
         return NULL;
     }
+    if (num > SIZE_MAX / size) {
+        /* prevent overflow */
+        errno = ENOMEM;
+        return NULL;
+    }
     return calloc(num, size);
 }