Fixes an issue computing the unpacked size of a chunk (#1184)

This fixes an issue when the y_sampling is larger than the height of a chunk (and the height is not 1). Further, adds casts, use larger type to avoid overflow Signed-off-by: Kimball Thurston <kdt3rd@gmail.com>
AcademySoftwareFoundation · Oct 19, 2021 · 7d6dff5 · 7d6dff5
1 parent f44097e
commit 7d6dff5
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 6 deletions.
diff --git a/src/lib/OpenEXRCore/chunk.c b/src/lib/OpenEXRCore/chunk.c
@@ -178,7 +178,12 @@ compute_chunk_unpack_size (
             if (curc->y_sampling > 1)
             {
                 if (height > 1)
-                    chansz /= ((uint64_t) curc->y_sampling);
+                {
+                    if (curc->y_sampling > height)
+                        chansz /= ((uint64_t) height);
+                    else
+                        chansz /= ((uint64_t) curc->y_sampling);
+                }
                 else if ((y % ((int) curc->y_sampling)) != 0)
                     chansz = 0;
             }

diff --git a/src/lib/OpenEXRCore/validation.c b/src/lib/OpenEXRCore/validation.c
@@ -151,11 +151,17 @@ validate_image_dimensions (
     /* isnormal will return true when par is 0, which should also be disallowed */
     if (!isnormal (par) || par < 1e-6f || par > 1e+6f)
         return f->print_error (
-            f, EXR_ERR_INVALID_ATTR, "Invalid pixel aspect ratio %g", (double)par);
+            f,
+            EXR_ERR_INVALID_ATTR,
+            "Invalid pixel aspect ratio %g",
+            (double) par);
 
     if (sww < 0.f)
         return f->print_error (
-            f, EXR_ERR_INVALID_ATTR, "Invalid screen window width %g", (double)sww);
+            f,
+            EXR_ERR_INVALID_ATTR,
+            "Invalid screen window width %g",
+            (double) sww);
 
     return EXR_ERR_SUCCESS;
 }
@@ -189,8 +195,8 @@ validate_channels (
             "At least one channel required");
 
     dw = curpart->data_window;
-    w  = dw.max.x - dw.min.x + 1;
-    h  = dw.max.y - dw.min.y + 1;
+    w  = (int64_t) dw.max.x - (int64_t) dw.min.x + 1;
+    h  = (int64_t) dw.max.y - (int64_t) dw.min.y + 1;
     for (int c = 0; c < channels->num_channels; ++c)
     {
         int32_t xsamp = channels->entries[c].x_sampling;

diff --git a/src/lib/OpenEXRUtil/ImfCheckFile.cpp b/src/lib/OpenEXRUtil/ImfCheckFile.cpp
@@ -1181,9 +1181,11 @@ bool readCoreScanlinePart(exr_context_t f, int part, bool reduceMemory, bool red
     if (rv != EXR_ERR_SUCCESS)
         return true;
 
-    for (int y = datawin.min.y; y <= datawin.max.y; y += lines_per_chunk)
+    for (uint64_t chunk = 0; chunk < height; chunk += lines_per_chunk)
     {
         exr_chunk_info_t cinfo = { 0 };
+        int y = ((int) chunk) + datawin.min.y;
+
         rv = exr_read_scanline_chunk_info (f, part, y, &cinfo);
         if (rv != EXR_ERR_SUCCESS)
         {