Skip to content

Commit

Permalink
Use size_t for DWA buffersize calculation (#901)
Browse files Browse the repository at this point in the history 
* Use size_t for DWA buffersize calculation

Signed-off-by: Peter Hillman <peterh@wetafx.co.nz>

* use Int64 instead of size_t for buffersize calculations

Signed-off-by: Peter Hillman <peterh@wetafx.co.nz>
  • Loading branch information
@peterhillman
peterhillman authored Jan 20, 2021
1 parent dd87e54 commit 0e08c95
Show file tree
Hide file tree
Showing 2 changed files with 33 additions and 29 deletions.
42 changes: 23 additions & 19 deletions src/lib/OpenEXR/ImfDwaCompressor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2932,10 +2932,10 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize)
// of channels we have.
//

size_t maxOutBufferSize = 0;
Int64 maxOutBufferSize = 0;
int numLossyDctChans = 0;
int unknownBufferSize = 0;
int rleBufferSize = 0;
Int64 unknownBufferSize = 0;
Int64 rleBufferSize = 0;

int maxLossyDctAcSize = (int)ceil ((float)numScanLines() / 8.0f) *
(int)ceil ((float)(_max[0] - _min[0] + 1) / 8.0f) *
Expand All @@ -2945,6 +2945,8 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize)
(int)ceil ((float)(_max[0] - _min[0] + 1) / 8.0f) *
sizeof (unsigned short);

Int64 pixelCount = static_cast<Int64>(numScanLines()) * static_cast<Int64>(_max[0] - _min[0] + 1);

for (unsigned int chan = 0; chan < _channelData.size(); ++chan)
{
switch (_channelData[chan].compression)
Expand All @@ -2971,8 +2973,7 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize)
// of the source data.
//

int rleAmount = 2 * numScanLines() * (_max[0] - _min[0] + 1) *
OPENEXR_IMF_NAMESPACE::pixelTypeSize (_channelData[chan].type);
Int64 rleAmount = 2 * pixelCount * OPENEXR_IMF_NAMESPACE::pixelTypeSize (_channelData[chan].type);

rleBufferSize += rleAmount;
}
Expand All @@ -2981,8 +2982,7 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize)

case UNKNOWN:

unknownBufferSize += numScanLines() * (_max[0] - _min[0] + 1) *
OPENEXR_IMF_NAMESPACE::pixelTypeSize (_channelData[chan].type);
unknownBufferSize += pixelCount * OPENEXR_IMF_NAMESPACE::pixelTypeSize (_channelData[chan].type);
break;

default:
Expand All @@ -2999,13 +2999,13 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize)
// which could take slightly more space
//

maxOutBufferSize += compressBound ((uLongf)rleBufferSize);
maxOutBufferSize += compressBound (rleBufferSize);

//
// And the same goes for the UNKNOWN data
//

maxOutBufferSize += compressBound ((uLongf)unknownBufferSize);
maxOutBufferSize += compressBound (unknownBufferSize);

//
// Allocate a zip/deflate compressor big enought to hold the DC data
Expand Down Expand Up @@ -3052,7 +3052,7 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize)
// to Huffman encoding
//

if (static_cast<size_t>(maxLossyDctAcSize * numLossyDctChans) > _packedAcBufferSize)
if (static_cast<Int64>(maxLossyDctAcSize * numLossyDctChans) > _packedAcBufferSize)
{
_packedAcBufferSize = maxLossyDctAcSize * numLossyDctChans;
if (_packedAcBuffer != 0)
Expand All @@ -3064,15 +3064,15 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize)
// _packedDcBuffer holds one quantized DCT coef per 8x8 block
//

if (static_cast<size_t>(maxLossyDctDcSize * numLossyDctChans) > _packedDcBufferSize)
if (static_cast<Int64>(maxLossyDctDcSize * numLossyDctChans) > _packedDcBufferSize)
{
_packedDcBufferSize = maxLossyDctDcSize * numLossyDctChans;
if (_packedDcBuffer != 0)
delete[] _packedDcBuffer;
_packedDcBuffer = new char[_packedDcBufferSize];
}

if (static_cast<size_t>(rleBufferSize) > _rleBufferSize)
if ( rleBufferSize > _rleBufferSize )
{
_rleBufferSize = rleBufferSize;
if (_rleBuffer != 0)
Expand All @@ -3091,7 +3091,7 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize)
// all in one swoop (for each compression scheme).
//

int planarUncBufferSize[NUM_COMPRESSOR_SCHEMES];
Int64 planarUncBufferSize[NUM_COMPRESSOR_SCHEMES];
for (int i=0; i<NUM_COMPRESSOR_SCHEMES; ++i)
planarUncBufferSize[i] = 0;

Expand All @@ -3103,14 +3103,12 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize)
break;

case RLE:
planarUncBufferSize[RLE] +=
numScanLines() * (_max[0] - _min[0] + 1) *
planarUncBufferSize[RLE] += pixelCount *
OPENEXR_IMF_NAMESPACE::pixelTypeSize (_channelData[chan].type);
break;

case UNKNOWN:
planarUncBufferSize[UNKNOWN] +=
numScanLines() * (_max[0] - _min[0] + 1) *
planarUncBufferSize[UNKNOWN] += pixelCount *
OPENEXR_IMF_NAMESPACE::pixelTypeSize (_channelData[chan].type);
break;

Expand All @@ -3128,16 +3126,22 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize)
if (planarUncBufferSize[UNKNOWN] > 0)
{
planarUncBufferSize[UNKNOWN] =
compressBound ((uLongf)planarUncBufferSize[UNKNOWN]);
compressBound (planarUncBufferSize[UNKNOWN]);
}

for (int i = 0; i < NUM_COMPRESSOR_SCHEMES; ++i)
{
if (static_cast<size_t>(planarUncBufferSize[i]) > _planarUncBufferSize[i])
if ( planarUncBufferSize[i] > _planarUncBufferSize[i])
{
_planarUncBufferSize[i] = planarUncBufferSize[i];
if (_planarUncBuffer[i] != 0)
delete[] _planarUncBuffer[i];

if (planarUncBufferSize[i] > std::numeric_limits<size_t>::max())
{
throw IEX_NAMESPACE::ArgExc("DWA buffers too large");
}

_planarUncBuffer[i] = new char[planarUncBufferSize[i]];
}
}
Expand Down
20 changes: 10 additions & 10 deletions src/lib/OpenEXR/ImfDwaCompressor.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,16 +169,16 @@ class DwaCompressor: public Compressor
std::vector<CscChannelSet> _cscSets;
std::vector<Classifier> _channelRules;

char *_packedAcBuffer;
size_t _packedAcBufferSize;
char *_packedDcBuffer;
size_t _packedDcBufferSize;
char *_rleBuffer;
size_t _rleBufferSize;
char *_outBuffer;
size_t _outBufferSize;
char *_planarUncBuffer[NUM_COMPRESSOR_SCHEMES];
size_t _planarUncBufferSize[NUM_COMPRESSOR_SCHEMES];
char* _packedAcBuffer;
Int64 _packedAcBufferSize;
char* _packedDcBuffer;
Int64 _packedDcBufferSize;
char* _rleBuffer;
Int64 _rleBufferSize;
char* _outBuffer;
Int64 _outBufferSize;
char* _planarUncBuffer[NUM_COMPRESSOR_SCHEMES];
Int64 _planarUncBufferSize[NUM_COMPRESSOR_SCHEMES];

Zip *_zip;
float _dwaCompressionLevel;
Expand Down

0 comments on commit 0e08c95

Please sign in to comment.