Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

exrcheck: rework 'reduceMemory' and 'reduceTime' modes #902

Merged
merged 3 commits into from
Jan 28, 2021
Merged

exrcheck: rework 'reduceMemory' and 'reduceTime' modes #902

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ee730ff
exrcheck: rework 'reducememory' and 'reducetime' modes
peterhillman Jan 20, 2021
ecded97
Merge branch 'master' into exrcheck_timeout_andmemory
peterhillman Jan 28, 2021
69aa673
merge branch 'master'
peterhillman Jan 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
174 changes: 107 additions & 67 deletions src/lib/OpenEXRUtil/ImfCheckFile.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,24 +28,14 @@ using std::vector;
using std::max;
using Imath::Box2i;


int
getStep( const Box2i &dw , bool reduceTime)
{

// limit to approximately 2,000,000 pixels or 2000 rows in fast mode
if (reduceTime)
{
size_t rowCount = (dw.max.y - dw.min.y + 1);
size_t pixelCount = rowCount * (dw.max.x - dw.min.x + 1);
return max( 1 , max ( static_cast<int>(pixelCount / 2000000) , static_cast<int>(rowCount / 2000) ) );
}
else
{
return 1;
}

}
//
// limits for reduceMemory mode
//
const int gMaxScanlineWidth= 1000000;
const int gMaxTilePixelsPerScanline = 8000000;
const int gMaxTileSize = 1000*1000;
const int gMaxSamplesPerDeepPixel = 1000;
const int gMaxSamplesPerScanline = 1<<12;

//
// read image or part using the Rgba interface
Expand All @@ -63,17 +53,15 @@ readRgba(T& in, bool reduceMemory , bool reduceTime)
int w = dw.max.x - dw.min.x + 1;
int dx = dw.min.x;

if (reduceMemory && w > (1 << 10))
if (reduceMemory && w > gMaxScanlineWidth )
{
return false;
}

Array<Rgba> pixels (w);
in.setFrameBuffer (&pixels[-dx], 1, 0);



int step = getStep( dw , reduceTime );
int step = 1;

//
// try reading scanlines. Continue reading scanlines
Expand All @@ -88,6 +76,15 @@ readRgba(T& in, bool reduceMemory , bool reduceTime)
catch(...)
{
threw = true;

//
// in reduceTime mode, fail immediately - the file is corrupt
//
if (reduceTime)
{
return threw;
}

}
}
}
Expand All @@ -113,7 +110,7 @@ readScanline(T& in, bool reduceMemory , bool reduceTime)
int w = dw.max.x - dw.min.x + 1;
int dx = dw.min.x;

if (reduceMemory && w > (1 << 10))
if (reduceMemory && w > gMaxScanlineWidth )
{
return false;
}
Expand Down Expand Up @@ -144,9 +141,7 @@ readScanline(T& in, bool reduceMemory , bool reduceTime)

in.setFrameBuffer(i);

int step = getStep( dw , reduceTime );


int step = 1;

//
// try reading scanlines. Continue reading scanlines
Expand All @@ -161,6 +156,15 @@ readScanline(T& in, bool reduceMemory , bool reduceTime)
catch(...)
{
threw = true;

//
// in reduceTime mode, fail immediately - the file is corrupt
//
if (reduceTime)
{
return threw;
}

}
}
}
Expand All @@ -183,7 +187,7 @@ readTileRgba( T& in,bool reduceMemory, bool reduceTime)
int w = dw.max.x - dw.min.x + 1;
int h = dw.max.y - dw.min.y + 1;

if ( (reduceMemory || reduceTime ) && h*w > 1000*1000)
if ( (reduceMemory || reduceTime ) && h*w > gMaxTileSize )
{
return false;
}
Expand Down Expand Up @@ -217,16 +221,14 @@ readTile(T& in, bool reduceMemory , bool reduceTime)
const Box2i& dw = in.header().dataWindow();

int w = dw.max.x - dw.min.x + 1;
int h = dw.max.y - dw.min.y + 1;
int dwx = dw.min.x;
int dwy = dw.min.y;
int numXLevels = in.numXLevels();
int numYLevels = in.numYLevels();

const TileDescription& td = in.header().tileDescription();


if (reduceMemory && w > (1 << 10))
if (reduceMemory && (w > gMaxScanlineWidth || (td.xSize*td.ySize) > gMaxTileSize) )
{
return false;
}
Expand Down Expand Up @@ -254,19 +256,8 @@ readTile(T& in, bool reduceMemory , bool reduceTime)

in.setFrameBuffer (i);


//
// limit to 2,000 tiles
//
size_t step = 1;

if (reduceTime && in.numXTiles(0) * in.numYTiles(0) > 2000)
{
step = max(1, (in.numXTiles(0) * in.numYTiles(0)) / 2000 );
}



size_t tileIndex =0;
bool isRipMap = td.mode == RIPMAP_LEVELS;

Expand Down Expand Up @@ -298,6 +289,14 @@ readTile(T& in, bool reduceMemory , bool reduceTime)
if (isRipMap || xlevel==ylevel)
{
threw = true;

//
// in reduceTime mode, fail immediately - the file is corrupt
//
if (reduceTime)
{
return threw;
}
}
}
}
Expand Down Expand Up @@ -328,9 +327,8 @@ bool readDeepScanLine(T& in,bool reduceMemory, bool reduceTime)

int w = dw.max.x - dw.min.x + 1;
int dwx = dw.min.x;
int dwy = dw.min.y;

if ( reduceMemory && w > (1 << 8) )
if ( reduceMemory && w > gMaxScanlineWidth )
{
return false;
}
Expand Down Expand Up @@ -372,11 +370,6 @@ bool readDeepScanLine(T& in,bool reduceMemory, bool reduceTime)
in.setFrameBuffer(frameBuffer);

int step = 1;
// only read 200 scanlines in fast mode
if (reduceTime)
{
step = max( 1 , (dw.max.y - dw.min.y + 1) / 200 );
}

vector<float> pixelBuffer;

Expand All @@ -396,7 +389,7 @@ bool readDeepScanLine(T& in,bool reduceMemory, bool reduceTime)
//
// don't read samples which require a lot of memory in reduceMemory mode
//
if (!reduceMemory || localSampleCount[j] <= 1000)
if (!reduceMemory || localSampleCount[j] <= gMaxSamplesPerDeepPixel )
{
bufferSize += localSampleCount[j];
}
Expand All @@ -406,7 +399,7 @@ bool readDeepScanLine(T& in,bool reduceMemory, bool reduceTime)
//
// limit total number of samples read in reduceMemory mode
//
if (!reduceMemory || bufferSize < 1<<10)
if (!reduceMemory || bufferSize < gMaxScanlineWidth )
{
//
// allocate sample buffer and set per-pixel pointers into buffer
Expand All @@ -419,7 +412,7 @@ bool readDeepScanLine(T& in,bool reduceMemory, bool reduceTime)
for (int k = 0; k < channelCount; k++)
{

if (reduceMemory && localSampleCount[j] > 1000)
if (reduceMemory && localSampleCount[j] > gMaxSamplesPerDeepPixel )
{
data[k][j] = nullptr;
}
Expand All @@ -438,6 +431,13 @@ bool readDeepScanLine(T& in,bool reduceMemory, bool reduceTime)
catch(...)
{
threw = true;
//
// in reduceTime mode, fail immediately - the file is corrupt
//
if (reduceTime)
{
return threw;
}
}
}

Expand Down Expand Up @@ -525,11 +525,6 @@ readDeepTile(T& in,bool reduceMemory , bool reduceTime)
in.setFrameBuffer(frameBuffer);
size_t step = 1;

if (reduceTime && in.numXTiles(0) * in.numYTiles(0) > 2000)
{
step = max(1, (in.numXTiles(0) * in.numYTiles(0)) / 2000 );
}

int tileIndex = 0;
bool isRipMap = td.mode == RIPMAP_LEVELS;

Expand Down Expand Up @@ -568,15 +563,15 @@ readDeepTile(T& in,bool reduceMemory , bool reduceTime)
{
for (int tx = 0 ; tx < tileWidth ; ++tx )
{
if (!reduceMemory || localSampleCount[ty][tx] < 100)
if (!reduceMemory || localSampleCount[ty][tx] < gMaxSamplesPerDeepPixel )
{
bufferSize += channelCount * localSampleCount[ty][tx];
}
}
}

// limit total samples allocated for this tile
if (!reduceMemory || bufferSize < 1<<12)
if (!reduceMemory || bufferSize < gMaxSamplesPerScanline )
{

pixelBuffer.resize( bufferSize );
Expand All @@ -586,7 +581,7 @@ readDeepTile(T& in,bool reduceMemory , bool reduceTime)
{
for (int tx = 0 ; tx < tileWidth ; ++tx )
{
if (!reduceMemory || localSampleCount[ty][tx] < 100)
if (!reduceMemory || localSampleCount[ty][tx] < gMaxSamplesPerDeepPixel )
{
for (int k = 0 ; k < channelCount ; ++k )
{
Expand Down Expand Up @@ -620,6 +615,13 @@ readDeepTile(T& in,bool reduceMemory , bool reduceTime)
if (isRipMap || xlevel==ylevel)
{
threw = true;
//
// in reduceTime mode, fail immediately - the file is corrupt
//
if (reduceTime)
{
return threw;
}
}

}
Expand Down Expand Up @@ -682,13 +684,32 @@ readMultiPart(MultiPartInputFile& in,bool reduceMemory,bool reduceTime)
threw = true;
}

bool widePart = false;
Box2i b = in.header( part ).dataWindow();
if (b.max.x - b.min.x > 1000000)
{
widePart = true;
}
bool widePart = false;
Box2i b = in.header( part ).dataWindow();

//
// very wide scanline parts take excessive memory to read.
// detect that here so that tests can be skipped when reduceMemory is set
//


if (b.max.x - b.min.x > gMaxScanlineWidth )
{
widePart = true;

}
//
// significant memory is also required to read a tiled part
// using the scanline interface with tall tiles - the scanlineAPI
// needs to allocate memory to store an entire row of tiles
//
if (isTiled(in.header( part ).type()))
{
if ( in.header( part ).tileDescription().ySize * (b.max.x-b.min.x+1) > gMaxTilePixelsPerScanline )
{
widePart = true;
}
}

if (!reduceMemory || !widePart)
{
Expand Down Expand Up @@ -856,7 +877,7 @@ class PtrIStream: public IStream



void resetInput(const char*fileName)
void resetInput(const char* /*fileName*/)
{
// do nothing: filename doesn't need to be 'reset' between calls
}
Expand Down Expand Up @@ -889,10 +910,29 @@ runChecks(T& source,bool reduceMemory,bool reduceTime)
MultiPartInputFile multi(source);
firstPartType = multi.header(0).type();
Box2i b = multi.header(0).dataWindow();
if (b.max.x - b.min.x > 1000000)

//
// scanline images with very wide parts take excessive memory to read
// detect that here so that tests can be skipped when reduceMemory is set
//
if (b.max.x - b.min.x > gMaxScanlineWidth )
{
firstPartWide = true;
}
//
// significant memory is also required to read a tiled file
// using the scanline interface with tall tiles - the scanlineAPI
// needs to allocate memory to store an entire row of tiles
//

if (isTiled(firstPartType))
{
if ( multi.header(0).tileDescription().ySize * (b.max.x-b.min.x+1) > gMaxTilePixelsPerScanline )
{
firstPartWide = true;
}
}


threw = readMultiPart(multi , reduceMemory , reduceTime);
}
Expand Down
13 changes: 7 additions & 6 deletions src/lib/OpenEXRUtil/ImfCheckFile.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,18 @@ OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_ENTER


//
// attempt to read the given file as an OpenEXR, using
// various OpenEXR read paths.
// attempt to read the given file as an OpenEXR, using various OpenEXR read paths.
// This can be used to validate correctness of the library, when running the library
// with a sanitizer or memory checker, as well as checking that a file is a correct OpenEXR
//
// returns true if the file read correctly using expected API calls, or false
// returns true if the file reads correctly using expected API calls, or false
// if an exception was thrown that indicates the file is invalid
//
// if reduceMemory and/or reduceTime are true, will avoid tests or inputs that are known to
// take large amounts of memory and/or time respectively. This may hide errors within the
// file or library
// if reduceMemory is true, will avoid tests or inputs that are known to
// take large amounts of memory. This may hide errors within the file or library.
//
// if reduceTime is true and an error is found within the file, then future tests are reduced for speed.
// This may hide errors within the library.
//
//

Expand Down