Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add memory monotonicity test over srcSize #2538

Merged
merged 2 commits into from
Mar 22, 2021
Merged

Add memory monotonicity test over srcSize #2538

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b9dd821
Add mem monotonicity test over srcSize
senhuang42 Mar 16, 2021
dff4a0e
Make ZSTD_estimateCCtxSize_internal() loop through all srcSize parame…
senhuang42 Mar 19, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions lib/compress/zstd_compress.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1390,15 +1390,23 @@ size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams)

static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel)
{
ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
return ZSTD_estimateCCtxSize_usingCParams(cParams);
int tier = 0;
size_t largestSize = 0;
static const unsigned long long srcSizeTiers[4] = {16 KB, 128 KB, 256 KB, ZSTD_CONTENTSIZE_UNKNOWN};
for (; tier < 4; ++tier) {
/* Choose the set of cParams for a given level across all srcSizes that give the largest cctxSize */
ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeTiers[tier], 0, ZSTD_cpm_noAttachDict);
largestSize = MAX(ZSTD_estimateCCtxSize_usingCParams(cParams), largestSize);
}
return largestSize;
}

size_t ZSTD_estimateCCtxSize(int compressionLevel)
{
int level;
size_t memBudget = 0;
for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) {
/* Ensure monotonically increasing memory usage as compression level increases */
size_t const newMB = ZSTD_estimateCCtxSize_internal(level);
if (newMB > memBudget) memBudget = newMB;
}
Expand Down
42 changes: 42 additions & 0 deletions tests/fuzzer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3181,6 +3181,48 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");

DISPLAYLEVEL(3, "test%3i : check compression mem usage monotonicity over levels for estimateCCtxSize() : ", testNb++);
{
int level = 1;
size_t prevSize = 0;
for (; level < ZSTD_maxCLevel(); ++level) {
size_t const currSize = ZSTD_estimateCCtxSize(level);
if (prevSize > currSize) {
DISPLAYLEVEL(3, "Error! previous cctx size: %zu at level: %d is larger than current cctx size: %zu at level: %d",
prevSize, level-1, currSize, level);
goto _output_error;
}
prevSize = currSize;
}
}
DISPLAYLEVEL(3, "OK \n");

DISPLAYLEVEL(3, "test%3i : check estimateCCtxSize() always larger or equal to ZSTD_estimateCCtxSize_usingCParams() : ", testNb++);
{
size_t const kSizeIncrement = 2 KB;
int level = -3;

for (; level <= ZSTD_maxCLevel(); ++level) {
size_t dictSize = 0;
for (; dictSize <= 256 KB; dictSize += 8 * kSizeIncrement) {
size_t srcSize = 2 KB;
for (; srcSize < 300 KB; srcSize += kSizeIncrement) {
ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, srcSize, dictSize);
size_t const cctxSizeUsingCParams = ZSTD_estimateCCtxSize_usingCParams(cParams);
size_t const cctxSizeUsingLevel = ZSTD_estimateCCtxSize(level);
if (cctxSizeUsingLevel < cctxSizeUsingCParams
|| ZSTD_isError(cctxSizeUsingCParams)
|| ZSTD_isError(cctxSizeUsingLevel)) {
DISPLAYLEVEL(3, "error! l: %d dict: %zu srcSize: %zu cctx size cpar: %zu, cctx size level: %zu\n",
level, dictSize, srcSize, cctxSizeUsingCParams, cctxSizeUsingLevel);
goto _output_error;
}
}
}
}
}
DISPLAYLEVEL(3, "OK \n");

#endif

_end:
Expand Down