Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

use adiosLog in blosc and sz #2968

Merged
merged 1 commit into from
Dec 3, 2021
Merged

use adiosLog in blosc and sz #2968

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 30 additions & 26 deletions source/adios2/operator/compress/CompressBlosc.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,23 +88,22 @@ size_t CompressBlosc::Operate(const char *dataIn, const Dims &blockStart,
value, "when setting Blosc clevel parameter\n"));
if (compressionLevel < 0 || compressionLevel > 9)
{
throw std::invalid_argument(
"ERROR: compression_level must be an "
"integer between 0 (default: no compression) and 9 "
"(more compression, more memory) inclusive, in call to "
"ADIOS2 Blosc Compress\n");
helper::Log("Operator", "CompressBlosc", "Operate",
"compression_level must be an integer between 0 "
"(no compression) and 9 (more compression, more "
"memory consumption) inclusive",
helper::EXCEPTION);
}
}
else if (key == "doshuffle")
{
auto itShuffle = m_Shuffles.find(value);
if (itShuffle == m_Shuffles.end())
{
throw std::invalid_argument(
"ERROR: invalid shuffle vale " + value +
" must be BLOSC_SHUFFLE, BLOSC_NOSHUFFLE or "
"BLOSC_BITSHUFFLE, "
" in call to ADIOS2 Blosc Compress\n");
helper::Log("Operator", "CompressBlosc", "Operate",
"Parameter doshuffle must be BLOSC_SHUFFLE, "
"BLOSC_NOSHUFFLE or BLOSC_BITSHUFFLE",
helper::EXCEPTION);
}
doShuffle = itShuffle->second;
}
Expand All @@ -118,11 +117,10 @@ size_t CompressBlosc::Operate(const char *dataIn, const Dims &blockStart,
compressor = value;
if (m_Compressors.count(compressor) == 0)
{
throw std::invalid_argument(
"ERROR: invalid compressor " + compressor +
" valid values: blosclz (default), lz4, lz4hc, "
"snappy, "
"zlib, or zstd, in call to ADIOS2 Blosc Compression\n");
helper::Log("Operator", "CompressBlosc", "Operate",
"Parameter compressor must be blosclz (default), "
"lz4, lz4hc, snappy, zlib, or zstd",
helper::EXCEPTION);
}
}
else if (key == "blocksize")
Expand Down Expand Up @@ -170,10 +168,10 @@ size_t CompressBlosc::Operate(const char *dataIn, const Dims &blockStart,
const int result = blosc_set_compressor(compressor.c_str());
if (result == -1)
{
throw std::invalid_argument(
"ERROR: invalid compressor " + compressor +
" check if supported by blosc build, in "
"call to ADIOS2 Blosc Compression\n");
helper::Log("Operator", "CompressBlosc", "Operate",
"blosc library linked does not support compressor " +
compressor,
helper::EXCEPTION);
}
blosc_set_nthreads(threads);
blosc_set_blocksize(blockSize);
Expand Down Expand Up @@ -245,7 +243,8 @@ size_t CompressBlosc::InverseOperate(const char *bufferIn, const size_t sizeIn,
}
else
{
throw("unknown blosc buffer version");
helper::Log("Operator", "CompressBlosc", "InverseOperate",
"corrupted compressed buffer", helper::EXCEPTION);
}

return 0;
Expand Down Expand Up @@ -273,7 +272,9 @@ size_t CompressBlosc::DecompressV1(const char *bufferIn, const size_t sizeIn,

if (sizeIn - bufferInOffset < sizeof(DataHeader))
{
throw("corrupted blosc buffer header." + m_VersionInfo + "\n");
helper::Log("Operator", "CompressBlosc", "InverseOperate",
"corrupted compressed buffer." + m_VersionInfo,
helper::EXCEPTION);
}
const bool isChunked =
reinterpret_cast<const DataHeader *>(bufferIn + bufferInOffset)
Expand All @@ -294,7 +295,9 @@ size_t CompressBlosc::DecompressV1(const char *bufferIn, const size_t sizeIn,
}
if (decompressedSize != sizeOut)
{
throw("corrupted blosc buffer." + m_VersionInfo + "\n");
helper::Log("Operator", "CompressBlosc", "InverseOperate",
"corrupted compressed buffer." + m_VersionInfo,
helper::EXCEPTION);
}
return sizeOut;
}
Expand Down Expand Up @@ -359,10 +362,11 @@ size_t CompressBlosc::DecompressChunkedFormat(const char *bufferIn,
currentOutputSize += static_cast<size_t>(decompressdSize);
else
{
throw std::runtime_error(
"ERROR: ADIOS2 Blosc Decompress failed. Decompressed chunk "
"results in zero decompressed bytes." +
m_VersionInfo + "\n");
helper::Log("Operator", "CompressBlosc",
"DecompressChunkedFormat",
"blosc decompress failed with zero buffer size. " +
m_VersionInfo,
helper::EXCEPTION);
}
inputOffset += static_cast<size_t>(max_inputDataSize);
}
Expand Down
60 changes: 34 additions & 26 deletions source/adios2/operator/compress/CompressSZ.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,9 +129,10 @@ size_t CompressSZ::Operate(const char *dataIn, const Dims &blockStart,
}
else
{
throw std::invalid_argument(
"ERROR: ADIOS2 operator unknown SZ parameter szMode: " +
it->second + "\n");
helper::Log("Operator", "CompressSZ", "Operate",
"Parameter szMode must be SZ_BEST_SPEED, "
"SZ_BEST_COMPRESSION or SZ_DEFAULT_COMPRESSION",
helper::EXCEPTION);
}
sz.szMode = szMode;
}
Expand Down Expand Up @@ -164,10 +165,10 @@ size_t CompressSZ::Operate(const char *dataIn, const Dims &blockStart,
}
else
{
throw std::invalid_argument("ERROR: ADIOS2 operator "
"unknown SZ parameter "
"errorBoundMode: " +
it->second + "\n");
helper::Log("Operator", "CompressSZ", "Operate",
"Parameter errorBoundMode must be ABS, REL, "
"ABS_AND_REL, ABS_OR_REL or PW_REL",
helper::EXCEPTION);
}
sz.errorBoundMode = errorBoundMode;
}
Expand All @@ -190,24 +191,24 @@ size_t CompressSZ::Operate(const char *dataIn, const Dims &blockStart,
else if (it->first == "pwr_type")
{
int pwr_type = SZ_PWR_MIN_TYPE;
if ((it->first == "MIN") || (it->first == "SZ_PWR_MIN_TYPE"))
if ((it->second == "MIN") || (it->second == "SZ_PWR_MIN_TYPE"))
{
pwr_type = SZ_PWR_MIN_TYPE;
}
else if ((it->first == "AVG") || (it->first == "SZ_PWR_AVG_TYPE"))
else if ((it->second == "AVG") || (it->second == "SZ_PWR_AVG_TYPE"))
{
pwr_type = SZ_PWR_AVG_TYPE;
}
else if ((it->first == "MAX") || (it->first == "SZ_PWR_MAX_TYPE"))
else if ((it->second == "MAX") || (it->second == "SZ_PWR_MAX_TYPE"))
{
pwr_type = SZ_PWR_MAX_TYPE;
}
else
{
throw std::invalid_argument("ERROR: ADIOS2 operator "
"unknown SZ parameter "
"pwr_type: " +
it->second + "\n");
helper::Log("Operator", "CompressSZ", "Operate",
"Parameter pwr_type must be MIN, SZ_PWR_MIN_TYPE, "
"AVG, SZ_PWR_AVG_TYPE, MAX or SZ_PWR_MAX_TYPE",
helper::EXCEPTION);
}
sz.pwr_type = pwr_type;
}
Expand Down Expand Up @@ -265,9 +266,9 @@ size_t CompressSZ::Operate(const char *dataIn, const Dims &blockStart,
}
else
{
throw std::invalid_argument("ERROR: ADIOS2 SZ Compression only support "
"double or float, type: " +
ToString(varType) + " is unsupported\n");
helper::Log("Operator", "CompressSZ", "Operate",
"SZ compressor only support float or double types",
helper::EXCEPTION);
}

convertedDims = ConvertDims(blockCount, varType, 5, true, 0);
Expand Down Expand Up @@ -309,7 +310,10 @@ size_t CompressSZ::InverseOperate(const char *bufferIn, const size_t sizeIn,
}
else
{
throw("unknown sz buffer version");
helper::Log("Operator", "CompressSZ", "InverseOperate",
"unknown sz buffer version, probably caused by corrupted "
"compressed buffer",
helper::EXCEPTION);
}

return 0;
Expand Down Expand Up @@ -372,8 +376,9 @@ size_t CompressSZ::DecompressV1(const char *bufferIn, const size_t sizeIn,
}
else
{
throw std::runtime_error(
"ERROR: data type must be either double or float in SZ\n");
helper::Log("Operator", "CompressSZ", "DecompressV1",
"SZ compressor only support float or double types",
helper::EXCEPTION);
}

const size_t dataSizeBytes =
Expand All @@ -388,8 +393,9 @@ size_t CompressSZ::DecompressV1(const char *bufferIn, const size_t sizeIn,

if (result == nullptr)
{
throw std::runtime_error("ERROR: SZ_decompress failed." +
m_VersionInfo + "\n");
helper::Log("Operator", "CompressSZ", "DecompressV1",
"decompression failed. " + m_VersionInfo,
helper::EXCEPTION);
}
std::memcpy(dataOut, result, dataSizeBytes);
free(result);
Expand Down Expand Up @@ -440,8 +446,9 @@ size_t CompressSZ::DecompressV2(const char *bufferIn, const size_t sizeIn,
}
else
{
throw std::runtime_error(
"ERROR: data type must be either double or float in SZ\n");
helper::Log("Operator", "CompressSZ", "DecompressV2",
"SZ compressor only support float or double types",
helper::EXCEPTION);
}

const size_t dataSizeBytes = helper::GetTotalSize(blockCount, dataTypeSize);
Expand All @@ -458,8 +465,9 @@ size_t CompressSZ::DecompressV2(const char *bufferIn, const size_t sizeIn,

if (result == nullptr)
{
throw std::runtime_error("ERROR: SZ_decompress failed." +
m_VersionInfo + "\n");
helper::Log("Operator", "CompressSZ", "DecompressV2",
"decompression failed. " + m_VersionInfo,
helper::EXCEPTION);
}
std::memcpy(dataOut, result, dataSizeBytes);
free(result);
Expand Down