Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add Benchmark output #271

Merged
merged 4 commits into from
Oct 7, 2024
Merged

Add Benchmark output #271

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e7cc6c1
Add Benchmark output
mkarunan Sep 19, 2024
14cf58e
Refactor
mkarunan Sep 19, 2024
7e52832
Address comments
mkarunan Sep 19, 2024
601ba2f
Merge branch 'benchmarking' into application-output
dlangbe Oct 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 105 additions & 0 deletions library/src/data_types.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,111 @@ namespace hiptensor
return "HIP_TYPE_NONE";
}
}

std::string opTypeToString(hiptensorOperator_t opType)
{
if(opType == HIPTENSOR_OP_IDENTITY)
{
return "HIPTENSOR_OP_IDENTITY";
}
else if(opType == HIPTENSOR_OP_SQRT)
{
return "HIPTENSOR_OP_SQRT";
}
else if(opType == HIPTENSOR_OP_ADD)
{
return "HIPTENSOR_OP_ADD";
}
else if(opType == HIPTENSOR_OP_MUL)
{
return "HIPTENSOR_OP_MUL";
}
else if(opType == HIPTENSOR_OP_MAX)
{
return "HIPTENSOR_OP_MAX";
}
else if(opType == HIPTENSOR_OP_MIN)
{
return "HIPTENSOR_OP_MIN";
}
else
{
return "HIPTENSOR_OP_UNKNOWN";
}
}

std::string algoTypeToString(hiptensorAlgo_t algoType)
{
if(algoType == HIPTENSOR_ALGO_ACTOR_CRITIC)
{
return "HIPTENSOR_ALGO_ACTOR_CRITIC";
}
else if(algoType == HIPTENSOR_ALGO_DEFAULT)
{
return "HIPTENSOR_ALGO_DEFAULT";
}
else if(algoType == HIPTENSOR_ALGO_DEFAULT_PATIENT)
{
return "HIPTENSOR_ALGO_DEFAULT_PATIENT";
}
else
{
return "HIPTENSOR_ALGO_UNKNOWN";
}
}

std::string logLevelToString(hiptensorLogLevel_t logLevel)
{
if(logLevel == HIPTENSOR_LOG_LEVEL_OFF)
{
return "HIPTENSOR_LOG_LEVEL_OFF";
}
else if(logLevel == HIPTENSOR_LOG_LEVEL_ERROR)
{
return "HIPTENSOR_LOG_LEVEL_ERROR";
}
else if(logLevel == HIPTENSOR_LOG_LEVEL_PERF_TRACE)
{
return "HIPTENSOR_LOG_LEVEL_PERF_TRACE";
}
else if(logLevel == HIPTENSOR_LOG_LEVEL_PERF_HINT)
{
return "HIPTENSOR_LOG_LEVEL_PERF_HINT";
}
else if(logLevel == HIPTENSOR_LOG_LEVEL_HEURISTICS_TRACE)
{
return "HIPTENSOR_LOG_LEVEL_HEURISTICS_TRACE";
}
else if(logLevel == HIPTENSOR_LOG_LEVEL_API_TRACE)
{
return "HIPTENSOR_LOG_LEVEL_API_TRACE";
}
else
{
return "HIPTENSOR_LOG_LEVEL_UNKNOWN";
}
}

std::string workSizePrefToString(hiptensorWorksizePreference_t workSize)
{
if(workSize == HIPTENSOR_WORKSPACE_MIN)
{
return "HIPTENSOR_WORKSPACE_MIN";
}
else if(workSize == HIPTENSOR_WORKSPACE_RECOMMENDED)
{
return "HIPTENSOR_WORKSPACE_RECOMMENDED";
}
else if(workSize == HIPTENSOR_WORKSPACE_MAX)
{
return "HIPTENSOR_WORKSPACE_MAX";
}
else
{
return "HIPTENSOR_WORKSPACE_UNKNOWN";
}
}

} // namespace hiptensor

bool operator==(hipDataType hipType, hiptensorComputeType_t computeType)
Expand Down
4 changes: 4 additions & 0 deletions library/src/include/data_types.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,10 @@ namespace hiptensor

std::string computeTypeToString(hiptensorComputeType_t computeType);
std::string hipTypeToString(hipDataType hipType);
std::string opTypeToString(hiptensorOperator_t opType);
std::string algoTypeToString(hiptensorAlgo_t algoType);
std::string logLevelToString(hiptensorLogLevel_t);
std::string workSizePrefToString(hiptensorWorksizePreference_t workSize);
} // namespace hiptensor

bool operator==(hipDataType hipType, hiptensorComputeType_t computeType);
Expand Down
203 changes: 192 additions & 11 deletions test/01_contraction/contraction_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,13 +78,102 @@ namespace hiptensor
mRunFlag = true;
mValidationResult = false;
mMaxRelativeError = 0.0;

mElapsedTimeMs = mTotalGFlops = mMeasuredTFlopsPerSec = mTotalBytes = 0.0;
}

ContractionResource* ContractionTest::getResource() const
{
return DataStorage::instance().get();
}

std::ostream& ContractionTest::printHeader(std::ostream& stream /* = std::cout */) const
{
return stream << "TypeA, TypeB, TypeC, "
<< "TypeD, TypeCompute, "
<< "Algorithm, Operator, "
<< "WorkSizePreference, LogLevel, "
<< "Lengths, Strides, Modes, Alpha,"
<< "Beta, elapsedMs, "
<< "Problem Size(GFlops), "
<< "TFlops/s, "
<< "TotalBytes, "
<< "Result" << std::endl;
}

std::ostream& ContractionTest::printKernel(std::ostream& stream) const
{
auto param = Base::GetParam();
auto testType = std::get<0>(param);
auto algorithm = std::get<1>(param);
auto operatorType = std::get<2>(param);
auto workSizePref = std::get<3>(param);
auto logLevel = std::get<4>(param);
auto lengths = std::get<5>(param);
auto strides = std::get<6>(param);
auto modes = std::get<7>(param);
auto alpha = std::get<8>(param);
auto beta = std::get<9>(param);

stream << hipTypeToString(testType[0]) << ", " << hipTypeToString(testType[1]) << ", " << hipTypeToString(testType[2]) << ", "
<< hipTypeToString(testType[3]) << ", " << computeTypeToString(convertToComputeType(testType[4])) << ", " << algoTypeToString(algorithm) << ", "
<< opTypeToString(operatorType) << ", " << workSizePrefToString(workSizePref) << ", " << logLevelToString(logLevel) << ", [";

for(int i = 0; i < lengths.size(); i++) {
stream << "[" ;
for(int j = 0; j < lengths[i].size(); j++) {
stream << lengths[i][j] << ", ";
}
stream << "], ";
}
stream << "], [";

if(!strides.empty()) {
for(int i = 0; i < strides.size(); i++) {
stream << "[" ;
for(int j = 0; j < strides[i].size(); j++) {
stream << strides[i][j] << ", ";
}
stream << "], ";
}
}
stream << "], [";

if(!modes.empty()) {
for(int i = 0; i < modes.size(); i++) {
stream << "[" ;
for(int j = 0; j < modes[i].size(); j++) {
stream << modes[i][j] << ", ";
}
stream << "],";
}
}
stream << "], " << alpha << "," << beta << ", ";

if(!mRunFlag)
{
stream << "n/a"
<< ", "
<< "n/a"
<< ", "
<< "n/a"
<< ", "
<< "n/a"
<< ", "
<< "SKIPPED" << std::endl;
}
else
{

stream << mElapsedTimeMs << ", " << mTotalGFlops << ", " << mMeasuredTFlopsPerSec
<< ", " << mTotalBytes << ", "
<<((bool)mValidationResult ? "PASSED" : "FAILED")
<< std::endl;
}

return stream;
}

void ContractionTest::SetUp()
{
// reset API log buffer
Expand Down Expand Up @@ -413,20 +502,21 @@ namespace hiptensor
void ContractionTest::reportResults(std::ostream& stream,
hipDataType DDataType,
hiptensorComputeType_t computeType,
bool omitHeader,
bool omitSkipped,
bool omitFailed,
bool omitPassed) const
{
if(!omitHeader)
{
printHeader(stream);
}

// Conditionally print outputs
if((mRunFlag || !omitSkipped) && (mValidationResult || !omitFailed)
&& (!mValidationResult || !omitPassed))
{
if(mPrintTypes)
{
ContractionTest::sAPILogBuff
<< "TypeA/B/C/D: " << hipTypeToString(DDataType)
<< ", ComputeType: " << computeTypeToString(computeType) << std::endl;
}
printKernel(stream);

stream << ContractionTest::sAPILogBuff.str();

Expand Down Expand Up @@ -658,6 +748,11 @@ namespace hiptensor

auto resource = getResource();

hipEvent_t startEvent, stopEvent;
CHECK_HIP_ERROR(hipEventCreate(&startEvent));
CHECK_HIP_ERROR(hipEventCreate(&stopEvent));
CHECK_HIP_ERROR(hipEventRecord(startEvent));

CHECK_HIPTENSOR_ERROR(hiptensorContraction(handle,
&plan,
(void*)&alphaBuf,
Expand All @@ -670,6 +765,54 @@ namespace hiptensor
worksize,
0 /* stream */));

CHECK_HIP_ERROR(hipEventRecord(stopEvent));
CHECK_HIP_ERROR(hipEventSynchronize(stopEvent))

auto timeMs = 0.0f;
CHECK_HIP_ERROR(hipEventElapsedTime(&timeMs, startEvent, stopEvent));

size_t totalLength = std::accumulate(d_ms_ns.mLengths.begin(),
d_ms_ns.mLengths.end(),
size_t(1),
std::multiplies<size_t>());

uint32_t hops = desc.mTensorMode[2].size() / 2;
auto iter = std::find(desc.mTensorMode[0].cbegin(), desc.mTensorMode[0].cend(), desc.mTensorMode[2][desc.mTensorMode[2].size() - 1]);
if(iter != desc.mTensorMode[0].cend())
{
auto offset = std::distance(desc.mTensorMode[0].cbegin(), iter);
totalLength *= std::accumulate(a_ms_ks.mLengths.begin() + offset,
a_ms_ks.mLengths.begin() + offset + hops,
size_t(1),
std::multiplies<size_t>());
}

mElapsedTimeMs = float64_t(timeMs);
mTotalGFlops = 2.0 * totalLength;
mMeasuredTFlopsPerSec = mTotalGFlops / mElapsedTimeMs;

size_t sizeA = std::accumulate(a_ms_ks.mLengths.begin(),
a_ms_ks.mLengths.end(),
hipDataTypeSize(ADataType),
std::multiplies<size_t>());

size_t sizeB = std::accumulate(b_ns_ks.mLengths.begin(),
b_ns_ks.mLengths.end(),
hipDataTypeSize(BDataType),
std::multiplies<size_t>());

size_t sizeD = std::accumulate(d_ms_ns.mLengths.begin(),
d_ms_ns.mLengths.end(),
hipDataTypeSize(DDataType),
std::multiplies<size_t>());

mTotalBytes = sizeA + sizeB + sizeD;
mTotalBytes += (betaBuf.mReal != 0.0) ? sizeD : 0;
mTotalBytes /= (1e9 * mElapsedTimeMs);

CHECK_HIP_ERROR(hipEventDestroy(startEvent));
CHECK_HIP_ERROR(hipEventDestroy(stopEvent));

auto& testOptions = HiptensorOptions::instance();

if(testOptions->performValidation())
Expand Down Expand Up @@ -699,12 +842,7 @@ namespace hiptensor
DDataType,
workspace));

size_t elementsCD = std::accumulate(d_ms_ns.mLengths.begin(),
d_ms_ns.mLengths.end(),
size_t{1},
std::multiplies<size_t>());

int sizeD = elementsCD * hipDataTypeSize(DDataType);
auto reference = resource->allocDevice(sizeD);
resource->copyData(reference, resource->hostD(), sizeD);

Expand All @@ -715,6 +853,47 @@ namespace hiptensor
size_t{1},
std::multiplies<size_t>());


size_t elementsCD = sizeD / hipDataTypeSize(ADataType);

if(DDataType == HIP_R_16F)
{
std::tie(mValidationResult, mMaxRelativeError)
= compareEqualLaunchKernel<_Float16>((_Float16*)resource->deviceD().get(),
(_Float16*)reference.get(),
elementsCD,
computeType,
tolerance);
}
else if(DDataType == HIP_R_16BF)
{
std::tie(mValidationResult, mMaxRelativeError)
= compareEqualLaunchKernel<hip_bfloat16>(
(hip_bfloat16*)resource->deviceD().get(),
(hip_bfloat16*)reference.get(),
elementsCD,
computeType,
tolerance);
}
else if(DDataType == HIP_R_32F || DDataType == HIP_C_32F)
{
std::tie(mValidationResult, mMaxRelativeError)
= compareEqualLaunchKernel<float>((float*)resource->deviceD().get(),
(float*)reference.get(),
elementsCD,
computeType,
tolerance);
}
else if(DDataType == HIP_R_64F || DDataType == HIP_C_64F)
{
std::tie(mValidationResult, mMaxRelativeError)
= compareEqualLaunchKernel<double>((double*)resource->deviceD().get(),
(double*)reference.get(),
elementsCD,
computeType,
tolerance);
}

auto eps = getEpsilon(computeType == HIPTENSOR_COMPUTE_64F ? HIPTENSOR_COMPUTE_64F
: HIPTENSOR_COMPUTE_32F);
double tolerance = 2 * nelems_k * eps;
Expand Down Expand Up @@ -780,6 +959,7 @@ namespace hiptensor
reportResults(std::cout,
DDataType,
computeType,
false,
loggingOptions->omitSkipped(),
loggingOptions->omitFailed(),
loggingOptions->omitPassed());
Expand All @@ -790,6 +970,7 @@ namespace hiptensor
reportResults(loggingOptions->ostream().fstream(),
DDataType,
computeType,
false,
loggingOptions->omitSkipped(),
loggingOptions->omitFailed(),
loggingOptions->omitPassed());
Expand Down
Loading