Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

YQ-3151 added issues for timeout and cancelled #8718

Merged
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 11 additions & 6 deletions ydb/core/kqp/executer_actor/kqp_executer_impl.h
Original file line number Diff line number Diff line change
Expand Up @@ -684,7 +684,7 @@ class TKqpExecuterBase : public TActorBootstrapped<TDerived> {
if (statusCode == Ydb::StatusIds::INTERNAL_ERROR) {
InternalError(issues);
} else if (statusCode == Ydb::StatusIds::TIMEOUT) {
TimeoutError(ev->Sender);
TimeoutError(ev->Sender, issues);
} else {
RuntimeError(NYql::NDq::DqStatusToYdbStatus(msg.GetStatusCode()), issues);
}
Expand Down Expand Up @@ -1706,29 +1706,34 @@ class TKqpExecuterBase : public TActorBootstrapped<TDerived> {
ReplyErrorAndDie(status, &issues);
}

void TimeoutError(TActorId abortSender) {
void TimeoutError(TActorId abortSender, NYql::TIssues issues) {
if (AlreadyReplied) {
LOG_E("Timeout when we already replied - not good" << Endl << TBackTrace().PrintToString() << Endl);
return;
}

const auto status = NYql::NDqProto::StatusIds::TIMEOUT;
const TString message = "Request timeout exceeded";
if (issues.Empty()) {
issues.AddIssue("Request timeout exceeded");
}

TerminateComputeActors(Ydb::StatusIds::TIMEOUT, message);
TerminateComputeActors(Ydb::StatusIds::TIMEOUT, issues);

AlreadyReplied = true;

LOG_E("Abort execution: " << NYql::NDqProto::StatusIds_StatusCode_Name(status) << "," << message);
LOG_E("Abort execution: " << NYql::NDqProto::StatusIds_StatusCode_Name(status) << ", " << issues.ToOneLineString());
if (ExecuterSpan) {
ExecuterSpan.EndError(TStringBuilder() << NYql::NDqProto::StatusIds_StatusCode_Name(status));
}

ResponseEv->Record.MutableResponse()->SetStatus(Ydb::StatusIds::TIMEOUT);
for (const auto& issue : issues) {
NYql::IssueToMessage(issue, ResponseEv->Record.MutableResponse()->AddIssues());
GrigoriyPA marked this conversation as resolved.
Show resolved Hide resolved
}

// TEvAbortExecution can come from either ComputeActor or SessionActor (== Target).
if (abortSender != Target) {
auto abortEv = MakeHolder<TEvKqp::TEvAbortExecution>(status, message);
auto abortEv = MakeHolder<TEvKqp::TEvAbortExecution>(status, issues);
this->Send(Target, abortEv.Release());
}

Expand Down
10 changes: 6 additions & 4 deletions ydb/core/kqp/proxy_service/kqp_proxy_service.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -148,12 +148,14 @@ class TKqpProxyService : public TActorBootstrapped<TKqpProxyService> {
struct TEvOnRequestTimeout: public TEventLocal<TEvOnRequestTimeout, EEv::EvOnRequestTimeout> {
ui64 RequestId;
TDuration Timeout;
TDuration InitialTimeout;
NYql::NDqProto::StatusIds::StatusCode Status;
int Round;

TEvOnRequestTimeout(ui64 requestId, TDuration timeout, NYql::NDqProto::StatusIds::StatusCode status, int round)
: RequestId(requestId)
, Timeout(timeout)
, InitialTimeout(timeout)
, Status(status)
, Round(round)
{}
Expand Down Expand Up @@ -1283,9 +1285,9 @@ class TKqpProxyService : public TActorBootstrapped<TKqpProxyService> {

const TKqpSessionInfo* info = LocalSessions->FindPtr(reqInfo->SessionId);
if (msg->Round == 0 && info) {
TString message = TStringBuilder()
<< "request's " << (msg->Status == NYql::NDqProto::StatusIds::TIMEOUT ? "timeout" : "cancelAfter")
<< " exceeded";
TString message = msg->Status == NYql::NDqProto::StatusIds::TIMEOUT
? (TStringBuilder() << "Request's timeout " << msg->Timeout.MilliSeconds() << "ms exceeded")
: (TStringBuilder() << "Request's canceled after " << msg->Timeout.MilliSeconds() << "ms");

Send(info->WorkerId, new TEvKqp::TEvAbortExecution(msg->Status, message));

Expand All @@ -1297,7 +1299,7 @@ class TKqpProxyService : public TActorBootstrapped<TKqpProxyService> {
}
} else {
TString message = TStringBuilder()
<< "Query did not complete within specified timeout, session id " << reqInfo->SessionId;
<< "Query did not complete within specified timeout " << msg->InitialTimeout.MilliSeconds() << "ms, session id " << reqInfo->SessionId;
ReplyProcessError(NYql::NDq::DqStatusToYdbStatus(msg->Status), message, requestId);
}
}
Expand Down
5 changes: 5 additions & 0 deletions ydb/core/kqp/run_script_actor/kqp_run_script_actor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,11 @@ class TRunScriptActor : public NActors::TActorBootstrapped<TRunScriptActor> {
WaitFinalizationRequest = true;
RunState = IsExecuting() ? ERunState::Finishing : RunState;

if (RunState == ERunState::Cancelling) {
NYql::TIssue cancelIssue("Request was canceled by user");
cancelIssue.SetCode(NYql::DEFAULT_ERROR, NYql::TSeverityIds::S_INFO);
}

auto scriptFinalizeRequest = std::make_unique<TEvScriptFinalizeRequest>(
GetFinalizationStatusFromRunState(), ExecutionId, Database, Status, GetExecStatusFromStatusCode(Status),
Issues, std::move(QueryStats), std::move(QueryPlan), std::move(QueryAst), LeaseGeneration
Expand Down
1 change: 1 addition & 0 deletions ydb/core/kqp/session_actor/kqp_query_state.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ std::unique_ptr<TEvTxProxySchemeCache::TEvNavigateKeySet> TKqpQueryState::BuildN


bool TKqpQueryState::SaveAndCheckCompileResult(TEvKqp::TEvCompileResponse* ev) {
CompilationRunning = false;
CompileResult = ev->CompileResult;
YQL_ENSURE(CompileResult);
MaxReadType = CompileResult->MaxReadType;
Expand Down
1 change: 1 addition & 0 deletions ydb/core/kqp/session_actor/kqp_query_state.h
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ class TKqpQueryState : public TNonCopyable {
bool KeepSession = false;
TIntrusiveConstPtr<NACLib::TUserToken> UserToken;
NActors::TMonotonic StartedAt;
bool CompilationRunning = false;

THashMap<NKikimr::TTableId, ui64> TableVersions;

Expand Down
19 changes: 13 additions & 6 deletions ydb/core/kqp/session_actor/kqp_session_actor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -524,6 +524,7 @@ class TKqpSessionActor : public TActorBootstrapped<TKqpSessionActor> {

void CompileQuery() {
YQL_ENSURE(QueryState);
QueryState->CompilationRunning = true;
auto ev = QueryState->BuildCompileRequest(CompilationCookie, GUCSettings);
LOG_D("Sending CompileQuery request");

Expand Down Expand Up @@ -1528,16 +1529,22 @@ class TKqpSessionActor : public TActorBootstrapped<TKqpSessionActor> {
TString logMsg = TStringBuilder() << "got TEvAbortExecution in " << CurrentStateFuncName();
LOG_I(logMsg << ", status: " << NYql::NDqProto::StatusIds_StatusCode_Name(msg.GetStatusCode()) << " send to: " << ExecuterId);

TString reason = TStringBuilder() << "Request timeout exceeded, cancelling after "
<< (AppData()->MonotonicTimeProvider->Now() - QueryState->StartedAt).MilliSeconds()
<< " milliseconds.";
auto issues = ev->Get()->GetIssues();
TStringBuilder reason = TStringBuilder() << "Cancelling after " << (AppData()->MonotonicTimeProvider->Now() - QueryState->StartedAt).MilliSeconds() << "ms";
if (QueryState->CompilationRunning) {
reason << " during compilation";
} else if (ExecuterId) {
reason << " during execution";
} else {
reason << " in " << CurrentStateFuncName();
}
issues.AddIssue(reason);

if (ExecuterId) {
auto abortEv = MakeHolder<TEvKqp::TEvAbortExecution>(msg.GetStatusCode(), reason);
auto abortEv = MakeHolder<TEvKqp::TEvAbortExecution>(msg.GetStatusCode(), issues);
Send(ExecuterId, abortEv.Release(), IEventHandle::FlagTrackDelivery);
} else {
const auto& issues = ev->Get()->GetIssues();
ReplyQueryError(NYql::NDq::DqStatusToYdbStatus(msg.GetStatusCode()), logMsg, MessageFromIssues(issues));
ReplyQueryError(NYql::NDq::DqStatusToYdbStatus(msg.GetStatusCode()), "", MessageFromIssues(issues));
}
}

Expand Down
10 changes: 9 additions & 1 deletion ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ class TDqComputeActorBase : public NActors::TActorBootstrapped<TDerived>
public:
void Bootstrap() {
try {
StartTime = TInstant::Now();
{
TStringBuilder prefixBuilder;
prefixBuilder << "SelfId: " << this->SelfId() << ", TxId: " << TxId << ", task: " << Task.GetId() << ". ";
Expand Down Expand Up @@ -1049,8 +1050,14 @@ class TDqComputeActorBase : public NActors::TActorBootstrapped<TDerived>
);
}

TStringBuilder reason = TStringBuilder() << "Task execution timeout ";
if (RuntimeSettings.Timeout) {
reason << RuntimeSettings.Timeout->MilliSeconds() << "ms ";
}
reason << "exceeded, terminating after " << (TInstant::Now() - StartTime).MilliSeconds() << "ms";

State = NDqProto::COMPUTE_STATE_FAILURE;
ReportStateAndMaybeDie(NYql::NDqProto::StatusIds::TIMEOUT, {TIssue("timeout exceeded")}, true);
ReportStateAndMaybeDie(NYql::NDqProto::StatusIds::TIMEOUT, {TIssue(reason)}, true);
break;
}
case EEvWakeupTag::PeriodicStatsTag: {
Expand Down Expand Up @@ -1928,6 +1935,7 @@ class TDqComputeActorBase : public NActors::TActorBootstrapped<TDerived>
NWilson::TSpan ComputeActorSpan;
TDuration SourceCpuTime;
private:
TInstant StartTime;
bool Running = true;
TInstant LastSendStatsTime;
bool PassExceptions = false;
Expand Down
Loading