Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

YQL-15941 no_llvm versions of purecalc and embedded #707

Merged
merged 2 commits into from
Dec 26, 2023
Merged

YQL-15941 no_llvm versions of purecalc and embedded #707

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d7b6641
init
vitstn Dec 25, 2023
1c41038
build
vitstn Dec 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ydb/library/yql/core/extract_predicate/ut/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ PEERDIR(
ydb/library/yql/providers/result/provider
ydb/library/yql/providers/yt/gateway/file
ydb/library/yql/providers/yt/provider
ydb/library/yql/providers/yt/codec/codegen
ydb/library/yql/providers/yt/comp_nodes/llvm
ydb/library/yql/minikql/comp_nodes/llvm
ydb/library/yql/minikql/invoke_builtins/llvm
ydb/library/yql/sql/pg
Expand Down
2 changes: 2 additions & 0 deletions ydb/library/yql/core/ut/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ PEERDIR(
ydb/library/yql/providers/result/provider
ydb/library/yql/providers/yt/gateway/file
ydb/library/yql/providers/yt/provider
ydb/library/yql/providers/yt/codec/codegen
ydb/library/yql/providers/yt/comp_nodes/llvm
ydb/library/yql/minikql/comp_nodes/llvm
ydb/library/yql/minikql/invoke_builtins/llvm
ydb/library/yql/sql/pg
Expand Down
4 changes: 4 additions & 0 deletions ydb/library/yql/minikql/codegen/codegen.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,10 @@ namespace {
}
}

bool ICodegen::IsCodegenAvailable() {
return true;
}

class TCodegen : public ICodegen, private llvm::JITEventListener {
public:
TCodegen(ETarget target, ESanitize sanitize)
Expand Down
2 changes: 2 additions & 0 deletions ydb/library/yql/minikql/codegen/codegen.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ class ICodegen {

using TSharedPtr = std::shared_ptr<ICodegen>;
static TSharedPtr MakeShared(ETarget target, ESanitize sanitize = ESanitize::Auto);

static bool IsCodegenAvailable();
};

}
Expand Down
25 changes: 25 additions & 0 deletions ydb/library/yql/minikql/codegen/no_llvm/codegen_dummy.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#include "codegen.h"

#include <util/generic/yexception.h>

namespace NYql {
namespace NCodegen {

ICodegen::TPtr ICodegen::Make(ETarget target, ESanitize sanitize) {
Y_UNUSED(target);
Y_UNUSED(sanitize);
throw yexception() << "Codegen is not available";
}

ICodegen::TSharedPtr ICodegen::MakeShared(ETarget target, ESanitize sanitize) {
Y_UNUSED(target);
Y_UNUSED(sanitize);
throw yexception() << "Codegen is not available";
}

bool ICodegen::IsCodegenAvailable() {
return false;
}

}
}
17 changes: 17 additions & 0 deletions ydb/library/yql/minikql/codegen/no_llvm/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
LIBRARY()

SRCDIR(
ydb/library/yql/minikql/codegen
)

ADDINCL(
ydb/library/yql/minikql/codegen
)

SRCS(
codegen_dummy.cpp
)

PROVIDES(MINIKQL_CODEGEN)

END()
2 changes: 1 addition & 1 deletion ydb/library/yql/minikql/codegen/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
RECURSE(llvm llvm14)
RECURSE(no_llvm llvm llvm14)
2 changes: 1 addition & 1 deletion ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -779,7 +779,7 @@ class TComputationPatternImpl final : public IComputationPattern {
#elif defined(MKQL_FORCE_USE_CODEGEN)
: Codegen(NYql::NCodegen::ICodegen::MakeShared(NYql::NCodegen::ETarget::Native))
#else
: Codegen(opts.OptLLVM != "OFF" || GetEnv(TString("MKQL_FORCE_USE_LLVM")) ? NYql::NCodegen::ICodegen::MakeShared(NYql::NCodegen::ETarget::Native) : NYql::NCodegen::ICodegen::TPtr())
: Codegen((NYql::NCodegen::ICodegen::IsCodegenAvailable() && opts.OptLLVM != "OFF") || GetEnv(TString("MKQL_FORCE_USE_LLVM")) ? NYql::NCodegen::ICodegen::MakeShared(NYql::NCodegen::ETarget::Native) : NYql::NCodegen::ICodegen::TPtr())
#endif
{
/// TODO: Enable JIT for AARCH64
Expand Down
16 changes: 0 additions & 16 deletions ydb/library/yql/providers/common/comp_nodes/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,22 +41,6 @@ PEERDIR(
ydb/library/yql/providers/common/schema/expr
)

IF (NOT MKQL_DISABLE_CODEGEN)
PEERDIR(
ydb/library/yql/minikql/codegen/llvm
contrib/libs/llvm12/lib/IR
contrib/libs/llvm12/lib/ExecutionEngine/MCJIT
contrib/libs/llvm12/lib/Linker
contrib/libs/llvm12/lib/Target/X86
contrib/libs/llvm12/lib/Target/X86/AsmParser
contrib/libs/llvm12/lib/Transforms/IPO
)
ELSE()
CFLAGS(
-DMKQL_DISABLE_CODEGEN
)
ENDIF()

YQL_LAST_ABI_VERSION()

END()
22 changes: 22 additions & 0 deletions ydb/library/yql/providers/yt/codec/codegen/no_llvm/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
LIBRARY()

ADDINCL(
ydb/library/yql/providers/yt/codec/codegen
)

SRCDIR(
ydb/library/yql/providers/yt/codec/codegen
)

SRCS(
yt_codec_cg_dummy.cpp
)

PEERDIR(
)

PROVIDES(YT_CODEC_CODEGEN)

YQL_LAST_ABI_VERSION()

END()
6 changes: 6 additions & 0 deletions ydb/library/yql/providers/yt/codec/codegen/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,14 @@ ENDIF()

YQL_LAST_ABI_VERSION()

PROVIDES(YT_CODEC_CODEGEN)

END()

RECURSE(
no_llvm
)

RECURSE_FOR_TESTS(
ut
)
38 changes: 38 additions & 0 deletions ydb/library/yql/providers/yt/codec/codegen/yt_codec_cg_dummy.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#include "yt_codec_cg.h"

#include <util/generic/yexception.h>

namespace NYql {

TString GetYtCodecBitCode() {
throw yexception() << "Codegen is not available";
}

void YtCodecAddMappings(NCodegen::ICodegen& codegen) {
Y_UNUSED(codegen);
throw yexception() << "Codegen is not available";
}

template<bool Flat>
THolder<IYtCodecCgWriter> MakeYtCodecCgWriter(const std::unique_ptr<NCodegen::ICodegen>& codegen, const void* cookie) {
Y_UNUSED(codegen);
Y_UNUSED(cookie);
throw yexception() << "Codegen is not available";
}

template THolder<IYtCodecCgWriter> MakeYtCodecCgWriter<true>(const std::unique_ptr<NCodegen::ICodegen>& codegen, const void* cookie);
template THolder<IYtCodecCgWriter> MakeYtCodecCgWriter<false>(const std::unique_ptr<NCodegen::ICodegen>& codegen, const void* cookie);

THolder<IYtCodecCgReader> MakeYtCodecCgReader(const std::unique_ptr<NCodegen::ICodegen>& codegen,
const NKikimr::NMiniKQL::THolderFactory& holderFactory, const void* cookie) {
Y_UNUSED(codegen);
Y_UNUSED(holderFactory);
Y_UNUSED(cookie);
throw yexception() << "Codegen is not available";
}

extern "C" void ThrowBadDecimal() {
throw yexception() << "Codegen is not available";
}

}
26 changes: 26 additions & 0 deletions ydb/library/yql/providers/yt/codec/ut/no_llvm/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
UNITTEST_FOR(ydb/library/yql/providers/yt/codec)

SRCDIR(
ydb/library/yql/providers/yt/codec/ut
)

SRCS(
yt_codec_io_ut.cpp
)

PEERDIR(
library/cpp/yson/node
ydb/library/yql/minikql/codegen/no_llvm
ydb/library/yql/minikql/computation/no_llvm
ydb/library/yql/public/udf/service/exception_policy
ydb/library/yql/sql
ydb/library/yql/sql/pg_dummy
ydb/library/yql/providers/common/codec
ydb/library/yql/providers/common/mkql
ydb/library/yql/providers/yt/lib/yson_helpers
ydb/library/yql/providers/yt/codec/codegen/no_llvm
)

YQL_LAST_ABI_VERSION()

END()
1 change: 1 addition & 0 deletions ydb/library/yql/providers/yt/codec/ut/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ PEERDIR(
ydb/library/yql/providers/common/codec
ydb/library/yql/providers/common/mkql
ydb/library/yql/providers/yt/lib/yson_helpers
ydb/library/yql/providers/yt/codec/codegen
)

YQL_LAST_ABI_VERSION()
Expand Down
7 changes: 2 additions & 5 deletions ydb/library/yql/providers/yt/codec/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,7 @@ PEERDIR(
ydb/library/yql/providers/yt/lib/skiff
)

IF (NOT MKQL_DISABLE_CODEGEN)
PEERDIR(
ydb/library/yql/providers/yt/codec/codegen
)
ELSE()
IF (MKQL_DISABLE_CODEGEN)
CFLAGS(
-DMKQL_DISABLE_CODEGEN
)
Expand All @@ -46,4 +42,5 @@ RECURSE(

RECURSE_FOR_TESTS(
ut
ut/no_llvm
)
4 changes: 2 additions & 2 deletions ydb/library/yql/providers/yt/codec/yt_codec_io.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1469,7 +1469,7 @@ void TMkqlReaderImpl::SetSpecs(const TMkqlIOSpecs& specs, const NKikimr::NMiniKQ
Buf_.SetStats(JobStats_);
if (Specs_->UseSkiff_) {
#ifndef MKQL_DISABLE_CODEGEN
if (Specs_->OptLLVM_ != "OFF") {
if (Specs_->OptLLVM_ != "OFF" && NCodegen::ICodegen::IsCodegenAvailable()) {
Decoder_.Reset(new TSkiffLLVMDecoder(Buf_, *Specs_, holderFactory));
}
else
Expand Down Expand Up @@ -1976,7 +1976,7 @@ void TMkqlWriterImpl::SetSpecs(const TMkqlIOSpecs& specs, const TVector<TString>

#ifndef MKQL_DISABLE_CODEGEN
THashMap<TStructType*, std::pair<llvm::Function*, llvm::Function*>> llvmFunctions;
if (Specs_->UseSkiff_ && Specs_->OptLLVM_ != "OFF") {
if (Specs_->UseSkiff_ && Specs_->OptLLVM_ != "OFF" && NCodegen::ICodegen::IsCodegenAvailable()) {
for (size_t i: xrange(Specs_->Outputs.size())) {
auto rowType = Specs_->Outputs[i].RowType;
if (rowType->GetMembersCount() != 0 && !llvmFunctions.contains(rowType)) {
Expand Down
2 changes: 1 addition & 1 deletion ydb/library/yql/providers/yt/comp_nodes/dq/dq_yt_block_reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -624,4 +624,4 @@ IComputationNode* CreateDqYtReadBlockWrapper(const TComputationNodeFactoryContex
{
return new TDqYtReadBlockWrapper(ctx, clusterName, token, inputSpec, samplingSpec, inputGroups, itemType, tableNames, std::move(tables), jobStats, inflight, timeout);
}
}
}
3 changes: 0 additions & 3 deletions ydb/library/yql/providers/yt/comp_nodes/dq/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ LIBRARY()
PEERDIR(
ydb/library/yql/minikql
ydb/library/yql/minikql/computation/llvm
ydb/library/yql/minikql/computation
ydb/library/yql/providers/yt/comp_nodes
ydb/library/yql/providers/yt/codec
ydb/library/yql/providers/common/codec
Expand Down Expand Up @@ -45,8 +44,6 @@ SRCS(
dq_yt_writer.cpp
)

INCLUDE(../../../../minikql/computation/header.ya.make.inc)

YQL_LAST_ABI_VERSION()


Expand Down
23 changes: 23 additions & 0 deletions ydb/library/yql/providers/yt/comp_nodes/llvm/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
LIBRARY()

OWNER(
g:yql
g:yql_ydb_core
)

NO_COMPILER_WARNINGS()

PEERDIR(
ydb/library/yql/minikql/codegen/llvm
ydb/library/yql/minikql/invoke_builtins/llvm
contrib/libs/llvm12/lib/IR
contrib/libs/llvm12/lib/ExecutionEngine/MCJIT
contrib/libs/llvm12/lib/Linker
contrib/libs/llvm12/lib/Target/X86
contrib/libs/llvm12/lib/Target/X86/AsmParser
contrib/libs/llvm12/lib/Transforms/IPO
)

INCLUDE(../ya.make.inc)

END()
23 changes: 23 additions & 0 deletions ydb/library/yql/providers/yt/comp_nodes/llvm14/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
LIBRARY()

OWNER(
g:yql
g:yql_ydb_core
)

NO_COMPILER_WARNINGS()

PEERDIR(
ydb/library/yql/minikql/codegen/llvm14
ydb/library/yql/minikql/invoke_builtins/llvm14
contrib/libs/llvm14/lib/IR
contrib/libs/llvm14/lib/ExecutionEngine/MCJIT
contrib/libs/llvm14/lib/Linker
contrib/libs/llvm14/lib/Target/X86
contrib/libs/llvm14/lib/Target/X86/AsmParser
contrib/libs/llvm14/lib/Transforms/IPO
)

INCLUDE(../ya.make.inc)

END()
16 changes: 16 additions & 0 deletions ydb/library/yql/providers/yt/comp_nodes/no_llvm/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
LIBRARY()

OWNER(
g:yql
g:yql_ydb_core
)

CXXFLAGS(-DMKQL_DISABLE_CODEGEN)

ADDINCL(GLOBAL ydb/library/yql/minikql/codegen/llvm_stub)

INCLUDE(../ya.make.inc)

PEERDIR(ydb/library/yql/minikql/invoke_builtins/no_llvm)

END()
Loading
Loading