YQL-15941 no_llvm versions of purecalc and embedded (ydb-platform#707)

* init

* build

ydb/library/yql/core/extract_predicate/ut/ya.make

@@ -18,6 +18,8 @@ PEERDIR(
     ydb/library/yql/providers/result/provider
     ydb/library/yql/providers/yt/gateway/file
     ydb/library/yql/providers/yt/provider
+    ydb/library/yql/providers/yt/codec/codegen
+    ydb/library/yql/providers/yt/comp_nodes/llvm
     ydb/library/yql/minikql/comp_nodes/llvm
     ydb/library/yql/minikql/invoke_builtins/llvm
     ydb/library/yql/sql/pg

ydb/library/yql/core/ut/ya.make

@@ -28,6 +28,8 @@ PEERDIR(
     ydb/library/yql/providers/result/provider
     ydb/library/yql/providers/yt/gateway/file
     ydb/library/yql/providers/yt/provider
+    ydb/library/yql/providers/yt/codec/codegen
+    ydb/library/yql/providers/yt/comp_nodes/llvm
     ydb/library/yql/minikql/comp_nodes/llvm
     ydb/library/yql/minikql/invoke_builtins/llvm
     ydb/library/yql/sql/pg

ydb/library/yql/minikql/codegen/codegen.cpp

@@ -230,6 +230,10 @@ namespace {
     }
 }
 
+bool ICodegen::IsCodegenAvailable() {
+    return true;
+}
+
 class TCodegen : public ICodegen, private llvm::JITEventListener {
 public:
     TCodegen(ETarget target, ESanitize sanitize)

ydb/library/yql/minikql/codegen/codegen.h

@@ -67,6 +67,8 @@ class ICodegen {
 
     using TSharedPtr = std::shared_ptr<ICodegen>;
     static TSharedPtr MakeShared(ETarget target, ESanitize sanitize = ESanitize::Auto);
+
+    static bool IsCodegenAvailable();
 };
 
 }

ydb/library/yql/minikql/codegen/no_llvm/codegen_dummy.cpp

@@ -0,0 +1,25 @@
+#include "codegen.h"
+
+#include <util/generic/yexception.h>
+
+namespace NYql {
+namespace NCodegen {
+
+ICodegen::TPtr ICodegen::Make(ETarget target, ESanitize sanitize) {
+    Y_UNUSED(target);
+    Y_UNUSED(sanitize);
+    throw yexception() << "Codegen is not available";
+}
+
+ICodegen::TSharedPtr ICodegen::MakeShared(ETarget target, ESanitize sanitize) {
+    Y_UNUSED(target);
+    Y_UNUSED(sanitize);
+    throw yexception() << "Codegen is not available";
+}
+
+bool ICodegen::IsCodegenAvailable() {
+    return false;
+}
+
+}
+}

ydb/library/yql/minikql/codegen/no_llvm/ya.make

@@ -0,0 +1,17 @@
+LIBRARY()
+
+SRCDIR(
+    ydb/library/yql/minikql/codegen
+)
+
+ADDINCL(
+    ydb/library/yql/minikql/codegen
+)
+
+SRCS(
+    codegen_dummy.cpp
+)
+
+PROVIDES(MINIKQL_CODEGEN)
+
+END()

ydb/library/yql/minikql/codegen/ya.make

@@ -1 +1 @@
-RECURSE(llvm llvm14)
+RECURSE(no_llvm llvm llvm14)

ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp

@@ -779,7 +779,7 @@ class TComputationPatternImpl final : public IComputationPattern {
 #elif defined(MKQL_FORCE_USE_CODEGEN)
         : Codegen(NYql::NCodegen::ICodegen::MakeShared(NYql::NCodegen::ETarget::Native))
 #else
-        : Codegen(opts.OptLLVM != "OFF" || GetEnv(TString("MKQL_FORCE_USE_LLVM")) ? NYql::NCodegen::ICodegen::MakeShared(NYql::NCodegen::ETarget::Native) : NYql::NCodegen::ICodegen::TPtr())
+        : Codegen((NYql::NCodegen::ICodegen::IsCodegenAvailable() && opts.OptLLVM != "OFF") || GetEnv(TString("MKQL_FORCE_USE_LLVM")) ? NYql::NCodegen::ICodegen::MakeShared(NYql::NCodegen::ETarget::Native) : NYql::NCodegen::ICodegen::TPtr())
 #endif
     {
     /// TODO: Enable JIT for AARCH64

ydb/library/yql/providers/common/comp_nodes/ya.make

@@ -41,22 +41,6 @@ PEERDIR(
     ydb/library/yql/providers/common/schema/expr
 )
 
-IF (NOT MKQL_DISABLE_CODEGEN)
-    PEERDIR(
-        ydb/library/yql/minikql/codegen/llvm
-        contrib/libs/llvm12/lib/IR
-        contrib/libs/llvm12/lib/ExecutionEngine/MCJIT
-        contrib/libs/llvm12/lib/Linker
-        contrib/libs/llvm12/lib/Target/X86
-        contrib/libs/llvm12/lib/Target/X86/AsmParser
-        contrib/libs/llvm12/lib/Transforms/IPO
-    )
-ELSE()
-    CFLAGS(
-        -DMKQL_DISABLE_CODEGEN
-    )
-ENDIF()
-
 YQL_LAST_ABI_VERSION()
 
 END()

ydb/library/yql/providers/yt/codec/codegen/no_llvm/ya.make

@@ -0,0 +1,22 @@
+LIBRARY()
+
+ADDINCL(
+    ydb/library/yql/providers/yt/codec/codegen
+)
+
+SRCDIR(
+    ydb/library/yql/providers/yt/codec/codegen
+)
+
+SRCS(
+    yt_codec_cg_dummy.cpp
+)
+
+PEERDIR(
+)
+
+PROVIDES(YT_CODEC_CODEGEN)
+
+YQL_LAST_ABI_VERSION()
+
+END()

ydb/library/yql/providers/yt/codec/codegen/ya.make

@@ -72,8 +72,14 @@ ENDIF()
 
 YQL_LAST_ABI_VERSION()
 
+PROVIDES(YT_CODEC_CODEGEN)
+
 END()
 
+RECURSE(
+    no_llvm
+)
+
 RECURSE_FOR_TESTS(
     ut
 )

ydb/library/yql/providers/yt/codec/codegen/yt_codec_cg_dummy.cpp

@@ -0,0 +1,38 @@
+#include "yt_codec_cg.h"
+
+#include <util/generic/yexception.h>
+
+namespace NYql {
+
+TString GetYtCodecBitCode() {
+    throw yexception() << "Codegen is not available";
+}
+
+void YtCodecAddMappings(NCodegen::ICodegen& codegen) {
+    Y_UNUSED(codegen);
+    throw yexception() << "Codegen is not available";
+}
+
+template<bool Flat>
+THolder<IYtCodecCgWriter> MakeYtCodecCgWriter(const std::unique_ptr<NCodegen::ICodegen>& codegen, const void* cookie) {
+    Y_UNUSED(codegen);
+    Y_UNUSED(cookie);
+    throw yexception() << "Codegen is not available";
+}
+
+template THolder<IYtCodecCgWriter> MakeYtCodecCgWriter<true>(const std::unique_ptr<NCodegen::ICodegen>& codegen, const void* cookie);
+template THolder<IYtCodecCgWriter> MakeYtCodecCgWriter<false>(const std::unique_ptr<NCodegen::ICodegen>& codegen, const void* cookie);
+
+THolder<IYtCodecCgReader> MakeYtCodecCgReader(const std::unique_ptr<NCodegen::ICodegen>& codegen,
+    const NKikimr::NMiniKQL::THolderFactory& holderFactory, const void* cookie) {
+    Y_UNUSED(codegen);
+    Y_UNUSED(holderFactory);
+    Y_UNUSED(cookie);
+    throw yexception() << "Codegen is not available";
+}
+
+extern "C" void ThrowBadDecimal() {
+    throw yexception() << "Codegen is not available";
+}
+
+}

ydb/library/yql/providers/yt/codec/ut/no_llvm/ya.make

@@ -0,0 +1,26 @@
+UNITTEST_FOR(ydb/library/yql/providers/yt/codec)
+
+SRCDIR(
+    ydb/library/yql/providers/yt/codec/ut
+)
+
+SRCS(
+    yt_codec_io_ut.cpp
+)
+
+PEERDIR(
+    library/cpp/yson/node
+    ydb/library/yql/minikql/codegen/no_llvm
+    ydb/library/yql/minikql/computation/no_llvm
+    ydb/library/yql/public/udf/service/exception_policy
+    ydb/library/yql/sql
+    ydb/library/yql/sql/pg_dummy
+    ydb/library/yql/providers/common/codec
+    ydb/library/yql/providers/common/mkql
+    ydb/library/yql/providers/yt/lib/yson_helpers
+    ydb/library/yql/providers/yt/codec/codegen/no_llvm
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()

ydb/library/yql/providers/yt/codec/ut/ya.make

@@ -13,6 +13,7 @@ PEERDIR(
     ydb/library/yql/providers/common/codec
     ydb/library/yql/providers/common/mkql
     ydb/library/yql/providers/yt/lib/yson_helpers
+    ydb/library/yql/providers/yt/codec/codegen
 )
 
 YQL_LAST_ABI_VERSION()

ydb/library/yql/providers/yt/codec/ya.make

@@ -26,11 +26,7 @@ PEERDIR(
     ydb/library/yql/providers/yt/lib/skiff
 )
 
-IF (NOT MKQL_DISABLE_CODEGEN)
-    PEERDIR(
-        ydb/library/yql/providers/yt/codec/codegen
-    )
-ELSE()
+IF (MKQL_DISABLE_CODEGEN)
     CFLAGS(
         -DMKQL_DISABLE_CODEGEN
     )
@@ -46,4 +42,5 @@ RECURSE(
 
 RECURSE_FOR_TESTS(
     ut
+    ut/no_llvm
 )

ydb/library/yql/providers/yt/codec/yt_codec_io.cpp

@@ -1469,7 +1469,7 @@ void TMkqlReaderImpl::SetSpecs(const TMkqlIOSpecs& specs, const NKikimr::NMiniKQ
     Buf_.SetStats(JobStats_);
     if (Specs_->UseSkiff_) {
 #ifndef MKQL_DISABLE_CODEGEN
-        if (Specs_->OptLLVM_ != "OFF") {
+        if (Specs_->OptLLVM_ != "OFF" && NCodegen::ICodegen::IsCodegenAvailable()) {
             Decoder_.Reset(new TSkiffLLVMDecoder(Buf_, *Specs_, holderFactory));
         }
         else
@@ -1976,7 +1976,7 @@ void TMkqlWriterImpl::SetSpecs(const TMkqlIOSpecs& specs, const TVector<TString>
 
 #ifndef MKQL_DISABLE_CODEGEN
     THashMap<TStructType*, std::pair<llvm::Function*, llvm::Function*>> llvmFunctions;
-    if (Specs_->UseSkiff_ && Specs_->OptLLVM_ != "OFF") {
+    if (Specs_->UseSkiff_ && Specs_->OptLLVM_ != "OFF" && NCodegen::ICodegen::IsCodegenAvailable()) {
         for (size_t i: xrange(Specs_->Outputs.size())) {
             auto rowType = Specs_->Outputs[i].RowType;
             if (rowType->GetMembersCount() != 0 && !llvmFunctions.contains(rowType)) {

ydb/library/yql/providers/yt/comp_nodes/dq/dq_yt_block_reader.cpp

@@ -624,4 +624,4 @@ IComputationNode* CreateDqYtReadBlockWrapper(const TComputationNodeFactoryContex
 {
     return new TDqYtReadBlockWrapper(ctx, clusterName, token, inputSpec, samplingSpec, inputGroups, itemType, tableNames, std::move(tables), jobStats, inflight, timeout);
 }
-}
+}

ydb/library/yql/providers/yt/comp_nodes/dq/ya.make

@@ -3,7 +3,6 @@ LIBRARY()
 PEERDIR(
     ydb/library/yql/minikql
     ydb/library/yql/minikql/computation/llvm
-    ydb/library/yql/minikql/computation
     ydb/library/yql/providers/yt/comp_nodes
     ydb/library/yql/providers/yt/codec
     ydb/library/yql/providers/common/codec
@@ -45,8 +44,6 @@ SRCS(
     dq_yt_writer.cpp
 )
 
-INCLUDE(../../../../minikql/computation/header.ya.make.inc)
-
 YQL_LAST_ABI_VERSION()
 
 

ydb/library/yql/providers/yt/comp_nodes/llvm/ya.make

@@ -0,0 +1,23 @@
+LIBRARY()
+
+OWNER(
+    g:yql
+    g:yql_ydb_core
+)
+
+NO_COMPILER_WARNINGS()
+
+PEERDIR(
+    ydb/library/yql/minikql/codegen/llvm
+    ydb/library/yql/minikql/invoke_builtins/llvm
+    contrib/libs/llvm12/lib/IR
+    contrib/libs/llvm12/lib/ExecutionEngine/MCJIT
+    contrib/libs/llvm12/lib/Linker
+    contrib/libs/llvm12/lib/Target/X86
+    contrib/libs/llvm12/lib/Target/X86/AsmParser
+    contrib/libs/llvm12/lib/Transforms/IPO
+)
+
+INCLUDE(../ya.make.inc)
+
+END()

ydb/library/yql/providers/yt/comp_nodes/llvm14/ya.make

@@ -0,0 +1,23 @@
+LIBRARY()
+
+OWNER(
+    g:yql
+    g:yql_ydb_core
+)
+
+NO_COMPILER_WARNINGS()
+
+PEERDIR(
+    ydb/library/yql/minikql/codegen/llvm14
+    ydb/library/yql/minikql/invoke_builtins/llvm14
+    contrib/libs/llvm14/lib/IR
+    contrib/libs/llvm14/lib/ExecutionEngine/MCJIT
+    contrib/libs/llvm14/lib/Linker
+    contrib/libs/llvm14/lib/Target/X86
+    contrib/libs/llvm14/lib/Target/X86/AsmParser
+    contrib/libs/llvm14/lib/Transforms/IPO
+)
+
+INCLUDE(../ya.make.inc)
+
+END()

ydb/library/yql/providers/yt/comp_nodes/no_llvm/ya.make

@@ -0,0 +1,16 @@
+LIBRARY()
+
+OWNER(
+    g:yql
+    g:yql_ydb_core
+)
+
+CXXFLAGS(-DMKQL_DISABLE_CODEGEN)
+
+ADDINCL(GLOBAL ydb/library/yql/minikql/codegen/llvm_stub)
+
+INCLUDE(../ya.make.inc)
+
+PEERDIR(ydb/library/yql/minikql/invoke_builtins/no_llvm)
+
+END()