YQL-15941 llvm14 initial commit

build/plugins/cp.py

@@ -4,14 +4,25 @@
 
 
 def oncopy(unit, *args):
-    keywords = {'RESULT': 1, 'KEEP_DIR_STRUCT': 0, 'DESTINATION': 1, 'FROM': 1}
+    keywords = {
+        'RESULT': 1,
+        'KEEP_DIR_STRUCT': 0,
+        'DESTINATION': 1,
+        'FROM': 1,
+        'OUTPUT_INCLUDES': -1,
+        'AUTO': 0,
+        'WITH_CONTEXT': 0,
+    }
 
     flat_args, spec_args = sort_by_keywords(keywords, args)
 
     dest_dir = spec_args['DESTINATION'][0] if 'DESTINATION' in spec_args else ''
     from_dir = spec_args['FROM'][0] if 'FROM' in spec_args else ''
+    output_includes = spec_args['OUTPUT_INCLUDES'] if 'OUTPUT_INCLUDES' in spec_args else None
     keep_struct = 'KEEP_DIR_STRUCT' in spec_args
     save_in_var = 'RESULT' in spec_args
+    auto = 'AUTO' in spec_args
+    with_context = 'WITH_CONTEXT' in spec_args
     targets = []
 
     for source in flat_args:
@@ -25,6 +36,12 @@ def oncopy(unit, *args):
         target_path = os.path.join(dest_dir, rel_path, filename)
         if save_in_var:
             targets.append(target_path)
-        unit.oncopy_file([source_path, target_path])
+        unit.oncopy_file(
+            [source_path, target_path]
+            + (['OUTPUT_INCLUDES'] + output_includes if output_includes else [])
+            + (['OUTPUT_INCLUDES', source_path] if with_context else [])
+            + (['AUTO'] if auto else [])
+        )
+
     if save_in_var:
         unit.set([spec_args["RESULT"][0], " ".join(targets)])

build/plugins/llvm_bc.py

@@ -2,10 +2,12 @@
 
 
 def onllvm_bc(unit, *args):
-    free_args, kwds = sort_by_keywords({'SYMBOLS': -1, 'NAME': 1, 'GENERATE_MACHINE_CODE': 0, 'NO_COMPILE': 0}, args)
+    free_args, kwds = sort_by_keywords(
+        {'SYMBOLS': -1, 'NAME': 1, 'GENERATE_MACHINE_CODE': 0, 'NO_COMPILE': 0, 'SUFFIX': 1}, args
+    )
     name = kwds['NAME'][0]
     symbols = kwds.get('SYMBOLS')
-    obj_suf = unit.get('OBJ_SUF')
+    obj_suf = kwds['SUFFIX'][0] if 'SUFFIX' in kwds else '' + unit.get('OBJ_SUF')
     skip_compile_step = 'NO_COMPILE' in kwds
     merged_bc = name + '_merged' + obj_suf + '.bc'
     out_bc = name + '_optimized' + obj_suf + '.bc'

ydb/core/engine/mkql_engine_flat_extfunc.cpp

@@ -2,6 +2,7 @@
 #include "mkql_engine_flat_impl.h"
 #include "mkql_keys.h"
 #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders_codegen.h>
 #include <ydb/library/yql/minikql/computation/mkql_computation_node_pack.h>
 #include <ydb/library/yql/minikql/computation/mkql_custom_list.h>
 #include <ydb/library/yql/minikql/mkql_node_cast.h>

ydb/core/kqp/runtime/kqp_compute.cpp

@@ -1,6 +1,7 @@
 #include "kqp_compute.h"
 
 #include <ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders_codegen.h>
 #include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h>
 #include <ydb/library/yql/minikql/mkql_node_cast.h>
 #include <ydb/library/yql/minikql/mkql_program_builder.h>

ydb/library/yql/core/spilling/storage/file_storage/ya.make

@@ -10,22 +10,6 @@ PEERDIR(
 
 NO_COMPILER_WARNINGS()
 
-IF (NOT MKQL_DISABLE_CODEGEN)
-    PEERDIR(
-        ydb/library/yql/minikql/codegen
-        contrib/libs/llvm12/lib/IR
-        contrib/libs/llvm12/lib/ExecutionEngine/MCJIT
-        contrib/libs/llvm12/lib/Linker
-        contrib/libs/llvm12/lib/Target/X86
-        contrib/libs/llvm12/lib/Target/X86/AsmParser
-        contrib/libs/llvm12/lib/Transforms/IPO
-    )
-ELSE()
-    CFLAGS(
-        -DMKQL_DISABLE_CODEGEN
-    )
-ENDIF()
-
 YQL_LAST_ABI_VERSION()
 
 END()

ydb/library/yql/core/spilling/storage/ya.make

@@ -11,22 +11,6 @@ PEERDIR(
 
 NO_COMPILER_WARNINGS()
 
-IF (NOT MKQL_DISABLE_CODEGEN)
-    PEERDIR(
-        ydb/library/yql/minikql/codegen
-        contrib/libs/llvm12/lib/IR
-        contrib/libs/llvm12/lib/ExecutionEngine/MCJIT
-        contrib/libs/llvm12/lib/Linker
-        contrib/libs/llvm12/lib/Target/X86
-        contrib/libs/llvm12/lib/Target/X86/AsmParser
-        contrib/libs/llvm12/lib/Transforms/IPO
-    )
-ELSE()
-    CFLAGS(
-        -DMKQL_DISABLE_CODEGEN
-    )
-ENDIF()
-
 YQL_LAST_ABI_VERSION()
 
 END()

ydb/library/yql/core/spilling/ya.make

@@ -23,22 +23,6 @@ PEERDIR(
 
 NO_COMPILER_WARNINGS()
 
-IF (NOT MKQL_DISABLE_CODEGEN)
-    PEERDIR(
-        ydb/library/yql/minikql/codegen
-        contrib/libs/llvm12/lib/IR
-        contrib/libs/llvm12/lib/ExecutionEngine/MCJIT
-        contrib/libs/llvm12/lib/Linker
-        contrib/libs/llvm12/lib/Target/X86
-        contrib/libs/llvm12/lib/Target/X86/AsmParser
-        contrib/libs/llvm12/lib/Transforms/IPO
-    )
-ELSE()
-    CFLAGS(
-        -DMKQL_DISABLE_CODEGEN
-    )
-ENDIF()
-
 YQL_LAST_ABI_VERSION()
 
 END()

ydb/library/yql/minikql/codegen/codegen.cpp

@@ -1,30 +1,5 @@
 #include "codegen.h"
-#include <llvm/ExecutionEngine/ExecutionEngine.h>
-#include <llvm/ExecutionEngine/JITEventListener.h>
-#include <llvm/ExecutionEngine/MCJIT.h>
-#include <llvm/IR/DiagnosticInfo.h>
-#include <llvm/IR/DiagnosticPrinter.h>
-#include <llvm/IR/LegacyPassManager.h>
-#include <llvm/IR/LLVMContext.h>
-#include <llvm/IR/Module.h>
-#include <llvm/IR/Verifier.h>
-#include <llvm/IRReader/IRReader.h>
-#include <llvm/Linker/Linker.h>
-#include <llvm-c/Disassembler.h>
-#include <llvm/Support/Host.h>
-#include <llvm/Support/ManagedStatic.h>
-#include <llvm/Support/SourceMgr.h>
-#include <llvm/Support/TargetSelect.h>
-#include <llvm/Support/Timer.h>
-#include <llvm/Support/ErrorHandling.h>
-#include <llvm/Transforms/IPO.h>
-#include <llvm/Transforms/IPO/PassManagerBuilder.h>
-#include <llvm/Transforms/Instrumentation.h>
-#include <llvm/Transforms/Instrumentation/AddressSanitizer.h>
-#include <llvm/Transforms/Instrumentation/MemorySanitizer.h>
-#include <llvm/Transforms/Instrumentation/ThreadSanitizer.h>
-#include <llvm/LinkAllPasses.h>
-
+#include "codegen_llvm_deps.h" // Y_IGNORE
 #include <contrib/libs/re2/re2/re2.h>
 
 #include <util/generic/maybe.h>
@@ -211,10 +186,16 @@ namespace NCodegen {
 
 namespace {
 
-    void FatalErrorHandler(void* user_data, const std::string& reason, bool gen_crash_diag) {
+    void FatalErrorHandler(void* user_data,
+#if LLVM_VERSION_MAJOR == 12
+    const std::string& reason
+#else
+    const char* reason
+#endif
+    , bool gen_crash_diag) {
         Y_UNUSED(user_data);
         Y_UNUSED(gen_crash_diag);
-        ythrow yexception() << "LLVM fatal error: " << reason.c_str();
+        ythrow yexception() << "LLVM fatal error: " << reason;
     }
 
     void AddAddressSanitizerPasses(const llvm::PassManagerBuilder& builder, llvm::legacy::PassManagerBase& pm) {
@@ -313,7 +294,9 @@ class TCodegen : public ICodegen, private llvm::JITEventListener {
         targetOptions.EnableFastISel = true;
         // init manually, this field was lost in llvm::TargetOptions ctor :/
         // make coverity happy
+#if LLVM_VERSION_MAJOR == 12
         targetOptions.StackProtectorGuardOffset = 0;
+#endif        
 
         std::string what;
         auto&& engineBuilder = llvm::EngineBuilder(std::move(module));

ydb/library/yql/minikql/codegen/codegen_llvm_deps.h.txt

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <llvm/ExecutionEngine/ExecutionEngine.h>
+#include <llvm/ExecutionEngine/JITEventListener.h>
+#include <llvm/ExecutionEngine/MCJIT.h>
+#include <llvm/IR/DiagnosticInfo.h>
+#include <llvm/IR/DiagnosticPrinter.h>
+#include <llvm/IR/LegacyPassManager.h>
+#include <llvm/IR/LLVMContext.h>
+#include <llvm/IR/Module.h>
+#include <llvm/IR/Verifier.h>
+#include <llvm/IRReader/IRReader.h>
+#include <llvm/Linker/Linker.h>
+#include <llvm-c/Disassembler.h>
+#include <llvm/Support/Host.h>
+#include <llvm/Support/ManagedStatic.h>
+#include <llvm/Support/SourceMgr.h>
+#include <llvm/Support/TargetSelect.h>
+#include <llvm/Support/Timer.h>
+#include <llvm/Support/ErrorHandling.h>
+#include <llvm/Transforms/IPO.h>
+#include <llvm/Transforms/IPO/PassManagerBuilder.h>
+#include <llvm/Transforms/Instrumentation.h>
+#include <llvm/Transforms/Instrumentation/AddressSanitizer.h>
+#include <llvm/Transforms/Instrumentation/MemorySanitizer.h>
+#include <llvm/Transforms/Instrumentation/ThreadSanitizer.h>
+#include <llvm/LinkAllPasses.h>

ydb/library/yql/minikql/codegen/codegen_ut.cpp

@@ -1,10 +1,6 @@
-#include "codegen.h"
+#include <ydb/library/yql/minikql/codegen/codegen.h>
 
-#include <llvm/IR/Constants.h>
-#include <llvm/IR/DerivedTypes.h>
-#include <llvm/IR/Instructions.h>
-#include <llvm/IR/LLVMContext.h>
-#include <llvm/IR/Module.h>
+#include <codegen_ut_llvm_deps.h> // Y_IGNORE
 
 #include <library/cpp/testing/unittest/registar.h>
 #include <library/cpp/resource/resource.h>

ydb/library/yql/minikql/codegen/codegen_ut_llvm_deps.h.txt

@@ -0,0 +1,7 @@
+#pragma once
+
+#include <llvm/IR/Constants.h>
+#include <llvm/IR/DerivedTypes.h>
+#include <llvm/IR/Instructions.h>
+#include <llvm/IR/LLVMContext.h>
+#include <llvm/IR/Module.h>

ydb/library/yql/minikql/codegen/llvm/ut/ya.make

@@ -0,0 +1,27 @@
+UNITTEST()
+
+PEERDIR(
+    contrib/libs/llvm12/lib/IR
+    contrib/libs/llvm12/lib/ExecutionEngine/MCJIT
+    contrib/libs/llvm12/lib/Linker
+    contrib/libs/llvm12/lib/Target/X86
+    contrib/libs/llvm12/lib/Target/X86/AsmParser
+    contrib/libs/llvm12/lib/Target/X86/Disassembler
+    contrib/libs/llvm12/lib/Transforms/IPO
+    contrib/libs/llvm12/lib/Transforms/ObjCARC
+)
+
+IF (OS_LINUX)
+    PEERDIR(
+        contrib/libs/llvm12/lib/ExecutionEngine/PerfJITEvents
+    )
+ENDIF()
+
+SET(LLVM_VER 12)
+
+INCLUDE(../../ut/ya.make.inc)
+
+PEERDIR(ydb/library/yql/minikql/codegen/llvm)
+
+END()
+

ydb/library/yql/minikql/codegen/llvm/ya.make

@@ -0,0 +1,26 @@
+LIBRARY()
+
+PEERDIR(
+    contrib/libs/llvm12/lib/IR
+    contrib/libs/llvm12/lib/ExecutionEngine/MCJIT
+    contrib/libs/llvm12/lib/Linker
+    contrib/libs/llvm12/lib/Target/X86
+    contrib/libs/llvm12/lib/Target/X86/AsmParser
+    contrib/libs/llvm12/lib/Target/X86/Disassembler
+    contrib/libs/llvm12/lib/Transforms/IPO
+    contrib/libs/llvm12/lib/Transforms/ObjCARC
+)
+
+IF (OS_LINUX)
+    PEERDIR(
+        contrib/libs/llvm12/lib/ExecutionEngine/PerfJITEvents
+    )
+ENDIF()
+
+INCLUDE(../ya.make.inc)
+
+END()
+
+RECURSE_FOR_TESTS(
+    ut
+)

ydb/library/yql/minikql/codegen/llvm14/ut/ya.make

@@ -0,0 +1,27 @@
+UNITTEST()
+
+PEERDIR(
+    contrib/libs/llvm14/lib/IR
+    contrib/libs/llvm14/lib/ExecutionEngine/MCJIT
+    contrib/libs/llvm14/lib/Linker
+    contrib/libs/llvm14/lib/Target/X86
+    contrib/libs/llvm14/lib/Target/X86/AsmParser
+    contrib/libs/llvm14/lib/Target/X86/Disassembler
+    contrib/libs/llvm14/lib/Transforms/IPO
+    contrib/libs/llvm14/lib/Transforms/ObjCARC
+)
+
+IF (OS_LINUX)
+    PEERDIR(
+        contrib/libs/llvm14/lib/ExecutionEngine/PerfJITEvents
+    )
+ENDIF()
+
+
+SET(LLVM_VER 14)
+
+INCLUDE(../../ut/ya.make.inc)
+
+PEERDIR(ydb/library/yql/minikql/codegen/llvm14)
+
+END()

ydb/library/yql/minikql/codegen/llvm14/ya.make

@@ -0,0 +1,26 @@
+LIBRARY()
+
+PEERDIR(
+    contrib/libs/llvm14/lib/IR
+    contrib/libs/llvm14/lib/ExecutionEngine/MCJIT
+    contrib/libs/llvm14/lib/Linker
+    contrib/libs/llvm14/lib/Target/X86
+    contrib/libs/llvm14/lib/Target/X86/AsmParser
+    contrib/libs/llvm14/lib/Target/X86/Disassembler
+    contrib/libs/llvm14/lib/Transforms/IPO
+    contrib/libs/llvm14/lib/Transforms/ObjCARC
+)
+
+IF (OS_LINUX)
+    PEERDIR(
+        contrib/libs/llvm14/lib/ExecutionEngine/PerfJITEvents
+    )
+ENDIF()
+
+INCLUDE(../ya.make.inc)
+
+END()
+
+RECURSE_FOR_TESTS(
+    ut
+)