Update Pony to use LLVM 12.0.0

- Update the LLVM submodule to `llvmorg-12.0.0` - Update the JIT code to work with LLVM 12. - Update custom optimizations to work with LLVM 12. - Move old JIT and optimization code to separate files for historical reference - Update handling LLVM library dependencies: - Consolidate LLVM library configuration in the root `CMakeLists.txt` file. - Deprecate the `llvm_config` cmake macro in favour of using individual components with `llvm_map_components_to_libnames`. - Include the in-progress work on Mac M1: - Update CMakeLists.txt to require C++14 (LLVM now does). - Add missing header to cpu.c that is required on Apple M1. - Fixes ponylang#1930
ergl · May 26, 2021 · 3b8d9b0 · 3b8d9b0
1 parent 847c834
commit 3b8d9b0
Show file tree

Hide file tree

Showing 28 changed files with 2,521 additions and 537 deletions.
diff --git a/.ci-dockerfiles/x86-64-unknown-linux-musl-builder/Dockerfile b/.ci-dockerfiles/x86-64-unknown-linux-musl-builder/Dockerfile
@@ -16,7 +16,7 @@ RUN apk update \
   zlib-dev \
   bash \
   curl \
-  py-pip \
+  py3-pip \
 && pip install cloudsmith-cli
 
 # add user pony in order to not run tests as root

diff --git a/.ci-dockerfiles/x86-64-unknown-linux-ubuntu18.04-builder/Dockerfile b/.ci-dockerfiles/x86-64-unknown-linux-ubuntu18.04-builder/Dockerfile
@@ -8,13 +8,14 @@ RUN apt-get update \
   curl \
   git \
   make \
-  python-pip \
+  python3-pip \
+  python3-setuptools \
   xz-utils \
   zlib1g-dev \
  && rm -rf /var/lib/apt/lists/* \
  && apt-get -y autoremove --purge \
  && apt-get -y clean \
- && pip install cloudsmith-cli
+ && pip3 install cloudsmith-cli
 
 # install a newer cmake
 RUN curl --output cmake-3.15.3-Linux-x86_64.sh https://cmake.org/files/v3.15/cmake-3.15.3-Linux-x86_64.sh \

diff --git a/.cirrus.yml b/.cirrus.yml
@@ -22,9 +22,9 @@ task:
       TRIPLE_OS: linux-gnu
   - name: "PR: x86-64-unknown-linux-ubuntu18.04"
     container:
-      image: ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu18.04-builder:20200507
+      image: ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu18.04-builder:20210427
     environment:
-      CACHE_BUSTER: 20210224
+      CACHE_BUSTER: 20210427
       TRIPLE_VENDOR: unknown
       TRIPLE_OS: linux-ubuntu18.04
   - name: "PR: x86-64-unknown-linux-ubuntu20.04"
@@ -36,7 +36,7 @@ task:
       TRIPLE_OS: linux-ubuntu20.04
   - name: "PR: x86-64-unknown-linux-musl"
     container:
-      image: ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20200421
+      image: ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20210420
     environment:
       CACHE_BUSTER: 20210224
       TRIPLE_VENDOR: unknown
@@ -136,13 +136,15 @@ task:
   only_if: $CIRRUS_PR != ''
 
   windows_container:
-    image: ponylang/ponyc-ci-x86-64-pc-windows-msvc-builder:20201208
+    image: ponylang/ponyc-ci-x86-64-pc-windows-msvc-builder:20210427
     os_version: 2019
     cpu: 8
     memory: 24
 
   name: "PR: x86-64-pc-windows-msvc"
 
+  timeout_in: 120m
+
   libs_cache:
     folder: build/libs
     fingerprint_script:
@@ -161,7 +163,7 @@ task:
   only_if: $CIRRUS_PR != ''
 
   windows_container:
-    image: ponylang/ponyc-ci-x86-64-pc-windows-msvc-builder:20201208
+    image: ponylang/ponyc-ci-x86-64-pc-windows-msvc-builder:20210427
     os_version: 2019
     cpu: 8
     memory: 24
@@ -261,9 +263,9 @@ task:
         TRIPLE_OS: linux-gnu
     - name: "nightly: x86-64-unknown-linux-ubuntu18.04"
       container:
-        image: ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu18.04-builder:20200507
+        image: ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu18.04-builder:20210427
       environment:
-        CACHE_BUSTER: 20210224
+        CACHE_BUSTER: 20210427
         TRIPLE_VENDOR: unknown
         TRIPLE_OS: linux-ubuntu18.04
     - name: "nightly: x86-64-unknown-linux-ubuntu20.04"
@@ -275,7 +277,7 @@ task:
         TRIPLE_OS: linux-ubuntu20.04
     - name: "nightly: x86-64-unknown-linux-musl"
       container:
-        image: ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20200421
+        image: ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20210420
       environment:
         CACHE_BUSTER: 20210224
         TRIPLE_VENDOR: unknown
@@ -407,9 +409,9 @@ task:
         TRIPLE_OS: linux-gnu
     - name: "release: x86-64-unknown-linux-ubuntu18.04"
       container:
-        image: ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu18.04-builder:20200507
+        image: ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu18.04-builder:20210427
       environment:
-        CACHE_BUSTER: 20200507
+        CACHE_BUSTER: 20210427
         TRIPLE_VENDOR: unknown
         TRIPLE_OS: linux-ubuntu18.04
     - name: "release: x86-64-unknown-linux-ubuntu20.04"
@@ -421,7 +423,7 @@ task:
         TRIPLE_OS: linux-ubuntu20.04
     - name: "release: x86-64-unknown-linux-musl"
       container:
-        image: ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20200421
+        image: ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20210420
       environment:
         CACHE_BUSTER: 20200421
         TRIPLE_VENDOR: unknown

diff --git a/.release-notes/3745.md b/.release-notes/3745.md
@@ -0,0 +1,3 @@
+## Update to LLVM 12.0.0
+
+We have updated the vendored LLVM submodule to tag `llvmorg-12.0.0`, and updated the build system, code generation, and JIT code to work with LLVM 12.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -18,6 +18,10 @@ endif()
 # Uncomment this to show build commands
 # set(CMAKE_VERBOSE_MAKEFILE ON)
 
+# We require C++14 (because LLVM does)
+set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
 # We require LLVM, Google Test and Google Benchmark
 if(NOT PONY_CROSS_LIBPONYRT)
     find_package(LLVM REQUIRED CONFIG PATHS "build/libs/lib/cmake/llvm" "build/libs/lib64/cmake/llvm" NO_DEFAULT_PATH)
@@ -103,6 +107,33 @@ if("${_compiler_arch}" STREQUAL "")
     set(_compiler_arch ${CMAKE_SYSTEM_PROCESSOR})
 endif()
 
+# LLVM component setup
+message("Compiler architecture is ${_compiler_arch}")
+
+if(NOT PONY_CROSS_LIBPONYRT)
+    set(LLVM_COMPONENTS
+        core
+        demangle
+        objcarcopts
+        orcjit
+        x86asmparser
+        x86codegen
+        x86desc
+        x86info
+        aarch64asmparser
+        aarch64codegen
+        aarch64desc
+        aarch64info
+        armasmparser
+        armcodegen
+        armdesc
+        arminfo
+    )
+
+    llvm_map_components_to_libnames(PONYC_LLVM_LIBS ${LLVM_COMPONENTS})
+    # message("PONYC_LLVM_LIBS: ${PONYC_LLVM_LIBS}")
+endif()
+
 # Required definitions.  We use these generators so that the defines are correct for both *nix (where the config applies at configuration time) and Windows (where the config applies at build time).
 add_compile_definitions(
     BUILD_COMPILER="${CMAKE_C_COMPILER_VERSION}"
@@ -165,8 +196,8 @@ else()
 endif()
 
 if(${CMAKE_HOST_SYSTEM_NAME} MATCHES "Darwin")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mmacosx-version-min=10.12 -DUSE_SCHEDULER_SCALING_PTHREADS")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++ -mmacosx-version-min=10.12")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mmacosx-version-min=10.14 -DUSE_SCHEDULER_SCALING_PTHREADS")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++ -mmacosx-version-min=10.14")
 endif()
 
 if(${CMAKE_HOST_SYSTEM_NAME} MATCHES "DragonFly")
@@ -175,9 +206,6 @@ if(${CMAKE_HOST_SYSTEM_NAME} MATCHES "DragonFly")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem /usr/local/cxx_atomics")
 endif()
 
-
-
-
 set(CMAKE_STATIC_LIBRARY_PREFIX "")
 
 if (PONY_CROSS_LIBPONYRT)

diff --git a/Makefile b/Makefile
@@ -2,7 +2,7 @@ config ?= release
 arch ?= native
 tune ?= generic
 build_flags ?= -j2
-llvm_archs ?= X86
+llvm_archs ?= X86;ARM;AArch64
 llvm_config ?= Release
 llc_arch ?= x86-64
 

diff --git a/benchmark/libponyc/CMakeLists.txt b/benchmark/libponyc/CMakeLists.txt
@@ -13,8 +13,6 @@ target_include_directories(libponyc.benchmarks
     PRIVATE ../../build/libs/include
 )
 
-llvm_config(libponyc.benchmarks all)
-
 target_link_directories(libponyc.benchmarks
     PRIVATE ../../build/libs/lib
 )
@@ -33,14 +31,12 @@ if (MSVC)
         PRIVATE Ws2_32
         PRIVATE Shlwapi
     )
-
-    get_target_property(TARGET_LIBRARIES libponyc.benchmarks LINK_LIBRARIES)
-    LIST(REMOVE_ITEM TARGET_LIBRARIES LTO)
-    set_property(TARGET libponyc.benchmarks PROPERTY LINK_LIBRARIES ${TARGET_LIBRARIES})
 elseif(${CMAKE_HOST_SYSTEM_NAME} MATCHES "Darwin")
     target_link_libraries(libponyc.benchmarks PRIVATE dl)
-elseif(${CMAKE_HOST_SYSTEM_NAME} MATCHES "BSD|DragonFly")
+elseif(${CMAKE_HOST_SYSTEM_NAME} MATCHES "BSD")
     target_link_libraries(libponyc.benchmarks PRIVATE execinfo)
+elseif(${CMAKE_HOST_SYSTEM_NAME} MATCHES "DragonFly")
+    target_link_libraries(libponyc.benchmarks PRIVATE execinfo atomic)
 else()
     target_link_libraries(libponyc.benchmarks PRIVATE atomic dl)
     target_link_options(libponyc.benchmarks PRIVATE "-static-libstdc++")
@@ -49,3 +45,7 @@ endif ()
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
     target_link_options(libponyc.benchmarks PRIVATE "-static-libgcc")
 endif()
+
+if(NOT PONY_CROSS_LIBPONYRT)
+    target_link_libraries(libponyc.benchmarks PRIVATE ${PONYC_LLVM_LIBS})
+endif()
diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
@@ -10,12 +10,12 @@ if(NOT DEFINED PONYC_LIBS_BUILD_TYPE)
 endif()
 
 ExternalProject_Add(gbenchmark
-    URL https://github.com/google/benchmark/archive/v1.5.2.tar.gz
+    URL https://github.com/google/benchmark/archive/v1.5.1.tar.gz
     CMAKE_ARGS -DCMAKE_BUILD_TYPE=${PONYC_LIBS_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} -DBENCHMARK_ENABLE_GTEST_TESTS=OFF -DCMAKE_CXX_FLAGS=-fpic --no-warn-unused-cli
 )
 
 ExternalProject_Add(googletest
-    URL https://github.com/google/googletest/archive/release-1.8.1.tar.gz
+    URL https://github.com/google/googletest/archive/release-1.10.0.tar.gz
     CMAKE_ARGS -DCMAKE_BUILD_TYPE=${PONYC_LIBS_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} -DCMAKE_CXX_FLAGS=-fpic -Dgtest_force_shared_crt=ON --no-warn-unused-cli
 )
 
@@ -30,8 +30,8 @@ install(TARGETS blake2
 
 find_package(Git)
 
-set(LLVM_DESIRED_HASH "c1a0a213378a458fbea1a5c77b315c7dce08fd05")
-set(PATCHES_DESIRED_HASH "9063f83d727bf042a1232420e168c1ea192bf6a2960d35e57123245b630eb923")
+set(LLVM_DESIRED_HASH "d28af7c654d8db0b68c175db5ce212d74fb5e9bc")
+set(PATCHES_DESIRED_HASH "3a655193262fd9b2e87340e096efcbd96726a07fe6dd42a263f3a4fc2dc0192e")
 
 if(GIT_FOUND)
     if(EXISTS "${PROJECT_SOURCE_DIR}/../.git")
@@ -65,12 +65,19 @@ if(GIT_FOUND)
     file(GLOB PONY_LLVM_PATCHES "${PROJECT_SOURCE_DIR}/llvm/patches/*.diff")
 
     # check to see if the patch hashes match
+    message("Checking patches ${PONY_LLVM_PATCHES}")
     set(PATCHES_ACTUAL_HASH "")
     foreach (PATCH ${PONY_LLVM_PATCHES})
-        file(SHA256 ${PATCH} patch_file_hash)
-        string(CONCAT PATCHES_ACTUAL_HASH patch_file_hash)
+        file(STRINGS ${PATCH} patch_file NEWLINE_CONSUME)
+        string(REPLACE "\n" " " patch_file ${patch_file})
+        string(SHA256 patch_file_hash ${patch_file})
+        # message("${PATCH}: '${patch_file_hash}'")
+        string(CONCAT PATCHES_ACTUAL_HASH ${PATCHES_ACTUAL_HASH} ${patch_file_hash})
+        # message("concat is '${PATCHES_ACTUAL_HASH}'")
     endforeach()
     string(SHA256 PATCHES_ACTUAL_HASH ${PATCHES_ACTUAL_HASH})
+    # message("Desired hash ${PATCHES_DESIRED_HASH}")
+    # message("Actual hash  ${PATCHES_ACTUAL_HASH}")
     if(NOT PATCHES_ACTUAL_HASH EQUAL "${PATCHES_DESIRED_HASH}")
         message(FATAL_ERROR "Patch hash actual ${PATCHES_ACTUAL_HASH} does not match desired ${PATCHES_DESIRED_HASH}")
     endif()
@@ -97,9 +104,7 @@ else()
     message(FATAL_ERROR "Git not found!")
 endif()
 
-if (NOT DEFINED LLVM_TARGETS_TO_BUILD)
-  set(LLVM_TARGETS_TO_BUILD X86)
-endif()
+message("Building targets: ${LLVM_TARGETS_TO_BUILD}")
 
 set(LLVM_ENABLE_BINDINGS OFF)
 set(LLVM_ENABLE_OCAMLDOC OFF)
@@ -112,4 +117,9 @@ set(LLVM_INCLUDE_TESTS OFF)
 set(LLVM_TOOL_REMARKS_SHLIB_BUILD OFF)
 set(LLVM_ENABLE_ZLIB OFF)
 
+if(${CMAKE_HOST_SYSTEM_NAME} MATCHES "Darwin")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mmacosx-version-min=10.14 -DUSE_SCHEDULER_SCALING_PTHREADS")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++ -mmacosx-version-min=10.14")
+endif()
+
 add_subdirectory(llvm/src/llvm)
diff --git a/lib/llvm/patches/2019-12-23-01-jit-eh-frames.diff b/lib/llvm/patches/2019-12-23-01-jit-eh-frames.diff
diff --git a/...llvm/patches/2020-01-07-01-c-exports.diff → ...llvm/patches/2020-07-28-01-c-exports.diff b/...llvm/patches/2020-01-07-01-c-exports.diff → ...llvm/patches/2020-07-28-01-c-exports.diff
@@ -1,8 +1,8 @@
 diff --git a/llvm/tools/llvm-shlib/CMakeLists.txt b/llvm/tools/llvm-shlib/CMakeLists.txt
-index 901f55c99d5..66c034470ad 100644
+index f3a2056f80d..3339dc58218 100644
 --- a/llvm/tools/llvm-shlib/CMakeLists.txt
 +++ b/llvm/tools/llvm-shlib/CMakeLists.txt
-@@ -91,9 +91,9 @@ if(LLVM_BUILD_LLVM_C_DYLIB AND NOT MSVC)
+@@ -79,9 +79,9 @@ if(LLVM_BUILD_LLVM_C_DYLIB AND NOT MSVC)
    # To get the export list for a single llvm library:
    # nm ${LIB_PATH} | awk "/T _LLVM/ { print $3 }" | sort -u | sed -e "s/^_//g" > ${LIB_PATH}.exports
 
@@ -14,20 +14,7 @@ index 901f55c99d5..66c034470ad 100644
    set(LIB_NAME ${LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}LLVM)
    set(LIB_PATH ${LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
    set(LIB_EXPORTS_PATH ${LIB_NAME}.exports)
-@@ -109,10 +109,10 @@ if(LLVM_BUILD_LLVM_C_DYLIB AND NOT MSVC)
-   add_custom_target(libLLVMCExports DEPENDS ${LLVM_EXPORTED_SYMBOL_FILE})
-
-   add_llvm_library(LLVM-C SHARED ${SOURCES} INSTALL_WITH_TOOLCHAIN)
--  
-+
-   target_link_libraries(LLVM-C PUBLIC LLVM)
-   add_dependencies(LLVM-C libLLVMCExports)
--  
-+
-   set_property(TARGET LLVM-C APPEND_STRING PROPERTY
-               LINK_FLAGS
-               " -compatibility_version 1 -current_version ${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH} -Wl,-reexport_library ${LIB_PATH}")
-@@ -137,11 +137,11 @@ if(MSVC)
+@@ -125,11 +125,11 @@ if(LLVM_BUILD_LLVM_C_DYLIB AND MSVC)
 
    # Set this name here, not used in multi conf loop,
    # but add script will pick the right one.
@@ -40,8 +27,8 @@ index 901f55c99d5..66c034470ad 100644
 +    list(APPEND FULL_LIB_NAMES ${LLVM_BINARY_DIR}/${CMAKE_CFG_INTDIR}/lib/${lib}.lib)
    endforeach()
 
-   # Need to seperate lib names with newlines.
-@@ -155,7 +155,7 @@ if(MSVC)
+   # Need to separate lib names with newlines.
+@@ -143,7 +143,7 @@ if(LLVM_BUILD_LLVM_C_DYLIB AND MSVC)
        # Write out the full lib names into file to be read by the python script.
        # One libsfile per build, the add_custom_command should expand
        # ${CMAKE_CFG_INTDIR} correctly and select the right one.
@@ -50,12 +37,12 @@ index 901f55c99d5..66c034470ad 100644
      endforeach()
    else()
      # Write out the full lib names into file to be read by the python script.
-@@ -165,7 +165,7 @@ if(MSVC)
+@@ -153,7 +153,7 @@ if(LLVM_BUILD_LLVM_C_DYLIB AND MSVC)
    # Generate the exports file dynamically.
    set(GEN_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/gen-msvc-exports.py)
 
 -  set(LLVM_EXPORTED_SYMBOL_FILE ${CMAKE_BINARY_DIR}/${CMAKE_CFG_INTDIR}/libllvm-c.exports)
 +  set(LLVM_EXPORTED_SYMBOL_FILE ${LLVM_BINARY_DIR}/${CMAKE_CFG_INTDIR}/libllvm-c.exports)
-
-   if(CMAKE_CROSSCOMPILING)
-     build_native_tool(llvm-nm llvm_nm)
+   if(NOT LLVM_NM)
+     if(CMAKE_CROSSCOMPILING)
+       build_native_tool(llvm-nm llvm_nm)
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		## Update to LLVM 12.0.0

		We have updated the vendored LLVM submodule to tag `llvmorg-12.0.0`, and updated the build system, code generation, and JIT code to work with LLVM 12.