Windows compilation: enable compiling expanded list of extensions in …

…envoy-static (#10542)

Co-authored-by: Sunjay Bhatia <sbhatia@pivotal.io>
Co-authored-by: William A Rowe Jr <wrowe@pivotal.io>
Signed-off-by: Sunjay Bhatia <sbhatia@pivotal.io>
Signed-off-by: William A Rowe Jr <wrowe@pivotal.io>

.azure-pipelines/pipelines.yml

@@ -132,17 +132,22 @@ jobs:
     steps:
       - powershell: |
           .\ci\windows_ci_setup.ps1
+          Write-Host "##vso[task.prependpath]$env:TOOLS_BIN_DIR\usr\bin"
+          Write-Host "##vso[task.prependpath]$env:VC_TOOLS_BIN_X64;$env:VC_CMAKE_PATH\CMake\bin;$env:VC_CMAKE_PATH\Ninja"
           Write-Host "##vso[task.prependpath]$env:TOOLS_BIN_DIR"
         displayName: "Install dependencies"
         env:
           TOOLS_BIN_DIR: $(Pipeline.Workspace)\bin
+          VC_CMAKE_PATH: "C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Enterprise\\Common7\\IDE\\CommonExtensions\\Microsoft\\CMake"
+          VC_TOOLS_BIN_X64: "C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Enterprise\\VC\\Tools\\MSVC\\14.25.28610\\bin\\HostX64\\x64"
 
       - bash: ci/windows_ci_steps.sh
         displayName: "Run Windows CI"
         env:
-          TMPDIR: $(Agent.TempDirectory)
           BAZEL_VC: "C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Enterprise\\VC"
-          BAZEL_SH: "C:\\Program Files\\Git\\bin\\bash.exe"
+          BAZEL_SH: $(Pipeline.Workspace)/bin/usr/bin/bash.exe
           BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
           BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
           GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+          MSYS2_ARG_CONV_EXCL: "*"
+          TMPDIR: $(Agent.TempDirectory)

.bazelrc

@@ -10,7 +10,7 @@
 # Startup options cannot be selected via config.
 startup --host_jvm_args=-Xmx2g
 
-build --workspace_status_command=bazel/get_workspace_status
+build --workspace_status_command="bash bazel/get_workspace_status"
 build --experimental_local_memory_estimate
 build --experimental_strict_action_env=true
 build --host_force_python=PY3
@@ -205,5 +205,46 @@ build:compdb --strip=always
 build:compdb --build_tag_filters=-nocompdb
 build:compdb --define=ENVOY_CONFIG_COMPILATION_DATABASE=1
 
+# Windows build quirks
+build:windows --action_env=TMPDIR
+build:windows --define signal_trace=disabled
+build:windows --define hot_restart=disabled
+build:windows --define tcmalloc=disabled
+build:windows --define manual_stamp=manual_stamp
+
+# Should not be required after upstream fix to bazel,
+# and already a no-op to linux/macos builds
+# see issue https://github.com/bazelbuild/rules_foreign_cc/issues/301
+build:windows --copt="-DCARES_STATICLIB"
+build:windows --copt="-DNGHTTP2_STATICLIB"
+build:windows --copt="-DCURL_STATICLIB"
+
+# Required to work around build defects on Windows MSVC cl
+# Unguarded gcc pragmas in quiche are not recognized by MSVC
+build:msvc-cl --copt="/wd4068"
+# Allows 'nodiscard' function return values to be discarded
+build:msvc-cl --copt="/wd4834"
+# Allows inline functions to be undefined
+build:msvc-cl --copt="/wd4506"
+build:msvc-cl --copt="-D_SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING"
+
+# Required to work around Windows clang-cl build defects
+# Ignore conflicting definitions of _WIN32_WINNT
+# Overriding __TIME__ etc is problematic (and is actually an invalid no-op)
+build:clang-cl --copt="-Wno-macro-redefined"
+build:clang-cl --copt="-Wno-builtin-macro-redefined"
+build:clang-cl --action_env=USE_CLANG_CL=1
+
+# Defaults to 'auto' - Off for windows, so override to linux behavior
+build:windows --enable_runfiles=yes
+
+# This should become adopted by bazel as the default
+build:windows --features=compiler_param_file
+
+# These options attempt to force a monolithic binary including the CRT
+build:windows --features=fully_static_link
+build:windows --features=static_link_msvcrt
+build:windows --dynamic_mode=off
+
 try-import %workspace%/clang.bazelrc
 try-import %workspace%/user.bazelrc

bazel/external/kafka_int32.patch

@@ -0,0 +1,27 @@
+--- DescribeGroupsResponse.json	2020-03-25 16:12:16.373302600 -0400
++++ DescribeGroupsResponse.json	2020-03-25 16:11:16.184156200 -0400
+@@ -63,7 +63,7 @@
+         { "name": "MemberAssignment", "type": "bytes", "versions": "0+",
+           "about": "The current assignment provided by the group leader." }
+       ]},
+-      { "name": "AuthorizedOperations", "type": "int32", "versions": "3+",  "default": "-2147483648",
++      { "name": "AuthorizedOperations", "type": "int32", "versions": "3+",  "default": "INT32_MIN",
+         "about": "32-bit bitfield to represent authorized operations for this group." }
+     ]}
+   ]
+
+--- MetadataResponse.json	2020-03-25 15:53:36.319161000 -0400
++++ MetadataResponse.json	2020-03-25 15:54:11.510400000 -0400
+@@ -81,10 +81,10 @@
+         { "name": "OfflineReplicas", "type": "[]int32", "versions": "5+", "ignorable": true,
+           "about": "The set of offline replicas of this partition." }
+       ]},
+-      { "name": "TopicAuthorizedOperations", "type": "int32", "versions": "8+", "default": "-2147483648",
++      { "name": "TopicAuthorizedOperations", "type": "int32", "versions": "8+", "default": "INT32_MIN",
+         "about": "32-bit bitfield to represent authorized operations for this topic." }
+     ]},
+-    { "name": "ClusterAuthorizedOperations", "type": "int32", "versions": "8+", "default": "-2147483648",
++    { "name": "ClusterAuthorizedOperations", "type": "int32", "versions": "8+", "default": "INT32_MIN",
+       "about": "32-bit bitfield to represent authorized operations for this cluster." }
+   ]
+ }

bazel/foreign_cc/BUILD

@@ -36,8 +36,6 @@ cc_library(
     ],
 )
 
-# autotools packages are unusable on Windows as-is
-# TODO: Consider our own luajit.BUILD file as we do with many other packages
 configure_make(
     name = "luajit",
     configure_command = "build.py",
@@ -51,10 +49,10 @@ configure_make(
     lib_source = "@com_github_luajit_luajit//:all",
     make_commands = [],
     out_include_dir = "include/luajit-2.1",
-    static_libraries = [
-        "libluajit-5.1.a",
-    ],
-    tags = ["skip_on_windows"],
+    static_libraries = select({
+        "//bazel:windows_x86_64": ["lua51.lib"],
+        "//conditions:default": ["libluajit-5.1.a"],
+    }),
 )
 
 configure_make(
@@ -70,10 +68,10 @@ configure_make(
     lib_source = "@com_github_moonjit_moonjit//:all",
     make_commands = [],
     out_include_dir = "include/moonjit-2.2",
-    static_libraries = [
-        "libluajit-5.1.a",
-    ],
-    tags = ["skip_on_windows"],
+    static_libraries = select({
+        "//bazel:windows_x86_64": ["lua51.lib"],
+        "//conditions:default": ["libluajit-5.1.a"],
+    }),
 )
 
 envoy_cmake_external(

bazel/foreign_cc/luajit.patch

@@ -57,7 +57,7 @@ new file mode 100755
 index 0000000..9c71271
 --- /dev/null
 +++ b/build.py
-@@ -0,0 +1,39 @@
+@@ -0,0 +1,56 @@
 +#!/usr/bin/env python
 +
 +import argparse
@@ -95,5 +95,22 @@ index 0000000..9c71271
 +
 +    os.system('make V=1 PREFIX="{}" install'.format(args.prefix))
 +
-+main()
++def win_main():
++    src_dir = os.path.dirname(os.path.realpath(__file__))
++    dst_dir = os.getcwd() + "/luajit"
++    shutil.copytree(src_dir, os.path.basename(src_dir))
++    os.chdir(os.path.basename(src_dir) + "/src")
++    os.system('msvcbuild.bat static')
++    os.makedirs(dst_dir + "/lib", exist_ok=True)
++    shutil.copy("lua51.lib", dst_dir + "/lib")
++    os.makedirs(dst_dir + "/include/luajit-2.1", exist_ok=True)
++    for header in ["lauxlib.h", "luaconf.h", "lua.h", "lua.hpp", "luajit.h", "lualib.h"]:
++      shutil.copy(header, dst_dir + "/include/luajit-2.1")
++    os.makedirs(dst_dir + "/bin", exist_ok=True)
++    shutil.copy("luajit.exe", dst_dir + "/bin")
++    
++if os.name == 'nt':
++  win_main()
++else:
++  main()
 +

bazel/foreign_cc/moonjit.patch

@@ -48,7 +48,7 @@ new file mode 100755
 index 0000000..9c71271
 --- /dev/null
 +++ b/build.py
-@@ -0,0 +1,39 @@
+@@ -0,0 +1,56 @@
 +#!/usr/bin/env python
 +
 +import argparse
@@ -86,5 +86,22 @@ index 0000000..9c71271
 +
 +    os.system('make V=1 PREFIX="{}" install'.format(args.prefix))
 +
-+main()
++def win_main():
++    src_dir = os.path.dirname(os.path.realpath(__file__))
++    dst_dir = os.getcwd() + "/moonjit"
++    shutil.copytree(src_dir, os.path.basename(src_dir))
++    os.chdir(os.path.basename(src_dir) + "/src")
++    os.system('msvcbuild.bat static')
++    os.makedirs(dst_dir + "/lib", exist_ok=True)
++    shutil.copy("lua51.lib", dst_dir + "/lib")
++    os.makedirs(dst_dir + "/include/moonjit-2.2", exist_ok=True)
++    for header in ["lauxlib.h", "luaconf.h", "lua.h", "lua.hpp", "luajit.h", "lualib.h"]:
++      shutil.copy(header, dst_dir + "/include/moonjit-2.2")
++    os.makedirs(dst_dir + "/bin", exist_ok=True)
++    shutil.copy("luajit.exe", dst_dir + "/bin")
++    
++if os.name == 'nt':
++  win_main()
++else:
++  main()
 +

bazel/repositories.bzl

@@ -5,8 +5,14 @@ load("@envoy_api//bazel:envoy_http_archive.bzl", "envoy_http_archive")
 load(":repository_locations.bzl", "REPOSITORY_LOCATIONS")
 load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language")
 
-# dict of {build recipe name: longform extension name,}
-PPC_SKIP_TARGETS = {"luajit": "envoy.filters.http.lua"}
+PPC_SKIP_TARGETS = ["envoy.filters.http.lua"]
+
+WINDOWS_SKIP_TARGETS = [
+    "envoy.tracers.dynamic_ot",
+    "envoy.tracers.lightstep",
+    "envoy.tracers.datadog",
+    "envoy.tracers.opencensus",
+]
 
 # Make all contents of an external repository accessible under a filegroup.  Used for external HTTP
 # archives, e.g. cares.
@@ -795,6 +801,7 @@ filegroup(
     http_archive(
         name = "kafka_source",
         build_file_content = KAFKASOURCE_BUILD_CONTENT,
+        patches = ["@envoy//bazel/external:kafka_int32.patch"],
         **REPOSITORY_LOCATIONS["kafka_source"]
     )
 

ci/build_setup.ps1

@@ -17,5 +17,5 @@ $env:ENVOY_SRCDIR = [System.IO.Path]::GetFullPath("$PSScriptRoot\..")
 echo "ENVOY_BAZEL_ROOT: $env:ENVOY_BAZEL_ROOT"
 echo "ENVOY_SRCDIR: $env:ENVOY_SRCDIR"
 
-$env:BAZEL_BASE_OPTIONS="--noworkspace_rc --output_base=$env:ENVOY_BAZEL_ROOT --bazelrc=$env:ENVOY_SRCDIR\windows\.bazelrc"
+$env:BAZEL_BASE_OPTIONS="--output_base=$env:ENVOY_BAZEL_ROOT"
 $env:BAZEL_BUILD_OPTIONS="--config=msvc-cl --features=compiler_param_file --strategy=Genrule=standalone --spawn_strategy=standalone --verbose_failures --jobs=$env:NUM_CPUS --show_task_finish --test_output=all $env:BAZEL_BUILD_EXTRA_OPTIONS $env:BAZEL_EXTRA_TEST_OPTIONS"

ci/windows_ci_setup.ps1

@@ -2,14 +2,6 @@
 # The list of installed software can be found at:
 # https://github.com/actions/virtual-environments/blob/master/images/win/Windows2019-Readme.md
 
-Add-Type -AssemblyName System.IO.Compression.FileSystem
-function Unzip
-{
-    param([string]$zipfile, [string]$outpath)
-
-    [System.IO.Compression.ZipFile]::ExtractToDirectory($zipfile, $outpath)
-}
-
 function Checksum
 {
     param([string]$filepath, [string]$expected, [string]$algorithm)
@@ -26,10 +18,23 @@ function Checksum
 mkdir "$env:TOOLS_BIN_DIR"
 $wc = New-Object System.Net.WebClient
 $wc.DownloadFile("https://github.com/bazelbuild/bazelisk/releases/download/v1.0/bazelisk-windows-amd64.exe", "$env:TOOLS_BIN_DIR\bazel.exe")
-$wc.DownloadFile("https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-win.zip", "$env:TOOLS_BIN_DIR\ninja-win.zip")
+# See https://sourceforge.net/projects/msys2/files/Base/x86_64/ for msys2 download source
+$wc.DownloadFile("http://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20190524.tar.xz", "$env:TEMP\msys2.tar.xz")
 
 # Check the SHA256 file hash of each downloaded file.
 Checksum $env:TOOLS_BIN_DIR\bazel.exe 96395ee9e3fb9f4499fcaffa8a94dd72b0748f495f366bc4be44dbf09d6827fc SHA256
-Checksum $env:TOOLS_BIN_DIR\ninja-win.zip 2d70010633ddaacc3af4ffbd21e22fae90d158674a09e132e06424ba3ab036e9 SHA256
+Checksum $env:TEMP\msys2.tar.xz 168e156fa9f00d90a8445676c023c63be6e82f71487f4e2688ab5cb13b345383 SHA256
 
-Unzip "$env:TOOLS_BIN_DIR\ninja-win.zip" "$env:TOOLS_BIN_DIR"
+# Unpack and install msys2 and required packages
+$tarpath="$env:ProgramFiles\Git\usr\bin\tar.exe"
+$msys2TarPathClean = "/$env:TEMP/msys2.tar.xz".replace(':', '').replace('\', '/')
+$outDirClean = "/$env:TOOLS_BIN_DIR".replace(':', '').replace('\', '/')
+&"$tarpath" -Jxf $msys2TarPathClean -C $outDirClean --strip-components=1
+# Add utils to the path for msys2 setup
+$env:PATH = "$env:TOOLS_BIN_DIR\usr\bin;$env:TOOLS_BIN_DIR\mingw64\bin;$env:PATH"
+bash.exe -c "pacman-key --init 2>&1"
+bash.exe -c "pacman-key --populate msys2 2>&1"
+bash.exe -c "pacman.exe -Syyuu --noconfirm 2>&1"
+bash.exe -c "pacman.exe -Syuu --noconfirm 2>&1"
+bash.exe -c "pacman.exe -S --noconfirm --needed compression diffutils patch 2>&1"
+bash.exe -c "pacman.exe -Scc --noconfirm 2>&1"

ci/windows_ci_steps.sh

@@ -20,14 +20,24 @@ df -h
 # rules_foreign_cc does not currently use bazel output/temp directories by default, it uses mktemp
 # which respects the value of the TMPDIR environment variable
 drive="$(readlink -f $TMPDIR | cut -d '/' -f2)"
-/c/windows/system32/cmd.exe "/c mklink /d $drive:\\$drive $drive:\\"
+if [ ! -e "/$drive/$drive" ]; then
+  /c/windows/system32/cmd.exe /c "mklink /d $drive:\\$drive $drive:\\"
+fi
 
-BAZEL_STARTUP_OPTIONS="--noworkspace_rc --bazelrc=windows/.bazelrc --output_base=c:/_eb"
+BAZEL_STARTUP_OPTIONS="--output_base=c:/_eb"
 BAZEL_BUILD_OPTIONS="-c opt --config=msvc-cl --show_task_finish --verbose_failures \
   --test_output=all ${BAZEL_BUILD_EXTRA_OPTIONS} ${BAZEL_EXTRA_TEST_OPTIONS}"
 
-bazel ${BAZEL_STARTUP_OPTIONS} build ${BAZEL_BUILD_OPTIONS} //bazel/... --build_tag_filters=-skip_on_windows
+# With all envoy-static and //test/ tree building, no need to test compile externals
+# bazel ${BAZEL_STARTUP_OPTIONS} build ${BAZEL_BUILD_OPTIONS} //bazel/... --build_tag_filters=-skip_on_windows
 
 bazel ${BAZEL_STARTUP_OPTIONS} build ${BAZEL_BUILD_OPTIONS} //source/exe:envoy-static --build_tag_filters=-skip_on_windows
 
-# bazel ${BAZEL_STARTUP_OPTIONS} test ${BAZEL_BUILD_OPTIONS} //test/... --test_tag_filters=-skip_on_windows --build_tests_only --test_summary=terse --test_output=errors
+# TODO(sunjayBhatia, wrowe): We are disabling building/running tests for now as the AZP pipelines
+# workers do not provide enough resources for us to produce fast enough or reliable enough builds.
+# Test compilation of known MSVC-compatible test sources
+# bazel ${BAZEL_STARTUP_OPTIONS} build ${BAZEL_BUILD_OPTIONS} //test/... --test_tag_filters=-skip_on_windows --build_tests_only
+
+# Test invocations of known-working tests on Windows
+# bazel ${BAZEL_STARTUP_OPTIONS} test ${BAZEL_BUILD_OPTIONS} //test/... --test_tag_filters=-skip_on_windows,-fails_on_windows --build_tests_only --test_summary=terse --test_output=errors
+

source/exe/BUILD

@@ -9,12 +9,8 @@ load(
     "envoy_cc_win32_library",
     "envoy_package",
 )
-load(
-    "//source/extensions:all_extensions.bzl",
-    "envoy_all_extensions",
-    "envoy_windows_extensions",
-)
-load("//bazel:repositories.bzl", "PPC_SKIP_TARGETS")
+load("//source/extensions:all_extensions.bzl", "envoy_all_extensions")
+load("//bazel:repositories.bzl", "PPC_SKIP_TARGETS", "WINDOWS_SKIP_TARGETS")
 
 envoy_package()
 
@@ -41,7 +37,7 @@ envoy_cc_library(
         "//source/server:server_lib",
         "//source/server:listener_hooks_lib",
     ] + select({
-        "//bazel:windows_x86_64": envoy_windows_extensions(),
+        "//bazel:windows_x86_64": envoy_all_extensions(WINDOWS_SKIP_TARGETS),
         "//bazel:linux_ppc": envoy_all_extensions(PPC_SKIP_TARGETS),
         "//conditions:default": envoy_all_extensions(),
     }),

source/exe/win32/platform_impl.cc

@@ -9,10 +9,9 @@ namespace Envoy {
 PlatformImpl::PlatformImpl()
     : thread_factory_(std::make_unique<Thread::ThreadFactoryImplWin32>()),
       file_system_(std::make_unique<Filesystem::InstanceImplWin32>()) {
-  const WORD wVersionRequested = MAKEWORD(2, 2);
-  WSADATA wsaData;
-  const int rc = ::WSAStartup(wVersionRequested, &wsaData);
-  RELEASE_ASSERT(rc == 0, "WSAStartup failed with error");
+  WSADATA wsa_data;
+  const WORD version_requested = MAKEWORD(2, 2);
+  RELEASE_ASSERT(WSAStartup(version_requested, &wsa_data) == 0, "WSAStartup failed with error");
 }
 
 PlatformImpl::~PlatformImpl() { ::WSACleanup(); }

source/extensions/all_extensions.bzl

@@ -1,5 +1,5 @@
 load("@bazel_skylib//lib:dicts.bzl", "dicts")
-load("@envoy_build_config//:extensions_build_config.bzl", "EXTENSIONS", "WINDOWS_EXTENSIONS")
+load("@envoy_build_config//:extensions_build_config.bzl", "EXTENSIONS")
 
 # These extensions are registered using the extension system but are required for the core Envoy build.
 # The map may be overridden by extensions specified in envoy_build_config.
@@ -9,14 +9,8 @@ _required_extensions = {
 }
 
 # Return all extensions to be compiled into Envoy.
-def envoy_all_extensions(blacklist = dict()):
+def envoy_all_extensions(blacklist = []):
     all_extensions = dicts.add(_required_extensions, EXTENSIONS)
 
     # These extensions can be removed on a site specific basis.
-    return [v for k, v in all_extensions.items() if not k in blacklist.values()]
-
-def envoy_windows_extensions():
-    all_extensions = dicts.add(_required_extensions, WINDOWS_EXTENSIONS)
-
-    # These extensions can be removed on a site specific basis.
-    return all_extensions.values()
+    return [v for k, v in all_extensions.items() if not k in blacklist]

source/extensions/clusters/redis/redis_cluster.cc

@@ -1,7 +1,5 @@
 #include "redis_cluster.h"
 
-#include <err.h>
-
 #include "envoy/config/cluster/redis/redis_cluster.pb.h"
 #include "envoy/config/cluster/redis/redis_cluster.pb.validate.h"
 #include "envoy/config/cluster/v3/cluster.pb.h"