From 6170ee48801683be2561c2d9062bd442aeb1ccd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Zasso?= Date: Sat, 29 Jan 2022 12:37:12 +0100 Subject: [PATCH 1/6] deps: update zlib to upstream fc5cfd78a3 Updated as described in doc/contributing/maintaining-zlib.md. --- deps/zlib/BUILD.gn | 257 +++++-- deps/zlib/DIR_METADATA | 3 + deps/zlib/OWNERS | 6 +- deps/zlib/README.chromium | 1 + deps/zlib/adler32.c | 12 +- deps/zlib/adler32_simd.c | 4 - deps/zlib/arm_features.c | 90 --- deps/zlib/chromeconf.h | 4 + deps/zlib/contrib/bench/zlib_bench.cc | 83 ++- deps/zlib/contrib/minizip/iowin32.c | 8 +- deps/zlib/contrib/minizip/miniunz.c | 13 +- deps/zlib/contrib/minizip/minizip.c | 7 +- deps/zlib/contrib/minizip/minizip.md | 9 + deps/zlib/contrib/optimizations/chunkcopy.h | 34 +- .../contrib/optimizations/inffast_chunk.c | 2 +- .../contrib/optimizations/insert_string.h | 69 +- deps/zlib/contrib/tests/DEPS | 4 + deps/zlib/contrib/tests/OWNERS | 1 + deps/zlib/contrib/tests/fuzzers/BUILD.gn | 38 +- deps/zlib/contrib/tests/fuzzers/OWNERS | 1 - .../contrib/tests/fuzzers/deflate_fuzzer.cc | 75 +- .../tests/fuzzers/streaming_inflate_fuzzer.cc | 74 ++ deps/zlib/contrib/tests/infcover.cc | 684 ++++++++++++++++++ deps/zlib/contrib/tests/infcover.h | 11 + deps/zlib/contrib/tests/run_all_unittests.cc | 14 + deps/zlib/contrib/tests/utils_unittest.cc | 521 +++++++++++++ deps/zlib/cpu_features.c | 167 +++++ deps/zlib/{arm_features.h => cpu_features.h} | 9 +- deps/zlib/crc32.c | 17 +- deps/zlib/crc32_simd.c | 4 - deps/zlib/crc_folding.c | 18 +- deps/zlib/deflate.c | 60 +- deps/zlib/fill_window_sse.c | 8 +- deps/zlib/google/BUILD.gn | 6 +- deps/zlib/google/DEPS | 1 + deps/zlib/google/OWNERS | 3 + deps/zlib/google/compression_utils.cc | 39 +- deps/zlib/google/compression_utils.h | 28 +- .../zlib/google/compression_utils_portable.cc | 4 +- deps/zlib/google/compression_utils_portable.h | 6 +- .../zlib/google/compression_utils_unittest.cc | 11 +- deps/zlib/google/test/data/README.md | 15 + .../test/data/test_posix_permissions.zip | Bin 0 -> 574 bytes deps/zlib/google/zip.cc | 223 +++--- deps/zlib/google/zip.h | 184 ++--- deps/zlib/google/zip_internal.cc | 240 ++++-- deps/zlib/google/zip_internal.h | 20 +- deps/zlib/google/zip_reader.cc | 109 ++- deps/zlib/google/zip_reader.h | 39 +- deps/zlib/google/zip_reader_unittest.cc | 58 +- deps/zlib/google/zip_unittest.cc | 446 +++++++++--- deps/zlib/google/zip_writer.cc | 344 ++++++--- deps/zlib/google/zip_writer.h | 129 +++- .../zlib/patches/0003-uninitializedjump.patch | 15 + deps/zlib/patches/0004-fix-uwp.patch | 22 + deps/zlib/patches/0005-infcover-gtest.patch | 405 +++++++++++ deps/zlib/patches/0006-fix-check_match.patch | 42 ++ .../0007-zero-init-deflate-window.patch | 40 + .../0008-minizip-zip-unzip-tools.patch | 98 +++ deps/zlib/simd_stub.c | 35 - deps/zlib/x86.c | 101 --- deps/zlib/x86.h | 16 - deps/zlib/zlib.gyp | 8 +- 63 files changed, 3913 insertions(+), 1082 deletions(-) create mode 100644 deps/zlib/DIR_METADATA delete mode 100644 deps/zlib/arm_features.c create mode 100644 deps/zlib/contrib/minizip/minizip.md create mode 100644 deps/zlib/contrib/tests/DEPS create mode 100644 deps/zlib/contrib/tests/fuzzers/streaming_inflate_fuzzer.cc create mode 100644 deps/zlib/contrib/tests/infcover.cc create mode 100644 deps/zlib/contrib/tests/infcover.h create mode 100644 deps/zlib/contrib/tests/run_all_unittests.cc create mode 100644 deps/zlib/contrib/tests/utils_unittest.cc create mode 100644 deps/zlib/cpu_features.c rename deps/zlib/{arm_features.h => cpu_features.h} (53%) create mode 100644 deps/zlib/google/test/data/README.md create mode 100644 deps/zlib/google/test/data/test_posix_permissions.zip create mode 100644 deps/zlib/patches/0003-uninitializedjump.patch create mode 100644 deps/zlib/patches/0004-fix-uwp.patch create mode 100644 deps/zlib/patches/0005-infcover-gtest.patch create mode 100644 deps/zlib/patches/0006-fix-check_match.patch create mode 100644 deps/zlib/patches/0007-zero-init-deflate-window.patch create mode 100644 deps/zlib/patches/0008-minizip-zip-unzip-tools.patch delete mode 100644 deps/zlib/simd_stub.c delete mode 100644 deps/zlib/x86.c delete mode 100644 deps/zlib/x86.h diff --git a/deps/zlib/BUILD.gn b/deps/zlib/BUILD.gn index 5f8873357d35d1..49f52e1f8b1e50 100644 --- a/deps/zlib/BUILD.gn +++ b/deps/zlib/BUILD.gn @@ -4,6 +4,10 @@ import("//build/config/compiler/compiler.gni") +if (build_with_chromium) { + import("//testing/test.gni") +} + if (current_cpu == "arm" || current_cpu == "arm64") { import("//build/config/arm.gni") } @@ -14,10 +18,47 @@ config("zlib_config") { config("zlib_internal_config") { defines = [ "ZLIB_IMPLEMENTATION" ] + + if (!is_debug) { + # Build code using -O3, see: crbug.com/1084371. + configs = [ "//build/config/compiler:optimize_speed" ] + } + if (is_debug || use_libfuzzer) { + # Enable zlib's asserts in debug and fuzzer builds. + defines += [ "ZLIB_DEBUG" ] + } + + if (is_win && !is_clang) { + # V8 supports building with msvc, these silence some warnings that + # causes compilation to fail (https://crbug.com/1255096). + cflags = [ + "/wd4244", + "/wd4100", + "/wd4702", + "/wd4127", + ] + } +} + +source_set("zlib_common_headers") { + sources = [ + "chromeconf.h", + "deflate.h", + "inffast.h", + "inffixed.h", + "inflate.h", + "inftrees.h", + "zconf.h", + "zlib.h", + "zutil.h", + ] } use_arm_neon_optimizations = false -if (current_cpu == "arm" || current_cpu == "arm64") { +if ((current_cpu == "arm" || current_cpu == "arm64") && + !(is_win && !is_clang)) { + # TODO(richard.townsend@arm.com): Optimizations temporarily disabled for + # Windows on Arm MSVC builds, see http://crbug.com/v8/10012. if (arm_use_neon) { use_arm_neon_optimizations = true } @@ -29,6 +70,11 @@ use_x86_x64_optimizations = config("zlib_adler32_simd_config") { if (use_x86_x64_optimizations) { defines = [ "ADLER32_SIMD_SSSE3" ] + if (is_win) { + defines += [ "X86_WINDOWS" ] + } else { + defines += [ "X86_NOT_WINDOWS" ] + } } if (use_arm_neon_optimizations) { @@ -55,16 +101,13 @@ source_set("zlib_adler32_simd") { "adler32_simd.c", "adler32_simd.h", ] - if (!is_debug) { - # Use optimize_speed (-O3) to output the _smallest_ code. - configs -= [ "//build/config/compiler:default_optimization" ] - configs += [ "//build/config/compiler:optimize_speed" ] - } } configs += [ ":zlib_internal_config" ] public_configs = [ ":zlib_adler32_simd_config" ] + + public_deps = [ ":zlib_common_headers" ] } if (use_arm_neon_optimizations) { @@ -78,6 +121,8 @@ if (use_arm_neon_optimizations) { defines += [ "ARMV8_OS_ANDROID" ] } else if (is_linux || is_chromeos) { defines += [ "ARMV8_OS_LINUX" ] + } else if (is_mac) { + defines += [ "ARMV8_OS_MACOS" ] } else if (is_fuchsia) { defines += [ "ARMV8_OS_FUCHSIA" ] } else if (is_win) { @@ -94,37 +139,23 @@ if (use_arm_neon_optimizations) { if (!is_ios) { include_dirs = [ "." ] - if (is_android) { - import("//build/config/android/config.gni") - if (defined(android_ndk_root) && android_ndk_root != "") { - deps = [ - "//third_party/android_ndk:cpu_features", - ] - } else { - assert(false, "CPU detection requires the Android NDK") - } - } else if (!is_win && !is_clang) { + if (!is_win && !is_clang) { assert(!use_thin_lto, "ThinLTO fails mixing different module-level targets") cflags_c = [ "-march=armv8-a+crc" ] } sources = [ - "arm_features.c", - "arm_features.h", "crc32_simd.c", "crc32_simd.h", ] - - if (!is_debug) { - configs -= [ "//build/config/compiler:default_optimization" ] - configs += [ "//build/config/compiler:optimize_speed" ] - } } configs += [ ":zlib_internal_config" ] public_configs = [ ":zlib_arm_crc32_config" ] + + public_deps = [ ":zlib_common_headers" ] } } @@ -139,6 +170,7 @@ config("zlib_inflate_chunk_simd_config") { if (use_arm_neon_optimizations) { defines = [ "INFLATE_CHUNK_SIMD_NEON" ] + if (current_cpu == "arm64") { defines += [ "INFLATE_CHUNK_READ_64LE" ] } @@ -157,22 +189,18 @@ source_set("zlib_inflate_chunk_simd") { "contrib/optimizations/inffast_chunk.h", "contrib/optimizations/inflate.c", ] - - if (use_arm_neon_optimizations && !is_debug) { - # Here we trade better performance on newer/bigger ARMv8 cores - # for less perf on ARMv7, per crbug.com/772870#c40 - configs -= [ "//build/config/compiler:default_optimization" ] - configs += [ "//build/config/compiler:optimize_speed" ] - } } + configs += [ ":zlib_internal_config" ] + + # Needed for MSVC, which is still supported by V8 and PDFium. zlib uses K&R C + # style function declarations, which triggers warning C4131. configs -= [ "//build/config/compiler:chromium_code" ] - configs += [ - ":zlib_internal_config", - "//build/config/compiler:no_chromium_code", - ] + configs += [ "//build/config/compiler:no_chromium_code" ] public_configs = [ ":zlib_inflate_chunk_simd_config" ] + + public_deps = [ ":zlib_common_headers" ] } config("zlib_crc32_simd_config") { @@ -201,6 +229,16 @@ source_set("zlib_crc32_simd") { configs += [ ":zlib_internal_config" ] public_configs = [ ":zlib_crc32_simd_config" ] + public_deps = [ ":zlib_common_headers" ] +} + +config("zlib_x86_simd_config") { + if (use_x86_x64_optimizations) { + defines = [ + "CRC32_SIMD_SSE42_PCLMUL", + "DEFLATE_FILL_WINDOW_SSE2", + ] + } } source_set("zlib_x86_simd") { @@ -218,17 +256,13 @@ source_set("zlib_x86_simd") { "-mpclmul", ] } - } else { - sources = [ - "simd_stub.c", - ] } - configs -= [ "//build/config/compiler:chromium_code" ] - configs += [ - ":zlib_internal_config", - "//build/config/compiler:no_chromium_code", - ] + configs += [ ":zlib_internal_config" ] + + public_configs = [ ":zlib_x86_simd_config" ] + + public_deps = [ ":zlib_common_headers" ] } config("zlib_warnings") { @@ -248,6 +282,8 @@ component("zlib") { "chromeconf.h", "compress.c", "contrib/optimizations/insert_string.h", + "cpu_features.c", + "cpu_features.h", "crc32.c", "crc32.h", "deflate.c", @@ -267,7 +303,6 @@ component("zlib") { "trees.c", "trees.h", "uncompr.c", - "x86.h", "zconf.h", "zlib.h", "zutil.c", @@ -277,6 +312,20 @@ component("zlib") { defines = [] deps = [] + if (!use_x86_x64_optimizations && !use_arm_neon_optimizations) { + # Apparently android_cronet bot builds with NEON disabled and + # we also should disable optimizations for iOS@x86 (a.k.a. simulator). + defines += [ "CPU_NO_SIMD" ] + } + + if (is_ios) { + # iOS@ARM is a special case where we always have NEON but don't check + # for crypto extensions. + # TODO(cavalcantii): verify what is the current state of CPU features + # shipped on latest iOS devices. + defines += [ "ARM_OS_IOS" ] + } + if (use_x86_x64_optimizations || use_arm_neon_optimizations) { deps += [ ":zlib_adler32_simd", @@ -284,7 +333,6 @@ component("zlib") { ] if (use_x86_x64_optimizations) { - sources += [ "x86.c" ] deps += [ ":zlib_crc32_simd" ] } else if (use_arm_neon_optimizations) { sources += [ "contrib/optimizations/slide_hash_neon.h" ] @@ -294,18 +342,29 @@ component("zlib") { sources += [ "inflate.c" ] } + deps += [ ":zlib_x86_simd" ] + + if (is_android) { + import("//build/config/android/config.gni") + if (defined(android_ndk_root) && android_ndk_root != "") { + deps += [ "//third_party/android_ndk:cpu_features" ] + } else { + assert(false, "CPU detection requires the Android NDK") + } + } + configs -= [ "//build/config/compiler:chromium_code" ] + configs += [ "//build/config/compiler:no_chromium_code" ] + + public_configs = [ ":zlib_config" ] + configs += [ ":zlib_internal_config", - "//build/config/compiler:no_chromium_code", # Must be after no_chromium_code for warning flags to be ordered correctly. ":zlib_warnings", ] - public_configs = [ ":zlib_config" ] - - deps += [ ":zlib_x86_simd" ] allow_circular_includes_from = deps } @@ -337,43 +396,113 @@ static_library("minizip") { ] } - if (is_mac || is_ios || is_android || is_nacl) { + if (is_apple || is_android || is_nacl) { # Mac, Android and the BSDs don't have fopen64, ftello64, or fseeko64. We # use fopen, ftell, and fseek instead on these systems. defines = [ "USE_FILE32API" ] } - deps = [ - ":zlib", - ] + deps = [ ":zlib" ] configs -= [ "//build/config/compiler:chromium_code" ] - configs += [ - "//build/config/compiler:no_chromium_code", + configs += [ "//build/config/compiler:no_chromium_code" ] + + public_configs = [ ":zlib_config" ] + configs += [ # Must be after no_chromium_code for warning flags to be ordered correctly. ":minizip_warnings", ] - - public_configs = [ ":zlib_config" ] } executable("zlib_bench") { include_dirs = [ "." ] - sources = [ - "contrib/bench/zlib_bench.cc", - ] - + sources = [ "contrib/bench/zlib_bench.cc" ] if (!is_debug) { configs -= [ "//build/config/compiler:default_optimization" ] configs += [ "//build/config/compiler:optimize_speed" ] } + deps = [ ":zlib" ] + configs -= [ "//build/config/compiler:chromium_code" ] configs += [ "//build/config/compiler:no_chromium_code" ] +} - deps = [ - ":zlib", - ] +if (!is_win || target_os != "winuwp") { + executable("minizip_bin") { + include_dirs = [ "." ] + + sources = [ "contrib/minizip/minizip.c" ] + + if (is_clang) { + cflags = [ "-Wno-incompatible-pointer-types-discards-qualifiers" ] + } + + if (!is_debug) { + configs -= [ "//build/config/compiler:default_optimization" ] + configs += [ "//build/config/compiler:optimize_speed" ] + } + + deps = [ ":minizip" ] + + configs -= [ "//build/config/compiler:chromium_code" ] + configs += [ "//build/config/compiler:no_chromium_code" ] + } + + executable("miniunz_bin") { + include_dirs = [ "." ] + + sources = [ "contrib/minizip/miniunz.c" ] + + if (is_clang) { + cflags = [ "-Wno-incompatible-pointer-types-discards-qualifiers" ] + } + + if (!is_debug) { + configs -= [ "//build/config/compiler:default_optimization" ] + configs += [ "//build/config/compiler:optimize_speed" ] + } + + deps = [ ":minizip" ] + + configs -= [ "//build/config/compiler:chromium_code" ] + configs += [ "//build/config/compiler:no_chromium_code" ] + } +} + +if (build_with_chromium) { + test("zlib_unittests") { + testonly = true + + sources = [ + "contrib/tests/infcover.cc", + "contrib/tests/infcover.h", + "contrib/tests/run_all_unittests.cc", + "contrib/tests/utils_unittest.cc", + "google/compression_utils_unittest.cc", + "google/zip_reader_unittest.cc", + "google/zip_unittest.cc", + ] + + data = [ "google/test/data/" ] + + deps = [ + ":zlib", + "google:compression_utils", + "google:zip", + "//base/test:test_support", + "//testing/gtest", + ] + + configs -= [ "//build/config/compiler:chromium_code" ] + configs += [ "//build/config/compiler:no_chromium_code" ] + + include_dirs = [ + "//third_party/googletest/src/googletest/include/gtest", + ".", + "google", + ] + } } diff --git a/deps/zlib/DIR_METADATA b/deps/zlib/DIR_METADATA new file mode 100644 index 00000000000000..d366dc732137cd --- /dev/null +++ b/deps/zlib/DIR_METADATA @@ -0,0 +1,3 @@ +monorail: { + component: "Internals" +} diff --git a/deps/zlib/OWNERS b/deps/zlib/OWNERS index 069bcd8c2a1445..ecffb59f051501 100644 --- a/deps/zlib/OWNERS +++ b/deps/zlib/OWNERS @@ -1,7 +1,5 @@ agl@chromium.org cavalcantii@chromium.org cblume@chromium.org -mtklein@chromium.org -scroggo@chromium.org - -# COMPONENT: Internals +noel@chromium.org +scroggo@google.com diff --git a/deps/zlib/README.chromium b/deps/zlib/README.chromium index 3d90f79be2ffc2..c3c1ef69ad4656 100644 --- a/deps/zlib/README.chromium +++ b/deps/zlib/README.chromium @@ -2,6 +2,7 @@ Name: zlib Short Name: zlib URL: http://zlib.net/ Version: 1.2.11 +CPEPrefix: cpe:/a:zlib:zlib:1.2.11 Security Critical: yes License: Custom license License File: LICENSE diff --git a/deps/zlib/adler32.c b/deps/zlib/adler32.c index a42f35fce2a2e0..8f8fbb904891ae 100644 --- a/deps/zlib/adler32.c +++ b/deps/zlib/adler32.c @@ -59,10 +59,8 @@ local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2)); # define MOD63(a) a %= BASE #endif -#if defined(ADLER32_SIMD_SSSE3) -#include "adler32_simd.h" -#include "x86.h" -#elif defined(ADLER32_SIMD_NEON) +#include "cpu_features.h" +#if defined(ADLER32_SIMD_SSSE3) || defined(ADLER32_SIMD_NEON) #include "adler32_simd.h" #endif @@ -76,10 +74,10 @@ uLong ZEXPORT adler32_z(adler, buf, len) unsigned n; #if defined(ADLER32_SIMD_SSSE3) - if (x86_cpu_enable_ssse3 && buf && len >= 64) + if (buf != Z_NULL && len >= 64 && x86_cpu_enable_ssse3) return adler32_simd_(adler, buf, len); #elif defined(ADLER32_SIMD_NEON) - if (buf && len >= 64) + if (buf != Z_NULL && len >= 64) return adler32_simd_(adler, buf, len); #endif @@ -108,7 +106,7 @@ uLong ZEXPORT adler32_z(adler, buf, len) */ if (buf == Z_NULL) { if (!len) /* Assume user is calling adler32(0, NULL, 0); */ - x86_check_features(); + cpu_check_features(); return 1L; } #else diff --git a/deps/zlib/adler32_simd.c b/deps/zlib/adler32_simd.c index f8b07297b93840..1354915cc099ad 100644 --- a/deps/zlib/adler32_simd.c +++ b/deps/zlib/adler32_simd.c @@ -50,13 +50,9 @@ #define NMAX 5552 #if defined(ADLER32_SIMD_SSSE3) -#ifndef __GNUC__ -#define __attribute__() -#endif #include -__attribute__((target("ssse3"))) uint32_t ZLIB_INTERNAL adler32_simd_( /* SSSE3 */ uint32_t adler, const unsigned char *buf, diff --git a/deps/zlib/arm_features.c b/deps/zlib/arm_features.c deleted file mode 100644 index f5641c39bbce97..00000000000000 --- a/deps/zlib/arm_features.c +++ /dev/null @@ -1,90 +0,0 @@ -/* arm_features.c -- ARM processor features detection. - * - * Copyright 2018 The Chromium Authors. All rights reserved. - * Use of this source code is governed by a BSD-style license that can be - * found in the Chromium source repository LICENSE file. - */ - -#include "arm_features.h" -#include "zutil.h" -#include - -int ZLIB_INTERNAL arm_cpu_enable_crc32 = 0; -int ZLIB_INTERNAL arm_cpu_enable_pmull = 0; - -#if defined(ARMV8_OS_ANDROID) || defined(ARMV8_OS_LINUX) || defined(ARMV8_OS_FUCHSIA) -#include -#endif - -#if defined(ARMV8_OS_ANDROID) -#include -#elif defined(ARMV8_OS_LINUX) -#include -#include -#elif defined(ARMV8_OS_FUCHSIA) -#include -#include -#include -#elif defined(ARMV8_OS_WINDOWS) -#include -#else -#error arm_features.c ARM feature detection in not defined for your platform -#endif - -static void _arm_check_features(void); - -#if defined(ARMV8_OS_ANDROID) || defined(ARMV8_OS_LINUX) || defined(ARMV8_OS_FUCHSIA) -static pthread_once_t cpu_check_inited_once = PTHREAD_ONCE_INIT; -void ZLIB_INTERNAL arm_check_features(void) -{ - pthread_once(&cpu_check_inited_once, _arm_check_features); -} -#elif defined(ARMV8_OS_WINDOWS) -static INIT_ONCE cpu_check_inited_once = INIT_ONCE_STATIC_INIT; -static BOOL CALLBACK _arm_check_features_forwarder(PINIT_ONCE once, PVOID param, PVOID* context) -{ - _arm_check_features(); - return TRUE; -} -void ZLIB_INTERNAL arm_check_features(void) -{ - InitOnceExecuteOnce(&cpu_check_inited_once, _arm_check_features_forwarder, - NULL, NULL); -} -#endif - -/* - * See http://bit.ly/2CcoEsr for run-time detection of ARM features and also - * crbug.com/931275 for android_getCpuFeatures() use in the Android sandbox. - */ -static void _arm_check_features(void) -{ -#if defined(ARMV8_OS_ANDROID) && defined(__aarch64__) - uint64_t features = android_getCpuFeatures(); - arm_cpu_enable_crc32 = !!(features & ANDROID_CPU_ARM64_FEATURE_CRC32); - arm_cpu_enable_pmull = !!(features & ANDROID_CPU_ARM64_FEATURE_PMULL); -#elif defined(ARMV8_OS_ANDROID) /* aarch32 */ - uint64_t features = android_getCpuFeatures(); - arm_cpu_enable_crc32 = !!(features & ANDROID_CPU_ARM_FEATURE_CRC32); - arm_cpu_enable_pmull = !!(features & ANDROID_CPU_ARM_FEATURE_PMULL); -#elif defined(ARMV8_OS_LINUX) && defined(__aarch64__) - unsigned long features = getauxval(AT_HWCAP); - arm_cpu_enable_crc32 = !!(features & HWCAP_CRC32); - arm_cpu_enable_pmull = !!(features & HWCAP_PMULL); -#elif defined(ARMV8_OS_LINUX) && (defined(__ARM_NEON) || defined(__ARM_NEON__)) - /* Query HWCAP2 for ARMV8-A SoCs running in aarch32 mode */ - unsigned long features = getauxval(AT_HWCAP2); - arm_cpu_enable_crc32 = !!(features & HWCAP2_CRC32); - arm_cpu_enable_pmull = !!(features & HWCAP2_PMULL); -#elif defined(ARMV8_OS_FUCHSIA) - uint32_t features; - zx_status_t rc = zx_system_get_features(ZX_FEATURE_KIND_CPU, &features); - if (rc != ZX_OK || (features & ZX_ARM64_FEATURE_ISA_ASIMD) == 0) - return; /* Report nothing if ASIMD(NEON) is missing */ - arm_cpu_enable_crc32 = !!(features & ZX_ARM64_FEATURE_ISA_CRC32); - arm_cpu_enable_pmull = !!(features & ZX_ARM64_FEATURE_ISA_PMULL); -#elif defined(ARMV8_OS_WINDOWS) - arm_cpu_enable_crc32 = IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE); - arm_cpu_enable_pmull = IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE); -#endif -} diff --git a/deps/zlib/chromeconf.h b/deps/zlib/chromeconf.h index 666093d696a583..5ecf29edbff5ae 100644 --- a/deps/zlib/chromeconf.h +++ b/deps/zlib/chromeconf.h @@ -192,4 +192,8 @@ #define arm_check_features Cr_z_arm_check_features #define armv8_crc32_little Cr_z_armv8_crc32_little +/* Symbols added by cpu_features.c */ +#define cpu_check_features Cr_z_cpu_check_features +#define x86_cpu_enable_sse2 Cr_z_x86_cpu_enable_sse2 + #endif /* THIRD_PARTY_ZLIB_CHROMECONF_H_ */ diff --git a/deps/zlib/contrib/bench/zlib_bench.cc b/deps/zlib/contrib/bench/zlib_bench.cc index 5dcdef09f36b76..252560ea9e0b18 100644 --- a/deps/zlib/contrib/bench/zlib_bench.cc +++ b/deps/zlib/contrib/bench/zlib_bench.cc @@ -15,7 +15,7 @@ * Note this code can be compiled outside of the Chromium build system against * the system zlib (-lz) with g++ or clang++ as follows: * - * g++|clang++ -O3 -Wall -std=c++11 -lstdc++ -lz zlib_bench.cc + * g++|clang++ -O3 -Wall -std=c++11 zlib_bench.cc -lstdc++ -lz */ #include @@ -38,7 +38,7 @@ void error_exit(const char* error, int code) { } inline char* string_data(std::string* s) { - return s->empty() ? 0 : &*s->begin(); + return s->empty() ? nullptr : &*s->begin(); } struct Data { @@ -99,10 +99,25 @@ const char* zlib_wrapper_name(zlib_wrapper type) { if (type == kWrapperZRAW) return "RAW"; error_exit("bad wrapper type", int(type)); - return 0; + return nullptr; +} + +static int zlib_strategy = Z_DEFAULT_STRATEGY; + +const char* zlib_level_strategy_name(int compression_level) { + if (compression_level == 0) + return ""; // strategy is meaningless at level 0 + if (zlib_strategy == Z_HUFFMAN_ONLY) + return "huffman "; + if (zlib_strategy == Z_RLE) + return "rle "; + if (zlib_strategy == Z_DEFAULT_STRATEGY) + return ""; + error_exit("bad strategy", zlib_strategy); + return nullptr; } -static int zlib_compression_level; +static int zlib_compression_level = Z_DEFAULT_COMPRESSION; void zlib_compress( const zlib_wrapper type, @@ -119,7 +134,7 @@ void zlib_compress( memset(&stream, 0, sizeof(stream)); int result = deflateInit2(&stream, zlib_compression_level, Z_DEFLATED, - zlib_stream_wrapper_type(type), MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY); + zlib_stream_wrapper_type(type), MAX_MEM_LEVEL, zlib_strategy); if (result != Z_OK) error_exit("deflateInit2 failed", result); @@ -178,14 +193,19 @@ void verify_equal(const char* input, size_t size, std::string* output) { exit(3); } -void zlib_file(const char* name, const zlib_wrapper type) { +void zlib_file(const char* name, const zlib_wrapper type, int width) { /* * Read the file data. */ const auto file = read_file_data_or_exit(name); const int length = static_cast(file.size); const char* data = file.data.get(); - printf("%-40s :\n", name); + + /* + * Report compression strategy and file name. + */ + const char* strategy = zlib_level_strategy_name(zlib_compression_level); + printf("%s%-40s :\n", strategy, name); /* * Chop the data into blocks. @@ -263,9 +283,9 @@ void zlib_file(const char* name, const zlib_wrapper type) { double inflate_rate_max = length * repeats / mega_byte / utime[0]; // type, block size, compression ratio, etc - printf("%s: [b %dM] bytes %6d -> %6u %4.1f%%", - zlib_wrapper_name(type), block_size / (1 << 20), length, - static_cast(output_length), output_length * 100.0 / length); + printf("%s: [b %dM] bytes %*d -> %*u %4.2f%%", + zlib_wrapper_name(type), block_size / (1 << 20), width, length, width, + unsigned(output_length), output_length * 100.0 / length); // compress / uncompress median (max) rates printf(" comp %5.1f (%5.1f) MB/s uncomp %5.1f (%5.1f) MB/s\n", @@ -276,18 +296,24 @@ static int argn = 1; char* get_option(int argc, char* argv[], const char* option) { if (argn < argc) - return !strcmp(argv[argn], option) ? argv[argn++] : 0; - return 0; + return !strcmp(argv[argn], option) ? argv[argn++] : nullptr; + return nullptr; } -bool get_compression(int argc, char* argv[], int* value) { +bool get_compression(int argc, char* argv[], int& value) { if (argn < argc) - *value = atoi(argv[argn++]); - return *value >= 1 && *value <= 9; + value = isdigit(argv[argn][0]) ? atoi(argv[argn++]) : -1; + return value >= 0 && value <= 9; +} + +void get_field_width(int argc, char* argv[], int& value) { + value = atoi(argv[argn++]); } void usage_exit(const char* program) { - printf("usage: %s gzip|zlib|raw [--compression 1:9] files...\n", program); + static auto* options = + "gzip|zlib|raw [--compression 0:9] [--huffman|--rle] [--field width]"; + printf("usage: %s %s files ...\n", program, options); exit(1); } @@ -302,15 +328,30 @@ int main(int argc, char* argv[]) { else usage_exit(argv[0]); - if (!get_option(argc, argv, "--compression")) - zlib_compression_level = Z_DEFAULT_COMPRESSION; - else if (!get_compression(argc, argv, &zlib_compression_level)) - usage_exit(argv[0]); + int file_size_field_width = 0; + + while (argn < argc && argv[argn][0] == '-') { + if (get_option(argc, argv, "--compression")) { + if (!get_compression(argc, argv, zlib_compression_level)) + usage_exit(argv[0]); + } else if (get_option(argc, argv, "--field")) { + get_field_width(argc, argv, file_size_field_width); + } else if (get_option(argc, argv, "--huffman")) { + zlib_strategy = Z_HUFFMAN_ONLY; + } else if (get_option(argc, argv, "--rle")) { + zlib_strategy = Z_RLE; + } else { + usage_exit(argv[0]); + } + } if (argn >= argc) usage_exit(argv[0]); + + if (file_size_field_width < 6) + file_size_field_width = 6; while (argn < argc) - zlib_file(argv[argn++], type); + zlib_file(argv[argn++], type, file_size_field_width); return 0; } diff --git a/deps/zlib/contrib/minizip/iowin32.c b/deps/zlib/contrib/minizip/iowin32.c index 246ceb91a13942..c6bc314b3c28af 100644 --- a/deps/zlib/contrib/minizip/iowin32.c +++ b/deps/zlib/contrib/minizip/iowin32.c @@ -31,14 +31,12 @@ #define _WIN32_WINNT 0x601 #endif -#if _WIN32_WINNT >= _WIN32_WINNT_WIN8 -// see Include/shared/winapifamily.h in the Windows Kit -#if defined(WINAPI_FAMILY_PARTITION) && (!(defined(IOWIN32_USING_WINRT_API))) -#if WINAPI_FAMILY_ONE_PARTITION(WINAPI_FAMILY, WINAPI_PARTITION_APP) +#if !defined(IOWIN32_USING_WINRT_API) +#if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP) +// Windows Store or Universal Windows Platform #define IOWIN32_USING_WINRT_API 1 #endif #endif -#endif voidpf ZCALLBACK win32_open_file_func OF((voidpf opaque, const char* filename, int mode)); uLong ZCALLBACK win32_read_file_func OF((voidpf opaque, voidpf stream, void* buf, uLong size)); diff --git a/deps/zlib/contrib/minizip/miniunz.c b/deps/zlib/contrib/minizip/miniunz.c index 3d65401be5cdd7..08737f689a96f3 100644 --- a/deps/zlib/contrib/minizip/miniunz.c +++ b/deps/zlib/contrib/minizip/miniunz.c @@ -12,7 +12,7 @@ Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) */ -#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) +#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) && (!defined(__ANDROID_API__)) #ifndef __USE_FILE_OFFSET64 #define __USE_FILE_OFFSET64 #endif @@ -27,7 +27,7 @@ #endif #endif -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) // In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions #define FOPEN_FUNC(filename, mode) fopen(filename, mode) #define FTELLO_FUNC(stream) ftello(stream) @@ -45,6 +45,7 @@ #include #include #include +#include #ifdef _WIN32 # include @@ -97,7 +98,7 @@ void change_file_date(filename,dosdate,tmu_date) SetFileTime(hFile,&ftm,&ftLastAcc,&ftm); CloseHandle(hFile); #else -#ifdef unix || __APPLE__ +#if defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) struct utimbuf ut; struct tm newdate; newdate.tm_sec = tmu_date.tm_sec; @@ -125,11 +126,9 @@ int mymkdir(dirname) const char* dirname; { int ret=0; -#ifdef _WIN32 +#if defined(_WIN32) ret = _mkdir(dirname); -#elif unix - ret = mkdir (dirname,0775); -#elif __APPLE__ +#elif defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) ret = mkdir (dirname,0775); #endif return ret; diff --git a/deps/zlib/contrib/minizip/minizip.c b/deps/zlib/contrib/minizip/minizip.c index 4288962ecef056..b794953c5c2393 100644 --- a/deps/zlib/contrib/minizip/minizip.c +++ b/deps/zlib/contrib/minizip/minizip.c @@ -12,8 +12,7 @@ Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) */ - -#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) +#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) && (!defined(__ANDROID_API__)) #ifndef __USE_FILE_OFFSET64 #define __USE_FILE_OFFSET64 #endif @@ -28,7 +27,7 @@ #endif #endif -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) // In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions #define FOPEN_FUNC(filename, mode) fopen(filename, mode) #define FTELLO_FUNC(stream) ftello(stream) @@ -94,7 +93,7 @@ uLong filetime(f, tmzip, dt) return ret; } #else -#ifdef unix || __APPLE__ +#if defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) uLong filetime(f, tmzip, dt) char *f; /* name of file to get info on */ tm_zip *tmzip; /* return value: access, modific. and creation times */ diff --git a/deps/zlib/contrib/minizip/minizip.md b/deps/zlib/contrib/minizip/minizip.md new file mode 100644 index 00000000000000..9f15dd23dec279 --- /dev/null +++ b/deps/zlib/contrib/minizip/minizip.md @@ -0,0 +1,9 @@ +Minizip is a library provided by //third_party/zlib [1]. Its zip and unzip +tools can be built in a developer checkout for testing purposes with: + +```shell + autoninja -C out/Release minizip_bin + autoninja -C out/Release miniunz_bin +``` + +[1] Upstream is https://github.com/madler/zlib/tree/master/contrib/minizip diff --git a/deps/zlib/contrib/optimizations/chunkcopy.h b/deps/zlib/contrib/optimizations/chunkcopy.h index 8bcdadad05a817..9c0b7cb06c2dcc 100644 --- a/deps/zlib/contrib/optimizations/chunkcopy.h +++ b/deps/zlib/contrib/optimizations/chunkcopy.h @@ -29,7 +29,6 @@ #include typedef uint8x16_t z_vec128i_t; #elif defined(INFLATE_CHUNK_SIMD_SSE2) -#pragma GCC target ("sse2") #include typedef __m128i z_vec128i_t; #else @@ -113,6 +112,10 @@ static inline unsigned char FAR* chunkcopy_core_safe( Assert(out + len <= limit, "chunk copy exceeds safety limit"); if ((limit - out) < (ptrdiff_t)CHUNKCOPY_CHUNK_SIZE) { const unsigned char FAR* Z_RESTRICT rfrom = from; + Assert((uintptr_t)out - (uintptr_t)from >= len, + "invalid restrict in chunkcopy_core_safe"); + Assert((uintptr_t)from - (uintptr_t)out >= len, + "invalid restrict in chunkcopy_core_safe"); if (len & 8) { Z_BUILTIN_MEMCPY(out, rfrom, 8); out += 8; @@ -339,6 +342,10 @@ static inline unsigned char FAR* chunkcopy_relaxed( unsigned char FAR* Z_RESTRICT out, const unsigned char FAR* Z_RESTRICT from, unsigned len) { + Assert((uintptr_t)out - (uintptr_t)from >= len, + "invalid restrict in chunkcopy_relaxed"); + Assert((uintptr_t)from - (uintptr_t)out >= len, + "invalid restrict in chunkcopy_relaxed"); return chunkcopy_core(out, from, len); } @@ -361,6 +368,11 @@ static inline unsigned char FAR* chunkcopy_safe( unsigned len, unsigned char FAR* limit) { Assert(out + len <= limit, "chunk copy exceeds safety limit"); + Assert((uintptr_t)out - (uintptr_t)from >= len, + "invalid restrict in chunkcopy_safe"); + Assert((uintptr_t)from - (uintptr_t)out >= len, + "invalid restrict in chunkcopy_safe"); + return chunkcopy_core_safe(out, from, len, limit); } @@ -407,6 +419,26 @@ static inline unsigned char FAR* chunkcopy_lapped_safe( return chunkcopy_lapped_relaxed(out, dist, len); } +/* TODO(cavalcanti): see crbug.com/1110083. */ +static inline unsigned char FAR* chunkcopy_safe_ugly(unsigned char FAR* out, + unsigned dist, + unsigned len, + unsigned char FAR* limit) { +#if defined(__GNUC__) && !defined(__clang__) + /* Speed is the same as using chunkcopy_safe + w/ GCC on ARM (tested gcc 6.3 and 7.5) and avoids + undefined behavior. + */ + return chunkcopy_core_safe(out, out - dist, len, limit); +#elif defined(__clang__) && defined(ARMV8_OS_ANDROID) && !defined(__aarch64__) + /* Seems to perform better on 32bit (i.e. Android). */ + return chunkcopy_core_safe(out, out - dist, len, limit); +#else + /* Seems to perform better on 64bit. */ + return chunkcopy_lapped_safe(out, dist, len, limit); +#endif +} + /* * The chunk-copy code above deals with writing the decoded DEFLATE data to * the output with SIMD methods to increase decode speed. Reading the input diff --git a/deps/zlib/contrib/optimizations/inffast_chunk.c b/deps/zlib/contrib/optimizations/inffast_chunk.c index 4099edf3961d74..4bacbc46f620a3 100644 --- a/deps/zlib/contrib/optimizations/inffast_chunk.c +++ b/deps/zlib/contrib/optimizations/inffast_chunk.c @@ -276,7 +276,7 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ the main copy is near the end. */ out = chunkunroll_relaxed(out, &dist, &len); - out = chunkcopy_safe(out, out - dist, len, limit); + out = chunkcopy_safe_ugly(out, dist, len, limit); } else { /* from points to window, so there is no risk of overlapping pointers requiring memset-like behaviour diff --git a/deps/zlib/contrib/optimizations/insert_string.h b/deps/zlib/contrib/optimizations/insert_string.h index 69eee3dc9e91fc..9f634ae351b2e5 100644 --- a/deps/zlib/contrib/optimizations/insert_string.h +++ b/deps/zlib/contrib/optimizations/insert_string.h @@ -4,45 +4,51 @@ * Use of this source code is governed by a BSD-style license that can be * found in the Chromium source repository LICENSE file. */ -#ifdef _MSC_VER + +#if defined(_MSC_VER) #define INLINE __inline #else #define INLINE inline #endif -/* Optimized insert_string block */ -#if defined(CRC32_SIMD_SSE42_PCLMUL) || defined(CRC32_ARMV8_CRC32) -#define TARGET_CPU_WITH_CRC +#include "cpu_features.h" + // clang-format off #if defined(CRC32_SIMD_SSE42_PCLMUL) - /* Required to make MSVC bot build pass. */ - #include - #if defined(__GNUC__) || defined(__clang__) - #undef TARGET_CPU_WITH_CRC + #include /* Required to make MSVC bot build pass. */ + + #if defined(__clang__) || defined(__GNUC__) #define TARGET_CPU_WITH_CRC __attribute__((target("sse4.2"))) + #else + #define TARGET_CPU_WITH_CRC #endif #define _cpu_crc32_u32 _mm_crc32_u32 #elif defined(CRC32_ARMV8_CRC32) - #include "arm_features.h" #if defined(__clang__) - #undef TARGET_CPU_WITH_CRC #define __crc32cw __builtin_arm_crc32cw + #elif defined(__GNUC__) + #define __crc32cw __builtin_aarch64_crc32cw #endif - #define _cpu_crc32_u32 __crc32cw - - #if defined(__aarch64__) + #if defined(__aarch64__) && defined(__clang__) #define TARGET_CPU_WITH_CRC __attribute__((target("crc"))) - #else // !defined(__aarch64__) + #elif defined(__aarch64__) && defined(__GNUC__) + #define TARGET_CPU_WITH_CRC __attribute__((target("+crc"))) + #elif defined(__clang__) // !defined(__aarch64__) #define TARGET_CPU_WITH_CRC __attribute__((target("armv8-a,crc"))) #endif // defined(__aarch64__) + + #define _cpu_crc32_u32 __crc32cw + #endif // clang-format on + +#if defined(TARGET_CPU_WITH_CRC) + TARGET_CPU_WITH_CRC -local INLINE Pos insert_string_optimized(deflate_state* const s, - const Pos str) { +local INLINE Pos insert_string_simd(deflate_state* const s, const Pos str) { Pos ret; unsigned *ip, val, h = 0; @@ -64,7 +70,8 @@ local INLINE Pos insert_string_optimized(deflate_state* const s, s->prev[str & s->w_mask] = ret; return ret; } -#endif /* Optimized insert_string block */ + +#endif // TARGET_CPU_WITH_CRC /* =========================================================================== * Update a hash value with the given input byte @@ -99,24 +106,22 @@ local INLINE Pos insert_string_c(deflate_state* const s, const Pos str) { } local INLINE Pos insert_string(deflate_state* const s, const Pos str) { -/* String dictionary insertion: faster symbol hashing has a positive impact - * on data compression speeds (around 20% on Intel and 36% on Arm Cortex big - * cores). - * A misfeature is that the generated compressed output will differ from - * vanilla zlib (even though it is still valid 'DEFLATE-d' content). +/* insert_string_simd string dictionary insertion: this SIMD symbol hashing + * significantly improves data compression speed. * - * We offer here a way to disable the optimization if there is the expectation - * that compressed content should match when compared to vanilla zlib. + * Note: the generated compressed output is a valid DEFLATE stream but will + * differ from vanilla zlib output ... */ -#if !defined(CHROMIUM_ZLIB_NO_CASTAGNOLI) - /* TODO(cavalcantii): unify CPU features code. */ -#if defined(CRC32_ARMV8_CRC32) - if (arm_cpu_enable_crc32) - return insert_string_optimized(s, str); -#elif defined(CRC32_SIMD_SSE42_PCLMUL) +#if defined(CHROMIUM_ZLIB_NO_CASTAGNOLI) +/* ... so this build-time option can used to disable the SIMD symbol hasher + * if matching vanilla zlib DEFLATE output is required. + */ (;) /* FALLTHOUGH */ +#elif defined(TARGET_CPU_WITH_CRC) && defined(CRC32_SIMD_SSE42_PCLMUL) if (x86_cpu_enable_simd) - return insert_string_optimized(s, str); -#endif + return insert_string_simd(s, str); +#elif defined(TARGET_CPU_WITH_CRC) && defined(CRC32_ARMV8_CRC32) + if (arm_cpu_enable_crc32) + return insert_string_simd(s, str); #endif return insert_string_c(s, str); } diff --git a/deps/zlib/contrib/tests/DEPS b/deps/zlib/contrib/tests/DEPS new file mode 100644 index 00000000000000..42751740686776 --- /dev/null +++ b/deps/zlib/contrib/tests/DEPS @@ -0,0 +1,4 @@ +include_rules = [ + "+testing/gtest", + "+base", +] diff --git a/deps/zlib/contrib/tests/OWNERS b/deps/zlib/contrib/tests/OWNERS index 9a2fb6fbdc6b3f..aa6a2d1bb3c041 100644 --- a/deps/zlib/contrib/tests/OWNERS +++ b/deps/zlib/contrib/tests/OWNERS @@ -1 +1,2 @@ cblume@chromium.org +cavalcantii@chromium.org diff --git a/deps/zlib/contrib/tests/fuzzers/BUILD.gn b/deps/zlib/contrib/tests/fuzzers/BUILD.gn index c46b66440073a5..10abe00c0515b6 100644 --- a/deps/zlib/contrib/tests/fuzzers/BUILD.gn +++ b/deps/zlib/contrib/tests/fuzzers/BUILD.gn @@ -9,37 +9,27 @@ group("fuzzers") { } fuzzer_test("zlib_uncompress_fuzzer") { - sources = [ - "uncompress_fuzzer.cc", - ] - deps = [ - "../../../:zlib", - ] + sources = [ "uncompress_fuzzer.cc" ] + deps = [ "../../../:zlib" ] } fuzzer_test("zlib_inflate_fuzzer") { - sources = [ - "inflate_fuzzer.cc", - ] - deps = [ - "../../../:zlib", - ] + sources = [ "inflate_fuzzer.cc" ] + deps = [ "../../../:zlib" ] +} + +fuzzer_test("zlib_streaming_inflate_fuzzer") { + sources = [ "streaming_inflate_fuzzer.cc" ] + deps = [ "../../../:zlib" ] + libfuzzer_options = [ "max_len=256000" ] } fuzzer_test("zlib_deflate_set_dictionary_fuzzer") { - sources = [ - "deflate_set_dictionary_fuzzer.cc", - ] - deps = [ - "../../../:zlib", - ] + sources = [ "deflate_set_dictionary_fuzzer.cc" ] + deps = [ "../../../:zlib" ] } fuzzer_test("zlib_deflate_fuzzer") { - sources = [ - "deflate_fuzzer.cc", - ] - deps = [ - "../../../:zlib", - ] + sources = [ "deflate_fuzzer.cc" ] + deps = [ "../../../:zlib" ] } diff --git a/deps/zlib/contrib/tests/fuzzers/OWNERS b/deps/zlib/contrib/tests/fuzzers/OWNERS index 6397ce69c70649..9a2fb6fbdc6b3f 100644 --- a/deps/zlib/contrib/tests/fuzzers/OWNERS +++ b/deps/zlib/contrib/tests/fuzzers/OWNERS @@ -1,2 +1 @@ cblume@chromium.org -mmoroz@chromium.org diff --git a/deps/zlib/contrib/tests/fuzzers/deflate_fuzzer.cc b/deps/zlib/contrib/tests/fuzzers/deflate_fuzzer.cc index 6098ff162ce1c0..c00e71551d80db 100644 --- a/deps/zlib/contrib/tests/fuzzers/deflate_fuzzer.cc +++ b/deps/zlib/contrib/tests/fuzzers/deflate_fuzzer.cc @@ -2,46 +2,73 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +#include #include #include +#include +#include #include -#include #include #include "third_party/zlib/zlib.h" -static Bytef buffer[256 * 1024] = {0}; +// Fuzzer builds often have NDEBUG set, so roll our own assert macro. +#define ASSERT(cond) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "%s:%d Assert failed: %s\n", __FILE__, __LINE__, #cond); \ + exit(1); \ + } \ + } while (0) -// Entry point for LibFuzzer. extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { - // zlib's deflate requires non-zero input sizes - if (!size) - return 0; - - // We need to strip the 'const' for zlib. - std::vector input_buffer{data, data+size}; - - uLongf buffer_length = static_cast(sizeof(buffer)); + FuzzedDataProvider fdp(data, size); + int level = fdp.PickValueInArray({0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + int windowBits = fdp.PickValueInArray({9, 10, 11, 12, 13, 14, 15}); + int memLevel = fdp.PickValueInArray({1, 2, 3, 4, 5, 6, 7, 8, 9}); + int strategy = fdp.PickValueInArray( + {Z_DEFAULT_STRATEGY, Z_FILTERED, Z_HUFFMAN_ONLY, Z_RLE, Z_FIXED}); + std::vector src = fdp.ConsumeRemainingBytes(); z_stream stream; - stream.next_in = input_buffer.data(); - stream.avail_in = size; - stream.total_in = size; - stream.next_out = buffer; - stream.avail_out = buffer_length; - stream.total_out = buffer_length; stream.zalloc = Z_NULL; stream.zfree = Z_NULL; - if (Z_OK != deflateInit(&stream, Z_DEFAULT_COMPRESSION)) { - deflateEnd(&stream); - assert(false); + // Compress the data one byte at a time to exercise the streaming code. + int ret = + deflateInit2(&stream, level, Z_DEFLATED, windowBits, memLevel, strategy); + ASSERT(ret == Z_OK); + std::vector compressed(src.size() * 2 + 1000); + stream.next_out = compressed.data(); + stream.avail_out = compressed.size(); + for (uint8_t b : src) { + stream.next_in = &b; + stream.avail_in = 1; + ret = deflate(&stream, Z_NO_FLUSH); + ASSERT(ret == Z_OK); } - - auto deflate_result = deflate(&stream, Z_NO_FLUSH); + stream.next_in = Z_NULL; + stream.avail_in = 0; + ret = deflate(&stream, Z_FINISH); + ASSERT(ret == Z_STREAM_END); + compressed.resize(compressed.size() - stream.avail_out); deflateEnd(&stream); - if (Z_OK != deflate_result) - assert(false); + + // Verify that the data decompresses correctly. + ret = inflateInit2(&stream, windowBits); + ASSERT(ret == Z_OK); + // Make room for at least one byte so it's never empty. + std::vector decompressed(src.size() + 1); + stream.next_in = compressed.data(); + stream.avail_in = compressed.size(); + stream.next_out = decompressed.data(); + stream.avail_out = decompressed.size(); + ret = inflate(&stream, Z_FINISH); + ASSERT(ret == Z_STREAM_END); + decompressed.resize(decompressed.size() - stream.avail_out); + inflateEnd(&stream); + + ASSERT(decompressed == src); return 0; } diff --git a/deps/zlib/contrib/tests/fuzzers/streaming_inflate_fuzzer.cc b/deps/zlib/contrib/tests/fuzzers/streaming_inflate_fuzzer.cc new file mode 100644 index 00000000000000..de4d216756cb42 --- /dev/null +++ b/deps/zlib/contrib/tests/fuzzers/streaming_inflate_fuzzer.cc @@ -0,0 +1,74 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include +#include +#include +#include + +#include "third_party/zlib/zlib.h" + +// Fuzzer builds often have NDEBUG set, so roll our own assert macro. +#define ASSERT(cond) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "%s:%d Assert failed: %s\n", __FILE__, __LINE__, #cond); \ + exit(1); \ + } \ + } while (0) + +// Entry point for LibFuzzer. +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + // Deflate data. + z_stream comp_strm; + comp_strm.zalloc = Z_NULL; + comp_strm.zfree = Z_NULL; + comp_strm.opaque = Z_NULL; + int ret = deflateInit(&comp_strm, Z_DEFAULT_COMPRESSION); + ASSERT(ret == Z_OK); + + size_t comp_buf_cap = deflateBound(&comp_strm, size); + uint8_t* comp_buf = (uint8_t*)malloc(comp_buf_cap); + ASSERT(comp_buf != nullptr); + comp_strm.next_out = comp_buf; + comp_strm.avail_out = comp_buf_cap; + comp_strm.next_in = (unsigned char*)data; + comp_strm.avail_in = size; + ret = deflate(&comp_strm, Z_FINISH); + ASSERT(ret == Z_STREAM_END); + size_t comp_sz = comp_buf_cap - comp_strm.avail_out; + + // Inflate comp_buf one chunk at a time. + z_stream decomp_strm; + decomp_strm.zalloc = Z_NULL; + decomp_strm.zfree = Z_NULL; + decomp_strm.opaque = Z_NULL; + ret = inflateInit(&decomp_strm); + ASSERT(ret == Z_OK); + decomp_strm.next_in = comp_buf; + decomp_strm.avail_in = comp_sz; + + while (decomp_strm.avail_in > 0) { + uint8_t decomp_buf[1024]; + decomp_strm.next_out = decomp_buf; + decomp_strm.avail_out = sizeof(decomp_buf); + ret = inflate(&decomp_strm, Z_FINISH); + ASSERT(ret == Z_OK || ret == Z_STREAM_END || ret == Z_BUF_ERROR); + + // Verify the output bytes. + size_t num_out = sizeof(decomp_buf) - decomp_strm.avail_out; + for (size_t i = 0; i < num_out; i++) { + ASSERT(decomp_buf[i] == data[decomp_strm.total_out - num_out + i]); + } + } + + ret = deflateEnd(&comp_strm); + ASSERT(ret == Z_OK); + free(comp_buf); + + inflateEnd(&decomp_strm); + ASSERT(ret == Z_OK); + + return 0; +} diff --git a/deps/zlib/contrib/tests/infcover.cc b/deps/zlib/contrib/tests/infcover.cc new file mode 100644 index 00000000000000..c5300a52111aa2 --- /dev/null +++ b/deps/zlib/contrib/tests/infcover.cc @@ -0,0 +1,684 @@ +/* infcover.c -- test zlib's inflate routines with full code coverage + * Copyright (C) 2011, 2016 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* to use, do: ./configure --cover && make cover */ +// clang-format off +#include "infcover.h" +#include +#include +#include + +#include "zlib.h" + +/* get definition of internal structure so we can mess with it (see pull()), + and so we can call inflate_trees() (see cover5()) */ +#define ZLIB_INTERNAL +#include "inftrees.h" +#include "inflate.h" + +/* XXX: use C++ streams instead of printf/fputs/etc due to portability + * as type sizes can vary between platforms. + */ +#include +#define local static + +/* XXX: hacking C assert and plugging into GTest. */ +#include "gtest.h" +#if defined(assert) +#undef assert +#define assert EXPECT_TRUE +#endif + +/* XXX: handle what is a reserved word in C++. */ +#define try try_f + +/* -- memory tracking routines -- */ + +/* + These memory tracking routines are provided to zlib and track all of zlib's + allocations and deallocations, check for LIFO operations, keep a current + and high water mark of total bytes requested, optionally set a limit on the + total memory that can be allocated, and when done check for memory leaks. + + They are used as follows: + + z_stream strm; + mem_setup(&strm) initializes the memory tracking and sets the + zalloc, zfree, and opaque members of strm to use + memory tracking for all zlib operations on strm + mem_limit(&strm, limit) sets a limit on the total bytes requested -- a + request that exceeds this limit will result in an + allocation failure (returns NULL) -- setting the + limit to zero means no limit, which is the default + after mem_setup() + mem_used(&strm, "msg") prints to stderr "msg" and the total bytes used + mem_high(&strm, "msg") prints to stderr "msg" and the high water mark + mem_done(&strm, "msg") ends memory tracking, releases all allocations + for the tracking as well as leaked zlib blocks, if + any. If there was anything unusual, such as leaked + blocks, non-FIFO frees, or frees of addresses not + allocated, then "msg" and information about the + problem is printed to stderr. If everything is + normal, nothing is printed. mem_done resets the + strm members to Z_NULL to use the default memory + allocation routines on the next zlib initialization + using strm. + */ + +/* these items are strung together in a linked list, one for each allocation */ +struct mem_item { + void *ptr; /* pointer to allocated memory */ + size_t size; /* requested size of allocation */ + struct mem_item *next; /* pointer to next item in list, or NULL */ +}; + +/* this structure is at the root of the linked list, and tracks statistics */ +struct mem_zone { + struct mem_item *first; /* pointer to first item in list, or NULL */ + size_t total, highwater; /* total allocations, and largest total */ + size_t limit; /* memory allocation limit, or 0 if no limit */ + int notlifo, rogue; /* counts of non-LIFO frees and rogue frees */ +}; + +/* memory allocation routine to pass to zlib */ +local void *mem_alloc(void *mem, unsigned count, unsigned size) +{ + void *ptr; + struct mem_item *item; + struct mem_zone *zone = static_cast(mem); + size_t len = count * (size_t)size; + + /* induced allocation failure */ + if (zone == NULL || (zone->limit && zone->total + len > zone->limit)) + return NULL; + + /* perform allocation using the standard library, fill memory with a + non-zero value to make sure that the code isn't depending on zeros */ + ptr = malloc(len); + if (ptr == NULL) + return NULL; + memset(ptr, 0xa5, len); + + /* create a new item for the list */ + item = static_cast(malloc(sizeof(struct mem_item))); + if (item == NULL) { + free(ptr); + return NULL; + } + item->ptr = ptr; + item->size = len; + + /* insert item at the beginning of the list */ + item->next = zone->first; + zone->first = item; + + /* update the statistics */ + zone->total += item->size; + if (zone->total > zone->highwater) + zone->highwater = zone->total; + + /* return the allocated memory */ + return ptr; +} + +/* memory free routine to pass to zlib */ +local void mem_free(void *mem, void *ptr) +{ + struct mem_item *item, *next; + struct mem_zone *zone = static_cast(mem); + + /* if no zone, just do a free */ + if (zone == NULL) { + free(ptr); + return; + } + + /* point next to the item that matches ptr, or NULL if not found -- remove + the item from the linked list if found */ + next = zone->first; + if (next) { + if (next->ptr == ptr) + zone->first = next->next; /* first one is it, remove from list */ + else { + do { /* search the linked list */ + item = next; + next = item->next; + } while (next != NULL && next->ptr != ptr); + if (next) { /* if found, remove from linked list */ + item->next = next->next; + zone->notlifo++; /* not a LIFO free */ + } + + } + } + + /* if found, update the statistics and free the item */ + if (next) { + zone->total -= next->size; + free(next); + } + + /* if not found, update the rogue count */ + else + zone->rogue++; + + /* in any case, do the requested free with the standard library function */ + free(ptr); +} + +/* set up a controlled memory allocation space for monitoring, set the stream + parameters to the controlled routines, with opaque pointing to the space */ +local void mem_setup(z_stream *strm) +{ + struct mem_zone *zone; + + zone = static_cast(malloc(sizeof(struct mem_zone))); + assert(zone != NULL); + zone->first = NULL; + zone->total = 0; + zone->highwater = 0; + zone->limit = 0; + zone->notlifo = 0; + zone->rogue = 0; + strm->opaque = zone; + strm->zalloc = mem_alloc; + strm->zfree = mem_free; +} + +/* set a limit on the total memory allocation, or 0 to remove the limit */ +local void mem_limit(z_stream *strm, size_t limit) +{ + struct mem_zone *zone = static_cast(strm->opaque); + + zone->limit = limit; +} + +/* show the current total requested allocations in bytes */ +local void mem_used(z_stream *strm, const char *prefix) +{ + struct mem_zone *zone = static_cast(strm->opaque); + + std::cout << prefix << ": " << zone->total << " allocated" << std::endl; +} + +/* show the high water allocation in bytes */ +local void mem_high(z_stream *strm, const char *prefix) +{ + struct mem_zone *zone = static_cast(strm->opaque); + + std::cout << prefix << ": " << zone->highwater << " high water mark" << std::endl; +} + +/* release the memory allocation zone -- if there are any surprises, notify */ +local void mem_done(z_stream *strm, const char *prefix) +{ + int count = 0; + struct mem_item *item, *next; + struct mem_zone *zone = static_cast(strm->opaque); + + /* show high water mark */ + mem_high(strm, prefix); + + /* free leftover allocations and item structures, if any */ + item = zone->first; + while (item != NULL) { + free(item->ptr); + next = item->next; + free(item); + item = next; + count++; + } + + /* issue alerts about anything unexpected */ + if (count || zone->total) + std::cout << "** " << prefix << ": " + << zone->total << " bytes in " + << count << " blocks not freed" + << std::endl; + + if (zone->notlifo) + std::cout << "** " << prefix << ": " + << zone->notlifo << " frees not LIFO" + << std::endl; + + if (zone->rogue) + std::cout << "** " << prefix << ": " + << zone->rogue << " frees not recognized" + << std::endl; + + /* free the zone and delete from the stream */ + free(zone); + strm->opaque = Z_NULL; + strm->zalloc = Z_NULL; + strm->zfree = Z_NULL; +} + +/* -- inflate test routines -- */ + +/* Decode a hexadecimal string, set *len to length, in[] to the bytes. This + decodes liberally, in that hex digits can be adjacent, in which case two in + a row writes a byte. Or they can be delimited by any non-hex character, + where the delimiters are ignored except when a single hex digit is followed + by a delimiter, where that single digit writes a byte. The returned data is + allocated and must eventually be freed. NULL is returned if out of memory. + If the length is not needed, then len can be NULL. */ +local unsigned char *h2b(const char *hex, unsigned *len) +{ + unsigned char *in, *re; + unsigned next, val; + + in = static_cast(malloc((strlen(hex) + 1) >> 1)); + if (in == NULL) + return NULL; + next = 0; + val = 1; + do { + if (*hex >= '0' && *hex <= '9') + val = (val << 4) + *hex - '0'; + else if (*hex >= 'A' && *hex <= 'F') + val = (val << 4) + *hex - 'A' + 10; + else if (*hex >= 'a' && *hex <= 'f') + val = (val << 4) + *hex - 'a' + 10; + else if (val != 1 && val < 32) /* one digit followed by delimiter */ + val += 240; /* make it look like two digits */ + if (val > 255) { /* have two digits */ + in[next++] = val & 0xff; /* save the decoded byte */ + val = 1; /* start over */ + } + } while (*hex++); /* go through the loop with the terminating null */ + if (len != NULL) + *len = next; + re = static_cast(realloc(in, next)); + return re == NULL ? in : re; +} + +/* generic inflate() run, where hex is the hexadecimal input data, what is the + text to include in an error message, step is how much input data to feed + inflate() on each call, or zero to feed it all, win is the window bits + parameter to inflateInit2(), len is the size of the output buffer, and err + is the error code expected from the first inflate() call (the second + inflate() call is expected to return Z_STREAM_END). If win is 47, then + header information is collected with inflateGetHeader(). If a zlib stream + is looking for a dictionary, then an empty dictionary is provided. + inflate() is run until all of the input data is consumed. */ +local void inf(const char *hex, const char *what, unsigned step, int win, unsigned len, + int err) +{ + int ret; + unsigned have; + unsigned char *in, *out; + z_stream strm, copy; + gz_header head; + + mem_setup(&strm); + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit2(&strm, win); + if (ret != Z_OK) { + mem_done(&strm, what); + return; + } + out = static_cast(malloc(len)); assert(out != NULL); + if (win == 47) { + head.extra = out; + head.extra_max = len; + head.name = out; + head.name_max = len; + head.comment = out; + head.comm_max = len; + ret = inflateGetHeader(&strm, &head); assert(ret == Z_OK); + } + in = h2b(hex, &have); assert(in != NULL); + if (step == 0 || step > have) + step = have; + strm.avail_in = step; + have -= step; + strm.next_in = in; + do { + strm.avail_out = len; + strm.next_out = out; + ret = inflate(&strm, Z_NO_FLUSH); assert(err == 9 || ret == err); + if (ret != Z_OK && ret != Z_BUF_ERROR && ret != Z_NEED_DICT) + break; + if (ret == Z_NEED_DICT) { + ret = inflateSetDictionary(&strm, in, 1); + assert(ret == Z_DATA_ERROR); + mem_limit(&strm, 1); + ret = inflateSetDictionary(&strm, out, 0); + assert(ret == Z_MEM_ERROR); + mem_limit(&strm, 0); + ((struct inflate_state *)strm.state)->mode = DICT; + ret = inflateSetDictionary(&strm, out, 0); + assert(ret == Z_OK); + ret = inflate(&strm, Z_NO_FLUSH); assert(ret == Z_BUF_ERROR); + } + ret = inflateCopy(©, &strm); assert(ret == Z_OK); + ret = inflateEnd(©); assert(ret == Z_OK); + err = 9; /* don't care next time around */ + have += strm.avail_in; + strm.avail_in = step > have ? have : step; + have -= strm.avail_in; + } while (strm.avail_in); + free(in); + free(out); + ret = inflateReset2(&strm, -8); assert(ret == Z_OK); + ret = inflateEnd(&strm); assert(ret == Z_OK); + mem_done(&strm, what); +} + +/* cover all of the lines in inflate.c up to inflate() */ +void cover_support(void) +{ + int ret; + z_stream strm; + + mem_setup(&strm); + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit(&strm); assert(ret == Z_OK); + mem_used(&strm, "inflate init"); + ret = inflatePrime(&strm, 5, 31); assert(ret == Z_OK); + ret = inflatePrime(&strm, -1, 0); assert(ret == Z_OK); + ret = inflateSetDictionary(&strm, Z_NULL, 0); + assert(ret == Z_STREAM_ERROR); + ret = inflateEnd(&strm); assert(ret == Z_OK); + mem_done(&strm, "prime"); + + inf("63 0", "force window allocation", 0, -15, 1, Z_OK); + inf("63 18 5", "force window replacement", 0, -8, 259, Z_OK); + inf("63 18 68 30 d0 0 0", "force split window update", 4, -8, 259, Z_OK); + inf("3 0", "use fixed blocks", 0, -15, 1, Z_STREAM_END); + inf("", "bad window size", 0, 1, 0, Z_STREAM_ERROR); + + mem_setup(&strm); + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit_(&strm, ZLIB_VERSION - 1, (int)sizeof(z_stream)); + assert(ret == Z_VERSION_ERROR); + mem_done(&strm, "wrong version"); + + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit(&strm); assert(ret == Z_OK); + ret = inflateEnd(&strm); assert(ret == Z_OK); + std::cout << "inflate built-in memory routines" << std::endl;; +} + +/* cover all inflate() header and trailer cases and code after inflate() */ +void cover_wrap(void) +{ + int ret; + z_stream strm, copy; + unsigned char dict[257]; + + ret = inflate(Z_NULL, 0); assert(ret == Z_STREAM_ERROR); + ret = inflateEnd(Z_NULL); assert(ret == Z_STREAM_ERROR); + ret = inflateCopy(Z_NULL, Z_NULL); assert(ret == Z_STREAM_ERROR); + std::cout << "inflate bad parameters" << std::endl; + + inf("1f 8b 0 0", "bad gzip method", 0, 31, 0, Z_DATA_ERROR); + inf("1f 8b 8 80", "bad gzip flags", 0, 31, 0, Z_DATA_ERROR); + inf("77 85", "bad zlib method", 0, 15, 0, Z_DATA_ERROR); + inf("8 99", "set window size from header", 0, 0, 0, Z_OK); + inf("78 9c", "bad zlib window size", 0, 8, 0, Z_DATA_ERROR); + inf("78 9c 63 0 0 0 1 0 1", "check adler32", 0, 15, 1, Z_STREAM_END); + inf("1f 8b 8 1e 0 0 0 0 0 0 1 0 0 0 0 0 0", "bad header crc", 0, 47, 1, + Z_DATA_ERROR); + inf("1f 8b 8 2 0 0 0 0 0 0 1d 26 3 0 0 0 0 0 0 0 0 0", "check gzip length", + 0, 47, 0, Z_STREAM_END); + inf("78 90", "bad zlib header check", 0, 47, 0, Z_DATA_ERROR); + inf("8 b8 0 0 0 1", "need dictionary", 0, 8, 0, Z_NEED_DICT); + inf("78 9c 63 0", "compute adler32", 0, 15, 1, Z_OK); + + mem_setup(&strm); + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit2(&strm, -8); + strm.avail_in = 2; + strm.next_in = (Bytef *)"\x63"; + strm.avail_out = 1; + strm.next_out = (Bytef *)&ret; + mem_limit(&strm, 1); + ret = inflate(&strm, Z_NO_FLUSH); assert(ret == Z_MEM_ERROR); + ret = inflate(&strm, Z_NO_FLUSH); assert(ret == Z_MEM_ERROR); + mem_limit(&strm, 0); + memset(dict, 0, 257); + ret = inflateSetDictionary(&strm, dict, 257); + assert(ret == Z_OK); + mem_limit(&strm, (sizeof(struct inflate_state) << 1) + 256); + ret = inflatePrime(&strm, 16, 0); assert(ret == Z_OK); + strm.avail_in = 2; + strm.next_in = (Bytef *)"\x80"; + ret = inflateSync(&strm); assert(ret == Z_DATA_ERROR); + ret = inflate(&strm, Z_NO_FLUSH); assert(ret == Z_STREAM_ERROR); + strm.avail_in = 4; + strm.next_in = (Bytef *)"\0\0\xff\xff"; + ret = inflateSync(&strm); assert(ret == Z_OK); + (void)inflateSyncPoint(&strm); + ret = inflateCopy(©, &strm); assert(ret == Z_MEM_ERROR); + mem_limit(&strm, 0); + ret = inflateUndermine(&strm, 1); assert(ret == Z_DATA_ERROR); + (void)inflateMark(&strm); + ret = inflateEnd(&strm); assert(ret == Z_OK); + mem_done(&strm, "miscellaneous, force memory errors"); +} + +/* input and output functions for inflateBack() */ +local unsigned pull(void *desc, unsigned char **buf) +{ + static unsigned int next = 0; + static unsigned char dat[] = {0x63, 0, 2, 0}; + struct inflate_state *state; + + if (desc == Z_NULL) { + next = 0; + return 0; /* no input (already provided at next_in) */ + } + state = reinterpret_cast(((z_stream *)desc)->state); + if (state != Z_NULL) + state->mode = SYNC; /* force an otherwise impossible situation */ + return next < sizeof(dat) ? (*buf = dat + next++, 1) : 0; +} + +local int push(void *desc, unsigned char *buf, unsigned len) +{ + buf += len; + return desc != Z_NULL; /* force error if desc not null */ +} + +/* cover inflateBack() up to common deflate data cases and after those */ +void cover_back(void) +{ + int ret; + z_stream strm; + unsigned char win[32768]; + + ret = inflateBackInit_(Z_NULL, 0, win, 0, 0); + assert(ret == Z_VERSION_ERROR); + ret = inflateBackInit(Z_NULL, 0, win); assert(ret == Z_STREAM_ERROR); + ret = inflateBack(Z_NULL, Z_NULL, Z_NULL, Z_NULL, Z_NULL); + assert(ret == Z_STREAM_ERROR); + ret = inflateBackEnd(Z_NULL); assert(ret == Z_STREAM_ERROR); + std::cout << "inflateBack bad parameters" << std::endl;; + + mem_setup(&strm); + ret = inflateBackInit(&strm, 15, win); assert(ret == Z_OK); + strm.avail_in = 2; + strm.next_in = (Bytef *)"\x03"; + ret = inflateBack(&strm, pull, Z_NULL, push, Z_NULL); + assert(ret == Z_STREAM_END); + /* force output error */ + strm.avail_in = 3; + strm.next_in = (Bytef *)"\x63\x00"; + ret = inflateBack(&strm, pull, Z_NULL, push, &strm); + assert(ret == Z_BUF_ERROR); + /* force mode error by mucking with state */ + ret = inflateBack(&strm, pull, &strm, push, Z_NULL); + assert(ret == Z_STREAM_ERROR); + ret = inflateBackEnd(&strm); assert(ret == Z_OK); + mem_done(&strm, "inflateBack bad state"); + + ret = inflateBackInit(&strm, 15, win); assert(ret == Z_OK); + ret = inflateBackEnd(&strm); assert(ret == Z_OK); + std::cout << "inflateBack built-in memory routines" << std::endl;; +} + +/* do a raw inflate of data in hexadecimal with both inflate and inflateBack */ +local int try(const char *hex, const char *id, int err) +{ + int ret; + unsigned len, size; + unsigned char *in, *out, *win; + char *prefix; + z_stream strm; + + /* convert to hex */ + in = h2b(hex, &len); + assert(in != NULL); + + /* allocate work areas */ + size = len << 3; + out = static_cast(malloc(size)); + assert(out != NULL); + win = static_cast(malloc(32768)); + assert(win != NULL); + prefix = static_cast(malloc(strlen(id) + 6)); + assert(prefix != NULL); + + /* first with inflate */ + strcpy(prefix, id); + strcat(prefix, "-late"); + mem_setup(&strm); + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit2(&strm, err < 0 ? 47 : -15); + assert(ret == Z_OK); + strm.avail_in = len; + strm.next_in = in; + do { + strm.avail_out = size; + strm.next_out = out; + ret = inflate(&strm, Z_TREES); + assert(ret != Z_STREAM_ERROR && ret != Z_MEM_ERROR); + if (ret == Z_DATA_ERROR || ret == Z_NEED_DICT) + break; + } while (strm.avail_in || strm.avail_out == 0); + if (err) { + assert(ret == Z_DATA_ERROR); + assert(strcmp(id, strm.msg) == 0); + } + inflateEnd(&strm); + mem_done(&strm, prefix); + + /* then with inflateBack */ + if (err >= 0) { + strcpy(prefix, id); + strcat(prefix, "-back"); + mem_setup(&strm); + ret = inflateBackInit(&strm, 15, win); + assert(ret == Z_OK); + strm.avail_in = len; + strm.next_in = in; + ret = inflateBack(&strm, pull, Z_NULL, push, Z_NULL); + assert(ret != Z_STREAM_ERROR); + if (err) { + assert(ret == Z_DATA_ERROR); + assert(strcmp(id, strm.msg) == 0); + } + inflateBackEnd(&strm); + mem_done(&strm, prefix); + } + + /* clean up */ + free(prefix); + free(win); + free(out); + free(in); + return ret; +} + +/* cover deflate data cases in both inflate() and inflateBack() */ +void cover_inflate(void) +{ + try("0 0 0 0 0", "invalid stored block lengths", 1); + try("3 0", "fixed", 0); + try("6", "invalid block type", 1); + try("1 1 0 fe ff 0", "stored", 0); + try("fc 0 0", "too many length or distance symbols", 1); + try("4 0 fe ff", "invalid code lengths set", 1); + try("4 0 24 49 0", "invalid bit length repeat", 1); + try("4 0 24 e9 ff ff", "invalid bit length repeat", 1); + try("4 0 24 e9 ff 6d", "invalid code -- missing end-of-block", 1); + try("4 80 49 92 24 49 92 24 71 ff ff 93 11 0", + "invalid literal/lengths set", 1); + try("4 80 49 92 24 49 92 24 f b4 ff ff c3 84", "invalid distances set", 1); + try("4 c0 81 8 0 0 0 0 20 7f eb b 0 0", "invalid literal/length code", 1); + try("2 7e ff ff", "invalid distance code", 1); + try("c c0 81 0 0 0 0 0 90 ff 6b 4 0", "invalid distance too far back", 1); + + /* also trailer mismatch just in inflate() */ + try("1f 8b 8 0 0 0 0 0 0 0 3 0 0 0 0 1", "incorrect data check", -1); + try("1f 8b 8 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 1", + "incorrect length check", -1); + try("5 c0 21 d 0 0 0 80 b0 fe 6d 2f 91 6c", "pull 17", 0); + try("5 e0 81 91 24 cb b2 2c 49 e2 f 2e 8b 9a 47 56 9f fb fe ec d2 ff 1f", + "long code", 0); + try("ed c0 1 1 0 0 0 40 20 ff 57 1b 42 2c 4f", "length extra", 0); + try("ed cf c1 b1 2c 47 10 c4 30 fa 6f 35 1d 1 82 59 3d fb be 2e 2a fc f c", + "long distance and extra", 0); + try("ed c0 81 0 0 0 0 80 a0 fd a9 17 a9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 " + "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 6", "window end", 0); + inf("2 8 20 80 0 3 0", "inflate_fast TYPE return", 0, -15, 258, + Z_STREAM_END); + inf("63 18 5 40 c 0", "window wrap", 3, -8, 300, Z_OK); +} + +/* XXX(cavalcantii): fix linking error due inflate_table. */ +/* cover remaining lines in inftrees.c */ +/* void cover_trees(void) */ +/* { */ +/* int ret; */ +/* unsigned bits; */ +/* unsigned short lens[16], work[16]; */ +/* code *next, table[ENOUGH_DISTS]; */ + +/* /\* we need to call inflate_table() directly in order to manifest not- */ +/* enough errors, since zlib insures that enough is always enough *\/ */ +/* for (bits = 0; bits < 15; bits++) */ +/* lens[bits] = (unsigned short)(bits + 1); */ +/* lens[15] = 15; */ +/* next = table; */ +/* bits = 15; */ +/* ret = inflate_table(DISTS, lens, 16, &next, &bits, work); */ +/* assert(ret == 1); */ +/* next = table; */ +/* bits = 1; */ +/* ret = inflate_table(DISTS, lens, 16, &next, &bits, work); */ +/* assert(ret == 1); */ +/* fputs("inflate_table not enough errors\n", stderr); */ +/* } */ + +/* cover remaining inffast.c decoding and window copying */ +void cover_fast(void) +{ + inf("e5 e0 81 ad 6d cb b2 2c c9 01 1e 59 63 ae 7d ee fb 4d fd b5 35 41 68" + " ff 7f 0f 0 0 0", "fast length extra bits", 0, -8, 258, Z_DATA_ERROR); + inf("25 fd 81 b5 6d 59 b6 6a 49 ea af 35 6 34 eb 8c b9 f6 b9 1e ef 67 49" + " 50 fe ff ff 3f 0 0", "fast distance extra bits", 0, -8, 258, + Z_DATA_ERROR); + inf("3 7e 0 0 0 0 0", "fast invalid distance code", 0, -8, 258, + Z_DATA_ERROR); + inf("1b 7 0 0 0 0 0", "fast invalid literal/length code", 0, -8, 258, + Z_DATA_ERROR); + inf("d c7 1 ae eb 38 c 4 41 a0 87 72 de df fb 1f b8 36 b1 38 5d ff ff 0", + "fast 2nd level codes and too far back", 0, -8, 258, Z_DATA_ERROR); + inf("63 18 5 8c 10 8 0 0 0 0", "very common case", 0, -8, 259, Z_OK); + inf("63 60 60 18 c9 0 8 18 18 18 26 c0 28 0 29 0 0 0", + "contiguous and wrap around window", 6, -8, 259, Z_OK); + inf("63 0 3 0 0 0 0 0", "copy direct from output", 0, -8, 259, + Z_STREAM_END); +} + +// clang-format on diff --git a/deps/zlib/contrib/tests/infcover.h b/deps/zlib/contrib/tests/infcover.h new file mode 100644 index 00000000000000..b3e112f9154cbe --- /dev/null +++ b/deps/zlib/contrib/tests/infcover.h @@ -0,0 +1,11 @@ +#ifndef __INF_COVER_H__ +#define __INF_COVER_H__ + +void cover_support(void); +void cover_wrap(void); +void cover_back(void); +void cover_inflate(void); +void cover_trees(void); +void cover_fast(void); + +#endif diff --git a/deps/zlib/contrib/tests/run_all_unittests.cc b/deps/zlib/contrib/tests/run_all_unittests.cc new file mode 100644 index 00000000000000..65ed57e7fd7099 --- /dev/null +++ b/deps/zlib/contrib/tests/run_all_unittests.cc @@ -0,0 +1,14 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/bind.h" +#include "base/test/launcher/unit_test_launcher.h" +#include "base/test/test_suite.h" + +int main(int argc, char** argv) { + base::TestSuite test_suite(argc, argv); + return base::LaunchUnitTests( + argc, argv, + base::BindOnce(&base::TestSuite::Run, base::Unretained(&test_suite))); +} diff --git a/deps/zlib/contrib/tests/utils_unittest.cc b/deps/zlib/contrib/tests/utils_unittest.cc new file mode 100644 index 00000000000000..468421efed5a78 --- /dev/null +++ b/deps/zlib/contrib/tests/utils_unittest.cc @@ -0,0 +1,521 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the Chromium source repository LICENSE file. + +#include "infcover.h" + +#include +#include + +#include "compression_utils_portable.h" +#include "gtest.h" +#include "zlib.h" + +void TestPayloads(size_t input_size, zlib_internal::WrapperType type) { + std::vector input; + input.reserve(input_size); + for (size_t i = 1; i <= input_size; ++i) + input.push_back(i & 0xff); + + // If it is big enough for GZIP, will work for other wrappers. + std::vector compressed( + zlib_internal::GzipExpectedCompressedSize(input.size())); + std::vector decompressed(input.size()); + + // Libcores's java/util/zip/Deflater default settings: ZLIB, + // DEFAULT_COMPRESSION and DEFAULT_STRATEGY. + unsigned long compressed_size = static_cast(compressed.size()); + int result = zlib_internal::CompressHelper( + type, compressed.data(), &compressed_size, input.data(), input.size(), + Z_DEFAULT_COMPRESSION, nullptr, nullptr); + ASSERT_EQ(result, Z_OK); + + unsigned long decompressed_size = + static_cast(decompressed.size()); + result = zlib_internal::UncompressHelper(type, decompressed.data(), + &decompressed_size, + compressed.data(), compressed_size); + ASSERT_EQ(result, Z_OK); + EXPECT_EQ(input, decompressed); +} + +TEST(ZlibTest, ZlibWrapper) { + // Minimal ZLIB wrapped short stream size is about 8 bytes. + for (size_t i = 1; i < 1024; ++i) + TestPayloads(i, zlib_internal::WrapperType::ZLIB); +} + +TEST(ZlibTest, GzipWrapper) { + // GZIP should be 12 bytes bigger than ZLIB wrapper. + for (size_t i = 1; i < 1024; ++i) + TestPayloads(i, zlib_internal::WrapperType::GZIP); +} + +TEST(ZlibTest, RawWrapper) { + // RAW has no wrapper (V8 Blobs is a known user), size + // should be payload_size + 2 for short payloads. + for (size_t i = 1; i < 1024; ++i) + TestPayloads(i, zlib_internal::WrapperType::ZRAW); +} + +TEST(ZlibTest, InflateCover) { + cover_support(); + cover_wrap(); + cover_back(); + cover_inflate(); + // TODO(cavalcantii): enable this last test. + // cover_trees(); + cover_fast(); +} + +TEST(ZlibTest, DeflateStored) { + const int no_compression = 0; + const zlib_internal::WrapperType type = zlib_internal::WrapperType::GZIP; + std::vector input(1 << 10, 42); + std::vector compressed( + zlib_internal::GzipExpectedCompressedSize(input.size())); + std::vector decompressed(input.size()); + unsigned long compressed_size = static_cast(compressed.size()); + int result = zlib_internal::CompressHelper( + type, compressed.data(), &compressed_size, input.data(), input.size(), + no_compression, nullptr, nullptr); + ASSERT_EQ(result, Z_OK); + + unsigned long decompressed_size = + static_cast(decompressed.size()); + result = zlib_internal::UncompressHelper(type, decompressed.data(), + &decompressed_size, + compressed.data(), compressed_size); + ASSERT_EQ(result, Z_OK); + EXPECT_EQ(input, decompressed); +} + +TEST(ZlibTest, StreamingInflate) { + uint8_t comp_buf[4096], decomp_buf[4096]; + z_stream comp_strm, decomp_strm; + int ret; + + std::vector src; + for (size_t i = 0; i < 1000; i++) { + for (size_t j = 0; j < 40; j++) { + src.push_back(j); + } + } + + // Deflate src into comp_buf. + comp_strm.zalloc = Z_NULL; + comp_strm.zfree = Z_NULL; + comp_strm.opaque = Z_NULL; + ret = deflateInit(&comp_strm, Z_BEST_COMPRESSION); + ASSERT_EQ(ret, Z_OK); + comp_strm.next_out = comp_buf; + comp_strm.avail_out = sizeof(comp_buf); + comp_strm.next_in = src.data(); + comp_strm.avail_in = src.size(); + ret = deflate(&comp_strm, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + size_t comp_sz = sizeof(comp_buf) - comp_strm.avail_out; + + // Inflate comp_buf one 4096-byte buffer at a time. + decomp_strm.zalloc = Z_NULL; + decomp_strm.zfree = Z_NULL; + decomp_strm.opaque = Z_NULL; + ret = inflateInit(&decomp_strm); + ASSERT_EQ(ret, Z_OK); + decomp_strm.next_in = comp_buf; + decomp_strm.avail_in = comp_sz; + + while (decomp_strm.avail_in > 0) { + decomp_strm.next_out = decomp_buf; + decomp_strm.avail_out = sizeof(decomp_buf); + ret = inflate(&decomp_strm, Z_FINISH); + ASSERT_TRUE(ret == Z_OK || ret == Z_STREAM_END || ret == Z_BUF_ERROR); + + // Verify the output bytes. + size_t num_out = sizeof(decomp_buf) - decomp_strm.avail_out; + for (size_t i = 0; i < num_out; i++) { + EXPECT_EQ(decomp_buf[i], src[decomp_strm.total_out - num_out + i]); + } + } + + // Cleanup memory (i.e. makes ASAN bot happy). + ret = deflateEnd(&comp_strm); + EXPECT_EQ(ret, Z_OK); + ret = inflateEnd(&decomp_strm); + EXPECT_EQ(ret, Z_OK); +} + +TEST(ZlibTest, CRCHashBitsCollision) { + // The CRC32c of the hex sequences 2a,14,14,14 and 2a,14,db,14 have the same + // lower 9 bits. Since longest_match doesn't check match[2], a bad match could + // be chosen when the number of hash bits is <= 9. For this reason, the number + // of hash bits must be set higher, regardless of the memlevel parameter, when + // using CRC32c hashing for string matching. See https://crbug.com/1113596 + + std::vector src = { + // Random byte; zlib doesn't match at offset 0. + 123, + + // This will look like 5-byte match. + 0x2a, + 0x14, + 0xdb, + 0x14, + 0x15, + + // Offer a 4-byte match to bump the next expected match length to 5. + 0x2a, + 0x14, + 0x14, + 0x14, + + 0x2a, + 0x14, + 0x14, + 0x14, + 0x15, + }; + + z_stream stream; + stream.zalloc = nullptr; + stream.zfree = nullptr; + + // Using a low memlevel to try to reduce the number of hash bits. Negative + // windowbits means raw deflate, i.e. without the zlib header. + int ret = deflateInit2(&stream, /*comp level*/ 2, /*method*/ Z_DEFLATED, + /*windowbits*/ -15, /*memlevel*/ 2, + /*strategy*/ Z_DEFAULT_STRATEGY); + ASSERT_EQ(ret, Z_OK); + std::vector compressed(100, '\0'); + stream.next_out = compressed.data(); + stream.avail_out = compressed.size(); + stream.next_in = src.data(); + stream.avail_in = src.size(); + ret = deflate(&stream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + compressed.resize(compressed.size() - stream.avail_out); + deflateEnd(&stream); + + ret = inflateInit2(&stream, /*windowbits*/ -15); + ASSERT_EQ(ret, Z_OK); + std::vector decompressed(src.size(), '\0'); + stream.next_in = compressed.data(); + stream.avail_in = compressed.size(); + stream.next_out = decompressed.data(); + stream.avail_out = decompressed.size(); + ret = inflate(&stream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + EXPECT_EQ(0U, stream.avail_out); + inflateEnd(&stream); + + EXPECT_EQ(src, decompressed); +} + +TEST(ZlibTest, CRCHashAssert) { + // The CRC32c of the hex sequences ff,ff,5e,6f and ff,ff,13,ff have the same + // lower 15 bits. This means longest_match's assert that match[2] == scan[2] + // won't hold. However, such hash collisions are only possible when one of the + // other four bytes also mismatch. This tests that zlib's assert handles this + // case. + + std::vector src = { + // Random byte; zlib doesn't match at offset 0. + 123, + + // This has the same hash as the last byte sequence, and the first two and + // last two bytes match; though the third and the fourth don't. + 0xff, + 0xff, + 0x5e, + 0x6f, + 0x12, + 0x34, + + // Offer a 5-byte match to bump the next expected match length to 6 + // (because the two first and two last bytes need to match). + 0xff, + 0xff, + 0x13, + 0xff, + 0x12, + + 0xff, + 0xff, + 0x13, + 0xff, + 0x12, + 0x34, + }; + + z_stream stream; + stream.zalloc = nullptr; + stream.zfree = nullptr; + + int ret = deflateInit2(&stream, /*comp level*/ 5, /*method*/ Z_DEFLATED, + /*windowbits*/ -15, /*memlevel*/ 8, + /*strategy*/ Z_DEFAULT_STRATEGY); + ASSERT_EQ(ret, Z_OK); + std::vector compressed(100, '\0'); + stream.next_out = compressed.data(); + stream.avail_out = compressed.size(); + stream.next_in = src.data(); + stream.avail_in = src.size(); + ret = deflate(&stream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + compressed.resize(compressed.size() - stream.avail_out); + deflateEnd(&stream); + + ret = inflateInit2(&stream, /*windowbits*/ -15); + ASSERT_EQ(ret, Z_OK); + std::vector decompressed(src.size(), '\0'); + stream.next_in = compressed.data(); + stream.avail_in = compressed.size(); + stream.next_out = decompressed.data(); + stream.avail_out = decompressed.size(); + ret = inflate(&stream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + EXPECT_EQ(0U, stream.avail_out); + inflateEnd(&stream); + + EXPECT_EQ(src, decompressed); +} + +// Fuzzer generated. +static const uint8_t checkMatchCrashData[] = { + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x1e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0x00, + 0x6e, 0x6e, 0x6e, 0x00, 0x00, 0x00, 0x00, 0x6e, 0x01, 0x39, 0x6e, 0x6e, + 0x00, 0x00, 0x00, 0x00, 0xf7, 0xff, 0x00, 0x00, 0x00, 0x00, 0x6e, 0x6e, + 0x00, 0x00, 0x0a, 0x9a, 0x00, 0x00, 0x6e, 0x6e, 0x6e, 0x2a, 0x00, 0x00, + 0x00, 0xd5, 0xf0, 0x00, 0x81, 0x02, 0xf3, 0xfd, 0xff, 0xab, 0xf3, 0x6e, + 0x7e, 0x04, 0x5b, 0xf6, 0x2a, 0x2c, 0xf8, 0x00, 0x54, 0xf3, 0xa5, 0x0e, + 0xfd, 0x6e, 0xff, 0x00, 0xfd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xa4, 0x0b, 0xa5, 0x2a, 0x0d, 0x10, 0x01, 0x26, 0xf6, 0x04, 0x0e, + 0xff, 0x6e, 0x6e, 0x6e, 0x76, 0x00, 0x00, 0x87, 0x01, 0xfe, 0x0d, 0xb6, + 0x6e, 0x6e, 0xf7, 0x00, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xfd, 0x00, 0x2a, 0x00, 0x00, 0x00, 0x00, 0x29, 0x00, 0x9b, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, + 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x6e, 0xff, 0xff, 0x00, + 0x00, 0xd5, 0xf0, 0x00, 0xff, 0x40, 0x7e, 0x0b, 0xa5, 0x10, 0x67, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0x40, 0x7e, 0x0b, 0xa5, 0x10, 0x67, + 0x7e, 0x32, 0x6e, 0x00, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x40, 0x0b, 0xa5, + 0x10, 0x67, 0x01, 0xfe, 0x0d, 0xb6, 0x2a, 0x00, 0x00, 0x58, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x6e, 0xfd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x3d, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xd6, 0x2d, 0x2d, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, + 0x8a, 0x8a, 0x8a, 0x8a, 0x66, 0x8a, 0x8a, 0x8a, 0xee, 0x1d, 0x00, 0x00, + 0x00, 0x02, 0x00, 0x00, 0x00, 0xee, 0x0a, 0x00, 0x00, 0x00, 0x54, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xf3, 0x00, 0x00, 0xff, 0xff, 0x23, 0x7e, 0x00, 0x1e, + 0x00, 0x00, 0xd5, 0xf0, 0x00, 0xff, 0x40, 0x0b, 0xa5, 0x10, 0x67, 0x01, + 0xfe, 0x0d, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, + 0x8a, 0x8a, 0x8a, 0x2d, 0x6e, 0x2d, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x0e, + 0xfb, 0x00, 0x10, 0x24, 0x00, 0x00, 0xfb, 0xff, 0x00, 0x00, 0xff, 0x1f, + 0xb3, 0x00, 0x04, 0x3d, 0x00, 0xee, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x3d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, + 0x01, 0x45, 0x3d, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x11, 0x21, 0x00, 0x1e, + 0x00, 0x0c, 0xb3, 0xfe, 0x0e, 0xee, 0x02, 0x00, 0x1d, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x6e, 0x6e, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x6e, 0x00, + 0x00, 0x87, 0x00, 0x33, 0x38, 0x6e, 0x6e, 0x6e, 0x6e, 0x6e, 0x00, 0x00, + 0x00, 0x38, 0x00, 0x00, 0xff, 0xff, 0xff, 0x04, 0x3f, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xf0, 0x00, 0xff, 0x00, 0x31, 0x13, 0x13, 0x13, + 0x0b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xab, 0x30, 0x83, 0x33, + 0x00, 0x00, 0x01, 0x05, 0x00, 0x00, 0xff, 0xff, 0x7d, 0xff, 0x00, 0x01, + 0x10, 0x0d, 0x2a, 0xa5, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x11, + 0x21, 0x00, 0xa5, 0x00, 0x68, 0x68, 0x68, 0x67, 0x00, 0x00, 0xff, 0xff, + 0x02, 0x00, 0x00, 0x68, 0x68, 0x68, 0x68, 0x00, 0x00, 0xfa, 0xff, 0xff, + 0x03, 0x01, 0xff, 0x02, 0x00, 0x00, 0x68, 0x68, 0x68, 0x68, 0x0a, 0x10, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, + 0x06, 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xfa, 0xff, 0xff, 0x08, 0xff, 0xff, 0xff, 0x00, 0x06, 0x04, + 0x00, 0xf8, 0xff, 0xff, 0x00, 0x01, 0x00, 0x00, 0x25, 0x00, 0x00, 0x00, + 0x00, 0xff, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x78, 0x00, 0x00, 0x01, 0x00, 0xff, 0xff, 0xff, 0x00, 0x06, 0x04, 0x6e, + 0x7e, 0x87, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x00, + 0x00, 0x6e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x87, 0x6e, 0x6e, 0x6e, + 0x00, 0x01, 0x38, 0xd5, 0xf0, 0x00, 0x00, 0x2a, 0xfe, 0x04, 0x5b, 0x0d, + 0xfd, 0x6e, 0x92, 0x28, 0xf9, 0xfb, 0xff, 0x07, 0xd2, 0xd6, 0x2d, 0x2d, + 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, 0x8a, + 0x8a, 0x8a, 0xc2, 0x91, 0x00, 0x5b, 0xef, 0xde, 0xf2, 0x6e, 0x6e, 0xfd, + 0x0c, 0x02, 0x91, 0x62, 0x91, 0xfd, 0x6e, 0x6e, 0xd3, 0x06, 0x00, 0x00, + 0x1e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x00, + 0xd5, 0xf0, 0x00, 0xff, 0x00, 0x00, 0x31, 0x13, 0x13, 0x13, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x04, 0x00, 0x13, 0x0a, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x6e, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x09, 0x00, 0x6a, 0x24, 0x26, 0x30, 0x01, 0x2e, 0x2a, 0xfe, + 0x04, 0x5b, 0x0d, 0xfd, 0x6e, 0x6e, 0xd7, 0x06, 0x6e, 0x6e, 0x6e, 0x00, + 0x00, 0xb1, 0xb1, 0xb1, 0xb1, 0x00, 0x00, 0x00, 0x6e, 0x5b, 0x00, 0x00, + 0x58, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x58, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1e, 0x00, 0x00, 0x00, 0x87, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x0b, + 0x00, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x24, 0x2a, 0x6e, 0x5c, 0x24, + 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x87, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xeb, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x00, 0x40, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x05, 0x00, 0x00, 0x00, 0x5d, 0x10, 0x6e, 0x6e, 0xa5, 0x2f, 0x00, 0x00, + 0x95, 0x87, 0x00, 0x6e}; + +TEST(ZlibTest, CheckMatchCrash) { + // See https://crbug.com/1113142. + z_stream stream; + stream.zalloc = nullptr; + stream.zfree = nullptr; + + // Low windowbits to hit window sliding also with a relatively small input. + int ret = deflateInit2(&stream, /*comp level*/ 5, /*method*/ Z_DEFLATED, + /*windowbits*/ -9, /*memlevel*/ 8, + /*strategy*/ Z_DEFAULT_STRATEGY); + ASSERT_EQ(ret, Z_OK); + + uint8_t compressed[sizeof(checkMatchCrashData) * 2]; + stream.next_out = compressed; + stream.avail_out = sizeof(compressed); + + for (size_t i = 0; i < sizeof(checkMatchCrashData); i++) { + ASSERT_GT(stream.avail_out, 0U); + stream.next_in = (uint8_t*)&checkMatchCrashData[i]; + stream.avail_in = 1; + ret = deflate(&stream, Z_NO_FLUSH); + ASSERT_EQ(ret, Z_OK); + } + + stream.next_in = nullptr; + stream.avail_in = 0; + ASSERT_GT(stream.avail_out, 0U); + ret = deflate(&stream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + size_t compressed_sz = sizeof(compressed) - stream.avail_out; + deflateEnd(&stream); + + uint8_t decompressed[sizeof(checkMatchCrashData)]; + ret = inflateInit2(&stream, -15); + ASSERT_EQ(ret, Z_OK); + stream.next_in = compressed; + stream.avail_in = compressed_sz; + stream.next_out = decompressed; + stream.avail_out = sizeof(decompressed); + ret = inflate(&stream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + inflateEnd(&stream); + ASSERT_EQ( + memcmp(checkMatchCrashData, decompressed, sizeof(checkMatchCrashData)), + 0); +} + +TEST(ZlibTest, DeflateRLEUninitUse) { + // MSan would complain about use of uninitialized values in deflate_rle if the + // window isn't zero-initialized. See crbug.com/1137613. Similar problems + // exist in other places in zlib, e.g. longest_match (crbug.com/1144420) but + // we don't have as nice test cases. + + int level = 9; + int windowBits = 9; + int memLevel = 8; + int strategy = Z_RLE; + const std::vector src{ + 0x31, 0x64, 0x38, 0x32, 0x30, 0x32, 0x30, 0x36, 0x65, 0x35, 0x38, 0x35, + 0x32, 0x61, 0x30, 0x36, 0x65, 0x35, 0x32, 0x66, 0x30, 0x34, 0x38, 0x37, + 0x61, 0x31, 0x38, 0x36, 0x37, 0x37, 0x31, 0x39, 0x0a, 0x65, 0x62, 0x00, + 0x9f, 0xff, 0xc6, 0xc6, 0xc6, 0xff, 0x09, 0x00, 0x62, 0x00, 0x9f, 0xff, + 0xc6, 0xc6, 0xc6, 0xff, 0x09, 0x00, 0x62, 0x00, 0x9f, 0xff, 0xc6, 0xc6, + 0xc6, 0xff, 0x09, 0x00, 0x62, 0x00, 0x9f, 0xff, 0xc6, 0xc6, 0xc6, 0x95, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x0e, 0x0a, 0x54, 0x52, + 0x58, 0x56, 0xab, 0x26, 0x13, 0x53, 0x5a, 0xb5, 0x30, 0xbb, 0x96, 0x44, + 0x80, 0xe6, 0xc5, 0x0a, 0xd0, 0x47, 0x7a, 0xa0, 0x4e, 0xbe, 0x30, 0xdc, + 0xa1, 0x08, 0x54, 0xe1, 0x51, 0xd1, 0xea, 0xef, 0xdb, 0xa1, 0x2d, 0xb4, + 0xb9, 0x58, 0xb1, 0x2f, 0xf0, 0xae, 0xbc, 0x07, 0xd1, 0xba, 0x7f, 0x14, + 0xa4, 0xde, 0x99, 0x7f, 0x4d, 0x3e, 0x25, 0xd9, 0xef, 0xee, 0x4f, 0x38, + 0x7b, 0xaf, 0x3f, 0x6b, 0x53, 0x5a, 0xcb, 0x1f, 0x97, 0xb5, 0x43, 0xa3, + 0xe8, 0xff, 0x09, 0x00, 0x62, 0x00, 0x9f, 0xff, 0xc6, 0xc6, 0xc6, 0xff, + 0x09, 0x00, 0x62, 0x00, 0x9f, 0xff, 0xc6, 0xc6, 0xc6, 0xff, 0x09, 0x00, + 0x62, 0x00, 0x9f, 0xff, 0xc6, 0xc6, 0xc6, 0xff, 0x09, 0x00, 0x62, 0x00, + 0x9f, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x3c, + 0x73, 0x70, 0x23, 0x87, 0xec, 0xf8, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xc1, 0x00, 0x00, 0x9f, 0xc6, 0xc6, 0xff, 0x09, 0x00, 0x62, 0x00, 0x9f, + 0xff, 0xc6, 0xc6, 0xc6, 0xff, 0x09, 0x00, 0x62, 0x00, 0x9f, 0xff, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, + }; + + z_stream stream; + stream.zalloc = Z_NULL; + stream.zfree = Z_NULL; + + // Compress the data one byte at a time to exercise the streaming code. + int ret = + deflateInit2(&stream, level, Z_DEFLATED, windowBits, memLevel, strategy); + ASSERT_EQ(ret, Z_OK); + std::vector compressed(src.size() * 2 + 1000); + stream.next_out = compressed.data(); + stream.avail_out = compressed.size(); + for (uint8_t b : src) { + stream.next_in = &b; + stream.avail_in = 1; + ret = deflate(&stream, Z_NO_FLUSH); + ASSERT_EQ(ret, Z_OK); + } + stream.next_in = Z_NULL; + stream.avail_in = 0; + ret = deflate(&stream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + deflateEnd(&stream); +} diff --git a/deps/zlib/cpu_features.c b/deps/zlib/cpu_features.c new file mode 100644 index 00000000000000..70f01beebe30b5 --- /dev/null +++ b/deps/zlib/cpu_features.c @@ -0,0 +1,167 @@ +/* cpu_features.c -- Processor features detection. + * + * Copyright 2018 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#include "cpu_features.h" +#include "zutil.h" + +#include +#if defined(_MSC_VER) +#include +#elif defined(ADLER32_SIMD_SSSE3) +#include +#endif + +/* TODO(cavalcantii): remove checks for x86_flags on deflate. + */ +#if defined(ARMV8_OS_MACOS) +/* crc32 is a baseline feature in ARMv8.1-A, and macOS running on arm64 is new + * enough that this can be assumed without runtime detection. */ +int ZLIB_INTERNAL arm_cpu_enable_crc32 = 1; +#else +int ZLIB_INTERNAL arm_cpu_enable_crc32 = 0; +#endif +int ZLIB_INTERNAL arm_cpu_enable_pmull = 0; +int ZLIB_INTERNAL x86_cpu_enable_sse2 = 0; +int ZLIB_INTERNAL x86_cpu_enable_ssse3 = 0; +int ZLIB_INTERNAL x86_cpu_enable_simd = 0; + +#ifndef CPU_NO_SIMD + +#if defined(ARMV8_OS_ANDROID) || defined(ARMV8_OS_LINUX) || defined(ARMV8_OS_FUCHSIA) +#include +#endif + +#if defined(ARMV8_OS_ANDROID) +#include +#elif defined(ARMV8_OS_LINUX) +#include +#include +#elif defined(ARMV8_OS_FUCHSIA) +#include +#include +#include +#elif defined(ARMV8_OS_WINDOWS) || defined(X86_WINDOWS) +#include +#elif !defined(_MSC_VER) +#include +#else +#error cpu_features.c CPU feature detection in not defined for your platform +#endif + +#if !defined(CPU_NO_SIMD) && !defined(ARMV8_OS_MACOS) && !defined(ARM_OS_IOS) +static void _cpu_check_features(void); +#endif + +#if defined(ARMV8_OS_ANDROID) || defined(ARMV8_OS_LINUX) || defined(ARMV8_OS_MACOS) || defined(ARMV8_OS_FUCHSIA) || defined(X86_NOT_WINDOWS) +#if !defined(ARMV8_OS_MACOS) +// _cpu_check_features() doesn't need to do anything on mac/arm since all +// features are known at build time, so don't call it. +// Do provide cpu_check_features() (with a no-op implementation) so that we +// don't have to make all callers of it check for mac/arm. +static pthread_once_t cpu_check_inited_once = PTHREAD_ONCE_INIT; +#endif +void ZLIB_INTERNAL cpu_check_features(void) +{ +#if !defined(ARMV8_OS_MACOS) + pthread_once(&cpu_check_inited_once, _cpu_check_features); +#endif +} +#elif defined(ARMV8_OS_WINDOWS) || defined(X86_WINDOWS) +static INIT_ONCE cpu_check_inited_once = INIT_ONCE_STATIC_INIT; +static BOOL CALLBACK _cpu_check_features_forwarder(PINIT_ONCE once, PVOID param, PVOID* context) +{ + _cpu_check_features(); + return TRUE; +} +void ZLIB_INTERNAL cpu_check_features(void) +{ + InitOnceExecuteOnce(&cpu_check_inited_once, _cpu_check_features_forwarder, + NULL, NULL); +} +#endif + +#if (defined(__ARM_NEON__) || defined(__ARM_NEON)) +/* + * iOS@ARM is a special case where we always have NEON but don't check + * for crypto extensions. + */ +#if !defined(ARMV8_OS_MACOS) && !defined(ARM_OS_IOS) +/* + * See http://bit.ly/2CcoEsr for run-time detection of ARM features and also + * crbug.com/931275 for android_getCpuFeatures() use in the Android sandbox. + */ +static void _cpu_check_features(void) +{ +#if defined(ARMV8_OS_ANDROID) && defined(__aarch64__) + uint64_t features = android_getCpuFeatures(); + arm_cpu_enable_crc32 = !!(features & ANDROID_CPU_ARM64_FEATURE_CRC32); + arm_cpu_enable_pmull = !!(features & ANDROID_CPU_ARM64_FEATURE_PMULL); +#elif defined(ARMV8_OS_ANDROID) /* aarch32 */ + uint64_t features = android_getCpuFeatures(); + arm_cpu_enable_crc32 = !!(features & ANDROID_CPU_ARM_FEATURE_CRC32); + arm_cpu_enable_pmull = !!(features & ANDROID_CPU_ARM_FEATURE_PMULL); +#elif defined(ARMV8_OS_LINUX) && defined(__aarch64__) + unsigned long features = getauxval(AT_HWCAP); + arm_cpu_enable_crc32 = !!(features & HWCAP_CRC32); + arm_cpu_enable_pmull = !!(features & HWCAP_PMULL); +#elif defined(ARMV8_OS_LINUX) && (defined(__ARM_NEON) || defined(__ARM_NEON__)) + /* Query HWCAP2 for ARMV8-A SoCs running in aarch32 mode */ + unsigned long features = getauxval(AT_HWCAP2); + arm_cpu_enable_crc32 = !!(features & HWCAP2_CRC32); + arm_cpu_enable_pmull = !!(features & HWCAP2_PMULL); +#elif defined(ARMV8_OS_FUCHSIA) + uint32_t features; + zx_status_t rc = zx_system_get_features(ZX_FEATURE_KIND_CPU, &features); + if (rc != ZX_OK || (features & ZX_ARM64_FEATURE_ISA_ASIMD) == 0) + return; /* Report nothing if ASIMD(NEON) is missing */ + arm_cpu_enable_crc32 = !!(features & ZX_ARM64_FEATURE_ISA_CRC32); + arm_cpu_enable_pmull = !!(features & ZX_ARM64_FEATURE_ISA_PMULL); +#elif defined(ARMV8_OS_WINDOWS) + arm_cpu_enable_crc32 = IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE); + arm_cpu_enable_pmull = IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE); +#endif +} +#endif +#elif defined(X86_NOT_WINDOWS) || defined(X86_WINDOWS) +/* + * iOS@x86 (i.e. emulator) is another special case where we disable + * SIMD optimizations. + */ +#ifndef CPU_NO_SIMD +/* On x86 we simply use a instruction to check the CPU features. + * (i.e. CPUID). + */ +static void _cpu_check_features(void) +{ + int x86_cpu_has_sse2; + int x86_cpu_has_ssse3; + int x86_cpu_has_sse42; + int x86_cpu_has_pclmulqdq; + int abcd[4]; + +#ifdef _MSC_VER + __cpuid(abcd, 1); +#else + __cpuid(1, abcd[0], abcd[1], abcd[2], abcd[3]); +#endif + + x86_cpu_has_sse2 = abcd[3] & 0x4000000; + x86_cpu_has_ssse3 = abcd[2] & 0x000200; + x86_cpu_has_sse42 = abcd[2] & 0x100000; + x86_cpu_has_pclmulqdq = abcd[2] & 0x2; + + x86_cpu_enable_sse2 = x86_cpu_has_sse2; + + x86_cpu_enable_ssse3 = x86_cpu_has_ssse3; + + x86_cpu_enable_simd = x86_cpu_has_sse2 && + x86_cpu_has_sse42 && + x86_cpu_has_pclmulqdq; +} +#endif +#endif +#endif diff --git a/deps/zlib/arm_features.h b/deps/zlib/cpu_features.h similarity index 53% rename from deps/zlib/arm_features.h rename to deps/zlib/cpu_features.h index 09fec259b1c924..c7b15c5597623f 100644 --- a/deps/zlib/arm_features.h +++ b/deps/zlib/cpu_features.h @@ -1,4 +1,4 @@ -/* arm_features.h -- ARM processor features detection. +/* cpu_features.h -- Processor features detection. * * Copyright 2018 The Chromium Authors. All rights reserved. * Use of this source code is governed by a BSD-style license that can be @@ -7,7 +7,12 @@ #include "zlib.h" +/* TODO(cavalcantii): remove checks for x86_flags on deflate. + */ extern int arm_cpu_enable_crc32; extern int arm_cpu_enable_pmull; +extern int x86_cpu_enable_sse2; +extern int x86_cpu_enable_ssse3; +extern int x86_cpu_enable_simd; -void arm_check_features(void); +void cpu_check_features(void); diff --git a/deps/zlib/crc32.c b/deps/zlib/crc32.c index e95b9087351c1a..d4c3248d98415b 100644 --- a/deps/zlib/crc32.c +++ b/deps/zlib/crc32.c @@ -29,13 +29,10 @@ #endif /* MAKECRCH */ #include "deflate.h" -#include "x86.h" +#include "cpu_features.h" #include "zutil.h" /* for STDC and FAR definitions */ -#if defined(CRC32_SIMD_SSE42_PCLMUL) -#include "crc32_simd.h" -#elif defined(CRC32_ARMV8_CRC32) -#include "arm_features.h" +#if defined(CRC32_SIMD_SSE42_PCLMUL) || defined(CRC32_ARMV8_CRC32) #include "crc32_simd.h" #endif @@ -226,7 +223,7 @@ unsigned long ZEXPORT crc32_z(crc, buf, len) */ if (buf == Z_NULL) { if (!len) /* Assume user is calling crc32(0, NULL, 0); */ - x86_check_features(); + cpu_check_features(); return 0UL; } @@ -289,7 +286,7 @@ unsigned long ZEXPORT crc32(crc, buf, len) */ if (buf == Z_NULL) { if (!len) /* Assume user is calling crc32(0, NULL, 0); */ - arm_check_features(); + cpu_check_features(); return 0UL; } @@ -500,25 +497,31 @@ uLong ZEXPORT crc32_combine64(crc1, crc2, len2) ZLIB_INTERNAL void crc_reset(deflate_state *const s) { +#ifdef CRC32_SIMD_SSE42_PCLMUL if (x86_cpu_enable_simd) { crc_fold_init(s); return; } +#endif s->strm->adler = crc32(0L, Z_NULL, 0); } ZLIB_INTERNAL void crc_finalize(deflate_state *const s) { +#ifdef CRC32_SIMD_SSE42_PCLMUL if (x86_cpu_enable_simd) s->strm->adler = crc_fold_512to32(s); +#endif } ZLIB_INTERNAL void copy_with_crc(z_streamp strm, Bytef *dst, long size) { +#ifdef CRC32_SIMD_SSE42_PCLMUL if (x86_cpu_enable_simd) { crc_fold_copy(strm->state, dst, strm->next_in, size); return; } +#endif zmemcpy(dst, strm->next_in, size); strm->adler = crc32(strm->adler, dst, size); } diff --git a/deps/zlib/crc32_simd.c b/deps/zlib/crc32_simd.c index 27481847e97b90..c8e5592f38ef1e 100644 --- a/deps/zlib/crc32_simd.c +++ b/deps/zlib/crc32_simd.c @@ -8,9 +8,6 @@ #include "crc32_simd.h" #if defined(CRC32_SIMD_SSE42_PCLMUL) -#ifndef __GNUC__ -#define __attribute__() -#endif /* * crc32_sse42_simd_(): compute the crc32 of the buffer, where the buffer @@ -24,7 +21,6 @@ #include #include -__attribute__((target("sse4.2,pclmul"))) uint32_t ZLIB_INTERNAL crc32_sse42_simd_( /* SSE4.2+PCLMUL */ const unsigned char *buf, z_size_t len, diff --git a/deps/zlib/crc_folding.c b/deps/zlib/crc_folding.c index 54f4b5c9401089..ee31d4918d6413 100644 --- a/deps/zlib/crc_folding.c +++ b/deps/zlib/crc_folding.c @@ -18,15 +18,13 @@ #include "deflate.h" +#ifdef CRC32_SIMD_SSE42_PCLMUL + #include #include #include #include -#ifndef __GNUC__ -#define __attribute__() -#endif - #define CRC_LOAD(s) \ do { \ __m128i xmm_crc0 = _mm_loadu_si128((__m128i *)s->crc0 + 0);\ @@ -43,7 +41,6 @@ _mm_storeu_si128((__m128i *)s->crc0 + 4, xmm_crc_part);\ } while (0); -__attribute__((target("sse4.2,pclmul"))) ZLIB_INTERNAL void crc_fold_init(deflate_state *const s) { CRC_LOAD(s) @@ -58,7 +55,6 @@ ZLIB_INTERNAL void crc_fold_init(deflate_state *const s) s->strm->adler = 0; } -__attribute__((target("sse4.2,pclmul"))) local void fold_1(deflate_state *const s, __m128i *xmm_crc0, __m128i *xmm_crc1, __m128i *xmm_crc2, __m128i *xmm_crc3) @@ -85,7 +81,6 @@ local void fold_1(deflate_state *const s, *xmm_crc3 = _mm_castps_si128(ps_res); } -__attribute__((target("sse4.2,pclmul"))) local void fold_2(deflate_state *const s, __m128i *xmm_crc0, __m128i *xmm_crc1, __m128i *xmm_crc2, __m128i *xmm_crc3) @@ -120,7 +115,6 @@ local void fold_2(deflate_state *const s, *xmm_crc3 = _mm_castps_si128(ps_res31); } -__attribute__((target("sse4.2,pclmul"))) local void fold_3(deflate_state *const s, __m128i *xmm_crc0, __m128i *xmm_crc1, __m128i *xmm_crc2, __m128i *xmm_crc3) @@ -161,7 +155,6 @@ local void fold_3(deflate_state *const s, *xmm_crc3 = _mm_castps_si128(ps_res32); } -__attribute__((target("sse4.2,pclmul"))) local void fold_4(deflate_state *const s, __m128i *xmm_crc0, __m128i *xmm_crc1, __m128i *xmm_crc2, __m128i *xmm_crc3) @@ -228,7 +221,6 @@ local const unsigned zalign(32) pshufb_shf_table[60] = { 0x0201008f,0x06050403,0x0a090807,0x0e0d0c0b /* shl 1 (16 -15)/shr15*/ }; -__attribute__((target("sse4.2,pclmul"))) local void partial_fold(deflate_state *const s, const size_t len, __m128i *xmm_crc0, __m128i *xmm_crc1, __m128i *xmm_crc2, __m128i *xmm_crc3, @@ -279,7 +271,6 @@ local void partial_fold(deflate_state *const s, const size_t len, *xmm_crc3 = _mm_castps_si128(ps_res); } -__attribute__((target("sse4.2,pclmul"))) ZLIB_INTERNAL void crc_fold_copy(deflate_state *const s, unsigned char *dst, const unsigned char *src, long len) { @@ -294,7 +285,7 @@ ZLIB_INTERNAL void crc_fold_copy(deflate_state *const s, goto partial; } - algn_diff = 0 - (uintptr_t)src & 0xF; + algn_diff = (0 - (uintptr_t)src) & 0xF; if (algn_diff) { xmm_crc_part = _mm_loadu_si128((__m128i *)src); _mm_storeu_si128((__m128i *)dst, xmm_crc_part); @@ -436,7 +427,6 @@ local const unsigned zalign(16) crc_mask2[4] = { 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }; -__attribute__((target("sse4.2,pclmul"))) unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state *const s) { const __m128i xmm_mask = _mm_load_si128((__m128i *)crc_mask); @@ -503,3 +493,5 @@ unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state *const s) return ~crc; CRC_SAVE(s) } + +#endif /* CRC32_SIMD_SSE42_PCLMUL */ diff --git a/deps/zlib/deflate.c b/deps/zlib/deflate.c index 201254ac1b664e..fc7ae45905fff1 100644 --- a/deps/zlib/deflate.c +++ b/deps/zlib/deflate.c @@ -50,7 +50,7 @@ /* @(#) $Id$ */ #include #include "deflate.h" -#include "x86.h" +#include "cpu_features.h" #include "contrib/optimizations/insert_string.h" #if (defined(__ARM_NEON__) || defined(__ARM_NEON)) @@ -60,6 +60,11 @@ #include "crc32_simd.h" #endif +#ifdef FASTEST +/* See http://crbug.com/1113596 */ +#error "FASTEST is not supported in Chromium's zlib." +#endif + const char deflate_copyright[] = " deflate 1.2.11 Copyright 1995-2017 Jean-loup Gailly and Mark Adler "; /* @@ -244,10 +249,8 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, // for all wrapper formats (e.g. RAW, ZLIB, GZIP). // Feature detection is not triggered while using RAW mode (i.e. we never // call crc32() with a NULL buffer). -#if defined(CRC32_ARMV8_CRC32) - arm_check_features(); -#elif defined(CRC32_SIMD_SSE42_PCLMUL) - x86_check_features(); +#if defined(CRC32_ARMV8_CRC32) || defined(CRC32_SIMD_SSE42_PCLMUL) + cpu_check_features(); #endif if (version == Z_NULL || version[0] != my_version[0] || @@ -306,10 +309,9 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, s->w_size = 1 << s->w_bits; s->w_mask = s->w_size - 1; - if (x86_cpu_enable_simd) { + s->hash_bits = memLevel + 7; + if ((x86_cpu_enable_simd || arm_cpu_enable_crc32) && s->hash_bits < 15) { s->hash_bits = 15; - } else { - s->hash_bits = memLevel + 7; } s->hash_size = 1 << s->hash_bits; @@ -319,7 +321,14 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, s->window = (Bytef *) ZALLOC(strm, s->w_size + window_padding, 2*sizeof(Byte)); + /* Avoid use of unitialized values in the window, see crbug.com/1137613 and + * crbug.com/1144420 */ + zmemzero(s->window, (s->w_size + window_padding) * (2 * sizeof(Byte))); s->prev = (Posf *) ZALLOC(strm, s->w_size, sizeof(Pos)); + /* Avoid use of uninitialized value, see: + * https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=11360 + */ + zmemzero(s->prev, s->w_size * sizeof(Pos)); s->head = (Posf *) ZALLOC(strm, s->hash_size, sizeof(Pos)); s->high_water = 0; /* nothing written to s->window yet */ @@ -1211,7 +1220,7 @@ ZLIB_INTERNAL unsigned deflate_read_buf(strm, buf, size) #ifdef GZIP if (strm->state->wrap == 2) copy_with_crc(strm, buf, len); - else + else #endif { zmemcpy(buf, strm->next_in, len); @@ -1344,7 +1353,16 @@ local uInt longest_match(s, cur_match) * necessary to put more guard bytes at the end of the window, or * to check more often for insufficient lookahead. */ - Assert(scan[2] == match[2], "scan[2]?"); + if (!x86_cpu_enable_simd && !arm_cpu_enable_crc32) { + Assert(scan[2] == match[2], "scan[2]?"); + } else { + /* When using CRC hashing, scan[2] and match[2] may mismatch, but in + * that case at least one of the other hashed bytes will mismatch + * also. Bytes 0 and 1 were already checked above, and we know there + * are at least four bytes to check otherwise the mismatch would have + * been found by the scan_end comparison above, so: */ + Assert(scan[2] == match[2] || scan[3] != match[3], "scan[2]??"); + } scan++, match++; do { } while (*(ushf*)(scan+=2) == *(ushf*)(match+=2) && @@ -1375,7 +1393,16 @@ local uInt longest_match(s, cur_match) * the hash keys are equal and that HASH_BITS >= 8. */ scan += 2, match++; - Assert(*scan == *match, "match[2]?"); + if (!x86_cpu_enable_simd && !arm_cpu_enable_crc32) { + Assert(*scan == *match, "match[2]?"); + } else { + /* When using CRC hashing, scan[2] and match[2] may mismatch, but in + * that case at least one of the other hashed bytes will mismatch + * also. Bytes 0 and 1 were already checked above, and we know there + * are at least four bytes to check otherwise the mismatch would have + * been found by the scan_end comparison above, so: */ + Assert(*scan == *match || scan[1] != match[1], "match[2]??"); + } /* We check for insufficient lookahead only every 8th comparison; * the 256th check will be made at strstart+258. @@ -1519,11 +1546,12 @@ local void fill_window_c(deflate_state *s); local void fill_window(deflate_state *s) { +#ifdef DEFLATE_FILL_WINDOW_SSE2 if (x86_cpu_enable_simd) { fill_window_sse(s); return; } - +#endif fill_window_c(s); } @@ -2035,7 +2063,13 @@ local block_state deflate_slow(s, flush) uInt max_insert = s->strstart + s->lookahead - MIN_MATCH; /* Do not insert strings in hash table beyond this. */ - check_match(s, s->strstart-1, s->prev_match, s->prev_length); + if (s->prev_match == -1) { + /* The window has slid one byte past the previous match, + * so the first byte cannot be compared. */ + check_match(s, s->strstart, s->prev_match+1, s->prev_length-1); + } else { + check_match(s, s->strstart-1, s->prev_match, s->prev_length); + } _tr_tally_dist(s, s->strstart -1 - s->prev_match, s->prev_length - MIN_MATCH, bflush); diff --git a/deps/zlib/fill_window_sse.c b/deps/zlib/fill_window_sse.c index e19154bc14d238..a841c99904baee 100644 --- a/deps/zlib/fill_window_sse.c +++ b/deps/zlib/fill_window_sse.c @@ -9,10 +9,10 @@ * For conditions of distribution and use, see copyright notice in zlib.h */ -#pragma GCC target ("sse2") -#include #include "deflate.h" +#ifdef DEFLATE_FILL_WINDOW_SSE2 + #define UPDATE_HASH(s,h,i) \ {\ if (s->level < 6) { \ @@ -29,6 +29,8 @@ extern int deflate_read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); +#include + void fill_window_sse(deflate_state *s) { const __m128i xmm_wsize = _mm_set1_epi16(s->w_size); @@ -176,3 +178,5 @@ void fill_window_sse(deflate_state *s) Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD, "not enough room for search"); } + +#endif /* DEFLATE_FILL_WINDOW_SSE2 */ diff --git a/deps/zlib/google/BUILD.gn b/deps/zlib/google/BUILD.gn index 4024836205f64c..c29e8927806235 100644 --- a/deps/zlib/google/BUILD.gn +++ b/deps/zlib/google/BUILD.gn @@ -28,10 +28,10 @@ if (build_with_chromium) { "compression_utils.h", ] deps = [ - ":compression_utils_portable", "//base", "//third_party/zlib", ] + public_deps = [ ":compression_utils_portable" ] } } @@ -42,7 +42,5 @@ static_library("compression_utils_portable") { "compression_utils_portable.cc", "compression_utils_portable.h", ] - deps = [ - "//third_party/zlib", - ] + public_deps = [ "//third_party/zlib" ] } diff --git a/deps/zlib/google/DEPS b/deps/zlib/google/DEPS index 144fbd149295b3..03f2cb950b9517 100644 --- a/deps/zlib/google/DEPS +++ b/deps/zlib/google/DEPS @@ -2,4 +2,5 @@ include_rules = [ '+base', '+build', '+testing', + "+third_party/zlib/zlib.h", ] diff --git a/deps/zlib/google/OWNERS b/deps/zlib/google/OWNERS index 1ca25314631124..411670ca13aa17 100644 --- a/deps/zlib/google/OWNERS +++ b/deps/zlib/google/OWNERS @@ -1,5 +1,8 @@ +fdegros@chromium.org +noel@chromium.org satorux@chromium.org # compression_utils* asvitkine@chromium.org isherman@chromium.org +cavalcantii@chromium.org diff --git a/deps/zlib/google/compression_utils.cc b/deps/zlib/google/compression_utils.cc index 9f63a840167d47..781c805d0a950b 100644 --- a/deps/zlib/google/compression_utils.cc +++ b/deps/zlib/google/compression_utils.cc @@ -5,16 +5,15 @@ #include "third_party/zlib/google/compression_utils.h" #include "base/bit_cast.h" -#include "base/logging.h" +#include "base/check_op.h" #include "base/process/memory.h" -#include "base/strings/string_piece.h" #include "base/sys_byteorder.h" #include "third_party/zlib/google/compression_utils_portable.h" namespace compression { -bool GzipCompress(base::StringPiece input, +bool GzipCompress(base::span input, char* output_buffer, size_t output_buffer_size, size_t* compressed_size, @@ -35,7 +34,11 @@ bool GzipCompress(base::StringPiece input, return true; } -bool GzipCompress(base::StringPiece input, std::string* output) { +bool GzipCompress(base::span input, std::string* output) { + return GzipCompress(base::as_bytes(input), output); +} + +bool GzipCompress(base::span input, std::string* output) { // Not using std::vector<> because allocation failures are recoverable, // which is hidden by std::vector<>. static_assert(sizeof(Bytef) == 1, ""); @@ -87,30 +90,44 @@ bool GzipUncompress(const std::string& input, std::string* output) { return false; } -bool GzipUncompress(base::StringPiece input, base::StringPiece output) { +bool GzipUncompress(base::span input, + base::span output) { + return GzipUncompress(base::as_bytes(input), base::as_bytes(output)); +} + +bool GzipUncompress(base::span input, + base::span output) { uLongf uncompressed_size = GetUncompressedSize(input); if (uncompressed_size > output.size()) return false; return zlib_internal::GzipUncompressHelper( bit_cast(output.data()), &uncompressed_size, bit_cast(input.data()), - static_cast(input.length())) == Z_OK; + static_cast(input.size())) == Z_OK; } -bool GzipUncompress(base::StringPiece input, std::string* output) { +bool GzipUncompress(base::span input, std::string* output) { + return GzipUncompress(base::as_bytes(input), output); +} + +bool GzipUncompress(base::span input, std::string* output) { // Disallow in-place usage, i.e., |input| using |*output| as underlying data. - DCHECK_NE(input.data(), output->data()); + DCHECK_NE(reinterpret_cast(input.data()), output->data()); uLongf uncompressed_size = GetUncompressedSize(input); output->resize(uncompressed_size); return zlib_internal::GzipUncompressHelper( bit_cast(output->data()), &uncompressed_size, bit_cast(input.data()), - static_cast(input.length())) == Z_OK; + static_cast(input.size())) == Z_OK; +} + +uint32_t GetUncompressedSize(base::span compressed_data) { + return GetUncompressedSize(base::as_bytes(compressed_data)); } -uint32_t GetUncompressedSize(base::StringPiece compressed_data) { +uint32_t GetUncompressedSize(base::span compressed_data) { return zlib_internal::GetGzipUncompressedSize( - bit_cast(compressed_data.data()), compressed_data.length()); + bit_cast(compressed_data.data()), compressed_data.size()); } } // namespace compression diff --git a/deps/zlib/google/compression_utils.h b/deps/zlib/google/compression_utils.h index 516220719ff43c..cca47be1efef6a 100644 --- a/deps/zlib/google/compression_utils.h +++ b/deps/zlib/google/compression_utils.h @@ -7,7 +7,7 @@ #include -#include "base/strings/string_piece.h" +#include "base/containers/span.h" namespace compression { @@ -18,7 +18,7 @@ namespace compression { // |malloc_fn| and |free_fn| are pointers to malloc() and free()-like functions, // or nullptr to use the standard ones. // Returns true for success. -bool GzipCompress(base::StringPiece input, +bool GzipCompress(base::span input, char* output_buffer, size_t output_buffer_size, size_t* compressed_size, @@ -29,27 +29,41 @@ bool GzipCompress(base::StringPiece input, // |input| and |output| are allowed to point to the same string (in-place // operation). // Returns true for success. -bool GzipCompress(base::StringPiece input, std::string* output); +bool GzipCompress(base::span input, std::string* output); + +// Like the above method, but using uint8_t instead. +bool GzipCompress(base::span input, std::string* output); // Uncompresses the data in |input| using gzip, storing the result in |output|. // |input| and |output| are allowed to be the same string (in-place operation). // Returns true for success. bool GzipUncompress(const std::string& input, std::string* output); -// Like the above method, but uses base::StringPiece to avoid allocations if +// Like the above method, but uses base::span to avoid allocations if // needed. |output|'s size must be at least as large as the return value from // GetUncompressedSize. // Returns true for success. -bool GzipUncompress(base::StringPiece input, base::StringPiece output); +bool GzipUncompress(base::span input, + base::span output); + +// Like the above method, but using uint8_t instead. +bool GzipUncompress(base::span input, + base::span output); // Uncompresses the data in |input| using gzip, and writes the results to // |output|, which must NOT be the underlying string of |input|, and is resized // if necessary. // Returns true for success. -bool GzipUncompress(base::StringPiece input, std::string* output); +bool GzipUncompress(base::span input, std::string* output); + +// Like the above method, but using uint8_t instead. +bool GzipUncompress(base::span input, std::string* output); // Returns the uncompressed size from GZIP-compressed |compressed_data|. -uint32_t GetUncompressedSize(base::StringPiece compressed_data); +uint32_t GetUncompressedSize(base::span compressed_data); + +// Like the above method, but using uint8_t instead. +uint32_t GetUncompressedSize(base::span compressed_data); } // namespace compression diff --git a/deps/zlib/google/compression_utils_portable.cc b/deps/zlib/google/compression_utils_portable.cc index 191e349e31ad30..331e41e1257253 100644 --- a/deps/zlib/google/compression_utils_portable.cc +++ b/deps/zlib/google/compression_utils_portable.cc @@ -5,7 +5,7 @@ * found in the Chromium source repository LICENSE file. */ -#include "third_party/zlib/google/compression_utils_portable.h" +#include "compression_utils_portable.h" #include #include @@ -84,7 +84,7 @@ int CompressHelper(WrapperType wrapper_type, int compression_level, void* (*malloc_fn)(size_t), void (*free_fn)(void*)) { - if (compression_level < 1 || compression_level > 9) { + if (compression_level < 0 || compression_level > 9) { compression_level = Z_DEFAULT_COMPRESSION; } diff --git a/deps/zlib/google/compression_utils_portable.h b/deps/zlib/google/compression_utils_portable.h index cd004e86cf515c..c1f377571fbba0 100644 --- a/deps/zlib/google/compression_utils_portable.h +++ b/deps/zlib/google/compression_utils_portable.h @@ -9,10 +9,14 @@ #include +/* TODO(cavalcantii): remove support for Chromium ever building with a system + * zlib. + */ #if defined(USE_SYSTEM_ZLIB) #include +/* AOSP build requires relative paths. */ #else -#include "third_party/zlib/zlib.h" +#include "zlib.h" #endif namespace zlib_internal { diff --git a/deps/zlib/google/compression_utils_unittest.cc b/deps/zlib/google/compression_utils_unittest.cc index b0e04b8c973633..415b9ab9228928 100644 --- a/deps/zlib/google/compression_utils_unittest.cc +++ b/deps/zlib/google/compression_utils_unittest.cc @@ -9,8 +9,7 @@ #include -#include "base/logging.h" -#include "base/stl_util.h" +#include "base/cxx17_backports.h" #include "testing/gtest/include/gtest/gtest.h" namespace compression { @@ -55,13 +54,9 @@ TEST(CompressionUtilsTest, GzipUncompression) { EXPECT_EQ(golden_data, uncompressed_data); } -TEST(CompressionUtilsTest, GzipUncompressionFromStringPieceToString) { - base::StringPiece compressed_data( - reinterpret_cast(kCompressedData), - base::size(kCompressedData)); - +TEST(CompressionUtilsTest, GzipUncompressionFromSpanToString) { std::string uncompressed_data; - EXPECT_TRUE(GzipUncompress(compressed_data, &uncompressed_data)); + EXPECT_TRUE(GzipUncompress(kCompressedData, &uncompressed_data)); std::string golden_data(reinterpret_cast(kData), base::size(kData)); diff --git a/deps/zlib/google/test/data/README.md b/deps/zlib/google/test/data/README.md new file mode 100644 index 00000000000000..c76648fb08a4ef --- /dev/null +++ b/deps/zlib/google/test/data/README.md @@ -0,0 +1,15 @@ +## test\_posix\_permissions.zip +Rebuild this zip by running: +``` +rm test_posix_permissions.zip && +mkdir z && +cd z && +touch 0.txt 1.txt 2.txt 3.txt && +chmod a+x 0.txt && +chmod o+x 1.txt && +chmod u+x 2.txt && +zip test_posix_permissions.zip * && +mv test_posix_permissions.zip .. && +cd .. && +rm -r z +``` diff --git a/deps/zlib/google/test/data/test_posix_permissions.zip b/deps/zlib/google/test/data/test_posix_permissions.zip new file mode 100644 index 0000000000000000000000000000000000000000..a058ba17445c37a6a55f8d2b99cf8ef0aad4e036 GIT binary patch literal 574 zcmWIWW@h1H0D+cVtq?E+O0Y7>Fc|2SRFs5IHQ&}+5?S(h6`mzJ!z~1 k8U+m_%8XjlxEE*?G`w&b#mWX$!@vTBI*beqe?S}t00Hb}KmY&$ literal 0 HcmV?d00001 diff --git a/deps/zlib/google/zip.cc b/deps/zlib/google/zip.cc index 907e5da72e974d..7c46718808d27f 100644 --- a/deps/zlib/google/zip.cc +++ b/deps/zlib/google/zip.cc @@ -4,7 +4,6 @@ #include "third_party/zlib/google/zip.h" -#include #include #include @@ -30,10 +29,6 @@ bool ExcludeNoFilesFilter(const base::FilePath& file_path) { return true; } -bool ExcludeHiddenFilesFilter(const base::FilePath& file_path) { - return !IsHiddenFile(file_path); -} - // Creates a directory at |extract_dir|/|entry_path|, including any parents. bool CreateDirectory(const base::FilePath& extract_dir, const base::FilePath& entry_path) { @@ -50,126 +45,128 @@ std::unique_ptr CreateFilePathWriterDelegate( class DirectFileAccessor : public FileAccessor { public: - explicit DirectFileAccessor(base::FilePath src_dir) : src_dir_(src_dir) {} + explicit DirectFileAccessor(base::FilePath src_dir) + : src_dir_(std::move(src_dir)) {} + ~DirectFileAccessor() override = default; - std::vector OpenFilesForReading( - const std::vector& paths) override { - std::vector files; - for (const auto& path : paths) { - base::File file; - if (base::PathExists(path) && !base::DirectoryExists(path)) { - file = base::File(path, base::File::FLAG_OPEN | base::File::FLAG_READ); + bool Open(const Paths paths, std::vector* const files) override { + DCHECK(files); + files->reserve(files->size() + paths.size()); + + for (const base::FilePath& path : paths) { + DCHECK(!path.IsAbsolute()); + const base::FilePath absolute_path = src_dir_.Append(path); + if (base::DirectoryExists(absolute_path)) { + files->emplace_back(); + LOG(ERROR) << "Cannot open '" << path << "': It is a directory"; + } else { + files->emplace_back(absolute_path, + base::File::FLAG_OPEN | base::File::FLAG_READ); + LOG_IF(ERROR, !files->back().IsValid()) + << "Cannot open '" << path << "'"; } - files.push_back(std::move(file)); } - return files; - } - bool DirectoryExists(const base::FilePath& file) override { - return base::DirectoryExists(file); + return true; } - std::vector ListDirectoryContent( - const base::FilePath& dir) override { - std::vector files; + bool List(const base::FilePath& path, + std::vector* const files, + std::vector* const subdirs) override { + DCHECK(!path.IsAbsolute()); + DCHECK(files); + DCHECK(subdirs); + base::FileEnumerator file_enumerator( - dir, false /* recursive */, + src_dir_.Append(path), false /* recursive */, base::FileEnumerator::FILES | base::FileEnumerator::DIRECTORIES); - for (base::FilePath path = file_enumerator.Next(); !path.value().empty(); - path = file_enumerator.Next()) { - files.push_back(DirectoryContentEntry(path, base::DirectoryExists(path))); + + while (!file_enumerator.Next().empty()) { + const base::FileEnumerator::FileInfo info = file_enumerator.GetInfo(); + (info.IsDirectory() ? subdirs : files) + ->push_back(path.Append(info.GetName())); } - return files; + + return true; } - base::Time GetLastModifiedTime(const base::FilePath& path) override { + bool GetInfo(const base::FilePath& path, Info* const info) override { + DCHECK(!path.IsAbsolute()); + DCHECK(info); + base::File::Info file_info; - if (!base::GetFileInfo(path, &file_info)) { - LOG(ERROR) << "Failed to retrieve file modification time for " - << path.value(); + if (!base::GetFileInfo(src_dir_.Append(path), &file_info)) { + LOG(ERROR) << "Cannot get info of '" << path << "'"; + return false; } - return file_info.last_modified; + + info->is_directory = file_info.is_directory; + info->last_modified = file_info.last_modified; + + return true; } private: - base::FilePath src_dir_; - - DISALLOW_COPY_AND_ASSIGN(DirectFileAccessor); + const base::FilePath src_dir_; }; } // namespace -ZipParams::ZipParams(const base::FilePath& src_dir, - const base::FilePath& dest_file) - : src_dir_(src_dir), - dest_file_(dest_file), - file_accessor_(new DirectFileAccessor(src_dir)) {} - -#if defined(OS_POSIX) -// Does not take ownership of |fd|. -ZipParams::ZipParams(const base::FilePath& src_dir, int dest_fd) - : src_dir_(src_dir), - dest_fd_(dest_fd), - file_accessor_(new DirectFileAccessor(src_dir)) {} -#endif +std::ostream& operator<<(std::ostream& out, const Progress& progress) { + return out << progress.bytes << " bytes, " << progress.files << " files, " + << progress.directories << " dirs, " << progress.errors + << " errors"; +} bool Zip(const ZipParams& params) { - // Using a pointer to avoid copies of a potentially large array. - const std::vector* files_to_add = ¶ms.files_to_zip(); - std::vector all_files; - if (files_to_add->empty()) { - // Include all files from the src_dir (modulo the src_dir itself and - // filtered and hidden files). - - files_to_add = &all_files; - // Using a list so we can call push_back while iterating. - std::list entries; - entries.push_back(FileAccessor::DirectoryContentEntry( - params.src_dir(), true /* is directory*/)); - const FilterCallback& filter_callback = params.filter_callback(); - for (auto iter = entries.begin(); iter != entries.end(); ++iter) { - const base::FilePath& entry_path = iter->path; - if (iter != entries.begin() && // Don't filter the root dir. - ((!params.include_hidden_files() && IsHiddenFile(entry_path)) || - (filter_callback && !filter_callback.Run(entry_path)))) { - continue; - } - - if (iter != entries.begin()) { // Exclude the root dir from the ZIP file. - // Make the path relative for AddEntryToZip. - base::FilePath relative_path; - bool success = - params.src_dir().AppendRelativePath(entry_path, &relative_path); - DCHECK(success); - all_files.push_back(relative_path); - } - - if (iter->is_directory) { - std::vector subentries = - params.file_accessor()->ListDirectoryContent(entry_path); - entries.insert(entries.end(), subentries.begin(), subentries.end()); - } - } - } + DirectFileAccessor default_accessor(params.src_dir); + FileAccessor* const file_accessor = params.file_accessor ?: &default_accessor; std::unique_ptr zip_writer; + #if defined(OS_POSIX) - if (params.dest_fd() != base::kInvalidPlatformFile) { - DCHECK(params.dest_file().empty()); - zip_writer = internal::ZipWriter::CreateWithFd( - params.dest_fd(), params.src_dir(), params.file_accessor()); + if (params.dest_fd != base::kInvalidPlatformFile) { + DCHECK(params.dest_file.empty()); + zip_writer = + internal::ZipWriter::CreateWithFd(params.dest_fd, file_accessor); if (!zip_writer) return false; } #endif + if (!zip_writer) { - zip_writer = internal::ZipWriter::Create( - params.dest_file(), params.src_dir(), params.file_accessor()); + zip_writer = internal::ZipWriter::Create(params.dest_file, file_accessor); if (!zip_writer) return false; } - return zip_writer->WriteEntries(*files_to_add); + + zip_writer->SetProgressCallback(params.progress_callback, + params.progress_period); + zip_writer->SetRecursive(params.recursive); + zip_writer->ContinueOnError(params.continue_on_error); + + if (!params.include_hidden_files || params.filter_callback) + zip_writer->SetFilterCallback(base::BindRepeating( + [](const ZipParams* const params, const base::FilePath& path) -> bool { + return (params->include_hidden_files || !IsHiddenFile(path)) && + (!params->filter_callback || + params->filter_callback.Run(params->src_dir.Append(path))); + }, + ¶ms)); + + if (params.src_files.empty()) { + // No source items are specified. Zip the entire source directory. + zip_writer->SetRecursive(true); + if (!zip_writer->AddDirectoryContents(base::FilePath())) + return false; + } else { + // Only zip the specified source items. + if (!zip_writer->AddMixedEntries(params.src_files)) + return false; + } + + return zip_writer->Close(); } bool Unzip(const base::FilePath& src_file, const base::FilePath& dest_dir) { @@ -179,28 +176,29 @@ bool Unzip(const base::FilePath& src_file, const base::FilePath& dest_dir) { bool UnzipWithFilterCallback(const base::FilePath& src_file, const base::FilePath& dest_dir, - const FilterCallback& filter_cb, + FilterCallback filter_cb, bool log_skipped_files) { base::File file(src_file, base::File::FLAG_OPEN | base::File::FLAG_READ); if (!file.IsValid()) { - DLOG(WARNING) << "Failed to open " << src_file.value(); + DLOG(WARNING) << "Cannot open '" << src_file << "'"; return false; } + return UnzipWithFilterAndWriters( file.GetPlatformFile(), base::BindRepeating(&CreateFilePathWriterDelegate, dest_dir), - base::BindRepeating(&CreateDirectory, dest_dir), filter_cb, + base::BindRepeating(&CreateDirectory, dest_dir), std::move(filter_cb), log_skipped_files); } bool UnzipWithFilterAndWriters(const base::PlatformFile& src_file, - const WriterFactory& writer_factory, - const DirectoryCreator& directory_creator, - const FilterCallback& filter_cb, + WriterFactory writer_factory, + DirectoryCreator directory_creator, + FilterCallback filter_cb, bool log_skipped_files) { ZipReader reader; if (!reader.OpenFromPlatformFile(src_file)) { - DLOG(WARNING) << "Failed to open src_file " << src_file; + DLOG(WARNING) << "Cannot open '" << src_file << "'"; return false; } while (reader.HasMore()) { @@ -239,32 +237,29 @@ bool UnzipWithFilterAndWriters(const base::PlatformFile& src_file, bool ZipWithFilterCallback(const base::FilePath& src_dir, const base::FilePath& dest_file, - const FilterCallback& filter_cb) { + FilterCallback filter_cb) { DCHECK(base::DirectoryExists(src_dir)); - ZipParams params(src_dir, dest_file); - params.set_filter_callback(filter_cb); - return Zip(params); + return Zip({.src_dir = src_dir, + .dest_file = dest_file, + .filter_callback = std::move(filter_cb)}); } -bool Zip(const base::FilePath& src_dir, const base::FilePath& dest_file, +bool Zip(const base::FilePath& src_dir, + const base::FilePath& dest_file, bool include_hidden_files) { - if (include_hidden_files) { - return ZipWithFilterCallback(src_dir, dest_file, - base::BindRepeating(&ExcludeNoFilesFilter)); - } else { - return ZipWithFilterCallback( - src_dir, dest_file, base::BindRepeating(&ExcludeHiddenFilesFilter)); - } + return Zip({.src_dir = src_dir, + .dest_file = dest_file, + .include_hidden_files = include_hidden_files}); } #if defined(OS_POSIX) bool ZipFiles(const base::FilePath& src_dir, - const std::vector& src_relative_paths, + Paths src_relative_paths, int dest_fd) { DCHECK(base::DirectoryExists(src_dir)); - ZipParams params(src_dir, dest_fd); - params.set_files_to_zip(src_relative_paths); - return Zip(params); + return Zip({.src_dir = src_dir, + .dest_fd = dest_fd, + .src_files = src_relative_paths}); } #endif // defined(OS_POSIX) diff --git a/deps/zlib/google/zip.h b/deps/zlib/google/zip.h index 4f64a8aca883a8..ecaecb1dc9663c 100644 --- a/deps/zlib/google/zip.h +++ b/deps/zlib/google/zip.h @@ -5,9 +5,13 @@ #ifndef THIRD_PARTY_ZLIB_GOOGLE_ZIP_H_ #define THIRD_PARTY_ZLIB_GOOGLE_ZIP_H_ +#include +#include +#include #include #include "base/callback.h" +#include "base/containers/span.h" #include "base/files/file_path.h" #include "base/files/platform_file.h" #include "base/time/time.h" @@ -21,99 +25,118 @@ namespace zip { class WriterDelegate; +// Paths passed as span to avoid copying them. +using Paths = base::span; + // Abstraction for file access operation required by Zip(). +// // Can be passed to the ZipParams for providing custom access to the files, // for example over IPC. -// If none is provided, the files are accessed directly. -// All parameters paths are expected to be absolute. +// +// All parameters paths are expected to be relative to the source directory. class FileAccessor { public: virtual ~FileAccessor() = default; - struct DirectoryContentEntry { - DirectoryContentEntry(const base::FilePath& path, bool is_directory) - : path(path), is_directory(is_directory) {} - base::FilePath path; + struct Info { bool is_directory = false; + base::Time last_modified; }; // Opens files specified in |paths|. // Directories should be mapped to invalid files. - virtual std::vector OpenFilesForReading( - const std::vector& paths) = 0; + virtual bool Open(Paths paths, std::vector* files) = 0; + + // Lists contents of a directory at |path|. + virtual bool List(const base::FilePath& path, + std::vector* files, + std::vector* subdirs) = 0; - virtual bool DirectoryExists(const base::FilePath& path) = 0; - virtual std::vector ListDirectoryContent( - const base::FilePath& dir_path) = 0; - virtual base::Time GetLastModifiedTime(const base::FilePath& path) = 0; + // Gets info about a file or directory. + virtual bool GetInfo(const base::FilePath& path, Info* info) = 0; }; -class ZipParams { - public: - ZipParams(const base::FilePath& src_dir, const base::FilePath& dest_file); -#if defined(OS_POSIX) - // Does not take ownership of |dest_fd|. - ZipParams(const base::FilePath& src_dir, int dest_fd); +// Progress of a ZIP creation operation. +struct Progress { + // Total number of bytes read from files getting zipped so far. + std::int64_t bytes = 0; - int dest_fd() const { return dest_fd_; } -#endif + // Number of file entries added to the ZIP so far. + // A file entry is added after its bytes have been processed. + int files = 0; - const base::FilePath& src_dir() const { return src_dir_; } - - const base::FilePath& dest_file() const { return dest_file_; } - - // Restricts the files actually zipped to the paths listed in - // |src_relative_paths|. They must be relative to the |src_dir| passed in the - // constructor and will be used as the file names in the created zip file. All - // source paths must be under |src_dir| in the file system hierarchy. - void set_files_to_zip(const std::vector& src_relative_paths) { - src_files_ = src_relative_paths; - } - const std::vector& files_to_zip() const { return src_files_; } - - using FilterCallback = base::RepeatingCallback; - void set_filter_callback(FilterCallback filter_callback) { - filter_callback_ = filter_callback; - } - const FilterCallback& filter_callback() const { return filter_callback_; } - - void set_include_hidden_files(bool include_hidden_files) { - include_hidden_files_ = include_hidden_files; - } - bool include_hidden_files() const { return include_hidden_files_; } - - // Sets a custom file accessor for file operations. Default is to directly - // access the files (with fopen and the rest). - // Useful in cases where running in a sandbox process and file access has to - // go through IPC, for example. - void set_file_accessor(std::unique_ptr file_accessor) { - file_accessor_ = std::move(file_accessor); - } - FileAccessor* file_accessor() const { return file_accessor_.get(); } - - private: - base::FilePath src_dir_; - - base::FilePath dest_file_; -#if defined(OS_POSIX) - int dest_fd_ = base::kInvalidPlatformFile; -#endif + // Number of directory entries added to the ZIP so far. + // A directory entry is added before items in it. + int directories = 0; + + // Number of errors encountered so far (files that cannot be opened, + // directories that cannot be listed). + int errors = 0; +}; - // The relative paths to the files that should be included in the zip file. If - // this is empty, all files in |src_dir_| are included. - std::vector src_files_; +// Prints Progress to output stream. +std::ostream& operator<<(std::ostream& out, const Progress& progress); - // Filter used to exclude files from the ZIP file. Only effective when - // |src_files_| is empty. - FilterCallback filter_callback_; +// Callback reporting the progress of a ZIP creation operation. +// +// This callback returns a boolean indicating whether the ZIP creation operation +// should continue. If it returns false once, then the ZIP creation operation is +// immediately cancelled and the callback won't be called again. +using ProgressCallback = base::RepeatingCallback; - // Whether hidden files should be included in the ZIP file. Only effective - // when |src_files_| is empty. - bool include_hidden_files_ = true; +using FilterCallback = base::RepeatingCallback; + +// ZIP creation parameters and options. +struct ZipParams { + // Source directory. Ignored if |file_accessor| is set. + base::FilePath src_dir; + + // Abstraction around file system access used to read files. + // If left null, an implementation that accesses files directly is used. + FileAccessor* file_accessor = nullptr; // Not owned - // Abstraction around file system access used to read files. An implementation - // that accesses files directly is provided by default. - std::unique_ptr file_accessor_; + // Destination file path. + // Either dest_file or dest_fd should be set, but not both. + base::FilePath dest_file; + +#if defined(OS_POSIX) + // Destination file passed a file descriptor. + // Either dest_file or dest_fd should be set, but not both. + int dest_fd = base::kInvalidPlatformFile; +#endif + + // The relative paths to the files and directories that should be included in + // the ZIP file. If this is empty, the whole contents of |src_dir| are + // included. + // + // These paths must be relative to |src_dir| and will be used as the file + // names in the created ZIP file. All files must be under |src_dir| in the + // file system hierarchy. + // + // All the paths in |src_files| are included in the created ZIP file, + // irrespective of |include_hidden_files| and |filter_callback|. + Paths src_files; + + // Filter used to exclude files from the ZIP file. This is only taken in + // account when recursively adding subdirectory contents. + FilterCallback filter_callback; + + // Optional progress reporting callback. + ProgressCallback progress_callback; + + // Progress reporting period. The final callback is always called when the ZIP + // creation operation completes. + base::TimeDelta progress_period; + + // Should add hidden files? This is only taken in account when recursively + // adding subdirectory contents. + bool include_hidden_files = true; + + // Should recursively add subdirectory contents? + bool recursive = false; + + // Should ignore errors when discovering files and zipping them? + bool continue_on_error = false; }; // Zip files specified into a ZIP archives. The source files and ZIP destination @@ -125,15 +148,15 @@ bool Zip(const ZipParams& params); // of src_dir will be at the root level of the created zip. For each file in // src_dir, include it only if the callback |filter_cb| returns true. Otherwise // omit it. -using FilterCallback = base::RepeatingCallback; bool ZipWithFilterCallback(const base::FilePath& src_dir, const base::FilePath& dest_file, - const FilterCallback& filter_cb); + FilterCallback filter_cb); // Convenience method for callers who don't need to set up the filter callback. // If |include_hidden_files| is true, files starting with "." are included. // Otherwise they are omitted. -bool Zip(const base::FilePath& src_dir, const base::FilePath& dest_file, +bool Zip(const base::FilePath& src_dir, + const base::FilePath& dest_file, bool include_hidden_files); #if defined(OS_POSIX) @@ -143,7 +166,7 @@ bool Zip(const base::FilePath& src_dir, const base::FilePath& dest_file, // file names in the created zip file. All source paths must be under |src_dir| // in the file system hierarchy. bool ZipFiles(const base::FilePath& src_dir, - const std::vector& src_relative_paths, + Paths src_relative_paths, int dest_fd); #endif // defined(OS_POSIX) @@ -152,10 +175,9 @@ bool ZipFiles(const base::FilePath& src_dir, // returns true. Otherwise omit it. // If |log_skipped_files| is true, files skipped during extraction are printed // to debug log. -using FilterCallback = base::RepeatingCallback; bool UnzipWithFilterCallback(const base::FilePath& zip_file, const base::FilePath& dest_dir, - const FilterCallback& filter_cb, + FilterCallback filter_cb, bool log_skipped_files); // Unzip the contents of zip_file, using the writers provided by writer_factory. @@ -168,9 +190,9 @@ typedef base::RepeatingCallback( WriterFactory; typedef base::RepeatingCallback DirectoryCreator; bool UnzipWithFilterAndWriters(const base::PlatformFile& zip_file, - const WriterFactory& writer_factory, - const DirectoryCreator& directory_creator, - const FilterCallback& filter_cb, + WriterFactory writer_factory, + DirectoryCreator directory_creator, + FilterCallback filter_cb, bool log_skipped_files); // Unzip the contents of zip_file into dest_dir. diff --git a/deps/zlib/google/zip_internal.cc b/deps/zlib/google/zip_internal.cc index 314740f5447c03..00e9eefe6c206b 100644 --- a/deps/zlib/google/zip_internal.cc +++ b/deps/zlib/google/zip_internal.cc @@ -5,10 +5,16 @@ #include "third_party/zlib/google/zip_internal.h" #include +#include #include +#include +#include "base/files/file_path.h" #include "base/logging.h" +#include "base/no_destructor.h" +#include "base/notreached.h" +#include "base/strings/string_util.h" #include "base/strings/utf_string_conversions.h" #if defined(USE_SYSTEM_MINIZIP) @@ -34,9 +40,9 @@ typedef struct { } WIN32FILE_IOWIN; // This function is derived from third_party/minizip/iowin32.c. -// Its only difference is that it treats the char* as UTF8 and +// Its only difference is that it treats the filename as UTF-8 and // uses the Unicode version of CreateFile. -void* ZipOpenFunc(void *opaque, const char* filename, int mode) { +void* ZipOpenFunc(void* opaque, const void* filename, int mode) { DWORD desired_access = 0, creation_disposition = 0; DWORD share_mode = 0, flags_and_attributes = 0; HANDLE file = 0; @@ -54,10 +60,11 @@ void* ZipOpenFunc(void *opaque, const char* filename, int mode) { creation_disposition = CREATE_ALWAYS; } - base::string16 filename16 = base::UTF8ToUTF16(filename); - if ((filename != NULL) && (desired_access != 0)) { - file = CreateFile(filename16.c_str(), desired_access, share_mode, - NULL, creation_disposition, flags_and_attributes, NULL); + if (filename != nullptr && desired_access != 0) { + file = CreateFileW( + base::UTF8ToWide(static_cast(filename)).c_str(), + desired_access, share_mode, nullptr, creation_disposition, + flags_and_attributes, nullptr); } if (file == INVALID_HANDLE_VALUE) @@ -77,11 +84,11 @@ void* ZipOpenFunc(void *opaque, const char* filename, int mode) { } #endif -#if defined(OS_POSIX) +#if defined(OS_POSIX) || defined(OS_FUCHSIA) // Callback function for zlib that opens a file stream from a file descriptor. // Since we do not own the file descriptor, dup it so that we can fdopen/fclose // a file stream. -void* FdOpenFileFunc(void* opaque, const char* filename, int mode) { +void* FdOpenFileFunc(void* opaque, const void* filename, int mode) { FILE* file = NULL; const char* mode_fopen = NULL; @@ -103,15 +110,15 @@ void* FdOpenFileFunc(void* opaque, const char* filename, int mode) { int FdCloseFileFunc(void* opaque, void* stream) { fclose(static_cast(stream)); - free(opaque); // malloc'ed in FillFdOpenFileFunc() + free(opaque); // malloc'ed in FillFdOpenFileFunc() return 0; } // Fills |pzlib_filecunc_def| appropriately to handle the zip file // referred to by |fd|. -void FillFdOpenFileFunc(zlib_filefunc_def* pzlib_filefunc_def, int fd) { - fill_fopen_filefunc(pzlib_filefunc_def); - pzlib_filefunc_def->zopen_file = FdOpenFileFunc; +void FillFdOpenFileFunc(zlib_filefunc64_def* pzlib_filefunc_def, int fd) { + fill_fopen64_filefunc(pzlib_filefunc_def); + pzlib_filefunc_def->zopen64_file = FdOpenFileFunc; pzlib_filefunc_def->zclose_file = FdCloseFileFunc; int* ptr_fd = static_cast(malloc(sizeof(fd))); *ptr_fd = fd; @@ -122,7 +129,7 @@ void FillFdOpenFileFunc(zlib_filefunc_def* pzlib_filefunc_def, int fd) { #if defined(OS_WIN) // Callback function for zlib that opens a file stream from a Windows handle. // Does not take ownership of the handle. -void* HandleOpenFileFunc(void* opaque, const char* filename, int mode) { +void* HandleOpenFileFunc(void* opaque, const void* /*filename*/, int mode) { WIN32FILE_IOWIN file_ret; file_ret.hf = static_cast(opaque); file_ret.error = 0; @@ -136,7 +143,7 @@ void* HandleOpenFileFunc(void* opaque, const char* filename, int mode) { } int HandleCloseFileFunc(void* opaque, void* stream) { - free(stream); // malloc'ed in HandleOpenFileFunc() + free(stream); // malloc'ed in HandleOpenFileFunc() return 0; } #endif @@ -146,8 +153,8 @@ int HandleCloseFileFunc(void* opaque, void* stream) { // expect their opaque parameters refer to this struct. struct ZipBuffer { const char* data; // weak - size_t length; - size_t offset; + ZPOS64_T length; + ZPOS64_T offset; }; // Opens the specified file. When this function returns a non-NULL pointer, zlib @@ -156,7 +163,7 @@ struct ZipBuffer { // given opaque parameter and returns it because this parameter stores all // information needed for uncompressing data. (This function does not support // writing compressed data and it returns NULL for this case.) -void* OpenZipBuffer(void* opaque, const char* /*filename*/, int mode) { +void* OpenZipBuffer(void* opaque, const void* /*filename*/, int mode) { if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER) != ZLIB_FILEFUNC_MODE_READ) { NOTREACHED(); return NULL; @@ -173,10 +180,11 @@ void* OpenZipBuffer(void* opaque, const char* /*filename*/, int mode) { uLong ReadZipBuffer(void* opaque, void* /*stream*/, void* buf, uLong size) { ZipBuffer* buffer = static_cast(opaque); DCHECK_LE(buffer->offset, buffer->length); - size_t remaining_bytes = buffer->length - buffer->offset; + ZPOS64_T remaining_bytes = buffer->length - buffer->offset; if (!buffer || !buffer->data || !remaining_bytes) return 0; - size = std::min(size, static_cast(remaining_bytes)); + if (size > remaining_bytes) + size = remaining_bytes; memcpy(buf, &buffer->data[buffer->offset], size); buffer->offset += size; return size; @@ -193,21 +201,23 @@ uLong WriteZipBuffer(void* /*opaque*/, } // Returns the offset from the beginning of the data. -long GetOffsetOfZipBuffer(void* opaque, void* /*stream*/) { +ZPOS64_T GetOffsetOfZipBuffer(void* opaque, void* /*stream*/) { ZipBuffer* buffer = static_cast(opaque); if (!buffer) return -1; - return static_cast(buffer->offset); + return buffer->offset; } // Moves the current offset to the specified position. -long SeekZipBuffer(void* opaque, void* /*stream*/, uLong offset, int origin) { +long SeekZipBuffer(void* opaque, + void* /*stream*/, + ZPOS64_T offset, + int origin) { ZipBuffer* buffer = static_cast(opaque); if (!buffer) return -1; if (origin == ZLIB_FILEFUNC_SEEK_CUR) { - buffer->offset = std::min(buffer->offset + static_cast(offset), - buffer->length); + buffer->offset = std::min(buffer->offset + offset, buffer->length); return 0; } if (origin == ZLIB_FILEFUNC_SEEK_END) { @@ -215,7 +225,7 @@ long SeekZipBuffer(void* opaque, void* /*stream*/, uLong offset, int origin) { return 0; } if (origin == ZLIB_FILEFUNC_SEEK_SET) { - buffer->offset = std::min(buffer->length, static_cast(offset)); + buffer->offset = std::min(buffer->length, offset); return 0; } NOTREACHED(); @@ -241,7 +251,7 @@ int GetErrorOfZipBuffer(void* /*opaque*/, void* /*stream*/) { // Returns a zip_fileinfo struct with the time represented by |file_time|. zip_fileinfo TimeToZipFileInfo(const base::Time& file_time) { base::Time::Exploded file_time_parts; - file_time.LocalExplode(&file_time_parts); + file_time.UTCExplode(&file_time_parts); zip_fileinfo zip_info = {}; if (file_time_parts.year >= 1980) { @@ -266,33 +276,33 @@ namespace zip { namespace internal { unzFile OpenForUnzipping(const std::string& file_name_utf8) { - zlib_filefunc_def* zip_func_ptrs = NULL; + zlib_filefunc64_def* zip_func_ptrs = nullptr; #if defined(OS_WIN) - zlib_filefunc_def zip_funcs; - fill_win32_filefunc(&zip_funcs); - zip_funcs.zopen_file = ZipOpenFunc; + zlib_filefunc64_def zip_funcs; + fill_win32_filefunc64(&zip_funcs); + zip_funcs.zopen64_file = ZipOpenFunc; zip_func_ptrs = &zip_funcs; #endif - return unzOpen2(file_name_utf8.c_str(), zip_func_ptrs); + return unzOpen2_64(file_name_utf8.c_str(), zip_func_ptrs); } -#if defined(OS_POSIX) +#if defined(OS_POSIX) || defined(OS_FUCHSIA) unzFile OpenFdForUnzipping(int zip_fd) { - zlib_filefunc_def zip_funcs; + zlib_filefunc64_def zip_funcs; FillFdOpenFileFunc(&zip_funcs, zip_fd); // Passing dummy "fd" filename to zlib. - return unzOpen2("fd", &zip_funcs); + return unzOpen2_64("fd", &zip_funcs); } #endif #if defined(OS_WIN) unzFile OpenHandleForUnzipping(HANDLE zip_handle) { - zlib_filefunc_def zip_funcs; - fill_win32_filefunc(&zip_funcs); - zip_funcs.zopen_file = HandleOpenFileFunc; + zlib_filefunc64_def zip_funcs; + fill_win32_filefunc64(&zip_funcs); + zip_funcs.zopen64_file = HandleOpenFileFunc; zip_funcs.zclose_file = HandleCloseFileFunc; zip_funcs.opaque = zip_handle; - return unzOpen2("fd", &zip_funcs); + return unzOpen2_64("fd", &zip_funcs); } #endif @@ -308,72 +318,152 @@ unzFile PrepareMemoryForUnzipping(const std::string& data) { buffer->length = data.length(); buffer->offset = 0; - zlib_filefunc_def zip_functions; - zip_functions.zopen_file = OpenZipBuffer; + zlib_filefunc64_def zip_functions; + zip_functions.zopen64_file = OpenZipBuffer; zip_functions.zread_file = ReadZipBuffer; zip_functions.zwrite_file = WriteZipBuffer; - zip_functions.ztell_file = GetOffsetOfZipBuffer; - zip_functions.zseek_file = SeekZipBuffer; + zip_functions.ztell64_file = GetOffsetOfZipBuffer; + zip_functions.zseek64_file = SeekZipBuffer; zip_functions.zclose_file = CloseZipBuffer; zip_functions.zerror_file = GetErrorOfZipBuffer; - zip_functions.opaque = static_cast(buffer); - return unzOpen2(NULL, &zip_functions); + zip_functions.opaque = buffer; + return unzOpen2_64(nullptr, &zip_functions); } zipFile OpenForZipping(const std::string& file_name_utf8, int append_flag) { - zlib_filefunc_def* zip_func_ptrs = NULL; + zlib_filefunc64_def* zip_func_ptrs = nullptr; #if defined(OS_WIN) - zlib_filefunc_def zip_funcs; - fill_win32_filefunc(&zip_funcs); - zip_funcs.zopen_file = ZipOpenFunc; + zlib_filefunc64_def zip_funcs; + fill_win32_filefunc64(&zip_funcs); + zip_funcs.zopen64_file = ZipOpenFunc; zip_func_ptrs = &zip_funcs; #endif - return zipOpen2(file_name_utf8.c_str(), - append_flag, - NULL, // global comment - zip_func_ptrs); + return zipOpen2_64(file_name_utf8.c_str(), append_flag, nullptr, + zip_func_ptrs); } #if defined(OS_POSIX) zipFile OpenFdForZipping(int zip_fd, int append_flag) { - zlib_filefunc_def zip_funcs; + zlib_filefunc64_def zip_funcs; FillFdOpenFileFunc(&zip_funcs, zip_fd); // Passing dummy "fd" filename to zlib. - return zipOpen2("fd", append_flag, NULL, &zip_funcs); + return zipOpen2_64("fd", append_flag, nullptr, &zip_funcs); } #endif bool ZipOpenNewFileInZip(zipFile zip_file, const std::string& str_path, - base::Time last_modified_time) { + base::Time last_modified_time, + Compression compression) { // Section 4.4.4 http://www.pkware.com/documents/casestudies/APPNOTE.TXT // Setting the Language encoding flag so the file is told to be in utf-8. const uLong LANGUAGE_ENCODING_FLAG = 0x1 << 11; - zip_fileinfo file_info = TimeToZipFileInfo(last_modified_time); - if (ZIP_OK != zipOpenNewFileInZip4(zip_file, // file - str_path.c_str(), // filename - &file_info, // zip_fileinfo - NULL, // extrafield_local, - 0u, // size_extrafield_local - NULL, // extrafield_global - 0u, // size_extrafield_global - NULL, // comment - Z_DEFLATED, // method - Z_DEFAULT_COMPRESSION, // level - 0, // raw - -MAX_WBITS, // windowBits - DEF_MEM_LEVEL, // memLevel - Z_DEFAULT_STRATEGY, // strategy - NULL, // password - 0, // crcForCrypting - 0, // versionMadeBy - LANGUAGE_ENCODING_FLAG)) { // flagBase - DLOG(ERROR) << "Could not open zip file entry " << str_path; + const zip_fileinfo file_info = TimeToZipFileInfo(last_modified_time); + const int err = zipOpenNewFileInZip4_64( + /*file=*/zip_file, + /*filename=*/str_path.c_str(), + /*zip_fileinfo=*/&file_info, + /*extrafield_local=*/nullptr, + /*size_extrafield_local=*/0u, + /*extrafield_global=*/nullptr, + /*size_extrafield_global=*/0u, + /*comment=*/nullptr, + /*method=*/compression, + /*level=*/Z_DEFAULT_COMPRESSION, + /*raw=*/0, + /*windowBits=*/-MAX_WBITS, + /*memLevel=*/DEF_MEM_LEVEL, + /*strategy=*/Z_DEFAULT_STRATEGY, + /*password=*/nullptr, + /*crcForCrypting=*/0, + /*versionMadeBy=*/0, + /*flagBase=*/LANGUAGE_ENCODING_FLAG, + /*zip64=*/1); + + if (err != ZIP_OK) { + DLOG(ERROR) << "Cannot open ZIP file entry '" << str_path + << "': zipOpenNewFileInZip4_64 returned " << err; return false; } + return true; } +Compression GetCompressionMethod(const base::FilePath& path) { + // Get the filename extension in lower case. + const base::FilePath::StringType ext = + base::ToLowerASCII(path.FinalExtension()); + + if (ext.empty()) + return kDeflated; + + using StringPiece = base::FilePath::StringPieceType; + + // Skip the leading dot. + StringPiece ext_without_dot = ext; + DCHECK_EQ(ext_without_dot.front(), FILE_PATH_LITERAL('.')); + ext_without_dot.remove_prefix(1); + + // Well known filename extensions of files that a likely to be already + // compressed. The extensions are in lower case without the leading dot. + static const base::NoDestructor< + std::unordered_set>> + exts(std::initializer_list{ + FILE_PATH_LITERAL("3g2"), // + FILE_PATH_LITERAL("3gp"), // + FILE_PATH_LITERAL("7z"), // + FILE_PATH_LITERAL("7zip"), // + FILE_PATH_LITERAL("aac"), // + FILE_PATH_LITERAL("avi"), // + FILE_PATH_LITERAL("bz"), // + FILE_PATH_LITERAL("bz2"), // + FILE_PATH_LITERAL("crx"), // + FILE_PATH_LITERAL("gif"), // + FILE_PATH_LITERAL("gz"), // + FILE_PATH_LITERAL("jar"), // + FILE_PATH_LITERAL("jpeg"), // + FILE_PATH_LITERAL("jpg"), // + FILE_PATH_LITERAL("lz"), // + FILE_PATH_LITERAL("m2v"), // + FILE_PATH_LITERAL("m4p"), // + FILE_PATH_LITERAL("m4v"), // + FILE_PATH_LITERAL("mng"), // + FILE_PATH_LITERAL("mov"), // + FILE_PATH_LITERAL("mp2"), // + FILE_PATH_LITERAL("mp3"), // + FILE_PATH_LITERAL("mp4"), // + FILE_PATH_LITERAL("mpe"), // + FILE_PATH_LITERAL("mpeg"), // + FILE_PATH_LITERAL("mpg"), // + FILE_PATH_LITERAL("mpv"), // + FILE_PATH_LITERAL("ogg"), // + FILE_PATH_LITERAL("ogv"), // + FILE_PATH_LITERAL("png"), // + FILE_PATH_LITERAL("qt"), // + FILE_PATH_LITERAL("rar"), // + FILE_PATH_LITERAL("taz"), // + FILE_PATH_LITERAL("tb2"), // + FILE_PATH_LITERAL("tbz"), // + FILE_PATH_LITERAL("tbz2"), // + FILE_PATH_LITERAL("tgz"), // + FILE_PATH_LITERAL("tlz"), // + FILE_PATH_LITERAL("tz"), // + FILE_PATH_LITERAL("tz2"), // + FILE_PATH_LITERAL("vob"), // + FILE_PATH_LITERAL("webm"), // + FILE_PATH_LITERAL("wma"), // + FILE_PATH_LITERAL("wmv"), // + FILE_PATH_LITERAL("xz"), // + FILE_PATH_LITERAL("z"), // + FILE_PATH_LITERAL("zip"), // + }); + + if (exts->count(ext_without_dot)) + return kStored; + + return kDeflated; +} + } // namespace internal } // namespace zip diff --git a/deps/zlib/google/zip_internal.h b/deps/zlib/google/zip_internal.h index 49fb902a7408e7..c7feba692be9fe 100644 --- a/deps/zlib/google/zip_internal.h +++ b/deps/zlib/google/zip_internal.h @@ -35,7 +35,7 @@ namespace internal { // Windows. unzFile OpenForUnzipping(const std::string& file_name_utf8); -#if defined(OS_POSIX) +#if defined(OS_POSIX) || defined(OS_FUCHSIA) // Opens the file referred to by |zip_fd| for unzipping. unzFile OpenFdForUnzipping(int zip_fd); #endif @@ -60,10 +60,24 @@ zipFile OpenForZipping(const std::string& file_name_utf8, int append_flag); zipFile OpenFdForZipping(int zip_fd, int append_flag); #endif -// Wrapper around zipOpenNewFileInZip4 which passes most common options. +// Compression methods. +enum Compression { + kStored = 0, // Stored (no compression) + kDeflated = Z_DEFLATED, // Deflated +}; + +// Adds a file (or directory) entry to the ZIP archive. bool ZipOpenNewFileInZip(zipFile zip_file, const std::string& str_path, - base::Time last_modified_time); + base::Time last_modified_time, + Compression compression); + +// Selects the best compression method for the given file. The heuristic is +// based on the filename extension. By default, the compression method is +// kDeflated. But if the given path has an extension indicating a well known +// file format which is likely to be already compressed (eg ZIP, RAR, JPG, +// PNG...) then the compression method is simply kStored. +Compression GetCompressionMethod(const base::FilePath& path); const int kZipMaxPath = 256; const int kZipBufSize = 8192; diff --git a/deps/zlib/google/zip_reader.cc b/deps/zlib/google/zip_reader.cc index 8b4bb4aee782bf..f546a90d2a0a10 100644 --- a/deps/zlib/google/zip_reader.cc +++ b/deps/zlib/google/zip_reader.cc @@ -9,11 +9,9 @@ #include "base/bind.h" #include "base/files/file.h" #include "base/logging.h" -#include "base/macros.h" -#include "base/single_thread_task_runner.h" #include "base/strings/string_util.h" #include "base/strings/utf_string_conversions.h" -#include "base/threading/thread_task_runner_handle.h" +#include "base/threading/sequenced_task_runner_handle.h" #include "build/build_config.h" #include "third_party/zlib/google/zip_internal.h" @@ -26,6 +24,10 @@ #endif // defined(OS_WIN) #endif // defined(USE_SYSTEM_MINIZIP) +#if defined(OS_POSIX) +#include +#endif + namespace zip { namespace { @@ -37,6 +39,10 @@ namespace { class StringWriterDelegate : public WriterDelegate { public: StringWriterDelegate(size_t max_read_bytes, std::string* output); + + StringWriterDelegate(const StringWriterDelegate&) = delete; + StringWriterDelegate& operator=(const StringWriterDelegate&) = delete; + ~StringWriterDelegate() override; // WriterDelegate methods: @@ -50,11 +56,11 @@ class StringWriterDelegate : public WriterDelegate { void SetTimeModified(const base::Time& time) override; + void SetPosixFilePermissions(int mode) override; + private: size_t max_read_bytes_; std::string* output_; - - DISALLOW_COPY_AND_ASSIGN(StringWriterDelegate); }; StringWriterDelegate::StringWriterDelegate(size_t max_read_bytes, @@ -81,6 +87,33 @@ void StringWriterDelegate::SetTimeModified(const base::Time& time) { // Do nothing. } +void StringWriterDelegate::SetPosixFilePermissions(int mode) { + // Do nothing. +} + +#if defined(OS_POSIX) +void SetPosixFilePermissions(int fd, int mode) { + base::stat_wrapper_t sb; + if (base::File::Fstat(fd, &sb)) { + return; + } + mode_t new_mode = sb.st_mode; + // Transfer the executable bit only if the file is readable. + if ((sb.st_mode & S_IRUSR) == S_IRUSR && (mode & S_IXUSR) == S_IXUSR) { + new_mode |= S_IXUSR; + } + if ((sb.st_mode & S_IRGRP) == S_IRGRP && (mode & S_IXGRP) == S_IXGRP) { + new_mode |= S_IXGRP; + } + if ((sb.st_mode & S_IROTH) == S_IROTH && (mode & S_IXOTH) == S_IXOTH) { + new_mode |= S_IXOTH; + } + if (new_mode != sb.st_mode) { + fchmod(fd, new_mode); + } +} +#endif + } // namespace // TODO(satorux): The implementation assumes that file names in zip files @@ -91,7 +124,8 @@ ZipReader::EntryInfo::EntryInfo(const std::string& file_name_in_zip, : file_path_(base::FilePath::FromUTF8Unsafe(file_name_in_zip)), is_directory_(false), is_unsafe_(false), - is_encrypted_(false) { + is_encrypted_(false), + posix_mode_(0) { original_size_ = raw_file_info.uncompressed_size; // Directory entries in zip files end with "/". @@ -102,7 +136,7 @@ ZipReader::EntryInfo::EntryInfo(const std::string& file_name_in_zip, is_unsafe_ = file_path_.ReferencesParent(); // We also consider that the file name is unsafe, if it's invalid UTF-8. - base::string16 file_name_utf16; + std::u16string file_name_utf16; if (!base::UTF8ToUTF16(file_name_in_zip.data(), file_name_in_zip.size(), &file_name_utf16)) { is_unsafe_ = true; @@ -130,8 +164,13 @@ ZipReader::EntryInfo::EntryInfo(const std::string& file_name_in_zip, exploded_time.second = raw_file_info.tmu_date.tm_sec; exploded_time.millisecond = 0; - if (!base::Time::FromLocalExploded(exploded_time, &last_modified_)) + if (!base::Time::FromUTCExploded(exploded_time, &last_modified_)) last_modified_ = base::Time::UnixEpoch(); + +#if defined(OS_POSIX) + posix_mode_ = + (raw_file_info.external_fa >> 16L) & (S_IRWXU | S_IRWXG | S_IRWXO); +#endif } ZipReader::ZipReader() { @@ -158,7 +197,7 @@ bool ZipReader::Open(const base::FilePath& zip_file_path) { bool ZipReader::OpenFromPlatformFile(base::PlatformFile zip_fd) { DCHECK(!zip_file_); -#if defined(OS_POSIX) +#if defined(OS_POSIX) || defined(OS_FUCHSIA) zip_file_ = internal::OpenFdForUnzipping(zip_fd); #elif defined(OS_WIN) zip_file_ = internal::OpenHandleForUnzipping(zip_fd); @@ -277,9 +316,11 @@ bool ZipReader::ExtractCurrentEntry(WriterDelegate* delegate, unzCloseCurrentFile(zip_file_); - if (entire_file_extracted && - current_entry_info()->last_modified() != base::Time::UnixEpoch()) { - delegate->SetTimeModified(current_entry_info()->last_modified()); + if (entire_file_extracted) { + delegate->SetPosixFilePermissions(current_entry_info()->posix_mode()); + if (current_entry_info()->last_modified() != base::Time::UnixEpoch()) { + delegate->SetTimeModified(current_entry_info()->last_modified()); + } } return entire_file_extracted; @@ -296,11 +337,11 @@ void ZipReader::ExtractCurrentEntryToFilePathAsync( // If this is a directory, just create it and return. if (current_entry_info()->is_directory()) { if (base::CreateDirectory(output_file_path)) { - base::ThreadTaskRunnerHandle::Get()->PostTask( + base::SequencedTaskRunnerHandle::Get()->PostTask( FROM_HERE, std::move(success_callback)); } else { DVLOG(1) << "Unzip failed: unable to create directory."; - base::ThreadTaskRunnerHandle::Get()->PostTask( + base::SequencedTaskRunnerHandle::Get()->PostTask( FROM_HERE, std::move(failure_callback)); } return; @@ -308,16 +349,16 @@ void ZipReader::ExtractCurrentEntryToFilePathAsync( if (unzOpenCurrentFile(zip_file_) != UNZ_OK) { DVLOG(1) << "Unzip failed: unable to open current zip entry."; - base::ThreadTaskRunnerHandle::Get()->PostTask(FROM_HERE, - std::move(failure_callback)); + base::SequencedTaskRunnerHandle::Get()->PostTask( + FROM_HERE, std::move(failure_callback)); return; } base::FilePath output_dir_path = output_file_path.DirName(); if (!base::CreateDirectory(output_dir_path)) { DVLOG(1) << "Unzip failed: unable to create containing directory."; - base::ThreadTaskRunnerHandle::Get()->PostTask(FROM_HERE, - std::move(failure_callback)); + base::SequencedTaskRunnerHandle::Get()->PostTask( + FROM_HERE, std::move(failure_callback)); return; } @@ -327,17 +368,17 @@ void ZipReader::ExtractCurrentEntryToFilePathAsync( if (!output_file.IsValid()) { DVLOG(1) << "Unzip failed: unable to create platform file at " << output_file_path.value(); - base::ThreadTaskRunnerHandle::Get()->PostTask(FROM_HERE, - std::move(failure_callback)); + base::SequencedTaskRunnerHandle::Get()->PostTask( + FROM_HERE, std::move(failure_callback)); return; } - base::ThreadTaskRunnerHandle::Get()->PostTask( + base::SequencedTaskRunnerHandle::Get()->PostTask( FROM_HERE, base::BindOnce(&ZipReader::ExtractChunk, weak_ptr_factory_.GetWeakPtr(), - Passed(std::move(output_file)), - std::move(success_callback), std::move(failure_callback), - progress_callback, 0 /* initial offset */)); + std::move(output_file), std::move(success_callback), + std::move(failure_callback), progress_callback, + 0 /* initial offset */)); } bool ZipReader::ExtractCurrentEntryToString(uint64_t max_read_bytes, @@ -434,12 +475,12 @@ void ZipReader::ExtractChunk(base::File output_file, progress_callback.Run(current_progress); - base::ThreadTaskRunnerHandle::Get()->PostTask( + base::SequencedTaskRunnerHandle::Get()->PostTask( FROM_HERE, base::BindOnce(&ZipReader::ExtractChunk, weak_ptr_factory_.GetWeakPtr(), - Passed(std::move(output_file)), - std::move(success_callback), std::move(failure_callback), - progress_callback, current_progress)); + std::move(output_file), std::move(success_callback), + std::move(failure_callback), progress_callback, + current_progress)); } } @@ -471,6 +512,12 @@ void FileWriterDelegate::SetTimeModified(const base::Time& time) { file_->SetTimes(base::Time::Now(), time); } +void FileWriterDelegate::SetPosixFilePermissions(int mode) { +#if defined(OS_POSIX) + zip::SetPosixFilePermissions(file_->GetPlatformFile(), mode); +#endif +} + // FilePathWriterDelegate ------------------------------------------------------ FilePathWriterDelegate::FilePathWriterDelegate( @@ -499,4 +546,10 @@ void FilePathWriterDelegate::SetTimeModified(const base::Time& time) { base::TouchFile(output_file_path_, base::Time::Now(), time); } +void FilePathWriterDelegate::SetPosixFilePermissions(int mode) { +#if defined(OS_POSIX) + zip::SetPosixFilePermissions(file_.GetPlatformFile(), mode); +#endif +} + } // namespace zip diff --git a/deps/zlib/google/zip_reader.h b/deps/zlib/google/zip_reader.h index d442d428591e50..c2d3bea804630b 100644 --- a/deps/zlib/google/zip_reader.h +++ b/deps/zlib/google/zip_reader.h @@ -14,7 +14,6 @@ #include "base/files/file.h" #include "base/files/file_path.h" #include "base/files/file_util.h" -#include "base/macros.h" #include "base/memory/weak_ptr.h" #include "base/time/time.h" @@ -42,6 +41,11 @@ class WriterDelegate { // Sets the last-modified time of the data. virtual void SetTimeModified(const base::Time& time) = 0; + + // Called with the POSIX file permissions of the data; POSIX implementations + // may apply some of the permissions (for example, the executable bit) to the + // output file. + virtual void SetPosixFilePermissions(int mode) = 0; }; // This class is used for reading zip files. A typical use case of this @@ -79,6 +83,9 @@ class ZipReader { EntryInfo(const std::string& filename_in_zip, const unz_file_info& raw_file_info); + EntryInfo(const EntryInfo&) = delete; + EntryInfo& operator=(const EntryInfo&) = delete; + // Returns the file path. The path is usually relative like // "foo/bar.txt", but if it's absolute, is_unsafe() returns true. const base::FilePath& file_path() const { return file_path_; } @@ -110,6 +117,9 @@ class ZipReader { // Returns true if the entry is encrypted. bool is_encrypted() const { return is_encrypted_; } + // Returns the posix file permissions of the entry. + int posix_mode() const { return posix_mode_; } + private: const base::FilePath file_path_; int64_t original_size_; @@ -117,10 +127,14 @@ class ZipReader { bool is_directory_; bool is_unsafe_; bool is_encrypted_; - DISALLOW_COPY_AND_ASSIGN(EntryInfo); + int posix_mode_; }; ZipReader(); + + ZipReader(const ZipReader&) = delete; + ZipReader& operator=(const ZipReader&) = delete; + ~ZipReader(); // Opens the zip file specified by |zip_file_path|. Returns true on @@ -225,8 +239,6 @@ class ZipReader { std::unique_ptr current_entry_info_; base::WeakPtrFactory weak_ptr_factory_{this}; - - DISALLOW_COPY_AND_ASSIGN(ZipReader); }; // A writer delegate that writes to a given File. @@ -240,6 +252,9 @@ class FileWriterDelegate : public WriterDelegate { // Constructs a FileWriterDelegate that takes ownership of |file|. explicit FileWriterDelegate(std::unique_ptr file); + FileWriterDelegate(const FileWriterDelegate&) = delete; + FileWriterDelegate& operator=(const FileWriterDelegate&) = delete; + // Truncates the file to the number of bytes written. ~FileWriterDelegate() override; @@ -255,6 +270,10 @@ class FileWriterDelegate : public WriterDelegate { // Sets the last-modified time of the data. void SetTimeModified(const base::Time& time) override; + // On POSIX systems, sets the file to be executable if the source file was + // executable. + void SetPosixFilePermissions(int mode) override; + // Return the actual size of the file. int64_t file_length() { return file_length_; } @@ -267,14 +286,16 @@ class FileWriterDelegate : public WriterDelegate { std::unique_ptr owned_file_; int64_t file_length_ = 0; - - DISALLOW_COPY_AND_ASSIGN(FileWriterDelegate); }; // A writer delegate that writes a file at a given path. class FilePathWriterDelegate : public WriterDelegate { public: explicit FilePathWriterDelegate(const base::FilePath& output_file_path); + + FilePathWriterDelegate(const FilePathWriterDelegate&) = delete; + FilePathWriterDelegate& operator=(const FilePathWriterDelegate&) = delete; + ~FilePathWriterDelegate() override; // WriterDelegate methods: @@ -289,11 +310,13 @@ class FilePathWriterDelegate : public WriterDelegate { // Sets the last-modified time of the data. void SetTimeModified(const base::Time& time) override; + // On POSIX systems, sets the file to be executable if the source file was + // executable. + void SetPosixFilePermissions(int mode) override; + private: base::FilePath output_file_path_; base::File file_; - - DISALLOW_COPY_AND_ASSIGN(FilePathWriterDelegate); }; } // namespace zip diff --git a/deps/zlib/google/zip_reader_unittest.cc b/deps/zlib/google/zip_reader_unittest.cc index 87190c7e0dee76..a15193b407acdd 100644 --- a/deps/zlib/google/zip_reader_unittest.cc +++ b/deps/zlib/google/zip_reader_unittest.cc @@ -12,18 +12,21 @@ #include #include "base/bind.h" +#include "base/check.h" +#include "base/cxx17_backports.h" #include "base/files/file.h" #include "base/files/file_util.h" #include "base/files/scoped_temp_dir.h" #include "base/hash/md5.h" -#include "base/logging.h" #include "base/path_service.h" #include "base/run_loop.h" -#include "base/stl_util.h" +#include "base/strings/string_piece.h" #include "base/strings/stringprintf.h" #include "base/strings/utf_string_conversions.h" +#include "base/test/bind.h" #include "base/test/task_environment.h" #include "base/time/time.h" +#include "build/build_config.h" #include "testing/gmock/include/gmock/gmock.h" #include "testing/gtest/include/gtest/gtest.h" #include "testing/platform_test.h" @@ -110,6 +113,7 @@ class MockWriterDelegate : public zip::WriterDelegate { MOCK_METHOD0(PrepareOutput, bool()); MOCK_METHOD2(WriteBytes, bool(const char*, int)); MOCK_METHOD1(SetTimeModified, void(const base::Time&)); + MOCK_METHOD1(SetPosixFilePermissions, void(int)); }; bool ExtractCurrentEntryToFilePath(zip::ZipReader* reader, @@ -287,7 +291,7 @@ TEST_F(ZipReaderTest, current_entry_info_RegularFile) { // The expected time stamp: 2009-05-29 06:22:20 base::Time::Exploded exploded = {}; // Zero-clear. - current_entry_info->last_modified().LocalExplode(&exploded); + current_entry_info->last_modified().UTCExplode(&exploded); EXPECT_EQ(2009, exploded.year); EXPECT_EQ(5, exploded.month); EXPECT_EQ(29, exploded.day_of_month); @@ -356,7 +360,7 @@ TEST_F(ZipReaderTest, current_entry_info_Directory) { // The expected time stamp: 2009-05-31 15:49:52 base::Time::Exploded exploded = {}; // Zero-clear. - current_entry_info->last_modified().LocalExplode(&exploded); + current_entry_info->last_modified().UTCExplode(&exploded); EXPECT_EQ(2009, exploded.year); EXPECT_EQ(5, exploded.month); EXPECT_EQ(31, exploded.day_of_month); @@ -510,12 +514,12 @@ TEST_F(ZipReaderTest, ExtractCurrentEntryToString) { if (i > 0) { // Exact byte read limit: must pass. EXPECT_TRUE(reader.ExtractCurrentEntryToString(i, &contents)); - EXPECT_EQ(base::StringPiece("0123456", i).as_string(), contents); + EXPECT_EQ(std::string(base::StringPiece("0123456", i)), contents); } // More than necessary byte read limit: must pass. EXPECT_TRUE(reader.ExtractCurrentEntryToString(16, &contents)); - EXPECT_EQ(base::StringPiece("0123456", i).as_string(), contents); + EXPECT_EQ(std::string(base::StringPiece("0123456", i)), contents); } reader.Close(); } @@ -563,6 +567,38 @@ TEST_F(ZipReaderTest, ExtractPartOfCurrentEntry) { reader.Close(); } +TEST_F(ZipReaderTest, ExtractPosixPermissions) { + base::ScopedTempDir temp_dir; + ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); + + ZipReader reader; + ASSERT_TRUE( + reader.Open(test_data_dir_.AppendASCII("test_posix_permissions.zip"))); + for (auto entry : {"0.txt", "1.txt", "2.txt", "3.txt"}) { + ASSERT_TRUE(LocateAndOpenEntry(&reader, base::FilePath::FromASCII(entry))); + FilePathWriterDelegate delegate(temp_dir.GetPath().AppendASCII(entry)); + ASSERT_TRUE(reader.ExtractCurrentEntry(&delegate, 10000)); + } + reader.Close(); + +#if defined(OS_POSIX) + // This assumes a umask of at least 0400. + int mode = 0; + EXPECT_TRUE(base::GetPosixFilePermissions( + temp_dir.GetPath().AppendASCII("0.txt"), &mode)); + EXPECT_EQ(mode & 0700, 0700); + EXPECT_TRUE(base::GetPosixFilePermissions( + temp_dir.GetPath().AppendASCII("1.txt"), &mode)); + EXPECT_EQ(mode & 0700, 0600); + EXPECT_TRUE(base::GetPosixFilePermissions( + temp_dir.GetPath().AppendASCII("2.txt"), &mode)); + EXPECT_EQ(mode & 0700, 0700); + EXPECT_TRUE(base::GetPosixFilePermissions( + temp_dir.GetPath().AppendASCII("3.txt"), &mode)); + EXPECT_EQ(mode & 0700, 0600); +#endif +} + // This test exposes http://crbug.com/430959, at least on OS X TEST_F(ZipReaderTest, DISABLED_LeakDetectionTest) { for (int i = 0; i < 100000; ++i) { @@ -616,6 +652,7 @@ TEST_F(ZipReaderTest, ExtractCurrentEntrySuccess) { .WillOnce(Return(true)); EXPECT_CALL(mock_writer, WriteBytes(_, _)) .WillRepeatedly(Return(true)); + EXPECT_CALL(mock_writer, SetPosixFilePermissions(_)); EXPECT_CALL(mock_writer, SetTimeModified(_)); base::FilePath target_path(FILE_PATH_LITERAL("foo/bar/quux.txt")); @@ -631,11 +668,10 @@ class FileWriterDelegateTest : public ::testing::Test { protected: void SetUp() override { ASSERT_TRUE(base::CreateTemporaryFile(&temp_file_path_)); - file_.Initialize(temp_file_path_, (base::File::FLAG_CREATE_ALWAYS | - base::File::FLAG_READ | - base::File::FLAG_WRITE | - base::File::FLAG_TEMPORARY | - base::File::FLAG_DELETE_ON_CLOSE)); + file_.Initialize(temp_file_path_, + (base::File::FLAG_CREATE_ALWAYS | base::File::FLAG_READ | + base::File::FLAG_WRITE | base::File::FLAG_WIN_TEMPORARY | + base::File::FLAG_DELETE_ON_CLOSE)); ASSERT_TRUE(file_.IsValid()); } diff --git a/deps/zlib/google/zip_unittest.cc b/deps/zlib/google/zip_unittest.cc index 7ea3c36c2a80f1..944930ffc84e39 100644 --- a/deps/zlib/google/zip_unittest.cc +++ b/deps/zlib/google/zip_unittest.cc @@ -16,16 +16,21 @@ #include "base/files/file_path.h" #include "base/files/file_util.h" #include "base/files/scoped_temp_dir.h" -#include "base/macros.h" +#include "base/logging.h" #include "base/path_service.h" #include "base/strings/string_util.h" #include "base/strings/stringprintf.h" +#include "base/test/bind.h" #include "build/build_config.h" #include "testing/gtest/include/gtest/gtest.h" #include "testing/platform_test.h" #include "third_party/zlib/google/zip.h" +#include "third_party/zlib/google/zip_internal.h" #include "third_party/zlib/google/zip_reader.h" +// Convenience macro to create a file path from a string literal. +#define FP(path) base::FilePath(FILE_PATH_LITERAL(path)) + namespace { bool CreateFile(const std::string& content, @@ -55,8 +60,8 @@ class VirtualFileSystem : public zip::FileAccessor { static constexpr char kBar2Content[] = "This is bar too."; VirtualFileSystem() { - base::FilePath test_dir(FILE_PATH_LITERAL("/test")); - base::FilePath foo_txt_path = test_dir.Append(FILE_PATH_LITERAL("foo.txt")); + base::FilePath test_dir; + base::FilePath foo_txt_path = test_dir.AppendASCII("foo.txt"); base::FilePath file_path; base::File file; @@ -64,62 +69,92 @@ class VirtualFileSystem : public zip::FileAccessor { DCHECK(success); files_[foo_txt_path] = std::move(file); - base::FilePath bar_dir = test_dir.Append(FILE_PATH_LITERAL("bar")); - base::FilePath bar1_txt_path = - bar_dir.Append(FILE_PATH_LITERAL("bar1.txt")); + base::FilePath bar_dir = test_dir.AppendASCII("bar"); + base::FilePath bar1_txt_path = bar_dir.AppendASCII("bar1.txt"); success = CreateFile(kBar1Content, &file_path, &file); DCHECK(success); files_[bar1_txt_path] = std::move(file); - base::FilePath bar2_txt_path = - bar_dir.Append(FILE_PATH_LITERAL("bar2.txt")); + base::FilePath bar2_txt_path = bar_dir.AppendASCII("bar2.txt"); success = CreateFile(kBar2Content, &file_path, &file); DCHECK(success); files_[bar2_txt_path] = std::move(file); - file_tree_[test_dir] = std::vector{ - DirectoryContentEntry(foo_txt_path, /*is_dir=*/false), - DirectoryContentEntry(bar_dir, /*is_dir=*/true)}; - file_tree_[bar_dir] = std::vector{ - DirectoryContentEntry(bar1_txt_path, /*is_dir=*/false), - DirectoryContentEntry(bar2_txt_path, /*is_dir=*/false)}; + file_tree_[base::FilePath()] = {{foo_txt_path}, {bar_dir}}; + file_tree_[bar_dir] = {{bar1_txt_path, bar2_txt_path}}; + file_tree_[foo_txt_path] = {}; + file_tree_[bar1_txt_path] = {}; + file_tree_[bar2_txt_path] = {}; } + + VirtualFileSystem(const VirtualFileSystem&) = delete; + VirtualFileSystem& operator=(const VirtualFileSystem&) = delete; + ~VirtualFileSystem() override = default; private: - std::vector OpenFilesForReading( - const std::vector& paths) override { - std::vector files; - for (const auto& path : paths) { - auto iter = files_.find(path); - files.push_back(iter == files_.end() ? base::File() - : std::move(iter->second)); + bool Open(const zip::Paths paths, + std::vector* const files) override { + DCHECK(files); + files->reserve(files->size() + paths.size()); + + for (const base::FilePath& path : paths) { + const auto it = files_.find(path); + if (it == files_.end()) { + files->emplace_back(); + } else { + EXPECT_TRUE(it->second.IsValid()); + files->push_back(std::move(it->second)); + } } - return files; - } - bool DirectoryExists(const base::FilePath& file) override { - return file_tree_.count(file) == 1 && files_.count(file) == 0; + return true; } - std::vector ListDirectoryContent( - const base::FilePath& dir) override { - auto iter = file_tree_.find(dir); - if (iter == file_tree_.end()) { - NOTREACHED(); - return std::vector(); + bool List(const base::FilePath& path, + std::vector* const files, + std::vector* const subdirs) override { + DCHECK(!path.IsAbsolute()); + DCHECK(files); + DCHECK(subdirs); + + const auto it = file_tree_.find(path); + if (it == file_tree_.end()) + return false; + + for (const base::FilePath& file : it->second.files) { + DCHECK(!file.empty()); + files->push_back(file); } - return iter->second; + + for (const base::FilePath& subdir : it->second.subdirs) { + DCHECK(!subdir.empty()); + subdirs->push_back(subdir); + } + + return true; } - base::Time GetLastModifiedTime(const base::FilePath& path) override { - return base::Time::FromDoubleT(172097977); // Some random date. + bool GetInfo(const base::FilePath& path, Info* const info) override { + DCHECK(!path.IsAbsolute()); + DCHECK(info); + + if (!file_tree_.count(path)) + return false; + + info->is_directory = !files_.count(path); + info->last_modified = + base::Time::FromDoubleT(172097977); // Some random date. + + return true; } - std::map> file_tree_; - std::map files_; + struct DirContents { + std::vector files, subdirs; + }; - DISALLOW_COPY_AND_ASSIGN(VirtualFileSystem); + std::map file_tree_; + std::map files_; }; // static @@ -130,10 +165,7 @@ constexpr char VirtualFileSystem::kBar2Content[]; // Make the test a PlatformTest to setup autorelease pools properly on Mac. class ZipTest : public PlatformTest { protected: - enum ValidYearType { - VALID_YEAR, - INVALID_YEAR - }; + enum ValidYearType { VALID_YEAR, INVALID_YEAR }; virtual void SetUp() { PlatformTest::SetUp(); @@ -142,38 +174,33 @@ class ZipTest : public PlatformTest { test_dir_ = temp_dir_.GetPath(); base::FilePath zip_path(test_dir_); - zip_contents_.insert(zip_path.Append(FILE_PATH_LITERAL("foo.txt"))); - zip_path = zip_path.Append(FILE_PATH_LITERAL("foo")); + zip_contents_.insert(zip_path.AppendASCII("foo.txt")); + zip_path = zip_path.AppendASCII("foo"); zip_contents_.insert(zip_path); - zip_contents_.insert(zip_path.Append(FILE_PATH_LITERAL("bar.txt"))); - zip_path = zip_path.Append(FILE_PATH_LITERAL("bar")); + zip_contents_.insert(zip_path.AppendASCII("bar.txt")); + zip_path = zip_path.AppendASCII("bar"); zip_contents_.insert(zip_path); - zip_contents_.insert(zip_path.Append(FILE_PATH_LITERAL("baz.txt"))); - zip_contents_.insert(zip_path.Append(FILE_PATH_LITERAL("quux.txt"))); - zip_contents_.insert(zip_path.Append(FILE_PATH_LITERAL(".hidden"))); + zip_contents_.insert(zip_path.AppendASCII("baz.txt")); + zip_contents_.insert(zip_path.AppendASCII("quux.txt")); + zip_contents_.insert(zip_path.AppendASCII(".hidden")); // Include a subset of files in |zip_file_list_| to test ZipFiles(). - zip_file_list_.push_back(base::FilePath(FILE_PATH_LITERAL("foo.txt"))); - zip_file_list_.push_back( - base::FilePath(FILE_PATH_LITERAL("foo/bar/quux.txt"))); - zip_file_list_.push_back( - base::FilePath(FILE_PATH_LITERAL("foo/bar/.hidden"))); + zip_file_list_.push_back(FP("foo.txt")); + zip_file_list_.push_back(FP("foo/bar/quux.txt")); + zip_file_list_.push_back(FP("foo/bar/.hidden")); } - virtual void TearDown() { - PlatformTest::TearDown(); - } + virtual void TearDown() { PlatformTest::TearDown(); } bool GetTestDataDirectory(base::FilePath* path) { bool success = base::PathService::Get(base::DIR_SOURCE_ROOT, path); EXPECT_TRUE(success); if (!success) return false; - *path = path->AppendASCII("third_party"); - *path = path->AppendASCII("zlib"); - *path = path->AppendASCII("google"); - *path = path->AppendASCII("test"); - *path = path->AppendASCII("data"); + for (const base::StringPiece s : + {"third_party", "zlib", "google", "test", "data"}) { + *path = path->AppendASCII(s); + } return true; } @@ -192,11 +219,12 @@ class ZipTest : public PlatformTest { ASSERT_TRUE(GetTestDataDirectory(&original_dir)); original_dir = original_dir.AppendASCII("test"); - base::FileEnumerator files(test_dir_, true, + base::FileEnumerator files( + test_dir_, true, base::FileEnumerator::FILES | base::FileEnumerator::DIRECTORIES); base::FilePath unzipped_entry_path = files.Next(); size_t count = 0; - while (!unzipped_entry_path.value().empty()) { + while (!unzipped_entry_path.empty()) { EXPECT_EQ(zip_contents_.count(unzipped_entry_path), 1U) << "Couldn't find " << unzipped_entry_path.value(); count++; @@ -204,24 +232,15 @@ class ZipTest : public PlatformTest { if (base::PathExists(unzipped_entry_path) && !base::DirectoryExists(unzipped_entry_path)) { // It's a file, check its contents are what we zipped. - // TODO(774156): figure out why the commented out EXPECT_TRUE below - // fails on the build bots (but not on the try-bots). base::FilePath relative_path; - bool append_relative_path_success = - test_dir_.AppendRelativePath(unzipped_entry_path, &relative_path); - if (!append_relative_path_success) { - LOG(ERROR) << "Append relative path failed, params: " - << test_dir_.value() << " and " - << unzipped_entry_path.value(); - } + ASSERT_TRUE( + test_dir_.AppendRelativePath(unzipped_entry_path, &relative_path)) + << "Cannot append relative path failed, params: '" << test_dir_ + << "' and '" << unzipped_entry_path << "'"; base::FilePath original_path = original_dir.Append(relative_path); - LOG(ERROR) << "Comparing original " << original_path.value() - << " and unzipped file " << unzipped_entry_path.value() - << " result: " - << base::ContentsEqual(original_path, unzipped_entry_path); - // EXPECT_TRUE(base::ContentsEqual(original_path, unzipped_entry_path)) - // << "Contents differ between original " << original_path.value() - // << " and unzipped file " << unzipped_entry_path.value(); + EXPECT_TRUE(base::ContentsEqual(original_path, unzipped_entry_path)) + << "Original file '" << original_path << "' and unzipped file '" + << unzipped_entry_path << "' have different contents"; } unzipped_entry_path = files.Next(); } @@ -265,11 +284,11 @@ class ZipTest : public PlatformTest { // supports, which is 2 seconds. Note that between this call to Time::Now() // and zip::Zip() the clock can advance a bit, hence the use of EXPECT_GE. base::Time::Exploded now_parts; - base::Time::Now().LocalExplode(&now_parts); + base::Time::Now().UTCExplode(&now_parts); now_parts.second = now_parts.second & ~1; now_parts.millisecond = 0; base::Time now_time; - EXPECT_TRUE(base::Time::FromLocalExploded(now_parts, &now_time)); + EXPECT_TRUE(base::Time::FromUTCExploded(now_parts, &now_time)); EXPECT_EQ(1, base::WriteFile(src_file, "1", 1)); EXPECT_TRUE(base::TouchFile(src_file, base::Time::Now(), test_mtime)); @@ -486,8 +505,8 @@ TEST_F(ZipTest, ZipFiles) { base::File zip_file(zip_name, base::File::FLAG_CREATE | base::File::FLAG_WRITE); ASSERT_TRUE(zip_file.IsValid()); - EXPECT_TRUE(zip::ZipFiles(src_dir, zip_file_list_, - zip_file.GetPlatformFile())); + EXPECT_TRUE( + zip::ZipFiles(src_dir, zip_file_list_, zip_file.GetPlatformFile())); zip_file.Close(); zip::ZipReader reader; @@ -523,8 +542,8 @@ TEST_F(ZipTest, UnzipFilesWithIncorrectSize) { for (int i = 0; i < 8; i++) { SCOPED_TRACE(base::StringPrintf("Processing %d.txt", i)); - base::FilePath file_path = temp_dir.AppendASCII( - base::StringPrintf("%d.txt", i)); + base::FilePath file_path = + temp_dir.AppendASCII(base::StringPrintf("%d.txt", i)); int64_t file_size = -1; EXPECT_TRUE(base::GetFileSize(file_path, &file_size)); EXPECT_EQ(static_cast(i), file_size); @@ -534,26 +553,265 @@ TEST_F(ZipTest, UnzipFilesWithIncorrectSize) { TEST_F(ZipTest, ZipWithFileAccessor) { base::FilePath zip_file; ASSERT_TRUE(base::CreateTemporaryFile(&zip_file)); - zip::ZipParams params(base::FilePath(FILE_PATH_LITERAL("/test")), zip_file); - params.set_file_accessor(std::make_unique()); + VirtualFileSystem file_accessor; + const zip::ZipParams params{.file_accessor = &file_accessor, + .dest_file = zip_file}; ASSERT_TRUE(zip::Zip(params)); base::ScopedTempDir scoped_temp_dir; ASSERT_TRUE(scoped_temp_dir.CreateUniqueTempDir()); const base::FilePath& temp_dir = scoped_temp_dir.GetPath(); ASSERT_TRUE(zip::Unzip(zip_file, temp_dir)); - base::FilePath bar_dir = temp_dir.Append(FILE_PATH_LITERAL("bar")); + base::FilePath bar_dir = temp_dir.AppendASCII("bar"); EXPECT_TRUE(base::DirectoryExists(bar_dir)); std::string file_content; - EXPECT_TRUE(base::ReadFileToString( - temp_dir.Append(FILE_PATH_LITERAL("foo.txt")), &file_content)); + EXPECT_TRUE( + base::ReadFileToString(temp_dir.AppendASCII("foo.txt"), &file_content)); EXPECT_EQ(VirtualFileSystem::kFooContent, file_content); - EXPECT_TRUE(base::ReadFileToString( - bar_dir.Append(FILE_PATH_LITERAL("bar1.txt")), &file_content)); + EXPECT_TRUE( + base::ReadFileToString(bar_dir.AppendASCII("bar1.txt"), &file_content)); EXPECT_EQ(VirtualFileSystem::kBar1Content, file_content); - EXPECT_TRUE(base::ReadFileToString( - bar_dir.Append(FILE_PATH_LITERAL("bar2.txt")), &file_content)); + EXPECT_TRUE( + base::ReadFileToString(bar_dir.AppendASCII("bar2.txt"), &file_content)); EXPECT_EQ(VirtualFileSystem::kBar2Content, file_content); } +// Tests progress reporting while zipping files. +TEST_F(ZipTest, ZipProgress) { + base::FilePath src_dir; + ASSERT_TRUE(GetTestDataDirectory(&src_dir)); + src_dir = src_dir.AppendASCII("test"); + + base::ScopedTempDir temp_dir; + ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); + base::FilePath zip_file = temp_dir.GetPath().AppendASCII("out.zip"); + + int progress_count = 0; + zip::Progress last_progress; + + zip::ProgressCallback progress_callback = + base::BindLambdaForTesting([&](const zip::Progress& progress) { + progress_count++; + LOG(INFO) << "Progress #" << progress_count << ": " << progress; + + // Progress should only go forwards. + EXPECT_GE(progress.bytes, last_progress.bytes); + EXPECT_GE(progress.files, last_progress.files); + EXPECT_GE(progress.directories, last_progress.directories); + + last_progress = progress; + return true; + }); + + EXPECT_TRUE(zip::Zip({.src_dir = src_dir, + .dest_file = zip_file, + .progress_callback = std::move(progress_callback)})); + + EXPECT_EQ(progress_count, 14); + EXPECT_EQ(last_progress.bytes, 13546); + EXPECT_EQ(last_progress.files, 5); + EXPECT_EQ(last_progress.directories, 2); + + TestUnzipFile(zip_file, true); +} + +// Tests throttling of progress reporting while zipping files. +TEST_F(ZipTest, ZipProgressPeriod) { + base::FilePath src_dir; + ASSERT_TRUE(GetTestDataDirectory(&src_dir)); + src_dir = src_dir.AppendASCII("test"); + + base::ScopedTempDir temp_dir; + ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); + base::FilePath zip_file = temp_dir.GetPath().AppendASCII("out.zip"); + + int progress_count = 0; + zip::Progress last_progress; + + zip::ProgressCallback progress_callback = + base::BindLambdaForTesting([&](const zip::Progress& progress) { + progress_count++; + LOG(INFO) << "Progress #" << progress_count << ": " << progress; + + // Progress should only go forwards. + EXPECT_GE(progress.bytes, last_progress.bytes); + EXPECT_GE(progress.files, last_progress.files); + EXPECT_GE(progress.directories, last_progress.directories); + + last_progress = progress; + return true; + }); + + EXPECT_TRUE(zip::Zip({.src_dir = src_dir, + .dest_file = zip_file, + .progress_callback = std::move(progress_callback), + .progress_period = base::Hours(1)})); + + // We expect only 2 progress reports: the first one, and the last one. + EXPECT_EQ(progress_count, 2); + EXPECT_EQ(last_progress.bytes, 13546); + EXPECT_EQ(last_progress.files, 5); + EXPECT_EQ(last_progress.directories, 2); + + TestUnzipFile(zip_file, true); +} + +// Tests cancellation while zipping files. +TEST_F(ZipTest, ZipCancel) { + base::FilePath src_dir; + ASSERT_TRUE(GetTestDataDirectory(&src_dir)); + src_dir = src_dir.AppendASCII("test"); + + base::ScopedTempDir temp_dir; + ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); + base::FilePath zip_file = temp_dir.GetPath().AppendASCII("out.zip"); + + // First: establish the number of possible interruption points. + int progress_count = 0; + + EXPECT_TRUE(zip::Zip({.src_dir = src_dir, + .dest_file = zip_file, + .progress_callback = base::BindLambdaForTesting( + [&progress_count](const zip::Progress&) { + progress_count++; + return true; + })})); + + EXPECT_EQ(progress_count, 14); + + // Second: exercise each and every interruption point. + for (int i = progress_count; i > 0; i--) { + int j = 0; + EXPECT_FALSE(zip::Zip({.src_dir = src_dir, + .dest_file = zip_file, + .progress_callback = base::BindLambdaForTesting( + [i, &j](const zip::Progress&) { + j++; + // Callback shouldn't be called again after + // having returned false once. + EXPECT_LE(j, i); + return j < i; + })})); + + EXPECT_EQ(j, i); + } +} + +// Tests zip::internal::GetCompressionMethod() +TEST_F(ZipTest, GetCompressionMethod) { + using zip::internal::GetCompressionMethod; + using zip::internal::kDeflated; + using zip::internal::kStored; + + EXPECT_EQ(GetCompressionMethod(FP("")), kDeflated); + EXPECT_EQ(GetCompressionMethod(FP("NoExtension")), kDeflated); + EXPECT_EQ(GetCompressionMethod(FP("Folder.zip").Append(FP("NoExtension"))), + kDeflated); + EXPECT_EQ(GetCompressionMethod(FP("Name.txt")), kDeflated); + EXPECT_EQ(GetCompressionMethod(FP("Name.zip")), kStored); + EXPECT_EQ(GetCompressionMethod(FP("Name....zip")), kStored); + EXPECT_EQ(GetCompressionMethod(FP("Name.zip")), kStored); + EXPECT_EQ(GetCompressionMethod(FP("NAME.ZIP")), kStored); + EXPECT_EQ(GetCompressionMethod(FP("Name.gz")), kStored); + EXPECT_EQ(GetCompressionMethod(FP("Name.tar.gz")), kStored); + EXPECT_EQ(GetCompressionMethod(FP("Name.tar")), kDeflated); + + // This one is controversial. + EXPECT_EQ(GetCompressionMethod(FP(".zip")), kStored); +} + +// Tests that files put inside a ZIP are effectively compressed. +TEST_F(ZipTest, Compressed) { + base::ScopedTempDir temp_dir; + ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); + + const base::FilePath src_dir = temp_dir.GetPath().AppendASCII("input"); + EXPECT_TRUE(base::CreateDirectory(src_dir)); + + // Create some dummy source files. + for (const base::StringPiece s : {"foo", "bar.txt", ".hidden"}) { + base::File f(src_dir.AppendASCII(s), + base::File::FLAG_CREATE | base::File::FLAG_WRITE); + ASSERT_TRUE(f.SetLength(5000)); + } + + // Zip the source files. + const base::FilePath dest_file = temp_dir.GetPath().AppendASCII("dest.zip"); + EXPECT_TRUE(zip::Zip({.src_dir = src_dir, + .dest_file = dest_file, + .include_hidden_files = true})); + + // Since the source files compress well, the destination ZIP file should be + // smaller than the source files. + int64_t dest_file_size; + ASSERT_TRUE(base::GetFileSize(dest_file, &dest_file_size)); + EXPECT_GT(dest_file_size, 300); + EXPECT_LT(dest_file_size, 1000); +} + +// Tests that a ZIP put inside a ZIP is simply stored instead of being +// compressed. +TEST_F(ZipTest, NestedZip) { + base::ScopedTempDir temp_dir; + ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); + + const base::FilePath src_dir = temp_dir.GetPath().AppendASCII("input"); + EXPECT_TRUE(base::CreateDirectory(src_dir)); + + // Create a dummy ZIP file. This is not a valid ZIP file, but for the purpose + // of this test, it doesn't really matter. + const int64_t src_size = 5000; + + { + base::File f(src_dir.AppendASCII("src.zip"), + base::File::FLAG_CREATE | base::File::FLAG_WRITE); + ASSERT_TRUE(f.SetLength(src_size)); + } + + // Zip the dummy ZIP file. + const base::FilePath dest_file = temp_dir.GetPath().AppendASCII("dest.zip"); + EXPECT_TRUE(zip::Zip({.src_dir = src_dir, .dest_file = dest_file})); + + // Since the dummy source (inner) ZIP file should simply be stored in the + // destination (outer) ZIP file, the destination file should be bigger than + // the source file, but not much bigger. + int64_t dest_file_size; + ASSERT_TRUE(base::GetFileSize(dest_file, &dest_file_size)); + EXPECT_GT(dest_file_size, src_size + 100); + EXPECT_LT(dest_file_size, src_size + 300); +} + +// Tests that there is no 2GB or 4GB limits. Tests that big files can be zipped +// (crbug.com/1207737) and that big ZIP files can be created +// (crbug.com/1221447). +TEST_F(ZipTest, DISABLED_BigFile) { + base::ScopedTempDir temp_dir; + ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); + + const base::FilePath src_dir = temp_dir.GetPath().AppendASCII("input"); + EXPECT_TRUE(base::CreateDirectory(src_dir)); + + // Create a big dummy ZIP file. This is not a valid ZIP file, but for the + // purpose of this test, it doesn't really matter. + const int64_t src_size = 5'000'000'000; + + { + base::File f(src_dir.AppendASCII("src.zip"), + base::File::FLAG_CREATE | base::File::FLAG_WRITE); + ASSERT_TRUE(f.SetLength(src_size)); + } + + // Zip the dummy ZIP file. + const base::FilePath dest_file = temp_dir.GetPath().AppendASCII("dest.zip"); + EXPECT_TRUE(zip::Zip({.src_dir = src_dir, .dest_file = dest_file})); + + // Since the dummy source (inner) ZIP file should simply be stored in the + // destination (outer) ZIP file, the destination file should be bigger than + // the source file, but not much bigger. + int64_t dest_file_size; + ASSERT_TRUE(base::GetFileSize(dest_file, &dest_file_size)); + EXPECT_GT(dest_file_size, src_size + 100); + EXPECT_LT(dest_file_size, src_size + 300); +} + } // namespace diff --git a/deps/zlib/google/zip_writer.cc b/deps/zlib/google/zip_writer.cc index 1f2f073b30cb32..201f1997b5b4fe 100644 --- a/deps/zlib/google/zip_writer.cc +++ b/deps/zlib/google/zip_writer.cc @@ -4,200 +4,316 @@ #include "third_party/zlib/google/zip_writer.h" +#include + #include "base/files/file.h" +#include "base/logging.h" +#include "base/strings/strcat.h" #include "base/strings/string_util.h" #include "third_party/zlib/google/zip_internal.h" namespace zip { namespace internal { -namespace { +class Redact { + public: + explicit Redact(const base::FilePath& path) : path_(path) {} + + friend std::ostream& operator<<(std::ostream& out, const Redact&& r) { + return LOG_IS_ON(INFO) ? out << "'" << r.path_ << "'" : out << "(redacted)"; + } + + private: + const base::FilePath& path_; +}; + +bool ZipWriter::ShouldContinue() { + if (!progress_callback_) + return true; + + const base::TimeTicks now = base::TimeTicks::Now(); + if (next_progress_report_time_ > now) + return true; + + next_progress_report_time_ = now + progress_period_; + if (progress_callback_.Run(progress_)) + return true; -// Numbers of pending entries that trigger writting them to the ZIP file. -constexpr size_t kMaxPendingEntriesCount = 50; + LOG(ERROR) << "Cancelling ZIP creation"; + return false; +} -bool AddFileContentToZip(zipFile zip_file, - base::File file, - const base::FilePath& file_path) { - int num_bytes; +bool ZipWriter::AddFileContent(const base::FilePath& path, base::File file) { char buf[zip::internal::kZipBufSize]; - do { - num_bytes = file.ReadAtCurrentPos(buf, zip::internal::kZipBufSize); - if (num_bytes > 0) { - if (zipWriteInFileInZip(zip_file, buf, num_bytes) != ZIP_OK) { - DLOG(ERROR) << "Could not write data to zip for path " - << file_path.value(); - return false; - } + while (ShouldContinue()) { + const int num_bytes = + file.ReadAtCurrentPos(buf, zip::internal::kZipBufSize); + + if (num_bytes < 0) { + PLOG(ERROR) << "Cannot read file " << Redact(path); + return false; } - } while (num_bytes > 0); - return true; + if (num_bytes == 0) + return true; + + if (zipWriteInFileInZip(zip_file_, buf, num_bytes) != ZIP_OK) { + PLOG(ERROR) << "Cannot write data from file " << Redact(path) + << " to ZIP"; + return false; + } + + progress_.bytes += num_bytes; + } + + return false; } -bool OpenNewFileEntry(zipFile zip_file, - const base::FilePath& path, - bool is_directory, - base::Time last_modified) { +bool ZipWriter::OpenNewFileEntry(const base::FilePath& path, + bool is_directory, + base::Time last_modified) { std::string str_path = path.AsUTF8Unsafe(); + #if defined(OS_WIN) base::ReplaceSubstringsAfterOffset(&str_path, 0u, "\\", "/"); #endif - if (is_directory) + + Compression compression = kDeflated; + + if (is_directory) { str_path += "/"; + } else { + compression = GetCompressionMethod(path); + } - return zip::internal::ZipOpenNewFileInZip(zip_file, str_path, last_modified); + return zip::internal::ZipOpenNewFileInZip(zip_file_, str_path, last_modified, + compression); } -bool CloseNewFileEntry(zipFile zip_file) { - return zipCloseFileInZip(zip_file) == ZIP_OK; +bool ZipWriter::CloseNewFileEntry() { + return zipCloseFileInZip(zip_file_) == ZIP_OK; } -bool AddFileEntryToZip(zipFile zip_file, - const base::FilePath& path, - base::File file) { - base::File::Info file_info; - if (!file.GetInfo(&file_info)) +bool ZipWriter::AddFileEntry(const base::FilePath& path, base::File file) { + base::File::Info info; + if (!file.GetInfo(&info)) return false; - if (!OpenNewFileEntry(zip_file, path, /*is_directory=*/false, - file_info.last_modified)) + if (!OpenNewFileEntry(path, /*is_directory=*/false, info.last_modified)) return false; - bool success = AddFileContentToZip(zip_file, std::move(file), path); - if (!CloseNewFileEntry(zip_file)) + if (!AddFileContent(path, std::move(file))) return false; - return success; + progress_.files++; + return CloseNewFileEntry(); } -bool AddDirectoryEntryToZip(zipFile zip_file, - const base::FilePath& path, - base::Time last_modified) { - return OpenNewFileEntry(zip_file, path, /*is_directory=*/true, - last_modified) && - CloseNewFileEntry(zip_file); -} +bool ZipWriter::AddDirectoryEntry(const base::FilePath& path) { + FileAccessor::Info info; + if (!file_accessor_->GetInfo(path, &info) || !info.is_directory) { + LOG(ERROR) << "Not a directory: " << Redact(path); + progress_.errors++; + return continue_on_error_; + } + + if (!OpenNewFileEntry(path, /*is_directory=*/true, info.last_modified)) + return false; + + if (!CloseNewFileEntry()) + return false; + + progress_.directories++; + if (!ShouldContinue()) + return false; -} // namespace + if (!recursive_) + return true; + + return AddDirectoryContents(path); +} #if defined(OS_POSIX) // static std::unique_ptr ZipWriter::CreateWithFd( int zip_file_fd, - const base::FilePath& root_dir, FileAccessor* file_accessor) { DCHECK(zip_file_fd != base::kInvalidPlatformFile); zipFile zip_file = internal::OpenFdForZipping(zip_file_fd, APPEND_STATUS_CREATE); + if (!zip_file) { - DLOG(ERROR) << "Couldn't create ZIP file for FD " << zip_file_fd; + DLOG(ERROR) << "Cannot create ZIP file for FD " << zip_file_fd; return nullptr; } - return std::unique_ptr( - new ZipWriter(zip_file, root_dir, file_accessor)); + + return std::unique_ptr(new ZipWriter(zip_file, file_accessor)); } #endif // static std::unique_ptr ZipWriter::Create( const base::FilePath& zip_file_path, - const base::FilePath& root_dir, FileAccessor* file_accessor) { DCHECK(!zip_file_path.empty()); zipFile zip_file = internal::OpenForZipping(zip_file_path.AsUTF8Unsafe(), APPEND_STATUS_CREATE); + if (!zip_file) { - DLOG(ERROR) << "Couldn't create ZIP file at path " << zip_file_path; + PLOG(ERROR) << "Cannot create ZIP file " << Redact(zip_file_path); return nullptr; } - return std::unique_ptr( - new ZipWriter(zip_file, root_dir, file_accessor)); -} - -ZipWriter::ZipWriter(zipFile zip_file, - const base::FilePath& root_dir, - FileAccessor* file_accessor) - : zip_file_(zip_file), root_dir_(root_dir), file_accessor_(file_accessor) {} -ZipWriter::~ZipWriter() { - DCHECK(pending_entries_.empty()); + return std::unique_ptr(new ZipWriter(zip_file, file_accessor)); } -bool ZipWriter::WriteEntries(const std::vector& paths) { - return AddEntries(paths) && Close(); -} +ZipWriter::ZipWriter(zipFile zip_file, FileAccessor* file_accessor) + : zip_file_(zip_file), file_accessor_(file_accessor) {} -bool ZipWriter::AddEntries(const std::vector& paths) { - DCHECK(zip_file_); - pending_entries_.insert(pending_entries_.end(), paths.begin(), paths.end()); - return FlushEntriesIfNeeded(/*force=*/false); +ZipWriter::~ZipWriter() { + if (zip_file_) + zipClose(zip_file_, nullptr); } bool ZipWriter::Close() { - bool success = FlushEntriesIfNeeded(/*force=*/true) && - zipClose(zip_file_, nullptr) == ZIP_OK; + const bool success = zipClose(zip_file_, nullptr) == ZIP_OK; zip_file_ = nullptr; + + // Call the progress callback one last time with the final progress status. + if (progress_callback_ && !progress_callback_.Run(progress_)) { + LOG(ERROR) << "Cancelling ZIP creation at the end"; + return false; + } + return success; } -bool ZipWriter::FlushEntriesIfNeeded(bool force) { - if (pending_entries_.size() < kMaxPendingEntriesCount && !force) - return true; +bool ZipWriter::AddMixedEntries(Paths paths) { + // Pointers to directory paths in |paths|. + std::vector directories; - while (pending_entries_.size() >= kMaxPendingEntriesCount || - (force && !pending_entries_.empty())) { - size_t entry_count = - std::min(pending_entries_.size(), kMaxPendingEntriesCount); - std::vector relative_paths; - std::vector absolute_paths; - relative_paths.insert(relative_paths.begin(), pending_entries_.begin(), - pending_entries_.begin() + entry_count); - for (auto iter = pending_entries_.begin(); - iter != pending_entries_.begin() + entry_count; ++iter) { - // The FileAccessor requires absolute paths. - absolute_paths.push_back(root_dir_.Append(*iter)); - } - pending_entries_.erase(pending_entries_.begin(), - pending_entries_.begin() + entry_count); - - // We don't know which paths are files and which ones are directories, and - // we want to avoid making a call to file_accessor_ for each entry. Open the - // files instead, invalid files are returned for directories. - std::vector files = - file_accessor_->OpenFilesForReading(absolute_paths); - DCHECK_EQ(files.size(), relative_paths.size()); - for (size_t i = 0; i < files.size(); i++) { + // Declared outside loop to reuse internal buffer. + std::vector files; + + // First pass. We don't know which paths are files and which ones are + // directories, and we want to avoid making a call to file_accessor_ for each + // path. Try to open all of the paths as files. We'll get invalid file + // descriptors for directories, and we'll process these directories in the + // second pass. + while (!paths.empty()) { + // Work with chunks of 50 paths at most. + const size_t n = std::min(paths.size(), 50); + const Paths relative_paths = paths.subspan(0, n); + paths = paths.subspan(n, paths.size() - n); + + files.clear(); + if (!file_accessor_->Open(relative_paths, &files) || files.size() != n) + return false; + + for (size_t i = 0; i < n; i++) { const base::FilePath& relative_path = relative_paths[i]; - const base::FilePath& absolute_path = absolute_paths[i]; - base::File file = std::move(files[i]); + DCHECK(!relative_path.empty()); + base::File& file = files[i]; + if (file.IsValid()) { - if (!AddFileEntryToZip(zip_file_, relative_path, std::move(file))) { - LOG(ERROR) << "Failed to write file " << relative_path.value() - << " to ZIP file."; + // It's a file. + if (!AddFileEntry(relative_path, std::move(file))) return false; - } } else { - // Missing file or directory case. - base::Time last_modified = - file_accessor_->GetLastModifiedTime(absolute_path); - if (last_modified.is_null()) { - LOG(ERROR) << "Failed to write entry " << relative_path.value() - << " to ZIP file."; - return false; - } - DCHECK(file_accessor_->DirectoryExists(absolute_path)); - if (!AddDirectoryEntryToZip(zip_file_, relative_path, last_modified)) { - LOG(ERROR) << "Failed to write directory " << relative_path.value() - << " to ZIP file."; - return false; - } + // It's probably a directory. Remember its path for the second pass. + directories.push_back(&relative_path); } } } + + // Second pass for directories discovered during the first pass. + for (const base::FilePath* const path : directories) { + DCHECK(path); + if (!AddDirectoryEntry(*path)) + return false; + } + return true; } +bool ZipWriter::AddFileEntries(Paths paths) { + // Declared outside loop to reuse internal buffer. + std::vector files; + + while (!paths.empty()) { + // Work with chunks of 50 paths at most. + const size_t n = std::min(paths.size(), 50); + const Paths relative_paths = paths.subspan(0, n); + paths = paths.subspan(n, paths.size() - n); + + DCHECK_EQ(relative_paths.size(), n); + + files.clear(); + if (!file_accessor_->Open(relative_paths, &files) || files.size() != n) + return false; + + for (size_t i = 0; i < n; i++) { + const base::FilePath& relative_path = relative_paths[i]; + DCHECK(!relative_path.empty()); + base::File& file = files[i]; + + if (!file.IsValid()) { + LOG(ERROR) << "Cannot open " << Redact(relative_path); + progress_.errors++; + + if (continue_on_error_) + continue; + + return false; + } + + if (!AddFileEntry(relative_path, std::move(file))) + return false; + } + } + + return true; +} + +bool ZipWriter::AddDirectoryEntries(Paths paths) { + for (const base::FilePath& path : paths) { + if (!AddDirectoryEntry(path)) + return false; + } + + return true; +} + +bool ZipWriter::AddDirectoryContents(const base::FilePath& path) { + std::vector files, subdirs; + + if (!file_accessor_->List(path, &files, &subdirs)) { + progress_.errors++; + return continue_on_error_; + } + + Filter(&files); + Filter(&subdirs); + + if (!AddFileEntries(files)) + return false; + + return AddDirectoryEntries(subdirs); +} + +void ZipWriter::Filter(std::vector* const paths) { + DCHECK(paths); + + if (!filter_callback_) + return; + + const auto end = std::remove_if(paths->begin(), paths->end(), + [this](const base::FilePath& path) { + return !filter_callback_.Run(path); + }); + paths->erase(end, paths->end()); +} + } // namespace internal } // namespace zip diff --git a/deps/zlib/google/zip_writer.h b/deps/zlib/google/zip_writer.h index bd2a727b94d408..fcc96275004af6 100644 --- a/deps/zlib/google/zip_writer.h +++ b/deps/zlib/google/zip_writer.h @@ -9,6 +9,7 @@ #include #include "base/files/file_path.h" +#include "base/time/time.h" #include "build/build_config.h" #include "third_party/zlib/google/zip.h" @@ -28,64 +29,126 @@ namespace internal { // performance reasons as these calls may be expensive when IPC based). // This class is so far internal and only used by zip.cc, but could be made // public if needed. +// +// All methods returning a bool return true on success and false on error. class ZipWriter { public: -// Creates a writer that will write a ZIP file to |zip_file_fd|/|zip_file| -// and which entries (specifies with AddEntries) are relative to |root_dir|. +// Creates a writer that will write a ZIP file to |zip_file_fd| or |zip_file| +// and which entries are relative to |file_accessor|'s source directory. // All file reads are performed using |file_accessor|. #if defined(OS_POSIX) static std::unique_ptr CreateWithFd(int zip_file_fd, - const base::FilePath& root_dir, FileAccessor* file_accessor); #endif + static std::unique_ptr Create(const base::FilePath& zip_file, - const base::FilePath& root_dir, FileAccessor* file_accessor); + + ZipWriter(const ZipWriter&) = delete; + ZipWriter& operator=(const ZipWriter&) = delete; + ~ZipWriter(); - // Writes the files at |paths| to the ZIP file and closes this Zip file. - // Note that the the FilePaths must be relative to |root_dir| specified in the - // Create method. - // Returns true if all entries were written successfuly. - bool WriteEntries(const std::vector& paths); + // Sets the optional progress callback. The callback is called once for each + // time |period|. The final callback is always called when the ZIP operation + // completes. + void SetProgressCallback(ProgressCallback callback, base::TimeDelta period) { + progress_callback_ = std::move(callback); + progress_period_ = std::move(period); + } + + // Should ignore missing files and directories? + void ContinueOnError(bool continue_on_error) { + continue_on_error_ = continue_on_error; + } + + // Sets the recursive flag, indicating whether the contents of subdirectories + // should be included. + void SetRecursive(bool b) { recursive_ = b; } + + // Sets the filter callback. + void SetFilterCallback(FilterCallback callback) { + filter_callback_ = std::move(callback); + } + + // Adds the contents of a directory. If the recursive flag is set, the + // contents of subdirectories are also added. + bool AddDirectoryContents(const base::FilePath& path); + + // Adds the entries at |paths| to the ZIP file. These can be a mixed bag of + // files and directories. If the recursive flag is set, the contents of + // subdirectories is also added. + bool AddMixedEntries(Paths paths); + + // Closes the ZIP file. + bool Close(); private: - ZipWriter(zipFile zip_file, - const base::FilePath& root_dir, - FileAccessor* file_accessor); + // Takes ownership of |zip_file|. + ZipWriter(zipFile zip_file, FileAccessor* file_accessor); - // Writes the pending entries to the ZIP file if there are at least - // |kMaxPendingEntriesCount| of them. If |force| is true, all pending entries - // are written regardless of how many there are. - // Returns false if writing an entry fails, true if no entry was written or - // there was no error writing entries. - bool FlushEntriesIfNeeded(bool force); + // Regularly called during processing to check whether zipping should continue + // or should be cancelled. + bool ShouldContinue(); - // Adds the files at |paths| to the ZIP file. These FilePaths must be relative - // to |root_dir| specified in the Create method. - bool AddEntries(const std::vector& paths); + // Adds file content to currently open file entry. + bool AddFileContent(const base::FilePath& path, base::File file); - // Closes the ZIP file. - // Returns true if successful, false otherwise (typically if an entry failed - // to be written). - bool Close(); + // Adds a file entry (including file contents). + bool AddFileEntry(const base::FilePath& path, base::File file); + + // Adds file entries. All the paths should be existing files. + bool AddFileEntries(Paths paths); + + // Adds a directory entry. If the recursive flag is set, the contents of this + // directory are also added. + bool AddDirectoryEntry(const base::FilePath& path); - // The entries that have been added but not yet written to the ZIP file. - std::vector pending_entries_; + // Adds directory entries. All the paths should be existing directories. If + // the recursive flag is set, the contents of these directories are also + // added. + bool AddDirectoryEntries(Paths paths); + + // Opens a file or directory entry. + bool OpenNewFileEntry(const base::FilePath& path, + bool is_directory, + base::Time last_modified); + + // Closes the currently open entry. + bool CloseNewFileEntry(); + + // Filters entries. + void Filter(std::vector* paths); // The actual zip file. zipFile zip_file_; - // Path to the directory entry paths are relative to. - base::FilePath root_dir_; - // Abstraction over file access methods used to read files. - FileAccessor* file_accessor_; + FileAccessor* const file_accessor_; + + // Progress stats. + Progress progress_; + + // Optional progress callback. + ProgressCallback progress_callback_; + + // Optional progress reporting period. + base::TimeDelta progress_period_; + + // Next time to report progress. + base::TimeTicks next_progress_report_time_ = base::TimeTicks::Now(); + + // Filter used to exclude files from the ZIP file. + FilterCallback filter_callback_; + + // Should recursively add directories? + bool recursive_ = false; - DISALLOW_COPY_AND_ASSIGN(ZipWriter); + // Should ignore missing files and directories? + bool continue_on_error_ = false; }; } // namespace internal } // namespace zip -#endif // THIRD_PARTY_ZLIB_GOOGLE_ZIP_WRITER_H_ \ No newline at end of file +#endif // THIRD_PARTY_ZLIB_GOOGLE_ZIP_WRITER_H_ diff --git a/deps/zlib/patches/0003-uninitializedjump.patch b/deps/zlib/patches/0003-uninitializedjump.patch new file mode 100644 index 00000000000000..7aae3238a5011a --- /dev/null +++ b/deps/zlib/patches/0003-uninitializedjump.patch @@ -0,0 +1,15 @@ +diff --git a/third_party/zlib/deflate.c b/third_party/zlib/deflate.c +index a39e62787862..c6053fd1c7ea 100644 +--- a/third_party/zlib/deflate.c ++++ b/third_party/zlib/deflate.c +@@ -318,6 +318,10 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, + s->w_size + window_padding, + 2*sizeof(Byte)); + s->prev = (Posf *) ZALLOC(strm, s->w_size, sizeof(Pos)); ++ /* Avoid use of uninitialized value, see: ++ * https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=11360 ++ */ ++ zmemzero(s->prev, s->w_size * sizeof(Pos)); + s->head = (Posf *) ZALLOC(strm, s->hash_size, sizeof(Pos)); + + s->high_water = 0; /* nothing written to s->window yet */ diff --git a/deps/zlib/patches/0004-fix-uwp.patch b/deps/zlib/patches/0004-fix-uwp.patch new file mode 100644 index 00000000000000..23145a7ae5357f --- /dev/null +++ b/deps/zlib/patches/0004-fix-uwp.patch @@ -0,0 +1,22 @@ +diff --git a/third_party/zlib/contrib/minizip/iowin32.c b/third_party/zlib/contrib/minizip/iowin32.c +index 246ceb91a139..c6bc314b3c28 100644 +--- a/third_party/zlib/contrib/minizip/iowin32.c ++++ b/third_party/zlib/contrib/minizip/iowin32.c +@@ -31,14 +31,12 @@ + #define _WIN32_WINNT 0x601 + #endif + +-#if _WIN32_WINNT >= _WIN32_WINNT_WIN8 +-// see Include/shared/winapifamily.h in the Windows Kit +-#if defined(WINAPI_FAMILY_PARTITION) && (!(defined(IOWIN32_USING_WINRT_API))) +-#if WINAPI_FAMILY_ONE_PARTITION(WINAPI_FAMILY, WINAPI_PARTITION_APP) ++#if !defined(IOWIN32_USING_WINRT_API) ++#if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP) ++// Windows Store or Universal Windows Platform + #define IOWIN32_USING_WINRT_API 1 + #endif + #endif +-#endif + + voidpf ZCALLBACK win32_open_file_func OF((voidpf opaque, const char* filename, int mode)); + uLong ZCALLBACK win32_read_file_func OF((voidpf opaque, voidpf stream, void* buf, uLong size)); diff --git a/deps/zlib/patches/0005-infcover-gtest.patch b/deps/zlib/patches/0005-infcover-gtest.patch new file mode 100644 index 00000000000000..f5443bd7d88e1c --- /dev/null +++ b/deps/zlib/patches/0005-infcover-gtest.patch @@ -0,0 +1,405 @@ +From 409594639f15d825202971db7a275023e05772ff Mon Sep 17 00:00:00 2001 +From: Adenilson Cavalcanti +Date: Tue, 28 Apr 2020 10:48:01 -0700 +Subject: [PATCH] Local Changes: - make C tests build as C++ code so we can + use gtest. - use gtest EXPECT_TRUE instead of C assert. - replace C + streams for C++ (portability issues). + +--- + test/infcover.c | 167 ++++++++++++++++++++++++++---------------------- + 1 file changed, 90 insertions(+), 77 deletions(-) + +diff --git a/test/infcover.c b/test/infcover.c +index 2be0164..a8c51c7 100644 +--- a/test/infcover.c ++++ b/test/infcover.c +@@ -4,11 +4,12 @@ + */ + + /* to use, do: ./configure --cover && make cover */ +- ++// clang-format off ++#include "infcover.h" + #include + #include + #include +-#include ++ + #include "zlib.h" + + /* get definition of internal structure so we can mess with it (see pull()), +@@ -17,8 +18,22 @@ + #include "inftrees.h" + #include "inflate.h" + ++/* XXX: use C++ streams instead of printf/fputs/etc due to portability ++ * as type sizes can vary between platforms. ++ */ ++#include + #define local static + ++/* XXX: hacking C assert and plugging into GTest. */ ++#include "gtest.h" ++#if defined(assert) ++#undef assert ++#define assert EXPECT_TRUE ++#endif ++ ++/* XXX: handle what is a reserved word in C++. */ ++#define try try_f ++ + /* -- memory tracking routines -- */ + + /* +@@ -72,7 +87,7 @@ local void *mem_alloc(void *mem, unsigned count, unsigned size) + { + void *ptr; + struct mem_item *item; +- struct mem_zone *zone = mem; ++ struct mem_zone *zone = static_cast(mem); + size_t len = count * (size_t)size; + + /* induced allocation failure */ +@@ -87,7 +102,7 @@ local void *mem_alloc(void *mem, unsigned count, unsigned size) + memset(ptr, 0xa5, len); + + /* create a new item for the list */ +- item = malloc(sizeof(struct mem_item)); ++ item = static_cast(malloc(sizeof(struct mem_item))); + if (item == NULL) { + free(ptr); + return NULL; +@@ -112,7 +127,7 @@ local void *mem_alloc(void *mem, unsigned count, unsigned size) + local void mem_free(void *mem, void *ptr) + { + struct mem_item *item, *next; +- struct mem_zone *zone = mem; ++ struct mem_zone *zone = static_cast(mem); + + /* if no zone, just do a free */ + if (zone == NULL) { +@@ -159,7 +174,7 @@ local void mem_setup(z_stream *strm) + { + struct mem_zone *zone; + +- zone = malloc(sizeof(struct mem_zone)); ++ zone = static_cast(malloc(sizeof(struct mem_zone))); + assert(zone != NULL); + zone->first = NULL; + zone->total = 0; +@@ -175,33 +190,33 @@ local void mem_setup(z_stream *strm) + /* set a limit on the total memory allocation, or 0 to remove the limit */ + local void mem_limit(z_stream *strm, size_t limit) + { +- struct mem_zone *zone = strm->opaque; ++ struct mem_zone *zone = static_cast(strm->opaque); + + zone->limit = limit; + } + + /* show the current total requested allocations in bytes */ +-local void mem_used(z_stream *strm, char *prefix) ++local void mem_used(z_stream *strm, const char *prefix) + { +- struct mem_zone *zone = strm->opaque; ++ struct mem_zone *zone = static_cast(strm->opaque); + +- fprintf(stderr, "%s: %lu allocated\n", prefix, zone->total); ++ std::cout << prefix << ": " << zone->total << " allocated" << std::endl; + } + + /* show the high water allocation in bytes */ +-local void mem_high(z_stream *strm, char *prefix) ++local void mem_high(z_stream *strm, const char *prefix) + { +- struct mem_zone *zone = strm->opaque; ++ struct mem_zone *zone = static_cast(strm->opaque); + +- fprintf(stderr, "%s: %lu high water mark\n", prefix, zone->highwater); ++ std::cout << prefix << ": " << zone->highwater << " high water mark" << std::endl; + } + + /* release the memory allocation zone -- if there are any surprises, notify */ +-local void mem_done(z_stream *strm, char *prefix) ++local void mem_done(z_stream *strm, const char *prefix) + { + int count = 0; + struct mem_item *item, *next; +- struct mem_zone *zone = strm->opaque; ++ struct mem_zone *zone = static_cast(strm->opaque); + + /* show high water mark */ + mem_high(strm, prefix); +@@ -218,13 +233,20 @@ local void mem_done(z_stream *strm, char *prefix) + + /* issue alerts about anything unexpected */ + if (count || zone->total) +- fprintf(stderr, "** %s: %lu bytes in %d blocks not freed\n", +- prefix, zone->total, count); ++ std::cout << "** " << prefix << ": " ++ << zone->total << " bytes in " ++ << count << " blocks not freed" ++ << std::endl; ++ + if (zone->notlifo) +- fprintf(stderr, "** %s: %d frees not LIFO\n", prefix, zone->notlifo); ++ std::cout << "** " << prefix << ": " ++ << zone->notlifo << " frees not LIFO" ++ << std::endl; ++ + if (zone->rogue) +- fprintf(stderr, "** %s: %d frees not recognized\n", +- prefix, zone->rogue); ++ std::cout << "** " << prefix << ": " ++ << zone->rogue << " frees not recognized" ++ << std::endl; + + /* free the zone and delete from the stream */ + free(zone); +@@ -247,7 +269,7 @@ local unsigned char *h2b(const char *hex, unsigned *len) + unsigned char *in, *re; + unsigned next, val; + +- in = malloc((strlen(hex) + 1) >> 1); ++ in = static_cast(malloc((strlen(hex) + 1) >> 1)); + if (in == NULL) + return NULL; + next = 0; +@@ -268,7 +290,7 @@ local unsigned char *h2b(const char *hex, unsigned *len) + } while (*hex++); /* go through the loop with the terminating null */ + if (len != NULL) + *len = next; +- re = realloc(in, next); ++ re = static_cast(realloc(in, next)); + return re == NULL ? in : re; + } + +@@ -281,7 +303,7 @@ local unsigned char *h2b(const char *hex, unsigned *len) + header information is collected with inflateGetHeader(). If a zlib stream + is looking for a dictionary, then an empty dictionary is provided. + inflate() is run until all of the input data is consumed. */ +-local void inf(char *hex, char *what, unsigned step, int win, unsigned len, ++local void inf(const char *hex, const char *what, unsigned step, int win, unsigned len, + int err) + { + int ret; +@@ -298,7 +320,7 @@ local void inf(char *hex, char *what, unsigned step, int win, unsigned len, + mem_done(&strm, what); + return; + } +- out = malloc(len); assert(out != NULL); ++ out = static_cast(malloc(len)); assert(out != NULL); + if (win == 47) { + head.extra = out; + head.extra_max = len; +@@ -347,7 +369,7 @@ local void inf(char *hex, char *what, unsigned step, int win, unsigned len, + } + + /* cover all of the lines in inflate.c up to inflate() */ +-local void cover_support(void) ++void cover_support(void) + { + int ret; + z_stream strm; +@@ -381,11 +403,11 @@ local void cover_support(void) + strm.next_in = Z_NULL; + ret = inflateInit(&strm); assert(ret == Z_OK); + ret = inflateEnd(&strm); assert(ret == Z_OK); +- fputs("inflate built-in memory routines\n", stderr); ++ std::cout << "inflate built-in memory routines" << std::endl;; + } + + /* cover all inflate() header and trailer cases and code after inflate() */ +-local void cover_wrap(void) ++void cover_wrap(void) + { + int ret; + z_stream strm, copy; +@@ -394,7 +416,7 @@ local void cover_wrap(void) + ret = inflate(Z_NULL, 0); assert(ret == Z_STREAM_ERROR); + ret = inflateEnd(Z_NULL); assert(ret == Z_STREAM_ERROR); + ret = inflateCopy(Z_NULL, Z_NULL); assert(ret == Z_STREAM_ERROR); +- fputs("inflate bad parameters\n", stderr); ++ std::cout << "inflate bad parameters" << std::endl; + + inf("1f 8b 0 0", "bad gzip method", 0, 31, 0, Z_DATA_ERROR); + inf("1f 8b 8 80", "bad gzip flags", 0, 31, 0, Z_DATA_ERROR); +@@ -415,9 +437,9 @@ local void cover_wrap(void) + strm.next_in = Z_NULL; + ret = inflateInit2(&strm, -8); + strm.avail_in = 2; +- strm.next_in = (void *)"\x63"; ++ strm.next_in = (Bytef *)"\x63"; + strm.avail_out = 1; +- strm.next_out = (void *)&ret; ++ strm.next_out = (Bytef *)&ret; + mem_limit(&strm, 1); + ret = inflate(&strm, Z_NO_FLUSH); assert(ret == Z_MEM_ERROR); + ret = inflate(&strm, Z_NO_FLUSH); assert(ret == Z_MEM_ERROR); +@@ -428,11 +450,11 @@ local void cover_wrap(void) + mem_limit(&strm, (sizeof(struct inflate_state) << 1) + 256); + ret = inflatePrime(&strm, 16, 0); assert(ret == Z_OK); + strm.avail_in = 2; +- strm.next_in = (void *)"\x80"; ++ strm.next_in = (Bytef *)"\x80"; + ret = inflateSync(&strm); assert(ret == Z_DATA_ERROR); + ret = inflate(&strm, Z_NO_FLUSH); assert(ret == Z_STREAM_ERROR); + strm.avail_in = 4; +- strm.next_in = (void *)"\0\0\xff\xff"; ++ strm.next_in = (Bytef *)"\0\0\xff\xff"; + ret = inflateSync(&strm); assert(ret == Z_OK); + (void)inflateSyncPoint(&strm); + ret = inflateCopy(©, &strm); assert(ret == Z_MEM_ERROR); +@@ -454,7 +476,7 @@ local unsigned pull(void *desc, unsigned char **buf) + next = 0; + return 0; /* no input (already provided at next_in) */ + } +- state = (void *)((z_stream *)desc)->state; ++ state = reinterpret_cast(((z_stream *)desc)->state); + if (state != Z_NULL) + state->mode = SYNC; /* force an otherwise impossible situation */ + return next < sizeof(dat) ? (*buf = dat + next++, 1) : 0; +@@ -467,7 +489,7 @@ local int push(void *desc, unsigned char *buf, unsigned len) + } + + /* cover inflateBack() up to common deflate data cases and after those */ +-local void cover_back(void) ++void cover_back(void) + { + int ret; + z_stream strm; +@@ -479,17 +501,17 @@ local void cover_back(void) + ret = inflateBack(Z_NULL, Z_NULL, Z_NULL, Z_NULL, Z_NULL); + assert(ret == Z_STREAM_ERROR); + ret = inflateBackEnd(Z_NULL); assert(ret == Z_STREAM_ERROR); +- fputs("inflateBack bad parameters\n", stderr); ++ std::cout << "inflateBack bad parameters" << std::endl;; + + mem_setup(&strm); + ret = inflateBackInit(&strm, 15, win); assert(ret == Z_OK); + strm.avail_in = 2; +- strm.next_in = (void *)"\x03"; ++ strm.next_in = (Bytef *)"\x03"; + ret = inflateBack(&strm, pull, Z_NULL, push, Z_NULL); + assert(ret == Z_STREAM_END); + /* force output error */ + strm.avail_in = 3; +- strm.next_in = (void *)"\x63\x00"; ++ strm.next_in = (Bytef *)"\x63\x00"; + ret = inflateBack(&strm, pull, Z_NULL, push, &strm); + assert(ret == Z_BUF_ERROR); + /* force mode error by mucking with state */ +@@ -500,11 +522,11 @@ local void cover_back(void) + + ret = inflateBackInit(&strm, 15, win); assert(ret == Z_OK); + ret = inflateBackEnd(&strm); assert(ret == Z_OK); +- fputs("inflateBack built-in memory routines\n", stderr); ++ std::cout << "inflateBack built-in memory routines" << std::endl;; + } + + /* do a raw inflate of data in hexadecimal with both inflate and inflateBack */ +-local int try(char *hex, char *id, int err) ++local int try(const char *hex, const char *id, int err) + { + int ret; + unsigned len, size; +@@ -518,11 +540,11 @@ local int try(char *hex, char *id, int err) + + /* allocate work areas */ + size = len << 3; +- out = malloc(size); ++ out = static_cast(malloc(size)); + assert(out != NULL); +- win = malloc(32768); ++ win = static_cast(malloc(32768)); + assert(win != NULL); +- prefix = malloc(strlen(id) + 6); ++ prefix = static_cast(malloc(strlen(id) + 6)); + assert(prefix != NULL); + + /* first with inflate */ +@@ -578,7 +600,7 @@ local int try(char *hex, char *id, int err) + } + + /* cover deflate data cases in both inflate() and inflateBack() */ +-local void cover_inflate(void) ++void cover_inflate(void) + { + try("0 0 0 0 0", "invalid stored block lengths", 1); + try("3 0", "fixed", 0); +@@ -613,32 +635,33 @@ local void cover_inflate(void) + inf("63 18 5 40 c 0", "window wrap", 3, -8, 300, Z_OK); + } + ++/* XXX(cavalcantii): fix linking error due inflate_table. */ + /* cover remaining lines in inftrees.c */ +-local void cover_trees(void) +-{ +- int ret; +- unsigned bits; +- unsigned short lens[16], work[16]; +- code *next, table[ENOUGH_DISTS]; +- +- /* we need to call inflate_table() directly in order to manifest not- +- enough errors, since zlib insures that enough is always enough */ +- for (bits = 0; bits < 15; bits++) +- lens[bits] = (unsigned short)(bits + 1); +- lens[15] = 15; +- next = table; +- bits = 15; +- ret = inflate_table(DISTS, lens, 16, &next, &bits, work); +- assert(ret == 1); +- next = table; +- bits = 1; +- ret = inflate_table(DISTS, lens, 16, &next, &bits, work); +- assert(ret == 1); +- fputs("inflate_table not enough errors\n", stderr); +-} ++/* void cover_trees(void) */ ++/* { */ ++/* int ret; */ ++/* unsigned bits; */ ++/* unsigned short lens[16], work[16]; */ ++/* code *next, table[ENOUGH_DISTS]; */ ++ ++/* /\* we need to call inflate_table() directly in order to manifest not- */ ++/* enough errors, since zlib insures that enough is always enough *\/ */ ++/* for (bits = 0; bits < 15; bits++) */ ++/* lens[bits] = (unsigned short)(bits + 1); */ ++/* lens[15] = 15; */ ++/* next = table; */ ++/* bits = 15; */ ++/* ret = inflate_table(DISTS, lens, 16, &next, &bits, work); */ ++/* assert(ret == 1); */ ++/* next = table; */ ++/* bits = 1; */ ++/* ret = inflate_table(DISTS, lens, 16, &next, &bits, work); */ ++/* assert(ret == 1); */ ++/* fputs("inflate_table not enough errors\n", stderr); */ ++/* } */ + + /* cover remaining inffast.c decoding and window copying */ +-local void cover_fast(void) ++void cover_fast(void) + { + inf("e5 e0 81 ad 6d cb b2 2c c9 01 1e 59 63 ae 7d ee fb 4d fd b5 35 41 68" + " ff 7f 0f 0 0 0", "fast length extra bits", 0, -8, 258, Z_DATA_ERROR); +@@ -658,14 +681,4 @@ local void cover_fast(void) + Z_STREAM_END); + } + +-int main(void) +-{ +- fprintf(stderr, "%s\n", zlibVersion()); +- cover_support(); +- cover_wrap(); +- cover_back(); +- cover_inflate(); +- cover_trees(); +- cover_fast(); +- return 0; +-} ++// clang-format on +-- +2.21.1 (Apple Git-122.3) + diff --git a/deps/zlib/patches/0006-fix-check_match.patch b/deps/zlib/patches/0006-fix-check_match.patch new file mode 100644 index 00000000000000..b21c363b1ec1e7 --- /dev/null +++ b/deps/zlib/patches/0006-fix-check_match.patch @@ -0,0 +1,42 @@ +From 8304bdda5293ffd5b3efce8e4f54904b387029d6 Mon Sep 17 00:00:00 2001 +From: Hans Wennborg +Date: Wed, 23 Sep 2020 16:36:38 +0200 +Subject: [PATCH] Avoid crashing in check_match when prev_match == -1 + +prev_match can be set to -1 after sliding the window. In that case, the +window has slid past the first byte of the last match, which means it +cannot be compared in check_match. + +This would cause zlib to crash on some inputs to deflate when built +with ZLIB_DEBUG enabled. + +Check for this situation and avoid crashing by not trying to compare +the first byte. + +Bug: 1113142 +--- + third_party/zlib/deflate.c | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) + +diff --git a/third_party/zlib/deflate.c b/third_party/zlib/deflate.c +index cfdd2f46b230..d70732ec6fc2 100644 +--- a/third_party/zlib/deflate.c ++++ b/third_party/zlib/deflate.c +@@ -2060,7 +2060,13 @@ local block_state deflate_slow(s, flush) + uInt max_insert = s->strstart + s->lookahead - MIN_MATCH; + /* Do not insert strings in hash table beyond this. */ + +- check_match(s, s->strstart-1, s->prev_match, s->prev_length); ++ if (s->prev_match == -1) { ++ /* The window has slid one byte past the previous match, ++ * so the first byte cannot be compared. */ ++ check_match(s, s->strstart, s->prev_match+1, s->prev_length-1); ++ } else { ++ check_match(s, s->strstart-1, s->prev_match, s->prev_length); ++ } + + _tr_tally_dist(s, s->strstart -1 - s->prev_match, + s->prev_length - MIN_MATCH, bflush); +-- +2.28.0.681.g6f77f65b4e-goog + diff --git a/deps/zlib/patches/0007-zero-init-deflate-window.patch b/deps/zlib/patches/0007-zero-init-deflate-window.patch new file mode 100644 index 00000000000000..9dbbf53a8c6b6e --- /dev/null +++ b/deps/zlib/patches/0007-zero-init-deflate-window.patch @@ -0,0 +1,40 @@ +From 92537ee19784e0e545f06d89b7d89ab532a18cff Mon Sep 17 00:00:00 2001 +From: Hans Wennborg +Date: Tue, 3 Nov 2020 15:54:09 +0100 +Subject: [PATCH] [zlib] Zero-initialize the window used for deflation + +Otherwise MSan complains about use-of-uninitialized values in the +window. +This happens in both regular deflate's longest_match and deflate_rle. + +Before crrev.com/822755 we used to suppress those reports, but it seems +better to fix it properly. That will also allow us to catch other +potential issues with MSan in these functions. + +The instances of this that we've seen only reproduce with +fill_window_sse(), not with the regular fill_window() function. Since +the former doesn't exist in upstream zlib, I'm not planning to send this +patch upstream. + +Bug: 1137613, 1144420 +--- + third_party/zlib/deflate.c | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/third_party/zlib/deflate.c b/third_party/zlib/deflate.c +index 8bf93e524875..fc7ae45905ff 100644 +--- a/third_party/zlib/deflate.c ++++ b/third_party/zlib/deflate.c +@@ -321,6 +321,9 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, + s->window = (Bytef *) ZALLOC(strm, + s->w_size + window_padding, + 2*sizeof(Byte)); ++ /* Avoid use of unitialized values in the window, see crbug.com/1137613 and ++ * crbug.com/1144420 */ ++ zmemzero(s->window, (s->w_size + window_padding) * (2 * sizeof(Byte))); + s->prev = (Posf *) ZALLOC(strm, s->w_size, sizeof(Pos)); + /* Avoid use of uninitialized value, see: + * https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=11360 +-- +2.29.1.341.ge80a0c044ae-goog + diff --git a/deps/zlib/patches/0008-minizip-zip-unzip-tools.patch b/deps/zlib/patches/0008-minizip-zip-unzip-tools.patch new file mode 100644 index 00000000000000..48ceb02d4c3781 --- /dev/null +++ b/deps/zlib/patches/0008-minizip-zip-unzip-tools.patch @@ -0,0 +1,98 @@ +From 0c7de17000659f4f79de878296892c46be0aff77 Mon Sep 17 00:00:00 2001 +From: Noel Gordon +Date: Wed, 26 May 2021 21:57:43 +1000 +Subject: [PATCH] Build minizip zip and unzip tools + +--- + third_party/zlib/contrib/minizip/miniunz.c | 13 ++++++------- + third_party/zlib/contrib/minizip/minizip.c | 7 +++---- + 2 files changed, 9 insertions(+), 11 deletions(-) + +diff --git a/third_party/zlib/contrib/minizip/miniunz.c b/third_party/zlib/contrib/minizip/miniunz.c +index 3d65401be5cd..08737f689a96 100644 +--- a/third_party/zlib/contrib/minizip/miniunz.c ++++ b/third_party/zlib/contrib/minizip/miniunz.c +@@ -12,7 +12,7 @@ + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + */ + +-#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) ++#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) && (!defined(__ANDROID_API__)) + #ifndef __USE_FILE_OFFSET64 + #define __USE_FILE_OFFSET64 + #endif +@@ -27,7 +27,7 @@ + #endif + #endif + +-#ifdef __APPLE__ ++#if defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) + // In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions + #define FOPEN_FUNC(filename, mode) fopen(filename, mode) + #define FTELLO_FUNC(stream) ftello(stream) +@@ -45,6 +45,7 @@ + #include + #include + #include ++#include + + #ifdef _WIN32 + # include +@@ -97,7 +98,7 @@ void change_file_date(filename,dosdate,tmu_date) + SetFileTime(hFile,&ftm,&ftLastAcc,&ftm); + CloseHandle(hFile); + #else +-#ifdef unix || __APPLE__ ++#if defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) + struct utimbuf ut; + struct tm newdate; + newdate.tm_sec = tmu_date.tm_sec; +@@ -125,11 +126,9 @@ int mymkdir(dirname) + const char* dirname; + { + int ret=0; +-#ifdef _WIN32 ++#if defined(_WIN32) + ret = _mkdir(dirname); +-#elif unix +- ret = mkdir (dirname,0775); +-#elif __APPLE__ ++#elif defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) + ret = mkdir (dirname,0775); + #endif + return ret; +diff --git a/third_party/zlib/contrib/minizip/minizip.c b/third_party/zlib/contrib/minizip/minizip.c +index 4288962ecef0..b794953c5c23 100644 +--- a/third_party/zlib/contrib/minizip/minizip.c ++++ b/third_party/zlib/contrib/minizip/minizip.c +@@ -12,8 +12,7 @@ + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + */ + +- +-#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) ++#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) && (!defined(__ANDROID_API__)) + #ifndef __USE_FILE_OFFSET64 + #define __USE_FILE_OFFSET64 + #endif +@@ -28,7 +27,7 @@ + #endif + #endif + +-#ifdef __APPLE__ ++#if defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) + // In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions + #define FOPEN_FUNC(filename, mode) fopen(filename, mode) + #define FTELLO_FUNC(stream) ftello(stream) +@@ -94,7 +93,7 @@ uLong filetime(f, tmzip, dt) + return ret; + } + #else +-#ifdef unix || __APPLE__ ++#if defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) + uLong filetime(f, tmzip, dt) + char *f; /* name of file to get info on */ + tm_zip *tmzip; /* return value: access, modific. and creation times */ +-- +2.31.1.818.g46aad6cb9e-goog + diff --git a/deps/zlib/simd_stub.c b/deps/zlib/simd_stub.c deleted file mode 100644 index c6d46051498f36..00000000000000 --- a/deps/zlib/simd_stub.c +++ /dev/null @@ -1,35 +0,0 @@ -/* simd_stub.c -- stub implementations -* Copyright (C) 2014 Intel Corporation -* For conditions of distribution and use, see copyright notice in zlib.h -*/ -#include - -#include "deflate.h" -#include "x86.h" - -int ZLIB_INTERNAL x86_cpu_enable_simd = 0; - -void ZLIB_INTERNAL crc_fold_init(deflate_state *const s) { - assert(0); -} - -void ZLIB_INTERNAL crc_fold_copy(deflate_state *const s, - unsigned char *dst, - const unsigned char *src, - long len) { - assert(0); -} - -unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state *const s) { - assert(0); - return 0; -} - -void ZLIB_INTERNAL fill_window_sse(deflate_state *s) -{ - assert(0); -} - -void x86_check_features(void) -{ -} diff --git a/deps/zlib/x86.c b/deps/zlib/x86.c deleted file mode 100644 index 7488ad08b976c8..00000000000000 --- a/deps/zlib/x86.c +++ /dev/null @@ -1,101 +0,0 @@ -/* - * x86 feature check - * - * Copyright (C) 2013 Intel Corporation. All rights reserved. - * Author: - * Jim Kukunas - * - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -#include "x86.h" -#include "zutil.h" - -int ZLIB_INTERNAL x86_cpu_enable_ssse3 = 0; -int ZLIB_INTERNAL x86_cpu_enable_simd = 0; - -#ifndef _MSC_VER -#include - -pthread_once_t cpu_check_inited_once = PTHREAD_ONCE_INIT; -static void _x86_check_features(void); - -void x86_check_features(void) -{ - pthread_once(&cpu_check_inited_once, _x86_check_features); -} - -static void _x86_check_features(void) -{ - int x86_cpu_has_sse2; - int x86_cpu_has_ssse3; - int x86_cpu_has_sse42; - int x86_cpu_has_pclmulqdq; - unsigned eax, ebx, ecx, edx; - - eax = 1; -#ifdef __i386__ - __asm__ __volatile__ ( - "xchg %%ebx, %1\n\t" - "cpuid\n\t" - "xchg %1, %%ebx\n\t" - : "+a" (eax), "=S" (ebx), "=c" (ecx), "=d" (edx) - ); -#else - __asm__ __volatile__ ( - "cpuid\n\t" - : "+a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx) - ); -#endif /* (__i386__) */ - - x86_cpu_has_sse2 = edx & 0x4000000; - x86_cpu_has_ssse3 = ecx & 0x000200; - x86_cpu_has_sse42 = ecx & 0x100000; - x86_cpu_has_pclmulqdq = ecx & 0x2; - - x86_cpu_enable_ssse3 = x86_cpu_has_ssse3; - - x86_cpu_enable_simd = x86_cpu_has_sse2 && - x86_cpu_has_sse42 && - x86_cpu_has_pclmulqdq; -} -#else -#include -#include - -static BOOL CALLBACK _x86_check_features(PINIT_ONCE once, - PVOID param, - PVOID *context); -static INIT_ONCE cpu_check_inited_once = INIT_ONCE_STATIC_INIT; - -void x86_check_features(void) -{ - InitOnceExecuteOnce(&cpu_check_inited_once, _x86_check_features, - NULL, NULL); -} - -static BOOL CALLBACK _x86_check_features(PINIT_ONCE once, - PVOID param, - PVOID *context) -{ - int x86_cpu_has_sse2; - int x86_cpu_has_ssse3; - int x86_cpu_has_sse42; - int x86_cpu_has_pclmulqdq; - int regs[4]; - - __cpuid(regs, 1); - - x86_cpu_has_sse2 = regs[3] & 0x4000000; - x86_cpu_has_ssse3 = regs[2] & 0x000200; - x86_cpu_has_sse42 = regs[2] & 0x100000; - x86_cpu_has_pclmulqdq = regs[2] & 0x2; - - x86_cpu_enable_ssse3 = x86_cpu_has_ssse3; - - x86_cpu_enable_simd = x86_cpu_has_sse2 && - x86_cpu_has_sse42 && - x86_cpu_has_pclmulqdq; - return TRUE; -} -#endif /* _MSC_VER */ diff --git a/deps/zlib/x86.h b/deps/zlib/x86.h deleted file mode 100644 index 7205d50265c356..00000000000000 --- a/deps/zlib/x86.h +++ /dev/null @@ -1,16 +0,0 @@ -/* x86.h -- check for x86 CPU features -* Copyright (C) 2013 Intel Corporation Jim Kukunas -* For conditions of distribution and use, see copyright notice in zlib.h -*/ - -#ifndef X86_H -#define X86_H - -#include "zlib.h" - -extern int x86_cpu_enable_ssse3; -extern int x86_cpu_enable_simd; - -void x86_check_features(void); - -#endif /* X86_H */ diff --git a/deps/zlib/zlib.gyp b/deps/zlib/zlib.gyp index 7cc472d63ad056..83733b1d5309f0 100644 --- a/deps/zlib/zlib.gyp +++ b/deps/zlib/zlib.gyp @@ -18,6 +18,8 @@ 'adler32.c', 'compress.c', 'contrib/optimizations/insert_string.h', + 'cpu_features.c', + 'cpu_features.h', 'crc32.c', 'crc32.h', 'deflate.c', @@ -37,7 +39,6 @@ 'trees.c', 'trees.h', 'uncompr.c', - 'x86.h', 'zconf.h', 'zlib.h', 'zutil.c', @@ -89,15 +90,12 @@ 'crc32_simd.h', 'crc_folding.c', 'fill_window_sse.c', - 'x86.c', ], 'conditions': [ ['target_arch=="x64"', { 'defines': [ 'INFLATE_CHUNK_READ_64LE' ], }], ], - }, { - 'sources': [ 'simd_stub.c', ], }], ['arm_fpu=="neon"', { 'defines': [ @@ -111,8 +109,6 @@ ['OS!="ios"', { 'defines': [ 'CRC32_ARMV8_CRC32' ], 'sources': [ - 'arm_features.c', - 'arm_features.h', 'crc32_simd.c', 'crc32_simd.h', ], From 56e6a5159f1d59326c5e39558d18feb17b97709a Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Fri, 3 Apr 2020 04:13:23 +0200 Subject: [PATCH 2/6] deps: fix zlib compilation for CPUs without SIMD features MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix the compile flags so that zlib can run on CPUs that do not have SSSE3/SSE4.2/etc. Do not compile zlib with flags that indicate that those features are available, and instead enable them selectively for functions that use them. There are probably better way to do this, e.g. through gyp file modifications as suggested in the issue. However, this patch should do just fine until that happens. Fixes: https://github.com/nodejs/node/issues/32553 PR-URL: https://github.com/nodejs/node/pull/32627 Reviewed-By: Gireesh Punathil Reviewed-By: Ben Noordhuis Reviewed-By: Tobias Nießen Reviewed-By: James M Snell --- deps/zlib/adler32_simd.c | 4 ++++ deps/zlib/crc32_simd.c | 4 ++++ deps/zlib/crc_folding.c | 12 ++++++++++++ 3 files changed, 20 insertions(+) diff --git a/deps/zlib/adler32_simd.c b/deps/zlib/adler32_simd.c index 1354915cc099ad..f8b07297b93840 100644 --- a/deps/zlib/adler32_simd.c +++ b/deps/zlib/adler32_simd.c @@ -50,9 +50,13 @@ #define NMAX 5552 #if defined(ADLER32_SIMD_SSSE3) +#ifndef __GNUC__ +#define __attribute__() +#endif #include +__attribute__((target("ssse3"))) uint32_t ZLIB_INTERNAL adler32_simd_( /* SSSE3 */ uint32_t adler, const unsigned char *buf, diff --git a/deps/zlib/crc32_simd.c b/deps/zlib/crc32_simd.c index c8e5592f38ef1e..27481847e97b90 100644 --- a/deps/zlib/crc32_simd.c +++ b/deps/zlib/crc32_simd.c @@ -8,6 +8,9 @@ #include "crc32_simd.h" #if defined(CRC32_SIMD_SSE42_PCLMUL) +#ifndef __GNUC__ +#define __attribute__() +#endif /* * crc32_sse42_simd_(): compute the crc32 of the buffer, where the buffer @@ -21,6 +24,7 @@ #include #include +__attribute__((target("sse4.2,pclmul"))) uint32_t ZLIB_INTERNAL crc32_sse42_simd_( /* SSE4.2+PCLMUL */ const unsigned char *buf, z_size_t len, diff --git a/deps/zlib/crc_folding.c b/deps/zlib/crc_folding.c index ee31d4918d6413..4e9c6e78529f7e 100644 --- a/deps/zlib/crc_folding.c +++ b/deps/zlib/crc_folding.c @@ -25,6 +25,10 @@ #include #include +#ifndef __GNUC__ +#define __attribute__() +#endif + #define CRC_LOAD(s) \ do { \ __m128i xmm_crc0 = _mm_loadu_si128((__m128i *)s->crc0 + 0);\ @@ -41,6 +45,7 @@ _mm_storeu_si128((__m128i *)s->crc0 + 4, xmm_crc_part);\ } while (0); +__attribute__((target("sse4.2,pclmul"))) ZLIB_INTERNAL void crc_fold_init(deflate_state *const s) { CRC_LOAD(s) @@ -55,6 +60,7 @@ ZLIB_INTERNAL void crc_fold_init(deflate_state *const s) s->strm->adler = 0; } +__attribute__((target("sse4.2,pclmul"))) local void fold_1(deflate_state *const s, __m128i *xmm_crc0, __m128i *xmm_crc1, __m128i *xmm_crc2, __m128i *xmm_crc3) @@ -81,6 +87,7 @@ local void fold_1(deflate_state *const s, *xmm_crc3 = _mm_castps_si128(ps_res); } +__attribute__((target("sse4.2,pclmul"))) local void fold_2(deflate_state *const s, __m128i *xmm_crc0, __m128i *xmm_crc1, __m128i *xmm_crc2, __m128i *xmm_crc3) @@ -115,6 +122,7 @@ local void fold_2(deflate_state *const s, *xmm_crc3 = _mm_castps_si128(ps_res31); } +__attribute__((target("sse4.2,pclmul"))) local void fold_3(deflate_state *const s, __m128i *xmm_crc0, __m128i *xmm_crc1, __m128i *xmm_crc2, __m128i *xmm_crc3) @@ -155,6 +163,7 @@ local void fold_3(deflate_state *const s, *xmm_crc3 = _mm_castps_si128(ps_res32); } +__attribute__((target("sse4.2,pclmul"))) local void fold_4(deflate_state *const s, __m128i *xmm_crc0, __m128i *xmm_crc1, __m128i *xmm_crc2, __m128i *xmm_crc3) @@ -221,6 +230,7 @@ local const unsigned zalign(32) pshufb_shf_table[60] = { 0x0201008f,0x06050403,0x0a090807,0x0e0d0c0b /* shl 1 (16 -15)/shr15*/ }; +__attribute__((target("sse4.2,pclmul"))) local void partial_fold(deflate_state *const s, const size_t len, __m128i *xmm_crc0, __m128i *xmm_crc1, __m128i *xmm_crc2, __m128i *xmm_crc3, @@ -271,6 +281,7 @@ local void partial_fold(deflate_state *const s, const size_t len, *xmm_crc3 = _mm_castps_si128(ps_res); } +__attribute__((target("sse4.2,pclmul"))) ZLIB_INTERNAL void crc_fold_copy(deflate_state *const s, unsigned char *dst, const unsigned char *src, long len) { @@ -427,6 +438,7 @@ local const unsigned zalign(16) crc_mask2[4] = { 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }; +__attribute__((target("sse4.2,pclmul"))) unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state *const s) { const __m128i xmm_mask = _mm_load_si128((__m128i *)crc_mask); From ec258fee16ef34b5206dc4ec2f49c52797c2698c Mon Sep 17 00:00:00 2001 From: raisinten Date: Fri, 16 Oct 2020 17:10:16 +0530 Subject: [PATCH 3/6] build: fix zlib inlining for IA-32 PR-URL: https://github.com/nodejs/node/pull/35679 Fixes: https://github.com/nodejs/node/issues/35629 Reviewed-By: Anna Henningsen Reviewed-By: Rich Trott --- deps/zlib/contrib/optimizations/chunkcopy.h | 1 + deps/zlib/fill_window_sse.c | 1 + 2 files changed, 2 insertions(+) diff --git a/deps/zlib/contrib/optimizations/chunkcopy.h b/deps/zlib/contrib/optimizations/chunkcopy.h index 9c0b7cb06c2dcc..6232fae37448ce 100644 --- a/deps/zlib/contrib/optimizations/chunkcopy.h +++ b/deps/zlib/contrib/optimizations/chunkcopy.h @@ -29,6 +29,7 @@ #include typedef uint8x16_t z_vec128i_t; #elif defined(INFLATE_CHUNK_SIMD_SSE2) +#pragma GCC target ("sse2") #include typedef __m128i z_vec128i_t; #else diff --git a/deps/zlib/fill_window_sse.c b/deps/zlib/fill_window_sse.c index a841c99904baee..e618534a23fb9a 100644 --- a/deps/zlib/fill_window_sse.c +++ b/deps/zlib/fill_window_sse.c @@ -29,6 +29,7 @@ extern int deflate_read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); +#pragma GCC target ("sse2") #include void fill_window_sse(deflate_state *s) From 634ad79cbfa7c9944a3f47452d0aa470dccf441b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Zasso?= Date: Sat, 29 Jan 2022 13:58:36 +0100 Subject: [PATCH 4/6] fixup! deps: update zlib to upstream fc5cfd78a3 --- deps/zlib/zlib.gyp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deps/zlib/zlib.gyp b/deps/zlib/zlib.gyp index 83733b1d5309f0..afc158e8046a68 100644 --- a/deps/zlib/zlib.gyp +++ b/deps/zlib/zlib.gyp @@ -95,6 +95,11 @@ ['target_arch=="x64"', { 'defines': [ 'INFLATE_CHUNK_READ_64LE' ], }], + ['OS=="win"', { + 'defines': [ 'X86_WINDOWS' ], + }, { + 'defines': [ 'X86_NOT_WINDOWS' ], + }] ], }], ['arm_fpu=="neon"', { From 582477893f00120a9c0d6e0b00df27a470d99229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Zasso?= Date: Sun, 30 Jan 2022 10:42:47 +0100 Subject: [PATCH 5/6] build: default ARM FPU to "neon" on ARM64 Windows --- configure.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configure.py b/configure.py index 1bd24a4a8e35db..01065079369c19 100755 --- a/configure.py +++ b/configure.py @@ -1143,6 +1143,9 @@ def configure_arm(o): if options.dest_os == 'android': arm_fpu = 'vfpv3' o['variables']['arm_version'] = '7' + + if flavor == 'win': + arm_fpu = 'neon' o['variables']['arm_fpu'] = options.arm_fpu or arm_fpu From a83aa1c7419685f2e712ba410bc8fb9ff7415a5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Zasso?= Date: Sun, 30 Jan 2022 10:47:04 +0100 Subject: [PATCH 6/6] fixup! build: default ARM FPU to "neon" on ARM64 Windows --- configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.py b/configure.py index 01065079369c19..72554bd8d6b65d 100755 --- a/configure.py +++ b/configure.py @@ -1143,7 +1143,7 @@ def configure_arm(o): if options.dest_os == 'android': arm_fpu = 'vfpv3' o['variables']['arm_version'] = '7' - + if flavor == 'win': arm_fpu = 'neon'