From ef4b710f730b09bb40e630e596874438358cdc98 Mon Sep 17 00:00:00 2001 From: Kimball Thurston Date: Tue, 25 Apr 2023 17:41:58 +1200 Subject: [PATCH] Switch to embedding libdeflate into EXRCore (#1387) Switches to embedding upstream (MIT license) libdeflate vs linking against external library, functions should be private / hidden by default. Like Imath, enables one to use a system version or force using an internally embedded version (OPENEXR_FORCE_INTERNAL_DEFLATE). Further, the repository used can be controlled with OPENEXR_DEFLATE_REPO and OPENEXR_DEFLATE_TAG cmake variables. This centralizes all IETF RFC 1950 (zlib) compression into the EXRCore library. This implies that the main C++ library depends on EXRCore. Signed-off-by: Kimball Thurston --- BUILD.bazel | 107 +++++++++++++++- bazel/third_party/Imath.BUILD | 6 +- bazel/third_party/libdeflate.BUILD | 58 +++++++++ bazel/third_party/openexr_deps.bzl | 13 +- bazel/third_party/zlib.BUILD | 98 --------------- cmake/CMakeLists.txt | 3 - cmake/LibraryDefine.cmake | 3 + cmake/OpenEXR.pc.in | 2 +- cmake/OpenEXRSetup.cmake | 137 ++++++++------------- docs/TechnicalIntroduction.rst | 12 +- docs/install.rst | 4 +- src/lib/OpenEXR/CMakeLists.txt | 2 +- src/lib/OpenEXR/ImfDwaCompressor.cpp | 105 ++++++++-------- src/lib/OpenEXR/ImfHeader.cpp | 16 ++- src/lib/OpenEXR/ImfIDManifest.cpp | 36 +++--- src/lib/OpenEXR/ImfIDManifestAttribute.cpp | 6 +- src/lib/OpenEXR/ImfPxr24Compressor.cpp | 43 +++---- src/lib/OpenEXR/ImfZip.cpp | 49 +++----- src/lib/OpenEXR/ImfZipCompressor.cpp | 1 - src/lib/OpenEXRCore/CMakeLists.txt | 14 ++- src/lib/OpenEXRCore/chunk.c | 4 +- src/lib/OpenEXRCore/compression.c | 127 +++++++++++++++++++ src/lib/OpenEXRCore/encoding.c | 13 +- src/lib/OpenEXRCore/internal_pxr24.c | 57 +++++---- src/lib/OpenEXRCore/internal_zip.c | 85 +++++++------ src/lib/OpenEXRCore/memory.c | 2 + src/lib/OpenEXRCore/openexr.h | 2 + src/lib/OpenEXRCore/openexr_compression.h | 50 ++++++++ src/lib/OpenEXRCore/unpack.c | 8 -- src/test/OpenEXRCoreTest/CMakeLists.txt | 3 + src/test/OpenEXRCoreTest/buffer.cpp | 38 ++++++ src/test/OpenEXRCoreTest/buffer.h | 9 ++ src/test/OpenEXRCoreTest/main.cpp | 2 + src/test/OpenEXRTest/testIDManifest.cpp | 18 +-- 34 files changed, 700 insertions(+), 433 deletions(-) create mode 100644 bazel/third_party/libdeflate.BUILD delete mode 100644 bazel/third_party/zlib.BUILD create mode 100644 src/lib/OpenEXRCore/compression.c create mode 100644 src/lib/OpenEXRCore/openexr_compression.h create mode 100644 src/test/OpenEXRCoreTest/buffer.cpp create mode 100644 src/test/OpenEXRCoreTest/buffer.h diff --git a/BUILD.bazel b/BUILD.bazel index 56facf391d..099c680e1b 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -53,9 +53,9 @@ expand_template( "@OPENEXR_NAMESPACE_CUSTOM@": "3.2.0", "@OPENEXR_PACKAGE_NAME@": "OpenEXR 3.2.0", "@OPENEXR_VERSION_EXTRA@": "", - "@OPENEXR_VERSION_MAJOR@": "3", - "@OPENEXR_VERSION_MINOR@": "2", - "@OPENEXR_VERSION_PATCH@": "0", + "@OpenEXR_VERSION_MAJOR@": "3", + "@OpenEXR_VERSION_MINOR@": "2", + "@OpenEXR_VERSION_PATCH@": "0", "@OPENEXR_VERSION@": "3.2.0", "#cmakedefine OPENEXR_ENABLE_API_VISIBILITY": "#define OPENEXR_ENABLE_API_VISIBILITY", "#cmakedefine OPENEXR_HAVE_LARGE_STACK 1": "/* #undef OPENEXR_HAVE_LARGE_STACK */", @@ -141,6 +141,105 @@ cc_library( deps = [":Iex"], ) +cc_library( + name = "OpenEXRCore", + srcs = [ + "src/lib/OpenEXRCore/backward_compatibility.h", + "src/lib/OpenEXRCore/internal_attr.h", + "src/lib/OpenEXRCore/internal_channel_list.h", + "src/lib/OpenEXRCore/internal_coding.h", + "src/lib/OpenEXRCore/internal_constants.h", + "src/lib/OpenEXRCore/internal_compress.h", + "src/lib/OpenEXRCore/internal_decompress.h", + "src/lib/OpenEXRCore/internal_file.h", + "src/lib/OpenEXRCore/internal_float_vector.h", + "src/lib/OpenEXRCore/internal_huf.h", + "src/lib/OpenEXRCore/internal_memory.h", + "src/lib/OpenEXRCore/internal_opaque.h", + "src/lib/OpenEXRCore/internal_posix_file_impl.h", + "src/lib/OpenEXRCore/internal_win32_file_impl.h", + "src/lib/OpenEXRCore/internal_preview.h", + "src/lib/OpenEXRCore/internal_string.h", + "src/lib/OpenEXRCore/internal_string_vector.h", + "src/lib/OpenEXRCore/internal_structs.h", + "src/lib/OpenEXRCore/internal_util.h", + "src/lib/OpenEXRCore/internal_xdr.h", + "src/lib/OpenEXRCore/internal_rle.c", + "src/lib/OpenEXRCore/internal_zip.c", + "src/lib/OpenEXRCore/internal_pxr24.c", + "src/lib/OpenEXRCore/internal_b44.c", + "src/lib/OpenEXRCore/internal_b44_table.c", + "src/lib/OpenEXRCore/internal_piz.c", + "src/lib/OpenEXRCore/internal_dwa.c", + "src/lib/OpenEXRCore/internal_huf.c", + "src/lib/OpenEXRCore/attributes.c", + "src/lib/OpenEXRCore/string.c", + "src/lib/OpenEXRCore/string_vector.c", + "src/lib/OpenEXRCore/float_vector.c", + "src/lib/OpenEXRCore/channel_list.c", + "src/lib/OpenEXRCore/opaque.c", + "src/lib/OpenEXRCore/preview.c", + "src/lib/OpenEXRCore/base.c", + "src/lib/OpenEXRCore/context.c", + "src/lib/OpenEXRCore/memory.c", + "src/lib/OpenEXRCore/internal_structs.c", + "src/lib/OpenEXRCore/part.c", + "src/lib/OpenEXRCore/part_attr.c", + "src/lib/OpenEXRCore/std_attr.c", + "src/lib/OpenEXRCore/parse_header.c", + "src/lib/OpenEXRCore/write_header.c", + "src/lib/OpenEXRCore/chunk.c", + "src/lib/OpenEXRCore/coding.c", + "src/lib/OpenEXRCore/compression.c", + "src/lib/OpenEXRCore/decoding.c", + "src/lib/OpenEXRCore/encoding.c", + "src/lib/OpenEXRCore/pack.c", + "src/lib/OpenEXRCore/unpack.c", + "src/lib/OpenEXRCore/validation.c", + "src/lib/OpenEXRCore/debug.c", + "src/lib/OpenEXR/OpenEXRConfig.h", + "src/lib/IlmThread/IlmThreadConfig.h", + ], + hdrs = [ + "src/lib/OpenEXRCore/openexr.h", + "src/lib/OpenEXRCore/openexr_attr.h", + "src/lib/OpenEXRCore/openexr_base.h", + "src/lib/OpenEXRCore/openexr_chunkio.h", + "src/lib/OpenEXRCore/openexr_coding.h", + "src/lib/OpenEXRCore/openexr_compression.h", + "src/lib/OpenEXRCore/openexr_conf.h", + "src/lib/OpenEXRCore/openexr_context.h", + "src/lib/OpenEXRCore/openexr_decode.h", + "src/lib/OpenEXRCore/openexr_debug.h", + "src/lib/OpenEXRCore/openexr_encode.h", + "src/lib/OpenEXRCore/openexr_errors.h", + "src/lib/OpenEXRCore/openexr_part.h", + "src/lib/OpenEXRCore/openexr_std_attr.h", + ], + copts = select({ + ":windows": [], + "//conditions:default": [ + "-Wno-error", + ], + }), + features = select({ + ":windows": ["windows_export_all_symbols"], + "//conditions:default": [], + }), + includes = ["src/lib/OpenEXRCore", "src/lib/OpenEXR", "src/lib/IlmThread"], + linkopts = + select({ + ":windows": [], + "//conditions:default": [ + "-pthread", + ], + }), + visibility = ["//visibility:public"], + deps = [ + "@Imath", "@libdeflate", + ], +) + cc_library( name = "OpenEXR", srcs = [ @@ -378,8 +477,8 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":IlmThread", + ":OpenEXRCore", "@Imath", - "@net_zlib_zlib//:zlib", ], ) diff --git a/bazel/third_party/Imath.BUILD b/bazel/third_party/Imath.BUILD index 4f65c67578..8e35e40b45 100644 --- a/bazel/third_party/Imath.BUILD +++ b/bazel/third_party/Imath.BUILD @@ -12,9 +12,9 @@ expand_template( "@IMATH_NAMESPACE_CUSTOM@": "0", "@IMATH_NAMESPACE@": "Imath", "@IMATH_PACKAGE_NAME@": "Imath 3.1.7", - "@IMATH_VERSION_MAJOR@": "3", - "@IMATH_VERSION_MINOR@": "1", - "@IMATH_VERSION_PATCH@": "7", + "@Imath_VERSION_MAJOR@": "3", + "@Imath_VERSION_MINOR@": "1", + "@Imath_VERSION_PATCH@": "7", "@IMATH_VERSION@": "3.1.7", "#cmakedefine IMATH_HALF_USE_LOOKUP_TABLE": "#define IMATH_HALF_USE_LOOKUP_TABLE", "#cmakedefine IMATH_ENABLE_API_VISIBILITY": "#define IMATH_ENABLE_API_VISIBILITY", diff --git a/bazel/third_party/libdeflate.BUILD b/bazel/third_party/libdeflate.BUILD new file mode 100644 index 0000000000..43ad6bf915 --- /dev/null +++ b/bazel/third_party/libdeflate.BUILD @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) Contributors to the OpenEXR Project. + +load("@rules_cc//cc:defs.bzl", "cc_library") + +licenses(["notice"]) + +exports_files( + glob(["lib/**"]) + ["common_defs.h"], + visibility = ["//visibility:public"], +) + +cc_library( + name = "libdeflate", + srcs = [ + "lib/arm/cpu_features.c", + "lib/x86/cpu_features.c", + "lib/utils.c", + "lib/deflate_compress.c", + "lib/deflate_decompress.c", + "lib/adler32.c", + "lib/zlib_compress.c", + "lib/zlib_decompress.c", + "lib/crc32.c", + "lib/gzip_compress.c", + "lib/gzip_decompress.c", + "lib/adler32_vec_template.h", + "lib/bt_matchfinder.h", + "lib/cpu_features_common.h", + "lib/crc32_multipliers.h", + "lib/crc32_tables.h", + "lib/decompress_template.h", + "lib/deflate_compress.h", + "lib/deflate_constants.h", + "lib/gzip_constants.h", + "lib/hc_matchfinder.h", + "lib/ht_matchfinder.h", + "lib/lib_common.h", + "lib/matchfinder_common.h", + "lib/zlib_constants.h", + "lib/arm/adler32_impl.h", + "lib/arm/cpu_features.h", + "lib/arm/crc32_impl.h", + "lib/arm/crc32_pmull_helpers.h", + "lib/arm/crc32_pmull_wide.h", + "lib/arm/matchfinder_impl.h", + "lib/x86/adler32_impl.h", + "lib/x86/cpu_features.h", + "lib/x86/crc32_impl.h", + "lib/x86/crc32_pclmul_template.h", + "lib/x86/decompress_impl.h", + "lib/x86/matchfinder_impl.h", + "common_defs.h", + ], + hdrs = ["libdeflate.h"], + includes = ["."], + visibility = ["//visibility:public"], + ) diff --git a/bazel/third_party/openexr_deps.bzl b/bazel/third_party/openexr_deps.bzl index f6038e300e..f0a62b4fed 100644 --- a/bazel/third_party/openexr_deps.bzl +++ b/bazel/third_party/openexr_deps.bzl @@ -11,14 +11,11 @@ def openexr_deps(): maybe( http_archive, - name = "net_zlib_zlib", - build_file = "@com_openexr//:bazel/third_party/zlib.BUILD", - sha256 = "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30", - strip_prefix = "zlib-1.2.13", - urls = [ - "https://mirror.bazel.build/zlib.net/zlib-1.2.13.tar.gz", - "https://zlib.net/zlib-1.2.13.tar.gz", - ], + name = "libdeflate", + build_file = "@com_openexr//:bazel/third_party/libdeflate.BUILD", + sha256 = "225d982bcaf553221c76726358d2ea139bb34913180b20823c782cede060affd", + strip_prefix = "libdeflate-1.18", + urls = ["https://github.com/ebiggers/libdeflate/archive/refs/tags/v1.18.tar.gz"], ) maybe( diff --git a/bazel/third_party/zlib.BUILD b/bazel/third_party/zlib.BUILD deleted file mode 100644 index 34f5987047..0000000000 --- a/bazel/third_party/zlib.BUILD +++ /dev/null @@ -1,98 +0,0 @@ -# Copied from https://github.com/protocolbuffers/protobuf/blob/master/third_party/zlib.BUILD - -# Copyright 2008 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Code generated by the Protocol Buffer compiler is owned by the owner -# of the input file used when generating it. This code is not -# standalone and requires a support library to be linked with it. This -# support library is itself covered by the above license. - -load("@rules_cc//cc:defs.bzl", "cc_library") - -licenses(["notice"]) # BSD/MIT-like license (for zlib) - -_ZLIB_HEADERS = [ - "crc32.h", - "deflate.h", - "gzguts.h", - "inffast.h", - "inffixed.h", - "inflate.h", - "inftrees.h", - "trees.h", - "zconf.h", - "zlib.h", - "zutil.h", -] - -_ZLIB_PREFIXED_HEADERS = ["zlib/include/" + hdr for hdr in _ZLIB_HEADERS] - -# In order to limit the damage from the `includes` propagation -# via `:zlib`, copy the public headers to a subdirectory and -# expose those. -genrule( - name = "copy_public_headers", - srcs = _ZLIB_HEADERS, - outs = _ZLIB_PREFIXED_HEADERS, - cmd = "cp $(SRCS) $(@D)/zlib/include/", -) - -cc_library( - name = "zlib", - srcs = [ - "adler32.c", - "compress.c", - "crc32.c", - "deflate.c", - "gzclose.c", - "gzlib.c", - "gzread.c", - "gzwrite.c", - "infback.c", - "inffast.c", - "inflate.c", - "inftrees.c", - "trees.c", - "uncompr.c", - "zutil.c", - # Include the un-prefixed headers in srcs to work - # around the fact that zlib isn't consistent in its - # choice of <> or "" delimiter when including itself. - ] + _ZLIB_HEADERS, - hdrs = _ZLIB_PREFIXED_HEADERS, - copts = select({ - "@bazel_tools//src/conditions:windows": [], - "//conditions:default": [ - "-Wno-unused-variable", - "-Wno-implicit-function-declaration", - ], - }), - includes = ["zlib/include/"], - visibility = ["//visibility:public"], -) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index ec51f639eb..fe3df7d4a1 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -209,9 +209,6 @@ if(OPENEXR_INSTALL_PKG_CONFIG) endif() set(exr_pthread_libs ${CMAKE_THREAD_LIBS_INIT}) endif() - if(NOT zlib_INTERNAL_DIR) - set(zlib_link "-lz") - endif() string(REPLACE ".in" "" pcout ${pcinfile}) configure_file(${pcinfile} ${CMAKE_CURRENT_BINARY_DIR}/${pcout} @ONLY) install( diff --git a/cmake/LibraryDefine.cmake b/cmake/LibraryDefine.cmake index 0e2dbf6c17..a242ec8519 100644 --- a/cmake/LibraryDefine.cmake +++ b/cmake/LibraryDefine.cmake @@ -27,6 +27,9 @@ function(OPENEXR_DEFINE_LIBRARY libname) PRIVATE cxx_std_${OPENEXR_CXX_STANDARD} INTERFACE cxx_std_11 ) + # we are embedding libdeflate + target_include_directories(${objlib} PRIVATE ${EXR_DEFLATE_INCLUDE_DIR}) + if(OPENEXR_CURLIB_PRIV_EXPORT AND BUILD_SHARED_LIBS) target_compile_definitions(${objlib} PRIVATE ${OPENEXR_CURLIB_PRIV_EXPORT}) if(WIN32) diff --git a/cmake/OpenEXR.pc.in b/cmake/OpenEXR.pc.in index c58420ec59..818480efc1 100644 --- a/cmake/OpenEXR.pc.in +++ b/cmake/OpenEXR.pc.in @@ -17,4 +17,4 @@ Version: @OPENEXR_VERSION@ Libs: @exr_pthread_libs@ -L${libdir} -lOpenEXR${libsuffix} -lOpenEXRUtil${libsuffix} -lOpenEXRCore${libsuffix} -lIex${libsuffix} -lIlmThread${libsuffix} Cflags: -I${includedir} -I${OpenEXR_includedir} @exr_pthread_cflags@ Requires: Imath -Libs.private: @zlib_link@ + diff --git a/cmake/OpenEXRSetup.cmake b/cmake/OpenEXRSetup.cmake index 458c1b8987..c697cbf4f6 100644 --- a/cmake/OpenEXRSetup.cmake +++ b/cmake/OpenEXRSetup.cmake @@ -157,100 +157,67 @@ if(OPENEXR_ENABLE_THREADING) endif() endif() -option(OPENEXR_FORCE_INTERNAL_ZLIB "Force using an internal zlib" OFF) -if (NOT OPENEXR_FORCE_INTERNAL_ZLIB) - if(NOT TARGET ZLIB::ZLIB) - find_package(ZLIB QUIET) - endif() -endif() -if(OPENEXR_FORCE_INTERNAL_ZLIB OR NOT TARGET ZLIB::ZLIB) - set(zlib_VER "1.2.11") - if(OPENEXR_FORCE_INTERNAL_ZLIB) - message(STATUS "Compiling internal copy of zlib version ${zlib_VER}") - else() - message(STATUS "zlib library not found, compiling ${zlib_VER}") - endif() - - # Unfortunately, zlib has an ancient cmake setup which does not include - # modern cmake exports, so we can't use the faster / more integrated - # FetchContent as we do for Imath below. There may be a way, but - # external project is just as easy - include(ExternalProject) +option(OPENEXR_FORCE_INTERNAL_DEFLATE "Force using an internal libdeflate" OFF) +set(OPENEXR_DEFLATE_REPO "https://github.com/ebiggers/libdeflate.git" CACHE STRING "Repo path for libdeflate source") +set(OPENEXR_DEFLATE_TAG "v1.18" CACHE STRING "Tag to use for libdeflate source repo (defaults to primary if empty)") - set(cmake_cc_arg) - if (CMAKE_CROSSCOMPILING) - set(cmake_cc_arg -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}) - endif () +if(NOT OPENEXR_FORCE_INTERNAL_DEFLATE) + #TODO: ^^ Release should not clone from main, this is a place holder + set(CMAKE_IGNORE_PATH "${CMAKE_CURRENT_BINARY_DIR}/_deps/deflate-src/config;${CMAKE_CURRENT_BINARY_DIR}/_deps/deflate-build/config") + include(FindPkgConfig) + pkg_check_modules(deflate IMPORTED_TARGET GLOBAL libdeflate) + set(CMAKE_IGNORE_PATH) +endif() - if(NOT (APPLE OR WIN32) AND NOT OPENEXR_FORCE_INTERNAL_ZLIB) - set(zlib_INTERNAL_DIR "${CMAKE_INSTALL_PREFIX}" CACHE PATH "zlib install dir") +if(NOT TARGET PkgConfig::deflate AND NOT deflate_FOUND) + if(OPENEXR_FORCE_INTERNAL_DEFLATE) + message(STATUS "libdeflate forced internal, installing from ${OPENEXR_DEFLATE_REPO} (${OPENEXR_DEFLATE_TAG})") else() - set(zlib_INTERNAL_DIR "${CMAKE_BINARY_DIR}/zlib-install" CACHE PATH "zlib install dir") + message(STATUS "libdeflate was not found, installing from ${OPENEXR_DEFLATE_REPO} (${OPENEXR_DEFLATE_TAG})") endif() - - # Need to set byproducts so ninja generator knows where the targets come from - ExternalProject_Add(zlib_external - GIT_REPOSITORY "https://github.com/madler/zlib.git" + include(FetchContent) + FetchContent_Declare(Deflate + GIT_REPOSITORY ${OPENEXR_DEFLATE_REPO} + GIT_TAG ${OPENEXR_DEFLATE_TAG} GIT_SHALLOW ON - GIT_TAG "v${zlib_VER}" - UPDATE_COMMAND "" - SOURCE_DIR zlib-src - BINARY_DIR zlib-build - INSTALL_DIR ${zlib_INTERNAL_DIR} - BUILD_BYPRODUCTS - "${zlib_INTERNAL_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}z${CMAKE_SHARED_LIBRARY_SUFFIX}" - "${zlib_INTERNAL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}z${CMAKE_STATIC_LIBRARY_SUFFIX}" - CMAKE_ARGS - -D CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -D CMAKE_POSITION_INDEPENDENT_CODE=ON - -D CMAKE_INSTALL_PREFIX:PATH= - -D CMAKE_GENERATOR:STRING=${CMAKE_GENERATOR} - ${cmake_cc_arg} - ) - - file(MAKE_DIRECTORY "${zlib_INTERNAL_DIR}") - file(MAKE_DIRECTORY "${zlib_INTERNAL_DIR}/include") - file(MAKE_DIRECTORY "${zlib_INTERNAL_DIR}/lib") - - if(WIN32) - set(zliblibname "zlib") - set(zlibstaticlibname "zlibstatic") - else() - set(zliblibname "z") - set(zlibstaticlibname "z") - endif() - - if(MSVC) - set(zlibpostfix "d") - endif() - - add_library(zlib_static STATIC IMPORTED GLOBAL) - add_dependencies(zlib_static zlib_external) - set_property(TARGET zlib_static PROPERTY - IMPORTED_LOCATION "${zlib_INTERNAL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${zlibstaticlibname}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) - set_property(TARGET zlib_static PROPERTY - IMPORTED_LOCATION_DEBUG "${zlib_INTERNAL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${zlibstaticlibname}${zlibpostfix}${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - target_include_directories(zlib_static INTERFACE "${zlib_INTERNAL_DIR}/include") - if(NOT (APPLE OR WIN32) AND BUILD_SHARED_LIBS AND NOT OPENEXR_FORCE_INTERNAL_ZLIB) - add_library(zlib_shared SHARED IMPORTED GLOBAL) - add_dependencies(zlib_shared zlib_external) - set_property(TARGET zlib_shared PROPERTY - IMPORTED_LOCATION "${zlib_INTERNAL_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${zliblibname}${CMAKE_SHARED_LIBRARY_SUFFIX}" - ) - set_property(TARGET zlib_static PROPERTY - IMPORTED_LOCATION_DEBUG "${zlib_INTERNAL_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}${zliblibname}${zlibpostfix}${CMAKE_SHARED_LIBRARY_SUFFIX}" - ) - target_include_directories(zlib_shared INTERFACE "${zlib_INTERNAL_DIR}/include") + FetchContent_GetProperties(Deflate) + if(NOT Deflate_POPULATED) + FetchContent_Populate(Deflate) endif() - if(NOT (APPLE OR WIN32) AND BUILD_SHARED_LIBS AND NOT OPENEXR_FORCE_INTERNAL_ZLIB) - add_library(ZLIB::ZLIB ALIAS zlib_shared) - else() - add_library(ZLIB::ZLIB ALIAS zlib_static) - endif() + # Rather than actually compile something, just embed the sources + # into exrcore. This could in theory cause issues when compiling as + # a static library into another application which also uses + # libdeflate but we switch the export sysmbol to hidden which should + # hide the symbols when linking... + set(EXR_DEFLATE_SOURCES + lib/arm/cpu_features.c + lib/x86/cpu_features.c + lib/utils.c + lib/deflate_compress.c + lib/deflate_decompress.c + lib/adler32.c + lib/zlib_compress.c + lib/zlib_decompress.c) + # don't need these + # lib/crc32.c + # lib/gzip_compress.c + # lib/gzip_decompress.c + file(READ ${deflate_SOURCE_DIR}/lib/lib_common.h DEFLATE_HIDE) + string(REPLACE default hidden DEFLATE_HIDE "${DEFLATE_HIDE}") + string(REPLACE "__declspec(dllexport)" "/**/" DEFLATE_HIDE "${DEFLATE_HIDE}") + file(WRITE ${deflate_SOURCE_DIR}/lib/lib_common.h "${DEFLATE_HIDE}") + + # cmake makes fetch content name lowercase for the properties (to deflate) + list(TRANSFORM EXR_DEFLATE_SOURCES PREPEND ${deflate_SOURCE_DIR}/) + set(EXR_DEFLATE_INCLUDE_DIR ${deflate_SOURCE_DIR}) + set(EXR_DEFLATE_LIB) +else() + set(EXR_DEFLATE_INCLUDE_DIR) + set(EXR_DEFLATE_LIB PkgConfig::deflate) + set(EXR_DEFLATE_SOURCES) endif() ####################################### diff --git a/docs/TechnicalIntroduction.rst b/docs/TechnicalIntroduction.rst index 9a92eeb8d1..7fca35810f 100644 --- a/docs/TechnicalIntroduction.rst +++ b/docs/TechnicalIntroduction.rst @@ -764,14 +764,14 @@ Supported compression schemes: input to the compressor is short, adding the header tends to offset any size reduction of the input.) * - ZIPS (lossless) - - Uses the open source zlib library for compression. Unlike ZIP - compression, this operates one scan line at a time. + - Uses the open source deflate library for IETF RFC 1950 compression. + Unlike ZIP compression, this operates one scan line at a time. * - ZIP (lossless) - Differences between horizontally adjacent pixels are compressed using the - open source zlib library. ZIP decompression is faster than PIZ - decompression, but ZIP compression is significantly slower. Photographic - images tend to shrink to between 45 and 55 percent of their uncompressed - size. + open source deflate library for IETF RFC 1950 compression. ZIP + decompression is faster than PIZ decompression, but ZIP may be + larger. Photographic images tend to shrink to between 45 and 55 + percent of their uncompressed size. Multi-resolution files are often used as texture maps for 3D renderers. For this application, fast read accesses are usually more diff --git a/docs/install.rst b/docs/install.rst index 15c628f8cc..9b589feb2e 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -70,8 +70,8 @@ Make sure these are installed on your system before building OpenEXR: * OpenEXR requires CMake version 3.12 or newer * C++ compiler that supports C++11 -* zlib -* Imath (auto fetched by CMake if not found) +* Imath (auto fetched by CMake if not found) (https://github.com/AcademySoftwareFoundation/openexr) +* libdeflate source code (auto fetched by CMake if not found) (https://github.com/ebiggers/libdeflate) The instructions that follow describe building OpenEXR with CMake. diff --git a/src/lib/OpenEXR/CMakeLists.txt b/src/lib/OpenEXR/CMakeLists.txt index a4e0c2497e..ba45a9c3fe 100644 --- a/src/lib/OpenEXR/CMakeLists.txt +++ b/src/lib/OpenEXR/CMakeLists.txt @@ -216,5 +216,5 @@ openexr_define_library(OpenEXR OpenEXR::Config OpenEXR::Iex OpenEXR::IlmThread - ZLIB::ZLIB + OpenEXR::OpenEXRCore ) diff --git a/src/lib/OpenEXR/ImfDwaCompressor.cpp b/src/lib/OpenEXR/ImfDwaCompressor.cpp index b4800a8208..ed79914402 100644 --- a/src/lib/OpenEXR/ImfDwaCompressor.cpp +++ b/src/lib/OpenEXR/ImfDwaCompressor.cpp @@ -139,7 +139,7 @@ # endif # define NOMINMAX #endif -#include +#include OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER @@ -2074,15 +2074,14 @@ DwaCompressor::compress ( if (*unknownUncompressedSize > 0) { - uLong inSize = static_cast (*unknownUncompressedSize); - uLong outSize = compressBound (inSize); - - if (Z_OK != ::compress2 ( - reinterpret_cast (outDataPtr), - &outSize, - reinterpret_cast (_planarUncBuffer[UNKNOWN]), - inSize, - 9)) + size_t outSize; + if (EXR_ERR_SUCCESS != exr_compress_buffer( + 9, // TODO: use default??? the old call to zlib had 9 hardcoded + _planarUncBuffer[UNKNOWN], + *unknownUncompressedSize, + outDataPtr, + exr_compress_max_buffer_size (*unknownUncompressedSize), + &outSize)) { throw IEX_NAMESPACE::BaseExc ("Data compression (zlib) failed."); } @@ -2114,16 +2113,15 @@ DwaCompressor::compress ( case DEFLATE: { - uLong sourceLen = static_cast (*totalAcUncompressedCount * sizeof (unsigned short)); - uLong destLen = compressBound (sourceLen); - - if (Z_OK != - ::compress2 ( - reinterpret_cast (outDataPtr), - &destLen, - reinterpret_cast (_packedAcBuffer), + size_t sourceLen = *totalAcUncompressedCount * sizeof (unsigned short); + size_t destLen; + if (EXR_ERR_SUCCESS != exr_compress_buffer( + 9, // TODO: use default??? the old call to zlib had 9 hardcoded + _packedAcBuffer, sourceLen, - 9)) + outDataPtr, + exr_compress_max_buffer_size (sourceLen), + &destLen)) { throw IEX_NAMESPACE::InputExc ( "Data compression (zlib) failed."); @@ -2166,15 +2164,14 @@ DwaCompressor::compress ( _planarUncBuffer[RLE], (signed char*) _rleBuffer); - uLong srcLen = static_cast (*rleUncompressedSize); - uLong dstLen = compressBound (srcLen); - - if (Z_OK != ::compress2 ( - reinterpret_cast (outDataPtr), - &dstLen, - reinterpret_cast (_rleBuffer), - srcLen, - 9)) + size_t dstLen; + if (EXR_ERR_SUCCESS != exr_compress_buffer( + 9, // TODO: use default??? the old call to zlib had 9 hardcoded + _rleBuffer, + *rleUncompressedSize, + outDataPtr, + exr_compress_max_buffer_size (*rleUncompressedSize), + &dstLen)) { throw IEX_NAMESPACE::BaseExc ("Error compressing RLE'd data."); } @@ -2405,14 +2402,12 @@ DwaCompressor::uncompress ( "(corrupt header)."); } - uLong inSize = static_cast (unknownCompressedSize); - uLong outSize = static_cast (unknownUncompressedSize); - - if (Z_OK != ::uncompress ( - reinterpret_cast (_planarUncBuffer[UNKNOWN]), - &outSize, - reinterpret_cast (compressedUnknownBuf), - inSize)) + if (EXR_ERR_SUCCESS != exr_uncompress_buffer ( + compressedUnknownBuf, + unknownCompressedSize, + _planarUncBuffer[UNKNOWN], + unknownUncompressedSize, + nullptr)) { throw IEX_NAMESPACE::BaseExc ("Error uncompressing UNKNOWN data."); } @@ -2449,14 +2444,14 @@ DwaCompressor::uncompress ( break; case DEFLATE: { - uLong destLen = static_cast (totalAcUncompressedCount * sizeof (unsigned short)); - uLong sourceLen = static_cast (acCompressedSize); - - if (Z_OK != ::uncompress ( - reinterpret_cast (_packedAcBuffer), - &destLen, - reinterpret_cast (compressedAcBuf), - sourceLen)) + size_t destLen; + + if (EXR_ERR_SUCCESS != exr_uncompress_buffer ( + compressedAcBuf, + acCompressedSize, + _packedAcBuffer, + totalAcUncompressedCount * sizeof (unsigned short), + &destLen)) { throw IEX_NAMESPACE::InputExc ( "Data decompression (zlib) failed."); @@ -2520,14 +2515,14 @@ DwaCompressor::uncompress ( "(corrupt header)."); } - uLong dstLen = static_cast (rleUncompressedSize); - uLong srcLen = static_cast (rleCompressedSize); + size_t dstLen; - if (Z_OK != ::uncompress ( - reinterpret_cast (_rleBuffer), - &dstLen, - reinterpret_cast (compressedRleBuf), - srcLen)) + if (EXR_ERR_SUCCESS != exr_uncompress_buffer ( + compressedRleBuf, + rleCompressedSize, + _rleBuffer, + rleUncompressedSize, + &dstLen)) { throw IEX_NAMESPACE::BaseExc ("Error uncompressing RLE data."); } @@ -2882,7 +2877,7 @@ DwaCompressor::initializeBuffers (size_t& outBufferSize) maxOutBufferSize += std::max ( 2lu * maxLossyDctAcSize + 65536lu, - static_cast (compressBound (static_cast (maxLossyDctAcSize)))); + static_cast (exr_compress_max_buffer_size (maxLossyDctAcSize))); numLossyDctChans++; break; @@ -2922,14 +2917,14 @@ DwaCompressor::initializeBuffers (size_t& outBufferSize) // which could take slightly more space // - maxOutBufferSize += static_cast (compressBound (static_cast (rleBufferSize))); + maxOutBufferSize += static_cast (exr_compress_max_buffer_size (rleBufferSize)); // // And the same goes for the UNKNOWN data // maxOutBufferSize += - static_cast (compressBound (static_cast (unknownBufferSize))); + static_cast (exr_compress_max_buffer_size (unknownBufferSize)); // // Allocate a zip/deflate compressor big enough to hold the DC data @@ -3048,7 +3043,7 @@ DwaCompressor::initializeBuffers (size_t& outBufferSize) if (planarUncBufferSize[UNKNOWN] > 0) { planarUncBufferSize[UNKNOWN] = static_cast ( - compressBound (static_cast (planarUncBufferSize[UNKNOWN]))); + exr_compress_max_buffer_size (planarUncBufferSize[UNKNOWN])); } for (int i = 0; i < NUM_COMPRESSOR_SCHEMES; ++i) diff --git a/src/lib/OpenEXR/ImfHeader.cpp b/src/lib/OpenEXR/ImfHeader.cpp index 57b8f6d2e1..ba2142a465 100644 --- a/src/lib/OpenEXR/ImfHeader.cpp +++ b/src/lib/OpenEXR/ImfHeader.cpp @@ -45,7 +45,7 @@ #include #include #include -#include +#include #include "ImfNamespace.h" #include "ImfTiledMisc.h" @@ -64,15 +64,13 @@ using IMATH_NAMESPACE::V2i; namespace { -static int s_DefaultZipCompressionLevel = 4; -static float s_DefaultDwaCompressionLevel = 45.f; - struct CompressionRecord { CompressionRecord () - : zip_level (s_DefaultZipCompressionLevel) - , dwa_level (s_DefaultDwaCompressionLevel) - {} + { + exr_get_default_zip_compression_level(&zip_level); + exr_get_default_dwa_compression_quality(&dwa_level); + } int zip_level; float dwa_level; }; @@ -259,13 +257,13 @@ sanityCheckDisplayWindow (int width, int height) void setDefaultZipCompressionLevel (int level) { - s_DefaultZipCompressionLevel = level; + exr_set_default_zip_compression_level (level); } void setDefaultDwaCompressionLevel (float level) { - s_DefaultDwaCompressionLevel = level; + exr_set_default_dwa_compression_quality (level); } Header::Header ( diff --git a/src/lib/OpenEXR/ImfIDManifest.cpp b/src/lib/OpenEXR/ImfIDManifest.cpp index 0117e4a49c..866c0b996b 100644 --- a/src/lib/OpenEXR/ImfIDManifest.cpp +++ b/src/lib/OpenEXR/ImfIDManifest.cpp @@ -11,7 +11,7 @@ #include "ImfXdr.h" #include #include -#include +#include #include #include @@ -536,14 +536,15 @@ IDManifest::IDManifest (const CompressedIDManifest& compressed) // decompress the compressed manifest // - vector uncomp (compressed._uncompressedDataSize); - uLong outSize = static_cast (compressed._uncompressedDataSize); - uLong inSize = static_cast (compressed._compressedDataSize); - if (Z_OK != ::uncompress ( - uncomp.data(), - &outSize, - reinterpret_cast (compressed._data), - inSize)) + vector uncomp (compressed._uncompressedDataSize); + size_t outSize; + size_t inSize = static_cast (compressed._compressedDataSize); + if (EXR_ERR_SUCCESS != exr_uncompress_buffer ( + compressed._data, + inSize, + uncomp.data(), + compressed._uncompressedDataSize, + &outSize)) { throw IEX_NAMESPACE::InputExc ( "IDManifest decompression (zlib) failed."); @@ -1062,16 +1063,21 @@ CompressedIDManifest::CompressedIDManifest (const IDManifest& manifest) manifest.serialize (serial); - uLong outputSize = static_cast (serial.size ()); + size_t outputSize = serial.size (); // // allocate a buffer which is guaranteed to be big enough for compression // - uLong compressedDataSize = compressBound (outputSize); - _data = (unsigned char*) malloc (compressedDataSize); - if (Z_OK != - ::compress ( - _data, &compressedDataSize, reinterpret_cast (serial.data ()), outputSize)) + size_t compressedBufferSize = exr_compress_max_buffer_size (outputSize); + size_t compressedDataSize; + _data = (unsigned char*) malloc (compressedBufferSize); + if (EXR_ERR_SUCCESS != exr_compress_buffer ( + -1, + serial.data (), + outputSize, + _data, + compressedBufferSize, + &compressedDataSize)) { throw IEX_NAMESPACE::InputExc ("ID manifest compression failed"); } diff --git a/src/lib/OpenEXR/ImfIDManifestAttribute.cpp b/src/lib/OpenEXR/ImfIDManifestAttribute.cpp index fa4f27d33f..53a9bfad0f 100644 --- a/src/lib/OpenEXR/ImfIDManifestAttribute.cpp +++ b/src/lib/OpenEXR/ImfIDManifestAttribute.cpp @@ -40,12 +40,12 @@ IDManifestAttribute::readValueFrom ( OPENEXR_IMF_INTERNAL_NAMESPACE::IStream& is, int size, int version) { - if (size < 4) + if (size < sizeof (uint64_t)) { throw IEX_NAMESPACE::InputExc ( "Invalid size field reading idmanifest attribute"); } - _value._compressedDataSize = size - 4; + _value._compressedDataSize = size - sizeof (uint64_t); if (_value._data) { @@ -65,7 +65,7 @@ IDManifestAttribute::readValueFrom ( // // allocate memory for compressed storage and read data // - _value._data = static_cast (malloc (size - 4)); + _value._data = static_cast (malloc (size - sizeof (uint64_t))); char* input = (char*) _value._data; Xdr::read (is, input, _value._compressedDataSize); } diff --git a/src/lib/OpenEXR/ImfPxr24Compressor.cpp b/src/lib/OpenEXR/ImfPxr24Compressor.cpp index e576395c28..90dfaad897 100644 --- a/src/lib/OpenEXR/ImfPxr24Compressor.cpp +++ b/src/lib/OpenEXR/ImfPxr24Compressor.cpp @@ -48,7 +48,7 @@ #include #include #include -#include +#include using namespace std; using namespace IMATH_NAMESPACE; @@ -160,8 +160,7 @@ Pxr24Compressor::Pxr24Compressor ( { size_t maxInBytes = uiMult (maxScanLineSize, numScanLines); - size_t maxOutBytes = uiAdd ( - uiAdd (maxInBytes, size_t (ceil (maxInBytes * 0.01))), size_t (100)); + size_t maxOutBytes = exr_compress_max_buffer_size (maxInBytes); _tmpBuffer = new unsigned char[maxInBytes]; _outBuffer = new char[maxOutBytes]; @@ -341,14 +340,16 @@ Pxr24Compressor::compress ( } } - uLong inBufferSize = static_cast (tmpBufferEnd - _tmpBuffer); - uLong outSize = compressBound (inBufferSize); + size_t inBufferSize = static_cast (tmpBufferEnd - _tmpBuffer); + size_t outSize = exr_compress_max_buffer_size (inBufferSize); - if (Z_OK != ::compress ( - reinterpret_cast (_outBuffer), - &outSize, - reinterpret_cast (_tmpBuffer), - inBufferSize)) + if (EXR_ERR_SUCCESS != exr_compress_buffer ( + -1, + _tmpBuffer, + inBufferSize, + _outBuffer, + outSize, + &outSize)) { throw IEX_NAMESPACE::BaseExc ("Data compression (zlib) failed."); } @@ -367,14 +368,14 @@ Pxr24Compressor::uncompress ( return 0; } - uLong tmpSize = static_cast (_maxScanLineSize * _numScanLines); + size_t tmpSize = static_cast (_maxScanLineSize * _numScanLines); - if (Z_OK != - ::uncompress ( - reinterpret_cast (_tmpBuffer), - &tmpSize, - reinterpret_cast (inPtr), - inSize)) + if (EXR_ERR_SUCCESS != exr_uncompress_buffer( + inPtr, + inSize, + _tmpBuffer, + tmpSize, + &tmpSize)) { throw IEX_NAMESPACE::InputExc ("Data decompression (zlib) failed."); } @@ -412,7 +413,7 @@ Pxr24Compressor::uncompress ( ptr[3] = ptr[2] + n; tmpBufferEnd = ptr[3] + n; - if (static_cast (tmpBufferEnd - _tmpBuffer) > tmpSize) + if (static_cast (tmpBufferEnd - _tmpBuffer) > tmpSize) notEnoughData (); for (int j = 0; j < n; ++j) @@ -437,7 +438,7 @@ Pxr24Compressor::uncompress ( ptr[1] = ptr[0] + n; tmpBufferEnd = ptr[1] + n; - if (static_cast (tmpBufferEnd - _tmpBuffer) > tmpSize) + if (static_cast (tmpBufferEnd - _tmpBuffer) > tmpSize) notEnoughData (); for (int j = 0; j < n; ++j) @@ -460,7 +461,7 @@ Pxr24Compressor::uncompress ( ptr[2] = ptr[1] + n; tmpBufferEnd = ptr[2] + n; - if (static_cast (tmpBufferEnd - _tmpBuffer) > tmpSize) + if (static_cast (tmpBufferEnd - _tmpBuffer) > tmpSize) notEnoughData (); for (int j = 0; j < n; ++j) @@ -483,7 +484,7 @@ Pxr24Compressor::uncompress ( } } - if (static_cast (tmpBufferEnd - _tmpBuffer) < tmpSize) tooMuchData (); + if (static_cast (tmpBufferEnd - _tmpBuffer) < tmpSize) tooMuchData (); outPtr = _outBuffer; return writePtr - _outBuffer; diff --git a/src/lib/OpenEXR/ImfZip.cpp b/src/lib/OpenEXR/ImfZip.cpp index 8dd53bea92..44ced27466 100644 --- a/src/lib/OpenEXR/ImfZip.cpp +++ b/src/lib/OpenEXR/ImfZip.cpp @@ -10,8 +10,7 @@ #include "ImfSimd.h" #include "ImfSystemSpecific.h" -#include -#include +#include OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER @@ -42,8 +41,7 @@ Zip::maxRawSize () size_t Zip::maxCompressedSize () { - return uiAdd ( - uiAdd (_maxRawSize, size_t (ceil (_maxRawSize * 0.01))), size_t (100)); + return exr_compress_max_buffer_size (_maxRawSize); } int @@ -93,18 +91,16 @@ Zip::compress (const char* raw, int rawSize, char* compressed) // // Compress the data using zlib // - - uLong inSize = static_cast (rawSize); - uLong outSize = compressBound (inSize); - - if (Z_OK != ::compress2 ( - reinterpret_cast (compressed), - &outSize, - reinterpret_cast (_tmpBuffer), - inSize, - _zipLevel)) + size_t outSize; + if (EXR_ERR_SUCCESS != exr_compress_buffer ( + _zipLevel, + _tmpBuffer, + rawSize, + compressed, + maxCompressedSize (), + &outSize)) { - throw IEX_NAMESPACE::BaseExc ("Data compression (zlib) failed."); + throw IEX_NAMESPACE::BaseExc ("Data compression failed."); } return outSize; @@ -330,23 +326,18 @@ auto interleave = interleave_scalar; int Zip::uncompress (const char* compressed, int compressedSize, char* raw) { - // - // Decompress the data using zlib - // - - uLong outSize = static_cast (_maxRawSize); - uLong inSize = static_cast (compressedSize); - - if (Z_OK != ::uncompress ( - reinterpret_cast (_tmpBuffer), - &outSize, - reinterpret_cast (compressed), - inSize)) + size_t outSize = 0; + if (EXR_ERR_SUCCESS != exr_uncompress_buffer ( + compressed, + (size_t)compressedSize, + _tmpBuffer, + _maxRawSize, + &outSize)) { - throw IEX_NAMESPACE::InputExc ("Data decompression (zlib) failed."); + throw IEX_NAMESPACE::InputExc ("Data decompression failed."); } - if (outSize == 0) { return outSize; } + if (outSize == 0) { return static_cast (outSize); } // // Predictor. diff --git a/src/lib/OpenEXR/ImfZipCompressor.cpp b/src/lib/OpenEXR/ImfZipCompressor.cpp index cb667e00c2..be5610c673 100644 --- a/src/lib/OpenEXR/ImfZipCompressor.cpp +++ b/src/lib/OpenEXR/ImfZipCompressor.cpp @@ -14,7 +14,6 @@ #include "ImfCheckedArithmetic.h" #include "ImfHeader.h" #include "ImfNamespace.h" -#include OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER diff --git a/src/lib/OpenEXRCore/CMakeLists.txt b/src/lib/OpenEXRCore/CMakeLists.txt index a590c6b8fb..be84689eb7 100644 --- a/src/lib/OpenEXRCore/CMakeLists.txt +++ b/src/lib/OpenEXRCore/CMakeLists.txt @@ -60,6 +60,7 @@ openexr_define_library(OpenEXRCore chunk.c coding.c + compression.c decoding.c encoding.c pack.c @@ -68,6 +69,8 @@ openexr_define_library(OpenEXRCore debug.c + ${EXR_DEFLATE_SOURCES} + HEADERS openexr.h @@ -75,6 +78,7 @@ openexr_define_library(OpenEXRCore openexr_base.h openexr_chunkio.h openexr_coding.h + openexr_compression.h openexr_conf.h openexr_context.h openexr_decode.h @@ -83,8 +87,6 @@ openexr_define_library(OpenEXRCore openexr_errors.h openexr_part.h openexr_std_attr.h - DEPENDENCIES - ZLIB::ZLIB PRIVATE_DEPS ${OPENEXR_EXTRA_MATH_LIB} ) @@ -97,3 +99,11 @@ if (TARGET Imath::ImathConfig) else() target_include_directories(OpenEXRCore PRIVATE ${IMATH_HEADER_ONLY_INCLUDE_DIRS}) endif() + +if (DEFINED EXR_DEFLATE_LIB) + if (BUILD_SHARED_LIBS) + target_link_libraries(OpenEXRCore PRIVATE ${EXR_DEFLATE_LIB}) + else() + target_link_libraries(OpenEXRCore PUBLIC ${EXR_DEFLATE_LIB}) + endif() +endif() diff --git a/src/lib/OpenEXRCore/chunk.c b/src/lib/OpenEXRCore/chunk.c index ae5cde3fcf..f0f760b714 100644 --- a/src/lib/OpenEXRCore/chunk.c +++ b/src/lib/OpenEXRCore/chunk.c @@ -456,7 +456,7 @@ reconstruct_chunk_table ( uint64_t offset_start, chunk_start, max_offset; uint64_t* curctable; const struct _internal_exr_part* curpart = NULL; - int maxidx, found_ci, computed_ci, partnum = 0; + int found_ci, computed_ci, partnum = 0; curpart = ctxt->parts[ctxt->num_parts - 1]; offset_start = curpart->chunk_table_offset; @@ -481,12 +481,10 @@ reconstruct_chunk_table ( if (rv != EXR_ERR_SUCCESS) return rv; chunk_start = curctable[0]; - maxidx = 0; for (int ci = 1; ci < curpart->chunk_count; ++ci) { if (curctable[ci] > chunk_start) { - maxidx = ci; chunk_start = curctable[ci]; } } diff --git a/src/lib/OpenEXRCore/compression.c b/src/lib/OpenEXRCore/compression.c new file mode 100644 index 0000000000..78821da1b2 --- /dev/null +++ b/src/lib/OpenEXRCore/compression.c @@ -0,0 +1,127 @@ +/* +** SPDX-License-Identifier: BSD-3-Clause +** Copyright Contributors to the OpenEXR Project. +*/ + +#include "openexr_compression.h" +#include "openexr_base.h" + +#include + +/* value Aras found to be better trade off of speed vs size */ +#define EXR_DEFAULT_ZLIB_COMPRESS_LEVEL 4 + +/**************************************/ + +size_t exr_compress_max_buffer_size (size_t in_bytes) +{ + size_t r, extra; + + r = libdeflate_zlib_compress_bound (NULL, in_bytes); + /* + * lib deflate has a message about needing a 9 byte boundary + * but is unclear if it actually adds that or not + * (see the comment on libdeflate_deflate_compress) + */ + if (r > (SIZE_MAX - 9)) + return (size_t)(SIZE_MAX); + r += 9; + + /* + * old library had uiAdd( uiAdd( in, ceil(in * 0.01) ), 100 ) + */ + extra = (in_bytes * (size_t)130); + if (extra < in_bytes) + return (size_t)(SIZE_MAX); + extra /= (size_t)128; + + if (extra > (SIZE_MAX - 100)) + return (size_t)(SIZE_MAX); + + if (extra > r) + r = extra; + return r; +} + +/**************************************/ + +exr_result_t exr_compress_buffer ( + int level, + const void *in, + size_t in_bytes, + void *out, + size_t out_bytes_avail, + size_t *actual_out ) +{ + struct libdeflate_compressor *comp; + + if (level < 0) + { + exr_get_default_zip_compression_level (&level); + /* truly unset anywhere */ + if (level < 0) + level = EXR_DEFAULT_ZLIB_COMPRESS_LEVEL; + } + + comp = libdeflate_alloc_compressor (level); + if (comp) + { + size_t outsz; + outsz = libdeflate_zlib_compress ( + comp, + in, + in_bytes, + out, + out_bytes_avail); + + libdeflate_free_compressor (comp); + + if (outsz != 0) + { + if (actual_out) + *actual_out = outsz; + return EXR_ERR_SUCCESS; + } + return EXR_ERR_OUT_OF_MEMORY; + } + return EXR_ERR_OUT_OF_MEMORY; +} + +/**************************************/ + +exr_result_t exr_uncompress_buffer ( + const void *in, + size_t in_bytes, + void *out, + size_t out_bytes_avail, + size_t *actual_out ) +{ + struct libdeflate_decompressor *decomp; + enum libdeflate_result res; + size_t actual_in_bytes; + + decomp = libdeflate_alloc_decompressor (); + if (decomp) + { + res = libdeflate_zlib_decompress_ex ( + decomp, + in, + in_bytes, + out, + out_bytes_avail, + &actual_in_bytes, + actual_out); + + libdeflate_free_decompressor (decomp); + + if (res == LIBDEFLATE_SUCCESS) + { + if (in_bytes == actual_in_bytes) + return EXR_ERR_SUCCESS; + /* it's an error to not consume the full buffer, right? */ + } + return EXR_ERR_CORRUPT_CHUNK; + } + return EXR_ERR_OUT_OF_MEMORY; +} + diff --git a/src/lib/OpenEXRCore/encoding.c b/src/lib/OpenEXRCore/encoding.c index 68f123a1bc..4cbdb06650 100644 --- a/src/lib/OpenEXRCore/encoding.c +++ b/src/lib/OpenEXRCore/encoding.c @@ -10,6 +10,8 @@ #include "internal_structs.h" #include "internal_xdr.h" +#include "openexr_compression.h" + /**************************************/ static exr_result_t @@ -24,9 +26,14 @@ default_compress_chunk (exr_encode_pipeline_t* encode) EXR_TRANSCODE_BUFFER_COMPRESSED, &(encode->compressed_buffer), &(encode->compressed_alloc_size), - (((size_t) encode->packed_bytes) * (size_t) 110) / ((size_t) 100) + - 65536); - if (rv != EXR_ERR_SUCCESS) return rv; + exr_compress_max_buffer_size( encode->packed_bytes )); + if (rv != EXR_ERR_SUCCESS) + return pctxt->print_error ( + pctxt, + rv, + "error allocating buffer %lu", + exr_compress_max_buffer_size (encode->packed_bytes)); + //return rv; switch (part->comp_type) { diff --git a/src/lib/OpenEXRCore/internal_pxr24.c b/src/lib/OpenEXRCore/internal_pxr24.c index f1928ad3b4..0aaa4a1e10 100644 --- a/src/lib/OpenEXRCore/internal_pxr24.c +++ b/src/lib/OpenEXRCore/internal_pxr24.c @@ -10,7 +10,7 @@ #include "internal_xdr.h" #include -#include +#include "openexr_compression.h" /**************************************/ @@ -92,7 +92,8 @@ apply_pxr24_impl (exr_encode_pipeline_t* encode) uint8_t* out = encode->scratch_buffer_1; uint64_t nOut = 0; const uint8_t* lastIn = encode->packed_buffer; - uLong compbufsz = (uLong) encode->compressed_alloc_size; + size_t compbufsz; + exr_result_t rv; for (int y = 0; y < encode->chunk.height; ++y) { @@ -215,24 +216,27 @@ apply_pxr24_impl (exr_encode_pipeline_t* encode) } } - if (Z_OK != compress ( - (Bytef*) encode->compressed_buffer, - &compbufsz, - (const Bytef*) encode->scratch_buffer_1, - (uLong) nOut)) - { - return EXR_ERR_CORRUPT_CHUNK; - } - if (compbufsz > encode->packed_bytes) + rv = exr_compress_buffer ( + -1, + encode->scratch_buffer_1, + nOut, + encode->compressed_buffer, + encode->compressed_alloc_size, + &compbufsz ); + + if (rv == EXR_ERR_SUCCESS) { - memcpy ( - encode->compressed_buffer, - encode->packed_buffer, - encode->packed_bytes); - compbufsz = (uLong) encode->packed_bytes; + if (compbufsz > encode->packed_bytes) + { + memcpy ( + encode->compressed_buffer, + encode->packed_buffer, + encode->packed_bytes); + compbufsz = encode->packed_bytes; + } + encode->compressed_bytes = compbufsz; } - encode->compressed_bytes = compbufsz; - return EXR_ERR_SUCCESS; + return rv; } exr_result_t @@ -262,8 +266,8 @@ undo_pxr24_impl ( void* scratch_data, uint64_t scratch_size) { - uLong outSize = (uLong) uncompressed_size; - int rstat; + size_t outSize; + exr_result_t rstat; uint8_t* out = uncompressed_data; uint64_t nOut = 0; uint64_t nDec = 0; @@ -271,13 +275,14 @@ undo_pxr24_impl ( if (scratch_size < uncompressed_size) return EXR_ERR_INVALID_ARGUMENT; - rstat = uncompress ( - (Bytef*) scratch_data, - &outSize, - (const Bytef*) compressed_data, - (uLong) comp_buf_size); + rstat = exr_uncompress_buffer ( + compressed_data, + comp_buf_size, + scratch_data, + uncompressed_size, + &outSize); - if (rstat != Z_OK) return EXR_ERR_CORRUPT_CHUNK; + if (rstat != EXR_ERR_SUCCESS) return rstat; for (int y = 0; y < decode->chunk.height; ++y) { diff --git a/src/lib/OpenEXRCore/internal_zip.c b/src/lib/OpenEXRCore/internal_zip.c index 1374cab0d0..7707b5ef73 100644 --- a/src/lib/OpenEXRCore/internal_zip.c +++ b/src/lib/OpenEXRCore/internal_zip.c @@ -13,7 +13,8 @@ #include #include #include -#include + +#include "openexr_compression.h" #if defined __SSE2__ || (_MSC_VER >= 1300 && (_M_IX86 || _M_X64)) # define IMF_HAVE_SSE2 1 @@ -235,35 +236,30 @@ undo_zip_impl ( void* scratch_data, uint64_t scratch_size) { - uLong outSize = (uLong) scratch_size; - int rstat; + size_t actual_out_bytes; + exr_result_t res; if (scratch_size < uncompressed_size) return EXR_ERR_INVALID_ARGUMENT; - rstat = uncompress ( - (Bytef*) scratch_data, - &outSize, - (const Bytef*) compressed_data, - (uLong) comp_buf_size); - if (rstat == Z_OK) + res = exr_uncompress_buffer ( + compressed_data, + comp_buf_size, + scratch_data, + scratch_size, + &actual_out_bytes); + + if (res == EXR_ERR_SUCCESS) { - if (outSize == uncompressed_size) + if (actual_out_bytes == uncompressed_size) { - reconstruct (scratch_data, outSize); - interleave (uncompressed_data, scratch_data, outSize); - rstat = EXR_ERR_SUCCESS; + reconstruct (scratch_data, actual_out_bytes); + interleave (uncompressed_data, scratch_data, actual_out_bytes); } else - { - rstat = EXR_ERR_CORRUPT_CHUNK; - } - } - else - { - rstat = EXR_ERR_CORRUPT_CHUNK; + res = EXR_ERR_CORRUPT_CHUNK; } - return (exr_result_t) rstat; + return res; } /**************************************/ @@ -307,8 +303,8 @@ apply_zip_impl (exr_encode_pipeline_t* encode) const uint8_t* raw = encode->packed_buffer; const uint8_t* stop = raw + encode->packed_bytes; int p, level; - uLong compbufsz = (uLong) encode->compressed_alloc_size; - exr_result_t rv = EXR_ERR_SUCCESS; + size_t compbufsz; + exr_result_t rv; rv = exr_get_zip_compression_level ( encode->context, encode->part_index, &level); @@ -334,25 +330,32 @@ apply_zip_impl (exr_encode_pipeline_t* encode) ++t1; } - if (Z_OK != compress2 ( - (Bytef*) encode->compressed_buffer, - &compbufsz, - (const Bytef*) encode->scratch_buffer_1, - (uLong) encode->packed_bytes, - level)) + rv = exr_compress_buffer ( + level, + encode->scratch_buffer_1, + encode->packed_bytes, + encode->compressed_buffer, + encode->compressed_alloc_size, + &compbufsz); + + if (rv == EXR_ERR_SUCCESS) { - return EXR_ERR_CORRUPT_CHUNK; + if (compbufsz > encode->packed_bytes) + { + memcpy ( + encode->compressed_buffer, + encode->packed_buffer, + encode->packed_bytes); + compbufsz = encode->packed_bytes; + } + encode->compressed_bytes = compbufsz; } - if (compbufsz > encode->packed_bytes) + else { - memcpy ( - encode->compressed_buffer, - encode->packed_buffer, - encode->packed_bytes); - compbufsz = encode->packed_bytes; + printf("ZIP: Unable to compress buffer %lu -> %lu @ level %d\n", encode->packed_bytes, encode->compressed_alloc_size, level); } - encode->compressed_bytes = compbufsz; - return EXR_ERR_SUCCESS; + + return rv; } exr_result_t @@ -366,7 +369,11 @@ internal_exr_apply_zip (exr_encode_pipeline_t* encode) &(encode->scratch_buffer_1), &(encode->scratch_alloc_size_1), encode->packed_bytes); - if (rv != EXR_ERR_SUCCESS) return rv; + if (rv != EXR_ERR_SUCCESS) + { + printf("ZIP: Unable to alloc scratch buffer\n"); + return rv; + } return apply_zip_impl (encode); } diff --git a/src/lib/OpenEXRCore/memory.c b/src/lib/OpenEXRCore/memory.c index b1b38333ec..dc98c7916b 100644 --- a/src/lib/OpenEXRCore/memory.c +++ b/src/lib/OpenEXRCore/memory.c @@ -10,6 +10,7 @@ #else # include #endif +#include /**************************************/ @@ -24,6 +25,7 @@ exr_set_default_memory_routines ( { _glob_alloc_func = alloc_func; _glob_free_func = free_func; + libdeflate_set_memory_allocator (alloc_func, free_func); } /**************************************/ diff --git a/src/lib/OpenEXRCore/openexr.h b/src/lib/OpenEXRCore/openexr.h index 668d647709..08eb92a460 100644 --- a/src/lib/OpenEXRCore/openexr.h +++ b/src/lib/OpenEXRCore/openexr.h @@ -20,6 +20,8 @@ #include "openexr_chunkio.h" +#include "openexr_compression.h" + #include "openexr_decode.h" #include "openexr_encode.h" diff --git a/src/lib/OpenEXRCore/openexr_compression.h b/src/lib/OpenEXRCore/openexr_compression.h new file mode 100644 index 0000000000..08470d4aa5 --- /dev/null +++ b/src/lib/OpenEXRCore/openexr_compression.h @@ -0,0 +1,50 @@ +/* +** SPDX-License-Identifier: BSD-3-Clause +** Copyright Contributors to the OpenEXR Project. +*/ + +#ifndef OPENEXR_CORE_COMPRESSION_H +#define OPENEXR_CORE_COMPRESSION_H + +#include "openexr_errors.h" + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** @file */ + +/** Computes a buffer that will be large enough to hold the compressed + * data. This may include some extra padding for headers / scratch */ +EXR_EXPORT +size_t exr_compress_max_buffer_size (size_t in_bytes); + +/** Compresses a buffer using a zlib style compression. + * + * If the level is -1, will use the default compression set to the library + * \ref exr_set_default_zip_compression_level + * data. This may include some extra padding for headers / scratch */ +EXR_EXPORT +exr_result_t exr_compress_buffer ( + int level, + const void *in, + size_t in_bytes, + void *out, + size_t out_bytes_avail, + size_t *actual_out); + +EXR_EXPORT +exr_result_t exr_uncompress_buffer ( + const void *in, + size_t in_bytes, + void *out, + size_t out_bytes_avail, + size_t *actual_out); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* OPENEXR_CORE_COMPRESSION_H */ diff --git a/src/lib/OpenEXRCore/unpack.c b/src/lib/OpenEXRCore/unpack.c index ac4e259f39..8a74a1db8f 100644 --- a/src/lib/OpenEXRCore/unpack.c +++ b/src/lib/OpenEXRCore/unpack.c @@ -397,14 +397,10 @@ unpack_16bit_3chan_planar (exr_decode_pipeline_t* decode) const uint16_t *in0, *in1, *in2; uint8_t * out0, *out1, *out2; int w, h; - int inc0, inc1, inc2; int linc0, linc1, linc2; w = decode->channels[0].width; h = decode->chunk.height; - inc0 = decode->channels[0].user_pixel_stride; - inc1 = decode->channels[1].user_pixel_stride; - inc2 = decode->channels[2].user_pixel_stride; linc0 = decode->channels[0].user_line_stride; linc1 = decode->channels[1].user_line_stride; linc2 = decode->channels[2].user_line_stride; @@ -451,14 +447,10 @@ unpack_half_to_float_3chan_planar (exr_decode_pipeline_t* decode) const uint16_t *in0, *in1, *in2; uint8_t * out0, *out1, *out2; int w, h; - int inc0, inc1, inc2; int linc0, linc1, linc2; w = decode->channels[0].width; h = decode->chunk.height; - inc0 = decode->channels[0].user_pixel_stride; - inc1 = decode->channels[1].user_pixel_stride; - inc2 = decode->channels[2].user_pixel_stride; linc0 = decode->channels[0].user_line_stride; linc1 = decode->channels[1].user_line_stride; linc2 = decode->channels[2].user_line_stride; diff --git a/src/test/OpenEXRCoreTest/CMakeLists.txt b/src/test/OpenEXRCoreTest/CMakeLists.txt index 4449de0b68..2d0b4f66b8 100644 --- a/src/test/OpenEXRCoreTest/CMakeLists.txt +++ b/src/test/OpenEXRCoreTest/CMakeLists.txt @@ -4,6 +4,8 @@ add_executable(OpenEXRCoreTest base_units.cpp base_units.h + buffer.cpp + buffer.h compression.cpp compression.h deep.cpp @@ -60,6 +62,7 @@ define_openexrcore_tests( testBaseLimits testBaseDebug testXDR + testBufferCompression testAttrSizes testAttrStrings diff --git a/src/test/OpenEXRCoreTest/buffer.cpp b/src/test/OpenEXRCoreTest/buffer.cpp new file mode 100644 index 0000000000..041befe0a4 --- /dev/null +++ b/src/test/OpenEXRCoreTest/buffer.cpp @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright Contributors to the OpenEXR Project. + +// Windows specific addition to prevent the indirect import of the redefined min/max macros +#if defined _WIN32 || defined _WIN64 +# ifdef NOMINMAX +# undef NOMINMAX +# endif +# define NOMINMAX +#endif + +#include + +#include "test_value.h" +#include + +void testBufferCompression (const std::string& tempdir) +{ + size_t bc0 = exr_compress_max_buffer_size(0); + size_t bc128 = exr_compress_max_buffer_size(128); + if (bc0 < 9) + EXRCORE_TEST_FAIL(bc0 < 9); + if (bc128 < 128 + 9) + EXRCORE_TEST_FAIL(bc0 < 9); + std::cout << "Max Buffer Size (0): " << bc0 << std::endl; + std::cout << "Max Buffer Size (128): " << bc128 << std::endl; + + std::vector buf, cbuf; + buf = {'O', 'p', 'e', 'n', 'E', 'X', 'R'}; + cbuf.resize( exr_compress_max_buffer_size (buf.size()) ); + size_t outsz; + EXRCORE_TEST_RVAL (exr_compress_buffer (9, buf.data(), buf.size(), &cbuf[0], cbuf.size(), &outsz)); + std::cout << "compressed size: " << outsz << std::endl; + EXRCORE_TEST_RVAL (exr_uncompress_buffer (cbuf.data(), outsz, &buf[0], buf.size(), &outsz)); + std::cout << "uncompressed size: " << outsz << std::endl; + if (buf[0] != 'O') + EXRCORE_TEST_FAIL(buf[0] != 'O'); +} diff --git a/src/test/OpenEXRCoreTest/buffer.h b/src/test/OpenEXRCoreTest/buffer.h new file mode 100644 index 0000000000..24a1df9542 --- /dev/null +++ b/src/test/OpenEXRCoreTest/buffer.h @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright Contributors to the OpenEXR Project. + +#ifndef OPENEXR_CORE_TEST_BUFFER_H +#define OPENEXR_CORE_TEST_BUFFER_H + +void testBufferCompression (const std::string& tempdir); + +#endif // OPENEXR_CORE_TEST_BUFFER_H diff --git a/src/test/OpenEXRCoreTest/main.cpp b/src/test/OpenEXRCoreTest/main.cpp index 359b0c2f3a..970bbd82c4 100644 --- a/src/test/OpenEXRCoreTest/main.cpp +++ b/src/test/OpenEXRCoreTest/main.cpp @@ -26,6 +26,7 @@ #include "general_attr.h" #include "read.h" #include "write.h" +#include "buffer.h" #if defined(ANDROID) || defined(__ANDROID_API__) # define IMF_TMP_DIR "/sdcard/" @@ -149,6 +150,7 @@ main (int argc, char* argv[]) TEST (testBaseLimits, "core"); TEST (testBaseDebug, "core"); TEST (testXDR, "core"); + TEST (testBufferCompression, "core"); TEST (testAttrSizes, "gen_attr"); TEST (testAttrStrings, "gen_attr"); diff --git a/src/test/OpenEXRTest/testIDManifest.cpp b/src/test/OpenEXRTest/testIDManifest.cpp index 11f42b521a..0841260a1e 100644 --- a/src/test/OpenEXRTest/testIDManifest.cpp +++ b/src/test/OpenEXRTest/testIDManifest.cpp @@ -16,7 +16,7 @@ #include #include "tmpDir.h" -#include +#include namespace IMF = OPENEXR_IMF_NAMESPACE; using namespace IMF; @@ -136,15 +136,17 @@ doReadWriteManifest (const IDManifest& mfst, const string& fn, bool dump) // // allocate a buffer which is guaranteed to be big enough for compression // - uLong sourceDataSize = static_cast (str.str ().size ()); - uLong compressedDataSize = compressBound (sourceDataSize); + size_t sourceDataSize = str.str ().size (); + size_t compressedDataSize = exr_compress_max_buffer_size (sourceDataSize); vector compressed (compressedDataSize); - ::compress ( - reinterpret_cast (compressed.data ()), - &compressedDataSize, - reinterpret_cast (str.str ().c_str ()), - sourceDataSize); + exr_compress_buffer ( + 6, + str.str ().c_str (), + sourceDataSize, + compressed.data (), + compressedDataSize, + &compressedDataSize); cerr << "simple zip size: " << compressedDataSize << ' '; #endif