diff --git a/Release/CMakeLists.txt b/Release/CMakeLists.txt index 563b13d0dc..4a5bdef421 100644 --- a/Release/CMakeLists.txt +++ b/Release/CMakeLists.txt @@ -18,6 +18,7 @@ enable_testing() set(WERROR ON CACHE BOOL "Treat Warnings as Errors.") set(CPPREST_EXCLUDE_WEBSOCKETS OFF CACHE BOOL "Exclude websockets functionality.") set(CPPREST_EXCLUDE_COMPRESSION OFF CACHE BOOL "Exclude compression functionality.") +set(CPPREST_EXCLUDE_BROTLI ON CACHE BOOL "Exclude Brotli compression functionality.") set(CPPREST_EXPORT_DIR cpprestsdk CACHE STRING "Directory to install CMake config files.") set(CPPREST_INSTALL_HEADERS ON CACHE BOOL "Install header files.") set(CPPREST_INSTALL ON CACHE BOOL "Add install commands.") @@ -62,6 +63,7 @@ include(cmake/cpprest_find_boost.cmake) include(cmake/cpprest_find_zlib.cmake) include(cmake/cpprest_find_openssl.cmake) include(cmake/cpprest_find_websocketpp.cmake) +include(cmake/cpprest_find_brotli.cmake) include(CheckIncludeFiles) include(GNUInstallDirs) diff --git a/Release/cmake/cpprest_find_brotli.cmake b/Release/cmake/cpprest_find_brotli.cmake new file mode 100644 index 0000000000..2ec59a9ad9 --- /dev/null +++ b/Release/cmake/cpprest_find_brotli.cmake @@ -0,0 +1,10 @@ +function(cpprest_find_brotli) + if(TARGET cpprestsdk_brotli_internal) + return() + endif() + + find_package(unofficial-brotli REQUIRED) + + add_library(cpprestsdk_brotli_internal INTERFACE) + target_link_libraries(cpprestsdk_brotli_internal INTERFACE unofficial::brotli::brotlienc unofficial::brotli::brotlidec unofficial::brotli::brotlicommon) +endfunction() diff --git a/Release/cmake/cpprestsdk-config.in.cmake b/Release/cmake/cpprestsdk-config.in.cmake index 522f8f7db1..8b5e8a6ff3 100644 --- a/Release/cmake/cpprestsdk-config.in.cmake +++ b/Release/cmake/cpprestsdk-config.in.cmake @@ -3,6 +3,10 @@ if(@CPPREST_USES_ZLIB@) find_dependency(ZLIB) endif() +if(@CPPREST_USES_BROTLI@) + find_dependency(unofficial-brotli) +endif() + if(@CPPREST_USES_OPENSSL@) find_dependency(OpenSSL) endif() diff --git a/Release/include/cpprest/asyncrt_utils.h b/Release/include/cpprest/asyncrt_utils.h index 21c92de294..2739a13e35 100644 --- a/Release/include/cpprest/asyncrt_utils.h +++ b/Release/include/cpprest/asyncrt_utils.h @@ -503,7 +503,7 @@ _ASYNCRTIMP const std::error_category & __cdecl linux_category(); /// /// Gets the one global instance of the current platform's error category. -/// +/// _ASYNCRTIMP const std::error_category & __cdecl platform_category(); /// diff --git a/Release/include/cpprest/details/http_helpers.h b/Release/include/cpprest/details/http_helpers.h index 596ac9efaa..ad01e2eb67 100644 --- a/Release/include/cpprest/details/http_helpers.h +++ b/Release/include/cpprest/details/http_helpers.h @@ -41,70 +41,4 @@ namespace details _ASYNCRTIMP size_t __cdecl add_chunked_delimiters(_Out_writes_(buffer_size) uint8_t *data, _In_ size_t buffer_size, size_t bytes_read); } - namespace compression - { - enum class compression_algorithm : int - { - deflate = 15, - gzip = 31, - invalid = 9999 - }; - - using data_buffer = std::vector; - - class stream_decompressor - { - public: - - static compression_algorithm to_compression_algorithm(const utility::string_t& alg) - { - if (_XPLATSTR("gzip") == alg) - { - return compression_algorithm::gzip; - } - else if (_XPLATSTR("deflate") == alg) - { - return compression_algorithm::deflate; - } - - return compression_algorithm::invalid; - } - - static utility::string_t known_algorithms() { return _XPLATSTR("deflate, gzip"); } - - _ASYNCRTIMP static bool __cdecl is_supported(); - - _ASYNCRTIMP stream_decompressor(compression_algorithm alg); - - _ASYNCRTIMP data_buffer decompress(const data_buffer& input); - - _ASYNCRTIMP data_buffer decompress(const uint8_t* input, size_t input_size); - - _ASYNCRTIMP bool has_error() const; - - private: - class stream_decompressor_impl; - std::shared_ptr m_pimpl; - }; - - class stream_compressor - { - public: - - _ASYNCRTIMP static bool __cdecl is_supported(); - - _ASYNCRTIMP stream_compressor(compression_algorithm alg); - - _ASYNCRTIMP data_buffer compress(const data_buffer& input, bool finish); - - _ASYNCRTIMP data_buffer compress(const uint8_t* input, size_t input_size, bool finish); - - _ASYNCRTIMP bool has_error() const; - - private: - class stream_compressor_impl; - std::shared_ptr m_pimpl; - }; - - } }}} diff --git a/Release/include/cpprest/http_client.h b/Release/include/cpprest/http_client.h index f5ad8fac70..9ccd19e61f 100644 --- a/Release/include/cpprest/http_client.h +++ b/Release/include/cpprest/http_client.h @@ -247,20 +247,22 @@ class http_client_config } /// - /// Checks if requesting a compressed response is turned on, the default is off. + /// Checks if requesting a compressed response using Content-Encoding is turned on, the default is off. /// - /// True if compressed response is enabled, false otherwise + /// True if a content-encoded compressed response is allowed, false otherwise bool request_compressed_response() const { return m_request_compressed; } /// - /// Request that the server responds with a compressed body. - /// If true, in cases where the server does not support compression, this will have no effect. + /// Request that the server respond with a compressed body using Content-Encoding; to use Transfer-Encoding, do not + /// set this, and specify a vector of pointers + /// to the set_decompress_factories method of the object for the request. + /// If true and the server does not support compression, this will have no effect. /// The response body is internally decompressed before the consumer receives the data. /// - /// True to turn on response body compression, false otherwise. + /// True to turn on content-encoded response body compression, false otherwise. /// Please note there is a performance cost due to copying the request data. Currently only supported on Windows and OSX. void set_request_compressed_response(bool request_compressed) { diff --git a/Release/include/cpprest/http_compression.h b/Release/include/cpprest/http_compression.h new file mode 100644 index 0000000000..13b183af00 --- /dev/null +++ b/Release/include/cpprest/http_compression.h @@ -0,0 +1,319 @@ +/*** + * Copyright (C) Microsoft. All rights reserved. + * Licensed under the MIT license. See LICENSE.txt file in the project root for full license information. + * + * =+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * + * HTTP Library: Compression and decompression interfaces + * + * For the latest on this and related APIs, please see: https://github.com/Microsoft/cpprestsdk + * + * =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + ****/ +#pragma once + +namespace web +{ +namespace http +{ +namespace compression +{ +/// +/// Hint as to whether a compress or decompress call is meant to be the last for a particular HTTP request or reply +/// +enum operation_hint +{ + is_last, // Used for the expected last compress() call, or for an expected single decompress() call + has_more // Used when further compress() calls will be made, or when multiple decompress() calls may be required +}; + +/// +/// Result structure for asynchronous compression and decompression operations +/// +struct operation_result +{ + size_t input_bytes_processed; // From the input buffer + size_t output_bytes_produced; // To the output buffer + bool done; // For compress, set when 'last' is true and there was enough space to complete compression; + // for decompress, set if the end of the decompression stream has been reached +}; + +/// +/// Compression interface for use with HTTP requests +/// +class compress_provider +{ +public: + virtual const utility::string_t& algorithm() const = 0; + virtual size_t compress(const uint8_t* input, + size_t input_size, + uint8_t* output, + size_t output_size, + operation_hint hint, + size_t& input_bytes_processed, + bool* done = nullptr) = 0; + virtual pplx::task compress( + const uint8_t* input, size_t input_size, uint8_t* output, size_t output_size, operation_hint hint) = 0; + virtual void reset() = 0; + virtual ~compress_provider() = default; +}; + +/// +/// Decompression interface for use with HTTP requests +/// +class decompress_provider +{ +public: + virtual const utility::string_t& algorithm() const = 0; + virtual size_t decompress(const uint8_t* input, + size_t input_size, + uint8_t* output, + size_t output_size, + operation_hint hint, + size_t& input_bytes_processed, + bool* done = nullptr) = 0; + virtual pplx::task decompress( + const uint8_t* input, size_t input_size, uint8_t* output, size_t output_size, operation_hint hint) = 0; + virtual void reset() = 0; + virtual ~decompress_provider() = default; +}; + +/// +/// Factory interface for compressors for use with received HTTP requests +/// +class compress_factory +{ +public: + virtual const utility::string_t& algorithm() const = 0; + virtual std::unique_ptr make_compressor() const = 0; + virtual ~compress_factory() = default; +}; + +/// +/// Factory interface for decompressors for use with HTTP requests +/// +class decompress_factory +{ +public: + virtual const utility::string_t& algorithm() const = 0; + virtual const uint16_t weight() const = 0; + virtual std::unique_ptr make_decompressor() const = 0; + virtual ~decompress_factory() = default; +}; + +/// +/// Built-in compression support +/// +namespace builtin +{ +/// +/// Test whether cpprestsdk was built with built-in compression support +/// True if cpprestsdk was built with built-in compression support, and false if not. +/// +_ASYNCRTIMP bool supported(); + +/// +// String constants for each built-in compression algorithm, for convenient use with the factory functions +/// +namespace algorithm +{ +constexpr utility::char_t *GZIP = _XPLATSTR("gzip"); +constexpr utility::char_t *DEFLATE = _XPLATSTR("deflate"); +constexpr utility::char_t *BROTLI = _XPLATSTR("br"); + +/// +/// Test whether cpprestsdk was built with built-in compression support and +/// the supplied string matches a supported built-in algorithm +/// The name of the algorithm to test for built-in support. +/// True if cpprestsdk was built with built-in compression support and +/// the supplied string matches a supported built-in algorithm, and false if not. +/// +_ASYNCRTIMP bool supported(const utility::string_t& algorithm); +}; + +/// +/// Factory function to instantiate a built-in compression provider with default parameters by compression algorithm +/// name. +/// +/// The name of the algorithm for which to instantiate a provider. +/// +/// A caller-owned pointer to a provider of the requested-type, or to nullptr if no such built-in type exists. +/// +_ASYNCRTIMP std::unique_ptr make_compressor(const utility::string_t& algorithm); + +/// +/// Factory function to instantiate a built-in decompression provider with default parameters by compression algorithm +/// name. +/// +/// The name of the algorithm for which to instantiate a provider. +/// +/// A caller-owned pointer to a provider of the requested-type, or to nullptr if no such built-in type exists. +/// +_ASYNCRTIMP std::unique_ptr make_decompressor(const utility::string_t& algorithm); + +/// +/// Factory function to obtain a pointer to a built-in compression provider factory by compression algorithm name. +/// +/// The name of the algorithm for which to find a factory. +/// +/// A caller-owned pointer to a provider of the requested-type, or to nullptr if no such built-in type exists. +/// +_ASYNCRTIMP std::shared_ptr get_compress_factory(const utility::string_t& algorithm); + +/// +/// Factory function to obtain a pointer to a built-in decompression provider factory by compression algorithm name. +/// +/// The name of the algorithm for which to find a factory. +/// +/// A caller-owned pointer to a provider of the requested-type, or to nullptr if no such built-in type exists. +/// +_ASYNCRTIMP std::shared_ptr get_decompress_factory(const utility::string_t& algorithm); + +/// +// Factory function to instantiate a built-in gzip compression provider with caller-selected parameters. +/// +/// +/// A caller-owned pointer to a gzip compression provider, or to nullptr if the library was built without built-in +/// compression support. +/// +_ASYNCRTIMP std::unique_ptr make_gzip_compressor(int compressionLevel, + int method, + int strategy, + int memLevel); + +/// +// Factory function to instantiate a built-in deflate compression provider with caller-selected parameters. +/// +/// +/// A caller-owned pointer to a deflate compression provider, or to nullptr if the library was built without built-in +/// compression support.. +/// +_ASYNCRTIMP std::unique_ptr make_deflate_compressor(int compressionLevel, + int method, + int strategy, + int memLevel); + +/// +// Factory function to instantiate a built-in Brotli compression provider with caller-selected parameters. +/// +/// +/// A caller-owned pointer to a Brotli compression provider, or to nullptr if the library was built without built-in +/// compression support. +/// +_ASYNCRTIMP std::unique_ptr make_brotli_compressor(uint32_t window, uint32_t quality, uint32_t mode); +} // namespace builtin + +/// +/// Factory function to instantiate a compression provider factory by compression algorithm name. +/// +/// The name of the algorithm supported by the factory. Must match that returned by the +/// web::http::compression::compress_provider type instantiated by the factory's make_compressor function. +/// The supplied string is copied, and thus need not remain valid once the call returns. +/// A factory function to be used to instantiate a compressor matching the factory's +/// reported algorithm. +/// +/// A pointer to a generic provider factory implementation configured with the supplied parameters. +/// +/// +/// This method may be used to conveniently instantiate a factory object for a caller-selected compress_provider. +/// That provider may be of the caller's own design, or it may be one of the built-in types. As such, this method may +/// be helpful when a caller wishes to build vectors containing a mix of custom and built-in providers. +/// +_ASYNCRTIMP std::shared_ptr make_compress_factory( + const utility::string_t& algorithm, std::function()> make_compressor); + +/// +/// Factory function to instantiate a decompression provider factory by compression algorithm name. +/// +/// The name of the algorithm supported by the factory. Must match that returned by the +/// web::http::compression::decompress_provider type instantiated by the factory's make_decompressor function. +/// The supplied string is copied, and thus need not remain valid once the call returns. +/// A numeric weight for the compression algorithm, times 1000, for use as a "quality value" when +/// requesting that the server send a compressed response. Valid values are between 0 and 1000, inclusive, where higher +/// values indicate more preferred algorithms, and 0 indicates that the algorithm is not allowed; values greater than +/// 1000 are treated as 1000. +/// A factory function to be used to instantiate a decompressor matching the factory's +/// reported algorithm. +/// +/// A pointer to a generic provider factory implementation configured with the supplied parameters. +/// +/// +/// This method may be used to conveniently instantiate a factory object for a caller-selected +/// decompress_provider. That provider may be of the caller's own design, or it may be one of the built-in +/// types. As such, this method may be helpful when a caller wishes to change the weights of built-in provider types, +/// to use custom providers without explicitly implementing a decompress_factory, or to build vectors containing +/// a mix of custom and built-in providers. +/// +_ASYNCRTIMP std::shared_ptr make_decompress_factory( + const utility::string_t& algorithm, + uint16_t weight, + std::function()> make_decompressor); + +namespace details +{ +/// +/// Header type enum for use with compressor and decompressor header parsing and building functions +/// +enum header_types +{ + transfer_encoding, + content_encoding, + te, + accept_encoding +}; + +/// +/// Factory function to instantiate an appropriate compression provider, if any. +/// +/// A TE or Accept-Encoding header to interpret. +/// Specifies the type of header whose contents are in the encoding parameter; valid values are +/// header_type::te and header_type::accept_encoding. +/// A compressor object of the caller's preferred (possibly custom) type, which is used if +/// possible. +/// A collection of factory objects for use in construction of an appropriate compressor, if +/// any. If empty or not supplied, the set of supported built-in compressors is used. +/// +/// A pointer to a compressor object that is acceptable per the supplied header, or to nullptr if no matching +/// algorithm is found. +/// +_ASYNCRTIMP std::unique_ptr get_compressor_from_header( + const utility::string_t& encoding, + header_types type, + const std::vector>& factories = std::vector>()); + +/// +/// Factory function to instantiate an appropriate decompression provider, if any. +/// +/// A Transfer-Encoding or Content-Encoding header to interpret. +/// Specifies the type of header whose contents are in the encoding parameter; valid values are +/// header_type::transfer_encoding and header_type::content_encoding. +/// A collection of factory objects for use in construction of an appropriate decompressor, +/// if any. If empty or not supplied, the set of supported built-in compressors is used. +/// +/// A pointer to a decompressor object that is acceptable per the supplied header, or to nullptr if no matching +/// algorithm is found. +/// +_ASYNCRTIMP std::unique_ptr get_decompressor_from_header( + const utility::string_t& encoding, + header_types type, + const std::vector>& factories = + std::vector>()); + +/// +/// Helper function to compose a TE or Accept-Encoding header with supported, and possibly ranked, compression +/// algorithms. +/// +/// Specifies the type of header to be built; valid values are header_type::te and +/// header_type::accept_encoding. +/// A collection of factory objects for use in header construction. If empty or not +/// supplied, the set of supported built-in compressors is used. +/// +/// A well-formed header, without the header name, specifying the acceptable ranked compression types. +/// +_ASYNCRTIMP utility::string_t build_supported_header(header_types type, + const std::vector>& factories = + std::vector>()); +} // namespace details +} // namespace compression +} // namespace http +} // namespace web diff --git a/Release/include/cpprest/http_headers.h b/Release/include/cpprest/http_headers.h index c2e35f6079..077266a7d5 100644 --- a/Release/include/cpprest/http_headers.h +++ b/Release/include/cpprest/http_headers.h @@ -219,7 +219,7 @@ class http_headers return false; } - return bind_impl(iter->second, value) || iter->second.empty(); + return details::bind_impl(iter->second, value) || iter->second.empty(); } /// @@ -285,9 +285,14 @@ class http_headers _ASYNCRTIMP void set_date(const utility::datetime& date); private: + // Headers are stored in a map with case insensitive key. + inner_container m_headers; +}; - template - bool bind_impl(const key_type &text, _t &ref) const +namespace details +{ + template + bool bind_impl(const key_type &text, _t &ref) { utility::istringstream_t iss(text); iss.imbue(std::locale::classic()); @@ -300,20 +305,18 @@ class http_headers return true; } - bool bind_impl(const key_type &text, utf16string &ref) const + template + bool bind_impl(const key_type &text, utf16string &ref) { ref = utility::conversions::to_utf16string(text); return true; } - bool bind_impl(const key_type &text, std::string &ref) const + template + bool bind_impl(const key_type &text, std::string &ref) { ref = utility::conversions::to_utf8string(text); return true; } - - // Headers are stored in a map with case insensitive key. - inner_container m_headers; -}; - +} }} diff --git a/Release/include/cpprest/http_msg.h b/Release/include/cpprest/http_msg.h index b85e98ff42..39649ef833 100644 --- a/Release/include/cpprest/http_msg.h +++ b/Release/include/cpprest/http_msg.h @@ -26,6 +26,7 @@ #include "cpprest/asyncrt_utils.h" #include "cpprest/streams.h" #include "cpprest/containerstream.h" +#include "cpprest/http_compression.h" namespace web { @@ -338,6 +339,38 @@ class http_msg_base /// const concurrency::streams::ostream & outstream() const { return m_outStream; } + /// + /// Sets the compressor for the message body + /// + void set_compressor(std::unique_ptr compressor) + { + m_compressor = std::move(compressor); + } + + /// + /// Gets the compressor for the message body, if any + /// + std::unique_ptr &compressor() + { + return m_compressor; + } + + /// + /// Sets the collection of factory classes for decompressors for use with the message body + /// + void set_decompress_factories(const std::vector> &factories) + { + m_decompressors = factories; + } + + /// + /// Gets the collection of factory classes for decompressors to be used to decompress the message body, if any + /// + const std::vector> &decompress_factories() + { + return m_decompressors; + } + const pplx::task_completion_event & _get_data_available() const { return m_data_available; } /// @@ -345,11 +378,25 @@ class http_msg_base /// _ASYNCRTIMP void _prepare_to_receive_data(); + + /// + /// Determine the remaining input stream length + /// + /// + /// std::numeric_limits::max() if the stream's remaining length cannot be determined + /// length if the stream's remaining length (which may be 0) can be determined + /// + /// + /// This routine should only be called after a msg (request/response) has been + /// completely constructed. + /// + _ASYNCRTIMP size_t _get_stream_length(); + /// /// Determine the content length /// /// - /// size_t::max if there is content with unknown length (transfer_encoding:chunked) + /// std::numeric_limits::max() if there is content with unknown length (transfer_encoding:chunked) /// 0 if there is no content /// length if there is content with known length /// @@ -359,8 +406,27 @@ class http_msg_base /// _ASYNCRTIMP size_t _get_content_length(); + /// + /// Determine the content length, and, if necessary, manage compression in the Transfer-Encoding header + /// + /// + /// std::numeric_limits::max() if there is content with unknown length (transfer_encoding:chunked) + /// 0 if there is no content + /// length if there is content with known length + /// + /// + /// This routine is like _get_content_length, except that it adds a compression algorithm to + /// the Trasfer-Length header if compression is configured. It throws if a Transfer-Encoding + /// header exists and does not match the one it generated. + /// + _ASYNCRTIMP size_t _get_content_length_and_set_compression(); + protected: + std::unique_ptr m_compressor; + std::unique_ptr m_decompressor; + std::vector> m_decompressors; + /// /// Stream to read the message body. /// By default this is an invalid stream. The user could set the instream on @@ -386,6 +452,8 @@ class http_msg_base /// The TCE is used to signal the availability of the message body. pplx::task_completion_event m_data_available; + + size_t _get_content_length(bool honor_compression); }; /// @@ -671,7 +739,7 @@ class http_response /// A readable, open asynchronous stream. /// A string holding the MIME type of the message body. /// - /// This cannot be used in conjunction with any other means of setting the body of the request. + /// This cannot be used in conjunction with any external means of setting the body of the request. /// The stream will not be read until the message is sent. /// void set_body(const concurrency::streams::istream &stream, const utility::string_t &content_type = _XPLATSTR("application/octet-stream")) @@ -687,7 +755,7 @@ class http_response /// The size of the data to be sent in the body. /// A string holding the MIME type of the message body. /// - /// This cannot be used in conjunction with any other means of setting the body of the request. + /// This cannot be used in conjunction with any external means of setting the body of the request. /// The stream will not be read until the message is sent. /// void set_body(const concurrency::streams::istream &stream, utility::size64_t content_length, const utility::string_t &content_type = _XPLATSTR("application/octet-stream")) @@ -1149,6 +1217,94 @@ class http_request return _m_impl->set_response_stream(stream); } + /// + /// Sets a compressor that will be used to compress the body of the HTTP message as it is sent. + /// + /// A pointer to an instantiated compressor of the desired type. + /// + /// This cannot be used in conjunction with any external means of compression. The Transfer-Encoding + /// header will be managed internally, and must not be set by the client. + /// + void set_compressor(std::unique_ptr compressor) + { + return _m_impl->set_compressor(std::move(compressor)); + } + + /// + /// Sets a compressor that will be used to compress the body of the HTTP message as it is sent. + /// + /// The built-in compression algorithm to use. + /// + /// True if a built-in compressor was instantiated, otherwise false. + /// + /// + /// This cannot be used in conjunction with any external means of compression. The Transfer-Encoding + /// header will be managed internally, and must not be set by the client. + /// + bool set_compressor(utility::string_t algorithm) + { + _m_impl->set_compressor(http::compression::builtin::make_compressor(algorithm)); + return (bool)_m_impl->compressor(); + } + + /// + /// Gets the compressor to be used to compress the message body, if any. + /// + /// + /// The compressor itself. + /// + std::unique_ptr &compressor() + { + return _m_impl->compressor(); + } + + /// + /// Sets the default collection of built-in factory classes for decompressors that may be used to + /// decompress the body of the HTTP message as it is received, effectively enabling decompression. + /// + /// The collection of factory classes for allowable decompressors. The + /// supplied vector itself need not remain valid after the call returns. + /// + /// This default collection is implied if request_compressed_response() is set in the associated + /// client::http_client_config and neither overload of this method has been called. + /// + /// This cannot be used in conjunction with any external means of decompression. The TE and Accept-Encoding + /// headers must not be set by the client, as they will be managed internally as appropriate. + /// + _ASYNCRTIMP void set_decompress_factories(); + + /// + /// Sets a collection of factory classes for decompressors that may be used to decompress the + /// body of the HTTP message as it is received, effectively enabling decompression. + /// + /// + /// If set, this collection takes the place of the built-in compression providers. It may contain + /// custom factory classes and/or factory classes for built-in providers, and may be used to adjust + /// the weights of the built-in providers, which default to 500 (i.e. "q=0.500"). + /// + /// This cannot be used in conjunction with any external means of decompression. The TE and Accept-Encoding + /// headers must not be set by the client, as they will be managed internally as appropriate. + /// + void set_decompress_factories(const std::vector> &factories) + { + return _m_impl->set_decompress_factories(factories); + } + + /// + /// Gets the collection of factory classes for decompressors to be used to decompress the message body, if any. + /// + /// + /// The collection of factory classes itself. + /// + /// + /// This cannot be used in conjunction with any external means of decompression. The TE + /// header must not be set by the client, as it will be managed internally. + /// + const std::vector> &decompress_factories() const + { + return _m_impl->decompress_factories(); + } + /// /// Defines a callback function that will be invoked for every chunk of data uploaded or downloaded /// as part of the request. diff --git a/Release/src/CMakeLists.txt b/Release/src/CMakeLists.txt index 050ff71fc2..bb3065e607 100644 --- a/Release/src/CMakeLists.txt +++ b/Release/src/CMakeLists.txt @@ -21,6 +21,7 @@ set(SOURCES http/common/internal_http_helpers.h http/common/http_helpers.cpp http/common/http_msg.cpp + http/common/http_compression.cpp http/listener/http_listener.cpp http/listener/http_listener_msg.cpp http/listener/http_server_api.cpp @@ -71,10 +72,19 @@ endif() # Compression component if(CPPREST_EXCLUDE_COMPRESSION) + if(NOT CPPREST_EXCLUDE_BROTLI) + message(FATAL_ERROR "Use of Brotli requires compression to be enabled") + endif() target_compile_definitions(cpprest PRIVATE -DCPPREST_EXCLUDE_COMPRESSION=1) else() cpprest_find_zlib() target_link_libraries(cpprest PRIVATE cpprestsdk_zlib_internal) + if(CPPREST_EXCLUDE_BROTLI) + target_compile_definitions(cpprest PRIVATE -DCPPREST_EXCLUDE_BROTLI=1) + else() + cpprest_find_brotli() + target_link_libraries(cpprest PRIVATE cpprestsdk_brotli_internal) + endif() endif() # PPLX component @@ -232,6 +242,7 @@ endif() if(CPPREST_INSTALL) set(CPPREST_USES_BOOST OFF) set(CPPREST_USES_ZLIB OFF) + set(CPPREST_USES_BROTLI OFF) set(CPPREST_USES_OPENSSL OFF) set(CPPREST_TARGETS cpprest) @@ -243,6 +254,10 @@ if(CPPREST_INSTALL) list(APPEND CPPREST_TARGETS cpprestsdk_zlib_internal) set(CPPREST_USES_ZLIB ON) endif() + if(TARGET cpprestsdk_brotli_internal) + list(APPEND CPPREST_TARGETS cpprestsdk_brotli_internal) + set(CPPREST_USES_BROTLI ON) + endif() if(TARGET cpprestsdk_openssl_internal) list(APPEND CPPREST_TARGETS cpprestsdk_openssl_internal) set(CPPREST_USES_OPENSSL ON) diff --git a/Release/src/http/client/http_client.cpp b/Release/src/http/client/http_client.cpp index 2963fae3c2..89eaa4941a 100644 --- a/Release/src/http/client/http_client.cpp +++ b/Release/src/http/client/http_client.cpp @@ -96,42 +96,57 @@ void request_context::report_exception(std::exception_ptr exceptionPtr) finish(); } -bool request_context::handle_content_encoding_compression() +bool request_context::handle_compression() { - if (web::http::details::compression::stream_decompressor::is_supported() && m_http_client->client_config().request_compressed_response()) + // If the response body is compressed we will read the encoding header and create a decompressor object which will later decompress the body + try { - // If the response body is compressed we will read the encoding header and create a decompressor object which will later decompress the body - auto&& headers = m_response.headers(); - auto it_ce = headers.find(web::http::header_names::content_encoding); - if (it_ce != headers.end()) + utility::string_t encoding; + http_headers &headers = m_response.headers(); + + // Note that some headers, for example "Transfer-Encoding: chunked", may legitimately not produce a decompressor + if (m_http_client->client_config().request_compressed_response() && headers.match(web::http::header_names::content_encoding, encoding)) + { + // Note that, while Transfer-Encoding (chunked only) is valid with Content-Encoding, + // we don't need to look for it here because winhttp de-chunks for us in that case + m_decompressor = compression::details::get_decompressor_from_header(encoding, compression::details::header_types::content_encoding, m_request.decompress_factories()); + } + else if (!m_request.decompress_factories().empty() && headers.match(web::http::header_names::transfer_encoding, encoding)) { - auto alg = web::http::details::compression::stream_decompressor::to_compression_algorithm(it_ce->second); - - if (alg != web::http::details::compression::compression_algorithm::invalid) - { - m_decompressor = utility::details::make_unique(alg); - } - else - { - report_exception( - http_exception("Unsupported compression algorithm in the Content-Encoding header: " - + utility::conversions::to_utf8string(it_ce->second))); - return false; - } + m_decompressor = compression::details::get_decompressor_from_header(encoding, compression::details::header_types::transfer_encoding, m_request.decompress_factories()); } } + catch (...) + { + report_exception(std::current_exception()); + return false; + } + return true; } -utility::string_t request_context::get_accept_encoding_header() const +utility::string_t request_context::get_compression_header() const { utility::string_t headers; - // Add the header needed to request a compressed response if supported on this platform and it has been specified in the config - if (web::http::details::compression::stream_decompressor::is_supported() - && m_http_client->client_config().request_compressed_response()) + + // Add the correct header needed to request a compressed response if supported + // on this platform and it has been specified in the config and/or request + if (m_http_client->client_config().request_compressed_response()) + { + if (!m_request.decompress_factories().empty() || web::http::compression::builtin::supported()) + { + // Accept-Encoding -- request Content-Encoding from the server + headers.append(header_names::accept_encoding + U(": ")); + headers.append(compression::details::build_supported_header(compression::details::header_types::accept_encoding, m_request.decompress_factories())); + headers.append(U("\r\n")); + } + } + else if (!m_request.decompress_factories().empty()) { - headers.append(U("Accept-Encoding: ")); - headers.append(web::http::details::compression::stream_decompressor::known_algorithms()); + // TE -- request Transfer-Encoding from the server + headers.append(header_names::connection + U(": TE\r\n") + // Required by Section 4.3 of RFC-7230 + header_names::te + U(": ")); + headers.append(compression::details::build_supported_header(compression::details::header_types::te, m_request.decompress_factories())); headers.append(U("\r\n")); } diff --git a/Release/src/http/client/http_client_asio.cpp b/Release/src/http/client/http_client_asio.cpp index e3ea15c9ff..7590497420 100644 --- a/Release/src/http/client/http_client_asio.cpp +++ b/Release/src/http/client/http_client_asio.cpp @@ -850,12 +850,12 @@ class asio_context final : public request_context, public std::enable_shared_fro extra_headers.append(ctx->generate_basic_auth_header()); } - extra_headers += utility::conversions::to_utf8string(ctx->get_accept_encoding_header()); + extra_headers += utility::conversions::to_utf8string(ctx->get_compression_header()); // Check user specified transfer-encoding. std::string transferencoding; if (ctx->m_request.headers().match(header_names::transfer_encoding, transferencoding) && - transferencoding == "chunked") + boost::icontains(transferencoding, U("chunked"))) { ctx->m_needChunked = true; } @@ -1394,7 +1394,7 @@ class asio_context final : public request_context, public std::enable_shared_fro if (boost::iequals(name, header_names::transfer_encoding)) { - needChunked = boost::iequals(value, U("chunked")); + needChunked = boost::icontains(value, U("chunked")); } if (boost::iequals(name, header_names::connection)) @@ -1415,7 +1415,7 @@ class asio_context final : public request_context, public std::enable_shared_fro // TCP stream - set it size_t max. m_response.headers().match(header_names::content_length, m_content_length); - if (!this->handle_content_encoding_compression()) + if (!this->handle_compression()) { // false indicates report_exception was called return; @@ -1518,6 +1518,49 @@ class asio_context final : public request_context, public std::enable_shared_fro } } + bool decompress(const uint8_t* input, size_t input_size, std::vector& output) + { + // Need to guard against attempting to decompress when we're already finished or encountered an error! + if (input == nullptr || input_size == 0) + { + return false; + } + + size_t processed; + size_t got; + size_t inbytes = 0; + size_t outbytes = 0; + bool done = false; + + try + { + output.resize(input_size * 3); + do + { + if (inbytes) + { + output.resize(output.size() + std::max(input_size, static_cast(1024))); + } + got = m_decompressor->decompress(input + inbytes, + input_size - inbytes, + output.data() + outbytes, + output.size() - outbytes, + web::http::compression::operation_hint::has_more, + processed, + &done); + inbytes += processed; + outbytes += got; + } while (got && !done); + output.resize(outbytes); + } + catch (...) + { + return false; + } + + return true; + } + void handle_chunk(const boost::system::error_code& ec, int to_read) { if (!ec) @@ -1550,10 +1593,11 @@ class asio_context final : public request_context, public std::enable_shared_fro const auto this_request = shared_from_this(); if (m_decompressor) { - auto decompressed = m_decompressor->decompress( - boost::asio::buffer_cast(m_body_buf.data()), to_read); + std::vector decompressed; - if (m_decompressor->has_error()) + bool boo = + decompress(boost::asio::buffer_cast(m_body_buf.data()), to_read, decompressed); + if (!boo) { report_exception(std::runtime_error("Failed to decompress the response body")); return; @@ -1574,8 +1618,7 @@ class asio_context final : public request_context, public std::enable_shared_fro { // Move the decompressed buffer into a shared_ptr to keep it alive until putn_nocopy completes. // When VS 2013 support is dropped, this should be changed to a unique_ptr plus a move capture. - using web::http::details::compression::data_buffer; - auto shared_decompressed = std::make_shared(std::move(decompressed)); + auto shared_decompressed = std::make_shared>(std::move(decompressed)); writeBuffer.putn_nocopy(shared_decompressed->data(), shared_decompressed->size()) .then([this_request, to_read, shared_decompressed AND_CAPTURE_MEMBER_FUNCTION_POINTERS]( @@ -1670,10 +1713,11 @@ class asio_context final : public request_context, public std::enable_shared_fro if (m_decompressor) { - auto decompressed = - m_decompressor->decompress(boost::asio::buffer_cast(m_body_buf.data()), read_size); + std::vector decompressed; - if (m_decompressor->has_error()) + bool boo = + decompress(boost::asio::buffer_cast(m_body_buf.data()), read_size, decompressed); + if (!boo) { this_request->report_exception(std::runtime_error("Failed to decompress the response body")); return; @@ -1704,8 +1748,7 @@ class asio_context final : public request_context, public std::enable_shared_fro { // Move the decompressed buffer into a shared_ptr to keep it alive until putn_nocopy completes. // When VS 2013 support is dropped, this should be changed to a unique_ptr plus a move capture. - using web::http::details::compression::data_buffer; - auto shared_decompressed = std::make_shared(std::move(decompressed)); + auto shared_decompressed = std::make_shared>(std::move(decompressed)); writeBuffer.putn_nocopy(shared_decompressed->data(), shared_decompressed->size()) .then([this_request, read_size, shared_decompressed AND_CAPTURE_MEMBER_FUNCTION_POINTERS]( @@ -1715,7 +1758,7 @@ class asio_context final : public request_context, public std::enable_shared_fro { writtenSize = op.get(); this_request->m_downloaded += static_cast(read_size); - this_request->m_body_buf.consume(writtenSize); + this_request->m_body_buf.consume(read_size); this_request->async_read_until_buffersize( static_cast(std::min( static_cast(this_request->m_http_client->client_config().chunksize()), diff --git a/Release/src/http/client/http_client_impl.h b/Release/src/http/client/http_client_impl.h index 7b6c974a0d..067233ab63 100644 --- a/Release/src/http/client/http_client_impl.h +++ b/Release/src/http/client/http_client_impl.h @@ -72,10 +72,10 @@ class request_context /// Set m_decompressor based on the response headers, or call report_exception /// false on failure - bool handle_content_encoding_compression(); + bool handle_compression(); /// Append an Accept-Encoding header if requested by the http_client settings - utility::string_t get_accept_encoding_header() const; + utility::string_t get_compression_header() const; concurrency::streams::streambuf _get_writebuffer(); @@ -95,7 +95,7 @@ class request_context // Registration for cancellation notification if enabled. pplx::cancellation_token_registration m_cancellationRegistration; - std::unique_ptr m_decompressor; + std::unique_ptr m_decompressor; protected: diff --git a/Release/src/http/client/http_client_winhttp.cpp b/Release/src/http/client/http_client_winhttp.cpp index a1f69ab215..0a135d039e 100644 --- a/Release/src/http/client/http_client_winhttp.cpp +++ b/Release/src/http/client/http_client_winhttp.cpp @@ -182,9 +182,10 @@ class memory_holder { uint8_t* m_externalData; std::vector m_internalData; + size_t m_size; public: - memory_holder() : m_externalData(nullptr) + memory_holder() : m_externalData(nullptr), m_size(0) { } @@ -197,10 +198,11 @@ class memory_holder m_externalData = nullptr; } - inline void reassign_to(_In_opt_ uint8_t *block) + inline void reassign_to(_In_opt_ uint8_t *block, size_t length) { assert(block != nullptr); m_externalData = block; + m_size = length; } inline bool is_internally_allocated() const @@ -212,6 +214,11 @@ class memory_holder { return is_internally_allocated() ? &m_internalData[0] : m_externalData ; } + + inline size_t size() const + { + return is_internally_allocated() ? m_internalData.size() : m_size; + } }; // Possible ways a message body can be sent/received. @@ -245,7 +252,7 @@ class winhttp_request_context final : public request_context if (block == nullptr) m_body_data.allocate_space(length); else - m_body_data.reassign_to(block); + m_body_data.reassign_to(block, length); } void allocate_reply_space(_In_opt_ uint8_t *block, size_t length) @@ -253,7 +260,7 @@ class winhttp_request_context final : public request_context if (block == nullptr) m_body_data.allocate_space(length); else - m_body_data.reassign_to(block); + m_body_data.reassign_to(block, length); } bool is_externally_allocated() const @@ -286,8 +293,224 @@ class winhttp_request_context final : public request_context std::shared_ptr m_self_reference; memory_holder m_body_data; + // Compress/decompress-related processing state lives here + class compression_state + { + public: + compression_state() + : m_acquired(nullptr), m_bytes_read(0), m_bytes_processed(0), m_needs_flush(false), m_started(false), m_done(false), m_chunked(false) + { + } + + // Minimal state for on-the-fly decoding of "chunked" encoded data + class _chunk_helper + { + public: + _chunk_helper() + : m_bytes_remaining(0), m_chunk_size(true), m_chunk_delim(false), m_expect_linefeed(false), m_ignore(false), m_trailer(false) + { + } + + // Returns true if the end of chunked data has been reached, specifically whether the 0-length + // chunk and its trailing delimiter has been processed. Otherwise, offset and length bound the + // portion of buffer that represents a contiguous (and possibly partial) chunk of consumable + // data; offset+length is the total number of bytes processed from the buffer on this pass. + bool process_buffer(uint8_t *buffer, size_t buffer_size, size_t &offset, size_t &length) + { + bool done = false; + size_t n = 0; + size_t l = 0; + + while (n < buffer_size) + { + if (m_ignore) + { + if (m_expect_linefeed) + { + _ASSERTE(m_chunk_delim && m_trailer); + if (buffer[n] != '\n') + { + // The data stream does not conform to "chunked" encoding + throw http_exception(status_codes::BadRequest, "Transfer-Encoding malformed trailer"); + } + + // Look for further trailer fields or the end of the stream + m_expect_linefeed = false; + m_trailer = false; + } + else if (buffer[n] == '\r') + { + if (!m_trailer) + { + // We're at the end of the data we need to ignore + _ASSERTE(m_chunk_size || m_chunk_delim); + m_ignore = false; + m_chunk_delim = false; // this is only set if we're at the end of the message + } // else we're at the end of a trailer field + m_expect_linefeed = true; + } + else if (m_chunk_delim) + { + // We're processing (and ignoring) a trailer field + m_trailer = true; + } + } + else if (m_expect_linefeed) + { + // We've already seen a carriage return; confirm the linefeed + if (buffer[n] != '\n') + { + // The data stream does not conform to "chunked" encoding + throw http_exception(status_codes::BadRequest, "Transfer-Encoding malformed delimiter"); + } + if (m_chunk_size) + { + if (!m_bytes_remaining) + { + // We're processing the terminating "empty" chunk; there's + // no data, we just need to confirm the final chunk delimiter, + // possibly ignoring a trailer part along the way + m_ignore = true; + m_chunk_delim = true; + } // else we move on to the chunk data itself + m_chunk_size = false; + } + else + { + // Now we move on to the next chunk size + _ASSERTE(!m_bytes_remaining); + if (m_chunk_delim) + { + // We expect a chunk size next + m_chunk_size = true; + } + else + { + // We just processed the end-of-input delimiter + done = true; + } + m_chunk_delim = false; + } + m_expect_linefeed = false; + } + else if (m_chunk_delim) + { + // We're processing a post-chunk delimiter + if (buffer[n] != '\r') + { + // The data stream does not conform to "chunked" encoding + throw http_exception(status_codes::BadRequest, "Transfer-Encoding malformed chunk delimiter"); + } + + // We found the carriage return; look for the linefeed + m_expect_linefeed = true; + } + else if (m_chunk_size) + { + // We're processing an ASCII hexadecimal chunk size + if (buffer[n] >= 'a' && buffer[n] <= 'f') + { + m_bytes_remaining *= 16; + m_bytes_remaining += 10 + buffer[n] - 'a'; + } + else if (buffer[n] >= 'A' && buffer[n] <= 'F') + { + m_bytes_remaining *= 16; + m_bytes_remaining += 10 + buffer[n] - 'A'; + } + else if (buffer[n] >= '0' && buffer[n] <= '9') + { + m_bytes_remaining *= 16; + m_bytes_remaining += buffer[n] - '0'; + } + else if (buffer[n] == '\r') + { + // We've reached the end of the size, and there's no chunk extention + m_expect_linefeed = true; + } + else if (buffer[n] == ';') + { + // We've reached the end of the size, and there's a chunk extention; + // we don't support extensions, so we ignore them per RFC + m_ignore = true; + } + else + { + // The data stream does not conform to "chunked" encoding + throw http_exception(status_codes::BadRequest, "Transfer-Encoding malformed chunk size or extension"); + } + } + else + { + if (m_bytes_remaining) + { + // We're at the offset of a chunk of consumable data; let the caller process it + l = std::min(m_bytes_remaining, buffer_size-n); + m_bytes_remaining -= l; + if (!m_bytes_remaining) + { + // We're moving on to the post-chunk delimiter + m_chunk_delim = true; + } + } + else + { + // We've previously processed the terminating empty chunk and its + // trailing delimiter; skip the entire buffer, and inform the caller + n = buffer_size; + done = true; + } + + // Let the caller process the result + break; + } + + // Move on to the next byte + n++; + } + + offset = n; + length = l; + return buffer_size ? done : (!m_bytes_remaining && !m_chunk_size && !m_chunk_delim); + } + + private: + size_t m_bytes_remaining; // the number of bytes remaining in the chunk we're currently processing + bool m_chunk_size; // if true, we're processing a chunk size or its trailing delimiter + bool m_chunk_delim; // if true, we're processing a delimiter between a chunk and the next chunk's size + bool m_expect_linefeed; // if true, we're processing a delimiter, and we've already seen its carriage return + bool m_ignore; // if true, we're processing a chunk extension or trailer, which we don't support + bool m_trailer; // if true, we're processing (and ignoring) a trailer field; m_ignore is also true + }; + + std::vector m_buffer; // we read data from the stream into this before compressing + uint8_t *m_acquired; // we use this in place of m_buffer if the stream has directly-accessible data available + size_t m_bytes_read; // we most recently read this many bytes, which may be less than m_buffer.size() + size_t m_bytes_processed; // we've compressed this many bytes of m_bytes_read so far + bool m_needs_flush; // we've read and compressed all bytes, but the compressor still has compressed bytes to give us + bool m_started; // we've sent at least some number of bytes to m_decompressor + bool m_done; // we've read, compressed, and consumed all bytes + bool m_chunked; // if true, we need to decode and decompress a transfer-encoded message + size_t m_chunk_bytes; // un-decompressed bytes remaining in the most-recently-obtained data from m_chunk + std::unique_ptr<_chunk_helper> m_chunk; + } m_compression_state; + void cleanup() { + if (m_compression_state.m_acquired != nullptr) + { + // We may still hold a piece of the buffer if we encountered an exception; release it here + if (m_decompressor) + { + _get_writebuffer().commit(0); + } + else + { + _get_readbuffer().release(m_compression_state.m_acquired, m_compression_state.m_bytes_processed); + } + m_compression_state.m_acquired = nullptr; + } + if(m_request_handle != nullptr) { auto tmp_handle = m_request_handle; @@ -898,7 +1121,16 @@ class winhttp_client final : public _http_client_communicator return; } - const size_t content_length = msg._get_impl()->_get_content_length(); + size_t content_length; + try + { + content_length = msg._get_impl()->_get_content_length_and_set_compression(); + } + catch (...) + { + request->report_exception(std::current_exception()); + return; + } if (content_length > 0) { if ( msg.method() == http::methods::GET || msg.method() == http::methods::HEAD ) @@ -910,9 +1142,11 @@ class winhttp_client final : public _http_client_communicator // There is a request body that needs to be transferred. if (content_length == std::numeric_limits::max()) { - // The content length is unknown and the application set a stream. This is an - // indication that we will use transfer encoding chunked. + // The content length is not set and the application set a stream. This is an + // indication that we will use transfer encoding chunked. We still want to + // know that stream's effective length if possible for memory efficiency. winhttp_context->m_bodyType = transfer_encoding_chunked; + winhttp_context->m_remaining_to_write = msg._get_impl()->_get_stream_length(); } else { @@ -923,7 +1157,11 @@ class winhttp_client final : public _http_client_communicator } utility::string_t flattened_headers = web::http::details::flatten_http_headers(headers); - flattened_headers += winhttp_context->get_accept_encoding_header(); + if (winhttp_context->m_request.method() == http::methods::GET) + { + // Prepare to request a compressed response from the server if necessary. + flattened_headers += winhttp_context->get_compression_header(); + } // Add headers. if(!flattened_headers.empty()) @@ -1062,19 +1300,36 @@ class winhttp_client final : public _http_client_communicator else { // If bytes read is less than the chunk size this request is done. + // Is it really, though? The WinHttpReadData docs suggest that less can be returned regardless... const size_t chunkSize = pContext->m_http_client->client_config().chunksize(); - if (bytesRead < chunkSize && !firstRead) + std::unique_ptr &decompressor = pContext->m_decompressor; + if (!decompressor && bytesRead < chunkSize && !firstRead) { pContext->complete_request(pContext->m_downloaded); } else { - auto writebuf = pContext->_get_writebuffer(); - pContext->allocate_reply_space(writebuf.alloc(chunkSize), chunkSize); + uint8_t *buffer; + + if (decompressor) + { + // m_buffer holds the compressed data; we'll decompress into the caller's buffer later + if (pContext->m_compression_state.m_buffer.capacity() < chunkSize) + { + pContext->m_compression_state.m_buffer.reserve(chunkSize); + } + buffer = pContext->m_compression_state.m_buffer.data(); + } + else + { + auto writebuf = pContext->_get_writebuffer(); + pContext->allocate_reply_space(writebuf.alloc(chunkSize), chunkSize); + buffer = pContext->m_body_data.get(); + } if (!WinHttpReadData( pContext->m_request_handle, - pContext->m_body_data.get(), + buffer, static_cast(chunkSize), nullptr)) { @@ -1087,11 +1342,38 @@ class winhttp_client final : public _http_client_communicator static void _transfer_encoding_chunked_write_data(_In_ winhttp_request_context * p_request_context) { - const size_t chunk_size = p_request_context->m_http_client->client_config().chunksize(); + size_t chunk_size; + std::unique_ptr &compressor = p_request_context->m_request.compressor(); - p_request_context->allocate_request_space(nullptr, chunk_size+http::details::chunked_encoding::additional_encoding_space); + // Set the chunk size up front; we need it before the lambda functions come into scope + if (compressor) + { + // We could allocate less than a chunk for the compressed data here, though that + // would result in more trips through this path for not-so-compressible data... + if (p_request_context->m_body_data.size() > http::details::chunked_encoding::additional_encoding_space) + { + // If we've previously allocated space for the compressed data, don't reduce it + chunk_size = p_request_context->m_body_data.size() - http::details::chunked_encoding::additional_encoding_space; + } + else if (p_request_context->m_remaining_to_write != std::numeric_limits::max()) + { + // Choose a semi-intelligent size based on how much total data is left to compress + chunk_size = std::min(static_cast(p_request_context->m_remaining_to_write)+128, p_request_context->m_http_client->client_config().chunksize()); + } + else + { + // Just base our allocation on the chunk size, since we don't have any other data available + chunk_size = p_request_context->m_http_client->client_config().chunksize(); + } + } + else + { + // We're not compressing; use the smaller of the remaining data (if known) and the configured (or default) chunk size + chunk_size = std::min(static_cast(p_request_context->m_remaining_to_write), p_request_context->m_http_client->client_config().chunksize()); + } + p_request_context->allocate_request_space(nullptr, chunk_size + http::details::chunked_encoding::additional_encoding_space); - auto after_read = [p_request_context, chunk_size](pplx::task op) + auto after_read = [p_request_context, chunk_size, &compressor](pplx::task op) { size_t bytes_read; try @@ -1102,7 +1384,10 @@ class winhttp_client final : public _http_client_communicator { // We have raw memory here writing to a memory stream so it is safe to wait // since it will always be non-blocking. - p_request_context->m_readBufferCopy->putn_nocopy(&p_request_context->m_body_data.get()[http::details::chunked_encoding::data_offset], bytes_read).wait(); + if (!compressor) + { + p_request_context->m_readBufferCopy->putn_nocopy(&p_request_context->m_body_data.get()[http::details::chunked_encoding::data_offset], bytes_read).wait(); + } } } catch (...) @@ -1115,7 +1400,21 @@ class winhttp_client final : public _http_client_communicator size_t offset = http::details::chunked_encoding::add_chunked_delimiters(p_request_context->m_body_data.get(), chunk_size + http::details::chunked_encoding::additional_encoding_space, bytes_read); + if (!compressor && p_request_context->m_remaining_to_write != std::numeric_limits::max()) + { + if (bytes_read == 0 && p_request_context->m_remaining_to_write) + { + // The stream ended earlier than we detected it should + http_exception ex(U("Unexpected end of request body stream encountered before expected length met.")); + p_request_context->report_exception(ex); + return; + } + p_request_context->m_remaining_to_write -= bytes_read; + } + // Stop writing chunks if we reached the end of the stream. + // Note that we could detect end-of-stream based on !m_remaining_to_write, and insert + // the last (0) chunk if we have enough extra space... though we currently don't. if (bytes_read == 0) { p_request_context->m_bodyType = no_body; @@ -1140,7 +1439,171 @@ class winhttp_client final : public _http_client_communicator } }; - p_request_context->_get_readbuffer().getn(&p_request_context->m_body_data.get()[http::details::chunked_encoding::data_offset], chunk_size).then(after_read); + if (compressor) + { + auto do_compress = [p_request_context, chunk_size, &compressor](pplx::task op) -> pplx::task + { + size_t bytes_read; + + try + { + bytes_read = op.get(); + } + catch (...) + { + return pplx::task_from_exception(std::current_exception()); + } + _ASSERTE(bytes_read >= 0); + + uint8_t *buffer = p_request_context->m_compression_state.m_acquired; + if (buffer == nullptr) + { + buffer = p_request_context->m_compression_state.m_buffer.data(); + } + + web::http::compression::operation_hint hint = web::http::compression::operation_hint::has_more; + + if (bytes_read) + { + // An actual read always resets compression state for the next chunk + _ASSERTE(p_request_context->m_compression_state.m_bytes_processed == p_request_context->m_compression_state.m_bytes_read); + _ASSERTE(!p_request_context->m_compression_state.m_needs_flush); + p_request_context->m_compression_state.m_bytes_read = bytes_read; + p_request_context->m_compression_state.m_bytes_processed = 0; + if (p_request_context->m_readBufferCopy) + { + // If we've been asked to keep a copy of the raw data for restarts, do so here, pre-compression + p_request_context->m_readBufferCopy->putn_nocopy(buffer, bytes_read).wait(); + } + if (p_request_context->m_remaining_to_write == bytes_read) + { + // We've read to the end of the stream; finalize here if possible. We'll + // decrement the remaining count as we actually process the read buffer. + hint = web::http::compression::operation_hint::is_last; + } + } + else if (p_request_context->m_compression_state.m_needs_flush) + { + // All input has been consumed, but we still need to collect additional compressed output; + // this is done (in theory it can be multiple times) as a finalizing operation + hint = web::http::compression::operation_hint::is_last; + } + else if (p_request_context->m_compression_state.m_bytes_processed == p_request_context->m_compression_state.m_bytes_read) + { + if (p_request_context->m_remaining_to_write && p_request_context->m_remaining_to_write != std::numeric_limits::max()) + { + // The stream ended earlier than we detected it should + return pplx::task_from_exception(http_exception(U("Unexpected end of request body stream encountered before expected length met."))); + } + + // We think we're done; inform the compression library so it can finalize and/or give us any pending compressed bytes. + // Note that we may end up here multiple times if m_needs_flush is set, until all compressed bytes are drained. + hint = web::http::compression::operation_hint::is_last; + } + // else we're still compressing bytes from the previous read + + _ASSERTE(p_request_context->m_compression_state.m_bytes_processed <= p_request_context->m_compression_state.m_bytes_read); + + uint8_t *in = buffer + p_request_context->m_compression_state.m_bytes_processed; + size_t inbytes = p_request_context->m_compression_state.m_bytes_read - p_request_context->m_compression_state.m_bytes_processed; + return compressor->compress(in, inbytes, &p_request_context->m_body_data.get()[http::details::chunked_encoding::data_offset], chunk_size, hint) + .then([p_request_context, bytes_read, hint, chunk_size](pplx::task op) -> pplx::task + { + http::compression::operation_result r; + + try + { + r = op.get(); + } + catch (...) + { + return pplx::task_from_exception(std::current_exception()); + } + + if (hint == web::http::compression::operation_hint::is_last) + { + // We're done reading all chunks, but the compressor may still have compressed bytes to drain from previous reads + _ASSERTE(r.done || r.output_bytes_produced == chunk_size); + p_request_context->m_compression_state.m_needs_flush = !r.done; + p_request_context->m_compression_state.m_done = r.done; + } + + // Update the number of bytes compressed in this read chunk; if it's been fully compressed, + // we'll reset m_bytes_processed and m_bytes_read after reading the next chunk + p_request_context->m_compression_state.m_bytes_processed += r.input_bytes_processed; + _ASSERTE(p_request_context->m_compression_state.m_bytes_processed <= p_request_context->m_compression_state.m_bytes_read); + if (p_request_context->m_remaining_to_write != std::numeric_limits::max()) + { + _ASSERTE(p_request_context->m_remaining_to_write >= r.input_bytes_processed); + p_request_context->m_remaining_to_write -= r.input_bytes_processed; + } + + if (p_request_context->m_compression_state.m_acquired != nullptr && p_request_context->m_compression_state.m_bytes_processed == p_request_context->m_compression_state.m_bytes_read) + { + // Release the acquired buffer back to the streambuf at the earliest possible point + p_request_context->_get_readbuffer().release(p_request_context->m_compression_state.m_acquired, p_request_context->m_compression_state.m_bytes_processed); + p_request_context->m_compression_state.m_acquired = nullptr; + } + + return pplx::task_from_result(r.output_bytes_produced); + }); + }; + + if (p_request_context->m_compression_state.m_bytes_processed < p_request_context->m_compression_state.m_bytes_read || p_request_context->m_compression_state.m_needs_flush) + { + // We're still working on data from a previous read; continue compression without reading new data + do_compress(pplx::task_from_result(0)).then(after_read); + } + else if (p_request_context->m_compression_state.m_done) + { + // We just need to send the last (zero-length) chunk; there's no sense in going through the compression path + after_read(pplx::task_from_result(0)); + } + else + { + size_t length; + + // We need to read from the input stream, then compress before sending + if (p_request_context->_get_readbuffer().acquire(p_request_context->m_compression_state.m_acquired, length)) + { + if (length == 0) + { + if (p_request_context->_get_readbuffer().exception()) + { + p_request_context->report_exception(p_request_context->_get_readbuffer().exception()); + return; + } + else if (p_request_context->m_remaining_to_write && p_request_context->m_remaining_to_write != std::numeric_limits::max()) + { + // Unexpected end-of-stream. + p_request_context->report_error(GetLastError(), _XPLATSTR("Outgoing HTTP body stream ended early.")); + return; + } + } + else if (length > p_request_context->m_remaining_to_write) + { + // The stream grew, but we won't + length = p_request_context->m_remaining_to_write; + } + + do_compress(pplx::task_from_result(length)).then(after_read); + } + else + { + length = std::min(static_cast(p_request_context->m_remaining_to_write), p_request_context->m_http_client->client_config().chunksize()); + if (p_request_context->m_compression_state.m_buffer.capacity() < length) + { + p_request_context->m_compression_state.m_buffer.reserve(length); + } + p_request_context->_get_readbuffer().getn(p_request_context->m_compression_state.m_buffer.data(), length).then(do_compress).then(after_read); + } + } + } + else + { + // We're not compressing; just read and chunk + p_request_context->_get_readbuffer().getn(&p_request_context->m_body_data.get()[http::details::chunked_encoding::data_offset], chunk_size).then(after_read); + } } static void _multiple_segment_write_data(_In_ winhttp_request_context * p_request_context) @@ -1272,7 +1735,21 @@ class winhttp_client final : public _http_client_communicator { return false; } + + // We successfully seeked back; now reset the compression state, if any, to match + if (p_request_context->m_request.compressor()) + { + try + { + p_request_context->m_request.compressor()->reset(); + } + catch (...) + { + return false; + } + } } + p_request_context->m_compression_state = winhttp_request_context::compression_state(); // If we got ERROR_WINHTTP_RESEND_REQUEST, the response header is not available, // we cannot call WinHttpQueryAuthSchemes and WinHttpSetCredentials. @@ -1346,7 +1823,16 @@ class winhttp_client final : public _http_client_communicator } // Reset the request body type since it might have already started sending. - const size_t content_length = request._get_impl()->_get_content_length(); + size_t content_length; + try + { + content_length = request._get_impl()->_get_content_length_and_set_compression(); + } + catch (...) + { + return false; + } + if (content_length > 0) { // There is a request body that needs to be transferred. @@ -1355,6 +1841,7 @@ class winhttp_client final : public _http_client_communicator // The content length is unknown and the application set a stream. This is an // indication that we will need to chunk the data. p_request_context->m_bodyType = transfer_encoding_chunked; + p_request_context->m_remaining_to_write = request._get_impl()->_get_stream_length(); } else { @@ -1551,11 +2038,17 @@ class winhttp_client final : public _http_client_communicator } } - if (!p_request_context->handle_content_encoding_compression()) + // Check whether the request is compressed, and if so, whether we're handling it. + if (!p_request_context->handle_compression()) { // false indicates report_exception was called return; } + if (p_request_context->m_decompressor && !p_request_context->m_http_client->client_config().request_compressed_response()) + { + p_request_context->m_compression_state.m_chunk = std::make_unique(); + p_request_context->m_compression_state.m_chunked = true; + } // Signal that the headers are available. p_request_context->complete_headers(); @@ -1582,25 +2075,30 @@ class winhttp_client final : public _http_client_communicator { // Status information contains pointer to DWORD containing number of bytes available. const DWORD num_bytes = *(PDWORD)statusInfo; + uint8_t *buffer; - if(num_bytes > 0) + if (num_bytes > 0) { if (p_request_context->m_decompressor) { - // Decompression is too slow to reliably do on this callback. Therefore we need to store it now in order to decompress it at a later stage in the flow. - // However, we want to eventually use the writebuf to store the decompressed body. Therefore we'll store the compressed body as an internal allocation in the request_context - p_request_context->allocate_reply_space(nullptr, num_bytes); + // Allocate space for the compressed data; we'll decompress it into the caller stream once it's been filled in + if (p_request_context->m_compression_state.m_buffer.capacity() < num_bytes) + { + p_request_context->m_compression_state.m_buffer.reserve(num_bytes); + } + buffer = p_request_context->m_compression_state.m_buffer.data(); } else { auto writebuf = p_request_context->_get_writebuffer(); p_request_context->allocate_reply_space(writebuf.alloc(num_bytes), num_bytes); + buffer = p_request_context->m_body_data.get(); } - // Read in body all at once. + // Read in available body data all at once. if(!WinHttpReadData( hRequestHandle, - p_request_context->m_body_data.get(), + buffer, num_bytes, nullptr)) { @@ -1610,6 +2108,21 @@ class winhttp_client final : public _http_client_communicator } else { + if (p_request_context->m_decompressor) + { + if (p_request_context->m_compression_state.m_chunked) + { + // We haven't seen the 0-length chunk and/or trailing delimiter that indicate the end of chunked input + p_request_context->report_exception(http_exception("Chunked response stream ended unexpectedly")); + return; + } + if (p_request_context->m_compression_state.m_started && !p_request_context->m_compression_state.m_done) + { + p_request_context->report_exception(http_exception("Received incomplete compressed stream")); + return; + } + } + // No more data available, complete the request. auto progress = p_request_context->m_request._get_impl()->_progress_handler(); if (progress) @@ -1647,68 +2160,240 @@ class winhttp_client final : public _http_client_communicator // If no bytes have been read, then this is the end of the response. if (bytesRead == 0) { + if (p_request_context->m_decompressor) + { + if (p_request_context->m_compression_state.m_chunked) + { + // We haven't seen the 0-length chunk and/or trailing delimiter that indicate the end of chunked input + p_request_context->report_exception(http_exception("Chunked response stream ended unexpectedly")); + return; + } + if (p_request_context->m_compression_state.m_started && !p_request_context->m_compression_state.m_done) + { + p_request_context->report_exception(http_exception("Received incomplete compressed stream")); + return; + } + } p_request_context->complete_request(p_request_context->m_downloaded); return; } auto writebuf = p_request_context->_get_writebuffer(); - // If we have compressed data it is stored in the local allocation of the p_request_context. We will store the decompressed buffer in the external allocation of the p_request_context. if (p_request_context->m_decompressor) { - web::http::details::compression::data_buffer decompressed = p_request_context->m_decompressor->decompress(p_request_context->m_body_data.get(), bytesRead); - - if (p_request_context->m_decompressor->has_error()) - { - p_request_context->report_exception(std::runtime_error("Failed to decompress the response body")); - return; - } + size_t chunk_size = std::max(static_cast(bytesRead), p_request_context->m_http_client->client_config().chunksize()); + p_request_context->m_compression_state.m_bytes_read = static_cast(bytesRead); + p_request_context->m_compression_state.m_chunk_bytes = 0; - // We've decompressed this chunk of the body, need to now store it in the writebuffer. - auto decompressed_size = decompressed.size(); + // Note, some servers seem to send a first chunk of body data that decompresses to nothing, but + // initializes the decompression state; this produces no decompressed output. Subsequent chunks + // will then begin emitting decompressed body data. - if (decompressed_size > 0) + // Oddly enough, WinHttp doesn't de-chunk for us if "chunked" isn't the only + // encoding, so we need to do so on the fly as we process the received data + auto process_buffer = [chunk_size](winhttp_request_context *c, size_t bytes_produced, bool outer) -> bool { - auto p = writebuf.alloc(decompressed_size); - p_request_context->allocate_reply_space(p, decompressed_size); - std::memcpy(p_request_context->m_body_data.get(), &decompressed[0], decompressed_size); - } - // Note, some servers seem to send a first chunk of body data that decompresses to nothing but initializes the zlib decryption state. This produces no decompressed output. - // Subsequent chunks will then begin emmiting decompressed body data. + if (!c->m_compression_state.m_chunk_bytes) + { + if (c->m_compression_state.m_chunked) + { + size_t offset; + bool done; + + // Process the next portion of this piece of the transfer-encoded message + done = c->m_compression_state.m_chunk->process_buffer(c->m_compression_state.m_buffer.data()+c->m_compression_state.m_bytes_processed, c->m_compression_state.m_bytes_read-c->m_compression_state.m_bytes_processed, offset, c->m_compression_state.m_chunk_bytes); + + // Skip chunk-related metadata; it isn't relevant to decompression + _ASSERTE(c->m_compression_state.m_bytes_processed+offset <= c->m_compression_state.m_bytes_read); + c->m_compression_state.m_bytes_processed += offset; + + if (!c->m_compression_state.m_chunk_bytes) + { + if (done) + { + // We've processed/validated all bytes in this transfer-encoded message. + // Note that we currently ignore "extra" trailing bytes, i.e. c->m_compression_state.m_bytes_processed < c->m_compression_state.m_bytes_read + if (c->m_compression_state.m_done) + { + c->complete_request(c->m_downloaded); + return false; + } + else if (!outer && bytes_produced != chunk_size) + { + throw http_exception("Transfer ended before decompression completed"); + } + } + else if (!outer && bytes_produced != chunk_size) + { + // There should be more data to receive; look for it + c->m_compression_state.m_bytes_processed = 0; + read_next_response_chunk(c, static_cast(c->m_compression_state.m_bytes_read)); + return false; + } + } + } + else + { + _ASSERTE(!c->m_compression_state.m_bytes_processed || c->m_compression_state.m_bytes_processed == c->m_compression_state.m_bytes_read); + if (c->m_compression_state.m_done) + { + // Decompression is done; complete the request + c->complete_request(c->m_downloaded); + return false; + } + else if (c->m_compression_state.m_bytes_processed != c->m_compression_state.m_bytes_read) + { + // We still have more data to process in the current buffer + c->m_compression_state.m_chunk_bytes = c->m_compression_state.m_bytes_read - c->m_compression_state.m_bytes_processed; + } + else if (!outer && bytes_produced != chunk_size) + { + // There should be more data to receive; look for it + c->m_compression_state.m_bytes_processed = 0; + read_next_response_chunk(c, static_cast(c->m_compression_state.m_bytes_read)); + return false; + } + // Otherwise, we've processed all bytes in the input buffer, but there's a good chance that + // there are still decompressed bytes to emit; we'll do so before reading the next chunk + } + } - bytesRead = static_cast(decompressed_size); - } + // We're still processing the current message chunk + return true; + }; - // If the data was allocated directly from the buffer then commit, otherwise we still - // need to write to the response stream buffer. - if (p_request_context->is_externally_allocated()) - { - writebuf.commit(bytesRead); - read_next_response_chunk(p_request_context.get(), bytesRead); - } - else - { - writebuf.putn_nocopy(p_request_context->m_body_data.get(), bytesRead).then( - [hRequestHandle, p_request_context, bytesRead] (pplx::task op) + Concurrency::details::_do_while([p_request_context, chunk_size, process_buffer]() -> pplx::task { - size_t written = 0; - try { written = op.get(); } + uint8_t *buffer; + + try + { + if (!process_buffer(p_request_context.get(), 0, true)) + { + // The chunked request has been completely processed (or contains no data in the first place) + return pplx::task_from_result(false); + } + } catch (...) { - p_request_context->report_exception(std::current_exception()); - return; + // The outer do-while requires an explicit task return to activate the then() clause + return pplx::task_from_exception(std::current_exception()); } - // If we couldn't write everything, it's time to exit. - if (written != bytesRead) + // If it's possible to know how much post-compression data we're expecting (for instance if we can discern how + // much total data the ostream can support, we could allocate (or at least attempt to acquire) based on that + p_request_context->m_compression_state.m_acquired = p_request_context->_get_writebuffer().alloc(chunk_size); + if (p_request_context->m_compression_state.m_acquired) { - p_request_context->report_exception(std::runtime_error("response stream unexpectedly failed to write the requested number of bytes")); - return; + buffer = p_request_context->m_compression_state.m_acquired; + } + else + { + // The streambuf couldn't accommodate our request; we'll use m_body_data's + // internal vector as temporary storage, then putn() to the caller's stream + p_request_context->allocate_reply_space(nullptr, chunk_size); + buffer = p_request_context->m_body_data.get(); } - read_next_response_chunk(p_request_context.get(), bytesRead); + uint8_t *in = p_request_context->m_compression_state.m_buffer.data() + p_request_context->m_compression_state.m_bytes_processed; + size_t inbytes = p_request_context->m_compression_state.m_chunk_bytes; + if (inbytes) + { + p_request_context->m_compression_state.m_started = true; + } + return p_request_context->m_decompressor->decompress(in, inbytes, buffer, chunk_size, web::http::compression::operation_hint::has_more).then( + [p_request_context, buffer, chunk_size, process_buffer] (pplx::task op) + { + auto r = op.get(); + auto keep_going = [&r, process_buffer](winhttp_request_context *c) -> pplx::task + { + _ASSERTE(r.input_bytes_processed <= c->m_compression_state.m_chunk_bytes); + c->m_compression_state.m_chunk_bytes -= r.input_bytes_processed; + c->m_compression_state.m_bytes_processed += r.input_bytes_processed; + c->m_compression_state.m_done = r.done; + + try + { + // See if we still have more work to do for this section and/or for the response in general + return pplx::task_from_result(process_buffer(c, r.output_bytes_produced, false)); + } + catch (...) + { + return pplx::task_from_exception(std::current_exception()); + } + }; + + _ASSERTE(p_request_context->m_compression_state.m_bytes_processed+r.input_bytes_processed <= p_request_context->m_compression_state.m_bytes_read); + + if (p_request_context->m_compression_state.m_acquired != nullptr) + { + // We decompressed directly into the output stream + p_request_context->m_compression_state.m_acquired = nullptr; + p_request_context->_get_writebuffer().commit(r.output_bytes_produced); + return keep_going(p_request_context.get()); + } + + // We decompressed into our own buffer; let the stream copy the data + return p_request_context->_get_writebuffer().putn_nocopy(buffer, r.output_bytes_produced).then([p_request_context, r, keep_going](pplx::task op) { + if (op.get() != r.output_bytes_produced) + { + return pplx::task_from_exception(std::runtime_error("Response stream unexpectedly failed to write the requested number of bytes")); + } + return keep_going(p_request_context.get()); + }); + }); + }).then([p_request_context](pplx::task op) + { + try + { + bool ignored = op.get(); + } + catch (...) + { + // We're only here to pick up any exception that may have been thrown, and to clean up if needed + if (p_request_context->m_compression_state.m_acquired) + { + p_request_context->_get_writebuffer().commit(0); + p_request_context->m_compression_state.m_acquired = nullptr; + } + p_request_context->report_exception(std::current_exception()); + } }); } + else + { + // If the data was allocated directly from the buffer then commit, otherwise we still + // need to write to the response stream buffer. + if (p_request_context->is_externally_allocated()) + { + writebuf.commit(bytesRead); + read_next_response_chunk(p_request_context.get(), bytesRead); + } + else + { + writebuf.putn_nocopy(p_request_context->m_body_data.get(), bytesRead).then( + [hRequestHandle, p_request_context, bytesRead] (pplx::task op) + { + size_t written = 0; + try { written = op.get(); } + catch (...) + { + p_request_context->report_exception(std::current_exception()); + return; + } + + // If we couldn't write everything, it's time to exit. + if (written != bytesRead) + { + p_request_context->report_exception(std::runtime_error("response stream unexpectedly failed to write the requested number of bytes")); + return; + } + + read_next_response_chunk(p_request_context.get(), bytesRead); + }); + } + } return; } } diff --git a/Release/src/http/common/http_compression.cpp b/Release/src/http/common/http_compression.cpp new file mode 100644 index 0000000000..e7e7f9a927 --- /dev/null +++ b/Release/src/http/common/http_compression.cpp @@ -0,0 +1,1117 @@ +/*** + * Copyright (C) Microsoft. All rights reserved. + * Licensed under the MIT license. See LICENSE.txt file in the project root for full license information. + * + * =+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * + * HTTP Library: Compression and decompression interfaces + * + * For the latest on this and related APIs, please see: https://github.com/Microsoft/cpprestsdk + * + * =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + ****/ + +#include "stdafx.h" + +// CPPREST_EXCLUDE_COMPRESSION is set if we're on a platform that supports compression but we want to explicitly disable +// it. CPPREST_EXCLUDE_BROTLI is set if we want to explicitly disable Brotli compression support. +// CPPREST_EXCLUDE_WEBSOCKETS is a flag that now essentially means "no external dependencies". TODO: Rename + +#if __APPLE__ +#include "TargetConditionals.h" +#if defined(TARGET_OS_MAC) +#if !defined(CPPREST_EXCLUDE_COMPRESSION) +#define CPPREST_HTTP_COMPRESSION +#endif // !defined(CPPREST_EXCLUDE_COMPRESSION) +#endif // defined(TARGET_OS_MAC) +#elif defined(_WIN32) && (!defined(WINAPI_FAMILY) || WINAPI_PARTITION_DESKTOP) +#if !defined(CPPREST_EXCLUDE_WEBSOCKETS) && !defined(CPPREST_EXCLUDE_COMPRESSION) +#define CPPREST_HTTP_COMPRESSION +#endif // !defined(CPPREST_EXCLUDE_WEBSOCKETS) && !defined(CPPREST_EXCLUDE_COMPRESSION) +#endif + +#if defined(CPPREST_HTTP_COMPRESSION) +#include +#if !defined(CPPREST_EXCLUDE_BROTLI) +#define CPPREST_BROTLI_COMPRESSION +#endif // CPPREST_EXCLUDE_BROTLI +#if defined(CPPREST_BROTLI_COMPRESSION) +#include +#include +#endif // CPPREST_BROTLI_COMPRESSION +#endif + +namespace web +{ +namespace http +{ +namespace compression +{ +namespace builtin +{ +#if defined(CPPREST_HTTP_COMPRESSION) +// A shared base class for the gzip and deflate compressors +class zlib_compressor_base : public compress_provider +{ +public: + static const utility::string_t GZIP; + static const utility::string_t DEFLATE; + + zlib_compressor_base(int windowBits, + int compressionLevel = Z_DEFAULT_COMPRESSION, + int method = Z_DEFLATED, + int strategy = Z_DEFAULT_STRATEGY, + int memLevel = MAX_MEM_LEVEL) + : m_algorithm(windowBits >= 16 ? GZIP : DEFLATE) + { + m_state = deflateInit2(&m_stream, compressionLevel, method, windowBits, memLevel, strategy); + } + + const utility::string_t& algorithm() const { return m_algorithm; } + + size_t compress(const uint8_t* input, + size_t input_size, + uint8_t* output, + size_t output_size, + operation_hint hint, + size_t& input_bytes_processed, + bool* done) + { + if (m_state == Z_STREAM_END || (hint != operation_hint::is_last && !input_size)) + { + input_bytes_processed = 0; + if (done) + { + *done = (m_state == Z_STREAM_END); + } + return 0; + } + + if (m_state != Z_OK && m_state != Z_BUF_ERROR && m_state != Z_STREAM_ERROR) + { + std::stringstream ss; + ss << "Prior unrecoverable compression stream error " << m_state; + throw std::runtime_error(std::move(ss.str())); + } + + if (input_size > std::numeric_limits::max() || output_size > std::numeric_limits::max()) + { + throw std::runtime_error("Compression input or output size out of range"); + } + + m_stream.next_in = const_cast(input); + m_stream.avail_in = static_cast(input_size); + m_stream.next_out = const_cast(output); + m_stream.avail_out = static_cast(output_size); + + m_state = deflate(&m_stream, (hint == operation_hint::is_last) ? Z_FINISH : Z_PARTIAL_FLUSH); + if (m_state != Z_OK && m_state != Z_STREAM_ERROR && + !(hint == operation_hint::is_last && (m_state == Z_STREAM_END || m_state == Z_BUF_ERROR))) + { + std::stringstream ss; + ss << "Unrecoverable compression stream error " << m_state; + throw std::runtime_error(std::move(ss.str())); + } + + input_bytes_processed = input_size - m_stream.avail_in; + if (done) + { + *done = (m_state == Z_STREAM_END); + } + return output_size - m_stream.avail_out; + } + + pplx::task compress( + const uint8_t* input, size_t input_size, uint8_t* output, size_t output_size, operation_hint hint) + { + operation_result r; + + try + { + r.output_bytes_produced = + compress(input, input_size, output, output_size, hint, r.input_bytes_processed, &r.done); + } + catch (...) + { + pplx::task_completion_event ev; + ev.set_exception(std::current_exception()); + return pplx::create_task(ev); + } + + return pplx::task_from_result(r); + } + + void reset() + { + m_state = deflateReset(&m_stream); + if (m_state != Z_OK) + { + std::stringstream ss; + ss << "Failed to reset zlib compressor " << m_state; + throw std::runtime_error(std::move(ss.str())); + } + } + + ~zlib_compressor_base() { (void)deflateEnd(&m_stream); } + +private: + int m_state{Z_BUF_ERROR}; + z_stream m_stream{0}; + const utility::string_t& m_algorithm; +}; + +const utility::string_t zlib_compressor_base::GZIP(algorithm::GZIP); +const utility::string_t zlib_compressor_base::DEFLATE(algorithm::DEFLATE); + +// A shared base class for the gzip and deflate decompressors +class zlib_decompressor_base : public decompress_provider +{ +public: + zlib_decompressor_base(int windowBits) + : m_algorithm(windowBits >= 16 ? zlib_compressor_base::GZIP : zlib_compressor_base::DEFLATE) + { + m_state = inflateInit2(&m_stream, windowBits); + } + + const utility::string_t& algorithm() const { return m_algorithm; } + + size_t decompress(const uint8_t* input, + size_t input_size, + uint8_t* output, + size_t output_size, + operation_hint hint, + size_t& input_bytes_processed, + bool* done) + { + if (m_state == Z_STREAM_END || !input_size) + { + input_bytes_processed = 0; + if (done) + { + *done = (m_state == Z_STREAM_END); + } + return 0; + } + + if (m_state != Z_OK && m_state != Z_BUF_ERROR && m_state != Z_STREAM_ERROR) + { + std::stringstream ss; + ss << "Prior unrecoverable decompression stream error " << m_state; + throw std::runtime_error(std::move(ss.str())); + } + + if (input_size > std::numeric_limits::max() || output_size > std::numeric_limits::max()) + { + throw std::runtime_error("Compression input or output size out of range"); + } + + m_stream.next_in = const_cast(input); + m_stream.avail_in = static_cast(input_size); + m_stream.next_out = const_cast(output); + m_stream.avail_out = static_cast(output_size); + + m_state = inflate(&m_stream, (hint == operation_hint::is_last) ? Z_FINISH : Z_PARTIAL_FLUSH); + if (m_state != Z_OK && m_state != Z_STREAM_ERROR && m_state != Z_STREAM_END && m_state != Z_BUF_ERROR) + { + // Z_BUF_ERROR is a success code for Z_FINISH, and the caller can continue as if operation_hint::is_last was + // not given + std::stringstream ss; + ss << "Unrecoverable decompression stream error " << m_state; + throw std::runtime_error(std::move(ss.str())); + } + + input_bytes_processed = input_size - m_stream.avail_in; + if (done) + { + *done = (m_state == Z_STREAM_END); + } + return output_size - m_stream.avail_out; + } + + pplx::task decompress( + const uint8_t* input, size_t input_size, uint8_t* output, size_t output_size, operation_hint hint) + { + operation_result r; + + try + { + r.output_bytes_produced = + decompress(input, input_size, output, output_size, hint, r.input_bytes_processed, &r.done); + } + catch (...) + { + pplx::task_completion_event ev; + ev.set_exception(std::current_exception()); + return pplx::create_task(ev); + } + + return pplx::task_from_result(r); + } + + void reset() + { + m_state = inflateReset(&m_stream); + if (m_state != Z_OK) + { + std::stringstream ss; + ss << "Failed to reset zlib decompressor " << m_state; + throw std::runtime_error(std::move(ss.str())); + } + } + + ~zlib_decompressor_base() { (void)inflateEnd(&m_stream); } + +private: + int m_state{Z_BUF_ERROR}; + z_stream m_stream{0}; + const utility::string_t& m_algorithm; +}; + +class gzip_compressor : public zlib_compressor_base +{ +public: + gzip_compressor() : zlib_compressor_base(31) // 15 is MAX_WBITS in zconf.h; add 16 for gzip + { + } + + gzip_compressor(int compressionLevel, int method, int strategy, int memLevel) + : zlib_compressor_base(31, compressionLevel, method, strategy, memLevel) + { + } +}; + +class gzip_decompressor : public zlib_decompressor_base +{ +public: + gzip_decompressor::gzip_decompressor() : zlib_decompressor_base(16) // gzip auto-detect + { + } +}; + +class deflate_compressor : public zlib_compressor_base +{ +public: + deflate_compressor() : zlib_compressor_base(15) // 15 is MAX_WBITS in zconf.h + { + } + + deflate_compressor(int compressionLevel, int method, int strategy, int memLevel) + : zlib_compressor_base(15, compressionLevel, method, strategy, memLevel) + { + } +}; + +class deflate_decompressor : public zlib_decompressor_base +{ +public: + deflate_decompressor() : zlib_decompressor_base(0) // deflate auto-detect + { + } +}; + +#if defined(CPPREST_BROTLI_COMPRESSION) +class brotli_compressor : public compress_provider +{ +public: + static const utility::string_t BROTLI; + + brotli_compressor(uint32_t window = BROTLI_DEFAULT_WINDOW, + uint32_t quality = BROTLI_DEFAULT_QUALITY, + uint32_t mode = BROTLI_DEFAULT_MODE) + : m_algorithm(BROTLI), m_window(window), m_quality(quality), m_mode(mode) + { + (void)reset(); + } + + const utility::string_t& algorithm() const { return m_algorithm; } + + size_t compress(const uint8_t* input, + size_t input_size, + uint8_t* output, + size_t output_size, + operation_hint hint, + size_t& input_bytes_processed, + bool* done) + { + if (m_done || (hint != operation_hint::is_last && !input_size)) + { + input_bytes_processed = 0; + if (done) + { + *done = m_done; + } + return 0; + } + + if (m_state != BROTLI_TRUE) + { + throw std::runtime_error("Prior unrecoverable compression stream error"); + } + + const uint8_t* next_in = input; + size_t avail_in; + uint8_t* next_out = output; + size_t avail_out = output_size; + size_t total_out; + + if (BrotliEncoderHasMoreOutput(m_stream) == BROTLI_TRUE) + { + avail_in = 0; + do + { + m_state = BrotliEncoderCompressStream(m_stream, + (hint == operation_hint::is_last) ? BROTLI_OPERATION_FINISH + : BROTLI_OPERATION_FLUSH, + &avail_in, + &next_in, + &avail_out, + &next_out, + &total_out); + } while (m_state == BROTLI_TRUE && avail_out && BrotliEncoderHasMoreOutput(m_stream) == BROTLI_TRUE); + } + + if (m_state == BROTLI_TRUE && avail_out) + { + avail_in = input_size; + do + { + m_state = BrotliEncoderCompressStream(m_stream, + (hint == operation_hint::is_last) ? BROTLI_OPERATION_FINISH + : BROTLI_OPERATION_FLUSH, + &avail_in, + &next_in, + &avail_out, + &next_out, + &total_out); + } while (m_state == BROTLI_TRUE && avail_out && BrotliEncoderHasMoreOutput(m_stream) == BROTLI_TRUE); + } + + if (m_state != BROTLI_TRUE) + { + throw std::runtime_error("Unrecoverable compression stream error"); + } + + if (hint == operation_hint::is_last) + { + m_done = (BrotliEncoderIsFinished(m_stream) == BROTLI_TRUE); + } + + input_bytes_processed = input_size - avail_in; + if (done) + { + *done = m_done; + } + return output_size - avail_out; + } + + pplx::task compress( + const uint8_t* input, size_t input_size, uint8_t* output, size_t output_size, operation_hint hint) + { + operation_result r; + + try + { + r.output_bytes_produced = + compress(input, input_size, output, output_size, hint, r.input_bytes_processed, &r.done); + } + catch (...) + { + pplx::task_completion_event ev; + ev.set_exception(std::current_exception()); + return pplx::create_task(ev); + } + + return pplx::task_from_result(r); + } + + void reset() + { + if (m_stream) + { + BrotliEncoderDestroyInstance(m_stream); + } + + m_stream = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr); + m_state = m_stream ? BROTLI_TRUE : BROTLI_FALSE; + + if (m_state == BROTLI_TRUE && m_window != BROTLI_DEFAULT_WINDOW) + { + m_state = BrotliEncoderSetParameter(m_stream, BROTLI_PARAM_LGWIN, m_window); + } + if (m_state == BROTLI_TRUE && m_quality != BROTLI_DEFAULT_QUALITY) + { + m_state = BrotliEncoderSetParameter(m_stream, BROTLI_PARAM_QUALITY, m_quality); + } + if (m_state == BROTLI_TRUE && m_mode != BROTLI_DEFAULT_MODE) + { + m_state = BrotliEncoderSetParameter(m_stream, BROTLI_PARAM_MODE, m_window); + } + + if (m_state != BROTLI_TRUE) + { + throw std::runtime_error("Failed to reset Brotli compressor"); + } + } + + ~brotli_compressor() + { + if (m_stream) + { + BrotliEncoderDestroyInstance(m_stream); + } + } + +private: + BROTLI_BOOL m_state{BROTLI_FALSE}; + BrotliEncoderState* m_stream{nullptr}; + bool m_done{false}; + uint32_t m_window; + uint32_t m_quality; + uint32_t m_mode; + const utility::string_t& m_algorithm; +}; + +const utility::string_t brotli_compressor::BROTLI(algorithm::BROTLI); + +class brotli_decompressor : public decompress_provider +{ +public: + brotli_decompressor() : m_algorithm(brotli_compressor::BROTLI) + { + try + { + reset(); + } + catch (...) + { + } + } + + const utility::string_t& algorithm() const { return m_algorithm; } + + size_t decompress(const uint8_t* input, + size_t input_size, + uint8_t* output, + size_t output_size, + operation_hint hint, + size_t& input_bytes_processed, + bool* done) + { + if (m_state == BROTLI_DECODER_RESULT_SUCCESS /* || !input_size*/) + { + input_bytes_processed = 0; + if (done) + { + *done = (m_state == BROTLI_DECODER_RESULT_SUCCESS); + } + return 0; + } + + if (m_state == BROTLI_DECODER_RESULT_ERROR) + { + throw std::runtime_error("Prior unrecoverable decompression stream error"); + } + + const uint8_t* next_in = input; + size_t avail_in = input_size; + uint8_t* next_out = output; + size_t avail_out = output_size; + size_t total_out; + + // N.B. we ignore 'hint' here. We could instead call BrotliDecoderDecompress() if it's set, but we'd either + // have to first allocate a guaranteed-large-enough buffer and then copy out of it, or we'd have to call + // reset() if it failed due to insufficient output buffer space (and we'd need to use + // BrotliDecoderGetErrorCode() to tell if that's why it failed) + m_state = BrotliDecoderDecompressStream(m_stream, &avail_in, &next_in, &avail_out, &next_out, &total_out); + if (m_state == BROTLI_DECODER_RESULT_ERROR) + { + throw std::runtime_error("Unrecoverable decompression stream error"); + } + + input_bytes_processed = input_size - avail_in; + if (done) + { + *done = (m_state == BROTLI_DECODER_RESULT_SUCCESS); + } + return output_size - avail_out; + } + + pplx::task decompress( + const uint8_t* input, size_t input_size, uint8_t* output, size_t output_size, operation_hint hint) + { + operation_result r; + + try + { + r.output_bytes_produced = + decompress(input, input_size, output, output_size, hint, r.input_bytes_processed, &r.done); + } + catch (...) + { + pplx::task_completion_event ev; + ev.set_exception(std::current_exception()); + return pplx::create_task(ev); + } + + return pplx::task_from_result(r); + } + + void reset() + { + if (m_stream) + { + BrotliDecoderDestroyInstance(m_stream); + } + + m_stream = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); + m_state = m_stream ? BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT : BROTLI_DECODER_RESULT_ERROR; + + if (m_state != BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) + { + throw std::runtime_error("Failed to reset Brotli decompressor"); + } + } + + ~brotli_decompressor() + { + if (m_stream) + { + BrotliDecoderDestroyInstance(m_stream); + } + } + +private: + BrotliDecoderResult m_state{BROTLI_DECODER_RESULT_ERROR}; + BrotliDecoderState* m_stream{nullptr}; + const utility::string_t& m_algorithm; +}; +#endif // CPPREST_BROTLI_COMPRESSION +#endif // CPPREST_HTTP_COMPRESSION + +// Generic internal implementation of the compress_factory API +class generic_compress_factory : public compress_factory +{ +public: + generic_compress_factory(const utility::string_t& algorithm, + std::function()> make_compressor) + : m_algorithm(algorithm), _make_compressor(make_compressor) + { + } + + const utility::string_t& algorithm() const { return m_algorithm; } + + std::unique_ptr make_compressor() const { return _make_compressor(); } + +private: + const utility::string_t m_algorithm; + std::function()> _make_compressor; +}; + +// Generic internal implementation of the decompress_factory API +class generic_decompress_factory : public decompress_factory +{ +public: + generic_decompress_factory(const utility::string_t& algorithm, + uint16_t weight, + std::function()> make_decompressor) + : m_algorithm(algorithm), m_weight(weight), _make_decompressor(make_decompressor) + { + } + + const utility::string_t& algorithm() const { return m_algorithm; } + + const uint16_t weight() const { return m_weight; } + + std::unique_ptr make_decompressor() const { return _make_decompressor(); } + +private: + const utility::string_t m_algorithm; + uint16_t m_weight; + std::function()> _make_decompressor; +}; + +// "Private" algorithm-to-factory tables for namespace static helpers +static const std::vector> g_compress_factories +#if defined(CPPREST_HTTP_COMPRESSION) + = {std::make_shared( + algorithm::GZIP, []() -> std::unique_ptr { return std::make_unique(); }), + std::make_shared( + algorithm::DEFLATE, + []() -> std::unique_ptr { return std::make_unique(); }), +#if defined(CPPREST_BROTLI_COMPRESSION) + std::make_shared( + algorithm::BROTLI, + []() -> std::unique_ptr { return std::make_unique(); }) +#endif // CPPREST_BROTLI_COMPRESSION +}; +#else // CPPREST_HTTP_COMPRESSION + ; +#endif // CPPREST_HTTP_COMPRESSION + +static const std::vector> g_decompress_factories +#if defined(CPPREST_HTTP_COMPRESSION) + = {std::make_shared( + algorithm::GZIP, + 500, + []() -> std::unique_ptr { return std::make_unique(); }), + std::make_shared( + algorithm::DEFLATE, + 500, + []() -> std::unique_ptr { return std::make_unique(); }), +#if defined(CPPREST_BROTLI_COMPRESSION) + std::make_shared( + algorithm::BROTLI, + 500, + []() -> std::unique_ptr { return std::make_unique(); }) +#endif // CPPREST_BROTLI_COMPRESSION +}; +#else // CPPREST_HTTP_COMPRESSION + ; +#endif // CPPREST_HTTP_COMPRESSION + +bool supported() { return !g_compress_factories.empty(); } + +bool algorithm::supported(const utility::string_t& algorithm) +{ + for (auto& factory : g_compress_factories) + { + if (utility::details::str_iequal(algorithm, factory->algorithm())) + { + return true; + } + } + + return false; +} + +static std::unique_ptr _make_compressor( + const std::vector>& factories, const utility::string_t& algorithm) +{ + for (auto& factory : factories) + { + if (factory && utility::details::str_iequal(algorithm, factory->algorithm())) + { + return factory->make_compressor(); + } + } + + return std::unique_ptr(); +} + +std::unique_ptr make_compressor(const utility::string_t& algorithm) +{ + return _make_compressor(g_compress_factories, algorithm); +} + +static std::unique_ptr _make_decompressor( + const std::vector>& factories, const utility::string_t& algorithm) +{ + for (auto& factory : factories) + { + if (factory && utility::details::str_iequal(algorithm, factory->algorithm())) + { + return factory->make_decompressor(); + } + } + + return std::unique_ptr(); +} + +std::unique_ptr make_decompressor(const utility::string_t& algorithm) +{ + return _make_decompressor(g_decompress_factories, algorithm); +} + +std::shared_ptr get_compress_factory(const utility::string_t& algorithm) +{ + for (auto& factory : g_compress_factories) + { + if (utility::details::str_iequal(algorithm, factory->algorithm())) + { + return factory; + } + } + + return std::shared_ptr(); +} + +std::shared_ptr get_decompress_factory(const utility::string_t& algorithm) +{ + for (auto& factory : g_decompress_factories) + { + if (utility::details::str_iequal(algorithm, factory->algorithm())) + { + return factory; + } + } + + return std::shared_ptr(); +} + +std::unique_ptr make_gzip_compressor(int compressionLevel, int method, int strategy, int memLevel) +{ +#if defined(CPPREST_HTTP_COMPRESSION) + return std::move(std::make_unique(compressionLevel, method, strategy, memLevel)); +#else // CPPREST_HTTP_COMPRESSION + return std::unique_ptr(); +#endif // CPPREST_HTTP_COMPRESSION +} + +std::unique_ptr make_deflate_compressor(int compressionLevel, int method, int strategy, int memLevel) +{ +#if defined(CPPREST_HTTP_COMPRESSION) + return std::move(std::make_unique(compressionLevel, method, strategy, memLevel)); +#else // CPPREST_HTTP_COMPRESSION + return std::unique_ptr(); +#endif // CPPREST_HTTP_COMPRESSION +} + +std::unique_ptr make_brotli_compressor(uint32_t window, uint32_t quality, uint32_t mode) +{ +#if defined(CPPREST_HTTP_COMPRESSION) && defined(CPPREST_BROTLI_COMPRESSION) + return std::move(std::make_unique(window, quality, mode)); +#else // CPPREST_BROTLI_COMPRESSION + return std::unique_ptr(); +#endif // CPPREST_BROTLI_COMPRESSION +} +} // namespace builtin + +std::shared_ptr make_compress_factory( + const utility::string_t& algorithm, std::function()> make_compressor) +{ + return std::make_shared(algorithm, make_compressor); +} + +std::shared_ptr make_decompress_factory( + const utility::string_t& algorithm, + uint16_t weight, + std::function()> make_decompressor) +{ + return std::make_shared(algorithm, weight, make_decompressor); +} + +namespace details +{ +namespace builtin +{ +const std::vector> get_decompress_factories() +{ + return web::http::compression::builtin::g_decompress_factories; +} +} // namespace builtin + +static bool is_http_whitespace(utility::char_t ch) { return ch == _XPLATSTR(' ') || ch == _XPLATSTR('\t'); } + +static void remove_surrounding_http_whitespace(const utility::string_t& encoding, size_t& start, size_t& length) +{ + while (length > 0 && is_http_whitespace(encoding.at(start))) + { + start++; + length--; + } + while (length > 0 && is_http_whitespace(encoding.at(start + length - 1))) + { + length--; + } +} + +std::unique_ptr get_compressor_from_header( + const utility::string_t& encoding, + header_types type, + const std::vector>& factories) +{ + const std::vector>& f = + factories.empty() ? web::http::compression::builtin::g_compress_factories : factories; + std::unique_ptr compressor; + struct _tuple + { + size_t start; + size_t length; + size_t rank; + } t; + std::vector<_tuple> tokens; + size_t highest; + size_t mark; + size_t end; + size_t n; + bool first; + + _ASSERTE(type == header_types::te || type == header_types::accept_encoding); + + // See https://tools.ietf.org/html/rfc7230#section-4.3 (TE) and + // https://tools.ietf.org/html/rfc7231#section-5.3.4 (Accept-Encoding) for details + + n = 0; + highest = 0; + first = true; + while (n != utility::string_t::npos) + { + // Tokenize by commas first + mark = encoding.find(_XPLATSTR(','), n); + t.start = n; + t.rank = static_cast(-1); + if (mark == utility::string_t::npos) + { + t.length = encoding.size() - n; + n = utility::string_t::npos; + } + else + { + t.length = mark - n; + n = mark + 1; + } + + // Then remove leading and trailing whitespace + remove_surrounding_http_whitespace(encoding, t.start, t.length); + + // Next split at the semicolon, if any, and deal with rank and additional whitespace + mark = encoding.find(_XPLATSTR(';'), t.start); + if (mark < t.start + t.length) + { + end = t.start + t.length - 1; + t.length = mark - t.start; + while (t.length > 0 && is_http_whitespace(encoding.at(t.start + t.length - 1))) + { + // Skip trailing whitespace in encoding type + t.length--; + } + if (mark < end) + { + // Check for an optional ranking, max. length "q=0.999" + mark = encoding.find(_XPLATSTR("q="), mark + 1); + if (mark != utility::string_t::npos && mark + 1 < end && end - mark <= 6) + { + // Determine ranking; leading whitespace has been implicitly skipped by find(). + // The ranking always starts with '1' or '0' per standard, and has at most 3 decimal places + mark += 1; + t.rank = 1000 * (encoding.at(mark + 1) - _XPLATSTR('0')); + if (mark + 2 < end && encoding.at(mark + 2) == _XPLATSTR('.')) + { + // This is a real number rank; convert decimal part to hundreds and apply it + size_t factor = 100; + mark += 2; + for (size_t i = mark + 1; i <= end; i++) + { + t.rank += (encoding.at(i) - _XPLATSTR('0')) * factor; + factor /= 10; + } + } + if (t.rank > 1000) + { + throw http_exception(status_codes::BadRequest, "Invalid q-value in header"); + } + } + } + } + + if (!t.length) + { + if (!first || n != utility::string_t::npos) + { + // An entirely empty header is OK per RFC, but an extraneous comma is not + throw http_exception(status_codes::BadRequest, "Empty field in header"); + } + return std::unique_ptr(); + } + + if (!compressor) + { + if (t.rank == static_cast(1000) || t.rank == static_cast(-1)) + { + // Immediately try to instantiate a compressor for any unranked or top-ranked algorithm + compressor = web::http::compression::builtin::_make_compressor(f, encoding.substr(t.start, t.length)); + } + else if (t.rank) + { + // Store off remaining ranked algorithms, sorting as we go + if (t.rank >= highest) + { + tokens.emplace_back(t); + highest = t.rank; + } + else + { + for (auto x = tokens.begin(); x != tokens.end(); x++) + { + if (t.rank <= x->rank) + { + tokens.emplace(x, t); + break; + } + } + } + } + // else a rank of 0 means "not permitted" + } + // else we've chosen a compressor; we're just validating the rest of the header + + first = false; + } + // Note: for Accept-Encoding, we don't currently explicitly handle "identity;q=0" and "*;q=0" + + if (compressor) + { + return std::move(compressor); + } + + // If we're here, we didn't match the caller's compressor above; + // try any that we saved off in order of highest to lowest rank + for (auto t = tokens.rbegin(); t != tokens.rend(); t++) + { + auto coding = encoding.substr(t->start, t->length); + + // N.B for TE, "trailers" will simply fail to instantiate a + // compressor; ditto for "*" and "identity" for Accept-Encoding + auto compressor = web::http::compression::builtin::_make_compressor(f, coding); + if (compressor) + { + return std::move(compressor); + } + if (type == header_types::accept_encoding && utility::details::str_iequal(coding, _XPLATSTR("identity"))) + { + // The client specified a preference for "no encoding" vs. anything else we might still have + return std::unique_ptr(); + } + } + + return std::unique_ptr(); +} + +std::unique_ptr get_decompressor_from_header( + const utility::string_t& encoding, + header_types type, + const std::vector>& factories) +{ + const std::vector>& f = + factories.empty() ? web::http::compression::builtin::g_decompress_factories : factories; + std::unique_ptr decompressor; + utility::string_t token; + size_t start; + size_t length; + size_t comma; + size_t n; + + _ASSERTE(type == header_types::transfer_encoding || type == header_types::content_encoding); + + n = 0; + while (n != utility::string_t::npos) + { + // Tokenize by commas first + comma = encoding.find(_XPLATSTR(','), n); + start = n; + if (comma == utility::string_t::npos) + { + length = encoding.size() - n; + n = utility::string_t::npos; + } + else + { + length = comma - n; + n = comma + 1; + } + + // Then remove leading and trailing whitespace + remove_surrounding_http_whitespace(encoding, start, length); + + if (!length) + { + throw http_exception(status_codes::BadRequest, "Empty field in header"); + } + + // Immediately try to instantiate a decompressor + token = encoding.substr(start, length); + auto d = web::http::compression::builtin::_make_decompressor(f, token); + if (d) + { + if (decompressor) + { + status_code code = status_codes::NotImplemented; + if (type == header_types::content_encoding) + { + code = status_codes::UnsupportedMediaType; + } + throw http_exception(code, "Multiple compression algorithms not supported for a single request"); + } + + // We found our decompressor; store it off while we process the rest of the header + decompressor = std::move(d); + } + else + { + if (n != utility::string_t::npos) + { + if (type == header_types::transfer_encoding && + utility::details::str_iequal(_XPLATSTR("chunked"), token)) + { + throw http_exception(status_codes::BadRequest, + "Chunked must come last in the Transfer-Encoding header"); + } + } + if (!decompressor && !f.empty() && (n != utility::string_t::npos || type == header_types::content_encoding)) + { + // The first encoding type did not match; throw an informative + // exception with an encoding-type-appropriate HTTP error code + status_code code = status_codes::NotImplemented; + if (type == header_types::content_encoding) + { + code = status_codes::UnsupportedMediaType; + } + throw http_exception(code, "Unsupported encoding type"); + } + } + } + + if (type == header_types::transfer_encoding && !utility::details::str_iequal(_XPLATSTR("chunked"), token)) + { + throw http_exception(status_codes::BadRequest, "Transfer-Encoding header missing chunked"); + } + + // Either the response is compressed and we have a decompressor that can handle it, or + // built-in compression is not enabled and we don't have an alternate set of decompressors + return std::move(decompressor); +} + +utility::string_t build_supported_header(header_types type, + const std::vector>& factories) +{ + const std::vector>& f = + factories.empty() ? web::http::compression::builtin::g_decompress_factories : factories; + utility::ostringstream_t os; + bool start; + + _ASSERTE(type == header_types::te || type == header_types::accept_encoding); + + // Add all specified algorithms and their weights to the header + start = true; + os.imbue(std::locale::classic()); + for each (auto& factory in f) + { + if (factory) + { + auto weight = factory->weight(); + + if (!start) + { + os << _XPLATSTR(", "); + } + os << factory->algorithm(); + if (weight <= 1000) + { + os << _XPLATSTR(";q=") << weight / 1000 << _XPLATSTR(".") << weight % 1000; + } + start = false; + } + } + + if (start && type == header_types::accept_encoding) + { + // Request that no encoding be applied + os << _XPLATSTR("identity;q=1, *;q=0"); + } + + return std::move(os.str()); +} +} // namespace details +} // namespace compression +} // namespace http +} // namespace web diff --git a/Release/src/http/common/http_helpers.cpp b/Release/src/http/common/http_helpers.cpp index 2c45e5961d..0d17c569eb 100644 --- a/Release/src/http/common/http_helpers.cpp +++ b/Release/src/http/common/http_helpers.cpp @@ -12,27 +12,6 @@ ****/ #include "stdafx.h" - -// CPPREST_EXCLUDE_COMPRESSION is set if we're on a platform that supports compression but we want to explicitly disable it. -// CPPREST_EXCLUDE_WEBSOCKETS is a flag that now essentially means "no external dependencies". TODO: Rename - -#if __APPLE__ -#include "TargetConditionals.h" -#if defined(TARGET_OS_MAC) -#if !defined(CPPREST_EXCLUDE_COMPRESSION) -#define CPPREST_HTTP_COMPRESSION -#endif // !defined(CPPREST_EXCLUDE_COMPRESSION) -#endif // defined(TARGET_OS_MAC) -#elif defined(_WIN32) && (!defined(WINAPI_FAMILY) || WINAPI_PARTITION_DESKTOP) -#if !defined(CPPREST_EXCLUDE_WEBSOCKETS) && !defined(CPPREST_EXCLUDE_COMPRESSION) -#define CPPREST_HTTP_COMPRESSION -#endif // !defined(CPPREST_EXCLUDE_WEBSOCKETS) && !defined(CPPREST_EXCLUDE_COMPRESSION) -#endif - -#if defined(CPPREST_HTTP_COMPRESSION) -#include -#endif - #include "internal_http_helpers.h" using namespace web; @@ -142,309 +121,5 @@ bool validate_method(const utility::string_t& method) return true; } -namespace compression -{ -#if defined(CPPREST_HTTP_COMPRESSION) - - class compression_base_impl - { - public: - compression_base_impl(compression_algorithm alg) : m_alg(alg), m_zLibState(Z_OK) - { - memset(&m_zLibStream, 0, sizeof(m_zLibStream)); - } - - size_t read_output(size_t input_offset, size_t available_input, size_t total_out_before, uint8_t* temp_buffer, data_buffer& output) - { - input_offset += (available_input - stream().avail_in); - auto out_length = stream().total_out - total_out_before; - output.insert(output.end(), temp_buffer, temp_buffer + out_length); - - return input_offset; - } - - bool is_complete() const - { - return state() == Z_STREAM_END; - } - - bool has_error() const - { - return !is_complete() && state() != Z_OK; - } - - int state() const - { - return m_zLibState; - } - - void set_state(int state) - { - m_zLibState = state; - } - - compression_algorithm algorithm() const - { - return m_alg; - } - - z_stream& stream() - { - return m_zLibStream; - } - - int to_zlib_alg(compression_algorithm alg) - { - return static_cast(alg); - } - - private: - const compression_algorithm m_alg; - - std::atomic m_zLibState{ Z_OK }; - z_stream m_zLibStream; - }; - - class stream_decompressor::stream_decompressor_impl : public compression_base_impl - { - public: - stream_decompressor_impl(compression_algorithm alg) : compression_base_impl(alg) - { - set_state(inflateInit2(&stream(), to_zlib_alg(alg))); - } - - ~stream_decompressor_impl() - { - inflateEnd(&stream()); - } - - data_buffer decompress(const uint8_t* input, size_t input_size) - { - if (input == nullptr || input_size == 0) - { - set_state(Z_BUF_ERROR); - return data_buffer(); - } - - // Need to guard against attempting to decompress when we're already finished or encountered an error! - if (is_complete() || has_error()) - { - set_state(Z_STREAM_ERROR); - return data_buffer(); - } - - const size_t BUFFER_SIZE = 1024; - unsigned char temp_buffer[BUFFER_SIZE]; - - data_buffer output; - output.reserve(input_size * 3); - - size_t input_offset{ 0 }; - - while (state() == Z_OK && input_offset < input_size) - { - auto total_out_before = stream().total_out; - - auto available_input = input_size - input_offset; - stream().next_in = const_cast(&input[input_offset]); - stream().avail_in = static_cast(available_input); - stream().next_out = temp_buffer; - stream().avail_out = BUFFER_SIZE; - - set_state(inflate(&stream(), Z_PARTIAL_FLUSH)); - - if (has_error()) - { - return data_buffer(); - } - - input_offset = read_output(input_offset, available_input, total_out_before, temp_buffer, output); - } - - return output; - } - }; - - class stream_compressor::stream_compressor_impl : public compression_base_impl - { - public: - stream_compressor_impl(compression_algorithm alg) : compression_base_impl(alg) - { - const int level = Z_DEFAULT_COMPRESSION; - if (alg == compression_algorithm::gzip) - { - set_state(deflateInit2(&stream(), level, Z_DEFLATED, to_zlib_alg(alg), MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY)); - } - else if (alg == compression_algorithm::deflate) - { - set_state(deflateInit(&stream(), level)); - } - } - - web::http::details::compression::data_buffer compress(const uint8_t* input, size_t input_size, bool finish) - { - if (input == nullptr || input_size == 0) - { - set_state(Z_BUF_ERROR); - return data_buffer(); - } - - if (state() != Z_OK) - { - set_state(Z_STREAM_ERROR); - return data_buffer(); - } - - data_buffer output; - output.reserve(input_size); - - const size_t BUFFER_SIZE = 1024; - uint8_t temp_buffer[BUFFER_SIZE]; - - size_t input_offset{ 0 }; - auto flush = Z_NO_FLUSH; - - while (flush == Z_NO_FLUSH) - { - auto total_out_before = stream().total_out; - auto available_input = input_size - input_offset; - - if (available_input == 0) - { - flush = finish ? Z_FINISH : Z_PARTIAL_FLUSH; - } - else - { - stream().avail_in = static_cast(available_input); - stream().next_in = const_cast(&input[input_offset]); - } - - do - { - stream().next_out = temp_buffer; - stream().avail_out = BUFFER_SIZE; - - set_state(deflate(&stream(), flush)); - - if (has_error()) - { - return data_buffer(); - } - - input_offset = read_output(input_offset, available_input, total_out_before, temp_buffer, output); - - } while (stream().avail_out == 0); - } - - return output; - } - - ~stream_compressor_impl() - { - deflateEnd(&stream()); - } - }; -#else // Stub impl for when compression is not supported - - class compression_base_impl - { - public: - bool has_error() const - { - return true; - } - }; - - class stream_compressor::stream_compressor_impl : public compression_base_impl - { - public: - stream_compressor_impl(compression_algorithm) {} - compression::data_buffer compress(const uint8_t* data, size_t size, bool) - { - return data_buffer(data, data + size); - } - }; - - class stream_decompressor::stream_decompressor_impl : public compression_base_impl - { - public: - stream_decompressor_impl(compression_algorithm) {} - compression::data_buffer decompress(const uint8_t* data, size_t size) - { - return data_buffer(data, data + size); - } - }; -#endif - - bool __cdecl stream_decompressor::is_supported() - { -#if !defined(CPPREST_HTTP_COMPRESSION) - return false; -#else - return true; -#endif - } - - stream_decompressor::stream_decompressor(compression_algorithm alg) - : m_pimpl(std::make_shared(alg)) - { - } - - compression::data_buffer stream_decompressor::decompress(const data_buffer& input) - { - if (input.empty()) - { - return data_buffer(); - } - - return m_pimpl->decompress(&input[0], input.size()); - } - - web::http::details::compression::data_buffer stream_decompressor::decompress(const uint8_t* input, size_t input_size) - { - return m_pimpl->decompress(input, input_size); - } - - bool stream_decompressor::has_error() const - { - return m_pimpl->has_error(); - } - - bool __cdecl stream_compressor::is_supported() - { -#if !defined(CPPREST_HTTP_COMPRESSION) - return false; -#else - return true; -#endif - } - - stream_compressor::stream_compressor(compression_algorithm alg) - : m_pimpl(std::make_shared(alg)) - { - - } - - compression::data_buffer stream_compressor::compress(const data_buffer& input, bool finish) - { - if (input.empty()) - { - return compression::data_buffer(); - } - - return m_pimpl->compress(&input[0], input.size(), finish); - } - - web::http::details::compression::data_buffer stream_compressor::compress(const uint8_t* input, size_t input_size, bool finish) - { - return m_pimpl->compress(input, input_size, finish); - } - - bool stream_compressor::has_error() const - { - return m_pimpl->has_error(); - } - -} // namespace compression } // namespace details }} // namespace web::http diff --git a/Release/src/http/common/http_msg.cpp b/Release/src/http/common/http_msg.cpp index 8f218cb913..d17d09dcef 100644 --- a/Release/src/http/common/http_msg.cpp +++ b/Release/src/http/common/http_msg.cpp @@ -310,36 +310,95 @@ void http_msg_base::_prepare_to_receive_data() // or media (like file) that the user can read from... } -size_t http_msg_base::_get_content_length() +size_t http_msg_base::_get_stream_length() +{ + auto &stream = instream(); + + if (stream.can_seek()) + { + auto offset = stream.tell(); + auto end = stream.seek(0, std::ios_base::end); + stream.seek(offset); + return static_cast(end - offset); + } + + return std::numeric_limits::max(); +} + +size_t http_msg_base::_get_content_length(bool honor_compression) { // An invalid response_stream indicates that there is no body if ((bool)instream()) { - size_t content_length = 0; + size_t content_length; utility::string_t transfer_encoding; - bool has_cnt_length = headers().match(header_names::content_length, content_length); - bool has_xfr_encode = headers().match(header_names::transfer_encoding, transfer_encoding); + if (headers().match(header_names::transfer_encoding, transfer_encoding)) + { + // Transfer encoding is set; it trumps any content length that may or may not be present + if (honor_compression && m_compressor) + { + http::http_headers tmp; + + // Build a header for comparison with the existing one + tmp.add(header_names::transfer_encoding, m_compressor->algorithm()); + tmp.add(header_names::transfer_encoding, _XPLATSTR("chunked")); + + if (!utility::details::str_iequal(transfer_encoding, tmp[header_names::transfer_encoding])) + { + // Some external entity added this header, and it doesn't match our + // expectations; bail out, since the caller's intentions are not clear + throw http_exception("Transfer-Encoding header is internally managed when compressing"); + } + } - if (has_xfr_encode) + return std::numeric_limits::max(); + } + + if (honor_compression && m_compressor) { + // A compressor is set; this implies transfer encoding, since we don't know the compressed length + // up front for content encoding. We return the uncompressed length if we can figure it out. + headers().add(header_names::transfer_encoding, m_compressor->algorithm()); + headers().add(header_names::transfer_encoding, _XPLATSTR("chunked")); return std::numeric_limits::max(); } - if (has_cnt_length) + if (headers().match(header_names::content_length, content_length)) + { + // An explicit content length is set; trust it, since we + // may not be required to send the stream's entire contents + return content_length; + } + + content_length = _get_stream_length(); + if (content_length != std::numeric_limits::max()) { + // The content length wasn't explcitly set, but we figured it out; + // use it, since sending this way is more efficient than chunking + headers().add(header_names::content_length, content_length); return content_length; } - // Neither is set. Assume transfer-encoding for now (until we have the ability to determine - // the length of the stream). + // We don't know the content length; we'll chunk the stream headers().add(header_names::transfer_encoding, _XPLATSTR("chunked")); return std::numeric_limits::max(); } + // There is no content return 0; } +size_t http_msg_base::_get_content_length_and_set_compression() +{ + return _get_content_length(true); +} + +size_t http_msg_base::_get_content_length() +{ + return _get_content_length(false); +} + // Helper function to inline continuation if possible. struct inline_continuation { @@ -1041,6 +1100,11 @@ details::_http_request::_http_request(std::unique_ptrset_decompress_factories(compression::details::builtin::get_decompress_factories()); +} + const http_version http_versions::HTTP_0_9 = { 0, 9 }; const http_version http_versions::HTTP_1_0 = { 1, 0 }; const http_version http_versions::HTTP_1_1 = { 1, 1 }; diff --git a/Release/src/http/common/internal_http_helpers.h b/Release/src/http/common/internal_http_helpers.h index 024ac84ac4..ef19612270 100644 --- a/Release/src/http/common/internal_http_helpers.h +++ b/Release/src/http/common/internal_http_helpers.h @@ -31,3 +31,19 @@ void trim_whitespace(std::basic_string &str) bool validate_method(const utility::string_t& method); }}} + +namespace web { namespace http { namespace compression { + +class compression::decompress_factory; + +namespace details { namespace builtin { + +/// +/// Helper function to get the set of built-in decompress factories +/// + +const std::vector> get_decompress_factories(); + +}} + +}}} diff --git a/Release/src/http/listener/http_server_httpsys.cpp b/Release/src/http/listener/http_server_httpsys.cpp index 8da74d0083..3adb4fb882 100644 --- a/Release/src/http/listener/http_server_httpsys.cpp +++ b/Release/src/http/listener/http_server_httpsys.cpp @@ -85,7 +85,7 @@ static utility::string_t HttpServerAPIKnownHeaders[] = U("Proxy-Authorization"), U("Referer"), U("Range"), - U("Te"), + U("TE"), U("Translate"), U("User-Agent"), U("Request-Maximum"), @@ -539,6 +539,7 @@ void windows_request_context::read_headers_io_completion(DWORD error_code, DWORD } else { + utility::string_t header; std::string badRequestMsg; try { @@ -557,6 +558,66 @@ void windows_request_context::read_headers_io_completion(DWORD error_code, DWORD m_msg.set_method(parse_request_method(m_request)); parse_http_headers(m_request->Headers, m_msg.headers()); + // See if we need to compress or decompress the incoming request body, and if so, prepare for it + try + { + if (m_msg.headers().match(header_names::transfer_encoding, header)) + { + try + { + m_decompressor = http::compression::details::get_decompressor_from_header(header, http::compression::details::header_types::transfer_encoding); + } + catch (http_exception &e) + { + if (e.error_code().value() != status_codes::NotImplemented) + { + // Something is wrong with the header; we'll fail here + throw; + } + // We could not find a decompressor; we'll see if the user's handler adds one later + m_decompress_header_type = http::compression::details::header_types::transfer_encoding; + m_decompress_header = std::move(header); + } + } + else if (m_msg.headers().match(header_names::content_encoding, header)) + { + try + { + m_decompressor = http::compression::details::get_decompressor_from_header(header, http::compression::details::header_types::content_encoding); + } + catch (http_exception &e) + { + if (e.error_code().value() != status_codes::UnsupportedMediaType) + { + // Something is wrong with the header; we'll fail here + throw; + } + // We could not find a decompressor; we'll see if the user's handler adds one later + m_decompress_header_type = http::compression::details::header_types::content_encoding; + m_decompress_header = std::move(header); + } + } + else if (m_msg.headers().match(header_names::te, header)) + { + // Note that init_response_headers throws away m_msg, so we need to set our compressor here. If + // the header contains all unsupported algorithms, it's not an error -- we just won't compress + m_compressor = http::compression::details::get_compressor_from_header(header, http::compression::details::header_types::te); + } + else if (m_msg.headers().match(header_names::accept_encoding, header)) + { + // This would require pre-compression of the input stream, since we MUST send Content-Length, so we'll (legally) ignore it + //m_compressor = http::compression::details::get_compressor_from_header(header, http::compression::details::header_types:accept_encoding); + } + } + catch (http_exception &e) + { + if (badRequestMsg.empty()) + { + // Respond with a reasonable message + badRequestMsg = e.what(); + } + } + m_msg._get_impl()->_set_http_version({ (uint8_t)m_request->Version.MajorVersion, (uint8_t)m_request->Version.MinorVersion }); // Retrieve the remote IP address @@ -601,12 +662,24 @@ void windows_request_context::read_headers_io_completion(DWORD error_code, DWORD void windows_request_context::read_request_body_chunk() { auto *pServer = static_cast(http_server_api::server_api()); + PVOID body; // The read_body_io_completion callback function m_overlapped.set_http_io_completion([this](DWORD error, DWORD nBytes){ read_body_io_completion(error, nBytes);}); auto request_body_buf = m_msg._get_impl()->outstream().streambuf(); - auto body = request_body_buf.alloc(CHUNK_SIZE); + if (!m_decompressor) + { + body = request_body_buf.alloc(CHUNK_SIZE); + } + else + { + if (m_compress_buffer.size() < CHUNK_SIZE) + { + m_compress_buffer.resize(CHUNK_SIZE); + } + body = m_compress_buffer.data(); + } // Once we allow users to set the output stream the following assert could fail. // At that time we would need compensation code that would allocate a buffer from the heap instead. @@ -626,7 +699,10 @@ void windows_request_context::read_request_body_chunk() { // There was no more data to read. CancelThreadpoolIo(pServer->m_threadpool_io); - request_body_buf.commit(0); + if (!m_decompressor) + { + request_body_buf.commit(0); + } if(error_code == ERROR_HANDLE_EOF) { m_msg._get_impl()->_complete(request_body_buf.in_avail()); @@ -647,17 +723,49 @@ void windows_request_context::read_body_io_completion(DWORD error_code, DWORD by if (error_code == NO_ERROR) { - request_body_buf.commit(bytes_read); + if (!m_decompressor) + { + request_body_buf.commit(bytes_read); + } + else + { + size_t got; + size_t used; + size_t total_used = 0; + + do + { + auto body = request_body_buf.alloc(CHUNK_SIZE); + try + { + got = m_decompressor->decompress(m_compress_buffer.data()+total_used, bytes_read-total_used, body, CHUNK_SIZE, http::compression::operation_hint::has_more, used, NULL); + } + catch (...) + { + request_body_buf.commit(0); + m_msg._get_impl()->_complete(0, std::current_exception()); + return; + } + request_body_buf.commit(got); + total_used += used; + } while (total_used != bytes_read); + } read_request_body_chunk(); } else if (error_code == ERROR_HANDLE_EOF) { - request_body_buf.commit(0); + if (!m_decompressor) + { + request_body_buf.commit(0); + } m_msg._get_impl()->_complete(request_body_buf.in_avail()); } else { - request_body_buf.commit(0); + if (!m_decompressor) + { + request_body_buf.commit(0); + } m_msg._get_impl()->_complete(0, std::make_exception_ptr(http_exception(error_code))); } } @@ -800,8 +908,53 @@ void windows_request_context::async_process_response() const std::string reason = utf16_to_utf8(m_response.reason_phrase()); win_api_response.pReason = reason.c_str(); win_api_response.ReasonLength = (USHORT)reason.size(); + size_t content_length; - size_t content_length = m_response._get_impl()->_get_content_length(); + if (m_compressor || m_response._get_impl()->compressor()) + { + if (m_response.headers().has(header_names::content_length)) + { + // Content-Length should not be sent with Transfer-Encoding + m_response.headers().remove(header_names::content_length); + } + if (!m_response._get_impl()->compressor()) + { + // Temporarily move the compressor to the reponse, so _get_content_length() will honor it + m_response._get_impl()->set_compressor(std::move(m_compressor)); + } // else one was already set from a callback, and we'll (blindly) use it + content_length = m_response._get_impl()->_get_content_length_and_set_compression(); + m_compressor = std::move(m_response._get_impl()->compressor()); + m_response._get_impl()->set_compressor(nullptr); + } + else + { + if (!m_decompress_header.empty()) + { + auto factories = m_response._get_impl()->decompress_factories(); + try + { + m_decompressor = http::compression::details::get_decompressor_from_header(m_decompress_header, m_decompress_header_type, factories); + m_decompress_header.clear(); + if (!m_decompressor) + { + http::status_code code = http::status_codes::NotImplemented; + if (m_decompress_header_type == http::compression::details::header_types::content_encoding) + { + code = status_codes::UnsupportedMediaType; + } + throw http_exception(code); + } + } + catch (http_exception &e) + { + // No matching decompressor was supplied via callback + CancelThreadpoolIo(pServer->m_threadpool_io); + cancel_request(std::make_exception_ptr(e)); + return; + } + } + content_length = m_response._get_impl()->_get_content_length(); + } m_headers = std::unique_ptr(new HTTP_UNKNOWN_HEADER[msl::safeint3::SafeInt(m_response.headers().size())]); m_headers_buffer.resize(msl::safeint3::SafeInt(m_response.headers().size()) * 2); @@ -822,8 +975,6 @@ void windows_request_context::async_process_response() // Send response callback function m_overlapped.set_http_io_completion([this](DWORD error, DWORD nBytes){ send_response_io_completion(error, nBytes);}); - m_remaining_to_write = content_length; - // Figure out how to send the entity body of the message. if (content_length == 0) { @@ -854,6 +1005,12 @@ void windows_request_context::async_process_response() _ASSERTE(content_length > 0); m_sending_in_chunks = (content_length != std::numeric_limits::max()); m_transfer_encoding = (content_length == std::numeric_limits::max()); + m_remaining_to_write = content_length; + if (content_length == std::numeric_limits::max()) + { + // Attempt to figure out the remaining length of the input stream + m_remaining_to_write = m_response._get_impl()->_get_stream_length(); + } StartThreadpoolIo(pServer->m_threadpool_io); const unsigned long error_code = HttpSendHttpResponse( @@ -901,11 +1058,12 @@ void windows_request_context::transmit_body() return; } + msl::safeint3::SafeInt safeCount = m_remaining_to_write; + size_t next_chunk_size = safeCount.Min(CHUNK_SIZE); + // In both cases here we could perform optimizations to try and use acquire on the streams to avoid an extra copy. if ( m_sending_in_chunks ) { - msl::safeint3::SafeInt safeCount = m_remaining_to_write; - size_t next_chunk_size = safeCount.Min(CHUNK_SIZE); m_body_data.resize(CHUNK_SIZE); streams::rawptr_buffer buf(&m_body_data[0], next_chunk_size); @@ -937,33 +1095,109 @@ void windows_request_context::transmit_body() else { // We're transfer-encoding... - const size_t body_data_length = CHUNK_SIZE+http::details::chunked_encoding::additional_encoding_space; - m_body_data.resize(body_data_length); - - streams::rawptr_buffer buf(&m_body_data[http::details::chunked_encoding::data_offset], body_data_length); - - m_response.body().read(buf, CHUNK_SIZE).then([this, body_data_length](pplx::task op) + if (m_compressor) { - size_t bytes_read = 0; + // ...and compressing. For simplicity, we allocate a buffer that's "too large to fail" while compressing. + const size_t body_data_length = 2*CHUNK_SIZE + http::details::chunked_encoding::additional_encoding_space; + m_body_data.resize(body_data_length); - // If an exception occurs surface the error to user on the server side - // and cancel the request so the client sees the error. - try - { - bytes_read = op.get(); - } catch (...) + // We'll read into a temporary buffer before compressing + if (m_compress_buffer.capacity() < next_chunk_size) { - cancel_request(std::current_exception()); - return; + m_compress_buffer.reserve(next_chunk_size); } - // Check whether this is the last one to send... - m_transfer_encoding = (bytes_read > 0); - size_t offset = http::details::chunked_encoding::add_chunked_delimiters(&m_body_data[0], body_data_length, bytes_read); + streams::rawptr_buffer buf(m_compress_buffer.data(), next_chunk_size); - auto data_length = bytes_read + (http::details::chunked_encoding::additional_encoding_space-offset); - send_entity_body(&m_body_data[offset], data_length); - }); + m_response.body().read(buf, next_chunk_size).then([this, body_data_length](pplx::task op) + { + size_t bytes_read = 0; + + // If an exception occurs surface the error to user on the server side + // and cancel the request so the client sees the error. + try + { + bytes_read = op.get(); + } + catch (...) + { + cancel_request(std::current_exception()); + return; + } + _ASSERTE(bytes_read >= 0); + + // Compress this chunk; if we read no data, allow the compressor to finalize its stream + http::compression::operation_hint hint = http::compression::operation_hint::has_more; + if (!bytes_read) + { + hint = http::compression::operation_hint::is_last; + } + m_compressor->compress(m_compress_buffer.data(), bytes_read, &m_body_data[http::details::chunked_encoding::data_offset], body_data_length, hint) + .then([this, bytes_read, body_data_length](pplx::task op) + { + http::compression::operation_result r; + + try + { + r = op.get(); + } + catch (...) + { + cancel_request(std::current_exception()); + return; + } + + if (r.input_bytes_processed != bytes_read || + r.output_bytes_produced == body_data_length - http::details::chunked_encoding::additional_encoding_space || + r.done != !bytes_read) + { + // We chose our parameters so that compression should + // never overflow body_data_length; fail if it does + cancel_request(std::make_exception_ptr(std::exception("Compressed data exceeds internal buffer size."))); + return; + } + + // Check whether this is the last one to send; note that this is a + // few lines of near-duplicate code with the non-compression path + _ASSERTE(bytes_read <= m_remaining_to_write); + m_remaining_to_write -= bytes_read; + m_transfer_encoding = (r.output_bytes_produced > 0); + size_t offset = http::details::chunked_encoding::add_chunked_delimiters(&m_body_data[0], body_data_length, r.output_bytes_produced); + send_entity_body(&m_body_data[offset], r.output_bytes_produced + http::details::chunked_encoding::additional_encoding_space - offset); + }); + }); + } + else + { + const size_t body_data_length = CHUNK_SIZE + http::details::chunked_encoding::additional_encoding_space; + m_body_data.resize(body_data_length); + + streams::rawptr_buffer buf(&m_body_data[http::details::chunked_encoding::data_offset], body_data_length); + + m_response.body().read(buf, next_chunk_size).then([this, body_data_length](pplx::task op) + { + size_t bytes_read = 0; + + // If an exception occurs surface the error to user on the server side + // and cancel the request so the client sees the error. + try + { + bytes_read = op.get(); + } + catch (...) + { + cancel_request(std::current_exception()); + return; + } + + // Check whether this is the last one to send... + m_transfer_encoding = (bytes_read > 0); + size_t offset = http::details::chunked_encoding::add_chunked_delimiters(&m_body_data[0], body_data_length, bytes_read); + + auto data_length = bytes_read + (http::details::chunked_encoding::additional_encoding_space - offset); + send_entity_body(&m_body_data[offset], data_length); + }); + } } } diff --git a/Release/src/http/listener/http_server_httpsys.h b/Release/src/http/listener/http_server_httpsys.h index 2523d62e4e..5a8cbd137b 100644 --- a/Release/src/http/listener/http_server_httpsys.h +++ b/Release/src/http/listener/http_server_httpsys.h @@ -143,6 +143,13 @@ struct windows_request_context : http::details::_http_server_context http_response m_response; std::exception_ptr m_except_ptr; + + std::vector m_compress_buffer; + std::unique_ptr m_compressor; + std::unique_ptr m_decompressor; + utility::string_t m_decompress_header; + http::compression::details::header_types m_decompress_header_type; + private: windows_request_context(const windows_request_context &); windows_request_context& operator=(const windows_request_context &); diff --git a/Release/tests/functional/http/client/CMakeLists.txt b/Release/tests/functional/http/client/CMakeLists.txt index 60804e1742..17cf4eff81 100644 --- a/Release/tests/functional/http/client/CMakeLists.txt +++ b/Release/tests/functional/http/client/CMakeLists.txt @@ -21,6 +21,7 @@ set(SOURCES status_code_reason_phrase_tests.cpp to_string_tests.cpp http_client_fuzz_tests.cpp + compression_tests.cpp stdafx.cpp ) diff --git a/Release/tests/functional/http/client/compression_tests.cpp b/Release/tests/functional/http/client/compression_tests.cpp new file mode 100644 index 0000000000..e9cc9ef661 --- /dev/null +++ b/Release/tests/functional/http/client/compression_tests.cpp @@ -0,0 +1,1401 @@ +/*** + * Copyright (C) Microsoft. All rights reserved. + * Licensed under the MIT license. See LICENSE.txt file in the project root for full license information. + * + * =+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * + * compression_tests.cpp + * + * Tests cases, including client/server, for the web::http::compression namespace. + * + * =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + ****/ + +#include "cpprest/details/http_helpers.h" +#include "cpprest/version.h" +#include "stdafx.h" +#include + +#ifndef __cplusplus_winrt +#include "cpprest/http_listener.h" +#endif + +using namespace web; +using namespace utility; +using namespace web::http; +using namespace web::http::client; + +using namespace tests::functional::http::utilities; + +namespace tests +{ +namespace functional +{ +namespace http +{ +namespace client +{ +SUITE(compression_tests) +{ + // A fake "pass-through" compressor/decompressor for testing + class fake_provider : public web::http::compression::compress_provider, + public web::http::compression::decompress_provider + { + public: + static const utility::string_t FAKE; + + fake_provider(size_t size = static_cast(-1)) : _size(size), _so_far(0), _done(false) {} + + virtual const utility::string_t& algorithm() const { return FAKE; } + + virtual size_t decompress(const uint8_t* input, + size_t input_size, + uint8_t* output, + size_t output_size, + web::http::compression::operation_hint hint, + size_t& input_bytes_processed, + bool* done) + { + size_t bytes; + + if (_done) + { + input_bytes_processed = 0; + if (*done) + { + *done = true; + } + return 0; + } + if (_size == static_cast(-1) || input_size > _size - _so_far) + { + std::stringstream ss; + ss << "Fake decompress - invalid data " << input_size << ", " << output_size << " with " << _so_far + << " / " << _size; + throw std::runtime_error(std::move(ss.str())); + } + bytes = std::min(input_size, output_size); + if (bytes) + { + memcpy(output, input, bytes); + } + _so_far += bytes; + _done = (_so_far == _size); + if (done) + { + *done = _done; + } + input_bytes_processed = bytes; + return input_bytes_processed; + } + + virtual pplx::task decompress( + const uint8_t* input, + size_t input_size, + uint8_t* output, + size_t output_size, + web::http::compression::operation_hint hint) + { + web::http::compression::operation_result r; + + try + { + r.output_bytes_produced = + decompress(input, input_size, output, output_size, hint, r.input_bytes_processed, &r.done); + } + catch (...) + { + pplx::task_completion_event ev; + ev.set_exception(std::current_exception()); + return pplx::create_task(ev); + } + + return pplx::task_from_result(r); + } + + virtual size_t compress(const uint8_t* input, + size_t input_size, + uint8_t* output, + size_t output_size, + web::http::compression::operation_hint hint, + size_t& input_bytes_processed, + bool* done) + { + size_t bytes; + + if (_done) + { + input_bytes_processed = 0; + if (*done) + { + *done = true; + } + return 0; + } + if (_size == static_cast(-1) || input_size > _size - _so_far) + { + std::stringstream ss; + ss << "Fake compress - invalid data " << input_size << ", " << output_size << " with " << _so_far + << " / " << _size; + throw std::runtime_error(std::move(ss.str())); + } + bytes = std::min(input_size, output_size); + if (bytes) + { + memcpy(output, input, bytes); + } + _so_far += bytes; + _done = (hint == web::http::compression::operation_hint::is_last && _so_far == _size); + if (done) + { + *done = _done; + } + input_bytes_processed = bytes; + return input_bytes_processed; + } + + virtual pplx::task compress( + const uint8_t* input, + size_t input_size, + uint8_t* output, + size_t output_size, + web::http::compression::operation_hint hint) + { + web::http::compression::operation_result r; + + try + { + r.output_bytes_produced = + compress(input, input_size, output, output_size, hint, r.input_bytes_processed, &r.done); + } + catch (...) + { + pplx::task_completion_event ev; + ev.set_exception(std::current_exception()); + return pplx::create_task(ev); + } + + return pplx::task_from_result(r); + } + + virtual void reset() + { + _done = false; + _so_far = 0; + } + + private: + size_t _size; + size_t _so_far; + bool _done; + }; + + const utility::string_t fake_provider::FAKE = _XPLATSTR("fake"); + + void compress_and_decompress( + const utility::string_t& algorithm, const size_t buffer_size, const size_t chunk_size, bool compressible) + { + std::unique_ptr compressor; + std::unique_ptr decompressor; + std::vector input_buffer; + std::vector cmp_buffer; + std::vector dcmp_buffer; + web::http::compression::operation_result r; + std::vector chunk_sizes; + Concurrency::task_group_status result; + size_t csize; + size_t dsize; + size_t i; + size_t nn; + + if (algorithm == fake_provider::FAKE) + { + compressor = std::make_unique(buffer_size); + decompressor = std::make_unique(buffer_size); + } + else + { + compressor = web::http::compression::builtin::make_compressor(algorithm); + decompressor = web::http::compression::builtin::make_decompressor(algorithm); + } + VERIFY_IS_TRUE((bool)compressor); + VERIFY_IS_TRUE((bool)decompressor); + + input_buffer.reserve(buffer_size); + for (size_t i = 0; i < buffer_size; ++i) + { + if (compressible) + { + input_buffer.push_back(static_cast('a' + i % 26)); + } + else + { + input_buffer.push_back(static_cast(std::rand())); + } + } + + // compress in chunks + csize = 0; + cmp_buffer.resize(buffer_size); // pessimistic (or not, for non-compressible data) + for (i = 0; i < buffer_size; i += chunk_size) + { + result = compressor + ->compress(input_buffer.data() + i, + std::min(chunk_size, buffer_size - i), + cmp_buffer.data() + csize, + std::min(chunk_size, buffer_size - csize), + web::http::compression::operation_hint::has_more) + .then([&r](web::http::compression::operation_result x) { r = x; }) + .wait(); + VERIFY_ARE_EQUAL(result, Concurrency::task_group_status::completed); + VERIFY_ARE_EQUAL(r.input_bytes_processed, std::min(chunk_size, buffer_size - i)); + VERIFY_ARE_EQUAL(r.done, false); + chunk_sizes.push_back(r.output_bytes_produced); + csize += r.output_bytes_produced; + } + if (i >= buffer_size) + { + size_t cmpsize = buffer_size; + do + { + if (csize == cmpsize) + { + // extend the output buffer if there may be more compressed bytes to retrieve + cmpsize += std::min(chunk_size, (size_t)200); + cmp_buffer.resize(cmpsize); + } + result = compressor + ->compress(NULL, + 0, + cmp_buffer.data() + csize, + std::min(chunk_size, cmpsize - csize), + web::http::compression::operation_hint::is_last) + .then([&r](web::http::compression::operation_result x) { r = x; }) + .wait(); + VERIFY_ARE_EQUAL(result, Concurrency::task_group_status::completed); + VERIFY_ARE_EQUAL(r.input_bytes_processed, 0); + chunk_sizes.push_back(r.output_bytes_produced); + csize += r.output_bytes_produced; + } while (csize == cmpsize); + VERIFY_ARE_EQUAL(r.done, true); + + // once more with no input, to assure no error and done + result = compressor->compress(NULL, 0, NULL, 0, web::http::compression::operation_hint::is_last) + .then([&r](web::http::compression::operation_result x) { r = x; }) + .wait(); + VERIFY_ARE_EQUAL(result, Concurrency::task_group_status::completed); + VERIFY_ARE_EQUAL(r.input_bytes_processed, 0); + VERIFY_ARE_EQUAL(r.output_bytes_produced, 0); + VERIFY_ARE_EQUAL(r.done, true); + } + cmp_buffer.resize(csize); // actual + + // decompress in as-compressed chunks + nn = 0; + dsize = 0; + dcmp_buffer.resize(buffer_size); + for (std::vector::iterator it = chunk_sizes.begin(); it != chunk_sizes.end(); ++it) + { + if (*it) + { + auto hint = web::http::compression::operation_hint::has_more; + if (it == chunk_sizes.begin()) + { + hint = web::http::compression::operation_hint::is_last; + } + result = decompressor + ->decompress(cmp_buffer.data() + nn, + *it, + dcmp_buffer.data() + dsize, + std::min(chunk_size, buffer_size - dsize), + hint) + .then([&r](web::http::compression::operation_result x) { r = x; }) + .wait(); + VERIFY_ARE_EQUAL(result, Concurrency::task_group_status::completed); + nn += *it; + dsize += r.output_bytes_produced; + } + } + VERIFY_ARE_EQUAL(csize, nn); + VERIFY_ARE_EQUAL(dsize, buffer_size); + VERIFY_ARE_EQUAL(input_buffer, dcmp_buffer); + VERIFY_IS_TRUE(r.done); + + // decompress again in fixed-size chunks + nn = 0; + dsize = 0; + decompressor->reset(); + memset(dcmp_buffer.data(), 0, dcmp_buffer.size()); + do + { + size_t n = std::min(chunk_size, csize - nn); + do + { + result = decompressor + ->decompress(cmp_buffer.data() + nn, + n, + dcmp_buffer.data() + dsize, + std::min(chunk_size, buffer_size - dsize), + web::http::compression::operation_hint::has_more) + .then([&r](web::http::compression::operation_result x) { r = x; }) + .wait(); + VERIFY_ARE_EQUAL(result, Concurrency::task_group_status::completed); + dsize += r.output_bytes_produced; + nn += r.input_bytes_processed; + n -= r.input_bytes_processed; + } while (n); + } while (nn < csize || !r.done); + VERIFY_ARE_EQUAL(csize, nn); + VERIFY_ARE_EQUAL(dsize, buffer_size); + VERIFY_ARE_EQUAL(input_buffer, dcmp_buffer); + VERIFY_IS_TRUE(r.done); + + // once more with no input, to assure no error and done + result = decompressor->decompress(NULL, 0, NULL, 0, web::http::compression::operation_hint::has_more) + .then([&r](web::http::compression::operation_result x) { r = x; }) + .wait(); + VERIFY_ARE_EQUAL(result, Concurrency::task_group_status::completed); + VERIFY_ARE_EQUAL(r.input_bytes_processed, 0); + VERIFY_ARE_EQUAL(r.output_bytes_produced, 0); + VERIFY_IS_TRUE(r.done); + + // decompress all at once + decompressor->reset(); + memset(dcmp_buffer.data(), 0, dcmp_buffer.size()); + result = decompressor + ->decompress(cmp_buffer.data(), + csize, + dcmp_buffer.data(), + dcmp_buffer.size(), + web::http::compression::operation_hint::is_last) + .then([&r](web::http::compression::operation_result x) { r = x; }) + .wait(); + VERIFY_ARE_EQUAL(result, Concurrency::task_group_status::completed); + VERIFY_ARE_EQUAL(r.output_bytes_produced, buffer_size); + VERIFY_ARE_EQUAL(input_buffer, dcmp_buffer); + + if (algorithm != fake_provider::FAKE) + { + // invalid decompress buffer, first and subsequent tries + cmp_buffer[0] = ~cmp_buffer[1]; + decompressor->reset(); + for (i = 0; i < 2; i++) + { + nn = 0; + try + { + result = decompressor + ->decompress(cmp_buffer.data(), + csize, + dcmp_buffer.data(), + dcmp_buffer.size(), + web::http::compression::operation_hint::is_last) + .then([&nn](web::http::compression::operation_result x) { nn++; }) + .wait(); + nn++; + } + catch (std::runtime_error) + { + } + VERIFY_ARE_EQUAL(nn, 0); + } + } + } + + void compress_test(const utility::string_t& algorithm) + { + size_t tuples[][2] = {{7999, 8192}, + {8192, 8192}, + {16001, 8192}, + {16384, 8192}, + {140000, 65536}, + {256 * 1024, 65536}, + {256 * 1024, 256 * 1024}, + {263456, 256 * 1024}}; + + for (int i = 0; i < sizeof(tuples) / sizeof(tuples[0]); i++) + { + for (int j = 0; j < 2; j++) + { + compress_and_decompress(algorithm, tuples[i][0], tuples[i][1], !!j); + } + } + } + + TEST_FIXTURE(uri_address, compress_and_decompress) + { + compress_test(fake_provider::FAKE); + if (web::http::compression::builtin::algorithm::supported(web::http::compression::builtin::algorithm::GZIP)) + { + compress_test(web::http::compression::builtin::algorithm::GZIP); + } + if (web::http::compression::builtin::algorithm::supported(web::http::compression::builtin::algorithm::DEFLATE)) + { + compress_test(web::http::compression::builtin::algorithm::DEFLATE); + } + if (web::http::compression::builtin::algorithm::supported(web::http::compression::builtin::algorithm::BROTLI)) + { + compress_test(web::http::compression::builtin::algorithm::BROTLI); + } + } + + TEST_FIXTURE(uri_address, compress_headers) + { + const utility::string_t _NONE = _XPLATSTR("none"); + + std::unique_ptr c; + std::unique_ptr d; + + std::shared_ptr fcf = web::http::compression::make_compress_factory( + fake_provider::FAKE, []() -> std::unique_ptr { + return std::make_unique(); + }); + std::vector> fcv; + fcv.push_back(fcf); + std::shared_ptr fdf = + web::http::compression::make_decompress_factory( + fake_provider::FAKE, 800, []() -> std::unique_ptr { + return std::make_unique(); + }); + std::vector> fdv; + fdv.push_back(fdf); + + std::shared_ptr ncf = web::http::compression::make_compress_factory( + _NONE, []() -> std::unique_ptr { + return std::make_unique(); + }); + std::vector> ncv; + ncv.push_back(ncf); + std::shared_ptr ndf = + web::http::compression::make_decompress_factory( + _NONE, 800, []() -> std::unique_ptr { + return std::make_unique(); + }); + std::vector> ndv; + ndv.push_back(ndf); + + // Supported algorithms + VERIFY_ARE_EQUAL( + web::http::compression::builtin::supported(), + web::http::compression::builtin::algorithm::supported(web::http::compression::builtin::algorithm::GZIP)); + VERIFY_ARE_EQUAL( + web::http::compression::builtin::supported(), + web::http::compression::builtin::algorithm::supported(web::http::compression::builtin::algorithm::DEFLATE)); + if (web::http::compression::builtin::algorithm::supported(web::http::compression::builtin::algorithm::BROTLI)) + { + VERIFY_IS_TRUE(web::http::compression::builtin::supported()); + } + VERIFY_IS_FALSE(web::http::compression::builtin::algorithm::supported(_XPLATSTR(""))); + VERIFY_IS_FALSE(web::http::compression::builtin::algorithm::supported(_XPLATSTR("foo"))); + + // Strings that double as both Transfer-Encoding and TE + std::vector encodings = {_XPLATSTR("gzip"), + _XPLATSTR("gZip "), + _XPLATSTR(" GZIP"), + _XPLATSTR(" gzip "), + _XPLATSTR(" gzip , chunked "), + _XPLATSTR(" gZip , chunked "), + _XPLATSTR("GZIP,chunked")}; + + // Similar, but geared to match a non-built-in algorithm + std::vector fake = {_XPLATSTR("fake"), + _XPLATSTR("faKe "), + _XPLATSTR(" FAKE"), + _XPLATSTR(" fake "), + _XPLATSTR(" fake , chunked "), + _XPLATSTR(" faKe , chunked "), + _XPLATSTR("FAKE,chunked")}; + + std::vector invalid = {_XPLATSTR(","), + _XPLATSTR(",gzip"), + _XPLATSTR("gzip,"), + _XPLATSTR(",gzip, chunked"), + _XPLATSTR(" ,gzip, chunked"), + _XPLATSTR("gzip, chunked,"), + _XPLATSTR("gzip, chunked, "), + _XPLATSTR("gzip,, chunked"), + _XPLATSTR("gzip , , chunked"), + _XPLATSTR("foo")}; + + std::vector invalid_tes = { + _XPLATSTR("deflate;q=0.5, gzip;q=2"), + _XPLATSTR("deflate;q=1.5, gzip;q=1"), + }; + + std::vector empty = {_XPLATSTR(""), _XPLATSTR(" ")}; + + // Repeat for Transfer-Encoding (which also covers part of TE) and Content-Encoding (which also covers all of + // Accept-Encoding) + for (int transfer = 0; transfer < 2; transfer++) + { + web::http::compression::details::header_types ctype = + transfer ? web::http::compression::details::header_types::te + : web::http::compression::details::header_types::accept_encoding; + web::http::compression::details::header_types dtype = + transfer ? web::http::compression::details::header_types::transfer_encoding + : web::http::compression::details::header_types::content_encoding; + + // No compression - Transfer-Encoding + d = web::http::compression::details::get_decompressor_from_header( + _XPLATSTR(" chunked "), web::http::compression::details::header_types::transfer_encoding); + VERIFY_IS_FALSE((bool)d); + + utility::string_t gzip(web::http::compression::builtin::algorithm::GZIP); + for (auto encoding = encodings.begin(); encoding != encodings.end(); encoding++) + { + bool has_comma = false; + + has_comma = encoding->find(_XPLATSTR(",")) != utility::string_t::npos; + + // Built-in only + c = web::http::compression::details::get_compressor_from_header(*encoding, ctype); + VERIFY_ARE_EQUAL((bool)c, web::http::compression::builtin::supported()); + if (c) + { + VERIFY_ARE_EQUAL(c->algorithm(), gzip); + } + + try + { + d = web::http::compression::details::get_decompressor_from_header(*encoding, dtype); + VERIFY_ARE_EQUAL((bool)d, web::http::compression::builtin::supported()); + if (d) + { + VERIFY_ARE_EQUAL(d->algorithm(), gzip); + } + } + catch (http_exception) + { + VERIFY_IS_TRUE(transfer == !has_comma); + } + } + + for (auto encoding = fake.begin(); encoding != fake.end(); encoding++) + { + bool has_comma = false; + + has_comma = encoding->find(_XPLATSTR(",")) != utility::string_t::npos; + + // Supplied compressor/decompressor + c = web::http::compression::details::get_compressor_from_header(*encoding, ctype, fcv); + VERIFY_IS_TRUE((bool)c); + VERIFY_IS_TRUE(c->algorithm() == fcf->algorithm()); + + try + { + d = web::http::compression::details::get_decompressor_from_header(*encoding, dtype, fdv); + VERIFY_IS_TRUE((bool)d); + VERIFY_IS_TRUE(d->algorithm() == fdf->algorithm()); + } + catch (http_exception) + { + VERIFY_IS_TRUE(transfer == !has_comma); + } + + // No matching compressor + c = web::http::compression::details::get_compressor_from_header(*encoding, ctype, ncv); + VERIFY_IS_FALSE((bool)c); + + try + { + d = web::http::compression::details::get_decompressor_from_header(*encoding, dtype, ndv); + VERIFY_IS_FALSE(true); + } + catch (http_exception) + { + } + } + + // Negative tests - invalid headers, no matching algorithm, etc. + for (auto encoding = invalid.begin(); encoding != invalid.end(); encoding++) + { + try + { + c = web::http::compression::details::get_compressor_from_header(*encoding, ctype); + VERIFY_IS_TRUE(encoding->find(_XPLATSTR(",")) == utility::string_t::npos); + VERIFY_IS_FALSE((bool)c); + } + catch (http_exception) + { + } + + try + { + d = web::http::compression::details::get_decompressor_from_header(*encoding, dtype); + VERIFY_IS_TRUE(!web::http::compression::builtin::supported() && + encoding->find(_XPLATSTR(",")) == utility::string_t::npos); + VERIFY_IS_FALSE((bool)d); + } + catch (http_exception) + { + } + } + + // Negative tests - empty headers + for (auto encoding = empty.begin(); encoding != empty.end(); encoding++) + { + c = web::http::compression::details::get_compressor_from_header(*encoding, ctype); + VERIFY_IS_FALSE((bool)c); + + try + { + d = web::http::compression::details::get_decompressor_from_header(*encoding, dtype); + VERIFY_IS_FALSE(true); + } + catch (http_exception) + { + } + } + + // Negative tests - invalid rankings + for (auto te = invalid_tes.begin(); te != invalid_tes.end(); te++) + { + try + { + c = web::http::compression::details::get_compressor_from_header(*te, ctype); + VERIFY_IS_FALSE(true); + } + catch (http_exception) + { + } + } + + utility::string_t builtin; + std::vector> dv; + + // Builtins + builtin = web::http::compression::details::build_supported_header(ctype); + if (transfer) + { + VERIFY_ARE_EQUAL(!builtin.empty(), web::http::compression::builtin::supported()); + } + else + { + VERIFY_IS_FALSE(builtin.empty()); + } + + // Null decompressor - effectively forces no compression algorithms + dv.push_back(std::shared_ptr()); + builtin = web::http::compression::details::build_supported_header(ctype, dv); + VERIFY_ARE_EQUAL((bool)transfer, builtin.empty()); + dv.pop_back(); + + if (web::http::compression::builtin::supported()) + { + dv.push_back(web::http::compression::builtin::get_decompress_factory( + web::http::compression::builtin::algorithm::GZIP)); + builtin = web::http::compression::details::build_supported_header(ctype, dv); // --> "gzip;q=1.0" + VERIFY_IS_FALSE(builtin.empty()); + } + else + { + builtin = _XPLATSTR("gzip;q=1.0"); + } + + // TE- and/or Accept-Encoding-specific test cases, regenerated for each pass + std::vector tes = { + builtin, + _XPLATSTR(" deflate;q=0.777 ,foo;q=0,gzip;q=0.9, bar;q=1.0, xxx;q=1 "), + _XPLATSTR("gzip ; q=1, deflate;q=0.5"), + _XPLATSTR("gzip;q=1.0, deflate;q=0.5"), + _XPLATSTR("deflate;q=0.5, gzip;q=1"), + _XPLATSTR("gzip,deflate;q=0.7"), + _XPLATSTR("trailers,gzip,deflate;q=0.7")}; + + for (int fake = 0; fake < 2; fake++) + { + if (fake) + { + // Switch built-in vs. supplied results the second time around + for (auto &te : tes) + { + te.replace(te.find(web::http::compression::builtin::algorithm::GZIP), + gzip.size(), + fake_provider::FAKE); + if (te.find(web::http::compression::builtin::algorithm::DEFLATE) != utility::string_t::npos) + { + te.replace(te.find(web::http::compression::builtin::algorithm::DEFLATE), + utility::string_t(web::http::compression::builtin::algorithm::DEFLATE).size(), + _NONE); + } + } + } + + for (auto te = tes.begin(); te != tes.end(); te++) + { + // Built-in only + c = web::http::compression::details::get_compressor_from_header(*te, ctype); + if (c) + { + VERIFY_IS_TRUE(web::http::compression::builtin::supported()); + VERIFY_IS_FALSE((bool)fake); + VERIFY_ARE_EQUAL(c->algorithm(), gzip); + } + else + { + VERIFY_IS_TRUE((bool)fake || !web::http::compression::builtin::supported()); + } + + // Supplied compressor - both matching and non-matching + c = web::http::compression::details::get_compressor_from_header(*te, ctype, fcv); + VERIFY_ARE_EQUAL((bool)c, (bool)fake); + if (c) + { + VERIFY_ARE_EQUAL(c->algorithm(), fake_provider::FAKE); + } + } + } + } + } + + template + class my_rawptr_buffer : public concurrency::streams::rawptr_buffer<_CharType> + { + public: + my_rawptr_buffer(const _CharType* data, size_t size) + : concurrency::streams::rawptr_buffer<_CharType>(data, size) + { + } + + // No acquire(), to force non-acquire compression client codepaths + virtual bool acquire(_Out_ _CharType*& ptr, _Out_ size_t& count) + { + ptr; + count; + return false; + } + + virtual void release(_Out_writes_(count) _CharType* ptr, _In_ size_t count) + { + ptr; + count; + } + + static concurrency::streams::basic_istream<_CharType> open_istream(const _CharType* data, size_t size) + { + return concurrency::streams::basic_istream<_CharType>( + concurrency::streams::streambuf<_CharType>(std::make_shared>(data, size))); + } + }; + + TEST_FIXTURE(uri_address, compress_client_server) + { + bool processed; + bool skip_transfer_put = false; + int transfer; + + size_t buffer_sizes[] = {0, 1, 3, 4, 4096, 65536, 100000, 157890}; + + std::vector> dfactories; + std::vector> cfactories; + +#if defined(_WIN32) && !defined(CPPREST_FORCE_HTTP_CLIENT_ASIO) + // Run a quick test to see if we're dealing with older/broken winhttp for compressed transfer encoding + { + test_http_server* p_server = nullptr; + std::unique_ptr scoped = + std::move(std::make_unique(m_uri)); + scoped->server()->next_request().then([&skip_transfer_put](pplx::task op) { + try + { + op.get()->reply(static_cast(status_codes::OK)); + } + catch (std::runtime_error) + { + // The test server throws if it's destructed with outstanding tasks, + // which will happen if winhttp responds 501 without informing us + VERIFY_IS_TRUE(skip_transfer_put); + } + }); + + http_client client(m_uri); + http_request msg(methods::PUT); + msg.set_compressor(std::make_unique(0)); + msg.set_body(concurrency::streams::rawptr_stream::open_istream((const uint8_t*)nullptr, 0)); + http_response rsp = client.request(msg).get(); + rsp.content_ready().wait(); + if (rsp.status_code() == status_codes::NotImplemented) + { + skip_transfer_put = true; + } + else + { + VERIFY_IS_TRUE(rsp.status_code() == status_codes::OK); + } + } +#endif // _WIN32 + + // Test decompression both explicitly through the test server and implicitly through the listener; + // this is the top-level loop in order to avoid thrashing the listeners more than necessary + for (int real = 0; real < 2; real++) + { + web::http::experimental::listener::http_listener listener; + std::unique_ptr scoped; + test_http_server* p_server = nullptr; + std::vector v; + size_t buffer_size; + + // Start the listener, and configure callbacks if necessary + if (real) + { + listener = std::move(web::http::experimental::listener::http_listener(m_uri)); + listener.open().wait(); + listener.support(methods::PUT, [&v, &dfactories, &processed](http_request request) { + utility::string_t encoding; + http_response rsp; + + if (request.headers().match(web::http::header_names::transfer_encoding, encoding) || + request.headers().match(web::http::header_names::content_encoding, encoding)) + { + if (encoding.find(fake_provider::FAKE) != utility::string_t::npos) + { + // This one won't be found by the server in the default set... + rsp._get_impl()->set_decompress_factories(dfactories); + } + } + processed = true; + rsp.set_status_code(status_codes::OK); + request.reply(rsp); + }); + listener.support( + methods::GET, [&v, &buffer_size, &cfactories, &processed, &transfer](http_request request) { + utility::string_t encoding; + http_response rsp; + bool done; + + if (transfer) + { +#if defined(_WIN32) && !defined(__cplusplus_winrt) && !defined(CPPREST_FORCE_HTTP_CLIENT_ASIO) + // Compression happens in the listener itself + done = request.headers().match(web::http::header_names::te, encoding); + VERIFY_IS_TRUE(done); + if (encoding.find(fake_provider::FAKE) != utility::string_t::npos) + { + // This one won't be found in the server's default set... + rsp._get_impl()->set_compressor(std::make_unique(buffer_size)); + } +#endif // _WIN32 + rsp.set_body( + concurrency::streams::rawptr_stream::open_istream(v.data(), v.size())); + } + else + { + std::unique_ptr c; + std::vector pre; + size_t used; + + done = request.headers().match(web::http::header_names::accept_encoding, encoding); + VERIFY_IS_TRUE(done); + pre.resize(v.size() + 128); + c = web::http::compression::details::get_compressor_from_header( + encoding, web::http::compression::details::header_types::accept_encoding, cfactories); + VERIFY_IS_TRUE((bool)c); + auto got = c->compress(v.data(), + v.size(), + pre.data(), + pre.size(), + web::http::compression::operation_hint::is_last, + used, + &done); + VERIFY_IS_TRUE(used == v.size()); + VERIFY_IS_TRUE(done); + + // Add a single pre-compressed stream, since Content-Encoding requires Content-Length + pre.resize(got); + rsp.headers().add(header_names::content_encoding, c->algorithm()); + rsp.set_body( + concurrency::streams::container_stream>::open_istream(pre)); + } + processed = true; + rsp.set_status_code(status_codes::OK); + request.reply(rsp); + }); + } + else + { + scoped = std::move(std::make_unique(m_uri)); + p_server = scoped->server(); + } + + // Test various buffer sizes + for (int sz = 0; sz < sizeof(buffer_sizes) / sizeof(buffer_sizes[0]); sz++) + { + std::vector algorithms; + std::map> dmap; + + buffer_size = buffer_sizes[sz]; + + dfactories.clear(); + cfactories.clear(); + + // Re-build the sets of compress and decompress factories, to account for the buffer size in our "fake" + // ones + if (web::http::compression::builtin::algorithm::supported( + web::http::compression::builtin::algorithm::GZIP)) + { + algorithms.push_back(web::http::compression::builtin::algorithm::GZIP); + dmap[web::http::compression::builtin::algorithm::GZIP] = + web::http::compression::builtin::get_decompress_factory( + web::http::compression::builtin::algorithm::GZIP); + dfactories.push_back(dmap[web::http::compression::builtin::algorithm::GZIP]); + cfactories.push_back(web::http::compression::builtin::get_compress_factory( + web::http::compression::builtin::algorithm::GZIP)); + } + if (web::http::compression::builtin::algorithm::supported( + web::http::compression::builtin::algorithm::DEFLATE)) + { + algorithms.push_back(web::http::compression::builtin::algorithm::DEFLATE); + dmap[web::http::compression::builtin::algorithm::DEFLATE] = + web::http::compression::builtin::get_decompress_factory( + web::http::compression::builtin::algorithm::DEFLATE); + dfactories.push_back(dmap[web::http::compression::builtin::algorithm::DEFLATE]); + cfactories.push_back(web::http::compression::builtin::get_compress_factory( + web::http::compression::builtin::algorithm::DEFLATE)); + } + if (web::http::compression::builtin::algorithm::supported( + web::http::compression::builtin::algorithm::BROTLI)) + { + algorithms.push_back(web::http::compression::builtin::algorithm::BROTLI); + dmap[web::http::compression::builtin::algorithm::BROTLI] = + web::http::compression::builtin::get_decompress_factory( + web::http::compression::builtin::algorithm::BROTLI); + dfactories.push_back(dmap[web::http::compression::builtin::algorithm::BROTLI]); + cfactories.push_back(web::http::compression::builtin::get_compress_factory( + web::http::compression::builtin::algorithm::BROTLI)); + } + algorithms.push_back(fake_provider::FAKE); + dmap[fake_provider::FAKE] = web::http::compression::make_decompress_factory( + fake_provider::FAKE, + 1000, + [buffer_size]() -> std::unique_ptr { + return std::make_unique(buffer_size); + }); + dfactories.push_back(dmap[fake_provider::FAKE]); + cfactories.push_back(web::http::compression::make_compress_factory( + fake_provider::FAKE, [buffer_size]() -> std::unique_ptr { + return std::make_unique(buffer_size); + })); + + v.resize(buffer_size); + + // Test compressible (net shrinking) and non-compressible (net growing) buffers + for (int compressible = 0; compressible < 2; compressible++) + { + for (size_t x = 0; x < buffer_size; x++) + { + if (compressible) + { + v[x] = static_cast('a' + x % 26); + } + else + { + v[x] = static_cast(std::rand()); + } + } + + // Test both Transfer-Encoding and Content-Encoding + for (transfer = 0; transfer < 2; transfer++) + { + web::http::client::http_client_config config; + config.set_request_compressed_response(!transfer); + http_client client(m_uri, config); + + // Test supported compression algorithms + for (auto& algorithm : algorithms) + { + // Test both GET and PUT + for (int put = 0; put < 2; put++) + { + if (transfer && put && skip_transfer_put) + { + continue; + } + + processed = false; + + if (put) + { + std::vector streams; + std::vector pre; + + if (transfer) + { + // Add a pair of non-compressed streams for Transfer-Encoding, one with and one + // without acquire/release support + streams.emplace_back(concurrency::streams::rawptr_stream::open_istream( + (const uint8_t*)v.data(), v.size())); + streams.emplace_back( + my_rawptr_buffer::open_istream(v.data(), v.size())); + } + else + { + bool done; + size_t used; + pre.resize(v.size() + 128); + + auto c = web::http::compression::builtin::make_compressor(algorithm); + if (algorithm == fake_provider::FAKE) + { + VERIFY_IS_FALSE((bool)c); + c = std::make_unique(buffer_size); + } + VERIFY_IS_TRUE((bool)c); + auto got = c->compress(v.data(), + v.size(), + pre.data(), + pre.size(), + web::http::compression::operation_hint::is_last, + used, + &done); + VERIFY_ARE_EQUAL(used, v.size()); + VERIFY_IS_TRUE(done); + + // Add a single pre-compressed stream, since Content-Encoding requires + // Content-Length + streams.emplace_back(concurrency::streams::rawptr_stream::open_istream( + pre.data(), got)); + } + + for (auto &stream : streams) + { + http_request msg(methods::PUT); + + processed = false; + + msg.set_body(stream); + if (transfer) + { + bool boo = msg.set_compressor(algorithm); + VERIFY_ARE_EQUAL(boo, algorithm != fake_provider::FAKE); + if (algorithm == fake_provider::FAKE) + { + msg.set_compressor(std::make_unique(buffer_size)); + } + } + else + { + msg.headers().add(header_names::content_encoding, algorithm); + } + + if (!real) + { + // We implement the decompression path in the server, to prove that valid, + // compressed data is sent + p_server->next_request().then([&](test_request* p_request) { + std::unique_ptr d; + std::vector vv; + utility::string_t header; + size_t used; + size_t got; + bool done; + + http_asserts::assert_test_request_equals( + p_request, methods::PUT, U("/")); + + if (transfer) + { + VERIFY_IS_FALSE(p_request->match_header( + header_names::content_encoding, header)); + done = p_request->match_header(header_names::transfer_encoding, + header); + VERIFY_IS_TRUE(done); + d = web::http::compression::details::get_decompressor_from_header( + header, + web::http::compression::details::header_types:: + transfer_encoding, + dfactories); + } + else + { + done = p_request->match_header(header_names::transfer_encoding, + header); + if (done) + { + VERIFY_IS_TRUE( + utility::details::str_iequal(_XPLATSTR("chunked"), header)); + } + done = + p_request->match_header(header_names::content_encoding, header); + VERIFY_IS_TRUE(done); + d = web::http::compression::details::get_decompressor_from_header( + header, + web::http::compression::details::header_types::content_encoding, + dfactories); + } +#if defined(_WIN32) && !defined(__cplusplus_winrt) && !defined(CPPREST_FORCE_HTTP_CLIENT_ASIO) + VERIFY_IS_TRUE((bool)d); +#else // _WIN32 + VERIFY_ARE_NOT_EQUAL((bool)d, !!transfer); +#endif // _WIN32 + + vv.resize(buffer_size + 128); + if (d) + { + got = d->decompress(p_request->m_body.data(), + p_request->m_body.size(), + vv.data(), + vv.size(), + web::http::compression::operation_hint::is_last, + used, + &done); + VERIFY_ARE_EQUAL(used, p_request->m_body.size()); + VERIFY_IS_TRUE(done); + } + else + { + memcpy(vv.data(), v.data(), v.size()); + got = v.size(); + } + VERIFY_ARE_EQUAL(buffer_size, got); + vv.resize(buffer_size); + VERIFY_ARE_EQUAL(v, vv); + processed = true; + + p_request->reply(static_cast(status_codes::OK)); + }); + } + + // Send the request + http_response rsp = client.request(msg).get(); + VERIFY_ARE_EQUAL(rsp.status_code(), status_codes::OK); + rsp.content_ready().wait(); + stream.close().wait(); + VERIFY_IS_TRUE(processed); + } + } + else + { + std::vector vv; + concurrency::streams::ostream stream = + concurrency::streams::rawptr_stream::open_ostream(vv.data(), + buffer_size); + http_request msg(methods::GET); + + std::vector> df = { + dmap[algorithm]}; + msg.set_decompress_factories(df); + + vv.resize(buffer_size + 128); // extra to ensure no overflow + + concurrency::streams::rawptr_buffer buf( + vv.data(), vv.size(), std::ios::out); + + if (!real) + { + p_server->next_request().then([&](test_request* p_request) { + std::map headers; + std::unique_ptr c; + utility::string_t header; + std::vector cmp; + size_t used; + size_t extra = 0; + size_t skip = 0; + size_t got; + bool done; + + std::string ext = ";x=y"; + std::string trailer = "a=b\r\nx=y\r\n"; + + http_asserts::assert_test_request_equals(p_request, methods::GET, U("/")); + + if (transfer) + { + // On Windows, someone along the way adds "Accept-Encoding: peerdist", + // so we can't unconditionally assert that Accept-Encoding is not + // present + done = p_request->match_header(header_names::accept_encoding, header); + VERIFY_IS_TRUE(!done || + header.find(algorithm) == utility::string_t::npos); + done = p_request->match_header(header_names::te, header); + if (done) + { + c = web::http::compression::details::get_compressor_from_header( + header, + web::http::compression::details::header_types::te, + cfactories); + } + + // Account for space for the chunk header and delimiters, plus a chunk + // extension and a chunked trailer part + extra = 2 * web::http::details::chunked_encoding:: + additional_encoding_space + + ext.size() + trailer.size(); + skip = web::http::details::chunked_encoding::data_offset + ext.size(); + } + else + { + VERIFY_IS_FALSE(p_request->match_header(header_names::te, header)); + done = p_request->match_header(header_names::accept_encoding, header); + VERIFY_IS_TRUE(done); + c = web::http::compression::details::get_compressor_from_header( + header, + web::http::compression::details::header_types::accept_encoding, + cfactories); + } +#if !defined __cplusplus_winrt + VERIFY_IS_TRUE((bool)c); +#else // __cplusplus_winrt + VERIFY_ARE_NOT_EQUAL((bool)c, !!transfer); +#endif // __cplusplus_winrt + cmp.resize(extra + buffer_size + 128); + if (c) + { + got = c->compress(v.data(), + v.size(), + cmp.data() + skip, + cmp.size() - extra, + web::http::compression::operation_hint::is_last, + used, + &done); + VERIFY_ARE_EQUAL(used, v.size()); + VERIFY_IS_TRUE(done); + } + else + { + memcpy(cmp.data() + skip, v.data(), v.size()); + got = v.size(); + } + if (transfer) + { + // Add delimiters for the first (and only) data chunk, plus the final + // 0-length chunk, and hack in a dummy chunk extension and a dummy + // trailer part. Note that we put *two* "0\r\n" in here in the 0-length + // case... and none of the parsers complain. + size_t total = + got + + web::http::details::chunked_encoding::additional_encoding_space + + ext.size(); + _ASSERTE(ext.size() >= 2); + if (got > ext.size() - 1) + { + cmp[total - 2] = + cmp[got + web::http::details::chunked_encoding::data_offset]; + } + if (got > ext.size() - 2) + { + cmp[total - 1] = + cmp[got + web::http::details::chunked_encoding::data_offset + + 1]; + } + size_t offset = + web::http::details::chunked_encoding::add_chunked_delimiters( + cmp.data(), total, got); + size_t offset2 = + web::http::details::chunked_encoding::add_chunked_delimiters( + cmp.data() + total - 7, + web::http::details::chunked_encoding::additional_encoding_space, + 0); + _ASSERTE( + offset2 == 7 && + web::http::details::chunked_encoding::additional_encoding_space - + 7 == + 5); + memcpy(cmp.data() + web::http::details::chunked_encoding::data_offset - + 2, + ext.data(), + ext.size()); + cmp[web::http::details::chunked_encoding::data_offset + ext.size() - + 2] = '\r'; + cmp[web::http::details::chunked_encoding::data_offset + ext.size() - + 1] = '\n'; + if (got > ext.size() - 1) + { + cmp[got + web::http::details::chunked_encoding::data_offset] = + cmp[total - 2]; + } + if (got > ext.size() - 2) + { + cmp[got + web::http::details::chunked_encoding::data_offset + 1] = + cmp[total - 1]; + } + cmp[total - 2] = '\r'; + cmp[total - 1] = '\n'; + memcpy(cmp.data() + total + 3, trailer.data(), trailer.size()); + cmp[total + trailer.size() + 3] = '\r'; + cmp[total + trailer.size() + 4] = '\n'; + cmp.erase(cmp.begin(), cmp.begin() + offset); + cmp.resize( + ext.size() + got + trailer.size() + + web::http::details::chunked_encoding::additional_encoding_space - + offset + 5); + if (c) + { + headers[header_names::transfer_encoding] = + c->algorithm() + _XPLATSTR(", chunked"); + } + else + { + headers[header_names::transfer_encoding] = _XPLATSTR("chunked"); + } + } + else + { + cmp.resize(got); + headers[header_names::content_encoding] = c->algorithm(); + } + processed = true; + + if (cmp.size()) + { + p_request->reply(static_cast(status_codes::OK), + utility::string_t(), + headers, + cmp); + } + else + { + p_request->reply(static_cast(status_codes::OK), + utility::string_t(), + headers); + } + }); + } + + // Common send and response processing code + http_response rsp = client.request(msg).get(); + VERIFY_ARE_EQUAL(rsp.status_code(), status_codes::OK); + VERIFY_NO_THROWS(rsp.content_ready().wait()); + + if (transfer) + { + VERIFY_IS_TRUE(rsp.headers().has(header_names::transfer_encoding)); + VERIFY_IS_FALSE(rsp.headers().has(header_names::content_encoding)); + } + else + { + utility::string_t header; + + VERIFY_IS_TRUE(rsp.headers().has(header_names::content_encoding)); + bool boo = rsp.headers().match(header_names::transfer_encoding, header); + if (boo) + { + VERIFY_IS_TRUE(utility::details::str_iequal(_XPLATSTR("chunked"), header)); + } + } + + size_t offset; + VERIFY_NO_THROWS(offset = rsp.body().read_to_end(buf).get()); + VERIFY_ARE_EQUAL(offset, buffer_size); + VERIFY_ARE_EQUAL(offset, static_cast(buf.getpos(std::ios::out))); + vv.resize(buffer_size); + VERIFY_ARE_EQUAL(v, vv); + buf.close(std::ios_base::out).wait(); + stream.close().wait(); + } + VERIFY_IS_TRUE(processed); + } + } + } + } + } + if (real) + { + listener.close().wait(); + } + } + } +} // SUITE(request_helper_tests) +} +} +} +} diff --git a/Release/tests/functional/http/client/outside_tests.cpp b/Release/tests/functional/http/client/outside_tests.cpp index 7c0438d48c..bec9b0c458 100644 --- a/Release/tests/functional/http/client/outside_tests.cpp +++ b/Release/tests/functional/http/client/outside_tests.cpp @@ -58,7 +58,7 @@ TEST_FIXTURE(uri_address, outside_cnn_dot_com) TEST_FIXTURE(uri_address, outside_wikipedia_compressed_http_response) { - if (web::http::details::compression::stream_decompressor::is_supported() == false) + if (web::http::compression::builtin::supported() == false) { // On platforms which do not support compressed http, nothing to check. return; diff --git a/Release/tests/functional/http/client/request_helper_tests.cpp b/Release/tests/functional/http/client/request_helper_tests.cpp index 32a49490ef..ee4ff762b6 100644 --- a/Release/tests/functional/http/client/request_helper_tests.cpp +++ b/Release/tests/functional/http/client/request_helper_tests.cpp @@ -45,7 +45,7 @@ TEST_FIXTURE(uri_address, do_not_fail_on_content_encoding_when_not_requested) TEST_FIXTURE(uri_address, fail_on_content_encoding_if_unsupported) { - if (web::http::details::compression::stream_compressor::is_supported()) + if (web::http::compression::builtin::supported()) { test_http_server::scoped_server scoped(m_uri); auto& server = *scoped.server(); @@ -63,7 +63,7 @@ TEST_FIXTURE(uri_address, fail_on_content_encoding_if_unsupported) TEST_FIXTURE(uri_address, send_accept_encoding) { - if (web::http::details::compression::stream_compressor::is_supported()) + if (web::http::compression::builtin::supported()) { test_http_server::scoped_server scoped(m_uri); auto& server = *scoped.server(); @@ -93,7 +93,11 @@ TEST_FIXTURE(uri_address, do_not_send_accept_encoding) std::atomic found_accept_encoding(true); server.next_request().then([&found_accept_encoding](test_request *p_request) { - found_accept_encoding = p_request->m_headers.find(header_names::accept_encoding) != p_request->m_headers.end(); + utility::string_t header; + + // On Windows, someone along the way (not us!) adds "Accept-Encoding: peerdist" + found_accept_encoding = + p_request->match_header(header_names::accept_encoding, header) && header != _XPLATSTR("peerdist"); p_request->reply(200, U("OK")); }); @@ -102,55 +106,6 @@ TEST_FIXTURE(uri_address, do_not_send_accept_encoding) VERIFY_IS_FALSE(found_accept_encoding); } -TEST_FIXTURE(uri_address, compress_and_decompress) -{ - if (web::http::details::compression::stream_compressor::is_supported()) - { - auto compress_and_decompress = [](web::http::details::compression::compression_algorithm alg) - { - auto compressor = std::make_shared(alg); - auto decompressor = std::make_shared(alg); - - const size_t buffer_size = 100; - const size_t split_pos = buffer_size / 2; - - web::http::details::compression::data_buffer input_buffer; - input_buffer.reserve(buffer_size); - - for (size_t i = 0; i < buffer_size; ++i) - { - input_buffer.push_back(static_cast(i)); - } - - web::http::details::compression::data_buffer buffer1(input_buffer.begin(), input_buffer.begin() + split_pos); - web::http::details::compression::data_buffer buffer2(input_buffer.begin() + split_pos, input_buffer.end()); - - auto compressed_data1 = compressor->compress(buffer1, false); - VERIFY_IS_FALSE(compressed_data1.empty()); - VERIFY_IS_FALSE(compressor->has_error()); - - auto compressed_data2 = compressor->compress(buffer2, true); - VERIFY_IS_FALSE(compressed_data2.empty()); - VERIFY_IS_FALSE(compressor->has_error()); - - auto decompressed_data1 = decompressor->decompress(compressed_data1); - VERIFY_IS_FALSE(decompressed_data1.empty()); - VERIFY_IS_FALSE(decompressor->has_error()); - - auto decompressed_data2 = decompressor->decompress(compressed_data2); - VERIFY_IS_FALSE(decompressed_data2.empty()); - VERIFY_IS_FALSE(decompressor->has_error()); - - decompressed_data1.insert(decompressed_data1.end(), decompressed_data2.begin(), decompressed_data2.end()); - - VERIFY_ARE_EQUAL(input_buffer, decompressed_data1); - }; - - compress_and_decompress(web::http::details::compression::compression_algorithm::gzip); - compress_and_decompress(web::http::details::compression::compression_algorithm::deflate); - } -} - TEST_FIXTURE(uri_address, non_rvalue_bodies) { test_http_server::scoped_server scoped(m_uri); diff --git a/Release/tests/functional/http/utilities/include/test_http_server.h b/Release/tests/functional/http/utilities/include/test_http_server.h index 84eb27d4ab..8220f0b331 100644 --- a/Release/tests/functional/http/utilities/include/test_http_server.h +++ b/Release/tests/functional/http/utilities/include/test_http_server.h @@ -46,6 +46,15 @@ class test_request return reply_impl(status_code, reason_phrase, headers, (void *)&data[0], data.size() * sizeof(utf8char)); } + unsigned long reply( + const unsigned short status_code, + const utility::string_t &reason_phrase, + const std::map &headers, + const std::vector &data) + { + return reply_impl(status_code, reason_phrase, headers, (void *)&data[0], data.size()); + } + unsigned long reply( const unsigned short status_code, const utility::string_t &reason_phrase, @@ -60,20 +69,12 @@ class test_request bool match_header(const utility::string_t & header_name, T & header_value) { auto iter = m_headers.find(header_name); - if (iter != m_headers.end()) - { - utility::istringstream_t iss(iter->second); - iss >> header_value; - if (iss.fail() || !iss.eof()) - { - return false; - } - return true; - } - else - { - return false; - } + if (iter == m_headers.end()) + { + return false; + } + + return web::http::details::bind_impl(iter->second, header_value) || iter->second.empty(); } // Request data. diff --git a/Release/tests/functional/http/utilities/test_http_server.cpp b/Release/tests/functional/http/utilities/test_http_server.cpp index cd359b1592..c4f848c009 100644 --- a/Release/tests/functional/http/utilities/test_http_server.cpp +++ b/Release/tests/functional/http/utilities/test_http_server.cpp @@ -160,7 +160,7 @@ static utility::string_t HttpServerAPIKnownHeaders[] = U("Proxy-Authorization"), U("Referer"), U("Range"), - U("Te"), + U("TE"), U("Translate"), U("User-Agent"), U("Request-Maximum"), @@ -345,7 +345,7 @@ class _test_http_server utility::string_t transfer_encoding; const bool has_transfer_encoding = p_test_request->match_header(U("Transfer-Encoding"), transfer_encoding); - if (has_transfer_encoding && transfer_encoding == U("chunked")) + if (has_transfer_encoding && transfer_encoding.find(U("chunked")) != std::string::npos) { content_length = 0; char buf[4096];