Added support for brotli response body compression

CMakeLists.txt

@@ -37,6 +37,7 @@ endif()
 ###############
 option(ENABLE_APIDB "Enable APIDB backend, as used by the OSM servers" ON)
 option(ENABLE_YAJL "Enable JSON output with the YAJL library" ON)
+option(ENABLE_BROTLI "Enable Brotli library" ON)
 option(ENABLE_FMT_HEADER "Enable FMT header only mode" ON)
 option(ENABLE_COVERAGE "Compile with coverage info collection" OFF)
 option(ENABLE_PROFILING "Compile with profiling" OFF)
@@ -102,6 +103,12 @@ endif()
 target_compile_definitions(cgimap_common_compiler_options INTERFACE
     HAVE_YAJL=$<BOOL:${YAJL_FOUND}>)
 
+if(ENABLE_BROTLI)
+    find_package(Brotli COMPONENTS encoder decoder common REQUIRED)
+endif()
+target_compile_definitions(cgimap_common_compiler_options INTERFACE
+    HAVE_BROTLI=$<BOOL:${Brotli_FOUND}>)
+
 find_package(Fcgi REQUIRED)
 target_compile_definitions(cgimap_common_compiler_options INTERFACE
     HAVE_FCGI=$<BOOL:${Fcgi_FOUND}>)

README

@@ -33,7 +33,7 @@ CGImap uses a PostgreSQL server for the APIDB backend.
 If you're running a Debian or Ubuntu system these can be installed
 using the following command:
 
-    sudo apt-get install libxml2-dev libpqxx-dev libfcgi-dev zlib1g-dev \
+    sudo apt-get install libxml2-dev libpqxx-dev libfcgi-dev zlib1g-dev libbrotli-dev \
       libboost-program-options-dev libfmt-dev libmemcached-dev libcrypto++-dev \
       libargon2-dev libyajl-dev
 

cmake/FindBrotli.cmake

@@ -0,0 +1,168 @@
+# A simple FindBrotli package for Cmake's find_package function.
+# Note: This find package doesn't have version support, as the version file doesn't seem to be installed on most systems.
+#
+# If you want to find the static packages instead of shared (the default), define BROTLI_USE_STATIC_LIBS as TRUE.
+# The targets will have the same names, but it will use the static libs.
+#
+# Valid find_package COMPONENTS names: "decoder", "encoder", and "common"
+# Note that if you're requiring "decoder" or "encoder", then "common" will be automatically added as required.
+#
+# Defines the libraries (if found): Brotli::decoder, Brotli::encoder, Brotli::common
+# and the includes path variable: Brotli_INCLUDE_DIR
+#
+# If it's failing to find the libraries, try setting BROTLI_ROOT_DIR to the folder containing your library & include dir.
+
+# If they asked for a specific version, warn/fail since we don't support it.
+# TODO: if they start distributing the version somewhere, implement finding it.
+# See https://github.com/google/brotli/issues/773#issuecomment-579133187
+if(Brotli_FIND_VERSION)
+	set(_brotli_version_error_msg "FindBrotli.cmake doesn't have version support!")
+	# If the package is required, throw a fatal error
+	# Otherwise, if not running quietly, we throw a warning
+	if(Brotli_FIND_REQUIRED)
+		message(FATAL_ERROR "${_brotli_version_error_msg}")
+	elseif(NOT Brotli_FIND_QUIETLY)
+		message(WARNING "${_brotli_version_error_msg}")
+	endif()
+endif()
+
+# Since both decoder & encoder require the common lib, force its requirement..
+# if the user is requiring either of those other libs.
+if(Brotli_FIND_REQUIRED_decoder OR Brotli_FIND_REQUIRED_encoder)
+	set(Brotli_FIND_REQUIRED_common TRUE)
+endif()
+
+# Support preference of static libs by adjusting CMAKE_FIND_LIBRARY_SUFFIXES
+# Credit to FindOpenSSL.cmake for this
+if(BROTLI_USE_STATIC_LIBS)
+  set(_brotli_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
+  if(WIN32)
+    set(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a ${CMAKE_FIND_LIBRARY_SUFFIXES})
+  else()
+    set(CMAKE_FIND_LIBRARY_SUFFIXES .a)
+  endif()
+endif()
+
+# Make PkgConfig optional, since some users (mainly Windows) don't have it.
+# But it's a lot more clean than manually using find_library.
+find_package(PkgConfig QUIET)
+
+# Only used if the PkgConfig libraries aren't used.
+find_path(Brotli_INCLUDE_DIR
+	NAMES
+		"brotli/decode.h"
+		"brotli/encode.h"
+	HINTS
+		${BROTLI_ROOT_DIR}
+	PATH_SUFFIXES
+		"include"
+		"includes"
+	DOC "The path to Brotli's include directory."
+)
+# Hides this var from the GUI
+mark_as_advanced(Brotli_INCLUDE_DIR)
+
+# Just used for PkgConfig stuff in the loop below
+set(_brotli_stat_str "")
+if(BROTLI_USE_STATIC_LIBS)
+	set(_brotli_stat_str "_STATIC")
+endif()
+
+# Each string here is "ComponentName;LiteralName" (the semi-colon is a delimiter)
+foreach(_listvar "common;common" "decoder;dec" "encoder;enc")
+	# Split the component name and literal library name from the listvar
+	list(GET _listvar 0 _component_name)
+	list(GET _listvar 1 _libname)
+
+	# NOTE: We can't rely on PkgConf for static libs since the upstream static lib support is broken
+	# See https://github.com/google/brotli/issues/795
+	# TODO: whenever their issue is fixed upstream, remove this "AND NOT BROTLI_USE_STATIC_LIBS" check
+	if(PKG_CONFIG_FOUND AND NOT BROTLI_USE_STATIC_LIBS)
+		# These need to be GLOBAL for MinGW when making ALIAS libraries against them.
+		# Have to postfix _STATIC on the name to tell PkgConfig to find the static libs.
+		pkg_check_modules(Brotli_${_component_name}${_brotli_stat_str} QUIET GLOBAL IMPORTED_TARGET libbrotli${_libname})
+	endif()
+
+	# Check if the target was already found by Pkgconf
+	if(TARGET PkgConfig::Brotli_${_component_name}${_brotli_stat_str})
+		# ALIAS since we don't want the PkgConfig namespace on the Cmake library (for end-users)
+		add_library(Brotli::${_component_name} ALIAS PkgConfig::Brotli_${_component_name}${_brotli_stat_str})
+
+		# Tells HANDLE_COMPONENTS we found the component
+		set(Brotli_${_component_name}_FOUND TRUE)
+		if(Brotli_FIND_REQUIRED_${_component_name})
+			# If the lib is required, we can add its literal path as a required var for FindPackageHandleStandardArgs
+			# Since it won't accept the PkgConfig targets
+			if(BROTLI_USE_STATIC_LIBS)
+				list(APPEND _brotli_req_vars "Brotli_${_component_name}_STATIC_LIBRARIES")
+			else()
+				list(APPEND _brotli_req_vars "Brotli_${_component_name}_LINK_LIBRARIES")
+			endif()
+		endif()
+
+		# Skip searching for the libs with find_library since it was already found by Pkgconf
+		continue()
+	endif()
+
+	if(Brotli_FIND_REQUIRED_${_component_name})
+		# If it's required, we can set the name used in find_library as a required var for FindPackageHandleStandardArgs
+		list(APPEND _brotli_req_vars "Brotli_${_component_name}")
+	endif()
+
+	list(APPEND _brotli_lib_names
+		"brotli${_libname}"
+		"libbrotli${_libname}"
+	)
+	if(BROTLI_USE_STATIC_LIBS)
+		# Postfix "-static" to the libnames since we're looking for static libs
+		list(TRANSFORM _brotli_lib_names APPEND "-static")
+	endif()
+
+	find_library(Brotli_${_component_name}
+		NAMES ${_brotli_lib_names}
+		HINTS ${BROTLI_ROOT_DIR}
+		PATH_SUFFIXES
+			"lib"
+			"lib64"
+			"libs"
+			"libs64"
+			"lib/x86_64-linux-gnu"
+	)
+	# Hide the library variable from the Cmake GUI
+	mark_as_advanced(Brotli_${_component_name})
+
+	# Unset since otherwise it'll stick around for the next loop and break things
+	unset(_brotli_lib_names)
+
+	# Check if find_library found the library
+	if(Brotli_${_component_name})
+		# Tells HANDLE_COMPONENTS we found the component
+		set(Brotli_${_component_name}_FOUND TRUE)
+
+		add_library("Brotli::${_component_name}" UNKNOWN IMPORTED)
+		# Attach the literal library and include dir to the IMPORTED target for the end-user
+		set_target_properties("Brotli::${_component_name}" PROPERTIES
+			INTERFACE_INCLUDE_DIRECTORIES "${Brotli_INCLUDE_DIR}"
+			IMPORTED_LOCATION "${Brotli_${_component_name}}"
+		)
+	else()
+		# Tells HANDLE_COMPONENTS we found the component
+		set(Brotli_${_component_name}_FOUND FALSE)
+	endif()
+endforeach()
+
+include(FindPackageHandleStandardArgs)
+# Sets Brotli_FOUND, and fails the find_package(Brotli) call if it was REQUIRED but missing libs.
+find_package_handle_standard_args(Brotli
+	FOUND_VAR
+		Brotli_FOUND
+	REQUIRED_VARS
+		Brotli_INCLUDE_DIR
+		${_brotli_req_vars}
+	HANDLE_COMPONENTS
+)
+
+# Restore the original find library ordering
+if(BROTLI_USE_STATIC_LIBS)
+  set(CMAKE_FIND_LIBRARY_SUFFIXES ${_brotli_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})
+endif()

docker/debian/Dockerfile_bookworm

@@ -5,7 +5,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update -qq && \
     apt-get install -y gcc g++ make cmake \
-       libfcgi-dev libxml2-dev libmemcached-dev \
+       libfcgi-dev libxml2-dev libmemcached-dev libbrotli-dev \
        libboost-program-options-dev libcrypto++-dev libyajl-dev \
        libpqxx-dev zlib1g-dev libargon2-dev libfmt-dev \
        postgresql-15 postgresql-server-dev-all \

docker/debian/Dockerfile_trixie

@@ -5,7 +5,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update -qq && \
     apt-get install -y gcc g++ make cmake \
-       libfcgi-dev libxml2-dev libmemcached-dev \
+       libfcgi-dev libxml2-dev libmemcached-dev libbrotli-dev \
        libboost-program-options-dev libcrypto++-dev libyajl-dev \
        libpqxx-dev zlib1g-dev libargon2-dev libfmt-dev \
        postgresql-16 postgresql-server-dev-all \

docker/ubuntu/Dockerfile

@@ -5,7 +5,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update -qq && \
     apt-get install -y gcc g++ make cmake \
-       libfcgi-dev libxml2-dev libmemcached-dev \
+       libfcgi-dev libxml2-dev libmemcached-dev libbrotli-dev \
        libboost-program-options-dev \
        libcrypto++-dev libyajl-dev \
        libpqxx-dev zlib1g-dev libargon2-dev libfmt-dev \

docker/ubuntu/Dockerfile2204

@@ -5,7 +5,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update -qq && \
     apt-get install -y gcc g++ make cmake \
-       libfcgi-dev libxml2-dev libmemcached-dev \
+       libfcgi-dev libxml2-dev libmemcached-dev libbrotli-dev \
        libboost-program-options-dev libcrypto++-dev libyajl-dev \
        libpqxx-dev zlib1g-dev libargon2-dev libfmt-dev \
        postgresql-14 postgresql-server-dev-all \

include/cgimap/brotli.hpp

@@ -0,0 +1,53 @@
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of openstreetmap-cgimap (https://github.com/zerebubuth/openstreetmap-cgimap/).
+ *
+ * Copyright (C) 2009-2023 by the CGImap developer community.
+ * For a full list of authors see the git log.
+ */
+
+
+#ifndef BROTLI_HPP
+#define BROTLI_HPP
+
+#if HAVE_BROTLI
+
+#include <functional>
+#include <memory>
+
+#include <brotli/decode.h>
+#include <brotli/encode.h>
+
+#include "cgimap/output_buffer.hpp"
+
+
+/**
+ * Compresses an output stream.
+ */
+class brotli_output_buffer : public output_buffer {
+public:
+
+  brotli_output_buffer(output_buffer& o);
+  brotli_output_buffer(const brotli_output_buffer &old) = delete;
+  ~brotli_output_buffer() override = default;
+  int write(const char *buffer, int len) override;
+  int written() override;
+  int close() override;
+  void flush() override;
+
+private:
+  int compress(const char *data, int data_length, bool last);
+
+  BrotliEncoderState *state_ = nullptr;
+  std::array<uint8_t, 16384> buff;
+
+  output_buffer& out;
+  // keep track of bytes written
+  size_t bytes_in = 0;
+  bool flushed{false};
+};
+
+#endif
+
+#endif

include/cgimap/http.hpp

@@ -21,6 +21,10 @@
 #ifdef HAVE_LIBZ
 #include "cgimap/zlib.hpp"
 #endif
+#if HAVE_BROTLI
+#include "cgimap/brotli.hpp"
+#endif
+
 #include "cgimap/output_buffer.hpp"
 
 /**
@@ -284,6 +288,17 @@ class gzip : public encoding {
 };
 #endif /* HAVE_LIBZ */
 
+#if HAVE_BROTLI
+
+class brotli : public encoding {
+public:
+  brotli() : encoding("br"){}
+  std::unique_ptr<output_buffer> buffer(output_buffer& out) override {
+    return std::make_unique<brotli_output_buffer>(out);
+  }
+};
+#endif
+
 /*
  * Parses an Accept-Encoding header and returns the chosen
  * encoding.

src/CMakeLists.txt

@@ -10,6 +10,7 @@ target_sources(cgimap_core PRIVATE
     backend.cpp
     backend.cpp
     bbox.cpp
+    brotli.cpp
     choose_formatter.cpp
     handler.cpp
     http.cpp
@@ -78,6 +79,9 @@ target_link_libraries(cgimap_core
     CryptoPP::CryptoPP
     Libmemcached::Libmemcached
     Argon2::Argon2
+    $<$<BOOL:${ENABLE_BROTLI}>:Brotli::common>
+    $<$<BOOL:${ENABLE_BROTLI}>:Brotli::encoder>
+    $<$<BOOL:${ENABLE_BROTLI}>:Brotli::decoder>
     $<$<BOOL:${ENABLE_YAJL}>:YAJL::YAJL>
     PQXX::PQXX)