From 31e7c475d6fa729589ec06ac6b1eb823a490e143 Mon Sep 17 00:00:00 2001 From: johha Date: Wed, 3 Jan 2024 14:54:31 +0100 Subject: [PATCH] Add experimental jemalloc support Jemalloc is an alternative malloc implementation which improves memory fragmentation. It also reduces the overall memory consumption and thus might be beneficial when running on puma. Further information can be found in this article about malloc and jemalloc: https://www.speedshop.co/2017/12/04/malloc-doubles-ruby-memory.html This change adds the jemalloc packages and introduces the config property `cc.experimental.use_jemalloc_memory_allocator` to enable jemalloc (default is false). --- config/blobs.yml | 4 ++++ .../templates/bin/cc_deployment_updater.erb | 4 ++++ .../templates/bin/cloud_controller_clock.erb | 4 ++++ jobs/cloud_controller_ng/spec | 6 ++++++ .../templates/bin/cloud_controller_ng.erb | 4 ++++ .../templates/bin/local_worker.erb | 4 ++++ .../templates/bin/cloud_controller_worker.erb | 3 +++ packages/jemalloc/README.md | 9 +++++++++ packages/jemalloc/packaging | 18 ++++++++++++++++++ packages/jemalloc/spec | 4 ++++ 10 files changed, 60 insertions(+) create mode 100644 packages/jemalloc/README.md create mode 100644 packages/jemalloc/packaging create mode 100644 packages/jemalloc/spec diff --git a/config/blobs.yml b/config/blobs.yml index 848e0ee9ae..01e69f2d80 100644 --- a/config/blobs.yml +++ b/config/blobs.yml @@ -2,6 +2,10 @@ expat/expat-2.5.0.tar.bz2: size: 569205 object_id: 970ccd16-75ac-4c55-5280-c00c4aa8f6cc sha: sha256:6f0e6e01f7b30025fa05c85fdad1e5d0ec7fd35d9f61b22f34998de11969ff67 +jemalloc/jemalloc-5.3.0.tar.bz2: + size: 736023 + object_id: 1ded0db4-5ba5-4b71-58d0-3b638b98f15b + sha: sha256:2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa mariadb_connector_c/mariadb-connector-c-3.3.5-src.tar.gz: size: 1392609 object_id: ca059f02-93a8-491b-74cb-44dcc0a59bbc diff --git a/jobs/cc_deployment_updater/templates/bin/cc_deployment_updater.erb b/jobs/cc_deployment_updater/templates/bin/cc_deployment_updater.erb index a4afd820f0..ea1a6a475c 100644 --- a/jobs/cc_deployment_updater/templates/bin/cc_deployment_updater.erb +++ b/jobs/cc_deployment_updater/templates/bin/cc_deployment_updater.erb @@ -7,4 +7,8 @@ cd /var/vcap/packages/cloud_controller_ng/cloud_controller_ng export RUBYOPT='--yjit' <% end %> +<% if link("cloud_controller_internal").p('cc.experimental.use_jemalloc_memory_allocator') %> + export LD_PRELOAD=/var/vcap/packages/jemalloc/lib/libjemalloc.so +<% end %> + exec bundle exec rake deployment_updater:start diff --git a/jobs/cloud_controller_clock/templates/bin/cloud_controller_clock.erb b/jobs/cloud_controller_clock/templates/bin/cloud_controller_clock.erb index 01332c0931..12ffb37234 100644 --- a/jobs/cloud_controller_clock/templates/bin/cloud_controller_clock.erb +++ b/jobs/cloud_controller_clock/templates/bin/cloud_controller_clock.erb @@ -7,4 +7,8 @@ cd /var/vcap/packages/cloud_controller_ng/cloud_controller_ng export RUBYOPT='--yjit' <% end %> +<% if link("cloud_controller_internal").p('cc.experimental.use_jemalloc_memory_allocator') %> +export LD_PRELOAD=/var/vcap/packages/jemalloc/lib/libjemalloc.so +<% end %> + exec bundle exec rake clock:start diff --git a/jobs/cloud_controller_ng/spec b/jobs/cloud_controller_ng/spec index 89e9e08e81..b2d05d8ff0 100644 --- a/jobs/cloud_controller_ng/spec +++ b/jobs/cloud_controller_ng/spec @@ -69,6 +69,7 @@ packages: - nginx - nginx_newrelic_plugin - libpq + - jemalloc - mariadb_connector_c - ruby-3.2 @@ -158,6 +159,7 @@ provides: - cc.external_port - cc.external_protocol - cc.experimental.use_yjit_compiler + - cc.experimental.use_jemalloc_memory_allocator - cc.internal_service_hostname - cc.log_db_queries - cc.logging.format.timestamp @@ -1187,6 +1189,10 @@ properties: description: "Use Ruby's YJIT compiler when running Cloud Controller API servers and workers. This feature is experimental and not recommended. Please review the drawbacks and benefits of YJIT before enabling." default: false + cc.experimental.use_jemalloc_memory_allocator: + description: "Enables jemalloc rather than malloc for Cloud Controller API servers and workers; however, it's experimental and not typically recommended, so review its pros and cons before use." + default: false + cc.experimental.use_puma_webserver: description: "Use Puma in place of Thin as the webserver. This may increase performance as Puma forks Cloud Controller processes to avoid relying on threads" default: false diff --git a/jobs/cloud_controller_ng/templates/bin/cloud_controller_ng.erb b/jobs/cloud_controller_ng/templates/bin/cloud_controller_ng.erb index ec3f7e9266..c207804091 100644 --- a/jobs/cloud_controller_ng/templates/bin/cloud_controller_ng.erb +++ b/jobs/cloud_controller_ng/templates/bin/cloud_controller_ng.erb @@ -18,5 +18,9 @@ echo 'Finished migrations and seeds' export RUBYOPT='--yjit' <% end %> +<% if p('cc.experimental.use_jemalloc_memory_allocator') %> +export LD_PRELOAD=/var/vcap/packages/jemalloc/lib/libjemalloc.so +<% end %> + exec /var/vcap/packages/cloud_controller_ng/cloud_controller_ng/bin/cloud_controller \ -c /var/vcap/jobs/cloud_controller_ng/config/cloud_controller_ng.yml diff --git a/jobs/cloud_controller_ng/templates/bin/local_worker.erb b/jobs/cloud_controller_ng/templates/bin/local_worker.erb index d02a2928b2..54a34ce580 100644 --- a/jobs/cloud_controller_ng/templates/bin/local_worker.erb +++ b/jobs/cloud_controller_ng/templates/bin/local_worker.erb @@ -9,5 +9,9 @@ wait_for_blobstore export RUBYOPT='--yjit' <% end %> +<% if p('cc.experimental.use_jemalloc_memory_allocator') %> +export LD_PRELOAD=/var/vcap/packages/jemalloc/lib/libjemalloc.so +<% end %> + cd /var/vcap/packages/cloud_controller_ng/cloud_controller_ng exec bundle exec rake "jobs:local[cc_api_worker.<%= spec.job.name %>.<%= spec.index %>.${INDEX}]" diff --git a/jobs/cloud_controller_worker/templates/bin/cloud_controller_worker.erb b/jobs/cloud_controller_worker/templates/bin/cloud_controller_worker.erb index 66bfbe01a6..d48eabbe85 100644 --- a/jobs/cloud_controller_worker/templates/bin/cloud_controller_worker.erb +++ b/jobs/cloud_controller_worker/templates/bin/cloud_controller_worker.erb @@ -9,6 +9,9 @@ wait_for_blobstore export RUBYOPT='--yjit' <% end %> +<% if link("cloud_controller_internal").p('cc.experimental.use_jemalloc_memory_allocator') %> +export LD_PRELOAD=/var/vcap/packages/jemalloc/lib/libjemalloc.so +<% end %> cd /var/vcap/packages/cloud_controller_ng/cloud_controller_ng exec bundle exec rake jobs:generic[cc_global_worker.<%= spec.job.name %>.<%= spec.index %>.${INDEX}] diff --git a/packages/jemalloc/README.md b/packages/jemalloc/README.md new file mode 100644 index 0000000000..d6c341b137 --- /dev/null +++ b/packages/jemalloc/README.md @@ -0,0 +1,9 @@ +jemalloc +============ +Jemalloc is a general purpose malloc implementation that emphasizes fragmentation avoidance and scalable concurrency support. + +This file can be downloaded from the following locations: + +| Filename | Download URL | +|------------------------| ------------ | +| jemalloc-5.3.0.tar.bz2 | https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2 | diff --git a/packages/jemalloc/packaging b/packages/jemalloc/packaging new file mode 100644 index 0000000000..755abb1fe5 --- /dev/null +++ b/packages/jemalloc/packaging @@ -0,0 +1,18 @@ +#!/bin/bash -eu + +function main() { + local jemalloc_version + jemalloc_version="5.3.0" + + mkdir -p "jemalloc-${jemalloc_version}" && tar xf "jemalloc/jemalloc-${jemalloc_version}.tar.bz2" -C "jemalloc-${jemalloc_version}" --strip-components=1 + + pushd "jemalloc-${jemalloc_version}" > /dev/null + autoconf + ./configure --prefix="${BOSH_INSTALL_TARGET}" + make dist + make install + + popd > /dev/null +} + +main diff --git a/packages/jemalloc/spec b/packages/jemalloc/spec new file mode 100644 index 0000000000..40ff49b708 --- /dev/null +++ b/packages/jemalloc/spec @@ -0,0 +1,4 @@ +--- +name: jemalloc +files: +- jemalloc/jemalloc-5.3.0.tar.bz2