Add experimental jemalloc support

Jemalloc is an alternative malloc implementation which improves memory fragmentation.
It also reduces the overall memory consumption and thus might be beneficial when running on puma.
Further information can be found in this article about malloc and jemalloc: https://www.speedshop.co/2017/12/04/malloc-doubles-ruby-memory.html

This change adds the jemalloc packages and introduces the config
property `cc.experimental.use_jemalloc_memory_allocator` to enable jemalloc (default is false).

config/blobs.yml

@@ -2,6 +2,10 @@ expat/expat-2.5.0.tar.bz2:
   size: 569205
   object_id: 970ccd16-75ac-4c55-5280-c00c4aa8f6cc
   sha: sha256:6f0e6e01f7b30025fa05c85fdad1e5d0ec7fd35d9f61b22f34998de11969ff67
+jemalloc/jemalloc-5.3.0.tar.bz2:
+  size: 736023
+  object_id: 1ded0db4-5ba5-4b71-58d0-3b638b98f15b
+  sha: sha256:2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa
 mariadb_connector_c/mariadb-connector-c-3.3.5-src.tar.gz:
   size: 1392609
   object_id: ca059f02-93a8-491b-74cb-44dcc0a59bbc

jobs/cc_deployment_updater/templates/bin/cc_deployment_updater.erb

@@ -7,4 +7,8 @@ cd /var/vcap/packages/cloud_controller_ng/cloud_controller_ng
 export RUBYOPT='--yjit'
 <% end %>
 
+<% if link("cloud_controller_internal").p('cc.experimental.use_jemalloc_memory_allocator') %>
+  export LD_PRELOAD=/var/vcap/packages/jemalloc/lib/libjemalloc.so
+<% end %>
+
 exec bundle exec rake deployment_updater:start

jobs/cloud_controller_clock/templates/bin/cloud_controller_clock.erb

@@ -7,4 +7,8 @@ cd /var/vcap/packages/cloud_controller_ng/cloud_controller_ng
 export RUBYOPT='--yjit'
 <% end %>
 
+<% if link("cloud_controller_internal").p('cc.experimental.use_jemalloc_memory_allocator') %>
+export LD_PRELOAD=/var/vcap/packages/jemalloc/lib/libjemalloc.so
+<% end %>
+
 exec bundle exec rake clock:start

jobs/cloud_controller_ng/spec

@@ -69,6 +69,7 @@ packages:
   - nginx
   - nginx_newrelic_plugin
   - libpq
+  - jemalloc
   - mariadb_connector_c
   - ruby-3.2
 
@@ -158,6 +159,7 @@ provides:
   - cc.external_port
   - cc.external_protocol
   - cc.experimental.use_yjit_compiler
+  - cc.experimental.use_jemalloc_memory_allocator
   - cc.internal_service_hostname
   - cc.log_db_queries
   - cc.logging.format.timestamp
@@ -1187,6 +1189,10 @@ properties:
     description: "Use Ruby's YJIT compiler when running Cloud Controller API servers and workers. This feature is experimental and not recommended. Please review the drawbacks and benefits of YJIT before enabling."
     default: false
 
+  cc.experimental.use_jemalloc_memory_allocator:
+    description: "Enables jemalloc rather than malloc for Cloud Controller API servers and workers; however, it's experimental and not typically recommended, so review its pros and cons before use."
+    default: false
+
   cc.experimental.use_puma_webserver:
     description: "Use Puma in place of Thin as the webserver. This may increase performance as Puma forks Cloud Controller processes to avoid relying on threads"
     default: false

jobs/cloud_controller_ng/templates/bin/cloud_controller_ng.erb

@@ -18,5 +18,9 @@ echo 'Finished migrations and seeds'
 export RUBYOPT='--yjit'
 <% end %>
 
+<% if p('cc.experimental.use_jemalloc_memory_allocator') %>
+export LD_PRELOAD=/var/vcap/packages/jemalloc/lib/libjemalloc.so
+<% end %>
+
 exec /var/vcap/packages/cloud_controller_ng/cloud_controller_ng/bin/cloud_controller \
   -c /var/vcap/jobs/cloud_controller_ng/config/cloud_controller_ng.yml

jobs/cloud_controller_ng/templates/bin/local_worker.erb

@@ -9,5 +9,9 @@ wait_for_blobstore
 export RUBYOPT='--yjit'
 <% end %>
 
+<% if p('cc.experimental.use_jemalloc_memory_allocator') %>
+export LD_PRELOAD=/var/vcap/packages/jemalloc/lib/libjemalloc.so
+<% end %>
+
 cd /var/vcap/packages/cloud_controller_ng/cloud_controller_ng
 exec bundle exec rake "jobs:local[cc_api_worker.<%= spec.job.name %>.<%= spec.index %>.${INDEX}]"

jobs/cloud_controller_worker/templates/bin/cloud_controller_worker.erb

@@ -9,6 +9,9 @@ wait_for_blobstore
 export RUBYOPT='--yjit'
 <% end %>
 
+<% if link("cloud_controller_internal").p('cc.experimental.use_jemalloc_memory_allocator') %>
+export LD_PRELOAD=/var/vcap/packages/jemalloc/lib/libjemalloc.so
+<% end %>
 
 cd /var/vcap/packages/cloud_controller_ng/cloud_controller_ng
 exec bundle exec rake jobs:generic[cc_global_worker.<%= spec.job.name %>.<%= spec.index %>.${INDEX}]

packages/jemalloc/README.md

@@ -0,0 +1,9 @@
+jemalloc
+============
+Jemalloc is a general purpose malloc implementation that emphasizes fragmentation avoidance and scalable concurrency support.
+
+This file can be downloaded from the following locations:
+
+| Filename               | Download URL |
+|------------------------| ------------ |
+| jemalloc-5.3.0.tar.bz2 | https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2 |

packages/jemalloc/packaging

@@ -0,0 +1,18 @@
+#!/bin/bash -eu
+
+function main() {
+  local jemalloc_version
+  jemalloc_version="5.3.0"
+
+  mkdir -p "jemalloc-${jemalloc_version}" && tar xf "jemalloc/jemalloc-${jemalloc_version}.tar.bz2" -C "jemalloc-${jemalloc_version}" --strip-components=1
+
+  pushd "jemalloc-${jemalloc_version}" > /dev/null
+    autoconf
+    ./configure --prefix="${BOSH_INSTALL_TARGET}"
+    make dist
+    make install
+
+  popd > /dev/null
+}
+
+main

packages/jemalloc/spec

@@ -0,0 +1,4 @@
+---
+name: jemalloc
+files:
+- jemalloc/jemalloc-5.3.0.tar.bz2