jemalloc.c

#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"

#include "jemalloc/internal/assert.h"
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/buf_writer.h"
#include "jemalloc/internal/ctl.h"
#include "jemalloc/internal/emap.h"
#include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/extent_mmap.h"
#include "jemalloc/internal/fxp.h"
#include "jemalloc/internal/san.h"
#include "jemalloc/internal/hook.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/log.h"
#include "jemalloc/internal/malloc_io.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/nstime.h"
#include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/safety_check.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/spin.h"
#include "jemalloc/internal/sz.h"
#include "jemalloc/internal/ticker.h"
#include "jemalloc/internal/thread_event.h"
#include "jemalloc/internal/util.h"

/******************************************************************************/
/* Data. */

/* Runtime configuration options. */
const char	*je_malloc_conf
#ifndef _WIN32
    JEMALLOC_ATTR(weak)
#endif
    ;
/*
 * The usual rule is that the closer to runtime you are, the higher priority
 * your configuration settings are (so the jemalloc config options get lower
 * priority than the per-binary setting, which gets lower priority than the /etc
 * setting, which gets lower priority than the environment settings).
 *
 * But it's a fairly common use case in some testing environments for a user to
 * be able to control the binary, but nothing else (e.g. a performancy canary
 * uses the production OS and environment variables, but can run any binary in
 * those circumstances).  For these use cases, it's handy to have an in-binary
 * mechanism for overriding environment variable settings, with the idea that if
 * the results are positive they get promoted to the official settings, and
 * moved from the binary to the environment variable.
 *
 * We don't actually want this to be widespread, so we'll give it a silly name
 * and not mention it in headers or documentation.
 */
const char	*je_malloc_conf_2_conf_harder
#ifndef _WIN32
    JEMALLOC_ATTR(weak)
#endif
    ;

const char *opt_malloc_conf_symlink = NULL;
const char *opt_malloc_conf_env_var = NULL;

bool	opt_abort =
#ifdef JEMALLOC_DEBUG
    true
#else
    false
#endif
    ;
bool	opt_abort_conf =
#ifdef JEMALLOC_DEBUG
    true
#else
    false
#endif
    ;
/* Intentionally default off, even with debug builds. */
bool	opt_confirm_conf = false;
const char	*opt_junk =
#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
    "true"
#else
    "false"
#endif
    ;
bool	opt_junk_alloc =
#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
    true
#else
    false
#endif
    ;
bool	opt_junk_free =
#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
    true
#else
    false
#endif
    ;
bool	opt_trust_madvise =
#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS
    false
#else
    true
#endif
    ;

bool opt_cache_oblivious =
#ifdef JEMALLOC_CACHE_OBLIVIOUS
    true
#else
    false
#endif
    ;

zero_realloc_action_t opt_zero_realloc_action =
#ifdef JEMALLOC_ZERO_REALLOC_DEFAULT_FREE
    zero_realloc_action_free
#else
    zero_realloc_action_alloc
#endif
    ;

atomic_zu_t zero_realloc_count = ATOMIC_INIT(0);

const char *const zero_realloc_mode_names[] = {
	"alloc",
	"free",
	"abort",
};

/*
 * These are the documented values for junk fill debugging facilities -- see the
 * man page.
 */
static const uint8_t junk_alloc_byte = 0xa5;
static const uint8_t junk_free_byte = 0x5a;

static void default_junk_alloc(void *ptr, size_t usize) {
	memset(ptr, junk_alloc_byte, usize);
}

static void default_junk_free(void *ptr, size_t usize) {
	memset(ptr, junk_free_byte, usize);
}

void (*JET_MUTABLE junk_alloc_callback)(void *ptr, size_t size) = &default_junk_alloc;
void (*JET_MUTABLE junk_free_callback)(void *ptr, size_t size) = &default_junk_free;
void (*JET_MUTABLE invalid_conf_abort)(void) = &abort;

bool	opt_utrace = false;
bool	opt_xmalloc = false;
bool	opt_experimental_infallible_new = false;
bool	opt_experimental_tcache_gc = false;
bool	opt_zero = false;
unsigned	opt_narenas = 0;
static fxp_t		opt_narenas_ratio = FXP_INIT_INT(4);

unsigned	ncpus;

unsigned opt_debug_double_free_max_scan =
    SAFETY_CHECK_DOUBLE_FREE_MAX_SCAN_DEFAULT;

size_t opt_calloc_madvise_threshold = 0;

/* Protects arenas initialization. */
static malloc_mutex_t arenas_lock;

/* The global hpa, and whether it's on. */
bool opt_hpa = false;
hpa_shard_opts_t opt_hpa_opts = HPA_SHARD_OPTS_DEFAULT;
sec_opts_t opt_hpa_sec_opts = SEC_OPTS_DEFAULT;

/*
 * Arenas that are used to service external requests.  Not all elements of the
 * arenas array are necessarily used; arenas are created lazily as needed.
 *
 * arenas[0..narenas_auto) are used for automatic multiplexing of threads and
 * arenas.  arenas[narenas_auto..narenas_total) are only used if the application
 * takes some action to create them and allocate from them.
 *
 * Points to an arena_t.
 */
JEMALLOC_ALIGNED(CACHELINE)
atomic_p_t		arenas[MALLOCX_ARENA_LIMIT];
static atomic_u_t	narenas_total; /* Use narenas_total_*(). */
/* Below three are read-only after initialization. */
static arena_t		*a0; /* arenas[0]. */
unsigned		narenas_auto;
unsigned		manual_arena_base;

malloc_init_t malloc_init_state = malloc_init_uninitialized;

/* False should be the common case.  Set to true to trigger initialization. */
bool			malloc_slow = true;

/* When malloc_slow is true, set the corresponding bits for sanity check. */
enum {
	flag_opt_junk_alloc	= (1U),
	flag_opt_junk_free	= (1U << 1),
	flag_opt_zero		= (1U << 2),
	flag_opt_utrace		= (1U << 3),
	flag_opt_xmalloc	= (1U << 4)
};
static uint8_t	malloc_slow_flags;

#ifdef JEMALLOC_THREADED_INIT
/* Used to let the initializing thread recursively allocate. */
#  define NO_INITIALIZER	((unsigned long)0)
#  define INITIALIZER		pthread_self()
#  define IS_INITIALIZER	(malloc_initializer == pthread_self())
static pthread_t		malloc_initializer = NO_INITIALIZER;
#else
#  define NO_INITIALIZER	false
#  define INITIALIZER		true
#  define IS_INITIALIZER	malloc_initializer
static bool			malloc_initializer = NO_INITIALIZER;
#endif

/* Used to avoid initialization races. */
#ifdef _WIN32
#if _WIN32_WINNT >= 0x0600
static malloc_mutex_t	init_lock = SRWLOCK_INIT;
#else
static malloc_mutex_t	init_lock;
static bool init_lock_initialized = false;

JEMALLOC_ATTR(constructor)
static void WINAPI
_init_init_lock(void) {
	/*
	 * If another constructor in the same binary is using mallctl to e.g.
	 * set up extent hooks, it may end up running before this one, and
	 * malloc_init_hard will crash trying to lock the uninitialized lock. So
	 * we force an initialization of the lock in malloc_init_hard as well.
	 * We don't try to care about atomicity of the accessed to the
	 * init_lock_initialized boolean, since it really only matters early in
	 * the process creation, before any separate thread normally starts
	 * doing anything.
	 */
	if (!init_lock_initialized) {
		malloc_mutex_init(&init_lock, "init", WITNESS_RANK_INIT,
		    malloc_mutex_rank_exclusive);
	}
	init_lock_initialized = true;
}

#ifdef _MSC_VER
#  pragma section(".CRT$XCU", read)
JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used)
static const void (WINAPI *init_init_lock)(void) = _init_init_lock;
#endif
#endif
#else
static malloc_mutex_t	init_lock = MALLOC_MUTEX_INITIALIZER;
#endif

typedef struct {
	void	*p;	/* Input pointer (as in realloc(p, s)). */
	size_t	s;	/* Request size. */
	void	*r;	/* Result pointer. */
} malloc_utrace_t;

#ifdef JEMALLOC_UTRACE
#  define UTRACE(a, b, c) do {						\
	if (unlikely(opt_utrace)) {					\
		int utrace_serrno = errno;				\
		malloc_utrace_t ut;					\
		ut.p = (a);						\
		ut.s = (b);						\
		ut.r = (c);						\
		UTRACE_CALL(&ut, sizeof(ut));				\
		errno = utrace_serrno;					\
	}								\
} while (0)
#else
#  define UTRACE(a, b, c)
#endif

/* Whether encountered any invalid config options. */
static bool had_conf_error = false;

/******************************************************************************/
/*
 * Function prototypes for static functions that are referenced prior to
 * definition.
 */

static bool	malloc_init_hard_a0(void);
static bool	malloc_init_hard(void);

/******************************************************************************/
/*
 * Begin miscellaneous support functions.
 */

JEMALLOC_ALWAYS_INLINE bool
malloc_init_a0(void) {
	if (unlikely(malloc_init_state == malloc_init_uninitialized)) {
		return malloc_init_hard_a0();
	}
	return false;
}

JEMALLOC_ALWAYS_INLINE bool
malloc_init(void) {
	if (unlikely(!malloc_initialized()) && malloc_init_hard()) {
		return true;
	}
	return false;
}

/*
 * The a0*() functions are used instead of i{d,}alloc() in situations that
 * cannot tolerate TLS variable access.
 */

static void *
a0ialloc(size_t size, bool zero, bool is_internal) {
	if (unlikely(malloc_init_a0())) {
		return NULL;
	}

	return iallocztm(TSDN_NULL, size, sz_size2index(size), zero, NULL,
	    is_internal, arena_get(TSDN_NULL, 0, true), true);
}

static void
a0idalloc(void *ptr, bool is_internal) {
	idalloctm(TSDN_NULL, ptr, NULL, NULL, is_internal, true);
}

void *
a0malloc(size_t size) {
	return a0ialloc(size, false, true);
}

void
a0dalloc(void *ptr) {
	a0idalloc(ptr, true);
}

/*
 * FreeBSD's libc uses the bootstrap_*() functions in bootstrap-sensitive
 * situations that cannot tolerate TLS variable access (TLS allocation and very
 * early internal data structure initialization).
 */

void *
bootstrap_malloc(size_t size) {
	if (unlikely(size == 0)) {
		size = 1;
	}

	return a0ialloc(size, false, false);
}

void *
bootstrap_calloc(size_t num, size_t size) {
	size_t num_size;

	num_size = num * size;
	if (unlikely(num_size == 0)) {
		assert(num == 0 || size == 0);
		num_size = 1;
	}

	return a0ialloc(num_size, true, false);
}

void
bootstrap_free(void *ptr) {
	if (unlikely(ptr == NULL)) {
		return;
	}

	a0idalloc(ptr, false);
}

void
arena_set(unsigned ind, arena_t *arena) {
	atomic_store_p(&arenas[ind], arena, ATOMIC_RELEASE);
}

static void
narenas_total_set(unsigned narenas) {
	atomic_store_u(&narenas_total, narenas, ATOMIC_RELEASE);
}

static void
narenas_total_inc(void) {
	atomic_fetch_add_u(&narenas_total, 1, ATOMIC_RELEASE);
}

unsigned
narenas_total_get(void) {
	return atomic_load_u(&narenas_total, ATOMIC_ACQUIRE);
}

/* Create a new arena and insert it into the arenas array at index ind. */
static arena_t *
arena_init_locked(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
	arena_t *arena;

	assert(ind <= narenas_total_get());
	if (ind >= MALLOCX_ARENA_LIMIT) {
		return NULL;
	}
	if (ind == narenas_total_get()) {
		narenas_total_inc();
	}

	/*
	 * Another thread may have already initialized arenas[ind] if it's an
	 * auto arena.
	 */
	arena = arena_get(tsdn, ind, false);
	if (arena != NULL) {
		assert(arena_is_auto(arena));
		return arena;
	}

	/* Actually initialize the arena. */
	arena = arena_new(tsdn, ind, config);

	return arena;
}

static void
arena_new_create_background_thread(tsdn_t *tsdn, unsigned ind) {
	if (ind == 0) {
		return;
	}

	if (have_background_thread) {
		if (background_thread_create(tsdn_tsd(tsdn), ind)) {
			malloc_printf("<jemalloc>: error in background thread "
				      "creation for arena %u. Abort.\n", ind);
			abort();
		}
	}
}

arena_t *
arena_init(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
	arena_t *arena;

	malloc_mutex_lock(tsdn, &arenas_lock);
	arena = arena_init_locked(tsdn, ind, config);
	malloc_mutex_unlock(tsdn, &arenas_lock);

	arena_new_create_background_thread(tsdn, ind);

	return arena;
}

static void
arena_bind(tsd_t *tsd, unsigned ind, bool internal) {
	arena_t *arena = arena_get(tsd_tsdn(tsd), ind, false);
	arena_nthreads_inc(arena, internal);

	if (internal) {
		tsd_iarena_set(tsd, arena);
	} else {
		tsd_arena_set(tsd, arena);
		/*
		 * While shard acts as a random seed, the cast below should
		 * not make much difference.
		 */
		uint8_t shard = (uint8_t)atomic_fetch_add_u(
		    &arena->binshard_next, 1, ATOMIC_RELAXED);
		tsd_binshards_t *bins = tsd_binshardsp_get(tsd);
		for (unsigned i = 0; i < SC_NBINS; i++) {
			assert(bin_infos[i].n_shards > 0 &&
			    bin_infos[i].n_shards <= BIN_SHARDS_MAX);
			bins->binshard[i] = shard % bin_infos[i].n_shards;
		}
	}
}

void
arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena) {
	assert(oldarena != NULL);
	assert(newarena != NULL);

	arena_nthreads_dec(oldarena, false);
	arena_nthreads_inc(newarena, false);
	tsd_arena_set(tsd, newarena);

	if (arena_nthreads_get(oldarena, false) == 0) {
		/* Purge if the old arena has no associated threads anymore. */
		arena_decay(tsd_tsdn(tsd), oldarena,
		    /* is_background_thread */ false, /* all */ true);
	}
}

static void
arena_unbind(tsd_t *tsd, unsigned ind, bool internal) {
	arena_t *arena;

	arena = arena_get(tsd_tsdn(tsd), ind, false);
	arena_nthreads_dec(arena, internal);

	if (internal) {
		tsd_iarena_set(tsd, NULL);
	} else {
		tsd_arena_set(tsd, NULL);
	}
}

/* Slow path, called only by arena_choose(). */
arena_t *
arena_choose_hard(tsd_t *tsd, bool internal) {
	arena_t *ret JEMALLOC_CC_SILENCE_INIT(NULL);

	if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena)) {
		unsigned choose = percpu_arena_choose();
		ret = arena_get(tsd_tsdn(tsd), choose, true);
		assert(ret != NULL);
		arena_bind(tsd, arena_ind_get(ret), false);
		arena_bind(tsd, arena_ind_get(ret), true);

		return ret;
	}

	if (narenas_auto > 1) {
		unsigned i, j, choose[2], first_null;
		bool is_new_arena[2];

		/*
		 * Determine binding for both non-internal and internal
		 * allocation.
		 *
		 *   choose[0]: For application allocation.
		 *   choose[1]: For internal metadata allocation.
		 */

		for (j = 0; j < 2; j++) {
			choose[j] = 0;
			is_new_arena[j] = false;
		}

		first_null = narenas_auto;
		malloc_mutex_lock(tsd_tsdn(tsd), &arenas_lock);
		assert(arena_get(tsd_tsdn(tsd), 0, false) != NULL);
		for (i = 1; i < narenas_auto; i++) {
			if (arena_get(tsd_tsdn(tsd), i, false) != NULL) {
				/*
				 * Choose the first arena that has the lowest
				 * number of threads assigned to it.
				 */
				for (j = 0; j < 2; j++) {
					if (arena_nthreads_get(arena_get(
					    tsd_tsdn(tsd), i, false), !!j) <
					    arena_nthreads_get(arena_get(
					    tsd_tsdn(tsd), choose[j], false),
					    !!j)) {
						choose[j] = i;
					}
				}
			} else if (first_null == narenas_auto) {
				/*
				 * Record the index of the first uninitialized
				 * arena, in case all extant arenas are in use.
				 *
				 * NB: It is possible for there to be
				 * discontinuities in terms of initialized
				 * versus uninitialized arenas, due to the
				 * "thread.arena" mallctl.
				 */
				first_null = i;
			}
		}

		for (j = 0; j < 2; j++) {
			if (arena_nthreads_get(arena_get(tsd_tsdn(tsd),
			    choose[j], false), !!j) == 0 || first_null ==
			    narenas_auto) {
				/*
				 * Use an unloaded arena, or the least loaded
				 * arena if all arenas are already initialized.
				 */
				if (!!j == internal) {
					ret = arena_get(tsd_tsdn(tsd),
					    choose[j], false);
				}
			} else {
				arena_t *arena;

				/* Initialize a new arena. */
				choose[j] = first_null;
				arena = arena_init_locked(tsd_tsdn(tsd),
				    choose[j], &arena_config_default);
				if (arena == NULL) {
					malloc_mutex_unlock(tsd_tsdn(tsd),
					    &arenas_lock);
					return NULL;
				}
				is_new_arena[j] = true;
				if (!!j == internal) {
					ret = arena;
				}
			}
			arena_bind(tsd, choose[j], !!j);
		}
		malloc_mutex_unlock(tsd_tsdn(tsd), &arenas_lock);

		for (j = 0; j < 2; j++) {
			if (is_new_arena[j]) {
				assert(choose[j] > 0);
				arena_new_create_background_thread(
				    tsd_tsdn(tsd), choose[j]);
			}
		}

	} else {
		ret = arena_get(tsd_tsdn(tsd), 0, false);
		arena_bind(tsd, 0, false);
		arena_bind(tsd, 0, true);
	}

	return ret;
}

void
iarena_cleanup(tsd_t *tsd) {
	arena_t *iarena;

	iarena = tsd_iarena_get(tsd);
	if (iarena != NULL) {
		arena_unbind(tsd, arena_ind_get(iarena), true);
	}
}

void
arena_cleanup(tsd_t *tsd) {
	arena_t *arena;

	arena = tsd_arena_get(tsd);
	if (arena != NULL) {
		arena_unbind(tsd, arena_ind_get(arena), false);
	}
}

static void
stats_print_atexit(void) {
	if (config_stats) {
		tsdn_t *tsdn;
		unsigned narenas, i;

		tsdn = tsdn_fetch();

		/*
		 * Merge stats from extant threads.  This is racy, since
		 * individual threads do not lock when recording tcache stats
		 * events.  As a consequence, the final stats may be slightly
		 * out of date by the time they are reported, if other threads
		 * continue to allocate.
		 */
		for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
			arena_t *arena = arena_get(tsdn, i, false);
			if (arena != NULL) {
				tcache_slow_t *tcache_slow;

				malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
				ql_foreach(tcache_slow, &arena->tcache_ql,
				    link) {
					tcache_stats_merge(tsdn,
					    tcache_slow->tcache, arena);
				}
				malloc_mutex_unlock(tsdn,
				    &arena->tcache_ql_mtx);
			}
		}
	}
	je_malloc_stats_print(NULL, NULL, opt_stats_print_opts);
}

/*
 * Ensure that we don't hold any locks upon entry to or exit from allocator
 * code (in a "broad" sense that doesn't count a reentrant allocation as an
 * entrance or exit).
 */
JEMALLOC_ALWAYS_INLINE void
check_entry_exit_locking(tsdn_t *tsdn) {
	if (!config_debug) {
		return;
	}
	if (tsdn_null(tsdn)) {
		return;
	}
	tsd_t *tsd = tsdn_tsd(tsdn);
	/*
	 * It's possible we hold locks at entry/exit if we're in a nested
	 * allocation.
	 */
	int8_t reentrancy_level = tsd_reentrancy_level_get(tsd);
	if (reentrancy_level != 0) {
		return;
	}
	witness_assert_lockless(tsdn_witness_tsdp_get(tsdn));
}

/*
 * End miscellaneous support functions.
 */
/******************************************************************************/
/*
 * Begin initialization functions.
 */

static char *
jemalloc_getenv(const char *name) {
#ifdef JEMALLOC_FORCE_GETENV
	return getenv(name);
#else
#  ifdef JEMALLOC_HAVE_SECURE_GETENV
	return secure_getenv(name);
#  else
#    ifdef JEMALLOC_HAVE_ISSETUGID
	if (issetugid() != 0) {
		return NULL;
	}
#    endif
	return getenv(name);
#  endif
#endif
}

static unsigned
malloc_ncpus(void) {
	long result;

#ifdef _WIN32
	SYSTEM_INFO si;
	GetSystemInfo(&si);
	result = si.dwNumberOfProcessors;
#elif defined(CPU_COUNT)
	/*
	 * glibc >= 2.6 has the CPU_COUNT macro.
	 *
	 * glibc's sysconf() uses isspace().  glibc allocates for the first time
	 * *before* setting up the isspace tables.  Therefore we need a
	 * different method to get the number of CPUs.
	 *
	 * The getaffinity approach is also preferred when only a subset of CPUs
	 * is available, to avoid using more arenas than necessary.
	 */
	{
#  if defined(__FreeBSD__) || defined(__DragonFly__)
		cpuset_t set;
#  else
		cpu_set_t set;
#  endif
#  if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
		sched_getaffinity(0, sizeof(set), &set);
#  else
		pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
#  endif
		result = CPU_COUNT(&set);
	}
#else
	result = sysconf(_SC_NPROCESSORS_ONLN);
#endif
	return ((result == -1) ? 1 : (unsigned)result);
}

/*
 * Ensure that number of CPUs is determistinc, i.e. it is the same based on:
 * - sched_getaffinity()
 * - _SC_NPROCESSORS_ONLN
 * - _SC_NPROCESSORS_CONF
 * Since otherwise tricky things is possible with percpu arenas in use.
 */
static bool
malloc_cpu_count_is_deterministic(void)
{
#ifdef _WIN32
	return true;
#else
	long cpu_onln = sysconf(_SC_NPROCESSORS_ONLN);
	long cpu_conf = sysconf(_SC_NPROCESSORS_CONF);
	if (cpu_onln != cpu_conf) {
		return false;
	}
#  if defined(CPU_COUNT)
#    if defined(__FreeBSD__) || defined(__DragonFly__)
	cpuset_t set;
#    else
	cpu_set_t set;
#    endif /* __FreeBSD__ */
#    if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
	sched_getaffinity(0, sizeof(set), &set);
#    else /* !JEMALLOC_HAVE_SCHED_SETAFFINITY */
	pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
#    endif /* JEMALLOC_HAVE_SCHED_SETAFFINITY */
	long cpu_affinity = CPU_COUNT(&set);
	if (cpu_affinity != cpu_conf) {
		return false;
	}
#  endif /* CPU_COUNT */
	return true;
#endif
}

static void
init_opt_stats_opts(const char *v, size_t vlen, char *dest) {
	size_t opts_len = strlen(dest);
	assert(opts_len <= stats_print_tot_num_options);

	for (size_t i = 0; i < vlen; i++) {
		switch (v[i]) {
#define OPTION(o, v, d, s) case o: break;
			STATS_PRINT_OPTIONS
#undef OPTION
		default: continue;
		}

		if (strchr(dest, v[i]) != NULL) {
			/* Ignore repeated. */
			continue;
		}

		dest[opts_len++] = v[i];
		dest[opts_len] = '\0';
		assert(opts_len <= stats_print_tot_num_options);
	}
	assert(opts_len == strlen(dest));
}

static void
malloc_conf_format_error(const char *msg, const char *begin, const char *end) {
	size_t len = end - begin + 1;
	len = len > BUFERROR_BUF ? BUFERROR_BUF : len;

	malloc_printf("<jemalloc>: %s -- %.*s\n", msg, (int)len, begin);
}

static bool
malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
    char const **v_p, size_t *vlen_p) {
	bool accept;
	const char *opts = *opts_p;

	*k_p = opts;

	for (accept = false; !accept;) {
		switch (*opts) {
		case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
		case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
		case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
		case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
		case 'Y': case 'Z':
		case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
		case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
		case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
		case 's': case 't': case 'u': case 'v': case 'w': case 'x':
		case 'y': case 'z':
		case '0': case '1': case '2': case '3': case '4': case '5':
		case '6': case '7': case '8': case '9':
		case '_':
			opts++;
			break;
		case ':':
			opts++;
			*klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p;
			*v_p = opts;
			accept = true;
			break;
		case '\0':
			if (opts != *opts_p) {
				malloc_conf_format_error(
				    "Conf string ends with key",
				    *opts_p, opts - 1);
				had_conf_error = true;
			}
			return true;
		default:
			malloc_conf_format_error(
			    "Malformed conf string", *opts_p, opts);
			had_conf_error = true;
			return true;
		}
	}

	for (accept = false; !accept;) {
		switch (*opts) {
		case ',':
			opts++;
			/*
			 * Look ahead one character here, because the next time
			 * this function is called, it will assume that end of
			 * input has been cleanly reached if no input remains,
			 * but we have optimistically already consumed the
			 * comma if one exists.
			 */
			if (*opts == '\0') {
				malloc_conf_format_error(
				    "Conf string ends with comma",
				    *opts_p, opts - 1);
				had_conf_error = true;
			}
			*vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p;
			accept = true;
			break;
		case '\0':
			*vlen_p = (uintptr_t)opts - (uintptr_t)*v_p;
			accept = true;
			break;
		default:
			opts++;
			break;
		}
	}

	*opts_p = opts;
	return false;
}

static void
malloc_abort_invalid_conf(void) {
	assert(opt_abort_conf);
	malloc_printf("<jemalloc>: Abort (abort_conf:true) on invalid conf "
	    "value (see above).\n");
	invalid_conf_abort();
}

static void
malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v,
    size_t vlen) {
	malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k,
	    (int)vlen, v);
	/* If abort_conf is set, error out after processing all options. */
	const char *experimental = "experimental_";
	if (strncmp(k, experimental, strlen(experimental)) == 0) {
		/* However, tolerate experimental features. */
		return;
	}
	had_conf_error = true;
}

static void
malloc_slow_flag_init(void) {
	/*
	 * Combine the runtime options into malloc_slow for fast path.  Called
	 * after processing all the options.
	 */
	malloc_slow_flags |= (opt_junk_alloc ? flag_opt_junk_alloc : 0)
	    | (opt_junk_free ? flag_opt_junk_free : 0)
	    | (opt_zero ? flag_opt_zero : 0)
	    | (opt_utrace ? flag_opt_utrace : 0)
	    | (opt_xmalloc ? flag_opt_xmalloc : 0);

	malloc_slow = (malloc_slow_flags != 0);
}

/* Number of sources for initializing malloc_conf */
#define MALLOC_CONF_NSOURCES 5

static const char *
obtain_malloc_conf(unsigned which_source, char readlink_buf[PATH_MAX + 1]) {
	if (config_debug) {
		static unsigned read_source = 0;
		/*
		 * Each source should only be read once, to minimize # of
		 * syscalls on init.
		 */
		assert(read_source == which_source);
		read_source++;
	}
	assert(which_source < MALLOC_CONF_NSOURCES);

	const char *ret;
	switch (which_source) {
	case 0:
		ret = config_malloc_conf;
		break;
	case 1:
		if (je_malloc_conf != NULL) {
			/* Use options that were compiled into the program. */
			ret = je_malloc_conf;
		} else {
			/* No configuration specified. */
			ret = NULL;
		}
		break;
	case 2: {
#ifndef JEMALLOC_CONFIG_FILE
		ret = NULL;
		break;
#else
		ssize_t linklen = 0;
#  ifndef _WIN32
		int saved_errno = errno;
		const char *linkname =
#    ifdef JEMALLOC_PREFIX
		    "/etc/"JEMALLOC_PREFIX"malloc.conf"
#    else
		    "/etc/malloc.conf"
#    endif
		    ;

		/*
		 * Try to use the contents of the "/etc/malloc.conf" symbolic
		 * link's name.
		 */
#    ifndef JEMALLOC_READLINKAT
		linklen = readlink(linkname, readlink_buf, PATH_MAX);
#    else
		linklen = readlinkat(AT_FDCWD, linkname, readlink_buf, PATH_MAX);
#    endif
		if (linklen == -1) {
			/* No configuration specified. */
			linklen = 0;
			/* Restore errno. */
			set_errno(saved_errno);
		}
#  endif
		readlink_buf[linklen] = '\0';
		ret = readlink_buf;
		break;
#endif
	} case 3: {
#ifndef JEMALLOC_CONFIG_ENV
		ret = NULL;
		break;
#else
		const char *envname =
#  ifdef JEMALLOC_PREFIX
			JEMALLOC_CPREFIX"MALLOC_CONF"
#  else
			"MALLOC_CONF"
#  endif
			;

		if ((ret = jemalloc_getenv(envname)) != NULL) {
			opt_malloc_conf_env_var = ret;
		} else {
			/* No configuration specified. */
			ret = NULL;
		}
		break;
#endif
	} case 4: {
		ret = je_malloc_conf_2_conf_harder;
		break;
	} default:
		not_reached();
		ret = NULL;
	}
	return ret;
}

static void
validate_hpa_settings(void) {
	if (!hpa_supported() || !opt_hpa) {
		return;
	}
	if (HUGEPAGE > HUGEPAGE_MAX_EXPECTED_SIZE) {
		had_conf_error = true;
		malloc_printf(
		    "<jemalloc>: huge page size (%zu) greater than expected."
		    "May not be supported or behave as expected.", HUGEPAGE);
	}
#ifndef JEMALLOC_HAVE_MADVISE_COLLAPSE
	if (opt_hpa_opts.hugify_sync) {
	       had_conf_error = true;
	       malloc_printf(
		   "<jemalloc>: hpa_hugify_sync config option is enabled, "
		   "but MADV_COLLAPSE support was not detected at build "
		   "time.");
	}
#endif
}

static void
malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
    bool initial_call, const char *opts_cache[MALLOC_CONF_NSOURCES],
    char readlink_buf[PATH_MAX + 1]) {
	static const char *opts_explain[MALLOC_CONF_NSOURCES] = {
		"string specified via --with-malloc-conf",
		"string pointed to by the global variable malloc_conf",
		"\"name\" of the file referenced by the symbolic link named "
		    "/etc/malloc.conf",
		"value of the environment variable MALLOC_CONF",
		"string pointed to by the global variable "
		    "malloc_conf_2_conf_harder",
	};
	unsigned i;
	const char *opts, *k, *v;
	size_t klen, vlen;

	for (i = 0; i < MALLOC_CONF_NSOURCES; i++) {
		/* Get runtime configuration. */
		if (initial_call) {
			opts_cache[i] = obtain_malloc_conf(i, readlink_buf);
		}
		opts = opts_cache[i];
		if (!initial_call && opt_confirm_conf) {
			malloc_printf(
			    "<jemalloc>: malloc_conf #%u (%s): \"%s\"\n",
			    i + 1, opts_explain[i], opts != NULL ? opts : "");
		}
		if (opts == NULL) {
			continue;
		}

		while (*opts != '\0' && !malloc_conf_next(&opts, &k, &klen, &v,
		    &vlen)) {

#define CONF_ERROR(msg, k, klen, v, vlen)				\
			if (!initial_call) {				\
				malloc_conf_error(			\
				    msg, k, klen, v, vlen);		\
				cur_opt_valid = false;			\
			}
#define CONF_CONTINUE	{						\
				if (!initial_call && opt_confirm_conf	\
				    && cur_opt_valid) {			\
					malloc_printf("<jemalloc>: -- "	\
					    "Set conf value: %.*s:%.*s"	\
					    "\n", (int)klen, k,		\
					    (int)vlen, v);		\
				}					\
				continue;				\
			}
#define CONF_MATCH(n)							\
	(sizeof(n)-1 == klen && strncmp(n, k, klen) == 0)
#define CONF_MATCH_VALUE(n)						\
	(sizeof(n)-1 == vlen && strncmp(n, v, vlen) == 0)
#define CONF_HANDLE_BOOL(o, n)						\
			if (CONF_MATCH(n)) {				\
				if (CONF_MATCH_VALUE("true")) {		\
					o = true;			\
				} else if (CONF_MATCH_VALUE("false")) {	\
					o = false;			\
				} else {				\
					CONF_ERROR("Invalid conf value",\
					    k, klen, v, vlen);		\
				}					\
				CONF_CONTINUE;				\
			}
      /*
       * One of the CONF_MIN macros below expands, in one of the use points,
       * to "unsigned integer < 0", which is always false, triggering the
       * GCC -Wtype-limits warning, which we disable here and re-enable below.
       */
      JEMALLOC_DIAGNOSTIC_PUSH
      JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS

#define CONF_DONT_CHECK_MIN(um, min)	false
#define CONF_CHECK_MIN(um, min)	((um) < (min))
#define CONF_DONT_CHECK_MAX(um, max)	false
#define CONF_CHECK_MAX(um, max)	((um) > (max))

#define CONF_VALUE_READ(max_t, result)					\
	      char *end;						\
	      set_errno(0);						\
	      result = (max_t)malloc_strtoumax(v, &end, 0);
#define CONF_VALUE_READ_FAIL()						\
	      (get_errno() != 0 || (uintptr_t)end - (uintptr_t)v != vlen)

#define CONF_HANDLE_T(t, max_t, o, n, min, max, check_min, check_max, clip) \
			if (CONF_MATCH(n)) {				\
				max_t mv;				\
				CONF_VALUE_READ(max_t, mv)		\
				if (CONF_VALUE_READ_FAIL()) {		\
					CONF_ERROR("Invalid conf value",\
					    k, klen, v, vlen);		\
				} else if (clip) {			\
					if (check_min(mv, (t)(min))) {	\
						o = (t)(min);		\
					} else if (			\
					    check_max(mv, (t)(max))) {	\
						o = (t)(max);		\
					} else {			\
						o = (t)mv;		\
					}				\
				} else {				\
					if (check_min(mv, (t)(min)) ||	\
					    check_max(mv, (t)(max))) {	\
						CONF_ERROR(		\
						    "Out-of-range "	\
						    "conf value",	\
						    k, klen, v, vlen);	\
					} else {			\
						o = (t)mv;		\
					}				\
				}					\
				CONF_CONTINUE;				\
			}
#define CONF_HANDLE_T_U(t, o, n, min, max, check_min, check_max, clip)	\
	      CONF_HANDLE_T(t, uintmax_t, o, n, min, max, check_min,	\
			    check_max, clip)
#define CONF_HANDLE_T_SIGNED(t, o, n, min, max, check_min, check_max, clip)\
	      CONF_HANDLE_T(t, intmax_t, o, n, min, max, check_min,	\
			    check_max, clip)

#define CONF_HANDLE_UNSIGNED(o, n, min, max, check_min, check_max,	\
    clip)								\
			CONF_HANDLE_T_U(unsigned, o, n, min, max,	\
			    check_min, check_max, clip)
#define CONF_HANDLE_SIZE_T(o, n, min, max, check_min, check_max, clip)	\
			CONF_HANDLE_T_U(size_t, o, n, min, max,		\
			    check_min, check_max, clip)
#define CONF_HANDLE_INT64_T(o, n, min, max, check_min, check_max, clip)	\
			CONF_HANDLE_T_SIGNED(int64_t, o, n, min, max,	\
			    check_min, check_max, clip)
#define CONF_HANDLE_UINT64_T(o, n, min, max, check_min, check_max, clip)\
			CONF_HANDLE_T_U(uint64_t, o, n, min, max,	\
			    check_min, check_max, clip)
#define CONF_HANDLE_SSIZE_T(o, n, min, max)				\
			CONF_HANDLE_T_SIGNED(ssize_t, o, n, min, max,	\
			    CONF_CHECK_MIN, CONF_CHECK_MAX, false)
#define CONF_HANDLE_CHAR_P(o, n, d)					\
			if (CONF_MATCH(n)) {				\
				size_t cpylen = (vlen <=		\
				    sizeof(o)-1) ? vlen :		\
				    sizeof(o)-1;			\
				strncpy(o, v, cpylen);			\
				o[cpylen] = '\0';			\
				CONF_CONTINUE;				\
			}

			bool cur_opt_valid = true;

			CONF_HANDLE_BOOL(opt_confirm_conf, "confirm_conf")
			if (initial_call) {
				continue;
			}

			CONF_HANDLE_BOOL(opt_abort, "abort")
			CONF_HANDLE_BOOL(opt_abort_conf, "abort_conf")
			CONF_HANDLE_BOOL(opt_cache_oblivious, "cache_oblivious")
			CONF_HANDLE_BOOL(opt_trust_madvise, "trust_madvise")
			if (strncmp("metadata_thp", k, klen) == 0) {
				int m;
				bool match = false;
				for (m = 0; m < metadata_thp_mode_limit; m++) {
					if (strncmp(metadata_thp_mode_names[m],
					    v, vlen) == 0) {
						opt_metadata_thp = m;
						match = true;
						break;
					}
				}
				if (!match) {
					CONF_ERROR("Invalid conf value",
					    k, klen, v, vlen);
				}
				CONF_CONTINUE;
			}
			CONF_HANDLE_BOOL(opt_retain, "retain")
			if (strncmp("dss", k, klen) == 0) {
				int m;
				bool match = false;
				for (m = 0; m < dss_prec_limit; m++) {
					if (strncmp(dss_prec_names[m], v, vlen)
					    == 0) {
						if (extent_dss_prec_set(m)) {
							CONF_ERROR(
							    "Error setting dss",
							    k, klen, v, vlen);
						} else {
							opt_dss =
							    dss_prec_names[m];
							match = true;
							break;
						}
					}
				}
				if (!match) {
					CONF_ERROR("Invalid conf value",
					    k, klen, v, vlen);
				}
				CONF_CONTINUE;
			}
			if (CONF_MATCH("narenas")) {
				if (CONF_MATCH_VALUE("default")) {
					opt_narenas = 0;
					CONF_CONTINUE;
				} else {
					CONF_HANDLE_UNSIGNED(opt_narenas,
					    "narenas", 1, UINT_MAX,
					    CONF_CHECK_MIN, CONF_DONT_CHECK_MAX,
					    /* clip */ false)
				}
			}
			if (CONF_MATCH("narenas_ratio")) {
				char *end;
				bool err = fxp_parse(&opt_narenas_ratio, v,
				    &end);
				if (err || (size_t)(end - v) != vlen) {
					CONF_ERROR("Invalid conf value",
					    k, klen, v, vlen);
				}
				CONF_CONTINUE;
			}
			if (CONF_MATCH("bin_shards")) {
				const char *bin_shards_segment_cur = v;
				size_t vlen_left = vlen;
				do {
					size_t size_start;
					size_t size_end;
					size_t nshards;
					bool err = multi_setting_parse_next(
					    &bin_shards_segment_cur, &vlen_left,
					    &size_start, &size_end, &nshards);
					if (err || bin_update_shard_size(
					    bin_shard_sizes, size_start,
					    size_end, nshards)) {
						CONF_ERROR(
						    "Invalid settings for "
						    "bin_shards", k, klen, v,
						    vlen);
						break;
					}
				} while (vlen_left > 0);
				CONF_CONTINUE;
			}
			CONF_HANDLE_SIZE_T(opt_bin_info_max_batched_size,
			    "max_batched_size", 0, SIZE_T_MAX,
			    CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX,
			    /* clip */ true)
			CONF_HANDLE_SIZE_T(opt_bin_info_remote_free_max_batch,
			    "remote_free_max_batch", 0,
			    BIN_REMOTE_FREE_ELEMS_MAX,
			    CONF_DONT_CHECK_MIN, CONF_CHECK_MAX,
			    /* clip */ true)
			CONF_HANDLE_SIZE_T(opt_bin_info_remote_free_max,
			    "remote_free_max", 0,
			    BIN_REMOTE_FREE_ELEMS_MAX,
			    CONF_DONT_CHECK_MIN, CONF_CHECK_MAX,
			    /* clip */ true)

			if (CONF_MATCH("tcache_ncached_max")) {
				bool err = tcache_bin_info_default_init(
				    v, vlen);
				if (err) {
					CONF_ERROR("Invalid settings for "
					    "tcache_ncached_max", k, klen, v,
					    vlen);
				}
				CONF_CONTINUE;
			}
			CONF_HANDLE_INT64_T(opt_mutex_max_spin,
			    "mutex_max_spin", -1, INT64_MAX, CONF_CHECK_MIN,
			    CONF_DONT_CHECK_MAX, false);
			CONF_HANDLE_SSIZE_T(opt_dirty_decay_ms,
			    "dirty_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) <
			    QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) :
			    SSIZE_MAX);
			CONF_HANDLE_SSIZE_T(opt_muzzy_decay_ms,
			    "muzzy_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) <
			    QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) :
			    SSIZE_MAX);
			CONF_HANDLE_BOOL(opt_stats_print, "stats_print")
			if (CONF_MATCH("stats_print_opts")) {
				init_opt_stats_opts(v, vlen,
				    opt_stats_print_opts);
				CONF_CONTINUE;
			}
			CONF_HANDLE_INT64_T(opt_stats_interval,
			    "stats_interval", -1, INT64_MAX,
			    CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, false)
			if (CONF_MATCH("stats_interval_opts")) {
				init_opt_stats_opts(v, vlen,
				    opt_stats_interval_opts);
				CONF_CONTINUE;
			}
			if (config_fill) {
				if (CONF_MATCH("junk")) {
					if (CONF_MATCH_VALUE("true")) {
						opt_junk = "true";
						opt_junk_alloc = opt_junk_free =
						    true;
					} else if (CONF_MATCH_VALUE("false")) {
						opt_junk = "false";
						opt_junk_alloc = opt_junk_free =
						    false;
					} else if (CONF_MATCH_VALUE("alloc")) {
						opt_junk = "alloc";
						opt_junk_alloc = true;
						opt_junk_free = false;
					} else if (CONF_MATCH_VALUE("free")) {
						opt_junk = "free";
						opt_junk_alloc = false;
						opt_junk_free = true;
					} else {
						CONF_ERROR(
						    "Invalid conf value",
						    k, klen, v, vlen);
					}
					CONF_CONTINUE;
				}
				CONF_HANDLE_BOOL(opt_zero, "zero")
			}
			if (config_utrace) {
				CONF_HANDLE_BOOL(opt_utrace, "utrace")
			}
			if (config_xmalloc) {
				CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc")
			}
			if (config_enable_cxx) {
				CONF_HANDLE_BOOL(
				    opt_experimental_infallible_new,
				    "experimental_infallible_new")
			}

			CONF_HANDLE_BOOL(opt_experimental_tcache_gc,
			    "experimental_tcache_gc")
			CONF_HANDLE_BOOL(opt_tcache, "tcache")
			CONF_HANDLE_SIZE_T(opt_tcache_max, "tcache_max",
			    0, TCACHE_MAXCLASS_LIMIT, CONF_DONT_CHECK_MIN,
			    CONF_CHECK_MAX, /* clip */ true)
			if (CONF_MATCH("lg_tcache_max")) {
				size_t m;
				CONF_VALUE_READ(size_t, m)
				if (CONF_VALUE_READ_FAIL()) {
					CONF_ERROR("Invalid conf value",
					    k, klen, v, vlen);
				} else {
					/* clip if necessary */
					if (m > TCACHE_LG_MAXCLASS_LIMIT) {
						m = TCACHE_LG_MAXCLASS_LIMIT;
					}
					opt_tcache_max = (size_t)1 << m;
				}
				CONF_CONTINUE;
			}
			/*
			 * Anyone trying to set a value outside -16 to 16 is
			 * deeply confused.
			 */
			CONF_HANDLE_SSIZE_T(opt_lg_tcache_nslots_mul,
			    "lg_tcache_nslots_mul", -16, 16)
			/* Ditto with values past 2048. */
			CONF_HANDLE_UNSIGNED(opt_tcache_nslots_small_min,
			    "tcache_nslots_small_min", 1, 2048,
			    CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
			CONF_HANDLE_UNSIGNED(opt_tcache_nslots_small_max,
			    "tcache_nslots_small_max", 1, 2048,
			    CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
			CONF_HANDLE_UNSIGNED(opt_tcache_nslots_large,
			    "tcache_nslots_large", 1, 2048,
			    CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
			CONF_HANDLE_SIZE_T(opt_tcache_gc_incr_bytes,
			    "tcache_gc_incr_bytes", 1024, SIZE_T_MAX,
			    CONF_CHECK_MIN, CONF_DONT_CHECK_MAX,
			    /* clip */ true)
			CONF_HANDLE_SIZE_T(opt_tcache_gc_delay_bytes,
			    "tcache_gc_delay_bytes", 0, SIZE_T_MAX,
			    CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX,
			    /* clip */ false)
			CONF_HANDLE_UNSIGNED(opt_lg_tcache_flush_small_div,
			    "lg_tcache_flush_small_div", 1, 16,
			    CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
			CONF_HANDLE_UNSIGNED(opt_lg_tcache_flush_large_div,
			    "lg_tcache_flush_large_div", 1, 16,
			    CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
			CONF_HANDLE_UNSIGNED(opt_debug_double_free_max_scan,
			    "debug_double_free_max_scan", 0, UINT_MAX,
			    CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX,
			    /* clip */ false)
			CONF_HANDLE_SIZE_T(opt_calloc_madvise_threshold,
			    "calloc_madvise_threshold", 0, SC_LARGE_MAXCLASS,
			    CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, /* clip */ false)

			/*
			 * The runtime option of oversize_threshold remains
			 * undocumented.  It may be tweaked in the next major
			 * release (6.0).  The default value 8M is rather
			 * conservative / safe.  Tuning it further down may
			 * improve fragmentation a bit more, but may also cause
			 * contention on the huge arena.
			 */
			CONF_HANDLE_SIZE_T(opt_oversize_threshold,
			    "oversize_threshold", 0, SC_LARGE_MAXCLASS,
			    CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, false)
			CONF_HANDLE_SIZE_T(opt_lg_extent_max_active_fit,
			    "lg_extent_max_active_fit", 0,
			    (sizeof(size_t) << 3), CONF_DONT_CHECK_MIN,
			    CONF_CHECK_MAX, false)

			if (strncmp("percpu_arena", k, klen) == 0) {
				bool match = false;
				for (int m = percpu_arena_mode_names_base; m <
				    percpu_arena_mode_names_limit; m++) {
					if (strncmp(percpu_arena_mode_names[m],
					    v, vlen) == 0) {
						if (!have_percpu_arena) {
							CONF_ERROR(
							    "No getcpu support",
							    k, klen, v, vlen);
						}
						opt_percpu_arena = m;
						match = true;
						break;
					}
				}
				if (!match) {
					CONF_ERROR("Invalid conf value",
					    k, klen, v, vlen);
				}
				CONF_CONTINUE;
			}
			CONF_HANDLE_BOOL(opt_background_thread,
			    "background_thread");
			CONF_HANDLE_SIZE_T(opt_max_background_threads,
					   "max_background_threads", 1,
					   opt_max_background_threads,
					   CONF_CHECK_MIN, CONF_CHECK_MAX,
					   true);
			CONF_HANDLE_BOOL(opt_hpa, "hpa")
			CONF_HANDLE_SIZE_T(opt_hpa_opts.slab_max_alloc,
			    "hpa_slab_max_alloc", PAGE, HUGEPAGE,
			    CONF_CHECK_MIN, CONF_CHECK_MAX, true);

			/*
			 * Accept either a ratio-based or an exact hugification
			 * threshold.
			 */
			CONF_HANDLE_SIZE_T(opt_hpa_opts.hugification_threshold,
			    "hpa_hugification_threshold", PAGE, HUGEPAGE,
			    CONF_CHECK_MIN, CONF_CHECK_MAX, true);
			if (CONF_MATCH("hpa_hugification_threshold_ratio")) {
				fxp_t ratio;
				char *end;
				bool err = fxp_parse(&ratio, v,
				    &end);
				if (err || (size_t)(end - v) != vlen
				    || ratio > FXP_INIT_INT(1)) {
					CONF_ERROR("Invalid conf value",
					    k, klen, v, vlen);
				} else {
					opt_hpa_opts.hugification_threshold =
					    fxp_mul_frac(HUGEPAGE, ratio);
				}
				CONF_CONTINUE;
			}

			CONF_HANDLE_UINT64_T(
			    opt_hpa_opts.hugify_delay_ms, "hpa_hugify_delay_ms",
			    0, 0, CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX,
			    false);

			CONF_HANDLE_BOOL(
			    opt_hpa_opts.hugify_sync, "hpa_hugify_sync");

			CONF_HANDLE_UINT64_T(
			    opt_hpa_opts.min_purge_interval_ms,
			    "hpa_min_purge_interval_ms", 0, 0,
			    CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false);

			CONF_HANDLE_SSIZE_T(
			    opt_hpa_opts.experimental_max_purge_nhp,
			    "experimental_hpa_max_purge_nhp", -1, SSIZE_MAX);

			if (CONF_MATCH("hpa_dirty_mult")) {
				if (CONF_MATCH_VALUE("-1")) {
					opt_hpa_opts.dirty_mult = (fxp_t)-1;
					CONF_CONTINUE;
				}
				fxp_t ratio;
				char *end;
				bool err = fxp_parse(&ratio, v,
				    &end);
				if (err || (size_t)(end - v) != vlen) {
					CONF_ERROR("Invalid conf value",
					    k, klen, v, vlen);
				} else {
					opt_hpa_opts.dirty_mult = ratio;
				}
				CONF_CONTINUE;
			}

			CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.nshards,
			    "hpa_sec_nshards", 0, 0, CONF_CHECK_MIN,
			    CONF_DONT_CHECK_MAX, true);
			CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_alloc,
			    "hpa_sec_max_alloc", PAGE, 0, CONF_CHECK_MIN,
			    CONF_DONT_CHECK_MAX, true);
			CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_bytes,
			    "hpa_sec_max_bytes", PAGE, 0, CONF_CHECK_MIN,
			    CONF_DONT_CHECK_MAX, true);
			CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.bytes_after_flush,
			    "hpa_sec_bytes_after_flush", PAGE, 0,
			    CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
			CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.batch_fill_extra,
			    "hpa_sec_batch_fill_extra", 0, HUGEPAGE_PAGES,
			    CONF_CHECK_MIN, CONF_CHECK_MAX, true);

			if (CONF_MATCH("slab_sizes")) {
				if (CONF_MATCH_VALUE("default")) {
					sc_data_init(sc_data);
					CONF_CONTINUE;
				}
				bool err;
				const char *slab_size_segment_cur = v;
				size_t vlen_left = vlen;
				do {
					size_t slab_start;
					size_t slab_end;
					size_t pgs;
					err = multi_setting_parse_next(
					    &slab_size_segment_cur,
					    &vlen_left, &slab_start, &slab_end,
					    &pgs);
					if (!err) {
						sc_data_update_slab_size(
						    sc_data, slab_start,
						    slab_end, (int)pgs);
					} else {
						CONF_ERROR("Invalid settings "
						    "for slab_sizes",
						    k, klen, v, vlen);
					}
				} while (!err && vlen_left > 0);
				CONF_CONTINUE;
			}
			if (config_prof) {
				CONF_HANDLE_BOOL(opt_prof, "prof")
				CONF_HANDLE_CHAR_P(opt_prof_prefix,
				    "prof_prefix", "jeprof")
				CONF_HANDLE_BOOL(opt_prof_active, "prof_active")
				CONF_HANDLE_BOOL(opt_prof_thread_active_init,
				    "prof_thread_active_init")
				CONF_HANDLE_SIZE_T(opt_lg_prof_sample,
				    "lg_prof_sample", 0, (sizeof(uint64_t) << 3)
				    - 1, CONF_DONT_CHECK_MIN, CONF_CHECK_MAX,
				    true)
				CONF_HANDLE_SIZE_T(opt_experimental_lg_prof_threshold,
				    "experimental_lg_prof_threshold", 0, (sizeof(uint64_t) << 3)
				    - 1, CONF_DONT_CHECK_MIN, CONF_CHECK_MAX,
				    true)
				CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum")
				CONF_HANDLE_UNSIGNED(opt_prof_bt_max, "prof_bt_max",
				    1, PROF_BT_MAX_LIMIT, CONF_CHECK_MIN, CONF_CHECK_MAX,
				    /* clip */ true)
				CONF_HANDLE_SSIZE_T(opt_lg_prof_interval,
				    "lg_prof_interval", -1,
				    (sizeof(uint64_t) << 3) - 1)
				CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump")
				CONF_HANDLE_BOOL(opt_prof_final, "prof_final")
				CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak")
				CONF_HANDLE_BOOL(opt_prof_leak_error,
				    "prof_leak_error")
				CONF_HANDLE_BOOL(opt_prof_log, "prof_log")
				CONF_HANDLE_BOOL(opt_prof_pid_namespace, "prof_pid_namespace")
				CONF_HANDLE_SSIZE_T(opt_prof_recent_alloc_max,
				    "prof_recent_alloc_max", -1, SSIZE_MAX)
				CONF_HANDLE_BOOL(opt_prof_stats, "prof_stats")
				CONF_HANDLE_BOOL(opt_prof_sys_thread_name,
				    "prof_sys_thread_name")
				if (CONF_MATCH("prof_time_resolution")) {
					if (CONF_MATCH_VALUE("default")) {
						opt_prof_time_res =
						    prof_time_res_default;
					} else if (CONF_MATCH_VALUE("high")) {
						if (!config_high_res_timer) {
							CONF_ERROR(
							    "No high resolution"
							    " timer support",
							    k, klen, v, vlen);
						} else {
							opt_prof_time_res =
							    prof_time_res_high;
						}
					} else {
						CONF_ERROR("Invalid conf value",
						    k, klen, v, vlen);
					}
					CONF_CONTINUE;
				}
				/*
				 * Undocumented.  When set to false, don't
				 * correct for an unbiasing bug in jeprof
				 * attribution.  This can be handy if you want
				 * to get consistent numbers from your binary
				 * across different jemalloc versions, even if
				 * those numbers are incorrect.  The default is
				 * true.
				 */
				CONF_HANDLE_BOOL(opt_prof_unbias, "prof_unbias")
			}
			if (config_log) {
				if (CONF_MATCH("log")) {
					size_t cpylen = (
					    vlen <= sizeof(log_var_names) ?
					    vlen : sizeof(log_var_names) - 1);
					strncpy(log_var_names, v, cpylen);
					log_var_names[cpylen] = '\0';
					CONF_CONTINUE;
				}
			}
			if (CONF_MATCH("thp")) {
				bool match = false;
				for (int m = 0; m < thp_mode_names_limit; m++) {
					if (strncmp(thp_mode_names[m],v, vlen)
					    == 0) {
						if (!have_madvise_huge && !have_memcntl) {
							CONF_ERROR(
							    "No THP support",
							    k, klen, v, vlen);
						}
						opt_thp = m;
						match = true;
						break;
					}
				}
				if (!match) {
					CONF_ERROR("Invalid conf value",
					    k, klen, v, vlen);
				}
				CONF_CONTINUE;
			}
			if (CONF_MATCH("zero_realloc")) {
				if (CONF_MATCH_VALUE("alloc")) {
					opt_zero_realloc_action
					    = zero_realloc_action_alloc;
				} else if (CONF_MATCH_VALUE("free")) {
					opt_zero_realloc_action
					    = zero_realloc_action_free;
				} else if (CONF_MATCH_VALUE("abort")) {
					opt_zero_realloc_action
					    = zero_realloc_action_abort;
				} else {
					CONF_ERROR("Invalid conf value",
					    k, klen, v, vlen);
				}
				CONF_CONTINUE;
			}
			if (config_uaf_detection &&
			    CONF_MATCH("lg_san_uaf_align")) {
				ssize_t a;
				CONF_VALUE_READ(ssize_t, a)
				if (CONF_VALUE_READ_FAIL() || a < -1) {
					CONF_ERROR("Invalid conf value",
					    k, klen, v, vlen);
				}
				if (a == -1) {
					opt_lg_san_uaf_align = -1;
					CONF_CONTINUE;
				}

				/* clip if necessary */
				ssize_t max_allowed = (sizeof(size_t) << 3) - 1;
				ssize_t min_allowed = LG_PAGE;
				if (a > max_allowed) {
					a = max_allowed;
				} else if (a < min_allowed) {
					a = min_allowed;
				}

				opt_lg_san_uaf_align = a;
				CONF_CONTINUE;
			}

			CONF_HANDLE_SIZE_T(opt_san_guard_small,
			    "san_guard_small", 0, SIZE_T_MAX,
			    CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false)
			CONF_HANDLE_SIZE_T(opt_san_guard_large,
			    "san_guard_large", 0, SIZE_T_MAX,
			    CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false)

			CONF_ERROR("Invalid conf pair", k, klen, v, vlen);
#undef CONF_ERROR
#undef CONF_CONTINUE
#undef CONF_MATCH
#undef CONF_MATCH_VALUE
#undef CONF_HANDLE_BOOL
#undef CONF_DONT_CHECK_MIN
#undef CONF_CHECK_MIN
#undef CONF_DONT_CHECK_MAX
#undef CONF_CHECK_MAX
#undef CONF_HANDLE_T
#undef CONF_HANDLE_T_U
#undef CONF_HANDLE_T_SIGNED
#undef CONF_HANDLE_UNSIGNED
#undef CONF_HANDLE_SIZE_T
#undef CONF_HANDLE_SSIZE_T
#undef CONF_HANDLE_CHAR_P
    /* Re-enable diagnostic "-Wtype-limits" */
    JEMALLOC_DIAGNOSTIC_POP
		}
		validate_hpa_settings();
		if (opt_abort_conf && had_conf_error) {
			malloc_abort_invalid_conf();
		}
	}
	atomic_store_b(&log_init_done, true, ATOMIC_RELEASE);
}

static bool
malloc_conf_init_check_deps(void) {
	if (opt_prof_leak_error && !opt_prof_final) {
		malloc_printf("<jemalloc>: prof_leak_error is set w/o "
		    "prof_final.\n");
		return true;
	}
	/* To emphasize in the stats output that opt is disabled when !debug. */
	if (!config_debug) {
		opt_debug_double_free_max_scan = 0;
	}

	return false;
}

static void
malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
    char readlink_buf[PATH_MAX + 1]) {
	const char *opts_cache[MALLOC_CONF_NSOURCES] = {NULL, NULL, NULL, NULL,
		NULL};

	/* The first call only set the confirm_conf option and opts_cache */
	malloc_conf_init_helper(NULL, NULL, true, opts_cache, readlink_buf);
	malloc_conf_init_helper(sc_data, bin_shard_sizes, false, opts_cache,
	    NULL);
	if (malloc_conf_init_check_deps()) {
		/* check_deps does warning msg only; abort below if needed. */
		if (opt_abort_conf) {
			malloc_abort_invalid_conf();
		}
	}
}

#undef MALLOC_CONF_NSOURCES

static bool
malloc_init_hard_needed(void) {
	if (malloc_initialized() || (IS_INITIALIZER && malloc_init_state ==
	    malloc_init_recursible)) {
		/*
		 * Another thread initialized the allocator before this one
		 * acquired init_lock, or this thread is the initializing
		 * thread, and it is recursively allocating.
		 */
		return false;
	}
#ifdef JEMALLOC_THREADED_INIT
	if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) {
		/* Busy-wait until the initializing thread completes. */
		spin_t spinner = SPIN_INITIALIZER;
		do {
			malloc_mutex_unlock(TSDN_NULL, &init_lock);
			spin_adaptive(&spinner);
			malloc_mutex_lock(TSDN_NULL, &init_lock);
		} while (!malloc_initialized());
		return false;
	}
#endif
	return true;
}

static bool
malloc_init_hard_a0_locked(void) {
	malloc_initializer = INITIALIZER;

	JEMALLOC_DIAGNOSTIC_PUSH
	JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
	sc_data_t sc_data = {0};
	JEMALLOC_DIAGNOSTIC_POP

	/*
	 * Ordering here is somewhat tricky; we need sc_boot() first, since that
	 * determines what the size classes will be, and then
	 * malloc_conf_init(), since any slab size tweaking will need to be done
	 * before sz_boot and bin_info_boot, which assume that the values they
	 * read out of sc_data_global are final.
	 */
	sc_boot(&sc_data);
	unsigned bin_shard_sizes[SC_NBINS];
	bin_shard_sizes_boot(bin_shard_sizes);
	/*
	 * prof_boot0 only initializes opt_prof_prefix.  We need to do it before
	 * we parse malloc_conf options, in case malloc_conf parsing overwrites
	 * it.
	 */
	if (config_prof) {
		prof_boot0();
	}
	char readlink_buf[PATH_MAX + 1];
	readlink_buf[0] = '\0';
	malloc_conf_init(&sc_data, bin_shard_sizes, readlink_buf);
	san_init(opt_lg_san_uaf_align);
	sz_boot(&sc_data, opt_cache_oblivious);
	bin_info_boot(&sc_data, bin_shard_sizes);

	if (opt_stats_print) {
		/* Print statistics at exit. */
		if (atexit(stats_print_atexit) != 0) {
			malloc_write("<jemalloc>: Error in atexit()\n");
			if (opt_abort) {
				abort();
			}
		}
	}

	if (stats_boot()) {
		return true;
	}
	if (pages_boot()) {
		return true;
	}
	if (base_boot(TSDN_NULL)) {
		return true;
	}
	/* emap_global is static, hence zeroed. */
	if (emap_init(&arena_emap_global, b0get(), /* zeroed */ true)) {
		return true;
	}
	if (extent_boot()) {
		return true;
	}
	if (ctl_boot()) {
		return true;
	}
	if (config_prof) {
		prof_boot1();
	}
	if (opt_hpa && !hpa_supported()) {
		malloc_printf("<jemalloc>: HPA not supported in the current "
		    "configuration; %s.",
		    opt_abort_conf ? "aborting" : "disabling");
		if (opt_abort_conf) {
			malloc_abort_invalid_conf();
		} else {
			opt_hpa = false;
		}
	}
	if (arena_boot(&sc_data, b0get(), opt_hpa)) {
		return true;
	}
	if (tcache_boot(TSDN_NULL, b0get())) {
		return true;
	}
	if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS,
	    malloc_mutex_rank_exclusive)) {
		return true;
	}
	hook_boot();
	/*
	 * Create enough scaffolding to allow recursive allocation in
	 * malloc_ncpus().
	 */
	narenas_auto = 1;
	manual_arena_base = narenas_auto + 1;
	memset(arenas, 0, sizeof(arena_t *) * narenas_auto);
	/*
	 * Initialize one arena here.  The rest are lazily created in
	 * arena_choose_hard().
	 */
	if (arena_init(TSDN_NULL, 0, &arena_config_default) == NULL) {
		return true;
	}
	a0 = arena_get(TSDN_NULL, 0, false);

	if (opt_hpa && !hpa_supported()) {
		malloc_printf("<jemalloc>: HPA not supported in the current "
		    "configuration; %s.",
		    opt_abort_conf ? "aborting" : "disabling");
		if (opt_abort_conf) {
			malloc_abort_invalid_conf();
		} else {
			opt_hpa = false;
		}
	} else if (opt_hpa) {
		hpa_shard_opts_t hpa_shard_opts = opt_hpa_opts;
		hpa_shard_opts.deferral_allowed = background_thread_enabled();
		if (pa_shard_enable_hpa(TSDN_NULL, &a0->pa_shard,
		    &hpa_shard_opts, &opt_hpa_sec_opts)) {
			return true;
		}
	}

	malloc_init_state = malloc_init_a0_initialized;

	size_t buf_len = strlen(readlink_buf);
	if (buf_len > 0) {
		void *readlink_allocated = a0ialloc(buf_len + 1, false, true);
		if (readlink_allocated != NULL) {
			memcpy(readlink_allocated, readlink_buf, buf_len + 1);
			opt_malloc_conf_symlink = readlink_allocated;
		}
	}

	return false;
}

static bool
malloc_init_hard_a0(void) {
	bool ret;

	malloc_mutex_lock(TSDN_NULL, &init_lock);
	ret = malloc_init_hard_a0_locked();
	malloc_mutex_unlock(TSDN_NULL, &init_lock);
	return ret;
}

/* Initialize data structures which may trigger recursive allocation. */
static bool
malloc_init_hard_recursible(void) {
	malloc_init_state = malloc_init_recursible;

	ncpus = malloc_ncpus();
	if (opt_percpu_arena != percpu_arena_disabled) {
		bool cpu_count_is_deterministic =
		    malloc_cpu_count_is_deterministic();
		if (!cpu_count_is_deterministic) {
			/*
			 * If # of CPU is not deterministic, and narenas not
			 * specified, disables per cpu arena since it may not
			 * detect CPU IDs properly.
			 */
			if (opt_narenas == 0) {
				opt_percpu_arena = percpu_arena_disabled;
				malloc_write("<jemalloc>: Number of CPUs "
				    "detected is not deterministic. Per-CPU "
				    "arena disabled.\n");
				if (opt_abort_conf) {
					malloc_abort_invalid_conf();
				}
				if (opt_abort) {
					abort();
				}
			}
		}
	}

#if (defined(JEMALLOC_HAVE_PTHREAD_ATFORK) && !defined(JEMALLOC_MUTEX_INIT_CB) \
    && !defined(JEMALLOC_ZONE) && !defined(_WIN32) && \
    !defined(__native_client__))
	/* LinuxThreads' pthread_atfork() allocates. */
	if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent,
	    jemalloc_postfork_child) != 0) {
		malloc_write("<jemalloc>: Error in pthread_atfork()\n");
		if (opt_abort) {
			abort();
		}
		return true;
	}
#endif

	if (background_thread_boot0()) {
		return true;
	}

	return false;
}

static unsigned
malloc_narenas_default(void) {
	assert(ncpus > 0);
	/*
	 * For SMP systems, create more than one arena per CPU by
	 * default.
	 */
	if (ncpus > 1) {
		fxp_t fxp_ncpus = FXP_INIT_INT(ncpus);
		fxp_t goal = fxp_mul(fxp_ncpus, opt_narenas_ratio);
		uint32_t int_goal = fxp_round_nearest(goal);
		if (int_goal == 0) {
			return 1;
		}
		return int_goal;
	} else {
		return 1;
	}
}

static percpu_arena_mode_t
percpu_arena_as_initialized(percpu_arena_mode_t mode) {
	assert(!malloc_initialized());
	assert(mode <= percpu_arena_disabled);

	if (mode != percpu_arena_disabled) {
		mode += percpu_arena_mode_enabled_base;
	}

	return mode;
}

static bool
malloc_init_narenas(void) {
	assert(ncpus > 0);

	if (opt_percpu_arena != percpu_arena_disabled) {
		if (!have_percpu_arena || malloc_getcpu() < 0) {
			opt_percpu_arena = percpu_arena_disabled;
			malloc_printf("<jemalloc>: perCPU arena getcpu() not "
			    "available. Setting narenas to %u.\n", opt_narenas ?
			    opt_narenas : malloc_narenas_default());
			if (opt_abort) {
				abort();
			}
		} else {
			if (ncpus >= MALLOCX_ARENA_LIMIT) {
				malloc_printf("<jemalloc>: narenas w/ percpu"
				    "arena beyond limit (%d)\n", ncpus);
				if (opt_abort) {
					abort();
				}
				return true;
			}
			/* NB: opt_percpu_arena isn't fully initialized yet. */
			if (percpu_arena_as_initialized(opt_percpu_arena) ==
			    per_phycpu_arena && ncpus % 2 != 0) {
				malloc_printf("<jemalloc>: invalid "
				    "configuration -- per physical CPU arena "
				    "with odd number (%u) of CPUs (no hyper "
				    "threading?).\n", ncpus);
				if (opt_abort)
					abort();
			}
			unsigned n = percpu_arena_ind_limit(
			    percpu_arena_as_initialized(opt_percpu_arena));
			if (opt_narenas < n) {
				/*
				 * If narenas is specified with percpu_arena
				 * enabled, actual narenas is set as the greater
				 * of the two. percpu_arena_choose will be free
				 * to use any of the arenas based on CPU
				 * id. This is conservative (at a small cost)
				 * but ensures correctness.
				 *
				 * If for some reason the ncpus determined at
				 * boot is not the actual number (e.g. because
				 * of affinity setting from numactl), reserving
				 * narenas this way provides a workaround for
				 * percpu_arena.
				 */
				opt_narenas = n;
			}
		}
	}
	if (opt_narenas == 0) {
		opt_narenas = malloc_narenas_default();
	}
	assert(opt_narenas > 0);

	narenas_auto = opt_narenas;
	/*
	 * Limit the number of arenas to the indexing range of MALLOCX_ARENA().
	 */
	if (narenas_auto >= MALLOCX_ARENA_LIMIT) {
		narenas_auto = MALLOCX_ARENA_LIMIT - 1;
		malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n",
		    narenas_auto);
	}
	narenas_total_set(narenas_auto);
	if (arena_init_huge(a0)) {
		narenas_total_inc();
	}
	manual_arena_base = narenas_total_get();

	return false;
}

static void
malloc_init_percpu(void) {
	opt_percpu_arena = percpu_arena_as_initialized(opt_percpu_arena);
}

static bool
malloc_init_hard_finish(void) {
	if (malloc_mutex_boot()) {
		return true;
	}

	malloc_init_state = malloc_init_initialized;
	malloc_slow_flag_init();

	return false;
}

static void
malloc_init_hard_cleanup(tsdn_t *tsdn, bool reentrancy_set) {
	malloc_mutex_assert_owner(tsdn, &init_lock);
	malloc_mutex_unlock(tsdn, &init_lock);
	if (reentrancy_set) {
		assert(!tsdn_null(tsdn));
		tsd_t *tsd = tsdn_tsd(tsdn);
		assert(tsd_reentrancy_level_get(tsd) > 0);
		post_reentrancy(tsd);
	}
}

static bool
malloc_init_hard(void) {
	tsd_t *tsd;

#if defined(_WIN32) && _WIN32_WINNT < 0x0600
	_init_init_lock();
#endif
	malloc_mutex_lock(TSDN_NULL, &init_lock);

#define UNLOCK_RETURN(tsdn, ret, reentrancy)		\
	malloc_init_hard_cleanup(tsdn, reentrancy);	\
	return ret;

	if (!malloc_init_hard_needed()) {
		UNLOCK_RETURN(TSDN_NULL, false, false)
	}

	if (malloc_init_state != malloc_init_a0_initialized &&
	    malloc_init_hard_a0_locked()) {
		UNLOCK_RETURN(TSDN_NULL, true, false)
	}

	malloc_mutex_unlock(TSDN_NULL, &init_lock);
	/* Recursive allocation relies on functional tsd. */
	tsd = malloc_tsd_boot0();
	if (tsd == NULL) {
		return true;
	}
	if (malloc_init_hard_recursible()) {
		return true;
	}

	malloc_mutex_lock(tsd_tsdn(tsd), &init_lock);
	/* Set reentrancy level to 1 during init. */
	pre_reentrancy(tsd, NULL);
	/* Initialize narenas before prof_boot2 (for allocation). */
	if (malloc_init_narenas()
	    || background_thread_boot1(tsd_tsdn(tsd), b0get())) {
		UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
	}
	if (config_prof && prof_boot2(tsd, b0get())) {
		UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
	}

	malloc_init_percpu();

	if (malloc_init_hard_finish()) {
		UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
	}
	post_reentrancy(tsd);
	malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock);

	witness_assert_lockless(witness_tsd_tsdn(
	    tsd_witness_tsdp_get_unsafe(tsd)));
	malloc_tsd_boot1();
	/* Update TSD after tsd_boot1. */
	tsd = tsd_fetch();
	if (opt_background_thread) {
		assert(have_background_thread);
		/*
		 * Need to finish init & unlock first before creating background
		 * threads (pthread_create depends on malloc).  ctl_init (which
		 * sets isthreaded) needs to be called without holding any lock.
		 */
		background_thread_ctl_init(tsd_tsdn(tsd));
		if (background_thread_create(tsd, 0)) {
			return true;
		}
	}
#undef UNLOCK_RETURN
	return false;
}

/*
 * End initialization functions.
 */
/******************************************************************************/
/*
 * Begin allocation-path internal functions and data structures.
 */

/*
 * Settings determined by the documented behavior of the allocation functions.
 */
typedef struct static_opts_s static_opts_t;
struct static_opts_s {
	/* Whether or not allocation size may overflow. */
	bool may_overflow;

	/*
	 * Whether or not allocations (with alignment) of size 0 should be
	 * treated as size 1.
	 */
	bool bump_empty_aligned_alloc;
	/*
	 * Whether to assert that allocations are not of size 0 (after any
	 * bumping).
	 */
	bool assert_nonempty_alloc;

	/*
	 * Whether or not to modify the 'result' argument to malloc in case of
	 * error.
	 */
	bool null_out_result_on_error;
	/* Whether to set errno when we encounter an error condition. */
	bool set_errno_on_error;

	/*
	 * The minimum valid alignment for functions requesting aligned storage.
	 */
	size_t min_alignment;

	/* The error string to use if we oom. */
	const char *oom_string;
	/* The error string to use if the passed-in alignment is invalid. */
	const char *invalid_alignment_string;

	/*
	 * False if we're configured to skip some time-consuming operations.
	 *
	 * This isn't really a malloc "behavior", but it acts as a useful
	 * summary of several other static (or at least, static after program
	 * initialization) options.
	 */
	bool slow;
	/*
	 * Return size.
	 */
	bool usize;
};

JEMALLOC_ALWAYS_INLINE void
static_opts_init(static_opts_t *static_opts) {
	static_opts->may_overflow = false;
	static_opts->bump_empty_aligned_alloc = false;
	static_opts->assert_nonempty_alloc = false;
	static_opts->null_out_result_on_error = false;
	static_opts->set_errno_on_error = false;
	static_opts->min_alignment = 0;
	static_opts->oom_string = "";
	static_opts->invalid_alignment_string = "";
	static_opts->slow = false;
	static_opts->usize = false;
}

typedef struct dynamic_opts_s dynamic_opts_t;
struct dynamic_opts_s {
	void **result;
	size_t usize;
	size_t num_items;
	size_t item_size;
	size_t alignment;
	bool zero;
	unsigned tcache_ind;
	unsigned arena_ind;
};

JEMALLOC_ALWAYS_INLINE void
dynamic_opts_init(dynamic_opts_t *dynamic_opts) {
	dynamic_opts->result = NULL;
	dynamic_opts->usize = 0;
	dynamic_opts->num_items = 0;
	dynamic_opts->item_size = 0;
	dynamic_opts->alignment = 0;
	dynamic_opts->zero = false;
	dynamic_opts->tcache_ind = TCACHE_IND_AUTOMATIC;
	dynamic_opts->arena_ind = ARENA_IND_AUTOMATIC;
}

/*
 * ind parameter is optional and is only checked and filled if alignment == 0;
 * return true if result is out of range.
 */
JEMALLOC_ALWAYS_INLINE bool
aligned_usize_get(size_t size, size_t alignment, size_t *usize, szind_t *ind,
    bool bump_empty_aligned_alloc) {
	assert(usize != NULL);
	if (alignment == 0) {
		if (ind != NULL) {
			*ind = sz_size2index(size);
			if (unlikely(*ind >= SC_NSIZES)) {
				return true;
			}
			*usize = sz_index2size(*ind);
			assert(*usize > 0 && *usize <= SC_LARGE_MAXCLASS);
			return false;
		}
		*usize = sz_s2u(size);
	} else {
		if (bump_empty_aligned_alloc && unlikely(size == 0)) {
			size = 1;
		}
		*usize = sz_sa2u(size, alignment);
	}
	if (unlikely(*usize == 0 || *usize > SC_LARGE_MAXCLASS)) {
		return true;
	}
	return false;
}

JEMALLOC_ALWAYS_INLINE bool
zero_get(bool guarantee, bool slow) {
	if (config_fill && slow && unlikely(opt_zero)) {
		return true;
	} else {
		return guarantee;
	}
}

/* Return true if a manual arena is specified and arena_get() OOMs. */
JEMALLOC_ALWAYS_INLINE bool
arena_get_from_ind(tsd_t *tsd, unsigned arena_ind, arena_t **arena_p) {
	if (arena_ind == ARENA_IND_AUTOMATIC) {
		/*
		 * In case of automatic arena management, we defer arena
		 * computation until as late as we can, hoping to fill the
		 * allocation out of the tcache.
		 */
		*arena_p = NULL;
	} else {
		*arena_p = arena_get(tsd_tsdn(tsd), arena_ind, true);
		if (unlikely(*arena_p == NULL) && arena_ind >= narenas_auto) {
			return true;
		}
	}
	return false;
}

/* ind is ignored if dopts->alignment > 0. */
JEMALLOC_ALWAYS_INLINE void *
imalloc_no_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd,
    size_t size, size_t usize, szind_t ind, bool slab) {
	/* Fill in the tcache. */
	tcache_t *tcache = tcache_get_from_ind(tsd, dopts->tcache_ind,
	    sopts->slow, /* is_alloc */ true);

	/* Fill in the arena. */
	arena_t *arena;
	if (arena_get_from_ind(tsd, dopts->arena_ind, &arena)) {
		return NULL;
	}

	if (unlikely(dopts->alignment != 0)) {
		return ipalloct_explicit_slab(tsd_tsdn(tsd), usize,
		    dopts->alignment, dopts->zero, slab, tcache, arena);
	}

	return iallocztm_explicit_slab(tsd_tsdn(tsd), size, ind, dopts->zero,
	    slab, tcache, false, arena, sopts->slow);
}

JEMALLOC_ALWAYS_INLINE void *
imalloc_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd,
    size_t usize, szind_t ind) {
	void *ret;

	dopts->alignment = prof_sample_align(usize, dopts->alignment);
	/*
	 * If the allocation is small enough that it would normally be allocated
	 * on a slab, we need to take additional steps to ensure that it gets
	 * its own extent instead.
	 */
	if (sz_can_use_slab(usize)) {
		assert((dopts->alignment & PROF_SAMPLE_ALIGNMENT_MASK) == 0);
		size_t bumped_usize = sz_sa2u(usize, dopts->alignment);
		szind_t bumped_ind = sz_size2index(bumped_usize);
		dopts->tcache_ind = TCACHE_IND_NONE;
		ret = imalloc_no_sample(sopts, dopts, tsd, bumped_usize,
		    bumped_usize, bumped_ind, /* slab */ false);
		if (unlikely(ret == NULL)) {
			return NULL;
		}
		arena_prof_promote(tsd_tsdn(tsd), ret, usize, bumped_usize);
	} else {
		ret = imalloc_no_sample(sopts, dopts, tsd, usize, usize, ind,
		    /* slab */ false);
	}
	assert(prof_sample_aligned(ret));

	return ret;
}

/*
 * Returns true if the allocation will overflow, and false otherwise.  Sets
 * *size to the product either way.
 */
JEMALLOC_ALWAYS_INLINE bool
compute_size_with_overflow(bool may_overflow, dynamic_opts_t *dopts,
    size_t *size) {
	/*
	 * This function is just num_items * item_size, except that we may have
	 * to check for overflow.
	 */

	if (!may_overflow) {
		assert(dopts->num_items == 1);
		*size = dopts->item_size;
		return false;
	}

	/* A size_t with its high-half bits all set to 1. */
	static const size_t high_bits = SIZE_T_MAX << (sizeof(size_t) * 8 / 2);

	*size = dopts->item_size * dopts->num_items;

	if (unlikely(*size == 0)) {
		return (dopts->num_items != 0 && dopts->item_size != 0);
	}

	/*
	 * We got a non-zero size, but we don't know if we overflowed to get
	 * there.  To avoid having to do a divide, we'll be clever and note that
	 * if both A and B can be represented in N/2 bits, then their product
	 * can be represented in N bits (without the possibility of overflow).
	 */
	if (likely((high_bits & (dopts->num_items | dopts->item_size)) == 0)) {
		return false;
	}
	if (likely(*size / dopts->item_size == dopts->num_items)) {
		return false;
	}
	return true;
}

JEMALLOC_ALWAYS_INLINE int
imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
	/* Where the actual allocated memory will live. */
	void *allocation = NULL;
	/* Filled in by compute_size_with_overflow below. */
	size_t size = 0;
	/*
	 * The zero initialization for ind is actually dead store, in that its
	 * value is reset before any branch on its value is taken.  Sometimes
	 * though, it's convenient to pass it as arguments before this point.
	 * To avoid undefined behavior then, we initialize it with dummy stores.
	 */
	szind_t ind = 0;
	/* usize will always be properly initialized. */
	size_t usize;

	/* Reentrancy is only checked on slow path. */
	int8_t reentrancy_level;

	/* Compute the amount of memory the user wants. */
	if (unlikely(compute_size_with_overflow(sopts->may_overflow, dopts,
	    &size))) {
		goto label_oom;
	}

	if (unlikely(dopts->alignment < sopts->min_alignment
	    || (dopts->alignment & (dopts->alignment - 1)) != 0)) {
		goto label_invalid_alignment;
	}

	/* This is the beginning of the "core" algorithm. */
	dopts->zero = zero_get(dopts->zero, sopts->slow);
	if (aligned_usize_get(size, dopts->alignment, &usize, &ind,
	    sopts->bump_empty_aligned_alloc)) {
		goto label_oom;
	}
	dopts->usize = usize;
	/* Validate the user input. */
	if (sopts->assert_nonempty_alloc) {
		assert (size != 0);
	}

	check_entry_exit_locking(tsd_tsdn(tsd));

	/*
	 * If we need to handle reentrancy, we can do it out of a
	 * known-initialized arena (i.e. arena 0).
	 */
	reentrancy_level = tsd_reentrancy_level_get(tsd);
	if (sopts->slow && unlikely(reentrancy_level > 0)) {
		/*
		 * We should never specify particular arenas or tcaches from
		 * within our internal allocations.
		 */
		assert(dopts->tcache_ind == TCACHE_IND_AUTOMATIC ||
		    dopts->tcache_ind == TCACHE_IND_NONE);
		assert(dopts->arena_ind == ARENA_IND_AUTOMATIC);
		dopts->tcache_ind = TCACHE_IND_NONE;
		/* We know that arena 0 has already been initialized. */
		dopts->arena_ind = 0;
	}

	/*
	 * If dopts->alignment > 0, then ind is still 0, but usize was computed
	 * in the previous if statement.  Down the positive alignment path,
	 * imalloc_no_sample and imalloc_sample will ignore ind.
	 */

	/* If profiling is on, get our profiling context. */
	if (config_prof && opt_prof) {
		bool prof_active = prof_active_get_unlocked();
		bool sample_event = te_prof_sample_event_lookahead(tsd, usize);
		prof_tctx_t *tctx = prof_alloc_prep(tsd, prof_active,
		    sample_event);

		emap_alloc_ctx_t alloc_ctx;
		if (likely(tctx == PROF_TCTX_SENTINEL)) {
			alloc_ctx.slab = sz_can_use_slab(usize);
			allocation = imalloc_no_sample(
			    sopts, dopts, tsd, usize, usize, ind,
			    alloc_ctx.slab);
		} else if (tctx != NULL) {
			allocation = imalloc_sample(
			    sopts, dopts, tsd, usize, ind);
			alloc_ctx.slab = false;
		} else {
			allocation = NULL;
		}

		if (unlikely(allocation == NULL)) {
			prof_alloc_rollback(tsd, tctx);
			goto label_oom;
		}
		prof_malloc(tsd, allocation, size, usize, &alloc_ctx, tctx);
	} else {
		assert(!opt_prof);
		allocation = imalloc_no_sample(sopts, dopts, tsd, size, usize,
		    ind, sz_can_use_slab(usize));
		if (unlikely(allocation == NULL)) {
			goto label_oom;
		}
	}

	/*
	 * Allocation has been done at this point.  We still have some
	 * post-allocation work to do though.
	 */

	thread_alloc_event(tsd, usize);

	assert(dopts->alignment == 0
	    || ((uintptr_t)allocation & (dopts->alignment - 1)) == ZU(0));

	assert(usize == isalloc(tsd_tsdn(tsd), allocation));

	if (config_fill && sopts->slow && !dopts->zero
	    && unlikely(opt_junk_alloc)) {
		junk_alloc_callback(allocation, usize);
	}

	if (sopts->slow) {
		UTRACE(0, size, allocation);
	}

	/* Success! */
	check_entry_exit_locking(tsd_tsdn(tsd));
	*dopts->result = allocation;
	return 0;

label_oom:
	if (unlikely(sopts->slow) && config_xmalloc && unlikely(opt_xmalloc)) {
		malloc_write(sopts->oom_string);
		abort();
	}

	if (sopts->slow) {
		UTRACE(NULL, size, NULL);
	}

	check_entry_exit_locking(tsd_tsdn(tsd));

	if (sopts->set_errno_on_error) {
		set_errno(ENOMEM);
	}

	if (sopts->null_out_result_on_error) {
		*dopts->result = NULL;
	}

	return ENOMEM;

	/*
	 * This label is only jumped to by one goto; we move it out of line
	 * anyways to avoid obscuring the non-error paths, and for symmetry with
	 * the oom case.
	 */
label_invalid_alignment:
	if (config_xmalloc && unlikely(opt_xmalloc)) {
		malloc_write(sopts->invalid_alignment_string);
		abort();
	}

	if (sopts->set_errno_on_error) {
		set_errno(EINVAL);
	}

	if (sopts->slow) {
		UTRACE(NULL, size, NULL);
	}

	check_entry_exit_locking(tsd_tsdn(tsd));

	if (sopts->null_out_result_on_error) {
		*dopts->result = NULL;
	}

	return EINVAL;
}

JEMALLOC_ALWAYS_INLINE bool
imalloc_init_check(static_opts_t *sopts, dynamic_opts_t *dopts) {
	if (unlikely(!malloc_initialized()) && unlikely(malloc_init())) {
		if (config_xmalloc && unlikely(opt_xmalloc)) {
			malloc_write(sopts->oom_string);
			abort();
		}
		UTRACE(NULL, dopts->num_items * dopts->item_size, NULL);
		set_errno(ENOMEM);
		*dopts->result = NULL;

		return false;
	}

	return true;
}

/* Returns the errno-style error code of the allocation. */
JEMALLOC_ALWAYS_INLINE int
imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) {
	if (tsd_get_allocates() && !imalloc_init_check(sopts, dopts)) {
		return ENOMEM;
	}

	/* We always need the tsd.  Let's grab it right away. */
	tsd_t *tsd = tsd_fetch();
	assert(tsd);
	if (likely(tsd_fast(tsd))) {
		/* Fast and common path. */
		tsd_assert_fast(tsd);
		sopts->slow = false;
		return imalloc_body(sopts, dopts, tsd);
	} else {
		if (!tsd_get_allocates() && !imalloc_init_check(sopts, dopts)) {
			return ENOMEM;
		}

		sopts->slow = true;
		return imalloc_body(sopts, dopts, tsd);
	}
}

JEMALLOC_NOINLINE
void *
malloc_default(size_t size) {
	void *ret;
	static_opts_t sopts;
	dynamic_opts_t dopts;

	/*
	 * This variant has logging hook on exit but not on entry.  It's callled
	 * only by je_malloc, below, which emits the entry one for us (and, if
	 * it calls us, does so only via tail call).
	 */

	static_opts_init(&sopts);
	dynamic_opts_init(&dopts);

	sopts.null_out_result_on_error = true;
	sopts.set_errno_on_error = true;
	sopts.oom_string = "<jemalloc>: Error in malloc(): out of memory\n";

	dopts.result = &ret;
	dopts.num_items = 1;
	dopts.item_size = size;

	imalloc(&sopts, &dopts);
	/*
	 * Note that this branch gets optimized away -- it immediately follows
	 * the check on tsd_fast that sets sopts.slow.
	 */
	if (sopts.slow) {
		uintptr_t args[3] = {size};
		hook_invoke_alloc(hook_alloc_malloc, ret, (uintptr_t)ret, args);
	}

	return ret;
}

/******************************************************************************/
/*
 * Begin malloc(3)-compatible functions.
 */

JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *
JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1)
je_malloc(size_t size) {
	LOG("core.malloc.entry", "size: %zu", size);

	void * ret = imalloc_fastpath(size, &malloc_default);

	LOG("core.malloc.exit", "result: %p", ret);
	return ret;
}

JEMALLOC_EXPORT int JEMALLOC_NOTHROW
JEMALLOC_ATTR(nonnull(1))
je_posix_memalign(void **memptr, size_t alignment, size_t size) {
	int ret;
	static_opts_t sopts;
	dynamic_opts_t dopts;

	LOG("core.posix_memalign.entry", "mem ptr: %p, alignment: %zu, "
	    "size: %zu", memptr, alignment, size);

	static_opts_init(&sopts);
	dynamic_opts_init(&dopts);

	sopts.bump_empty_aligned_alloc = true;
	sopts.min_alignment = sizeof(void *);
	sopts.oom_string =
	    "<jemalloc>: Error allocating aligned memory: out of memory\n";
	sopts.invalid_alignment_string =
	    "<jemalloc>: Error allocating aligned memory: invalid alignment\n";

	dopts.result = memptr;
	dopts.num_items = 1;
	dopts.item_size = size;
	dopts.alignment = alignment;

	ret = imalloc(&sopts, &dopts);
	if (sopts.slow) {
		uintptr_t args[3] = {(uintptr_t)memptr, (uintptr_t)alignment,
			(uintptr_t)size};
		hook_invoke_alloc(hook_alloc_posix_memalign, *memptr,
		    (uintptr_t)ret, args);
	}

	LOG("core.posix_memalign.exit", "result: %d, alloc ptr: %p", ret,
	    *memptr);

	return ret;
}

JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *
JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(2)
je_aligned_alloc(size_t alignment, size_t size) {
	void *ret;

	static_opts_t sopts;
	dynamic_opts_t dopts;

	LOG("core.aligned_alloc.entry", "alignment: %zu, size: %zu\n",
	    alignment, size);

	static_opts_init(&sopts);
	dynamic_opts_init(&dopts);

	sopts.bump_empty_aligned_alloc = true;
	sopts.null_out_result_on_error = true;
	sopts.set_errno_on_error = true;
	sopts.min_alignment = 1;
	sopts.oom_string =
	    "<jemalloc>: Error allocating aligned memory: out of memory\n";
	sopts.invalid_alignment_string =
	    "<jemalloc>: Error allocating aligned memory: invalid alignment\n";

	dopts.result = &ret;
	dopts.num_items = 1;
	dopts.item_size = size;
	dopts.alignment = alignment;

	imalloc(&sopts, &dopts);
	if (sopts.slow) {
		uintptr_t args[3] = {(uintptr_t)alignment, (uintptr_t)size};
		hook_invoke_alloc(hook_alloc_aligned_alloc, ret,
		    (uintptr_t)ret, args);
	}

	LOG("core.aligned_alloc.exit", "result: %p", ret);

	return ret;
}

JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *
JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2)
je_calloc(size_t num, size_t size) {
	void *ret;
	static_opts_t sopts;
	dynamic_opts_t dopts;

	LOG("core.calloc.entry", "num: %zu, size: %zu", num, size);

	static_opts_init(&sopts);
	dynamic_opts_init(&dopts);

	sopts.may_overflow = true;
	sopts.null_out_result_on_error = true;
	sopts.set_errno_on_error = true;
	sopts.oom_string = "<jemalloc>: Error in calloc(): out of memory\n";

	dopts.result = &ret;
	dopts.num_items = num;
	dopts.item_size = size;
	dopts.zero = true;

	imalloc(&sopts, &dopts);
	if (sopts.slow) {
		uintptr_t args[3] = {(uintptr_t)num, (uintptr_t)size};
		hook_invoke_alloc(hook_alloc_calloc, ret, (uintptr_t)ret, args);
	}

	LOG("core.calloc.exit", "result: %p", ret);

	return ret;
}

JEMALLOC_ALWAYS_INLINE void
ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
	if (!slow_path) {
		tsd_assert_fast(tsd);
	}
	check_entry_exit_locking(tsd_tsdn(tsd));
	if (tsd_reentrancy_level_get(tsd) != 0) {
		assert(slow_path);
	}

	assert(ptr != NULL);
	assert(malloc_initialized() || IS_INITIALIZER);

	emap_alloc_ctx_t alloc_ctx;
	emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr,
	    &alloc_ctx);
	assert(alloc_ctx.szind != SC_NSIZES);

	size_t usize = sz_index2size(alloc_ctx.szind);
	if (config_prof && opt_prof) {
		prof_free(tsd, ptr, usize, &alloc_ctx);
	}

	if (likely(!slow_path)) {
		idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false,
		    false);
	} else {
		if (config_fill && slow_path && opt_junk_free) {
			junk_free_callback(ptr, usize);
		}
		idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false,
		    true);
	}
	thread_dalloc_event(tsd, usize);
}

JEMALLOC_ALWAYS_INLINE void
isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) {
	if (!slow_path) {
		tsd_assert_fast(tsd);
	}
	check_entry_exit_locking(tsd_tsdn(tsd));
	if (tsd_reentrancy_level_get(tsd) != 0) {
		assert(slow_path);
	}

	assert(ptr != NULL);
	assert(malloc_initialized() || IS_INITIALIZER);

	emap_alloc_ctx_t alloc_ctx;
	if (!config_prof) {
		alloc_ctx.szind = sz_size2index(usize);
		alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
	} else {
		if (likely(!prof_sample_aligned(ptr))) {
			/*
			 * When the ptr is not page aligned, it was not sampled.
			 * usize can be trusted to determine szind and slab.
			 */
			alloc_ctx.szind = sz_size2index(usize);
			alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
		} else if (opt_prof) {
			emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global,
			    ptr, &alloc_ctx);

			if (config_opt_safety_checks) {
				/* Small alloc may have !slab (sampled). */
				if (unlikely(alloc_ctx.szind !=
				    sz_size2index(usize))) {
					safety_check_fail_sized_dealloc(
					    /* current_dealloc */ true, ptr,
					    /* true_size */ sz_index2size(
					    alloc_ctx.szind),
					    /* input_size */ usize);
				}
			}
		} else {
			alloc_ctx.szind = sz_size2index(usize);
			alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
		}
	}
	bool fail = maybe_check_alloc_ctx(tsd, ptr, &alloc_ctx);
	if (fail) {
		/*
		 * This is a heap corruption bug.  In real life we'll crash; for
		 * the unit test we just want to avoid breaking anything too
		 * badly to get a test result out.  Let's leak instead of trying
		 * to free.
		 */
		return;
	}

	if (config_prof && opt_prof) {
		prof_free(tsd, ptr, usize, &alloc_ctx);
	}
	if (likely(!slow_path)) {
		isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, &alloc_ctx,
		    false);
	} else {
		if (config_fill && slow_path && opt_junk_free) {
			junk_free_callback(ptr, usize);
		}
		isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, &alloc_ctx,
		    true);
	}
	thread_dalloc_event(tsd, usize);
}

JEMALLOC_NOINLINE
void
free_default(void *ptr) {
	UTRACE(ptr, 0, 0);
	if (likely(ptr != NULL)) {
		/*
		 * We avoid setting up tsd fully (e.g. tcache, arena binding)
		 * based on only free() calls -- other activities trigger the
		 * minimal to full transition.  This is because free() may
		 * happen during thread shutdown after tls deallocation: if a
		 * thread never had any malloc activities until then, a
		 * fully-setup tsd won't be destructed properly.
		 */
		tsd_t *tsd = tsd_fetch_min();
		check_entry_exit_locking(tsd_tsdn(tsd));

		if (likely(tsd_fast(tsd))) {
			tcache_t *tcache = tcache_get_from_ind(tsd,
			    TCACHE_IND_AUTOMATIC, /* slow */ false,
			    /* is_alloc */ false);
			ifree(tsd, ptr, tcache, /* slow */ false);
		} else {
			tcache_t *tcache = tcache_get_from_ind(tsd,
			    TCACHE_IND_AUTOMATIC, /* slow */ true,
			    /* is_alloc */ false);
			uintptr_t args_raw[3] = {(uintptr_t)ptr};
			hook_invoke_dalloc(hook_dalloc_free, ptr, args_raw);
			ifree(tsd, ptr, tcache, /* slow */ true);
		}

		check_entry_exit_locking(tsd_tsdn(tsd));
	}
}

JEMALLOC_EXPORT void JEMALLOC_NOTHROW
je_free(void *ptr) {
	LOG("core.free.entry", "ptr: %p", ptr);

	je_free_impl(ptr);

	LOG("core.free.exit", "");
}

JEMALLOC_EXPORT void JEMALLOC_NOTHROW
je_free_sized(void *ptr, size_t size) {
	LOG("core.free_sized.entry", "ptr: %p, size: %zu", ptr, size);

	je_sdallocx_noflags(ptr, size);

	LOG("core.free_sized.exit", "");
}

JEMALLOC_EXPORT void JEMALLOC_NOTHROW
je_free_aligned_sized(void *ptr, size_t alignment, size_t size) {
	return je_sdallocx(ptr, size, /* flags */ MALLOCX_ALIGN(alignment));
}

/*
 * End malloc(3)-compatible functions.
 */
/******************************************************************************/
/*
 * Begin non-standard override functions.
 */

#ifdef JEMALLOC_OVERRIDE_MEMALIGN
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *
JEMALLOC_ATTR(malloc)
je_memalign(size_t alignment, size_t size) {
	void *ret;
	static_opts_t sopts;
	dynamic_opts_t dopts;

	LOG("core.memalign.entry", "alignment: %zu, size: %zu\n", alignment,
	    size);

	static_opts_init(&sopts);
	dynamic_opts_init(&dopts);

	sopts.bump_empty_aligned_alloc = true;
	sopts.min_alignment = 1;
	sopts.oom_string =
	    "<jemalloc>: Error allocating aligned memory: out of memory\n";
	sopts.invalid_alignment_string =
	    "<jemalloc>: Error allocating aligned memory: invalid alignment\n";
	sopts.null_out_result_on_error = true;

	dopts.result = &ret;
	dopts.num_items = 1;
	dopts.item_size = size;
	dopts.alignment = alignment;

	imalloc(&sopts, &dopts);
	if (sopts.slow) {
		uintptr_t args[3] = {alignment, size};
		hook_invoke_alloc(hook_alloc_memalign, ret, (uintptr_t)ret,
		    args);
	}

	LOG("core.memalign.exit", "result: %p", ret);
	return ret;
}
#endif

#ifdef JEMALLOC_OVERRIDE_VALLOC
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *
JEMALLOC_ATTR(malloc)
je_valloc(size_t size) {
	void *ret;

	static_opts_t sopts;
	dynamic_opts_t dopts;

	LOG("core.valloc.entry", "size: %zu\n", size);

	static_opts_init(&sopts);
	dynamic_opts_init(&dopts);

	sopts.null_out_result_on_error = true;
	sopts.min_alignment = PAGE;
	sopts.oom_string =
	    "<jemalloc>: Error allocating aligned memory: out of memory\n";
	sopts.invalid_alignment_string =
	    "<jemalloc>: Error allocating aligned memory: invalid alignment\n";

	dopts.result = &ret;
	dopts.num_items = 1;
	dopts.item_size = size;
	dopts.alignment = PAGE;

	imalloc(&sopts, &dopts);
	if (sopts.slow) {
		uintptr_t args[3] = {size};
		hook_invoke_alloc(hook_alloc_valloc, ret, (uintptr_t)ret, args);
	}

	LOG("core.valloc.exit", "result: %p\n", ret);
	return ret;
}
#endif

#ifdef JEMALLOC_OVERRIDE_PVALLOC
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *
JEMALLOC_ATTR(malloc)
je_pvalloc(size_t size) {
	void *ret;

	static_opts_t sopts;
	dynamic_opts_t dopts;

	LOG("core.pvalloc.entry", "size: %zu\n", size);

	static_opts_init(&sopts);
	dynamic_opts_init(&dopts);

	sopts.null_out_result_on_error = true;
	sopts.min_alignment = PAGE;
	sopts.oom_string =
	    "<jemalloc>: Error allocating aligned memory: out of memory\n";
	sopts.invalid_alignment_string =
	    "<jemalloc>: Error allocating aligned memory: invalid alignment\n";

	dopts.result = &ret;
	dopts.num_items = 1;
	/*
	 * This is the only difference from je_valloc - size is rounded up to
	 * a PAGE multiple.
	 */
	dopts.item_size = PAGE_CEILING(size);
	dopts.alignment = PAGE;

	imalloc(&sopts, &dopts);
	if (sopts.slow) {
		uintptr_t args[3] = {size};
		hook_invoke_alloc(hook_alloc_pvalloc, ret, (uintptr_t)ret,
		    args);
	}

	LOG("core.pvalloc.exit", "result: %p\n", ret);
	return ret;
}
#endif

#if defined(JEMALLOC_IS_MALLOC) && defined(JEMALLOC_GLIBC_MALLOC_HOOK)
/*
 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
 * to inconsistently reference libc's malloc(3)-compatible functions
 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541).
 *
 * These definitions interpose hooks in glibc.  The functions are actually
 * passed an extra argument for the caller return address, which will be
 * ignored.
 */
#include <features.h> // defines __GLIBC__ if we are compiling against glibc

JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free;
JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc;
JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc;
#  ifdef JEMALLOC_GLIBC_MEMALIGN_HOOK
JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) =
    je_memalign;
#  endif

#  ifdef __GLIBC__
/*
 * To enable static linking with glibc, the libc specific malloc interface must
 * be implemented also, so none of glibc's malloc.o functions are added to the
 * link.
 */
#    define ALIAS(je_fn)	__attribute__((alias (#je_fn), used))
/* To force macro expansion of je_ prefix before stringification. */
#    define PREALIAS(je_fn)	ALIAS(je_fn)
#    ifdef JEMALLOC_OVERRIDE___LIBC_CALLOC
void *__libc_calloc(size_t n, size_t size) PREALIAS(je_calloc);
#    endif
#    ifdef JEMALLOC_OVERRIDE___LIBC_FREE
void __libc_free(void* ptr) PREALIAS(je_free);
#    endif
#    ifdef JEMALLOC_OVERRIDE___LIBC_FREE_SIZED
void __libc_free_sized(void* ptr, size_t size) PREALIAS(je_free_sized);
#    endif
#    ifdef JEMALLOC_OVERRIDE___LIBC_FREE_ALIGNED_SIZED
void __libc_free_aligned_sized(
    void* ptr, size_t alignment, size_t size) PREALIAS(je_free_aligned_sized);
#    endif
#    ifdef JEMALLOC_OVERRIDE___LIBC_MALLOC
void *__libc_malloc(size_t size) PREALIAS(je_malloc);
#    endif
#    ifdef JEMALLOC_OVERRIDE___LIBC_MEMALIGN
void *__libc_memalign(size_t align, size_t s) PREALIAS(je_memalign);
#    endif
#    ifdef JEMALLOC_OVERRIDE___LIBC_REALLOC
void *__libc_realloc(void* ptr, size_t size) PREALIAS(je_realloc);
#    endif
#    ifdef JEMALLOC_OVERRIDE___LIBC_VALLOC
void *__libc_valloc(size_t size) PREALIAS(je_valloc);
#    endif
#    ifdef JEMALLOC_OVERRIDE___LIBC_PVALLOC
void *__libc_pvalloc(size_t size) PREALIAS(je_pvalloc);
#    endif
#    ifdef JEMALLOC_OVERRIDE___POSIX_MEMALIGN
int __posix_memalign(void** r, size_t a, size_t s) PREALIAS(je_posix_memalign);
#    endif
#    undef PREALIAS
#    undef ALIAS
#  endif
#endif

/*
 * End non-standard override functions.
 */
/******************************************************************************/
/*
 * Begin non-standard functions.
 */

JEMALLOC_ALWAYS_INLINE unsigned
mallocx_tcache_get(int flags) {
	if (likely((flags & MALLOCX_TCACHE_MASK) == 0)) {
		return TCACHE_IND_AUTOMATIC;
	} else if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) {
		return TCACHE_IND_NONE;
	} else {
		return MALLOCX_TCACHE_GET(flags);
	}
}

JEMALLOC_ALWAYS_INLINE unsigned
mallocx_arena_get(int flags) {
	if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
		return MALLOCX_ARENA_GET(flags);
	} else {
		return ARENA_IND_AUTOMATIC;
	}
}

#ifdef JEMALLOC_EXPERIMENTAL_SMALLOCX_API

#define JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) x ## y
#define JEMALLOC_SMALLOCX_CONCAT_HELPER2(x, y)  \
  JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y)

typedef struct {
	void *ptr;
	size_t size;
} smallocx_return_t;

JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
smallocx_return_t JEMALLOC_NOTHROW
/*
 * The attribute JEMALLOC_ATTR(malloc) cannot be used due to:
 *  - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86488
 */
JEMALLOC_SMALLOCX_CONCAT_HELPER2(je_smallocx_, JEMALLOC_VERSION_GID_IDENT)
  (size_t size, int flags) {
	/*
	 * Note: the attribute JEMALLOC_ALLOC_SIZE(1) cannot be
	 * used here because it makes writing beyond the `size`
	 * of the `ptr` undefined behavior, but the objective
	 * of this function is to allow writing beyond `size`
	 * up to `smallocx_return_t::size`.
	 */
	smallocx_return_t ret;
	static_opts_t sopts;
	dynamic_opts_t dopts;

	LOG("core.smallocx.entry", "size: %zu, flags: %d", size, flags);

	static_opts_init(&sopts);
	dynamic_opts_init(&dopts);

	sopts.assert_nonempty_alloc = true;
	sopts.null_out_result_on_error = true;
	sopts.oom_string = "<jemalloc>: Error in mallocx(): out of memory\n";
	sopts.usize = true;

	dopts.result = &ret.ptr;
	dopts.num_items = 1;
	dopts.item_size = size;
	if (unlikely(flags != 0)) {
		dopts.alignment = MALLOCX_ALIGN_GET(flags);
		dopts.zero = MALLOCX_ZERO_GET(flags);
		dopts.tcache_ind = mallocx_tcache_get(flags);
		dopts.arena_ind = mallocx_arena_get(flags);
	}

	imalloc(&sopts, &dopts);
	assert(dopts.usize == je_nallocx(size, flags));
	ret.size = dopts.usize;

	LOG("core.smallocx.exit", "result: %p, size: %zu", ret.ptr, ret.size);
	return ret;
}
#undef JEMALLOC_SMALLOCX_CONCAT_HELPER
#undef JEMALLOC_SMALLOCX_CONCAT_HELPER2
#endif

JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *
JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1)
je_mallocx(size_t size, int flags) {
	void *ret;
	static_opts_t sopts;
	dynamic_opts_t dopts;

	LOG("core.mallocx.entry", "size: %zu, flags: %d", size, flags);

	static_opts_init(&sopts);
	dynamic_opts_init(&dopts);

	sopts.assert_nonempty_alloc = true;
	sopts.null_out_result_on_error = true;
	sopts.oom_string = "<jemalloc>: Error in mallocx(): out of memory\n";

	dopts.result = &ret;
	dopts.num_items = 1;
	dopts.item_size = size;
	if (unlikely(flags != 0)) {
		dopts.alignment = MALLOCX_ALIGN_GET(flags);
		dopts.zero = MALLOCX_ZERO_GET(flags);
		dopts.tcache_ind = mallocx_tcache_get(flags);
		dopts.arena_ind = mallocx_arena_get(flags);
	}

	imalloc(&sopts, &dopts);
	if (sopts.slow) {
		uintptr_t args[3] = {size, flags};
		hook_invoke_alloc(hook_alloc_mallocx, ret, (uintptr_t)ret,
		    args);
	}

	LOG("core.mallocx.exit", "result: %p", ret);
	return ret;
}

static void *
irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize,
    size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena,
    prof_tctx_t *tctx, hook_ralloc_args_t *hook_args) {
	void *p;

	if (tctx == NULL) {
		return NULL;
	}

	alignment = prof_sample_align(usize, alignment);
	/*
	 * If the allocation is small enough that it would normally be allocated
	 * on a slab, we need to take additional steps to ensure that it gets
	 * its own extent instead.
	 */
	if (sz_can_use_slab(usize)) {
		size_t bumped_usize = sz_sa2u(usize, alignment);
		p = iralloct_explicit_slab(tsdn, old_ptr, old_usize,
		    bumped_usize, alignment, zero, /* slab */ false,
		    tcache, arena, hook_args);
		if (p == NULL) {
			return NULL;
		}
		arena_prof_promote(tsdn, p, usize, bumped_usize);
	} else {
		p = iralloct_explicit_slab(tsdn, old_ptr, old_usize, usize,
		    alignment, zero, /* slab */ false, tcache, arena,
		    hook_args);
	}
	assert(prof_sample_aligned(p));

	return p;
}

JEMALLOC_ALWAYS_INLINE void *
irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,
    size_t alignment, size_t usize, bool zero, tcache_t *tcache,
    arena_t *arena, emap_alloc_ctx_t *alloc_ctx,
    hook_ralloc_args_t *hook_args) {
	prof_info_t old_prof_info;
	prof_info_get_and_reset_recent(tsd, old_ptr, alloc_ctx, &old_prof_info);
	bool prof_active = prof_active_get_unlocked();
	bool sample_event = te_prof_sample_event_lookahead(tsd, usize);
	prof_tctx_t *tctx = prof_alloc_prep(tsd, prof_active, sample_event);
	void *p;
	if (unlikely(tctx != PROF_TCTX_SENTINEL)) {
		p = irallocx_prof_sample(tsd_tsdn(tsd), old_ptr, old_usize,
		    usize, alignment, zero, tcache, arena, tctx, hook_args);
	} else {
		p = iralloct(tsd_tsdn(tsd), old_ptr, old_usize, size, alignment,
		    usize, zero, tcache, arena, hook_args);
	}
	if (unlikely(p == NULL)) {
		prof_alloc_rollback(tsd, tctx);
		return NULL;
	}
	assert(usize == isalloc(tsd_tsdn(tsd), p));
	prof_realloc(tsd, p, size, usize, tctx, prof_active, old_ptr,
	    old_usize, &old_prof_info, sample_event);

	return p;
}

static void *
do_rallocx(void *ptr, size_t size, int flags, bool is_realloc) {
	void *p;
	tsd_t *tsd;
	size_t usize;
	size_t old_usize;
	size_t alignment = MALLOCX_ALIGN_GET(flags);
	arena_t *arena;

	assert(ptr != NULL);
	assert(size != 0);
	assert(malloc_initialized() || IS_INITIALIZER);
	tsd = tsd_fetch();
	check_entry_exit_locking(tsd_tsdn(tsd));

	bool zero = zero_get(MALLOCX_ZERO_GET(flags), /* slow */ true);

	unsigned arena_ind = mallocx_arena_get(flags);
	if (arena_get_from_ind(tsd, arena_ind, &arena)) {
		goto label_oom;
	}

	unsigned tcache_ind = mallocx_tcache_get(flags);
	tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind,
	    /* slow */ true, /* is_alloc */ true);

	emap_alloc_ctx_t alloc_ctx;
	emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr,
	    &alloc_ctx);
	assert(alloc_ctx.szind != SC_NSIZES);
	old_usize = sz_index2size(alloc_ctx.szind);
	assert(old_usize == isalloc(tsd_tsdn(tsd), ptr));
	if (aligned_usize_get(size, alignment, &usize, NULL, false)) {
		goto label_oom;
	}

	hook_ralloc_args_t hook_args = {is_realloc, {(uintptr_t)ptr, size,
		flags, 0}};
	if (config_prof && opt_prof) {
		p = irallocx_prof(tsd, ptr, old_usize, size, alignment, usize,
		    zero, tcache, arena, &alloc_ctx, &hook_args);
		if (unlikely(p == NULL)) {
			goto label_oom;
		}
	} else {
		p = iralloct(tsd_tsdn(tsd), ptr, old_usize, size, alignment,
		    usize, zero, tcache, arena, &hook_args);
		if (unlikely(p == NULL)) {
			goto label_oom;
		}
		assert(usize == isalloc(tsd_tsdn(tsd), p));
	}
	assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
	thread_alloc_event(tsd, usize);
	thread_dalloc_event(tsd, old_usize);

	UTRACE(ptr, size, p);
	check_entry_exit_locking(tsd_tsdn(tsd));

	if (config_fill && unlikely(opt_junk_alloc) && usize > old_usize
	    && !zero) {
		size_t excess_len = usize - old_usize;
		void *excess_start = (void *)((byte_t *)p + old_usize);
		junk_alloc_callback(excess_start, excess_len);
	}

	return p;
label_oom:
	if (is_realloc) {
		set_errno(ENOMEM);
	}
	if (config_xmalloc && unlikely(opt_xmalloc)) {
		malloc_write("<jemalloc>: Error in rallocx(): out of memory\n");
		abort();
	}
	UTRACE(ptr, size, 0);
	check_entry_exit_locking(tsd_tsdn(tsd));

	return NULL;
}

JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *
JEMALLOC_ALLOC_SIZE(2)
je_rallocx(void *ptr, size_t size, int flags) {
	LOG("core.rallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr,
	    size, flags);
	void *ret = do_rallocx(ptr, size, flags, false);
	LOG("core.rallocx.exit", "result: %p", ret);
	return ret;
}

static void *
do_realloc_nonnull_zero(void *ptr) {
	if (config_stats) {
		atomic_fetch_add_zu(&zero_realloc_count, 1, ATOMIC_RELAXED);
	}
	if (opt_zero_realloc_action == zero_realloc_action_alloc) {
		/*
		 * The user might have gotten an alloc setting while expecting a
		 * free setting.  If that's the case, we at least try to
		 * reduce the harm, and turn off the tcache while allocating, so
		 * that we'll get a true first fit.
		 */
		return do_rallocx(ptr, 1, MALLOCX_TCACHE_NONE, true);
	} else if (opt_zero_realloc_action == zero_realloc_action_free) {
		UTRACE(ptr, 0, 0);
		tsd_t *tsd = tsd_fetch();
		check_entry_exit_locking(tsd_tsdn(tsd));

		tcache_t *tcache = tcache_get_from_ind(tsd,
		    TCACHE_IND_AUTOMATIC, /* slow */ true,
		    /* is_alloc */ false);
		uintptr_t args[3] = {(uintptr_t)ptr, 0};
		hook_invoke_dalloc(hook_dalloc_realloc, ptr, args);
		ifree(tsd, ptr, tcache, true);

		check_entry_exit_locking(tsd_tsdn(tsd));
		return NULL;
	} else {
		safety_check_fail("Called realloc(non-null-ptr, 0) with "
		    "zero_realloc:abort set\n");
		/* In real code, this will never run; the safety check failure
		 * will call abort.  In the unit test, we just want to bail out
		 * without corrupting internal state that the test needs to
		 * finish.
		 */
		return NULL;
	}
}

JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *
JEMALLOC_ALLOC_SIZE(2)
je_realloc(void *ptr, size_t size) {
	LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size);

	if (likely(ptr != NULL && size != 0)) {
		void *ret = do_rallocx(ptr, size, 0, true);
		LOG("core.realloc.exit", "result: %p", ret);
		return ret;
	} else if (ptr != NULL && size == 0) {
		void *ret = do_realloc_nonnull_zero(ptr);
		LOG("core.realloc.exit", "result: %p", ret);
		return ret;
	} else {
		/* realloc(NULL, size) is equivalent to malloc(size). */
		void *ret;

		static_opts_t sopts;
		dynamic_opts_t dopts;

		static_opts_init(&sopts);
		dynamic_opts_init(&dopts);

		sopts.null_out_result_on_error = true;
		sopts.set_errno_on_error = true;
		sopts.oom_string =
		    "<jemalloc>: Error in realloc(): out of memory\n";

		dopts.result = &ret;
		dopts.num_items = 1;
		dopts.item_size = size;

		imalloc(&sopts, &dopts);
		if (sopts.slow) {
			uintptr_t args[3] = {(uintptr_t)ptr, size};
			hook_invoke_alloc(hook_alloc_realloc, ret,
			    (uintptr_t)ret, args);
		}
		LOG("core.realloc.exit", "result: %p", ret);
		return ret;
	}
}

JEMALLOC_ALWAYS_INLINE size_t
ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
    size_t extra, size_t alignment, bool zero) {
	size_t newsize;

	if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero,
	    &newsize)) {
		return old_usize;
	}

	return newsize;
}

static size_t
ixallocx_prof_sample(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
    size_t extra, size_t alignment, bool zero, prof_tctx_t *tctx) {
	/* Sampled allocation needs to be page aligned. */
	if (tctx == NULL || !prof_sample_aligned(ptr)) {
		return old_usize;
	}

	return ixallocx_helper(tsdn, ptr, old_usize, size, extra, alignment,
	    zero);
}

JEMALLOC_ALWAYS_INLINE size_t
ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
    size_t extra, size_t alignment, bool zero, emap_alloc_ctx_t *alloc_ctx) {
	/*
	 * old_prof_info is only used for asserting that the profiling info
	 * isn't changed by the ixalloc() call.
	 */
	prof_info_t old_prof_info;
	prof_info_get(tsd, ptr, alloc_ctx, &old_prof_info);

	/*
	 * usize isn't knowable before ixalloc() returns when extra is non-zero.
	 * Therefore, compute its maximum possible value and use that in
	 * prof_alloc_prep() to decide whether to capture a backtrace.
	 * prof_realloc() will use the actual usize to decide whether to sample.
	 */
	size_t usize_max;
	if (aligned_usize_get(size + extra, alignment, &usize_max, NULL,
	    false)) {
		/*
		 * usize_max is out of range, and chances are that allocation
		 * will fail, but use the maximum possible value and carry on
		 * with prof_alloc_prep(), just in case allocation succeeds.
		 */
		usize_max = SC_LARGE_MAXCLASS;
	}
	bool prof_active = prof_active_get_unlocked();
	bool sample_event = te_prof_sample_event_lookahead(tsd, usize_max);
	prof_tctx_t *tctx = prof_alloc_prep(tsd, prof_active, sample_event);

	size_t usize;
	if (unlikely(tctx != PROF_TCTX_SENTINEL)) {
		usize = ixallocx_prof_sample(tsd_tsdn(tsd), ptr, old_usize,
		    size, extra, alignment, zero, tctx);
	} else {
		usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
		    extra, alignment, zero);
	}

	/*
	 * At this point we can still safely get the original profiling
	 * information associated with the ptr, because (a) the edata_t object
	 * associated with the ptr still lives and (b) the profiling info
	 * fields are not touched.  "(a)" is asserted in the outer je_xallocx()
	 * function, and "(b)" is indirectly verified below by checking that
	 * the alloc_tctx field is unchanged.
	 */
	prof_info_t prof_info;
	if (usize == old_usize) {
		prof_info_get(tsd, ptr, alloc_ctx, &prof_info);
		prof_alloc_rollback(tsd, tctx);
	} else {
		prof_info_get_and_reset_recent(tsd, ptr, alloc_ctx, &prof_info);
		assert(usize <= usize_max);
		sample_event = te_prof_sample_event_lookahead(tsd, usize);
		prof_realloc(tsd, ptr, size, usize, tctx, prof_active, ptr,
		    old_usize, &prof_info, sample_event);
	}

	assert(old_prof_info.alloc_tctx == prof_info.alloc_tctx);
	return usize;
}

JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
	tsd_t *tsd;
	size_t usize, old_usize;
	size_t alignment = MALLOCX_ALIGN_GET(flags);
	bool zero = zero_get(MALLOCX_ZERO_GET(flags), /* slow */ true);

	LOG("core.xallocx.entry", "ptr: %p, size: %zu, extra: %zu, "
	    "flags: %d", ptr, size, extra, flags);

	assert(ptr != NULL);
	assert(size != 0);
	assert(SIZE_T_MAX - size >= extra);
	assert(malloc_initialized() || IS_INITIALIZER);
	tsd = tsd_fetch();
	check_entry_exit_locking(tsd_tsdn(tsd));

	/*
	 * old_edata is only for verifying that xallocx() keeps the edata_t
	 * object associated with the ptr (though the content of the edata_t
	 * object can be changed).
	 */
	edata_t *old_edata = emap_edata_lookup(tsd_tsdn(tsd),
	    &arena_emap_global, ptr);

	emap_alloc_ctx_t alloc_ctx;
	emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr,
	    &alloc_ctx);
	assert(alloc_ctx.szind != SC_NSIZES);
	old_usize = sz_index2size(alloc_ctx.szind);
	assert(old_usize == isalloc(tsd_tsdn(tsd), ptr));
	/*
	 * The API explicitly absolves itself of protecting against (size +
	 * extra) numerical overflow, but we may need to clamp extra to avoid
	 * exceeding SC_LARGE_MAXCLASS.
	 *
	 * Ordinarily, size limit checking is handled deeper down, but here we
	 * have to check as part of (size + extra) clamping, since we need the
	 * clamped value in the above helper functions.
	 */
	if (unlikely(size > SC_LARGE_MAXCLASS)) {
		usize = old_usize;
		goto label_not_resized;
	}
	if (unlikely(SC_LARGE_MAXCLASS - size < extra)) {
		extra = SC_LARGE_MAXCLASS - size;
	}

	if (config_prof && opt_prof) {
		usize = ixallocx_prof(tsd, ptr, old_usize, size, extra,
		    alignment, zero, &alloc_ctx);
	} else {
		usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
		    extra, alignment, zero);
	}

	/*
	 * xallocx() should keep using the same edata_t object (though its
	 * content can be changed).
	 */
	assert(emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr)
	    == old_edata);

	if (unlikely(usize == old_usize)) {
		goto label_not_resized;
	}
	thread_alloc_event(tsd, usize);
	thread_dalloc_event(tsd, old_usize);

	if (config_fill && unlikely(opt_junk_alloc) && usize > old_usize &&
	    !zero) {
		size_t excess_len = usize - old_usize;
		void *excess_start = (void *)((byte_t *)ptr + old_usize);
		junk_alloc_callback(excess_start, excess_len);
	}
label_not_resized:
	if (unlikely(!tsd_fast(tsd))) {
		uintptr_t args[4] = {(uintptr_t)ptr, size, extra, flags};
		hook_invoke_expand(hook_expand_xallocx, ptr, old_usize,
		    usize, (uintptr_t)usize, args);
	}

	UTRACE(ptr, size, ptr);
	check_entry_exit_locking(tsd_tsdn(tsd));

	LOG("core.xallocx.exit", "result: %zu", usize);
	return usize;
}

JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
JEMALLOC_ATTR(pure)
je_sallocx(const void *ptr, int flags) {
	size_t usize;
	tsdn_t *tsdn;

	LOG("core.sallocx.entry", "ptr: %p, flags: %d", ptr, flags);

	assert(malloc_initialized() || IS_INITIALIZER);
	assert(ptr != NULL);

	tsdn = tsdn_fetch();
	check_entry_exit_locking(tsdn);

	if (config_debug || force_ivsalloc) {
		usize = ivsalloc(tsdn, ptr);
		assert(force_ivsalloc || usize != 0);
	} else {
		usize = isalloc(tsdn, ptr);
	}

	check_entry_exit_locking(tsdn);

	LOG("core.sallocx.exit", "result: %zu", usize);
	return usize;
}

JEMALLOC_EXPORT void JEMALLOC_NOTHROW
je_dallocx(void *ptr, int flags) {
	LOG("core.dallocx.entry", "ptr: %p, flags: %d", ptr, flags);

	assert(ptr != NULL);
	assert(malloc_initialized() || IS_INITIALIZER);

	tsd_t *tsd = tsd_fetch_min();
	bool fast = tsd_fast(tsd);
	check_entry_exit_locking(tsd_tsdn(tsd));

	unsigned tcache_ind = mallocx_tcache_get(flags);
	tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind, !fast,
	    /* is_alloc */ false);

	UTRACE(ptr, 0, 0);
	if (likely(fast)) {
		tsd_assert_fast(tsd);
		ifree(tsd, ptr, tcache, false);
	} else {
		uintptr_t args_raw[3] = {(uintptr_t)ptr, flags};
		hook_invoke_dalloc(hook_dalloc_dallocx, ptr, args_raw);
		ifree(tsd, ptr, tcache, true);
	}
	check_entry_exit_locking(tsd_tsdn(tsd));

	LOG("core.dallocx.exit", "");
}

JEMALLOC_ALWAYS_INLINE size_t
inallocx(tsdn_t *tsdn, size_t size, int flags) {
	check_entry_exit_locking(tsdn);
	size_t usize;
	/* In case of out of range, let the user see it rather than fail. */
	aligned_usize_get(size, MALLOCX_ALIGN_GET(flags), &usize, NULL, false);
	check_entry_exit_locking(tsdn);
	return usize;
}

JEMALLOC_NOINLINE void
sdallocx_default(void *ptr, size_t size, int flags) {
	assert(ptr != NULL);
	assert(malloc_initialized() || IS_INITIALIZER);

	tsd_t *tsd = tsd_fetch_min();
	bool fast = tsd_fast(tsd);
	size_t usize = inallocx(tsd_tsdn(tsd), size, flags);
	check_entry_exit_locking(tsd_tsdn(tsd));

	unsigned tcache_ind = mallocx_tcache_get(flags);
	tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind, !fast,
	    /* is_alloc */ false);

	UTRACE(ptr, 0, 0);
	if (likely(fast)) {
		tsd_assert_fast(tsd);
		isfree(tsd, ptr, usize, tcache, false);
	} else {
		uintptr_t args_raw[3] = {(uintptr_t)ptr, size, flags};
		hook_invoke_dalloc(hook_dalloc_sdallocx, ptr, args_raw);
		isfree(tsd, ptr, usize, tcache, true);
	}
	check_entry_exit_locking(tsd_tsdn(tsd));
}

JEMALLOC_EXPORT void JEMALLOC_NOTHROW
je_sdallocx(void *ptr, size_t size, int flags) {
	LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr,
		size, flags);

	je_sdallocx_impl(ptr, size, flags);

	LOG("core.sdallocx.exit", "");
}

JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
JEMALLOC_ATTR(pure)
je_nallocx(size_t size, int flags) {
	size_t usize;
	tsdn_t *tsdn;

	assert(size != 0);

	if (unlikely(malloc_init())) {
		LOG("core.nallocx.exit", "result: %zu", ZU(0));
		return 0;
	}

	tsdn = tsdn_fetch();
	check_entry_exit_locking(tsdn);

	usize = inallocx(tsdn, size, flags);
	if (unlikely(usize > SC_LARGE_MAXCLASS)) {
		LOG("core.nallocx.exit", "result: %zu", ZU(0));
		return 0;
	}

	check_entry_exit_locking(tsdn);
	LOG("core.nallocx.exit", "result: %zu", usize);
	return usize;
}

JEMALLOC_EXPORT int JEMALLOC_NOTHROW
je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,
    size_t newlen) {
	int ret;
	tsd_t *tsd;

	LOG("core.mallctl.entry", "name: %s", name);

	if (unlikely(malloc_init())) {
		LOG("core.mallctl.exit", "result: %d", EAGAIN);
		return EAGAIN;
	}

	tsd = tsd_fetch();
	check_entry_exit_locking(tsd_tsdn(tsd));
	ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen);
	check_entry_exit_locking(tsd_tsdn(tsd));

	LOG("core.mallctl.exit", "result: %d", ret);
	return ret;
}

JEMALLOC_EXPORT int JEMALLOC_NOTHROW
je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) {
	int ret;

	LOG("core.mallctlnametomib.entry", "name: %s", name);

	if (unlikely(malloc_init())) {
		LOG("core.mallctlnametomib.exit", "result: %d", EAGAIN);
		return EAGAIN;
	}

	tsd_t *tsd = tsd_fetch();
	check_entry_exit_locking(tsd_tsdn(tsd));
	ret = ctl_nametomib(tsd, name, mibp, miblenp);
	check_entry_exit_locking(tsd_tsdn(tsd));

	LOG("core.mallctlnametomib.exit", "result: %d", ret);
	return ret;
}

JEMALLOC_EXPORT int JEMALLOC_NOTHROW
je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
  void *newp, size_t newlen) {
	int ret;
	tsd_t *tsd;

	LOG("core.mallctlbymib.entry", "");

	if (unlikely(malloc_init())) {
		LOG("core.mallctlbymib.exit", "result: %d", EAGAIN);
		return EAGAIN;
	}

	tsd = tsd_fetch();
	check_entry_exit_locking(tsd_tsdn(tsd));
	ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen);
	check_entry_exit_locking(tsd_tsdn(tsd));
	LOG("core.mallctlbymib.exit", "result: %d", ret);
	return ret;
}

#define STATS_PRINT_BUFSIZE 65536
JEMALLOC_EXPORT void JEMALLOC_NOTHROW
je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
    const char *opts) {
	tsdn_t *tsdn;

	LOG("core.malloc_stats_print.entry", "");

	tsdn = tsdn_fetch();
	check_entry_exit_locking(tsdn);

	if (config_debug) {
		stats_print(write_cb, cbopaque, opts);
	} else {
		buf_writer_t buf_writer;
		buf_writer_init(tsdn, &buf_writer, write_cb, cbopaque, NULL,
		    STATS_PRINT_BUFSIZE);
		stats_print(buf_writer_cb, &buf_writer, opts);
		buf_writer_terminate(tsdn, &buf_writer);
	}

	check_entry_exit_locking(tsdn);
	LOG("core.malloc_stats_print.exit", "");
}
#undef STATS_PRINT_BUFSIZE

JEMALLOC_ALWAYS_INLINE size_t
je_malloc_usable_size_impl(JEMALLOC_USABLE_SIZE_CONST void *ptr) {
	assert(malloc_initialized() || IS_INITIALIZER);

	tsdn_t *tsdn = tsdn_fetch();
	check_entry_exit_locking(tsdn);

	size_t ret;
	if (unlikely(ptr == NULL)) {
		ret = 0;
	} else {
		if (config_debug || force_ivsalloc) {
			ret = ivsalloc(tsdn, ptr);
			assert(force_ivsalloc || ret != 0);
		} else {
			ret = isalloc(tsdn, ptr);
		}
	}
	check_entry_exit_locking(tsdn);

	return ret;
}

JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) {
	LOG("core.malloc_usable_size.entry", "ptr: %p", ptr);

	size_t ret = je_malloc_usable_size_impl(ptr);

	LOG("core.malloc_usable_size.exit", "result: %zu", ret);
	return ret;
}

#ifdef JEMALLOC_HAVE_MALLOC_SIZE
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
je_malloc_size(const void *ptr) {
	LOG("core.malloc_size.entry", "ptr: %p", ptr);

	size_t ret = je_malloc_usable_size_impl(ptr);

	LOG("core.malloc_size.exit", "result: %zu", ret);
	return ret;
}
#endif

static void
batch_alloc_prof_sample_assert(tsd_t *tsd, size_t batch, size_t usize) {
	assert(config_prof && opt_prof);
	bool prof_sample_event = te_prof_sample_event_lookahead(tsd,
	    batch * usize);
	assert(!prof_sample_event);
	size_t surplus;
	prof_sample_event = te_prof_sample_event_lookahead_surplus(tsd,
	    (batch + 1) * usize, &surplus);
	assert(prof_sample_event);
	assert(surplus < usize);
}

size_t
batch_alloc(void **ptrs, size_t num, size_t size, int flags) {
	LOG("core.batch_alloc.entry",
	    "ptrs: %p, num: %zu, size: %zu, flags: %d", ptrs, num, size, flags);

	tsd_t *tsd = tsd_fetch();
	check_entry_exit_locking(tsd_tsdn(tsd));

	size_t filled = 0;

	if (unlikely(tsd == NULL || tsd_reentrancy_level_get(tsd) > 0)) {
		goto label_done;
	}

	size_t alignment = MALLOCX_ALIGN_GET(flags);
	size_t usize;
	if (aligned_usize_get(size, alignment, &usize, NULL, false)) {
		goto label_done;
	}
	szind_t ind = sz_size2index(usize);
	bool zero = zero_get(MALLOCX_ZERO_GET(flags), /* slow */ true);

	/*
	 * The cache bin and arena will be lazily initialized; it's hard to
	 * know in advance whether each of them needs to be initialized.
	 */
	cache_bin_t *bin = NULL;
	arena_t *arena = NULL;

	size_t nregs = 0;
	if (likely(ind < SC_NBINS)) {
		nregs = bin_infos[ind].nregs;
		assert(nregs > 0);
	}

	while (filled < num) {
		size_t batch = num - filled;
		size_t surplus = SIZE_MAX; /* Dead store. */
		bool prof_sample_event = config_prof && opt_prof
		    && prof_active_get_unlocked()
		    && te_prof_sample_event_lookahead_surplus(tsd,
		    batch * usize, &surplus);

		if (prof_sample_event) {
			/*
			 * Adjust so that the batch does not trigger prof
			 * sampling.
			 */
			batch -= surplus / usize + 1;
			batch_alloc_prof_sample_assert(tsd, batch, usize);
		}

		size_t progress = 0;

		if (likely(ind < SC_NBINS) && batch >= nregs) {
			if (arena == NULL) {
				unsigned arena_ind = mallocx_arena_get(flags);
				if (arena_get_from_ind(tsd, arena_ind,
				    &arena)) {
					goto label_done;
				}
				if (arena == NULL) {
					arena = arena_choose(tsd, NULL);
				}
				if (unlikely(arena == NULL)) {
					goto label_done;
				}
			}
			size_t arena_batch = batch - batch % nregs;
			size_t n = arena_fill_small_fresh(tsd_tsdn(tsd), arena,
			    ind, ptrs + filled, arena_batch, zero);
			progress += n;
			filled += n;
		}

		unsigned tcache_ind = mallocx_tcache_get(flags);
		tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind,
		    /* slow */ true, /* is_alloc */ true);
		if (likely(tcache != NULL &&
		    ind < tcache_nbins_get(tcache->tcache_slow) &&
		    !tcache_bin_disabled(ind, &tcache->bins[ind],
		    tcache->tcache_slow)) && progress < batch) {
			if (bin == NULL) {
				bin = &tcache->bins[ind];
			}
			/*
			 * If we don't have a tcache bin, we don't want to
			 * immediately give up, because there's the possibility
			 * that the user explicitly requested to bypass the
			 * tcache, or that the user explicitly turned off the
			 * tcache; in such cases, we go through the slow path,
			 * i.e. the mallocx() call at the end of the while loop.
			 */
			if (bin != NULL) {
				size_t bin_batch = batch - progress;
				/*
				 * n can be less than bin_batch, meaning that
				 * the cache bin does not have enough memory.
				 * In such cases, we rely on the slow path,
				 * i.e. the mallocx() call at the end of the
				 * while loop, to fill in the cache, and in the
				 * next iteration of the while loop, the tcache
				 * will contain a lot of memory, and we can
				 * harvest them here.  Compared to the
				 * alternative approach where we directly go to
				 * the arena bins here, the overhead of our
				 * current approach should usually be minimal,
				 * since we never try to fetch more memory than
				 * what a slab contains via the tcache.  An
				 * additional benefit is that the tcache will
				 * not be empty for the next allocation request.
				 */
				size_t n = cache_bin_alloc_batch(bin, bin_batch,
				    ptrs + filled);
				if (config_stats) {
					bin->tstats.nrequests += n;
				}
				if (zero) {
					for (size_t i = 0; i < n; ++i) {
						memset(ptrs[filled + i], 0,
						    usize);
					}
				}
				if (config_prof && opt_prof
				    && unlikely(ind >= SC_NBINS)) {
					for (size_t i = 0; i < n; ++i) {
						prof_tctx_reset_sampled(tsd,
						    ptrs[filled + i]);
					}
				}
				progress += n;
				filled += n;
			}
		}

		/*
		 * For thread events other than prof sampling, trigger them as
		 * if there's a single allocation of size (n * usize).  This is
		 * fine because:
		 * (a) these events do not alter the allocation itself, and
		 * (b) it's possible that some event would have been triggered
		 *     multiple times, instead of only once, if the allocations
		 *     were handled individually, but it would do no harm (or
		 *     even be beneficial) to coalesce the triggerings.
		 */
		thread_alloc_event(tsd, progress * usize);

		if (progress < batch || prof_sample_event) {
			void *p = je_mallocx(size, flags);
			if (p == NULL) { /* OOM */
				break;
			}
			if (progress == batch) {
				assert(prof_sampled(tsd, p));
			}
			ptrs[filled++] = p;
		}
	}

label_done:
	check_entry_exit_locking(tsd_tsdn(tsd));
	LOG("core.batch_alloc.exit", "result: %zu", filled);
	return filled;
}

/*
 * End non-standard functions.
 */
/******************************************************************************/
/*
 * The following functions are used by threading libraries for protection of
 * malloc during fork().
 */

/*
 * If an application creates a thread before doing any allocation in the main
 * thread, then calls fork(2) in the main thread followed by memory allocation
 * in the child process, a race can occur that results in deadlock within the
 * child: the main thread may have forked while the created thread had
 * partially initialized the allocator.  Ordinarily jemalloc prevents
 * fork/malloc races via the following functions it registers during
 * initialization using pthread_atfork(), but of course that does no good if
 * the allocator isn't fully initialized at fork time.  The following library
 * constructor is a partial solution to this problem.  It may still be possible
 * to trigger the deadlock described above, but doing so would involve forking
 * via a library constructor that runs before jemalloc's runs.
 */
#ifndef JEMALLOC_JET
JEMALLOC_ATTR(constructor)
static void
jemalloc_constructor(void) {
	malloc_init();
}
#endif

#ifndef JEMALLOC_MUTEX_INIT_CB
void
jemalloc_prefork(void)
#else
JEMALLOC_EXPORT void
_malloc_prefork(void)
#endif
{
	tsd_t *tsd;
	unsigned i, j, narenas;
	arena_t *arena;

#ifdef JEMALLOC_MUTEX_INIT_CB
	if (!malloc_initialized()) {
		return;
	}
#endif
	assert(malloc_initialized());

	tsd = tsd_fetch();

	narenas = narenas_total_get();

	witness_prefork(tsd_witness_tsdp_get(tsd));
	/* Acquire all mutexes in a safe order. */
	ctl_prefork(tsd_tsdn(tsd));
	tcache_prefork(tsd_tsdn(tsd));
	malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock);
	if (have_background_thread) {
		background_thread_prefork0(tsd_tsdn(tsd));
	}
	prof_prefork0(tsd_tsdn(tsd));
	if (have_background_thread) {
		background_thread_prefork1(tsd_tsdn(tsd));
	}
	/* Break arena prefork into stages to preserve lock order. */
	for (i = 0; i < 9; i++) {
		for (j = 0; j < narenas; j++) {
			if ((arena = arena_get(tsd_tsdn(tsd), j, false)) !=
			    NULL) {
				switch (i) {
				case 0:
					arena_prefork0(tsd_tsdn(tsd), arena);
					break;
				case 1:
					arena_prefork1(tsd_tsdn(tsd), arena);
					break;
				case 2:
					arena_prefork2(tsd_tsdn(tsd), arena);
					break;
				case 3:
					arena_prefork3(tsd_tsdn(tsd), arena);
					break;
				case 4:
					arena_prefork4(tsd_tsdn(tsd), arena);
					break;
				case 5:
					arena_prefork5(tsd_tsdn(tsd), arena);
					break;
				case 6:
					arena_prefork6(tsd_tsdn(tsd), arena);
					break;
				case 7:
					arena_prefork7(tsd_tsdn(tsd), arena);
					break;
				case 8:
					arena_prefork8(tsd_tsdn(tsd), arena);
					break;
				default: not_reached();
				}
			}
		}

	}
	prof_prefork1(tsd_tsdn(tsd));
	stats_prefork(tsd_tsdn(tsd));
	tsd_prefork(tsd);
}

#ifndef JEMALLOC_MUTEX_INIT_CB
void
jemalloc_postfork_parent(void)
#else
JEMALLOC_EXPORT void
_malloc_postfork(void)
#endif
{
	tsd_t *tsd;
	unsigned i, narenas;

#ifdef JEMALLOC_MUTEX_INIT_CB
	if (!malloc_initialized()) {
		return;
	}
#endif
	assert(malloc_initialized());

	tsd = tsd_fetch();

	tsd_postfork_parent(tsd);

	witness_postfork_parent(tsd_witness_tsdp_get(tsd));
	/* Release all mutexes, now that fork() has completed. */
	stats_postfork_parent(tsd_tsdn(tsd));
	for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
		arena_t *arena;

		if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) {
			arena_postfork_parent(tsd_tsdn(tsd), arena);
		}
	}
	prof_postfork_parent(tsd_tsdn(tsd));
	if (have_background_thread) {
		background_thread_postfork_parent(tsd_tsdn(tsd));
	}
	malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock);
	tcache_postfork_parent(tsd_tsdn(tsd));
	ctl_postfork_parent(tsd_tsdn(tsd));
}

void
jemalloc_postfork_child(void) {
	tsd_t *tsd;
	unsigned i, narenas;

	assert(malloc_initialized());

	tsd = tsd_fetch();

	tsd_postfork_child(tsd);

	witness_postfork_child(tsd_witness_tsdp_get(tsd));
	/* Release all mutexes, now that fork() has completed. */
	stats_postfork_child(tsd_tsdn(tsd));
	for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
		arena_t *arena;

		if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) {
			arena_postfork_child(tsd_tsdn(tsd), arena);
		}
	}
	prof_postfork_child(tsd_tsdn(tsd));
	if (have_background_thread) {
		background_thread_postfork_child(tsd_tsdn(tsd));
	}
	malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock);
	tcache_postfork_child(tsd_tsdn(tsd));
	ctl_postfork_child(tsd_tsdn(tsd));
}

/******************************************************************************/