#define JEMALLOC_CTL_C_

#include "jemalloc/internal/jemalloc_preamble.h"

#include "jemalloc/internal/jemalloc_internal_includes.h"

#include "jemalloc/internal/assert.h"

#include "jemalloc/internal/ctl.h"

#include "jemalloc/internal/extent_dss.h"

#include "jemalloc/internal/extent_mmap.h"

#include "jemalloc/internal/mutex.h"

#include "jemalloc/internal/nstime.h"

#include "jemalloc/internal/sc.h"

#include "jemalloc/internal/util.h"

/******************************************************************************/

/* Data. */

/*

 * ctl_mtx protects the following:

 * - ctl_stats->*

 */

static malloc_mutex_t	ctl_mtx;

static bool		ctl_initialized;

static ctl_stats_t	*ctl_stats;

static ctl_arenas_t	*ctl_arenas;

/******************************************************************************/

/* Helpers for named and indexed nodes. */

static const ctl_named_node_t *

ctl_named_node(const ctl_node_t *node) {

	return ((node->named) ? (const ctl_named_node_t *)node : NULL);

}

static const ctl_named_node_t *

ctl_named_children(const ctl_named_node_t *node, size_t index) {

	const ctl_named_node_t *children = ctl_named_node(node->children);

	return (children ? &children[index] : NULL);

}

static const ctl_indexed_node_t *

ctl_indexed_node(const ctl_node_t *node) {

	return (!node->named ? (const ctl_indexed_node_t *)node : NULL);

}

/******************************************************************************/

/* Function prototypes for non-inline static functions. */

#define CTL_PROTO(n)							\

static int	n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,	\

    void *oldp, size_t *oldlenp, void *newp, size_t newlen);

#define INDEX_PROTO(n)							\

static const ctl_named_node_t	*n##_index(tsdn_t *tsdn,		\

    const size_t *mib, size_t miblen, size_t i);

CTL_PROTO(version)

CTL_PROTO(epoch)

CTL_PROTO(background_thread)

CTL_PROTO(max_background_threads)

CTL_PROTO(thread_tcache_enabled)

CTL_PROTO(thread_tcache_flush)

CTL_PROTO(thread_prof_name)

CTL_PROTO(thread_prof_active)

CTL_PROTO(thread_arena)

CTL_PROTO(thread_allocated)

CTL_PROTO(thread_allocatedp)

CTL_PROTO(thread_deallocated)

CTL_PROTO(thread_deallocatedp)

CTL_PROTO(config_cache_oblivious)

CTL_PROTO(config_debug)

CTL_PROTO(config_fill)

CTL_PROTO(config_lazy_lock)

CTL_PROTO(config_malloc_conf)

CTL_PROTO(config_opt_safety_checks)

CTL_PROTO(config_prof)

CTL_PROTO(config_prof_libgcc)

CTL_PROTO(config_prof_libunwind)

CTL_PROTO(config_stats)

CTL_PROTO(config_utrace)

CTL_PROTO(config_xmalloc)

CTL_PROTO(opt_abort)

CTL_PROTO(opt_abort_conf)

CTL_PROTO(opt_confirm_conf)

CTL_PROTO(opt_metadata_thp)

CTL_PROTO(opt_retain)

CTL_PROTO(opt_dss)

CTL_PROTO(opt_narenas)

CTL_PROTO(opt_percpu_arena)

CTL_PROTO(opt_oversize_threshold)

CTL_PROTO(opt_background_thread)

CTL_PROTO(opt_max_background_threads)

CTL_PROTO(opt_dirty_decay_ms)

CTL_PROTO(opt_muzzy_decay_ms)

CTL_PROTO(opt_stats_print)

CTL_PROTO(opt_stats_print_opts)

CTL_PROTO(opt_junk)

CTL_PROTO(opt_zero)

CTL_PROTO(opt_utrace)

CTL_PROTO(opt_xmalloc)

CTL_PROTO(opt_tcache)

CTL_PROTO(opt_thp)

CTL_PROTO(opt_lg_extent_max_active_fit)

CTL_PROTO(opt_lg_tcache_max)

CTL_PROTO(opt_prof)

CTL_PROTO(opt_prof_prefix)

CTL_PROTO(opt_prof_active)

CTL_PROTO(opt_prof_thread_active_init)

CTL_PROTO(opt_lg_prof_sample)

CTL_PROTO(opt_lg_prof_interval)

CTL_PROTO(opt_prof_gdump)

CTL_PROTO(opt_prof_final)

CTL_PROTO(opt_prof_leak)

CTL_PROTO(opt_prof_accum)

CTL_PROTO(tcache_create)

CTL_PROTO(tcache_flush)

CTL_PROTO(tcache_destroy)

CTL_PROTO(arena_i_initialized)

CTL_PROTO(arena_i_decay)

CTL_PROTO(arena_i_purge)

CTL_PROTO(arena_i_reset)

CTL_PROTO(arena_i_destroy)

CTL_PROTO(arena_i_dss)

CTL_PROTO(arena_i_dirty_decay_ms)

CTL_PROTO(arena_i_muzzy_decay_ms)

CTL_PROTO(arena_i_extent_hooks)

CTL_PROTO(arena_i_retain_grow_limit)

INDEX_PROTO(arena_i)

CTL_PROTO(arenas_bin_i_size)

CTL_PROTO(arenas_bin_i_nregs)

CTL_PROTO(arenas_bin_i_slab_size)

CTL_PROTO(arenas_bin_i_nshards)

INDEX_PROTO(arenas_bin_i)

CTL_PROTO(arenas_lextent_i_size)

INDEX_PROTO(arenas_lextent_i)

CTL_PROTO(arenas_narenas)

CTL_PROTO(arenas_dirty_decay_ms)

CTL_PROTO(arenas_muzzy_decay_ms)

CTL_PROTO(arenas_quantum)

CTL_PROTO(arenas_page)

CTL_PROTO(arenas_tcache_max)

CTL_PROTO(arenas_nbins)

CTL_PROTO(arenas_nhbins)

CTL_PROTO(arenas_nlextents)

CTL_PROTO(arenas_create)

CTL_PROTO(arenas_lookup)

CTL_PROTO(prof_thread_active_init)

CTL_PROTO(prof_active)

CTL_PROTO(prof_dump)

CTL_PROTO(prof_gdump)

CTL_PROTO(prof_reset)

CTL_PROTO(prof_interval)

CTL_PROTO(lg_prof_sample)

CTL_PROTO(prof_log_start)

CTL_PROTO(prof_log_stop)

CTL_PROTO(stats_arenas_i_small_allocated)

CTL_PROTO(stats_arenas_i_small_nmalloc)

CTL_PROTO(stats_arenas_i_small_ndalloc)

CTL_PROTO(stats_arenas_i_small_nrequests)

CTL_PROTO(stats_arenas_i_small_nfills)

CTL_PROTO(stats_arenas_i_small_nflushes)

CTL_PROTO(stats_arenas_i_large_allocated)

CTL_PROTO(stats_arenas_i_large_nmalloc)

CTL_PROTO(stats_arenas_i_large_ndalloc)

CTL_PROTO(stats_arenas_i_large_nrequests)

CTL_PROTO(stats_arenas_i_large_nfills)

CTL_PROTO(stats_arenas_i_large_nflushes)

CTL_PROTO(stats_arenas_i_bins_j_nmalloc)

CTL_PROTO(stats_arenas_i_bins_j_ndalloc)

CTL_PROTO(stats_arenas_i_bins_j_nrequests)

CTL_PROTO(stats_arenas_i_bins_j_curregs)

CTL_PROTO(stats_arenas_i_bins_j_nfills)

CTL_PROTO(stats_arenas_i_bins_j_nflushes)

CTL_PROTO(stats_arenas_i_bins_j_nslabs)

CTL_PROTO(stats_arenas_i_bins_j_nreslabs)

CTL_PROTO(stats_arenas_i_bins_j_curslabs)

CTL_PROTO(stats_arenas_i_bins_j_nonfull_slabs)

INDEX_PROTO(stats_arenas_i_bins_j)

CTL_PROTO(stats_arenas_i_lextents_j_nmalloc)

CTL_PROTO(stats_arenas_i_lextents_j_ndalloc)

CTL_PROTO(stats_arenas_i_lextents_j_nrequests)

CTL_PROTO(stats_arenas_i_lextents_j_curlextents)

INDEX_PROTO(stats_arenas_i_lextents_j)

CTL_PROTO(stats_arenas_i_extents_j_ndirty)

CTL_PROTO(stats_arenas_i_extents_j_nmuzzy)

CTL_PROTO(stats_arenas_i_extents_j_nretained)

CTL_PROTO(stats_arenas_i_extents_j_dirty_bytes)

CTL_PROTO(stats_arenas_i_extents_j_muzzy_bytes)

CTL_PROTO(stats_arenas_i_extents_j_retained_bytes)

INDEX_PROTO(stats_arenas_i_extents_j)

CTL_PROTO(stats_arenas_i_nthreads)

CTL_PROTO(stats_arenas_i_uptime)

CTL_PROTO(stats_arenas_i_dss)

CTL_PROTO(stats_arenas_i_dirty_decay_ms)

CTL_PROTO(stats_arenas_i_muzzy_decay_ms)

CTL_PROTO(stats_arenas_i_pactive)

CTL_PROTO(stats_arenas_i_pdirty)

CTL_PROTO(stats_arenas_i_pmuzzy)

CTL_PROTO(stats_arenas_i_mapped)

CTL_PROTO(stats_arenas_i_retained)

CTL_PROTO(stats_arenas_i_extent_avail)

CTL_PROTO(stats_arenas_i_dirty_npurge)

CTL_PROTO(stats_arenas_i_dirty_nmadvise)

CTL_PROTO(stats_arenas_i_dirty_purged)

CTL_PROTO(stats_arenas_i_muzzy_npurge)

CTL_PROTO(stats_arenas_i_muzzy_nmadvise)

CTL_PROTO(stats_arenas_i_muzzy_purged)

CTL_PROTO(stats_arenas_i_base)

CTL_PROTO(stats_arenas_i_internal)

CTL_PROTO(stats_arenas_i_metadata_thp)

CTL_PROTO(stats_arenas_i_tcache_bytes)

CTL_PROTO(stats_arenas_i_resident)

CTL_PROTO(stats_arenas_i_abandoned_vm)

INDEX_PROTO(stats_arenas_i)

CTL_PROTO(stats_allocated)

CTL_PROTO(stats_active)

CTL_PROTO(stats_background_thread_num_threads)

CTL_PROTO(stats_background_thread_num_runs)

CTL_PROTO(stats_background_thread_run_interval)

CTL_PROTO(stats_metadata)

CTL_PROTO(stats_metadata_thp)

CTL_PROTO(stats_resident)

CTL_PROTO(stats_mapped)

CTL_PROTO(stats_retained)

CTL_PROTO(experimental_hooks_install)

CTL_PROTO(experimental_hooks_remove)

CTL_PROTO(experimental_utilization_query)

CTL_PROTO(experimental_utilization_batch_query)

CTL_PROTO(experimental_arenas_i_pactivep)

INDEX_PROTO(experimental_arenas_i)

#define MUTEX_STATS_CTL_PROTO_GEN(n)					\

CTL_PROTO(stats_##n##_num_ops)						\

CTL_PROTO(stats_##n##_num_wait)						\

CTL_PROTO(stats_##n##_num_spin_acq)					\

CTL_PROTO(stats_##n##_num_owner_switch)					\

CTL_PROTO(stats_##n##_total_wait_time)					\

CTL_PROTO(stats_##n##_max_wait_time)					\

CTL_PROTO(stats_##n##_max_num_thds)

/* Global mutexes. */

#define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(mutexes_##mtx)

MUTEX_PROF_GLOBAL_MUTEXES

#undef OP

/* Per arena mutexes. */

#define OP(mtx) MUTEX_STATS_CTL_PROTO_GEN(arenas_i_mutexes_##mtx)

MUTEX_PROF_ARENA_MUTEXES

#undef OP

/* Arena bin mutexes. */

MUTEX_STATS_CTL_PROTO_GEN(arenas_i_bins_j_mutex)

#undef MUTEX_STATS_CTL_PROTO_GEN

CTL_PROTO(stats_mutexes_reset)

/******************************************************************************/

/* mallctl tree. */

#define NAME(n)	{true},	n

#define CHILD(t, c)							\

	sizeof(c##_node) / sizeof(ctl_##t##_node_t),			\

	(ctl_node_t *)c##_node,						\

	NULL

#define CTL(c)	0, NULL, c##_ctl

/*

 * Only handles internal indexed nodes, since there are currently no external

 * ones.

 */

#define INDEX(i)	{false},	i##_index

static const ctl_named_node_t	thread_tcache_node[] = {

	{NAME("enabled"),	CTL(thread_tcache_enabled)},

	{NAME("flush"),		CTL(thread_tcache_flush)}

};

static const ctl_named_node_t	thread_prof_node[] = {

	{NAME("name"),		CTL(thread_prof_name)},

	{NAME("active"),	CTL(thread_prof_active)}

};

static const ctl_named_node_t	thread_node[] = {

	{NAME("arena"),		CTL(thread_arena)},

	{NAME("allocated"),	CTL(thread_allocated)},

	{NAME("allocatedp"),	CTL(thread_allocatedp)},

	{NAME("deallocated"),	CTL(thread_deallocated)},

	{NAME("deallocatedp"),	CTL(thread_deallocatedp)},

	{NAME("tcache"),	CHILD(named, thread_tcache)},

	{NAME("prof"),		CHILD(named, thread_prof)}

};

static const ctl_named_node_t	config_node[] = {

	{NAME("cache_oblivious"), CTL(config_cache_oblivious)},

	{NAME("debug"),		CTL(config_debug)},

	{NAME("fill"),		CTL(config_fill)},

	{NAME("lazy_lock"),	CTL(config_lazy_lock)},

	{NAME("malloc_conf"),	CTL(config_malloc_conf)},

	{NAME("opt_safety_checks"),	CTL(config_opt_safety_checks)},

	{NAME("prof"),		CTL(config_prof)},

	{NAME("prof_libgcc"),	CTL(config_prof_libgcc)},

	{NAME("prof_libunwind"), CTL(config_prof_libunwind)},

	{NAME("stats"),		CTL(config_stats)},

	{NAME("utrace"),	CTL(config_utrace)},

	{NAME("xmalloc"),	CTL(config_xmalloc)}

};

static const ctl_named_node_t opt_node[] = {

	{NAME("abort"),		CTL(opt_abort)},

	{NAME("abort_conf"),	CTL(opt_abort_conf)},

	{NAME("confirm_conf"),	CTL(opt_confirm_conf)},

	{NAME("metadata_thp"),	CTL(opt_metadata_thp)},

	{NAME("retain"),	CTL(opt_retain)},

	{NAME("dss"),		CTL(opt_dss)},

	{NAME("narenas"),	CTL(opt_narenas)},

	{NAME("percpu_arena"),	CTL(opt_percpu_arena)},

	{NAME("oversize_threshold"),	CTL(opt_oversize_threshold)},

	{NAME("background_thread"),	CTL(opt_background_thread)},

	{NAME("max_background_threads"),	CTL(opt_max_background_threads)},

	{NAME("dirty_decay_ms"), CTL(opt_dirty_decay_ms)},

	{NAME("muzzy_decay_ms"), CTL(opt_muzzy_decay_ms)},

	{NAME("stats_print"),	CTL(opt_stats_print)},

	{NAME("stats_print_opts"),	CTL(opt_stats_print_opts)},

	{NAME("junk"),		CTL(opt_junk)},

	{NAME("zero"),		CTL(opt_zero)},

	{NAME("utrace"),	CTL(opt_utrace)},

	{NAME("xmalloc"),	CTL(opt_xmalloc)},

	{NAME("tcache"),	CTL(opt_tcache)},

	{NAME("thp"),		CTL(opt_thp)},

	{NAME("lg_extent_max_active_fit"), CTL(opt_lg_extent_max_active_fit)},

	{NAME("lg_tcache_max"),	CTL(opt_lg_tcache_max)},

	{NAME("prof"),		CTL(opt_prof)},

	{NAME("prof_prefix"),	CTL(opt_prof_prefix)},

	{NAME("prof_active"),	CTL(opt_prof_active)},

	{NAME("prof_thread_active_init"), CTL(opt_prof_thread_active_init)},

	{NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)},

	{NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)},

	{NAME("prof_gdump"),	CTL(opt_prof_gdump)},

	{NAME("prof_final"),	CTL(opt_prof_final)},

	{NAME("prof_leak"),	CTL(opt_prof_leak)},

	{NAME("prof_accum"),	CTL(opt_prof_accum)}

};

static const ctl_named_node_t	tcache_node[] = {

	{NAME("create"),	CTL(tcache_create)},

	{NAME("flush"),		CTL(tcache_flush)},

	{NAME("destroy"),	CTL(tcache_destroy)}

};

static const ctl_named_node_t arena_i_node[] = {

	{NAME("initialized"),	CTL(arena_i_initialized)},

	{NAME("decay"),		CTL(arena_i_decay)},

	{NAME("purge"),		CTL(arena_i_purge)},

	{NAME("reset"),		CTL(arena_i_reset)},

	{NAME("destroy"),	CTL(arena_i_destroy)},

	{NAME("dss"),		CTL(arena_i_dss)},

	{NAME("dirty_decay_ms"), CTL(arena_i_dirty_decay_ms)},

	{NAME("muzzy_decay_ms"), CTL(arena_i_muzzy_decay_ms)},

	{NAME("extent_hooks"),	CTL(arena_i_extent_hooks)},

	{NAME("retain_grow_limit"),	CTL(arena_i_retain_grow_limit)}

};

static const ctl_named_node_t super_arena_i_node[] = {

	{NAME(""),		CHILD(named, arena_i)}

};

static const ctl_indexed_node_t arena_node[] = {

	{INDEX(arena_i)}

};

static const ctl_named_node_t arenas_bin_i_node[] = {

	{NAME("size"),		CTL(arenas_bin_i_size)},

	{NAME("nregs"),		CTL(arenas_bin_i_nregs)},

	{NAME("slab_size"),	CTL(arenas_bin_i_slab_size)},

	{NAME("nshards"),	CTL(arenas_bin_i_nshards)}

};

static const ctl_named_node_t super_arenas_bin_i_node[] = {

	{NAME(""),		CHILD(named, arenas_bin_i)}

};

static const ctl_indexed_node_t arenas_bin_node[] = {

	{INDEX(arenas_bin_i)}

};

static const ctl_named_node_t arenas_lextent_i_node[] = {

	{NAME("size"),		CTL(arenas_lextent_i_size)}

};

static const ctl_named_node_t super_arenas_lextent_i_node[] = {

	{NAME(""),		CHILD(named, arenas_lextent_i)}

};

static const ctl_indexed_node_t arenas_lextent_node[] = {

	{INDEX(arenas_lextent_i)}

};

static const ctl_named_node_t arenas_node[] = {

	{NAME("narenas"),	CTL(arenas_narenas)},

	{NAME("dirty_decay_ms"), CTL(arenas_dirty_decay_ms)},

	{NAME("muzzy_decay_ms"), CTL(arenas_muzzy_decay_ms)},

	{NAME("quantum"),	CTL(arenas_quantum)},

	{NAME("page"),		CTL(arenas_page)},

	{NAME("tcache_max"),	CTL(arenas_tcache_max)},

	{NAME("nbins"),		CTL(arenas_nbins)},

	{NAME("nhbins"),	CTL(arenas_nhbins)},

	{NAME("bin"),		CHILD(indexed, arenas_bin)},

	{NAME("nlextents"),	CTL(arenas_nlextents)},

	{NAME("lextent"),	CHILD(indexed, arenas_lextent)},

	{NAME("create"),	CTL(arenas_create)},

	{NAME("lookup"),	CTL(arenas_lookup)}

};

static const ctl_named_node_t	prof_node[] = {

	{NAME("thread_active_init"), CTL(prof_thread_active_init)},

	{NAME("active"),	CTL(prof_active)},

	{NAME("dump"),		CTL(prof_dump)},

	{NAME("gdump"),		CTL(prof_gdump)},

	{NAME("reset"),		CTL(prof_reset)},

	{NAME("interval"),	CTL(prof_interval)},

	{NAME("lg_sample"),	CTL(lg_prof_sample)},

	{NAME("log_start"),	CTL(prof_log_start)},

	{NAME("log_stop"),	CTL(prof_log_stop)}

};

static const ctl_named_node_t stats_arenas_i_small_node[] = {

	{NAME("allocated"),	CTL(stats_arenas_i_small_allocated)},

	{NAME("nmalloc"),	CTL(stats_arenas_i_small_nmalloc)},

	{NAME("ndalloc"),	CTL(stats_arenas_i_small_ndalloc)},

	{NAME("nrequests"),	CTL(stats_arenas_i_small_nrequests)},

	{NAME("nfills"),	CTL(stats_arenas_i_small_nfills)},

	{NAME("nflushes"),	CTL(stats_arenas_i_small_nflushes)}

};

static const ctl_named_node_t stats_arenas_i_large_node[] = {

	{NAME("allocated"),	CTL(stats_arenas_i_large_allocated)},

	{NAME("nmalloc"),	CTL(stats_arenas_i_large_nmalloc)},

	{NAME("ndalloc"),	CTL(stats_arenas_i_large_ndalloc)},

	{NAME("nrequests"),	CTL(stats_arenas_i_large_nrequests)},

	{NAME("nfills"),	CTL(stats_arenas_i_large_nfills)},

	{NAME("nflushes"),	CTL(stats_arenas_i_large_nflushes)}

};

#define MUTEX_PROF_DATA_NODE(prefix)					\

static const ctl_named_node_t stats_##prefix##_node[] = {		\

	{NAME("num_ops"),						\

	 CTL(stats_##prefix##_num_ops)},				\

	{NAME("num_wait"),						\

	 CTL(stats_##prefix##_num_wait)},				\

	{NAME("num_spin_acq"),						\

	 CTL(stats_##prefix##_num_spin_acq)},				\

	{NAME("num_owner_switch"),					\

	 CTL(stats_##prefix##_num_owner_switch)},			\

	{NAME("total_wait_time"),					\

	 CTL(stats_##prefix##_total_wait_time)},			\

	{NAME("max_wait_time"),						\

	 CTL(stats_##prefix##_max_wait_time)},				\

	{NAME("max_num_thds"),						\

	 CTL(stats_##prefix##_max_num_thds)}				\

	/* Note that # of current waiting thread not provided. */	\

};

MUTEX_PROF_DATA_NODE(arenas_i_bins_j_mutex)

static const ctl_named_node_t stats_arenas_i_bins_j_node[] = {

	{NAME("nmalloc"),	CTL(stats_arenas_i_bins_j_nmalloc)},

	{NAME("ndalloc"),	CTL(stats_arenas_i_bins_j_ndalloc)},

	{NAME("nrequests"),	CTL(stats_arenas_i_bins_j_nrequests)},

	{NAME("curregs"),	CTL(stats_arenas_i_bins_j_curregs)},

	{NAME("nfills"),	CTL(stats_arenas_i_bins_j_nfills)},

	{NAME("nflushes"),	CTL(stats_arenas_i_bins_j_nflushes)},

	{NAME("nslabs"),	CTL(stats_arenas_i_bins_j_nslabs)},

	{NAME("nreslabs"),	CTL(stats_arenas_i_bins_j_nreslabs)},

	{NAME("curslabs"),	CTL(stats_arenas_i_bins_j_curslabs)},

	{NAME("nonfull_slabs"),	CTL(stats_arenas_i_bins_j_nonfull_slabs)},

	{NAME("mutex"),		CHILD(named, stats_arenas_i_bins_j_mutex)}

};

static const ctl_named_node_t super_stats_arenas_i_bins_j_node[] = {

	{NAME(""),		CHILD(named, stats_arenas_i_bins_j)}

};

static const ctl_indexed_node_t stats_arenas_i_bins_node[] = {

	{INDEX(stats_arenas_i_bins_j)}

};

static const ctl_named_node_t stats_arenas_i_lextents_j_node[] = {

	{NAME("nmalloc"),	CTL(stats_arenas_i_lextents_j_nmalloc)},

	{NAME("ndalloc"),	CTL(stats_arenas_i_lextents_j_ndalloc)},

	{NAME("nrequests"),	CTL(stats_arenas_i_lextents_j_nrequests)},

	{NAME("curlextents"),	CTL(stats_arenas_i_lextents_j_curlextents)}

};

static const ctl_named_node_t super_stats_arenas_i_lextents_j_node[] = {

	{NAME(""),		CHILD(named, stats_arenas_i_lextents_j)}

};

static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = {

	{INDEX(stats_arenas_i_lextents_j)}

};

static const ctl_named_node_t stats_arenas_i_extents_j_node[] = {

	{NAME("ndirty"),	CTL(stats_arenas_i_extents_j_ndirty)},

	{NAME("nmuzzy"),	CTL(stats_arenas_i_extents_j_nmuzzy)},

	{NAME("nretained"),	CTL(stats_arenas_i_extents_j_nretained)},

	{NAME("dirty_bytes"),	CTL(stats_arenas_i_extents_j_dirty_bytes)},

	{NAME("muzzy_bytes"),	CTL(stats_arenas_i_extents_j_muzzy_bytes)},

	{NAME("retained_bytes"), CTL(stats_arenas_i_extents_j_retained_bytes)}

};

static const ctl_named_node_t super_stats_arenas_i_extents_j_node[] = {

	{NAME(""),		CHILD(named, stats_arenas_i_extents_j)}

};

static const ctl_indexed_node_t stats_arenas_i_extents_node[] = {

	{INDEX(stats_arenas_i_extents_j)}

};

#define OP(mtx)  MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##mtx)

MUTEX_PROF_ARENA_MUTEXES

#undef OP

static const ctl_named_node_t stats_arenas_i_mutexes_node[] = {

#define OP(mtx) {NAME(#mtx), CHILD(named, stats_arenas_i_mutexes_##mtx)},

MUTEX_PROF_ARENA_MUTEXES

#undef OP

};

static const ctl_named_node_t stats_arenas_i_node[] = {

	{NAME("nthreads"),	CTL(stats_arenas_i_nthreads)},

	{NAME("uptime"),	CTL(stats_arenas_i_uptime)},

	{NAME("dss"),		CTL(stats_arenas_i_dss)},

	{NAME("dirty_decay_ms"), CTL(stats_arenas_i_dirty_decay_ms)},

	{NAME("muzzy_decay_ms"), CTL(stats_arenas_i_muzzy_decay_ms)},

	{NAME("pactive"),	CTL(stats_arenas_i_pactive)},

	{NAME("pdirty"),	CTL(stats_arenas_i_pdirty)},

	{NAME("pmuzzy"),	CTL(stats_arenas_i_pmuzzy)},

	{NAME("mapped"),	CTL(stats_arenas_i_mapped)},

	{NAME("retained"),	CTL(stats_arenas_i_retained)},

	{NAME("extent_avail"),	CTL(stats_arenas_i_extent_avail)},

	{NAME("dirty_npurge"),	CTL(stats_arenas_i_dirty_npurge)},

	{NAME("dirty_nmadvise"), CTL(stats_arenas_i_dirty_nmadvise)},

	{NAME("dirty_purged"),	CTL(stats_arenas_i_dirty_purged)},

	{NAME("muzzy_npurge"),	CTL(stats_arenas_i_muzzy_npurge)},

	{NAME("muzzy_nmadvise"), CTL(stats_arenas_i_muzzy_nmadvise)},

	{NAME("muzzy_purged"),	CTL(stats_arenas_i_muzzy_purged)},

	{NAME("base"),		CTL(stats_arenas_i_base)},

	{NAME("internal"),	CTL(stats_arenas_i_internal)},

	{NAME("metadata_thp"),	CTL(stats_arenas_i_metadata_thp)},

	{NAME("tcache_bytes"),	CTL(stats_arenas_i_tcache_bytes)},

	{NAME("resident"),	CTL(stats_arenas_i_resident)},

	{NAME("abandoned_vm"),	CTL(stats_arenas_i_abandoned_vm)},

	{NAME("small"),		CHILD(named, stats_arenas_i_small)},

	{NAME("large"),		CHILD(named, stats_arenas_i_large)},

	{NAME("bins"),		CHILD(indexed, stats_arenas_i_bins)},

	{NAME("lextents"),	CHILD(indexed, stats_arenas_i_lextents)},

	{NAME("extents"),	CHILD(indexed, stats_arenas_i_extents)},

	{NAME("mutexes"),	CHILD(named, stats_arenas_i_mutexes)}

};

static const ctl_named_node_t super_stats_arenas_i_node[] = {

	{NAME(""),		CHILD(named, stats_arenas_i)}

};

static const ctl_indexed_node_t stats_arenas_node[] = {

	{INDEX(stats_arenas_i)}

};

static const ctl_named_node_t stats_background_thread_node[] = {

	{NAME("num_threads"),	CTL(stats_background_thread_num_threads)},

	{NAME("num_runs"),	CTL(stats_background_thread_num_runs)},

	{NAME("run_interval"),	CTL(stats_background_thread_run_interval)}

};

#define OP(mtx) MUTEX_PROF_DATA_NODE(mutexes_##mtx)

MUTEX_PROF_GLOBAL_MUTEXES

#undef OP

static const ctl_named_node_t stats_mutexes_node[] = {

#define OP(mtx) {NAME(#mtx), CHILD(named, stats_mutexes_##mtx)},

MUTEX_PROF_GLOBAL_MUTEXES

#undef OP

	{NAME("reset"),		CTL(stats_mutexes_reset)}

};

#undef MUTEX_PROF_DATA_NODE

static const ctl_named_node_t stats_node[] = {

	{NAME("allocated"),	CTL(stats_allocated)},

	{NAME("active"),	CTL(stats_active)},

	{NAME("metadata"),	CTL(stats_metadata)},

	{NAME("metadata_thp"),	CTL(stats_metadata_thp)},

	{NAME("resident"),	CTL(stats_resident)},

	{NAME("mapped"),	CTL(stats_mapped)},

	{NAME("retained"),	CTL(stats_retained)},

	{NAME("background_thread"),

	 CHILD(named, stats_background_thread)},

	{NAME("mutexes"),	CHILD(named, stats_mutexes)},

	{NAME("arenas"),	CHILD(indexed, stats_arenas)}

};

static const ctl_named_node_t experimental_hooks_node[] = {

	{NAME("install"),	CTL(experimental_hooks_install)},

	{NAME("remove"),	CTL(experimental_hooks_remove)}

};

static const ctl_named_node_t experimental_utilization_node[] = {

	{NAME("query"),		CTL(experimental_utilization_query)},

	{NAME("batch_query"),	CTL(experimental_utilization_batch_query)}

};

static const ctl_named_node_t experimental_arenas_i_node[] = {

	{NAME("pactivep"),	CTL(experimental_arenas_i_pactivep)}

};

static const ctl_named_node_t super_experimental_arenas_i_node[] = {

	{NAME(""),		CHILD(named, experimental_arenas_i)}

};

static const ctl_indexed_node_t experimental_arenas_node[] = {

	{INDEX(experimental_arenas_i)}

};

static const ctl_named_node_t experimental_node[] = {

	{NAME("hooks"),		CHILD(named, experimental_hooks)},

	{NAME("utilization"),	CHILD(named, experimental_utilization)},

	{NAME("arenas"),	CHILD(indexed, experimental_arenas)}

};

static const ctl_named_node_t	root_node[] = {

	{NAME("version"),	CTL(version)},

	{NAME("epoch"),		CTL(epoch)},

	{NAME("background_thread"),	CTL(background_thread)},

	{NAME("max_background_threads"),	CTL(max_background_threads)},

	{NAME("thread"),	CHILD(named, thread)},

	{NAME("config"),	CHILD(named, config)},

	{NAME("opt"),		CHILD(named, opt)},

	{NAME("tcache"),	CHILD(named, tcache)},

	{NAME("arena"),		CHILD(indexed, arena)},

	{NAME("arenas"),	CHILD(named, arenas)},

	{NAME("prof"),		CHILD(named, prof)},

	{NAME("stats"),		CHILD(named, stats)},

	{NAME("experimental"),	CHILD(named, experimental)}

};

static const ctl_named_node_t super_root_node[] = {

	{NAME(""),		CHILD(named, root)}

};

#undef NAME

#undef CHILD

#undef CTL

#undef INDEX

/******************************************************************************/

/*

 * Sets *dst + *src non-atomically.  This is safe, since everything is

 * synchronized by the ctl mutex.

 */

static void

ctl_accum_arena_stats_u64(arena_stats_u64_t *dst, arena_stats_u64_t *src) {

#ifdef JEMALLOC_ATOMIC_U64

	uint64_t cur_dst = atomic_load_u64(dst, ATOMIC_RELAXED);

	uint64_t cur_src = atomic_load_u64(src, ATOMIC_RELAXED);

	atomic_store_u64(dst, cur_dst + cur_src, ATOMIC_RELAXED);

#else

	*dst += *src;

#endif

}

/* Likewise: with ctl mutex synchronization, reading is simple. */

static uint64_t

ctl_arena_stats_read_u64(arena_stats_u64_t *p) {

#ifdef JEMALLOC_ATOMIC_U64

	return atomic_load_u64(p, ATOMIC_RELAXED);

#else

	return *p;

#endif

}

static void

accum_atomic_zu(atomic_zu_t *dst, atomic_zu_t *src) {

	size_t cur_dst = atomic_load_zu(dst, ATOMIC_RELAXED);

	size_t cur_src = atomic_load_zu(src, ATOMIC_RELAXED);

	atomic_store_zu(dst, cur_dst + cur_src, ATOMIC_RELAXED);

}

/******************************************************************************/

static unsigned

arenas_i2a_impl(size_t i, bool compat, bool validate) {

	unsigned a;

	switch (i) {

	case MALLCTL_ARENAS_ALL:

		a = 0;

		break;

	case MALLCTL_ARENAS_DESTROYED:

		a = 1;

		break;

	default:

		if (compat && i == ctl_arenas->narenas) {

			/*

			 * Provide deprecated backward compatibility for

			 * accessing the merged stats at index narenas rather

			 * than via MALLCTL_ARENAS_ALL.  This is scheduled for

			 * removal in 6.0.0.

			 */

			a = 0;

		} else if (validate && i >= ctl_arenas->narenas) {

			a = UINT_MAX;

		} else {

			/*

			 * This function should never be called for an index

			 * more than one past the range of indices that have

			 * initialized ctl data.

			 */

			assert(i < ctl_arenas->narenas || (!validate && i ==

			    ctl_arenas->narenas));

			a = (unsigned)i + 2;

		}

		break;

	}

	return a;

}

static unsigned

arenas_i2a(size_t i) {

	return arenas_i2a_impl(i, true, false);

}

static ctl_arena_t *

arenas_i_impl(tsd_t *tsd, size_t i, bool compat, bool init) {

	ctl_arena_t *ret;

	assert(!compat || !init);

	ret = ctl_arenas->arenas[arenas_i2a_impl(i, compat, false)];

	if (init && ret == NULL) {

		if (config_stats) {

			struct container_s {

				ctl_arena_t		ctl_arena;

				ctl_arena_stats_t	astats;

			};

			struct container_s *cont =

			    (struct container_s *)base_alloc(tsd_tsdn(tsd),

			    b0get(), sizeof(struct container_s), QUANTUM);

			if (cont == NULL) {

				return NULL;

			}

			ret = &cont->ctl_arena;

			ret->astats = &cont->astats;

		} else {

			ret = (ctl_arena_t *)base_alloc(tsd_tsdn(tsd), b0get(),

			    sizeof(ctl_arena_t), QUANTUM);

			if (ret == NULL) {

				return NULL;

			}

		}

		ret->arena_ind = (unsigned)i;

		ctl_arenas->arenas[arenas_i2a_impl(i, compat, false)] = ret;

	}

	assert(ret == NULL || arenas_i2a(ret->arena_ind) == arenas_i2a(i));

	return ret;

}

static ctl_arena_t *

arenas_i(size_t i) {

	ctl_arena_t *ret = arenas_i_impl(tsd_fetch(), i, true, false);

	assert(ret != NULL);

	return ret;

}

static void

ctl_arena_clear(ctl_arena_t *ctl_arena) {

	ctl_arena->nthreads = 0;

	ctl_arena->dss = dss_prec_names[dss_prec_limit];

	ctl_arena->dirty_decay_ms = -1;

	ctl_arena->muzzy_decay_ms = -1;

	ctl_arena->pactive = 0;

	ctl_arena->pdirty = 0;

	ctl_arena->pmuzzy = 0;

	if (config_stats) {

		memset(&ctl_arena->astats->astats, 0, sizeof(arena_stats_t));

		ctl_arena->astats->allocated_small = 0;

		ctl_arena->astats->nmalloc_small = 0;

		ctl_arena->astats->ndalloc_small = 0;

		ctl_arena->astats->nrequests_small = 0;

		ctl_arena->astats->nfills_small = 0;

		ctl_arena->astats->nflushes_small = 0;

		memset(ctl_arena->astats->bstats, 0, SC_NBINS *

		    sizeof(bin_stats_t));

		memset(ctl_arena->astats->lstats, 0, (SC_NSIZES - SC_NBINS) *

		    sizeof(arena_stats_large_t));

		memset(ctl_arena->astats->estats, 0, SC_NPSIZES *

		    sizeof(arena_stats_extents_t));

	}

}

static void

ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) {

	unsigned i;

	if (config_stats) {

		arena_stats_merge(tsdn, arena, &ctl_arena->nthreads,

		    &ctl_arena->dss, &ctl_arena->dirty_decay_ms,

		    &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive,

		    &ctl_arena->pdirty, &ctl_arena->pmuzzy,

		    &ctl_arena->astats->astats, ctl_arena->astats->bstats,

		    ctl_arena->astats->lstats, ctl_arena->astats->estats);

		for (i = 0; i < SC_NBINS; i++) {

			ctl_arena->astats->allocated_small +=

			    ctl_arena->astats->bstats[i].curregs *

			    sz_index2size(i);

			ctl_arena->astats->nmalloc_small +=

			    ctl_arena->astats->bstats[i].nmalloc;

			ctl_arena->astats->ndalloc_small +=

			    ctl_arena->astats->bstats[i].ndalloc;

			ctl_arena->astats->nrequests_small +=

			    ctl_arena->astats->bstats[i].nrequests;

			ctl_arena->astats->nfills_small +=

			    ctl_arena->astats->bstats[i].nfills;

			ctl_arena->astats->nflushes_small +=

			    ctl_arena->astats->bstats[i].nflushes;

		}

	} else {

		arena_basic_stats_merge(tsdn, arena, &ctl_arena->nthreads,

		    &ctl_arena->dss, &ctl_arena->dirty_decay_ms,

		    &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive,

		    &ctl_arena->pdirty, &ctl_arena->pmuzzy);

	}

}

static void

ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena,

    bool destroyed) {

	unsigned i;

	if (!destroyed) {

		ctl_sdarena->nthreads += ctl_arena->nthreads;

		ctl_sdarena->pactive += ctl_arena->pactive;

		ctl_sdarena->pdirty += ctl_arena->pdirty;

		ctl_sdarena->pmuzzy += ctl_arena->pmuzzy;

	} else {

		assert(ctl_arena->nthreads == 0);

		assert(ctl_arena->pactive == 0);

		assert(ctl_arena->pdirty == 0);

		assert(ctl_arena->pmuzzy == 0);

	}

	if (config_stats) {

		ctl_arena_stats_t *sdstats = ctl_sdarena->astats;

		ctl_arena_stats_t *astats = ctl_arena->astats;

		if (!destroyed) {

			accum_atomic_zu(&sdstats->astats.mapped,

			    &astats->astats.mapped);

			accum_atomic_zu(&sdstats->astats.retained,

			    &astats->astats.retained);

			accum_atomic_zu(&sdstats->astats.extent_avail,

			    &astats->astats.extent_avail);

		}

		ctl_accum_arena_stats_u64(&sdstats->astats.decay_dirty.npurge,

		    &astats->astats.decay_dirty.npurge);

		ctl_accum_arena_stats_u64(&sdstats->astats.decay_dirty.nmadvise,

		    &astats->astats.decay_dirty.nmadvise);

		ctl_accum_arena_stats_u64(&sdstats->astats.decay_dirty.purged,

		    &astats->astats.decay_dirty.purged);

		ctl_accum_arena_stats_u64(&sdstats->astats.decay_muzzy.npurge,

		    &astats->astats.decay_muzzy.npurge);

		ctl_accum_arena_stats_u64(&sdstats->astats.decay_muzzy.nmadvise,

		    &astats->astats.decay_muzzy.nmadvise);

		ctl_accum_arena_stats_u64(&sdstats->astats.decay_muzzy.purged,

		    &astats->astats.decay_muzzy.purged);

#define OP(mtx) malloc_mutex_prof_merge(				\

		    &(sdstats->astats.mutex_prof_data[			\

		        arena_prof_mutex_##mtx]),			\

		    &(astats->astats.mutex_prof_data[			\

		        arena_prof_mutex_##mtx]));

MUTEX_PROF_ARENA_MUTEXES

#undef OP

		if (!destroyed) {