Skip to content

Commit 69ecba7

Browse files
michaelsproulmichaelsproul
committed
Add jemalloc support
1 parent 8600645 commit 69ecba7

8 files changed

Lines changed: 138 additions & 5 deletions

File tree

‎.cargo/config.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
[env]
2+
# Set the number of arenas to 16 when using jemalloc.
3+
JEMALLOC_SYS_WITH_MALLOC_CONF = "abort_conf:true,narenas:16"
4+

‎Cargo.lock‎

Lines changed: 40 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Makefile‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,11 @@ BUILD_PATH_AARCH64 = "target/$(AARCH64_TAG)/release"
1414
PINNED_NIGHTLY ?= nightly
1515
CLIPPY_PINNED_NIGHTLY=nightly-2022-05-19
1616

17+
# List of features to use when building natively. Can be overriden via the environment.
18+
FEATURES ?= jemalloc
19+
1720
# List of features to use when cross-compiling. Can be overridden via the environment.
18-
CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx
21+
CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx,jemalloc
1922

2023
# Cargo profile for Cross builds. Default is for local builds, CI uses an override.
2124
CROSS_PROFILE ?= release

‎common/malloc_utils/Cargo.toml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,16 @@ version = "0.1.0"
44
authors = ["Paul Hauner <paul@paulhauner.com>"]
55
edition = "2021"
66

7-
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
8-
97
[dependencies]
108
lighthouse_metrics = { path = "../lighthouse_metrics" }
119
lazy_static = "1.4.0"
1210
libc = "0.2.79"
1311
parking_lot = "0.12.0"
12+
jemallocator = { version = "0.5.0", optional = true, features = ["background_threads"] }
13+
jemalloc-ctl = { version = "0.5.0", optional = true }
1414

1515
[features]
1616
mallinfo2 = []
17+
jemalloc = ["jemallocator", "jemallocator/stats", "jemalloc-ctl"]
18+
jemalloc-stats = ["jemallocator/stats"]
19+
jemalloc-profiling = ["jemallocator/profiling"]

‎common/malloc_utils/src/jemalloc.rs‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
//! Set the allocator to `jemalloc`.
2+
//!
3+
//! Due to `jemalloc` requiring configuration at compile time or immediately upon runtime
4+
//! initialisation it is configured via a Cargo config file in `.cargo/config.toml`.
5+
//!
6+
//! The `jemalloc` tuning can be overriden by:
7+
//!
8+
//! A) `JEMALLOC_SYS_WITH_MALLOC_CONF` at compile-time.
9+
//! B) `_RJEM_MALLOC_CONF` at runtime.
10+
use jemalloc_ctl::{arenas, epoch, stats, Error};
11+
use lazy_static::lazy_static;
12+
use lighthouse_metrics::{set_gauge, try_create_int_gauge, IntGauge};
13+
14+
#[global_allocator]
15+
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
16+
17+
// Metrics for jemalloc.
18+
lazy_static! {
19+
pub static ref NUM_ARENAS: lighthouse_metrics::Result<IntGauge> =
20+
try_create_int_gauge("jemalloc_num_arenas", "The number of arenas in use");
21+
pub static ref BYTES_ALLOCATED: lighthouse_metrics::Result<IntGauge> =
22+
try_create_int_gauge("jemalloc_bytes_allocated", "Equivalent to stats.allocated");
23+
pub static ref BYTES_ACTIVE: lighthouse_metrics::Result<IntGauge> =
24+
try_create_int_gauge("jemalloc_bytes_active", "Equivalent to stats.active");
25+
pub static ref BYTES_MAPPED: lighthouse_metrics::Result<IntGauge> =
26+
try_create_int_gauge("jemalloc_bytes_mapped", "Equivalent to stats.mapped");
27+
pub static ref BYTES_METADATA: lighthouse_metrics::Result<IntGauge> =
28+
try_create_int_gauge("jemalloc_bytes_metadata", "Equivalent to stats.metadata");
29+
pub static ref BYTES_RESIDENT: lighthouse_metrics::Result<IntGauge> =
30+
try_create_int_gauge("jemalloc_bytes_resident", "Equivalent to stats.resident");
31+
pub static ref BYTES_RETAINED: lighthouse_metrics::Result<IntGauge> =
32+
try_create_int_gauge("jemalloc_bytes_retained", "Equivalent to stats.retained");
33+
}
34+
35+
pub fn scrape_jemalloc_metrics() {
36+
scrape_jemalloc_metrics_fallible().unwrap()
37+
}
38+
39+
pub fn scrape_jemalloc_metrics_fallible() -> Result<(), Error> {
40+
// Advance the epoch so that the underlying statistics are updated.
41+
epoch::advance()?;
42+
43+
set_gauge(&NUM_ARENAS, arenas::narenas::read()? as i64);
44+
set_gauge(&BYTES_ALLOCATED, stats::allocated::read()? as i64);
45+
set_gauge(&BYTES_ACTIVE, stats::active::read()? as i64);
46+
set_gauge(&BYTES_MAPPED, stats::mapped::read()? as i64);
47+
set_gauge(&BYTES_METADATA, stats::metadata::read()? as i64);
48+
set_gauge(&BYTES_RESIDENT, stats::resident::read()? as i64);
49+
set_gauge(&BYTES_RETAINED, stats::retained::read()? as i64);
50+
51+
Ok(())
52+
}

‎common/malloc_utils/src/lib.rs‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,17 +24,36 @@
2424
//! detecting `glibc` are best-effort. If this crate throws errors about undefined external
2525
//! functions, then try to compile with the `not_glibc_interface` module.
2626
27-
#[cfg(all(target_os = "linux", not(target_env = "musl")))]
27+
#[cfg(all(
28+
target_os = "linux",
29+
not(any(target_env = "musl", feature = "jemalloc"))
30+
))]
2831
mod glibc;
2932

33+
#[cfg(feature = "jemalloc")]
34+
mod jemalloc;
35+
3036
pub use interface::*;
3137

32-
#[cfg(all(target_os = "linux", not(target_env = "musl")))]
38+
#[cfg(all(
39+
target_os = "linux",
40+
not(any(target_env = "musl", feature = "jemalloc"))
41+
))]
3342
mod interface {
3443
pub use crate::glibc::configure_glibc_malloc as configure_memory_allocator;
3544
pub use crate::glibc::scrape_mallinfo_metrics as scrape_allocator_metrics;
3645
}
3746

47+
#[cfg(feature = "jemalloc")]
48+
mod interface {
49+
#[allow(dead_code)]
50+
pub fn configure_memory_allocator() -> Result<(), String> {
51+
Ok(())
52+
}
53+
54+
pub use crate::jemalloc::scrape_jemalloc_metrics as scrape_allocator_metrics;
55+
}
56+
3857
#[cfg(any(not(target_os = "linux"), target_env = "musl"))]
3958
mod interface {
4059
#[allow(dead_code, clippy::unnecessary_wraps)]

‎lighthouse/Cargo.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ gnosis = []
2424
slasher-mdbx = ["slasher/mdbx"]
2525
# Support slasher LMDB backend.
2626
slasher-lmdb = ["slasher/lmdb"]
27+
# Use jemalloc.
28+
jemalloc = ["malloc_utils/jemalloc"]
2729

2830
[dependencies]
2931
beacon_node = { "path" = "../beacon_node" }

‎lighthouse/src/main.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,14 @@ fn bls_library_name() -> &'static str {
3131
}
3232
}
3333

34+
fn allocator_name() -> &'static str {
35+
if cfg!(feature = "jemalloc") {
36+
"jemalloc"
37+
} else {
38+
"system"
39+
}
40+
}
41+
3442
fn main() {
3543
// Enable backtraces unless a RUST_BACKTRACE value has already been explicitly provided.
3644
if std::env::var("RUST_BACKTRACE").is_err() {
@@ -51,10 +59,12 @@ fn main() {
5159
"{}\n\
5260
BLS library: {}\n\
5361
SHA256 hardware acceleration: {}\n\
62+
Allocator: {}\n\
5463
Specs: mainnet (true), minimal ({}), gnosis ({})",
5564
VERSION.replace("Lighthouse/", ""),
5665
bls_library_name(),
5766
have_sha_extensions(),
67+
allocator_name(),
5868
cfg!(feature = "spec-minimal"),
5969
cfg!(feature = "gnosis"),
6070
).as_str()

0 commit comments

Comments
 (0)