Skip to content
This repository has been archived by the owner on Dec 9, 2018. It is now read-only.

Commit

Permalink
add cargo feature to enable jemalloc (disabled by default)
Browse files Browse the repository at this point in the history
  • Loading branch information
Jorge Aparicio committed Jun 6, 2015
1 parent 3f7ffd9 commit 1b7bfd8
Show file tree
Hide file tree
Showing 4 changed files with 254 additions and 12 deletions.
3 changes: 3 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,9 @@ The `remove-mno-compact-eh-flag.patch` only matters if you are targeting the
The `optional-backtrace.patch` modifies Rust source code to allow disabling backtrace support via a
cargo feature.

The `optional-jemalloc.patch` modifies Rust source code to allow picking between malloc and
jemalloc via a cargo feature.

## Known limitations

### Huge binaries in debug profile
Expand Down
221 changes: 209 additions & 12 deletions cargo-ify.patch
View file
Original file line number Diff line number Diff line change
@@ -1,34 +1,226 @@
diff --git a/src/liballoc/Cargo.toml b/src/liballoc/Cargo.toml
new file mode 100644
index 0000000..4123179
index 0000000..e84471e
--- /dev/null
+++ b/src/liballoc/Cargo.toml
@@ -0,0 +1,19 @@
@@ -0,0 +1,18 @@
+[package]
+authors = ["The Rust Project Developers"]
+build = "build.rs"
+name = "alloc"
+version = "0.0.0"
+
+[lib]
+name = "alloc"
+path = "lib.rs"
+
+[features]
+jemalloc = []
+
+[dependencies.core]
+path = "../libcore"
+
+[dependencies.libc]
+path = "../liblibc"
diff --git a/src/liballoc/build.rs b/src/liballoc/build.rs
new file mode 100644
index 0000000..3d92f8c
--- /dev/null
+++ b/src/liballoc/build.rs
@@ -0,0 +1,187 @@
+#![allow(dead_code)]
+
+use std::fs::File;
+use std::io::Read;
+use std::ops::Range;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::{env, fs, iter};
+
+fn main() {
+ let ref ctxt = Ctxt::new().unwrap();
+
+ println!("cargo:rustc-link-search=native={}", ctxt.dst().display());
+
+ match () {
+ #[cfg(feature = "jemalloc")]
+ () => jemalloc(ctxt),
+ #[cfg(not(feature = "jemalloc"))]
+ () => {},
+ }
+}
+
+struct Cflags<'a> {
+ host: Option<std::str::Split<'a, char>>,
+ target: iter::Peekable<std::str::Split<'a, char>>,
+}
+
+impl<'a> Iterator for Cflags<'a> {
+ type Item = &'a str;
+
+# TODO jemalloc support, in both static and dynamic flavors
+#[features]
+#default = ["jemalloc"]
+#jemalloc = []
+ // Host flags may look like: "-O2 -pipe", target flags may look like "-Wall -fPIC $(CFLAGS)"
+ // Collecting this iterator should give you: ["-Wall", "-fPIC", "-O2", "-pipe"]
+ fn next(&mut self) -> Option<&'a str> {
+ if self.target.peek() == Some(&"$(CFLAGS)") {
+ self.host.as_mut().and_then(|it| it.next()).or_else(|| {
+ self.target.next(); // drop $(CFLAGS)
+ self.target.next()
+ })
+ } else {
+ self.target.next()
+ }
+ }
+}
+
+struct Ctxt {
+ ar: String,
+ cc: String,
+ cfg: String,
+ cfg_cflags: Range<usize>,
+ cflags: Option<String>,
+ dst: PathBuf,
+ host: String,
+ src: PathBuf,
+ target: String,
+}
+
+impl Ctxt {
+ fn new() -> Result<Ctxt, String> {
+ // mips-unknown-linux-gnu
+ let target = try!(env::var("TARGET").map_err(|_| "cargo bug: $TARGET not set"));
+
+ // mips_unknown_linux_gnu
+ let target_ = target.replace("-", "_");
+
+ // x86_64-unknown-linux-gnu
+ let host = try!(env::var("HOST").map_err(|_| "cargo bug: $HOST not set"));
+
+ // mips-openwrt-linux-ar
+ let ar = {
+ let ref var = format!("AR_{}", target_);
+ try!(env::var(var).map_err(|_| format!("error: {} not set", var)))
+ };
+
+ // mips-openwrt-linux-gcc
+ let cc = {
+ let ref var = format!("CC_{}", target_);
+ try!(env::var(var).map_err(|_| format!("error: {} not set", var)))
+ };
+
+ let src = PathBuf::from(try!(env::var("CARGO_MANIFEST_DIR").map_err(|_| {
+ "cargo bug: $CARGO_MANIFEST_DIR not set"
+ })));
+
+ let dst = PathBuf::from(try!(env::var("OUT_DIR").map_err(|_| {
+ "cargo bug: $OUT_DIR not set"
+ })));
+
+ let cflags = env::var("CFLAGS").ok();
+
+ let mut cfg = String::new();
+ try! {
+ File::open(format!("../../mk/cfg/{}.mk", target))
+ .and_then(|mut f| {
+ f.read_to_string(&mut cfg).map(|_| {})
+ })
+ .map_err(|_| format!("error: couldn't read {}.mk", target))
+ }
+
+ // CFG_JEMALLOC_CFLAGS_$(TARGET_) := $(CFLAGS)
+ let cfg_cflags = try!(cfg.find("CFG_JEMALLOC_CFLAGS_").and_then(|sol| {
+ cfg[sol..].find(":= ").and_then(|start| {
+ cfg[sol+start..].find("\n").and_then(|end| {
+ Some(sol+start+":= ".len()..sol+start+end)
+ })
+ })
+ }).ok_or(format!("couldn't find CFLAGS in {}.mk", target)));
+
+ Ok(Ctxt {
+ ar: ar,
+ cc: cc,
+ cfg: cfg,
+ cfg_cflags: cfg_cflags,
+ cflags: cflags,
+ dst: dst,
+ host: host,
+ src: src,
+ target: target,
+ })
+ }
+
+ fn ar(&self) -> &str {
+ &self.ar
+ }
+
+ fn cc(&self) -> &str {
+ &self.cc
+ }
+
+ fn cflags(&self) -> Cflags {
+ Cflags {
+ host: self.cflags.as_ref().map(|s| s.split(' ')),
+ target: self.cfg[self.cfg_cflags.clone()].trim().split(' ').peekable()
+ }
+ }
+
+ fn dst(&self) -> &Path {
+ &self.dst
+ }
+
+ fn host(&self) -> &str {
+ &self.host
+ }
+
+ fn src(&self) -> &Path {
+ &self.src
+ }
+
+ fn target(&self) -> &str {
+ &self.target
+ }
+}
+
+/// Build `libjemalloc`
+// see mk/rt.mk
+fn jemalloc(ctxt: &Ctxt) {
+ let ar = ctxt.ar();
+ let ref dst = ctxt.dst().join("jemalloc");
+ let ref src = ctxt.src().join("../jemalloc");
+
+ fs::create_dir_all(dst).ok();
+
+ assert! {
+ Command::new(src.join("configure"))
+ .current_dir(dst)
+ .arg("--with-jemalloc-prefix=je_")
+ .arg("--disable-fill") // see CFG_JEMALLOC_FLAGS at mk/main.mk
+ .arg(format!("--build={}", ctxt.host()))
+ .arg(format!("--host={}", ctxt.target()))
+ .arg(format!("CC={} {}", ctxt.cc(), ctxt.cflags().collect::<Vec<_>>().connect(" ")))
+ .arg(format!("AR={}", ar))
+ .arg(format!("RANLIB={} s", ar))
+ .arg(format!("CPPFLAGS=-I {}", ctxt.src().join("../rt/").display()))
+ .arg("EXTRA_CFLAGS=-g1 -ffunction-sections -fdata-sections")
+ .status().unwrap().success()
+ }
+
+ assert! {
+ Command::new("make")
+ .arg("-C").arg(dst)
+ .arg("build_lib_static")
+ .status().unwrap().success()
+ }
+
+ fs::copy(dst.join("lib/libjemalloc_pic.a"), ctxt.dst().join("libjemalloc.a")).unwrap();
+
+ println!("cargo:rustc-link-lib=static=jemalloc");
+}
diff --git a/src/libcollections/Cargo.toml b/src/libcollections/Cargo.toml
new file mode 100644
index 0000000..7ccfae4
index 0000000..a421ffc
--- /dev/null
+++ b/src/libcollections/Cargo.toml
@@ -0,0 +1,17 @@
@@ -0,0 +1,20 @@
+[package]
+authors = ["The Rust Project Developers"]
+name = "collections"
Expand All @@ -38,6 +230,9 @@ index 0000000..7ccfae4
+name = "collections"
+path = "lib.rs"
+
+[features]
+jemalloc = ["alloc/jemalloc"]
+
+[dependencies.alloc]
+path = "../liballoc"
+
Expand Down Expand Up @@ -127,10 +322,10 @@ index 0000000..52dfb2e
+path = "../libcore"
diff --git a/src/libstd/Cargo.toml b/src/libstd/Cargo.toml
new file mode 100644
index 0000000..b0a1e20
index 0000000..0c25516
--- /dev/null
+++ b/src/libstd/Cargo.toml
@@ -0,0 +1,37 @@
@@ -0,0 +1,38 @@
+[package]
+authors = ["The Rust Project Developers"]
+build = "build.rs"
Expand All @@ -143,6 +338,7 @@ index 0000000..b0a1e20
+
+[features]
+backtrace = []
+jemalloc = ["alloc/jemalloc", "collections/jemalloc"]
+#default = ["backtrace"]
+
+[build_dependencies]
Expand Down Expand Up @@ -170,10 +366,10 @@ index 0000000..b0a1e20
+path = "../librustc_unicode"
diff --git a/src/libstd/build.rs b/src/libstd/build.rs
new file mode 100644
index 0000000..1660ece
index 0000000..13400eb
--- /dev/null
+++ b/src/libstd/build.rs
@@ -0,0 +1,307 @@
@@ -0,0 +1,308 @@
+#![allow(dead_code)]
+
+extern crate gcc;
Expand Down Expand Up @@ -264,6 +460,7 @@ index 0000000..1660ece
+ let src = PathBuf::from(try!(env::var("CARGO_MANIFEST_DIR").map_err(|_| {
+ "cargo bug: $CARGO_MANIFEST_DIR not set"
+ })));
+
+ let dst = PathBuf::from(try!(env::var("OUT_DIR").map_err(|_| {
+ "cargo bug: $OUT_DIR not set"
+ })));
Expand Down
11 changes: 11 additions & 0 deletions fetch-and-patch-rust.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,19 @@ unzip ${HASH}.zip
mv compiler-rt-${HASH} compiler-rt
rm ${HASH}.zip

# fetch jemalloc
# XXX as above this is not 100% correct, but the module hasn't been updated in
# months
HASH=e24a1a025a1f214e40eedafe3b9c7b1d69937922
rmdir jemalloc
wget "https://github.com/rust-lang/jemalloc/archive/${HASH}.zip"
unzip ${HASH}.zip
mv jemalloc-${HASH} jemalloc
rm ${HASH}.zip

# patch
cd ..
curl -s https://raw.githubusercontent.com/japaric/std-with-cargo/master/cargo-ify.patch | patch -p1
curl -s https://raw.githubusercontent.com/japaric/std-with-cargo/master/optional-backtrace.patch | patch -p1
curl -s https://raw.githubusercontent.com/japaric/std-with-cargo/master/optional-jemalloc.patch | patch -p1
curl -s https://raw.githubusercontent.com/japaric/std-with-cargo/master/remove-mno-compact-eh-flag.patch | patch -p1
31 changes: 31 additions & 0 deletions optional-jemalloc.patch
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
diff --git a/src/liballoc/heap.rs b/src/liballoc/heap.rs
index e155dc8..5ce5a9c 100644
--- a/src/liballoc/heap.rs
+++ b/src/liballoc/heap.rs
@@ -197,7 +197,7 @@ mod imp {

#[cfg(all(not(feature = "external_funcs"),
not(feature = "external_crate"),
- jemalloc))]
+ feature = "jemalloc"))]
mod imp {
use core::option::Option;
use core::option::Option::None;
@@ -278,7 +278,7 @@ mod imp {

#[cfg(all(not(feature = "external_funcs"),
not(feature = "external_crate"),
- not(jemalloc),
+ not(feature = "jemalloc"),
unix))]
mod imp {
use core::cmp;
@@ -342,7 +342,7 @@ mod imp {

#[cfg(all(not(feature = "external_funcs"),
not(feature = "external_crate"),
- not(jemalloc),
+ not(feature = "jemalloc"),
windows))]
mod imp {
use libc::{c_void, size_t};

0 comments on commit 1b7bfd8

Please sign in to comment.