From fcb913efe0914e3561538b6911eecb95d93e42d1 Mon Sep 17 00:00:00 2001 From: Huang-Huang Bao Date: Tue, 19 Nov 2024 02:03:13 +0800 Subject: [PATCH] doc: update for aya and drop the libbpf-skel support Also update nix scripts. --- Cargo.toml | 6 +++--- README.md | 32 +++++++++++++++++++++++++------- build.rs | 33 +++++++++++++++++++++++---------- docs/guide/cross.md | 25 +++++++++++++++++++------ docs/guide/openwrt.md | 1 - flake.nix | 9 ++++++--- nix/cross-package.nix | 25 +++++++++---------------- nix/package.nix | 4 ---- 8 files changed, 85 insertions(+), 50 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3a83a6e..fafb990 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,13 +17,13 @@ opt-level = "z" strip = true [features] -default = ["aya"] +default = ["aya", "pkg-config"] # Enable IPv6 NAPT ipv6 = [] # Enable Aya BPF loader, requires Rust>=1.80.0 -aya = ["dep:aya", "dep:pkg-config"] +aya = ["dep:aya"] # Enable libbpf BPF loader -libbpf = ["dep:libbpf-rs", "dep:libbpf-sys", "dep:pkg-config"] +libbpf = ["dep:libbpf-rs", "dep:libbpf-sys"] # Enable libbpf skeleton-wrapped BPF loader, requires unpublish self_cell for now libbpf-skel = [ "dep:libbpf-rs", diff --git a/README.md b/README.md index 89beeed..7de7a7f 100644 --- a/README.md +++ b/README.md @@ -20,20 +20,24 @@ For implementation details, see documentations under [reference](./docs/referenc ## Requirement - Linux kernel >= 5.15 (compiled with BPF and BTF support) on target machine -- `clang` compiling BPF C code -- `cargo` and `rustfmt` for building +- Rust toolchain (`cargo` etc.) +- `clang` for compiling BPF C code +- `libbpf` headers +- (optional) `pkg-config` to locate `libbpf` headers -**aya** (default) +`"aya"` loader (default) -- `libbpf` headers -- `llvm-strip` +- `llvm-strip` for stripping compiled BPF object -**libbpf** +`"libbpf"` loader +- `rustfmt` for formatting generated code - `clang` libs for bindgen - `libelf` from elfutils - `zlib` +The `"libbpf-skel"` loader is served as reference purpose and you should just use `aya` or `libbpf` instead. + It's also required the eBPF JIT implementation for target architecture in kernel has implemented support for BPF-to-BPF calls, which is not the case for MIPS and other architectures have less interests. This application is only tested to work on x86-64 or aarch64. See also [OpenWrt guide](./docs/guide/openwrt.md) for pitfalls running this on OpenWrt. @@ -42,6 +46,7 @@ See also [OpenWrt guide](./docs/guide/openwrt.md) for pitfalls running this on O ```shell cargo install --git https://github.com/EHfive/einat-ebpf.git +cargo install --git https://github.com/EHfive/einat-ebpf.git ``` You can also enable IPv6 NAT66 feature with `--features ipv6` flag, however it would increase load time of eBPF programs to about 4 times. @@ -59,7 +64,20 @@ For NixOS, you can use module [`github:EHfive/einat-ebpf#nixosModules.default`]( For OpenWrt, there are [openwrt-einat-ebpf](https://github.com/muink/openwrt-einat-ebpf) and [luci-app-einat](https://github.com/muink/luci-app-einat) by @muink. -See also [cross-compilation guide](./docs/guide/cross.md) for cross-compilation on Debian/Debian-based distros. +See also [cross-compilation guide](./docs/guide/cross-aya.md) for cross-compilation on Debian/Debian-based distros. + +### Build Environment Variables + +| Name | Example Value | Note | +| ---------------------- | -------------------------- | ----------------------------------------------- | +| `EINAT_BPF_CFLAGS` | `-I/usr/include/` | Specify extra CFLAGS for BPF object compilation | +| `LIBBPF_NO_PKG_CONFIG` | `1` | Disable [pkg_config lookup] of libbpf. | + +[pkg_config lookup]: (https://docs.rs/pkg-config/0.3.31/pkg_config/index.html#environment-variables) + +You can combine `LIBBPF_NO_PKG_CONFIG` and `EINAT_BPF_CFLAGS` to specify include flag of libbpf headers manually. + +See also [build.rs](./build.rs) for reference. ## Usage diff --git a/build.rs b/build.rs index afa8edf..00b999d 100644 --- a/build.rs +++ b/build.rs @@ -40,21 +40,35 @@ fn einat_obj_build() { // compile BPF C code let mut cmd = Command::new("clang"); + cmd.args(c_args()); + + if let Some(cflags) = option_env!("EINAT_BPF_CFLAGS") { + cmd.arg(cflags); + } + + // Specify environment variable LIBBPF_NO_PKG_CONFIG=1 to disable pkg-config lookup. + // Or just disable the "pkg-config" feature. + #[cfg(feature = "pkg-config")] + match pkg_config::probe_library("libbpf") { + Ok(libbpf) => { + let includes = libbpf + .include_paths + .into_iter() + .map(|i| format!("-I{}", i.to_string_lossy())); + cmd.args(includes); + } + Err(e) => { + eprintln!("Can not locate libbpf with pkg-config: {}", e) + } + } + let target = if env::var("CARGO_CFG_TARGET_ENDIAN").unwrap() == "little" { "bpfel" } else { "bpfeb" }; - let libbpf = pkg_config::probe_library("libbpf").unwrap(); - - let includes = libbpf - .include_paths - .into_iter() - .map(|i| format!("-I{}", i.to_string_lossy())); - - cmd.args(c_args()) - .arg("-target") + cmd.arg("-target") .arg(target) .arg("-g") .arg("-O2") @@ -62,7 +76,6 @@ fn einat_obj_build() { .arg(SRC) .arg("-o") .arg(bpf_obj) - .args(includes) .status() .expect("compile BPF object failed"); diff --git a/docs/guide/cross.md b/docs/guide/cross.md index 18b902b..e5ea7ae 100644 --- a/docs/guide/cross.md +++ b/docs/guide/cross.md @@ -2,12 +2,15 @@ This guide gives example for cross-compiling for aarch64, replace "aarch64" and "arm64" with respective architecture identifier tokens for cross-compiling for other architectures. -## Build Dependencies +The "aya" loader is used by default. +To enable only the "libbpf" loader, specify Cargo flags `--no-default-features --features aya,pkg-config`. -Install `libelf` and `zlib1g` as it's required by `libbpf`. Install `gcc-aarch64-linux-gnu` for cross linking. Install `clang` for bindgen and compile eBPF C code in this project. +### Build Dependencies + +Install `gcc-aarch64-linux-gnu` for cross linking. Install `clang` for bindgen and compile eBPF C code in this project. ``` -apt install libelf-dev zlib1g-dev gcc-aarch64-linux-gnu clang +apt install gcc-aarch64-linux-gnu clang ``` Install `rustup` to get Rust>=1.74, see https://www.rust-lang.org/tools/install. Also make sure `rustfmt` is installed as it's used by `libbpf-cargo`. @@ -18,15 +21,20 @@ Add required target to Rust toolchain: rustup target add aarch64-unknown-linux-gnu ``` -## Target Dependencies +### Target Dependencies + +For "libbpf" loader, you would also need to install `libelf` and `zlib1g` as it's required by `libbpf`. ``` dpkg --add-architecture arm64 apt update +# We only need libbpf headers +apt install libbpf-dev:arm64 +# For "libbpf" loader only apt install libelf-dev:arm64 zlib1g-dev:arm64 ``` -## Environment Variables +### Environment Variables ``` export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER="/usr/bin/aarch64-linux-gnu-gcc" @@ -38,7 +46,12 @@ export CC_aarch64_unknown_linux_gnu="/usr/bin/aarch64-linux-gnu-gcc" export CFLAGS_aarch64_unknown_linux_gnu="-I /usr/include/aarch64-linux-gnu -L /usr/lib/aarch64-linux-gnu" ``` -## Build static binary +Specify `EINAT_BPF_CFLAGS` if einat build script failed to locate libbpf headers. +``` +export EINAT_BPF_CFLAGS="-I /usr/include/aarch64-linux-gnu" +``` + +### Build static binary ``` cd einat-ebpf diff --git a/docs/guide/openwrt.md b/docs/guide/openwrt.md index 8275a39..c09e9bc 100644 --- a/docs/guide/openwrt.md +++ b/docs/guide/openwrt.md @@ -34,7 +34,6 @@ Additional kernel configs required, you might need to add these to kernel config ``` CONFIG_BPF_SYSCALL=y CONFIG_BPF_JIT=y -CONFIG_NET_CLS_BPF=y CONFIG_NET_ACT_BPF=y ``` diff --git a/flake.nix b/flake.nix index b18da76..f421894 100644 --- a/flake.nix +++ b/flake.nix @@ -34,9 +34,7 @@ crossPackage' = import ./nix/cross-package.nix { inherit naersk; }; overlay = final: prev: { - einat = defaultPackage' { - pkgs = (import nixpkgs) { inherit (prev) system; }; - }; + einat = defaultPackage' prev; }; module = { @@ -60,6 +58,11 @@ crossPackage = { ... }@args: crossPackage' ({ inherit pkgs; } // args); in { + legacyPackages = (import nixpkgs) { + inherit system; + overlays = [ overlay ]; + }; + packages = { default = defaultPackage; ipv6 = defaultPackage; diff --git a/nix/cross-package.nix b/nix/cross-package.nix index 3b3245e..3119093 100644 --- a/nix/cross-package.nix +++ b/nix/cross-package.nix @@ -44,34 +44,23 @@ let #required by libelf zstd ] - ) ++ [ - pkgs.libbpf - ]; + ); buildInputsSearchFlags = map (dep: "-L${lib.getLib dep}/lib") buildInputs; in naersk'.buildPackage { src = ../.; gitSubmodules = true; - depsBuildBuild = with pkgs; [ - # build dependencies of cargo build depenceies libbpf-cargo -> libbpf-sys - stdenv.cc - ]; nativeBuildInputs = with pkgs; [ pkg-config - # required by `libbpf_cargo::SkeletonBuilder` - rustfmt - ## build dependencies of cargo build depenceies libbpf-cargo -> libbpf-sys - stdenv.cc.libc - zlib - elfutils - ## build dependencies of libbpf-sys on target platform - # for cross linking libelf and zlib, and make libbpf - crossPkgs.stdenv.cc # compile BPF C code llvmPackages.clang-unwrapped llvmPackages.bintools-unwrapped + + ## build dependencies of libbpf-sys on target platform + # for cross linking libelf and zlib, and make libbpf + crossPkgs.stdenv.cc ]; inherit buildInputs; strictDeps = true; @@ -99,6 +88,9 @@ naersk'.buildPackage { "-mno-outline-atomics" ]; + LIBBPF_NO_PKG_CONFIG = 1; + EINAT_BPF_CFLAGS = "-I${pkgs.libbpf}/include"; + "CC_${targetUnderscore}" = crossCC; "CARGO_TARGET_${targetUnderscoreUpper}_LINKER" = crossCC; @@ -115,6 +107,7 @@ naersk'.buildPackage { preBuild = '' export BINDGEN_EXTRA_CLANG_ARGS_${targetUnderscore}="''${NIX_CFLAGS_COMPILE}"; + # Avoid adding host dependencies to CFLAGS and LDFLAGS for build platform if [[ ${pkgs.stdenv.cc.suffixSalt} != ${crossPkgs.stdenv.cc.suffixSalt} ]]; then export NIX_CC_WRAPPER_TARGET_HOST_${pkgs.stdenv.cc.suffixSalt}=""; diff --git a/nix/package.nix b/nix/package.nix index 829a55e..d6dcca7 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -3,7 +3,6 @@ lib, rustPlatform, pkg-config, - rustfmt, llvmPackages, libbpf, elfutils, @@ -15,12 +14,9 @@ naersk.buildPackage { nativeBuildInputs = [ pkg-config - rustfmt llvmPackages.clang-unwrapped llvmPackages.bintools-unwrapped rustPlatform.bindgenHook - elfutils - zlib ]; buildInputs = [