From f791f34fc4837a0974048e92f5f08da4a053aa81 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Fri, 31 May 2024 13:49:39 +0200 Subject: [PATCH 1/3] rust: add static_call support Add static_call support by mirroring how C does. When the platform does not support static calls (right now, that means that it is not x86), then the function pointer is loaded from a global and called. Otherwise, we generate a call to a trampoline function, and objtool is used to make these calls patchable at runtime. Signed-off-by: Alice Ryhl --- rust/kernel/lib.rs | 1 + rust/kernel/static_call.rs | 90 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 rust/kernel/static_call.rs diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index fbd91a48ff8bc5..d534b117895564 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -38,6 +38,7 @@ pub mod net; pub mod prelude; pub mod print; mod static_assert; +pub mod static_call; #[doc(hidden)] pub mod std_vendor; pub mod str; diff --git a/rust/kernel/static_call.rs b/rust/kernel/static_call.rs new file mode 100644 index 00000000000000..154a25cf111630 --- /dev/null +++ b/rust/kernel/static_call.rs @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Logic for static calls. + +#[macro_export] +#[doc(hidden)] +macro_rules! ty_underscore_for { + ($arg:expr) => { + _ + }; +} + +#[doc(hidden)] +#[repr(transparent)] +pub struct AddressableStaticCallKey { + _ptr: *const bindings::static_call_key, +} +unsafe impl Sync for AddressableStaticCallKey {} +impl AddressableStaticCallKey { + pub const fn new(ptr: *const bindings::static_call_key) -> Self { + Self { _ptr: ptr } + } +} + +#[cfg(CONFIG_HAVE_STATIC_CALL)] +#[doc(hidden)] +#[macro_export] +macro_rules! _static_call { + ($name:ident($($args:expr),* $(,)?)) => {{ + // Symbol mangling will give this symbol a unique name. + #[cfg(CONFIG_HAVE_STATIC_CALL_INLINE)] + #[link_section = ".discard.addressable"] + #[used] + static __ADDRESSABLE: $crate::static_call::AddressableStaticCallKey = unsafe { + $crate::static_call::AddressableStaticCallKey::new(::core::ptr::addr_of!( + $crate::macros::paste! { $crate::bindings:: [<__SCK__ $name >]; } + )) + }; + + let fn_ptr: unsafe extern "C" fn($($crate::static_call::ty_underscore_for!($args)),*) -> _ = + $crate::macros::paste! { $crate::bindings:: [<__SCT__ $name >]; }; + (fn_ptr)($($args),*) + }}; +} + +#[cfg(not(CONFIG_HAVE_STATIC_CALL))] +#[doc(hidden)] +#[macro_export] +macro_rules! _static_call { + ($name:ident($($args:expr),* $(,)?)) => {{ + let void_ptr_fn: *mut ::core::ffi::c_void = $crate::macros::paste! { $crate::bindings:: [<__SCK__ $name >]; }.func; + + let fn_ptr: unsafe extern "C" fn($($crate::static_call::ty_underscore_for!($args)),*) -> _ = if true { + ::core::mem::transmute(void_ptr_fn) + } else { + // This is dead code, but it influences type inference on `fn_ptr` so that we transmute + // the function pointer to the right type. + $crate::macros::paste! { $crate::bindings:: [<__SCT__ $name >]; } + }; + + (fn_ptr)($($args),*) + }}; +} + +/// Statically call a global function. +/// +/// # Safety +/// +/// This macro will call the provided function. It is up to the caller to uphold the safety +/// guarantees of the function. +/// +/// # Examples +/// +/// ```ignore +/// fn call_static() { +/// unsafe { +/// static_call! { your_static_call() }; +/// } +/// } +/// ``` +#[macro_export] +macro_rules! static_call { + // Forward to the real implementation. Separated like this so that we don't have to duplicate + // the documentation. + ($($args:tt)*) => { $crate::static_call::_static_call! { $($args)* } }; +} + +pub use {_static_call, static_call, ty_underscore_for}; From 38720c0449960c6dd027b1a5084db31fac34e55f Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Fri, 31 May 2024 16:52:26 +0200 Subject: [PATCH 2/3] rust: add static_key_false Add just enough support for static key so that we can use it from tracepoints. Tracepoints rely on `static_key_false` even though it is deprecated, so we add the same functionality to Rust. Signed-off-by: Alice Ryhl --- rust/kernel/lib.rs | 1 + rust/kernel/static_key.rs | 150 ++++++++++++++++++++++++++++++++++++++ scripts/Makefile.build | 2 +- 3 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 rust/kernel/static_key.rs diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index d534b117895564..22e1fedd0774b3 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -39,6 +39,7 @@ pub mod prelude; pub mod print; mod static_assert; pub mod static_call; +pub mod static_key; #[doc(hidden)] pub mod std_vendor; pub mod str; diff --git a/rust/kernel/static_key.rs b/rust/kernel/static_key.rs new file mode 100644 index 00000000000000..511aa90e80128d --- /dev/null +++ b/rust/kernel/static_key.rs @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Logic for static keys. + +use crate::bindings::*; + +#[doc(hidden)] +#[macro_export] +#[cfg(target_arch = "x86_64")] +#[cfg(not(CONFIG_HAVE_RUST_ASM_GOTO))] +macro_rules! _static_key_false { + ($key:path, $keytyp:ty, $field:ident) => {{ + let mut output = 1u32; + + core::arch::asm!( + r#" + 1: .byte 0x0f,0x1f,0x44,0x00,0x00 + + .pushsection __jump_table, "aw" + .balign 8 + .long 1b - . + .long 3f - . + .quad {0} + {1} - . + .popsection + + 2: mov {2:e}, 0 + 3: + "#, + sym $key, + const ::core::mem::offset_of!($keytyp, $field), + inout(reg) output, + ); + + output != 0 + }}; +} + +#[doc(hidden)] +#[macro_export] +#[cfg(target_arch = "x86_64")] +#[cfg(CONFIG_HAVE_RUST_ASM_GOTO)] +macro_rules! _static_key_false { + ($key:path, $keytyp:ty, $field:ident) => {'my_label: { + core::arch::asm!( + r#" + 1: .byte 0x0f,0x1f,0x44,0x00,0x00 + + .pushsection __jump_table, "aw" + .balign 8 + .long 1b - . + .long {0} - . + .quad {1} + {2} - . + .popsection + "#, + label { + break 'my_label true; + }, + sym $key, + const ::core::mem::offset_of!($keytyp, $field), + ); + + break 'my_label false; + }}; +} + +#[doc(hidden)] +#[macro_export] +#[cfg(target_arch = "aarch64")] +#[cfg(not(CONFIG_HAVE_RUST_ASM_GOTO))] +macro_rules! _static_key_false { + ($key:path, $keytyp:ty, $field:ident) => {{ + let mut output = 1u32; + + core::arch::asm!( + r#" + 1: nop + + .pushsection __jump_table, "aw" + .align 3 + .long 1b - ., 3f - . + .quad {0} + {1} - . + .popsection + + 2: mov {2:w}, 0 + 3: + "#, + sym $key, + const ::core::mem::offset_of!($keytyp, $field), + inout(reg) output + ); + + output != 0 + }}; +} + +#[doc(hidden)] +#[macro_export] +#[cfg(target_arch = "aarch64")] +#[cfg(CONFIG_HAVE_RUST_ASM_GOTO)] +macro_rules! _static_key_false { + ($key:path, $keytyp:ty, $field:ident) => {'my_label: { + core::arch::asm!( + r#" + 1: nop + + .pushsection __jump_table, "aw" + .align 3 + .long 1b - ., {0} - . + .quad {1} + {2} - . + .popsection + "#, + label { + break 'my_label true; + }, + sym $key, + const ::core::mem::offset_of!($keytyp, $field), + ); + + break 'my_label false; + }}; +} + +/// Branch based on a static key. +/// +/// Takes three arguments: +/// +/// * `key` - the path to the static variable containing the `static_key`. +/// * `keytyp` - the type of `key`. +/// * `field` - the name of the field of `key` that contains the `static_key`. +#[macro_export] +macro_rules! static_key_false { + // Forward to the real implementation. Separated like this so that we don't have to duplicate + // the documentation. + ($key:path, $keytyp:ty, $field:ident) => {{ + // Assert that `$key` has type `$keytyp` and that `$key.$field` has type `static_key`. + // + // SAFETY: We know that `$key` is a static because otherwise the inline assembly will not + // compile. The raw pointers created in this block are in-bounds of `$key`. + static _TY_ASSERT: () = unsafe { + let key: *const $keytyp = ::core::ptr::addr_of!($key); + let _: *const $crate::bindings::static_key = ::core::ptr::addr_of!((*key).$field); + }; + + $crate::static_key::_static_key_false! { $key, $keytyp, $field } + }}; +} + +pub use {_static_key_false, static_key_false}; diff --git a/scripts/Makefile.build b/scripts/Makefile.build index efacca63c89767..60197c1c063f09 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -263,7 +263,7 @@ $(obj)/%.lst: $(obj)/%.c FORCE # Compile Rust sources (.rs) # --------------------------------------------------------------------------- -rust_allowed_features := new_uninit +rust_allowed_features := asm_const,asm_goto,new_uninit # `--out-dir` is required to avoid temporaries being created by `rustc` in the # current working directory, which may be not accessible in the out-of-tree From 9d0346a9a212d1ad8929b690305b959da5395053 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Fri, 31 May 2024 16:52:37 +0200 Subject: [PATCH 3/3] rust: add tracepoint support Make it possible to have Rust code call into tracepoints defined by C code. It is still required that the tracepoint is declared in a C header, and that this header is included in the input to bindgen. Signed-off-by: Alice Ryhl --- rust/bindings/bindings_helper.h | 1 + rust/bindings/lib.rs | 15 ++++++ rust/helpers.c | 18 +++++++ rust/kernel/lib.rs | 1 + rust/kernel/tracepoint.rs | 94 +++++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+) create mode 100644 rust/kernel/tracepoint.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index ddb5644d4fd900..d442f9ccfc2c43 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs index 40ddaee50d8bd8..48856761d6822e 100644 --- a/rust/bindings/lib.rs +++ b/rust/bindings/lib.rs @@ -48,3 +48,18 @@ mod bindings_helper { } pub use bindings_raw::*; + +/// Rust version of the C macro `rcu_dereference_raw`. +/// +/// The rust helper only works with void pointers, but this wrapper method makes it work with any +/// pointer type using pointer casts. +/// +/// # Safety +/// +/// This method has the same safety requirements as the C macro of the same name. +#[inline(always)] +pub unsafe fn rcu_dereference_raw(p: *const *mut T) -> *mut T { + // SAFETY: This helper calls into the C macro, so the caller promises to uphold the safety + // requirements. + unsafe { __rcu_dereference_raw(p as *mut *mut _) as *mut T } +} diff --git a/rust/helpers.c b/rust/helpers.c index 2c37a0f5d7a84f..3b0877eb7f366c 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -165,6 +165,24 @@ rust_helper_krealloc(const void *objp, size_t new_size, gfp_t flags) } EXPORT_SYMBOL_GPL(rust_helper_krealloc); +void rust_helper_preempt_enable_notrace(void) +{ + preempt_enable_notrace(); +} +EXPORT_SYMBOL_GPL(rust_helper_preempt_enable_notrace); + +void rust_helper_preempt_disable_notrace(void) +{ + preempt_disable_notrace(); +} +EXPORT_SYMBOL_GPL(rust_helper_preempt_disable_notrace); + +void *rust_helper___rcu_dereference_raw(void **p) +{ + return rcu_dereference_raw(p); +} +EXPORT_SYMBOL_GPL(rust_helper___rcu_dereference_raw); + /* * `bindgen` binds the C `size_t` type as the Rust `usize` type, so we can * use it in contexts where Rust expects a `usize` like slice (array) indices. diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 22e1fedd0774b3..3f3b280bb437a6 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -46,6 +46,7 @@ pub mod str; pub mod sync; pub mod task; pub mod time; +pub mod tracepoint; pub mod types; pub mod workqueue; diff --git a/rust/kernel/tracepoint.rs b/rust/kernel/tracepoint.rs new file mode 100644 index 00000000000000..6ce8bd43d2418d --- /dev/null +++ b/rust/kernel/tracepoint.rs @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Logic for tracepoints. + +/// Declare the Rust entry point for a tracepoint. +#[macro_export] +macro_rules! declare_trace { + ($($(#[$attr:meta])* $pub:vis fn $name:ident($($argname:ident : $argtyp:ty),* $(,)?);)*) => {$( + $( #[$attr] )* + #[inline(always)] + $pub unsafe fn $name($($argname : $argtyp),*) { + #[cfg(CONFIG_TRACEPOINTS)] + { + use $crate::bindings::*; + + // SAFETY: This macro only compiles if $name is a real tracepoint, and if it is a + // real tracepoint, then it is okay to query the static key. + let should_trace = unsafe { + $crate::macros::paste! { + $crate::static_key::static_key_false!( + [< __tracepoint_ $name >], + $crate::bindings::tracepoint, + key + ) + } + }; + + if should_trace { + // TODO: cpu_online(raw_smp_processor_id()) + let cond = true; + $crate::tracepoint::do_trace!($name($($argname : $argtyp),*), cond); + } + } + + #[cfg(not(CONFIG_TRACEPOINTS))] + { + // If tracepoints are disabled, insert a trivial use of each argument + // to avoid unused argument warnings. + $( let _unused = $argname; )* + } + } + )*} +} + +#[doc(hidden)] +#[macro_export] +macro_rules! do_trace { + ($name:ident($($argname:ident : $argtyp:ty),* $(,)?), $cond:expr) => {{ + if !$cond { + return; + } + + // SAFETY: This call is balanced with the call below. + unsafe { $crate::bindings::preempt_disable_notrace() }; + + // SAFETY: This calls the tracepoint with the provided arguments. The caller of the Rust + // wrapper guarantees that this is okay. + #[cfg(CONFIG_HAVE_STATIC_CALL)] + unsafe { + let it_func_ptr: *mut $crate::bindings::tracepoint_func = + $crate::bindings::rcu_dereference_raw( + ::core::ptr::addr_of!( + $crate::macros::concat_idents!(__tracepoint_, $name).funcs + ) + ); + + if !it_func_ptr.is_null() { + let __data = (*it_func_ptr).data; + $crate::macros::paste! { + $crate::static_call::static_call! { + [< tp_func_ $name >] (__data, $($argname),*) + }; + } + } + } + + // SAFETY: This calls the tracepoint with the provided arguments. The caller of the Rust + // wrapper guarantees that this is okay. + #[cfg(not(CONFIG_HAVE_STATIC_CALL))] + unsafe { + $crate::macros::concat_idents!(__traceiter_, $name)( + ::core::ptr::null_mut(), + $($argname),* + ); + } + + // SAFETY: This call is balanced with the call above. + unsafe { $crate::bindings::preempt_enable_notrace() }; + }} +} + +pub use {declare_trace, do_trace};