Fix loading of renderdoc library (#140)

* Fix loading of RenderDoc library

It turns out this crate has been loading the RenderDoc API incorrectly:

> To do this you'll use your platforms dynamic library functions to see
> if the library is open already - e.g. `GetModuleHandle` on Windows, or
> or `dlopen` with the `RTLD_NOW | RTLD_NOLOAD` flags if available on
> *nix systems. On most platforms you can just search for the module
> name - `renderdoc.dll` on Windows, or `librenderdoc.so` on Linux, or
> `libVkLayer_GLES_RenderDoc.so` on Android should be sufficient here,
> so you don’t need to know the path to where RenderDoc is running from.
> This will vary by platform however so consult your platform’s OS
> documentation. Then you can use `GetProcAddress` or `dlsym` to fetch
> the `RENDERDOC_GetAPI` function using the typedef above.

This was reported long ago as an issue, and I'm addressing this now with
this commit.

It does change the runtime behavior of the library, though: the
application no longer attempts to load RenderDoc into memory if not
injected from the outside, so `RenderDoc::new()` will now return `Err`
when previously it would have succeeded.

* Restore integration and doctest support in CI

This commit adds an unstable `ci` feature which reverts back the
previous non-compliant opening method because it enables tests to run in
CI without needing to painstakingly extract the test binary names from
`cargo build --tests --message-format json` and spawn each one in
`renderdoccmd capture $NAME`. Besides, this strategy is impossible
anyway because doctests compiled on-the-fly by `rustdoc` and do not have
external binaries with which to launch with RenderDoc. We also cannot
make the `RTLD_NOLOAD` conditional, i.e. `#[cfg(any(test, doctest))]`
and `#[cfg(not(any(test, doctest)))]`, due to:

rust-lang/rust#67295

As such, this internal crate feature will have to do.

Cargo.toml

@@ -19,6 +19,12 @@ all-features = true
 [badges]
 circle-ci = { repository = "ebkalderon/renderdoc-rs" }
 
+[features]
+default = []
+
+# Private feature only intended for doctests in CI
+ci = []
+
 [dependencies]
 bitflags = "1.0"
 float-cmp = "0.9"

examples/triangle.rs

@@ -38,7 +38,7 @@ async fn run(event_loop: EventLoop<()>, window: Window) {
         .await
         .expect("Failed to create device");
 
-    let mut rd: RenderDoc<V110> = RenderDoc::new().unwrap();
+    let mut rd: RenderDoc<V110> = RenderDoc::new().expect("RenderDoc is not running");
     rd.set_focus_toggle_keys(&[InputButton::F]);
     rd.set_capture_keys(&[InputButton::C]);
 

src/version.rs

@@ -1,31 +1,14 @@
 //! Entry points for the RenderDoc API.
 
-use std::os::raw::c_void;
-use std::path::Path;
+use std::ffi::c_void;
+use std::ptr;
 
 use libloading::{Library, Symbol};
 use once_cell::sync::OnceCell;
 use renderdoc_sys::RENDERDOC_API_1_4_1;
 
 use crate::error::Error;
 
-static RD_LIB: OnceCell<Library> = OnceCell::new();
-
-#[cfg(windows)]
-fn get_path() -> &'static Path {
-    Path::new("renderdoc.dll")
-}
-
-#[cfg(all(unix, not(target_os = "android")))]
-fn get_path() -> &'static Path {
-    Path::new("librenderdoc.so")
-}
-
-#[cfg(target_os = "android")]
-fn get_path() -> &'static Path {
-    Path::new("libVkLayer_GLES_RenderDoc.so")
-}
-
 /// Entry point for the RenderDoc API.
 pub type Entry = RENDERDOC_API_1_4_1;
 
@@ -55,13 +38,6 @@ pub enum VersionCode {
     V141 = 10401,
 }
 
-/// Initializes a new instance of the RenderDoc API.
-///
-/// # Safety
-///
-/// This function is not thread-safe and should not be called on multiple threads at once.
-type GetApiFn = unsafe extern "C" fn(ver: VersionCode, out: *mut *mut c_void) -> i32;
-
 /// Entry point into the RenderDoc API.
 pub trait Version {
     /// Minimum compatible version number.
@@ -73,18 +49,48 @@ pub trait Version {
     ///
     /// This function is not thread-safe and should not be called on multiple threads at once.
     fn load() -> Result<*mut Entry, Error> {
-        use std::ptr;
+        static LIBRARY: OnceCell<Library> = OnceCell::new();
+
+        type GetApiFn = unsafe extern "C" fn(ver: u32, out: *mut *mut c_void) -> i32;
+
+        #[cfg(windows)]
+        let lib_path = "renderdoc.dll";
+        #[cfg(all(unix, not(target_os = "android")))]
+        let lib_path = "librenderdoc.so";
+        #[cfg(target_os = "android")]
+        let lib_path = "libVkLayer_GLES_RenderDoc.so";
 
         unsafe {
-            let lib = RD_LIB
-                .get_or_try_init(|| Library::new(get_path()))
+            #[cfg(not(feature = "ci"))]
+            #[cfg(unix)]
+            let lib = LIBRARY
+                .get_or_try_init(|| {
+                    // TODO: Use constant from `libloading`, once added upstream.
+                    const RTLD_NOLOAD: i32 = 0x4;
+
+                    let flags = libloading::os::unix::RTLD_NOW | RTLD_NOLOAD;
+                    libloading::os::unix::Library::open(Some(lib_path), flags).map(Into::into)
+                })
+                .map_err(Error::library)?;
+
+            #[cfg(not(feature = "ci"))]
+            #[cfg(windows)]
+            let lib = LIBRARY
+                .get_or_try_init(|| {
+                    libloading::os::windows::Library::open_already_loaded(lib_path).map(Into::into)
+                })
+                .map_err(Error::library)?;
+
+            #[cfg(feature = "ci")]
+            let lib = LIBRARY
+                .get_or_try_init(|| Library::new(lib_path))
                 .map_err(Error::library)?;
 
             let get_api: Symbol<GetApiFn> =
                 lib.get(b"RENDERDOC_GetAPI\0").map_err(Error::symbol)?;
 
             let mut obj = ptr::null_mut();
-            match get_api(Self::VERSION, &mut obj) {
+            match get_api(Self::VERSION as u32, &mut obj) {
                 1 => Ok(obj as *mut Entry),
                 _ => Err(Error::no_compatible_api()),
             }