Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

asset: WasmAssetIo #703

Merged
merged 5 commits into from
Oct 20, 2020
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions crates/bevy_asset/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,11 @@ log = { version = "0.4", features = ["release_max_level_info"] }
notify = { version = "5.0.0-pre.2", optional = true }
parking_lot = "0.11.0"
rand = "0.7.3"
async-trait = "0.1.40"
futures-lite = "1.4.0"

[target.'cfg(target_arch = "wasm32")'.dependencies]
wasm-bindgen = { version = "0.2" }
web-sys = { version = "0.3", features = ["Request", "Window", "Response"]}
wasm-bindgen-futures = "0.4"
js-sys = "0.3"
29 changes: 15 additions & 14 deletions crates/bevy_asset/src/asset_server.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
use crate::{
path::{AssetPath, AssetPathId, SourcePathId},
Asset, AssetIo, AssetIoError, AssetLifecycle, AssetLifecycleChannel, AssetLifecycleEvent,
AssetLoader, Assets, FileAssetIo, Handle, HandleId, HandleUntyped, LabelId, LoadContext,
LoadState, RefChange, RefChangeChannel, SourceInfo, SourceMeta,
AssetLoader, Assets, Handle, HandleId, HandleUntyped, LabelId, LoadContext, LoadState,
RefChange, RefChangeChannel, SourceInfo, SourceMeta,
};
use anyhow::Result;
use bevy_ecs::Res;
Expand Down Expand Up @@ -35,8 +35,8 @@ pub(crate) struct AssetRefCounter {
pub(crate) ref_counts: Arc<RwLock<HashMap<HandleId, usize>>>,
}

pub struct AssetServerInternal<TAssetIo: AssetIo = FileAssetIo> {
pub(crate) asset_io: TAssetIo,
pub struct AssetServerInternal {
pub(crate) asset_io: Box<dyn AssetIo>,
pub(crate) asset_ref_counter: AssetRefCounter,
pub(crate) asset_sources: Arc<RwLock<HashMap<SourcePathId, SourceInfo>>>,
pub(crate) asset_lifecycles: Arc<RwLock<HashMap<Uuid, Box<dyn AssetLifecycle>>>>,
Expand All @@ -47,20 +47,20 @@ pub struct AssetServerInternal<TAssetIo: AssetIo = FileAssetIo> {
}

/// Loads assets from the filesystem on background threads
pub struct AssetServer<TAssetIo: AssetIo = FileAssetIo> {
pub(crate) server: Arc<AssetServerInternal<TAssetIo>>,
pub struct AssetServer {
pub(crate) server: Arc<AssetServerInternal>,
}

impl<TAssetIo: AssetIo> Clone for AssetServer<TAssetIo> {
impl Clone for AssetServer {
fn clone(&self) -> Self {
Self {
server: self.server.clone(),
}
}
}

impl<TAssetIo: AssetIo> AssetServer<TAssetIo> {
pub fn new(source_io: TAssetIo, task_pool: TaskPool) -> Self {
impl AssetServer {
pub fn new<T: AssetIo>(source_io: T, task_pool: TaskPool) -> Self {
AssetServer {
server: Arc::new(AssetServerInternal {
loaders: Default::default(),
Expand All @@ -70,7 +70,7 @@ impl<TAssetIo: AssetIo> AssetServer<TAssetIo> {
handle_to_path: Default::default(),
asset_lifecycles: Default::default(),
task_pool,
asset_io: source_io,
asset_io: Box::new(source_io),
}),
}
}
Expand Down Expand Up @@ -180,7 +180,7 @@ impl<TAssetIo: AssetIo> AssetServer<TAssetIo> {
}

// TODO: properly set failed LoadState in all failure cases
fn load_sync<'a, P: Into<AssetPath<'a>>>(
async fn load_async<'a, P: Into<AssetPath<'a>>>(
&self,
path: P,
force: bool,
Expand Down Expand Up @@ -221,17 +221,18 @@ impl<TAssetIo: AssetIo> AssetServer<TAssetIo> {
};

// load the asset bytes
let bytes = self.server.asset_io.load_path(asset_path.path())?;
let bytes = self.server.asset_io.load_path(asset_path.path()).await?;

// load the asset source using the corresponding AssetLoader
let mut load_context = LoadContext::new(
asset_path.path(),
&self.server.asset_ref_counter.channel,
&self.server.asset_io,
&*self.server.asset_io,
version,
);
asset_loader
.load(&bytes, &mut load_context)
.await
.map_err(AssetServerError::AssetLoaderError)?;

// if version has changed since we loaded and grabbed a lock, return. theres is a newer version being loaded
Expand Down Expand Up @@ -291,7 +292,7 @@ impl<TAssetIo: AssetIo> AssetServer<TAssetIo> {
self.server
.task_pool
.spawn(async move {
server.load_sync(owned_path, force).unwrap();
server.load_async(owned_path, force).await.unwrap();
})
.detach();
asset_path.into()
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
use crate::{filesystem_watcher::FilesystemWatcher, AssetIo, AssetIoError, AssetServer};
use anyhow::Result;
use async_trait::async_trait;
use bevy_ecs::Res;
use bevy_utils::HashSet;
use crossbeam_channel::TryRecvError;
Expand All @@ -10,33 +12,6 @@ use std::{
path::{Path, PathBuf},
sync::Arc,
};
use thiserror::Error;

use crate::{filesystem_watcher::FilesystemWatcher, AssetServer};

/// Errors that occur while loading assets
#[derive(Error, Debug)]
pub enum AssetIoError {
#[error("Path not found")]
NotFound(PathBuf),
#[error("Encountered an io error while loading asset.")]
Io(#[from] io::Error),
#[error("Failed to watch path")]
PathWatchError(PathBuf),
}

/// Handles load requests from an AssetServer
pub trait AssetIo: Send + Sync + 'static {
fn load_path(&self, path: &Path) -> Result<Vec<u8>, AssetIoError>;
fn save_path(&self, path: &Path, bytes: &[u8]) -> Result<(), AssetIoError>;
fn read_directory(
&self,
path: &Path,
) -> Result<Box<dyn Iterator<Item = PathBuf>>, AssetIoError>;
fn is_directory(&self, path: &Path) -> bool;
fn watch_path_for_changes(&self, path: &Path) -> Result<(), AssetIoError>;
fn watch_for_changes(&self) -> Result<(), AssetIoError>;
}

pub struct FileAssetIo {
root_path: PathBuf,
Expand Down Expand Up @@ -67,8 +42,9 @@ impl FileAssetIo {
}
}

#[async_trait]
impl AssetIo for FileAssetIo {
fn load_path(&self, path: &Path) -> Result<Vec<u8>, AssetIoError> {
async fn load_path(&self, path: &Path) -> Result<Vec<u8>, AssetIoError> {
let mut bytes = Vec::new();
match File::open(self.root_path.join(path)) {
Ok(mut file) => {
Expand Down Expand Up @@ -98,15 +74,6 @@ impl AssetIo for FileAssetIo {
)))
}

fn save_path(&self, path: &Path, bytes: &[u8]) -> Result<(), AssetIoError> {
let path = self.root_path.join(path);
if let Some(parent_path) = path.parent() {
fs::create_dir_all(parent_path)?;
}

Ok(fs::write(self.root_path.join(path), bytes)?)
}

fn watch_path_for_changes(&self, path: &Path) -> Result<(), AssetIoError> {
#[cfg(feature = "filesystem_watcher")]
{
Expand Down Expand Up @@ -139,7 +106,13 @@ impl AssetIo for FileAssetIo {
#[cfg(feature = "filesystem_watcher")]
pub fn filesystem_watcher_system(asset_server: Res<AssetServer>) {
let mut changed = HashSet::default();
let watcher = asset_server.server.asset_io.filesystem_watcher.read();
let asset_io =
if let Some(asset_io) = asset_server.server.asset_io.downcast_ref::<FileAssetIo>() {
asset_io
} else {
return;
};
let watcher = asset_io.filesystem_watcher.read();
if let Some(ref watcher) = *watcher {
loop {
let event = match watcher.receiver.try_recv() {
Expand All @@ -155,9 +128,7 @@ pub fn filesystem_watcher_system(asset_server: Res<AssetServer>) {
{
for path in paths.iter() {
if !changed.contains(path) {
let relative_path = path
.strip_prefix(&asset_server.server.asset_io.root_path)
.unwrap();
let relative_path = path.strip_prefix(&asset_io.root_path).unwrap();
let _ = asset_server.load_untracked(relative_path, true);
}
}
Expand Down
45 changes: 45 additions & 0 deletions crates/bevy_asset/src/io/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#[cfg(not(target_arch = "wasm32"))]
mod file_asset_io;
#[cfg(target_arch = "wasm32")]
mod wasm_asset_io;

#[cfg(not(target_arch = "wasm32"))]
pub use file_asset_io::*;
#[cfg(target_arch = "wasm32")]
pub use wasm_asset_io::*;

use anyhow::Result;
use async_trait::async_trait;
use downcast_rs::{impl_downcast, Downcast};
use std::{
io,
path::{Path, PathBuf},
};
use thiserror::Error;

/// Errors that occur while loading assets
#[derive(Error, Debug)]
pub enum AssetIoError {
#[error("Path not found")]
NotFound(PathBuf),
#[error("Encountered an io error while loading asset.")]
Io(#[from] io::Error),
#[error("Failed to watch path")]
PathWatchError(PathBuf),
}

/// Handles load requests from an AssetServer
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
pub trait AssetIo: Downcast + Send + Sync + 'static {
async fn load_path(&self, path: &Path) -> Result<Vec<u8>, AssetIoError>;
fn read_directory(
&self,
path: &Path,
) -> Result<Box<dyn Iterator<Item = PathBuf>>, AssetIoError>;
fn is_directory(&self, path: &Path) -> bool;
fn watch_path_for_changes(&self, path: &Path) -> Result<(), AssetIoError>;
fn watch_for_changes(&self) -> Result<(), AssetIoError>;
}

impl_downcast!(AssetIo);
54 changes: 54 additions & 0 deletions crates/bevy_asset/src/io/wasm_asset_io.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
use crate::{AssetIo, AssetIoError};
use anyhow::Result;
use async_trait::async_trait;
use js_sys::Uint8Array;
use std::path::{Path, PathBuf};
use wasm_bindgen::JsCast;
use wasm_bindgen_futures::JsFuture;
use web_sys::Response;

pub struct WasmAssetIo {
root_path: PathBuf,
}

impl WasmAssetIo {
pub fn new<P: AsRef<Path>>(path: P) -> Self {
WasmAssetIo {
root_path: path.as_ref().to_owned(),
}
}
}

#[async_trait(?Send)]
impl AssetIo for WasmAssetIo {
async fn load_path(&self, path: &Path) -> Result<Vec<u8>, AssetIoError> {
let path = self.root_path.join(path);
let window = web_sys::window().unwrap();

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will not work in web workers and WorkerGlobalScope should be used instead. However, web-sys doesn't seem to provide any abstraction to access this cross-environments as WindowOrWorkerGlobalScope mixin is not supported.

I think the best option would be to access js_sys::global() and try casting it to either Window or WorkerGlobalScope, whichever succeeds. Similar fix was done in rustwasm/gloo#106, but it's kind of ugly. Maybe it could land in bevy_utils?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

haha anything specific about web-sys / webworkers is going to be out of my depth at the moment. but that sounds reasonable to me. given that we know this works outside of webworkers, im inclined to merge as-is for now. Then someone who knows what they're doing (like you) can follow up with webworker compat.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Regarding block_on, I think it would be best to make loaders async as it is the only way to make it work on single threaded wasm. GLTF loading with sub assets would still be broken, but that can be fixed by making schedule async as in the wasm threads PR.

+1 for making loaders async - I did it on my branch and it was very simple change

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was hesitant to make asset loaders async because I didn't want consumers to need the #[async_trait] proc-macro. But:

  1. atelier-assets uses async loaders
  2. they do it without requiring #[async_trait] in consumers by using BoxFuture
  3. even if async_trait was required, thats not a huge loss relative to what we get

I'll make the change

Copy link
Member

@mrk-its mrk-its Oct 19, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

here is my implementation, for example working gltf loader: mrk-its@05bf8ae

BTW I didn't know how to conditionally enable '?Send' without copying whole struct definition, that's why I've ended with wrapping web_sys types with unsafe impl Send :) good to know about #[cfg_attr] :)

let resp_value = JsFuture::from(window.fetch_with_str(path.to_str().unwrap()))
.await
.unwrap();
let resp: Response = resp_value.dyn_into().unwrap();
let data = JsFuture::from(resp.array_buffer().unwrap()).await.unwrap();
let bytes = Uint8Array::new(&data).to_vec();
Ok(bytes)
}

fn read_directory(
&self,
_path: &Path,
) -> Result<Box<dyn Iterator<Item = PathBuf>>, AssetIoError> {
Ok(Box::new(std::iter::empty::<PathBuf>()))
}

fn watch_path_for_changes(&self, _path: &Path) -> Result<(), AssetIoError> {
Ok(())
}

fn watch_for_changes(&self) -> Result<(), AssetIoError> {
Ok(())
}

fn is_directory(&self, path: &Path) -> bool {
self.root_path.join(path).is_dir()
}
}
10 changes: 7 additions & 3 deletions crates/bevy_asset/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
mod asset_server;
mod assets;
#[cfg(feature = "filesystem_watcher")]
#[cfg(all(feature = "filesystem_watcher", not(target_arch = "wasm32")))]
mod filesystem_watcher;
mod handle;
mod info;
Expand Down Expand Up @@ -37,7 +37,7 @@ use bevy_type_registry::RegisterType;
pub struct AssetPlugin;

pub struct AssetServerSettings {
asset_folder: String,
pub asset_folder: String,
}

impl Default for AssetServerSettings {
Expand All @@ -61,7 +61,11 @@ impl Plugin for AssetPlugin {
let settings = app
.resources_mut()
.get_or_insert_with(AssetServerSettings::default);

#[cfg(not(target_arch = "wasm32"))]
let source = FileAssetIo::new(&settings.asset_folder);
#[cfg(target_arch = "wasm32")]
let source = WasmAssetIo::new(&settings.asset_folder);
AssetServer::new(source, task_pool)
};

Expand All @@ -74,7 +78,7 @@ impl Plugin for AssetPlugin {
asset_server::free_unused_assets_system.system(),
);

#[cfg(feature = "filesystem_watcher")]
#[cfg(all(feature = "filesystem_watcher", not(target_arch = "wasm32")))]
app.add_system_to_stage(stage::LOAD_ASSETS, io::filesystem_watcher_system.system());
}
}
12 changes: 8 additions & 4 deletions crates/bevy_asset/src/loader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,18 @@ use crate::{
use anyhow::Result;
use bevy_ecs::{Res, ResMut, Resource};
use bevy_type_registry::{TypeUuid, TypeUuidDynamic};
use bevy_utils::HashMap;
use bevy_utils::{BoxedFuture, HashMap};
use crossbeam_channel::{Receiver, Sender};
use downcast_rs::{impl_downcast, Downcast};
use std::path::Path;

/// A loader for an asset source
pub trait AssetLoader: Send + Sync + 'static {
fn load(&self, bytes: &[u8], load_context: &mut LoadContext) -> Result<(), anyhow::Error>;
fn load<'a>(
&'a self,
bytes: &'a [u8],
load_context: &'a mut LoadContext,
) -> BoxedFuture<'a, Result<(), anyhow::Error>>;
fn extensions(&self) -> &[&str];
}

Expand Down Expand Up @@ -94,8 +98,8 @@ impl<'a> LoadContext<'a> {
Handle::strong(id.into(), self.ref_change_channel.sender.clone())
}

pub fn read_asset_bytes<P: AsRef<Path>>(&self, path: P) -> Result<Vec<u8>, AssetIoError> {
self.asset_io.load_path(path.as_ref())
pub async fn read_asset_bytes<P: AsRef<Path>>(&self, path: P) -> Result<Vec<u8>, AssetIoError> {
self.asset_io.load_path(path.as_ref()).await
}

pub fn get_asset_metas(&self) -> Vec<AssetMeta> {
Expand Down
1 change: 1 addition & 0 deletions crates/bevy_audio/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ bevy_app = { path = "../bevy_app", version = "0.2.1" }
bevy_asset = { path = "../bevy_asset", version = "0.2.1" }
bevy_ecs = { path = "../bevy_ecs", version = "0.2.1" }
bevy_type_registry = { path = "../bevy_type_registry", version = "0.2.1" }
bevy_utils = { path = "../bevy_utils", version = "0.2.1" }

# other
anyhow = "1.0"
Expand Down
Loading