Merge pull request #86 from NREL/vehicle-fetching-caching

adding new from_github vehicle method

rust/fastsim-core/Cargo.toml

@@ -39,6 +39,9 @@ include_dir = "0.7.3"
 itertools = "0.12.0"
 ndarray-stats = "0.5.1"
 tempfile = "3.8.1"
+url = "2.5.0"
+ureq = "2.9.1"
+isahc = "1.7.2"
 
 [package.metadata]
 include = [

rust/fastsim-core/src/cycle.rs

@@ -636,6 +636,7 @@ pub struct RustCycle {
 impl SerdeAPI for RustCycle {
     const ACCEPTED_BYTE_FORMATS: &'static [&'static str] = &["yaml", "json", "bin", "csv"];
     const ACCEPTED_STR_FORMATS: &'static [&'static str] = &["yaml", "json", "csv"];
+    const CACHE_FOLDER: &'static str = &"cycles";
 
     fn init(&mut self) -> anyhow::Result<()> {
         self.init_checks()

rust/fastsim-core/src/imports.rs

@@ -13,3 +13,4 @@ pub(crate) use std::path::PathBuf;
 
 pub(crate) use crate::traits::*;
 pub(crate) use crate::utils::*;
+pub(crate) use crate::vehicle_utils::*;

rust/fastsim-core/src/traits.rs

@@ -1,9 +1,11 @@
 use crate::imports::*;
 use std::collections::HashMap;
+use ureq;
 
 pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> {
     const ACCEPTED_BYTE_FORMATS: &'static [&'static str] = &["yaml", "json", "bin"];
     const ACCEPTED_STR_FORMATS: &'static [&'static str] = &["yaml", "json"];
+    const CACHE_FOLDER: &'static str = &"";
 
     /// Specialized code to execute upon initialization
     fn init(&mut self) -> anyhow::Result<()> {
@@ -189,6 +191,83 @@ pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> {
         bincode_de.init()?;
         Ok(bincode_de)
     }
+
+    /// Instantiates an object from a url.  Accepts yaml and json file types  
+    /// # Arguments  
+    /// - url: URL (either as a string or url type) to object  
+    /// Note: The URL needs to be a URL pointing directly to a file, for example
+    /// a raw github URL.
+    fn from_url<S: AsRef<str>>(url: S) -> anyhow::Result<Self> {
+        let url = url::Url::parse(url.as_ref())?;
+        let format = url
+            .path_segments()
+            .and_then(|segments| segments.last())
+            .and_then(|filename| Path::new(filename).extension())
+            .and_then(OsStr::to_str)
+            .with_context(|| "Could not parse file format from URL: {url:?}")?;
+        let response = ureq::get(url.as_ref()).call()?.into_reader();
+        Self::from_reader(response, format)
+    }
+
+    /// Takes an instantiated Rust object and saves it in the FASTSim data directory in
+    /// a rust_objects folder.  
+    /// WARNING: If there is a file already in the data subdirectory with the
+    /// same name, it will be replaced by the new file.  
+    /// # Arguments  
+    /// - self (rust object)  
+    /// - file_path: path to file within subdirectory. If only the file name is
+    /// listed, file will sit directly within the subdirectory of
+    /// the FASTSim data directory. If a path is given, the file will live
+    /// within the path specified, within the subdirectory CACHE_FOLDER of the
+    /// FASTSim data directory.
+    fn to_cache<P: AsRef<Path>>(&self, file_path: P) -> anyhow::Result<()> {
+        let file_name = file_path
+            .as_ref()
+            .file_name()
+            .with_context(|| "Could not determine file name")?
+            .to_str()
+            .context("Could not determine file name.")?;
+        let mut subpath = PathBuf::new();
+        let file_path_internal = file_path
+            .as_ref()
+            .to_str()
+            .context("Could not determine file name.")?;
+        if file_name == file_path_internal {
+            subpath = PathBuf::from(Self::CACHE_FOLDER);
+        } else {
+            subpath = Path::new(Self::CACHE_FOLDER).join(
+                file_path_internal
+                    .strip_suffix(file_name)
+                    .context("Could not determine path to subdirectory.")?,
+            );
+        }
+        let data_subdirectory = create_project_subdir(subpath)
+            .with_context(|| "Could not find or build Fastsim data subdirectory.")?;
+        let file_path = data_subdirectory.join(file_name);
+        self.to_file(file_path)
+    }
+
+    /// Instantiates a Rust object from the subdirectory within the FASTSim data
+    /// directory corresponding to the Rust Object ("vehices" for a RustVehice,
+    /// "cycles" for a RustCycle, and the root folder of the data directory for
+    /// all other objects).  
+    /// # Arguments  
+    /// - file_path: subpath to object, including file name, within subdirectory.
+    ///   If the file sits directly in the subdirectory, this will just be the
+    ///   file name.  
+    /// Note: This function will work for all objects cached using the
+    /// to_cache() method. If a file has been saved manually to a different
+    /// subdirectory than the correct one for the object type (for instance a
+    /// RustVehicle saved within a subdirectory other than "vehicles" using the
+    /// utils::url_to_cache() function), then from_cache() will not be able to
+    /// find and instantiate the object. Instead, use the from_file method, and
+    /// use the utils::path_to_cache() to find the FASTSim data directory
+    /// location if needed.
+    fn from_cache<P: AsRef<Path>>(file_path: P) -> anyhow::Result<Self> {
+        let full_file_path = Path::new(Self::CACHE_FOLDER).join(file_path);
+        let path_including_directory = path_to_cache()?.join(full_file_path);
+        Self::from_file(path_including_directory)
+    }
 }
 
 pub trait ApproxEq<Rhs = Self> {

rust/fastsim-core/src/utils.rs

@@ -7,6 +7,7 @@ use ndarray::*;
 use ndarray_stats::QuantileExt;
 use regex::Regex;
 use std::collections::HashSet;
+use url::Url;
 
 use crate::imports::*;
 #[cfg(feature = "pyo3")]
@@ -525,6 +526,59 @@ pub fn create_project_subdir<P: AsRef<Path>>(subpath: P) -> anyhow::Result<PathB
     Ok(path)
 }
 
+/// Returns the path to the OS-specific data directory, if it exists.
+pub fn path_to_cache() -> anyhow::Result<PathBuf> {
+    let proj_dirs = ProjectDirs::from("gov", "NREL", "fastsim").ok_or_else(|| {
+        anyhow!("Could not build path to project directory: \"gov.NREL.fastsim\"")
+    })?;
+    Ok(PathBuf::from(proj_dirs.config_dir()))
+}
+
+/// Deletes FASTSim data directory, clearing its contents. If subpath is
+/// provided, will only delete the subdirectory pointed to by the subpath,
+/// rather than deleting the whole data directory. If the subpath is an empty
+/// string, deletes the entire FASTSim directory.     
+/// USE WITH CAUTION, as this function deletes ALL objects stored in the FASTSim
+/// data directory or provided subdirectory.  
+/// # Arguments  
+/// - subpath: Subpath to a subdirectory within the FASTSim data directory. If
+///   an empty string, the function will delete the whole FASTSim data
+///   directory, clearing all its contents.  
+/// Note: it is not possible to delete single files using this function, only
+/// directories. If a single file needs deleting, the path_to_cache() function
+/// can be used to find the FASTSim data directory location. The file can then
+/// be found and manually deleted.
+pub fn clear_cache<P: AsRef<Path>>(subpath: P) -> anyhow::Result<()> {
+    let path = path_to_cache()?.join(subpath);
+    Ok(std::fs::remove_dir_all(path)?)
+}
+
+/// takes an object from a url and saves it in the FASTSim data directory in a
+/// rust_objects folder  
+/// WARNING: if there is a file already in the data subdirectory with the same
+/// name, it will be replaced by the new file   
+/// to save to a folder other than rust_objects, define constant CACHE_FOLDER to
+/// be the desired folder name  
+/// # Arguments  
+/// - url: url (either as a string or url type) to object  
+/// - subpath: path to subdirectory within FASTSim data directory. Suggested
+/// paths are "vehicles" for a RustVehicle, "cycles" for a RustCycle, and
+/// "rust_objects" for other Rust objects.  
+/// Note: In order for the file to be save in the proper format, the URL needs
+/// to be a URL pointing directly to a file, for example a raw github URL.
+pub fn url_to_cache<S: AsRef<str>, P: AsRef<Path>>(url: S, subpath: P) -> anyhow::Result<()> {
+    let url = Url::parse(url.as_ref())?;
+    let file_name = url
+        .path_segments()
+        .and_then(|segments| segments.last())
+        .with_context(|| "Could not parse filename from URL: {url:?}")?;
+    let data_subdirectory = create_project_subdir(subpath)
+        .with_context(|| "Could not find or build Fastsim data subdirectory.")?;
+    let file_path = data_subdirectory.join(file_name);
+    download_file_from_url(url.as_ref(), &file_path)?;
+    Ok(())
+}
+
 #[cfg(feature = "pyo3")]
 pub mod array_wrappers {
     use crate::proc_macros::add_pyo3_api;
@@ -732,4 +786,28 @@ mod tests {
         let y_lookup = interpolate_vectors(&x, &xs.to_vec(), &ys.to_vec(), false);
         assert_eq!(expected_y_lookup, y_lookup);
     }
+
+    #[test]
+    fn test_path_to_cache() {
+        let path = path_to_cache().unwrap();
+        println!("{:?}", path);
+    }
+
+    #[test]
+    fn test_clear_cache() {
+        let temp_sub_dir = tempfile::TempDir::new_in(create_project_subdir("").unwrap()).unwrap();
+        let sub_dir_path = temp_sub_dir.path().to_str().unwrap();
+        let still_exists_before = std::fs::metadata(sub_dir_path).is_ok();
+        assert_eq!(still_exists_before, true);
+        url_to_cache("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", "").unwrap();
+        clear_cache(sub_dir_path).unwrap();
+        let still_exists = std::fs::metadata(sub_dir_path).is_ok();
+        assert_eq!(still_exists, false);
+        let path_to_vehicle = path_to_cache()
+            .unwrap()
+            .join("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml");
+        let vehicle_still_exists = std::fs::metadata(&path_to_vehicle).is_ok();
+        assert_eq!(vehicle_still_exists, true);
+        std::fs::remove_file(path_to_vehicle).unwrap();
+    }
 }

rust/fastsim-core/src/vehicle.rs

@@ -545,6 +545,8 @@ pub struct RustVehicle {
 
 /// RustVehicle rust methods
 impl RustVehicle {
+    const VEHICLE_DIRECTORY_URL: &'static str =
+        &"https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/";
     /// Sets the following parameters:
     /// - `ess_mass_kg`
     /// - `mc_mass_kg`
@@ -1023,6 +1025,43 @@ impl RustVehicle {
         v.set_derived().unwrap();
         v
     }
+
+    /// Downloads specified vehicle from FASTSim vehicle repo or url and
+    /// instantiates it into a RustVehicle. Notes in vehicle.doc the origin of
+    /// the vehicle. Returns vehicle.  
+    /// # Arguments  
+    /// - vehicle_file_name: file name for vehicle to be downloaded, including
+    ///   path from url directory or FASTSim repository (if applicable)  
+    /// - url: url for vehicle repository where vehicle will be downloaded from,
+    ///   if None, assumed to be downloaded from vehicle FASTSim repo  
+    /// Note: The URL needs to be a URL pointing directly to a file, for example
+    /// a raw github URL, split up so that the "url" argument is the path to the
+    /// directory, and the "vehicle_file_name" is the path within the directory
+    /// to the file.  
+    /// Note: If downloading from the FASTSim Vehicle Repo, the
+    /// vehicle_file_name should include the path to the file from the root of
+    /// the Repo, as listed in the output of the
+    /// vehicle_utils::fetch_github_list() function.  
+    /// Note: the url should not include the file name, only the path to the
+    /// file or a root directory of the file.
+    pub fn from_github_or_url<S: AsRef<str>>(
+        vehicle_file_name: S,
+        url: Option<S>,
+    ) -> anyhow::Result<Self> {
+        let url_internal = match url {
+            Some(s) => {
+                s.as_ref().trim_end_matches('/').to_owned()
+                    + "/"
+                    + &vehicle_file_name.as_ref().trim_start_matches('/')
+            }
+            None => Self::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name.as_ref(),
+        };
+        let mut vehicle =
+            Self::from_url(&url_internal).with_context(|| "Could not parse vehicle from url")?;
+        let vehicle_origin = "Vehicle from ".to_owned() + url_internal.as_str();
+        vehicle.doc = Some(vehicle_origin);
+        Ok(vehicle)
+    }
 }
 
 impl Default for RustVehicle {
@@ -1051,9 +1090,33 @@ impl Default for RustVehicle {
 }
 
 impl SerdeAPI for RustVehicle {
+    const CACHE_FOLDER: &'static str = &"vehicles";
+
     fn init(&mut self) -> anyhow::Result<()> {
         self.set_derived()
     }
+
+    /// instantiates a vehicle from a url, and notes in vehicle.doc the origin
+    /// of the vehicle.  
+    /// accepts yaml and json file types  
+    /// # Arguments  
+    /// - url: URL (either as a string or url type) to object  
+    /// Note: The URL needs to be a URL pointing directly to a file, for example
+    /// a raw github URL.
+    fn from_url<S: AsRef<str>>(url: S) -> anyhow::Result<Self> {
+        let url = url::Url::parse(url.as_ref())?;
+        let format = url
+            .path_segments()
+            .and_then(|segments| segments.last())
+            .and_then(|filename| Path::new(filename).extension())
+            .and_then(OsStr::to_str)
+            .with_context(|| "Could not parse file format from URL: {url:?}")?;
+        let response = ureq::get(url.as_ref()).call()?.into_reader();
+        let mut vehicle = Self::from_reader(response, format)?;
+        let vehicle_origin = "Vehicle from ".to_owned() + url.as_ref();
+        vehicle.doc = Some(vehicle_origin);
+        Ok(vehicle)
+    }
 }
 
 #[cfg(test)]