From 3ed06c869efd84dc317f58286b53ff206840fe93 Mon Sep 17 00:00:00 2001 From: Christopher Rabotin Date: Sat, 14 Dec 2024 10:59:36 -0700 Subject: [PATCH 1/4] Add TDM KVN parsing + Tracking data downsampling (low pass filter) + filter by tracker I also noticed that the overlapping may no longer work with the new tracking data structure, so I added a warning. --- src/io/mod.rs | 2 +- src/od/msr/mod.rs | 4 +- src/od/msr/trackingdata/io_ccsds_tdm.rs | 216 ++++++++++++++ .../io_parquet.rs} | 143 +--------- src/od/msr/trackingdata/mod.rs | 265 ++++++++++++++++++ src/od/simulator/arc.rs | 3 + 6 files changed, 490 insertions(+), 143 deletions(-) create mode 100644 src/od/msr/trackingdata/io_ccsds_tdm.rs rename src/od/msr/{data_arc.rs => trackingdata/io_parquet.rs} (70%) create mode 100644 src/od/msr/trackingdata/mod.rs diff --git a/src/io/mod.rs b/src/io/mod.rs index ca2d321f..4e241227 100644 --- a/src/io/mod.rs +++ b/src/io/mod.rs @@ -186,7 +186,7 @@ pub enum InputOutputError { }, #[snafu(display("missing required data {which}"))] MissingData { which: String }, - #[snafu(display("unknown data column `{which}`"))] + #[snafu(display("unknown data `{which}`"))] UnsupportedData { which: String }, #[snafu(display("{action} encountered a Parquet error: {source}"))] ParquetError { diff --git a/src/od/msr/mod.rs b/src/od/msr/mod.rs index 61d2414f..cd0ccf8c 100644 --- a/src/od/msr/mod.rs +++ b/src/od/msr/mod.rs @@ -16,11 +16,11 @@ along with this program. If not, see . */ -mod data_arc; pub mod measurement; pub mod sensitivity; +mod trackingdata; mod types; -pub use data_arc::TrackingDataArc; pub use measurement::Measurement; +pub use trackingdata::TrackingDataArc; pub use types::MeasurementType; diff --git a/src/od/msr/trackingdata/io_ccsds_tdm.rs b/src/od/msr/trackingdata/io_ccsds_tdm.rs new file mode 100644 index 00000000..a6c72238 --- /dev/null +++ b/src/od/msr/trackingdata/io_ccsds_tdm.rs @@ -0,0 +1,216 @@ +/* + Nyx, blazing fast astrodynamics + Copyright (C) 2018-onwards Christopher Rabotin + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +use crate::io::ExportCfg; +use crate::io::{InputOutputError, StdIOSnafu}; +use crate::od::msr::{Measurement, MeasurementType}; +use hifitime::prelude::Epoch; +use hifitime::TimeScale; +use snafu::ResultExt; +use std::collections::{BTreeMap, HashMap}; +use std::error::Error; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; + +use super::TrackingDataArc; + +impl TrackingDataArc { + /// Loads a tracking arc from its serialization in CCSDS TDM. + pub fn from_tdm>( + path: P, + aliases: Option>, + ) -> Result { + let file = File::open(&path).context(StdIOSnafu { + action: "opening CCSDS TDM file for tracking arc", + })?; + + let mut measurements = BTreeMap::new(); + + let reader = BufReader::new(file); + + let mut in_data_section = false; + let mut current_tracker = String::new(); + let mut time_system = TimeScale::UTC; + + for line in reader.lines() { + let line = line.context(StdIOSnafu { + action: "reading CCSDS TDM file", + })?; + let line = line.trim(); + + if line == "DATA_START" { + in_data_section = true; + continue; + } else if line == "DATA_STOP" { + in_data_section = false; + } + + if !in_data_section { + if line.starts_with("PARTICIPANT_1") { + current_tracker = line.split('=').nth(1).unwrap_or("").trim().to_string(); + // If aliases are provided, try to map them. + if let Some(aliases) = &aliases { + if let Some(alias) = aliases.get(¤t_tracker) { + current_tracker = alias.clone(); + } + } + } + if line.starts_with("TIME_SYSTEM") { + let ts = line.split('=').nth(1).unwrap_or("UTC").trim(); + match ts { + "UTC" => time_system = TimeScale::UTC, + "TAI" => time_system = TimeScale::TAI, + "GPS" => time_system = TimeScale::GPST, + _ => { + return Err(InputOutputError::UnsupportedData { + which: format!("time scale `{ts}` not supported"), + }) + } + } + } + continue; + } + + if let Some((mtype, epoch, value)) = parse_measurement_line(line, time_system)? { + measurements + .entry(epoch) + .or_insert_with(|| Measurement { + tracker: current_tracker.clone(), + epoch, + data: HashMap::new(), + }) + .data + .insert(mtype, value); + } + } + + Ok(Self { + measurements, + source: Some(path.as_ref().to_path_buf().display().to_string()), + }) + } + + /// Store this tracking arc to a CCSDS TDM file, with optional metadata and a timestamp appended to the filename. + pub fn to_tdm>( + &self, + path: P, + cfg: ExportCfg, + ) -> Result> { + todo!() + } +} + +fn parse_measurement_line( + line: &str, + time_system: TimeScale, +) -> Result, InputOutputError> { + let parts: Vec<&str> = line.split('=').collect(); + if parts.len() != 2 { + return Ok(None); + } + + let (mtype_str, data) = (parts[0].trim(), parts[1].trim()); + let mtype = match mtype_str { + "RANGE" => MeasurementType::Range, + "DOPPLER_INSTANTANEOUS" | "DOPPLER_INTEGRATED" => MeasurementType::Doppler, + "ANGLE_1" => MeasurementType::Azimuth, + "ANGLE_2" => MeasurementType::Elevation, + _ => { + return Err(InputOutputError::UnsupportedData { + which: mtype_str.to_string(), + }) + } + }; + + let data_parts: Vec<&str> = data.split_whitespace().collect(); + if data_parts.len() != 2 { + return Ok(None); + } + + let epoch = + Epoch::from_gregorian_str(&format!("{} {time_system}", data_parts[0])).map_err(|e| { + InputOutputError::Inconsistency { + msg: format!("{e} when parsing epoch"), + } + })?; + + let value = data_parts[1] + .parse::() + .map_err(|e| InputOutputError::UnsupportedData { + which: format!("`{}` is not a float: {e}", data_parts[1]), + })?; + + Ok(Some((mtype, epoch, value))) +} + +#[cfg(test)] +mod ut_tdm { + extern crate pretty_env_logger as pel; + use hifitime::TimeUnits; + use std::{collections::HashMap, path::PathBuf}; + + use crate::od::msr::TrackingDataArc; + + #[test] + fn test_tdm_no_alias() { + let path: PathBuf = [ + env!("CARGO_MANIFEST_DIR"), + "output_data", + "demo_tracking_arc.tdm", + ] + .iter() + .collect(); + + let tdm = TrackingDataArc::from_tdm(path, None).unwrap(); + println!("{tdm}"); + } + + #[test] + fn test_tdm_with_alias() { + pel::init(); + let mut aliases = HashMap::new(); + aliases.insert("DSS-65 Madrid".to_string(), "DSN Madrid".to_string()); + + let path: PathBuf = [ + env!("CARGO_MANIFEST_DIR"), + "output_data", + "demo_tracking_arc.tdm", + ] + .iter() + .collect(); + + let tdm = TrackingDataArc::from_tdm(path, Some(aliases)).unwrap(); + println!("{tdm}"); + + let tdm_failed_downsample = tdm.clone().downsample(10.seconds()); + assert_eq!( + tdm_failed_downsample.len(), + tdm.len(), + "downsampling should have failed because it's upsampling" + ); + + let tdm_downsample = tdm.clone().downsample(10.minutes()); + println!("{tdm_downsample}"); + assert_eq!( + tdm_downsample.len(), + tdm.len() / 10 + 1, + "downsampling has wrong sample count" + ); + } +} diff --git a/src/od/msr/data_arc.rs b/src/od/msr/trackingdata/io_parquet.rs similarity index 70% rename from src/od/msr/data_arc.rs rename to src/od/msr/trackingdata/io_parquet.rs index 031aef5c..81f9f3dc 100644 --- a/src/od/msr/data_arc.rs +++ b/src/od/msr/trackingdata/io_parquet.rs @@ -15,10 +15,10 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -use super::{measurement::Measurement, MeasurementType}; use crate::io::watermark::pq_writer; use crate::io::{ArrowSnafu, InputOutputError, MissingDataSnafu, ParquetSnafu, StdIOSnafu}; use crate::io::{EmptyDatasetSnafu, ExportCfg}; +use crate::od::msr::{Measurement, MeasurementType}; use arrow::array::{Array, Float64Builder, StringBuilder}; use arrow::datatypes::{DataType, Field, Schema}; use arrow::record_batch::RecordBatch; @@ -27,34 +27,22 @@ use arrow::{ datatypes, record_batch::RecordBatchReader, }; -use core::fmt; -use hifitime::prelude::{Duration, Epoch}; +use hifitime::prelude::Epoch; use hifitime::TimeScale; -use indexmap::IndexSet; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use parquet::arrow::ArrowWriter; use snafu::{ensure, ResultExt}; use std::collections::{BTreeMap, HashMap}; use std::error::Error; use std::fs::File; -use std::ops::Bound::{Excluded, Included, Unbounded}; -use std::ops::RangeBounds; use std::path::{Path, PathBuf}; use std::sync::Arc; -/// Tracking data storing all of measurements as a B-Tree. -#[derive(Clone, Default)] -pub struct TrackingDataArc { - /// All measurements in this data arc - pub measurements: BTreeMap, - /// Source file if loaded from a file or saved to a file. - pub source: Option, -} +use super::TrackingDataArc; impl TrackingDataArc { /// Loads a tracking arc from its serialization in parquet. pub fn from_parquet>(path: P) -> Result { - // Read the file since we closed it earlier let file = File::open(&path).context(StdIOSnafu { action: "opening file for tracking arc", })?; @@ -225,96 +213,6 @@ impl TrackingDataArc { source: Some(path.as_ref().to_path_buf().display().to_string()), }) } - - /// Returns the unique list of aliases in this tracking data arc - pub fn unique_aliases(&self) -> IndexSet { - self.unique().0 - } - - /// Returns the unique measurement types in this tracking data arc - pub fn unique_types(&self) -> IndexSet { - self.unique().1 - } - - /// Returns the unique trackers and unique measurement types in this data arc - pub fn unique(&self) -> (IndexSet, IndexSet) { - let mut aliases = IndexSet::new(); - let mut types = IndexSet::new(); - for msr in self.measurements.values() { - aliases.insert(msr.tracker.clone()); - for k in msr.data.keys() { - types.insert(*k); - } - } - (aliases, types) - } - - /// Returns the start epoch of this tracking arc - pub fn start_epoch(&self) -> Option { - self.measurements.first_key_value().map(|(k, _)| *k) - } - - /// Returns the end epoch of this tracking arc - pub fn end_epoch(&self) -> Option { - self.measurements.last_key_value().map(|(k, _)| *k) - } - - /// Returns the number of measurements in this data arc - pub fn len(&self) -> usize { - self.measurements.len() - } - - /// Returns whether this arc has no measurements. - pub fn is_empty(&self) -> bool { - self.measurements.is_empty() - } - - /// Returns the minimum duration between two subsequent measurements. - /// This is important to correctly set up the propagator and not miss any measurement. - pub fn min_duration_sep(&self) -> Option { - if self.is_empty() { - None - } else { - let mut min_sep = Duration::MAX; - let mut prev_epoch = self.start_epoch().unwrap(); - for (epoch, _) in self.measurements.iter().skip(1) { - let this_sep = *epoch - prev_epoch; - min_sep = min_sep.min(this_sep); - prev_epoch = *epoch; - } - Some(min_sep) - } - } - - /// Returns a new tracking arc that only contains measurements that fall within the given epoch range. - pub fn filter_by_epoch>(mut self, bound: R) -> Self { - self.measurements = self - .measurements - .range(bound) - .map(|(epoch, msr)| (*epoch, msr.clone())) - .collect::>(); - self - } - - /// Returns a new tracking arc that only contains measurements that fall within the given offset from the first epoch - pub fn filter_by_offset>(self, bound: R) -> Self { - if self.is_empty() { - return self; - } - // Rebuild an epoch bound. - let start = match bound.start_bound() { - Unbounded => self.start_epoch().unwrap(), - Included(offset) | Excluded(offset) => self.start_epoch().unwrap() + *offset, - }; - - let end = match bound.end_bound() { - Unbounded => self.end_epoch().unwrap(), - Included(offset) | Excluded(offset) => self.end_epoch().unwrap() - *offset, - }; - - self.filter_by_epoch(start..end) - } - /// Store this tracking arc to a parquet file. pub fn to_parquet_simple>(&self, path: P) -> Result> { self.to_parquet(path, ExportCfg::default()) @@ -434,38 +332,3 @@ impl TrackingDataArc { Ok(path_buf) } } - -impl fmt::Display for TrackingDataArc { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.is_empty() { - write!(f, "Empty tracking arc") - } else { - let start = self.start_epoch().unwrap(); - let end = self.end_epoch().unwrap(); - let src = match &self.source { - Some(src) => format!(" (source: {src})"), - None => String::new(), - }; - write!( - f, - "Tracking arc with {} measurements of type {:?} over {} (from {start} to {end}) with trackers {:?}{src}", - self.len(), - self.unique_types(), - end - start, - self.unique_aliases() - ) - } - } -} - -impl fmt::Debug for TrackingDataArc { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{self} @ {self:p}") - } -} - -impl PartialEq for TrackingDataArc { - fn eq(&self, other: &Self) -> bool { - self.measurements == other.measurements - } -} diff --git a/src/od/msr/trackingdata/mod.rs b/src/od/msr/trackingdata/mod.rs new file mode 100644 index 00000000..80a9e672 --- /dev/null +++ b/src/od/msr/trackingdata/mod.rs @@ -0,0 +1,265 @@ +/* + Nyx, blazing fast astrodynamics + Copyright (C) 2018-onwards Christopher Rabotin + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ +use super::{measurement::Measurement, MeasurementType}; +use core::fmt; +use hifitime::prelude::{Duration, Epoch}; +use indexmap::IndexSet; +use std::collections::{BTreeMap, HashMap}; +use std::ops::Bound::{Excluded, Included, Unbounded}; +use std::ops::RangeBounds; + +mod io_ccsds_tdm; +mod io_parquet; + +/// Tracking data storing all of measurements as a B-Tree. +/// It inherently does NOT support multiple concurrent measurements from several trackers. +#[derive(Clone, Default)] +pub struct TrackingDataArc { + /// All measurements in this data arc + pub measurements: BTreeMap, // BUG: Consider a map of tracking to epoch! + /// Source file if loaded from a file or saved to a file. + pub source: Option, +} + +impl TrackingDataArc { + /// Returns the unique list of aliases in this tracking data arc + pub fn unique_aliases(&self) -> IndexSet { + self.unique().0 + } + + /// Returns the unique measurement types in this tracking data arc + pub fn unique_types(&self) -> IndexSet { + self.unique().1 + } + + /// Returns the unique trackers and unique measurement types in this data arc + pub fn unique(&self) -> (IndexSet, IndexSet) { + let mut aliases = IndexSet::new(); + let mut types = IndexSet::new(); + for msr in self.measurements.values() { + aliases.insert(msr.tracker.clone()); + for k in msr.data.keys() { + types.insert(*k); + } + } + (aliases, types) + } + + /// Returns the start epoch of this tracking arc + pub fn start_epoch(&self) -> Option { + self.measurements.first_key_value().map(|(k, _)| *k) + } + + /// Returns the end epoch of this tracking arc + pub fn end_epoch(&self) -> Option { + self.measurements.last_key_value().map(|(k, _)| *k) + } + + /// Returns the number of measurements in this data arc + pub fn len(&self) -> usize { + self.measurements.len() + } + + /// Returns whether this arc has no measurements. + pub fn is_empty(&self) -> bool { + self.measurements.is_empty() + } + + /// Returns the minimum duration between two subsequent measurements. + /// This is important to correctly set up the propagator and not miss any measurement. + pub fn min_duration_sep(&self) -> Option { + if self.is_empty() { + None + } else { + let mut min_sep = Duration::MAX; + let mut prev_epoch = self.start_epoch().unwrap(); + for (epoch, _) in self.measurements.iter().skip(1) { + let this_sep = *epoch - prev_epoch; + min_sep = min_sep.min(this_sep); + prev_epoch = *epoch; + } + Some(min_sep) + } + } + + /// Returns a new tracking arc that only contains measurements that fall within the given epoch range. + pub fn filter_by_epoch>(mut self, bound: R) -> Self { + self.measurements = self + .measurements + .range(bound) + .map(|(epoch, msr)| (*epoch, msr.clone())) + .collect::>(); + self + } + + /// Returns a new tracking arc that only contains measurements that fall within the given offset from the first epoch + pub fn filter_by_offset>(self, bound: R) -> Self { + if self.is_empty() { + return self; + } + // Rebuild an epoch bound. + let start = match bound.start_bound() { + Unbounded => self.start_epoch().unwrap(), + Included(offset) | Excluded(offset) => self.start_epoch().unwrap() + *offset, + }; + + let end = match bound.end_bound() { + Unbounded => self.end_epoch().unwrap(), + Included(offset) | Excluded(offset) => self.end_epoch().unwrap() - *offset, + }; + + self.filter_by_epoch(start..end) + } + + /// Returns a new tracking arc that only contains measurements from the desired tracker. + pub fn filter_by_tracker(mut self, tracker: String) -> Self { + self.measurements = self + .measurements + .iter() + .filter_map(|(epoch, msr)| { + if msr.tracker == tracker { + Some((*epoch, msr.clone())) + } else { + None + } + }) + .collect::>(); + self + } + + /// Downsamples the tracking data to a lower frequency using a simple moving average low-pass filter followed by decimation, + /// returning new `TrackingDataArc` with downsampled measurements. + /// + /// It provides a computationally efficient approach to reduce the sampling rate while mitigating aliasing effects. + /// + /// # Algorithm + /// + /// 1. A simple moving average filter is applied as a low-pass filter. + /// 2. Decimation is performed by selecting every Nth sample after filtering. + /// + /// # Advantages + /// + /// - Computationally efficient, suitable for large datasets common in spaceflight applications. + /// - Provides basic anti-aliasing, crucial for preserving signal integrity in orbit determination and tracking. + /// - Maintains phase information, important for accurate timing in spacecraft state estimation. + /// + /// # Limitations + /// + /// - The frequency response is not as sharp as more sophisticated filters (e.g., FIR, IIR). + /// - May not provide optimal stopband attenuation for high-precision applications. + /// + /// ## Considerations for Spaceflight Applications + /// + /// - Suitable for initial data reduction in ground station tracking pipelines. + /// - Adequate for many orbit determination and tracking tasks where computational speed is prioritized. + /// - For high-precision applications (e.g., interplanetary navigation), consider using more advanced filtering techniques. + /// + pub fn downsample(self, target_step: Duration) -> Self { + if self.is_empty() { + return self; + } + let current_step = self.min_duration_sep().unwrap(); + + if current_step >= target_step { + warn!("cannot downsample tracking data from {current_step} to {target_step} (that would be upsampling)"); + return self; + } + + let current_hz = 1.0 / current_step.to_seconds(); + let target_hz = 1.0 / target_step.to_seconds(); + + // Simple moving average as low-pass filter + let window_size = (current_hz / target_hz).round() as usize; + + info!("downsampling tracking data from {current_step} ({current_hz:.6} Hz) to {target_step} ({target_hz:.6} Hz) (N = {window_size})"); + + let mut result = TrackingDataArc { + source: self.source.clone(), + ..Default::default() + }; + + let measurements: Vec<_> = self.measurements.iter().collect(); + + for (i, (epoch, _)) in measurements.iter().enumerate().step_by(window_size) { + let start = if i >= window_size / 2 { + i - window_size / 2 + } else { + 0 + }; + let end = (i + window_size / 2 + 1).min(measurements.len()); + let window = &measurements[start..end]; + + let mut filtered_measurement = Measurement { + tracker: window[0].1.tracker.clone(), + epoch: **epoch, + data: HashMap::new(), + }; + + // Apply moving average filter for each measurement type + for mtype in self.unique_types() { + let sum: f64 = window.iter().filter_map(|(_, m)| m.data.get(&mtype)).sum(); + let count = window + .iter() + .filter(|(_, m)| m.data.contains_key(&mtype)) + .count(); + + if count > 0 { + filtered_measurement.data.insert(mtype, sum / count as f64); + } + } + + result.measurements.insert(**epoch, filtered_measurement); + } + result + } +} + +impl fmt::Display for TrackingDataArc { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.is_empty() { + write!(f, "Empty tracking arc") + } else { + let start = self.start_epoch().unwrap(); + let end = self.end_epoch().unwrap(); + let src = match &self.source { + Some(src) => format!(" (source: {src})"), + None => String::new(), + }; + write!( + f, + "Tracking arc with {} measurements of type {:?} over {} (from {start} to {end}) with trackers {:?}{src}", + self.len(), + self.unique_types(), + end - start, + self.unique_aliases() + ) + } + } +} + +impl fmt::Debug for TrackingDataArc { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self} @ {self:p}") + } +} + +impl PartialEq for TrackingDataArc { + fn eq(&self, other: &Self) -> bool { + self.measurements == other.measurements + } +} diff --git a/src/od/simulator/arc.rs b/src/od/simulator/arc.rs index caa59a60..c55ecae2 100644 --- a/src/od/simulator/arc.rs +++ b/src/od/simulator/arc.rs @@ -288,6 +288,9 @@ impl TrackingArcSim { if let Some(cfg) = self.configs.get(name) { if let Some(scheduler) = cfg.scheduler { info!("Building schedule for {name}"); + if scheduler.handoff == Handoff::Overlap { + warn!("Overlapping measurements on {name} is no longer supported on identical epochs."); + } built_cfg.get_mut(name).unwrap().scheduler = None; built_cfg.get_mut(name).unwrap().strands = Some(Vec::new()); From 6983a675feb5d140949b5dab52c94b589ce424b7 Mon Sep 17 00:00:00 2001 From: Christopher Rabotin Date: Sat, 14 Dec 2024 12:05:34 -0700 Subject: [PATCH 2/4] Add TDM exporting --- .github/workflows/python.yml | 215 ---------------------- src/io/watermark.rs | 2 +- src/md/trajectory/sc_traj.rs | 38 ++-- src/od/msr/trackingdata/io_ccsds_tdm.rs | 228 +++++++++++++++++------- tests/orbit_determination/simulator.rs | 53 +++++- 5 files changed, 238 insertions(+), 298 deletions(-) delete mode 100644 .github/workflows/python.yml diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml deleted file mode 100644 index dd6dc6ff..00000000 --- a/.github/workflows/python.yml +++ /dev/null @@ -1,215 +0,0 @@ -# name: Python -# on: -# push: -# branches: -# - main -# - master -# tags: -# - '*' -# pull_request: -# workflow_dispatch: - -# permissions: -# contents: read - -# jobs: -# linux: -# runs-on: ubuntu-latest -# strategy: -# matrix: -# target: [x86_64] -# steps: -# - uses: actions/checkout@v3 -# - uses: actions/setup-python@v4 -# with: -# python-version: "3.11" - -# - name: Build wheels -# uses: PyO3/maturin-action@v1 -# with: -# target: ${{ matrix.target }} -# args: --release --out dist --find-interpreter -F python -# sccache: 'true' -# rust-toolchain: 1.74 - -# - name: Upload wheels -# uses: actions/upload-artifact@v3 -# with: -# name: wheels -# path: dist - -# - name: pytest x86_64 -# if: ${{ matrix.target == 'x86_64' }} -# run: | -# set -e -# ls -lh dist -# pip install nyx_space --find-links dist --force-reinstall -v --no-cache-dir -# pip install pytest numpy pandas plotly pyarrow scipy pyyaml -# pytest - -# - name: Upload python tests HTMLs -# uses: actions/upload-artifact@v3 -# with: -# name: od-plots -# path: output_data/*.html - -# windows: -# runs-on: windows-latest -# strategy: -# matrix: -# target: [x64] -# steps: -# - uses: actions/checkout@v3 -# - uses: actions/setup-python@v4 -# with: -# python-version: "3.11" -# architecture: ${{ matrix.target }} -# - name: Build wheels -# uses: PyO3/maturin-action@v1 -# with: -# target: ${{ matrix.target }} -# args: --release --out dist --find-interpreter -F python -# rust-toolchain: 1.74 -# - name: Upload wheels -# uses: actions/upload-artifact@v3 -# with: -# name: wheels -# path: dist - -# - name: pytest -# shell: bash -# run: | -# set -e -# ls -lh dist -# pip install --find-links dist --force-reinstall nyx_space -# pip install pytest numpy pandas plotly pyarrow pyyaml -# pytest - -# macos: -# runs-on: macos-latest -# strategy: -# matrix: -# target: [x86_64, aarch64] -# steps: -# - uses: actions/checkout@v3 -# - uses: actions/setup-python@v4 -# with: -# python-version: "3.11" - -# - name: Update cargo packages -# run: cargo update - -# - name: Build wheels -# uses: PyO3/maturin-action@v1 -# with: -# target: ${{ matrix.target }} -# args: --release --out dist --find-interpreter -F python -# sccache: 'true' -# rust-toolchain: 1.74 - -# - name: Upload wheels -# uses: actions/upload-artifact@v3 -# with: -# name: wheels -# path: dist - -# - name: pytest -# if: ${{ !startsWith(matrix.target, 'aarch64') }} -# shell: bash -# run: | -# set -e -# ls -lh dist -# pip install --find-links dist --force-reinstall nyx_space -# pip install pytest numpy pandas plotly pyarrow pyyaml -# pytest - -# sdist: -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v3 -# - name: Build sdist -# uses: PyO3/maturin-action@v1 -# with: -# command: sdist -# args: --out dist -# rust-toolchain: 1.74 -# - name: Upload sdist -# uses: actions/upload-artifact@v3 -# with: -# name: wheels -# path: dist - -# release: -# name: Release -# runs-on: ubuntu-latest -# if: github.ref_type == 'tag' -# needs: [linux, windows, macos, sdist] -# steps: -# - uses: actions/download-artifact@v3 -# with: -# name: wheels -# - name: Publish to PyPI -# uses: PyO3/maturin-action@v1 -# env: -# MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} -# with: -# command: upload -# args: --skip-existing * - -# packaging: -# permissions: write-all -# runs-on: ubuntu-latest -# needs: [linux] -# steps: -# - name: Check out code -# uses: actions/checkout@v2 - -# - name: Set up Docker Buildx -# uses: docker/setup-buildx-action@v2 -# with: -# buildkitd-flags: --debug - -# - name: Cache Docker layers -# uses: actions/cache@v2 -# with: -# path: /tmp/.buildx-cache -# key: ${{ runner.os }}-buildx-${{ github.sha }} -# restore-keys: | -# ${{ runner.os }}-buildx- - -# - name: Build development image -# run: docker build -f Dockerfile.dev -t nyx-build . - -# - name: Run development container and build the package -# run: docker run --name nyx-builder nyx-build - -# - name: Copy built package from container to host -# run: docker cp nyx-builder:/app/target/wheels ./dist - -# - name: Get short SHA -# id: short-sha -# run: echo "::set-output name=sha::$(echo ${GITHUB_SHA::8})" - -# - name: Get the version -# id: get_version -# run: echo ::set-output name=VERSION::${GITHUB_REF#refs/tags/} - -# - name: Login to GitHub Container Registry -# uses: docker/login-action@v2 -# with: -# registry: ghcr.io -# username: ${{ github.actor }} -# password: ${{ secrets.GITHUB_TOKEN }} - -# - name: Build and push Docker image with built package -# uses: docker/build-push-action@v4 -# with: -# context: . -# file: ./Dockerfile -# push: true -# tags: | -# ghcr.io/nyx-space/nyx-fds:${{ steps.short-sha.outputs.sha }} -# ghcr.io/nyx-space/nyx-fds:${{ github.ref == 'refs/heads/master' && 'latest' || steps.short-sha.outputs.sha }} -# ghcr.io/nyx-space/nyx-fds:${{ startsWith(github.ref, 'refs/tags/') && steps.get_version.outputs.VERSION || steps.short-sha.outputs.sha }} -# cache-from: type=local,src=/tmp/.buildx-cache -# cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/src/io/watermark.rs b/src/io/watermark.rs index 3c095b6a..657bff51 100644 --- a/src/io/watermark.rs +++ b/src/io/watermark.rs @@ -60,5 +60,5 @@ pub(crate) fn pq_writer(metadata: Option>) -> Option String { - format!("Nyx v{}", build::PKG_VERSION) + format!("Nyx Space v{}", build::PKG_VERSION) } diff --git a/src/md/trajectory/sc_traj.rs b/src/md/trajectory/sc_traj.rs index d29b4c58..794d134b 100644 --- a/src/md/trajectory/sc_traj.rs +++ b/src/md/trajectory/sc_traj.rs @@ -332,6 +332,19 @@ impl Traj { // Write mandatory metadata writeln!(writer, "CCSDS_OMM_VERS = 2.0").map_err(err_hdlr)?; + + writeln!( + writer, + "COMMENT Built by {} -- https://nyxspace.com/\n", + prj_name_ver() + ) + .map_err(err_hdlr)?; + writeln!( + writer, + "COMMENT Nyx Space provided under the AGPL v3 open source license -- https://nyxspace.com/pricing\n" + ) + .map_err(err_hdlr)?; + writeln!( writer, "CREATION_DATE = {}", @@ -350,9 +363,9 @@ impl Traj { writeln!(writer, "META_START").map_err(err_hdlr)?; // Write optional metadata if let Some(object_name) = metadata.get("object_name") { - writeln!(writer, "OBJECT_NAME = {}", object_name).map_err(err_hdlr)?; + writeln!(writer, "\tOBJECT_NAME = {}", object_name).map_err(err_hdlr)?; } else if let Some(object_name) = &self.name { - writeln!(writer, "OBJECT_NAME = {}", object_name).map_err(err_hdlr)?; + writeln!(writer, "\tOBJECT_NAME = {}", object_name).map_err(err_hdlr)?; } let first_orbit = states[0].orbit; @@ -369,7 +382,7 @@ impl Traj { let ref_frame = frame_str.replace(center, " "); writeln!( writer, - "REF_FRAME = {}", + "\tREF_FRAME = {}", match ref_frame.trim() { "J2000" => "ICRF", _ => ref_frame.trim(), @@ -377,44 +390,37 @@ impl Traj { ) .map_err(err_hdlr)?; - writeln!(writer, "CENTER_NAME = {center}",).map_err(err_hdlr)?; + writeln!(writer, "\tCENTER_NAME = {center}",).map_err(err_hdlr)?; - writeln!(writer, "TIME_SYSTEM = {}", first_orbit.epoch.time_scale).map_err(err_hdlr)?; + writeln!(writer, "\tTIME_SYSTEM = {}", first_orbit.epoch.time_scale).map_err(err_hdlr)?; writeln!( writer, - "START_TIME = {}", + "\tSTART_TIME = {}", Formatter::new(states[0].epoch(), iso8601_no_ts) ) .map_err(err_hdlr)?; writeln!( writer, - "USEABLE_START_TIME = {}", + "\tUSEABLE_START_TIME = {}", Formatter::new(states[0].epoch(), iso8601_no_ts) ) .map_err(err_hdlr)?; writeln!( writer, - "USEABLE_STOP_TIME = {}", + "\tUSEABLE_STOP_TIME = {}", Formatter::new(states[states.len() - 1].epoch(), iso8601_no_ts) ) .map_err(err_hdlr)?; writeln!( writer, - "STOP_TIME = {}", + "\tSTOP_TIME = {}", Formatter::new(states[states.len() - 1].epoch(), iso8601_no_ts) ) .map_err(err_hdlr)?; writeln!(writer, "META_STOP\n").map_err(err_hdlr)?; - writeln!( - writer, - "COMMENT Generated by {} provided in AGPLv3 license -- https://nyxspace.com/\n", - prj_name_ver() - ) - .map_err(err_hdlr)?; - for sc_state in &states { let state = sc_state.orbit; writeln!( diff --git a/src/od/msr/trackingdata/io_ccsds_tdm.rs b/src/od/msr/trackingdata/io_ccsds_tdm.rs index a6c72238..beafa58f 100644 --- a/src/od/msr/trackingdata/io_ccsds_tdm.rs +++ b/src/od/msr/trackingdata/io_ccsds_tdm.rs @@ -16,17 +16,20 @@ along with this program. If not, see . */ +use crate::io::watermark::prj_name_ver; use crate::io::ExportCfg; use crate::io::{InputOutputError, StdIOSnafu}; use crate::od::msr::{Measurement, MeasurementType}; +use hifitime::efmt::{Format, Formatter}; use hifitime::prelude::Epoch; use hifitime::TimeScale; use snafu::ResultExt; use std::collections::{BTreeMap, HashMap}; -use std::error::Error; use std::fs::File; -use std::io::{BufRead, BufReader}; +use std::io::Write; +use std::io::{BufRead, BufReader, BufWriter}; use std::path::{Path, PathBuf}; +use std::str::FromStr; use super::TrackingDataArc; @@ -107,12 +110,169 @@ impl TrackingDataArc { } /// Store this tracking arc to a CCSDS TDM file, with optional metadata and a timestamp appended to the filename. - pub fn to_tdm>( - &self, + pub fn to_tdm_file>( + mut self, + spacecraft_name: String, + aliases: Option>, path: P, cfg: ExportCfg, - ) -> Result> { - todo!() + ) -> Result { + if self.is_empty() { + return Err(InputOutputError::MissingData { + which: " - empty tracking data cannot be exported to TDM".to_string(), + }); + } + + // Filter epochs if needed. + if cfg.start_epoch.is_some() && cfg.end_epoch.is_some() { + self = self.filter_by_epoch(cfg.start_epoch.unwrap()..cfg.end_epoch.unwrap()); + } else if cfg.start_epoch.is_some() { + self = self.filter_by_epoch(cfg.start_epoch.unwrap()..); + } else if cfg.end_epoch.is_some() { + self = self.filter_by_epoch(..cfg.end_epoch.unwrap()); + } + + let tick = Epoch::now().unwrap(); + info!("Exporting tracking data to CCSDS TDM file..."); + + // Grab the path here before we move stuff. + let path_buf = cfg.actual_path(path); + + let metadata = cfg.metadata.unwrap_or_default(); + + let file = File::create(&path_buf).context(StdIOSnafu { + action: "creating CCSDS TDM file for tracking arc", + })?; + let mut writer = BufWriter::new(file); + + let err_hdlr = |source| InputOutputError::StdIOError { + source, + action: "writing data to TDM file", + }; + + // Epoch formmatter. + let iso8601_no_ts = Format::from_str("%Y-%m-%dT%H:%M:%S.%f").unwrap(); + + // Write mandatory metadata + writeln!(writer, "CCSDS_TDM_VERS = 2.0").map_err(err_hdlr)?; + writeln!( + writer, + "\nCOMMENT Build by {} -- https://nyxspace.com", + prj_name_ver() + ) + .map_err(err_hdlr)?; + writeln!( + writer, + "COMMENT Nyx Space provided under the AGPL v3 open source license -- https://nyxspace.com/pricing\n" + ) + .map_err(err_hdlr)?; + writeln!( + writer, + "CREATION_DATE = {}", + Formatter::new(Epoch::now().unwrap(), iso8601_no_ts) + ) + .map_err(err_hdlr)?; + writeln!( + writer, + "ORIGINATOR = {}\n", + metadata + .get("originator") + .unwrap_or(&"Nyx Space".to_string()) + ) + .map_err(err_hdlr)?; + + // Create a new meta section for each tracker + // Get unique trackers and process each one separately + let trackers = self.unique_aliases(); + + for tracker in trackers { + let tracker_data = self.clone().filter_by_tracker(tracker.clone()); + + writeln!(writer, "META_START").map_err(err_hdlr)?; + writeln!(writer, "\tTIME_SYSTEM = UTC").map_err(err_hdlr)?; + writeln!( + writer, + "\tSTART_TIME = {}", + Formatter::new(tracker_data.start_epoch().unwrap(), iso8601_no_ts) + ) + .map_err(err_hdlr)?; + writeln!( + writer, + "\tSTOP_TIME = {}", + Formatter::new(tracker_data.end_epoch().unwrap(), iso8601_no_ts) + ) + .map_err(err_hdlr)?; + + writeln!( + writer, + "\tPARTICIPANT_1 = {}", + if let Some(aliases) = &aliases { + if let Some(alias) = aliases.get(&tracker) { + alias + } else { + &tracker + } + } else { + &tracker + } + ) + .map_err(err_hdlr)?; + + writeln!(writer, "\tPARTICIPANT_2 = {spacecraft_name}").map_err(err_hdlr)?; + + // Add additional metadata, could include timetag ref for example. + for (k, v) in &metadata { + if k != "originator" { + writeln!(writer, "\t{k} = {v}").map_err(err_hdlr)?; + } + } + + if self.unique_types().contains(&MeasurementType::Range) { + writeln!(writer, "\tRANGE_UNITS = km").map_err(err_hdlr)?; + } + + if self.unique_types().contains(&MeasurementType::Azimuth) + || self.unique_types().contains(&MeasurementType::Elevation) + { + writeln!(writer, "\tANGLE_TYPE = AZEL").map_err(err_hdlr)?; + } + + writeln!(writer, "META_STOP\n").map_err(err_hdlr)?; + + // Write the data section + writeln!(writer, "DATA_START").map_err(err_hdlr)?; + + // Process measurements for this tracker + for (epoch, measurement) in tracker_data.measurements { + for (mtype, value) in &measurement.data { + let type_str = match mtype { + MeasurementType::Range => "RANGE", + MeasurementType::Doppler => "DOPPLER_INTEGRATED", + MeasurementType::Azimuth => "ANGLE_1", + MeasurementType::Elevation => "ANGLE_2", + }; + + writeln!( + writer, + "\t{:<20} = {:<23}\t{:.12}", + type_str, + Formatter::new(epoch, iso8601_no_ts), + value + ) + .map_err(err_hdlr)?; + } + } + + writeln!(writer, "DATA_STOP\n").map_err(err_hdlr)?; + } + + #[allow(clippy::writeln_empty_string)] + writeln!(writer, "").map_err(err_hdlr)?; + + // Return the path this was written to + let tock_time = Epoch::now().unwrap() - tick; + info!("CCSDS TDM written to {} in {tock_time}", path_buf.display()); + Ok(path_buf) } } @@ -158,59 +318,3 @@ fn parse_measurement_line( Ok(Some((mtype, epoch, value))) } - -#[cfg(test)] -mod ut_tdm { - extern crate pretty_env_logger as pel; - use hifitime::TimeUnits; - use std::{collections::HashMap, path::PathBuf}; - - use crate::od::msr::TrackingDataArc; - - #[test] - fn test_tdm_no_alias() { - let path: PathBuf = [ - env!("CARGO_MANIFEST_DIR"), - "output_data", - "demo_tracking_arc.tdm", - ] - .iter() - .collect(); - - let tdm = TrackingDataArc::from_tdm(path, None).unwrap(); - println!("{tdm}"); - } - - #[test] - fn test_tdm_with_alias() { - pel::init(); - let mut aliases = HashMap::new(); - aliases.insert("DSS-65 Madrid".to_string(), "DSN Madrid".to_string()); - - let path: PathBuf = [ - env!("CARGO_MANIFEST_DIR"), - "output_data", - "demo_tracking_arc.tdm", - ] - .iter() - .collect(); - - let tdm = TrackingDataArc::from_tdm(path, Some(aliases)).unwrap(); - println!("{tdm}"); - - let tdm_failed_downsample = tdm.clone().downsample(10.seconds()); - assert_eq!( - tdm_failed_downsample.len(), - tdm.len(), - "downsampling should have failed because it's upsampling" - ); - - let tdm_downsample = tdm.clone().downsample(10.minutes()); - println!("{tdm_downsample}"); - assert_eq!( - tdm_downsample.len(), - tdm.len() / 10 + 1, - "downsampling has wrong sample count" - ); - } -} diff --git a/tests/orbit_determination/simulator.rs b/tests/orbit_determination/simulator.rs index 20234444..a04b0522 100644 --- a/tests/orbit_determination/simulator.rs +++ b/tests/orbit_determination/simulator.rs @@ -5,6 +5,7 @@ use nyx_space::od::prelude::*; use nyx_space::od::simulator::TrackingArcSim; use nyx_space::od::simulator::TrkConfig; use std::collections::BTreeMap; +use std::collections::HashMap; use std::env; use std::path::PathBuf; use std::str::FromStr; @@ -117,12 +118,56 @@ fn continuous_tracking(almanac: Arc) { println!("[{}] {arc}", output_fn.to_string_lossy()); // Now read this file back in. - let arc_concrete = TrackingDataArc::from_parquet(output_fn).unwrap(); + let arc_rtn = TrackingDataArc::from_parquet(output_fn).unwrap(); - println!("{arc_concrete}"); + println!("{arc_rtn}"); assert_eq!(arc.measurements.len(), 116); // Check that we've loaded all of the measurements - assert_eq!(arc_concrete.measurements.len(), arc.measurements.len()); - assert_eq!(arc_concrete.unique(), arc.unique()); + assert_eq!(arc_rtn.measurements.len(), arc.measurements.len()); + assert_eq!(arc_rtn.unique(), arc.unique()); + + // Serialize as TDM + let path: PathBuf = [env!("CARGO_MANIFEST_DIR"), "output_data", "simple_arc.tdm"] + .iter() + .collect(); + + let mut aliases = HashMap::new(); + aliases.insert("Demo Ground Station".to_string(), "Fake GS".to_string()); + + let output_fn = arc + .clone() + .to_tdm_file( + "MySpacecraft".to_string(), + Some(aliases), + path, + ExportCfg::default(), + ) + .unwrap(); + + // Read back from TDM + let arc_tdm = TrackingDataArc::from_tdm(output_fn, None).unwrap(); + println!("{arc_tdm}"); + + // Check everything but the source, since it'll be set when read from TDM. + assert_eq!(arc_tdm.len(), arc.len()); + assert_eq!(arc_tdm.start_epoch(), arc.start_epoch()); + assert_eq!(arc_tdm.end_epoch(), arc.end_epoch()); + assert_eq!(arc_tdm.unique(), arc.unique()); + + // Test the downsampling + let tdm_failed_downsample = arc_tdm.clone().downsample(10.seconds()); + assert_eq!( + tdm_failed_downsample.len(), + arc_tdm.len(), + "downsampling should have failed because it's upsampling" + ); + + let tdm_downsample = arc_tdm.clone().downsample(10.minutes()); + println!("{tdm_downsample}"); + assert_eq!( + tdm_downsample.len(), + arc_tdm.len() / 10 + 1, + "downsampling has wrong sample count" + ); } From bcf020bf1b1c3190ee44080f656cd9bf18e0dbe7 Mon Sep 17 00:00:00 2001 From: Christopher Rabotin Date: Sat, 14 Dec 2024 12:09:03 -0700 Subject: [PATCH 3/4] Switch to IndexMap for measurements to ensure consistent iteration --- src/od/msr/measurement.rs | 7 +++---- src/od/msr/trackingdata/io_ccsds_tdm.rs | 3 ++- src/od/msr/trackingdata/io_parquet.rs | 3 ++- src/od/msr/trackingdata/mod.rs | 6 +++--- tests/orbit_determination/simulator.rs | 12 +++++++++++- 5 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/od/msr/measurement.rs b/src/od/msr/measurement.rs index 9a03792f..b3778c32 100644 --- a/src/od/msr/measurement.rs +++ b/src/od/msr/measurement.rs @@ -18,9 +18,8 @@ use super::MeasurementType; use hifitime::Epoch; -use indexmap::IndexSet; +use indexmap::{IndexMap, IndexSet}; use nalgebra::{allocator::Allocator, DefaultAllocator, DimName, OVector}; -use std::collections::HashMap; use std::fmt; /// A type-agnostic simultaneous measurement storage structure. Allows storing any number of simultaneous measurement of a given taker. @@ -31,7 +30,7 @@ pub struct Measurement { /// Epoch of the measurement pub epoch: Epoch, /// All measurements made simultaneously - pub data: HashMap, + pub data: IndexMap, } impl Measurement { @@ -39,7 +38,7 @@ impl Measurement { Self { tracker, epoch, - data: HashMap::new(), + data: IndexMap::new(), } } diff --git a/src/od/msr/trackingdata/io_ccsds_tdm.rs b/src/od/msr/trackingdata/io_ccsds_tdm.rs index beafa58f..a786bb38 100644 --- a/src/od/msr/trackingdata/io_ccsds_tdm.rs +++ b/src/od/msr/trackingdata/io_ccsds_tdm.rs @@ -23,6 +23,7 @@ use crate::od::msr::{Measurement, MeasurementType}; use hifitime::efmt::{Format, Formatter}; use hifitime::prelude::Epoch; use hifitime::TimeScale; +use indexmap::IndexMap; use snafu::ResultExt; use std::collections::{BTreeMap, HashMap}; use std::fs::File; @@ -96,7 +97,7 @@ impl TrackingDataArc { .or_insert_with(|| Measurement { tracker: current_tracker.clone(), epoch, - data: HashMap::new(), + data: IndexMap::new(), }) .data .insert(mtype, value); diff --git a/src/od/msr/trackingdata/io_parquet.rs b/src/od/msr/trackingdata/io_parquet.rs index 81f9f3dc..fd55b0b8 100644 --- a/src/od/msr/trackingdata/io_parquet.rs +++ b/src/od/msr/trackingdata/io_parquet.rs @@ -29,6 +29,7 @@ use arrow::{ }; use hifitime::prelude::Epoch; use hifitime::TimeScale; +use indexmap::IndexMap; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use parquet::arrow::ArrowWriter; use snafu::{ensure, ResultExt}; @@ -177,7 +178,7 @@ impl TrackingDataArc { let mut measurement = Measurement { epoch, tracker: tracking_device.value(i).to_string(), - data: HashMap::new(), + data: IndexMap::new(), }; if range_avail { diff --git a/src/od/msr/trackingdata/mod.rs b/src/od/msr/trackingdata/mod.rs index 80a9e672..d11e5d11 100644 --- a/src/od/msr/trackingdata/mod.rs +++ b/src/od/msr/trackingdata/mod.rs @@ -18,8 +18,8 @@ use super::{measurement::Measurement, MeasurementType}; use core::fmt; use hifitime::prelude::{Duration, Epoch}; -use indexmap::IndexSet; -use std::collections::{BTreeMap, HashMap}; +use indexmap::{IndexMap, IndexSet}; +use std::collections::BTreeMap; use std::ops::Bound::{Excluded, Included, Unbounded}; use std::ops::RangeBounds; @@ -207,7 +207,7 @@ impl TrackingDataArc { let mut filtered_measurement = Measurement { tracker: window[0].1.tracker.clone(), epoch: **epoch, - data: HashMap::new(), + data: IndexMap::new(), }; // Apply moving average filter for each measurement type diff --git a/tests/orbit_determination/simulator.rs b/tests/orbit_determination/simulator.rs index a04b0522..958488ca 100644 --- a/tests/orbit_determination/simulator.rs +++ b/tests/orbit_determination/simulator.rs @@ -79,7 +79,17 @@ fn continuous_tracking(almanac: Arc) { let mut devices = BTreeMap::new(); for gs in GroundStation::load_many(ground_station_file).unwrap() { - devices.insert(gs.name.clone(), gs); + devices.insert( + gs.name.clone(), + gs.with_msr_type( + MeasurementType::Azimuth, + StochasticNoise::default_angle_deg(), + ) + .with_msr_type( + MeasurementType::Elevation, + StochasticNoise::default_angle_deg(), + ), + ); } // Load the tracking configuration from the test data. From 442cc4f986993d66d9a3fefe70989d1d77b138b4 Mon Sep 17 00:00:00 2001 From: Christopher Rabotin Date: Sat, 14 Dec 2024 13:22:19 -0700 Subject: [PATCH 4/4] Switch trk_arc test to high sample to test a more realistic situation of 1 second down to 10 second downsampling --- data/tests/config/tracking_cfg.yaml | 4 ++-- examples/04_lro_od/plot_msr.py | 10 ++++++++-- tests/orbit_determination/simulator.rs | 24 ++++++++++++++++++------ tests/orbit_determination/trackingarc.rs | 2 +- 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/data/tests/config/tracking_cfg.yaml b/data/tests/config/tracking_cfg.yaml index 1cedd3d3..f74731e6 100644 --- a/data/tests/config/tracking_cfg.yaml +++ b/data/tests/config/tracking_cfg.yaml @@ -4,7 +4,7 @@ Demo ground station: cadence: Continuous min_samples: 10 sample_alignment: null - sampling: 1 min + sampling: 1 s Canberra: scheduler: @@ -12,4 +12,4 @@ Canberra: cadence: Continuous min_samples: 10 sample_alignment: 10 s - sampling: 1 min + sampling: 1 s diff --git a/examples/04_lro_od/plot_msr.py b/examples/04_lro_od/plot_msr.py index 018c79a3..cb77e16a 100644 --- a/examples/04_lro_od/plot_msr.py +++ b/examples/04_lro_od/plot_msr.py @@ -1,8 +1,11 @@ +import click import polars as pl import plotly.express as px -if __name__ == "__main__": - df = pl.read_parquet("./04_lro_simulated_tracking.parquet") +@click.command +@click.option("-p", "--path", type=str, default="./04_lro_simulated_tracking.parquet") +def main(path: str): + df = pl.read_parquet(path) df = df.with_columns(pl.col("Epoch (UTC)").str.to_datetime("%Y-%m-%dT%H:%M:%S%.f")).sort( "Epoch (UTC)", descending=False @@ -21,3 +24,6 @@ # Plot each measurement kind for msr_col_name in ["Range (km)", "Doppler (km/s)"]: px.scatter(df, x="Epoch (UTC)", y=msr_col_name, color="Tracking device").show() + +if __name__ == "__main__": + main() diff --git a/tests/orbit_determination/simulator.rs b/tests/orbit_determination/simulator.rs index 958488ca..9b71f29a 100644 --- a/tests/orbit_determination/simulator.rs +++ b/tests/orbit_determination/simulator.rs @@ -132,7 +132,7 @@ fn continuous_tracking(almanac: Arc) { println!("{arc_rtn}"); - assert_eq!(arc.measurements.len(), 116); + assert_eq!(arc.measurements.len(), 7723); // Check that we've loaded all of the measurements assert_eq!(arc_rtn.measurements.len(), arc.measurements.len()); assert_eq!(arc_rtn.unique(), arc.unique()); @@ -149,7 +149,7 @@ fn continuous_tracking(almanac: Arc) { .clone() .to_tdm_file( "MySpacecraft".to_string(), - Some(aliases), + Some(aliases.clone()), path, ExportCfg::default(), ) @@ -166,18 +166,30 @@ fn continuous_tracking(almanac: Arc) { assert_eq!(arc_tdm.unique(), arc.unique()); // Test the downsampling - let tdm_failed_downsample = arc_tdm.clone().downsample(10.seconds()); + let tdm_failed_downsample = arc_tdm.clone().downsample(0.1.seconds()); assert_eq!( tdm_failed_downsample.len(), arc_tdm.len(), "downsampling should have failed because it's upsampling" ); - let tdm_downsample = arc_tdm.clone().downsample(10.minutes()); - println!("{tdm_downsample}"); + let arc_downsample = arc_tdm.clone().downsample(10.seconds()); + println!("{arc_downsample}"); assert_eq!( - tdm_downsample.len(), + arc_downsample.len(), arc_tdm.len() / 10 + 1, "downsampling has wrong sample count" ); + + let path: PathBuf = [ + env!("CARGO_MANIFEST_DIR"), + "output_data", + "simple_arc_downsampled.parquet", + ] + .iter() + .collect(); + + arc_downsample + .to_parquet(path, ExportCfg::default()) + .unwrap(); } diff --git a/tests/orbit_determination/trackingarc.rs b/tests/orbit_determination/trackingarc.rs index 111baabe..be4e595b 100644 --- a/tests/orbit_determination/trackingarc.rs +++ b/tests/orbit_determination/trackingarc.rs @@ -129,7 +129,7 @@ fn trk_simple( let arc = trk.generate_measurements(almanac).unwrap(); // Regression - assert_eq!(arc.measurements.len(), 197); + assert_eq!(arc.measurements.len(), 12803); // And serialize to disk let path: PathBuf = [