fetchNASIS("components"): use duplicates argument and custom profile ID for many:1 relationships of legend, mapunit to datamapunit

NEWS.md

@@ -1,3 +1,9 @@
+# soilDB 2.7.2 (2022-06-24)
+
+ * `fetchNASIS("components')`: Fix and refactor ( **breaking change** from 2.7.1, which introduced a bug/inconsistency) of recent change; `duplicates` argument is now _required_ to merge in data from mapunit/legend tables (where many:1 relationships between legend/mapunit and datamapunit are possible). In 2.7.1 possibly incomplete mapunit/legend tables could be joined to SoilProfileCollection result (if and only if the tables were populated in selected set/local DB). Does not change historic (<=2.7.0) default behavior. Thanks to @dylanbeaudette for suggesting use of `get_component_correlation_data_from_NASIS_db()` here.
+
+ * Several fixes for Roxygen documentation (notably for `ROSETTA()` and various NASIS web report related functions) that were missing `@export` tags. Several previously-exported functions missed being explicitly exported in the new Roxygen-generate NAMESPACE. These unintentional omissions from 2.7.1 NAMESPACE have been resolved.
+
 # soilDB 2.7.1 (2022-06-10)
    - `get_SDA_coecoclass()` better handling of `NULL` `ecoclassref`; support for filtering on `ecoclasstypename`; `not_assigned_value` now applies to `ecoclassname`, `ecoclasstypename` and `ecoclassref` columns in addition to `ecoclassid`; Thanks to Andy Paolucci and Jason Nemecek. Also, added additional columns from legend/mapunit tables (`areasymbol`, `lkey`, `muname`).
    - `fetchNASIS(from="components")` now returns mapunit and legend information (if loaded in local NASIS database); results now contain `mustatus` and `repdmu` which can be used to remove components from additional mapunits and non-representative data mapunits; thanks to Nathan Roe

R/fetchNASIS.R

@@ -56,9 +56,11 @@
 #' @param lab should the `phlabresults` child table be fetched with
 #' site/pedon/horizon data (default: `FALSE`)
 #' @param fill include pedon or component records without horizon data in result? (default: `FALSE`)
+#' @param dropAdditional Used only for `from='components'` with `duplicates=TRUE`. Prevent "duplication" of `mustatus=="additional"`  mapunits? Default: `TRUE`
+#' @param dropNonRepresentative Used only for `from='components'` with `duplicates=TRUE`. Prevent "duplication" of non-representative data mapunits? Default: `TRUE`
+#' @param duplicates Used only for `from='components'`. Duplicate components for all instances of use (i.e. one for each legend data mapunit is used on; optionally for additional mapunits, and/or non-representative data mapunits?)
 #' @param stringsAsFactors deprecated
-#' @param dsn Optional: path to local SQLite database containing NASIS
-#' table structure; default: `NULL`
+#' @param dsn Optional: path to local SQLite database containing NASIS table structure; default: `NULL`
 #' @return A SoilProfileCollection object
 #' @seealso `get_component_data_from_NASIS()`
 #' @author D. E. Beaudette, J. M. Skovlin, S.M. Roecker, A.G. Brown
@@ -73,6 +75,9 @@ fetchNASIS <- function(from = 'pedons',
                        mixColors = TRUE,
                        lab = FALSE,
                        fill = FALSE,
+                       dropAdditional = TRUE,
+                       dropNonRepresentative = TRUE,
+                       duplicates = FALSE,
                        stringsAsFactors = NULL,
                        dsn = NULL) {
 
@@ -114,7 +119,10 @@ fetchNASIS <- function(from = 'pedons',
                                   rmHzErrors = rmHzErrors,
                                   nullFragsAreZero = nullFragsAreZero,
                                   fill = fill,
-                                  dsn = dsn)
+                                  dsn = dsn,
+                                  dropAdditional = dropAdditional,
+                                  dropNotRepresentative = dropNonRepresentative,
+                                  duplicates = duplicates)
   }
 
   if (from == 'pedon_report') {

R/fetchNASIS_components.R

@@ -7,9 +7,11 @@
                                    nullFragsAreZero = TRUE,
                                    fill = FALSE,
                                    stringsAsFactors = NULL,
-                                   dsn = NULL) {
+                                   dsn = NULL,
+                                   dropAdditional = TRUE,
+                                   dropNotRepresentative = TRUE,
+                                   duplicates = FALSE) {
 
-
   if (!missing(stringsAsFactors) && is.logical(stringsAsFactors)) {
     .Deprecated(msg = sprintf("stringsAsFactors argument is deprecated.\nSetting package option with `NASISDomainsAsFactor(%s)`", stringsAsFactors))
     NASISDomainsAsFactor(stringsAsFactors)
@@ -22,23 +24,9 @@
 
   # optionally legend and mapunit information are included if in local DB/selected set
   #   includes possible results for rep and non-rep DMUs and any mustatus
-  f.lg         <- get_legend_from_NASIS(SS = SS, dsn = dsn)
-  mu.q <- "SELECT ng.grpname, liid, lmapunitiid, lmapunitiid AS mukey, areatypename,
-           nationalmusym, muiid, musym, muname, mukind, mutype, muacres, mustatus,
-           dmuinvesintens, farmlndcl, dmuiid, repdmu 
-           FROM area a 
-            INNER JOIN areatype at ON at.areatypeiid = a.areatypeiidref
-            INNER JOIN legend_View_1 l ON l.areaiidref = a.areaiid 
-            INNER JOIN lmapunit_View_1 lmu ON lmu.liidref = l.liid 
-            INNER JOIN mapunit_View_1 mu ON mu.muiid = lmu.muiidref
-            INNER JOIN nasisgroup ng ON ng.grpiid = mu.grpiidref
-            INNER JOIN correlation_View_1 cor ON cor.muiidref = mu.muiid
-            INNER JOIN datamapunit_View_1 dmu ON dmu.dmuiid = cor.dmuiidref"
-  if (!SS) {
-    mu.q <- gsub("_View_1", "", mu.q)
+  if (duplicates) {
+    f.corr       <- get_component_correlation_data_from_NASIS_db(SS = SS, dsn = dsn, dropAdditional = dropAdditional, dropNotRepresentative = dropNotRepresentative)
   }
-  f.mu         <- uncode(dbQueryNASIS(NASIS(dsn = dsn), mu.q), dsn = dsn)
-
   # load data in pieces
   f.comp       <- get_component_data_from_NASIS_db(SS = SS, dsn = dsn, nullFragsAreZero = nullFragsAreZero)
   f.chorizon   <- get_component_horizon_data_from_NASIS_db(SS = SS, fill = fill, dsn = dsn, nullFragsAreZero = nullFragsAreZero)
@@ -80,26 +68,50 @@
     assign('component.hz.problems', value = bad.ids, envir = soilDB.env)
   }
 
+  # diagnostics and restrictions
+  # 2021-11-30: subset to hide aqp warnings for <- methods
+  f.diaghz2 <- f.diaghz[which(f.diaghz$coiid %in% f.chorizon$coiid),]
+  f.restrict2 <- f.restrict[which(f.restrict$coiid %in% f.chorizon$coiid),]
+
   if (nrow(f.chorizon) > 0) {
-    # upgrade to SoilProfilecollection
-    depths(f.chorizon) <- coiid ~ hzdept_r + hzdepb_r
+    if (duplicates) {
+      f.chorizon <- merge(f.chorizon, f.comp[,c("coiid","dmuiid")], by = "coiid", all.x = TRUE, all.y = TRUE, sort = FALSE)
+      f.chorizon <- merge(f.corr[,c("dmuiid","muiid","lmapunitiid")], f.chorizon, all.y = TRUE, by = "dmuiid", sort = FALSE)
+      f.chorizon$coiidcmb <- paste0(f.chorizon$lmapunitiid, ":", f.chorizon$muiid, ":", f.chorizon$dmuiid, ":", f.chorizon$coiid)
+
+      f.diaghz2 <- merge(f.diaghz2, f.comp[,c("coiid","dmuiid")], by = "coiid", all.x = TRUE, all.y = TRUE, sort = FALSE)
+      f.diaghz2 <- merge(f.corr[,c("dmuiid","muiid","lmapunitiid")], f.diaghz2, all.y = TRUE, by = "dmuiid", sort = FALSE)
+      f.diaghz2$coiidcmb <- paste0(f.diaghz2$lmapunitiid, ":", f.diaghz2$muiid, ":", f.diaghz2$dmuiid, ":", f.diaghz2$coiid)
+
+      f.restrict2 <- merge(f.restrict2, f.comp[,c("coiid","dmuiid")], by = "coiid", all.x = TRUE, all.y = TRUE, sort = FALSE)
+      f.restrict2 <- merge(f.corr[,c("dmuiid","muiid","lmapunitiid")], f.restrict2, all.y = TRUE, by = "dmuiid", sort = FALSE)
+      f.restrict2$coiidcmb <- paste0(f.restrict2$lmapunitiid, ":", f.restrict2$muiid, ":", f.restrict2$dmuiid, ":", f.restrict2$coiid)
+    }
+
+    if (duplicates) {
+      # use combined coiid (lmapunitiid, muiid, dmuiid, coiid) under name coiidcmb
+      depths(f.chorizon) <- coiidcmb ~ hzdept_r + hzdepb_r
+      site(f.chorizon) <- ~ dmuiid + muiid + lmapunitiid + coiid
+    } else {
+      # upgrade to SoilProfilecollection
+      depths(f.chorizon) <- coiid ~ hzdept_r + hzdepb_r
+    }
+
   } else {
     stop("No horizon data in NASIS component query result.", call. = FALSE)
   }
 
   # add site data to object
   site(f.chorizon) <- f.comp # left-join via coiid
 
-  # add mapunit data to object if any
-  if (!is.null(f.mu) && nrow(f.mu) > 0) {
-    site(f.chorizon) <- f.mu # left-join via dmuiid
+  if (duplicates && !is.null(f.corr) && nrow(f.corr) > 0) {
+    site(f.chorizon) <- f.corr # left-join via dmuiid, muiid, lmapunitiid
   }
 
-  # add legend data to object if any
-  if (!is.null(f.lg) && nrow(f.lg) > 0) {
-    site(f.chorizon) <- f.lg # left-join via liid
-  }
-
+  # add diagnostic features and restrictions to SPC
+  diagnostic_hz(f.chorizon) <- f.diaghz2
+  restrictions(f.chorizon) <- f.restrict2
+
   ## 2017-3-13: short-circuits need testing, consider pre-marking mistakes before parsing
   ## 2021-10-28: TODO: harmonize strategies for .formatXXXXString methods and ID variables
   .SD <- NULL
@@ -131,14 +143,6 @@
   if (nrow(ov) > 0) {
     site(f.chorizon) <- ov
   }
-  # 2021-11-30: subset to hide aqp warnings for <- methods
-
-  # add diagnostic features to SPC
-  diagnostic_hz(f.chorizon) <- f.diaghz[which(f.diaghz$coiid %in% f.chorizon$coiid),]
-
-  # add restrictions to SPC
-  # required new setter in aqp SPC object (AGB added 2019/12/23)
-  restrictions(f.chorizon) <- f.restrict[which(f.restrict$coiid %in% f.chorizon$coiid),]
 
   # print any messages on possible data quality problems:
   if (exists('component.hz.problems', envir = soilDB.env)) {

R/get_component_data_from_NASIS_db.R

@@ -180,7 +180,7 @@ get_component_correlation_data_from_NASIS_db <- function(SS = TRUE,
     NASISDomainsAsFactor(stringsAsFactors)
   }
 
-  q <- "SELECT lmapunitiid, mu.muiid, musym, nationalmusym, mu.muname, mukind, mutype, mustatus, muacres, farmlndcl, repdmu, dmuiid, areasymbol, areaname, ssastatus, cordate
+  q <- "SELECT lmapunitiid, lmapunitiid AS mukey, mu.muiid, musym, nationalmusym, mu.muname, mukind, mutype, mustatus, muacres, farmlndcl, repdmu, dmuiid, areasymbol, areaname, ssastatus, cordate
 
   FROM  mapunit_View_1 AS mu