#69 First light for geotrace

* handle_ru_geotrace() retains original format (GeoJSON or WKT) and extracts lon, lat, alt from the first point of the geotrace.
* Add package data with example geofields, use in tests and examples, document
* Add source for geolocation test form as odkbuild and xml
* Add ODKC_TEST_FID_WKT to GHA
* RMD skeleton updated with comments re geofields
* Bump dev version

.github/workflows/main.yml

@@ -59,6 +59,7 @@ jobs:
       ODKC_TEST_FID_ZIP: ${{ secrets.ODKC_TEST_FID_ZIP }}
       ODKC_TEST_FID_ATT: ${{ secrets.ODKC_TEST_FID_ATT }}
       ODKC_TEST_FID_GAP: ${{ secrets.ODKC_TEST_FID_GAP }}
+      ODKC_TEST_FID_WKT: ${{ secrets.ODKC_TEST_FID_WKT }}
       ODKC_TEST_UN: ${{ secrets.ODKC_TEST_UN }}
       ODKC_TEST_PW: ${{ secrets.ODKC_TEST_PW }}
       ODKC_VERSION: ${{ secrets.ODKC_VERSION }}

DESCRIPTION

@@ -1,7 +1,7 @@
 Type: Package
 Package: ruODK
 Title: An R Client for the ODK Central API
-Version: 0.6.6.9021
+Version: 0.6.6.9022
 Authors@R: 
     c(person(given = c("Florian", "W."),
              family = "Mayer",
@@ -52,7 +52,7 @@ Imports:
     rlist (>= 0.4.6.1),
     stringr (>= 1.4.0),
     tibble (>= 2.1.3),
-    tidyr (>= 1.0.2),
+    tidyr (>= 1.0.3),
     tidyselect (>= 1.0.0),
     xml2 (>= 1.2.2)
 Suggests: 

NAMESPACE

@@ -40,6 +40,7 @@ export(get_test_url)
 export(handle_ru_attachments)
 export(handle_ru_datetimes)
 export(handle_ru_geopoints)
+export(handle_ru_geotraces)
 export(odata_metadata_get)
 export(odata_service_get)
 export(odata_submission_get)
@@ -59,6 +60,7 @@ export(ru_msg_warn)
 export(ru_settings)
 export(ru_setup)
 export(split_geopoint)
+export(split_geotrace)
 export(submission_detail)
 export(submission_export)
 export(submission_get)

R/data.R

@@ -357,3 +357,58 @@
 #' @family included
 #' @encoding UTF-8
 "fq_zip_taxa"
+
+#' The form_schema of a form containing geofields in GeoJSON.
+#'
+#' \lifecycle{stable}
+#'
+#' @source \code{\link{form_schema}}
+#' run on the test form
+#' `system.file("extdata", "Locations.xml", package = "ruODK")`.
+#' @family included
+#' @encoding UTF-8
+"geo_fs"
+
+#' The unparsed submissions of a form containing geofields in GeoJSON.
+#'
+#' \lifecycle{stable}
+#'
+#' @source \code{\link{odata_submission_get}(wkt=FALSE, parse=FALSE)}
+#' run on the test form
+#' `system.file("extdata", "Locations.xml", package = "ruODK")`.
+#' @family included
+#' @encoding UTF-8
+"geo_gj_raw"
+
+#' The parsed submissions of a form containing geofields in GeoJSON.
+#'
+#' \lifecycle{stable}
+#'
+#' @source \code{\link{odata_submission_get}(wkt=FALSE, parse=TRUE)}
+#' run on the test form
+#' `system.file("extdata", "Locations.xml", package = "ruODK")`.
+#' @family included
+#' @encoding UTF-8
+"geo_gj"
+
+#' The unparsed submissions of a form containing geofields in WKT.
+#'
+#' \lifecycle{stable}
+#'
+#' @source \code{\link{odata_submission_get}(wkt=TRUE, parse=FALSE)}
+#' run on the test form
+#' `system.file("extdata", "Locations.xml", package = "ruODK")`.
+#' @family included
+#' @encoding UTF-8
+"geo_wkt_raw"
+
+#' The parsed submissions of a form containing geofields in WKT.
+#'
+#' \lifecycle{stable}
+#'
+#' @source \code{\link{odata_submission_get}(wkt=TRUE, parse=TRUE)}
+#' run on the test form
+#' `system.file("extdata", "Locations.xml", package = "ruODK")`.
+#' @family included
+#' @encoding UTF-8
+"geo_wkt"

R/handle_ru_geopoints.R

@@ -16,43 +16,48 @@
 #'   ```
 #' @param form_schema The `form_schema` for the submissions.
 #'   E.g. the output of `ruODK::form_schema()`.
-#' @param wkt Whether to expect nested lists of GeoJSON (if FALSE) or WKT
-#'   strings (if TRUE), default: FALSE.
-#'   See \code{\link{odata_submission_get}} parameter \code{wkt}.
+#' @template param-wkt
 #' @template param-verbose
 #' @return The submissions tibble with all geopoints retained in their original
 #'   format, plus columns of their coordinate components as provided by
 #'   \code{\link{split_geopoint}}.
 #' @export
 #' @family utilities
 #' @examples
-#' \dontrun{
 #' library(magrittr)
-#' data("fq_raw")
-#' data("fq_form_schema")
+#' data("gep_fs")
+#' data("geo_gj_raw")
+#' data("geo_wkt_raw")
 #'
-#' fq_with_geo <- fq_raw %>%
-#'   ruODK::odata_submission_rectangle() %>%
-#'   ruODK::handle_ru_geopoints(form_schema = fq_form_schema)
+#' # GeoJSON
+#' geo_gj_parsed <- geo_gj_raw %>%
+#'   ruODK::odata_submission_rectangle(form_schema = geo_fs) %>%
+#'   ruODK::handle_ru_geopoints(form_schema = geo_fs, wkt = FALSE)
 #'
-#' dplyr::glimpse(fq_with_geo)
-#' }
+#' dplyr::glimpse(geo_gj_parsed)
+#'
+#' # WKT
+#' geo_wkt_parsed <- geo_wkt_raw %>%
+#'   ruODK::odata_submission_rectangle(form_schema = geo_fs) %>%
+#'   ruODK::handle_ru_geopoints(form_schema = geo_fs, wkt = TRUE)
+#'
+#' dplyr::glimpse(geo_wkt_parsed)
 handle_ru_geopoints <- function(data,
                                 form_schema,
                                 wkt = FALSE,
                                 verbose = get_ru_verbose()) {
   # Find Geopoint columns
-  gp_cols <- form_schema %>%
+  geo_cols <- form_schema %>%
     dplyr::filter(type == "geopoint") %>%
     magrittr::extract2("ruodk_name") %>%
     intersect(names(data))
 
   if (verbose == TRUE) {
-    x <- paste(gp_cols, collapse = ", ")
+    x <- paste(geo_cols, collapse = ", ")
     ru_msg_info(glue::glue("Found geopoints: {x}."))
   }
 
-  for (colname in gp_cols) {
+  for (colname in geo_cols) {
     if (colname %in% names(data)) {
       if (verbose == TRUE) ru_msg_info(glue::glue("Parsing {colname}..."))
       data <- data %>% split_geopoint(as.character(colname), wkt = wkt)

R/handle_ru_geotraces.R

@@ -0,0 +1,72 @@
+#' Split all geotraces of a submission tibble into their components.
+#'
+#' \lifecycle{stable}
+#'
+#' @details For a given tibble of submissions, find all columns which are listed
+#' in the form schema as type \code{geotrace}, and extract their components.
+#' Extracted components are longitude (X), latitude (Y), altitude (Z, where
+#' given), and accuracy (M, where given) of the first point of the geotrace.
+#'
+#' The original column is retained to allow parsing into other spatially
+#' enabled formats.
+#' @param data Submissions rectangled into a tibble. E.g. the output of
+#'   ```
+#'   ruODK::odata_submission_get(parse = FALSE) %>%
+#'   ruODK::odata_submission_rectangle(form_schema = ...)
+#'   ```
+#' @param form_schema The `form_schema` for the submissions.
+#'   E.g. the output of `ruODK::form_schema()`.
+#' @template param-wkt
+#' @template param-verbose
+#' @return The submissions tibble with all geotraces retained in their original
+#'   format, plus columns of their first point's coordinate components as
+#'   provided by \code{\link{split_geotrace}}.
+#' @export
+#' @family utilities
+#' @examples
+#' \dontrun{
+#' library(magrittr)
+#' data("gep_fs")
+#' data("geo_wkt_raw")
+#' data("geo_gj_raw")
+#'
+#' # GeoJSON
+#' geo_gj_parsed <- geo_gj_raw %>%
+#'   ruODK::odata_submission_rectangle(form_schema = geo_fs) %>%
+#'   ruODK::handle_ru_geotraces(form_schema = geo_fs, wkt = FALSE)
+#'
+#' dplyr::glimpse(geo_gj_parsed)
+#'
+#' # WKT
+#' geo_wkt_parsed <- geo_wkt_raw %>%
+#'   ruODK::odata_submission_rectangle(form_schema = geo_fs) %>%
+#'   ruODK::handle_ru_geotraces(form_schema = geo_fs, wkt = TRUE)
+#'
+#' dplyr::glimpse(geo_wkt_parsed)
+#' }
+handle_ru_geotraces <- function(data,
+                                form_schema,
+                                wkt = FALSE,
+                                verbose = get_ru_verbose()) {
+  # Find Geotrace columns
+  geo_cols <- form_schema %>%
+    dplyr::filter(type == "geotrace") %>%
+    magrittr::extract2("ruodk_name") %>%
+    intersect(names(data))
+
+  if (verbose == TRUE) {
+    x <- paste(geo_cols, collapse = ", ")
+    ru_msg_info(glue::glue("Found geotraces: {x}."))
+  }
+
+  for (colname in geo_cols) {
+    if (colname %in% names(data)) {
+      if (verbose == TRUE) ru_msg_info(glue::glue("Parsing {colname}..."))
+      data <- data %>% split_geotrace(as.character(colname), wkt = wkt)
+    }
+  }
+
+  data
+}
+
+# usethis::use_test("handle_ru_geotraces")

R/odata_submission_get.R

@@ -61,9 +61,6 @@
 #' @param wkt If TRUE, geospatial data will be returned as WKT (Well Known Text)
 #'   strings. Default: FALSE, returns GeoJSON structures.
 #'   Note that accuracy is only returned through GeoJSON.
-#'   \code{\link{handle_ru_geopoints}} parses `geopoint` WKT into
-#'   longitude, latitude, and altitude, prefixed by the original field name to
-#'   avoid naming conflicts between possibly multiple geopoints.
 #' @param parse Whether to parse submission data based on form schema.
 #'   Dates and datetimes will be parsed into local time.
 #'   Attachments will be downloaded, and the field updated to the local file
@@ -223,19 +220,11 @@ odata_submission_get <- function(table = "Submissions",
     odkc_version = odkc_version
   )
 
-  # WKT seatbelt
-  # if (wkt == FALSE &&
-  #     parse == TRUE &&
-  #     ("geotrace" %in% unique(fs$type) |
-  #      "geoshape" %in% unique(fs$type))) {
-  #   ru_msg_warn("Form has geotrace or geoshape, use either wkt=T or parse=F")
-  # }
-
   #----------------------------------------------------------------------------#
   # Parse submission data
   if (verbose == TRUE) ru_msg_info("Parsing submissions...")
 
-  # Rectangle, handle date/times, attachments, geopoints.
+  # Rectangle, handle date/times, attachments, geopoints, geotraces, geoshapes.
   sub <- sub %>%
     odata_submission_rectangle(form_schema = fs, verbose = verbose) %>%
     handle_ru_datetimes(form_schema = fs, verbose = verbose) %>%
@@ -257,16 +246,17 @@ odata_submission_get <- function(table = "Submissions",
         .
       }
     } %>%
-    # input can be geojson or wkt, tell handlers through wkt=wkt
-    # keep original and add (name suffixed) lon/lat/alt/acc, sfg st point
-    handle_ru_geopoints(form_schema = fs, wkt = wkt, verbose = verbose)
-
-    # handle_ru_geotraces(data = ., form_schema = fs, wkt = wkt, verbose = verbose)
-    # keep original and add (name suffixed) first point lon/lat/alt/acc, sfg, st line
-
-    # handle_ru_geoshapes(data = ., form_schema = fs, wkt = wkt, verbose = verbose)
-    # keep original and add (name suffixed) first point lon/lat/alt/acc, sfg, st poly
-
+    handle_ru_geopoints(
+      form_schema = fs,
+      wkt = wkt,
+      verbose = verbose
+    ) %>%
+    handle_ru_geotraces(
+      form_schema = fs,
+      wkt = wkt,
+      verbose = verbose
+    )
+    # %>% handle_ru_geoshapes(form_schema = fs, wkt = wkt, verbose = verbose)
 
   #
   # End parse submission data

R/ruODK.R

@@ -6,7 +6,6 @@
 #' `ruODK` is "pipe-friendly" and re-exports `\%>\%` and `\%||\%`, but does not
 #'  require their use.
 #'
-#' @importFrom rlang %||%
 #' @keywords internal
 "_PACKAGE"
 

R/split_geopoint.R

@@ -5,34 +5,62 @@
 #' @details This function is used by \code{\link{handle_ru_geopoints}}
 #' on all \code{geopoint} fields as per \code{\link{form_schema}}.
 #'
-#' @param data (dataframe) A dataframe with a column of type WKT POINT
-#' @param colname (chr) The name of the WKT POINT column.
+#' @param data (dataframe) A dataframe with a geopoint column.
+#' @param colname (chr) The name of the geopoint column.
 #'   This column will be retained.
+#' @template param-wkt
 #' @return The given dataframe with the WKT POINT column <colname>, plus
 #'   three new columns, `<colname>_longitude`, `<colname>_latitude`,
 #'   `<colname>_altitude`.
 #'   The three new columns are prefixed with the original `colname` to avoid
 #'   naming conflicts with any other geopoint columns.
+#'
 #' @export
 #' @family utilities
 #' @examples
-#' df <- tibble::tibble(
+#' \dontrun{
+#' df_wkt <- tibble::tibble(
 #'   stuff = c("asd", "sdf", "sdf"),
 #'   loc = c(
 #'     "POINT (115.99 -32.12 20.01)",
 #'     "POINT (116.12 -33.34 15.23)",
-#'     "POINT (114.00 -31.56 23.56)"
+#'     "POINT (114.01 -31.56 23.56)"
 #'   )
 #' )
-#' df_split <- df %>% split_geopoint("loc", wkt=TRUE)
+#' df_wkt_split <- df %>% split_geopoint("loc", wkt=TRUE)
 #' testthat::expect_equal(
-#'   names(df_split),
+#'   names(df_wkt_split),
 #'   c("stuff", "loc", "loc_longitude", "loc_latitude", "loc_altitude")
 #' )
+#'
+#' # With package data
+#' data("gep_fs")
+#' data("geo_wkt_raw")
+#' data("geo_gj_raw")
+#'
+#' # Find variable names of geopoints
+#' geo_fields <- geo_fs %>%
+#'   dplyr::filter(type  == "geopoint") %>%
+#'   magrittr::extract2("ruodk_name")
+#' geo_fields[1] # First geotrace in data: point_location_point_gps
+#'
+#' # Rectangle but don't parse submission data (GeoJSON and WKT)
+#' geo_gj_rt <- geo_gj_raw %>%
+#'   odata_submission_rectangle(form_schema = geo_fs)
+#' geo_wkt_rt <- geo_wkt_raw %>%
+#'   odata_submission_rectangle(form_schema = geo_fs)
+#'
+#' # Data with first geopoint split
+#' gj_first_gt <- split_geopoint(geo_gj_rt, geo_fields[1], wkt = FALSE)
+#' gj_first_gt$point_location_point_gps_longitude
+#'
+#' wkt_first_gt <- split_geopoint(geo_wkt_rt, geo_fields[1], wkt = TRUE)
+#' wkt_first_gt$point_location_point_gps_longitude
+#' }
 split_geopoint <- function(data, colname, wkt = FALSE) {
   if (wkt == FALSE) {
     # GeoJSON
-    # Extract coords into programmatically generated variable names
+    # Task: Extract coordinates into programmatically generated variable names
     # Step 1: tidyr::hoist() extracts but can't assign with :=
     data %>%
       tidyr::hoist(
@@ -43,7 +71,7 @@ split_geopoint <- function(data, colname, wkt = FALSE) {
         XXX_accuracy = list("properties", "accuracy"),
         .remove = FALSE
       ) %>%
-      # Step 2: dplyr::mutate_at() can programmatically manipulate vars
+      # Step 2: dplyr::mutate_at() can programmatically manipulate variables
       dplyr::rename_at(
         dplyr::vars(dplyr::starts_with("XXX")),
         list( ~ stringr::str_replace(., "XXX", colname))