streamlining

.Rbuildignore

@@ -92,6 +92,8 @@ vignettes/join_by_closest.Rmd
 vignettes/wqx3_development_plan.Rmd
 vignettes/read_waterdata_functions.Rmd
 vignettes/dataretrieval_discrete_changes_images/*
+vignettes/Wide data example.xlsx
+vignettes/messyData.png
 ^ci$
 ^public$
 ^docker$

R/dataRetrievals-package.R

@@ -9,7 +9,7 @@ See: https://api.waterdata.usgs.gov/signup"
   packageStartupMessage("dataRetrieval ", dataRetrieval_version,"
 Extended Documentation: https://doi-usgs.github.io/dataRetrieval
 Learn about the new functions that are replacing NWIS functions here:
-https://water.code-pages.usgs.gov/dataRetrieval/articles/read_waterdata_functions.html",
+https://doi-usgs.github.io/dataRetrieval/articles/read_waterdata_functions.html",
 token_message)
 }
 

R/read_waterdata_daily.R

@@ -88,29 +88,10 @@ read_waterdata_daily <- function(monitoring_location_id = NA_character_,
   output_id <- "daily_id"
 
   args <- mget(names(formals()))
-  args[["service"]] <-  service
+  return_list <- get_ogc_data(args,
+                              output_id, 
+                              service)
 
-  args <- switch_arg_id(args, 
-                        id_name = output_id, 
-                        service = service)
-
-  args[["properties"]] <- switch_properties_id(properties, 
-                                               id_name = output_id, 
-                                               service = service)
-
-  args[["convertType"]] <- NULL
-
-  dv_req <- do.call(construct_api_requests, args)
-
-  return_list <- walk_pages(dv_req, max_results)
-
-  return_list <- deal_with_empty(return_list, properties, service)
-
-  if(convertType) return_list <- cleanup_cols(return_list,
-                                              service = "daily")
-
-  return_list <- rejigger_cols(return_list, properties, output_id)
-
   return_list <- return_list[order(return_list$time, return_list$monitoring_location_id), ]
 
   return(return_list)

R/read_waterdata_monitoring_location.R

@@ -139,23 +139,10 @@ read_waterdata_monitoring_location <- function(monitoring_location_id = NA_chara
   output_id <- "monitoring_location_id"
 
   args <- mget(names(formals()))
-  args[["service"]] <-  service
-
-  args <- switch_arg_id(args, 
-                        id_name = output_id, 
-                        service = service)
-
-  args[["properties"]] <- switch_properties_id(properties, 
-                                               id_name = output_id, 
-                                               service = service)
-
-  site_req <- do.call(construct_api_requests, args)
-
-  return_list <- walk_pages(site_req, max_results)
-
-  return_list <- deal_with_empty(return_list, properties, service)
-
-  return_list <- rejigger_cols(return_list, properties, output_id)
+  args[["convertType"]] <- FALSE
+  return_list <- get_ogc_data(args,
+                              output_id, 
+                              service)
 
   return(return_list)
 }

R/read_waterdata_samples.R

@@ -339,21 +339,28 @@ explode_query <- function(baseURL, POST = FALSE, x){
 #'                                group = "Biological")
 #' observedProperties <- check_waterdata_sample_params("observedproperty",
 #'                                   text = "phosphorus")
-#' 
+#' ref_list <- check_waterdata_sample_params("reference-list")
 #' }
 check_waterdata_sample_params <- function(service = "characteristicgroup",
                                      ...){
 
   service_options <- c("characteristicgroup", "states", "counties",
                        "countries", "sitetype", "samplemedia",
-                       "characteristics", "observedproperty")
+                       "characteristics", "observedproperty",
+                       "reference-list")
 
   match.arg(service, choices = service_options, several.ok = FALSE)
 
   check_group_req <- httr2::request("https://api.waterdata.usgs.gov") |> 
-    httr2::req_url_path_append("samples-data",
-                               "codeservice",
-                               service) |> 
+    httr2::req_url_path_append("samples-data") 
+
+  if(service != "reference-list"){
+    check_group_req <- check_group_req |> 
+      httr2::req_url_path_append("codeservice")
+  }
+
+  check_group_req <- check_group_req |>
+    httr2::req_url_path_append(service) |> 
     httr2::req_user_agent(default_ua()) |> 
     httr2::req_url_query(mimeType = "application/json") 
 
@@ -365,7 +372,7 @@ check_waterdata_sample_params <- function(service = "characteristicgroup",
     check_group_req <- httr2::req_url_query(check_group_req,
                                             !!!params)
   }
-  
+
   message("GET: ", check_group_req$url) 
 
   check_group <- httr2::req_perform(check_group_req) |> 
@@ -506,7 +513,7 @@ summarize_waterdata_samples <- function(monitoringLocationIdentifier){
     df$firstActivity <- as.Date(df$firstActivity)
     df$mostRecentActivity <- as.Date(df$mostRecentActivity)
   }
-  
+
   attr(df, "url") <- baseURL$url
   attr(df, "queryTime") <- Sys.time()
 

R/read_waterdata_ts_meta.R

@@ -81,25 +81,9 @@ read_waterdata_ts_meta <- function(monitoring_location_id = NA_character_,
   output_id <- "time_series_id"
 
   args <- mget(names(formals()))
-  args[["service"]] <-  service
-
-  args <- switch_arg_id(args, id_name = output_id, service = service)
-
-  args[["convertType"]] <- NULL
-
-  args[["properties"]] <- switch_properties_id(properties, 
-                                               id_name = output_id, 
-                                               service = service)
-
-  req_ts_meta <- do.call(construct_api_requests, args)
-
-  return_list <- walk_pages(req_ts_meta, max_results)
-
-  return_list <- deal_with_empty(return_list, properties, service)
-
-  if(convertType) return_list <- cleanup_cols(return_list)
-
-  return_list <- rejigger_cols(return_list, properties, output_id)
+  return_list <- get_ogc_data(args,
+                              output_id, 
+                              service)
 
   return(return_list)
 

R/walk_pages.R

@@ -19,6 +19,7 @@
 #' 
 deal_with_empty <- function(return_list, properties, service){
   if(nrow(return_list) == 0){
+
     if(all(is.na(properties))){
       schema <- check_OGC_requests(endpoint = service, type = "schema")
       properties <- names(schema$properties)
@@ -213,6 +214,8 @@ get_resp_data <- function(resp) {
 #' @return data.frame with attributes
 walk_pages <- function(req, max_results){
 
+  message("Requesting:\n", req$url)
+
   if(is.na(max_results)){
     resps <- httr2::req_perform_iterative(req, 
                                           next_req = next_req_url, 
@@ -240,8 +243,50 @@ walk_pages <- function(req, max_results){
     return_list <- get_resp_data(resps)
   }
 
+  return(return_list)
+}
+
+
+#' Coordinate the request and retrieval of OGC calls
+#' 
+#' @param args arguments from individual functions
+#' @param output_id Name of id column to return
+#' @param service Endpoint name.
+#' @param max_results
+#' 
+#' @noRd
+#' @return data.frame with attributes
+get_ogc_data <- function(args,
+                         output_id, 
+                         service){
+
+  args[["service"]] <-  service
+  max_results <- args[["max_results"]]
+  args[["max_results"]] <- NULL
+  args <- switch_arg_id(args, 
+                        id_name = output_id, 
+                        service = service)
+
+  properties <- args[["properties"]]
+  args[["properties"]] <- switch_properties_id(properties, 
+                                               id_name = output_id, 
+                                               service = service)
+  convertType <- args[["convertType"]] 
+  args[["convertType"]] <- NULL
+
+  req <- do.call(construct_api_requests, args)
+
+  return_list <- walk_pages(req, max_results)
+
+  return_list <- deal_with_empty(return_list, properties, service)
+
+  if(convertType) return_list <- cleanup_cols(return_list, service = service)
+
+  return_list <- rejigger_cols(return_list, properties, output_id)
+
   attr(return_list, "request") <- req
   attr(return_list, "queryTime") <- Sys.time()
-
   return(return_list)
 }
+
+

code.json

@@ -89,5 +89,51 @@
     "date": {
       "metadataLastUpdated": "2022-12-06"
     }
+  },
+  {
+    "name": "dataRetrieval",
+    "organization": "U.S. Geological Survey",
+    "description": "This R package is designed to obtain USGS or EPA water quality sample data, streamflow data, and metadata directly from web services",
+    "version": "2.7.19",
+    "status": "Development",
+
+    "permissions": {
+      "usageType": "openSource",
+      "licenses": [
+        {
+          "name": "Public Domain, CC0-1.0",
+          "URL": "https://code.usgs.gov/water/dataRetrieval/-/raw/2.7.19/LICENSE.md"
+        }
+      ]
+    },
+
+    "homepageURL": "https://code.usgs.gov/water/dataRetrieval/-/tags/2.7.19",
+    "downloadURL": "https://code.usgs.gov/water/dataRetrieval/-/archive/2.7.19/dataRetrieval-2.7.19.zip",
+    "disclaimerURL": "https://code.usgs.gov/water/dataRetrieval/-/raw/2.7.19/DISCLAIMER.md",
+    "repositoryURL": "https://code.usgs.gov/water/dataRetrieval.git",
+
+    "vcs": "git",
+
+    "laborHours": 500,
+
+    "tags": [
+      "R",
+      "rstats",
+	  "USGS",
+	  "water"
+    ],
+
+    "languages": [
+      "R"
+    ],
+
+    "contact": {
+      "name": "Laura De Cicco",
+      "email": "ldecicco@usgs.gov"
+    },
+
+    "date": {
+      "metadataLastUpdated": "2025-06-17"
+    }
   }
 ]