From ce7800352b1b467d80a9178a65e06371b4cebbc0 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 16 Apr 2025 14:17:58 -0500 Subject: [PATCH 001/117] Beginning work on new API services --- DESCRIPTION | 4 +- NAMESPACE | 3 + R/read_USGS_dv.R | 356 +++++++++++++++++++++++++++++++++++ R/read_USGS_samples.R | 16 +- man/check_OGC_requests.Rd | 31 +++ man/construct_dv_requests.Rd | 44 +++++ man/read_USGS_dv.Rd | 55 ++++++ 7 files changed, 502 insertions(+), 7 deletions(-) create mode 100644 R/read_USGS_dv.R create mode 100644 man/check_OGC_requests.Rd create mode 100644 man/construct_dv_requests.Rd create mode 100644 man/read_USGS_dv.Rd diff --git a/DESCRIPTION b/DESCRIPTION index 52258002..e5c2da35 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -29,9 +29,7 @@ Authors@R: c( comment=c(ORCID = "0000-0002-9775-6861"))) Description: Collection of functions to help retrieve U.S. Geological Survey and U.S. Environmental Protection Agency water quality and - hydrology data from web services. Data are discovered from - National Water Information System and . - Water quality data are obtained from the Water Quality Portal . + hydrology data from web services. License: CC0 Copyright: This software is in the public domain because it contains materials that originally came from the United States Geological Survey, an agency of diff --git a/NAMESPACE b/NAMESPACE index d3130088..990b0777 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -3,11 +3,13 @@ export(addWaterYear) export(calcWaterYear) export(checkWQPdates) +export(check_OGC_requests) export(check_param) export(constructNWISURL) export(constructUseURL) export(constructWQPURL) export(construct_USGS_sample_request) +export(construct_dv_requests) export(countyCd) export(countyCdLookup) export(create_NWIS_bib) @@ -42,6 +44,7 @@ export(readNWISuv) export(readWQPdata) export(readWQPqw) export(readWQPsummary) +export(read_USGS_dv) export(read_USGS_samples) export(renameNWISColumns) export(setAccess) diff --git a/R/read_USGS_dv.R b/R/read_USGS_dv.R new file mode 100644 index 00000000..a474aea5 --- /dev/null +++ b/R/read_USGS_dv.R @@ -0,0 +1,356 @@ +#' Get USGS Daily Data +#' +#' Daily data provide one data value to represent water conditions for the day. +#' Throughout much of the history of the USGS, the primary water data available +#' was daily data collected manually at the monitoring location once each day. +#' With improved availability of computer storage and automated transmission of +#' data, the daily data published today are generally a statistical summary or +#' metric of the continuous data collected each day, such as the daily mean, +#' minimum, or maximum value. Daily data are automatically calculated from the +#' continuous data of the same parameter code and are described by parameter code +#' and a statistic code. These data have also been referred to as "daily values" +#' or "DV". +#' +#' @export +#' @examples +#' site <- "USGS-02238500" +#' pcode <- "00060" +#' dv_data_sf <- read_USGS_dv(monitoring_location_id = site, +#' parameter_code = "00060", limit = 1000) +#' +#' dv_data <- read_USGS_dv(monitoring_location_id = site, +#' parameter_code = "00060", no_sf = TRUE) +#' +#' sites <- c("USGS-01491000", "USGS-01645000") +#' multi_site <- read_USGS_dv(monitoring_location_id = sites, +#' parameter_code = c("00060", "00065")) +#' +#' bbox_data <- read_USGS_dv(bbox = c(-83, 36.5, -81, 38.5), +#' parameter_code = "00060") +#' +read_USGS_dv <- function(monitoring_location_id = NA_character_, + parameter_code = NA_character_, + statistic_id = NA_character_, + properties = c("monitoring_location_id", + "parameter_code", + "statistic_id", + "time", + "value", + "unit_of_measure", + "approval_status", + "qualifier"), + bbox = NA, + timeseries_id = NA_character_, + id = NA_character_, + approval_status = NA_character_, + unit_of_measure = NA_character_, + qualifier = NA_character_, + value = NA, + last_modified = NA_character_, + limit = 10000, + crs = NA_character_, + bbox_crs = NA_character_, + skipGeometry = NA, + sortby = NA_character_, + offset = NA, + datetime = NA_character_, + filter = NA_character_, + no_sf = FALSE){ + + use_sf <- all(pkg.env$local_sf, !no_sf) + page <- 1 + + if(!use_sf){ + skipGeometry <- TRUE + } + + dv_req <- construct_dv_requests(monitoring_location_id = monitoring_location_id, + parameter_code = parameter_code, + statistic_id = statistic_id, + properties = properties, + bbox = bbox, + timeseries_id = timeseries_id, + id = id, + approval_status = approval_status, + unit_of_measure = unit_of_measure, + qualifier = qualifier, + value = value, + last_modified = last_modified, + limit = limit, + crs = crs, + bbox_crs = bbox_crs, + skipGeometry = skipGeometry, + sortby = sortby, + offset = offset, + datetime = datetime, + filter = filter) + + return_list <- walk_pages(dv_req, use_sf) + + return(return_list) +} + +cleanup_cols <- function(df){ + + if("qualifier" %in% names(df)){ + df$qualifier <- vapply(X = df$qualifier, + FUN = function(x) paste(x, collapse = ","), + FUN.VALUE = c(NA_character_)) + } + + if("time" %in% names(df)){ + df$time <- as.Date(df$time) + } + + if("value" %in% names(df)){ + df$value <- as.numeric(df$value) + } + df +} + +walk_pages <- function(req, use_sf) { + + walk_pages_recursive( + req = req, + page = 1, + contents = list(), + use_sf + ) +} + +# Change to https://httr2.r-lib.org/reference/req_perform_iterative.html +walk_pages_recursive <- function(req, page, contents, use_sf) { + + message("GET: ", req$url) + + returned_contents <- httr2::req_perform(req) + + if(httr2::resp_status(returned_contents) != 200){ + if(httr2::resp_status(returned_contents) == 429){ + stop("You hit your daily limit for requests. To increase the limit, + see: https://api.waterdata.usgs.gov/docs/ogcapi/keys/") + } + } + + header_info <- httr2::resp_headers(returned_contents) + message("Remaining requests for the day:", header_info$`x-ratelimit-remaining`) + + contents[[page]] <- returned_contents |> + httr2::resp_body_string() + + json_content <- jsonlite::fromJSON(contents[[page]] ) + + next_page <- json_content$links[, "rel", drop = TRUE] == "next" + + if (!any(next_page)) { + + if(use_sf){ + return_df <- lapply(contents, function(x) { + df_i <- sf::read_sf(x) |> + cleanup_cols() + df_i + }) |> + do.call(what = rbind) + } else { + return_df <- lapply(contents, function(x) { + df_i <- jsonlite::fromJSON(x)[["features"]][["properties"]] |> + cleanup_cols() + df_i + }) |> + do.call(what = rbind) + } + + return(return_df) + + } else { + + Tailcall( + walk_pages_recursive, + httr2::request(json_content$links[, "href", drop = TRUE][next_page]) |> + httr2::req_user_agent(default_ua()) |> + httr2::req_headers(`Accept-Encoding` = c("compress", "gzip"), + `X-Api-Key` = Sys.getenv("API_USGS_PAT")) , + page + 1, + contents, + use_sf + ) + } +} + +#' Create DV url +#' +#' Main documentation: \url{https://api.waterdata.usgs.gov/ogcapi/v0/}, +#' Swagger docs: \url{https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html}. +#' +#' @export +#' @param monitoring_location_id description +#' @examples +#' site <- "USGS-02238500" +#' pcode <- "00060" +#' req_dv <- construct_dv_requests(monitoring_location_id = site, +#' parameter_code = "00060") +#' +#' req_dv <- construct_dv_requests(monitoring_location_id = site, +#' parameter_code = c("00060", "00065")) +#' +#' sites <- c("USGS-01491000", "USGS-01645000") +#' req_dv <- construct_dv_requests(monitoring_location_id = sites, +#' parameter_code = c("00060", "00065")) +#' +construct_dv_requests <- function(monitoring_location_id = NA_character_, + parameter_code = NA_character_, + statistic_id = NA_character_, + properties = c("monitoring_location_id", + "parameter_code", + "statistic_id", + "time", + "value", + "unit_of_measure", + "approval_status", + "qualifier"), + bbox = NA, + timeseries_id = NA_character_, + id = NA_character_, + approval_status = NA_character_, + unit_of_measure = NA_character_, + qualifier = NA_character_, + value = NA, + last_modified = NA_character_, + limit = 10000, + crs = NA_character_, + bbox_crs = NA_character_, + skipGeometry = FALSE, + sortby = "time", + offset = NA, + datetime = NA_character_, + filter = NA_character_ + ){ + + match.arg(properties, choices = c("id", + "timeseries_id", + "monitoring_location_id", + "parameter_code", + "statistic_id", + "time", + "value", + "unit_of_measure", + "approval_status", + "qualifier", + "last_modified", NA_character_), + several.ok = TRUE) + + baseURL <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> + httr2::req_url_path_append("daily", "items") |> + httr2::req_user_agent(default_ua()) + + token <- Sys.getenv("API_USGS_PAT") + + if(isTRUE(length(monitoring_location_id) > 1)){ + filter <- c(filter[!is.na(filter)], + paste0("monitoring_location_id IN ('", + paste0(monitoring_location_id, + collapse = "', '"), "')")) + monitoring_location_id <- NA_character_ + } + + if(isTRUE(length(parameter_code) > 1)){ + filter <- c(filter[!is.na(filter)], + paste0("parameter_code IN ('", + paste0(parameter_code, + collapse = "', '"), "')")) + parameter_code <- NA_character_ + } + + POST <- FALSE + + baseURL <- explode_query(baseURL, POST = POST, + list(monitoring_location_id = monitoring_location_id, + parameter_code = parameter_code, + statistic_id = statistic_id, + timeseries_id = timeseries_id, + id = id, + approval_status = approval_status, + unit_of_measure = unit_of_measure, + qualifier = qualifier, + value = value, + last_modified = last_modified, + limit = limit, + crs = crs, + bbox_crs = bbox_crs, + skipGeometry = skipGeometry, + sortby = sortby, + offset = offset, + datetime = datetime, + filter = filter, + f = "json", + lang = "en-US")) + + if(all(!is.na(bbox))){ + browser() + baseURL <- httr2::req_body_json_modify(baseURL, + bbox = bbox, + .multi = "comma") + } + + if(POST){ + baseURL <- httr2::req_body_json_modify(baseURL, + properties = properties) + } else { + baseURL <- httr2::req_url_query(baseURL, + properties = properties, + .multi = "comma") + } + + + if(token != ""){ + baseURL <- baseURL |> + httr2::req_headers(`X-Api-Key` = token, + `Accept-Encoding` = c("compress", "gzip")) + } else { + baseURL <- baseURL |> + httr2::req_headers(`Accept-Encoding` = c("compress", "gzip")) + } + + return(baseURL) +} + + +#' Check OGC requests +#' +#' @param endpoint Character, can be "daily", "timeseries-metadata" +#' @param type Character, can be "queryables", "schema" +#' @export +#' @return list +#' @examples +#' +#' dv_queryables <- check_OGC_requests(endpoint = "daily", +#' type = "queryables") +#' dv_schema <- check_OGC_requests(endpoint = "daily", +#' type = "schema") +#' ts_meta_queryables <- check_OGC_requests(endpoint = "timeseries-metadata", +#' type = "queryables") +#' ts_meta_schema <- check_OGC_requests(endpoint = "timeseries-metadata", +#' type = "schema") +#' +check_OGC_requests <- function(endpoint = "daily", + type = "queryables"){ + + match.arg(endpoint, c("daily", "timeseries-metadata")) + match.arg(type, c("queryables", "schema")) + + check_req <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> + httr2::req_url_path_append(endpoint) |> + httr2::req_url_path_append(type) |> + httr2::req_user_agent(default_ua()) |> + httr2::req_url_query(f = "json", + lang = "en-US") + + message("GET: ", check_req$url) + + query_ret <- httr2::req_perform(check_req) |> + httr2::resp_body_string() |> + jsonlite::fromJSON() + + return(query_ret) + +} \ No newline at end of file diff --git a/R/read_USGS_samples.R b/R/read_USGS_samples.R index acac89ab..3a3f4b53 100644 --- a/R/read_USGS_samples.R +++ b/R/read_USGS_samples.R @@ -292,7 +292,7 @@ check_profile <- function(dataProfile, profile_convert){ return(dataProfile) } -explode_query <- function(baseURL, x){ +explode_query <- function(baseURL, POST = FALSE, x){ if(!is.list(x)){ return(baseURL) @@ -300,9 +300,15 @@ explode_query <- function(baseURL, x){ if(any(!is.na(x))){ x <- Filter(Negate(anyNA), x) - baseURL <- httr2::req_url_query(baseURL, - !!!x, - .multi = "explode") + if(POST){ + baseURL <- httr2::req_body_json(req = baseURL, + data = x) + } else { + baseURL <- httr2::req_url_query(baseURL, + !!!x, + .multi = "explode") + } + } return(baseURL) } @@ -359,6 +365,8 @@ check_param <- function(service = "characteristicgroup", !!!params) } + message("GET: ", check_group_req$url) + check_group <- httr2::req_perform(check_group_req) |> httr2::resp_body_string() |> jsonlite::fromJSON() diff --git a/man/check_OGC_requests.Rd b/man/check_OGC_requests.Rd new file mode 100644 index 00000000..72ca6720 --- /dev/null +++ b/man/check_OGC_requests.Rd @@ -0,0 +1,31 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/read_USGS_dv.R +\name{check_OGC_requests} +\alias{check_OGC_requests} +\title{Check OGC requests} +\usage{ +check_OGC_requests(endpoint = "daily", type = "queryables") +} +\arguments{ +\item{endpoint}{Character, can be "daily", "timeseries-metadata"} + +\item{type}{Character, can be "queryables", "schema"} +} +\value{ +list +} +\description{ +Check OGC requests +} +\examples{ + +dv_queryables <- check_OGC_requests(endpoint = "daily", + type = "queryables") +dv_schema <- check_OGC_requests(endpoint = "daily", + type = "schema") +ts_meta_queryables <- check_OGC_requests(endpoint = "timeseries-metadata", + type = "queryables") +ts_meta_schema <- check_OGC_requests(endpoint = "timeseries-metadata", + type = "schema") + +} diff --git a/man/construct_dv_requests.Rd b/man/construct_dv_requests.Rd new file mode 100644 index 00000000..aaaa94d0 --- /dev/null +++ b/man/construct_dv_requests.Rd @@ -0,0 +1,44 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/read_USGS_dv.R +\name{construct_dv_requests} +\alias{construct_dv_requests} +\title{Create DV url} +\usage{ +construct_dv_requests( + monitoring_location_id = NA_character_, + parameter_code = NA_character_, + statistic_id = NA_character_, + properties = c("id", "timeseries_id", "monitoring_location_id", "parameter_code", + "statistic_id", "time", "value", "unit_of_measure", "approval_status", "qualifier", + "last_modified"), + bbox = NA, + timeseries_id = NA_character_, + id = NA_character_, + approval_status = NA_character_, + unit_of_measure = NA_character_, + qualifier = NA_character_, + value = NA, + last_modified = NA_character_, + limit = 10000, + crs = NA_character_, + bbox_crs = NA_character_, + skipGeometry = FALSE, + sortby = NA_character_, + offset = NA, + datetime = NA_character_ +) +} +\arguments{ +\item{monitoring_location_id}{description} +} +\description{ +Main documentation: \url{https://api.waterdata.usgs.gov/ogcapi/v0/}, +Swagger docs: \url{https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html}. +} +\examples{ +site <- "USGS-02238500" +pcode <- "00060" +req_dv <- construct_dv_requests(monitoring_location_id = site, + parameter_code = "00060") + +} diff --git a/man/read_USGS_dv.Rd b/man/read_USGS_dv.Rd new file mode 100644 index 00000000..bcc41eed --- /dev/null +++ b/man/read_USGS_dv.Rd @@ -0,0 +1,55 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/read_USGS_dv.R +\name{read_USGS_dv} +\alias{read_USGS_dv} +\title{Get USGS Daily Data} +\usage{ +read_USGS_dv( + monitoring_location_id = NA_character_, + parameter_code = NA_character_, + statistic_id = NA_character_, + properties = c("monitoring_location_id", "parameter_code", "statistic_id", "time", + "value", "unit_of_measure", "approval_status", "qualifier"), + bbox = NA, + timeseries_id = NA_character_, + id = NA_character_, + approval_status = NA_character_, + unit_of_measure = NA_character_, + qualifier = NA_character_, + value = NA, + last_modified = NA_character_, + limit = 10000, + crs = NA_character_, + bbox_crs = NA_character_, + skipGeometry = NA, + sortby = NA_character_, + offset = NA, + datetime = NA_character_, + no_sf = FALSE +) +} +\description{ +Daily data provide one data value to represent water conditions for the day. +Throughout much of the history of the USGS, the primary water data available +was daily data collected manually at the monitoring location once each day. +With improved availability of computer storage and automated transmission of +data, the daily data published today are generally a statistical summary or +metric of the continuous data collected each day, such as the daily mean, +minimum, or maximum value. Daily data are automatically calculated from the +continuous data of the same parameter code and are described by parameter code +and a statistic code. These data have also been referred to as "daily values" +or "DV". +} +\examples{ +site <- "USGS-02238500" +pcode <- "00060" +dv_data_sf <- read_USGS_dv(monitoring_location_id = site, + parameter_code = "00060", limit = 1000) + +dv_data <- read_USGS_dv(monitoring_location_id = site, + parameter_code = "00060", no_sf = TRUE) + +bbox_data <- read_USGS_dv(bbox = c(-83, 36.5, -81, 38.5), + parameter_code = "00060") + +} From 65382a408899298b5882a8971c98c576e73aeb8c Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 17 Apr 2025 09:05:21 -0500 Subject: [PATCH 002/117] change messages --- R/read_USGS_dv.R | 63 ++++++++++++++++++++++++++++-------- man/construct_dv_requests.Rd | 20 +++++++++--- man/read_USGS_dv.Rd | 11 ++++++- 3 files changed, 74 insertions(+), 20 deletions(-) diff --git a/R/read_USGS_dv.R b/R/read_USGS_dv.R index a474aea5..ac278cb0 100644 --- a/R/read_USGS_dv.R +++ b/R/read_USGS_dv.R @@ -16,14 +16,18 @@ #' site <- "USGS-02238500" #' pcode <- "00060" #' dv_data_sf <- read_USGS_dv(monitoring_location_id = site, -#' parameter_code = "00060", limit = 1000) +#' parameter_code = "00060", +#' datetime = c("2021-01-01", "2022-01-01")) #' #' dv_data <- read_USGS_dv(monitoring_location_id = site, #' parameter_code = "00060", no_sf = TRUE) #' #' sites <- c("USGS-01491000", "USGS-01645000") -#' multi_site <- read_USGS_dv(monitoring_location_id = sites, -#' parameter_code = c("00060", "00065")) +#' start_date <- "2021-01-01" +#' end_date <- "2022-01-01" +#' req_dv <- read_USGS_dv(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), +#' parameter_code = c("00060", "00065"), +#' datetime = c("2021-01-01", "2022-01-01")) #' #' bbox_data <- read_USGS_dv(bbox = c(-83, 36.5, -81, 38.5), #' parameter_code = "00060") @@ -57,6 +61,8 @@ read_USGS_dv <- function(monitoring_location_id = NA_character_, filter = NA_character_, no_sf = FALSE){ + message("Function in development, use at your own risk.") + use_sf <- all(pkg.env$local_sf, !no_sf) page <- 1 @@ -127,13 +133,13 @@ walk_pages_recursive <- function(req, page, contents, use_sf) { if(httr2::resp_status(returned_contents) != 200){ if(httr2::resp_status(returned_contents) == 429){ - stop("You hit your daily limit for requests. To increase the limit, + stop("You hit your hourly limit for requests. To increase the limit, see: https://api.waterdata.usgs.gov/docs/ogcapi/keys/") } } header_info <- httr2::resp_headers(returned_contents) - message("Remaining requests for the day:", header_info$`x-ratelimit-remaining`) + message("Remaining requests this hour:", header_info$`x-ratelimit-remaining`) contents[[page]] <- returned_contents |> httr2::resp_body_string() @@ -164,12 +170,21 @@ walk_pages_recursive <- function(req, page, contents, use_sf) { } else { + make_request <- httr2::request(json_content$links[, "href", drop = TRUE][next_page]) |> + httr2::req_user_agent(default_ua()) + + if( Sys.getenv("API_USGS_PAT") != ""){ + make_request <- make_request |> + httr2::req_headers(`Accept-Encoding` = c("compress", "gzip"), + `X-Api-Key` = Sys.getenv("API_USGS_PAT")) + } else { + make_request <- make_request |> + httr2::req_headers(`Accept-Encoding` = c("compress", "gzip")) + } + Tailcall( walk_pages_recursive, - httr2::request(json_content$links[, "href", drop = TRUE][next_page]) |> - httr2::req_user_agent(default_ua()) |> - httr2::req_headers(`Accept-Encoding` = c("compress", "gzip"), - `X-Api-Key` = Sys.getenv("API_USGS_PAT")) , + make_request, page + 1, contents, use_sf @@ -194,8 +209,11 @@ walk_pages_recursive <- function(req, page, contents, use_sf) { #' parameter_code = c("00060", "00065")) #' #' sites <- c("USGS-01491000", "USGS-01645000") +#' start_date <- "2018-01-01" +#' end_date <- "2022-01-01" #' req_dv <- construct_dv_requests(monitoring_location_id = sites, -#' parameter_code = c("00060", "00065")) +#' parameter_code = c("00060", "00065"), +#' datetime = c(start_date, end_date)) #' construct_dv_requests <- function(monitoring_location_id = NA_character_, parameter_code = NA_character_, @@ -245,6 +263,8 @@ construct_dv_requests <- function(monitoring_location_id = NA_character_, token <- Sys.getenv("API_USGS_PAT") + ########################################################### + # This is all going to be done eventually with a POST CQL request: if(isTRUE(length(monitoring_location_id) > 1)){ filter <- c(filter[!is.na(filter)], paste0("monitoring_location_id IN ('", @@ -260,6 +280,22 @@ construct_dv_requests <- function(monitoring_location_id = NA_character_, collapse = "', '"), "')")) parameter_code <- NA_character_ } + ########################################################### + + if(!any(is.na(datetime))){ + if(length(datetime) == 1){ + datetime <- format(datetime, format = "%Y-%m-%dT%H:%M:%SZ") + } else if (length(datetime) == 2) { + datetime <- as.POSIXct(datetime) + datetime <- paste0(vapply(datetime, FUN = function(x) { + format(x, format = "%Y-%m-%dT%H:%M:%SZ")}, + FUN.VALUE = c(NA_character_) + ), collapse = "/") + datetime <- gsub("NA", "..", datetime) + } else { + stop("datetime should only include 1-2 values") + } + } POST <- FALSE @@ -286,15 +322,14 @@ construct_dv_requests <- function(monitoring_location_id = NA_character_, lang = "en-US")) if(all(!is.na(bbox))){ - browser() - baseURL <- httr2::req_body_json_modify(baseURL, + baseURL <- httr2::req_url_query(baseURL, bbox = bbox, .multi = "comma") } if(POST){ - baseURL <- httr2::req_body_json_modify(baseURL, - properties = properties) + baseURL <- httr2::req_body_json(baseURL, + properties = properties) } else { baseURL <- httr2::req_url_query(baseURL, properties = properties, diff --git a/man/construct_dv_requests.Rd b/man/construct_dv_requests.Rd index aaaa94d0..684f339f 100644 --- a/man/construct_dv_requests.Rd +++ b/man/construct_dv_requests.Rd @@ -8,9 +8,8 @@ construct_dv_requests( monitoring_location_id = NA_character_, parameter_code = NA_character_, statistic_id = NA_character_, - properties = c("id", "timeseries_id", "monitoring_location_id", "parameter_code", - "statistic_id", "time", "value", "unit_of_measure", "approval_status", "qualifier", - "last_modified"), + properties = c("monitoring_location_id", "parameter_code", "statistic_id", "time", + "value", "unit_of_measure", "approval_status", "qualifier"), bbox = NA, timeseries_id = NA_character_, id = NA_character_, @@ -23,9 +22,10 @@ construct_dv_requests( crs = NA_character_, bbox_crs = NA_character_, skipGeometry = FALSE, - sortby = NA_character_, + sortby = "time", offset = NA, - datetime = NA_character_ + datetime = NA_character_, + filter = NA_character_ ) } \arguments{ @@ -41,4 +41,14 @@ pcode <- "00060" req_dv <- construct_dv_requests(monitoring_location_id = site, parameter_code = "00060") +req_dv <- construct_dv_requests(monitoring_location_id = site, + parameter_code = c("00060", "00065")) + +sites <- c("USGS-01491000", "USGS-01645000") +start_date <- "2018-01-01" +end_date <- "2022-01-01" +req_dv <- construct_dv_requests(monitoring_location_id = sites, + parameter_code = c("00060", "00065"), + datetime = c(start_date, end_date)) + } diff --git a/man/read_USGS_dv.Rd b/man/read_USGS_dv.Rd index bcc41eed..effd72fb 100644 --- a/man/read_USGS_dv.Rd +++ b/man/read_USGS_dv.Rd @@ -25,6 +25,7 @@ read_USGS_dv( sortby = NA_character_, offset = NA, datetime = NA_character_, + filter = NA_character_, no_sf = FALSE ) } @@ -44,11 +45,19 @@ or "DV". site <- "USGS-02238500" pcode <- "00060" dv_data_sf <- read_USGS_dv(monitoring_location_id = site, - parameter_code = "00060", limit = 1000) + parameter_code = "00060", + datetime = c("2021-01-01", "2022-01-01")) dv_data <- read_USGS_dv(monitoring_location_id = site, parameter_code = "00060", no_sf = TRUE) +sites <- c("USGS-01491000", "USGS-01645000") +start_date <- "2021-01-01" +end_date <- "2022-01-01" +req_dv <- read_USGS_dv(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), + parameter_code = c("00060", "00065"), + datetime = c("2021-01-01", "2022-01-01")) + bbox_data <- read_USGS_dv(bbox = c(-83, 36.5, -81, 38.5), parameter_code = "00060") From b8bef41ba327c8d9f25ad8e66c9fa5a6ba2c2b0f Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 17 Apr 2025 09:52:52 -0500 Subject: [PATCH 003/117] Adding parameters. --- R/read_USGS_dv.R | 95 +++++++++++++++++++++++++++++------- R/read_USGS_samples.R | 2 +- man/check_OGC_requests.Rd | 6 ++- man/construct_dv_requests.Rd | 76 ++++++++++++++++++++++++++++- man/read_USGS_dv.Rd | 94 ++++++++++++++++++++++++++++++++--- 5 files changed, 244 insertions(+), 29 deletions(-) diff --git a/R/read_USGS_dv.R b/R/read_USGS_dv.R index ac278cb0..3b42c095 100644 --- a/R/read_USGS_dv.R +++ b/R/read_USGS_dv.R @@ -12,7 +12,12 @@ #' or "DV". #' #' @export -#' @examples +#' @inheritParams construct_dv_requests +#' @param no_sf Boolean, whether or not to return an "sf" object. TRUE returns +#' a basic data frame, FALSE returns a "sf" object. +#' @examplesIf is_dataRetrieval_user() +#' +#' \donttest{ #' site <- "USGS-02238500" #' pcode <- "00060" #' dv_data_sf <- read_USGS_dv(monitoring_location_id = site, @@ -20,18 +25,19 @@ #' datetime = c("2021-01-01", "2022-01-01")) #' #' dv_data <- read_USGS_dv(monitoring_location_id = site, -#' parameter_code = "00060", no_sf = TRUE) +#' parameter_code = "00060", +#' no_sf = TRUE) #' #' sites <- c("USGS-01491000", "USGS-01645000") #' start_date <- "2021-01-01" #' end_date <- "2022-01-01" -#' req_dv <- read_USGS_dv(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), -#' parameter_code = c("00060", "00065"), -#' datetime = c("2021-01-01", "2022-01-01")) -#' -#' bbox_data <- read_USGS_dv(bbox = c(-83, 36.5, -81, 38.5), -#' parameter_code = "00060") +#' #req_dv <- read_USGS_dv(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), +#' # parameter_code = c("00060", "00065"), +#' # datetime = c("2021-01-01", "2022-01-01")) #' +#' # bbox_data <- read_USGS_dv(bbox = c(-83, 36.5, -81, 38.5), +#' # parameter_code = "00060") +#' } read_USGS_dv <- function(monitoring_location_id = NA_character_, parameter_code = NA_character_, statistic_id = NA_character_, @@ -55,7 +61,6 @@ read_USGS_dv <- function(monitoring_location_id = NA_character_, crs = NA_character_, bbox_crs = NA_character_, skipGeometry = NA, - sortby = NA_character_, offset = NA, datetime = NA_character_, filter = NA_character_, @@ -86,13 +91,14 @@ read_USGS_dv <- function(monitoring_location_id = NA_character_, crs = crs, bbox_crs = bbox_crs, skipGeometry = skipGeometry, - sortby = sortby, offset = offset, datetime = datetime, filter = filter) return_list <- walk_pages(dv_req, use_sf) + return_list <- return_list[order(return_list$time, return_list$monitoring_location_id), ] + return(return_list) } @@ -198,7 +204,63 @@ walk_pages_recursive <- function(req, page, contents, use_sf) { #' Swagger docs: \url{https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html}. #' #' @export -#' @param monitoring_location_id description +#' @param monitoring_location_id A unique identifier representing a single monitoring +#' location. This corresponds to the id field in the sites endpoint. Monitoring +#' location IDs are created by combining the agency code of the agency responsible +#' for the monitoring location (e.g. USGS) with the ID number of the monitoring +#' location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500). +#' @param bbox Only features that have a geometry that intersects the bounding +#' box are selected.The bounding box is provided as four or six numbers, depending +#' on whether the coordinate reference system includes a vertical axis (height or +#' depth). +#' @param crs Indicates the coordinate reference system for the results. +#' @param bbox-crs Indicates the coordinate reference system for the given bbox +#' coordinates. +#' @param properties The properties that should be included for each feature. The +#' parameter value is a comma-separated list of property names. Available values: +#' id, timeseries_id, monitoring_location_id, parameter_code, statistic_id, time, +#' value, unit_of_measure, approval_status, qualifier, last_modified. +#' @param skipGeometry This option can be used to skip response geometries for +#' each feature. +#' @param offset The optional offset parameter indicates the index within the +#' result set from which the server shall begin presenting results in the response +#' document. The first element has an index of 0 (default). +#' @param datetime Either a date-time or an interval. Only features that have a +#' temporal property that intersects the value of datetime are selected. If a +#' feature has multiple temporal properties, it is the decision of the server +#' whether only a single temporal property is used to determine the extent or +#' all relevant temporal properties. +#' @param id A universally unique identifier (UUID) representing a single version +#' of a record. It is not stable over time. Every time the record is refreshed in +#' our database (which may happen as part of normal operations and does not imply +#' any change to the data itself) a new ID will be generated. To uniquely identify +#' a single observation over time, compare the time and timeseries_id fields; each +#' timeseries will only have a single observation at a given time. +#' @param timeseries_id A unique identifier representing a single timeseries. +#' This corresponds to the id field in the timeseries-metadata endpoint. +#' @param parameter_code Parameter codes are 5-digit codes used to identify the +#' constituent measured and the units of measure. +#' @param statistic_id A code corresponding to the statistic an observation represents. +#' Example codes include 00001 (max), 00002 (min), and 00003 (mean). +#' @param time The date an observation represents. +#' @param value The value of the observation. Values are transmitted as strings +#' in the JSON response format in order to preserve precision. +#' @param unit_of_measure A human-readable description of the units of measurement +#' associated with an observation. +#' @param approval_status Some of the data that you have obtained from this U.S. +#' Geological Survey database may not have received Director's approval. Any such +#' data values are qualified as provisional and are subject to revision. Provisional +#' data are released on the condition that neither the USGS nor the United States +#' Government may be held liable for any damages resulting from its use. This field +#' reflects the approval status of each record, and is either "Approved", meaning +#' processing review has been completed and the data is approved for publication, +#' or "Provisional" and subject to revision. +#' @param qualifier This field indicates any qualifiers associated with an observation, +#' for instance if a sensor may have been impacted by ice or if values were estimated. +#' @param last_modified The last time a record was refreshed in our database. This +#' may happen due to regular operational processes and does not necessarily indicate +#' anything about the measurement has changed. You can query this field using +#' date-times or intervals. #' @examples #' site <- "USGS-02238500" #' pcode <- "00060" @@ -238,7 +300,6 @@ construct_dv_requests <- function(monitoring_location_id = NA_character_, crs = NA_character_, bbox_crs = NA_character_, skipGeometry = FALSE, - sortby = "time", offset = NA, datetime = NA_character_, filter = NA_character_ @@ -314,7 +375,6 @@ construct_dv_requests <- function(monitoring_location_id = NA_character_, crs = crs, bbox_crs = bbox_crs, skipGeometry = skipGeometry, - sortby = sortby, offset = offset, datetime = datetime, filter = filter, @@ -356,7 +416,9 @@ construct_dv_requests <- function(monitoring_location_id = NA_character_, #' @param type Character, can be "queryables", "schema" #' @export #' @return list -#' @examples +#' @examplesIf is_dataRetrieval_user() +#' +#' \donttest{ #' #' dv_queryables <- check_OGC_requests(endpoint = "daily", #' type = "queryables") @@ -366,7 +428,7 @@ construct_dv_requests <- function(monitoring_location_id = NA_character_, #' type = "queryables") #' ts_meta_schema <- check_OGC_requests(endpoint = "timeseries-metadata", #' type = "schema") -#' +#' } check_OGC_requests <- function(endpoint = "daily", type = "queryables"){ @@ -383,8 +445,7 @@ check_OGC_requests <- function(endpoint = "daily", message("GET: ", check_req$url) query_ret <- httr2::req_perform(check_req) |> - httr2::resp_body_string() |> - jsonlite::fromJSON() + httr2::resp_body_json() return(query_ret) diff --git a/R/read_USGS_samples.R b/R/read_USGS_samples.R index 3a3f4b53..98276ddd 100644 --- a/R/read_USGS_samples.R +++ b/R/read_USGS_samples.R @@ -235,7 +235,7 @@ construct_USGS_sample_request <- function(monitoringLocationIdentifier = NA, must all be defined, or none defined.") } - baseURL <- explode_query(baseURL, + baseURL <- explode_query(baseURL, POST = FALSE, list(hydrologicUnit = hydrologicUnit, projectIdentifier = projectIdentifier, recordIdentifierUserSupplied = recordIdentifierUserSupplied, diff --git a/man/check_OGC_requests.Rd b/man/check_OGC_requests.Rd index 72ca6720..e91f88ea 100644 --- a/man/check_OGC_requests.Rd +++ b/man/check_OGC_requests.Rd @@ -18,6 +18,9 @@ list Check OGC requests } \examples{ +\dontshow{if (is_dataRetrieval_user()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} + +\donttest{ dv_queryables <- check_OGC_requests(endpoint = "daily", type = "queryables") @@ -27,5 +30,6 @@ ts_meta_queryables <- check_OGC_requests(endpoint = "timeseries-metadata", type = "queryables") ts_meta_schema <- check_OGC_requests(endpoint = "timeseries-metadata", type = "schema") - +} +\dontshow{\}) # examplesIf} } diff --git a/man/construct_dv_requests.Rd b/man/construct_dv_requests.Rd index 684f339f..63bf73d1 100644 --- a/man/construct_dv_requests.Rd +++ b/man/construct_dv_requests.Rd @@ -22,14 +22,86 @@ construct_dv_requests( crs = NA_character_, bbox_crs = NA_character_, skipGeometry = FALSE, - sortby = "time", offset = NA, datetime = NA_character_, filter = NA_character_ ) } \arguments{ -\item{monitoring_location_id}{description} +\item{monitoring_location_id}{A unique identifier representing a single monitoring +location. This corresponds to the id field in the sites endpoint. Monitoring +location IDs are created by combining the agency code of the agency responsible +for the monitoring location (e.g. USGS) with the ID number of the monitoring +location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500).} + +\item{parameter_code}{Parameter codes are 5-digit codes used to identify the +constituent measured and the units of measure.} + +\item{statistic_id}{A code corresponding to the statistic an observation represents. +Example codes include 00001 (max), 00002 (min), and 00003 (mean).} + +\item{properties}{The properties that should be included for each feature. The +parameter value is a comma-separated list of property names. Available values: +id, timeseries_id, monitoring_location_id, parameter_code, statistic_id, time, +value, unit_of_measure, approval_status, qualifier, last_modified.} + +\item{bbox}{Only features that have a geometry that intersects the bounding +box are selected.The bounding box is provided as four or six numbers, depending +on whether the coordinate reference system includes a vertical axis (height or +depth).} + +\item{timeseries_id}{A unique identifier representing a single timeseries. +This corresponds to the id field in the timeseries-metadata endpoint.} + +\item{id}{A universally unique identifier (UUID) representing a single version +of a record. It is not stable over time. Every time the record is refreshed in +our database (which may happen as part of normal operations and does not imply +any change to the data itself) a new ID will be generated. To uniquely identify +a single observation over time, compare the time and timeseries_id fields; each +timeseries will only have a single observation at a given time.} + +\item{approval_status}{Some of the data that you have obtained from this U.S. +Geological Survey database may not have received Director's approval. Any such +data values are qualified as provisional and are subject to revision. Provisional +data are released on the condition that neither the USGS nor the United States +Government may be held liable for any damages resulting from its use. This field +reflects the approval status of each record, and is either "Approved", meaning +processing review has been completed and the data is approved for publication, +or "Provisional" and subject to revision.} + +\item{unit_of_measure}{A human-readable description of the units of measurement +associated with an observation.} + +\item{qualifier}{This field indicates any qualifiers associated with an observation, +for instance if a sensor may have been impacted by ice or if values were estimated.} + +\item{value}{The value of the observation. Values are transmitted as strings +in the JSON response format in order to preserve precision.} + +\item{last_modified}{The last time a record was refreshed in our database. This +may happen due to regular operational processes and does not necessarily indicate +anything about the measurement has changed. You can query this field using +date-times or intervals.} + +\item{crs}{Indicates the coordinate reference system for the results.} + +\item{skipGeometry}{This option can be used to skip response geometries for +each feature.} + +\item{offset}{The optional offset parameter indicates the index within the +result set from which the server shall begin presenting results in the response +document. The first element has an index of 0 (default).} + +\item{datetime}{Either a date-time or an interval. Only features that have a +temporal property that intersects the value of datetime are selected. If a +feature has multiple temporal properties, it is the decision of the server +whether only a single temporal property is used to determine the extent or +all relevant temporal properties.} + +\item{bbox-crs}{Indicates the coordinate reference system for the given bbox +coordinates.} + +\item{time}{The date an observation represents.} } \description{ Main documentation: \url{https://api.waterdata.usgs.gov/ogcapi/v0/}, diff --git a/man/read_USGS_dv.Rd b/man/read_USGS_dv.Rd index effd72fb..5a61d9d5 100644 --- a/man/read_USGS_dv.Rd +++ b/man/read_USGS_dv.Rd @@ -22,13 +22,86 @@ read_USGS_dv( crs = NA_character_, bbox_crs = NA_character_, skipGeometry = NA, - sortby = NA_character_, offset = NA, datetime = NA_character_, filter = NA_character_, no_sf = FALSE ) } +\arguments{ +\item{monitoring_location_id}{A unique identifier representing a single monitoring +location. This corresponds to the id field in the sites endpoint. Monitoring +location IDs are created by combining the agency code of the agency responsible +for the monitoring location (e.g. USGS) with the ID number of the monitoring +location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500).} + +\item{parameter_code}{Parameter codes are 5-digit codes used to identify the +constituent measured and the units of measure.} + +\item{statistic_id}{A code corresponding to the statistic an observation represents. +Example codes include 00001 (max), 00002 (min), and 00003 (mean).} + +\item{properties}{The properties that should be included for each feature. The +parameter value is a comma-separated list of property names. Available values: +id, timeseries_id, monitoring_location_id, parameter_code, statistic_id, time, +value, unit_of_measure, approval_status, qualifier, last_modified.} + +\item{bbox}{Only features that have a geometry that intersects the bounding +box are selected.The bounding box is provided as four or six numbers, depending +on whether the coordinate reference system includes a vertical axis (height or +depth).} + +\item{timeseries_id}{A unique identifier representing a single timeseries. +This corresponds to the id field in the timeseries-metadata endpoint.} + +\item{id}{A universally unique identifier (UUID) representing a single version +of a record. It is not stable over time. Every time the record is refreshed in +our database (which may happen as part of normal operations and does not imply +any change to the data itself) a new ID will be generated. To uniquely identify +a single observation over time, compare the time and timeseries_id fields; each +timeseries will only have a single observation at a given time.} + +\item{approval_status}{Some of the data that you have obtained from this U.S. +Geological Survey database may not have received Director's approval. Any such +data values are qualified as provisional and are subject to revision. Provisional +data are released on the condition that neither the USGS nor the United States +Government may be held liable for any damages resulting from its use. This field +reflects the approval status of each record, and is either "Approved", meaning +processing review has been completed and the data is approved for publication, +or "Provisional" and subject to revision.} + +\item{unit_of_measure}{A human-readable description of the units of measurement +associated with an observation.} + +\item{qualifier}{This field indicates any qualifiers associated with an observation, +for instance if a sensor may have been impacted by ice or if values were estimated.} + +\item{value}{The value of the observation. Values are transmitted as strings +in the JSON response format in order to preserve precision.} + +\item{last_modified}{The last time a record was refreshed in our database. This +may happen due to regular operational processes and does not necessarily indicate +anything about the measurement has changed. You can query this field using +date-times or intervals.} + +\item{crs}{Indicates the coordinate reference system for the results.} + +\item{skipGeometry}{This option can be used to skip response geometries for +each feature.} + +\item{offset}{The optional offset parameter indicates the index within the +result set from which the server shall begin presenting results in the response +document. The first element has an index of 0 (default).} + +\item{datetime}{Either a date-time or an interval. Only features that have a +temporal property that intersects the value of datetime are selected. If a +feature has multiple temporal properties, it is the decision of the server +whether only a single temporal property is used to determine the extent or +all relevant temporal properties.} + +\item{no_sf}{Boolean, whether or not to return an "sf" object. TRUE returns +a basic data frame, FALSE returns a "sf" object.} +} \description{ Daily data provide one data value to represent water conditions for the day. Throughout much of the history of the USGS, the primary water data available @@ -42,6 +115,9 @@ and a statistic code. These data have also been referred to as "daily values" or "DV". } \examples{ +\dontshow{if (is_dataRetrieval_user()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} + +\donttest{ site <- "USGS-02238500" pcode <- "00060" dv_data_sf <- read_USGS_dv(monitoring_location_id = site, @@ -49,16 +125,18 @@ dv_data_sf <- read_USGS_dv(monitoring_location_id = site, datetime = c("2021-01-01", "2022-01-01")) dv_data <- read_USGS_dv(monitoring_location_id = site, - parameter_code = "00060", no_sf = TRUE) + parameter_code = "00060", + no_sf = TRUE) sites <- c("USGS-01491000", "USGS-01645000") start_date <- "2021-01-01" end_date <- "2022-01-01" -req_dv <- read_USGS_dv(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), - parameter_code = c("00060", "00065"), - datetime = c("2021-01-01", "2022-01-01")) - -bbox_data <- read_USGS_dv(bbox = c(-83, 36.5, -81, 38.5), - parameter_code = "00060") +#req_dv <- read_USGS_dv(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), +# parameter_code = c("00060", "00065"), +# datetime = c("2021-01-01", "2022-01-01")) +# bbox_data <- read_USGS_dv(bbox = c(-83, 36.5, -81, 38.5), +# parameter_code = "00060") +} +\dontshow{\}) # examplesIf} } From 7af6779756e7e27e4614b8cb9f956a822ce51bd7 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 17 Apr 2025 12:02:07 -0500 Subject: [PATCH 004/117] fixing some warnings --- R/readWQPdata.R | 72 ++++++++++++++++++------------------ R/readWQPqw.R | 8 ++-- R/read_USGS_dv.R | 10 +++-- man/construct_dv_requests.Rd | 14 ++++--- man/readWQPdata.Rd | 72 ++++++++++++++++++------------------ man/readWQPqw.Rd | 8 ++-- man/read_USGS_dv.Rd | 8 ++++ 7 files changed, 104 insertions(+), 88 deletions(-) diff --git a/R/readWQPdata.R b/R/readWQPdata.R index 01655a66..15d4d6e1 100644 --- a/R/readWQPdata.R +++ b/R/readWQPdata.R @@ -95,11 +95,11 @@ #' attr(pHData, "url") #' #' # WQX3: -#' pHData_wqx3 <- readWQPdata(siteid = "USGS-04024315", -#' characteristicName = nameToUse, -#' service = "ResultWQX3", -#' dataProfile = "basicPhysChem") -#' attr(pHData_wqx3, "url") +#' #pHData_wqx3 <- readWQPdata(siteid = "USGS-04024315", +#' # characteristicName = nameToUse, +#' # service = "ResultWQX3", +#' # dataProfile = "basicPhysChem") +#' #attr(pHData_wqx3, "url") #' #' # More examples: #' # querying by county @@ -111,15 +111,15 @@ #' #' attr(DeWitt, "url") #' -#' DeWitt_wqx3 <- readWQPdata( -#' statecode = "Illinois", -#' countycode = "DeWitt", -#' characteristicName = "Nitrogen", -#' service = "ResultWQX3", -#' dataProfile = "basicPhysChem", -#' ignore_attributes = TRUE) -#' -#' attr(DeWitt_wqx3, "url") +#' #DeWitt_wqx3 <- readWQPdata( +#' # statecode = "Illinois", +#' # countycode = "DeWitt", +#' # characteristicName = "Nitrogen", +#' # service = "ResultWQX3", +#' # dataProfile = "basicPhysChem", +#' # ignore_attributes = TRUE) +#' # +#' #attr(DeWitt_wqx3, "url") #' #' # Data profile: "Sampling Activity" #' activity <- readWQPdata( @@ -128,11 +128,11 @@ #' ) #' attr(activity, "url") #' -#' activity_wqx3 <- readWQPdata( -#' siteid = "USGS-04024315", -#' service = "ActivityWQX3" -#' ) -#' attr(activity_wqx3, "url") +#' # activity_wqx3 <- readWQPdata( +#' # siteid = "USGS-04024315", +#' # service = "ActivityWQX3" +#' # ) +#' # attr(activity_wqx3, "url") #' #' Dane_activity <- readWQPdata( #' statecode = "Wisconsin", @@ -143,14 +143,14 @@ #' ) #' attr(Dane_activity, "url") #' -#' Dane_activity_wqx3 <- readWQPdata( -#' statecode = "Wisconsin", -#' countycode = "Dane", -#' startDateLo = "2023-01-01", -#' startDateHi = "2023-12-31", -#' service = "ActivityWQX3" -#' ) -#' attr(Dane_activity_wqx3, "url") +#' # Dane_activity_wqx3 <- readWQPdata( +#' # statecode = "Wisconsin", +#' # countycode = "Dane", +#' # startDateLo = "2023-01-01", +#' # startDateHi = "2023-12-31", +#' # service = "ActivityWQX3" +#' # ) +#' # attr(Dane_activity_wqx3, "url") #' #' ######################################################## #' # Additional examples: @@ -199,11 +199,11 @@ #' dataProfile = "narrowResult" #' ) #' -#' samp_narrow_wqx3 <- readWQPdata( -#' siteid = "USGS-04024315", -#' service = "ResultWQX3", -#' dataProfile = "narrow" -#' ) +#' # samp_narrow_wqx3 <- readWQPdata( +#' # siteid = "USGS-04024315", +#' # service = "ResultWQX3", +#' # dataProfile = "narrow" +#' # ) #' #' #' # Data profiles: "Sampling Activity" @@ -240,10 +240,10 @@ #' service = "Result", #' dataProfile = "narrowResult" ) #' -#' rawPHsites <- readWQPdata(siteid = c("USGS-05406450", "USGS-05427949", "WIDNR_WQX-133040"), -#' characteristicName = "pH", -#' service = "ResultWQX3", -#' dataProfile = "narrow" ) +#' # rawPHsites <- readWQPdata(siteid = c("USGS-05406450", "USGS-05427949", "WIDNR_WQX-133040"), +#' # characteristicName = "pH", +#' # service = "ResultWQX3", +#' # dataProfile = "narrow" ) #' #' } readWQPdata <- function(..., diff --git a/R/readWQPqw.R b/R/readWQPqw.R index 647722c0..903a57ca 100644 --- a/R/readWQPqw.R +++ b/R/readWQPqw.R @@ -63,10 +63,10 @@ #' ncol(pHsites_legacy) #' attr(pHsites_legacy, "url") #' -#' pHsites_modern <- readWQPqw(c("USGS-05406450", "USGS-05427949", "WIDNR_WQX-133040"), -#' "pH", "", "", legacy = FALSE) -#' ncol(pHsites_modern) -#' attr(pHsites_modern, "url") +#' # pHsites_modern <- readWQPqw(c("USGS-05406450", "USGS-05427949", "WIDNR_WQX-133040"), +#' # "pH", "", "", legacy = FALSE) +#' # ncol(pHsites_modern) +#' # attr(pHsites_modern, "url") #' #' nwisEx <- readWQPqw("USGS-04024000", c("34247", "30234", "32104", "34220"), "", "2022-12-20") #' diff --git a/R/read_USGS_dv.R b/R/read_USGS_dv.R index 3b42c095..b2068a5d 100644 --- a/R/read_USGS_dv.R +++ b/R/read_USGS_dv.R @@ -214,7 +214,7 @@ walk_pages_recursive <- function(req, page, contents, use_sf) { #' on whether the coordinate reference system includes a vertical axis (height or #' depth). #' @param crs Indicates the coordinate reference system for the results. -#' @param bbox-crs Indicates the coordinate reference system for the given bbox +#' @param bbox_crs Indicates the coordinate reference system for the given bbox #' coordinates. #' @param properties The properties that should be included for each feature. The #' parameter value is a comma-separated list of property names. Available values: @@ -261,6 +261,10 @@ walk_pages_recursive <- function(req, page, contents, use_sf) { #' may happen due to regular operational processes and does not necessarily indicate #' anything about the measurement has changed. You can query this field using #' date-times or intervals. +#' @param limit The optional limit parameter limits the number of items that are +#' presented in the response document. Only items are counted that are on the +#' first level of the collection in the response document. Nested objects +#' contained within the explicitly requested items shall not be counted. #' @examples #' site <- "USGS-02238500" #' pcode <- "00060" @@ -301,8 +305,7 @@ construct_dv_requests <- function(monitoring_location_id = NA_character_, bbox_crs = NA_character_, skipGeometry = FALSE, offset = NA, - datetime = NA_character_, - filter = NA_character_ + datetime = NA_character_ ){ match.arg(properties, choices = c("id", @@ -323,6 +326,7 @@ construct_dv_requests <- function(monitoring_location_id = NA_character_, httr2::req_user_agent(default_ua()) token <- Sys.getenv("API_USGS_PAT") + filter <- NA_character_ ########################################################### # This is all going to be done eventually with a POST CQL request: diff --git a/man/construct_dv_requests.Rd b/man/construct_dv_requests.Rd index 63bf73d1..dae15d14 100644 --- a/man/construct_dv_requests.Rd +++ b/man/construct_dv_requests.Rd @@ -23,8 +23,7 @@ construct_dv_requests( bbox_crs = NA_character_, skipGeometry = FALSE, offset = NA, - datetime = NA_character_, - filter = NA_character_ + datetime = NA_character_ ) } \arguments{ @@ -83,8 +82,16 @@ may happen due to regular operational processes and does not necessarily indicat anything about the measurement has changed. You can query this field using date-times or intervals.} +\item{limit}{The optional limit parameter limits the number of items that are +presented in the response document. Only items are counted that are on the +first level of the collection in the response document. Nested objects +contained within the explicitly requested items shall not be counted.} + \item{crs}{Indicates the coordinate reference system for the results.} +\item{bbox_crs}{Indicates the coordinate reference system for the given bbox +coordinates.} + \item{skipGeometry}{This option can be used to skip response geometries for each feature.} @@ -98,9 +105,6 @@ feature has multiple temporal properties, it is the decision of the server whether only a single temporal property is used to determine the extent or all relevant temporal properties.} -\item{bbox-crs}{Indicates the coordinate reference system for the given bbox -coordinates.} - \item{time}{The date an observation represents.} } \description{ diff --git a/man/readWQPdata.Rd b/man/readWQPdata.Rd index 16175970..e0552366 100644 --- a/man/readWQPdata.Rd +++ b/man/readWQPdata.Rd @@ -117,11 +117,11 @@ attr(pHData, "queryTime") attr(pHData, "url") # WQX3: -pHData_wqx3 <- readWQPdata(siteid = "USGS-04024315", - characteristicName = nameToUse, - service = "ResultWQX3", - dataProfile = "basicPhysChem") -attr(pHData_wqx3, "url") +#pHData_wqx3 <- readWQPdata(siteid = "USGS-04024315", +# characteristicName = nameToUse, +# service = "ResultWQX3", +# dataProfile = "basicPhysChem") +#attr(pHData_wqx3, "url") # More examples: # querying by county @@ -133,15 +133,15 @@ DeWitt <- readWQPdata( attr(DeWitt, "url") -DeWitt_wqx3 <- readWQPdata( - statecode = "Illinois", - countycode = "DeWitt", - characteristicName = "Nitrogen", - service = "ResultWQX3", - dataProfile = "basicPhysChem", - ignore_attributes = TRUE) - -attr(DeWitt_wqx3, "url") +#DeWitt_wqx3 <- readWQPdata( +# statecode = "Illinois", +# countycode = "DeWitt", +# characteristicName = "Nitrogen", +# service = "ResultWQX3", +# dataProfile = "basicPhysChem", +# ignore_attributes = TRUE) +# +#attr(DeWitt_wqx3, "url") # Data profile: "Sampling Activity" activity <- readWQPdata( @@ -150,11 +150,11 @@ activity <- readWQPdata( ) attr(activity, "url") -activity_wqx3 <- readWQPdata( - siteid = "USGS-04024315", - service = "ActivityWQX3" -) -attr(activity_wqx3, "url") +# activity_wqx3 <- readWQPdata( +# siteid = "USGS-04024315", +# service = "ActivityWQX3" +# ) +# attr(activity_wqx3, "url") Dane_activity <- readWQPdata( statecode = "Wisconsin", @@ -165,14 +165,14 @@ Dane_activity <- readWQPdata( ) attr(Dane_activity, "url") -Dane_activity_wqx3 <- readWQPdata( - statecode = "Wisconsin", - countycode = "Dane", - startDateLo = "2023-01-01", - startDateHi = "2023-12-31", - service = "ActivityWQX3" -) -attr(Dane_activity_wqx3, "url") +# Dane_activity_wqx3 <- readWQPdata( +# statecode = "Wisconsin", +# countycode = "Dane", +# startDateLo = "2023-01-01", +# startDateHi = "2023-12-31", +# service = "ActivityWQX3" +# ) +# attr(Dane_activity_wqx3, "url") ######################################################## # Additional examples: @@ -221,11 +221,11 @@ samp_narrow <- readWQPdata( dataProfile = "narrowResult" ) -samp_narrow_wqx3 <- readWQPdata( - siteid = "USGS-04024315", - service = "ResultWQX3", - dataProfile = "narrow" -) +# samp_narrow_wqx3 <- readWQPdata( +# siteid = "USGS-04024315", +# service = "ResultWQX3", +# dataProfile = "narrow" +# ) # Data profiles: "Sampling Activity" @@ -262,10 +262,10 @@ rawPHsites_legacy <- readWQPdata(siteid = c("USGS-05406450", "USGS-05427949", "W service = "Result", dataProfile = "narrowResult" ) -rawPHsites <- readWQPdata(siteid = c("USGS-05406450", "USGS-05427949", "WIDNR_WQX-133040"), - characteristicName = "pH", - service = "ResultWQX3", - dataProfile = "narrow" ) +# rawPHsites <- readWQPdata(siteid = c("USGS-05406450", "USGS-05427949", "WIDNR_WQX-133040"), +# characteristicName = "pH", +# service = "ResultWQX3", +# dataProfile = "narrow" ) } \dontshow{\}) # examplesIf} diff --git a/man/readWQPqw.Rd b/man/readWQPqw.Rd index 23fd4ed4..089b9781 100644 --- a/man/readWQPqw.Rd +++ b/man/readWQPqw.Rd @@ -89,10 +89,10 @@ pHsites_legacy <- readWQPqw(c("USGS-05406450", "USGS-05427949", "WIDNR_WQX-13304 ncol(pHsites_legacy) attr(pHsites_legacy, "url") -pHsites_modern <- readWQPqw(c("USGS-05406450", "USGS-05427949", "WIDNR_WQX-133040"), - "pH", "", "", legacy = FALSE) -ncol(pHsites_modern) -attr(pHsites_modern, "url") +# pHsites_modern <- readWQPqw(c("USGS-05406450", "USGS-05427949", "WIDNR_WQX-133040"), +# "pH", "", "", legacy = FALSE) +# ncol(pHsites_modern) +# attr(pHsites_modern, "url") nwisEx <- readWQPqw("USGS-04024000", c("34247", "30234", "32104", "34220"), "", "2022-12-20") diff --git a/man/read_USGS_dv.Rd b/man/read_USGS_dv.Rd index 5a61d9d5..3034a5a0 100644 --- a/man/read_USGS_dv.Rd +++ b/man/read_USGS_dv.Rd @@ -84,8 +84,16 @@ may happen due to regular operational processes and does not necessarily indicat anything about the measurement has changed. You can query this field using date-times or intervals.} +\item{limit}{The optional limit parameter limits the number of items that are +presented in the response document. Only items are counted that are on the +first level of the collection in the response document. Nested objects +contained within the explicitly requested items shall not be counted.} + \item{crs}{Indicates the coordinate reference system for the results.} +\item{bbox_crs}{Indicates the coordinate reference system for the given bbox +coordinates.} + \item{skipGeometry}{This option can be used to skip response geometries for each feature.} From 1314210974940949ea158b4d628a0b61349933bb Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 17 Apr 2025 12:06:53 -0500 Subject: [PATCH 005/117] remove filter arg --- R/read_USGS_dv.R | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/R/read_USGS_dv.R b/R/read_USGS_dv.R index b2068a5d..c46c7d2b 100644 --- a/R/read_USGS_dv.R +++ b/R/read_USGS_dv.R @@ -63,7 +63,6 @@ read_USGS_dv <- function(monitoring_location_id = NA_character_, skipGeometry = NA, offset = NA, datetime = NA_character_, - filter = NA_character_, no_sf = FALSE){ message("Function in development, use at your own risk.") @@ -92,8 +91,7 @@ read_USGS_dv <- function(monitoring_location_id = NA_character_, bbox_crs = bbox_crs, skipGeometry = skipGeometry, offset = offset, - datetime = datetime, - filter = filter) + datetime = datetime) return_list <- walk_pages(dv_req, use_sf) From 66fb60fb1b2de4d5b5413b610bfd0a116b9b69d2 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 18 Apr 2025 16:10:45 -0500 Subject: [PATCH 006/117] WQP workign? --- R/readWQPdata.R | 28 ++++++++++++++-------------- man/readWQPdata.Rd | 28 ++++++++++++++-------------- man/read_USGS_dv.Rd | 1 - 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/R/readWQPdata.R b/R/readWQPdata.R index 15d4d6e1..81e460a3 100644 --- a/R/readWQPdata.R +++ b/R/readWQPdata.R @@ -95,11 +95,11 @@ #' attr(pHData, "url") #' #' # WQX3: -#' #pHData_wqx3 <- readWQPdata(siteid = "USGS-04024315", -#' # characteristicName = nameToUse, -#' # service = "ResultWQX3", -#' # dataProfile = "basicPhysChem") -#' #attr(pHData_wqx3, "url") +#' pHData_wqx3 <- readWQPdata(siteid = "USGS-04024315", +#' characteristicName = nameToUse, +#' service = "ResultWQX3", +#' dataProfile = "basicPhysChem") +#' attr(pHData_wqx3, "url") #' #' # More examples: #' # querying by county @@ -111,15 +111,15 @@ #' #' attr(DeWitt, "url") #' -#' #DeWitt_wqx3 <- readWQPdata( -#' # statecode = "Illinois", -#' # countycode = "DeWitt", -#' # characteristicName = "Nitrogen", -#' # service = "ResultWQX3", -#' # dataProfile = "basicPhysChem", -#' # ignore_attributes = TRUE) -#' # -#' #attr(DeWitt_wqx3, "url") +#' DeWitt_wqx3 <- readWQPdata( +#' statecode = "Illinois", +#' countycode = "DeWitt", +#' characteristicName = "Nitrogen", +#' service = "ResultWQX3", +#' dataProfile = "basicPhysChem", +#' ignore_attributes = TRUE) +#' +#' attr(DeWitt_wqx3, "url") #' #' # Data profile: "Sampling Activity" #' activity <- readWQPdata( diff --git a/man/readWQPdata.Rd b/man/readWQPdata.Rd index e0552366..e58c0d83 100644 --- a/man/readWQPdata.Rd +++ b/man/readWQPdata.Rd @@ -117,11 +117,11 @@ attr(pHData, "queryTime") attr(pHData, "url") # WQX3: -#pHData_wqx3 <- readWQPdata(siteid = "USGS-04024315", -# characteristicName = nameToUse, -# service = "ResultWQX3", -# dataProfile = "basicPhysChem") -#attr(pHData_wqx3, "url") +pHData_wqx3 <- readWQPdata(siteid = "USGS-04024315", + characteristicName = nameToUse, + service = "ResultWQX3", + dataProfile = "basicPhysChem") +attr(pHData_wqx3, "url") # More examples: # querying by county @@ -133,15 +133,15 @@ DeWitt <- readWQPdata( attr(DeWitt, "url") -#DeWitt_wqx3 <- readWQPdata( -# statecode = "Illinois", -# countycode = "DeWitt", -# characteristicName = "Nitrogen", -# service = "ResultWQX3", -# dataProfile = "basicPhysChem", -# ignore_attributes = TRUE) -# -#attr(DeWitt_wqx3, "url") +DeWitt_wqx3 <- readWQPdata( + statecode = "Illinois", + countycode = "DeWitt", + characteristicName = "Nitrogen", + service = "ResultWQX3", + dataProfile = "basicPhysChem", + ignore_attributes = TRUE) + +attr(DeWitt_wqx3, "url") # Data profile: "Sampling Activity" activity <- readWQPdata( diff --git a/man/read_USGS_dv.Rd b/man/read_USGS_dv.Rd index 3034a5a0..46371a82 100644 --- a/man/read_USGS_dv.Rd +++ b/man/read_USGS_dv.Rd @@ -24,7 +24,6 @@ read_USGS_dv( skipGeometry = NA, offset = NA, datetime = NA_character_, - filter = NA_character_, no_sf = FALSE ) } From cc7a76d851bc637ca29c54a8fcd0f8ec6fd71be6 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 6 May 2025 15:20:18 -0500 Subject: [PATCH 007/117] Some updates to dv --- DESCRIPTION | 3 +- R/read_USGS_dv.R | 218 +++++++++++++++++++++----------------- inst/templates/param.CQL2 | 7 ++ inst/templates/post.CQL2 | 8 ++ man/read_USGS_dv.Rd | 16 ++- 5 files changed, 147 insertions(+), 105 deletions(-) create mode 100644 inst/templates/param.CQL2 create mode 100644 inst/templates/post.CQL2 diff --git a/DESCRIPTION b/DESCRIPTION index e5c2da35..ccfa5546 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -44,7 +44,8 @@ Imports: xml2, readr (>= 1.4.0), jsonlite, - httr2 + httr2, + whisker Suggests: covr, dplyr, diff --git a/R/read_USGS_dv.R b/R/read_USGS_dv.R index c46c7d2b..c7b3526c 100644 --- a/R/read_USGS_dv.R +++ b/R/read_USGS_dv.R @@ -28,15 +28,13 @@ #' parameter_code = "00060", #' no_sf = TRUE) #' -#' sites <- c("USGS-01491000", "USGS-01645000") -#' start_date <- "2021-01-01" -#' end_date <- "2022-01-01" -#' #req_dv <- read_USGS_dv(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), -#' # parameter_code = c("00060", "00065"), -#' # datetime = c("2021-01-01", "2022-01-01")) +#' multi_site <- read_USGS_dv(monitoring_location_id = c("USGS-01491000", +#' "USGS-01645000"), +#' parameter_code = c("00060", "00010"), +#' datetime = c("2023-01-01", "2024-01-01")) #' -#' # bbox_data <- read_USGS_dv(bbox = c(-83, 36.5, -81, 38.5), -#' # parameter_code = "00060") +#' bbox_data <- read_USGS_dv(bbox = c(-83, 36.5, -81, 38.5), +#' parameter_code = "00060") #' } read_USGS_dv <- function(monitoring_location_id = NA_character_, parameter_code = NA_character_, @@ -68,8 +66,7 @@ read_USGS_dv <- function(monitoring_location_id = NA_character_, message("Function in development, use at your own risk.") use_sf <- all(pkg.env$local_sf, !no_sf) - page <- 1 - + if(!use_sf){ skipGeometry <- TRUE } @@ -130,8 +127,13 @@ walk_pages <- function(req, use_sf) { # Change to https://httr2.r-lib.org/reference/req_perform_iterative.html walk_pages_recursive <- function(req, page, contents, use_sf) { - - message("GET: ", req$url) + + url_method <- "GET" + if(!is.null(req$body)){ + url_method <- "POST" + body <- req$body + } + message(url_method, ": ", req$url) returned_contents <- httr2::req_perform(req) @@ -174,17 +176,8 @@ walk_pages_recursive <- function(req, page, contents, use_sf) { } else { - make_request <- httr2::request(json_content$links[, "href", drop = TRUE][next_page]) |> - httr2::req_user_agent(default_ua()) - - if( Sys.getenv("API_USGS_PAT") != ""){ - make_request <- make_request |> - httr2::req_headers(`Accept-Encoding` = c("compress", "gzip"), - `X-Api-Key` = Sys.getenv("API_USGS_PAT")) - } else { - make_request <- make_request |> - httr2::req_headers(`Accept-Encoding` = c("compress", "gzip")) - } + make_request <- httr2::req_url(req = req, + url = json_content$links[, "href", drop = TRUE][next_page]) Tailcall( walk_pages_recursive, @@ -306,111 +299,148 @@ construct_dv_requests <- function(monitoring_location_id = NA_character_, datetime = NA_character_ ){ - match.arg(properties, choices = c("id", - "timeseries_id", - "monitoring_location_id", - "parameter_code", - "statistic_id", - "time", - "value", - "unit_of_measure", - "approval_status", - "qualifier", - "last_modified", NA_character_), + schema <- check_OGC_requests(endpoint = "daily", + type = "schema") + all_properties <- names(schema$properties) + + match.arg(properties, choices = c(all_properties, NA_character_), several.ok = TRUE) + if(all(properties %in% all_properties[!all_properties %in% c("id", "geometry")])) { + # Cleans up URL if we're asking for everything + properties <- NA_character_ + } + baseURL <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> httr2::req_url_path_append("daily", "items") |> - httr2::req_user_agent(default_ua()) + httr2::req_user_agent(default_ua()) |> + httr2::req_headers(`Accept-Encoding` = c("compress", "gzip")) token <- Sys.getenv("API_USGS_PAT") - filter <- NA_character_ - - ########################################################### - # This is all going to be done eventually with a POST CQL request: - if(isTRUE(length(monitoring_location_id) > 1)){ - filter <- c(filter[!is.na(filter)], - paste0("monitoring_location_id IN ('", - paste0(monitoring_location_id, - collapse = "', '"), "')")) - monitoring_location_id <- NA_character_ - } - - if(isTRUE(length(parameter_code) > 1)){ - filter <- c(filter[!is.na(filter)], - paste0("parameter_code IN ('", - paste0(parameter_code, - collapse = "', '"), "')")) - parameter_code <- NA_character_ - } - ########################################################### - if(!any(is.na(datetime))){ - if(length(datetime) == 1){ - datetime <- format(datetime, format = "%Y-%m-%dT%H:%M:%SZ") - } else if (length(datetime) == 2) { - datetime <- as.POSIXct(datetime) - datetime <- paste0(vapply(datetime, FUN = function(x) { - format(x, format = "%Y-%m-%dT%H:%M:%SZ")}, - FUN.VALUE = c(NA_character_) - ), collapse = "/") - datetime <- gsub("NA", "..", datetime) - } else { - stop("datetime should only include 1-2 values") - } + if(token != ""){ + baseURL <- baseURL |> + httr2::req_headers_redacted(`X-Api-Key` = token) } POST <- FALSE - baseURL <- explode_query(baseURL, POST = POST, - list(monitoring_location_id = monitoring_location_id, - parameter_code = parameter_code, - statistic_id = statistic_id, - timeseries_id = timeseries_id, - id = id, - approval_status = approval_status, - unit_of_measure = unit_of_measure, - qualifier = qualifier, - value = value, - last_modified = last_modified, + template_path_post <- system.file("templates/post.CQL2", package = "dataRetrieval") + template_post <- readChar(template_path_post, file.info(template_path_post)$size) + + post_params <- explode_post(list(monitoring_location_id = monitoring_location_id, + parameter_code = parameter_code, + statistic_id = statistic_id, + timeseries_id = timeseries_id, + id = id, + approval_status = approval_status, + unit_of_measure = unit_of_measure, + qualifier = qualifier, + value = value)) + + if(length(post_params) > 0){ + POST = TRUE + } + + datetime <- format_api_dates(datetime) + + baseURL <- explode_query(baseURL, POST = FALSE, + list(last_modified = last_modified, limit = limit, crs = crs, bbox_crs = bbox_crs, skipGeometry = skipGeometry, offset = offset, datetime = datetime, - filter = filter, f = "json", lang = "en-US")) - + if(all(!is.na(bbox))){ baseURL <- httr2::req_url_query(baseURL, bbox = bbox, .multi = "comma") } - if(POST){ - baseURL <- httr2::req_body_json(baseURL, - properties = properties) - } else { + if(!all(is.na(properties))){ baseURL <- httr2::req_url_query(baseURL, properties = properties, - .multi = "comma") + .multi = "comma") } - - if(token != ""){ + if(POST){ baseURL <- baseURL |> - httr2::req_headers(`X-Api-Key` = token, - `Accept-Encoding` = c("compress", "gzip")) + httr2::req_headers(`Content-Type` = "application/query-cql-json") + + post_params <- list( + "params" = unname(post_params) + ) + + x <- whisker::whisker.render(template_post, post_params) + baseURL <- httr2::req_body_raw(baseURL, x) + } else { - baseURL <- baseURL |> - httr2::req_headers(`Accept-Encoding` = c("compress", "gzip")) + baseURL <- explode_query(baseURL, POST = FALSE, + list(monitoring_location_id = monitoring_location_id, + parameter_code = parameter_code, + statistic_id = statistic_id, + timeseries_id = timeseries_id, + id = id, + approval_status = approval_status, + unit_of_measure = unit_of_measure, + qualifier = qualifier, + value = value)) } return(baseURL) } +format_api_dates <- function(datetime){ + + if(!any(is.na(datetime))){ + if(length(datetime) == 1){ + datetime <- format(datetime, format = "%Y-%m-%dT%H:%M:%SZ") + } else if (length(datetime) == 2) { + datetime <- as.POSIXct(datetime) + datetime <- paste0(vapply(datetime, FUN = function(x) { + format(x, format = "%Y-%m-%dT%H:%M:%SZ")}, + FUN.VALUE = c(NA_character_) + ), collapse = "/") + datetime <- gsub("NA", "..", datetime) + } else { + stop("datetime should only include 1-2 values") + } + } + return(datetime) +} + +explode_post <- function(ls){ + + ls <- Filter(Negate(anyNA), ls) + params <- NULL + + if(max(lengths(ls)) > 1) { + + for(i in seq_along(ls)){ + params[names(ls[i])] <- cql2_param(ls[i]) + } + + if(length(params) > 1){ + params[seq_along(1:(length(params)-1))] <- paste0(params[seq_along(1:(length(params)-1))], ",") + } + } + return(params) +} + +cql2_param <- function(parameter){ + template_path <- system.file("templates/param.CQL2", package = "dataRetrieval") + template <- readChar(template_path, file.info(template_path)$size) + + parameters <- paste0(unlist(parameter), collapse = '", "') + parameters <- paste0('"', parameters, '"') + parameter_list <- list("property" = names(parameter), + "parameter" = parameters) + return(whisker::whisker.render(template, parameter_list)) +} #' Check OGC requests #' @@ -443,9 +473,7 @@ check_OGC_requests <- function(endpoint = "daily", httr2::req_user_agent(default_ua()) |> httr2::req_url_query(f = "json", lang = "en-US") - - message("GET: ", check_req$url) - + query_ret <- httr2::req_perform(check_req) |> httr2::resp_body_json() diff --git a/inst/templates/param.CQL2 b/inst/templates/param.CQL2 new file mode 100644 index 00000000..305cd380 --- /dev/null +++ b/inst/templates/param.CQL2 @@ -0,0 +1,7 @@ + { + "op": "in", + "args": [ + {"property": "{{{property}}}"}, + [ {{{parameter}}} ] + ] + } \ No newline at end of file diff --git a/inst/templates/post.CQL2 b/inst/templates/post.CQL2 new file mode 100644 index 00000000..c948c51b --- /dev/null +++ b/inst/templates/post.CQL2 @@ -0,0 +1,8 @@ +{ + "op": "and", + "args": [ + {{#params}} + {{{.}}} + {{/params}} + ] +} \ No newline at end of file diff --git a/man/read_USGS_dv.Rd b/man/read_USGS_dv.Rd index 46371a82..d23b29dc 100644 --- a/man/read_USGS_dv.Rd +++ b/man/read_USGS_dv.Rd @@ -135,15 +135,13 @@ dv_data <- read_USGS_dv(monitoring_location_id = site, parameter_code = "00060", no_sf = TRUE) -sites <- c("USGS-01491000", "USGS-01645000") -start_date <- "2021-01-01" -end_date <- "2022-01-01" -#req_dv <- read_USGS_dv(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), -# parameter_code = c("00060", "00065"), -# datetime = c("2021-01-01", "2022-01-01")) - -# bbox_data <- read_USGS_dv(bbox = c(-83, 36.5, -81, 38.5), -# parameter_code = "00060") +multi_site <- read_USGS_dv(monitoring_location_id = c("USGS-01491000", + "USGS-01645000"), + parameter_code = c("00060", "00010"), + datetime = c("2023-01-01", "2024-01-01")) + +bbox_data <- read_USGS_dv(bbox = c(-83, 36.5, -81, 38.5), + parameter_code = "00060") } \dontshow{\}) # examplesIf} } From b8505dc3b47594cec9e974be83314d4ec89cf722 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 6 May 2025 15:20:57 -0500 Subject: [PATCH 008/117] add whisker to docker --- docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index a5fa12e5..c4adc2df 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -24,6 +24,7 @@ RUN apt-get update -qq && apt-get -y --no-install-recommends install \ r-cran-maps \ r-cran-leaflet \ r-cran-readxl \ + r-cran-whisker \ && apt-get install -y pandoc \ && rm -rf /var/lib/apt/lists/* From 52249898cef8a5f0dc0d4b7e7aed1081a0816936 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 6 May 2025 16:38:37 -0500 Subject: [PATCH 009/117] getting POST dvs to work --- R/read_USGS_dv.R | 32 ++++++++++++++++++++------------ man/read_USGS_dv.Rd | 8 ++++++-- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/R/read_USGS_dv.R b/R/read_USGS_dv.R index c7b3526c..c66efa25 100644 --- a/R/read_USGS_dv.R +++ b/R/read_USGS_dv.R @@ -15,6 +15,8 @@ #' @inheritParams construct_dv_requests #' @param no_sf Boolean, whether or not to return an "sf" object. TRUE returns #' a basic data frame, FALSE returns a "sf" object. +#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function +#' will convert the data to dates, datetimes, #' @examplesIf is_dataRetrieval_user() #' #' \donttest{ @@ -33,7 +35,7 @@ #' parameter_code = c("00060", "00010"), #' datetime = c("2023-01-01", "2024-01-01")) #' -#' bbox_data <- read_USGS_dv(bbox = c(-83, 36.5, -81, 38.5), +#' bbox_data <- read_USGS_dv(bbox = c(-83, 36.5, -82.5, 37.5), #' parameter_code = "00060") #' } read_USGS_dv <- function(monitoring_location_id = NA_character_, @@ -61,7 +63,8 @@ read_USGS_dv <- function(monitoring_location_id = NA_character_, skipGeometry = NA, offset = NA, datetime = NA_character_, - no_sf = FALSE){ + no_sf = FALSE, + convertType = TRUE){ message("Function in development, use at your own risk.") @@ -92,17 +95,25 @@ read_USGS_dv <- function(monitoring_location_id = NA_character_, return_list <- walk_pages(dv_req, use_sf) + if(convertType) return_list <- cleanup_cols(return_list) + return_list <- return_list[order(return_list$time, return_list$monitoring_location_id), ] + if(!"id" %in% properties){ + return_list <- return_list[, names(return_list)[!names(return_list) %in% "id"]] + } + return(return_list) } cleanup_cols <- function(df){ if("qualifier" %in% names(df)){ - df$qualifier <- vapply(X = df$qualifier, - FUN = function(x) paste(x, collapse = ","), - FUN.VALUE = c(NA_character_)) + if(!all(is.na(df$qualifier))){ + df$qualifier <- vapply(X = df$qualifier, + FUN = function(x) paste(x, collapse = ", "), + FUN.VALUE = c(NA_character_)) + } } if("time" %in% names(df)){ @@ -158,16 +169,12 @@ walk_pages_recursive <- function(req, page, contents, use_sf) { if(use_sf){ return_df <- lapply(contents, function(x) { - df_i <- sf::read_sf(x) |> - cleanup_cols() - df_i + df_i <- sf::read_sf(x) }) |> do.call(what = rbind) } else { return_df <- lapply(contents, function(x) { - df_i <- jsonlite::fromJSON(x)[["features"]][["properties"]] |> - cleanup_cols() - df_i + df_i <- jsonlite::fromJSON(x)[["features"]][["properties"]] }) |> do.call(what = rbind) } @@ -306,7 +313,7 @@ construct_dv_requests <- function(monitoring_location_id = NA_character_, match.arg(properties, choices = c(all_properties, NA_character_), several.ok = TRUE) - if(all(properties %in% all_properties[!all_properties %in% c("id", "geometry")])) { + if(all(all_properties[!all_properties %in% c("id", "geometry")] %in% properties)) { # Cleans up URL if we're asking for everything properties <- NA_character_ } @@ -346,6 +353,7 @@ construct_dv_requests <- function(monitoring_location_id = NA_character_, baseURL <- explode_query(baseURL, POST = FALSE, list(last_modified = last_modified, + properties = properties, limit = limit, crs = crs, bbox_crs = bbox_crs, diff --git a/man/read_USGS_dv.Rd b/man/read_USGS_dv.Rd index d23b29dc..56bef52e 100644 --- a/man/read_USGS_dv.Rd +++ b/man/read_USGS_dv.Rd @@ -24,7 +24,8 @@ read_USGS_dv( skipGeometry = NA, offset = NA, datetime = NA_character_, - no_sf = FALSE + no_sf = FALSE, + convertType = TRUE ) } \arguments{ @@ -108,6 +109,9 @@ all relevant temporal properties.} \item{no_sf}{Boolean, whether or not to return an "sf" object. TRUE returns a basic data frame, FALSE returns a "sf" object.} + +\item{convertType}{logical, defaults to \code{TRUE}. If \code{TRUE}, the function +will convert the data to dates, datetimes,} } \description{ Daily data provide one data value to represent water conditions for the day. @@ -140,7 +144,7 @@ multi_site <- read_USGS_dv(monitoring_location_id = c("USGS-01491000", parameter_code = c("00060", "00010"), datetime = c("2023-01-01", "2024-01-01")) -bbox_data <- read_USGS_dv(bbox = c(-83, 36.5, -81, 38.5), +bbox_data <- read_USGS_dv(bbox = c(-83, 36.5, -82.5, 37.5), parameter_code = "00060") } \dontshow{\}) # examplesIf} From 61b4dd89ac63ddf169f781bf907c0a0cd832c932 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 7 May 2025 14:12:01 -0500 Subject: [PATCH 010/117] user doesn't need offset --- R/read_USGS_dv.R | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/R/read_USGS_dv.R b/R/read_USGS_dv.R index c66efa25..03aa2d11 100644 --- a/R/read_USGS_dv.R +++ b/R/read_USGS_dv.R @@ -16,7 +16,7 @@ #' @param no_sf Boolean, whether or not to return an "sf" object. TRUE returns #' a basic data frame, FALSE returns a "sf" object. #' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function -#' will convert the data to dates, datetimes, +#' will convert the data to dates and qualifier to string vector. #' @examplesIf is_dataRetrieval_user() #' #' \donttest{ @@ -35,8 +35,6 @@ #' parameter_code = c("00060", "00010"), #' datetime = c("2023-01-01", "2024-01-01")) #' -#' bbox_data <- read_USGS_dv(bbox = c(-83, 36.5, -82.5, 37.5), -#' parameter_code = "00060") #' } read_USGS_dv <- function(monitoring_location_id = NA_character_, parameter_code = NA_character_, @@ -61,7 +59,6 @@ read_USGS_dv <- function(monitoring_location_id = NA_character_, crs = NA_character_, bbox_crs = NA_character_, skipGeometry = NA, - offset = NA, datetime = NA_character_, no_sf = FALSE, convertType = TRUE){ @@ -90,7 +87,6 @@ read_USGS_dv <- function(monitoring_location_id = NA_character_, crs = crs, bbox_crs = bbox_crs, skipGeometry = skipGeometry, - offset = offset, datetime = datetime) return_list <- walk_pages(dv_req, use_sf) @@ -136,7 +132,6 @@ walk_pages <- function(req, use_sf) { ) } -# Change to https://httr2.r-lib.org/reference/req_perform_iterative.html walk_pages_recursive <- function(req, page, contents, use_sf) { url_method <- "GET" @@ -302,7 +297,6 @@ construct_dv_requests <- function(monitoring_location_id = NA_character_, crs = NA_character_, bbox_crs = NA_character_, skipGeometry = FALSE, - offset = NA, datetime = NA_character_ ){ @@ -318,17 +312,7 @@ construct_dv_requests <- function(monitoring_location_id = NA_character_, properties <- NA_character_ } - baseURL <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> - httr2::req_url_path_append("daily", "items") |> - httr2::req_user_agent(default_ua()) |> - httr2::req_headers(`Accept-Encoding` = c("compress", "gzip")) - - token <- Sys.getenv("API_USGS_PAT") - - if(token != ""){ - baseURL <- baseURL |> - httr2::req_headers_redacted(`X-Api-Key` = token) - } + baseURL <- setup_api("daily") POST <- FALSE @@ -358,10 +342,7 @@ construct_dv_requests <- function(monitoring_location_id = NA_character_, crs = crs, bbox_crs = bbox_crs, skipGeometry = skipGeometry, - offset = offset, - datetime = datetime, - f = "json", - lang = "en-US")) + datetime = datetime)) if(all(!is.na(bbox))){ baseURL <- httr2::req_url_query(baseURL, @@ -402,6 +383,25 @@ construct_dv_requests <- function(monitoring_location_id = NA_character_, return(baseURL) } +setup_api <- function(service){ + baseURL <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> + httr2::req_url_path_append(service, "items") |> + httr2::req_user_agent(default_ua()) |> + httr2::req_headers(`Accept-Encoding` = c("compress", "gzip")) + + token <- Sys.getenv("API_USGS_PAT") + + if(token != ""){ + baseURL <- baseURL |> + httr2::req_headers_redacted(`X-Api-Key` = token) + } + + baseURL <- explode_query(baseURL, POST = FALSE, + list(f = "json", + lang = "en-US")) + +} + format_api_dates <- function(datetime){ if(!any(is.na(datetime))){ From bec30c5f4c6b2d21695e5a12e382cdf288d908fe Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 7 May 2025 15:22:31 -0500 Subject: [PATCH 011/117] Rejigger structure and add vignette --- NAMESPACE | 3 +- R/construct_api_requests.R | 349 +++++++++++++ R/read_USGS_dv.R | 461 +++--------------- R/read_USGS_ts_meta.R | 125 +++++ R/walk_pages.R | 88 ++++ man/check_OGC_requests.Rd | 2 +- ..._requests.Rd => construct_api_requests.Rd} | 84 +++- man/read_NWIS_ts_meta.Rd | 108 ++++ man/read_USGS_dv.Rd | 24 +- vignettes/read_USGS_functions.Rmd | 118 +++++ 10 files changed, 947 insertions(+), 415 deletions(-) create mode 100644 R/construct_api_requests.R create mode 100644 R/read_USGS_ts_meta.R create mode 100644 R/walk_pages.R rename man/{construct_dv_requests.Rd => construct_api_requests.Rd} (62%) create mode 100644 man/read_NWIS_ts_meta.Rd create mode 100644 vignettes/read_USGS_functions.Rmd diff --git a/NAMESPACE b/NAMESPACE index 990b0777..5baa34a5 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -9,7 +9,7 @@ export(constructNWISURL) export(constructUseURL) export(constructWQPURL) export(construct_USGS_sample_request) -export(construct_dv_requests) +export(construct_api_requests) export(countyCd) export(countyCdLookup) export(create_NWIS_bib) @@ -44,6 +44,7 @@ export(readNWISuv) export(readWQPdata) export(readWQPqw) export(readWQPsummary) +export(read_NWIS_ts_meta) export(read_USGS_dv) export(read_USGS_samples) export(renameNWISColumns) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R new file mode 100644 index 00000000..a2f8738f --- /dev/null +++ b/R/construct_api_requests.R @@ -0,0 +1,349 @@ +#' Create API url +#' +#' Main documentation: \url{https://api.waterdata.usgs.gov/ogcapi/v0/}, +#' Swagger docs: \url{https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html}. +#' +#' @export +#' @param monitoring_location_id A unique identifier representing a single monitoring +#' location. This corresponds to the id field in the sites endpoint. Monitoring +#' location IDs are created by combining the agency code of the agency responsible +#' for the monitoring location (e.g. USGS) with the ID number of the monitoring +#' location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500). +#' @param bbox Only features that have a geometry that intersects the bounding +#' box are selected.The bounding box is provided as four or six numbers, depending +#' on whether the coordinate reference system includes a vertical axis (height or +#' depth). +#' @param crs Indicates the coordinate reference system for the results. +#' @param bbox_crs Indicates the coordinate reference system for the given bbox +#' coordinates. +#' @param properties The properties that should be included for each feature. The +#' parameter value is a comma-separated list of property names. Available values: +#' id, timeseries_id, monitoring_location_id, parameter_code, statistic_id, time, +#' value, unit_of_measure, approval_status, qualifier, last_modified. +#' @param skipGeometry This option can be used to skip response geometries for +#' each feature. +#' @param offset The optional offset parameter indicates the index within the +#' result set from which the server shall begin presenting results in the response +#' document. The first element has an index of 0 (default). +#' @param datetime Either a date-time or an interval. Only features that have a +#' temporal property that intersects the value of datetime are selected. If a +#' feature has multiple temporal properties, it is the decision of the server +#' whether only a single temporal property is used to determine the extent or +#' all relevant temporal properties. +#' @param id A universally unique identifier (UUID) representing a single version +#' of a record. It is not stable over time. Every time the record is refreshed in +#' our database (which may happen as part of normal operations and does not imply +#' any change to the data itself) a new ID will be generated. To uniquely identify +#' a single observation over time, compare the time and timeseries_id fields; each +#' timeseries will only have a single observation at a given time. +#' @param timeseries_id A unique identifier representing a single timeseries. +#' This corresponds to the id field in the timeseries-metadata endpoint. +#' @param parameter_code Parameter codes are 5-digit codes used to identify the +#' constituent measured and the units of measure. +#' @param parameter_name A human-understandable name corresponding to parameter_code. +#' @param statistic_id A code corresponding to the statistic an observation represents. +#' Example codes include 00001 (max), 00002 (min), and 00003 (mean). +#' @param time The date an observation represents. +#' @param value The value of the observation. Values are transmitted as strings +#' in the JSON response format in order to preserve precision. +#' @param unit_of_measure A human-readable description of the units of measurement +#' associated with an observation. +#' @param approval_status Some of the data that you have obtained from this U.S. +#' Geological Survey database may not have received Director's approval. Any such +#' data values are qualified as provisional and are subject to revision. Provisional +#' data are released on the condition that neither the USGS nor the United States +#' Government may be held liable for any damages resulting from its use. This field +#' reflects the approval status of each record, and is either "Approved", meaning +#' processing review has been completed and the data is approved for publication, +#' or "Provisional" and subject to revision. +#' @param qualifier This field indicates any qualifiers associated with an observation, +#' for instance if a sensor may have been impacted by ice or if values were estimated. +#' @param last_modified The last time a record was refreshed in our database. This +#' may happen due to regular operational processes and does not necessarily indicate +#' anything about the measurement has changed. You can query this field using +#' date-times or intervals. +#' @param begin The datetime of the earliest observation in the timeseries. +#' Together with end, this field represents the period of record of a timeseries. +#' Note that some timeseries may have large gaps in their collection record. +#' @param end The datetime of the most recent observation in the timeseries. +#' Data returned by this endpoint updates at most once per day, and potentially +#' less frequently than that, and as such there may be more recent observations +#' within a timeseries than the timeseries end value reflects. Together with begin, +#' this field represents the period of record of a timeseries. It is additionally +#' used to determine whether a timeseries is "active". +#' @param computation_period_identifier Indicates the period of data used for +#' any statistical computations. +#' @param computation_identifier Indicates whether the data from this timeseries +#' represent a specific statistical computation. +#' @param thresholds Thresholds represent known numeric limits for a timeseries, +#' for example the historic maximum value for a parameter or a level below which +#' a sensor is non-operative. These thresholds are sometimes used to automatically +#' determine if an observation is erroneous due to sensor error, and therefore +#' shouldn't be included in the timeseries. +#' @param sublocation_identifier An optional human-readable identifier used to +#' specify where measurements are recorded at a monitoring location. +#' @param primary A flag identifying if the timeseries is a "primary" timeseries. +#' "Primary" timeseries (which have this flag) are standard observations which +#' undergo Bureau review and approval processes. Non-primary timeseries, which +#' will have missing values for "primary", are provisional datasets made available +#' to meet the need for timely best science and to assist with daily operations +#' which need real-time information. Non-primary timeseries data are only retained +#' by this system for 120 days. See the USGS Provisional Data Statement for more information. +#' @param web_description A description of what this timeseries represents, as +#' used by WDFN and other USGS data dissemination products. +#' @param limit The optional limit parameter limits the number of items that are +#' presented in the response document. Only items are counted that are on the +#' first level of the collection in the response document. Nested objects +#' contained within the explicitly requested items shall not be counted. +#' @keywords internal +#' @examples +#' site <- "USGS-02238500" +#' pcode <- "00060" +#' req_dv <- construct_api_requests("daily", +#' monitoring_location_id = site, +#' parameter_code = "00060") +#' +#' req_dv <- construct_api_requests("daily", +#' monitoring_location_id = site, +#' parameter_code = c("00060", "00065")) +#' +#' sites <- c("USGS-01491000", "USGS-01645000") +#' start_date <- "2018-01-01" +#' end_date <- "2022-01-01" +#' req_dv <- construct_api_requests("daily", +#' monitoring_location_id = sites, +#' parameter_code = c("00060", "00065"), +#' datetime = c(start_date, end_date)) +#' +construct_api_requests <- function(service, + monitoring_location_id = NA_character_, + parameter_code = NA_character_, + statistic_id = NA_character_, + properties = c("monitoring_location_id", + "parameter_code", + "statistic_id", + "time", + "value", + "unit_of_measure", + "approval_status", + "qualifier"), + bbox = NA, + timeseries_id = NA_character_, + id = NA_character_, + approval_status = NA_character_, + unit_of_measure = NA_character_, + qualifier = NA_character_, + value = NA, + last_modified = NA_character_, + limit = 10000, + crs = NA_character_, + bbox_crs = NA_character_, + skipGeometry = FALSE, + datetime = NA_character_, + begin = NA_character_, + end = NA_character_, + primary = NA_character_, + parameter_name = NA_character_, + thresholds = NA, + sublocation_identifier = NA_character_, + computation_period_identifier = NA_character_, + computation_identifier = NA_character_, + web_description = NA_character_){ + + schema <- check_OGC_requests(endpoint = service, + type = "schema") + all_properties <- names(schema$properties) + + match.arg(properties, choices = c(all_properties, NA_character_), + several.ok = TRUE) + + if(all(all_properties[!all_properties %in% c("id", "geometry")] %in% properties)) { + # Cleans up URL if we're asking for everything + properties <- NA_character_ + } + + baseURL <- setup_api(service) + + POST <- FALSE + + template_path_post <- system.file("templates/post.CQL2", package = "dataRetrieval") + template_post <- readChar(template_path_post, file.info(template_path_post)$size) + + post_params <- explode_post(list(monitoring_location_id = monitoring_location_id, + parameter_code = parameter_code, + statistic_id = statistic_id, + timeseries_id = timeseries_id, + id = id, + approval_status = approval_status, + unit_of_measure = unit_of_measure, + qualifier = qualifier, + value = value, + parameter_name = parameter_name + +)) + + if(length(post_params) > 0){ + POST = TRUE + } + + datetime <- format_api_dates(datetime) + + baseURL <- explode_query(baseURL, POST = FALSE, + list(last_modified = last_modified, + begin = begin, + end = end, + primary = primary, + computation_period_identifier = computation_period_identifier, + computation_identifier = computation_identifier, + web_description = web_description, + properties = properties, + limit = limit, + crs = crs, + `bbox-crs` = bbox_crs, + skipGeometry = skipGeometry, + datetime = datetime)) + + if(all(!is.na(bbox))){ + baseURL <- httr2::req_url_query(baseURL, + bbox = bbox, + .multi = "comma") + } + + if(!all(is.na(properties))){ + baseURL <- httr2::req_url_query(baseURL, + properties = properties, + .multi = "comma") + } + + if(POST){ + baseURL <- baseURL |> + httr2::req_headers(`Content-Type` = "application/query-cql-json") + + post_params <- list( + "params" = unname(post_params) + ) + + x <- whisker::whisker.render(template_post, post_params) + baseURL <- httr2::req_body_raw(baseURL, x) + + } else { + baseURL <- explode_query(baseURL, POST = FALSE, + list(monitoring_location_id = monitoring_location_id, + parameter_code = parameter_code, + statistic_id = statistic_id, + timeseries_id = timeseries_id, + id = id, + approval_status = approval_status, + unit_of_measure = unit_of_measure, + qualifier = qualifier, + value = value)) + } + + return(baseURL) +} + +setup_api <- function(service){ + baseURL <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> + httr2::req_url_path_append(service, "items") |> + httr2::req_user_agent(default_ua()) |> + httr2::req_headers(`Accept-Encoding` = c("compress", "gzip")) + + token <- Sys.getenv("API_USGS_PAT") + + if(token != ""){ + baseURL <- baseURL |> + httr2::req_headers_redacted(`X-Api-Key` = token) + } + + baseURL <- explode_query(baseURL, POST = FALSE, + list(f = "json", + lang = "en-US")) + +} + +format_api_dates <- function(datetime){ + + if(!any(is.na(datetime))){ + if(length(datetime) == 1){ + datetime <- format(datetime, format = "%Y-%m-%dT%H:%M:%SZ") + } else if (length(datetime) == 2) { + datetime <- as.POSIXct(datetime) + datetime <- paste0(vapply(datetime, FUN = function(x) { + format(x, format = "%Y-%m-%dT%H:%M:%SZ")}, + FUN.VALUE = c(NA_character_) + ), collapse = "/") + datetime <- gsub("NA", "..", datetime) + } else { + stop("datetime should only include 1-2 values") + } + } + return(datetime) +} + +explode_post <- function(ls){ + + ls <- Filter(Negate(anyNA), ls) + params <- NULL + + if(max(lengths(ls)) > 1) { + + for(i in seq_along(ls)){ + params[names(ls[i])] <- cql2_param(ls[i]) + } + + if(length(params) > 1){ + params[seq_along(1:(length(params)-1))] <- paste0(params[seq_along(1:(length(params)-1))], ",") + } + } + return(params) +} + +cql2_param <- function(parameter){ + template_path <- system.file("templates/param.CQL2", package = "dataRetrieval") + template <- readChar(template_path, file.info(template_path)$size) + + parameters <- paste0(unlist(parameter), collapse = '", "') + parameters <- paste0('"', parameters, '"') + parameter_list <- list("property" = names(parameter), + "parameter" = parameters) + return(whisker::whisker.render(template, parameter_list)) +} + +#' Check OGC requests +#' +#' @param endpoint Character, can be "daily", "timeseries-metadata" +#' @param type Character, can be "queryables", "schema" +#' @export +#' @return list +#' @examplesIf is_dataRetrieval_user() +#' +#' \donttest{ +#' +#' dv_queryables <- check_OGC_requests(endpoint = "daily", +#' type = "queryables") +#' dv_schema <- check_OGC_requests(endpoint = "daily", +#' type = "schema") +#' ts_meta_queryables <- check_OGC_requests(endpoint = "timeseries-metadata", +#' type = "queryables") +#' ts_meta_schema <- check_OGC_requests(endpoint = "timeseries-metadata", +#' type = "schema") +#' } +check_OGC_requests <- function(endpoint = "daily", + type = "queryables"){ + + match.arg(endpoint, c("daily", "timeseries-metadata")) + match.arg(type, c("queryables", "schema")) + + check_req <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> + httr2::req_url_path_append(endpoint) |> + httr2::req_url_path_append(type) |> + httr2::req_user_agent(default_ua()) |> + httr2::req_url_query(f = "json", + lang = "en-US") + + query_ret <- httr2::req_perform(check_req) |> + httr2::resp_body_json() + + return(query_ret) + +} \ No newline at end of file diff --git a/R/read_USGS_dv.R b/R/read_USGS_dv.R index 03aa2d11..0acb3d83 100644 --- a/R/read_USGS_dv.R +++ b/R/read_USGS_dv.R @@ -12,191 +12,6 @@ #' or "DV". #' #' @export -#' @inheritParams construct_dv_requests -#' @param no_sf Boolean, whether or not to return an "sf" object. TRUE returns -#' a basic data frame, FALSE returns a "sf" object. -#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function -#' will convert the data to dates and qualifier to string vector. -#' @examplesIf is_dataRetrieval_user() -#' -#' \donttest{ -#' site <- "USGS-02238500" -#' pcode <- "00060" -#' dv_data_sf <- read_USGS_dv(monitoring_location_id = site, -#' parameter_code = "00060", -#' datetime = c("2021-01-01", "2022-01-01")) -#' -#' dv_data <- read_USGS_dv(monitoring_location_id = site, -#' parameter_code = "00060", -#' no_sf = TRUE) -#' -#' multi_site <- read_USGS_dv(monitoring_location_id = c("USGS-01491000", -#' "USGS-01645000"), -#' parameter_code = c("00060", "00010"), -#' datetime = c("2023-01-01", "2024-01-01")) -#' -#' } -read_USGS_dv <- function(monitoring_location_id = NA_character_, - parameter_code = NA_character_, - statistic_id = NA_character_, - properties = c("monitoring_location_id", - "parameter_code", - "statistic_id", - "time", - "value", - "unit_of_measure", - "approval_status", - "qualifier"), - bbox = NA, - timeseries_id = NA_character_, - id = NA_character_, - approval_status = NA_character_, - unit_of_measure = NA_character_, - qualifier = NA_character_, - value = NA, - last_modified = NA_character_, - limit = 10000, - crs = NA_character_, - bbox_crs = NA_character_, - skipGeometry = NA, - datetime = NA_character_, - no_sf = FALSE, - convertType = TRUE){ - - message("Function in development, use at your own risk.") - - use_sf <- all(pkg.env$local_sf, !no_sf) - - if(!use_sf){ - skipGeometry <- TRUE - } - - dv_req <- construct_dv_requests(monitoring_location_id = monitoring_location_id, - parameter_code = parameter_code, - statistic_id = statistic_id, - properties = properties, - bbox = bbox, - timeseries_id = timeseries_id, - id = id, - approval_status = approval_status, - unit_of_measure = unit_of_measure, - qualifier = qualifier, - value = value, - last_modified = last_modified, - limit = limit, - crs = crs, - bbox_crs = bbox_crs, - skipGeometry = skipGeometry, - datetime = datetime) - - return_list <- walk_pages(dv_req, use_sf) - - if(convertType) return_list <- cleanup_cols(return_list) - - return_list <- return_list[order(return_list$time, return_list$monitoring_location_id), ] - - if(!"id" %in% properties){ - return_list <- return_list[, names(return_list)[!names(return_list) %in% "id"]] - } - - return(return_list) -} - -cleanup_cols <- function(df){ - - if("qualifier" %in% names(df)){ - if(!all(is.na(df$qualifier))){ - df$qualifier <- vapply(X = df$qualifier, - FUN = function(x) paste(x, collapse = ", "), - FUN.VALUE = c(NA_character_)) - } - } - - if("time" %in% names(df)){ - df$time <- as.Date(df$time) - } - - if("value" %in% names(df)){ - df$value <- as.numeric(df$value) - } - df -} - -walk_pages <- function(req, use_sf) { - - walk_pages_recursive( - req = req, - page = 1, - contents = list(), - use_sf - ) -} - -walk_pages_recursive <- function(req, page, contents, use_sf) { - - url_method <- "GET" - if(!is.null(req$body)){ - url_method <- "POST" - body <- req$body - } - message(url_method, ": ", req$url) - - returned_contents <- httr2::req_perform(req) - - if(httr2::resp_status(returned_contents) != 200){ - if(httr2::resp_status(returned_contents) == 429){ - stop("You hit your hourly limit for requests. To increase the limit, - see: https://api.waterdata.usgs.gov/docs/ogcapi/keys/") - } - } - - header_info <- httr2::resp_headers(returned_contents) - message("Remaining requests this hour:", header_info$`x-ratelimit-remaining`) - - contents[[page]] <- returned_contents |> - httr2::resp_body_string() - - json_content <- jsonlite::fromJSON(contents[[page]] ) - - next_page <- json_content$links[, "rel", drop = TRUE] == "next" - - if (!any(next_page)) { - - if(use_sf){ - return_df <- lapply(contents, function(x) { - df_i <- sf::read_sf(x) - }) |> - do.call(what = rbind) - } else { - return_df <- lapply(contents, function(x) { - df_i <- jsonlite::fromJSON(x)[["features"]][["properties"]] - }) |> - do.call(what = rbind) - } - - return(return_df) - - } else { - - make_request <- httr2::req_url(req = req, - url = json_content$links[, "href", drop = TRUE][next_page]) - - Tailcall( - walk_pages_recursive, - make_request, - page + 1, - contents, - use_sf - ) - } -} - -#' Create DV url -#' -#' Main documentation: \url{https://api.waterdata.usgs.gov/ogcapi/v0/}, -#' Swagger docs: \url{https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html}. -#' -#' @export #' @param monitoring_location_id A unique identifier representing a single monitoring #' location. This corresponds to the id field in the sites endpoint. Monitoring #' location IDs are created by combining the agency code of the agency responsible @@ -254,237 +69,107 @@ walk_pages_recursive <- function(req, page, contents, use_sf) { #' may happen due to regular operational processes and does not necessarily indicate #' anything about the measurement has changed. You can query this field using #' date-times or intervals. +#' @param crs Indicates the coordinate reference system for the results. #' @param limit The optional limit parameter limits the number of items that are #' presented in the response document. Only items are counted that are on the #' first level of the collection in the response document. Nested objects #' contained within the explicitly requested items shall not be counted. -#' @examples +#' @param no_sf Boolean, whether or not to return an "sf" object. TRUE returns +#' a basic data frame, FALSE returns a "sf" object. +#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function +#' will convert the data to dates and qualifier to string vector. +#' @examplesIf is_dataRetrieval_user() +#' +#' \donttest{ #' site <- "USGS-02238500" #' pcode <- "00060" -#' req_dv <- construct_dv_requests(monitoring_location_id = site, -#' parameter_code = "00060") +#' dv_data_sf <- read_USGS_dv(monitoring_location_id = site, +#' parameter_code = "00060", +#' datetime = c("2021-01-01", "2022-01-01")) +#' +#' dv_data_sf <- read_USGS_dv(monitoring_location_id = site, +#' parameter_code = "00060", +#' properties = c("monitoring_location_id", +#' "value", +#' "time"), +#' datetime = c("2021-01-01", "2022-01-01")) #' -#' req_dv <- construct_dv_requests(monitoring_location_id = site, -#' parameter_code = c("00060", "00065")) +#' dv_data <- read_USGS_dv(monitoring_location_id = site, +#' parameter_code = "00060", +#' no_sf = TRUE) #' -#' sites <- c("USGS-01491000", "USGS-01645000") -#' start_date <- "2018-01-01" -#' end_date <- "2022-01-01" -#' req_dv <- construct_dv_requests(monitoring_location_id = sites, -#' parameter_code = c("00060", "00065"), -#' datetime = c(start_date, end_date)) +#' multi_site <- read_USGS_dv(monitoring_location_id = c("USGS-01491000", +#' "USGS-01645000"), +#' parameter_code = c("00060", "00010"), +#' datetime = c("2023-01-01", "2024-01-01")) #' -construct_dv_requests <- function(monitoring_location_id = NA_character_, - parameter_code = NA_character_, - statistic_id = NA_character_, - properties = c("monitoring_location_id", - "parameter_code", - "statistic_id", - "time", - "value", - "unit_of_measure", - "approval_status", - "qualifier"), - bbox = NA, - timeseries_id = NA_character_, - id = NA_character_, - approval_status = NA_character_, - unit_of_measure = NA_character_, - qualifier = NA_character_, - value = NA, - last_modified = NA_character_, - limit = 10000, - crs = NA_character_, - bbox_crs = NA_character_, - skipGeometry = FALSE, - datetime = NA_character_ - ){ +#' } +read_USGS_dv <- function(monitoring_location_id = NA_character_, + parameter_code = NA_character_, + statistic_id = NA_character_, + properties = c("monitoring_location_id", + "parameter_code", + "statistic_id", + "time", + "value", + "unit_of_measure", + "approval_status", + "qualifier"), + bbox = NA, + timeseries_id = NA_character_, + id = NA_character_, + approval_status = NA_character_, + unit_of_measure = NA_character_, + qualifier = NA_character_, + value = NA, + last_modified = NA_character_, + limit = 10000, + crs = NA_character_, + bbox_crs = NA_character_, + skipGeometry = NA, + datetime = NA_character_, + no_sf = FALSE, + convertType = TRUE){ - schema <- check_OGC_requests(endpoint = "daily", - type = "schema") - all_properties <- names(schema$properties) + message("Function in development, use at your own risk.") - match.arg(properties, choices = c(all_properties, NA_character_), - several.ok = TRUE) + use_sf <- all(pkg.env$local_sf, !no_sf) - if(all(all_properties[!all_properties %in% c("id", "geometry")] %in% properties)) { - # Cleans up URL if we're asking for everything - properties <- NA_character_ + if(!use_sf){ + skipGeometry <- TRUE } - baseURL <- setup_api("daily") - - POST <- FALSE - - template_path_post <- system.file("templates/post.CQL2", package = "dataRetrieval") - template_post <- readChar(template_path_post, file.info(template_path_post)$size) - - post_params <- explode_post(list(monitoring_location_id = monitoring_location_id, + dv_req <- construct_api_requests(service = "daily", + monitoring_location_id = monitoring_location_id, parameter_code = parameter_code, statistic_id = statistic_id, + properties = properties, + bbox = bbox, timeseries_id = timeseries_id, id = id, approval_status = approval_status, unit_of_measure = unit_of_measure, qualifier = qualifier, - value = value)) - - if(length(post_params) > 0){ - POST = TRUE - } - - datetime <- format_api_dates(datetime) - - baseURL <- explode_query(baseURL, POST = FALSE, - list(last_modified = last_modified, - properties = properties, - limit = limit, - crs = crs, - bbox_crs = bbox_crs, - skipGeometry = skipGeometry, - datetime = datetime)) + value = value, + last_modified = last_modified, + limit = limit, + crs = crs, + bbox_crs = bbox_crs, + skipGeometry = skipGeometry, + datetime = datetime) - if(all(!is.na(bbox))){ - baseURL <- httr2::req_url_query(baseURL, - bbox = bbox, - .multi = "comma") - } - - if(!all(is.na(properties))){ - baseURL <- httr2::req_url_query(baseURL, - properties = properties, - .multi = "comma") - } - - if(POST){ - baseURL <- baseURL |> - httr2::req_headers(`Content-Type` = "application/query-cql-json") - - post_params <- list( - "params" = unname(post_params) - ) - - x <- whisker::whisker.render(template_post, post_params) - baseURL <- httr2::req_body_raw(baseURL, x) - - } else { - baseURL <- explode_query(baseURL, POST = FALSE, - list(monitoring_location_id = monitoring_location_id, - parameter_code = parameter_code, - statistic_id = statistic_id, - timeseries_id = timeseries_id, - id = id, - approval_status = approval_status, - unit_of_measure = unit_of_measure, - qualifier = qualifier, - value = value)) - } - - return(baseURL) -} - -setup_api <- function(service){ - baseURL <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> - httr2::req_url_path_append(service, "items") |> - httr2::req_user_agent(default_ua()) |> - httr2::req_headers(`Accept-Encoding` = c("compress", "gzip")) - - token <- Sys.getenv("API_USGS_PAT") - - if(token != ""){ - baseURL <- baseURL |> - httr2::req_headers_redacted(`X-Api-Key` = token) - } + return_list <- walk_pages(dv_req, use_sf) - baseURL <- explode_query(baseURL, POST = FALSE, - list(f = "json", - lang = "en-US")) + if(convertType) return_list <- cleanup_cols(return_list) -} - -format_api_dates <- function(datetime){ + return_list <- return_list[order(return_list$time, return_list$monitoring_location_id), ] - if(!any(is.na(datetime))){ - if(length(datetime) == 1){ - datetime <- format(datetime, format = "%Y-%m-%dT%H:%M:%SZ") - } else if (length(datetime) == 2) { - datetime <- as.POSIXct(datetime) - datetime <- paste0(vapply(datetime, FUN = function(x) { - format(x, format = "%Y-%m-%dT%H:%M:%SZ")}, - FUN.VALUE = c(NA_character_) - ), collapse = "/") - datetime <- gsub("NA", "..", datetime) - } else { - stop("datetime should only include 1-2 values") - } + if(!"id" %in% properties){ + return_list <- return_list[, names(return_list)[!names(return_list) %in% "id"]] } - return(datetime) -} - -explode_post <- function(ls){ - ls <- Filter(Negate(anyNA), ls) - params <- NULL - - if(max(lengths(ls)) > 1) { - - for(i in seq_along(ls)){ - params[names(ls[i])] <- cql2_param(ls[i]) - } - - if(length(params) > 1){ - params[seq_along(1:(length(params)-1))] <- paste0(params[seq_along(1:(length(params)-1))], ",") - } - } - return(params) + return(return_list) } -cql2_param <- function(parameter){ - template_path <- system.file("templates/param.CQL2", package = "dataRetrieval") - template <- readChar(template_path, file.info(template_path)$size) - - parameters <- paste0(unlist(parameter), collapse = '", "') - parameters <- paste0('"', parameters, '"') - parameter_list <- list("property" = names(parameter), - "parameter" = parameters) - return(whisker::whisker.render(template, parameter_list)) -} -#' Check OGC requests -#' -#' @param endpoint Character, can be "daily", "timeseries-metadata" -#' @param type Character, can be "queryables", "schema" -#' @export -#' @return list -#' @examplesIf is_dataRetrieval_user() -#' -#' \donttest{ -#' -#' dv_queryables <- check_OGC_requests(endpoint = "daily", -#' type = "queryables") -#' dv_schema <- check_OGC_requests(endpoint = "daily", -#' type = "schema") -#' ts_meta_queryables <- check_OGC_requests(endpoint = "timeseries-metadata", -#' type = "queryables") -#' ts_meta_schema <- check_OGC_requests(endpoint = "timeseries-metadata", -#' type = "schema") -#' } -check_OGC_requests <- function(endpoint = "daily", - type = "queryables"){ - - match.arg(endpoint, c("daily", "timeseries-metadata")) - match.arg(type, c("queryables", "schema")) - - check_req <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> - httr2::req_url_path_append(endpoint) |> - httr2::req_url_path_append(type) |> - httr2::req_user_agent(default_ua()) |> - httr2::req_url_query(f = "json", - lang = "en-US") - query_ret <- httr2::req_perform(check_req) |> - httr2::resp_body_json() - - return(query_ret) - -} \ No newline at end of file diff --git a/R/read_USGS_ts_meta.R b/R/read_USGS_ts_meta.R new file mode 100644 index 00000000..db19dfa1 --- /dev/null +++ b/R/read_USGS_ts_meta.R @@ -0,0 +1,125 @@ +#' Get USGS Timeseries metadata +#' +#' Daily data and continuous measurements are grouped into timeseries, which +#' represent a collection of observations of a single parameter, potentially +#' aggregated using a standard statistic, at a single site. This endpoint provides +#' metadata about those timeseries, including their operational thresholds, units +#' of measurement, and when the earliest and most recent observations in a timeseries +#' occurred. +#' +#' @export +#' @param parameter_code Parameter codes are 5-digit codes used to identify the +#' constituent measured and the units of measure. +#' @param parameter_name A human-understandable name corresponding to parameter_code. +#' @param statistic_id A code corresponding to the statistic an observation represents. +#' Example codes include 00001 (max), 00002 (min), and 00003 (mean). +#' @param last_modified The last time a record was refreshed in our database. This +#' may happen due to regular operational processes and does not necessarily indicate +#' anything about the measurement has changed. You can query this field using +#' date-times or intervals. +#' @param begin The datetime of the earliest observation in the timeseries. +#' Together with end, this field represents the period of record of a timeseries. +#' Note that some timeseries may have large gaps in their collection record. +#' @param end The datetime of the most recent observation in the timeseries. +#' Data returned by this endpoint updates at most once per day, and potentially +#' less frequently than that, and as such there may be more recent observations +#' within a timeseries than the timeseries end value reflects. Together with begin, +#' this field represents the period of record of a timeseries. It is additionally +#' used to determine whether a timeseries is "active". +#' @param computation_period_identifier Indicates the period of data used for +#' any statistical computations. +#' @param computation_identifier Indicates whether the data from this timeseries +#' represent a specific statistical computation. +#' @param thresholds Thresholds represent known numeric limits for a timeseries, +#' for example the historic maximum value for a parameter or a level below which +#' a sensor is non-operative. These thresholds are sometimes used to automatically +#' determine if an observation is erroneous due to sensor error, and therefore +#' shouldn't be included in the timeseries. +#' @param sublocation_identifier An optional human-readable identifier used to +#' specify where measurements are recorded at a monitoring location. +#' @param primary A flag identifying if the timeseries is a "primary" timeseries. +#' "Primary" timeseries (which have this flag) are standard observations which +#' undergo Bureau review and approval processes. Non-primary timeseries, which +#' will have missing values for "primary", are provisional datasets made available +#' to meet the need for timely best science and to assist with daily operations +#' which need real-time information. Non-primary timeseries data are only retained +#' by this system for 120 days. See the USGS Provisional Data Statement for more information. +#' @param web_description A description of what this timeseries represents, as +#' used by WDFN and other USGS data dissemination products. +#' @param crs Indicates the coordinate reference system for the results. +#' @param limit The optional limit parameter limits the number of items that are +#' presented in the response document. Only items are counted that are on the +#' first level of the collection in the response document. Nested objects +#' contained within the explicitly requested items shall not be counted. +#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function +#' will convert the data to dates and qualifier to string vector. +#' @examplesIf is_dataRetrieval_user() +#' +#' \donttest{ +#' site <- "USGS-02238500" +#' meta_1 <- read_NWIS_ts_meta(monitoring_location_id = site) +#' } +read_NWIS_ts_meta <- function(monitoring_location_id = NA_character_, + parameter_code = NA_character_, + parameter_name = NA_character_, + properties = c("monitoring_location_id", + "unit_of_measure", + "parameter_name", + "begin", + "end", + "primary", + "computation_identifier", + "sublocation_identifier", + "id", + "last_modified", + "web_description"), + limit = 10000, + statistic_id = NA_character_, + last_modified = NA_character_, + begin = NA_character_, + end = NA_character_, + unit_of_measure = NA_character_, + computation_period_identifier = NA_character_, + computation_identifier = NA_character_, + thresholds = NA, + sublocation_identifier = NA_character_, + primary = NA_character_, + id = NA_character_, + web_description = NA_character_, + use_sf = TRUE, + convertType = FALSE){ + + message("Function in development, use at your own risk.") + + req_ts_meta <- construct_api_requests("timeseries-metadata", + monitoring_location_id = monitoring_location_id, + parameter_code = parameter_code, + parameter_name = parameter_name, + properties = properties, + statistic_id = statistic_id, + last_modified = last_modified, + begin = begin, + end = end, + limit = limit, + unit_of_measure = unit_of_measure, + computation_period_identifier = computation_period_identifier, + computation_identifier = computation_identifier, + thresholds = thresholds, + sublocation_identifier = sublocation_identifier, + primary = primary, + id = id, + web_description = web_description) + + return_list <- walk_pages(req_ts_meta, use_sf) + + return_list <- return_list[, properties] + + if(convertType) return_list <- cleanup_cols(return_list) + + if(!"id" %in% properties){ + return_list <- return_list[, names(return_list)[!names(return_list) %in% "id"]] + } + + return(return_list) + +} \ No newline at end of file diff --git a/R/walk_pages.R b/R/walk_pages.R new file mode 100644 index 00000000..51a6cdb3 --- /dev/null +++ b/R/walk_pages.R @@ -0,0 +1,88 @@ +cleanup_cols <- function(df){ + + if("qualifier" %in% names(df)){ + if(!all(is.na(df$qualifier))){ + df$qualifier <- vapply(X = df$qualifier, + FUN = function(x) paste(x, collapse = ", "), + FUN.VALUE = c(NA_character_)) + } + } + + if("time" %in% names(df)){ + df$time <- as.Date(df$time) + } + + if("value" %in% names(df)){ + df$value <- as.numeric(df$value) + } + df +} + +walk_pages <- function(req, use_sf) { + + walk_pages_recursive( + req = req, + page = 1, + contents = list(), + use_sf + ) +} + +walk_pages_recursive <- function(req, page, contents, use_sf) { + + url_method <- "GET" + if(!is.null(req$body)){ + url_method <- "POST" + body <- req$body + } + message(url_method, ": ", req$url) + + returned_contents <- httr2::req_perform(req) + + if(httr2::resp_status(returned_contents) != 200){ + if(httr2::resp_status(returned_contents) == 429){ + stop("You hit your hourly limit for requests. To increase the limit, + see: https://api.waterdata.usgs.gov/docs/ogcapi/keys/") + } + } + + header_info <- httr2::resp_headers(returned_contents) + message("Remaining requests this hour:", header_info$`x-ratelimit-remaining`) + + contents[[page]] <- returned_contents |> + httr2::resp_body_string() + + json_content <- jsonlite::fromJSON(contents[[page]] ) + + next_page <- json_content$links[, "rel", drop = TRUE] == "next" + + if (!any(next_page)) { + + if(use_sf){ + return_df <- lapply(contents, function(x) { + df_i <- sf::read_sf(x) + }) |> + do.call(what = rbind) + } else { + return_df <- lapply(contents, function(x) { + df_i <- jsonlite::fromJSON(x)[["features"]][["properties"]] + }) |> + do.call(what = rbind) + } + + return(return_df) + + } else { + + make_request <- httr2::req_url(req = req, + url = json_content$links[, "href", drop = TRUE][next_page]) + + Tailcall( + walk_pages_recursive, + make_request, + page + 1, + contents, + use_sf + ) + } +} \ No newline at end of file diff --git a/man/check_OGC_requests.Rd b/man/check_OGC_requests.Rd index e91f88ea..cdf04d90 100644 --- a/man/check_OGC_requests.Rd +++ b/man/check_OGC_requests.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_USGS_dv.R +% Please edit documentation in R/construct_api_requests.R \name{check_OGC_requests} \alias{check_OGC_requests} \title{Check OGC requests} diff --git a/man/construct_dv_requests.Rd b/man/construct_api_requests.Rd similarity index 62% rename from man/construct_dv_requests.Rd rename to man/construct_api_requests.Rd index dae15d14..f6645796 100644 --- a/man/construct_dv_requests.Rd +++ b/man/construct_api_requests.Rd @@ -1,10 +1,11 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_USGS_dv.R -\name{construct_dv_requests} -\alias{construct_dv_requests} -\title{Create DV url} +% Please edit documentation in R/construct_api_requests.R +\name{construct_api_requests} +\alias{construct_api_requests} +\title{Create API url} \usage{ -construct_dv_requests( +construct_api_requests( + service, monitoring_location_id = NA_character_, parameter_code = NA_character_, statistic_id = NA_character_, @@ -22,8 +23,16 @@ construct_dv_requests( crs = NA_character_, bbox_crs = NA_character_, skipGeometry = FALSE, - offset = NA, - datetime = NA_character_ + datetime = NA_character_, + begin = NA_character_, + end = NA_character_, + primary = NA_character_, + parameter_name = NA_character_, + thresholds = NA, + sublocation_identifier = NA_character_, + computation_period_identifier = NA_character_, + computation_identifier = NA_character_, + web_description = NA_character_ ) } \arguments{ @@ -95,16 +104,55 @@ coordinates.} \item{skipGeometry}{This option can be used to skip response geometries for each feature.} -\item{offset}{The optional offset parameter indicates the index within the -result set from which the server shall begin presenting results in the response -document. The first element has an index of 0 (default).} - \item{datetime}{Either a date-time or an interval. Only features that have a temporal property that intersects the value of datetime are selected. If a feature has multiple temporal properties, it is the decision of the server whether only a single temporal property is used to determine the extent or all relevant temporal properties.} +\item{begin}{The datetime of the earliest observation in the timeseries. +Together with end, this field represents the period of record of a timeseries. +Note that some timeseries may have large gaps in their collection record.} + +\item{end}{The datetime of the most recent observation in the timeseries. +Data returned by this endpoint updates at most once per day, and potentially +less frequently than that, and as such there may be more recent observations +within a timeseries than the timeseries end value reflects. Together with begin, +this field represents the period of record of a timeseries. It is additionally +used to determine whether a timeseries is "active".} + +\item{primary}{A flag identifying if the timeseries is a "primary" timeseries. +"Primary" timeseries (which have this flag) are standard observations which +undergo Bureau review and approval processes. Non-primary timeseries, which +will have missing values for "primary", are provisional datasets made available +to meet the need for timely best science and to assist with daily operations +which need real-time information. Non-primary timeseries data are only retained +by this system for 120 days. See the USGS Provisional Data Statement for more information.} + +\item{parameter_name}{A human-understandable name corresponding to parameter_code.} + +\item{thresholds}{Thresholds represent known numeric limits for a timeseries, +for example the historic maximum value for a parameter or a level below which +a sensor is non-operative. These thresholds are sometimes used to automatically +determine if an observation is erroneous due to sensor error, and therefore +shouldn't be included in the timeseries.} + +\item{sublocation_identifier}{An optional human-readable identifier used to +specify where measurements are recorded at a monitoring location.} + +\item{computation_period_identifier}{Indicates the period of data used for +any statistical computations.} + +\item{computation_identifier}{Indicates whether the data from this timeseries +represent a specific statistical computation.} + +\item{web_description}{A description of what this timeseries represents, as +used by WDFN and other USGS data dissemination products.} + +\item{offset}{The optional offset parameter indicates the index within the +result set from which the server shall begin presenting results in the response +document. The first element has an index of 0 (default).} + \item{time}{The date an observation represents.} } \description{ @@ -114,17 +162,21 @@ Swagger docs: \url{https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html}. \examples{ site <- "USGS-02238500" pcode <- "00060" -req_dv <- construct_dv_requests(monitoring_location_id = site, - parameter_code = "00060") +req_dv <- construct_api_requests("daily", + monitoring_location_id = site, + parameter_code = "00060") -req_dv <- construct_dv_requests(monitoring_location_id = site, - parameter_code = c("00060", "00065")) +req_dv <- construct_api_requests("daily", + monitoring_location_id = site, + parameter_code = c("00060", "00065")) sites <- c("USGS-01491000", "USGS-01645000") start_date <- "2018-01-01" end_date <- "2022-01-01" -req_dv <- construct_dv_requests(monitoring_location_id = sites, +req_dv <- construct_api_requests("daily", + monitoring_location_id = sites, parameter_code = c("00060", "00065"), datetime = c(start_date, end_date)) } +\keyword{internal} diff --git a/man/read_NWIS_ts_meta.Rd b/man/read_NWIS_ts_meta.Rd new file mode 100644 index 00000000..1eda79b6 --- /dev/null +++ b/man/read_NWIS_ts_meta.Rd @@ -0,0 +1,108 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/read_USGS_ts_meta.R +\name{read_NWIS_ts_meta} +\alias{read_NWIS_ts_meta} +\title{Get USGS Timeseries metadata} +\usage{ +read_NWIS_ts_meta( + monitoring_location_id = NA_character_, + parameter_code = NA_character_, + parameter_name = NA_character_, + properties = c("monitoring_location_id", "unit_of_measure", "parameter_name", "begin", + "end", "primary", "computation_identifier", "sublocation_identifier", "id", + "last_modified", "web_description"), + limit = 10000, + statistic_id = NA_character_, + last_modified = NA_character_, + begin = NA_character_, + end = NA_character_, + unit_of_measure = NA_character_, + computation_period_identifier = NA_character_, + computation_identifier = NA_character_, + thresholds = NA, + sublocation_identifier = NA_character_, + primary = NA_character_, + id = NA_character_, + web_description = NA_character_, + use_sf = TRUE, + convertType = FALSE +) +} +\arguments{ +\item{parameter_code}{Parameter codes are 5-digit codes used to identify the +constituent measured and the units of measure.} + +\item{parameter_name}{A human-understandable name corresponding to parameter_code.} + +\item{limit}{The optional limit parameter limits the number of items that are +presented in the response document. Only items are counted that are on the +first level of the collection in the response document. Nested objects +contained within the explicitly requested items shall not be counted.} + +\item{statistic_id}{A code corresponding to the statistic an observation represents. +Example codes include 00001 (max), 00002 (min), and 00003 (mean).} + +\item{last_modified}{The last time a record was refreshed in our database. This +may happen due to regular operational processes and does not necessarily indicate +anything about the measurement has changed. You can query this field using +date-times or intervals.} + +\item{begin}{The datetime of the earliest observation in the timeseries. +Together with end, this field represents the period of record of a timeseries. +Note that some timeseries may have large gaps in their collection record.} + +\item{end}{The datetime of the most recent observation in the timeseries. +Data returned by this endpoint updates at most once per day, and potentially +less frequently than that, and as such there may be more recent observations +within a timeseries than the timeseries end value reflects. Together with begin, +this field represents the period of record of a timeseries. It is additionally +used to determine whether a timeseries is "active".} + +\item{computation_period_identifier}{Indicates the period of data used for +any statistical computations.} + +\item{computation_identifier}{Indicates whether the data from this timeseries +represent a specific statistical computation.} + +\item{thresholds}{Thresholds represent known numeric limits for a timeseries, +for example the historic maximum value for a parameter or a level below which +a sensor is non-operative. These thresholds are sometimes used to automatically +determine if an observation is erroneous due to sensor error, and therefore +shouldn't be included in the timeseries.} + +\item{sublocation_identifier}{An optional human-readable identifier used to +specify where measurements are recorded at a monitoring location.} + +\item{primary}{A flag identifying if the timeseries is a "primary" timeseries. +"Primary" timeseries (which have this flag) are standard observations which +undergo Bureau review and approval processes. Non-primary timeseries, which +will have missing values for "primary", are provisional datasets made available +to meet the need for timely best science and to assist with daily operations +which need real-time information. Non-primary timeseries data are only retained +by this system for 120 days. See the USGS Provisional Data Statement for more information.} + +\item{web_description}{A description of what this timeseries represents, as +used by WDFN and other USGS data dissemination products.} + +\item{convertType}{logical, defaults to \code{TRUE}. If \code{TRUE}, the function +will convert the data to dates and qualifier to string vector.} + +\item{crs}{Indicates the coordinate reference system for the results.} +} +\description{ +Daily data and continuous measurements are grouped into timeseries, which +represent a collection of observations of a single parameter, potentially +aggregated using a standard statistic, at a single site. This endpoint provides +metadata about those timeseries, including their operational thresholds, units +of measurement, and when the earliest and most recent observations in a timeseries +occurred. +} +\examples{ +\dontshow{if (is_dataRetrieval_user()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} + +\donttest{ +site <- "USGS-02238500" +meta_1 <- read_NWIS_ts_meta(monitoring_location_id = site) +} +\dontshow{\}) # examplesIf} +} diff --git a/man/read_USGS_dv.Rd b/man/read_USGS_dv.Rd index 56bef52e..72b92504 100644 --- a/man/read_USGS_dv.Rd +++ b/man/read_USGS_dv.Rd @@ -22,7 +22,6 @@ read_USGS_dv( crs = NA_character_, bbox_crs = NA_character_, skipGeometry = NA, - offset = NA, datetime = NA_character_, no_sf = FALSE, convertType = TRUE @@ -97,10 +96,6 @@ coordinates.} \item{skipGeometry}{This option can be used to skip response geometries for each feature.} -\item{offset}{The optional offset parameter indicates the index within the -result set from which the server shall begin presenting results in the response -document. The first element has an index of 0 (default).} - \item{datetime}{Either a date-time or an interval. Only features that have a temporal property that intersects the value of datetime are selected. If a feature has multiple temporal properties, it is the decision of the server @@ -111,7 +106,13 @@ all relevant temporal properties.} a basic data frame, FALSE returns a "sf" object.} \item{convertType}{logical, defaults to \code{TRUE}. If \code{TRUE}, the function -will convert the data to dates, datetimes,} +will convert the data to dates and qualifier to string vector.} + +\item{offset}{The optional offset parameter indicates the index within the +result set from which the server shall begin presenting results in the response +document. The first element has an index of 0 (default).} + +\item{time}{The date an observation represents.} } \description{ Daily data provide one data value to represent water conditions for the day. @@ -134,7 +135,14 @@ pcode <- "00060" dv_data_sf <- read_USGS_dv(monitoring_location_id = site, parameter_code = "00060", datetime = c("2021-01-01", "2022-01-01")) - + +dv_data_sf <- read_USGS_dv(monitoring_location_id = site, + parameter_code = "00060", + properties = c("monitoring_location_id", + "value", + "time"), + datetime = c("2021-01-01", "2022-01-01")) + dv_data <- read_USGS_dv(monitoring_location_id = site, parameter_code = "00060", no_sf = TRUE) @@ -144,8 +152,6 @@ multi_site <- read_USGS_dv(monitoring_location_id = c("USGS-01491000", parameter_code = c("00060", "00010"), datetime = c("2023-01-01", "2024-01-01")) -bbox_data <- read_USGS_dv(bbox = c(-83, 36.5, -82.5, 37.5), - parameter_code = "00060") } \dontshow{\}) # examplesIf} } diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd new file mode 100644 index 00000000..9d0ea4de --- /dev/null +++ b/vignettes/read_USGS_functions.Rmd @@ -0,0 +1,118 @@ +--- +title: "Introducing read_USGS_dv" +author: Laura A. DeCicco +editor_options: + chunk_output_type: console +output: + rmarkdown::html_vignette: + toc: true + number_sections: true +vignette: > + %\VignetteIndexEntry{Introducing read_USGS_dv} + \usepackage[utf8]{inputenc} + %\VignetteEngine{knitr::rmarkdown} +--- + + +```{r setup, include=FALSE, message=FALSE} +library(knitr) +library(dataRetrieval) +library(dplyr) +library(ggplot2) + +options(continue = " ", + width = 50) + +knitr::opts_chunk$set( + echo = TRUE, + message = TRUE, + fig.height = 4, + fig.width = 7 +) +``` + + +As we bid adieu to the NWIS web services, we welcome a new web service offering, the USGS Water Data OGC APIs. + +# New USGS data access + +This is a modern access point for USGS data. The USGS is planning to modernize all web services in the near future. For each of these updates, `dataRetrieval` will create a new function to access the new services. + +# Daily Values + +To access these services on a web browser, go to . + +The `read_USGS_dv` function will eventually replace the `readNWISdv` function. Generally, both functions look pretty similar: + +```{r} +library(dataRetrieval) + +dv_legacy <- readNWISdv(siteNumbers = "01491000", + parameterCd = c("00060", "00010"), + statCd = "00003", + startDate = "2023-01-01", + endDate = "2025-05-06") + +nrow(dv_legacy) + +dv_modern <- read_USGS_dv(monitoring_location_id = "USGS-01491000", + parameter_code = c("00060", "00010"), + statistic_id = "00003", + datetime = c("2023-01-01", "2025-05-06")) + +nrow(dv_modern) +``` +The first thing you might notice is that the legacy service serves data in a "wide" format, which means there are multiple observations in a single row of the data frame. The new services return the data in a "long" format, so that only one observation is returned per row. This follows tidy principles and is often preferred in scripting languages. We'll look at the returned data with a `ggplot2` plot to show a very simple example of the power of a long format: + +```{r} +library(ggplot2) +ggplot(data = dv_modern) + + geom_point(aes(x = time, y = value, + color = approval_status)) + + facet_grid(parameter_code ~ ., scale = "free") + + theme_bw() + +``` + +## Additional new features + +* Another feature of the modern returned data frame is that it has a "geometry" column which is an `sf` class and can therefore be used in `sf` processes. + +* Flexible column selection: + +```{r} +dv_2 <- read_USGS_dv(monitoring_location_id = "USGS-01491000", + parameter_code = c("00060"), + properties = c("monitoring_location_id", + "value", + "time"), + statistic_id = "00003", + datetime = c("2022-10-01", "2023-10-01")) +head(dv_2) +``` + +* When you look at the help file for the new function `?read_USGS_dv`, you'll also notice there are many more arguments. These are mostly set by default to NA. You DO NOT need to specify all of these parameters. + + + + +# API Tokens + + + +# Notes on dataRetrieval development + +## New Features + +### Style + +New functions will use a "snake case", such as "read_USGS_samples". Older functions use camel case, such as "readNWISdv". The difference is the underscore between words. This should be a handy way to tell the difference between newer modern data access, and the older traditional functions. + +### Structure + +Historically, we allowed users to customize their queries via the `...` argument structure. With `...`, users needed to know the exact names of query parameters before using the function. Now, the new functions will include **ALL** possible arguments that the web service APIs support. This will allow users to use tab-autocompletes (available in RStudio and other IDEs). **Users will need to understand that it is not advisable to specify all of these parameters. The systems can get bogged down with redundant query parameters.** We expect this will be easier for users, but it might take some time to smooth out the documentation and test usability. There may be additional consequences, such as users won't be able to build up argument lists to pass into the function. + +### Dependencies + +Under the hood, `dataRetrieval` changed the dependency from `httr` to `httr2`. `httr2` is the modern R package for web requests that is actively developed/maintained. As we develop functions for the modern USGS web services, we'll continue to explore updating package dependencies. Since the new services offer geospatial output, we have recommend using the `sf` package. The `whisker` package was also included to help create POST CQL2 queries. + From cb8ac47f8426716fffb269bb9a323a7b5d1ee452 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 7 May 2025 15:46:46 -0500 Subject: [PATCH 012/117] Get rid of some warnings --- R/construct_api_requests.R | 3 --- R/read_USGS_dv.R | 8 +------- R/read_USGS_ts_meta.R | 33 ++++++++++++++++++++++++++++++--- man/construct_api_requests.Rd | 4 ---- man/read_NWIS_ts_meta.Rd | 30 ++++++++++++++++++++++++++---- man/read_USGS_dv.Rd | 8 -------- 6 files changed, 57 insertions(+), 29 deletions(-) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index a2f8738f..bea4c88a 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -22,9 +22,6 @@ #' value, unit_of_measure, approval_status, qualifier, last_modified. #' @param skipGeometry This option can be used to skip response geometries for #' each feature. -#' @param offset The optional offset parameter indicates the index within the -#' result set from which the server shall begin presenting results in the response -#' document. The first element has an index of 0 (default). #' @param datetime Either a date-time or an interval. Only features that have a #' temporal property that intersects the value of datetime are selected. If a #' feature has multiple temporal properties, it is the decision of the server diff --git a/R/read_USGS_dv.R b/R/read_USGS_dv.R index 0acb3d83..e8096cf6 100644 --- a/R/read_USGS_dv.R +++ b/R/read_USGS_dv.R @@ -30,9 +30,6 @@ #' value, unit_of_measure, approval_status, qualifier, last_modified. #' @param skipGeometry This option can be used to skip response geometries for #' each feature. -#' @param offset The optional offset parameter indicates the index within the -#' result set from which the server shall begin presenting results in the response -#' document. The first element has an index of 0 (default). #' @param datetime Either a date-time or an interval. Only features that have a #' temporal property that intersects the value of datetime are selected. If a #' feature has multiple temporal properties, it is the decision of the server @@ -74,8 +71,6 @@ #' presented in the response document. Only items are counted that are on the #' first level of the collection in the response document. Nested objects #' contained within the explicitly requested items shall not be counted. -#' @param no_sf Boolean, whether or not to return an "sf" object. TRUE returns -#' a basic data frame, FALSE returns a "sf" object. #' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function #' will convert the data to dates and qualifier to string vector. #' @examplesIf is_dataRetrieval_user() @@ -128,12 +123,11 @@ read_USGS_dv <- function(monitoring_location_id = NA_character_, bbox_crs = NA_character_, skipGeometry = NA, datetime = NA_character_, - no_sf = FALSE, convertType = TRUE){ message("Function in development, use at your own risk.") - use_sf <- all(pkg.env$local_sf, !no_sf) + use_sf <- all(pkg.env$local_sf) if(!use_sf){ skipGeometry <- TRUE diff --git a/R/read_USGS_ts_meta.R b/R/read_USGS_ts_meta.R index db19dfa1..2c1d8cf0 100644 --- a/R/read_USGS_ts_meta.R +++ b/R/read_USGS_ts_meta.R @@ -8,9 +8,21 @@ #' occurred. #' #' @export +#' @param monitoring_location_id A unique identifier representing a single monitoring +#' location. This corresponds to the id field in the sites endpoint. Monitoring +#' location IDs are created by combining the agency code of the agency responsible +#' for the monitoring location (e.g. USGS) with the ID number of the monitoring +#' location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500). +#' @param properties The properties that should be included for each feature. The +#' parameter value is a comma-separated list of property names. Available values: +#' geometry, id, unit_of_measure, parameter_name, parameter_code, statistic_id, +#' last_modified, begin, end, computation_period_identifier, computation_identifier, +#' thresholds, sublocation_identifier, primary, monitoring_location_id, web_description. #' @param parameter_code Parameter codes are 5-digit codes used to identify the #' constituent measured and the units of measure. #' @param parameter_name A human-understandable name corresponding to parameter_code. +#' @param unit_of_measure A human-readable description of the units of measurement +#' associated with an observation. #' @param statistic_id A code corresponding to the statistic an observation represents. #' Example codes include 00001 (max), 00002 (min), and 00003 (mean). #' @param last_modified The last time a record was refreshed in our database. This @@ -53,22 +65,30 @@ #' contained within the explicitly requested items shall not be counted. #' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function #' will convert the data to dates and qualifier to string vector. +#' @param skipGeometry This option can be used to skip response geometries for +#' each feature. #' @examplesIf is_dataRetrieval_user() #' #' \donttest{ #' site <- "USGS-02238500" #' meta_1 <- read_NWIS_ts_meta(monitoring_location_id = site) +#' +#' meta_multi <- read_NWIS_ts_meta(monitoring_location_id = c("USGS-01491000", +#' "USGS-01645000"), +#' parameter_code = c("00060", "00010")) #' } read_NWIS_ts_meta <- function(monitoring_location_id = NA_character_, parameter_code = NA_character_, parameter_name = NA_character_, properties = c("monitoring_location_id", "unit_of_measure", + "parameter_code", "parameter_name", + "statistic_id", + "computation_identifier", "begin", "end", "primary", - "computation_identifier", "sublocation_identifier", "id", "last_modified", @@ -86,11 +106,17 @@ read_NWIS_ts_meta <- function(monitoring_location_id = NA_character_, primary = NA_character_, id = NA_character_, web_description = NA_character_, - use_sf = TRUE, + skipGeometry = NA, convertType = FALSE){ message("Function in development, use at your own risk.") + use_sf <- all(pkg.env$local_sf) + + if(!use_sf){ + skipGeometry <- TRUE + } + req_ts_meta <- construct_api_requests("timeseries-metadata", monitoring_location_id = monitoring_location_id, parameter_code = parameter_code, @@ -108,7 +134,8 @@ read_NWIS_ts_meta <- function(monitoring_location_id = NA_character_, sublocation_identifier = sublocation_identifier, primary = primary, id = id, - web_description = web_description) + web_description = web_description, + skipGeometry = skipGeometry) return_list <- walk_pages(req_ts_meta, use_sf) diff --git a/man/construct_api_requests.Rd b/man/construct_api_requests.Rd index f6645796..4c789e03 100644 --- a/man/construct_api_requests.Rd +++ b/man/construct_api_requests.Rd @@ -149,10 +149,6 @@ represent a specific statistical computation.} \item{web_description}{A description of what this timeseries represents, as used by WDFN and other USGS data dissemination products.} -\item{offset}{The optional offset parameter indicates the index within the -result set from which the server shall begin presenting results in the response -document. The first element has an index of 0 (default).} - \item{time}{The date an observation represents.} } \description{ diff --git a/man/read_NWIS_ts_meta.Rd b/man/read_NWIS_ts_meta.Rd index 1eda79b6..0fbfb6e9 100644 --- a/man/read_NWIS_ts_meta.Rd +++ b/man/read_NWIS_ts_meta.Rd @@ -8,9 +8,9 @@ read_NWIS_ts_meta( monitoring_location_id = NA_character_, parameter_code = NA_character_, parameter_name = NA_character_, - properties = c("monitoring_location_id", "unit_of_measure", "parameter_name", "begin", - "end", "primary", "computation_identifier", "sublocation_identifier", "id", - "last_modified", "web_description"), + properties = c("monitoring_location_id", "unit_of_measure", "parameter_code", + "parameter_name", "statistic_id", "computation_identifier", "begin", "end", + "primary", "sublocation_identifier", "id", "last_modified", "web_description"), limit = 10000, statistic_id = NA_character_, last_modified = NA_character_, @@ -24,16 +24,28 @@ read_NWIS_ts_meta( primary = NA_character_, id = NA_character_, web_description = NA_character_, - use_sf = TRUE, + skipGeometry = NA, convertType = FALSE ) } \arguments{ +\item{monitoring_location_id}{A unique identifier representing a single monitoring +location. This corresponds to the id field in the sites endpoint. Monitoring +location IDs are created by combining the agency code of the agency responsible +for the monitoring location (e.g. USGS) with the ID number of the monitoring +location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500).} + \item{parameter_code}{Parameter codes are 5-digit codes used to identify the constituent measured and the units of measure.} \item{parameter_name}{A human-understandable name corresponding to parameter_code.} +\item{properties}{The properties that should be included for each feature. The +parameter value is a comma-separated list of property names. Available values: +geometry, id, unit_of_measure, parameter_name, parameter_code, statistic_id, +last_modified, begin, end, computation_period_identifier, computation_identifier, +thresholds, sublocation_identifier, primary, monitoring_location_id, web_description.} + \item{limit}{The optional limit parameter limits the number of items that are presented in the response document. Only items are counted that are on the first level of the collection in the response document. Nested objects @@ -58,6 +70,9 @@ within a timeseries than the timeseries end value reflects. Together with begin, this field represents the period of record of a timeseries. It is additionally used to determine whether a timeseries is "active".} +\item{unit_of_measure}{A human-readable description of the units of measurement +associated with an observation.} + \item{computation_period_identifier}{Indicates the period of data used for any statistical computations.} @@ -84,6 +99,9 @@ by this system for 120 days. See the USGS Provisional Data Statement for more in \item{web_description}{A description of what this timeseries represents, as used by WDFN and other USGS data dissemination products.} +\item{skipGeometry}{This option can be used to skip response geometries for +each feature.} + \item{convertType}{logical, defaults to \code{TRUE}. If \code{TRUE}, the function will convert the data to dates and qualifier to string vector.} @@ -103,6 +121,10 @@ occurred. \donttest{ site <- "USGS-02238500" meta_1 <- read_NWIS_ts_meta(monitoring_location_id = site) + +meta_multi <- read_NWIS_ts_meta(monitoring_location_id = c("USGS-01491000", + "USGS-01645000"), + parameter_code = c("00060", "00010")) } \dontshow{\}) # examplesIf} } diff --git a/man/read_USGS_dv.Rd b/man/read_USGS_dv.Rd index 72b92504..3007cfb2 100644 --- a/man/read_USGS_dv.Rd +++ b/man/read_USGS_dv.Rd @@ -23,7 +23,6 @@ read_USGS_dv( bbox_crs = NA_character_, skipGeometry = NA, datetime = NA_character_, - no_sf = FALSE, convertType = TRUE ) } @@ -102,16 +101,9 @@ feature has multiple temporal properties, it is the decision of the server whether only a single temporal property is used to determine the extent or all relevant temporal properties.} -\item{no_sf}{Boolean, whether or not to return an "sf" object. TRUE returns -a basic data frame, FALSE returns a "sf" object.} - \item{convertType}{logical, defaults to \code{TRUE}. If \code{TRUE}, the function will convert the data to dates and qualifier to string vector.} -\item{offset}{The optional offset parameter indicates the index within the -result set from which the server shall begin presenting results in the response -document. The first element has an index of 0 (default).} - \item{time}{The date an observation represents.} } \description{ From 4db8f73346c344bfc2143730203327e1a9519de4 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 7 May 2025 16:36:18 -0500 Subject: [PATCH 013/117] cleanup a few more things --- .Rbuildignore | 1 + R/read_USGS_ts_meta.R | 4 +++ _pkgdown.yml | 16 ++++++---- docker/Dockerfile | 1 + man/read_NWIS_ts_meta.Rd | 8 +++-- vignettes/read_USGS_functions.Rmd | 53 +++++++++++++++++++++++++++++++ 6 files changed, 74 insertions(+), 9 deletions(-) diff --git a/.Rbuildignore b/.Rbuildignore index 91be94e2..8c7cdb4f 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -90,6 +90,7 @@ vignettes/Status.Rmd vignettes/long_to_wide.Rmd vignettes/join_by_closest.Rmd vignettes/wqx3_development_plan.Rmd +vignettes/read_USGS_functions.Rmd vignettes/dataretrieval_discrete_changes_images/* ^ci$ ^public$ diff --git a/R/read_USGS_ts_meta.R b/R/read_USGS_ts_meta.R index 2c1d8cf0..c784b60f 100644 --- a/R/read_USGS_ts_meta.R +++ b/R/read_USGS_ts_meta.R @@ -38,6 +38,8 @@ #' within a timeseries than the timeseries end value reflects. Together with begin, #' this field represents the period of record of a timeseries. It is additionally #' used to determine whether a timeseries is "active". +#' @param id A unique identifier representing a single timeseries. This corresponds +#' to the id field in the timeseries-metadata endpoint. #' @param computation_period_identifier Indicates the period of data used for #' any statistical computations. #' @param computation_identifier Indicates whether the data from this timeseries @@ -106,6 +108,7 @@ read_NWIS_ts_meta <- function(monitoring_location_id = NA_character_, primary = NA_character_, id = NA_character_, web_description = NA_character_, + crs = NA_character_, skipGeometry = NA, convertType = FALSE){ @@ -134,6 +137,7 @@ read_NWIS_ts_meta <- function(monitoring_location_id = NA_character_, sublocation_identifier = sublocation_identifier, primary = primary, id = id, + crs = crs, web_description = web_description, skipGeometry = skipGeometry) diff --git a/_pkgdown.yml b/_pkgdown.yml index ed98dac0..79712d39 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -64,6 +64,15 @@ navbar: - icon: fa-github fa-lg href: https://github.com/DOI-USGS/dataRetrieval reference: + - title: USGS new data services + desc: Functions to retrieve USGS data from new services. + contents: + - read_USGS_samples + - read_USGS_dv + - read_USGS_ts_meta + - summarize_USGS_samples + - construct_USGS_sample_request + - check_param - title: National Water Information System (NWIS) desc: Functions to retrieve (USGS) NWIS data. These will be slowly phased out and replaced with the read_USGS family of functions. contents: @@ -91,13 +100,6 @@ reference: - whatWQPmetrics - readWQPsummary - wqp_check_status - - title: USGS new data services - desc: Functions to retrieve USGS data from new services. - contents: - - read_USGS_samples - - summarize_USGS_samples - - construct_USGS_sample_request - - check_param - title: National Ground-Water Monitoring Network desc: Functions to retrieve NGWMN data. contents: diff --git a/docker/Dockerfile b/docker/Dockerfile index c4adc2df..740f1e12 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -25,6 +25,7 @@ RUN apt-get update -qq && apt-get -y --no-install-recommends install \ r-cran-leaflet \ r-cran-readxl \ r-cran-whisker \ + r-cran-ggplot2 \ && apt-get install -y pandoc \ && rm -rf /var/lib/apt/lists/* diff --git a/man/read_NWIS_ts_meta.Rd b/man/read_NWIS_ts_meta.Rd index 0fbfb6e9..512e83b8 100644 --- a/man/read_NWIS_ts_meta.Rd +++ b/man/read_NWIS_ts_meta.Rd @@ -24,6 +24,7 @@ read_NWIS_ts_meta( primary = NA_character_, id = NA_character_, web_description = NA_character_, + crs = NA_character_, skipGeometry = NA, convertType = FALSE ) @@ -96,16 +97,19 @@ to meet the need for timely best science and to assist with daily operations which need real-time information. Non-primary timeseries data are only retained by this system for 120 days. See the USGS Provisional Data Statement for more information.} +\item{id}{A unique identifier representing a single timeseries. This corresponds +to the id field in the timeseries-metadata endpoint.} + \item{web_description}{A description of what this timeseries represents, as used by WDFN and other USGS data dissemination products.} +\item{crs}{Indicates the coordinate reference system for the results.} + \item{skipGeometry}{This option can be used to skip response geometries for each feature.} \item{convertType}{logical, defaults to \code{TRUE}. If \code{TRUE}, the function will convert the data to dates and qualifier to string vector.} - -\item{crs}{Indicates the coordinate reference system for the results.} } \description{ Daily data and continuous measurements are grouped into timeseries, which diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 9d0ea4de..bda34cf5 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -62,6 +62,7 @@ dv_modern <- read_USGS_dv(monitoring_location_id = "USGS-01491000", nrow(dv_modern) ``` + The first thing you might notice is that the legacy service serves data in a "wide" format, which means there are multiple observations in a single row of the data frame. The new services return the data in a "long" format, so that only one observation is returned per row. This follows tidy principles and is often preferred in scripting languages. We'll look at the returned data with a `ggplot2` plot to show a very simple example of the power of a long format: ```{r} @@ -93,11 +94,63 @@ head(dv_2) * When you look at the help file for the new function `?read_USGS_dv`, you'll also notice there are many more arguments. These are mostly set by default to NA. You DO NOT need to specify all of these parameters. +* Flexible data/last_modified + + + +# Time Series Metadata + +To access these services on a web browser, go to . + +The `read_USGS_ts_meta` function will eventually replace the `whatNWISdata` function. Generally, both functions look pretty similar: + +```{r} +library(dataRetrieval) + +data_legacy <- whatNWISdata(siteNumbers = "01491000", + parameterCd = c("00060", "00010")) +``` + +```{r echo=FALSE} + +knitr::kable(data_legacy[,c("parm_cd", + "stat_cd", + "begin_date", + "end_date", + "count_nu")]) + +``` + +```{r} +data_modern <- read_NWIS_ts_meta(monitoring_location_id = "USGS-01491000", + parameter_code = c("00060", "00010")) + +``` + +```{r echo=FALSE} + +knitr::kable(data_modern[,c("parameter_code", + "parameter_name", + "statistic_id", + "computation_identifier", + "begin", + "end")]) +``` +## Additional new features # API Tokens +If you find yourself running into limits, you can request an API token here: + +Then save your token in your .Renviron file like this: + +``` +API_USGS_PAT = "my_super_secret_token" +``` + +You can use `usethis::edit_r_environ()` to edit find and open your .Renviron file. You will need to restart R for that variable to be recognized. # Notes on dataRetrieval development From b390fdeb35aebe6fc054c4a9971b6d0a427ad1cb Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 8 May 2025 08:22:41 -0500 Subject: [PATCH 014/117] Doh! --- NAMESPACE | 2 +- R/read_USGS_ts_meta.R | 6 +++--- man/{read_NWIS_ts_meta.Rd => read_USGS_ts_meta.Rd} | 10 +++++----- vignettes/read_USGS_functions.Rmd | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) rename man/{read_NWIS_ts_meta.Rd => read_USGS_ts_meta.Rd} (96%) diff --git a/NAMESPACE b/NAMESPACE index 5baa34a5..d29e0f40 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -44,9 +44,9 @@ export(readNWISuv) export(readWQPdata) export(readWQPqw) export(readWQPsummary) -export(read_NWIS_ts_meta) export(read_USGS_dv) export(read_USGS_samples) +export(read_USGS_ts_meta) export(renameNWISColumns) export(setAccess) export(stateCd) diff --git a/R/read_USGS_ts_meta.R b/R/read_USGS_ts_meta.R index c784b60f..561fae72 100644 --- a/R/read_USGS_ts_meta.R +++ b/R/read_USGS_ts_meta.R @@ -73,13 +73,13 @@ #' #' \donttest{ #' site <- "USGS-02238500" -#' meta_1 <- read_NWIS_ts_meta(monitoring_location_id = site) +#' meta_1 <- read_USGS_ts_meta(monitoring_location_id = site) #' -#' meta_multi <- read_NWIS_ts_meta(monitoring_location_id = c("USGS-01491000", +#' meta_multi <- read_USGS_ts_meta(monitoring_location_id = c("USGS-01491000", #' "USGS-01645000"), #' parameter_code = c("00060", "00010")) #' } -read_NWIS_ts_meta <- function(monitoring_location_id = NA_character_, +read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, parameter_code = NA_character_, parameter_name = NA_character_, properties = c("monitoring_location_id", diff --git a/man/read_NWIS_ts_meta.Rd b/man/read_USGS_ts_meta.Rd similarity index 96% rename from man/read_NWIS_ts_meta.Rd rename to man/read_USGS_ts_meta.Rd index 512e83b8..4ca4cf9a 100644 --- a/man/read_NWIS_ts_meta.Rd +++ b/man/read_USGS_ts_meta.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/read_USGS_ts_meta.R -\name{read_NWIS_ts_meta} -\alias{read_NWIS_ts_meta} +\name{read_USGS_ts_meta} +\alias{read_USGS_ts_meta} \title{Get USGS Timeseries metadata} \usage{ -read_NWIS_ts_meta( +read_USGS_ts_meta( monitoring_location_id = NA_character_, parameter_code = NA_character_, parameter_name = NA_character_, @@ -124,9 +124,9 @@ occurred. \donttest{ site <- "USGS-02238500" -meta_1 <- read_NWIS_ts_meta(monitoring_location_id = site) +meta_1 <- read_USGS_ts_meta(monitoring_location_id = site) -meta_multi <- read_NWIS_ts_meta(monitoring_location_id = c("USGS-01491000", +meta_multi <- read_USGS_ts_meta(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), parameter_code = c("00060", "00010")) } diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index bda34cf5..452c58cd 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -122,7 +122,7 @@ knitr::kable(data_legacy[,c("parm_cd", ``` ```{r} -data_modern <- read_NWIS_ts_meta(monitoring_location_id = "USGS-01491000", +data_modern <- read_USGS_ts_meta(monitoring_location_id = "USGS-01491000", parameter_code = c("00060", "00010")) ``` From 0a3d1b50d2b0615940b03f0ef5ae3923fce4ebe0 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 8 May 2025 08:31:57 -0500 Subject: [PATCH 015/117] update to time-series switch --- R/construct_api_requests.R | 19 ++++++++++--------- R/read_USGS_dv.R | 6 +++--- R/read_USGS_ts_meta.R | 2 +- man/construct_api_requests.Rd | 4 ++-- man/read_USGS_dv.Rd | 4 ++-- 5 files changed, 18 insertions(+), 17 deletions(-) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index bea4c88a..dbd92361 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -33,7 +33,7 @@ #' any change to the data itself) a new ID will be generated. To uniquely identify #' a single observation over time, compare the time and timeseries_id fields; each #' timeseries will only have a single observation at a given time. -#' @param timeseries_id A unique identifier representing a single timeseries. +#' @param time_series_id A unique identifier representing a single timeseries. #' This corresponds to the id field in the timeseries-metadata endpoint. #' @param parameter_code Parameter codes are 5-digit codes used to identify the #' constituent measured and the units of measure. @@ -125,7 +125,7 @@ construct_api_requests <- function(service, "approval_status", "qualifier"), bbox = NA, - timeseries_id = NA_character_, + time_series_id = NA_character_, id = NA_character_, approval_status = NA_character_, unit_of_measure = NA_character_, @@ -169,15 +169,13 @@ construct_api_requests <- function(service, post_params <- explode_post(list(monitoring_location_id = monitoring_location_id, parameter_code = parameter_code, statistic_id = statistic_id, - timeseries_id = timeseries_id, + time_series_id = time_series_id, id = id, approval_status = approval_status, unit_of_measure = unit_of_measure, qualifier = qualifier, value = value, - parameter_name = parameter_name - -)) + parameter_name = parameter_name)) if(length(post_params) > 0){ POST = TRUE @@ -228,12 +226,13 @@ construct_api_requests <- function(service, list(monitoring_location_id = monitoring_location_id, parameter_code = parameter_code, statistic_id = statistic_id, - timeseries_id = timeseries_id, + time_series_id = time_series_id, id = id, approval_status = approval_status, unit_of_measure = unit_of_measure, qualifier = qualifier, - value = value)) + value = value, + parameter_name = parameter_name)) } return(baseURL) @@ -328,7 +327,9 @@ cql2_param <- function(parameter){ check_OGC_requests <- function(endpoint = "daily", type = "queryables"){ - match.arg(endpoint, c("daily", "timeseries-metadata")) + #https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/server/getCollections + + match.arg(endpoint, c("daily", "time-series-metadata", "sites")) match.arg(type, c("queryables", "schema")) check_req <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> diff --git a/R/read_USGS_dv.R b/R/read_USGS_dv.R index e8096cf6..c2690dca 100644 --- a/R/read_USGS_dv.R +++ b/R/read_USGS_dv.R @@ -41,7 +41,7 @@ #' any change to the data itself) a new ID will be generated. To uniquely identify #' a single observation over time, compare the time and timeseries_id fields; each #' timeseries will only have a single observation at a given time. -#' @param timeseries_id A unique identifier representing a single timeseries. +#' @param time_series_id A unique identifier representing a single timeseries. #' This corresponds to the id field in the timeseries-metadata endpoint. #' @param parameter_code Parameter codes are 5-digit codes used to identify the #' constituent measured and the units of measure. @@ -111,7 +111,7 @@ read_USGS_dv <- function(monitoring_location_id = NA_character_, "approval_status", "qualifier"), bbox = NA, - timeseries_id = NA_character_, + time_series_id = NA_character_, id = NA_character_, approval_status = NA_character_, unit_of_measure = NA_character_, @@ -139,7 +139,7 @@ read_USGS_dv <- function(monitoring_location_id = NA_character_, statistic_id = statistic_id, properties = properties, bbox = bbox, - timeseries_id = timeseries_id, + time_series_id = time_series_id, id = id, approval_status = approval_status, unit_of_measure = unit_of_measure, diff --git a/R/read_USGS_ts_meta.R b/R/read_USGS_ts_meta.R index 561fae72..ef24aa56 100644 --- a/R/read_USGS_ts_meta.R +++ b/R/read_USGS_ts_meta.R @@ -120,7 +120,7 @@ read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, skipGeometry <- TRUE } - req_ts_meta <- construct_api_requests("timeseries-metadata", + req_ts_meta <- construct_api_requests("time-series-metadata", monitoring_location_id = monitoring_location_id, parameter_code = parameter_code, parameter_name = parameter_name, diff --git a/man/construct_api_requests.Rd b/man/construct_api_requests.Rd index 4c789e03..0e884ddd 100644 --- a/man/construct_api_requests.Rd +++ b/man/construct_api_requests.Rd @@ -12,7 +12,7 @@ construct_api_requests( properties = c("monitoring_location_id", "parameter_code", "statistic_id", "time", "value", "unit_of_measure", "approval_status", "qualifier"), bbox = NA, - timeseries_id = NA_character_, + time_series_id = NA_character_, id = NA_character_, approval_status = NA_character_, unit_of_measure = NA_character_, @@ -58,7 +58,7 @@ box are selected.The bounding box is provided as four or six numbers, depending on whether the coordinate reference system includes a vertical axis (height or depth).} -\item{timeseries_id}{A unique identifier representing a single timeseries. +\item{time_series_id}{A unique identifier representing a single timeseries. This corresponds to the id field in the timeseries-metadata endpoint.} \item{id}{A universally unique identifier (UUID) representing a single version diff --git a/man/read_USGS_dv.Rd b/man/read_USGS_dv.Rd index 3007cfb2..c464de73 100644 --- a/man/read_USGS_dv.Rd +++ b/man/read_USGS_dv.Rd @@ -11,7 +11,7 @@ read_USGS_dv( properties = c("monitoring_location_id", "parameter_code", "statistic_id", "time", "value", "unit_of_measure", "approval_status", "qualifier"), bbox = NA, - timeseries_id = NA_character_, + time_series_id = NA_character_, id = NA_character_, approval_status = NA_character_, unit_of_measure = NA_character_, @@ -49,7 +49,7 @@ box are selected.The bounding box is provided as four or six numbers, depending on whether the coordinate reference system includes a vertical axis (height or depth).} -\item{timeseries_id}{A unique identifier representing a single timeseries. +\item{time_series_id}{A unique identifier representing a single timeseries. This corresponds to the id field in the timeseries-metadata endpoint.} \item{id}{A universally unique identifier (UUID) representing a single version From f12d71f8c82f07a2a089943bdf21164a334ad6f9 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 8 May 2025 08:55:43 -0500 Subject: [PATCH 016/117] make check_param more explicit --- NAMESPACE | 2 +- R/construct_api_requests.R | 33 ++++++++---- R/dataRetrievals-package.R | 4 +- R/readNWISdata.R | 4 +- R/read_USGS_samples.R | 51 ++++++++++--------- _pkgdown.yml | 3 +- man/check_OGC_requests.Rd | 5 +- ...k_param.Rd => check_USGS_sample_params.Rd} | 22 ++++---- man/construct_USGS_sample_request.Rd | 15 +++--- man/countyCd.Rd | 2 +- man/read_USGS_samples.Rd | 14 ++--- man/stateCd.Rd | 2 +- vignettes/samples_data.Rmd | 22 ++++---- 13 files changed, 97 insertions(+), 82 deletions(-) rename man/{check_param.Rd => check_USGS_sample_params.Rd} (61%) diff --git a/NAMESPACE b/NAMESPACE index d29e0f40..bbd1e1ad 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -4,7 +4,7 @@ export(addWaterYear) export(calcWaterYear) export(checkWQPdates) export(check_OGC_requests) -export(check_param) +export(check_USGS_sample_params) export(constructNWISURL) export(constructUseURL) export(constructWQPURL) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index dbd92361..3245beb6 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -310,6 +310,7 @@ cql2_param <- function(parameter){ #' @param endpoint Character, can be "daily", "timeseries-metadata" #' @param type Character, can be "queryables", "schema" #' @export +#' @keywords internal #' @return list #' @examplesIf is_dataRetrieval_user() #' @@ -319,22 +320,36 @@ cql2_param <- function(parameter){ #' type = "queryables") #' dv_schema <- check_OGC_requests(endpoint = "daily", #' type = "schema") -#' ts_meta_queryables <- check_OGC_requests(endpoint = "timeseries-metadata", +#' ts_meta_queryables <- check_OGC_requests(endpoint = "time-series-metadata", #' type = "queryables") -#' ts_meta_schema <- check_OGC_requests(endpoint = "timeseries-metadata", +#' ts_meta_schema <- check_OGC_requests(endpoint = "time-series-metadata", #' type = "schema") #' } check_OGC_requests <- function(endpoint = "daily", type = "queryables"){ - #https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/server/getCollections - - match.arg(endpoint, c("daily", "time-series-metadata", "sites")) match.arg(type, c("queryables", "schema")) - check_req <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> + check_collections <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/server/getCollections") + + check_endpoints <- basic_request(check_collections) + services <- sapply(check_endpoints$tags, function(x) x[["name"]]) + + match.arg(endpoint, services) + + url_base <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> httr2::req_url_path_append(endpoint) |> - httr2::req_url_path_append(type) |> + httr2::req_url_path_append(type) + + query_ret <- basic_request(url_base) + + return(query_ret) + +} + +basic_request <- function(url_base){ + + check_req <- url_base |> httr2::req_user_agent(default_ua()) |> httr2::req_url_query(f = "json", lang = "en-US") @@ -342,6 +357,4 @@ check_OGC_requests <- function(endpoint = "daily", query_ret <- httr2::req_perform(check_req) |> httr2::resp_body_json() - return(query_ret) - -} \ No newline at end of file +} diff --git a/R/dataRetrievals-package.R b/R/dataRetrievals-package.R index 3afb244a..f093c204 100644 --- a/R/dataRetrievals-package.R +++ b/R/dataRetrievals-package.R @@ -90,7 +90,7 @@ NULL #' US State Code Lookup Table #' #' Classic lookup table for states. Has been replaced in functions with -#' \code{check_param("states")}. +#' \code{check_USGS_sample_params("states")}. #' #' @name stateCd #' @return stateCd data frame. @@ -112,7 +112,7 @@ NULL #' US County Code Lookup Table #' #' Classic lookup table for counties. Has been replaced in functions with -#' \code{check_param("counties")}. +#' \code{check_USGS_sample_params("counties")}. #' #' @name countyCd #' @return countyCd data frame. diff --git a/R/readNWISdata.R b/R/readNWISdata.R index 5ee38883..8b4c1827 100644 --- a/R/readNWISdata.R +++ b/R/readNWISdata.R @@ -307,7 +307,7 @@ stateCdLookup <- function(input, outputType <- match.arg(outputType, c("postal", "fullName", "id", "fips")) - states <- check_param("states") + states <- check_USGS_sample_params("states") country <- match.arg(country, choices = unique(states$countryCode), several.ok = FALSE) states <- states[states$countryCode == country,] @@ -379,7 +379,7 @@ countyCdLookup <- function(state, county, outputType = "fips") { stop("Only one state allowed in countyCdLookup.") } - counties <- check_param("counties") + counties <- check_USGS_sample_params("counties") # first turn state into stateCd postal name state_postal <- stateCdLookup(state, diff --git a/R/read_USGS_samples.R b/R/read_USGS_samples.R index 98276ddd..5444e63b 100644 --- a/R/read_USGS_samples.R +++ b/R/read_USGS_samples.R @@ -14,9 +14,9 @@ #' numbers without an agency prefix are assumed to have the prefix USGS. #' @param activityMediaName Sample media refers to the environmental medium that #' was sampled or analyzed. See available options by running -#' \code{check_param("samplemedia")$activityMedia}. +#' \code{check_USGS_sample_params("samplemedia")$activityMedia}. #' @param siteTypeCode Site type code query parameter. See available -#' options by running \code{check_param("sitetype")$typeCode}. +#' options by running \code{check_USGS_sample_params("sitetype")$typeCode}. #' @param boundingBox North and South are latitude values; East and West are longitude values. #' A vector of 4 (west, south, east, north) is expected. #' An example would be: c(-92.8, 44.2, -88.9, 46.0). @@ -35,7 +35,7 @@ #' records that match the date. #' @param characteristicGroup Characteristic group is a broad category describing the sample. #' See available options by running -#' \code{check_param("characteristicgroup")$characteristicGroup}. +#' \code{check_USGS_sample_params("characteristicgroup")$characteristicGroup}. #' @param characteristicUserSupplied Observed property is the USGS term for the #' constituent sampled and the property name gives a detailed description of what #' was sampled. Observed property is mapped to characteristicUserSupplied and replaces @@ -45,21 +45,21 @@ #' \url{https://waterdata.usgs.gov/code-dictionary/}. #' @param characteristic Characteristic is a specific category describing the sample. #' See available options by running -#' \code{check_param("characteristics")$characteristicName}. +#' \code{check_USGS_sample_params("characteristics")$characteristicName}. #' @param stateFips State query parameter. To get a list of available state fips, -#' run \code{check_param("states")}. The "fips" can be created using the function +#' run \code{check_USGS_sample_params("states")}. The "fips" can be created using the function #' \code{stateCdLookup} - for example: \code{stateCdLookup("WI", "fips")}. #' FIPs codes for states take the format: #' CountryAbbrev:StateNumber, like US:55 for Wisconsin. #' @param countyFips County query parameter. To get a list of available counties, -#' run \code{check_param("counties")}. The "Fips" can be created using the function +#' run \code{check_USGS_sample_params("counties")}. The "Fips" can be created using the function #' \code{countyCdLookup} - for example: \code{countyCdLookup("WI", "Dane", "fips")} #' for Dane County, WI. #' FIPs codes for counties take the format: #' CountryAbbrev:StateNumber:CountyNumber, like US:55:025 for Dane County, WI. #' @param countryFips Country query parameter. Do not set redundant parameters. #' If another query parameter contains the country information, leave this parameter -#' set to the default NA. See available options by running \code{check_param("countries")}, +#' set to the default NA. See available options by running \code{check_USGS_sample_params("countries")}, #' where the "id" field contains the value to use in the countryFips input. #' @param projectIdentifier Project identifier query parameter. This information #' would be needed from prior project information. @@ -68,7 +68,7 @@ #' @param siteTypeName Site type name query parameter. See available #' options by running \code{check_param("sitetype")$typeName}. #' @param usgsPCode USGS parameter code. See available options by running -#' \code{check_param("characteristics")$parameterCode}. +#' \code{check_USGS_sample_params("characteristics")$parameterCode}. #' @param pointLocationLatitude Latitude for a point/radius query (decimal degrees). Must be used #' with pointLocationLongitude and pointLocationWithinMiles. #' @param pointLocationLongitude Longitude for a point/radius query (decimal degrees). Must be used @@ -85,6 +85,7 @@ #' "project" and "projectmonitoringlocationweight". Options for "organizations" are: #' "organization" and "count". #' @export +#' @keywords internal #' @return data frame returned from web service call. #' #' @examplesIf is_dataRetrieval_user() @@ -175,36 +176,36 @@ construct_USGS_sample_request <- function(monitoringLocationIdentifier = NA, if(all(!is.na(siteTypeCode))){ siteTypeCode <- match.arg(siteTypeCode, - check_param("sitetype")$typeCode, + check_USGS_sample_params("sitetype")$typeCode, several.ok = TRUE) } if(all(!is.na(activityMediaName))){ activityMediaName <- match.arg(activityMediaName, - check_param("samplemedia")$activityMedia, + check_USGS_sample_params("samplemedia")$activityMedia, several.ok = TRUE) } if(all(!is.na(characteristicGroup))){ characteristicGroup <- match.arg(characteristicGroup, - check_param("characteristicgroup")$characteristicGroup, + check_USGS_sample_params("characteristicgroup")$characteristicGroup, several.ok = TRUE) } if(all(!is.na(countryFips))){ countryFips <- match.arg(countryFips, - check_param("countries")$countryCode, + check_USGS_sample_params("countries")$countryCode, several.ok = TRUE) } if(all(!is.na(siteTypeName))){ siteTypeName <- match.arg(siteTypeName, - check_param("sitetype")$typeLongName, + check_USGS_sample_params("sitetype")$typeLongName, several.ok = TRUE) } if(all(!is.na(stateFips))){ - states <- check_param("states") + states <- check_USGS_sample_params("states") state_codes <- paste(states$countryCode, states$fipsCode, sep = ":") stateFips <- match.arg(stateFips, state_codes, @@ -212,10 +213,10 @@ construct_USGS_sample_request <- function(monitoringLocationIdentifier = NA, } if(all(!is.na(countyFips))){ - states <- check_param("states") + states <- check_USGS_sample_params("states") state_codes <- paste(states$countryCode, states$fipsCode, sep = ":") - counties <- check_param("counties") + counties <- check_USGS_sample_params("counties") state_cd <- stats::setNames(states$fipsCode, states$stateAbbrev) county_codes <- paste(counties$countryCode, @@ -328,19 +329,19 @@ explode_query <- function(baseURL, POST = FALSE, x){ #' @examplesIf is_dataRetrieval_user() #' #' \donttest{ -#' groups <- check_param("characteristicgroup") -#' states <- check_param("states") -#' countries <- check_param("countries") -#' counties <- check_param("counties") -#' sitetypes <- check_param("sitetype") -#' samplemedia <- check_param("samplemedia") -#' characteristics <- check_param("characteristics", +#' groups <- check_USGS_sample_params("characteristicgroup") +#' states <- check_USGS_sample_params("states") +#' countries <- check_USGS_sample_params("countries") +#' counties <- check_USGS_sample_params("counties") +#' sitetypes <- check_USGS_sample_params("sitetype") +#' samplemedia <- check_USGS_sample_params("samplemedia") +#' characteristics <- check_USGS_sample_params("characteristics", #' group = "Biological") -#' observedProperties <- check_param("observedproperty", +#' observedProperties <- check_USGS_sample_params("observedproperty", #' text = "phosphorus") #' #' } -check_param <- function(service = "characteristicgroup", +check_USGS_sample_params <- function(service = "characteristicgroup", ...){ service_options <- c("characteristicgroup", "states", "counties", diff --git a/_pkgdown.yml b/_pkgdown.yml index 79712d39..a7e40331 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -71,8 +71,7 @@ reference: - read_USGS_dv - read_USGS_ts_meta - summarize_USGS_samples - - construct_USGS_sample_request - - check_param + - check_USGS_sample_params - title: National Water Information System (NWIS) desc: Functions to retrieve (USGS) NWIS data. These will be slowly phased out and replaced with the read_USGS family of functions. contents: diff --git a/man/check_OGC_requests.Rd b/man/check_OGC_requests.Rd index cdf04d90..03ce253c 100644 --- a/man/check_OGC_requests.Rd +++ b/man/check_OGC_requests.Rd @@ -26,10 +26,11 @@ dv_queryables <- check_OGC_requests(endpoint = "daily", type = "queryables") dv_schema <- check_OGC_requests(endpoint = "daily", type = "schema") -ts_meta_queryables <- check_OGC_requests(endpoint = "timeseries-metadata", +ts_meta_queryables <- check_OGC_requests(endpoint = "time-series-metadata", type = "queryables") -ts_meta_schema <- check_OGC_requests(endpoint = "timeseries-metadata", +ts_meta_schema <- check_OGC_requests(endpoint = "time-series-metadata", type = "schema") } \dontshow{\}) # examplesIf} } +\keyword{internal} diff --git a/man/check_param.Rd b/man/check_USGS_sample_params.Rd similarity index 61% rename from man/check_param.Rd rename to man/check_USGS_sample_params.Rd index 63deb385..eb387605 100644 --- a/man/check_param.Rd +++ b/man/check_USGS_sample_params.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/read_USGS_samples.R -\name{check_param} -\alias{check_param} +\name{check_USGS_sample_params} +\alias{check_USGS_sample_params} \title{Check values from codeservice} \usage{ -check_param(service = "characteristicgroup", ...) +check_USGS_sample_params(service = "characteristicgroup", ...) } \arguments{ \item{service}{Options are: "characteristicgroup", "states", "counties", @@ -24,15 +24,15 @@ Call a service to check on values from: \dontshow{if (is_dataRetrieval_user()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} \donttest{ -groups <- check_param("characteristicgroup") -states <- check_param("states") -countries <- check_param("countries") -counties <- check_param("counties") -sitetypes <- check_param("sitetype") -samplemedia <- check_param("samplemedia") -characteristics <- check_param("characteristics", +groups <- check_USGS_sample_params("characteristicgroup") +states <- check_USGS_sample_params("states") +countries <- check_USGS_sample_params("countries") +counties <- check_USGS_sample_params("counties") +sitetypes <- check_USGS_sample_params("sitetype") +samplemedia <- check_USGS_sample_params("samplemedia") +characteristics <- check_USGS_sample_params("characteristics", group = "Biological") -observedProperties <- check_param("observedproperty", +observedProperties <- check_USGS_sample_params("observedproperty", text = "phosphorus") } diff --git a/man/construct_USGS_sample_request.Rd b/man/construct_USGS_sample_request.Rd index 21e194fe..9f773a77 100644 --- a/man/construct_USGS_sample_request.Rd +++ b/man/construct_USGS_sample_request.Rd @@ -36,7 +36,7 @@ for example: AZ014-320821110580701, CAX01-15304600, USGS-040851385. Location numbers without an agency prefix are assumed to have the prefix USGS.} \item{siteTypeCode}{Site type code query parameter. See available -options by running \code{check_param("sitetype")$typeCode}.} +options by running \code{check_USGS_sample_params("sitetype")$typeCode}.} \item{boundingBox}{North and South are latitude values; East and West are longitude values. A vector of 4 (west, south, east, north) is expected. @@ -51,11 +51,11 @@ was sampled or analyzed.} \item{characteristicGroup}{Characteristic group is a broad category describing the sample. See available options by running -\code{check_param("characteristicgroup")$characteristicGroup}.} +\code{check_USGS_sample_params("characteristicgroup")$characteristicGroup}.} \item{characteristic}{Characteristic is a specific category describing the sample. See available options by running -\code{check_param("characteristics")$characteristicName}.} +\code{check_USGS_sample_params("characteristics")$characteristicName}.} \item{characteristicUserSupplied}{Observed property is the USGS term for the constituent sampled and the property name gives a detailed description of what @@ -77,17 +77,17 @@ records that match the date.} \item{countryFips}{Country query parameter. Do not set redundant parameters. If another query parameter contains the country information, leave this parameter -set to the default NA. See available options by running \code{check_param("countries")}, +set to the default NA. See available options by running \code{check_USGS_sample_params("countries")}, where the "id" field contains the value to use in the countryFips input.} \item{stateFips}{State query parameter. To get a list of available state fips, -run \code{check_param("states")}. The "fips" can be created using the function +run \code{check_USGS_sample_params("states")}. The "fips" can be created using the function \code{stateCdLookup} - for example: \code{stateCdLookup("WI", "fips")}. FIPs codes for states take the format: CountryAbbrev:StateNumber, like US:55 for Wisconsin.} \item{countyFips}{County query parameter. To get a list of available counties, -run \code{check_param("counties")}. The "Fips" can be created using the function +run \code{check_USGS_sample_params("counties")}. The "Fips" can be created using the function \code{countyCdLookup} - for example: \code{countyCdLookup("WI", "Dane", "fips")} for Dane County, WI. FIPs codes for counties take the format: @@ -103,7 +103,7 @@ information would be needed from the data supplier.} options by running \code{check_param("sitetype")$typeName}.} \item{usgsPCode}{USGS parameter code. See available options by running -\code{check_param("characteristics")$parameterCode}.} +\code{check_USGS_sample_params("characteristics")$parameterCode}.} \item{pointLocationLatitude}{Latitude for a point/radius query (decimal degrees). Must be used with pointLocationLongitude and pointLocationWithinMiles.} @@ -150,3 +150,4 @@ rawData <- importWQP(req) } \dontshow{\}) # examplesIf} } +\keyword{internal} diff --git a/man/countyCd.Rd b/man/countyCd.Rd index a7b6e57d..9fd7e511 100644 --- a/man/countyCd.Rd +++ b/man/countyCd.Rd @@ -18,7 +18,7 @@ COUNTY_ID \tab character \tab County id \cr } \description{ Classic lookup table for counties. Has been replaced in functions with -\code{check_param("counties")}. +\code{check_USGS_sample_params("counties")}. } \examples{ head(countyCd) diff --git a/man/read_USGS_samples.Rd b/man/read_USGS_samples.Rd index bcf26166..975f66a5 100644 --- a/man/read_USGS_samples.Rd +++ b/man/read_USGS_samples.Rd @@ -37,7 +37,7 @@ for example: AZ014-320821110580701, CAX01-15304600, USGS-040851385. Location numbers without an agency prefix are assumed to have the prefix USGS.} \item{siteTypeCode}{Site type code query parameter. See available -options by running \code{check_param("sitetype")$typeCode}.} +options by running \code{check_USGS_sample_params("sitetype")$typeCode}.} \item{boundingBox}{North and South are latitude values; East and West are longitude values. A vector of 4 (west, south, east, north) is expected. @@ -52,11 +52,11 @@ was sampled or analyzed.} \item{characteristicGroup}{Characteristic group is a broad category describing the sample. See available options by running -\code{check_param("characteristicgroup")$characteristicGroup}.} +\code{check_USGS_sample_params("characteristicgroup")$characteristicGroup}.} \item{characteristic}{Characteristic is a specific category describing the sample. See available options by running -\code{check_param("characteristics")$characteristicName}.} +\code{check_USGS_sample_params("characteristics")$characteristicName}.} \item{characteristicUserSupplied}{Observed property is the USGS term for the constituent sampled and the property name gives a detailed description of what @@ -78,17 +78,17 @@ records that match the date.} \item{countryFips}{Country query parameter. Do not set redundant parameters. If another query parameter contains the country information, leave this parameter -set to the default NA. See available options by running \code{check_param("countries")}, +set to the default NA. See available options by running \code{check_USGS_sample_params("countries")}, where the "id" field contains the value to use in the countryFips input.} \item{stateFips}{State query parameter. To get a list of available state fips, -run \code{check_param("states")}. The "fips" can be created using the function +run \code{check_USGS_sample_params("states")}. The "fips" can be created using the function \code{stateCdLookup} - for example: \code{stateCdLookup("WI", "fips")}. FIPs codes for states take the format: CountryAbbrev:StateNumber, like US:55 for Wisconsin.} \item{countyFips}{County query parameter. To get a list of available counties, -run \code{check_param("counties")}. The "Fips" can be created using the function +run \code{check_USGS_sample_params("counties")}. The "Fips" can be created using the function \code{countyCdLookup} - for example: \code{countyCdLookup("WI", "Dane", "fips")} for Dane County, WI. FIPs codes for counties take the format: @@ -104,7 +104,7 @@ information would be needed from the data supplier.} options by running \code{check_param("sitetype")$typeName}.} \item{usgsPCode}{USGS parameter code. See available options by running -\code{check_param("characteristics")$parameterCode}.} +\code{check_USGS_sample_params("characteristics")$parameterCode}.} \item{pointLocationLatitude}{Latitude for a point/radius query (decimal degrees). Must be used with pointLocationLongitude and pointLocationWithinMiles.} diff --git a/man/stateCd.Rd b/man/stateCd.Rd index a61aad42..0e595174 100644 --- a/man/stateCd.Rd +++ b/man/stateCd.Rd @@ -17,7 +17,7 @@ STATENS \tab character \tab Geographic Names Information System Identifier (GNI } \description{ Classic lookup table for states. Has been replaced in functions with -\code{check_param("states")}. +\code{check_USGS_sample_params("states")}. } \examples{ head(stateCd) diff --git a/vignettes/samples_data.Rmd b/vignettes/samples_data.Rmd index 14c67de5..32d9043b 100644 --- a/vignettes/samples_data.Rmd +++ b/vignettes/samples_data.Rmd @@ -259,7 +259,7 @@ map_it(point_sites) ### countyFips County query parameter. To get a list of available counties, -run `check_param("counties")`. The "Fips" values can be created using the function `countyCdLookup`. +run `check_USGS_sample_params("counties")`. The "Fips" values can be created using the function `countyCdLookup`. ```{r} dane_county <- countyCdLookup("WI", "Dane", @@ -280,7 +280,7 @@ map_it(county_sites) ### stateFips State query parameter. To get a list of available state fips values, -run `check_param("states")`. The "fips" values can be created using the function +run `check_USGS_sample_params("states")`. The "fips" values can be created using the function `stateCdLookup`. ```{r} @@ -307,7 +307,7 @@ Additional parameters can be included to limit the results coming back from a re Site type code query parameter. ```{r} -site_type_info <- check_param("sitetype") +site_type_info <- check_USGS_sample_params("sitetype") site_type_info$typeCode ``` @@ -324,7 +324,7 @@ site_type_info$typeLongName Sample media refers to the environmental medium that was sampled or analyzed. ```{r} -media_info <- check_param("samplemedia") +media_info <- check_USGS_sample_params("samplemedia") media_info$activityMedia ``` @@ -333,34 +333,34 @@ media_info$activityMedia Characteristic group is a broad category describing the sample measurement. The options for this parameter generally follow the values described in the Water Quality Portal [User Guide](https://www.waterqualitydata.us/portal_userguide), but not always. ```{r} -group_info <- check_param("characteristicgroup") +group_info <- check_USGS_sample_params("characteristicgroup") group_info$characteristicGroup ``` ### characteristic -Characteristic is a specific category describing the sample. See `check_param("characteristics")` for a full list, below is a small sample: +Characteristic is a specific category describing the sample. See `check_USGS_sample_params("characteristics")` for a full list, below is a small sample: ```{r} -characteristic_info <- check_param("characteristics") +characteristic_info <- check_USGS_sample_params("characteristics") head(unique(characteristic_info$characteristicName)) ``` ### characteristicUserSupplied -Observed property is the USGS term for the constituent sampled and the property name gives a detailed description of what was sampled. Observed Property is mapped to characteristicUserSupplied, and replaces the parameter name and pcode USGS previously used to describe discrete sample data. See `check_param("observedproperty")` for a full list, below is a small sample: +Observed property is the USGS term for the constituent sampled and the property name gives a detailed description of what was sampled. Observed Property is mapped to characteristicUserSupplied, and replaces the parameter name and pcode USGS previously used to describe discrete sample data. See `check_USGS_sample_params("observedproperty")` for a full list, below is a small sample: ```{r} -char_us <- check_param("observedproperty") +char_us <- check_USGS_sample_params("observedproperty") head(char_us$observedProperty) ``` ### usgsPCode -USGS parameter code. See `check_param("characteristics")` for a full list, below is a small sample: +USGS parameter code. See `check_USGS_sample_params("characteristics")` for a full list, below is a small sample: ```{r} -characteristic_info <- check_param("characteristics") +characteristic_info <- check_USGS_sample_params("characteristics") head(unique(characteristic_info$parameterCode)) ``` From 8418a426f03799dadbebe19acdfbc5a979df0aec Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 8 May 2025 09:07:30 -0500 Subject: [PATCH 017/117] forgot a rename --- tests/testthat/tests_samples.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/tests_samples.R b/tests/testthat/tests_samples.R index d23c1e0a..2c87d4ef 100644 --- a/tests/testthat/tests_samples.R +++ b/tests/testthat/tests_samples.R @@ -10,7 +10,7 @@ test_that("General samples-data retrievals work using WQP tests", { # testing lists: startDate <- as.Date("2022-01-01") - secchi_ops <- check_param("observedproperty", + secchi_ops <- check_USGS_sample_params("observedproperty", text = "secchi") state_fips <- paste0("US:", stateCdLookup("WI", "id")) From 742d6e49d0b7eed935bc6a652b64dfc09d215b79 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 8 May 2025 14:11:14 -0500 Subject: [PATCH 018/117] Change to md style roxygen --- DESCRIPTION | 7 +- R/checkWQPdates.R | 4 +- R/citations.R | 4 +- R/constructNWISURL.R | 14 +-- R/construct_api_requests.R | 135 ++++++++++++-------------- R/dataRetrievals-package.R | 10 +- R/importNGWMN_wml2.R | 6 +- R/importRDB1.R | 8 +- R/importWQP.R | 8 +- R/importWaterML1.R | 6 +- R/readNGWMNdata.R | 16 ++-- R/readNWISdata.R | 32 +++---- R/readNWISdv.R | 12 +-- R/readNWISpCode.R | 6 +- R/readNWISsite.R | 2 +- R/readNWISunit.R | 96 +++++++++---------- R/readWQPdata.R | 10 +- R/readWQPdots.R | 2 +- R/readWQPqw.R | 12 +-- R/read_USGS_dv.R | 97 +++++-------------- R/read_USGS_samples.R | 36 +++---- R/read_USGS_ts_meta.R | 93 ++++++------------ R/renameColumns.R | 6 +- R/setAccess.R | 2 +- R/walk_pages.R | 10 +- R/whatNWISdata.R | 8 +- R/whatNWISsites.R | 4 +- R/whatWQPdata.R | 12 +-- R/whatWQPsites.R | 20 ++-- man/addWaterYear.Rd | 6 +- man/construct_USGS_sample_request.Rd | 24 ++--- man/construct_api_requests.Rd | 137 ++++++++++----------------- man/findNLDI.Rd | 4 +- man/importWQP.Rd | 2 +- man/importWaterML1.Rd | 4 +- man/pcode_to_name.Rd | 4 +- man/readNWISdata.Rd | 22 ++--- man/readNWISdv.Rd | 10 +- man/readNWISgwl.Rd | 9 +- man/readNWISmeas.Rd | 2 +- man/readNWISpCode.Rd | 16 ++-- man/readNWISpeak.Rd | 4 +- man/readNWISrating.Rd | 2 +- man/readNWISsite.Rd | 80 ++++++++-------- man/readNWISstat.Rd | 2 +- man/readNWISuv.Rd | 8 +- man/readWQPdata.Rd | 17 ++-- man/readWQPqw.Rd | 4 +- man/readWQPsummary.Rd | 6 +- man/read_USGS_dv.Rd | 103 ++++++++------------ man/read_USGS_samples.Rd | 24 ++--- man/read_USGS_ts_meta.Rd | 96 ++++++++----------- man/renameNWISColumns.Rd | 2 +- man/whatNWISdata.Rd | 2 +- man/whatWQPdata.Rd | 16 ++-- man/wqpSpecials.Rd | 6 +- man/wqp_check_status.Rd | 4 +- 57 files changed, 557 insertions(+), 737 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index ccfa5546..77a1428b 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -37,7 +37,7 @@ Copyright: This software is in the public domain because it contains materials Depends: R (>= 4.1.0) Imports: - curl, + curl (>= 6.0.0), lubridate (>= 1.5.0), stats, utils, @@ -45,16 +45,17 @@ Imports: readr (>= 1.4.0), jsonlite, httr2, - whisker + whisker, + sf Suggests: covr, dplyr, knitr, rmarkdown, - sf, testthat Encoding: UTF-8 BuildVignettes: true VignetteBuilder: knitr BugReports: https://github.com/DOI-USGS/dataRetrieval/issues RoxygenNote: 7.3.2 +Roxygen: list(markdown = TRUE) diff --git a/R/checkWQPdates.R b/R/checkWQPdates.R index 99abf198..3033cb2e 100644 --- a/R/checkWQPdates.R +++ b/R/checkWQPdates.R @@ -1,7 +1,7 @@ #' Date Check for Water Quality Portal #' -#' Checks date format for inputs to the Water Quality Portal. Used in \code{readWQPqw} -#' and \code{readWQPdata}. +#' Checks date format for inputs to the Water Quality Portal. Used in `readWQPqw` +#' and `readWQPdata`. #' #' @param values named list with arguments to send to the Water Quality Portal #' @return values named list with corrected arguments to send to the Water Quality Portal diff --git a/R/citations.R b/R/citations.R index aedbd5ba..b6d5b21d 100644 --- a/R/citations.R +++ b/R/citations.R @@ -2,7 +2,7 @@ #' #' Uses attributes from the NWIS functions to create data citations. #' -#' See \code{?bibentry} for more information. +#' See `?bibentry` for more information. #' #' @param x Any data returned from an NWIS function, must #' include "queryTime" and "url" attributes, which should @@ -48,7 +48,7 @@ create_NWIS_bib <- function(x){ #' #' Uses attributes from the WQP functions to create data citations. #' -#' See \code{?bibentry} for more information. +#' See `?bibentry` for more information. #' #' @param x Any data returned from an NWIS function, must #' include "queryTime" and "url" attributes, which should diff --git a/R/constructNWISURL.R b/R/constructNWISURL.R index f163f651..bcb4ecbf 100644 --- a/R/constructNWISURL.R +++ b/R/constructNWISURL.R @@ -22,21 +22,21 @@ #' momentary problem with the internet connection). It is possible to safely use the "tsv" option, #' but the user must carefully check the results to see if the data returns matches #' what is expected. The default is therefore "xml". -#' @param expanded logical defaults to \code{TRUE}. If \code{TRUE}, retrieves additional +#' @param expanded logical defaults to `TRUE`. If `TRUE`, retrieves additional #' information, only applicable for qw data. #' @param ratingType can be "base", "corr", or "exsa". Only applies to rating curve data. #' @param statReportType character Only used for statistics service requests. Time #' division for statistics: daily, monthly, or annual. Default is daily. #' Note that daily provides statistics for each calendar day over the specified #' range of water years, i.e. no more than 366 data points will be returned for -#' each site/parameter. Use \code{readNWISdata} or \code{readNWISdv} for daily averages. +#' each site/parameter. Use `readNWISdata` or `readNWISdv` for daily averages. #' Also note that "annual" returns statistics for the calendar year. Use -#' \code{readNWISdata} for water years. Monthly and yearly +#' `readNWISdata` for water years. Monthly and yearly #' provide statistics for each month and year within the range individually. #' @param statType character Only used for statistics service requests. Type(s) #' of statistics to output for daily values. Default is mean, which is the only #' option for monthly and yearly report types. See the statistics service documentation -#' at \url{https://waterservices.usgs.gov/docs/statistics/} for a +#' at for a #' full list of codes. #' @keywords data import USGS web service #' @return url string @@ -304,7 +304,7 @@ constructNWISURL <- function(siteNumbers, #' Construct WQP url for data retrieval #' -#' Construct WQP url for data retrieval. This function gets the data from here: \url{https://www.waterqualitydata.us} +#' Construct WQP url for data retrieval. This function gets the data from here: #' #' @param siteNumbers string or vector of strings USGS site number. #' @param parameterCd string or vector of USGS parameter code. This is usually an 5 digit number. @@ -433,7 +433,7 @@ constructWQPURL <- function(siteNumbers, #' Construct URL for NWIS water use data service #' -#' Reconstructs URLs to retrieve data from here: \url{https://waterdata.usgs.gov/nwis/wu} +#' Reconstructs URLs to retrieve data from here: #' #' @param years integer Years for data retrieval. Must be years ending in 0 or 5, #' or "ALL", which retrieves all available years. @@ -490,4 +490,4 @@ constructUseURL <- function(years, stateCd, countyCd, categories) { .multi = "comma") return(baseURL) -} \ No newline at end of file +} diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index 3245beb6..a05eb11d 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -1,93 +1,41 @@ #' Create API url #' -#' Main documentation: \url{https://api.waterdata.usgs.gov/ogcapi/v0/}, -#' Swagger docs: \url{https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html}. +#' Main documentation: , +#' Swagger docs: . #' #' @export -#' @param monitoring_location_id A unique identifier representing a single monitoring -#' location. This corresponds to the id field in the sites endpoint. Monitoring -#' location IDs are created by combining the agency code of the agency responsible -#' for the monitoring location (e.g. USGS) with the ID number of the monitoring -#' location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500). +#' @param monitoring_location_id `r get_params("daily")$monitoring_location_id` +#' @param parameter_code `r get_params("daily")$parameter_code` +#' @param statistic_id `r get_params("daily")$statistic_id` +#' @param time `r get_params("daily")$time` +#' @param value `r get_params("daily")$value` +#' @param unit_of_measure `r get_params("daily")$unit_of_measure` +#' @param approval_status `r get_params("daily")$approval_status` +#' @param last_modified `r get_params("daily")$last_modified` +#' @param time_series_id `r get_params("daily")$time_series_id` +#' @param qualifier `r get_params("daily")$qualifier` +#' @param id `r get_params("daily")$id` +#' @param parameter_name `r get_params("time-series-metadata")$parameter_name` +#' @param computation_identifier `r get_params("time-series-metadata")$computation_identifier` +#' @param computation_period_identifier `r get_params("time-series-metadata")$computation_period_identifier` +#' @param sublocation_identifier `r get_params("time-series-metadata")$sublocation_identifier` +#' @param begin `r get_params("time-series-metadata")$begin` +#' @param end `r get_params("time-series-metadata")$end` +#' @param thresholds `r get_params("time-series-metadata")$thresholds` +#' @param primary `r get_params("time-series-metadata")$primary` +#' @param web_description `r get_params("time-series-metadata")$web_description` #' @param bbox Only features that have a geometry that intersects the bounding #' box are selected.The bounding box is provided as four or six numbers, depending #' on whether the coordinate reference system includes a vertical axis (height or #' depth). #' @param crs Indicates the coordinate reference system for the results. -#' @param bbox_crs Indicates the coordinate reference system for the given bbox -#' coordinates. #' @param properties The properties that should be included for each feature. The #' parameter value is a comma-separated list of property names. Available values: #' id, timeseries_id, monitoring_location_id, parameter_code, statistic_id, time, #' value, unit_of_measure, approval_status, qualifier, last_modified. #' @param skipGeometry This option can be used to skip response geometries for -#' each feature. -#' @param datetime Either a date-time or an interval. Only features that have a -#' temporal property that intersects the value of datetime are selected. If a -#' feature has multiple temporal properties, it is the decision of the server -#' whether only a single temporal property is used to determine the extent or -#' all relevant temporal properties. -#' @param id A universally unique identifier (UUID) representing a single version -#' of a record. It is not stable over time. Every time the record is refreshed in -#' our database (which may happen as part of normal operations and does not imply -#' any change to the data itself) a new ID will be generated. To uniquely identify -#' a single observation over time, compare the time and timeseries_id fields; each -#' timeseries will only have a single observation at a given time. -#' @param time_series_id A unique identifier representing a single timeseries. -#' This corresponds to the id field in the timeseries-metadata endpoint. -#' @param parameter_code Parameter codes are 5-digit codes used to identify the -#' constituent measured and the units of measure. -#' @param parameter_name A human-understandable name corresponding to parameter_code. -#' @param statistic_id A code corresponding to the statistic an observation represents. -#' Example codes include 00001 (max), 00002 (min), and 00003 (mean). -#' @param time The date an observation represents. -#' @param value The value of the observation. Values are transmitted as strings -#' in the JSON response format in order to preserve precision. -#' @param unit_of_measure A human-readable description of the units of measurement -#' associated with an observation. -#' @param approval_status Some of the data that you have obtained from this U.S. -#' Geological Survey database may not have received Director's approval. Any such -#' data values are qualified as provisional and are subject to revision. Provisional -#' data are released on the condition that neither the USGS nor the United States -#' Government may be held liable for any damages resulting from its use. This field -#' reflects the approval status of each record, and is either "Approved", meaning -#' processing review has been completed and the data is approved for publication, -#' or "Provisional" and subject to revision. -#' @param qualifier This field indicates any qualifiers associated with an observation, -#' for instance if a sensor may have been impacted by ice or if values were estimated. -#' @param last_modified The last time a record was refreshed in our database. This -#' may happen due to regular operational processes and does not necessarily indicate -#' anything about the measurement has changed. You can query this field using -#' date-times or intervals. -#' @param begin The datetime of the earliest observation in the timeseries. -#' Together with end, this field represents the period of record of a timeseries. -#' Note that some timeseries may have large gaps in their collection record. -#' @param end The datetime of the most recent observation in the timeseries. -#' Data returned by this endpoint updates at most once per day, and potentially -#' less frequently than that, and as such there may be more recent observations -#' within a timeseries than the timeseries end value reflects. Together with begin, -#' this field represents the period of record of a timeseries. It is additionally -#' used to determine whether a timeseries is "active". -#' @param computation_period_identifier Indicates the period of data used for -#' any statistical computations. -#' @param computation_identifier Indicates whether the data from this timeseries -#' represent a specific statistical computation. -#' @param thresholds Thresholds represent known numeric limits for a timeseries, -#' for example the historic maximum value for a parameter or a level below which -#' a sensor is non-operative. These thresholds are sometimes used to automatically -#' determine if an observation is erroneous due to sensor error, and therefore -#' shouldn't be included in the timeseries. -#' @param sublocation_identifier An optional human-readable identifier used to -#' specify where measurements are recorded at a monitoring location. -#' @param primary A flag identifying if the timeseries is a "primary" timeseries. -#' "Primary" timeseries (which have this flag) are standard observations which -#' undergo Bureau review and approval processes. Non-primary timeseries, which -#' will have missing values for "primary", are provisional datasets made available -#' to meet the need for timely best science and to assist with daily operations -#' which need real-time information. Non-primary timeseries data are only retained -#' by this system for 120 days. See the USGS Provisional Data Statement for more information. -#' @param web_description A description of what this timeseries represents, as -#' used by WDFN and other USGS data dissemination products. +#' each feature. The returning object will be a data frame with no spatial +#' information. #' @param limit The optional limit parameter limits the number of items that are #' presented in the response document. Only items are counted that are on the #' first level of the collection in the response document. Nested objects @@ -154,6 +102,12 @@ construct_api_requests <- function(service, match.arg(properties, choices = c(all_properties, NA_character_), several.ok = TRUE) + use_sf <- all(pkg.env$local_sf) + + if(!use_sf){ + skipGeometry <- TRUE + } + if(all(all_properties[!all_properties %in% c("id", "geometry")] %in% properties)) { # Cleans up URL if we're asking for everything properties <- NA_character_ @@ -358,3 +312,32 @@ basic_request <- function(url_base){ httr2::resp_body_json() } + +# Create descriptions dynamically +get_description <- function(service){ + + check_collections <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/server/getCollections") + + check_endpoints <- basic_request(check_collections) + tags <- check_endpoints[["tags"]] + + service_index <- which(sapply(tags, function(x){ + x$name == service + })) + + tags[[service_index]][["description"]] + +} + +# Create descriptions dynamically +get_params <- function(service){ + + check_queryables_req <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> + httr2::req_url_path_append(service) |> + httr2::req_url_path_append("schema") + + check_queryables <- basic_request(check_queryables_req) + params <- sapply(check_queryables$properties, function(x) x[["description"]]) + +} + diff --git a/R/dataRetrievals-package.R b/R/dataRetrievals-package.R index f093c204..62abe374 100644 --- a/R/dataRetrievals-package.R +++ b/R/dataRetrievals-package.R @@ -15,13 +15,13 @@ Extended Documentation: https://doi-usgs.github.io/dataRetrieval") #' that originally came from the United States Geological Survey, an agency of #' the United States Department of Interior. For more information, see the #' official USGS copyright policy at -#' \url{https://www.usgs.gov/information-policies-and-instructions/copyrights-and-credits}\cr +#' \cr #' LazyLoad: \tab yes\cr #' } #' #' Retrieval functions for USGS and EPA hydrologic and water quality data. #' -#' Please see \url{https://doi-usgs.github.io/dataRetrieval/} for more information. +#' Please see for more information. #' #' @name dataRetrieval #' @aliases dataRetrieval-package @@ -60,7 +60,7 @@ NULL #' Data to convert USGS parameter code to characteristic names #' #' Data pulled from Water Quality Portal on December 20, 2021. The data was pulled from -#' \url{https://www.waterqualitydata.us/Codes/public_srsnames/?mimeType=csv}. +#' . #' #' @name pCodeToName #' @return pCodeToName data frame with information about USGS parameters and how they @@ -90,7 +90,7 @@ NULL #' US State Code Lookup Table #' #' Classic lookup table for states. Has been replaced in functions with -#' \code{check_USGS_sample_params("states")}. +#' `check_USGS_sample_params("states")`. #' #' @name stateCd #' @return stateCd data frame. @@ -112,7 +112,7 @@ NULL #' US County Code Lookup Table #' #' Classic lookup table for counties. Has been replaced in functions with -#' \code{check_USGS_sample_params("counties")}. +#' `check_USGS_sample_params("counties")`. #' #' @name countyCd #' @return countyCd data frame. diff --git a/R/importNGWMN_wml2.R b/R/importNGWMN_wml2.R index 2f0541f1..a9782fa9 100644 --- a/R/importNGWMN_wml2.R +++ b/R/importNGWMN_wml2.R @@ -4,12 +4,12 @@ #' but the general functionality is correct. #' #' @param input character or raw, containing the url for the retrieval or a path to the data file, or raw XML. -#' @param asDateTime logical, if \code{TRUE} returns date and time as POSIXct, if \code{FALSE}, character +#' @param asDateTime logical, if `TRUE` returns date and time as POSIXct, if `FALSE`, character #' @param tz character to set timezone attribute of dateTime. Default is "UTC", and converts the #' date times to UTC, properly accounting for daylight savings times based on the data's provided time zone offset. #' Possible values to provide are "America/New_York", "America/Chicago", "America/Denver", "America/Los_Angeles", #' "America/Anchorage", as well as the following which do not use daylight savings time: "America/Honolulu", -#' "America/Jamaica", "America/Managua", "America/Phoenix", and "America/Metlakatla". See also \code{OlsonNames()} +#' "America/Jamaica", "America/Managua", "America/Phoenix", and "America/Metlakatla". See also `OlsonNames()` #' for more information on time zones. #' @return mergedDF a data frame source, time, value, uom, uomTitle, comment, gmlID #' @export @@ -138,7 +138,7 @@ importNGWMN <- function(input, asDateTime = FALSE, tz = "UTC") { #' Anything defined as a default, is returned as an attribute of that data frame. #' #' @param input XML with only the wml2:MeasurementTimeseries node and children -#' @param asDateTime logical, if \code{TRUE} returns date and time as POSIXct, if \code{FALSE}, character +#' @param asDateTime logical, if `TRUE` returns date and time as POSIXct, if `FALSE`, character #' @param tz character to set timezone attribute of datetime. Default is an empty quote, which converts the #' datetimes to UTC (properly accounting for daylight savings times based on the data's provided time zone offset). #' Possible values are "America/New_York", "America/Chicago", "America/Denver", "America/Los_Angeles", diff --git a/R/importRDB1.R b/R/importRDB1.R index 36419be4..1ca2476f 100644 --- a/R/importRDB1.R +++ b/R/importRDB1.R @@ -6,14 +6,14 @@ #' recommended to use the RDB format for importing multi-site data. #' #' @param obs_url character containing the url for the retrieval or a file path to the data file. -#' @param asDateTime logical, if \code{TRUE} returns date and time as POSIXct, if \code{FALSE}, Date +#' @param asDateTime logical, if `TRUE` returns date and time as POSIXct, if `FALSE`, Date #' @param tz character to set timezone attribute of datetime. Default converts the datetimes to UTC #' (properly accounting for daylight savings times based on the data's provided tz_cd column). #' Recommended US values include "UTC", "America/New_York", "America/Chicago", "America/Denver", #' "America/Los_Angeles", "America/Anchorage", "America/Honolulu", "America/Jamaica", "America/Managua", #' "America/Phoenix", and "America/Metlakatla". -#' For a complete list, see \url{https://en.wikipedia.org/wiki/List_of_tz_database_time_zones} -#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the +#' For a complete list, see +#' @param convertType logical, defaults to `TRUE`. If `TRUE`, the #' function will convert the data to dates, datetimes, #' numerics based on a standard algorithm. If false, everything is returned as a character #' @return A data frame with the following columns: @@ -22,7 +22,7 @@ #' agency_cd \tab character \tab The NWIS code for the agency reporting the data\cr #' site_no \tab character \tab The USGS site number \cr #' datetime \tab POSIXct \tab The date and time of the value converted to -#' UTC (if asDateTime = \code{TRUE}), \cr +#' UTC (if asDateTime = `TRUE`), \cr #' \tab character \tab or raw character string (if asDateTime = FALSE) \cr #' tz_cd \tab character \tab The time zone code for datetime \cr #' code \tab character \tab Any codes that qualify the corresponding value\cr diff --git a/R/importWQP.R b/R/importWQP.R index c245afb0..a8cd9793 100644 --- a/R/importWQP.R +++ b/R/importWQP.R @@ -8,17 +8,17 @@ #' Possible values include "America/New_York","America/Chicago", "America/Denver","America/Los_Angeles", #' "America/Anchorage","America/Honolulu","America/Jamaica","America/Managua", #' "America/Phoenix", and "America/Metlakatla" -#' @param csv logical. Is the data coming back with a csv or tsv format. Default is \code{FALSE}. +#' @param csv logical. Is the data coming back with a csv or tsv format. Default is `FALSE`. #' Currently, the summary service does not support tsv, for other services tsv is the safer choice. -#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function +#' @param convertType logical, defaults to `TRUE`. If `TRUE`, the function #' will convert the data to dates, datetimes, #' numerics based on a standard algorithm. If false, everything is returned as a character. #' @return retval dataframe raw data returned from the Water Quality Portal. Additionally, #' a POSIXct dateTime column is supplied for #' start and end times, and converted to UTC. See -#' \url{https://www.waterqualitydata.us/portal_userguide/} for more information. +#' for more information. #' @export -#' @seealso \code{\link{readWQPdata}}, \code{\link{readWQPqw}}, \code{\link{whatWQPsites}} +#' @seealso [readWQPdata()], [readWQPqw()], [whatWQPsites()] #' @examplesIf is_dataRetrieval_user() #' # These examples require an internet connection to run #' diff --git a/R/importWaterML1.R b/R/importWaterML1.R index da9ba20b..2ad98d46 100644 --- a/R/importWaterML1.R +++ b/R/importWaterML1.R @@ -5,13 +5,13 @@ #' #' @param obs_url character or raw, containing the url for the retrieval or a #' file path to the data file, or raw XML. -#' @param asDateTime logical, if \code{TRUE} returns date and time as POSIXct, if \code{FALSE}, Date +#' @param asDateTime logical, if `TRUE` returns date and time as POSIXct, if `FALSE`, Date #' @param tz character to set timezone attribute of datetime. Default converts the datetimes to UTC #' (properly accounting for daylight savings times based on the data's provided tz_cd column). #' Recommended US values include "UTC", "America/New_York", "America/Chicago", "America/Denver", #' "America/Los_Angeles", "America/Anchorage", "America/Honolulu", "America/Jamaica", "America/Managua", #' "America/Phoenix", and "America/Metlakatla". -#' For a complete list, see \url{https://en.wikipedia.org/wiki/List_of_tz_database_time_zones} +#' For a complete list, see #' @return A data frame with the following columns: #' \tabular{lll}{ #' Name \tab Type \tab Description \cr @@ -39,7 +39,7 @@ #' queryTime \tab POSIXct \tab The time the data was returned \cr #' } #' -#' @seealso \code{\link{renameNWISColumns}} +#' @seealso [renameNWISColumns()] #' @export #' @examplesIf is_dataRetrieval_user() #' site_id <- "02177000" diff --git a/R/readNGWMNdata.R b/R/readNGWMNdata.R index 5c283e81..bc8704bf 100644 --- a/R/readNGWMNdata.R +++ b/R/readNGWMNdata.R @@ -2,14 +2,14 @@ #' #' Only water level data and site locations and names are currently available through the web service. #' @param service char Service for the request - "observation" and "featureOfInterest" are implemented. -#' @param \dots Other parameters to supply, namely \code{siteNumbers} or \code{bbox} -#' @param asDateTime logical if \code{TRUE}, will convert times to POSIXct format. Currently defaults to -#' \code{FALSE} since time zone information is not included. +#' @param \dots Other parameters to supply, namely `siteNumbers` or `bbox` +#' @param asDateTime logical if `TRUE`, will convert times to POSIXct format. Currently defaults to +#' `FALSE` since time zone information is not included. #' @param tz character to set timezone attribute of dateTime. Default is "UTC", and converts the #' date times to UTC, properly accounting for daylight savings times based on the data's provided time zone offset. #' Possible values to provide are "America/New_York", "America/Chicago", "America/Denver", "America/Los_Angeles", #' "America/Anchorage", as well as the following which do not use daylight savings time: "America/Honolulu", -#' "America/Jamaica", "America/Managua", "America/Phoenix", and "America/Metlakatla". See also \code{OlsonNames()} +#' "America/Jamaica", "America/Managua", "America/Phoenix", and "America/Metlakatla". See also `OlsonNames()` #' for more information on time zones. #' @export #' @examplesIf is_dataRetrieval_user() @@ -98,15 +98,15 @@ readNGWMNdata <- function(service, ..., asDateTime = TRUE, tz = "UTC") { #' Retrieve groundwater levels from the National Ground Water Monitoring Network. #' #' @param siteNumbers character Vector of feature IDs formatted with agency code and site number -#' separated by a period or semicolon, e.g. \code{USGS.404159100494601}. +#' separated by a period or semicolon, e.g. `USGS.404159100494601`. #' @param asDateTime logical Should dates and times be converted to date/time objects, -#' or returned as character? Defaults to \code{TRUE}. Must be set to \code{FALSE} if a site +#' or returned as character? Defaults to `TRUE`. Must be set to `FALSE` if a site #' contains non-standard dates. #' @param tz character to set timezone attribute of dateTime. Default is "UTC", and converts the #' date times to UTC, properly accounting for daylight savings times based on the data's provided time zone offset. #' Possible values to provide are "America/New_York", "America/Chicago", "America/Denver", "America/Los_Angeles", #' "America/Anchorage", as well as the following which do not use daylight savings time: "America/Honolulu", -#' "America/Jamaica", "America/Managua", "America/Phoenix", and "America/Metlakatla". See also \code{OlsonNames()} +#' "America/Jamaica", "America/Managua", "America/Phoenix", and "America/Metlakatla". See also `OlsonNames()` #' for more information on time zones. #' @export #' @@ -139,7 +139,7 @@ readNGWMNlevels <- function(siteNumbers, asDateTime = TRUE, tz = "UTC") { #' Retrieve site data from the National Ground Water Monitoring Network. #' #' @param siteNumbers character Vector of feature IDs formatted with agency code and site number -#' separated by a period or semicolon, e.g. \code{USGS.404159100494601}. +#' separated by a period or semicolon, e.g. `USGS.404159100494601`. #' #' @export #' @return A data frame the following columns: diff --git a/R/readNWISdata.R b/R/readNWISdata.R index 8b4c1827..d1772ff0 100644 --- a/R/readNWISdata.R +++ b/R/readNWISdata.R @@ -1,20 +1,20 @@ #' General Data Import from NWIS #' #' Returns data from the NWIS web service. -#' Arguments to the function should be based on \url{https://waterservices.usgs.gov} service calls. +#' Arguments to the function should be based on service calls. #' See examples below for ideas of constructing queries. #' -#' @param asDateTime logical, if \code{TRUE} returns date and time as POSIXct, if \code{FALSE}, Date -#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the +#' @param asDateTime logical, if `TRUE` returns date and time as POSIXct, if `FALSE`, Date +#' @param convertType logical, defaults to `TRUE`. If `TRUE`, the #' function will convert the data to dates, datetimes, #' numerics based on a standard algorithm. If false, everything is returned as a character #' @param tz character to set timezone attribute of dateTime. Default is "UTC", and converts the #' date times to UTC, properly accounting for daylight savings times based on the data's provided tz_cd column. #' Possible values to provide are "America/New_York", "America/Chicago", "America/Denver", "America/Los_Angeles", #' "America/Anchorage", as well as the following which do not use daylight savings time: "America/Honolulu", -#' "America/Jamaica", "America/Managua", "America/Phoenix", and "America/Metlakatla". See also \code{OlsonNames()} +#' "America/Jamaica", "America/Managua", "America/Phoenix", and "America/Metlakatla". See also `OlsonNames()` #' for more information on time zones. -#' @param \dots see \url{https://waterservices.usgs.gov/docs/site-service/} for +#' @param \dots see for #' a complete list of options. A #' list of arguments can also be supplied. One important argument to include is #' "service". Possible values are "iv" @@ -22,14 +22,14 @@ #' "dv" (for daily values), "gwlevels" (for groundwater levels), #' "site" (for site service), "measurement", and "stat" (for #' statistics service). Note: "measurement" calls go to: -#' \url{https://nwis.waterdata.usgs.gov/usa/nwis} for data requests, and use different call requests schemes. +#' for data requests, and use different call requests schemes. #' The statistics service has a limited selection of arguments -#' (see \url{https://waterservices.usgs.gov/docs/site-service/}). +#' (see ). #' #' @details This function requires users to create their own arguments #' based on the NWIS web services. It is a more complicated function to use -#' compared to other NWIS functions such as \code{\link{readNWISdv}}, \code{\link{readNWISuv}}, -#' \code{\link{readNWISgwl}}, etc. However, this function adds a lot of +#' compared to other NWIS functions such as [readNWISdv()], [readNWISuv()], +#' [readNWISgwl()], etc. However, this function adds a lot of #' flexibility to the possible queries. This function will also behave exactly #' as NWIS when it comes to date queries. NWIS by default will only return the latest #' value for the daily and instantaneous services. So if you do not provide @@ -71,7 +71,7 @@ #' queryTime \tab POSIXct \tab The time the data was returned \cr #' } #' -#' @seealso \code{\link{renameNWISColumns}}, \code{\link{importWaterML1}}, \code{\link{importRDB1}} +#' @seealso [renameNWISColumns()], [importWaterML1()], [importRDB1()] #' @export #' @examplesIf is_dataRetrieval_user() #' \donttest{ @@ -281,8 +281,8 @@ https://cran.r-project.org/web/packages/dataRetrieval/vignettes/qwdata_changes.h #' State code look up #' -#' Function to simplify finding state and state code definitions. Used in \code{readNWISdata} -#' and \code{readWQPdata}. +#' Function to simplify finding state and state code definitions. Used in `readNWISdata` +#' and `readWQPdata`. #' #' @param input could be character (full name, abbreviation, id), or numeric (id) #' @param country description @@ -345,8 +345,8 @@ stateCdLookup <- function(input, #' US county code look up #' -#' Function to simplify finding county and county code definitions. Used in \code{readNWISdata} -#' and \code{readNWISuse}. Currently only has US counties. +#' Function to simplify finding county and county code definitions. Used in `readNWISdata` +#' and `readNWISuse`. Currently only has US counties. #' #' @param state could be character (full name, abbreviation, id), or numeric (id) #' @param county could be character (name, with or without "County") or numeric (id) @@ -425,7 +425,7 @@ countyCdLookup <- function(state, county, outputType = "fips") { } #' -#' Format and organize NWIS arguments that are passed in as \code{...}. +#' Format and organize NWIS arguments that are passed in as `...`. #' #' @keywords internal readNWISdots <- function(...) { @@ -584,4 +584,4 @@ convertLists <- function(...) { list(...)[sapply(list(...), class) != "list"] ) # get the non-list parts return(matchReturn) -} \ No newline at end of file +} diff --git a/R/readNWISdv.R b/R/readNWISdv.R index 95fb5e76..4c5ac90d 100644 --- a/R/readNWISdv.R +++ b/R/readNWISdv.R @@ -1,18 +1,18 @@ #' Daily Value USGS NWIS Data Retrieval #' #' Imports data from NWIS daily web service. This function gets the data from here: -#' \url{https://waterservices.usgs.gov/docs/dv-service/daily-values-service-details/} +#' #' Inputs to this function are just USGS site ids, USGS parameter codes, -#' USGS statistic codes, and start and end date. For a more complex query, use \code{\link{readNWISdata}}, +#' USGS statistic codes, and start and end date. For a more complex query, use [readNWISdata()], #' with an argument service = "dv". #' Data coming the daily web services are aggregates of the instantaneous #' (sensor) web services. Not all statistical codes are available for all data. -#' Use the function \code{\link{whatNWISdata}} to discover what data +#' Use the function [whatNWISdata()] to discover what data #' is available for a USGS site. The column data_type_cd with the values "dv" -#' returned from \code{\link{whatNWISdata}}) are available from this service. +#' returned from [whatNWISdata()]) are available from this service. #' #' More information on the web service can be found here: -#' \url{https://waterservices.usgs.gov/test-tools}, choosing the +#' , choosing the #' "Daily Value Service". #' #' @param siteNumbers character USGS site number. This is usually an 8 digit number. @@ -49,7 +49,7 @@ #' queryTime \tab POSIXct \tab The time the data was returned \cr #' } #' -#' @seealso \code{\link{renameNWISColumns}}, \code{\link{importWaterML1}} +#' @seealso [renameNWISColumns()], [importWaterML1()] #' @export #' @keywords data import USGS web service #' @examplesIf is_dataRetrieval_user() diff --git a/R/readNWISpCode.R b/R/readNWISpCode.R index 1bb77bd7..93b9431a 100644 --- a/R/readNWISpCode.R +++ b/R/readNWISpCode.R @@ -1,10 +1,10 @@ #' USGS Parameter Data Retrieval #' #' Imports data from NWIS about measured parameter based on user-supplied parameter code or codes. -#' This function gets the data from here: \url{https://nwis.waterdata.usgs.gov/nwis/pmcodes} +#' This function gets the data from here: #' #' @param parameterCd character of USGS parameter codes (or multiple parameter codes). These are 5 digit number codes, -#' more information can be found here: \url{https://help.waterdata.usgs.gov/}. To get a +#' more information can be found here: . To get a #' complete list of all current parameter codes in the USGS, use "all" as the input. #' @keywords data import USGS web service #' @return parameterData data frame with the following information: @@ -19,7 +19,7 @@ #' } #' #' @export -#' @seealso \code{\link{importRDB1}} +#' @seealso [importRDB1()] #' @examples #' #' paramINFO <- readNWISpCode(c("01075", "00060", "00931")) diff --git a/R/readNWISsite.R b/R/readNWISsite.R index 5898ff32..c1b110f9 100644 --- a/R/readNWISsite.R +++ b/R/readNWISsite.R @@ -1,6 +1,6 @@ #' USGS Site File Data Retrieval #' -#' Imports data from USGS site file site. This function gets data from here: \url{https://waterservices.usgs.gov/} +#' Imports data from USGS site file site. This function gets data from here: #' #' @param siteNumbers character USGS site number (or multiple sites). This is usually an 8 digit number #' @keywords data import USGS web service diff --git a/R/readNWISunit.R b/R/readNWISunit.R index b3fea426..63568b14 100644 --- a/R/readNWISunit.R +++ b/R/readNWISunit.R @@ -1,32 +1,32 @@ #' Instantaneous value data retrieval from USGS (NWIS) #' #' Imports data from NWIS web service. This function gets the data from here: -#' \url{https://waterservices.usgs.gov/docs/instantaneous-values/instantaneous-values-details/} +#' #' Inputs to this function are just USGS site ids, USGS parameter codes, -#' and start and end date. For a more complex query, use \code{\link{readNWISdata}}, +#' and start and end date. For a more complex query, use [readNWISdata()], #' including an arguement service="uv". #' Not all parameter codes are available for all data. -#' Use the function \code{\link{whatNWISdata}} to discover what data +#' Use the function [whatNWISdata()] to discover what data #' is available for a USGS site. The column data_type_cd with the values "uv" -#' returned from \code{\link{whatNWISdata}}) are available from this service. +#' returned from [whatNWISdata()]) are available from this service. #' #' More information on the web service can be found here: -#' \url{https://waterservices.usgs.gov/test-tools}, choosing the +#' , choosing the #' "Instantaneous Value Service". #' #' @param siteNumbers character USGS site number (or multiple sites). This is usually an 8 digit number #' @param parameterCd character USGS parameter code. This is usually an 5 digit number. #' @param startDate character starting date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates #' retrieval for the earliest possible record. Simple date arguments are specified in local time. -#' See more information here: \url{https://waterservices.usgs.gov/docs/instantaneous-values/}. +#' See more information here: . #' @param endDate character ending date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates #' retrieval for the latest possible record. Simple date arguments are specified in local time. -#' See more information here: \url{https://waterservices.usgs.gov/docs/instantaneous-values/}. +#' See more information here: . #' @param tz character to set timezone attribute of dateTime. Default is "UTC", and converts the #' date times to UTC, properly accounting for daylight savings times based on the data's provided tz_cd column. #' Possible values to provide are "America/New_York", "America/Chicago", "America/Denver", "America/Los_Angeles", #' "America/Anchorage", as well as the following which do not use daylight savings time: "America/Honolulu", -#' "America/Jamaica", "America/Managua", "America/Phoenix", and "America/Metlakatla". See also \code{OlsonNames()} +#' "America/Jamaica", "America/Managua", "America/Phoenix", and "America/Metlakatla". See also `OlsonNames()` #' for more information on time zones. #' @keywords data import USGS web service #' @return A data frame with the following columns: @@ -55,7 +55,7 @@ #' queryTime \tab POSIXct \tab The time the data was returned \cr #' } #' -#' @seealso \code{\link{renameNWISColumns}}, \code{\link{importWaterML1}} +#' @seealso [renameNWISColumns()], [importWaterML1()] #' @export #' @examplesIf is_dataRetrieval_user() #' site_id <- "05114000" @@ -107,7 +107,7 @@ readNWISuv <- function(siteNumbers, parameterCd, startDate = "", endDate = "", t #' Peak flow data from USGS (NWIS) #' #' Reads peak flow from NWISweb. Data is retrieved from -#' \url{https://waterdata.usgs.gov/nwis}. +#' . #' In some cases, the specific date of the peak data is not know. This function #' will default to #' converting complete dates to a "Date" object, and converting incomplete dates to @@ -123,10 +123,10 @@ readNWISuv <- function(siteNumbers, parameterCd, startDate = "", endDate = "", t #' @param endDate character ending date for data retrieval in the form YYYY-MM-DD. #' Default is "" which indicates #' retrieval for the latest possible record. -#' @param asDateTime logical default to \code{TRUE}. When \code{TRUE}, the peak_dt column is converted -#' to a Date object, and incomplete dates are removed. When \code{FALSE}, no +#' @param asDateTime logical default to `TRUE`. When `TRUE`, the peak_dt column is converted +#' to a Date object, and incomplete dates are removed. When `FALSE`, no #' columns are removed, but no dates are converted. -#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function +#' @param convertType logical, defaults to `TRUE`. If `TRUE`, the function #' will convert the data to dates, datetimes, #' numerics based on a standard algorithm. If false, everything is returned as a character #' @return A data frame with the following columns: @@ -137,7 +137,7 @@ readNWISuv <- function(siteNumbers, parameterCd, startDate = "", endDate = "", t #' peak_dt \tab Date \tab Date of peak streamflow \cr #' peak_tm \tab character \tab Time of peak streamflow as character \cr #' peak_va \tab numeric \tab Annual peak streamflow value in cfs \cr -#' peak_cd \tab character \tab Peak Discharge-Qualification codes (see \code{comment} +#' peak_cd \tab character \tab Peak Discharge-Qualification codes (see `comment` #' for more information) \cr #' gage_ht \tab numeric \tab Gage height for the associated peak streamflow in feet \cr #' gage_ht_cd \tab character \tab Gage height qualification codes \cr @@ -159,7 +159,7 @@ readNWISuv <- function(siteNumbers, parameterCd, startDate = "", endDate = "", t #' comment \tab character \tab Header comments from the RDB file \cr #' siteInfo \tab data.frame \tab A data frame containing information on the requested sites \cr #' } -#' @seealso \code{\link{constructNWISURL}}, \code{\link{importRDB1}} +#' @seealso [constructNWISURL()], [importRDB1()] #' @export #' @examplesIf is_dataRetrieval_user() #' site_ids <- c("01594440", "040851325") @@ -226,22 +226,22 @@ readNWISpeak <- function(siteNumbers, #' Rating table for an active USGS streamgage retrieval #' #' Reads current rating table for an active USGS streamgage from NWISweb. -#' Data is retrieved from \url{https://waterdata.usgs.gov/nwis}. +#' Data is retrieved from . #' #' @param siteNumber character USGS site number. This is usually an 8 digit number #' @param type character can be "base", "corr", or "exsa" -#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function +#' @param convertType logical, defaults to `TRUE`. If `TRUE`, the function #' will convert the data to dates, datetimes, #' numerics based on a standard algorithm. If false, everything is returned as a character -#' @return A data frame. If \code{type} is "base, " then the columns are +#' @return A data frame. If `type` is "base, " then the columns are #' INDEP, typically the gage height, in feet; DEP, typically the streamflow, #' in cubic feet per second; and STOR, where "*" indicates that the pair are -#' a fixed point of the rating curve. If \code{type} is "exsa, " then an +#' a fixed point of the rating curve. If `type` is "exsa, " then an #' additional column, SHIFT, is included that indicates the current shift in -#' the rating for that value of INDEP. If \code{type} is "corr, " then the +#' the rating for that value of INDEP. If `type` is "corr, " then the #' columns are INDEP, typically the gage height, in feet; CORR, the correction #' for that value; and CORRINDEP, the corrected value for CORR.\cr -#' If \code{type} is "base, " then the data frame has an attribute called "RATING" +#' If `type` is "base, " then the data frame has an attribute called "RATING" #' that describes the rating curve is included. #' #' There are also several useful attributes attached to the data frame: @@ -256,7 +256,7 @@ readNWISpeak <- function(siteNumbers, #' #' @note Not all active USGS streamgages have traditional rating curves that #' relate flow to stage. -#' @seealso \code{\link{constructNWISURL}}, \code{\link{importRDB1}} +#' @seealso [constructNWISURL()], [importRDB1()] #' @export #' @examplesIf is_dataRetrieval_user() #' site_id <- "01594440" @@ -296,8 +296,8 @@ readNWISrating <- function(siteNumber, type = "base", convertType = TRUE) { #' Surface-water measurement data retrieval from USGS (NWIS) #' -#' Reads surface-water measurement data from NWISweb. Data is retrieved from \url{https://waterdata.usgs.gov/nwis}. -#' See \url{https://waterdata.usgs.gov/usa/nwis/sw} for details about surface water. +#' Reads surface-water measurement data from NWISweb. Data is retrieved from . +#' See for details about surface water. #' #' @param siteNumbers character USGS site number (or multiple sites). This is usually an 8 digit number #' @param startDate character starting date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates @@ -308,10 +308,10 @@ readNWISrating <- function(siteNumber, type = "base", convertType = TRUE) { #' date times to UTC, properly accounting for daylight savings times based on the data's provided tz_cd column. #' Possible values to provide are "America/New_York", "America/Chicago", "America/Denver", "America/Los_Angeles", #' "America/Anchorage", as well as the following which do not use daylight savings time: "America/Honolulu", -#' "America/Jamaica", "America/Managua", "America/Phoenix", and "America/Metlakatla". See also \code{OlsonNames()} +#' "America/Jamaica", "America/Managua", "America/Phoenix", and "America/Metlakatla". See also `OlsonNames()` #' for more information on time zones. #' @param expanded logical. Whether or not (TRUE or FALSE) to call the expanded data. -#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function will +#' @param convertType logical, defaults to `TRUE`. If `TRUE`, the function will #' convert the data to dates, datetimes, #' numerics based on a standard algorithm. If false, everything is returned as a character #' @return A data frame with at least the following columns: @@ -326,8 +326,8 @@ readNWISrating <- function(siteNumber, type = "base", convertType = TRUE) { #' tz_cd \tab character \tab The time zone code for the measurement_dt column \cr #' } #' -#' See \url{https://waterdata.usgs.gov/usa/nwis/sw} for details about surface water, and -#' \url{https://waterdata.usgs.gov/nwis/help?output_formats_help} +#' See for details about surface water, and +#' #' for help on the columns and codes. #' #' There are also several useful attributes attached to the data frame: @@ -339,7 +339,7 @@ readNWISrating <- function(siteNumber, type = "base", convertType = TRUE) { #' siteInfo \tab data.frame \tab A data frame containing information on the requested sites \cr #' tz_cd_reported \tab The originally reported time zone \cr #' } -#' @seealso \code{\link{constructNWISURL}}, \code{\link{importRDB1}} +#' @seealso [constructNWISURL()], [importRDB1()] #' @export #' @examplesIf is_dataRetrieval_user() #' site_ids <- c("01594440", "040851325") @@ -427,25 +427,25 @@ readNWISmeas <- function(siteNumbers, #' Groundwater level measurements retrieval from USGS (NWIS) #' #' Imports groundwater level data from NWIS web service. This function gets the data from here: -#' \url{https://waterservices.usgs.gov/docs/groundwater-levels/groundwater-levels-details/} +#' #' Inputs to this function are just USGS site ids, USGS parameter codes, -#' and start and end date. For a more complex query, use \code{\link{readNWISdata}}, +#' and start and end date. For a more complex query, use [readNWISdata()], #' including an argument service="gwlevels". #' Not all parameter codes are available for all data. -#' Use the function \code{\link{whatNWISdata}} to discover what data +#' Use the function [whatNWISdata()] to discover what data #' is available for a USGS site. The column data_type_cd with the values "gw" -#' returned from \code{\link{whatNWISdata}}) are available from this service. +#' returned from [whatNWISdata()]) are available from this service. #' #' More information on the web service can be found here: -#' \url{https://waterservices.usgs.gov/test-tools}, choosing the +#' , choosing the #' "Groundwater Levels Value Service". #' #' #' Mixed date/times come back from the service -#' depending on the year that the data was collected. See \url{https://waterdata.usgs.gov/usa/nwis/gw} +#' depending on the year that the data was collected. See #' for details about groundwater. By default the returned dates are converted to date objects, unless convertType #' is specified as FALSE. Sites with non-standard date formats (i.e. lacking a day) can be affected (see examples). -#' See \url{https://waterservices.usgs.gov/docs/groundwater-levels/} for more information. +#' See for more information. #' #' @param siteNumbers character USGS site number (or multiple sites). This is usually an 8 digit number #' @param startDate character starting date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates @@ -453,14 +453,14 @@ readNWISmeas <- function(siteNumbers, #' @param endDate character ending date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates #' retrieval for the latest possible record. #' @param parameterCd character USGS parameter code. This is usually an 5 digit number. Default is "". -#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the +#' @param convertType logical, defaults to `TRUE`. If `TRUE`, the #' function will convert the data to dates, datetimes, #' numerics based on a standard algorithm. If false, everything is returned as a character #' @param tz character to set timezone attribute of dateTime. Default is "UTC", and converts the #' date times to UTC, properly accounting for daylight savings times based on the data's provided tz_cd column. #' Possible values to provide are "America/New_York", "America/Chicago", "America/Denver", "America/Los_Angeles", #' "America/Anchorage", as well as the following which do not use daylight savings time: "America/Honolulu", -#' "America/Jamaica", "America/Managua", "America/Phoenix", and "America/Metlakatla". See also \code{OlsonNames()} +#' "America/Jamaica", "America/Managua", "America/Phoenix", and "America/Metlakatla". See also `OlsonNames()` #' for more information on time zones. #' @return A data frame with the following columns: #' \tabular{lll}{ @@ -486,7 +486,7 @@ readNWISmeas <- function(siteNumbers, #' siteInfo \tab data.frame \tab A data frame containing information on the requested sites \cr #' } #' -#' @seealso \code{\link{constructNWISURL}}, \code{\link{importRDB1}} +#' @seealso [constructNWISURL()], [importRDB1()] #' @export #' @examplesIf is_dataRetrieval_user() #' site_id <- "434400121275801" @@ -550,7 +550,7 @@ readNWISgwl <- function(siteNumbers, #' Site statistics retrieval from USGS (NWIS) #' #' Retrieves site statistics from the USGS Statistics Web Service beta. -#' See \url{https://waterservices.usgs.gov/docs/statistics/} for more information. +#' See for more information. #' #' @param siteNumbers character USGS site number (or multiple sites). This is usually an 8 digit number. #' @param parameterCd character USGS parameter code. This is usually a 5 digit number. @@ -563,7 +563,7 @@ readNWISgwl <- function(siteNumbers, #' which indicates retrieval for the latest possible record. For daily data, this #' indicates the end of the period #' the statistics will be computed over. The same restrictions as startDate apply. -#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function will convert the data to +#' @param convertType logical, defaults to `TRUE`. If `TRUE`, the function will convert the data to #' numerics based on a standard algorithm. Years, months, and days (if appliccable) are also returned as numerics #' in separate columns. If convertType is false, everything is returned as a character. #' @param statReportType character time division for statistics: daily, monthly, or annual. Default is daily. @@ -576,7 +576,7 @@ readNWISgwl <- function(siteNumbers, #' @param statType character type(s) of statistics to output for daily values. #' Default is mean, which is the only #' option for monthly and yearly report types. See the statistics service documentation -#' at \url{https://waterservices.usgs.gov/docs/statistics/} for a full list of codes. +#' at for a full list of codes. #' @return A data frame with the following columns: #' \tabular{lll}{ #' Name \tab Type \tab Description \cr @@ -586,7 +586,7 @@ readNWISgwl <- function(siteNumbers, #' #' Other columns will be present depending on statReportType and statType #' } -#' @seealso \code{\link{constructNWISURL}}, \code{\link{importRDB1}} +#' @seealso [constructNWISURL()], [importRDB1()] #' @export #' @examplesIf is_dataRetrieval_user() #' \donttest{ @@ -660,13 +660,13 @@ readNWISstat <- function(siteNumbers, parameterCd, startDate = "", endDate = "", #' Water use data retrieval from USGS (NWIS) #' #' Retrieves water use data from USGS Water Use Data for the Nation. See -#' \url{https://waterdata.usgs.gov/nwis/wu} for +#' for #' more information. All available use categories for the supplied arguments are retrieved. #' #' @param stateCd could be character (full name, abbreviation, id), or numeric (id). #' Only one is accepted per query. #' @param countyCd could be character (name, with or without "County", or "ALL"), -#' numeric (id), or \code{NULL}, which will +#' numeric (id), or `NULL`, which will #' return state or national data depending on the stateCd argument. "ALL" may #' also be supplied, which will return data #' for every county in a state. Can be a vector of counties in the same state. @@ -676,14 +676,14 @@ readNWISstat <- function(siteNumbers, parameterCd, startDate = "", endDate = "", #' Specific categories must be supplied as two- #' letter abbreviations as seen in the URL when using the NWIS water use web interface. Note that #' there are different codes for national and state level data. -#' @param convertType logical defaults to \code{TRUE}. If \code{TRUE}, the function +#' @param convertType logical defaults to `TRUE`. If `TRUE`, the function #' will convert the data to #' numerics based on a standard algorithm. Years, months, and days (if appliccable) are #' also returned as numerics #' in separate columns. If convertType is false, everything is returned as a character. #' @param transform logical only intended for use with national data. Defaults to -#' \code{FALSE}, with data being returned as -#' presented by the web service. If \code{TRUE}, data will be transformed and +#' `FALSE`, with data being returned as +#' presented by the web service. If `TRUE`, data will be transformed and #' returned with column names, which will reformat #' national data to be similar to state data. #' @return A data frame with at least the year of record, and all available diff --git a/R/readWQPdata.R b/R/readWQPdata.R index 81e460a3..19a6dbea 100644 --- a/R/readWQPdata.R +++ b/R/readWQPdata.R @@ -1,7 +1,7 @@ #' General Data Import from Water Quality Portal #' #' Imports data from Water Quality Portal web service. This function gets the data from here: -#' \url{https://www.waterqualitydata.us}. +#' . #' #' This function uses \dots as a query input, which can be very flexible, but also #' has a steeper learning curve. For a quick overview, scroll down to the Examples @@ -41,7 +41,7 @@ #' } #' #' -#' @param \dots see \url{https://www.waterqualitydata.us/webservices_documentation} for a complete list of options. +#' @param \dots see for a complete list of options. #' A list of arguments can also be supplied. For more information see the above #' description for this help file. One way to figure out how to construct a WQP query is to go to the "Advanced" #' form in the Water Quality Portal. Use the form to discover what parameters are available. Once the query is @@ -63,11 +63,11 @@ #' "America/Anchorage", as well as the following which do not use daylight savings #' time: "America/Honolulu", #' "America/Jamaica","America/Managua","America/Phoenix", and "America/Metlakatla". -#' See also \code{OlsonNames()} +#' See also `OlsonNames()` #' for more information on time zones. #' @param ignore_attributes logical to choose to ignore fetching site and status -#' attributes. Default is \code{FALSE}. -#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function +#' attributes. Default is `FALSE`. +#' @param convertType logical, defaults to `TRUE`. If `TRUE`, the function #' will convert the data to dates, datetimes, #' numerics based on a standard algorithm. If false, everything is returned as a character. #' @keywords data import WQP web service diff --git a/R/readWQPdots.R b/R/readWQPdots.R index 29c4bfe2..49d49589 100644 --- a/R/readWQPdots.R +++ b/R/readWQPdots.R @@ -1,5 +1,5 @@ #' -#' Format and organize WQP arguments that are passed in as \code{...}. +#' Format and organize WQP arguments that are passed in as `...`. #' #' @keywords internal readWQPdots <- function(..., legacy = TRUE) { diff --git a/R/readWQPqw.R b/R/readWQPqw.R index 903a57ca..faaca270 100644 --- a/R/readWQPqw.R +++ b/R/readWQPqw.R @@ -1,7 +1,7 @@ #' Raw Data Import for Water Quality Portal #' #' Imports data from the Water Quality Portal. -#' This function gets the data from here: \url{https://www.waterqualitydata.us}. There +#' This function gets the data from here: . There #' are four required input arguments: siteNumbers, parameterCd, startDate, and endDate. #' parameterCd can either be a USGS 5-digit code, or a characteristic name. The sites can be #' either USGS, or other Water Quality Portal offered sites. It is required to use the 'full' @@ -24,15 +24,15 @@ #' "America/Anchorage", as well as the following which do not use daylight savings #' time: "America/Honolulu", #' "America/Jamaica","America/Managua","America/Phoenix", and "America/Metlakatla". -#' See also \code{OlsonNames()} +#' See also `OlsonNames()` #' for more information on time zones. #' @param querySummary logical to look at number of records and unique sites that #' will be returned from this query. -#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function +#' @param convertType logical, defaults to `TRUE`. If `TRUE`, the function #' will convert the data to dates, datetimes, #' numerics based on a standard algorithm. If false, everything is returned as a character. #' @param ignore_attributes logical to choose to ignore fetching site and parameter -#' attributes. Default is \code{FALSE}. +#' attributes. Default is `FALSE`. #' @param legacy Logical. If TRUE, uses legacy WQP services. Default is TRUE. #' Setting legacy = FALSE uses WQX3.0 WQP services, which are in-development, use with caution. #' @keywords data import USGS web service @@ -47,8 +47,8 @@ #' queryTime \tab POSIXct \tab The time the data was returned \cr #' } #' @export -#' @seealso \code{\link{readWQPdata}}, \code{\link{whatWQPsites}}, -#' and \code{\link{importWQP}} +#' @seealso [readWQPdata()], [whatWQPsites()], +#' and [importWQP()] #' @examplesIf is_dataRetrieval_user() #' \donttest{ #' rawPcode <- readWQPqw("USGS-01594440", "01075", "", "") diff --git a/R/read_USGS_dv.R b/R/read_USGS_dv.R index c2690dca..be866424 100644 --- a/R/read_USGS_dv.R +++ b/R/read_USGS_dv.R @@ -1,77 +1,36 @@ #' Get USGS Daily Data #' -#' Daily data provide one data value to represent water conditions for the day. -#' Throughout much of the history of the USGS, the primary water data available -#' was daily data collected manually at the monitoring location once each day. -#' With improved availability of computer storage and automated transmission of -#' data, the daily data published today are generally a statistical summary or -#' metric of the continuous data collected each day, such as the daily mean, -#' minimum, or maximum value. Daily data are automatically calculated from the -#' continuous data of the same parameter code and are described by parameter code -#' and a statistic code. These data have also been referred to as "daily values" -#' or "DV". +#' Description `r get_description("daily")` #' #' @export -#' @param monitoring_location_id A unique identifier representing a single monitoring -#' location. This corresponds to the id field in the sites endpoint. Monitoring -#' location IDs are created by combining the agency code of the agency responsible -#' for the monitoring location (e.g. USGS) with the ID number of the monitoring -#' location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500). +#' @param monitoring_location_id `r get_params("daily")$monitoring_location_id` +#' @param parameter_code `r get_params("daily")$parameter_code` +#' @param datetime `r get_params("daily")$datetime` +#' @param statistic_id `r get_params("daily")$statistic_id` +#' @param time `r get_params("daily")$time` +#' @param value `r get_params("daily")$value` +#' @param unit_of_measure `r get_params("daily")$unit_of_measure` +#' @param approval_status `r get_params("daily")$approval_status` +#' @param last_modified `r get_params("daily")$last_modified` +#' @param time_series_id `r get_params("daily")$time_series_id` +#' @param qualifier `r get_params("daily")$qualifier` +#' @param id `r get_params("daily")$id` +#' @param properties The properties that should be included for each feature. +#' The parameter value is a comma-separated list of property names. Available options are +#' `r schema <- check_OGC_requests(endpoint = "daily", type = "schema"); paste(names(schema$properties), collapse = ", ")` #' @param bbox Only features that have a geometry that intersects the bounding #' box are selected.The bounding box is provided as four or six numbers, depending #' on whether the coordinate reference system includes a vertical axis (height or -#' depth). -#' @param crs Indicates the coordinate reference system for the results. -#' @param bbox_crs Indicates the coordinate reference system for the given bbox -#' coordinates. -#' @param properties The properties that should be included for each feature. The -#' parameter value is a comma-separated list of property names. Available values: -#' id, timeseries_id, monitoring_location_id, parameter_code, statistic_id, time, -#' value, unit_of_measure, approval_status, qualifier, last_modified. -#' @param skipGeometry This option can be used to skip response geometries for -#' each feature. -#' @param datetime Either a date-time or an interval. Only features that have a -#' temporal property that intersects the value of datetime are selected. If a -#' feature has multiple temporal properties, it is the decision of the server -#' whether only a single temporal property is used to determine the extent or -#' all relevant temporal properties. -#' @param id A universally unique identifier (UUID) representing a single version -#' of a record. It is not stable over time. Every time the record is refreshed in -#' our database (which may happen as part of normal operations and does not imply -#' any change to the data itself) a new ID will be generated. To uniquely identify -#' a single observation over time, compare the time and timeseries_id fields; each -#' timeseries will only have a single observation at a given time. -#' @param time_series_id A unique identifier representing a single timeseries. -#' This corresponds to the id field in the timeseries-metadata endpoint. -#' @param parameter_code Parameter codes are 5-digit codes used to identify the -#' constituent measured and the units of measure. -#' @param statistic_id A code corresponding to the statistic an observation represents. -#' Example codes include 00001 (max), 00002 (min), and 00003 (mean). -#' @param time The date an observation represents. -#' @param value The value of the observation. Values are transmitted as strings -#' in the JSON response format in order to preserve precision. -#' @param unit_of_measure A human-readable description of the units of measurement -#' associated with an observation. -#' @param approval_status Some of the data that you have obtained from this U.S. -#' Geological Survey database may not have received Director's approval. Any such -#' data values are qualified as provisional and are subject to revision. Provisional -#' data are released on the condition that neither the USGS nor the United States -#' Government may be held liable for any damages resulting from its use. This field -#' reflects the approval status of each record, and is either "Approved", meaning -#' processing review has been completed and the data is approved for publication, -#' or "Provisional" and subject to revision. -#' @param qualifier This field indicates any qualifiers associated with an observation, -#' for instance if a sensor may have been impacted by ice or if values were estimated. -#' @param last_modified The last time a record was refreshed in our database. This -#' may happen due to regular operational processes and does not necessarily indicate -#' anything about the measurement has changed. You can query this field using -#' date-times or intervals. +#' depth). Coordinates are assumed to be in crs 4326. #' @param crs Indicates the coordinate reference system for the results. #' @param limit The optional limit parameter limits the number of items that are #' presented in the response document. Only items are counted that are on the #' first level of the collection in the response document. Nested objects #' contained within the explicitly requested items shall not be counted. -#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function +#' @param skipGeometry This option can be used to skip response geometries for +#' each feature. The returning object will be a data frame with no spatial +#' information. +#' @param convertType logical, defaults to `TRUE`. If `TRUE`, the function #' will convert the data to dates and qualifier to string vector. #' @examplesIf is_dataRetrieval_user() #' @@ -82,7 +41,7 @@ #' parameter_code = "00060", #' datetime = c("2021-01-01", "2022-01-01")) #' -#' dv_data_sf <- read_USGS_dv(monitoring_location_id = site, +#' dv_data_trim <- read_USGS_dv(monitoring_location_id = site, #' parameter_code = "00060", #' properties = c("monitoring_location_id", #' "value", @@ -91,7 +50,7 @@ #' #' dv_data <- read_USGS_dv(monitoring_location_id = site, #' parameter_code = "00060", -#' no_sf = TRUE) +#' skipGeometry = TRUE) #' #' multi_site <- read_USGS_dv(monitoring_location_id = c("USGS-01491000", #' "USGS-01645000"), @@ -120,19 +79,12 @@ read_USGS_dv <- function(monitoring_location_id = NA_character_, last_modified = NA_character_, limit = 10000, crs = NA_character_, - bbox_crs = NA_character_, skipGeometry = NA, datetime = NA_character_, convertType = TRUE){ message("Function in development, use at your own risk.") - use_sf <- all(pkg.env$local_sf) - - if(!use_sf){ - skipGeometry <- TRUE - } - dv_req <- construct_api_requests(service = "daily", monitoring_location_id = monitoring_location_id, parameter_code = parameter_code, @@ -148,11 +100,10 @@ read_USGS_dv <- function(monitoring_location_id = NA_character_, last_modified = last_modified, limit = limit, crs = crs, - bbox_crs = bbox_crs, skipGeometry = skipGeometry, datetime = datetime) - return_list <- walk_pages(dv_req, use_sf) + return_list <- walk_pages(dv_req) if(convertType) return_list <- cleanup_cols(return_list) diff --git a/R/read_USGS_samples.R b/R/read_USGS_samples.R index 5444e63b..87beab69 100644 --- a/R/read_USGS_samples.R +++ b/R/read_USGS_samples.R @@ -1,12 +1,12 @@ #' Construct request for USGS Samples Data #' #' This function creates the call for discrete water quality samples data -#' service described at \url{https://waterdata.usgs.gov/download-samples}. +#' service described at . #' Note: all possible arguments are included, but it is strongly recommended #' to only use the NECESSARY arguments. Leave unnecessary arguments as the default #' NA. #' -#' See also: \url{https://api.waterdata.usgs.gov/samples-data/docs}. +#' See also: . #' #' @param monitoringLocationIdentifier A monitoring location identifier has two parts: the agency code #' and the location number, separated by a dash (-). Location identifiers should be separated with commas, @@ -14,9 +14,9 @@ #' numbers without an agency prefix are assumed to have the prefix USGS. #' @param activityMediaName Sample media refers to the environmental medium that #' was sampled or analyzed. See available options by running -#' \code{check_USGS_sample_params("samplemedia")$activityMedia}. +#' `check_USGS_sample_params("samplemedia")$activityMedia`. #' @param siteTypeCode Site type code query parameter. See available -#' options by running \code{check_USGS_sample_params("sitetype")$typeCode}. +#' options by running `check_USGS_sample_params("sitetype")$typeCode`. #' @param boundingBox North and South are latitude values; East and West are longitude values. #' A vector of 4 (west, south, east, north) is expected. #' An example would be: c(-92.8, 44.2, -88.9, 46.0). @@ -35,40 +35,40 @@ #' records that match the date. #' @param characteristicGroup Characteristic group is a broad category describing the sample. #' See available options by running -#' \code{check_USGS_sample_params("characteristicgroup")$characteristicGroup}. +#' `check_USGS_sample_params("characteristicgroup")$characteristicGroup`. #' @param characteristicUserSupplied Observed property is the USGS term for the #' constituent sampled and the property name gives a detailed description of what #' was sampled. Observed property is mapped to characteristicUserSupplied and replaces #' the parameter name and pcode USGS #' previously used to describe discrete sample data. Find more information in the #' Observed Properties and Parameter Codes section of the Code Dictionary found here: -#' \url{https://waterdata.usgs.gov/code-dictionary/}. +#' . #' @param characteristic Characteristic is a specific category describing the sample. #' See available options by running -#' \code{check_USGS_sample_params("characteristics")$characteristicName}. +#' `check_USGS_sample_params("characteristics")$characteristicName`. #' @param stateFips State query parameter. To get a list of available state fips, -#' run \code{check_USGS_sample_params("states")}. The "fips" can be created using the function -#' \code{stateCdLookup} - for example: \code{stateCdLookup("WI", "fips")}. +#' run `check_USGS_sample_params("states")`. The "fips" can be created using the function +#' `stateCdLookup` - for example: `stateCdLookup("WI", "fips")`. #' FIPs codes for states take the format: #' CountryAbbrev:StateNumber, like US:55 for Wisconsin. #' @param countyFips County query parameter. To get a list of available counties, -#' run \code{check_USGS_sample_params("counties")}. The "Fips" can be created using the function -#' \code{countyCdLookup} - for example: \code{countyCdLookup("WI", "Dane", "fips")} +#' run `check_USGS_sample_params("counties")`. The "Fips" can be created using the function +#' `countyCdLookup` - for example: `countyCdLookup("WI", "Dane", "fips")` #' for Dane County, WI. #' FIPs codes for counties take the format: #' CountryAbbrev:StateNumber:CountyNumber, like US:55:025 for Dane County, WI. #' @param countryFips Country query parameter. Do not set redundant parameters. #' If another query parameter contains the country information, leave this parameter -#' set to the default NA. See available options by running \code{check_USGS_sample_params("countries")}, +#' set to the default NA. See available options by running `check_USGS_sample_params("countries")`, #' where the "id" field contains the value to use in the countryFips input. #' @param projectIdentifier Project identifier query parameter. This information #' would be needed from prior project information. #' @param recordIdentifierUserSupplied Record identifier, user supplied identifier. This #' information would be needed from the data supplier. #' @param siteTypeName Site type name query parameter. See available -#' options by running \code{check_param("sitetype")$typeName}. +#' options by running `check_param("sitetype")$typeName`. #' @param usgsPCode USGS parameter code. See available options by running -#' \code{check_USGS_sample_params("characteristics")$parameterCode}. +#' `check_USGS_sample_params("characteristics")$parameterCode`. #' @param pointLocationLatitude Latitude for a point/radius query (decimal degrees). Must be used #' with pointLocationLongitude and pointLocationWithinMiles. #' @param pointLocationLongitude Longitude for a point/radius query (decimal degrees). Must be used @@ -318,7 +318,7 @@ explode_query <- function(baseURL, POST = FALSE, x){ #' Check values from codeservice #' #' Call a service to check on values from: -#' \url{https://api.waterdata.usgs.gov/samples-data/codeservice/docs}. +#' . #' #' @param service Options are: "characteristicgroup", "states", "counties", #' "countries", "sitetype", "samplemedia", "characteristics", "observedproperty" @@ -379,7 +379,7 @@ check_USGS_sample_params <- function(service = "characteristicgroup", #' USGS Samples Data #' #' This function creates the call and gets the data for discrete water quality samples data -#' service described at \url{https://waterdata.usgs.gov/download-samples}. +#' service described at . #' #' @inheritParams construct_USGS_sample_request #' @param tz character to set timezone attribute of datetime. Default is UTC @@ -466,7 +466,7 @@ read_USGS_samples <- function(monitoringLocationIdentifier = NA, #' USGS Samples Summary Data #' #' This function creates the call and gets the data for discrete water quality samples summary data -#' service described at \url{https://api.waterdata.usgs.gov/samples-data/docs}. +#' service described at . #' #' @param monitoringLocationIdentifier A monitoring location identifier has two parts, #' separated by a dash (-): the agency code and the location number. Location identifiers should be separated with commas, @@ -507,4 +507,4 @@ summarize_USGS_samples <- function(monitoringLocationIdentifier){ attr(df, "queryTime") <- Sys.time() return(df) -} \ No newline at end of file +} diff --git a/R/read_USGS_ts_meta.R b/R/read_USGS_ts_meta.R index ef24aa56..c7868d17 100644 --- a/R/read_USGS_ts_meta.R +++ b/R/read_USGS_ts_meta.R @@ -1,74 +1,40 @@ #' Get USGS Timeseries metadata #' -#' Daily data and continuous measurements are grouped into timeseries, which -#' represent a collection of observations of a single parameter, potentially -#' aggregated using a standard statistic, at a single site. This endpoint provides -#' metadata about those timeseries, including their operational thresholds, units -#' of measurement, and when the earliest and most recent observations in a timeseries -#' occurred. +#' Description `r get_description("time-series-metadata")` #' #' @export -#' @param monitoring_location_id A unique identifier representing a single monitoring -#' location. This corresponds to the id field in the sites endpoint. Monitoring -#' location IDs are created by combining the agency code of the agency responsible -#' for the monitoring location (e.g. USGS) with the ID number of the monitoring -#' location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500). -#' @param properties The properties that should be included for each feature. The -#' parameter value is a comma-separated list of property names. Available values: -#' geometry, id, unit_of_measure, parameter_name, parameter_code, statistic_id, -#' last_modified, begin, end, computation_period_identifier, computation_identifier, -#' thresholds, sublocation_identifier, primary, monitoring_location_id, web_description. -#' @param parameter_code Parameter codes are 5-digit codes used to identify the -#' constituent measured and the units of measure. -#' @param parameter_name A human-understandable name corresponding to parameter_code. -#' @param unit_of_measure A human-readable description of the units of measurement -#' associated with an observation. -#' @param statistic_id A code corresponding to the statistic an observation represents. -#' Example codes include 00001 (max), 00002 (min), and 00003 (mean). -#' @param last_modified The last time a record was refreshed in our database. This -#' may happen due to regular operational processes and does not necessarily indicate -#' anything about the measurement has changed. You can query this field using -#' date-times or intervals. -#' @param begin The datetime of the earliest observation in the timeseries. -#' Together with end, this field represents the period of record of a timeseries. -#' Note that some timeseries may have large gaps in their collection record. -#' @param end The datetime of the most recent observation in the timeseries. -#' Data returned by this endpoint updates at most once per day, and potentially -#' less frequently than that, and as such there may be more recent observations -#' within a timeseries than the timeseries end value reflects. Together with begin, -#' this field represents the period of record of a timeseries. It is additionally -#' used to determine whether a timeseries is "active". -#' @param id A unique identifier representing a single timeseries. This corresponds -#' to the id field in the timeseries-metadata endpoint. -#' @param computation_period_identifier Indicates the period of data used for -#' any statistical computations. -#' @param computation_identifier Indicates whether the data from this timeseries -#' represent a specific statistical computation. -#' @param thresholds Thresholds represent known numeric limits for a timeseries, -#' for example the historic maximum value for a parameter or a level below which -#' a sensor is non-operative. These thresholds are sometimes used to automatically -#' determine if an observation is erroneous due to sensor error, and therefore -#' shouldn't be included in the timeseries. -#' @param sublocation_identifier An optional human-readable identifier used to -#' specify where measurements are recorded at a monitoring location. -#' @param primary A flag identifying if the timeseries is a "primary" timeseries. -#' "Primary" timeseries (which have this flag) are standard observations which -#' undergo Bureau review and approval processes. Non-primary timeseries, which -#' will have missing values for "primary", are provisional datasets made available -#' to meet the need for timely best science and to assist with daily operations -#' which need real-time information. Non-primary timeseries data are only retained -#' by this system for 120 days. See the USGS Provisional Data Statement for more information. -#' @param web_description A description of what this timeseries represents, as -#' used by WDFN and other USGS data dissemination products. +#' @param monitoring_location_id `r get_params("time-series-metadata")$monitoring_location_id` +#' @param parameter_code `r get_params("time-series-metadata")$parameter_code` +#' @param parameter_name `r get_params("time-series-metadata")$parameter_name` +#' @param statistic_id `r get_params("time-series-metadata")$statistic_id` +#' @param computation_identifier `r get_params("time-series-metadata")$computation_identifier` +#' @param computation_period_identifier `r get_params("time-series-metadata")$computation_period_identifier` +#' @param sublocation_identifier `r get_params("time-series-metadata")$sublocation_identifier` +#' @param last_modified `r get_params("time-series-metadata")$last_modified` +#' @param begin `r get_params("time-series-metadata")$begin` +#' @param end `r get_params("time-series-metadata")$end` +#' @param thresholds `r get_params("time-series-metadata")$thresholds` +#' @param unit_of_measure `r get_params("time-series-metadata")$unit_of_measure` +#' @param primary `r get_params("time-series-metadata")$primary` +#' @param web_description `r get_params("time-series-metadata")$web_description` +#' @param properties The properties that should be included for each feature. +#' The parameter value is a comma-separated list of property names. Available options are +#' `r schema <- check_OGC_requests(endpoint = "time-series-metadata", type = "schema"); paste(names(schema$properties), collapse = ", ")` +#' @param id `r get_params("time-series-metadata")$id` +#' @param bbox Only features that have a geometry that intersects the bounding +#' box are selected.The bounding box is provided as four or six numbers, depending +#' on whether the coordinate reference system includes a vertical axis (height or +#' depth). Coordinates are assumed to be in crs 4326. #' @param crs Indicates the coordinate reference system for the results. #' @param limit The optional limit parameter limits the number of items that are #' presented in the response document. Only items are counted that are on the #' first level of the collection in the response document. Nested objects #' contained within the explicitly requested items shall not be counted. -#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function +#' @param convertType logical, defaults to `TRUE`. If `TRUE`, the function #' will convert the data to dates and qualifier to string vector. #' @param skipGeometry This option can be used to skip response geometries for -#' each feature. +#' each feature. The returning object will be a data frame with no spatial +#' information. #' @examplesIf is_dataRetrieval_user() #' #' \donttest{ @@ -96,6 +62,7 @@ read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, "last_modified", "web_description"), limit = 10000, + bbox = NA, statistic_id = NA_character_, last_modified = NA_character_, begin = NA_character_, @@ -140,8 +107,8 @@ read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, crs = crs, web_description = web_description, skipGeometry = skipGeometry) - - return_list <- walk_pages(req_ts_meta, use_sf) + + return_list <- walk_pages(req_ts_meta) return_list <- return_list[, properties] @@ -153,4 +120,4 @@ read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, return(return_list) -} \ No newline at end of file +} diff --git a/R/renameColumns.R b/R/renameColumns.R index c0254923..69d58a52 100644 --- a/R/renameColumns.R +++ b/R/renameColumns.R @@ -18,8 +18,8 @@ #' @param p72019 the base name for parameter code 72019. #' @param \dots named arguments for the base name for any other parameter code. The #' form of the name must be like pXXXXX, where XXXXX is the parameter code. -#' @return A dataset like \code{data} with selected columns renamed. -#' @note The following statistics codes are converted by \code{renameNWISColumns}. +#' @return A dataset like `data` with selected columns renamed. +#' @note The following statistics codes are converted by `renameNWISColumns`. #' \describe{ #' \item{00000}{Instantaneous Value, suffix: Inst} #' \item{00001}{Maximum value, suffix: Max} @@ -34,7 +34,7 @@ #' \item{00023}{Tidal high-low value, suffix: HiLoTide} #' \item{00024}{Tidal low-low value, suffix: LoLoTide} #' } -#' @seealso \code{\link{readNWISdv}}, \code{\link{readNWISuv}} +#' @seealso [readNWISdv()], [readNWISuv()] #' @keywords manip IO #' @export #' @examples diff --git a/R/setAccess.R b/R/setAccess.R index d5019be1..643ac7ed 100644 --- a/R/setAccess.R +++ b/R/setAccess.R @@ -1,7 +1,7 @@ #' Set data endpoint #' #' access Indicate which dataRetrieval access code -#' you want to use options: \code{c('public','internal')} +#' you want to use options: `c('public','internal')` #' #' @param access code for data access. Options are: "public","internal","cooperator", or "USGS". #' \itemize{ diff --git a/R/walk_pages.R b/R/walk_pages.R index 51a6cdb3..b2d3de1c 100644 --- a/R/walk_pages.R +++ b/R/walk_pages.R @@ -18,23 +18,25 @@ cleanup_cols <- function(df){ df } -walk_pages <- function(req, use_sf) { +walk_pages <- function(req) { walk_pages_recursive( req = req, page = 1, - contents = list(), - use_sf + contents = list() ) } -walk_pages_recursive <- function(req, page, contents, use_sf) { +walk_pages_recursive <- function(req, page, contents) { url_method <- "GET" if(!is.null(req$body)){ url_method <- "POST" body <- req$body } + + use_sf <- !grepl("skipGeometry=true", req$url, ignore.case = TRUE) + message(url_method, ": ", req$url) returned_contents <- httr2::req_perform(req) diff --git a/R/whatNWISdata.R b/R/whatNWISdata.R index da3f8c2c..35ad061e 100644 --- a/R/whatNWISdata.R +++ b/R/whatNWISdata.R @@ -1,19 +1,19 @@ #' USGS data availability #' #' Imports a table of available parameters, period of record, and count. See -#' \url{https://waterservices.usgs.gov/docs/site-service/} +#' #' for more information. #' -#' @param \dots see \url{https://waterservices.usgs.gov/docs/site-service/} +#' @param \dots see #' for a complete list of options. A list of arguments can also be supplied. -#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function will +#' @param convertType logical, defaults to `TRUE`. If `TRUE`, the function will #' convert the data to dates, datetimes, #' numerics based on a standard algorithm. If false, everything is returned as a character #' @keywords data import USGS web service #' #' @details This function requires users to create their own arguments #' based on the NWIS web services. It is a more complicated function to use -#' compared to other NWIS functions such as \code{\link{readNWISdv}}, \code{\link{readNWISuv}}, +#' compared to other NWIS functions such as [readNWISdv()], [readNWISuv()], #' etc. However, this function adds a lot of #' flexibility to the possible queries. If the "service" argument is included, #' the results will be filtered to the proper data_type_cd. This is a great diff --git a/R/whatNWISsites.R b/R/whatNWISsites.R index 81253036..9e5544ee 100644 --- a/R/whatNWISsites.R +++ b/R/whatNWISsites.R @@ -1,10 +1,10 @@ #' Site Data Import from NWIS #' #' Returns a list of sites from the NWIS web service. This function gets the data from: -#' \url{https://waterservices.usgs.gov/docs/site-service/}. +#' . #' Mapper format is used #' -#' @param \dots see \url{https://waterservices.usgs.gov/docs/site-service/} +#' @param \dots see #' for a complete list of options. A list (or lists) can also be supplied. #' #' @return A data frame with at least the following columns: diff --git a/R/whatWQPdata.R b/R/whatWQPdata.R index 6f125f11..c4934dce 100644 --- a/R/whatWQPdata.R +++ b/R/whatWQPdata.R @@ -151,9 +151,9 @@ whatWQPmetrics <- function(..., #' Data Available from Water Quality Portal #' #' Returns a list of sites from the Water Quality Portal web service. This function gets -#' the data from: \url{https://www.waterqualitydata.us}. +#' the data from: . #' Arguments to the function should be based on -#' \url{https://www.waterqualitydata.us/webservices_documentation}. +#' . #' The information returned from whatWQPdata describes the #' available data at the WQP sites, and some metadata on the sites themselves. #' For example, a row is returned for each individual site that fulfills this @@ -161,13 +161,13 @@ whatWQPmetrics <- function(..., #' are available for the query. It does not break those results down by any finer #' grain. For example, if you ask for "Nutrients" (characteristicGroup), you will #' not learn what specific nutrients are available at that site. For that -#' kind of data discovery see \code{readWQPsummary}. +#' kind of data discovery see `readWQPsummary`. #' -#' @param \dots see \url{https://www.waterqualitydata.us/webservices_documentation} for +#' @param \dots see for #' a complete list of options. A list of arguments can also be supplied. #' One way to figure out how to construct a WQP query is to go to the "Advanced" #' form in the Water Quality Portal: -#' \url{https://www.waterqualitydata.us/#mimeType=csv&providers=NWIS&providers=STORET} +#' #' Use the form to discover what parameters are available. Once the query is #' set in the form, scroll down to the "Query URL". You will see the parameters #' after "https://www.waterqualitydata.us/#". For example, if you chose "Nutrient" @@ -175,7 +175,7 @@ whatWQPmetrics <- function(..., #' in the Query URL. The corresponding argument for dataRetrieval is #' characteristicType = "Nutrient". dataRetrieval users do not need to include #' mimeType, and providers is optional (these arguments are picked automatically). -#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the function +#' @param convertType logical, defaults to `TRUE`. If `TRUE`, the function #' will convert the data to dates, datetimes, #' numerics based on a standard algorithm. If false, everything is returned as a character. #' @keywords data import WQP web service diff --git a/R/whatWQPsites.R b/R/whatWQPsites.R index 18fbcebf..27d39ea0 100644 --- a/R/whatWQPsites.R +++ b/R/whatWQPsites.R @@ -1,18 +1,18 @@ #' Site Data Import from Water Quality Portal #' #' Returns a list of sites from the Water Quality Portal web service. This function -#' gets the data from: \url{https://www.waterqualitydata.us}. +#' gets the data from: . #' Arguments to the function should be based on -#' \url{https://www.waterqualitydata.us/webservices_documentation}. The return from +#' . The return from #' this function returns the basic metadata on WQP sites. It is -#' generally faster than the \code{\link{whatWQPdata}} function, but does +#' generally faster than the [whatWQPdata()] function, but does #' not return information on what data was collected at the site. #' -#' @param \dots see \url{https://www.waterqualitydata.us/webservices_documentation} +#' @param \dots see #' for a complete list of options. A list of arguments can also be supplied. #' One way to figure out how to construct a WQP query is to go to the "Advanced" #' form in the Water Quality Portal: -#' \url{https://www.waterqualitydata.us/#mimeType=csv&providers=NWIS&providers=STORET} +#' #' Use the form to discover what parameters are available. Once the query is #' set in the form, scroll down to the "Query URL". You will see the parameters #' after "https://www.waterqualitydata.us/#". For example, if you chose "Nutrient" @@ -22,7 +22,7 @@ #' mimeType, and providers is optional (these arguments are picked automatically). #' @param legacy Logical. If TRUE, uses legacy WQP services. Default is TRUE. #' Setting legacy = FALSE uses WQX3.0 WQP services, which are in-development, use with caution. -#' @param convertType logical, defaults to \code{TRUE}. If \code{TRUE}, the +#' @param convertType logical, defaults to `TRUE`. If `TRUE`, the #' function will convert the data to dates, datetimes, #' numerics based on a standard algorithm. If false, everything is returned as a character. #' @keywords data import WQP web service @@ -96,17 +96,17 @@ whatWQPsites <- function(..., legacy = TRUE, convertType = TRUE) { #' Summary of Data Available from Water Quality Portal #' #' Returns a list of sites with year-by-year information on what data is available. -#' The function gets the data from: \url{https://www.waterqualitydata.us}. +#' The function gets the data from: . #' Arguments to the function should be based on -#' \url{https://www.waterqualitydata.us/webservices_documentation}. +#' . #' The information returned from this function describes the #' available data at the WQP sites, and some metadata on the sites themselves. #' -#' @param \dots see \url{https://www.waterqualitydata.us/webservices_documentation} +#' @param \dots see #' for a complete list of options. A list of arguments can also be supplied. #' One way to figure out how to construct a WQP query is to go to the "Advanced" #' form in the Water Quality Portal: -#' \url{https://www.waterqualitydata.us/#mimeType=csv&providers=NWIS&providers=STORET} +#' #' Use the form to discover what parameters are available. Once the query is #' set in the form, scroll down to the "Query URL". You will see the parameters #' after "https://www.waterqualitydata.us/#". For example, if you chose "Nutrient" diff --git a/man/addWaterYear.Rd b/man/addWaterYear.Rd index 0aad52e5..789f974e 100644 --- a/man/addWaterYear.Rd +++ b/man/addWaterYear.Rd @@ -9,13 +9,13 @@ addWaterYear(rawData) \arguments{ \item{rawData}{the daily- or unit-values datset retrieved from NWISweb. Must have at least one of the following columns to add the new water year columns: -`dateTime`, `Date`, `ActivityStartDate`, or `ActivityEndDate`. The date column(s) +\code{dateTime}, \code{Date}, \code{ActivityStartDate}, or \code{ActivityEndDate}. The date column(s) can be character, POSIXct, Date. They cannot be numeric.} } \value{ data.frame with an additional integer column with "WY" appended to the -date column name. For WQP, there will be 2 columns: `ActivityStartDateWY` and -`ActivityEndDateWY`. +date column name. For WQP, there will be 2 columns: \code{ActivityStartDateWY} and +\code{ActivityEndDateWY}. } \description{ Add a column to the dataRetrieval data frame with the water year. WQP diff --git a/man/construct_USGS_sample_request.Rd b/man/construct_USGS_sample_request.Rd index 9f773a77..896f7375 100644 --- a/man/construct_USGS_sample_request.Rd +++ b/man/construct_USGS_sample_request.Rd @@ -43,7 +43,7 @@ A vector of 4 (west, south, east, north) is expected. An example would be: c(-92.8, 44.2, -88.9, 46.0).} \item{hydrologicUnit}{Hydrologic Unit Codes (HUCs) identify physical areas -within the US that drain to a certain portion of the stream network. +within the US that drain to a certain portion of the stream network. This filter accepts values containing 2, 4, 6, 8, 10 or 12 digits.} \item{activityMediaName}{Sample media refers to the environmental medium that @@ -54,7 +54,7 @@ See available options by running \code{check_USGS_sample_params("characteristicgroup")$characteristicGroup}.} \item{characteristic}{Characteristic is a specific category describing the sample. -See available options by running +See available options by running \code{check_USGS_sample_params("characteristics")$characteristicName}.} \item{characteristicUserSupplied}{Observed property is the USGS term for the @@ -75,22 +75,22 @@ than the value entered for activityStartDateLower. Can be an R Date object, or a string with format YYYY-MM-DD. The logic is inclusive, i.e. it will also return records that match the date.} -\item{countryFips}{Country query parameter. Do not set redundant parameters. +\item{countryFips}{Country query parameter. Do not set redundant parameters. If another query parameter contains the country information, leave this parameter set to the default NA. See available options by running \code{check_USGS_sample_params("countries")}, where the "id" field contains the value to use in the countryFips input.} -\item{stateFips}{State query parameter. To get a list of available state fips, +\item{stateFips}{State query parameter. To get a list of available state fips, run \code{check_USGS_sample_params("states")}. The "fips" can be created using the function -\code{stateCdLookup} - for example: \code{stateCdLookup("WI", "fips")}. -FIPs codes for states take the format: +\code{stateCdLookup} - for example: \code{stateCdLookup("WI", "fips")}. +FIPs codes for states take the format: CountryAbbrev:StateNumber, like US:55 for Wisconsin.} \item{countyFips}{County query parameter. To get a list of available counties, run \code{check_USGS_sample_params("counties")}. The "Fips" can be created using the function -\code{countyCdLookup} - for example: \code{countyCdLookup("WI", "Dane", "fips")} +\code{countyCdLookup} - for example: \code{countyCdLookup("WI", "Dane", "fips")} for Dane County, WI. -FIPs codes for counties take the format: +FIPs codes for counties take the format: CountryAbbrev:StateNumber:CountyNumber, like US:55:025 for Dane County, WI.} \item{projectIdentifier}{Project identifier query parameter. This information @@ -102,7 +102,7 @@ information would be needed from the data supplier.} \item{siteTypeName}{Site type name query parameter. See available options by running \code{check_param("sitetype")$typeName}.} -\item{usgsPCode}{USGS parameter code. See available options by running +\item{usgsPCode}{USGS parameter code. See available options by running \code{check_USGS_sample_params("characteristics")$parameterCode}.} \item{pointLocationLatitude}{Latitude for a point/radius query (decimal degrees). Must be used @@ -117,10 +117,10 @@ with pointLocationLatitude and pointLocationLongitude} \item{dataType}{Options include: "Results", "Monitoring locations", "Activities", "Projects", and "Organizations".} -\item{dataProfile}{Profile depends on type. Options for "results" dataType are: +\item{dataProfile}{Profile depends on type. Options for "results" dataType are: "fullphyschem", "basicphyschem", "fullbio", "basicbio", "narrow", -"resultdetectionquantitationlimit", "labsampleprep", "count". Options for "locations" are: -"site" and "count". Options for "activities" are "sampact", "actmetric", "actgroup", +"resultdetectionquantitationlimit", "labsampleprep", "count". Options for "locations" are: +"site" and "count". Options for "activities" are "sampact", "actmetric", "actgroup", and "count". Options for "projects" are: "project" and "projectmonitoringlocationweight". Options for "organizations" are: "organization" and "count".} diff --git a/man/construct_api_requests.Rd b/man/construct_api_requests.Rd index 0e884ddd..9f1953f8 100644 --- a/man/construct_api_requests.Rd +++ b/man/construct_api_requests.Rd @@ -36,17 +36,11 @@ construct_api_requests( ) } \arguments{ -\item{monitoring_location_id}{A unique identifier representing a single monitoring -location. This corresponds to the id field in the sites endpoint. Monitoring -location IDs are created by combining the agency code of the agency responsible -for the monitoring location (e.g. USGS) with the ID number of the monitoring -location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500).} +\item{monitoring_location_id}{A unique identifier representing a single monitoring location. This corresponds to the \code{id} field in the \code{sites} endpoint. Monitoring location IDs are created by combining the agency code of the agency responsible for the monitoring location (e.g. USGS) with the ID number of the monitoring location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500).} -\item{parameter_code}{Parameter codes are 5-digit codes used to identify the -constituent measured and the units of measure.} +\item{parameter_code}{Parameter codes are 5-digit codes used to identify the constituent measured and the units of measure. A complete list of parameter codes and associated groupings can be found at \url{https://help.waterdata.usgs.gov/codes-and-parameters/parameters}.} -\item{statistic_id}{A code corresponding to the statistic an observation represents. -Example codes include 00001 (max), 00002 (min), and 00003 (mean).} +\item{statistic_id}{A code corresponding to the statistic an observation represents. Example codes include 00001 (max), 00002 (min), and 00003 (mean). A complete list of codes and their descriptions can be found at \url{https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=\%25&fmt=html}.} \item{properties}{The properties that should be included for each feature. The parameter value is a comma-separated list of property names. Available values: @@ -58,38 +52,29 @@ box are selected.The bounding box is provided as four or six numbers, depending on whether the coordinate reference system includes a vertical axis (height or depth).} -\item{time_series_id}{A unique identifier representing a single timeseries. -This corresponds to the id field in the timeseries-metadata endpoint.} +\item{time_series_id}{A unique identifier representing a single time series. This corresponds to the \code{id} field in the \code{time-series-metadata} endpoint.} -\item{id}{A universally unique identifier (UUID) representing a single version -of a record. It is not stable over time. Every time the record is refreshed in -our database (which may happen as part of normal operations and does not imply -any change to the data itself) a new ID will be generated. To uniquely identify -a single observation over time, compare the time and timeseries_id fields; each -timeseries will only have a single observation at a given time.} +\item{id}{A universally unique identifier (UUID) representing a single version of a record. It is not stable over time. Every time the record is refreshed in our database (which may happen as part of normal operations and does not imply any change to the data itself) a new ID will be generated. To uniquely identify a single observation over time, compare the \code{time} and \code{time_series_id} fields; each time series will only have a single observation at a given \code{time}.} -\item{approval_status}{Some of the data that you have obtained from this U.S. -Geological Survey database may not have received Director's approval. Any such -data values are qualified as provisional and are subject to revision. Provisional -data are released on the condition that neither the USGS nor the United States -Government may be held liable for any damages resulting from its use. This field -reflects the approval status of each record, and is either "Approved", meaning -processing review has been completed and the data is approved for publication, -or "Provisional" and subject to revision.} +\item{approval_status}{Some of the data that you have obtained from this U.S. Geological Survey database may not have received Director's approval. Any such data values are qualified as provisional and are subject to revision. Provisional data are released on the condition that neither the USGS nor the United States Government may be held liable for any damages resulting from its use. This field reflects the approval status of each record, and is either "Approved", meaining processing review has been completed and the data is approved for publication, or "Provisional" and subject to revision. For more information about provisional data, go to \url{https://waterdata.usgs.gov/provisional-data-statement/}.} -\item{unit_of_measure}{A human-readable description of the units of measurement -associated with an observation.} +\item{unit_of_measure}{A human-readable description of the units of measurement associated with an observation.} -\item{qualifier}{This field indicates any qualifiers associated with an observation, -for instance if a sensor may have been impacted by ice or if values were estimated.} +\item{qualifier}{This field indicates any qualifiers associated with an observation, for instance if a sensor may have been impacted by ice or if values were estimated.} -\item{value}{The value of the observation. Values are transmitted as strings -in the JSON response format in order to preserve precision.} +\item{value}{The value of the observation. Values are transmitted as strings in the JSON response format in order to preserve precision.} -\item{last_modified}{The last time a record was refreshed in our database. This -may happen due to regular operational processes and does not necessarily indicate -anything about the measurement has changed. You can query this field using -date-times or intervals.} +\item{last_modified}{The last time a record was refreshed in our database. This may happen due to regular operational processes and does not necessarily indicate anything about the measurement has changed. +You can query this field using date-times or intervals, adhering to RFC 3339, or using ISO 8601 duration objects. Intervals may be bounded or half-bounded (double-dots at start or end). +Examples: +\itemize{ +\item A date-time: "2018-02-12T23:20:50Z" +\item A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" +\item Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" +\item Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours +} + +Only features that have a \code{last_modified} that intersects the value of datetime are selected. If a feature has multiple temporal properties, it is the decision of the server whether only a single temporal property is used to determine the extent or all relevant temporal properties.} \item{limit}{The optional limit parameter limits the number of items that are presented in the response document. Only items are counted that are on the @@ -98,58 +83,38 @@ contained within the explicitly requested items shall not be counted.} \item{crs}{Indicates the coordinate reference system for the results.} -\item{bbox_crs}{Indicates the coordinate reference system for the given bbox -coordinates.} - \item{skipGeometry}{This option can be used to skip response geometries for -each feature.} - -\item{datetime}{Either a date-time or an interval. Only features that have a -temporal property that intersects the value of datetime are selected. If a -feature has multiple temporal properties, it is the decision of the server -whether only a single temporal property is used to determine the extent or -all relevant temporal properties.} - -\item{begin}{The datetime of the earliest observation in the timeseries. -Together with end, this field represents the period of record of a timeseries. -Note that some timeseries may have large gaps in their collection record.} - -\item{end}{The datetime of the most recent observation in the timeseries. -Data returned by this endpoint updates at most once per day, and potentially -less frequently than that, and as such there may be more recent observations -within a timeseries than the timeseries end value reflects. Together with begin, -this field represents the period of record of a timeseries. It is additionally -used to determine whether a timeseries is "active".} - -\item{primary}{A flag identifying if the timeseries is a "primary" timeseries. -"Primary" timeseries (which have this flag) are standard observations which -undergo Bureau review and approval processes. Non-primary timeseries, which -will have missing values for "primary", are provisional datasets made available -to meet the need for timely best science and to assist with daily operations -which need real-time information. Non-primary timeseries data are only retained -by this system for 120 days. See the USGS Provisional Data Statement for more information.} - -\item{parameter_name}{A human-understandable name corresponding to parameter_code.} - -\item{thresholds}{Thresholds represent known numeric limits for a timeseries, -for example the historic maximum value for a parameter or a level below which -a sensor is non-operative. These thresholds are sometimes used to automatically -determine if an observation is erroneous due to sensor error, and therefore -shouldn't be included in the timeseries.} - -\item{sublocation_identifier}{An optional human-readable identifier used to -specify where measurements are recorded at a monitoring location.} - -\item{computation_period_identifier}{Indicates the period of data used for -any statistical computations.} - -\item{computation_identifier}{Indicates whether the data from this timeseries -represent a specific statistical computation.} - -\item{web_description}{A description of what this timeseries represents, as -used by WDFN and other USGS data dissemination products.} - -\item{time}{The date an observation represents.} +each feature. The returning object will be a data frame with no spatial +information.} + +\item{begin}{The datetime of the earliest observation in the time series. Together with \code{end}, this field represents the period of record of a time series. Note that some time series may have large gaps in their collection record. This field is currently in the local time of the monitoring location. We intend to update this \strong{in version v0} to use UTC with a time zone.} + +\item{end}{The datetime of the most recent observation in the time series. Data returned by this endpoint updates at most once per day, and potentially less frequently than that, and as such there may be more recent observations within a time series than the time series \code{end} value reflects. Together with \code{begin}, this field represents the period of record of a time series. It is additionally used to determine whether a time series is "active". We intend to update this \strong{in version v0} to use UTC with a time zone.} + +\item{primary}{A flag identifying if the time series is a "primary" time series. "Primary" time series (which have this flag) are standard observations which undergo \href{https://www.usgs.gov/survey-manual/5028-fundamental-science-practices-review-and-approval-scientific-data-release}{Bureau review and approval processes}. Non-primary time series, which will have missing values for "primary", are provisional datasets made available to meet the need for timely best science and to assist with daily operations which need real-time information. Non-primary time series data are only retained by this system for 120 days. See the \href{https://waterdata.usgs.gov/provisional-data-statement/}{USGS Provisional Data Statement} for more information.} + +\item{parameter_name}{A human-understandable name corresponding to \code{parameter_code}.} + +\item{thresholds}{Thresholds represent known numeric limits for a time series, for example the historic maximum value for a parameter or a level below which a sensor is non-operative. These thresholds are sometimes used to automatically determine if an observation is erroneous due to sensor error, and therefore shouldn't be included in the time series.} + +\item{sublocation_identifier}{An optional human-readable identifier used to specify where measurements are recorded at a monitoring location.} + +\item{computation_period_identifier}{Indicates the period of data used for any statistical computations.} + +\item{computation_identifier}{Indicates whether the data from this time series represent a specific statistical computation.} + +\item{web_description}{A description of what this time series represents, as used by WDFN and other USGS data dissemination products.} + +\item{time}{The date an observation represents. You can query this field using date-times or intervals, adhering to RFC 3339, or using ISO 8601 duration objects. Intervals may be bounded or half-bounded (double-dots at start or end). +Examples: +\itemize{ +\item A date-time: "2018-02-12T23:20:50Z" +\item A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" +\item Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" +\item Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours +} + +Only features that have a \code{time} that intersects the value of datetime are selected. If a feature has multiple temporal properties, it is the decision of the server whether only a single temporal property is used to determine the extent or all relevant temporal properties.} } \description{ Main documentation: \url{https://api.waterdata.usgs.gov/ogcapi/v0/}, diff --git a/man/findNLDI.Rd b/man/findNLDI.Rd index d1a6763d..dcc74fa7 100644 --- a/man/findNLDI.Rd +++ b/man/findNLDI.Rd @@ -48,8 +48,8 @@ be explicitly requested.} \item{distance_km}{numeric. Define how far to look along the navigation path in kilometers (default = 100)} -\item{no_sf}{if available, should `sf` be used for parsing, -defaults to `TRUE` if `sf` is locally installed} +\item{no_sf}{if available, should \code{sf} be used for parsing, +defaults to \code{TRUE} if \code{sf} is locally installed} \item{warn}{(default TRUE) should warnings be printed} } diff --git a/man/importWQP.Rd b/man/importWQP.Rd index 352aab77..54c7b2ab 100644 --- a/man/importWQP.Rd +++ b/man/importWQP.Rd @@ -51,5 +51,5 @@ STORETdata_char <- importWQP(STORETex, convertType = FALSE) \dontshow{\}) # examplesIf} } \seealso{ -\code{\link{readWQPdata}}, \code{\link{readWQPqw}}, \code{\link{whatWQPsites}} +\code{\link[=readWQPdata]{readWQPdata()}}, \code{\link[=readWQPqw]{readWQPqw()}}, \code{\link[=whatWQPsites]{whatWQPsites()}} } diff --git a/man/importWaterML1.Rd b/man/importWaterML1.Rd index 79ed544e..3f0c5f75 100644 --- a/man/importWaterML1.Rd +++ b/man/importWaterML1.Rd @@ -25,7 +25,7 @@ A data frame with the following columns: Name \tab Type \tab Description \cr agency_cd \tab character \tab The NWIS code for the agency reporting the data\cr site_no \tab character \tab The USGS site number \cr - \tab POSIXct \tab The date and time of the value converted to UTC (if asDateTime = TRUE), \cr +\tab POSIXct \tab The date and time of the value converted to UTC (if asDateTime = TRUE), \cr \tab character \tab or raw character string (if asDateTime = FALSE) \cr tz_cd \tab character \tab The time zone code for \cr code \tab character \tab Any codes that qualify the corresponding value\cr @@ -107,5 +107,5 @@ importFile <- importWaterML1(fullPath, TRUE) \dontshow{\}) # examplesIf} } \seealso{ -\code{\link{renameNWISColumns}} +\code{\link[=renameNWISColumns]{renameNWISColumns()}} } diff --git a/man/pcode_to_name.Rd b/man/pcode_to_name.Rd index 963802ee..dd3e1a4e 100644 --- a/man/pcode_to_name.Rd +++ b/man/pcode_to_name.Rd @@ -7,12 +7,12 @@ pcode_to_name(parameterCd = "all") } \arguments{ -\item{parameterCd}{character that contains the code for a character vector +\item{parameterCd}{character that contains the code for a character vector of 5-digit parameter codes. Default is "all" which will return a complete list of parameter codes that have been mapped to a characteristic name.} } \value{ -a data frame with columns "parm_cd", "description", +a data frame with columns "parm_cd", "description", "characteristicname", "measureunitcode", "resultsamplefraction", "resulttemperaturebasis", "resultstatisticalbasis", "resulttimebasis", "resultweightbasis", "resultparticlesizebasis", "last_rev_dt" diff --git a/man/readNWISdata.Rd b/man/readNWISdata.Rd index d1ea37e7..c1fb917a 100644 --- a/man/readNWISdata.Rd +++ b/man/readNWISdata.Rd @@ -11,7 +11,7 @@ readNWISdata(..., asDateTime = TRUE, convertType = TRUE, tz = "UTC") a complete list of options. A list of arguments can also be supplied. One important argument to include is "service". Possible values are "iv" -(for instantaneous), +(for instantaneous), "dv" (for daily values), "gwlevels" (for groundwater levels), "site" (for site service), "measurement", and "stat" (for statistics service). Note: "measurement" calls go to: @@ -39,10 +39,10 @@ Name \tab Type \tab Description \cr agency \tab character \tab The NWIS code for the agency reporting the data\cr site \tab character \tab The USGS site number \cr dateTime \tab POSIXct \tab The date and time (if applicable) of the measurement, - converted to UTC for unit value data. R only allows one time zone attribute per column. For unit data - spanning a time zone change, converting the data to UTC solves this problem. For daily data, - the time zone attribute is the time zone of the first returned measurement. - \cr +converted to UTC for unit value data. R only allows one time zone attribute per column. For unit data +spanning a time zone change, converting the data to UTC solves this problem. For daily data, +the time zone attribute is the time zone of the first returned measurement. +\cr tz_cd \tab character \tab The time zone code for dateTime column\cr code \tab character \tab Any codes that qualify the corresponding value\cr value \tab numeric \tab The numeric value for the parameter \cr @@ -71,17 +71,17 @@ See examples below for ideas of constructing queries. \details{ This function requires users to create their own arguments based on the NWIS web services. It is a more complicated function to use -compared to other NWIS functions such as \code{\link{readNWISdv}}, \code{\link{readNWISuv}}, -\code{\link{readNWISgwl}}, etc. However, this function adds a lot of -flexibility to the possible queries. This function will also behave exactly +compared to other NWIS functions such as \code{\link[=readNWISdv]{readNWISdv()}}, \code{\link[=readNWISuv]{readNWISuv()}}, +\code{\link[=readNWISgwl]{readNWISgwl()}}, etc. However, this function adds a lot of +flexibility to the possible queries. This function will also behave exactly as NWIS when it comes to date queries. NWIS by default will only return the latest value for the daily and instantaneous services. So if you do not provide a starting date, you will only get back the latest value. If you want the full period of record, you can use "startDate = '1900-01-01'". Other options for dates are periods, such as "period = 'P7D'" which translates to a period of 7 days. For period, use only a positive ISO-8601 duration format, which should -not be expressed in periods of less than a day, or in increments of months M or years Y. -period returns data for a site generally from now to a time in the past. +not be expressed in periods of less than a day, or in increments of months M or years Y. +period returns data for a site generally from now to a time in the past. Note that when period is used all data up to the most recent value are returned. } \examples{ @@ -212,5 +212,5 @@ peak_data <- readNWISdata( \dontshow{\}) # examplesIf} } \seealso{ -\code{\link{renameNWISColumns}}, \code{\link{importWaterML1}}, \code{\link{importRDB1}} +\code{\link[=renameNWISColumns]{renameNWISColumns()}}, \code{\link[=importWaterML1]{importWaterML1()}}, \code{\link[=importRDB1]{importRDB1()}} } diff --git a/man/readNWISdv.Rd b/man/readNWISdv.Rd index 8c2f9d80..d4c34204 100644 --- a/man/readNWISdv.Rd +++ b/man/readNWISdv.Rd @@ -54,16 +54,16 @@ queryTime \tab POSIXct \tab The time the data was returned \cr } } \description{ -Imports data from NWIS daily web service. This function gets the data from here: +Imports data from NWIS daily web service. This function gets the data from here: \url{https://waterservices.usgs.gov/docs/dv-service/daily-values-service-details/} Inputs to this function are just USGS site ids, USGS parameter codes, -USGS statistic codes, and start and end date. For a more complex query, use \code{\link{readNWISdata}}, +USGS statistic codes, and start and end date. For a more complex query, use \code{\link[=readNWISdata]{readNWISdata()}}, with an argument service = "dv". Data coming the daily web services are aggregates of the instantaneous (sensor) web services. Not all statistical codes are available for all data. -Use the function \code{\link{whatNWISdata}} to discover what data +Use the function \code{\link[=whatNWISdata]{whatNWISdata()}} to discover what data is available for a USGS site. The column data_type_cd with the values "dv" -returned from \code{\link{whatNWISdata}}) are available from this service. +returned from \code{\link[=whatNWISdata]{whatNWISdata()}}) are available from this service. } \details{ More information on the web service can be found here: @@ -99,7 +99,7 @@ notActive <- readNWISdv(site, "00060", "2014-01-01", "2014-01-07") \dontshow{\}) # examplesIf} } \seealso{ -\code{\link{renameNWISColumns}}, \code{\link{importWaterML1}} +\code{\link[=renameNWISColumns]{renameNWISColumns()}}, \code{\link[=importWaterML1]{importWaterML1()}} } \keyword{USGS} \keyword{data} diff --git a/man/readNWISgwl.Rd b/man/readNWISgwl.Rd index c354b76c..00d19b10 100644 --- a/man/readNWISgwl.Rd +++ b/man/readNWISgwl.Rd @@ -64,19 +64,18 @@ siteInfo \tab data.frame \tab A data frame containing information on the request Imports groundwater level data from NWIS web service. This function gets the data from here: \url{https://waterservices.usgs.gov/docs/groundwater-levels/groundwater-levels-details/} Inputs to this function are just USGS site ids, USGS parameter codes, -and start and end date. For a more complex query, use \code{\link{readNWISdata}}, +and start and end date. For a more complex query, use \code{\link[=readNWISdata]{readNWISdata()}}, including an argument service="gwlevels". Not all parameter codes are available for all data. -Use the function \code{\link{whatNWISdata}} to discover what data +Use the function \code{\link[=whatNWISdata]{whatNWISdata()}} to discover what data is available for a USGS site. The column data_type_cd with the values "gw" -returned from \code{\link{whatNWISdata}}) are available from this service. +returned from \code{\link[=whatNWISdata]{whatNWISdata()}}) are available from this service. } \details{ More information on the web service can be found here: \url{https://waterservices.usgs.gov/test-tools}, choosing the "Groundwater Levels Value Service". - Mixed date/times come back from the service depending on the year that the data was collected. See \url{https://waterdata.usgs.gov/usa/nwis/gw} for details about groundwater. By default the returned dates are converted to date objects, unless convertType @@ -99,5 +98,5 @@ data5 <- readNWISgwl("263819081585801", parameterCd = "72019") \dontshow{\}) # examplesIf} } \seealso{ -\code{\link{constructNWISURL}}, \code{\link{importRDB1}} +\code{\link[=constructNWISURL]{constructNWISURL()}}, \code{\link[=importRDB1]{importRDB1()}} } diff --git a/man/readNWISmeas.Rd b/man/readNWISmeas.Rd index 01872efb..ba99378c 100644 --- a/man/readNWISmeas.Rd +++ b/man/readNWISmeas.Rd @@ -79,5 +79,5 @@ Meas07227500.exRaw <- readNWISmeas("07227500", expanded = TRUE, convertType = FA \dontshow{\}) # examplesIf} } \seealso{ -\code{\link{constructNWISURL}}, \code{\link{importRDB1}} +\code{\link[=constructNWISURL]{constructNWISURL()}}, \code{\link[=importRDB1]{importRDB1()}} } diff --git a/man/readNWISpCode.Rd b/man/readNWISpCode.Rd index 8e608816..8b5887cb 100644 --- a/man/readNWISpCode.Rd +++ b/man/readNWISpCode.Rd @@ -14,13 +14,13 @@ complete list of all current parameter codes in the USGS, use "all" as the input \value{ parameterData data frame with the following information: \tabular{lll}{ - Name \tab Type \tab Description\cr - parameter_cd \tab character \tab 5-digit USGS parameter code \cr - parameter_group_nm \tab character \tab USGS parameter group name\cr - parameter_nm \tab character \tab USGS parameter name\cr - casrn \tab character \tab Chemical Abstracts Service (CAS) Registry Number\cr - srsname \tab character \tab Substance Registry Services Name\cr - parameter_units \tab character \tab Parameter units\cr +Name \tab Type \tab Description\cr +parameter_cd \tab character \tab 5-digit USGS parameter code \cr +parameter_group_nm \tab character \tab USGS parameter group name\cr +parameter_nm \tab character \tab USGS parameter name\cr +casrn \tab character \tab Chemical Abstracts Service (CAS) Registry Number\cr +srsname \tab character \tab Substance Registry Services Name\cr +parameter_units \tab character \tab Parameter units\cr } } \description{ @@ -39,7 +39,7 @@ one_extra <- readNWISpCode(c("01075", "12345")) } } \seealso{ -\code{\link{importRDB1}} +\code{\link[=importRDB1]{importRDB1()}} } \keyword{USGS} \keyword{data} diff --git a/man/readNWISpeak.Rd b/man/readNWISpeak.Rd index 63ae13d5..d875785c 100644 --- a/man/readNWISpeak.Rd +++ b/man/readNWISpeak.Rd @@ -71,7 +71,7 @@ In some cases, the specific date of the peak data is not know. This function will default to converting complete dates to a "Date" object, and converting incomplete dates to "NA". If those incomplete dates are -needed, set the `asDateTime` argument to FALSE. No dates will be converted to +needed, set the \code{asDateTime} argument to FALSE. No dates will be converted to R Date objects. } \examples{ @@ -86,5 +86,5 @@ peakdata <- readNWISpeak(stations, convertType = FALSE) \dontshow{\}) # examplesIf} } \seealso{ -\code{\link{constructNWISURL}}, \code{\link{importRDB1}} +\code{\link[=constructNWISURL]{constructNWISURL()}}, \code{\link[=importRDB1]{importRDB1()}} } diff --git a/man/readNWISrating.Rd b/man/readNWISrating.Rd index 60541ad7..9166cb2f 100644 --- a/man/readNWISrating.Rd +++ b/man/readNWISrating.Rd @@ -55,5 +55,5 @@ attr(data, "RATING") \dontshow{\}) # examplesIf} } \seealso{ -\code{\link{constructNWISURL}}, \code{\link{importRDB1}} +\code{\link[=constructNWISURL]{constructNWISURL()}}, \code{\link[=importRDB1]{importRDB1()}} } diff --git a/man/readNWISsite.Rd b/man/readNWISsite.Rd index 11696f47..4393ce28 100644 --- a/man/readNWISsite.Rd +++ b/man/readNWISsite.Rd @@ -15,46 +15,46 @@ A data frame with at least the following columns: Name \tab Type \tab Description \cr agency_cd \tab character \tab The NWIS code for the agency reporting the data\cr site_no \tab character \tab The USGS site number \cr - station_nm \tab character \tab Site name \cr - site_tp_cd \tab character \tab Site type \cr - lat_va \tab numeric \tab DMS latitude \cr - long_va \tab numeric \tab DMS longitude \cr - dec_lat_va \tab numeric \tab Decimal latitude \cr - dec_long_va \tab numeric \tab Decimal longitude \cr - coord_meth_cd \tab character \tab Latitude-longitude method \cr - coord_acy_cd \tab character \tab Latitude-longitude accuracy \cr - coord_datum_cd \tab character \tab Latitude-longitude datum \cr - dec_coord_datum_cd \tab character \tab Decimal Latitude-longitude datum \cr - district_cd \tab character \tab District code \cr - state_cd \tab character \tab State code \cr - county_cd \tab character \tab County code \cr - country_cd \tab character \tab Country code \cr - land_net_ds \tab character \tab Land net location description \cr - map_nm \tab character \tab Name of location map \cr - map_scale_fc \tab character \tab Scale of location map \cr - alt_va \tab numeric \tab Altitude of Gage/land surface \cr - alt_meth_cd \tab character \tab Method altitude determined \cr - alt_acy_va \tab numeric \tab Altitude accuracy \cr - alt_datum_cd \tab character \tab Altitude datum \cr - huc_cd \tab character \tab Hydrologic unit code \cr - basin_cd \tab character \tab Drainage basin code \cr - topo_cd \tab character \tab Topographic setting code \cr - instruments_cd \tab character \tab Flags for instruments at site \cr - construction_dt \tab character \tab Date of first construction \cr - inventory_dt \tab character \tab Date site established or inventoried \cr - drain_area_va \tab numeric \tab Drainage area \cr - contrib_drain_area_va \tab numeric \tab Contributing drainage area \cr - tz_cd \tab character \tab Time Zone abbreviation \cr - local_time_fg \tab character \tab Site honors Daylight Savings Time \cr - reliability_cd \tab character \tab Data reliability code \cr - gw_file_cd \tab character \tab Data-other GW files \cr - nat_aqfr_cd \tab character \tab National aquifer code \cr - aqfr_cd \tab character \tab Local aquifer code \cr - aqfr_type_cd \tab character \tab Local aquifer type code \cr - well_depth_va \tab numeric \tab Well depth \cr - hole_depth_va \tab numeric \tab Hole depth \cr - depth_src_cd \tab character \tab Source of depth data \cr - project_no \tab character \tab Project number \cr +station_nm \tab character \tab Site name \cr +site_tp_cd \tab character \tab Site type \cr +lat_va \tab numeric \tab DMS latitude \cr +long_va \tab numeric \tab DMS longitude \cr +dec_lat_va \tab numeric \tab Decimal latitude \cr +dec_long_va \tab numeric \tab Decimal longitude \cr +coord_meth_cd \tab character \tab Latitude-longitude method \cr +coord_acy_cd \tab character \tab Latitude-longitude accuracy \cr +coord_datum_cd \tab character \tab Latitude-longitude datum \cr +dec_coord_datum_cd \tab character \tab Decimal Latitude-longitude datum \cr +district_cd \tab character \tab District code \cr +state_cd \tab character \tab State code \cr +county_cd \tab character \tab County code \cr +country_cd \tab character \tab Country code \cr +land_net_ds \tab character \tab Land net location description \cr +map_nm \tab character \tab Name of location map \cr +map_scale_fc \tab character \tab Scale of location map \cr +alt_va \tab numeric \tab Altitude of Gage/land surface \cr +alt_meth_cd \tab character \tab Method altitude determined \cr +alt_acy_va \tab numeric \tab Altitude accuracy \cr +alt_datum_cd \tab character \tab Altitude datum \cr +huc_cd \tab character \tab Hydrologic unit code \cr +basin_cd \tab character \tab Drainage basin code \cr +topo_cd \tab character \tab Topographic setting code \cr +instruments_cd \tab character \tab Flags for instruments at site \cr +construction_dt \tab character \tab Date of first construction \cr +inventory_dt \tab character \tab Date site established or inventoried \cr +drain_area_va \tab numeric \tab Drainage area \cr +contrib_drain_area_va \tab numeric \tab Contributing drainage area \cr +tz_cd \tab character \tab Time Zone abbreviation \cr +local_time_fg \tab character \tab Site honors Daylight Savings Time \cr +reliability_cd \tab character \tab Data reliability code \cr +gw_file_cd \tab character \tab Data-other GW files \cr +nat_aqfr_cd \tab character \tab National aquifer code \cr +aqfr_cd \tab character \tab Local aquifer code \cr +aqfr_type_cd \tab character \tab Local aquifer type code \cr +well_depth_va \tab numeric \tab Well depth \cr +hole_depth_va \tab numeric \tab Hole depth \cr +depth_src_cd \tab character \tab Source of depth data \cr +project_no \tab character \tab Project number \cr } There are also several useful attributes attached to the data frame: diff --git a/man/readNWISstat.Rd b/man/readNWISstat.Rd index 860e8e76..b747a558 100644 --- a/man/readNWISstat.Rd +++ b/man/readNWISstat.Rd @@ -91,5 +91,5 @@ x <- readNWISstat( \dontshow{\}) # examplesIf} } \seealso{ -\code{\link{constructNWISURL}}, \code{\link{importRDB1}} +\code{\link[=constructNWISURL]{constructNWISURL()}}, \code{\link[=importRDB1]{importRDB1()}} } diff --git a/man/readNWISuv.Rd b/man/readNWISuv.Rd index 20e0b9fa..6a83daac 100644 --- a/man/readNWISuv.Rd +++ b/man/readNWISuv.Rd @@ -57,12 +57,12 @@ queryTime \tab POSIXct \tab The time the data was returned \cr Imports data from NWIS web service. This function gets the data from here: \url{https://waterservices.usgs.gov/docs/instantaneous-values/instantaneous-values-details/} Inputs to this function are just USGS site ids, USGS parameter codes, -and start and end date. For a more complex query, use \code{\link{readNWISdata}}, +and start and end date. For a more complex query, use \code{\link[=readNWISdata]{readNWISdata()}}, including an arguement service="uv". Not all parameter codes are available for all data. -Use the function \code{\link{whatNWISdata}} to discover what data +Use the function \code{\link[=whatNWISdata]{whatNWISdata()}} to discover what data is available for a USGS site. The column data_type_cd with the values "uv" -returned from \code{\link{whatNWISdata}}) are available from this service. +returned from \code{\link[=whatNWISdata]{whatNWISdata()}}) are available from this service. } \details{ More information on the web service can be found here: @@ -100,7 +100,7 @@ GMTdata <- readNWISuv( \dontshow{\}) # examplesIf} } \seealso{ -\code{\link{renameNWISColumns}}, \code{\link{importWaterML1}} +\code{\link[=renameNWISColumns]{renameNWISColumns()}}, \code{\link[=importWaterML1]{importWaterML1()}} } \keyword{USGS} \keyword{data} diff --git a/man/readWQPdata.Rd b/man/readWQPdata.Rd index e58c0d83..e740e9cc 100644 --- a/man/readWQPdata.Rd +++ b/man/readWQPdata.Rd @@ -15,9 +15,9 @@ readWQPdata( } \arguments{ \item{\dots}{see \url{https://www.waterqualitydata.us/webservices_documentation} for a complete list of options. -A list of arguments can also be supplied. For more information see the above -description for this help file. One way to figure out how to construct a WQP query is to go to the "Advanced" -form in the Water Quality Portal. Use the form to discover what parameters are available. Once the query is +A list of arguments can also be supplied. For more information see the above +description for this help file. One way to figure out how to construct a WQP query is to go to the "Advanced" +form in the Water Quality Portal. Use the form to discover what parameters are available. Once the query is set in the form, scroll down to the "Query URL". You will see the parameters after "https://www.waterqualitydata.us/#". For example, if you chose "Nutrient" in the Characteristic Group dropdown, you will see characteristicType=Nutrient @@ -28,7 +28,7 @@ mimeType, and providers is optional (these arguments are picked automatically).} \item{service}{character. See Details for more information.} \item{querySummary}{logical to only return the number of records and unique sites that -will be returned from this query. Choosing TRUE is deprecated, readWQPsummary +will be returned from this query. Choosing TRUE is deprecated, readWQPsummary is recommended instead.} \item{tz}{character to set timezone attribute of dateTime. Default is "UTC", and converts the @@ -66,15 +66,12 @@ Imports data from Water Quality Portal web service. This function gets the data \url{https://www.waterqualitydata.us}. } \details{ -This function uses \dots as a query input, which can be very flexible, but also +This function uses \dots as a query input, which can be very flexible, but also has a steeper learning curve. For a quick overview, scroll down to the Examples in this help file to see many query options. - - There are currently 10 legacy options for data provided by the Water Quality Portal: - Legacy: \tabular{lll}{ @@ -93,9 +90,9 @@ Organization Data \tab Organization \tab /data/Organization/search \cr There are 4 WQX3 options. These are still in-development, and should be used with caution. - + \tabular{llll}{ -WQP Radio Button \tab service argument \tab Base URL \tab dataProfile \cr +WQP Radio Button \tab service argument \tab Base URL \tab dataProfile \cr Monitoring Locations \tab StationWQX3 \tab /wqx3/Station/search \tab \cr Full Physical Chemical \tab ResultWQX3 \tab /wqx3/Result/search \tab fullPhysChem \cr Narrow \tab ResultWQX3 \tab /wqx3/Result/search \tab narrow \cr diff --git a/man/readWQPqw.Rd b/man/readWQPqw.Rd index 089b9781..3148bc96 100644 --- a/man/readWQPqw.Rd +++ b/man/readWQPqw.Rd @@ -103,8 +103,8 @@ DO <- readWQPqw(siteNumbers = "USGS-05288705", \dontshow{\}) # examplesIf} } \seealso{ -\code{\link{readWQPdata}}, \code{\link{whatWQPsites}}, -and \code{\link{importWQP}} +\code{\link[=readWQPdata]{readWQPdata()}}, \code{\link[=whatWQPsites]{whatWQPsites()}}, +and \code{\link[=importWQP]{importWQP()}} } \keyword{USGS} \keyword{data} diff --git a/man/readWQPsummary.Rd b/man/readWQPsummary.Rd index 08f28856..8c4b9e1d 100644 --- a/man/readWQPsummary.Rd +++ b/man/readWQPsummary.Rd @@ -8,11 +8,11 @@ readWQPsummary(...) } \arguments{ \item{\dots}{see \url{https://www.waterqualitydata.us/webservices_documentation} - for a complete list of options. A list of arguments can also be supplied. -One way to figure out how to construct a WQP query is to go to the "Advanced" +for a complete list of options. A list of arguments can also be supplied. +One way to figure out how to construct a WQP query is to go to the "Advanced" form in the Water Quality Portal: \url{https://www.waterqualitydata.us/#mimeType=csv&providers=NWIS&providers=STORET} -Use the form to discover what parameters are available. Once the query is +Use the form to discover what parameters are available. Once the query is set in the form, scroll down to the "Query URL". You will see the parameters after "https://www.waterqualitydata.us/#". For example, if you chose "Nutrient" in the Characteristic Group dropdown, you will see characteristicType=Nutrient diff --git a/man/read_USGS_dv.Rd b/man/read_USGS_dv.Rd index c464de73..0860bd51 100644 --- a/man/read_USGS_dv.Rd +++ b/man/read_USGS_dv.Rd @@ -20,67 +20,50 @@ read_USGS_dv( last_modified = NA_character_, limit = 10000, crs = NA_character_, - bbox_crs = NA_character_, skipGeometry = NA, datetime = NA_character_, convertType = TRUE ) } \arguments{ -\item{monitoring_location_id}{A unique identifier representing a single monitoring -location. This corresponds to the id field in the sites endpoint. Monitoring -location IDs are created by combining the agency code of the agency responsible -for the monitoring location (e.g. USGS) with the ID number of the monitoring -location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500).} +\item{monitoring_location_id}{A unique identifier representing a single monitoring location. This corresponds to the \code{id} field in the \code{sites} endpoint. Monitoring location IDs are created by combining the agency code of the agency responsible for the monitoring location (e.g. USGS) with the ID number of the monitoring location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500).} -\item{parameter_code}{Parameter codes are 5-digit codes used to identify the -constituent measured and the units of measure.} +\item{parameter_code}{Parameter codes are 5-digit codes used to identify the constituent measured and the units of measure. A complete list of parameter codes and associated groupings can be found at \url{https://help.waterdata.usgs.gov/codes-and-parameters/parameters}.} -\item{statistic_id}{A code corresponding to the statistic an observation represents. -Example codes include 00001 (max), 00002 (min), and 00003 (mean).} +\item{statistic_id}{A code corresponding to the statistic an observation represents. Example codes include 00001 (max), 00002 (min), and 00003 (mean). A complete list of codes and their descriptions can be found at \url{https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=\%25&fmt=html}.} -\item{properties}{The properties that should be included for each feature. The -parameter value is a comma-separated list of property names. Available values: -id, timeseries_id, monitoring_location_id, parameter_code, statistic_id, time, -value, unit_of_measure, approval_status, qualifier, last_modified.} +\item{properties}{The properties that should be included for each feature. +The parameter value is a comma-separated list of property names. Available options are +geometry, id, time_series_id, monitoring_location_id, parameter_code, statistic_id, time, value, unit_of_measure, approval_status, qualifier, last_modified} \item{bbox}{Only features that have a geometry that intersects the bounding box are selected.The bounding box is provided as four or six numbers, depending on whether the coordinate reference system includes a vertical axis (height or -depth).} +depth). Coordinates are assumed to be in crs 4326.} -\item{time_series_id}{A unique identifier representing a single timeseries. -This corresponds to the id field in the timeseries-metadata endpoint.} +\item{time_series_id}{A unique identifier representing a single time series. This corresponds to the \code{id} field in the \code{time-series-metadata} endpoint.} -\item{id}{A universally unique identifier (UUID) representing a single version -of a record. It is not stable over time. Every time the record is refreshed in -our database (which may happen as part of normal operations and does not imply -any change to the data itself) a new ID will be generated. To uniquely identify -a single observation over time, compare the time and timeseries_id fields; each -timeseries will only have a single observation at a given time.} +\item{id}{A universally unique identifier (UUID) representing a single version of a record. It is not stable over time. Every time the record is refreshed in our database (which may happen as part of normal operations and does not imply any change to the data itself) a new ID will be generated. To uniquely identify a single observation over time, compare the \code{time} and \code{time_series_id} fields; each time series will only have a single observation at a given \code{time}.} -\item{approval_status}{Some of the data that you have obtained from this U.S. -Geological Survey database may not have received Director's approval. Any such -data values are qualified as provisional and are subject to revision. Provisional -data are released on the condition that neither the USGS nor the United States -Government may be held liable for any damages resulting from its use. This field -reflects the approval status of each record, and is either "Approved", meaning -processing review has been completed and the data is approved for publication, -or "Provisional" and subject to revision.} +\item{approval_status}{Some of the data that you have obtained from this U.S. Geological Survey database may not have received Director's approval. Any such data values are qualified as provisional and are subject to revision. Provisional data are released on the condition that neither the USGS nor the United States Government may be held liable for any damages resulting from its use. This field reflects the approval status of each record, and is either "Approved", meaining processing review has been completed and the data is approved for publication, or "Provisional" and subject to revision. For more information about provisional data, go to \url{https://waterdata.usgs.gov/provisional-data-statement/}.} -\item{unit_of_measure}{A human-readable description of the units of measurement -associated with an observation.} +\item{unit_of_measure}{A human-readable description of the units of measurement associated with an observation.} -\item{qualifier}{This field indicates any qualifiers associated with an observation, -for instance if a sensor may have been impacted by ice or if values were estimated.} +\item{qualifier}{This field indicates any qualifiers associated with an observation, for instance if a sensor may have been impacted by ice or if values were estimated.} -\item{value}{The value of the observation. Values are transmitted as strings -in the JSON response format in order to preserve precision.} +\item{value}{The value of the observation. Values are transmitted as strings in the JSON response format in order to preserve precision.} -\item{last_modified}{The last time a record was refreshed in our database. This -may happen due to regular operational processes and does not necessarily indicate -anything about the measurement has changed. You can query this field using -date-times or intervals.} +\item{last_modified}{The last time a record was refreshed in our database. This may happen due to regular operational processes and does not necessarily indicate anything about the measurement has changed. +You can query this field using date-times or intervals, adhering to RFC 3339, or using ISO 8601 duration objects. Intervals may be bounded or half-bounded (double-dots at start or end). +Examples: +\itemize{ +\item A date-time: "2018-02-12T23:20:50Z" +\item A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" +\item Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" +\item Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours +} + +Only features that have a \code{last_modified} that intersects the value of datetime are selected. If a feature has multiple temporal properties, it is the decision of the server whether only a single temporal property is used to determine the extent or all relevant temporal properties.} \item{limit}{The optional limit parameter limits the number of items that are presented in the response document. Only items are counted that are on the @@ -89,34 +72,28 @@ contained within the explicitly requested items shall not be counted.} \item{crs}{Indicates the coordinate reference system for the results.} -\item{bbox_crs}{Indicates the coordinate reference system for the given bbox -coordinates.} - \item{skipGeometry}{This option can be used to skip response geometries for -each feature.} +each feature. The returning object will be a data frame with no spatial +information.} -\item{datetime}{Either a date-time or an interval. Only features that have a -temporal property that intersects the value of datetime are selected. If a -feature has multiple temporal properties, it is the decision of the server -whether only a single temporal property is used to determine the extent or -all relevant temporal properties.} +\item{datetime}{} \item{convertType}{logical, defaults to \code{TRUE}. If \code{TRUE}, the function will convert the data to dates and qualifier to string vector.} -\item{time}{The date an observation represents.} +\item{time}{The date an observation represents. You can query this field using date-times or intervals, adhering to RFC 3339, or using ISO 8601 duration objects. Intervals may be bounded or half-bounded (double-dots at start or end). +Examples: +\itemize{ +\item A date-time: "2018-02-12T23:20:50Z" +\item A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" +\item Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" +\item Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours +} + +Only features that have a \code{time} that intersects the value of datetime are selected. If a feature has multiple temporal properties, it is the decision of the server whether only a single temporal property is used to determine the extent or all relevant temporal properties.} } \description{ -Daily data provide one data value to represent water conditions for the day. -Throughout much of the history of the USGS, the primary water data available -was daily data collected manually at the monitoring location once each day. -With improved availability of computer storage and automated transmission of -data, the daily data published today are generally a statistical summary or -metric of the continuous data collected each day, such as the daily mean, -minimum, or maximum value. Daily data are automatically calculated from the -continuous data of the same parameter code and are described by parameter code -and a statistic code. These data have also been referred to as "daily values" -or "DV". +Description Daily data provide one data value to represent water conditions for the day. Throughout much of the history of the USGS, the primary water data available was daily data collected manually at the monitoring location once each day. With improved availability of computer storage and automated transmission of data, the daily data published today are generally a statistical summary or metric of the continuous data collected each day, such as the daily mean, minimum, or maximum value. Daily data are automatically calculated from the continuous data of the same parameter code and are described by parameter code and a statistic code. These data have also been referred to as “daily values” or “DV”. } \examples{ \dontshow{if (is_dataRetrieval_user()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} @@ -128,7 +105,7 @@ dv_data_sf <- read_USGS_dv(monitoring_location_id = site, parameter_code = "00060", datetime = c("2021-01-01", "2022-01-01")) -dv_data_sf <- read_USGS_dv(monitoring_location_id = site, +dv_data_trim <- read_USGS_dv(monitoring_location_id = site, parameter_code = "00060", properties = c("monitoring_location_id", "value", @@ -137,7 +114,7 @@ dv_data_sf <- read_USGS_dv(monitoring_location_id = site, dv_data <- read_USGS_dv(monitoring_location_id = site, parameter_code = "00060", - no_sf = TRUE) + skipGeometry = TRUE) multi_site <- read_USGS_dv(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), diff --git a/man/read_USGS_samples.Rd b/man/read_USGS_samples.Rd index 975f66a5..2695a24b 100644 --- a/man/read_USGS_samples.Rd +++ b/man/read_USGS_samples.Rd @@ -44,7 +44,7 @@ A vector of 4 (west, south, east, north) is expected. An example would be: c(-92.8, 44.2, -88.9, 46.0).} \item{hydrologicUnit}{Hydrologic Unit Codes (HUCs) identify physical areas -within the US that drain to a certain portion of the stream network. +within the US that drain to a certain portion of the stream network. This filter accepts values containing 2, 4, 6, 8, 10 or 12 digits.} \item{activityMediaName}{Sample media refers to the environmental medium that @@ -55,7 +55,7 @@ See available options by running \code{check_USGS_sample_params("characteristicgroup")$characteristicGroup}.} \item{characteristic}{Characteristic is a specific category describing the sample. -See available options by running +See available options by running \code{check_USGS_sample_params("characteristics")$characteristicName}.} \item{characteristicUserSupplied}{Observed property is the USGS term for the @@ -76,22 +76,22 @@ than the value entered for activityStartDateLower. Can be an R Date object, or a string with format YYYY-MM-DD. The logic is inclusive, i.e. it will also return records that match the date.} -\item{countryFips}{Country query parameter. Do not set redundant parameters. +\item{countryFips}{Country query parameter. Do not set redundant parameters. If another query parameter contains the country information, leave this parameter set to the default NA. See available options by running \code{check_USGS_sample_params("countries")}, where the "id" field contains the value to use in the countryFips input.} -\item{stateFips}{State query parameter. To get a list of available state fips, +\item{stateFips}{State query parameter. To get a list of available state fips, run \code{check_USGS_sample_params("states")}. The "fips" can be created using the function -\code{stateCdLookup} - for example: \code{stateCdLookup("WI", "fips")}. -FIPs codes for states take the format: +\code{stateCdLookup} - for example: \code{stateCdLookup("WI", "fips")}. +FIPs codes for states take the format: CountryAbbrev:StateNumber, like US:55 for Wisconsin.} \item{countyFips}{County query parameter. To get a list of available counties, run \code{check_USGS_sample_params("counties")}. The "Fips" can be created using the function -\code{countyCdLookup} - for example: \code{countyCdLookup("WI", "Dane", "fips")} +\code{countyCdLookup} - for example: \code{countyCdLookup("WI", "Dane", "fips")} for Dane County, WI. -FIPs codes for counties take the format: +FIPs codes for counties take the format: CountryAbbrev:StateNumber:CountyNumber, like US:55:025 for Dane County, WI.} \item{projectIdentifier}{Project identifier query parameter. This information @@ -103,7 +103,7 @@ information would be needed from the data supplier.} \item{siteTypeName}{Site type name query parameter. See available options by running \code{check_param("sitetype")$typeName}.} -\item{usgsPCode}{USGS parameter code. See available options by running +\item{usgsPCode}{USGS parameter code. See available options by running \code{check_USGS_sample_params("characteristics")$parameterCode}.} \item{pointLocationLatitude}{Latitude for a point/radius query (decimal degrees). Must be used @@ -118,10 +118,10 @@ with pointLocationLatitude and pointLocationLongitude} \item{dataType}{Options include: "Results", "Monitoring locations", "Activities", "Projects", and "Organizations".} -\item{dataProfile}{Profile depends on type. Options for "results" dataType are: +\item{dataProfile}{Profile depends on type. Options for "results" dataType are: "fullphyschem", "basicphyschem", "fullbio", "basicbio", "narrow", -"resultdetectionquantitationlimit", "labsampleprep", "count". Options for "locations" are: -"site" and "count". Options for "activities" are "sampact", "actmetric", "actgroup", +"resultdetectionquantitationlimit", "labsampleprep", "count". Options for "locations" are: +"site" and "count". Options for "activities" are "sampact", "actmetric", "actgroup", and "count". Options for "projects" are: "project" and "projectmonitoringlocationweight". Options for "organizations" are: "organization" and "count".} diff --git a/man/read_USGS_ts_meta.Rd b/man/read_USGS_ts_meta.Rd index 4ca4cf9a..2bf69b3e 100644 --- a/man/read_USGS_ts_meta.Rd +++ b/man/read_USGS_ts_meta.Rd @@ -12,6 +12,7 @@ read_USGS_ts_meta( "parameter_name", "statistic_id", "computation_identifier", "begin", "end", "primary", "sublocation_identifier", "id", "last_modified", "web_description"), limit = 10000, + bbox = NA, statistic_id = NA_character_, last_modified = NA_character_, begin = NA_character_, @@ -30,94 +31,71 @@ read_USGS_ts_meta( ) } \arguments{ -\item{monitoring_location_id}{A unique identifier representing a single monitoring -location. This corresponds to the id field in the sites endpoint. Monitoring -location IDs are created by combining the agency code of the agency responsible -for the monitoring location (e.g. USGS) with the ID number of the monitoring -location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500).} +\item{monitoring_location_id}{A unique identifier representing a single monitoring location. This corresponds to the \code{id} field in the \code{sites} endpoint. Monitoring location IDs are created by combining the agency code of the agency responsible for the monitoring location (e.g. USGS) with the ID number of the monitoring location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500).} -\item{parameter_code}{Parameter codes are 5-digit codes used to identify the -constituent measured and the units of measure.} +\item{parameter_code}{Parameter codes are 5-digit codes used to identify the constituent measured and the units of measure. A complete list of parameter codes and associated groupings can be found at \url{https://help.waterdata.usgs.gov/codes-and-parameters/parameters}.} -\item{parameter_name}{A human-understandable name corresponding to parameter_code.} +\item{parameter_name}{A human-understandable name corresponding to \code{parameter_code}.} -\item{properties}{The properties that should be included for each feature. The -parameter value is a comma-separated list of property names. Available values: -geometry, id, unit_of_measure, parameter_name, parameter_code, statistic_id, -last_modified, begin, end, computation_period_identifier, computation_identifier, -thresholds, sublocation_identifier, primary, monitoring_location_id, web_description.} +\item{properties}{The properties that should be included for each feature. +The parameter value is a comma-separated list of property names. Available options are +geometry, id, unit_of_measure, parameter_name, parameter_code, statistic_id, last_modified, begin, end, computation_period_identifier, computation_identifier, thresholds, sublocation_identifier, primary, monitoring_location_id, web_description} \item{limit}{The optional limit parameter limits the number of items that are presented in the response document. Only items are counted that are on the first level of the collection in the response document. Nested objects contained within the explicitly requested items shall not be counted.} -\item{statistic_id}{A code corresponding to the statistic an observation represents. -Example codes include 00001 (max), 00002 (min), and 00003 (mean).} +\item{bbox}{Only features that have a geometry that intersects the bounding +box are selected.The bounding box is provided as four or six numbers, depending +on whether the coordinate reference system includes a vertical axis (height or +depth). Coordinates are assumed to be in crs 4326.} + +\item{statistic_id}{A code corresponding to the statistic an observation represents. Example codes include 00001 (max), 00002 (min), and 00003 (mean). A complete list of codes and their descriptions can be found at \url{https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=\%25&fmt=html}.} + +\item{last_modified}{The last time a record was refreshed in our database. This may happen due to regular operational processes and does not necessarily indicate anything about the measurement has changed. +You can query this field using date-times or intervals, adhering to RFC 3339, or using ISO 8601 duration objects. Intervals may be bounded or half-bounded (double-dots at start or end). +Examples: +\itemize{ +\item A date-time: "2018-02-12T23:20:50Z" +\item A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" +\item Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" +\item Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours +} -\item{last_modified}{The last time a record was refreshed in our database. This -may happen due to regular operational processes and does not necessarily indicate -anything about the measurement has changed. You can query this field using -date-times or intervals.} +Only features that have a \code{last_modified} that intersects the value of datetime are selected. If a feature has multiple temporal properties, it is the decision of the server whether only a single temporal property is used to determine the extent or all relevant temporal properties.} -\item{begin}{The datetime of the earliest observation in the timeseries. -Together with end, this field represents the period of record of a timeseries. -Note that some timeseries may have large gaps in their collection record.} +\item{begin}{The datetime of the earliest observation in the time series. Together with \code{end}, this field represents the period of record of a time series. Note that some time series may have large gaps in their collection record. This field is currently in the local time of the monitoring location. We intend to update this \strong{in version v0} to use UTC with a time zone.} -\item{end}{The datetime of the most recent observation in the timeseries. -Data returned by this endpoint updates at most once per day, and potentially -less frequently than that, and as such there may be more recent observations -within a timeseries than the timeseries end value reflects. Together with begin, -this field represents the period of record of a timeseries. It is additionally -used to determine whether a timeseries is "active".} +\item{end}{The datetime of the most recent observation in the time series. Data returned by this endpoint updates at most once per day, and potentially less frequently than that, and as such there may be more recent observations within a time series than the time series \code{end} value reflects. Together with \code{begin}, this field represents the period of record of a time series. It is additionally used to determine whether a time series is "active". We intend to update this \strong{in version v0} to use UTC with a time zone.} -\item{unit_of_measure}{A human-readable description of the units of measurement -associated with an observation.} +\item{unit_of_measure}{A human-readable description of the units of measurement associated with an observation.} -\item{computation_period_identifier}{Indicates the period of data used for -any statistical computations.} +\item{computation_period_identifier}{Indicates the period of data used for any statistical computations.} -\item{computation_identifier}{Indicates whether the data from this timeseries -represent a specific statistical computation.} +\item{computation_identifier}{Indicates whether the data from this time series represent a specific statistical computation.} -\item{thresholds}{Thresholds represent known numeric limits for a timeseries, -for example the historic maximum value for a parameter or a level below which -a sensor is non-operative. These thresholds are sometimes used to automatically -determine if an observation is erroneous due to sensor error, and therefore -shouldn't be included in the timeseries.} +\item{thresholds}{Thresholds represent known numeric limits for a time series, for example the historic maximum value for a parameter or a level below which a sensor is non-operative. These thresholds are sometimes used to automatically determine if an observation is erroneous due to sensor error, and therefore shouldn't be included in the time series.} -\item{sublocation_identifier}{An optional human-readable identifier used to -specify where measurements are recorded at a monitoring location.} +\item{sublocation_identifier}{An optional human-readable identifier used to specify where measurements are recorded at a monitoring location.} -\item{primary}{A flag identifying if the timeseries is a "primary" timeseries. -"Primary" timeseries (which have this flag) are standard observations which -undergo Bureau review and approval processes. Non-primary timeseries, which -will have missing values for "primary", are provisional datasets made available -to meet the need for timely best science and to assist with daily operations -which need real-time information. Non-primary timeseries data are only retained -by this system for 120 days. See the USGS Provisional Data Statement for more information.} +\item{primary}{A flag identifying if the time series is a "primary" time series. "Primary" time series (which have this flag) are standard observations which undergo \href{https://www.usgs.gov/survey-manual/5028-fundamental-science-practices-review-and-approval-scientific-data-release}{Bureau review and approval processes}. Non-primary time series, which will have missing values for "primary", are provisional datasets made available to meet the need for timely best science and to assist with daily operations which need real-time information. Non-primary time series data are only retained by this system for 120 days. See the \href{https://waterdata.usgs.gov/provisional-data-statement/}{USGS Provisional Data Statement} for more information.} -\item{id}{A unique identifier representing a single timeseries. This corresponds -to the id field in the timeseries-metadata endpoint.} +\item{id}{A unique identifier representing a single time series. This corresponds to the \code{id} field in the \code{time-series-metadata} endpoint.} -\item{web_description}{A description of what this timeseries represents, as -used by WDFN and other USGS data dissemination products.} +\item{web_description}{A description of what this time series represents, as used by WDFN and other USGS data dissemination products.} \item{crs}{Indicates the coordinate reference system for the results.} \item{skipGeometry}{This option can be used to skip response geometries for -each feature.} +each feature. The returning object will be a data frame with no spatial +information.} \item{convertType}{logical, defaults to \code{TRUE}. If \code{TRUE}, the function will convert the data to dates and qualifier to string vector.} } \description{ -Daily data and continuous measurements are grouped into timeseries, which -represent a collection of observations of a single parameter, potentially -aggregated using a standard statistic, at a single site. This endpoint provides -metadata about those timeseries, including their operational thresholds, units -of measurement, and when the earliest and most recent observations in a timeseries -occurred. +Description Daily data and continuous measurements are grouped into time series, which represent a collection of observations of a single parameter, potentially aggregated using a standard statistic, at a single site. This endpoint provides metadata about those time series, including their operational thresholds, units of measurement, and when the earliest and most recent observations in a time series occurred. } \examples{ \dontshow{if (is_dataRetrieval_user()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} diff --git a/man/renameNWISColumns.Rd b/man/renameNWISColumns.Rd index 9ee611f1..55244ecb 100644 --- a/man/renameNWISColumns.Rd +++ b/man/renameNWISColumns.Rd @@ -86,7 +86,7 @@ names(newNames) } } \seealso{ -\code{\link{readNWISdv}}, \code{\link{readNWISuv}} +\code{\link[=readNWISdv]{readNWISdv()}}, \code{\link[=readNWISuv]{readNWISuv()}} } \keyword{IO} \keyword{manip} diff --git a/man/whatNWISdata.Rd b/man/whatNWISdata.Rd index 120ee997..225ab92f 100644 --- a/man/whatNWISdata.Rd +++ b/man/whatNWISdata.Rd @@ -65,7 +65,7 @@ for more information. \details{ This function requires users to create their own arguments based on the NWIS web services. It is a more complicated function to use -compared to other NWIS functions such as \code{\link{readNWISdv}}, \code{\link{readNWISuv}}, +compared to other NWIS functions such as \code{\link[=readNWISdv]{readNWISdv()}}, \code{\link[=readNWISuv]{readNWISuv()}}, etc. However, this function adds a lot of flexibility to the possible queries. If the "service" argument is included, the results will be filtered to the proper data_type_cd. This is a great diff --git a/man/whatWQPdata.Rd b/man/whatWQPdata.Rd index b6fedafd..9e8d8f7f 100644 --- a/man/whatWQPdata.Rd +++ b/man/whatWQPdata.Rd @@ -9,10 +9,10 @@ whatWQPdata(..., convertType = TRUE) \arguments{ \item{\dots}{see \url{https://www.waterqualitydata.us/webservices_documentation} for a complete list of options. A list of arguments can also be supplied. -One way to figure out how to construct a WQP query is to go to the "Advanced" +One way to figure out how to construct a WQP query is to go to the "Advanced" form in the Water Quality Portal: \url{https://www.waterqualitydata.us/#mimeType=csv&providers=NWIS&providers=STORET} -Use the form to discover what parameters are available. Once the query is +Use the form to discover what parameters are available. Once the query is set in the form, scroll down to the "Query URL". You will see the parameters after "https://www.waterqualitydata.us/#". For example, if you chose "Nutrient" in the Characteristic Group dropdown, you will see characteristicType=Nutrient @@ -25,8 +25,8 @@ will convert the data to dates, datetimes, numerics based on a standard algorithm. If false, everything is returned as a character.} } \value{ -A data frame that returns basic data availability such as -sites, number of results, and number of sampling activities from the +A data frame that returns basic data availability such as +sites, number of results, and number of sampling activities from the query parameters for the Water Quality Portal. } \description{ @@ -36,11 +36,11 @@ Arguments to the function should be based on \url{https://www.waterqualitydata.us/webservices_documentation}. The information returned from whatWQPdata describes the available data at the WQP sites, and some metadata on the sites themselves. -For example, a row is returned for each individual site that fulfills this -query. In that we can learn how many sampling activities and results -are available for the query. It does not break those results down by any finer +For example, a row is returned for each individual site that fulfills this +query. In that we can learn how many sampling activities and results +are available for the query. It does not break those results down by any finer grain. For example, if you ask for "Nutrients" (characteristicGroup), you will -not learn what specific nutrients are available at that site. For that +not learn what specific nutrients are available at that site. For that kind of data discovery see \code{readWQPsummary}. } \examples{ diff --git a/man/wqpSpecials.Rd b/man/wqpSpecials.Rd index fda2a399..369ef387 100644 --- a/man/wqpSpecials.Rd +++ b/man/wqpSpecials.Rd @@ -15,10 +15,10 @@ whatWQPsites(..., legacy = TRUE, convertType = TRUE) \arguments{ \item{\dots}{see \url{https://www.waterqualitydata.us/webservices_documentation} for a complete list of options. A list of arguments can also be supplied. -One way to figure out how to construct a WQP query is to go to the "Advanced" +One way to figure out how to construct a WQP query is to go to the "Advanced" form in the Water Quality Portal: \url{https://www.waterqualitydata.us/#mimeType=csv&providers=NWIS&providers=STORET} -Use the form to discover what parameters are available. Once the query is +Use the form to discover what parameters are available. Once the query is set in the form, scroll down to the "Query URL". You will see the parameters after "https://www.waterqualitydata.us/#". For example, if you chose "Nutrient" in the Characteristic Group dropdown, you will see characteristicType=Nutrient @@ -45,7 +45,7 @@ gets the data from: \url{https://www.waterqualitydata.us}. Arguments to the function should be based on \url{https://www.waterqualitydata.us/webservices_documentation}. The return from this function returns the basic metadata on WQP sites. It is -generally faster than the \code{\link{whatWQPdata}} function, but does +generally faster than the \code{\link[=whatWQPdata]{whatWQPdata()}} function, but does not return information on what data was collected at the site. } \examples{ diff --git a/man/wqp_check_status.Rd b/man/wqp_check_status.Rd index f46c172e..ca3e4737 100644 --- a/man/wqp_check_status.Rd +++ b/man/wqp_check_status.Rd @@ -15,8 +15,8 @@ a list generated from the WQP describing what data was returned. } \description{ -The information from this request is only available for a -limited time after the original query from the WQP. In the +The information from this request is only available for a +limited time after the original query from the WQP. In the readWQPdata and readWQPqw functions, the results from this function will be attached as an attribute to the data. } From 245c4921acaf3f169be1a01571828ddd09f5a315 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 8 May 2025 15:52:03 -0500 Subject: [PATCH 019/117] adding site service --- NAMESPACE | 1 + R/construct_api_requests.R | 120 +++++--------------- R/read_USGS_dv.R | 2 +- R/read_USGS_sites.R | 176 ++++++++++++++++++++++++++++++ R/walk_pages.R | 3 +- _pkgdown.yml | 1 + man/construct_api_requests.Rd | 88 +-------------- man/read_USGS_dv.Rd | 11 +- man/read_USGS_sites.Rd | 170 +++++++++++++++++++++++++++++ vignettes/read_USGS_functions.Rmd | 22 +++- 10 files changed, 414 insertions(+), 180 deletions(-) create mode 100644 R/read_USGS_sites.R create mode 100644 man/read_USGS_sites.Rd diff --git a/NAMESPACE b/NAMESPACE index bbd1e1ad..64296e5a 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -46,6 +46,7 @@ export(readWQPqw) export(readWQPsummary) export(read_USGS_dv) export(read_USGS_samples) +export(read_USGS_sites) export(read_USGS_ts_meta) export(renameNWISColumns) export(setAccess) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index a05eb11d..4efb8a07 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -4,26 +4,8 @@ #' Swagger docs: . #' #' @export -#' @param monitoring_location_id `r get_params("daily")$monitoring_location_id` -#' @param parameter_code `r get_params("daily")$parameter_code` -#' @param statistic_id `r get_params("daily")$statistic_id` -#' @param time `r get_params("daily")$time` -#' @param value `r get_params("daily")$value` -#' @param unit_of_measure `r get_params("daily")$unit_of_measure` -#' @param approval_status `r get_params("daily")$approval_status` -#' @param last_modified `r get_params("daily")$last_modified` -#' @param time_series_id `r get_params("daily")$time_series_id` -#' @param qualifier `r get_params("daily")$qualifier` -#' @param id `r get_params("daily")$id` -#' @param parameter_name `r get_params("time-series-metadata")$parameter_name` -#' @param computation_identifier `r get_params("time-series-metadata")$computation_identifier` -#' @param computation_period_identifier `r get_params("time-series-metadata")$computation_period_identifier` -#' @param sublocation_identifier `r get_params("time-series-metadata")$sublocation_identifier` -#' @param begin `r get_params("time-series-metadata")$begin` -#' @param end `r get_params("time-series-metadata")$end` -#' @param thresholds `r get_params("time-series-metadata")$thresholds` -#' @param primary `r get_params("time-series-metadata")$primary` -#' @param web_description `r get_params("time-series-metadata")$web_description` +#' @param service Which service +#' @param ... Extra parameters from the specific services. #' @param bbox Only features that have a geometry that intersects the bounding #' box are selected.The bounding box is provided as four or six numbers, depending #' on whether the coordinate reference system includes a vertical axis (height or @@ -61,39 +43,12 @@ #' datetime = c(start_date, end_date)) #' construct_api_requests <- function(service, - monitoring_location_id = NA_character_, - parameter_code = NA_character_, - statistic_id = NA_character_, - properties = c("monitoring_location_id", - "parameter_code", - "statistic_id", - "time", - "value", - "unit_of_measure", - "approval_status", - "qualifier"), - bbox = NA, - time_series_id = NA_character_, - id = NA_character_, - approval_status = NA_character_, - unit_of_measure = NA_character_, - qualifier = NA_character_, - value = NA, - last_modified = NA_character_, - limit = 10000, - crs = NA_character_, - bbox_crs = NA_character_, - skipGeometry = FALSE, - datetime = NA_character_, - begin = NA_character_, - end = NA_character_, - primary = NA_character_, - parameter_name = NA_character_, - thresholds = NA, - sublocation_identifier = NA_character_, - computation_period_identifier = NA_character_, - computation_identifier = NA_character_, - web_description = NA_character_){ + properties = NA_character_, + bbox = NA, + crs = NA_character_, + limit = 10000, + skipGeometry = FALSE, + ...){ schema <- check_OGC_requests(endpoint = service, type = "schema") @@ -120,37 +75,32 @@ construct_api_requests <- function(service, template_path_post <- system.file("templates/post.CQL2", package = "dataRetrieval") template_post <- readChar(template_path_post, file.info(template_path_post)$size) - post_params <- explode_post(list(monitoring_location_id = monitoring_location_id, - parameter_code = parameter_code, - statistic_id = statistic_id, - time_series_id = time_series_id, - id = id, - approval_status = approval_status, - unit_of_measure = unit_of_measure, - qualifier = qualifier, - value = value, - parameter_name = parameter_name)) + single_params <- c("datetime", "last_modified", "begin", "end", "time") + + full_list <- list(...) + get_list <- full_list[names(full_list) %in% single_params] + + get_list[["crs"]] <- crs + get_list[["skipGeometry"]] <- skipGeometry + get_list[["limit"]] <- limit + + post_list <- full_list[!names(full_list) %in% single_params] + + post_params <- explode_post(post_list) if(length(post_params) > 0){ POST = TRUE } - datetime <- format_api_dates(datetime) + time_periods <- c("last_modified", "datetime", "time") + if(any(time_periods %in% names(get_list))){ + for(i in time_periods){ + get_list[[i]] <- format_api_dates(get_list[[i]]) + full_list[[i]] <- format_api_dates(full_list[[i]]) + } + } - baseURL <- explode_query(baseURL, POST = FALSE, - list(last_modified = last_modified, - begin = begin, - end = end, - primary = primary, - computation_period_identifier = computation_period_identifier, - computation_identifier = computation_identifier, - web_description = web_description, - properties = properties, - limit = limit, - crs = crs, - `bbox-crs` = bbox_crs, - skipGeometry = skipGeometry, - datetime = datetime)) + baseURL <- explode_query(baseURL, POST = FALSE, get_list) if(all(!is.na(bbox))){ baseURL <- httr2::req_url_query(baseURL, @@ -176,17 +126,7 @@ construct_api_requests <- function(service, baseURL <- httr2::req_body_raw(baseURL, x) } else { - baseURL <- explode_query(baseURL, POST = FALSE, - list(monitoring_location_id = monitoring_location_id, - parameter_code = parameter_code, - statistic_id = statistic_id, - time_series_id = time_series_id, - id = id, - approval_status = approval_status, - unit_of_measure = unit_of_measure, - qualifier = qualifier, - value = value, - parameter_name = parameter_name)) + baseURL <- explode_query(baseURL, POST = FALSE, full_list) } return(baseURL) @@ -213,7 +153,7 @@ setup_api <- function(service){ format_api_dates <- function(datetime){ - if(!any(is.na(datetime))){ + if(!any(isTRUE(is.na(datetime)) | isTRUE(is.null(datetime)))){ if(length(datetime) == 1){ datetime <- format(datetime, format = "%Y-%m-%dT%H:%M:%SZ") } else if (length(datetime) == 2) { diff --git a/R/read_USGS_dv.R b/R/read_USGS_dv.R index be866424..7c8afaf0 100644 --- a/R/read_USGS_dv.R +++ b/R/read_USGS_dv.R @@ -5,7 +5,7 @@ #' @export #' @param monitoring_location_id `r get_params("daily")$monitoring_location_id` #' @param parameter_code `r get_params("daily")$parameter_code` -#' @param datetime `r get_params("daily")$datetime` +#' @param datetime `r get_params("daily")$time` #' @param statistic_id `r get_params("daily")$statistic_id` #' @param time `r get_params("daily")$time` #' @param value `r get_params("daily")$value` diff --git a/R/read_USGS_sites.R b/R/read_USGS_sites.R new file mode 100644 index 00000000..c0a36aea --- /dev/null +++ b/R/read_USGS_sites.R @@ -0,0 +1,176 @@ +#' Get USGS Daily Data +#' +#' Description `r get_description("daily")` +#' +#' @export +#' @param monitoring_location_id `r get_params("sites")$id` +#' @param agency_code `r get_params("sites")$agency_code` +#' @param agency_name `r get_params("sites")$agency_name` +#' @param monitoring_location_number `r get_params("sites")$monitoring_location_number` +#' @param monitoring_location_name `r get_params("sites")$monitoring_location_name` +#' @param district_code `r get_params("sites")$district_code` +#' @param state_name `r get_params("sites")$state_name` +#' @param county_code `r get_params("sites")$county_code` +#' @param county_name `r get_params("sites")$county_name` +#' @param country_code `r get_params("sites")$country_code` +#' @param country_name `r get_params("sites")$country_name` +#' @param state_code `r get_params("sites")$state_code` +#' @param minor_civil_division_code `r get_params("sites")$minor_civil_division_code` +#' @param site_type_code `r get_params("sites")$site_type_code` +#' @param site_type `r get_params("sites")$site_type` +#' @param hydrologic_unit_code `r get_params("sites")$hydrologic_unit_code` +#' @param basin_code `r get_params("sites")$basin_code` +#' @param altitude `r get_params("sites")$altitude` +#' @param altitude_accuracy `r get_params("sites")$altitude_accuracy` +#' @param altitude_method_code `r get_params("sites")$altitude_method_code` +#' @param altitude_method_name `r get_params("sites")$altitude_method_name` +#' @param vertical_datum `r get_params("sites")$vertical_datum` +#' @param vertical_datum_name `r get_params("sites")$vertical_datum_name` +#' @param horizontal_positional_accuracy_code `r get_params("sites")$horizontal_positional_accuracy_code` +#' @param horizontal_positional_accuracy `r get_params("sites")$horizontal_positional_accuracy` +#' @param horizontal_position_method_code `r get_params("sites")$horizontal_position_method_code` +#' @param horizontal_position_method_name `r get_params("sites")$horizontal_position_method_name` +#' @param original_horizontal_datum `r get_params("sites")$original_horizontal_datum` +#' @param original_horizontal_datum_name `r get_params("sites")$original_horizontal_datum_name` +#' @param drainage_area `r get_params("sites")$drainage_area` +#' @param contributing_drainage_area `r get_params("sites")$contributing_drainage_area` +#' @param time_zone_abbreviation `r get_params("sites")$time_zone_abbreviation` +#' @param uses_daylight_savings `r get_params("sites")$uses_daylight_savings` +#' @param construction_date `r get_params("sites")$construction_date` +#' @param aquifer_code `r get_params("sites")$aquifer_code` +#' @param national_aquifer_code `r get_params("sites")$national_aquifer_code` +#' @param aquifer_type_code `r get_params("sites")$aquifer_type_code` +#' @param well_constructed_depth `r get_params("sites")$well_constructed_depth` +#' @param hole_constructed_depth `r get_params("sites")$hole_constructed_depth` +#' @param depth_source_code `r get_params("sites")$depth_source_code` +#' @param properties The properties that should be included for each feature. +#' The parameter value is a comma-separated list of property names. Available options are +#' `r schema <- check_OGC_requests(endpoint = "sites", type = "schema"); paste(names(schema$properties), collapse = ", ")` +#' @param bbox Only features that have a geometry that intersects the bounding +#' box are selected.The bounding box is provided as four or six numbers, depending +#' on whether the coordinate reference system includes a vertical axis (height or +#' depth). Coordinates are assumed to be in crs 4326. +#' @param crs Indicates the coordinate reference system for the results. +#' @param limit The optional limit parameter limits the number of items that are +#' presented in the response document. Only items are counted that are on the +#' first level of the collection in the response document. Nested objects +#' contained within the explicitly requested items shall not be counted. +#' @param skipGeometry This option can be used to skip response geometries for +#' each feature. The returning object will be a data frame with no spatial +#' information. +#' @param convertType logical, defaults to `TRUE`. If `TRUE`, the function +#' will convert the data to dates and qualifier to string vector. +#' @examplesIf is_dataRetrieval_user() +#' +#' \donttest{ +#' site <- "USGS-02238500" +#' site_info <- read_USGS_sites(monitoring_location_id = site) +#' +#' site_info_no_sf <- read_USGS_sites(monitoring_location_id = site, +#' skipGeometry = TRUE) +#' +#' multi_site <- read_USGS_sites(state_name = "Wisconsin") +#' +#' } +read_USGS_sites <- function(monitoring_location_id = NA_character_, + agency_code = NA_character_, + agency_name = NA_character_, + monitoring_location_number = NA_character_, + monitoring_location_name = NA_character_, + district_code = NA_character_, + country_code = NA_character_, + country_name = NA_character_, + state_code = NA_character_, + state_name = NA_character_, + county_code = NA_character_, + county_name = NA_character_, + minor_civil_division_code = NA_character_, + site_type_code = NA_character_, + site_type = NA_character_, + hydrologic_unit_code = NA_character_, + basin_code = NA_character_, + altitude = NA_character_, + altitude_accuracy = NA_character_, + altitude_method_code = NA_character_, + altitude_method_name = NA_character_, + vertical_datum = NA_character_, + vertical_datum_name = NA_character_, + horizontal_positional_accuracy_code = NA_character_, + horizontal_positional_accuracy = NA_character_, + horizontal_position_method_code = NA_character_, + horizontal_position_method_name = NA_character_, + original_horizontal_datum = NA_character_, + original_horizontal_datum_name = NA_character_, + drainage_area = NA_character_, + contributing_drainage_area = NA_character_, + time_zone_abbreviation = NA_character_, + uses_daylight_savings = NA_character_, + construction_date = NA_character_, + aquifer_code = NA_character_, + national_aquifer_code = NA_character_, + aquifer_type_code = NA_character_, + well_constructed_depth = NA_character_, + hole_constructed_depth = NA_character_, + depth_source_code = NA_character_, + properties = NA_character_, + bbox = NA, + limit = 10000, + crs = NA_character_, + skipGeometry = NA, + convertType = TRUE){ + + message("Function in development, use at your own risk.") + + site_req <- construct_api_requests(service = "sites", + id = monitoring_location_id, + agency_code = agency_code, + agency_name = agency_name, + monitoring_location_number = monitoring_location_number, + monitoring_location_name = monitoring_location_name, + district_code = district_code, + country_code = country_code, + country_name = country_name, + state_code = state_code, + state_name = state_name, + county_code = county_code, + county_name = county_name, + minor_civil_division_code = minor_civil_division_code, + site_type_code = site_type_code, + site_type = site_type, + hydrologic_unit_code = hydrologic_unit_code, + basin_code = basin_code, + altitude = altitude, + altitude_accuracy = altitude_accuracy, + altitude_method_code = altitude_method_code, + altitude_method_name = altitude_method_name, + vertical_datum = vertical_datum, + vertical_datum_name = vertical_datum_name, + horizontal_positional_accuracy_code = horizontal_positional_accuracy_code, + horizontal_positional_accuracy = horizontal_positional_accuracy, + horizontal_position_method_code = horizontal_position_method_code, + horizontal_position_method_name = horizontal_position_method_name, + original_horizontal_datum = original_horizontal_datum, + original_horizontal_datum_name = original_horizontal_datum_name, + drainage_area = drainage_area, + contributing_drainage_area = contributing_drainage_area, + time_zone_abbreviation = time_zone_abbreviation, + uses_daylight_savings = uses_daylight_savings, + construction_date = construction_date, + aquifer_code = aquifer_code, + national_aquifer_code = national_aquifer_code, + aquifer_type_code = aquifer_type_code, + well_constructed_depth = well_constructed_depth, + hole_constructed_depth = hole_constructed_depth, + depth_source_code = depth_source_code, + limit = limit, + bbox = bbox, + crs = crs, + skipGeometry = skipGeometry, + properties = properties) + + return_list <- walk_pages(site_req) + + if(convertType) return_list <- cleanup_cols(return_list) + + return(return_list) +} diff --git a/R/walk_pages.R b/R/walk_pages.R index b2d3de1c..1dd04fe3 100644 --- a/R/walk_pages.R +++ b/R/walk_pages.R @@ -83,8 +83,7 @@ walk_pages_recursive <- function(req, page, contents) { walk_pages_recursive, make_request, page + 1, - contents, - use_sf + contents ) } } \ No newline at end of file diff --git a/_pkgdown.yml b/_pkgdown.yml index a7e40331..7dc91fd1 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -70,6 +70,7 @@ reference: - read_USGS_samples - read_USGS_dv - read_USGS_ts_meta + - read_USGS_sites - summarize_USGS_samples - check_USGS_sample_params - title: National Water Information System (NWIS) diff --git a/man/construct_api_requests.Rd b/man/construct_api_requests.Rd index 9f1953f8..9cdf218f 100644 --- a/man/construct_api_requests.Rd +++ b/man/construct_api_requests.Rd @@ -6,41 +6,16 @@ \usage{ construct_api_requests( service, - monitoring_location_id = NA_character_, - parameter_code = NA_character_, - statistic_id = NA_character_, - properties = c("monitoring_location_id", "parameter_code", "statistic_id", "time", - "value", "unit_of_measure", "approval_status", "qualifier"), + properties = NA_character_, bbox = NA, - time_series_id = NA_character_, - id = NA_character_, - approval_status = NA_character_, - unit_of_measure = NA_character_, - qualifier = NA_character_, - value = NA, - last_modified = NA_character_, - limit = 10000, crs = NA_character_, - bbox_crs = NA_character_, + limit = 10000, skipGeometry = FALSE, - datetime = NA_character_, - begin = NA_character_, - end = NA_character_, - primary = NA_character_, - parameter_name = NA_character_, - thresholds = NA, - sublocation_identifier = NA_character_, - computation_period_identifier = NA_character_, - computation_identifier = NA_character_, - web_description = NA_character_ + ... ) } \arguments{ -\item{monitoring_location_id}{A unique identifier representing a single monitoring location. This corresponds to the \code{id} field in the \code{sites} endpoint. Monitoring location IDs are created by combining the agency code of the agency responsible for the monitoring location (e.g. USGS) with the ID number of the monitoring location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500).} - -\item{parameter_code}{Parameter codes are 5-digit codes used to identify the constituent measured and the units of measure. A complete list of parameter codes and associated groupings can be found at \url{https://help.waterdata.usgs.gov/codes-and-parameters/parameters}.} - -\item{statistic_id}{A code corresponding to the statistic an observation represents. Example codes include 00001 (max), 00002 (min), and 00003 (mean). A complete list of codes and their descriptions can be found at \url{https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=\%25&fmt=html}.} +\item{service}{Which service} \item{properties}{The properties that should be included for each feature. The parameter value is a comma-separated list of property names. Available values: @@ -52,69 +27,18 @@ box are selected.The bounding box is provided as four or six numbers, depending on whether the coordinate reference system includes a vertical axis (height or depth).} -\item{time_series_id}{A unique identifier representing a single time series. This corresponds to the \code{id} field in the \code{time-series-metadata} endpoint.} - -\item{id}{A universally unique identifier (UUID) representing a single version of a record. It is not stable over time. Every time the record is refreshed in our database (which may happen as part of normal operations and does not imply any change to the data itself) a new ID will be generated. To uniquely identify a single observation over time, compare the \code{time} and \code{time_series_id} fields; each time series will only have a single observation at a given \code{time}.} - -\item{approval_status}{Some of the data that you have obtained from this U.S. Geological Survey database may not have received Director's approval. Any such data values are qualified as provisional and are subject to revision. Provisional data are released on the condition that neither the USGS nor the United States Government may be held liable for any damages resulting from its use. This field reflects the approval status of each record, and is either "Approved", meaining processing review has been completed and the data is approved for publication, or "Provisional" and subject to revision. For more information about provisional data, go to \url{https://waterdata.usgs.gov/provisional-data-statement/}.} - -\item{unit_of_measure}{A human-readable description of the units of measurement associated with an observation.} - -\item{qualifier}{This field indicates any qualifiers associated with an observation, for instance if a sensor may have been impacted by ice or if values were estimated.} - -\item{value}{The value of the observation. Values are transmitted as strings in the JSON response format in order to preserve precision.} - -\item{last_modified}{The last time a record was refreshed in our database. This may happen due to regular operational processes and does not necessarily indicate anything about the measurement has changed. -You can query this field using date-times or intervals, adhering to RFC 3339, or using ISO 8601 duration objects. Intervals may be bounded or half-bounded (double-dots at start or end). -Examples: -\itemize{ -\item A date-time: "2018-02-12T23:20:50Z" -\item A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" -\item Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" -\item Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours -} - -Only features that have a \code{last_modified} that intersects the value of datetime are selected. If a feature has multiple temporal properties, it is the decision of the server whether only a single temporal property is used to determine the extent or all relevant temporal properties.} +\item{crs}{Indicates the coordinate reference system for the results.} \item{limit}{The optional limit parameter limits the number of items that are presented in the response document. Only items are counted that are on the first level of the collection in the response document. Nested objects contained within the explicitly requested items shall not be counted.} -\item{crs}{Indicates the coordinate reference system for the results.} - \item{skipGeometry}{This option can be used to skip response geometries for each feature. The returning object will be a data frame with no spatial information.} -\item{begin}{The datetime of the earliest observation in the time series. Together with \code{end}, this field represents the period of record of a time series. Note that some time series may have large gaps in their collection record. This field is currently in the local time of the monitoring location. We intend to update this \strong{in version v0} to use UTC with a time zone.} - -\item{end}{The datetime of the most recent observation in the time series. Data returned by this endpoint updates at most once per day, and potentially less frequently than that, and as such there may be more recent observations within a time series than the time series \code{end} value reflects. Together with \code{begin}, this field represents the period of record of a time series. It is additionally used to determine whether a time series is "active". We intend to update this \strong{in version v0} to use UTC with a time zone.} - -\item{primary}{A flag identifying if the time series is a "primary" time series. "Primary" time series (which have this flag) are standard observations which undergo \href{https://www.usgs.gov/survey-manual/5028-fundamental-science-practices-review-and-approval-scientific-data-release}{Bureau review and approval processes}. Non-primary time series, which will have missing values for "primary", are provisional datasets made available to meet the need for timely best science and to assist with daily operations which need real-time information. Non-primary time series data are only retained by this system for 120 days. See the \href{https://waterdata.usgs.gov/provisional-data-statement/}{USGS Provisional Data Statement} for more information.} - -\item{parameter_name}{A human-understandable name corresponding to \code{parameter_code}.} - -\item{thresholds}{Thresholds represent known numeric limits for a time series, for example the historic maximum value for a parameter or a level below which a sensor is non-operative. These thresholds are sometimes used to automatically determine if an observation is erroneous due to sensor error, and therefore shouldn't be included in the time series.} - -\item{sublocation_identifier}{An optional human-readable identifier used to specify where measurements are recorded at a monitoring location.} - -\item{computation_period_identifier}{Indicates the period of data used for any statistical computations.} - -\item{computation_identifier}{Indicates whether the data from this time series represent a specific statistical computation.} - -\item{web_description}{A description of what this time series represents, as used by WDFN and other USGS data dissemination products.} - -\item{time}{The date an observation represents. You can query this field using date-times or intervals, adhering to RFC 3339, or using ISO 8601 duration objects. Intervals may be bounded or half-bounded (double-dots at start or end). -Examples: -\itemize{ -\item A date-time: "2018-02-12T23:20:50Z" -\item A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" -\item Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" -\item Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours -} - -Only features that have a \code{time} that intersects the value of datetime are selected. If a feature has multiple temporal properties, it is the decision of the server whether only a single temporal property is used to determine the extent or all relevant temporal properties.} +\item{...}{Extra parameters from the specific services.} } \description{ Main documentation: \url{https://api.waterdata.usgs.gov/ogcapi/v0/}, diff --git a/man/read_USGS_dv.Rd b/man/read_USGS_dv.Rd index 0860bd51..26a9b294 100644 --- a/man/read_USGS_dv.Rd +++ b/man/read_USGS_dv.Rd @@ -76,7 +76,16 @@ contained within the explicitly requested items shall not be counted.} each feature. The returning object will be a data frame with no spatial information.} -\item{datetime}{} +\item{datetime}{The date an observation represents. You can query this field using date-times or intervals, adhering to RFC 3339, or using ISO 8601 duration objects. Intervals may be bounded or half-bounded (double-dots at start or end). +Examples: +\itemize{ +\item A date-time: "2018-02-12T23:20:50Z" +\item A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" +\item Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" +\item Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours +} + +Only features that have a \code{time} that intersects the value of datetime are selected. If a feature has multiple temporal properties, it is the decision of the server whether only a single temporal property is used to determine the extent or all relevant temporal properties.} \item{convertType}{logical, defaults to \code{TRUE}. If \code{TRUE}, the function will convert the data to dates and qualifier to string vector.} diff --git a/man/read_USGS_sites.Rd b/man/read_USGS_sites.Rd new file mode 100644 index 00000000..7dd0438f --- /dev/null +++ b/man/read_USGS_sites.Rd @@ -0,0 +1,170 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/read_USGS_sites.R +\name{read_USGS_sites} +\alias{read_USGS_sites} +\title{Get USGS Daily Data} +\usage{ +read_USGS_sites( + monitoring_location_id = NA_character_, + agency_code = NA_character_, + agency_name = NA_character_, + monitoring_location_number = NA_character_, + monitoring_location_name = NA_character_, + district_code = NA_character_, + country_code = NA_character_, + country_name = NA_character_, + state_code = NA_character_, + state_name = NA_character_, + county_code = NA_character_, + county_name = NA_character_, + minor_civil_division_code = NA_character_, + site_type_code = NA_character_, + site_type = NA_character_, + hydrologic_unit_code = NA_character_, + basin_code = NA_character_, + altitude = NA_character_, + altitude_accuracy = NA_character_, + altitude_method_code = NA_character_, + altitude_method_name = NA_character_, + vertical_datum = NA_character_, + vertical_datum_name = NA_character_, + horizontal_positional_accuracy_code = NA_character_, + horizontal_positional_accuracy = NA_character_, + horizontal_position_method_code = NA_character_, + horizontal_position_method_name = NA_character_, + original_horizontal_datum = NA_character_, + original_horizontal_datum_name = NA_character_, + drainage_area = NA_character_, + contributing_drainage_area = NA_character_, + time_zone_abbreviation = NA_character_, + uses_daylight_savings = NA_character_, + construction_date = NA_character_, + aquifer_code = NA_character_, + national_aquifer_code = NA_character_, + aquifer_type_code = NA_character_, + well_constructed_depth = NA_character_, + hole_constructed_depth = NA_character_, + depth_source_code = NA_character_, + bbox = NA, + limit = 10000, + crs = NA_character_, + skipGeometry = NA, + convertType = TRUE +) +} +\arguments{ +\item{monitoring_location_id}{A unique identifier representing a single monitoring location. This corresponds to the \code{id} field in the \code{sites} endpoint. Monitoring location IDs are created by combining the agency code of the agency responsible for the monitoring location (e.g. USGS) with the ID number of the monitoring location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500).} + +\item{agency_code}{The agency that is reporting the data. Agency codes are fixed values assigned by the National Water Information System (NWIS). A list of agency codes is available \href{https://help.waterdata.usgs.gov/code/agency_cd_query?fmt=html}{at this link}.} + +\item{agency_name}{The name of the agency that is reporting the data.} + +\item{monitoring_location_number}{Each site in the USGS data base has a unique 8- to 15-digit identification number. Site numbers are assigned \href{https://help.waterdata.usgs.gov/faq/sites/do-station-numbers-have-any-particular-meaning}{based on this logic}.} + +\item{monitoring_location_name}{This is the official name of the site in the database. For well information this can be a district-assigned local number.} + +\item{district_code}{The Water Science Centers (WSCs) across the United States use the FIPS state code as the district code. In some case, sites and samples may be managed by a water science center that is adjacent to the state in which the site actually resides. For example a site may have a district code of 30 which translates to Montana, but the state code could be 56 for Wyoming because that is where the site actually is located.} + +\item{country_code}{The code for the country in which the site is located.} + +\item{country_name}{The name of the country in which the site is located.} + +\item{state_code}{State code. A \href{https://www2.census.gov/geo/docs/reference/state.txt}{two-digit ANSI code} (formerly FIPS code) as defined by the American National Standards Institute, to define States and equivalents. A three-digit ANSI code is used to define counties and county equivalents. \href{https://www.census.gov/library/reference/code-lists/ansi.html#states}{A lookup table is available.} The only countries with political subdivisions other than the US are Mexico and Canada. The Mexican states have US state codes ranging from 81-86 and Canadian provinces have state codes ranging from 90-98.} + +\item{minor_civil_division_code}{Codes for primary governmental or administrative divisions of the county or county equivalent in which the site is located.} + +\item{site_type_code}{A code describing the hydrologic setting of the site. \href{https://help.waterdata.usgs.gov/code/site_tp_query?fmt=html}{A list of codes is available.}} + +\item{site_type}{A description of the hydrologic setting of the site. \href{https://help.waterdata.usgs.gov/code/site_tp_query?fmt=html}{A list of codes is available.}} + +\item{hydrologic_unit_code}{The United States is divided and sub-divided into successively smaller hydrologic units which are classified into four levels: regions, sub-regions, accounting units, and cataloging units. The hydrologic units are arranged within each other, from the smallest (cataloging units) to the largest (regions). Each hydrologic unit is identified by a unique hydrologic unit code (HUC) consisting of two to eight digits based on the four levels of classification in the hydrologic unit system.} + +\item{basin_code}{The Basin Code or "drainage basin code" is a two-digit code that further subdivides the 8-digit hydrologic-unit code. The drainage basin code is defined by the USGS State Office where the site is located.} + +\item{altitude}{Altitude of the site referenced to the specified Vertical Datum.} + +\item{altitude_accuracy}{Accuracy of the altitude, in feet. An accuracy of +/- 0.1 foot would be entered as “.1”. Many altitudes are interpolated from the contours on topographic maps; accuracies determined in this way are generally entered as one-half of the contour interval.} + +\item{altitude_method_code}{Codes representing the method used to measure altitude. \href{https://help.waterdata.usgs.gov/code/alt_meth_cd_query?fmt=html}{A list of codes is available.}} + +\item{altitude_method_name}{The name of the the method used to measure altitude. \href{https://help.waterdata.usgs.gov/code/alt_meth_cd_query?fmt=html}{A list of codes is available.}} + +\item{vertical_datum}{The datum used to determine altitude and vertical position at the site. \href{https://help.waterdata.usgs.gov/code/alt_datum_cd_query?fmt=html}{A list of codes is available.}} + +\item{vertical_datum_name}{The datum used to determine altitude and vertical position at the site. \href{https://help.waterdata.usgs.gov/code/alt_datum_cd_query?fmt=html}{A list of codes is available.}} + +\item{horizontal_positional_accuracy_code}{Indicates the accuracy of the latitude longitude values. \href{https://help.waterdata.usgs.gov/code/coord_acy_cd_query?fmt=html}{A list of codes is available.}} + +\item{horizontal_positional_accuracy}{Indicates the accuracy of the latitude longitude values. \href{https://help.waterdata.usgs.gov/code/coord_acy_cd_query?fmt=html}{A list of codes is available.}} + +\item{horizontal_position_method_code}{Indicates the method used to determine latitude longitude values. \href{https://help.waterdata.usgs.gov/code/coord_meth_cd_query?fmt=html}{A list of codes is available.}} + +\item{horizontal_position_method_name}{Indicates the method used to determine latitude longitude values. \href{https://help.waterdata.usgs.gov/code/coord_meth_cd_query?fmt=html}{A list of codes is available.}} + +\item{original_horizontal_datum}{Coordinates are published in EPSG:4326 / WGS84 / World Geodetic System 1984. This field indicates the original datum used to determine coordinates before they were converted. \href{https://help.waterdata.usgs.gov/code/coord_datum_cd_query?fmt=html}{A list of codes is available.}} + +\item{original_horizontal_datum_name}{Coordinates are published in EPSG:4326 / WGS84 / World Geodetic System 1984. This field indicates the original datum used to determine coordinates before they were converted. \href{https://help.waterdata.usgs.gov/code/coord_datum_cd_query?fmt=html}{A list of codes is available.}} + +\item{drainage_area}{The area enclosed by a topographic divide from which direct surface runoff from precipitation normally drains by gravity into the stream above that point.} + +\item{contributing_drainage_area}{The contributing drainage area of a lake, stream, wetland, or estuary site, in square miles. This item should be present only if the contributing area is different from the total drainage area. This situation can occur when part of the drainage area consists of very porous soil or depressions that either allow all runoff to enter the groundwater or traps the water in ponds so that rainfall does not contribute to runoff. A transbasin diversion can also affect the total drainage area.} + +\item{time_zone_abbreviation}{A short code describing the time zone used by a site.} + +\item{uses_daylight_savings}{A flag indicating whether or not a site uses daylight savings.} + +\item{construction_date}{Date the well was completed.} + +\item{aquifer_code}{Local aquifers in the USGS water resources data base are identified by a geohydrologic unit code (a three-digit number related to the age of the formation, followed by a 4 or 5 character abbreviation for the geologic unit or aquifer name). \href{https://help.waterdata.usgs.gov/faq/groundwater/local-aquifer-description}{Additional information is available at this link.}} + +\item{national_aquifer_code}{National aquifers are the principal aquifers or aquifer systems in the United States, defined as regionally extensive aquifers or aquifer systems that have the potential to be used as a source of potable water. Not all groundwater sites can be associated with a National Aquifer. Such sites will not be retrieved using this search criteria. \href{https://help.waterdata.usgs.gov/code/nat_aqfr_query?fmt=html}{A list of National aquifer codes and names is available.}} + +\item{aquifer_type_code}{Groundwater occurs in aquifers under two different conditions. Where water only partly fills an aquifer, the upper surface is free to rise and decline. These aquifers are referred to as unconfined (or water-table) aquifers. Where water completely fills an aquifer that is overlain by a confining bed, the aquifer is referred to as a confined (or artesian) aquifer. When a confined aquifer is penetrated by a well, the water level in the well will rise above the top of the aquifer (but not necessarily above land surface). \href{https://help.waterdata.usgs.gov/faq/groundwater/local-aquifer-description}{Additional information is available at this link.}} + +\item{well_constructed_depth}{The depth of the finished well, in feet below land surface datum. Note: Not all groundwater sites have information on Well Depth. Such sites will not be retrieved using this search criteria.} + +\item{hole_constructed_depth}{The total depth to which the hole is drilled, in feet below land surface datum. Note: Not all groundwater sites have information on Hole Depth. Such sites will not be retrieved using this search criteria.} + +\item{depth_source_code}{A code indicating the source of water-level data. \href{https://help.waterdata.usgs.gov/code/water_level_src_cd_query?fmt=html}{A list of codes is available.}} + +\item{bbox}{Only features that have a geometry that intersects the bounding +box are selected.The bounding box is provided as four or six numbers, depending +on whether the coordinate reference system includes a vertical axis (height or +depth). Coordinates are assumed to be in crs 4326.} + +\item{limit}{The optional limit parameter limits the number of items that are +presented in the response document. Only items are counted that are on the +first level of the collection in the response document. Nested objects +contained within the explicitly requested items shall not be counted.} + +\item{crs}{Indicates the coordinate reference system for the results.} + +\item{skipGeometry}{This option can be used to skip response geometries for +each feature. The returning object will be a data frame with no spatial +information.} + +\item{convertType}{logical, defaults to \code{TRUE}. If \code{TRUE}, the function +will convert the data to dates and qualifier to string vector.} + +\item{properties}{The properties that should be included for each feature. +The parameter value is a comma-separated list of property names. Available options are +geometry, id, agency_code, agency_name, monitoring_location_number, monitoring_location_name, district_code, country_code, country_name, state_code, state_name, county_code, county_name, minor_civil_division_code, site_type_code, site_type, hydrologic_unit_code, basin_code, altitude, altitude_accuracy, altitude_method_code, altitude_method_name, vertical_datum, vertical_datum_name, horizontal_positional_accuracy_code, horizontal_positional_accuracy, horizontal_position_method_code, horizontal_position_method_name, original_horizontal_datum, original_horizontal_datum_name, drainage_area, contributing_drainage_area, time_zone_abbreviation, uses_daylight_savings, construction_date, aquifer_code, national_aquifer_code, aquifer_type_code, well_constructed_depth, hole_constructed_depth, depth_source_code} +} +\description{ +Description Daily data provide one data value to represent water conditions for the day. Throughout much of the history of the USGS, the primary water data available was daily data collected manually at the monitoring location once each day. With improved availability of computer storage and automated transmission of data, the daily data published today are generally a statistical summary or metric of the continuous data collected each day, such as the daily mean, minimum, or maximum value. Daily data are automatically calculated from the continuous data of the same parameter code and are described by parameter code and a statistic code. These data have also been referred to as “daily values” or “DV”. +} +\examples{ +\dontshow{if (is_dataRetrieval_user()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} + +\donttest{ +site <- "USGS-02238500" +site_info <- read_USGS_sites(monitoring_location_id = site) + +site_info_no_sf <- read_USGS_sites(monitoring_location_id = site, + skipGeometry = TRUE) + +multi_site <- read_USGS_sites(state_name = "Wisconsin") + +} +\dontshow{\}) # examplesIf} +} diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 452c58cd..189582b4 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -1,5 +1,5 @@ --- -title: "Introducing read_USGS_dv" +title: "Introduction to New USGS Services" author: Laura A. DeCicco editor_options: chunk_output_type: console @@ -8,7 +8,7 @@ output: toc: true number_sections: true vignette: > - %\VignetteIndexEntry{Introducing read_USGS_dv} + %\VignetteIndexEntry{Introduction to New USGS Services} \usepackage[utf8]{inputenc} %\VignetteEngine{knitr::rmarkdown} --- @@ -105,8 +105,6 @@ To access these services on a web browser, go to From 7bf35c02525881e7b9f2e9c1d5b2faf742d76df6 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 8 May 2025 15:52:46 -0500 Subject: [PATCH 020/117] with updates --- man/read_USGS_sites.Rd | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/man/read_USGS_sites.Rd b/man/read_USGS_sites.Rd index 7dd0438f..2bec27a5 100644 --- a/man/read_USGS_sites.Rd +++ b/man/read_USGS_sites.Rd @@ -45,6 +45,7 @@ read_USGS_sites( well_constructed_depth = NA_character_, hole_constructed_depth = NA_character_, depth_source_code = NA_character_, + properties = NA_character_, bbox = NA, limit = 10000, crs = NA_character_, @@ -71,6 +72,12 @@ read_USGS_sites( \item{state_code}{State code. A \href{https://www2.census.gov/geo/docs/reference/state.txt}{two-digit ANSI code} (formerly FIPS code) as defined by the American National Standards Institute, to define States and equivalents. A three-digit ANSI code is used to define counties and county equivalents. \href{https://www.census.gov/library/reference/code-lists/ansi.html#states}{A lookup table is available.} The only countries with political subdivisions other than the US are Mexico and Canada. The Mexican states have US state codes ranging from 81-86 and Canadian provinces have state codes ranging from 90-98.} +\item{state_name}{The name of the state or state equivalent in which the site is located.} + +\item{county_code}{The code for the county or county equivalent (parish, borough, etc.) in which the site is located. \href{https://help.waterdata.usgs.gov/code/county_query?fmt=html}{A list of codes is available.}} + +\item{county_name}{The name of the county or county equivalent (parish, borough, etc.) in which the site is located. \href{https://help.waterdata.usgs.gov/code/county_query?fmt=html}{A list of codes is available.}} + \item{minor_civil_division_code}{Codes for primary governmental or administrative divisions of the county or county equivalent in which the site is located.} \item{site_type_code}{A code describing the hydrologic setting of the site. \href{https://help.waterdata.usgs.gov/code/site_tp_query?fmt=html}{A list of codes is available.}} @@ -127,6 +134,10 @@ read_USGS_sites( \item{depth_source_code}{A code indicating the source of water-level data. \href{https://help.waterdata.usgs.gov/code/water_level_src_cd_query?fmt=html}{A list of codes is available.}} +\item{properties}{The properties that should be included for each feature. +The parameter value is a comma-separated list of property names. Available options are +geometry, id, agency_code, agency_name, monitoring_location_number, monitoring_location_name, district_code, country_code, country_name, state_code, state_name, county_code, county_name, minor_civil_division_code, site_type_code, site_type, hydrologic_unit_code, basin_code, altitude, altitude_accuracy, altitude_method_code, altitude_method_name, vertical_datum, vertical_datum_name, horizontal_positional_accuracy_code, horizontal_positional_accuracy, horizontal_position_method_code, horizontal_position_method_name, original_horizontal_datum, original_horizontal_datum_name, drainage_area, contributing_drainage_area, time_zone_abbreviation, uses_daylight_savings, construction_date, aquifer_code, national_aquifer_code, aquifer_type_code, well_constructed_depth, hole_constructed_depth, depth_source_code} + \item{bbox}{Only features that have a geometry that intersects the bounding box are selected.The bounding box is provided as four or six numbers, depending on whether the coordinate reference system includes a vertical axis (height or @@ -145,10 +156,6 @@ information.} \item{convertType}{logical, defaults to \code{TRUE}. If \code{TRUE}, the function will convert the data to dates and qualifier to string vector.} - -\item{properties}{The properties that should be included for each feature. -The parameter value is a comma-separated list of property names. Available options are -geometry, id, agency_code, agency_name, monitoring_location_number, monitoring_location_name, district_code, country_code, country_name, state_code, state_name, county_code, county_name, minor_civil_division_code, site_type_code, site_type, hydrologic_unit_code, basin_code, altitude, altitude_accuracy, altitude_method_code, altitude_method_name, vertical_datum, vertical_datum_name, horizontal_positional_accuracy_code, horizontal_positional_accuracy, horizontal_position_method_code, horizontal_position_method_name, original_horizontal_datum, original_horizontal_datum_name, drainage_area, contributing_drainage_area, time_zone_abbreviation, uses_daylight_savings, construction_date, aquifer_code, national_aquifer_code, aquifer_type_code, well_constructed_depth, hole_constructed_depth, depth_source_code} } \description{ Description Daily data provide one data value to represent water conditions for the day. Throughout much of the history of the USGS, the primary water data available was daily data collected manually at the monitoring location once each day. With improved availability of computer storage and automated transmission of data, the daily data published today are generally a statistical summary or metric of the continuous data collected each day, such as the daily mean, minimum, or maximum value. Daily data are automatically calculated from the continuous data of the same parameter code and are described by parameter code and a statistic code. These data have also been referred to as “daily values” or “DV”. From c5e6c14ae5f8439bee43d7d282942306c8d81fd9 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 8 May 2025 15:57:21 -0500 Subject: [PATCH 021/117] Started news --- NEWS | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/NEWS b/NEWS index 624a63fd..1e3be500 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,9 @@ +dataRetrieval 2.7.19 +=================== +* Added read_USGS_dv, read_USGS_sites, read_USGS_ts_meta to access +new USGS web services. +* Added whisker and sf as dependencies. + dataRetrieval 2.7.18 =================== * Switched from httr to httr2 From 4e3d663ece25c62feb271e7dca8a172472a16d66 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 9 May 2025 09:20:48 -0500 Subject: [PATCH 022/117] Update to monitoring-locations --- R/construct_api_requests.R | 7 +++ R/read_USGS_dv.R | 23 +++------ R/read_USGS_sites.R | 100 +++++++++++++++++++------------------ R/read_USGS_ts_meta.R | 42 +++++----------- R/walk_pages.R | 11 ++++ man/read_USGS_dv.Rd | 9 ++-- man/read_USGS_sites.Rd | 57 ++++++++++----------- man/read_USGS_ts_meta.Rd | 18 ++++--- 8 files changed, 134 insertions(+), 133 deletions(-) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index 4efb8a07..696cdadb 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -66,6 +66,13 @@ construct_api_requests <- function(service, if(all(all_properties[!all_properties %in% c("id", "geometry")] %in% properties)) { # Cleans up URL if we're asking for everything properties <- NA_character_ + } else { + if(all(!is.na(properties))){ + properties <- gsub("-", "_", properties) + properties <- properties[!properties %in% c("id", + "geometry", + paste0(gsub("-", "_", service), "_id"))] + } } baseURL <- setup_api(service) diff --git a/R/read_USGS_dv.R b/R/read_USGS_dv.R index 7c8afaf0..5b1dfeec 100644 --- a/R/read_USGS_dv.R +++ b/R/read_USGS_dv.R @@ -14,7 +14,7 @@ #' @param last_modified `r get_params("daily")$last_modified` #' @param time_series_id `r get_params("daily")$time_series_id` #' @param qualifier `r get_params("daily")$qualifier` -#' @param id `r get_params("daily")$id` +#' @param daily_id `r get_params("daily")$id` #' @param properties The properties that should be included for each feature. #' The parameter value is a comma-separated list of property names. Available options are #' `r schema <- check_OGC_requests(endpoint = "daily", type = "schema"); paste(names(schema$properties), collapse = ", ")` @@ -61,17 +61,10 @@ read_USGS_dv <- function(monitoring_location_id = NA_character_, parameter_code = NA_character_, statistic_id = NA_character_, - properties = c("monitoring_location_id", - "parameter_code", - "statistic_id", - "time", - "value", - "unit_of_measure", - "approval_status", - "qualifier"), + properties = NA_character_, bbox = NA, time_series_id = NA_character_, - id = NA_character_, + daily_id = NA_character_, approval_status = NA_character_, unit_of_measure = NA_character_, qualifier = NA_character_, @@ -85,14 +78,16 @@ read_USGS_dv <- function(monitoring_location_id = NA_character_, message("Function in development, use at your own risk.") - dv_req <- construct_api_requests(service = "daily", + service <- "daily" + + dv_req <- construct_api_requests(service = service, monitoring_location_id = monitoring_location_id, parameter_code = parameter_code, statistic_id = statistic_id, properties = properties, bbox = bbox, time_series_id = time_series_id, - id = id, + id = daily_id, approval_status = approval_status, unit_of_measure = unit_of_measure, qualifier = qualifier, @@ -109,9 +104,7 @@ read_USGS_dv <- function(monitoring_location_id = NA_character_, return_list <- return_list[order(return_list$time, return_list$monitoring_location_id), ] - if(!"id" %in% properties){ - return_list <- return_list[, names(return_list)[!names(return_list) %in% "id"]] - } + return_list <- rejigger_cols(return_list, properties, service) return(return_list) } diff --git a/R/read_USGS_sites.R b/R/read_USGS_sites.R index c0a36aea..b11881cd 100644 --- a/R/read_USGS_sites.R +++ b/R/read_USGS_sites.R @@ -1,51 +1,51 @@ #' Get USGS Daily Data #' -#' Description `r get_description("daily")` +#' Description `r get_description("monitoring-locations")` #' #' @export -#' @param monitoring_location_id `r get_params("sites")$id` -#' @param agency_code `r get_params("sites")$agency_code` -#' @param agency_name `r get_params("sites")$agency_name` -#' @param monitoring_location_number `r get_params("sites")$monitoring_location_number` -#' @param monitoring_location_name `r get_params("sites")$monitoring_location_name` -#' @param district_code `r get_params("sites")$district_code` -#' @param state_name `r get_params("sites")$state_name` -#' @param county_code `r get_params("sites")$county_code` -#' @param county_name `r get_params("sites")$county_name` -#' @param country_code `r get_params("sites")$country_code` -#' @param country_name `r get_params("sites")$country_name` -#' @param state_code `r get_params("sites")$state_code` -#' @param minor_civil_division_code `r get_params("sites")$minor_civil_division_code` -#' @param site_type_code `r get_params("sites")$site_type_code` -#' @param site_type `r get_params("sites")$site_type` -#' @param hydrologic_unit_code `r get_params("sites")$hydrologic_unit_code` -#' @param basin_code `r get_params("sites")$basin_code` -#' @param altitude `r get_params("sites")$altitude` -#' @param altitude_accuracy `r get_params("sites")$altitude_accuracy` -#' @param altitude_method_code `r get_params("sites")$altitude_method_code` -#' @param altitude_method_name `r get_params("sites")$altitude_method_name` -#' @param vertical_datum `r get_params("sites")$vertical_datum` -#' @param vertical_datum_name `r get_params("sites")$vertical_datum_name` -#' @param horizontal_positional_accuracy_code `r get_params("sites")$horizontal_positional_accuracy_code` -#' @param horizontal_positional_accuracy `r get_params("sites")$horizontal_positional_accuracy` -#' @param horizontal_position_method_code `r get_params("sites")$horizontal_position_method_code` -#' @param horizontal_position_method_name `r get_params("sites")$horizontal_position_method_name` -#' @param original_horizontal_datum `r get_params("sites")$original_horizontal_datum` -#' @param original_horizontal_datum_name `r get_params("sites")$original_horizontal_datum_name` -#' @param drainage_area `r get_params("sites")$drainage_area` -#' @param contributing_drainage_area `r get_params("sites")$contributing_drainage_area` -#' @param time_zone_abbreviation `r get_params("sites")$time_zone_abbreviation` -#' @param uses_daylight_savings `r get_params("sites")$uses_daylight_savings` -#' @param construction_date `r get_params("sites")$construction_date` -#' @param aquifer_code `r get_params("sites")$aquifer_code` -#' @param national_aquifer_code `r get_params("sites")$national_aquifer_code` -#' @param aquifer_type_code `r get_params("sites")$aquifer_type_code` -#' @param well_constructed_depth `r get_params("sites")$well_constructed_depth` -#' @param hole_constructed_depth `r get_params("sites")$hole_constructed_depth` -#' @param depth_source_code `r get_params("sites")$depth_source_code` +#' @param monitoring_location_id `r get_params("monitoring-locations")$id` +#' @param agency_code `r get_params("monitoring-locations")$agency_code` +#' @param agency_name `r get_params("monitoring-locations")$agency_name` +#' @param monitoring_location_number `r get_params("monitoring-locations")$monitoring_location_number` +#' @param monitoring_location_name `r get_params("monitoring-locations")$monitoring_location_name` +#' @param district_code `r get_params("monitoring-locations")$district_code` +#' @param state_name `r get_params("monitoring-locations")$state_name` +#' @param county_code `r get_params("monitoring-locations")$county_code` +#' @param county_name `r get_params("monitoring-locations")$county_name` +#' @param country_code `r get_params("monitoring-locations")$country_code` +#' @param country_name `r get_params("monitoring-locations")$country_name` +#' @param state_code `r get_params("monitoring-locations")$state_code` +#' @param minor_civil_division_code `r get_params("monitoring-locations")$minor_civil_division_code` +#' @param site_type_code `r get_params("monitoring-locations")$site_type_code` +#' @param site_type `r get_params("monitoring-locations")$site_type` +#' @param hydrologic_unit_code `r get_params("monitoring-locations")$hydrologic_unit_code` +#' @param basin_code `r get_params("monitoring-locations")$basin_code` +#' @param altitude `r get_params("monitoring-locations")$altitude` +#' @param altitude_accuracy `r get_params("monitoring-locations")$altitude_accuracy` +#' @param altitude_method_code `r get_params("monitoring-locations")$altitude_method_code` +#' @param altitude_method_name `r get_params("monitoring-locations")$altitude_method_name` +#' @param vertical_datum `r get_params("monitoring-locations")$vertical_datum` +#' @param vertical_datum_name `r get_params("monitoring-locations")$vertical_datum_name` +#' @param horizontal_positional_accuracy_code `r get_params("monitoring-locations")$horizontal_positional_accuracy_code` +#' @param horizontal_positional_accuracy `r get_params("monitoring-locations")$horizontal_positional_accuracy` +#' @param horizontal_position_method_code `r get_params("monitoring-locations")$horizontal_position_method_code` +#' @param horizontal_position_method_name `r get_params("monitoring-locations")$horizontal_position_method_name` +#' @param original_horizontal_datum `r get_params("monitoring-locations")$original_horizontal_datum` +#' @param original_horizontal_datum_name `r get_params("monitoring-locations")$original_horizontal_datum_name` +#' @param drainage_area `r get_params("monitoring-locations")$drainage_area` +#' @param contributing_drainage_area `r get_params("monitoring-locations")$contributing_drainage_area` +#' @param time_zone_abbreviation `r get_params("monitoring-locations")$time_zone_abbreviation` +#' @param uses_daylight_savings `r get_params("monitoring-locations")$uses_daylight_savings` +#' @param construction_date `r get_params("monitoring-locations")$construction_date` +#' @param aquifer_code `r get_params("monitoring-locations")$aquifer_code` +#' @param national_aquifer_code `r get_params("monitoring-locations")$national_aquifer_code` +#' @param aquifer_type_code `r get_params("monitoring-locations")$aquifer_type_code` +#' @param well_constructed_depth `r get_params("monitoring-locations")$well_constructed_depth` +#' @param hole_constructed_depth `r get_params("monitoring-locations")$hole_constructed_depth` +#' @param depth_source_code `r get_params("monitoring-locations")$depth_source_code` #' @param properties The properties that should be included for each feature. #' The parameter value is a comma-separated list of property names. Available options are -#' `r schema <- check_OGC_requests(endpoint = "sites", type = "schema"); paste(names(schema$properties), collapse = ", ")` +#' `r schema <- check_OGC_requests(endpoint = "monitoring-locations", type = "schema"); paste(names(schema$properties), collapse = ", ")` #' @param bbox Only features that have a geometry that intersects the bounding #' box are selected.The bounding box is provided as four or six numbers, depending #' on whether the coordinate reference system includes a vertical axis (height or @@ -58,13 +58,16 @@ #' @param skipGeometry This option can be used to skip response geometries for #' each feature. The returning object will be a data frame with no spatial #' information. -#' @param convertType logical, defaults to `TRUE`. If `TRUE`, the function -#' will convert the data to dates and qualifier to string vector. #' @examplesIf is_dataRetrieval_user() #' #' \donttest{ #' site <- "USGS-02238500" #' site_info <- read_USGS_sites(monitoring_location_id = site) +#' +#' site_slim <- read_USGS_sites(monitoring_location_id = site, +#' properties = c("monitoring_locations_id", +#' "state_name", +#' "country_name")) #' #' site_info_no_sf <- read_USGS_sites(monitoring_location_id = site, #' skipGeometry = TRUE) @@ -116,12 +119,13 @@ read_USGS_sites <- function(monitoring_location_id = NA_character_, bbox = NA, limit = 10000, crs = NA_character_, - skipGeometry = NA, - convertType = TRUE){ + skipGeometry = NA){ message("Function in development, use at your own risk.") - site_req <- construct_api_requests(service = "sites", + service <- "monitoring-locations" + + site_req <- construct_api_requests(service = service, id = monitoring_location_id, agency_code = agency_code, agency_name = agency_name, @@ -170,7 +174,7 @@ read_USGS_sites <- function(monitoring_location_id = NA_character_, return_list <- walk_pages(site_req) - if(convertType) return_list <- cleanup_cols(return_list) + return_list <- rejigger_cols(return_list, properties, service) return(return_list) } diff --git a/R/read_USGS_ts_meta.R b/R/read_USGS_ts_meta.R index c7868d17..97166127 100644 --- a/R/read_USGS_ts_meta.R +++ b/R/read_USGS_ts_meta.R @@ -20,7 +20,7 @@ #' @param properties The properties that should be included for each feature. #' The parameter value is a comma-separated list of property names. Available options are #' `r schema <- check_OGC_requests(endpoint = "time-series-metadata", type = "schema"); paste(names(schema$properties), collapse = ", ")` -#' @param id `r get_params("time-series-metadata")$id` +#' @param time_series_id `r get_params("time-series-metadata")$id` #' @param bbox Only features that have a geometry that intersects the bounding #' box are selected.The bounding box is provided as four or six numbers, depending #' on whether the coordinate reference system includes a vertical axis (height or @@ -43,24 +43,16 @@ #' #' meta_multi <- read_USGS_ts_meta(monitoring_location_id = c("USGS-01491000", #' "USGS-01645000"), -#' parameter_code = c("00060", "00010")) +#' parameter_code = c("00060", "00010"), +#' properties = c("monitoring_location_id", +#' "parameter_code", +#' "begin", +#' "end")) #' } read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, parameter_code = NA_character_, parameter_name = NA_character_, - properties = c("monitoring_location_id", - "unit_of_measure", - "parameter_code", - "parameter_name", - "statistic_id", - "computation_identifier", - "begin", - "end", - "primary", - "sublocation_identifier", - "id", - "last_modified", - "web_description"), + properties = NA_character_, limit = 10000, bbox = NA, statistic_id = NA_character_, @@ -73,7 +65,7 @@ read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, thresholds = NA, sublocation_identifier = NA_character_, primary = NA_character_, - id = NA_character_, + time_series_id = NA_character_, web_description = NA_character_, crs = NA_character_, skipGeometry = NA, @@ -81,13 +73,9 @@ read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, message("Function in development, use at your own risk.") - use_sf <- all(pkg.env$local_sf) + service = "time-series-metadata" - if(!use_sf){ - skipGeometry <- TRUE - } - - req_ts_meta <- construct_api_requests("time-series-metadata", + req_ts_meta <- construct_api_requests(service, monitoring_location_id = monitoring_location_id, parameter_code = parameter_code, parameter_name = parameter_name, @@ -103,20 +91,16 @@ read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, thresholds = thresholds, sublocation_identifier = sublocation_identifier, primary = primary, - id = id, + id = time_series_id, crs = crs, web_description = web_description, skipGeometry = skipGeometry) return_list <- walk_pages(req_ts_meta) - - return_list <- return_list[, properties] - + if(convertType) return_list <- cleanup_cols(return_list) - if(!"id" %in% properties){ - return_list <- return_list[, names(return_list)[!names(return_list) %in% "id"]] - } + return_list <- rejigger_cols(return_list, properties, service) return(return_list) diff --git a/R/walk_pages.R b/R/walk_pages.R index 1dd04fe3..7a605622 100644 --- a/R/walk_pages.R +++ b/R/walk_pages.R @@ -1,3 +1,14 @@ +rejigger_cols <- function(df, properties, service){ + new_id <- paste0(gsub("-", "_", service), "_id") + names(df)[names(df) == "id"] <- new_id + + if(!all(is.na(properties))){ + properties[properties == "id"] <- new_id + df <- df[, properties] + } + df +} + cleanup_cols <- function(df){ if("qualifier" %in% names(df)){ diff --git a/man/read_USGS_dv.Rd b/man/read_USGS_dv.Rd index 26a9b294..f31105ad 100644 --- a/man/read_USGS_dv.Rd +++ b/man/read_USGS_dv.Rd @@ -8,11 +8,10 @@ read_USGS_dv( monitoring_location_id = NA_character_, parameter_code = NA_character_, statistic_id = NA_character_, - properties = c("monitoring_location_id", "parameter_code", "statistic_id", "time", - "value", "unit_of_measure", "approval_status", "qualifier"), + properties = NA_character_, bbox = NA, time_series_id = NA_character_, - id = NA_character_, + daily_id = NA_character_, approval_status = NA_character_, unit_of_measure = NA_character_, qualifier = NA_character_, @@ -26,7 +25,7 @@ read_USGS_dv( ) } \arguments{ -\item{monitoring_location_id}{A unique identifier representing a single monitoring location. This corresponds to the \code{id} field in the \code{sites} endpoint. Monitoring location IDs are created by combining the agency code of the agency responsible for the monitoring location (e.g. USGS) with the ID number of the monitoring location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500).} +\item{monitoring_location_id}{A unique identifier representing a single monitoring location. This corresponds to the \code{id} field in the \code{monitoring-locations} endpoint. Monitoring location IDs are created by combining the agency code of the agency responsible for the monitoring location (e.g. USGS) with the ID number of the monitoring location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500).} \item{parameter_code}{Parameter codes are 5-digit codes used to identify the constituent measured and the units of measure. A complete list of parameter codes and associated groupings can be found at \url{https://help.waterdata.usgs.gov/codes-and-parameters/parameters}.} @@ -43,7 +42,7 @@ depth). Coordinates are assumed to be in crs 4326.} \item{time_series_id}{A unique identifier representing a single time series. This corresponds to the \code{id} field in the \code{time-series-metadata} endpoint.} -\item{id}{A universally unique identifier (UUID) representing a single version of a record. It is not stable over time. Every time the record is refreshed in our database (which may happen as part of normal operations and does not imply any change to the data itself) a new ID will be generated. To uniquely identify a single observation over time, compare the \code{time} and \code{time_series_id} fields; each time series will only have a single observation at a given \code{time}.} +\item{daily_id}{A universally unique identifier (UUID) representing a single version of a record. It is not stable over time. Every time the record is refreshed in our database (which may happen as part of normal operations and does not imply any change to the data itself) a new ID will be generated. To uniquely identify a single observation over time, compare the \code{time} and \code{time_series_id} fields; each time series will only have a single observation at a given \code{time}.} \item{approval_status}{Some of the data that you have obtained from this U.S. Geological Survey database may not have received Director's approval. Any such data values are qualified as provisional and are subject to revision. Provisional data are released on the condition that neither the USGS nor the United States Government may be held liable for any damages resulting from its use. This field reflects the approval status of each record, and is either "Approved", meaining processing review has been completed and the data is approved for publication, or "Provisional" and subject to revision. For more information about provisional data, go to \url{https://waterdata.usgs.gov/provisional-data-statement/}.} diff --git a/man/read_USGS_sites.Rd b/man/read_USGS_sites.Rd index 2bec27a5..d92bae70 100644 --- a/man/read_USGS_sites.Rd +++ b/man/read_USGS_sites.Rd @@ -49,46 +49,45 @@ read_USGS_sites( bbox = NA, limit = 10000, crs = NA_character_, - skipGeometry = NA, - convertType = TRUE + skipGeometry = NA ) } \arguments{ -\item{monitoring_location_id}{A unique identifier representing a single monitoring location. This corresponds to the \code{id} field in the \code{sites} endpoint. Monitoring location IDs are created by combining the agency code of the agency responsible for the monitoring location (e.g. USGS) with the ID number of the monitoring location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500).} +\item{monitoring_location_id}{A unique identifier representing a single monitoring location. This corresponds to the \code{id} field in the \code{monitoring-locations} endpoint. Monitoring location IDs are created by combining the agency code of the agency responsible for the monitoring location (e.g. USGS) with the ID number of the monitoring location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500).} \item{agency_code}{The agency that is reporting the data. Agency codes are fixed values assigned by the National Water Information System (NWIS). A list of agency codes is available \href{https://help.waterdata.usgs.gov/code/agency_cd_query?fmt=html}{at this link}.} \item{agency_name}{The name of the agency that is reporting the data.} -\item{monitoring_location_number}{Each site in the USGS data base has a unique 8- to 15-digit identification number. Site numbers are assigned \href{https://help.waterdata.usgs.gov/faq/sites/do-station-numbers-have-any-particular-meaning}{based on this logic}.} +\item{monitoring_location_number}{Each monitoring location in the USGS data base has a unique 8- to 15-digit identification number. Monitoring location numbers are assigned \href{https://help.waterdata.usgs.gov/faq/sites/do-station-numbers-have-any-particular-meaning}{based on this logic}.} -\item{monitoring_location_name}{This is the official name of the site in the database. For well information this can be a district-assigned local number.} +\item{monitoring_location_name}{This is the official name of the monitoring location in the database. For well information this can be a district-assigned local number.} -\item{district_code}{The Water Science Centers (WSCs) across the United States use the FIPS state code as the district code. In some case, sites and samples may be managed by a water science center that is adjacent to the state in which the site actually resides. For example a site may have a district code of 30 which translates to Montana, but the state code could be 56 for Wyoming because that is where the site actually is located.} +\item{district_code}{The Water Science Centers (WSCs) across the United States use the FIPS state code as the district code. In some case, monitoring locations and samples may be managed by a water science center that is adjacent to the state in which the monitoring location actually resides. For example a monitoring location may have a district code of 30 which translates to Montana, but the state code could be 56 for Wyoming because that is where the monitoring location actually is located.} -\item{country_code}{The code for the country in which the site is located.} +\item{country_code}{The code for the country in which the monitoring location is located.} -\item{country_name}{The name of the country in which the site is located.} +\item{country_name}{The name of the country in which the monitoring location is located.} \item{state_code}{State code. A \href{https://www2.census.gov/geo/docs/reference/state.txt}{two-digit ANSI code} (formerly FIPS code) as defined by the American National Standards Institute, to define States and equivalents. A three-digit ANSI code is used to define counties and county equivalents. \href{https://www.census.gov/library/reference/code-lists/ansi.html#states}{A lookup table is available.} The only countries with political subdivisions other than the US are Mexico and Canada. The Mexican states have US state codes ranging from 81-86 and Canadian provinces have state codes ranging from 90-98.} -\item{state_name}{The name of the state or state equivalent in which the site is located.} +\item{state_name}{The name of the state or state equivalent in which the monitoring location is located.} -\item{county_code}{The code for the county or county equivalent (parish, borough, etc.) in which the site is located. \href{https://help.waterdata.usgs.gov/code/county_query?fmt=html}{A list of codes is available.}} +\item{county_code}{The code for the county or county equivalent (parish, borough, etc.) in which the monitoring location is located. \href{https://help.waterdata.usgs.gov/code/county_query?fmt=html}{A list of codes is available.}} -\item{county_name}{The name of the county or county equivalent (parish, borough, etc.) in which the site is located. \href{https://help.waterdata.usgs.gov/code/county_query?fmt=html}{A list of codes is available.}} +\item{county_name}{The name of the county or county equivalent (parish, borough, etc.) in which the monitoring location is located. \href{https://help.waterdata.usgs.gov/code/county_query?fmt=html}{A list of codes is available.}} -\item{minor_civil_division_code}{Codes for primary governmental or administrative divisions of the county or county equivalent in which the site is located.} +\item{minor_civil_division_code}{Codes for primary governmental or administrative divisions of the county or county equivalent in which the monitoring location is located.} -\item{site_type_code}{A code describing the hydrologic setting of the site. \href{https://help.waterdata.usgs.gov/code/site_tp_query?fmt=html}{A list of codes is available.}} +\item{site_type_code}{A code describing the hydrologic setting of the monitoring location. \href{https://help.waterdata.usgs.gov/code/site_tp_query?fmt=html}{A list of codes is available.}} -\item{site_type}{A description of the hydrologic setting of the site. \href{https://help.waterdata.usgs.gov/code/site_tp_query?fmt=html}{A list of codes is available.}} +\item{site_type}{A description of the hydrologic setting of the monitoring location. \href{https://help.waterdata.usgs.gov/code/site_tp_query?fmt=html}{A list of codes is available.}} \item{hydrologic_unit_code}{The United States is divided and sub-divided into successively smaller hydrologic units which are classified into four levels: regions, sub-regions, accounting units, and cataloging units. The hydrologic units are arranged within each other, from the smallest (cataloging units) to the largest (regions). Each hydrologic unit is identified by a unique hydrologic unit code (HUC) consisting of two to eight digits based on the four levels of classification in the hydrologic unit system.} -\item{basin_code}{The Basin Code or "drainage basin code" is a two-digit code that further subdivides the 8-digit hydrologic-unit code. The drainage basin code is defined by the USGS State Office where the site is located.} +\item{basin_code}{The Basin Code or "drainage basin code" is a two-digit code that further subdivides the 8-digit hydrologic-unit code. The drainage basin code is defined by the USGS State Office where the monitoring location is located.} -\item{altitude}{Altitude of the site referenced to the specified Vertical Datum.} +\item{altitude}{Altitude of the monitoring location referenced to the specified Vertical Datum.} \item{altitude_accuracy}{Accuracy of the altitude, in feet. An accuracy of +/- 0.1 foot would be entered as “.1”. Many altitudes are interpolated from the contours on topographic maps; accuracies determined in this way are generally entered as one-half of the contour interval.} @@ -96,9 +95,9 @@ read_USGS_sites( \item{altitude_method_name}{The name of the the method used to measure altitude. \href{https://help.waterdata.usgs.gov/code/alt_meth_cd_query?fmt=html}{A list of codes is available.}} -\item{vertical_datum}{The datum used to determine altitude and vertical position at the site. \href{https://help.waterdata.usgs.gov/code/alt_datum_cd_query?fmt=html}{A list of codes is available.}} +\item{vertical_datum}{The datum used to determine altitude and vertical position at the monitoring location. \href{https://help.waterdata.usgs.gov/code/alt_datum_cd_query?fmt=html}{A list of codes is available.}} -\item{vertical_datum_name}{The datum used to determine altitude and vertical position at the site. \href{https://help.waterdata.usgs.gov/code/alt_datum_cd_query?fmt=html}{A list of codes is available.}} +\item{vertical_datum_name}{The datum used to determine altitude and vertical position at the monitoring location. \href{https://help.waterdata.usgs.gov/code/alt_datum_cd_query?fmt=html}{A list of codes is available.}} \item{horizontal_positional_accuracy_code}{Indicates the accuracy of the latitude longitude values. \href{https://help.waterdata.usgs.gov/code/coord_acy_cd_query?fmt=html}{A list of codes is available.}} @@ -114,23 +113,23 @@ read_USGS_sites( \item{drainage_area}{The area enclosed by a topographic divide from which direct surface runoff from precipitation normally drains by gravity into the stream above that point.} -\item{contributing_drainage_area}{The contributing drainage area of a lake, stream, wetland, or estuary site, in square miles. This item should be present only if the contributing area is different from the total drainage area. This situation can occur when part of the drainage area consists of very porous soil or depressions that either allow all runoff to enter the groundwater or traps the water in ponds so that rainfall does not contribute to runoff. A transbasin diversion can also affect the total drainage area.} +\item{contributing_drainage_area}{The contributing drainage area of a lake, stream, wetland, or estuary monitoring location, in square miles. This item should be present only if the contributing area is different from the total drainage area. This situation can occur when part of the drainage area consists of very porous soil or depressions that either allow all runoff to enter the groundwater or traps the water in ponds so that rainfall does not contribute to runoff. A transbasin diversion can also affect the total drainage area.} -\item{time_zone_abbreviation}{A short code describing the time zone used by a site.} +\item{time_zone_abbreviation}{A short code describing the time zone used by a monitoring location.} -\item{uses_daylight_savings}{A flag indicating whether or not a site uses daylight savings.} +\item{uses_daylight_savings}{A flag indicating whether or not a monitoring location uses daylight savings.} \item{construction_date}{Date the well was completed.} \item{aquifer_code}{Local aquifers in the USGS water resources data base are identified by a geohydrologic unit code (a three-digit number related to the age of the formation, followed by a 4 or 5 character abbreviation for the geologic unit or aquifer name). \href{https://help.waterdata.usgs.gov/faq/groundwater/local-aquifer-description}{Additional information is available at this link.}} -\item{national_aquifer_code}{National aquifers are the principal aquifers or aquifer systems in the United States, defined as regionally extensive aquifers or aquifer systems that have the potential to be used as a source of potable water. Not all groundwater sites can be associated with a National Aquifer. Such sites will not be retrieved using this search criteria. \href{https://help.waterdata.usgs.gov/code/nat_aqfr_query?fmt=html}{A list of National aquifer codes and names is available.}} +\item{national_aquifer_code}{National aquifers are the principal aquifers or aquifer systems in the United States, defined as regionally extensive aquifers or aquifer systems that have the potential to be used as a source of potable water. Not all groundwater monitoring locations can be associated with a National Aquifer. Such monitoring locations will not be retrieved using this search criteria. \href{https://help.waterdata.usgs.gov/code/nat_aqfr_query?fmt=html}{A list of National aquifer codes and names is available.}} \item{aquifer_type_code}{Groundwater occurs in aquifers under two different conditions. Where water only partly fills an aquifer, the upper surface is free to rise and decline. These aquifers are referred to as unconfined (or water-table) aquifers. Where water completely fills an aquifer that is overlain by a confining bed, the aquifer is referred to as a confined (or artesian) aquifer. When a confined aquifer is penetrated by a well, the water level in the well will rise above the top of the aquifer (but not necessarily above land surface). \href{https://help.waterdata.usgs.gov/faq/groundwater/local-aquifer-description}{Additional information is available at this link.}} -\item{well_constructed_depth}{The depth of the finished well, in feet below land surface datum. Note: Not all groundwater sites have information on Well Depth. Such sites will not be retrieved using this search criteria.} +\item{well_constructed_depth}{The depth of the finished well, in feet below land surface datum. Note: Not all groundwater monitoring locations have information on Well Depth. Such monitoring locations will not be retrieved using this search criteria.} -\item{hole_constructed_depth}{The total depth to which the hole is drilled, in feet below land surface datum. Note: Not all groundwater sites have information on Hole Depth. Such sites will not be retrieved using this search criteria.} +\item{hole_constructed_depth}{The total depth to which the hole is drilled, in feet below land surface datum. Note: Not all groundwater monitoring locations have information on Hole Depth. Such monitoring locations will not be retrieved using this search criteria.} \item{depth_source_code}{A code indicating the source of water-level data. \href{https://help.waterdata.usgs.gov/code/water_level_src_cd_query?fmt=html}{A list of codes is available.}} @@ -153,12 +152,9 @@ contained within the explicitly requested items shall not be counted.} \item{skipGeometry}{This option can be used to skip response geometries for each feature. The returning object will be a data frame with no spatial information.} - -\item{convertType}{logical, defaults to \code{TRUE}. If \code{TRUE}, the function -will convert the data to dates and qualifier to string vector.} } \description{ -Description Daily data provide one data value to represent water conditions for the day. Throughout much of the history of the USGS, the primary water data available was daily data collected manually at the monitoring location once each day. With improved availability of computer storage and automated transmission of data, the daily data published today are generally a statistical summary or metric of the continuous data collected each day, such as the daily mean, minimum, or maximum value. Daily data are automatically calculated from the continuous data of the same parameter code and are described by parameter code and a statistic code. These data have also been referred to as “daily values” or “DV”. +Description Location information is basic information about the monitoring location including the name, identifier, agency responsible for data collection, and the date the location was established. It also includes information about the type of location, such as stream, lake, or groundwater, and geographic information about the location, such as state, county, latitude and longitude, and hydrologic unit code (HUC). } \examples{ \dontshow{if (is_dataRetrieval_user()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} @@ -167,6 +163,11 @@ Description Daily data provide one data value to represent water conditions for site <- "USGS-02238500" site_info <- read_USGS_sites(monitoring_location_id = site) +site_slim <- read_USGS_sites(monitoring_location_id = site, + properties = c("monitoring_locations_id", + "state_name", + "country_name")) + site_info_no_sf <- read_USGS_sites(monitoring_location_id = site, skipGeometry = TRUE) diff --git a/man/read_USGS_ts_meta.Rd b/man/read_USGS_ts_meta.Rd index 2bf69b3e..3dfd9023 100644 --- a/man/read_USGS_ts_meta.Rd +++ b/man/read_USGS_ts_meta.Rd @@ -8,9 +8,7 @@ read_USGS_ts_meta( monitoring_location_id = NA_character_, parameter_code = NA_character_, parameter_name = NA_character_, - properties = c("monitoring_location_id", "unit_of_measure", "parameter_code", - "parameter_name", "statistic_id", "computation_identifier", "begin", "end", - "primary", "sublocation_identifier", "id", "last_modified", "web_description"), + properties = NA_character_, limit = 10000, bbox = NA, statistic_id = NA_character_, @@ -23,7 +21,7 @@ read_USGS_ts_meta( thresholds = NA, sublocation_identifier = NA_character_, primary = NA_character_, - id = NA_character_, + time_series_id = NA_character_, web_description = NA_character_, crs = NA_character_, skipGeometry = NA, @@ -31,7 +29,7 @@ read_USGS_ts_meta( ) } \arguments{ -\item{monitoring_location_id}{A unique identifier representing a single monitoring location. This corresponds to the \code{id} field in the \code{sites} endpoint. Monitoring location IDs are created by combining the agency code of the agency responsible for the monitoring location (e.g. USGS) with the ID number of the monitoring location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500).} +\item{monitoring_location_id}{A unique identifier representing a single monitoring location. This corresponds to the \code{id} field in the \code{monitoring-locations} endpoint. Monitoring location IDs are created by combining the agency code of the agency responsible for the monitoring location (e.g. USGS) with the ID number of the monitoring location (e.g. 02238500), separated by a hyphen (e.g. USGS-02238500).} \item{parameter_code}{Parameter codes are 5-digit codes used to identify the constituent measured and the units of measure. A complete list of parameter codes and associated groupings can be found at \url{https://help.waterdata.usgs.gov/codes-and-parameters/parameters}.} @@ -81,7 +79,7 @@ Only features that have a \code{last_modified} that intersects the value of date \item{primary}{A flag identifying if the time series is a "primary" time series. "Primary" time series (which have this flag) are standard observations which undergo \href{https://www.usgs.gov/survey-manual/5028-fundamental-science-practices-review-and-approval-scientific-data-release}{Bureau review and approval processes}. Non-primary time series, which will have missing values for "primary", are provisional datasets made available to meet the need for timely best science and to assist with daily operations which need real-time information. Non-primary time series data are only retained by this system for 120 days. See the \href{https://waterdata.usgs.gov/provisional-data-statement/}{USGS Provisional Data Statement} for more information.} -\item{id}{A unique identifier representing a single time series. This corresponds to the \code{id} field in the \code{time-series-metadata} endpoint.} +\item{time_series_id}{A unique identifier representing a single time series. This corresponds to the \code{id} field in the \code{time-series-metadata} endpoint.} \item{web_description}{A description of what this time series represents, as used by WDFN and other USGS data dissemination products.} @@ -95,7 +93,7 @@ information.} will convert the data to dates and qualifier to string vector.} } \description{ -Description Daily data and continuous measurements are grouped into time series, which represent a collection of observations of a single parameter, potentially aggregated using a standard statistic, at a single site. This endpoint provides metadata about those time series, including their operational thresholds, units of measurement, and when the earliest and most recent observations in a time series occurred. +Description Daily data and continuous measurements are grouped into time series, which represent a collection of observations of a single parameter, potentially aggregated using a standard statistic, at a single monitoring location. This endpoint provides metadata about those time series, including their operational thresholds, units of measurement, and when the earliest and most recent observations in a time series occurred. } \examples{ \dontshow{if (is_dataRetrieval_user()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} @@ -106,7 +104,11 @@ meta_1 <- read_USGS_ts_meta(monitoring_location_id = site) meta_multi <- read_USGS_ts_meta(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), - parameter_code = c("00060", "00010")) + parameter_code = c("00060", "00010"), + properties = c("monitoring_location_id", + "parameter_code", + "begin", + "end")) } \dontshow{\}) # examplesIf} } From ec1784de19a933366f743c3d8efa4ed42f50bf53 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 9 May 2025 09:46:42 -0500 Subject: [PATCH 023/117] hide datetime --- R/read_USGS_dv.R | 13 ++++++------- man/read_USGS_dv.Rd | 21 +++++---------------- 2 files changed, 11 insertions(+), 23 deletions(-) diff --git a/R/read_USGS_dv.R b/R/read_USGS_dv.R index 5b1dfeec..292be1f3 100644 --- a/R/read_USGS_dv.R +++ b/R/read_USGS_dv.R @@ -5,7 +5,6 @@ #' @export #' @param monitoring_location_id `r get_params("daily")$monitoring_location_id` #' @param parameter_code `r get_params("daily")$parameter_code` -#' @param datetime `r get_params("daily")$time` #' @param statistic_id `r get_params("daily")$statistic_id` #' @param time `r get_params("daily")$time` #' @param value `r get_params("daily")$value` @@ -39,14 +38,14 @@ #' pcode <- "00060" #' dv_data_sf <- read_USGS_dv(monitoring_location_id = site, #' parameter_code = "00060", -#' datetime = c("2021-01-01", "2022-01-01")) +#' time = c("2021-01-01", "2022-01-01")) #' #' dv_data_trim <- read_USGS_dv(monitoring_location_id = site, #' parameter_code = "00060", #' properties = c("monitoring_location_id", #' "value", #' "time"), -#' datetime = c("2021-01-01", "2022-01-01")) +#' time = c("2021-01-01", "2022-01-01")) #' #' dv_data <- read_USGS_dv(monitoring_location_id = site, #' parameter_code = "00060", @@ -55,7 +54,7 @@ #' multi_site <- read_USGS_dv(monitoring_location_id = c("USGS-01491000", #' "USGS-01645000"), #' parameter_code = c("00060", "00010"), -#' datetime = c("2023-01-01", "2024-01-01")) +#' time = c("2023-01-01", "2024-01-01")) #' #' } read_USGS_dv <- function(monitoring_location_id = NA_character_, @@ -73,7 +72,7 @@ read_USGS_dv <- function(monitoring_location_id = NA_character_, limit = 10000, crs = NA_character_, skipGeometry = NA, - datetime = NA_character_, + time = NA_character_, convertType = TRUE){ message("Function in development, use at your own risk.") @@ -92,11 +91,11 @@ read_USGS_dv <- function(monitoring_location_id = NA_character_, unit_of_measure = unit_of_measure, qualifier = qualifier, value = value, + time = time, last_modified = last_modified, limit = limit, crs = crs, - skipGeometry = skipGeometry, - datetime = datetime) + skipGeometry = skipGeometry) return_list <- walk_pages(dv_req) diff --git a/man/read_USGS_dv.Rd b/man/read_USGS_dv.Rd index f31105ad..cf93ae75 100644 --- a/man/read_USGS_dv.Rd +++ b/man/read_USGS_dv.Rd @@ -20,7 +20,7 @@ read_USGS_dv( limit = 10000, crs = NA_character_, skipGeometry = NA, - datetime = NA_character_, + time = NA_character_, convertType = TRUE ) } @@ -75,7 +75,7 @@ contained within the explicitly requested items shall not be counted.} each feature. The returning object will be a data frame with no spatial information.} -\item{datetime}{The date an observation represents. You can query this field using date-times or intervals, adhering to RFC 3339, or using ISO 8601 duration objects. Intervals may be bounded or half-bounded (double-dots at start or end). +\item{time}{The date an observation represents. You can query this field using date-times or intervals, adhering to RFC 3339, or using ISO 8601 duration objects. Intervals may be bounded or half-bounded (double-dots at start or end). Examples: \itemize{ \item A date-time: "2018-02-12T23:20:50Z" @@ -88,17 +88,6 @@ Only features that have a \code{time} that intersects the value of datetime are \item{convertType}{logical, defaults to \code{TRUE}. If \code{TRUE}, the function will convert the data to dates and qualifier to string vector.} - -\item{time}{The date an observation represents. You can query this field using date-times or intervals, adhering to RFC 3339, or using ISO 8601 duration objects. Intervals may be bounded or half-bounded (double-dots at start or end). -Examples: -\itemize{ -\item A date-time: "2018-02-12T23:20:50Z" -\item A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" -\item Half-bounded intervals: "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" -\item Duration objects: "P1M" for data from the past month or "PT36H" for the last 36 hours -} - -Only features that have a \code{time} that intersects the value of datetime are selected. If a feature has multiple temporal properties, it is the decision of the server whether only a single temporal property is used to determine the extent or all relevant temporal properties.} } \description{ Description Daily data provide one data value to represent water conditions for the day. Throughout much of the history of the USGS, the primary water data available was daily data collected manually at the monitoring location once each day. With improved availability of computer storage and automated transmission of data, the daily data published today are generally a statistical summary or metric of the continuous data collected each day, such as the daily mean, minimum, or maximum value. Daily data are automatically calculated from the continuous data of the same parameter code and are described by parameter code and a statistic code. These data have also been referred to as “daily values” or “DV”. @@ -111,14 +100,14 @@ site <- "USGS-02238500" pcode <- "00060" dv_data_sf <- read_USGS_dv(monitoring_location_id = site, parameter_code = "00060", - datetime = c("2021-01-01", "2022-01-01")) + time = c("2021-01-01", "2022-01-01")) dv_data_trim <- read_USGS_dv(monitoring_location_id = site, parameter_code = "00060", properties = c("monitoring_location_id", "value", "time"), - datetime = c("2021-01-01", "2022-01-01")) + time = c("2021-01-01", "2022-01-01")) dv_data <- read_USGS_dv(monitoring_location_id = site, parameter_code = "00060", @@ -127,7 +116,7 @@ dv_data <- read_USGS_dv(monitoring_location_id = site, multi_site <- read_USGS_dv(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), parameter_code = c("00060", "00010"), - datetime = c("2023-01-01", "2024-01-01")) + time = c("2023-01-01", "2024-01-01")) } \dontshow{\}) # examplesIf} From b85cb3633ab2239cb60268d300b0e7517241d96c Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 9 May 2025 09:48:18 -0500 Subject: [PATCH 024/117] update to vignette --- vignettes/read_USGS_functions.Rmd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 189582b4..583a55e8 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -58,7 +58,7 @@ nrow(dv_legacy) dv_modern <- read_USGS_dv(monitoring_location_id = "USGS-01491000", parameter_code = c("00060", "00010"), statistic_id = "00003", - datetime = c("2023-01-01", "2025-05-06")) + time = c("2023-01-01", "2025-05-06")) nrow(dv_modern) ``` @@ -88,7 +88,7 @@ dv_2 <- read_USGS_dv(monitoring_location_id = "USGS-01491000", "value", "time"), statistic_id = "00003", - datetime = c("2022-10-01", "2023-10-01")) + time = c("2022-10-01", "2023-10-01")) head(dv_2) ``` From c71e3111bf88e434b512b240b52902e587a4f4f1 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 9 May 2025 10:38:44 -0500 Subject: [PATCH 025/117] cleanup pages? --- .gitlab-ci.yml | 2 ++ R/read_USGS_sites.R | 2 +- R/read_USGS_ts_meta.R | 3 ++- man/read_USGS_sites.Rd | 2 +- man/read_USGS_ts_meta.Rd | 3 ++- pkgdown/extra.css | 34 +++++++++++++++++++--------------- 6 files changed, 27 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9112a7f2..2609354d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -98,6 +98,8 @@ pages: cache: [] script: - Rscript -e 'pkgdown::build_site(override = list(destination = "public"))' + - Rscript -e 'file.copy(from = "./public/articles/logo.png", to = "./public/reference/logo.png") +' artifacts: paths: - $PAGES_OUTDIR diff --git a/R/read_USGS_sites.R b/R/read_USGS_sites.R index b11881cd..b7fe6645 100644 --- a/R/read_USGS_sites.R +++ b/R/read_USGS_sites.R @@ -1,4 +1,4 @@ -#' Get USGS Daily Data +#' Get USGS Site Data #' #' Description `r get_description("monitoring-locations")` #' diff --git a/R/read_USGS_ts_meta.R b/R/read_USGS_ts_meta.R index 97166127..60fffa30 100644 --- a/R/read_USGS_ts_meta.R +++ b/R/read_USGS_ts_meta.R @@ -47,7 +47,8 @@ #' properties = c("monitoring_location_id", #' "parameter_code", #' "begin", -#' "end")) +#' "end"), +#' skipGeometry = TRUE) #' } read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, parameter_code = NA_character_, diff --git a/man/read_USGS_sites.Rd b/man/read_USGS_sites.Rd index d92bae70..20d94691 100644 --- a/man/read_USGS_sites.Rd +++ b/man/read_USGS_sites.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/read_USGS_sites.R \name{read_USGS_sites} \alias{read_USGS_sites} -\title{Get USGS Daily Data} +\title{Get USGS Site Data} \usage{ read_USGS_sites( monitoring_location_id = NA_character_, diff --git a/man/read_USGS_ts_meta.Rd b/man/read_USGS_ts_meta.Rd index 3dfd9023..34a4e0a0 100644 --- a/man/read_USGS_ts_meta.Rd +++ b/man/read_USGS_ts_meta.Rd @@ -108,7 +108,8 @@ meta_multi <- read_USGS_ts_meta(monitoring_location_id = c("USGS-01491000", properties = c("monitoring_location_id", "parameter_code", "begin", - "end")) + "end"), + skipGeometry = TRUE) } \dontshow{\}) # examplesIf} } diff --git a/pkgdown/extra.css b/pkgdown/extra.css index 52e2c5da..fb14c3a9 100644 --- a/pkgdown/extra.css +++ b/pkgdown/extra.css @@ -1,27 +1,27 @@ -/* ================INSTRUCTIONS=================*/ +/* ================INSTRUCTIONS=================*/ /* By changing the info below, you can reduce the size of the logo or hide the search box. You can also override the standard font characteristics if you would like to use your own custom styles. In order for your changes to work though, you MUST include a reference in your HTML pages to both the common CSS file and custom CSS file in that order. Instructions are provided below for customizing these classes. */ -/* =============Width===============*/ +/* =============Width===============*/ /* BY DEFAULT, THERE IS NO MAX WIDTH: If you want the want to restrict the width of the page, remove the comment out slashes and astricks surrounding the ".tmp-container {max-width: 1170px;}". you can change the 1170px to a smaller/larger max width if you'd like */ - + /* .tmp-container { max-width: 1170px; } */ -/* =============Search===============*/ +/* =============Search===============*/ /* BY DEFAULT, Search box is displayed: If you want the to hide the search, remove the comment out slashes and astricks surrounding the ".search-box {display:none;}" below. */ - + /* #search-box { display: none; } */ -/* =============LOGO===============*/ +/* =============LOGO===============*/ /* THE DEFAULT LOGO HEIGHT IS 65PX: If you want the logo to be smaller (50px), comment out the ".logo-header img {height 65px;}" below and remove the comment out slashes and astricks surrounding the ".logo-header img {height: 50px...margin-top: 18px;}" and the header search input (so the search box size is reduced too). 50px is the MINIMUM HEIGHT for the logo. */ .logo-header img { @@ -30,7 +30,7 @@ If you want the logo to be smaller (50px), comment out the ".logo-header img {he /* .logo-header img { height: 50px; -} +} .header-search input[type="search"] { height: 30px; margin-top: 16px; @@ -42,7 +42,7 @@ If you want the logo to be smaller (50px), comment out the ".logo-header img {he } */ -/* =============STANDARD CONTENT===============*/ +/* =============STANDARD CONTENT===============*/ /* TO CHANGE THE TEXT SIZE OF THE CONTENT, FONT, ETC: By default, USGS has set the font size, family, etc. in order to provide a consistent size for content across all pages. If you would prefer not to have any of these pre-defined formats, you can change them below. NOTE: header and footer will not be changed. */ #maincontent { @@ -55,19 +55,19 @@ By default, USGS has set the font size, family, etc. in order to provide a consi padding-right: 15px; } -/* =============SEARCH===============*/ +/* =============SEARCH===============*/ /* THIS HIDES THE SEARCH BOX ON VERY SMALL DEVICES: For simplification, search bar is visible on larger screens but is hidden on small screens. If you would prefer not to have the search box at all, you can remove the "@media (max-width:500px) {" and the second closing "}". below */ @media (max-width:500px) { -.header-search form { +.header-search form { display: none} } -/* =============SOCIAL MEDIA===============*/ +/* =============SOCIAL MEDIA===============*/ /* If you would prefer not to have the social media links, you can remove the comment out slashes and astricks surrounding the content below */ - /* .footer-social-links { + /* .footer-social-links { display: none} */ - + @charset "UTF-8"; /* CSS Document */ @@ -81,7 +81,7 @@ footer, #navbar { -webkit-box-sizing: border-box; -moz-box-sizing: border-box; box-sizing: border-box; -} +} footer, header, main, nav, div { display: block; @@ -109,7 +109,7 @@ footer, header, main, nav, div { } hr { - width: 100%; + width: 100%; margin-top: 42px; clear: both; } @@ -352,6 +352,10 @@ Content: "\f082"; color: white !important; } +.text-muted { + color: white !important; +} + .navbar-nav a:hover { text-decoration: underline; color: white !important; From af255afe485c470a765f305107d6760c59feef08 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 9 May 2025 10:55:22 -0500 Subject: [PATCH 026/117] wayward quote --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2609354d..381de0e5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -99,7 +99,6 @@ pages: script: - Rscript -e 'pkgdown::build_site(override = list(destination = "public"))' - Rscript -e 'file.copy(from = "./public/articles/logo.png", to = "./public/reference/logo.png") -' artifacts: paths: - $PAGES_OUTDIR From 9c131ef9e0280f03a779ed6d918f0637220ff3c9 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 9 May 2025 11:15:47 -0500 Subject: [PATCH 027/117] it wasn't a wayward quote...it was a wayward end-of-line --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 381de0e5..76402775 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -98,7 +98,7 @@ pages: cache: [] script: - Rscript -e 'pkgdown::build_site(override = list(destination = "public"))' - - Rscript -e 'file.copy(from = "./public/articles/logo.png", to = "./public/reference/logo.png") + - Rscript -e 'file.copy(from = "./public/articles/logo.png", to = "./public/reference/logo.png")' artifacts: paths: - $PAGES_OUTDIR From 64701466dff9eb58600b626956cff01cbec17f3d Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 16 May 2025 13:43:38 -0500 Subject: [PATCH 028/117] rename --- NAMESPACE | 4 ++-- NEWS | 2 +- R/construct_api_requests.R | 2 +- R/readNWISdv.R | 4 ++++ R/readNWISsite.R | 5 +++++ R/{read_USGS_dv.R => read_USGS_daily.R} | 10 +++++----- ...S_sites.R => read_USGS_monitoring_location.R} | 10 +++++----- R/read_USGS_ts_meta.R | 2 +- man/{read_USGS_dv.Rd => read_USGS_daily.Rd} | 16 ++++++++-------- ...sites.Rd => read_USGS_monitoring_location.Rd} | 16 ++++++++-------- man/read_USGS_ts_meta.Rd | 2 +- vignettes/read_USGS_functions.Rmd | 12 ++++++------ 12 files changed, 47 insertions(+), 38 deletions(-) rename R/{read_USGS_dv.R => read_USGS_daily.R} (93%) rename R/{read_USGS_sites.R => read_USGS_monitoring_location.R} (96%) rename man/{read_USGS_dv.Rd => read_USGS_daily.Rd} (95%) rename man/{read_USGS_sites.Rd => read_USGS_monitoring_location.Rd} (96%) diff --git a/NAMESPACE b/NAMESPACE index 64296e5a..9145e4cc 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -44,9 +44,9 @@ export(readNWISuv) export(readWQPdata) export(readWQPqw) export(readWQPsummary) -export(read_USGS_dv) +export(read_USGS_daily) +export(read_USGS_monitoring_location) export(read_USGS_samples) -export(read_USGS_sites) export(read_USGS_ts_meta) export(renameNWISColumns) export(setAccess) diff --git a/NEWS b/NEWS index 1e3be500..c8981de2 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,6 @@ dataRetrieval 2.7.19 =================== -* Added read_USGS_dv, read_USGS_sites, read_USGS_ts_meta to access +* Added read_USGS_daily, read_USGS_monitoring_location, read_USGS_ts_meta to access new USGS web services. * Added whisker and sf as dependencies. diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index 696cdadb..fe25fe86 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -276,7 +276,7 @@ get_description <- function(service){ } -# Create descriptions dynamically +# Create parameter descriptions dynamically get_params <- function(service){ check_queryables_req <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> diff --git a/R/readNWISdv.R b/R/readNWISdv.R index 4c5ac90d..592d2691 100644 --- a/R/readNWISdv.R +++ b/R/readNWISdv.R @@ -83,6 +83,10 @@ readNWISdv <- function(siteNumbers, endDate = "", statCd = "00003") { + .Deprecated(new = "read_USGS_daily", + package = "dataRetrieval", + msg = "NWIS servers are slated for decommission. Please begin to migrate to read_USGS_daily.") + url <- constructNWISURL( siteNumbers = siteNumbers, parameterCd = parameterCd, diff --git a/R/readNWISsite.R b/R/readNWISsite.R index c1b110f9..d3fa4de1 100644 --- a/R/readNWISsite.R +++ b/R/readNWISsite.R @@ -67,6 +67,11 @@ #' } readNWISsite <- function(siteNumbers) { + .Deprecated(new = "read_USGS_monitoring_location", + package = "dataRetrieval", + msg = "NWIS servers are slated for decommission. Please begin to migrate to read_USGS_monitoring_location") + + baseURL <- httr2::request(pkg.env[["site"]]) urlSitefile <- httr2::req_url_query(baseURL, siteOutput = "Expanded", diff --git a/R/read_USGS_dv.R b/R/read_USGS_daily.R similarity index 93% rename from R/read_USGS_dv.R rename to R/read_USGS_daily.R index 292be1f3..c6fe142c 100644 --- a/R/read_USGS_dv.R +++ b/R/read_USGS_daily.R @@ -36,28 +36,28 @@ #' \donttest{ #' site <- "USGS-02238500" #' pcode <- "00060" -#' dv_data_sf <- read_USGS_dv(monitoring_location_id = site, +#' dv_data_sf <- read_USGS_daily(monitoring_location_id = site, #' parameter_code = "00060", #' time = c("2021-01-01", "2022-01-01")) #' -#' dv_data_trim <- read_USGS_dv(monitoring_location_id = site, +#' dv_data_trim <- read_USGS_daily(monitoring_location_id = site, #' parameter_code = "00060", #' properties = c("monitoring_location_id", #' "value", #' "time"), #' time = c("2021-01-01", "2022-01-01")) #' -#' dv_data <- read_USGS_dv(monitoring_location_id = site, +#' dv_data <- read_USGS_daily(monitoring_location_id = site, #' parameter_code = "00060", #' skipGeometry = TRUE) #' -#' multi_site <- read_USGS_dv(monitoring_location_id = c("USGS-01491000", +#' multi_site <- read_USGS_daily(monitoring_location_id = c("USGS-01491000", #' "USGS-01645000"), #' parameter_code = c("00060", "00010"), #' time = c("2023-01-01", "2024-01-01")) #' #' } -read_USGS_dv <- function(monitoring_location_id = NA_character_, +read_USGS_daily <- function(monitoring_location_id = NA_character_, parameter_code = NA_character_, statistic_id = NA_character_, properties = NA_character_, diff --git a/R/read_USGS_sites.R b/R/read_USGS_monitoring_location.R similarity index 96% rename from R/read_USGS_sites.R rename to R/read_USGS_monitoring_location.R index b7fe6645..1f280e7f 100644 --- a/R/read_USGS_sites.R +++ b/R/read_USGS_monitoring_location.R @@ -62,20 +62,20 @@ #' #' \donttest{ #' site <- "USGS-02238500" -#' site_info <- read_USGS_sites(monitoring_location_id = site) +#' site_info <- read_USGS_monitoring_location(monitoring_location_id = site) #' -#' site_slim <- read_USGS_sites(monitoring_location_id = site, +#' site_slim <- read_USGS_monitoring_location(monitoring_location_id = site, #' properties = c("monitoring_locations_id", #' "state_name", #' "country_name")) #' -#' site_info_no_sf <- read_USGS_sites(monitoring_location_id = site, +#' site_info_no_sf <- read_USGS_monitoring_location(monitoring_location_id = site, #' skipGeometry = TRUE) #' -#' multi_site <- read_USGS_sites(state_name = "Wisconsin") +#' multi_site <- read_USGS_monitoring_location(state_name = "Wisconsin") #' #' } -read_USGS_sites <- function(monitoring_location_id = NA_character_, +read_USGS_monitoring_location <- function(monitoring_location_id = NA_character_, agency_code = NA_character_, agency_name = NA_character_, monitoring_location_number = NA_character_, diff --git a/R/read_USGS_ts_meta.R b/R/read_USGS_ts_meta.R index 60fffa30..a36ac44a 100644 --- a/R/read_USGS_ts_meta.R +++ b/R/read_USGS_ts_meta.R @@ -1,4 +1,4 @@ -#' Get USGS Timeseries metadata +#' Get USGS Time Series Metadata #' #' Description `r get_description("time-series-metadata")` #' diff --git a/man/read_USGS_dv.Rd b/man/read_USGS_daily.Rd similarity index 95% rename from man/read_USGS_dv.Rd rename to man/read_USGS_daily.Rd index cf93ae75..44b087e8 100644 --- a/man/read_USGS_dv.Rd +++ b/man/read_USGS_daily.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_USGS_dv.R -\name{read_USGS_dv} -\alias{read_USGS_dv} +% Please edit documentation in R/read_USGS_daily.R +\name{read_USGS_daily} +\alias{read_USGS_daily} \title{Get USGS Daily Data} \usage{ -read_USGS_dv( +read_USGS_daily( monitoring_location_id = NA_character_, parameter_code = NA_character_, statistic_id = NA_character_, @@ -98,22 +98,22 @@ Description Daily data provide one data value to represent water conditions for \donttest{ site <- "USGS-02238500" pcode <- "00060" -dv_data_sf <- read_USGS_dv(monitoring_location_id = site, +dv_data_sf <- read_USGS_daily(monitoring_location_id = site, parameter_code = "00060", time = c("2021-01-01", "2022-01-01")) -dv_data_trim <- read_USGS_dv(monitoring_location_id = site, +dv_data_trim <- read_USGS_daily(monitoring_location_id = site, parameter_code = "00060", properties = c("monitoring_location_id", "value", "time"), time = c("2021-01-01", "2022-01-01")) -dv_data <- read_USGS_dv(monitoring_location_id = site, +dv_data <- read_USGS_daily(monitoring_location_id = site, parameter_code = "00060", skipGeometry = TRUE) -multi_site <- read_USGS_dv(monitoring_location_id = c("USGS-01491000", +multi_site <- read_USGS_daily(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), parameter_code = c("00060", "00010"), time = c("2023-01-01", "2024-01-01")) diff --git a/man/read_USGS_sites.Rd b/man/read_USGS_monitoring_location.Rd similarity index 96% rename from man/read_USGS_sites.Rd rename to man/read_USGS_monitoring_location.Rd index 20d94691..0e6975c1 100644 --- a/man/read_USGS_sites.Rd +++ b/man/read_USGS_monitoring_location.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_USGS_sites.R -\name{read_USGS_sites} -\alias{read_USGS_sites} +% Please edit documentation in R/read_USGS_monitoring_location.R +\name{read_USGS_monitoring_location} +\alias{read_USGS_monitoring_location} \title{Get USGS Site Data} \usage{ -read_USGS_sites( +read_USGS_monitoring_location( monitoring_location_id = NA_character_, agency_code = NA_character_, agency_name = NA_character_, @@ -161,17 +161,17 @@ Description Location information is basic information about the monitoring locat \donttest{ site <- "USGS-02238500" -site_info <- read_USGS_sites(monitoring_location_id = site) +site_info <- read_USGS_monitoring_location(monitoring_location_id = site) -site_slim <- read_USGS_sites(monitoring_location_id = site, +site_slim <- read_USGS_monitoring_location(monitoring_location_id = site, properties = c("monitoring_locations_id", "state_name", "country_name")) -site_info_no_sf <- read_USGS_sites(monitoring_location_id = site, +site_info_no_sf <- read_USGS_monitoring_location(monitoring_location_id = site, skipGeometry = TRUE) -multi_site <- read_USGS_sites(state_name = "Wisconsin") +multi_site <- read_USGS_monitoring_location(state_name = "Wisconsin") } \dontshow{\}) # examplesIf} diff --git a/man/read_USGS_ts_meta.Rd b/man/read_USGS_ts_meta.Rd index 34a4e0a0..a71bf55d 100644 --- a/man/read_USGS_ts_meta.Rd +++ b/man/read_USGS_ts_meta.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/read_USGS_ts_meta.R \name{read_USGS_ts_meta} \alias{read_USGS_ts_meta} -\title{Get USGS Timeseries metadata} +\title{Get USGS Time Series Metadata} \usage{ read_USGS_ts_meta( monitoring_location_id = NA_character_, diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 583a55e8..26da9cfd 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -42,7 +42,7 @@ This is a modern access point for USGS data. The USGS is planning to modernize a To access these services on a web browser, go to . -The `read_USGS_dv` function will eventually replace the `readNWISdv` function. Generally, both functions look pretty similar: +The `read_USGS_daily` function will eventually replace the `readNWISdv` function. Generally, both functions look pretty similar: ```{r} library(dataRetrieval) @@ -55,7 +55,7 @@ dv_legacy <- readNWISdv(siteNumbers = "01491000", nrow(dv_legacy) -dv_modern <- read_USGS_dv(monitoring_location_id = "USGS-01491000", +dv_modern <- read_USGS_daily(monitoring_location_id = "USGS-01491000", parameter_code = c("00060", "00010"), statistic_id = "00003", time = c("2023-01-01", "2025-05-06")) @@ -82,7 +82,7 @@ ggplot(data = dv_modern) + * Flexible column selection: ```{r} -dv_2 <- read_USGS_dv(monitoring_location_id = "USGS-01491000", +dv_2 <- read_USGS_daily(monitoring_location_id = "USGS-01491000", parameter_code = c("00060"), properties = c("monitoring_location_id", "value", @@ -92,7 +92,7 @@ dv_2 <- read_USGS_dv(monitoring_location_id = "USGS-01491000", head(dv_2) ``` -* When you look at the help file for the new function `?read_USGS_dv`, you'll also notice there are many more arguments. These are mostly set by default to NA. You DO NOT need to specify all of these parameters. +* When you look at the help file for the new function `?read_USGS_daily`, you'll also notice there are many more arguments. These are mostly set by default to NA. You DO NOT need to specify all of these parameters. * Flexible data/last_modified @@ -140,7 +140,7 @@ knitr::kable(data_modern[,c("parameter_code", # Site Information -The `read_USGS_sites` function will eventually replace the `readNWISsite` function. Generally, both functions look pretty similar: +The `read_USGS_monitoring_location` function will eventually replace the `readNWISsite` function. Generally, both functions look pretty similar: ```{r} sites_legacy <- readNWISsite(siteNumbers = "01491000") @@ -148,7 +148,7 @@ names(sites_legacy) ``` ```{r} -sites_modern <- read_USGS_sites(monitoring_location_id = "USGS-01491000") +sites_modern <- read_USGS_monitoring_location(monitoring_location_id = "USGS-01491000") names(sites_modern) ``` From 6b30cfff6a5cc925be69b7ef5902862788d5c88e Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 20 May 2025 08:42:50 -0500 Subject: [PATCH 029/117] Update _pkgdown.yml Co-authored-by: Joe Zemmels (he/him) --- _pkgdown.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_pkgdown.yml b/_pkgdown.yml index 7dc91fd1..4274568f 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -70,7 +70,7 @@ reference: - read_USGS_samples - read_USGS_dv - read_USGS_ts_meta - - read_USGS_sites + - read_USGS_monitoring_location - summarize_USGS_samples - check_USGS_sample_params - title: National Water Information System (NWIS) From 6ccb817561c4529031d529a0a4e2f43084867b75 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 20 May 2025 08:42:58 -0500 Subject: [PATCH 030/117] Update _pkgdown.yml Co-authored-by: Joe Zemmels (he/him) --- _pkgdown.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_pkgdown.yml b/_pkgdown.yml index 4274568f..c915c93e 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -68,7 +68,7 @@ reference: desc: Functions to retrieve USGS data from new services. contents: - read_USGS_samples - - read_USGS_dv + - read_USGS_daily - read_USGS_ts_meta - read_USGS_monitoring_location - summarize_USGS_samples From ced59594233a9a4e8fea299b0fbe714d5f8ce1c6 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 20 May 2025 09:06:33 -0500 Subject: [PATCH 031/117] docs match code --- man/read_USGS_samples.Rd | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/man/read_USGS_samples.Rd b/man/read_USGS_samples.Rd index 9cd3fd32..2695a24b 100644 --- a/man/read_USGS_samples.Rd +++ b/man/read_USGS_samples.Rd @@ -27,8 +27,7 @@ read_USGS_samples( pointLocationWithinMiles = NA, dataType = "results", dataProfile = NA, - tz = "UTC", - convertType = TRUE + tz = "UTC" ) } \arguments{ @@ -132,10 +131,6 @@ and "count". Options for "projects" are: Possible values include "America/New_York","America/Chicago", "America/Denver","America/Los_Angeles", "America/Anchorage","America/Honolulu","America/Jamaica","America/Managua", "America/Phoenix", and "America/Metlakatla"} - -\item{convertType}{logical, defaults to \code{TRUE}. If \code{TRUE}, the function -will convert the data to dates, datetimes, -numerics based on a standard algorithm. If false, everything is returned as a character.} } \description{ This function creates the call and gets the data for discrete water quality samples data From 05f031d9287ef06f305ddc33054bf12e858b46eb Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 20 May 2025 16:30:19 -0500 Subject: [PATCH 032/117] put basic_request in all queries --- R/construct_api_requests.R | 52 +++++++++++++++++-------------- vignettes/read_USGS_functions.Rmd | 2 +- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index fe25fe86..596cca47 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -140,21 +140,10 @@ construct_api_requests <- function(service, } setup_api <- function(service){ + baseURL <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> httr2::req_url_path_append(service, "items") |> - httr2::req_user_agent(default_ua()) |> - httr2::req_headers(`Accept-Encoding` = c("compress", "gzip")) - - token <- Sys.getenv("API_USGS_PAT") - - if(token != ""){ - baseURL <- baseURL |> - httr2::req_headers_redacted(`X-Api-Key` = token) - } - - baseURL <- explode_query(baseURL, POST = FALSE, - list(f = "json", - lang = "en-US")) + basic_request() } @@ -233,8 +222,11 @@ check_OGC_requests <- function(endpoint = "daily", check_collections <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/server/getCollections") - check_endpoints <- basic_request(check_collections) - services <- sapply(check_endpoints$tags, function(x) x[["name"]]) + check_endpoints_req <- basic_request(check_collections) + query_ret <- httr2::req_perform(check_endpoints_req) |> + httr2::resp_body_json() + + services <- sapply(query_ret$tags, function(x) x[["name"]]) match.arg(endpoint, services) @@ -242,7 +234,10 @@ check_OGC_requests <- function(endpoint = "daily", httr2::req_url_path_append(endpoint) |> httr2::req_url_path_append(type) - query_ret <- basic_request(url_base) + req <- basic_request(url_base) + + query_ret <- httr2::req_perform(req) |> + httr2::resp_body_json() return(query_ret) @@ -250,13 +245,20 @@ check_OGC_requests <- function(endpoint = "daily", basic_request <- function(url_base){ - check_req <- url_base |> + req <- url_base |> httr2::req_user_agent(default_ua()) |> + httr2::req_headers(`Accept-Encoding` = c("compress", "gzip")) |> httr2::req_url_query(f = "json", lang = "en-US") - query_ret <- httr2::req_perform(check_req) |> - httr2::resp_body_json() + token <- Sys.getenv("API_USGS_PAT") + + if(token != ""){ + req <- req |> + httr2::req_headers_redacted(`X-Api-Key` = token) + } + + return(req) } @@ -265,8 +267,10 @@ get_description <- function(service){ check_collections <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/server/getCollections") - check_endpoints <- basic_request(check_collections) - tags <- check_endpoints[["tags"]] + check_endpoints_req <- basic_request(check_collections) + query_ret <- httr2::req_perform(check_endpoints_req) |> + httr2::resp_body_json() + tags <- query_ret[["tags"]] service_index <- which(sapply(tags, function(x){ x$name == service @@ -283,8 +287,10 @@ get_params <- function(service){ httr2::req_url_path_append(service) |> httr2::req_url_path_append("schema") - check_queryables <- basic_request(check_queryables_req) - params <- sapply(check_queryables$properties, function(x) x[["description"]]) + check_queryables_req <- basic_request(check_queryables_req) + query_ret <- httr2::req_perform(check_queryables_req) |> + httr2::resp_body_json() + params <- sapply(query_ret$properties, function(x) x[["description"]]) } diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 26da9cfd..82b3298c 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -100,7 +100,7 @@ head(dv_2) # Time Series Metadata -To access these services on a web browser, go to . +To access these services on a web browser, go to . The `read_USGS_ts_meta` function will eventually replace the `whatNWISdata` function. Generally, both functions look pretty similar: From 85be645ed5ebc0b1ae867d1dbd611c123ec15d3b Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 21 May 2025 15:10:28 -0500 Subject: [PATCH 033/117] Update R/walk_pages.R Co-authored-by: Joe Zemmels (he/him) --- R/walk_pages.R | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/R/walk_pages.R b/R/walk_pages.R index 7a605622..226158d4 100644 --- a/R/walk_pages.R +++ b/R/walk_pages.R @@ -60,7 +60,9 @@ walk_pages_recursive <- function(req, page, contents) { } header_info <- httr2::resp_headers(returned_contents) - message("Remaining requests this hour:", header_info$`x-ratelimit-remaining`) + if(Sys.getenv("API_USGS_PAT") != ""){ + message("Remaining requests this hour:", header_info$`x-ratelimit-remaining`) + } contents[[page]] <- returned_contents |> httr2::resp_body_string() From 5e4a64edc8e884c9ca945684f25d772d4ce3e33d Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 22 May 2025 13:57:35 -0500 Subject: [PATCH 034/117] Add descriptions --- vignettes/read_USGS_functions.Rmd | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 82b3298c..f84c674e 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -40,9 +40,11 @@ This is a modern access point for USGS data. The USGS is planning to modernize a # Daily Values +`r dataRetrieval:::get_description("daily")` + To access these services on a web browser, go to . -The `read_USGS_daily` function will eventually replace the `readNWISdv` function. Generally, both functions look pretty similar: +The `read_USGS_daily` function will eventually replace the `readNWISdv` function. Generally, both functions look pretty similar, though we've updated argument names to align with new Water Mission Area conventions:: ```{r} library(dataRetrieval) @@ -55,19 +57,19 @@ dv_legacy <- readNWISdv(siteNumbers = "01491000", nrow(dv_legacy) -dv_modern <- read_USGS_daily(monitoring_location_id = "USGS-01491000", +daily_modern <- read_USGS_daily(monitoring_location_id = "USGS-01491000", parameter_code = c("00060", "00010"), statistic_id = "00003", time = c("2023-01-01", "2025-05-06")) -nrow(dv_modern) +nrow(daily_modern) ``` -The first thing you might notice is that the legacy service serves data in a "wide" format, which means there are multiple observations in a single row of the data frame. The new services return the data in a "long" format, so that only one observation is returned per row. This follows tidy principles and is often preferred in scripting languages. We'll look at the returned data with a `ggplot2` plot to show a very simple example of the power of a long format: +The first thing you might notice is that the legacy service serves data in a "wide" format, which means there are multiple observations in a single row of the data frame (see: [Pivot Data](articles/long_to_wide.html)). The new services return the data in a "long" format, so that only one observation is returned per row. This follows tidy principles and is often preferred in scripting languages. We'll look at the returned data with a `ggplot2` plot to show a very simple example of the power of a long format: ```{r} library(ggplot2) -ggplot(data = dv_modern) + +ggplot(data = daily_modern) + geom_point(aes(x = time, y = value, color = approval_status)) + facet_grid(parameter_code ~ ., scale = "free") + @@ -100,6 +102,8 @@ head(dv_2) # Time Series Metadata +`r dataRetrieval:::get_description("time-series-metadata")` + To access these services on a web browser, go to . The `read_USGS_ts_meta` function will eventually replace the `whatNWISdata` function. Generally, both functions look pretty similar: @@ -136,10 +140,10 @@ knitr::kable(data_modern[,c("parameter_code", ``` -## Additional new features - # Site Information +`r dataRetrieval:::get_description("monitoring-locations")` + The `read_USGS_monitoring_location` function will eventually replace the `readNWISsite` function. Generally, both functions look pretty similar: ```{r} From a8f725383f8edbbea5c54bd9427fb807d9f7c357 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 22 May 2025 14:42:55 -0500 Subject: [PATCH 035/117] Fixes from reviewers --- R/construct_api_requests.R | 25 ++++++++----- R/read_USGS_daily.R | 34 ++++++----------- R/read_USGS_monitoring_location.R | 55 +++------------------------- R/read_USGS_ts_meta.R | 33 +++++------------ man/check_OGC_requests.Rd | 2 +- man/construct_api_requests.Rd | 8 +--- man/read_USGS_daily.Rd | 7 ++-- man/read_USGS_monitoring_location.Rd | 3 -- man/read_USGS_ts_meta.Rd | 7 +--- 9 files changed, 51 insertions(+), 123 deletions(-) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index 596cca47..f7a891c6 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -10,11 +10,9 @@ #' box are selected.The bounding box is provided as four or six numbers, depending #' on whether the coordinate reference system includes a vertical axis (height or #' depth). -#' @param crs Indicates the coordinate reference system for the results. #' @param properties The properties that should be included for each feature. The -#' parameter value is a comma-separated list of property names. Available values: -#' id, timeseries_id, monitoring_location_id, parameter_code, statistic_id, time, -#' value, unit_of_measure, approval_status, qualifier, last_modified. +#' parameter value is a comma-separated list of property names which depend on the +#' service being called. #' @param skipGeometry This option can be used to skip response geometries for #' each feature. The returning object will be a data frame with no spatial #' information. @@ -45,7 +43,6 @@ construct_api_requests <- function(service, properties = NA_character_, bbox = NA, - crs = NA_character_, limit = 10000, skipGeometry = FALSE, ...){ @@ -85,9 +82,13 @@ construct_api_requests <- function(service, single_params <- c("datetime", "last_modified", "begin", "end", "time") full_list <- list(...) - get_list <- full_list[names(full_list) %in% single_params] - get_list[["crs"]] <- crs + if(all(is.na(full_list)) & is.na(bbox)){ + warning("No filtering arguments specified.") + } + + get_list <- full_list[names(full_list) %in% single_params] + get_list[["skipGeometry"]] <- skipGeometry get_list[["limit"]] <- limit @@ -99,7 +100,7 @@ construct_api_requests <- function(service, POST = TRUE } - time_periods <- c("last_modified", "datetime", "time") + time_periods <- c("last_modified", "datetime", "time", "begin", "end") if(any(time_periods %in% names(get_list))){ for(i in time_periods){ get_list[[i]] <- format_api_dates(get_list[[i]]) @@ -151,7 +152,11 @@ format_api_dates <- function(datetime){ if(!any(isTRUE(is.na(datetime)) | isTRUE(is.null(datetime)))){ if(length(datetime) == 1){ - datetime <- format(datetime, format = "%Y-%m-%dT%H:%M:%SZ") + if(grepl("P", datetime, ignore.case = TRUE)){ + return(datetime) + } else { + datetime <- format(datetime, format = "%Y-%m-%dT%H:%M:%SZ") + } } else if (length(datetime) == 2) { datetime <- as.POSIXct(datetime) datetime <- paste0(vapply(datetime, FUN = function(x) { @@ -197,7 +202,7 @@ cql2_param <- function(parameter){ #' Check OGC requests #' -#' @param endpoint Character, can be "daily", "timeseries-metadata" +#' @param endpoint Character, can be any existing collection #' @param type Character, can be "queryables", "schema" #' @export #' @keywords internal diff --git a/R/read_USGS_daily.R b/R/read_USGS_daily.R index c6fe142c..780e0673 100644 --- a/R/read_USGS_daily.R +++ b/R/read_USGS_daily.R @@ -21,7 +21,6 @@ #' box are selected.The bounding box is provided as four or six numbers, depending #' on whether the coordinate reference system includes a vertical axis (height or #' depth). Coordinates are assumed to be in crs 4326. -#' @param crs Indicates the coordinate reference system for the results. #' @param limit The optional limit parameter limits the number of items that are #' presented in the response document. Only items are counted that are on the #' first level of the collection in the response document. Nested objects @@ -50,6 +49,10 @@ #' dv_data <- read_USGS_daily(monitoring_location_id = site, #' parameter_code = "00060", #' skipGeometry = TRUE) +#' +#' dv_data_period <- read_USGS_daily(monitoring_location_id = site, +#' parameter_code = "00060", +#' time = "P7D") #' #' multi_site <- read_USGS_daily(monitoring_location_id = c("USGS-01491000", #' "USGS-01645000"), @@ -70,33 +73,20 @@ read_USGS_daily <- function(monitoring_location_id = NA_character_, value = NA, last_modified = NA_character_, limit = 10000, - crs = NA_character_, skipGeometry = NA, time = NA_character_, convertType = TRUE){ message("Function in development, use at your own risk.") - + service <- "daily" - - dv_req <- construct_api_requests(service = service, - monitoring_location_id = monitoring_location_id, - parameter_code = parameter_code, - statistic_id = statistic_id, - properties = properties, - bbox = bbox, - time_series_id = time_series_id, - id = daily_id, - approval_status = approval_status, - unit_of_measure = unit_of_measure, - qualifier = qualifier, - value = value, - time = time, - last_modified = last_modified, - limit = limit, - crs = crs, - skipGeometry = skipGeometry) - + args <- mget(names(formals())) + args[["id"]] <- args[["daily_id"]] + args[["daily_id"]] <- NULL + args[["convertType"]] <- NULL + args[["service"]] <- service + dv_req <- do.call(construct_api_requests, args) + return_list <- walk_pages(dv_req) if(convertType) return_list <- cleanup_cols(return_list) diff --git a/R/read_USGS_monitoring_location.R b/R/read_USGS_monitoring_location.R index 1f280e7f..529e73fd 100644 --- a/R/read_USGS_monitoring_location.R +++ b/R/read_USGS_monitoring_location.R @@ -50,7 +50,6 @@ #' box are selected.The bounding box is provided as four or six numbers, depending #' on whether the coordinate reference system includes a vertical axis (height or #' depth). Coordinates are assumed to be in crs 4326. -#' @param crs Indicates the coordinate reference system for the results. #' @param limit The optional limit parameter limits the number of items that are #' presented in the response document. Only items are counted that are on the #' first level of the collection in the response document. Nested objects @@ -118,60 +117,18 @@ read_USGS_monitoring_location <- function(monitoring_location_id = NA_character_ properties = NA_character_, bbox = NA, limit = 10000, - crs = NA_character_, skipGeometry = NA){ message("Function in development, use at your own risk.") service <- "monitoring-locations" - site_req <- construct_api_requests(service = service, - id = monitoring_location_id, - agency_code = agency_code, - agency_name = agency_name, - monitoring_location_number = monitoring_location_number, - monitoring_location_name = monitoring_location_name, - district_code = district_code, - country_code = country_code, - country_name = country_name, - state_code = state_code, - state_name = state_name, - county_code = county_code, - county_name = county_name, - minor_civil_division_code = minor_civil_division_code, - site_type_code = site_type_code, - site_type = site_type, - hydrologic_unit_code = hydrologic_unit_code, - basin_code = basin_code, - altitude = altitude, - altitude_accuracy = altitude_accuracy, - altitude_method_code = altitude_method_code, - altitude_method_name = altitude_method_name, - vertical_datum = vertical_datum, - vertical_datum_name = vertical_datum_name, - horizontal_positional_accuracy_code = horizontal_positional_accuracy_code, - horizontal_positional_accuracy = horizontal_positional_accuracy, - horizontal_position_method_code = horizontal_position_method_code, - horizontal_position_method_name = horizontal_position_method_name, - original_horizontal_datum = original_horizontal_datum, - original_horizontal_datum_name = original_horizontal_datum_name, - drainage_area = drainage_area, - contributing_drainage_area = contributing_drainage_area, - time_zone_abbreviation = time_zone_abbreviation, - uses_daylight_savings = uses_daylight_savings, - construction_date = construction_date, - aquifer_code = aquifer_code, - national_aquifer_code = national_aquifer_code, - aquifer_type_code = aquifer_type_code, - well_constructed_depth = well_constructed_depth, - hole_constructed_depth = hole_constructed_depth, - depth_source_code = depth_source_code, - limit = limit, - bbox = bbox, - crs = crs, - skipGeometry = skipGeometry, - properties = properties) - + args <- mget(names(formals())) + args[["service"]] <- service + args[["id"]] <- args[["monitoring_location_id"]] + args[["monitoring_location_id"]] <- NULL + site_req <- do.call(construct_api_requests, args) + return_list <- walk_pages(site_req) return_list <- rejigger_cols(return_list, properties, service) diff --git a/R/read_USGS_ts_meta.R b/R/read_USGS_ts_meta.R index a36ac44a..13249a54 100644 --- a/R/read_USGS_ts_meta.R +++ b/R/read_USGS_ts_meta.R @@ -20,12 +20,11 @@ #' @param properties The properties that should be included for each feature. #' The parameter value is a comma-separated list of property names. Available options are #' `r schema <- check_OGC_requests(endpoint = "time-series-metadata", type = "schema"); paste(names(schema$properties), collapse = ", ")` -#' @param time_series_id `r get_params("time-series-metadata")$id` +#' @param time_series_metadata_id `r get_params("time-series-metadata")$id` #' @param bbox Only features that have a geometry that intersects the bounding #' box are selected.The bounding box is provided as four or six numbers, depending #' on whether the coordinate reference system includes a vertical axis (height or #' depth). Coordinates are assumed to be in crs 4326. -#' @param crs Indicates the coordinate reference system for the results. #' @param limit The optional limit parameter limits the number of items that are #' presented in the response document. Only items are counted that are on the #' first level of the collection in the response document. Nested objects @@ -66,9 +65,8 @@ read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, thresholds = NA, sublocation_identifier = NA_character_, primary = NA_character_, - time_series_id = NA_character_, + time_series_metadata_id = NA_character_, web_description = NA_character_, - crs = NA_character_, skipGeometry = NA, convertType = FALSE){ @@ -76,27 +74,14 @@ read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, service = "time-series-metadata" - req_ts_meta <- construct_api_requests(service, - monitoring_location_id = monitoring_location_id, - parameter_code = parameter_code, - parameter_name = parameter_name, - properties = properties, - statistic_id = statistic_id, - last_modified = last_modified, - begin = begin, - end = end, - limit = limit, - unit_of_measure = unit_of_measure, - computation_period_identifier = computation_period_identifier, - computation_identifier = computation_identifier, - thresholds = thresholds, - sublocation_identifier = sublocation_identifier, - primary = primary, - id = time_series_id, - crs = crs, - web_description = web_description, - skipGeometry = skipGeometry) + args <- mget(names(formals())) + args[["service"]] <- service + args[["id"]] <- args[["time_series_id"]] + args[["time_series_metadata_id"]] <- NULL + args[["convertType"]] <- NULL + req_ts_meta <- do.call(construct_api_requests, args) + return_list <- walk_pages(req_ts_meta) if(convertType) return_list <- cleanup_cols(return_list) diff --git a/man/check_OGC_requests.Rd b/man/check_OGC_requests.Rd index 03ce253c..623379b7 100644 --- a/man/check_OGC_requests.Rd +++ b/man/check_OGC_requests.Rd @@ -7,7 +7,7 @@ check_OGC_requests(endpoint = "daily", type = "queryables") } \arguments{ -\item{endpoint}{Character, can be "daily", "timeseries-metadata"} +\item{endpoint}{Character, can be any existing collection} \item{type}{Character, can be "queryables", "schema"} } diff --git a/man/construct_api_requests.Rd b/man/construct_api_requests.Rd index 9cdf218f..e240213b 100644 --- a/man/construct_api_requests.Rd +++ b/man/construct_api_requests.Rd @@ -8,7 +8,6 @@ construct_api_requests( service, properties = NA_character_, bbox = NA, - crs = NA_character_, limit = 10000, skipGeometry = FALSE, ... @@ -18,17 +17,14 @@ construct_api_requests( \item{service}{Which service} \item{properties}{The properties that should be included for each feature. The -parameter value is a comma-separated list of property names. Available values: -id, timeseries_id, monitoring_location_id, parameter_code, statistic_id, time, -value, unit_of_measure, approval_status, qualifier, last_modified.} +parameter value is a comma-separated list of property names which depend on the +service being called.} \item{bbox}{Only features that have a geometry that intersects the bounding box are selected.The bounding box is provided as four or six numbers, depending on whether the coordinate reference system includes a vertical axis (height or depth).} -\item{crs}{Indicates the coordinate reference system for the results.} - \item{limit}{The optional limit parameter limits the number of items that are presented in the response document. Only items are counted that are on the first level of the collection in the response document. Nested objects diff --git a/man/read_USGS_daily.Rd b/man/read_USGS_daily.Rd index 44b087e8..a3cf02a2 100644 --- a/man/read_USGS_daily.Rd +++ b/man/read_USGS_daily.Rd @@ -18,7 +18,6 @@ read_USGS_daily( value = NA, last_modified = NA_character_, limit = 10000, - crs = NA_character_, skipGeometry = NA, time = NA_character_, convertType = TRUE @@ -69,8 +68,6 @@ presented in the response document. Only items are counted that are on the first level of the collection in the response document. Nested objects contained within the explicitly requested items shall not be counted.} -\item{crs}{Indicates the coordinate reference system for the results.} - \item{skipGeometry}{This option can be used to skip response geometries for each feature. The returning object will be a data frame with no spatial information.} @@ -112,6 +109,10 @@ dv_data_trim <- read_USGS_daily(monitoring_location_id = site, dv_data <- read_USGS_daily(monitoring_location_id = site, parameter_code = "00060", skipGeometry = TRUE) + +dv_data_period <- read_USGS_daily(monitoring_location_id = site, + parameter_code = "00060", + time = "P7D") multi_site <- read_USGS_daily(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), diff --git a/man/read_USGS_monitoring_location.Rd b/man/read_USGS_monitoring_location.Rd index 0e6975c1..f3ca7d70 100644 --- a/man/read_USGS_monitoring_location.Rd +++ b/man/read_USGS_monitoring_location.Rd @@ -48,7 +48,6 @@ read_USGS_monitoring_location( properties = NA_character_, bbox = NA, limit = 10000, - crs = NA_character_, skipGeometry = NA ) } @@ -147,8 +146,6 @@ presented in the response document. Only items are counted that are on the first level of the collection in the response document. Nested objects contained within the explicitly requested items shall not be counted.} -\item{crs}{Indicates the coordinate reference system for the results.} - \item{skipGeometry}{This option can be used to skip response geometries for each feature. The returning object will be a data frame with no spatial information.} diff --git a/man/read_USGS_ts_meta.Rd b/man/read_USGS_ts_meta.Rd index a71bf55d..ed6044a0 100644 --- a/man/read_USGS_ts_meta.Rd +++ b/man/read_USGS_ts_meta.Rd @@ -21,9 +21,8 @@ read_USGS_ts_meta( thresholds = NA, sublocation_identifier = NA_character_, primary = NA_character_, - time_series_id = NA_character_, + time_series_metadata_id = NA_character_, web_description = NA_character_, - crs = NA_character_, skipGeometry = NA, convertType = FALSE ) @@ -79,12 +78,10 @@ Only features that have a \code{last_modified} that intersects the value of date \item{primary}{A flag identifying if the time series is a "primary" time series. "Primary" time series (which have this flag) are standard observations which undergo \href{https://www.usgs.gov/survey-manual/5028-fundamental-science-practices-review-and-approval-scientific-data-release}{Bureau review and approval processes}. Non-primary time series, which will have missing values for "primary", are provisional datasets made available to meet the need for timely best science and to assist with daily operations which need real-time information. Non-primary time series data are only retained by this system for 120 days. See the \href{https://waterdata.usgs.gov/provisional-data-statement/}{USGS Provisional Data Statement} for more information.} -\item{time_series_id}{A unique identifier representing a single time series. This corresponds to the \code{id} field in the \code{time-series-metadata} endpoint.} +\item{time_series_metadata_id}{A unique identifier representing a single time series. This corresponds to the \code{id} field in the \code{time-series-metadata} endpoint.} \item{web_description}{A description of what this time series represents, as used by WDFN and other USGS data dissemination products.} -\item{crs}{Indicates the coordinate reference system for the results.} - \item{skipGeometry}{This option can be used to skip response geometries for each feature. The returning object will be a data frame with no spatial information.} From cdf79fb80f3f7ec4d629fa1c453c96a859005d80 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 22 May 2025 15:45:47 -0500 Subject: [PATCH 036/117] Get rid of Tailcall --- DESCRIPTION | 12 +++++++++++- R/construct_api_requests.R | 2 +- R/dataRetrievals-package.R | 10 ++++++++-- R/read_USGS_daily.R | 3 ++- R/walk_pages.R | 14 +++++++------- man/construct_api_requests.Rd | 2 +- man/read_USGS_daily.Rd | 3 ++- 7 files changed, 32 insertions(+), 14 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 77a1428b..e3908ca8 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -26,7 +26,17 @@ Authors@R: c( comment=c(ORCID = "0000-0003-2521-5043")), person("Lee", "Stanish", role="ctb", email = "lstanish@usgs.gov", - comment=c(ORCID = "0000-0002-9775-6861"))) + comment=c(ORCID = "0000-0002-9775-6861")), + person("Joeseph", "Zemmels", role="ctb", + email = "jzemmels@usgs.gov", + comment=c(ORCID = "0009-0008-1463-6313")), + person("Elise", "Hinman", role="ctb", + email = "ehinman@usgs.gov", + comment=c(ORCID = "0000-0001-5396-1583")), + person("Michael", "Mahoney", role="ctb", + email = "mjmahoney@usgs.gov", + comment=c(ORCID = "0000-0003-2402-304X")) + ) Description: Collection of functions to help retrieve U.S. Geological Survey and U.S. Environmental Protection Agency water quality and hydrology data from web services. diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index f7a891c6..3304ab80 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -4,7 +4,7 @@ #' Swagger docs: . #' #' @export -#' @param service Which service +#' @param service Which service available on . #' @param ... Extra parameters from the specific services. #' @param bbox Only features that have a geometry that intersects the bounding #' box are selected.The bounding box is provided as four or six numbers, depending diff --git a/R/dataRetrievals-package.R b/R/dataRetrievals-package.R index 62abe374..97c6627d 100644 --- a/R/dataRetrievals-package.R +++ b/R/dataRetrievals-package.R @@ -1,8 +1,14 @@ .onAttach <- function(libname, pkgname) { if (!interactive()) return() - dataRetrieval_version = utils::packageVersion("dataRetrieval") + dataRetrieval_version <- utils::packageVersion("dataRetrieval") + token_message <- "" + if(Sys.getenv("API_USGS_PAT") == ""){ + token_message <- "Consider adding an API_USGS_PAT for new USGS functions. +See: https://api.waterdata.usgs.gov/signup" + } packageStartupMessage("dataRetrieval ", dataRetrieval_version," -Extended Documentation: https://doi-usgs.github.io/dataRetrieval") +Extended Documentation: https://doi-usgs.github.io/dataRetrieval", + token_message) } #' Retrieval functions for USGS and EPA data diff --git a/R/read_USGS_daily.R b/R/read_USGS_daily.R index 780e0673..a74e0e33 100644 --- a/R/read_USGS_daily.R +++ b/R/read_USGS_daily.R @@ -54,9 +54,10 @@ #' parameter_code = "00060", #' time = "P7D") #' -#' multi_site <- read_USGS_daily(monitoring_location_id = c("USGS-01491000", +#' multi_site <- read_USGS_daily(monitoring_location_id = c("USGS-01491000", #' "USGS-01645000"), #' parameter_code = c("00060", "00010"), +#' limit = 500, #' time = c("2023-01-01", "2024-01-01")) #' #' } diff --git a/R/walk_pages.R b/R/walk_pages.R index 226158d4..5d9a8db3 100644 --- a/R/walk_pages.R +++ b/R/walk_pages.R @@ -91,12 +91,12 @@ walk_pages_recursive <- function(req, page, contents) { make_request <- httr2::req_url(req = req, url = json_content$links[, "href", drop = TRUE][next_page]) - - Tailcall( - walk_pages_recursive, - make_request, - page + 1, - contents - ) + page <- page + 1 + while (TRUE){ + return(walk_pages_recursive(req = make_request, + page = page, + contents = contents)) + } + } } \ No newline at end of file diff --git a/man/construct_api_requests.Rd b/man/construct_api_requests.Rd index e240213b..3e75315f 100644 --- a/man/construct_api_requests.Rd +++ b/man/construct_api_requests.Rd @@ -14,7 +14,7 @@ construct_api_requests( ) } \arguments{ -\item{service}{Which service} +\item{service}{Which service available on \url{https://api.waterdata.usgs.gov/ogcapi/v0/}.} \item{properties}{The properties that should be included for each feature. The parameter value is a comma-separated list of property names which depend on the diff --git a/man/read_USGS_daily.Rd b/man/read_USGS_daily.Rd index a3cf02a2..d5129f6e 100644 --- a/man/read_USGS_daily.Rd +++ b/man/read_USGS_daily.Rd @@ -114,9 +114,10 @@ dv_data_period <- read_USGS_daily(monitoring_location_id = site, parameter_code = "00060", time = "P7D") -multi_site <- read_USGS_daily(monitoring_location_id = c("USGS-01491000", +multi_site <- read_USGS_daily(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), parameter_code = c("00060", "00010"), + limit = 500, time = c("2023-01-01", "2024-01-01")) } From aa7d987cbc7276247465a9014ef25a80147577b7 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 23 May 2025 08:58:52 -0500 Subject: [PATCH 037/117] Update vignettes/read_USGS_functions.Rmd Co-authored-by: Elise Hinman <121896266+ehinman@users.noreply.github.com> --- vignettes/read_USGS_functions.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index f84c674e..15659f69 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -181,7 +181,7 @@ New functions will use a "snake case", such as "read_USGS_samples". Older functi ### Structure -Historically, we allowed users to customize their queries via the `...` argument structure. With `...`, users needed to know the exact names of query parameters before using the function. Now, the new functions will include **ALL** possible arguments that the web service APIs support. This will allow users to use tab-autocompletes (available in RStudio and other IDEs). **Users will need to understand that it is not advisable to specify all of these parameters. The systems can get bogged down with redundant query parameters.** We expect this will be easier for users, but it might take some time to smooth out the documentation and test usability. There may be additional consequences, such as users won't be able to build up argument lists to pass into the function. +Historically, we allowed users to customize their queries via the `...` argument structure. With `...`, users needed to know the exact names of query parameters before using the function. Now, the new functions will include **ALL** possible arguments that the web service APIs support. This will allow users to use tab-autocompletes (available in RStudio and other IDEs). **Users will need to understand that they are not required to specify all of these parameters. In fact, it is not advisable: the systems can get bogged down with redundant query parameters.** We expect this will be easier for users, but it might take some time to smooth out the documentation and test usability. There may be additional consequences, such as users won't be able to build up argument lists to pass into the function. ### Dependencies From f5dc93e6c4cb06be4d9df6b9460dde8480136322 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 23 May 2025 09:11:48 -0500 Subject: [PATCH 038/117] fix bbox error --- R/construct_api_requests.R | 2 +- R/read_USGS_daily.R | 4 ++-- R/read_USGS_monitoring_location.R | 6 +++--- R/read_USGS_ts_meta.R | 4 ++-- man/read_USGS_daily.Rd | 4 ++-- man/read_USGS_monitoring_location.Rd | 6 +++--- man/read_USGS_ts_meta.Rd | 4 ++-- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index 3304ab80..0d72ae0a 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -83,7 +83,7 @@ construct_api_requests <- function(service, full_list <- list(...) - if(all(is.na(full_list)) & is.na(bbox)){ + if(all(is.na(full_list)) & all(is.na(bbox))){ warning("No filtering arguments specified.") } diff --git a/R/read_USGS_daily.R b/R/read_USGS_daily.R index a74e0e33..1d4c0779 100644 --- a/R/read_USGS_daily.R +++ b/R/read_USGS_daily.R @@ -14,8 +14,8 @@ #' @param time_series_id `r get_params("daily")$time_series_id` #' @param qualifier `r get_params("daily")$qualifier` #' @param daily_id `r get_params("daily")$id` -#' @param properties The properties that should be included for each feature. -#' The parameter value is a comma-separated list of property names. Available options are +#' @param properties A vector of requested columns to be returned from the query. +#' Available options are: #' `r schema <- check_OGC_requests(endpoint = "daily", type = "schema"); paste(names(schema$properties), collapse = ", ")` #' @param bbox Only features that have a geometry that intersects the bounding #' box are selected.The bounding box is provided as four or six numbers, depending diff --git a/R/read_USGS_monitoring_location.R b/R/read_USGS_monitoring_location.R index 529e73fd..7e3934ca 100644 --- a/R/read_USGS_monitoring_location.R +++ b/R/read_USGS_monitoring_location.R @@ -43,9 +43,9 @@ #' @param well_constructed_depth `r get_params("monitoring-locations")$well_constructed_depth` #' @param hole_constructed_depth `r get_params("monitoring-locations")$hole_constructed_depth` #' @param depth_source_code `r get_params("monitoring-locations")$depth_source_code` -#' @param properties The properties that should be included for each feature. -#' The parameter value is a comma-separated list of property names. Available options are -#' `r schema <- check_OGC_requests(endpoint = "monitoring-locations", type = "schema"); paste(names(schema$properties), collapse = ", ")` +#' @param properties A vector of requested columns to be returned from the query. +#' Available options are: +#' `r schema <- check_OGC_requests(endpoint = "monitoring-locations", type = "schema"); paste(names(schema$properties), collapse = ", ")`. #' @param bbox Only features that have a geometry that intersects the bounding #' box are selected.The bounding box is provided as four or six numbers, depending #' on whether the coordinate reference system includes a vertical axis (height or diff --git a/R/read_USGS_ts_meta.R b/R/read_USGS_ts_meta.R index 13249a54..6c4b94a6 100644 --- a/R/read_USGS_ts_meta.R +++ b/R/read_USGS_ts_meta.R @@ -17,8 +17,8 @@ #' @param unit_of_measure `r get_params("time-series-metadata")$unit_of_measure` #' @param primary `r get_params("time-series-metadata")$primary` #' @param web_description `r get_params("time-series-metadata")$web_description` -#' @param properties The properties that should be included for each feature. -#' The parameter value is a comma-separated list of property names. Available options are +#' @param properties A vector of requested columns to be returned from the query. +#' Available options are: #' `r schema <- check_OGC_requests(endpoint = "time-series-metadata", type = "schema"); paste(names(schema$properties), collapse = ", ")` #' @param time_series_metadata_id `r get_params("time-series-metadata")$id` #' @param bbox Only features that have a geometry that intersects the bounding diff --git a/man/read_USGS_daily.Rd b/man/read_USGS_daily.Rd index d5129f6e..849ab96a 100644 --- a/man/read_USGS_daily.Rd +++ b/man/read_USGS_daily.Rd @@ -30,8 +30,8 @@ read_USGS_daily( \item{statistic_id}{A code corresponding to the statistic an observation represents. Example codes include 00001 (max), 00002 (min), and 00003 (mean). A complete list of codes and their descriptions can be found at \url{https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=\%25&fmt=html}.} -\item{properties}{The properties that should be included for each feature. -The parameter value is a comma-separated list of property names. Available options are +\item{properties}{A vector of requested columns to be returned from the query. +Available options are: geometry, id, time_series_id, monitoring_location_id, parameter_code, statistic_id, time, value, unit_of_measure, approval_status, qualifier, last_modified} \item{bbox}{Only features that have a geometry that intersects the bounding diff --git a/man/read_USGS_monitoring_location.Rd b/man/read_USGS_monitoring_location.Rd index f3ca7d70..ba74a298 100644 --- a/man/read_USGS_monitoring_location.Rd +++ b/man/read_USGS_monitoring_location.Rd @@ -132,9 +132,9 @@ read_USGS_monitoring_location( \item{depth_source_code}{A code indicating the source of water-level data. \href{https://help.waterdata.usgs.gov/code/water_level_src_cd_query?fmt=html}{A list of codes is available.}} -\item{properties}{The properties that should be included for each feature. -The parameter value is a comma-separated list of property names. Available options are -geometry, id, agency_code, agency_name, monitoring_location_number, monitoring_location_name, district_code, country_code, country_name, state_code, state_name, county_code, county_name, minor_civil_division_code, site_type_code, site_type, hydrologic_unit_code, basin_code, altitude, altitude_accuracy, altitude_method_code, altitude_method_name, vertical_datum, vertical_datum_name, horizontal_positional_accuracy_code, horizontal_positional_accuracy, horizontal_position_method_code, horizontal_position_method_name, original_horizontal_datum, original_horizontal_datum_name, drainage_area, contributing_drainage_area, time_zone_abbreviation, uses_daylight_savings, construction_date, aquifer_code, national_aquifer_code, aquifer_type_code, well_constructed_depth, hole_constructed_depth, depth_source_code} +\item{properties}{A vector of requested columns to be returned from the query. +Available options are: +geometry, id, agency_code, agency_name, monitoring_location_number, monitoring_location_name, district_code, country_code, country_name, state_code, state_name, county_code, county_name, minor_civil_division_code, site_type_code, site_type, hydrologic_unit_code, basin_code, altitude, altitude_accuracy, altitude_method_code, altitude_method_name, vertical_datum, vertical_datum_name, horizontal_positional_accuracy_code, horizontal_positional_accuracy, horizontal_position_method_code, horizontal_position_method_name, original_horizontal_datum, original_horizontal_datum_name, drainage_area, contributing_drainage_area, time_zone_abbreviation, uses_daylight_savings, construction_date, aquifer_code, national_aquifer_code, aquifer_type_code, well_constructed_depth, hole_constructed_depth, depth_source_code.} \item{bbox}{Only features that have a geometry that intersects the bounding box are selected.The bounding box is provided as four or six numbers, depending diff --git a/man/read_USGS_ts_meta.Rd b/man/read_USGS_ts_meta.Rd index ed6044a0..e2a41912 100644 --- a/man/read_USGS_ts_meta.Rd +++ b/man/read_USGS_ts_meta.Rd @@ -34,8 +34,8 @@ read_USGS_ts_meta( \item{parameter_name}{A human-understandable name corresponding to \code{parameter_code}.} -\item{properties}{The properties that should be included for each feature. -The parameter value is a comma-separated list of property names. Available options are +\item{properties}{A vector of requested columns to be returned from the query. +Available options are: geometry, id, unit_of_measure, parameter_name, parameter_code, statistic_id, last_modified, begin, end, computation_period_identifier, computation_identifier, thresholds, sublocation_identifier, primary, monitoring_location_id, web_description} \item{limit}{The optional limit parameter limits the number of items that are From 28d964e37a99d4243186d8450ab7bc7d82007429 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 23 May 2025 09:37:18 -0500 Subject: [PATCH 039/117] fixed slash --- R/construct_api_requests.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index 0d72ae0a..b2f52ec5 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -152,7 +152,8 @@ format_api_dates <- function(datetime){ if(!any(isTRUE(is.na(datetime)) | isTRUE(is.null(datetime)))){ if(length(datetime) == 1){ - if(grepl("P", datetime, ignore.case = TRUE)){ + if(grepl("P", datetime, ignore.case = TRUE) | + grepl("/", datetime)){ return(datetime) } else { datetime <- format(datetime, format = "%Y-%m-%dT%H:%M:%SZ") From 8ad2db96f10e3a53216b4a7ce59ff7e3e4cbd087 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 23 May 2025 11:54:04 -0500 Subject: [PATCH 040/117] deal with empty returns --- R/read_USGS_daily.R | 4 +++- R/read_USGS_monitoring_location.R | 2 ++ R/read_USGS_ts_meta.R | 2 ++ R/walk_pages.R | 29 ++++++++++++++++++++--------- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/R/read_USGS_daily.R b/R/read_USGS_daily.R index 1d4c0779..5d644492 100644 --- a/R/read_USGS_daily.R +++ b/R/read_USGS_daily.R @@ -90,8 +90,10 @@ read_USGS_daily <- function(monitoring_location_id = NA_character_, return_list <- walk_pages(dv_req) + return_list <- deal_with_empty(return_list, properties, service) + if(convertType) return_list <- cleanup_cols(return_list) - + return_list <- return_list[order(return_list$time, return_list$monitoring_location_id), ] return_list <- rejigger_cols(return_list, properties, service) diff --git a/R/read_USGS_monitoring_location.R b/R/read_USGS_monitoring_location.R index 7e3934ca..3c838a8b 100644 --- a/R/read_USGS_monitoring_location.R +++ b/R/read_USGS_monitoring_location.R @@ -131,6 +131,8 @@ read_USGS_monitoring_location <- function(monitoring_location_id = NA_character_ return_list <- walk_pages(site_req) + return_list <- deal_with_empty(return_list, properties, service) + return_list <- rejigger_cols(return_list, properties, service) return(return_list) diff --git a/R/read_USGS_ts_meta.R b/R/read_USGS_ts_meta.R index 6c4b94a6..fd2cd519 100644 --- a/R/read_USGS_ts_meta.R +++ b/R/read_USGS_ts_meta.R @@ -84,6 +84,8 @@ read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, return_list <- walk_pages(req_ts_meta) + return_list <- deal_with_empty(return_list, properties, service) + if(convertType) return_list <- cleanup_cols(return_list) return_list <- rejigger_cols(return_list, properties, service) diff --git a/R/walk_pages.R b/R/walk_pages.R index 5d9a8db3..4915a2b8 100644 --- a/R/walk_pages.R +++ b/R/walk_pages.R @@ -1,3 +1,16 @@ +deal_with_empty <- function(return_list, properties, service){ + if(nrow(return_list) == 0){ + if(is.na(properties)){ + schema <- check_OGC_requests(endpoint = service, type = "schema") + properties <- names(schema$properties) + } + return_list <- data.frame(matrix(nrow = 0, ncol = length(properties))) + names(return_list) <- properties + } + + return(return_list) +} + rejigger_cols <- function(df, properties, service){ new_id <- paste0(gsub("-", "_", service), "_id") names(df)[names(df) == "id"] <- new_id @@ -50,15 +63,9 @@ walk_pages_recursive <- function(req, page, contents) { message(url_method, ": ", req$url) - returned_contents <- httr2::req_perform(req) - - if(httr2::resp_status(returned_contents) != 200){ - if(httr2::resp_status(returned_contents) == 429){ - stop("You hit your hourly limit for requests. To increase the limit, - see: https://api.waterdata.usgs.gov/docs/ogcapi/keys/") - } - } - + returned_contents <- req |> + httr2::req_perform() + header_info <- httr2::resp_headers(returned_contents) if(Sys.getenv("API_USGS_PAT") != ""){ message("Remaining requests this hour:", header_info$`x-ratelimit-remaining`) @@ -69,6 +76,10 @@ walk_pages_recursive <- function(req, page, contents) { json_content <- jsonlite::fromJSON(contents[[page]] ) + if(isTRUE(json_content[["numberReturned"]] == 0)){ + return(data.frame()) + } + next_page <- json_content$links[, "rel", drop = TRUE] == "next" if (!any(next_page)) { From 1703cec22d4c727f8ed6eda71727deb86a3796e3 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 23 May 2025 15:04:41 -0500 Subject: [PATCH 041/117] Rejigger to use more native httr2 functions --- R/construct_api_requests.R | 20 ++++- R/dataRetrievals-package.R | 2 +- R/walk_pages.R | 95 ++++++++++-------------- man/read_USGS_ts_meta.Rd | 2 +- tests/testthat/tests_general.R | 16 ++++ tests/testthat/tests_userFriendly_fxns.R | 28 +++++++ 6 files changed, 103 insertions(+), 60 deletions(-) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index b2f52ec5..b48b4bce 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -229,7 +229,9 @@ check_OGC_requests <- function(endpoint = "daily", check_collections <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/server/getCollections") check_endpoints_req <- basic_request(check_collections) - query_ret <- httr2::req_perform(check_endpoints_req) |> + + query_ret <- check_endpoints_req |> + httr2::req_perform() |> httr2::resp_body_json() services <- sapply(query_ret$tags, function(x) x[["name"]]) @@ -242,20 +244,32 @@ check_OGC_requests <- function(endpoint = "daily", req <- basic_request(url_base) - query_ret <- httr2::req_perform(req) |> + query_ret <- req |> + httr2::req_perform() |> httr2::resp_body_json() return(query_ret) } +error_body <- function(resp) { + status <- httr2::resp_status(resp) + if(status == 429){ + x <- httr2::resp_body_json(resp)$error + return(x[["message"]]) + } else if (status == 403){ + return("Query request denied. Possible reasons include query being exceeding server limits.") + } +} + basic_request <- function(url_base){ req <- url_base |> httr2::req_user_agent(default_ua()) |> httr2::req_headers(`Accept-Encoding` = c("compress", "gzip")) |> httr2::req_url_query(f = "json", - lang = "en-US") + lang = "en-US") |> + httr2::req_error(body = error_body) token <- Sys.getenv("API_USGS_PAT") diff --git a/R/dataRetrievals-package.R b/R/dataRetrievals-package.R index 97c6627d..b92eda0e 100644 --- a/R/dataRetrievals-package.R +++ b/R/dataRetrievals-package.R @@ -3,7 +3,7 @@ dataRetrieval_version <- utils::packageVersion("dataRetrieval") token_message <- "" if(Sys.getenv("API_USGS_PAT") == ""){ - token_message <- "Consider adding an API_USGS_PAT for new USGS functions. + token_message <- "\nConsider adding an API_USGS_PAT for new USGS functions. See: https://api.waterdata.usgs.gov/signup" } packageStartupMessage("dataRetrieval ", dataRetrieval_version," diff --git a/R/walk_pages.R b/R/walk_pages.R index 4915a2b8..8a30c516 100644 --- a/R/walk_pages.R +++ b/R/walk_pages.R @@ -39,75 +39,60 @@ cleanup_cols <- function(df){ if("value" %in% names(df)){ df$value <- as.numeric(df$value) } + + if("contributing_drainage_area" %in% names(df)){ + df$contributing_drainage_area <- as.numeric(df$contributing_drainage_area) + } + df } -walk_pages <- function(req) { - - walk_pages_recursive( - req = req, - page = 1, - contents = list() - ) -} +next_req_url <- function(resp, req) { -walk_pages_recursive <- function(req, page, contents) { + body <- httr2::resp_body_json(resp) - url_method <- "GET" - if(!is.null(req$body)){ - url_method <- "POST" - body <- req$body + if(isTRUE(body[["numberReturned"]] == 0)){ + return(data.frame()) } - use_sf <- !grepl("skipGeometry=true", req$url, ignore.case = TRUE) - - message(url_method, ": ", req$url) - - returned_contents <- req |> - httr2::req_perform() - - header_info <- httr2::resp_headers(returned_contents) + header_info <- httr2::resp_headers(resp) if(Sys.getenv("API_USGS_PAT") != ""){ message("Remaining requests this hour:", header_info$`x-ratelimit-remaining`) } - contents[[page]] <- returned_contents |> - httr2::resp_body_string() - - json_content <- jsonlite::fromJSON(contents[[page]] ) - - if(isTRUE(json_content[["numberReturned"]] == 0)){ - return(data.frame()) + links <- body$links + if(any(sapply(links, function(x) x$rel) == "next")){ + next_index <- which(sapply(links, function(x) x$rel) == "next") + + next_url <- links[[next_index]][["href"]] + return(httr2::req_url(req = req, url = next_url)) + } else { + return(NULL) } +} + +get_resp_data <- function(resp) { - next_page <- json_content$links[, "rel", drop = TRUE] == "next" + use_sf <- !grepl("skipGeometry=true", resp$url, ignore.case = TRUE) - if (!any(next_page)) { - - if(use_sf){ - return_df <- lapply(contents, function(x) { - df_i <- sf::read_sf(x) - }) |> - do.call(what = rbind) - } else { - return_df <- lapply(contents, function(x) { - df_i <- jsonlite::fromJSON(x)[["features"]][["properties"]] - }) |> - do.call(what = rbind) - } - - return(return_df) - + if(use_sf){ + return_df <- sf::read_sf(httr2::resp_body_string(resp)) } else { - - make_request <- httr2::req_url(req = req, - url = json_content$links[, "href", drop = TRUE][next_page]) - page <- page + 1 - while (TRUE){ - return(walk_pages_recursive(req = make_request, - page = page, - contents = contents)) - } + return_df <- jsonlite::fromJSON(httr2::resp_body_string(resp))[["features"]][["properties"]] + } + return(return_df) + +} + +walk_pages <- function(req){ + resps <- httr2::req_perform_iterative(req, next_req = next_req_url) + + return_list <- data.frame() + for(resp in resps){ + df1 <- get_resp_data(resp) + return_list <- rbind(return_list, df1) } -} \ No newline at end of file + + return(return_list) +} diff --git a/man/read_USGS_ts_meta.Rd b/man/read_USGS_ts_meta.Rd index e2a41912..33aa2a44 100644 --- a/man/read_USGS_ts_meta.Rd +++ b/man/read_USGS_ts_meta.Rd @@ -36,7 +36,7 @@ read_USGS_ts_meta( \item{properties}{A vector of requested columns to be returned from the query. Available options are: -geometry, id, unit_of_measure, parameter_name, parameter_code, statistic_id, last_modified, begin, end, computation_period_identifier, computation_identifier, thresholds, sublocation_identifier, primary, monitoring_location_id, web_description} +geometry, id, unit_of_measure, parameter_name, parameter_code, statistic_id, last_modified, begin, end, computation_period_identifier, computation_identifier, thresholds, sublocation_identifier, primary, monitoring_location_id, web_description, parameter_description} \item{limit}{The optional limit parameter limits the number of items that are presented in the response document. Only items are counted that are on the diff --git a/tests/testthat/tests_general.R b/tests/testthat/tests_general.R index 3e35a42b..563ddd46 100644 --- a/tests/testthat/tests_general.R +++ b/tests/testthat/tests_general.R @@ -126,6 +126,17 @@ test_that("General NWIS retrievals working", { dailyData <- readNWISdata(args) expect_lt(nrow(dailyData), nrow(instData)) + + args2 <- list( + monitoring_location_id = "USGS-05114000", + parameter_code = "00060", + time = c("2014-05-01", endDate = "2014-05-01") + ) + + daily_USGS <- do.call(read_USGS_daily, args2) + expect_lt(nrow(daily_USGS), nrow(instData)) + + args <- list(stateCd = "OH", parameterCd = "00665") sites <- whatNWISsites(args) expect_type(sites, "list") @@ -182,6 +193,11 @@ test_that("General NWIS retrievals working", { ) expect_equal(2, nrow(multi_huc)) + multi_hucs <- c("07130007", "07130011") + multi_huc_new <- read_USGS_monitoring_location( + hydrologic_unit_code = multi_hucs + ) + expect_equal(2, length(unique(multi_huc_new$hydrologic_unit_code))) peak_data <- readNWISdata( service = "peak", diff --git a/tests/testthat/tests_userFriendly_fxns.R b/tests/testthat/tests_userFriendly_fxns.R index a996c1f6..d21b9742 100644 --- a/tests/testthat/tests_userFriendly_fxns.R +++ b/tests/testthat/tests_userFriendly_fxns.R @@ -122,6 +122,12 @@ test_that("NWIS dv tests", { rawDailyQ <- readNWISdv(siteNumber, pCode, startDate, endDate) expect_is(rawDailyQ$Date, "Date") + + raw_USGS_daily <- read_USGS_daily(monitoring_location_id = paste0("USGS-", siteNumber), + parameter_code = pCode, + time = c(startDate, endDate)) + expect_is(raw_USGS_daily$time, "Date") + rawDailyQAndTempMeanMax <- readNWISdv(siteNumber, c("00010", "00060"), startDate, endDate, @@ -130,6 +136,14 @@ test_that("NWIS dv tests", { expect_true(length(grep("00060", names(rawDailyQAndTempMeanMax))) >= 2 & length(grep("00010", names(rawDailyQAndTempMeanMax))) >= 2) + raw_USGS_TempMeanMax <- read_USGS_daily(monitoring_location_id = paste0("USGS-", siteNumber), + parameter_code = c("00010", "00060"), + time = c(startDate, endDate), + statistic_id = c("00001", "00003")) + + expect_true(length(unique(raw_USGS_TempMeanMax$parameter_code)) == 2) + expect_true(length(unique(raw_USGS_TempMeanMax$statistic_id)) == 2) + expect_true(length(unique(raw_USGS_TempMeanMax$monitoring_location_id)) == 1) rawDailyMultiSites <- readNWISdv(c("01491000", "01645000"), c("00010", "00060"), @@ -138,9 +152,23 @@ test_that("NWIS dv tests", { ) expect_true(length(unique(rawDailyMultiSites$site_no)) > 1) + raw_USGS_MultiSites <- read_USGS_daily(monitoring_location_id = paste0("USGS-", + c("01491000", "01645000")), + parameter_code = c("00010", "00060"), + time = c(startDate, endDate), + statistic_id = c("00001", "00003")) + + expect_true(length(unique(raw_USGS_MultiSites$monitoring_location_id)) == 2) + site <- "05212700" notActive <- readNWISdv(site, "00060", "2014-01-01", "2014-01-07") expect_true(nrow(notActive) == 0) + + notActiveUSGS <- read_USGS_daily(monitoring_location_id = paste0("USGS-", site), + parameter_code = "00060", + time = c("2014-01-01", "2014-01-07")) + expect_true(nrow(notActiveUSGS) == 0) + }) test_that("WQP qw tests", { From 358fdc4c8fe79bfa265fc5662e696c58bcb5dac8 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 23 May 2025 15:30:52 -0500 Subject: [PATCH 042/117] fix empty return (again!) --- R/walk_pages.R | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/R/walk_pages.R b/R/walk_pages.R index 8a30c516..a1c48997 100644 --- a/R/walk_pages.R +++ b/R/walk_pages.R @@ -52,7 +52,7 @@ next_req_url <- function(resp, req) { body <- httr2::resp_body_json(resp) if(isTRUE(body[["numberReturned"]] == 0)){ - return(data.frame()) + return(NULL) } header_info <- httr2::resp_headers(resp) @@ -73,6 +73,12 @@ next_req_url <- function(resp, req) { get_resp_data <- function(resp) { + body <- httr2::resp_body_json(resp) + + if(isTRUE(body[["numberReturned"]] == 0)){ + return(data.frame()) + } + use_sf <- !grepl("skipGeometry=true", resp$url, ignore.case = TRUE) if(use_sf){ @@ -86,13 +92,25 @@ get_resp_data <- function(resp) { } walk_pages <- function(req){ - resps <- httr2::req_perform_iterative(req, next_req = next_req_url) + + resps <- httr2::req_perform_iterative(req, next_req = next_req_url) + failures <- resps |> + httr2::resps_failures() |> + httr2::resps_requests() + + if(length(failures) > 0){ + message("There were", length(failures), "failed requests.") + } + return_list <- data.frame() for(resp in resps){ df1 <- get_resp_data(resp) return_list <- rbind(return_list, df1) } + attr(return_list, "request") <- req + attr(return_list, "queryTime") <- Sys.time() + return(return_list) } From be46567f01bbc0dbffdb2d6093d7999415179d10 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 23 May 2025 16:08:15 -0500 Subject: [PATCH 043/117] add bbox example --- R/construct_api_requests.R | 18 ++++++++++-------- R/read_USGS_monitoring_location.R | 2 ++ man/read_USGS_monitoring_location.Rd | 2 ++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index b48b4bce..1c810b72 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -177,14 +177,16 @@ explode_post <- function(ls){ ls <- Filter(Negate(anyNA), ls) params <- NULL - if(max(lengths(ls)) > 1) { - - for(i in seq_along(ls)){ - params[names(ls[i])] <- cql2_param(ls[i]) - } - - if(length(params) > 1){ - params[seq_along(1:(length(params)-1))] <- paste0(params[seq_along(1:(length(params)-1))], ",") + if(length(ls) > 0){ + if(max(lengths(ls)) > 1) { + + for(i in seq_along(ls)){ + params[names(ls[i])] <- cql2_param(ls[i]) + } + + if(length(params) > 1){ + params[seq_along(1:(length(params)-1))] <- paste0(params[seq_along(1:(length(params)-1))], ",") + } } } return(params) diff --git a/R/read_USGS_monitoring_location.R b/R/read_USGS_monitoring_location.R index 3c838a8b..51c985b8 100644 --- a/R/read_USGS_monitoring_location.R +++ b/R/read_USGS_monitoring_location.R @@ -73,6 +73,8 @@ #' #' multi_site <- read_USGS_monitoring_location(state_name = "Wisconsin") #' +#' bbox_vals = c(-94.00, 35.0, -93.5, 35.5) +#' multi_site <- read_USGS_monitoring_location(bbox = bbox_vals) #' } read_USGS_monitoring_location <- function(monitoring_location_id = NA_character_, agency_code = NA_character_, diff --git a/man/read_USGS_monitoring_location.Rd b/man/read_USGS_monitoring_location.Rd index ba74a298..34858c98 100644 --- a/man/read_USGS_monitoring_location.Rd +++ b/man/read_USGS_monitoring_location.Rd @@ -170,6 +170,8 @@ site_info_no_sf <- read_USGS_monitoring_location(monitoring_location_id = site, multi_site <- read_USGS_monitoring_location(state_name = "Wisconsin") +bbox_vals = c(-94.00, 35.0, -93.5, 35.5) +multi_site <- read_USGS_monitoring_location(bbox = bbox_vals) } \dontshow{\}) # examplesIf} } From a0a51570fe72a1309fad7aafa65fcde7201122df Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 27 May 2025 12:52:40 -0500 Subject: [PATCH 044/117] Update R/construct_api_requests.R Co-authored-by: Elise Hinman <121896266+ehinman@users.noreply.github.com> --- R/construct_api_requests.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index 1c810b72..5e28f19b 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -260,7 +260,7 @@ error_body <- function(resp) { x <- httr2::resp_body_json(resp)$error return(x[["message"]]) } else if (status == 403){ - return("Query request denied. Possible reasons include query being exceeding server limits.") + return("Query request denied. Possible reasons include query exceeding server limits.") } } From 5e78cd50c97fa5773fe733558d7054f49270444b Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 27 May 2025 14:10:41 -0500 Subject: [PATCH 045/117] check if offset is too big --- R/construct_api_requests.R | 8 +++++--- R/walk_pages.R | 7 +++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index 5e28f19b..94769371 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -51,9 +51,11 @@ construct_api_requests <- function(service, type = "schema") all_properties <- names(schema$properties) - match.arg(properties, choices = c(all_properties, NA_character_), - several.ok = TRUE) - + if(!is.na(properties)){ + match.arg(properties, choices = all_properties, + several.ok = TRUE) + } + use_sf <- all(pkg.env$local_sf) if(!use_sf){ diff --git a/R/walk_pages.R b/R/walk_pages.R index a1c48997..e2ecd22b 100644 --- a/R/walk_pages.R +++ b/R/walk_pages.R @@ -65,6 +65,13 @@ next_req_url <- function(resp, req) { next_index <- which(sapply(links, function(x) x$rel) == "next") next_url <- links[[next_index]][["href"]] + + offset <- as.integer(sub("(?i).*?\\boffset=?\\s*(\\d+).*", "\\1", next_url)) + if(isTRUE(offset > 40000)){ + warning("Not all data was returned! Split up the query for best results.") + return(NULL) + } + return(httr2::req_url(req = req, url = next_url)) } else { return(NULL) From 14870297c6cf37de339d3898017405f777acf46b Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 27 May 2025 14:40:05 -0500 Subject: [PATCH 046/117] Don't limit to 20 pages. --- R/walk_pages.R | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/R/walk_pages.R b/R/walk_pages.R index e2ecd22b..c3c9655c 100644 --- a/R/walk_pages.R +++ b/R/walk_pages.R @@ -100,7 +100,9 @@ get_resp_data <- function(resp) { walk_pages <- function(req){ - resps <- httr2::req_perform_iterative(req, next_req = next_req_url) + resps <- httr2::req_perform_iterative(req, + next_req = next_req_url, + max_reqs = Inf) failures <- resps |> httr2::resps_failures() |> From f9e766ba63e5f8d46e298366275859a81bce447e Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 27 May 2025 14:47:45 -0500 Subject: [PATCH 047/117] Adding some bbox text --- R/read_USGS_daily.R | 4 +++- R/read_USGS_monitoring_location.R | 4 +++- R/read_USGS_ts_meta.R | 4 +++- man/read_USGS_daily.Rd | 4 +++- man/read_USGS_monitoring_location.Rd | 4 +++- man/read_USGS_ts_meta.Rd | 4 +++- 6 files changed, 18 insertions(+), 6 deletions(-) diff --git a/R/read_USGS_daily.R b/R/read_USGS_daily.R index 5d644492..3a5f96ad 100644 --- a/R/read_USGS_daily.R +++ b/R/read_USGS_daily.R @@ -20,7 +20,9 @@ #' @param bbox Only features that have a geometry that intersects the bounding #' box are selected.The bounding box is provided as four or six numbers, depending #' on whether the coordinate reference system includes a vertical axis (height or -#' depth). Coordinates are assumed to be in crs 4326. +#' depth). Coordinates are assumed to be in crs 4326. The expected format is a numeric +#' vector structured: c(xmin,ymin,xmax,ymax). Another way to think of it is c(Western-most longitude, +#' Southern-most latitude, Eastern-most longitude, Northern-most longitude). #' @param limit The optional limit parameter limits the number of items that are #' presented in the response document. Only items are counted that are on the #' first level of the collection in the response document. Nested objects diff --git a/R/read_USGS_monitoring_location.R b/R/read_USGS_monitoring_location.R index 51c985b8..8c577236 100644 --- a/R/read_USGS_monitoring_location.R +++ b/R/read_USGS_monitoring_location.R @@ -49,7 +49,9 @@ #' @param bbox Only features that have a geometry that intersects the bounding #' box are selected.The bounding box is provided as four or six numbers, depending #' on whether the coordinate reference system includes a vertical axis (height or -#' depth). Coordinates are assumed to be in crs 4326. +#' depth). Coordinates are assumed to be in crs 4326. The expected format is a numeric +#' vector structured: c(xmin,ymin,xmax,ymax). Another way to think of it is c(Western-most longitude, +#' Southern-most latitude, Eastern-most longitude, Northern-most longitude). #' @param limit The optional limit parameter limits the number of items that are #' presented in the response document. Only items are counted that are on the #' first level of the collection in the response document. Nested objects diff --git a/R/read_USGS_ts_meta.R b/R/read_USGS_ts_meta.R index fd2cd519..887da8ee 100644 --- a/R/read_USGS_ts_meta.R +++ b/R/read_USGS_ts_meta.R @@ -24,7 +24,9 @@ #' @param bbox Only features that have a geometry that intersects the bounding #' box are selected.The bounding box is provided as four or six numbers, depending #' on whether the coordinate reference system includes a vertical axis (height or -#' depth). Coordinates are assumed to be in crs 4326. +#' depth). Coordinates are assumed to be in crs 4326. The expected format is a numeric +#' vector structured: c(xmin,ymin,xmax,ymax). Another way to think of it is c(Western-most longitude, +#' Southern-most latitude, Eastern-most longitude, Northern-most longitude). #' @param limit The optional limit parameter limits the number of items that are #' presented in the response document. Only items are counted that are on the #' first level of the collection in the response document. Nested objects diff --git a/man/read_USGS_daily.Rd b/man/read_USGS_daily.Rd index 849ab96a..3b4c8185 100644 --- a/man/read_USGS_daily.Rd +++ b/man/read_USGS_daily.Rd @@ -37,7 +37,9 @@ geometry, id, time_series_id, monitoring_location_id, parameter_code, statistic_ \item{bbox}{Only features that have a geometry that intersects the bounding box are selected.The bounding box is provided as four or six numbers, depending on whether the coordinate reference system includes a vertical axis (height or -depth). Coordinates are assumed to be in crs 4326.} +depth). Coordinates are assumed to be in crs 4326. The expected format is a numeric +vector structured: c(xmin,ymin,xmax,ymax). Another way to think of it is c(Western-most longitude, +Southern-most latitude, Eastern-most longitude, Northern-most longitude).} \item{time_series_id}{A unique identifier representing a single time series. This corresponds to the \code{id} field in the \code{time-series-metadata} endpoint.} diff --git a/man/read_USGS_monitoring_location.Rd b/man/read_USGS_monitoring_location.Rd index 34858c98..1da7369d 100644 --- a/man/read_USGS_monitoring_location.Rd +++ b/man/read_USGS_monitoring_location.Rd @@ -139,7 +139,9 @@ geometry, id, agency_code, agency_name, monitoring_location_number, monitoring_l \item{bbox}{Only features that have a geometry that intersects the bounding box are selected.The bounding box is provided as four or six numbers, depending on whether the coordinate reference system includes a vertical axis (height or -depth). Coordinates are assumed to be in crs 4326.} +depth). Coordinates are assumed to be in crs 4326. The expected format is a numeric +vector structured: c(xmin,ymin,xmax,ymax). Another way to think of it is c(Western-most longitude, +Southern-most latitude, Eastern-most longitude, Northern-most longitude).} \item{limit}{The optional limit parameter limits the number of items that are presented in the response document. Only items are counted that are on the diff --git a/man/read_USGS_ts_meta.Rd b/man/read_USGS_ts_meta.Rd index 33aa2a44..ced421a7 100644 --- a/man/read_USGS_ts_meta.Rd +++ b/man/read_USGS_ts_meta.Rd @@ -46,7 +46,9 @@ contained within the explicitly requested items shall not be counted.} \item{bbox}{Only features that have a geometry that intersects the bounding box are selected.The bounding box is provided as four or six numbers, depending on whether the coordinate reference system includes a vertical axis (height or -depth). Coordinates are assumed to be in crs 4326.} +depth). Coordinates are assumed to be in crs 4326. The expected format is a numeric +vector structured: c(xmin,ymin,xmax,ymax). Another way to think of it is c(Western-most longitude, +Southern-most latitude, Eastern-most longitude, Northern-most longitude).} \item{statistic_id}{A code corresponding to the statistic an observation represents. Example codes include 00001 (max), 00002 (min), and 00003 (mean). A complete list of codes and their descriptions can be found at \url{https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=\%25&fmt=html}.} From b8ded262c6e16e38fcd35156b142a017c76dd9e5 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 27 May 2025 14:50:21 -0500 Subject: [PATCH 048/117] fix error --- R/construct_api_requests.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index 94769371..cd8209e8 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -51,7 +51,7 @@ construct_api_requests <- function(service, type = "schema") all_properties <- names(schema$properties) - if(!is.na(properties)){ + if(!all(is.na(properties))){ match.arg(properties, choices = all_properties, several.ok = TRUE) } From c8cf3aae2b49e70348f5186310b61318a0258880 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 28 May 2025 12:49:32 -0500 Subject: [PATCH 049/117] Add a generalized retrieval function --- NAMESPACE | 1 + R/construct_api_requests.R | 12 ++--- R/read_USGS_data.R | 80 ++++++++++++++++++++++++++++++++++ man/read_USGS_data.Rd | 52 ++++++++++++++++++++++ tests/testthat/tests_general.R | 33 ++++++++++++++ 5 files changed, 172 insertions(+), 6 deletions(-) create mode 100644 R/read_USGS_data.R create mode 100644 man/read_USGS_data.Rd diff --git a/NAMESPACE b/NAMESPACE index 9145e4cc..73f14c7c 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -45,6 +45,7 @@ export(readWQPdata) export(readWQPqw) export(readWQPsummary) export(read_USGS_daily) +export(read_USGS_data) export(read_USGS_monitoring_location) export(read_USGS_samples) export(read_USGS_ts_meta) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index cd8209e8..4dbf7f5f 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -242,11 +242,10 @@ check_OGC_requests <- function(endpoint = "daily", match.arg(endpoint, services) - url_base <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> + req <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> httr2::req_url_path_append(endpoint) |> - httr2::req_url_path_append(type) - - req <- basic_request(url_base) + httr2::req_url_path_append(type) |> + basic_request() query_ret <- req |> httr2::req_perform() |> @@ -309,11 +308,12 @@ get_params <- function(service){ check_queryables_req <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> httr2::req_url_path_append(service) |> - httr2::req_url_path_append("schema") + httr2::req_url_path_append("schema") |> + basic_request() - check_queryables_req <- basic_request(check_queryables_req) query_ret <- httr2::req_perform(check_queryables_req) |> httr2::resp_body_json() + params <- sapply(query_ret$properties, function(x) x[["description"]]) } diff --git a/R/read_USGS_data.R b/R/read_USGS_data.R new file mode 100644 index 00000000..ca795a21 --- /dev/null +++ b/R/read_USGS_data.R @@ -0,0 +1,80 @@ +#' Generalized USGS data retrieval function +#' +#' Function that allows complex CQL queries. +#' See +#' for more information. +#' +#' @export +#' @param service character, can be any existing collection such +#' as "daily", "monitoring-locations", "time-series-metadata" +#' @param CQL A string in a Common Query Language format. +#' @param \dots Additional arguments to send to the request. +#' @examplesIf is_dataRetrieval_user() +#' +#' \donttest{ +#' cql <- '{ +#' "op": "and", +#' "args": [ +#' { +#' "op": "in", +#' "args": [ +#' { "property": "parameter_code" }, +#' [ "00060", "00065" ] +#' ] +#' }, +#' { +#' "op": "in", +#' "args": [ +#' { "property": "monitoring_location_id" }, +#' [ "USGS-07367300", "USGS-03277200" ] +#' ] +#' } +#' ] +#' }' +#' +#' dv_data <- read_USGS_data(service = "daily", +#' CQL = cql, +#' time = c("2023-01-01", "2024-01-01")) +#' +#' } +read_USGS_data <- function(service, + CQL, + ..., + convertType = TRUE){ + + message("Function in development, use at your own risk.") + + check_collections <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/server/getCollections") + + check_endpoints_req <- basic_request(check_collections) |> + httr2::req_perform() |> + httr2::resp_body_json() + + endpoints <- sapply(check_endpoints_req$tags, function(x) x[["name"]]) + + match.arg(service, endpoints) + + args <- list(...) + args[["service"]] <- service + + data_req <- suppressWarnings(do.call(construct_api_requests, args)) + + data_req <- data_req |> + httr2::req_headers(`Content-Type` = "application/query-cql-json") |> + httr2::req_body_raw(CQL) + + return_list <- walk_pages(data_req) + + return_list <- deal_with_empty(return_list, properties, service) + + if(convertType) return_list <- cleanup_cols(return_list) + + return_list <- return_list[order(return_list$time, return_list$monitoring_location_id), ] + + return_list <- rejigger_cols(return_list, args[["properties"]], service) + + return(return_list) +} + + + diff --git a/man/read_USGS_data.Rd b/man/read_USGS_data.Rd new file mode 100644 index 00000000..c6b0fefe --- /dev/null +++ b/man/read_USGS_data.Rd @@ -0,0 +1,52 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/read_USGS_data.R +\name{read_USGS_data} +\alias{read_USGS_data} +\title{Generalized USGS data retrieval function} +\usage{ +read_USGS_data(service, CQL, ..., convertType = TRUE) +} +\arguments{ +\item{service}{character, can be any existing collection such +as "daily", "monitoring-locations", "time-series-metadata"} + +\item{CQL}{A string in a Common Query Language format.} + +\item{\dots}{Additional arguments to send to the request.} +} +\description{ +Function that allows complex CQL queries. +See \url{https://api.waterdata.usgs.gov/docs/ogcapi/complex-queries/} +for more information. +} +\examples{ +\dontshow{if (is_dataRetrieval_user()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} + +\donttest{ +cql <- '{ +"op": "and", +"args": [ + { + "op": "in", + "args": [ + { "property": "parameter_code" }, + [ "00060", "00065" ] + ] + }, + { + "op": "in", + "args": [ + { "property": "monitoring_location_id" }, + [ "USGS-07367300", "USGS-03277200" ] + ] + } +] +}' + +dv_data <- read_USGS_data(service = "daily", + CQL = cql, + time = c("2023-01-01", "2024-01-01")) + +} +\dontshow{\}) # examplesIf} +} diff --git a/tests/testthat/tests_general.R b/tests/testthat/tests_general.R index 563ddd46..74df9528 100644 --- a/tests/testthat/tests_general.R +++ b/tests/testthat/tests_general.R @@ -1,5 +1,38 @@ context("General functions") +test_that("General USGS retrievals working", { + testthat::skip_on_cran() + + cql <- '{ + "op": "and", + "args": [ + { + "op": "in", + "args": [ + { "property": "parameter_code" }, + [ "00060", "00065" ] + ] + }, + { + "op": "in", + "args": [ + { "property": "monitoring_location_id" }, + [ "USGS-07367300", "USGS-03277200" ] + ] + } + ] + }' + + dv_data <- read_USGS_data(service = "daily", + CQL = cql, + time = c("2023-01-01", "2024-01-01")) + expect_equal(as.Date(c("2023-01-01", "2024-01-01")), + range(dv_data$time)) + expect_true(all(unique(dv_data$monitoring_location_id) %in% + c("USGS-07367300", "USGS-03277200"))) + +}) + test_that("General NWIS retrievals working", { testthat::skip_on_cran() From fe90e7b57321505af576dbb30a1a33e8dcf3eb48 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 28 May 2025 13:02:05 -0500 Subject: [PATCH 050/117] add another test and doc --- R/read_USGS_data.R | 8 +++++++- man/read_USGS_data.Rd | 3 +++ tests/testthat/tests_general.R | 26 ++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/R/read_USGS_data.R b/R/read_USGS_data.R index ca795a21..9c9e03ce 100644 --- a/R/read_USGS_data.R +++ b/R/read_USGS_data.R @@ -8,6 +8,8 @@ #' @param service character, can be any existing collection such #' as "daily", "monitoring-locations", "time-series-metadata" #' @param CQL A string in a Common Query Language format. +#' @param convertType logical, defaults to `TRUE`. If `TRUE`, the function +#' will convert the data to dates and qualifier to string vector. #' @param \dots Additional arguments to send to the request. #' @examplesIf is_dataRetrieval_user() #' @@ -57,6 +59,10 @@ read_USGS_data <- function(service, args <- list(...) args[["service"]] <- service + if(!"properties" %in% names(args)){ + args[["properties"]] <- NA_character_ + } + data_req <- suppressWarnings(do.call(construct_api_requests, args)) data_req <- data_req |> @@ -65,7 +71,7 @@ read_USGS_data <- function(service, return_list <- walk_pages(data_req) - return_list <- deal_with_empty(return_list, properties, service) + return_list <- deal_with_empty(return_list, args[["properties"]], service) if(convertType) return_list <- cleanup_cols(return_list) diff --git a/man/read_USGS_data.Rd b/man/read_USGS_data.Rd index c6b0fefe..85406a3e 100644 --- a/man/read_USGS_data.Rd +++ b/man/read_USGS_data.Rd @@ -13,6 +13,9 @@ as "daily", "monitoring-locations", "time-series-metadata"} \item{CQL}{A string in a Common Query Language format.} \item{\dots}{Additional arguments to send to the request.} + +\item{convertType}{logical, defaults to \code{TRUE}. If \code{TRUE}, the function +will convert the data to dates and qualifier to string vector.} } \description{ Function that allows complex CQL queries. diff --git a/tests/testthat/tests_general.R b/tests/testthat/tests_general.R index 74df9528..9891b39d 100644 --- a/tests/testthat/tests_general.R +++ b/tests/testthat/tests_general.R @@ -30,6 +30,32 @@ test_that("General USGS retrievals working", { range(dv_data$time)) expect_true(all(unique(dv_data$monitoring_location_id) %in% c("USGS-07367300", "USGS-03277200"))) + + + cql_not_active <- '{ + "op": "and", + "args": [ + { + "op": "in", + "args": [ + { "property": "parameter_code" }, + [ "00060", "00065" ] + ] + }, + { + "op": "in", + "args": [ + { "property": "monitoring_location_id" }, + [ "USGS-05212700"] + ] + } + ] + }' + + notActiveUSGS <- read_USGS_data(CQL = cql_not_active, + service = "daily", + time = c("2014-01-01", "2014-01-07")) + }) From f2c8d1231635b0f31be5374f224e02f54a401429 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 28 May 2025 17:03:07 -0500 Subject: [PATCH 051/117] add some internal documentation --- R/construct_api_requests.R | 76 +++++++++++++++++++++++- _pkgdown.yml | 13 ++-- tests/testthat/tests_general.R | 58 ++++++++++-------- tests/testthat/tests_userFriendly_fxns.R | 43 ++++++++------ 4 files changed, 137 insertions(+), 53 deletions(-) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index cd8209e8..fd383892 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -194,6 +194,19 @@ explode_post <- function(ls){ return(params) } +#' Create CQL parameters +#' +#' Helps to give more informative messages on some errors. +#' +#' @param parameter named vector +#' @noRd +#' @return list +#' @examples +#' +#' parameter <- list("monitoring_location_id" = c("USGS-02238500", +#' "USGS-01491000")) +#' dataRetrieval:::cql2_param(parameter) +#' cql2_param <- function(parameter){ template_path <- system.file("templates/param.CQL2", package = "dataRetrieval") template <- readChar(template_path, file.info(template_path)$size) @@ -256,6 +269,22 @@ check_OGC_requests <- function(endpoint = "daily", } +#' Custom Error Messages +#' +#' Helps to give more informative messages on some errors. +#' +#' @param resp httr2 response +#' @return list +#' @noRd +#' @examplesIf is_dataRetrieval_user() +#' +#' \donttest{ +#' check_collections <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/server/getCollections") +#' collect_request <- dataRetrieval:::basic_request(check_collections) +#' query_ret <- httr2::req_perform(collect_request) +#' dataRetrieval:::error_body(query_ret) +#' } +#' error_body <- function(resp) { status <- httr2::resp_status(resp) if(status == 429){ @@ -266,6 +295,23 @@ error_body <- function(resp) { } } + +#' Basic request to API services +#' +#' Automatically includes json format, gzip encoding, dataRetrieval +#' user agents, and the X-Api-Key token if available. +#' +#' @param url_base httr2 request +#' @return list +#' @noRd +#' @examplesIf is_dataRetrieval_user() +#' +#' \donttest{ +#' check_collections <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/server/getCollections") +#' collect_request <- dataRetrieval:::basic_request(check_collections) +#' collect_request +#' } +#' basic_request <- function(url_base){ req <- url_base |> @@ -286,7 +332,20 @@ basic_request <- function(url_base){ } -# Create descriptions dynamically +#' Create service descriptions dynamically +#' +#' This function populates the parameter descriptions. +#' +#' @param service Character, can be any of the endpoints +#' @return list +#' @noRd +#' @examplesIf is_dataRetrieval_user() +#' +#' \donttest{ +#' ml_desc <- dataRetrieval:::get_description("monitoring-locations") +#' ml_desc +#' } +#' get_description <- function(service){ check_collections <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/server/getCollections") @@ -304,7 +363,20 @@ get_description <- function(service){ } -# Create parameter descriptions dynamically +#' Create parameter descriptions dynamically +#' +#' This function populates the parameter descriptions. +#' +#' @param service Character, can be any of the endpoints +#' @return list +#' @noRd +#' @examplesIf is_dataRetrieval_user() +#' +#' \donttest{ +#' ml <- dataRetrieval:::get_params("monitoring-locations") +#' ml$national_aquifer_code +#' } +#' get_params <- function(service){ check_queryables_req <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> diff --git a/_pkgdown.yml b/_pkgdown.yml index c915c93e..43122431 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -26,12 +26,6 @@ navbar: href: articles/dataRetrieval.html - text: Function Help href: reference/index.html - - text: Large Data Pulls - menu: - - text: Scripting Approach - href: articles/wqp_large_pull_script.html - - text: Pipeline Approach - href: articles/wqp_large_pull_targets.html - text: Water Quality Data menu: - text: Samples Data @@ -46,12 +40,14 @@ navbar: menu: - text: Tutorial href: articles/tutorial.html - - text: Changes to QW - href: articles/qwdata_changes.html - text: Pivot Data href: articles/long_to_wide.html - text: Join by closest date href: articles/join_by_closest.html + - text: Large Request Scripting Approach + href: articles/wqp_large_pull_script.html + - text: Large Request Pipeline Approach + href: articles/wqp_large_pull_targets.html - text: Stat Service href: articles/statsServiceMap.html - text: NLDI Interface @@ -71,6 +67,7 @@ reference: - read_USGS_daily - read_USGS_ts_meta - read_USGS_monitoring_location + - read_USGS_data - summarize_USGS_samples - check_USGS_sample_params - title: National Water Information System (NWIS) diff --git a/tests/testthat/tests_general.R b/tests/testthat/tests_general.R index 563ddd46..8d6d6c1e 100644 --- a/tests/testthat/tests_general.R +++ b/tests/testthat/tests_general.R @@ -42,14 +42,21 @@ test_that("General NWIS retrievals working", { startDate <- as.Date("2013-10-01") endDate <- as.Date("2014-09-30") - waterYear <- readNWISdata( - bBox = c(-83, 36.5, -81, 38.5), - parameterCd = "00010", - service = "dv", - startDate = startDate, - endDate = endDate + # waterYear <- readNWISdata( + # bBox = c(-83, 36.5, -81, 38.5), + # parameterCd = "00010", + # service = "dv", + # startDate = startDate, + # endDate = endDate + # ) + # expect_is(waterYear$dateTime, "POSIXct") + + waterYear <- read_USGS_daily( + bbox = c(-83, 36.5, -81, 38.5), + parameter_code = "00010", + time = c(startDate, endDate) ) - expect_is(waterYear$dateTime, "POSIXct") + expect_is(waterYear$time, "Date") siteInfo <- readNWISdata( stateCd = "WI", @@ -117,15 +124,15 @@ test_that("General NWIS retrievals working", { instData <- readNWISdata(args) - args <- list( - sites = "05114000", service = "dv", - parameterCd = "00060", - startDate = "2014-05-01", - endDate = "2014-05-01" - ) - - dailyData <- readNWISdata(args) - expect_lt(nrow(dailyData), nrow(instData)) + # args <- list( + # sites = "05114000", service = "dv", + # parameterCd = "00060", + # startDate = "2014-05-01", + # endDate = "2014-05-01" + # ) + # + # dailyData <- readNWISdata(args) + # expect_lt(nrow(dailyData), nrow(instData)) args2 <- list( monitoring_location_id = "USGS-05114000", @@ -183,16 +190,17 @@ test_that("General NWIS retrievals working", { "url" ))) - multi_hucs <- c("07130007", "07130011") - multi_huc <- dataRetrieval::readNWISdata( - huc = multi_hucs, - parameterCd = "63680", - startDate = "2015-06-18", - endDate = "2015-06-18", - service = "dv" - ) - expect_equal(2, nrow(multi_huc)) + # multi_hucs <- c("07130007", "07130011") + # multi_huc <- dataRetrieval::readNWISdata( + # huc = multi_hucs, + # parameterCd = "63680", + # startDate = "2015-06-18", + # endDate = "2015-06-18", + # service = "dv" + # ) + # expect_equal(2, nrow(multi_huc)) + # HUC isn't available in the "daily" service: multi_hucs <- c("07130007", "07130011") multi_huc_new <- read_USGS_monitoring_location( hydrologic_unit_code = multi_hucs diff --git a/tests/testthat/tests_userFriendly_fxns.R b/tests/testthat/tests_userFriendly_fxns.R index d21b9742..c1612610 100644 --- a/tests/testthat/tests_userFriendly_fxns.R +++ b/tests/testthat/tests_userFriendly_fxns.R @@ -86,9 +86,16 @@ test_that("peak, rating curves, surface-water measurements", { siteINFO <- readNWISsite("05114000") expect_is(siteINFO$agency_cd, "character") expect_equal(siteINFO$site_no, "05114000") + + siteINFO_USGS <- read_USGS_monitoring_location("USGS-05114000") + expect_is(siteINFO_USGS$agency_code, "character") + expect_equal(siteINFO_USGS$monitoring_locations_id, "USGS-05114000") - siteINFOMulti <- readNWISsite(c("05114000", "09423350")) - expect_true(nrow(siteINFOMulti) == 2) + # siteINFOMulti <- readNWISsite(c("05114000", "09423350")) + # expect_true(nrow(siteINFOMulti) == 2) + + siteINFOMulti_USGS <- read_USGS_monitoring_location(c("USGS-05114000", "USGS-09423350")) + expect_true(nrow(siteINFOMulti_USGS) == 2) Meas07227500.ex <- readNWISmeas("07227500", expanded = TRUE) expect_is(Meas07227500.ex$measurement_dt, "Date") @@ -120,8 +127,8 @@ test_that("NWIS dv tests", { endDate <- "2012-06-30" pCode <- "00060" - rawDailyQ <- readNWISdv(siteNumber, pCode, startDate, endDate) - expect_is(rawDailyQ$Date, "Date") + # rawDailyQ <- readNWISdv(siteNumber, pCode, startDate, endDate) + # expect_is(rawDailyQ$Date, "Date") raw_USGS_daily <- read_USGS_daily(monitoring_location_id = paste0("USGS-", siteNumber), parameter_code = pCode, @@ -129,12 +136,12 @@ test_that("NWIS dv tests", { expect_is(raw_USGS_daily$time, "Date") - rawDailyQAndTempMeanMax <- readNWISdv(siteNumber, c("00010", "00060"), - startDate, endDate, - statCd = c("00001", "00003") - ) - expect_true(length(grep("00060", names(rawDailyQAndTempMeanMax))) >= 2 & - length(grep("00010", names(rawDailyQAndTempMeanMax))) >= 2) + # rawDailyQAndTempMeanMax <- readNWISdv(siteNumber, c("00010", "00060"), + # startDate, endDate, + # statCd = c("00001", "00003") + # ) + # expect_true(length(grep("00060", names(rawDailyQAndTempMeanMax))) >= 2 & + # length(grep("00010", names(rawDailyQAndTempMeanMax))) >= 2) raw_USGS_TempMeanMax <- read_USGS_daily(monitoring_location_id = paste0("USGS-", siteNumber), parameter_code = c("00010", "00060"), @@ -145,12 +152,12 @@ test_that("NWIS dv tests", { expect_true(length(unique(raw_USGS_TempMeanMax$statistic_id)) == 2) expect_true(length(unique(raw_USGS_TempMeanMax$monitoring_location_id)) == 1) - rawDailyMultiSites <- readNWISdv(c("01491000", "01645000"), - c("00010", "00060"), - startDate, endDate, - statCd = c("00001", "00003") - ) - expect_true(length(unique(rawDailyMultiSites$site_no)) > 1) + # rawDailyMultiSites <- readNWISdv(c("01491000", "01645000"), + # c("00010", "00060"), + # startDate, endDate, + # statCd = c("00001", "00003") + # ) + # expect_true(length(unique(rawDailyMultiSites$site_no)) > 1) raw_USGS_MultiSites <- read_USGS_daily(monitoring_location_id = paste0("USGS-", c("01491000", "01645000")), @@ -161,8 +168,8 @@ test_that("NWIS dv tests", { expect_true(length(unique(raw_USGS_MultiSites$monitoring_location_id)) == 2) site <- "05212700" - notActive <- readNWISdv(site, "00060", "2014-01-01", "2014-01-07") - expect_true(nrow(notActive) == 0) + # notActive <- readNWISdv(site, "00060", "2014-01-01", "2014-01-07") + # expect_true(nrow(notActive) == 0) notActiveUSGS <- read_USGS_daily(monitoring_location_id = paste0("USGS-", site), parameter_code = "00060", From be06d16a88c6de37b109b1abea6a65ebaaa68bd2 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 30 May 2025 14:32:32 -0500 Subject: [PATCH 052/117] cleanup api urls --- R/AAA.R | 1 + R/construct_api_requests.R | 43 +++++++++++++++++++++++++------------- R/read_USGS_data.R | 10 +++------ R/walk_pages.R | 2 +- 4 files changed, 33 insertions(+), 23 deletions(-) diff --git a/R/AAA.R b/R/AAA.R index 762bff68..21abed3c 100644 --- a/R/AAA.R +++ b/R/AAA.R @@ -4,6 +4,7 @@ pkg.env <- new.env() suppressMessages(setAccess("public")) pkg.env$nldi_base <- "https://api.water.usgs.gov/nldi/linked-data/" pkg.env$local_sf <- requireNamespace("sf", quietly = TRUE) + options("dataRetrieval" = list("api_version" = "v0")) } diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index 4dbf7f5f..a77eca54 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -142,9 +142,16 @@ construct_api_requests <- function(service, return(baseURL) } +base_url <- function(){ + + httr2::request("https://api.waterdata.usgs.gov/ogcapi/") |> + httr2::req_url_path_append(getOption("dataRetrieval")$api_version) +} + setup_api <- function(service){ - baseURL <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> + baseURL <- base_url() |> + httr2::req_url_path_append("collections") |> httr2::req_url_path_append(service, "items") |> basic_request() @@ -230,19 +237,14 @@ check_OGC_requests <- function(endpoint = "daily", match.arg(type, c("queryables", "schema")) - check_collections <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/server/getCollections") - - check_endpoints_req <- basic_request(check_collections) - - query_ret <- check_endpoints_req |> - httr2::req_perform() |> - httr2::resp_body_json() + query_ret <- get_collection() services <- sapply(query_ret$tags, function(x) x[["name"]]) match.arg(endpoint, services) - req <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> + req <- base_url() |> + httr2::req_url_path_append("collections") |> httr2::req_url_path_append(endpoint) |> httr2::req_url_path_append(type) |> basic_request() @@ -287,12 +289,9 @@ basic_request <- function(url_base){ # Create descriptions dynamically get_description <- function(service){ + + query_ret <- get_collection() - check_collections <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/server/getCollections") - - check_endpoints_req <- basic_request(check_collections) - query_ret <- httr2::req_perform(check_endpoints_req) |> - httr2::resp_body_json() tags <- query_ret[["tags"]] service_index <- which(sapply(tags, function(x){ @@ -303,10 +302,24 @@ get_description <- function(service){ } +get_collection <- function(){ + + check_collections <- base_url() |> + httr2::req_url_path_append("openapi?f=html#/server/getCollections") + + check_endpoints_req <- basic_request(check_collections) + + query_ret <- httr2::req_perform(check_endpoints_req) |> + httr2::resp_body_json() + + return(query_ret) +} + # Create parameter descriptions dynamically get_params <- function(service){ - check_queryables_req <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/collections") |> + check_queryables_req <- base_url() |> + httr2::req_url_path_append("collections") |> httr2::req_url_path_append(service) |> httr2::req_url_path_append("schema") |> basic_request() diff --git a/R/read_USGS_data.R b/R/read_USGS_data.R index 9c9e03ce..71f82220 100644 --- a/R/read_USGS_data.R +++ b/R/read_USGS_data.R @@ -45,14 +45,10 @@ read_USGS_data <- function(service, convertType = TRUE){ message("Function in development, use at your own risk.") + + query_req <- get_collection() - check_collections <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/server/getCollections") - - check_endpoints_req <- basic_request(check_collections) |> - httr2::req_perform() |> - httr2::resp_body_json() - - endpoints <- sapply(check_endpoints_req$tags, function(x) x[["name"]]) + endpoints <- sapply(query_req$tags, function(x) x[["name"]]) match.arg(service, endpoints) diff --git a/R/walk_pages.R b/R/walk_pages.R index c3c9655c..739a2f56 100644 --- a/R/walk_pages.R +++ b/R/walk_pages.R @@ -57,7 +57,7 @@ next_req_url <- function(resp, req) { header_info <- httr2::resp_headers(resp) if(Sys.getenv("API_USGS_PAT") != ""){ - message("Remaining requests this hour:", header_info$`x-ratelimit-remaining`) + message("Remaining requests this hour:", header_info$`x-ratelimit-remaining`, " ") } links <- body$links From 38e64018b7dcff22060edb7210fe30e8b9eb053c Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Mon, 2 Jun 2025 10:54:35 -0500 Subject: [PATCH 053/117] add some roxygen --- R/construct_api_requests.R | 85 +++++++++++++++++++++++++++++-- tests/testthat/tests_general.R | 91 ++++++++++++++++------------------ tests/testthat/tests_nldi.R | 2 +- 3 files changed, 124 insertions(+), 54 deletions(-) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index aa0a38c0..e368959e 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -78,9 +78,6 @@ construct_api_requests <- function(service, POST <- FALSE - template_path_post <- system.file("templates/post.CQL2", package = "dataRetrieval") - template_post <- readChar(template_path_post, file.info(template_path_post)$size) - single_params <- c("datetime", "last_modified", "begin", "end", "time") full_list <- list(...) @@ -132,6 +129,9 @@ construct_api_requests <- function(service, "params" = unname(post_params) ) + template_path_post <- system.file("templates/post.CQL2", package = "dataRetrieval") + template_post <- readChar(template_path_post, file.info(template_path_post)$size) + x <- whisker::whisker.render(template_post, post_params) baseURL <- httr2::req_body_raw(baseURL, x) @@ -142,12 +142,32 @@ construct_api_requests <- function(service, return(baseURL) } +#' Setup the request for the OGC API requests +#' +#' @noRd +#' @return httr2 request +#' @examplesIf is_dataRetrieval_user() +#' +#' \donttest{ +#' request <- dataRetrieval:::base_url() +#' request +#' } base_url <- function(){ httr2::request("https://api.waterdata.usgs.gov/ogcapi/") |> httr2::req_url_path_append(getOption("dataRetrieval")$api_version) } +#' Setup the request for a particular endpoint collection +#' +#' @noRd +#' @return httr2 request +#' @examplesIf is_dataRetrieval_user() +#' +#' \donttest{ +#' request <- dataRetrieval:::setup_api("daily") +#' request +#' } setup_api <- function(service){ baseURL <- base_url() |> @@ -157,6 +177,31 @@ setup_api <- function(service){ } +#' Format the date request +#' +#' Users will want to give either start/end dates or +#' period requests. +#' +#' +#' @noRd +#' @return character vector with a length of either 1 or 2. +#' @examples +#' +#' start_end <- c("2021-01-01", "2022-01-01") +#' dataRetrieval:::format_api_dates(start_end) +#' +#' period <- "P7D" +#' dataRetrieval:::format_api_dates(period) +#' +#' start <- c("2021-01-01", NA) +#' dataRetrieval:::format_api_dates(start) +#' +#' end <- c(NA, "2021-01-01") +#' dataRetrieval:::format_api_dates(end) +#' +#' start_end <- as.POSIXct(c("2021-01-01 12:15:00", "2022-01-01 16:45")) +#' dataRetrieval:::format_api_dates(start_end) +#' format_api_dates <- function(datetime){ if(!any(isTRUE(is.na(datetime)) | isTRUE(is.null(datetime)))){ @@ -181,6 +226,21 @@ format_api_dates <- function(datetime){ return(datetime) } +#' Turn request list into POST body cql +#' +#' @noRd +#' @return character vector of CQL filters +#' @examplesIf is_dataRetrieval_user() +#' +#' \donttest{ +#' +#' query_list <- list(monitoring_location_id = c("USGS-01491000", +#' "USGS-01645000"), +#' parameter_code = c("00060", "00010")) +#' +#' dataRetrieval:::explode_post(query_list) +#' +#' } explode_post <- function(ls){ ls <- Filter(Negate(anyNA), ls) @@ -280,7 +340,9 @@ check_OGC_requests <- function(endpoint = "daily", #' @examplesIf is_dataRetrieval_user() #' #' \donttest{ -#' check_collections <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/server/getCollections") +#' check_collections <- dataRetrieval:::base_url() |> +#' httr2::req_url_path_append("openapi?f=html#/server/getCollections") +#' #' collect_request <- dataRetrieval:::basic_request(check_collections) #' query_ret <- httr2::req_perform(collect_request) #' dataRetrieval:::error_body(query_ret) @@ -308,7 +370,8 @@ error_body <- function(resp) { #' @examplesIf is_dataRetrieval_user() #' #' \donttest{ -#' check_collections <- httr2::request("https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/server/getCollections") +#' check_collections <- dataRetrieval:::base_url() |> +#' httr2::req_url_path_append("openapi?f=html#/server/getCollections") #' collect_request <- dataRetrieval:::basic_request(check_collections) #' collect_request #' } @@ -361,6 +424,18 @@ get_description <- function(service){ } +#' Get collection response +#' +#' +#' @return httr2 response +#' @noRd +#' @examplesIf is_dataRetrieval_user() +#' +#' \donttest{ +#' collection <- dataRetrieval:::get_collection() +#' collection +#' } +#' get_collection <- function(){ check_collections <- base_url() |> diff --git a/tests/testthat/tests_general.R b/tests/testthat/tests_general.R index d2a04666..8b110bbf 100644 --- a/tests/testthat/tests_general.R +++ b/tests/testthat/tests_general.R @@ -314,57 +314,52 @@ test_that("whatNWISdata", { test_that("General WQP retrievals working", { testthat::skip_on_cran() nameToUse <- "pH" - # pHData <- readWQPdata(siteid = "USGS-04024315", - # characteristicName = nameToUse, - # service = "ResultWQX3") - # expect_is(pHData$Activity_StartDateTime, "POSIXct") + pHData <- readWQPdata(siteid = "USGS-04024315", + characteristicName = nameToUse, + service = "ResultWQX3") + expect_is(pHData$Activity_StartDateTime, "POSIXct") # # # testing lists: - # startDate <- as.Date("2022-01-01") - # secchi.names <- c("Depth, Secchi disk depth", - # "Secchi depth", - # "Water transparency, Secchi disc", - # "Depth, Secchi disk depth (choice list)") - # # "Transparency, Secchi tube with disk", - # # "Secchi Reading Condition (choice list)", - # # "Depth, Secchi disk visible at bottom (Y/N) (choice list)") - # - # args_2 <- list( - # "startDateLo" = startDate, - # "startDateHi" = "2024-01-01", - # statecode = "WI", - # characteristicName = secchi.names - # ) - # + startDate <- as.Date("2022-01-01") + secchi.names <- c("Depth, Secchi disk depth", + "Secchi depth", + "Water transparency, Secchi disc", + "Depth, Secchi disk depth (choice list)") + # "Transparency, Secchi tube with disk", + # "Secchi Reading Condition (choice list)", + # "Depth, Secchi disk visible at bottom (Y/N) (choice list)") + + args_2 <- list( + "startDateLo" = startDate, + "startDateHi" = "2024-01-01", + statecode = "WI", + characteristicName = secchi.names + ) + # # Testing multiple lists: - # arg_3 <- list( - # "startDateLo" = startDate, - # "startDateHi" = "2023-12-31" - # ) - # arg_4 <- list( - # statecode = "WI", - # characteristicName = secchi.names - # ) - # - # lakeData <- readWQPdata(args_2, ignore_attributes = TRUE) - # expect_true(nrow(lakeData) > 0) - # lakeSites <- whatWQPsites(args_2) - # expect_type(lakeSites, "list") - # - # wqp.summary_no_atts <- readWQPdata( - # siteid = "USGS-04024315", - # characteristicName = nameToUse, - # ignore_attributes = TRUE, - # service = "ResultWQX3" - # ) - # expect_true(!all(c("siteInfo", "variableInfo") %in% names(attributes(wqp.summary_no_atts)))) - # - # rawPcode <- readWQPqw("USGS-01594440", "01075", "", "", legacy = FALSE) - # expect_true(all(c("url", "queryTime", "siteInfo", "headerInfo") %in% - # names(attributes(rawPcode)))) - # - # # This means wqp_check_status was called: - # expect_true("dataProviders" %in% names(attr(rawPcode, "headerInfo"))) + arg_3 <- list( + "startDateLo" = startDate, + "startDateHi" = "2023-12-31" + ) + arg_4 <- list( + statecode = "WI", + characteristicName = secchi.names + ) + + wqp.summary_no_atts <- readWQPdata( + siteid = "USGS-04024315", + characteristicName = nameToUse, + ignore_attributes = TRUE, + service = "ResultWQX3" + ) + expect_true(!all(c("siteInfo", "variableInfo") %in% names(attributes(wqp.summary_no_atts)))) + + rawPcode <- readWQPqw("USGS-01594440", "01075", "", "", legacy = FALSE) + expect_true(all(c("url", "queryTime", "siteInfo", "headerInfo") %in% + names(attributes(rawPcode)))) + + # This means wqp_check_status was called: + expect_true("dataProviders" %in% names(attr(rawPcode, "headerInfo"))) rawPcode2 <- readWQPqw("USGS-01594440", "01075", "", "", ignore_attributes = TRUE) expect_true(all(!c( "queryTime", "siteInfo") %in% diff --git a/tests/testthat/tests_nldi.R b/tests/testthat/tests_nldi.R index 44de3544..7aae6d05 100644 --- a/tests/testthat/tests_nldi.R +++ b/tests/testthat/tests_nldi.R @@ -84,7 +84,7 @@ test_that("NLDI navigation sources...", { expect_error(findNLDI(nwis = "11120000", nav = c("DT"), warn = FALSE)) expect_error(findNLDI(nwis = "11120000", nav = c("DT", "UM"), warn = FALSE)) # WARNING: Data not found - expect_warning(findNLDI(comid = 101, nav = "UM", find = "nwis", warn = TRUE)) + # expect_warning(findNLDI(comid = 101, nav = "UM", find = "nwis", warn = TRUE)) }) test_that("NLDI find sources...", { From c841abb4fe6f34a838b4a13fa4e83a02cd113f92 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Mon, 2 Jun 2025 13:13:03 -0500 Subject: [PATCH 054/117] add more docs --- R/walk_pages.R | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/R/walk_pages.R b/R/walk_pages.R index 739a2f56..cc2ab8f7 100644 --- a/R/walk_pages.R +++ b/R/walk_pages.R @@ -1,6 +1,21 @@ +#' Return a data frame if there's an empty response +#' +#' +#' @return data.frame +#' @noRd +#' @examples +#' +#' df <- dataRetrieval:::deal_with_empty(data.frame(NULL), +#' properties = c("time", "value"), +#' service = "daily") +#' +#' df2 <- dataRetrieval:::deal_with_empty(data.frame(NULL), +#' properties = NA, +#' service = "daily") +#' deal_with_empty <- function(return_list, properties, service){ if(nrow(return_list) == 0){ - if(is.na(properties)){ + if(all(is.na(properties))){ schema <- check_OGC_requests(endpoint = service, type = "schema") properties <- names(schema$properties) } From 10b7cfe23a76557f2cfc36df16b84e5f8301ea8a Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Mon, 2 Jun 2025 13:50:13 -0500 Subject: [PATCH 055/117] more docs --- R/read_USGS_daily.R | 3 +- R/walk_pages.R | 92 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/R/read_USGS_daily.R b/R/read_USGS_daily.R index 3a5f96ad..2abb31a8 100644 --- a/R/read_USGS_daily.R +++ b/R/read_USGS_daily.R @@ -94,7 +94,8 @@ read_USGS_daily <- function(monitoring_location_id = NA_character_, return_list <- deal_with_empty(return_list, properties, service) - if(convertType) return_list <- cleanup_cols(return_list) + if(convertType) return_list <- cleanup_cols(return_list, + service = "daily") return_list <- return_list[order(return_list$time, return_list$monitoring_location_id), ] diff --git a/R/walk_pages.R b/R/walk_pages.R index cc2ab8f7..a5476b2e 100644 --- a/R/walk_pages.R +++ b/R/walk_pages.R @@ -1,5 +1,9 @@ #' Return a data frame if there's an empty response #' +#' @param return_list data frame returned from walk_pages +#' @param properties A vector of requested columns +#' @param service character, can be any existing collection such +#' as "daily", "monitoring-locations", "time-series-metadata" #' #' @return data.frame #' @noRd @@ -26,6 +30,28 @@ deal_with_empty <- function(return_list, properties, service){ return(return_list) } +#' Rejigger and rename +#' +#' Reorder columns based on users property input. +#' Add "service" prefix to returned "id" column. +#' This allows better integration with other endpoints. +#' +#' @param df data frame returned from walk_pages +#' @param properties A vector of requested columns +#' @param service character, can be any existing collection such +#' as "daily", "monitoring-locations", "time-series-metadata" +#' +#' @return data.frame +#' @noRd +#' @examples +#' +#' df <- dataRetrieval:::deal_with_empty(data.frame(NULL), +#' properties = c("time", "value", "id"), +#' service = "daily") +#' df2 <- dataRetrieval:::rejigger_cols(df, +#' properties = c("value", "id", "time"), +#' service = "daily") +#' rejigger_cols <- function(df, properties, service){ new_id <- paste0(gsub("-", "_", service), "_id") names(df)[names(df) == "id"] <- new_id @@ -37,7 +63,31 @@ rejigger_cols <- function(df, properties, service){ df } -cleanup_cols <- function(df){ + +#' Convert columns if needed +#' +#' These are columns that have caused problems in testing. +#' Mostly if the columns are empty on 1 page, but not the next. +#' The qualifier column also comes back as a list column. This +#' is fine for many, but others prefer a character column. +#' +#' +#' @param df data frame returned from walk_pages +#' @param service character, can be any existing collection such +#' as "daily" +#' @return data.frame +#' @noRd +#' @examples +#' +#' df <- dataRetrieval:::deal_with_empty(data.frame(NULL), +#' properties = c("time", "value", "id", "qualifier"), +#' service = "daily") +#' df2 <- dataRetrieval:::rejigger_cols(df, +#' properties = c("value", "id", "time", "qualifier"), +#' service = "daily") +#' df3 <- dataRetrieval:::cleanup_cols(df2) +#' +cleanup_cols <- function(df, service = "daily"){ if("qualifier" %in% names(df)){ if(!all(is.na(df$qualifier))){ @@ -48,7 +98,10 @@ cleanup_cols <- function(df){ } if("time" %in% names(df)){ - df$time <- as.Date(df$time) + if(service == "daily"){ + df$time <- as.Date(df$time) + } + # leave some room here for POSIXct in the other services. } if("value" %in% names(df)){ @@ -62,6 +115,17 @@ cleanup_cols <- function(df){ df } +#' Next request URL +#' +#' Custom function to find the "next" URL from the API response. +#' @seealso [httr2::req_perform_iterative] +#' +#' @param resp httr2 response from last request +#' @param req httr2 request from last time +#' +#' @noRd +#' @return the url for the next request +#' next_req_url <- function(resp, req) { body <- httr2::resp_body_json(resp) @@ -81,11 +145,15 @@ next_req_url <- function(resp, req) { next_url <- links[[next_index]][["href"]] + ################################################ + # This offset check will be going away + # offset should be replaced by "cursor" eventually. offset <- as.integer(sub("(?i).*?\\boffset=?\\s*(\\d+).*", "\\1", next_url)) if(isTRUE(offset > 40000)){ warning("Not all data was returned! Split up the query for best results.") return(NULL) } + ################################################ return(httr2::req_url(req = req, url = next_url)) } else { @@ -93,6 +161,16 @@ next_req_url <- function(resp, req) { } } +#' Get single response data frame +#' +#' Depending on skipGeometry to decide to use sf or not. +#' +#' @noRd +#' +#' @param resp httr2 response from last request +#' +#' @return data.frame +#' get_resp_data <- function(resp) { body <- httr2::resp_body_json(resp) @@ -113,12 +191,21 @@ get_resp_data <- function(resp) { } +#' Walk through the pages +#' +#' @param req httr2 initial request +#' +#' @noRd +#' @return data.frame with attributes walk_pages <- function(req){ resps <- httr2::req_perform_iterative(req, next_req = next_req_url, max_reqs = Inf) + ###################################### + # So far I haven't tested this because I haven't had + # individual failures failures <- resps |> httr2::resps_failures() |> httr2::resps_requests() @@ -126,6 +213,7 @@ walk_pages <- function(req){ if(length(failures) > 0){ message("There were", length(failures), "failed requests.") } + ###################################### return_list <- data.frame() for(resp in resps){ From 1a74342d683d13e6ed46e7393bd51be6efdb65e5 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Mon, 2 Jun 2025 14:19:02 -0500 Subject: [PATCH 056/117] Add deprecation and make Status page more prominent in pkgdown site. --- R/AAA.R | 7 +++++++ R/readNWISdata.R | 11 +++++++++++ R/readNWISpCode.R | 3 +++ R/readNWISunit.R | 9 +++++++++ R/whatNWISdata.R | 5 +++++ R/whatNWISsites.R | 5 +++++ _pkgdown.yml | 6 ++++-- 7 files changed, 44 insertions(+), 2 deletions(-) diff --git a/R/AAA.R b/R/AAA.R index 21abed3c..1200e175 100644 --- a/R/AAA.R +++ b/R/AAA.R @@ -52,3 +52,10 @@ discrete water quality data newer than March 11, 2024. For additional details, see: https://doi-usgs.github.io/dataRetrieval/articles/Status.html") } + +new_nwis_message <- function(){ + return("ALERT: All NWIS services are slated for decommission. +For up-to-date information, see: +https://doi-usgs.github.io/dataRetrieval/articles/Status.html") +} + diff --git a/R/readNWISdata.R b/R/readNWISdata.R index 4b070827..3a5f3f62 100644 --- a/R/readNWISdata.R +++ b/R/readNWISdata.R @@ -221,6 +221,17 @@ Please see vignette('qwdata_changes', package = 'dataRetrieval') for more information. https://cran.r-project.org/web/packages/dataRetrieval/vignettes/qwdata_changes.html" ) + } else if (service == "dv"){ + .Deprecated(new = "read_USGS_daily", + package = "dataRetrieval", + msg = "NWIS servers are slated for decommission. Please begin to migrate to read_USGS_daily.") + + } else if (service == "site"){ + .Deprecated(new = "read_USGS_monitoring_location", + package = "dataRetrieval", + msg = "NWIS servers are slated for decommission. Please begin to migrate to read_USGS_monitoring_location") + } else { + message(new_nwis_message()) } baseURL <- httr2::request(pkg.env[[service]]) diff --git a/R/readNWISpCode.R b/R/readNWISpCode.R index 93b9431a..065d5a5e 100644 --- a/R/readNWISpCode.R +++ b/R/readNWISpCode.R @@ -31,6 +31,9 @@ #' #' } readNWISpCode <- function(parameterCd) { + + message(new_nwis_message()) + parameterCd.orig <- parameterCd parameterCd <- parameterCd[!is.na(parameterCd)] baseURL <- httr2::request(pkg.env[["pCode"]]) diff --git a/R/readNWISunit.R b/R/readNWISunit.R index 63568b14..f97a2df7 100644 --- a/R/readNWISunit.R +++ b/R/readNWISunit.R @@ -175,6 +175,8 @@ readNWISpeak <- function(siteNumbers, asDateTime = TRUE, convertType = TRUE) { + message(new_nwis_message()) + # Doesn't seem to be a peak xml service url <- constructNWISURL( siteNumbers = siteNumbers, @@ -266,6 +268,7 @@ readNWISpeak <- function(siteNumbers, #' } readNWISrating <- function(siteNumber, type = "base", convertType = TRUE) { + message(new_nwis_message()) # No rating xml service url <- constructNWISURL(siteNumber, service = "rating", ratingType = type) @@ -357,6 +360,7 @@ readNWISmeas <- function(siteNumbers, expanded = FALSE, convertType = TRUE) { + message(new_nwis_message()) # Doesn't seem to be a WaterML1 format option url <- constructNWISURL( siteNumbers = siteNumbers, @@ -505,6 +509,9 @@ readNWISgwl <- function(siteNumbers, endDate = "", parameterCd = NA, convertType = TRUE, tz = "UTC") { + + message(new_nwis_message()) + url <- constructNWISURL( siteNumbers = siteNumbers, parameterCd = parameterCd, @@ -616,6 +623,7 @@ readNWISgwl <- function(siteNumbers, readNWISstat <- function(siteNumbers, parameterCd, startDate = "", endDate = "", convertType = TRUE, statReportType = "daily", statType = "mean") { + message(new_nwis_message()) # check for NAs in site numbers if (any(is.na(siteNumbers))) { siteNumbers <- siteNumbers[!is.na(siteNumbers)] @@ -720,6 +728,7 @@ readNWISuse <- function(stateCd, categories = "ALL", convertType = TRUE, transform = FALSE) { + message(new_nwis_message()) countyID <- NULL countyCd <- countyCd[countyCd != ""] diff --git a/R/whatNWISdata.R b/R/whatNWISdata.R index 07dc0a6c..578dc2e8 100644 --- a/R/whatNWISdata.R +++ b/R/whatNWISdata.R @@ -90,6 +90,11 @@ whatNWISdata <- function(..., convertType = TRUE) { matchReturn <- convertLists(...) prewarned <- FALSE + + .Deprecated(new = "read_USGS_ts_meta", + package = "dataRetrieval", + msg = "NWIS servers are slated for decommission. Please begin to migrate to read_USGS_ts_meta") + if ("service" %in% names(matchReturn)) { service <- matchReturn$service diff --git a/R/whatNWISsites.R b/R/whatNWISsites.R index 221a5216..32ef026a 100644 --- a/R/whatNWISsites.R +++ b/R/whatNWISsites.R @@ -35,6 +35,11 @@ #' } whatNWISsites <- function(...) { + .Deprecated(new = "read_USGS_monitoring_location", + package = "dataRetrieval", + msg = "NWIS servers are slated for decommission. Please begin to migrate to read_USGS_monitoring_location") + + matchReturn <- convertLists(...) if ("service" %in% names(matchReturn)) { service <- matchReturn$service diff --git a/_pkgdown.yml b/_pkgdown.yml index 43122431..3162e95e 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -22,8 +22,8 @@ template: pkgdown-nav-height: 125px navbar: left: - - text: Background - href: articles/dataRetrieval.html + - text: Status + href: articles/Status.html - text: Function Help href: reference/index.html - text: Water Quality Data @@ -40,6 +40,8 @@ navbar: menu: - text: Tutorial href: articles/tutorial.html + - text: Background + href: articles/dataRetrieval.html - text: Pivot Data href: articles/long_to_wide.html - text: Join by closest date From 85ffddfd68b0d18809fbd74ccbc11366564fadaf Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Mon, 2 Jun 2025 16:20:39 -0500 Subject: [PATCH 057/117] Update tests and examples --- R/readNWISdata.R | 35 +--------- R/readNWISdv.R | 31 ++------- R/readNWISsite.R | 11 ++-- R/whatNWISdata.R | 26 ++------ R/whatNWISsites.R | 9 ++- man/readNWISdata.Rd | 35 +--------- man/readNWISdv.Rd | 29 +-------- man/readNWISsite.Rd | 9 ++- man/whatNWISdata.Rd | 24 +------ man/whatNWISsites.Rd | 7 +- tests/testthat/tests_general.R | 82 +++++++++--------------- tests/testthat/tests_userFriendly_fxns.R | 26 +------- 12 files changed, 65 insertions(+), 259 deletions(-) diff --git a/R/readNWISdata.R b/R/readNWISdata.R index 3a5f3f62..8179fcde 100644 --- a/R/readNWISdata.R +++ b/R/readNWISdata.R @@ -71,13 +71,12 @@ #' queryTime \tab POSIXct \tab The time the data was returned \cr #' } #' -#' @seealso [renameNWISColumns()], [importWaterML1()], [importRDB1()] +#' @seealso [read_USGS_data()] #' @export #' @examplesIf is_dataRetrieval_user() #' \donttest{ #' # Examples not run for time considerations #' -#' dataTemp <- readNWISdata(stateCd = "OH", parameterCd = "00010", service = "dv") #' instFlow <- readNWISdata( #' sites = "05114000", service = "iv", #' parameterCd = "00060", @@ -96,26 +95,7 @@ #' service = "iv", parameterCd = "00060" #' ) #' -#' bBoxEx <- readNWISdata(bBox = c(-83, 36.5, -81, 38.5), parameterCd = "00010") #' -#' startDate <- as.Date("2013-10-01") -#' endDate <- as.Date("2014-09-30") -#' waterYear <- readNWISdata( -#' bBox = c(-83, 36.5, -82.5, 36.75), -#' parameterCd = "00010", -#' service = "dv", -#' startDate = startDate, -#' endDate = endDate -#' ) -#' -#' siteInfo <- readNWISdata( -#' stateCd = "WI", parameterCd = "00010", -#' hasDataTypeCd = "iv", service = "site" -#' ) -#' temp <- readNWISdata( -#' bBox = c(-83, 36.5, -82.5, 36.75), parameterCd = "00010", service = "site", -#' seriesCatalogOutput = TRUE -#' ) #' GWL <- readNWISdata(site_no = c("392725077582401", #' "375907091432201"), #' parameterCd = "62610", @@ -160,19 +140,6 @@ #' ) #' allDailyStats_2 <- readNWISdata(arg.list, service = "stat") #' -#' # use county names to get data -#' dailyStaffordVA <- readNWISdata( -#' stateCd = "Virginia", -#' countyCd = "Stafford", -#' parameterCd = "00060", -#' startDate = "2015-01-01", -#' endDate = "2015-01-30" -#' ) -#' va_counties <- c("51001", "51003", "51005", "51007", "51009", "51011", "51013", "51015") -#' va_counties_data <- readNWISdata( -#' startDate = "2015-01-01", endDate = "2015-12-31", -#' parameterCd = "00060", countycode = va_counties -#' ) #' #' site_id <- "01594440" #' rating_curve <- readNWISdata(service = "rating", site_no = site_id, file_type = "base") diff --git a/R/readNWISdv.R b/R/readNWISdv.R index 592d2691..6f4564e4 100644 --- a/R/readNWISdv.R +++ b/R/readNWISdv.R @@ -49,34 +49,13 @@ #' queryTime \tab POSIXct \tab The time the data was returned \cr #' } #' -#' @seealso [renameNWISColumns()], [importWaterML1()] +#' @seealso [read_USGS_daily()] #' @export #' @keywords data import USGS web service -#' @examplesIf is_dataRetrieval_user() -#' site_id <- "04085427" -#' startDate <- "2012-01-01" -#' endDate <- "2012-06-30" -#' pCode <- "00060" -#' \donttest{ -#' rawDailyQ <- readNWISdv(site_id, pCode, startDate, endDate) -#' rawDailyQAndTempMeanMax <- readNWISdv(site_id, c("00010", "00060"), -#' startDate, endDate, -#' statCd = c("00001", "00003") -#' ) -#' rawDailyQAndTempMeanMax <- renameNWISColumns(rawDailyQAndTempMeanMax) -#' rawDailyMultiSites <- readNWISdv(c("01491000", "01645000"), c("00010", "00060"), -#' startDate, endDate, -#' statCd = c("00001", "00003") -#' ) -#' # Site with no data: -#' x <- readNWISdv("10258500", "00060", "2014-09-08", "2014-09-14") -#' names(attributes(x)) -#' attr(x, "siteInfo") -#' attr(x, "variableInfo") -#' -#' site <- "05212700" -#' notActive <- readNWISdv(site, "00060", "2014-01-01", "2014-01-07") -#' } +#' @examples +#' +#' # see ?read_USGS_daily +#' readNWISdv <- function(siteNumbers, parameterCd, startDate = "", diff --git a/R/readNWISsite.R b/R/readNWISsite.R index d3fa4de1..e8d44160 100644 --- a/R/readNWISsite.R +++ b/R/readNWISsite.R @@ -59,12 +59,11 @@ #' comment \tab character \tab Header comments from the RDB file \cr #' } #' @export -#' @examplesIf is_dataRetrieval_user() -#' \donttest{ -#' -#' siteINFO <- readNWISsite("05114000") -#' siteINFOMulti <- readNWISsite(c("05114000", "09423350")) -#' } +#' @seealso [read_USGS_monitoring_location()] +#' @examples +#' +#' # see ?read_USGS_monitoring_location +#' readNWISsite <- function(siteNumbers) { .Deprecated(new = "read_USGS_monitoring_location", diff --git a/R/whatNWISdata.R b/R/whatNWISdata.R index 578dc2e8..8a5274dd 100644 --- a/R/whatNWISdata.R +++ b/R/whatNWISdata.R @@ -63,29 +63,11 @@ #' queryTime \tab POSIXct \tab The time the data was returned \cr #' } #' @export -#' @examplesIf is_dataRetrieval_user() -#' \donttest{ -#' -#' availableData <- whatNWISdata(siteNumber = "05114000") +#' @seealso [read_USGS_ts_meta()] +#' @examples +#' +#' # see ?read_USGS_ts_meta #' -#' # To find just unit value ('instantaneous') data: -#' uvData <- whatNWISdata(siteNumber = "05114000", -#' service = "uv") -#' uvDataMulti <- whatNWISdata(siteNumber = c("05114000", "09423350"), -#' service = c("uv", "dv")) -#' flowAndTemp <- whatNWISdata( -#' stateCd = "WI", service = "dv", -#' parameterCd = c("00060", "00010"), -#' statCd = "00003" -#' ) -#' sites <- whatNWISdata(stateCd = "WI", -#' parameterCd = "00060", -#' siteType = "ST", -#' service = "site") -#' -#' sites <- whatNWISdata(stateCd = "WI", -#' service = "gwlevels") -#' } whatNWISdata <- function(..., convertType = TRUE) { matchReturn <- convertLists(...) diff --git a/R/whatNWISsites.R b/R/whatNWISsites.R index 32ef026a..c6d7df08 100644 --- a/R/whatNWISsites.R +++ b/R/whatNWISsites.R @@ -26,13 +26,12 @@ #' queryTime \tab POSIXct \tab The time the data was returned \cr #' } #' @export +#' @seealso [read_USGS_monitoring_location()] #' #' @examples -#' \donttest{ -#' -#' siteListPhos <- whatNWISsites(stateCd = "OH", parameterCd = "00665") -#' oneSite <- whatNWISsites(sites = "05114000") -#' } +#' +#' # see ?read_USGS_monitoring_location +#' whatNWISsites <- function(...) { .Deprecated(new = "read_USGS_monitoring_location", diff --git a/man/readNWISdata.Rd b/man/readNWISdata.Rd index c1fb917a..6d7d90db 100644 --- a/man/readNWISdata.Rd +++ b/man/readNWISdata.Rd @@ -89,7 +89,6 @@ Note that when period is used all data up to the most recent value are returned. \donttest{ # Examples not run for time considerations -dataTemp <- readNWISdata(stateCd = "OH", parameterCd = "00010", service = "dv") instFlow <- readNWISdata( sites = "05114000", service = "iv", parameterCd = "00060", @@ -108,26 +107,7 @@ multiSite <- readNWISdata( service = "iv", parameterCd = "00060" ) -bBoxEx <- readNWISdata(bBox = c(-83, 36.5, -81, 38.5), parameterCd = "00010") -startDate <- as.Date("2013-10-01") -endDate <- as.Date("2014-09-30") -waterYear <- readNWISdata( - bBox = c(-83, 36.5, -82.5, 36.75), - parameterCd = "00010", - service = "dv", - startDate = startDate, - endDate = endDate -) - -siteInfo <- readNWISdata( - stateCd = "WI", parameterCd = "00010", - hasDataTypeCd = "iv", service = "site" -) -temp <- readNWISdata( - bBox = c(-83, 36.5, -82.5, 36.75), parameterCd = "00010", service = "site", - seriesCatalogOutput = TRUE -) GWL <- readNWISdata(site_no = c("392725077582401", "375907091432201"), parameterCd = "62610", @@ -172,19 +152,6 @@ arg.list <- list( ) allDailyStats_2 <- readNWISdata(arg.list, service = "stat") -# use county names to get data -dailyStaffordVA <- readNWISdata( - stateCd = "Virginia", - countyCd = "Stafford", - parameterCd = "00060", - startDate = "2015-01-01", - endDate = "2015-01-30" -) -va_counties <- c("51001", "51003", "51005", "51007", "51009", "51011", "51013", "51015") -va_counties_data <- readNWISdata( - startDate = "2015-01-01", endDate = "2015-12-31", - parameterCd = "00060", countycode = va_counties -) site_id <- "01594440" rating_curve <- readNWISdata(service = "rating", site_no = site_id, file_type = "base") @@ -212,5 +179,5 @@ peak_data <- readNWISdata( \dontshow{\}) # examplesIf} } \seealso{ -\code{\link[=renameNWISColumns]{renameNWISColumns()}}, \code{\link[=importWaterML1]{importWaterML1()}}, \code{\link[=importRDB1]{importRDB1()}} +\code{\link[=read_USGS_data]{read_USGS_data()}} } diff --git a/man/readNWISdv.Rd b/man/readNWISdv.Rd index d4c34204..2972328d 100644 --- a/man/readNWISdv.Rd +++ b/man/readNWISdv.Rd @@ -71,35 +71,12 @@ More information on the web service can be found here: "Daily Value Service". } \examples{ -\dontshow{if (is_dataRetrieval_user()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} -site_id <- "04085427" -startDate <- "2012-01-01" -endDate <- "2012-06-30" -pCode <- "00060" -\donttest{ -rawDailyQ <- readNWISdv(site_id, pCode, startDate, endDate) -rawDailyQAndTempMeanMax <- readNWISdv(site_id, c("00010", "00060"), - startDate, endDate, - statCd = c("00001", "00003") -) -rawDailyQAndTempMeanMax <- renameNWISColumns(rawDailyQAndTempMeanMax) -rawDailyMultiSites <- readNWISdv(c("01491000", "01645000"), c("00010", "00060"), - startDate, endDate, - statCd = c("00001", "00003") -) -# Site with no data: -x <- readNWISdv("10258500", "00060", "2014-09-08", "2014-09-14") -names(attributes(x)) -attr(x, "siteInfo") -attr(x, "variableInfo") -site <- "05212700" -notActive <- readNWISdv(site, "00060", "2014-01-01", "2014-01-07") -} -\dontshow{\}) # examplesIf} +# see ?read_USGS_daily + } \seealso{ -\code{\link[=renameNWISColumns]{renameNWISColumns()}}, \code{\link[=importWaterML1]{importWaterML1()}} +\code{\link[=read_USGS_daily]{read_USGS_daily()}} } \keyword{USGS} \keyword{data} diff --git a/man/readNWISsite.Rd b/man/readNWISsite.Rd index 4393ce28..0c0895d5 100644 --- a/man/readNWISsite.Rd +++ b/man/readNWISsite.Rd @@ -69,13 +69,12 @@ comment \tab character \tab Header comments from the RDB file \cr Imports data from USGS site file site. This function gets data from here: \url{https://waterservices.usgs.gov/} } \examples{ -\dontshow{if (is_dataRetrieval_user()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} -\donttest{ -siteINFO <- readNWISsite("05114000") -siteINFOMulti <- readNWISsite(c("05114000", "09423350")) +# see ?read_USGS_monitoring_location + } -\dontshow{\}) # examplesIf} +\seealso{ +\code{\link[=read_USGS_monitoring_location]{read_USGS_monitoring_location()}} } \keyword{USGS} \keyword{data} diff --git a/man/whatNWISdata.Rd b/man/whatNWISdata.Rd index 225ab92f..8ff2f31b 100644 --- a/man/whatNWISdata.Rd +++ b/man/whatNWISdata.Rd @@ -73,30 +73,12 @@ function to use before a large data set, by filtering down the number of sites that have useful data. } \examples{ -\dontshow{if (is_dataRetrieval_user()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} -\donttest{ -availableData <- whatNWISdata(siteNumber = "05114000") +# see ?read_USGS_ts_meta -# To find just unit value ('instantaneous') data: -uvData <- whatNWISdata(siteNumber = "05114000", - service = "uv") -uvDataMulti <- whatNWISdata(siteNumber = c("05114000", "09423350"), - service = c("uv", "dv")) -flowAndTemp <- whatNWISdata( - stateCd = "WI", service = "dv", - parameterCd = c("00060", "00010"), - statCd = "00003" -) -sites <- whatNWISdata(stateCd = "WI", - parameterCd = "00060", - siteType = "ST", - service = "site") - -sites <- whatNWISdata(stateCd = "WI", - service = "gwlevels") } -\dontshow{\}) # examplesIf} +\seealso{ +\code{\link[=read_USGS_ts_meta]{read_USGS_ts_meta()}} } \keyword{USGS} \keyword{data} diff --git a/man/whatNWISsites.Rd b/man/whatNWISsites.Rd index 7f6408f3..89d8cf3b 100644 --- a/man/whatNWISsites.Rd +++ b/man/whatNWISsites.Rd @@ -36,9 +36,10 @@ Returns a list of sites from the NWIS web service. This function gets the data f Mapper format is used } \examples{ -\donttest{ -siteListPhos <- whatNWISsites(stateCd = "OH", parameterCd = "00665") -oneSite <- whatNWISsites(sites = "05114000") +# see ?read_USGS_monitoring_location + } +\seealso{ +\code{\link[=read_USGS_monitoring_location]{read_USGS_monitoring_location()}} } diff --git a/tests/testthat/tests_general.R b/tests/testthat/tests_general.R index 8b110bbf..c5d75f71 100644 --- a/tests/testthat/tests_general.R +++ b/tests/testthat/tests_general.R @@ -99,24 +99,6 @@ test_that("General NWIS retrievals working", { bBoxEx <- readNWISdata(bBox = c(-83, 36.5, -81, 38.5), parameterCd = "00010") expect_true(length(unique(bBoxEx$site_no)) > 1) - startDate <- as.Date("2013-10-01") - endDate <- as.Date("2014-09-30") - # waterYear <- readNWISdata( - # bBox = c(-83, 36.5, -81, 38.5), - # parameterCd = "00010", - # service = "dv", - # startDate = startDate, - # endDate = endDate - # ) - # expect_is(waterYear$dateTime, "POSIXct") - - waterYear <- read_USGS_daily( - bbox = c(-83, 36.5, -81, 38.5), - parameter_code = "00010", - time = c(startDate, endDate) - ) - expect_is(waterYear$time, "Date") - siteInfo <- readNWISdata( stateCd = "WI", parameterCd = "00010", @@ -183,16 +165,6 @@ test_that("General NWIS retrievals working", { instData <- readNWISdata(args) - # args <- list( - # sites = "05114000", service = "dv", - # parameterCd = "00060", - # startDate = "2014-05-01", - # endDate = "2014-05-01" - # ) - # - # dailyData <- readNWISdata(args) - # expect_lt(nrow(dailyData), nrow(instData)) - args2 <- list( monitoring_location_id = "USGS-05114000", parameter_code = "00060", @@ -201,23 +173,34 @@ test_that("General NWIS retrievals working", { daily_USGS <- do.call(read_USGS_daily, args2) expect_lt(nrow(daily_USGS), nrow(instData)) + + ohio <- read_USGS_monitoring_location(state_name = "Ohio", + site_type_code = "ST") + bbox <- sf::st_bbox(ohio) + what_sites <- read_USGS_ts_meta(parameter_code = "00665", + bbox = as.numeric(bbox)) + expect_true(all(c("monitoring_location_id", + "begin", "end", "parameter_name") %in% names(what_sites))) - - args <- list(stateCd = "OH", parameterCd = "00665") - sites <- whatNWISsites(args) - expect_type(sites, "list") + huc <- read_USGS_monitoring_location(hydrologic_unit_code = "02080202") + expect_true(nrow(huc) > 0) # Test counties: - dailyStaffordVA <- readNWISdata( - stateCd = "Virginia", - countyCd = "Stafford", - parameterCd = "00060", - startDate = "2015-01-01", - endDate = "2015-01-30" + + county_code <- countyCdLookup(state = "Virginia", county = "Stafford") + stafford <- read_USGS_monitoring_location(county_code = "179", + state_code = "51") + stafford_bbox <- sf::st_bbox(stafford) + + dailyStaffordVA <- read_USGS_daily( + bbox = as.numeric(stafford_bbox), + parameter_code = "00060", + time = c("2015-01-01", "2015-01-30") ) expect_gt(nrow(dailyStaffordVA), 1) - AS <- readNWISdata(stateCd = "AS", service = "site") + # America Samoa? + AS <- read_USGS_monitoring_location(state_name = "American Samoa") expect_gt(nrow(AS), 0) site_id <- "01594440" @@ -249,22 +232,15 @@ test_that("General NWIS retrievals working", { "url" ))) - # multi_hucs <- c("07130007", "07130011") - # multi_huc <- dataRetrieval::readNWISdata( - # huc = multi_hucs, - # parameterCd = "63680", - # startDate = "2015-06-18", - # endDate = "2015-06-18", - # service = "dv" - # ) - # expect_equal(2, nrow(multi_huc)) - - # HUC isn't available in the "daily" service: multi_hucs <- c("07130007", "07130011") - multi_huc_new <- read_USGS_monitoring_location( - hydrologic_unit_code = multi_hucs + multi_huc_sites <- read_USGS_monitoring_location(hydrologic_unit_code = multi_hucs) + + multi_huc <- read_USGS_daily(bbox = as.numeric(sf::st_bbox(multi_huc_sites)), + parameter_code = "63680", + statistic_id = "00003", + time = c("2015-06-18", "2015-06-18") ) - expect_equal(2, length(unique(multi_huc_new$hydrologic_unit_code))) + expect_equal(4, length(unique(multi_huc$monitoring_location_id))) peak_data <- readNWISdata( service = "peak", diff --git a/tests/testthat/tests_userFriendly_fxns.R b/tests/testthat/tests_userFriendly_fxns.R index c1612610..f10baa1e 100644 --- a/tests/testthat/tests_userFriendly_fxns.R +++ b/tests/testthat/tests_userFriendly_fxns.R @@ -91,9 +91,7 @@ test_that("peak, rating curves, surface-water measurements", { expect_is(siteINFO_USGS$agency_code, "character") expect_equal(siteINFO_USGS$monitoring_locations_id, "USGS-05114000") - # siteINFOMulti <- readNWISsite(c("05114000", "09423350")) - # expect_true(nrow(siteINFOMulti) == 2) - + siteINFOMulti_USGS <- read_USGS_monitoring_location(c("USGS-05114000", "USGS-09423350")) expect_true(nrow(siteINFOMulti_USGS) == 2) @@ -127,22 +125,11 @@ test_that("NWIS dv tests", { endDate <- "2012-06-30" pCode <- "00060" - # rawDailyQ <- readNWISdv(siteNumber, pCode, startDate, endDate) - # expect_is(rawDailyQ$Date, "Date") - raw_USGS_daily <- read_USGS_daily(monitoring_location_id = paste0("USGS-", siteNumber), parameter_code = pCode, time = c(startDate, endDate)) expect_is(raw_USGS_daily$time, "Date") - - # rawDailyQAndTempMeanMax <- readNWISdv(siteNumber, c("00010", "00060"), - # startDate, endDate, - # statCd = c("00001", "00003") - # ) - # expect_true(length(grep("00060", names(rawDailyQAndTempMeanMax))) >= 2 & - # length(grep("00010", names(rawDailyQAndTempMeanMax))) >= 2) - raw_USGS_TempMeanMax <- read_USGS_daily(monitoring_location_id = paste0("USGS-", siteNumber), parameter_code = c("00010", "00060"), time = c(startDate, endDate), @@ -152,13 +139,6 @@ test_that("NWIS dv tests", { expect_true(length(unique(raw_USGS_TempMeanMax$statistic_id)) == 2) expect_true(length(unique(raw_USGS_TempMeanMax$monitoring_location_id)) == 1) - # rawDailyMultiSites <- readNWISdv(c("01491000", "01645000"), - # c("00010", "00060"), - # startDate, endDate, - # statCd = c("00001", "00003") - # ) - # expect_true(length(unique(rawDailyMultiSites$site_no)) > 1) - raw_USGS_MultiSites <- read_USGS_daily(monitoring_location_id = paste0("USGS-", c("01491000", "01645000")), parameter_code = c("00010", "00060"), @@ -168,9 +148,7 @@ test_that("NWIS dv tests", { expect_true(length(unique(raw_USGS_MultiSites$monitoring_location_id)) == 2) site <- "05212700" - # notActive <- readNWISdv(site, "00060", "2014-01-01", "2014-01-07") - # expect_true(nrow(notActive) == 0) - + notActiveUSGS <- read_USGS_daily(monitoring_location_id = paste0("USGS-", site), parameter_code = "00060", time = c("2014-01-01", "2014-01-07")) From def03d839315b88dce657d330c17fed71b3a22de Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 3 Jun 2025 08:35:31 -0500 Subject: [PATCH 058/117] add data --- _pkgdown.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/_pkgdown.yml b/_pkgdown.yml index 3162e95e..dce8c165 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -68,6 +68,7 @@ reference: - read_USGS_samples - read_USGS_daily - read_USGS_ts_meta + - read_USGS_data - read_USGS_monitoring_location - read_USGS_data - summarize_USGS_samples From eb3e372e52dac5c26f0663dd46bbf9d997c0cfab Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 3 Jun 2025 08:46:54 -0500 Subject: [PATCH 059/117] Update readme with new flow --- README.Rmd | 16 +++++++++------- README.md | 35 ++++++++++++++++++++--------------- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/README.Rmd b/README.Rmd index c0509aa8..a3636ef7 100644 --- a/README.Rmd +++ b/README.Rmd @@ -27,7 +27,7 @@ library(dataRetrieval) [![](http://cranlogs.r-pkg.org/badges/dataRetrieval)](https://cran.r-project.org/package=dataRetrieval) [![](http://cranlogs.r-pkg.org/badges/grand-total/dataRetrieval)](https://cran.r-project.org/package=dataRetrieval) -The `dataRetrieval` package was created to simplify the process of loading hydrologic data into the R environment. It is designed to retrieve the major data types of U.S. Geological Survey (USGS) hydrology data that are available on the Web, as well as data from the Water Quality Portal (WQP), which currently houses water quality data from the Environmental Protection Agency (EPA), U.S. Department of Agriculture (USDA), and USGS. Direct USGS data is obtained from a service called the National Water Information System (NWIS). +The `dataRetrieval` package was created to simplify the process of loading hydrologic data into the R environment. It is designed to retrieve the major data types of U.S. Geological Survey (USGS) hydrology data that are available on the Web, as well as data from the Water Quality Portal (WQP), which currently houses water quality data from the Environmental Protection Agency (EPA), U.S. Department of Agriculture (USDA), and USGS. # Introduction @@ -37,17 +37,19 @@ If you have additional questions about these changes, email CompTools@usgs.gov. # What would you like to do? -1. Get instantaneous USGS discharge data. Start here: `?readNWISuv` +1. Get instantaneous USGS data (for example, discharge sensor data). Start here: `?readNWISuv` -2. Get daily USGS discharge data. Start here: `?readNWISdv` +2. Get daily USGS data (for example, mean daily discharge). Start here: `?read_USGS_daily` 3. Get USGS groundwater data. Start here: `?readNWISgwl` -4. Get discrete water quality data. Start here: `?readWQPdata` +4. Get discrete water quality data from a cooperative service that integrates publicly available water-quality data from the USGS, EPA, and over 400 state, federal, tribal, and local agencies. Start here: `?readWQPdata` -4. Discover USGS data (not including discrete water quality data). Start here: `?whatNWISdata` +5. Get USGS discrete water quality data. Start here: `?read_USGS_samples` -6. Find Hydro Network-Linked Data Index (NLDI) data. Start here: `?findNLDI` +6. Discover USGS time series data. Start here: `?read_USGS_ts_meta` + +7. Find Hydro Network-Linked Data Index (NLDI) data. Start here: `?findNLDI` For additional tutorials, see: @@ -160,7 +162,7 @@ Water Quality Portal. Washington (DC): National Water Quality Monitoring Council # Package Support -The Water Mission Area of the USGS supports the development and maintenance of `dataRetrieval`, and most likely further into the future. Resources are available primarily for maintenance and responding to user questions. Priorities on the development of new features are determined by the `dataRetrieval` development team. This software was last released with USGS record: IP-147158. +The Water Mission Area of the USGS supports the development and maintenance of `dataRetrieval`, and most likely further into the future. Resources are available primarily for maintenance and responding to user questions. Priorities on the development of new features are determined by the `dataRetrieval` development team. ```{r disclaimer, child="DISCLAIMER.md", eval=TRUE} ``` diff --git a/README.md b/README.md index 153c822c..a66f84fc 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,7 @@ retrieve the major data types of U.S. Geological Survey (USGS) hydrology data that are available on the Web, as well as data from the Water Quality Portal (WQP), which currently houses water quality data from the Environmental Protection Agency (EPA), U.S. Department of Agriculture -(USDA), and USGS. Direct USGS data is obtained from a service called the -National Water Information System (NWIS). +(USDA), and USGS. # Introduction @@ -28,18 +27,25 @@ If you have additional questions about these changes, email # What would you like to do? -1. Get instantaneous USGS discharge data. Start here: `?readNWISuv` +1. Get instantaneous USGS data (for example, discharge sensor data). + Start here: `?readNWISuv` -2. Get daily USGS discharge data. Start here: `?readNWISdv` +2. Get daily USGS data (for example, mean daily discharge). Start here: + `?read_USGS_daily` 3. Get USGS groundwater data. Start here: `?readNWISgwl` -4. Get discrete water quality data. Start here: `?readWQPdata` +4. Get discrete water quality data from a cooperative service that + integrates publicly available water-quality data from the USGS, EPA, + and over 400 state, federal, tribal, and local agencies. Start here: + `?readWQPdata` -5. Discover USGS data (not including discrete water quality data). - Start here: `?whatNWISdata` +5. Get USGS discrete water quality data. Start here: + `?read_USGS_samples` -6. Find Hydro Network-Linked Data Index (NLDI) data. Start here: +6. Discover USGS time series data. Start here: `?read_USGS_ts_meta` + +7. Find Hydro Network-Linked Data Index (NLDI) data. Start here: `?findNLDI` For additional tutorials, see: @@ -170,15 +176,15 @@ NWIScitation <- create_NWIS_bib(dv) NWIScitation #> U.S. Geological Survey (2025). _National Water Information System data #> available on the World Wide Web (USGS Water Data for the Nation)_. -#> doi:10.5066/F7P55KJN , Accessed Mar -#> 25, 2025, +#> doi:10.5066/F7P55KJN , Accessed Jun +#> 03, 2025, #> . print(NWIScitation, style = "Bibtex") #> @Manual{, #> title = {National Water Information System data available on the World Wide Web (USGS Water Data for the Nation)}, #> author = {{U.S. Geological Survey}}, #> doi = {10.5066/F7P55KJN}, -#> note = {Accessed Mar 25, 2025}, +#> note = {Accessed Jun 03, 2025}, #> year = {2025}, #> url = {https://waterservices.usgs.gov/nwis/dv/?site=09010500&format=waterml%2C1.1&ParameterCd=00060&StatCd=00003&startDT=1851-01-01}, #> } @@ -202,14 +208,14 @@ WQPcitation <- create_WQP_bib(SC) WQPcitation #> National Water Quality Monitoring Council (2025). _Water Quality #> Portal_. doi:10.5066/P9QRKUVJ , -#> Accessed Mar 25, 2025, +#> Accessed Jun 03, 2025, #> . print(WQPcitation, style = "Bibtex") #> @Manual{, #> title = {Water Quality Portal}, #> author = {{National Water Quality Monitoring Council}}, #> doi = {10.5066/P9QRKUVJ}, -#> note = {Accessed Mar 25, 2025}, +#> note = {Accessed Jun 03, 2025}, #> year = {2025}, #> url = {https://www.waterqualitydata.us/data/Result/search?siteid=USGS-05288705&count=no&pCode=00300&mimeType=csv}, #> } @@ -229,8 +235,7 @@ The Water Mission Area of the USGS supports the development and maintenance of `dataRetrieval`, and most likely further into the future. Resources are available primarily for maintenance and responding to user questions. Priorities on the development of new features are determined -by the `dataRetrieval` development team. This software was last released -with USGS record: IP-147158. +by the `dataRetrieval` development team. # Disclaimer From 52fe279fb7473958db61c1d813cfb30978afca7f Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 3 Jun 2025 09:34:44 -0500 Subject: [PATCH 060/117] eliminate test warnings --- R/construct_api_requests.R | 2 +- R/readNWISunit.R | 10 +- tests/testthat/tests_general.R | 142 ++++++++++++----------- tests/testthat/tests_userFriendly_fxns.R | 6 +- 4 files changed, 83 insertions(+), 77 deletions(-) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index e368959e..44898483 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -111,7 +111,7 @@ construct_api_requests <- function(service, if(all(!is.na(bbox))){ baseURL <- httr2::req_url_query(baseURL, - bbox = bbox, + bbox = as.numeric(bbox), .multi = "comma") } diff --git a/R/readNWISunit.R b/R/readNWISunit.R index f97a2df7..13728aa3 100644 --- a/R/readNWISunit.R +++ b/R/readNWISunit.R @@ -209,7 +209,7 @@ readNWISpeak <- function(siteNumbers, } - siteInfo <- suppressMessages(readNWISsite(siteNumbers)) + siteInfo <- suppressWarnings(readNWISsite(siteNumbers)) siteInfo <- merge( x = unique(data[, c("agency_cd", "site_no")]), y = siteInfo, @@ -287,7 +287,7 @@ readNWISrating <- function(siteNumber, type = "base", convertType = TRUE) { attr(data, "RATING") <- Rat } - siteInfo <- suppressMessages(readNWISsite(siteNumbers = siteNumber)) + siteInfo <- suppressWarnings(readNWISsite(siteNumbers = siteNumber)) attr(data, "siteInfo") <- siteInfo attr(data, "variableInfo") <- NULL @@ -408,7 +408,7 @@ readNWISmeas <- function(siteNumbers, } - siteInfo <- suppressMessages(readNWISsite(siteNumbers)) + siteInfo <- suppressWarnings(readNWISsite(siteNumbers)) siteInfo <- merge( x = unique(data[, c("agency_cd", "site_no")]), y = siteInfo, @@ -541,7 +541,7 @@ readNWISgwl <- function(siteNumbers, data$lev_dt <- as.Date(data$lev_dt) } } - siteInfo <- suppressMessages(readNWISsite(siteNumbers)) + siteInfo <- suppressWarnings(readNWISsite(siteNumbers)) siteInfo <- merge( x = unique(data[, c("agency_cd", "site_no")]), y = siteInfo, @@ -649,7 +649,7 @@ readNWISstat <- function(siteNumbers, parameterCd, startDate = "", endDate = "", convertType = convertType ) - siteInfo <- suppressMessages(readNWISsite(siteNumbers)) + siteInfo <- suppressWarnings(readNWISsite(siteNumbers)) if (nrow(data) > 0) { siteInfo <- merge( diff --git a/tests/testthat/tests_general.R b/tests/testthat/tests_general.R index c5d75f71..400116a1 100644 --- a/tests/testthat/tests_general.R +++ b/tests/testthat/tests_general.R @@ -55,6 +55,7 @@ test_that("General USGS retrievals working", { notActiveUSGS <- read_USGS_data(CQL = cql_not_active, service = "daily", time = c("2014-01-01", "2014-01-07")) + expect_true(nrow(notActiveUSGS) == 0) }) @@ -96,16 +97,18 @@ test_that("General NWIS retrievals working", { expect_error(readNWISdata(), "No arguments supplied") expect_error(readNWISdata(siteNumber = NA), "NA's are not allowed in query") - bBoxEx <- readNWISdata(bBox = c(-83, 36.5, -81, 38.5), parameterCd = "00010") - expect_true(length(unique(bBoxEx$site_no)) > 1) + bBox_inventory <- read_USGS_ts_meta(bbox = c(-83, 38, -82.5, 38.5), + parameter_code = "00010") + + expect_true(length(unique(bBox_inventory$monitoring_location_id)) > 1) - siteInfo <- readNWISdata( - stateCd = "WI", - parameterCd = "00010", - hasDataTypeCd = "iv", - service = "site" - ) - expect_is(siteInfo$station_nm, "character") + siteInfo <- read_USGS_monitoring_location(state_name = "Wisconsin") + + timeseriesInfo <- read_USGS_ts_meta(bbox = sf::st_bbox(siteInfo), + parameter_code = "00010", + computation_period_identifier = "Points" ) + + expect_is(timeseriesInfo$begin, "POSIXct") gw_data <- readNWISdata( stateCd = "AL", @@ -178,7 +181,7 @@ test_that("General NWIS retrievals working", { site_type_code = "ST") bbox <- sf::st_bbox(ohio) what_sites <- read_USGS_ts_meta(parameter_code = "00665", - bbox = as.numeric(bbox)) + bbox = bbox) expect_true(all(c("monitoring_location_id", "begin", "end", "parameter_name") %in% names(what_sites))) @@ -187,13 +190,16 @@ test_that("General NWIS retrievals working", { # Test counties: - county_code <- countyCdLookup(state = "Virginia", county = "Stafford") - stafford <- read_USGS_monitoring_location(county_code = "179", - state_code = "51") + county_code_stafford <- countyCdLookup(state = "Virginia", + county = "Stafford", + outputType = "id") + state_code_va <- stateCdLookup(input = "Virginia", outputType = "id") + stafford <- read_USGS_monitoring_location(county_code = county_code_stafford, + state_code = state_code_va) stafford_bbox <- sf::st_bbox(stafford) dailyStaffordVA <- read_USGS_daily( - bbox = as.numeric(stafford_bbox), + bbox = stafford_bbox, parameter_code = "00060", time = c("2015-01-01", "2015-01-30") ) @@ -235,7 +241,7 @@ test_that("General NWIS retrievals working", { multi_hucs <- c("07130007", "07130011") multi_huc_sites <- read_USGS_monitoring_location(hydrologic_unit_code = multi_hucs) - multi_huc <- read_USGS_daily(bbox = as.numeric(sf::st_bbox(multi_huc_sites)), + multi_huc <- read_USGS_daily(bbox = sf::st_bbox(multi_huc_sites), parameter_code = "63680", statistic_id = "00003", time = c("2015-06-18", "2015-06-18") @@ -258,33 +264,35 @@ test_that("General NWIS retrievals working", { test_that("whatNWISdata", { # no service specified: - availableData <- whatNWISdata(siteNumber = "05114000") - expect_equal(ncol(availableData), 24) + availableData <- read_USGS_ts_meta(monitoring_location_id = "USGS-05114000") + expect_equal(ncol(availableData), 17) - uvData <- whatNWISdata(siteNumber = "05114000", service = "uv") - expect_equal(unique(uvData$data_type_cd), "uv") + uvData <- read_USGS_ts_meta(monitoring_location_id = "USGS-05114000", + computation_period_identifier = c("Points")) + expect_equal(unique(uvData$computation_period_identifier), "Points") # multiple services - uvDataMulti <- whatNWISdata( - siteNumber = c("05114000", "09423350"), - service = c("uv", "dv") - ) - expect_true(all(unique(uvDataMulti$data_type_cd) %in% c("uv", "dv"))) + uvDataMulti <- read_USGS_ts_meta(monitoring_location_id = c("USGS-05114000", + "USGS-09423350"), + computation_period_identifier = c("Daily", + "Points")) + + expect_true(all(unique(uvDataMulti$computation_period_identifier) %in% c("Daily", + "Points"))) # state codes: - flowAndTemp <- whatNWISdata( - stateCd = "WI", service = c("uv", "dv"), - parameterCd = c("00060", "00010"), - statCd = "00003" - ) - expect_true(all(unique(flowAndTemp$data_type_cd) %in% c("uv", "dv"))) - expect_true(all(unique(flowAndTemp$parm_cd) %in% c("00060", "00010"))) - expect_true(all(unique(flowAndTemp$stat_cd) %in% c("00003", NA))) - - # site service - sites <- whatNWISdata(stateCd = "WI", service = "site") - expect_true(all(c("gw", "sv", "qw", "dv", "pk", "uv") - %in% unique(sites$data_type_cd))) + wi_sites <- read_USGS_monitoring_location(state_name = "Wisconsin") + flow_and_temp <- read_USGS_ts_meta(bbox = sf::st_bbox(wi_sites), + parameter_code = c("00060", "00010"), + statistic_id = "00003", + computation_period_identifier = c("Daily", + "Points")) + + expect_true(all(unique(flow_and_temp$computation_period_identifier) %in% c("Daily", + "Points"))) + expect_true(all(unique(flow_and_temp$parameter_code) %in% c("00060", "00010"))) + expect_true(all(unique(flow_and_temp$statistic_id) %in% c("00003"))) + }) test_that("General WQP retrievals working", { @@ -344,32 +352,32 @@ test_that("General WQP retrievals working", { # This means wqp_check_status wasn't called: expect_false("dataProviders" %in% names(attr(rawPcode2, "headerInfo"))) - # pHData <- readWQPdata(siteid = "USGS-04024315", - # characteristicName = "pH", - # service = "ResultWQX3") - # expect_true(all(c("url", "queryTime", "siteInfo", "headerInfo") %in% - # names(attributes(pHData)))) - # + pHData <- readWQPdata(siteid = "USGS-04024315", + characteristicName = "pH", + service = "ResultWQX3") + expect_true(all(c("url", "queryTime", "siteInfo", "headerInfo") %in% + names(attributes(pHData)))) + # # This means wqp_check_status was called: - # expect_true("dataProviders" %in% names(attr(pHData, "headerInfo"))) - # - # pHData2 <- readWQPdata(siteid = "USGS-04024315", - # characteristicName = "pH", - # ignore_attributes = TRUE, - # service = "ResultWQX3") - # expect_true(all(!c("queryTime", "siteInfo") %in% - # names(attributes(pHData2)))) - # + expect_true("dataProviders" %in% names(attr(pHData, "headerInfo"))) + + pHData2 <- readWQPdata(siteid = "USGS-04024315", + characteristicName = "pH", + ignore_attributes = TRUE, + service = "ResultWQX3") + expect_true(all(!c("queryTime", "siteInfo") %in% + names(attributes(pHData2)))) + # # This means wqp_check_status was called: - # expect_false("dataProviders" %in% names(attr(pHData2, "headerInfo"))) - # - # rawPcode <- readWQPqw("USGS-01594440", "01075", - # ignore_attributes = TRUE, legacy = FALSE) - # headerInfo <- attr(rawPcode, "headerInfo") - # wqp_request_id <- headerInfo$`wqp-request-id` - # count_info <- wqp_check_status(wqp_request_id) - # - # expect_true("dataProviders" %in% names(count_info)) + expect_false("dataProviders" %in% names(attr(pHData2, "headerInfo"))) + + rawPcode <- readWQPqw("USGS-01594440", "01075", + ignore_attributes = TRUE, legacy = FALSE) + headerInfo <- attr(rawPcode, "headerInfo") + wqp_request_id <- headerInfo$`wqp-request-id` + count_info <- wqp_check_status(wqp_request_id) + + expect_true("dataProviders" %in% names(count_info)) }) @@ -421,18 +429,16 @@ test_that("whatWQPdata working", { context("whatNWISsites") test_that("whatNWISsites working", { testthat::skip_on_cran() - siteListPhos <- whatNWISsites(stateCd = "OH", parameterCd = "00665") + siteListOhio <- read_USGS_monitoring_location(state_name = "Ohio") + siteListPhos <- read_USGS_ts_meta(bbox = sf::st_bbox(siteListOhio), + parameter_code = "00665") expect_true(nrow(siteListPhos) > 0) expect_true(is.numeric(siteListPhos$dec_lat_va)) - bboxSites <- whatNWISsites(bbox = c(-92.5, 45.4, -87, 47), parameterCd = "00060") + bboxSites <- read_USGS_ts_meta(bbox = c(-92.5, 45.4, -87, 47), + parameter_code = "00060") expect_true(nrow(bboxSites) > 0) - expect_true(is.numeric(bboxSites$dec_lat_va)) - #gwlevels: - info <- whatNWISsites(stateCd = "NY", service="gwlevels") - expect_true(nrow(info) > 0) - expect_equal(attr(info, "url"), "https://waterservices.usgs.gov/nwis/site/?stateCd=NY&hasDataTypeCd=gw&format=mapper") }) context("readWQPdots") diff --git a/tests/testthat/tests_userFriendly_fxns.R b/tests/testthat/tests_userFriendly_fxns.R index f10baa1e..fa9026f9 100644 --- a/tests/testthat/tests_userFriendly_fxns.R +++ b/tests/testthat/tests_userFriendly_fxns.R @@ -120,17 +120,17 @@ test_that("peak, rating curves, surface-water measurements", { test_that("NWIS dv tests", { testthat::skip_on_cran() - siteNumber <- "04085427" + siteNumber <- "USGS-04085427" startDate <- "2012-01-01" endDate <- "2012-06-30" pCode <- "00060" - raw_USGS_daily <- read_USGS_daily(monitoring_location_id = paste0("USGS-", siteNumber), + raw_USGS_daily <- read_USGS_daily(monitoring_location_id = siteNumber, parameter_code = pCode, time = c(startDate, endDate)) expect_is(raw_USGS_daily$time, "Date") - raw_USGS_TempMeanMax <- read_USGS_daily(monitoring_location_id = paste0("USGS-", siteNumber), + raw_USGS_TempMeanMax <- read_USGS_daily(monitoring_location_id = siteNumber, parameter_code = c("00010", "00060"), time = c(startDate, endDate), statistic_id = c("00001", "00003")) From b45aefcb125fa1444eff5de4fdcddef1784be546 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 3 Jun 2025 10:08:39 -0500 Subject: [PATCH 061/117] Update message and convert more tests --- R/AAA.R | 3 +- R/construct_api_requests.R | 7 +++ tests/testthat/tests_general.R | 8 ++-- tests/testthat/tests_userFriendly_fxns.R | 57 +++++++++++++++--------- 4 files changed, 49 insertions(+), 26 deletions(-) diff --git a/R/AAA.R b/R/AAA.R index 1200e175..ec650725 100644 --- a/R/AAA.R +++ b/R/AAA.R @@ -54,7 +54,8 @@ https://doi-usgs.github.io/dataRetrieval/articles/Status.html") } new_nwis_message <- function(){ - return("ALERT: All NWIS services are slated for decommission. + return("ALERT: All NWIS services are slated for decommission +and dataRetrieval functions will be updated to use new API endpoints. For up-to-date information, see: https://doi-usgs.github.io/dataRetrieval/articles/Status.html") } diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index 44898483..1016f590 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -202,8 +202,15 @@ setup_api <- function(service){ #' start_end <- as.POSIXct(c("2021-01-01 12:15:00", "2022-01-01 16:45")) #' dataRetrieval:::format_api_dates(start_end) #' +#' start_end2 <- c("2021-01-01 12:15:00", "") +#' dataRetrieval:::format_api_dates(start_end2) +#' format_api_dates <- function(datetime){ + if(is.character(datetime)){ + datetime[datetime == ""] <- NA + } + if(!any(isTRUE(is.na(datetime)) | isTRUE(is.null(datetime)))){ if(length(datetime) == 1){ if(grepl("P", datetime, ignore.case = TRUE) | diff --git a/tests/testthat/tests_general.R b/tests/testthat/tests_general.R index 400116a1..e42d3260 100644 --- a/tests/testthat/tests_general.R +++ b/tests/testthat/tests_general.R @@ -261,7 +261,7 @@ test_that("General NWIS retrievals working", { expect_lt(nrow(peak_data), 100000) }) -test_that("whatNWISdata", { +test_that("read_USGS_ts_meta", { # no service specified: availableData <- read_USGS_ts_meta(monitoring_location_id = "USGS-05114000") @@ -426,14 +426,14 @@ test_that("whatWQPdata working", { expect_is(lakeSites$activityCount, "numeric") }) -context("whatNWISsites") -test_that("whatNWISsites working", { +context("read_USGS_ts_meta") +test_that("read_USGS_ts_meta working", { testthat::skip_on_cran() siteListOhio <- read_USGS_monitoring_location(state_name = "Ohio") siteListPhos <- read_USGS_ts_meta(bbox = sf::st_bbox(siteListOhio), parameter_code = "00665") expect_true(nrow(siteListPhos) > 0) - expect_true(is.numeric(siteListPhos$dec_lat_va)) + expect_is(siteListPhos$begin, "POSIXct") bboxSites <- read_USGS_ts_meta(bbox = c(-92.5, 45.4, -87, 47), parameter_code = "00060") diff --git a/tests/testthat/tests_userFriendly_fxns.R b/tests/testthat/tests_userFriendly_fxns.R index fa9026f9..c14d8892 100644 --- a/tests/testthat/tests_userFriendly_fxns.R +++ b/tests/testthat/tests_userFriendly_fxns.R @@ -99,8 +99,13 @@ test_that("peak, rating curves, surface-water measurements", { expect_is(Meas07227500.ex$measurement_dt, "Date") expect_is(Meas07227500.ex$measurement_dateTime, "POSIXct") - expect_equal(nrow(whatNWISdata(siteNumber = "10312000", parameterCd = "50286")), 0) - expect_equal(ncol(whatNWISdata(siteNumber = "10312000", parameterCd = "50286")), 24) + expect_equal(nrow(read_USGS_ts_meta(monitoring_location_id = "USGS-10312000", + parameter_code = "50286")), 0) + expect_equal(ncol(read_USGS_ts_meta(monitoring_location_id = "USGS-10312000", + parameter_code = "50286", + properties = c("geometry", "id", + "unit_of_measure", + "parameter_name"))), 4) url <- httr2::request("https://waterservices.usgs.gov/nwis/site/?format=rdb&seriesCatalogOutput=true&sites=05114000") x <- importRDB1(url) @@ -117,7 +122,7 @@ test_that("peak, rating curves, surface-water measurements", { convertType = FALSE)) }) -test_that("NWIS dv tests", { +test_that("read_USGS_daily", { testthat::skip_on_cran() siteNumber <- "USGS-04085427" @@ -139,8 +144,7 @@ test_that("NWIS dv tests", { expect_true(length(unique(raw_USGS_TempMeanMax$statistic_id)) == 2) expect_true(length(unique(raw_USGS_TempMeanMax$monitoring_location_id)) == 1) - raw_USGS_MultiSites <- read_USGS_daily(monitoring_location_id = paste0("USGS-", - c("01491000", "01645000")), + raw_USGS_MultiSites <- read_USGS_daily(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), parameter_code = c("00010", "00060"), time = c(startDate, endDate), statistic_id = c("00001", "00003")) @@ -363,37 +367,48 @@ test_that("calcWaterYear can handle missing values", { }) -context("Construct NWIS urls") -test_that("Construct NWIS urls", { +context("construct_api_requests") +test_that("Construct USGS urls", { testthat::skip_on_cran() - siteNumber <- "01594440" - startDate <- "1985-01-01" + siteNumber <- "USGS-01594440" + startDate <- "2024-01-01" endDate <- "" pCode <- c("00060", "00010") - url_daily <- constructNWISURL(siteNumber, pCode, - startDate, endDate, "dv", - statCd = c("00003", "00001") - ) + url_daily <- construct_api_requests(service = "daily", + monitoring_location_id = siteNumber, + parameter_code = pCode, + time = c(startDate, endDate), + statistic_id = c("00003", "00001")) # nolint start: line_length_linter expect_equal(url_daily$url, - "https://waterservices.usgs.gov/nwis/dv/?site=01594440&format=waterml%2C1.1&ParameterCd=00060%2C00010&StatCd=00003%2C00001&startDT=1985-01-01") + "https://api.waterdata.usgs.gov/ogcapi/v0/collections/daily/items?f=json&lang=en-US&time=2024-01-01T00%3A00%3A00Z%2F..&skipGeometry=FALSE&limit=10000") - url_unit <- constructNWISURL(siteNumber, pCode, "2012-06-28", "2012-06-30", "iv") + url_works <- dataRetrieval:::walk_pages(url_daily) + expect_true(nrow(url_works) > 0) + + url_ts_meta <- construct_api_requests(monitoring_location_id = siteNumber, + parameter_code = pCode, + service = "time-series-metadata") expect_equal( - url_unit$url, - "https://nwis.waterservices.usgs.gov/nwis/iv/?site=01594440&format=waterml%2C1.1&ParameterCd=00060%2C00010&startDT=2012-06-28&endDT=2012-06-30" + url_ts_meta$url, + "https://api.waterdata.usgs.gov/ogcapi/v0/collections/time-series-metadata/items?f=json&lang=en-US&skipGeometry=FALSE&limit=10000" ) + + url_works_ts <- dataRetrieval:::walk_pages(url_ts_meta) + expect_true(nrow(url_works_ts) > 0) - url_daily_tsv <- constructNWISURL(siteNumber, pCode, startDate, endDate, "dv", - statCd = c("00003", "00001"), format = "tsv" - ) + url_ml <- construct_api_requests(id = siteNumber, + service = "monitoring-locations") - expect_equal(url_daily_tsv$url, "https://waterservices.usgs.gov/nwis/dv/?site=01594440&format=rdb%2C1.0&ParameterCd=00060%2C00010&StatCd=00003%2C00001&startDT=1985-01-01") + expect_equal(url_ml$url, "https://api.waterdata.usgs.gov/ogcapi/v0/collections/monitoring-locations/items?f=json&lang=en-US&skipGeometry=FALSE&limit=10000&id=USGS-01594440") + url_works_ml <- dataRetrieval:::walk_pages(url_ml) + expect_true(nrow(url_works_ml) > 0) + url_use <- constructUseURL( years = c(1990, 1995), stateCd = "Ohio", From 8ea1fb33c7dee1001c54c4f09d926428dd559e0d Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 3 Jun 2025 10:13:03 -0500 Subject: [PATCH 062/117] last test cleanup --- tests/testthat/tests_userFriendly_fxns.R | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/testthat/tests_userFriendly_fxns.R b/tests/testthat/tests_userFriendly_fxns.R index c14d8892..e65ea7c0 100644 --- a/tests/testthat/tests_userFriendly_fxns.R +++ b/tests/testthat/tests_userFriendly_fxns.R @@ -83,15 +83,10 @@ test_that("peak, rating curves, surface-water measurements", { data <- readNWISmeas(siteNumbers) expect_is(data$agency_cd, "character") - siteINFO <- readNWISsite("05114000") - expect_is(siteINFO$agency_cd, "character") - expect_equal(siteINFO$site_no, "05114000") - siteINFO_USGS <- read_USGS_monitoring_location("USGS-05114000") expect_is(siteINFO_USGS$agency_code, "character") expect_equal(siteINFO_USGS$monitoring_locations_id, "USGS-05114000") - siteINFOMulti_USGS <- read_USGS_monitoring_location(c("USGS-05114000", "USGS-09423350")) expect_true(nrow(siteINFOMulti_USGS) == 2) From 96c19881b60ef37714dcad10a478d0c03e04d367 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 3 Jun 2025 13:16:57 -0500 Subject: [PATCH 063/117] clean up message --- R/AAA.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/AAA.R b/R/AAA.R index ec650725..32340d30 100644 --- a/R/AAA.R +++ b/R/AAA.R @@ -55,7 +55,7 @@ https://doi-usgs.github.io/dataRetrieval/articles/Status.html") new_nwis_message <- function(){ return("ALERT: All NWIS services are slated for decommission -and dataRetrieval functions will be updated to use new API endpoints. +and new dataRetrieval functions will be added. For up-to-date information, see: https://doi-usgs.github.io/dataRetrieval/articles/Status.html") } From 06293bf70921a375867d2e721cad42d6de63399e Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 3 Jun 2025 13:28:55 -0500 Subject: [PATCH 064/117] fix NLDI test --- R/findNLDI.R | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/R/findNLDI.R b/R/findNLDI.R index fb109aef..c32e0d1b 100644 --- a/R/findNLDI.R +++ b/R/findNLDI.R @@ -91,15 +91,14 @@ get_nldi <- function(url, type = "", use_sf = FALSE, warn = TRUE) { # If successful ... if (res$status_code == 200) { # Interpret as text - d <- httr2::resp_body_string(res) - - if (d == "") { - + if(length(res$body) > 0){ + d <- httr2::resp_body_string(res) + } else { if(warn){ warning("No data returned for: ", url, call. = FALSE) } - return(NULL) + return(NULL) } if (use_sf) { From 8150512947dda591afdf528ad56f3cc7c16ea717 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 3 Jun 2025 14:28:39 -0500 Subject: [PATCH 065/117] make sure this works on gitlab --- vignettes/read_USGS_functions.Rmd | 156 +++++++++++++++++------------- 1 file changed, 87 insertions(+), 69 deletions(-) diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 15659f69..94166a10 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -25,38 +25,111 @@ options(continue = " ", knitr::opts_chunk$set( echo = TRUE, - message = TRUE, + message = FALSE, + warning = FALSE, fig.height = 4, fig.width = 7 ) ``` -As we bid adieu to the NWIS web services, we welcome a new web service offering, the USGS Water Data OGC APIs. +As we bid adieu to the NWIS web services, we welcome a host of new web service offering, the [USGS Water Data OGC APIs](https://api.waterdata.usgs.gov/ogcapi/v0/). This is a modern access point for USGS water data. The USGS is planning to modernize **all** web services in the near future. For each of these updates, `dataRetrieval` will create a new function to access the new services. -# New USGS data access +This document will introduce each new function (note as time goes on, we'll update this document to include additional functions). Scroll down for tips on what you'll need to do to convert your legacy scripts to use these modern functions. + +Note the timeline for the NWIS servers being shut down is currently very uncertain. We'd recommend incorporating these new functions as soon as possible to avoid future headaches. + +# Monitoring Location + +The `read_USGS_monitoring_location` function replaces the `readNWISsite` function. + +`r dataRetrieval:::get_description("monitoring-locations")` + +To access these services on a web browser, go to . + +Here is a simple example of requesting one known USGS site: + +```{r} +sites_information <- read_USGS_monitoring_location(monitoring_location_id = "USGS-01491000") +``` + +The output includes the following information: + +```{r echo=FALSE} +knitr::kable(t(sites_information)) +``` + +Maybe that is more information than you need. You can specify which columns get returned with the "properties" argument: + +```{r} +site_info <- read_USGS_monitoring_location(monitoring_location_id = "USGS-01491000", + properties = c("monitoring_locations_id", + "site_type", + "drainage_area", + "monitoring_location_name")) + +knitr::kable(site_info) + +``` + +Any of those original outputs can also be inputs to the function! So let's say we want to find all the stream sites in Wisconsin: + +```{r} +sites_wi <- read_USGS_monitoring_location(state_name = "Wisconsin", + site_type = "Stream") +``` + +The returned data includes a column "geometry" which is a collection of simple feature (sf) points. This allows for seamless integration with the `sf` package. Here are 2 quick examples of using the `sf` object in ggplot2 and leaflet: + +```{r} +library(ggplot2) + +ggplot(data = sites_wi) + + geom_sf() + + theme_minimal() +``` + +```{r} +library(leaflet) +leaflet_crs <- "+proj=longlat +datum=WGS84" #default leaflet crs +leaflet(data = sites_wi |> + sf::st_transform(crs = leaflet_crs)) |> + addProviderTiles("CartoDB.Positron") |> + addCircleMarkers(popup = ~monitoring_location_name, + radius = 0.1, + opacity = 1) + +``` + +# Time Series Metadata + +The `read_USGS_ts_meta` function replaces the `whatNWISdata` function. + +`r dataRetrieval:::get_description("time-series-metadata")` + +To access these services on a web browser, go to . + + +```{r} +ts_available <- read_USGS_ts_meta(monitoring_location_id = "USGS-01491000", + parameter_code = c("00060", "00010")) + +``` -This is a modern access point for USGS data. The USGS is planning to modernize all web services in the near future. For each of these updates, `dataRetrieval` will create a new function to access the new services. # Daily Values +The `read_USGS_daily` function replaces the `readNWISdv` function. + `r dataRetrieval:::get_description("daily")` To access these services on a web browser, go to . -The `read_USGS_daily` function will eventually replace the `readNWISdv` function. Generally, both functions look pretty similar, though we've updated argument names to align with new Water Mission Area conventions:: + ```{r} library(dataRetrieval) -dv_legacy <- readNWISdv(siteNumbers = "01491000", - parameterCd = c("00060", "00010"), - statCd = "00003", - startDate = "2023-01-01", - endDate = "2025-05-06") - -nrow(dv_legacy) - daily_modern <- read_USGS_daily(monitoring_location_id = "USGS-01491000", parameter_code = c("00060", "00010"), statistic_id = "00003", @@ -65,7 +138,7 @@ daily_modern <- read_USGS_daily(monitoring_location_id = "USGS-01491000", nrow(daily_modern) ``` -The first thing you might notice is that the legacy service serves data in a "wide" format, which means there are multiple observations in a single row of the data frame (see: [Pivot Data](articles/long_to_wide.html)). The new services return the data in a "long" format, so that only one observation is returned per row. This follows tidy principles and is often preferred in scripting languages. We'll look at the returned data with a `ggplot2` plot to show a very simple example of the power of a long format: +The first thing you might notice is that the new service serves data in a "long" format, which means there is just a single observations per row of the data frame (see: [Pivot Data](articles/long_to_wide.html)). ```{r} library(ggplot2) @@ -100,62 +173,7 @@ head(dv_2) -# Time Series Metadata - -`r dataRetrieval:::get_description("time-series-metadata")` -To access these services on a web browser, go to . - -The `read_USGS_ts_meta` function will eventually replace the `whatNWISdata` function. Generally, both functions look pretty similar: - -```{r} -data_legacy <- whatNWISdata(siteNumbers = "01491000", - parameterCd = c("00060", "00010")) -``` - -```{r echo=FALSE} - -knitr::kable(data_legacy[,c("parm_cd", - "stat_cd", - "begin_date", - "end_date", - "count_nu")]) - -``` - -```{r} -data_modern <- read_USGS_ts_meta(monitoring_location_id = "USGS-01491000", - parameter_code = c("00060", "00010")) - -``` - -```{r echo=FALSE} - -knitr::kable(data_modern[,c("parameter_code", - "parameter_name", - "statistic_id", - "computation_identifier", - "begin", - "end")]) - -``` - -# Site Information - -`r dataRetrieval:::get_description("monitoring-locations")` - -The `read_USGS_monitoring_location` function will eventually replace the `readNWISsite` function. Generally, both functions look pretty similar: - -```{r} -sites_legacy <- readNWISsite(siteNumbers = "01491000") -names(sites_legacy) -``` - -```{r} -sites_modern <- read_USGS_monitoring_location(monitoring_location_id = "USGS-01491000") -names(sites_modern) - -``` # API Tokens From eab31be6b420e9d9d1642f66ea5a25b7562b27e2 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 3 Jun 2025 15:26:43 -0500 Subject: [PATCH 066/117] Remove bbox from daily --- R/read_USGS_daily.R | 7 --- vignettes/read_USGS_functions.Rmd | 97 ++++++++++++++++++------------- 2 files changed, 58 insertions(+), 46 deletions(-) diff --git a/R/read_USGS_daily.R b/R/read_USGS_daily.R index 3a5f96ad..34f5f757 100644 --- a/R/read_USGS_daily.R +++ b/R/read_USGS_daily.R @@ -17,12 +17,6 @@ #' @param properties A vector of requested columns to be returned from the query. #' Available options are: #' `r schema <- check_OGC_requests(endpoint = "daily", type = "schema"); paste(names(schema$properties), collapse = ", ")` -#' @param bbox Only features that have a geometry that intersects the bounding -#' box are selected.The bounding box is provided as four or six numbers, depending -#' on whether the coordinate reference system includes a vertical axis (height or -#' depth). Coordinates are assumed to be in crs 4326. The expected format is a numeric -#' vector structured: c(xmin,ymin,xmax,ymax). Another way to think of it is c(Western-most longitude, -#' Southern-most latitude, Eastern-most longitude, Northern-most longitude). #' @param limit The optional limit parameter limits the number of items that are #' presented in the response document. Only items are counted that are on the #' first level of the collection in the response document. Nested objects @@ -67,7 +61,6 @@ read_USGS_daily <- function(monitoring_location_id = NA_character_, parameter_code = NA_character_, statistic_id = NA_character_, properties = NA_character_, - bbox = NA, time_series_id = NA_character_, daily_id = NA_character_, approval_status = NA_character_, diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 94166a10..f8a0af6f 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -33,11 +33,37 @@ knitr::opts_chunk$set( ``` -As we bid adieu to the NWIS web services, we welcome a host of new web service offering, the [USGS Water Data OGC APIs](https://api.waterdata.usgs.gov/ogcapi/v0/). This is a modern access point for USGS water data. The USGS is planning to modernize **all** web services in the near future. For each of these updates, `dataRetrieval` will create a new function to access the new services. +As we bid adieu to the NWIS web services, we welcome a host of new web service offering: the [USGS Water Data OGC APIs](https://api.waterdata.usgs.gov/ogcapi/v0/). This is a modern access point for USGS water data. The USGS is planning to modernize **all** web services in the near future. For each of these updates, `dataRetrieval` will create a new function to access the new services. -This document will introduce each new function (note as time goes on, we'll update this document to include additional functions). Scroll down for tips on what you'll need to do to convert your legacy scripts to use these modern functions. +This document will introduce each new function (note as time goes on, we'll update this document to include additional functions). Scroll down for tips on what you'll need to do to convert your legacy scripts to use these modern functions. Note the timeline for the NWIS servers being shut down is currently very uncertain. We'd recommend incorporating these new functions as soon as possible to avoid future headaches. -Note the timeline for the NWIS servers being shut down is currently very uncertain. We'd recommend incorporating these new functions as soon as possible to avoid future headaches. +# New Features + +Each new "API endpoint" will deliver a new type of USGS data. Currently the available endpoints are "monitoring-locations", "time-series-metadata", and "daily". All of these new endpoints offer some new features that the NWIS services did not have: + +* Users can pick which columns are returned with the "properties" argument. Available properties can be discovered with the `check_OGC_requests` function: + +```{r} +daily_schema <- check_OGC_requests(endpoint = "daily", type = "schema") +names(daily_schema$properties) + +``` + +* Any column that is returned can also be queried on! The main `dataRetrieval` functions will take vector inputs. Each query is appended as a boolean AND. For instance, if you ask for a vector of monitoring location ids and a vector of parameter codes, you will get data back for just those monitoring locations AND the specified parameter codes. Towards the bottom of this document, we'll discuss ways to make more specific queries using "Common Query Language" or CQL. + +* Data is returned with a "geometry" column, which is a simple feature object, allowing the data to be integrated with the `sf` package and associated geospatial workflows. + +* When you look at the help file for the new functions, you’ll also notice there are many more arguments. These are mostly set by default to NA. You **DO NOT** need to (and most like should not!) specify all of these parameters. + +* You can register an API key for use with USGS water data APIs. There are now limits on how many queries can be requested per IP address per hour. If you find yourself running into limits, you can request an API token here: + +Then save your token in your .Renviron file like this: + +``` +API_USGS_PAT = "my_super_secret_token" +``` + +You can use `usethis::edit_r_environ()` to edit find and open your .Renviron file. You will need to restart R for that variable to be recognized. # Monitoring Location @@ -116,6 +142,17 @@ ts_available <- read_USGS_ts_meta(monitoring_location_id = "USGS-01491000", ``` +The returning tables gives information on all available time series. For instance, we can pull a few columns out and see when each time series started, ended, or was last modified. + +```{r echo=FALSE} +ts_available1 <- ts_available[,c("parameter_name", "statistic_id", "begin", "end", "last_modified")] +ts_available1 <- sf::st_drop_geometry(ts_available1) +ts_available1$begin <- as.Date(ts_available1$begin) +ts_available1$end <- as.Date(ts_available1$end) +ts_available1$last_modified <- as.Date(ts_available1$last_modified) +knitr::kable(ts_available1) + +``` # Daily Values @@ -150,43 +187,7 @@ ggplot(data = daily_modern) + ``` -## Additional new features - -* Another feature of the modern returned data frame is that it has a "geometry" column which is an `sf` class and can therefore be used in `sf` processes. -* Flexible column selection: - -```{r} -dv_2 <- read_USGS_daily(monitoring_location_id = "USGS-01491000", - parameter_code = c("00060"), - properties = c("monitoring_location_id", - "value", - "time"), - statistic_id = "00003", - time = c("2022-10-01", "2023-10-01")) -head(dv_2) -``` - -* When you look at the help file for the new function `?read_USGS_daily`, you'll also notice there are many more arguments. These are mostly set by default to NA. You DO NOT need to specify all of these parameters. - -* Flexible data/last_modified - - - - - - -# API Tokens - -If you find yourself running into limits, you can request an API token here: - -Then save your token in your .Renviron file like this: - -``` -API_USGS_PAT = "my_super_secret_token" -``` - -You can use `usethis::edit_r_environ()` to edit find and open your .Renviron file. You will need to restart R for that variable to be recognized. # Notes on dataRetrieval development @@ -205,3 +206,21 @@ Historically, we allowed users to customize their queries via the `...` argument Under the hood, `dataRetrieval` changed the dependency from `httr` to `httr2`. `httr2` is the modern R package for web requests that is actively developed/maintained. As we develop functions for the modern USGS web services, we'll continue to explore updating package dependencies. Since the new services offer geospatial output, we have recommend using the `sf` package. The `whisker` package was also included to help create POST CQL2 queries. +## Developmental workflow + +CRAN-stable documentation will be available on the GitHub pages: + + +In-development documentation will be available on the USGS GitLab pages: + + +Development of `dataRetrieval` will happen on a git branch called "develop". The "develop" branch will only move to the "main" branch when we submit to CRAN, unless there are bug fixes that pertain to the CRAN release. The "develop" branch WILL change frequently, and there are no promises of future behavior. Users must accept that they are using those functions at their own risk. If you willing to accept this risk, the installation instructions are: + +```{r eval=FALSE} +library(remotes) + +install_github("DOI-USGS/dataRetrieval", + ref = "develop") + +``` + From 5b011c844dab6cf2e8c37cce92a5d03bb41b1cce Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 3 Jun 2025 15:30:39 -0500 Subject: [PATCH 067/117] Add API key info to readme --- README.Rmd | 12 ++++++++++++ README.md | 15 +++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/README.Rmd b/README.Rmd index a3636ef7..5e6af2d7 100644 --- a/README.Rmd +++ b/README.Rmd @@ -72,6 +72,18 @@ install.packages("dataRetrieval") The USGS is planning to modernize all web services in the near future. For each of these updates, `dataRetrieval` will create a new function to access the new services. +## API Token + +You can register an API key for use with USGS water data APIs. There are now limits on how many queries can be requested per IP address per hour. If you find yourself running into limits, you can request an API token here: + +Then save your token in your .Renviron file like this: + +``` +API_USGS_PAT = "my_super_secret_token" +``` + +You can use `usethis::edit_r_environ()` to edit find and open your .Renviron file. You will need to restart R for that variable to be recognized. + ## New Features ### Style diff --git a/README.md b/README.md index a66f84fc..e00bd3bf 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,21 @@ The USGS is planning to modernize all web services in the near future. For each of these updates, `dataRetrieval` will create a new function to access the new services. +## API Token + +You can register an API key for use with USGS water data APIs. There are +now limits on how many queries can be requested per IP address per hour. +If you find yourself running into limits, you can request an API token +here: + +Then save your token in your .Renviron file like this: + + API_USGS_PAT = "my_super_secret_token" + +You can use `usethis::edit_r_environ()` to edit find and open your +.Renviron file. You will need to restart R for that variable to be +recognized. + ## New Features ### Style From b142641cfec91c20551bbb2dc4856462bd54ea46 Mon Sep 17 00:00:00 2001 From: Elise Hinman Date: Tue, 3 Jun 2025 15:32:27 -0500 Subject: [PATCH 068/117] initial edits --- vignettes/Status.Rmd | 83 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 75 insertions(+), 8 deletions(-) diff --git a/vignettes/Status.Rmd b/vignettes/Status.Rmd index 1afc2f8e..413a4008 100644 --- a/vignettes/Status.Rmd +++ b/vignettes/Status.Rmd @@ -15,9 +15,82 @@ editor_options: chunk_output_type: console --- -This page will be updated frequently with information about the status of dataRetrieval services. +# Overview + +Welcome to the `dataRetrieval` status page. This page will be updated frequently with information about the status of `dataRetrieval` services. + +Please contact the Computational Tools team at CompTools@usgs.gov with questions. + +# New Water Data APIs + +
+ What is an application programming interface (API)? + When you run a `dataRetrieval` function like `read_USGS_samples()` or `readNWISdv()`, you are actually generating a URL that contains within it specifications of the dataset in which you are interested (e.g. which monitoring locations, characteristics, pcodes, start and end dates, etc.). The format of that URL is special: it is parsed by a USGS-specific API web service, which translates the request into a database call. It then packages the response object from the database (with the data) into a format that can be sent to and then unpacked by the user. `dataRetrieval` takes a lot of the guesswork out of this process by generating the URL, sending it to the API, and wrangling the response object into a tabular dataset. +
+
+The USGS is in the process of creating new, [publicly available APIs](https://api.waterdata.usgs.gov/) (a.k.a. "web services") to replace the existing [WaterServices](https://waterservices.usgs.gov/). `dataRetrieval` relies upon these web services to provide monitoring location information and water quantity/quality datasets. As new web services come online, `dataRetrieval` will be furnished with new functions to accommodate these changes. + +The table below shows the **existing** NWIS functions and, if applicable, their slated replacements. Note that several functions do not have direct replacements because the new services to support them do not yet exist. We will update this table as new services and functions come online. Also note that some new functions may only be available on the "develop" branch of `dataRetrieval` (`remotes::install_github("DOI-USGS/dataRetrieval", ref = "develop")`). + +```{r echo=FALSE} +df <- data.frame( + Legacy = c( + "readNWISdv", + "readNWISsite", + "whatNWISsites", + "", + "readNWISuv", + "readNWISrating", + "readNWISstat", + "readNWISmeas", + "readNWISpeak", + "readNWISgwl", + "readNWISuse", + "readNWISdata", + "whatNWISdata", + "readNWISpCode", + "readNWISqw" + ), + New = c( + "read_USGS_daily", + "read_USGS_monitoring_location", + "read_USGS_ts_meta", + "read_USGS_data", + rep("", 10), + "read_USGS_samples" + ), + "Available on (branch)" = c("develop", "develop", "develop", "develop", rep("", 10), "main (CRAN)") +) + +knitr::kable(df, col.names = c("WaterServices (legacy) function", "Water Data (new) function", "Available on (branch name)")) + +``` + +If you want to learn more about the new water data APIs, check out the ["What's new with WDFN APIs?" blog post](https://waterdata.usgs.gov/blog/api-whats-new-wdfn-apis/), as well as the [documentation](https://api.waterdata.usgs.gov/docs/) available on api.waterdata.usgs.gov. + +## API Keys + +Do you make *a lot* of `dataRetrieval` requests per day? ...On the order of more than 50 function calls per hour? As you switch your workflows over to the new Water Data API functions, consider grabbing yourself an API key, which will bump your limit up to 1,000 requests per hour. Head to the [sign up page](https://api.waterdata.usgs.gov/signup) to get a token. + +One you have your API key, add it to your `.Renviron` file like this: + +```{r} +API_USGS_PAT = "[your api key]" +``` + +Replace [your api key] with the alphanumeric code provided by the sign-up page. + +# Discrete Data + +Beyond the Water Services to Water Data transition, discrete water quality datasets have had some recent updates as well. + +## Samples Data + +## WQP + + + -Contact CompTools@usgs.gov with additional questions. # Overview @@ -47,12 +120,6 @@ New USGS data (post March 11, 2024) **are temporarily not accessible** on the ** However, new USGS data **are accessible** in a pre-release (*beta*) version of the [WQP web page](https://www.waterqualitydata.us/beta/) and new [wqx3 web services](https://waterqualitydata.us/wqx3/). Data are available in the "WQX version 3.0 format" (WQX = [Water Quality Exchange](https://exchangenetwork.net/data-exchange/wqx/)) for these new "Data Profiles" (how the data is formatted by the WQP): -* Monitoring Location -* Results - Narrow -* Results - Full Physical Chemical -* Results - Basic Physical Chemical -* Sampling Activity - Guidance on how to use the new web page and web services are available in the [User Guide](https://www.waterqualitydata.us/beta/portal_userguide/) and [Web Services Guide](https://www.waterqualitydata.us/beta/webservices_documentation/). Additional profiles will continue to be added over time. **Disclaimer:** During the beta period, users may encounter bugs or identify issues with the implementation of the WQX 3.0 format: we welcome (and encourage!) your feedback to help improve these offerings, just send an email to WQX@epa.gov. From 78717620b1e985ffdb3e2e06700b1c6ff24a013b Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 3 Jun 2025 15:37:25 -0500 Subject: [PATCH 069/117] add discrete stuff --- man/read_USGS_daily.Rd | 8 -------- vignettes/read_USGS_functions.Rmd | 3 ++- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/man/read_USGS_daily.Rd b/man/read_USGS_daily.Rd index 3b4c8185..9ce44cb0 100644 --- a/man/read_USGS_daily.Rd +++ b/man/read_USGS_daily.Rd @@ -9,7 +9,6 @@ read_USGS_daily( parameter_code = NA_character_, statistic_id = NA_character_, properties = NA_character_, - bbox = NA, time_series_id = NA_character_, daily_id = NA_character_, approval_status = NA_character_, @@ -34,13 +33,6 @@ read_USGS_daily( Available options are: geometry, id, time_series_id, monitoring_location_id, parameter_code, statistic_id, time, value, unit_of_measure, approval_status, qualifier, last_modified} -\item{bbox}{Only features that have a geometry that intersects the bounding -box are selected.The bounding box is provided as four or six numbers, depending -on whether the coordinate reference system includes a vertical axis (height or -depth). Coordinates are assumed to be in crs 4326. The expected format is a numeric -vector structured: c(xmin,ymin,xmax,ymax). Another way to think of it is c(Western-most longitude, -Southern-most latitude, Eastern-most longitude, Northern-most longitude).} - \item{time_series_id}{A unique identifier representing a single time series. This corresponds to the \code{id} field in the \code{time-series-metadata} endpoint.} \item{daily_id}{A universally unique identifier (UUID) representing a single version of a record. It is not stable over time. Every time the record is refreshed in our database (which may happen as part of normal operations and does not imply any change to the data itself) a new ID will be generated. To uniquely identify a single observation over time, compare the \code{time} and \code{time_series_id} fields; each time series will only have a single observation at a given \code{time}.} diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index f8a0af6f..21216c67 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -187,8 +187,9 @@ ggplot(data = daily_modern) + ``` +# Discrete Samples - +Discrete USGS water quality can be accessed via the `read_USGS_samples` function. While this is a new, modern USGS endpoint, it is not served in the same infrastructure as the rest of these new advertised functions. See [Samples Data](articles/samples_data.html)) for information on accessing USGS discrete water quality data. # Notes on dataRetrieval development From 3bf91ce947d502361e34b83ed4fd187bd011c728 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Tue, 3 Jun 2025 15:57:37 -0500 Subject: [PATCH 070/117] put bbox back in daily --- R/read_USGS_daily.R | 7 +++++++ man/read_USGS_daily.Rd | 8 ++++++++ tests/testthat/tests_userFriendly_fxns.R | 20 ++++++++++---------- vignettes/read_USGS_functions.Rmd | 13 ++++++------- 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/R/read_USGS_daily.R b/R/read_USGS_daily.R index f1706a67..d6341f5f 100644 --- a/R/read_USGS_daily.R +++ b/R/read_USGS_daily.R @@ -17,6 +17,12 @@ #' @param properties A vector of requested columns to be returned from the query. #' Available options are: #' `r schema <- check_OGC_requests(endpoint = "daily", type = "schema"); paste(names(schema$properties), collapse = ", ")` +#' @param bbox Only features that have a geometry that intersects the bounding +#' box are selected.The bounding box is provided as four or six numbers, depending +#' on whether the coordinate reference system includes a vertical axis (height or +#' depth). Coordinates are assumed to be in crs 4326. The expected format is a numeric +#' vector structured: c(xmin,ymin,xmax,ymax). Another way to think of it is c(Western-most longitude, +#' Southern-most latitude, Eastern-most longitude, Northern-most longitude). #' @param limit The optional limit parameter limits the number of items that are #' presented in the response document. Only items are counted that are on the #' first level of the collection in the response document. Nested objects @@ -71,6 +77,7 @@ read_USGS_daily <- function(monitoring_location_id = NA_character_, limit = 10000, skipGeometry = NA, time = NA_character_, + bbox = NA, convertType = TRUE){ message("Function in development, use at your own risk.") diff --git a/man/read_USGS_daily.Rd b/man/read_USGS_daily.Rd index 9ce44cb0..f30acb71 100644 --- a/man/read_USGS_daily.Rd +++ b/man/read_USGS_daily.Rd @@ -19,6 +19,7 @@ read_USGS_daily( limit = 10000, skipGeometry = NA, time = NA_character_, + bbox = NA, convertType = TRUE ) } @@ -77,6 +78,13 @@ Examples: Only features that have a \code{time} that intersects the value of datetime are selected. If a feature has multiple temporal properties, it is the decision of the server whether only a single temporal property is used to determine the extent or all relevant temporal properties.} +\item{bbox}{Only features that have a geometry that intersects the bounding +box are selected.The bounding box is provided as four or six numbers, depending +on whether the coordinate reference system includes a vertical axis (height or +depth). Coordinates are assumed to be in crs 4326. The expected format is a numeric +vector structured: c(xmin,ymin,xmax,ymax). Another way to think of it is c(Western-most longitude, +Southern-most latitude, Eastern-most longitude, Northern-most longitude).} + \item{convertType}{logical, defaults to \code{TRUE}. If \code{TRUE}, the function will convert the data to dates and qualifier to string vector.} } diff --git a/tests/testthat/tests_userFriendly_fxns.R b/tests/testthat/tests_userFriendly_fxns.R index e65ea7c0..4d2b39ef 100644 --- a/tests/testthat/tests_userFriendly_fxns.R +++ b/tests/testthat/tests_userFriendly_fxns.R @@ -160,16 +160,16 @@ test_that("WQP qw tests", { nameToUse <- "Specific conductance" pcodeToUse <- "00095" - # INFO_WQP <- readWQPqw("USGS-04024315", pcodeToUse, - # startDate = "", endDate = "", legacy = FALSE) - # expect_is(INFO_WQP$Activity_StartDateTime, "POSIXct") - # - # INFO2 <- readWQPqw("WIDNR_WQX-10032762", nameToUse, - # startDate = "", endDate = "", legacy = FALSE) - # expect_is(INFO2$Activity_StartDateTime, "POSIXct") - # - # df <- readWQPqw("USGS-04193500", parameterCd = "00665", legacy = FALSE) - # expect_true(nrow(df) > 0) + INFO_WQP <- readWQPqw("USGS-04024315", pcodeToUse, + startDate = "", endDate = "", legacy = FALSE) + expect_is(INFO_WQP$Activity_StartDateTime, "POSIXct") + + INFO2 <- readWQPqw("WIDNR_WQX-10032762", nameToUse, + startDate = "", endDate = "", legacy = FALSE) + expect_is(INFO2$Activity_StartDateTime, "POSIXct") + + df <- readWQPqw("USGS-04193500", parameterCd = "00665", legacy = FALSE) + expect_true(nrow(df) > 0) df2 <- readWQPqw("USGS-05427718", parameterCd = "all") expect_true(nrow(df2) > 0) diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 21216c67..4b23dcb2 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -41,6 +41,12 @@ This document will introduce each new function (note as time goes on, we'll upda Each new "API endpoint" will deliver a new type of USGS data. Currently the available endpoints are "monitoring-locations", "time-series-metadata", and "daily". All of these new endpoints offer some new features that the NWIS services did not have: +* Any column that is returned can also be queried on! The main `dataRetrieval` functions will take vector inputs. Each query is appended as a boolean AND. For instance, if you ask for a vector of monitoring location ids and a vector of parameter codes, you will get data back for just those monitoring locations AND the specified parameter codes. Towards the bottom of this document, we'll discuss ways to make more specific queries using "Common Query Language" or CQL. + +* Data is returned with a "geometry" column, which is a simple feature object, allowing the data to be integrated with the `sf` package and associated geospatial workflows. + +* When you look at the help file for the new functions, you’ll also notice there are many more arguments. These are mostly set by default to NA. You **DO NOT** need to (and most likely should not!) specify all of these parameters. + * Users can pick which columns are returned with the "properties" argument. Available properties can be discovered with the `check_OGC_requests` function: ```{r} @@ -49,12 +55,6 @@ names(daily_schema$properties) ``` -* Any column that is returned can also be queried on! The main `dataRetrieval` functions will take vector inputs. Each query is appended as a boolean AND. For instance, if you ask for a vector of monitoring location ids and a vector of parameter codes, you will get data back for just those monitoring locations AND the specified parameter codes. Towards the bottom of this document, we'll discuss ways to make more specific queries using "Common Query Language" or CQL. - -* Data is returned with a "geometry" column, which is a simple feature object, allowing the data to be integrated with the `sf` package and associated geospatial workflows. - -* When you look at the help file for the new functions, you’ll also notice there are many more arguments. These are mostly set by default to NA. You **DO NOT** need to (and most like should not!) specify all of these parameters. - * You can register an API key for use with USGS water data APIs. There are now limits on how many queries can be requested per IP address per hour. If you find yourself running into limits, you can request an API token here: Then save your token in your .Renviron file like this: @@ -62,7 +62,6 @@ Then save your token in your .Renviron file like this: ``` API_USGS_PAT = "my_super_secret_token" ``` - You can use `usethis::edit_r_environ()` to edit find and open your .Renviron file. You will need to restart R for that variable to be recognized. # Monitoring Location From 2dc44aefa4cad2515b9056cd2dd34569b675fb66 Mon Sep 17 00:00:00 2001 From: Elise Hinman Date: Tue, 3 Jun 2025 17:30:52 -0500 Subject: [PATCH 071/117] more edits to status vignette --- vignettes/Status.Rmd | 230 ++++++------------------------------------- 1 file changed, 32 insertions(+), 198 deletions(-) diff --git a/vignettes/Status.Rmd b/vignettes/Status.Rmd index 413a4008..fc76787e 100644 --- a/vignettes/Status.Rmd +++ b/vignettes/Status.Rmd @@ -35,6 +35,7 @@ The table below shows the **existing** NWIS functions and, if applicable, their ```{r echo=FALSE} df <- data.frame( Legacy = c( + "readNWISqw (deprecated)", "readNWISdv", "readNWISsite", "whatNWISsites", @@ -48,18 +49,17 @@ df <- data.frame( "readNWISuse", "readNWISdata", "whatNWISdata", - "readNWISpCode", - "readNWISqw" + "readNWISpCode" ), New = c( + "read_USGS_samples", "read_USGS_daily", "read_USGS_monitoring_location", "read_USGS_ts_meta", "read_USGS_data", - rep("", 10), - "read_USGS_samples" + rep("", 10) ), - "Available on (branch)" = c("develop", "develop", "develop", "develop", rep("", 10), "main (CRAN)") + "Available on (branch)" = c("main (CRAN)", "develop", "develop", "develop", "develop", rep("", 10)) ) knitr::kable(df, col.names = c("WaterServices (legacy) function", "Water Data (new) function", "Available on (branch name)")) @@ -82,52 +82,21 @@ Replace [your api key] with the alphanumeric code provided by the sign-up page. # Discrete Data -Beyond the Water Services to Water Data transition, discrete water quality datasets have had some recent updates as well. +In March 2024, NWIS **discrete water quality** services were "frozen": any public data retrieval using `readNWISqw()` no longer included any new data. Concurrently, the main [Water Quality Portal (WQP) API](https://www.waterqualitydata.us/) stopped serving new and updated USGS data. Now, new and updated data are available from the [USGS Samples API](https://waterdata.usgs.gov/download-samples/#dataProfile=site) (for USGS data only) or in the [beta version](https://www.waterqualitydata.us/beta/) of the WQP (both USGS and non-USGS data). -## Samples Data - -## WQP - - - - - -# Overview - -On March 11, 2024, NWIS **discrete water quality** services were "frozen": any public data retrieval using `readNWISqw()` did not include any new data. As of dataRetrieval v2.7.17, `readNWISqw()` has been retired and replaced by `readWQPqw()`. Learn more about the change and where to find the new samples data in our [blog](https://waterdata.usgs.gov/blog/changes-to-sample-data/). - -If you have additional questions about the NWIS qw data service, email CompTools@usgs.gov. - -## Latest CRAN update - -* dataRetrieval now uses `httr2` instead of `httr` under the hood for constructing web service calls. The `httr2` package is considered the modern replacement for `httr`, and has support going forward (unlike `httr`). Depending on how you perform the package updates, you may need to install `httr2`. - -* dataRetrieval will give a message with the requested URL each time a web service call is made. These can be hidden by wrapping dataRetrieval calls with `suppressMessages`. That being said, it is very informative to see exactly where the data is coming from. - -* The output of the "constructURL..." functions are now `httr2` requests instead of a character string. - -* The update to `httr2` will give us more flexibility to set up the eventual replacements to the NWIS web services. Over the next year, expect to see some new and major updates to USGS data access. dataRetrieval WILL stay on the cutting edge for accessing new USGS APIs. - -* WQP continues to default to the legacy system (that does not include recent USGS discrete sample data). To access the most recent USGS data from the new "WQX3" services use the function `readWQPqw` and set the argument `legacy=FALSE` or use the function `readWQPdata` and set the argument `service = "ResultWQX3"`. Why aren't the new services set as default? This is because the WQP itself still considers those services "beta", and therefore performance is not guaranteed. +What does this mean for water quality data users of `dataRetrieval`? Check out the sections below for more specifics. -* Finally, saving the best for last! There's a new set of functions that access the new USGS "samples-data" services. This is a USGS-specific service for discrete sample data. If you are only interested in USGS discrete water quality data (as opposed to USGS AND non-USGS discrete water quality data available from Water Quality Portal), you can use the `read_USGS_samples` function. - -Read more about it here: - -## Locating USGS data using the Water Quality Portal - -New USGS data (post March 11, 2024) **are temporarily not accessible** on the **main** Water Quality Portal (WQP) page (www.waterqualitydata.us). Data are still being collected, but are not available on this webpage. This limited availability is expected to last a few months. +## Samples Data -However, new USGS data **are accessible** in a pre-release (*beta*) version of the [WQP web page](https://www.waterqualitydata.us/beta/) and new [wqx3 web services](https://waterqualitydata.us/wqx3/). Data are available in the "WQX version 3.0 format" (WQX = [Water Quality Exchange](https://exchangenetwork.net/data-exchange/wqx/)) for these new "Data Profiles" (how the data is formatted by the WQP): +There's a new set of functions that access the USGS "samples-data" services! If you are only interested in USGS discrete water quality data, you can use the `read_USGS_samples` function. -Guidance on how to use the new web page and web services are available in the [User Guide](https://www.waterqualitydata.us/beta/portal_userguide/) and [Web Services Guide](https://www.waterqualitydata.us/beta/webservices_documentation/). Additional profiles will continue to be added over time. +Read more about it in the vignette, [Introducing read_USGS_samples](https://doi-usgs.github.io/dataRetrieval/articles/samples_data.html). -**Disclaimer:** During the beta period, users may encounter bugs or identify issues with the implementation of the WQX 3.0 format: we welcome (and encourage!) your feedback to help improve these offerings, just send an email to WQX@epa.gov. - -The current WQP data profiles (available on the main Water Quality Portal web pages and from the current web services, https://www.waterqualitydata.us) deliver data in "WQX version 2.0" (what we're referring to as the "legacy") format. These will remain available for a period of time after the rollout of version 3.0. Eventually they will be retired, but there is not yet an estimated time line. +## WQP -# What to expect: dataRetrieval specific +`dataRetrieval` WQP functions continue to default to the legacy system (that does not include post-March 2024 USGS discrete sample data). To access the most recent USGS data from the new "WQX3" services in `dataRetrieval`, use the function `readWQPqw` and set the argument `legacy=FALSE` or use the function `readWQPdata` and set the argument `service = "ResultWQX3"`. New services aren't currently set as the default because the WQP team still considers these services "beta", and therefore performance is not guaranteed. Users may encounter bugs or identify issues with the implementation of the new services: we welcome (and encourage!) your feedback to help improve these offerings, just send an email to WQX@epa.gov. +The table below provides a summary of the current state of WQP functions in `dataRetrieval`. ```{r echo=FALSE} df <- data.frame(Function = c("readWQPdata", @@ -136,14 +105,12 @@ df <- data.frame(Function = c("readWQPdata", "whatWQPmetrics", "whatWQPsamples", "whatWQPdata", - "readNWISqw", "readWQPsummary", "whatNWISdata"), Status = c("Set to legacy options by default. WQX3 options available.", "Set to legacy options by default. WQX3 options available.", "Set to legacy options by default. WQX3 options available.", rep("Currently only available via legacy services.", 3), - "Retired.", "Does not have accurate information for USGS data.", "Does not have accurate information for qw data.")) @@ -151,174 +118,41 @@ knitr::kable(df) ``` -## readWQPqw - -The `readWQPqw()` function is generally advertised as a user-friendly function since it only works with a known list of sites, parameter codes or characteristic names, and start/end dates. - -As of `dataRetrieval` 2.7.17, this function will use the default WQX version 2 dataProfile, specified by the `legacy = TRUE` argument. Setting `legacy = FALSE` will return the WQX 3.0 "narrow" dataProfile. Keep in mind the 2.0 profiles will eventually be retired. For any more flexibility, users will need to use the `readWQPdata()` function. - -An example of a WQX 3.0 return: - -```{r eval=FALSE} -library(dataRetrieval) -rawPcode <- readWQPqw(siteNumbers = "USGS-01594440", - parameterCd = "01075", - legacy = FALSE) -``` - -Compared to using the WQX 2.0 legacy results: - -```{r eval=FALSE} -rawPcode_legacy <- readWQPqw(siteNumbers = "USGS-01594440", - parameterCd = "01075", - legacy = TRUE) - -``` - -## readWQPdata +### Leveraging the beta WQP services -The `readWQPdata()` function is the most flexible function to get WQP data. Currently there are 11 legacy options and 5 options that use the new WQX 3.0 profiles. Note that `readWQPdata()` does not leverage a `legacy` argument to specify which profile version the user would like returned, but instead relies on the user's specification of `service` and `dataProfile` arguments. - -### WQX 3.0 - -There are currently three WQX 3.0 "services" available: ResultWQX, StationWQX and ActivityWQX. The "ResultWQX" service has multiple available "dataProfiles". +The beta WQP offers data in the "WQX version 3.0 format" (WQX = [Water Quality Exchange](https://exchangenetwork.net/data-exchange/wqx/)) using new "data profiles" (how the data are formatted by the WQP). There are currently three WQX 3.0 "services" available on beta: ResultWQX3, StationWQX3 and ActivityWQX3. The "ResultWQX3" service has multiple available data profiles: | Service | dataProfile | | -------------- | ---------- | -| StationWQX | | -| ResultWQX | fullPhysChem | -| ResultWQX | basicPhysChem | -| ResultWQX | narrow | -| ActivityWQX | | +| StationWQX3 | | +| ResultWQX3 | fullPhysChem | +| ResultWQX3 | basicPhysChem | +| ResultWQX3 | narrow | +| ActivityWQX3 | | -Examples: +For example, if you wish to leverage a specific "ResultWQX3" data profile using the beta services, your code might look something like this, using the very flexible `readWQPdata` function: -```{r eval=FALSE} +```{r, eval = FALSE} data_full <- readWQPdata(siteid = "USGS-04024315", characteristicName = "pH", dataProfile = "fullPhysChem", service = "ResultWQX3") - -data_basic <- readWQPdata(siteid = "USGS-04024315", - characteristicName = "pH", - dataProfile = "basicPhysChem", - service = "ResultWQX3") - -data_narrow <- readWQPdata(siteid = "USGS-04024315", - characteristicName = "pH", - dataProfile = "narrow", - service = "ResultWQX3") - -data_sites <- readWQPdata(siteid = "USGS-04024315", - characteristicName = "pH", - service = "StationWQX3") - ``` -### WQX 2.0 - Legacy - -There are 8 services available from the legacy WQP. The Station and Result legacy services can still be accessed, but users should move to StationWQX, ResultWQX, and ActivityWQX. As other former services become available in WQX 3.0, we will update these documents. - -| Service | dataProfile | WQX 3.0 service "analog" | -| -------------- | ---------- | ---------- | -| Station | | StationWQX | -| Result | resultPhysChem | ResultWQX | -| Result | biological | -| Result | narrowResult | ResultWQX | -| Activity | activityAll | ActivityWQX | -| ActivityMetric | | | -| Project | | | -| ProjectMonitoringLocationWeighting | | | -| ResultDetectionQuantitationLimit | | | -| BiologicalMetric | | | - -Examples: - -```{r eval=FALSE} -# Data profiles: "Organization Data" (legacy) -org_data <- readWQPdata( - statecode = "WI", - countycode = "Dane", - service = "Organization" -) - -# Data profiles: "Project Data" (legacy) -project_data <- readWQPdata( - statecode = "WI", - countycode = "Dane", - service = "Project" -) - -# Data profiles: "Project Monitoring Location Weighting Data" (legacy) -proj_mlwd <- readWQPdata( - statecode = "WI", - countycode = "Dane", - service = "ProjectMonitoringLocationWeighting" -) - -# Data profiles: "Sample Results (physical/chemical metadata)" (legacy) -samp_data <- readWQPdata( - siteid = "USGS-04024315", - dataProfile = "resultPhysChem", - service = "Result" -) - -# Data profiles: "Sample Results (biological metadata)" (legacy) -samp_bio <- readWQPdata( - siteid = "USGS-04024315", - dataProfile = "biological", - service = "Result" -) - -# Data profiles: "Sample Results (narrow)" (legacy) -samp_narrow <- readWQPdata( - siteid = "USGS-04024315", - dataProfile = "narrowResult", - service = "Result" -) - -# Data profiles: "Sampling Activity" (legacy) -samp_activity <- readWQPdata( - siteid = "USGS-04024315", - dataProfile = "activityAll", - service = "Activity" -) - - -# Data profile: "Sampling Activity Metrics" (legacy) -act_metrics <- readWQPdata( - statecode = "WI", - countycode = "Dane", - service = "ActivityMetric" -) - -# Data profile: "Result Detection Quantitation Limit Data" (legacy) -dl_data <- readWQPdata( - siteid = "USGS-04024315", - service = "ResultDetectionQuantitationLimit" -) +On the other hand, the "StationWQX3" service requires no `dataProfile` argument: +```{r, eval = FALSE} +data_station <- readWQPdata(siteid = "USGS-04024315", + characteristicName = "pH", + service = "StationWQX3") ``` -## whatNWISdata - - -NWIS discrete water quality services are "frozen": the returned data availability will also be frozen ONLY for "qw" data_type_cd results. All other data types should not be affected. - -When the NWIS services are decommissioned (likely in 2025): there will no longer be any "qw" information provided in the output of `whatNWISdata`. Discrete water-quality availability will be available via WQP services. More information will be provided as we learn more. - -Here's an example of what will change: - -```{r eval=FALSE} -what_NWIS <- whatNWISdata(siteNumber = "05114000") -nrow(what_NWIS) -[1] 407 -nrow(what_NWIS[what_NWIS$data_type_cd == "qw",]) -[1] 381 -``` +Guidance on how to use the new web page and web services are available in the [User Guide](https://www.waterqualitydata.us/beta/portal_userguide/) and [Web Services Guide](https://www.waterqualitydata.us/beta/webservices_documentation/). Additional profiles will continue to be added over time. -So for site "05114000", there are 381 NWIS qw parameters that have been measured. Since mid-March 2024, the data availability for those 381 parameters are frozen...even if new data are collected. Eventually those 381 rows of data will not be returned, only 26 rows of data will be returned (407-381). +## NWIS "QW" Data -New services/functions are being developed to replace the lost functionality so check back here for updated information. +As of dataRetrieval v2.7.17, `readNWISqw()` has been retired and replaced by `readWQPqw()`. The `readWQPqw()` function is generally advertised as a user-friendly function since it only works with a known list of sites, parameter codes or characteristic names, and start/end dates. Learn more about the change and where to find the new samples data in the dataretrieval [Changes to NWIS QW services vignette](https://doi-usgs.github.io/dataRetrieval/articles/qwdata_changes.html) and the WDFN blog on [Improved Public Delivery of Water Quality and Field Samples](https://waterdata.usgs.gov/blog/changes-to-sample-data/). +### A note on `whatNWISdata` and "QW" data +Currently, any "QW" data summaries provided by `whatNWISdata` are very likely out of date and incorrect: do not use this function to query by `data_type_cd = "qw"`. When the NWIS services are decommissioned (likely in 2025) there will no longer be any "qw" information provided in the output of `whatNWISdata`. Discrete water-quality data will be available via WQP services and USGS Samples only. More information will be provided as we learn more. From 6eb3ca039331772d33bae31f618a2b0dbceab20f Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 4 Jun 2025 08:44:53 -0500 Subject: [PATCH 072/117] Vignette updates --- R/read_USGS_data.R | 5 ++- _pkgdown.yml | 2 + vignettes/read_USGS_functions.Rmd | 69 ++++++++++++++++++++++++++++--- 3 files changed, 69 insertions(+), 7 deletions(-) diff --git a/R/read_USGS_data.R b/R/read_USGS_data.R index 71f82220..58b2c3cc 100644 --- a/R/read_USGS_data.R +++ b/R/read_USGS_data.R @@ -71,7 +71,10 @@ read_USGS_data <- function(service, if(convertType) return_list <- cleanup_cols(return_list) - return_list <- return_list[order(return_list$time, return_list$monitoring_location_id), ] + # Add other time series services when they come online + if(service %in% c("daily")){ + return_list <- return_list[order(return_list$time, return_list$monitoring_location_id), ] + } return_list <- rejigger_cols(return_list, args[["properties"]], service) diff --git a/_pkgdown.yml b/_pkgdown.yml index dce8c165..102b4999 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -40,6 +40,8 @@ navbar: menu: - text: Tutorial href: articles/tutorial.html + - text: New USGS Functions + href: articles/read_USGS_functions.html - text: Background href: articles/dataRetrieval.html - text: Pivot Data diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 4b23dcb2..7666e69f 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -64,7 +64,11 @@ API_USGS_PAT = "my_super_secret_token" ``` You can use `usethis::edit_r_environ()` to edit find and open your .Renviron file. You will need to restart R for that variable to be recognized. -# Monitoring Location +# New Functions + +As new API endpoints come online, this section will be updated with any `dataRetrieval` function that is created. + +## Monitoring Location The `read_USGS_monitoring_location` function replaces the `readNWISsite` function. @@ -126,7 +130,7 @@ leaflet(data = sites_wi |> ``` -# Time Series Metadata +## Time Series Metadata The `read_USGS_ts_meta` function replaces the `whatNWISdata` function. @@ -153,7 +157,7 @@ knitr::kable(ts_available1) ``` -# Daily Values +## Daily Values The `read_USGS_daily` function replaces the `readNWISdv` function. @@ -169,12 +173,12 @@ library(dataRetrieval) daily_modern <- read_USGS_daily(monitoring_location_id = "USGS-01491000", parameter_code = c("00060", "00010"), statistic_id = "00003", - time = c("2023-01-01", "2025-05-06")) + time = c("2023-10-01", "2024-09-30")) nrow(daily_modern) ``` -The first thing you might notice is that the new service serves data in a "long" format, which means there is just a single observations per row of the data frame (see: [Pivot Data](articles/long_to_wide.html)). +The first thing you might notice is that the new service serves data in a "long" format, which means there is just a single observations per row of the data frame (see: [Pivot Data](articles/long_to_wide.html)). Many functions will be easier and more efficient to work with a "long" data frame. For instance, here we can see how `ggplot2` can use the parameter_code column to create a "facet" plot: ```{r} library(ggplot2) @@ -186,7 +190,60 @@ ggplot(data = daily_modern) + ``` -# Discrete Samples +## General Retrieval + +The `read_USGS_data` function replaces the `readNWISdata` function. + +The new APIs can handle complex requests. For those queries, users will need to construct their own request using Common Query Language. There's an excellent article . + +Let's try to find sites in Wisconsin and Minnesota that have a drainage area greater than 1000 mi^2. + +```{r} +cql <- '{ + "op": "and", + "args": [ + { + "op": "in", + "args": [ + { "property": "state_name" }, + [ "Wisconsin", "Minnesota" ] + ] + }, + { + "op": ">", + "args": [ + { "property": "drainage_area" }, + 1000 + ] + } + ] +}' + +sites_mn_wi <- read_USGS_data(service = "monitoring-locations", + CQL = cql) + +``` + +Let's see what that looks like: + +```{r} +leaflet_crs <- "+proj=longlat +datum=WGS84" #default leaflet crs +pal <- colorNumeric("viridis", sites_mn_wi$drainage_area) +leaflet(data = sites_mn_wi |> + sf::st_transform(crs = leaflet_crs)) |> + addProviderTiles("CartoDB.Positron") |> + addCircleMarkers(popup = ~monitoring_location_name, + color = ~ pal(drainage_area), + radius = 0.1, + opacity = 1) |> + addLegend(pal = pal, + position = "bottomleft", + title = "Drainage Area", + values = ~drainage_area) + +``` + +## Discrete Samples Discrete USGS water quality can be accessed via the `read_USGS_samples` function. While this is a new, modern USGS endpoint, it is not served in the same infrastructure as the rest of these new advertised functions. See [Samples Data](articles/samples_data.html)) for information on accessing USGS discrete water quality data. From 5ca8d67bb4a6d59a13dc22596532a60a453e14cc Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 4 Jun 2025 08:57:09 -0500 Subject: [PATCH 073/117] Fixes #754 --- vignettes/nldi.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vignettes/nldi.Rmd b/vignettes/nldi.Rmd index c2e873e1..7476756b 100644 --- a/vignettes/nldi.Rmd +++ b/vignettes/nldi.Rmd @@ -31,7 +31,7 @@ opts_chunk$set( ) ``` -This post will demonstrate `dataRetrieval` functions to query the [Network Linked Data Index](https://labs.waterdata.usgs.gov/about-nldi/index.html) (NLDI). +This post will demonstrate `dataRetrieval` functions to query the [Network Linked Data Index](https://api.water.usgs.gov/nldi/swagger-ui/index.html) (NLDI). The NLDI provides a information backbone to navigate the NHDPlusV2 network and discover features indexed to the network. This process of feature discovery mirrors web-based navigation tools like Google Maps. From 283a7994a621366f5311ec93f2e4b555aba0ecd7 Mon Sep 17 00:00:00 2001 From: Elise Hinman Date: Wed, 4 Jun 2025 09:07:22 -0500 Subject: [PATCH 074/117] small wordsmithing, rearranging, add a bit more about API key --- vignettes/Status.Rmd | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/vignettes/Status.Rmd b/vignettes/Status.Rmd index fc76787e..996a0605 100644 --- a/vignettes/Status.Rmd +++ b/vignettes/Status.Rmd @@ -30,7 +30,7 @@ Please contact the Computational Tools team at CompTools@usgs.gov with questions
The USGS is in the process of creating new, [publicly available APIs](https://api.waterdata.usgs.gov/) (a.k.a. "web services") to replace the existing [WaterServices](https://waterservices.usgs.gov/). `dataRetrieval` relies upon these web services to provide monitoring location information and water quantity/quality datasets. As new web services come online, `dataRetrieval` will be furnished with new functions to accommodate these changes. -The table below shows the **existing** NWIS functions and, if applicable, their slated replacements. Note that several functions do not have direct replacements because the new services to support them do not yet exist. We will update this table as new services and functions come online. Also note that some new functions may only be available on the "develop" branch of `dataRetrieval` (`remotes::install_github("DOI-USGS/dataRetrieval", ref = "develop")`). +The table below shows the **existing** NWIS functions and, if applicable, their slated replacements. Note that several functions do not have direct replacements because the new services to support them do not yet exist. We will update this table as new services and functions come online. Also note that some new functions may only be available on the "develop" branch of `dataRetrieval` (`remotes::install_github("DOI-USGS/dataRetrieval", ref = "develop")`). More information on each function's specifications (complete with examples) are available in the [package index](https://doi-usgs.github.io/dataRetrieval/reference/index.html) and/or function documentation (e.g. `?read_USGS_daily`) ```{r echo=FALSE} df <- data.frame( @@ -70,7 +70,7 @@ If you want to learn more about the new water data APIs, check out the ["What's ## API Keys -Do you make *a lot* of `dataRetrieval` requests per day? ...On the order of more than 50 function calls per hour? As you switch your workflows over to the new Water Data API functions, consider grabbing yourself an API key, which will bump your limit up to 1,000 requests per hour. Head to the [sign up page](https://api.waterdata.usgs.gov/signup) to get a token. +Do you make *a lot* of `dataRetrieval` WaterServices calls (e.g. using functions like `readNWISdv`, `readNWISuv`) per day? ...On the order of more than 50 function calls per hour? As you switch your workflows over to the new Water Data API functions, consider grabbing yourself an API key, which will bump your limit up to 1,000 requests per hour. Head to the [sign up page](https://api.waterdata.usgs.gov/signup) to get a token. One you have your API key, add it to your `.Renviron` file like this: @@ -78,23 +78,23 @@ One you have your API key, add it to your `.Renviron` file like this: API_USGS_PAT = "[your api key]" ``` -Replace [your api key] with the alphanumeric code provided by the sign-up page. +Replace [your api key] with the alphanumeric code provided by the sign-up page. That's it! `dataRetrieval` will look for an `.Renviron` file in your directories and use it for making web service calls. # Discrete Data -In March 2024, NWIS **discrete water quality** services were "frozen": any public data retrieval using `readNWISqw()` no longer included any new data. Concurrently, the main [Water Quality Portal (WQP) API](https://www.waterqualitydata.us/) stopped serving new and updated USGS data. Now, new and updated data are available from the [USGS Samples API](https://waterdata.usgs.gov/download-samples/#dataProfile=site) (for USGS data only) or in the [beta version](https://www.waterqualitydata.us/beta/) of the WQP (both USGS and non-USGS data). +In March 2024, NWIS **discrete water quality** services were "frozen": any public data retrieval using `readNWISqw()` no longer included any new data. Concurrently, the main [Water Quality Portal (WQP) API](https://www.waterqualitydata.us/) stopped serving new and updated USGS data (we will refer to this set of web services as "legacy"). Now, new and updated data are available from the [USGS Samples API](https://waterdata.usgs.gov/download-samples/#dataProfile=site) (for USGS data only) or in the [beta version](https://www.waterqualitydata.us/beta/) of the WQP (both USGS and non-USGS data). What does this mean for water quality data users of `dataRetrieval`? Check out the sections below for more specifics. ## Samples Data -There's a new set of functions that access the USGS "samples-data" services! If you are only interested in USGS discrete water quality data, you can use the `read_USGS_samples` function. +There's a new set of functions that access the USGS "samples-data" services! If you are **only** interested in USGS discrete water quality data, you can use the `read_USGS_samples` function. Read more about it in the vignette, [Introducing read_USGS_samples](https://doi-usgs.github.io/dataRetrieval/articles/samples_data.html). ## WQP -`dataRetrieval` WQP functions continue to default to the legacy system (that does not include post-March 2024 USGS discrete sample data). To access the most recent USGS data from the new "WQX3" services in `dataRetrieval`, use the function `readWQPqw` and set the argument `legacy=FALSE` or use the function `readWQPdata` and set the argument `service = "ResultWQX3"`. New services aren't currently set as the default because the WQP team still considers these services "beta", and therefore performance is not guaranteed. Users may encounter bugs or identify issues with the implementation of the new services: we welcome (and encourage!) your feedback to help improve these offerings, just send an email to WQX@epa.gov. +`dataRetrieval` WQP functions continue to default to the legacy system (that does not include post-March 2024 USGS discrete sample data). The new replacement services aren't currently set as the default because the WQP team still considers these services "beta", and therefore performance is not guaranteed. Users may encounter bugs or identify issues with the implementation of the new services: we welcome (and encourage!) your feedback to help improve these offerings, just send an email to WQX@epa.gov. The table below provides a summary of the current state of WQP functions in `dataRetrieval`. @@ -130,7 +130,9 @@ The beta WQP offers data in the "WQX version 3.0 format" (WQX = [Water Quality E | ResultWQX3 | narrow | | ActivityWQX3 | | -For example, if you wish to leverage a specific "ResultWQX3" data profile using the beta services, your code might look something like this, using the very flexible `readWQPdata` function: +**Quickstart:** To access the most recent USGS data from the new services in `dataRetrieval`, use the function `readWQPqw` and set the argument `legacy=FALSE` or use the function `readWQPdata` and set the argument `service = "ResultWQX3"`. + +If you wish to leverage a specific "ResultWQX3" data profile using the beta services, your code might look something like this, using the very flexible `readWQPdata` function: ```{r, eval = FALSE} data_full <- readWQPdata(siteid = "USGS-04024315", @@ -153,6 +155,6 @@ Guidance on how to use the new web page and web services are available in the [U As of dataRetrieval v2.7.17, `readNWISqw()` has been retired and replaced by `readWQPqw()`. The `readWQPqw()` function is generally advertised as a user-friendly function since it only works with a known list of sites, parameter codes or characteristic names, and start/end dates. Learn more about the change and where to find the new samples data in the dataretrieval [Changes to NWIS QW services vignette](https://doi-usgs.github.io/dataRetrieval/articles/qwdata_changes.html) and the WDFN blog on [Improved Public Delivery of Water Quality and Field Samples](https://waterdata.usgs.gov/blog/changes-to-sample-data/). -### A note on `whatNWISdata` and "QW" data +### A note on `whatNWISdata` and "qw" data -Currently, any "QW" data summaries provided by `whatNWISdata` are very likely out of date and incorrect: do not use this function to query by `data_type_cd = "qw"`. When the NWIS services are decommissioned (likely in 2025) there will no longer be any "qw" information provided in the output of `whatNWISdata`. Discrete water-quality data will be available via WQP services and USGS Samples only. More information will be provided as we learn more. +Currently, any "qw" data summaries provided by `whatNWISdata` are very likely out of date and incorrect: do not use this function to query by `data_type_cd = "qw"`. When the NWIS services are decommissioned (likely in 2025) there will no longer be any "qw" information provided in the output of `whatNWISdata`. Discrete water-quality data will be available via WQP services and USGS Samples only. More information will be provided as we learn more. From c450de606249a5522a6686f7bcf714a39e95b839 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 4 Jun 2025 09:08:22 -0500 Subject: [PATCH 075/117] take author off to encourage collaboration --- vignettes/read_USGS_functions.Rmd | 1 - 1 file changed, 1 deletion(-) diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 15659f69..56e8b078 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -1,6 +1,5 @@ --- title: "Introduction to New USGS Services" -author: Laura A. DeCicco editor_options: chunk_output_type: console output: From f3b8f89b8959426bd03821e0270241b6034ae780 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 4 Jun 2025 09:10:59 -0500 Subject: [PATCH 076/117] better link --- vignettes/nldi.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vignettes/nldi.Rmd b/vignettes/nldi.Rmd index 7476756b..1a18a8ec 100644 --- a/vignettes/nldi.Rmd +++ b/vignettes/nldi.Rmd @@ -31,7 +31,7 @@ opts_chunk$set( ) ``` -This post will demonstrate `dataRetrieval` functions to query the [Network Linked Data Index](https://api.water.usgs.gov/nldi/swagger-ui/index.html) (NLDI). +This post will demonstrate `dataRetrieval` functions to query the [Network Linked Data Index](https://waterdata.usgs.gov/blog/nldi-intro/) (NLDI). The NLDI provides a information backbone to navigate the NHDPlusV2 network and discover features indexed to the network. This process of feature discovery mirrors web-based navigation tools like Google Maps. From 4c2870366e3729dbd40f473e1b9d9aaf929edcca Mon Sep 17 00:00:00 2001 From: Elise Hinman Date: Wed, 4 Jun 2025 09:14:41 -0500 Subject: [PATCH 077/117] correct one more QW, add reference to USGS services vignette - no link yet --- vignettes/Status.Rmd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vignettes/Status.Rmd b/vignettes/Status.Rmd index 996a0605..f18b27bc 100644 --- a/vignettes/Status.Rmd +++ b/vignettes/Status.Rmd @@ -30,7 +30,7 @@ Please contact the Computational Tools team at CompTools@usgs.gov with questions
The USGS is in the process of creating new, [publicly available APIs](https://api.waterdata.usgs.gov/) (a.k.a. "web services") to replace the existing [WaterServices](https://waterservices.usgs.gov/). `dataRetrieval` relies upon these web services to provide monitoring location information and water quantity/quality datasets. As new web services come online, `dataRetrieval` will be furnished with new functions to accommodate these changes. -The table below shows the **existing** NWIS functions and, if applicable, their slated replacements. Note that several functions do not have direct replacements because the new services to support them do not yet exist. We will update this table as new services and functions come online. Also note that some new functions may only be available on the "develop" branch of `dataRetrieval` (`remotes::install_github("DOI-USGS/dataRetrieval", ref = "develop")`). More information on each function's specifications (complete with examples) are available in the [package index](https://doi-usgs.github.io/dataRetrieval/reference/index.html) and/or function documentation (e.g. `?read_USGS_daily`) +The table below shows the **existing** NWIS functions and, if applicable, their slated replacements. Note that several functions do not have direct replacements because the new services to support them do not yet exist. We will update this table as new services and functions come online. Also note that some new functions may only be available on the "develop" branch of `dataRetrieval` (`remotes::install_github("DOI-USGS/dataRetrieval", ref = "develop")`). More information on each function's specifications (complete with examples) are available in the [Introduction to New USGS Services vignette](add link here), the [package index](https://doi-usgs.github.io/dataRetrieval/reference/index.html) and/or function documentation (e.g. `?read_USGS_daily`). ```{r echo=FALSE} df <- data.frame( @@ -151,7 +151,7 @@ data_station <- readWQPdata(siteid = "USGS-04024315", Guidance on how to use the new web page and web services are available in the [User Guide](https://www.waterqualitydata.us/beta/portal_userguide/) and [Web Services Guide](https://www.waterqualitydata.us/beta/webservices_documentation/). Additional profiles will continue to be added over time. -## NWIS "QW" Data +## NWIS "qw" Data As of dataRetrieval v2.7.17, `readNWISqw()` has been retired and replaced by `readWQPqw()`. The `readWQPqw()` function is generally advertised as a user-friendly function since it only works with a known list of sites, parameter codes or characteristic names, and start/end dates. Learn more about the change and where to find the new samples data in the dataretrieval [Changes to NWIS QW services vignette](https://doi-usgs.github.io/dataRetrieval/articles/qwdata_changes.html) and the WDFN blog on [Improved Public Delivery of Water Quality and Field Samples](https://waterdata.usgs.gov/blog/changes-to-sample-data/). From 77b5c230d0fa0a50375959ac186f666f9dd75b31 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 4 Jun 2025 09:37:51 -0500 Subject: [PATCH 078/117] Add help section --- vignettes/read_USGS_functions.Rmd | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 0fdc128c..1664c463 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -280,3 +280,21 @@ install_github("DOI-USGS/dataRetrieval", ``` +# HELP! + +That's a lot of new information and changes. There are certainly going to be scripts that have been passed down through the years that will start breaking once the NWIS servers are decommissioned. + +Check back on the documentation often: + +Peruse the "Additional Articles", when we find common issues people have with converting their old workflows, we will try to add articles to clarrify new workflows. + +Currently, you might be interested in: + +* [General Tutorial](articles/Tutorial.html) + +* [Pivot Help](articles/long_to_wide.html) + +* [Joining by closest date](articles/join_by_closest.html) + +If you have additional questions, email comptools@usgs.gov. General questions and bug reports can be reported here: + From aaffa73265185a61dd708d7930bfb1cd060c4204 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 4 Jun 2025 10:23:28 -0500 Subject: [PATCH 079/117] Update vignettes/read_USGS_functions.Rmd Co-authored-by: Cee Nell --- vignettes/read_USGS_functions.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 1664c463..cd946cde 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -32,7 +32,7 @@ knitr::opts_chunk$set( ``` -As we bid adieu to the NWIS web services, we welcome a host of new web service offering: the [USGS Water Data OGC APIs](https://api.waterdata.usgs.gov/ogcapi/v0/). This is a modern access point for USGS water data. The USGS is planning to modernize **all** web services in the near future. For each of these updates, `dataRetrieval` will create a new function to access the new services. +As we bid adieu to the NWIS web services, we welcome a host of new web service offering: the [USGS Water Data OGC APIs](https://api.waterdata.usgs.gov/ogcapi/v0/). This is a modern access point for USGS water data. The USGS is [planning to modernize **all** web services](https://waterdata.usgs.gov/blog/api-whats-new-wdfn-apis/) in the near future. For each of these updates, `dataRetrieval` will create a new function to access the new services. This document will introduce each new function (note as time goes on, we'll update this document to include additional functions). Scroll down for tips on what you'll need to do to convert your legacy scripts to use these modern functions. Note the timeline for the NWIS servers being shut down is currently very uncertain. We'd recommend incorporating these new functions as soon as possible to avoid future headaches. From 2b0ea51046192a5e893351b82b4bc8150382de4a Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 4 Jun 2025 10:23:45 -0500 Subject: [PATCH 080/117] Update vignettes/read_USGS_functions.Rmd Co-authored-by: Cee Nell --- vignettes/read_USGS_functions.Rmd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index cd946cde..30430b36 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -119,7 +119,9 @@ ggplot(data = sites_wi) + ```{r} library(leaflet) + leaflet_crs <- "+proj=longlat +datum=WGS84" #default leaflet crs + leaflet(data = sites_wi |> sf::st_transform(crs = leaflet_crs)) |> addProviderTiles("CartoDB.Positron") |> From 69b5d047dd600caa9582c1da1385ab5fb1707186 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 4 Jun 2025 10:23:56 -0500 Subject: [PATCH 081/117] Update vignettes/read_USGS_functions.Rmd Co-authored-by: Cee Nell --- vignettes/read_USGS_functions.Rmd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 30430b36..836357ed 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -229,7 +229,9 @@ Let's see what that looks like: ```{r} leaflet_crs <- "+proj=longlat +datum=WGS84" #default leaflet crs + pal <- colorNumeric("viridis", sites_mn_wi$drainage_area) + leaflet(data = sites_mn_wi |> sf::st_transform(crs = leaflet_crs)) |> addProviderTiles("CartoDB.Positron") |> From 095c5d50ac884305e6b939d04e661e1b60fe028b Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 4 Jun 2025 10:34:02 -0500 Subject: [PATCH 082/117] Update vignettes/read_USGS_functions.Rmd Co-authored-by: Joe Zemmels (he/him) --- vignettes/read_USGS_functions.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 836357ed..92ca4e72 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -42,7 +42,7 @@ Each new "API endpoint" will deliver a new type of USGS data. Currently the avai * Any column that is returned can also be queried on! The main `dataRetrieval` functions will take vector inputs. Each query is appended as a boolean AND. For instance, if you ask for a vector of monitoring location ids and a vector of parameter codes, you will get data back for just those monitoring locations AND the specified parameter codes. Towards the bottom of this document, we'll discuss ways to make more specific queries using "Common Query Language" or CQL. -* Data is returned with a "geometry" column, which is a simple feature object, allowing the data to be integrated with the `sf` package and associated geospatial workflows. +* **Provides [Simple Features](https://en.wikipedia.org/wiki/Simple_Features) functionality**: data is returned with a "geometry" column, which is a simple feature object, allowing the data to be integrated with the [`sf`](https://r-spatial.github.io/sf/) package and associated geospatial workflows. * When you look at the help file for the new functions, you’ll also notice there are many more arguments. These are mostly set by default to NA. You **DO NOT** need to (and most likely should not!) specify all of these parameters. From da75aec43f6fdeb91740f8659ce73f5b33fb04c9 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 4 Jun 2025 10:34:31 -0500 Subject: [PATCH 083/117] Update vignettes/read_USGS_functions.Rmd Co-authored-by: Joe Zemmels (he/him) --- vignettes/read_USGS_functions.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 92ca4e72..36c0f754 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -40,7 +40,7 @@ This document will introduce each new function (note as time goes on, we'll upda Each new "API endpoint" will deliver a new type of USGS data. Currently the available endpoints are "monitoring-locations", "time-series-metadata", and "daily". All of these new endpoints offer some new features that the NWIS services did not have: -* Any column that is returned can also be queried on! The main `dataRetrieval` functions will take vector inputs. Each query is appended as a boolean AND. For instance, if you ask for a vector of monitoring location ids and a vector of parameter codes, you will get data back for just those monitoring locations AND the specified parameter codes. Towards the bottom of this document, we'll discuss ways to make more specific queries using "Common Query Language" or CQL. +* **Supports [Contextual Query Language](https://www.loc.gov/standards/sru/cql/) (CQL) syntax**: any column can also be queried on! The main `dataRetrieval::read_USGS_*` functions will take vector inputs. Each query is appended as a boolean AND. For instance, if you ask for a vector of monitoring location ids and a vector of parameter codes, you will get data back for just those monitoring locations AND the specified parameter codes. Towards the bottom of this document, we'll discuss ways to make more specific queries using Contextual Query Language. * **Provides [Simple Features](https://en.wikipedia.org/wiki/Simple_Features) functionality**: data is returned with a "geometry" column, which is a simple feature object, allowing the data to be integrated with the [`sf`](https://r-spatial.github.io/sf/) package and associated geospatial workflows. From f478f2ffd25cb1464d1de3246eaeaea42c2691e9 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 4 Jun 2025 10:34:53 -0500 Subject: [PATCH 084/117] Update vignettes/read_USGS_functions.Rmd Co-authored-by: Joe Zemmels (he/him) --- vignettes/read_USGS_functions.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 36c0f754..63cd2b2a 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -44,7 +44,7 @@ Each new "API endpoint" will deliver a new type of USGS data. Currently the avai * **Provides [Simple Features](https://en.wikipedia.org/wiki/Simple_Features) functionality**: data is returned with a "geometry" column, which is a simple feature object, allowing the data to be integrated with the [`sf`](https://r-spatial.github.io/sf/) package and associated geospatial workflows. -* When you look at the help file for the new functions, you’ll also notice there are many more arguments. These are mostly set by default to NA. You **DO NOT** need to (and most likely should not!) specify all of these parameters. +* **Enables more flexible queries**: when you look at the help file for the new functions, you’ll also notice there are many more arguments. These are mostly set by default to NA. You **DO NOT** need to (and most likely should not!) specify all of these parameters. * Users can pick which columns are returned with the "properties" argument. Available properties can be discovered with the `check_OGC_requests` function: From 7bb6683bb1dca3aa12aa187a8cd12b8ca30acd08 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 4 Jun 2025 10:35:34 -0500 Subject: [PATCH 085/117] Update vignettes/read_USGS_functions.Rmd Co-authored-by: Joe Zemmels (he/him) --- vignettes/read_USGS_functions.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 63cd2b2a..c1411f3c 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -193,7 +193,7 @@ ggplot(data = daily_modern) + ## General Retrieval -The `read_USGS_data` function replaces the `readNWISdata` function. +The `read_USGS_data` function replaces the `readNWISdata` function. This is a lower-level, generalized function for querying any of the API endpoints. The new APIs can handle complex requests. For those queries, users will need to construct their own request using Common Query Language. There's an excellent article . From 8943ec375aeecd44f436e49e0dc3a244f60b44da Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 4 Jun 2025 13:41:01 -0500 Subject: [PATCH 086/117] Update README.Rmd Co-authored-by: Elise Hinman <121896266+ehinman@users.noreply.github.com> --- README.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.Rmd b/README.Rmd index 5e6af2d7..df5e58e1 100644 --- a/README.Rmd +++ b/README.Rmd @@ -47,7 +47,7 @@ If you have additional questions about these changes, email CompTools@usgs.gov. 5. Get USGS discrete water quality data. Start here: `?read_USGS_samples` -6. Discover USGS time series data. Start here: `?read_USGS_ts_meta` +6. Discover USGS time series metadata. Start here: `?read_USGS_ts_meta` 7. Find Hydro Network-Linked Data Index (NLDI) data. Start here: `?findNLDI` From 4c090c08fb48bfc193eb6ddfd49c4dda189320de Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 4 Jun 2025 13:41:25 -0500 Subject: [PATCH 087/117] Update vignettes/read_USGS_functions.Rmd Co-authored-by: Elise Hinman <121896266+ehinman@users.noreply.github.com> --- vignettes/read_USGS_functions.Rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index c1411f3c..ba3332de 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -38,7 +38,7 @@ This document will introduce each new function (note as time goes on, we'll upda # New Features -Each new "API endpoint" will deliver a new type of USGS data. Currently the available endpoints are "monitoring-locations", "time-series-metadata", and "daily". All of these new endpoints offer some new features that the NWIS services did not have: +Each new "API endpoint" will deliver a new type of USGS data. Currently the available endpoints are "monitoring-locations", "time-series-metadata", and "daily". All of these endpoints offer some new features that the NWIS services did not have: * **Supports [Contextual Query Language](https://www.loc.gov/standards/sru/cql/) (CQL) syntax**: any column can also be queried on! The main `dataRetrieval::read_USGS_*` functions will take vector inputs. Each query is appended as a boolean AND. For instance, if you ask for a vector of monitoring location ids and a vector of parameter codes, you will get data back for just those monitoring locations AND the specified parameter codes. Towards the bottom of this document, we'll discuss ways to make more specific queries using Contextual Query Language. From 35c918e392e17f508203f8ace0fe5672c2775bfc Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 4 Jun 2025 13:42:04 -0500 Subject: [PATCH 088/117] Update vignettes/read_USGS_functions.Rmd Co-authored-by: Cee Nell --- vignettes/read_USGS_functions.Rmd | 1 + 1 file changed, 1 insertion(+) diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index ba3332de..f6fca0f4 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -183,6 +183,7 @@ The first thing you might notice is that the new service serves data in a "long" ```{r} library(ggplot2) + ggplot(data = daily_modern) + geom_point(aes(x = time, y = value, color = approval_status)) + From 942fc3b6da8974a856d58bb6fe7ba002d427d014 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 4 Jun 2025 15:27:35 -0500 Subject: [PATCH 089/117] First pass updates to tutorial --- vignettes/tutorial.Rmd | 269 ++++++++++++++++------------------------- 1 file changed, 102 insertions(+), 167 deletions(-) diff --git a/vignettes/tutorial.Rmd b/vignettes/tutorial.Rmd index 34c15676..414e9c34 100644 --- a/vignettes/tutorial.Rmd +++ b/vignettes/tutorial.Rmd @@ -5,10 +5,12 @@ date: "`r format(Sys.time(), '%B %d, %Y')`" output: rmarkdown::html_vignette vignette: > - %\VignetteEngine{knitr::rmarkdown} %\VignetteIndexEntry{dataRetrieval Tutorial} %\VignetteDepends{dplyr} \usepackage[utf8]{inputenc} + %\VignetteEngine{knitr::rmarkdown} +editor_options: + chunk_output_type: console --- ```{r setup, include=FALSE} @@ -47,7 +49,7 @@ vignette("dataRetrieval", package = "dataRetrieval") Additionally, each function has a help file. These can be accessed by typing a question mark, followed by the function name in the R console: ```{r echo=TRUE, eval=FALSE} -?readNWISuv +?read_USGS_daily ``` Each function's help file has working examples to demonstrate the usage. The examples may have comments "## Not run". These examples CAN be run, they just are not run by the CRAN maintainers due to the external service calls. @@ -89,11 +91,14 @@ There are many types of data served from NWIS. To understand how the services ar * NWIS has traditionally been the source for all USGS water data -* NWIS will be retired (scheduled late 2026): +* NWIS will be retired (scheduled 2026, but uncertain): * * USGS functions will slowly replace NWIS functions - * `read_USGS_samples` is the first replacement + * `read_USGS_samples` has replaced `readNWISqw` + * `read_USGS_daily` can replace `readNWISdv` + * `read_USGS_monitoring_location` can replace `readNWISsite` + * `read_USGS_ts_meta` can replace `whatNWISdata` * Discrete water quality data: * WQP functions should be used when accessing non-USGS discrete water quality data @@ -102,19 +107,19 @@ There are many types of data served from NWIS. To understand how the services ar # NWIS Data: Current NWIS offerings -| data_type_cd |Function| Data description | +| data_type_cd |Function| Data description | Replacement Function | |--------|:-------|------:|-------:| -|uv|[readNWISuv](https://doi-usgs.github.io/dataRetrieval/reference/readNWISuv.html)|Continuous data| -|dv|[readNWISdv](https://doi-usgs.github.io/dataRetrieval/reference/readNWISdv.html)|Daily aggregated | -|gwlevels|[readNWISgwl](https://doi-usgs.github.io/dataRetrieval/reference/readNWISgwl.html)|Groundwater levels | -|site|[readNWISsite](https://doi-usgs.github.io/dataRetrieval/reference/readNWISsite.html)|Site metadata| -|pcode|[readNWISpCode](https://doi-usgs.github.io/dataRetrieval/reference/readNWISpCode.html)|Parameter code metadata | -|stat|[readNWISstat](https://doi-usgs.github.io/dataRetrieval/reference/readNWISstat.html)| Site statistics | -|rating|[readNWISrating](https://doi-usgs.github.io/dataRetrieval/reference/readNWISrating.html)| Rating curves| -|peak|[readNWISpeak](https://doi-usgs.github.io/dataRetrieval/reference/readNWISpeak.html)|Peak flow| -|use|[readNWISuse](https://doi-usgs.github.io/dataRetrieval/reference/readNWISuse.html)|Water Use| -|meas|[readNWISmeas](https://doi-usgs.github.io/dataRetrieval/reference/readNWISmeas.html)|Discrete surface water| -| | [readNWISdata](https://doi-usgs.github.io/dataRetrieval/reference/readNWISdata.html) | General data import for NWIS| +|uv|[readNWISuv](https://doi-usgs.github.io/dataRetrieval/reference/readNWISuv.html)|Continuous data| None yet | +|dv|[readNWISdv](https://doi-usgs.github.io/dataRetrieval/reference/readNWISdv.html)|Daily aggregated | [read_USGS_daily](https://doi-usgs.github.io/dataRetrieval/reference/read_USGS_daily.html) | +|gwlevels|[readNWISgwl](https://doi-usgs.github.io/dataRetrieval/reference/readNWISgwl.html)|Groundwater levels | None yet | +|site|[readNWISsite](https://doi-usgs.github.io/dataRetrieval/reference/readNWISsite.html)|Site metadata| [read_USGS_monitoring_location](https://doi-usgs.github.io/dataRetrieval/reference/read_USGS_monitoring_location.html) | +|pcode|[readNWISpCode](https://doi-usgs.github.io/dataRetrieval/reference/readNWISpCode.html)|Parameter code metadata | None yet | +|stat|[readNWISstat](https://doi-usgs.github.io/dataRetrieval/reference/readNWISstat.html)| Site statistics | None yet | +|rating|[readNWISrating](https://doi-usgs.github.io/dataRetrieval/reference/readNWISrating.html)| Rating curves| None yet | +|peak|[readNWISpeak](https://doi-usgs.github.io/dataRetrieval/reference/readNWISpeak.html)|Peak flow| None yet | +|use|[readNWISuse](https://doi-usgs.github.io/dataRetrieval/reference/readNWISuse.html)|Water Use| None yet | +|meas|[readNWISmeas](https://doi-usgs.github.io/dataRetrieval/reference/readNWISmeas.html)|Discrete surface water| None yet | +| | [readNWISdata](https://doi-usgs.github.io/dataRetrieval/reference/readNWISdata.html) | General data import for NWIS| [read_USGS_data](https://doi-usgs.github.io/dataRetrieval/reference/read_USGS_data.html) | ## USGS Basic Retrievals @@ -142,20 +147,14 @@ df <- data.frame( names(df) <- c("Parameter Codes", "Short Name") -knitr::kable(df) -``` - - - -```{r echo=FALSE, eval=TRUE} -df <- data.frame( +df2 <- data.frame( pCode = c("00001", "00002", "00003", "00008"), shName = c("Maximum", "Minimum", "Mean", "Median") ) -names(df) <- c("Statistic Codes", "Short Name") +names(df2) <- c("Statistic Codes", "Short Name") -knitr::kable(df) +knitr::kable(list(df, df2)) ``` @@ -197,17 +196,14 @@ You can use the "user-friendly" functions. These functions take the same 4 input Let's start by asking for discharge (parameter code = 00060) at a site right next to the old USGS office in Wisconsin (Pheasant Branch Creek). ```{r echo=TRUE, eval=TRUE} -siteNo <- "05427948" +siteNo <- "USGS-05427948" pCode <- "00060" start.date <- "2023-10-01" end.date <- "2024-09-30" -pheasant <- readNWISdv( - siteNumbers = siteNo, - parameterCd = pCode, - startDate = start.date, - endDate = end.date -) +pheasant <- read_USGS_daily(monitoring_location_id = siteNo, + parameter_code = pCode, + time = c(start.date, end.date)) ``` From the Pheasant Creek example, let's look at the data. The column names are: @@ -217,53 +213,28 @@ names(pheasant) ``` -The names of the columns are based on the parameter and statistic codes. In many cases, you can clean up the names with the convenience function `renameNWISColumns`: - -```{r echo=TRUE, eval=TRUE} -pheasant <- renameNWISColumns(pheasant) -names(pheasant) -``` - -The returned data also has several attributes attached to the data frame. To see what the attributes are: - -```{r echo=TRUE, eval=TRUE} -names(attributes(pheasant)) -``` - -Each `dataRetrieval` return should have the attributes: url, siteInfo, and variableInfo. Additional attributes are available depending on the data service. - -To access the attributes: - -```{r echo=TRUE, eval=TRUE} -url <- attr(pheasant, "url") -url -``` - -[Raw Data](`r url`) - -Make a simple plot to see the data: +Let's make a simple plot to see the data: ```{r echo=TRUE, eval=TRUE, fig.height=3.5} library(ggplot2) ts <- ggplot( data = pheasant, - aes(Date, Flow) -) + + aes(time, value)) + geom_line() ts ``` -Then use the attributes attached to the data frame to create better labels: +Then we can use the `readNWISpCode` and `read_USGS_monitoring_location` functions to create better labels: ```{r echo=TRUE, eval=TRUE, fig.height=3.5} -parameterInfo <- attr(pheasant, "variableInfo") -siteInfo <- attr(pheasant, "siteInfo") +parameterInfo <- readNWISpCode(pCode) +siteInfo <- read_USGS_monitoring_location(siteNo) ts <- ts + xlab("") + - ylab(parameterInfo$variableDescription) + - ggtitle(siteInfo$station_nm) + ylab(parameterInfo$parameter_nm) + + ggtitle(siteInfo$monitoring_location_name) ts ``` @@ -273,25 +244,26 @@ The most common question the dataRetrieval team gets is: "I KNOW this site has data but it's not coming out of dataRetrieval! Where's my data?" -The best way to verify you are calling your data correctly, use the `whatNWISdata` function to find out the data_type_cd (which will tell you the service you need to call), the parameter/stat codes available at that site, and the period of record. All rows that have "qw" in the column data_type_cd will come from the Water Quality Portal. +First verify that the data you think is available is actually associated with the location. For time series data, use the `read_NWIS_ts_meta` function to find out the available time series data. ```{r echo=TRUE} library(dplyr) -site <- "05407000" -data_available <- whatNWISdata(siteNumber = site) +site <- "USGS-05407000" +ts_data_available <- read_USGS_ts_meta(monitoring_location_id = site) + -data_available_NWIS <- data_available |> - select(data_type_cd, parm_cd, stat_cd, - begin_date, end_date, count_nu) |> - filter(!data_type_cd %in% c("qw", "ad")) |> - arrange(data_type_cd) +data_available <- ts_data_available |> + sf::st_drop_geometry() |> + mutate(begin = as.Date(begin), + end = as.Date(end)) |> + select(parameter_name, parameter_code, statistic_id, computation_identifier, + begin, end) ``` -This is the only available data from NWIS for site `r site`. ```{r echo=FALSE} -datatable(data_available_NWIS, +datatable(data_available, rownames = FALSE, options = list(pageLength = 7, lengthChange = FALSE, @@ -300,42 +272,61 @@ datatable(data_available_NWIS, ``` -The data_type_cd can be used to figure out where to request data: - -```{r echo=FALSE} -df <- data.frame(data_type_cd = c("dv", "uv", "pk", "sv", "gwl"), - readNWIS = c("readNWISdv", "readNWISuv", - "readNWISpeak", "readNWISmeas", "readNWISgwl"), - readNWISdata = c('readNWISdata(..., service = "dv")', - 'readNWISdata(..., service = "iv")', - 'readNWISdata(..., service = "peak")', - 'Not available', - 'readNWISdata(..., service = "gwlevels")')) -knitr::kable(df) -``` -So to get all the NWIS data from the above site: +The time series that have "Instantaneous" in the computation_identifier column will be available in the instantaneous data service (currently `readNWISuv`), and the rest of the data will be available in the daily service (`read_USGS_daily`). ```{r eval=FALSE, echo=TRUE} -dv_pcodes <- data_available_NWIS$parm_cd[data_available_NWIS$data_type_cd == "dv"] -stat_cds <- data_available_NWIS$stat_cd[data_available_NWIS$data_type_cd == "dv"] +dv_pcodes <- data_available$parameter_code[data_available$computation_identifier != "Instantaneous"] +stat_cds <- data_available$statistic_id[data_available$computation_identifier != "Instantaneous"] -dv_data <- readNWISdv(siteNumbers = site, - parameterCd = unique(dv_pcodes), - statCd = unique(stat_cds)) +dv_data <- read_USGS_daily(monitoring_location_id = site, + parameter_code = unique(dv_pcodes), + statistic_id = unique(stat_cds)) -uv_pcodes <- data_available_NWIS$parm_cd[data_available_NWIS$data_type_cd == "uv"] +uv_pcodes <- data_available$parameter_code[data_available$computation_identifier == "Instantaneous"] -uv_data <- readNWISuv(siteNumbers = site, +uv_data <- readNWISuv(siteNumbers = gsub("USGS-", "", site), parameterCd = unique(uv_pcodes)) -peak_data <- readNWISpeak(site) +peak_data <- readNWISpeak(gsub("USGS-", "", site)) + +``` + +For discrete water quality data, use the `summarize_USGS_samples` function: + +```{r echo=TRUE} +discrete_data_available_all <- summarize_USGS_samples(site) + +discrete_data_available <- discrete_data_available_all |> + select(parameter_name = characteristicUserSupplied, + begin = firstActivity, end = mostRecentActivity, + count = resultCount) + +``` + +```{r echo=FALSE} +datatable(discrete_data_available, + rownames = FALSE, + options = list(pageLength = 7, + lengthChange = FALSE, + searching = FALSE) +) + +``` + +The discrete water quality data can be accessed with the `read_USGS_samples` function: +```{r eval=FALSE, echo=TRUE} +samples_data <- read_USGS_samples(monitoringLocationIdentifier = site, + dataProfile = "basicphyschem") ``` + # Water Quality Portal (WQP) `dataRetrieval` also allows users to access data from the [Water Quality Portal](http://www.waterqualitydata.us/). The WQP houses data from multiple agencies; while USGS data comes from the NWIS database, EPA data comes from the STORET database (this includes many state, tribal, NGO, and academic groups). The WQP brings data from all these organizations together and provides it in a single format that has a more verbose output than NWIS. +This tutorial will use the modern WQX3 format. This is still considered "beta", but it is the best way to get up-to-date multi-agency data. + The single user-friendly function is `readWQPqw`. This function will take a site or vector of sites in the first argument "siteNumbers". USGS sites need to add "USGS-" before the site number. The 2nd argument "parameterCd". Although it is called "parameterCd", it can take EITHER a USGS 5-digit parameter code OR a characterisitc name (this is what non-USGS databases use). Leaving "parameterCd" as empty quotes will return all data for a site. @@ -343,22 +334,25 @@ The 2nd argument "parameterCd". Although it is called "parameterCd", it can take So we could get all the water quality data for site `r site` like this: ```{r eval=FALSE, echo=TRUE} -qw_data_all <- readWQPqw(siteNumbers = paste0("USGS-", site), - parameterCd = "") +qw_data_all <- readWQPqw(siteNumbers = site, + parameterCd = "", + legacy = FALSE) ``` or 1 parameter code: ```{r eval=FALSE, echo=TRUE} -qw_data_00095 <- readWQPqw(siteNumbers = paste0("USGS-", site), - parameterCd = "00095") +qw_data_00095 <- readWQPqw(siteNumbers = site, + parameterCd = "00095", + legacy = FALSE) ``` or 1 characteristic name: ```{r eval=FALSE, echo=TRUE} -qw_data_sp <- readWQPqw(siteNumbers = paste0("USGS-", site), - parameterCd = "Specific conductance") +qw_data_sp <- readWQPqw(siteNumbers = site, + parameterCd = "Specific conductance", + legacy = FALSE) ``` @@ -366,22 +360,16 @@ qw_data_sp <- readWQPqw(siteNumbers = paste0("USGS-", site), This is all great when you know your site numbers. What do you do when you don't? -There are 2 `dataRetrieval` functions that help with discover in NWIS: +There are 2 `dataRetrieval` functions that help with USGS data discovery: -* `whatNWISsites` finds sites within a specified filter (quicker) -* `whatNWISdata` summarizes the data within the specified filter (more information) +* `read_USGS_monitoring_location` finds sites within a specified filter +* `read_USGS_ts_meta` summarizes the time series meta data And 2 functions that help with discover in WQP: * `readWQPsummary` summarizes the data available within the WQP by year. * `whatWQPdata` summarizes the data available within the WQP. -There are several ways to specify the requests. The best way to discover how flexible the USGS web services are is to click on the links and see all of the filtering options: -[http://waterservices.usgs.gov/](http://waterservices.usgs.gov/) - -```{r echo=FALSE, out.width = "500px"} -knitr::include_graphics("waterservices.png") -``` Available geographic filters are individual site(s), a single state, a bounding box, or a HUC (hydrologic unit code). See examples for those services by looking at the help page for the `readNWISdata` and `readWQPdata` functions: @@ -389,11 +377,9 @@ Here are a few examples: ```{r eval=FALSE} # Daily temperature in Ohio -dataTemp <- readNWISdata( - stateCd = "OH", - parameterCd = "00010", - service = "dv" -) +ohio_sites <- read_USGS_monitoring_location(state_name = "Ohio") +ohio_ts_meta <- read_USGS_ts_meta(bbox = sf::st_bbox(ohio_sites), + parameter_code = "00010") # Real-time discharge at a site instFlow <- readNWISdata( @@ -405,12 +391,6 @@ instFlow <- readNWISdata( tz = "America/Chicago" ) -# Temperature within a bounding box: -bBoxEx <- readNWISdata( - bBox = c(-83, 36.5, -81, 38.5), - parameterCd = "00010" -) - # Groundwater levels within a HUC: groundwaterHUC <- readNWISdata( huc = "02070010", @@ -418,51 +398,6 @@ groundwaterHUC <- readNWISdata( ) ``` - -### Arizona Example - -For example, let's see which sites ever measured phosphorus at least 100 times over at least 20 years in Arizona. Water quality data is exclusively found in WQP functions. - -```{r az, echo=TRUE} -AZ_sites <- readWQPsummary( - statecode = "AZ", - siteType = "Stream" -) - -az_phos_summary <- AZ_sites |> - mutate(ResultCount = as.numeric(ResultCount), - Lat = as.numeric(MonitoringLocationLatitude), - Lon = as.numeric(MonitoringLocationLongitude)) |> - rename(Site = MonitoringLocationIdentifier) |> - group_by(Site, Lat, Lon) |> - summarise(min_year = min(YearSummarized), - max_year = max(YearSummarized), - count = sum(ResultCount)) |> - mutate(POR = max_year - min_year) |> - filter(count > 100, - POR >= 20) |> - arrange(desc(count)) |> - ungroup() - -``` - - -```{r echo=TRUE, eval=TRUE, fig.height=4} -library(leaflet) - -leaflet(data = az_phos_summary) %>% - addProviderTiles("CartoDB.Positron") %>% - addCircleMarkers(~Lon, ~Lat, - color = "red", radius = 3, stroke = FALSE, - fillOpacity = 0.8, opacity = 0.8, - popup = ~Site - ) -``` - - - - - # Time/Time zone discussion * The arguments for all `dataRetrieval` functions concerning dates (startDate, endDate) can be R Date objects, or character strings, as long as the string is in the form "YYYY-MM-DD". From 69c7d1b28ee45077ddcd2afd47510dc365ddfbf5 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 4 Jun 2025 15:31:04 -0500 Subject: [PATCH 090/117] some updates from reviewers --- vignettes/read_USGS_functions.Rmd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 1664c463..c83ecae1 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -32,7 +32,7 @@ knitr::opts_chunk$set( ``` -As we bid adieu to the NWIS web services, we welcome a host of new web service offering: the [USGS Water Data OGC APIs](https://api.waterdata.usgs.gov/ogcapi/v0/). This is a modern access point for USGS water data. The USGS is planning to modernize **all** web services in the near future. For each of these updates, `dataRetrieval` will create a new function to access the new services. +As we bid adieu to the NWIS web services, we welcome a host of new web service offering: the [USGS Water Data OGC APIs](https://api.waterdata.usgs.gov/ogcapi/v0/). This is a modern access point for USGS water data. The USGS is planning to modernize **all** web services in the near future. For each of these updates, `dataRetrieval` will create a new function to access the new services and deprecate functions for accessing legacy services. This document will introduce each new function (note as time goes on, we'll update this document to include additional functions). Scroll down for tips on what you'll need to do to convert your legacy scripts to use these modern functions. Note the timeline for the NWIS servers being shut down is currently very uncertain. We'd recommend incorporating these new functions as soon as possible to avoid future headaches. @@ -252,7 +252,7 @@ Discrete USGS water quality can be accessed via the `read_USGS_samples` function ### Style -New functions will use a "snake case", such as "read_USGS_samples". Older functions use camel case, such as "readNWISdv". The difference is the underscore between words. This should be a handy way to tell the difference between newer modern data access, and the older traditional functions. +New functions will use a "snake case", such as `read_USGS_samples`. Older functions use camel case, such as `readNWISdv`. The difference is the underscore between words. This should be a handy way to tell the difference between newer modern data access, and the older traditional functions. ### Structure From 4972ee0a37be4be556b0f7f98d140c35744608a9 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 4 Jun 2025 16:09:03 -0500 Subject: [PATCH 091/117] vignette updates --- README.Rmd | 2 +- README.md | 11 ++++---- vignettes/read_USGS_functions.Rmd | 45 +++++++++++++++++++------------ 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/README.Rmd b/README.Rmd index df5e58e1..ae80bcc8 100644 --- a/README.Rmd +++ b/README.Rmd @@ -47,7 +47,7 @@ If you have additional questions about these changes, email CompTools@usgs.gov. 5. Get USGS discrete water quality data. Start here: `?read_USGS_samples` -6. Discover USGS time series metadata. Start here: `?read_USGS_ts_meta` +6. Get metadata about USGS time series data, including instantaneous and daily data. Start here: `?read_USGS_ts_meta` 7. Find Hydro Network-Linked Data Index (NLDI) data. Start here: `?findNLDI` diff --git a/README.md b/README.md index e00bd3bf..a9a35dda 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,8 @@ If you have additional questions about these changes, email 5. Get USGS discrete water quality data. Start here: `?read_USGS_samples` -6. Discover USGS time series data. Start here: `?read_USGS_ts_meta` +6. Get metadata about USGS time series data, including instantaneous + and daily data. Start here: `?read_USGS_ts_meta` 7. Find Hydro Network-Linked Data Index (NLDI) data. Start here: `?findNLDI` @@ -192,14 +193,14 @@ NWIScitation #> U.S. Geological Survey (2025). _National Water Information System data #> available on the World Wide Web (USGS Water Data for the Nation)_. #> doi:10.5066/F7P55KJN , Accessed Jun -#> 03, 2025, +#> 04, 2025, #> . print(NWIScitation, style = "Bibtex") #> @Manual{, #> title = {National Water Information System data available on the World Wide Web (USGS Water Data for the Nation)}, #> author = {{U.S. Geological Survey}}, #> doi = {10.5066/F7P55KJN}, -#> note = {Accessed Jun 03, 2025}, +#> note = {Accessed Jun 04, 2025}, #> year = {2025}, #> url = {https://waterservices.usgs.gov/nwis/dv/?site=09010500&format=waterml%2C1.1&ParameterCd=00060&StatCd=00003&startDT=1851-01-01}, #> } @@ -223,14 +224,14 @@ WQPcitation <- create_WQP_bib(SC) WQPcitation #> National Water Quality Monitoring Council (2025). _Water Quality #> Portal_. doi:10.5066/P9QRKUVJ , -#> Accessed Jun 03, 2025, +#> Accessed Jun 04, 2025, #> . print(WQPcitation, style = "Bibtex") #> @Manual{, #> title = {Water Quality Portal}, #> author = {{National Water Quality Monitoring Council}}, #> doi = {10.5066/P9QRKUVJ}, -#> note = {Accessed Jun 03, 2025}, +#> note = {Accessed Jun 04, 2025}, #> year = {2025}, #> url = {https://www.waterqualitydata.us/data/Result/search?siteid=USGS-05288705&count=no&pCode=00300&mimeType=csv}, #> } diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 8c7bdfac..eda725e9 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -5,7 +5,7 @@ editor_options: output: rmarkdown::html_vignette: toc: true - number_sections: true + number_sections: false vignette: > %\VignetteIndexEntry{Introduction to New USGS Services} \usepackage[utf8]{inputenc} @@ -31,20 +31,23 @@ knitr::opts_chunk$set( ) ``` -As we bid adieu to the NWIS web services, we welcome a host of new web service offering: the [USGS Water Data OGC APIs](https://api.waterdata.usgs.gov/ogcapi/v0/). This is a modern access point for USGS water data. The USGS is [planning to modernize **all** web services](https://waterdata.usgs.gov/blog/api-whats-new-wdfn-apis/) in the near future. For each of these updates, `dataRetrieval` will create a new function to access the new services and deprecate functions for accessing the legacy services. -This document will introduce each new function (note as time goes on, we'll update this document to include additional functions). Scroll down for tips on what you'll need to do to convert your legacy scripts to use these modern functions. Note the timeline for the NWIS servers being shut down is currently very uncertain. We'd recommend incorporating these new functions as soon as possible to avoid future headaches. +As we bid adieu to the NWIS web services, we welcome a host of new web service offering: the [USGS Water Data OGC APIs](https://api.waterdata.usgs.gov/ogcapi/v0/). This is a modern access point for USGS water data. The USGS will be modernizing [**all** of the NWIS web services](https://waterdata.usgs.gov/blog/api-whats-new-wdfn-apis/) in the near future. For each of these updates, `dataRetrieval` will create a new function to access the new services and deprecate functions for accessing the legacy services. + +This document will introduce each new function (note as time goes on, we'll update this document to include additional functions). Note the timeline for the NWIS servers being shut down is currently very uncertain. We'd recommend incorporating these new functions as soon as possible to avoid future headaches. # New Features -Each new "API endpoint" will deliver a new type of USGS data. Currently the available endpoints are "monitoring-locations", "time-series-metadata", and "daily". All of these endpoints offer some new features that the NWIS services did not have: +Each new "API endpoint" will deliver a new type of USGS data. Currently the available endpoints are "monitoring-locations", "time-series-metadata", and "daily". All of these endpoints offer some new features that the legacy services did not have: + +## Flexible Queries -* **Supports [Contextual Query Language](https://www.loc.gov/standards/sru/cql/) (CQL) syntax**: any column can also be queried on! The main `dataRetrieval::read_USGS_*` functions will take vector inputs. Each query is appended as a boolean AND. For instance, if you ask for a vector of monitoring location ids and a vector of parameter codes, you will get data back for just those monitoring locations AND the specified parameter codes. Towards the bottom of this document, we'll discuss ways to make more specific queries using Contextual Query Language. +When you look at the help file for the new functions, you’ll notice there are many more arguments. These are mostly set by default to `NA`. You **DO NOT** need to (and most likely should not!) specify all of these parameters. The requested filters are appended as Boolean "AND"s, meaning if you specify a vector of monitoring locations and parameter codes, you will only get back those specified monitoring locations with the specified parameter codes. -* **Provides [Simple Features](https://en.wikipedia.org/wiki/Simple_Features) functionality**: data is returned with a "geometry" column, which is a simple feature object, allowing the data to be integrated with the [`sf`](https://r-spatial.github.io/sf/) package and associated geospatial workflows. +A side bonus of this is that since all of the potential arguments are defined, your favorite integrated development environment (IDE) will almost certainly have an autocomplete feature to let you tab through the potential options. -* **Enables more flexible queries**: when you look at the help file for the new functions, you’ll also notice there are many more arguments. These are mostly set by default to NA. You **DO NOT** need to (and most likely should not!) specify all of these parameters. +## Flexible Columns Returned -* Users can pick which columns are returned with the "properties" argument. Available properties can be discovered with the `check_OGC_requests` function: +Users can pick which columns are returned with the "properties" argument. Available properties can be discovered with the `check_OGC_requests` function: ```{r} daily_schema <- check_OGC_requests(endpoint = "daily", type = "schema") @@ -52,14 +55,24 @@ names(daily_schema$properties) ``` -* You can register an API key for use with USGS water data APIs. There are now limits on how many queries can be requested per IP address per hour. If you find yourself running into limits, you can request an API token here: +## API Tokens + +You can register an API key for use with USGS water data APIs. There are now limits on how many queries can be requested per IP address per hour. If you find yourself running into limits, you can request an API token here: Then save your token in your .Renviron file like this: ``` API_USGS_PAT = "my_super_secret_token" ``` -You can use `usethis::edit_r_environ()` to edit find and open your .Renviron file. You will need to restart R for that variable to be recognized. +You can use `usethis::edit_r_environ()` to edit find and open your .Renviron file. You will need to restart R for that variable to be recognized. You should not add this file to git projects or generally share your API key. Anyone else using your API key will count against the number of requests available to you! + +## Contextual Query Language 2 Support + +Supports [Contextual Query Language](https://www.loc.gov/standards/sru/cql/) (CQL2) syntax for flexible queries. We'll show how to use the `read_USGS_data` function to make specific queries using Contextual Query Language (2). + +## Simple Features + +Provides [Simple Features](https://en.wikipedia.org/wiki/Simple_Features) functionality. The data is returned with a "geometry" column, which is a simple feature object, allowing the data to be integrated with the [`sf`](https://r-spatial.github.io/sf/) package and associated geospatial workflows. # New Functions @@ -170,11 +183,9 @@ To access these services on a web browser, go to . +The new APIs can handle complex requests. For those queries, users will need to construct their own request using Contextual Query Language (CQL2). There's an excellent article . Let's try to find sites in Wisconsin and Minnesota that have a drainage area greater than 1000 mi^2. @@ -263,7 +274,7 @@ Historically, we allowed users to customize their queries via the `...` argument ### Dependencies -Under the hood, `dataRetrieval` changed the dependency from `httr` to `httr2`. `httr2` is the modern R package for web requests that is actively developed/maintained. As we develop functions for the modern USGS web services, we'll continue to explore updating package dependencies. Since the new services offer geospatial output, we have recommend using the `sf` package. The `whisker` package was also included to help create POST CQL2 queries. +Under the hood, `dataRetrieval` changed the dependency from `httr` to `httr2`. `httr2` is the modern R package for web requests that is actively developed/maintained. As we develop functions for the modern USGS web services, we'll continue to explore updating package dependencies. Since the new services offer geospatial output, we also now require the `sf` package. The `whisker` package was also included to help create POST CQL2 queries. ## Developmental workflow From 8946acfe9c9c33e7bdbc604d0cb11255188dad64 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Wed, 4 Jun 2025 16:19:18 -0500 Subject: [PATCH 092/117] Clean up language --- vignettes/tutorial.Rmd | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vignettes/tutorial.Rmd b/vignettes/tutorial.Rmd index 414e9c34..dbe1faae 100644 --- a/vignettes/tutorial.Rmd +++ b/vignettes/tutorial.Rmd @@ -76,8 +76,8 @@ Functions in `dataRetrieval` look like `readNWISdv`, `readNWISuv`, `readWQPqw`, + "read" will access full data sets + "what" will access data availability * _Middle_: "NWIS", "USGS", "WQP": - + NWIS functions get data from NWIS web services. - + USGS functions are the functions that will eventually replace the NWIS functions. These pull from modern USGS API services. + + NWIS functions get data from legacy NWIS web services. + + USGS functions are the functions that will eventually replace the legacy NWIS functions. These pull from modern USGS API services. + WQP functions are for discrete water-quality data from the Water Quality Portal. * _Suffix_: "data" or other: + Functions that end in "data": These are flexible, powerful functions that allow complex user queries. @@ -91,7 +91,7 @@ There are many types of data served from NWIS. To understand how the services ar * NWIS has traditionally been the source for all USGS water data -* NWIS will be retired (scheduled 2026, but uncertain): +* Legacy NWIS services will be retired (scheduled 2026, but uncertain): * * USGS functions will slowly replace NWIS functions From 42c920ebb7922bace0e8f3fd2c6334d0ddcda501 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 5 Jun 2025 10:55:05 -0500 Subject: [PATCH 093/117] Added max_results argument --- R/construct_api_requests.R | 23 +++++++---- R/read_USGS_daily.R | 46 ++++++++++++---------- R/read_USGS_data.R | 8 +++- R/read_USGS_monitoring_location.R | 36 +++++++++++++---- R/read_USGS_ts_meta.R | 21 ++++++---- R/walk_pages.R | 58 +++++++++++++++------------- man/construct_api_requests.Rd | 3 +- man/read_USGS_daily.Rd | 17 +++++--- man/read_USGS_monitoring_location.Rd | 24 +++++++++--- man/read_USGS_ts_meta.Rd | 33 +++++++++------- 10 files changed, 170 insertions(+), 99 deletions(-) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index 1016f590..5e4f8639 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -43,7 +43,8 @@ construct_api_requests <- function(service, properties = NA_character_, bbox = NA, - limit = 10000, + limit = NA, + max_results = NA, skipGeometry = FALSE, ...){ @@ -56,12 +57,6 @@ construct_api_requests <- function(service, several.ok = TRUE) } - use_sf <- all(pkg.env$local_sf) - - if(!use_sf){ - skipGeometry <- TRUE - } - if(all(all_properties[!all_properties %in% c("id", "geometry")] %in% properties)) { # Cleans up URL if we're asking for everything properties <- NA_character_ @@ -89,7 +84,19 @@ construct_api_requests <- function(service, get_list <- full_list[names(full_list) %in% single_params] get_list[["skipGeometry"]] <- skipGeometry - get_list[["limit"]] <- limit + + if(is.na(limit)){ + if(!is.na(max_results)){ + get_list[["limit"]] <- max_results + } else { + get_list[["limit"]] <- 10000 + } + } else { + if(!is.na(max_results)){ + if(limit > max_results) stop("limit cannot be greater than max_result") + } + get_list[["limit"]] <- limit + } post_list <- full_list[!names(full_list) %in% single_params] diff --git a/R/read_USGS_daily.R b/R/read_USGS_daily.R index d6341f5f..3d9e99d8 100644 --- a/R/read_USGS_daily.R +++ b/R/read_USGS_daily.R @@ -23,10 +23,13 @@ #' depth). Coordinates are assumed to be in crs 4326. The expected format is a numeric #' vector structured: c(xmin,ymin,xmax,ymax). Another way to think of it is c(Western-most longitude, #' Southern-most latitude, Eastern-most longitude, Northern-most longitude). -#' @param limit The optional limit parameter limits the number of items that are -#' presented in the response document. Only items are counted that are on the -#' first level of the collection in the response document. Nested objects -#' contained within the explicitly requested items shall not be counted. +#' @param limit The optional limit parameter is used to control the subset of the +#' selected features that should be returned in each page. The maximum allowable +#' limit is 10000. It may be beneficial to set this number lower if your internet +#' connection is spotty. The default (`NA`) will set the limit to the maximum +#' allowable limit for the service. +#' @param max_resuts The optional maximum number of rows to return. This value +#' must be less than the requested limit. #' @param skipGeometry This option can be used to skip response geometries for #' each feature. The returning object will be a data frame with no spatial #' information. @@ -57,28 +60,29 @@ #' time = "P7D") #' #' multi_site <- read_USGS_daily(monitoring_location_id = c("USGS-01491000", -#' "USGS-01645000"), +#' "USGS-01645000"), #' parameter_code = c("00060", "00010"), #' limit = 500, #' time = c("2023-01-01", "2024-01-01")) #' #' } read_USGS_daily <- function(monitoring_location_id = NA_character_, - parameter_code = NA_character_, - statistic_id = NA_character_, - properties = NA_character_, - time_series_id = NA_character_, - daily_id = NA_character_, - approval_status = NA_character_, - unit_of_measure = NA_character_, - qualifier = NA_character_, - value = NA, - last_modified = NA_character_, - limit = 10000, - skipGeometry = NA, - time = NA_character_, - bbox = NA, - convertType = TRUE){ + parameter_code = NA_character_, + statistic_id = NA_character_, + properties = NA_character_, + time_series_id = NA_character_, + daily_id = NA_character_, + approval_status = NA_character_, + unit_of_measure = NA_character_, + qualifier = NA_character_, + value = NA, + last_modified = NA_character_, + skipGeometry = NA, + time = NA_character_, + bbox = NA, + limit = NA, + max_results = NA, + convertType = TRUE){ message("Function in development, use at your own risk.") @@ -90,7 +94,7 @@ read_USGS_daily <- function(monitoring_location_id = NA_character_, args[["service"]] <- service dv_req <- do.call(construct_api_requests, args) - return_list <- walk_pages(dv_req) + return_list <- walk_pages(dv_req, max_results) return_list <- deal_with_empty(return_list, properties, service) diff --git a/R/read_USGS_data.R b/R/read_USGS_data.R index 58b2c3cc..e8f52c49 100644 --- a/R/read_USGS_data.R +++ b/R/read_USGS_data.R @@ -65,7 +65,13 @@ read_USGS_data <- function(service, httr2::req_headers(`Content-Type` = "application/query-cql-json") |> httr2::req_body_raw(CQL) - return_list <- walk_pages(data_req) + if("max_results" %in% names(args)){ + max_results <- args[["max_results"]] + } else { + max_results <- NA + } + + return_list <- walk_pages(data_req, max_results) return_list <- deal_with_empty(return_list, args[["properties"]], service) diff --git a/R/read_USGS_monitoring_location.R b/R/read_USGS_monitoring_location.R index 8c577236..71ec5b21 100644 --- a/R/read_USGS_monitoring_location.R +++ b/R/read_USGS_monitoring_location.R @@ -52,10 +52,13 @@ #' depth). Coordinates are assumed to be in crs 4326. The expected format is a numeric #' vector structured: c(xmin,ymin,xmax,ymax). Another way to think of it is c(Western-most longitude, #' Southern-most latitude, Eastern-most longitude, Northern-most longitude). -#' @param limit The optional limit parameter limits the number of items that are -#' presented in the response document. Only items are counted that are on the -#' first level of the collection in the response document. Nested objects -#' contained within the explicitly requested items shall not be counted. +#' @param limit The optional limit parameter is used to control the subset of the +#' selected features that should be returned in each page. The maximum allowable +#' limit is 10000. It may be beneficial to set this number lower if your internet +#' connection is spotty. The default (`NA`) will set the limit to the maximum +#' allowable limit for the service. +#' @param max_resuts The optional maximum number of rows to return. This value +#' must be less than the requested limit. #' @param skipGeometry This option can be used to skip response geometries for #' each feature. The returning object will be a data frame with no spatial #' information. @@ -66,9 +69,15 @@ #' site_info <- read_USGS_monitoring_location(monitoring_location_id = site) #' #' site_slim <- read_USGS_monitoring_location(monitoring_location_id = site, -#' properties = c("monitoring_locations_id", +#' properties = c("monitoring_location_id", #' "state_name", #' "country_name")) +#' +#' site_slim_no_sf_slim <- read_USGS_monitoring_location(monitoring_location_id = site, +#' properties = c("monitoring_location_id", +#' "state_name", +#' "country_name"), +#' skipGeometry = TRUE) #' #' site_info_no_sf <- read_USGS_monitoring_location(monitoring_location_id = site, #' skipGeometry = TRUE) @@ -77,6 +86,10 @@ #' #' bbox_vals = c(-94.00, 35.0, -93.5, 35.5) #' multi_site <- read_USGS_monitoring_location(bbox = bbox_vals) +#' multi_site_n_100 <- read_USGS_monitoring_location(bbox = bbox_vals, +#' max_results = 100) +#' multi_site_limit_100 <- read_USGS_monitoring_location(bbox = bbox_vals, +#' limit = 100) #' } read_USGS_monitoring_location <- function(monitoring_location_id = NA_character_, agency_code = NA_character_, @@ -120,7 +133,8 @@ read_USGS_monitoring_location <- function(monitoring_location_id = NA_character_ depth_source_code = NA_character_, properties = NA_character_, bbox = NA, - limit = 10000, + limit = NA, + max_results = NA, skipGeometry = NA){ message("Function in development, use at your own risk.") @@ -131,13 +145,21 @@ read_USGS_monitoring_location <- function(monitoring_location_id = NA_character_ args[["service"]] <- service args[["id"]] <- args[["monitoring_location_id"]] args[["monitoring_location_id"]] <- NULL + + # help with monitoring_location(s) confusion + # User can put in either in the property vector: + properties[properties == "monitoring_location_id"] <- "monitoring_locations_id" + args[["properties"]][args[["properties"]] == "monitoring_location_id"] <- "monitoring_locations_id" + site_req <- do.call(construct_api_requests, args) - return_list <- walk_pages(site_req) + return_list <- walk_pages(site_req, max_results) return_list <- deal_with_empty(return_list, properties, service) return_list <- rejigger_cols(return_list, properties, service) + # They will get back the singular though, to work with the other services. + names(return_list)[(names(return_list) == "monitoring_locations_id")] <- "monitoring_location_id" return(return_list) } diff --git a/R/read_USGS_ts_meta.R b/R/read_USGS_ts_meta.R index 887da8ee..7d99f04d 100644 --- a/R/read_USGS_ts_meta.R +++ b/R/read_USGS_ts_meta.R @@ -27,10 +27,13 @@ #' depth). Coordinates are assumed to be in crs 4326. The expected format is a numeric #' vector structured: c(xmin,ymin,xmax,ymax). Another way to think of it is c(Western-most longitude, #' Southern-most latitude, Eastern-most longitude, Northern-most longitude). -#' @param limit The optional limit parameter limits the number of items that are -#' presented in the response document. Only items are counted that are on the -#' first level of the collection in the response document. Nested objects -#' contained within the explicitly requested items shall not be counted. +#' @param limit The optional limit parameter is used to control the subset of the +#' selected features that should be returned in each page. The maximum allowable +#' limit is 10000. It may be beneficial to set this number lower if your internet +#' connection is spotty. The default (`NA`) will set the limit to the maximum +#' allowable limit for the service. +#' @param max_resuts The optional maximum number of rows to return. This value +#' must be less than the requested limit. #' @param convertType logical, defaults to `TRUE`. If `TRUE`, the function #' will convert the data to dates and qualifier to string vector. #' @param skipGeometry This option can be used to skip response geometries for @@ -48,15 +51,14 @@ #' properties = c("monitoring_location_id", #' "parameter_code", #' "begin", -#' "end"), +#' "end", +#' "time_series_id"), #' skipGeometry = TRUE) #' } read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, parameter_code = NA_character_, parameter_name = NA_character_, properties = NA_character_, - limit = 10000, - bbox = NA, statistic_id = NA_character_, last_modified = NA_character_, begin = NA_character_, @@ -70,6 +72,9 @@ read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, time_series_metadata_id = NA_character_, web_description = NA_character_, skipGeometry = NA, + limit = NA, + max_results = NA, + bbox = NA, convertType = FALSE){ message("Function in development, use at your own risk.") @@ -84,7 +89,7 @@ read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, args[["convertType"]] <- NULL req_ts_meta <- do.call(construct_api_requests, args) - return_list <- walk_pages(req_ts_meta) + return_list <- walk_pages(req_ts_meta, max_results) return_list <- deal_with_empty(return_list, properties, service) diff --git a/R/walk_pages.R b/R/walk_pages.R index a5476b2e..0354d010 100644 --- a/R/walk_pages.R +++ b/R/walk_pages.R @@ -180,12 +180,11 @@ get_resp_data <- function(resp) { } use_sf <- !grepl("skipGeometry=true", resp$url, ignore.case = TRUE) + return_df <- sf::read_sf(httr2::resp_body_string(resp)) - if(use_sf){ - return_df <- sf::read_sf(httr2::resp_body_string(resp)) - } else { - return_df <- jsonlite::fromJSON(httr2::resp_body_string(resp))[["features"]][["properties"]] - } + if(!use_sf){ + return_df <- sf::st_drop_geometry(return_df) + } return(return_df) @@ -197,28 +196,33 @@ get_resp_data <- function(resp) { #' #' @noRd #' @return data.frame with attributes -walk_pages <- function(req){ - - resps <- httr2::req_perform_iterative(req, - next_req = next_req_url, - max_reqs = Inf) - - ###################################### - # So far I haven't tested this because I haven't had - # individual failures - failures <- resps |> - httr2::resps_failures() |> - httr2::resps_requests() - - if(length(failures) > 0){ - message("There were", length(failures), "failed requests.") - } - ###################################### - - return_list <- data.frame() - for(resp in resps){ - df1 <- get_resp_data(resp) - return_list <- rbind(return_list, df1) +walk_pages <- function(req, max_resuts){ + + if(is.na(max_resuts)){ + resps <- httr2::req_perform_iterative(req, + next_req = next_req_url, + max_reqs = Inf) + ###################################### + # So far I haven't tested this because I haven't had + # individual failures + failures <- resps |> + httr2::resps_failures() |> + httr2::resps_requests() + + if(length(failures) > 0){ + message("There were", length(failures), "failed requests.") + } + + return_list <- data.frame() + for(resp in resps){ + df1 <- get_resp_data(resp) + return_list <- rbind(return_list, df1) + } + + ###################################### + } else { + resps <- httr2::req_perform(req) + return_list <- get_resp_data(resps) } attr(return_list, "request") <- req diff --git a/man/construct_api_requests.Rd b/man/construct_api_requests.Rd index 3e75315f..0c56fc39 100644 --- a/man/construct_api_requests.Rd +++ b/man/construct_api_requests.Rd @@ -8,7 +8,8 @@ construct_api_requests( service, properties = NA_character_, bbox = NA, - limit = 10000, + limit = NA, + max_results = NA, skipGeometry = FALSE, ... ) diff --git a/man/read_USGS_daily.Rd b/man/read_USGS_daily.Rd index f30acb71..2c287304 100644 --- a/man/read_USGS_daily.Rd +++ b/man/read_USGS_daily.Rd @@ -16,10 +16,11 @@ read_USGS_daily( qualifier = NA_character_, value = NA, last_modified = NA_character_, - limit = 10000, skipGeometry = NA, time = NA_character_, bbox = NA, + limit = NA, + max_results = NA, convertType = TRUE ) } @@ -58,11 +59,6 @@ Examples: Only features that have a \code{last_modified} that intersects the value of datetime are selected. If a feature has multiple temporal properties, it is the decision of the server whether only a single temporal property is used to determine the extent or all relevant temporal properties.} -\item{limit}{The optional limit parameter limits the number of items that are -presented in the response document. Only items are counted that are on the -first level of the collection in the response document. Nested objects -contained within the explicitly requested items shall not be counted.} - \item{skipGeometry}{This option can be used to skip response geometries for each feature. The returning object will be a data frame with no spatial information.} @@ -85,8 +81,17 @@ depth). Coordinates are assumed to be in crs 4326. The expected format is a nume vector structured: c(xmin,ymin,xmax,ymax). Another way to think of it is c(Western-most longitude, Southern-most latitude, Eastern-most longitude, Northern-most longitude).} +\item{limit}{The optional limit parameter is used to control the subset of the +selected features that should be returned in each page. The maximum allowable +limit is 10000. It may be beneficial to set this number lower if your internet +connection is spotty. The default (\code{NA}) will set the limit to the maximum +allowable limit for the service.} + \item{convertType}{logical, defaults to \code{TRUE}. If \code{TRUE}, the function will convert the data to dates and qualifier to string vector.} + +\item{max_resuts}{The optional maximum number of rows to return. This value +must be less than the requested limit.} } \description{ Description Daily data provide one data value to represent water conditions for the day. Throughout much of the history of the USGS, the primary water data available was daily data collected manually at the monitoring location once each day. With improved availability of computer storage and automated transmission of data, the daily data published today are generally a statistical summary or metric of the continuous data collected each day, such as the daily mean, minimum, or maximum value. Daily data are automatically calculated from the continuous data of the same parameter code and are described by parameter code and a statistic code. These data have also been referred to as “daily values” or “DV”. diff --git a/man/read_USGS_monitoring_location.Rd b/man/read_USGS_monitoring_location.Rd index 1da7369d..f262e76d 100644 --- a/man/read_USGS_monitoring_location.Rd +++ b/man/read_USGS_monitoring_location.Rd @@ -47,7 +47,8 @@ read_USGS_monitoring_location( depth_source_code = NA_character_, properties = NA_character_, bbox = NA, - limit = 10000, + limit = NA, + max_results = NA, skipGeometry = NA ) } @@ -143,14 +144,18 @@ depth). Coordinates are assumed to be in crs 4326. The expected format is a nume vector structured: c(xmin,ymin,xmax,ymax). Another way to think of it is c(Western-most longitude, Southern-most latitude, Eastern-most longitude, Northern-most longitude).} -\item{limit}{The optional limit parameter limits the number of items that are -presented in the response document. Only items are counted that are on the -first level of the collection in the response document. Nested objects -contained within the explicitly requested items shall not be counted.} +\item{limit}{The optional limit parameter is used to control the subset of the +selected features that should be returned in each page. The maximum allowable +limit is 10000. It may be beneficial to set this number lower if your internet +connection is spotty. The default (\code{NA}) will set the limit to the maximum +allowable limit for the service.} \item{skipGeometry}{This option can be used to skip response geometries for each feature. The returning object will be a data frame with no spatial information.} + +\item{max_resuts}{The optional maximum number of rows to return. This value +must be less than the requested limit.} } \description{ Description Location information is basic information about the monitoring location including the name, identifier, agency responsible for data collection, and the date the location was established. It also includes information about the type of location, such as stream, lake, or groundwater, and geographic information about the location, such as state, county, latitude and longitude, and hydrologic unit code (HUC). @@ -163,9 +168,15 @@ site <- "USGS-02238500" site_info <- read_USGS_monitoring_location(monitoring_location_id = site) site_slim <- read_USGS_monitoring_location(monitoring_location_id = site, - properties = c("monitoring_locations_id", + properties = c("monitoring_location_id", "state_name", "country_name")) + +site_slim_no_sf_slim <- read_USGS_monitoring_location(monitoring_location_id = site, + properties = c("monitoring_location_id", + "state_name", + "country_name"), + skipGeometry = TRUE) site_info_no_sf <- read_USGS_monitoring_location(monitoring_location_id = site, skipGeometry = TRUE) @@ -174,6 +185,7 @@ multi_site <- read_USGS_monitoring_location(state_name = "Wisconsin") bbox_vals = c(-94.00, 35.0, -93.5, 35.5) multi_site <- read_USGS_monitoring_location(bbox = bbox_vals) +multi_site <- read_USGS_monitoring_location(bbox = bbox_vals, max_results = 100) } \dontshow{\}) # examplesIf} } diff --git a/man/read_USGS_ts_meta.Rd b/man/read_USGS_ts_meta.Rd index ced421a7..1cfb6bbc 100644 --- a/man/read_USGS_ts_meta.Rd +++ b/man/read_USGS_ts_meta.Rd @@ -9,8 +9,6 @@ read_USGS_ts_meta( parameter_code = NA_character_, parameter_name = NA_character_, properties = NA_character_, - limit = 10000, - bbox = NA, statistic_id = NA_character_, last_modified = NA_character_, begin = NA_character_, @@ -24,6 +22,9 @@ read_USGS_ts_meta( time_series_metadata_id = NA_character_, web_description = NA_character_, skipGeometry = NA, + limit = NA, + bbox = NA, + max_results = NA, convertType = FALSE ) } @@ -38,18 +39,6 @@ read_USGS_ts_meta( Available options are: geometry, id, unit_of_measure, parameter_name, parameter_code, statistic_id, last_modified, begin, end, computation_period_identifier, computation_identifier, thresholds, sublocation_identifier, primary, monitoring_location_id, web_description, parameter_description} -\item{limit}{The optional limit parameter limits the number of items that are -presented in the response document. Only items are counted that are on the -first level of the collection in the response document. Nested objects -contained within the explicitly requested items shall not be counted.} - -\item{bbox}{Only features that have a geometry that intersects the bounding -box are selected.The bounding box is provided as four or six numbers, depending -on whether the coordinate reference system includes a vertical axis (height or -depth). Coordinates are assumed to be in crs 4326. The expected format is a numeric -vector structured: c(xmin,ymin,xmax,ymax). Another way to think of it is c(Western-most longitude, -Southern-most latitude, Eastern-most longitude, Northern-most longitude).} - \item{statistic_id}{A code corresponding to the statistic an observation represents. Example codes include 00001 (max), 00002 (min), and 00003 (mean). A complete list of codes and their descriptions can be found at \url{https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=\%25&fmt=html}.} \item{last_modified}{The last time a record was refreshed in our database. This may happen due to regular operational processes and does not necessarily indicate anything about the measurement has changed. @@ -88,8 +77,24 @@ Only features that have a \code{last_modified} that intersects the value of date each feature. The returning object will be a data frame with no spatial information.} +\item{limit}{The optional limit parameter is used to control the subset of the +selected features that should be returned in each page. The maximum allowable +limit is 10000. It may be beneficial to set this number lower if your internet +connection is spotty. The default (\code{NA}) will set the limit to the maximum +allowable limit for the service.} + +\item{bbox}{Only features that have a geometry that intersects the bounding +box are selected.The bounding box is provided as four or six numbers, depending +on whether the coordinate reference system includes a vertical axis (height or +depth). Coordinates are assumed to be in crs 4326. The expected format is a numeric +vector structured: c(xmin,ymin,xmax,ymax). Another way to think of it is c(Western-most longitude, +Southern-most latitude, Eastern-most longitude, Northern-most longitude).} + \item{convertType}{logical, defaults to \code{TRUE}. If \code{TRUE}, the function will convert the data to dates and qualifier to string vector.} + +\item{max_resuts}{The optional maximum number of rows to return. This value +must be less than the requested limit.} } \description{ Description Daily data and continuous measurements are grouped into time series, which represent a collection of observations of a single parameter, potentially aggregated using a standard statistic, at a single monitoring location. This endpoint provides metadata about those time series, including their operational thresholds, units of measurement, and when the earliest and most recent observations in a time series occurred. From 3575b4cd148c66bfdbc3d16c985eb83ea08cf449 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 5 Jun 2025 11:22:21 -0500 Subject: [PATCH 094/117] =?UTF-8?q?Learned=20how=20to=20spell=20results=20?= =?UTF-8?q?=F0=9F=98=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- R/read_USGS_daily.R | 20 ++++++++++---------- R/read_USGS_monitoring_location.R | 2 +- R/read_USGS_ts_meta.R | 6 ++++-- man/read_USGS_daily.Rd | 26 +++++++++++++------------- man/read_USGS_monitoring_location.Rd | 11 +++++++---- man/read_USGS_ts_meta.Rd | 13 +++++++------ 6 files changed, 42 insertions(+), 36 deletions(-) diff --git a/R/read_USGS_daily.R b/R/read_USGS_daily.R index 3d9e99d8..371973c2 100644 --- a/R/read_USGS_daily.R +++ b/R/read_USGS_daily.R @@ -28,7 +28,7 @@ #' limit is 10000. It may be beneficial to set this number lower if your internet #' connection is spotty. The default (`NA`) will set the limit to the maximum #' allowable limit for the service. -#' @param max_resuts The optional maximum number of rows to return. This value +#' @param max_results The optional maximum number of rows to return. This value #' must be less than the requested limit. #' @param skipGeometry This option can be used to skip response geometries for #' each feature. The returning object will be a data frame with no spatial @@ -41,8 +41,8 @@ #' site <- "USGS-02238500" #' pcode <- "00060" #' dv_data_sf <- read_USGS_daily(monitoring_location_id = site, -#' parameter_code = "00060", -#' time = c("2021-01-01", "2022-01-01")) +#' parameter_code = "00060", +#' time = c("2021-01-01", "2022-01-01")) #' #' dv_data_trim <- read_USGS_daily(monitoring_location_id = site, #' parameter_code = "00060", @@ -52,18 +52,18 @@ #' time = c("2021-01-01", "2022-01-01")) #' #' dv_data <- read_USGS_daily(monitoring_location_id = site, -#' parameter_code = "00060", -#' skipGeometry = TRUE) +#' parameter_code = "00060", +#' skipGeometry = TRUE) #' #' dv_data_period <- read_USGS_daily(monitoring_location_id = site, -#' parameter_code = "00060", -#' time = "P7D") +#' parameter_code = "00060", +#' time = "P7D") #' #' multi_site <- read_USGS_daily(monitoring_location_id = c("USGS-01491000", #' "USGS-01645000"), -#' parameter_code = c("00060", "00010"), -#' limit = 500, -#' time = c("2023-01-01", "2024-01-01")) +#' parameter_code = c("00060", "00010"), +#' limit = 500, +#' time = c("2023-01-01", "2024-01-01")) #' #' } read_USGS_daily <- function(monitoring_location_id = NA_character_, diff --git a/R/read_USGS_monitoring_location.R b/R/read_USGS_monitoring_location.R index 71ec5b21..eb9fc5a8 100644 --- a/R/read_USGS_monitoring_location.R +++ b/R/read_USGS_monitoring_location.R @@ -57,7 +57,7 @@ #' limit is 10000. It may be beneficial to set this number lower if your internet #' connection is spotty. The default (`NA`) will set the limit to the maximum #' allowable limit for the service. -#' @param max_resuts The optional maximum number of rows to return. This value +#' @param max_results The optional maximum number of rows to return. This value #' must be less than the requested limit. #' @param skipGeometry This option can be used to skip response geometries for #' each feature. The returning object will be a data frame with no spatial diff --git a/R/read_USGS_ts_meta.R b/R/read_USGS_ts_meta.R index 7d99f04d..8f876f6c 100644 --- a/R/read_USGS_ts_meta.R +++ b/R/read_USGS_ts_meta.R @@ -32,7 +32,7 @@ #' limit is 10000. It may be beneficial to set this number lower if your internet #' connection is spotty. The default (`NA`) will set the limit to the maximum #' allowable limit for the service. -#' @param max_resuts The optional maximum number of rows to return. This value +#' @param max_results The optional maximum number of rows to return. This value #' must be less than the requested limit. #' @param convertType logical, defaults to `TRUE`. If `TRUE`, the function #' will convert the data to dates and qualifier to string vector. @@ -46,7 +46,7 @@ #' meta_1 <- read_USGS_ts_meta(monitoring_location_id = site) #' #' meta_multi <- read_USGS_ts_meta(monitoring_location_id = c("USGS-01491000", -#' "USGS-01645000"), +#' "USGS-01645000"), #' parameter_code = c("00060", "00010"), #' properties = c("monitoring_location_id", #' "parameter_code", @@ -97,6 +97,8 @@ read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, return_list <- rejigger_cols(return_list, properties, service) + names(return_list)[(names(return_list) == "time_series_metadata_id")] <- "time_series_id" + return(return_list) } diff --git a/man/read_USGS_daily.Rd b/man/read_USGS_daily.Rd index 2c287304..aa7a7f10 100644 --- a/man/read_USGS_daily.Rd +++ b/man/read_USGS_daily.Rd @@ -87,11 +87,11 @@ limit is 10000. It may be beneficial to set this number lower if your internet connection is spotty. The default (\code{NA}) will set the limit to the maximum allowable limit for the service.} +\item{max_results}{The optional maximum number of rows to return. This value +must be less than the requested limit.} + \item{convertType}{logical, defaults to \code{TRUE}. If \code{TRUE}, the function will convert the data to dates and qualifier to string vector.} - -\item{max_resuts}{The optional maximum number of rows to return. This value -must be less than the requested limit.} } \description{ Description Daily data provide one data value to represent water conditions for the day. Throughout much of the history of the USGS, the primary water data available was daily data collected manually at the monitoring location once each day. With improved availability of computer storage and automated transmission of data, the daily data published today are generally a statistical summary or metric of the continuous data collected each day, such as the daily mean, minimum, or maximum value. Daily data are automatically calculated from the continuous data of the same parameter code and are described by parameter code and a statistic code. These data have also been referred to as “daily values” or “DV”. @@ -103,8 +103,8 @@ Description Daily data provide one data value to represent water conditions for site <- "USGS-02238500" pcode <- "00060" dv_data_sf <- read_USGS_daily(monitoring_location_id = site, - parameter_code = "00060", - time = c("2021-01-01", "2022-01-01")) + parameter_code = "00060", + time = c("2021-01-01", "2022-01-01")) dv_data_trim <- read_USGS_daily(monitoring_location_id = site, parameter_code = "00060", @@ -114,18 +114,18 @@ dv_data_trim <- read_USGS_daily(monitoring_location_id = site, time = c("2021-01-01", "2022-01-01")) dv_data <- read_USGS_daily(monitoring_location_id = site, - parameter_code = "00060", - skipGeometry = TRUE) + parameter_code = "00060", + skipGeometry = TRUE) dv_data_period <- read_USGS_daily(monitoring_location_id = site, - parameter_code = "00060", - time = "P7D") + parameter_code = "00060", + time = "P7D") multi_site <- read_USGS_daily(monitoring_location_id = c("USGS-01491000", - "USGS-01645000"), - parameter_code = c("00060", "00010"), - limit = 500, - time = c("2023-01-01", "2024-01-01")) + "USGS-01645000"), + parameter_code = c("00060", "00010"), + limit = 500, + time = c("2023-01-01", "2024-01-01")) } \dontshow{\}) # examplesIf} diff --git a/man/read_USGS_monitoring_location.Rd b/man/read_USGS_monitoring_location.Rd index f262e76d..f8be4bad 100644 --- a/man/read_USGS_monitoring_location.Rd +++ b/man/read_USGS_monitoring_location.Rd @@ -150,12 +150,12 @@ limit is 10000. It may be beneficial to set this number lower if your internet connection is spotty. The default (\code{NA}) will set the limit to the maximum allowable limit for the service.} +\item{max_results}{The optional maximum number of rows to return. This value +must be less than the requested limit.} + \item{skipGeometry}{This option can be used to skip response geometries for each feature. The returning object will be a data frame with no spatial information.} - -\item{max_resuts}{The optional maximum number of rows to return. This value -must be less than the requested limit.} } \description{ Description Location information is basic information about the monitoring location including the name, identifier, agency responsible for data collection, and the date the location was established. It also includes information about the type of location, such as stream, lake, or groundwater, and geographic information about the location, such as state, county, latitude and longitude, and hydrologic unit code (HUC). @@ -185,7 +185,10 @@ multi_site <- read_USGS_monitoring_location(state_name = "Wisconsin") bbox_vals = c(-94.00, 35.0, -93.5, 35.5) multi_site <- read_USGS_monitoring_location(bbox = bbox_vals) -multi_site <- read_USGS_monitoring_location(bbox = bbox_vals, max_results = 100) +multi_site_n_100 <- read_USGS_monitoring_location(bbox = bbox_vals, + max_results = 100) +multi_site_limit_100 <- read_USGS_monitoring_location(bbox = bbox_vals, + limit = 100) } \dontshow{\}) # examplesIf} } diff --git a/man/read_USGS_ts_meta.Rd b/man/read_USGS_ts_meta.Rd index 1cfb6bbc..e2825bd6 100644 --- a/man/read_USGS_ts_meta.Rd +++ b/man/read_USGS_ts_meta.Rd @@ -23,8 +23,8 @@ read_USGS_ts_meta( web_description = NA_character_, skipGeometry = NA, limit = NA, - bbox = NA, max_results = NA, + bbox = NA, convertType = FALSE ) } @@ -83,6 +83,9 @@ limit is 10000. It may be beneficial to set this number lower if your internet connection is spotty. The default (\code{NA}) will set the limit to the maximum allowable limit for the service.} +\item{max_results}{The optional maximum number of rows to return. This value +must be less than the requested limit.} + \item{bbox}{Only features that have a geometry that intersects the bounding box are selected.The bounding box is provided as four or six numbers, depending on whether the coordinate reference system includes a vertical axis (height or @@ -92,9 +95,6 @@ Southern-most latitude, Eastern-most longitude, Northern-most longitude).} \item{convertType}{logical, defaults to \code{TRUE}. If \code{TRUE}, the function will convert the data to dates and qualifier to string vector.} - -\item{max_resuts}{The optional maximum number of rows to return. This value -must be less than the requested limit.} } \description{ Description Daily data and continuous measurements are grouped into time series, which represent a collection of observations of a single parameter, potentially aggregated using a standard statistic, at a single monitoring location. This endpoint provides metadata about those time series, including their operational thresholds, units of measurement, and when the earliest and most recent observations in a time series occurred. @@ -107,12 +107,13 @@ site <- "USGS-02238500" meta_1 <- read_USGS_ts_meta(monitoring_location_id = site) meta_multi <- read_USGS_ts_meta(monitoring_location_id = c("USGS-01491000", - "USGS-01645000"), + "USGS-01645000"), parameter_code = c("00060", "00010"), properties = c("monitoring_location_id", "parameter_code", "begin", - "end"), + "end", + "time_series_id"), skipGeometry = TRUE) } \dontshow{\}) # examplesIf} From 978dcdbdd51956d867723417a04eb6fa46e4e23f Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 5 Jun 2025 11:35:08 -0500 Subject: [PATCH 095/117] Still learning to spell --- R/walk_pages.R | 4 ++-- tests/testthat/tests_userFriendly_fxns.R | 13 +++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/R/walk_pages.R b/R/walk_pages.R index 0354d010..031a08d4 100644 --- a/R/walk_pages.R +++ b/R/walk_pages.R @@ -196,9 +196,9 @@ get_resp_data <- function(resp) { #' #' @noRd #' @return data.frame with attributes -walk_pages <- function(req, max_resuts){ +walk_pages <- function(req, max_results){ - if(is.na(max_resuts)){ + if(is.na(max_results)){ resps <- httr2::req_perform_iterative(req, next_req = next_req_url, max_reqs = Inf) diff --git a/tests/testthat/tests_userFriendly_fxns.R b/tests/testthat/tests_userFriendly_fxns.R index 4d2b39ef..77b7083c 100644 --- a/tests/testthat/tests_userFriendly_fxns.R +++ b/tests/testthat/tests_userFriendly_fxns.R @@ -83,11 +83,12 @@ test_that("peak, rating curves, surface-water measurements", { data <- readNWISmeas(siteNumbers) expect_is(data$agency_cd, "character") - siteINFO_USGS <- read_USGS_monitoring_location("USGS-05114000") + siteINFO_USGS <- read_USGS_monitoring_location(monitoring_location_id = "USGS-05114000") expect_is(siteINFO_USGS$agency_code, "character") - expect_equal(siteINFO_USGS$monitoring_locations_id, "USGS-05114000") + expect_equal(siteINFO_USGS$monitoring_location_id, "USGS-05114000") - siteINFOMulti_USGS <- read_USGS_monitoring_location(c("USGS-05114000", "USGS-09423350")) + siteINFOMulti_USGS <- read_USGS_monitoring_location(monitoring_location_id = c("USGS-05114000", + "USGS-09423350")) expect_true(nrow(siteINFOMulti_USGS) == 2) Meas07227500.ex <- readNWISmeas("07227500", expanded = TRUE) @@ -381,7 +382,7 @@ test_that("Construct USGS urls", { expect_equal(url_daily$url, "https://api.waterdata.usgs.gov/ogcapi/v0/collections/daily/items?f=json&lang=en-US&time=2024-01-01T00%3A00%3A00Z%2F..&skipGeometry=FALSE&limit=10000") - url_works <- dataRetrieval:::walk_pages(url_daily) + url_works <- dataRetrieval:::walk_pages(url_daily, max_results = 1) expect_true(nrow(url_works) > 0) url_ts_meta <- construct_api_requests(monitoring_location_id = siteNumber, @@ -393,7 +394,7 @@ test_that("Construct USGS urls", { "https://api.waterdata.usgs.gov/ogcapi/v0/collections/time-series-metadata/items?f=json&lang=en-US&skipGeometry=FALSE&limit=10000" ) - url_works_ts <- dataRetrieval:::walk_pages(url_ts_meta) + url_works_ts <- dataRetrieval:::walk_pages(url_ts_meta, max_results = 1) expect_true(nrow(url_works_ts) > 0) url_ml <- construct_api_requests(id = siteNumber, @@ -401,7 +402,7 @@ test_that("Construct USGS urls", { expect_equal(url_ml$url, "https://api.waterdata.usgs.gov/ogcapi/v0/collections/monitoring-locations/items?f=json&lang=en-US&skipGeometry=FALSE&limit=10000&id=USGS-01594440") - url_works_ml <- dataRetrieval:::walk_pages(url_ml) + url_works_ml <- dataRetrieval:::walk_pages(url_ml, max_results = 1) expect_true(nrow(url_works_ml) > 0) url_use <- constructUseURL( From 783a8201786f512a30a7816c5c6abafc6c5b4d5f Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 5 Jun 2025 12:10:33 -0500 Subject: [PATCH 096/117] add some tips --- vignettes/read_USGS_functions.Rmd | 75 +++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index eda725e9..a748aac8 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -74,6 +74,81 @@ Supports [Contextual Query Language](https://www.loc.gov/standards/sru/cql/) (CQ Provides [Simple Features](https://en.wikipedia.org/wiki/Simple_Features) functionality. The data is returned with a "geometry" column, which is a simple feature object, allowing the data to be integrated with the [`sf`](https://r-spatial.github.io/sf/) package and associated geospatial workflows. +# Lessons Learned + +This section will initially be a random stream of consciousness on lessons learned while developing these functions and playing with the services. + +## Query limits + +A semi-common way to find a lot of data in the past would have been to use a monitoring location query to get a huge list of sites, and then use that huge list of sites (maybe winnowing it down a little) to get the data. These new services return a 403 error if your request is too big ("web server understands your request but refuses to authorize it"). This is true whether or not the request is a GET or POST request (something that is taken care of under the hood), and seems to be a character limit of the overall request. Roughly, it seems like if you were requesting more than 250 monitoring locations, the response will immediately return with a 403 error. + +There are at least 2 ways to deal with this. One is to manually split the data requests and bind the results together later. The other is to use the bounding box of the initial request as an input to the data request. Potentially some sites would need to be filtered out later using this method. + +Example: + +```{r} +ohio <- read_USGS_monitoring_location(state_name = "Ohio", + site_type_code = "ST") + +``` + +There are `r nrow(ohio)` rows returned that are stream sites in Ohio. If we tried to ask for all the discharge data over the last 7 days from that list of sites: + +``` +ohio_discharge <- read_USGS_daily(monitoring_location_id = ohio$monitoring_location_id, + parameter_code = "00060", + time = "P7D") +Error in `req_perform()`: +! HTTP 403 Forbidden. +• Query request denied. Possible reasons include query exceeding server limits. +``` + +We could use the fact that the `ohio` data frame contains geospatial information, create a bounding box, and ask for that data like this: + +```{r} +ohio_discharge <- read_USGS_daily(bbox = sf::st_bbox(ohio), + parameter_code = "00060", + time = "P7D") + +``` + +A reasonable `r nrow(ohio_discharge)` are returned with the bounding box query. + +Maybe you have a list of sites that are scattered around the country. The bounding box method might not be ideal. There are several ways to loop through a set of sites, here is one simple example: + +```{r} +big_vector_of_sites <- ohio$monitoring_location_id + +site_list <- split(big_vector_of_sites, ceiling(seq_along(big_vector_of_sites)/200)) + +data_returned <- data.frame() +for(sites in site_list){ + df_sites <- read_USGS_daily(monitoring_location_id = sites, + parameter_code = "00060", + time = "P7D") + if(nrow(df_sites) == 0){ + next + } + data_returned <- rbind(data_returned, df_sites) +} + +``` + +Note there is fewer data returned in `data_returned` because those sites are already filtered down to just "Stream" sites. The bounding box results `ohio_discharge` contained other types of monitoring location types. + +## Result limits + +There's a hard cap at 50,000 rows returned per one request. This means that for 1 `dataRetrieval` request, only 50,000 rows will be returned even if there is more data! If you know you are making a big request, it will be up to you to split up your request into "reasonable" chunks. Note that sometimes you'll notice a big request gets chunked up and you can see that it actually made a bunch of requests - this is done automatically (it's called paging), and the 50,000 limit is still in effect for the total number returned from all the pages. + +## limit vs max_results + +A user can specify a `limit` or `max_results`. + +The `max_results` argument defines how many rows are returned (assuming the data has at least `max_results` rows to return). This can be used as a handy way to make sure you aren't requesting a ton of data, perhaps to do some initial coding or troubleshooting. + +The `limit` argument defines how many rows are returned per page of data, but does NOT affect the overall number of rows returned. With a good internet connection, you can probably get away with ignoring this argument. By default it will be set to the highest value that the services allow. The reason you might want to change this argument is that it might be easier on a spotty internet connection to page through smaller sets of data. + + # New Functions As new API endpoints come online, this section will be updated with any `dataRetrieval` function that is created. From 4379cf77b6896f83b49c8b2d95ad1dadc1086419 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 5 Jun 2025 13:01:50 -0500 Subject: [PATCH 097/117] I think pandoc is already in the base image --- docker/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 740f1e12..404040c3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -26,7 +26,6 @@ RUN apt-get update -qq && apt-get -y --no-install-recommends install \ r-cran-readxl \ r-cran-whisker \ r-cran-ggplot2 \ - && apt-get install -y pandoc \ && rm -rf /var/lib/apt/lists/* From e54f7c26d78ddb8365f1bb51a296f7fd20091978 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 5 Jun 2025 13:52:21 -0500 Subject: [PATCH 098/117] Working pkgdown links --- vignettes/read_USGS_functions.Rmd | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index a748aac8..1c0cf51b 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -379,11 +379,11 @@ Peruse the "Additional Articles", when we find common issues people have with co Currently, you might be interested in: -* [General Tutorial](articles/Tutorial.html) +* [General Tutorial](tutorial.html) -* [Pivot Help](articles/long_to_wide.html) +* [Pivot Help](long_to_wide.html) -* [Joining by closest date](articles/join_by_closest.html) +* [Joining by closest date](join_by_closest.html) If you have additional questions, email comptools@usgs.gov. General questions and bug reports can be reported here: From 8d2fe118688e73b86ef2f23414bca51f9729b547 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 5 Jun 2025 14:19:30 -0500 Subject: [PATCH 099/117] stop testing on ci --- tests/testthat/tests_general.R | 2 +- tests/testthat/tests_userFriendly_fxns.R | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/testthat/tests_general.R b/tests/testthat/tests_general.R index e42d3260..4cd73e48 100644 --- a/tests/testthat/tests_general.R +++ b/tests/testthat/tests_general.R @@ -62,7 +62,7 @@ test_that("General USGS retrievals working", { test_that("General NWIS retrievals working", { testthat::skip_on_cran() - + skip_on_ci() multiSite <- readNWISdata( sites = c("04025500", "040263491"), service = "iv", parameterCd = "00060", diff --git a/tests/testthat/tests_userFriendly_fxns.R b/tests/testthat/tests_userFriendly_fxns.R index 77b7083c..ed36dce7 100644 --- a/tests/testthat/tests_userFriendly_fxns.R +++ b/tests/testthat/tests_userFriendly_fxns.R @@ -2,7 +2,7 @@ context("Unit values") test_that("Unit value data returns correct types", { testthat::skip_on_cran() - + skip_on_ci() siteNumber <- "05114000" parameterCd <- "00060" startDate <- "2014-10-10" @@ -68,7 +68,7 @@ test_that("Unit value data returns correct types", { context("Peak, rating, meas, site") test_that("peak, rating curves, surface-water measurements", { testthat::skip_on_cran() - + skip_on_ci() siteNumbers <- c("01594440", "040851325") data <- readNWISpeak(siteNumbers) expect_is(data$agency_cd, "character") @@ -158,6 +158,7 @@ test_that("read_USGS_daily", { test_that("WQP qw tests", { testthat::skip_on_cran() + skip_on_ci() nameToUse <- "Specific conductance" pcodeToUse <- "00095" @@ -185,6 +186,7 @@ test_that("WQP qw tests", { context("readNWISstat tests") test_that("readNWISstat tests", { testthat::skip_on_cran() + skip_on_ci() data <- readNWISstat( siteNumbers = c("02171500"), parameterCd = c("00010", "00060"), @@ -218,6 +220,7 @@ test_that("readNWISstat tests", { context("readNWISuse tests") test_that("readNWISuse tests", { testthat::skip_on_cran() + skip_on_ci() dc <- readNWISuse( years = c(2000, 2005, 2010), stateCd = "DC", countyCd = NULL From 030b246526bfc09089ed0dbb488dea4aaa361398 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 5 Jun 2025 15:03:44 -0500 Subject: [PATCH 100/117] move this after the property fiddling --- R/construct_api_requests.R | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index 5e4f8639..340a8ace 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -52,11 +52,6 @@ construct_api_requests <- function(service, type = "schema") all_properties <- names(schema$properties) - if(!all(is.na(properties))){ - match.arg(properties, choices = all_properties, - several.ok = TRUE) - } - if(all(all_properties[!all_properties %in% c("id", "geometry")] %in% properties)) { # Cleans up URL if we're asking for everything properties <- NA_character_ @@ -69,6 +64,11 @@ construct_api_requests <- function(service, } } + if(!all(is.na(properties))){ + match.arg(properties, choices = all_properties, + several.ok = TRUE) + } + baseURL <- setup_api(service) POST <- FALSE From 95746570ffc302d34660bef4509888bbc2d8db45 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 6 Jun 2025 09:57:39 -0500 Subject: [PATCH 101/117] Add a function to deal with id crazyness --- R/construct_api_requests.R | 126 ++++++++++++++++++++++----- R/read_USGS_daily.R | 20 +++-- R/read_USGS_monitoring_location.R | 22 +++-- R/read_USGS_ts_meta.R | 22 +++-- R/walk_pages.R | 29 +++--- man/read_USGS_monitoring_location.Rd | 2 - man/read_USGS_ts_meta.Rd | 4 +- 7 files changed, 164 insertions(+), 61 deletions(-) diff --git a/R/construct_api_requests.R b/R/construct_api_requests.R index 340a8ace..3d2279f2 100644 --- a/R/construct_api_requests.R +++ b/R/construct_api_requests.R @@ -48,27 +48,6 @@ construct_api_requests <- function(service, skipGeometry = FALSE, ...){ - schema <- check_OGC_requests(endpoint = service, - type = "schema") - all_properties <- names(schema$properties) - - if(all(all_properties[!all_properties %in% c("id", "geometry")] %in% properties)) { - # Cleans up URL if we're asking for everything - properties <- NA_character_ - } else { - if(all(!is.na(properties))){ - properties <- gsub("-", "_", properties) - properties <- properties[!properties %in% c("id", - "geometry", - paste0(gsub("-", "_", service), "_id"))] - } - } - - if(!all(is.na(properties))){ - match.arg(properties, choices = all_properties, - several.ok = TRUE) - } - baseURL <- setup_api(service) POST <- FALSE @@ -184,6 +163,111 @@ setup_api <- function(service){ } +#' Switch endpoint id arg +#' +#' @noRd +#' @return list +#' @examples +#' +#' l1 <- list("id" = "1234") +#' dataRetrieval:::switch_arg_id(l1, +#' id_name = "monitoring_location_id", +#' service = "monitoring-locations") +#' +#' l2 <- list("monitoring_location_id" = "1234") +#' dataRetrieval:::switch_arg_id(l2, +#' id_name = "monitoring_location_id", +#' service = "monitoring-locations") +#' +#' l3 <- list("monitoring_locations_id" = "1234") +#' dataRetrieval:::switch_arg_id(l3, +#' id_name = "monitoring_location_id", +#' service = "monitoring-locations") +#' +switch_arg_id <- function(ls, id_name, service){ + + service_id <- paste0(gsub("-", "_", service), "_id") + if(!"id" %in% names(ls)){ + if(service_id %in% names(ls)){ + ls[["id"]] <- ls[[service_id]] + } else { + ls[["id"]] <- ls[[id_name]] + } + } + + ls[[service_id]] <- NULL + ls[[id_name]] <- NULL + return(ls) +} + +#' Switch properties id +#' +#' @noRd +#' @return list +#' @examples +#' +#' properties <- c("id", "state_name", "country_name") +#' dataRetrieval:::switch_properties_id(properties, +#' id_name = "monitoring_location_id", +#' service = "monitoring-locations") +#' +#' properties2 <- c("monitoring_location_id", "state_name", "country_name") +#' dataRetrieval:::switch_properties_id(properties2, +#' id_name = "monitoring_location_id", +#' service = "monitoring-locations") +#' +#' properties3 <- c("monitoring_locations_id", "state_name", "country_name") +#' dataRetrieval:::switch_properties_id(properties3, +#' id_name = "monitoring_location_id", +#' service = "monitoring-locations") +switch_properties_id <- function(properties, id_name, service){ + + service_id <- paste0(gsub("-", "_", service), "_id") + + last_letter <- substr(service, nchar(service), nchar(service)) + if(last_letter == "s"){ + service_singluar <- substr(service,1, nchar(service)-1) + service_id_singular <- paste0(gsub("-", "_", service_singluar), "_id") + } else { + service_id_singular <- "" + } + + if(!"id" %in% properties){ + if(service_id %in% properties){ + properties[properties == service_id] <- "id" + + } else if(service_id_singular %in% properties) { + properties[properties == service_id_singular] <- "id" + } else { + properties[properties == id_name] <- "id" + } + } + + schema <- check_OGC_requests(endpoint = service, + type = "schema") + all_properties <- names(schema$properties) + + if(all(all_properties[!all_properties %in% c("id", "geometry")] %in% properties)) { + # Cleans up URL if we're asking for everything + properties <- NA_character_ + } else { + if(all(!is.na(properties))){ + properties <- gsub("-", "_", properties) + properties <- properties[!properties %in% c("id", + "geometry", + paste0(gsub("-", "_", service), "_id"))] + } + } + + if(!all(is.na(properties))){ + match.arg(properties, choices = all_properties, + several.ok = TRUE) + } + + return(properties) +} + + #' Format the date request #' #' Users will want to give either start/end dates or diff --git a/R/read_USGS_daily.R b/R/read_USGS_daily.R index 371973c2..87947a84 100644 --- a/R/read_USGS_daily.R +++ b/R/read_USGS_daily.R @@ -87,11 +87,21 @@ read_USGS_daily <- function(monitoring_location_id = NA_character_, message("Function in development, use at your own risk.") service <- "daily" + output_id <- "daily_id" + args <- mget(names(formals())) - args[["id"]] <- args[["daily_id"]] - args[["daily_id"]] <- NULL - args[["convertType"]] <- NULL args[["service"]] <- service + + args <- switch_arg_id(args, + id_name = output_id, + service = service) + + args[["properties"]] <- switch_properties_id(properties, + id_name = output_id, + service = service) + + args[["convertType"]] <- NULL + dv_req <- do.call(construct_api_requests, args) return_list <- walk_pages(dv_req, max_results) @@ -100,11 +110,11 @@ read_USGS_daily <- function(monitoring_location_id = NA_character_, if(convertType) return_list <- cleanup_cols(return_list, service = "daily") + + return_list <- rejigger_cols(return_list, properties, output_id) return_list <- return_list[order(return_list$time, return_list$monitoring_location_id), ] - return_list <- rejigger_cols(return_list, properties, service) - return(return_list) } diff --git a/R/read_USGS_monitoring_location.R b/R/read_USGS_monitoring_location.R index eb9fc5a8..cb0dba35 100644 --- a/R/read_USGS_monitoring_location.R +++ b/R/read_USGS_monitoring_location.R @@ -82,8 +82,6 @@ #' site_info_no_sf <- read_USGS_monitoring_location(monitoring_location_id = site, #' skipGeometry = TRUE) #' -#' multi_site <- read_USGS_monitoring_location(state_name = "Wisconsin") -#' #' bbox_vals = c(-94.00, 35.0, -93.5, 35.5) #' multi_site <- read_USGS_monitoring_location(bbox = bbox_vals) #' multi_site_n_100 <- read_USGS_monitoring_location(bbox = bbox_vals, @@ -140,16 +138,18 @@ read_USGS_monitoring_location <- function(monitoring_location_id = NA_character_ message("Function in development, use at your own risk.") service <- "monitoring-locations" + output_id <- "monitoring_location_id" args <- mget(names(formals())) args[["service"]] <- service - args[["id"]] <- args[["monitoring_location_id"]] - args[["monitoring_location_id"]] <- NULL - # help with monitoring_location(s) confusion - # User can put in either in the property vector: - properties[properties == "monitoring_location_id"] <- "monitoring_locations_id" - args[["properties"]][args[["properties"]] == "monitoring_location_id"] <- "monitoring_locations_id" + args <- switch_arg_id(args, + id_name = output_id, + service = service) + + args[["properties"]] <- switch_properties_id(properties, + id_name = output_id, + service = service) site_req <- do.call(construct_api_requests, args) @@ -157,9 +157,7 @@ read_USGS_monitoring_location <- function(monitoring_location_id = NA_character_ return_list <- deal_with_empty(return_list, properties, service) - return_list <- rejigger_cols(return_list, properties, service) - # They will get back the singular though, to work with the other services. - names(return_list)[(names(return_list) == "monitoring_locations_id")] <- "monitoring_location_id" - + return_list <- rejigger_cols(return_list, properties, output_id) + return(return_list) } diff --git a/R/read_USGS_ts_meta.R b/R/read_USGS_ts_meta.R index 8f876f6c..e2f7efd2 100644 --- a/R/read_USGS_ts_meta.R +++ b/R/read_USGS_ts_meta.R @@ -20,7 +20,7 @@ #' @param properties A vector of requested columns to be returned from the query. #' Available options are: #' `r schema <- check_OGC_requests(endpoint = "time-series-metadata", type = "schema"); paste(names(schema$properties), collapse = ", ")` -#' @param time_series_metadata_id `r get_params("time-series-metadata")$id` +#' @param time_series_id `r get_params("time-series-metadata")$id` #' @param bbox Only features that have a geometry that intersects the bounding #' box are selected.The bounding box is provided as four or six numbers, depending #' on whether the coordinate reference system includes a vertical axis (height or @@ -69,7 +69,7 @@ read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, thresholds = NA, sublocation_identifier = NA_character_, primary = NA_character_, - time_series_metadata_id = NA_character_, + time_series_id = NA_character_, web_description = NA_character_, skipGeometry = NA, limit = NA, @@ -80,13 +80,19 @@ read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, message("Function in development, use at your own risk.") service = "time-series-metadata" + output_id <- "time_series_id" args <- mget(names(formals())) - args[["service"]] <- service - args[["id"]] <- args[["time_series_id"]] - args[["time_series_metadata_id"]] <- NULL + + args <- switch_arg_id(args, id_name = output_id, service = service) + args[["convertType"]] <- NULL + + args[["properties"]] <- switch_properties_id(properties, + id_name = output_id, + service = service) + req_ts_meta <- do.call(construct_api_requests, args) return_list <- walk_pages(req_ts_meta, max_results) @@ -94,10 +100,8 @@ read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, return_list <- deal_with_empty(return_list, properties, service) if(convertType) return_list <- cleanup_cols(return_list) - - return_list <- rejigger_cols(return_list, properties, service) - - names(return_list)[(names(return_list) == "time_series_metadata_id")] <- "time_series_id" + + return_list <- rejigger_cols(return_list, properties, output_id) return(return_list) diff --git a/R/walk_pages.R b/R/walk_pages.R index 031a08d4..6d19cc4e 100644 --- a/R/walk_pages.R +++ b/R/walk_pages.R @@ -46,19 +46,28 @@ deal_with_empty <- function(return_list, properties, service){ #' @examples #' #' df <- dataRetrieval:::deal_with_empty(data.frame(NULL), -#' properties = c("time", "value", "id"), -#' service = "daily") +#' properties = c("state_code", "county_code", "id"), +#' service = "monitoring-locations") #' df2 <- dataRetrieval:::rejigger_cols(df, -#' properties = c("value", "id", "time"), -#' service = "daily") -#' -rejigger_cols <- function(df, properties, service){ - new_id <- paste0(gsub("-", "_", service), "_id") - names(df)[names(df) == "id"] <- new_id - +#' properties = c("state_code", "id", "county_code"), +#' output_id = "monitoring_location_id") +#' +#' df3 <- dataRetrieval:::rejigger_cols(df, +#' properties = c("state_code", "monitoring_location_id", "county_code"), +#' output_id = "monitoring_location_id") +#' +rejigger_cols <- function(df, properties, output_id){ + if(!all(is.na(properties))){ - properties[properties == "id"] <- new_id + + if(!"id" %in% properties){ + names(df)[(names(df) == "id")] <- output_id + properties[properties == "id"] <- output_id + } + df <- df[, properties] + } else { + names(df)[(names(df) == "id")] <- output_id } df } diff --git a/man/read_USGS_monitoring_location.Rd b/man/read_USGS_monitoring_location.Rd index f8be4bad..58d3fa69 100644 --- a/man/read_USGS_monitoring_location.Rd +++ b/man/read_USGS_monitoring_location.Rd @@ -181,8 +181,6 @@ site_slim_no_sf_slim <- read_USGS_monitoring_location(monitoring_location_id = s site_info_no_sf <- read_USGS_monitoring_location(monitoring_location_id = site, skipGeometry = TRUE) -multi_site <- read_USGS_monitoring_location(state_name = "Wisconsin") - bbox_vals = c(-94.00, 35.0, -93.5, 35.5) multi_site <- read_USGS_monitoring_location(bbox = bbox_vals) multi_site_n_100 <- read_USGS_monitoring_location(bbox = bbox_vals, diff --git a/man/read_USGS_ts_meta.Rd b/man/read_USGS_ts_meta.Rd index e2825bd6..9a4e7387 100644 --- a/man/read_USGS_ts_meta.Rd +++ b/man/read_USGS_ts_meta.Rd @@ -19,7 +19,7 @@ read_USGS_ts_meta( thresholds = NA, sublocation_identifier = NA_character_, primary = NA_character_, - time_series_metadata_id = NA_character_, + time_series_id = NA_character_, web_description = NA_character_, skipGeometry = NA, limit = NA, @@ -69,7 +69,7 @@ Only features that have a \code{last_modified} that intersects the value of date \item{primary}{A flag identifying if the time series is a "primary" time series. "Primary" time series (which have this flag) are standard observations which undergo \href{https://www.usgs.gov/survey-manual/5028-fundamental-science-practices-review-and-approval-scientific-data-release}{Bureau review and approval processes}. Non-primary time series, which will have missing values for "primary", are provisional datasets made available to meet the need for timely best science and to assist with daily operations which need real-time information. Non-primary time series data are only retained by this system for 120 days. See the \href{https://waterdata.usgs.gov/provisional-data-statement/}{USGS Provisional Data Statement} for more information.} -\item{time_series_metadata_id}{A unique identifier representing a single time series. This corresponds to the \code{id} field in the \code{time-series-metadata} endpoint.} +\item{time_series_id}{A unique identifier representing a single time series. This corresponds to the \code{id} field in the \code{time-series-metadata} endpoint.} \item{web_description}{A description of what this time series represents, as used by WDFN and other USGS data dissemination products.} From 4cb2b6a66be5b273a090ade4fe28f7ce0e0c7653 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 6 Jun 2025 10:26:47 -0500 Subject: [PATCH 102/117] plural --- R/walk_pages.R | 14 +++++++--- vignettes/read_USGS_functions.Rmd | 43 ++++++++++++++++++++++++++++--- 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/R/walk_pages.R b/R/walk_pages.R index 6d19cc4e..8a2d8ed9 100644 --- a/R/walk_pages.R +++ b/R/walk_pages.R @@ -59,12 +59,18 @@ deal_with_empty <- function(return_list, properties, service){ rejigger_cols <- function(df, properties, output_id){ if(!all(is.na(properties))){ - if(!"id" %in% properties){ - names(df)[(names(df) == "id")] <- output_id - properties[properties == "id"] <- output_id + if(output_id %in% properties){ + names(df)[(names(df) == "id")] <- output_id + } else { + # just in case users become aware of the singular/plural issue + # where the endpoint name is plural, but input to other endpoints are singular + plural <- gsub("_id", "s_id", output_id) + if(plural %in% properties){ + names(df)[(names(df) == "id")] <- plural + } + } } - df <- df[, properties] } else { names(df)[(names(df) == "id")] <- output_id diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_USGS_functions.Rmd index 1c0cf51b..d1eb03bc 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_USGS_functions.Rmd @@ -66,9 +66,9 @@ API_USGS_PAT = "my_super_secret_token" ``` You can use `usethis::edit_r_environ()` to edit find and open your .Renviron file. You will need to restart R for that variable to be recognized. You should not add this file to git projects or generally share your API key. Anyone else using your API key will count against the number of requests available to you! -## Contextual Query Language 2 Support +## Contextual Query Language Support -Supports [Contextual Query Language](https://www.loc.gov/standards/sru/cql/) (CQL2) syntax for flexible queries. We'll show how to use the `read_USGS_data` function to make specific queries using Contextual Query Language (2). +Supports [Contextual Query Language](https://www.loc.gov/standards/sru/cql/) (CQL2) syntax for flexible queries. We'll show how to use the `read_USGS_data` function to make specific CQL2 queries. ## Simple Features @@ -148,6 +148,41 @@ The `max_results` argument defines how many rows are returned (assuming the data The `limit` argument defines how many rows are returned per page of data, but does NOT affect the overall number of rows returned. With a good internet connection, you can probably get away with ignoring this argument. By default it will be set to the highest value that the services allow. The reason you might want to change this argument is that it might be easier on a spotty internet connection to page through smaller sets of data. +## id + +Each API endpoint natively returns a column named "id". The results of the "id" column can be used as inputs into other endpoints, **HOWEVER** the input in those functions have different names. For example, the "id" column of the monitoring location endpoint is considered the "monitoring_location_id" when used as an input to any of the other functions. + +Therefore, `dataRetrieval` functions will rename the "id" column to whatever it is referred to in other functions. Here are the id translations: + +```{r echo=FALSE} +df <- dplyr::tibble(Function = c("read_USGS_monitoring_location", + "read_USGS_ts_meta", + "read_USGS_daily"), + "ID returned" = c("monitoring_location_id", + "time_series_id", + "daily_id")) + +knitr::kable(df) +``` + +If a user would prefer the columns to come back as "id", they can specify that using the `properties` argument: + +```{r} +site <- "USGS-02238500" + +site_1 <- read_USGS_monitoring_location(monitoring_location_id = site, + properties = c("monitoring_location_id", + "state_name", + "country_name")) +names(site_1) +site_2 <- read_USGS_monitoring_location(monitoring_location_id = site, + properties = c("id", + "state_name", + "country_name")) +names(site_2) + + +``` # New Functions @@ -177,7 +212,7 @@ Maybe that is more information than you need. You can specify which columns get ```{r} site_info <- read_USGS_monitoring_location(monitoring_location_id = "USGS-01491000", - properties = c("monitoring_locations_id", + properties = c("monitoring_location_id", "site_type", "drainage_area", "monitoring_location_name")) @@ -375,7 +410,7 @@ That's a lot of new information and changes. There are certainly going to be scr Check back on the documentation often: -Peruse the "Additional Articles", when we find common issues people have with converting their old workflows, we will try to add articles to clarrify new workflows. +Peruse the "Additional Articles", when we find common issues people have with converting their old workflows, we will try to add articles to clarify new workflows. Currently, you might be interested in: From 5cc4e05cb91e678ebb5d5173ff2109ef61110572 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 12 Jun 2025 08:49:27 -0500 Subject: [PATCH 103/117] Allow character urls --- R/importRDB1.R | 5 +++++ R/importWQP.R | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/R/importRDB1.R b/R/importRDB1.R index 1ca2476f..0fcdfe4f 100644 --- a/R/importRDB1.R +++ b/R/importRDB1.R @@ -99,6 +99,11 @@ importRDB1 <- function(obs_url, tz <- match.arg(tz, OlsonNames()) + if (is.character(obs_url) && + grepl("(https)://[^ /$.?#].[^\\s]*", obs_url)){ + obs_url <- httr2::request(obs_url) + } + if(inherits(obs_url, "httr2_request")){ doc <- getWebServiceData(obs_url) diff --git a/R/importWQP.R b/R/importWQP.R index a8cd9793..570f5131 100644 --- a/R/importWQP.R +++ b/R/importWQP.R @@ -45,18 +45,24 @@ importWQP <- function(obs_url, tz = "UTC", tz <- "UTC" } + if (is.character(obs_url) && + grepl("(https)://[^ /$.?#].[^\\s]*", obs_url)){ + obs_url <- httr2::request(obs_url) + } + if (inherits(obs_url, "httr2_request")) { doc <- getWebServiceData(obs_url) if (is.null(doc)) { return(invisible(NULL)) } headerInfo <- attr(doc, "headerInfo") - + } else { doc <- obs_url } last_chars <- as.character(substr(doc, nchar(doc)-1, nchar(doc))) + if(last_chars != c("\n")){ doc <- paste0(doc, "\n") } From dee77cf3ab09f4f5f05db362e6bac08931aa3917 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 12 Jun 2025 08:57:30 -0500 Subject: [PATCH 104/117] chickening out on removing all deprecated function examples. --- R/readNWISdv.R | 7 +++++++ R/readNWISsite.R | 1 + R/whatNWISdata.R | 10 ++++++++++ R/whatNWISsites.R | 2 ++ man/readNWISdv.Rd | 7 +++++++ man/readNWISsite.Rd | 1 + man/whatNWISdata.Rd | 10 ++++++++++ man/whatNWISsites.Rd | 2 ++ 8 files changed, 40 insertions(+) diff --git a/R/readNWISdv.R b/R/readNWISdv.R index 6f4564e4..acd70335 100644 --- a/R/readNWISdv.R +++ b/R/readNWISdv.R @@ -56,6 +56,13 @@ #' #' # see ?read_USGS_daily #' +#' #site_id <- "04085427" +#' #startDate <- "2012-01-01" +#' #endDate <- "2012-06-30" +#' #pCode <- "00060" +#' # +#' #rawDailyQ <- readNWISdv(site_id, pCode, startDate, endDate) +#' readNWISdv <- function(siteNumbers, parameterCd, startDate = "", diff --git a/R/readNWISsite.R b/R/readNWISsite.R index e8d44160..722b512f 100644 --- a/R/readNWISsite.R +++ b/R/readNWISsite.R @@ -63,6 +63,7 @@ #' @examples #' #' # see ?read_USGS_monitoring_location +#' # siteINFOMulti <- readNWISsite(c("05114000", "09423350")) #' readNWISsite <- function(siteNumbers) { diff --git a/R/whatNWISdata.R b/R/whatNWISdata.R index 8a5274dd..13e29f76 100644 --- a/R/whatNWISdata.R +++ b/R/whatNWISdata.R @@ -68,6 +68,16 @@ #' #' # see ?read_USGS_ts_meta #' +#' #site1 <- whatWQPsamples(siteid = "USGS-01594440") +#' +#' #type <- "Stream" +#' +#' #sites <- whatWQPsamples(countycode = "US:55:025", siteType = type) +#' +#' #lakeSites_samples <- whatWQPsamples(siteType = "Lake, Reservoir, Impoundment", +#' # countycode = "US:55:025") +#' +#' whatNWISdata <- function(..., convertType = TRUE) { matchReturn <- convertLists(...) diff --git a/R/whatNWISsites.R b/R/whatNWISsites.R index c6d7df08..cb15ce73 100644 --- a/R/whatNWISsites.R +++ b/R/whatNWISsites.R @@ -31,6 +31,8 @@ #' @examples #' #' # see ?read_USGS_monitoring_location +#' #siteListPhos <- whatNWISsites(stateCd = "OH", parameterCd = "00665") +#' #oneSite <- whatNWISsites(sites = "05114000") #' whatNWISsites <- function(...) { diff --git a/man/readNWISdv.Rd b/man/readNWISdv.Rd index 2972328d..83d36b8b 100644 --- a/man/readNWISdv.Rd +++ b/man/readNWISdv.Rd @@ -74,6 +74,13 @@ More information on the web service can be found here: # see ?read_USGS_daily +#site_id <- "04085427" +#startDate <- "2012-01-01" +#endDate <- "2012-06-30" +#pCode <- "00060" +# +#rawDailyQ <- readNWISdv(site_id, pCode, startDate, endDate) + } \seealso{ \code{\link[=read_USGS_daily]{read_USGS_daily()}} diff --git a/man/readNWISsite.Rd b/man/readNWISsite.Rd index 0c0895d5..f38434cf 100644 --- a/man/readNWISsite.Rd +++ b/man/readNWISsite.Rd @@ -71,6 +71,7 @@ Imports data from USGS site file site. This function gets data from here: \url{h \examples{ # see ?read_USGS_monitoring_location +# siteINFOMulti <- readNWISsite(c("05114000", "09423350")) } \seealso{ diff --git a/man/whatNWISdata.Rd b/man/whatNWISdata.Rd index 8ff2f31b..a27c1a35 100644 --- a/man/whatNWISdata.Rd +++ b/man/whatNWISdata.Rd @@ -76,6 +76,16 @@ of sites that have useful data. # see ?read_USGS_ts_meta +#site1 <- whatWQPsamples(siteid = "USGS-01594440") + +#type <- "Stream" + +#sites <- whatWQPsamples(countycode = "US:55:025", siteType = type) + +#lakeSites_samples <- whatWQPsamples(siteType = "Lake, Reservoir, Impoundment", +# countycode = "US:55:025") + + } \seealso{ \code{\link[=read_USGS_ts_meta]{read_USGS_ts_meta()}} diff --git a/man/whatNWISsites.Rd b/man/whatNWISsites.Rd index 89d8cf3b..3f580bac 100644 --- a/man/whatNWISsites.Rd +++ b/man/whatNWISsites.Rd @@ -38,6 +38,8 @@ Mapper format is used \examples{ # see ?read_USGS_monitoring_location +#siteListPhos <- whatNWISsites(stateCd = "OH", parameterCd = "00665") +#oneSite <- whatNWISsites(sites = "05114000") } \seealso{ From 1e405d668ea6e07757282345453bf0346332bb1d Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 12 Jun 2025 15:44:21 -0500 Subject: [PATCH 105/117] Getting ready for CRAN --- DESCRIPTION | 2 +- R/readNWISpCode.R | 6 +++--- R/read_USGS_samples.R | 4 ++-- inst/CITATION | 26 ++++++++++++++++++-------- man/construct_USGS_sample_request.Rd | 2 +- man/read_USGS_samples.Rd | 2 +- 6 files changed, 26 insertions(+), 16 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index e3908ca8..1bdf2b4a 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: dataRetrieval Type: Package Title: Retrieval Functions for USGS and EPA Hydrology and Water Quality Data -Version: 2.7.18.9002 +Version: 2.7.19 Authors@R: c( person("Laura", "DeCicco", role = c("aut","cre"), email = "ldecicco@usgs.gov", diff --git a/R/readNWISpCode.R b/R/readNWISpCode.R index 065d5a5e..6eb2f80c 100644 --- a/R/readNWISpCode.R +++ b/R/readNWISpCode.R @@ -3,9 +3,9 @@ #' Imports data from NWIS about measured parameter based on user-supplied parameter code or codes. #' This function gets the data from here: #' -#' @param parameterCd character of USGS parameter codes (or multiple parameter codes). These are 5 digit number codes, -#' more information can be found here: . To get a -#' complete list of all current parameter codes in the USGS, use "all" as the input. +#' @param parameterCd character of USGS parameter codes (or multiple parameter codes). +#' These are 5 digit number codes. To get a complete list of all current parameter +#' codes in the USGS, use "all" as the input. #' @keywords data import USGS web service #' @return parameterData data frame with the following information: #' \tabular{lll}{ diff --git a/R/read_USGS_samples.R b/R/read_USGS_samples.R index 49d92350..032955b0 100644 --- a/R/read_USGS_samples.R +++ b/R/read_USGS_samples.R @@ -1,7 +1,7 @@ #' Construct request for USGS Samples Data #' #' This function creates the call for discrete water quality samples data -#' service described at . +#' service described at . #' Note: all possible arguments are included, but it is strongly recommended #' to only use the NECESSARY arguments. Leave unnecessary arguments as the default #' NA. @@ -379,7 +379,7 @@ check_USGS_sample_params <- function(service = "characteristicgroup", #' USGS Samples Data #' #' This function creates the call and gets the data for discrete water quality samples data -#' service described at . +#' service described at . #' #' @inheritParams construct_USGS_sample_request #' @param tz character to set timezone attribute of datetime. Default is UTC diff --git a/inst/CITATION b/inst/CITATION index dd9cf7c1..0a3803b3 100644 --- a/inst/CITATION +++ b/inst/CITATION @@ -6,32 +6,42 @@ bibentry(bibtype = "Manual", email = "ldecicco@usgs.gov", comment=c(ORCID="0000-0002-3915-9487")), person("Robert", "Hirsch", role = c("aut"), - comment=c(ORCID="0000-0002-4534-075X")), + comment=c(ORCID = "0000-0002-4534-075X")), person("David","Lorenz", role=c("aut")), person("Jordan", "Read", role = c("ctb")), person("Jordan", "Walker", role = c("ctb")), person("Lindsay","Platt", role=c("ctb")), person("David","Watkins", role=c("aut"), email = "wwatkins@usgs.gov", - comment=c(ORCID="0000-0002-7544-0700")), + comment=c(ORCID = "0000-0002-7544-0700")), person("David", "Blodgett", role="aut", - comment=c(ORCID="0000-0001-9489-1710"), + comment=c(ORCID = "0000-0001-9489-1710"), email = "dblodgett@usgs.gov"), person("Mike", "Johnson", role=c("aut"), email = "mikecp11@gmail.com", - comment=c(ORCID="0000-0002-5288-8350")), + comment=c(ORCID = "0000-0002-5288-8350")), person("Aliesha", "Krall", role="ctb", email = "akrall@usgs.gov", - comment=c(ORCID="0000-0003-2521-5043")), + comment=c(ORCID = "0000-0003-2521-5043")), person("Lee", "Stanish", role="ctb", email = "lstanish@usgs.gov", - comment=c(ORCID = "0000-0002-9775-6861"))), + comment=c(ORCID = "0000-0002-9775-6861")), + person("Joeseph", "Zemmels", role="ctb", + email = "jzemmels@usgs.gov", + comment=c(ORCID = "0009-0008-1463-6313")), + person("Elise", "Hinman", role="ctb", + email = "ehinman@usgs.gov", + comment=c(ORCID = "0000-0001-5396-1583")), + person("Michael", "Mahoney", role="ctb", + email = "mjmahoney@usgs.gov", + comment=c(ORCID = "0000-0003-2402-304X")) + ), title = "dataRetrieval: R packages for discovering and retrieving water data available from U.S. federal hydrologic web services", publisher = "U.S. Geological Survey", address="Reston, VA", - version = "2.7.18", + version = "2.7.19", institution = "U.S. Geological Survey", year = 2025, doi = "10.5066/P9X4L3GE", - textVersion = "De Cicco, L.A., Hirsch, R.M., Lorenz, D., Watkins, W.D., Johnson, M., 2025, dataRetrieval: R packages for discovering and retrieving water data available from Federal hydrologic web services, v.2.7.18, doi:10.5066/P9X4L3GE" + textVersion = "De Cicco, L.A., Hirsch, R.M., Lorenz, D., Watkins, W.D., Johnson, M., 2025, dataRetrieval: R packages for discovering and retrieving water data available from Federal hydrologic web services, v.2.7.19, doi:10.5066/P9X4L3GE" ) diff --git a/man/construct_USGS_sample_request.Rd b/man/construct_USGS_sample_request.Rd index 896f7375..d3b89415 100644 --- a/man/construct_USGS_sample_request.Rd +++ b/man/construct_USGS_sample_request.Rd @@ -130,7 +130,7 @@ data frame returned from web service call. } \description{ This function creates the call for discrete water quality samples data -service described at \url{https://waterdata.usgs.gov/download-samples}. +service described at \url{https://waterdata.usgs.gov/download-samples/}. Note: all possible arguments are included, but it is strongly recommended to only use the NECESSARY arguments. Leave unnecessary arguments as the default NA. diff --git a/man/read_USGS_samples.Rd b/man/read_USGS_samples.Rd index 2695a24b..0af207d2 100644 --- a/man/read_USGS_samples.Rd +++ b/man/read_USGS_samples.Rd @@ -134,7 +134,7 @@ Possible values include "America/New_York","America/Chicago", "America/Denver"," } \description{ This function creates the call and gets the data for discrete water quality samples data -service described at \url{https://waterdata.usgs.gov/download-samples}. +service described at \url{https://waterdata.usgs.gov/download-samples/}. } \examples{ \dontshow{if (is_dataRetrieval_user()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} From 068e7bc6d0c3da20777f2f18324f7fc376941349 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Thu, 12 Jun 2025 16:09:58 -0500 Subject: [PATCH 106/117] lose the scary message --- R/read_USGS_daily.R | 2 -- R/read_USGS_data.R | 2 -- R/read_USGS_monitoring_location.R | 4 +--- R/read_USGS_samples.R | 3 +-- R/read_USGS_ts_meta.R | 4 +--- man/readNWISpCode.Rd | 6 +++--- 6 files changed, 6 insertions(+), 15 deletions(-) diff --git a/R/read_USGS_daily.R b/R/read_USGS_daily.R index 87947a84..4e92fa74 100644 --- a/R/read_USGS_daily.R +++ b/R/read_USGS_daily.R @@ -84,8 +84,6 @@ read_USGS_daily <- function(monitoring_location_id = NA_character_, max_results = NA, convertType = TRUE){ - message("Function in development, use at your own risk.") - service <- "daily" output_id <- "daily_id" diff --git a/R/read_USGS_data.R b/R/read_USGS_data.R index e8f52c49..935db0c9 100644 --- a/R/read_USGS_data.R +++ b/R/read_USGS_data.R @@ -43,8 +43,6 @@ read_USGS_data <- function(service, CQL, ..., convertType = TRUE){ - - message("Function in development, use at your own risk.") query_req <- get_collection() diff --git a/R/read_USGS_monitoring_location.R b/R/read_USGS_monitoring_location.R index cb0dba35..f46998c7 100644 --- a/R/read_USGS_monitoring_location.R +++ b/R/read_USGS_monitoring_location.R @@ -134,9 +134,7 @@ read_USGS_monitoring_location <- function(monitoring_location_id = NA_character_ limit = NA, max_results = NA, skipGeometry = NA){ - - message("Function in development, use at your own risk.") - + service <- "monitoring-locations" output_id <- "monitoring_location_id" diff --git a/R/read_USGS_samples.R b/R/read_USGS_samples.R index 032955b0..f10febd7 100644 --- a/R/read_USGS_samples.R +++ b/R/read_USGS_samples.R @@ -484,8 +484,7 @@ read_USGS_samples <- function(monitoringLocationIdentifier = NA, #' #' } summarize_USGS_samples <- function(monitoringLocationIdentifier){ - message("Function in development, use at your own risk.") - + if(length(monitoringLocationIdentifier) > 1){ stop("Summary service only available for one site at a time.") } diff --git a/R/read_USGS_ts_meta.R b/R/read_USGS_ts_meta.R index e2f7efd2..1a9ef7db 100644 --- a/R/read_USGS_ts_meta.R +++ b/R/read_USGS_ts_meta.R @@ -76,9 +76,7 @@ read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, max_results = NA, bbox = NA, convertType = FALSE){ - - message("Function in development, use at your own risk.") - + service = "time-series-metadata" output_id <- "time_series_id" diff --git a/man/readNWISpCode.Rd b/man/readNWISpCode.Rd index 8b5887cb..ebeef88b 100644 --- a/man/readNWISpCode.Rd +++ b/man/readNWISpCode.Rd @@ -7,9 +7,9 @@ readNWISpCode(parameterCd) } \arguments{ -\item{parameterCd}{character of USGS parameter codes (or multiple parameter codes). These are 5 digit number codes, -more information can be found here: \url{https://help.waterdata.usgs.gov/}. To get a -complete list of all current parameter codes in the USGS, use "all" as the input.} +\item{parameterCd}{character of USGS parameter codes (or multiple parameter codes). +These are 5 digit number codes. To get a complete list of all current parameter +codes in the USGS, use "all" as the input.} } \value{ parameterData data frame with the following information: From 7dae1e2ab166ac962b0668db0cb0b08d88f52959 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 13 Jun 2025 10:32:52 -0500 Subject: [PATCH 107/117] NAME CHANGE! --- .github/workflows/R-CMD-check.yaml | 1 + .github/workflows/pkgdown.yaml | 1 + .github/workflows/test-coverage.yaml | 1 + NAMESPACE | 10 ++-- R/readNWISdata.R | 10 ++-- R/readNWISdv.R | 8 +-- R/readNWISsite.R | 8 +-- R/{read_USGS_data.R => read_waterdata.R} | 6 +-- ...ad_USGS_daily.R => read_waterdata_daily.R} | 12 ++--- ...R => read_waterdata_monitoring_location.R} | 16 +++--- ...SGS_samples.R => read_waterdata_samples.R} | 10 ++-- ...SGS_ts_meta.R => read_waterdata_ts_meta.R} | 6 +-- man/check_USGS_sample_params.Rd | 2 +- man/construct_USGS_sample_request.Rd | 2 +- man/readNWISdata.Rd | 2 +- man/readNWISdv.Rd | 4 +- man/readNWISsite.Rd | 4 +- man/{read_USGS_data.Rd => read_waterdata.Rd} | 12 ++--- ..._USGS_daily.Rd => read_waterdata_daily.Rd} | 18 +++---- ... => read_waterdata_monitoring_location.Rd} | 22 ++++---- ...S_samples.Rd => read_waterdata_samples.Rd} | 14 ++--- ...S_ts_meta.Rd => read_waterdata_ts_meta.Rd} | 12 ++--- man/summarize_USGS_samples.Rd | 2 +- tests/testthat/tests_general.R | 50 +++++++++--------- tests/testthat/tests_samples.R | 30 +++++------ vignettes/Status.Rmd | 18 +++---- vignettes/long_to_wide.Rmd | 2 +- ...tions.Rmd => read_waterdata_functions.Rmd} | 52 +++++++++---------- vignettes/samples_data.Rmd | 26 +++++----- vignettes/tutorial.Rmd | 42 +++++++-------- 30 files changed, 203 insertions(+), 200 deletions(-) rename R/{read_USGS_data.R => read_waterdata.R} (94%) rename R/{read_USGS_daily.R => read_waterdata_daily.R} (92%) rename R/{read_USGS_monitoring_location.R => read_waterdata_monitoring_location.R} (93%) rename R/{read_USGS_samples.R => read_waterdata_samples.R} (98%) rename R/{read_USGS_ts_meta.R => read_waterdata_ts_meta.R} (95%) rename man/{read_USGS_data.Rd => read_waterdata.Rd} (82%) rename man/{read_USGS_daily.Rd => read_waterdata_daily.Rd} (94%) rename man/{read_USGS_monitoring_location.Rd => read_waterdata_monitoring_location.Rd} (95%) rename man/{read_USGS_samples.Rd => read_waterdata_samples.Rd} (95%) rename man/{read_USGS_ts_meta.Rd => read_waterdata_ts_meta.Rd} (96%) rename vignettes/{read_USGS_functions.Rmd => read_waterdata_functions.Rmd} (84%) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index b6aa1e55..15254f9b 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -24,6 +24,7 @@ jobs: env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + API_USGS_PAT: ${{ secrets.API_USGS_PAT }} R_KEEP_PKG_SOURCE: yes CUSTOM_DR_UA: 'GitHub_CI' diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml index 7c20a4cb..ac3360ad 100644 --- a/.github/workflows/pkgdown.yaml +++ b/.github/workflows/pkgdown.yaml @@ -28,6 +28,7 @@ jobs: group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }} env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + API_USGS_PAT: ${{ secrets.API_USGS_PAT }} CUSTOM_DR_UA: 'GitHub_CI' steps: - uses: actions/checkout@581d62f320f2a4043a1ea6ac77290c60d27485cc diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index ce05609b..98e4b9e6 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -14,6 +14,7 @@ jobs: env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + API_USGS_PAT: ${{ secrets.API_USGS_PAT }} R_KEEP_PKG_SOURCE: yes CUSTOM_DR_UA: 'GitHub_CI' diff --git a/NAMESPACE b/NAMESPACE index 73f14c7c..622427bc 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -44,11 +44,11 @@ export(readNWISuv) export(readWQPdata) export(readWQPqw) export(readWQPsummary) -export(read_USGS_daily) -export(read_USGS_data) -export(read_USGS_monitoring_location) -export(read_USGS_samples) -export(read_USGS_ts_meta) +export(read_waterdata) +export(read_waterdata_daily) +export(read_waterdata_monitoring_location) +export(read_waterdata_samples) +export(read_waterdata_ts_meta) export(renameNWISColumns) export(setAccess) export(stateCd) diff --git a/R/readNWISdata.R b/R/readNWISdata.R index 8179fcde..6533b236 100644 --- a/R/readNWISdata.R +++ b/R/readNWISdata.R @@ -71,7 +71,7 @@ #' queryTime \tab POSIXct \tab The time the data was returned \cr #' } #' -#' @seealso [read_USGS_data()] +#' @seealso [read_waterdata_data()] #' @export #' @examplesIf is_dataRetrieval_user() #' \donttest{ @@ -189,14 +189,14 @@ for more information. https://cran.r-project.org/web/packages/dataRetrieval/vignettes/qwdata_changes.html" ) } else if (service == "dv"){ - .Deprecated(new = "read_USGS_daily", + .Deprecated(new = "read_waterdata_daily", package = "dataRetrieval", - msg = "NWIS servers are slated for decommission. Please begin to migrate to read_USGS_daily.") + msg = "NWIS servers are slated for decommission. Please begin to migrate to read_waterdata_daily.") } else if (service == "site"){ - .Deprecated(new = "read_USGS_monitoring_location", + .Deprecated(new = "read_waterdata_monitoring_location", package = "dataRetrieval", - msg = "NWIS servers are slated for decommission. Please begin to migrate to read_USGS_monitoring_location") + msg = "NWIS servers are slated for decommission. Please begin to migrate to read_waterdata_monitoring_location") } else { message(new_nwis_message()) } diff --git a/R/readNWISdv.R b/R/readNWISdv.R index acd70335..c67a773c 100644 --- a/R/readNWISdv.R +++ b/R/readNWISdv.R @@ -49,12 +49,12 @@ #' queryTime \tab POSIXct \tab The time the data was returned \cr #' } #' -#' @seealso [read_USGS_daily()] +#' @seealso [read_waterdata_daily()] #' @export #' @keywords data import USGS web service #' @examples #' -#' # see ?read_USGS_daily +#' # see ?read_waterdata_daily #' #' #site_id <- "04085427" #' #startDate <- "2012-01-01" @@ -69,9 +69,9 @@ readNWISdv <- function(siteNumbers, endDate = "", statCd = "00003") { - .Deprecated(new = "read_USGS_daily", + .Deprecated(new = "read_waterdata_daily", package = "dataRetrieval", - msg = "NWIS servers are slated for decommission. Please begin to migrate to read_USGS_daily.") + msg = "NWIS servers are slated for decommission. Please begin to migrate to read_waterdata_daily.") url <- constructNWISURL( siteNumbers = siteNumbers, diff --git a/R/readNWISsite.R b/R/readNWISsite.R index 722b512f..4ab14e63 100644 --- a/R/readNWISsite.R +++ b/R/readNWISsite.R @@ -59,17 +59,17 @@ #' comment \tab character \tab Header comments from the RDB file \cr #' } #' @export -#' @seealso [read_USGS_monitoring_location()] +#' @seealso [read_waterdata_monitoring_location()] #' @examples #' -#' # see ?read_USGS_monitoring_location +#' # see ?read_waterdata_monitoring_location #' # siteINFOMulti <- readNWISsite(c("05114000", "09423350")) #' readNWISsite <- function(siteNumbers) { - .Deprecated(new = "read_USGS_monitoring_location", + .Deprecated(new = "read_waterdata_monitoring_location", package = "dataRetrieval", - msg = "NWIS servers are slated for decommission. Please begin to migrate to read_USGS_monitoring_location") + msg = "NWIS servers are slated for decommission. Please begin to migrate to read_waterdata_monitoring_location") baseURL <- httr2::request(pkg.env[["site"]]) diff --git a/R/read_USGS_data.R b/R/read_waterdata.R similarity index 94% rename from R/read_USGS_data.R rename to R/read_waterdata.R index 935db0c9..b19e9a7a 100644 --- a/R/read_USGS_data.R +++ b/R/read_waterdata.R @@ -1,4 +1,4 @@ -#' Generalized USGS data retrieval function +#' Generalized USGS Water Data API retrieval function #' #' Function that allows complex CQL queries. #' See @@ -34,12 +34,12 @@ #' ] #' }' #' -#' dv_data <- read_USGS_data(service = "daily", +#' dv_data <- read_waterdata(service = "daily", #' CQL = cql, #' time = c("2023-01-01", "2024-01-01")) #' #' } -read_USGS_data <- function(service, +read_waterdata <- function(service, CQL, ..., convertType = TRUE){ diff --git a/R/read_USGS_daily.R b/R/read_waterdata_daily.R similarity index 92% rename from R/read_USGS_daily.R rename to R/read_waterdata_daily.R index 4e92fa74..2ac5f01c 100644 --- a/R/read_USGS_daily.R +++ b/R/read_waterdata_daily.R @@ -40,33 +40,33 @@ #' \donttest{ #' site <- "USGS-02238500" #' pcode <- "00060" -#' dv_data_sf <- read_USGS_daily(monitoring_location_id = site, +#' dv_data_sf <- read_waterdata_daily(monitoring_location_id = site, #' parameter_code = "00060", #' time = c("2021-01-01", "2022-01-01")) #' -#' dv_data_trim <- read_USGS_daily(monitoring_location_id = site, +#' dv_data_trim <- read_waterdata_daily(monitoring_location_id = site, #' parameter_code = "00060", #' properties = c("monitoring_location_id", #' "value", #' "time"), #' time = c("2021-01-01", "2022-01-01")) #' -#' dv_data <- read_USGS_daily(monitoring_location_id = site, +#' dv_data <- read_waterdata_daily(monitoring_location_id = site, #' parameter_code = "00060", #' skipGeometry = TRUE) #' -#' dv_data_period <- read_USGS_daily(monitoring_location_id = site, +#' dv_data_period <- read_waterdata_daily(monitoring_location_id = site, #' parameter_code = "00060", #' time = "P7D") #' -#' multi_site <- read_USGS_daily(monitoring_location_id = c("USGS-01491000", +#' multi_site <- read_waterdata_daily(monitoring_location_id = c("USGS-01491000", #' "USGS-01645000"), #' parameter_code = c("00060", "00010"), #' limit = 500, #' time = c("2023-01-01", "2024-01-01")) #' #' } -read_USGS_daily <- function(monitoring_location_id = NA_character_, +read_waterdata_daily <- function(monitoring_location_id = NA_character_, parameter_code = NA_character_, statistic_id = NA_character_, properties = NA_character_, diff --git a/R/read_USGS_monitoring_location.R b/R/read_waterdata_monitoring_location.R similarity index 93% rename from R/read_USGS_monitoring_location.R rename to R/read_waterdata_monitoring_location.R index f46998c7..9489a1bb 100644 --- a/R/read_USGS_monitoring_location.R +++ b/R/read_waterdata_monitoring_location.R @@ -66,30 +66,30 @@ #' #' \donttest{ #' site <- "USGS-02238500" -#' site_info <- read_USGS_monitoring_location(monitoring_location_id = site) +#' site_info <- read_waterdata_monitoring_location(monitoring_location_id = site) #' -#' site_slim <- read_USGS_monitoring_location(monitoring_location_id = site, +#' site_slim <- read_waterdata_monitoring_location(monitoring_location_id = site, #' properties = c("monitoring_location_id", #' "state_name", #' "country_name")) #' -#' site_slim_no_sf_slim <- read_USGS_monitoring_location(monitoring_location_id = site, +#' site_slim_no_sf_slim <- read_waterdata_monitoring_location(monitoring_location_id = site, #' properties = c("monitoring_location_id", #' "state_name", #' "country_name"), #' skipGeometry = TRUE) #' -#' site_info_no_sf <- read_USGS_monitoring_location(monitoring_location_id = site, +#' site_info_no_sf <- read_waterdata_monitoring_location(monitoring_location_id = site, #' skipGeometry = TRUE) #' #' bbox_vals = c(-94.00, 35.0, -93.5, 35.5) -#' multi_site <- read_USGS_monitoring_location(bbox = bbox_vals) -#' multi_site_n_100 <- read_USGS_monitoring_location(bbox = bbox_vals, +#' multi_site <- read_waterdata_monitoring_location(bbox = bbox_vals) +#' multi_site_n_100 <- read_waterdata_monitoring_location(bbox = bbox_vals, #' max_results = 100) -#' multi_site_limit_100 <- read_USGS_monitoring_location(bbox = bbox_vals, +#' multi_site_limit_100 <- read_waterdata_monitoring_location(bbox = bbox_vals, #' limit = 100) #' } -read_USGS_monitoring_location <- function(monitoring_location_id = NA_character_, +read_waterdata_monitoring_location <- function(monitoring_location_id = NA_character_, agency_code = NA_character_, agency_name = NA_character_, monitoring_location_number = NA_character_, diff --git a/R/read_USGS_samples.R b/R/read_waterdata_samples.R similarity index 98% rename from R/read_USGS_samples.R rename to R/read_waterdata_samples.R index f10febd7..e225efe0 100644 --- a/R/read_USGS_samples.R +++ b/R/read_waterdata_samples.R @@ -392,24 +392,24 @@ check_USGS_sample_params <- function(service = "characteristicgroup", #' @examplesIf is_dataRetrieval_user() #' #' \donttest{ -#' ph_data <- read_USGS_samples( +#' ph_data <- read_waterdata_samples( #' monitoringLocationIdentifier = "USGS-04074950", #' characteristicUserSupplied = "pH, water, unfiltered, field", #' activityStartDateUpper = "2000-01-01", #' dataProfile = "narrow") #' #' nameToUse <- "pH" -#' pHData <- read_USGS_samples(monitoringLocationIdentifier = "USGS-04024315", +#' pHData <- read_waterdata_samples(monitoringLocationIdentifier = "USGS-04024315", #' characteristic = nameToUse) #' ncol(pHData) #' attr(pHData, "url") #' attr(pHData, "queryTime") #' -#' summary_data <- read_USGS_samples(monitoringLocationIdentifier = "USGS-04024315", +#' summary_data <- read_waterdata_samples(monitoringLocationIdentifier = "USGS-04024315", #' dataType = "projects") #' #' } -read_USGS_samples <- function(monitoringLocationIdentifier = NA, +read_waterdata_samples <- function(monitoringLocationIdentifier = NA, siteTypeCode = NA, boundingBox = NA, hydrologicUnit = NA, @@ -506,4 +506,4 @@ summarize_USGS_samples <- function(monitoringLocationIdentifier){ attr(df, "queryTime") <- Sys.time() return(df) -} \ No newline at end of file +} diff --git a/R/read_USGS_ts_meta.R b/R/read_waterdata_ts_meta.R similarity index 95% rename from R/read_USGS_ts_meta.R rename to R/read_waterdata_ts_meta.R index 1a9ef7db..1dc701f7 100644 --- a/R/read_USGS_ts_meta.R +++ b/R/read_waterdata_ts_meta.R @@ -43,9 +43,9 @@ #' #' \donttest{ #' site <- "USGS-02238500" -#' meta_1 <- read_USGS_ts_meta(monitoring_location_id = site) +#' meta_1 <- read_waterdata_ts_meta(monitoring_location_id = site) #' -#' meta_multi <- read_USGS_ts_meta(monitoring_location_id = c("USGS-01491000", +#' meta_multi <- read_waterdata_ts_meta(monitoring_location_id = c("USGS-01491000", #' "USGS-01645000"), #' parameter_code = c("00060", "00010"), #' properties = c("monitoring_location_id", @@ -55,7 +55,7 @@ #' "time_series_id"), #' skipGeometry = TRUE) #' } -read_USGS_ts_meta <- function(monitoring_location_id = NA_character_, +read_waterdata_ts_meta <- function(monitoring_location_id = NA_character_, parameter_code = NA_character_, parameter_name = NA_character_, properties = NA_character_, diff --git a/man/check_USGS_sample_params.Rd b/man/check_USGS_sample_params.Rd index eb387605..338b84e0 100644 --- a/man/check_USGS_sample_params.Rd +++ b/man/check_USGS_sample_params.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_USGS_samples.R +% Please edit documentation in R/read_waterdata_samples.R \name{check_USGS_sample_params} \alias{check_USGS_sample_params} \title{Check values from codeservice} diff --git a/man/construct_USGS_sample_request.Rd b/man/construct_USGS_sample_request.Rd index d3b89415..58ab4cbe 100644 --- a/man/construct_USGS_sample_request.Rd +++ b/man/construct_USGS_sample_request.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_USGS_samples.R +% Please edit documentation in R/read_waterdata_samples.R \name{construct_USGS_sample_request} \alias{construct_USGS_sample_request} \title{Construct request for USGS Samples Data} diff --git a/man/readNWISdata.Rd b/man/readNWISdata.Rd index 6d7d90db..cb7116f6 100644 --- a/man/readNWISdata.Rd +++ b/man/readNWISdata.Rd @@ -179,5 +179,5 @@ peak_data <- readNWISdata( \dontshow{\}) # examplesIf} } \seealso{ -\code{\link[=read_USGS_data]{read_USGS_data()}} +\code{\link[=read_waterdata_data]{read_waterdata_data()}} } diff --git a/man/readNWISdv.Rd b/man/readNWISdv.Rd index 83d36b8b..bd131c90 100644 --- a/man/readNWISdv.Rd +++ b/man/readNWISdv.Rd @@ -72,7 +72,7 @@ More information on the web service can be found here: } \examples{ -# see ?read_USGS_daily +# see ?read_waterdata_daily #site_id <- "04085427" #startDate <- "2012-01-01" @@ -83,7 +83,7 @@ More information on the web service can be found here: } \seealso{ -\code{\link[=read_USGS_daily]{read_USGS_daily()}} +\code{\link[=read_waterdata_daily]{read_waterdata_daily()}} } \keyword{USGS} \keyword{data} diff --git a/man/readNWISsite.Rd b/man/readNWISsite.Rd index f38434cf..4447faab 100644 --- a/man/readNWISsite.Rd +++ b/man/readNWISsite.Rd @@ -70,12 +70,12 @@ Imports data from USGS site file site. This function gets data from here: \url{h } \examples{ -# see ?read_USGS_monitoring_location +# see ?read_waterdata_monitoring_location # siteINFOMulti <- readNWISsite(c("05114000", "09423350")) } \seealso{ -\code{\link[=read_USGS_monitoring_location]{read_USGS_monitoring_location()}} +\code{\link[=read_waterdata_monitoring_location]{read_waterdata_monitoring_location()}} } \keyword{USGS} \keyword{data} diff --git a/man/read_USGS_data.Rd b/man/read_waterdata.Rd similarity index 82% rename from man/read_USGS_data.Rd rename to man/read_waterdata.Rd index 85406a3e..544cf63f 100644 --- a/man/read_USGS_data.Rd +++ b/man/read_waterdata.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_USGS_data.R -\name{read_USGS_data} -\alias{read_USGS_data} -\title{Generalized USGS data retrieval function} +% Please edit documentation in R/read_waterdata.R +\name{read_waterdata} +\alias{read_waterdata} +\title{Generalized USGS Water Data API retrieval function} \usage{ -read_USGS_data(service, CQL, ..., convertType = TRUE) +read_waterdata(service, CQL, ..., convertType = TRUE) } \arguments{ \item{service}{character, can be any existing collection such @@ -46,7 +46,7 @@ cql <- '{ ] }' -dv_data <- read_USGS_data(service = "daily", +dv_data <- read_waterdata(service = "daily", CQL = cql, time = c("2023-01-01", "2024-01-01")) diff --git a/man/read_USGS_daily.Rd b/man/read_waterdata_daily.Rd similarity index 94% rename from man/read_USGS_daily.Rd rename to man/read_waterdata_daily.Rd index aa7a7f10..5a29c4ed 100644 --- a/man/read_USGS_daily.Rd +++ b/man/read_waterdata_daily.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_USGS_daily.R -\name{read_USGS_daily} -\alias{read_USGS_daily} +% Please edit documentation in R/read_waterdata_daily.R +\name{read_waterdata_daily} +\alias{read_waterdata_daily} \title{Get USGS Daily Data} \usage{ -read_USGS_daily( +read_waterdata_daily( monitoring_location_id = NA_character_, parameter_code = NA_character_, statistic_id = NA_character_, @@ -102,26 +102,26 @@ Description Daily data provide one data value to represent water conditions for \donttest{ site <- "USGS-02238500" pcode <- "00060" -dv_data_sf <- read_USGS_daily(monitoring_location_id = site, +dv_data_sf <- read_waterdata_daily(monitoring_location_id = site, parameter_code = "00060", time = c("2021-01-01", "2022-01-01")) -dv_data_trim <- read_USGS_daily(monitoring_location_id = site, +dv_data_trim <- read_waterdata_daily(monitoring_location_id = site, parameter_code = "00060", properties = c("monitoring_location_id", "value", "time"), time = c("2021-01-01", "2022-01-01")) -dv_data <- read_USGS_daily(monitoring_location_id = site, +dv_data <- read_waterdata_daily(monitoring_location_id = site, parameter_code = "00060", skipGeometry = TRUE) -dv_data_period <- read_USGS_daily(monitoring_location_id = site, +dv_data_period <- read_waterdata_daily(monitoring_location_id = site, parameter_code = "00060", time = "P7D") -multi_site <- read_USGS_daily(monitoring_location_id = c("USGS-01491000", +multi_site <- read_waterdata_daily(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), parameter_code = c("00060", "00010"), limit = 500, diff --git a/man/read_USGS_monitoring_location.Rd b/man/read_waterdata_monitoring_location.Rd similarity index 95% rename from man/read_USGS_monitoring_location.Rd rename to man/read_waterdata_monitoring_location.Rd index 58d3fa69..40cd6e0b 100644 --- a/man/read_USGS_monitoring_location.Rd +++ b/man/read_waterdata_monitoring_location.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_USGS_monitoring_location.R -\name{read_USGS_monitoring_location} -\alias{read_USGS_monitoring_location} +% Please edit documentation in R/read_waterdata_monitoring_location.R +\name{read_waterdata_monitoring_location} +\alias{read_waterdata_monitoring_location} \title{Get USGS Site Data} \usage{ -read_USGS_monitoring_location( +read_waterdata_monitoring_location( monitoring_location_id = NA_character_, agency_code = NA_character_, agency_name = NA_character_, @@ -165,27 +165,27 @@ Description Location information is basic information about the monitoring locat \donttest{ site <- "USGS-02238500" -site_info <- read_USGS_monitoring_location(monitoring_location_id = site) +site_info <- read_waterdata_monitoring_location(monitoring_location_id = site) -site_slim <- read_USGS_monitoring_location(monitoring_location_id = site, +site_slim <- read_waterdata_monitoring_location(monitoring_location_id = site, properties = c("monitoring_location_id", "state_name", "country_name")) -site_slim_no_sf_slim <- read_USGS_monitoring_location(monitoring_location_id = site, +site_slim_no_sf_slim <- read_waterdata_monitoring_location(monitoring_location_id = site, properties = c("monitoring_location_id", "state_name", "country_name"), skipGeometry = TRUE) -site_info_no_sf <- read_USGS_monitoring_location(monitoring_location_id = site, +site_info_no_sf <- read_waterdata_monitoring_location(monitoring_location_id = site, skipGeometry = TRUE) bbox_vals = c(-94.00, 35.0, -93.5, 35.5) -multi_site <- read_USGS_monitoring_location(bbox = bbox_vals) -multi_site_n_100 <- read_USGS_monitoring_location(bbox = bbox_vals, +multi_site <- read_waterdata_monitoring_location(bbox = bbox_vals) +multi_site_n_100 <- read_waterdata_monitoring_location(bbox = bbox_vals, max_results = 100) -multi_site_limit_100 <- read_USGS_monitoring_location(bbox = bbox_vals, +multi_site_limit_100 <- read_waterdata_monitoring_location(bbox = bbox_vals, limit = 100) } \dontshow{\}) # examplesIf} diff --git a/man/read_USGS_samples.Rd b/man/read_waterdata_samples.Rd similarity index 95% rename from man/read_USGS_samples.Rd rename to man/read_waterdata_samples.Rd index 0af207d2..b23fbd4e 100644 --- a/man/read_USGS_samples.Rd +++ b/man/read_waterdata_samples.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_USGS_samples.R -\name{read_USGS_samples} -\alias{read_USGS_samples} +% Please edit documentation in R/read_waterdata_samples.R +\name{read_waterdata_samples} +\alias{read_waterdata_samples} \title{USGS Samples Data} \usage{ -read_USGS_samples( +read_waterdata_samples( monitoringLocationIdentifier = NA, siteTypeCode = NA, boundingBox = NA, @@ -140,20 +140,20 @@ service described at \url{https://waterdata.usgs.gov/download-samples/}. \dontshow{if (is_dataRetrieval_user()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} \donttest{ -ph_data <- read_USGS_samples( +ph_data <- read_waterdata_samples( monitoringLocationIdentifier = "USGS-04074950", characteristicUserSupplied = "pH, water, unfiltered, field", activityStartDateUpper = "2000-01-01", dataProfile = "narrow") nameToUse <- "pH" -pHData <- read_USGS_samples(monitoringLocationIdentifier = "USGS-04024315", +pHData <- read_waterdata_samples(monitoringLocationIdentifier = "USGS-04024315", characteristic = nameToUse) ncol(pHData) attr(pHData, "url") attr(pHData, "queryTime") -summary_data <- read_USGS_samples(monitoringLocationIdentifier = "USGS-04024315", +summary_data <- read_waterdata_samples(monitoringLocationIdentifier = "USGS-04024315", dataType = "projects") } diff --git a/man/read_USGS_ts_meta.Rd b/man/read_waterdata_ts_meta.Rd similarity index 96% rename from man/read_USGS_ts_meta.Rd rename to man/read_waterdata_ts_meta.Rd index 9a4e7387..4847f0a4 100644 --- a/man/read_USGS_ts_meta.Rd +++ b/man/read_waterdata_ts_meta.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_USGS_ts_meta.R -\name{read_USGS_ts_meta} -\alias{read_USGS_ts_meta} +% Please edit documentation in R/read_waterdata_ts_meta.R +\name{read_waterdata_ts_meta} +\alias{read_waterdata_ts_meta} \title{Get USGS Time Series Metadata} \usage{ -read_USGS_ts_meta( +read_waterdata_ts_meta( monitoring_location_id = NA_character_, parameter_code = NA_character_, parameter_name = NA_character_, @@ -104,9 +104,9 @@ Description Daily data and continuous measurements are grouped into time series, \donttest{ site <- "USGS-02238500" -meta_1 <- read_USGS_ts_meta(monitoring_location_id = site) +meta_1 <- read_waterdata_ts_meta(monitoring_location_id = site) -meta_multi <- read_USGS_ts_meta(monitoring_location_id = c("USGS-01491000", +meta_multi <- read_waterdata_ts_meta(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), parameter_code = c("00060", "00010"), properties = c("monitoring_location_id", diff --git a/man/summarize_USGS_samples.Rd b/man/summarize_USGS_samples.Rd index 92a0023d..3541ed4c 100644 --- a/man/summarize_USGS_samples.Rd +++ b/man/summarize_USGS_samples.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_USGS_samples.R +% Please edit documentation in R/read_waterdata_samples.R \name{summarize_USGS_samples} \alias{summarize_USGS_samples} \title{USGS Samples Summary Data} diff --git a/tests/testthat/tests_general.R b/tests/testthat/tests_general.R index 4cd73e48..97469582 100644 --- a/tests/testthat/tests_general.R +++ b/tests/testthat/tests_general.R @@ -23,7 +23,7 @@ test_that("General USGS retrievals working", { ] }' - dv_data <- read_USGS_data(service = "daily", + dv_data <- read_waterdata(service = "daily", CQL = cql, time = c("2023-01-01", "2024-01-01")) expect_equal(as.Date(c("2023-01-01", "2024-01-01")), @@ -52,7 +52,7 @@ test_that("General USGS retrievals working", { ] }' - notActiveUSGS <- read_USGS_data(CQL = cql_not_active, + notActiveUSGS <- read_waterdata(CQL = cql_not_active, service = "daily", time = c("2014-01-01", "2014-01-07")) expect_true(nrow(notActiveUSGS) == 0) @@ -97,14 +97,14 @@ test_that("General NWIS retrievals working", { expect_error(readNWISdata(), "No arguments supplied") expect_error(readNWISdata(siteNumber = NA), "NA's are not allowed in query") - bBox_inventory <- read_USGS_ts_meta(bbox = c(-83, 38, -82.5, 38.5), + bBox_inventory <- read_waterdata_ts_meta(bbox = c(-83, 38, -82.5, 38.5), parameter_code = "00010") expect_true(length(unique(bBox_inventory$monitoring_location_id)) > 1) - siteInfo <- read_USGS_monitoring_location(state_name = "Wisconsin") + siteInfo <- read_waterdata_monitoring_location(state_name = "Wisconsin") - timeseriesInfo <- read_USGS_ts_meta(bbox = sf::st_bbox(siteInfo), + timeseriesInfo <- read_waterdata_ts_meta(bbox = sf::st_bbox(siteInfo), parameter_code = "00010", computation_period_identifier = "Points" ) @@ -174,18 +174,18 @@ test_that("General NWIS retrievals working", { time = c("2014-05-01", endDate = "2014-05-01") ) - daily_USGS <- do.call(read_USGS_daily, args2) + daily_USGS <- do.call(read_waterdata_daily, args2) expect_lt(nrow(daily_USGS), nrow(instData)) - ohio <- read_USGS_monitoring_location(state_name = "Ohio", + ohio <- read_waterdata_monitoring_location(state_name = "Ohio", site_type_code = "ST") bbox <- sf::st_bbox(ohio) - what_sites <- read_USGS_ts_meta(parameter_code = "00665", + what_sites <- read_waterdata_ts_meta(parameter_code = "00665", bbox = bbox) expect_true(all(c("monitoring_location_id", "begin", "end", "parameter_name") %in% names(what_sites))) - huc <- read_USGS_monitoring_location(hydrologic_unit_code = "02080202") + huc <- read_waterdata_monitoring_location(hydrologic_unit_code = "02080202") expect_true(nrow(huc) > 0) # Test counties: @@ -194,11 +194,11 @@ test_that("General NWIS retrievals working", { county = "Stafford", outputType = "id") state_code_va <- stateCdLookup(input = "Virginia", outputType = "id") - stafford <- read_USGS_monitoring_location(county_code = county_code_stafford, + stafford <- read_waterdata_monitoring_location(county_code = county_code_stafford, state_code = state_code_va) stafford_bbox <- sf::st_bbox(stafford) - dailyStaffordVA <- read_USGS_daily( + dailyStaffordVA <- read_waterdata_daily( bbox = stafford_bbox, parameter_code = "00060", time = c("2015-01-01", "2015-01-30") @@ -206,7 +206,7 @@ test_that("General NWIS retrievals working", { expect_gt(nrow(dailyStaffordVA), 1) # America Samoa? - AS <- read_USGS_monitoring_location(state_name = "American Samoa") + AS <- read_waterdata_monitoring_location(state_name = "American Samoa") expect_gt(nrow(AS), 0) site_id <- "01594440" @@ -239,9 +239,9 @@ test_that("General NWIS retrievals working", { ))) multi_hucs <- c("07130007", "07130011") - multi_huc_sites <- read_USGS_monitoring_location(hydrologic_unit_code = multi_hucs) + multi_huc_sites <- read_waterdata_monitoring_location(hydrologic_unit_code = multi_hucs) - multi_huc <- read_USGS_daily(bbox = sf::st_bbox(multi_huc_sites), + multi_huc <- read_waterdata_daily(bbox = sf::st_bbox(multi_huc_sites), parameter_code = "63680", statistic_id = "00003", time = c("2015-06-18", "2015-06-18") @@ -261,18 +261,18 @@ test_that("General NWIS retrievals working", { expect_lt(nrow(peak_data), 100000) }) -test_that("read_USGS_ts_meta", { +test_that("read_waterdata_ts_meta", { # no service specified: - availableData <- read_USGS_ts_meta(monitoring_location_id = "USGS-05114000") + availableData <- read_waterdata_ts_meta(monitoring_location_id = "USGS-05114000") expect_equal(ncol(availableData), 17) - uvData <- read_USGS_ts_meta(monitoring_location_id = "USGS-05114000", + uvData <- read_waterdata_ts_meta(monitoring_location_id = "USGS-05114000", computation_period_identifier = c("Points")) expect_equal(unique(uvData$computation_period_identifier), "Points") # multiple services - uvDataMulti <- read_USGS_ts_meta(monitoring_location_id = c("USGS-05114000", + uvDataMulti <- read_waterdata_ts_meta(monitoring_location_id = c("USGS-05114000", "USGS-09423350"), computation_period_identifier = c("Daily", "Points")) @@ -281,8 +281,8 @@ test_that("read_USGS_ts_meta", { "Points"))) # state codes: - wi_sites <- read_USGS_monitoring_location(state_name = "Wisconsin") - flow_and_temp <- read_USGS_ts_meta(bbox = sf::st_bbox(wi_sites), + wi_sites <- read_waterdata_monitoring_location(state_name = "Wisconsin") + flow_and_temp <- read_waterdata_ts_meta(bbox = sf::st_bbox(wi_sites), parameter_code = c("00060", "00010"), statistic_id = "00003", computation_period_identifier = c("Daily", @@ -426,16 +426,16 @@ test_that("whatWQPdata working", { expect_is(lakeSites$activityCount, "numeric") }) -context("read_USGS_ts_meta") -test_that("read_USGS_ts_meta working", { +context("read_waterdata_ts_meta") +test_that("read_waterdata_ts_meta working", { testthat::skip_on_cran() - siteListOhio <- read_USGS_monitoring_location(state_name = "Ohio") - siteListPhos <- read_USGS_ts_meta(bbox = sf::st_bbox(siteListOhio), + siteListOhio <- read_waterdata_monitoring_location(state_name = "Ohio") + siteListPhos <- read_waterdata_ts_meta(bbox = sf::st_bbox(siteListOhio), parameter_code = "00665") expect_true(nrow(siteListPhos) > 0) expect_is(siteListPhos$begin, "POSIXct") - bboxSites <- read_USGS_ts_meta(bbox = c(-92.5, 45.4, -87, 47), + bboxSites <- read_waterdata_ts_meta(bbox = c(-92.5, 45.4, -87, 47), parameter_code = "00060") expect_true(nrow(bboxSites) > 0) diff --git a/tests/testthat/tests_samples.R b/tests/testthat/tests_samples.R index 2c87d4ef..e3ddef11 100644 --- a/tests/testthat/tests_samples.R +++ b/tests/testthat/tests_samples.R @@ -4,7 +4,7 @@ context("General functions") test_that("General samples-data retrievals work using WQP tests", { testthat::skip_on_cran() nameToUse <- "pH" - pHData <- read_USGS_samples(monitoringLocationIdentifier = "USGS-04024315", + pHData <- read_waterdata_samples(monitoringLocationIdentifier = "USGS-04024315", characteristic = nameToUse) expect_is(pHData$Activity_StartDateTime, "POSIXct") @@ -14,7 +14,7 @@ test_that("General samples-data retrievals work using WQP tests", { text = "secchi") state_fips <- paste0("US:", stateCdLookup("WI", "id")) - lakeData <- read_USGS_samples(activityStartDateLower = startDate, + lakeData <- read_waterdata_samples(activityStartDateLower = startDate, activityStartDateUpper = "2024-01-01", stateFips = "US:55", characteristicUserSupplied = secchi_ops$observedProperty, @@ -22,20 +22,20 @@ test_that("General samples-data retrievals work using WQP tests", { expect_true(nrow(lakeData) > 0) - lakeSites <- read_USGS_samples(monitoringLocationIdentifier = unique(lakeData$Location_Identifier), + lakeSites <- read_waterdata_samples(monitoringLocationIdentifier = unique(lakeData$Location_Identifier), dataType = "locations", dataProfile = "site") expect_type(lakeSites, "list") - rawPcode <- read_USGS_samples(monitoringLocationIdentifier = "USGS-01594440", + rawPcode <- read_waterdata_samples(monitoringLocationIdentifier = "USGS-01594440", usgsPCode = "01075") expect_true(all(c("url", "queryTime", "headerInfo") %in% names(attributes(rawPcode)))) - pHData <- read_USGS_samples(monitoringLocationIdentifier = "USGS-04024315", + pHData <- read_waterdata_samples(monitoringLocationIdentifier = "USGS-04024315", characteristic = "pH", dataProfile = "narrow") @@ -50,7 +50,7 @@ context("samples-data samples") test_that("samples-data activities working", { testthat::skip_on_cran() - activityInfo <- read_USGS_samples(monitoringLocationIdentifier = "USGS-01594440", + activityInfo <- read_waterdata_samples(monitoringLocationIdentifier = "USGS-01594440", dataType = "activities") expect_true(nrow(activityInfo) > 0) }) @@ -60,7 +60,7 @@ test_that("samples-data project working", { testthat::skip_on_cran() type <- "Stream" - projectInfo <- read_USGS_samples(countyFips = countyCdLookup("WI", "Dane"), + projectInfo <- read_waterdata_samples(countyFips = countyCdLookup("WI", "Dane"), siteTypeName = type, dataType = "projects") expect_true(ncol(projectInfo) >= 0) @@ -79,13 +79,13 @@ test_that("profiles", { testthat::skip_on_cran() # Data profiles: "Organization Data" - org_data <- read_USGS_samples( + org_data <- read_waterdata_samples( countyFips = countyCdLookup("WI", "Dane"), dataType = "organizations" ) # Data profiles: "Site Data Only" - site_data <- read_USGS_samples( + site_data <- read_waterdata_samples( countyFips = countyCdLookup("WI", "Dane"), dataType = "locations" ) @@ -93,7 +93,7 @@ test_that("profiles", { expect_true(all(c("ProviderName", "Location_Identifier") %in% names(site_data))) # Data profiles: "Project Data" - project_data <- read_USGS_samples( + project_data <- read_waterdata_samples( countyFips = countyCdLookup("WI", "Dane"), dataType = "projects" ) @@ -104,7 +104,7 @@ test_that("profiles", { ) %in% names(project_data))) # Data profiles: "Project Monitoring Location Weighting Data" - proj_mlwd <- read_USGS_samples( + proj_mlwd <- read_waterdata_samples( countyFips = countyCdLookup("WI", "Dane"), dataType = "projects", dataProfile = "projectmonitoringlocationweight" @@ -116,7 +116,7 @@ test_that("profiles", { ) %in% names(proj_mlwd))) # Data profiles: "Sample Results (biological metadata)" - samp_bio <- read_USGS_samples( + samp_bio <- read_waterdata_samples( monitoringLocationIdentifier = "USGS-04024315", dataProfile = "basicbio", dataType = "results" @@ -128,7 +128,7 @@ test_that("profiles", { ) %in% names(samp_bio))) # Data profiles: "Sample Results (narrow)" - samp_narrow <- read_USGS_samples( + samp_narrow <- read_waterdata_samples( monitoringLocationIdentifier = "USGS-04024315", dataProfile = "narrow", dataType = "results" @@ -140,7 +140,7 @@ test_that("profiles", { ) %in% names(samp_narrow))) # Data profiles: "Sampling Activity" - samp_activity <- read_USGS_samples( + samp_activity <- read_waterdata_samples( monitoringLocationIdentifier = "USGS-04024315", dataProfile = "sampact", # Sampling Activities dataType = "activities" @@ -152,7 +152,7 @@ test_that("profiles", { ) %in% names(samp_activity))) # Data profile: "Result Detection Quantitation Limit Data" - dl_data <- read_USGS_samples( + dl_data <- read_waterdata_samples( monitoringLocationIdentifier = "USGS-04024315", dataType = "results", dataProfile = "resultdetectionquantitationlimit" diff --git a/vignettes/Status.Rmd b/vignettes/Status.Rmd index f18b27bc..a258c888 100644 --- a/vignettes/Status.Rmd +++ b/vignettes/Status.Rmd @@ -25,12 +25,12 @@ Please contact the Computational Tools team at CompTools@usgs.gov with questions
What is an application programming interface (API)? - When you run a `dataRetrieval` function like `read_USGS_samples()` or `readNWISdv()`, you are actually generating a URL that contains within it specifications of the dataset in which you are interested (e.g. which monitoring locations, characteristics, pcodes, start and end dates, etc.). The format of that URL is special: it is parsed by a USGS-specific API web service, which translates the request into a database call. It then packages the response object from the database (with the data) into a format that can be sent to and then unpacked by the user. `dataRetrieval` takes a lot of the guesswork out of this process by generating the URL, sending it to the API, and wrangling the response object into a tabular dataset. + When you run a `dataRetrieval` function like `read_waterdata_samples()` or `readNWISdv()`, you are actually generating a URL that contains within it specifications of the dataset in which you are interested (e.g. which monitoring locations, characteristics, pcodes, start and end dates, etc.). The format of that URL is special: it is parsed by a USGS-specific API web service, which translates the request into a database call. It then packages the response object from the database (with the data) into a format that can be sent to and then unpacked by the user. `dataRetrieval` takes a lot of the guesswork out of this process by generating the URL, sending it to the API, and wrangling the response object into a tabular dataset.

The USGS is in the process of creating new, [publicly available APIs](https://api.waterdata.usgs.gov/) (a.k.a. "web services") to replace the existing [WaterServices](https://waterservices.usgs.gov/). `dataRetrieval` relies upon these web services to provide monitoring location information and water quantity/quality datasets. As new web services come online, `dataRetrieval` will be furnished with new functions to accommodate these changes. -The table below shows the **existing** NWIS functions and, if applicable, their slated replacements. Note that several functions do not have direct replacements because the new services to support them do not yet exist. We will update this table as new services and functions come online. Also note that some new functions may only be available on the "develop" branch of `dataRetrieval` (`remotes::install_github("DOI-USGS/dataRetrieval", ref = "develop")`). More information on each function's specifications (complete with examples) are available in the [Introduction to New USGS Services vignette](add link here), the [package index](https://doi-usgs.github.io/dataRetrieval/reference/index.html) and/or function documentation (e.g. `?read_USGS_daily`). +The table below shows the **existing** NWIS functions and, if applicable, their slated replacements. Note that several functions do not have direct replacements because the new services to support them do not yet exist. We will update this table as new services and functions come online. Also note that some new functions may only be available on the "develop" branch of `dataRetrieval` (`remotes::install_github("DOI-USGS/dataRetrieval", ref = "develop")`). More information on each function's specifications (complete with examples) are available in the [Introduction to New USGS Services vignette](add link here), the [package index](https://doi-usgs.github.io/dataRetrieval/reference/index.html) and/or function documentation (e.g. `?read_waterdata_daily`). ```{r echo=FALSE} df <- data.frame( @@ -52,11 +52,11 @@ df <- data.frame( "readNWISpCode" ), New = c( - "read_USGS_samples", - "read_USGS_daily", - "read_USGS_monitoring_location", - "read_USGS_ts_meta", - "read_USGS_data", + "read_waterdata_samples", + "read_waterdata_daily", + "read_waterdata_monitoring_location", + "read_waterdata_ts_meta", + "read_waterdata", rep("", 10) ), "Available on (branch)" = c("main (CRAN)", "develop", "develop", "develop", "develop", rep("", 10)) @@ -88,9 +88,9 @@ What does this mean for water quality data users of `dataRetrieval`? Check out t ## Samples Data -There's a new set of functions that access the USGS "samples-data" services! If you are **only** interested in USGS discrete water quality data, you can use the `read_USGS_samples` function. +There's a new set of functions that access the USGS "samples-data" services! If you are **only** interested in USGS discrete water quality data, you can use the `read_waterdata_samples` function. -Read more about it in the vignette, [Introducing read_USGS_samples](https://doi-usgs.github.io/dataRetrieval/articles/samples_data.html). +Read more about it in the vignette, [Introducing read_waterdata_samples](https://doi-usgs.github.io/dataRetrieval/articles/samples_data.html). ## WQP diff --git a/vignettes/long_to_wide.Rmd b/vignettes/long_to_wide.Rmd index c1b3404c..d3ecceb9 100644 --- a/vignettes/long_to_wide.Rmd +++ b/vignettes/long_to_wide.Rmd @@ -68,7 +68,7 @@ sites <- c("USGS-04027000", "USGS-04063700") characteristic_names <- c("Phosphorus as phosphorus, water, filtered", "Orthophosphate as phosphorus, water, filtered" ) -nutrient_data <- read_USGS_samples(monitoringLocationIdentifier = sites, +nutrient_data <- read_waterdata_samples(monitoringLocationIdentifier = sites, characteristicUserSupplied = characteristic_names, dataProfile = "basicphyschem") diff --git a/vignettes/read_USGS_functions.Rmd b/vignettes/read_waterdata_functions.Rmd similarity index 84% rename from vignettes/read_USGS_functions.Rmd rename to vignettes/read_waterdata_functions.Rmd index d1eb03bc..3713f452 100644 --- a/vignettes/read_USGS_functions.Rmd +++ b/vignettes/read_waterdata_functions.Rmd @@ -1,5 +1,5 @@ --- -title: "Introduction to New USGS Services" +title: "Introduction to New USGS Water Data APIs" editor_options: chunk_output_type: console output: @@ -31,13 +31,13 @@ knitr::opts_chunk$set( ) ``` -As we bid adieu to the NWIS web services, we welcome a host of new web service offering: the [USGS Water Data OGC APIs](https://api.waterdata.usgs.gov/ogcapi/v0/). This is a modern access point for USGS water data. The USGS will be modernizing [**all** of the NWIS web services](https://waterdata.usgs.gov/blog/api-whats-new-wdfn-apis/) in the near future. For each of these updates, `dataRetrieval` will create a new function to access the new services and deprecate functions for accessing the legacy services. +As we bid adieu to the NWIS web services, we welcome a host of new web service offering: the [USGS Water Data APIs](https://api.waterdata.usgs.gov/ogcapi/v0/). This is a modern access point for USGS water data. The USGS will be modernizing [all of the NWIS web services](https://waterdata.usgs.gov/blog/api-whats-new-wdfn-apis/) in the near future. For each of these updates, `dataRetrieval` will create a new function to access the new services and deprecate functions for accessing the legacy services. -This document will introduce each new function (note as time goes on, we'll update this document to include additional functions). Note the timeline for the NWIS servers being shut down is currently very uncertain. We'd recommend incorporating these new functions as soon as possible to avoid future headaches. +This document will introduce each new function (note as time goes on, we'll update this document to include additional functions). The timeline for the NWIS servers being shut down is currently very uncertain. We'd recommend incorporating these new functions as soon as possible to avoid future headaches. # New Features -Each new "API endpoint" will deliver a new type of USGS data. Currently the available endpoints are "monitoring-locations", "time-series-metadata", and "daily". All of these endpoints offer some new features that the legacy services did not have: +Each new "API endpoint" will deliver a new type of USGS water data. Currently the available endpoints are "monitoring-locations", "time-series-metadata", and "daily". All of these endpoints offer some new features that the legacy services did not have: ## Flexible Queries @@ -68,7 +68,7 @@ You can use `usethis::edit_r_environ()` to edit find and open your .Renviron fil ## Contextual Query Language Support -Supports [Contextual Query Language](https://www.loc.gov/standards/sru/cql/) (CQL2) syntax for flexible queries. We'll show how to use the `read_USGS_data` function to make specific CQL2 queries. +Supports [Contextual Query Language](https://www.loc.gov/standards/sru/cql/) (CQL2) syntax for flexible queries. We'll show how to use the `read_waterdata` function to make specific CQL2 queries. ## Simple Features @@ -87,7 +87,7 @@ There are at least 2 ways to deal with this. One is to manually split the data r Example: ```{r} -ohio <- read_USGS_monitoring_location(state_name = "Ohio", +ohio <- read_waterdata_monitoring_location(state_name = "Ohio", site_type_code = "ST") ``` @@ -95,7 +95,7 @@ ohio <- read_USGS_monitoring_location(state_name = "Ohio", There are `r nrow(ohio)` rows returned that are stream sites in Ohio. If we tried to ask for all the discharge data over the last 7 days from that list of sites: ``` -ohio_discharge <- read_USGS_daily(monitoring_location_id = ohio$monitoring_location_id, +ohio_discharge <- read_waterdata_daily(monitoring_location_id = ohio$monitoring_location_id, parameter_code = "00060", time = "P7D") Error in `req_perform()`: @@ -106,7 +106,7 @@ Error in `req_perform()`: We could use the fact that the `ohio` data frame contains geospatial information, create a bounding box, and ask for that data like this: ```{r} -ohio_discharge <- read_USGS_daily(bbox = sf::st_bbox(ohio), +ohio_discharge <- read_waterdata_daily(bbox = sf::st_bbox(ohio), parameter_code = "00060", time = "P7D") @@ -123,7 +123,7 @@ site_list <- split(big_vector_of_sites, ceiling(seq_along(big_vector_of_sites)/2 data_returned <- data.frame() for(sites in site_list){ - df_sites <- read_USGS_daily(monitoring_location_id = sites, + df_sites <- read_waterdata_daily(monitoring_location_id = sites, parameter_code = "00060", time = "P7D") if(nrow(df_sites) == 0){ @@ -155,9 +155,9 @@ Each API endpoint natively returns a column named "id". The results of the "id" Therefore, `dataRetrieval` functions will rename the "id" column to whatever it is referred to in other functions. Here are the id translations: ```{r echo=FALSE} -df <- dplyr::tibble(Function = c("read_USGS_monitoring_location", - "read_USGS_ts_meta", - "read_USGS_daily"), +df <- dplyr::tibble(Function = c("read_waterdata_monitoring_location", + "read_waterdata_ts_meta", + "read_waterdata_daily"), "ID returned" = c("monitoring_location_id", "time_series_id", "daily_id")) @@ -170,12 +170,12 @@ If a user would prefer the columns to come back as "id", they can specify that u ```{r} site <- "USGS-02238500" -site_1 <- read_USGS_monitoring_location(monitoring_location_id = site, +site_1 <- read_waterdata_monitoring_location(monitoring_location_id = site, properties = c("monitoring_location_id", "state_name", "country_name")) names(site_1) -site_2 <- read_USGS_monitoring_location(monitoring_location_id = site, +site_2 <- read_waterdata_monitoring_location(monitoring_location_id = site, properties = c("id", "state_name", "country_name")) @@ -190,7 +190,7 @@ As new API endpoints come online, this section will be updated with any `dataRet ## Monitoring Location -The `read_USGS_monitoring_location` function replaces the `readNWISsite` function. +The `read_waterdata_monitoring_location` function replaces the `readNWISsite` function. `r dataRetrieval:::get_description("monitoring-locations")` @@ -199,7 +199,7 @@ To access these services on a web browser, go to ## Time Series Metadata -The `read_USGS_ts_meta` function replaces the `whatNWISdata` function. +The `read_waterdata_ts_meta` function replaces the `whatNWISdata` function. `r dataRetrieval:::get_description("time-series-metadata")` @@ -262,7 +262,7 @@ To access these services on a web browser, go to . @@ -340,7 +340,7 @@ cql <- '{ ] }' -sites_mn_wi <- read_USGS_data(service = "monitoring-locations", +sites_mn_wi <- read_waterdata(service = "monitoring-locations", CQL = cql) ``` @@ -368,7 +368,7 @@ leaflet(data = sites_mn_wi |> ## Discrete Samples -Discrete USGS water quality can be accessed via the `read_USGS_samples` function. While this is a new, modern USGS endpoint, it is not served in the same infrastructure as the rest of these new advertised functions. See [Samples Data](articles/samples_data.html)) for information on accessing USGS discrete water quality data. +Discrete USGS water quality can be accessed via the `read_waterdata_samples` function. While this is a new, modern USGS endpoint, it is not served in the same infrastructure as the rest of these new advertised functions. See [Samples Data](articles/samples_data.html)) for information on accessing USGS discrete water quality data. # Notes on dataRetrieval development @@ -376,7 +376,7 @@ Discrete USGS water quality can be accessed via the `read_USGS_samples` function ### Style -New functions will use a "snake case", such as `read_USGS_samples`. Older functions use camel case, such as `readNWISdv`. The difference is the underscore between words. This should be a handy way to tell the difference between newer modern data access, and the older traditional functions. +New functions will use a "snake case", such as `read_waterdata_samples`. Older functions use camel case, such as `readNWISdv`. The difference is the underscore between words. This should be a handy way to tell the difference between newer modern data access, and the older traditional functions. ### Structure diff --git a/vignettes/samples_data.Rmd b/vignettes/samples_data.Rmd index 32d9043b..1b60d1e5 100644 --- a/vignettes/samples_data.Rmd +++ b/vignettes/samples_data.Rmd @@ -1,5 +1,5 @@ --- -title: "Introducing read_USGS_samples" +title: "Introducing read_waterdata_samples" author: Laura A. DeCicco editor_options: chunk_output_type: console @@ -8,7 +8,7 @@ output: toc: true number_sections: true vignette: > - %\VignetteIndexEntry{Introducing read_USGS_samples} + %\VignetteIndexEntry{Introducing read_waterdata_samples} \usepackage[utf8]{inputenc} %\VignetteEngine{knitr::rmarkdown} --- @@ -91,7 +91,7 @@ This is a modern access point for USGS discrete water quality data. The USGS is ### Style -New functions will use a "snake case", such as "read_USGS_samples". Older functions use camel case, such as "readNWISdv". The difference is the underscore between words. This should be a handy way to tell the difference between newer modern data access, and the older traditional functions. +New functions will use a "snake case", such as "read_waterdata_samples". Older functions use camel case, such as "readNWISdv". The difference is the underscore between words. This should be a handy way to tell the difference between newer modern data access, and the older traditional functions. ### Structure @@ -137,11 +137,11 @@ DT::datatable(formatted_data_at_site, rownames = FALSE) We see there's `r data_at_site$resultCount[data_at_site$characteristicUserSupplied == "Phosphorus as phosphorus, water, unfiltered"]` filtered phosphorus values available. Note that if we ask for a simple characteristic = "Phosphorus", we'd get back both filtered and unfiltered, which might not be appropriate to mix together in an analysis. "characteristicUserSupplied" allows us to query by a very specific set of data. It is similar to a long-form USGS parameter code. -To get that data, use the `read_USGS_samples` function: +To get that data, use the `read_waterdata_samples` function: ```{r} user_char <- "Phosphorus as phosphorus, water, unfiltered" -phos_data <- read_USGS_samples(monitoringLocationIdentifier = site, +phos_data <- read_waterdata_samples(monitoringLocationIdentifier = site, characteristicUserSupplied = user_char) ``` @@ -150,7 +150,7 @@ Inspecting phos_data, there are `r ncol(phos_data)` columns (!). That is because Instead of using the "Full physical chemical" profile, we could ask for the "Narrow" profile, which contains fewer columns: ```{r} -phos_narrow <- read_USGS_samples(monitoringLocationIdentifier = site, +phos_narrow <- read_waterdata_samples(monitoringLocationIdentifier = site, characteristicUserSupplied = user_char, dataProfile = "narrow") ``` @@ -206,7 +206,7 @@ North and south are latitude values; east and west are longitude values. A vecto bbox <- c(-90.8, 44.2, -89.9, 45.0) user_char <- "Phosphorus as phosphorus, water, unfiltered" -bbox_sites <- read_USGS_samples(boundingBox = bbox, +bbox_sites <- read_waterdata_samples(boundingBox = bbox, characteristicUserSupplied = user_char, dataType = "locations", dataProfile = "site") @@ -224,7 +224,7 @@ Hydrologic Unit Codes (HUCs) identify physical areas within the US that drain to ```{r} -huc_sites <- read_USGS_samples(hydrologicUnit = "070700", +huc_sites <- read_waterdata_samples(hydrologicUnit = "070700", characteristicUserSupplied = user_char, dataType = "locations", dataProfile = "site") @@ -242,7 +242,7 @@ map_it(huc_sites) Location latitude (pointLocationLatitude) and longitude (pointLocationLongitude), and the radius (pointLocationWithinMiles) are required for this geographic filter: ```{r} -point_sites <- read_USGS_samples(pointLocationLatitude = 43.074680, +point_sites <- read_waterdata_samples(pointLocationLatitude = 43.074680, pointLocationLongitude = -89.428054, pointLocationWithinMiles = 20, characteristicUserSupplied = user_char, @@ -265,7 +265,7 @@ run `check_USGS_sample_params("counties")`. The "Fips" values can be created usi dane_county <- countyCdLookup("WI", "Dane", outputType = "fips") -county_sites <- read_USGS_samples(countyFips = dane_county, +county_sites <- read_waterdata_samples(countyFips = dane_county, characteristicUserSupplied = user_char, dataType = "locations", dataProfile = "site") @@ -286,7 +286,7 @@ run `check_USGS_sample_params("states")`. The "fips" values can be created using ```{r} state_fip <- stateCdLookup("WI", outputType = "fips") -state_sites <- read_USGS_samples(stateFips = state_fip, +state_sites <- read_waterdata_samples(stateFips = state_fip, characteristicUserSupplied = user_char, dataType = "locations", dataProfile = "site") @@ -379,7 +379,7 @@ Specify one or both of these fields to filter on the activity start date. The se For instance, let's grab Wisconsin sites that measured phosphorus in October or November of 2024: ```{r} -state_sites_recent <- read_USGS_samples(stateFips = state_fip, +state_sites_recent <- read_waterdata_samples(stateFips = state_fip, characteristicUserSupplied = user_char, dataType = "locations", activityStartDateLower = "2024-10-01", @@ -401,7 +401,7 @@ The above examples showed how to find sites within a geographic filter. We can u ```{r} dane_county <- countyCdLookup("WI", "Dane") -county_lake_sites <- read_USGS_samples(countyFips = dane_county, +county_lake_sites <- read_waterdata_samples(countyFips = dane_county, characteristicUserSupplied = user_char, siteTypeName = "Lake, Reservoir, Impoundment", dataType = "locations", diff --git a/vignettes/tutorial.Rmd b/vignettes/tutorial.Rmd index dbe1faae..e189728f 100644 --- a/vignettes/tutorial.Rmd +++ b/vignettes/tutorial.Rmd @@ -49,7 +49,7 @@ vignette("dataRetrieval", package = "dataRetrieval") Additionally, each function has a help file. These can be accessed by typing a question mark, followed by the function name in the R console: ```{r echo=TRUE, eval=FALSE} -?read_USGS_daily +?read_waterdata_daily ``` Each function's help file has working examples to demonstrate the usage. The examples may have comments "## Not run". These examples CAN be run, they just are not run by the CRAN maintainers due to the external service calls. @@ -95,14 +95,14 @@ There are many types of data served from NWIS. To understand how the services ar * * USGS functions will slowly replace NWIS functions - * `read_USGS_samples` has replaced `readNWISqw` - * `read_USGS_daily` can replace `readNWISdv` - * `read_USGS_monitoring_location` can replace `readNWISsite` - * `read_USGS_ts_meta` can replace `whatNWISdata` + * `read_waterdata_samples` has replaced `readNWISqw` + * `read_waterdata_daily` can replace `readNWISdv` + * `read_waterdata_monitoring_location` can replace `readNWISsite` + * `read_waterdata_ts_meta` can replace `whatNWISdata` * Discrete water quality data: * WQP functions should be used when accessing non-USGS discrete water quality data - * `read_USGS_samples` should be used for USGS data + * `read_waterdata_samples` should be used for USGS data # NWIS Data: Current NWIS offerings @@ -110,16 +110,16 @@ There are many types of data served from NWIS. To understand how the services ar | data_type_cd |Function| Data description | Replacement Function | |--------|:-------|------:|-------:| |uv|[readNWISuv](https://doi-usgs.github.io/dataRetrieval/reference/readNWISuv.html)|Continuous data| None yet | -|dv|[readNWISdv](https://doi-usgs.github.io/dataRetrieval/reference/readNWISdv.html)|Daily aggregated | [read_USGS_daily](https://doi-usgs.github.io/dataRetrieval/reference/read_USGS_daily.html) | +|dv|[readNWISdv](https://doi-usgs.github.io/dataRetrieval/reference/readNWISdv.html)|Daily aggregated | [read_waterdata_daily](https://doi-usgs.github.io/dataRetrieval/reference/read_waterdata_daily.html) | |gwlevels|[readNWISgwl](https://doi-usgs.github.io/dataRetrieval/reference/readNWISgwl.html)|Groundwater levels | None yet | -|site|[readNWISsite](https://doi-usgs.github.io/dataRetrieval/reference/readNWISsite.html)|Site metadata| [read_USGS_monitoring_location](https://doi-usgs.github.io/dataRetrieval/reference/read_USGS_monitoring_location.html) | +|site|[readNWISsite](https://doi-usgs.github.io/dataRetrieval/reference/readNWISsite.html)|Site metadata| [read_waterdata_monitoring_location](https://doi-usgs.github.io/dataRetrieval/reference/read_waterdata_monitoring_location.html) | |pcode|[readNWISpCode](https://doi-usgs.github.io/dataRetrieval/reference/readNWISpCode.html)|Parameter code metadata | None yet | |stat|[readNWISstat](https://doi-usgs.github.io/dataRetrieval/reference/readNWISstat.html)| Site statistics | None yet | |rating|[readNWISrating](https://doi-usgs.github.io/dataRetrieval/reference/readNWISrating.html)| Rating curves| None yet | |peak|[readNWISpeak](https://doi-usgs.github.io/dataRetrieval/reference/readNWISpeak.html)|Peak flow| None yet | |use|[readNWISuse](https://doi-usgs.github.io/dataRetrieval/reference/readNWISuse.html)|Water Use| None yet | |meas|[readNWISmeas](https://doi-usgs.github.io/dataRetrieval/reference/readNWISmeas.html)|Discrete surface water| None yet | -| | [readNWISdata](https://doi-usgs.github.io/dataRetrieval/reference/readNWISdata.html) | General data import for NWIS| [read_USGS_data](https://doi-usgs.github.io/dataRetrieval/reference/read_USGS_data.html) | +| | [readNWISdata](https://doi-usgs.github.io/dataRetrieval/reference/readNWISdata.html) | General data import for NWIS| [read_waterdata](https://doi-usgs.github.io/dataRetrieval/reference/read_waterdata.html) | ## USGS Basic Retrievals @@ -201,7 +201,7 @@ pCode <- "00060" start.date <- "2023-10-01" end.date <- "2024-09-30" -pheasant <- read_USGS_daily(monitoring_location_id = siteNo, +pheasant <- read_waterdata_daily(monitoring_location_id = siteNo, parameter_code = pCode, time = c(start.date, end.date)) ``` @@ -224,12 +224,12 @@ ts <- ggplot( ts ``` -Then we can use the `readNWISpCode` and `read_USGS_monitoring_location` functions to create better labels: +Then we can use the `readNWISpCode` and `read_waterdata_monitoring_location` functions to create better labels: ```{r echo=TRUE, eval=TRUE, fig.height=3.5} parameterInfo <- readNWISpCode(pCode) -siteInfo <- read_USGS_monitoring_location(siteNo) +siteInfo <- read_waterdata_monitoring_location(siteNo) ts <- ts + xlab("") + @@ -250,7 +250,7 @@ First verify that the data you think is available is actually associated with th library(dplyr) site <- "USGS-05407000" -ts_data_available <- read_USGS_ts_meta(monitoring_location_id = site) +ts_data_available <- read_waterdata_ts_meta(monitoring_location_id = site) data_available <- ts_data_available |> @@ -272,13 +272,13 @@ datatable(data_available, ``` -The time series that have "Instantaneous" in the computation_identifier column will be available in the instantaneous data service (currently `readNWISuv`), and the rest of the data will be available in the daily service (`read_USGS_daily`). +The time series that have "Instantaneous" in the computation_identifier column will be available in the instantaneous data service (currently `readNWISuv`), and the rest of the data will be available in the daily service (`read_waterdata_daily`). ```{r eval=FALSE, echo=TRUE} dv_pcodes <- data_available$parameter_code[data_available$computation_identifier != "Instantaneous"] stat_cds <- data_available$statistic_id[data_available$computation_identifier != "Instantaneous"] -dv_data <- read_USGS_daily(monitoring_location_id = site, +dv_data <- read_waterdata_daily(monitoring_location_id = site, parameter_code = unique(dv_pcodes), statistic_id = unique(stat_cds)) @@ -313,10 +313,10 @@ datatable(discrete_data_available, ``` -The discrete water quality data can be accessed with the `read_USGS_samples` function: +The discrete water quality data can be accessed with the `read_waterdata_samples` function: ```{r eval=FALSE, echo=TRUE} -samples_data <- read_USGS_samples(monitoringLocationIdentifier = site, +samples_data <- read_waterdata_samples(monitoringLocationIdentifier = site, dataProfile = "basicphyschem") ``` @@ -362,8 +362,8 @@ This is all great when you know your site numbers. What do you do when you don't There are 2 `dataRetrieval` functions that help with USGS data discovery: -* `read_USGS_monitoring_location` finds sites within a specified filter -* `read_USGS_ts_meta` summarizes the time series meta data +* `read_waterdata_monitoring_location` finds sites within a specified filter +* `read_waterdata_ts_meta` summarizes the time series meta data And 2 functions that help with discover in WQP: @@ -377,8 +377,8 @@ Here are a few examples: ```{r eval=FALSE} # Daily temperature in Ohio -ohio_sites <- read_USGS_monitoring_location(state_name = "Ohio") -ohio_ts_meta <- read_USGS_ts_meta(bbox = sf::st_bbox(ohio_sites), +ohio_sites <- read_waterdata_monitoring_location(state_name = "Ohio") +ohio_ts_meta <- read_waterdata_ts_meta(bbox = sf::st_bbox(ohio_sites), parameter_code = "00010") # Real-time discharge at a site From 8638c2fb0333245543cb0a7ba94dfd67f5d6b86e Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 13 Jun 2025 10:51:37 -0500 Subject: [PATCH 108/117] Add warning to setAccess --- NAMESPACE | 2 +- R/readNWISdata.R | 2 +- R/read_waterdata_samples.R | 4 ++-- R/setAccess.R | 6 ++++++ R/whatNWISdata.R | 8 ++++---- R/whatNWISsites.R | 8 ++++---- man/readNWISdata.Rd | 2 +- ...ize_USGS_samples.Rd => summarize_waterdata_samples.Rd} | 8 ++++---- man/whatNWISdata.Rd | 4 ++-- man/whatNWISsites.Rd | 4 ++-- tests/testthat/tests_samples.R | 4 ++-- vignettes/samples_data.Rmd | 8 ++++---- 12 files changed, 33 insertions(+), 27 deletions(-) rename man/{summarize_USGS_samples.Rd => summarize_waterdata_samples.Rd} (83%) diff --git a/NAMESPACE b/NAMESPACE index 622427bc..1908b9d7 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -53,7 +53,7 @@ export(renameNWISColumns) export(setAccess) export(stateCd) export(stateCdLookup) -export(summarize_USGS_samples) +export(summarize_waterdata_samples) export(whatNWISdata) export(whatNWISsites) export(whatWQPdata) diff --git a/R/readNWISdata.R b/R/readNWISdata.R index 6533b236..f56ccfcc 100644 --- a/R/readNWISdata.R +++ b/R/readNWISdata.R @@ -71,7 +71,7 @@ #' queryTime \tab POSIXct \tab The time the data was returned \cr #' } #' -#' @seealso [read_waterdata_data()] +#' @seealso [read_waterdata()] #' @export #' @examplesIf is_dataRetrieval_user() #' \donttest{ diff --git a/R/read_waterdata_samples.R b/R/read_waterdata_samples.R index e225efe0..94edd496 100644 --- a/R/read_waterdata_samples.R +++ b/R/read_waterdata_samples.R @@ -480,10 +480,10 @@ read_waterdata_samples <- function(monitoringLocationIdentifier = NA, #' \donttest{ #' monitoringLocationIdentifier <- "USGS-04074950" #' -#' what_data <- summarize_USGS_samples(monitoringLocationIdentifier) +#' what_data <- summarize_waterdata_samples(monitoringLocationIdentifier) #' #' } -summarize_USGS_samples <- function(monitoringLocationIdentifier){ +summarize_waterdata_samples <- function(monitoringLocationIdentifier){ if(length(monitoringLocationIdentifier) > 1){ stop("Summary service only available for one site at a time.") diff --git a/R/setAccess.R b/R/setAccess.R index 643ac7ed..ac33a195 100644 --- a/R/setAccess.R +++ b/R/setAccess.R @@ -27,12 +27,18 @@ setAccess <- function(access = "public") { if (access == "internal") { pkg.env$access <- "3" message("setting access to internal") + warning("Internal access is slated for decommision. +Please contact comptools@usgs.gov for more information.") } else if (access == "cooperator") { pkg.env$access <- "1" message("setting access to cooperator") + warning("Cooperator access is slated for decommision. +Please contact comptools@usgs.gov for more information.") } else if (access == "USGS") { pkg.env$access <- "2" message("setting access to all USGS Water Science Centers") + warning("Water Science Center access is slated for decommision. +Please contact comptools@usgs.gov for more information.") } else { pkg.env$access <- NULL message("setting access to public") diff --git a/R/whatNWISdata.R b/R/whatNWISdata.R index 13e29f76..dfdf1a23 100644 --- a/R/whatNWISdata.R +++ b/R/whatNWISdata.R @@ -63,10 +63,10 @@ #' queryTime \tab POSIXct \tab The time the data was returned \cr #' } #' @export -#' @seealso [read_USGS_ts_meta()] +#' @seealso [read_waterdata_ts_meta()] #' @examples #' -#' # see ?read_USGS_ts_meta +#' # see ?read_waterdata_ts_meta #' #' #site1 <- whatWQPsamples(siteid = "USGS-01594440") #' @@ -83,9 +83,9 @@ whatNWISdata <- function(..., convertType = TRUE) { prewarned <- FALSE - .Deprecated(new = "read_USGS_ts_meta", + .Deprecated(new = "read_waterdata_ts_meta", package = "dataRetrieval", - msg = "NWIS servers are slated for decommission. Please begin to migrate to read_USGS_ts_meta") + msg = "NWIS servers are slated for decommission. Please begin to migrate to read_waterdata_ts_meta") if ("service" %in% names(matchReturn)) { service <- matchReturn$service diff --git a/R/whatNWISsites.R b/R/whatNWISsites.R index cb15ce73..8ece54c8 100644 --- a/R/whatNWISsites.R +++ b/R/whatNWISsites.R @@ -26,19 +26,19 @@ #' queryTime \tab POSIXct \tab The time the data was returned \cr #' } #' @export -#' @seealso [read_USGS_monitoring_location()] +#' @seealso [read_waterdata_monitoring_location()] #' #' @examples #' -#' # see ?read_USGS_monitoring_location +#' # see ?read_waterdata_monitoring_location #' #siteListPhos <- whatNWISsites(stateCd = "OH", parameterCd = "00665") #' #oneSite <- whatNWISsites(sites = "05114000") #' whatNWISsites <- function(...) { - .Deprecated(new = "read_USGS_monitoring_location", + .Deprecated(new = "read_waterdata_monitoring_location", package = "dataRetrieval", - msg = "NWIS servers are slated for decommission. Please begin to migrate to read_USGS_monitoring_location") + msg = "NWIS servers are slated for decommission. Please begin to migrate to read_waterdata_monitoring_location") matchReturn <- convertLists(...) diff --git a/man/readNWISdata.Rd b/man/readNWISdata.Rd index cb7116f6..1201b6ac 100644 --- a/man/readNWISdata.Rd +++ b/man/readNWISdata.Rd @@ -179,5 +179,5 @@ peak_data <- readNWISdata( \dontshow{\}) # examplesIf} } \seealso{ -\code{\link[=read_waterdata_data]{read_waterdata_data()}} +\code{\link[=read_waterdata]{read_waterdata()}} } diff --git a/man/summarize_USGS_samples.Rd b/man/summarize_waterdata_samples.Rd similarity index 83% rename from man/summarize_USGS_samples.Rd rename to man/summarize_waterdata_samples.Rd index 3541ed4c..488e7718 100644 --- a/man/summarize_USGS_samples.Rd +++ b/man/summarize_waterdata_samples.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/read_waterdata_samples.R -\name{summarize_USGS_samples} -\alias{summarize_USGS_samples} +\name{summarize_waterdata_samples} +\alias{summarize_waterdata_samples} \title{USGS Samples Summary Data} \usage{ -summarize_USGS_samples(monitoringLocationIdentifier) +summarize_waterdata_samples(monitoringLocationIdentifier) } \arguments{ \item{monitoringLocationIdentifier}{A monitoring location identifier has two parts, @@ -25,7 +25,7 @@ service described at \url{https://api.waterdata.usgs.gov/samples-data/docs}. \donttest{ monitoringLocationIdentifier <- "USGS-04074950" -what_data <- summarize_USGS_samples(monitoringLocationIdentifier) +what_data <- summarize_waterdata_samples(monitoringLocationIdentifier) } \dontshow{\}) # examplesIf} diff --git a/man/whatNWISdata.Rd b/man/whatNWISdata.Rd index a27c1a35..fcc94de5 100644 --- a/man/whatNWISdata.Rd +++ b/man/whatNWISdata.Rd @@ -74,7 +74,7 @@ of sites that have useful data. } \examples{ -# see ?read_USGS_ts_meta +# see ?read_waterdata_ts_meta #site1 <- whatWQPsamples(siteid = "USGS-01594440") @@ -88,7 +88,7 @@ of sites that have useful data. } \seealso{ -\code{\link[=read_USGS_ts_meta]{read_USGS_ts_meta()}} +\code{\link[=read_waterdata_ts_meta]{read_waterdata_ts_meta()}} } \keyword{USGS} \keyword{data} diff --git a/man/whatNWISsites.Rd b/man/whatNWISsites.Rd index 3f580bac..b4310020 100644 --- a/man/whatNWISsites.Rd +++ b/man/whatNWISsites.Rd @@ -37,11 +37,11 @@ Mapper format is used } \examples{ -# see ?read_USGS_monitoring_location +# see ?read_waterdata_monitoring_location #siteListPhos <- whatNWISsites(stateCd = "OH", parameterCd = "00665") #oneSite <- whatNWISsites(sites = "05114000") } \seealso{ -\code{\link[=read_USGS_monitoring_location]{read_USGS_monitoring_location()}} +\code{\link[=read_waterdata_monitoring_location]{read_waterdata_monitoring_location()}} } diff --git a/tests/testthat/tests_samples.R b/tests/testthat/tests_samples.R index e3ddef11..869734d8 100644 --- a/tests/testthat/tests_samples.R +++ b/tests/testthat/tests_samples.R @@ -66,11 +66,11 @@ test_that("samples-data project working", { expect_true(ncol(projectInfo) >= 0) }) -context("summary_USGS_samples") +context("summary_waterdata_samples") test_that("summary_USGS_samples working", { testthat::skip_on_cran() - site1 <- summarize_USGS_samples(monitoringLocationIdentifier = "USGS-01594440") + site1 <- summarize_waterdata_samples(monitoringLocationIdentifier = "USGS-01594440") expect_is(site1, "data.frame") }) diff --git a/vignettes/samples_data.Rmd b/vignettes/samples_data.Rmd index 1b60d1e5..7df9d779 100644 --- a/vignettes/samples_data.Rmd +++ b/vignettes/samples_data.Rmd @@ -112,13 +112,13 @@ And here is a link to the web service documentation: ## Retrieving data from a known site -Let's say we have a USGS site. We can check the data available at that site using `summarize_USGS_samples` like this: +Let's say we have a USGS site. We can check the data available at that site using `summarize_waterdata_samples` like this: ```{r} library(dataRetrieval) site <- "USGS-04183500" -data_at_site <- summarize_USGS_samples(monitoringLocationIdentifier = site) +data_at_site <- summarize_waterdata_samples(monitoringLocationIdentifier = site) ``` @@ -409,13 +409,13 @@ county_lake_sites <- read_waterdata_samples(countyFips = dane_county, ``` -There are only `r nrow(county_lake_sites)` lake sites measuring phosphorus in Dane County, WI. We can get a summary of the data at each site using the `summarize_USGS_samples` function. This function only accepts 1 site at a time: +There are only `r nrow(county_lake_sites)` lake sites measuring phosphorus in Dane County, WI. We can get a summary of the data at each site using the `summarize_waterdata_samples` function. This function only accepts 1 site at a time: ```{r message=FALSE} all_data <- data.frame() for(i in county_lake_sites$Location_Identifier){ - avail_i <- summarize_USGS_samples(monitoringLocationIdentifier = i) + avail_i <- summarize_waterdata_samples(monitoringLocationIdentifier = i) all_data <- avail_i |> filter(characteristicUserSupplied == user_char) |> bind_rows(all_data) From 1d9d8f79db4a9b73356ec26863571b47baff1bb9 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 13 Jun 2025 11:01:47 -0500 Subject: [PATCH 109/117] Update the readme --- NEWS | 5 ++++- README.Rmd | 10 +++++----- README.md | 26 +++++++++++++------------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/NEWS b/NEWS index c8981de2..98d03abd 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,11 @@ dataRetrieval 2.7.19 =================== -* Added read_USGS_daily, read_USGS_monitoring_location, read_USGS_ts_meta to access +* Added read_waterdata_daily, read_waterdata_monitoring_location, read_waterdata_ts_meta to access new USGS web services. * Added whisker and sf as dependencies. +* Renamed read_USGS_sample to read_waterdata_sample. +* Renamed summarize_USGS_sample to summarize_waterdata_sample. +* Added warning to setAccess for non-public endpoints. dataRetrieval 2.7.18 =================== diff --git a/README.Rmd b/README.Rmd index ae80bcc8..af74e5a4 100644 --- a/README.Rmd +++ b/README.Rmd @@ -39,15 +39,15 @@ If you have additional questions about these changes, email CompTools@usgs.gov. 1. Get instantaneous USGS data (for example, discharge sensor data). Start here: `?readNWISuv` -2. Get daily USGS data (for example, mean daily discharge). Start here: `?read_USGS_daily` +2. Get daily USGS data (for example, mean daily discharge). Start here: `?read_waterdata_daily` 3. Get USGS groundwater data. Start here: `?readNWISgwl` 4. Get discrete water quality data from a cooperative service that integrates publicly available water-quality data from the USGS, EPA, and over 400 state, federal, tribal, and local agencies. Start here: `?readWQPdata` -5. Get USGS discrete water quality data. Start here: `?read_USGS_samples` +5. Get USGS discrete water quality data. Start here: `?read_waterdata_samples` -6. Get metadata about USGS time series data, including instantaneous and daily data. Start here: `?read_USGS_ts_meta` +6. Get metadata about USGS time series data, including instantaneous and daily data. Start here: `?read_waterdata_ts_meta` 7. Find Hydro Network-Linked Data Index (NLDI) data. Start here: `?findNLDI` @@ -88,7 +88,7 @@ You can use `usethis::edit_r_environ()` to edit find and open your .Renviron fil ### Style -New functions will use a "snake case", such as "read_USGS_samples". Older functions use camel case, such as "readNWISdv". The difference is the underscore between words. This should be a handy way to tell the difference between newer modern data access, and the older traditional functions. +New functions will use a "snake case", such as "read_waterdata_samples". Older functions use camel case, such as "readNWISdv". The difference is the underscore between words. This should be a handy way to tell the difference between newer modern data access, and the older traditional functions. ### Structure @@ -96,7 +96,7 @@ Historically, we allowed users to customize their queries via the `...` argument ### Dependencies -As we develop functions for the modern USGS web services, we'll continue to explore updating package dependencies. +As we develop functions for the modern USGS Water Data APIs, we'll continue to explore updating package dependencies. ### Developmental workflow diff --git a/README.md b/README.md index a9a35dda..5125410f 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ If you have additional questions about these changes, email Start here: `?readNWISuv` 2. Get daily USGS data (for example, mean daily discharge). Start here: - `?read_USGS_daily` + `?read_waterdata_daily` 3. Get USGS groundwater data. Start here: `?readNWISgwl` @@ -41,10 +41,10 @@ If you have additional questions about these changes, email `?readWQPdata` 5. Get USGS discrete water quality data. Start here: - `?read_USGS_samples` + `?read_waterdata_samples` 6. Get metadata about USGS time series data, including instantaneous - and daily data. Start here: `?read_USGS_ts_meta` + and daily data. Start here: `?read_waterdata_ts_meta` 7. Find Hydro Network-Linked Data Index (NLDI) data. Start here: `?findNLDI` @@ -94,7 +94,7 @@ recognized. ### Style -New functions will use a “snake case”, such as “read_USGS_samples”. +New functions will use a “snake case”, such as “read_waterdata_samples”. Older functions use camel case, such as “readNWISdv”. The difference is the underscore between words. This should be a handy way to tell the difference between newer modern data access, and the older traditional @@ -116,8 +116,8 @@ able to build up argument lists to pass into the function. ### Dependencies -As we develop functions for the modern USGS web services, we’ll continue -to explore updating package dependencies. +As we develop functions for the modern USGS Water Data APIs, we’ll +continue to explore updating package dependencies. ### Developmental workflow @@ -157,17 +157,17 @@ citation(package = "dataRetrieval") #> #> De Cicco, L.A., Hirsch, R.M., Lorenz, D., Watkins, W.D., Johnson, M., #> 2025, dataRetrieval: R packages for discovering and retrieving water -#> data available from Federal hydrologic web services, v.2.7.18, +#> data available from Federal hydrologic web services, v.2.7.19, #> doi:10.5066/P9X4L3GE #> #> A BibTeX entry for LaTeX users is #> #> @Manual{, -#> author = {Laura DeCicco and Robert Hirsch and David Lorenz and Jordan Read and Jordan Walker and Lindsay Platt and David Watkins and David Blodgett and Mike Johnson and Aliesha Krall and Lee Stanish}, +#> author = {Laura DeCicco and Robert Hirsch and David Lorenz and Jordan Read and Jordan Walker and Lindsay Platt and David Watkins and David Blodgett and Mike Johnson and Aliesha Krall and Lee Stanish and Joeseph Zemmels and Elise Hinman and Michael Mahoney}, #> title = {dataRetrieval: R packages for discovering and retrieving water data available from U.S. federal hydrologic web services}, #> publisher = {U.S. Geological Survey}, #> address = {Reston, VA}, -#> version = {2.7.18}, +#> version = {2.7.19}, #> institution = {U.S. Geological Survey}, #> year = {2025}, #> doi = {10.5066/P9X4L3GE}, @@ -193,14 +193,14 @@ NWIScitation #> U.S. Geological Survey (2025). _National Water Information System data #> available on the World Wide Web (USGS Water Data for the Nation)_. #> doi:10.5066/F7P55KJN , Accessed Jun -#> 04, 2025, +#> 13, 2025, #> . print(NWIScitation, style = "Bibtex") #> @Manual{, #> title = {National Water Information System data available on the World Wide Web (USGS Water Data for the Nation)}, #> author = {{U.S. Geological Survey}}, #> doi = {10.5066/F7P55KJN}, -#> note = {Accessed Jun 04, 2025}, +#> note = {Accessed Jun 13, 2025}, #> year = {2025}, #> url = {https://waterservices.usgs.gov/nwis/dv/?site=09010500&format=waterml%2C1.1&ParameterCd=00060&StatCd=00003&startDT=1851-01-01}, #> } @@ -224,14 +224,14 @@ WQPcitation <- create_WQP_bib(SC) WQPcitation #> National Water Quality Monitoring Council (2025). _Water Quality #> Portal_. doi:10.5066/P9QRKUVJ , -#> Accessed Jun 04, 2025, +#> Accessed Jun 13, 2025, #> . print(WQPcitation, style = "Bibtex") #> @Manual{, #> title = {Water Quality Portal}, #> author = {{National Water Quality Monitoring Council}}, #> doi = {10.5066/P9QRKUVJ}, -#> note = {Accessed Jun 04, 2025}, +#> note = {Accessed Jun 13, 2025}, #> year = {2025}, #> url = {https://www.waterqualitydata.us/data/Result/search?siteid=USGS-05288705&count=no&pCode=00300&mimeType=csv}, #> } From b9cbf4ea991b9bfc3a50c33baa62c01ee08f624e Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 13 Jun 2025 11:19:17 -0500 Subject: [PATCH 110/117] missed a few? --- .Rbuildignore | 2 +- tests/testthat/tests_userFriendly_fxns.R | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.Rbuildignore b/.Rbuildignore index 8c7cdb4f..a4686bfb 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -90,7 +90,7 @@ vignettes/Status.Rmd vignettes/long_to_wide.Rmd vignettes/join_by_closest.Rmd vignettes/wqx3_development_plan.Rmd -vignettes/read_USGS_functions.Rmd +vignettes/read_waterdata_functions.Rmd vignettes/dataretrieval_discrete_changes_images/* ^ci$ ^public$ diff --git a/tests/testthat/tests_userFriendly_fxns.R b/tests/testthat/tests_userFriendly_fxns.R index ed36dce7..f81af9a9 100644 --- a/tests/testthat/tests_userFriendly_fxns.R +++ b/tests/testthat/tests_userFriendly_fxns.R @@ -83,11 +83,11 @@ test_that("peak, rating curves, surface-water measurements", { data <- readNWISmeas(siteNumbers) expect_is(data$agency_cd, "character") - siteINFO_USGS <- read_USGS_monitoring_location(monitoring_location_id = "USGS-05114000") + siteINFO_USGS <- read_waterdata_monitoring_location(monitoring_location_id = "USGS-05114000") expect_is(siteINFO_USGS$agency_code, "character") expect_equal(siteINFO_USGS$monitoring_location_id, "USGS-05114000") - siteINFOMulti_USGS <- read_USGS_monitoring_location(monitoring_location_id = c("USGS-05114000", + siteINFOMulti_USGS <- read_waterdata_monitoring_location(monitoring_location_id = c("USGS-05114000", "USGS-09423350")) expect_true(nrow(siteINFOMulti_USGS) == 2) @@ -95,9 +95,9 @@ test_that("peak, rating curves, surface-water measurements", { expect_is(Meas07227500.ex$measurement_dt, "Date") expect_is(Meas07227500.ex$measurement_dateTime, "POSIXct") - expect_equal(nrow(read_USGS_ts_meta(monitoring_location_id = "USGS-10312000", + expect_equal(nrow(read_waterdata_ts_meta(monitoring_location_id = "USGS-10312000", parameter_code = "50286")), 0) - expect_equal(ncol(read_USGS_ts_meta(monitoring_location_id = "USGS-10312000", + expect_equal(ncol(read_waterdata_ts_meta(monitoring_location_id = "USGS-10312000", parameter_code = "50286", properties = c("geometry", "id", "unit_of_measure", @@ -118,7 +118,7 @@ test_that("peak, rating curves, surface-water measurements", { convertType = FALSE)) }) -test_that("read_USGS_daily", { +test_that("read_waterdata_daily", { testthat::skip_on_cran() siteNumber <- "USGS-04085427" @@ -126,12 +126,12 @@ test_that("read_USGS_daily", { endDate <- "2012-06-30" pCode <- "00060" - raw_USGS_daily <- read_USGS_daily(monitoring_location_id = siteNumber, + raw_USGS_daily <- read_waterdata_daily(monitoring_location_id = siteNumber, parameter_code = pCode, time = c(startDate, endDate)) expect_is(raw_USGS_daily$time, "Date") - raw_USGS_TempMeanMax <- read_USGS_daily(monitoring_location_id = siteNumber, + raw_USGS_TempMeanMax <- read_waterdata_daily(monitoring_location_id = siteNumber, parameter_code = c("00010", "00060"), time = c(startDate, endDate), statistic_id = c("00001", "00003")) @@ -140,7 +140,7 @@ test_that("read_USGS_daily", { expect_true(length(unique(raw_USGS_TempMeanMax$statistic_id)) == 2) expect_true(length(unique(raw_USGS_TempMeanMax$monitoring_location_id)) == 1) - raw_USGS_MultiSites <- read_USGS_daily(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), + raw_USGS_MultiSites <- read_waterdata_daily(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), parameter_code = c("00010", "00060"), time = c(startDate, endDate), statistic_id = c("00001", "00003")) @@ -149,7 +149,7 @@ test_that("read_USGS_daily", { site <- "05212700" - notActiveUSGS <- read_USGS_daily(monitoring_location_id = paste0("USGS-", site), + notActiveUSGS <- read_waterdata_daily(monitoring_location_id = paste0("USGS-", site), parameter_code = "00060", time = c("2014-01-01", "2014-01-07")) expect_true(nrow(notActiveUSGS) == 0) From da86ce1497c97d56888873ade19e81985bb25e61 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 13 Jun 2025 11:42:24 -0500 Subject: [PATCH 111/117] missed a few --- NAMESPACE | 6 +- R/read_waterdata_samples.R | 132 ++++++++++++++---- ...ms.Rd => check_waterdata_sample_params.Rd} | 22 +-- ... => construct_waterdata_sample_request.Rd} | 22 +-- man/read_waterdata_samples.Rd | 41 +++++- man/summarize_waterdata_samples.Rd | 3 + tests/testthat/tests_samples.R | 4 +- tests/testthat/tests_userFriendly_fxns.R | 16 +-- vignettes/samples_data.Rmd | 22 +-- vignettes/tutorial.Rmd | 4 +- 10 files changed, 187 insertions(+), 85 deletions(-) rename man/{check_USGS_sample_params.Rd => check_waterdata_sample_params.Rd} (59%) rename man/{construct_USGS_sample_request.Rd => construct_waterdata_sample_request.Rd} (89%) diff --git a/NAMESPACE b/NAMESPACE index 1908b9d7..c396b414 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -4,12 +4,12 @@ export(addWaterYear) export(calcWaterYear) export(checkWQPdates) export(check_OGC_requests) -export(check_USGS_sample_params) +export(check_waterdata_sample_params) export(constructNWISURL) export(constructUseURL) export(constructWQPURL) -export(construct_USGS_sample_request) export(construct_api_requests) +export(construct_waterdata_sample_request) export(countyCd) export(countyCdLookup) export(create_NWIS_bib) @@ -44,6 +44,7 @@ export(readNWISuv) export(readWQPdata) export(readWQPqw) export(readWQPsummary) +export(read_USGS_samples) export(read_waterdata) export(read_waterdata_daily) export(read_waterdata_monitoring_location) @@ -53,6 +54,7 @@ export(renameNWISColumns) export(setAccess) export(stateCd) export(stateCdLookup) +export(summarize_USGS_samples) export(summarize_waterdata_samples) export(whatNWISdata) export(whatNWISsites) diff --git a/R/read_waterdata_samples.R b/R/read_waterdata_samples.R index 94edd496..0875d49b 100644 --- a/R/read_waterdata_samples.R +++ b/R/read_waterdata_samples.R @@ -14,9 +14,9 @@ #' numbers without an agency prefix are assumed to have the prefix USGS. #' @param activityMediaName Sample media refers to the environmental medium that #' was sampled or analyzed. See available options by running -#' `check_USGS_sample_params("samplemedia")$activityMedia`. +#' `check_waterdata_sample_params("samplemedia")$activityMedia`. #' @param siteTypeCode Site type code query parameter. See available -#' options by running `check_USGS_sample_params("sitetype")$typeCode`. +#' options by running `check_waterdata_sample_params("sitetype")$typeCode`. #' @param boundingBox North and South are latitude values; East and West are longitude values. #' A vector of 4 (west, south, east, north) is expected. #' An example would be: c(-92.8, 44.2, -88.9, 46.0). @@ -35,7 +35,7 @@ #' records that match the date. #' @param characteristicGroup Characteristic group is a broad category describing the sample. #' See available options by running -#' `check_USGS_sample_params("characteristicgroup")$characteristicGroup`. +#' `check_waterdata_sample_params("characteristicgroup")$characteristicGroup`. #' @param characteristicUserSupplied Observed property is the USGS term for the #' constituent sampled and the property name gives a detailed description of what #' was sampled. Observed property is mapped to characteristicUserSupplied and replaces @@ -45,21 +45,21 @@ #' . #' @param characteristic Characteristic is a specific category describing the sample. #' See available options by running -#' `check_USGS_sample_params("characteristics")$characteristicName`. +#' `check_waterdata_sample_params("characteristics")$characteristicName`. #' @param stateFips State query parameter. To get a list of available state fips, -#' run `check_USGS_sample_params("states")`. The "fips" can be created using the function +#' run `check_waterdata_sample_params("states")`. The "fips" can be created using the function #' `stateCdLookup` - for example: `stateCdLookup("WI", "fips")`. #' FIPs codes for states take the format: #' CountryAbbrev:StateNumber, like US:55 for Wisconsin. #' @param countyFips County query parameter. To get a list of available counties, -#' run `check_USGS_sample_params("counties")`. The "Fips" can be created using the function +#' run `check_waterdata_sample_params("counties")`. The "Fips" can be created using the function #' `countyCdLookup` - for example: `countyCdLookup("WI", "Dane", "fips")` #' for Dane County, WI. #' FIPs codes for counties take the format: #' CountryAbbrev:StateNumber:CountyNumber, like US:55:025 for Dane County, WI. #' @param countryFips Country query parameter. Do not set redundant parameters. #' If another query parameter contains the country information, leave this parameter -#' set to the default NA. See available options by running `check_USGS_sample_params("countries")`, +#' set to the default NA. See available options by running `check_waterdata_sample_params("countries")`, #' where the "id" field contains the value to use in the countryFips input. #' @param projectIdentifier Project identifier query parameter. This information #' would be needed from prior project information. @@ -68,7 +68,7 @@ #' @param siteTypeName Site type name query parameter. See available #' options by running `check_param("sitetype")$typeName`. #' @param usgsPCode USGS parameter code. See available options by running -#' `check_USGS_sample_params("characteristics")$parameterCode`. +#' `check_waterdata_sample_params("characteristics")$parameterCode`. #' @param pointLocationLatitude Latitude for a point/radius query (decimal degrees). Must be used #' with pointLocationLongitude and pointLocationWithinMiles. #' @param pointLocationLongitude Longitude for a point/radius query (decimal degrees). Must be used @@ -91,13 +91,13 @@ #' @examplesIf is_dataRetrieval_user() #' #' \donttest{ -#' req <- construct_USGS_sample_request( +#' req <- construct_waterdata_sample_request( #' monitoringLocationIdentifier = "USGS-04074950", #' characteristicUserSupplied = "pH, water, unfiltered, field") #' rawData <- importWQP(req) #' #' } -construct_USGS_sample_request <- function(monitoringLocationIdentifier = NA, +construct_waterdata_sample_request <- function(monitoringLocationIdentifier = NA, siteTypeCode = NA, boundingBox = NA, hydrologicUnit = NA, @@ -176,36 +176,36 @@ construct_USGS_sample_request <- function(monitoringLocationIdentifier = NA, if(all(!is.na(siteTypeCode))){ siteTypeCode <- match.arg(siteTypeCode, - check_USGS_sample_params("sitetype")$typeCode, + check_waterdata_sample_params("sitetype")$typeCode, several.ok = TRUE) } if(all(!is.na(activityMediaName))){ activityMediaName <- match.arg(activityMediaName, - check_USGS_sample_params("samplemedia")$activityMedia, + check_waterdata_sample_params("samplemedia")$activityMedia, several.ok = TRUE) } if(all(!is.na(characteristicGroup))){ characteristicGroup <- match.arg(characteristicGroup, - check_USGS_sample_params("characteristicgroup")$characteristicGroup, + check_waterdata_sample_params("characteristicgroup")$characteristicGroup, several.ok = TRUE) } if(all(!is.na(countryFips))){ countryFips <- match.arg(countryFips, - check_USGS_sample_params("countries")$countryCode, + check_waterdata_sample_params("countries")$countryCode, several.ok = TRUE) } if(all(!is.na(siteTypeName))){ siteTypeName <- match.arg(siteTypeName, - check_USGS_sample_params("sitetype")$typeLongName, + check_waterdata_sample_params("sitetype")$typeLongName, several.ok = TRUE) } if(all(!is.na(stateFips))){ - states <- check_USGS_sample_params("states") + states <- check_waterdata_sample_params("states") state_codes <- paste(states$countryCode, states$fipsCode, sep = ":") stateFips <- match.arg(stateFips, state_codes, @@ -213,10 +213,10 @@ construct_USGS_sample_request <- function(monitoringLocationIdentifier = NA, } if(all(!is.na(countyFips))){ - states <- check_USGS_sample_params("states") + states <- check_waterdata_sample_params("states") state_codes <- paste(states$countryCode, states$fipsCode, sep = ":") - counties <- check_USGS_sample_params("counties") + counties <- check_waterdata_sample_params("counties") state_cd <- stats::setNames(states$fipsCode, states$stateAbbrev) county_codes <- paste(counties$countryCode, @@ -329,19 +329,19 @@ explode_query <- function(baseURL, POST = FALSE, x){ #' @examplesIf is_dataRetrieval_user() #' #' \donttest{ -#' groups <- check_USGS_sample_params("characteristicgroup") -#' states <- check_USGS_sample_params("states") -#' countries <- check_USGS_sample_params("countries") -#' counties <- check_USGS_sample_params("counties") -#' sitetypes <- check_USGS_sample_params("sitetype") -#' samplemedia <- check_USGS_sample_params("samplemedia") -#' characteristics <- check_USGS_sample_params("characteristics", +#' groups <- check_waterdata_sample_params("characteristicgroup") +#' states <- check_waterdata_sample_params("states") +#' countries <- check_waterdata_sample_params("countries") +#' counties <- check_waterdata_sample_params("counties") +#' sitetypes <- check_waterdata_sample_params("sitetype") +#' samplemedia <- check_waterdata_sample_params("samplemedia") +#' characteristics <- check_waterdata_sample_params("characteristics", #' group = "Biological") -#' observedProperties <- check_USGS_sample_params("observedproperty", +#' observedProperties <- check_waterdata_sample_params("observedproperty", #' text = "phosphorus") #' #' } -check_USGS_sample_params <- function(service = "characteristicgroup", +check_waterdata_sample_params <- function(service = "characteristicgroup", ...){ service_options <- c("characteristicgroup", "states", "counties", @@ -381,14 +381,14 @@ check_USGS_sample_params <- function(service = "characteristicgroup", #' This function creates the call and gets the data for discrete water quality samples data #' service described at . #' -#' @inheritParams construct_USGS_sample_request +#' @inheritParams construct_waterdata_sample_request #' @param tz character to set timezone attribute of datetime. Default is UTC #' (properly accounting for daylight savings times based on the data's provided tz_cd column). #' Possible values include "America/New_York","America/Chicago", "America/Denver","America/Los_Angeles", #' "America/Anchorage","America/Honolulu","America/Jamaica","America/Managua", #' "America/Phoenix", and "America/Metlakatla" #' @export -#' +#' @rdname read_waterdata_samples #' @examplesIf is_dataRetrieval_user() #' #' \donttest{ @@ -433,7 +433,7 @@ read_waterdata_samples <- function(monitoringLocationIdentifier = NA, dataProfile = NA, tz = "UTC"){ - request_url <- construct_USGS_sample_request(monitoringLocationIdentifier = monitoringLocationIdentifier, + request_url <- construct_waterdata_sample_request(monitoringLocationIdentifier = monitoringLocationIdentifier, siteTypeCode = siteTypeCode, boundingBox = boundingBox, hydrologicUnit = hydrologicUnit, @@ -474,7 +474,7 @@ read_waterdata_samples <- function(monitoringLocationIdentifier = NA, #' numbers without an agency prefix are assumed to have the prefix USGS. #' @export #' @return data frame with summary of data available based on the monitoringLocationIdentifier -#' +#' @rdname summarize_waterdata_samples #' @examplesIf is_dataRetrieval_user() #' #' \donttest{ @@ -507,3 +507,73 @@ summarize_waterdata_samples <- function(monitoringLocationIdentifier){ return(df) } + + +#' @rdname read_waterdata_samples +#' @export +read_USGS_samples <- function(monitoringLocationIdentifier = NA, + siteTypeCode = NA, + boundingBox = NA, + hydrologicUnit = NA, + activityMediaName = NA, + characteristicGroup = NA, + characteristic = NA, + characteristicUserSupplied = NA, + activityStartDateLower = NA, + activityStartDateUpper = NA, + countryFips = NA, + stateFips = NA, + countyFips = NA, + projectIdentifier = NA, + recordIdentifierUserSupplied = NA, + siteTypeName = NA, + usgsPCode = NA, + pointLocationLatitude = NA, + pointLocationLongitude = NA, + pointLocationWithinMiles = NA, + dataType = "results", + dataProfile = NA, + tz = "UTC"){ + + .Deprecated(new = "read_waterdata_samples", + package = "dataRetrieval", + msg = "Function has been renamed. Please begin to migrate to read_waterdata_samples") + + + read_waterdata_samples(monitoringLocationIdentifier = monitoringLocationIdentifier, + siteTypeCode = siteTypeCode, + boundingBox = boundingBox, + hydrologicUnit = hydrologicUnit, + activityMediaName = activityMediaName, + characteristicGroup = characteristicGroup, + characteristic = characteristic, + characteristicUserSupplied = characteristicUserSupplied, + activityStartDateLower = activityStartDateLower, + activityStartDateUpper = activityStartDateUpper, + countryFips = countryFips, + stateFips = stateFips, + countyFips = countyFips, + projectIdentifier = projectIdentifier, + recordIdentifierUserSupplied = recordIdentifierUserSupplied, + siteTypeName = siteTypeName, + usgsPCode = usgsPCode, + pointLocationLatitude = pointLocationLatitude, + pointLocationLongitude = pointLocationLongitude, + pointLocationWithinMiles = pointLocationWithinMiles, + dataType = dataType, + dataProfile = dataProfile, + tz = tz) +} + + +#' @rdname summarize_waterdata_samples +#' @export +summarize_USGS_samples <- function(monitoringLocationIdentifier){ + + .Deprecated(new = "summarize_waterdata_samples", + package = "dataRetrieval", + msg = "Function has been renamed. Please begin to migrate to summarize_waterdata_samples") + + summarize_waterdata_samples(monitoringLocationIdentifier) +} + diff --git a/man/check_USGS_sample_params.Rd b/man/check_waterdata_sample_params.Rd similarity index 59% rename from man/check_USGS_sample_params.Rd rename to man/check_waterdata_sample_params.Rd index 338b84e0..d8b4a9eb 100644 --- a/man/check_USGS_sample_params.Rd +++ b/man/check_waterdata_sample_params.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/read_waterdata_samples.R -\name{check_USGS_sample_params} -\alias{check_USGS_sample_params} +\name{check_waterdata_sample_params} +\alias{check_waterdata_sample_params} \title{Check values from codeservice} \usage{ -check_USGS_sample_params(service = "characteristicgroup", ...) +check_waterdata_sample_params(service = "characteristicgroup", ...) } \arguments{ \item{service}{Options are: "characteristicgroup", "states", "counties", @@ -24,15 +24,15 @@ Call a service to check on values from: \dontshow{if (is_dataRetrieval_user()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} \donttest{ -groups <- check_USGS_sample_params("characteristicgroup") -states <- check_USGS_sample_params("states") -countries <- check_USGS_sample_params("countries") -counties <- check_USGS_sample_params("counties") -sitetypes <- check_USGS_sample_params("sitetype") -samplemedia <- check_USGS_sample_params("samplemedia") -characteristics <- check_USGS_sample_params("characteristics", +groups <- check_waterdata_sample_params("characteristicgroup") +states <- check_waterdata_sample_params("states") +countries <- check_waterdata_sample_params("countries") +counties <- check_waterdata_sample_params("counties") +sitetypes <- check_waterdata_sample_params("sitetype") +samplemedia <- check_waterdata_sample_params("samplemedia") +characteristics <- check_waterdata_sample_params("characteristics", group = "Biological") -observedProperties <- check_USGS_sample_params("observedproperty", +observedProperties <- check_waterdata_sample_params("observedproperty", text = "phosphorus") } diff --git a/man/construct_USGS_sample_request.Rd b/man/construct_waterdata_sample_request.Rd similarity index 89% rename from man/construct_USGS_sample_request.Rd rename to man/construct_waterdata_sample_request.Rd index 58ab4cbe..6835e7f2 100644 --- a/man/construct_USGS_sample_request.Rd +++ b/man/construct_waterdata_sample_request.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/read_waterdata_samples.R -\name{construct_USGS_sample_request} -\alias{construct_USGS_sample_request} +\name{construct_waterdata_sample_request} +\alias{construct_waterdata_sample_request} \title{Construct request for USGS Samples Data} \usage{ -construct_USGS_sample_request( +construct_waterdata_sample_request( monitoringLocationIdentifier = NA, siteTypeCode = NA, boundingBox = NA, @@ -36,7 +36,7 @@ for example: AZ014-320821110580701, CAX01-15304600, USGS-040851385. Location numbers without an agency prefix are assumed to have the prefix USGS.} \item{siteTypeCode}{Site type code query parameter. See available -options by running \code{check_USGS_sample_params("sitetype")$typeCode}.} +options by running \code{check_waterdata_sample_params("sitetype")$typeCode}.} \item{boundingBox}{North and South are latitude values; East and West are longitude values. A vector of 4 (west, south, east, north) is expected. @@ -51,11 +51,11 @@ was sampled or analyzed.} \item{characteristicGroup}{Characteristic group is a broad category describing the sample. See available options by running -\code{check_USGS_sample_params("characteristicgroup")$characteristicGroup}.} +\code{check_waterdata_sample_params("characteristicgroup")$characteristicGroup}.} \item{characteristic}{Characteristic is a specific category describing the sample. See available options by running -\code{check_USGS_sample_params("characteristics")$characteristicName}.} +\code{check_waterdata_sample_params("characteristics")$characteristicName}.} \item{characteristicUserSupplied}{Observed property is the USGS term for the constituent sampled and the property name gives a detailed description of what @@ -77,17 +77,17 @@ records that match the date.} \item{countryFips}{Country query parameter. Do not set redundant parameters. If another query parameter contains the country information, leave this parameter -set to the default NA. See available options by running \code{check_USGS_sample_params("countries")}, +set to the default NA. See available options by running \code{check_waterdata_sample_params("countries")}, where the "id" field contains the value to use in the countryFips input.} \item{stateFips}{State query parameter. To get a list of available state fips, -run \code{check_USGS_sample_params("states")}. The "fips" can be created using the function +run \code{check_waterdata_sample_params("states")}. The "fips" can be created using the function \code{stateCdLookup} - for example: \code{stateCdLookup("WI", "fips")}. FIPs codes for states take the format: CountryAbbrev:StateNumber, like US:55 for Wisconsin.} \item{countyFips}{County query parameter. To get a list of available counties, -run \code{check_USGS_sample_params("counties")}. The "Fips" can be created using the function +run \code{check_waterdata_sample_params("counties")}. The "Fips" can be created using the function \code{countyCdLookup} - for example: \code{countyCdLookup("WI", "Dane", "fips")} for Dane County, WI. FIPs codes for counties take the format: @@ -103,7 +103,7 @@ information would be needed from the data supplier.} options by running \code{check_param("sitetype")$typeName}.} \item{usgsPCode}{USGS parameter code. See available options by running -\code{check_USGS_sample_params("characteristics")$parameterCode}.} +\code{check_waterdata_sample_params("characteristics")$parameterCode}.} \item{pointLocationLatitude}{Latitude for a point/radius query (decimal degrees). Must be used with pointLocationLongitude and pointLocationWithinMiles.} @@ -142,7 +142,7 @@ See also: \url{https://api.waterdata.usgs.gov/samples-data/docs}. \dontshow{if (is_dataRetrieval_user()) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} \donttest{ -req <- construct_USGS_sample_request( +req <- construct_waterdata_sample_request( monitoringLocationIdentifier = "USGS-04074950", characteristicUserSupplied = "pH, water, unfiltered, field") rawData <- importWQP(req) diff --git a/man/read_waterdata_samples.Rd b/man/read_waterdata_samples.Rd index b23fbd4e..6eb9081f 100644 --- a/man/read_waterdata_samples.Rd +++ b/man/read_waterdata_samples.Rd @@ -2,6 +2,7 @@ % Please edit documentation in R/read_waterdata_samples.R \name{read_waterdata_samples} \alias{read_waterdata_samples} +\alias{read_USGS_samples} \title{USGS Samples Data} \usage{ read_waterdata_samples( @@ -29,6 +30,32 @@ read_waterdata_samples( dataProfile = NA, tz = "UTC" ) + +read_USGS_samples( + monitoringLocationIdentifier = NA, + siteTypeCode = NA, + boundingBox = NA, + hydrologicUnit = NA, + activityMediaName = NA, + characteristicGroup = NA, + characteristic = NA, + characteristicUserSupplied = NA, + activityStartDateLower = NA, + activityStartDateUpper = NA, + countryFips = NA, + stateFips = NA, + countyFips = NA, + projectIdentifier = NA, + recordIdentifierUserSupplied = NA, + siteTypeName = NA, + usgsPCode = NA, + pointLocationLatitude = NA, + pointLocationLongitude = NA, + pointLocationWithinMiles = NA, + dataType = "results", + dataProfile = NA, + tz = "UTC" +) } \arguments{ \item{monitoringLocationIdentifier}{A monitoring location identifier has two parts: the agency code @@ -37,7 +64,7 @@ for example: AZ014-320821110580701, CAX01-15304600, USGS-040851385. Location numbers without an agency prefix are assumed to have the prefix USGS.} \item{siteTypeCode}{Site type code query parameter. See available -options by running \code{check_USGS_sample_params("sitetype")$typeCode}.} +options by running \code{check_waterdata_sample_params("sitetype")$typeCode}.} \item{boundingBox}{North and South are latitude values; East and West are longitude values. A vector of 4 (west, south, east, north) is expected. @@ -52,11 +79,11 @@ was sampled or analyzed.} \item{characteristicGroup}{Characteristic group is a broad category describing the sample. See available options by running -\code{check_USGS_sample_params("characteristicgroup")$characteristicGroup}.} +\code{check_waterdata_sample_params("characteristicgroup")$characteristicGroup}.} \item{characteristic}{Characteristic is a specific category describing the sample. See available options by running -\code{check_USGS_sample_params("characteristics")$characteristicName}.} +\code{check_waterdata_sample_params("characteristics")$characteristicName}.} \item{characteristicUserSupplied}{Observed property is the USGS term for the constituent sampled and the property name gives a detailed description of what @@ -78,17 +105,17 @@ records that match the date.} \item{countryFips}{Country query parameter. Do not set redundant parameters. If another query parameter contains the country information, leave this parameter -set to the default NA. See available options by running \code{check_USGS_sample_params("countries")}, +set to the default NA. See available options by running \code{check_waterdata_sample_params("countries")}, where the "id" field contains the value to use in the countryFips input.} \item{stateFips}{State query parameter. To get a list of available state fips, -run \code{check_USGS_sample_params("states")}. The "fips" can be created using the function +run \code{check_waterdata_sample_params("states")}. The "fips" can be created using the function \code{stateCdLookup} - for example: \code{stateCdLookup("WI", "fips")}. FIPs codes for states take the format: CountryAbbrev:StateNumber, like US:55 for Wisconsin.} \item{countyFips}{County query parameter. To get a list of available counties, -run \code{check_USGS_sample_params("counties")}. The "Fips" can be created using the function +run \code{check_waterdata_sample_params("counties")}. The "Fips" can be created using the function \code{countyCdLookup} - for example: \code{countyCdLookup("WI", "Dane", "fips")} for Dane County, WI. FIPs codes for counties take the format: @@ -104,7 +131,7 @@ information would be needed from the data supplier.} options by running \code{check_param("sitetype")$typeName}.} \item{usgsPCode}{USGS parameter code. See available options by running -\code{check_USGS_sample_params("characteristics")$parameterCode}.} +\code{check_waterdata_sample_params("characteristics")$parameterCode}.} \item{pointLocationLatitude}{Latitude for a point/radius query (decimal degrees). Must be used with pointLocationLongitude and pointLocationWithinMiles.} diff --git a/man/summarize_waterdata_samples.Rd b/man/summarize_waterdata_samples.Rd index 488e7718..c7c8972b 100644 --- a/man/summarize_waterdata_samples.Rd +++ b/man/summarize_waterdata_samples.Rd @@ -2,9 +2,12 @@ % Please edit documentation in R/read_waterdata_samples.R \name{summarize_waterdata_samples} \alias{summarize_waterdata_samples} +\alias{summarize_USGS_samples} \title{USGS Samples Summary Data} \usage{ summarize_waterdata_samples(monitoringLocationIdentifier) + +summarize_USGS_samples(monitoringLocationIdentifier) } \arguments{ \item{monitoringLocationIdentifier}{A monitoring location identifier has two parts, diff --git a/tests/testthat/tests_samples.R b/tests/testthat/tests_samples.R index 869734d8..da145998 100644 --- a/tests/testthat/tests_samples.R +++ b/tests/testthat/tests_samples.R @@ -10,7 +10,7 @@ test_that("General samples-data retrievals work using WQP tests", { # testing lists: startDate <- as.Date("2022-01-01") - secchi_ops <- check_USGS_sample_params("observedproperty", + secchi_ops <- check_waterdata_sample_params("observedproperty", text = "secchi") state_fips <- paste0("US:", stateCdLookup("WI", "id")) @@ -67,7 +67,7 @@ test_that("samples-data project working", { }) context("summary_waterdata_samples") -test_that("summary_USGS_samples working", { +test_that("summary_waterdata_samples working", { testthat::skip_on_cran() site1 <- summarize_waterdata_samples(monitoringLocationIdentifier = "USGS-01594440") diff --git a/tests/testthat/tests_userFriendly_fxns.R b/tests/testthat/tests_userFriendly_fxns.R index f81af9a9..ba58c224 100644 --- a/tests/testthat/tests_userFriendly_fxns.R +++ b/tests/testthat/tests_userFriendly_fxns.R @@ -126,26 +126,26 @@ test_that("read_waterdata_daily", { endDate <- "2012-06-30" pCode <- "00060" - raw_USGS_daily <- read_waterdata_daily(monitoring_location_id = siteNumber, + raw_waterdata_daily <- read_waterdata_daily(monitoring_location_id = siteNumber, parameter_code = pCode, time = c(startDate, endDate)) - expect_is(raw_USGS_daily$time, "Date") + expect_is(raw_waterdata_daily$time, "Date") - raw_USGS_TempMeanMax <- read_waterdata_daily(monitoring_location_id = siteNumber, + raw_waterdata_TempMeanMax <- read_waterdata_daily(monitoring_location_id = siteNumber, parameter_code = c("00010", "00060"), time = c(startDate, endDate), statistic_id = c("00001", "00003")) - expect_true(length(unique(raw_USGS_TempMeanMax$parameter_code)) == 2) - expect_true(length(unique(raw_USGS_TempMeanMax$statistic_id)) == 2) - expect_true(length(unique(raw_USGS_TempMeanMax$monitoring_location_id)) == 1) + expect_true(length(unique(raw_waterdata_TempMeanMax$parameter_code)) == 2) + expect_true(length(unique(raw_waterdata_TempMeanMax$statistic_id)) == 2) + expect_true(length(unique(raw_waterdata_TempMeanMax$monitoring_location_id)) == 1) - raw_USGS_MultiSites <- read_waterdata_daily(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), + raw_waterdata_MultiSites <- read_waterdata_daily(monitoring_location_id = c("USGS-01491000", "USGS-01645000"), parameter_code = c("00010", "00060"), time = c(startDate, endDate), statistic_id = c("00001", "00003")) - expect_true(length(unique(raw_USGS_MultiSites$monitoring_location_id)) == 2) + expect_true(length(unique(raw_waterdata_MultiSites$monitoring_location_id)) == 2) site <- "05212700" diff --git a/vignettes/samples_data.Rmd b/vignettes/samples_data.Rmd index 7df9d779..8895130d 100644 --- a/vignettes/samples_data.Rmd +++ b/vignettes/samples_data.Rmd @@ -259,7 +259,7 @@ map_it(point_sites) ### countyFips County query parameter. To get a list of available counties, -run `check_USGS_sample_params("counties")`. The "Fips" values can be created using the function `countyCdLookup`. +run `check_waterdata_sample_params("counties")`. The "Fips" values can be created using the function `countyCdLookup`. ```{r} dane_county <- countyCdLookup("WI", "Dane", @@ -280,7 +280,7 @@ map_it(county_sites) ### stateFips State query parameter. To get a list of available state fips values, -run `check_USGS_sample_params("states")`. The "fips" values can be created using the function +run `check_waterdata_sample_params("states")`. The "fips" values can be created using the function `stateCdLookup`. ```{r} @@ -307,7 +307,7 @@ Additional parameters can be included to limit the results coming back from a re Site type code query parameter. ```{r} -site_type_info <- check_USGS_sample_params("sitetype") +site_type_info <- check_waterdata_sample_params("sitetype") site_type_info$typeCode ``` @@ -324,7 +324,7 @@ site_type_info$typeLongName Sample media refers to the environmental medium that was sampled or analyzed. ```{r} -media_info <- check_USGS_sample_params("samplemedia") +media_info <- check_waterdata_sample_params("samplemedia") media_info$activityMedia ``` @@ -333,34 +333,34 @@ media_info$activityMedia Characteristic group is a broad category describing the sample measurement. The options for this parameter generally follow the values described in the Water Quality Portal [User Guide](https://www.waterqualitydata.us/portal_userguide), but not always. ```{r} -group_info <- check_USGS_sample_params("characteristicgroup") +group_info <- check_waterdata_sample_params("characteristicgroup") group_info$characteristicGroup ``` ### characteristic -Characteristic is a specific category describing the sample. See `check_USGS_sample_params("characteristics")` for a full list, below is a small sample: +Characteristic is a specific category describing the sample. See `check_waterdata_sample_params("characteristics")` for a full list, below is a small sample: ```{r} -characteristic_info <- check_USGS_sample_params("characteristics") +characteristic_info <- check_waterdata_sample_params("characteristics") head(unique(characteristic_info$characteristicName)) ``` ### characteristicUserSupplied -Observed property is the USGS term for the constituent sampled and the property name gives a detailed description of what was sampled. Observed Property is mapped to characteristicUserSupplied, and replaces the parameter name and pcode USGS previously used to describe discrete sample data. See `check_USGS_sample_params("observedproperty")` for a full list, below is a small sample: +Observed property is the USGS term for the constituent sampled and the property name gives a detailed description of what was sampled. Observed Property is mapped to characteristicUserSupplied, and replaces the parameter name and pcode USGS previously used to describe discrete sample data. See `check_waterdata_sample_params("observedproperty")` for a full list, below is a small sample: ```{r} -char_us <- check_USGS_sample_params("observedproperty") +char_us <- check_waterdata_sample_params("observedproperty") head(char_us$observedProperty) ``` ### usgsPCode -USGS parameter code. See `check_USGS_sample_params("characteristics")` for a full list, below is a small sample: +USGS parameter code. See `check_waterdata_sample_params("characteristics")` for a full list, below is a small sample: ```{r} -characteristic_info <- check_USGS_sample_params("characteristics") +characteristic_info <- check_waterdata_sample_params("characteristics") head(unique(characteristic_info$parameterCode)) ``` diff --git a/vignettes/tutorial.Rmd b/vignettes/tutorial.Rmd index e189728f..68e2605f 100644 --- a/vignettes/tutorial.Rmd +++ b/vignettes/tutorial.Rmd @@ -291,10 +291,10 @@ peak_data <- readNWISpeak(gsub("USGS-", "", site)) ``` -For discrete water quality data, use the `summarize_USGS_samples` function: +For discrete water quality data, use the `summarize_waterdata_samples` function: ```{r echo=TRUE} -discrete_data_available_all <- summarize_USGS_samples(site) +discrete_data_available_all <- summarize_waterdata_samples(site) discrete_data_available <- discrete_data_available_all |> select(parameter_name = characteristicUserSupplied, From cdcdd791f9d661690fb1c44068d93e8e8d13082b Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 13 Jun 2025 11:54:30 -0500 Subject: [PATCH 112/117] fix state code stuff --- R/dataRetrievals-package.R | 4 ++-- R/readNWISdata.R | 4 ++-- R/read_waterdata_samples.R | 4 +--- man/countyCd.Rd | 2 +- man/stateCd.Rd | 2 +- 5 files changed, 7 insertions(+), 9 deletions(-) diff --git a/R/dataRetrievals-package.R b/R/dataRetrievals-package.R index b92eda0e..cd205de7 100644 --- a/R/dataRetrievals-package.R +++ b/R/dataRetrievals-package.R @@ -96,7 +96,7 @@ NULL #' US State Code Lookup Table #' #' Classic lookup table for states. Has been replaced in functions with -#' `check_USGS_sample_params("states")`. +#' `check_waterdata_sample_params("states")`. #' #' @name stateCd #' @return stateCd data frame. @@ -118,7 +118,7 @@ NULL #' US County Code Lookup Table #' #' Classic lookup table for counties. Has been replaced in functions with -#' `check_USGS_sample_params("counties")`. +#' `check_USGS_waterdata_params("counties")`. #' #' @name countyCd #' @return countyCd data frame. diff --git a/R/readNWISdata.R b/R/readNWISdata.R index f56ccfcc..d0c47fc4 100644 --- a/R/readNWISdata.R +++ b/R/readNWISdata.R @@ -288,7 +288,7 @@ stateCdLookup <- function(input, outputType <- match.arg(outputType, c("postal", "fullName", "id", "fips")) - states <- check_USGS_sample_params("states") + states <- check_waterdata_sample_params("states") country <- match.arg(country, choices = unique(states$countryCode), several.ok = FALSE) states <- states[states$countryCode == country,] @@ -360,7 +360,7 @@ countyCdLookup <- function(state, county, outputType = "fips") { stop("Only one state allowed in countyCdLookup.") } - counties <- check_USGS_sample_params("counties") + counties <- check_waterdata_sample_params("counties") # first turn state into stateCd postal name state_postal <- stateCdLookup(state, diff --git a/R/read_waterdata_samples.R b/R/read_waterdata_samples.R index 0875d49b..d41f0448 100644 --- a/R/read_waterdata_samples.R +++ b/R/read_waterdata_samples.R @@ -365,9 +365,7 @@ check_waterdata_sample_params <- function(service = "characteristicgroup", check_group_req <- httr2::req_url_query(check_group_req, !!!params) } - - message("GET: ", check_group_req$url) - + check_group <- httr2::req_perform(check_group_req) |> httr2::resp_body_string() |> jsonlite::fromJSON() diff --git a/man/countyCd.Rd b/man/countyCd.Rd index 9fd7e511..5fcac308 100644 --- a/man/countyCd.Rd +++ b/man/countyCd.Rd @@ -18,7 +18,7 @@ COUNTY_ID \tab character \tab County id \cr } \description{ Classic lookup table for counties. Has been replaced in functions with -\code{check_USGS_sample_params("counties")}. +\code{check_USGS_waterdata_params("counties")}. } \examples{ head(countyCd) diff --git a/man/stateCd.Rd b/man/stateCd.Rd index 0e595174..b8eaea3d 100644 --- a/man/stateCd.Rd +++ b/man/stateCd.Rd @@ -17,7 +17,7 @@ STATENS \tab character \tab Geographic Names Information System Identifier (GNI } \description{ Classic lookup table for states. Has been replaced in functions with -\code{check_USGS_sample_params("states")}. +\code{check_waterdata_sample_params("states")}. } \examples{ head(stateCd) From 0315f53e49ea9e847f5508113cf994c8545c397f Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 13 Jun 2025 13:08:24 -0500 Subject: [PATCH 113/117] Fix pkgdown --- _pkgdown.yml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/_pkgdown.yml b/_pkgdown.yml index 102b4999..20530cc1 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -40,8 +40,8 @@ navbar: menu: - text: Tutorial href: articles/tutorial.html - - text: New USGS Functions - href: articles/read_USGS_functions.html + - text: USGS Water Data APIs + href: articles/read_waterdata_functions.html - text: Background href: articles/dataRetrieval.html - text: Pivot Data @@ -67,16 +67,15 @@ reference: - title: USGS new data services desc: Functions to retrieve USGS data from new services. contents: - - read_USGS_samples - - read_USGS_daily - - read_USGS_ts_meta - - read_USGS_data - - read_USGS_monitoring_location - - read_USGS_data - - summarize_USGS_samples - - check_USGS_sample_params + - read_waterdata_samples + - read_waterdata_daily + - read_waterdata_ts_meta + - read_waterdata_monitoring_location + - read_waterdata + - summarize_waterdata_samples + - check_waterdata_sample_params - title: National Water Information System (NWIS) - desc: Functions to retrieve (USGS) NWIS data. These will be slowly phased out and replaced with the read_USGS family of functions. + desc: Functions to retrieve (USGS) NWIS data. These will be slowly phased out and replaced with the read_waterdata family of functions. contents: - readNWISdv - readNWISuv From 5ddc0d075831395d8695fd909c2003674ac07945 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 13 Jun 2025 13:13:10 -0500 Subject: [PATCH 114/117] Does that help? --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 76402775..fc5dec06 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,6 +31,7 @@ variables: NOT_CRAN: "true" PAGES_OUTDIR: "$CI_PROJECT_DIR/public" CUSTOM_DR_UA: "GitLab_CI" + API_USGS_PAT: "${API_USGS_PAT}" build-image: stage: build From a2775c5a21ce2ea38b755ad5c5917d9e3c42b5a6 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Fri, 13 Jun 2025 15:34:42 -0500 Subject: [PATCH 115/117] Update NEWS Co-authored-by: Elise Hinman <121896266+ehinman@users.noreply.github.com> --- NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 98d03abd..4a0a1f4e 100644 --- a/NEWS +++ b/NEWS @@ -4,7 +4,7 @@ dataRetrieval 2.7.19 new USGS web services. * Added whisker and sf as dependencies. * Renamed read_USGS_sample to read_waterdata_sample. -* Renamed summarize_USGS_sample to summarize_waterdata_sample. +* Renamed summarize_USGS_samples to summarize_waterdata_samples. * Added warning to setAccess for non-public endpoints. dataRetrieval 2.7.18 From 9805ce14c621b40bac8c3ace790eb734f39ed568 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Mon, 16 Jun 2025 07:55:28 -0500 Subject: [PATCH 116/117] Update NEWS Co-authored-by: Elise Hinman <121896266+ehinman@users.noreply.github.com> --- NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 4a0a1f4e..2e656a3a 100644 --- a/NEWS +++ b/NEWS @@ -3,7 +3,7 @@ dataRetrieval 2.7.19 * Added read_waterdata_daily, read_waterdata_monitoring_location, read_waterdata_ts_meta to access new USGS web services. * Added whisker and sf as dependencies. -* Renamed read_USGS_sample to read_waterdata_sample. +* Renamed read_USGS_samples to read_waterdata_samples. * Renamed summarize_USGS_samples to summarize_waterdata_samples. * Added warning to setAccess for non-public endpoints. From 63a988787adfb191567505fe705409486e2806d6 Mon Sep 17 00:00:00 2001 From: Laura DeCicco Date: Mon, 16 Jun 2025 08:35:17 -0500 Subject: [PATCH 117/117] Update intro vignette --- inst/extdata/temperatureAndFlow.RData | Bin 2174 -> 51243 bytes vignettes/dataRetrieval.Rmd | 278 +++++++++----------------- 2 files changed, 91 insertions(+), 187 deletions(-) diff --git a/inst/extdata/temperatureAndFlow.RData b/inst/extdata/temperatureAndFlow.RData index 80062797ae53223fb15f7152d3d1f663dd465bec..7e77e64214dded02767f69f2d139ac3c77ce2ae0 100644 GIT binary patch literal 51243 zcmdp-WmgA-HRBmj!|c0xa$h!5xCb;z5GL0*kxDA_*ZZ5ZqyLhakb--Gbk| zxqsk(zSHMCr)R3Ft81)#db(-AXs`ZlC`SRPpSAmcK8GYANlCp?j+6<;DSv{(RT9=D3ZthXS_5f(`RtdhOQ!sKxPmI>W%Jq$73c}v(Z!) z&x^6yUNGXy8n8J73VaM+(WfPIt7$S~Fdf>_h}H&Vfu8QuG@F2}8Btaj8q7(tr7?K? z1*8gPxj?#P&zgOG?YuadT;&-VP6AaXJOvw!efcU8pAoOBCDtnX&f8S9g zar9xF$x=ZupHs?ur1rD^=B|24+{E!h1cMauKb}8uw8I{mM{WD|XAm%6^_L zP00oy>hm$rv-S?eWuh#LUl9v#m)}_M;m2dOikZ}}!(}hf_7%wGDy>mvr5bx#VisCzz17C3X{$JK408PbKlo&CWZ7WavdZTiZD#ZRbWM+P8!Qwrh-YC2pKhCY_@$M`NNT(BUOI2>c{!J|7B6A72a$}|Fsva#Vjz;vG4id=okgvmy?&nENf{ER8 z5_r>3YL9Uio&pd8M6HYk$;%QmV$;tYS@S}vHY(&RQrL)-(V6q!S@sX6YgP!MIL2!; z@nQd(QvoV2=wn^?$PYgRP^tk5G7vSCv+$3y2bj%Kv10VaqMf;j!O(KDjyuE%9Ra-P z7S*Kt`8Xj3tUub+$>O?-y|opM6`>U2F3~a*=>YO%)<2PyOm^ZXF^t7&U3;Q*p#^1{oFVZ`mkxe!iw*?MQ$qqYpbMM2> zG-9Pz>0aDYLa!nFoRTdA<;zrN)l=Lmu9Da#IHNo>CFPLu$+n^<8r?rb`RcT}$8MWh$xl5D2wcsbAo20zwG*q-nMqagPvO<>@jYq&-#Zw@@Ahkv2=ShxY z=4FkoZQUF%v|>B#<2(DMn$+ryr>Rq7*ln(3V9zRgyEYF0gijIYZij zFdt$rBv%Zz(qx^@QP0GXXU80Y8k-sVcgELg$P9F&9KO0Xm+tD zOJd{?RZ-VueT%8UN<2A;99W;N<{a@Ld&okp^J}7VQW$0?K4p{3s-2jgvN?#zV~d6t z`%#+GgTR9Y`E+fIcd}JtcNr_q18Z5oRbI`iw)}NB;oKm63R`zV`}f>o$(l9hS+hL} zA2XSft^KyV+?uR9x^jol)G4N5vVz5eCq2xFuw5iASVo<*k#?pLDF}gkYh0!BGpSv) ze35|}>tmHahds!Ui!XGMnKq1dAX&zxq>b*uo+bZq`G(+^3JDd57$epnD-_193qk}sD@7tq z(i)i#2rR6G=jWMYjK#KgP8xfd;vk)_bHkb}0jsGCWv)RKT^c)K`QL}jb^-Bk`;t0R zrfAU1*LSBS+PfYBl56op3dDJZF@G2s_|0vawj@|$mba-W1ET5t*Rj5OfuiZCaJ zTz>HwfMbLM`EC1Nf0qA>CWDToSN{6DK^+?%((o=g<7arvsz8{xGx>4%D_35AZrq+N z+;=^aw4hFr)2#p+WZQ)BO}U;;=V%)PVy_QA(Y~>ytqHLtTF>Ocg;6qbSO>kV_2a2_ zCY?X%dEN0sI}INEHqmG<7<)T>=fK5lK^}E8R%;uRLHh5M9f1${bAxh#NJYZ3wCZ{@ z^j=3cTK<7#!;5}t;ylJ=IO3PY<6|-&YCFjhqRn%CvE2S5s~aJMkvpu}LLefXIT<N9tRQQ9+5v^}&>AHW%-)-Kg-D^?ZwmIIQ*&e3bsmXYhY-h5z zAhh(EPJvjy)T61c!%!lO+0svWR9(_hs&6;h% zBBPJSjoksn#$LZs;wTzyjK-rnO1W@-nXl8VjMd0z@0zmXW-{Y49kF_J+v)Q#a7<8& zlC^uZ$m*`8B@8EU0|Xd>k~``d@PwIb`1NsPy6)67D4{E31?i^!VkwIh<~%Q-Yr{1W zDMmSFyGdmbxkeiZmvBWmW1u1tc~{4V?#UHE+E9ZCGL#?``rRZJrU)6Xe6BA0j7IK6 z_R?DWy-~tJ3A4;j>i3bHJT~?SUyyXoEF*!mS==RdWAsTf^IK-&x-tk<8+B79jt+?0)t4bPG}wpOwTiH*&U@cJYfqkWb_MToq*_U#vCV9 zBc-S+yX#z{x9JT2_8%ch04!$5V51I*5)bB0Vl`O^P)w;M$*z<%npnx6gqq-iyw)Ve zJ>w#3Z7tilMjB&NFfqCW_Jf7iDst}ZK`7#4nl%vLh6M*tC1ybN0Ej80xry#EQz>A7L6|HKq+Xl z9~r0X6B!-l=X8o1p>xgG$1NbFzJzP`6S3ABH)WOk%BWlLX_j_ zjC-SI%J_GwE^I5%IM1yj9R*VP#J`5&QWs-7aNdby$tl;cl)mfc#7bi!_FI*MPG_L# zGuCK`Fw5`m1DWLBD0i!ERo7*=QFpVT_TYTuk;0BOJ*utYIv%{J>$Sn)sWqsLso;;$ z6ibrrRdZvT4d&l53=&|lXEiTnyjvI}Vpq?Tu zr>pPxQ@$j9%@XM&6vAeO`>B`Lji8LlA{wjIh!RW#uaJpx*5V4u)2{g7ufh>z4{abU zYh=KdKrlVF;HK&qz-zD34nAM2jtd$f+S4Z=6IQvAp^OQhFCn#i%+h+GH1(2)`RtXmXr3Mx8v0^jg*p_2bf85N;S~zcc+1W~^*8ne2wOkRdTP z1?WX1?dI2 ziY~R}LMDmU7Zxw}#+M}OHp(~> z_YjX;w!?lc5^`fR^icR2zVup{u~C>R7tCnVFBTBzS*)$R z`{b79NhP>$<|H}tTC~JI!V2(x-4uh>@uLUhyRmXF7>7kM|F3il#7L7$b4X|~ZgS_l zOCda&-f$->V%Eqp8eTGyQ?Lw%!qIF}R4eYookw|snhf5mOm%vJKAGo$IULv-O9t_* zStEaFjw-T@=D*>@J{Gix4vT<*5%0Z6RIGTBJ(!492q!7#(b6dWAfcnw8>k%P^B%Um z`}+9nF~WwdX7mu!RPuP=*mtA=&e?7dVapX#k(Bmi&N`HDo_=f1Td|_%G}QP47rrB6 zT2kxdp_>!=qSE;%QwZ*EAQGWQq?su6Sye1ok+2p(kZ)emos=q7nJC)<2dsGvME8QZ zEUJtUxh5dyF>f!((sz*IhXyHYDLmg{A2m#wc}$-4>19zVNR<rQBRFWC- zcKlkdT06>u@q4dTU)ORd3b4P`myr*{MQDp~vZlrEw;R(^m?}&cL#u-MGXIEka$?>8aPv?_+PP=yc`4N}4x&vf06 zu5o)Ur8C(Wd2N(KXH6ZL4>NXrDB0E!RN>);*3BJ~ecYYV)>r!3TNUxd?#%^!DQnSIUCl8Hb&g=doZ zci;k6VreRi9ZEbrp)@)S+uEq&nCd*OqHqBdsowlP-GyFJ4Xixn2t?*1+o4naTI`}s zQenF9fMv}V}>-aJmG5BkTQ>==L@zCz@wT7*kL{m zes55eF}$_@QH92m-yC|D<|MK#H4<`%GM5y->rTnMFSKJYvOowjdz7^wbss3R=pdKm z$12Mjvn}3f<7ZOjJ?MNQX$#=!z=TIBmDJAb(CujK9PDnQoLBwE({vN{mIo5dYIKd-V^nLy~XowYxzX=8ztiEd$ zzIHU~>xteHhS;i&6iWWHBu-?f#`|+_XI+2QVry;eW`};ed@=Y&Xp*w+kj2M^d)X>F zEweo49e$==%1QuykMLc7nL4kG3Os&#lxdxIMARJ19w|N=E$4l1==7b)amyqD8fK0W ztL0|}v8-@#k9sHhT3ZWa+x+`p%z49N>&&_e>7)}Y20NvU0{tqWOFOt+oS5M#NfRx8 z#Ncz@hOCG`^b%3zI$Yw?3kmIr^`la!Pv0%$lzuQQbZlDmmM-_!yf&mlF|e3Wt^C%& zS=?X%+Un1KWzq|mAA?_jwo;U=0Ft;DS?^r{Bx<%wj7SLlYs)FOE3DF+v=G)5ZlMD) zA~(+v$21SSFN0Gngi4>0;=kketk8eEFUlBYrXm^EiRn&fP>FUVEUeA$Vi|`6h+?VC z$kU~|hv+F)%$e@Q*>XKDG@4B}Ixj7o`jVnKb&;e|G|;Zqa-HEGgr{2o$MQvh5*UV91PDtxFb?GS7;9;{P2A=^dKR+qS%9K&j!{^IN@G_8 z-Ej%8!J2;Mg;5Lr>+7#!UFlKQTJz%Bm$?=-DUhi`V_wDE4>cL`O^z`u1{TZl%y=8w zw#L9i4ItswbE7`B?O8_!S{esdF2i#F*&%Kmw7P$M5<6`61S_P%kvF^S_7d$KFPyTQ zx;6;hhjCV$L2`8h7BIwa0}OXB36uDj)J;-EI=Fl2mrM~}F%&N#eK!D2Kq~UbpqAl9 zz9focw7JYmlMlh9vWcMqfY>jMA=unXJ5TC}U)cH#1n@_R?2}{VNhw~1dF}lKvS|ey z>H+j9xk$$r7S9-qab0a(Qak0sWbU@+-Y9V>!}BOEJ+s@-h8?I-+Y} zZcxR%Pp*@Qc#sXi^c7mw<}y5c60vzx|FPPDPT{;n`t*PW3rHG){5DykgwKBSfkz&5 z{?MF|*&Y&ljaZd#@+jnXWKm6f@nlt)5#NtDAh+hUS?8(|m36f;+9>neeP^WQk)=&f zG8X(N{n3vJVrz-L4C$Vw;cH6Z%7VqOaYVeqUzTT^@(=oyo!m2vB#B!u9GyYd!T3k~ z55I_VR~locvbtlKq~vRHtbF)ErnxHI++0=@E4%TC)^kT6#69gHl?fc0rG#UNq zY<*}0G|9Mcahc@Nn#v`K8hREb?6Jxhb8IYfEUT#@($k};A|BHnO=yvnksV!eTUy?h z-D7dw>#l8z#C#{zW(Jh7x`-OX-#LnjkJM!K^Q9?%lF7M2Kql{)kG177)$-j`!iQq% zwjqXq@!Lc@dE=?*jCmDgHl6FY`jU4ZX`|P{!a(ZZ{y|?@XvWj%+>po^Pc>tO1rpA65iz3>>`a{{?1K_>ATO0#MMTUbg%p0}(kQU^;b<9;GQbchP{d^gwNtBX z$??ky>ckV(5z{5|$lYhw=&nThou!ghmrOe`xjA!sjfTDDN%uWRao3+u zr`Nh!V}$Ok%}_~=awYEQiEsnKL&DJo9r5)NT$YJ#iI1xr3eAR*)~sov(xWyTWOKb5 zGLG{EaG_&M`&@{n^zp<`u>(^_E?i9JQ4vCR2KJXr487kQbu@~ zb^I-s&2H}Nu{lpOjo@7q_LCF=6 znvTG0GFzKa@l_a-&Wy-n2gAJkDGp^qu%T1>O+0T0ArAsdOfi6%=gpZm($y7eUYxlk z>9rjpt|I?cxm4aJq_(w)+T!`B_@hmYkKEOTa*nb4b`}{%wCU`?eH4tg9##61vFSHb zVPop1?urTo^xbX5myCb#Pm|DZo|?mf%p;_PmkE@t z_vbMw7FlMYbxja`5S~!)auWDCsW-6yD6)FX77rzDuc})5_FXlvB@Lb=5X{mWO;of)4Z;Ekg`C_rnLHW- zE03&j{ksV!ei<+y^v=vjr%i}e2469CI~b;!l-lmO;(Rlt+syJwYLzVd;Jvs19y>?e zf~)ynb#5FW3fueE;t8|D`=ZC7$oy_lXt|-x!}`yP9UalQZL)6=w)tYNr`?18kosa0 zDWNJ}yGT0xXoLh$BFimeZNYPY1hBbWuQ(a-8!K%WSeR0xhm=|3yPBj7tgXQRl*!%K zRy#rnKU^`Vvr(LlW{f#l!5~&a9iz{>ISEXJ6w&7(8ab7w^gr^RLUJ7uEV*^NpM?}{ zn?Yr?I2EKraxPX&`ctP3`hBC;dl5YKS(z7YdZ?S|&b6e&R(O&dI3A_h@;jS$8qx7v zNg#Gnia61_IJZ42AeZ&ZXbMj-pG@PvR8spx0=xR&hin8tg&3Z9iTvJc_^dv)j@S>I zEXzH^7Rg5?>(K=Ie>POBkeIE;9&lqi0p`*i3ChxN?I#YK z<ercQtM@Q1S&DbC3$3 z)_iGjk&HENheTzR0ZnxWI&az~JodKYoZk~}tQfduS!@fqG;Bo7Q8G!46LPx4Mh;<+dUFIyGClIw|)+ovdK1oS@Ju(G7``ny?lsZuH6Pm&b)dFRP z-muhRcTF|1^pWKX?BWYMii~E`OW~!f+OB_PsfuuqWHM%h?D6UJ_6n~4>MZSpo;t`> zpO<)l(htEs?=pUis3y05L12TEIbP6VkzozP8ue`siL{U1rbksnrbX?_rcv9$)W6#| z0T>fs-dG8PB5sX~3ENC!v%RT662GADRcU2xJY8j1##i z_tY3yJ4y!4Q3yejF6w-Y*H5n+5l9RhEV6D&8Aa}jMG>JEgk8S;U)G&ZR+>R%$S85C zilBVrX{u*IWI-CWX$N_Zk4zDRLaqj`FMEL^-#VFI8%lD}%$qPMLdBMQFZy49N*`H= zIdZ8uIAUCLiV%jD#jqaXPP2&@yO^85kv@~vGf!H;k3ekG>~b!R3gD66)!251y)9#f{FdT^z? z?aO~L;V7|dKJ*h5MnP+MO(HDZ<1ME9kC~+j_kAtH^WfEFa`oyTGD5xjbn^ zj*0kJ0eH=}s$WQMN;5QF%fcNw!V$J0k)Y<3NJS(p8NXGj2HYA_*ctHjMc5eAn0s9 zNwr=jP%p#ibXlk^Uu}CYmweg2drZ{kx9l#j!EcQ1{k~Xl=H-W#^xGlcXeY44O*U7| zn7MG+YGFNe-;cKK7i4=AEtYTpGlzA%w|bE)LUL&W^p?|?JXga{Ixd)1X4R7=>!;}U z1lc=d@}Z<3rQU_Uyu6RRMVFMs@|U?7ppP6oLaGb|VB5QZ4`26Wc_)?Gl6$FzvITd^ z=iLeIQ{{cV^QsB1$Bt>Wg?APrzcb|ZC{G!% zXpr+pvIB>!2+^g8IXS>Azf4DyU<$9wnHxxUM*Qtl@aDEp=;pOo5QPeUNvu7n6xx2v z1)-ajC)uA6k@PmWM87XWEZAeR@KZ~Z$h2jmv?Q53^VE|tDN1ev z@jf0A9&kln1ME-b3ek)8jHM-5+?_0Exw+i8h(!^Tt+{HD_1e zVzQ(GzLn9q|5}#uwvIIqcaTa9SFY*?k=)aJ3S;157CD;*nA}kig zyY6-o$yc5f=FBA(;^FCg9T3?)qB27QMl-%C5+eFSbMs8rTj$H1JpkVzW8Q~&azCHP zZ*8pP@?Iogg>M5<9yEbbAPAlIUE9??%QJW%C%O}d;Pa8bRS1n}np7xJPH!5ucdc5H zubN{zrT2Jlnu*X;sR(D!MV)e|md>7b;1L7%FluT2w1g4$%-H%B&i1s#WLZsQ?$CLc z_DP=>k(2WFUlZe?k666$=I1N#K|%`ovei~Cj*!o|b)c9x>C-L^(>0B%+vi@0S?azh zzue(@PX2V0`n)9I`*a(J%%3A4Z8&Sw42CPMXL~A z7jleFdap-AmRFGZ^V7mtVix^DVTk_;$o>iR{s~t92_B>ne0t+=BmattI&d+N*YHGD zG5-f7){Y^!WbM6B7JK?*Hf``>&M1f0X=3^MBO%N4I}e_(%JHH26o~ z{|KP`&yx|}e?0mZNEOz9UaS5$kNsah@81b9`d6o;e@CkR)vW4Y$W{MBzWEm{{l73* z{skEEugd)YRgmZsM}x%2wESXYJGQ9B1-yM4 zF}2r&Pa(?$S2mf8{(JzOJ)D_i)V~LQJe`?)o>_dJF}$`KjoSP^SaI5)tg}&ZD_j=% zSs7YZn#7%(<#G81Fs<4!o~;3nH1{)arl;aUW%6*pd!}8|vvH-A zWS+-PWi#ot0Ro$6h{`n(cDy3nV_gc2F6RJ=CWN1<_`*L2ICzr3vFQ zU}su9_B#o3k#%?GuP#N`*?S&k-vXJNm!^MplsUA{^j*HLrP68(qT#qamT(Hp*}DiBvWSD|yV|B4 z#=%4ROWpZnz89CsuUl5FWgL}jA+N8H&!*=WxaqIHUo&4{(21hxFr5-*Tv~6z%1|ZsVr`{LL5TP zakv(gwOS7M)N(MHy(z>w@jL_l+>~%KdMbn8;-uZo?wrmXG*pN`m)`q{|JvMr;Saht z^ELGvI@Y^;FLzHOZ#h|jvu|k@FS8gOaMU|{T&K;f-xd9C!`{Bzem{KLVkKEfDZWLd z463GPQCJ42&rMtBg;Qr+oOrRR{YD)@5_=MNi}9S@H&s#v%c#8|3cx9!Lq}W7hS>9; z4J6=wi~E8D+SKx8*1vLw;vLu?dQpEatG*!RSiBR$+YXc34#Hx66uv*b%uWN`bT9y4 zoY-En(F#=_daRAI&Yl-%G%lVO>w@p5@NmW-NQTF^9ydyP1!|T+Y3xR(V=gaPmU5WX~XgkwbTvf112xxyI+Xg(A0Y z-0vQK{HU~jefjV$FTa1AEB(GYm)PT(H&9jl)Uu7&q&tWEn#J8;7qAMyHeemU+}*fN z-41KY^G);qdIUll=1h9t8K1knWdGB2Xz*jE1$^xd3w-fSGc{ps2-~LayD{x`$21tf ze)QG&<#Y0&ZXKlb+!L>TV9@D4cYb;K?KkrsrWG}>N&kiIz%GtX8vV6{&HDhOlcle_ z)}M!jn&KZ+b_@*5YaSg8#*@?;O?up?Xe{LRBOUZrcQUU|f8_)bZEBL)>I5p@uMah( zUJnZ4wn$tNSevLnW?ns{o^lgX3SM0DuIjlx2i7OYy4dH(4!i#P?5f`} z5=s~yqA@bunsmps4PJIZ_Xw)V znRzI@;kRROT>f&-TT+F3Oe9ogyu=+Fb%2haEMCm=y4ol(dD-hJLmj$CcXTtY09;;nQs@GrD^$8yXKh4tT)k| zCDmc#?kpeoB~y3n$ei;t$CLYZ!?2frkC#!;w$b3F`|%~)9rYyhb|jm|ANHW0n_c%3 zYMeWN12t}mj@#?)-4Y3Zp^o(Jo6kRE(!Ij$JbqE$9#qQ+l<9`lQ*i>0h@7=1@3Cid4sbr zNVmfkfzQCW#Sfx1mQTzdSFCnVt@;Ojwkc~(MI|roi*B+VpO_swEOUdBuL-_vx)t3s zx8&uNd#QLWQ67rrId-T_b*RoiGYHGkuv-$R~9nDf(feC1eF;B?uq>s(yOT!pz(v8?^myUM=`fAYbF&8plY zw`23t(-NS&e6B#>=jcm8C*Ljh-`q`NlwX$!W?)NK>_0X-4!lo)w|j60l~b@DWBLMu zR1E*0rp4FBhT$oX6mzw*is9{FcQLnZr7wOMjEe{z5#P(*N1~jY$IJbB(Ge4(&!=fl z{r0}N`S*>@pPK8?F$C{02SI7V6~BTY^=I+Q=l1Jhk8gwhtEu0=eNCua8UMkKRm^ga z`bv8G8<3p3*-1?(yK*G==GMv3+iB7LF3+KP2&AW`Mfze~?R!cjnjpw8rjF&}5gBu0 z7ew2-dp1z+|CVz2ZhSJxkh=}^aPq#T-qZ|KLXhdmk+>hdThn%~%|xwMGKEer`C{Dc zxm({xVhrAubKCEJwsC&3L7OgLObn2-R^-&5k57JYciE>$KI-dG& zJy_I&0|`Ek>*F2Nrpp7DSu`2(CgSQC-WY9SV_nfuk{xi1yncUvn3BidP`vz5;`w^a z?q#pAh~lJuhRk^J7oQNnSaV0sR0~&!+2QtFZ`HSvT}Rf%;FPQQv@p7(T)W&p{+v? zY-&nsDODPs!wEFo2CRahdu%MC2c@T~@@&s(TH9dW2&ny@DfxWlx7@L_3C^|+f&tlS z$lA+E$4k4dj)ah=)F=uz6=@}JK*ElQ?JCc6K4^?7mq0})fG>Nfo#y3z#(l|@=9||i zqbI>a)&_Pq#1SZkAa&Un4qFYeO*1=_HOMKWMFQLq^9PJkkI9XGDZXvO5 zoX&pp>}Yt1zbk)@C+;RnsHWf9(7;aUQ=-yR4{urn)|>bu9D08o8qhaXc(oz$Mdz1H zMYXpd=DoAfO2evWOjTp=z7>ot6-V{qM`S}Mp7$H7qu-<_?kHg6_baxxKC5W%Rrcqt zS3Sd>YS%bvG~}10)JNU?LVMT{>QdC>Y1m;rweMq=>l~4P{jsDAv&YO&Thk$nqKrg80&4o$XCNNJxWoLF3>Sn)gD zt0uT%ewJ_K7X!T&YhHZ=p#+wt;?BAc(ifSRO@3%eKdb8c(zA~5hosSk2F#s23*5Dt z;paG45v~nN!00Rz+}Iz-p(5AjdUm&CbQt!&^unci0{q|r$zJTKaRR5CV27+B*O+buvj-aCKF5>wq6>t>` zs0rG`t+(>k8)7d4^q9{{>eX`kE2tavpO#dU$R%=8)@^knF!OHUql3dy*W`Po8Cg$V z=fUceKIGEY!wCS{fKki>gFf9AM$zJ~*VRw+E7VP}mouEO&9dq?YmraOY2^f(&P1A0@s{e_E2j-YoUgoAfk)+nA=7FjHaHN}(lBOPVj| z6Z@?P@(DI$Pv$&zw?gA!k^tz>KzcV*Avqj9)W53e-+zk8MQe#^$7C*m#7eeEsU-u< zAugpwUVaVDbMoo}Eekrb|= zqow9Hq$#R?I0*J&Czo)y``fxp35X&r#2}yLoP}p|FW2g9wJ^Vh1Unq}!1=fQL+o+i;*itC^vaMr@_KE7Hsm{bx3qSBGO%ZU;+$*Ti4SrV$0bc+s! zuUA~~zt_`q=cIg@=sIz~BYf{(2OD3U(};=>o)SB~b(3j}pCvs#Eq;U5SRX1)U!MrG zz?bgXJG2R~Y1artc3G8MSPnF>EE}w;NnGj*D1K;3jsMAGFe7No{%$t=YB%iGBoIDZ z{PVcTbzU?j-1LNoZD@;zZ~dnrZlx=t=A67Q9S-$LmHJ^O)RAK!`CS}|Rybaq_pLwR z8~zoIf*z#**?l+F(#IvWNQC@)g+TM0k-=|}QGw4ky+coZm({y11EP2Xkr~ncBby>Q z5ojy%pE#I^s#hd?W#EsbhXFc~)_XyZ#Y5z#t2Ns^Nlz+G7#{+jr_Xk;kb`o)u- zwZl5-Wgzb4tS|0S?v#hJ=-co`42;xX`Yd=1hY08Pjc5bus@u1p<9%F>ySl8o3GBS= z^OF&d{uyetfQnve%V9+=n_T&+Q<#%!-mzA9GWZyGGdi`?{6~xoT6%tNe71 zr*6w0-HE|LVwznZ4iH}ST9`Tp_xn6vtZ4?2Q{Z-JAdfVxt8#yUntPW#!O9P{NwR<=ZCKG zlg!8LW{Y#+)M@Q6(4)z6mkH;CdTsl&#qEbmIYJYjlP1L=7(`c`jcJYZs8MXyTpFlyuGCVX_uRQa10ySy0TloAoA zLK87>YY$|GwCK;W)%ISTZ1T{G+gbEnACF;N_AI=A6_X1C!kUqC+`2KP#K?rE_qD1h z`{{#&83RvWmJ8+g-<^NvzN$86tN6RX9z%whk)Kx1Ul|X%wv|pWpT3_sTmQReA&}Q+ zg%rvkF!Sv2Nv#(tg^~V77qDbfPJ_yLDvqYaK%l|fyuL&;S;-9 zYo7g=ycM6Ho0r$J<4k|VMNBDO<@O)b1fC-==!<)pNU2Ilq?ctiwrwXHt3Q)|)y{pL z-T6xMx;CZhzLjr5I{tgBz9)KO#@{urx(F1B#4i4PCYMhaUypgbdfKm8tU-fdr9+Y0 z62B+yew(5IsNi!SS3#QG<~D*&hQn3Yyztlf6+anxjg>SU=~{qlX&@#^W%}zk+^W4= z^j}?iy2fOS8|~(lKWlLHD#(K>B1dI09+qFn??&9Tgi}0tIW1bOFg_T*?KpU!s)O+e zsluCL;Qn^oucnq^5_zq|)w^w1{vznoFU z7qA+d}FC&4+hyyeA zgzC{}aY;XHA$^b^EWcde1mhucPxo!x$8u8JsR5nTH>8Dl8(3G^%L6~E``)YMAIgnb zwzw&8ambUzb6u^x^0hU9Ul2(IR2O%fV&Jt>eBB1yGzewP6icrMJPEd#H>|dB?iv4> z|Io10nyvs^Q2#0i0N*MOOI#hjfpBiJ03Hn3Y;Kul|JbCr`>VHa?}QC=qwF)0>aob` zL=A^mx@|l}1@%XKRFD6B^AWKO!2Rp<_cSWINEp`L%5o-V?_E`X z)Im+T%&EJ5+;>J%L2z#~`rN8!-_%8Tu{*k`U^*w^w;J_ZpknpnPSF9io2=j8k(b)1 zp>8{GndENRpFaIhmUk$r`%qVoHbBLRIhD7q4p!nH(qUcg9>LBCg;eqe!j{te^B5l8 z6%|9NS*UwrcRpZee~2Ds@PBA}^FXNf|9|}U?v@*sTM<$w+_I&y6e-M&Yl+H|B`Fh< zq?(ZIbMC!JSsHE%StcQ(nxxXgoKk~qQzS|?HJHiPG-jCX%sIc;>GS>lb*D4Sd7anm z`Fy+{&&T8Wa+&QPi`qVn_+WCnf=l?lJywjP6&@9s1uSs6tIk~W4kk#5i{O~BbqkS;t&U}Zbq1O6x zhKI97=mW2hzv@R`ynXKK*1k-v4ak5y$S^oMp04=DEQ5m3)kSUy~ZK``wFGigUY!rNUZ?>cd%^gI`#0 z8WX~eQk}e&9VVBOVPhaZ0qJ}`}Wtx^^&E0|rdMK4>2?EP3X zgr~@y{d8l6L67dwJcv!aKCn zN+%2wzI-`PXeK-hi(>xfr1QP=d=t$nL9A&L7MhTlcr5Yd?8#d{JVZ0=Ff;wPF)s7O z1)C>^`#wkYmxgWpag?1vm|dn?A`ad-<5&;>^<@57!R<9SscGf|t=;y4NqNo4;|J}A+JzvS?8BzS;t-3D5sW{Y|2{Ps%JgZu>a*ye)OTLcQFkY zw;qa4))-i5N#0ncJj+rIXFBBk+g0H*czyf7IwLal#UB)pqJP=5wUddGYsvM;ztO5D z5;7V#ZcU3nrnc@zQefu4j3wnKH&#NgAG*Ag>B;JK0{*?5VwM(F%)A?rwr)fCnMAi! z;p0zRF6S^^?YOObjR~EN(qT%3-du{;f&r{LbKRII+g4JtIv3g-G&fh&^XuAkW&Wnl zyABPEtL_%(->?6%Y*a(0kKUFJZ};0a{^h%lV8<4l=-g#rlEpXmFNj~yMwFk4?npkUvG3w0%Z3(n z+KvyA8z0Yf_1mFSs~P2nlR3AM(SUA! zBsI3gwp`jB;xPJ~$EW+dCEKjtu9M65CNn2}hl>|i_zZ4%d!Q<8mw&$ovHv(RLrQHonxs3#xUN!w; z(ERUIGCbj}w;5?onJSX~k0XA4Y9@8f8tO!G=b1%^TExR>WnMqe`N#5$lBVUnx6yy> zUOl>sSz)34b>(&Ttk>^zeixSpxG*jrGdR~uh`2K5bU7t@b z&0}lVXwU2zJHr?Z_;g?c7YeMKENZ$#q-%FFQj!>m<*lctg}3izsE!ja7aF&Vn1O%% z6Y6!?-JKNRN=tzQ-^Jheei`TbsRFV$I#E^=p(L;4$riqtv9s^bvHZIH^T$t_OFp|r zYNb836+Afj*W?c*A}DrI>x?j^x^?=&DXZr>OJ`#Z+ppedAUncNH7b%J-%I`uGj2-X zJvjnG<*EOLc5jdSvLW#KmrJTIlq%iUcM`P=j=vAjTX$v4u2w>L!5eMIZD)SuC!})i z{)lU@jz4hF=iMWNo0(~&y(CYA#@cNk2DsNB8i=0#N!Z%UQ#;1LcJ))5SP{R+_5GVe z&nHb=-INE;!-w}I2R3P3`bAeOdAR81y*){X&`;xCIi9ga$M1gYvVE_rc$!4oa#ZUN z|1y5&%7!kjHOlBcE1N>k2irt}58mAI#*Y_2bjX<$HW}O=z_uKvZ$5b3He%O?J?#@|4T2cJtx%8F||G;(hjM%DeJSjM!eGe^z_np>D=)!(+?+(Zq<+ z%ICT&S*LWCzK5>ftAxhdW(7xVGP_6(cD=K2H*KF}ZInOTiT28(O5uU|H>Ka7OjqP9 zJHG46d~*u9X=|7HBa=5CI9gAXuQuoS_dYt@20wrNb#~c2`TS(-#}M={Uzc^k9?Iq? ztn%;XvrQFlTi2a?em-S^a&v=Z-;F@{fWz*at+@kFeiOhL{WI~jwcGw$sZLK^MOLr| zrxRJqTSD|nT_u|9848@Bgfjn;EgslLql7aHlG++HIIoc1yd}1#=u%#!rPrfvo?uCYf93dYfUhRWYPEQNmpQ%mnD|zfng(1UdzdfN zJoJef!W;4DkZ(z~MeBMJitpJwzG^`{h+j{!B8%zW_FtE?C`YJzqKpwuXt^lT!Yi?W zYimiO%Ve%a9hA_DlibiIF7XIeUzCxf0hcHBiSF(b7Fof?No{&8K^S$3D1&c?#`21+ zyb>>R8HcGlqKpAeXoIM`gdF0JZtck`Q4hR~S?~fa;OkxJa?zolRgK2oGk>lZ>G}1Gqq=z0f$V~5Hgi`nTD_n~ON10ELBOiN^17OztkWk)` zC48q|*=!A0Bsu9d40LKateo&Ega0n*_^@Smq1%_0f(M-g(9^Daj&N~-B5aN9o?AM$ ziqG&fKAD!w4YZ66zVU|NVH#2!b%Nhv$$2rGv0?-E7mMQUZa+7ENrMnxDBQzDU%7iV z{^&<*jb(9~9?FWY)Ql-D%kbw7obqgPgZHFlDT@(VD0G>7sy0!wI%HC^djsRrTR~8d zBvzU1jP4gtN)ECZ`;6yOD!3`yEYuI?8@4`c^Hhdh8;k!>-kL94b`V~6PBJN3$#MfX z2&tED3wIh6M7R^1fB3>hX0mFhn;f%ZTCB2qxi1`O=6D0#bPdZ$+q$00a7ixDERwhG zeHDp+m7SV9<(7D(o>Qs`yH-lCHXAd8^@Zwg$+}Wb+^0!xy8t25y(9lKOV*?lk!Fa^ z?yaT;jn$+He37-TLb4w%!LYSR#5dExX7#FR`^Q|c6ixW=n38%-k?;6S)~%VH_L;Dr zC?_2@fgW>7VNo!T)IPt*(rftIq$H}E9gFUEZ6v*ySi6(&cjlmMQ>snXORp`cW0p8c zjW|nh{x*bYJw|nS)*hrmEZUv(1v%Tk+#h2YDqk27<#46+v1&wW>Z*y6GLlziE{&qE z?8UF&mRf5+#%kY$eBI&B8Aa|ol|Y4R0~F?avS~_Lp!; zds#mgSDmsAs&aaR<@}QL2BLAE?k%<%rexhY-TUeZOqJ9yuE8+65~fZj26=0VF_ z!BUp3WuK2cG$%^U4)(A7<`!I=X+C=$s^pGnM_{`jvZ#~~EBnZG-tx#8@NncX6Y$oH z`m`L4Y*cf#Vw3Oemm4R?kPWJ}G4CTysXF2N->Wxv-hWUAJ5>(TeBT)m#~J>qTB9W% zq_Fo<4rL&OzPasoFb^%zQaWS{jvi#lRT{tFUslGc@}3^j1#w`zzbjDa5JOcMsJcM? zVFXQR6@LT{tnB1(61jC#{E)?&YYBT_ghZ_==jT%xks6r$yn}j_HQ&}uLqTzR(g|)! z2}o6|ddV3ZxjR&Gb^#xxfZ`!OO=_FQ-87*U&9wQ9c-39ly1zY3w4VM(oZWF*Ik^eZ zn}_%@j+JHXH`3(a`Y)GpdO9QHKg(ymtJA^MLp9uBod+1KIu8WmizUU}5?dVWlkH z9<{Ods_WqBR|ju{Y^tUO7Rzft<%->NTE4+9Li5~M<}Q&Wpn7B*)aca6{)$IwDYwX2 zL*}-BviYx9>811g`^fFaLQ*foFq{0`m~VZA?xye1-j;)Q7|XV+Vf!CqSMrk*Do5!? znN9<%@aW;OFKKd~87tk8efbsbvL?-JWM4r=L_v|G~t{Lq|A#!!)``7o>j3HED@z?PW#E z2epEbURjeii*yP3xK^qJvBVAmd$=>x z|HDS)VLgid+FuLC;UmF}3YbU0^rd+SXL6rlq&#sD2q|y5z?Zv? zbb5fat~oOF9@UD0Rl#j?k|u1wx}ZCbQ-^{R zPgh_y)|a68P+}w~;+P?a>mbq3r{oYF_G_C`d4|A7n; zn2$$eX^i8XFmOxuYQ-@bwhvjE*-wHNdhtDq?tUWrC_R-T^&jiXNS>9$0(v$D{P-x@ zT@phrb>dE(;B)fnr@)MGY7;^v>&Ec-IheW&>R2!y|nl ztqV_nGJLYe=^^_}?%YJ&rrGPTkFY_TxJ-t|Y?p^Uc^jSO6vep(LXmUC1j^#_Pb=JS zgy<`>CQgDR8sAY|6tiKrjlFM_@wtt71iPCqS`q;{Km4a}qI(;h>C>*|=;cQCA`O6T zqR51K88m0*r-fdcFx!SL&1|9WR#yR$qM1!+c#j1JPu9L8a&IvBNM`OmQWZ0}O1dF8 zG=i`ud}3eI#F-FQ=VZzz0+T4fO>04L|cmx3khqndzbw zZQIkbnA0PgfRZTI`H^}Q6GgEo*ej_AuTM9O*C3vkQPyf|);-KX4Ib!+`KD-Wx3nJ# z(SwVuy^Mv69tcdUqyI0j+69B|3a|{=8dicuz3A!0|Br-1XPYx%JHbM;$$fQwf*TTj zyE4vg?{@Yah;&Y4A!FY5Wggh^C5>qkE3djf@(uO%pIEXv)f%OW2IpB_A6%<$ux<=F znrT4FIFsWNGMV5FP;L@AE8pX8h$rqa1>c`(`@4ovm5<^PJE*|vBK-!ZUb;aw39?dJ z^q*WEBIeu}D>5yVlRk!#&7r5@fDTkl)RL0Yx0Ml(n9&h z-RoYbr;O&4Y8~Ei92?Eoqkzm3u2!}QjN?L-gq5Z=ubn{zdRxo=;KVoBmI(TGHAstw znc303S?qy{)EV%Q?VbFT3P=XI1HPDp@{O7MDtl;r0|&Z@Z(1l5+E*t8gTxVOl4u2^ zWFj+pLI0;R*h7D!BiTxsiNnAt7X8(uofD8rf43c}-dhZB0?!E_6vnjY_G6~iL#L?$ z0A1t3qC$CGTdPyPat=6~*T=a9K5B`7beAw#yV*oZaZcXB2dT=FU_TKqSdJO6;1Mfy z3l^fs3AFU8Qz}&Q(^m6`%nF5)oh)06Y-~O1>`wTK1gW~9FWkw^ot{d^>(Y&`?+LFF zg>ip}9pqRgC1*p9=fn{pAhAc>B8zkWO42`KlRk2#_awHfnuu?VwHh2&-RKK1!?Y`+ zArSxAZk!+8s!3f(p!z(Zv;rQrAt&ttQe>{I`pjO%mq0XdX&K8+qf0J2`6CL++t#BS zn32ldk~h!fEFz2|C8{ix<19Dfvwe-{l}v7=br-5R)hQ+zQOHNbG!h<`f;R)os`s$>c z9EWVc*yPX!ade-++gOoK2ET%m&A{SyVxN6Ut<--SqQ4+Xxv|*gcS9so>Vm45>@ALH zhB&fM5MVsY59wsz1)&24AP3s(?x~k^l$S-}xiJwKoY6&&Z@8kRg#1&TD!`iLa)&Itq#G31yrx#iQ2({O zIjvqEe+bYY*j;Fnd7Z$#bIOEYtC_95f(KJgHXBe2%`5VoQrb3H8I)xgY+}$8u0iA|q$2Cg1D6mn1urKqaE>7c@9vDW$#OKiq>h9M4;(D7 z1IV4F{yhmT=+7*Jef2ve@xcp0_+@63dyPex+S^&#4eG6KB?;ATgC*msm(4Q zLci7^>pbwv2i^zReO|rt{LabwSTsmA0ue09FPfDcn1-%!8TZ97kHHgOj;!#WAx+Bj zKX>BzgfRkB7gl|M-rBh@>Z8|qj|m^2dTT(OGc#uH(tmY{VuHig``p|0sajD4ezQs8 ztfxDmktPtGJEFs{M@ML97V_EqjFm%p@k6-_pwR85Y-R_bOaz?C5bh5LreWu|!59ya zn=B3ZPyLRL_-2DQDEw^n`tS9LkWYec{|qP{>OPa-7F-P6KUPC+He=@bOnc245>hfm zd`pkiHH#&`vDlG?BCScx&6K%E=nO0Q28tGu}l)r+IC*I#D@G(ZeS}vIN zMZ-&oR^+Ar@@69wYPpk;;uVmpUUdue!)n(%w%`KMOTSPY_&~{vdM$_>C;{c^`q%dC?5) zSJ?@*^+N~HOWEWb-tV&5>)$BX-Ae!b=$pF#9d)|1{D1sri^;jKB6qk6AOP7rRktdH zz&tP3?4x(o4P_BcsE+uBTT&aAr|iol^@XfKqf08Q=LTAXul=lB^)k6IQ*S+S97>nz z78{KT*oo-O(`=fl1~%`&>d;~P&M6%G)$tm^7bg4)V42#9WvvPAnp9HRzcb`xm#D5d{tp|<{`jH}o>n&2Pr_axP9Olc^xB4%jGQU;NmO+TP; zY)7Xo;{%4SBEa1s#3K-;r*K(rGA5-|E9~H10IZn50E|>-({JbVNy@wZOK&Zx)W0+2 zT^EM3{|r95MCZv-a5R|rJ~y?h)xN&cb=%-k|WZx=#p9S-YlKodUVr0031O* zgr%32{GI;LaO7gN!ZJ%#sxyu{YaeXlF0!L;#H^}Zgn8LTv(J0?Ka;6x#$tWP{9E4c z>m&bT{Xqi`&}TDrX0plb#L!MU*?N0EFBY}Qw0AsuW-*u(zA4k^l^T4l3mrtR*P{%c zmPoqhXoYtt1bhmxn@PD3;Wi&xq*_D$!LsJ!n?^E=c%*QyU|NmrP*o%f zG$|CWSOSEE)+@7tCXRhGy5r~y4dp9Sg(`9OeLZ@cXFDr>Tq0WX0NaL_t;E9`*v`?& zRsEVc$_5E@bOIYtJHo{O`T!LdvsYI;;Kk=Z+#!oD`JT~8vwoXCQm{UQW4(DPxXFkH zw0U%q{wR5~50VLV4xsepXcMkaHZP3Q;*D5aeuApxa&5u~Kh1c*M zIp$H_UgeI)ApZ(wEO4E z2mrJ=S|HT{0{#og@}Rn6MrQ*UROMTOVo*l~r{mSqIPHXbxxjgnj~oKSnpOv6RbH7* zFwy&%n_dP7>rWVQ9O|Uq+LC(tG84zj#5a)ldx2`EFIKuqnUeV*>4*;bDxBFN2<0sT zxMt|^uSi#CJoQRDG3>2Q0zM=|4bIEq)mB2bVS!$+Y$o>}^W5`wKB-3G+m5uet28*( zi6(@PxjXEjOaM)Q%V||jLv>1;OR_y5K*lH*gM-~DV84EfDjhG^WK%x;zbxA+D^)&Q9f?-#BBHPg5|V+;w&?8g|t zfr(=HrQ-Rz6<@BYN7zJe>t1~UDYB&`>3z`>;_R0e<;ct}x;L_LX4&T((v6w-;FKNx z&0P^mvk4id9h@BnpL=EI;QtfQsy5Mp2PlBL_D6r^(SITxAi_dnNHr0sp9&2)H|8=> z;tCu)Wes3UmHmk`v}+*P>P0jif`2N_=>pyHhBUNJPOw$t*dZL%it8M$RY5w8}EKzVp#-@MYKAB&|vb!rr_@gfsD&^09F z6CbD1!pi>*gE>|Nv*r%Ii(VO~sLe6Af0WIMyv-Ag!-N;I3@#~(qHIF8&PVYoknRPQLN6=k8s~#FNFUX9n zv9bIll3S{p$SV{zsm$gy-=Tz~H+xp~d8ZnQyu#6_%4~tk+wj}B$N}$G1OLU#7 zsXjD^U1?f;_K`wSw}o<{B|ZykGEp-%lRKqBoH5d6j3Q?NwY{7zINZ}n74LIMO;eVX zfx`VhUI!FHH@H)UEkC|YHpimxbGdX&{^XD~6j?V-68Ozm&RPylR+VvH8jq%*7;2Cn zZI9@p>S~Y~i8l$4U5paNnaF8{a1@h$mfOHVMUs=~Iy!JCkarr5sT?hx=W*zN&R1vat}OQ3ct z%8${YfVzGXDMJn&dP`twlFhujr%q)_qrbr*4?`%z&u0zAm-PoyJ<{ zLNz9oWzadN5;#kPMEB`~k5NC~l12ao11xh~HA4A}`baP3@7Fnx$sq>^sV_6jqGfED(AtFWJEtd>HCpeyP@gE>u*5+4EFB8|1KlD&-Z|r4|6k zIw3>7dT2Y}O(Q$4Vl{vf(jnm-;7?Br(`+nUFDt*$CdpOPWOHm2kK83EABZ4JOXd6+ z>Z+=Tf;bIMPu+~-AZy4hq)R0bGwmba@t#(sva)no@EhcB_$FK`8Zs^mco?o!TR2vO zJk^A+#H3G>Qtv=^xRi>u{=Z5Tzhj_@qC;m@JrPzPKe_6s$AiL`7cz za%dTJ_y1W*oae@)gwaO};+S70Ww5)xqUhXW7V&7IFiU!~(nsj2c@H~Pd{a7}+i*Gc z5|`~~JPRtm-r2WOl_LqCvZD2QbeXZ-NY8NI<)tJT5H1^-!giD@th~%Y%-H>=`J(JNFDep zDTm*oQf%ZW!PfM-B&+;XCE_c%x*99hNuTasKySMBQH~TUjUGVO{j2*ZM+;?Hi46;w z4?~MUNdhGBncU;p+h53^PwVsPK#G7hqq0EtNUoF0a__;l!UxNucJPUQ#>xaAFksFp z+|Ed-n>Y70mM54hKyW6QC~WsjKY36aorLt1{A0b?dWk}150*SSs#-l1N38F<(}EGJDqw#;_@ zveW1$*An=?4d?up<3Qb0(IA!jd}j9}q`$Ykl-*XnNj)<19Q+=sT51E7vap-v(*IP3 zejMu)93E3%YDNjZh`nn(#Q1u&I{2q}qjHaJ?(PPl9lZ_Ai{TnE!8LgsZeq?nH3!-e z>7cMdBO5U^pBtE(S}}+Mt^1fBiLm^#GJX9-2g>*w)6@M3=-u>ZqoNmpiL`6+l3uLR z1lm)(;8MXculW^MjNXn)Zn}H%ksv@s9G6XCz!Ri;1tKS09jKVvgiaZ!-g`PF5-b9A z4p60Rq{)0Pdf)7qH6hPs>+r9*0jFzDnzp%LgpXJcW;-XBxP){8g+j~fI41pH^;0_# zjUOa!xZ}{w4Fp0&(SgZkr&sXq2k@IV_}>n;oAY>#iSek!|Nllyu0o<)Ga&~CouGDk z)NJP)Y$Gq?lV->m=lVPTb~W|B9FF62S;k7ZK-ow>nMmXhSp&+g%uybH;fB+a)@;g= z6Z_?R5|v1YihQLZ{>lwjbh>1bha^n=)Od`t=|Y} z!C1zBjHVy-Fa`9V%q!qM_LDRvyPm|@`e~m4K9U$xSJyP^k44qQ7<6rB7)-=14x$Ou zA5@;bzHq<>^*dj8lyUloE2n`*#*QdsOk;bhDM3JjG~Ymu{0hIA$23G4lhi#gBn8w? zz^FVB$--(nPW03@IWOzJ)3+kOO&FDdW#Tpwa1omhWK+Hxi9erqH69d%QcY3pcm0;m zbP?Na`zFAsL^a1zbwm#{?(@5E#N#FJtR^qkst8@Ria7Rmj*`rhS3B=qp64 z*yw~mlGXzi$cX3gYk*#D;K6=D)X}9pxW_*9fH1EI%1mmj=9U&?reLh8cN;Th44^^w zspyUrW7|a;W_X8!ZLv|yU&2TWY!7eAJ`kWtb1@d+ue@S`o#ep)TgQ8#ESbP>kdY%3 z064!1YG}-X2M7AYr+6?Mku*#;SI7t!Sb9A;PL4VAmW-pn^T^>#O`hW#nDk|8^h>>a zQRiGLI&5Fb5vO*G?(j03uuWSAV^G!1Y0{c~*@CdKr^nAw-xmj@(%PaTPV^Wvi!5)B z#r+iO^0L_$6;)&G0OYaid(u`wctjepsh<+{2+hxENuq4;a=OiR#v0-m#4gtgdW8n( zpSs(bqXgo|4j@vV{Fr?|4>!aWyk^0f?{b z1<3Nw8p|Txgwxdhm10f}C{KW+@f4T^QKJ#=z>R+3^ae`8v8)P!?<8_xSm_=ZLuEMW zkOJ`lY5?HNuYD;~Htno@3v_szWxZ&i-QKqVmvX3?za*K!$8<=OsfCX$M*~H+KM<*j zD8tk-dacuw3qQB2qtvy3C076p1)kY*S0QDKV^#sEp#O2wfeqc+c-80w>R&%abGx&9 z^Up!Dd+{1-H7migOZiJZK~}tD8Zf%mVe$v*8=l zB7^Fp(@c9KzR@s$D^q&{6yP^Go*E>Y56?(eTV&8$2IK$@Ma%9%uWn#A1_KMqtPpX^ zyhjBlFcne4C$yZ_ky;l|#Fv4x`64|eN9=+RaXUUAlAJYUn%EU5S91O`cX~4zrLC!B zPu+K|t;{78fN;B3_Q1Ay7XIw0m~czc(kDQn!qlbW&to8>hz@s2*0c$^~&rrs|hiGj&^8*(qLG! z%AJ1y$93$I2f7I_+Jb<&LprHLjx<3v#-j@rtacbL!EH6+|9luQ{M|J1jNR z3jK}?DDeGZK`ILim}9R71Lv4f%|_1-q9lD~iHHb@!&BQoL%Oi02;|@`DICI~PYoWF z1H|32jVj;DUsW%S+W29+$OU-i`FyNr5s)O^?UA0{!)JJ^fUNjQb{uVcXI*rfYF$;P zewf+wG9>xvFp!GL*MSK=LAvK~!Y4pY_ERn`e)l zCbAe6I$x2^fQIY&gN_2Z;g#8~oQAPLQw@dw#*$t9JUHU%?+^6*zDSGZNB_I{bQ^RLbmxGq?W*H<;GjpIG!km| z!rcXWfDOq7KB>0hyC*m~;@|sdW`(kZ%z8R-TGWBPD9bVhtxhFSpkdw@0qJhqk`X`U z9hYQnelz%-TlgaYzvHCcLO?jsIbcn~g+6R}s^B0Xz6lzHqq%0vq-q1ckwzC{RtJ3O zH)J4gCk^$Z8QkZ+YsU}`u%2&_r#po~uAi|)k-)WjAut;W-m`hDsrd>Ct#YdL-zGG9wwyGi3$=1HQBXRfDh#<6unK$Luq@_?U$8+7mH244nki|8_KI^0W${ z81Z&=>TIb7Y-8uhs{&+xH-RN}=$9^huQa4wN0 zv8Y!L_86^qcm=u^j7;#Zg(Kg|)6Ok1hQ^~)QGgiz9x{e`%8YcOz|NQIbF1~rw4gV7 zH}_T+VrzLDP63klTqZnJo(ZkUYJh-X6OWzXc@~890lgy@`g_`o4+-!+pva7w=~K?= ztrh{p0>LZg;(p z?uD~D*c?DS_RXh|`c>@Ubf3Q3C?xfl2{l}!+sR^wAFOLrG&^+}G4fX?gN_nvS3gs417??#wiR3S z9fmBxA8Q{XFUvgIdQLP2`dHu_(#5Vq!b@n~L@D0m&R;75W1K*J2Bwx-a z;t$YVfU^G&A~RLTQ@=cQhsF?>+-N@;@HK%`A2EgsxqN}4@;L}i>`!4Ua7{>m$u04x zpI5Bt7vO{M$fjAvrhb#r=R zbQxsnWh`O8L4)cBZk6C}xX|*pdegU(qdE3N^do7^O9i7U$s}tv=FxylKo#7WL+!PV zQ)25(6>d8(!bcxi5znp&85?;|#f~ZgO-u~P;+6ujgKyxIdWO)=(W>7DmBudy);R0~ z`Z|JukABGq^b))X>^txh?@+Smy}UKIIw^`{z_K+vg4wA>f4H?3m}SFvy4-rUN9*tt zj08Wkv#4*veoM6o`g{4$U|Ts7&vZ3ipjFMy)7$Q!Y5 zUsMk@Ug`0O1HBymuu1g{4x<5xf$Ic$)?BB0EhrT`r-p8w0L#S&O=F`#9$s55+6H@8 zej|ZxP7G8u|J+IHhfTM1njXdib~nLM!!^->9OYZN*T-UX0NF4u+{s!YsMI6uC^#EO zRjJY=RFBHQec)hFNfntW_ct7!`HGA$-aswQd~p0-9Ny+ff6#^tI8N}q<`oh z>X7NAqtX@|FjR5}vIR)00dkSMOCPD@8Uy|vh=n;lUXc|R4bua(GlqQrQu+Mj1rs;H z;GindyHHUjR$spiG@mR^Q>x_a0ii;fo=u{~G~HwW-oK{|V(#QPs^ANK?YQF5bMj3 zYOplYUq*UEMB*9{%2q6OUZ8R5ovwIv=xKM$%Vmy0mN#ZzV_tg{@VW?!sxxq0XNbk} z?C#8s1Fc&?1mb7(wu2`R0~^Y#C)u!HunqovjRXTuy*M3}|2SGri=i4C(!CPVQFC|7 zD6%n~i8dRgy#zD?H1I`QKQvWh>Eh*m6zgwO(2jluUXQNJrbLd3h!iTkY63Km3|}KL zIk*MOIcY*}C9UtHa8`m}QQtpZHOinFD~Maabwh7)3o|^MeDpO9B3%%7-!pF!?lcy7 zy<*Z`<(v3y%Wor?wL1y4dda{xZ&vjZK7tph|A@5Kh?N{Zz6e?xaB<9{soK^Tff{iC zwT^+$qYc1m*AASk5EyYFDT2dVX(1WmXTTo#y6msEX*A;N^p zYQWw8`A=O}+`(3Fkb~B6%!Hx5R`f64}IWBLYXq&3}>- zg|r`=yvi`;?abB8JF^N(xKwAHRC~eer2nXz;{J{I5)LG2cK&=oKl&U>jTl(V8?mEV z)JV_rWV_p5JkF+fw+y#5d0?qLm!qdN;q$VjGFz`ab#9RrG2!}eY~R%4B)e5cCjt-; zUc}zbb*9*E-gvoc=wHI<60Ope^QO$Kq$H!(M;_Q>UIcsZaHVvcDAuUpbSrmXPj^L3 z&}DR!h->{@kr}#)_oB@0(Ox0nis%^cav8{z2_^Oz`iD@fYLtTYS!<|@Ez;T;;JteA>^?6+#eGN90>noPeIyH|JM?M5qD z#4ru=g_wS^adpRObY=QI=tT1UpZti1bHR=S9~j(7?p4sk+O5rk`jOQCxPcbg#0=&Q zBSQxkAV{DeKUodi@|`?u7Y}gq*po|1I)t_yb1Cqj#o^OnFTw zgOWad=X;V(>+chBWt`5sdD@;~vTbXzsC21ZPRh%!j zTiuNJ!FOwLo+qB`rzukg%+~_e7tO;NBdFOFYeIDCG;Uja4(-ff?M{J4L zq-OyWY`;vM#!5k>{*rVi{>t$fxR<~;^*`JKCg5*@79m~njHCk3p%9Qmsl{OP<7V;-O~ZzqRo>^6Cno zR?bPel@t(wW3L4qhH;2@Ib{nzruaU3`0Jt$6rM=IRns9r?TyiY+Eu{FIb1{rMxJVo z*R-IXctUqRI6$+5^s7GPY)K;vJ(tR-2GsMZ$G!qt>oVdCpd2~^q*JVsTcjgaNFue* zCZ0O*uY$P3O0i>-Lr;f`MUmhQz7}9&g^E=#Iq71vJ_-OsY~=(MU!TBiN_e4Chh+r2 z%FX!+YsBJkM_c7aoKOXTq!?7lZu~?Z5T0@jn6`3(KN9QuzI**%ioAh4q+8hPH^9*# zgyz-@MG4i={zMau;Nf`mHGzcPA12)xd)~YH>Bgm9a;FY+J^B&ZkqC@k@zV%c=7F(Y zd#s;+GGLEiy2S==GD=^j+jD9R*_+-^YSINeJlG$Bvy9U77WbX{iqtG-QJYEu2tzqF zX_M4NF-+wZ9r0X(UoYdhY%lMUJrjui>~qA@JqoAo)>m0|3Kh%lX_qq2@z(-^^!|G@ zT=6x)ZEFRsX~h&fq)Vo@R6Y~D5yiWxtSus z!R|p-%!vSGZ8|e;g<9_oNrlc8c7|!6*3kj&>2VxNm=?NmXXgRoxJhP!F@wjAe?bj9 zn=8`N6)T9@_g?|h3pSp%N4|rEDc1BKe1_FGHsdeAc)U!(5f35dh~gI4a4`$(?ZeyQ zW%|~btwDLOEEwSZ6zJu>$$7t8al!}L=!qReQ@>-IXz+gE&){OrB^w5tI@C! z`b6K(R**@{c@b$^A!7+bIXxUy|M10j$tjktwMXi?m~D#KBetih4LhOK86VK4Q|9$M ztTmwzY`>jkRJD2v%n_Dl41D}2bZZ4JrlGOvRE(?x=7<3RYo7A5w=WJ=b+sd}=SVDA z3@hU)(X?arlsRax4SB_|`8LF991iT}Yr^f<1oVT|oY7I&?^s@Cw&He^M1L-hsvDlt zDXmqyXMsCWVeSr{2xT?ex*+OrQPCQ)0&}fi0qReJH)7R3SdXgYpWBA*PG>3?<9ly@ z4PveN^NfsoBRJ=Q-ZE8u{#iO&9#`nO%3E-m?uHZy0USex4}tSsrUoFjb=^c0PPMYQ zT9%sw<%ECEWQYLmQ_}vD{|&xC@Ik=COc^QcRVOUV;CI(xru}vh{3KW56d2mIx$!00 zoWrll9KM62eoMed+)>+LlPM+V8E@sk+=uX^D$W(?if2}6!`pc(cgMl)rC zLpB)AgsN9HO*@8A*G^n2gP&a^V_tY~7Aa_);}Rle2iztOBK#!fc3P5CD>ZG&gc+!Z z>V_CZLuD6U2wn&4<$yA?aZseZCO~B8Cms0pJ78p!%E4*M`m`KTkv4#~QhcI|lER>f7LMKw1^}Rh6IZC5HDfuY78|%we%oC>+31+&Tx8uMn*U+k%UnkB6vK(2B%KEA4S%vs)ycl>FA z6x!&Ip_&tdGP)I@iWL0bf-68R!fcxbsSgFR%kw8I+FOR*60aEb(Xf-L>D#f}%|EPV zsr32#@r#wSz{rp8-^j{I%}Iv5BfpNwlssV5gYGbSZPMm`oiY$kVzPB=h{zWt0XXgi z+A8RW1xKa-@iNOeq26*v(oO&w7df1BzMEt8XKeMWR@eb7BGc=66a@(~ro2 zPo$f?p+c-yzDP-yR`M!9H^{2_Zv&gTEjdn_lJKjx7I3-fVV3FlgsmMVV3YW5 zP+85>5EY%)`&yFCwt`IfhYt9IPBp<7`?l3AdN;DSZzN~~Tp~LJ=vfe8W56f>jU9;C z@Q9=KZA}`GrKl+G^SJ-G;4oF4zv50Ha*7B3O2=nYnL`ixc~XY82KGCT(3S)onq4A) z_o371TGxwMRN5)#8eVF;OGrC=DguWw$pn!bIljT8n6_~Uu$pyX8~vv)lghYTfTl-44_U-<vgV*^2G@++4%%dCKcr8xliEt^skEKR zYx&@4S-|L2Bi>FAju)9QGaK{9qfSym2?lbF^!z$mZi!zMH)Diih`HPl^A}+13TdpR=}n*uKSH!JVPp>Acu$vQ6T zBI^D`L$nHfxuk7}`}>^QUw%p4#yO3H&_w;A?@XeyCU-13bECzw(j zcS3l7)+Zns!ojbImfwa1pgnJB7H|tIFbRrZq_y197m2AWVpAQLiSEi?d@0>VuH?Xk zopLKYDJ2{?O-`FJxDk$uQ^xY zi{fCMEa~E8N$_93622fl*+-9jEud5MKZOG($q7Joaz*}1I9=?}M?UGeHu0C8^04_JLqu=oEX_s1^gvu%y%DJ6KzBpdX3BPucG-F8sr&LB?k zYBLZB;J+4H!B-9>zJdLIMgmmAVjiGA<`%6foU5!+yD)FN=sU^jf#dzd>YUh(U_}aL zL-*E1Jq2D6X`k?9?s4U*p!EA!@ETn~?_Y##J^}bHI-kji3Ib?5CeZ3Y=W z(qrMVrp~-Eez1?+Wh`+%pT#(SAr|#WXWFOk@@xl#&oQNHjvg2!{a9qaq>mi=nm`bO zh^FI1w~jH7j6;*tE2zg)0)!4=XXWSH-~;!0<+fBlN9x88*gOOW4wJsJ{gp$}uZN(D z2;`V-&lMr)Ia7Np{e0N_9qr2PE3(RW>f8etPg8RXIO+rPk=RD=rQBH3Kke)d?m)$< z0_5yaw)i(tx`6)eNXt{cqtQnFr}{W2UemGaZ~$=f1T!?XJ;z^3bpG%{2DZm3ns#%< zYw(7|81)4iG?H^BGgB++w7KnRbUC0T(0OhG&O*Gam1S^F3^=TY3tFfq$nl2!wYXG! zoJGnIt{Eh~RT+vwAAl`14L{Ed_xYj^0KWKgk*TF}5wPjAbi^|>wz&&9g_q;OWM3mg zL*TK=zR2DUMCAXg?9bz&eBU=?SFcJzLqPtRY*nGgArK6WJL; zwve(_7-i2gmMq!EK4ckVEHh@gf7kT+KL7k)&p*#!&8sqF?)$#Z>o|||IF1WBgaXgz z_O4}Hd@|;TGy`k{Ij3=P>L>>1gT;uvd9B~HE`cP{V6stbw4lxt9RIZY?QgN9UH+|c zKj38gh3T1v?Ob7;_yu);-k7c@@Bh|7e$yWhhc(x?fL2sRK~hZ}9)!1yX{2%8d`EPu zrX{APkd7ws3T65=s=Ji~eyEVt?-brmv8bkHb$3xRJAkqQjw$^D-i`Dpq?@Z`7cGv} z3Ot9$N-t3(c+WOWFv}Pd(nMBt!uu$5(TadugR9vvTOGucD;PbZBz|WNFU=o9ZHx*C zzz8!H0yqempFFt<6(tmTu+jRmkY1fc+;xnVMrNU&pm5Z1RaAB8Kqzx!S$K}8=jTHc54WO!aDKBQt`L^&_ zAd_(D!y)K=G#w(*EE`^H)`LZ#r%U2%WqgZvpkvDfKq*6zC{jWg4Fxb3m<}RlcR)qG z*fIxr1*1iLq}d3~mccQ%z+|wHeQ#x?g2~fXaHAbmQnm-DHg=x5E;}pssbj>C>}y-AS;;|zYw@E zeK6$<$c;e-z#J^2#!9dc9w#`_FRYM`@|maYVbho$rnx{818vMKJWiL9>~pNfkYZ}H zce(=HrLO`${7aEZLjwwuZ%(m<@g>1R0kR5}R$G*1ZW25?6uhX=MCR8_RJOVSeDzwP z1~G%LwrI~s6d4QTomLJQ&^m6Tod6RJhT=?Tpsf&w(}^*Y`=${kEfVv)6-duoF$rrk zV)#s^oE~i7P9}4Hy?676n3o?1**AY-a?sVYyw@a5_hvK}pMhczsEN_wURH2131$Fo z$o*3L&9QE-5a5-bO!vGAuk4g#bpd6?cFDB=!6Dz#$MH$Fsb8pgD z&dm380ud}Lea8XJix>}dpsX_T7x6hQ5`L3hpG6rjZNod}i)X?kjN%Q*G{q4nAH&mH zEAy8a`BfT!^B)q2TPBno(koz_=G6F>O;YAl z9wIMk1m=VLil`t5#QY5oiVHQFriFnD^de*}QyENrFgfJgF=FpRqN5OQeP~QM5NMa1 zZ#N@-dXc>ZW<94iC~7b<60%i`@1oiT>W=Va2gMousyHV5?opKlrVVdg5N$bdB&RfV zs0NnXB59A&h=PL-29p8AgnY(p8uf7eHGDf@=f2AU+X$+z1Ookwa!DIP4$>CF#oGh& zTCzJpcEC;%l?*mnSm@RPaGy{gXEXR57=~1#h^RLNP~4;q&O z;jrm;kUYs?zFq`Kn4A{t<0&FuLb=}{HGF6{JdKoXHPje?4La2;47iB)G?XY_wfZi@ zxEmfz%H}Ciz@(ENETVe@3@Scjc~c?DUSVA^MprkkGXO-AKyr&{&ngw*Gs` zFNlMLomcq@3`Zf0=+=N&hg+f76en-+d2l$sup5H+3ZH^*k#_$T8Y|6Zlm&!(vCaU7 zxVBx@e*!bfDnI%~GJ;7`G(vmtW}9O^qjDO0D!4%-52*P?d0>;dyuwxHfw{e>%DlM@ zg5UnlfkvDRbfz3ObJr9D;yA_tHxCy>vfEh=5ui7WUrdG&MFoc8--L=N0{{Rk%~OGU zd#TwOD~1)|bn>Ryqm&R|(WDG;tGr?Jyg=MX1D(asQO7pkuTjIxfaf$^#VLv3$leCT zmfz{5z1r(%d**3hSqm6tCJ_M6_XFhe^3fo?>@~LqfE3JbB7?cUpR`ukTNQ3K-zc+9 z*=8+(EWyC{HtKsph^X4d-daRF;CA?v8KzXW*?J=Bz#`&!NgZY^s;0)*3z&~4(C5aR z%*p_T^B$uEfYIOJ_uOvo);*LiKq*8kj*hv3I#kMynaII`dERE%yWrXaMmnN>=POW# z`!xQEs(vrmM(F`R_eCyh>o|D0{2Q#x>tbr|eI7WT{IX2+!z9whn9$*)Z+w|B-jW+_n?w$$wTEf* zdNCfbHtbomxmz-^i8c`swhNNgzYVhBC-Aiplb-Xn*wgAF)Wl%DPhm!+;j+;2Y?LJM zKb3iujhqFOx*o9CV0f09B^IL(%+%6Q5egRmOcf70qj+p1bKNz};${9)%x{VQd#pkx z!QUZ1R~b+%1V)jP4%we<-l2{=rvgWg14@svaw%V)sWJQn%D}iO6mwQ^mc{`pPt5sA zCi;&!g>&w!#rE(7G%!M4L(b!|c1a|?k9lPgX{ZNZt%Wuz`_~}L6L!^VFG-+|(87L9 zmXjSdw~?Q>IhY#tzij2(#7uC@JkVM|Kh?4F9S@5;tC~&{2A5Dp6hJ0a)$Gc<`i*$= zoj*-{raR5xCl!JDYtz7jRLzp8{>13sudOxXpBzRfBWLQvl;l zeFoI;L)2KaRbET<56UEX=PAJVEdc(3DROrbqkaUxw9aSA9kDW~Y}Ha6Xd@6m_7Fx6 zh?bx-$$1)}gK&5;AXypTJs4GCUZ2ESzRmsbmj>n=nB#BakLWixKPjaElZSvDl_cmS z_-X+&h}>20OL<$8e|Fz#d)(V8ONKla5uJO7fvZw(z*P~`44%`GvkgEB!0a>i0nk7= zC5$lE*iE-97I7vEm{N*7X>UVx5y4#&0|EJ%ey_))-$Y3HP2&@A=?Nx)-c$lduJ8^7 zDWD_((`g9R3oOrpXB24D5j{HrK0{;_jH#N41mI$q`2`vv7y9$r6>d$0>9 z0rzPWXq8}Q!3|*N*8+JPQW9o66uyC!q>z#YcrVTMKIB@mO1hJe^<;Vsu)z4;U1#_Q zKluM#i=&xf{^x-P;XRdLmWww2_eQ)WV~8G9u}iQS39(27l6@f43mg7NcLBnF1&CW= z%&|@_FnSfWL$wKwU6d@aqoYKalYxFZ8Sg|A zSIHtQlbT%^BVfeW5J`~_6J3|AVazb7C<7vtp-AY-jUwTOgMj8Af1(7u0{VTk-XPa^ zG+Y`=0DnTll!i%#V4%pG(&*iQCz=u}sl$z4&!YHw3gs%nJW8DyS(&@NzxJwo>=}R% z0+hqg0%5UCg3 zEZ+dB&^EYh);T~R!;PwTyFJzh%xNc4j|APCt=zsdC*0#7K-|Rxlwo{L^qU$>{{&q5 zyfTxw9=(SWM@7!!Pj9vF>R%Xxp<>oZm4>a9TjTEeK>3kilx0w59d5#O+ynhlq|vI@ zHW=<4@AYtaVmudA0gTq$CPR?3ab zpATajS@dMMfaZ1+5fTMY^vWv1Z38DD5QAL>aQwh7)Sd?UgTBLZ?5;Ld5mlp(1Cc%M zOie4DNuQ2jV9T0d0VWY&VBO@;#xgNwY?Oz?{Pv4pr!m$ytUxuZUcnf7q0Gc&Y}}Z> z97*_-VssQHk~9w12OOy>zGJ+*@|v`V=gR$zvP&XYuK6xHAb}<{ewqRu9yYj4;w2TY zC4foRDQJXs-AG$75YFXiv#BU`y`o4%G{+G zI?!nFZqy6Xq1wxgEFdW-9WstWagq?T^^&H`C^#q?^^)d81Pvf4CpDfd8AViwBvC5B z5}HDUyrfhB%m!YhQ!2ctMu1^QaS;QMS0HNp_96ygG4O(W5d#1hc;T~%=>adGE%8N6 zbHFRXC`f{YTTr>#0N>kB!p+qaOsfg)-6%Avv0_Y%L2%NB<7sAX*B7rKe=;z1u&?o> z;O4<>d6`fjV~KYf2L{~7_81F*XmMP%4TzFN7L(rrRRd8BjEVsDh}m<=YYj#e6zwc*HC^xjwlp2adGJ8cHJC=?D|G1Ao;2U7waplVP9-a})Rh0bu}ZA5jjFbVERhnKHnd})(Y z-9YnK0*wJcU(|~pcM@x~Va5sUm@>P$Q4A349pzgHW?g%X&LjJScmV$>;PTwG z1v6n%ODXXjE(oS0A80#Mq?-3;*v6#_$ABwsIHwR6IEH67PXST54*%?d_^*A9sU&%z zI>NpO*aI^<(5M%MC||AplT!fu{nuaOAN?8+ZjCg6Id6huL!fW0MG(? z=gwjuCT+*>Ic3a(l}W{)Zpb7YBS2ZtF zL1@8Dg&@CNY-jqB0RH!YBA7aQ5Pnd~jrdfB0J#mjLQK_-JC$7a?sH};nFUOO#q2UZnyNTX7Cq4Kc|I?fnQVXH%;+9S@Yl4g#Jmf1{ZSsG3XxV$z3E=yb+BRG(v+71Qx=9Y@&6xD$KrR_g=IEpuBm{H!z1O1hZf+0I=LqCkAvBFs?*L!MYzMnV8O_~L7?@U3OsVaukQkmWXX1q% zV%%AT+5k~TE1}DDLs=w-4dBjXAkJQ@761?>be(P}5!6yqMrk(`O=57DkdY6m@~MzK z6aN(Mc*vk&_(@=(AxNR%_{oY@jX7Q~9csKHCJVS=J22287|_%5!{BBhE+G>45bxX4 zAQ;e#ut8XF9A{=Nz?3Q5JTVemA-g6%*Y%LYt*8pK?6wGbx{W#KNvAbHP~fqjAo z5K$m0WQn$n&SdI6dF^2_yK!mE|8CHVbu=NM#+GTzYQ5;L%S5rP*oXCgvdldg?B$_G zZLn5zK2AYEmqn^$kt6RSzM1~(O_{2=5s>1w7hfl}(@)c_VLC=#kiJclM(wRxS1@=|db4|i`h|W3 z9%la`a#6B@SuBuLt96m`5>{}J_ppQL(sTqE5i6RKzNf?E>Frk+ox((Ee(eXfd6`>Q z3w+-4c=WSXW+vxRZg{lOF7A5%8dF4pga1GD7?3m}L6qKK#z8MZ&IQ^*kWP4OZM*s}^rwt!$-m=Wb<_s~X~#wkQ0kYHZkg|%OJ$sjkBck5WqkXxek(cIP=wga3F5lj?Wn|T>C&4X8vi|1rHAp`U1i- z>gR~vr1X{Ca$I7nQ-*(0ua;7F)Yh|J4?BCyfED--P7noxu^=^TVphxmrmB(7JAebJ zi|VM5@5W4#5@Urw3gOa+z^vc=neDb`D+EPra1#;jUlOvLDI`9u;TjdC?E$E1D)5gT zj8S6vGp&yBCx8MVy;-jTmlZFD1e6(F@>55J;pVgw7%z(PUqt3suifrW)LY>nqJ*^T zFyXZxuzfYo6-0SBn(V@qZ9e?ZGVW6!_pi4l9Ra=V0^>gl*8F@!6sZ;rFGx6M_C+b! z73afHS)XwSjzZxjM57)|LOV`+F(piNq@wm*Lly}S^fJ101k@WfW;f|!4u*+D1AC0$ z*@kG+2$*zI_47HI9ol!`=t>3YAo2r2ig2gCS}2IgnzD(!B!oNWZ98TyTX6YVF|4w4$a7aabV0nL9EB2O`vr*@9qC zLBt$Q0^phu8=@Ha=12MsvjSXjiP#W;c?HsWSGmBmXoQzt zRQ#}Gm<99eeTWn%xt7igq~T>;Ifp1-PFy5f?_{%CYwb0x&yHbhVU}ArNm&p)_G5oK zRnS-bi&p$tNx>hcbE71cd8J_v*&A1ZgeKe0jFVRksrbKtF;M=VYH$r*RDyF^OzalX z;Ks9LEyefxFsCKk?cU7S*!%MV!Eb;VO*#q+&rxwrY`q=h60)qq%ZzrF zD%ma|set=eLaz2s{J-1xHS6C4?%fDT&so$pPNmwjZVR;6fMDPS`WuQRH}m<%m9Ktz zAyu;KCp9p{B!yD0p&jUdM5*gH01KxVGuEig&CEmiuJb=g=3~VioQ5hV+k z&>HJv7O9;{E>NThroj!M&Asb@t_H35W4x(0{a54ArnwAInSB+d+C=|u|M#XNSN?H9 zkN%JU@P^^RZZ^If2$rG)qo3^a=13;at*_Pte_WtEsMT_(?`$D9&FgLhni3p2aJG3eP;IdH^-zY2)>Ur^0TunOLf zDU(cyLUdjx@>fHL2J#f)c3^+dItZ?2ycsi-M#u_por$)aC~CuVrNSeqXhSSPm*9by!ct}ws$Z%!+G)<4<@Q+VYc?PFm6kM_y>x4z0th^9U8#CAdeGuKXs zzH60__XgzAXha)<^yZ&56>AP6RG6VXk~h9(Fjv@4z`6e^-`#=z+3G!r4Fd*FFzn4n zMsXrywxq!~1$04*RJqX}!UgbT+YTkjN!EtVE50FLL9X4?tAthzFfYh6Y4DwZ>RB#U zI(b))NpQE1xN-w@SWDQ}_L!#~%UebF!LmGVFr^=3kY+2MZ-H^N4B>Gz@F*VfxUUpe zR_4J09Eu)BwCiBc8W-O-#)pFRwtpm-ACrFt0gGyq_h?jck_HJjPpSQC^&w{!s|;_e zUp093Z(>pKZ(;!gFwj>ZMZ0e)@Mco1Pb-539Ds9KpxJYW+caJ_hGTMx!lp zlN-kU4x8702x5m63qKYTep;sxMuV4ypcmPB@V=_;&n1};lq*L2YaK>19_t z5f-?)jjUyEqm=_1o|Z&(Z>{mh4Iz)cQ{dj_@XvQA^&S;U<>vf_Bzrw;sf-!^Cg&Hk z+mX-eRsEt|(K_5?;!Ds<@>+%@`x}hAw9LX~?yp-klE^UE@Bx<3;@8Ws?${eF&+#w1 z9J4Gnu61m6E_u+Tn=znzS{7bw^UY}`$@$l-xyZx*8J%+E7g9ezR)4(ew0AvY?UHp$ z@fR1Bhrx@^5B3`nkE=e2ooC>`+hU!UG2jKE^ZUbEa1EdlhxR6@oyfKZMyUu*FNTInz1-J?&ndh0@8_k>vsAPUo!18{q}sl zMZV(;Z~6PU>;=CInL>BJ@KUm-7YDg9p1Um&>u-?_rXqD8zEG1h4!l>mW<0MybmyC` z15P6=zc@g*H10^6{=I7bKbF~CUy^>e5&kPblpQDFq#ok8skK4W|5*I_D!Rirvxy`4 znn4R&OZ#AOu7A{&XVw7^mUw&s5%B{LAsunNO;~x_u$7Zt04*30E&MPME3Z-qRrCYZmL~=gN zh4e4mXCKV&mh{kl+8al*F1=I!u_ktfxY`&`Elp1(3puX!?kw|R3n_1Zhqj1i77f0- zVz^@5@a{-(&(m&%4Ex6I#28P;pu`HgyfmP27Kykq?mDvV0)xGlRJcT&yHZG~wbCCA z_kS-LE%HS~?F9xdEoT-vSocYH7xRYRRI=MiEl4)rR6fD5QtZV=Dj10?3@>damF=|4 z=^NOt1|RwP*^Wb9r~^S(pde~ghFQpLNhpP(EY$)IbAyp-RrPZ-EY9l(*Z&v{85}(` zu5!3T%>k~o0rUB!@YTV^g;8U@*7xWxGWZ(O8n7dd%06P07Re*<4OBZpeWP?-q?;#^ zN~|@Wm^}=|Ox&Tyt{gs7skb!Fo){qLeGi$uHeX}^wriRN7nb>vD6e0Aya`;|=I>TKm)RliM1R&**Me#rNj2z4&h%fOHtQf92?rr%6()%!#LftvU_+DSEi zGx%q8a8J2>?)PF{+_|{iE1X4$0Og$wL(b3DboG*XQ?5;-vo9AjYtk z{z$xMR@=1808Wueo%VmfXiPK~7C})I6p@T)>_5jOe+cS$s2Z;LZGvg3LgU@1QL#!d z_`OcM8Y#;MK1612Y8e{0-7dz&)A73cGp1jQZR-wa8D!l@dx;8HbCHA(i#XI}PEqe9 zKI@Y6qc?3~3*{~;c@bxRzb@BrD;qRM-H>XH@wLEd>xmnmG5_r?S__FRq{NmcPh{1^ zDZYa%_*q!vuLy`%Q#*|3=91zsd}uP-E*{m&yel90IWF4Z*GB4=6y}et-V;0iIwQps zDGiropIv&^EPHvpNM${1HFSzQ`?8aK^~Bhz5Qa9N$Lxwi-j^ zw!87~R@ECgwAwyhg;pD)eydQ{>GDQjJpR_&J_8zOSQ5n1lNyI9yfal4v~u|qdzQ~Z ze07Bt*Oc_CQl7g%X?<-5oS1x%^$^4RC}Uz~i!(f0BTR#B;)jt_73#649eI_Wm&c}c z-FZ_!}TV|$!7p)of@{@9w>^FN;6&zirge*Al~Xv{nMiw9a+cTDqA zp8Zme@@qI&Eb9wutEuU0gPqveE4X|v_`JMKZ|;RN&%(1`a*$+?7^*_o$%TzC8gC@4Ood5t)B%mxhymNba(&x9LpuiZQlI( z6aAkCA9vs*+}R1I&nTRc$kpOt|1NyoN3)rlm0>ivq2_nUB7!qThRRmFulAbK|Bf=g zy2H6{SC*n7=IkAARHgj9eZrPtvz^(#6aR{ngL85M@htig+dDxe*)u|?IpVi)r-i`L z$((f6nfFJ(i}j?a%RKSvLh`n91Uc}AhldmYV}JjscK+Xq#vL;WM!Wk;9`Grc+|tS1 z=vm)*jQYWUNV)d?^g7)gYNY2~HrN(AA~1=mG3Rdmpf1PxR^gP9*l#T?pD;CogZ)(V z|J$*~8$G>E=Te@`E!(S<{9;lm>i;P_^;%1b}@VDyf?nY zl-29_YKF2sq*gYlKDC2>xD6Bp?dH*PLuD(^N@;1ur`Kz zl4n!K>OK18hdmZL+n!d#(_cfHnB*@pj~cKGGY^FH?SCO-j;zZa(;xmGs`Q`RD{u8Y zs%mul^=RC(xIx@!*3*9u9&HAGM zzwSxh2{Zo_r+C7rp*^-A2{ZTi2lw7U<$_a=1$;e}Op?XT|398{JX0>V^JJVWvJISe zEP8rvRz#`uO<1liCE>Ab`cAQsVUab0=s4RKm(V|>5!SC(SEW|It?g%ig0bP5LS7-A zlUYZxg|kw%BEqHm_E^J@O!+UbNQPsHC8FuQX|FG#mbL=)6tpZpCBM@}e097l>qwQ7 z`_R|ZvwWpiY^d6IZS?jNmR;iOkx%aHi}$5@Lytv1)l^D*f2{vgQj2EROn>A1W|j*= zu@1zu3=xfoQk2La?~xaTDuUn*e!f7f?j^}3nWjH+6lm@psZ6g$;%$n+rP zh@+vbk+@O_elZ8XqUxb~_{AS^AYT7Qf(A>TU5WpxO2>wE)2<;z-JglKr-dydj_YjZ z8+=f;I+o_AIUy#oi&%_Xe)&N7J}2irKhxb8Dn^sMuBBv`%*vn?*haiKc8upo@`>(O z`)jcR-9MTgEzT8{>sd>?e=q5|5MSlXt{|Kj!WJHNNM+J?+B?=dzz+dEJd41Er5*3M z{>;?GRwbiHRPZ$4QeK_tr+%4J${!|cq>h^mZ!zvwrz?GIQ!iPeY6Y@oo%PjiWr>Dq z=qi)ybShGNxFr2`!{L_hUiE#~it684D@PnxVQ*?S+_!v;Ksu0j;!pe8*Zj3zg_9?4 zbWh;pghCF*$nD4tEpDDC&JjNHmr~ixVQ%;Izq@A|@r;(ZmaM20>QEcYa}KoG^HE1L z@&9Sk``gV5@Gw75ie1OOzAA1m(f*oey1|AzlsSR=$}ZQj%KKyYTD{tLkNG`pQ7x$c zSeSE7=?_J|G77!EUL89Zl@Pe@bxPUJ{@Kz)L;6Uv@~i8!^)A8|520R zg#0(Sr+#di+eT4hnpD}C+<(~Xf?azvbsn^a6vT;Bovd+rj!{RcpCXoFqCz^al>0Y) z4kUP*#smiM-;lbU6&ewm?ktliw90kVzl7e=LU-HPvvK0!>p9X#sC~L|nf;K2!)(`k z#BX~m8%s*Zz@Qs}ZWX6yNvV&M&G-jjJ+P0p31kUA z_UpiR*LTM3n}WQSCe3oO-)r2661Judh9(|DP6G|k!5PkAOz58ZM07-THTFt^14B0?{$bFS z3HX5dOxPYrY+dtYvU#RBnKX9Z?2JN#5&x0Ng^^~PENt8>FjtQe79MPn7ZjLg?JjEjHQTQHVZA(Za_{trWk$7H&ZVrA4+1#2qN`V+XHcp| z&(BNtBkGpT3;mrEml{b~dKqmQC!F{e+8Z%k-3Q}b2uc%=Q8RtdWppckb8>8R#+B$O z7`wgFidXwdPw9Ai*w$7!q`M10S9kszrObL;Mc}(g**cj^Db@0uu3x;atx)V2GyM-o z)JLuc?@ldPk0^!zs&1;Ghc*5xhvAwMvmYI8DE&gu3!^*oZ1XPntdeDlOnc5&-ybVa zy&|r&fxi)RX@BQ4zir;{OLxt#W^3=|A?2*cK4zc)|3Bi{Et6pjTVpTcqlV?)nNWP| zn#%p{{Cn3HkDpOqa#yg|j?3>HDw?Srz07!6X%zD$NueZGGj2#b@{0R~hR-G@A)6AY zy{6PJ37YN$sWhTl#3khPV{T`KrH7nn*f(utc4E)`VoUI`PH4*Vv^GaQI_eTse%q&a9_tXl=^OM!D=Yn5niA!w9q#1gM7>45*8XR4ZX4uu zky>xnA@P#z{Pw9#uShPp`-4sVTON~w8pVn3w3etdPIhVuC*U?u}(_LUo3mqxm6bZf&5ExW@Gm4k>=Jv zPt~_G-^dv!)-Hx#viM&0Gc)p`cU-1Pkp6a)dwChdE_k#~ebUG#cRg%lq)q#w`W$(=F#W?xeW}nvdK%s(fO+ z{*G0>f)};6b7IHe|5JDB8&tQj9e(e>nuB)Tt~Z}Kbl+h=7+nx2efdLbXWYF(<451Q zCNqgOk41_Ob0Kpc<$uDdPRCr)BXjiq?;PlRbndRoUA=h|E1?Y!_Si4O-F&LSp53Og zY1RwrdZPlpUaYKpZ97e)y9_PW#^S07=$AxYbo@mLg)u$3AeomHyW{9uz1ivLDQ-3? zI}blXDWlP3dIIJlyW%1UPKV?Q*)2cj!S9*%BVi*qbVf`9BRhdOf}1Vp_igU&h}~*X zBxqfB_l@8jG&s0=^y=8tyxGS%Vpisgg5rpAJbu9Tle&m)dg4i0!Di1?ZEKO~nvkIQ zkezw`JgF23n96udrPc%&{b^t76}k1Xog3e{R92qLSd8Lar}z!>C!|8mM&Y;M4=>bT zP1~>2o|>R=lcS%1N$^{xMd+HVrk#Ly3~${fyME$cCNg5-CP9F2$|uE^k`ljlaG%ep z4x6ocOkccu00!e7Eu(N3zVf94a+F^MNj~rA>3>^T_oU7dy3T9~=w4Mx(O;^s7$%$l z3~aXRQuuXv`@!E=V+f<4^54UZl4j}TcrFRlb`0`tgxIM((Uyhiow-a$EgD)VmnR8( zd=4BOjk=#-IP0X@ve=Mp_w%o0^)sr4c*T9?7>A`4U0)}O)v%!>`kYX}%}%2*3C_%+{3WZ?^Vf-*GFGN#R~?!n$=(4dMacTKVI93OqPI z9k;8{rEbTzwb=7t}U0sj+1qpPns4&C=|oqOP(Du)M!ZLABLvW0LvdX5ch>!Pd@ zhZ92{^n1#LV6OH(idR1SSEBwV5;bz>%Rw6gjKs|&V-%-$Eq1>1OoH8Rq+O$W*y4_w zJ*mKvofvU2JSm+Bee!1eR{!*t!pt*9Y*=aux{&#tElY*)eg)10AWHPw3~&foM_#0U1eF4w@S6>GXW z?c_CBegu6CKiZIRysmNe+3urPZ2jA#SIOh?M9w23p5lgOL0V8csefHVU#TXwBc#=i zOcxrz0`12S=UB)liv`#cPEd!k$!3pUVx(kTTN;Y7{Cr0kKYq@ABU>pRxJ0oGEt!vm zs_uGby_D>{{^MeM7fwS>&WGPmJ&m^Hk-92*UoZG_>|o>^XKtl~o64QC4BA>=BxXx? z&Y|AF(NR*e8SU$bEu3AqOek2LkpVI%yVZi8NooM!!ox-?!qu0VDII-Gj{C)~NQo${`;%nd* zS?x=~td`1w=+q8_{LEa``LH8u)tCfn)Ix?h)+$%U?%E8VskUbO~uG|yjN3{8XW7NJ3NU#fvE@3pCcJCJS{ z&$oyrx{}U@vFo5ib zUhl?cM!81WK?gtPF$84xN3Y&d&dcq1@9&LvezAMWzndHDjtj=WDrFQWbpMy^fc3J>IO9@3p8#NE1|p1qm?K3}x(~A+dnU|8?* zj4cAg>{9(ZMV3TfUz*;xwUqYTAme;h!C+`kILyH+HZO#OJ{Pip(9MS_WqIyTjInK3 z%AQk>RFsugdUm${oxa{rW%9M#PlDg-+a{;?y?wF>C0S5#YV#2X&j0*7n8qb=?w*O6 z&GrxtzK1;;?g2M9Nrt4*mm|!#trD@ znUNC05E&XHV zMxT;mXt>Vyv1=k5TcmvvahsEcH`rcWu{x(bEHie5BmGg$$jNu60#vyrdlmU?%+`27 z5+Yw!O;Rq`Z10>w!$@NLA~x>iRQF$7^wTpZdh-c(vE!=!Ew8>iWqu(1{TA~f3Ps6X z;uwHC@l2`8N)H%7C$~PW#Jpj7+Xb8I{+Zn{{cQ@9Y2^!}Rg4QY!$YsHA2Eu37LHU? zJ`=R|U|c4fAM3x? zku}+KB0%#(v50Q8|jx-|n`MPtx-7tt?|GuWY z^(6v3lA-QLx7&@8#gA^!%*lIY`FT<{QTpwd-zs`DtrE1)61l*m=ixV zjS}jQMVesqj)6l#Jd7;lD*u$s1+Ezps^p~C&l=#K$PeYwQ#jJI;si3w2^+=!&o2(h zvzyJB{a2?l{^ZoBzt^y&jQ^x}rN#?RI7p6(l?r{!I_ns_TwOqYknU-PqJL>vF5^T; zw5`||Cv`>DT+myPx-Dudb%H(j*q}Bv)bM47^$~kH^zQKTQJlUKZ?uUAmoniA=k-zQ z#2?Tk9e(;v=ftA&1ycUV%kt&?OMFuA+$%1^7^d%<-Uuv(E!zH9?se%De2I#<)?cz? z+|wFg$@=(X-oA6xvHf%D3eK&56R!>q59YbI*XLgT6a3Lv!u_WBhWN|iQ(rX1+CLt$ z5gnF0p<5`)|CHc;WJW6d!rDV8g-^!&N`|=aVk-^1HyyUiOQ$%GR~s&40+DxSWGtG8 zTc2j?6+hBQWY^V7KK|S_H>&fPCcIkYn~dF+8pmrI4OY0OwHTeEs(=%*9-mGee_Hx3 zzu8`c zrIR{tplKbCI+obIuXIz8l7=XGTHXC2VyjZ*^Gh*v=ix6w5^V+cK=PJ_wIN!eD(6MBdM{7K9`a=E#Bf~ zk+ZABM{f)WTrlZCu^Zkh3D%j-kZ#-Oe8`ZXR9q|DJK(v0TX~1)bOlgBYMj2QUW8Tn zjaIxRbQZR`sUDlmQ&6s|`geCi)#emJUuY1sDY+>h+jUhE*X7(Ka_i^*^gAama$OSd zdbcl?Q$LygorvxQt87xLr(;J-SGu|o?u=MZRCWH!^3zU>w8i0e+VTOFkX@R&+Vw+X z348nUx}@7~NbP;RmKDs|fF}y>2-s?CTt2kT#=$+y&EXSPa86ee@D2GcrqS* zF1+~9YvGsCmSoE}xRKLa$>z<^c9bfrPnQXDKk!OF(tkl_E+djXh5z(R;mke4#hr$P-aXazJ>`A@6asDu#eKHjtd@T7SJMA0vNMpX1c*QVPN9RMlHy(A| zF+I}Nb0_cAaNoJ5#A2mw;m2n4mE3M6e_3rWIt4^P)6O^6O9LOjZR}IOU*ajmb8?Tg zG^=s=7^7ZWW4PC#W(xD2{?Kn@ICe5Wy=(I56%^)FpEG6N`%wJs$G7`Qa-ycMFecFv(MNK7dr7RO^QW=r zSN8H6xo-1oMa~|osxKe+-srS~8Ofba)<&?zDe%K z50vrG+dTvi0?4fUy7A*CkDln8+hGsvV>Gt-EeazVl0%D++?hF(Bfl8iwFP!000002JMXxt*d*ZSP5RWr4 zV2SbIM=Uk1E7TdOMY>xe<r9Rr3L6k*Z{sO7ab6k*!x*q%x`5WIwCfCN+!9%LD0F z|Bp~#L``bSm6|xMGMxK6_uTWiYV|$1m08(J_H6YCf)Egb9RVQ_w1=U9{i{O=2@zWk zTCB3m48^J$Y)LQtpj_P(gy30w#+emfWHKP34Q^7 z0sad775D&r06qjCy8JgmfCP7dJHP>O02~Ad!69%6+zIXkhrwZR7q|=D4eka~coIAXo&ryUr@>di zSHMwl6g&f-0bd1Q1$JO6Zkcl|znh3bU7M1H6)st10fn2&;|;g|M_>@Pr0!OtA#=b)bfPs7hua2)<5 z*h%lZdTG?<1nFTfeeC*|A(M!g!hGsuFH4wDAnqLE%|X8aIS>08#G8eTVV)%AL3zc& zs9(nM53_I|aW<@B-CE`EWAM)j|3g*2%73@Ee8y zgMr^D_=Eq}fEoolnCQS6bZsk26f#39h znrk=)uzmpR>A(#`PmKc|Y$PfFN!U{ZMDxScn9zX?Ka-RvH4XMl$wnL^yHuFpaw0*H4L-V zP*7g5OSmlfjectw&do!X;I~8!|2uiFT>*XylrLC8oIE(6Z_iH=KaaX{ z;HOys8{;62fmyNLfb*3sEMJ`ejm*5@FfLLS>6@DtSagj<&ryz>uX zPy4~|A4#%%h;?s4ZbF}h%wYX}$PD~uV1FNcAH2cI^?fg|f$u@Ck)7aWim$=Ho_8hf zGhI)Qx%?kMzDa)E_51@1U8yj7U=e{bqUn9kKo~tKfRQKXklk?~=;B1fmSN6M{VX$VD1@Cj3UzaRv z_huroXIE#K%&2QBGvhUL$Bge(>j`~NGZTe+;WVcIj%F>za#fNXYh)iaEyb`_Hg6>*Ik_Oo_P;383r)o}TfrsX?S#Lp z7pwRRZ!cHH(b?`^(Ob`@Gx z&*_!+w~k}CmVQgCut!y$trv?Xv!ER!?b7zP!RlJWDa9x-HCI-wTwU>K?5Ism&@{|8 z*N*31<@UYp{)7&>+sNSwThmieEG1raKKDo4)12+CD~6_Qms>x#VR#5oR*gGF3tX2t ziwm1aU*nI7jT=r(j84hlx&GQROop?T!IVm|T-}R_Ma!AI@mj*<98I-zRA4&>W2U%l zf1I=$?Vh-znQBQfc5FknwymjRkugiOSBoZ7t9pTR-M7LFtx(9bJL_N4f$tRk{%CPA<-M;qjKM>6%5wpVF)L zNv9y|w%5bKH@Y5r3ihs+75f^uKTLtvb@dISTGvebYFEml<3m&oCYrVFpD@)Dt*YpJ z8l364c&F-c>zUPv={>JXth}4Jz1!l{KCi?|)pn}s?$PBqJL$gLn&+`9`Whd0)6&$I z%R0clyF+oGvG=_{z0bL8*1EBty}PB-O)T6%>&Wxh#}^L%4M%P`rOq<|0D-1P ALI3~& diff --git a/vignettes/dataRetrieval.Rmd b/vignettes/dataRetrieval.Rmd index 1a03416c..ebaf2df5 100644 --- a/vignettes/dataRetrieval.Rmd +++ b/vignettes/dataRetrieval.Rmd @@ -39,15 +39,14 @@ A quick workflow for USGS `dataRetrieval` functions: ```{r workflow, echo=TRUE,eval=FALSE} library(dataRetrieval) # Choptank River near Greensboro, MD -siteNumber <- "01491000" -ChoptankInfo <- readNWISsite(siteNumber) +siteNumber <- "USGS-01491000" +ChoptankInfo <- read_waterdata_monitoring_location(siteNumber) parameterCd <- "00060" # Raw daily data: -rawDailyData <- readNWISdv( - siteNumber, parameterCd, - "1980-01-01", "2010-01-01" -) +rawDailyData <- read_waterdata_daily(monitoring_location_id = siteNumber, + parameter_code = parameterCd, + time = c("1980-01-01", "2010-01-01")) pCode <- readNWISpCode(parameterCd) @@ -59,8 +58,8 @@ Table 1 describes the functions available in the `dataRetrieval` package. ```{r echo=FALSE} Functions <- c( - "readNWISdata", - "readNWISdv", + "read_waterdata", + "read_waterdata_daily", "readNWISuv", "readNWISrating", "readNWISmeas", @@ -69,9 +68,11 @@ Functions <- c( "readNWISuse", "readNWISstat", "readNWISpCode", - "readNWISsite", + "read_waterdata_monitoring_location", + "read_waterdata_samples", + "summarize_waterdata_samples", "whatNWISsites", - "whatNWISdata", + "read_waterdata_ts_meta", "readWQPdata", "readWQPqw", "whatWQPsites", @@ -80,27 +81,9 @@ Functions <- c( "whatWQPmetrics", "whatWQPsamples" ) -Arguments <- c( - "service, tz='UTC', ...", # readNWISdata - "statCd='00003'", # readNWISdv - "tz='UTC'", # readNWISuv - "type='base", # readNWISrating - "tz='UTC'", # readNWISmeas - "", # readNWISpeak - "tz='UTC'", # readNWISgwl - "stateCd, countyCd, years='ALL', categories='ALL'", # readNWISuse - "statReportType='daily', statType='mean'", # readNWISstat - "", # readNWISpCode - "", # readNWISsite - "...", # whatNWISsites - "service, ...", # whatNWISdata - "...", - "", # readWQPdata - "...", - "...", "...", "...", "..." -) # whatWQPsites + Description <- c( - "Data using user-specified queries", # readNWISdata + "Time series data using user-specified queries", # readNWISdata "Daily values", # readNWISdv "Instantaneous values", # readNWISuv "Rating table for active streamgage", # readNWISrating @@ -110,7 +93,9 @@ Description <- c( "Water use", # readNWISuse "Statistical service", # readNWISstat "Parameter code information", # readNWISpCode - "Site information", # readNWISsite + "Site information", # read_waterdata_monitoring_location + "Discrete UGSS water quality data", # read_waterdata_samples + "Discrete USGS water quality summary", "Site search using user-specified queries", "Data availability", "User-specified queries", @@ -121,28 +106,20 @@ Description <- c( "Metric availability", "Sample availability" ) -Source <- c(rep("NWIS", 13), rep("WQP", 7)) -Site <- c( - "opt.", rep("req.", 6), "", - rep("req.", 4), "opt.", "opt.", "req.", rep("opt.", 5) -) -parameterCd <- c( - "opt.", rep("req.", 2), - rep("", 5), "req.", "req.", - rep("", 2), rep("opt.", 2), "req.", rep("", 5) -) -start <- c( - "opt.", rep("req.", 2), "", - rep("req.", 3), "", "req.", rep("", 5), "req.", rep("opt.", 5) -) +Source <- c("USGS Water Data API", + "USGS Water Data API", + rep("NWIS", 8), + "USGS Water Data API", + "USGS Samples Data", + "USGS Samples Data", + "NWIS", + "USGS Water Data API", + rep("WQP", 7)) + data.df <- data.frame( Name = Functions, `Data Returned` = Description, - siteNumbers = Site, - parameterCd = parameterCd, - `startDate \n endDate` = start, - Arguments, Source, stringsAsFactors = FALSE ) @@ -155,7 +132,7 @@ The arguments `startDate` and `endDate` have defaults to request the maximum dat # USGS Web Retrievals -In this section, examples of National Water Information System (NWIS) retrievals show how to get raw data into R. This data includes [site information](#site-information), measured [parameter information](#parameter-information), historical [daily values](#daily-data), [unit values](#unit-data) (which include real-time data but can also include other sensor data stored at regular time intervals), [groundwater level data](#groundwater-level-data), [peak flow data](#peak-flow-data), [rating curve data](#rating-curve-data), [surface-water measurement data](#surface-water-measurement-data), [water use data](#water-use-data), and [statistics data](#statistics-data). The section [Embedded Metadata](#embedded-metadata) shows instructions for getting metadata that is attached to each returned data frame. +In this section we'll show how to get raw data into R. This data includes [site information](#site-information), measured [parameter information](#parameter-information), historical [daily values](#daily-data), [unit values](#unit-data) (which include real-time data but can also include other sensor data stored at regular time intervals), [groundwater level data](#groundwater-level-data), [peak flow data](#peak-flow-data), [rating curve data](#rating-curve-data), [surface-water measurement data](#surface-water-measurement-data), [water use data](#water-use-data), and [statistics data](#statistics-data). The section [Embedded Metadata](#embedded-metadata) shows instructions for getting metadata that is attached to each returned data frame. The USGS organizes hydrologic data in a standard structure. Streamgages are located throughout the United States, and each streamgage has a unique ID (referred in this document and throughout the `dataRetrieval` package as `siteNumber`). Often (but not always), these ID's are 8 digits for surface-water sites and 15 digits for groundwater sites. The first step to finding data is discovering this `siteNumber`. There are many ways to do this, one is the [National Water Information System: Mapper](https://maps.waterdata.usgs.gov/mapper/index.html). @@ -206,92 +183,72 @@ There are occasions where NWIS values are not reported as numbers, instead there ## Site Information -### readNWISsite +### read_waterdata_monitoring_location -Use the `readNWISsite` function to obtain all of the information available for a particular USGS site (or sites) such as full station name, drainage area, latitude, and longitude. `readNWISsite` can also access information about multiple sites with a vector input. +Use the `read_waterdata_monitoring_location` function to obtain all of the information available for a particular USGS site (or sites) such as full station name, drainage area, latitude, and longitude. `read_waterdata_monitoring_location` can also access information about multiple sites with a vector input. ```{r getSite, echo=TRUE, eval=FALSE} -siteNumbers <- c("01491000", "01645000") -siteINFO <- readNWISsite(siteNumbers) +siteNumbers <- c("USGS-01491000", "USGS-01645000") +siteINFO <- read_waterdata_monitoring_location(siteNumbers) ``` Site information is obtained from: -[https://waterservices.usgs.gov/docs/site-service/](https://waterservices.usgs.gov/docs/site-service/) + -Information on the returned data can be found with the `comment` function as described in the [Metadata](#embedded-metadata) section. - -```{r siteNames3, echo=TRUE, eval=FALSE} -comment(siteINFO) -``` +### read_waterdata_ts_meta +To discover what time series data is available for a particular USGS site, including measured parameters, period of record, and number of samples (count), use the `read_waterdata_ts_meta` function. -### whatNWISdata - -To discover what data is available for a particular USGS site, including measured parameters, period of record, and number of samples (count), use the `whatNWISdata` function. It is possible to limit the retrieval information to a subset of services. The possible choices for services are: "dv" (daily values), "uv", or "iv" (unit values), "qw" (water-quality), "sv" (sites visits), "pk" (peak measurements), "gw" (groundwater levels), "ad" (sites included in USGS Annual Water Data Reports External Link), "aw" (sites monitored by the USGS Active Groundwater Level Network External Link), and "id" (historical instantaneous values). - -In the following example, we limit the retrieved data to only daily data. The default for "service" is `all`, which returns all of the available data for that site. Likewise, there are arguments for parameter code (`parameterCd`) and statistic code (`statCd`) to filter the results. The default for both is to return all possible values (`all`). The returned `count_nu` for "uv" data is the count of days with returned data, not the actual count of returned values. +In the following example, we limit the retrieved data to only daily data. ```{r getSiteExtended, echo=TRUE, eval=FALSE} # Continuing from the previous example: # This pulls out just the daily, mean data: -dailyDataAvailable <- whatNWISdata( - siteNumber = siteNumbers, - service = "dv", - statCd = "00003" +dailyDataAvailable <- read_waterdata_ts_meta( + monitoring_location_id = siteNumbers, + computation_period_identifier = "Daily", + statistic_id = "00003" ) ``` -```{r echo=FALSE} +```{r echo=FALSE, eval=FALSE} + +tableData <- dailyDataAvailable[c("monitoring_location_id", + "parameter_description", + "unit_of_measure", + "begin", "end")] + +tableData$begin <- as.Date(tableData$begin) +tableData$end <- as.Date(tableData$end) +tableData <- sf::st_drop_geometry(tableData) + + +knitr::kable(tableData, + caption = "Table 4: Reformatted version of output from the whatNWISdata function for the Choptank River near Greensboro, MD, and from Seneca Creek at Dawsonville, MD from the daily values service [Some columns deleted for space considerations]") -tableData <- data.frame( - siteNumbers = c( - "01491000", - "01491000", - "01645000", - "01491000", - "01491000", - "01491000" - ), - srsname = c( - "Temperature, water", - "Stream flow, mean daily", - "Stream flow, mean daily", - "Specific conductance", - "Suspended sediment concentration (SSC)", - "Suspended sediment discharge" - ), - startDate = c( - "2010-10-01", - "1948-01-01", - "1930-09-26", - "2010-10-01", - "1980-10-01", - "1980-10-01" - ), - endDate = c( - "2012-05-09", - "2017-05-17", - "2017-05-17", - "2012-05-09", - "1991-09-30", - "1991-09-30" - ), - count = c("529", "25340", "31646", "527", "4017", "4017"), - units = c("deg C", "ft3/s", "ft3/s", "uS/cm @25C", "mg/l", "tons/day"), - stringsAsFactors = FALSE -) -# nolint start -kable(tableData, - caption = "Table 4: Reformatted version of output from the whatNWISdata function for the Choptank River near Greensboro, MD, and from Seneca Creek at Dawsonville, MD from the daily values service [Some columns deleted for space considerations]" -) # nolint end ``` -See [Creating Tables](#creating-tables-in-microsoft-software-from-r) for instructions on converting an R data frame to a table in Microsoft® software Excel or Word to display a data availability table similar to Table 4. Excel, Microsoft, PowerPoint, Windows, and Word are registered trademarks of Microsoft Corporation in the United States and other countries. +Table 4: Reformatted version of output from the whatNWISdata function for the Choptank River near Greensboro, MD, and from Seneca Creek at Dawsonville, MD from the daily values service [Some columns deleted for space considerations] + +|monitoring_location_id |parameter_description |unit_of_measure |begin |end | +|:----------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------|:---------------|:----------|:----------| +|USGS-01491000 |Specific conductance, water, unfiltered, microsiemens per centimeter at 25 degrees Celsius |uS/cm |2010-10-01 |2012-05-09 | +|USGS-01491000 |Dissolved oxygen, water, unfiltered, milligrams per liter |mg/l |2023-04-21 |2025-06-15 | +|USGS-01491000 |Discharge, cubic feet per second |ft^3/s |1948-01-01 |2025-06-15 | +|USGS-01645000 |Discharge, cubic feet per second |ft^3/s |1930-09-26 |2025-06-15 | +|USGS-01491000 |Suspended sediment concentration, milligrams per liter |mg/l |1980-10-01 |1991-09-29 | +|USGS-01491000 |Suspended sediment discharge, short tons per day |tons/day |1980-10-01 |1991-09-29 | +|USGS-01491000 |Nitrate plus nitrite, water, in situ, milligrams per liter as nitrogen |mg/l |2023-08-02 |2025-06-14 | +|USGS-01491000 |Temperature, water, degrees Celsius |degC |2023-04-21 |2025-06-15 | +|USGS-01491000 |Turbidity, water, unfiltered, monochrome near infra-red LED light, 780-900 nm, detection angle 90 +-2.5 degrees, formazin nephelometric units (FNU) |_FNU |2023-04-21 |2025-06-14 | +|USGS-01491000 |Temperature, water, degrees Celsius |degC |2010-10-01 |2012-05-09 | +|USGS-01491000 |Specific conductance, water, unfiltered, microsiemens per centimeter at 25 degrees Celsius |uS/cm |2023-04-21 |2025-06-14 | + ## Parameter Information @@ -313,29 +270,31 @@ The dates (start and end) must be in the format "YYYY-MM-DD" (note: the user mus ```{r label=getNWISDaily, echo=TRUE, eval=FALSE} # Choptank River near Greensboro, MD: -siteNumber <- "01491000" +siteNumber <- "USSG-01491000" parameterCd <- "00060" # Discharge startDate <- "2009-10-01" endDate <- "2012-09-30" -discharge <- readNWISdv(siteNumber, parameterCd, startDate, endDate) +discharge <- read_waterdata_daily(monitoring_location_id = siteNumber, + parameter_code = parameterCd, + time = c(startDate, endDate)) ``` -The column "datetime" in the returned data frame is automatically imported as a variable of class "Date" in R. Each requested parameter has a value and remark code column. The names of these columns depend on the requested parameter and stat code combinations. USGS daily value qualification codes are often "A" (approved for publication) or "P" (provisional data subject to revision). +The column "time" in the returned data frame is automatically imported as a variable of class "Date" in R. Another example would be a request for mean and maximum daily temperature and discharge in early 2012: ```{r label=getNWIStemperature, echo=TRUE, eval=FALSE} -siteNumber <- "01491000" +siteNumber <- "USGS-01491000" parameterCd <- c("00010", "00060") # Temperature and discharge statCd <- c("00001", "00003") # Mean and maximum startDate <- "2012-01-01" endDate <- "2012-05-01" -temperatureAndFlow <- readNWISdv(siteNumber, parameterCd, - startDate, endDate, - statCd = statCd -) +temperatureAndFlow <- read_waterdata_daily(monitoring_location_id = siteNumber, + parameter_code = parameterCd, + statistic_id = statCd, + time = c(startDate, endDate)) ``` ```{r label=getNWIStemperature2, echo=FALSE, eval=TRUE} @@ -345,51 +304,34 @@ fullPath <- file.path(filePath, fileName) load(fullPath) ``` -The column names can be shortened and simplified using the `renameNWISColumns` function. This is not necessary, but may streamline subsequent data analysis and presentation. Site information, daily statistic information, and measured parameter information is attached to the data frame as attributes. This is discussed further in the [metadata](#embedded-metadata) section. - - -```{r label=renameColumns, echo=TRUE} -names(temperatureAndFlow) - -temperatureAndFlow <- renameNWISColumns(temperatureAndFlow) -names(temperatureAndFlow) -``` - -```{r label=attr1, echo=TRUE} -# Information about the data frame attributes: -names(attributes(temperatureAndFlow)) - -statInfo <- attr(temperatureAndFlow, "statisticInfo") -variableInfo <- attr(temperatureAndFlow, "variableInfo") -siteInfo <- attr(temperatureAndFlow, "siteInfo") -``` - - An example of plotting the above data: ```{r} -variableInfo <- attr(temperatureAndFlow, "variableInfo") -siteInfo <- attr(temperatureAndFlow, "siteInfo") + +temperature <- temperatureAndFlow[temperatureAndFlow$parameter_code == "00010",] +temperature <- temperature[temperature$statistic_id == "00001",] + +flow <- temperatureAndFlow[temperatureAndFlow$parameter_code == "00060",] par(mar = c(5, 5, 5, 5)) # sets the size of the plot window -plot(temperatureAndFlow$Date, temperatureAndFlow$Wtemp_Max, - ylab = variableInfo$parameter_desc[1], +plot(temperature$time, temperature$value, + ylab = "Maximum Temperture [C]", xlab = "" ) par(new = TRUE) -plot(temperatureAndFlow$Date, - temperatureAndFlow$Flow, +plot(flow$time, + flow$value, col = "red", type = "l", xaxt = "n", yaxt = "n", xlab = "", ylab = "", axes = FALSE ) axis(4, col = "red", col.axis = "red") -mtext(variableInfo$parameter_desc[2], side = 4, line = 3, col = "red") -title(paste(siteInfo$station_nm, "2012")) -legend("topleft", variableInfo$param_units, +mtext("Discharge [ft3/s]", side = 4, line = 3, col = "red") +title("CHOPTANK RIVER NEAR GREENSBORO, MD") +legend("topleft", unique(temperatureAndFlow$unit_of_measure), col = c("black", "red"), lty = c(NA, 1), pch = c(1, NA) ) @@ -642,44 +584,6 @@ sites <- whatWQPmetrics(countycode = "US:55:025", siteType = type) ``` -# Embedded Metadata - -All data frames returned from the Web services have some form of associated metadata. This information is included as attributes to the data frame. All data frames will have a `url` (returning a character of the url used to obtain the data), `siteInfo` (returning a data frame with information on sites), and `queryTime` (returning a POSIXct datetime) attributes. For example, the url and query time used to obtain the data can be found as follows: - -```{r meta1, eval=FALSE} - -attr(dischargeWI, "url") - -attr(dischargeWI, "queryTime") - -siteInfo <- attr(dischargeWI, "siteInfo") -``` - -Depending on the format that the data was obtained (RDB, WaterML1, etc), there will be additional information embedded in the data frame as attributes. To discover the available attributes: - -```{r meta2, eval=FALSE} -names(attributes(dischargeWI)) -``` - -For data obtained from `readNWISuv`, `readNWISdv`, `readNWISgwl` there are two attributes that are particularly useful: `siteInfo` and `variableInfo`. - -```{r meta3, eval=FALSE} - -siteInfo <- attr(dischargeWI, "siteInfo") - -variableInfo <- attr(dischargeWI, "variableInfo") -``` - -Data obtained from `readNWISpeak`, `readNWISmeas`, and `readNWISrating`, the `comment` attribute is useful. - -```{r meta5, eval=FALSE} -comment(peakData) - -# Which is equivalent to: -attr(peakData, "comment") -``` - - # Getting Started in R This section describes the options for downloading and installing the `dataRetrieval` package.