Chk tbls (#147)

* warning user if tables are missing

* look for missing tables as well

* update internal data

* fixl ikn

* add in ability to specify path

* increment version, news and notes

* specify namespace

Author: boshek

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: tidyhydat
 Title: Extract and Tidy Canadian 'Hydrometric' Data
-Version: 0.5.1
+Version: 0.5.2
 Authors@R: c(person("Sam", "Albers", email = "sam.albers@gov.bc.ca", role = c("aut", "cre"),
     comment = c(ORCID = "0000-0002-9270-7884")),
     person("David", "Hutchinson", email = "david.hutchinson@canada.ca", role = "ctb"),

NEWS.md

@@ -1,3 +1,6 @@
+# tidyhydat 0.5.2
+- add internal function `hy_check` to verify that HYDAT contains all the right tables and that those tables contain data. 
+
 # tidyhydat 0.5.1
 - Replace `class(x) ==` with `inherits`
 - Fix bug and added corresponding tests where a request for multiple stations to `realtime_dd` would fail if any data was missing

R/cli_style.R

@@ -30,3 +30,11 @@ info <- function(msg){
 }
 
 
+red_message <- function(msg) {
+  message(crayon::red(cli::symbol$cross), " ", msg)
+}
+
+green_message <- function(msg){
+  message(crayon::green(cli::symbol$tick), " ", msg)
+}
+

R/download.R

@@ -117,5 +117,34 @@ download_hydat <- function(dl_hydat_here = NULL) {
     congrats("HYDAT successfully downloaded")
   } else(not_done("HYDAT not successfully downloaded"))
 
+  hy_check()
+  
   invisible(TRUE)
 }
+
+hy_check <- function(hydat_path = NULL) {
+  con <- hy_src(hydat_path)
+  on.exit(hy_src_disconnect(con))
+  
+  have_tbls <- dplyr::src_tbls(con)
+  
+  tbl_diff <- setdiff(hy_expected_tbls(), have_tbls)
+  if (!rlang::is_empty(tbl_diff)) {
+    red_message("The following tables are missing from HYDAT")
+    red_message(paste0(tbl_diff, "\n"))
+  }
+  
+  
+  invisible(lapply(have_tbls, function(x) {
+    tbl_rows <- dplyr::tbl(con, x) %>% 
+      utils::head(1) %>% 
+      dplyr::collect() %>% 
+      nrow()
+    
+    if(tbl_rows == 0) {
+      red_message(paste0(x, " table has no data."))
+    } 
+  }))
+  
+}
+

R/tidyhydat-package.R

@@ -16,7 +16,7 @@
 #'  \itemize{
 #'   \item use the \code{download_hydat()} function.
 #'   \item If that fails you can download directly from this link:
-#'   \url{https://collaboration.cmc.ec.gc.ca/cmc/hydrometrics/www}
+#'   \url{https://collaboration.cmc.ec.gc.ca/cmc/hydrometrics/www/}
 #'   }
 #' 
 #' For more information on tidy data please see

R/utils.R

@@ -229,3 +229,17 @@ pull_station_number <- function(.data){
   unique(.data$STATION_NUMBER)
 }
 
+
+## expected tables
+hy_expected_tbls <- function() {
+  c("AGENCY_LIST", "ANNUAL_INSTANT_PEAKS", "ANNUAL_STATISTICS", 
+    "CONCENTRATION_SYMBOLS", "DATA_SYMBOLS", "DATA_TYPES", "DATUM_LIST", 
+    "DLY_FLOWS", "DLY_LEVELS", "MEASUREMENT_CODES", "OPERATION_CODES", 
+    "PEAK_CODES", "PRECISION_CODES", "REGIONAL_OFFICE_LIST", "SAMPLE_REMARK_CODES", 
+    "SED_DATA_TYPES", "SED_DLY_LOADS", "SED_DLY_SUSCON", "SED_SAMPLES", 
+    "SED_SAMPLES_PSD", "SED_VERTICAL_LOCATION", "SED_VERTICAL_SYMBOLS", 
+    "STATIONS", "STN_DATA_COLLECTION", "STN_DATA_RANGE", "STN_DATUM_CONVERSION", 
+    "STN_DATUM_UNRELATED", "STN_OPERATION_SCHEDULE", "STN_REGULATION", 
+    "STN_REMARKS", "STN_REMARK_CODES", "STN_STATUS_CODES", "VERSION"
+  )
+}

cran-comments.md

@@ -1,7 +1,25 @@
-tidhydat 0.5.1
+tidyhydat 0.5.2
+=========================
+## NEWS
+- add internal function `hy_check` to verify that HYDAT contains all the right tables and that those tables contain data. 
+
+## Test environments
+* win-builder (via `devtools::check_win_devel()` and `devtools::check_win_release()`)
+* local Windows 10, R 4.0.2 (via R CMD check --as-cran)
+* ubuntu-20.04, r: 'release' (github actions)
+* ubuntu-20.04, r: 'devel' (github actions)
+* macOS,        r: 'release' (github actions)
+* windows,      r: 'release' (github actions)
+* Fedora Linux, R-devel, clang, gfortran - r-hub
+* Debian Linux, R-release, GCC (debian-gcc-release) - r-hub
+* Windows Server 2008 R2 SP1, R-devel, 32/64 bit - r-hub
+
+
+
+tidyhydat 0.5.1
 =========================
 ## Re-submission note
-- Fix all non-secure or borken links
+- Fix all non-secure or broken links
 
 ## NEWS
 - Replace `class(x) ==` with `inherits`

data-raw/HYDAT_internal_data/allstations.csv

@@ -4382,7 +4382,7 @@ STATION_NUMBER,STATION_NAME,PROV_TERR_STATE_LOC,HYD_STATUS,REAL_TIME,LATITUDE,LO
 05EC006,WHITE EARTH CREEK NEAR SMOKY LAKE,AB,DISCONTINUED,FALSE,54.11555862426758,-112.3000030517578,America/Edmonton,-7,Etc/GMT+7
 05EC007,REDWATER RIVER NEAR VIMY,AB,DISCONTINUED,FALSE,54.09416961669922,-113.56639099121094,America/Edmonton,-7,Etc/GMT+7
 05ED001,STONY CREEK NEAR SADDLE LAKE,AB,DISCONTINUED,FALSE,54.02444076538086,-111.69833374023437,America/Edmonton,-7,Etc/GMT+7
-05ED002,ATIMOSWE CREEK NEAR ELK POINT,AB,ACTIVE,FALSE,53.88880157470703,-110.9210968017578,America/Edmonton,-7,Etc/GMT+7
+05ED002,ATIMOSWE CREEK NEAR ELK POINT,AB,ACTIVE,TRUE,53.88880157470703,-110.9210968017578,America/Edmonton,-7,Etc/GMT+7
 05ED003,MOOSEHILLS CREEK NEAR ELK POINT,AB,DISCONTINUED,FALSE,53.934478759765625,-110.77713775634766,America/Edmonton,-7,Etc/GMT+7
 05EE001,VERMILION RIVER NEAR MANNVILLE,AB,DISCONTINUED,FALSE,53.375,-111.1750030517578,America/Edmonton,-7,Etc/GMT+7
 05EE002,VERMILION RIVER AT LEA PARK,AB,DISCONTINUED,FALSE,53.65555953979492,-110.3388900756836,America/Edmonton,-7,Etc/GMT+7
@@ -4740,7 +4740,7 @@ STATION_NUMBER,STATION_NAME,PROV_TERR_STATE_LOC,HYD_STATUS,REAL_TIME,LATITUDE,LO
 05KH004,BIRCH RIVER BELOW BRACKEN DAM NEAR THE PAS,MB,DISCONTINUED,FALSE,53.82500076293945,-101.3249969482422,America/Winnipeg,-6,Etc/GMT+6
 05KH006,ROCKY LAKE NEAR WANLESS,MB,DISCONTINUED,FALSE,54.17055892944336,-101.39443969726562,America/Winnipeg,-6,Etc/GMT+6
 05KH008,SASKATCHEWAN RIVER NEAR MANITOBA BOUNDARY,SK,DISCONTINUED,FALSE,53.8861083984375,-101.8277816772461,America/Regina,-6,Etc/GMT+6
-05KH009,SASKATCHEWAN RIVER OLD CHANNEL,SK,DISCONTINUED,FALSE,53.81388854980469,-102.97222137451172,America/Regina,-6,Etc/GMT+6
+05KH009,SASKATCHEWAN RIVER OLD CHANNEL,SK,ACTIVE,TRUE,53.81388854980469,-102.97222137451172,America/Regina,-6,Etc/GMT+6
 05KH010,SASKATCHEWAN RIVER BELOW CUMBERLAND HOUSE,SK,DISCONTINUED,FALSE,53.883331298828125,-101.9333267211914,America/Regina,-6,Etc/GMT+6
 05KH011,DRAGLINE CHANNEL NEAR SQUAW RAPIDS,SK,DISCONTINUED,FALSE,53.83332824707031,-102.94027709960937,America/Regina,-6,Etc/GMT+6
 05KH012,TEARING RIVER NEAR CUMBERLAND HOUSE,SK,DISCONTINUED,FALSE,53.95000076293945,-102.1833267211914,America/Regina,-6,Etc/GMT+6
@@ -5227,7 +5227,7 @@ STATION_NUMBER,STATION_NAME,PROV_TERR_STATE_LOC,HYD_STATUS,REAL_TIME,LATITUDE,LO
 05QD012,LAKE 261 OUTLET NEAR KENORA,ON,DISCONTINUED,FALSE,49.70832824707031,-93.6999969482422,America/Winnipeg,-6,Etc/GMT+6
 05QD013,LAKE 265 OUTLET NEAR KENORA,ON,DISCONTINUED,FALSE,49.720829010009766,-93.633331298828125,America/Winnipeg,-6,Etc/GMT+6
 05QD015,LAKE 226 OUTLET NEAR KENORA,ON,DISCONTINUED,FALSE,49.68888854980469,-93.7375030517578,America/Winnipeg,-6,Etc/GMT+6
-05QD016,WABIGOON RIVER AT DRYDEN,ON,ACTIVE,FALSE,49.82917022705078,-92.8708267211914,America/Winnipeg,-6,Etc/GMT+6
+05QD016,WABIGOON RIVER AT DRYDEN,ON,ACTIVE,FALSE,49.782169342041016,-92.84284210205078,America/Winnipeg,-6,Etc/GMT+6
 05QD017,LAKE 223 OUTLET NEAR KENORA,ON,DISCONTINUED,FALSE,49.69889068603516,-93.7138900756836,America/Winnipeg,-6,Etc/GMT+6
 05QD018,LAKE 224 OUTLET NEAR KENORA,ON,DISCONTINUED,FALSE,49.692501068115234,-93.71111297607422,America/Winnipeg,-6,Etc/GMT+6
 05QD019,LAKE 225 OUTLET NEAR KENORA,ON,DISCONTINUED,FALSE,49.6875,-93.7138900756836,America/Winnipeg,-6,Etc/GMT+6
@@ -5368,7 +5368,7 @@ STATION_NUMBER,STATION_NAME,PROV_TERR_STATE_LOC,HYD_STATUS,REAL_TIME,LATITUDE,LO
 06CA005,MONTREAL LAKE NEAR MOLANOSA,SK,DISCONTINUED,FALSE,54.13750076293945,-105.6972198486328,America/Regina,-6,Etc/GMT+6
 06CA006,MONTREAL LAKE NEAR WEYAKWIN,SK,ACTIVE,FALSE,54.40972137451172,-105.7236099243164,America/Regina,-6,Etc/GMT+6
 06CA007,KINGSMERE LAKE AT THE OUTLET STRUCTURE,SK,DISCONTINUED,FALSE,54.036109924316406,-106.41999816894533,America/Regina,-6,Etc/GMT+6
-06CA008,CREAN LAKE AT THE WARDEN STATION,SK,DISCONTINUED,FALSE,54.03110885620117,-106.24694061279295,America/Regina,-6,Etc/GMT+6
+06CA008,CREAN LAKE AT THE WARDEN STATION,SK,ACTIVE,TRUE,54.03121948242188,-106.2469711303711,America/Regina,-6,Etc/GMT+6
 06CA009,BEARTRAP CREEK NEAR WASKESIU LAKE,SK,DISCONTINUED,FALSE,53.89556121826172,-106.1555633544922,America/Regina,-6,Etc/GMT+6
 06CB003,NEMEIBEN LAKE NEAR LA RONGE,SK,DISCONTINUED,FALSE,55.26055908203125,-105.36943817138672,America/Regina,-6,Etc/GMT+6
 06CC001,SMOOTHSTONE RIVER BELOW EMMELINE LAKE,SK,ACTIVE,FALSE,55.163330078125,-106.20693969726562,America/Regina,-6,Etc/GMT+6
@@ -5650,7 +5650,7 @@ STATION_NUMBER,STATION_NAME,PROV_TERR_STATE_LOC,HYD_STATUS,REAL_TIME,LATITUDE,LO
 07NC009,SLAVE RIVER (RESDELTA CHANNEL) NEAR FORT RESOLUTION,NT,DISCONTINUED,FALSE,61.28472137451172,-113.58193969726562,America/Yellowknife,-7,Etc/GMT+7
 07OB005,MEANDER RIVER AT OUTLET HUTCH LAKE,AB,DISCONTINUED,FALSE,58.77082824707031,-117.3847198486328,America/Edmonton,-7,Etc/GMT+7
 07OB007,HUTCH LAKE TRIBUTARY NEAR HIGH LEVEL,AB,DISCONTINUED,FALSE,58.71805953979492,-117.24082946777344,America/Edmonton,-7,Etc/GMT+7
-07PA001,BUFFALO RIVER AT HIGHWAY NO. 5,NT,ACTIVE,TRUE,60.71366882324219,-114.90389251708984,America/Yellowknife,-7,Etc/GMT+7
+07PA001,BUFFALO RIVER AT HIGHWAY NO. 5,NT,DISCONTINUED,FALSE,60.71366882324219,-114.90389251708984,America/Yellowknife,-7,Etc/GMT+7
 07PA002,WHITESAND RIVER NEAR ALTA/NWT BOUNDARY,NT,DISCONTINUED,FALSE,60.00471878051758,-115.57888793945312,America/Yellowknife,-7,Etc/GMT+7
 07PB001,GREAT SLAVE LAKE AT FORT RESOLUTION,NT,DISCONTINUED,FALSE,61.16389083862305,-113.67916870117187,America/Yellowknife,-7,Etc/GMT+7
 07PB002,LITTLE BUFFALO RIVER BELOW HIGHWAY NO. 5,NT,DISCONTINUED,FALSE,60.0502815246582,-112.6977767944336,America/Yellowknife,-7,Etc/GMT+7
@@ -6033,13 +6033,14 @@ STATION_NUMBER,STATION_NAME,PROV_TERR_STATE_LOC,HYD_STATUS,REAL_TIME,LATITUDE,LO
 08HD012,SPRINGER CREEK NEAR SAYWARD,BC,DISCONTINUED,FALSE,50.34027862548828,-125.9055633544922,America/Vancouver,-8,Etc/GMT+8
 08HD013,BARRON CREEK NEAR BLOEDEL,BC,DISCONTINUED,FALSE,50.11222076416016,-125.336669921875,America/Vancouver,-8,Etc/GMT+8
 08HD014,BLACK CREEK AT STURGESS ROAD,BC,DISCONTINUED,FALSE,49.80805969238281,-125.14250183105467,America/Vancouver,-8,Etc/GMT+8
-08HD015,SALMON RIVER ABOVE CAMPBELL LAKE DIVERSION,BC,ACTIVE,TRUE,50.09041976928711,-125.67822265625,America/Vancouver,-8,Etc/GMT+8
+08HD015,SALMON RIVER ABOVE CAMPBELL LAKE DIVERSION,BC,ACTIVE,FALSE,50.09041976928711,-125.67822265625,America/Vancouver,-8,Etc/GMT+8
 08HD016,HYACINTHE CREEK ON QUADRA ISLAND,BC,DISCONTINUED,FALSE,50.133888244628906,-125.25360870361328,America/Vancouver,-8,Etc/GMT+8
 08HD017,BASIL BROOK ON CORTES ISLAND,BC,DISCONTINUED,FALSE,50.11528015136719,-124.91583251953124,America/Vancouver,-8,Etc/GMT+8
 08HD020,SALMON RIVER DIVERSION NEAR CAMPBELL RIVER,BC,DISCONTINUED,FALSE,50.09366989135742,-125.66974639892578,America/Vancouver,-8,Etc/GMT+8
-08HD022,CAMPBELL RIVER AT CAMPBELL RIVER LODGE,BC,ACTIVE,FALSE,50.03364181518555,-125.267578125,America/Vancouver,-8,Etc/GMT+8
+08HD022,CAMPBELL RIVER AT CAMPBELL RIVER LODGE,BC,ACTIVE,TRUE,50.03364181518555,-125.267578125,America/Vancouver,-8,Etc/GMT+8
 08HD029,CAMPBELL RIVER (SPILLWAY FLOWS AT JOHN HART DAM),BC,ACTIVE,FALSE,50.04333114624024,-125.33708190917967,America/Vancouver,-8,Etc/GMT+8
 08HD030,PIGGOTT CREEK AT MOUNT WASHINGTON,BC,DISCONTINUED,FALSE,49.75,-125.34027862548828,America/Vancouver,-8,Etc/GMT+8
+08HDX03,CAMPBELL RIVER NEAR CAMPBELL RIVER (TEST STATION),BC,ACTIVE,TRUE,50.03731155395508,-125.2874984741211,America/Vancouver,-8,Etc/GMT+8
 08HE001,MARBLE RIVER AT OUTLET OF ALICE LAKE,BC,DISCONTINUED,FALSE,50.52389144897461,-127.43083190917967,America/Vancouver,-8,Etc/GMT+8
 08HE002,MARBLE RIVER AT OUTLET OF VICTORIA LAKE,BC,DISCONTINUED,FALSE,50.43027877807617,-127.40389251708984,America/Vancouver,-8,Etc/GMT+8
 08HE003,BENSON RIVER NEAR PORT ALICE,BC,DISCONTINUED,FALSE,50.4011116027832,-127.29778289794922,America/Vancouver,-8,Etc/GMT+8
@@ -6506,7 +6507,7 @@ STATION_NUMBER,STATION_NAME,PROV_TERR_STATE_LOC,HYD_STATUS,REAL_TIME,LATITUDE,LO
 08LG053,WALLOPER CREEK ABOVE WALLOPER LAKE,BC,DISCONTINUED,FALSE,50.48556137084961,-120.54444122314452,America/Vancouver,-8,Etc/GMT+8
 08LG054,SKEIKUT CREEK ABOVE INTAKE,BC,DISCONTINUED,FALSE,50.324440002441406,-121.22860717773437,America/Vancouver,-8,Etc/GMT+8
 08LG055,BETHSAIDA CREEK ABOVE HIGHLAND VALLEY ROAD,BC,DISCONTINUED,FALSE,50.47999954223633,-121.03333282470705,America/Vancouver,-8,Etc/GMT+8
-08LG056,GUICHON CREEK ABOVE TUNKWA LAKE DIVERSION,BC,ACTIVE,FALSE,50.60783004760742,-120.91085815429687,America/Vancouver,-8,Etc/GMT+8
+08LG056,GUICHON CREEK ABOVE TUNKWA LAKE DIVERSION,BC,ACTIVE,TRUE,50.60783004760742,-120.91085815429687,America/Vancouver,-8,Etc/GMT+8
 08LG057,MIDDAY CREEK ABOVE DIVERSIONS,BC,DISCONTINUED,FALSE,50.00582885742188,-120.9816665649414,America/Vancouver,-8,Etc/GMT+8
 08LG058,MIDDAY CREEK NEAR THE MOUTH,BC,DISCONTINUED,FALSE,49.97777938842773,-120.94000244140624,America/Vancouver,-8,Etc/GMT+8
 08LG059,MIDDAY CREEK ABOVE 800 M CONTOUR,BC,DISCONTINUED,FALSE,49.99639129638672,-120.9847183227539,America/Vancouver,-8,Etc/GMT+8
@@ -6653,7 +6654,7 @@ STATION_NUMBER,STATION_NAME,PROV_TERR_STATE_LOC,HYD_STATUS,REAL_TIME,LATITUDE,LO
 08MF066,CINQUEFOIL CREEK ABOVE DIVERSIONS,BC,DISCONTINUED,FALSE,50.63444137573242,-121.77027893066406,America/Vancouver,-8,Etc/GMT+8
 08MF067,KOPP CREEK ABOVE MENZ CREEK,BC,DISCONTINUED,FALSE,49.38417053222656,-121.39111328125,America/Vancouver,-8,Etc/GMT+8
 08MF071,BOULDER CREEK NEAR THE MOUTH,BC,DISCONTINUED,FALSE,49.249168395996094,-121.5999984741211,America/Vancouver,-8,Etc/GMT+8
-08MG001,CHEHALIS RIVER NEAR HARRISON MILLS,BC,ACTIVE,FALSE,49.30002975463867,-121.9377670288086,America/Vancouver,-8,Etc/GMT+8
+08MG001,CHEHALIS RIVER NEAR HARRISON MILLS,BC,ACTIVE,TRUE,49.30002975463867,-121.9377670288086,America/Vancouver,-8,Etc/GMT+8
 08MG003,GREEN RIVER NEAR PEMBERTON,BC,DISCONTINUED,FALSE,50.28194046020508,-122.85138702392578,America/Vancouver,-8,Etc/GMT+8
 08MG004,GREEN RIVER NEAR RAINBOW,BC,DISCONTINUED,FALSE,50.17082977294922,-122.91667175292967,America/Vancouver,-8,Etc/GMT+8
 08MG006,RUTHERFORD CREEK NEAR PEMBERTON,BC,DISCONTINUED,FALSE,50.26667022705078,-122.87222290039062,America/Vancouver,-8,Etc/GMT+8
@@ -7554,7 +7555,7 @@ STATION_NUMBER,STATION_NAME,PROV_TERR_STATE_LOC,HYD_STATUS,REAL_TIME,LATITUDE,LO
 08PA009,ROSS RESERVOIR NEAR NEWHALEM,WA,ACTIVE,FALSE,48.73278045654297,-121.06722259521484,America/Los_Angeles,-8,Etc/GMT+8
 08PA010,LIGHTNING LAKE NEAR MANNING PARK,BC,DISCONTINUED,FALSE,49.06222152709961,-120.82611083984376,America/Vancouver,-8,Etc/GMT+8
 09AA003,LINDEMAN LAKE NEAR BENNETT,BC,DISCONTINUED,FALSE,59.83332824707031,-135.01278686523438,America/Vancouver,-8,Etc/GMT+8
-09AA006,ATLIN RIVER NEAR ATLIN,BC,ACTIVE,FALSE,59.595359802246094,-133.81442260742187,America/Vancouver,-8,Etc/GMT+8
+09AA006,ATLIN RIVER NEAR ATLIN,BC,ACTIVE,TRUE,59.595359802246094,-133.81442260742187,America/Vancouver,-8,Etc/GMT+8
 09AA007,LUBBOCK RIVER NEAR ATLIN,YT,DISCONTINUED,FALSE,60.08110809326172,-133.85833740234375,America/Whitehorse,-8,Etc/GMT+8
 09AA008,PINE CREEK NEAR ATLIN,BC,DISCONTINUED,FALSE,59.56111145019531,-133.66555786132812,America/Vancouver,-8,Etc/GMT+8
 09AA009,WATSON RIVER NEAR CARCROSS,YT,DISCONTINUED,FALSE,60.21667098999024,-134.73056030273438,America/Whitehorse,-8,Etc/GMT+8