Skip to content
This repository was archived by the owner on Oct 12, 2023. It is now read-only.

Commit 5f1afc7

Browse files
zfengmszfengms
authored
Feature/configfromobj support programmatically created credentials and cluster configs (#168)
* support programmatically generated credentials/cluster config * update docs for programmatically generated config * reformat code * styling fixes * combine credentials cluster config methods * fix set credentials issue and test input * do not create az_config.json * update messages * move githubAuthenticationToken from cluster config to credentials
1 parent bc529f0 commit 5f1afc7

19 files changed

+281
-134
lines changed

‎CHANGELOG.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11
# Change Log
2+
## [0.6.1] 2017-11-13
3+
### Added
4+
- Support for users to use programmatically generated credentials and cluster config
5+
26
## [0.6.0] 2017-11-03
37
### Added
48
- Support for users to run custom versions of R via Docker containers

‎R/cluster.R

Lines changed: 25 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -1,67 +1,3 @@
1-
#' Creates a credentials file for rAzureBatch package authentication
2-
#'
3-
#' @param fileName Credentials file name
4-
#' @param ... Further named parameters
5-
#' \itemize{
6-
#' \item{"batchAccount"}: {Batch account name for Batch Service authentication.}
7-
#' \item{"batchKey"}: {Batch account key for signing REST signatures.}
8-
#' \item{"batchUrl"}: {Batch service url for account.}
9-
#' \item{"storageAccount"}: {Storage account for storing output results.}
10-
#' \item{"storageKey"}: {Storage account key for storage service authentication.}
11-
#'}
12-
#' @return The request to the Batch service was successful.
13-
#' @examples {
14-
#' generateCredentialsConfig("test_config.json")
15-
#' generateCredentialsConfig("test_config.json", batchAccount = "testbatchaccount",
16-
#' batchKey = "test_batch_account_key", batchUrl = "http://testbatchaccount.azure.com",
17-
#' storageAccount = "teststorageaccount", storageKey = "test_storage_account_key")
18-
#' }
19-
#' @export
20-
generateCredentialsConfig <- function(fileName, ...) {
21-
args <- list(...)
22-
23-
batchAccount <-
24-
ifelse(is.null(args$batchAccount),
25-
"batch_account_name",
26-
args$batchAccount)
27-
batchKey <-
28-
ifelse(is.null(args$batchKey), "batch_account_key", args$batchKey)
29-
batchUrl <-
30-
ifelse(is.null(args$batchUrl), "batch_account_url", args$batchUrl)
31-
32-
storageName <-
33-
ifelse(is.null(args$storageAccount),
34-
"storage_account_name",
35-
args$storageAccount)
36-
storageKey <-
37-
ifelse(is.null(args$storageKey),
38-
"storage_account_key",
39-
args$storageKey)
40-
41-
if (!file.exists(paste0(getwd(), "/", fileName))) {
42-
config <- list(
43-
batchAccount = list(
44-
name = batchAccount,
45-
key = batchKey,
46-
url = batchUrl
47-
),
48-
storageAccount = list(name = storageName,
49-
key = storageKey)
50-
)
51-
52-
configJson <-
53-
jsonlite::toJSON(config, auto_unbox = TRUE, pretty = TRUE)
54-
write(configJson, file = paste0(getwd(), "/", fileName))
55-
56-
print(
57-
sprintf(
58-
"A config file has been generated %s. Please enter your Batch credentials.",
59-
paste0(getwd(), "/", fileName)
60-
)
61-
)
62-
}
63-
}
64-
651
#' Creates a configuration file for the user's cluster setup.
662
#'
673
#' @param fileName Cluster settings file name
@@ -90,8 +26,7 @@ generateClusterConfig <- function(fileName) {
9026
rPackages = list(
9127
cran = vector(),
9228
github = vector(),
93-
bioconductor = vector(),
94-
githubAuthenticationToken = ""
29+
bioconductor = vector()
9530
),
9631
commandLine = vector()
9732
)
@@ -114,7 +49,7 @@ generateClusterConfig <- function(fileName) {
11449

11550
#' Creates an Azure cloud-enabled cluster.
11651
#'
117-
#' @param clusterSetting Cluster configuration's file name
52+
#' @param clusterSetting Cluster configuration object or file name
11853
#' @param fullName A boolean flag for checking the file full name
11954
#' @param wait A boolean flag to wait for all nodes to boot up
12055
#' @param resourceFiles A list of files that Batch will download to the compute node before running the command line
@@ -130,12 +65,21 @@ makeCluster <-
13065
fullName = FALSE,
13166
wait = TRUE,
13267
resourceFiles = list()) {
133-
if (fullName) {
134-
poolConfig <- rjson::fromJSON(file = paste0(clusterSetting))
135-
}
136-
else {
137-
poolConfig <-
138-
rjson::fromJSON(file = paste0(getwd(), "/", clusterSetting))
68+
if (class(clusterSetting) == "character") {
69+
if (fullName) {
70+
poolConfig <- rjson::fromJSON(file = paste0(clusterSetting))
71+
}
72+
else {
73+
poolConfig <-
74+
rjson::fromJSON(file = paste0(getwd(), "/", clusterSetting))
75+
}
76+
} else if (class(clusterSetting) == "list") {
77+
poolConfig <- clusterSetting
78+
} else {
79+
stop(sprintf(
80+
"cluster setting type is not supported: %s\n",
81+
class(clusterSetting)
82+
))
13983
}
14084

14185
config <- getOption("az_config")
@@ -199,11 +143,13 @@ makeCluster <-
199143
containerInstallCommand <- c(
200144
paste0(
201145
"wget https://raw.githubusercontent.com/Azure/doAzureParallel/",
202-
"master/inst/startup/cluster_setup.sh"),
146+
"master/inst/startup/cluster_setup.sh"
147+
),
203148
"chmod u+x cluster_setup.sh",
204149
paste0(
205150
"wget https://raw.githubusercontent.com/Azure/doAzureParallel/",
206-
"master/inst/startup/install_bioconductor.R"),
151+
"master/inst/startup/install_bioconductor.R"
152+
),
207153
"chmod u+x install_bioconductor.R",
208154
installAndStartContainerCommand
209155
)
@@ -220,14 +166,13 @@ makeCluster <-
220166
}
221167

222168
environmentSettings <- NULL
223-
if (!is.null(poolConfig$rPackages) &&
224-
!is.null(poolConfig$rPackages$githubAuthenticationToken) &&
225-
poolConfig$rPackages$githubAuthenticationToken != "") {
169+
if (!is.null(config$githubAuthenticationToken) &&
170+
config$githubAuthenticationToken != "") {
226171
environmentSettings <-
227172
list(
228173
list(
229174
name = "GITHUB_PAT",
230-
value = poolConfig$rPackages$githubAuthenticationToken
175+
value = config$githubAuthenticationToken
231176
)
232177
)
233178
}
@@ -375,24 +320,7 @@ makeCluster <-
375320
stopCluster <- function(cluster) {
376321
rAzureBatch::deletePool(cluster$poolId)
377322

378-
print(sprintf("Your %s cluster has been destroyed.", cluster$poolId))
379-
}
380-
381-
#' Set azure credentials to R session.
382-
#'
383-
#' @param fileName The cluster configuration that was created in \code{makeCluster}
384-
#'
385-
#' @export
386-
setCredentials <- function(fileName = "az_config.json") {
387-
if (file.exists(fileName)) {
388-
config <- rjson::fromJSON(file = paste0(fileName))
389-
}
390-
else{
391-
config <- rjson::fromJSON(file = paste0(getwd(), "/", fileName))
392-
}
393-
394-
options("az_config" = config)
395-
print("Your azure credentials have been set.")
323+
print(sprintf("Your %s cluster is being deleted.", cluster$poolId))
396324
}
397325

398326
getPoolWorkers <- function(poolId, ...) {

‎R/credentials.R

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
#' Creates a credentials file for rAzureBatch package authentication
2+
#'
3+
#' @param fileName Credentials file name
4+
#' @param ... Further named parameters
5+
#' \itemize{
6+
#' \item{"batchAccount"}: {Batch account name for Batch Service authentication.}
7+
#' \item{"batchKey"}: {Batch account key for signing REST signatures.}
8+
#' \item{"batchUrl"}: {Batch service url for account.}
9+
#' \item{"storageAccount"}: {Storage account for storing output results.}
10+
#' \item{"storageKey"}: {Storage account key for storage service authentication.}
11+
#'}
12+
#' @return The request to the Batch service was successful.
13+
#' @examples {
14+
#' generateCredentialsConfig("test_config.json")
15+
#' generateCredentialsConfig("test_config.json", batchAccount = "testbatchaccount",
16+
#' batchKey = "test_batch_account_key", batchUrl = "http://testbatchaccount.azure.com",
17+
#' storageAccount = "teststorageaccount", storageKey = "test_storage_account_key")
18+
#' }
19+
#' @export
20+
generateCredentialsConfig <- function(fileName, ...) {
21+
args <- list(...)
22+
23+
batchAccount <-
24+
ifelse(is.null(args$batchAccount),
25+
"batch_account_name",
26+
args$batchAccount)
27+
batchKey <-
28+
ifelse(is.null(args$batchKey), "batch_account_key", args$batchKey)
29+
batchUrl <-
30+
ifelse(is.null(args$batchUrl), "batch_account_url", args$batchUrl)
31+
32+
storageName <-
33+
ifelse(is.null(args$storageAccount),
34+
"storage_account_name",
35+
args$storageAccount)
36+
storageKey <-
37+
ifelse(is.null(args$storageKey),
38+
"storage_account_key",
39+
args$storageKey)
40+
41+
githubAuthenticationToken <-
42+
ifelse(is.null(args$githubAuthenticationToken),
43+
"",
44+
args$githubAuthenticationToken)
45+
46+
if (!file.exists(paste0(getwd(), "/", fileName))) {
47+
config <- list(
48+
batchAccount = list(name = batchAccount,
49+
key = batchKey,
50+
url = batchUrl),
51+
storageAccount = list(name = storageName,
52+
key = storageKey),
53+
githubAuthenticationToken = githubAuthenticationToken
54+
)
55+
56+
configJson <-
57+
jsonlite::toJSON(config, auto_unbox = TRUE, pretty = TRUE)
58+
write(configJson, file = paste0(getwd(), "/", fileName))
59+
60+
print(
61+
sprintf(
62+
"A config file has been generated %s. Please enter your Batch credentials.",
63+
paste0(getwd(), "/", fileName)
64+
)
65+
)
66+
}
67+
}
68+
69+
#' Set azure credentials to R session from credentials object or json file.
70+
#'
71+
#' @param credentials The credentials object or json file
72+
#'
73+
#' @export
74+
setCredentials <- function(credentials = "az_config.json") {
75+
if (class(credentials) == "character") {
76+
fileName <- credentials
77+
if (file.exists(fileName)) {
78+
config <- rjson::fromJSON(file = paste0(fileName))
79+
}
80+
else{
81+
config <- rjson::fromJSON(file = paste0(getwd(), "/", fileName))
82+
}
83+
} else if (class(credentials) == "list") {
84+
config <- credentials
85+
} else {
86+
stop(sprintf(
87+
"credentials type is not supported: %s\n",
88+
class(clusterSetting)
89+
))
90+
}
91+
92+
options("az_config" = config)
93+
print("Your azure credentials have been set.")
94+
}

‎R/validationUtilities.R

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,7 @@ validationClass <- R6::R6Class(
2323
}
2424
},
2525
# Validating cluster configuration files below doAzureParallel version 0.3.2
26-
isValidDeprecatedClusterConfig = function(clusterFilePath) {
27-
if (file.exists(clusterFilePath)) {
28-
poolConfig <- rjson::fromJSON(file = clusterFilePath)
29-
}
30-
else{
31-
poolConfig <-
32-
rjson::fromJSON(file = file.path(getwd(), clusterFilePath))
33-
}
34-
26+
isValidDeprecatedClusterConfig = function(poolConfig) {
3527
if (is.null(poolConfig$pool$poolSize)) {
3628
stop("Missing poolSize entry")
3729
}
@@ -85,12 +77,22 @@ validationClass <- R6::R6Class(
8577

8678
TRUE
8779
},
88-
isValidClusterConfig = function(clusterFilePath) {
89-
if (file.exists(clusterFilePath)) {
90-
pool <- rjson::fromJSON(file = clusterFilePath)
91-
}
92-
else{
93-
pool <- rjson::fromJSON(file = file.path(getwd(), clusterFilePath))
80+
isValidClusterConfig = function(clusterSetting) {
81+
if (class(clusterSetting) == "character") {
82+
clusterFilePath <- clusterSetting
83+
if (file.exists(clusterFilePath)) {
84+
pool <- rjson::fromJSON(file = clusterFilePath)
85+
}
86+
else{
87+
pool <- rjson::fromJSON(file = file.path(getwd(), clusterFilePath))
88+
}
89+
} else if (class(clusterSetting) == "list") {
90+
pool <- clusterSetting
91+
} else {
92+
stop(sprintf(
93+
"cluster setting type is not supported: %s\n",
94+
class(clusterSetting)
95+
))
9496
}
9597

9698
if (is.null(pool$poolSize)) {

‎README.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,8 @@ Use your credential config JSON file to enter your credentials.
142142
"storageAccount": {
143143
"name": <Azure Storage Account Name>,
144144
"key": <Azure Storage Account Key>
145-
}
145+
},
146+
"githubAuthenticationToken": {}
146147
}
147148
```
148149
Learn more:
@@ -170,8 +171,7 @@ Use your pool configuration JSON file to define your pool in Azure.
170171
},
171172
"rPackages": {
172173
"cran": ["some_cran_package", "some_other_cran_package"],
173-
"github": ["username/some_github_package", "another_username/some_other_github_package"],
174-
"githubAuthenticationToken": {}
174+
"github": ["username/some_github_package", "another_username/some_other_github_package"]
175175
},
176176
"commandLine": []
177177
}

‎docs/30-customize-cluster.md

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,7 @@ Specifying a docker container is done by updating your cluster.json file. Simply
2626
"rPackages": {
2727
"cran": [],
2828
"github": [],
29-
"bioconductor": [],
30-
"githubAuthenticationToken": ""
29+
"bioconductor": []
3130
},
3231
"commandLine": []
3332
}

0 commit comments

Comments
 (0)