From fcd676f8a74f5d6f3468967b3bcb77eec926a049 Mon Sep 17 00:00:00 2001 From: endre-seqera Date: Fri, 26 Jan 2024 15:23:40 +0100 Subject: [PATCH 1/2] feat: remove storage account name if present from azure path Signed-off-by: endre-seqera --- .../cloud/azure/file/AzPathFactory.groovy | 5 ++++ .../cloud/azure/file/AzPathFactoryTest.groovy | 28 ++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzPathFactory.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzPathFactory.groovy index d81f37a52a..443a50c139 100644 --- a/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzPathFactory.groovy +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzPathFactory.groovy @@ -48,6 +48,11 @@ class AzPathFactory extends FileSystemPathFactory { if( uri.startsWith('az:///') ) throw new IllegalArgumentException("Invalid Azure path URI - make sure the schema prefix does not container more than two slash characters - offending value: $uri") + final storageAccountName = AzConfig.getConfig().storage().accountName + if (uri.contains(storageAccountName)){ + uri = uri.replace("${storageAccountName}.","") + } + final storageConfigEnv = AzConfig.getConfig().storage().getEnv() final activeDirectoryConfigEnv = AzConfig.getConfig().activeDirectory().getEnv() diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzPathFactoryTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzPathFactoryTest.groovy index a5fb0f4eca..0588170751 100644 --- a/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzPathFactoryTest.groovy +++ b/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzPathFactoryTest.groovy @@ -38,7 +38,7 @@ class AzPathFactoryTest extends Specification { cleanup: Global.session = null - + where: AZ_URI | CONTAINER | BLOB 'az://my-data/foo/bar' | 'my-data' | 'foo/bar' @@ -46,6 +46,32 @@ class AzPathFactoryTest extends Specification { } + def 'should create az azure path and remove storage account name if present' () { + given: + def CONFIG = [azure: [ + storage: [ + accountKey: System.getenv('AZURE_STORAGE_ACCOUNT_KEY'), + accountName: System.getenv('AZURE_STORAGE_ACCOUNT_NAME'), + ] + ]] + Global.session = Mock(Session) { getConfig() >> CONFIG } + and: + + when: + def path = AzPathFactory.parse(AZ_URI) + then: + path instanceof AzPath + (path as AzPath).containerName == CONTAINER + (path as AzPath).blobName() == BLOB + + cleanup: + Global.session = null + + where: + storageAccount | AZ_URI | CONTAINER | BLOB + System.getenv('AZURE_STORAGE_ACCOUNT_NAME') | "az://${storageAccount}.my-data/foo/bar" | 'my-data' | 'foo/bar' + System.getenv('AZURE_STORAGE_ACCOUNT_NAME') | "az://${storageAccount}.my-data/data/*{1,2}.fq.gz"| 'my-data' | 'data/*{1,2}.fq.gz' + } def 'should throw illegal path' () { given: From 4704d9579a5aed209f992de43844366d333b338a Mon Sep 17 00:00:00 2001 From: endre-seqera Date: Fri, 26 Jan 2024 17:46:34 +0100 Subject: [PATCH 2/2] impr: only remove storage account name from the beginning of the path Signed-off-by: endre-seqera --- .../nextflow/cloud/azure/file/AzPathFactory.groovy | 4 ++-- .../cloud/azure/file/AzPathFactoryTest.groovy | 12 ++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzPathFactory.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzPathFactory.groovy index 443a50c139..70d57d9723 100644 --- a/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzPathFactory.groovy +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzPathFactory.groovy @@ -49,8 +49,8 @@ class AzPathFactory extends FileSystemPathFactory { throw new IllegalArgumentException("Invalid Azure path URI - make sure the schema prefix does not container more than two slash characters - offending value: $uri") final storageAccountName = AzConfig.getConfig().storage().accountName - if (uri.contains(storageAccountName)){ - uri = uri.replace("${storageAccountName}.","") + if (uri.startsWith("az://${storageAccountName}.") ) { + uri = uri.replace("az://${storageAccountName}.","az://") } final storageConfigEnv = AzConfig.getConfig().storage().getEnv() diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzPathFactoryTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzPathFactoryTest.groovy index 0588170751..8c6a9e03ba 100644 --- a/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzPathFactoryTest.groovy +++ b/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzPathFactoryTest.groovy @@ -46,7 +46,7 @@ class AzPathFactoryTest extends Specification { } - def 'should create az azure path and remove storage account name if present' () { + def 'should create az azure path and remove storage account from the beginning of the path, if present' () { given: def CONFIG = [azure: [ storage: [ @@ -68,9 +68,13 @@ class AzPathFactoryTest extends Specification { Global.session = null where: - storageAccount | AZ_URI | CONTAINER | BLOB - System.getenv('AZURE_STORAGE_ACCOUNT_NAME') | "az://${storageAccount}.my-data/foo/bar" | 'my-data' | 'foo/bar' - System.getenv('AZURE_STORAGE_ACCOUNT_NAME') | "az://${storageAccount}.my-data/data/*{1,2}.fq.gz"| 'my-data' | 'data/*{1,2}.fq.gz' + storageAccount | AZ_URI | CONTAINER | BLOB + System.getenv('AZURE_STORAGE_ACCOUNT_NAME') | "az://${storageAccount}.my-data/foo/bar" | 'my-data' | 'foo/bar' + System.getenv('AZURE_STORAGE_ACCOUNT_NAME') | "az://${storageAccount}.my-data/data/*{1,2}.fq.gz" | 'my-data' | 'data/*{1,2}.fq.gz' + System.getenv('AZURE_STORAGE_ACCOUNT_NAME') | "az://${storageAccount}.my-data/${storageAccount}.bar" | 'my-data' | "${storageAccount}.bar" + System.getenv('AZURE_STORAGE_ACCOUNT_NAME') | "az://${storageAccount}.${storageAccount}./foo/bar" | "${storageAccount}." | 'foo/bar' + System.getenv('AZURE_STORAGE_ACCOUNT_NAME') | "az://my-data/${storageAccount}.txt" | 'my-data' | "${storageAccount}.txt" + } def 'should throw illegal path' () {