From af3f71fae4bba4fe2c836e1726a45dc13c7a4b69 Mon Sep 17 00:00:00 2001 From: oandreeva-nv Date: Mon, 18 Sep 2023 12:44:23 -0700 Subject: [PATCH 1/5] Add ability to specify directory for cloud storage --- src/filesystem/api.cc | 11 +++++++++- src/filesystem/api.h | 9 +++++++++ src/filesystem/implementations/as.h | 27 +++++++++++++++---------- src/filesystem/implementations/common.h | 17 +++++++++++++++- src/filesystem/implementations/gcs.h | 13 ++++++++---- src/filesystem/implementations/local.h | 11 +++++++--- src/filesystem/implementations/s3.h | 18 ++++++++++++----- 7 files changed, 81 insertions(+), 25 deletions(-) diff --git a/src/filesystem/api.cc b/src/filesystem/api.cc index 4f3883ccc..b97826acb 100644 --- a/src/filesystem/api.cc +++ b/src/filesystem/api.cc @@ -619,7 +619,16 @@ MakeTemporaryDirectory(const FileSystemType type, std::string* temp_dir) { std::shared_ptr fs; RETURN_IF_ERROR(fsm_.GetFileSystem(type, fs)); - return fs->MakeTemporaryDirectory(temp_dir); + return fs->MakeTemporaryDirectory("/tmp", temp_dir); +} + +Status +MakeTemporaryDirectory( + const FileSystemType type, std::string& dir_path, std::string* temp_dir) +{ + std::shared_ptr fs; + RETURN_IF_ERROR(fsm_.GetFileSystem(type, fs)); + return fs->MakeTemporaryDirectory(dir_path, temp_dir); } Status diff --git a/src/filesystem/api.h b/src/filesystem/api.h index e7711ce8e..36adca35a 100644 --- a/src/filesystem/api.h +++ b/src/filesystem/api.h @@ -198,6 +198,15 @@ Status MakeDirectory(const std::string& dir, const bool recursive); /// \return Error status Status MakeTemporaryDirectory(const FileSystemType type, std::string* temp_dir); +/// Create a temporary directory of the specified filesystem type +/// in a provided location. +/// \param type The type of the filesystem. +/// \param dir_path The specified path. +/// \param temp_dir Returns the path to the temporary directory. +/// \return Error status +Status MakeTemporaryDirectory( + const FileSystemType type, std::string& dir_path, std::string* temp_dir); + /// Delete a path. /// \param path The path to the directory or file. /// \return Error status diff --git a/src/filesystem/implementations/as.h b/src/filesystem/implementations/as.h index 13eb80d99..0d4d47f68 100644 --- a/src/filesystem/implementations/as.h +++ b/src/filesystem/implementations/as.h @@ -94,7 +94,8 @@ class ASFileSystem : public FileSystem { const std::string& path, const char* contents, const size_t content_len) override; Status MakeDirectory(const std::string& dir, const bool recursive) override; - Status MakeTemporaryDirectory(std::string* temp_dir) override; + Status MakeTemporaryDirectory( + std::string& dir_path, std::string* temp_dir) override; Status DeletePath(const std::string& path) override; private: @@ -441,17 +442,20 @@ ASFileSystem::LocalizePath( "AS file localization not yet implemented " + path); } - std::string folder_template = "/tmp/folderXXXXXX"; - char* tmp_folder = mkdtemp(const_cast(folder_template.c_str())); - if (tmp_folder == nullptr) { - return Status( - Status::Code::INTERNAL, - "Failed to create local temp folder: " + folder_template + - ", errno:" + strerror(errno)); - } + // Create a local directory for azure model store. + // If ENV variable are not set, creates a temporary directory + // under `/tmp` with the format: "folderXXXXXX". + // Otherwise, will create a folder under specified directory with the same + // format. + std::string env_mount_dir = + GetEnvironmentVariableOrDefault("TRITON_AZURE_MOUNT_DIRECTORY", "/tmp"); + std::string tmp_folder; + RETURN_IF_ERROR(triton::core::MakeTemporaryDirectory( + FileSystemType::LOCAL, std::string(env_mount_dir), &tmp_folder)); + localized->reset(new LocalizedPath(path, tmp_folder)); - std::string dest(folder_template); + std::string dest(tmp_folder); std::string container, blob; RETURN_IF_ERROR(ParsePath(path, &container, &blob)); @@ -495,7 +499,8 @@ ASFileSystem::MakeDirectory(const std::string& dir, const bool recursive) } Status -ASFileSystem::MakeTemporaryDirectory(std::string* temp_dir) +ASFileSystem::MakeTemporaryDirectory( + std::string& dir_path, std::string* temp_dir) { return Status( Status::Code::UNSUPPORTED, diff --git a/src/filesystem/implementations/common.h b/src/filesystem/implementations/common.h index 5f45444dd..f7962357b 100644 --- a/src/filesystem/implementations/common.h +++ b/src/filesystem/implementations/common.h @@ -91,7 +91,8 @@ class FileSystem { const size_t content_len) = 0; virtual Status MakeDirectory( const std::string& dir, const bool recursive) = 0; - virtual Status MakeTemporaryDirectory(std::string* temp_dir) = 0; + virtual Status MakeTemporaryDirectory( + std::string dir_path, std::string* temp_dir) = 0; virtual Status DeletePath(const std::string& path) = 0; }; @@ -106,4 +107,18 @@ AppendSlash(const std::string& name) return (name + "/"); } +/// Helper function to get the value of the environment variable, +/// or default value if not set. +/// +/// \param variable_name The name of the environment variable. +/// \param default_value The default value. +/// \return The environment variable or the default value if not set. +std::string +GetEnvironmentVariableOrDefault( + const std::string& variable_name, const std::string& default_value) +{ + const char* value = getenv(variable_name.c_str()); + return value ? value : default_value; +} + }} // namespace triton::core diff --git a/src/filesystem/implementations/gcs.h b/src/filesystem/implementations/gcs.h index 56c3d8d34..8f9ad23c3 100644 --- a/src/filesystem/implementations/gcs.h +++ b/src/filesystem/implementations/gcs.h @@ -83,7 +83,8 @@ class GCSFileSystem : public FileSystem { const std::string& path, const char* contents, const size_t content_len) override; Status MakeDirectory(const std::string& dir, const bool recursive) override; - Status MakeTemporaryDirectory(std::string* temp_dir) override; + Status MakeTemporaryDirectory( + std::string& dir_path, std::string* temp_dir) override; Status DeletePath(const std::string& path) override; private: @@ -380,9 +381,12 @@ GCSFileSystem::LocalizePath( "GCS file localization not yet implemented " + path); } + // Create a local directory for GCS model store. + std::string env_mount_dir = + GetEnvironmentVariableOrDefault("TRITON_GCS_MOUNT_DIRECTORY", "/tmp"); std::string tmp_folder; - RETURN_IF_ERROR( - triton::core::MakeTemporaryDirectory(FileSystemType::LOCAL, &tmp_folder)); + RETURN_IF_ERROR(triton::core::MakeTemporaryDirectory( + FileSystemType::LOCAL, std::string(env_mount_dir), &tmp_folder)); localized->reset(new LocalizedPath(path, tmp_folder)); @@ -480,7 +484,8 @@ GCSFileSystem::MakeDirectory(const std::string& dir, const bool recursive) } Status -GCSFileSystem::MakeTemporaryDirectory(std::string* temp_dir) +GCSFileSystem::MakeTemporaryDirectory( + std::string& dir_path, std::string* temp_dir) { return Status( Status::Code::UNSUPPORTED, diff --git a/src/filesystem/implementations/local.h b/src/filesystem/implementations/local.h index f6deeb5e6..bb416b0ab 100644 --- a/src/filesystem/implementations/local.h +++ b/src/filesystem/implementations/local.h @@ -57,7 +57,8 @@ class LocalFileSystem : public FileSystem { const std::string& path, const char* contents, const size_t content_len) override; Status MakeDirectory(const std::string& dir, const bool recursive) override; - Status MakeTemporaryDirectory(std::string* temp_dir) override; + Status MakeTemporaryDirectory( + std::string& dir_path, std::string* temp_dir) override; Status DeletePath(const std::string& path) override; }; @@ -280,7 +281,8 @@ LocalFileSystem::MakeDirectory(const std::string& dir, const bool recursive) } Status -LocalFileSystem::MakeTemporaryDirectory(std::string* temp_dir) +LocalFileSystem::MakeTemporaryDirectory( + std::string& dir_path, std::string* temp_dir) { #ifdef _WIN32 char temp_path[MAX_PATH + 1]; @@ -314,7 +316,10 @@ LocalFileSystem::MakeTemporaryDirectory(std::string* temp_dir) "Failed to create local temp folder: " + *temp_dir); } #else - std::string folder_template = "/tmp/folderXXXXXX"; + if (dir_path.empty()) { + dir_path = "/tmp"; + } + std::string folder_template = JoinPath({dir_path, "folderXXXXXX"}); char* res = mkdtemp(const_cast(folder_template.c_str())); if (res == nullptr) { return Status( diff --git a/src/filesystem/implementations/s3.h b/src/filesystem/implementations/s3.h index fe6da18c3..336e827a5 100644 --- a/src/filesystem/implementations/s3.h +++ b/src/filesystem/implementations/s3.h @@ -159,7 +159,8 @@ class S3FileSystem : public FileSystem { const std::string& path, const char* contents, const size_t content_len) override; Status MakeDirectory(const std::string& dir, const bool recursive) override; - Status MakeTemporaryDirectory(std::string* temp_dir) override; + Status MakeTemporaryDirectory( + std::string& dir_path, std::string* temp_dir) override; Status DeletePath(const std::string& path) override; private: @@ -652,10 +653,16 @@ S3FileSystem::LocalizePath( effective_path = path; } - // Create temporary directory + // Create a local directory for AWS model store. + // If ENV variable are not set, creates a temporary directory + // under `/tmp` with the format: "folderXXXXXX". + // Otherwise, will create a folder under specified directory with the same + // format. + std::string env_mount_dir = + GetEnvironmentVariableOrDefault("TRITON_AWS_MOUNT_DIRECTORY", "/tmp"); std::string tmp_folder; - RETURN_IF_ERROR( - triton::core::MakeTemporaryDirectory(FileSystemType::LOCAL, &tmp_folder)); + RETURN_IF_ERROR(triton::core::MakeTemporaryDirectory( + FileSystemType::LOCAL, std::string(env_mount_dir), &tmp_folder)); // Specify contents to be downloaded std::set contents; @@ -774,7 +781,8 @@ S3FileSystem::MakeDirectory(const std::string& dir, const bool recursive) } Status -S3FileSystem::MakeTemporaryDirectory(std::string* temp_dir) +S3FileSystem::MakeTemporaryDirectory( + std::string& dir_path, std::string* temp_dir) { return Status( Status::Code::UNSUPPORTED, From 16f9de74905b380cdf1bfa52dde5b4239201f750 Mon Sep 17 00:00:00 2001 From: oandreeva-nv Date: Mon, 18 Sep 2023 12:47:45 -0700 Subject: [PATCH 2/5] Follow up correction for env_mount_dir --- src/filesystem/implementations/as.h | 2 +- src/filesystem/implementations/gcs.h | 2 +- src/filesystem/implementations/s3.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/filesystem/implementations/as.h b/src/filesystem/implementations/as.h index 0d4d47f68..1928ea521 100644 --- a/src/filesystem/implementations/as.h +++ b/src/filesystem/implementations/as.h @@ -451,7 +451,7 @@ ASFileSystem::LocalizePath( GetEnvironmentVariableOrDefault("TRITON_AZURE_MOUNT_DIRECTORY", "/tmp"); std::string tmp_folder; RETURN_IF_ERROR(triton::core::MakeTemporaryDirectory( - FileSystemType::LOCAL, std::string(env_mount_dir), &tmp_folder)); + FileSystemType::LOCAL, env_mount_dir, &tmp_folder)); localized->reset(new LocalizedPath(path, tmp_folder)); diff --git a/src/filesystem/implementations/gcs.h b/src/filesystem/implementations/gcs.h index 8f9ad23c3..82a0fb259 100644 --- a/src/filesystem/implementations/gcs.h +++ b/src/filesystem/implementations/gcs.h @@ -386,7 +386,7 @@ GCSFileSystem::LocalizePath( GetEnvironmentVariableOrDefault("TRITON_GCS_MOUNT_DIRECTORY", "/tmp"); std::string tmp_folder; RETURN_IF_ERROR(triton::core::MakeTemporaryDirectory( - FileSystemType::LOCAL, std::string(env_mount_dir), &tmp_folder)); + FileSystemType::LOCAL, env_mount_dir, &tmp_folder)); localized->reset(new LocalizedPath(path, tmp_folder)); diff --git a/src/filesystem/implementations/s3.h b/src/filesystem/implementations/s3.h index 336e827a5..1786fe140 100644 --- a/src/filesystem/implementations/s3.h +++ b/src/filesystem/implementations/s3.h @@ -662,7 +662,7 @@ S3FileSystem::LocalizePath( GetEnvironmentVariableOrDefault("TRITON_AWS_MOUNT_DIRECTORY", "/tmp"); std::string tmp_folder; RETURN_IF_ERROR(triton::core::MakeTemporaryDirectory( - FileSystemType::LOCAL, std::string(env_mount_dir), &tmp_folder)); + FileSystemType::LOCAL, env_mount_dir, &tmp_folder)); // Specify contents to be downloaded std::set contents; From 998723190242ee14a51f774bc6383c4e4528996a Mon Sep 17 00:00:00 2001 From: oandreeva-nv Date: Mon, 18 Sep 2023 14:55:59 -0700 Subject: [PATCH 3/5] Fixing built issue, added const char --- src/filesystem/api.cc | 4 ++-- src/filesystem/implementations/as.h | 8 ++++---- src/filesystem/implementations/common.h | 3 +++ src/filesystem/implementations/gcs.h | 8 ++++---- src/filesystem/implementations/local.h | 4 ++-- src/filesystem/implementations/s3.h | 8 ++++---- 6 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/filesystem/api.cc b/src/filesystem/api.cc index b97826acb..2734eaf48 100644 --- a/src/filesystem/api.cc +++ b/src/filesystem/api.cc @@ -619,12 +619,12 @@ MakeTemporaryDirectory(const FileSystemType type, std::string* temp_dir) { std::shared_ptr fs; RETURN_IF_ERROR(fsm_.GetFileSystem(type, fs)); - return fs->MakeTemporaryDirectory("/tmp", temp_dir); + return fs->MakeTemporaryDirectory(kDefaultMountDirectory, temp_dir); } Status MakeTemporaryDirectory( - const FileSystemType type, std::string& dir_path, std::string* temp_dir) + const FileSystemType type, std::string dir_path, std::string* temp_dir) { std::shared_ptr fs; RETURN_IF_ERROR(fsm_.GetFileSystem(type, fs)); diff --git a/src/filesystem/implementations/as.h b/src/filesystem/implementations/as.h index 1928ea521..fc449475a 100644 --- a/src/filesystem/implementations/as.h +++ b/src/filesystem/implementations/as.h @@ -95,7 +95,7 @@ class ASFileSystem : public FileSystem { const size_t content_len) override; Status MakeDirectory(const std::string& dir, const bool recursive) override; Status MakeTemporaryDirectory( - std::string& dir_path, std::string* temp_dir) override; + std::string dir_path, std::string* temp_dir) override; Status DeletePath(const std::string& path) override; private: @@ -447,8 +447,8 @@ ASFileSystem::LocalizePath( // under `/tmp` with the format: "folderXXXXXX". // Otherwise, will create a folder under specified directory with the same // format. - std::string env_mount_dir = - GetEnvironmentVariableOrDefault("TRITON_AZURE_MOUNT_DIRECTORY", "/tmp"); + std::string env_mount_dir = GetEnvironmentVariableOrDefault( + "TRITON_AZURE_MOUNT_DIRECTORY", kDefaultMountDirectory); std::string tmp_folder; RETURN_IF_ERROR(triton::core::MakeTemporaryDirectory( FileSystemType::LOCAL, env_mount_dir, &tmp_folder)); @@ -500,7 +500,7 @@ ASFileSystem::MakeDirectory(const std::string& dir, const bool recursive) Status ASFileSystem::MakeTemporaryDirectory( - std::string& dir_path, std::string* temp_dir) + std::string dir_path, std::string* temp_dir) { return Status( Status::Code::UNSUPPORTED, diff --git a/src/filesystem/implementations/common.h b/src/filesystem/implementations/common.h index f7962357b..54afb284b 100644 --- a/src/filesystem/implementations/common.h +++ b/src/filesystem/implementations/common.h @@ -65,6 +65,9 @@ namespace triton { namespace core { +// Default folder for temporary local cache +constexpr char kDefaultMountDirectory[] = "/tmp"; + // FileSystem interface that all file system implementation should inherit from. // To add new file system support, the implementation should be added and made // visible to FileSystemManager in api.cc diff --git a/src/filesystem/implementations/gcs.h b/src/filesystem/implementations/gcs.h index 82a0fb259..0ffda004d 100644 --- a/src/filesystem/implementations/gcs.h +++ b/src/filesystem/implementations/gcs.h @@ -84,7 +84,7 @@ class GCSFileSystem : public FileSystem { const size_t content_len) override; Status MakeDirectory(const std::string& dir, const bool recursive) override; Status MakeTemporaryDirectory( - std::string& dir_path, std::string* temp_dir) override; + std::string dir_path, std::string* temp_dir) override; Status DeletePath(const std::string& path) override; private: @@ -382,8 +382,8 @@ GCSFileSystem::LocalizePath( } // Create a local directory for GCS model store. - std::string env_mount_dir = - GetEnvironmentVariableOrDefault("TRITON_GCS_MOUNT_DIRECTORY", "/tmp"); + std::string env_mount_dir = GetEnvironmentVariableOrDefault( + "TRITON_GCS_MOUNT_DIRECTORY", kDefaultMountDirectory); std::string tmp_folder; RETURN_IF_ERROR(triton::core::MakeTemporaryDirectory( FileSystemType::LOCAL, env_mount_dir, &tmp_folder)); @@ -485,7 +485,7 @@ GCSFileSystem::MakeDirectory(const std::string& dir, const bool recursive) Status GCSFileSystem::MakeTemporaryDirectory( - std::string& dir_path, std::string* temp_dir) + std::string dir_path, std::string* temp_dir) { return Status( Status::Code::UNSUPPORTED, diff --git a/src/filesystem/implementations/local.h b/src/filesystem/implementations/local.h index bb416b0ab..be22accc9 100644 --- a/src/filesystem/implementations/local.h +++ b/src/filesystem/implementations/local.h @@ -58,7 +58,7 @@ class LocalFileSystem : public FileSystem { const size_t content_len) override; Status MakeDirectory(const std::string& dir, const bool recursive) override; Status MakeTemporaryDirectory( - std::string& dir_path, std::string* temp_dir) override; + std::string dir_path, std::string* temp_dir) override; Status DeletePath(const std::string& path) override; }; @@ -282,7 +282,7 @@ LocalFileSystem::MakeDirectory(const std::string& dir, const bool recursive) Status LocalFileSystem::MakeTemporaryDirectory( - std::string& dir_path, std::string* temp_dir) + std::string dir_path, std::string* temp_dir) { #ifdef _WIN32 char temp_path[MAX_PATH + 1]; diff --git a/src/filesystem/implementations/s3.h b/src/filesystem/implementations/s3.h index 1786fe140..48e1cbc27 100644 --- a/src/filesystem/implementations/s3.h +++ b/src/filesystem/implementations/s3.h @@ -160,7 +160,7 @@ class S3FileSystem : public FileSystem { const size_t content_len) override; Status MakeDirectory(const std::string& dir, const bool recursive) override; Status MakeTemporaryDirectory( - std::string& dir_path, std::string* temp_dir) override; + std::string dir_path, std::string* temp_dir) override; Status DeletePath(const std::string& path) override; private: @@ -658,8 +658,8 @@ S3FileSystem::LocalizePath( // under `/tmp` with the format: "folderXXXXXX". // Otherwise, will create a folder under specified directory with the same // format. - std::string env_mount_dir = - GetEnvironmentVariableOrDefault("TRITON_AWS_MOUNT_DIRECTORY", "/tmp"); + std::string env_mount_dir = GetEnvironmentVariableOrDefault( + "TRITON_AWS_MOUNT_DIRECTORY", kDefaultMountDirectory); std::string tmp_folder; RETURN_IF_ERROR(triton::core::MakeTemporaryDirectory( FileSystemType::LOCAL, env_mount_dir, &tmp_folder)); @@ -782,7 +782,7 @@ S3FileSystem::MakeDirectory(const std::string& dir, const bool recursive) Status S3FileSystem::MakeTemporaryDirectory( - std::string& dir_path, std::string* temp_dir) + std::string dir_path, std::string* temp_dir) { return Status( Status::Code::UNSUPPORTED, From 244faf2ef507224422380e622ce5f985bc479fe7 Mon Sep 17 00:00:00 2001 From: oandreeva-nv Date: Mon, 18 Sep 2023 15:11:44 -0700 Subject: [PATCH 4/5] Synced remote and fixed build issue --- src/filesystem/api.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/filesystem/api.h b/src/filesystem/api.h index 36adca35a..8ee0b373e 100644 --- a/src/filesystem/api.h +++ b/src/filesystem/api.h @@ -205,7 +205,7 @@ Status MakeTemporaryDirectory(const FileSystemType type, std::string* temp_dir); /// \param temp_dir Returns the path to the temporary directory. /// \return Error status Status MakeTemporaryDirectory( - const FileSystemType type, std::string& dir_path, std::string* temp_dir); + const FileSystemType type, std::string dir_path, std::string* temp_dir); /// Delete a path. /// \param path The path to the directory or file. From 50b93d8a634a9f5f5758db5e11c2e3572ae92be6 Mon Sep 17 00:00:00 2001 From: oandreeva-nv Date: Mon, 18 Sep 2023 15:14:08 -0700 Subject: [PATCH 5/5] use specified const char in MakeTemporaryDirectory --- src/filesystem/implementations/local.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/filesystem/implementations/local.h b/src/filesystem/implementations/local.h index be22accc9..8e60054a6 100644 --- a/src/filesystem/implementations/local.h +++ b/src/filesystem/implementations/local.h @@ -317,7 +317,7 @@ LocalFileSystem::MakeTemporaryDirectory( } #else if (dir_path.empty()) { - dir_path = "/tmp"; + dir_path = kDefaultMountDirectory; } std::string folder_template = JoinPath({dir_path, "folderXXXXXX"}); char* res = mkdtemp(const_cast(folder_template.c_str()));