add support for DMAPI xattr based GLACIER storage class

Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com>

add tests for tape info parsing

Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com>

remove auto-code formatting

Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com>

use querystring

Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com>

fix sending HTTP headers on all the status

Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com>

address PR comments

Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com>

address PR comments

Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com>

Separate out implicit restore status from implicit storage class

Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com>

update tests

Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com>

Author: tangledbytes

.gitignore

@@ -23,6 +23,8 @@ config-local.js
 *.sublime*
 .DS_Store
 heapdump-*
+.cache
+.clangd
 
 ## PRIVATE
 *.pem

config.js

@@ -880,6 +880,44 @@ config.NSFS_GLACIER_EXPIRY_TZ = 'LOCAL';
 // the request will be used
 config.NSFS_GLACIER_EXPIRY_TIME_OF_DAY = '';
 
+// If set to to true, NooBaa will attempt to read DMAPI
+// xattrs
+config.NSFS_GLACIER_DMAPI_ENABLE = false;
+
+// NSFS_GLACIER_DMAPI_IMPLICIT_RESTORE_STATUS if enabled then
+// NooBaa will derive restore status of the files based on DMAPI
+// xattr IF there are no explicit restore status attributes on
+// the file.
+config.NSFS_GLACIER_DMAPI_IMPLICIT_RESTORE_STATUS = false;
+
+// If set to true then NooBaa will consider DMAPI extended attributes
+// in conjuction with NooBaa's `user.storage_class` extended attribute
+// to determine state of an object.
+//
+// NOTE:NSFS_GLACIER_DMAPI_ENABLE should be enabled to use this.
+config.NSFS_GLACIER_DMAPI_IMPLICIT_SC = false;
+
+// NSFS_GLACIER_DMAPI_ALLOW_NOOBAA_TAKEOVER allows NooBaa to take over lifecycle
+// management of an object which was originally NOT managed by NooBaa.
+//
+// NOTE:NSFS_GLACIER_DMAPI_ENABLE and NSFS_GLACIER_USE_DMAPI should be enabled to use this.
+config.NSFS_GLACIER_DMAPI_ALLOW_NOOBAA_TAKEOVER = false;
+
+// NSFS_GLACIER_DMAPI_TPS_HTTP_HEADER_ENABLE if true will add additional HTTP headers
+// `config.NSFS_GLACIER_DMAPI_TPS_HTTP_HEADER` based on `dmapi.IBMTPS` EA.
+//
+// NOTE:NSFS_GLACIER_DMAPI_ENABLE should be enabled to use this.
+config.NSFS_GLACIER_DMAPI_TPS_HTTP_HEADER_ENABLE = false;
+config.NSFS_GLACIER_DMAPI_TPS_HTTP_HEADER = 'x-tape-meta-copy';
+
+// NSFS_GLACIER_DMAPI_PMIG_DAYS controls the "virtual"/fake expiry
+// days that will be shown if we detect a glacier object whose life-
+// cycle NSFS doesn't controls
+//
+// This is initialized to be the same as S3_RESTORE_REQUEST_MAX_DAYS
+// but can be overridden to any numberical value
+config.NSFS_GLACIER_DMAPI_PMIG_DAYS = config.S3_RESTORE_REQUEST_MAX_DAYS;
+
 config.NSFS_STATFS_CACHE_SIZE = 10000;
 config.NSFS_STATFS_CACHE_EXPIRY_MS = 1 * 1000;
 

src/endpoint/s3/s3_utils.js

@@ -2,6 +2,7 @@
 'use strict';
 
 const _ = require('lodash');
+const querystring = require('querystring');
 
 const dbg = require('../../util/debug_module')(__filename);
 const S3Error = require('./s3_errors').S3Error;
@@ -325,6 +326,13 @@ function set_response_object_md(res, object_md) {
 
         res.setHeader('x-amz-restore', restore);
     }
+    if (config.NSFS_GLACIER_DMAPI_TPS_HTTP_HEADER_ENABLE) {
+        object_md.restore_status?.tape_info?.forEach?.((meta, idx) => {
+            // @ts-ignore - For some TS check doesn't like "meta" being passed to querystring
+            const header = querystring.stringify(meta);
+            res.setHeader(`${config.NSFS_GLACIER_DMAPI_TPS_HTTP_HEADER}-${idx}`, header);
+        });
+    }
 }
 
 /** set_response_headers_get_object_attributes is based on set_response_object_md

src/manage_nsfs/manage_nsfs_glacier.js

@@ -5,8 +5,7 @@ const path = require('path');
 const { PersistentLogger } = require('../util/persistent_logger');
 const config = require('../../config');
 const nb_native = require('../util/nb_native');
-const { GlacierBackend } = require('../sdk/nsfs_glacier_backend/backend');
-const { getGlacierBackend } = require('../sdk/nsfs_glacier_backend/helper');
+const { Glacier } = require('../sdk/glacier');
 const native_fs_utils = require('../util/native_fs_utils');
 const { is_desired_time, record_current_time } = require('./manage_nsfs_cli_utils');
 
@@ -17,14 +16,14 @@ async function process_migrations() {
     const fs_context = native_fs_utils.get_process_fs_context();
     const lock_path = path.join(config.NSFS_GLACIER_LOGS_DIR, CLUSTER_LOCK);
     await native_fs_utils.lock_and_run(fs_context, lock_path, async () => {
-        const backend = getGlacierBackend();
+        const backend = Glacier.getBackend();
 
         if (
             await backend.low_free_space() ||
-            await time_exceeded(fs_context, config.NSFS_GLACIER_MIGRATE_INTERVAL, GlacierBackend.MIGRATE_TIMESTAMP_FILE)
+            await time_exceeded(fs_context, config.NSFS_GLACIER_MIGRATE_INTERVAL, Glacier.MIGRATE_TIMESTAMP_FILE)
         ) {
             await run_glacier_migrations(fs_context, backend);
-            const timestamp_file_path = path.join(config.NSFS_GLACIER_LOGS_DIR, GlacierBackend.MIGRATE_TIMESTAMP_FILE);
+            const timestamp_file_path = path.join(config.NSFS_GLACIER_LOGS_DIR, Glacier.MIGRATE_TIMESTAMP_FILE);
             await record_current_time(fs_context, timestamp_file_path);
         }
     });
@@ -34,46 +33,46 @@ async function process_migrations() {
  * run_tape_migrations reads the migration WALs and attempts to migrate the
  * files mentioned in the WAL.
  * @param {nb.NativeFSContext} fs_context
- * @param {import('../sdk/nsfs_glacier_backend/backend').GlacierBackend} backend
+ * @param {import('../sdk/glacier').Glacier} backend
  */
 async function run_glacier_migrations(fs_context, backend) {
-    await run_glacier_operation(fs_context, GlacierBackend.MIGRATE_WAL_NAME, backend.migrate.bind(backend));
+    await run_glacier_operation(fs_context, Glacier.MIGRATE_WAL_NAME, backend.migrate.bind(backend));
 }
 
 async function process_restores() {
     const fs_context = native_fs_utils.get_process_fs_context();
     const lock_path = path.join(config.NSFS_GLACIER_LOGS_DIR, CLUSTER_LOCK);
     await native_fs_utils.lock_and_run(fs_context, lock_path, async () => {
-        const backend = getGlacierBackend();
+        const backend = Glacier.getBackend();
 
         if (
             await backend.low_free_space() ||
-            !(await time_exceeded(fs_context, config.NSFS_GLACIER_RESTORE_INTERVAL, GlacierBackend.RESTORE_TIMESTAMP_FILE))
+            !(await time_exceeded(fs_context, config.NSFS_GLACIER_RESTORE_INTERVAL, Glacier.RESTORE_TIMESTAMP_FILE))
         ) return;
 
 
         await run_glacier_restore(fs_context, backend);
-        const timestamp_file_path = path.join(config.NSFS_GLACIER_LOGS_DIR, GlacierBackend.RESTORE_TIMESTAMP_FILE);
+        const timestamp_file_path = path.join(config.NSFS_GLACIER_LOGS_DIR, Glacier.RESTORE_TIMESTAMP_FILE);
         await record_current_time(fs_context, timestamp_file_path);
     });
 }
 
 /**
  * run_tape_restore reads the restore WALs and attempts to restore the
  * files mentioned in the WAL.
- * @param {nb.NativeFSContext} fs_context 
- * @param {import('../sdk/nsfs_glacier_backend/backend').GlacierBackend} backend
+ * @param {nb.NativeFSContext} fs_context
+ * @param {import('../sdk/glacier').Glacier} backend
  */
 async function run_glacier_restore(fs_context, backend) {
-    await run_glacier_operation(fs_context, GlacierBackend.RESTORE_WAL_NAME, backend.restore.bind(backend));
+    await run_glacier_operation(fs_context, Glacier.RESTORE_WAL_NAME, backend.restore.bind(backend));
 }
 
 async function process_expiry() {
     const fs_context = native_fs_utils.get_process_fs_context();
     const lock_path = path.join(config.NSFS_GLACIER_LOGS_DIR, SCAN_LOCK);
     await native_fs_utils.lock_and_run(fs_context, lock_path, async () => {
-        const backend = getGlacierBackend();
-        const timestamp_file_path = path.join(config.NSFS_GLACIER_LOGS_DIR, GlacierBackend.EXPIRY_TIMESTAMP_FILE);
+        const backend = Glacier.getBackend();
+        const timestamp_file_path = path.join(config.NSFS_GLACIER_LOGS_DIR, Glacier.EXPIRY_TIMESTAMP_FILE);
         if (
             await backend.low_free_space() ||
             await is_desired_time(
@@ -126,6 +125,8 @@ async function time_exceeded(fs_context, interval, timestamp_file) {
  */
 async function run_glacier_operation(fs_context, log_namespace, cb) {
     const log = new PersistentLogger(config.NSFS_GLACIER_LOGS_DIR, log_namespace, { locking: 'EXCLUSIVE' });
+
+    fs_context = prepare_galcier_fs_context(fs_context);
     try {
         await log.process(async (entry, failure_recorder) => cb(fs_context, entry, failure_recorder));
     } catch (error) {
@@ -135,6 +136,28 @@ async function run_glacier_operation(fs_context, log_namespace, cb) {
     }
 }
 
+/**
+ * prepare_galcier_fs_context returns a shallow copy of given
+ * fs_context with backend set to 'GPFS'.
+ *
+ * NOTE: The function will throw error if it detects that libgfs
+ * isn't loaded.
+ *
+ * @param {nb.NativeFSContext} fs_context
+ * @returns {nb.NativeFSContext}
+ */
+function prepare_galcier_fs_context(fs_context) {
+    if (config.NSFS_GLACIER_DMAPI_ENABLE) {
+        if (!nb_native().fs.gpfs) {
+            throw new Error('cannot use DMAPI xattrs: libgpfs not loaded');
+        }
+
+        return { ...fs_context, backend: 'GPFS', use_dmapi: true };
+    }
+
+    return { ...fs_context };
+}
+
 exports.process_migrations = process_migrations;
 exports.process_restores = process_restores;
 exports.process_expiry = process_expiry;

src/native/fs/fs_napi.cpp

@@ -46,6 +46,13 @@
 #define GPFS_XATTR_PREFIX "gpfs"
 #define GPFS_DOT_ENCRYPTION_EA "Encryption"
 #define GPFS_ENCRYPTION_XATTR_NAME GPFS_XATTR_PREFIX "." GPFS_DOT_ENCRYPTION_EA
+#define GPFS_DMAPI_XATTR_PREFIX "dmapi"
+#define GPFS_DMAPI_DOT_IBMOBJ_EA "IBMObj"
+#define GPFS_DMAPI_DOT_IBMPMIG_EA "IBMPMig"
+#define GPFS_DMAPI_DOT_IBMTPS_EA "IBMTPS"
+#define GPFS_DMAPI_XATTR_TAPE_INDICATOR GPFS_DMAPI_XATTR_PREFIX "." GPFS_DMAPI_DOT_IBMOBJ_EA
+#define GPFS_DMAPI_XATTR_TAPE_PREMIG GPFS_DMAPI_XATTR_PREFIX "." GPFS_DMAPI_DOT_IBMPMIG_EA
+#define GPFS_DMAPI_XATTR_TAPE_TPS GPFS_DMAPI_XATTR_PREFIX "." GPFS_DMAPI_DOT_IBMTPS_EA
 
 // This macro should be used after openning a file
 // it will autoclose the file using AutoCloser and will throw an error in case of failures
@@ -244,6 +251,11 @@ parse_open_flags(std::string flags)
 }
 
 const static std::vector<std::string> GPFS_XATTRS{ GPFS_ENCRYPTION_XATTR_NAME };
+const static std::vector<std::string> GPFS_DMAPI_XATTRS{
+    GPFS_DMAPI_XATTR_TAPE_INDICATOR,
+    GPFS_DMAPI_XATTR_TAPE_PREMIG,
+    GPFS_DMAPI_XATTR_TAPE_TPS,
+};
 const static std::vector<std::string> USER_XATTRS{
     "user.content_type",
     "user.content_md5",
@@ -461,9 +473,14 @@ get_fd_xattr(int fd, XattrMap& xattr, const std::vector<std::string>& xattr_keys
 }
 
 static int
-get_fd_gpfs_xattr(int fd, XattrMap& xattr, int& gpfs_error)
+get_fd_gpfs_xattr(int fd, XattrMap& xattr, int& gpfs_error, bool use_dmapi)
 {
-    for (auto const& key : GPFS_XATTRS) {
+    auto gpfs_xattrs { GPFS_XATTRS };
+    if (use_dmapi) {
+        gpfs_xattrs.insert(gpfs_xattrs.end(), GPFS_DMAPI_XATTRS.begin(), GPFS_DMAPI_XATTRS.end());
+    }
+
+    for (auto const& key : gpfs_xattrs) {
         gpfsRequest_t gpfsGetXattrRequest;
         build_gpfs_get_ea_request(&gpfsGetXattrRequest, key);
         int r = dlsym_gpfs_fcntl(fd, &gpfsGetXattrRequest);
@@ -603,6 +620,8 @@ struct FSWorker : public Napi::AsyncWorker
     // NOTE: If _do_ctime_check = false, then some functions will fallback to using mtime check
     bool _do_ctime_check;
 
+    bool _use_dmapi;
+
     FSWorker(const Napi::CallbackInfo& info)
         : AsyncWorker(info.Env())
         , _deferred(Napi::Promise::Deferred::New(info.Env()))
@@ -616,6 +635,7 @@ struct FSWorker : public Napi::AsyncWorker
         , _should_add_thread_capabilities(false)
         , _supplemental_groups()
         , _do_ctime_check(false)
+        , _use_dmapi(false)
     {
         for (int i = 0; i < (int)info.Length(); ++i) _args_ref.Set(i, info[i]);
         if (info[0].ToBoolean()) {
@@ -635,6 +655,7 @@ struct FSWorker : public Napi::AsyncWorker
                 _report_fs_stats = Napi::Persistent(fs_context.Get("report_fs_stats").As<Napi::Function>());
             }
             _do_ctime_check = fs_context.Get("do_ctime_check").ToBoolean();
+            _use_dmapi = fs_context.Get("use_dmapi").ToBoolean();
         }
     }
     void Begin(std::string desc)
@@ -798,7 +819,7 @@ struct Stat : public FSWorker
         if (!_use_lstat) {
             SYSCALL_OR_RETURN(get_fd_xattr(fd, _xattr, _xattr_get_keys));
             if (use_gpfs_lib()) {
-                GPFS_FCNTL_OR_RETURN(get_fd_gpfs_xattr(fd, _xattr, gpfs_error));
+                GPFS_FCNTL_OR_RETURN(get_fd_gpfs_xattr(fd, _xattr, gpfs_error, _use_dmapi));
             }
         }
 
@@ -1226,7 +1247,7 @@ struct Readfile : public FSWorker
         if (_read_xattr) {
             SYSCALL_OR_RETURN(get_fd_xattr(fd, _xattr, _xattr_get_keys));
             if (use_gpfs_lib()) {
-                GPFS_FCNTL_OR_RETURN(get_fd_gpfs_xattr(fd, _xattr, gpfs_error));
+                GPFS_FCNTL_OR_RETURN(get_fd_gpfs_xattr(fd, _xattr, gpfs_error, _use_dmapi));
             }
         }
 
@@ -1757,7 +1778,7 @@ struct FileStat : public FSWrapWorker<FileWrap>
         SYSCALL_OR_RETURN(fstat(fd, &_stat_res));
         SYSCALL_OR_RETURN(get_fd_xattr(fd, _xattr, _xattr_get_keys));
         if (use_gpfs_lib()) {
-            GPFS_FCNTL_OR_RETURN(get_fd_gpfs_xattr(fd, _xattr, gpfs_error));
+            GPFS_FCNTL_OR_RETURN(get_fd_gpfs_xattr(fd, _xattr, gpfs_error, _use_dmapi));
         }
 
         if (_do_ctime_check) {