add support for expiring objects on GET and run

migrate if migrate log size exceeds.

Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com>

Author: tangledbytes

config.js

@@ -946,6 +946,16 @@ config.NSFS_LOW_FREE_SPACE_MB_UNLEASH = 10 * 1024;
 // operations safely.
 config.NSFS_LOW_FREE_SPACE_PERCENT_UNLEASH = 0.10;
 
+// NSFS_GLACIER_GET_FORCE_EXPIRE if set to true then any restored item in the GLACIER
+// storage class will expire as soon as first GET request is received for it or
+// if the previous restore time has exceed, whichever is the earlier.
+config.NSFS_GLACIER_FORCE_EXPIRE_ON_GET = false;
+
+// NSFS_GLACIER_MIGRATE_LOG_THRESHOLD controls that how big the migration log file should be
+// Once this size is exceeded, migrate calls are supposed to kick in regardless of configured
+// interval
+config.NSFS_GLACIER_MIGRATE_LOG_THRESHOLD = 50 * 1024;
+
 // anonymous account name
 config.ANONYMOUS_ACCOUNT_NAME = 'anonymous';
 

src/manage_nsfs/manage_nsfs_glacier.js

@@ -20,7 +20,8 @@ async function process_migrations() {
 
         if (
             await backend.low_free_space() ||
-            await time_exceeded(fs_context, config.NSFS_GLACIER_MIGRATE_INTERVAL, Glacier.MIGRATE_TIMESTAMP_FILE)
+            await time_exceeded(fs_context, config.NSFS_GLACIER_MIGRATE_INTERVAL, Glacier.MIGRATE_TIMESTAMP_FILE) ||
+            await migrate_log_exceeds_threshold()
         ) {
             await run_glacier_migrations(fs_context, backend);
             const timestamp_file_path = path.join(config.NSFS_GLACIER_LOGS_DIR, Glacier.MIGRATE_TIMESTAMP_FILE);
@@ -115,6 +116,20 @@ async function time_exceeded(fs_context, interval, timestamp_file) {
     return false;
 }
 
+/**
+ * migrate_log_exceeds_threshold returns true if the underlying backend
+ * decides that the migrate log size has exceeded the given size threshold.
+ * @param {number} [threshold]
+ * @returns {Promise<boolean>}
+ */
+async function migrate_log_exceeds_threshold(threshold = config.NSFS_GLACIER_MIGRATE_LOG_THRESHOLD) {
+    const log = new PersistentLogger(config.NSFS_GLACIER_LOGS_DIR, Glacier.MIGRATE_WAL_NAME, { locking: null });
+    await log._open();
+
+    const { size } = await log.fh.stat(log.fs_context);
+    return size > threshold;
+}
+
 /**
  * run_glacier_operations takes a log_namespace and a callback and executes the
  * callback on each log file in that namespace. It will also generate a failure

src/sdk/namespace_fs.js

@@ -1151,6 +1151,8 @@ class NamespaceFS {
                 }
             }
 
+            await this._glacier_force_expire_on_get(fs_context, file_path, file, stat);
+
             await file.close(fs_context);
             file = null;
             object_sdk.throw_if_aborted();
@@ -3539,6 +3541,30 @@ class NamespaceFS {
         return Math.max(in_bytes, free_from_percentage);
     }
 
+    /**
+     * _glacier_force_expire_on_get expires a object if the object has storage
+     * class set to GLACIER and NooBaa is configured for forced get based
+     * eviction
+     * @param {nb.NativeFSContext} fs_context
+     * @param {string} file_path 
+     * @param {nb.NativeFile} file 
+     * @param {nb.NativeFSStats} stat
+     */
+    async _glacier_force_expire_on_get(fs_context, file_path, file, stat) {
+        if (!config.NSFS_GLACIER_FORCE_EXPIRE_ON_GET) return;
+
+        const storage_class = s3_utils.parse_storage_class(stat.xattr[Glacier.STORAGE_CLASS_XATTR]);
+        if (storage_class !== s3_utils.STORAGE_CLASS_GLACIER) return;
+
+        // Remove all the restore related xattrs
+        await file.replacexattr(fs_context, {
+            // Set date to 1970-01-01 to force expiry
+            [Glacier.XATTR_RESTORE_EXPIRY]: new Date(0).toISOString()
+        }, Glacier.XATTR_RESTORE_REQUEST);
+
+        await this.append_to_migrate_wal(file_path);
+    }
+
     async append_to_migrate_wal(entry) {
         if (!config.NSFS_GLACIER_LOGS_ENABLED) return;
 

src/util/file_reader.js

@@ -17,8 +17,12 @@ class NewlineReaderFilePathEntry {
 
 class NewlineReader {
     /**
-     * NewlineReader allows to read a file line by line while at max holding one line + 4096 bytes
-     * in memory.
+     * Newline character code
+     */
+    static NL_CODE = 10;
+
+    /**
+     * NewlineReader allows to read a file line by line.
      * @param {nb.NativeFSContext} fs_context 
      * @param {string} filepath 
      * @param {{
@@ -66,7 +70,7 @@ class NewlineReader {
         while (!this.eof) {
             // extract next line if terminated in current buffer
             if (this.start < this.end) {
-                const term_idx = this.buf.subarray(this.start, this.end).indexOf(10);
+                const term_idx = this.buf.subarray(this.start, this.end).indexOf(NewlineReader.NL_CODE);
                 if (term_idx >= 0) {
                     if (this.overflow_state) {
                         console.warn('line too long finally terminated:', this.info());