Add mount option { autoPersist: true } to IDBFS mount. (#21938)

* Add new setting -sIDBFS_AUTO_PERSIST which changes the semantics of an IndedexDB mount to automatically persist the VFS to IndexedDB after closing any file that has been written to. This enables users to avoid needing to call fsync() to persist files to the filesystem.

* Kick off persisting the filesystem on more operations. Fix test_idbfs_sync to properly delete IndexedDB state for the test if test fails.

* Update settings_reference.rst

* Propagate injected node_ops to the newly created child node

* Add [link] directive

* Move IDBFS_AUTO_PERSIST to a mount option rather than a -sSETTING.

* Address review.

* Add ChangeLog.md entry.

Author: juj

ChangeLog.md

@@ -23,6 +23,9 @@ See docs/process.md for more on how version tagging works.
 - The JSPI feature now uses the updated browser API for JSPI (available in
   Chrome v126+). To support older versions of Chrome use Emscripten version
   3.1.60 or earlier.
+- IDBFS mount has gained a new option { autoPersist: true }, which if passed,
+  changes the semantics of the IDBFS mount to automatically persist any changes
+  made to the filesystem. (#21938)
 
 3.1.60 - 05/20/24
 -----------------

site/source/docs/api_reference/Filesystem-API.rst

@@ -113,6 +113,8 @@ The *IDBFS* file system implements the :js:func:`FS.syncfs` interface, which whe
 
 This is provided to overcome the limitation that browsers do not offer synchronous APIs for persistent storage, and so (by default) all writes exist only temporarily in-memory.
 
+If the mount option `autoPersist: true` is passed when mounting IDBFS, then whenever any changes are made to the IDBFS directory tree, they will be automatically persisted to the IndexedDB backend. This lets users avoid needing to manually call `FS.syncfs` to persist changes to the IDBFS mounted directory tree.
+
 .. _filesystem-api-workerfs:
 
 WORKERFS

src/library_idbfs.js

@@ -23,8 +23,83 @@ addToLibrary({
     },
     DB_VERSION: 21,
     DB_STORE_NAME: 'FILE_DATA',
-    // reuse all of the core MEMFS functionality
-    mount: (...args) => MEMFS.mount(...args),
+
+    // Queues a new VFS -> IDBFS synchronization operation
+    queuePersist: (mount) => {
+      function onPersistComplete() {
+        if (mount.idbPersistState === 'again') startPersist(); // If a new sync request has appeared in between, kick off a new sync
+        else mount.idbPersistState = 0; // Otherwise reset sync state back to idle to wait for a new sync later
+      }
+      function startPersist() {
+        mount.idbPersistState = 'idb'; // Mark that we are currently running a sync operation
+        IDBFS.syncfs(mount, /*populate:*/false, onPersistComplete);
+      }
+
+      if (!mount.idbPersistState) {
+        // Programs typically write/copy/move multiple files in the in-memory
+        // filesystem within a single app frame, so when a filesystem sync
+        // command is triggered, do not start it immediately, but only after
+        // the current frame is finished. This way all the modified files
+        // inside the main loop tick will be batched up to the same sync.
+        mount.idbPersistState = setTimeout(startPersist, 0);
+      } else if (mount.idbPersistState === 'idb') {
+        // There is an active IndexedDB sync operation in-flight, but we now
+        // have accumulated more files to sync. We should therefore queue up
+        // a new sync after the current one finishes so that all writes
+        // will be properly persisted.
+        mount.idbPersistState = 'again';
+      }
+    },
+
+    mount: (mount) => {
+      // reuse core MEMFS functionality
+      var mnt = MEMFS.mount(mount);
+      // If the automatic IDBFS persistence option has been selected, then automatically persist
+      // all modifications to the filesystem as they occur.
+      if (mount?.opts?.autoPersist) {
+        mnt.idbPersistState = 0; // IndexedDB sync starts in idle state
+        var memfs_node_ops = mnt.node_ops;
+        mnt.node_ops = Object.assign({}, mnt.node_ops); // Clone node_ops to inject write tracking
+        mnt.node_ops.mknod = (parent, name, mode, dev) => {
+          var node = memfs_node_ops.mknod(parent, name, mode, dev);
+          // Propagate injected node_ops to the newly created child node
+          node.node_ops = mnt.node_ops;
+          // Remember for each IDBFS node which IDBFS mount point they came from so we know which mount to persist on modification.
+          node.idbfs_mount = mnt.mount;
+          // Remember original MEMFS stream_ops for this node
+          node.memfs_stream_ops = node.stream_ops;
+          // Clone stream_ops to inject write tracking
+          node.stream_ops = Object.assign({}, node.stream_ops);
+
+          // Track all file writes
+          node.stream_ops.write = (stream, buffer, offset, length, position, canOwn) => {
+            // This file has been modified, we must persist IndexedDB when this file closes
+            stream.node.isModified = true;
+            return node.memfs_stream_ops.write(stream, buffer, offset, length, position, canOwn);
+          };
+
+          // Persist IndexedDB on file close
+          node.stream_ops.close = (stream) => {
+            var n = stream.node;
+            if (n.isModified) {
+              IDBFS.queuePersist(n.idbfs_mount);
+              n.isModified = false;
+            }
+            if (n.memfs_stream_ops.close) return n.memfs_stream_ops.close(stream);
+          };
+
+          return node;
+        };
+        // Also kick off persisting the filesystem on other operations that modify the filesystem.
+        mnt.node_ops.mkdir   = (...args) => (IDBFS.queuePersist(mnt.mount), memfs_node_ops.mkdir(...args));
+        mnt.node_ops.rmdir   = (...args) => (IDBFS.queuePersist(mnt.mount), memfs_node_ops.rmdir(...args));
+        mnt.node_ops.symlink = (...args) => (IDBFS.queuePersist(mnt.mount), memfs_node_ops.symlink(...args));
+        mnt.node_ops.unlink  = (...args) => (IDBFS.queuePersist(mnt.mount), memfs_node_ops.unlink(...args));
+        mnt.node_ops.rename  = (...args) => (IDBFS.queuePersist(mnt.mount), memfs_node_ops.rename(...args));
+      }
+      return mnt;
+    },
+
     syncfs: (mount, populate, callback) => {
       IDBFS.getLocalSet(mount, (err, local) => {
         if (err) return callback(err);

test/fs/test_idbfs_sync.c

@@ -15,7 +15,7 @@
 
 int result = 1;
 
-void success() {
+void report_result() {
   REPORT_RESULT(result);
 #ifdef FORCE_EXIT
   emscripten_force_exit(0);
@@ -133,7 +133,18 @@ void test() {
 
 #endif
 
-#if EXTRA_WORK
+  // If the test failed, then delete test files from IndexedDB so that the test
+  // runner will not leak test state to subsequent tests that reuse this same
+  // file.
+  if (result != 1) {
+    unlink("/working1/empty.txt");
+    unlink("/working1/waka.txt");
+    unlink("/working1/moar.txt");
+    rmdir("/working1/dir");
+    EM_ASM(FS.syncfs(function(){})); // And persist deleted changes
+  }
+
+#if EXTRA_WORK && !FIRST
   EM_ASM(
     for (var i = 0; i < 100; i++) {
       FS.syncfs(function (err) {
@@ -144,24 +155,32 @@ void test() {
   );
 #endif
 
+#ifdef IDBFS_AUTO_PERSIST
+  report_result();
+#else
   // sync from memory state to persisted and then
-  // run 'success'
+  // run 'report_result'
   EM_ASM(
     // Ensure IndexedDB is closed at exit.
     Module['onExit'] = function() {
       assert(Object.keys(IDBFS.dbs).length == 0);
     };
     FS.syncfs(function (err) {
       assert(!err);
-      ccall('success', 'v');
+      ccall('report_result', 'v');
     });
   );
+#endif
 }
 
 int main() {
   EM_ASM(
     FS.mkdir('/working1');
-    FS.mount(IDBFS, {}, '/working1');
+    FS.mount(IDBFS, { 
+#ifdef IDBFS_AUTO_PERSIST
+      autoPersist: true
+#endif
+    }, '/working1');
 
 #if !FIRST
     // syncfs(true, f) should not break on already-existing directories:

test/test_browser.py

@@ -1354,18 +1354,14 @@ def test_fflush(self):
   @parameterized({
     '': ([],),
     'extra': (['-DEXTRA_WORK'],),
+    'autopersist': (['-DIDBFS_AUTO_PERSIST'],),
+    'force_exit': (['-sEXIT_RUNTIME', '-DFORCE_EXIT'],),
   })
-  def test_fs_idbfs_sync(self, extra):
+  def test_fs_idbfs_sync(self, args):
     self.set_setting('DEFAULT_LIBRARY_FUNCS_TO_INCLUDE', '$ccall')
     secret = str(time.time())
-    self.btest('fs/test_idbfs_sync.c', '1', args=['-lidbfs.js', '-DFIRST', f'-DSECRET="{secret}"', '-sEXPORTED_FUNCTIONS=_main,_test,_success', '-lidbfs.js'])
-    self.btest('fs/test_idbfs_sync.c', '1', args=['-lidbfs.js', f'-DSECRET="{secret }"', '-sEXPORTED_FUNCTIONS=_main,_test,_success', '-lidbfs.js'] + extra)
-
-  def test_fs_idbfs_sync_force_exit(self):
-    self.set_setting('DEFAULT_LIBRARY_FUNCS_TO_INCLUDE', '$ccall')
-    secret = str(time.time())
-    self.btest('fs/test_idbfs_sync.c', '1', args=['-lidbfs.js', '-DFIRST', f'-DSECRET="{secret}"', '-sEXPORTED_FUNCTIONS=_main,_test,_success', '-sEXIT_RUNTIME', '-DFORCE_EXIT', '-lidbfs.js'])
-    self.btest('fs/test_idbfs_sync.c', '1', args=['-lidbfs.js', f'-DSECRET="{secret }"', '-sEXPORTED_FUNCTIONS=_main,_test,_success', '-sEXIT_RUNTIME', '-DFORCE_EXIT', '-lidbfs.js'])
+    self.btest('fs/test_idbfs_sync.c', '1', args=['-lidbfs.js', f'-DSECRET="{secret}"', '-sEXPORTED_FUNCTIONS=_main,_test,_report_result', '-lidbfs.js'] + args + ['-DFIRST'])
+    self.btest('fs/test_idbfs_sync.c', '1', args=['-lidbfs.js', f'-DSECRET="{secret}"', '-sEXPORTED_FUNCTIONS=_main,_test,_report_result', '-lidbfs.js'] + args)
 
   def test_fs_idbfs_fsync(self):
     # sync from persisted state into memory before main()