WasmFS JS API: Implement mkdev (#19978)

This PR implements FS.mkdev and it's helpers (registerDevice,
makedev, major, and minor). Existing tests were modified to
pass with WasmFS.

In syscalls.cpp also fix a minor bug with poll to get the test passing.

Author: jameshu15869

.circleci/config.yml

@@ -547,6 +547,10 @@ jobs:
             wasmfs.test_readdir
             wasmfs.test_readdir_unlink
             wasmfs.test_unistd_pipe
+            wasmfs.test_unistd_io
+            wasmfs.test_unistd_curdir
+            wasmfs.test_poll
+            wasmfs.test_fs_64bit
             wasmfs.test_fs_write
             wasmfs.test_fs_writev
             wasmfs.test_fs_writev_rawfs

emcc.py

@@ -2358,6 +2358,7 @@ def phase_linker_setup(options, state, newargs):
       # included, as the entire JS library can refer to things that require
       # these exports.)
       settings.REQUIRED_EXPORTS += [
+        'wasmfs_create_file',
         '_wasmfs_mount',
         '_wasmfs_unmount',
         '_wasmfs_read_file',

src/library_wasmfs.js

@@ -18,6 +18,8 @@ addToLibrary({
   // data needs to be preloaded (and it would be invalid to do so, as any
   // further additions to wasmFSPreloadedFiles|Dirs would be ignored).
   $wasmFSPreloadingFlushed: false,
+  $wasmFSDevices: {},
+  $wasmFSDeviceStreams: {},
 
   $FS__postset: `
 FS.init();
@@ -59,6 +61,10 @@ FS.createPreloadedFile = FS_createPreloadedFile;
 #endif
     'malloc',
     'free',
+    'wasmfs_create_jsimpl_backend',
+    '$wasmFS$backends',
+    '$wasmFSDevices',
+    '$wasmFSDeviceStreams'
 #endif
   ],
   $FS : {
@@ -392,7 +398,104 @@ FS.createPreloadedFile = FS_createPreloadedFile;
         return __wasmfs_mknod(pathBuffer, mode, dev);
       }));
     },
-    // TODO: mkdev
+    makedev: (ma, mi) => ((ma) << 8 | (mi)),
+    registerDevice(dev, ops) {
+      var backendPointer = _wasmfs_create_jsimpl_backend();
+      var definedOps = {
+        userRead: ops.read,
+        userWrite: ops.write,
+
+        allocFile: (file) => {
+          wasmFSDeviceStreams[file] = {}
+        },
+        freeFile: (file) => {
+          wasmFSDeviceStreams[file] = undefined;
+        },
+        getSize: (file) => {},
+        read: (file, buffer, length, offset) => {
+          var bufferArray = Module.HEAP8.subarray(buffer, buffer + length);
+          try {
+            var bytesRead = definedOps.userRead(wasmFSDeviceStreams[file], bufferArray, 0, length, offset);
+          } catch (e) {
+            return -e.errno;
+          }
+          Module.HEAP8.set(bufferArray, buffer);
+          return bytesRead;
+        },
+        write: (file, buffer, length, offset) => {
+          var bufferArray = Module.HEAP8.subarray(buffer, buffer + length);
+          try {
+            var bytesWritten = definedOps.userWrite(wasmFSDeviceStreams[file], bufferArray, 0, length, offset);
+          } catch (e) {
+            return -e.errno;
+          }
+          Module.HEAP8.set(bufferArray, buffer);
+          return bytesWritten;       
+        },
+      };
+
+      wasmFS$backends[backendPointer] = definedOps;
+      wasmFSDevices[dev] = backendPointer;
+    },
+    createDevice(parent, name, input, output) {
+      if (typeof parent != 'string') {
+        // The old API allowed parents to be objects, which do not exist in WasmFS.
+        throw new Error("Only string paths are accepted");
+      }
+      var path = PATH.join2(parent, name);
+      var mode = FS_getMode(!!input, !!output);
+      if (!FS.createDevice.major) FS.createDevice.major = 64;
+      var dev = FS.makedev(FS.createDevice.major++, 0);
+      // Create a fake device with a set of stream ops to emulate
+      // the old API's createDevice().
+      FS.registerDevice(dev, {
+        read(stream, buffer, offset, length, pos /* ignored */) {
+          var bytesRead = 0;
+          for (var i = 0; i < length; i++) {
+            var result;
+            try {
+              result = input();
+            } catch (e) {
+              throw new FS.ErrnoError({{{ cDefs.EIO }}});
+            }
+            if (result === undefined && bytesRead === 0) {
+              throw new FS.ErrnoError({{{ cDefs.EAGAIN }}});
+            }
+            if (result === null || result === undefined) break;
+            bytesRead++;
+            buffer[offset+i] = result;
+          }
+          return bytesRead;
+        },
+        write(stream, buffer, offset, length, pos) {
+          for (var i = 0; i < length; i++) {
+            try {
+              output(buffer[offset+i]);
+            } catch (e) {
+              throw new FS.ErrnoError({{{ cDefs.EIO }}});
+            }
+          }
+          return i;
+        }
+      });
+      return FS.mkdev(path, mode, dev);
+    },
+    // mode is an optional argument, which will be set to 0666 if not passed in.
+    mkdev(path, mode, dev) {
+      if (typeof dev === 'undefined') {
+        dev = mode;
+        mode = 438 /* 0666 */;
+      }
+
+      var deviceBackend = wasmFSDevices[dev];
+      if (!deviceBackend) {
+        throw new Error("Invalid device ID.");
+      }
+
+      return FS.handleError(withStackSave(() => (
+        _wasmfs_create_file(stringToUTF8OnStack(path), mode, deviceBackend)
+      )));
+    },
     rename(oldPath, newPath) {
       return FS.handleError(withStackSave(() => {
         var oldPathBuffer = stringToUTF8OnStack(oldPath);

src/library_wasmfs_jsimpl.js

@@ -29,6 +29,9 @@ addToLibrary({
 #if ASSERTIONS
     assert(wasmFS$backends[backend]);
 #endif
+    if (!wasmFS$backends[backend].write) {
+      return -{{{ cDefs.EINVAL }}};
+    }
     return wasmFS$backends[backend].write(file, buffer, length, offset);
   },
 
@@ -37,6 +40,9 @@ addToLibrary({
 #if ASSERTIONS
     assert(wasmFS$backends[backend]);
 #endif
+    if (!wasmFS$backends[backend].read) {
+      return -{{{ cDefs.EINVAL }}};
+    }
     return wasmFS$backends[backend].read(file, buffer, length, offset);
   },
 

system/include/emscripten/wasmfs.h

@@ -72,6 +72,9 @@ backend_t wasmfs_create_node_backend(const char* root __attribute__((nonnull)));
 // thread.
 backend_t wasmfs_create_opfs_backend(void);
 
+// Creates a generic JSIMPL backend in the new file system.
+backend_t wasmfs_create_jsimpl_backend(void);
+
 backend_t wasmfs_create_icase_backend(backend_t backend);
 
 // Similar to fflush(0), but also flushes all internal buffers inside WasmFS.

system/lib/wasmfs/js_impl_backend.h

@@ -125,4 +125,12 @@ class JSImplBackend : public Backend {
   }
 };
 
+extern "C" {
+
+backend_t wasmfs_create_jsimpl_backend(void) {
+  return wasmFS.addBackend(std::make_unique<JSImplBackend>());
+}
+
+} // extern "C"
+
 } // namespace wasmfs

system/lib/wasmfs/syscalls.cpp

@@ -1379,12 +1379,13 @@ int __syscall_poll(intptr_t fds_, int nfds, int timeout) {
     if (openFile) {
       mask = 0;
       auto flags = openFile->locked().getFlags();
+      auto accessMode = flags & O_ACCMODE;
       auto readBit = pollfd->events & POLLOUT;
-      if (readBit && (flags == O_WRONLY || flags == O_RDWR)) {
+      if (readBit && (accessMode == O_WRONLY || accessMode == O_RDWR)) {
         mask |= readBit;
       }
       auto writeBit = pollfd->events & POLLIN;
-      if (writeBit && (flags == O_RDONLY || flags == O_RDWR)) {
+      if (writeBit && (accessMode == O_RDONLY || accessMode == O_RDWR)) {
         // If there is data in the file, then there is also the ability to read.
         // TODO: Does this need to consider the position as well? That is, if
         // the position is at the end, we can't read from the current position

test/core/test_poll.c

@@ -9,14 +9,23 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <poll.h>
+#include <emscripten.h>
 
 int main() {
+  EM_ASM(
+    var dummy_device = FS.makedev(64, 0);
+    FS.registerDevice(dummy_device, {});
+
+    FS.createDataFile('/', 'file', 'abcdef', true, true, false);
+    FS.mkdev('/device', dummy_device);
+  );
+  
   struct pollfd multi[5];
-  multi[0].fd = open("/file", O_RDONLY, 0777);
-  multi[1].fd = open("/device", O_RDONLY, 0777);
+  multi[0].fd = open("/file", O_RDWR, 0777);
+  multi[1].fd = open("/device", O_RDWR, 0777);
   multi[2].fd = 123;
-  multi[3].fd = open("/file", O_RDONLY, 0777);
-  multi[4].fd = open("/file", O_RDONLY, 0777);
+  multi[3].fd = open("/file", O_RDWR, 0777);
+  multi[4].fd = open("/file", O_RDWR, 0777);
   multi[0].events = POLLIN | POLLOUT | POLLNVAL | POLLERR;
   multi[1].events = POLLIN | POLLOUT | POLLNVAL | POLLERR;
   multi[2].events = POLLIN | POLLOUT | POLLNVAL | POLLERR;
@@ -26,7 +35,13 @@ int main() {
   printf("ret: %d\n", poll(multi, 5, 123));
   printf("errno: %d\n", errno);
   printf("multi[0].revents: %d\n", multi[0].revents == (POLLIN | POLLOUT));
+#if WASMFS
+  // TODO: Add support for POLLIN. The size of the device is 0 in WasmFS, so 
+  // devices cannot POLLIN.
+  printf("multi[1].revents: %d\n", multi[1].revents == POLLOUT);
+#else
   printf("multi[1].revents: %d\n", multi[1].revents == (POLLIN | POLLOUT));
+#endif
   printf("multi[2].revents: %d\n", multi[2].revents == POLLNVAL);
   printf("multi[3].revents: %d\n", multi[3].revents == 0);
   printf("multi[4].revents: %d\n", multi[4].revents == POLLOUT);

test/fs/test_64bit.c

@@ -8,15 +8,14 @@
 #include <emscripten.h>
 #include <stdio.h>
 #include <assert.h>
+#include <errno.h>
 
 int main(int argc, char *argv[])
 {
   EM_ASM({
     var counter = FS.makedev(64, 0);
 
     FS.registerDevice(counter, {
-      open: function(stream) {},
-      close: function(stream) {},
       read: function(stream, buffer, offset, length, position) {
         for (var i = 0; i < length; ++i) {
           buffer[offset + i] = position + i;

test/test_core.py

@@ -5808,13 +5808,8 @@ def test_fcntl_misc(self):
     self.do_run_in_out_file_test('fcntl/test_fcntl_misc.c')
 
   def test_poll(self):
-    self.add_pre_run('''
-      var dummy_device = FS.makedev(64, 0);
-      FS.registerDevice(dummy_device, {});
-
-      FS.createDataFile('/', 'file', 'abcdef', true, true, false);
-      FS.mkdev('/device', dummy_device);
-    ''')
+    if self.get_setting('WASMFS'):
+      self.set_setting('FORCE_FILESYSTEM')
     self.do_core_test('test_poll.c')
 
   def test_statvfs(self):
@@ -6079,6 +6074,8 @@ def test_fs_writev(self):
     self.do_runf(test_file('fs/test_writev.c'), 'success')
 
   def test_fs_64bit(self):
+    if self.get_setting('WASMFS'):
+      self.set_setting('FORCE_FILESYSTEM')
     self.do_runf(test_file('fs/test_64bit.c'), 'success')
 
   def test_sigalrm(self):
@@ -6113,6 +6110,8 @@ def test_unistd_access(self):
 
   def test_unistd_curdir(self):
     self.uses_es6 = True
+    if self.get_setting('WASMFS'):
+      self.set_setting('FORCE_FILESYSTEM')
     self.do_run_in_out_file_test('unistd/curdir.c')
 
   @also_with_noderawfs