[Wasm64] Better support + testing for fetch API (#19276)

Author: sbc100

src/Fetch.js

@@ -241,17 +241,17 @@ function fetchXHR(fetch, onsuccess, onerror, onprogress, onreadystatechange) {
   var fetch_attr = fetch + {{{ C_STRUCTS.emscripten_fetch_t.__attributes }}};
   var requestMethod = UTF8ToString(fetch_attr);
   if (!requestMethod) requestMethod = 'GET';
-  var userData = HEAPU32[fetch + {{{ C_STRUCTS.emscripten_fetch_t.userData }}} >> 2];
-  var fetchAttributes = HEAPU32[fetch_attr + {{{ C_STRUCTS.emscripten_fetch_attr_t.attributes }}} >> 2];
-  var timeoutMsecs = HEAPU32[fetch_attr + {{{ C_STRUCTS.emscripten_fetch_attr_t.timeoutMSecs }}} >> 2];
-  var withCredentials = !!HEAPU32[fetch_attr + {{{ C_STRUCTS.emscripten_fetch_attr_t.withCredentials }}} >> 2];
-  var destinationPath = HEAPU32[fetch_attr + {{{ C_STRUCTS.emscripten_fetch_attr_t.destinationPath }}} >> 2];
-  var userName = HEAPU32[fetch_attr + {{{ C_STRUCTS.emscripten_fetch_attr_t.userName }}} >> 2];
-  var password = HEAPU32[fetch_attr + {{{ C_STRUCTS.emscripten_fetch_attr_t.password }}} >> 2];
-  var requestHeaders = HEAPU32[fetch_attr + {{{ C_STRUCTS.emscripten_fetch_attr_t.requestHeaders }}} >> 2];
-  var overriddenMimeType = HEAPU32[fetch_attr + {{{ C_STRUCTS.emscripten_fetch_attr_t.overriddenMimeType }}} >> 2];
-  var dataPtr = HEAPU32[fetch_attr + {{{ C_STRUCTS.emscripten_fetch_attr_t.requestData }}} >> 2];
-  var dataLength = HEAPU32[fetch_attr + {{{ C_STRUCTS.emscripten_fetch_attr_t.requestDataSize }}} >> 2];
+  var userData = {{{ makeGetValue('fetch_attr', C_STRUCTS.emscripten_fetch_t.userData, '*') }}};
+  var fetchAttributes = {{{ makeGetValue('fetch_attr', C_STRUCTS.emscripten_fetch_attr_t.attributes, 'u32') }}};
+  var timeoutMsecs = {{{ makeGetValue('fetch_attr', C_STRUCTS.emscripten_fetch_attr_t.timeoutMSecs, 'u32') }}};
+  var withCredentials = !!{{{ makeGetValue('fetch_attr', C_STRUCTS.emscripten_fetch_attr_t.withCredentials, 'u8') }}};
+  var destinationPath = {{{ makeGetValue('fetch_attr', C_STRUCTS.emscripten_fetch_attr_t.destinationPath, '*') }}};
+  var userName = {{{ makeGetValue('fetch_attr', C_STRUCTS.emscripten_fetch_attr_t.userName, '*') }}};
+  var password = {{{ makeGetValue('fetch_attr', C_STRUCTS.emscripten_fetch_attr_t.password, '*') }}};
+  var requestHeaders = {{{ makeGetValue('fetch_attr', C_STRUCTS.emscripten_fetch_attr_t.requestHeaders, '*') }}};
+  var overriddenMimeType = {{{ makeGetValue('fetch_attr', C_STRUCTS.emscripten_fetch_attr_t.overriddenMimeType, '*') }}};
+  var dataPtr = {{{ makeGetValue('fetch_attr', C_STRUCTS.emscripten_fetch_attr_t.requestData, '*') }}};
+  var dataLength = {{{ makeGetValue('fetch_attr', C_STRUCTS.emscripten_fetch_attr_t.requestDataSize, '*') }}};
 
   var fetchAttrLoadToMemory = !!(fetchAttributes & {{{ cDefs.EMSCRIPTEN_FETCH_LOAD_TO_MEMORY }}});
   var fetchAttrStreamData = !!(fetchAttributes & {{{ cDefs.EMSCRIPTEN_FETCH_STREAM_DATA }}});
@@ -289,11 +289,11 @@ function fetchXHR(fetch, onsuccess, onerror, onprogress, onreadystatechange) {
   }
   if (requestHeaders) {
     for (;;) {
-      var key = HEAPU32[requestHeaders >> 2];
+      var key = {{{ makeGetValue('requestHeaders', 0, '*') }}};
       if (!key) break;
-      var value = HEAPU32[requestHeaders + 4 >> 2];
+      var value = {{{ makeGetValue('requestHeaders', POINTER_SIZE, '*') }}};
       if (!value) break;
-      requestHeaders += 8;
+      requestHeaders += {{{ 2 * POINTER_SIZE }}};
       var keyStr = UTF8ToString(key);
       var valueStr = UTF8ToString(value);
 #if FETCH_DEBUG
@@ -302,7 +302,7 @@ function fetchXHR(fetch, onsuccess, onerror, onprogress, onreadystatechange) {
       xhr.setRequestHeader(keyStr, valueStr);
     }
   }
-  var id = HEAPU32[fetch + {{{ C_STRUCTS.emscripten_fetch_t.id }}} >> 2];
+  var id = {{{ makeGetValue('fetch', C_STRUCTS.emscripten_fetch_t.id, 'u32') }}};
   Fetch.xhrs[id] = xhr;
   var data = (dataPtr && dataLength) ? HEAPU8.slice(dataPtr, dataPtr + dataLength) : null;
   // TODO: Support specifying custom headers to the request.

src/generated_struct_info64.json

@@ -1336,28 +1336,28 @@
             "value": 8
         },
         "emscripten_fetch_attr_t": {
-            "__size__": 152,
+            "__size__": 144,
             "attributes": 72,
-            "destinationPath": 96,
+            "destinationPath": 88,
             "onerror": 48,
             "onprogress": 56,
             "onreadystatechange": 64,
             "onsuccess": 40,
-            "overriddenMimeType": 128,
-            "password": 112,
-            "requestData": 136,
-            "requestDataSize": 144,
-            "requestHeaders": 120,
+            "overriddenMimeType": 120,
+            "password": 104,
+            "requestData": 128,
+            "requestDataSize": 136,
+            "requestHeaders": 112,
             "requestMethod": 0,
-            "timeoutMSecs": 80,
+            "timeoutMSecs": 76,
             "userData": 32,
-            "userName": 104,
-            "withCredentials": 88
+            "userName": 96,
+            "withCredentials": 80
         },
         "emscripten_fetch_t": {
             "__attributes": 128,
             "__proxyState": 124,
-            "__size__": 280,
+            "__size__": 272,
             "data": 24,
             "dataOffset": 40,
             "id": 0,

system/include/emscripten/fetch.h

@@ -85,7 +85,7 @@ typedef struct emscripten_fetch_attr_t {
 
   // Specifies the amount of time the request can take before failing due to a
   // timeout.
-  unsigned long timeoutMSecs;
+  uint32_t timeoutMSecs;
 
   // Indicates whether cross-site access control requests should be made using
   // credentials.

test/test_browser.py

@@ -124,6 +124,26 @@ def metafunc(self, with_wasm64):
   return metafunc
 
 
+def also_with_wasm2js_or_wasm64(f):
+  assert callable(f)
+
+  def metafunc(self, with_wasm2js, with_wasm64):
+    if with_wasm2js:
+      self.set_setting('WASM', 0)
+      f(self)
+    elif with_wasm64:
+      self.set_setting('MEMORY64')
+      self.emcc_args.append('-Wno-experimental')
+      f(self)
+    else:
+      f(self)
+
+  metafunc._parameterize = {'': (False, False),
+                            'wasm2js': (True, False),
+                            'wasm64': (False, True)}
+  return metafunc
+
+
 def shell_with_script(shell_file, output_file, replacement):
   shell = read_file(path_from_root('src', shell_file))
   create_file(output_file, shell.replace('{{{ SCRIPT }}}', replacement))
@@ -4602,7 +4622,7 @@ def test_preallocated_heap(self):
     self.btest_exit('test_preallocated_heap.cpp', args=['-sWASM=0', '-sINITIAL_MEMORY=16MB', '-sABORTING_MALLOC=0', '--shell-file', test_file('test_preallocated_heap_shell.html')])
 
   # Tests emscripten_fetch() usage to XHR data directly to memory without persisting results to IndexedDB.
-  @also_with_wasm2js
+  @also_with_wasm2js_or_wasm64
   def test_fetch_to_memory(self):
     # Test error reporting in the negative case when the file URL doesn't exist. (http 404)
     self.btest_exit('fetch/to_memory.cpp',
@@ -4626,29 +4646,29 @@ def test_fetch_from_thread(self, args):
                     args=args + ['-pthread', '-sPROXY_TO_PTHREAD', '-sFETCH_DEBUG', '-sFETCH', '-DFILE_DOES_NOT_EXIST'],
                     also_wasm2js=True)
 
-  @also_with_wasm2js
+  @also_with_wasm2js_or_wasm64
   def test_fetch_to_indexdb(self):
     shutil.copyfile(test_file('gears.png'), 'gears.png')
     self.btest_exit('fetch/to_indexeddb.cpp',
                     args=['-sFETCH_DEBUG', '-sFETCH'])
 
   # Tests emscripten_fetch() usage to persist an XHR into IndexedDB and subsequently load up from there.
-  @also_with_wasm2js
+  @also_with_wasm2js_or_wasm64
   def test_fetch_cached_xhr(self):
     shutil.copyfile(test_file('gears.png'), 'gears.png')
     self.btest_exit('fetch/cached_xhr.cpp',
                     args=['-sFETCH_DEBUG', '-sFETCH'])
 
   # Tests that response headers get set on emscripten_fetch_t values.
   @no_firefox('https://github.com/emscripten-core/emscripten/issues/16868')
-  @also_with_wasm2js
+  @also_with_wasm2js_or_wasm64
   @requires_threads
   def test_fetch_response_headers(self):
     shutil.copyfile(test_file('gears.png'), 'gears.png')
     self.btest_exit('fetch/response_headers.cpp', args=['-sFETCH_DEBUG', '-sFETCH', '-pthread', '-sPROXY_TO_PTHREAD'])
 
   # Test emscripten_fetch() usage to stream a XHR in to memory without storing the full file in memory
-  @also_with_wasm2js
+  @also_with_wasm2js_or_wasm64
   def test_fetch_stream_file(self):
     self.skipTest('moz-chunked-arraybuffer was firefox-only and has been removed')
     # Strategy: create a large 128MB file, and compile with a small 16MB Emscripten heap, so that the tested file
@@ -4674,6 +4694,7 @@ def test_fetch_xhr_abort(self):
   # Tests emscripten_fetch() usage in synchronous mode when used from the main
   # thread proxied to a Worker with -sPROXY_TO_PTHREAD option.
   @no_firefox('https://github.com/emscripten-core/emscripten/issues/16868')
+  @also_with_wasm64
   @requires_threads
   def test_fetch_sync_xhr(self):
     shutil.copyfile(test_file('gears.png'), 'gears.png')
@@ -4682,6 +4703,7 @@ def test_fetch_sync_xhr(self):
   # Tests emscripten_fetch() usage when user passes none of the main 3 flags (append/replace/no_download).
   # In that case, in append is implicitly understood.
   @no_firefox('https://github.com/emscripten-core/emscripten/issues/16868')
+  @also_with_wasm64
   @requires_threads
   def test_fetch_implicit_append(self):
     shutil.copyfile(test_file('gears.png'), 'gears.png')