Fix many remaining bugs with >2gb address space (#20099)

Author: sbc100

.circleci/config.yml

@@ -618,9 +618,8 @@ jobs:
           title: "wasm64"
           test_targets: "
             wasm64
-            core_2gb.test_em_asm
-            core_2gb.test_embind_unsigned_bigint
-            core_2gb.test_embind_no_rtti
+            core_2gb.test_*em_asm*
+            core_2gb.test_*embind*
             wasm64l.test_bigswitch
             other.test_memory64_proxies
             other.test_failing_growth_wasm64"

emcc.py

@@ -1781,8 +1781,11 @@ def set_max_memory():
       diagnostics.warning('unused-command-line-argument', 'MAXIMUM_MEMORY is only meaningful with ALLOW_MEMORY_GROWTH')
     settings.MAXIMUM_MEMORY = settings.INITIAL_MEMORY
 
-  # INITIAL_MEMORY sets a lower bound for MAXIMUM_MEMORY
   if 'MAXIMUM_MEMORY' not in user_settings:
+    if settings.ALLOW_MEMORY_GROWTH and settings.INITIAL_MEMORY > 2 * 1024 * 1024 * 1024:
+        settings.MAXIMUM_MEMORY = 4 * 1024 * 1024 * 1024
+
+    # INITIAL_MEMORY sets a lower bound for MAXIMUM_MEMORY
     if settings.INITIAL_MEMORY > settings.MAXIMUM_MEMORY:
       settings.MAXIMUM_MEMORY = settings.INITIAL_MEMORY
 
@@ -2875,10 +2878,12 @@ def get_full_import_name(name):
   if settings.WASM2JS:
     if settings.GENERATE_SOURCE_MAP:
       exit_with_error('wasm2js does not support source maps yet (debug in wasm for now)')
-    if settings.WASM_BIGINT:
-      exit_with_error('wasm2js does not support WASM_BIGINT')
     if settings.MEMORY64:
       exit_with_error('wasm2js does not support MEMORY64')
+    if settings.WASM_BIGINT:
+      exit_with_error('wasm2js does not support WASM_BIGINT')
+    if settings.CAN_ADDRESS_2GB:
+      exit_with_error('wasm2js does not support >2gb address space')
 
   if settings.NODE_CODE_CACHING:
     if settings.WASM_ASYNC_COMPILATION:

src/jsifier.js

@@ -160,7 +160,7 @@ function runJSify() {
         args = newArgs.join(',');
       }
 
-      if ((sig[0] == 'j' && i53abi) || (sig[0] == 'p' && WASM_BIGINT)) {
+      if ((sig[0] == 'j' && i53abi) || (sig[0] == 'p' && MEMORY64)) {
         // For functions that where we need to mutate the return value, we
         // also need to wrap the body in an inner function.
         if (oneliner) {

src/library.js

@@ -2872,34 +2872,31 @@ addToLibrary({
     while (ch = HEAPU8[sigPtr++]) {
 #if ASSERTIONS
       var chr = String.fromCharCode(ch);
-      var validChars = ['d', 'f', 'i'];
+      var validChars = ['d', 'f', 'i', 'p'];
 #if WASM_BIGINT
       // In WASM_BIGINT mode we support passing i64 values as bigint.
       validChars.push('j');
-#endif
-#if MEMORY64
-      // In MEMORY64 mode we also support passing i64 pointer types which
-      // get automatically converted to int53/Double.
-      validChars.push('p');
 #endif
       assert(validChars.includes(chr), `Invalid character ${ch}("${chr}") in readEmAsmArgs! Use only [${validChars}], and do not specify "v" for void return argument.`);
 #endif
       // Floats are always passed as doubles, so all types except for 'i'
       // are 8 bytes and require alignment.
-      buf += (ch != {{{ charCode('i') }}}) && buf % 8 ? 4 : 0;
-      readEmAsmArgsArray.push(
-#if MEMORY64
-        // Special case for pointers under wasm64 which we read as int53 Numbers.
-        ch == {{{ charCode('p') }}} ?  {{{ makeGetValue('buf', 0, '*') }}} :
+      var wide = (ch != {{{ charCode('i') }}});
+#if !MEMORY64
+      wide &= (ch != {{{ charCode('p') }}});
 #endif
+      buf += wide && (buf % 8) ? 4 : 0;
+      readEmAsmArgsArray.push(
+        // Special case for pointers under wasm64 or CAN_ADDRESS_2GB mode.
+        ch == {{{ charCode('p') }}} ? {{{ makeGetValue('buf', 0, '*') }}} :
 #if WASM_BIGINT
-        ch == {{{ charCode('j') }}} ?  {{{ makeGetValue('buf', 0, 'i64') }}} :
+        ch == {{{ charCode('j') }}} ? {{{ makeGetValue('buf', 0, 'i64') }}} :
 #endif
         ch == {{{ charCode('i') }}} ?
           {{{ makeGetValue('buf', 0, 'i32') }}} :
           {{{ makeGetValue('buf', 0, 'double') }}}
       );
-      buf += ch == {{{ charCode('i') }}} ? 4 : 8;
+      buf += wide ? 8 : 4;
     }
     return readEmAsmArgsArray;
   },

src/library_int53.js

@@ -24,7 +24,8 @@ addToLibrary({
     {{{ makeSetValue('ptr', 4, '(num - lower)/4294967296', 'u32') }}};
 #if ASSERTIONS
     var deserialized = (num >= 0) ? readI53FromU64(ptr) : readI53FromI64(ptr);
-    if (deserialized != num) warnOnce(`writeI53ToI64() out of range: serialized JS Number ${num} to Wasm heap as bytes lo=${ptrToString(HEAPU32[ptr/4])}, hi=${ptrToString(HEAPU32[ptr/4+1])}, which deserializes back to ${deserialized} instead!`);
+    var offset = {{{ getHeapOffset('ptr', 'u32') }}};
+    if (deserialized != num) warnOnce(`writeI53ToI64() out of range: serialized JS Number ${num} to Wasm heap as bytes lo=${ptrToString(HEAPU32[offset])}, hi=${ptrToString(HEAPU32[offset+1])}, which deserializes back to ${deserialized} instead!`);
 #endif
   },
 
@@ -48,9 +49,9 @@ addToLibrary({
   $writeI53ToI64Signaling: (ptr, num) => {
     if (num > 0x7FFFFFFFFFFFFFFF || num < -0x8000000000000000) {
 #if ASSERTIONS
-      throw 'RangeError in writeI53ToI64Signaling(): input value ' + num + ' is out of range of int64';
+      throw `RangeError in writeI53ToI64Signaling(): input value ${num} is out of range of int64`;
 #else
-      throw 'RangeError:' + num;
+      throw `RangeError: ${num}`;
 #endif
     }
     writeI53ToI64(ptr, num);
@@ -76,9 +77,9 @@ addToLibrary({
   $writeI53ToU64Signaling: (ptr, num) => {
     if (num < 0 || num > 0xFFFFFFFFFFFFFFFF) {
 #if ASSERTIONS
-      throw 'RangeError in writeI53ToU64Signaling(): input value ' + num + ' is out of range of uint64';
+      throw `RangeError in writeI53ToU64Signaling(): input value ${num} is out of range of uint64`;
 #else
-      throw 'RangeError:'+num;
+      throw `RangeError: ${num}`;
 #endif
     }
     writeI53ToI64(ptr, num);

src/library_pthread.js

@@ -163,7 +163,7 @@ var LibraryPThread = {
 
     threadStatusAsString(pthreadPtr) {
       var profilerBlock = {{{ makeGetValue('pthreadPtr', C_STRUCTS.pthread.profilerBlock, POINTER_TYPE) }}};
-      var status = (profilerBlock == 0) ? 0 : Atomics.load(HEAPU32, (profilerBlock + {{{ C_STRUCTS.thread_profiler_block.threadStatus }}} ) >> 2);
+      var status = (profilerBlock == 0) ? 0 : Atomics.load(HEAPU32, {{{ getHeapOffset('profilerBlock + ' + C_STRUCTS.thread_profiler_block.threadStatus, 'i32') }}});
       return PThread.threadStatusToString(status);
     },
 #endif
@@ -972,7 +972,7 @@ var LibraryPThread = {
       // values we serialize for proxying. TODO: pack this?
       var serializedNumCallArgs = numCallArgs {{{ WASM_BIGINT ? "* 2" : "" }}};
       var args = stackAlloc(serializedNumCallArgs * 8);
-      var b = args >> 3;
+      var b = {{{ getHeapOffset('args', 'i64') }}};
       for (var i = 0; i < numCallArgs; i++) {
         var arg = outerArgs[2 + i];
 #if WASM_BIGINT
@@ -1008,7 +1008,7 @@ var LibraryPThread = {
     numCallArgs /= 2;
 #endif
     proxiedJSCallArgs.length = numCallArgs;
-    var b = args >> 3;
+    var b = {{{ getHeapOffset('args', 'i64') }}};
     for (var i = 0; i < numCallArgs; i++) {
 #if WASM_BIGINT
       if (HEAP64[b + 2*i]) {
@@ -1233,13 +1233,13 @@ var LibraryPThread = {
   _emscripten_thread_mailbox_await: (pthread_ptr) => {
     if (typeof Atomics.waitAsync === 'function') {
       // TODO: How to make this work with wasm64?
-      var wait = Atomics.waitAsync(HEAP32, pthread_ptr >> 2, pthread_ptr);
+      var wait = Atomics.waitAsync(HEAP32, {{{ getHeapOffset('pthread_ptr', 'i32') }}}, pthread_ptr);
 #if ASSERTIONS
       assert(wait.async);
 #endif
       wait.value.then(checkMailbox);
       var waitingAsync = pthread_ptr + {{{ C_STRUCTS.pthread.waiting_async }}};
-      Atomics.store(HEAP32, waitingAsync >> 2, 1);
+      Atomics.store(HEAP32, {{{ getHeapOffset('waitingAsync', 'i32') }}}, 1);
     }
   },
 

src/library_wasm_worker.js

@@ -294,15 +294,15 @@ Atomics.waitAsync = (i32a, index, value, maxWaitMilliseconds) => {
 
   emscripten_atomic_wait_async__deps: ['$atomicWaitStates', '$liveAtomicWaitAsyncs', '$liveAtomicWaitAsyncCounter', '$jstoi_q'],
   emscripten_atomic_wait_async: (addr, val, asyncWaitFinished, userData, maxWaitMilliseconds) => {
-    let wait = Atomics.waitAsync(HEAP32, addr >> 2, val, maxWaitMilliseconds);
+    let wait = Atomics.waitAsync(HEAP32, {{{ getHeapOffset('addr', 'i32') }}}, val, maxWaitMilliseconds);
     if (!wait.async) return atomicWaitStates.indexOf(wait.value);
     // Increment waitAsync generation counter, account for wraparound in case
     // application does huge amounts of waitAsyncs per second (not sure if
     // possible?)
     // Valid counterrange: 0...2^31-1
     let counter = liveAtomicWaitAsyncCounter;
     liveAtomicWaitAsyncCounter = Math.max(0, (liveAtomicWaitAsyncCounter+1)|0);
-    liveAtomicWaitAsyncs[counter] = addr >> 2;
+    liveAtomicWaitAsyncs[counter] = addr;
     wait.value.then((value) => {
       if (liveAtomicWaitAsyncs[counter]) {
         delete liveAtomicWaitAsyncs[counter];
@@ -323,13 +323,14 @@ Atomics.waitAsync = (i32a, index, value, maxWaitMilliseconds) => {
       warnOnce(`Attempted to call emscripten_atomic_cancel_wait_async() with an invalid wait token value ${waitToken}`);
     }
 #endif
-    if (liveAtomicWaitAsyncs[waitToken]) {
+    var address = liveAtomicWaitAsyncs[waitToken];
+    if (address) {
       // Notify the waitAsync waiters on the memory location, so that JavaScript
       // garbage collection can occur.
       // See https://github.com/WebAssembly/threads/issues/176
       // This has the unfortunate effect of causing spurious wakeup of all other
       // waiters at the address (which causes a small performance loss).
-      Atomics.notify(HEAP32, liveAtomicWaitAsyncs[waitToken]);
+      Atomics.notify(HEAP32, {{{ getHeapOffset('address', 'i32') }}});
       delete liveAtomicWaitAsyncs[waitToken];
       return {{{ cDefs.EMSCRIPTEN_RESULT_SUCCESS }}};
     }
@@ -341,19 +342,18 @@ Atomics.waitAsync = (i32a, index, value, maxWaitMilliseconds) => {
   emscripten_atomic_cancel_all_wait_asyncs: () => {
     let waitAsyncs = Object.values(liveAtomicWaitAsyncs);
     waitAsyncs.forEach((address) => {
-      Atomics.notify(HEAP32, address);
+      Atomics.notify(HEAP32, {{{ getHeapOffset('address', 'i32') }}});
     });
     liveAtomicWaitAsyncs = {};
     return waitAsyncs.length;
   },
 
   emscripten_atomic_cancel_all_wait_asyncs_at_address__deps: ['$liveAtomicWaitAsyncs'],
   emscripten_atomic_cancel_all_wait_asyncs_at_address: (address) => {
-    address >>= 2;
     let numCancelled = 0;
     Object.keys(liveAtomicWaitAsyncs).forEach((waitToken) => {
       if (liveAtomicWaitAsyncs[waitToken] == address) {
-        Atomics.notify(HEAP32, address);
+        Atomics.notify(HEAP32, {{{ getHeapOffset('address', 'i32') }}});
         delete liveAtomicWaitAsyncs[waitToken];
         numCancelled++;
       }

system/include/emscripten/em_asm.h

@@ -69,33 +69,27 @@ void emscripten_asm_const_async_on_main_thread(
 
 // We can use the generic selection C11 feature (that clang supports pre-C11
 // as an extension) to emulate function overloading in C.
-// All other types, including *all* pointer types go through the default case.
-// This means we need to use a different default type for `__wasm64__` where
-// pointers are 64-bits wide, and we also need to include more non-default
-// cases, for example, we don't want `short` to end up using the wider default.
-#if __wasm64__
+// All other types, including *all* pointer types go through the default case
+#ifdef __wasm64__
+#define LONG_CODE 'j'
+#else
+#define LONG_CODE 'i'
+#endif
 #define _EM_ASM_SIG_CHAR(x) _Generic((x), \
     float: 'f', \
     double: 'd', \
     char: 'i', \
     unsigned char: 'i', \
     unsigned short: 'i', \
     unsigned int: 'i', \
-    unsigned long: 'j', \
+    unsigned long: LONG_CODE, \
     unsigned long long: 'j', \
     signed char: 'i', \
     signed short: 'i', \
     signed int: 'i', \
-    signed long: 'j', \
+    signed long: LONG_CODE, \
     signed long long: 'j', \
     default: 'p')
-#else
-#define _EM_ASM_SIG_CHAR(x) _Generic((x), \
-    float: 'f', \
-    double: 'd', \
-    long long: 'j', \
-    default: 'i')
-#endif
 
 // This indirection is needed to allow us to concatenate computed results, e.g.
 //   #define BAR(N) _EM_ASM_CONCATENATE(FOO_, N)
@@ -170,11 +164,7 @@ template<> struct __em_asm_sig<bool> { static const char value = 'i'; };
 template<> struct __em_asm_sig<wchar_t> { static const char value = 'i'; };
 template<> struct __em_asm_sig<long long> { static const char value = 'j'; };
 template<> struct __em_asm_sig<unsigned long long> { static const char value = 'j'; };
-#if __wasm64__
 template<typename T> struct __em_asm_sig<T*> { static const char value = 'p'; };
-#else
-template<typename T> struct __em_asm_sig<T*> { static const char value = 'i'; };
-#endif
 
 // Explicit support for enums, they're passed as int via variadic arguments.
 template<bool> struct __em_asm_if { };

system/lib/emmalloc.c

@@ -484,7 +484,7 @@ static bool claim_more_memory(size_t numBytes)
     return false;
   }
 #ifdef EMMALLOC_VERBOSE
-  MAIN_THREAD_ASYNC_EM_ASM(console.log('claim_more_memory: claimed 0x' + ($0>>>0).toString(16) + ' - 0x' + ($1>>>0).toString(16) + ' (' + ($2>>>0) + ' bytes) via sbrk()'), startPtr, startPtr + numBytes, numBytes);
+  MAIN_THREAD_ASYNC_EM_ASM(console.log('claim_more_memory: claimed ' + ptrToString($0) + ' - ' + ptrToString($1) + ' (' + ($2>>>0) + ' bytes) via sbrk()'), startPtr, startPtr + numBytes, numBytes);
 #endif
   assert(HAS_ALIGNMENT(startPtr, alignof(size_t)));
   uint8_t *endPtr = startPtr + numBytes;
@@ -634,7 +634,7 @@ static void *attempt_allocate(Region *freeRegion, size_t alignment, size_t size)
 #endif
 
 #ifdef EMMALLOC_VERBOSE
-  MAIN_THREAD_ASYNC_EM_ASM(console.log('attempt_allocate - succeeded allocating memory, region ptr=0x' + ($0>>>0).toString(16) + ', align=' + $1 + ', payload size=' + ($2>>>0) + ' bytes)'), freeRegion, alignment, size);
+  MAIN_THREAD_ASYNC_EM_ASM(console.log('attempt_allocate - succeeded allocating memory, region ptr=' + ptrToString($0) + ', align=' + $1 + ', payload size=' + ($2>>>0) + ' bytes)'), freeRegion, alignment, size);
 #endif
 
   return (uint8_t*)freeRegion + sizeof(size_t);
@@ -661,6 +661,10 @@ static size_t validate_alloc_size(size_t size)
   return validatedSize;
 }
 
+#ifdef EMMALLOC_VERBOSE
+EM_JS_DEPS(deps, "$ptrToString");
+#endif
+
 static void *allocate_memory(size_t alignment, size_t size)
 {
   ASSERT_MALLOC_IS_ACQUIRED();

test/common.py

@@ -339,8 +339,12 @@ def metafunc(self, with_wasm64):
 def can_do_standalone(self, impure=False):
   # Pure standalone engines don't support MEMORY64 yet.  Even with MEMORY64=2 (lowered)
   # the WASI APIs that take pointer values don't have 64-bit variants yet.
-  if self.get_setting('MEMORY64') and not impure:
-    return False
+  if not impure:
+    if self.get_setting('MEMORY64'):
+      return False
+    # This is way to detect the core_2gb test mode in test_core.py
+    if self.get_setting('INITIAL_MEMORY') == '2200mb':
+      return False
   return self.is_wasm() and \
       self.get_setting('STACK_OVERFLOW_CHECK', 0) < 2 and \
       not self.get_setting('MINIMAL_RUNTIME') and \
@@ -1399,7 +1403,8 @@ def _test_dylink_dso_needed(self, do_run):
     self.clear_setting('SIDE_MODULE')
 
     # XXX in wasm each lib load currently takes 5MB; default INITIAL_MEMORY=16MB is thus not enough
-    self.set_setting('INITIAL_MEMORY', '32mb')
+    if not self.has_changed_setting('INITIAL_MEMORY'):
+      self.set_setting('INITIAL_MEMORY', '32mb')
 
     so = '.wasm' if self.is_wasm() else '.js'