Merge pull request #1522 from bettio/bif-error-in-call_ext-opcodes

opcodesswitch: fix BIF error handling in `CALL_EXT` (and similar)

Check return value for `term_invalid_term()` otherwise error will not be raised,
and invalid terms will appear.

These changes are made under both the "Apache 2.0" and the "GNU Lesser General
Public License 2.1 or later" license terms (dual license).

SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later

Author: bettio

CHANGELOG.md

@@ -40,6 +40,8 @@ integers
 - Fixed support for setting esp32 boot_path in NVS.
 - Fixed race conditions in network:start/stop.
 - Fixed crash calling network:sta_rssi(), when network not up.
+- Fix error handling when calling `min` and `max` with code compiled before OTP-26: there was a
+bug when handling errors from BIFs used as NIFs (when called with `CALL_EXT` and similar opcodes)`
 
 ## [0.6.5] - 2024-10-15
 

src/libAtomVM/opcodesswitch.h

@@ -1990,19 +1990,23 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb)
                             // Support compilers < OTP26 that generate CALL_EXT
                             // for min/2 and max/2
                             const struct Bif *bif = EXPORTED_FUNCTION_TO_BIF(func);
+                            term return_value;
                             switch (arity) {
                                 case 0:
-                                    x_regs[0] = bif->bif0_ptr(ctx);
+                                    return_value = bif->bif0_ptr(ctx);
                                     break;
                                 case 1:
-                                    x_regs[0] = bif->bif1_ptr(ctx, 0, x_regs[0]);
+                                    return_value = bif->bif1_ptr(ctx, 0, x_regs[0]);
                                     break;
                                 case 2:
-                                    x_regs[0] = bif->bif2_ptr(ctx, 0, x_regs[0], x_regs[1]);
+                                    return_value = bif->bif2_ptr(ctx, 0, x_regs[0], x_regs[1]);
                                     break;
                                 default:
                                     fprintf(stderr, "Invalid arity %" PRIu32 " for bif\n", arity);
+                                    AVM_ABORT();
                             }
+                            PROCESS_MAYBE_TRAP_RETURN_VALUE_RESTORE_PC(return_value, orig_pc);
+                            x_regs[0] = return_value;
 
                             break;
                         }
@@ -2090,19 +2094,23 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb)
                             ctx->e += (n_words + 1);
 
                             const struct Bif *bif = EXPORTED_FUNCTION_TO_BIF(func);
+                            term return_value;
                             switch (arity) {
                                 case 0:
-                                    x_regs[0] = bif->bif0_ptr(ctx);
+                                    return_value = bif->bif0_ptr(ctx);
                                     break;
                                 case 1:
-                                    x_regs[0] = bif->bif1_ptr(ctx, 0, x_regs[0]);
+                                    return_value = bif->bif1_ptr(ctx, 0, x_regs[0]);
                                     break;
                                 case 2:
-                                    x_regs[0] = bif->bif2_ptr(ctx, 0, x_regs[0], x_regs[1]);
+                                    return_value = bif->bif2_ptr(ctx, 0, x_regs[0], x_regs[1]);
                                     break;
                                 default:
                                     fprintf(stderr, "Invalid arity %" PRIu32 " for bif\n", arity);
+                                    AVM_ABORT();
                             }
+                            PROCESS_MAYBE_TRAP_RETURN_VALUE_LAST(return_value);
+                            x_regs[0] = return_value;
 
                             DO_RETURN();
 
@@ -3541,19 +3549,23 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb)
                             // Support compilers < OTP26 that generate CALL_EXT_ONLY
                             // for min/2 and max/2
                             const struct Bif *bif = EXPORTED_FUNCTION_TO_BIF(func);
+                            term return_value;
                             switch (arity) {
                                 case 0:
-                                    x_regs[0] = bif->bif0_ptr(ctx);
+                                    return_value = bif->bif0_ptr(ctx);
                                     break;
                                 case 1:
-                                    x_regs[0] = bif->bif1_ptr(ctx, 0, x_regs[0]);
+                                    return_value = bif->bif1_ptr(ctx, 0, x_regs[0]);
                                     break;
                                 case 2:
-                                    x_regs[0] = bif->bif2_ptr(ctx, 0, x_regs[0], x_regs[1]);
+                                    return_value = bif->bif2_ptr(ctx, 0, x_regs[0], x_regs[1]);
                                     break;
                                 default:
                                     fprintf(stderr, "Invalid arity %" PRIu32 " for bif\n", arity);
+                                    AVM_ABORT();
                             }
+                            PROCESS_MAYBE_TRAP_RETURN_VALUE_LAST(return_value);
+                            x_regs[0] = return_value;
 
                             DO_RETURN();