opcodesswitch: fix BIF error handling in CALL_EXT (and similar)

BIF return value wasn't checked for errors, so there was a chance that
invalid terms might appear around.

As of v0.6.x was very unlikely since this required an out of memory with
`min` and `max` functions.

Signed-off-by: Davide Bettio <davide@uninstall.it>

Author: bettio

CHANGELOG.md

@@ -33,6 +33,8 @@ certain VM instructions are used.
 - Fixed compilation with latest debian gcc-arm-none-eabi
 - Fix `network:stop/0` on ESP32 so the network can be started again
 - Fix a memory corruption caused by `binary:split/2,3`
+- Fix error handling when calling `min` and `max` with code compiled before OTP-26: there was a
+bug when handling errors from BIFs used as NIFs (when called with `CALL_EXT` and similar opcodes)`
 
 ## [0.6.5] - 2024-10-15
 

src/libAtomVM/opcodesswitch.h

@@ -1990,19 +1990,23 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb)
                             // Support compilers < OTP26 that generate CALL_EXT
                             // for min/2 and max/2
                             const struct Bif *bif = EXPORTED_FUNCTION_TO_BIF(func);
+                            term return_value;
                             switch (arity) {
                                 case 0:
-                                    x_regs[0] = bif->bif0_ptr(ctx);
+                                    return_value = bif->bif0_ptr(ctx);
                                     break;
                                 case 1:
-                                    x_regs[0] = bif->bif1_ptr(ctx, 0, x_regs[0]);
+                                    return_value = bif->bif1_ptr(ctx, 0, x_regs[0]);
                                     break;
                                 case 2:
-                                    x_regs[0] = bif->bif2_ptr(ctx, 0, x_regs[0], x_regs[1]);
+                                    return_value = bif->bif2_ptr(ctx, 0, x_regs[0], x_regs[1]);
                                     break;
                                 default:
                                     fprintf(stderr, "Invalid arity %" PRIu32 " for bif\n", arity);
+                                    AVM_ABORT();
                             }
+                            PROCESS_MAYBE_TRAP_RETURN_VALUE_RESTORE_PC(return_value, orig_pc);
+                            x_regs[0] = return_value;
 
                             break;
                         }
@@ -2090,19 +2094,23 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb)
                             ctx->e += (n_words + 1);
 
                             const struct Bif *bif = EXPORTED_FUNCTION_TO_BIF(func);
+                            term return_value;
                             switch (arity) {
                                 case 0:
-                                    x_regs[0] = bif->bif0_ptr(ctx);
+                                    return_value = bif->bif0_ptr(ctx);
                                     break;
                                 case 1:
-                                    x_regs[0] = bif->bif1_ptr(ctx, 0, x_regs[0]);
+                                    return_value = bif->bif1_ptr(ctx, 0, x_regs[0]);
                                     break;
                                 case 2:
-                                    x_regs[0] = bif->bif2_ptr(ctx, 0, x_regs[0], x_regs[1]);
+                                    return_value = bif->bif2_ptr(ctx, 0, x_regs[0], x_regs[1]);
                                     break;
                                 default:
                                     fprintf(stderr, "Invalid arity %" PRIu32 " for bif\n", arity);
+                                    AVM_ABORT();
                             }
+                            PROCESS_MAYBE_TRAP_RETURN_VALUE_LAST(return_value);
+                            x_regs[0] = return_value;
 
                             DO_RETURN();
 
@@ -3535,19 +3543,23 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb)
                             // Support compilers < OTP26 that generate CALL_EXT_ONLY
                             // for min/2 and max/2
                             const struct Bif *bif = EXPORTED_FUNCTION_TO_BIF(func);
+                            term return_value;
                             switch (arity) {
                                 case 0:
-                                    x_regs[0] = bif->bif0_ptr(ctx);
+                                    return_value = bif->bif0_ptr(ctx);
                                     break;
                                 case 1:
-                                    x_regs[0] = bif->bif1_ptr(ctx, 0, x_regs[0]);
+                                    return_value = bif->bif1_ptr(ctx, 0, x_regs[0]);
                                     break;
                                 case 2:
-                                    x_regs[0] = bif->bif2_ptr(ctx, 0, x_regs[0], x_regs[1]);
+                                    return_value = bif->bif2_ptr(ctx, 0, x_regs[0], x_regs[1]);
                                     break;
                                 default:
                                     fprintf(stderr, "Invalid arity %" PRIu32 " for bif\n", arity);
+                                    AVM_ABORT();
                             }
+                            PROCESS_MAYBE_TRAP_RETURN_VALUE_LAST(return_value);
+                            x_regs[0] = return_value;
 
                             DO_RETURN();