Fix crash when decoding extended literals with a large index

Fix a bug where the VM could crash if an extended literal with a large index
was encountered.

Signed-off-by: Paul Guyot <pguyot@kallisys.net>

Author: pguyot

CHANGELOG.md

@@ -38,7 +38,7 @@ instead
 - ESP32: content of `boot.avm` partition is not truncated anymore
 - ESP32: `Fixed gpio:set_int` to accept any pin, not only pin 2
 - Fix memory corruption in `unicode:characters_to_binary`
-- Fix handling of large literal indexes
+- Fix handling of large literal indexes and large extended literal indexes
 - `unicode:characters_to_list`: fixed bogus out_of_memory error on some platforms such as ESP32
 - Fix crash in Elixir library when doing `inspect(:atom)`
 - General inspect() compliance with Elixir behavior (but there are still some minor differences)

src/libAtomVM/opcodesswitch.h

@@ -180,11 +180,22 @@ typedef dreg_t dreg_gc_safe_t;
         case COMPACT_EXTENDED:                                                          \
             switch (first_byte) {                                                       \
                 case COMPACT_EXTENDED_LITERAL: {                                        \
-                    uint8_t ext = ((*(decode_pc)++) & 0xF);                             \
-                    if (ext == 0x8) {                                                   \
-                        (decode_pc)++;                                                  \
-                    } else if (ext != 0) {                                              \
-                        AVM_ABORT();                                                    \
+                    uint8_t first_extended_byte = *(decode_pc)++;                       \
+                    switch (((first_extended_byte) >> 3) & 0x3) {                       \
+                        case 0:                                                         \
+                        case 2:                                                         \
+                            break;                                                      \
+                                                                                        \
+                        case 1:                                                         \
+                            (decode_pc)++;                                              \
+                            break;                                                      \
+                                                                                        \
+                        case 3: {                                                       \
+                            uint8_t sz = (first_extended_byte >> 5) + 2;                \
+                            decode_pc += sz;                                            \
+                            break;                                                      \
+                        }                                                               \
+                        default: UNREACHABLE(); /* help gcc 8.4 */                      \
                     }                                                                   \
                     break;                                                              \
                 }                                                                       \
@@ -594,14 +605,28 @@ static void destroy_extended_registers(Context *ctx, unsigned int live)
             switch (first_byte) {                                                                                       \
                 case COMPACT_EXTENDED_LITERAL: {                                                                        \
                     uint8_t first_extended_byte = *(decode_pc)++;                                                       \
-                    if (!(first_extended_byte & 0xF)) {                                                                 \
-                        dest_term = module_load_literal(mod, first_extended_byte >> 4, ctx);                            \
-                    } else if ((first_extended_byte & 0xF) == 0x8) {                                                    \
-                        uint8_t byte_1 = *(decode_pc)++;                                                                \
-                        uint16_t index = (((uint16_t) first_extended_byte & 0xE0) << 3) | byte_1;                       \
-                        dest_term = module_load_literal(mod, index, ctx);                                               \
-                    } else {                                                                                            \
-                        VM_ABORT();                                                                                     \
+                    switch (((first_extended_byte) >> 3) & 0x3) {                                                       \
+                        case 0:                                                                                         \
+                        case 2:                                                                                         \
+                            dest_term = module_load_literal(mod, first_extended_byte >> 4, ctx);                        \
+                            break;                                                                                      \
+                        case 1: {                                                                                       \
+                            uint8_t byte_1 = *(decode_pc)++;                                                            \
+                            uint16_t index = (((uint16_t) first_extended_byte & 0xE0) << 3) | byte_1;                   \
+                            dest_term = module_load_literal(mod, index, ctx);                                           \
+                            break;                                                                                      \
+                        }                                                                                               \
+                        case 3: {                                                                                       \
+                            uint8_t sz = (first_extended_byte >> 5) + 2;                                                \
+                            avm_int_t val = 0;                                                                          \
+                            for (uint8_t vi = 0; vi < sz; vi++) {                                                       \
+                                val <<= 8;                                                                              \
+                                val |= *(decode_pc)++;                                                                  \
+                            }                                                                                           \
+                            dest_term = module_load_literal(mod, val, ctx);                                             \
+                            break;                                                                                      \
+                        }                                                                                               \
+                        default: UNREACHABLE(); /* help gcc 8.4 */                                                      \
                     }                                                                                                   \
                     if (UNLIKELY(term_is_invalid_term(dest_term))) {                                                    \
                         RAISE_ERROR(OUT_OF_MEMORY_ATOM);                                                                \

tests/erlang_tests/CMakeLists.txt

@@ -322,6 +322,7 @@ compile_erlang(minuspow63plustwoabs)
 compile_erlang(literal_test0)
 compile_erlang(literal_test1)
 compile_erlang(literal_test2)
+compile_erlang(test_extended_literal_large)
 
 compile_erlang(test_list_eq)
 compile_erlang(test_tuple_eq)
@@ -791,6 +792,7 @@ add_custom_target(erlang_test_modules DEPENDS
     literal_test0.beam
     literal_test1.beam
     literal_test2.beam
+    test_extended_literal_large.beam
 
     test_list_eq.beam
     test_tuple_eq.beam