Merge pull request #1294 from pguyot/w39/add-support-for-LARGE_EXT_TUPLE

bettio · bettio · commit 389ec97e4ada · 2024-09-29T20:20:37.000+02:00
Add support for LARGE_TUPLE_EXT (fix #1292) These changes are made under both the "Apache 2.0" and the "GNU Lesser General Public License 2.1 or later" license terms (dual license). SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -28,6 +28,7 @@ also non string parameters (e.g. `Enum.join([1, 2], ",")`
 - Support for `io_lib:latin1_char_list/1`
 - Add support to Elixir for `Keyword.split/2`
 - Support for `binary:split/3` and `string:find/2,3`
+- Support for large tuples (more than 255 elements) in external terms.
 
 ### Changed
 
diff --git a/src/libAtomVM/externalterm.c b/src/libAtomVM/externalterm.c
@@ -38,6 +38,7 @@
 #define INTEGER_EXT 98
 #define ATOM_EXT 100
 #define SMALL_TUPLE_EXT 104
+#define LARGE_TUPLE_EXT 105
 #define NIL_EXT 106
 #define STRING_EXT 107
 #define LIST_EXT 108
@@ -281,15 +282,20 @@ static int serialize_term(uint8_t *buf, term t, GlobalContext *glb)
 
     } else if (term_is_tuple(t)) {
         size_t arity = term_get_tuple_arity(t);
-        if (arity > 255) {
-            fprintf(stderr, "Tuple arity greater than 255: %zu\n", arity);
-            AVM_ABORT();
-        }
+        size_t k;
         if (!IS_NULL_PTR(buf)) {
-            buf[0] = SMALL_TUPLE_EXT;
-            buf[1] = (int8_t) arity;
+            if (arity < 256) {
+                buf[0] = SMALL_TUPLE_EXT;
+                buf[1] = (int8_t) arity;
+                k = 2;
+            } else {
+                buf[0] = LARGE_TUPLE_EXT;
+                WRITE_32_UNALIGNED(buf + 1, (int32_t) arity);
+                k = 5;
+            }
+        } else {
+            k = arity < 256 ? 2 : 5;
         }
-        size_t k = 2;
         for (size_t i = 0; i < arity; ++i) {
             term e = term_get_tuple_element(t, i);
             k += serialize_term(IS_NULL_PTR(buf) ? NULL : buf + k, e, glb);
@@ -479,13 +485,20 @@ static term parse_external_terms(const uint8_t *external_term_buf, size_t *eterm
             return term_from_atom_index(global_atom_id);
         }
 
-        case SMALL_TUPLE_EXT: {
-            uint8_t arity = external_term_buf[1];
+        case SMALL_TUPLE_EXT:
+        case LARGE_TUPLE_EXT: {
+            size_t arity;
+            int buf_pos;
+            if (external_term_buf[0] == SMALL_TUPLE_EXT) {
+                arity = external_term_buf[1];
+                buf_pos = 2;
+            } else {
+                arity = READ_32_UNALIGNED(external_term_buf + 1);
+                buf_pos = 5;
+            }
             term tuple = term_alloc_tuple(arity, heap);
 
-            int buf_pos = 2;
-
-            for (int i = 0; i < arity; i++) {
+            for (size_t i = 0; i < arity; i++) {
                 size_t element_size;
                 term put_value = parse_external_terms(external_term_buf + buf_pos, &element_size, copy, heap, glb);
                 if (UNLIKELY(term_is_invalid_term(put_value))) {
@@ -713,20 +726,33 @@ static int calculate_heap_usage(const uint8_t *external_term_buf, size_t remaini
             return 0;
         }
 
-        case SMALL_TUPLE_EXT: {
-            if (UNLIKELY(remaining < 1)) {
-                return INVALID_TERM_SIZE;
+        case SMALL_TUPLE_EXT:
+        case LARGE_TUPLE_EXT: {
+            size_t arity;
+            size_t buf_pos;
+            if (external_term_buf[0] == SMALL_TUPLE_EXT) {
+                if (UNLIKELY(remaining < 1)) {
+                    return INVALID_TERM_SIZE;
+                }
+                remaining--;
+                arity = external_term_buf[1];
+                buf_pos = 2;
+            } else {
+                if (UNLIKELY(remaining < 5)) {
+                    return INVALID_TERM_SIZE;
+                }
+                remaining -= 5;
+                arity = READ_32_UNALIGNED(external_term_buf + 1);
+                buf_pos = 5;
             }
-            remaining--;
-            uint8_t arity = external_term_buf[1];
+
             if (UNLIKELY(remaining < arity)) {
                 return INVALID_TERM_SIZE;
             }
 
             int heap_usage = 1;
-            size_t buf_pos = 2;
 
-            for (int i = 0; i < arity; i++) {
+            for (size_t i = 0; i < arity; i++) {
                 size_t element_size = 0;
                 int u = calculate_heap_usage(external_term_buf + buf_pos, remaining, &element_size, copy);
                 if (UNLIKELY(u == INVALID_TERM_SIZE)) {
diff --git a/tests/erlang_tests/test_binary_to_term.erl b/tests/erlang_tests/test_binary_to_term.erl
@@ -122,6 +122,48 @@ start() ->
             57, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 48,
             49, 50, 51, 52, 53>>
     ),
+    test_reverse(
+        {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
+            26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
+            48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69,
+            70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91,
+            92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110,
+            111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127,
+            128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144,
+            145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161,
+            162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178,
+            179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195,
+            196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212,
+            213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229,
+            230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246,
+            247, 248, 249, 250, 251, 252, 253, 254, 255, 256},
+        <<131, 105, 0, 0, 1, 0, 97, 1, 97, 2, 97, 3, 97, 4, 97, 5, 97, 6, 97, 7, 97, 8, 97, 9, 97,
+            10, 97, 11, 97, 12, 97, 13, 97, 14, 97, 15, 97, 16, 97, 17, 97, 18, 97, 19, 97, 20, 97,
+            21, 97, 22, 97, 23, 97, 24, 97, 25, 97, 26, 97, 27, 97, 28, 97, 29, 97, 30, 97, 31, 97,
+            32, 97, 33, 97, 34, 97, 35, 97, 36, 97, 37, 97, 38, 97, 39, 97, 40, 97, 41, 97, 42, 97,
+            43, 97, 44, 97, 45, 97, 46, 97, 47, 97, 48, 97, 49, 97, 50, 97, 51, 97, 52, 97, 53, 97,
+            54, 97, 55, 97, 56, 97, 57, 97, 58, 97, 59, 97, 60, 97, 61, 97, 62, 97, 63, 97, 64, 97,
+            65, 97, 66, 97, 67, 97, 68, 97, 69, 97, 70, 97, 71, 97, 72, 97, 73, 97, 74, 97, 75, 97,
+            76, 97, 77, 97, 78, 97, 79, 97, 80, 97, 81, 97, 82, 97, 83, 97, 84, 97, 85, 97, 86, 97,
+            87, 97, 88, 97, 89, 97, 90, 97, 91, 97, 92, 97, 93, 97, 94, 97, 95, 97, 96, 97, 97, 97,
+            98, 97, 99, 97, 100, 97, 101, 97, 102, 97, 103, 97, 104, 97, 105, 97, 106, 97, 107, 97,
+            108, 97, 109, 97, 110, 97, 111, 97, 112, 97, 113, 97, 114, 97, 115, 97, 116, 97, 117,
+            97, 118, 97, 119, 97, 120, 97, 121, 97, 122, 97, 123, 97, 124, 97, 125, 97, 126, 97,
+            127, 97, 128, 97, 129, 97, 130, 97, 131, 97, 132, 97, 133, 97, 134, 97, 135, 97, 136,
+            97, 137, 97, 138, 97, 139, 97, 140, 97, 141, 97, 142, 97, 143, 97, 144, 97, 145, 97,
+            146, 97, 147, 97, 148, 97, 149, 97, 150, 97, 151, 97, 152, 97, 153, 97, 154, 97, 155,
+            97, 156, 97, 157, 97, 158, 97, 159, 97, 160, 97, 161, 97, 162, 97, 163, 97, 164, 97,
+            165, 97, 166, 97, 167, 97, 168, 97, 169, 97, 170, 97, 171, 97, 172, 97, 173, 97, 174,
+            97, 175, 97, 176, 97, 177, 97, 178, 97, 179, 97, 180, 97, 181, 97, 182, 97, 183, 97,
+            184, 97, 185, 97, 186, 97, 187, 97, 188, 97, 189, 97, 190, 97, 191, 97, 192, 97, 193,
+            97, 194, 97, 195, 97, 196, 97, 197, 97, 198, 97, 199, 97, 200, 97, 201, 97, 202, 97,
+            203, 97, 204, 97, 205, 97, 206, 97, 207, 97, 208, 97, 209, 97, 210, 97, 211, 97, 212,
+            97, 213, 97, 214, 97, 215, 97, 216, 97, 217, 97, 218, 97, 219, 97, 220, 97, 221, 97,
+            222, 97, 223, 97, 224, 97, 225, 97, 226, 97, 227, 97, 228, 97, 229, 97, 230, 97, 231,
+            97, 232, 97, 233, 97, 234, 97, 235, 97, 236, 97, 237, 97, 238, 97, 239, 97, 240, 97,
+            241, 97, 242, 97, 243, 97, 244, 97, 245, 97, 246, 97, 247, 97, 248, 97, 249, 97, 250,
+            97, 251, 97, 252, 97, 253, 97, 254, 97, 255, 98, 0, 0, 1, 0>>
+    ),
     ok = test_external_function(),
 
     {32768, 6} = erlang:binary_to_term(<<131, 98, 0, 0, 128, 0, 127>>, [used]),
