Skip to content

Optimize memory_scan_and_copy and memory_scan_and_rewrite #1771

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 24, 2025
Merged

Optimize memory_scan_and_copy and memory_scan_and_rewrite #1771

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
331 changes: 150 additions & 181 deletions src/libAtomVM/memory.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -580,137 +580,121 @@ static void memory_scan_and_copy(HeapFragment *old_fragment, term *mem_start, co

while (ptr < mem_end) {
term t = *ptr;
switch (t & TERM_PRIMARY_MASK) {
case TERM_PRIMARY_IMMED:
TRACE("Found immediate (%" TERM_X_FMT ")\n", t);
ptr++;
break;
case TERM_PRIMARY_CP: {
TRACE("Found boxed header (%" TERM_X_FMT ")\n", t);

size_t arity = term_get_size_from_boxed_header(t);
switch (t & TERM_BOXED_TAG_MASK) {
case TERM_BOXED_TUPLE: {
TRACE("- Boxed is tuple (%" TERM_X_FMT "), arity: %i\n", t, (int) arity);

for (size_t i = 1; i <= arity; i++) {
TRACE("-- Elem: %" TERM_X_FMT "\n", ptr[i]);
ptr[i] = memory_shallow_copy_term(old_fragment, ptr[i], &new_heap, move);
}
break;
}

if (term_is_atom(t)) {
TRACE("Found atom (%" TERM_X_FMT ")\n", t);
ptr++;

} else if (term_is_integer(t)) {
TRACE("Found integer (%" TERM_X_FMT ")\n", t);
ptr++;

} else if (term_is_nil(t)) {
TRACE("Found NIL (%" TERM_X_FMT ")\n", t);
ptr++;

} else if (term_is_local_pid(t)) {
TRACE("Found PID (%" TERM_X_FMT ")\n", t);
ptr++;

} else if (term_is_local_port(t)) {
TRACE("Found port (%" TERM_X_FMT ")\n", t);
ptr++;

} else if ((t & 0x3) == 0x0) {
TRACE("Found boxed header (%" TERM_X_FMT ")\n", t);

size_t arity = term_get_size_from_boxed_header(t);
switch (t & TERM_BOXED_TAG_MASK) {
case TERM_BOXED_TUPLE: {
TRACE("- Boxed is tuple (%" TERM_X_FMT "), arity: %i\n", t, (int) arity);

for (size_t i = 1; i <= arity; i++) {
TRACE("-- Elem: %" TERM_X_FMT "\n", ptr[i]);
ptr[i] = memory_shallow_copy_term(old_fragment, ptr[i], &new_heap, move);
case TERM_BOXED_BIN_MATCH_STATE: {
TRACE("- Found bin match state.\n");
ptr[1] = memory_shallow_copy_term(old_fragment, ptr[1], &new_heap, move);
break;
}
break;
}

case TERM_BOXED_BIN_MATCH_STATE: {
TRACE("- Found bin match state.\n");
ptr[1] = memory_shallow_copy_term(old_fragment, ptr[1], &new_heap, move);
break;
}
case TERM_BOXED_POSITIVE_INTEGER:
TRACE("- Found boxed pos int.\n");
break;

case TERM_BOXED_POSITIVE_INTEGER:
TRACE("- Found boxed pos int.\n");
break;
case TERM_BOXED_REF:
TRACE("- Found ref.\n");
break;

case TERM_BOXED_REF:
TRACE("- Found ref.\n");
break;
case TERM_BOXED_EXTERNAL_PID:
TRACE("- Found external pid.\n");
break;

case TERM_BOXED_EXTERNAL_PID:
TRACE("- Found external pid.\n");
break;
case TERM_BOXED_EXTERNAL_PORT:
TRACE("- Found external port.\n");
break;

case TERM_BOXED_EXTERNAL_PORT:
TRACE("- Found external port.\n");
break;
case TERM_BOXED_EXTERNAL_REF:
TRACE("- Found external ref.\n");
break;

case TERM_BOXED_EXTERNAL_REF:
TRACE("- Found external ref.\n");
break;

case TERM_BOXED_FUN: {
TRACE("- Found fun, size: %i.\n", (int) arity);
case TERM_BOXED_FUN: {
TRACE("- Found fun, size: %i.\n", (int) arity);

// first term is the boxed header, followed by module and fun index.
// first term is the boxed header, followed by module and fun index.

for (size_t i = 3; i <= arity; i++) {
TRACE("-- Frozen: %" TERM_X_FMT "\n", ptr[i]);
ptr[i] = memory_shallow_copy_term(old_fragment, ptr[i], &new_heap, move);
for (size_t i = 3; i <= arity; i++) {
TRACE("-- Frozen: %" TERM_X_FMT "\n", ptr[i]);
ptr[i] = memory_shallow_copy_term(old_fragment, ptr[i], &new_heap, move);
}
break;
}
break;
}

case TERM_BOXED_FLOAT:
TRACE("- Found float.\n");
break;
case TERM_BOXED_FLOAT:
TRACE("- Found float.\n");
break;

case TERM_BOXED_REFC_BINARY: {
TRACE("- Found refc binary.\n");
term ref = ((term) ptr) | TERM_PRIMARY_BOXED;
if (!term_refc_binary_is_const(ref)) {
*mso_list = term_list_init_prepend(ptr + REFC_BINARY_CONS_OFFSET, ref, *mso_list);
refc_binary_increment_refcount((struct RefcBinary *) term_refc_binary_ptr(ref));
}
break;
}

case TERM_BOXED_REFC_BINARY: {
TRACE("- Found refc binary.\n");
term ref = ((term) ptr) | TERM_PRIMARY_BOXED;
if (!term_refc_binary_is_const(ref)) {
*mso_list = term_list_init_prepend(ptr + REFC_BINARY_CONS_OFFSET, ref, *mso_list);
refc_binary_increment_refcount((struct RefcBinary *) term_refc_binary_ptr(ref));
case TERM_BOXED_SUB_BINARY: {
TRACE("- Found sub binary.\n");
ptr[3] = memory_shallow_copy_term(old_fragment, ptr[3], &new_heap, move);
break;
}
break;
}

case TERM_BOXED_SUB_BINARY: {
TRACE("- Found sub binary.\n");
ptr[3] = memory_shallow_copy_term(old_fragment, ptr[3], &new_heap, move);
break;
case TERM_BOXED_HEAP_BINARY:
TRACE("- Found binary.\n");
break;

case TERM_BOXED_MAP: {
TRACE("- Found map.\n");
size_t map_size = arity - 1;
size_t keys_offset = term_get_map_keys_offset();
size_t value_offset = term_get_map_value_offset();
TRACE("-- Map keys: %" TERM_X_FMT "\n", ptr[keys_offset]);
ptr[keys_offset] = memory_shallow_copy_term(old_fragment, ptr[keys_offset], &new_heap, move);
for (size_t i = value_offset; i < value_offset + map_size; ++i) {
TRACE("-- Map Value: %" TERM_X_FMT "\n", ptr[i]);
ptr[i] = memory_shallow_copy_term(old_fragment, ptr[i], &new_heap, move);
}
} break;

default:
fprintf(stderr, "- Found unknown boxed type: %" TERM_X_FMT "\n", (t >> 2) & 0xF);
AVM_ABORT();
}

case TERM_BOXED_HEAP_BINARY:
TRACE("- Found binary.\n");
break;

case TERM_BOXED_MAP: {
TRACE("- Found map.\n");
size_t map_size = arity - 1;
size_t keys_offset = term_get_map_keys_offset();
size_t value_offset = term_get_map_value_offset();
TRACE("-- Map keys: %" TERM_X_FMT "\n", ptr[keys_offset]);
ptr[keys_offset] = memory_shallow_copy_term(old_fragment, ptr[keys_offset], &new_heap, move);
for (size_t i = value_offset; i < value_offset + map_size; ++i) {
TRACE("-- Map Value: %" TERM_X_FMT "\n", ptr[i]);
ptr[i] = memory_shallow_copy_term(old_fragment, ptr[i], &new_heap, move);
}
} break;

default:
fprintf(stderr, "- Found unknown boxed type: %" TERM_X_FMT "\n", (t >> 2) & 0xF);
AVM_ABORT();
ptr += arity + 1;
break;
}

ptr += arity + 1;

} else if (term_is_nonempty_list(t)) {
TRACE("Found nonempty list (%p)\n", (void *) t);
*ptr = memory_shallow_copy_term(old_fragment, t, &new_heap, move);
ptr++;

} else if (term_is_boxed(t)) {
TRACE("Found boxed (%p)\n", (void *) t);
*ptr = memory_shallow_copy_term(old_fragment, t, &new_heap, move);
ptr++;

} else {
fprintf(stderr, "bug: found unknown term type: 0x%" TERM_X_FMT "\n", t);
AVM_ABORT();
case TERM_PRIMARY_LIST:
TRACE("Found nonempty list (%p)\n", (void *) t);
*ptr = memory_shallow_copy_term(old_fragment, t, &new_heap, move);
ptr++;
break;
case TERM_PRIMARY_BOXED:
TRACE("Found boxed (%p)\n", (void *) t);
*ptr = memory_shallow_copy_term(old_fragment, t, &new_heap, move);
ptr++;
break;
default:
UNREACHABLE();
}
}

Expand Down Expand Up @@ -835,90 +819,75 @@ HOT_FUNC static inline bool memory_heap_fragment_contains_pointer(HeapFragment *

HOT_FUNC static term memory_shallow_copy_term(HeapFragment *old_fragment, term t, term **new_heap, bool move)
{
if (term_is_atom(t)) {
return t;

} else if (term_is_integer(t)) {
return t;

} else if (term_is_nil(t)) {
return t;

} else if (term_is_local_pid(t)) {
return t;

} else if (term_is_local_port(t)) {
return t;

} else if (term_is_cp(t)) {
// CP is valid only on stack
return t;

} else if (term_is_catch_label(t)) {
// catch label is valid only on stack
return t;

} else if (term_is_boxed(t)) {
term *boxed_value = term_to_term_ptr(t);
// Do not GC terms from messages until the message is destroyed
if (old_fragment != NULL && !memory_heap_fragment_contains_pointer(old_fragment, boxed_value)) {
switch (t & TERM_PRIMARY_MASK) {
case TERM_PRIMARY_IMMED:
return t;
}

if (memory_is_moved_marker(boxed_value)) {
return memory_dereference_moved_marker(boxed_value);
}
case TERM_PRIMARY_CP:
// CP is valid only on stack
// catch label is valid only on stack
return t;

int boxed_size = term_boxed_size(t) + 1;
case TERM_PRIMARY_BOXED: {
term *boxed_value = term_to_term_ptr(t);
// Do not GC terms from messages until the message is destroyed
if (old_fragment != NULL && !memory_heap_fragment_contains_pointer(old_fragment, boxed_value)) {
return t;
}

// It must be an empty tuple, so we are not going to use moved markers.
// Empty tuples memory is too small to store moved markers.
// However it is also required to avoid boxed terms duplication.
// So instead all empty tuples will reference the same boxed term.
if (boxed_size == 1) {
return ((term) &empty_tuple) | TERM_PRIMARY_BOXED;
}
if (memory_is_moved_marker(boxed_value)) {
return memory_dereference_moved_marker(boxed_value);
}

term *dest = *new_heap;
for (int i = 0; i < boxed_size; i++) {
dest[i] = boxed_value[i];
}
*new_heap += boxed_size;
int boxed_size = term_boxed_size(t) + 1;

term new_term = ((term) dest) | TERM_PRIMARY_BOXED;
// It must be an empty tuple, so we are not going to use moved markers.
// Empty tuples memory is too small to store moved markers.
// However it is also required to avoid boxed terms duplication.
// So instead all empty tuples will reference the same boxed term.
if (boxed_size == 1) {
return ((term) &empty_tuple) | TERM_PRIMARY_BOXED;
}

if (move) {
memory_replace_with_moved_marker(boxed_value, new_term);
}
term *dest = *new_heap;
for (int i = 0; i < boxed_size; i++) {
dest[i] = boxed_value[i];
}
*new_heap += boxed_size;

return new_term;
term new_term = ((term) dest) | TERM_PRIMARY_BOXED;

} else if (term_is_nonempty_list(t)) {
term *list_ptr = term_get_list_ptr(t);
if (old_fragment != NULL && !memory_heap_fragment_contains_pointer(old_fragment, list_ptr)) {
return t;
}
if (move) {
memory_replace_with_moved_marker(boxed_value, new_term);
}

if (memory_is_moved_marker(list_ptr)) {
return memory_dereference_moved_marker(list_ptr);
return new_term;
}
case TERM_PRIMARY_LIST: {
term *list_ptr = term_get_list_ptr(t);
if (old_fragment != NULL && !memory_heap_fragment_contains_pointer(old_fragment, list_ptr)) {
return t;
}

term *dest = *new_heap;
dest[0] = list_ptr[0];
dest[1] = list_ptr[1];
*new_heap += 2;
if (memory_is_moved_marker(list_ptr)) {
return memory_dereference_moved_marker(list_ptr);
}

term new_term = ((term) dest) | 0x1;
term *dest = *new_heap;
dest[0] = list_ptr[0];
dest[1] = list_ptr[1];
*new_heap += 2;

if (move) {
memory_replace_with_moved_marker(list_ptr, new_term);
}
term new_term = ((term) dest) | 0x1;

return new_term;
if (move) {
memory_replace_with_moved_marker(list_ptr, new_term);
}

} else {
fprintf(stderr, "Unexpected term. Term is: %" TERM_X_FMT "\n", t);
AVM_ABORT();
return new_term;
}
default:
UNREACHABLE();
}
}

Expand Down
2 changes: 1 addition & 1 deletion src/libAtomVM/term.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ extern "C" {
#define TERM_PRIMARY_CP 0x0
#define TERM_PRIMARY_LIST 0x1
#define TERM_PRIMARY_BOXED 0x2
// #define TERM_PRIMARY_IMMED 0x3
#define TERM_PRIMARY_IMMED 0x3

#define TERM_BOXED_VALUE_TAG _Pragma ("TERM_BOXED_VALUE_TAG is deprecated, use TERM_PRIMARY_BOXED instead") TERM_PRIMARY_BOXED

Expand Down
Loading