[CORE] Change mmap flag judgment in getMmapped

[zengdage]

Change getMmapped to strictly check MEM_MMAP flag instead of MEM_ALLOCATED bitmask.

I found that some memory allocated via native malloc is also registered in mapallmem with the MEM_ALLOCATED flag, and later released through custom free routines.

[ptitSeb]

Have you tested other things with this pathc, espcialy 32bits apps/games?

[zengdage]

Have you tested other things with this pathc, espcialy 32bits apps/games?

I’m going to test box32 apps and provide more accurate change descriptions.

[zengdage]

Have you tested other things with this pathc, espcialy 32bits apps/games?

I created a test case to reproduce the issue I encountered when running x86_64 Electron on RISC-V Linux via Box64 (command: box64 ./electron -v).

When BOX64_MALLOC_HACK=2 is enabled, memory blocks allocated internally by Box64 are also tagged with the MEM_ALLOCATED flag. As a result, these blocks get incorrectly freed via the x86_64 custom free routine, triggering a crash.

I believe using only the MEM_ALLOCATED flag to decide whether to invoke the native free or the x86_64 custom free is unsafe. Similarly, relying solely on the MEM_MMAP flag for this branching logic is not fully reliable either. The correctness depends entirely on the underlying implementation of the x86_64 custom allocator:

• If the x86_64 custom allocator backs its allocations with mmap, then checking MEM_MMAP would work safely.
• If it uses brk instead, those allocated blocks will lack the MEM_MMAP flag, and the memory will mistakenly be released via native free, causing exceptions.

In my opinion, the most robust solution would be to explicitly track every memory block allocated through the x86_64 custom memory allocator. However, this approach could introduce performance overhead.

What would you recommend as the proper fix here?

g++ -o malloc_custom_detect malloc_custom_detect.cpp -ldl

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <dlfcn.h>
#include <sys/mman.h>
#include <sys/wait.h>
#include <unistd.h>
#include <new>

/* ---------- magic header ---------- */
#define CUSTOM_MAGIC  0xDEADBEEFCAFEBABEULL
#define PAGE_SIZE     4096

typedef struct {
    uint64_t magic;
    size_t   user_size;   /* requested size */
    size_t   mmap_size;   /* actual mmap'd size (for munmap) */
} alloc_header_t;

#define HEADER_SIZE  (sizeof(alloc_header_t))

/* Round up to page boundary */
static size_t page_align(size_t sz) {
    return (sz + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
}

/* ---------- custom allocator: mmap per allocation ---------- */
static void* custom_alloc(size_t sz, const char* tag) {
    size_t total    = HEADER_SIZE + sz;
    size_t map_size = page_align(total);

    void* raw = mmap(nullptr, map_size, PROT_READ | PROT_WRITE,
                     MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
    if (raw == MAP_FAILED) {
        fprintf(stderr, "[CUSTOM] %s(%zu): mmap failed!\n", tag, sz);
        return nullptr;
    }

    alloc_header_t* hdr = (alloc_header_t*)raw;
    hdr->magic     = CUSTOM_MAGIC;
    hdr->user_size = sz;
    hdr->mmap_size = map_size;

    void* user_ptr = (char*)raw + HEADER_SIZE;
    fprintf(stderr, "[CUSTOM] %s(%zu) = %p  [mmap %zu bytes at %p]\n",
            tag, sz, user_ptr, map_size, raw);
    return user_ptr;
}

static void custom_dealloc(void* user_ptr, const char* tag) {
    if (!user_ptr) return;

    alloc_header_t* hdr = (alloc_header_t*)((char*)user_ptr - HEADER_SIZE);
    if (hdr->magic != CUSTOM_MAGIC) {
        fprintf(stderr, "[CUSTOM] FATAL: %s(%p) — magic mismatch! "
                        "got=0x%llx, expected=0x%llx. "
                        "Pointer was NOT allocated by custom allocator!\n",
                tag, user_ptr,
                (unsigned long long)hdr->magic,
                (unsigned long long)CUSTOM_MAGIC);
        abort();
    }

    size_t map_size = hdr->mmap_size;
    fprintf(stderr, "[CUSTOM] %s(%p)  [munmap %zu bytes at %p]\n",
            tag, user_ptr, map_size, (void*)hdr);
    hdr->magic = 0;
    munmap(hdr, map_size);
}

/* ---------- early-alloc buffer for dlsym bootstrap ---------- */
static char early_buf[8192];
static int  early_used = 0;

/* ---------- override malloc/free ---------- */
extern "C" {

void* malloc(size_t sz) {
    return custom_alloc(sz, "malloc");
}

void free(void* p) {
    if (!p) return;
    if ((char*)p >= early_buf && (char*)p < early_buf + sizeof(early_buf))
        return;
    custom_dealloc(p, "free");
}

void* calloc(size_t n, size_t sz) {
    /*
     * dlsym internally calls calloc before we can resolve anything.
     * Use static early_buf for those early calls.
     */
    size_t total = n * sz;
    if (total == 0) total = 1;

    /* Try early buffer first during bootstrap */
    if (early_used + (int)total <= (int)sizeof(early_buf)) {
        void* p = early_buf + early_used;
        early_used += ((total + 15) & ~15);  /* align */
        memset(p, 0, total);
        return p;
    }

    void* p = custom_alloc(total, "calloc");
    if (p) memset(p, 0, total);
    return p;
}

void* realloc(void* p, size_t sz) {
    if (!p) return custom_alloc(sz, "realloc");
    if ((char*)p >= early_buf && (char*)p < early_buf + sizeof(early_buf)) {
        /* Can't realloc early_buf, just allocate new */
        return custom_alloc(sz, "realloc(early)");
    }

    alloc_header_t* old_hdr = (alloc_header_t*)((char*)p - HEADER_SIZE);
    if (old_hdr->magic != CUSTOM_MAGIC) return nullptr;

    size_t old_sz = old_hdr->user_size;
    void* new_p = custom_alloc(sz, "realloc");
    if (new_p) {
        memcpy(new_p, p, old_sz < sz ? old_sz : sz);
        custom_dealloc(p, "realloc-free-old");
    }
    return new_p;
}

}  /* extern "C" */

/* ---------- override C++ operators ---------- */

/* _Znwm */
void* operator new(size_t sz) {
    return custom_alloc(sz, "_Znwm");
}

/* _Znam */
void* operator new[](size_t sz) {
    return custom_alloc(sz, "_Znam");
}

/* _ZdlPv */
void operator delete(void* p) noexcept {
    if (p) custom_dealloc(p, "_ZdlPv");
}

/* _ZdaPv */
void operator delete[](void* p) noexcept {
    if (p) custom_dealloc(p, "_ZdaPv");
}

/* _ZdlPvm */
void operator delete(void* p, size_t sz) noexcept {
    (void)sz;
    if (p) custom_dealloc(p, "_ZdlPvm");
}

/* _ZdaPvm */
void operator delete[](void* p, size_t sz) noexcept {
    (void)sz;
    if (p) custom_dealloc(p, "_ZdaPvm");
}

/* Resolve glibc's real malloc for simulating "glibc-allocated" pointers */
static void* (*real_malloc)(size_t) = nullptr;

static void resolve_glibc_malloc() {
    if (!real_malloc)
        real_malloc = (void*(*)(size_t))dlsym(RTLD_NEXT, "malloc");
}

/* --- Normal round-trip: custom new/delete (no abort) --- */
static void test_custom_new_delete() {
    fprintf(stderr, "\n=== TEST: custom _Znwm/_Znam round-trip (mmap-backed) ===\n");

    int* q1 = new int(42);
    fprintf(stderr, "  new int(42) = %p, val=%d\n", q1, *q1);

    char* q2 = new char[256];
    memset(q2, 'X', 255);
    q2[255] = '\0';
    fprintf(stderr, "  new char[256] = %p\n", q2);

    delete q1;
    delete[] q2;
    fprintf(stderr, "  OK\n");
}

int main() {
    int pass = 0, total = 0;

    test_custom_new_delete();

    fprintf(stderr, "\n=== Result: %d/%d abort-detection tests passed ===\n", pass, total);
    return (pass == total) ? 0 : 1;
}

[ptitSeb]

Mmmm, you have a x86_64 custom allocator that use brk ?