From 75da4e9600743cfc095b5202f98c21d01c8a1cb2 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Mon, 16 May 2016 11:54:14 -0400 Subject: [PATCH] Flip freemap (and rename to allocmap) Remove the `memset(0xff)` when allocating a region. --- src/gc-debug.c | 8 ++++---- src/gc-pages.c | 30 ++++++++++++++++-------------- src/gc.c | 6 +++--- src/gc.h | 2 +- 4 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/gc-debug.c b/src/gc-debug.c index 1a6d5e0637a39..fd3bcf4324c9d 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -37,7 +37,7 @@ JL_DLLEXPORT jl_taggedvalue_t *jl_gc_find_taggedvalue_pool(char *p, size_t *osiz size_t ofs = p - page_begin; int pg_idx = page_index(r, page_begin); // Check if this is a free page - if (r->freemap[pg_idx / 32] & (uint32_t)(1 << (pg_idx % 32))) + if (!(r->allocmap[pg_idx / 32] & (uint32_t)(1 << (pg_idx % 32)))) return NULL; jl_gc_pagemeta_t *pagemeta = &r->meta[pg_idx]; int osize = pagemeta->osize; @@ -134,10 +134,10 @@ static void clear_mark(int bits) if (!region->pages) break; for (int pg_i = 0; pg_i < region->pg_cnt / 32; pg_i++) { - uint32_t line = region->freemap[pg_i]; - if (!!~line) { + uint32_t line = region->allocmap[pg_i]; + if (line) { for (int j = 0; j < 32; j++) { - if (!((line >> j) & 1)) { + if ((line >> j) & 1) { jl_gc_pagemeta_t *pg = page_metadata(region->pages[pg_i*32 + j].data + GC_PAGE_OFFSET); jl_tls_states_t *ptls = jl_all_task_states[pg->thread_n].ptls; diff --git a/src/gc-pages.c b/src/gc-pages.c index 732a1d555abf0..f3b301c6a36cb 100644 --- a/src/gc-pages.c +++ b/src/gc-pages.c @@ -77,7 +77,7 @@ static void jl_gc_alloc_region(region_t *region) { int pg_cnt = region_pg_cnt; const size_t pages_sz = sizeof(jl_gc_page_t) * pg_cnt; - const size_t freemap_sz = sizeof(uint32_t) * pg_cnt / 32; + const size_t allocmap_sz = sizeof(uint32_t) * pg_cnt / 32; char *mem = NULL; while (1) { if (__likely((mem = jl_gc_try_alloc_region(pg_cnt)))) @@ -95,17 +95,16 @@ static void jl_gc_alloc_region(region_t *region) } } region->pages = (jl_gc_page_t*)mem; - region->freemap = (uint32_t*)(mem + pages_sz); - region->meta = (jl_gc_pagemeta_t*)(mem + pages_sz +freemap_sz); + region->allocmap = (uint32_t*)(mem + pages_sz); + region->meta = (jl_gc_pagemeta_t*)(mem + pages_sz +allocmap_sz); region->lb = 0; region->ub = 0; region->pg_cnt = pg_cnt; #ifdef _OS_WINDOWS_ - VirtualAlloc(region->freemap, pg_cnt / 8, MEM_COMMIT, PAGE_READWRITE); + VirtualAlloc(region->allocmap, pg_cnt / 8, MEM_COMMIT, PAGE_READWRITE); VirtualAlloc(region->meta, pg_cnt * sizeof(jl_gc_pagemeta_t), MEM_COMMIT, PAGE_READWRITE); #endif - memset(region->freemap, 0xff, pg_cnt / 8); } NOINLINE void *jl_gc_alloc_page(void) @@ -119,7 +118,7 @@ NOINLINE void *jl_gc_alloc_page(void) if (region->pages == NULL) jl_gc_alloc_region(region); for (i = region->lb; i < region->pg_cnt / 32; i++) { - if (region->freemap[i]) + if (~region->allocmap[i]) break; } if (i == region->pg_cnt / 32) { @@ -139,15 +138,15 @@ NOINLINE void *jl_gc_alloc_page(void) region->ub = i; #if defined(_COMPILER_MINGW_) - int j = __builtin_ffs(region->freemap[i]) - 1; + int j = __builtin_ffs(~region->allocmap[i]) - 1; #elif defined(_COMPILER_MICROSOFT_) unsigned long j; - _BitScanForward(&j, region->freemap[i]); + _BitScanForward(&j, ~region->allocmap[i]); #else - int j = ffs(region->freemap[i]) - 1; + int j = ffs(~region->allocmap[i]) - 1; #endif - region->freemap[i] &= ~(uint32_t)(1 << j); + region->allocmap[i] |= (uint32_t)(1 << j); void *ptr = region->pages[i * 32 + j].data; #ifdef _OS_WINDOWS_ VirtualAlloc(ptr, GC_PAGE_SZ, MEM_COMMIT, PAGE_READWRITE); @@ -174,8 +173,8 @@ void jl_gc_free_page(void *p) } assert(i < REGION_COUNT && region->pages != NULL); uint32_t msk = (uint32_t)(1 << (pg_idx % 32)); - assert(!(region->freemap[pg_idx/32] & msk)); - region->freemap[pg_idx/32] ^= msk; + assert(region->allocmap[pg_idx/32] & msk); + region->allocmap[pg_idx/32] ^= msk; free(region->meta[pg_idx].ages); // tell the OS we don't need these pages right now size_t decommit_size = GC_PAGE_SZ; @@ -185,10 +184,13 @@ void jl_gc_free_page(void *p) decommit_size = jl_page_size; p = (void*)((uintptr_t)region->pages[pg_idx].data & ~(jl_page_size - 1)); // round down to the nearest page pg_idx = page_index(region, p); - if (pg_idx + n_pages > region->pg_cnt) goto no_decommit; + if (pg_idx + n_pages > region->pg_cnt) + goto no_decommit; for (; n_pages--; pg_idx++) { msk = (uint32_t)(1 << ((pg_idx % 32))); - if (!(region->freemap[pg_idx/32] & msk)) goto no_decommit; + if (region->allocmap[pg_idx / 32] & msk) { + goto no_decommit; + } } } #ifdef _OS_WINDOWS_ diff --git a/src/gc.c b/src/gc.c index 2e17434f2cc5a..3eba38be9d2d8 100644 --- a/src/gc.c +++ b/src/gc.c @@ -806,11 +806,11 @@ static void sweep_pool_region(gcval_t ***pfl, int region_i, int sweep_mask) int ub = 0; int lb = region->lb; for (int pg_i = 0; pg_i <= region->ub; pg_i++) { - uint32_t line = region->freemap[pg_i]; - if (!!~line) { + uint32_t line = region->allocmap[pg_i]; + if (line) { ub = pg_i; for (int j = 0; j < 32; j++) { - if (!((line >> j) & 1)) { + if ((line >> j) & 1) { jl_gc_pagemeta_t *pg = ®ion->meta[pg_i*32 + j]; int p_n = pg->pool_n; int t_n = pg->thread_n; diff --git a/src/gc.h b/src/gc.h index 295d502bf8aa1..f6276661050e0 100644 --- a/src/gc.h +++ b/src/gc.h @@ -167,7 +167,7 @@ typedef struct { // Tag: sizeof_jl_taggedvalue_t // Data: <= osize - sizeof_jl_taggedvalue_t jl_gc_page_t *pages; // [pg_cnt]; must be first, to preserve page alignment - uint32_t *freemap; // [pg_cnt / 32] + uint32_t *allocmap; // [pg_cnt / 32] jl_gc_pagemeta_t *meta; // [pg_cnt] int pg_cnt; // store a lower bound of the first free page in each region