diff -r -c ./libphobos/core/thread.d ./libphobos/core/thread.d *** ./libphobos/core/thread.d 2011-04-29 22:13:02.000000000 +0900 --- ./libphobos/core/thread.d 2011-05-13 01:19:57.453125000 +0900 *************** *** 91,97 **** extern (C) void* rt_stackTop(); extern (C) void rt_moduleTlsCtor(); extern (C) void rt_moduleTlsDtor(); ! extern (C) void rt_processGCMarks(void[]); void* getStackBottom() --- 91,98 ---- extern (C) void* rt_stackTop(); extern (C) void rt_moduleTlsCtor(); extern (C) void rt_moduleTlsDtor(); ! extern (C) void rt_processGCMarks(void[],size_t); ! extern (C) size_t rt_get_blkcache_offset(void[]) ; void* getStackBottom() *************** *** 1361,1366 **** --- 1362,1368 ---- auto pstart = cast(void*) &_tlsstart; auto pend = cast(void*) &_tlsend; m_tls = pstart[0 .. pend - pstart]; + blkcache_offset = rt_get_blkcache_offset(m_tls) ; } } *************** *** 1526,1531 **** --- 1528,1534 ---- } + size_t blkcache_offset ; Context m_main; Context* m_curr; bool m_lock; *************** *** 2609,2615 **** { for( Thread t = Thread.sm_tbeg; t; t = t.next ) { ! rt_processGCMarks(t.m_tls); } } --- 2612,2618 ---- { for( Thread t = Thread.sm_tbeg; t; t = t.next ) { ! rt_processGCMarks(t.m_tls, t.blkcache_offset); } } diff -r -c ./libphobos/gc/gcx.d ./libphobos/gc/gcx.d *** ./libphobos/gc/gcx.d 2011-04-29 22:13:02.000000000 +0900 --- ./libphobos/gc/gcx.d 2011-05-07 17:13:34.187500000 +0900 *************** *** 48,53 **** --- 48,63 ---- debug (COLLECT_PRINTF) import core.stdc.stdio : printf; debug private import core.stdc.stdio; + debug(PRINTF) void printFreeInfo(Pool* pool) + { + uint nReallyFree; + foreach(i; 0..pool.npages) { + if(pool.pagetable[i] >= B_FREE) nReallyFree++; + } + + printf("Pool %p: %d really free, %d supposedly be free\n", pool, nReallyFree, pool.freepages); + } + private { enum USE_CACHE = true; *************** *** 306,312 **** if (pool) { ! auto biti = cast(size_t)(p - pool.baseAddr) / 16; oldb = gcx.getBits(pool, biti); } --- 316,322 ---- if (pool) { ! auto biti = cast(size_t)(p - pool.baseAddr) / pool.divisor; oldb = gcx.getBits(pool, biti); } *************** *** 341,347 **** if (pool) { ! auto biti = cast(size_t)(p - pool.baseAddr) / 16; oldb = gcx.getBits(pool, biti); gcx.setBits(pool, biti, mask); --- 351,357 ---- if (pool) { ! auto biti = cast(size_t)(p - pool.baseAddr) / pool.divisor; oldb = gcx.getBits(pool, biti); gcx.setBits(pool, biti, mask); *************** *** 377,383 **** if (pool) { ! auto biti = cast(size_t)(p - pool.baseAddr) / 16; oldb = gcx.getBits(pool, biti); gcx.clrBits(pool, biti, mask); --- 387,393 ---- if (pool) { ! auto biti = cast(size_t)(p - pool.baseAddr) / pool.divisor; oldb = gcx.getBits(pool, biti); gcx.clrBits(pool, biti, mask); *************** *** 467,480 **** * of full pages freed. Perhaps this should instead be the amount of * memory freed. */ ! gcx.newPool(1); state = 2; } else state = 1; continue; case 1: ! gcx.newPool(1); state = 2; continue; case 2: --- 477,490 ---- * of full pages freed. Perhaps this should instead be the amount of * memory freed. */ ! gcx.newPool(1,false); state = 2; } else state = 1; continue; case 1: ! gcx.newPool(1, false); state = 2; continue; case 2: *************** *** 511,517 **** Pool *pool = gcx.findPool(p); assert(pool); ! gcx.setBits(pool, cast(size_t)(p - pool.baseAddr) / 16, bits); } return p; } --- 521,527 ---- Pool *pool = gcx.findPool(p); assert(pool); ! gcx.setBits(pool, cast(size_t)(p - pool.baseAddr) / pool.divisor, bits); } return p; } *************** *** 602,608 **** if (pool) { ! auto biti = cast(size_t)(p - pool.baseAddr) / 16; if (bits) { --- 612,618 ---- if (pool) { ! auto biti = cast(size_t)(p - pool.baseAddr) / pool.divisor; if (bits) { *************** *** 642,647 **** --- 652,658 ---- { debug (MEMSTOMP) memset(p + size, 0xF2, psize - size); pool.freePages(pagenum + newsz, psz - newsz); + pool.updateOffsets(pagenum); } if(alloc_size) *alloc_size = newsz * PAGESIZE; *************** *** 657,665 **** --- 668,680 ---- if (i == pagenum + newsz) { debug (MEMSTOMP) memset(p + psize, 0xF0, size - psize); + debug(PRINTF) printFreeInfo(pool); memset(&pool.pagetable[pagenum + psz], B_PAGEPLUS, newsz - psz); + pool.updateOffsets(pagenum); if(alloc_size) *alloc_size = newsz * PAGESIZE; + pool.freepages -= (newsz - psz); + debug(PRINTF) printFreeInfo(pool); return p; } if (i == pool.ncommitted) *************** *** 686,692 **** if (pool) { ! auto biti = cast(size_t)(p - pool.baseAddr) / 16; if (bits) { --- 701,707 ---- if (pool) { ! auto biti = cast(size_t)(p - pool.baseAddr) / pool.divisor; if (bits) { *************** *** 795,800 **** --- 810,817 ---- return 0; debug (MEMSTOMP) memset(p + psize, 0xF0, (psz + sz) * PAGESIZE - psize); memset(pool.pagetable + pagenum + psz, B_PAGEPLUS, sz); + pool.updateOffsets(pagenum); + pool.freepages -= sz; if (p == gcx.cached_size_key) gcx.cached_size_val = (psz + sz) * PAGESIZE; if (p == gcx.cached_info_key) *************** *** 862,867 **** --- 879,885 ---- // private void freeNoSync(void *p) { + debug(PRINTF) printf("Freeing %p\n", cast(size_t) p); assert (p); Pool* pool; *************** *** 876,894 **** sentinel_Invariant(p); p = sentinel_sub(p); pagenum = cast(size_t)(p - pool.baseAddr) / PAGESIZE; ! biti = cast(size_t)(p - pool.baseAddr) / 16; gcx.clrBits(pool, biti, BlkAttr.ALL_BITS); bin = cast(Bins)pool.pagetable[pagenum]; if (bin == B_PAGE) // if large alloc { size_t npages; - size_t n; // Free pages ! npages = 1; ! n = pagenum; ! while (++n < pool.ncommitted && pool.pagetable[n] == B_PAGEPLUS) ! npages++; debug (MEMSTOMP) memset(p, 0xF2, npages * PAGESIZE); pool.freePages(pagenum, npages); } --- 894,912 ---- sentinel_Invariant(p); p = sentinel_sub(p); pagenum = cast(size_t)(p - pool.baseAddr) / PAGESIZE; ! ! debug(PRINTF) printf("pool base = %p, PAGENUM = %d of %d / %d, bin = %d\n", pool.baseAddr, pagenum, pool.ncommitted, pool.npages, pool.pagetable[pagenum]); ! debug(PRINTF) if(pool.isLargeObject) printf("Block size = %d\n", pool.bPageOffsets[pagenum]); ! biti = cast(size_t)(p - pool.baseAddr) / pool.divisor; ! gcx.clrBits(pool, biti, BlkAttr.ALL_BITS); bin = cast(Bins)pool.pagetable[pagenum]; if (bin == B_PAGE) // if large alloc { size_t npages; // Free pages ! npages = pool.bPageOffsets[pagenum]; debug (MEMSTOMP) memset(p, 0xF2, npages * PAGESIZE); pool.freePages(pagenum, npages); } *************** *** 1758,1766 **** } else if (bin == B_PAGEPLUS) { ! do ! { --pn, offset -= PAGESIZE; ! } while (cast(Bins)pool.pagetable[pn] == B_PAGEPLUS); return pool.baseAddr + (offset & (offset.max ^ (PAGESIZE-1))); } --- 1776,1784 ---- } else if (bin == B_PAGEPLUS) { ! auto pageOffset = pool.bPageOffsets[pn]; ! offset -= pageOffset * PAGESIZE; ! pn -= pageOffset; return pool.baseAddr + (offset & (offset.max ^ (PAGESIZE-1))); } *************** *** 1796,1812 **** bin = cast(Bins)pool.pagetable[pagenum]; size = binsize[bin]; if (bin == B_PAGE) ! { size_t npages = pool.ncommitted; ! ubyte* pt; ! size_t i; ! ! pt = &pool.pagetable[0]; ! for (i = pagenum + 1; i < npages; i++) ! { ! if (pt[i] != B_PAGEPLUS) ! break; ! } ! size = (i - pagenum) * PAGESIZE; } cached_size_key = p; cached_size_val = size; --- 1814,1821 ---- bin = cast(Bins)pool.pagetable[pagenum]; size = binsize[bin]; if (bin == B_PAGE) ! { ! size = pool.bPageOffsets[pagenum] * PAGESIZE; } cached_size_key = p; cached_size_val = size; *************** *** 1843,1852 **** } else if (bin == B_PAGEPLUS) { ! do ! { --pn, offset -= PAGESIZE; ! } while (cast(Bins)pool.pagetable[pn] == B_PAGEPLUS); ! info.base = pool.baseAddr + (offset & (offset.max ^ (PAGESIZE-1))); // fix bin for use by size calc below --- 1852,1860 ---- } else if (bin == B_PAGEPLUS) { ! auto pageOffset = pool.bPageOffsets[pn]; ! offset = pageOffset * PAGESIZE; ! pn -= pageOffset; info.base = pool.baseAddr + (offset & (offset.max ^ (PAGESIZE-1))); // fix bin for use by size calc below *************** *** 1859,1875 **** info.size = binsize[bin]; if (bin == B_PAGE) ! { size_t npages = pool.ncommitted; ! ubyte* pt; ! size_t i; ! ! pt = &pool.pagetable[0]; ! for (i = pn + 1; i < npages; i++) ! { ! if (pt[i] != B_PAGEPLUS) ! break; ! } ! info.size = (i - pn) * PAGESIZE; } //////////////////////////////////////////////////////////////////// --- 1867,1874 ---- info.size = binsize[bin]; if (bin == B_PAGE) ! { ! info.size = pool.bPageOffsets[pn] * PAGESIZE; } //////////////////////////////////////////////////////////////////// *************** *** 1879,1885 **** // reset the offset to the base pointer, otherwise the bits // are the bits for the pointer, which may be garbage offset = cast(size_t)(info.base - pool.baseAddr); ! info.attr = getBits(pool, cast(size_t)(offset / 16)); cached_info_key = p; cached_info_val = info; --- 1878,1884 ---- // reset the offset to the base pointer, otherwise the bits // are the bits for the pointer, which may be garbage offset = cast(size_t)(info.base - pool.baseAddr); ! info.attr = getBits(pool, cast(size_t)(offset / pool.divisor)); cached_info_key = p; cached_info_val = info; *************** *** 1943,1949 **** size_t reserve(size_t size) { size_t npages = (size + PAGESIZE - 1) / PAGESIZE; ! Pool* pool = newPool(npages); if (!pool || pool.extendPages(npages) == OPFAIL) return 0; --- 1942,1950 ---- size_t reserve(size_t size) { size_t npages = (size + PAGESIZE - 1) / PAGESIZE; ! ! // Assume reserve() is for small objects. ! Pool* pool = newPool(npages, false); if (!pool || pool.extendPages(npages) == OPFAIL) return 0; *************** *** 1956,1977 **** */ void minimize() { size_t n; size_t pn; Pool* pool; size_t ncommitted; for (n = 0; n < npools; n++) { pool = pooltable[n]; ! ncommitted = pool.ncommitted; ! for (pn = 0; pn < ncommitted; pn++) ! { ! if (cast(Bins)pool.pagetable[pn] != B_FREE) ! break; ! } ! if (pn < ncommitted) ! continue; pool.Dtor(); cstdlib.free(pool); memmove(pooltable + n, --- 1957,1974 ---- */ void minimize() { + debug(PRINTF) printf("Minimizing.\n"); size_t n; size_t pn; Pool* pool; size_t ncommitted; + Outer: for (n = 0; n < npools; n++) { pool = pooltable[n]; ! debug(PRINTF) printFreeInfo(pool); ! if(pool.freepages < pool.npages) continue; pool.Dtor(); cstdlib.free(pool); memmove(pooltable + n, *************** *** 1981,1986 **** --- 1978,1984 ---- } minAddr = pooltable[0].baseAddr; maxAddr = pooltable[npools - 1].topAddr; + debug(PRINTF) printf("Done minimizing.\n"); } *************** *** 1990,1995 **** --- 1988,1995 ---- */ void *bigAlloc(size_t size, size_t *alloc_size = null) { + debug(PRINTF) printf("In bigAlloc. Size: %d\n", size); + Pool* pool; size_t npages; size_t n; *************** *** 2009,2014 **** --- 2009,2015 ---- for (n = 0; n < npools; n++) { pool = pooltable[n]; + if(!pool.isLargeObject || pool.freepages < npages) continue; pn = pool.allocPages(npages); if (pn != OPFAIL) goto L1; *************** *** 2028,2034 **** // Release empty pools to prevent bloat minimize(); // Allocate new pool ! pool = newPool(npages); if (!pool) { state = 2; continue; --- 2029,2035 ---- // Release empty pools to prevent bloat minimize(); // Allocate new pool ! pool = newPool(npages, true); if (!pool) { state = 2; continue; *************** *** 2040,2046 **** // Release empty pools to prevent bloat minimize(); // Allocate new pool ! pool = newPool(npages); if (!pool) { if (collected) --- 2041,2047 ---- // Release empty pools to prevent bloat minimize(); // Allocate new pool ! pool = newPool(npages, true); if (!pool) { if (collected) *************** *** 2059,2068 **** --- 2060,2076 ---- } L1: + debug(PRINTF) printFreeInfo(pool); pool.pagetable[pn] = B_PAGE; if (npages > 1) memset(&pool.pagetable[pn + 1], B_PAGEPLUS, npages - 1); + pool.updateOffsets(pn); + pool.freepages -= npages; + + debug(PRINTF) printFreeInfo(pool); + p = pool.baseAddr + pn * PAGESIZE; + debug(PRINTF) printf("Got large alloc: %p, pt = %d, np = %d\n", p, pool.pagetable[pn], npages); memset(cast(char *)p + size, 0, npages * PAGESIZE - size); debug (MEMSTOMP) memset(p, 0xF1, size); if(alloc_size) *************** *** 2080,2086 **** * Sort it into pooltable[]. * Return null if failed. */ ! Pool *newPool(size_t npages) { Pool* pool; Pool** newpooltable; --- 2088,2094 ---- * Sort it into pooltable[]. * Return null if failed. */ ! Pool *newPool(size_t npages, bool isLargeObject) { Pool* pool; Pool** newpooltable; *************** *** 2121,2127 **** pool = cast(Pool *)cstdlib.calloc(1, Pool.sizeof); if (pool) { ! pool.initialize(npages); if (!pool.baseAddr) goto Lerr; --- 2129,2135 ---- pool = cast(Pool *)cstdlib.calloc(1, Pool.sizeof); if (pool) { ! pool.initialize(npages, isLargeObject); if (!pool.baseAddr) goto Lerr; *************** *** 2171,2176 **** --- 2179,2185 ---- for (n = 0; n < npools; n++) { pool = pooltable[n]; + if(pool.isLargeObject) continue; pn = pool.allocPages(1); if (pn != OPFAIL) goto L1; *************** *** 2179,2184 **** --- 2188,2194 ---- L1: pool.pagetable[pn] = cast(ubyte)bin; + pool.freepages--; // Convert page to free list size_t size = binsize[bin]; *************** *** 2229,2250 **** // Adjust bit to be at start of allocated memory block if (bin < B_PAGE) { ! biti = (offset & notbinsize[bin]) >> 4; //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); } else if (bin == B_PAGE) { ! biti = (offset & notbinsize[bin]) >> 4; //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); } else if (bin == B_PAGEPLUS) { ! do ! { --pn; ! } while (cast(Bins)pool.pagetable[pn] == B_PAGEPLUS); ! biti = pn * (PAGESIZE / 16); pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); } --- 2239,2258 ---- // Adjust bit to be at start of allocated memory block if (bin < B_PAGE) { ! biti = (offset & notbinsize[bin]) / pool.divisor; //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); } else if (bin == B_PAGE) { ! biti = (offset & notbinsize[bin]) / pool.divisor; //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); } else if (bin == B_PAGEPLUS) { ! pn -= pool.bPageOffsets[pn]; ! biti = pn * (PAGESIZE / pool.divisor); pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); } *************** *** 2263,2269 **** pool.scan.set(biti); changes = 1; } ! debug (LOGGING) log_parent(sentinel_add(pool.baseAddr + biti * 16), sentinel_add(pbot)); } } } --- 2271,2277 ---- pool.scan.set(biti); changes = 1; } ! debug (LOGGING) log_parent(sentinel_add(pool.baseAddr + biti * pool.divisor), sentinel_add(pbot)); } } } *************** *** 2392,2397 **** --- 2400,2407 ---- pool.freebits.zero(); } + debug(COLLECT_PRINTF) printf("Set bits\n"); + // Mark each free entry, so it doesn't get scanned for (n = 0; n < B_PAGE; n++) { *************** *** 2403,2408 **** --- 2413,2420 ---- } } + debug(COLLECT_PRINTF) printf("Marked free entries.\n"); + for (n = 0; n < npools; n++) { pool = pooltable[n]; *************** *** 2413,2418 **** --- 2425,2431 ---- { if (!noStack) { + debug(COLLECT_PRINTF) printf("scanning multithreaded stack.\n"); // Scan stacks and registers for each paused thread thread_scanAll( &mark, stackTop ); } *************** *** 2465,2482 **** } *b = 0; ! auto o = pool.baseAddr + (b - bbase) * (typeof(bitm).sizeof*8) * 16; if (!(bitm & 0xFFFF)) { bitm >>= 16; ! o += 16 * 16; } if (!(bitm & 0xFF)) { bitm >>= 8; ! o += 8 * 16; } ! for (; bitm; o += 16, bitm >>= 1) { if (!(bitm & 1)) continue; --- 2478,2495 ---- } *b = 0; ! auto o = pool.baseAddr + (b - bbase) * (typeof(bitm).sizeof*8) * pool.divisor; if (!(bitm & 0xFFFF)) { bitm >>= 16; ! o += 16 * pool.divisor; } if (!(bitm & 0xFF)) { bitm >>= 8; ! o += 8 * pool.divisor; } ! for (; bitm; o += pool.divisor, bitm >>= 1) { if (!(bitm & 1)) continue; *************** *** 2491,2502 **** { if (bin == B_PAGEPLUS) { ! while (pool.pagetable[pn - 1] != B_PAGE) ! pn--; } ! auto u = 1; ! while (pn + u < pool.ncommitted && pool.pagetable[pn + u] == B_PAGEPLUS) ! u++; mark(o, o + u * PAGESIZE); } } --- 2504,2512 ---- { if (bin == B_PAGEPLUS) { ! pn -= pool.bPageOffsets[pn]; } ! auto u = pool.bPageOffsets[pn]; mark(o, o + u * PAGESIZE); } } *************** *** 2517,2585 **** pool = pooltable[n]; auto bbase = pool.mark.base(); auto ncommitted = pool.ncommitted; - for (pn = 0; pn < ncommitted; pn++, bbase += PAGESIZE / (32 * 16)) - { - Bins bin = cast(Bins)pool.pagetable[pn]; - - if (bin < B_PAGE) - { byte* p; - byte* ptop; - size_t biti; - size_t bitstride; - auto size = binsize[bin]; - - p = pool.baseAddr + pn * PAGESIZE; - ptop = p + PAGESIZE; - biti = pn * (PAGESIZE/16); - bitstride = size / 16; - - version(none) // BUG: doesn't work because freebits() must also be cleared - { - // If free'd entire page - if (bbase[0] == 0 && bbase[1] == 0 && bbase[2] == 0 && bbase[3] == 0 && - bbase[4] == 0 && bbase[5] == 0 && bbase[6] == 0 && bbase[7] == 0) - { - for (; p < ptop; p += size, biti += bitstride) - { - if (pool.finals.nbits && pool.finals.testClear(biti)) - rt_finalize(cast(List *)sentinel_add(p), false/*noStack > 0*/); - gcx.clrBits(pool, biti, BlkAttr.ALL_BITS); - - List *list = cast(List *)p; - //debug(PRINTF) printf("\tcollecting %p\n", list); - log_free(sentinel_add(list)); ! debug (MEMSTOMP) memset(p, 0xF3, size); ! } ! pool.pagetable[pn] = B_FREE; ! freed += PAGESIZE; ! //debug(PRINTF) printf("freeing entire page %d\n", pn); ! continue; ! } ! } ! for (; p < ptop; p += size, biti += bitstride) ! { ! if (!pool.mark.test(biti)) ! { ! sentinel_Invariant(sentinel_add(p)); ! ! pool.freebits.set(biti); ! if (pool.finals.nbits && pool.finals.testClear(biti)) ! rt_finalize(cast(List *)sentinel_add(p), false/*noStack > 0*/); ! clrBits(pool, biti, BlkAttr.ALL_BITS); ! ! List *list = cast(List *)p; ! debug(PRINTF) printf("\tcollecting %p\n", list); ! log_free(sentinel_add(list)); ! ! debug (MEMSTOMP) memset(p, 0xF3, size); ! ! freed += size; ! } ! } ! } ! else if (bin == B_PAGE) ! { size_t biti = pn * (PAGESIZE / 16); if (!pool.mark.test(biti)) { byte *p = pool.baseAddr + pn * PAGESIZE; --- 2527,2540 ---- pool = pooltable[n]; auto bbase = pool.mark.base(); auto ncommitted = pool.ncommitted; ! if(pool.isLargeObject) ! { ! for(pn = 0; pn < ncommitted; pn++) ! { ! Bins bin = cast(Bins)pool.pagetable[pn]; ! if(bin > B_PAGE) continue; ! size_t biti = pn; if (!pool.mark.test(biti)) { byte *p = pool.baseAddr + pn * PAGESIZE; *************** *** 2593,2603 **** --- 2548,2561 ---- log_free(sentinel_add(p)); pool.pagetable[pn] = B_FREE; freedpages++; + pool.freepages++; + debug (MEMSTOMP) memset(p, 0xF3, PAGESIZE); while (pn + 1 < ncommitted && pool.pagetable[pn + 1] == B_PAGEPLUS) { pn++; pool.pagetable[pn] = B_FREE; + pool.freepages++; freedpages++; debug (MEMSTOMP) *************** *** 2607,2612 **** --- 2565,2639 ---- } } } + + continue; + } + else + { + + for (pn = 0; pn < ncommitted; pn++, bbase += PAGESIZE / (32 * 16)) + { + Bins bin = cast(Bins)pool.pagetable[pn]; + + if (bin < B_PAGE) + { byte* p; + byte* ptop; + size_t biti; + size_t bitstride; + auto size = binsize[bin]; + + p = pool.baseAddr + pn * PAGESIZE; + ptop = p + PAGESIZE; + biti = pn * (PAGESIZE/16); + bitstride = size / 16; + + version(none) // BUG: doesn't work because freebits() must also be cleared + { + // If free'd entire page + if (bbase[0] == 0 && bbase[1] == 0 && bbase[2] == 0 && bbase[3] == 0 && + bbase[4] == 0 && bbase[5] == 0 && bbase[6] == 0 && bbase[7] == 0) + { + for (; p < ptop; p += size, biti += bitstride) + { + if (pool.finals.nbits && pool.finals.testClear(biti)) + rt_finalize(cast(List *)sentinel_add(p), false/*noStack > 0*/); + gcx.clrBits(pool, biti, BlkAttr.ALL_BITS); + + List *list = cast(List *)p; + //debug(PRINTF) printf("\tcollecting %p\n", list); + log_free(sentinel_add(list)); + + debug (MEMSTOMP) memset(p, 0xF3, size); + } + pool.pagetable[pn] = B_FREE; + freed += PAGESIZE; + pool.freepages++; + //debug(PRINTF) printf("freeing entire page %d\n", pn); + continue; + } + } + for (; p < ptop; p += size, biti += bitstride) + { + if (!pool.mark.test(biti)) + { + sentinel_Invariant(sentinel_add(p)); + + pool.freebits.set(biti); + if (pool.finals.nbits && pool.finals.testClear(biti)) + rt_finalize(cast(List *)sentinel_add(p), false/*noStack > 0*/); + clrBits(pool, biti, BlkAttr.ALL_BITS); + + List *list = cast(List *)p; + debug(PRINTF) printf("\tcollecting %p\n", list); + log_free(sentinel_add(list)); + + debug (MEMSTOMP) memset(p, 0xF3, size); + + freed += size; + } + } + } + } } } *************** *** 2621,2626 **** --- 2648,2654 ---- size_t ncommitted; pool = pooltable[n]; + if(pool.isLargeObject) continue; ncommitted = pool.ncommitted; for (pn = 0; pn < ncommitted; pn++) { *************** *** 2642,2647 **** --- 2670,2676 ---- goto Lnotfree; } pool.pagetable[pn] = B_FREE; + pool.freepages++; recoveredpages++; continue; *************** *** 2689,2695 **** { assert(p == cast(void*)((cast(size_t)p) & notbinsize[bins])); // return true if the block is not marked. ! return !(pool.mark.test(offset / 16)); } } return false; // not collecting or pointer is a valid argument. --- 2718,2724 ---- { assert(p == cast(void*)((cast(size_t)p) & notbinsize[bins])); // return true if the block is not marked. ! return !(pool.mark.test(offset / pool.divisor)); } } return false; // not collecting or pointer is a valid argument. *************** *** 2916,2927 **** GCBits appendable; // entries that are appendable size_t npages; size_t ncommitted; // ncommitted <= npages ubyte* pagetable; ! void initialize(size_t npages) { size_t poolsize; //debug(PRINTF) printf("Pool::Pool(%u)\n", npages); --- 2945,2965 ---- GCBits appendable; // entries that are appendable size_t npages; + size_t freepages; // The number of pages not in use. size_t ncommitted; // ncommitted <= npages ubyte* pagetable; + bool isLargeObject; + + // This tracks how far back we have to go to find the nearest B_PAGE at + // a smaller address than a B_PAGEPLUS. To save space, we use a uint. + // This limits individual allocations to 16 terabytes, assuming a 4k + // pagesize. + uint* bPageOffsets; ! void initialize(size_t npages, bool isLargeObject) { + this.isLargeObject = isLargeObject; size_t poolsize; //debug(PRINTF) printf("Pool::Pool(%u)\n", npages); *************** *** 2942,2960 **** } //assert(baseAddr); topAddr = baseAddr + poolsize; ! mark.alloc(cast(size_t)poolsize / 16); ! scan.alloc(cast(size_t)poolsize / 16); ! freebits.alloc(cast(size_t)poolsize / 16); ! noscan.alloc(cast(size_t)poolsize / 16); ! appendable.alloc(cast(size_t)poolsize / 16); pagetable = cast(ubyte*)cstdlib.malloc(npages); if (!pagetable) onOutOfMemoryError(); memset(pagetable, B_UNCOMMITTED, npages); this.npages = npages; ncommitted = 0; } --- 2980,3009 ---- } //assert(baseAddr); topAddr = baseAddr + poolsize; + auto div = this.divisor; + auto nbits = cast(size_t)poolsize / div; ! mark.alloc(nbits); ! scan.alloc(nbits); ! freebits.alloc(nbits); ! noscan.alloc(nbits); ! appendable.alloc(nbits); pagetable = cast(ubyte*)cstdlib.malloc(npages); if (!pagetable) onOutOfMemoryError(); + + if(isLargeObject) + { + bPageOffsets = cast(uint*)cstdlib.malloc(npages * uint.sizeof); + if (!bPageOffsets) + onOutOfMemoryError(); + } + memset(pagetable, B_UNCOMMITTED, npages); this.npages = npages; + this.freepages = npages; ncommitted = 0; } *************** *** 2985,2990 **** --- 3034,3042 ---- if (pagetable) cstdlib.free(pagetable); + if(bPageOffsets) + cstdlib.free(bPageOffsets); + mark.Dtor(); scan.Dtor(); freebits.Dtor(); *************** *** 3021,3026 **** --- 3073,3096 ---- } } + // The divisor used for determining bit indices. + size_t divisor() { + return isLargeObject ? PAGESIZE : 16; + } + + void updateOffsets(size_t fromWhere) + { + assert(pagetable[fromWhere] == B_PAGE); + size_t pn = fromWhere + 1; + for(uint offset = 1; pn < ncommitted; pn++, offset++) + { + if(pagetable[pn] != B_PAGEPLUS) break; + bPageOffsets[pn] = offset; + } + + // Store the size of the block in bPageOffsets[fromWhere]. + bPageOffsets[fromWhere] = cast(uint) (pn - fromWhere); + } /** * Allocate n pages from Pool. *************** *** 3028,3033 **** --- 3098,3104 ---- */ size_t allocPages(size_t n) { + if(freepages < n) return OPFAIL; size_t i; size_t n2; *************** *** 3043,3049 **** --- 3114,3128 ---- } } else + { n2 = n; + if(pagetable[i] == B_PAGE) + { + // Then we have the offset information. We can skip a + // whole bunch of stuff. + i += bPageOffsets[i] - 1; + } + } } return extendPages(n); } *************** *** 3117,3123 **** */ void freePages(size_t pagenum, size_t npages) { ! memset(&pagetable[pagenum], B_FREE, npages); } --- 3196,3211 ---- */ void freePages(size_t pagenum, size_t npages) { ! //memset(&pagetable[pagenum], B_FREE, npages); ! for(size_t i = pagenum; i < npages + pagenum; i++) ! { ! if(pagetable[i] < B_FREE) ! { ! freepages++; ! } ! ! pagetable[i] = B_FREE; ! } } diff -r -c ./libphobos/rt/lifetime.d ./libphobos//rt/lifetime.d *** ./libphobos/rt/lifetime.d 2011-04-29 22:13:02.000000000 +0900 --- ./libphobos//rt/lifetime.d 2011-05-13 01:21:38.593750000 +0900 *************** *** 418,431 **** } // we expect this to be called with the lock in place ! extern(C) void rt_processGCMarks(void[] tls) { // called after the mark routine to eliminate block cache data when it // might be ready to sweep debug(PRINTF) printf("processing GC Marks, %x\n", tls.ptr); ! auto cache = *cast(BlkInfo **)(tls.ptr + __blkcache_offset); if(cache) { debug(PRINTF) foreach(i; 0 .. N_CACHE_BLOCKS) --- 418,436 ---- } + extern(C) size_t rt_get_blkcache_offset(void[] tls) + { + return( (cast(void *)&__blkcache_storage) - tls.ptr ) ; + } + // we expect this to be called with the lock in place ! extern(C) void rt_processGCMarks(void[] tls, size_t blkcache_offset) { // called after the mark routine to eliminate block cache data when it // might be ready to sweep debug(PRINTF) printf("processing GC Marks, %x\n", tls.ptr); ! auto cache = *cast(BlkInfo **)(tls.ptr + blkcache_offset); if(cache) { debug(PRINTF) foreach(i; 0 .. N_CACHE_BLOCKS)