diff --git a/src/gc.c b/src/gc.c index a67783c741b43..e77ee90a34353 100644 --- a/src/gc.c +++ b/src/gc.c @@ -2087,24 +2087,47 @@ STATIC_INLINE void gc_mark_objarray(jl_ptls_t ptls, jl_value_t *obj_parent, jl_v jl_value_t *new_obj; // Decide whether need to chunk objary (void)jl_assume(step > 0); - size_t nobjs = (obj_end - obj_begin) / step; - if (nobjs > MAX_REFS_AT_ONCE) { - jl_gc_chunk_t c = {GC_objary_chunk, obj_parent, obj_begin + step * MAX_REFS_AT_ONCE, - obj_end, NULL, NULL, - step, nptr}; - gc_chunkqueue_push(mq, &c); - obj_end = obj_begin + step * MAX_REFS_AT_ONCE; + if ((nptr & 0x2) == 0x2) { + // pre-scan this object: most of this object should be old, so look for + // the first young object before starting this chunk + // (this also would be valid for young objects, but probably less beneficial) + for (; obj_begin < obj_end; obj_begin += step) { + new_obj = *obj_begin; + if (new_obj != NULL) { + verify_parent2("obj array", obj_parent, obj_begin, "elem(%d)", + gc_slot_to_arrayidx(obj_parent, obj_begin)); + jl_taggedvalue_t *o = jl_astaggedvalue(new_obj); + if (!gc_old(o->header)) + nptr |= 1; + if (!gc_marked(o->header)) + break; + gc_heap_snapshot_record_array_edge(obj_parent, &new_obj); + } + } + } + size_t too_big = (obj_end - obj_begin) / MAX_REFS_AT_ONCE > step; // use this order of operations to avoid idiv + jl_value_t **scan_end = obj_end; + if (too_big) { + scan_end = obj_begin + step * MAX_REFS_AT_ONCE; } - for (; obj_begin < obj_end; obj_begin += step) { + for (; obj_begin < scan_end; obj_begin += step) { new_obj = *obj_begin; if (new_obj != NULL) { verify_parent2("obj array", obj_parent, obj_begin, "elem(%d)", - gc_slot_to_arrayidx(obj_parent, obj_begin)); + gc_slot_to_arrayidx(obj_parent, obj_begin)); gc_try_claim_and_push(mq, new_obj, &nptr); gc_heap_snapshot_record_array_edge(obj_parent, &new_obj); } } - gc_mark_push_remset(ptls, obj_parent, nptr); + if (too_big) { + jl_gc_chunk_t c = {GC_objary_chunk, obj_parent, scan_end, + obj_end, NULL, NULL, + step, nptr}; + gc_chunkqueue_push(mq, &c); + } + else { + gc_mark_push_remset(ptls, obj_parent, nptr); + } } // Mark array with 8bit field descriptors @@ -2116,14 +2139,36 @@ STATIC_INLINE void gc_mark_array8(jl_ptls_t ptls, jl_value_t *ary8_parent, jl_va jl_value_t *new_obj; size_t elsize = ((jl_array_t *)ary8_parent)->elsize / sizeof(jl_value_t *); assert(elsize > 0); - // Decide whether need to chunk ary8 - size_t nrefs = (ary8_end - ary8_begin) / elsize; - if (nrefs > MAX_REFS_AT_ONCE) { - jl_gc_chunk_t c = {GC_ary8_chunk, ary8_parent, ary8_begin + elsize * MAX_REFS_AT_ONCE, - ary8_end, elem_begin, elem_end, - 0, nptr}; - gc_chunkqueue_push(mq, &c); - ary8_end = ary8_begin + elsize * MAX_REFS_AT_ONCE; + // Decide whether need to chunk objary + if ((nptr & 0x2) == 0x2) { + // pre-scan this object: most of this object should be old, so look for + // the first young object before starting this chunk + // (this also would be valid for young objects, but probably less beneficial) + for (; ary8_begin < ary8_end; ary8_begin += elsize) { + int early_end = 0; + for (uint8_t *pindex = elem_begin; pindex < elem_end; pindex++) { + new_obj = ary8_begin[*pindex]; + if (new_obj != NULL) { + verify_parent2("array", ary8_parent, &new_obj, "elem(%d)", + gc_slot_to_arrayidx(ary8_parent, ary8_begin)); + jl_taggedvalue_t *o = jl_astaggedvalue(new_obj); + if (!gc_old(o->header)) + nptr |= 1; + if (!gc_marked(o->header)){ + early_end = 1; + break; + } + gc_heap_snapshot_record_array_edge(ary8_parent, &new_obj); + } + } + if (early_end) + break; + } + } + size_t too_big = (ary8_end - ary8_begin) / MAX_REFS_AT_ONCE > elsize; // use this order of operations to avoid idiv + jl_value_t **scan_end = ary8_end; + if (too_big) { + scan_end = ary8_begin + elsize * MAX_REFS_AT_ONCE; } for (; ary8_begin < ary8_end; ary8_begin += elsize) { for (uint8_t *pindex = elem_begin; pindex < elem_end; pindex++) { @@ -2136,7 +2181,15 @@ STATIC_INLINE void gc_mark_array8(jl_ptls_t ptls, jl_value_t *ary8_parent, jl_va } } } - gc_mark_push_remset(ptls, ary8_parent, nptr); + if (too_big) { + jl_gc_chunk_t c = {GC_objary_chunk, ary8_parent, scan_end, + ary8_end, elem_begin, elem_end, + 0, nptr}; + gc_chunkqueue_push(mq, &c); + } + else { + gc_mark_push_remset(ptls, ary8_parent, nptr); + } } // Mark array with 16bit field descriptors @@ -2148,16 +2201,38 @@ STATIC_INLINE void gc_mark_array16(jl_ptls_t ptls, jl_value_t *ary16_parent, jl_ jl_value_t *new_obj; size_t elsize = ((jl_array_t *)ary16_parent)->elsize / sizeof(jl_value_t *); assert(elsize > 0); - // Decide whether need to chunk ary16 - size_t nrefs = (ary16_end - ary16_begin) / elsize; - if (nrefs > MAX_REFS_AT_ONCE) { - jl_gc_chunk_t c = {GC_ary16_chunk, ary16_parent, ary16_begin + elsize * MAX_REFS_AT_ONCE, - ary16_end, elem_begin, elem_end, - 0, nptr}; - gc_chunkqueue_push(mq, &c); - ary16_end = ary16_begin + elsize * MAX_REFS_AT_ONCE; + // Decide whether need to chunk objary + if ((nptr & 0x2) == 0x2) { + // pre-scan this object: most of this object should be old, so look for + // the first young object before starting this chunk + // (this also would be valid for young objects, but probably less beneficial) + for (; ary16_begin < ary16_end; ary16_begin += elsize) { + int early_end = 0; + for (uint16_t *pindex = elem_begin; pindex < elem_end; pindex++) { + new_obj = ary16_begin[*pindex]; + if (new_obj != NULL) { + verify_parent2("array", ary16_parent, &new_obj, "elem(%d)", + gc_slot_to_arrayidx(ary16_parent, ary16_begin)); + jl_taggedvalue_t *o = jl_astaggedvalue(new_obj); + if (!gc_old(o->header)) + nptr |= 1; + if (!gc_marked(o->header)){ + early_end = 1; + break; + } + gc_heap_snapshot_record_array_edge(ary16_parent, &new_obj); + } + } + if (early_end) + break; + } + } + size_t too_big = (ary16_end - ary16_begin) / MAX_REFS_AT_ONCE > elsize; // use this order of operations to avoid idiv + jl_value_t **scan_end = ary16_end; + if (too_big) { + scan_end = ary16_begin + elsize * MAX_REFS_AT_ONCE; } - for (; ary16_begin < ary16_end; ary16_begin += elsize) { + for (; ary16_begin < scan_end; ary16_begin += elsize) { for (uint16_t *pindex = elem_begin; pindex < elem_end; pindex++) { new_obj = ary16_begin[*pindex]; if (new_obj != NULL) { @@ -2168,7 +2243,15 @@ STATIC_INLINE void gc_mark_array16(jl_ptls_t ptls, jl_value_t *ary16_parent, jl_ } } } - gc_mark_push_remset(ptls, ary16_parent, nptr); + if (too_big) { + jl_gc_chunk_t c = {GC_objary_chunk, ary16_parent, scan_end, + ary16_end, elem_begin, elem_end, + elsize, nptr}; + gc_chunkqueue_push(mq, &c); + } + else { + gc_mark_push_remset(ptls, ary16_parent, nptr); + } } // Mark chunk of large array