diff --git a/src/gc.c b/src/gc.c index aff8958fa6074..3b85964d1eaa7 100644 --- a/src/gc.c +++ b/src/gc.c @@ -2061,7 +2061,8 @@ STATIC_INLINE void gc_mark_objarray(jl_ptls_t ptls, jl_value_t *obj_parent, jl_v // the first young object before starting this chunk // (this also would be valid for young objects, but probably less beneficial) for (; obj_begin < obj_end; obj_begin += step) { - new_obj = *obj_begin; + jl_value_t **slot = obj_begin; + new_obj = *slot; if (new_obj != NULL) { verify_parent2("obj array", obj_parent, obj_begin, "elem(%d)", gc_slot_to_arrayidx(obj_parent, obj_begin)); @@ -2070,7 +2071,7 @@ STATIC_INLINE void gc_mark_objarray(jl_ptls_t ptls, jl_value_t *obj_parent, jl_v nptr |= 1; if (!gc_marked(o->header)) break; - gc_heap_snapshot_record_array_edge(obj_parent, &new_obj); + gc_heap_snapshot_record_array_edge(obj_parent, slot); } } } @@ -2092,12 +2093,13 @@ STATIC_INLINE void gc_mark_objarray(jl_ptls_t ptls, jl_value_t *obj_parent, jl_v } } for (; obj_begin < scan_end; obj_begin += step) { + jl_value_t **slot = obj_begin; new_obj = *obj_begin; if (new_obj != NULL) { verify_parent2("obj array", obj_parent, obj_begin, "elem(%d)", gc_slot_to_arrayidx(obj_parent, obj_begin)); gc_try_claim_and_push(mq, new_obj, &nptr); - gc_heap_snapshot_record_array_edge(obj_parent, &new_obj); + gc_heap_snapshot_record_array_edge(obj_parent, slot); } } if (too_big) { @@ -2128,7 +2130,8 @@ STATIC_INLINE void gc_mark_array8(jl_ptls_t ptls, jl_value_t *ary8_parent, jl_va for (; ary8_begin < ary8_end; ary8_begin += elsize) { int early_end = 0; for (uint8_t *pindex = elem_begin; pindex < elem_end; pindex++) { - new_obj = ary8_begin[*pindex]; + jl_value_t **slot = &ary8_begin[*pindex]; + new_obj = *slot; if (new_obj != NULL) { verify_parent2("array", ary8_parent, &new_obj, "elem(%d)", gc_slot_to_arrayidx(ary8_parent, ary8_begin)); @@ -2139,7 +2142,7 @@ STATIC_INLINE void gc_mark_array8(jl_ptls_t ptls, jl_value_t *ary8_parent, jl_va early_end = 1; break; } - gc_heap_snapshot_record_array_edge(ary8_parent, &new_obj); + gc_heap_snapshot_record_array_edge(ary8_parent, slot); } } if (early_end) @@ -2165,12 +2168,13 @@ STATIC_INLINE void gc_mark_array8(jl_ptls_t ptls, jl_value_t *ary8_parent, jl_va } for (; ary8_begin < ary8_end; ary8_begin += elsize) { for (uint8_t *pindex = elem_begin; pindex < elem_end; pindex++) { - new_obj = ary8_begin[*pindex]; + jl_value_t **slot = &ary8_begin[*pindex]; + new_obj = *slot; if (new_obj != NULL) { verify_parent2("array", ary8_parent, &new_obj, "elem(%d)", gc_slot_to_arrayidx(ary8_parent, ary8_begin)); gc_try_claim_and_push(mq, new_obj, &nptr); - gc_heap_snapshot_record_array_edge(ary8_parent, &new_obj); + gc_heap_snapshot_record_array_edge(ary8_parent, slot); } } } @@ -2193,7 +2197,34 @@ STATIC_INLINE void gc_mark_array16(jl_ptls_t ptls, jl_value_t *ary16_parent, jl_ jl_gc_markqueue_t *mq = &ptls->mark_queue; jl_value_t *new_obj; size_t elsize = ((jl_array_t *)ary16_parent)->elsize / sizeof(jl_value_t *); - // Decide whether need to chunk ary16 + assert(elsize > 0); + // Decide whether need to chunk objary + if ((nptr & 0x2) == 0x2) { + // pre-scan this object: most of this object should be old, so look for + // the first young object before starting this chunk + // (this also would be valid for young objects, but probably less beneficial) + for (; ary16_begin < ary16_end; ary16_begin += elsize) { + int early_end = 0; + for (uint16_t *pindex = elem_begin; pindex < elem_end; pindex++) { + jl_value_t **slot = &ary16_begin[*pindex]; + new_obj = *slot; + if (new_obj != NULL) { + verify_parent2("array", ary16_parent, &new_obj, "elem(%d)", + gc_slot_to_arrayidx(ary16_parent, ary16_begin)); + jl_taggedvalue_t *o = jl_astaggedvalue(new_obj); + if (!gc_old(o->header)) + nptr |= 1; + if (!gc_marked(o->header)){ + early_end = 1; + break; + } + gc_heap_snapshot_record_array_edge(ary16_parent, slot); + } + } + if (early_end) + break; + } + } size_t too_big = (ary16_end - ary16_begin) / GC_CHUNK_BATCH_SIZE > elsize; // use this order of operations to avoid idiv jl_value_t **scan_end = ary16_end; int pushed_chunk = 0; @@ -2213,12 +2244,13 @@ STATIC_INLINE void gc_mark_array16(jl_ptls_t ptls, jl_value_t *ary16_parent, jl_ } for (; ary16_begin < scan_end; ary16_begin += elsize) { for (uint16_t *pindex = elem_begin; pindex < elem_end; pindex++) { - new_obj = ary16_begin[*pindex]; + jl_value_t **slot = &ary16_begin[*pindex]; + new_obj = *slot; if (new_obj != NULL) { verify_parent2("array", ary16_parent, &new_obj, "elem(%d)", gc_slot_to_arrayidx(ary16_parent, ary16_begin)); gc_try_claim_and_push(mq, new_obj, &nptr); - gc_heap_snapshot_record_array_edge(ary16_parent, &new_obj); + gc_heap_snapshot_record_array_edge(ary16_parent, slot); } } }