Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Only add big objarray to remset once #49315

Merged
merged 5 commits into from
Apr 20, 2023
Merged

Only add big objarray to remset once #49315

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a67e4a5
Only add big objarray to remset once
gbaraldi Apr 10, 2023
101ea90
Apply suggestions
gbaraldi Apr 10, 2023
e214f30
Apply suggestions from code review
gbaraldi Apr 12, 2023
65ff6e9
Merge branch 'master' into mark-loop-fix
gbaraldi Apr 17, 2023
906b54b
Merge branch 'master' into mark-loop-fix
gbaraldi Apr 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 112 additions & 29 deletions src/gc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2087,24 +2087,47 @@ STATIC_INLINE void gc_mark_objarray(jl_ptls_t ptls, jl_value_t *obj_parent, jl_v
jl_value_t *new_obj;
// Decide whether need to chunk objary
(void)jl_assume(step > 0);
size_t nobjs = (obj_end - obj_begin) / step;
if (nobjs > MAX_REFS_AT_ONCE) {
jl_gc_chunk_t c = {GC_objary_chunk, obj_parent, obj_begin + step * MAX_REFS_AT_ONCE,
obj_end, NULL, NULL,
step, nptr};
gc_chunkqueue_push(mq, &c);
obj_end = obj_begin + step * MAX_REFS_AT_ONCE;
if ((nptr & 0x2) == 0x2) {
// pre-scan this object: most of this object should be old, so look for
// the first young object before starting this chunk
// (this also would be valid for young objects, but probably less beneficial)
for (; obj_begin < obj_end; obj_begin += step) {
new_obj = *obj_begin;
if (new_obj != NULL) {
verify_parent2("obj array", obj_parent, obj_begin, "elem(%d)",
gc_slot_to_arrayidx(obj_parent, obj_begin));
jl_taggedvalue_t *o = jl_astaggedvalue(new_obj);
if (!gc_old(o->header))
nptr |= 1;
if (!gc_marked(o->header))
break;
gc_heap_snapshot_record_array_edge(obj_parent, &new_obj);
}
}
}
size_t too_big = (obj_end - obj_begin) / MAX_REFS_AT_ONCE > step; // use this order of operations to avoid idiv
jl_value_t **scan_end = obj_end;
if (too_big) {
scan_end = obj_begin + step * MAX_REFS_AT_ONCE;
}
for (; obj_begin < obj_end; obj_begin += step) {
for (; obj_begin < scan_end; obj_begin += step) {
new_obj = *obj_begin;
if (new_obj != NULL) {
verify_parent2("obj array", obj_parent, obj_begin, "elem(%d)",
gc_slot_to_arrayidx(obj_parent, obj_begin));
gc_slot_to_arrayidx(obj_parent, obj_begin));
gc_try_claim_and_push(mq, new_obj, &nptr);
gc_heap_snapshot_record_array_edge(obj_parent, &new_obj);
}
}
gc_mark_push_remset(ptls, obj_parent, nptr);
if (too_big) {
jl_gc_chunk_t c = {GC_objary_chunk, obj_parent, scan_end,
obj_end, NULL, NULL,
step, nptr};
gc_chunkqueue_push(mq, &c);
}
else {
gc_mark_push_remset(ptls, obj_parent, nptr);
}
}

// Mark array with 8bit field descriptors
Expand All @@ -2116,14 +2139,36 @@ STATIC_INLINE void gc_mark_array8(jl_ptls_t ptls, jl_value_t *ary8_parent, jl_va
jl_value_t *new_obj;
size_t elsize = ((jl_array_t *)ary8_parent)->elsize / sizeof(jl_value_t *);
assert(elsize > 0);
// Decide whether need to chunk ary8
size_t nrefs = (ary8_end - ary8_begin) / elsize;
if (nrefs > MAX_REFS_AT_ONCE) {
jl_gc_chunk_t c = {GC_ary8_chunk, ary8_parent, ary8_begin + elsize * MAX_REFS_AT_ONCE,
ary8_end, elem_begin, elem_end,
0, nptr};
gc_chunkqueue_push(mq, &c);
ary8_end = ary8_begin + elsize * MAX_REFS_AT_ONCE;
// Decide whether need to chunk objary
if ((nptr & 0x2) == 0x2) {
// pre-scan this object: most of this object should be old, so look for
// the first young object before starting this chunk
// (this also would be valid for young objects, but probably less beneficial)
for (; ary8_begin < ary8_end; ary8_begin += elsize) {
int early_end = 0;
for (uint8_t *pindex = elem_begin; pindex < elem_end; pindex++) {
new_obj = ary8_begin[*pindex];
if (new_obj != NULL) {
verify_parent2("array", ary8_parent, &new_obj, "elem(%d)",
gc_slot_to_arrayidx(ary8_parent, ary8_begin));
jl_taggedvalue_t *o = jl_astaggedvalue(new_obj);
if (!gc_old(o->header))
nptr |= 1;
if (!gc_marked(o->header)){
early_end = 1;
break;
}
gc_heap_snapshot_record_array_edge(ary8_parent, &new_obj);
}
}
if (early_end)
break;
}
}
size_t too_big = (ary8_end - ary8_begin) / MAX_REFS_AT_ONCE > elsize; // use this order of operations to avoid idiv
jl_value_t **scan_end = ary8_end;
if (too_big) {
scan_end = ary8_begin + elsize * MAX_REFS_AT_ONCE;
}
for (; ary8_begin < ary8_end; ary8_begin += elsize) {
for (uint8_t *pindex = elem_begin; pindex < elem_end; pindex++) {
Expand All @@ -2136,7 +2181,15 @@ STATIC_INLINE void gc_mark_array8(jl_ptls_t ptls, jl_value_t *ary8_parent, jl_va
}
}
}
gc_mark_push_remset(ptls, ary8_parent, nptr);
if (too_big) {
jl_gc_chunk_t c = {GC_objary_chunk, ary8_parent, scan_end,
ary8_end, elem_begin, elem_end,
0, nptr};
gc_chunkqueue_push(mq, &c);
}
else {
gc_mark_push_remset(ptls, ary8_parent, nptr);
}
}

// Mark array with 16bit field descriptors
Expand All @@ -2148,16 +2201,38 @@ STATIC_INLINE void gc_mark_array16(jl_ptls_t ptls, jl_value_t *ary16_parent, jl_
jl_value_t *new_obj;
size_t elsize = ((jl_array_t *)ary16_parent)->elsize / sizeof(jl_value_t *);
assert(elsize > 0);
// Decide whether need to chunk ary16
size_t nrefs = (ary16_end - ary16_begin) / elsize;
if (nrefs > MAX_REFS_AT_ONCE) {
jl_gc_chunk_t c = {GC_ary16_chunk, ary16_parent, ary16_begin + elsize * MAX_REFS_AT_ONCE,
ary16_end, elem_begin, elem_end,
0, nptr};
gc_chunkqueue_push(mq, &c);
ary16_end = ary16_begin + elsize * MAX_REFS_AT_ONCE;
// Decide whether need to chunk objary
if ((nptr & 0x2) == 0x2) {
// pre-scan this object: most of this object should be old, so look for
// the first young object before starting this chunk
// (this also would be valid for young objects, but probably less beneficial)
for (; ary16_begin < ary16_end; ary16_begin += elsize) {
int early_end = 0;
for (uint16_t *pindex = elem_begin; pindex < elem_end; pindex++) {
new_obj = ary16_begin[*pindex];
if (new_obj != NULL) {
verify_parent2("array", ary16_parent, &new_obj, "elem(%d)",
gc_slot_to_arrayidx(ary16_parent, ary16_begin));
jl_taggedvalue_t *o = jl_astaggedvalue(new_obj);
if (!gc_old(o->header))
nptr |= 1;
if (!gc_marked(o->header)){
early_end = 1;
break;
}
gc_heap_snapshot_record_array_edge(ary16_parent, &new_obj);
}
}
if (early_end)
break;
}
}
size_t too_big = (ary16_end - ary16_begin) / MAX_REFS_AT_ONCE > elsize; // use this order of operations to avoid idiv
jl_value_t **scan_end = ary16_end;
if (too_big) {
scan_end = ary16_begin + elsize * MAX_REFS_AT_ONCE;
}
for (; ary16_begin < ary16_end; ary16_begin += elsize) {
for (; ary16_begin < scan_end; ary16_begin += elsize) {
for (uint16_t *pindex = elem_begin; pindex < elem_end; pindex++) {
new_obj = ary16_begin[*pindex];
if (new_obj != NULL) {
Expand All @@ -2168,7 +2243,15 @@ STATIC_INLINE void gc_mark_array16(jl_ptls_t ptls, jl_value_t *ary16_parent, jl_
}
}
}
gc_mark_push_remset(ptls, ary16_parent, nptr);
if (too_big) {
jl_gc_chunk_t c = {GC_objary_chunk, ary16_parent, scan_end,
ary16_end, elem_begin, elem_end,
elsize, nptr};
gc_chunkqueue_push(mq, &c);
}
else {
gc_mark_push_remset(ptls, ary16_parent, nptr);
}
}

// Mark chunk of large array
Expand Down