From 32c975098521e830ce706b67e7232a007c0846c7 Mon Sep 17 00:00:00 2001 From: Albert Mingkun Yang Date: Wed, 28 Aug 2024 13:28:01 +0000 Subject: [PATCH] 8339160: [BACKOUT] JDK-8338440 Parallel: Improve fragmentation mitigation in Full GC Reviewed-by: tschatzl --- .../share/gc/parallel/psParallelCompact.cpp | 705 ++++++++---------- .../share/gc/parallel/psParallelCompact.hpp | 77 +- 2 files changed, 359 insertions(+), 423 deletions(-) diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.cpp b/src/hotspot/share/gc/parallel/psParallelCompact.cpp index a4884afe25472..4bff8f8a7d06a 100644 --- a/src/hotspot/share/gc/parallel/psParallelCompact.cpp +++ b/src/hotspot/share/gc/parallel/psParallelCompact.cpp @@ -126,82 +126,73 @@ ParallelCompactData::RegionData::dc_claimed = 0x8U << dc_shift; const ParallelCompactData::RegionData::region_sz_t ParallelCompactData::RegionData::dc_completed = 0xcU << dc_shift; -bool ParallelCompactData::RegionData::is_clear() { - return (_destination == nullptr) && - (_source_region == 0) && - (_partial_obj_addr == nullptr) && - (_partial_obj_size == 0) && - (_dc_and_los == 0) && - (_shadow_state == 0); -} - -#ifdef ASSERT -void ParallelCompactData::RegionData::verify_clear() { - assert(_destination == nullptr, "inv"); - assert(_source_region == 0, "inv"); - assert(_partial_obj_addr == nullptr, "inv"); - assert(_partial_obj_size == 0, "inv"); - assert(_dc_and_los == 0, "inv"); - assert(_shadow_state == 0, "inv"); -} -#endif - SpaceInfo PSParallelCompact::_space_info[PSParallelCompact::last_space_id]; SpanSubjectToDiscoveryClosure PSParallelCompact::_span_based_discoverer; ReferenceProcessor* PSParallelCompact::_ref_processor = nullptr; -void SplitInfo::record(size_t split_region_idx, HeapWord* split_point, size_t preceding_live_words) { - assert(split_region_idx != 0, "precondition"); - - // Obj denoted by split_point will be deferred to the next space. - assert(split_point != nullptr, "precondition"); - - const ParallelCompactData& sd = PSParallelCompact::summary_data(); +void SplitInfo::record(size_t src_region_idx, size_t partial_obj_size, + HeapWord* destination) +{ + assert(src_region_idx != 0, "invalid src_region_idx"); + assert(partial_obj_size != 0, "invalid partial_obj_size argument"); + assert(destination != nullptr, "invalid destination argument"); - PSParallelCompact::RegionData* split_region_ptr = sd.region(split_region_idx); - assert(preceding_live_words < split_region_ptr->data_size(), "inv"); + _src_region_idx = src_region_idx; + _partial_obj_size = partial_obj_size; + _destination = destination; - HeapWord* preceding_destination = split_region_ptr->destination(); - assert(preceding_destination != nullptr, "inv"); + // These fields may not be updated below, so make sure they're clear. + assert(_dest_region_addr == nullptr, "should have been cleared"); + assert(_first_src_addr == nullptr, "should have been cleared"); - // How many regions does the preceding part occupy - uint preceding_destination_count; - if (preceding_live_words == 0) { - preceding_destination_count = 0; - } else { - if (preceding_destination + preceding_live_words > sd.region_align_up(preceding_destination)) { - preceding_destination_count = 2; - } else { - preceding_destination_count = 1; + // Determine the number of destination regions for the partial object. + HeapWord* const last_word = destination + partial_obj_size - 1; + const ParallelCompactData& sd = PSParallelCompact::summary_data(); + HeapWord* const beg_region_addr = sd.region_align_down(destination); + HeapWord* const end_region_addr = sd.region_align_down(last_word); + + if (beg_region_addr == end_region_addr) { + // One destination region. + _destination_count = 1; + if (end_region_addr == destination) { + // The destination falls on a region boundary, thus the first word of the + // partial object will be the first word copied to the destination region. + _dest_region_addr = end_region_addr; + _first_src_addr = sd.region_to_addr(src_region_idx); } + } else { + // Two destination regions. When copied, the partial object will cross a + // destination region boundary, so a word somewhere within the partial + // object will be the first word copied to the second destination region. + _destination_count = 2; + _dest_region_addr = end_region_addr; + const size_t ofs = pointer_delta(end_region_addr, destination); + assert(ofs < _partial_obj_size, "sanity"); + _first_src_addr = sd.region_to_addr(src_region_idx) + ofs; } - - _split_region_idx = split_region_idx; - _split_point = split_point; - _preceding_live_words = preceding_live_words; - _preceding_destination = preceding_destination; - _preceding_destination_count = preceding_destination_count; } void SplitInfo::clear() { - _split_region_idx = 0; - _split_point = nullptr; - _preceding_live_words = 0; - _preceding_destination = nullptr; - _preceding_destination_count = 0; + _src_region_idx = 0; + _partial_obj_size = 0; + _destination = nullptr; + _destination_count = 0; + _dest_region_addr = nullptr; + _first_src_addr = nullptr; assert(!is_valid(), "sanity"); } #ifdef ASSERT void SplitInfo::verify_clear() { - assert(_split_region_idx == 0, "not clear"); - assert(_split_point == nullptr, "not clear"); - assert(_preceding_live_words == 0, "not clear"); - assert(_preceding_destination == nullptr, "not clear"); - assert(_preceding_destination_count == 0, "not clear"); + assert(_src_region_idx == 0, "not clear"); + assert(_partial_obj_size == 0, "not clear"); + assert(_destination == nullptr, "not clear"); + assert(_destination_count == 0, "not clear"); + assert(_dest_region_addr == nullptr, "not clear"); + assert(_first_src_addr == nullptr, "not clear"); } #endif // #ifdef ASSERT @@ -304,105 +295,110 @@ ParallelCompactData::summarize_dense_prefix(HeapWord* beg, HeapWord* end) } } -// The total live words on src_region would overflow the target space, so find -// the overflowing object and record the split point. The invariant is that an -// obj should not cross space boundary. -HeapWord* ParallelCompactData::summarize_split_space(size_t src_region, - SplitInfo& split_info, - HeapWord* const destination, - HeapWord* const target_end, - HeapWord** target_next) { +// Find the point at which a space can be split and, if necessary, record the +// split point. +// +// If the current src region (which overflowed the destination space) doesn't +// have a partial object, the split point is at the beginning of the current src +// region (an "easy" split, no extra bookkeeping required). +// +// If the current src region has a partial object, the split point is in the +// region where that partial object starts (call it the split_region). If +// split_region has a partial object, then the split point is just after that +// partial object (a "hard" split where we have to record the split data and +// zero the partial_obj_size field). With a "hard" split, we know that the +// partial_obj ends within split_region because the partial object that caused +// the overflow starts in split_region. If split_region doesn't have a partial +// obj, then the split is at the beginning of split_region (another "easy" +// split). +HeapWord* +ParallelCompactData::summarize_split_space(size_t src_region, + SplitInfo& split_info, + HeapWord* destination, + HeapWord* target_end, + HeapWord** target_next) +{ assert(destination <= target_end, "sanity"); assert(destination + _region_data[src_region].data_size() > target_end, "region should not fit into target space"); assert(is_region_aligned(target_end), "sanity"); + size_t split_region = src_region; + HeapWord* split_destination = destination; size_t partial_obj_size = _region_data[src_region].partial_obj_size(); if (destination + partial_obj_size > target_end) { - assert(partial_obj_size > 0, "inv"); - // The overflowing obj is from a previous region. - // - // source-regions: + // The split point is just after the partial object (if any) in the + // src_region that contains the start of the object that overflowed the + // destination space. // - // *************** - // | A|AA | - // *************** - // ^ - // | split-point + // Find the start of the "overflow" object and set split_region to the + // region containing it. + HeapWord* const overflow_obj = _region_data[src_region].partial_obj_addr(); + split_region = addr_to_region_idx(overflow_obj); + + // Clear the source_region field of all destination regions whose first word + // came from data after the split point (a non-null source_region field + // implies a region must be filled). // - // dest-region: + // An alternative to the simple loop below: clear during post_compact(), + // which uses memcpy instead of individual stores, and is easy to + // parallelize. (The downside is that it clears the entire RegionData + // object as opposed to just one field.) // - // ******** - // |~~~~A | - // ******** - // ^^ - // || target-space-end - // | - // | destination + // post_compact() would have to clear the summary data up to the highest + // address that was written during the summary phase, which would be // - // AAA would overflow target-space. + // max(top, max(new_top, clear_top)) // - HeapWord* overflowing_obj = _region_data[src_region].partial_obj_addr(); - size_t split_region = addr_to_region_idx(overflowing_obj); + // where clear_top is a new field in SpaceInfo. Would have to set clear_top + // to target_end. + const RegionData* const sr = region(split_region); + const size_t beg_idx = + addr_to_region_idx(region_align_up(sr->destination() + + sr->partial_obj_size())); + const size_t end_idx = addr_to_region_idx(target_end); - // The number of live words before the overflowing object on this split region - size_t preceding_live_words; - if (is_region_aligned(overflowing_obj)) { - preceding_live_words = 0; - } else { - // Words accounted by the overflowing object on the split region - size_t overflowing_size = pointer_delta(region_align_up(overflowing_obj), overflowing_obj); - preceding_live_words = region(split_region)->data_size() - overflowing_size; - } - - split_info.record(split_region, overflowing_obj, preceding_live_words); - - HeapWord* src_region_start = region_to_addr(src_region); - HeapWord* new_top = destination - pointer_delta(src_region_start, overflowing_obj); - - // If the overflowing obj was relocated to its original destination, - // those destination regions would have their source_region set. Now that - // this overflowing obj is relocated somewhere else, reset the - // source_region. - { - size_t range_start = addr_to_region_idx(region_align_up(new_top)); - size_t range_end = addr_to_region_idx(region_align_up(destination)); - for (size_t i = range_start; i < range_end; ++i) { - region(i)->set_source_region(0); - } + log_develop_trace(gc, compaction)("split: clearing source_region field in [" SIZE_FORMAT ", " SIZE_FORMAT ")", beg_idx, end_idx); + for (size_t idx = beg_idx; idx < end_idx; ++idx) { + _region_data[idx].set_source_region(0); } - // Update new top of target space - *target_next = new_top; + // Set split_destination and partial_obj_size to reflect the split region. + split_destination = sr->destination(); + partial_obj_size = sr->partial_obj_size(); + } - return overflowing_obj; + // The split is recorded only if a partial object extends onto the region. + if (partial_obj_size != 0) { + _region_data[split_region].set_partial_obj_size(0); + split_info.record(split_region, partial_obj_size, split_destination); } - // Obj-iteration to locate the overflowing obj - HeapWord* region_start = region_to_addr(src_region); - HeapWord* region_end = region_start + RegionSize; - HeapWord* cur_addr = region_start + partial_obj_size; - size_t live_words = partial_obj_size; + // Setup the continuation addresses. + *target_next = split_destination + partial_obj_size; + HeapWord* const source_next = region_to_addr(split_region) + partial_obj_size; - while (true) { - assert(cur_addr < region_end, "inv"); - cur_addr = PSParallelCompact::mark_bitmap()->find_obj_beg(cur_addr, region_end); - // There must be an overflowing obj in this region - assert(cur_addr < region_end, "inv"); + if (log_develop_is_enabled(Trace, gc, compaction)) { + const char * split_type = partial_obj_size == 0 ? "easy" : "hard"; + log_develop_trace(gc, compaction)("%s split: src=" PTR_FORMAT " src_c=" SIZE_FORMAT " pos=" SIZE_FORMAT, + split_type, p2i(source_next), split_region, partial_obj_size); + log_develop_trace(gc, compaction)("%s split: dst=" PTR_FORMAT " dst_c=" SIZE_FORMAT " tn=" PTR_FORMAT, + split_type, p2i(split_destination), + addr_to_region_idx(split_destination), + p2i(*target_next)); - oop obj = cast_to_oop(cur_addr); - size_t obj_size = obj->size(); - if (destination + live_words + obj_size > target_end) { - // Found the overflowing obj - split_info.record(src_region, cur_addr, live_words); - *target_next = destination + live_words; - return cur_addr; + if (partial_obj_size != 0) { + HeapWord* const po_beg = split_info.destination(); + HeapWord* const po_end = po_beg + split_info.partial_obj_size(); + log_develop_trace(gc, compaction)("%s split: po_beg=" PTR_FORMAT " " SIZE_FORMAT " po_end=" PTR_FORMAT " " SIZE_FORMAT, + split_type, + p2i(po_beg), addr_to_region_idx(po_beg), + p2i(po_end), addr_to_region_idx(po_end)); } - - live_words += obj_size; - cur_addr += obj_size; } + + return source_next; } size_t ParallelCompactData::live_words_in_space(const MutableSpace* space, @@ -454,57 +450,70 @@ bool ParallelCompactData::summarize(SplitInfo& split_info, const size_t end_region = addr_to_region_idx(region_align_up(source_end)); HeapWord *dest_addr = target_beg; - for (/* empty */; cur_region < end_region; cur_region++) { - size_t words = _region_data[cur_region].data_size(); - - // Skip empty ones - if (words == 0) { - continue; - } - - if (split_info.is_split(cur_region)) { - assert(words > split_info.preceding_live_words(), "inv"); - words -= split_info.preceding_live_words(); - } - + while (cur_region < end_region) { + // The destination must be set even if the region has no data. _region_data[cur_region].set_destination(dest_addr); - // If cur_region does not fit entirely into the target space, find a point - // at which the source space can be 'split' so that part is copied to the - // target space and the rest is copied elsewhere. - if (dest_addr + words > target_end) { - assert(source_next != nullptr, "source_next is null when splitting"); - *source_next = summarize_split_space(cur_region, split_info, dest_addr, - target_end, target_next); - return false; - } + size_t words = _region_data[cur_region].data_size(); + if (words > 0) { + // If cur_region does not fit entirely into the target space, find a point + // at which the source space can be 'split' so that part is copied to the + // target space and the rest is copied elsewhere. + if (dest_addr + words > target_end) { + assert(source_next != nullptr, "source_next is null when splitting"); + *source_next = summarize_split_space(cur_region, split_info, dest_addr, + target_end, target_next); + return false; + } - uint destination_count = split_info.is_split(cur_region) - ? split_info.preceding_destination_count() - : 0; + // Compute the destination_count for cur_region, and if necessary, update + // source_region for a destination region. The source_region field is + // updated if cur_region is the first (left-most) region to be copied to a + // destination region. + // + // The destination_count calculation is a bit subtle. A region that has + // data that compacts into itself does not count itself as a destination. + // This maintains the invariant that a zero count means the region is + // available and can be claimed and then filled. + uint destination_count = 0; + if (split_info.is_split(cur_region)) { + // The current region has been split: the partial object will be copied + // to one destination space and the remaining data will be copied to + // another destination space. Adjust the initial destination_count and, + // if necessary, set the source_region field if the partial object will + // cross a destination region boundary. + destination_count = split_info.destination_count(); + if (destination_count == 2) { + size_t dest_idx = addr_to_region_idx(split_info.dest_region_addr()); + _region_data[dest_idx].set_source_region(cur_region); + } + } - HeapWord* const last_addr = dest_addr + words - 1; - const size_t dest_region_1 = addr_to_region_idx(dest_addr); - const size_t dest_region_2 = addr_to_region_idx(last_addr); + HeapWord* const last_addr = dest_addr + words - 1; + const size_t dest_region_1 = addr_to_region_idx(dest_addr); + const size_t dest_region_2 = addr_to_region_idx(last_addr); + + // Initially assume that the destination regions will be the same and + // adjust the value below if necessary. Under this assumption, if + // cur_region == dest_region_2, then cur_region will be compacted + // completely into itself. + destination_count += cur_region == dest_region_2 ? 0 : 1; + if (dest_region_1 != dest_region_2) { + // Destination regions differ; adjust destination_count. + destination_count += 1; + // Data from cur_region will be copied to the start of dest_region_2. + _region_data[dest_region_2].set_source_region(cur_region); + } else if (is_region_aligned(dest_addr)) { + // Data from cur_region will be copied to the start of the destination + // region. + _region_data[dest_region_1].set_source_region(cur_region); + } - // Initially assume that the destination regions will be the same and - // adjust the value below if necessary. Under this assumption, if - // cur_region == dest_region_2, then cur_region will be compacted - // completely into itself. - destination_count += cur_region == dest_region_2 ? 0 : 1; - if (dest_region_1 != dest_region_2) { - // Destination regions differ; adjust destination_count. - destination_count += 1; - // Data from cur_region will be copied to the start of dest_region_2. - _region_data[dest_region_2].set_source_region(cur_region); - } else if (is_region_aligned(dest_addr)) { - // Data from cur_region will be copied to the start of the destination - // region. - _region_data[dest_region_1].set_source_region(cur_region); + _region_data[cur_region].set_destination_count(destination_count); + dest_addr += words; } - _region_data[cur_region].set_destination_count(destination_count); - dest_addr += words; + ++cur_region; } *target_next = dest_addr; @@ -512,12 +521,12 @@ bool ParallelCompactData::summarize(SplitInfo& split_info, } #ifdef ASSERT -void ParallelCompactData::verify_clear() { - for (uint cur_idx = 0; cur_idx < region_count(); ++cur_idx) { - if (!region(cur_idx)->is_clear()) { - log_warning(gc)("Uncleared Region: %u", cur_idx); - region(cur_idx)->verify_clear(); - } +void ParallelCompactData::verify_clear() +{ + const size_t* const beg = (const size_t*) _region_vspace->committed_low_addr(); + const size_t* const end = (const size_t*) _region_vspace->committed_high_addr(); + for (const size_t* p = beg; p < end; ++p) { + assert(*p == 0, "not zero"); } } #endif // #ifdef ASSERT @@ -684,13 +693,6 @@ void PSParallelCompact::post_compact() } } -#ifdef ASSERT - { - mark_bitmap()->verify_clear(); - summary_data().verify_clear(); - } -#endif - ParCompactionManager::flush_all_string_dedup_requests(); MutableSpace* const eden_space = _space_info[eden_space_id].space(); @@ -877,10 +879,10 @@ void PSParallelCompact::summary_phase() bool maximum_compaction = check_maximum_compaction(total_live_words, old_space, full_region_prefix_end); - HeapWord* dense_prefix_end = maximum_compaction - ? full_region_prefix_end - : compute_dense_prefix_for_old_space(old_space, - full_region_prefix_end); + HeapWord* dense_prefix_end = + maximum_compaction ? full_region_prefix_end + : compute_dense_prefix_for_old_space(old_space, + full_region_prefix_end); SpaceId id = old_space_id; _space_info[id].set_dense_prefix(dense_prefix_end); @@ -888,8 +890,6 @@ void PSParallelCompact::summary_phase() fill_dense_prefix_end(id); _summary_data.summarize_dense_prefix(old_space->bottom(), dense_prefix_end); } - - // Compacting objs inn [dense_prefix_end, old_space->top()) _summary_data.summarize(_space_info[id].split_info(), dense_prefix_end, old_space->top(), nullptr, dense_prefix_end, old_space->end(), @@ -1549,30 +1549,6 @@ void PSParallelCompact::forward_to_new_addr() { WorkerTask("PSForward task"), _num_workers(num_workers) {} - static void forward_objs_in_range(ParCompactionManager* cm, - HeapWord* start, - HeapWord* end, - HeapWord* destination) { - HeapWord* cur_addr = start; - HeapWord* new_addr = destination; - - while (cur_addr < end) { - cur_addr = mark_bitmap()->find_obj_beg(cur_addr, end); - if (cur_addr >= end) { - return; - } - assert(mark_bitmap()->is_marked(cur_addr), "inv"); - oop obj = cast_to_oop(cur_addr); - if (new_addr != cur_addr) { - cm->preserved_marks()->push_if_necessary(obj, obj->mark()); - obj->forward_to(cast_to_oop(new_addr)); - } - size_t obj_size = obj->size(); - new_addr += obj_size; - cur_addr += obj_size; - } - } - void work(uint worker_id) override { ParCompactionManager* cm = ParCompactionManager::gc_thread_compaction_manager(worker_id); for (uint id = old_space_id; id < last_space_id; ++id) { @@ -1584,8 +1560,6 @@ void PSParallelCompact::forward_to_new_addr() { continue; } - const SplitInfo& split_info = _space_info[SpaceId(id)].split_info(); - size_t dense_prefix_region = _summary_data.addr_to_region_idx(dense_prefix_addr); size_t top_region = _summary_data.addr_to_region_idx(_summary_data.region_align_up(top)); size_t start_region; @@ -1605,19 +1579,24 @@ void PSParallelCompact::forward_to_new_addr() { HeapWord* region_start = _summary_data.region_to_addr(cur_region); HeapWord* region_end = region_start + ParallelCompactData::RegionSize; - - if (split_info.is_split(cur_region)) { - // Part 1: will be relocated to space-1 - HeapWord* preceding_destination = split_info.preceding_destination(); - HeapWord* split_point = split_info.split_point(); - forward_objs_in_range(cm, region_start + live_words, split_point, preceding_destination + live_words); - - // Part 2: will be relocated to space-2 - HeapWord* destination = region_ptr->destination(); - forward_objs_in_range(cm, split_point, region_end, destination); - } else { - HeapWord* destination = region_ptr->destination(); - forward_objs_in_range(cm, region_start + live_words, region_end, destination + live_words); + HeapWord* cur_addr = region_start + live_words; + + HeapWord* destination = region_ptr->destination(); + while (cur_addr < region_end) { + cur_addr = mark_bitmap()->find_obj_beg(cur_addr, region_end); + if (cur_addr >= region_end) { + break; + } + assert(mark_bitmap()->is_marked(cur_addr), "inv"); + HeapWord* new_addr = destination + live_words; + oop obj = cast_to_oop(cur_addr); + if (new_addr != cur_addr) { + cm->preserved_marks()->push_if_necessary(obj, obj->mark()); + obj->forward_to(cast_to_oop(new_addr)); + } + size_t obj_size = obj->size(); + live_words += obj_size; + cur_addr += obj_size; } } } @@ -1649,16 +1628,13 @@ void PSParallelCompact::verify_forward() { break; } assert(mark_bitmap()->is_marked(cur_addr), "inv"); - assert(bump_ptr <= _space_info[bump_ptr_space].new_top(), "inv"); // Move to the space containing cur_addr if (bump_ptr == _space_info[bump_ptr_space].new_top()) { bump_ptr = space(space_id(cur_addr))->bottom(); bump_ptr_space = space_id(bump_ptr); } oop obj = cast_to_oop(cur_addr); - if (cur_addr == bump_ptr) { - assert(!obj->is_forwarded(), "inv"); - } else { + if (cur_addr != bump_ptr) { assert(obj->forwardee() == cast_to_oop(bump_ptr), "inv"); } bump_ptr += obj->size(); @@ -1968,10 +1944,15 @@ PSParallelCompact::SpaceId PSParallelCompact::space_id(HeapWord* addr) { } // Skip over count live words starting from beg, and return the address of the -// next live word. Callers must also ensure that there are enough live words in -// the range [beg, end) to skip. -HeapWord* PSParallelCompact::skip_live_words(HeapWord* beg, HeapWord* end, size_t count) +// next live word. Unless marked, the word corresponding to beg is assumed to +// be dead. Callers must either ensure beg does not correspond to the middle of +// an object, or account for those live words in some other way. Callers must +// also ensure that there are enough live words in the range [beg, end) to skip. +HeapWord* +PSParallelCompact::skip_live_words(HeapWord* beg, HeapWord* end, size_t count) { + assert(count > 0, "sanity"); + ParMarkBitMap* m = mark_bitmap(); HeapWord* cur_addr = beg; while (true) { @@ -1987,94 +1968,69 @@ HeapWord* PSParallelCompact::skip_live_words(HeapWord* beg, HeapWord* end, size_ } } -// On filling a destination region (dest-region), we need to know the location -// of the word that will be at the start of the dest-region after compaction. -// A dest-region can have one or more source regions, but only the first -// source-region contains this location. This location is retrieved by calling -// `first_src_addr` on a dest-region. -// Conversely, a source-region has a dest-region which holds the destination of -// the first live word on this source-region, based on which the destination -// for the rest of live words can be derived. -// -// Note: -// There is some complication due to space-boundary-fragmentation (an obj can't -// cross space-boundary) -- a source-region may be split and behave like two -// distinct regions with their own dest-region, as depicted below. -// -// source-region: region-n -// -// ********************** -// | A|A~~~~B|B | -// ********************** -// n-1 n n+1 -// -// AA, BB denote two live objs. ~~~~ denotes unknown number of live objs. -// -// Assuming the dest-region for region-n is the final region before -// old-space-end and its first-live-word is the middle of AA, the heap content -// will look like the following after compaction: -// -// ************** ************* -// A|A~~~~ | |BB | -// ************** ************* -// ^ ^ -// | old-space-end | eden-space-start -// -// Therefore, in this example, region-n will have two dest-regions, one for -// the final region in old-space and the other for the first region in -// eden-space. -// To handle this special case, we introduce the concept of split-region, whose -// contents are relocated to two spaces. `SplitInfo` captures all necessary -// info about the split, the first part, spliting-point, and the second part. HeapWord* PSParallelCompact::first_src_addr(HeapWord* const dest_addr, SpaceId src_space_id, size_t src_region_idx) { - const size_t RegionSize = ParallelCompactData::RegionSize; + assert(summary_data().is_region_aligned(dest_addr), "not aligned"); + + const SplitInfo& split_info = _space_info[src_space_id].split_info(); + if (split_info.dest_region_addr() == dest_addr) { + // The partial object ending at the split point contains the first word to + // be copied to dest_addr. + return split_info.first_src_addr(); + } + const ParallelCompactData& sd = summary_data(); - assert(sd.is_region_aligned(dest_addr), "precondition"); + ParMarkBitMap* const bitmap = mark_bitmap(); + const size_t RegionSize = ParallelCompactData::RegionSize; + assert(sd.is_region_aligned(dest_addr), "not aligned"); const RegionData* const src_region_ptr = sd.region(src_region_idx); - assert(src_region_ptr->data_size() > 0, "src region cannot be empty"); - const size_t partial_obj_size = src_region_ptr->partial_obj_size(); HeapWord* const src_region_destination = src_region_ptr->destination(); - HeapWord* const region_start = sd.region_to_addr(src_region_idx); - HeapWord* const region_end = sd.region_to_addr(src_region_idx) + RegionSize; + assert(dest_addr >= src_region_destination, "wrong src region"); + assert(src_region_ptr->data_size() > 0, "src region cannot be empty"); - // Identify the actual destination for the first live words on this region, - // taking split-region into account. - HeapWord* region_start_destination; - const SplitInfo& split_info = _space_info[src_space_id].split_info(); - if (split_info.is_split(src_region_idx)) { - // The second part of this split region; use the recorded split point. - if (dest_addr == src_region_destination) { - return split_info.split_point(); + HeapWord* const src_region_beg = sd.region_to_addr(src_region_idx); + HeapWord* const src_region_end = src_region_beg + RegionSize; + + HeapWord* addr = src_region_beg; + if (dest_addr == src_region_destination) { + // Return the first live word in the source region. + if (partial_obj_size == 0) { + addr = bitmap->find_obj_beg(addr, src_region_end); + assert(addr < src_region_end, "no objects start in src region"); } - region_start_destination = split_info.preceding_destination(); - } else { - region_start_destination = src_region_destination; + return addr; } - // Calculate the offset to be skipped - size_t words_to_skip = pointer_delta(dest_addr, region_start_destination); + // Must skip some live data. + size_t words_to_skip = dest_addr - src_region_destination; + assert(src_region_ptr->data_size() > words_to_skip, "wrong src region"); - HeapWord* result; - if (partial_obj_size > words_to_skip) { - result = region_start + words_to_skip; - } else { - words_to_skip -= partial_obj_size; - result = skip_live_words(region_start + partial_obj_size, region_end, words_to_skip); + if (partial_obj_size >= words_to_skip) { + // All the live words to skip are part of the partial object. + addr += words_to_skip; + if (partial_obj_size == words_to_skip) { + // Find the first live word past the partial object. + addr = bitmap->find_obj_beg(addr, src_region_end); + assert(addr < src_region_end, "wrong src region"); + } + return addr; } - if (split_info.is_split(src_region_idx)) { - assert(result < split_info.split_point(), "postcondition"); - } else { - assert(result < region_end, "postcondition"); + // Skip over the partial object (if any). + if (partial_obj_size != 0) { + words_to_skip -= partial_obj_size; + addr += partial_obj_size; } - return result; + // Skip over live words due to objects that start in the region. + addr = skip_live_words(addr, src_region_end, words_to_skip); + assert(addr < src_region_end, "wrong src region"); + return addr; } void PSParallelCompact::decrement_destination_counts(ParCompactionManager* cm, @@ -2125,7 +2081,10 @@ size_t PSParallelCompact::next_src_region(MoveAndUpdateClosure& closure, HeapWord*& src_space_top, HeapWord* end_addr) { + typedef ParallelCompactData::RegionData RegionData; + ParallelCompactData& sd = PSParallelCompact::summary_data(); + const size_t region_size = ParallelCompactData::RegionSize; size_t src_region_idx = 0; @@ -2133,8 +2092,8 @@ size_t PSParallelCompact::next_src_region(MoveAndUpdateClosure& closure, HeapWord* const src_aligned_up = sd.region_align_up(end_addr); RegionData* src_region_ptr = sd.addr_to_region_ptr(src_aligned_up); HeapWord* const top_aligned_up = sd.region_align_up(src_space_top); - const RegionData* const top_region_ptr = sd.addr_to_region_ptr(top_aligned_up); - + const RegionData* const top_region_ptr = + sd.addr_to_region_ptr(top_aligned_up); while (src_region_ptr < top_region_ptr && src_region_ptr->data_size() == 0) { ++src_region_ptr; } @@ -2151,50 +2110,59 @@ size_t PSParallelCompact::next_src_region(MoveAndUpdateClosure& closure, } // Switch to a new source space and find the first non-empty region. - uint space_id = src_space_id + 1; + unsigned int space_id = src_space_id + 1; assert(space_id < last_space_id, "not enough spaces"); - for (/* empty */; space_id < last_space_id; ++space_id) { - HeapWord* bottom = _space_info[space_id].space()->bottom(); - HeapWord* top = _space_info[space_id].space()->top(); - // Skip empty space - if (bottom == top) { - continue; - } - - // Identify the first region that contains live words in this space - size_t cur_region = sd.addr_to_region_idx(bottom); - size_t end_region = sd.addr_to_region_idx(sd.region_align_up(top)); - - for (/* empty */ ; cur_region < end_region; ++cur_region) { - RegionData* cur = sd.region(cur_region); - if (cur->live_obj_size() > 0) { - HeapWord* region_start_addr = sd.region_to_addr(cur_region); - HeapWord* region_end_addr = region_start_addr + ParallelCompactData::RegionSize; - HeapWord* first_live_word = mark_bitmap()->find_obj_beg(region_start_addr, region_end_addr); - assert(first_live_word < region_end_addr, "inv"); - - src_space_id = SpaceId(space_id); - src_space_top = top; - closure.set_source(first_live_word); - return cur_region; + HeapWord* const destination = closure.destination(); + + do { + MutableSpace* space = _space_info[space_id].space(); + HeapWord* const bottom = space->bottom(); + const RegionData* const bottom_cp = sd.addr_to_region_ptr(bottom); + + // Iterate over the spaces that do not compact into themselves. + if (bottom_cp->destination() != bottom) { + HeapWord* const top_aligned_up = sd.region_align_up(space->top()); + const RegionData* const top_cp = sd.addr_to_region_ptr(top_aligned_up); + + for (const RegionData* src_cp = bottom_cp; src_cp < top_cp; ++src_cp) { + if (src_cp->live_obj_size() > 0) { + // Found it. + assert(src_cp->destination() == destination, + "first live obj in the space must match the destination"); + assert(src_cp->partial_obj_size() == 0, + "a space cannot begin with a partial obj"); + + src_space_id = SpaceId(space_id); + src_space_top = space->top(); + const size_t src_region_idx = sd.region(src_cp); + closure.set_source(sd.region_to_addr(src_region_idx)); + return src_region_idx; + } else { + assert(src_cp->data_size() == 0, "sanity"); + } } } - } + } while (++space_id < last_space_id); - ShouldNotReachHere(); + assert(false, "no source region was found"); + return 0; } HeapWord* PSParallelCompact::partial_obj_end(HeapWord* region_start_addr) { ParallelCompactData& sd = summary_data(); assert(sd.is_region_aligned(region_start_addr), "precondition"); - // Use per-region partial_obj_size to locate the end of the obj, that extends - // to region_start_addr. + // Use per-region partial_obj_size to locate the end of the obj, that extends to region_start_addr. + SplitInfo& split_info = _space_info[space_id(region_start_addr)].split_info(); size_t start_region_idx = sd.addr_to_region_idx(region_start_addr); size_t end_region_idx = sd.region_count(); size_t accumulated_size = 0; for (size_t region_idx = start_region_idx; region_idx < end_region_idx; ++region_idx) { + if (split_info.is_split(region_idx)) { + accumulated_size += split_info.partial_obj_size(); + break; + } size_t cur_partial_obj_size = sd.region(region_idx)->partial_obj_size(); accumulated_size += cur_partial_obj_size; if (cur_partial_obj_size != ParallelCompactData::RegionSize) { @@ -2204,8 +2172,6 @@ HeapWord* PSParallelCompact::partial_obj_end(HeapWord* region_start_addr) { return region_start_addr + accumulated_size; } -// Use region_idx as the destination region, and evacuate all live objs on its -// source regions to this destination region. void PSParallelCompact::fill_region(ParCompactionManager* cm, MoveAndUpdateClosure& closure, size_t region_idx) { ParMarkBitMap* const bitmap = mark_bitmap(); @@ -2226,43 +2192,20 @@ void PSParallelCompact::fill_region(ParCompactionManager* cm, MoveAndUpdateClosu src_region_idx += 1; } - // source-region: - // - // ********** - // | ~~~ | - // ********** - // ^ - // |-- closure.source() / first_src_addr - // - // - // ~~~ : live words - // - // destination-region: - // - // ********** - // | | - // ********** - // ^ - // |-- region-start if (bitmap->is_unmarked(closure.source())) { - // An object overflows the previous destination region, so this - // destination region should copy the remainder of the object or as much as - // will fit. + // The first source word is in the middle of an object; copy the remainder + // of the object or as much as will fit. The fact that pointer updates were + // deferred will be noted when the object header is processed. HeapWord* const old_src_addr = closure.source(); { HeapWord* region_start = sd.region_align_down(closure.source()); HeapWord* obj_start = bitmap->find_obj_beg_reverse(region_start, closure.source()); HeapWord* obj_end; - if (obj_start != closure.source()) { - assert(bitmap->is_marked(obj_start), "inv"); - // Found the actual obj-start, try to find the obj-end using either - // size() if this obj is completely contained in the current region. + if (bitmap->is_marked(obj_start)) { HeapWord* next_region_start = region_start + ParallelCompactData::RegionSize; HeapWord* partial_obj_start = (next_region_start >= src_space_top) ? nullptr : sd.addr_to_region_ptr(next_region_start)->partial_obj_addr(); - // This obj extends to next region iff partial_obj_addr of the *next* - // region is the same as obj-start. if (partial_obj_start == obj_start) { // This obj extends to next region. obj_end = partial_obj_end(next_region_start); @@ -2279,41 +2222,39 @@ void PSParallelCompact::fill_region(ParCompactionManager* cm, MoveAndUpdateClosu } if (closure.is_full()) { - decrement_destination_counts(cm, src_space_id, src_region_idx, closure.source()); + decrement_destination_counts(cm, src_space_id, src_region_idx, + closure.source()); closure.complete_region(dest_addr, region_ptr); return; } - // Finished copying without using up the current destination-region HeapWord* const end_addr = sd.region_align_down(closure.source()); if (sd.region_align_down(old_src_addr) != end_addr) { - assert(sd.region_align_up(old_src_addr) == end_addr, "only one region"); // The partial object was copied from more than one source region. decrement_destination_counts(cm, src_space_id, src_region_idx, end_addr); // Move to the next source region, possibly switching spaces as well. All // args except end_addr may be modified. - src_region_idx = next_src_region(closure, src_space_id, src_space_top, end_addr); + src_region_idx = next_src_region(closure, src_space_id, src_space_top, + end_addr); } } - // Handle the rest obj-by-obj, where we know obj-start. do { HeapWord* cur_addr = closure.source(); HeapWord* const end_addr = MIN2(sd.region_align_up(cur_addr + 1), src_space_top); - // To handle the case where the final obj in source region extends to next region. - HeapWord* final_obj_start = (end_addr == src_space_top) + HeapWord* partial_obj_start = (end_addr == src_space_top) ? nullptr : sd.addr_to_region_ptr(end_addr)->partial_obj_addr(); - // Apply closure on objs inside [cur_addr, end_addr) + // apply closure on objs inside [cur_addr, end_addr) do { cur_addr = bitmap->find_obj_beg(cur_addr, end_addr); if (cur_addr == end_addr) { break; } size_t obj_size; - if (final_obj_start == cur_addr) { + if (partial_obj_start == cur_addr) { obj_size = pointer_delta(partial_obj_end(end_addr), cur_addr); } else { // This obj doesn't extend into next region; size() is safe to use. @@ -2324,7 +2265,8 @@ void PSParallelCompact::fill_region(ParCompactionManager* cm, MoveAndUpdateClosu } while (cur_addr < end_addr && !closure.is_full()); if (closure.is_full()) { - decrement_destination_counts(cm, src_space_id, src_region_idx, closure.source()); + decrement_destination_counts(cm, src_space_id, src_region_idx, + closure.source()); closure.complete_region(dest_addr, region_ptr); return; } @@ -2333,7 +2275,8 @@ void PSParallelCompact::fill_region(ParCompactionManager* cm, MoveAndUpdateClosu // Move to the next source region, possibly switching spaces as well. All // args except end_addr may be modified. - src_region_idx = next_src_region(closure, src_space_id, src_space_top, end_addr); + src_region_idx = next_src_region(closure, src_space_id, src_space_top, + end_addr); } while (true); } diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.hpp b/src/hotspot/share/gc/parallel/psParallelCompact.hpp index 878a09e283b18..3f487ec3ef483 100644 --- a/src/hotspot/share/gc/parallel/psParallelCompact.hpp +++ b/src/hotspot/share/gc/parallel/psParallelCompact.hpp @@ -116,45 +116,51 @@ class SplitInfo // Return true if this split info is valid (i.e., if a split has been // recorded). The very first region cannot have a partial object and thus is // never split, so 0 is the 'invalid' value. - bool is_valid() const { return _split_region_idx > 0; } + bool is_valid() const { return _src_region_idx > 0; } // Return true if this split holds data for the specified source region. - inline bool is_split(size_t region_idx) const; + inline bool is_split(size_t source_region) const; - // Obj at the split point doesn't fit the previous space and will be relocated to the next space. - HeapWord* split_point() const { return _split_point; } + // The index of the split region, the size of the partial object on that + // region and the destination of the partial object. + size_t partial_obj_size() const { return _partial_obj_size; } + HeapWord* destination() const { return _destination; } - // Number of live words before the split point on this region. - size_t preceding_live_words() const { return _preceding_live_words; } + // The destination count of the partial object referenced by this split + // (either 1 or 2). This must be added to the destination count of the + // remainder of the source region. + unsigned int destination_count() const { return _destination_count; } - // A split region has two "destinations", living in two spaces. This method - // returns the first one -- destination for the first live word on - // this split region. - HeapWord* preceding_destination() const { - assert(_preceding_destination != nullptr, "inv"); - return _preceding_destination; - } + // If a word within the partial object will be written to the first word of a + // destination region, this is the address of the destination region; + // otherwise this is null. + HeapWord* dest_region_addr() const { return _dest_region_addr; } - // Number of regions the preceding live words are relocated into. - uint preceding_destination_count() const { return _preceding_destination_count; } + // If a word within the partial object will be written to the first word of a + // destination region, this is the address of that word within the partial + // object; otherwise this is null. + HeapWord* first_src_addr() const { return _first_src_addr; } - void record(size_t split_region_idx, HeapWord* split_point, size_t preceding_live_words); + // Record the data necessary to split the region src_region_idx. + void record(size_t src_region_idx, size_t partial_obj_size, + HeapWord* destination); void clear(); DEBUG_ONLY(void verify_clear();) private: - size_t _split_region_idx; - HeapWord* _split_point; - size_t _preceding_live_words; - HeapWord* _preceding_destination; - uint _preceding_destination_count; + size_t _src_region_idx; + size_t _partial_obj_size; + HeapWord* _destination; + unsigned int _destination_count; + HeapWord* _dest_region_addr; + HeapWord* _first_src_addr; }; inline bool SplitInfo::is_split(size_t region_idx) const { - return _split_region_idx == region_idx && is_valid(); + return _src_region_idx == region_idx && is_valid(); } class SpaceInfo @@ -209,18 +215,10 @@ class ParallelCompactData class RegionData { public: - // Destination for the first live word in this region. - // Therefore, the new addr for every live obj on this region can be calculated as: - // - // new_addr := _destination + live_words_offset(old_addr); - // - // where, live_words_offset is the number of live words accumulated from - // region-start to old_addr. + // Destination address of the region. HeapWord* destination() const { return _destination; } - // A destination region can have multiple source regions; only the first - // one is recorded. Since all live objs are slided down, subsequent source - // regions can be found via plain heap-region iteration. + // The first region containing data destined for this region. size_t source_region() const { return _source_region; } // Reuse _source_region to store the corresponding shadow region index @@ -315,11 +313,8 @@ class ParallelCompactData // Return to the normal path here inline void shadow_to_normal(); - int shadow_state() { return _shadow_state; } - - bool is_clear(); - void verify_clear() NOT_DEBUG_RETURN; + int shadow_state() { return _shadow_state; } private: // The type used to represent object sizes within a region. @@ -878,10 +873,7 @@ class MoveAndUpdateClosure: public StackObj { size_t words_remaining() const { return _words_remaining; } bool is_full() const { return _words_remaining == 0; } HeapWord* source() const { return _source; } - void set_source(HeapWord* addr) { - assert(addr != nullptr, "precondition"); - _source = addr; - } + void set_source(HeapWord* addr) { _source = addr; } // If the object will fit (size <= words_remaining()), copy it to the current // destination, update the interior oops and the start array. @@ -910,8 +902,9 @@ inline size_t MoveAndUpdateClosure::calculate_words_remaining(size_t region) { HeapWord* dest_addr = PSParallelCompact::summary_data().region_to_addr(region); PSParallelCompact::SpaceId dest_space_id = PSParallelCompact::space_id(dest_addr); HeapWord* new_top = PSParallelCompact::new_top(dest_space_id); - return MIN2(pointer_delta(new_top, dest_addr), - ParallelCompactData::RegionSize); + assert(dest_addr < new_top, "sanity"); + + return MIN2(pointer_delta(new_top, dest_addr), ParallelCompactData::RegionSize); } inline