8357471: GenShen: Share collector reserves between young and old

src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp

@@ -175,12 +175,24 @@ void ShenandoahGenerationalHeuristics::choose_collection_set(ShenandoahCollectio
   bool doing_promote_in_place = (humongous_regions_promoted + regular_regions_promoted_in_place > 0);
   if (doing_promote_in_place || (preselected_candidates > 0) || (immediate_percent <= ShenandoahImmediateThreshold)) {
     // Only young collections need to prime the collection set.
+    bool need_to_finalize_mixed = false;
     if (_generation->is_young()) {
-      heap->old_generation()->heuristics()->prime_collection_set(collection_set);
+      need_to_finalize_mixed = heap->old_generation()->heuristics()->prime_collection_set(collection_set);
     }
 
     // Call the subclasses to add young-gen regions into the collection set.
     choose_collection_set_from_regiondata(collection_set, candidates, cand_idx, immediate_garbage + free);
+
+    if (_generation->is_young()) {
+      // Especially when young-gen trigger is expedited in order to finish mixed evacuations, there may not be
+      // enough consolidated garbage to make effective use of young-gen evacuation reserve.  If there is still
+      // young-gen reserve available following selection of the young-gen collection set, see if we can use
+      // this memory to expand the old-gen evacuation collection set.
+      need_to_finalize_mixed |= heap->old_generation()->heuristics()->top_off_collection_set();
+      if (need_to_finalize_mixed) {
+        heap->old_generation()->heuristics()->finalize_mixed_evacs();
+      }
+    }
   }
 
   if (collection_set->has_old_regions()) {

src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp

@@ -53,93 +53,134 @@ void ShenandoahGlobalHeuristics::choose_global_collection_set(ShenandoahCollecti
                                                               size_t cur_young_garbage) const {
   auto heap = ShenandoahGenerationalHeap::heap();
   size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
-  size_t capacity = heap->young_generation()->max_capacity();
+  size_t young_capacity = heap->young_generation()->max_capacity();
+  size_t old_capacity = heap->old_generation()->max_capacity();
   size_t garbage_threshold = region_size_bytes * ShenandoahGarbageThreshold / 100;
   size_t ignore_threshold = region_size_bytes * ShenandoahIgnoreGarbageThreshold / 100;
   const uint tenuring_threshold = heap->age_census()->tenuring_threshold();
 
   size_t young_evac_reserve = heap->young_generation()->get_evacuation_reserve();
   size_t old_evac_reserve = heap->old_generation()->get_evacuation_reserve();
-  size_t max_young_cset = (size_t) (young_evac_reserve / ShenandoahEvacWaste);
-  size_t young_cur_cset = 0;
-  size_t max_old_cset = (size_t) (old_evac_reserve / ShenandoahOldEvacWaste);
-  size_t old_cur_cset = 0;
-
-  // Figure out how many unaffiliated young regions are dedicated to mutator and to evacuator.  Allow the young
-  // collector's unaffiliated regions to be transferred to old-gen if old-gen has more easily reclaimed garbage
-  // than young-gen.  At the end of this cycle, any excess regions remaining in old-gen will be transferred back
-  // to young.  Do not transfer the mutator's unaffiliated regions to old-gen.  Those must remain available
-  // to the mutator as it needs to be able to consume this memory during concurrent GC.
 
   size_t unaffiliated_young_regions = heap->young_generation()->free_unaffiliated_regions();
   size_t unaffiliated_young_memory = unaffiliated_young_regions * region_size_bytes;
-
-  if (unaffiliated_young_memory > max_young_cset) {
-    size_t unaffiliated_mutator_memory = unaffiliated_young_memory - max_young_cset;
-    unaffiliated_young_memory -= unaffiliated_mutator_memory;
-    unaffiliated_young_regions = unaffiliated_young_memory / region_size_bytes; // round down
-    unaffiliated_young_memory = unaffiliated_young_regions * region_size_bytes;
+  size_t unaffiliated_old_regions = heap->old_generation()->free_unaffiliated_regions();
+  size_t unaffiliated_old_memory = unaffiliated_old_regions * region_size_bytes;
+
+  // Figure out how many unaffiliated regions are dedicated to Collector and OldCollector reserves.  Let these
+  // be shuffled between young and old generations in order to expedite evacuation of whichever regions have the
+  // most garbage, regardless of whether these garbage-first regions reside in young or old generation.
+  // Excess reserves will be transferred back to the mutator after collection set has been chosen.  At the end
+  // of evacuation, any reserves not consumed by evacuation will also be transferred to the mutator free set.
+  size_t shared_reserve_regions = 0;
+  if (young_evac_reserve > unaffiliated_young_memory) {
+    young_evac_reserve -= unaffiliated_young_memory;
+    shared_reserve_regions += unaffiliated_young_memory / region_size_bytes;
+  } else {
+    size_t delta_regions = young_evac_reserve / region_size_bytes;
+    shared_reserve_regions += delta_regions;
+    young_evac_reserve -= delta_regions * region_size_bytes;
+  }
+  if (old_evac_reserve > unaffiliated_old_memory) {
+    old_evac_reserve -= unaffiliated_old_memory;
+    shared_reserve_regions += unaffiliated_old_memory / region_size_bytes;
+  } else {
+    size_t delta_regions = old_evac_reserve / region_size_bytes;
+    shared_reserve_regions += delta_regions;
+    old_evac_reserve -= delta_regions * region_size_bytes;
   }
 
-  // We'll affiliate these unaffiliated regions with either old or young, depending on need.
-  max_young_cset -= unaffiliated_young_memory;
+  size_t shared_reserves = shared_reserve_regions * region_size_bytes;
+  size_t committed_from_shared_reserves = 0;
+  size_t max_young_cset = (size_t) (young_evac_reserve / ShenandoahEvacWaste);
+  size_t young_cur_cset = 0;
+  size_t max_old_cset = (size_t) (old_evac_reserve / ShenandoahOldEvacWaste);
+  size_t old_cur_cset = 0;
 
-  // Keep track of how many regions we plan to transfer from young to old.
-  size_t regions_transferred_to_old = 0;
+  size_t promo_bytes = 0;
+  size_t old_evac_bytes = 0;
+  size_t young_evac_bytes = 0;
 
-  size_t free_target = (capacity * ShenandoahMinFreeThreshold) / 100 + max_young_cset;
+  size_t max_total_cset = (max_young_cset + max_old_cset +
+                           (size_t) (shared_reserve_regions * region_size_bytes) / ShenandoahOldEvacWaste);
+  size_t free_target = ((young_capacity + old_capacity) * ShenandoahMinFreeThreshold) / 100 + max_total_cset;
   size_t min_garbage = (free_target > actual_free) ? (free_target - actual_free) : 0;
 
   log_info(gc, ergo)("Adaptive CSet Selection for GLOBAL. Max Young Evacuation: %zu"
-                     "%s, Max Old Evacuation: %zu%s, Actual Free: %zu%s.",
+                     "%s, Max Old Evacuation: %zu%s, Discretionary additional evacuation: %zu%s, Actual Free: %zu%s.",
                      byte_size_in_proper_unit(max_young_cset), proper_unit_for_byte_size(max_young_cset),
                      byte_size_in_proper_unit(max_old_cset), proper_unit_for_byte_size(max_old_cset),
+                     byte_size_in_proper_unit(shared_reserves), proper_unit_for_byte_size(shared_reserves),
                      byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free));
 
+  size_t cur_garbage = cur_young_garbage;
   for (size_t idx = 0; idx < size; idx++) {
     ShenandoahHeapRegion* r = data[idx].get_region();
     assert(!cset->is_preselected(r->index()), "There should be no preselected regions during GLOBAL GC");
     bool add_region = false;
+    size_t region_garbage = r->garbage();
+    size_t new_garbage = cur_garbage + region_garbage;
+    bool add_regardless = (region_garbage > ignore_threshold) && (new_garbage < min_garbage);
     if (r->is_old() || (r->age() >= tenuring_threshold)) {
-      size_t new_cset = old_cur_cset + r->get_live_data_bytes();
-      if ((r->garbage() > garbage_threshold)) {
-        while ((new_cset > max_old_cset) && (unaffiliated_young_regions > 0)) {
-          unaffiliated_young_regions--;
-          regions_transferred_to_old++;
+      if (add_regardless || (region_garbage > garbage_threshold)) {
+        size_t live_bytes = r->get_live_data_bytes();
+        size_t new_cset = old_cur_cset + r->get_live_data_bytes();
+        // May need multiple reserve regions to evacuate a single region, depending on live data bytes and ShenandoahOldEvacWaste
+        size_t orig_max_old_cset = max_old_cset;
+        size_t proposed_old_region_consumption = 0;
+        while ((new_cset > max_old_cset) && (committed_from_shared_reserves < shared_reserves)) {
+          committed_from_shared_reserves += region_size_bytes;
+          proposed_old_region_consumption++;
           max_old_cset += region_size_bytes / ShenandoahOldEvacWaste;
         }
-      }
-      if ((new_cset <= max_old_cset) && (r->garbage() > garbage_threshold)) {
-        add_region = true;
-        old_cur_cset = new_cset;
+        // We already know: add_regardless || region_garbage > garbage_threshold
+        if (new_cset <= max_old_cset) {
+          add_region = true;
+          old_cur_cset = new_cset;
+          cur_garbage = new_garbage;
+          if (r->is_old()) {
+            old_evac_bytes += live_bytes;
+          } else {
+            promo_bytes += live_bytes;
+          }
+        } else {
+          // We failed to sufficiently expand old, so unwind proposed expansion
+          max_old_cset = orig_max_old_cset;
+          committed_from_shared_reserves -= proposed_old_region_consumption * region_size_bytes;
+        }
       }
     } else {
       assert(r->is_young() && (r->age() < tenuring_threshold), "DeMorgan's law (assuming r->is_affiliated)");
-      size_t new_cset = young_cur_cset + r->get_live_data_bytes();
-      size_t region_garbage = r->garbage();
-      size_t new_garbage = cur_young_garbage + region_garbage;
-      bool add_regardless = (region_garbage > ignore_threshold) && (new_garbage < min_garbage);
-
-      if (add_regardless || (r->garbage() > garbage_threshold)) {
-        while ((new_cset > max_young_cset) && (unaffiliated_young_regions > 0)) {
-          unaffiliated_young_regions--;
+      if (add_regardless || (region_garbage > garbage_threshold)) {
+        size_t live_bytes = r->get_live_data_bytes();
+        size_t new_cset = young_cur_cset + live_bytes;
+        // May need multiple reserve regions to evacuate a single region, depending on live data bytes and ShenandoahEvacWaste
+        size_t orig_max_young_cset = max_young_cset;
+        size_t proposed_young_region_consumption = 0;
+        while ((new_cset > max_young_cset) && (committed_from_shared_reserves < shared_reserves)) {
+          committed_from_shared_reserves += region_size_bytes;
+          proposed_young_region_consumption++;
           max_young_cset += region_size_bytes / ShenandoahEvacWaste;
         }
-      }
-      if ((new_cset <= max_young_cset) && (add_regardless || (region_garbage > garbage_threshold))) {
-        add_region = true;
-        young_cur_cset = new_cset;
-        cur_young_garbage = new_garbage;
+        // We already know: add_regardless || region_garbage > garbage_threshold
+        if (new_cset <= max_young_cset) {
+          add_region = true;
+          young_cur_cset = new_cset;
+          cur_garbage = new_garbage;
+          young_evac_bytes += live_bytes;
+        } else {
+          // We failed to sufficiently expand young, so unwind proposed expansion
+          max_young_cset = orig_max_young_cset;
+          committed_from_shared_reserves -= proposed_young_region_consumption * region_size_bytes;
+        }
       }
     }
     if (add_region) {
       cset->add_region(r);
     }
   }
 
-  if (regions_transferred_to_old > 0) {
-    heap->generation_sizer()->force_transfer_to_old(regions_transferred_to_old);
-    heap->young_generation()->set_evacuation_reserve(young_evac_reserve - regions_transferred_to_old * region_size_bytes);
-    heap->old_generation()->set_evacuation_reserve(old_evac_reserve + regions_transferred_to_old * region_size_bytes);
-  }
+  heap->young_generation()->set_evacuation_reserve((size_t) (young_evac_bytes * ShenandoahEvacWaste));
+  heap->old_generation()->set_evacuation_reserve((size_t) (old_evac_bytes * ShenandoahOldEvacWaste));
+  heap->old_generation()->set_promoted_reserve((size_t) (promo_bytes * ShenandoahPromoEvacWaste));
 }