8341977: Replace predicate walking and cloning code for Loop Peeling …

…with a predicate visitor

Reviewed-by: kvn, epeter

src/hotspot/share/opto/loopTransform.cpp

@@ -762,7 +762,7 @@ void PhaseIdealLoop::do_peeling(IdealLoopTree *loop, Node_List &old_new) {
 
   // Step 1: Clone the loop body.  The clone becomes the peeled iteration.
   //         The pre-loop illegally has 2 control users (old & new loops).
-  const uint idx_before_clone = Compile::current()->unique();
+  const uint first_node_index_in_cloned_loop_body = Compile::current()->unique();
   LoopNode* outer_loop_head = head->skip_strip_mined();
   clone_loop(loop, old_new, dom_depth(outer_loop_head), ControlAroundStripMined);
 
@@ -815,19 +815,8 @@ void PhaseIdealLoop::do_peeling(IdealLoopTree *loop, Node_List &old_new) {
 
   // Step 5: Assertion Predicates initialization
   if (counted_loop && UseLoopPredicate) {
-    CountedLoopNode *cl_head = head->as_CountedLoop();
-    Node* init = cl_head->init_trip();
-    Node* stride = cl_head->stride();
-    IdealLoopTree* outer_loop = get_loop(outer_loop_head);
-    const Predicates predicates(new_head->in(LoopNode::EntryControl));
-    initialize_assertion_predicates_for_peeled_loop(predicates.loop_predicate_block(),
-                                                    outer_loop_head, dd_outer_loop_head,
-                                                    init, stride, outer_loop,
-                                                    idx_before_clone, old_new);
-    initialize_assertion_predicates_for_peeled_loop(predicates.profiled_loop_predicate_block(),
-                                                    outer_loop_head, dd_outer_loop_head,
-                                                    init, stride, outer_loop,
-                                                    idx_before_clone, old_new);
+    initialize_assertion_predicates_for_peeled_loop(new_head->as_CountedLoop(), head->as_CountedLoop(),
+                                                    first_node_index_in_cloned_loop_body, old_new);
  }
 
   // Now force out all loop-invariant dominating tests.  The optimizer
@@ -1434,7 +1423,6 @@ bool PhaseIdealLoop::assertion_predicate_has_loop_opaque_node(IfNode* iff) {
   assert(stride == 0 || init != 0, "init should be there every time stride is");
   return init != 0;
 }
-#endif // ASSERT
 
 void PhaseIdealLoop::count_opaque_loop_nodes(Node* n, uint& init, uint& stride) {
   init = 0;
@@ -1460,6 +1448,7 @@ void PhaseIdealLoop::count_opaque_loop_nodes(Node* n, uint& init, uint& stride)
     }
   }
 }
+#endif // ASSERT
 
 // Create an Initialized Assertion Predicate from the template_assertion_predicate
 IfTrueNode* PhaseIdealLoop::create_initialized_assertion_predicate(IfNode* template_assertion_predicate, Node* new_init,
@@ -1978,55 +1967,32 @@ void PhaseIdealLoop::copy_assertion_predicates_to_post_loop(LoopNode* main_loop_
   }
 }
 
-void PhaseIdealLoop::initialize_assertion_predicates_for_peeled_loop(const PredicateBlock* predicate_block,
-                                                                     LoopNode* outer_loop_head,
-                                                                     const int dd_outer_loop_head, Node* init,
-                                                                     Node* stride, IdealLoopTree* outer_loop,
-                                                                     const uint idx_before_clone,
-                                                                     const Node_List &old_new) {
-  if (!predicate_block->has_parse_predicate()) {
-    return;
-  }
-  Node* input_proj = outer_loop_head->in(LoopNode::EntryControl);
-  const Node* parse_predicate_uncommon_trap = predicate_block->parse_predicate()->uncommon_trap();
-  Node* next_regular_predicate_proj = predicate_block->skip_parse_predicate();
-  while (next_regular_predicate_proj->is_IfProj()) {
-    IfNode* iff = next_regular_predicate_proj->in(0)->as_If();
-    ProjNode* uncommon_proj = iff->proj_out(1 - next_regular_predicate_proj->as_Proj()->_con);
-    if (uncommon_proj->unique_ctrl_out() != parse_predicate_uncommon_trap) {
-      // Does not belong to this Predicate Block anymore.
-      break;
-    }
-    Node* bol = iff->in(1);
-    assert(!bol->is_OpaqueInitializedAssertionPredicate(), "should not find an Initialized Assertion Predicate");
-    if (bol->is_OpaqueTemplateAssertionPredicate()) {
-      // Initialize from Template Assertion Predicate.
-      input_proj = create_initialized_assertion_predicate(iff, init, stride, input_proj);
-
-      // Rewire any control inputs from the old Assertion Predicates above the peeled iteration down to the initialized
-      // Assertion Predicates above the peeled loop.
-      for (DUIterator i = next_regular_predicate_proj->outs(); next_regular_predicate_proj->has_out(i); i++) {
-        Node* dependent = next_regular_predicate_proj->out(i);
-        Node* new_node = old_new[dependent->_idx];
-
-        if (!dependent->is_CFG() &&
-            dependent->_idx < idx_before_clone &&  // old node
-            new_node != nullptr &&                 // cloned
-            new_node->_idx >= idx_before_clone) {  // for peeling
-          // The old nodes from the peeled loop still point to the predicate above the peeled loop.
-          // We need to rewire the dependencies to the newly Initialized Assertion Predicates.
-          _igvn.replace_input_of(dependent, 0, input_proj);
-          --i; // correct for just deleted predicate->out(i)
-        }
-      }
-    }
-    next_regular_predicate_proj = iff->in(0);
-  }
-
-  _igvn.replace_input_of(outer_loop_head, LoopNode::EntryControl, input_proj);
-  set_idom(outer_loop_head, input_proj, dd_outer_loop_head);
+void PhaseIdealLoop::initialize_assertion_predicates_for_peeled_loop(CountedLoopNode* peeled_loop_head,
+                                                                     CountedLoopNode* remaining_loop_head,
+                                                                     const uint first_node_index_in_cloned_loop_body,
+                                                                     const Node_List& old_new) {
+  const NodeInOriginalLoopBody node_in_original_loop_body(first_node_index_in_cloned_loop_body, old_new);
+  create_assertion_predicates_at_loop(peeled_loop_head, remaining_loop_head, node_in_original_loop_body);
 }
 
+void PhaseIdealLoop::create_assertion_predicates_at_loop(CountedLoopNode* source_loop_head,
+                                                         CountedLoopNode* target_loop_head,
+                                                         const NodeInLoopBody& _node_in_loop_body) {
+  Node* init = target_loop_head->init_trip();
+  Node* stride = target_loop_head->stride();
+  LoopNode* target_outer_loop_head = target_loop_head->skip_strip_mined();
+  Node* target_loop_entry = target_outer_loop_head->in(LoopNode::EntryControl);
+  CreateAssertionPredicatesVisitor create_assertion_predicates_visitor(init, stride, target_loop_entry, this,
+                                                                       _node_in_loop_body);
+  Node* source_loop_entry = source_loop_head->skip_strip_mined()->in(LoopNode::EntryControl);
+  PredicateIterator predicate_iterator(source_loop_entry);
+  predicate_iterator.for_each(create_assertion_predicates_visitor);
+  if (create_assertion_predicates_visitor.has_created_predicates()) {
+    IfTrueNode* last_created_predicate_success_proj = create_assertion_predicates_visitor.last_created_success_proj();
+    _igvn.replace_input_of(target_outer_loop_head, LoopNode::EntryControl, last_created_predicate_success_proj);
+    set_idom(target_outer_loop_head, last_created_predicate_success_proj, dom_depth(target_outer_loop_head));
+  }
+}
 //------------------------------do_unroll--------------------------------------
 // Unroll the loop body one step - make each trip do 2 iterations.
 void PhaseIdealLoop::do_unroll(IdealLoopTree *loop, Node_List &old_new, bool adjust_min_trip) {

src/hotspot/share/opto/loopnode.hpp

@@ -952,18 +952,22 @@ class PhaseIdealLoop : public PhaseTransform {
                                               Node* zero_trip_guard_proj_post, const Node_List& old_new);
   Node* clone_template_assertion_predicate(IfNode* iff, Node* new_init, Node* predicate, Node* uncommon_proj, Node* control,
                                            IdealLoopTree* outer_loop, Node* new_control);
+ public:
   IfTrueNode* create_initialized_assertion_predicate(IfNode* template_assertion_predicate, Node* new_init,
                                                      Node* new_stride, Node* control);
-  static void count_opaque_loop_nodes(Node* n, uint& init, uint& stride);
+ private:
+  DEBUG_ONLY(static void count_opaque_loop_nodes(Node* n, uint& init, uint& stride);)
   DEBUG_ONLY(static bool assertion_predicate_has_loop_opaque_node(IfNode* iff);)
   static void get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque = false);
   void update_main_loop_assertion_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, int stride_con);
   void copy_assertion_predicates_to_post_loop(LoopNode* main_loop_head, CountedLoopNode* post_loop_head,
                                               Node* stride);
-  void initialize_assertion_predicates_for_peeled_loop(const PredicateBlock* predicate_block, LoopNode* outer_loop_head,
-                                                       int dd_outer_loop_head, Node* init, Node* stride,
-                                                       IdealLoopTree* outer_loop, uint idx_before_clone,
+  void initialize_assertion_predicates_for_peeled_loop(CountedLoopNode* peeled_loop_head,
+                                                       CountedLoopNode* remaining_loop_head,
+                                                       uint first_node_index_in_cloned_loop_body,
                                                        const Node_List& old_new);
+  void create_assertion_predicates_at_loop(CountedLoopNode* source_loop_head, CountedLoopNode* target_loop_head,
+                                           const NodeInLoopBody& _node_in_loop_body);
   void insert_loop_limit_check_predicate(ParsePredicateSuccessProj* loop_limit_check_parse_proj, Node* cmp_limit,
                                          Node* bol);
   void log_loop_tree();

src/hotspot/share/opto/predicates.cpp

@@ -138,6 +138,21 @@ bool RuntimePredicate::is_predicate(Node* node, Deoptimization::DeoptReason deop
   return RegularPredicateWithUCT::is_predicate(node, deopt_reason);
 }
 
+// Rewire any non-CFG nodes dependent on this Template Assertion Predicate (i.e. with a control input to this
+// Template Assertion Predicate) to the 'target_predicate' based on the 'data_in_loop_body' check.
+void TemplateAssertionPredicate::rewire_loop_data_dependencies(IfTrueNode* target_predicate,
+                                                               const NodeInLoopBody& data_in_loop_body,
+                                                               PhaseIdealLoop* phase) const {
+  for (DUIterator i = _success_proj->outs(); _success_proj->has_out(i); i++) {
+    Node* output = _success_proj->out(i);
+    if (!output->is_CFG() && data_in_loop_body.check(output)) {
+      phase->igvn().replace_input_of(output, 0, target_predicate);
+      --i; // account for the just deleted output
+    }
+  }
+}
+
+
 // Template Assertion Predicates always have the dedicated OpaqueTemplateAssertionPredicate to identify them.
 bool TemplateAssertionPredicate::is_predicate(Node* node) {
   if (!may_be_assertion_predicate_if(node)) {
@@ -712,3 +727,25 @@ void Predicates::dump_for_loop(LoopNode* loop_node) {
   dump_at(loop_node->skip_strip_mined()->in(LoopNode::EntryControl));
 }
 #endif // NOT PRODUCT
+
+// Keep track of whether we are in the correct Predicate Block where Template Assertion Predicates can be found.
+// The PredicateIterator will always start at the loop entry and first visits the Loop Limit Check Predicate Block.
+void CreateAssertionPredicatesVisitor::visit(const ParsePredicate& parse_predicate) {
+  Deoptimization::DeoptReason deopt_reason = parse_predicate.head()->deopt_reason();
+  if (deopt_reason == Deoptimization::Reason_predicate ||
+      deopt_reason == Deoptimization::Reason_profile_predicate) {
+    _has_hoisted_check_parse_predicates = true;
+  }
+}
+
+void CreateAssertionPredicatesVisitor::visit(const TemplateAssertionPredicate& template_assertion_predicate) {
+  if (!_has_hoisted_check_parse_predicates) {
+    // Only process if we are in the correct Predicate Block.
+    return;
+  }
+  IfNode* template_head = template_assertion_predicate.head();
+  IfTrueNode* initialized_predicate = _phase->create_initialized_assertion_predicate(template_head, _init, _stride,
+                                                                                     _new_control);
+  template_assertion_predicate.rewire_loop_data_dependencies(initialized_predicate, _node_in_loop_body, _phase);
+  _new_control = initialized_predicate;
+}

src/hotspot/share/opto/predicates.hpp

@@ -248,6 +248,12 @@ class PredicateVisitor : StackObj {
   }
 };
 
+// Interface to check whether a node is in a loop body or not.
+class NodeInLoopBody : public StackObj {
+ public:
+  virtual bool check(Node* node) const = 0;
+};
+
 // Class to represent Assertion Predicates with a HaltNode instead of an UCT (i.e. either an Initialized Assertion
 // Predicate or a Template Assertion Predicate created after the initial one at Loop Predication).
 class AssertionPredicatesWithHalt : public StackObj {
@@ -393,6 +399,8 @@ class TemplateAssertionPredicate : public Predicate {
     return _success_proj;
   }
 
+  void rewire_loop_data_dependencies(IfTrueNode* target_predicate, const NodeInLoopBody& data_in_loop_body,
+                                     PhaseIdealLoop* phase) const;
   static bool is_predicate(Node* node);
 };
 
@@ -910,4 +918,70 @@ class Predicates : public StackObj {
 #endif // NOT PRODUCT
 };
 
+// This class checks whether a node is in the original loop body and not the cloned one.
+class NodeInOriginalLoopBody : public NodeInLoopBody {
+  const uint _first_node_index_in_cloned_loop_body;
+  const Node_List& _old_new;
+
+ public:
+  NodeInOriginalLoopBody(const uint first_node_index_in_cloned_loop_body, const Node_List& old_new)
+      : _first_node_index_in_cloned_loop_body(first_node_index_in_cloned_loop_body),
+        _old_new(old_new) {}
+  NONCOPYABLE(NodeInOriginalLoopBody);
+
+  // Check if 'node' is not a cloned node (i.e. "< _first_node_index_in_cloned_loop_body") and if we've created a
+  // clone from 'node' (i.e. _old_new entry is non-null). Then we know that 'node' belongs to the original loop body.
+  bool check(Node* node) const override {
+    if (node->_idx < _first_node_index_in_cloned_loop_body) {
+      Node* cloned_node = _old_new[node->_idx];
+      return cloned_node != nullptr && cloned_node->_idx >= _first_node_index_in_cloned_loop_body;
+    } else {
+      return false;
+    }
+  }
+};
+
+// Visitor to create Initialized Assertion Predicates at a target loop from Template Assertion Predicates from a source
+// loop. This visitor can be used in combination with a PredicateIterator.
+class CreateAssertionPredicatesVisitor : public PredicateVisitor {
+  Node* const _init;
+  Node* const _stride;
+  Node* const _old_target_loop_entry;
+  Node* _new_control;
+  PhaseIdealLoop* const _phase;
+  bool _has_hoisted_check_parse_predicates;
+  const NodeInLoopBody& _node_in_loop_body;
+
+ public:
+  CreateAssertionPredicatesVisitor(Node* init, Node* stride, Node* new_control, PhaseIdealLoop* phase,
+                                   const NodeInLoopBody& node_in_loop_body)
+      : _init(init),
+        _stride(stride),
+        _old_target_loop_entry(new_control),
+        _new_control(new_control),
+        _phase(phase),
+        _has_hoisted_check_parse_predicates(false),
+        _node_in_loop_body(node_in_loop_body) {}
+  NONCOPYABLE(CreateAssertionPredicatesVisitor);
+
+  using PredicateVisitor::visit;
+
+  void visit(const ParsePredicate& parse_predicate) override;
+  void visit(const TemplateAssertionPredicate& template_assertion_predicate) override;
+
+  // Did we create any new Initialized Assertion Predicates?
+  bool has_created_predicates() const {
+    return _new_control != _old_target_loop_entry;
+  }
+
+  // Return the last created node by this visitor or the originally provided 'new_control' to the visitor if there was
+  // no new node created (i.e. no Template Assertion Predicates found).
+  IfTrueNode* last_created_success_proj() const {
+    assert(has_created_predicates(), "should only be queried if new nodes have been created");
+    assert(_new_control->unique_ctrl_out_or_null() == nullptr, "no control outputs, yet");
+    assert(_new_control->is_IfTrue(), "Assertion Predicates only have IfTrue on success proj");
+    return _new_control->as_IfTrue();
+  }
+};
+
 #endif // SHARE_OPTO_PREDICATES_HPP