8344171: Clone and initialize Assertion Predicates in order instead o…

…f in reverse-order

src/hotspot/share/opto/loopTransform.cpp

@@ -1753,20 +1753,11 @@ void PhaseIdealLoop::create_assertion_predicates_at_loop(CountedLoopNode* source
                                                           CountedLoopNode* target_loop_head,
                                                           const NodeInLoopBody& _node_in_loop_body,
                                                           const bool clone_template) {
-  Node* init = target_loop_head->init_trip();
-  Node* stride = target_loop_head->stride();
-  LoopNode* target_outer_loop_head = target_loop_head->skip_strip_mined();
-  Node* target_loop_entry = target_outer_loop_head->in(LoopNode::EntryControl);
-  CreateAssertionPredicatesVisitor create_assertion_predicates_visitor(init, stride, target_loop_entry, this,
-                                                                       _node_in_loop_body, clone_template);
+  CreateAssertionPredicatesVisitor create_assertion_predicates_visitor(target_loop_head, this, _node_in_loop_body,
+                                                                       clone_template);
   Node* source_loop_entry = source_loop_head->skip_strip_mined()->in(LoopNode::EntryControl);
   PredicateIterator predicate_iterator(source_loop_entry);
   predicate_iterator.for_each(create_assertion_predicates_visitor);
-  if (create_assertion_predicates_visitor.has_created_predicates()) {
-    IfTrueNode* last_created_predicate_success_proj = create_assertion_predicates_visitor.last_created_success_proj();
-    _igvn.replace_input_of(target_outer_loop_head, LoopNode::EntryControl, last_created_predicate_success_proj);
-    set_idom(target_outer_loop_head, last_created_predicate_success_proj, dom_depth(target_outer_loop_head));
-  }
 }
 //------------------------------do_unroll--------------------------------------
 // Unroll the loop body one step - make each trip do 2 iterations.

src/hotspot/share/opto/loopnode.hpp

@@ -1363,6 +1363,12 @@ class PhaseIdealLoop : public PhaseTransform {
  public:
   void register_control(Node* n, IdealLoopTree *loop, Node* pred, bool update_body = true);
 
+  // Replace the control input of 'node' with 'new_control' and set the dom depth to the one of 'new_control'.
+  void replace_control_same_depth(Node* node, Node* new_control) {
+    _igvn.replace_input_of(node, 0, new_control);
+    set_idom(node, new_control, dom_depth(new_control));
+  }
+
   void replace_loop_entry(LoopNode* loop_head, Node* new_entry) {
     _igvn.replace_input_of(loop_head, LoopNode::EntryControl, new_entry);
     set_idom(loop_head, new_entry, dom_depth(new_entry));

src/hotspot/share/opto/predicates.cpp

@@ -879,6 +879,19 @@ void Predicates::dump_for_loop(LoopNode* loop_node) {
 }
 #endif // NOT PRODUCT
 
+CreateAssertionPredicatesVisitor::CreateAssertionPredicatesVisitor(CountedLoopNode* target_loop_head,
+                                                                   PhaseIdealLoop* phase,
+                                                                   const NodeInLoopBody& node_in_loop_body,
+                                                                   const bool clone_template)
+    : _init(target_loop_head->init_trip()),
+      _stride(target_loop_head->stride()),
+      _old_target_loop_entry(target_loop_head->skip_strip_mined()->in(LoopNode::EntryControl)),
+      _current_predicate_chain_head(target_loop_head->skip_strip_mined()), // Initially no predicates, yet.
+      _phase(phase),
+      _has_hoisted_check_parse_predicates(false),
+      _node_in_loop_body(node_in_loop_body),
+      _clone_template(clone_template) {}
+
 // Keep track of whether we are in the correct Predicate Block where Template Assertion Predicates can be found.
 // The PredicateIterator will always start at the loop entry and first visits the Loop Limit Check Predicate Block.
 void CreateAssertionPredicatesVisitor::visit(const ParsePredicate& parse_predicate) {
@@ -895,30 +908,48 @@ void CreateAssertionPredicatesVisitor::visit(const TemplateAssertionPredicate& t
     return;
   }
   if (_clone_template) {
-    _new_control = clone_template_and_replace_init_input(template_assertion_predicate);
+    IfTrueNode* cloned_template_success_proj = clone_template_and_replace_init_input(template_assertion_predicate);
+    initialize_from_template(template_assertion_predicate, cloned_template_success_proj);
+    _current_predicate_chain_head = cloned_template_success_proj->in(0);
+  } else {
+    IfTrueNode* initialized_success_proj = initialize_from_template(template_assertion_predicate, _old_target_loop_entry);
+    _current_predicate_chain_head = initialized_success_proj->in(0);
   }
-  _new_control = initialize_from_template(template_assertion_predicate);
 }
 
 // Create an Initialized Assertion Predicate from the provided Template Assertion Predicate.
 IfTrueNode* CreateAssertionPredicatesVisitor::initialize_from_template(
-    const TemplateAssertionPredicate& template_assertion_predicate) const {
+    const TemplateAssertionPredicate& template_assertion_predicate, Node* new_control) const {
   DEBUG_ONLY(template_assertion_predicate.verify();)
   IfNode* template_head = template_assertion_predicate.head();
   InitializedAssertionPredicateCreator initialized_assertion_predicate(_phase);
-  IfTrueNode* initialized_predicate = initialized_assertion_predicate.create_from_template(template_head,_new_control,
+  IfTrueNode* initialized_predicate = initialized_assertion_predicate.create_from_template(template_head, new_control,
                                                                                            _init, _stride);
   DEBUG_ONLY(InitializedAssertionPredicate::verify(initialized_predicate);)
   template_assertion_predicate.rewire_loop_data_dependencies(initialized_predicate, _node_in_loop_body, _phase);
+  rewire_to_old_predicate_chain_head(initialized_predicate);
   return initialized_predicate;
 }
 
 // Clone the provided 'template_assertion_predicate' and set '_init' as new input for the OpaqueLoopInitNode.
 IfTrueNode* CreateAssertionPredicatesVisitor::clone_template_and_replace_init_input(
     const TemplateAssertionPredicate& template_assertion_predicate) {
   OpaqueLoopInitNode* opaque_init = new OpaqueLoopInitNode(_phase->C, _init);
-  _phase->register_new_node(opaque_init, _new_control);
-  return template_assertion_predicate.clone_and_replace_init(_new_control, opaque_init, _phase);
+  _phase->register_new_node(opaque_init, _old_target_loop_entry);
+  return template_assertion_predicate.clone_and_replace_init(_old_target_loop_entry, opaque_init, _phase);
+}
+
+// Rewire the newly created predicates to the old predicate chain head (i.e. '_current_predicate_chain_head') by
+// rewiring its control input from '_old_target_loop_entry' to 'initialized_assertion_predicate_success_proj'.
+void CreateAssertionPredicatesVisitor::rewire_to_old_predicate_chain_head(
+    Node* initialized_assertion_predicate_success_proj) const {
+  if (_current_predicate_chain_head->is_Loop()) {
+    assert(_current_predicate_chain_head->in(LoopNode::EntryControl) == _old_target_loop_entry, "must be old loop entry");
+    _phase->replace_loop_entry(_current_predicate_chain_head->as_Loop(), initialized_assertion_predicate_success_proj);
+  } else {
+    assert(_current_predicate_chain_head->in(0) == _old_target_loop_entry, "must be old loop entry");
+    _phase->replace_control_same_depth(_current_predicate_chain_head, initialized_assertion_predicate_success_proj);
+  }
 }
 
 // Clone the Template Assertion Predicate and set a new input for the OpaqueLoopStrideNode.
@@ -951,9 +982,8 @@ IfTrueNode* UpdateStrideForAssertionPredicates::initialize_from_updated_template
 void UpdateStrideForAssertionPredicates::connect_initialized_assertion_predicate(
     Node* new_control_out, IfTrueNode* initialized_success_proj) const {
   if (new_control_out->is_Loop()) {
-    _phase->igvn().replace_input_of(new_control_out, LoopNode::EntryControl, initialized_success_proj);
+    _phase->replace_loop_entry(new_control_out->as_Loop(), initialized_success_proj);
   } else {
-    _phase->igvn().replace_input_of(new_control_out, 0, initialized_success_proj);
+    _phase->replace_control_same_depth(new_control_out, initialized_success_proj);
   }
-  _phase->set_idom(new_control_out, initialized_success_proj, _phase->dom_depth(new_control_out));
 }

src/hotspot/share/opto/predicates.hpp

@@ -984,46 +984,26 @@ class CreateAssertionPredicatesVisitor : public PredicateVisitor {
   Node* const _init;
   Node* const _stride;
   Node* const _old_target_loop_entry;
-  Node* _new_control;
+  Node* _current_predicate_chain_head;
   PhaseIdealLoop* const _phase;
   bool _has_hoisted_check_parse_predicates;
   const NodeInLoopBody& _node_in_loop_body;
   const bool _clone_template;
 
   IfTrueNode* clone_template_and_replace_init_input(const TemplateAssertionPredicate& template_assertion_predicate);
-  IfTrueNode* initialize_from_template(const TemplateAssertionPredicate& template_assertion_predicate) const;
+  IfTrueNode* initialize_from_template(const TemplateAssertionPredicate& template_assertion_predicate,
+                                       Node* new_control) const;
+  void rewire_to_old_predicate_chain_head(Node* initialized_assertion_predicate_success_proj) const;
 
  public:
-  CreateAssertionPredicatesVisitor(Node* init, Node* stride, Node* new_control, PhaseIdealLoop* phase,
-                                   const NodeInLoopBody& node_in_loop_body, const bool clone_template)
-      : _init(init),
-        _stride(stride),
-        _old_target_loop_entry(new_control),
-        _new_control(new_control),
-        _phase(phase),
-        _has_hoisted_check_parse_predicates(false),
-        _node_in_loop_body(node_in_loop_body),
-        _clone_template(clone_template) {}
+  CreateAssertionPredicatesVisitor(CountedLoopNode* target_loop_head, PhaseIdealLoop* phase,
+                                   const NodeInLoopBody& node_in_loop_body, bool clone_template);
   NONCOPYABLE(CreateAssertionPredicatesVisitor);
 
   using PredicateVisitor::visit;
 
   void visit(const ParsePredicate& parse_predicate) override;
   void visit(const TemplateAssertionPredicate& template_assertion_predicate) override;
-
-  // Did we create any new Initialized Assertion Predicates?
-  bool has_created_predicates() const {
-    return _new_control != _old_target_loop_entry;
-  }
-
-  // Return the last created node by this visitor or the originally provided 'new_control' to the visitor if there was
-  // no new node created (i.e. no Template Assertion Predicates found).
-  IfTrueNode* last_created_success_proj() const {
-    assert(has_created_predicates(), "should only be queried if new nodes have been created");
-    assert(_new_control->unique_ctrl_out_or_null() == nullptr, "no control outputs, yet");
-    assert(_new_control->is_IfTrue(), "Assertion Predicates only have IfTrue on success proj");
-    return _new_control->as_IfTrue();
-  }
 };
 
 // This visitor collects all Template Assertion Predicates If nodes or the corresponding Opaque nodes, depending on the