Revert "8344035: Replace predicate walking code in Loop Unswitching w…

…ith a predicate visitor"

This reverts commit 5509336.

src/hotspot/share/opto/loopPredicate.cpp

@@ -272,15 +272,150 @@ void PhaseIdealLoop::fix_cloned_data_node_controls(const ProjNode* old_uncommon_
   orig_to_clone.iterate_all(orig_clone_action);
 }
 
-// Put all OpaqueTemplateAssertionPredicate nodes on a list, starting at 'predicate' and going up in the tree.
-void PhaseIdealLoop::get_opaque_template_assertion_predicate_nodes(ParsePredicateSuccessProj* parse_predicate_proj,
-                                                                   Unique_Node_List& list) {
+IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj,
+                                                                     Node* new_entry, Deoptimization::DeoptReason reason,
+                                                                     const bool slow_loop) {
+
+  IfProjNode* new_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, new_entry, reason, Op_ParsePredicate,
+                                                               slow_loop);
+  assert(new_predicate_proj->is_IfTrue(), "the success projection of a Parse Predicate is a true projection");
+  ParsePredicateNode* parse_predicate = new_predicate_proj->in(0)->as_ParsePredicate();
+  return new_predicate_proj;
+}
+
+// Clones Template Assertion Predicates to both unswitched loops starting at 'old_predicate_proj' by following its
+// control inputs. It also rewires the control edges of data nodes with dependencies in the loop from the old predicates
+// to the new  cloned predicates.
+void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
+                                                                   ParsePredicateSuccessProj* old_parse_predicate_proj,
+                                                                   ParsePredicateNode* true_path_loop_parse_predicate,
+                                                                   ParsePredicateNode* false_path_loop_parse_predicate) {
+  // Push the original Template Assertion Predicates on a list to later process them in reverse order to keep the
+  // original predicate order.
+  Unique_Node_List list;
+  get_template_assertion_predicates(old_parse_predicate_proj, list);
+
+  Node_List to_process;
+  for (int i = list.size() - 1; i >= 0; i--) {
+    IfTrueNode* template_assertion_predicate_success_proj = list.at(i)->as_IfTrue();
+    assert(template_assertion_predicate_success_proj->in(0)->is_If(), "must be If node");
+
+    IfTrueNode* true_path_loop_proj =
+        clone_assertion_predicate_for_unswitched_loops(template_assertion_predicate_success_proj,
+                                                       true_path_loop_parse_predicate);
+    IfTrueNode* false_path_loop_proj =
+        clone_assertion_predicate_for_unswitched_loops(template_assertion_predicate_success_proj,
+                                                       false_path_loop_parse_predicate);
+
+    // Update control dependent data nodes.
+    for (DUIterator j = template_assertion_predicate_success_proj->outs();
+         template_assertion_predicate_success_proj->has_out(j);
+         j++) {
+      Node* true_path_loop_node = template_assertion_predicate_success_proj->out(j);
+      if (loop->is_member(get_loop(ctrl_or_self(true_path_loop_node)))) {
+        assert(true_path_loop_node->in(0) == template_assertion_predicate_success_proj, "only control edge");
+        Node* false_path_loop_node = old_new[true_path_loop_node->_idx];
+        assert(false_path_loop_node->in(0) == template_assertion_predicate_success_proj, "only control edge");
+        _igvn.replace_input_of(true_path_loop_node, 0, true_path_loop_proj);
+        to_process.push(false_path_loop_node);
+        --j;
+      }
+    }
+    // Have to delay updates to the false path loop so uses of predicate are not modified while we iterate on them.
+    while (to_process.size() > 0) {
+      Node* slow_node = to_process.pop();
+      _igvn.replace_input_of(slow_node, 0, false_path_loop_proj);
+    }
+  }
+}
+
+// Put all Template Assertion Predicate projections on a list, starting at 'predicate' and going up in the tree. If 'get_opaque'
+// is set, then the OpaqueTemplateAssertionPredicateNode nodes of the Assertion Predicates are put on the list instead
+// of the projections.
+void PhaseIdealLoop::get_template_assertion_predicates(ParsePredicateSuccessProj* parse_predicate_proj, Unique_Node_List& list,
+                                                       const bool get_opaque) {
   Deoptimization::DeoptReason deopt_reason = parse_predicate_proj->in(0)->as_ParsePredicate()->deopt_reason();
   PredicateBlockIterator predicate_iterator(parse_predicate_proj, deopt_reason);
-  OpaqueTemplateAssertionPredicateCollector opaque_template_assertion_predicate_collector(list);
-  predicate_iterator.for_each(opaque_template_assertion_predicate_collector);
+  TemplateAssertionPredicateCollector template_assertion_predicate_collector(list, get_opaque);
+  predicate_iterator.for_each(template_assertion_predicate_collector);
+}
+
+// Clone an Assertion Predicate for an unswitched loop. OpaqueLoopInit and OpaqueLoopStride nodes are cloned and uncommon
+// traps are kept for the predicate (a Halt node is used later when creating pre/main/post loops and copying this cloned
+// predicate again).
+IfTrueNode*
+PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(IfTrueNode* template_assertion_predicate_success_proj,
+                                                               ParsePredicateNode* unswitched_loop_parse_predicate) {
+  TemplateAssertionPredicate template_assertion_predicate(template_assertion_predicate_success_proj);
+  IfTrueNode* template_success_proj = template_assertion_predicate.clone(unswitched_loop_parse_predicate->in(0), this);
+  _igvn.replace_input_of(unswitched_loop_parse_predicate, 0, template_success_proj);
+  set_idom(unswitched_loop_parse_predicate, template_success_proj, dom_depth(template_success_proj));
+  return template_success_proj;
+}
+
+// Clone the old Parse Predicates and Assertion Predicates before the unswitch If to the unswitched loops after the
+// unswitch If.
+void PhaseIdealLoop::clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new,
+                                                                             IfProjNode*& true_path_loop_entry,
+                                                                             IfProjNode*& false_path_loop_entry) {
+  LoopNode* head = loop->_head->as_Loop();
+  Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl);
+
+  const Predicates predicates(entry);
+  clone_loop_predication_predicates_to_unswitched_loop(loop, old_new, predicates.loop_predicate_block(),
+                                                       Deoptimization::Reason_predicate, true_path_loop_entry, false_path_loop_entry);
+  clone_loop_predication_predicates_to_unswitched_loop(loop, old_new, predicates.profiled_loop_predicate_block(),
+                                                       Deoptimization::Reason_profile_predicate, true_path_loop_entry, false_path_loop_entry);
+
+  const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block();
+  if (loop_limit_check_predicate_block->has_parse_predicate() && !head->is_CountedLoop()) {
+    // Don't clone the Loop Limit Check Parse Predicate if we already have a counted loop (a Loop Limit Check Predicate
+    // is only created when converting a LoopNode to a CountedLoopNode).
+    clone_parse_predicate_to_unswitched_loops(loop_limit_check_predicate_block, Deoptimization::Reason_loop_limit_check,
+                                              true_path_loop_entry, false_path_loop_entry);
+  }
+}
+
+// Clone the Parse Predicate and Template Assertion Predicates of a Loop Predication related Predicate Block.
+void PhaseIdealLoop::clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
+                                                                          const PredicateBlock* predicate_block,
+                                                                          Deoptimization::DeoptReason reason,
+                                                                          IfProjNode*& true_path_loop_entry,
+                                                                          IfProjNode*& false_path_loop_entry) {
+  if (predicate_block->has_parse_predicate()) {
+    // We currently only clone Assertion Predicates if there are Parse Predicates. This is not entirely correct and will
+    // be changed with the complete fix for Assertion Predicates.
+    clone_parse_predicate_to_unswitched_loops(predicate_block, reason, true_path_loop_entry, false_path_loop_entry);
+    assert(true_path_loop_entry->in(0)->is_ParsePredicate() && false_path_loop_entry->in(0)->is_ParsePredicate(),
+           "must be success projections of the cloned Parse Predicates");
+    clone_assertion_predicates_to_unswitched_loop(loop, old_new, predicate_block->parse_predicate_success_proj(),
+                                                  true_path_loop_entry->in(0)->as_ParsePredicate(),
+                                                  false_path_loop_entry->in(0)->as_ParsePredicate());
+  }
 }
 
+void PhaseIdealLoop::clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block,
+                                                               Deoptimization::DeoptReason reason,
+                                                               IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred) {
+  assert(predicate_block->has_parse_predicate(), "must have parse predicate");
+  ParsePredicateSuccessProj* parse_predicate_proj = predicate_block->parse_predicate_success_proj();
+  iffast_pred = clone_parse_predicate_to_unswitched_loop(parse_predicate_proj, iffast_pred, reason, false);
+  check_cloned_parse_predicate_for_unswitching(iffast_pred, true);
+
+  ifslow_pred = clone_parse_predicate_to_unswitched_loop(parse_predicate_proj, ifslow_pred, reason, true);
+  check_cloned_parse_predicate_for_unswitching(ifslow_pred, false);
+}
+
+#ifndef PRODUCT
+void PhaseIdealLoop::check_cloned_parse_predicate_for_unswitching(const Node* new_entry, const bool is_fast_loop) {
+  assert(new_entry != nullptr, "IfTrue or IfFalse after clone predicate");
+  if (TraceLoopPredicate) {
+    tty->print("Parse Predicate cloned to %s loop: ", is_fast_loop ? "fast" : "slow");
+    new_entry->in(0)->dump();
+  }
+}
+#endif
+
 //------------------------------Invariance-----------------------------------
 // Helper class for loop_predication_impl to compute invariance on the fly and
 // clone invariants.

src/hotspot/share/opto/loopUnswitch.cpp

@@ -236,47 +236,18 @@ class OriginalLoop : public StackObj {
         _phase(loop->_phase) {}
   NONCOPYABLE(OriginalLoop);
 
-  // Unswitch the original loop on the invariant loop selector by creating a true-path-loop and a false-path-loop.
-  // Remove the unswitch candidate If from both unswitched loop versions which are now covered by the loop selector If.
-  void unswitch(const UnswitchedLoopSelector& unswitched_loop_selector) {
-    const uint first_false_path_loop_node_index = _phase->C->unique();
-    clone_loop(unswitched_loop_selector);
-
-    move_parse_and_template_assertion_predicates_to_unswitched_loops(first_false_path_loop_node_index);
-    DEBUG_ONLY(verify_unswitched_loop_versions(_loop->_head->as_Loop(), unswitched_loop_selector);)
-
-    _phase->recompute_dom_depth();
-    remove_unswitch_candidate_from_loops(unswitched_loop_selector);
-  }
-
  private:
-  void clone_loop(const UnswitchedLoopSelector& unswitched_loop_selector) {
-    _phase->clone_loop(_loop, _old_new, _phase->dom_depth(_loop_head),
-                       PhaseIdealLoop::CloneIncludesStripMined, unswitched_loop_selector.selector());
-    fix_loop_entries(unswitched_loop_selector);
-  }
-
-  void fix_loop_entries(const UnswitchedLoopSelector& unswitched_loop_selector) {
-    _phase->replace_loop_entry(_loop_head, unswitched_loop_selector.true_path_loop_proj());
+  void fix_loop_entries(IfProjNode* true_path_loop_entry, IfProjNode* false_path_loop_entry) {
+    _phase->replace_loop_entry(_loop_head, true_path_loop_entry);
     LoopNode* false_path_loop_strip_mined_head = old_to_new(_loop_head)->as_Loop();
-    _phase->replace_loop_entry(false_path_loop_strip_mined_head,
-                               unswitched_loop_selector.false_path_loop_proj());
+    _phase->replace_loop_entry(false_path_loop_strip_mined_head, false_path_loop_entry);
   }
 
-  // Moves the Parse And Template Assertion Predicates to the true and false path loop. They are inserted between the
-  // loop heads and the loop selector If projections. The old Parse and Template Assertion Predicates before
-  // the unswitched loop selector are killed.
-  void move_parse_and_template_assertion_predicates_to_unswitched_loops(const uint first_false_path_loop_node_index) const {
-    const NodeInOriginalLoopBody node_in_true_path_loop_body(first_false_path_loop_node_index, _old_new);
-    const NodeInClonedLoopBody node_in_false_path_loop_body(first_false_path_loop_node_index);
-    CloneUnswitchedLoopPredicatesVisitor
-    clone_unswitched_loop_predicates_visitor(_loop_head, old_to_new(_loop_head)->as_Loop(), node_in_true_path_loop_body,
-                                             node_in_false_path_loop_body, _phase);
-    PredicateIterator predicate_iterator(_loop_head->in(LoopNode::EntryControl));
-    predicate_iterator.for_each(clone_unswitched_loop_predicates_visitor);
+  Node* old_to_new(const Node* old) const {
+    return _old_new[old->_idx];
   }
 
-  #ifdef ASSERT
+#ifdef ASSERT
   void verify_unswitched_loop_versions(LoopNode* true_path_loop_head,
                                        const UnswitchedLoopSelector& unswitched_loop_selector) const {
     verify_unswitched_loop_version(true_path_loop_head, unswitched_loop_selector.true_path_loop_proj());
@@ -292,10 +263,6 @@ class OriginalLoop : public StackObj {
   }
 #endif // ASSERT
 
-  Node* old_to_new(const Node* old) const {
-    return _old_new[old->_idx];
-  }
-
   // Remove the unswitch candidate If nodes in both unswitched loop versions which are now dominated by the loop selector
   // If node. Keep the true-path-path in the true-path-loop and the false-path-path in the false-path-loop by setting
   // the bool input accordingly. The unswitch candidate If nodes are folded in the next IGVN round.
@@ -309,6 +276,28 @@ class OriginalLoop : public StackObj {
     _phase->dominated_by(unswitched_loop_selector.false_path_loop_proj(), unswitching_candidate_clone);
   }
 
+ public:
+  // Unswitch the original loop on the invariant loop selector by creating a true-path-loop and a false-path-loop.
+  // Remove the unswitch candidate If from both unswitched loop versions which are now covered by the loop selector If.
+  void unswitch(const UnswitchedLoopSelector& unswitched_loop_selector) {
+    _phase->clone_loop(_loop, _old_new, _phase->dom_depth(_loop_head),
+                       PhaseIdealLoop::CloneIncludesStripMined, unswitched_loop_selector.selector());
+
+    // At this point, the selector If projections are the corresponding loop entries.
+    // clone_parse_and_assertion_predicates_to_unswitched_loop() could clone additional predicates after the selector
+    // If projections. The loop entries are updated accordingly.
+    IfProjNode* true_path_loop_entry = unswitched_loop_selector.true_path_loop_proj();
+    IfProjNode* false_path_loop_entry = unswitched_loop_selector.false_path_loop_proj();
+    _phase->clone_parse_and_assertion_predicates_to_unswitched_loop(_loop, _old_new,
+                                                                    true_path_loop_entry, false_path_loop_entry);
+
+    fix_loop_entries(true_path_loop_entry, false_path_loop_entry);
+
+    DEBUG_ONLY(verify_unswitched_loop_versions(_loop->_head->as_Loop(), unswitched_loop_selector);)
+
+    _phase->recompute_dom_depth();
+    remove_unswitch_candidate_from_loops(unswitched_loop_selector);
+  }
 };
 
 // See comments below file header for more information about Loop Unswitching.

src/hotspot/share/opto/loopnode.cpp

@@ -4470,15 +4470,15 @@ void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(Ideal
     const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block();
     if (profiled_loop_predicate_block->has_parse_predicate()) {
       ParsePredicateSuccessProj* parse_predicate_proj = profiled_loop_predicate_block->parse_predicate_success_proj();
-      get_opaque_template_assertion_predicate_nodes(parse_predicate_proj, useful_predicates);
+      get_template_assertion_predicates(parse_predicate_proj, useful_predicates, true);
     }
   }
 
   if (UseLoopPredicate) {
     const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block();
     if (loop_predicate_block->has_parse_predicate()) {
       ParsePredicateSuccessProj* parse_predicate_proj = loop_predicate_block->parse_predicate_success_proj();
-      get_opaque_template_assertion_predicate_nodes(parse_predicate_proj, useful_predicates);
+      get_template_assertion_predicates(parse_predicate_proj, useful_predicates, true);
     }
   }
 }

src/hotspot/share/opto/loopnode.hpp

@@ -942,8 +942,7 @@ class PhaseIdealLoop : public PhaseTransform {
   static void ensure_zero_trip_guard_proj(Node* node, bool is_main_loop);
 #endif
  private:
-  static void get_opaque_template_assertion_predicate_nodes(ParsePredicateSuccessProj* parse_predicate_proj,
-                                                            Unique_Node_List& list);
+  static void get_template_assertion_predicates(ParsePredicateSuccessProj* parse_predicate_proj, Unique_Node_List& list, bool get_opaque = false);
   void update_main_loop_assertion_predicates(CountedLoopNode* main_loop_head);
   void initialize_assertion_predicates_for_peeled_loop(CountedLoopNode* peeled_loop_head,
                                                        CountedLoopNode* remaining_loop_head,
@@ -1658,7 +1657,28 @@ class PhaseIdealLoop : public PhaseTransform {
     _nodes_required = UINT_MAX;
   }
 
+ public:
+  // Clone Parse Predicates to slow and fast loop when unswitching a loop
+  void clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new,
+                                                               IfProjNode*& true_path_loop_entry,
+                                                               IfProjNode*& false_path_loop_entry);
  private:
+  void clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
+                                                            const PredicateBlock* predicate_block,
+                                                            Deoptimization::DeoptReason reason,
+                                                            IfProjNode*& true_path_loop_entry,
+                                                            IfProjNode*& false_path_loop_entry);
+  void clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block, Deoptimization::DeoptReason reason,
+                                                 IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred);
+  IfProjNode* clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry,
+                                                       Deoptimization::DeoptReason reason, bool slow_loop);
+  void clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
+                                                     ParsePredicateSuccessProj* old_parse_predicate_proj,
+                                                     ParsePredicateNode* true_path_loop_parse_predicate,
+                                                     ParsePredicateNode* false_path_loop_parse_predicate);
+  IfTrueNode* clone_assertion_predicate_for_unswitched_loops(IfTrueNode* template_assertion_predicate_success_proj,
+                                                             ParsePredicateNode* unswitched_loop_parse_predicate);
+  static void check_cloned_parse_predicate_for_unswitching(const Node* new_entry, bool is_fast_loop) PRODUCT_RETURN;
 
   bool _created_loop_node;
   DEBUG_ONLY(void dump_idoms(Node* early, Node* wrong_lca);)