8327110: Refactor create_bool_from_template_assertion_predicate() to …

…separate class and fix pure cloning cases used for Loop Unswitching and Split If

src/hotspot/share/opto/loopPredicate.cpp

@@ -218,7 +218,7 @@ IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessPro
 void PhaseIdealLoop::set_ctrl_of_nodes_with_same_ctrl(Node* start_node, ProjNode* old_uncommon_proj,
                                                       Node* new_uncommon_proj) {
   ResourceMark rm;
-  Unique_Node_List nodes_with_same_ctrl = find_nodes_with_same_ctrl(start_node, old_uncommon_proj);
+  const Unique_Node_List nodes_with_same_ctrl = find_nodes_with_same_ctrl(start_node, old_uncommon_proj);
   for (uint i = 0; i < nodes_with_same_ctrl.size(); i++) {
     Node* node = nodes_with_same_ctrl[i];
     if (node->in(0) == old_uncommon_proj) {
@@ -365,12 +365,15 @@ void PhaseIdealLoop::get_assertion_predicates(Node* predicate, Unique_Node_List&
 // Clone an Assertion Predicate for an unswitched loop. OpaqueLoopInit and OpaqueLoopStride nodes are cloned and uncommon
 // traps are kept for the predicate (a Halt node is used later when creating pre/main/post loops and copying this cloned
 // predicate again).
-IfProjNode* PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(Node* iff, IfProjNode* predicate,
+IfProjNode* PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(Node* template_assertion_predicate,
+                                                                           IfProjNode* predicate,
                                                                            Deoptimization::DeoptReason reason,
                                                                            ParsePredicateSuccessProj* parse_predicate_proj) {
-  Node* bol = create_bool_from_template_assertion_predicate(iff, nullptr, nullptr, parse_predicate_proj);
-  IfProjNode* if_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, iff->Opcode(), false);
-  _igvn.replace_input_of(if_proj->in(0), 1, bol);
+  TemplateAssertionPredicateExpression template_assertion_predicate_expression(
+      template_assertion_predicate->in(1)->as_Opaque4());
+  Opaque4Node* cloned_opaque4_node = template_assertion_predicate_expression.clone(parse_predicate_proj, this);
+  IfProjNode* if_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, template_assertion_predicate->Opcode(), false);
+  _igvn.replace_input_of(if_proj->in(0), 1, cloned_opaque4_node);
   _igvn.replace_input_of(parse_predicate_proj->in(0), 0, if_proj);
   set_idom(parse_predicate_proj->in(0), if_proj, dom_depth(if_proj));
   return if_proj;

src/hotspot/share/opto/loopnode.hpp

@@ -1659,7 +1659,7 @@ class PhaseIdealLoop : public PhaseTransform {
                                                      Deoptimization::DeoptReason reason, IfProjNode* old_predicate_proj,
                                                      ParsePredicateSuccessProj* fast_loop_parse_predicate_proj,
                                                      ParsePredicateSuccessProj* slow_loop_parse_predicate_proj);
-  IfProjNode* clone_assertion_predicate_for_unswitched_loops(Node* iff, IfProjNode* predicate,
+  IfProjNode* clone_assertion_predicate_for_unswitched_loops(Node* template_assertion_predicate, IfProjNode* predicate,
                                                              Deoptimization::DeoptReason reason,
                                                              ParsePredicateSuccessProj* parse_predicate_proj);
   static void check_cloned_parse_predicate_for_unswitching(const Node* new_entry, bool is_fast_loop) PRODUCT_RETURN;
@@ -1673,6 +1673,12 @@ class PhaseIdealLoop : public PhaseTransform {
   bool created_loop_node()     { return _created_loop_node; }
   void register_new_node(Node* n, Node* blk);
 
+  Node* clone_and_register(Node* n, Node* ctrl) {
+    n = n->clone();
+    register_new_node(n, ctrl);
+    return n;
+  }
+
 #ifdef ASSERT
   void dump_bad_graph(const char* msg, Node* n, Node* early, Node* LCA);
 #endif
@@ -1880,6 +1886,13 @@ class PathFrequency {
   float to(Node* n);
 };
 
+// Interface to transform OpaqueLoopInit and OpaqueLoopStride nodes of a Template Assertion Predicate Expression.
+class TransformStrategyForOpaqueLoopNodes : public StackObj {
+ public:
+  virtual Node* transform_opaque_init(OpaqueLoopInitNode* opaque_init) = 0;
+  virtual Node* transform_opaque_stride(OpaqueLoopStrideNode* opaque_stride) = 0;
+};
+
 // Class to clone a data node graph by taking a list of data nodes. This is done in 2 steps:
 //   1. Clone the data nodes
 //   2. Fix the cloned data inputs pointing to the old nodes to the cloned inputs by using an old->new mapping.
@@ -1906,7 +1919,10 @@ class DataNodeGraph : public StackObj {
  private:
   void clone(Node* node, Node* new_ctrl);
   void clone_data_nodes(Node* new_ctrl);
+  void clone_data_nodes_and_transform_opaque_loop_nodes(TransformStrategyForOpaqueLoopNodes& transform_strategy,
+                                                        Node* new_ctrl);
   void rewire_clones_to_cloned_inputs();
+  void transform_opaque_node(TransformStrategyForOpaqueLoopNodes& transform_strategy, Node* node);
 
  public:
   // Clone the provided data node collection and rewire the clones in such a way to create an identical graph copy.
@@ -1917,5 +1933,17 @@ class DataNodeGraph : public StackObj {
     rewire_clones_to_cloned_inputs();
     return _orig_to_new;
   }
+
+  // Create a copy of the provided data node collection by doing the following:
+  // Clone all non-OpaqueLoop* nodes and rewire them to create an identical subgraph copy. For the OpaqueLoop* nodes,
+  // apply the provided transformation strategy and include the transformed node into the subgraph copy to get a complete
+  // "cloned-and-transformed" graph copy. For all newly cloned nodes (which could also be new OpaqueLoop* nodes), set
+  // `new_ctrl` as ctrl.
+  const OrigToNewHashtable& clone_with_opaque_loop_transform_strategy(TransformStrategyForOpaqueLoopNodes& transform_strategy,
+                                                                      Node* new_ctrl) {
+    clone_data_nodes_and_transform_opaque_loop_nodes(transform_strategy, new_ctrl);
+    rewire_clones_to_cloned_inputs();
+    return _orig_to_new;
+  }
 };
 #endif // SHARE_OPTO_LOOPNODE_HPP

src/hotspot/share/opto/loopopts.cpp

@@ -4523,3 +4523,30 @@ void DataNodeGraph::rewire_clones_to_cloned_inputs() {
     }
   });
 }
+
+// Clone all non-OpaqueLoop* nodes and apply the provided transformation strategy for OpaqueLoop* nodes.
+// Set 'new_ctrl' as ctrl for all cloned non-OpaqueLoop* nodes.
+void DataNodeGraph::clone_data_nodes_and_transform_opaque_loop_nodes(TransformStrategyForOpaqueLoopNodes& transform_strategy,
+                                                                     Node* new_ctrl) {
+  for (uint i = 0; i < _data_nodes.size(); i++) {
+    Node* data_node = _data_nodes[i];
+    if (data_node->is_Opaque1()) {
+      transform_opaque_node(transform_strategy, data_node);
+    } else {
+      clone(data_node, new_ctrl);
+    }
+  }
+}
+
+void DataNodeGraph::transform_opaque_node(TransformStrategyForOpaqueLoopNodes& transform_strategy, Node* node) {
+  const uint next_idx = _phase->C->unique();
+  Node* transformed_node;
+  if (node->is_OpaqueLoopInit()) {
+    transformed_node = transform_strategy.transform_opaque_init(node->as_OpaqueLoopInit());
+  } else {
+    assert(node->is_OpaqueLoopStride(), "must be OpaqueLoopStrideNode");
+    transformed_node = transform_strategy.transform_opaque_stride(node->as_OpaqueLoopStride());
+  }
+  // Add an orig->new mapping to correctly update the inputs of the copied graph in rewire_clones_to_cloned_inputs().
+  _orig_to_new.put(node, transformed_node);
+}

src/hotspot/share/opto/node.hpp

@@ -134,6 +134,9 @@ class NegNode;
 class NegVNode;
 class NeverBranchNode;
 class Opaque1Node;
+class OpaqueLoopInitNode;
+class OpaqueLoopStrideNode;
+class Opaque4Node;
 class OuterStripMinedLoopNode;
 class OuterStripMinedLoopEndNode;
 class Node;
@@ -786,9 +789,12 @@ class Node {
     DEFINE_CLASS_ID(ClearArray, Node, 14)
     DEFINE_CLASS_ID(Halt,     Node, 15)
     DEFINE_CLASS_ID(Opaque1,  Node, 16)
-    DEFINE_CLASS_ID(Move,     Node, 17)
-    DEFINE_CLASS_ID(LShift,   Node, 18)
-    DEFINE_CLASS_ID(Neg,      Node, 19)
+      DEFINE_CLASS_ID(OpaqueLoopInit, Opaque1, 0)
+      DEFINE_CLASS_ID(OpaqueLoopStride, Opaque1, 1)
+    DEFINE_CLASS_ID(Opaque4,  Node, 17)
+    DEFINE_CLASS_ID(Move,     Node, 18)
+    DEFINE_CLASS_ID(LShift,   Node, 19)
+    DEFINE_CLASS_ID(Neg,      Node, 20)
 
     _max_classes  = ClassMask_Neg
   };
@@ -955,6 +961,9 @@ class Node {
   DEFINE_CLASS_QUERY(NegV)
   DEFINE_CLASS_QUERY(NeverBranch)
   DEFINE_CLASS_QUERY(Opaque1)
+  DEFINE_CLASS_QUERY(Opaque4)
+  DEFINE_CLASS_QUERY(OpaqueLoopInit)
+  DEFINE_CLASS_QUERY(OpaqueLoopStride)
   DEFINE_CLASS_QUERY(OuterStripMinedLoop)
   DEFINE_CLASS_QUERY(OuterStripMinedLoopEnd)
   DEFINE_CLASS_QUERY(Parm)

src/hotspot/share/opto/opaquenode.hpp

@@ -60,13 +60,15 @@ class Opaque1Node : public Node {
 class OpaqueLoopInitNode : public Opaque1Node {
   public:
   OpaqueLoopInitNode(Compile* C, Node *n) : Opaque1Node(C, n) {
+    init_class_id(Class_OpaqueLoopInit);
   }
   virtual int Opcode() const;
 };
 
 class OpaqueLoopStrideNode : public Opaque1Node {
   public:
   OpaqueLoopStrideNode(Compile* C, Node *n) : Opaque1Node(C, n) {
+    init_class_id(Class_OpaqueLoopStride);
   }
   virtual int Opcode() const;
 };
@@ -120,6 +122,7 @@ class Opaque3Node : public Node {
 class Opaque4Node : public Node {
   public:
   Opaque4Node(Compile* C, Node *tst, Node* final_tst) : Node(nullptr, tst, final_tst) {
+    init_class_id(Class_Opaque4);
     init_flags(Flag_is_macro);
     C->add_macro_node(this);
   }

src/hotspot/share/opto/predicates.cpp

@@ -24,6 +24,8 @@
 
 #include "precompiled.hpp"
 #include "opto/callnode.hpp"
+#include "opto/loopnode.hpp"
+#include "opto/node.hpp"
 #include "opto/predicates.hpp"
 
 // Walk over all Initialized Assertion Predicates and return the entry into the first Initialized Assertion Predicate
@@ -147,3 +149,171 @@ Node* PredicateBlock::skip_regular_predicates(Node* regular_predicate_proj, Deop
   }
   return entry;
 }
+
+TemplateAssertionPredicateExpression::TemplateAssertionPredicateExpression(Opaque4Node* opaque4_node)
+    : _opaque4_node(opaque4_node) {}
+
+// This strategy clones the OpaqueLoopInit and OpaqueLoopStride nodes
+class CloneStrategy : public TransformStrategyForOpaqueLoopNodes {
+  PhaseIdealLoop* const _phase;
+  Node* const _new_ctrl;
+
+ public:
+  CloneStrategy(PhaseIdealLoop* phase, Node* new_ctrl)
+      : _phase(phase),
+        _new_ctrl(new_ctrl) {}
+  NONCOPYABLE(CloneStrategy);
+
+  Node* transform_opaque_init(OpaqueLoopInitNode* opaque_init) override {
+    return _phase->clone_and_register(opaque_init, _new_ctrl)->as_OpaqueLoopInit();
+  }
+
+  Node* transform_opaque_stride(OpaqueLoopStrideNode* opaque_stride) override {
+    return _phase->clone_and_register(opaque_stride, _new_ctrl)->as_OpaqueLoopStride();
+  }
+};
+
+// Creates an identical clone of this Template Assertion Predicate Expression (i.e.cloning all nodes from the Opaque4Node
+// to and including the OpaqueLoop* nodes). The cloned nodes are rewired to reflect the same graph structure as found for
+// this Template Assertion Predicate Expression. The cloned nodes get 'new_ctrl' as ctrl. There is no other update done
+// for the cloned nodes. Return the newly cloned Opaque4Node.
+Opaque4Node* TemplateAssertionPredicateExpression::clone(Node* new_ctrl, PhaseIdealLoop* phase) {
+  CloneStrategy clone_init_and_stride_strategy(phase, new_ctrl);
+  return clone(clone_init_and_stride_strategy, new_ctrl, phase);
+}
+
+// Class to collect data nodes from a source to target nodes by following the inputs of the source node recursively.
+// The class takes a node filter to decide which input nodes to follow and a target node predicate to start backtracking
+// from. All nodes found on all path from source->target(s) are stored in a set (without duplicates).
+class DataNodesOnPathToTargets : public StackObj {
+  typedef bool (*NodeCheck)(const Node*);
+
+  NodeCheck _node_filter; // Node filter function to decide if we should process a node or not while searching for targets.
+  NodeCheck _is_target_node; // Function to decide if a node is a target node (i.e. where we should start backtracking).
+  Node_Stack _stack; // Stack that stores entries of the form: [Node* node, int next_unvisited_input_index_of_node].
+  VectorSet _visited; // Ensure that we are not visiting a node twice in the DFS walk which could happen with diamonds.
+  Unique_Node_List _collected_nodes; // The resulting node collection of all nodes on paths from source->target(s).
+
+ public:
+  DataNodesOnPathToTargets(NodeCheck node_filter, NodeCheck is_target_node)
+      : _node_filter(node_filter),
+        _is_target_node(is_target_node),
+        _stack(2) {}
+  NONCOPYABLE(DataNodesOnPathToTargets);
+
+  // Collect all input nodes from 'start_node'->target(s) by applying the node filter to discover new input nodes and
+  // the target node predicate to stop discovering more inputs and start backtracking. The implementation is done
+  // with an iterative DFS walk including a visited set to avoid redundant double visits which could consume a lot of
+  // unnecessary time.
+  const Unique_Node_List& collect(Node* start_node) {
+    assert(_collected_nodes.size() == 0, "should not call this method twice in a row");
+    assert(!_is_target_node(start_node), "no trivial paths where start node is also a target");
+
+    push_unvisited_node(start_node);
+    while (_stack.is_nonempty()) {
+      Node* current = _stack.node();
+      _visited.set(current->_idx);
+      if (_is_target_node(current)) {
+        // Target node? Do not visit its inputs and begin backtracking.
+        _collected_nodes.push(current);
+        pop_target_node_and_collect_predecessor();
+      } else if (!push_next_unvisited_input()) {
+        // All inputs visited. Continue backtracking.
+        pop_node_and_maybe_collect_predecessor();
+      }
+    }
+    return _collected_nodes;
+  }
+
+ private:
+  // The predecessor (just below the target node (currently on top) on the stack) is also on the path from
+  // start->target. Collect it.
+  void pop_target_node_and_collect_predecessor() {
+    _stack.pop();
+    assert(_stack.is_nonempty(), "target nodes should not be start nodes");
+    _collected_nodes.push(_stack.node());
+  }
+
+  // Push the next unvisited input node of the current node on top of the stack by using its stored associated input index:
+  //
+  //      I1  I2  I3        Stack:
+  //       \  |  /          [current, 2] // Index 2 means that I1 was visited before. The next unvisited input is I2.
+  //   Y   current                       // Visit I2 by pushing a new entry [I2, 1] and update the index [current, 2] ->
+  //    \  /                             // [current, 3] to visit I3 once 'current' is on top of stack again.
+  //     X                  [X, 3]       // Index 3 points past input array which means that there are no more inputs to
+  //                                     // visit. Once X is on top of stack again, we are done with 'X' and pop it.
+  //
+  // If an input was already collected before (i.e. part of start->target), then the current node is part of some kind
+  // of diamond with it:
+  //
+  //        C3       X
+  //       /  \     /
+  //     C2   current
+  //      \  /
+  //       C1
+  //
+  // Cx means collected. Since C3 is already collected (and thus already visited), we add `current` to the collected list.
+  // We continue the DFS with X which could potentially also be on a start->target path but that is not known yet.
+  // Collect the current node as well since it must also be on a path from start->target.
+  //
+  // This method returns true if there is an unvisited input and return false otherwise if all inputs have been visited.
+  bool push_next_unvisited_input() {
+    Node* current = _stack.node();
+    const uint next_unvisited_input_index = _stack.index();
+    for (uint i = next_unvisited_input_index; i < current->req(); i++) {
+      Node* input = current->in(i);
+      if (_node_filter(input)) {
+        if (!_visited.test(input->_idx)) { // Avoid double visits which could take a long time to process.
+          // About to visit i, at next visit: visit i+1
+          push_input_and_update_current_index(input, i + 1);
+          return true;
+        } else if (_collected_nodes.member(input)) {
+          // Diamond case, see description above.
+          // Input node part of start->target? Then current node (i.e. a predecessor of input) is also on path. Collect it.
+          _collected_nodes.push(current);
+        }
+      }
+    }
+    return false;
+  }
+
+  // Update the index of the current node on top of the stack with the next unvisited input index and push the new
+  // input which is visited next in the DFS order.
+  void push_input_and_update_current_index(Node* input, uint next_unvisited_input_index) {
+    _stack.set_index(next_unvisited_input_index);
+    push_unvisited_node(input);
+  }
+
+  // Push the next unvisited node in the DFS order with index 1 since this node needs to visit all its inputs.
+  void push_unvisited_node(Node* next_to_visit) {
+    _stack.push(next_to_visit, 1);
+  }
+
+  // If the current node on top of the stack is on a path from start->target(s), then also collect the predecessor node
+  // before popping the current node.
+  void pop_node_and_maybe_collect_predecessor() {
+    Node* current_node = _stack.node();
+    _stack.pop();
+    if (_stack.is_nonempty() && _collected_nodes.member(current_node)) {
+      // Current node was part of start->target? Then predecessor (i.e. newly on top of stack) is also on path. Collect it.
+      _collected_nodes.push(_stack.node());
+    }
+  }
+};
+
+// Clones this Template Assertion Predicate Expression and applies the given strategy to transform the OpaqueLoop* nodes.
+Opaque4Node* TemplateAssertionPredicateExpression::clone(TransformStrategyForOpaqueLoopNodes& transform_strategy,
+                                                         Node* new_ctrl, PhaseIdealLoop* phase) {
+  ResourceMark rm;
+  auto is_opaque_loop_node = [](const Node* node) {
+    return node->is_Opaque1();
+  };
+  DataNodesOnPathToTargets data_nodes_on_path_to_targets(TemplateAssertionPredicateExpression::maybe_contains,
+                                                         is_opaque_loop_node);
+  const Unique_Node_List& collected_nodes = data_nodes_on_path_to_targets.collect(_opaque4_node);
+  DataNodeGraph data_node_graph(collected_nodes, phase);
+  const OrigToNewHashtable& orig_to_new = data_node_graph.clone_with_opaque_loop_transform_strategy(transform_strategy, new_ctrl);
+  assert(orig_to_new.contains(_opaque4_node), "must exist");
+  Node* opaque4_clone = *orig_to_new.get(_opaque4_node);
+  return opaque4_clone->as_Opaque4();
+}