From 634513c5e1d25425a68c0d2d370e326d7511d633 Mon Sep 17 00:00:00 2001 From: Christian Hagedorn Date: Fri, 24 Nov 2023 17:00:50 +0100 Subject: [PATCH] Refactor Template Assertion Predicate Bool creation and Split If --- src/hotspot/share/opto/loopPredicate.cpp | 8 +- src/hotspot/share/opto/loopTransform.cpp | 150 ++--------- src/hotspot/share/opto/loopnode.hpp | 19 +- src/hotspot/share/opto/node.hpp | 6 + src/hotspot/share/opto/opaquenode.hpp | 2 + src/hotspot/share/opto/predicates.cpp | 316 +++++++++++++++++++++++ src/hotspot/share/opto/predicates.hpp | 41 +++ src/hotspot/share/opto/split_if.cpp | 114 ++++++-- 8 files changed, 499 insertions(+), 157 deletions(-) diff --git a/src/hotspot/share/opto/loopPredicate.cpp b/src/hotspot/share/opto/loopPredicate.cpp index d569bfcfab568..c2f4e0608ed5e 100644 --- a/src/hotspot/share/opto/loopPredicate.cpp +++ b/src/hotspot/share/opto/loopPredicate.cpp @@ -396,9 +396,13 @@ void PhaseIdealLoop::get_assertion_predicates(Node* predicate, Unique_Node_List& IfProjNode* PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(Node* iff, IfProjNode* predicate, Deoptimization::DeoptReason reason, ParsePredicateSuccessProj* parse_predicate_proj) { - Node* bol = create_bool_from_template_assertion_predicate(iff, nullptr, nullptr, parse_predicate_proj); + Node* opaque_node = iff->in(1); + TemplateAssertionPredicateBool template_assertion_predicate_bool(opaque_node->in(1)); + BoolNode* bol = template_assertion_predicate_bool.clone(parse_predicate_proj, this); + opaque_node = clone_and_register(opaque_node, parse_predicate_proj); + _igvn.replace_input_of(opaque_node, 1, bol); IfProjNode* if_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, iff->Opcode(), false); - _igvn.replace_input_of(if_proj->in(0), 1, bol); + _igvn.replace_input_of(if_proj->in(0), 1, opaque_node); _igvn.replace_input_of(parse_predicate_proj->in(0), 0, if_proj); set_idom(parse_predicate_proj->in(0), if_proj, dom_depth(if_proj)); return if_proj; diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp index 22386b144c223..95023337ae303 100644 --- a/src/hotspot/share/opto/loopTransform.cpp +++ b/src/hotspot/share/opto/loopTransform.cpp @@ -1385,40 +1385,6 @@ void PhaseIdealLoop::copy_assertion_predicates_to_main_loop_helper(const Predica } } -// Is 'n' a node that can be found on the input chain of a Template Assertion Predicate bool (i.e. between a Template -// Assertion Predicate If node and the OpaqueLoop* nodes)? -static bool is_part_of_template_assertion_predicate_bool(Node* n) { - int op = n->Opcode(); - return (n->is_Bool() || - n->is_Cmp() || - op == Op_AndL || - op == Op_OrL || - op == Op_RShiftL || - op == Op_LShiftL || - op == Op_LShiftI || - op == Op_AddL || - op == Op_AddI || - op == Op_MulL || - op == Op_MulI || - op == Op_SubL || - op == Op_SubI || - op == Op_ConvI2L || - op == Op_CastII); -} - -bool PhaseIdealLoop::subgraph_has_opaque(Node* n) { - if (n->Opcode() == Op_OpaqueLoopInit || n->Opcode() == Op_OpaqueLoopStride) { - return true; - } - if (!is_part_of_template_assertion_predicate_bool(n)) { - return false; - } - uint init; - uint stride; - count_opaque_loop_nodes(n, init, stride); - return init != 0 || stride != 0; -} - bool PhaseIdealLoop::assertion_predicate_has_loop_opaque_node(IfNode* iff) { uint init; uint stride; @@ -1462,101 +1428,21 @@ void PhaseIdealLoop::count_opaque_loop_nodes(Node* n, uint& init, uint& stride) wq.push(n); for (uint i = 0; i < wq.size(); i++) { Node* n = wq.at(i); - if (is_part_of_template_assertion_predicate_bool(n)) { - for (uint j = 1; j < n->req(); j++) { - Node* m = n->in(j); - if (m != nullptr) { - wq.push(m); - } - } - continue; - } - if (n->Opcode() == Op_OpaqueLoopInit) { - init++; - } else if (n->Opcode() == Op_OpaqueLoopStride) { - stride++; - } - } -} - -// Create a new Bool node from the provided Template Assertion Predicate. -// Unswitched loop: new_init and new_stride are both null. Clone OpaqueLoopInit and OpaqueLoopStride. -// Otherwise: Replace found OpaqueLoop* nodes with new_init and new_stride, respectively. -Node* PhaseIdealLoop::create_bool_from_template_assertion_predicate(Node* template_assertion_predicate, Node* new_init, - Node* new_stride, Node* control) { - Node_Stack to_clone(2); - Node* opaque4 = template_assertion_predicate->in(1); - assert(opaque4->Opcode() == Op_Opaque4, "must be Opaque4"); - to_clone.push(opaque4, 1); - uint current = C->unique(); - Node* result = nullptr; - bool is_unswitched_loop = new_init == nullptr && new_stride == nullptr; - assert(new_init != nullptr || is_unswitched_loop, "new_init must be set when new_stride is non-null"); - // Look for the opaque node to replace with the new value - // and clone everything in between. We keep the Opaque4 node - // so the duplicated predicates are eliminated once loop - // opts are over: they are here only to keep the IR graph - // consistent. - do { - Node* n = to_clone.node(); - uint i = to_clone.index(); - Node* m = n->in(i); - if (is_part_of_template_assertion_predicate_bool(m)) { - to_clone.push(m, 1); - continue; - } - if (m->is_Opaque1()) { - if (n->_idx < current) { - n = n->clone(); - register_new_node(n, control); - } - int op = m->Opcode(); - if (op == Op_OpaqueLoopInit) { - if (is_unswitched_loop && m->_idx < current && new_init == nullptr) { - new_init = m->clone(); - register_new_node(new_init, control); - } - n->set_req(i, new_init); + if (AssertionPredicateBoolOpcodes::is_valid(n)) { + if (n->Opcode() == Op_OpaqueLoopInit) { + init++; + } else if (n->Opcode() == Op_OpaqueLoopStride) { + stride++; } else { - assert(op == Op_OpaqueLoopStride, "unexpected opaque node"); - if (is_unswitched_loop && m->_idx < current && new_stride == nullptr) { - new_stride = m->clone(); - register_new_node(new_stride, control); - } - if (new_stride != nullptr) { - n->set_req(i, new_stride); - } - } - to_clone.set_node(n); - } - while (true) { - Node* cur = to_clone.node(); - uint j = to_clone.index(); - if (j+1 < cur->req()) { - to_clone.set_index(j+1); - break; - } - to_clone.pop(); - if (to_clone.size() == 0) { - result = cur; - break; - } - Node* next = to_clone.node(); - j = to_clone.index(); - if (next->in(j) != cur) { - assert(cur->_idx >= current || next->in(j)->Opcode() == Op_Opaque1, "new node or Opaque1 being replaced"); - if (next->_idx < current) { - next = next->clone(); - register_new_node(next, control); - to_clone.set_node(next); + for (uint j = 1; j < n->req(); j++) { + Node* m = n->in(j); + if (m != nullptr) { + wq.push(m); + } } - next->set_req(j, cur); } } - } while (result == nullptr); - assert(result->_idx >= current, "new node expected"); - assert(!is_unswitched_loop || new_init != nullptr, "new_init must always be found and cloned"); - return result; + } } // Clone an Assertion Predicate for the main loop. new_init and new_stride are set as new inputs. Since the predicates @@ -1564,11 +1450,21 @@ Node* PhaseIdealLoop::create_bool_from_template_assertion_predicate(Node* templa Node* PhaseIdealLoop::clone_assertion_predicate_and_initialize(Node* iff, Node* new_init, Node* new_stride, Node* predicate, Node* uncommon_proj, Node* control, IdealLoopTree* outer_loop, Node* input_proj) { - Node* result = create_bool_from_template_assertion_predicate(iff, new_init, new_stride, control); + Node* opaque_node = iff->in(1); + TemplateAssertionPredicateBool template_assertion_predicate_bool(opaque_node->in(1)); + BoolNode* new_bool; + if (new_stride == nullptr) { + new_bool = template_assertion_predicate_bool.clone_and_replace_init(control, new_init, this); + } else { + new_bool = template_assertion_predicate_bool.clone_and_replace_opaque_loop_nodes(control, new_init, new_stride, this); + } + opaque_node = clone_and_register(opaque_node, control); + _igvn.replace_input_of(opaque_node, 1, new_bool); + Node* proj = predicate->clone(); Node* other_proj = uncommon_proj->clone(); Node* new_iff = iff->clone(); - new_iff->set_req(1, result); + new_iff->set_req(1, opaque_node); proj->set_req(0, new_iff); other_proj->set_req(0, new_iff); Node* frame = new ParmNode(C->start(), TypeFunc::FramePtr); diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 4a34d7e49baca..d095b1f6aadaa 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -946,9 +946,6 @@ class PhaseIdealLoop : public PhaseTransform { Node* uncommon_proj, Node* control, IdealLoopTree* outer_loop, Node* input_proj); static void count_opaque_loop_nodes(Node* n, uint& init, uint& stride); - static bool subgraph_has_opaque(Node* n); - Node* create_bool_from_template_assertion_predicate(Node* template_assertion_predicate, Node* new_init, Node* new_stride, - Node* control); static bool assertion_predicate_has_loop_opaque_node(IfNode* iff); static void get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque = false); void update_main_loop_assertion_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, int stride_con); @@ -1647,6 +1644,12 @@ class PhaseIdealLoop : public PhaseTransform { bool created_loop_node() { return _created_loop_node; } void register_new_node(Node* n, Node* blk); + Node* clone_and_register(Node* n, Node* ctrl) { + n = n->clone(); + register_new_node(n, ctrl); + return n; + } + #ifdef ASSERT void dump_bad_graph(const char* msg, Node* n, Node* early, Node* LCA); #endif @@ -1723,14 +1726,12 @@ class PhaseIdealLoop : public PhaseTransform { void finish_clone_loop(Node_List* split_if_set, Node_List* split_bool_set, Node_List* split_cex_set); - bool clone_cmp_down(Node* n, const Node* blk1, const Node* blk2); - - void clone_loadklass_nodes_at_cmp_index(const Node* n, Node* cmp, int i); - - bool clone_cmp_loadklass_down(Node* n, const Node* blk1, const Node* blk2); - bool at_relevant_ctrl(Node* n, const Node* blk1, const Node* blk2); + bool clone_cmp_loadklass_down(Node* n, const Node* blk1, const Node* blk2); + void clone_loadklass_nodes_at_cmp_index(const Node* n, Node* cmp, int i); + bool clone_cmp_down(Node* n, const Node* blk1, const Node* blk2); + void clone_template_assertion_predicate_bool_down_if_related(Node* n); Node* similar_subtype_check(const Node* x, Node* r_in); diff --git a/src/hotspot/share/opto/node.hpp b/src/hotspot/share/opto/node.hpp index 0541409a31269..6dc4059a66f40 100644 --- a/src/hotspot/share/opto/node.hpp +++ b/src/hotspot/share/opto/node.hpp @@ -134,6 +134,8 @@ class NegNode; class NegVNode; class NeverBranchNode; class Opaque1Node; +class OpaqueLoopInitNode; +class OpaqueLoopStrideNode; class OuterStripMinedLoopNode; class OuterStripMinedLoopEndNode; class Node; @@ -783,6 +785,8 @@ class Node { DEFINE_CLASS_ID(ClearArray, Node, 14) DEFINE_CLASS_ID(Halt, Node, 15) DEFINE_CLASS_ID(Opaque1, Node, 16) + DEFINE_CLASS_ID(OpaqueLoopInit, Opaque1, 0) + DEFINE_CLASS_ID(OpaqueLoopStride, Opaque1, 1) DEFINE_CLASS_ID(Move, Node, 17) DEFINE_CLASS_ID(LShift, Node, 18) DEFINE_CLASS_ID(Neg, Node, 19) @@ -952,6 +956,8 @@ class Node { DEFINE_CLASS_QUERY(NegV) DEFINE_CLASS_QUERY(NeverBranch) DEFINE_CLASS_QUERY(Opaque1) + DEFINE_CLASS_QUERY(OpaqueLoopInit) + DEFINE_CLASS_QUERY(OpaqueLoopStride) DEFINE_CLASS_QUERY(OuterStripMinedLoop) DEFINE_CLASS_QUERY(OuterStripMinedLoopEnd) DEFINE_CLASS_QUERY(Parm) diff --git a/src/hotspot/share/opto/opaquenode.hpp b/src/hotspot/share/opto/opaquenode.hpp index 75b67fdea09ce..7e968b78d31cf 100644 --- a/src/hotspot/share/opto/opaquenode.hpp +++ b/src/hotspot/share/opto/opaquenode.hpp @@ -60,6 +60,7 @@ class Opaque1Node : public Node { class OpaqueLoopInitNode : public Opaque1Node { public: OpaqueLoopInitNode(Compile* C, Node *n) : Opaque1Node(C, n) { + init_class_id(Class_OpaqueLoopInit); } virtual int Opcode() const; }; @@ -67,6 +68,7 @@ class OpaqueLoopInitNode : public Opaque1Node { class OpaqueLoopStrideNode : public Opaque1Node { public: OpaqueLoopStrideNode(Compile* C, Node *n) : Opaque1Node(C, n) { + init_class_id(Class_OpaqueLoopStride); } virtual int Opcode() const; }; diff --git a/src/hotspot/share/opto/predicates.cpp b/src/hotspot/share/opto/predicates.cpp index c38ff4bcf024b..01470a5bf4eb7 100644 --- a/src/hotspot/share/opto/predicates.cpp +++ b/src/hotspot/share/opto/predicates.cpp @@ -25,6 +25,7 @@ #include "precompiled.hpp" #include "opto/callnode.hpp" #include "opto/predicates.hpp" +#include "opto/subnode.hpp" // Walk over all Initialized Assertion Predicates and return the entry into the first Initialized Assertion Predicate // (i.e. not belonging to an Initialized Assertion Predicate anymore) @@ -147,3 +148,318 @@ Node* PredicateBlock::skip_regular_predicates(Node* regular_predicate_proj, Deop } return entry; } + +TemplateAssertionPredicateBool::TemplateAssertionPredicateBool(Node* source_bool) : _source_bool(source_bool->as_Bool()) { +#ifdef ASSERT + // During IGVN, we could have multiple outputs of the _source_bool, for example, when the backedge of the loop of + // this Template Assertion Predicate is about to die and the CastII on the last value bool already folded to a + // constant (i.e. no OpaqueLoop* nodes anymore). Then IGVN could already have commoned up the bool with the bool of + // one of the Hoisted Check Predicates. Just check that the Template Assertion Predicate is one of the outputs. + bool has_template_output = false; + for (DUIterator_Fast imax, i = source_bool->fast_outs(imax); i < imax; i++) { + Node* out = source_bool->fast_out(i); + if (out->Opcode() == Op_Opaque4) { + has_template_output = true; + break; + } + } + assert(has_template_output, "must find Template Assertion Predicate as output"); +#endif // ASSERT +} + +// Stack used when performing DFS on Template Assertion Predicate bools. The DFS traversal visits non-CFG inputs of a +// node in increasing node index order (i.e. first visiting the input node at index 1). Each time a new node is visited, +// it is inserted on top of the stack. Each node n in the stack maintains a node input index i, denoted as [n, i]: +// +// i = 0, [n, 0]: +// n is currently being visited in the DFS traversal and was newly added to the stack +// +// i > 0, [n, i]: +// Let node s be the next node being visited after node n in the DFS traversal. The following holds: +// n->in(i) = s +class DFSNodeStack : public StackObj { + Node_Stack _stack; + static const uint _no_inputs_visited_yet = 0; + + public: + explicit DFSNodeStack(BoolNode* template_bool) + : _stack(2) { + _stack.push(template_bool, _no_inputs_visited_yet); + } + + // Push the next unvisited input of the current node on top of the stack and return true. The visiting order of node + // inputs is in increasing node input index order. If there are no unvisited inputs left, do nothing and return false. + bool push_next_unvisited_input() { + Node* node_on_top = top(); + increment_top_node_input_index(); + const uint next_unvisited_input = _stack.index(); + for (uint index = next_unvisited_input; index < node_on_top->req(); index++) { + Node* input = node_on_top->in(index); + if (AssertionPredicateBoolOpcodes::is_valid(input)) { + // We only care about related inputs. + _stack.set_index(index); + _stack.push(input, _no_inputs_visited_yet); + return true; + } + } + return false; + } + + Node* top() const { + return _stack.node(); + } + + uint node_index_to_previously_visited_parent() const { + return _stack.index(); + } + + bool is_not_empty() const { + return _stack.size() > 0; + } + + void pop() { + _stack.pop(); + } + + void increment_top_node_input_index() { + _stack.set_index(_stack.index() + 1); + } + + void replace_top_with(Node* node) { + _stack.set_node(node); + } +}; + +// Interface to transform OpaqueLoop* nodes of Template Assertion Predicate bools. The transformations must return a +// new or different existing node. +class TransformOpaqueLoopNodes : public StackObj { + public: + virtual Node* transform_opaque_init(OpaqueLoopInitNode* opaque_init) = 0; + virtual Node* transform_opaque_stride(OpaqueLoopStrideNode* opaque_stride) = 0; +}; + +// Class to clone an Assertion Predicate bool. The BoolNode and all the nodes up to but excluding the OpaqueLoop* nodes +// are cloned. The OpaqueLoop* nodes are transformed by the provided strategy (e.g. cloned or replaced). +class CloneAssertionPredicateBool : public StackObj { + DFSNodeStack _stack; + PhaseIdealLoop* _phase; + uint _index_before_cloning; + Node* _ctrl_for_clones; + DEBUG_ONLY(bool _found_init;) + + // Transform opaque_loop_node with the provided strategy. The transformation must return a new or an existing node + // other than the OpaqueLoop* node itself. + Node* transform_opaque_loop_node(const Node* opaque_loop_node, TransformOpaqueLoopNodes* transform_opaque_nodes) { + Node* transformed_node; + if (opaque_loop_node->is_OpaqueLoopInit()) { + DEBUG_ONLY(_found_init = true;) + transformed_node = transform_opaque_nodes->transform_opaque_init(opaque_loop_node->as_OpaqueLoopInit()); + } else { + transformed_node = transform_opaque_nodes->transform_opaque_stride(opaque_loop_node->as_OpaqueLoopStride()); + } + assert(transformed_node != opaque_loop_node, "OpaqueLoop*Node must have been transformed"); + return transformed_node; + } + + void pop_opaque_loop_node(Node* transformed_opaque_loop_node) { + _stack.pop(); + assert(_stack.is_not_empty(), "must not be empty when popping an OpaqueLoop*Node"); + if (must_clone_top_node(transformed_opaque_loop_node)) { + clone_top_node(transformed_opaque_loop_node); + } else { + set_req_of_clone_to_parent(transformed_opaque_loop_node); + } + } + + // Must only clone top node (i.e. child of 'previously_visited_parent') if not yet cloned (could visit this node a + // second time in DFS when coming back) and parent was already cloned or transformed (i.e. child node is on the chain + // to an OpaqueLoop* node and therefore needs to be cloned). + bool must_clone_top_node(Node* previously_visited_parent) { + Node* child_of_last_visited_parent = _stack.top(); + const uint node_index_to_previously_visited_parent = _stack.node_index_to_previously_visited_parent(); + return child_of_last_visited_parent->_idx < _index_before_cloning && + child_of_last_visited_parent->in(node_index_to_previously_visited_parent) != previously_visited_parent; + } + + // Clone the node currently on top of the stack (i.e. a descendant of an OpaqueLoop* node) and set + // 'previously_visited_parent' (also a clone) as new input at the input index stored with the top node. Replace the + // current node on top with the newly cloned version. + void clone_top_node(Node* previously_visited_parent) { + Node* child_of_last_visited_parent = _stack.top(); + const uint node_index_to_previously_visited_parent = _stack.node_index_to_previously_visited_parent(); + Node* clone = _phase->clone_and_register(child_of_last_visited_parent, _ctrl_for_clones); + clone->set_req(node_index_to_previously_visited_parent, previously_visited_parent); + _stack.replace_top_with(clone); + } + + // Child of 'previously_visited_parent' (a clone) was already cloned before. We only need to rewire the child to + // 'previously_visited_parent'. + void set_req_of_clone_to_parent(Node* previously_visited_parent) const { + Node* child_of_parent = _stack.top(); + const uint index_to_previously_visited_parent = _stack.node_index_to_previously_visited_parent(); + child_of_parent->set_req(index_to_previously_visited_parent, previously_visited_parent); + } + + // Pop a node from the stack and clone the new node on top if the parent node was cloned or transformed before (i.e. + // a node on the chain to an OpaqueLoop* node). The clone is set as new node on top. If the new node on top was + // already cloned, check if the previously visited parent was also cloned. If so, we do not need to clone the new + // node on top but can just connect it to the previously visited cloned parent. Finally, increment the input index + // to visit the next unvisited input of the current top node. + void pop_node() { + Node* previously_visited_parent = _stack.top(); + _stack.pop(); + if (_stack.is_not_empty()) { + if (must_clone_top_node(previously_visited_parent)) { + clone_top_node(previously_visited_parent); + } else if (is_cloned_node(previously_visited_parent)) { + rewire_top_node_to(previously_visited_parent); + } // else: Node is not part on the chain to an OpaqueLoop* nod + } + } + + bool is_cloned_node(Node* node) const { + return node->_idx >= _index_before_cloning; + } + + void rewire_top_node_to(Node* previously_visited_parent) { + _stack.top()->set_req(_stack.node_index_to_previously_visited_parent(), previously_visited_parent); + } + + public: + CloneAssertionPredicateBool(BoolNode* template_bool, Node* ctrl_for_clones, PhaseIdealLoop* phase) + : _stack(template_bool), + _phase(phase), + _index_before_cloning(phase->C->unique()), + _ctrl_for_clones(ctrl_for_clones) + DEBUG_ONLY(COMMA _found_init(false)) {} + + // Look for the OpaqueLoop* nodes to transform them with the strategy defined with 'transform_opaque_loop_nodes'. + // Clone all nodes in between. + BoolNode* clone(TransformOpaqueLoopNodes* transform_opaque_loop_nodes) { + Node* current; + while (_stack.is_not_empty()) { + current = _stack.top(); + if (current->is_Opaque1()) { + Node* transformed_node = transform_opaque_loop_node(current, transform_opaque_loop_nodes); + pop_opaque_loop_node(transformed_node); + } else if (!_stack.push_next_unvisited_input()) { + pop_node(); + } + } + assert(current->is_Bool() && current->_idx >= _index_before_cloning, "new BoolNode expected"); + assert(_found_init, "OpaqueLoopInitNode must always be found"); + return current->as_Bool(); + } +}; + +// This class caches a single OpaqueLoopInitNode and OpaqueLoopStrideNode. If the node is not cached, yet, we clone it +// and store the clone in the cache to be return for subsequent calls. +class CachedOpaqueLoopNodes { + OpaqueLoopInitNode* _cached_opaque_new_init; + OpaqueLoopStrideNode* _cached_new_opaque_stride; + PhaseIdealLoop* _phase; + Node* _new_ctrl; + + public: + CachedOpaqueLoopNodes(PhaseIdealLoop* phase, Node* new_ctrl) + : _cached_opaque_new_init(nullptr), + _cached_new_opaque_stride(nullptr), + _phase(phase), + _new_ctrl(new_ctrl) {} + + OpaqueLoopInitNode* clone_if_not_cached(OpaqueLoopInitNode* opaque_init) { + if (_cached_opaque_new_init == nullptr) { + _cached_opaque_new_init = _phase->clone_and_register(opaque_init, _new_ctrl)->as_OpaqueLoopInit(); + } + return _cached_opaque_new_init; + } + + OpaqueLoopStrideNode* clone_if_not_cached(OpaqueLoopStrideNode* opaque_stride) { + if (_cached_new_opaque_stride == nullptr) { + _cached_new_opaque_stride = _phase->clone_and_register(opaque_stride, _new_ctrl)->as_OpaqueLoopStride(); + } + return _cached_new_opaque_stride; + } +}; + +// The transformations of this class clone the existing OpaqueLoop* nodes without any other update. +class CloneOpaqueLoopNodes : public TransformOpaqueLoopNodes { + CachedOpaqueLoopNodes _cached_opaque_loop_nodes; + + public: + CloneOpaqueLoopNodes(PhaseIdealLoop* phase, Node* new_ctrl) + : _cached_opaque_loop_nodes(phase, new_ctrl) {} + + Node* transform_opaque_init(OpaqueLoopInitNode* opaque_init) override { + return _cached_opaque_loop_nodes.clone_if_not_cached(opaque_init); + } + + Node* transform_opaque_stride(OpaqueLoopStrideNode* opaque_stride) override { + return _cached_opaque_loop_nodes.clone_if_not_cached(opaque_stride); + } +}; + +// Clones this Template Assertion Predicate bool. This includes all nodes from the BoolNode to the OpaqueLoop* nodes. +// The cloned nodes are not updated. +BoolNode* TemplateAssertionPredicateBool::clone(Node* new_ctrl, PhaseIdealLoop* phase) { + CloneOpaqueLoopNodes clone_opaque_loop_nodes(phase, new_ctrl); + CloneAssertionPredicateBool clone_assertion_predicate_bool(_source_bool, new_ctrl, phase); + return clone_assertion_predicate_bool.clone(&clone_opaque_loop_nodes); +} + +// The transformations of this class clone the existing OpaqueLoop* nodes. The newly cloned OpaqueLoopInitNode +// additionally gets a new input node. +class CloneWithNewInit : public TransformOpaqueLoopNodes { + Node* _new_init; + CachedOpaqueLoopNodes _cached_opaque_loop_nodes; + + public: + CloneWithNewInit(PhaseIdealLoop* phase, Node* new_ctrl, Node* new_init) + : _new_init(new_init), + _cached_opaque_loop_nodes(phase, new_ctrl) {} + + Node* transform_opaque_init(OpaqueLoopInitNode* opaque_init) override { + return _new_init; + } + + Node* transform_opaque_stride(OpaqueLoopStrideNode* opaque_stride) override { + return _cached_opaque_loop_nodes.clone_if_not_cached(opaque_stride); + } +}; + +// Clones this Template Assertion Predicate bool. This includes all nodes from the BoolNode to the OpaqueLoop* nodes. +// The newly cloned OpaqueLoopInitNodes additionally get 'new_opaque_init_input' as a new input. The other nodes are +// not updated. +BoolNode* TemplateAssertionPredicateBool::clone_and_replace_init(Node* new_ctrl, Node* new_init, + PhaseIdealLoop* phase) { + CloneWithNewInit clone_with_new_init(phase, new_ctrl, new_init); + CloneAssertionPredicateBool clone_assertion_predicate_bool(_source_bool, new_ctrl, phase); + return clone_assertion_predicate_bool.clone(&clone_with_new_init); +} + + +// The transformations of this class fold the OpaqueLoop* nodes by returning their inputs. +class ReplaceOpaqueLoopNodes : public TransformOpaqueLoopNodes { + Node* _new_init; + Node* _new_stride; + + public: + ReplaceOpaqueLoopNodes(Node* new_init, Node* new_stride) : _new_init(new_init), _new_stride(new_stride) {} + + Node* transform_opaque_init(OpaqueLoopInitNode* opaque_init) override { + return _new_init; + } + + Node* transform_opaque_stride(OpaqueLoopStrideNode* opaque_stride) override { + return _new_stride; + } +}; + +// Clones this Template Assertion Predicate bool. This includes all nodes from the BoolNode to the OpaqueLoop* nodes. +// The cloned nodes are not updated. +BoolNode* TemplateAssertionPredicateBool::clone_and_replace_opaque_loop_nodes(Node* new_ctrl, Node* new_init, + Node* new_stride, PhaseIdealLoop* phase) { + ReplaceOpaqueLoopNodes replaceOpaqueLoopNodes(new_init, new_stride); + CloneAssertionPredicateBool clone_assertion_predicate_bool(_source_bool, new_ctrl, phase); + return clone_assertion_predicate_bool.clone(&replaceOpaqueLoopNodes); +} diff --git a/src/hotspot/share/opto/predicates.hpp b/src/hotspot/share/opto/predicates.hpp index 89d8bbad030d5..2c891a568273f 100644 --- a/src/hotspot/share/opto/predicates.hpp +++ b/src/hotspot/share/opto/predicates.hpp @@ -26,6 +26,8 @@ #define SHARE_OPTO_PREDICATES_HPP #include "opto/cfgnode.hpp" +#include "opto/loopnode.hpp" +#include "opto/opaquenode.hpp" /* * There are different kinds of predicates throughout the code. We differentiate between the following predicates: @@ -263,6 +265,45 @@ class RuntimePredicate : public StackObj { static bool is_success_proj(Node* node, Deoptimization::DeoptReason deopt_reason); }; +// Utility class for querying Assertion Predicate bool opcodes. +class AssertionPredicateBoolOpcodes : public StackObj { + public: + // Is 'n' a node that could be part of an Assertion Predicate bool (i.e. could be found on the input chain of an + // Assertion Predicate bool up to and including the OpaqueLoop* nodes)? + static bool is_valid(const Node* n) { + const int opcode = n->Opcode(); + return (opcode == Op_OpaqueLoopInit || + opcode == Op_OpaqueLoopStride || + n->is_Bool() || + n->is_Cmp() || + opcode == Op_AndL || + opcode == Op_OrL || + opcode == Op_RShiftL || + opcode == Op_LShiftL || + opcode == Op_LShiftI || + opcode == Op_AddL || + opcode == Op_AddI || + opcode == Op_MulL || + opcode == Op_MulI || + opcode == Op_SubL || + opcode == Op_SubI || + opcode == Op_ConvI2L || + opcode == Op_CastII); + } +}; + +// Class that represents either the BoolNode for the initial value or the last value of a Template Assertion Predicate. +class TemplateAssertionPredicateBool : public StackObj { + BoolNode* _source_bool; + + public: + explicit TemplateAssertionPredicateBool(Node* source_bool); + + BoolNode* clone(Node* new_ctrl, PhaseIdealLoop* phase); + BoolNode* clone_and_replace_init(Node* new_ctrl, Node* new_init, PhaseIdealLoop* phase); + BoolNode* clone_and_replace_opaque_loop_nodes(Node* new_ctrl, Node* new_init, Node* new_stride, PhaseIdealLoop* phase); +}; + // This class represents a Predicate Block (i.e. either a Loop Predicate Block, a Profiled Loop Predicate Block, // or a Loop Limit Check Predicate Block). It contains zero or more Regular Predicates followed by a Parse Predicate // which, however, does not need to exist (we could already have decided to remove Parse Predicates for this loop). diff --git a/src/hotspot/share/opto/split_if.cpp b/src/hotspot/share/opto/split_if.cpp index 5b6462307bb4d..6dd96883b535d 100644 --- a/src/hotspot/share/opto/split_if.cpp +++ b/src/hotspot/share/opto/split_if.cpp @@ -30,7 +30,7 @@ #include "opto/loopnode.hpp" #include "opto/movenode.hpp" #include "opto/opaquenode.hpp" - +#include "opto/predicates.hpp" //------------------------------split_thru_region------------------------------ // Split Node 'n' through merge point. @@ -95,24 +95,7 @@ bool PhaseIdealLoop::split_up( Node *n, Node *blk1, Node *blk2 ) { return true; } - if (subgraph_has_opaque(n)) { - Unique_Node_List wq; - wq.push(n); - for (uint i = 0; i < wq.size(); i++) { - Node* m = wq.at(i); - if (m->is_If()) { - assert(assertion_predicate_has_loop_opaque_node(m->as_If()), "opaque node not reachable from if?"); - Node* bol = create_bool_from_template_assertion_predicate(m, nullptr, nullptr, m->in(0)); - _igvn.replace_input_of(m, 1, bol); - } else { - assert(!m->is_CFG(), "not CFG expected"); - for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax; j++) { - Node* u = m->fast_out(j); - wq.push(u); - } - } - } - } + clone_template_assertion_predicate_bool_down_if_related(n); if (n->Opcode() == Op_OpaqueZeroTripGuard) { // If this Opaque1 is part of the zero trip guard for a loop: @@ -426,6 +409,99 @@ bool PhaseIdealLoop::clone_cmp_down(Node* n, const Node* blk1, const Node* blk2) return false; } +// This class clones Template Assertion Predicates bools down as part of the Split If optimization. +class CloneTemplateAssertionPredicateBoolDown { + PhaseIdealLoop* _phase; + + // Check if 'n' belongs to the init or last value Template Assertion Predicate bool, including the OpaqueLoop* nodes. + static bool is_part_of_template_assertion_predicate_bool(Node* n) { + if (AssertionPredicateBoolOpcodes::is_valid(n)) { + ResourceMark rm; + Unique_Node_List list; + list.push(n); + for (uint i = 0; i < list.size(); i++) { + Node* next = list.at(i); + const int opcode = next->Opcode(); + if (opcode == Op_OpaqueLoopInit || opcode == Op_OpaqueLoopStride) { + return true; + } else if (AssertionPredicateBoolOpcodes::is_valid(next)) { + push_non_null_inputs(list, next); + } + } + } + return false; + } + + static void push_non_null_inputs(Unique_Node_List& list, const Node* n) { + for (uint i = 1; i < n->req(); i++) { + Node* input = n->in(i); + if (input != nullptr) { + list.push(input); + } + } + } + + // OpaqueLoop*Node cloned OpaqueLoop*Node + // | | + // ... cloned ... + // | | + // template_assertion_predicate_bool_node ===> cloned template_assertion_predicate_bool_node + // | | + // template_assertion_predicate template_assertion_predicate + void clone_template_assertion_predicate_bool(Node* n) { + Unique_Node_List list; + list.push(n); + + for (uint i = 0; i < list.size(); i++) { + Node* next = list.at(i); + if (is_template_assertion_predicate(next)) { + clone_template_assertion_predicate_bool_and_replace(next); + } else { + assert(!next->is_CFG(), "no CFG expected in Template Assertion Predicate bool outputs"); + push_outputs(list, next); + } + } + } + + static bool is_template_assertion_predicate(Node* n) { + return n->is_If() && n->in(1)->Opcode() == Op_Opaque4; + } + + void clone_template_assertion_predicate_bool_and_replace(Node* template_assertion_predicate) { + Node* new_ctrl = template_assertion_predicate->in(0); + Node* opaque_node = template_assertion_predicate->in(1); + TemplateAssertionPredicateBool template_assertion_predicate_bool(opaque_node->in(1)); + BoolNode* cloned_bool = template_assertion_predicate_bool.clone(new_ctrl, _phase); + _phase->igvn().replace_input_of(opaque_node, 1, cloned_bool); + } + + static void push_outputs(Unique_Node_List& list, const Node* n) { + for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { + Node* out = n->fast_out(j); + list.push(out); + } + } + + public: + CloneTemplateAssertionPredicateBoolDown(PhaseIdealLoop* phase) : _phase(phase) {} + + // 'n' could be a node on the input chain from an OpaqueLoop* node (or an OpaqueLoop* node itself) to a Template Assertion + // Predicate bool. In this case, we need to clone the bool and the entire input chain including the OpaqueLoop* nodes + // down. This avoids creating the phi node inside the input chain. Otherwise, we could not find the OpaqueLoop* nodes + // anymore when trying to access them for a Template Assertion Predicate (pattern matching does not expect phis). + void clone_if_part_of_template_assertion_predicate_bool(Node* n) { + if (!is_part_of_template_assertion_predicate_bool(n)) { + return; + } + clone_template_assertion_predicate_bool(n); + } +}; + +void PhaseIdealLoop::clone_template_assertion_predicate_bool_down_if_related(Node* n) { + CloneTemplateAssertionPredicateBoolDown clone_down(this); + clone_down.clone_if_part_of_template_assertion_predicate_bool(n); +} + //------------------------------register_new_node------------------------------ void PhaseIdealLoop::register_new_node( Node *n, Node *blk ) { assert(!n->is_CFG(), "must be data node");