diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 4c71ed9a898e6..406013d360bfc 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -169,18 +169,20 @@ Node *PhaseIdealLoop::get_early_ctrl_for_expensive(Node *n, Node* earliest) { return earliest; } - while (1) { - Node *next = ctl; - // Moving the node out of a loop on the projection of a If - // confuses loop predication. So once we hit a Loop in a If branch + while (true) { + Node* next = ctl; + // Moving the node out of a loop on the projection of an If + // confuses Loop Predication. So, once we hit a loop in an If branch // that doesn't branch to an UNC, we stop. The code that process // expensive nodes will notice the loop and skip over it to try to // move the node further up. - if (ctl->is_CountedLoop() && ctl->in(1) != nullptr && ctl->in(1)->in(0) != nullptr && ctl->in(1)->in(0)->is_If()) { - if (!ctl->in(1)->as_Proj()->is_uncommon_trap_if_pattern()) { - break; - } - next = idom(ctl->in(1)->in(0)); + PredicateEntryIterator predicate_iterator(ctl->is_CountedLoop() ? ctl->in(LoopNode::EntryControl) : ctl); + if (predicate_iterator.has_next()) { + // Skip all predicates to not confuse Loop Predication. + do { + next = predicate_iterator.next_entry(); + } while (predicate_iterator.has_next()); + assert(is_dominator(next, ctl), "must be dominating after skipping predicates"); } else if (ctl->is_Proj()) { // We only move it up along a projection if the projection is // the single control projection for its parent: same code path, @@ -6368,31 +6370,16 @@ void PhaseIdealLoop::build_loop_late_post_work(Node *n, bool pinned) { if (least != early) { // Move the node above predicates as far up as possible so a - // following pass of loop predication doesn't hoist a predicate + // following pass of Loop Predication doesn't hoist a predicate // that depends on it above that node. - Node* new_ctrl = least; - for (;;) { - if (!new_ctrl->is_Proj()) { - break; - } - CallStaticJavaNode* call = new_ctrl->as_Proj()->is_uncommon_trap_if_pattern(); - if (call == nullptr) { - break; - } - int req = call->uncommon_trap_request(); - Deoptimization::DeoptReason trap_reason = Deoptimization::trap_request_reason(req); - if (trap_reason != Deoptimization::Reason_loop_limit_check && - trap_reason != Deoptimization::Reason_predicate && - trap_reason != Deoptimization::Reason_profile_predicate) { - break; - } - Node* c = new_ctrl->in(0)->in(0); - if (is_dominator(c, early) && c != early) { + PredicateEntryIterator predicate_iterator(least); + while (predicate_iterator.has_next()) { + Node* next_predicate_entry = predicate_iterator.next_entry(); + if (is_dominator(next_predicate_entry, early) && next_predicate_entry != early) { break; } - new_ctrl = c; + least = next_predicate_entry; } - least = new_ctrl; } // Try not to place code on a loop entry projection // which can inhibit range check elimination. diff --git a/src/hotspot/share/opto/predicates.cpp b/src/hotspot/share/opto/predicates.cpp index 9f782f34bdbb8..5b0de2e02d5f0 100644 --- a/src/hotspot/share/opto/predicates.cpp +++ b/src/hotspot/share/opto/predicates.cpp @@ -32,29 +32,29 @@ // (i.e. not belonging to an Initialized Assertion Predicate anymore) Node* AssertionPredicatesWithHalt::find_entry(Node* start_proj) { Node* entry = start_proj; - while (is_assertion_predicate_success_proj(entry)) { + while (AssertionPredicateWithHalt::is_predicate(entry)) { entry = entry->in(0)->in(0); } return entry; } -bool AssertionPredicatesWithHalt::is_assertion_predicate_success_proj(const Node* predicate_proj) { - if (predicate_proj == nullptr || !predicate_proj->is_IfProj() || !predicate_proj->in(0)->is_If()) { +bool AssertionPredicateWithHalt::is_predicate(const Node* maybe_success_proj) { + if (maybe_success_proj == nullptr || !maybe_success_proj->is_IfProj() || !maybe_success_proj->in(0)->is_If()) { return false; } - return has_assertion_predicate_opaque(predicate_proj) && has_halt(predicate_proj); + return has_assertion_predicate_opaque(maybe_success_proj) && has_halt(maybe_success_proj); } // Check if the If node of `predicate_proj` has an Opaque4 (Template Assertion Predicate) or an // OpaqueInitializedAssertionPredicate (Initialized Assertion Predicate) node as input. -bool AssertionPredicatesWithHalt::has_assertion_predicate_opaque(const Node* predicate_proj) { +bool AssertionPredicateWithHalt::has_assertion_predicate_opaque(const Node* predicate_proj) { IfNode* iff = predicate_proj->in(0)->as_If(); Node* bol = iff->in(1); return bol->is_Opaque4() || bol->is_OpaqueInitializedAssertionPredicate(); } // Check if the other projection (UCT projection) of `success_proj` has a Halt node as output. -bool AssertionPredicatesWithHalt::has_halt(const Node* success_proj) { +bool AssertionPredicateWithHalt::has_halt(const Node* success_proj) { ProjNode* other_proj = success_proj->as_IfProj()->other_if_proj(); return other_proj->outcnt() == 1 && other_proj->unique_out()->Opcode() == Op_Halt; } @@ -72,7 +72,15 @@ ParsePredicateNode* ParsePredicate::init_parse_predicate(Node* parse_predicate_p return nullptr; } -Deoptimization::DeoptReason RuntimePredicate::uncommon_trap_reason(IfProjNode* if_proj) { +bool ParsePredicate::is_predicate(Node* maybe_success_proj) { + if (!maybe_success_proj->is_IfProj()) { + return false; + } + IfNode* if_node = maybe_success_proj->in(0)->as_If(); + return if_node->is_ParsePredicate(); +} + +Deoptimization::DeoptReason RegularPredicateWithUCT::uncommon_trap_reason(IfProjNode* if_proj) { CallStaticJavaNode* uct_call = if_proj->is_uncommon_trap_if_pattern(); if (uct_call == nullptr) { return Deoptimization::Reason_none; @@ -80,8 +88,20 @@ Deoptimization::DeoptReason RuntimePredicate::uncommon_trap_reason(IfProjNode* i return Deoptimization::trap_request_reason(uct_call->uncommon_trap_request()); } -bool RuntimePredicate::is_success_proj(Node* node, Deoptimization::DeoptReason deopt_reason) { - if (may_be_runtime_predicate_if(node)) { +bool RegularPredicateWithUCT::is_predicate(Node* maybe_success_proj) { + if (may_be_predicate_if(maybe_success_proj)) { + IfProjNode* success_proj = maybe_success_proj->as_IfProj(); + const Deoptimization::DeoptReason deopt_reason = uncommon_trap_reason(success_proj); + return (deopt_reason == Deoptimization::Reason_loop_limit_check || + deopt_reason == Deoptimization::Reason_predicate || + deopt_reason == Deoptimization::Reason_profile_predicate); + } else { + return false; + } +} + +bool RegularPredicateWithUCT::is_predicate(Node* node, Deoptimization::DeoptReason deopt_reason) { + if (may_be_predicate_if(node)) { return deopt_reason == uncommon_trap_reason(node->as_IfProj()); } else { return false; @@ -89,7 +109,7 @@ bool RuntimePredicate::is_success_proj(Node* node, Deoptimization::DeoptReason d } // A Runtime Predicate must have an If or a RangeCheck node, while the If should not be a zero trip guard check. -bool RuntimePredicate::may_be_runtime_predicate_if(Node* node) { +bool RegularPredicateWithUCT::may_be_predicate_if(Node* node) { if (node->is_IfProj()) { const IfNode* if_node = node->in(0)->as_If(); const int opcode_if = if_node->Opcode(); @@ -101,6 +121,10 @@ bool RuntimePredicate::may_be_runtime_predicate_if(Node* node) { return false; } +bool RuntimePredicate::is_success_proj(Node* node, Deoptimization::DeoptReason deopt_reason) { + return RegularPredicateWithUCT::is_predicate(node, deopt_reason); +} + ParsePredicateIterator::ParsePredicateIterator(const Predicates& predicates) : _current_index(0) { const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); if (loop_limit_check_predicate_block->has_parse_predicate()) { @@ -356,3 +380,18 @@ bool TemplateAssertionPredicateExpressionNode::is_in_expression(Node* node) { bool TemplateAssertionPredicateExpressionNode::is_template_assertion_predicate(Node* node) { return node->is_If() && node->in(1)->is_Opaque4(); } + +// Is current node pointed to by iterator a predicate? +bool PredicateEntryIterator::has_next() const { + return ParsePredicate::is_predicate(_current) || + RegularPredicateWithUCT::is_predicate(_current) || + AssertionPredicateWithHalt::is_predicate(_current); +} + +// Skip the current predicate pointed to by iterator by returning the input into the predicate. This could possibly be +// a non-predicate node. +Node* PredicateEntryIterator::next_entry() { + assert(has_next(), "current must be predicate"); + _current = _current->in(0)->in(0); + return _current; +} diff --git a/src/hotspot/share/opto/predicates.hpp b/src/hotspot/share/opto/predicates.hpp index bf12aeb6a5163..f6ea26618359f 100644 --- a/src/hotspot/share/opto/predicates.hpp +++ b/src/hotspot/share/opto/predicates.hpp @@ -26,6 +26,7 @@ #define SHARE_OPTO_PREDICATES_HPP #include "opto/cfgnode.hpp" +#include "opto/connode.hpp" #include "opto/opaquenode.hpp" /* @@ -199,9 +200,6 @@ class AssertionPredicatesWithHalt : public StackObj { Node* _entry; static Node* find_entry(Node* start_proj); - static bool has_assertion_predicate_opaque(const Node* predicate_proj); - static bool has_halt(const Node* success_proj); - static bool is_assertion_predicate_success_proj(const Node* predicate_proj); public: AssertionPredicatesWithHalt(Node* assertion_predicate_proj) : _entry(find_entry(assertion_predicate_proj)) {} @@ -213,6 +211,30 @@ class AssertionPredicatesWithHalt : public StackObj { } }; +// Class to represent a single Assertion Predicate with a HaltNode. This could either be: +// - A Template Assertion Predicate. +// - A Initialized Assertion Predicate. +// Note that all other Regular Predicates have an UCT node. +class AssertionPredicateWithHalt : public StackObj { + static bool has_assertion_predicate_opaque(const Node* predicate_proj); + static bool has_halt(const Node* success_proj); + public: + static bool is_predicate(const Node* maybe_success_proj); +}; + +// Class to represent a single Regular Predicate with an UCT. This could either be: +// - A Runtime Predicate +// - A Template Assertion Predicate +// Note that all other Regular Predicates have a Halt node. +class RegularPredicateWithUCT : public StackObj { + static Deoptimization::DeoptReason uncommon_trap_reason(IfProjNode* if_proj); + static bool may_be_predicate_if(Node* node); + + public: + static bool is_predicate(Node* maybe_success_proj); + static bool is_predicate(Node* node, Deoptimization::DeoptReason deopt_reason); +}; + // Class to represent a Parse Predicate. class ParsePredicate : public StackObj { ParsePredicateSuccessProj* _success_proj; @@ -253,13 +275,12 @@ class ParsePredicate : public StackObj { assert(is_valid(), "must be valid"); return _success_proj; } + + static bool is_predicate(Node* maybe_success_proj); }; -// Utility class for queries on Runtime Predicates. +// Class to represent a Runtime Predicate. class RuntimePredicate : public StackObj { - static Deoptimization::DeoptReason uncommon_trap_reason(IfProjNode* if_proj); - static bool may_be_runtime_predicate_if(Node* node); - public: static bool is_success_proj(Node* node, Deoptimization::DeoptReason deopt_reason); }; @@ -473,4 +494,17 @@ class ParsePredicateIterator : public StackObj { ParsePredicateNode* next(); }; + +// Special predicate iterator that can be used to walk through predicate entries, regardless whether the predicate +// belongs to the same loop or not (i.e. leftovers from already folded nodes). The iterator returns the next entry +// to a predicate. +class PredicateEntryIterator : public StackObj { + Node* _current; + + public: + PredicateEntryIterator(Node* start) : _current(start) {}; + + bool has_next() const; + Node* next_entry(); +}; #endif // SHARE_OPTO_PREDICATES_HPP