From 4a035a371c7f750af041c37964af805f4552359a Mon Sep 17 00:00:00 2001 From: Christian Hagedorn Date: Mon, 27 Nov 2023 09:36:50 +0100 Subject: [PATCH] Full fix --- src/hotspot/share/opto/cfgnode.cpp | 64 ++ src/hotspot/share/opto/cfgnode.hpp | 88 ++- src/hotspot/share/opto/classes.hpp | 2 + src/hotspot/share/opto/compile.cpp | 5 - src/hotspot/share/opto/compile.hpp | 18 +- src/hotspot/share/opto/graphKit.cpp | 7 +- src/hotspot/share/opto/ifnode.cpp | 35 +- src/hotspot/share/opto/loopPredicate.cpp | 412 ++++--------- src/hotspot/share/opto/loopTransform.cpp | 514 +++------------- src/hotspot/share/opto/loopUnswitch.cpp | 217 ++++++- src/hotspot/share/opto/loopnode.cpp | 302 ++++------ src/hotspot/share/opto/loopnode.hpp | 106 +--- src/hotspot/share/opto/loopopts.cpp | 36 +- src/hotspot/share/opto/node.cpp | 3 - src/hotspot/share/opto/node.hpp | 5 +- src/hotspot/share/opto/opaquenode.cpp | 20 + src/hotspot/share/opto/opaquenode.hpp | 13 + src/hotspot/share/opto/predicates.cpp | 724 +++++++++++++++++++---- src/hotspot/share/opto/predicates.hpp | 576 +++++++++++++++--- src/hotspot/share/opto/split_if.cpp | 39 +- 20 files changed, 1893 insertions(+), 1293 deletions(-) diff --git a/src/hotspot/share/opto/cfgnode.cpp b/src/hotspot/share/opto/cfgnode.cpp index 7bffd75d3f624..948c1189c7846 100644 --- a/src/hotspot/share/opto/cfgnode.cpp +++ b/src/hotspot/share/opto/cfgnode.cpp @@ -39,6 +39,7 @@ #include "opto/narrowptrnode.hpp" #include "opto/mulnode.hpp" #include "opto/phaseX.hpp" +#include "opto/predicates.hpp" #include "opto/regalloc.hpp" #include "opto/regmask.hpp" #include "opto/runtime.hpp" @@ -2814,6 +2815,69 @@ const RegMask &GotoNode::out_RegMask() const { return RegMask::Empty; } +TemplateAssertionPredicateNode::TemplateAssertionPredicateNode(Node* control, BoolNode* bool_init_value, + BoolNode* bool_last_value, + const int initialized_init_value_opcode, + const int initialized_last_value_opcode) + : Node(control, bool_init_value, bool_last_value), + _initialized_init_value_opcode(initialized_init_value_opcode), + _initialized_last_value_opcode(initialized_last_value_opcode), + _useless(false) { + assert(initialized_init_value_opcode == Op_If || initialized_init_value_opcode == Op_RangeCheck, "invalid opcode"); + assert(initialized_last_value_opcode == Op_If || initialized_last_value_opcode == Op_RangeCheck, "invalid opcode"); + init_class_id(Class_TemplateAssertionPredicate); +} + +IfNode* TemplateAssertionPredicateNode::create_initialized_assertion_predicate( + Node* control, OpaqueAssertionPredicateNode* opaque_bool, + AssertionPredicateType initialized_assertion_predicate_type) const { + bool create_if_node = true; + switch (initialized_assertion_predicate_type) { + case AssertionPredicateType::Init_value: + create_if_node = _initialized_init_value_opcode == Op_If; + break; + case AssertionPredicateType::Last_value: + create_if_node = _initialized_last_value_opcode == Op_If; + break; + default: + assert(false, "invalid Assertion Predicate type"); + } + return create_if_node ? + new IfNode(control, opaque_bool, PROB_MAX, COUNT_UNKNOWN NOT_PRODUCT(COMMA initialized_assertion_predicate_type)) : + new RangeCheckNode(control, opaque_bool, PROB_MAX, COUNT_UNKNOWN NOT_PRODUCT(COMMA initialized_assertion_predicate_type)); +} + + +uint TemplateAssertionPredicateNode::index_for_bool_input(const BoolNode* bool_input) const { + if (bool_input == in(TemplateAssertionPredicateNode::InitValue)) { + return TemplateAssertionPredicateNode::InitValue; + } else { + assert(bool_input == in(TemplateAssertionPredicateNode::LastValue), "must be a bool input"); + return TemplateAssertionPredicateNode::LastValue; + } +} + +Node* TemplateAssertionPredicateNode::Identity(PhaseGVN* phase) { + if (phase->C->post_loop_opts_phase() || _useless) { + return in(0); + } else { + phase->C->record_for_post_loop_opts_igvn(this); + return this; + } +} + +const Type* TemplateAssertionPredicateNode::Value(PhaseGVN* phase) const { + return phase->type(in(0)); +} + +#ifndef PRODUCT +void TemplateAssertionPredicateNode::dump_spec(outputStream* st) const { + if (_useless) { + st->print("#useless "); + } +} +#endif // NOT PRODUCT + //============================================================================= const RegMask &JumpNode::out_RegMask() const { return RegMask::Empty; diff --git a/src/hotspot/share/opto/cfgnode.hpp b/src/hotspot/share/opto/cfgnode.hpp index bc6f181aa0755..061d8ac85ce41 100644 --- a/src/hotspot/share/opto/cfgnode.hpp +++ b/src/hotspot/share/opto/cfgnode.hpp @@ -56,7 +56,9 @@ class CatchProjNode; class JProjNode; class JumpProjNode; class SCMemProjNode; +class OpaqueAssertionPredicateNode; class PhaseIdealLoop; +enum class AssertionPredicateType; // The success projection of a Parse Predicate is always an IfTrueNode and the uncommon projection an IfFalseNode typedef IfTrueNode ParsePredicateSuccessProj; @@ -289,6 +291,45 @@ class GotoNode : public Node { virtual const RegMask &out_RegMask() const; }; +// This node represents a Template Assertion Predicate with two bools as input which can be used to create an +// Initialized Assertion Predicate from (more information can be found in the summary at predicates.hpp). +// This node is folded either after loop opts or once the associated CountedLoopNode is removed. +class TemplateAssertionPredicateNode : public Node { + int _initialized_init_value_opcode; + int _initialized_last_value_opcode; + bool _useless; // If this node becomes useless, it can be cleaned up by Identity(). + + virtual uint size_of() const {return sizeof(*this);} + public: + // Named bool inputs + enum { + InitValue = 1, + LastValue = 2 + }; + + TemplateAssertionPredicateNode(Node* control, BoolNode* bool_init_value, BoolNode* bool_last_value, + int initialized_init_value_opcode, int initialized_last_value_opcode); + + void mark_useless() { + _useless = true; + } + + IfNode* create_initialized_assertion_predicate(Node* control, OpaqueAssertionPredicateNode* opaque_bool, + AssertionPredicateType initialized_assertion_predicate_type) const; + uint index_for_bool_input(const BoolNode* bool_input) const; + + virtual int Opcode() const; + virtual bool pinned() const { return true; } + virtual bool is_CFG() const { return true; } + virtual uint hash() const { return NO_HASH; } // CFG nodes do not hash + virtual bool depends_only_on_test() const { return false; } + virtual const Type* bottom_type() const { return Type::CONTROL; } + virtual Node* Identity(PhaseGVN* phase); + virtual const Type* Value(PhaseGVN* phase) const; + + NOT_PRODUCT(void dump_spec(outputStream* st) const;) +}; + //------------------------------CProjNode-------------------------------------- // control projection for node that produces multiple control-flow paths class CProjNode : public ProjNode { @@ -318,11 +359,23 @@ class MultiBranchNode : public MultiNode { //------------------------------IfNode----------------------------------------- // Output selected Control, based on a boolean test class IfNode : public MultiBranchNode { + public: + float _prob; // Probability of true path being taken. + float _fcnt; // Frequency counter + + private: + NOT_PRODUCT(AssertionPredicateType _assertion_predicate_type;) + + void init_node(Node* control, Node* bol) { + init_class_id(Class_If); + init_req(0, control); + init_req(1, bol); + } + // Size is bigger to hold the probability field. However, _prob does not // change the semantics so it does not appear in the hash & cmp functions. virtual uint size_of() const { return sizeof(*this); } -private: // Helper methods for fold_compares bool cmpi_folds(PhaseIterGVN* igvn, bool fold_ne = false); bool is_ctrl_folds(Node* ctrl, PhaseIterGVN* igvn); @@ -413,14 +466,9 @@ class IfNode : public MultiBranchNode { // Magic manifest probabilities such as 0.83, 0.7, ... can be found in // gen_subtype_check() and catch_inline_exceptions(). - float _prob; // Probability of true path being taken. - float _fcnt; // Frequency counter - IfNode( Node *control, Node *b, float p, float fcnt ) - : MultiBranchNode(2), _prob(p), _fcnt(fcnt) { - init_class_id(Class_If); - init_req(0,control); - init_req(1,b); - } + IfNode(Node* control, Node* bol, float p, float fcnt); + NOT_PRODUCT(IfNode(Node* control, Node* bol, float p, float fcnt, AssertionPredicateType assertion_predicate_type);) + virtual int Opcode() const; virtual bool pinned() const { return true; } virtual const Type *bottom_type() const { return TypeTuple::IFBOTH; } @@ -447,13 +495,20 @@ class IfNode : public MultiBranchNode { class RangeCheckNode : public IfNode { private: - int is_range_check(Node* &range, Node* &index, jint &offset); + int is_range_check(Node*& range, Node*& index, jint& offset); public: - RangeCheckNode(Node* control, Node *b, float p, float fcnt) - : IfNode(control, b, p, fcnt) { + RangeCheckNode(Node* control, Node* bol, float p, float fcnt) + : IfNode(control, bol, p, fcnt) { + init_class_id(Class_RangeCheck); + } + +#ifndef PRODUCT + RangeCheckNode(Node* control, Node* bol, float p, float fcnt, AssertionPredicateType assertion_predicate_type) + : IfNode(control, bol, p, fcnt, assertion_predicate_type) { init_class_id(Class_RangeCheck); } +#endif // NOT_PRODUCT virtual int Opcode() const; virtual Node* Ideal(PhaseGVN *phase, bool can_reshape); @@ -461,11 +516,11 @@ class RangeCheckNode : public IfNode { // Special node that denotes a Parse Predicate added during parsing. A Parse Predicate serves as placeholder to later // create Regular Predicates (Runtime Predicates with possible Assertion Predicates) above it. Together they form a -// Predicate Block. The Parse Predicate and Regular Predicates share the same uncommon trap. +// Predicate Block. The Parse Predicate and Runtime Predicates share the same uncommon trap. // There are three kinds of Parse Predicates: // Loop Parse Predicate, Profiled Loop Parse Predicate (both used by Loop Predication), and Loop Limit Check Parse // Predicate (used for integer overflow checks when creating a counted loop). -// More information about predicates can be found in loopPredicate.cpp. +// More information about predicates can be found in predicates.hpp. class ParsePredicateNode : public IfNode { Deoptimization::DeoptReason _deopt_reason; bool _useless; // If the associated loop dies, this parse predicate becomes useless and can be cleaned up by Value(). @@ -490,6 +545,11 @@ class ParsePredicateNode : public IfNode { _useless = false; } + // Return the uncommon trap If projection of this Parse Predicate. + ParsePredicateUncommonProj* uncommon_proj() const { + return proj_out(0)->as_IfFalse(); + } + Node* uncommon_trap() const; Node* Ideal(PhaseGVN* phase, bool can_reshape) { diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp index 7d1a2cd1db0e8..43c4e39f81174 100644 --- a/src/hotspot/share/opto/classes.hpp +++ b/src/hotspot/share/opto/classes.hpp @@ -271,6 +271,7 @@ macro(OpaqueLoopStride) macro(OpaqueZeroTripGuard) macro(Opaque3) macro(Opaque4) +macro(OpaqueAssertionPredicate) macro(ProfileBoolean) macro(OrI) macro(OrL) @@ -362,6 +363,7 @@ macro(SubL) macro(TailCall) macro(TailJump) macro(MacroLogicV) +macro(TemplateAssertionPredicate) macro(ThreadLocal) macro(Unlock) macro(URShiftB) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 8a5e98eda9218..d5a567a823e5f 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -389,9 +389,6 @@ void Compile::remove_useless_node(Node* dead) { if (dead->is_expensive()) { remove_expensive_node(dead); } - if (dead->Opcode() == Op_Opaque4) { - remove_template_assertion_predicate_opaq(dead); - } if (dead->is_ParsePredicate()) { remove_parse_predicate(dead->as_ParsePredicate()); } @@ -443,7 +440,6 @@ void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_Lis remove_useless_nodes(_macro_nodes, useful); // remove useless macro nodes remove_useless_nodes(_parse_predicates, useful); // remove useless Parse Predicate nodes - remove_useless_nodes(_template_assertion_predicate_opaqs, useful); // remove useless Assertion Predicate opaque nodes remove_useless_nodes(_expensive_nodes, useful); // remove useless expensive nodes remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass remove_useless_unstable_if_traps(useful); // remove useless unstable_if traps @@ -640,7 +636,6 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci, _intrinsics (comp_arena(), 0, 0, nullptr), _macro_nodes (comp_arena(), 8, 0, nullptr), _parse_predicates (comp_arena(), 8, 0, nullptr), - _template_assertion_predicate_opaqs (comp_arena(), 8, 0, nullptr), _expensive_nodes (comp_arena(), 8, 0, nullptr), _for_post_loop_igvn(comp_arena(), 8, 0, nullptr), _unstable_if_traps (comp_arena(), 8, 0, nullptr), diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index af4eb6b1ee01a..ca8afcb2949d1 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -365,7 +365,6 @@ class Compile : public Phase { GrowableArray _intrinsics; // List of intrinsics. GrowableArray _macro_nodes; // List of nodes which need to be expanded before matching. GrowableArray _parse_predicates; // List of Parse Predicates. - GrowableArray _template_assertion_predicate_opaqs; // List of Opaque4 nodes for Template Assertion Predicates. GrowableArray _expensive_nodes; // List of nodes that are expensive to compute and that we'd better not let the GVN freely common GrowableArray _for_post_loop_igvn; // List of nodes for IGVN after loop opts are over GrowableArray _unstable_if_traps; // List of ifnodes after IGVN @@ -717,16 +716,12 @@ class Compile : public Phase { int macro_count() const { return _macro_nodes.length(); } int parse_predicate_count() const { return _parse_predicates.length(); } - int template_assertion_predicate_count() const { return _template_assertion_predicate_opaqs.length(); } int expensive_count() const { return _expensive_nodes.length(); } int coarsened_count() const { return _coarsened_locks.length(); } Node* macro_node(int idx) const { return _macro_nodes.at(idx); } - ParsePredicateNode* parse_predicate(int idx) const { return _parse_predicates.at(idx); } - Node* template_assertion_predicate_opaq_node(int idx) const { - return _template_assertion_predicate_opaqs.at(idx); - } + const GrowableArray& parse_predicates() const { return _parse_predicates; } Node* expensive_node(int idx) const { return _expensive_nodes.at(idx); } @@ -762,17 +757,6 @@ class Compile : public Phase { } } - void add_template_assertion_predicate_opaq(Node* n) { - assert(!_template_assertion_predicate_opaqs.contains(n), - "duplicate entry in template assertion predicate opaque4 list"); - _template_assertion_predicate_opaqs.append(n); - } - - void remove_template_assertion_predicate_opaq(Node* n) { - if (template_assertion_predicate_count() > 0) { - _template_assertion_predicate_opaqs.remove_if_existing(n); - } - } void add_coarsened_locks(GrowableArray& locks); void remove_coarsened_lock(Node* n); bool coarsened_locks_consistent(); diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp index b271dd6f63ceb..b949244996113 100644 --- a/src/hotspot/share/opto/graphKit.cpp +++ b/src/hotspot/share/opto/graphKit.cpp @@ -4061,10 +4061,11 @@ void GraphKit::add_parse_predicate(Deoptimization::DeoptReason reason, const int void GraphKit::add_parse_predicates(int nargs) { if (UseLoopPredicate) { add_parse_predicate(Deoptimization::Reason_predicate, nargs); + if (UseProfiledLoopPredicate) { + add_parse_predicate(Deoptimization::Reason_profile_predicate, nargs); + } } - if (UseProfiledLoopPredicate) { - add_parse_predicate(Deoptimization::Reason_profile_predicate, nargs); - } + // Loop Limit Check Predicate should be near the loop. add_parse_predicate(Deoptimization::Reason_loop_limit_check, nargs); } diff --git a/src/hotspot/share/opto/ifnode.cpp b/src/hotspot/share/opto/ifnode.cpp index 2751e8912624f..86215cf278cd0 100644 --- a/src/hotspot/share/opto/ifnode.cpp +++ b/src/hotspot/share/opto/ifnode.cpp @@ -47,6 +47,24 @@ extern uint explicit_null_checks_elided; #endif +IfNode::IfNode(Node* control, Node* bol, float p, float fcnt) + : MultiBranchNode(2), + _prob(p), + _fcnt(fcnt) + NOT_PRODUCT(COMMA _assertion_predicate_type(AssertionPredicateType::None)) { + init_node(control, bol); +} + +#ifndef PRODUCT +IfNode::IfNode(Node* control, Node* bol, float p, float fcnt, AssertionPredicateType assertion_predicate_type) + : MultiBranchNode(2), + _prob(p), + _fcnt(fcnt), + _assertion_predicate_type(assertion_predicate_type) { + init_node(control, bol); +} +#endif // NOT_PRODUCT + //============================================================================= //------------------------------Value------------------------------------------ // Return a tuple for whichever arm of the IF is reachable @@ -1783,8 +1801,18 @@ bool IfNode::is_zero_trip_guard() const { #ifndef PRODUCT //------------------------------dump_spec-------------------------------------- -void IfNode::dump_spec(outputStream *st) const { - st->print("P=%f, C=%f",_prob,_fcnt); +void IfNode::dump_spec(outputStream* st) const { + switch (_assertion_predicate_type) { + case AssertionPredicateType::Init_value: + st->print("#Init Value Assertion Predicate "); + break; + case AssertionPredicateType::Last_value: + st->print("#Last Value Assertion Predicate "); + break; + default: + break; + } + st->print("P=%f, C=%f",_prob, _fcnt); } #endif @@ -2067,6 +2095,9 @@ void ParsePredicateNode::dump_spec(outputStream* st) const { default: fatal("unknown kind"); } + if (_useless) { + st->print("#useless "); + } } #endif // NOT PRODUCT diff --git a/src/hotspot/share/opto/loopPredicate.cpp b/src/hotspot/share/opto/loopPredicate.cpp index c2f4e0608ed5e..ecb9ba456292f 100644 --- a/src/hotspot/share/opto/loopPredicate.cpp +++ b/src/hotspot/share/opto/loopPredicate.cpp @@ -30,15 +30,11 @@ #include "opto/castnode.hpp" #include "opto/connode.hpp" #include "opto/convertnode.hpp" -#include "opto/loopnode.hpp" #include "opto/matcher.hpp" #include "opto/mulnode.hpp" -#include "opto/opaquenode.hpp" #include "opto/predicates.hpp" #include "opto/rootnode.hpp" #include "opto/subnode.hpp" -#include -#include /* * The general idea of Loop Predication is to hoist a check inside a loop body by inserting a Hoisted Check Predicate with @@ -64,8 +60,7 @@ void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred, } } -//------------------------------create_new_if_for_predicate------------------------ -// create a new if above the uct_if_pattern for the predicate to be promoted. +// Creates a new if above a Parse Predicate with the same uct_if_pattern: // // before after // ---------- ---------- @@ -98,41 +93,39 @@ void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred, // // We will create a region to guard the uct call if there is no one there. // The continuation projection (if_cont) of the new_iff is returned which -// is an IfTrue projection. This code is also used to clone predicates to cloned loops. -IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, - Deoptimization::DeoptReason reason, +// is an IfTrue projection. +IfTrueNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessProj* parse_predicate_success_proj, + Node* new_entry, Deoptimization::DeoptReason reason, const int opcode, const bool rewire_uncommon_proj_phi_inputs) { - assert(parse_predicate_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!"); - ParsePredicateNode* parse_predicate = parse_predicate_proj->in(0)->as_ParsePredicate(); - - ProjNode* uncommon_proj = parse_predicate->proj_out(false); - Node* uct_region = uncommon_proj->unique_ctrl_out(); - assert(uct_region->is_Region() || uct_region->is_Call(), "must be a region or call uct"); + assert(parse_predicate_success_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!"); + ParsePredicateNode* parse_predicate = parse_predicate_success_proj->in(0)->as_ParsePredicate(); + ParsePredicateUncommonProj* uncommon_proj = parse_predicate->uncommon_proj(); + Node* uncommon_trap = parse_predicate->uncommon_trap(); uint proj_index = 1; // region's edge corresponding to uncommon_proj - if (!uct_region->is_Region()) { // create a region to guard the call - assert(uct_region->is_Call(), "must be call uct"); - CallNode* call = uct_region->as_Call(); + if (!uncommon_trap->is_Region()) { // create a region to guard the call + assert(uncommon_trap->is_Call(), "must be call uct"); + CallNode* call = uncommon_trap->as_Call(); IdealLoopTree* loop = get_loop(call); - uct_region = new RegionNode(1); + uncommon_trap = new RegionNode(1); Node* uncommon_proj_orig = uncommon_proj; - uncommon_proj = uncommon_proj->clone()->as_Proj(); + uncommon_proj = uncommon_proj->clone()->as_IfFalse(); register_control(uncommon_proj, loop, parse_predicate); - uct_region->add_req(uncommon_proj); - register_control(uct_region, loop, uncommon_proj); - _igvn.replace_input_of(call, 0, uct_region); + uncommon_trap->add_req(uncommon_proj); + register_control(uncommon_trap, loop, uncommon_proj); + _igvn.replace_input_of(call, 0, uncommon_trap); // When called from beautify_loops() idom is not constructed yet. if (_idom != nullptr) { - set_idom(call, uct_region, dom_depth(uct_region)); + set_idom(call, uncommon_trap, dom_depth(uncommon_trap)); } // Move nodes pinned on the projection or whose control is set to // the projection to the region. - lazy_replace(uncommon_proj_orig, uct_region); + lazy_replace(uncommon_proj_orig, uncommon_trap); } else { // Find region's edge corresponding to uncommon_proj - for (; proj_index < uct_region->req(); proj_index++) - if (uct_region->in(proj_index) == uncommon_proj) break; - assert(proj_index < uct_region->req(), "sanity"); + for (; proj_index < uncommon_trap->req(); proj_index++) + if (uncommon_trap->in(proj_index) == uncommon_proj) break; + assert(proj_index < uncommon_trap->req(), "sanity"); } Node* entry = parse_predicate->in(0); @@ -157,22 +150,23 @@ IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessPro fatal("no other If variant here"); } register_control(new_iff, lp, entry); - IfProjNode* if_cont = new IfTrueNode(new_iff); - IfProjNode* if_uct = new IfFalseNode(new_iff); + + IfTrueNode* if_cont = new IfTrueNode(new_iff); + IfFalseNode* if_uct = new IfFalseNode(new_iff); register_control(if_cont, lp, new_iff); - register_control(if_uct, get_loop(uct_region), new_iff); + register_control(if_uct, get_loop(uncommon_trap), new_iff); - _igvn.add_input_to(uct_region, if_uct); + _igvn.add_input_to(uncommon_trap, if_uct); // If rgn has phis add new edges which has the same // value as on original uncommon_proj pass. - assert(uct_region->in(uct_region->req() - 1) == if_uct, "new edge should be last"); + assert(uncommon_trap->in(uncommon_trap->req() - 1) == if_uct, "new edge should be last"); bool has_phi = false; - for (DUIterator_Fast imax, i = uct_region->fast_outs(imax); i < imax; i++) { - Node* use = uct_region->fast_out(i); + for (DUIterator_Fast imax, i = uncommon_trap->fast_outs(imax); i < imax; i++) { + Node* use = uncommon_trap->fast_out(i); if (use->is_Phi() && use->outcnt() > 0) { - assert(use->in(0) == uct_region, ""); + assert(use->in(0) == uncommon_trap, ""); _igvn.rehash_node_delayed(use); Node* phi_input = use->in(proj_index); @@ -193,7 +187,7 @@ IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessPro has_phi = true; } } - assert(!has_phi || uct_region->req() > 3, "no phis when region is created"); + assert(!has_phi || uncommon_trap->req() > 3, "no phis when region is created"); if (new_entry == nullptr) { // Attach if_cont to iff @@ -205,12 +199,12 @@ IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessPro // When called from beautify_loops() idom is not constructed yet. if (_idom != nullptr) { - Node* ridom = idom(uct_region); + Node* ridom = idom(uncommon_trap); Node* nrdom = dom_lca_internal(ridom, new_iff); - set_idom(uct_region, nrdom, dom_depth(uct_region)); + set_idom(uncommon_trap, nrdom, dom_depth(uncommon_trap)); } - return if_cont->as_IfProj(); + return if_cont; } // Update ctrl and control inputs of all data nodes starting from 'node' to 'new_ctrl' which have 'old_ctrl' as @@ -299,176 +293,6 @@ void PhaseIdealLoop::rewire_inputs_of_clones_to_clones(Node* new_ctrl, Node* clo } } -IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj, - Node* new_entry, Deoptimization::DeoptReason reason, - const bool slow_loop) { - - IfProjNode* new_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, new_entry, reason, Op_ParsePredicate, - slow_loop); - assert(new_predicate_proj->is_IfTrue(), "the success projection of a Parse Predicate is a true projection"); - ParsePredicateNode* parse_predicate = new_predicate_proj->in(0)->as_ParsePredicate(); - return new_predicate_proj; -} - -// Clones Assertion Predicates to both unswitched loops starting at 'old_predicate_proj' by following its control inputs. -// It also rewires the control edges of data nodes with dependencies in the loop from the old predicates to the new -// cloned predicates. -void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, - Deoptimization::DeoptReason reason, - IfProjNode* old_predicate_proj, - ParsePredicateSuccessProj* fast_loop_parse_predicate_proj, - ParsePredicateSuccessProj* slow_loop_parse_predicate_proj) { - assert(fast_loop_parse_predicate_proj->in(0)->is_ParsePredicate() && - slow_loop_parse_predicate_proj->in(0)->is_ParsePredicate(), "sanity check"); - // Only need to clone range check predicates as those can be changed and duplicated by inserting pre/main/post loops - // and doing loop unrolling. Push the original predicates on a list to later process them in reverse order to keep the - // original predicate order. - Unique_Node_List list; - get_assertion_predicates(old_predicate_proj, list); - - Node_List to_process; - IfNode* iff = old_predicate_proj->in(0)->as_If(); - IfProjNode* uncommon_proj = iff->proj_out(1 - old_predicate_proj->as_Proj()->_con)->as_IfProj(); - // Process in reverse order such that 'create_new_if_for_predicate' can be used in - // 'clone_assertion_predicate_for_unswitched_loops' and the original order is maintained. - for (int i = list.size() - 1; i >= 0; i--) { - Node* predicate = list.at(i); - assert(predicate->in(0)->is_If(), "must be If node"); - iff = predicate->in(0)->as_If(); - assert(predicate->is_Proj() && predicate->as_Proj()->is_IfProj(), "predicate must be a projection of an if node"); - IfProjNode* predicate_proj = predicate->as_IfProj(); - - IfProjNode* fast_proj = clone_assertion_predicate_for_unswitched_loops(iff, predicate_proj, reason, fast_loop_parse_predicate_proj); - assert(assertion_predicate_has_loop_opaque_node(fast_proj->in(0)->as_If()), "must find Assertion Predicate for fast loop"); - IfProjNode* slow_proj = clone_assertion_predicate_for_unswitched_loops(iff, predicate_proj, reason, slow_loop_parse_predicate_proj); - assert(assertion_predicate_has_loop_opaque_node(slow_proj->in(0)->as_If()), "must find Assertion Predicate for slow loop"); - - // Update control dependent data nodes. - for (DUIterator j = predicate->outs(); predicate->has_out(j); j++) { - Node* fast_node = predicate->out(j); - if (loop->is_member(get_loop(ctrl_or_self(fast_node)))) { - assert(fast_node->in(0) == predicate, "only control edge"); - Node* slow_node = old_new[fast_node->_idx]; - assert(slow_node->in(0) == predicate, "only control edge"); - _igvn.replace_input_of(fast_node, 0, fast_proj); - to_process.push(slow_node); - --j; - } - } - // Have to delay updates to the slow loop so uses of predicate are not modified while we iterate on them. - while (to_process.size() > 0) { - Node* slow_node = to_process.pop(); - _igvn.replace_input_of(slow_node, 0, slow_proj); - } - } -} - -// Put all Assertion Predicate projections on a list, starting at 'predicate' and going up in the tree. If 'get_opaque' -// is set, then the Opaque4 nodes of the Assertion Predicates are put on the list instead of the projections. -void PhaseIdealLoop::get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque) { - ParsePredicateNode* parse_predicate = predicate->in(0)->as_ParsePredicate(); - ProjNode* uncommon_proj = parse_predicate->proj_out(1 - predicate->as_Proj()->_con); - Node* rgn = uncommon_proj->unique_ctrl_out(); - assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct"); - predicate = parse_predicate->in(0); - while (predicate != nullptr && predicate->is_Proj() && predicate->in(0)->is_If()) { - IfNode* iff = predicate->in(0)->as_If(); - uncommon_proj = iff->proj_out(1 - predicate->as_Proj()->_con); - if (uncommon_proj->unique_ctrl_out() != rgn) { - break; - } - if (iff->in(1)->Opcode() == Op_Opaque4 && assertion_predicate_has_loop_opaque_node(iff)) { - if (get_opaque) { - // Collect the predicate Opaque4 node. - list.push(iff->in(1)); - } else { - // Collect the predicate projection. - list.push(predicate); - } - } - predicate = predicate->in(0)->in(0); - } -} - -// Clone an Assertion Predicate for an unswitched loop. OpaqueLoopInit and OpaqueLoopStride nodes are cloned and uncommon -// traps are kept for the predicate (a Halt node is used later when creating pre/main/post loops and copying this cloned -// predicate again). -IfProjNode* PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(Node* iff, IfProjNode* predicate, - Deoptimization::DeoptReason reason, - ParsePredicateSuccessProj* parse_predicate_proj) { - Node* opaque_node = iff->in(1); - TemplateAssertionPredicateBool template_assertion_predicate_bool(opaque_node->in(1)); - BoolNode* bol = template_assertion_predicate_bool.clone(parse_predicate_proj, this); - opaque_node = clone_and_register(opaque_node, parse_predicate_proj); - _igvn.replace_input_of(opaque_node, 1, bol); - IfProjNode* if_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, iff->Opcode(), false); - _igvn.replace_input_of(if_proj->in(0), 1, opaque_node); - _igvn.replace_input_of(parse_predicate_proj->in(0), 0, if_proj); - set_idom(parse_predicate_proj->in(0), if_proj, dom_depth(if_proj)); - return if_proj; -} - -// Clone the old Parse Predicates and Assertion Predicates before the unswitch If to the unswitched loops after the -// unswitch If. -void PhaseIdealLoop::clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new, - IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred) { - LoopNode* head = loop->_head->as_Loop(); - Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl); - - const Predicates predicates(entry); - clone_loop_predication_predicates_to_unswitched_loop(loop, old_new, predicates.loop_predicate_block(), - Deoptimization::Reason_predicate, iffast_pred, ifslow_pred); - clone_loop_predication_predicates_to_unswitched_loop(loop, old_new, predicates.profiled_loop_predicate_block(), - Deoptimization::Reason_profile_predicate, iffast_pred, ifslow_pred); - - const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); - if (loop_limit_check_predicate_block->has_parse_predicate() && !head->is_CountedLoop()) { - // Don't clone the Loop Limit Check Parse Predicate if we already have a counted loop (a Loop Limit Check Predicate - // is only created when converting a LoopNode to a CountedLoopNode). - clone_parse_predicate_to_unswitched_loops(loop_limit_check_predicate_block, Deoptimization::Reason_loop_limit_check, - iffast_pred, ifslow_pred); - } -} - -// Clone the Parse Predicate and Template Assertion Predicates of a Loop Predication related Predicate Block. -void PhaseIdealLoop::clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, - const PredicateBlock* predicate_block, - Deoptimization::DeoptReason reason, - IfProjNode*& iffast_pred, - IfProjNode*& ifslow_pred) { - if (predicate_block->has_parse_predicate()) { - // We currently only clone Assertion Predicates if there are Parse Predicates. This is not entirely correct and will - // be changed with the complete fix for Assertion Predicates. - clone_parse_predicate_to_unswitched_loops(predicate_block, reason, iffast_pred, ifslow_pred); - assert(iffast_pred->in(0)->is_ParsePredicate() && ifslow_pred->in(0)->is_ParsePredicate(), - "must be success projections of the cloned Parse Predicates"); - clone_assertion_predicates_to_unswitched_loop(loop, old_new, reason, predicate_block->parse_predicate_success_proj(), - iffast_pred->as_IfTrue(), ifslow_pred->as_IfTrue()); - } -} - -void PhaseIdealLoop::clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block, - Deoptimization::DeoptReason reason, - IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred) { - assert(predicate_block->has_parse_predicate(), "must have parse predicate"); - ParsePredicateSuccessProj* parse_predicate_proj = predicate_block->parse_predicate_success_proj(); - iffast_pred = clone_parse_predicate_to_unswitched_loop(parse_predicate_proj, iffast_pred, reason, false); - check_cloned_parse_predicate_for_unswitching(iffast_pred, true); - - ifslow_pred = clone_parse_predicate_to_unswitched_loop(parse_predicate_proj, ifslow_pred, reason, true); - check_cloned_parse_predicate_for_unswitching(ifslow_pred, false); -} - -#ifndef PRODUCT -void PhaseIdealLoop::check_cloned_parse_predicate_for_unswitching(const Node* new_entry, const bool is_fast_loop) { - assert(new_entry != nullptr, "IfTrue or IfFalse after clone predicate"); - if (TraceLoopPredicate) { - tty->print("Parse Predicate cloned to %s loop: ", is_fast_loop ? "fast" : "slow"); - new_entry->in(0)->dump(); - } -} -#endif - //------------------------------Invariance----------------------------------- // Helper class for loop_predication_impl to compute invariance on the fly and // clone invariants. @@ -786,8 +610,8 @@ bool IdealLoopTree::is_range_check_if(IfProjNode* if_success_proj, PhaseIdealLoo // max(scale*i + offset) = scale*(limit-stride) + offset // (2) stride*scale < 0 // max(scale*i + offset) = scale*init + offset -BoolNode* PhaseIdealLoop::rc_predicate(IdealLoopTree* loop, Node* ctrl, int scale, Node* offset, Node* init, - Node* limit, jint stride, Node* range, bool upper, bool& overflow) { +BoolNode* PhaseIdealLoop::rc_predicate(Node* ctrl, const int scale, Node* offset, Node* init, Node* limit, + const jint stride, Node* range, const bool upper, bool& overflow) { jint con_limit = (limit != nullptr && limit->is_Con()) ? limit->get_int() : 0; jint con_init = init->is_Con() ? init->get_int() : 0; jint con_offset = offset->is_Con() ? offset->get_int() : 0; @@ -1175,32 +999,34 @@ void PhaseIdealLoop::loop_predication_follow_branches(Node *n, IdealLoopTree *lo bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNode* if_success_proj, ParsePredicateSuccessProj* parse_predicate_proj, CountedLoopNode* cl, ConNode* zero, Invariance& invar, Deoptimization::DeoptReason reason) { - // Following are changed to nonnull when a predicate can be hoisted - IfProjNode* new_predicate_proj = nullptr; - IfNode* iff = if_success_proj->in(0)->as_If(); - Node* test = iff->in(1); - if (!test->is_Bool()) { //Conv2B, ... + IfNode* iff = if_success_proj->in(0)->as_If(); + Node* test = iff->in(1); + if (!test->is_Bool()) { // Conv2B, ... return false; } BoolNode* bol = test->as_Bool(); if (invar.is_invariant(bol)) { // Invariant test - new_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, - reason, - iff->Opcode()); - Node* ctrl = new_predicate_proj->in(0)->as_If()->in(0); - BoolNode* new_predicate_bol = invar.clone(bol, ctrl)->as_Bool(); + IfProjNode* hoisted_check_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, + iff->Opcode()); + Node* ctrl = hoisted_check_predicate_proj->in(0)->as_If()->in(0); + BoolNode* hoisted_check_predicate_bool = invar.clone(bol, ctrl)->as_Bool(); // Negate test if necessary (Parse Predicates always have IfTrue as success projection and IfFalse as uncommon trap) bool negated = false; if (if_success_proj->is_IfFalse()) { - new_predicate_bol = new BoolNode(new_predicate_bol->in(1), new_predicate_bol->_test.negate()); - register_new_node(new_predicate_bol, ctrl); + hoisted_check_predicate_bool = new BoolNode(hoisted_check_predicate_bool->in(1), + hoisted_check_predicate_bool->_test.negate()); + register_new_node(hoisted_check_predicate_bool, ctrl); negated = true; } - IfNode* new_predicate_iff = new_predicate_proj->in(0)->as_If(); + IfNode* new_predicate_iff = hoisted_check_predicate_proj->in(0)->as_If(); _igvn.hash_delete(new_predicate_iff); - new_predicate_iff->set_req(1, new_predicate_bol); + new_predicate_iff->set_req(1, hoisted_check_predicate_bool); + invar.map_ctrl(if_success_proj, hoisted_check_predicate_proj); // Mark hoisted check as invariant + + // Eliminate the old If in the loop body. + dominated_by(hoisted_check_predicate_proj, iff, negated); #ifndef PRODUCT if (TraceLoopPredicate) { tty->print("Predicate invariant if%s: %d ", negated ? " negated" : "", new_predicate_iff->_idx); @@ -1239,7 +1065,8 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod // Perform cloning to keep Invariance state correct since the // late schedule will place invariant things in the loop. - Node* ctrl = parse_predicate_proj->in(0)->as_If()->in(0); + ParsePredicateNode* parse_predicate = parse_predicate_proj->in(0)->as_ParsePredicate(); + Node* ctrl = parse_predicate->in(0); rng = invar.clone(rng, ctrl); if (offset && offset != zero) { assert(invar.is_invariant(offset), "offset must be loop invariant"); @@ -1248,31 +1075,39 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod // If predicate expressions may overflow in the integer range, longs are used. bool overflow = false; // Test the lower bound - BoolNode* lower_bound_bol = rc_predicate(loop, ctrl, scale, offset, init, limit, stride, rng, false, overflow); + BoolNode* lower_bound_bol = rc_predicate(ctrl, scale, offset, init, limit, stride, rng, false, overflow); const int if_opcode = iff->Opcode(); IfProjNode* lower_bound_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : if_opcode); IfNode* lower_bound_iff = lower_bound_proj->in(0)->as_If(); _igvn.hash_delete(lower_bound_iff); lower_bound_iff->set_req(1, lower_bound_bol); - if (TraceLoopPredicate) tty->print_cr("lower bound check if: %d", lower_bound_iff->_idx); + if (TraceLoopPredicate) { + tty->print_cr("lower bound check if: %d", lower_bound_iff->_idx); + } // Test the upper bound - BoolNode* upper_bound_bol = rc_predicate(loop, lower_bound_proj, scale, offset, init, limit, stride, rng, true, - overflow); + BoolNode* upper_bound_bol = rc_predicate(lower_bound_proj, scale, offset, init, limit, stride, rng, true, overflow); IfProjNode* upper_bound_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : if_opcode); assert(upper_bound_proj->in(0)->as_If()->in(0) == lower_bound_proj, "should dominate"); IfNode* upper_bound_iff = upper_bound_proj->in(0)->as_If(); _igvn.hash_delete(upper_bound_iff); upper_bound_iff->set_req(1, upper_bound_bol); - if (TraceLoopPredicate) tty->print_cr("upper bound check if: %d", lower_bound_iff->_idx); + if (TraceLoopPredicate) { + tty->print_cr("upper bound check if: %d", upper_bound_iff->_idx); + } - // Fall through into rest of the cleanup code which will move any dependent nodes to the skeleton predicates of the - // upper bound test. We always need to create skeleton predicates in order to properly remove dead loops when later - // splitting the predicated loop into (unreachable) sub-loops (i.e. done by unrolling, peeling, pre/main/post etc.). - new_predicate_proj = add_template_assertion_predicate(iff, loop, if_success_proj, parse_predicate_proj, upper_bound_proj, scale, - offset, init, limit, stride, rng, overflow, reason); + // Each newly created Hoisted Check Predicate is accompanied by two Template Assertion Predicates. Later, we initialize + // them by making a copy of them when splitting a loop into sub loops. The Assertion Predicates ensure that dead sub + // loops are removed properly. + NewTemplateAssertionPredicate new_template_assertion_predicate(cl, this); + TemplateAssertionPredicateNode* template_assertion_predicate = + new_template_assertion_predicate.create(if_opcode, upper_bound_proj, scale, offset, rng); + + replace_control_same_loop(parse_predicate, template_assertion_predicate); + eliminate_old_range_check(if_success_proj, template_assertion_predicate); + invar.map_ctrl(if_success_proj, template_assertion_predicate); // Mark hoisted check as invariant #ifndef PRODUCT if (TraceLoopOpts && !TraceLoopPredicate) { @@ -1285,63 +1120,17 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod // with uncommon trap. return false; } - assert(new_predicate_proj != nullptr, "sanity"); - // Success - attach condition (new_predicate_bol) to predicate if - invar.map_ctrl(if_success_proj, new_predicate_proj); // so that invariance test can be appropriate - - // Eliminate the old If in the loop body - dominated_by(new_predicate_proj, iff, if_success_proj->_con != new_predicate_proj->_con); C->set_major_progress(); return true; } -// Each newly created Hoisted Check Predicate is accompanied by two Template Assertion Predicates. Later, we initialize -// them by making a copy of them when splitting a loop into sub loops. The Assertion Predicates ensure that dead sub -// loops are removed properly. -IfProjNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj, - ParsePredicateSuccessProj* parse_predicate_proj, - IfProjNode* upper_bound_proj, const int scale, Node* offset, - Node* init, Node* limit, const jint stride, - Node* rng, bool& overflow, Deoptimization::DeoptReason reason) { - // First predicate for the initial value on first loop iteration - Node* opaque_init = new OpaqueLoopInitNode(C, init); - register_new_node(opaque_init, upper_bound_proj); - bool negate = (if_proj->_con != parse_predicate_proj->_con); - BoolNode* bol = rc_predicate(loop, upper_bound_proj, scale, offset, opaque_init, limit, stride, rng, - (stride > 0) != (scale > 0), overflow); - Node* opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); // This will go away once loop opts are over - C->add_template_assertion_predicate_opaq(opaque_bol); - register_new_node(opaque_bol, upper_bound_proj); - IfProjNode* new_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode()); - _igvn.replace_input_of(new_proj->in(0), 1, opaque_bol); - assert(opaque_init->outcnt() > 0, "should be used"); - - // Second predicate for init + (current stride - initial stride) - // This is identical to the previous predicate initially but as - // unrolling proceeds current stride is updated. - Node* init_stride = loop->_head->as_CountedLoop()->stride(); - Node* opaque_stride = new OpaqueLoopStrideNode(C, init_stride); - register_new_node(opaque_stride, new_proj); - Node* max_value = new SubINode(opaque_stride, init_stride); - register_new_node(max_value, new_proj); - max_value = new AddINode(opaque_init, max_value); - register_new_node(max_value, new_proj); - // init + (current stride - initial stride) is within the loop so narrow its type by leveraging the type of the iv Phi - max_value = new CastIINode(max_value, loop->_head->as_CountedLoop()->phi()->bottom_type()); - register_new_node(max_value, parse_predicate_proj); - - bol = rc_predicate(loop, new_proj, scale, offset, max_value, limit, stride, rng, (stride > 0) != (scale > 0), - overflow); - opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); - C->add_template_assertion_predicate_opaq(opaque_bol); - register_new_node(opaque_bol, new_proj); - new_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode()); - _igvn.replace_input_of(new_proj->in(0), 1, opaque_bol); - assert(max_value->outcnt() > 0, "should be used"); - assert(assertion_predicate_has_loop_opaque_node(new_proj->in(0)->as_If()), "unexpected"); - - return new_proj; +void PhaseIdealLoop::eliminate_old_range_check(IfProjNode* if_proj, + TemplateAssertionPredicateNode* template_assertion_predicate) { + ConINode* true_con = _igvn.intcon(1); + set_ctrl(true_con, C->root()); + _igvn.replace_input_of(if_proj->in(0), 1, true_con); + rewire_safe_outputs_to_dominator(if_proj, template_assertion_predicate); } // Insert Hoisted Check Predicates for null checks and range checks and additional Template Assertion Predicates for @@ -1354,10 +1143,6 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree* loop) { return false; } - if (head->is_OuterStripMinedLoop()) { - return false; - } - CountedLoopNode *cl = nullptr; if (head->is_valid_counted_loop(T_INT)) { cl = head->as_CountedLoop(); @@ -1419,7 +1204,7 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree* loop) { bool hoisted = false; // true if at least one proj is promoted - if (can_create_loop_predicates(profiled_loop_predicate_block)) { + if (can_create_loop_predicates(predicates)) { while (if_proj_list.size() > 0) { Node* n = if_proj_list.pop(); @@ -1505,42 +1290,47 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree* loop) { return hoisted; } -// We cannot add Loop Predicates if: -// (1) Already added Profiled Loop Predicates (Loop Predicates and Profiled Loop Predicates can be dependent +// We cannot create Loop Predicates if: +// (1) There is no Loop Parse Predicate. +// (2) Already added Profiled Loop Predicates (Loop Predicates and Profiled Loop Predicates can be dependent // through a data node, and thus we should only add new Profiled Loop Predicates which are below Loop Predicates // in the graph). -// (2) There are currently no Profiled Loop Predicates, but we have a data node with a control dependency on the Loop +// (3) There are currently no Profiled Loop Predicates, but we have a data node with a control dependency on the Loop // Parse Predicate (could happen, for example, if we've removed an earlier created Profiled Loop Predicate with // dominated_by()). We should not create a Loop Predicate for a check that is dependent on this data node because // the Loop Predicate would end up above the data node with its dependency on the Loop Parse Predicate below. This // would become unschedulable. However, we can still hoist the check as Profiled Loop Predicate which would end up // below the Loop Parse Predicate. -bool PhaseIdealLoop::can_create_loop_predicates(const PredicateBlock* profiled_loop_predicate_block) const { - bool has_profiled_loop_predicate_block = profiled_loop_predicate_block != nullptr; - bool can_create_loop_predicates = true; - if (has_profiled_loop_predicate_block - && (profiled_loop_predicate_block->has_runtime_predicates() // (1) - || profiled_loop_predicate_block->entry()->outcnt() != 1)) { // (2) - can_create_loop_predicates = false; +bool PhaseIdealLoop::can_create_loop_predicates(const Predicates& predicates) const { + const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block(); + if (!loop_predicate_block->has_parse_predicate()) { // (1) + return false; } - return can_create_loop_predicates; + + const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block(); + if (profiled_loop_predicate_block->has_runtime_predicates() // (2) + || loop_predicate_block->parse_predicate_success_proj()->outcnt() != 1) {// (3) + return false; + } + return true; } //------------------------------loop_predication-------------------------------- // driver routine for loop predication optimization -bool IdealLoopTree::loop_predication( PhaseIdealLoop *phase) { +bool IdealLoopTree::loop_predication(PhaseIdealLoop* phase) { bool hoisted = false; // Recursively promote predicates if (_child) { hoisted = _child->loop_predication( phase); } - // self + // Self if (can_apply_loop_predication()) { hoisted |= phase->loop_predication_impl(this); } - if (_next) { //sibling + // Sibling + if (_next) { hoisted |= _next->loop_predication( phase); } @@ -1548,5 +1338,9 @@ bool IdealLoopTree::loop_predication( PhaseIdealLoop *phase) { } bool IdealLoopTree::can_apply_loop_predication() { - return _head->is_Loop() && !_irreducible && !tail()->is_top(); + return !_head->is_Root() && + _head->is_Loop() && + !_head->is_OuterStripMinedLoop() && + !_irreducible && + !tail()->is_top(); } diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp index 95023337ae303..b25a3a7066d72 100644 --- a/src/hotspot/share/opto/loopTransform.cpp +++ b/src/hotspot/share/opto/loopTransform.cpp @@ -722,7 +722,7 @@ void PhaseIdealLoop::do_peeling(IdealLoopTree *loop, Node_List &old_new) { // Step 1: Clone the loop body. The clone becomes the peeled iteration. // The pre-loop illegally has 2 control users (old & new loops). - const uint idx_before_clone = Compile::current()->unique(); + const uint first_peeled_loop_node_index = C->unique(); LoopNode* outer_loop_head = head->skip_strip_mined(); clone_loop(loop, old_new, dom_depth(outer_loop_head), ControlAroundStripMined); @@ -775,20 +775,9 @@ void PhaseIdealLoop::do_peeling(IdealLoopTree *loop, Node_List &old_new) { // Step 5: Assertion Predicates initialization if (counted_loop && UseLoopPredicate) { - CountedLoopNode *cl_head = head->as_CountedLoop(); - Node* init = cl_head->init_trip(); - Node* stride = cl_head->stride(); - IdealLoopTree* outer_loop = get_loop(outer_loop_head); - const Predicates predicates(new_head->in(LoopNode::EntryControl)); - initialize_assertion_predicates_for_peeled_loop(predicates.loop_predicate_block(), - outer_loop_head, dd_outer_loop_head, - init, stride, outer_loop, - idx_before_clone, old_new); - initialize_assertion_predicates_for_peeled_loop(predicates.profiled_loop_predicate_block(), - outer_loop_head, dd_outer_loop_head, - init, stride, outer_loop, - idx_before_clone, old_new); - } + move_assertion_predicates_from_cloned_loop(new_head->as_CountedLoop(), head->as_CountedLoop(), + first_peeled_loop_node_index, old_new); + } // Now force out all loop-invariant dominating tests. The optimizer // finds some, but we _know_ they are all useless. @@ -797,6 +786,39 @@ void PhaseIdealLoop::do_peeling(IdealLoopTree *loop, Node_List &old_new) { loop->record_for_igvn(); } +// Creates new Assertion Predicates at the 'target_loop_head' (cloned loop). A new Template Assertion Predicate is +// inserted with the new init and stride values of the target loop for each existing Template Assertion Predicate found +// at 'source_loop_head (original loop). For each new Template Assertion Predicate, an Initialized Assertion Predicate +// for the new init and stride value is created. +void PhaseIdealLoop::clone_assertion_predicates_from_original_loop(CountedLoopNode* source_loop_head, + CountedLoopNode* target_loop_head, + const uint first_cloned_loop_node_index) { + AssertionPredicates assertion_predicates(source_loop_head, this); + NodeInClonedLoop node_in_cloned_loop(first_cloned_loop_node_index); + assertion_predicates.clone_to_loop(target_loop_head, &node_in_cloned_loop); +} + +// Creates new Assertion Predicates at the 'target_loop_head' (original loop). A new Template Assertion Predicate is +// inserted with the new init and stride values of the target loop for each existing Template Assertion Predicate found +// at 'source_loop_head' (cloned loop). For each new Template Assertion Predicate, an init and last value Initialized +// Assertion Predicate is created. The existing Template Assertion Predicates at the source loop are removed. +void PhaseIdealLoop::move_assertion_predicates_from_cloned_loop(CountedLoopNode* source_loop_head, + CountedLoopNode* target_loop_head, + const uint first_cloned_loop_node_index, + Node_List& old_new) { + AssertionPredicates assertion_predicates(source_loop_head, this); + NodeInOriginalLoop node_in_original_loop(first_cloned_loop_node_index, &old_new); + assertion_predicates.move_to_loop(target_loop_head, &node_in_original_loop); +} + +// Update existing Assertion Predicates at the loop being unrolled denoted by 'loop_head' with the provided stride value. +// The templates are first updated and then new Initialized Assertion Predicates are created based on the updated +// templates. The previously existing Initialized Assertion are no longer needed and killed. +void PhaseIdealLoop::update_assertion_predicates_during_unroll(CountedLoopNode* loop_head) { + AssertionPredicates assertion_predicates(loop_head, this); + assertion_predicates.update(loop_head->stride_con() * 2); +} + //------------------------------policy_maximally_unroll------------------------ // Calculate the exact loop trip-count and return TRUE if loop can be fully, // i.e. maximally, unrolled, otherwise return FALSE. When TRUE, the estimated @@ -1159,7 +1181,7 @@ bool IdealLoopTree::policy_range_check(PhaseIdealLoop* phase, bool provisional, continue; // dead constant test } if (!bol->is_Bool()) { - assert(bol->Opcode() == Op_Conv2B, "predicate check only"); + assert(bol->Opcode() == Op_OpaqueAssertionPredicate, "Initialized Assertion Predicate check only"); continue; } if (bol->as_Bool()->_test._test == BoolTest::ne) { @@ -1292,216 +1314,6 @@ Node* PhaseIdealLoop::cast_incr_before_loop(Node* incr, Node* ctrl, Node* loop) return nullptr; } -#ifdef ASSERT -void PhaseIdealLoop::ensure_zero_trip_guard_proj(Node* node, bool is_main_loop) { - assert(node->is_IfProj(), "must be the zero trip guard If node"); - Node* zer_bol = node->in(0)->in(1); - assert(zer_bol != nullptr && zer_bol->is_Bool(), "must be Bool"); - Node* zer_cmp = zer_bol->in(1); - assert(zer_cmp != nullptr && zer_cmp->Opcode() == Op_CmpI, "must be CmpI"); - // For the main loop, the opaque node is the second input to zer_cmp, for the post loop it's the first input node - Node* zer_opaq = zer_cmp->in(is_main_loop ? 2 : 1); - assert(zer_opaq != nullptr && zer_opaq->Opcode() == Op_OpaqueZeroTripGuard, "must be OpaqueZeroTripGuard"); -} -#endif - -// Make two copies of each Template Assertion Predicate before the pre-loop and add them to the main-loop. One remains -// a template while the other one is initialized with the initial value of the loop induction variable. The Initialized -// Assertion Predicates ensures that the main-loop is removed if some type ranges of Cast or Convert nodes become -// impossible and are replaced by top (i.e. a sign that the main-loop is dead). -void PhaseIdealLoop::copy_assertion_predicates_to_main_loop_helper(const PredicateBlock* predicate_block, Node* init, - Node* stride, IdealLoopTree* outer_loop, - LoopNode* outer_main_head, const uint dd_main_head, - const uint idx_before_pre_post, - const uint idx_after_post_before_pre, - Node* zero_trip_guard_proj_main, - Node* zero_trip_guard_proj_post, - const Node_List &old_new) { - if (predicate_block->has_parse_predicate()) { -#ifdef ASSERT - ensure_zero_trip_guard_proj(zero_trip_guard_proj_main, true); - ensure_zero_trip_guard_proj(zero_trip_guard_proj_post, false); -#endif - Node* predicate_proj = predicate_block->parse_predicate_success_proj(); - IfNode* iff = predicate_proj->in(0)->as_If(); - ProjNode* uncommon_proj = iff->proj_out(1 - predicate_proj->as_Proj()->_con); - Node* rgn = uncommon_proj->unique_ctrl_out(); - assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct"); - predicate_proj = iff->in(0); - Node* current_proj = outer_main_head->in(LoopNode::EntryControl); - Node* prev_proj = current_proj; - Node* opaque_init = new OpaqueLoopInitNode(C, init); - register_new_node(opaque_init, outer_main_head->in(LoopNode::EntryControl)); - Node* opaque_stride = new OpaqueLoopStrideNode(C, stride); - register_new_node(opaque_stride, outer_main_head->in(LoopNode::EntryControl)); - - while (predicate_proj != nullptr && predicate_proj->is_Proj() && predicate_proj->in(0)->is_If()) { - iff = predicate_proj->in(0)->as_If(); - uncommon_proj = iff->proj_out(1 - predicate_proj->as_Proj()->_con); - if (uncommon_proj->unique_ctrl_out() != rgn) - break; - if (iff->in(1)->Opcode() == Op_Opaque4) { - assert(assertion_predicate_has_loop_opaque_node(iff), "unexpected"); - // Clone the Assertion Predicate twice and initialize one with the initial - // value of the loop induction variable. Leave the other predicate - // to be initialized when increasing the stride during loop unrolling. - prev_proj = clone_assertion_predicate_and_initialize(iff, opaque_init, nullptr, predicate_proj, uncommon_proj, - current_proj, outer_loop, prev_proj); - assert(assertion_predicate_has_loop_opaque_node(prev_proj->in(0)->as_If()), ""); - - prev_proj = clone_assertion_predicate_and_initialize(iff, init, stride, predicate_proj, uncommon_proj, - current_proj, outer_loop, prev_proj); - assert(!assertion_predicate_has_loop_opaque_node(prev_proj->in(0)->as_If()), ""); - - // Rewire any control inputs from the cloned Assertion Predicates down to the main and post loop for data nodes - // that are part of the main loop (and were cloned to the pre and post loop). - for (DUIterator i = predicate_proj->outs(); predicate_proj->has_out(i); i++) { - Node* loop_node = predicate_proj->out(i); - Node* pre_loop_node = old_new[loop_node->_idx]; - // Change the control if 'loop_node' is part of the main loop. If there is an old->new mapping and the index of - // 'pre_loop_node' is greater than idx_before_pre_post, then we know that 'loop_node' was cloned and is part of - // the main loop (and 'pre_loop_node' is part of the pre loop). - if (!loop_node->is_CFG() && (pre_loop_node != nullptr && pre_loop_node->_idx > idx_after_post_before_pre)) { - // 'loop_node' is a data node and part of the main loop. Rewire the control to the projection of the zero-trip guard if node - // of the main loop that is immediately preceding the cloned predicates. - _igvn.replace_input_of(loop_node, 0, zero_trip_guard_proj_main); - --i; - } else if (loop_node->_idx > idx_before_pre_post && loop_node->_idx < idx_after_post_before_pre) { - // 'loop_node' is a data node and part of the post loop. Rewire the control to the projection of the zero-trip guard if node - // of the post loop that is immediately preceding the post loop header node (there are no cloned predicates for the post loop). - assert(pre_loop_node == nullptr, "a node belonging to the post loop should not have an old_new mapping at this stage"); - _igvn.replace_input_of(loop_node, 0, zero_trip_guard_proj_post); - --i; - } - } - - // Remove the Assertion Predicate from the pre-loop - _igvn.replace_input_of(iff, 1, _igvn.intcon(1)); - } - predicate_proj = predicate_proj->in(0)->in(0); - } - _igvn.replace_input_of(outer_main_head, LoopNode::EntryControl, prev_proj); - set_idom(outer_main_head, prev_proj, dd_main_head); - } -} - -bool PhaseIdealLoop::assertion_predicate_has_loop_opaque_node(IfNode* iff) { - uint init; - uint stride; - count_opaque_loop_nodes(iff->in(1)->in(1), init, stride); -#ifdef ASSERT - ResourceMark rm; - Unique_Node_List wq; - wq.clear(); - wq.push(iff->in(1)->in(1)); - uint verif_init = 0; - uint verif_stride = 0; - for (uint i = 0; i < wq.size(); i++) { - Node* n = wq.at(i); - int op = n->Opcode(); - if (!n->is_CFG()) { - if (n->Opcode() == Op_OpaqueLoopInit) { - verif_init++; - } else if (n->Opcode() == Op_OpaqueLoopStride) { - verif_stride++; - } else { - for (uint j = 1; j < n->req(); j++) { - Node* m = n->in(j); - if (m != nullptr) { - wq.push(m); - } - } - } - } - } - assert(init == verif_init && stride == verif_stride, "missed opaque node"); -#endif - assert(stride == 0 || init != 0, "init should be there every time stride is"); - return init != 0; -} - -void PhaseIdealLoop::count_opaque_loop_nodes(Node* n, uint& init, uint& stride) { - init = 0; - stride = 0; - ResourceMark rm; - Unique_Node_List wq; - wq.push(n); - for (uint i = 0; i < wq.size(); i++) { - Node* n = wq.at(i); - if (AssertionPredicateBoolOpcodes::is_valid(n)) { - if (n->Opcode() == Op_OpaqueLoopInit) { - init++; - } else if (n->Opcode() == Op_OpaqueLoopStride) { - stride++; - } else { - for (uint j = 1; j < n->req(); j++) { - Node* m = n->in(j); - if (m != nullptr) { - wq.push(m); - } - } - } - } - } -} - -// Clone an Assertion Predicate for the main loop. new_init and new_stride are set as new inputs. Since the predicates -// cannot fail at runtime, Halt nodes are inserted instead of uncommon traps. -Node* PhaseIdealLoop::clone_assertion_predicate_and_initialize(Node* iff, Node* new_init, Node* new_stride, Node* predicate, - Node* uncommon_proj, Node* control, IdealLoopTree* outer_loop, - Node* input_proj) { - Node* opaque_node = iff->in(1); - TemplateAssertionPredicateBool template_assertion_predicate_bool(opaque_node->in(1)); - BoolNode* new_bool; - if (new_stride == nullptr) { - new_bool = template_assertion_predicate_bool.clone_and_replace_init(control, new_init, this); - } else { - new_bool = template_assertion_predicate_bool.clone_and_replace_opaque_loop_nodes(control, new_init, new_stride, this); - } - opaque_node = clone_and_register(opaque_node, control); - _igvn.replace_input_of(opaque_node, 1, new_bool); - - Node* proj = predicate->clone(); - Node* other_proj = uncommon_proj->clone(); - Node* new_iff = iff->clone(); - new_iff->set_req(1, opaque_node); - proj->set_req(0, new_iff); - other_proj->set_req(0, new_iff); - Node* frame = new ParmNode(C->start(), TypeFunc::FramePtr); - register_new_node(frame, C->start()); - // It's impossible for the predicate to fail at runtime. Use a Halt node. - Node* halt = new HaltNode(other_proj, frame, "duplicated predicate failed which is impossible"); - _igvn.add_input_to(C->root(), halt); - new_iff->set_req(0, input_proj); - - register_control(new_iff, outer_loop == _ltree_root ? _ltree_root : outer_loop->_parent, input_proj); - register_control(proj, outer_loop == _ltree_root ? _ltree_root : outer_loop->_parent, new_iff); - register_control(other_proj, _ltree_root, new_iff); - register_control(halt, _ltree_root, other_proj); - return proj; -} - -void PhaseIdealLoop::copy_assertion_predicates_to_main_loop(CountedLoopNode* pre_head, Node* init, Node* stride, - IdealLoopTree* outer_loop, LoopNode* outer_main_head, - const uint dd_main_head, const uint idx_before_pre_post, - const uint idx_after_post_before_pre, - Node* zero_trip_guard_proj_main, - Node* zero_trip_guard_proj_post, - const Node_List &old_new) { - if (UseLoopPredicate) { - Node* entry = pre_head->in(LoopNode::EntryControl); - const Predicates predicates(entry); - copy_assertion_predicates_to_main_loop_helper(predicates.loop_predicate_block(), init, stride, outer_loop, - outer_main_head, dd_main_head, idx_before_pre_post, - idx_after_post_before_pre, zero_trip_guard_proj_main, - zero_trip_guard_proj_post, old_new); - copy_assertion_predicates_to_main_loop_helper(predicates.profiled_loop_predicate_block(), init, stride, - outer_loop, outer_main_head, dd_main_head, idx_before_pre_post, - idx_after_post_before_pre, zero_trip_guard_proj_main, - zero_trip_guard_proj_post, old_new); - } -} - //------------------------------insert_pre_post_loops-------------------------- // Insert pre and post loops. If peel_only is set, the pre-loop can not have // more iterations added. It acts as a 'peel' only, no lower-bound RCE, no @@ -1550,9 +1362,16 @@ void PhaseIdealLoop::insert_pre_post_loops(IdealLoopTree *loop, Node_List &old_n // Add the post loop const uint idx_before_pre_post = Compile::current()->unique(); CountedLoopNode *post_head = nullptr; - Node* post_incr = incr; - Node* main_exit = insert_post_loop(loop, old_new, main_head, main_end, post_incr, limit, post_head); - const uint idx_after_post_before_pre = Compile::current()->unique(); + Node* main_exit = insert_post_loop(loop, old_new, main_head, main_end, post_head); +#ifdef ASSERT + // Should only find Assertion Predicates in post loop + Node* post_loop_entry = post_head->in(LoopNode::EntryControl); + VerifyOnlyAssertionPredicates::verify(post_loop_entry); + Predicates predicates(post_loop_entry); + IfNode* zero_trip_if = predicates.entry()->in(0)->as_If(); + assert(zero_trip_if->is_zero_trip_guard(), + "must be newly created zero-trip guard of post loop"); +#endif // ASSERT //------------------------------ // Step B: Create Pre-Loop. @@ -1567,6 +1386,8 @@ void PhaseIdealLoop::insert_pre_post_loops(IdealLoopTree *loop, Node_List &old_n outer_loop = loop->_parent; assert(outer_loop->_head == outer_main_head, "broken loop tree"); } + + const uint first_pre_loop_node_index = Compile::current()->unique(); uint dd_main_head = dom_depth(outer_main_head); clone_loop(loop, old_new, dd_main_head, ControlAroundStripMined); CountedLoopNode* pre_head = old_new[main_head->_idx]->as_CountedLoop(); @@ -1647,11 +1468,7 @@ void PhaseIdealLoop::insert_pre_post_loops(IdealLoopTree *loop, Node_List &old_n // CastII for the main loop: Node* castii = cast_incr_before_loop(pre_incr, min_taken, main_head); assert(castii != nullptr, "no castII inserted"); - assert(post_head->in(1)->is_IfProj(), "must be zero-trip guard If node projection of the post loop"); - copy_assertion_predicates_to_main_loop(pre_head, castii, stride, outer_loop, outer_main_head, dd_main_head, - idx_before_pre_post, idx_after_post_before_pre, min_taken, post_head->in(1), - old_new); - copy_assertion_predicates_to_post_loop(outer_main_head, post_head, post_incr, stride); + move_assertion_predicates_from_cloned_loop(pre_head, main_head, first_pre_loop_node_index, old_new); // Step B4: Shorten the pre-loop to run only 1 iteration (for now). // RCE and alignment may change this later. @@ -1768,13 +1585,9 @@ void PhaseIdealLoop::insert_vector_post_loop(IdealLoopTree *loop, Node_List &old // mark this loop as processed main_head->mark_has_atomic_post_loop(); - Node *incr = main_end->incr(); - Node *limit = main_end->limit(); - // In this case we throw away the result as we are not using it to connect anything else. CountedLoopNode *post_head = nullptr; - insert_post_loop(loop, old_new, main_head, main_end, incr, limit, post_head); - copy_assertion_predicates_to_post_loop(main_head->skip_strip_mined(), post_head, incr, main_head->stride()); + insert_post_loop(loop, old_new, main_head, main_end, post_head); // It's difficult to be precise about the trip-counts // for post loops. They are usually very short, @@ -1789,9 +1602,8 @@ void PhaseIdealLoop::insert_vector_post_loop(IdealLoopTree *loop, Node_List &old //------------------------------insert_post_loop------------------------------- // Insert post loops. Add a post loop to the given loop passed. -Node *PhaseIdealLoop::insert_post_loop(IdealLoopTree* loop, Node_List& old_new, - CountedLoopNode* main_head, CountedLoopEndNode* main_end, - Node*& incr, Node* limit, CountedLoopNode*& post_head) { +Node* PhaseIdealLoop::insert_post_loop(IdealLoopTree* loop, Node_List& old_new, CountedLoopNode* main_head, + CountedLoopEndNode* main_end, CountedLoopNode*& post_head) { IfNode* outer_main_end = main_end; IdealLoopTree* outer_loop = loop; if (main_head->is_strip_mined()) { @@ -1809,6 +1621,7 @@ Node *PhaseIdealLoop::insert_post_loop(IdealLoopTree* loop, Node_List& old_new, // Step A1: Clone the loop body of main. The clone becomes the post-loop. // The main loop pre-header illegally has 2 control users (old & new loops). + const uint first_post_loop_node_index = C->unique(); clone_loop(loop, old_new, dd_main_exit, ControlAroundStripMined); assert(old_new[main_end->_idx]->Opcode() == Op_CountedLoopEnd, ""); post_head = old_new[main_head->_idx]->as_CountedLoop(); @@ -1833,9 +1646,11 @@ Node *PhaseIdealLoop::insert_post_loop(IdealLoopTree* loop, Node_List& old_new, // (the previous loop trip-counter exit value) because we will be changing // the exit value (via additional unrolling) so we cannot constant-fold away the zero // trip guard until all unrolling is done. - Node *zer_opaq = new OpaqueZeroTripGuardNode(C, incr, main_end->test_trip()); - Node *zer_cmp = new CmpINode(zer_opaq, limit); - Node *zer_bol = new BoolNode(zer_cmp, main_end->test_trip()); + Node* incr = main_end->incr(); + Node* limit = main_end->limit(); + Node* zer_opaq = new OpaqueZeroTripGuardNode(C, incr, main_end->test_trip()); + Node* zer_cmp = new CmpINode(zer_opaq, limit); + Node* zer_bol = new BoolNode(zer_cmp, main_end->test_trip()); register_new_node(zer_opaq, new_main_exit); register_new_node(zer_cmp, new_main_exit); register_new_node(zer_bol, new_main_exit); @@ -1880,7 +1695,7 @@ Node *PhaseIdealLoop::insert_post_loop(IdealLoopTree* loop, Node_List& old_new, // CastII for the new post loop: incr = cast_incr_before_loop(zer_opaq->in(1), zer_taken, post_head); assert(incr != nullptr, "no castII inserted"); - + clone_assertion_predicates_from_original_loop(main_head, post_head, first_post_loop_node_index); return new_main_exit; } @@ -1892,131 +1707,6 @@ bool IdealLoopTree::is_invariant(Node* n) const { return !is_member(_phase->get_loop(n_c)); } -// Search the Assertion Predicates added by loop predication and/or range check elimination and update them according -// to the new stride. -void PhaseIdealLoop::update_main_loop_assertion_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, - const int stride_con) { - Node* entry = ctrl; - Node* prev_proj = ctrl; - LoopNode* outer_loop_head = loop_head->skip_strip_mined(); - IdealLoopTree* outer_loop = get_loop(outer_loop_head); - - // Compute the value of the loop induction variable at the end of the - // first iteration of the unrolled loop: init + new_stride_con - init_inc - int new_stride_con = stride_con * 2; - Node* max_value = _igvn.intcon(new_stride_con); - set_ctrl(max_value, C->root()); - - while (entry != nullptr && entry->is_Proj() && entry->in(0)->is_If()) { - IfNode* iff = entry->in(0)->as_If(); - ProjNode* proj = iff->proj_out(1 - entry->as_Proj()->_con); - if (proj->unique_ctrl_out()->Opcode() != Op_Halt) { - break; - } - if (iff->in(1)->Opcode() == Op_Opaque4) { - if (!assertion_predicate_has_loop_opaque_node(iff)) { - // No OpaqueLoop* node? Then it's one of the two Initialized Assertion Predicates: - // - For the initial access a[init] - // - For the last access a[init+old_stride-orig_stride] - // We could keep the one for the initial access but we do not know which one we currently have here. Just kill both. - // We will create new Initialized Assertion Predicates from the Template Assertion Predicates below: - // - For the initial access a[init] (same as before) - // - For the last access a[init+new_stride-orig_stride] (with the new unroll stride) - _igvn.replace_input_of(iff, 1, iff->in(1)->in(2)); - } else { - // Template Assertion Predicate: Clone it to create initialized version with new stride. - prev_proj = clone_assertion_predicate_and_initialize(iff, init, max_value, entry, proj, ctrl, outer_loop, - prev_proj); - assert(!assertion_predicate_has_loop_opaque_node(prev_proj->in(0)->as_If()), "unexpected"); - } - } - entry = entry->in(0)->in(0); - } - if (prev_proj != ctrl) { - _igvn.replace_input_of(outer_loop_head, LoopNode::EntryControl, prev_proj); - set_idom(outer_loop_head, prev_proj, dom_depth(outer_loop_head)); - } -} - -// Go over the Assertion Predicates of the main loop and make a copy for the post loop with its initial iv value and -// stride as inputs. -void PhaseIdealLoop::copy_assertion_predicates_to_post_loop(LoopNode* main_loop_head, CountedLoopNode* post_loop_head, - Node* init, Node* stride) { - Node* post_loop_entry = post_loop_head->in(LoopNode::EntryControl); - Node* main_loop_entry = main_loop_head->in(LoopNode::EntryControl); - IdealLoopTree* post_loop = get_loop(post_loop_head); - - Node* ctrl = main_loop_entry; - Node* prev_proj = post_loop_entry; - while (ctrl != nullptr && ctrl->is_Proj() && ctrl->in(0)->is_If()) { - IfNode* iff = ctrl->in(0)->as_If(); - ProjNode* proj = iff->proj_out(1 - ctrl->as_Proj()->_con); - if (proj->unique_ctrl_out()->Opcode() != Op_Halt) { - break; - } - if (iff->in(1)->Opcode() == Op_Opaque4 && assertion_predicate_has_loop_opaque_node(iff)) { - prev_proj = clone_assertion_predicate_and_initialize(iff, init, stride, ctrl, proj, post_loop_entry, - post_loop, prev_proj); - assert(!assertion_predicate_has_loop_opaque_node(prev_proj->in(0)->as_If()), "unexpected"); - } - ctrl = ctrl->in(0)->in(0); - } - if (prev_proj != post_loop_entry) { - _igvn.replace_input_of(post_loop_head, LoopNode::EntryControl, prev_proj); - set_idom(post_loop_head, prev_proj, dom_depth(post_loop_head)); - } -} - -void PhaseIdealLoop::initialize_assertion_predicates_for_peeled_loop(const PredicateBlock* predicate_block, - LoopNode* outer_loop_head, - const int dd_outer_loop_head, Node* init, - Node* stride, IdealLoopTree* outer_loop, - const uint idx_before_clone, - const Node_List &old_new) { - if (!predicate_block->has_parse_predicate()) { - return; - } - Node* control = outer_loop_head->in(LoopNode::EntryControl); - Node* input_proj = control; - - const Node* parse_predicate_uncommon_trap = predicate_block->parse_predicate()->uncommon_trap(); - Node* next_regular_predicate_proj = predicate_block->skip_parse_predicate(); - while (next_regular_predicate_proj->is_IfProj()) { - IfNode* iff = next_regular_predicate_proj->in(0)->as_If(); - ProjNode* uncommon_proj = iff->proj_out(1 - next_regular_predicate_proj->as_Proj()->_con); - if (uncommon_proj->unique_ctrl_out() != parse_predicate_uncommon_trap) { - // Does not belong to this Predicate Block anymore. - break; - } - if (iff->in(1)->Opcode() == Op_Opaque4) { - assert(assertion_predicate_has_loop_opaque_node(iff), "unexpected"); - input_proj = clone_assertion_predicate_and_initialize(iff, init, stride, next_regular_predicate_proj, uncommon_proj, control, - outer_loop, input_proj); - - // Rewire any control inputs from the old Assertion Predicates above the peeled iteration down to the initialized - // Assertion Predicates above the peeled loop. - for (DUIterator i = next_regular_predicate_proj->outs(); next_regular_predicate_proj->has_out(i); i++) { - Node* dependent = next_regular_predicate_proj->out(i); - Node* new_node = old_new[dependent->_idx]; - - if (!dependent->is_CFG() && - dependent->_idx < idx_before_clone && // old node - new_node != nullptr && // cloned - new_node->_idx >= idx_before_clone) { // for peeling - // The old nodes from the peeled loop still point to the predicate above the peeled loop. - // We need to rewire the dependencies to the newly Initialized Assertion Predicates. - _igvn.replace_input_of(dependent, 0, input_proj); - --i; // correct for just deleted predicate->out(i) - } - } - } - next_regular_predicate_proj = iff->in(0); - } - - _igvn.replace_input_of(outer_loop_head, LoopNode::EntryControl, input_proj); - set_idom(outer_loop_head, input_proj, dd_outer_loop_head); -} - //------------------------------do_unroll-------------------------------------- // Unroll the loop body one step - make each trip do 2 iterations. void PhaseIdealLoop::do_unroll(IdealLoopTree *loop, Node_List &old_new, bool adjust_min_trip) { @@ -2080,7 +1770,7 @@ void PhaseIdealLoop::do_unroll(IdealLoopTree *loop, Node_List &old_new, bool adj assert(old_trip_count > 1 && (!adjust_min_trip || stride_p <= MIN2(max_jint / 2 - 2, MAX2(1<<3, Matcher::max_vector_size(T_BYTE)) * loop_head->unrolled_count())), "sanity"); - update_main_loop_assertion_predicates(ctrl, loop_head, init, stride_con); + update_assertion_predicates_during_unroll(loop_head); // Adjust loop limit to keep valid iterations number after unroll. // Use (limit - stride) instead of (((limit - init)/stride) & (-2))*stride @@ -2705,36 +2395,6 @@ bool PhaseIdealLoop::is_scaled_iv_plus_extra_offset(Node* exp1, Node* offset3, N return false; } -// Same as PhaseIdealLoop::duplicate_predicates() but for range checks -// eliminated by iteration splitting. -Node* PhaseIdealLoop::add_range_check_elimination_assertion_predicate(IdealLoopTree* loop, - Node* ctrl, const int scale_con, - Node* offset, Node* limit, jint stride_con, - Node* value) { - bool overflow = false; - BoolNode* bol = rc_predicate(loop, ctrl, scale_con, offset, value, nullptr, stride_con, - limit, (stride_con > 0) != (scale_con > 0), overflow); - Node* opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); - register_new_node(opaque_bol, ctrl); - IfNode* new_iff = nullptr; - if (overflow) { - new_iff = new IfNode(ctrl, opaque_bol, PROB_MAX, COUNT_UNKNOWN); - } else { - new_iff = new RangeCheckNode(ctrl, opaque_bol, PROB_MAX, COUNT_UNKNOWN); - } - register_control(new_iff, loop->_parent, ctrl); - Node* iffalse = new IfFalseNode(new_iff); - register_control(iffalse, _ltree_root, new_iff); - ProjNode* iftrue = new IfTrueNode(new_iff); - register_control(iftrue, loop->_parent, new_iff); - Node *frame = new ParmNode(C->start(), TypeFunc::FramePtr); - register_new_node(frame, C->start()); - Node* halt = new HaltNode(iffalse, frame, "range check predicate failed which is impossible"); - register_control(halt, _ltree_root, iffalse); - _igvn.add_input_to(C->root(), halt); - return iftrue; -} - //------------------------------do_range_check--------------------------------- // Eliminate range-checks and other trip-counter vs loop-invariant tests. void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) { @@ -2769,7 +2429,7 @@ void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) { } // Need to find the main-loop zero-trip guard - Node *ctrl = cl->skip_assertion_predicates_with_halt(); + Node *ctrl = cl->skip_assertion_predicates(); Node *iffm = ctrl->in(0); Node *opqzm = iffm->in(1)->in(1)->in(2); assert(opqzm->in(1) == main_limit, "do not understand situation"); @@ -2814,7 +2474,8 @@ void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) { set_ctrl(mini, C->root()); Node* loop_entry = cl->skip_strip_mined()->in(LoopNode::EntryControl); - assert(loop_entry->is_Proj() && loop_entry->in(0)->is_If(), "if projection only"); + assert((loop_entry->is_Proj() && loop_entry->in(0)->is_If()) || loop_entry->is_TemplateAssertionPredicate(), + "if projection or Template Assertion Predicate"); // Check loop body for tests of trip-counter plus loop-invariant vs loop-variant. for (uint i = 0; i < loop->_body.size(); i++) { @@ -2913,34 +2574,7 @@ void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) { if (b_test._test == BoolTest::lt) { // Range checks always use lt // The underflow and overflow limits: 0 <= scale*I+offset < limit add_constraint(stride_con, lscale_con, offset, zero, limit, pre_ctrl, &pre_limit, &main_limit); - Node* init = cl->init_trip(); - Node* opaque_init = new OpaqueLoopInitNode(C, init); - register_new_node(opaque_init, loop_entry); - - // Initialized Assertion Predicate for the value of the initial main-loop. - loop_entry = add_range_check_elimination_assertion_predicate(loop, loop_entry, scale_con, int_offset, - int_limit, stride_con, init); - assert(!assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected"); - - // Add two Template Assertion Predicates to create new Initialized Assertion Predicates from when either - // unrolling or splitting this main-loop further. - loop_entry = add_range_check_elimination_assertion_predicate(loop, loop_entry, scale_con, int_offset, - int_limit, stride_con, opaque_init); - assert(assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected"); - - Node* opaque_stride = new OpaqueLoopStrideNode(C, cl->stride()); - register_new_node(opaque_stride, loop_entry); - Node* max_value = new SubINode(opaque_stride, cl->stride()); - register_new_node(max_value, loop_entry); - max_value = new AddINode(opaque_init, max_value); - register_new_node(max_value, loop_entry); - // init + (current stride - initial stride) is within the loop so narrow its type by leveraging the type of the iv Phi - max_value = new CastIINode(max_value, loop->_head->as_CountedLoop()->phi()->bottom_type()); - register_new_node(max_value, loop_entry); - loop_entry = add_range_check_elimination_assertion_predicate(loop, loop_entry, scale_con, int_offset, - int_limit, stride_con, max_value); - assert(assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected"); - + loop_entry = add_range_check_elimination_assertion_predicates(loop, scale_con, int_offset, int_limit); } else { if (PrintOpto) { tty->print_cr("missed RCE opportunity"); @@ -3051,6 +2685,18 @@ void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) { return; } +TemplateAssertionPredicateNode* PhaseIdealLoop::add_range_check_elimination_assertion_predicates( + IdealLoopTree* loop, int scale, Node* offset, Node* range) { + CountedLoopNode* loop_head = loop->_head->as_CountedLoop(); + DEBUG_ONLY(uint node_index_before = C->unique()); + AssertionPredicates assertion_predicates(loop_head, this); + assertion_predicates.create(Op_RangeCheck, scale, offset, range); + TemplateAssertionPredicateNode* template_assertion_predicate = + loop_head->skip_strip_mined()->in(LoopNode::EntryControl)->as_TemplateAssertionPredicate(); + assert(node_index_before <= template_assertion_predicate->_idx, "must be the newly created one"); + return template_assertion_predicate; +} + bool IdealLoopTree::compute_has_range_checks() const { assert(_head->is_CountedLoop(), ""); for (uint i = 0; i < _body.size(); i++) { @@ -3137,7 +2783,7 @@ void IdealLoopTree::adjust_loop_exit_prob(PhaseIdealLoop *phase) { #ifdef ASSERT static CountedLoopNode* locate_pre_from_main(CountedLoopNode* main_loop) { assert(!main_loop->is_main_no_pre_loop(), "Does not have a pre loop"); - Node* ctrl = main_loop->skip_assertion_predicates_with_halt(); + Node* ctrl = main_loop->skip_assertion_predicates(); assert(ctrl->Opcode() == Op_IfTrue || ctrl->Opcode() == Op_IfFalse, ""); Node* iffm = ctrl->in(0); assert(iffm->Opcode() == Op_If, ""); @@ -3176,7 +2822,7 @@ void IdealLoopTree::remove_main_post_loops(CountedLoopNode *cl, PhaseIdealLoop * } assert(locate_pre_from_main(main_head) == cl, "bad main loop"); - Node* main_iff = main_head->skip_assertion_predicates_with_halt()->in(0); + Node* main_iff = main_head->skip_assertion_predicates()->in(0); // Remove the Opaque1Node of the pre loop and make it execute all iterations phase->_igvn.replace_input_of(pre_cmp, 2, pre_cmp->in(2)->in(2)); diff --git a/src/hotspot/share/opto/loopUnswitch.cpp b/src/hotspot/share/opto/loopUnswitch.cpp index fde1d2ad3526d..5e0651e82d94c 100644 --- a/src/hotspot/share/opto/loopUnswitch.cpp +++ b/src/hotspot/share/opto/loopUnswitch.cpp @@ -200,6 +200,39 @@ void PhaseIdealLoop::do_unswitching(IdealLoopTree* loop, Node_List& old_new) { C->set_major_progress(); } +// Class to create a new Parse Predicate for the fast loop. +class NewFastLoopParsePredicate : public NewParsePredicate { + public: + ParsePredicateSuccessProj* create(PhaseIdealLoop* phase, Node* new_entry, + ParsePredicateSuccessProj* old_parse_predicate_success_proj) override { + ParsePredicateNode* parse_predicate = old_parse_predicate_success_proj->in(0)->as_ParsePredicate(); +#ifndef PRODUCT + if (TraceLoopPredicate) { + tty->print("Created %d ParsePredicated for fast loop", parse_predicate->_idx); + parse_predicate->dump(); + } +#endif + return phase->create_new_if_for_predicate(old_parse_predicate_success_proj, new_entry, parse_predicate->deopt_reason(), + Op_ParsePredicate, false); + } +}; + +// Class to create a new Parse Predicate for the slow loop. +class NewSlowLoopParsePredicate : public NewParsePredicate { + public: + ParsePredicateSuccessProj* create(PhaseIdealLoop* phase, Node* new_entry, + ParsePredicateSuccessProj* old_parse_predicate_success_proj) override { + ParsePredicateNode* parse_predicate = old_parse_predicate_success_proj->in(0)->as_ParsePredicate(); +#ifndef PRODUCT + if (TraceLoopPredicate) { + tty->print("Created %d ParsePredicated for slow loop", parse_predicate->_idx); + parse_predicate->dump(); + } +#endif + return phase->create_new_if_for_predicate(old_parse_predicate_success_proj, new_entry, parse_predicate->deopt_reason(), + Op_ParsePredicate, true); + } +}; // Class to create an If node that selects if the fast or the slow loop should be executed at runtime. class UnswitchedLoopSelector { @@ -268,6 +301,173 @@ class UnswitchedLoopSelector { } }; +// Class to either represent the fast or the slow loop. +class UnswitchedLoop : public StackObj { + Node* _entry; + NewParsePredicate* _new_parse_predicate; + TemplateAssertionPredicateDataOutput* _node_in_target_loop; + PhaseIdealLoop* _phase; + PredicateInserter _predicate_inserter; + + public: + UnswitchedLoop(IfProjNode* unswitch_if_proj, LoopNode* unswitched_loop_head, NewParsePredicate* new_parse_predicate, + TemplateAssertionPredicateDataOutput* node_in_target_loop, PhaseIdealLoop* phase) + : _entry(unswitch_if_proj), + _new_parse_predicate(new_parse_predicate), + _node_in_target_loop(node_in_target_loop), + _phase(phase), + _predicate_inserter(unswitched_loop_head, phase) {} + + // Clone the Template Assertion Predicate to this loop. + void clone_to(TemplateAssertionPredicate& template_assertion_predicate) { + TemplateAssertionPredicate cloned_template = + template_assertion_predicate.clone(_entry, _node_in_target_loop, _phase); + _predicate_inserter.insert(cloned_template); + } + + // Clone the Parse Predicate to this loop. + void clone_to(ParsePredicate& parse_predicate) { + ParsePredicate cloned_parse_predicate = parse_predicate.clone(_entry, _new_parse_predicate, _phase); + _predicate_inserter.insert(cloned_parse_predicate); + } + +}; + +// This class represents the slow loop. +class SlowLoop : public StackObj { + NewSlowLoopParsePredicate _new_parse_predicate; + NodeInClonedLoop _node_in_slow_loop; + UnswitchedLoop _unswitched_loop; + + public: + SlowLoop(IfFalseNode* unswitch_if_proj, LoopNode* slow_loop_head, uint first_slow_loop_index, PhaseIdealLoop* phase) + : _new_parse_predicate(), + _node_in_slow_loop(first_slow_loop_index), + _unswitched_loop(unswitch_if_proj, slow_loop_head, &_new_parse_predicate, &_node_in_slow_loop, phase) {} + + void clone_to(TemplateAssertionPredicate& template_assertion_predicate) { + _unswitched_loop.clone_to(template_assertion_predicate); + } + + void clone_to(ParsePredicate& parse_predicate) { + _unswitched_loop.clone_to(parse_predicate); + } +}; + +// This class represents the fast loop. +class FastLoop : public StackObj { + NewFastLoopParsePredicate _new_parse_predicate; + NodeInOriginalLoop _node_in_fast_loop; + UnswitchedLoop _unswitched_loop; + PhaseIterGVN* _igvn; + + public: + FastLoop(IfTrueNode* unswitch_if_proj, LoopNode* fast_loop_head, uint first_slow_loop_index, Node_List* old_new, + PhaseIdealLoop* phase) + : _new_parse_predicate(), + _node_in_fast_loop(first_slow_loop_index, old_new), + _unswitched_loop(unswitch_if_proj, fast_loop_head, &_new_parse_predicate, &_node_in_fast_loop, phase), + _igvn(&phase->igvn()) {} + + void clone_to(TemplateAssertionPredicate& template_assertion_predicate) { + _unswitched_loop.clone_to(template_assertion_predicate); + template_assertion_predicate.kill(_igvn); + } + + void clone_to(ParsePredicate& parse_predicate) { + _unswitched_loop.clone_to(parse_predicate); + parse_predicate.kill(_igvn); + } +}; + +// This visitor visits the Parse Predicates and Template Assertion Predicates of the original not-yet-unswitched loop. +// A Template Assertion Predicate is always copied while Parse Predicates are sometimes skipped. +class ClonePredicates : public PredicateVisitor { + FastLoop _fast_loop; + SlowLoop _slow_loop; + bool _can_clone_parse_predicates; + + // Check if the original loop entry has any outside the loop body non-CFG dependencies. If that is the case, we cannot + // clone the Parse Predicates for the following reason: + // + // When peeling or partial peeling a loop, we could have non-CFG nodes N being pinned on some CFG nodes in the peeled + // section. If all CFG nodes in the peeled section are folded in the next IGVN phase, then the pinned non-CFG nodes N + // end up at the loop entry of the original loop. If we still find Parse Predicates at that point (i.e. not decided to + // remove all Parse Predicates and give up on further Loop Predication, yet), then these Parse Predicates can be cloned + // to the fast and slow loop when unswitching this loop at some point: + // + // Some CFG Node Some CFG node + // | Some CFG node / | \ + // Parse Predicates IGVN | unswitch N1 Unswitch If N2 + // | ===> Parse Predicates ===> / \ + // peeled CFG node / | \ Parse Predicates Parse Predicates + // / | \ N1 loop N2 | | + // N1 loop N2 fast loop slow loop + // + // This allows some more checks to be hoisted out of the fast and slow loop (i.e. creating Hoisted Check Predicates + // between the loop head and the Unswitch If). If one of these new Hoisted Check Predicates fail at runtime, we + // deoptimize and jump to the start of the loop in the interpreter, assuming that no statement was executed in the loop + // body, yet. However, the pinned statements N (N1, N2 in the figure above) could have already been executed with + // potentially visible side effects (i.e. storing a field). This is wrong. To fix that, we must move these pinned + // non-CFG nodes BELOW the Hoisted Check Predicates of the unswitched loops. This is done for nodes being part of the + // fast and slow loop body (i.e. part of the original loop to be unswitched). But we could also have non-CFG nodes that + // are not part of the loop body (i.e. only part of the originally peeled section). To force such a node to be executed + // after the newly created Hoisted Check Predicates of the fast/slow loop, we would need to clone it such that we can + // move the original node to the fast loop and the clone to the slow loop (or vice versa).to the slow and the fast loop. + // This, however, is not supported (yet). Therefore, we cannot clone the Parse Predicates to the fast and slow loop in + // this case. + static bool has_loop_entry_no_outside_loop_dependencies(Node* original_loop_entry, uint first_slow_loop_node_index) { + const uint entry_outcnt = original_loop_entry->outcnt(); + assert(entry_outcnt >= 1, "must have at least unswitch If, fast, and slow loop head after cloning the loop"); + if (entry_outcnt > 1) { + // For each data out node: + // Check if there is a slow node <-> fast node mapping (i.e. the node was part of the original loop to be unswitched) + // If we find a node without such a mapping, then it was not part of the original loop to be unswitched. + // In this case, we found a data dependency which disallows the cloning of Parse Predicates. + // Instead of actually mapping nodes, we can just count the number of slow loop nodes (which can only be mapped + // to a unique fast loop node), including the loop header nodes and multiply it by 2. This should be equal to the + // number of out nodes of 'original_entry' minus one for the unswitch If. + const uint slow_loop_node_count = count_slow_loop_nodes(original_loop_entry, first_slow_loop_node_index); + return slow_loop_node_count * 2 == entry_outcnt - 1; + } + return true; + } + + static uint count_slow_loop_nodes(Node* original_loop_entry, uint first_slow_loop_node_index) { + uint slow_loop_node_count = 0; + for (DUIterator_Fast imax, i = original_loop_entry->fast_outs(imax); i < imax; i++) { + Node* out = original_loop_entry->fast_out(i); + if (out->_idx >= first_slow_loop_node_index) { + slow_loop_node_count++; + } + } + return slow_loop_node_count; + } + + public: + using PredicateVisitor::visit; + + ClonePredicates(UnswitchedLoopSelector& unswitched_loop_selector, LoopNode* fast_loop_head, + uint first_slow_loop_index, Node_List* old_new, PhaseIdealLoop* phase) + : _fast_loop(unswitched_loop_selector.fast_loop_proj(), fast_loop_head, first_slow_loop_index, old_new, phase), + _slow_loop(unswitched_loop_selector.slow_loop_proj(), old_new->at(fast_loop_head->_idx)->as_Loop(), + first_slow_loop_index, phase), + _can_clone_parse_predicates(has_loop_entry_no_outside_loop_dependencies( + unswitched_loop_selector.entry(), first_slow_loop_index)) {} + + void visit(TemplateAssertionPredicate& template_assertion_predicate) override { + _slow_loop.clone_to(template_assertion_predicate); + _fast_loop.clone_to(template_assertion_predicate); + } + + void visit(ParsePredicate& parse_predicate) override { + if (_can_clone_parse_predicates) { + _slow_loop.clone_to(parse_predicate); + _fast_loop.clone_to(parse_predicate); + } + } +}; + // Class to unswitch a loop and create predicates at the new loops. The newly cloned loop becomes the slow loop while // the original loop becomes the fast loop. class OriginalLoop { @@ -293,21 +493,20 @@ class OriginalLoop { const uint first_slow_loop_node_index = _phase->C->unique(); _phase->clone_loop(_loop, *_old_new, _phase->dom_depth(_loop_head), PhaseIdealLoop::CloneIncludesStripMined, loop_selector_if); - // Fast (true) and Slow (false) control - IfProjNode* iffast_pred = unswitched_loop_selector.fast_loop_proj(); - IfProjNode* ifslow_pred = unswitched_loop_selector.slow_loop_proj(); - _phase->clone_parse_and_assertion_predicates_to_unswitched_loop(_loop, *_old_new, iffast_pred, ifslow_pred); - - fix_loop_entries(iffast_pred, ifslow_pred); + fix_loop_entries(unswitched_loop_selector); + ClonePredicates clone_predicates(unswitched_loop_selector, _strip_mined_loop_head, first_slow_loop_node_index, + _old_new, _phase); + PredicatesForLoop predicates_for_loop(unswitched_loop_selector.entry(), &clone_predicates); + predicates_for_loop.for_each(); DEBUG_ONLY(verify_unswitched_loops(_loop_head, unswitched_loop_selector, _old_new);) return loop_selector_if; } - void fix_loop_entries(IfProjNode* iffast_pred, IfProjNode* ifslow_pred) { - _phase->replace_loop_entry(_strip_mined_loop_head, iffast_pred); + void fix_loop_entries(const UnswitchedLoopSelector& unswitched_loop_selector) { + _phase->replace_loop_entry(_strip_mined_loop_head, unswitched_loop_selector.fast_loop_proj()); LoopNode* slow_loop_strip_mined_head = _old_new->at(_strip_mined_loop_head->_idx)->as_Loop(); - _phase->replace_loop_entry(slow_loop_strip_mined_head, ifslow_pred); + _phase->replace_loop_entry(slow_loop_strip_mined_head, unswitched_loop_selector.slow_loop_proj()); } #ifdef ASSERT diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index e9655b329e162..22008442dfa7e 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -170,64 +170,68 @@ Node *PhaseIdealLoop::get_early_ctrl_for_expensive(Node *n, Node* earliest) { } while (1) { - Node *next = ctl; + Node* next = ctl; // Moving the node out of a loop on the projection of a If // confuses loop predication. So once we hit a Loop in a If branch // that doesn't branch to an UNC, we stop. The code that process // expensive nodes will notice the loop and skip over it to try to // move the node further up. - if (ctl->is_CountedLoop() && ctl->in(1) != nullptr && ctl->in(1)->in(0) != nullptr && ctl->in(1)->in(0)->is_If()) { - if (!ctl->in(1)->as_Proj()->is_uncommon_trap_if_pattern()) { - break; - } - next = idom(ctl->in(1)->in(0)); - } else if (ctl->is_Proj()) { - // We only move it up along a projection if the projection is - // the single control projection for its parent: same code path, - // if it's a If with UNC or fallthrough of a call. - Node* parent_ctl = ctl->in(0); - if (parent_ctl == nullptr) { - break; - } else if (parent_ctl->is_CountedLoopEnd() && parent_ctl->as_CountedLoopEnd()->loopnode() != nullptr) { - next = parent_ctl->as_CountedLoopEnd()->loopnode()->init_control(); - } else if (parent_ctl->is_If()) { - if (!ctl->as_Proj()->is_uncommon_trap_if_pattern()) { - break; - } - assert(idom(ctl) == parent_ctl, "strange"); - next = idom(parent_ctl); - } else if (ctl->is_CatchProj()) { - if (ctl->as_Proj()->_con != CatchProjNode::fall_through_index) { - break; - } - assert(parent_ctl->in(0)->in(0)->is_Call(), "strange graph"); - next = parent_ctl->in(0)->in(0)->in(0); - } else { - // Check if parent control has a single projection (this - // control is the only possible successor of the parent - // control). If so, we can try to move the node above the - // parent control. - int nb_ctl_proj = 0; - for (DUIterator_Fast imax, i = parent_ctl->fast_outs(imax); i < imax; i++) { - Node *p = parent_ctl->fast_out(i); - if (p->is_Proj() && p->is_CFG()) { - nb_ctl_proj++; - if (nb_ctl_proj > 1) { - break; + + PredicateEntryIterator predicate_iterator(ctl->is_CountedLoop() ? ctl->in(LoopNode::EntryControl) : ctl); + if (predicate_iterator.has_next()) { + // Skip all predicates to not confuse Loop Predication. + do { + next = predicate_iterator.next_predicate_entry(); + } while (predicate_iterator.has_next()); + assert(is_dominator(next, ctl), "must be dominating after skipping predicates"); + } else if (ctl->is_CountedLoop()) { + // Loop without predicates. + assert(idom(ctl) == ctl->in(LoopNode::EntryControl), "entry to loop must be dominator"); + next = ctl->in(LoopNode::EntryControl); + } else { + if (ctl->is_Proj()) { + // We only move it up along a projection if the projection is + // the single control projection for its parent: same code path, + // if it's a If with UNC or fallthrough of a call. + Node* parent_ctl = ctl->in(0); + if (parent_ctl->is_CountedLoopEnd() && parent_ctl->as_CountedLoopEnd()->loopnode() != nullptr) { + next = parent_ctl->as_CountedLoopEnd()->loopnode()->init_control(); + } else if (parent_ctl->is_If()) { + assert(idom(ctl) == parent_ctl, "strange"); + next = idom(parent_ctl); + } else if (ctl->is_CatchProj()) { + if (ctl->as_Proj()->_con != CatchProjNode::fall_through_index) { + break; + } + assert(parent_ctl->in(0)->in(0)->is_Call(), "strange graph"); + next = parent_ctl->in(0)->in(0)->in(0); + } else { + // Check if parent control has a single projection (this + // control is the only possible successor of the parent + // control). If so, we can try to move the node above the + // parent control. + int nb_ctl_proj = 0; + for (DUIterator_Fast imax, i = parent_ctl->fast_outs(imax); i < imax; i++) { + Node* p = parent_ctl->fast_out(i); + if (p->is_Proj() && p->is_CFG()) { + nb_ctl_proj++; + if (nb_ctl_proj > 1) { + break; + } } } - } - if (nb_ctl_proj > 1) { - break; + if (nb_ctl_proj > 1) { + break; + } + assert(parent_ctl->is_Start() || parent_ctl->is_MemBar() || parent_ctl->is_Call() || + BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(parent_ctl), "unexpected node"); + assert(idom(ctl) == parent_ctl, "strange"); + next = idom(parent_ctl); } - assert(parent_ctl->is_Start() || parent_ctl->is_MemBar() || parent_ctl->is_Call() || - BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(parent_ctl), "unexpected node"); - assert(idom(ctl) == parent_ctl, "strange"); - next = idom(parent_ctl); + } else { + next = idom(ctl); } - } else { - next = idom(ctl); } if (next->is_Root() || next->is_Start() || dom_depth(next) < min_dom_depth) { break; @@ -1071,9 +1075,9 @@ bool PhaseIdealLoop::create_loop_nest(IdealLoopTree* loop, Node_List &old_new) { if (UseLoopPredicate) { add_parse_predicate(Deoptimization::Reason_predicate, inner_head, outer_ilt, cloned_sfpt); - } - if (UseProfiledLoopPredicate) { - add_parse_predicate(Deoptimization::Reason_profile_predicate, inner_head, outer_ilt, cloned_sfpt); + if (UseProfiledLoopPredicate) { + add_parse_predicate(Deoptimization::Reason_profile_predicate, inner_head, outer_ilt, cloned_sfpt); + } } add_parse_predicate(Deoptimization::Reason_loop_limit_check, inner_head, outer_ilt, cloned_sfpt); } @@ -1802,6 +1806,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ Node *init_control = x->in(LoopNode::EntryControl); + const PredicateBlock loop_limit_check_predicate_block(init_control, Deoptimization::Reason_loop_limit_check); int sov = check_stride_overflow(stride_m, limit_t, iv_bt); // If sov==0, limit's type always satisfies the condition, for // example, when it is an array length. @@ -1812,9 +1817,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ assert(!x->as_Loop()->is_loop_nest_inner_loop(), "loop was transformed"); // Generate loop's limit check. // Loop limit check predicate should be near the loop. - const Predicates predicates(init_control); - const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); - if (!loop_limit_check_predicate_block->has_parse_predicate()) { + if (!loop_limit_check_predicate_block.has_parse_predicate()) { // The limit check predicate is not generated if this method trapped here before. #ifdef ASSERT if (TraceLoopLimitCheck) { @@ -1826,8 +1829,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ return false; } - ParsePredicateNode* loop_limit_check_parse_predicate = loop_limit_check_predicate_block->parse_predicate(); - if (!is_dominator(get_ctrl(limit), loop_limit_check_parse_predicate->in(0))) { + if (!is_dominator(get_ctrl(limit), loop_limit_check_predicate_block.entry())) { return false; } @@ -1855,9 +1857,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ // 'ne' can be replaced with 'gt' only when init > limit. bt = BoolTest::gt; } else { - const Predicates predicates(init_control); - const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); - if (!loop_limit_check_predicate_block->has_parse_predicate()) { + if (!loop_limit_check_predicate_block.has_parse_predicate()) { // The limit check predicate is not generated if this method trapped here before. #ifdef ASSERT if (TraceLoopLimitCheck) { @@ -1869,10 +1869,8 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ return false; } - ParsePredicateNode* loop_limit_check_parse_predicate = loop_limit_check_predicate_block->parse_predicate(); - Node* parse_predicate_entry = loop_limit_check_parse_predicate->in(0); - if (!is_dominator(get_ctrl(limit), parse_predicate_entry) || - !is_dominator(get_ctrl(init_trip), parse_predicate_entry)) { + if (!is_dominator(get_ctrl(limit), loop_limit_check_predicate_block.entry()) || + !is_dominator(get_ctrl(init_trip), loop_limit_check_predicate_block.entry())) { return false; } @@ -2260,11 +2258,37 @@ void LoopNode::verify_strip_mined(int expect_skeleton) const { } #endif +// The visitor visits all Template Assertion Predicates and kills them by marking them useless. They will be removed +// during next round of IGVN. +class KillTemplateAssertionPredicates : public PredicateVisitor { + PhaseIterGVN* _igvn; + public: + using PredicateVisitor::visit; + + KillTemplateAssertionPredicates(PhaseIterGVN* igvn) : _igvn(igvn) {} + + void visit(TemplateAssertionPredicate& template_assertion_predicate) override { + template_assertion_predicate.kill(_igvn); + } +}; + //============================================================================= //------------------------------Ideal------------------------------------------ // Return a node which is more "ideal" than the current node. // Attempt to convert into a counted-loop. -Node *CountedLoopNode::Ideal(PhaseGVN *phase, bool can_reshape) { +Node* CountedLoopNode::Ideal(PhaseGVN* phase, bool can_reshape) { + if (can_reshape && phase->type(in(LoopBackControl)) == Type::TOP) { + // Since this loop is collapsing, we also need to remove the associated Template Assertion Predicates above this loop + // (no more loop opts for this loop). Otherwise, the Template Assertion Predicates could wrongly end up at the loop + // entry of another counted loop which did not have any predicates before (could happen for example, when partially + // peeling a loop which loses its Parse Predicates due to nodes in between which are later optimized away + converting + // this loop into a counted loop). When further splitting this counted loop during loop opts into sub loops, we + // erroneously create Initialized Assertion Predicates for the sub loops with the init and stride values of the + // previously collapsed loop. These can then fail at runtime because they check the wrong values. + KillTemplateAssertionPredicates kill_template_assertion_predicates(phase->is_IterGVN()); + PredicatesForLoop predicates_for_loop(skip_strip_mined()->in(EntryControl), &kill_template_assertion_predicates); + predicates_for_loop.for_each(); + } return RegionNode::Ideal(phase, can_reshape); } @@ -2581,14 +2605,14 @@ SafePointNode* CountedLoopNode::outer_safepoint() const { return l->outer_safepoint(); } -Node* CountedLoopNode::skip_assertion_predicates_with_halt() { +Node* CountedLoopNode::skip_assertion_predicates() { Node* ctrl = in(LoopNode::EntryControl); if (is_main_loop()) { ctrl = skip_strip_mined()->in(LoopNode::EntryControl); } - if (is_main_loop() || is_post_loop()) { - AssertionPredicatesWithHalt assertion_predicates(ctrl); - return assertion_predicates.entry(); + if (ctrl != nullptr && (is_main_loop() || is_post_loop())) { + const Predicates predicates(ctrl); + return predicates.entry(); } return ctrl; } @@ -4062,109 +4086,15 @@ void PhaseIdealLoop::log_loop_tree() { } } -// Eliminate all Parse and Template Assertion Predicates that are not associated with a loop anymore. The eliminated -// predicates will be removed during the next round of IGVN. -void PhaseIdealLoop::eliminate_useless_predicates() { - if (C->parse_predicate_count() == 0 && C->template_assertion_predicate_count() == 0) { - return; // No predicates left. - } - - eliminate_useless_parse_predicates(); - eliminate_useless_template_assertion_predicates(); -} - -// Eliminate all Parse Predicates that do not belong to a loop anymore by marking them useless. These will be removed -// during the next round of IGVN. +// Eliminate all Parse Predicates which have no connection to a loop head. These cannot be used anymore at this point and +// might block other optimizations. void PhaseIdealLoop::eliminate_useless_parse_predicates() { - mark_all_parse_predicates_useless(); - if (C->has_loops()) { - mark_loop_associated_parse_predicates_useful(); + if (C->parse_predicate_count() == 0) { + return; // no predicates left } - add_useless_parse_predicates_to_igvn_worklist(); -} -void PhaseIdealLoop::mark_all_parse_predicates_useless() const { - for (int i = 0; i < C->parse_predicate_count(); i++) { - C->parse_predicate(i)->mark_useless(); - } -} - -void PhaseIdealLoop::mark_loop_associated_parse_predicates_useful() { - for (LoopTreeIterator iterator(_ltree_root); !iterator.done(); iterator.next()) { - IdealLoopTree* loop = iterator.current(); - if (loop->can_apply_loop_predication()) { - mark_useful_parse_predicates_for_loop(loop); - } - } -} - -void PhaseIdealLoop::mark_useful_parse_predicates_for_loop(IdealLoopTree* loop) { - Node* entry = loop->_head->in(LoopNode::EntryControl); - const Predicates predicates(entry); - ParsePredicateIterator iterator(predicates); - while (iterator.has_next()) { - iterator.next()->mark_useful(); - } -} - -void PhaseIdealLoop::add_useless_parse_predicates_to_igvn_worklist() { - for (int i = 0; i < C->parse_predicate_count(); i++) { - ParsePredicateNode* parse_predicate_node = C->parse_predicate(i); - if (parse_predicate_node->is_useless()) { - _igvn._worklist.push(parse_predicate_node); - } - } -} - - -// Eliminate all Template Assertion Predicates that do not belong to their originally associated loop anymore by -// replacing the Opaque4 node of the If node with true. These nodes will be removed during the next round of IGVN. -void PhaseIdealLoop::eliminate_useless_template_assertion_predicates() { - Unique_Node_List useful_predicates; - if (C->has_loops()) { - collect_useful_template_assertion_predicates(useful_predicates); - } - eliminate_useless_template_assertion_predicates(useful_predicates); -} - -void PhaseIdealLoop::collect_useful_template_assertion_predicates(Unique_Node_List& useful_predicates) { - for (LoopTreeIterator iterator(_ltree_root); !iterator.done(); iterator.next()) { - IdealLoopTree* loop = iterator.current(); - if (loop->can_apply_loop_predication()) { - collect_useful_template_assertion_predicates_for_loop(loop, useful_predicates); - } - } -} - -void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(IdealLoopTree* loop, - Unique_Node_List &useful_predicates) { - Node* entry = loop->_head->in(LoopNode::EntryControl); - const Predicates predicates(entry); - if (UseProfiledLoopPredicate) { - const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block(); - if (profiled_loop_predicate_block->has_parse_predicate()) { - IfProjNode* parse_predicate_proj = profiled_loop_predicate_block->parse_predicate_success_proj(); - get_assertion_predicates(parse_predicate_proj, useful_predicates, true); - } - } - - if (UseLoopPredicate) { - const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block(); - if (loop_predicate_block->has_parse_predicate()) { - IfProjNode* parse_predicate_proj = loop_predicate_block->parse_predicate_success_proj(); - get_assertion_predicates(parse_predicate_proj, useful_predicates, true); - } - } -} - -void PhaseIdealLoop::eliminate_useless_template_assertion_predicates(Unique_Node_List& useful_predicates) { - for (int i = 0; i < C->template_assertion_predicate_count(); i++) { - Node* opaque4 = C->template_assertion_predicate_opaq_node(i); - assert(opaque4->Opcode() == Op_Opaque4, "must be"); - if (!useful_predicates.member(opaque4)) { // not in the useful list - _igvn.replace_node(opaque4, opaque4->in(2)); - } - } + EliminateUselessParsePredicates eliminator(&_igvn, _ltree_root); + eliminator.eliminate(); } // If a post or main loop is removed due to an assert predicate, the opaque that guards the loop is not needed anymore @@ -4512,7 +4442,7 @@ void PhaseIdealLoop::build_and_optimize() { // Some parser-inserted loop predicates could never be used by loop // predication or they were moved away from loop during some optimizations. // For example, peeling. Eliminate them before next loop optimizations. - eliminate_useless_predicates(); + eliminate_useless_parse_predicates(); #ifndef PRODUCT C->verify_graph_edges(); @@ -4970,6 +4900,11 @@ bool IdealLoopTree::verify_tree(IdealLoopTree* loop_verify) const { assert(ctrl != nullptr && ctrl->is_CFG(), "sane loop in-ctrl"); assert(back != nullptr && back->is_CFG(), "sane loop backedge"); cl->loopexit(); // assert implied + + if (cl->is_main_loop() || cl->is_post_loop()) { + // Should only find Assertion Predicates in main or post loop + DEBUG_ONLY(VerifyOnlyAssertionPredicates::verify(ctrl);) + } } // Broken part of VerifyLoopOptimizations (E) @@ -5703,7 +5638,7 @@ Node* CountedLoopNode::is_canonical_loop_entry() { if (!is_main_loop() && !is_post_loop()) { return nullptr; } - Node* ctrl = skip_assertion_predicates_with_halt(); + Node* ctrl = skip_assertion_predicates(); if (ctrl == nullptr || (!ctrl->is_IfTrue() && !ctrl->is_IfFalse())) { return nullptr; @@ -5749,7 +5684,7 @@ CountedLoopEndNode* CountedLoopNode::find_pre_loop_end() { return nullptr; } - Node* p_f = skip_assertion_predicates_with_halt()->in(0)->in(0); + Node* p_f = skip_assertion_predicates()->in(0)->in(0); if (!p_f->is_IfFalse() || !p_f->in(0)->is_CountedLoopEnd()) { return nullptr; } @@ -6182,29 +6117,14 @@ void PhaseIdealLoop::build_loop_late_post_work(Node *n, bool pinned) { // Move the node above predicates as far up as possible so a // following pass of loop predication doesn't hoist a predicate // that depends on it above that node. - Node* new_ctrl = least; - for (;;) { - if (!new_ctrl->is_Proj()) { - break; - } - CallStaticJavaNode* call = new_ctrl->as_Proj()->is_uncommon_trap_if_pattern(); - if (call == nullptr) { - break; - } - int req = call->uncommon_trap_request(); - Deoptimization::DeoptReason trap_reason = Deoptimization::trap_request_reason(req); - if (trap_reason != Deoptimization::Reason_loop_limit_check && - trap_reason != Deoptimization::Reason_predicate && - trap_reason != Deoptimization::Reason_profile_predicate) { - break; - } - Node* c = new_ctrl->in(0)->in(0); - if (is_dominator(c, early) && c != early) { + PredicateEntryIterator predicate_iterator(least); + while (predicate_iterator.has_next()) { + Node* next_predicate_entry = predicate_iterator.next_predicate_entry(); + if (is_dominator(next_predicate_entry, early) && next_predicate_entry != early) { break; } - new_ctrl = c; + least = next_predicate_entry; } - least = new_ctrl; } // Try not to place code on a loop entry projection // which can inhibit range check elimination. diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index c91ed6ae9e3b0..9e9860fc5cc9c 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -35,13 +35,15 @@ class CmpNode; class BaseCountedLoopEndNode; class CountedLoopNode; +class PredicateBlock; class IdealLoopTree; class LoopNode; class Node; +class TemplateAssertionPredicateDataOutput; class OuterStripMinedLoopEndNode; -class PredicateBlock; class PathFrequency; class PhaseIdealLoop; +class Predicates; class VectorSet; class Invariance; struct small_cache; @@ -322,7 +324,7 @@ class CountedLoopNode : public BaseCountedLoopNode { virtual IfFalseNode* outer_loop_exit() const; virtual SafePointNode* outer_safepoint() const; - Node* skip_assertion_predicates_with_halt(); + Node* skip_assertion_predicates(); virtual BasicType bt() const { return T_INT; @@ -930,31 +932,6 @@ class PhaseIdealLoop : public PhaseTransform { Node* cast_incr_before_loop(Node* incr, Node* ctrl, Node* loop); -#ifdef ASSERT - void ensure_zero_trip_guard_proj(Node* node, bool is_main_loop); -#endif - void copy_assertion_predicates_to_main_loop_helper(const PredicateBlock* predicate_block, Node* init, Node* stride, - IdealLoopTree* outer_loop, LoopNode* outer_main_head, - uint dd_main_head, uint idx_before_pre_post, - uint idx_after_post_before_pre, Node* zero_trip_guard_proj_main, - Node* zero_trip_guard_proj_post, const Node_List &old_new); - void copy_assertion_predicates_to_main_loop(CountedLoopNode* pre_head, Node* init, Node* stride, IdealLoopTree* outer_loop, - LoopNode* outer_main_head, uint dd_main_head, uint idx_before_pre_post, - uint idx_after_post_before_pre, Node* zero_trip_guard_proj_main, - Node* zero_trip_guard_proj_post, const Node_List& old_new); - Node* clone_assertion_predicate_and_initialize(Node* iff, Node* new_init, Node* new_stride, Node* predicate, - Node* uncommon_proj, Node* control, IdealLoopTree* outer_loop, - Node* input_proj); - static void count_opaque_loop_nodes(Node* n, uint& init, uint& stride); - static bool assertion_predicate_has_loop_opaque_node(IfNode* iff); - static void get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque = false); - void update_main_loop_assertion_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, int stride_con); - void copy_assertion_predicates_to_post_loop(LoopNode* main_loop_head, CountedLoopNode* post_loop_head, Node* init, - Node* stride); - void initialize_assertion_predicates_for_peeled_loop(const PredicateBlock* predicate_block, LoopNode* outer_loop_head, - int dd_outer_loop_head, Node* init, Node* stride, - IdealLoopTree* outer_loop, uint idx_before_clone, - const Node_List& old_new); void insert_loop_limit_check_predicate(ParsePredicateSuccessProj* loop_limit_check_parse_proj, Node* cmp_limit, Node* bol); #ifdef ASSERT @@ -1280,15 +1257,19 @@ class PhaseIdealLoop : public PhaseTransform { // Generate code to do a loop peel for the given loop (and body). // old_new is a temp array. void do_peeling( IdealLoopTree *loop, Node_List &old_new ); + void clone_assertion_predicates_from_original_loop(CountedLoopNode* source_loop_head, CountedLoopNode* target_loop_head, + uint first_cloned_loop_node_index); + void move_assertion_predicates_from_cloned_loop(CountedLoopNode* source_loop_head, CountedLoopNode* target_loop_head, + uint first_cloned_loop_node_index, Node_List& old_new); + void update_assertion_predicates_during_unroll(CountedLoopNode* loop_head); // Add pre and post loops around the given loop. These loops are used // during RCE, unrolling and aligning loops. void insert_pre_post_loops( IdealLoopTree *loop, Node_List &old_new, bool peel_only ); // Add post loop after the given loop. - Node *insert_post_loop(IdealLoopTree* loop, Node_List& old_new, - CountedLoopNode* main_head, CountedLoopEndNode* main_end, - Node*& incr, Node* limit, CountedLoopNode*& post_head); + Node* insert_post_loop(IdealLoopTree* loop, Node_List& old_new, CountedLoopNode* main_head, + CountedLoopEndNode* main_end, CountedLoopNode*& post_head); // Add a vector post loop between a vector main loop and the current post loop void insert_vector_post_loop(IdealLoopTree *loop, Node_List &old_new); @@ -1330,7 +1311,7 @@ class PhaseIdealLoop : public PhaseTransform { bool* p_short_scale, int depth); // Create a new if above the uncommon_trap_if_pattern for the predicate to be promoted - IfProjNode* create_new_if_for_predicate(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, + IfTrueNode* create_new_if_for_predicate(ParsePredicateSuccessProj* parse_predicate_success_proj, Node* new_entry, Deoptimization::DeoptReason reason, int opcode, bool rewire_uncommon_proj_phi_inputs = false); @@ -1347,14 +1328,19 @@ class PhaseIdealLoop : public PhaseTransform { public: void register_control(Node* n, IdealLoopTree *loop, Node* pred, bool update_body = true); + void replace_control_same_loop(Node* n, Node* new_control) { + _igvn.replace_input_of(n, 0, new_control); + set_idom(n, new_control, dom_depth(new_control)); + } + void replace_loop_entry(LoopNode* loop_head, Node* new_entry) { _igvn.replace_input_of(loop_head, LoopNode::EntryControl, new_entry); set_idom(loop_head, new_entry, dom_depth(new_entry)); } // Construct a range check for a predicate if - BoolNode* rc_predicate(IdealLoopTree* loop, Node* ctrl, int scale, Node* offset, Node* init, Node* limit, - jint stride, Node* range, bool upper, bool& overflow); + BoolNode* rc_predicate(Node* ctrl, int scale, Node* offset, Node* init, Node* limit, jint stride, Node* range, + bool upper, bool& overflow); // Implementation of the loop predication to promote checks outside the loop bool loop_predication_impl(IdealLoopTree *loop); @@ -1363,36 +1349,18 @@ class PhaseIdealLoop : public PhaseTransform { bool loop_predication_impl_helper(IdealLoopTree* loop, IfProjNode* if_success_proj, ParsePredicateSuccessProj* parse_predicate_proj, CountedLoopNode* cl, ConNode* zero, Invariance& invar, Deoptimization::DeoptReason reason); - bool can_create_loop_predicates(const PredicateBlock* profiled_loop_predicate_block) const; + bool can_create_loop_predicates(const Predicates& predicates) const; bool loop_predication_should_follow_branches(IdealLoopTree* loop, float& loop_trip_cnt); void loop_predication_follow_branches(Node *c, IdealLoopTree *loop, float loop_trip_cnt, PathFrequency& pf, Node_Stack& stack, VectorSet& seen, Node_List& if_proj_list); - IfProjNode* add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj, - ParsePredicateSuccessProj* parse_predicate_proj, - IfProjNode* upper_bound_proj, int scale, Node* offset, Node* init, Node* limit, - jint stride, Node* rng, bool& overflow, Deoptimization::DeoptReason reason); - Node* add_range_check_elimination_assertion_predicate(IdealLoopTree* loop, Node* predicate_proj, int scale_con, - Node* offset, Node* limit, jint stride_con, Node* value); - // Helper function to collect predicate for eliminating the useless ones - void eliminate_useless_predicates(); + TemplateAssertionPredicateNode* add_range_check_elimination_assertion_predicates(IdealLoopTree* loop, int scale, + Node* offset, Node* range); void eliminate_useless_parse_predicates(); - void mark_all_parse_predicates_useless() const; - void mark_loop_associated_parse_predicates_useful(); - static void mark_useful_parse_predicates_for_loop(IdealLoopTree* loop); - void add_useless_parse_predicates_to_igvn_worklist(); - - void eliminate_useless_template_assertion_predicates(); - void collect_useful_template_assertion_predicates(Unique_Node_List& useful_predicates); - static void collect_useful_template_assertion_predicates_for_loop(IdealLoopTree* loop, Unique_Node_List& useful_predicates); - void eliminate_useless_template_assertion_predicates(Unique_Node_List& useful_predicates); - void eliminate_useless_zero_trip_guard(); - bool has_control_dependencies_from_predicates(LoopNode* head) const; - void verify_fast_loop(LoopNode* head, const ProjNode* proj_true) const NOT_DEBUG_RETURN; public: // Change the control input of expensive nodes to allow commoning by // IGVN when it is guaranteed to not result in a more frequent @@ -1410,7 +1378,7 @@ class PhaseIdealLoop : public PhaseTransform { // Create a slow version of the loop by cloning the loop // and inserting an if to select fast-slow versions. // Return the inserted if. - IfNode* create_slow_version_of_loop(IdealLoopTree* loop, Node_List& old_new, IfNode* unswitch_iff); + IfNode* create_slow_version_of_loop(IdealLoopTree* loop, Node_List& old_new, IfNode* unswitching_candidate); // Clone loop with an invariant test (that does not exit) and // insert a clone of the test that selects which version to @@ -1506,7 +1474,8 @@ class PhaseIdealLoop : public PhaseTransform { Node *has_local_phi_input( Node *n ); // Mark an IfNode as being dominated by a prior test, // without actually altering the CFG (and hence IDOM info). - void dominated_by(IfProjNode* prevdom, IfNode* iff, bool flip = false, bool exclude_loop_predicate = false); + void dominated_by(IfProjNode* prevdom, IfNode* iff, bool flip = false, bool exclude_predicates = false); + void rewire_safe_outputs_to_dominator(Node* source, Node* dominator); // Split Node 'n' through merge point RegionNode* split_thru_region(Node* n, RegionNode* region); @@ -1617,28 +1586,6 @@ class PhaseIdealLoop : public PhaseTransform { _nodes_required = UINT_MAX; } - public: - // Clone Parse Predicates to slow and fast loop when unswitching a loop - void clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new, - IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred); - private: - void clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, - const PredicateBlock* predicate_block, - Deoptimization::DeoptReason reason, IfProjNode*& iffast_pred, - IfProjNode*& ifslow_pred); - void clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block, Deoptimization::DeoptReason reason, - IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred); - IfProjNode* clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, - Deoptimization::DeoptReason reason, bool slow_loop); - void clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, - Deoptimization::DeoptReason reason, IfProjNode* old_predicate_proj, - ParsePredicateSuccessProj* fast_loop_parse_predicate_proj, - ParsePredicateSuccessProj* slow_loop_parse_predicate_proj); - IfProjNode* clone_assertion_predicate_for_unswitched_loops(Node* iff, IfProjNode* predicate, - Deoptimization::DeoptReason reason, - ParsePredicateSuccessProj* parse_predicate_proj); - static void check_cloned_parse_predicate_for_unswitching(const Node* new_entry, bool is_fast_loop) PRODUCT_RETURN; - bool _created_loop_node; DEBUG_ONLY(void dump_idoms(Node* early, Node* wrong_lca);) NOT_PRODUCT(void dump_idoms_in_reverse(const Node* n, const Node_List& idom_list) const;) @@ -1738,10 +1685,11 @@ class PhaseIdealLoop : public PhaseTransform { void clone_template_assertion_predicate_bool_down_if_related(Node* n); Node* similar_subtype_check(const Node* x, Node* r_in); - void update_addp_chain_base(Node* x, Node* old_base, Node* new_base); bool can_move_to_inner_loop(Node* n, LoopNode* n_loop, Node* x); + void eliminate_old_range_check(IfProjNode* if_proj, + TemplateAssertionPredicateNode* template_assertion_predicate); }; diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index ea1bdc933d0f7..17cac4a2c28ae 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -305,7 +305,7 @@ bool PhaseIdealLoop::loop_phi_backedge_type_contains_zero(const Node* phi_diviso // Replace the dominated test with an obvious true or false. Place it on the // IGVN worklist for later cleanup. Move control-dependent data Nodes on the // live path up to the dominating control. -void PhaseIdealLoop::dominated_by(IfProjNode* prevdom, IfNode* iff, bool flip, bool exclude_loop_predicate) { +void PhaseIdealLoop::dominated_by(IfProjNode* prevdom, IfNode* iff, bool flip, bool exclude_predicates) { if (VerifyLoopOptimizations && PrintOpto) { tty->print_cr("dominating test"); } // prevdom is the dominating projection of the dominating test. @@ -349,7 +349,7 @@ void PhaseIdealLoop::dominated_by(IfProjNode* prevdom, IfNode* iff, bool flip, b ProjNode* dp_proj = dp->as_Proj(); ProjNode* unc_proj = iff->proj_out(1 - dp_proj->_con)->as_Proj(); - if (exclude_loop_predicate && + if (exclude_predicates && (unc_proj->is_uncommon_trap_proj(Deoptimization::Reason_predicate) != nullptr || unc_proj->is_uncommon_trap_proj(Deoptimization::Reason_profile_predicate) != nullptr || unc_proj->is_uncommon_trap_proj(Deoptimization::Reason_range_check) != nullptr)) { @@ -359,22 +359,26 @@ void PhaseIdealLoop::dominated_by(IfProjNode* prevdom, IfNode* iff, bool flip, b return; // Let IGVN transformation change control dependence. } - IdealLoopTree* old_loop = get_loop(dp); + rewire_safe_outputs_to_dominator(dp, prevdom); +} + +void PhaseIdealLoop::rewire_safe_outputs_to_dominator(Node* source, Node* dominator) { + IdealLoopTree* old_loop = get_loop(source); - for (DUIterator_Fast imax, i = dp->fast_outs(imax); i < imax; i++) { - Node* cd = dp->fast_out(i); // Control-dependent node + for (DUIterator_Fast imax, i = source->fast_outs(imax); i < imax; i++) { + Node* out = source->fast_out(i); // Control-dependent node // Do not rewire Div and Mod nodes which could have a zero divisor to avoid skipping their zero check. - if (cd->depends_only_on_test() && _igvn.no_dependent_zero_check(cd)) { - assert(cd->in(0) == dp, ""); - _igvn.replace_input_of(cd, 0, prevdom); - set_early_ctrl(cd, false); - IdealLoopTree* new_loop = get_loop(get_ctrl(cd)); + if (out->depends_only_on_test() && _igvn.no_dependent_zero_check(out)) { + assert(out->in(0) == source, "must be control dependent on source"); + _igvn.replace_input_of(out, 0, dominator); + set_early_ctrl(out, false); + IdealLoopTree* new_loop = get_loop(get_ctrl(out)); if (old_loop != new_loop) { if (!old_loop->_child) { - old_loop->_body.yank(cd); + old_loop->_body.yank(out); } if (!new_loop->_child) { - new_loop->_body.push(cd); + new_loop->_body.push(out); } } --i; @@ -774,9 +778,6 @@ Node *PhaseIdealLoop::conditional_move( Node *region ) { } }//for Node* bol = iff->in(1); - if (bol->Opcode() == Op_Opaque4) { - return nullptr; // Ignore loop predicate checks (the Opaque4 ensures they will go away) - } assert(bol->Opcode() == Op_Bool, "Unexpected node"); int cmp_op = bol->in(1)->Opcode(); if (cmp_op == Op_SubTypeCheck) { // SubTypeCheck expansion expects an IfNode @@ -1652,6 +1653,7 @@ void PhaseIdealLoop::try_sink_out_of_loop(Node* n) { !n->is_MergeMem() && !n->is_CMove() && n->Opcode() != Op_Opaque4 && + n->Opcode() != Op_OpaqueAssertionPredicate && !n->is_Type()) { Node *n_ctrl = get_ctrl(n); IdealLoopTree *n_loop = get_loop(n_ctrl); @@ -1946,14 +1948,14 @@ Node* PhaseIdealLoop::clone_iff(PhiNode* phi) { if (b->is_Phi()) { _igvn.replace_input_of(phi, i, clone_iff(b->as_Phi())); } else { - assert(b->is_Bool() || b->Opcode() == Op_Opaque4, ""); + assert(b->is_Bool() || b->Opcode() == Op_Opaque4 || b->Opcode() == Op_OpaqueAssertionPredicate, ""); } } Node* n = phi->in(1); Node* sample_opaque = nullptr; Node *sample_bool = nullptr; - if (n->Opcode() == Op_Opaque4) { + if (n->Opcode() == Op_Opaque4 || n->Opcode() == Op_OpaqueAssertionPredicate) { sample_opaque = n; sample_bool = n->in(1); assert(sample_bool->is_Bool(), "wrong type"); diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp index 44c37768cdc7d..9f4d483c9cd11 100644 --- a/src/hotspot/share/opto/node.cpp +++ b/src/hotspot/share/opto/node.cpp @@ -610,9 +610,6 @@ void Node::destruct(PhaseValues* phase) { if (is_expensive()) { compile->remove_expensive_node(this); } - if (Opcode() == Op_Opaque4) { - compile->remove_template_assertion_predicate_opaq(this); - } if (is_ParsePredicate()) { compile->remove_parse_predicate(as_ParsePredicate()); } diff --git a/src/hotspot/share/opto/node.hpp b/src/hotspot/share/opto/node.hpp index 6dc4059a66f40..b64c18f1e5262 100644 --- a/src/hotspot/share/opto/node.hpp +++ b/src/hotspot/share/opto/node.hpp @@ -169,6 +169,7 @@ class State; class StoreNode; class SubNode; class SubTypeCheckNode; +class TemplateAssertionPredicateNode; class Type; class TypeNode; class UnlockNode; @@ -790,6 +791,7 @@ class Node { DEFINE_CLASS_ID(Move, Node, 17) DEFINE_CLASS_ID(LShift, Node, 18) DEFINE_CLASS_ID(Neg, Node, 19) + DEFINE_CLASS_ID(TemplateAssertionPredicate, Node, 20) _max_classes = ClassMask_Neg }; @@ -989,6 +991,7 @@ class Node { DEFINE_CLASS_QUERY(StoreVector) DEFINE_CLASS_QUERY(StoreVectorScatter) DEFINE_CLASS_QUERY(ShiftV) + DEFINE_CLASS_QUERY(TemplateAssertionPredicate) DEFINE_CLASS_QUERY(Unlock) #undef DEFINE_CLASS_QUERY @@ -1046,7 +1049,7 @@ class Node { bool for_post_loop_opts_igvn() const { return (_flags & Flag_for_post_loop_opts_igvn) != 0; } - // Is 'n' possibly a loop entry (i.e. a Parse Predicate projection)? + // Is 'n' possibly a loop entry before loop opts (i.e. a Parse Predicate projection)? static bool may_be_loop_entry(Node* n) { return n != nullptr && n->is_IfProj() && n->in(0)->is_ParsePredicate(); } diff --git a/src/hotspot/share/opto/opaquenode.cpp b/src/hotspot/share/opto/opaquenode.cpp index 5075ccd46649e..0a279ace05861 100644 --- a/src/hotspot/share/opto/opaquenode.cpp +++ b/src/hotspot/share/opto/opaquenode.cpp @@ -23,6 +23,7 @@ */ #include "precompiled.hpp" +#include "opto/connode.hpp" #include "opto/loopnode.hpp" #include "opto/opaquenode.hpp" #include "opto/phaseX.hpp" @@ -102,6 +103,25 @@ const Type* Opaque4Node::Value(PhaseGVN* phase) const { return phase->type(in(1)); } +const Type* OpaqueAssertionPredicateNode::Value(PhaseGVN* phase) const { + return phase->type(in(1)); +} + +Node* OpaqueAssertionPredicateNode::Identity(PhaseGVN* phase) { + if (phase->C->post_loop_opts_phase()) { + // Initialized Assertion Predicates must always evaluate to true. Therefore, we get rid of them in product builds + // as they are useless. In debug builds we keep them as additional verification code. +#ifdef ASSERT + return in(1); +#else + return phase->intcon(1); +#endif // ASSERT + } else { + phase->C->record_for_post_loop_opts_igvn(this); + } + return this; +} + //============================================================================= uint ProfileBooleanNode::hash() const { return NO_HASH; } diff --git a/src/hotspot/share/opto/opaquenode.hpp b/src/hotspot/share/opto/opaquenode.hpp index 7e968b78d31cf..8d68b3c2f2c67 100644 --- a/src/hotspot/share/opto/opaquenode.hpp +++ b/src/hotspot/share/opto/opaquenode.hpp @@ -131,6 +131,19 @@ class Opaque4Node : public Node { virtual const Type* Value(PhaseGVN* phase) const; }; +// This node is used for Initialized Assertion Predicate bool nodes. Initialized Assertion Predicates must always evaluate +// to true. Therefore, we get rid of them in product builds as they are useless. In debug builds we keep them as additional +// verification code (i.e. removing this node and use the BoolNode input instead). +class OpaqueAssertionPredicateNode : public Node { + public: + OpaqueAssertionPredicateNode(BoolNode* bol) : Node(nullptr, bol) {} + + virtual int Opcode() const; + virtual Node* Identity(PhaseGVN* phase); + virtual const Type* Value(PhaseGVN* phase) const; + virtual const Type* bottom_type() const { return TypeInt::BOOL; } +}; + //------------------------------ProfileBooleanNode------------------------------- // A node represents value profile for a boolean during parsing. diff --git a/src/hotspot/share/opto/predicates.cpp b/src/hotspot/share/opto/predicates.cpp index 01470a5bf4eb7..c12af385b1954 100644 --- a/src/hotspot/share/opto/predicates.cpp +++ b/src/hotspot/share/opto/predicates.cpp @@ -23,38 +23,11 @@ */ #include "precompiled.hpp" +#include "opto/addnode.hpp" #include "opto/callnode.hpp" +#include "opto/castnode.hpp" #include "opto/predicates.hpp" -#include "opto/subnode.hpp" - -// Walk over all Initialized Assertion Predicates and return the entry into the first Initialized Assertion Predicate -// (i.e. not belonging to an Initialized Assertion Predicate anymore) -Node* AssertionPredicatesWithHalt::find_entry(Node* start_proj) { - Node* entry = start_proj; - while (is_assertion_predicate_success_proj(entry)) { - entry = entry->in(0)->in(0); - } - return entry; -} - -bool AssertionPredicatesWithHalt::is_assertion_predicate_success_proj(const Node* predicate_proj) { - if (predicate_proj == nullptr || !predicate_proj->is_IfProj() || !predicate_proj->in(0)->is_If()) { - return false; - } - return has_opaque4(predicate_proj) && has_halt(predicate_proj); -} - -// Check if the If node of `predicate_proj` has an Opaque4 node as input. -bool AssertionPredicatesWithHalt::has_opaque4(const Node* predicate_proj) { - IfNode* iff = predicate_proj->in(0)->as_If(); - return iff->in(1)->Opcode() == Op_Opaque4; -} - -// Check if the other projection (UCT projection) of `success_proj` has a Halt node as output. -bool AssertionPredicatesWithHalt::has_halt(const Node* success_proj) { - ProjNode* other_proj = success_proj->as_IfProj()->other_if_proj(); - return other_proj->outcnt() == 1 && other_proj->unique_out()->Opcode() == Op_Halt; -} +#include "opto/rootnode.hpp" // Returns the Parse Predicate node if the provided node is a Parse Predicate success proj. Otherwise, return null. ParsePredicateNode* ParsePredicate::init_parse_predicate(Node* parse_predicate_proj, @@ -69,17 +42,60 @@ ParsePredicateNode* ParsePredicate::init_parse_predicate(Node* parse_predicate_p return nullptr; } -Deoptimization::DeoptReason RuntimePredicate::uncommon_trap_reason(IfProjNode* if_proj) { - CallStaticJavaNode* uct_call = if_proj->is_uncommon_trap_if_pattern(); - if (uct_call == nullptr) { - return Deoptimization::Reason_none; +void EliminateUselessParsePredicates::eliminate() { + mark_all_parse_predicates_useless(); + for (LoopTreeIterator iterator(_ltree_root); !iterator.done(); iterator.next()) { + IdealLoopTree* loop = iterator.current(); + mark_parse_predicates_useful(loop); + } + add_useless_predicates_to_igvn_worklist(); +} + +void EliminateUselessParsePredicates::mark_all_parse_predicates_useless() { + for (ParsePredicateNode* parse_predicate : C->parse_predicates()) { + parse_predicate->mark_useless(); + } +} + +// This visitor marks all visited Parse Predicates useful. +class ParsePredicateUsefulMarker : public PredicateVisitor { + public: + using PredicateVisitor::visit; + + void visit(ParsePredicate& parse_predicate) override { + parse_predicate.head()->mark_useful(); + } +}; + +// Mark all Parse Predicates 'loop' as useful. If 'loop' represents an outer strip mined loop, we can skip it because +// we have already processed the predicates before when we visited its counted (inner) loop. +void EliminateUselessParsePredicates::mark_parse_predicates_useful(IdealLoopTree* loop) { + if (loop->can_apply_loop_predication()) { + ParsePredicateUsefulMarker useful_marker; + Node* entry = loop->_head->as_Loop()->skip_strip_mined()->in(LoopNode::EntryControl); + PredicatesForLoop predicates_for_loop(entry, &useful_marker); + predicates_for_loop.for_each(); + } +} + +void EliminateUselessParsePredicates::add_useless_predicates_to_igvn_worklist() { + for (ParsePredicateNode* parse_predicate : C->parse_predicates()) { + if (parse_predicate->is_useless()) { + _igvn->_worklist.push(parse_predicate); } - return Deoptimization::trap_request_reason(uct_call->uncommon_trap_request()); + } } -bool RuntimePredicate::is_success_proj(Node* node, Deoptimization::DeoptReason deopt_reason) { - if (may_be_runtime_predicate_if(node)) { - return deopt_reason == uncommon_trap_reason(node->as_IfProj()); +bool RuntimePredicate::is_success_proj(Node* maybe_success_proj) { + if (may_be_runtime_predicate_if(maybe_success_proj)) { + IfProjNode* success_proj = maybe_success_proj->as_IfProj(); + if (is_being_folded_without_uncommon_proj(success_proj)) { + return true; + } + const Deoptimization::DeoptReason deopt_reason = uncommon_trap_reason(success_proj); + return (deopt_reason == Deoptimization::Reason_loop_limit_check || + deopt_reason == Deoptimization::Reason_predicate || + deopt_reason == Deoptimization::Reason_profile_predicate); } else { return false; } @@ -88,9 +104,9 @@ bool RuntimePredicate::is_success_proj(Node* node, Deoptimization::DeoptReason d // A Runtime Predicate must have an If or a RangeCheck node, while the If should not be a zero trip guard check. bool RuntimePredicate::may_be_runtime_predicate_if(Node* node) { if (node->is_IfProj()) { - const IfNode* if_node = node->in(0)->as_If(); + const Node* if_node = node->in(0); const int opcode_if = if_node->Opcode(); - if ((opcode_if == Op_If && !if_node->is_zero_trip_guard()) + if ((opcode_if == Op_If && !if_node->as_If()->is_zero_trip_guard()) || opcode_if == Op_RangeCheck) { return true; } @@ -98,72 +114,211 @@ bool RuntimePredicate::may_be_runtime_predicate_if(Node* node) { return false; } -ParsePredicateIterator::ParsePredicateIterator(const Predicates& predicates) : _current_index(0) { - const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); - if (loop_limit_check_predicate_block->has_parse_predicate()) { - _parse_predicates.push(loop_limit_check_predicate_block->parse_predicate()); - } - if (UseProfiledLoopPredicate) { - const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block(); - if (profiled_loop_predicate_block->has_parse_predicate()) { - _parse_predicates.push(profiled_loop_predicate_block->parse_predicate()); +// Has the If node only the success projection left due to already folding the uncommon projection because of a constant +// bool input? This can happen during IGVN. Treat this case as being a Runtime Predicate to not miss other Predicates +// above this node when iterating through them. +bool RuntimePredicate::is_being_folded_without_uncommon_proj(const IfProjNode* success_proj) { + IfNode* if_node = success_proj->in(0)->as_If(); + return if_node->in(1)->is_ConI() && if_node->outcnt() == 1; +} + +Deoptimization::DeoptReason RuntimePredicate::uncommon_trap_reason(IfProjNode* if_proj) { + CallStaticJavaNode* uct_call = if_proj->is_uncommon_trap_if_pattern(); + if (uct_call == nullptr) { + return Deoptimization::Reason_none; } + return Deoptimization::trap_request_reason(uct_call->uncommon_trap_request()); +} + +bool RuntimePredicate::is_success_proj(Node* maybe_success_proj, Deoptimization::DeoptReason deopt_reason) { + if (may_be_runtime_predicate_if(maybe_success_proj)) { + IfProjNode* success_proj = maybe_success_proj->as_IfProj(); + return is_being_folded_without_uncommon_proj(success_proj) + || deopt_reason == uncommon_trap_reason(success_proj); + } else { + return false; } - if (UseLoopPredicate) { - const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block(); - if (loop_predicate_block->has_parse_predicate()) { - _parse_predicates.push(loop_predicate_block->parse_predicate()); - } +} + +// Insert a new predicate above the current tail of the chain. The control input of the current tail is set to the new +// predicate. The new predicate becomes the new tail of this chain. +void PredicateInserter::insert(Predicate& new_predicate) { + Node* new_entry_for_tail = new_predicate.tail(); + if (_ctrl_out->is_Loop()) { + _phase->replace_loop_entry(_ctrl_out->as_Loop(), new_entry_for_tail); + } else { + _phase->replace_control_same_loop(_ctrl_out, new_entry_for_tail); } + _ctrl_out = new_predicate.head(); } -ParsePredicateNode* ParsePredicateIterator::next() { - assert(has_next(), "always check has_next() first"); - return _parse_predicates.at(_current_index++); +// Insert an existing predicate into the chain by setting it as new tail. There is no control update involved. +void PredicateInserter::skip(Predicate& existing_predicate) { + _ctrl_out = existing_predicate.head(); } -#ifdef ASSERT -// Check that the block has at most one Parse Predicate and that we only find Regular Predicate nodes (i.e. IfProj, -// If, or RangeCheck nodes). -void PredicateBlock::verify_block() { - Node* next = _parse_predicate.entry(); // Skip unique Parse Predicate of this block if present - while (next != _entry) { - assert(!next->is_ParsePredicate(), "can only have one Parse Predicate in a block"); - const int opcode = next->Opcode(); - assert(next->is_IfProj() || opcode == Op_If || opcode == Op_RangeCheck, - "Regular Predicates consist of an IfProj and an If or RangeCheck node"); - assert(opcode != Op_If || !next->as_If()->is_zero_trip_guard(), "should not be zero trip guard"); - next = next->in(0); +// This visitor clones all visited Template Assertion Predicates and sets a new input for the cloned OpaqueLoopInitNodes. +// Afterward, we initialize the template by creating an InitializedAssertionPredicate for the init and last value. +class CloneAndInitAssertionPredicates : public PredicateVisitor { + Node* _old_target_loop_entry; + Node* _new_init; + PhaseIdealLoop* _phase; + TemplateAssertionPredicateDataOutput* _node_in_target_loop; + PredicateInserter _predicate_inserter; + + TemplateAssertionPredicate create_new_template(TemplateAssertionPredicate& template_assertion_predicate) { + TemplateAssertionPredicate new_template = + template_assertion_predicate.clone_update_opaque_init(_old_target_loop_entry, _new_init, _node_in_target_loop, + _phase); + _predicate_inserter.insert(new_template); + return new_template; } + + public: + using PredicateVisitor::visit; + + CloneAndInitAssertionPredicates(CountedLoopNode* target_loop_head, TemplateAssertionPredicateDataOutput* node_in_target_loop, + PhaseIdealLoop* phase) + : _old_target_loop_entry(target_loop_head->skip_strip_mined()->in(LoopNode::EntryControl)), + _new_init(target_loop_head->init_trip()), + _phase(phase), + _node_in_target_loop(node_in_target_loop), + _predicate_inserter(target_loop_head, phase) {} + + void visit(TemplateAssertionPredicate& template_assertion_predicate) override { + TemplateAssertionPredicate new_template = create_new_template(template_assertion_predicate); + new_template.initialize(_phase, _predicate_inserter); + } +}; + +// Creates new Assertion Predicates at the 'target_loop_head'. A new Template Assertion Predicate is inserted with the +// new init and stride values of the target loop for each existing Template Assertion Predicate found at source loop. +// For each new Template Assertion Predicate, an init and last value Initialized Assertion Predicate is created. +void AssertionPredicates::clone_to_loop(CountedLoopNode* target_loop_head, TemplateAssertionPredicateDataOutput* node_in_target_loop) { + CloneAndInitAssertionPredicates clone_and_init_assertion_predicates(target_loop_head, node_in_target_loop, _phase); + Node* source_loop_entry = _source_loop_head->skip_strip_mined()->in(LoopNode::EntryControl); + PredicatesForLoop predicates_for_loop(source_loop_entry, &clone_and_init_assertion_predicates); + predicates_for_loop.for_each(); } -#endif // ASSERT -// Walk over all Regular Predicates of this block (if any) and return the first node not belonging to the block -// anymore (i.e. entry to the first Regular Predicate in this block if any or `regular_predicate_proj` otherwise). -Node* PredicateBlock::skip_regular_predicates(Node* regular_predicate_proj, Deoptimization::DeoptReason deopt_reason) { - Node* entry = regular_predicate_proj; - while (RuntimePredicate::is_success_proj(entry, deopt_reason)) { - assert(entry->in(0)->as_If(), "must be If node"); - entry = entry->in(0)->in(0); +// This visitor clones all visited Template Assertion Predicates and sets a new input for the cloned OpaqueLoopInitNodes. +// Afterward, we initialize the template by creating an InitializedAssertionPredicate for the init and last value. +// The visited Template Assertion Predicates are killed. +class MoveAndInitAssertionPredicates : public PredicateVisitor { + CloneAndInitAssertionPredicates _clone_and_init_assertion_predicates; + PhaseIterGVN* _igvn; + + public: + using PredicateVisitor::visit; + + MoveAndInitAssertionPredicates(CountedLoopNode* target_loop_head, TemplateAssertionPredicateDataOutput* node_in_target_loop, + PhaseIdealLoop* phase) + : _clone_and_init_assertion_predicates(target_loop_head, node_in_target_loop, phase), + _igvn(&phase->igvn()) {} + + void visit(TemplateAssertionPredicate& template_assertion_predicate) override { + _clone_and_init_assertion_predicates.visit(template_assertion_predicate); + template_assertion_predicate.kill(_igvn); } - return entry; +}; + +// Creates new Assertion Predicates at the 'target_loop_head'. A new Template Assertion Predicate is inserted with the +// new init and stride values of the target loop for each existing Template Assertion Predicate found at source loop. +// For each new Template Assertion Predicate, an init and last value Initialized Assertion Predicate is created. +// The existing Template Assertion Predicates at the source loop are removed. +void AssertionPredicates::move_to_loop(CountedLoopNode* target_loop_head, TemplateAssertionPredicateDataOutput* node_in_target_loop) { + MoveAndInitAssertionPredicates move_and_init_assertion_predicates(target_loop_head, node_in_target_loop, _phase); + Node* source_loop_entry = _source_loop_head->skip_strip_mined()->in(LoopNode::EntryControl); + PredicatesForLoop predicates_for_loop(source_loop_entry, &move_and_init_assertion_predicates); + predicates_for_loop.for_each(); +} + +// Creates a single new Template Assertion Predicate at the source loop with its init and stride value together with an +// Initialized Assertion Predicate for the init and last value. +void AssertionPredicates::create(const int if_opcode, const int scale, Node* offset, Node* range) { + PredicateInserter predicate_inserter(_source_loop_head, _phase); + TemplateAssertionPredicate template_assertion_predicate = create_new_template(if_opcode, scale, offset, range, + predicate_inserter); + template_assertion_predicate.initialize(_phase, predicate_inserter); +} + +TemplateAssertionPredicate AssertionPredicates::create_new_template(const int if_opcode, const int scale, Node* offset, + Node* range, PredicateInserter& predicate_inserter) { + LoopNode* loop_head = _source_loop_head->skip_strip_mined(); + NewTemplateAssertionPredicate new_template_assertion_predicate(_source_loop_head, _phase); + TemplateAssertionPredicate template_assertion_predicate( + new_template_assertion_predicate.create(if_opcode, loop_head->in(LoopNode::EntryControl), scale, offset, + range)); + predicate_inserter.insert(template_assertion_predicate); + return template_assertion_predicate; } -TemplateAssertionPredicateBool::TemplateAssertionPredicateBool(Node* source_bool) : _source_bool(source_bool->as_Bool()) { +// This visitor updates all visited Template Assertion Predicates by setting a new input for the OpaqueLoopStrideNodes. +// Afterward, we initialize the updated template by creating an Initialized Assertion Predicate for the init and last +// value. The old Initialized Assertion Predicates are killed. +class UpdateAndInitAssertionPredicates : public PredicateVisitor { + Node* _new_stride; + PhaseIdealLoop* _phase; + uint _index_before_visit; + PredicateInserter _predicate_inserter; + + public: + using PredicateVisitor::visit; + + UpdateAndInitAssertionPredicates(Node* new_stride, CountedLoopNode* loop_head, PhaseIdealLoop* phase) + : _new_stride(new_stride), + _phase(phase), + _index_before_visit(phase->C->unique()), + _predicate_inserter(loop_head, phase) {} + + void visit(TemplateAssertionPredicate& template_assertion_predicate) override { + template_assertion_predicate.replace_opaque_stride_input(_new_stride, &_phase->igvn()); + _predicate_inserter.skip(template_assertion_predicate); + template_assertion_predicate.initialize(_phase, _predicate_inserter); + } + + void visit(InitializedAssertionPredicate& initialized_assertion_predicate) override { + if (initialized_assertion_predicate.head()->_idx < _index_before_visit) { + initialized_assertion_predicate.kill(&_phase->igvn()); + } + } +}; + +// Update existing Assertion Predicates at the source loop with the provided stride value. The templates are first +// updated and then new Initialized Assertion Predicates are created based on the updated templates. The previously +// existing Initialized Assertion are no longer needed and killed. +void AssertionPredicates::update(const int new_stride_con) { + Node* new_stride = create_stride(new_stride_con); + UpdateAndInitAssertionPredicates update_assertion_predicates(new_stride, _source_loop_head, _phase); + Node* source_loop_entry = _source_loop_head->skip_strip_mined()->in(LoopNode::EntryControl); + PredicatesForLoop predicates_for_loop(source_loop_entry, &update_assertion_predicates); + predicates_for_loop.for_each(); +} + +Node* AssertionPredicates::create_stride(const int stride_con) { + Node* new_stride = _phase->igvn().intcon(stride_con); + _phase->set_ctrl(new_stride, _phase->C->root()); + return new_stride; +} + +TemplateAssertionPredicateBool::TemplateAssertionPredicateBool(Node* source_bool) : _source_bool(source_bool->isa_Bool()) { #ifdef ASSERT - // During IGVN, we could have multiple outputs of the _source_bool, for example, when the backedge of the loop of - // this Template Assertion Predicate is about to die and the CastII on the last value bool already folded to a - // constant (i.e. no OpaqueLoop* nodes anymore). Then IGVN could already have commoned up the bool with the bool of - // one of the Hoisted Check Predicates. Just check that the Template Assertion Predicate is one of the outputs. - bool has_template_output = false; - for (DUIterator_Fast imax, i = source_bool->fast_outs(imax); i < imax; i++) { - Node* out = source_bool->fast_out(i); - if (out->Opcode() == Op_Opaque4) { - has_template_output = true; - break; + // We could have already folded the BoolNode to a constant. + if (is_not_dead()) { + // During IGVN, we could have multiple outputs of the _source_bool, for example, when the backedge of the loop of + // this Template Assertion Predicate is about to die and the CastII on the last value bool already folded to a + // constant (i.e. no OpaqueLoop* nodes anymore). Then IGVN could already have commoned up the bool with the bool of + // one of the Hoisted Check Predicates. Just check that the Template Assertion Predicate is one of the outputs. + bool has_template_output = false; + for (DUIterator_Fast imax, i = source_bool->fast_outs(imax); i < imax; i++) { + Node* out = source_bool->fast_out(i); + if (out->is_TemplateAssertionPredicate()) { + has_template_output = true; + break; + } } + assert(has_template_output, "must find Template Assertion Predicate as output"); } - assert(has_template_output, "must find Template Assertion Predicate as output"); #endif // ASSERT } @@ -331,7 +486,7 @@ class CloneAssertionPredicateBool : public StackObj { _phase(phase), _index_before_cloning(phase->C->unique()), _ctrl_for_clones(ctrl_for_clones) - DEBUG_ONLY(COMMA _found_init(false)) {} + DEBUG_ONLY(COMMA _found_init(false)) {} // Look for the OpaqueLoop* nodes to transform them with the strategy defined with 'transform_opaque_loop_nodes'. // Clone all nodes in between. @@ -402,6 +557,7 @@ class CloneOpaqueLoopNodes : public TransformOpaqueLoopNodes { // Clones this Template Assertion Predicate bool. This includes all nodes from the BoolNode to the OpaqueLoop* nodes. // The cloned nodes are not updated. BoolNode* TemplateAssertionPredicateBool::clone(Node* new_ctrl, PhaseIdealLoop* phase) { + assert(is_not_dead(), "must not be dead"); CloneOpaqueLoopNodes clone_opaque_loop_nodes(phase, new_ctrl); CloneAssertionPredicateBool clone_assertion_predicate_bool(_source_bool, new_ctrl, phase); return clone_assertion_predicate_bool.clone(&clone_opaque_loop_nodes); @@ -409,17 +565,21 @@ BoolNode* TemplateAssertionPredicateBool::clone(Node* new_ctrl, PhaseIdealLoop* // The transformations of this class clone the existing OpaqueLoop* nodes. The newly cloned OpaqueLoopInitNode // additionally gets a new input node. -class CloneWithNewInit : public TransformOpaqueLoopNodes { - Node* _new_init; +class CloneWithNewOpaqueInitInput : public TransformOpaqueLoopNodes { + PhaseIdealLoop* _phase; + Node* _new_opaque_init_input; CachedOpaqueLoopNodes _cached_opaque_loop_nodes; public: - CloneWithNewInit(PhaseIdealLoop* phase, Node* new_ctrl, Node* new_init) - : _new_init(new_init), + CloneWithNewOpaqueInitInput(PhaseIdealLoop* phase, Node* new_ctrl, Node* new_opaque_init_input) + : _phase(phase), + _new_opaque_init_input(new_opaque_init_input), _cached_opaque_loop_nodes(phase, new_ctrl) {} Node* transform_opaque_init(OpaqueLoopInitNode* opaque_init) override { - return _new_init; + Node* new_opaque_init = _cached_opaque_loop_nodes.clone_if_not_cached(opaque_init); + _phase->igvn().replace_input_of(new_opaque_init, 1, _new_opaque_init_input); + return new_opaque_init; } Node* transform_opaque_stride(OpaqueLoopStrideNode* opaque_stride) override { @@ -430,36 +590,376 @@ class CloneWithNewInit : public TransformOpaqueLoopNodes { // Clones this Template Assertion Predicate bool. This includes all nodes from the BoolNode to the OpaqueLoop* nodes. // The newly cloned OpaqueLoopInitNodes additionally get 'new_opaque_init_input' as a new input. The other nodes are // not updated. -BoolNode* TemplateAssertionPredicateBool::clone_and_replace_init(Node* new_ctrl, Node* new_init, +BoolNode* TemplateAssertionPredicateBool::clone_update_opaque_init(Node* new_ctrl, Node* new_opaque_init_input, PhaseIdealLoop* phase) { - CloneWithNewInit clone_with_new_init(phase, new_ctrl, new_init); + assert(is_not_dead(), "must not be dead"); + CloneWithNewOpaqueInitInput clone_with_new_opaque_init_input(phase, new_ctrl, new_opaque_init_input); CloneAssertionPredicateBool clone_assertion_predicate_bool(_source_bool, new_ctrl, phase); - return clone_assertion_predicate_bool.clone(&clone_with_new_init); + return clone_assertion_predicate_bool.clone(&clone_with_new_opaque_init_input); } - // The transformations of this class fold the OpaqueLoop* nodes by returning their inputs. -class ReplaceOpaqueLoopNodes : public TransformOpaqueLoopNodes { - Node* _new_init; - Node* _new_stride; +class RemoveOpaqueLoopNodes : public TransformOpaqueLoopNodes { + PhaseIdealLoop* _phase; public: - ReplaceOpaqueLoopNodes(Node* new_init, Node* new_stride) : _new_init(new_init), _new_stride(new_stride) {} - Node* transform_opaque_init(OpaqueLoopInitNode* opaque_init) override { - return _new_init; + return opaque_init->in(1); } Node* transform_opaque_stride(OpaqueLoopStrideNode* opaque_stride) override { - return _new_stride; + return opaque_stride->in(1); } }; // Clones this Template Assertion Predicate bool. This includes all nodes from the BoolNode to the OpaqueLoop* nodes. -// The cloned nodes are not updated. -BoolNode* TemplateAssertionPredicateBool::clone_and_replace_opaque_loop_nodes(Node* new_ctrl, Node* new_init, - Node* new_stride, PhaseIdealLoop* phase) { - ReplaceOpaqueLoopNodes replaceOpaqueLoopNodes(new_init, new_stride); +// The OpaqueLoop* nodes are not cloned but replaced by their input nodes (i.e. folding the OpaqueLoop* nodes away). +BoolNode* TemplateAssertionPredicateBool::clone_and_fold_opaque_loop_nodes(Node* new_ctrl, PhaseIdealLoop* phase) { + assert(is_not_dead(), "must not be dead"); + RemoveOpaqueLoopNodes remove_opaque_loop_nodes; CloneAssertionPredicateBool clone_assertion_predicate_bool(_source_bool, new_ctrl, phase); - return clone_assertion_predicate_bool.clone(&replaceOpaqueLoopNodes); + return clone_assertion_predicate_bool.clone(&remove_opaque_loop_nodes); +} + +// Visitor to visit an OpaqueLoopStride node of a Template Assertion Predicate bool. +class OpaqueLoopStrideVisitor : public StackObj { + public: + virtual void visit(OpaqueLoopStrideNode* opaque_stride) = 0; +}; + +// This visitor replaces the input of OpaqueLoopStride nodes in Template Assertion Predicate bools with a new node. +class ReplaceOpaqueStrideInput : public OpaqueLoopStrideVisitor { + PhaseIterGVN* _igvn; + Node* _new_opaque_stride_input; + + public: + ReplaceOpaqueStrideInput(PhaseIterGVN* igvn, Node* new_opaque_stride_input) + : _igvn(igvn), + _new_opaque_stride_input(new_opaque_stride_input) {} + + void visit(OpaqueLoopStrideNode* opaque_stride) override { + _igvn->replace_input_of(opaque_stride, 1, _new_opaque_stride_input); + } +}; + +// This class looks for OpaqueLoopStride nodes in Template Assertion Predicate bools and visits them. +class OpaqueLoopStrideNodes : public StackObj { + DFSNodeStack _stack; + + public: + OpaqueLoopStrideNodes(BoolNode* template_bool) : _stack(template_bool) {} + + void findAndVisit(OpaqueLoopStrideVisitor* visitor) { + while (_stack.is_not_empty()) { + Node* current = _stack.top(); + if (current->is_OpaqueLoopStride()) { + visitor->visit(current->as_OpaqueLoopStride()); + _stack.pop(); + } else if (!_stack.push_next_unvisited_input()) { + _stack.pop(); + } + } + } +}; + +// Sets 'new_opaque_stride_input' as new input of the OpaqueLoopStride node of this Template Assertion Predicate bool. +void TemplateAssertionPredicateBool::replace_opaque_stride_input(Node* new_opaque_stride_input, PhaseIterGVN* igvn) { + assert(is_not_dead(), "must not be dead"); + ReplaceOpaqueStrideInput replace_opaque_stride_input(igvn, new_opaque_stride_input); + OpaqueLoopStrideNodes opaque_loop_stride_nodes(_source_bool); + opaque_loop_stride_nodes.findAndVisit(&replace_opaque_stride_input); +} + +#ifdef ASSERT +// This visitor asserts that there are no OpaqueLoopStride nodes in Template Assertion Predicate bools. +class VerifyNoOpaqueStride : public OpaqueLoopStrideVisitor { + public: + void visit(OpaqueLoopStrideNode* opaque_stride) override { + assert(false, "should not find OpaqueLoopStrideNode"); + } +}; + +// Visit all nodes of this Template Assertion Predicate bool. Verifies that we do not visit any OpaqueLoopStrideNode. +void TemplateAssertionPredicateBool::verify_no_opaque_stride() { + VerifyNoOpaqueStride verify_no_opaque_stride; + OpaqueLoopStrideNodes opaque_loop_stride_nodes(_source_bool); + opaque_loop_stride_nodes.findAndVisit(&verify_no_opaque_stride); +} +#endif // ASSERT + +TemplateAssertionPredicate +TemplateAssertionPredicate::create_and_init(Node* new_ctrl, BoolNode* new_init_bool, Node* new_last_value, + TemplateAssertionPredicateDataOutput* node_in_target_loop, PhaseIdealLoop* phase) { + TemplateAssertionPredicateNode* cloned_template = _template_assertion_predicate->clone()->as_TemplateAssertionPredicate(); + update_data_dependencies_to_clone(cloned_template, node_in_target_loop, phase); + init_new_template(cloned_template, new_ctrl, new_init_bool, new_last_value, phase); + return { cloned_template->as_TemplateAssertionPredicate() }; +} + +void TemplateAssertionPredicate::init_new_template(TemplateAssertionPredicateNode* cloned_template, Node* new_ctrl, + BoolNode* new_init_bool, Node* new_last_value, + PhaseIdealLoop* phase) { + phase->igvn().replace_input_of(cloned_template, TemplateAssertionPredicateNode::InitValue, new_init_bool); + phase->igvn().replace_input_of(cloned_template, TemplateAssertionPredicateNode::LastValue, new_last_value); + phase->igvn().replace_input_of(cloned_template, 0, new_ctrl); + phase->register_control(cloned_template, phase->get_loop(new_ctrl), new_ctrl); +} + +// Update any data dependencies from this template to the new template if it meets the requirements checked with +// 'node_in_target_loop'. +void TemplateAssertionPredicate::update_data_dependencies_to_clone(TemplateAssertionPredicateNode* cloned_template, + TemplateAssertionPredicateDataOutput* node_in_target_loop, + PhaseIdealLoop* phase) { + for (DUIterator_Fast imax, i = _template_assertion_predicate->fast_outs(imax); i < imax; i++) { + Node* node = _template_assertion_predicate->fast_out(i); + if (!node->is_CFG() && node_in_target_loop->must_update(node)) { + phase->igvn().replace_input_of(node, 0, cloned_template); + --i; + --imax; + } + } +} + +// This class creates a new Initialized Assertion Predicate. +class CreateInitializedAssertionPredicate { + PhaseIdealLoop* _phase; + + // Create a new If or RangeCheck node to represent an Initialized Assertion Predicate and return it. + IfNode* create_if_node(TemplateAssertionPredicateNode* template_assertion_predicate, Node* new_ctrl, BoolNode* new_bool, + IdealLoopTree* loop, AssertionPredicateType assertion_predicate_type) const { + OpaqueAssertionPredicateNode* opaque_assertion_predicate_node = new OpaqueAssertionPredicateNode(new_bool); + _phase->register_new_node(opaque_assertion_predicate_node,new_ctrl); + IfNode* if_node = template_assertion_predicate->create_initialized_assertion_predicate( + new_ctrl, opaque_assertion_predicate_node, assertion_predicate_type); + _phase->register_control(if_node, loop, new_ctrl); + return if_node; + } + + void create_halt_node(IfFalseNode* fail_proj, IdealLoopTree* loop) { + StartNode* start_node = _phase->C->start(); + Node* frame = new ParmNode(start_node, TypeFunc::FramePtr); + _phase->register_new_node(frame, start_node); + Node* halt = new HaltNode(fail_proj, frame, "Assertion Predicate cannot fail"); + _phase->igvn().add_input_to(_phase->C->root(), halt); + _phase->register_control(halt, loop, fail_proj); + } + + // Create the out nodes of a newly created Initialized Assertion Predicate If node which includes the projections and + // the dedicated Halt node. + IfTrueNode* create_if_proj_nodes(IfNode* if_node, IdealLoopTree* loop) { + IfTrueNode* success_proj = new IfTrueNode(if_node); + IfFalseNode* fail_proj = new IfFalseNode(if_node); + _phase->register_control(success_proj, loop, if_node); + _phase->register_control(fail_proj, loop, if_node); + create_halt_node(fail_proj, loop); + return success_proj; + } + public: + explicit CreateInitializedAssertionPredicate(PhaseIdealLoop* phase) : _phase(phase) {} + + InitializedAssertionPredicate create(TemplateAssertionPredicateNode* template_assertion_predicate, Node* new_ctrl, + BoolNode* new_bool, AssertionPredicateType assertion_predicate_type) { + IdealLoopTree* loop = _phase->get_loop(new_ctrl); + IfNode* if_node = create_if_node(template_assertion_predicate, new_ctrl, new_bool, loop, assertion_predicate_type); + return { create_if_proj_nodes(if_node, loop) }; + } +}; + +void TemplateAssertionPredicate::create_initialized_predicate(Node* new_ctrl, PhaseIdealLoop* phase, + TemplateAssertionPredicateBool& template_bool, + AssertionPredicateType assertion_predicate_type, + PredicateInserter& predicate_inserter) { + CreateInitializedAssertionPredicate create_initialized_assertion_predicate(phase); + BoolNode* new_bool = template_bool.clone_and_fold_opaque_loop_nodes(new_ctrl, phase); + InitializedAssertionPredicate initialized_assertion_predicate = + create_initialized_assertion_predicate.create(_template_assertion_predicate, new_ctrl, new_bool, + assertion_predicate_type); + predicate_inserter.insert(initialized_assertion_predicate); +} + +Node* TemplateAssertionPredicateBools::create_last_value(Node* new_ctrl, OpaqueLoopInitNode* opaque_init) { + Node* init_stride = _loop_head->stride(); + Node* opaque_stride = new OpaqueLoopStrideNode(_phase->C, init_stride); + _phase->register_new_node(opaque_stride, new_ctrl); + Node* last_value = new SubINode(opaque_stride, init_stride); + _phase->register_new_node(last_value, new_ctrl); + last_value = new AddINode(opaque_init, last_value); + _phase->register_new_node(last_value, new_ctrl); + // init + (current stride - initial stride) is within the loop so narrow its type by leveraging the type of the iv Phi + last_value = new CastIINode(last_value, _loop_head->phi()->bottom_type()); + _phase->register_new_node(last_value, new_ctrl); + return last_value; +} + +TemplateAssertionPredicateNode* NewTemplateAssertionPredicate::create(const int if_opcode, Node* new_ctrl, const int scale, + Node* offset, Node* range) { + OpaqueLoopInitNode* opaque_init = create_opaque_init(new_ctrl); + + TemplateAssertionPredicateBools template_assertion_predicate_bools(_loop_head, scale, offset, range, _phase); + bool overflow_init_value; + BoolNode* bool_init_value = template_assertion_predicate_bools.create_for_init_value(new_ctrl, opaque_init, + overflow_init_value); + bool overflow_last_value; + BoolNode* bool_last_value = template_assertion_predicate_bools.create_for_last_value(new_ctrl, opaque_init, + overflow_last_value); + + return create_template_assertion_predicate(if_opcode, new_ctrl, overflow_init_value, bool_init_value, + overflow_last_value, bool_last_value); +} + +OpaqueLoopInitNode* NewTemplateAssertionPredicate::create_opaque_init(Node* loop_entry) { + OpaqueLoopInitNode* opaque_init = new OpaqueLoopInitNode(_phase->C, _loop_head->init_trip()); + _phase->register_new_node(opaque_init, loop_entry); + return opaque_init; +} + +TemplateAssertionPredicateNode* +NewTemplateAssertionPredicate::create_template_assertion_predicate(const int if_opcode, Node* new_ctrl, + bool overflow_init_value, BoolNode* bool_init_value, + bool overflow_last_value, BoolNode* bool_last_value) { + TemplateAssertionPredicateNode* template_assertion_predicate_node + = new TemplateAssertionPredicateNode(new_ctrl, bool_init_value, bool_last_value, + overflow_init_value ? Op_If : if_opcode, + overflow_last_value ? Op_If : if_opcode); + _phase->register_control(template_assertion_predicate_node, _phase->get_loop(new_ctrl), new_ctrl); + return template_assertion_predicate_node; +} + +// We have an Initialized Assertion Predicate if the bool input of the IfNode is an OpaqueAssertionPredicate or a ConI +// node (could be found during IGVN when this node is being folded) and we find a HaltNode on the uncommon projection path. +// If an Initialized Assertion Predicate is being folded and has already lost its uncommon projection with the HaltNode, +// (i.e. the IfNode has only the success projection left), then we treat it as Runtime Predicate. +bool InitializedAssertionPredicate::is_success_proj(const Node* success_proj) { + if (success_proj->is_IfTrue()) { + Node* if_node = success_proj->in(0); + if (if_node->is_If() && if_node->outcnt() == 2) { + return has_opaque_or_con(if_node->as_If()) && has_halt(success_proj); + } + } + return false; +} + +// Check if the If node has an OpaqueAssertionPredicate or a ConI node as bool input. The latter case could happen when +// an Initialized Assertion Predicate is about to be folded during IGVN. +bool InitializedAssertionPredicate::has_opaque_or_con(const IfNode* if_node) { + Node* bool_input = if_node->in(1); + return bool_input->is_ConI() || bool_input->Opcode() == Op_OpaqueAssertionPredicate; +} + +// Check if the other projection (UCT projection) of `success_proj` has a Halt node as output. +bool InitializedAssertionPredicate::has_halt(const Node* success_proj) { + ProjNode* other_proj = success_proj->as_IfProj()->other_if_proj(); + return other_proj->outcnt() == 1 && other_proj->unique_out()->Opcode() == Op_Halt; +} + +#ifdef ASSERT +// Check that the block has at most one Parse Predicate and that we only find Regular Predicate nodes (i.e. IfProj, +// If, RangeCheck, or TemplateAssertionPredicate nodes. +void PredicateBlock::verify_block() { + Node* next = _parse_predicate.entry(); // Skip unique Parse Predicate of this block if present + while (next != _entry) { + assert(!next->is_ParsePredicate(), "can only have one Parse Predicate in a block"); + const int opcode = next->Opcode(); + assert(next->is_IfProj() || next->is_TemplateAssertionPredicate() || opcode == Op_If || opcode == Op_RangeCheck, + "Regular Predicates consist of an IfProj and an If or RangeCheck or a TemplateAssertionPredicate node"); + assert(opcode != Op_If || !next->as_If()->is_zero_trip_guard(), "should not be zero trip guard"); + next = next->in(0); + } +} +#endif // ASSERT + +// Walk over all Regular Predicates of this block (if any) and return the first node not belonging to the block +// anymore (i.e. entry to the first Regular Predicate in this block if any or `regular_predicate_proj` otherwise). +Node* PredicateBlock::skip_regular_predicates(Node* regular_predicate_proj, Deoptimization::DeoptReason deopt_reason) { + PredicateVisitor do_nothing_visitor; + RegularPredicateInBlockIterator regular_predicate_in_block_iterator(regular_predicate_proj, deopt_reason, + &do_nothing_visitor); + return regular_predicate_in_block_iterator.for_each(); +} + +// Applies the PredicateVisitor to each Regular Predicate in this block. +Node* PredicateInBlockIterator::for_each() { + Node* entry = _start_node; + if (entry->is_IfTrue() && entry->in(0)->is_ParsePredicate()) { + ParsePredicate parse_predicate(entry, _deopt_reason); + if (parse_predicate.is_valid()) { + _predicate_visitor->visit(parse_predicate); + entry = parse_predicate.entry(); + } else { + // Parse Predicate belonging to a different Predicate Block. + return entry; + } + } + + RegularPredicateInBlockIterator regular_predicate_in_block_iterator(entry, _deopt_reason, _predicate_visitor); + return regular_predicate_in_block_iterator.for_each(); +} + +// Applies the PredicateVisitor to each Regular Predicate in this block. +Node* RegularPredicateInBlockIterator::for_each() { + Node* entry = _start_node; + while (true) { + if (entry->is_TemplateAssertionPredicate()) { + TemplateAssertionPredicate template_assertion_predicate(entry->as_TemplateAssertionPredicate()); + _predicate_visitor->visit(template_assertion_predicate); + entry = template_assertion_predicate.entry(); + } else if (RuntimePredicate::is_success_proj(entry, _deopt_reason)) { + RuntimePredicate runtime_predicate(entry->as_IfProj()); + _predicate_visitor->visit(runtime_predicate); + entry = runtime_predicate.entry(); + } else if (InitializedAssertionPredicate::is_success_proj(entry)) { + InitializedAssertionPredicate initialized_assertion_predicate(entry->as_IfTrue()); + _predicate_visitor->visit(initialized_assertion_predicate); + entry = initialized_assertion_predicate.entry(); + } else { + // Either a Parse Predicate or not a Regular Predicate. In both cases, the node does not belong to this block. + break; + } + } + return entry; +} + +// Applies the PredicateVisitor to each predicate for this loop. +void PredicatesForLoop::for_each() { + Node* entry_to_block = for_each(_start_node, Deoptimization::Reason_loop_limit_check); + + if (UseLoopPredicate) { + if (UseProfiledLoopPredicate) { + entry_to_block = for_each(entry_to_block, Deoptimization::Reason_profile_predicate); + } + for_each(entry_to_block, Deoptimization::Reason_predicate); + } +} + +Node* PredicatesForLoop::for_each(Node* current, Deoptimization::DeoptReason deopt_reason) { + PredicateInBlockIterator predicate_in_block_iterator(current, deopt_reason, _predicate_visitor); + return predicate_in_block_iterator.for_each(); +} + +PredicateEntryIterator::PredicateEntryIterator(Node* start) + : _current(start) {} + +// Is current node pointed at by iterator a predicate tail? +bool PredicateEntryIterator::has_next() const { + if (_current->is_TemplateAssertionPredicate()) { + return true; + } else if (_current->is_IfProj()) { + IfNode* if_node = _current->in(0)->as_If(); + return (if_node->is_ParsePredicate() || + RuntimePredicate::is_success_proj(_current) || + InitializedAssertionPredicate::is_success_proj(_current)); + } + return false; +} + +// Skip the current predicate pointed at by iterator by returning the input into the predicate. This could possibly be +// a non-predicate node. +Node* PredicateEntryIterator::next_predicate_entry() { + assert(has_next(), "current must be predicate"); + if (_current->is_TemplateAssertionPredicate()) { + _current = _current->in(0); + } else { + _current = _current->in(0)->in(0); + } + return _current; } diff --git a/src/hotspot/share/opto/predicates.hpp b/src/hotspot/share/opto/predicates.hpp index 2c891a568273f..0a30f9e5425e7 100644 --- a/src/hotspot/share/opto/predicates.hpp +++ b/src/hotspot/share/opto/predicates.hpp @@ -153,69 +153,95 @@ * * * Initially, before applying any loop-splitting optimizations, we find the following structure after Loop Predication - * (predicates inside square brackets [] do not need to exist if there are no checks to hoist): + * (predicates inside square brackets [] do not need to exist if there are no checks to hoist or if the hoisted check + * is not a range check and does not need a Template Assertion Predicate): * - * [Loop Predicate 1 + two Template Assertion Predicates] \ - * [Loop Predicate 2 + two Template Assertion Predicates] | - * ... | Loop Predicate Block - * [Loop Predicate n + two Template Assertion Predicates] | - * Loop Parse Predicate / + * [Loop Predicate 1 [+ Template Assertion Predicate 1]] \ + * [Loop Predicate 2 [+ Template Assertion Predicate 2]] | + * ... | Loop Predicate Block + * [Loop Predicate n [+ Template Assertion Predicate n]] | + * Loop Parse Predicate / * - * [Profiled Loop Predicate 1 + two Template Assertion Predicates] \ - * [Profiled Loop Predicate 2 + two Template Assertion Predicates] | Profiled Loop - * ... | Predicate Block - * [Profiled Loop Predicate m + two Template Assertion Predicates] | - * Profiled Loop Parse Predicate / + * [Profiled Loop Predicate 1 [+ Template Assertion Predicate 1]] \ + * [Profiled Loop Predicate 2 [+ Template Assertion Predicate 2]] | Profiled Loop + * ... | Predicate Block + * [Profiled Loop Predicate m [+ Template Assertion Predicate n]] | + * Profiled Loop Parse Predicate / * - * [Loop Limit Check Predicate] (at most one) \ Loop Limit Check - * Loop Limit Check Parse Predicate / Predicate Block + * [Loop Limit Check Predicate] (at most one) \ Loop Limit Check + * Loop Limit Check Parse Predicate / Predicate Block * Loop Head * * As an example, let's look at how the predicate structure looks for the main-loop after creating pre/main/post loops * and applying Range Check Elimination (the order is insignificant): * * Main Loop entry (zero-trip) guard - * [For Loop Predicate 1: Two Template + two Initialized Assertion Predicates] - * [For Loop Predicate 2: Two Template + two Initialized Assertion Predicates] + * [For Loop Predicate 1: Template + two Initialized Assertion Predicates] + * [For Loop Predicate 2: Template + two Initialized Assertion Predicates] * ... - * [For Loop Predicate n: Two Template + two Initialized Assertion Predicates] + * [For Loop Predicate n: Template + two Initialized Assertion Predicates] * - * [For Profiled Loop Predicate 1: Two Template + two Initialized Assertion Predicates] - * [For Profiled Loop Predicate 2: Two Template + two Initialized Assertion Predicates] + * [For Profiled Loop Predicate 1: Template + two Initialized Assertion Predicates] + * [For Profiled Loop Predicate 2: Template + two Initialized Assertion Predicates] * ... - * [For Profiled Loop Predicate m: Two Template + two Initialized Assertion Predicates] + * [For Profiled Loop Predicate m: Template + two Initialized Assertion Predicates] * - * (after unrolling, we have two Initialized Assertion Predicates for the Assertion Predicates of Range Check Elimination) - * [For Range Check Elimination Check 1: Two Templates + one Initialized Assertion Predicate] - * [For Range Check Elimination Check 2: Two Templates + one Initialized Assertion Predicate] + * [For Range Check Elimination Check 1: Template + two Initialized Assertion Predicate] + * [For Range Check Elimination Check 2: Template + two Initialized Assertion Predicate] * ... - * [For Range Check Elimination Check k: Two Templates + one Initialized Assertion Predicate] + * [For Range Check Elimination Check k: Template + two Initialized Assertion Predicate] * Main Loop Head */ +class InitializedAssertionPredicate; +class ParsePredicate; +class Predicates; +class PredicateVisitor; +class RuntimePredicate; +class TemplateAssertionPredicate; + +enum class AssertionPredicateType { + None, + Init_value, + Last_value +}; -// Class to represent Assertion Predicates with a HaltNode instead of an UCT (i.e. either an Initialized Assertion -// Predicate or a Template Assertion Predicate created after the initial one at Loop Predication). -class AssertionPredicatesWithHalt : public StackObj { - Node* _entry; - - static Node* find_entry(Node* start_proj); - static bool has_opaque4(const Node* predicate_proj); - static bool has_halt(const Node* success_proj); - static bool is_assertion_predicate_success_proj(const Node* predicate_proj); +// Interface to represent a C2 predicate. A predicate is either represented by a single CFG node or an If/IfProjs pair. +class Predicate : public StackObj { + public: + // Return the unique entry CFG node into the predicate. + virtual Node* entry() const = 0; + + // Return the head node of the predicate which is either: + // - The single CFG node if the predicate has only a single CFG node (i.e. Template Assertion Predicate) + // - The If node if the predicate is an If/IfProjs pair. + virtual Node* head() const = 0; + + // Return the tail node of the predicate which is either: + // - The single CFG node if the predicate has only a single CFG node (i.e. Template Assertion Predicate) + // - The IfProj success node if the predicate is an If/IfProjs pair. + virtual Node* tail() const = 0; +}; +// Interface to create a new Parse Predicate (clone of an old Parse Predicate) for either the fast or slow loop. +// The fast loop needs to create some additional clones while the slow loop can reuse old nodes. +class NewParsePredicate : public StackObj { public: - AssertionPredicatesWithHalt(Node* assertion_predicate_proj) : _entry(find_entry(assertion_predicate_proj)) {} + virtual ParsePredicateSuccessProj* create(PhaseIdealLoop* phase, Node* new_entry, + ParsePredicateSuccessProj* old_parse_predicate_success_proj) = 0; +}; - // Returns the control input node into the first assertion predicate If. If there are no assertion predicates, it - // returns the same node initially passed to the constructor. - Node* entry() const { - return _entry; - } +// Generic predicate visitor that does nothing. Subclass this visitor to add customized actions for each predicate. +class PredicateVisitor : StackObj { + public: + virtual void visit(TemplateAssertionPredicate& template_assertion_predicate) {} + virtual void visit(ParsePredicate& parse_predicate) {} + virtual void visit(RuntimePredicate& runtime_predicate) {} + virtual void visit(InitializedAssertionPredicate& initialized_assertion_predicate) {} }; // Class to represent a Parse Predicate. -class ParsePredicate : public StackObj { +class ParsePredicate : public Predicate { ParsePredicateSuccessProj* _success_proj; ParsePredicateNode* _parse_predicate_node; Node* _entry; @@ -235,7 +261,7 @@ class ParsePredicate : public StackObj { // Returns the control input node into this Parse Predicate if it is valid. Otherwise, it returns the passed node // into the constructor of this class. - Node* entry() const { + Node* entry() const override { return _entry; } @@ -245,24 +271,117 @@ class ParsePredicate : public StackObj { return _parse_predicate_node != nullptr; } - ParsePredicateNode* node() const { + ParsePredicateNode* head() const override { assert(is_valid(), "must be valid"); return _parse_predicate_node; } - ParsePredicateSuccessProj* success_proj() const { + ParsePredicateSuccessProj* tail() const override { assert(is_valid(), "must be valid"); return _success_proj; } + + ParsePredicate clone(Node* new_ctrl, NewParsePredicate* new_parse_predicate, PhaseIdealLoop* phase) { + ParsePredicateSuccessProj* success_proj = new_parse_predicate->create(phase, new_ctrl, _success_proj); + ParsePredicateNode* new_parse_predicate_node = success_proj->in(0)->as_ParsePredicate(); +#ifdef ASSERT + assert(_parse_predicate_node->uncommon_trap() == new_parse_predicate_node->uncommon_trap(), "same uncommon trap"); + assert(_parse_predicate_node->deopt_reason() == new_parse_predicate_node->deopt_reason(), "same deopt reason trap"); +#endif // ASSERT + return { success_proj, new_parse_predicate_node->deopt_reason() }; + } + + // Kill this Parse Predicate by marking it as useless. The Parse Predicate will be removed during the next round of IGVN. + void kill(PhaseIterGVN* igvn) { + _parse_predicate_node->mark_useless(); + igvn->_worklist.push(_parse_predicate_node); + } }; -// Utility class for queries on Runtime Predicates. -class RuntimePredicate : public StackObj { +// Eliminate all useless Parse Predicates by marking them as useless and adding them to the IGVN worklist. These are +// then removed in the next IGVN round. +class EliminateUselessParsePredicates : public StackObj { + Compile* C; + PhaseIterGVN* _igvn; + IdealLoopTree* _ltree_root; + + void mark_all_parse_predicates_useless(); + static void mark_parse_predicates_useful(IdealLoopTree* loop); + void add_useless_predicates_to_igvn_worklist(); + + public: + EliminateUselessParsePredicates(PhaseIterGVN* igvn, IdealLoopTree* ltree_root) + : C(igvn->C), + _igvn(igvn), + _ltree_root(ltree_root) {} + + void eliminate(); +}; + +// Class to represent a Runtime Predicate. +class RuntimePredicate : public Predicate { + IfProjNode* _success_proj; + IfNode* _if_node; + static Deoptimization::DeoptReason uncommon_trap_reason(IfProjNode* if_proj); static bool may_be_runtime_predicate_if(Node* node); + static bool is_being_folded_without_uncommon_proj(const IfProjNode* success_proj); public: - static bool is_success_proj(Node* node, Deoptimization::DeoptReason deopt_reason); + explicit RuntimePredicate(IfProjNode* success_proj) + : _success_proj(success_proj), + _if_node(success_proj->in(0)->as_If()) { + assert(is_success_proj(success_proj), "must be valid"); + } + + Node* entry() const override { + return _if_node->in(0); + } + + IfNode* head() const override { + return _if_node; + } + + Node* tail() const override { + return _success_proj; + } + + static bool is_success_proj(Node* maybe_success_proj); + static bool is_success_proj(Node* maybe_success_proj, Deoptimization::DeoptReason deopt_reason); +}; + +// This class can be used to insert predicates in the graph above a loop. It can either insert new predicates or skip +// over existing predicates in the graph. The class acts as an iterator and always maintains a link to the current +// control output where a new predicate could possibly be inserted. +class PredicateInserter : public StackObj { + Node* _ctrl_out; // The current control output if a new predicate is inserted in the graph. + PhaseIdealLoop* _phase; + + public: + PredicateInserter(LoopNode* loop_node, PhaseIdealLoop* phase) : _ctrl_out(loop_node->skip_strip_mined()), _phase(phase) {} + + void insert(Predicate& new_predicate); + void skip(Predicate& existing_predicate); +}; + +// The class offers different actions for Assertion Predicates belonging to a source loop. +class AssertionPredicates : public StackObj { + CountedLoopNode* _source_loop_head; + PhaseIdealLoop* _phase; + + TemplateAssertionPredicate create_new_template(int if_opcode, int scale, Node* offset, Node* range, + PredicateInserter& predicate_inserter); + Node* create_stride(int stride_con); + + public: + AssertionPredicates(CountedLoopNode* source_loop_head, PhaseIdealLoop* phase) + : _source_loop_head(source_loop_head), + _phase(phase) {} + + void clone_to_loop(CountedLoopNode* target_loop_head, TemplateAssertionPredicateDataOutput* node_in_target_loop); + void move_to_loop(CountedLoopNode* target_loop_head, TemplateAssertionPredicateDataOutput* node_in_target_loop); + void create(int if_opcode, int scale, Node* offset, Node* range); + void update(int new_stride_con); }; // Utility class for querying Assertion Predicate bool opcodes. @@ -299,14 +418,248 @@ class TemplateAssertionPredicateBool : public StackObj { public: explicit TemplateAssertionPredicateBool(Node* source_bool); + // The last value bool could already be a constant (i.e. dead) when the CastII node between the Template Assertion + // Predicate and the OpaqueLoop* nodes was replaced by a constant. In this case, _source_bool is null. + bool is_not_dead() const { + return _source_bool != nullptr; + } + BoolNode* clone(Node* new_ctrl, PhaseIdealLoop* phase); - BoolNode* clone_and_replace_init(Node* new_ctrl, Node* new_init, PhaseIdealLoop* phase); - BoolNode* clone_and_replace_opaque_loop_nodes(Node* new_ctrl, Node* new_init, Node* new_stride, PhaseIdealLoop* phase); + BoolNode* clone_update_opaque_init(Node* new_ctrl, Node* new_opaque_init_input, PhaseIdealLoop* phase); + BoolNode* clone_and_fold_opaque_loop_nodes(Node* new_ctrl, PhaseIdealLoop* phase); + void replace_opaque_stride_input(Node* new_opaque_stride_input, PhaseIterGVN* igvn); + DEBUG_ONLY(void verify_no_opaque_stride()); +}; + +// Class to represent a Template Assertion Predicate. +class TemplateAssertionPredicate : public Predicate { + TemplateAssertionPredicateNode* _template_assertion_predicate; + TemplateAssertionPredicateBool _init_value_bool; + TemplateAssertionPredicateBool _last_value_bool; + + TemplateAssertionPredicate create_and_init(Node* new_ctrl, BoolNode* new_init_bool, Node* new_last_value, + TemplateAssertionPredicateDataOutput* node_in_target_loop, PhaseIdealLoop* phase); + static void init_new_template(TemplateAssertionPredicateNode* cloned_template, Node* new_ctrl, BoolNode* new_init_bool, + Node* new_last_value, PhaseIdealLoop* phase); + void update_data_dependencies_to_clone(TemplateAssertionPredicateNode* cloned_template_assertion_predicate, + TemplateAssertionPredicateDataOutput* node_in_target_loop, PhaseIdealLoop* phase); + void create_initialized_predicate(Node* new_ctrl, PhaseIdealLoop* phase, TemplateAssertionPredicateBool &template_bool, + AssertionPredicateType assertion_predicate_type, PredicateInserter& predicate_inserter); + + public: + TemplateAssertionPredicate(TemplateAssertionPredicateNode* template_assertion_predicate) + : _template_assertion_predicate(template_assertion_predicate), + _init_value_bool(TemplateAssertionPredicateBool(template_assertion_predicate->in(TemplateAssertionPredicateNode::InitValue))), + _last_value_bool(TemplateAssertionPredicateBool(template_assertion_predicate->in(TemplateAssertionPredicateNode::LastValue))) + {} + + Node* entry() const override { + return _template_assertion_predicate->in(0); + } + + TemplateAssertionPredicateNode* head() const override { + return _template_assertion_predicate; + } + + TemplateAssertionPredicateNode* tail() const override { + return _template_assertion_predicate; + } + + // Clones this Template Assertion Predicate which will also clone its bools. The cloned nodes are not updated in any way. + TemplateAssertionPredicate clone(Node* new_ctrl, TemplateAssertionPredicateDataOutput* node_in_target_loop, PhaseIdealLoop* phase) { + BoolNode* new_init_bool = _init_value_bool.clone(new_ctrl, phase); + Node* new_last_value; + if (_last_value_bool.is_not_dead()) { + new_last_value = _last_value_bool.clone(new_ctrl, phase); + } else { + new_last_value = phase->igvn().intcon(1); + } + return create_and_init(new_ctrl, new_init_bool, new_last_value, node_in_target_loop, phase); + } + + // Same as clone() but the cloned OpaqueLoopInitNode nodes will get 'new_opaque_cloned nodes' as new input. + TemplateAssertionPredicate clone_update_opaque_init(Node* new_ctrl, Node* new_opaque_init_input, + TemplateAssertionPredicateDataOutput* node_in_target_loop, + PhaseIdealLoop* phase) { + BoolNode* new_init_bool = _init_value_bool.clone_update_opaque_init(new_ctrl, new_opaque_init_input, phase); + Node* new_last_value; + if (_last_value_bool.is_not_dead()) { + new_last_value = _last_value_bool.clone_update_opaque_init(new_ctrl, new_opaque_init_input, phase); + } else { + new_last_value = phase->igvn().intcon(1); + } + return create_and_init(new_ctrl, new_init_bool, new_last_value, node_in_target_loop, phase); + } + + // Replace the input of the OpaqueLoopStrideNode of the last value bool of this Template Assertion Predicate with + // the provided 'new_opaque_stride_input'. + void replace_opaque_stride_input(Node* new_opaque_stride_input, PhaseIterGVN* igvn) { + // Only last value bool has OpaqueLoopStrideNode. + DEBUG_ONLY(_init_value_bool.verify_no_opaque_stride()); + if (_last_value_bool.is_not_dead()) { + _last_value_bool.replace_opaque_stride_input(new_opaque_stride_input, igvn); + } + } + + // Create an Initialized Assertion Predicate from this Template Assertion Predicate for the init and the last value. + // This is done by cloning the Template Assertion Predicate bools and removing the OpaqueLoop* nodes (i.e. folding + // them away and using their inputs instead). + void initialize(PhaseIdealLoop* phase, PredicateInserter& predicate_inserter) { + Node* new_ctrl = entry(); + if (_last_value_bool.is_not_dead()) { + create_initialized_predicate(new_ctrl, phase, _last_value_bool, AssertionPredicateType::Last_value, + predicate_inserter); + } + create_initialized_predicate(new_ctrl, phase, _init_value_bool, AssertionPredicateType::Init_value, predicate_inserter); + } + + // Kill this Template Assertion Predicate by marking it as useless. The Template Assertion Predicate will be removed + // during the next round of IGVN. + void kill(PhaseIterGVN* igvn) { + _template_assertion_predicate->mark_useless(); + igvn->_worklist.push(_template_assertion_predicate); + } +}; + +// Class to create bool nodes for a new Template Assertion Predicate. +class TemplateAssertionPredicateBools : public StackObj { + PhaseIdealLoop* _phase; + CountedLoopNode* _loop_head; + jint _stride; + int _scale; + Node* _offset; + Node* _range; + bool _upper; + + Node* create_last_value(Node* new_ctrl, OpaqueLoopInitNode* opaque_init); + + public: + TemplateAssertionPredicateBools(CountedLoopNode* loop_head, int scale, Node* offset, Node* range, + PhaseIdealLoop* phase) + : _phase(phase), + _loop_head(loop_head), + _stride(_loop_head->stride()->get_int()), + _scale(scale), + _offset(offset), + _range(range), + _upper((_stride > 0) != (_scale > 0)) // Make sure rc_predicate() chooses "scale*init + offset" case. + {} + + BoolNode* create_for_init_value(Node* new_ctrl, OpaqueLoopInitNode* opaque_init, bool& overflow) { + return _phase->rc_predicate(new_ctrl, _scale, _offset, opaque_init, nullptr, _stride, _range, _upper, + overflow); + } + + BoolNode* create_for_last_value(Node* new_ctrl, OpaqueLoopInitNode* opaque_init, bool& overflow) { + Node* last_value = create_last_value(new_ctrl, opaque_init); + return _phase->rc_predicate(new_ctrl, _scale, _offset, last_value, nullptr, _stride, _range, _upper, + overflow); + } +}; + +// Class to create a new Template Assertion Predicate and insert it into the graph. +class NewTemplateAssertionPredicate : public StackObj { + CountedLoopNode* _loop_head; + PhaseIdealLoop* _phase; + + OpaqueLoopInitNode* create_opaque_init(Node* loop_entry); + TemplateAssertionPredicateNode* create_template_assertion_predicate(int if_opcode, Node* loop_entry, + bool overflow_init_value, BoolNode* bool_init_value, + bool overflow_last_value, BoolNode* bool_last_value); + + public: + explicit NewTemplateAssertionPredicate(CountedLoopNode* loop_head, PhaseIdealLoop* phase) + : _loop_head(loop_head), + _phase(phase) {} + + TemplateAssertionPredicateNode* create(int if_opcode, Node* new_ctrl, int scale, Node* offset, Node* range); + +}; + +// Interface to check if an output data node of a Template Assertion Predicate node must be updated to the newly cloned +// Template Assertion Predicate node. This decision is done based on whether the output node belongs to the original, +// not yet cloned loop body or not. This can be achieved by comparing node indices and/or old->new mappings. +class TemplateAssertionPredicateDataOutput : public StackObj { + public: + virtual bool must_update(Node* output_data_node) = 0; +}; + +// This class returns true if the output data node is part of the cloned loop body. +class NodeInClonedLoop : public TemplateAssertionPredicateDataOutput { + uint _first_node_index_in_cloned_loop; + + public: + explicit NodeInClonedLoop(uint first_node_index_in_cloned_loop) + : _first_node_index_in_cloned_loop(first_node_index_in_cloned_loop) {} + + bool must_update(Node* output_data_node) override { + return output_data_node->_idx >= _first_node_index_in_cloned_loop; + } +}; + +// This class returns true if the output data node belongs to the original loop body. +class NodeInOriginalLoop : public TemplateAssertionPredicateDataOutput { + uint _first_node_index_in_cloned_loop; + Node_List* _old_new; + + public: + explicit NodeInOriginalLoop(uint first_node_index_in_cloned_loop, Node_List* old_new) + : _first_node_index_in_cloned_loop(first_node_index_in_cloned_loop), + _old_new(old_new) {} + + // Check if 'output_data_node' is not a cloned node (i.e. < _first_node_index_in_cloned_loop) and if we've created a + // clone from it (with _old_new). If there is a clone, we know that 'output_data_node' belongs to the original loop. + bool must_update(Node* output_data_node) override { + if (output_data_node->_idx < _first_node_index_in_cloned_loop) { + Node* cloned_node = (*_old_new)[output_data_node->_idx]; + return cloned_node != nullptr && cloned_node->_idx >= _first_node_index_in_cloned_loop; + } else { + return false; + } + } +}; + +// Class to represent an Initialized Assertion Predicate. +class InitializedAssertionPredicate : public Predicate { + IfTrueNode* _success_proj; + IfNode* _if_node; + + static bool has_opaque_or_con(const IfNode* if_node); + static bool has_halt(const Node* success_proj); + + public: + InitializedAssertionPredicate(IfTrueNode* success_proj) + : _success_proj(success_proj), + _if_node(success_proj->in(0)->as_If()) {} + + Node* entry() const override { + return _if_node->in(0); + } + + IfNode* head() const override { + return _if_node; + } + + IfTrueNode* tail() const override { + return _success_proj; + }; + + static bool is_success_proj(const Node* success_proj); + + // Kill this Initialized Assertion Predicate by setting the bool input of the If node representing it to true. + // The Initialized Assertion Predicate will be removed during the next round of IGVN. + void kill(PhaseIterGVN* igvn) { + igvn->replace_input_of(_if_node, 1, igvn->intcon(1)); + } }; // This class represents a Predicate Block (i.e. either a Loop Predicate Block, a Profiled Loop Predicate Block, // or a Loop Limit Check Predicate Block). It contains zero or more Regular Predicates followed by a Parse Predicate // which, however, does not need to exist (we could already have decided to remove Parse Predicates for this loop). +// Use this class for queries about the predicates in this block. For iteration inside a Predicate Block, use +// BlockPredicateIterator. +// For a loop that was split from another loop, we will only find Assertion Predicates. We group them together as a +// single Predicate Block without a Parse Predicate (Assertion Predicates cannot be mapped to an uncommon trap). class PredicateBlock : public StackObj { ParsePredicate _parse_predicate; // Could be missing. Node* _entry; @@ -322,11 +675,12 @@ class PredicateBlock : public StackObj { } // Returns the control input node into this Regular Predicate block. This is either: - // - The control input to the first If node in the block representing a Runtime Predicate if there is at least one - // Runtime Predicate. - // - The control input node into the ParsePredicate node if there is only a Parse Predicate and no Runtime Predicate. - // - The same node initially passed to the constructor if this Regular Predicate block is empty (i.e. no Parse - // Predicate or Runtime Predicate). + // - The control input to the first If node in the block representing a Regular Predicate if we've created at least one + // Runtime Predicate during Loop Predication (could be a Runtime Predicate or a Template Assertion Predicate if + // we've already folded the Runtime Predicate away). + // - The control input node to the Parse Predicate if there is only a Parse Predicate and no Regular Predicate. + // - The same node initially passed to the constructor if this Predicate block is empty (i.e. no Parse or Regular + // Predicate). Node* entry() const { return _entry; } @@ -339,44 +693,34 @@ class PredicateBlock : public StackObj { return _parse_predicate.is_valid(); } - ParsePredicateNode* parse_predicate() const { - return _parse_predicate.node(); - } - - ParsePredicateSuccessProj* parse_predicate_success_proj() const { - return _parse_predicate.success_proj(); - } - bool has_runtime_predicates() const { return _parse_predicate.entry() != _entry; } - // Returns either: - // - The entry to the Parse Predicate if present. - // - The last Runtime Predicate success projection if Parse Predicate is not present. - // - The entry to this Regular Predicate Block if the block is empty. - Node* skip_parse_predicate() const { - return _parse_predicate.entry(); + ParsePredicateSuccessProj* parse_predicate_success_proj() const { + assert(has_parse_predicate(), "must be valid"); + return _parse_predicate.tail(); } }; -// This class takes a loop entry node and finds all the available predicates for the loop. +// This class represents all predicates before a loop. Use this class for queries about the predicates in this block. +// For iteration, either use the class PredicatesForLoop. class Predicates : public StackObj { - Node* _loop_entry; PredicateBlock _loop_limit_check_predicate_block; PredicateBlock _profiled_loop_predicate_block; PredicateBlock _loop_predicate_block; Node* _entry; public: - Predicates(Node* loop_entry) - : _loop_entry(loop_entry), - _loop_limit_check_predicate_block(loop_entry, Deoptimization::Reason_loop_limit_check), + explicit Predicates(Node* loop_entry) + : _loop_limit_check_predicate_block(loop_entry, Deoptimization::Reason_loop_limit_check), _profiled_loop_predicate_block(_loop_limit_check_predicate_block.entry(), Deoptimization::Reason_profile_predicate), _loop_predicate_block(_profiled_loop_predicate_block.entry(), Deoptimization::Reason_predicate), - _entry(_loop_predicate_block.entry()) {} + _entry(_loop_predicate_block.entry()) { + assert(loop_entry != nullptr, "must not be null"); + } // Returns the control input the first predicate if there are any predicates. If there are no predicates, the same // node initially passed to the constructor is returned. @@ -395,24 +739,90 @@ class Predicates : public StackObj { const PredicateBlock* loop_limit_check_predicate_block() const { return &_loop_limit_check_predicate_block; } +}; - bool has_any() const { - return _entry != _loop_entry; - } +// Iterator that applies a PredicateVisitor to each Regular Predicate in the block specified by the deopt_reason. +class RegularPredicateInBlockIterator : public StackObj { + Deoptimization::DeoptReason _deopt_reason; + Node* _start_node; + PredicateVisitor* _predicate_visitor; + + public: + RegularPredicateInBlockIterator(Node* start_node, Deoptimization::DeoptReason deopt_reason, + PredicateVisitor* predicate_visitor) + : _deopt_reason(deopt_reason), + _start_node(start_node), + _predicate_visitor(predicate_visitor) {} + + Node* for_each(); +}; + +// Iterator that applies a PredicateVisitor to each predicate in the block specified by the deopt_reason. +class PredicateInBlockIterator : public StackObj { + Node* _start_node; + Deoptimization::DeoptReason _deopt_reason; + PredicateVisitor* _predicate_visitor; + + public: + PredicateInBlockIterator(Node* start_node, Deoptimization::DeoptReason deopt_reason, PredicateVisitor* predicate_visitor) + : _start_node(start_node), + _deopt_reason(deopt_reason), + _predicate_visitor(predicate_visitor) {} + + Node* for_each(); +}; + +// Predicate iterator that applies a PredicateVisitor to each predicate belonging to the same loop to which the passed +// node belongs to. +class PredicatesForLoop : public StackObj { + Node* _start_node; + PredicateVisitor* _predicate_visitor; + + Node* for_each(Node* next, Deoptimization::DeoptReason deopt_reason); + + public: + PredicatesForLoop(Node* start_node, PredicateVisitor* predicate_visitor) + : _start_node(start_node), + _predicate_visitor(predicate_visitor) {} + + void for_each(); }; -// This class iterates over the Parse Predicates of a loop. -class ParsePredicateIterator : public StackObj { - GrowableArray _parse_predicates; - int _current_index; +// Special predicate iterator that can be used to walk through predicates, regardless if they all belong to the same +// loop or not (i.e. leftovers from already folded nodes). The iterator always returns the next entry to a +// predicate. +class PredicateEntryIterator : public StackObj { + Node* _current; public: - ParsePredicateIterator(const Predicates& predicates); + PredicateEntryIterator(Node* start); - bool has_next() const { - return _current_index < _parse_predicates.length(); + bool has_next() const; + Node* next_predicate_entry(); +}; + +#ifdef ASSERT +// This class verifies that there are no Parse Predicates and no Runtime Predicates. +class VerifyOnlyAssertionPredicates : public PredicateVisitor { + public: + using PredicateVisitor::visit; + + void visit(ParsePredicate& parse_predicate) override { + parse_predicate.head()->dump(); + assert(false, "should not find Parse Predicate"); } - ParsePredicateNode* next(); + void visit(RuntimePredicate& runtime_predicate) override { + runtime_predicate.head()->dump(); + assert(false, "should not find Runtime Predicate"); + } + + static void verify(Node* loop_entry) { + VerifyOnlyAssertionPredicates verify_only_assertion_predicates; + PredicatesForLoop predicates_for_loop(loop_entry, &verify_only_assertion_predicates); + predicates_for_loop.for_each(); + } }; +#endif // ASSERT + #endif // SHARE_OPTO_PREDICATES_HPP diff --git a/src/hotspot/share/opto/split_if.cpp b/src/hotspot/share/opto/split_if.cpp index 6dd96883b535d..f10d1b9dd2771 100644 --- a/src/hotspot/share/opto/split_if.cpp +++ b/src/hotspot/share/opto/split_if.cpp @@ -322,7 +322,7 @@ bool PhaseIdealLoop::clone_cmp_down(Node* n, const Node* blk1, const Node* blk2) assert( bol->is_Bool(), "" ); if (bol->outcnt() == 1) { Node* use = bol->unique_out(); - if (use->Opcode() == Op_Opaque4) { + if (use->Opcode() == Op_Opaque4 || use->Opcode() == Op_OpaqueAssertionPredicate) { if (use->outcnt() == 1) { Node* iff = use->unique_out(); assert(iff->is_If(), "unexpected node type"); @@ -333,8 +333,9 @@ bool PhaseIdealLoop::clone_cmp_down(Node* n, const Node* blk1, const Node* blk2) } } else { // We might see an Opaque1 from a loop limit check here - assert(use->is_If() || use->is_CMove() || use->Opcode() == Op_Opaque1 || use->is_AllocateArray(), "unexpected node type"); - Node *use_c = (use->is_If() || use->is_AllocateArray()) ? use->in(0) : get_ctrl(use); + bool is_control_use = use->is_If() || use->is_AllocateArray() || use->is_TemplateAssertionPredicate(); + assert(is_control_use || use->is_CMove() || use->Opcode() == Op_Opaque1, "unexpected node type"); + Node *use_c = (is_control_use) ? use->in(0) : get_ctrl(use); if (use_c == blk1 || use_c == blk2) { assert(use->is_CMove(), "unexpected node type"); continue; @@ -352,7 +353,20 @@ bool PhaseIdealLoop::clone_cmp_down(Node* n, const Node* blk1, const Node* blk2) for (DUIterator j = bol->outs(); bol->has_out(j); j++) { Node* u = bol->out(j); // Uses are either IfNodes, CMoves or Opaque4 - if (u->Opcode() == Op_Opaque4) { + if (u->is_TemplateAssertionPredicate()) { + TemplateAssertionPredicateNode* template_assertion_predicate = u->as_TemplateAssertionPredicate(); + assert(bol->outcnt() == 1, "must be unique"); + Node* cloned_bool = bol->clone(); + uint template_bool_input; + if (template_assertion_predicate->in(TemplateAssertionPredicateNode::InitValue) == bol) { + template_bool_input = TemplateAssertionPredicateNode::InitValue; + } else { + template_bool_input = TemplateAssertionPredicateNode::LastValue; + } + register_new_node(cloned_bool, template_assertion_predicate); + _igvn.replace_input_of(template_assertion_predicate, template_bool_input, cloned_bool); + --j; + } else if (u->Opcode() == Op_Opaque4 || u->Opcode() == Op_OpaqueAssertionPredicate) { assert(u->in(1) == bol, "bad input"); for (DUIterator_Last kmin, k = u->last_outs(kmin); k >= kmin; --k) { Node* iff = u->last_out(k); @@ -454,8 +468,8 @@ class CloneTemplateAssertionPredicateBoolDown { for (uint i = 0; i < list.size(); i++) { Node* next = list.at(i); - if (is_template_assertion_predicate(next)) { - clone_template_assertion_predicate_bool_and_replace(next); + if (next->is_TemplateAssertionPredicate()) { + clone_template_assertion_predicate_bool_and_replace(next->as_TemplateAssertionPredicate(), n->as_Bool()); } else { assert(!next->is_CFG(), "no CFG expected in Template Assertion Predicate bool outputs"); push_outputs(list, next); @@ -463,16 +477,13 @@ class CloneTemplateAssertionPredicateBoolDown { } } - static bool is_template_assertion_predicate(Node* n) { - return n->is_If() && n->in(1)->Opcode() == Op_Opaque4; - } - - void clone_template_assertion_predicate_bool_and_replace(Node* template_assertion_predicate) { + void clone_template_assertion_predicate_bool_and_replace(TemplateAssertionPredicateNode* template_assertion_predicate, + BoolNode* template_assertion_predicate_bool_node) { Node* new_ctrl = template_assertion_predicate->in(0); - Node* opaque_node = template_assertion_predicate->in(1); - TemplateAssertionPredicateBool template_assertion_predicate_bool(opaque_node->in(1)); + TemplateAssertionPredicateBool template_assertion_predicate_bool(template_assertion_predicate_bool_node); BoolNode* cloned_bool = template_assertion_predicate_bool.clone(new_ctrl, _phase); - _phase->igvn().replace_input_of(opaque_node, 1, cloned_bool); + const uint bool_index = template_assertion_predicate->index_for_bool_input(template_assertion_predicate_bool_node); + _phase->igvn().replace_input_of(template_assertion_predicate, bool_index, cloned_bool); } static void push_outputs(Unique_Node_List& list, const Node* n) {