Full fix

src/hotspot/share/opto/cfgnode.cpp

@@ -39,6 +39,7 @@
 #include "opto/narrowptrnode.hpp"
 #include "opto/mulnode.hpp"
 #include "opto/phaseX.hpp"
+#include "opto/predicates.hpp"
 #include "opto/regalloc.hpp"
 #include "opto/regmask.hpp"
 #include "opto/runtime.hpp"
@@ -2814,6 +2815,69 @@ const RegMask &GotoNode::out_RegMask() const {
   return RegMask::Empty;
 }
 
+TemplateAssertionPredicateNode::TemplateAssertionPredicateNode(Node* control, BoolNode* bool_init_value,
+                                                               BoolNode* bool_last_value,
+                                                               const int initialized_init_value_opcode,
+                                                               const int initialized_last_value_opcode)
+    : Node(control, bool_init_value, bool_last_value),
+      _initialized_init_value_opcode(initialized_init_value_opcode),
+      _initialized_last_value_opcode(initialized_last_value_opcode),
+      _useless(false) {
+  assert(initialized_init_value_opcode == Op_If || initialized_init_value_opcode == Op_RangeCheck, "invalid opcode");
+  assert(initialized_last_value_opcode == Op_If || initialized_last_value_opcode == Op_RangeCheck, "invalid opcode");
+  init_class_id(Class_TemplateAssertionPredicate);
+}
+
+IfNode* TemplateAssertionPredicateNode::create_initialized_assertion_predicate(
+    Node* control, OpaqueAssertionPredicateNode* opaque_bool,
+    AssertionPredicateType initialized_assertion_predicate_type) const {
+  bool create_if_node = true;
+  switch (initialized_assertion_predicate_type) {
+    case AssertionPredicateType::Init_value:
+      create_if_node = _initialized_init_value_opcode == Op_If;
+      break;
+    case AssertionPredicateType::Last_value:
+      create_if_node = _initialized_last_value_opcode == Op_If;
+      break;
+    default:
+      assert(false, "invalid Assertion Predicate type");
+  }
+  return create_if_node ?
+         new IfNode(control, opaque_bool, PROB_MAX, COUNT_UNKNOWN NOT_PRODUCT(COMMA initialized_assertion_predicate_type)) :
+         new RangeCheckNode(control, opaque_bool, PROB_MAX, COUNT_UNKNOWN NOT_PRODUCT(COMMA initialized_assertion_predicate_type));
+}
+
+
+uint TemplateAssertionPredicateNode::index_for_bool_input(const BoolNode* bool_input) const {
+  if (bool_input == in(TemplateAssertionPredicateNode::InitValue)) {
+    return TemplateAssertionPredicateNode::InitValue;
+  } else {
+    assert(bool_input == in(TemplateAssertionPredicateNode::LastValue), "must be a bool input");
+    return TemplateAssertionPredicateNode::LastValue;
+  }
+}
+
+Node* TemplateAssertionPredicateNode::Identity(PhaseGVN* phase) {
+  if (phase->C->post_loop_opts_phase() || _useless) {
+    return in(0);
+  } else {
+    phase->C->record_for_post_loop_opts_igvn(this);
+    return this;
+  }
+}
+
+const Type* TemplateAssertionPredicateNode::Value(PhaseGVN* phase) const {
+  return phase->type(in(0));
+}
+
+#ifndef PRODUCT
+void TemplateAssertionPredicateNode::dump_spec(outputStream* st) const {
+  if (_useless) {
+    st->print("#useless ");
+  }
+}
+#endif // NOT PRODUCT
+
 //=============================================================================
 const RegMask &JumpNode::out_RegMask() const {
   return RegMask::Empty;

src/hotspot/share/opto/cfgnode.hpp

@@ -56,7 +56,9 @@ class       CatchProjNode;
 class     JProjNode;
 class       JumpProjNode;
 class     SCMemProjNode;
+class  OpaqueAssertionPredicateNode;
 class PhaseIdealLoop;
+enum class AssertionPredicateType;
 
 // The success projection of a Parse Predicate is always an IfTrueNode and the uncommon projection an IfFalseNode
 typedef IfTrueNode ParsePredicateSuccessProj;
@@ -289,6 +291,45 @@ class GotoNode : public Node {
   virtual const RegMask &out_RegMask() const;
 };
 
+// This node represents a Template Assertion Predicate with two bools as input which can be used to create an
+// Initialized Assertion Predicate from (more information can be found in the summary at predicates.hpp).
+// This node is folded either after loop opts or once the associated CountedLoopNode is removed.
+class TemplateAssertionPredicateNode : public Node {
+  int _initialized_init_value_opcode;
+  int _initialized_last_value_opcode;
+  bool _useless; // If this node becomes useless, it can be cleaned up by Identity().
+
+  virtual uint size_of() const {return sizeof(*this);}
+ public:
+  // Named bool inputs
+  enum {
+    InitValue = 1,
+    LastValue = 2
+  };
+
+  TemplateAssertionPredicateNode(Node* control, BoolNode* bool_init_value, BoolNode* bool_last_value,
+                                 int initialized_init_value_opcode, int initialized_last_value_opcode);
+
+  void mark_useless() {
+    _useless = true;
+  }
+
+  IfNode* create_initialized_assertion_predicate(Node* control, OpaqueAssertionPredicateNode* opaque_bool,
+                                                 AssertionPredicateType initialized_assertion_predicate_type) const;
+  uint index_for_bool_input(const BoolNode* bool_input) const;
+
+  virtual int Opcode() const;
+  virtual bool pinned() const { return true; }
+  virtual bool is_CFG() const { return true; }
+  virtual uint hash() const { return NO_HASH; }  // CFG nodes do not hash
+  virtual bool depends_only_on_test() const { return false; }
+  virtual const Type* bottom_type() const { return Type::CONTROL; }
+  virtual Node* Identity(PhaseGVN* phase);
+  virtual const Type* Value(PhaseGVN* phase) const;
+
+  NOT_PRODUCT(void dump_spec(outputStream* st) const;)
+};
+
 //------------------------------CProjNode--------------------------------------
 // control projection for node that produces multiple control-flow paths
 class CProjNode : public ProjNode {
@@ -318,11 +359,23 @@ class MultiBranchNode : public MultiNode {
 //------------------------------IfNode-----------------------------------------
 // Output selected Control, based on a boolean test
 class IfNode : public MultiBranchNode {
+ public:
+  float _prob;                           // Probability of true path being taken.
+  float _fcnt;                           // Frequency counter
+
+ private:
+  NOT_PRODUCT(AssertionPredicateType _assertion_predicate_type;)
+
+  void init_node(Node* control, Node* bol) {
+    init_class_id(Class_If);
+    init_req(0, control);
+    init_req(1, bol);
+  }
+
   // Size is bigger to hold the probability field.  However, _prob does not
   // change the semantics so it does not appear in the hash & cmp functions.
   virtual uint size_of() const { return sizeof(*this); }
 
-private:
   // Helper methods for fold_compares
   bool cmpi_folds(PhaseIterGVN* igvn, bool fold_ne = false);
   bool is_ctrl_folds(Node* ctrl, PhaseIterGVN* igvn);
@@ -413,14 +466,9 @@ class IfNode : public MultiBranchNode {
   // Magic manifest probabilities such as 0.83, 0.7, ... can be found in
   // gen_subtype_check() and catch_inline_exceptions().
 
-  float _prob;                  // Probability of true path being taken.
-  float _fcnt;                  // Frequency counter
-  IfNode( Node *control, Node *b, float p, float fcnt )
-    : MultiBranchNode(2), _prob(p), _fcnt(fcnt) {
-    init_class_id(Class_If);
-    init_req(0,control);
-    init_req(1,b);
-  }
+  IfNode(Node* control, Node* bol, float p, float fcnt);
+  NOT_PRODUCT(IfNode(Node* control, Node* bol, float p, float fcnt, AssertionPredicateType assertion_predicate_type);)
+
   virtual int Opcode() const;
   virtual bool pinned() const { return true; }
   virtual const Type *bottom_type() const { return TypeTuple::IFBOTH; }
@@ -447,25 +495,32 @@ class IfNode : public MultiBranchNode {
 
 class RangeCheckNode : public IfNode {
 private:
-  int is_range_check(Node* &range, Node* &index, jint &offset);
+  int is_range_check(Node*& range, Node*& index, jint& offset);
 
 public:
-  RangeCheckNode(Node* control, Node *b, float p, float fcnt)
-    : IfNode(control, b, p, fcnt) {
+  RangeCheckNode(Node* control, Node* bol, float p, float fcnt)
+      : IfNode(control, bol, p, fcnt) {
+    init_class_id(Class_RangeCheck);
+  }
+
+#ifndef PRODUCT
+  RangeCheckNode(Node* control, Node* bol, float p, float fcnt, AssertionPredicateType assertion_predicate_type)
+      : IfNode(control, bol, p, fcnt, assertion_predicate_type) {
     init_class_id(Class_RangeCheck);
   }
+#endif // NOT_PRODUCT
 
   virtual int Opcode() const;
   virtual Node* Ideal(PhaseGVN *phase, bool can_reshape);
 };
 
 // Special node that denotes a Parse Predicate added during parsing. A Parse Predicate serves as placeholder to later
 // create Regular Predicates (Runtime Predicates with possible Assertion Predicates) above it. Together they form a
-// Predicate Block. The Parse Predicate and Regular Predicates share the same uncommon trap.
+// Predicate Block. The Parse Predicate and Runtime Predicates share the same uncommon trap.
 // There are three kinds of Parse Predicates:
 // Loop Parse Predicate, Profiled Loop Parse Predicate (both used by Loop Predication), and Loop Limit Check Parse
 // Predicate (used for integer overflow checks when creating a counted loop).
-// More information about predicates can be found in loopPredicate.cpp.
+// More information about predicates can be found in predicates.hpp.
 class ParsePredicateNode : public IfNode {
   Deoptimization::DeoptReason _deopt_reason;
   bool _useless; // If the associated loop dies, this parse predicate becomes useless and can be cleaned up by Value().
@@ -490,6 +545,11 @@ class ParsePredicateNode : public IfNode {
     _useless = false;
   }
 
+  // Return the uncommon trap If projection of this Parse Predicate.
+  ParsePredicateUncommonProj* uncommon_proj() const {
+    return proj_out(0)->as_IfFalse();
+  }
+
   Node* uncommon_trap() const;
 
   Node* Ideal(PhaseGVN* phase, bool can_reshape) {

src/hotspot/share/opto/classes.hpp

@@ -271,6 +271,7 @@ macro(OpaqueLoopStride)
 macro(OpaqueZeroTripGuard)
 macro(Opaque3)
 macro(Opaque4)
+macro(OpaqueAssertionPredicate)
 macro(ProfileBoolean)
 macro(OrI)
 macro(OrL)
@@ -362,6 +363,7 @@ macro(SubL)
 macro(TailCall)
 macro(TailJump)
 macro(MacroLogicV)
+macro(TemplateAssertionPredicate)
 macro(ThreadLocal)
 macro(Unlock)
 macro(URShiftB)

src/hotspot/share/opto/compile.cpp

@@ -389,9 +389,6 @@ void Compile::remove_useless_node(Node* dead) {
   if (dead->is_expensive()) {
     remove_expensive_node(dead);
   }
-  if (dead->Opcode() == Op_Opaque4) {
-    remove_template_assertion_predicate_opaq(dead);
-  }
   if (dead->is_ParsePredicate()) {
     remove_parse_predicate(dead->as_ParsePredicate());
   }
@@ -443,7 +440,6 @@ void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_Lis
 
   remove_useless_nodes(_macro_nodes,        useful); // remove useless macro nodes
   remove_useless_nodes(_parse_predicates,   useful); // remove useless Parse Predicate nodes
-  remove_useless_nodes(_template_assertion_predicate_opaqs, useful); // remove useless Assertion Predicate opaque nodes
   remove_useless_nodes(_expensive_nodes,    useful); // remove useless expensive nodes
   remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass
   remove_useless_unstable_if_traps(useful);          // remove useless unstable_if traps
@@ -640,7 +636,6 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci,
                   _intrinsics        (comp_arena(), 0, 0, nullptr),
                   _macro_nodes       (comp_arena(), 8, 0, nullptr),
                   _parse_predicates  (comp_arena(), 8, 0, nullptr),
-                  _template_assertion_predicate_opaqs (comp_arena(), 8, 0, nullptr),
                   _expensive_nodes   (comp_arena(), 8, 0, nullptr),
                   _for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
                   _unstable_if_traps (comp_arena(), 8, 0, nullptr),

src/hotspot/share/opto/compile.hpp

@@ -365,7 +365,6 @@ class Compile : public Phase {
   GrowableArray<CallGenerator*> _intrinsics;    // List of intrinsics.
   GrowableArray<Node*>  _macro_nodes;           // List of nodes which need to be expanded before matching.
   GrowableArray<ParsePredicateNode*> _parse_predicates; // List of Parse Predicates.
-  GrowableArray<Node*>  _template_assertion_predicate_opaqs; // List of Opaque4 nodes for Template Assertion Predicates.
   GrowableArray<Node*>  _expensive_nodes;       // List of nodes that are expensive to compute and that we'd better not let the GVN freely common
   GrowableArray<Node*>  _for_post_loop_igvn;    // List of nodes for IGVN after loop opts are over
   GrowableArray<UnstableIfTrap*> _unstable_if_traps;        // List of ifnodes after IGVN
@@ -717,16 +716,12 @@ class Compile : public Phase {
 
   int           macro_count()             const { return _macro_nodes.length(); }
   int           parse_predicate_count()   const { return _parse_predicates.length(); }
-  int           template_assertion_predicate_count() const { return _template_assertion_predicate_opaqs.length(); }
   int           expensive_count()         const { return _expensive_nodes.length(); }
   int           coarsened_count()         const { return _coarsened_locks.length(); }
 
   Node*         macro_node(int idx)       const { return _macro_nodes.at(idx); }
-  ParsePredicateNode* parse_predicate(int idx) const { return _parse_predicates.at(idx); }
 
-  Node* template_assertion_predicate_opaq_node(int idx) const {
-    return _template_assertion_predicate_opaqs.at(idx);
-  }
+  const GrowableArray<ParsePredicateNode*>& parse_predicates() const { return _parse_predicates; }
 
   Node*         expensive_node(int idx)   const { return _expensive_nodes.at(idx); }
 
@@ -762,17 +757,6 @@ class Compile : public Phase {
     }
   }
 
-  void add_template_assertion_predicate_opaq(Node* n) {
-    assert(!_template_assertion_predicate_opaqs.contains(n),
-           "duplicate entry in template assertion predicate opaque4 list");
-    _template_assertion_predicate_opaqs.append(n);
-  }
-
-  void remove_template_assertion_predicate_opaq(Node* n) {
-    if (template_assertion_predicate_count() > 0) {
-      _template_assertion_predicate_opaqs.remove_if_existing(n);
-    }
-  }
   void add_coarsened_locks(GrowableArray<AbstractLockNode*>& locks);
   void remove_coarsened_lock(Node* n);
   bool coarsened_locks_consistent();

src/hotspot/share/opto/graphKit.cpp

@@ -4061,10 +4061,11 @@ void GraphKit::add_parse_predicate(Deoptimization::DeoptReason reason, const int
 void GraphKit::add_parse_predicates(int nargs) {
   if (UseLoopPredicate) {
     add_parse_predicate(Deoptimization::Reason_predicate, nargs);
+    if (UseProfiledLoopPredicate) {
+      add_parse_predicate(Deoptimization::Reason_profile_predicate, nargs);
+    }
   }
-  if (UseProfiledLoopPredicate) {
-    add_parse_predicate(Deoptimization::Reason_profile_predicate, nargs);
-  }
+
   // Loop Limit Check Predicate should be near the loop.
   add_parse_predicate(Deoptimization::Reason_loop_limit_check, nargs);
 }

src/hotspot/share/opto/ifnode.cpp

@@ -47,6 +47,24 @@
 extern uint explicit_null_checks_elided;
 #endif
 
+IfNode::IfNode(Node* control, Node* bol, float p, float fcnt)
+    : MultiBranchNode(2),
+      _prob(p),
+      _fcnt(fcnt)
+      NOT_PRODUCT(COMMA _assertion_predicate_type(AssertionPredicateType::None)) {
+  init_node(control, bol);
+}
+
+#ifndef PRODUCT
+IfNode::IfNode(Node* control, Node* bol, float p, float fcnt, AssertionPredicateType assertion_predicate_type)
+    : MultiBranchNode(2),
+      _prob(p),
+      _fcnt(fcnt),
+      _assertion_predicate_type(assertion_predicate_type) {
+  init_node(control, bol);
+}
+#endif // NOT_PRODUCT
+
 //=============================================================================
 //------------------------------Value------------------------------------------
 // Return a tuple for whichever arm of the IF is reachable
@@ -1783,8 +1801,18 @@ bool IfNode::is_zero_trip_guard() const {
 
 #ifndef PRODUCT
 //------------------------------dump_spec--------------------------------------
-void IfNode::dump_spec(outputStream *st) const {
-  st->print("P=%f, C=%f",_prob,_fcnt);
+void IfNode::dump_spec(outputStream* st) const {
+  switch (_assertion_predicate_type) {
+    case AssertionPredicateType::Init_value:
+      st->print("#Init Value Assertion Predicate ");
+      break;
+    case AssertionPredicateType::Last_value:
+      st->print("#Last Value Assertion Predicate ");
+      break;
+    default:
+      break;
+  }
+  st->print("P=%f, C=%f",_prob, _fcnt);
 }
 #endif
 
@@ -2067,6 +2095,9 @@ void ParsePredicateNode::dump_spec(outputStream* st) const {
     default:
       fatal("unknown kind");
   }
+  if (_useless) {
+    st->print("#useless ");
+  }
 }
 
 #endif // NOT PRODUCT