Skip to content

Commit

Permalink
Refactor Template Assertion Predicate Bool creation and Split If
Browse files Browse the repository at this point in the history
  • Loading branch information
@chhagedorn
chhagedorn committed Nov 24, 2023
1 parent 8db7bad commit 634513c
Show file tree
Hide file tree
Showing 8 changed files with 499 additions and 157 deletions.
8 changes: 6 additions & 2 deletions src/hotspot/share/opto/loopPredicate.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -396,9 +396,13 @@ void PhaseIdealLoop::get_assertion_predicates(Node* predicate, Unique_Node_List&
IfProjNode* PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(Node* iff, IfProjNode* predicate,
Deoptimization::DeoptReason reason,
ParsePredicateSuccessProj* parse_predicate_proj) {
Node* bol = create_bool_from_template_assertion_predicate(iff, nullptr, nullptr, parse_predicate_proj);
Node* opaque_node = iff->in(1);
TemplateAssertionPredicateBool template_assertion_predicate_bool(opaque_node->in(1));
BoolNode* bol = template_assertion_predicate_bool.clone(parse_predicate_proj, this);
opaque_node = clone_and_register(opaque_node, parse_predicate_proj);
_igvn.replace_input_of(opaque_node, 1, bol);
IfProjNode* if_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, iff->Opcode(), false);
_igvn.replace_input_of(if_proj->in(0), 1, bol);
_igvn.replace_input_of(if_proj->in(0), 1, opaque_node);
_igvn.replace_input_of(parse_predicate_proj->in(0), 0, if_proj);
set_idom(parse_predicate_proj->in(0), if_proj, dom_depth(if_proj));
return if_proj;
Expand Down
150 changes: 23 additions & 127 deletions src/hotspot/share/opto/loopTransform.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1385,40 +1385,6 @@ void PhaseIdealLoop::copy_assertion_predicates_to_main_loop_helper(const Predica
}
}

// Is 'n' a node that can be found on the input chain of a Template Assertion Predicate bool (i.e. between a Template
// Assertion Predicate If node and the OpaqueLoop* nodes)?
static bool is_part_of_template_assertion_predicate_bool(Node* n) {
int op = n->Opcode();
return (n->is_Bool() ||
n->is_Cmp() ||
op == Op_AndL ||
op == Op_OrL ||
op == Op_RShiftL ||
op == Op_LShiftL ||
op == Op_LShiftI ||
op == Op_AddL ||
op == Op_AddI ||
op == Op_MulL ||
op == Op_MulI ||
op == Op_SubL ||
op == Op_SubI ||
op == Op_ConvI2L ||
op == Op_CastII);
}

bool PhaseIdealLoop::subgraph_has_opaque(Node* n) {
if (n->Opcode() == Op_OpaqueLoopInit || n->Opcode() == Op_OpaqueLoopStride) {
return true;
}
if (!is_part_of_template_assertion_predicate_bool(n)) {
return false;
}
uint init;
uint stride;
count_opaque_loop_nodes(n, init, stride);
return init != 0 || stride != 0;
}

bool PhaseIdealLoop::assertion_predicate_has_loop_opaque_node(IfNode* iff) {
uint init;
uint stride;
Expand Down Expand Up @@ -1462,113 +1428,43 @@ void PhaseIdealLoop::count_opaque_loop_nodes(Node* n, uint& init, uint& stride)
wq.push(n);
for (uint i = 0; i < wq.size(); i++) {
Node* n = wq.at(i);
if (is_part_of_template_assertion_predicate_bool(n)) {
for (uint j = 1; j < n->req(); j++) {
Node* m = n->in(j);
if (m != nullptr) {
wq.push(m);
}
}
continue;
}
if (n->Opcode() == Op_OpaqueLoopInit) {
init++;
} else if (n->Opcode() == Op_OpaqueLoopStride) {
stride++;
}
}
}

// Create a new Bool node from the provided Template Assertion Predicate.
// Unswitched loop: new_init and new_stride are both null. Clone OpaqueLoopInit and OpaqueLoopStride.
// Otherwise: Replace found OpaqueLoop* nodes with new_init and new_stride, respectively.
Node* PhaseIdealLoop::create_bool_from_template_assertion_predicate(Node* template_assertion_predicate, Node* new_init,
Node* new_stride, Node* control) {
Node_Stack to_clone(2);
Node* opaque4 = template_assertion_predicate->in(1);
assert(opaque4->Opcode() == Op_Opaque4, "must be Opaque4");
to_clone.push(opaque4, 1);
uint current = C->unique();
Node* result = nullptr;
bool is_unswitched_loop = new_init == nullptr && new_stride == nullptr;
assert(new_init != nullptr || is_unswitched_loop, "new_init must be set when new_stride is non-null");
// Look for the opaque node to replace with the new value
// and clone everything in between. We keep the Opaque4 node
// so the duplicated predicates are eliminated once loop
// opts are over: they are here only to keep the IR graph
// consistent.
do {
Node* n = to_clone.node();
uint i = to_clone.index();
Node* m = n->in(i);
if (is_part_of_template_assertion_predicate_bool(m)) {
to_clone.push(m, 1);
continue;
}
if (m->is_Opaque1()) {
if (n->_idx < current) {
n = n->clone();
register_new_node(n, control);
}
int op = m->Opcode();
if (op == Op_OpaqueLoopInit) {
if (is_unswitched_loop && m->_idx < current && new_init == nullptr) {
new_init = m->clone();
register_new_node(new_init, control);
}
n->set_req(i, new_init);
if (AssertionPredicateBoolOpcodes::is_valid(n)) {
if (n->Opcode() == Op_OpaqueLoopInit) {
init++;
} else if (n->Opcode() == Op_OpaqueLoopStride) {
stride++;
} else {
assert(op == Op_OpaqueLoopStride, "unexpected opaque node");
if (is_unswitched_loop && m->_idx < current && new_stride == nullptr) {
new_stride = m->clone();
register_new_node(new_stride, control);
}
if (new_stride != nullptr) {
n->set_req(i, new_stride);
}
}
to_clone.set_node(n);
}
while (true) {
Node* cur = to_clone.node();
uint j = to_clone.index();
if (j+1 < cur->req()) {
to_clone.set_index(j+1);
break;
}
to_clone.pop();
if (to_clone.size() == 0) {
result = cur;
break;
}
Node* next = to_clone.node();
j = to_clone.index();
if (next->in(j) != cur) {
assert(cur->_idx >= current || next->in(j)->Opcode() == Op_Opaque1, "new node or Opaque1 being replaced");
if (next->_idx < current) {
next = next->clone();
register_new_node(next, control);
to_clone.set_node(next);
for (uint j = 1; j < n->req(); j++) {
Node* m = n->in(j);
if (m != nullptr) {
wq.push(m);
}
}
next->set_req(j, cur);
}
}
} while (result == nullptr);
assert(result->_idx >= current, "new node expected");
assert(!is_unswitched_loop || new_init != nullptr, "new_init must always be found and cloned");
return result;
}
}

// Clone an Assertion Predicate for the main loop. new_init and new_stride are set as new inputs. Since the predicates
// cannot fail at runtime, Halt nodes are inserted instead of uncommon traps.
Node* PhaseIdealLoop::clone_assertion_predicate_and_initialize(Node* iff, Node* new_init, Node* new_stride, Node* predicate,
Node* uncommon_proj, Node* control, IdealLoopTree* outer_loop,
Node* input_proj) {
Node* result = create_bool_from_template_assertion_predicate(iff, new_init, new_stride, control);
Node* opaque_node = iff->in(1);
TemplateAssertionPredicateBool template_assertion_predicate_bool(opaque_node->in(1));
BoolNode* new_bool;
if (new_stride == nullptr) {
new_bool = template_assertion_predicate_bool.clone_and_replace_init(control, new_init, this);
} else {
new_bool = template_assertion_predicate_bool.clone_and_replace_opaque_loop_nodes(control, new_init, new_stride, this);
}
opaque_node = clone_and_register(opaque_node, control);
_igvn.replace_input_of(opaque_node, 1, new_bool);

Node* proj = predicate->clone();
Node* other_proj = uncommon_proj->clone();
Node* new_iff = iff->clone();
new_iff->set_req(1, result);
new_iff->set_req(1, opaque_node);
proj->set_req(0, new_iff);
other_proj->set_req(0, new_iff);
Node* frame = new ParmNode(C->start(), TypeFunc::FramePtr);
Expand Down
19 changes: 10 additions & 9 deletions src/hotspot/share/opto/loopnode.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -946,9 +946,6 @@ class PhaseIdealLoop : public PhaseTransform {
Node* uncommon_proj, Node* control, IdealLoopTree* outer_loop,
Node* input_proj);
static void count_opaque_loop_nodes(Node* n, uint& init, uint& stride);
static bool subgraph_has_opaque(Node* n);
Node* create_bool_from_template_assertion_predicate(Node* template_assertion_predicate, Node* new_init, Node* new_stride,
Node* control);
static bool assertion_predicate_has_loop_opaque_node(IfNode* iff);
static void get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque = false);
void update_main_loop_assertion_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, int stride_con);
Expand Down Expand Up @@ -1647,6 +1644,12 @@ class PhaseIdealLoop : public PhaseTransform {
bool created_loop_node() { return _created_loop_node; }
void register_new_node(Node* n, Node* blk);

Node* clone_and_register(Node* n, Node* ctrl) {
n = n->clone();
register_new_node(n, ctrl);
return n;
}

#ifdef ASSERT
void dump_bad_graph(const char* msg, Node* n, Node* early, Node* LCA);
#endif
Expand Down Expand Up @@ -1723,14 +1726,12 @@ class PhaseIdealLoop : public PhaseTransform {

void finish_clone_loop(Node_List* split_if_set, Node_List* split_bool_set, Node_List* split_cex_set);

bool clone_cmp_down(Node* n, const Node* blk1, const Node* blk2);

void clone_loadklass_nodes_at_cmp_index(const Node* n, Node* cmp, int i);

bool clone_cmp_loadklass_down(Node* n, const Node* blk1, const Node* blk2);

bool at_relevant_ctrl(Node* n, const Node* blk1, const Node* blk2);

bool clone_cmp_loadklass_down(Node* n, const Node* blk1, const Node* blk2);
void clone_loadklass_nodes_at_cmp_index(const Node* n, Node* cmp, int i);
bool clone_cmp_down(Node* n, const Node* blk1, const Node* blk2);
void clone_template_assertion_predicate_bool_down_if_related(Node* n);

Node* similar_subtype_check(const Node* x, Node* r_in);

Expand Down
6 changes: 6 additions & 0 deletions src/hotspot/share/opto/node.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,8 @@ class NegNode;
class NegVNode;
class NeverBranchNode;
class Opaque1Node;
class OpaqueLoopInitNode;
class OpaqueLoopStrideNode;
class OuterStripMinedLoopNode;
class OuterStripMinedLoopEndNode;
class Node;
Expand Down Expand Up @@ -783,6 +785,8 @@ class Node {
DEFINE_CLASS_ID(ClearArray, Node, 14)
DEFINE_CLASS_ID(Halt, Node, 15)
DEFINE_CLASS_ID(Opaque1, Node, 16)
DEFINE_CLASS_ID(OpaqueLoopInit, Opaque1, 0)
DEFINE_CLASS_ID(OpaqueLoopStride, Opaque1, 1)
DEFINE_CLASS_ID(Move, Node, 17)
DEFINE_CLASS_ID(LShift, Node, 18)
DEFINE_CLASS_ID(Neg, Node, 19)
Expand Down Expand Up @@ -952,6 +956,8 @@ class Node {
DEFINE_CLASS_QUERY(NegV)
DEFINE_CLASS_QUERY(NeverBranch)
DEFINE_CLASS_QUERY(Opaque1)
DEFINE_CLASS_QUERY(OpaqueLoopInit)
DEFINE_CLASS_QUERY(OpaqueLoopStride)
DEFINE_CLASS_QUERY(OuterStripMinedLoop)
DEFINE_CLASS_QUERY(OuterStripMinedLoopEnd)
DEFINE_CLASS_QUERY(Parm)
Expand Down
2 changes: 2 additions & 0 deletions src/hotspot/share/opto/opaquenode.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,13 +60,15 @@ class Opaque1Node : public Node {
class OpaqueLoopInitNode : public Opaque1Node {
public:
OpaqueLoopInitNode(Compile* C, Node *n) : Opaque1Node(C, n) {
init_class_id(Class_OpaqueLoopInit);
}
virtual int Opcode() const;
};

class OpaqueLoopStrideNode : public Opaque1Node {
public:
OpaqueLoopStrideNode(Compile* C, Node *n) : Opaque1Node(C, n) {
init_class_id(Class_OpaqueLoopStride);
}
virtual int Opcode() const;
};
Expand Down
Loading

0 comments on commit 634513c

Please sign in to comment.