From 4f9c88b9428c74692c28ef7cd95f861d423cc6af Mon Sep 17 00:00:00 2001 From: jiangfeilong Date: Mon, 24 Jun 2024 23:22:49 +0800 Subject: [PATCH] riscv port for JEP 475 --- .../gc/g1/g1BarrierSetAssembler_riscv.cpp | 274 ++++++--- .../gc/g1/g1BarrierSetAssembler_riscv.hpp | 25 +- src/hotspot/cpu/riscv/gc/g1/g1_riscv.ad | 580 ++++++++++++++++++ src/hotspot/cpu/riscv/riscv.ad | 19 +- 4 files changed, 809 insertions(+), 89 deletions(-) create mode 100644 src/hotspot/cpu/riscv/gc/g1/g1_riscv.ad diff --git a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp index 062f80290626f..d6e77e3ff92cd 100644 --- a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp @@ -1,6 +1,6 @@ /* * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -39,7 +39,10 @@ #include "c1/c1_LIRAssembler.hpp" #include "c1/c1_MacroAssembler.hpp" #include "gc/g1/c1/g1BarrierSetC1.hpp" -#endif +#endif // COMPILER1 +#ifdef COMPILER2 +#include "gc/g1/c2/g1BarrierSetC2.hpp" +#endif // COMPILER2 #define __ masm-> @@ -96,6 +99,55 @@ void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* mas __ pop_reg(saved_regs, sp); } +static void generate_queue_test_and_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime, + const Register thread, const Register value, const Register tmp1, const Register tmp2) { + // Can we store a value in the given thread's buffer? + // (The index field is typed as size_t.) + __ ld(tmp1, Address(thread, in_bytes(index_offset))); // tmp1 := *(index address) + __ beqz(tmp1, runtime); // jump to runtime if index == 0 (full buffer) + // The buffer is not full, store value into it. + __ sub(tmp1, tmp1, wordSize); // tmp1 := next index + __ sd(tmp1, Address(thread, in_bytes(index_offset))); // *(index address) := next index + __ ld(tmp2, Address(thread, in_bytes(buffer_offset))); // tmp2 := buffer address + __ add(tmp2, tmp2, tmp1); + __ sd(value, Address(tmp2)); // *(buffer address + next index) := value +} + +static void generate_pre_barrier_fast_path(MacroAssembler* masm, + const Register thread, + const Register tmp1) { + Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset())); + // Is marking active? + if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { + __ lwu(tmp1, in_progress); + } else { + assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); + __ lbu(tmp1, in_progress); + } +} + +static void generate_pre_barrier_slow_path(MacroAssembler* masm, + const Register obj, + const Register pre_val, + const Register thread, + const Register tmp1, + const Register tmp2, + Label& done, + Label& runtime) { + // Do we need to load the previous value? + if (obj != noreg) { + __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW); + } + // Is the previous value null? + __ beqz(pre_val, done, true); + generate_queue_test_and_insertion(masm, + G1ThreadLocalData::satb_mark_queue_index_offset(), + G1ThreadLocalData::satb_mark_queue_buffer_offset(), + runtime, + thread, pre_val, tmp1, tmp2); + __ j(done); +} + void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, Register obj, Register pre_val, @@ -116,43 +168,10 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, assert_different_registers(obj, pre_val, tmp1, tmp2); assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register"); - Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset())); - Address index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset())); - Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset())); - - // Is marking active? - if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { // 4-byte width - __ lwu(tmp1, in_progress); - } else { - assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); - __ lbu(tmp1, in_progress); - } + generate_pre_barrier_fast_path(masm, thread, tmp1); + // If marking is not active (*(mark queue active address) == 0), jump to done __ beqz(tmp1, done); - - // Do we need to load the previous value? - if (obj != noreg) { - __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW); - } - - // Is the previous value null? - __ beqz(pre_val, done); - - // Can we store original value in the thread's buffer? - // Is index == 0? - // (The index field is typed as size_t.) - - __ ld(tmp1, index); // tmp := *index_adr - __ beqz(tmp1, runtime); // tmp == 0? - // If yes, goto runtime - - __ sub(tmp1, tmp1, wordSize); // tmp := tmp - wordSize - __ sd(tmp1, index); // *index_adr := tmp - __ ld(tmp2, buffer); - __ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr - - // Record the previous value - __ sd(pre_val, Address(tmp1, 0)); - __ j(done); + generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp1, tmp2, done, runtime); __ bind(runtime); @@ -171,6 +190,49 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, } +static void generate_post_barrier_fast_path(MacroAssembler* masm, + const Register store_addr, + const Register new_val, + const Register tmp1, + const Register tmp2, + Label& done, + bool new_val_may_be_null) { + // Does store cross heap regions? + __ xorr(tmp1, store_addr, new_val); // tmp1 := store address ^ new value + __ srli(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes) + __ beqz(tmp1, done); + // Crosses regions, storing null? + if (new_val_may_be_null) { + __ beqz(new_val, done); + } + // Storing region crossing non-null, is card already dirty? + __ srli(tmp1, store_addr, CardTable::card_shift()); // tmp1 := card address relative to card table base + __ load_byte_map_base(tmp2); // tmp2 := card table base address + __ add(tmp1, tmp1, tmp2); // tmp1 := card address + __ lbu(tmp2, Address(tmp1)); // tmp2 := card +} + +static void generate_post_barrier_slow_path(MacroAssembler* masm, + const Register thread, + const Register tmp1, + const Register tmp2, + Label& done, + Label& runtime) { + __ membar(MacroAssembler::StoreLoad); // StoreLoad membar + __ lbu(tmp2, Address(tmp1)); // tmp2 := card + __ beqz(tmp2, done, true); + // Storing a region crossing, non-null oop, card is clean. + // Dirty card and log. + STATIC_ASSERT(CardTable::dirty_card_val() == 0); + __ sb(zr, Address(tmp1)); // *(card address) := dirty_card_val + generate_queue_test_and_insertion(masm, + G1ThreadLocalData::dirty_card_queue_index_offset(), + G1ThreadLocalData::dirty_card_queue_buffer_offset(), + runtime, + thread, tmp1, tmp2, t0); + __ j(done); +} + void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, Register store_addr, Register new_val, @@ -179,73 +241,119 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, Register tmp2) { assert(thread == xthread, "must be"); assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, t0); - assert(store_addr != noreg && new_val != noreg && tmp1 != noreg && - tmp2 != noreg, "expecting a register"); - - Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset())); - Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset())); - - BarrierSet* bs = BarrierSet::barrier_set(); - CardTableBarrierSet* ctbs = barrier_set_cast(bs); + assert(store_addr != noreg && new_val != noreg && tmp1 != noreg && tmp2 != noreg, + "expecting a register"); Label done; Label runtime; - // Does store cross heap regions? + generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, done, true /* new_val_may_be_null */); + // If card is young, jump to done (tmp2 holds the card value) + __ mv(t0, (int)G1CardTable::g1_young_card_val()); + __ beq(tmp2, t0, done); // card == young_card_val? + generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, done, runtime); - __ xorr(tmp1, store_addr, new_val); - __ srli(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); - __ beqz(tmp1, done); + __ bind(runtime); + // save the live input values + RegSet saved = RegSet::of(store_addr); + __ push_reg(saved, sp); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp1, thread); + __ pop_reg(saved, sp); - // crosses regions, storing null? + __ bind(done); +} - __ beqz(new_val, done); +#if defined(COMPILER2) - // storing region crossing non-null, is card already dirty? +static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) { + SaveLiveRegisters save_registers(masm, stub); + if (c_rarg0 != arg) { + __ mv(c_rarg0, arg); + } + __ mv(c_rarg1, xthread); + __ mv(t0, runtime_path); + __ jalr(t0); +} - const Register card_addr = tmp1; +void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm, + Register obj, + Register pre_val, + Register thread, + Register tmp1, + Register tmp2, + G1PreBarrierStubC2* stub) { + assert(thread == xthread, "must be"); + assert_different_registers(obj, pre_val, tmp1, tmp2); + assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register"); - __ srli(card_addr, store_addr, CardTable::card_shift()); + stub->initialize_registers(obj, pre_val, thread, tmp1, tmp2); - // get the address of the card - __ load_byte_map_base(tmp2); - __ add(card_addr, card_addr, tmp2); - __ lbu(tmp2, Address(card_addr)); - __ mv(t0, (int)G1CardTable::g1_young_card_val()); - __ beq(tmp2, t0, done); + generate_pre_barrier_fast_path(masm, thread, tmp1); + // If marking is active (*(mark queue active address) != 0), jump to stub (slow path) + __ bnez(tmp1, *stub->entry(), true); - assert((int)CardTable::dirty_card_val() == 0, "must be 0"); + __ bind(*stub->continuation()); +} - __ membar(MacroAssembler::StoreLoad); +void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm, + G1PreBarrierStubC2* stub) const { + Assembler::InlineSkippedInstructionsCounter skip_counter(masm); + Label runtime; + Register obj = stub->obj(); + Register pre_val = stub->pre_val(); + Register thread = stub->thread(); + Register tmp1 = stub->tmp1(); + Register tmp2 = stub->tmp2(); - __ lbu(tmp2, Address(card_addr)); - __ beqz(tmp2, done); + __ bind(*stub->entry()); + generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp1, tmp2, *stub->continuation(), runtime); - // storing a region crossing, non-null oop, card is clean. - // dirty card and log. + __ bind(runtime); + generate_c2_barrier_runtime_call(masm, stub, pre_val, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry)); + __ j(*stub->continuation()); +} - __ sb(zr, Address(card_addr)); +void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm, + Register store_addr, + Register new_val, + Register thread, + Register tmp1, + Register tmp2, + G1PostBarrierStubC2* stub) { + assert(thread == xthread, "must be"); + assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, t0); + assert(store_addr != noreg && new_val != noreg && tmp1 != noreg && tmp2 != noreg, + "expecting a register"); - __ ld(t0, queue_index); - __ beqz(t0, runtime); - __ sub(t0, t0, wordSize); - __ sd(t0, queue_index); + stub->initialize_registers(thread, tmp1, tmp2); - __ ld(tmp2, buffer); - __ add(t0, tmp2, t0); - __ sd(card_addr, Address(t0, 0)); - __ j(done); + bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0; + generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, *stub->continuation(), new_val_may_be_null); + // If card is not young, jump to stub (slow path) (tmp2 holds the card value) + __ mv(t0, (int)G1CardTable::g1_young_card_val()); + __ bne(tmp2, t0, *stub->entry(), true); - __ bind(runtime); - // save the live input values - RegSet saved = RegSet::of(store_addr); - __ push_reg(saved, sp); - __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread); - __ pop_reg(saved, sp); + __ bind(*stub->continuation()); +} - __ bind(done); +void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm, + G1PostBarrierStubC2* stub) const { + Assembler::InlineSkippedInstructionsCounter skip_counter(masm); + Label runtime; + Register thread = stub->thread(); + Register tmp1 = stub->tmp1(); // tmp1 holds the card address. + Register tmp2 = stub->tmp2(); + + __ bind(*stub->entry()); + generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, *stub->continuation(), runtime); + + __ bind(runtime); + generate_c2_barrier_runtime_call(masm, stub, tmp1, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry)); + __ j(*stub->continuation()); } +#endif // COMPILER2 + void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp2) { bool on_oop = is_reference_type(type); diff --git a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp index 96568994079dd..c7bee2ef6f3a8 100644 --- a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp +++ b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp @@ -1,6 +1,6 @@ /* * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -36,6 +36,8 @@ class LIR_Assembler; class StubAssembler; class G1PreBarrierStub; class G1PostBarrierStub; +class G1PreBarrierStubC2; +class G1PostBarrierStubC2; class G1BarrierSetAssembler: public ModRefBarrierSetAssembler { protected: @@ -72,6 +74,27 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler { void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm); #endif +#ifdef COMPILER2 + void g1_write_barrier_pre_c2(MacroAssembler* masm, + Register obj, + Register pre_val, + Register thread, + Register tmp1, + Register tmp2, + G1PreBarrierStubC2* c2_stub); + void generate_c2_pre_barrier_stub(MacroAssembler* masm, + G1PreBarrierStubC2* stub) const; + void g1_write_barrier_post_c2(MacroAssembler* masm, + Register store_addr, + Register new_val, + Register thread, + Register tmp1, + Register tmp2, + G1PostBarrierStubC2* c2_stub); + void generate_c2_post_barrier_stub(MacroAssembler* masm, + G1PostBarrierStubC2* stub) const; +#endif + void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp2); }; diff --git a/src/hotspot/cpu/riscv/gc/g1/g1_riscv.ad b/src/hotspot/cpu/riscv/gc/g1/g1_riscv.ad new file mode 100644 index 0000000000000..10d51aa9056e2 --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/g1/g1_riscv.ad @@ -0,0 +1,580 @@ +// +// Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 2024, Huawei Technologies Co., Ltd. All rights reserved. +// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +// +// This code is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License version 2 only, as +// published by the Free Software Foundation. +// +// This code is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// version 2 for more details (a copy is included in the LICENSE file that +// accompanied this code). +// +// You should have received a copy of the GNU General Public License version +// 2 along with this work; if not, write to the Free Software Foundation, +// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +// +// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +// or visit www.oracle.com if you need additional information or have any +// questions. +// + +source_hpp %{ + +#include "gc/g1/c2/g1BarrierSetC2.hpp" +#include "gc/shared/gc_globals.hpp" + +%} + +source %{ + +#include "gc/g1/g1BarrierSetAssembler_riscv.hpp" +#include "gc/g1/g1BarrierSetRuntime.hpp" + +static void write_barrier_pre(MacroAssembler* masm, + const MachNode* node, + Register obj, + Register pre_val, + Register tmp1, + Register tmp2, + RegSet preserve = RegSet(), + RegSet no_preserve = RegSet()) { + if (!G1PreBarrierStubC2::needs_barrier(node)) { + return; + } + Assembler::InlineSkippedInstructionsCounter skip_counter(masm); + G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler()); + G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node); + for (RegSetIterator reg = preserve.begin(); *reg != noreg; ++reg) { + stub->preserve(*reg); + } + for (RegSetIterator reg = no_preserve.begin(); *reg != noreg; ++reg) { + stub->dont_preserve(*reg); + } + g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, xthread, tmp1, tmp2, stub); +} + +static void write_barrier_post(MacroAssembler* masm, + const MachNode* node, + Register store_addr, + Register new_val, + Register tmp1, + Register tmp2) { + if (!G1PostBarrierStubC2::needs_barrier(node)) { + return; + } + Assembler::InlineSkippedInstructionsCounter skip_counter(masm); + G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler()); + G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node); + g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, xthread, tmp1, tmp2, stub); +} + +%} + +instruct g1StoreP(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3) +%{ + predicate(UseG1GC && n->as_Store()->barrier_data() != 0); + match(Set mem (StoreP mem src)); + effect(TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(STORE_COST); + format %{ "sd $src, $mem\t# ptr" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + write_barrier_pre(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $src$$Register) /* preserve */); + __ sd($src$$Register, Address($mem$$Register)); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $src$$Register /* new_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(istore_reg_mem); +%} + +instruct g1StoreN(indirect mem, iRegN src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3) +%{ + predicate(UseG1GC && n->as_Store()->barrier_data() != 0); + match(Set mem (StoreN mem src)); + effect(TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(STORE_COST); + format %{ "sw $src, $mem\t# compressed ptr" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + write_barrier_pre(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $src$$Register) /* preserve */); + __ sw($src$$Register, Address($mem$$Register)); + if ((barrier_data() & G1C2BarrierPost) != 0) { + if ((barrier_data() & G1C2BarrierPostNotNull) == 0) { + __ decode_heap_oop($tmp1$$Register, $src$$Register); + } else { + __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register); + } + } + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $tmp1$$Register /* new_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(istore_reg_mem); +%} + +instruct g1EncodePAndStoreN(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3) +%{ + predicate(UseG1GC && n->as_Store()->barrier_data() != 0); + match(Set mem (StoreN mem (EncodeP src))); + effect(TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(STORE_COST); + format %{ "encode_heap_oop $tmp1, $src\n\t" + "sw $tmp1, $mem\t# compressed ptr" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + write_barrier_pre(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $src$$Register) /* preserve */); + if ((barrier_data() & G1C2BarrierPostNotNull) == 0) { + __ encode_heap_oop($tmp1$$Register, $src$$Register); + } else { + __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register); + } + __ sw($tmp1$$Register, Address($mem$$Register)); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $src$$Register /* new_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(istore_reg_mem); +%} + +instruct g1CompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); + match(Set res (CompareAndExchangeP mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(2 * VOLATILE_REF_COST); + format %{ "cmpxchg $res = $mem, $oldval, $newval\t# ptr" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + // Pass $oldval to the pre-barrier (instead of loading from $mem), because + // $oldval is the only value that can be overwritten. + // The same holds for g1CompareAndSwapP and its Acq variant. + write_barrier_pre(masm, this, + noreg /* obj */, + $oldval$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ mv($tmp1$$Register, $oldval$$Register); + __ mv($tmp2$$Register, $newval$$Register); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $tmp2$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3) +%{ + predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0); + match(Set res (CompareAndExchangeP mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(VOLATILE_REF_COST); + format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# ptr" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + // Pass $oldval to the pre-barrier (instead of loading from $mem), because + // $oldval is the only value that can be overwritten. + // The same holds for g1CompareAndSwapP and its Acq variant. + write_barrier_pre(masm, this, + noreg /* obj */, + $oldval$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ mv($tmp1$$Register, $oldval$$Register); + __ mv($tmp2$$Register, $newval$$Register); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $tmp2$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); + match(Set res (CompareAndExchangeN mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(2 * VOLATILE_REF_COST); + format %{ "cmpxchg $res = $mem, $oldval, $newval\t# narrow oop" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + write_barrier_pre(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ mv($tmp1$$Register, $oldval$$Register); + __ mv($tmp2$$Register, $newval$$Register); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::uint32, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register); + __ decode_heap_oop($tmp2$$Register); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $tmp2$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3) +%{ + predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0); + match(Set res (CompareAndExchangeN mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(VOLATILE_REF_COST); + format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# narrow oop" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + write_barrier_pre(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ mv($tmp1$$Register, $oldval$$Register); + __ mv($tmp2$$Register, $newval$$Register); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::uint32, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register); + __ decode_heap_oop($tmp2$$Register); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $tmp2$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndSwapP(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegP oldval) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); + match(Set res (CompareAndSwapP mem (Binary oldval newval))); + match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(2 * VOLATILE_REF_COST); + format %{ "cmpxchg $mem, $oldval, $newval\t# (ptr)\n\t" + "mv $res, $res == $oldval" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + write_barrier_pre(masm, this, + noreg /* obj */, + $oldval$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ mv($tmp1$$Register, $oldval$$Register); + __ mv($tmp2$$Register, $newval$$Register); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $tmp2$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegP oldval) +%{ + predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0); + match(Set res (CompareAndSwapP mem (Binary oldval newval))); + match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(VOLATILE_REF_COST); + format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (ptr)\n\t" + "mv $res, $res == $oldval" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + write_barrier_pre(masm, this, + noreg /* obj */, + $oldval$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ mv($tmp1$$Register, $oldval$$Register); + __ mv($tmp2$$Register, $newval$$Register); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $tmp2$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndSwapN(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); + match(Set res (CompareAndSwapN mem (Binary oldval newval))); + match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(2 * VOLATILE_REF_COST); + format %{ "cmpxchg $mem, $oldval, $newval\t# (narrow oop)\n\t" + "mv $res, $res == $oldval" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + write_barrier_pre(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ mv($tmp1$$Register, $oldval$$Register); + __ mv($tmp2$$Register, $newval$$Register); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::uint32, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + __ decode_heap_oop($tmp2$$Register); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $tmp2$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval) +%{ + predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0); + match(Set res (CompareAndSwapN mem (Binary oldval newval))); + match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(VOLATILE_REF_COST); + format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (narrow oop)\n\t" + "mv $res, $res == $oldval" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + write_barrier_pre(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ mv($tmp1$$Register, $oldval$$Register); + __ mv($tmp2$$Register, $newval$$Register); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::uint32, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + __ decode_heap_oop($tmp2$$Register); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $tmp2$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1GetAndSetP(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); + match(Set preval (GetAndSetP mem newval)); + effect(TEMP preval, TEMP tmp1, TEMP tmp2); + ins_cost(2 * VOLATILE_REF_COST); + format %{ "atomic_xchg $preval, $newval, [$mem]" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($mem$$Register, $newval$$Register); + write_barrier_pre(masm, this, + $mem$$Register /* obj */, + $preval$$Register /* pre_val (as a temporary register) */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */, + RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */); + __ atomic_xchg($preval$$Register, $newval$$Register, $mem$$Register); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $newval$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */); + %} + ins_pipe(pipe_serial); +%} + +instruct g1GetAndSetPAcq(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval) +%{ + predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0); + match(Set preval (GetAndSetP mem newval)); + effect(TEMP preval, TEMP tmp1, TEMP tmp2); + ins_cost(VOLATILE_REF_COST); + format %{ "atomic_xchg_acq $preval, $newval, [$mem]" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($mem$$Register, $newval$$Register); + write_barrier_pre(masm, this, + $mem$$Register /* obj */, + $preval$$Register /* pre_val (as a temporary register) */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */, + RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */); + __ atomic_xchgal($preval$$Register, $newval$$Register, $mem$$Register); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $newval$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */); + %} + ins_pipe(pipe_serial); +%} + +instruct g1GetAndSetN(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); + match(Set preval (GetAndSetN mem newval)); + effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(2 * VOLATILE_REF_COST); + format %{ "atomic_xchgwu $preval, $newval, [$mem]" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($mem$$Register, $newval$$Register); + write_barrier_pre(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */); + __ atomic_xchgwu($preval$$Register, $newval$$Register, $mem$$Register); + __ decode_heap_oop($tmp1$$Register, $newval$$Register); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $tmp1$$Register /* new_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_serial); +%} + +instruct g1GetAndSetNAcq(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval) +%{ + predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0); + match(Set preval (GetAndSetN mem newval)); + effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(VOLATILE_REF_COST); + format %{ "atomic_xchgwu_acq $preval, $newval, [$mem]" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($mem$$Register, $newval$$Register); + write_barrier_pre(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */); + __ atomic_xchgalwu($preval$$Register, $newval$$Register, $mem$$Register); + __ decode_heap_oop($tmp1$$Register, $newval$$Register); + write_barrier_post(masm, this, + $mem$$Register /* store_addr */, + $tmp1$$Register /* new_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_serial); +%} + +instruct g1LoadP(iRegPNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2) +%{ + predicate(UseG1GC && n->as_Load()->barrier_data() != 0); + match(Set dst (LoadP mem)); + effect(TEMP dst, TEMP tmp1, TEMP tmp2); + ins_cost(LOAD_COST + BRANCH_COST); + format %{ "ld $dst, $mem\t# ptr" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + __ ld($dst$$Register, Address($mem$$Register)); + write_barrier_pre(masm, this, + noreg /* obj */, + $dst$$Register /* pre_val */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */); + %} + ins_pipe(iload_reg_mem); +%} + +instruct g1LoadN(iRegNNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3) +%{ + predicate(UseG1GC && n->as_Load()->barrier_data() != 0); + match(Set dst (LoadN mem)); + effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(VOLATILE_REF_COST); + format %{ "lwu $dst, $mem\t# compressed ptr" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + __ lwu($dst$$Register, Address($mem$$Register)); + if ((barrier_data() & G1C2BarrierPre) != 0) { + __ decode_heap_oop($tmp1$$Register, $dst$$Register); + write_barrier_pre(masm, this, + noreg /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + } + %} + ins_pipe(iload_reg_mem); +%} diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index 1e26c2bf14202..fdcba7b05cb02 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -2212,7 +2212,8 @@ bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) { assert_cond(m != nullptr); if (is_vshift_con_pattern(n, m) || // ShiftV src (ShiftCntV con) is_vector_bitwise_not_pattern(n, m) || - is_vector_scalar_bitwise_pattern(n, m)) { + is_vector_scalar_bitwise_pattern(n, m) || + is_encode_and_store_pattern(n, m)) { mstack.push(m, Visit); return true; } @@ -4773,6 +4774,7 @@ instruct loadP(iRegPNoSp dst, memory mem) // Load Compressed Pointer instruct loadN(iRegNNoSp dst, memory mem) %{ + predicate(n->as_Load()->barrier_data() == 0); match(Set dst (LoadN mem)); ins_cost(LOAD_COST); @@ -5208,6 +5210,7 @@ instruct storeimmP0(immP0 zero, memory mem) // Store Compressed Pointer instruct storeN(iRegN src, memory mem) %{ + predicate(n->as_Store()->barrier_data() == 0); match(Set mem (StoreN mem src)); ins_cost(STORE_COST); @@ -5222,6 +5225,7 @@ instruct storeN(iRegN src, memory mem) instruct storeImmN0(immN0 zero, memory mem) %{ + predicate(n->as_Store()->barrier_data() == 0); match(Set mem (StoreN mem zero)); ins_cost(STORE_COST); @@ -5412,6 +5416,7 @@ instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); match(Set res (CompareAndSwapN mem (Binary oldval newval))); ins_cost(LOAD_COST + STORE_COST + ALU_COST * 8 + BRANCH_COST * 4); @@ -5533,7 +5538,7 @@ instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP new instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval) %{ - predicate(needs_acquiring_load_reserved(n)); + predicate(needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == 0); match(Set res (CompareAndSwapN mem (Binary oldval newval))); @@ -5641,6 +5646,7 @@ instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL ne instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); match(Set res (CompareAndExchangeN mem (Binary oldval newval))); ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 3); @@ -5774,7 +5780,7 @@ instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval) %{ - predicate(needs_acquiring_load_reserved(n)); + predicate(needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == 0); match(Set res (CompareAndExchangeN mem (Binary oldval newval))); @@ -5902,6 +5908,7 @@ instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL ne instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 4); @@ -6033,7 +6040,7 @@ instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval) %{ - predicate(needs_acquiring_load_reserved(n)); + predicate(needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == 0); match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); @@ -6105,6 +6112,8 @@ instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set prev (GetAndSetN mem newv)); ins_cost(ALU_COST); @@ -6170,7 +6179,7 @@ instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{ - predicate(needs_acquiring_load_reserved(n)); + predicate(needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == 0); match(Set prev (GetAndSetN mem newv));