diff --git a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp index 062f80290626f..49c63be3e523e 100644 --- a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp @@ -1,6 +1,6 @@ /* * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -39,7 +39,10 @@ #include "c1/c1_LIRAssembler.hpp" #include "c1/c1_MacroAssembler.hpp" #include "gc/g1/c1/g1BarrierSetC1.hpp" -#endif +#endif // COMPILER1 +#ifdef COMPILER2 +#include "gc/g1/c2/g1BarrierSetC2.hpp" +#endif // COMPILER2 #define __ masm-> @@ -96,6 +99,37 @@ void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* mas __ pop_reg(saved_regs, sp); } +static void generate_queue_test_and_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime, + const Register thread, const Register value, const Register tmp1, const Register tmp2) { + // Can we store a value in the given thread's buffer? + // (The index field is typed as size_t.) + __ ld(tmp1, Address(thread, in_bytes(index_offset))); // tmp1 := *(index address) + __ beqz(tmp1, runtime); // jump to runtime if index == 0 (full buffer) + // The buffer is not full, store value into it. + __ sub(tmp1, tmp1, wordSize); // tmp1 := next index + __ sd(tmp1, Address(thread, in_bytes(index_offset))); // *(index address) := next index + __ ld(tmp2, Address(thread, in_bytes(buffer_offset))); // tmp2 := buffer address + __ sd(value, Address(tmp2, tmp1)); // *(buffer address + next index) := value +} + +static Register generate_marking_active_test(MacroAssembler* masm, const Register thread, const Register tmp1) { + Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset())); + if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { + __ lwu(tmp1, in_progress); // tmp1 := *(mark queue active address) + } else { + assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); + __ lbu(tmp1, in_progress); // tmp1 := *(mark queue active address) + } + return tmp1; +} + +static Register generate_pre_val_not_null_test(MacroAssembler* masm, const Register obj, const Register pre_val) { + if (obj != noreg) { + __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW); // pre_val := previous value + } + return pre_val; +} + void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, Register obj, Register pre_val, @@ -116,42 +150,15 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, assert_different_registers(obj, pre_val, tmp1, tmp2); assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register"); - Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset())); - Address index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset())); - Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset())); + Register is_marking_active = generate_marking_active_test(masm, thread, tmp1); - // Is marking active? - if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { // 4-byte width - __ lwu(tmp1, in_progress); - } else { - assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); - __ lbu(tmp1, in_progress); - } __ beqz(tmp1, done); - // Do we need to load the previous value? - if (obj != noreg) { - __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW); - } - - // Is the previous value null? - __ beqz(pre_val, done); - - // Can we store original value in the thread's buffer? - // Is index == 0? - // (The index field is typed as size_t.) - - __ ld(tmp1, index); // tmp := *index_adr - __ beqz(tmp1, runtime); // tmp == 0? - // If yes, goto runtime - - __ sub(tmp1, tmp1, wordSize); // tmp := tmp - wordSize - __ sd(tmp1, index); // *index_adr := tmp - __ ld(tmp2, buffer); - __ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr - - // Record the previous value - __ sd(pre_val, Address(tmp1, 0)); + generate_queue_test_and_insertion(masm, + G1ThreadLocalData::satb_mark_queue_index_offset(), + G1ThreadLocalData::satb_mark_queue_buffer_offset(), + runtime, + thread, pre_val, tmp1, tmp2); __ j(done); __ bind(runtime); @@ -171,6 +178,31 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, } +static Register generate_region_crossing_test(MacroAssembler* masm, const Register store_addr, const Register new_val, const Register tmp1) { + __ xorr(tmp1, store_addr, new_val); // tmp1 := store address ^ new value + __ srli(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes) + return tmp1; +} + +static Register generate_card_young_test(MacroAssembler* masm, const Register store_addr, const Register tmp1, const Register tmp2) { + __ srli(tmp1, store_addr, CardTable::card_shift()); // tmp1 := card address relative to card table base + __ load_byte_map_base(tmp2); // tmp2 := card table base address + __ add(tmp1, tmp1, tmp2); // tmp1 := card address + __ lbu(tmp2, Address(tmp1)); // tmp2 := card + return tmp2; +} + +static Register generate_card_clean_test(MacroAssembler* masm, const Register tmp1 /* card address */, const Register tmp2) { + __ membar(Assembler::StoreLoad); // StoreLoad membar + __ lbu(tmp2, Address(tmp1)); // tmp2 := card + return tmp2; +} + +static void generate_dirty_card(MacroAssembler* masm, const Register tmp1 /* card address */) { + STATIC_ASSERT(CardTable::dirty_card_val() == 0); + __ sb(zr, Address(tmp1)); // *(card address) := dirty_card_val +} + void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, Register store_addr, Register new_val, @@ -182,70 +214,160 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, assert(store_addr != noreg && new_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register"); - Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset())); - Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset())); - - BarrierSet* bs = BarrierSet::barrier_set(); - CardTableBarrierSet* ctbs = barrier_set_cast(bs); - Label done; Label runtime; - // Does store cross heap regions? - - __ xorr(tmp1, store_addr, new_val); - __ srli(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); - __ beqz(tmp1, done); + Register is_region_crossing = generate_region_crossing_test(masm, store_addr, new_val, tmp1); + __ beqz(is_region_crossing, done); // crosses regions, storing null? - __ beqz(new_val, done); - // storing region crossing non-null, is card already dirty? + Register card_val = generate_card_young_test(masm, store_addr, tmp1, tmp2); + __ mv(t0, (int)G1CardTable::g1_young_card_val()); + __ beq(card_val, t0, done); // card == young_card_val? + // From here on, tmp1 holds the card address. - const Register card_addr = tmp1; + Register is_card_clean = generate_card_clean_test(masm, tmp1, tmp2); + __ beqz(is_card_clean, done); - __ srli(card_addr, store_addr, CardTable::card_shift()); + generate_queue_test_and_insertion(masm, + G1ThreadLocalData::dirty_card_queue_index_offset(), + G1ThreadLocalData::dirty_card_queue_buffer_offset(), + runtime, + thread, tmp1, tmp2, t0); + __ j(done); - // get the address of the card - __ load_byte_map_base(tmp2); - __ add(card_addr, card_addr, tmp2); - __ lbu(tmp2, Address(card_addr)); - __ mv(t0, (int)G1CardTable::g1_young_card_val()); - __ beq(tmp2, t0, done); + __ bind(runtime); + // save the live input values + RegSet saved = RegSet::of(store_addr); + __ push_reg(saved, sp); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp1, thread); + __ pop_reg(saved, sp); - assert((int)CardTable::dirty_card_val() == 0, "must be 0"); + __ bind(done); +} - __ membar(MacroAssembler::StoreLoad); +#if defined(COMPILER2) - __ lbu(tmp2, Address(card_addr)); - __ beqz(tmp2, done); +#undef __ +#define __ masm-> - // storing a region crossing, non-null oop, card is clean. - // dirty card and log. +static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) { + SaveLiveRegisters save_registers(masm, stub); + if (c_rarg0 != arg) { + __ mv(c_rarg0, arg); + } + __ mv(c_rarg1, xthread); + __ mv(t0, runtime_path); + __ jalr(t0); +} - __ sb(zr, Address(card_addr)); +void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm, + Register obj, + Register pre_val, + Register thread, + Register tmp1, + Register tmp2, + G1PreBarrierStubC2* stub) { + assert(thread == xthread, "must be"); + assert_different_registers(obj, pre_val, tmp1, tmp2); + assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register"); - __ ld(t0, queue_index); - __ beqz(t0, runtime); - __ sub(t0, t0, wordSize); - __ sd(t0, queue_index); + stub->initialize_registers(obj, pre_val, thread, tmp1, tmp2); - __ ld(tmp2, buffer); - __ add(t0, tmp2, t0); - __ sd(card_addr, Address(t0, 0)); - __ j(done); + Register is_marking_active = generate_marking_active_test(masm, thread, tmp1); + __ bnez(is_marking_active, *stub->entry()); + + __ bind(*stub->continuation()); +} + +void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm, + G1PreBarrierStubC2* stub) const { + Assembler::InlineSkippedInstructionsCounter skip_counter(masm); + Label runtime; + Register obj = stub->obj(); + Register pre_val = stub->pre_val(); + Register thread = stub->thread(); + Register tmp1 = stub->tmp1(); + Register tmp2 = stub->tmp2(); + + __ bind(*stub->entry()); + + Register is_pre_val_not_null = generate_pre_val_not_null_test(masm, obj, pre_val); + __ beqz(is_pre_val_not_null, *stub->continuation()); + + generate_queue_test_and_insertion(masm, + G1ThreadLocalData::satb_mark_queue_index_offset(), + G1ThreadLocalData::satb_mark_queue_buffer_offset(), + runtime, + thread, pre_val, tmp1, tmp2); + __ j(*stub->continuation()); __ bind(runtime); - // save the live input values - RegSet saved = RegSet::of(store_addr); - __ push_reg(saved, sp); - __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread); - __ pop_reg(saved, sp); + generate_c2_barrier_runtime_call(masm, stub, pre_val, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry)); + __ j(*stub->continuation()); +} - __ bind(done); +void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm, + Register store_addr, + Register new_val, + Register thread, + Register tmp1, + Register tmp2, + G1PostBarrierStubC2* stub) { + assert(thread == xthread, "must be"); + assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, t0); + assert(store_addr != noreg && new_val != noreg && tmp1 != noreg && tmp2 != noreg, + "expecting a register"); + + stub->initialize_registers(thread, tmp1, tmp2); + + Register is_region_crossing = generate_region_crossing_test(masm, store_addr, new_val, tmp1); + __ beqz(is_region_crossing, *stub->continuation()); + + // crosses regions, storing null? + if ((stub->barrier_data() & G1C2BarrierPostNotNull) == 0) { + __ beqz(new_val, *stub->continuation()); + } + + Register card_val = generate_card_young_test(masm, store_addr, tmp1, tmp2); + // From here on, tmp1 holds the card address. + __ mv(t0, (int)G1CardTable::g1_young_card_val()); + __ bne(card_val, t0, *stub->entry()) + + __ bind(*stub->continuation()); } +void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm, + G1PostBarrierStubC2* stub) const { + Assembler::InlineSkippedInstructionsCounter skip_counter(masm); + Label runtime; + Register thread = stub->thread(); + Register tmp1 = stub->tmp1(); // tmp1 holds the card address. + Register tmp2 = stub->tmp2(); + + __ bind(*stub->entry()); + + Register is_card_clean = generate_card_clean_test(masm, tmp1, tmp2); + __ beqz(is_card_clean, *stub->continuation()); + + generate_dirty_card(masm, tmp1); + + generate_queue_test_and_insertion(masm, + G1ThreadLocalData::dirty_card_queue_index_offset(), + G1ThreadLocalData::dirty_card_queue_buffer_offset(), + runtime, + thread, tmp1, tmp2, t0); + __ j(*stub->continuation()); + + __ bind(runtime); + generate_c2_barrier_runtime_call(masm, stub, tmp1, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry)); + __ j(*stub->continuation()); +} + +#endif // COMPILER2 + void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp2) { bool on_oop = is_reference_type(type); diff --git a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp index 96568994079dd..c7bee2ef6f3a8 100644 --- a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp +++ b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp @@ -1,6 +1,6 @@ /* * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -36,6 +36,8 @@ class LIR_Assembler; class StubAssembler; class G1PreBarrierStub; class G1PostBarrierStub; +class G1PreBarrierStubC2; +class G1PostBarrierStubC2; class G1BarrierSetAssembler: public ModRefBarrierSetAssembler { protected: @@ -72,6 +74,27 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler { void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm); #endif +#ifdef COMPILER2 + void g1_write_barrier_pre_c2(MacroAssembler* masm, + Register obj, + Register pre_val, + Register thread, + Register tmp1, + Register tmp2, + G1PreBarrierStubC2* c2_stub); + void generate_c2_pre_barrier_stub(MacroAssembler* masm, + G1PreBarrierStubC2* stub) const; + void g1_write_barrier_post_c2(MacroAssembler* masm, + Register store_addr, + Register new_val, + Register thread, + Register tmp1, + Register tmp2, + G1PostBarrierStubC2* c2_stub); + void generate_c2_post_barrier_stub(MacroAssembler* masm, + G1PostBarrierStubC2* stub) const; +#endif + void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp2); }; diff --git a/src/hotspot/cpu/riscv/gc/g1/g1_riscv.ad b/src/hotspot/cpu/riscv/gc/g1/g1_riscv.ad new file mode 100644 index 0000000000000..df1c1ba55f528 --- /dev/null +++ b/src/hotspot/cpu/riscv/gc/g1/g1_riscv.ad @@ -0,0 +1,574 @@ +// +// Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 2024, Huawei Technologies Co., Ltd. All rights reserved. +// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +// +// This code is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License version 2 only, as +// published by the Free Software Foundation. +// +// This code is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// version 2 for more details (a copy is included in the LICENSE file that +// accompanied this code). +// +// You should have received a copy of the GNU General Public License version +// 2 along with this work; if not, write to the Free Software Foundation, +// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +// +// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +// or visit www.oracle.com if you need additional information or have any +// questions. +// + +source_hpp %{ + +#include "gc/g1/c2/g1BarrierSetC2.hpp" +#include "gc/shared/gc_globals.hpp" + +%} + +source %{ + +#include "gc/g1/g1BarrierSetAssembler_riscv.hpp" +#include "gc/g1/g1BarrierSetRuntime.hpp" + +static void g1_pre_write_barrier(MacroAssembler* masm, + const MachNode* node, + Register obj, + Register pre_val, + Register tmp1, + Register tmp2, + RegSet preserve = RegSet(), + RegSet no_preserve = RegSet()) { + if (!G1PreBarrierStubC2::needs_barrier(node)) { + return; + } + Assembler::InlineSkippedInstructionsCounter skip_counter(masm); + G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler()); + G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node); + for (RegSetIterator reg = preserve.begin(); *reg != noreg; ++reg) { + stub->preserve(*reg); + } + for (RegSetIterator reg = no_preserve.begin(); *reg != noreg; ++reg) { + stub->dont_preserve(*reg); + } + g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, xthread, tmp1, tmp2, stub); +} + +static void g1_post_write_barrier(MacroAssembler* masm, + const MachNode* node, + Register store_addr, + Register new_val, + Register tmp1, + Register tmp2) { + if (!G1PostBarrierStubC2::needs_barrier(node)) { + return; + } + Assembler::InlineSkippedInstructionsCounter skip_counter(masm); + G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler()); + G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node); + g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, xthread, tmp1, tmp2, stub); +} + +%} + +instruct g1StoreP(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3) +%{ + predicate(UseG1GC && n->as_Store()->barrier_data() != 0); + match(Set mem (StoreP mem src)); + effect(TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(STORE_COST); + format %{ "sd $src, $mem\t# ptr" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + g1_pre_write_barrier(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $src$$Register) /* preserve */); + __ sd($src$$Register, $mem$$Register); + g1_post_write_barrier(masm, this, + $mem$$Register /* store_addr */, + $src$$Register /* new_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(istore_reg_mem); +%} + +instruct g1StoreN(indirect mem, iRegN src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3) +%{ + predicate(UseG1GC && n->as_Store()->barrier_data() != 0); + match(Set mem (StoreN mem src)); + effect(TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(STORE_COST); + format %{ "sw $src, $mem\t# compressed ptr" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + g1_pre_write_barrier(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $src$$Register) /* preserve */); + __ sw($src$$Register, $mem$$Register); + if ((barrier_data() & G1C2BarrierPost) != 0) { + if ((barrier_data() & G1C2BarrierPostNotNull) == 0) { + __ decode_heap_oop($tmp1$$Register, $src$$Register); + } else { + __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register); + } + } + g1_post_write_barrier(masm, this, + $mem$$Register /* store_addr */, + $tmp1$$Register /* new_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(istore_reg_mem); +%} + +instruct g1EncodePAndStoreN(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3) +%{ + predicate(UseG1GC && n->as_Store()->barrier_data() != 0); + match(Set mem (StoreN mem (EncodeP src))); + effect(TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(STORE_COST); + format %{ "encode_heap_oop $tmp1, $src\n\t" + "sw $tmp1, $mem\t# compressed ptr" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + g1_pre_write_barrier(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $src$$Register) /* preserve */); + if ((barrier_data() & G1C2BarrierPostNotNull) == 0) { + __ encode_heap_oop($tmp1$$Register, $src$$Register); + } else { + __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register); + } + __ sw($tmp1$$Register, $mem$$Register); + g1_post_write_barrier(masm, this, + $mem$$Register /* store_addr */, + $src$$Register /* new_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(istore_reg_mem); +%} + +instruct g1CompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); + match(Set res (CompareAndExchangeP mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(2 * VOLATILE_REF_COST); + format %{ "cmpxchg $res = $mem, $oldval, $newval\t# ptr" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + g1_pre_write_barrier(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ mv($tmp1$$Register, $oldval$$Register); + __ mv($tmp2$$Register, $newval$$Register); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register); + g1_post_write_barrier(masm, this, + $mem$$Register /* store_addr */, + $tmp2$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3) +%{ + predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0); + match(Set res (CompareAndExchangeP mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(VOLATILE_REF_COST); + format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# ptr" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + g1_pre_write_barrier(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ mv($tmp1$$Register, $oldval$$Register); + __ mv($tmp2$$Register, $newval$$Register); + __ mcpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register); + g1_post_write_barrier(masm, this, + $mem$$Register /* store_addr */, + $tmp2$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); + match(Set res (CompareAndExchangeN mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(2 * VOLATILE_REF_COST); + format %{ "cmpxchg $res = $mem, $oldval, $newval\t# narrow oop" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + g1_pre_write_barrier(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ mv($tmp1$$Register, $oldval$$Register); + __ mv($tmp2$$Register, $newval$$Register); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::uint32, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register); + __ decode_heap_oop($tmp2$$Register); + g1_post_write_barrier(masm, this, + $mem$$Register /* store_addr */, + $tmp2$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3) +%{ + predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0); + match(Set res (CompareAndExchangeN mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(VOLATILE_REF_COST); + format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# narrow oop" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + g1_pre_write_barrier(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ mv($tmp1$$Register, $oldval$$Register); + __ mv($tmp2$$Register, $newval$$Register); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::uint32, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register); + __ decode_heap_oop($tmp2$$Register); + g1_post_write_barrier(masm, this, + $mem$$Register /* store_addr */, + $tmp2$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndSwapP(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegP oldval) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); + match(Set res (CompareAndSwapP mem (Binary oldval newval))); + match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); + ins_cost(2 * VOLATILE_REF_COST); + format %{ "cmpxchg $mem, $oldval, $newval\t# (ptr)\n\t" + "mv $res, $res == $oldva" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + g1_pre_write_barrier(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ mv($tmp1$$Register, $oldval$$Register); + __ mv($tmp2$$Register, $newval$$Register); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + g1_post_write_barrier(masm, this, + $mem$$Register /* store_addr */, + $tmp2$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegP oldval) +%{ + predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0); + match(Set res (CompareAndSwapP mem (Binary oldval newval))); + match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(VOLATILE_REF_COST); + format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (ptr)\n\t" + "mv $res, $res == $oldval" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + g1_pre_write_barrier(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ mv($tmp1$$Register, $oldval$$Register); + __ mv($tmp2$$Register, $newval$$Register); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + g1_post_write_barrier(masm, this, + $mem$$Register /* store_addr */, + $tmp2$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndSwapN(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); + match(Set res (CompareAndSwapN mem (Binary oldval newval))); + match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(2 * VOLATILE_REF_COST); + format %{ "cmpxchg $mem, $oldval, $newval\t# (narrow oop)\n\t" + "mv $res, $res == $oldval" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + g1_pre_write_barrier(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ mv($tmp1$$Register, $oldval$$Register); + __ mv($tmp2$$Register, $newval$$Register); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::uint32, + /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $register$$Register, + /*result as bool*/ true); + __ decode_heap_oop($tmp2$$Register); + g1_post_write_barrier(masm, this, + $mem$$Register /* store_addr */, + $tmp2$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1CompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval) +%{ + predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0); + match(Set res (CompareAndSwapN mem (Binary oldval newval))); + match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); + effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(VOLATILE_REF_COST); + format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (narrow oop)\n\t" + "mv $res, $res == $oldval" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($oldval$$Register, $mem$$Register); + assert_different_registers($newval$$Register, $mem$$Register); + g1_pre_write_barrier(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, + RegSet::of($res$$Register) /* no_preserve */); + __ mv($tmp1$$Register, $oldval$$Register); + __ mv($tmp2$$Register, $newval$$Register); + __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::uint32, + /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register, + /*result as bool*/ true); + __ decode_heap_oop($tmp2$$Register); + g1_post_write_barrier(masm, this, + $mem$$Register /* store_addr */, + $tmp2$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_slow); +%} + +instruct g1XChgP(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); + match(Set preval (GetAndSetP mem newval)); + effect(TEMP preval, TEMP tmp1, TEMP tmp2); + ins_cost(2 * VOLATILE_REF_COST); + format %{ "atomic_xchg $preval, $newval, [$mem]" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($mem$$Register, $newval$$Register); + g1_pre_write_barrier(masm, this, + $mem$$Register /* obj */, + $preval$$Register /* pre_val (as a temporary register) */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */, + RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */); + __ atomic_xchg($preval$$Register, $newval$$Register, $mem$$Register); + g1_post_write_barrier(masm, this, + $mem$$Register /* store_addr */, + $newval$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */); + %} + ins_pipe(pipe_serial); +%} + +instruct g1XChgPAcq(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval) +%{ + predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0); + match(Set preval (GetAndSetP mem newval)); + effect(TEMP preval, TEMP tmp1, TEMP tmp2); + ins_cost(VOLATILE_REF_COST); + format %{ "atomic_xchg_acq $preval, $newval, [$mem]" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($mem$$Register, $newval$$Register); + g1_pre_write_barrier(masm, this, + $mem$$Register /* obj */, + $preval$$Register /* pre_val (as a temporary register) */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */, + RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */); + __ atomic_xchgal($preval$$Register, $newval$$Register, $mem$$Register); + g1_post_write_barrier(masm, this, + $mem$$Register /* store_addr */, + $newval$$Register /* new_val */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */); + %} + ins_pipe(pipe_serial); +%} + +instruct g1XChgN(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval) +%{ + predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0); + match(Set preval (GetAndSetN mem newval)); + effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(2 * VOLATILE_REF_COST); + format %{ "atomic_xchgwu $preval, $newval, [$mem]" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($mem$$Register, $newval$$Register); + g1_pre_write_barrier(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */); + __ atomic_xchgwu($preval$$Register, $newval$$Register, $mem$$Register); + __ decode_heap_oop($tmp1$$Register, $newval$$Register); + g1_post_write_barrier(masm, this, + $mem$$Register /* store_addr */, + $tmp1$$Register /* new_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_serial); +%} + +instruct g1XChgNAcq(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval) +%{ + predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0); + match(Set preval (GetAndSetN mem newval)); + effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(VOLATILE_REF_COST); + format %{ "atomic_xchgwu_acq $preval, $newval, [$mem]" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + assert_different_registers($mem$$Register, $newval$$Register); + g1_pre_write_barrier(masm, this, + $mem$$Register /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */, + RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */); + __ atomic_xchgalwu($preval$$Register, $newval$$Register, $mem$$Register); + __ decode_heap_oop($tmp1$$Register, $newval$$Register); + g1_post_write_barrier(masm, this, + $mem$$Register /* store_addr */, + $tmp1$$Register /* new_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + %} + ins_pipe(pipe_serial); +%} + +instruct g1LoadP(iRegPNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr) +%{ + predicate(UseG1GC && n->as_Load()->barrier_data() != 0); + match(Set dst (LoadP mem)); + effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr); + ins_cost(LOAD_COST + BRANCH_COST); + format %{ "ld $dst, $mem\t# ptr" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + __ ld($dst$$Register, Address($mem$$Register)); + g1_pre_write_barrier(masm, this, + noreg /* obj */, + $dst$$Register /* pre_val */, + $tmp1$$Register /* tmp1 */, + $tmp2$$Register /* tmp2 */); + %} + ins_pipe(iload_reg_mem); +%} + +instruct g1LoadN(iRegNNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3) +%{ + predicate(UseG1GC && n->as_Load()->barrier_data() != 0); + match(Set dst (LoadN mem)); + effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3); + ins_cost(VOLATILE_REF_COST); + format %{ "lwu $dst, $mem\t# compressed ptr" %} + ins_encode %{ + guarantee($mem$$disp == 0, "impossible encoding"); + __ lwu($dst$$Register, Address($mem$$Register)); + if ((barrier_data() & G1C2BarrierPre) != 0) { + __ decode_heap_oop($tmp1$$Register, $dst$$Register); + g1_pre_write_barrier(masm, this, + noreg /* obj */, + $tmp1$$Register /* pre_val */, + $tmp2$$Register /* tmp1 */, + $tmp3$$Register /* tmp2 */); + } + %} + ins_pipe(iload_reg_mem); +%} diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index 5405314875898..a463c8a33f21d 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -1,7 +1,7 @@ // // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved. // Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. -// Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved. +// Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it @@ -2197,7 +2197,8 @@ bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) { assert_cond(m != nullptr); if (is_vshift_con_pattern(n, m) || // ShiftV src (ShiftCntV con) is_vector_bitwise_not_pattern(n, m) || - is_vector_scalar_bitwise_pattern(n, m)) { + is_vector_scalar_bitwise_pattern(n, m) || + is_encode_and_store_pattern(n, m)) { mstack.push(m, Visit); return true; } @@ -4744,6 +4745,7 @@ instruct loadP(iRegPNoSp dst, memory mem) // Load Compressed Pointer instruct loadN(iRegNNoSp dst, memory mem) %{ + predicate(n->as_Load()->barrier_data() == 0); match(Set dst (LoadN mem)); ins_cost(LOAD_COST); @@ -5179,6 +5181,7 @@ instruct storeimmP0(immP0 zero, memory mem) // Store Compressed Pointer instruct storeN(iRegN src, memory mem) %{ + predicate(n->as_Store()->barrier_data() == 0); match(Set mem (StoreN mem src)); ins_cost(STORE_COST); @@ -5193,6 +5196,7 @@ instruct storeN(iRegN src, memory mem) instruct storeImmN0(immN0 zero, memory mem) %{ + predicate(n->as_Store()->barrier_data() == 0); match(Set mem (StoreN mem zero)); ins_cost(STORE_COST); @@ -5383,6 +5387,7 @@ instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); match(Set res (CompareAndSwapN mem (Binary oldval newval))); ins_cost(LOAD_COST + STORE_COST + ALU_COST * 8 + BRANCH_COST * 4); @@ -5504,7 +5509,7 @@ instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP new instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval) %{ - predicate(needs_acquiring_load_reserved(n)); + predicate(needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == 0); match(Set res (CompareAndSwapN mem (Binary oldval newval))); @@ -5612,6 +5617,7 @@ instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL ne instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); match(Set res (CompareAndExchangeN mem (Binary oldval newval))); ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 3); @@ -5745,7 +5751,7 @@ instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval) %{ - predicate(needs_acquiring_load_reserved(n)); + predicate(needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == 0); match(Set res (CompareAndExchangeN mem (Binary oldval newval))); @@ -5873,6 +5879,7 @@ instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL ne instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 4); @@ -6004,7 +6011,7 @@ instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval) %{ - predicate(needs_acquiring_load_reserved(n)); + predicate(needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == 0); match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); @@ -6076,6 +6083,8 @@ instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{ + predicate(n->as_LoadStore()->barrier_data() == 0); + match(Set prev (GetAndSetN mem newv)); ins_cost(ALU_COST); @@ -6141,7 +6150,7 @@ instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{ - predicate(needs_acquiring_load_reserved(n)); + predicate(needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == 0); match(Set prev (GetAndSetN mem newv));