Skip to content

Commit

Permalink
riscv port for JEP 475
Browse files Browse the repository at this point in the history
  • Loading branch information
@feilongjiang
feilongjiang committed Sep 5, 2024
1 parent 9821e79 commit 1c012cf
Show file tree
Hide file tree
Showing 4 changed files with 811 additions and 89 deletions.
274 changes: 191 additions & 83 deletions src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -39,7 +39,10 @@
#include "c1/c1_LIRAssembler.hpp"
#include "c1/c1_MacroAssembler.hpp"
#include "gc/g1/c1/g1BarrierSetC1.hpp"
#endif
#endif // COMPILER1
#ifdef COMPILER2
#include "gc/g1/c2/g1BarrierSetC2.hpp"
#endif // COMPILER2

#define __ masm->

Expand Down Expand Up @@ -96,6 +99,55 @@ void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* mas
__ pop_reg(saved_regs, sp);
}

static void generate_queue_test_and_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
const Register thread, const Register value, const Register tmp1, const Register tmp2) {
// Can we store a value in the given thread's buffer?
// (The index field is typed as size_t.)
__ ld(tmp1, Address(thread, in_bytes(index_offset))); // tmp1 := *(index address)
__ beqz(tmp1, runtime); // jump to runtime if index == 0 (full buffer)
// The buffer is not full, store value into it.
__ sub(tmp1, tmp1, wordSize); // tmp1 := next index
__ sd(tmp1, Address(thread, in_bytes(index_offset))); // *(index address) := next index
__ ld(tmp2, Address(thread, in_bytes(buffer_offset))); // tmp2 := buffer address
__ add(tmp2, tmp2, tmp1);
__ sd(value, Address(tmp2)); // *(buffer address + next index) := value
}

static void generate_pre_barrier_fast_path(MacroAssembler* masm,
const Register thread,
const Register tmp1) {
Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
// Is marking active?
if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
__ lwu(tmp1, in_progress);
} else {
assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
__ lbu(tmp1, in_progress);
}
}

static void generate_pre_barrier_slow_path(MacroAssembler* masm,
const Register obj,
const Register pre_val,
const Register thread,
const Register tmp1,
const Register tmp2,
Label& done,
Label& runtime) {
// Do we need to load the previous value?
if (obj != noreg) {
__ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
}
// Is the previous value null?
__ beqz(pre_val, done, true);
generate_queue_test_and_insertion(masm,
G1ThreadLocalData::satb_mark_queue_index_offset(),
G1ThreadLocalData::satb_mark_queue_buffer_offset(),
runtime,
thread, pre_val, tmp1, tmp2);
__ j(done);
}

void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
Register obj,
Register pre_val,
Expand All @@ -116,43 +168,10 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
assert_different_registers(obj, pre_val, tmp1, tmp2);
assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");

Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
Address index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));

// Is marking active?
if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { // 4-byte width
__ lwu(tmp1, in_progress);
} else {
assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
__ lbu(tmp1, in_progress);
}
generate_pre_barrier_fast_path(masm, thread, tmp1);
// If marking is not active (*(mark queue active address) == 0), jump to done
__ beqz(tmp1, done);

// Do we need to load the previous value?
if (obj != noreg) {
__ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
}

// Is the previous value null?
__ beqz(pre_val, done);

// Can we store original value in the thread's buffer?
// Is index == 0?
// (The index field is typed as size_t.)

__ ld(tmp1, index); // tmp := *index_adr
__ beqz(tmp1, runtime); // tmp == 0?
// If yes, goto runtime

__ sub(tmp1, tmp1, wordSize); // tmp := tmp - wordSize
__ sd(tmp1, index); // *index_adr := tmp
__ ld(tmp2, buffer);
__ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr

// Record the previous value
__ sd(pre_val, Address(tmp1, 0));
__ j(done);
generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp1, tmp2, done, runtime);

__ bind(runtime);

Expand All @@ -171,6 +190,49 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,

}

static void generate_post_barrier_fast_path(MacroAssembler* masm,
const Register store_addr,
const Register new_val,
const Register tmp1,
const Register tmp2,
Label& done,
bool new_val_may_be_null) {
// Does store cross heap regions?
__ xorr(tmp1, store_addr, new_val); // tmp1 := store address ^ new value
__ srli(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes)
__ beqz(tmp1, done);
// Crosses regions, storing null?
if (new_val_may_be_null) {
__ beqz(new_val, done);
}
// Storing region crossing non-null, is card young?
__ srli(tmp1, store_addr, CardTable::card_shift()); // tmp1 := card address relative to card table base
__ load_byte_map_base(tmp2); // tmp2 := card table base address
__ add(tmp1, tmp1, tmp2); // tmp1 := card address
__ lbu(tmp2, Address(tmp1)); // tmp2 := card
}

static void generate_post_barrier_slow_path(MacroAssembler* masm,
const Register thread,
const Register tmp1,
const Register tmp2,
Label& done,
Label& runtime) {
__ membar(MacroAssembler::StoreLoad); // StoreLoad membar
__ lbu(tmp2, Address(tmp1)); // tmp2 := card
__ beqz(tmp2, done, true);
// Storing a region crossing, non-null oop, card is clean.
// Dirty card and log.
STATIC_ASSERT(CardTable::dirty_card_val() == 0);
__ sb(zr, Address(tmp1)); // *(card address) := dirty_card_val
generate_queue_test_and_insertion(masm,
G1ThreadLocalData::dirty_card_queue_index_offset(),
G1ThreadLocalData::dirty_card_queue_buffer_offset(),
runtime,
thread, tmp1, tmp2, t0);
__ j(done);
}

void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
Register store_addr,
Register new_val,
Expand All @@ -179,73 +241,119 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
Register tmp2) {
assert(thread == xthread, "must be");
assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, t0);
assert(store_addr != noreg && new_val != noreg && tmp1 != noreg &&
tmp2 != noreg, "expecting a register");

Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));

BarrierSet* bs = BarrierSet::barrier_set();
CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
assert(store_addr != noreg && new_val != noreg && tmp1 != noreg && tmp2 != noreg,
"expecting a register");

Label done;
Label runtime;

// Does store cross heap regions?
generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, done, true /* new_val_may_be_null */);
// If card is young, jump to done (tmp2 holds the card value)
__ mv(t0, (int)G1CardTable::g1_young_card_val());
__ beq(tmp2, t0, done); // card == young_card_val?
generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, done, runtime);

__ xorr(tmp1, store_addr, new_val);
__ srli(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes);
__ beqz(tmp1, done);
__ bind(runtime);
// save the live input values
RegSet saved = RegSet::of(store_addr);
__ push_reg(saved, sp);
__ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp1, thread);
__ pop_reg(saved, sp);

// crosses regions, storing null?
__ bind(done);
}

__ beqz(new_val, done);
#if defined(COMPILER2)

// storing region crossing non-null, is card already dirty?
static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) {
SaveLiveRegisters save_registers(masm, stub);
if (c_rarg0 != arg) {
__ mv(c_rarg0, arg);
}
__ mv(c_rarg1, xthread);
__ mv(t0, runtime_path);
__ jalr(t0);
}

const Register card_addr = tmp1;
void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm,
Register obj,
Register pre_val,
Register thread,
Register tmp1,
Register tmp2,
G1PreBarrierStubC2* stub) {
assert(thread == xthread, "must be");
assert_different_registers(obj, pre_val, tmp1, tmp2);
assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");

__ srli(card_addr, store_addr, CardTable::card_shift());
stub->initialize_registers(obj, pre_val, thread, tmp1, tmp2);

// get the address of the card
__ load_byte_map_base(tmp2);
__ add(card_addr, card_addr, tmp2);
__ lbu(tmp2, Address(card_addr));
__ mv(t0, (int)G1CardTable::g1_young_card_val());
__ beq(tmp2, t0, done);
generate_pre_barrier_fast_path(masm, thread, tmp1);
// If marking is active (*(mark queue active address) != 0), jump to stub (slow path)
__ bnez(tmp1, *stub->entry(), true);

assert((int)CardTable::dirty_card_val() == 0, "must be 0");
__ bind(*stub->continuation());
}

__ membar(MacroAssembler::StoreLoad);
void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm,
G1PreBarrierStubC2* stub) const {
Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
Label runtime;
Register obj = stub->obj();
Register pre_val = stub->pre_val();
Register thread = stub->thread();
Register tmp1 = stub->tmp1();
Register tmp2 = stub->tmp2();

__ lbu(tmp2, Address(card_addr));
__ beqz(tmp2, done);
__ bind(*stub->entry());
generate_pre_barrier_slow_path(masm, obj, pre_val, thread, tmp1, tmp2, *stub->continuation(), runtime);

// storing a region crossing, non-null oop, card is clean.
// dirty card and log.
__ bind(runtime);
generate_c2_barrier_runtime_call(masm, stub, pre_val, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry));
__ j(*stub->continuation());
}

__ sb(zr, Address(card_addr));
void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
Register store_addr,
Register new_val,
Register thread,
Register tmp1,
Register tmp2,
G1PostBarrierStubC2* stub) {
assert(thread == xthread, "must be");
assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, t0);
assert(store_addr != noreg && new_val != noreg && tmp1 != noreg && tmp2 != noreg,
"expecting a register");

__ ld(t0, queue_index);
__ beqz(t0, runtime);
__ sub(t0, t0, wordSize);
__ sd(t0, queue_index);
stub->initialize_registers(thread, tmp1, tmp2);

__ ld(tmp2, buffer);
__ add(t0, tmp2, t0);
__ sd(card_addr, Address(t0, 0));
__ j(done);
bool new_val_may_be_null = (stub->barrier_data() & G1C2BarrierPostNotNull) == 0;
generate_post_barrier_fast_path(masm, store_addr, new_val, tmp1, tmp2, *stub->continuation(), new_val_may_be_null);
// If card is not young, jump to stub (slow path) (tmp2 holds the card value)
__ mv(t0, (int)G1CardTable::g1_young_card_val());
__ bne(tmp2, t0, *stub->entry(), true);

__ bind(runtime);
// save the live input values
RegSet saved = RegSet::of(store_addr);
__ push_reg(saved, sp);
__ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
__ pop_reg(saved, sp);
__ bind(*stub->continuation());
}

__ bind(done);
void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
G1PostBarrierStubC2* stub) const {
Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
Label runtime;
Register thread = stub->thread();
Register tmp1 = stub->tmp1(); // tmp1 holds the card address.
Register tmp2 = stub->tmp2();

__ bind(*stub->entry());
generate_post_barrier_slow_path(masm, thread, tmp1, tmp2, *stub->continuation(), runtime);

__ bind(runtime);
generate_c2_barrier_runtime_call(masm, stub, tmp1, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
__ j(*stub->continuation());
}

#endif // COMPILER2

void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp2) {
bool on_oop = is_reference_type(type);
Expand Down
25 changes: 24 additions & 1 deletion src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.hpp
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
* Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -36,6 +36,8 @@ class LIR_Assembler;
class StubAssembler;
class G1PreBarrierStub;
class G1PostBarrierStub;
class G1PreBarrierStubC2;
class G1PostBarrierStubC2;

class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
protected:
Expand Down Expand Up @@ -72,6 +74,27 @@ class G1BarrierSetAssembler: public ModRefBarrierSetAssembler {
void generate_c1_post_barrier_runtime_stub(StubAssembler* sasm);
#endif

#ifdef COMPILER2
void g1_write_barrier_pre_c2(MacroAssembler* masm,
Register obj,
Register pre_val,
Register thread,
Register tmp1,
Register tmp2,
G1PreBarrierStubC2* c2_stub);
void generate_c2_pre_barrier_stub(MacroAssembler* masm,
G1PreBarrierStubC2* stub) const;
void g1_write_barrier_post_c2(MacroAssembler* masm,
Register store_addr,
Register new_val,
Register thread,
Register tmp1,
Register tmp2,
G1PostBarrierStubC2* c2_stub);
void generate_c2_post_barrier_stub(MacroAssembler* masm,
G1PostBarrierStubC2* stub) const;
#endif

void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp2);
};
Expand Down
Loading

0 comments on commit 1c012cf

Please sign in to comment.