Skip to content

Commit

Permalink
riscv port for JEP 475
Browse files Browse the repository at this point in the history
  • Loading branch information
@feilongjiang
feilongjiang committed Jul 3, 2024
1 parent 30c36a3 commit dbd3f9f
Show file tree
Hide file tree
Showing 5 changed files with 819 additions and 85 deletions.
1 change: 1 addition & 0 deletions make/hotspot/gensrc/GensrcAdlc.gmk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,7 @@ ifeq ($(call check-jvm-feature, compiler2), true)
ifeq ($(call check-jvm-feature, g1gc), true)
AD_SRC_FILES += $(call uniq, $(wildcard $(foreach d, $(AD_SRC_ROOTS), \
$d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/g1/g1_$(HOTSPOT_TARGET_CPU).ad \
$d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/g1/g1_$(HOTSPOT_TARGET_CPU_ARCH).ad \
)))
endif

Expand Down
285 changes: 206 additions & 79 deletions src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -39,7 +39,10 @@
#include "c1/c1_LIRAssembler.hpp"
#include "c1/c1_MacroAssembler.hpp"
#include "gc/g1/c1/g1BarrierSetC1.hpp"
#endif
#endif // COMPILER1
#ifdef COMPILER2
#include "gc/g1/c2/g1BarrierSetC2.hpp"
#endif // COMPILER2

#define __ masm->

Expand Down Expand Up @@ -96,6 +99,38 @@ void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* mas
__ pop_reg(saved_regs, sp);
}

static void generate_queue_test_and_insertion(MacroAssembler* masm, ByteSize index_offset, ByteSize buffer_offset, Label& runtime,
const Register thread, const Register value, const Register tmp1, const Register tmp2) {
// Can we store a value in the given thread's buffer?
// (The index field is typed as size_t.)
__ ld(tmp1, Address(thread, in_bytes(index_offset))); // tmp1 := *(index address)
__ beqz(tmp1, runtime); // jump to runtime if index == 0 (full buffer)
// The buffer is not full, store value into it.
__ sub(tmp1, tmp1, wordSize); // tmp1 := next index
__ sd(tmp1, Address(thread, in_bytes(index_offset))); // *(index address) := next index
__ ld(tmp2, Address(thread, in_bytes(buffer_offset))); // tmp2 := buffer address
__ add(tmp2, tmp2, tmp1);
__ sd(value, Address(tmp2)); // *(buffer address + next index) := value
}

static Register generate_marking_active_test(MacroAssembler* masm, const Register thread, const Register tmp1) {
Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
__ lwu(tmp1, in_progress); // tmp1 := *(mark queue active address)
} else {
assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
__ lbu(tmp1, in_progress); // tmp1 := *(mark queue active address)
}
return tmp1;
}

static Register generate_pre_val_not_null_test(MacroAssembler* masm, const Register obj, const Register pre_val) {
if (obj != noreg) {
__ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW); // pre_val := previous value
}
return pre_val;
}

void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
Register obj,
Register pre_val,
Expand All @@ -116,42 +151,17 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
assert_different_registers(obj, pre_val, tmp1, tmp2);
assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");

Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
Address index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));

// Is marking active?
if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { // 4-byte width
__ lwu(tmp1, in_progress);
} else {
assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
__ lbu(tmp1, in_progress);
}
Register is_marking_active = generate_marking_active_test(masm, thread, tmp1);
__ beqz(tmp1, done);

// Do we need to load the previous value?
if (obj != noreg) {
__ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
}

// Is the previous value null?
__ beqz(pre_val, done);

// Can we store original value in the thread's buffer?
// Is index == 0?
// (The index field is typed as size_t.)

__ ld(tmp1, index); // tmp := *index_adr
__ beqz(tmp1, runtime); // tmp == 0?
// If yes, goto runtime

__ sub(tmp1, tmp1, wordSize); // tmp := tmp - wordSize
__ sd(tmp1, index); // *index_adr := tmp
__ ld(tmp2, buffer);
__ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr
Register is_pre_val_not_null = generate_pre_val_not_null_test(masm, obj, pre_val);
__ beqz(is_pre_val_not_null, done);

// Record the previous value
__ sd(pre_val, Address(tmp1, 0));
generate_queue_test_and_insertion(masm,
G1ThreadLocalData::satb_mark_queue_index_offset(),
G1ThreadLocalData::satb_mark_queue_buffer_offset(),
runtime,
thread, pre_val, tmp1, tmp2);
__ j(done);

__ bind(runtime);
Expand All @@ -171,6 +181,31 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,

}

static Register generate_region_crossing_test(MacroAssembler* masm, const Register store_addr, const Register new_val, const Register tmp1) {
__ xorr(tmp1, store_addr, new_val); // tmp1 := store address ^ new value
__ srli(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes); // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes)
return tmp1;
}

static Register generate_card_young_test(MacroAssembler* masm, const Register store_addr, const Register tmp1, const Register tmp2) {
__ srli(tmp1, store_addr, CardTable::card_shift()); // tmp1 := card address relative to card table base
__ load_byte_map_base(tmp2); // tmp2 := card table base address
__ add(tmp1, tmp1, tmp2); // tmp1 := card address
__ lbu(tmp2, Address(tmp1)); // tmp2 := card
return tmp2;
}

static Register generate_card_clean_test(MacroAssembler* masm, const Register tmp1 /* card address */, const Register tmp2) {
__ membar(MacroAssembler::StoreLoad); // StoreLoad membar
__ lbu(tmp2, Address(tmp1)); // tmp2 := card
return tmp2;
}

static void generate_dirty_card(MacroAssembler* masm, const Register tmp1 /* card address */) {
STATIC_ASSERT(CardTable::dirty_card_val() == 0);
__ sb(zr, Address(tmp1)); // *(card address) := dirty_card_val
}

void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
Register store_addr,
Register new_val,
Expand All @@ -179,73 +214,165 @@ void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
Register tmp2) {
assert(thread == xthread, "must be");
assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, t0);
assert(store_addr != noreg && new_val != noreg && tmp1 != noreg &&
tmp2 != noreg, "expecting a register");

Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));

BarrierSet* bs = BarrierSet::barrier_set();
CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
assert(store_addr != noreg && new_val != noreg && tmp1 != noreg && tmp2 != noreg,
"expecting a register");

Label done;
Label runtime;

// Does store cross heap regions?

__ xorr(tmp1, store_addr, new_val);
__ srli(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes);
__ beqz(tmp1, done);
Register is_region_crossing = generate_region_crossing_test(masm, store_addr, new_val, tmp1);
__ beqz(is_region_crossing, done);

// crosses regions, storing null?

__ beqz(new_val, done);

// storing region crossing non-null, is card already dirty?

const Register card_addr = tmp1;

__ srli(card_addr, store_addr, CardTable::card_shift());

// get the address of the card
__ load_byte_map_base(tmp2);
__ add(card_addr, card_addr, tmp2);
__ lbu(tmp2, Address(card_addr));
Register card_val = generate_card_young_test(masm, store_addr, tmp1, tmp2);
// From here on, tmp1 holds the card address.
__ mv(t0, (int)G1CardTable::g1_young_card_val());
__ beq(tmp2, t0, done);

assert((int)CardTable::dirty_card_val() == 0, "must be 0");

__ membar(MacroAssembler::StoreLoad);
__ beq(card_val, t0, done); // card == young_card_val?

__ lbu(tmp2, Address(card_addr));
__ beqz(tmp2, done);
Register is_card_clean = generate_card_clean_test(masm, tmp1, tmp2);
__ beqz(is_card_clean, done);

// storing a region crossing, non-null oop, card is clean.
// dirty card and log.

__ sb(zr, Address(card_addr));

__ ld(t0, queue_index);
__ beqz(t0, runtime);
__ sub(t0, t0, wordSize);
__ sd(t0, queue_index);
generate_dirty_card(masm, tmp1);

__ ld(tmp2, buffer);
__ add(t0, tmp2, t0);
__ sd(card_addr, Address(t0, 0));
generate_queue_test_and_insertion(masm,
G1ThreadLocalData::dirty_card_queue_index_offset(),
G1ThreadLocalData::dirty_card_queue_buffer_offset(),
runtime,
thread, tmp1, tmp2, t0);
__ j(done);

__ bind(runtime);
// save the live input values
RegSet saved = RegSet::of(store_addr);
__ push_reg(saved, sp);
__ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread);
__ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), tmp1, thread);
__ pop_reg(saved, sp);

__ bind(done);
}

#if defined(COMPILER2)

#undef __
#define __ masm->

static void generate_c2_barrier_runtime_call(MacroAssembler* masm, G1BarrierStubC2* stub, const Register arg, const address runtime_path) {
SaveLiveRegisters save_registers(masm, stub);
if (c_rarg0 != arg) {
__ mv(c_rarg0, arg);
}
__ mv(c_rarg1, xthread);
__ mv(t0, runtime_path);
__ jalr(t0);
}

void G1BarrierSetAssembler::g1_write_barrier_pre_c2(MacroAssembler* masm,
Register obj,
Register pre_val,
Register thread,
Register tmp1,
Register tmp2,
G1PreBarrierStubC2* stub) {
assert(thread == xthread, "must be");
assert_different_registers(obj, pre_val, tmp1, tmp2);
assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");

stub->initialize_registers(obj, pre_val, thread, tmp1, tmp2);

Register is_marking_active = generate_marking_active_test(masm, thread, tmp1);
__ bnez(is_marking_active, *stub->entry(), true);

__ bind(*stub->continuation());
}

void G1BarrierSetAssembler::generate_c2_pre_barrier_stub(MacroAssembler* masm,
G1PreBarrierStubC2* stub) const {
Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
Label runtime;
Register obj = stub->obj();
Register pre_val = stub->pre_val();
Register thread = stub->thread();
Register tmp1 = stub->tmp1();
Register tmp2 = stub->tmp2();

__ bind(*stub->entry());

Register is_pre_val_not_null = generate_pre_val_not_null_test(masm, obj, pre_val);
__ beqz(is_pre_val_not_null, *stub->continuation(), true);

generate_queue_test_and_insertion(masm,
G1ThreadLocalData::satb_mark_queue_index_offset(),
G1ThreadLocalData::satb_mark_queue_buffer_offset(),
runtime,
thread, pre_val, tmp1, tmp2);
__ j(*stub->continuation());

__ bind(runtime);
generate_c2_barrier_runtime_call(masm, stub, pre_val, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry));
__ j(*stub->continuation());
}

void G1BarrierSetAssembler::g1_write_barrier_post_c2(MacroAssembler* masm,
Register store_addr,
Register new_val,
Register thread,
Register tmp1,
Register tmp2,
G1PostBarrierStubC2* stub) {
assert(thread == xthread, "must be");
assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, t0);
assert(store_addr != noreg && new_val != noreg && tmp1 != noreg && tmp2 != noreg,
"expecting a register");

stub->initialize_registers(thread, tmp1, tmp2);

Register is_region_crossing = generate_region_crossing_test(masm, store_addr, new_val, tmp1);
__ beqz(is_region_crossing, *stub->continuation());

// crosses regions, storing null?
if ((stub->barrier_data() & G1C2BarrierPostNotNull) == 0) {
__ beqz(new_val, *stub->continuation());
}

Register card_val = generate_card_young_test(masm, store_addr, tmp1, tmp2);
// From here on, tmp1 holds the card address.
__ mv(t0, (int)G1CardTable::g1_young_card_val());
__ bne(card_val, t0, *stub->entry(), true);

__ bind(*stub->continuation());
}

void G1BarrierSetAssembler::generate_c2_post_barrier_stub(MacroAssembler* masm,
G1PostBarrierStubC2* stub) const {
Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
Label runtime;
Register thread = stub->thread();
Register tmp1 = stub->tmp1(); // tmp1 holds the card address.
Register tmp2 = stub->tmp2();

__ bind(*stub->entry());

Register is_card_clean = generate_card_clean_test(masm, tmp1, tmp2);
__ beqz(is_card_clean, *stub->continuation(), true);

generate_dirty_card(masm, tmp1);

generate_queue_test_and_insertion(masm,
G1ThreadLocalData::dirty_card_queue_index_offset(),
G1ThreadLocalData::dirty_card_queue_buffer_offset(),
runtime,
thread, tmp1, tmp2, t0);
__ j(*stub->continuation());

__ bind(runtime);
generate_c2_barrier_runtime_call(masm, stub, tmp1, CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry));
__ j(*stub->continuation());
}

#endif // COMPILER2

void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Register dst, Address src, Register tmp1, Register tmp2) {
bool on_oop = is_reference_type(type);
Expand Down
Loading

0 comments on commit dbd3f9f

Please sign in to comment.