RISC-V: Implementation: JEP 450: Compact Object Headers (Experimental)

src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp

@@ -33,6 +33,7 @@
 #include "c1/c1_Runtime1.hpp"
 #include "classfile/javaClasses.hpp"
 #include "nativeInst_riscv.hpp"
+#include "runtime/objectMonitor.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "vmreg_riscv.inline.hpp"
 
@@ -226,8 +227,11 @@ void MonitorExitStub::emit_code(LIR_Assembler* ce) {
 }
 
 void LoadKlassStub::emit_code(LIR_Assembler* ce) {
-  // Currently not needed.
-  Unimplemented();
+  assert(UseCompactObjectHeaders, "Only use with compact object headers");
+  __ bind(_entry);
+  Register d = _result->as_register();
+  __ ld(d, Address(d, OM_OFFSET_NO_MONITOR_VALUE_TAG(header)));
+  __ j(_continuation);
 }
 
 // Implementation of patching:

src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp

@@ -194,7 +194,10 @@ void LIR_Assembler::arraycopy_type_check(Register src, Register src_pos, Registe
   // We don't know the array types are compatible
   if (basic_type != T_OBJECT) {
     // Simple test for basic type arrays
-    if (UseCompressedClassPointers) {
+    if (UseCompactObjectHeaders) {
+      __ load_nklass_compact(tmp, src, t1 /* tmp */);
+      __ load_nklass_compact(t0, dst, t1 /* tmp */);
+    } else if (UseCompressedClassPointers) {
       __ lwu(tmp, Address(src, oopDesc::klass_offset_in_bytes()));
       __ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
     } else {
@@ -254,32 +257,17 @@ void LIR_Assembler::arraycopy_assert(Register src, Register dst, Register tmp, c
     // dst type is exactly the expected type and the src type is a
     // subtype which we can't check or src is the same array as dst
     // but not necessarily exactly of type default_type.
-    Label known_ok, halt;
+    Label known_ok, cont, halt;
     __ mov_metadata(tmp, default_type->constant_encoding());
-    if (UseCompressedClassPointers) {
-      __ encode_klass_not_null(tmp);
-    }
 
     if (basic_type != T_OBJECT) {
-      if (UseCompressedClassPointers) {
-        __ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
-      } else {
-        __ ld(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
-      }
-      __ bne(tmp, t0, halt);
-      if (UseCompressedClassPointers) {
-        __ lwu(t0, Address(src, oopDesc::klass_offset_in_bytes()));
-      } else {
-        __ ld(t0, Address(src, oopDesc::klass_offset_in_bytes()));
-      }
-      __ beq(tmp, t0, known_ok);
+      __ cmp_klass(dst, tmp, t0, t1, cont);
+      __ j(halt);
+
+      __ bind(cont);
+      __ cmp_klass(src, tmp, t0, t1, known_ok);
     } else {
-      if (UseCompressedClassPointers) {
-        __ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
-      } else {
-        __ ld(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
-      }
-      __ beq(tmp, t0, known_ok);
+      __ cmp_klass(dst, tmp, t0, t1, known_ok);
       __ beq(src, dst, known_ok);
     }
     __ bind(halt);

src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp

@@ -1524,7 +1524,18 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
   }
 
   if (UseCompressedClassPointers) {
-    __ lwu(result, Address(obj, oopDesc::klass_offset_in_bytes()));
+    if (UseCompactObjectHeaders) {
+      // Check if we can take the (common) fast path, if obj is unlocked
+      __ ld(result, Address(obj, oopDesc::mark_offset_in_bytes()));
+      __ test_bit(t0, result, exact_log2(markWord::monitor_value));
+      __ bnez(t0, *op->stub()->entry(), /* is_far */ true);
+      __ bind(*op->stub()->continuation());
+
+      // Shift to get proper narrow Klass*.
+      __ srli(result, result, markWord::klass_shift);
+    } else {
+      __ lwu(result, Address(obj, oopDesc::klass_offset_in_bytes()));
+    }
     __ decode_klass_not_null(result);
   } else {
     __ ld(result, Address(obj, oopDesc::klass_offset_in_bytes()));

src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp

@@ -172,16 +172,30 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
   mv(tmp1, (int32_t)(intptr_t)markWord::prototype().value());
   sd(tmp1, Address(obj, oopDesc::mark_offset_in_bytes()));
 
-  if (UseCompressedClassPointers) { // Take care not to kill klass
-    encode_klass_not_null(tmp1, klass, tmp2);
-    sw(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
+  if (UseCompactObjectHeaders) {
+    ld(tmp1, Address(klass, Klass::prototype_header_offset()));
+    sd(tmp1, Address(obj, oopDesc::mark_offset_in_bytes()));
   } else {
-    sd(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+    // This assumes that all prototype bits fit in an int32_t
+    mv(tmp1, (int32_t)(intptr_t)markWord::prototype().value());
+    sd(tmp1, Address(obj, oopDesc::mark_offset_in_bytes()));
+
+    if (UseCompressedClassPointers) { // Take care not to kill klass
+      encode_klass_not_null(tmp1, klass, tmp2);
+      sw(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
+    } else {
+      sd(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+    }
   }
 
   if (len->is_valid()) {
     sw(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
-  } else if (UseCompressedClassPointers) {
+    if (UseCompactObjectHeaders) {
+      // With compact headers, arrays have a 32bit alignment gap after the length.
+      assert(arrayOopDesc::length_offset_in_bytes() == 8, "check length offset");
+      sw(zr, Address(obj, arrayOopDesc::length_offset_in_bytes() + sizeof(jint)));
+    }
+  } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
     store_klass_gap(obj, zr);
   }
 }

src/hotspot/cpu/riscv/c2_CodeStubs_riscv.cpp

@@ -99,4 +99,15 @@ void C2HandleAnonOMOwnerStub::emit(C2_MacroAssembler& masm) {
   __ j(continuation());
 }
 
+int C2LoadNKlassStub::max_size() const {
+  return 8;
+}
+
+void C2LoadNKlassStub::emit(C2_MacroAssembler& masm) {
+  __ bind(entry());
+  Register d = dst();
+  __ ld(d, Address(d, OM_OFFSET_NO_MONITOR_VALUE_TAG(header)));
+  __ j(continuation());
+}
+
 #undef __

src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp

@@ -1958,6 +1958,26 @@ void C2_MacroAssembler::expand_bits_l_v(Register dst, Register src, Register mas
   expand_bits_v(dst, src, mask, /* is_long */ true);
 }
 
+void C2_MacroAssembler::load_nklass_compact(Register dst, Register obj, int disp) {
+  C2LoadNKlassStub* stub = new (Compile::current()->comp_arena()) C2LoadNKlassStub(dst);
+  Compile::current()->output()->add_stub(stub);
+
+  // Note: Don't clobber obj anywhere in that method!
+
+  // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
+  // obj-start, so that we can load from the object's mark-word instead. Usually the address
+  // comes as obj-start in obj and klass_offset_in_bytes in disp. However, sometimes C2
+  // emits code that pre-computes obj-start + klass_offset_in_bytes into a register, and
+  // then passes that register as obj and 0 in disp. The following code extracts the base
+  // and offset to load the mark-word.
+  int offset = oopDesc::mark_offset_in_bytes() + disp - oopDesc::klass_offset_in_bytes();
+  ld(dst, Address(obj, offset));
+  test_bit(t0, dst, exact_log2(markWord::monitor_value));
+  bnez(t0, stub->entry(), /* is_far */ true);
+  bind(stub->continuation());
+  srli(dst, dst, markWord::klass_shift);
+}
+
 void C2_MacroAssembler::element_compare(Register a1, Register a2, Register result, Register cnt, Register tmp1, Register tmp2,
                                         VectorRegister vr1, VectorRegister vr2, VectorRegister vrs, bool islatin, Label &DONE) {
   Label loop;

src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp

@@ -176,6 +176,7 @@
 
   void signum_fp_v(VectorRegister dst, VectorRegister one, BasicType bt, int vlen);
 
+  void load_nklass_compact(Register dst, Register src, int disp);
 
   // intrinsic methods implemented by rvv instructions
 

src/hotspot/cpu/riscv/macroAssembler_riscv.cpp

@@ -2055,10 +2055,15 @@ void MacroAssembler::orptr(Address adr, RegisterOrConstant src, Register tmp1, R
   sd(tmp1, adr);
 }
 
+// Compare object klass with 'trial_klass' and if equal branch to label 'L'. Fall through otherwise.
 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp1, Register tmp2, Label &L) {
   assert_different_registers(oop, trial_klass, tmp1, tmp2);
   if (UseCompressedClassPointers) {
-    lwu(tmp1, Address(oop, oopDesc::klass_offset_in_bytes()));
+    if (UseCompactObjectHeaders) {
+      load_nklass_compact(tmp1, oop, tmp2);
+    } else {
+      lwu(tmp1, Address(oop, oopDesc::klass_offset_in_bytes()));
+    }
     if (CompressedKlassPointers::base() == nullptr) {
       slli(tmp1, tmp1, CompressedKlassPointers::shift());
       beq(trial_klass, tmp1, L);
@@ -2249,10 +2254,35 @@ void MacroAssembler::encode_heap_oop(Register d, Register s) {
   }
 }
 
+// Loads the obj's Klass* into dst.
+// Preserves src.
+void MacroAssembler::load_nklass_compact(Register dst, Register src, Register tmp) {
+  assert(UseCompressedClassPointers, "expects UseCompressedClassPointers");
+  assert_different_registers(dst, tmp);
+  assert_different_registers(src, tmp);
+
+  Label fast;
+
+  // Check if we can take the (common) fast path, if obj is unlocked.
+  ld(dst, Address(src, oopDesc::mark_offset_in_bytes()));
+  test_bit(tmp, dst, exact_log2(markWord::monitor_value));
+  beqz(tmp, fast);
+
+  // Fetch displaced header
+  ld(dst, Address(dst, OM_OFFSET_NO_MONITOR_VALUE_TAG(header)));
+
+  // Fast-path: shift and decode Klass*.
+  bind(fast);
+  srli(dst, dst, markWord::klass_shift);
+}
+
 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
   assert_different_registers(dst, tmp);
   assert_different_registers(src, tmp);
-  if (UseCompressedClassPointers) {
+  if (UseCompactObjectHeaders) {
+    load_nklass_compact(dst, src, tmp);
+    decode_klass_not_null(dst, tmp);
+  } else if (UseCompressedClassPointers) {
     lwu(dst, Address(src, oopDesc::klass_offset_in_bytes()));
     decode_klass_not_null(dst, tmp);
   } else {
@@ -2263,6 +2293,7 @@ void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
   // FIXME: Should this be a store release? concurrent gcs assumes
   // klass length is valid if klass field is not null.
+  assert(!UseCompactObjectHeaders, "not with compact headers");
   if (UseCompressedClassPointers) {
     encode_klass_not_null(src, tmp);
     sw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
@@ -2272,6 +2303,7 @@ void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
 }
 
 void MacroAssembler::store_klass_gap(Register dst, Register src) {
+  assert(!UseCompactObjectHeaders, "not with compact headers");
   if (UseCompressedClassPointers) {
     // Store to klass gap in destination
     sw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));

src/hotspot/cpu/riscv/macroAssembler_riscv.hpp

@@ -194,6 +194,7 @@ class MacroAssembler: public Assembler {
                       Address src, Register tmp1, Register tmp2);
   void access_store_at(BasicType type, DecoratorSet decorators, Address dst,
                        Register val, Register tmp1, Register tmp2, Register tmp3);
+  void load_nklass_compact(Register dst, Register src, Register tmp = t0);
   void load_klass(Register dst, Register src, Register tmp = t0);
   void store_klass(Register dst, Register src, Register tmp = t0);
   void cmp_klass(Register oop, Register trial_klass, Register tmp1, Register tmp2, Label &L);

src/hotspot/cpu/riscv/riscv.ad

@@ -4801,6 +4801,7 @@ instruct loadKlass(iRegPNoSp dst, memory mem)
 // Load Narrow Klass Pointer
 instruct loadNKlass(iRegNNoSp dst, memory mem)
 %{
+  predicate(!UseCompactObjectHeaders);
   match(Set dst (LoadNKlass mem));
 
   ins_cost(LOAD_COST);
@@ -4813,6 +4814,23 @@ instruct loadNKlass(iRegNNoSp dst, memory mem)
   ins_pipe(iload_reg_mem);
 %}
 
+instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory mem)
+%{
+  predicate(UseCompactObjectHeaders);
+  match(Set dst (LoadNKlass mem));
+  effect(TEMP_DEF dst);
+
+  ins_cost(LOAD_COST + ALU_COST * 3 + BRANCH_COST);
+  format %{ "lwu  $dst, $mem\t# compressed class ptr, #@loadNKlassCompactHeaders" %}
+
+  ins_encode %{
+    assert($mem$$index$$Register == noreg, "expect no index");
+    __ load_nklass_compact($dst$$Register, $mem$$base$$Register, $$mem$$disp);
+  %}
+
+  ins_pipe(pipe_slow);
+%}
+
 // Load Float
 instruct loadF(fRegF dst, memory mem)
 %{

src/hotspot/cpu/riscv/templateTable_riscv.cpp

@@ -3576,12 +3576,22 @@ void TemplateTable::_new() {
 
     // The object is initialized before the header. If the object size is
     // zero, go directly to the header initialization.
-    __ sub(x13, x13, sizeof(oopDesc));
+    if (UseCompactObjectHeaders) {
+      assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong, "oop base offset must be 8-byte-aligned"));
+      __ sub(x13, x13, oopDesc::base_offset_in_bytes());
+    } else {
+      __ sub(x13, x13, sizeof(oopDesc));
+    }
     __ beqz(x13, initialize_header);
 
     // Initialize object fields
     {
-      __ add(x12, x10, sizeof(oopDesc));
+      if (UseCompactObjectHeaders) {
+        assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong, "oop base offset must be 8-byte-aligned"));
+        __ add(x12, x10, oopDesc::base_offset_in_bytes());
+      } else {
+        __ add(x12, x10, sizeof(oopDesc));
+      }
       Label loop;
       __ bind(loop);
       __ sd(zr, Address(x12));
@@ -3590,12 +3600,17 @@ void TemplateTable::_new() {
       __ bnez(x13, loop);
     }
 
-    // initialize object hader only.
+    // initialize object header only.
     __ bind(initialize_header);
-    __ mv(t0, (intptr_t)markWord::prototype().value());
-    __ sd(t0, Address(x10, oopDesc::mark_offset_in_bytes()));
-    __ store_klass_gap(x10, zr);   // zero klass gap for compressed oops
-    __ store_klass(x10, x14);      // store klass last
+    if (UseCompactObjectHeaders) {
+      __ ld(t0, Address(x14, Klass::prototype_header_offset()));
+      __ sd(t0, Address(x10, oopDesc::mark_offset_in_bytes()));
+    } else {
+      __ mv(t0, (intptr_t)markWord::prototype().value());
+      __ sd(t0, Address(x10, oopDesc::mark_offset_in_bytes()));
+      __ store_klass_gap(x10, zr);  // zero klass gap for compressed oops
+      __ store_klass(x10, x14);     // store klass last
+    }
 
     {
       SkipIfEqual skip(_masm, &DTraceAllocProbes, false);