First prototype

src/hotspot/share/ci/ciField.cpp

@@ -106,6 +106,7 @@ ciField::ciField(ciInstanceKlass* klass, int index, Bytecodes::Code bc) :
   _name = (ciSymbol*)ciEnv::current(THREAD)->get_symbol(name);
 
   _is_null_free = false;
+  _null_marker_offset = -1;
 
   // Get the field's declared holder.
   //
@@ -221,7 +222,7 @@ ciField::ciField(fieldDescriptor *fd) :
 // Special copy constructor used to flatten inline type fields by
 // copying the fields of the inline type to a new holder klass.
 ciField::ciField(ciField* field, ciInstanceKlass* holder, int offset, bool is_final) {
-  assert(field->holder()->is_inlinetype(), "should only be used for inline type field flattening");
+  assert(field->holder()->is_inlinetype() || field->holder()->is_abstract(), "should only be used for inline type field flattening");
   // Set the is_final flag
   jint final = is_final ? JVM_ACC_FINAL : ~JVM_ACC_FINAL;
   AccessFlags flags(field->flags().as_int() & final);
@@ -240,6 +241,7 @@ ciField::ciField(ciField* field, ciInstanceKlass* holder, int offset, bool is_fi
   assert(!field->is_flat(), "field must not be flat");
   _is_flat = false;
   _is_null_free = field->_is_null_free;
+  _null_marker_offset = field->_null_marker_offset;
   _original_holder = (field->_original_holder != nullptr) ? field->_original_holder : field->_holder;
 }
 
@@ -287,11 +289,17 @@ void ciField::initialize_from(fieldDescriptor* fd) {
   // Get the flags, offset, and canonical holder of the field.
   _flags = ciFlags(fd->access_flags(), fd->field_flags().is_stable(), fd->field_status().is_initialized_final_update());
   _offset = fd->offset();
-  Klass* field_holder = fd->field_holder();
+  InstanceKlass* field_holder = fd->field_holder();
   assert(field_holder != nullptr, "null field_holder");
   _holder = CURRENT_ENV->get_instance_klass(field_holder);
   _is_flat = fd->is_flat();
   _is_null_free = fd->is_null_free_inline_type();
+  if (fd->has_null_marker()) {
+    InlineLayoutInfo* li = field_holder->inline_layout_info_adr(fd->index());
+    _null_marker_offset = li->null_marker_offset();
+  } else {
+    _null_marker_offset = -1;
+  }
   _original_holder = nullptr;
 
   // Check to see if the field is constant.
@@ -492,6 +500,7 @@ void ciField::print() {
   }
   tty->print(" is_flat=%s", bool_to_str(_is_flat));
   tty->print(" is_null_free=%s", bool_to_str(_is_null_free));
+  tty->print(" null_marker_offset=%s", bool_to_str(_null_marker_offset));
   tty->print(">");
 }
 

src/hotspot/share/ci/ciField.hpp

@@ -52,6 +52,7 @@ class ciField : public ArenaObj {
   bool             _is_constant;
   bool             _is_flat;
   bool             _is_null_free;
+  int              _null_marker_offset;
   ciMethod*        _known_to_link_with_put;
   ciInstanceKlass* _known_to_link_with_get;
   ciConstant       _constant_value;
@@ -175,6 +176,7 @@ class ciField : public ArenaObj {
   bool is_transient            () const { return flags().is_transient(); }
   bool is_flat                 () const { return _is_flat; }
   bool is_null_free            () const { return _is_null_free; }
+  int null_marker_offset       () const { return _null_marker_offset; }
 
   // The field is modified outside of instance initializer methods
   // (or class/initializer methods if the field is static).

src/hotspot/share/ci/ciInstanceKlass.cpp

@@ -536,7 +536,8 @@ GrowableArray<ciField*>* ciInstanceKlass::compute_nonstatic_fields_impl(Growable
   }
 
   // allocate the array:
-  if (flen == 0) {
+  // TODO why do we need this?
+  if (flen == 0 && !is_inlinetype()) {
     return nullptr;  // return nothing if none are locally declared
   }
   if (super_fields != nullptr) {

src/hotspot/share/classfile/classFileParser.hpp

@@ -212,7 +212,6 @@ class ClassFileParser {
   bool _has_contended_fields;
 
   bool _has_inline_type_fields;
-  bool _has_null_marker_offsets;
   bool _is_naturally_atomic;
   bool _must_be_atomic;
   bool _is_implicitly_constructible;

src/hotspot/share/code/nmethod.cpp

@@ -3812,6 +3812,12 @@ void nmethod::print_nmethod_labels(outputStream* stream, address block_begin, bo
       }
       if (!did_name)
         stream->print("%s", type2name(t));
+      if ((*sig)._offset == -1) {
+        // TODO it's also -1 if it's not from a flat field
+        //stream->print(" IS INIT");
+      } else  if ((*sig)._sort_offset != (*sig)._offset) {
+        stream->print(" NULL MARKER");
+      }
     }
     if (at_old_sp) {
       stream->print("  (%s of caller)", spname);

src/hotspot/share/oops/fieldStreams.hpp

@@ -297,6 +297,10 @@ class HierarchicalFieldStream : public StackObj  {
   bool is_flat() const {
     return _current_stream.is_flat();
   }
+
+  bool is_null_free_inline_type() {
+    return _current_stream.is_null_free_inline_type();
+  }
 };
 
 #endif // SHARE_OOPS_FIELDSTREAMS_HPP

src/hotspot/share/oops/inlineKlass.cpp

@@ -294,28 +294,49 @@ Klass* InlineKlass::value_array_klass_or_null() {
 //
 // Value classes could also have fields in abstract super value classes.
 // Use a HierarchicalFieldStream to get them as well.
-int InlineKlass::collect_fields(GrowableArray<SigEntry>* sig, int base_off) {
+int InlineKlass::collect_fields(GrowableArray<SigEntry>* sig, int base_off, int null_marker_offset) {
   int count = 0;
   SigEntry::add_entry(sig, T_METADATA, name(), base_off);
+  int max_offset = 0;
   for (HierarchicalFieldStream<JavaFieldStream> fs(this); !fs.done(); fs.next()) {
     if (fs.access_flags().is_static()) continue;
     int offset = base_off + fs.offset() - (base_off > 0 ? first_field_offset() : 0);
     // TODO 8284443 Use different heuristic to decide what should be scalarized in the calling convention
     if (fs.is_flat()) {
       // Resolve klass of flat field and recursively collect fields
+      int null_marker_offset = -1;
+      if (!fs.is_null_free_inline_type()) {
+        // TODO can we use null_marker_offset() instead?
+        InlineLayoutInfo* li = inline_layout_info_adr(fs.index());
+        null_marker_offset = li->null_marker_offset();
+      }
       Klass* vk = get_inline_type_field_klass(fs.index());
-      count += InlineKlass::cast(vk)->collect_fields(sig, offset);
+      count += InlineKlass::cast(vk)->collect_fields(sig, offset, null_marker_offset);
     } else {
       BasicType bt = Signature::basic_type(fs.signature());
       SigEntry::add_entry(sig, bt, fs.signature(), offset);
       count += type2size[bt];
     }
+    max_offset = MAX2(max_offset, offset);
   }
   int offset = base_off + size_helper()*HeapWordSize - (base_off > 0 ? first_field_offset() : 0);
+  // Null markers are no real fields, add them manually at the end (C2 relies on this) of the flat fields
+  if (null_marker_offset != -1) {
+    tty->print_cr("MARKER %d %d %d", null_marker_offset, offset, max_offset + 1);
+    SigEntry::add_entry(sig, T_BOOLEAN, name(), null_marker_offset, max_offset + 1);
+    count++;
+  }
   SigEntry::add_entry(sig, T_VOID, name(), offset);
   if (base_off == 0) {
     sig->sort(SigEntry::compare);
   }
+  /*
+  tty->print_cr("##\n");
+  print();
+  for (int i = 0; i < sig->length(); ++i) {
+    tty->print_cr("%s %d %d", type2name(sig->at(i)._bt), sig->at(i)._offset, sig->at(i)._sort_offset);
+  }
+  */
   assert(sig->at(0)._bt == T_METADATA && sig->at(sig->length()-1)._bt == T_VOID, "broken structure");
   return count;
 }

src/hotspot/share/oops/inlineKlass.hpp

@@ -178,7 +178,7 @@ class InlineKlass: public InstanceKlass {
   virtual void metaspace_pointers_do(MetaspaceClosure* it);
 
  private:
-  int collect_fields(GrowableArray<SigEntry>* sig, int base_off = 0);
+  int collect_fields(GrowableArray<SigEntry>* sig, int base_off = 0, int null_marker_offset = -1);
 
   void cleanup_blobs();
 

src/hotspot/share/oops/methodData.cpp

@@ -524,7 +524,7 @@ void BranchData::post_initialize(BytecodeStream* stream, MethodData* mdo) {
 void BranchData::print_data_on(outputStream* st, const char* extra) const {
   print_shared(st, "BranchData", extra);
   if (data()->flags()) {
-    tty->cr();
+    st->cr();
     tab(st);
   }
   st->print_cr("taken(%u) displacement(%d)",

src/hotspot/share/opto/callnode.cpp

@@ -524,8 +524,13 @@ void JVMState::format(PhaseRegAlloc *regalloc, const Node *n, outputStream* st)
           fld_node = mcall->in(first_ind+j);
           if (iklass != nullptr) {
             st->print(", [");
-            cifield = iklass->nonstatic_field_at(j);
-            cifield->print_name_on(st);
+            if (j < (uint)iklass->nof_nonstatic_fields()) {
+              cifield = iklass->nonstatic_field_at(j);
+              cifield->print_name_on(st);
+            } else {
+              // Must be a null marker
+              st->print("null marker");
+            }
             format_helper(regalloc, st, fld_node, ":", j, &scobjs);
           } else {
             format_helper(regalloc, st, fld_node, ", [", j, &scobjs);

src/hotspot/share/opto/inlinetypenode.cpp

@@ -257,6 +257,37 @@ bool InlineTypeNode::field_is_null_free(uint index) const {
   return field->is_null_free();
 }
 
+int InlineTypeNode::field_null_marker_offset(uint index) const {
+  assert(index < field_count(), "index out of bounds");
+  ciField* field = inline_klass()->declared_nonstatic_field_at(index);
+  assert(field->is_flat(), "must be an inline type");
+  return field->null_marker_offset();
+}
+
+
+// TODO implement with a worklist
+static uint helper(InlineTypeNode* vt, Unique_Node_List& worklist, Node_List& null_markers, SafePointNode* sfpt) {
+  uint cnt = 0;
+  for (uint i = 0; i < vt->field_count(); ++i) {
+    Node* value = vt->field_value(i);
+    if (vt->field_is_flat(i)) {
+      cnt += helper(value->as_InlineType(), worklist, null_markers, sfpt);
+      if (!vt->field_is_null_free(i)) {
+        null_markers.push(value->as_InlineType()->get_is_init());
+      }
+    } else {
+      if (value->is_InlineType()) {
+        // Add inline type field to the worklist to process later
+        worklist.push(value);
+      }
+      sfpt->add_req(value);
+      cnt++;
+    }
+  }
+  return cnt;
+}
+
+
 void InlineTypeNode::make_scalar_in_safepoint(PhaseIterGVN* igvn, Unique_Node_List& worklist, SafePointNode* sfpt) {
   // We should not scalarize larvals in debug info of their constructor calls because their fields could still be
   // updated. If we scalarize and update the fields in the constructor, the updates won't be visible in the caller after
@@ -271,17 +302,12 @@ void InlineTypeNode::make_scalar_in_safepoint(PhaseIterGVN* igvn, Unique_Node_Li
   }
 
   ciInlineKlass* vk = inline_klass();
-  uint nfields = vk->nof_nonstatic_fields();
+  vk->nof_nonstatic_fields();
   JVMState* jvms = sfpt->jvms();
   // Replace safepoint edge by SafePointScalarObjectNode and add field values
   assert(jvms != nullptr, "missing JVMS");
   uint first_ind = (sfpt->req() - jvms->scloff());
-  SafePointScalarObjectNode* sobj = new SafePointScalarObjectNode(type()->isa_instptr(),
-                                                                  nullptr,
-                                                                  first_ind,
-                                                                  sfpt->jvms()->depth(),
-                                                                  nfields);
-  sobj->init_req(0, igvn->C->root());
+
   // Nullable inline types have an IsInit field that needs
   // to be checked before using the field values.
   const TypeInt* tinit = igvn->type(get_is_init())->isa_int();
@@ -292,16 +318,21 @@ void InlineTypeNode::make_scalar_in_safepoint(PhaseIterGVN* igvn, Unique_Node_Li
   }
   // Iterate over the inline type fields in order of increasing
   // offset and add the field values to the safepoint.
-  for (uint j = 0; j < nfields; ++j) {
-    int offset = vk->nonstatic_field_at(j)->offset_in_bytes();
-    Node* value = field_value_by_offset(offset, true /* include flat inline type fields */);
-    if (value->is_InlineType()) {
-      // Add inline type field to the worklist to process later
-      worklist.push(value);
-    }
-    sfpt->add_req(value);
+  Node_List null_markers;
+  uint nfields = helper(this, worklist, null_markers, sfpt);
+
+  for (uint i = 0; i < null_markers.size(); ++i) {
+    Node* is_init = null_markers.at(i);
+    sfpt->add_req(is_init);
+    nfields++;
   }
   jvms->set_endoff(sfpt->req());
+  SafePointScalarObjectNode* sobj = new SafePointScalarObjectNode(type()->isa_instptr(),
+                                                                  nullptr,
+                                                                  first_ind,
+                                                                  sfpt->jvms()->depth(),
+                                                                  nfields);
+  sobj->init_req(0, igvn->C->root());
   sobj = igvn->transform(sobj)->as_SafePointScalarObject();
   igvn->rehash_node_delayed(sfpt);
   for (uint i = jvms->debug_start(); i < jvms->debug_end(); i++) {
@@ -455,7 +486,8 @@ void InlineTypeNode::load(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass*
       value = make_default_impl(kit->gvn(), ft->as_inline_klass(), visited);
     } else if (field_is_flat(i)) {
       // Recursively load the flat inline type field
-      value = make_from_flat_impl(kit, ft->as_inline_klass(), base, ptr, holder, offset, decorators, visited);
+      assert(field_is_null_free(i) == (field_null_marker_offset(i) == -1), "inconsistency");
+      value = make_from_flat_impl(kit, ft->as_inline_klass(), base, ptr, holder, offset, field_null_marker_offset(i), decorators, visited);
     } else {
       const TypeOopPtr* oop_ptr = kit->gvn().type(base)->isa_oopptr();
       bool is_array = (oop_ptr->isa_aryptr() != nullptr);
@@ -498,7 +530,7 @@ void InlineTypeNode::load(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass*
   }
 }
 
-void InlineTypeNode::store_flat(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset, DecoratorSet decorators) const {
+void InlineTypeNode::store_flat(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset, int null_marker_offset, DecoratorSet decorators) const {
   if (kit->gvn().type(base)->isa_aryptr()) {
     kit->C->set_flat_accesses();
   }
@@ -507,6 +539,13 @@ void InlineTypeNode::store_flat(GraphKit* kit, Node* base, Node* ptr, ciInstance
   if (holder == nullptr) {
     holder = inline_klass();
   }
+  if (null_marker_offset != -1) {
+    // Nullable flat field, read the null marker
+    Node* adr = kit->basic_plus_adr(base, null_marker_offset);
+    const TypePtr* adr_type = kit->gvn().type(adr)->isa_ptr();
+    int alias_idx = kit->C->get_alias_index(adr_type);
+    kit->store_to_memory(kit->control(), adr, get_is_init(), T_BOOLEAN, alias_idx, MemNode::unordered);
+  }
   holder_offset -= inline_klass()->first_field_offset();
   store(kit, base, ptr, holder, holder_offset, -1, decorators);
 }
@@ -520,7 +559,8 @@ void InlineTypeNode::store(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass
     ciType* ft = field_type(i);
     if (field_is_flat(i)) {
       // Recursively store the flat inline type field
-      value->as_InlineType()->store_flat(kit, base, ptr, holder, offset, decorators);
+      assert(field_is_null_free(i) == (field_null_marker_offset(i) == -1), "inconsistency");
+      value->as_InlineType()->store_flat(kit, base, ptr, holder, offset, field_null_marker_offset(i), decorators);
     } else {
       // Store field value to memory
       const TypePtr* adr_type = field_adr_type(base, offset, holder, decorators, kit->gvn());
@@ -946,23 +986,34 @@ InlineTypeNode* InlineTypeNode::make_from_oop_impl(GraphKit* kit, Node* oop, ciI
   return gvn.transform(vt)->as_InlineType();
 }
 
-InlineTypeNode* InlineTypeNode::make_from_flat(GraphKit* kit, ciInlineKlass* vk, Node* obj, Node* ptr, ciInstanceKlass* holder, int holder_offset, DecoratorSet decorators) {
+InlineTypeNode* InlineTypeNode::make_from_flat(GraphKit* kit, ciInlineKlass* vk, Node* obj, Node* ptr, ciInstanceKlass* holder, int holder_offset, int null_marker_offset, DecoratorSet decorators) {
   GrowableArray<ciType*> visited;
   visited.push(vk);
-  return make_from_flat_impl(kit, vk, obj, ptr, holder, holder_offset, decorators, visited);
+  return make_from_flat_impl(kit, vk, obj, ptr, holder, holder_offset, null_marker_offset, decorators, visited);
 }
 
 // GraphKit wrapper for the 'make_from_flat' method
-InlineTypeNode* InlineTypeNode::make_from_flat_impl(GraphKit* kit, ciInlineKlass* vk, Node* obj, Node* ptr, ciInstanceKlass* holder, int holder_offset, DecoratorSet decorators, GrowableArray<ciType*>& visited) {
+InlineTypeNode* InlineTypeNode::make_from_flat_impl(GraphKit* kit, ciInlineKlass* vk, Node* obj, Node* ptr, ciInstanceKlass* holder, int holder_offset, int null_marker_offset, DecoratorSet decorators, GrowableArray<ciType*>& visited) {
   if (kit->gvn().type(obj)->isa_aryptr()) {
     kit->C->set_flat_accesses();
   }
+
+  bool null_free = (null_marker_offset == -1);
+
   // Create and initialize an InlineTypeNode by loading all field values from
   // a flat inline type field at 'holder_offset' or from an inline type array.
-  InlineTypeNode* vt = make_uninitialized(kit->gvn(), vk);
+  InlineTypeNode* vt = make_uninitialized(kit->gvn(), vk, null_free);
   // The inline type is flattened into the object without an oop header. Subtract the
   // offset of the first field to account for the missing header when loading the values.
   holder_offset -= vk->first_field_offset();
+
+  if (null_marker_offset != -1) {
+    // Nullable flat field, read the null marker
+    Node* adr = kit->basic_plus_adr(obj, null_marker_offset);
+    Node* is_init = kit->make_load(nullptr, adr, TypeInt::BOOL, T_BOOLEAN, MemNode::unordered);
+    vt->set_req(IsInit, is_init);
+  }
+
   vt->load(kit, obj, ptr, holder, visited, holder_offset, decorators);
   assert(vt->is_loaded(&kit->gvn()) != obj, "holder oop should not be used as flattened inline type oop");
   return kit->gvn().transform(vt)->as_InlineType();
@@ -1121,6 +1172,10 @@ void InlineTypeNode::pass_fields(GraphKit* kit, Node* n, uint& base_input, bool
     if (field_is_flat(i)) {
       // Flat inline type field
       arg->as_InlineType()->pass_fields(kit, n, base_input, in);
+      if (!field_is_null_free(i)) {
+        assert(field_null_marker_offset(i) != -1, "inconsistency");
+        n->init_req(base_input++, arg->as_InlineType()->get_is_init());
+      }
     } else {
       if (arg->is_InlineType()) {
         // Non-flat inline type field
@@ -1179,8 +1234,21 @@ void InlineTypeNode::initialize_fields(GraphKit* kit, MultiNode* multi, uint& ba
     Node* parm = nullptr;
     if (field_is_flat(i)) {
       // Flat inline type field
-      InlineTypeNode* vt = make_uninitialized(gvn, type->as_inline_klass());
+      InlineTypeNode* vt = make_uninitialized(gvn, type->as_inline_klass(), field_is_null_free(i));
       vt->initialize_fields(kit, multi, base_input, in, true, null_check_region, visited);
+      if (!field_is_null_free(i)) {
+        assert(in, "FIXME");
+        assert(field_null_marker_offset(i) != -1, "inconsistency");
+        Node* is_init = nullptr;
+        if (multi->is_Start()) {
+          is_init = gvn.transform(new ParmNode(multi->as_Start(), base_input));
+        } else {
+          // TODO add a test for this
+          is_init = multi->as_Call()->in(base_input);
+        }
+        vt->set_req(IsInit, is_init);
+        base_input++;
+      }
       parm = gvn.transform(vt);
     } else {
       if (multi->is_Start()) {