Fix array gap

src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp

@@ -1261,7 +1261,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
                       len,
                       tmp1,
                       tmp2,
-                      arrayOopDesc::header_size(op->type()),
+                      UseCompactObjectHeaders ? arrayOopDesc::base_offset_in_bytes(op->type()) : arrayOopDesc::header_size(op->type()),
                       array_element_size(op->type()),
                       op->klass()->as_register(),
                       *op->stub()->entry());

src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp

@@ -318,19 +318,32 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
   cmp(len, rscratch1);
   br(Assembler::HS, slow_case);
 
+  // header_size is already base offset with UseCompactObjectHeaders
+  int base_offset = UseCompactObjectHeaders ? header_size : (header_size * BytesPerWord);
+
   const Register arr_size = t2; // okay to be the same
   // align object end
-  mov(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask);
+  mov(arr_size, (int32_t)base_offset + MinObjAlignmentInBytesMask);
   add(arr_size, arr_size, len, ext::uxtw, f);
   andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
 
   try_allocate(obj, arr_size, 0, t1, t2, slow_case);
 
   initialize_header(obj, klass, len, t1, t2);
 
+  assert(is_aligned(base_offset, BytesPerWord) || UseCompactObjectHeaders, "must be aligned or with UseCompactObjectHeaders");
+  if (UseCompactObjectHeaders && !is_aligned(base_offset, BytesPerWord)) {
+    // Clear leading 4 bytes, if necessary.
+    // TODO: This could perhaps go into initialize_body() and also clear the leading 4 bytes
+    // for non-array objects, thereby replacing the klass-gap clearing code in initialize_header().
+    assert(is_aligned(base_offset, BytesPerInt), "must be 4-byte aligned");
+    strw(zr, Address(obj, base_offset));
+    base_offset += BytesPerInt;
+  }
+
   // clear rest of allocated space
   const Register len_zero = len;
-  initialize_body(obj, arr_size, header_size * BytesPerWord, len_zero);
+  initialize_body(obj, arr_size, base_offset, len_zero);
 
   membar(StoreStore);
 

src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp

@@ -1598,7 +1598,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
                       len,
                       tmp1,
                       tmp2,
-                      arrayOopDesc::header_size(op->type()),
+                      UseCompactObjectHeaders ? arrayOopDesc::base_offset_in_bytes(op->type()) : arrayOopDesc::header_size(op->type()),
                       array_element_size(op->type()),
                       op->klass()->as_register(),
                       *op->stub()->entry());

src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp

@@ -309,22 +309,33 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
   cmpptr(len, (int32_t)max_array_allocation_length);
   jcc(Assembler::above, slow_case);
 
+  // header_size is already base offset with UseCompactObjectHeaders
+  int base_offset = UseCompactObjectHeaders ? header_size : (header_size * BytesPerWord);
+
   const Register arr_size = t2; // okay to be the same
   // align object end
-  movptr(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask);
+  movptr(arr_size, (int32_t)base_offset + MinObjAlignmentInBytesMask);
   lea(arr_size, Address(arr_size, len, f));
   andptr(arr_size, ~MinObjAlignmentInBytesMask);
 
   try_allocate(obj, arr_size, 0, t1, t2, slow_case);
 
   initialize_header(obj, klass, len, t1, t2);
 
+  assert(is_aligned(base_offset, BytesPerWord) || UseCompactObjectHeaders, "must be aligned or with UseCompactObjectHeaders");
+  if (UseCompactObjectHeaders && !is_aligned(base_offset, BytesPerWord)) {
+    // Clear leading 4 bytes, if necessary.
+    // TODO: This could perhaps go into initialize_body() and also clear the leading 4 bytes
+    // for non-array objects, thereby replacing the klass-gap clearing code in initialize_header().
+    assert(is_aligned(base_offset, BytesPerInt), "must be 4-byte aligned");
+    movl(Address(obj, base_offset), 0);
+    base_offset += BytesPerInt;
+  }
+  assert(is_aligned(base_offset, BytesPerWord), "must be word aligned");
+
   // clear rest of allocated space
   const Register len_zero = len;
-  if (UseCompactObjectHeaders) {
-    assert(header_size == 2, "check array header size");
-  }
-  initialize_body(obj, arr_size, header_size * BytesPerWord, len_zero);
+  initialize_body(obj, arr_size, base_offset, len_zero);
 
   if (CURRENT_ENV->dtrace_alloc_probes()) {
     assert(obj == rax, "must be");

src/hotspot/share/gc/parallel/mutableNUMASpace.cpp

@@ -104,7 +104,8 @@ void MutableNUMASpace::ensure_parsability() {
             size_t touched_words = words_to_fill;
 #ifndef ASSERT
             if (!ZapUnusedHeapArea) {
-              touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
+              touched_words = MIN2((size_t)align_object_size(UseCompactObjectHeaders ?
+                arrayOopDesc::int_array_header_size() : typeArrayOopDesc::header_size(T_INT)),
                 touched_words);
             }
 #endif

src/hotspot/share/gc/parallel/psPromotionLAB.cpp

@@ -46,7 +46,7 @@ void PSPromotionLAB::initialize(MemRegion lab) {
 
   // Initialize after VM starts up because header_size depends on compressed
   // oops.
-  filler_header_size = align_object_size(typeArrayOopDesc::header_size(T_INT));
+  filler_header_size = align_object_size(UseCompactObjectHeaders ? arrayOopDesc::int_array_header_size() : typeArrayOopDesc::header_size(T_INT));
 
   // We can be initialized to a zero size!
   if (free() > 0) {
@@ -89,13 +89,13 @@ void PSPromotionLAB::flush() {
     filler_oop->set_klass(Universe::intArrayKlassObj());
   }
   const size_t array_length =
-    pointer_delta(tlab_end, top()) - typeArrayOopDesc::header_size(T_INT);
+    pointer_delta(tlab_end, top()) - (UseCompactObjectHeaders ? arrayOopDesc::int_array_header_size() : typeArrayOopDesc::header_size(T_INT));
   assert( (array_length * (HeapWordSize/sizeof(jint))) < (size_t)max_jint, "array too big in PSPromotionLAB");
   filler_oop->set_length((int)(array_length * (HeapWordSize/sizeof(jint))));
 
 #ifdef ASSERT
   // Note that we actually DO NOT want to use the aligned header size!
-  HeapWord* elt_words = ((HeapWord*)filler_oop) + typeArrayOopDesc::header_size(T_INT);
+  HeapWord* elt_words = ((HeapWord*)filler_oop) + (UseCompactObjectHeaders ? arrayOopDesc::int_array_header_size() : typeArrayOopDesc::header_size(T_INT));
   Copy::fill_to_words(elt_words, array_length, 0xDEAABABE);
 #endif
 

src/hotspot/share/gc/shared/collectedHeap.cpp

@@ -355,14 +355,14 @@ size_t CollectedHeap::max_tlab_size() const {
   // We actually lose a little by dividing first,
   // but that just makes the TLAB  somewhat smaller than the biggest array,
   // which is fine, since we'll be able to fill that.
-  size_t max_int_size = typeArrayOopDesc::header_size(T_INT) +
+  size_t max_int_size = (UseCompactObjectHeaders ? arrayOopDesc::int_array_header_size() : typeArrayOopDesc::header_size(T_INT)) +
               sizeof(jint) *
               ((juint) max_jint / (size_t) HeapWordSize);
   return align_down(max_int_size, MinObjAlignment);
 }
 
 size_t CollectedHeap::filler_array_hdr_size() {
-  return align_object_offset(arrayOopDesc::header_size(T_INT)); // align to Long
+  return align_object_offset(UseCompactObjectHeaders ? arrayOopDesc::int_array_header_size() : arrayOopDesc::header_size(T_INT)); // align to Long
 }
 
 size_t CollectedHeap::filler_array_min_size() {

src/hotspot/share/gc/shared/memAllocator.cpp

@@ -433,7 +433,9 @@ MemRegion ObjArrayAllocator::obj_memory_range(oop obj) const {
     return MemAllocator::obj_memory_range(obj);
   }
   ArrayKlass* array_klass = ArrayKlass::cast(_klass);
-  const size_t hs = arrayOopDesc::header_size(array_klass->element_type());
+  const size_t hs = UseCompactObjectHeaders ?
+      align_up(arrayOopDesc::base_offset_in_bytes(array_klass->element_type()), HeapWordSize) / HeapWordSize :
+      arrayOopDesc::header_size(array_klass->element_type());
   return MemRegion(((HeapWord*)obj) + hs, _word_size - hs);
 }
 

src/hotspot/share/gc/shared/plab.cpp

@@ -44,7 +44,9 @@ PLAB::PLAB(size_t desired_plab_sz_) :
   _end(NULL), _hard_end(NULL), _allocated(0), _wasted(0), _undo_wasted(0)
 {
   // ArrayOopDesc::header_size depends on command line initialization.
-  AlignmentReserve = oopDesc::header_size() > MinObjAlignment ? align_object_size(arrayOopDesc::header_size(T_INT)) : 0;
+  AlignmentReserve = oopDesc::header_size() > MinObjAlignment ?  align_object_size(
+      UseCompactObjectHeaders ? arrayOopDesc::int_array_header_size() : arrayOopDesc::header_size(T_INT)
+    ) : 0;
   assert(min_size() > AlignmentReserve,
          "Minimum PLAB size " SIZE_FORMAT " must be larger than alignment reserve " SIZE_FORMAT " "
          "to be able to contain objects", min_size(), AlignmentReserve);

src/hotspot/share/gc/shared/space.cpp

@@ -645,7 +645,7 @@ void ContiguousSpace::allocate_temporary_filler(int factor) {
   }
   size = align_object_size(size);
 
-  const size_t array_header_size = typeArrayOopDesc::header_size(T_INT);
+  const size_t array_header_size = UseCompactObjectHeaders ? arrayOopDesc::int_array_header_size() : typeArrayOopDesc::header_size(T_INT);
   if (size >= align_object_size(array_header_size)) {
     size_t length = (size - array_header_size) * (HeapWordSize / sizeof(jint));
     // allocate uninitialized int array

src/hotspot/share/gc/shared/threadLocalAllocBuffer.hpp

@@ -143,7 +143,7 @@ class ThreadLocalAllocBuffer: public CHeapObj<mtThread> {
 
   // Reserve space at the end of TLAB
   static size_t end_reserve() {
-    int reserve_size = typeArrayOopDesc::header_size(T_INT);
+    int reserve_size = UseCompactObjectHeaders ? arrayOopDesc::int_array_header_size() : typeArrayOopDesc::header_size(T_INT);
     return MAX2(reserve_size, _reserve_for_allocation_prefetch);
   }
   static size_t alignment_reserve()              { return align_object_size(end_reserve()); }

src/hotspot/share/oops/arrayOop.hpp

@@ -51,8 +51,10 @@ class arrayOopDesc : public oopDesc {
   // Returns the aligned header_size_in_bytes.  This is not equivalent to
   // sizeof(arrayOopDesc) which should not appear in the code.
   static int header_size_in_bytes() {
-    size_t hs = align_up(length_offset_in_bytes() + sizeof(int),
-                              HeapWordSize);
+    size_t hs = UseCompactObjectHeaders?
+                length_offset_in_bytes() + sizeof(int) :
+                align_up(length_offset_in_bytes() + sizeof(int),
+                         HeapWordSize);
 #ifdef ASSERT
     // make sure it isn't called before UseCompressedOops is initialized.
     static size_t arrayoopdesc_hs = 0;
@@ -66,6 +68,13 @@ class arrayOopDesc : public oopDesc {
   // aligned 0 mod 8.  The typeArrayOop itself must be aligned at least this
   // strongly.
   static bool element_type_should_be_aligned(BasicType type) {
+#ifdef _LP64
+    if (UseCompactObjectHeaders) {
+      if (type == T_OBJECT || type == T_ARRAY) {
+        return !UseCompressedOops;
+      }
+    }
+#endif
     return type == T_DOUBLE || type == T_LONG;
   }
 
@@ -81,7 +90,12 @@ class arrayOopDesc : public oopDesc {
 
   // Returns the offset of the first element.
   static int base_offset_in_bytes(BasicType type) {
-    return header_size(type) * HeapWordSize;
+    if (UseCompactObjectHeaders) {
+      size_t hs = header_size_in_bytes();
+      return (int)(element_type_should_be_aligned(type) ? align_up(hs, BytesPerLong) : hs);
+    } else {
+      return header_size(type) * HeapWordSize;
+    }
   }
 
   // Returns the address of the first element. The elements in the array will not
@@ -121,12 +135,19 @@ class arrayOopDesc : public oopDesc {
   // Returns the header size in words aligned to the requirements of the
   // array object type.
   static int header_size(BasicType type) {
+    assert(!UseCompactObjectHeaders, "not used with UseCompactObjectHeaders");
     size_t typesize_in_bytes = header_size_in_bytes();
     return (int)(element_type_should_be_aligned(type)
       ? align_object_offset(typesize_in_bytes/HeapWordSize)
       : typesize_in_bytes/HeapWordSize);
   }
 
+  // Used for heap filling with UseCompactObjectHeaders
+  static int int_array_header_size() {
+    assert(UseCompactObjectHeaders, "used with UseCompactObjectHeaders");
+    return align_up(header_size_in_bytes(), HeapWordSize) / HeapWordSize;
+  }
+
   // Return the maximum length of an array of BasicType.  The length can passed
   // to typeArrayOop::object_size(scale, length, header_size) without causing an
   // overflow. We also need to make sure that this will not overflow a size_t on
@@ -135,16 +156,17 @@ class arrayOopDesc : public oopDesc {
     assert(type >= 0 && type < T_CONFLICT, "wrong type");
     assert(type2aelembytes(type) != 0, "wrong type");
 
+    int hdr_size_in_words = UseCompactObjectHeaders ? (align_up(base_offset_in_bytes(type), HeapWordSize) / HeapWordSize) : header_size(type);
     const size_t max_element_words_per_size_t =
-      align_down((SIZE_MAX/HeapWordSize - header_size(type)), MinObjAlignment);
+      align_down((SIZE_MAX/HeapWordSize - hdr_size_in_words), MinObjAlignment);
     const size_t max_elements_per_size_t =
       HeapWordSize * max_element_words_per_size_t / type2aelembytes(type);
     if ((size_t)max_jint < max_elements_per_size_t) {
       // It should be ok to return max_jint here, but parts of the code
       // (CollectedHeap, Klass::oop_oop_iterate(), and more) uses an int for
       // passing around the size (in words) of an object. So, we need to avoid
       // overflowing an int when we add the header. See CRs 4718400 and 7110613.
-      return align_down(max_jint - header_size(type), MinObjAlignment);
+      return align_down(max_jint - hdr_size_in_words, MinObjAlignment);
     }
     return (int32_t)max_elements_per_size_t;
   }

src/hotspot/share/oops/objArrayOop.hpp

@@ -51,6 +51,7 @@ class objArrayOopDesc : public arrayOopDesc {
 private:
   // Give size of objArrayOop in HeapWords minus the header
   static int array_size(int length) {
+    assert(!UseCompactObjectHeaders, "sanity");
     const uint OopsPerHeapWord = HeapWordSize/heapOopSize;
     assert(OopsPerHeapWord >= 1 && (HeapWordSize % heapOopSize == 0),
            "Else the following (new) computation would be in error");
@@ -97,11 +98,19 @@ class objArrayOopDesc : public arrayOopDesc {
 
   static int object_size(int length) {
     // This returns the object size in HeapWords.
-    uint asz = array_size(length);
-    uint osz = align_object_size(header_size() + asz);
-    assert(osz >= asz,   "no overflow");
-    assert((int)osz > 0, "no overflow");
-    return (int)osz;
+    if (UseCompactObjectHeaders) {
+      size_t asz = (size_t)length * heapOopSize;
+      size_t size_words = heap_word_size(base_offset_in_bytes() + asz);
+      size_t osz = align_object_size(size_words);
+      assert(osz < max_jint, "no overflow");
+      return (int)osz;
+    } else {
+      uint asz = array_size(length);
+      uint osz = align_object_size(header_size() + asz);
+      assert(osz >= asz,   "no overflow");
+      assert((int)osz > 0, "no overflow");
+      return (int)osz;
+    }
   }
 
   Klass* element_klass();

src/hotspot/share/opto/runtime.cpp

@@ -314,12 +314,24 @@ JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_nozero_C(Klass* array_type, int len
     // Zero array here if the caller is deoptimized.
     int size = ((typeArrayOop)result)->object_size();
     BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
-    const size_t hs = arrayOopDesc::header_size(elem_type);
-    // Align to next 8 bytes to avoid trashing arrays's length.
-    const size_t aligned_hs = align_object_offset(hs);
     HeapWord* obj = (HeapWord*)result;
-    if (aligned_hs > hs) {
-      Copy::zero_to_words(obj+hs, aligned_hs-hs);
+    size_t aligned_hs = 0;
+    if (UseCompactObjectHeaders) {
+      size_t hs_bytes = arrayOopDesc::base_offset_in_bytes(elem_type);
+      assert(is_aligned(hs_bytes, BytesPerInt), "must be 4 byte aligned");
+      if (!is_aligned(hs_bytes, BytesPerLong)) {
+        *reinterpret_cast<jint*>(reinterpret_cast<char*>(obj) + hs_bytes) = 0;
+        hs_bytes += BytesPerInt;
+      }
+      assert(is_aligned(hs_bytes, BytesPerLong), "must be 8-byte aligned");
+      aligned_hs = hs_bytes / BytesPerLong;
+    } else {
+      const size_t hs = arrayOopDesc::header_size(elem_type);
+      // Align to next 8 bytes to avoid trashing arrays's length.
+      aligned_hs = align_object_offset(hs);
+      if (aligned_hs > hs) {
+        Copy::zero_to_words(obj+hs, aligned_hs-hs);
+      }
     }
     // Optimized zeroing.
     Copy::fill_to_aligned_words(obj+aligned_hs, size-aligned_hs);

src/hotspot/share/opto/type.cpp

@@ -4542,7 +4542,9 @@ void TypeAryPtr::dump2( Dict &d, uint depth, outputStream *st ) const {
   }
 
   if( _offset != 0 ) {
-    int header_size = objArrayOopDesc::header_size() * wordSize;
+    int header_size = UseCompactObjectHeaders ?
+                        arrayOopDesc::base_offset_in_bytes(elem()->basic_type()) :
+                        objArrayOopDesc::header_size() * wordSize;
     if( _offset == OffsetTop )       st->print("+undefined");
     else if( _offset == OffsetBot )  st->print("+any");
     else if( _offset < header_size ) st->print("+%d", _offset);

src/hotspot/share/prims/unsafe.cpp

@@ -59,7 +59,8 @@
 
 
 #define MAX_OBJECT_SIZE \
-  ( arrayOopDesc::header_size(T_DOUBLE) * HeapWordSize \
+  ( (UseCompactObjectHeaders ? arrayOopDesc::base_offset_in_bytes(T_DOUBLE) \
+    : (arrayOopDesc::header_size(T_DOUBLE) * HeapWordSize)) \
     + ((julong)max_jint * sizeof(double)) )
 
 

src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/memory/Universe.java

@@ -207,6 +207,11 @@ public void printOn(PrintStream tty) {
   // aligned 0 mod 8.  The typeArrayOop itself must be aligned at least this
   // strongly.
   public static boolean elementTypeShouldBeAligned(BasicType type) {
+    if (VM.getVM().isLP64() && VM.getVM().isCompactObjectHeadersEnabled()) {
+      if (type == BasicType.T_OBJECT || type == BasicType.T_ARRAY) {
+        return !VM.getVM().isCompressedOopsEnabled();
+      }
+    }
     return type == BasicType.T_DOUBLE || type == BasicType.T_LONG;
   }
 

src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/Array.java

@@ -59,7 +59,9 @@ private static long headerSizeInBytes() {
     if (headerSize != 0) {
       return headerSize;
     }
-    if (VM.getVM().isCompressedKlassPointersEnabled()) {
+    if (VM.getVM().isCompactObjectHeadersEnabled()) {
+      headerSize = lengthOffsetInBytes() + VM.getVM().getIntSize();
+    } else if (VM.getVM().isCompressedKlassPointersEnabled()) {
       headerSize = typeSize;
     } else {
       headerSize = VM.getVM().alignUp(typeSize + VM.getVM().getIntSize(),
@@ -76,7 +78,7 @@ private static long headerSize(BasicType type) {
     }
   }
 
-  private long lengthOffsetInBytes() {
+  private static long lengthOffsetInBytes() {
     if (lengthOffsetInBytes != 0) {
       return lengthOffsetInBytes;
     }
@@ -108,7 +110,16 @@ public long getObjectSize() {
   }
 
   public static long baseOffsetInBytes(BasicType type) {
-    return headerSize(type) * VM.getVM().getHeapWordSize();
+    if (VM.getVM().isCompactObjectHeadersEnabled()) {
+      long typeSizeInBytes = headerSizeInBytes();
+      if (Universe.elementTypeShouldBeAligned(type)) {
+        return VM.getVM().alignUp(typeSizeInBytes, VM.getVM().getHeapWordSize());
+      } else {
+        return typeSizeInBytes;
+      }
+    } else {
+      return headerSize(type) * VM.getVM().getHeapWordSize();
+    }
   }
 
   public boolean isArray()             { return true; }