Bug 1635692 - Store trace kind in nursery cell header and remove nurs…

…ery cell kind flags r=sfink

Differential Revision: https://phabricator.services.mozilla.com/D74022

js/src/gc/Cell.h

@@ -115,19 +115,9 @@ class CellHeader {
   // compacting GC and is now a RelocationOverlay.
   static constexpr uintptr_t FORWARD_BIT = Bit(0);
 
-  // When a Cell is in the nursery, this will indicate if it is a JSString (1)
-  // or JSObject/BigInt (0). When not in nursery, this bit is still reserved for
-  // JSString to use as JSString::NON_ATOM bit. This may be removed by Bug
-  // 1376646.
-  static constexpr uintptr_t JSSTRING_BIT = Bit(1);
-
-  // When a Cell is in the nursery, this will indicate if it is a BigInt (1)
-  // or JSObject/JSString (0).
-  static constexpr uintptr_t BIGINT_BIT = Bit(2);
+  // Bits 1 and 2 are currently unused.
 
   bool isForwarded() const { return header_ & FORWARD_BIT; }
-  bool isString() const { return header_ & JSSTRING_BIT; }
-  bool isBigInt() const { return header_ & BIGINT_BIT; }
 
   uintptr_t flags() const { return header_ & RESERVED_MASK; }
 
@@ -190,16 +180,6 @@ struct alignas(gc::CellAlignBytes) Cell {
     return reinterpret_cast<const CellHeader*>(this)->isForwarded();
   }
 
-  inline bool nurseryCellIsString() const {
-    MOZ_ASSERT(!isTenured());
-    return reinterpret_cast<const CellHeader*>(this)->isString();
-  }
-
-  inline bool nurseryCellIsBigInt() const {
-    MOZ_ASSERT(!isTenured());
-    return reinterpret_cast<const CellHeader*>(this)->isBigInt();
-  }
-
   template <class T>
   inline bool is() const {
     return getTraceKind() == JS::MapTypeToTraceKind<T>::kind;
@@ -404,7 +384,7 @@ JS::Zone* Cell::nurseryZone() const {
 }
 
 JS::Zone* Cell::nurseryZoneFromAnyThread() const {
-  return NurseryCellHeader::from(this)->zone;
+  return NurseryCellHeader::from(this)->zone();
 }
 
 #ifdef DEBUG
@@ -417,19 +397,8 @@ inline JS::TraceKind Cell::getTraceKind() const {
                                      asTenured().getTraceKind());
     return asTenured().getTraceKind();
   }
-  if (nurseryCellIsString()) {
-    MOZ_ASSERT_IF(isForwarded(), UninlinedForwarded(this)->getTraceKind() ==
-                                     JS::TraceKind::String);
-    return JS::TraceKind::String;
-  }
-  if (nurseryCellIsBigInt()) {
-    MOZ_ASSERT_IF(isForwarded(), UninlinedForwarded(this)->getTraceKind() ==
-                                     JS::TraceKind::BigInt);
-    return JS::TraceKind::BigInt;
-  }
-  MOZ_ASSERT_IF(isForwarded(), UninlinedForwarded(this)->getTraceKind() ==
-                                   JS::TraceKind::Object);
-  return JS::TraceKind::Object;
+
+  return NurseryCellHeader::from(this)->traceKind();
 }
 
 /* static */ MOZ_ALWAYS_INLINE bool Cell::needWriteBarrierPre(JS::Zone* zone) {

js/src/gc/GC.cpp

@@ -7998,17 +7998,7 @@ JS_FRIEND_API void js::gc::AssertGCThingHasType(js::gc::Cell* cell,
   }
 
   MOZ_ASSERT(IsCellPointerValid(cell));
-
-  if (IsInsideNursery(cell)) {
-    MOZ_ASSERT(kind == (cell->nurseryCellIsString()
-                            ? JS::TraceKind::String
-                            : cell->nurseryCellIsBigInt()
-                                  ? JS::TraceKind::BigInt
-                                  : JS::TraceKind::Object));
-    return;
-  }
-
-  MOZ_ASSERT(MapAllocToTraceKind(cell->asTenured().getAllocKind()) == kind);
+  MOZ_ASSERT(cell->getTraceKind() == kind);
 }
 #endif
 

js/src/gc/Heap.h

@@ -845,7 +845,29 @@ static const int32_t ChunkStoreBufferOffsetFromLastByte =
 
 // Cell header stored before all nursery cells.
 struct alignas(gc::CellAlignBytes) NurseryCellHeader {
-  JS::Zone* const zone;
+  // Store zone pointer with the trace kind in the lowest three bits.
+  const uintptr_t zoneAndTraceKind;
+
+  // We only need to store a subset of trace kinds so this doesn't cover the
+  // full range.
+  static const uintptr_t TraceKindMask = 3;
+
+  static uintptr_t MakeValue(JS::Zone* const zone, JS::TraceKind kind) {
+    MOZ_ASSERT(uintptr_t(kind) < TraceKindMask);
+    MOZ_ASSERT((uintptr_t(zone) & TraceKindMask) == 0);
+    return uintptr_t(zone) | uintptr_t(kind);
+  }
+
+  NurseryCellHeader(JS::Zone* const zone, JS::TraceKind kind)
+      : zoneAndTraceKind(MakeValue(zone, kind)) {}
+
+  JS::Zone* zone() const {
+    return reinterpret_cast<JS::Zone*>(zoneAndTraceKind & ~TraceKindMask);
+  }
+
+  JS::TraceKind traceKind() const {
+    return JS::TraceKind(zoneAndTraceKind & TraceKindMask);
+  }
 
   static const NurseryCellHeader* from(const Cell* cell) {
     MOZ_ASSERT(IsInsideNursery(cell));
@@ -854,6 +876,13 @@ struct alignas(gc::CellAlignBytes) NurseryCellHeader {
   }
 };
 
+static_assert(uintptr_t(JS::TraceKind::Object) <=
+              NurseryCellHeader::TraceKindMask);
+static_assert(uintptr_t(JS::TraceKind::String) <=
+              NurseryCellHeader::TraceKindMask);
+static_assert(uintptr_t(JS::TraceKind::BigInt) <=
+              NurseryCellHeader::TraceKindMask);
+
 } /* namespace gc */
 
 namespace debug {

js/src/gc/Nursery.cpp

@@ -466,7 +466,7 @@ Cell* js::Nursery::allocateCell(Zone* zone, size_t size, JS::TraceKind kind) {
     return nullptr;
   }
 
-  new (ptr) NurseryCellHeader{zone};
+  new (ptr) NurseryCellHeader(zone, kind);
 
   auto cell =
       reinterpret_cast<Cell*>(uintptr_t(ptr) + sizeof(NurseryCellHeader));

js/src/jit/CompileWrappers.cpp

@@ -7,6 +7,7 @@
 #include "jit/CompileWrappers.h"
 
 #include "gc/GC.h"
+#include "gc/Heap.h"
 #include "jit/Ion.h"
 #include "jit/JitRealm.h"
 
@@ -163,6 +164,10 @@ void CompileZone::setMinorGCShouldCancelIonCompilations() {
   rt->gc.storeBuffer().setShouldCancelIonCompilations();
 }
 
+uintptr_t CompileZone::nurseryCellHeader(JS::TraceKind kind) {
+  return gc::NurseryCellHeader::MakeValue(zone(), kind);
+}
+
 JS::Realm* CompileRealm::realm() { return reinterpret_cast<JS::Realm*>(this); }
 
 /* static */

js/src/jit/CompileWrappers.h

@@ -82,6 +82,8 @@ class CompileZone {
   bool canNurseryAllocateStrings();
   bool canNurseryAllocateBigInts();
   void setMinorGCShouldCancelIonCompilations();
+
+  uintptr_t nurseryCellHeader(JS::TraceKind kind);
 };
 
 class JitRealm;

js/src/jit/MacroAssembler.cpp

@@ -599,9 +599,9 @@ void MacroAssembler::nurseryAllocateObject(Register result, Register temp,
   MOZ_ASSERT(totalSize < INT32_MAX);
   MOZ_ASSERT(totalSize % gc::CellAlignBytes == 0);
 
-  bumpPointerAllocate(result, temp, fail, zone,
-                      zone->addressOfNurseryPosition(),
-                      zone->addressOfNurseryCurrentEnd(), totalSize);
+  bumpPointerAllocate(
+      result, temp, fail, zone, zone->addressOfNurseryPosition(),
+      zone->addressOfNurseryCurrentEnd(), JS::TraceKind::Object, totalSize);
 
   if (nDynamicSlots) {
     computeEffectiveAddress(Address(result, thingSize), temp);
@@ -739,7 +739,8 @@ void MacroAssembler::nurseryAllocateString(Register result, Register temp,
 
   bumpPointerAllocate(result, temp, fail, zone,
                       zone->addressOfStringNurseryPosition(),
-                      zone->addressOfStringNurseryCurrentEnd(), thingSize);
+                      zone->addressOfStringNurseryCurrentEnd(),
+                      JS::TraceKind::String, thingSize);
 }
 
 // Inline version of Nursery::allocateBigInt.
@@ -755,12 +756,14 @@ void MacroAssembler::nurseryAllocateBigInt(Register result, Register temp,
 
   bumpPointerAllocate(result, temp, fail, zone,
                       zone->addressOfBigIntNurseryPosition(),
-                      zone->addressOfBigIntNurseryCurrentEnd(), thingSize);
+                      zone->addressOfBigIntNurseryCurrentEnd(),
+                      JS::TraceKind::BigInt, thingSize);
 }
 
 void MacroAssembler::bumpPointerAllocate(Register result, Register temp,
                                          Label* fail, CompileZone* zone,
                                          void* posAddr, const void* curEndAddr,
+                                         JS::TraceKind traceKind,
                                          uint32_t size) {
   uint32_t totalSize = size + Nursery::nurseryCellHeaderSize();
   MOZ_ASSERT(totalSize < INT32_MAX, "Nursery allocation too large");
@@ -784,7 +787,7 @@ void MacroAssembler::bumpPointerAllocate(Register result, Register temp,
   branchPtr(Assembler::Below, Address(temp, endOffset.value()), result, fail);
   storePtr(result, Address(temp, 0));
   subPtr(Imm32(size), result);
-  storePtr(ImmPtr(zone),
+  storePtr(ImmWord(zone->nurseryCellHeader(traceKind)),
            Address(result, -js::Nursery::nurseryCellHeaderSize()));
 
   if (GetJitContext()->runtime->geckoProfiler().enabled()) {
@@ -1557,9 +1560,7 @@ void MacroAssembler::initializeBigInt64(Scalar::Type type, Register bigInt,
                                         Register64 val) {
   MOZ_ASSERT(Scalar::isBigIntType(type));
 
-  uint32_t flags = BigInt::TYPE_FLAGS;
-
-  store32(Imm32(flags), Address(bigInt, BigInt::offsetOfFlags()));
+  store32(Imm32(0), Address(bigInt, BigInt::offsetOfFlags()));
 
   Label done, nonZero;
   branch64(Assembler::NotEqual, val, Imm64(0), &nonZero);
@@ -1575,7 +1576,7 @@ void MacroAssembler::initializeBigInt64(Scalar::Type type, Register bigInt,
     Label isPositive;
     branch64(Assembler::GreaterThan, val, Imm64(0), &isPositive);
     {
-      store32(Imm32(BigInt::signBitMask() | flags),
+      store32(Imm32(BigInt::signBitMask()),
               Address(bigInt, BigInt::offsetOfFlags()));
       neg64(val);
     }

js/src/jit/MacroAssembler.h

@@ -3394,7 +3394,8 @@ class MacroAssembler : public MacroAssemblerSpecific {
                              Label* fail);
   void bumpPointerAllocate(Register result, Register temp, Label* fail,
                            CompileZone* zone, void* posAddr,
-                           const void* curEddAddr, uint32_t size);
+                           const void* curEddAddr, JS::TraceKind traceKind,
+                           uint32_t size);
 
   void freeListAllocate(Register result, Register temp, gc::AllocKind allocKind,
                         Label* fail);

js/src/vm/BigIntType.h

@@ -43,8 +43,6 @@ class BigInt final : public js::gc::Cell {
  public:
   using Digit = uintptr_t;
 
-  static constexpr uintptr_t TYPE_FLAGS = js::gc::CellHeader::BIGINT_BIT;
-
  private:
   using Header = js::gc::CellHeaderWithLengthAndFlags;
 
@@ -65,7 +63,7 @@ class BigInt final : public js::gc::Cell {
   };
 
   void setLengthAndFlags(uint32_t len, uint32_t flags) {
-    header_.setLengthAndFlags(len, flags | TYPE_FLAGS);
+    header_.setLengthAndFlags(len, flags);
   }
 
  public:

js/src/vm/StringType.h

@@ -263,7 +263,7 @@ class JSString : public js::gc::Cell {
   static_assert(js::gc::CellFlagBitsReservedForGC <= 3,
                 "JSString::flags must reserve enough bits for Cell");
 
-  static const uint32_t NON_ATOM_BIT = js::gc::CellHeader::JSSTRING_BIT;
+  static const uint32_t NON_ATOM_BIT = js::Bit(1);
   static const uint32_t LINEAR_BIT = js::Bit(4);
   static const uint32_t DEPENDENT_BIT = js::Bit(5);
   static const uint32_t INLINE_CHARS_BIT = js::Bit(6);
@@ -287,10 +287,7 @@ class JSString : public js::gc::Cell {
       NON_ATOM_BIT | LINEAR_BIT | DEPENDENT_BIT;
 
   static const uint32_t TYPE_FLAGS_MASK =
-      js::BitMask(9) - js::BitMask(3) + js::gc::CellHeader::JSSTRING_BIT;
-
-  static_assert((TYPE_FLAGS_MASK & js::gc::CellHeader::BIGINT_BIT) == 0,
-                "BigInt bit must not be used for Strings");
+      js::BitMask(9) - js::BitMask(3) + NON_ATOM_BIT;
 
   static const uint32_t LATIN1_CHARS_BIT = js::Bit(9);