[CodeView] Add a random access type visitor.

This adds a visitor that is capable of accessing type
records randomly and caching intermediate results that it
learns about during partial linear scans.  This yields
amortized O(1) access to a type stream even though type
streams cannot normally be indexed.

Differential Revision: https://reviews.llvm.org/D33009

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@302936 91177308-0d34-0410-b5e6-96231b3b80d8

include/llvm/DebugInfo/CodeView/RandomAccessTypeVisitor.h

@@ -0,0 +1,103 @@
+//===- RandomAccessTypeVisitor.h ------------------------------ *- C++ --*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_CODEVIEW_RANDOMACCESSTYPEVISITOR_H
+#define LLVM_DEBUGINFO_CODEVIEW_RANDOMACCESSTYPEVISITOR_H
+
+#include "llvm/ADT/TinyPtrVector.h"
+#include "llvm/DebugInfo/CodeView/CVTypeVisitor.h"
+#include "llvm/DebugInfo/CodeView/TypeDatabase.h"
+#include "llvm/DebugInfo/CodeView/TypeDatabaseVisitor.h"
+#include "llvm/DebugInfo/CodeView/TypeDeserializer.h"
+#include "llvm/DebugInfo/CodeView/TypeIndex.h"
+#include "llvm/DebugInfo/CodeView/TypeRecord.h"
+#include "llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h"
+#include "llvm/Support/Error.h"
+
+namespace llvm {
+namespace codeview {
+
+class TypeDatabase;
+class TypeServerHandler;
+class TypeVisitorCallbacks;
+
+/// \brief Provides amortized O(1) random access to a CodeView type stream.
+/// Normally to access a type from a type stream, you must know its byte
+/// offset into the type stream, because type records are variable-lengthed.
+/// However, this is not the way we prefer to access them.  For example, given
+/// a symbol record one of the fields may be the TypeIndex of the symbol's
+/// type record.  Or given a type record such as an array type, there might
+/// be a TypeIndex for the element type.  Sequential access is perfect when
+/// we're just dumping every entry, but it's very poor for real world usage.
+///
+/// Type streams in PDBs contain an additional field which is a list of pairs
+/// containing indices and their corresponding offsets, roughly every ~8KB of
+/// record data.  This general idea need not be confined to PDBs though.  By
+/// supplying such an array, the producer of a type stream can allow the
+/// consumer much better access time, because the consumer can find the nearest
+/// index in this array, and do a linear scan forward only from there.
+///
+/// RandomAccessTypeVisitor implements this algorithm, but additionally goes one
+/// step further by caching offsets of every record that has been visited at
+/// least once.  This way, even repeated visits of the same record will never
+/// require more than one linear scan.  For a type stream of N elements divided
+/// into M chunks of roughly equal size, this yields a worst case lookup time
+/// of O(N/M) and an amortized time of O(1).
+class RandomAccessTypeVisitor {
+  typedef FixedStreamArray<TypeIndexOffset> PartialOffsetArray;
+
+public:
+  RandomAccessTypeVisitor(const CVTypeArray &Types, uint32_t NumRecords,
+                          PartialOffsetArray PartialOffsets);
+
+  Error visitTypeIndex(TypeIndex Index, TypeVisitorCallbacks &Callbacks);
+
+  const TypeDatabase &database() const { return Database; }
+
+private:
+  Error visitRangeForType(TypeIndex TI);
+  Error visitRange(TypeIndex Begin, uint32_t BeginOffset, TypeIndex End);
+
+  /// Visited records get automatically added to the type database.
+  TypeDatabase Database;
+
+  /// The type array to allow random access visitation of.
+  const CVTypeArray &Types;
+
+  /// The database visitor which adds new records to the database.
+  TypeDatabaseVisitor DatabaseVisitor;
+
+  /// The deserializer which deserializes new records.
+  TypeDeserializer Deserializer;
+
+  /// The visitation callback pipeline to use.  By default this contains a
+  /// deserializer and a type database visitor.  But the callback specified
+  /// in the constructor is also added.
+  TypeVisitorCallbackPipeline Pipeline;
+
+  /// The visitor used to visit the internal pipeline for deserialization and
+  /// database maintenance.
+  CVTypeVisitor InternalVisitor;
+
+  /// A vector mapping type indices to type offset.  For every record that has
+  /// been visited, contains the absolute offset of that record in the record
+  /// array.
+  std::vector<uint32_t> KnownOffsets;
+
+  /// An array of index offsets for the given type stream, allowing log(N)
+  /// lookups of a type record by index.  Similar to KnownOffsets but only
+  /// contains offsets for some type indices, some of which may not have
+  /// ever been visited.
+  PartialOffsetArray PartialOffsets;
+};
+
+} // end namespace codeview
+} // end namespace llvm
+
+#endif // LLVM_DEBUGINFO_CODEVIEW_RANDOMACCESSTYPEVISITOR_H

include/llvm/DebugInfo/CodeView/TypeDatabase.h

@@ -24,13 +24,13 @@ class TypeDatabase {
   friend class RandomAccessTypeVisitor;
 
 public:
-  explicit TypeDatabase(uint32_t ExpectedSize);
+  explicit TypeDatabase(uint32_t Capacity);
 
-  /// Gets the type index for the next type record.
-  TypeIndex getNextTypeIndex() const;
+  /// Records the name of a type, and reserves its type index.
+  TypeIndex appendType(StringRef Name, const CVType &Data);
 
   /// Records the name of a type, and reserves its type index.
-  void recordType(StringRef Name, const CVType &Data);
+  void recordType(StringRef Name, TypeIndex Index, const CVType &Data);
 
   /// Saves the name in a StringSet and creates a stable StringRef.
   StringRef saveTypeName(StringRef TypeName);
@@ -40,43 +40,29 @@ class TypeDatabase {
   const CVType &getTypeRecord(TypeIndex Index) const;
   CVType &getTypeRecord(TypeIndex Index);
 
-  bool containsTypeIndex(TypeIndex Index) const;
+  bool contains(TypeIndex Index) const;
 
   uint32_t size() const;
+  uint32_t capacity() const;
+  bool empty() const;
+
+  TypeIndex getAppendIndex() const;
 
-protected:
-  uint32_t toArrayIndex(TypeIndex Index) const;
+private:
+  void grow();
 
   BumpPtrAllocator Allocator;
 
+  uint32_t Count = 0;
+
   /// All user defined type records in .debug$T live in here. Type indices
   /// greater than 0x1000 are user defined. Subtract 0x1000 from the index to
   /// index into this vector.
   SmallVector<StringRef, 10> CVUDTNames;
   SmallVector<CVType, 10> TypeRecords;
 
   StringSaver TypeNameStorage;
-};
-
-class RandomAccessTypeDatabase : private TypeDatabase {
-public:
-  explicit RandomAccessTypeDatabase(uint32_t ExpectedSize);
-
-  /// Records the name of a type, and reserves its type index.
-  void recordType(StringRef Name, TypeIndex Index, const CVType &Data);
 
-  using TypeDatabase::saveTypeName;
-
-  StringRef getTypeName(TypeIndex Index) const;
-
-  const CVType &getTypeRecord(TypeIndex Index) const;
-  CVType &getTypeRecord(TypeIndex Index);
-
-  bool containsTypeIndex(TypeIndex Index) const;
-
-  uint32_t size() const;
-
-private:
   BitVector ValidRecords;
 };
 }

include/llvm/DebugInfo/CodeView/TypeDatabaseVisitor.h

@@ -24,8 +24,6 @@ namespace codeview {
 class TypeDatabaseVisitor : public TypeVisitorCallbacks {
 public:
   explicit TypeDatabaseVisitor(TypeDatabase &TypeDB) : TypeDB(&TypeDB) {}
-  explicit TypeDatabaseVisitor(RandomAccessTypeDatabase &TypeDB)
-      : TypeDB(&TypeDB) {}
 
   /// Paired begin/end actions for all types. Receives all record data,
   /// including the fixed-length record prefix.
@@ -53,9 +51,9 @@ class TypeDatabaseVisitor : public TypeVisitorCallbacks {
   StringRef Name;
   /// Current type index.  Only valid before visitTypeEnd, and if we are
   /// visiting a random access type database.
-  TypeIndex CurrentTypeIndex;
+  Optional<TypeIndex> CurrentTypeIndex;
 
-  PointerUnion<TypeDatabase *, RandomAccessTypeDatabase *> TypeDB;
+  TypeDatabase *TypeDB;
 };
 
 } // end namespace codeview

include/llvm/DebugInfo/CodeView/TypeDeserializer.h

@@ -46,6 +46,10 @@ class TypeDeserializer : public TypeVisitorCallbacks {
     return Mapping->Mapping.visitTypeBegin(Record);
   }
 
+  Error visitTypeBegin(CVType &Record, TypeIndex Index) override {
+    return visitTypeBegin(Record);
+  }
+
   Error visitTypeEnd(CVType &Record) override {
     assert(Mapping && "Not in a type mapping!");
     auto EC = Mapping->Mapping.visitTypeEnd(Record);

include/llvm/DebugInfo/CodeView/TypeDumpVisitor.h

@@ -45,6 +45,7 @@ class TypeDumpVisitor : public TypeVisitorCallbacks {
   /// Paired begin/end actions for all types. Receives all record data,
   /// including the fixed-length record prefix.
   Error visitTypeBegin(CVType &Record) override;
+  Error visitTypeBegin(CVType &Record, TypeIndex Index) override;
   Error visitTypeEnd(CVType &Record) override;
   Error visitMemberBegin(CVMemberRecord &Record) override;
   Error visitMemberEnd(CVMemberRecord &Record) override;

include/llvm/DebugInfo/CodeView/TypeIndex.h

@@ -242,6 +242,13 @@ class TypeIndex {
   support::ulittle32_t Index;
 };
 
+// Used for pseudo-indexing an array of type records.  An array of such records
+// sorted by TypeIndex can allow log(N) lookups even though such a type record
+// stream does not provide random access.
+struct TypeIndexOffset {
+  TypeIndex Type;
+  support::ulittle32_t Offset;
+};
 }
 }
 

include/llvm/DebugInfo/PDB/Native/RawTypes.h

@@ -73,13 +73,6 @@ struct SecMapEntry {
   support::ulittle32_t SecByteLength; // Byte count of the segment or group.
 };
 
-// Used for serialized hash table in TPI stream.
-// In the reference, it is an array of TI and cbOff pair.
-struct TypeIndexOffset {
-  codeview::TypeIndex Type;
-  support::ulittle32_t Offset;
-};
-
 /// Some of the values are stored in bitfields.  Since this needs to be portable
 /// across compilers and architectures (big / little endian in particular) we
 /// can't use the actual structures below, but must instead do the shifting

include/llvm/DebugInfo/PDB/Native/TpiStream.h

@@ -47,7 +47,7 @@ class TpiStream {
   uint32_t getHashKeySize() const;
   uint32_t getNumHashBuckets() const;
   FixedStreamArray<support::ulittle32_t> getHashValues() const;
-  FixedStreamArray<TypeIndexOffset> getTypeIndexOffsets() const;
+  FixedStreamArray<codeview::TypeIndexOffset> getTypeIndexOffsets() const;
   HashTable &getHashAdjusters();
 
   codeview::CVTypeRange types(bool *HadError) const;
@@ -62,7 +62,7 @@ class TpiStream {
 
   std::unique_ptr<BinaryStream> HashStream;
   FixedStreamArray<support::ulittle32_t> HashValues;
-  FixedStreamArray<TypeIndexOffset> TypeIndexOffsets;
+  FixedStreamArray<codeview::TypeIndexOffset> TypeIndexOffsets;
   HashTable HashAdjusters;
 
   const TpiStreamHeader *Header;

include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h

@@ -75,7 +75,7 @@ class TpiStreamBuilder {
   Optional<PdbRaw_TpiVer> VerHeader;
   std::vector<ArrayRef<uint8_t>> TypeRecords;
   std::vector<uint32_t> TypeHashes;
-  std::vector<TypeIndexOffset> TypeIndexOffsets;
+  std::vector<codeview::TypeIndexOffset> TypeIndexOffsets;
   uint32_t HashStreamIndex = kInvalidStreamIndex;
   std::unique_ptr<BinaryByteStream> HashValueStream;
 

include/llvm/Support/BinaryStreamArray.h

@@ -139,6 +139,7 @@ class VarStreamArrayIterator
   }
 
   uint32_t offset() const { return AbsOffset; }
+  uint32_t getRecordLength() const { return ThisLen; }
 
 private:
   void moveToEnd() {
@@ -294,6 +295,8 @@ template <typename T> class FixedStreamArray {
   friend class FixedStreamArrayIterator<T>;
 
 public:
+  typedef FixedStreamArrayIterator<T> Iterator;
+
   FixedStreamArray() = default;
   explicit FixedStreamArray(BinaryStreamRef Stream) : Stream(Stream) {
     assert(Stream.getLength() % sizeof(T) == 0);
@@ -371,7 +374,7 @@ class FixedStreamArrayIterator
   }
 
   FixedStreamArrayIterator<T> &operator-=(std::ptrdiff_t N) {
-    assert(Index >= N);
+    assert(std::ptrdiff_t(Index) >= N);
     Index -= N;
     return *this;
   }

lib/DebugInfo/CodeView/CMakeLists.txt

@@ -13,6 +13,7 @@ add_llvm_library(LLVMDebugInfoCodeView
   ModuleDebugFragmentVisitor.cpp
   ModuleDebugInlineeLinesFragment.cpp
   ModuleDebugLineFragment.cpp
+  RandomAccessTypeVisitor.cpp
   RecordSerialization.cpp
   StringTable.cpp
   SymbolRecordMapping.cpp

lib/DebugInfo/CodeView/CVTypeVisitor.cpp

@@ -26,8 +26,7 @@ CVTypeVisitor::CVTypeVisitor(TypeVisitorCallbacks &Callbacks)
     : Callbacks(Callbacks) {}
 
 template <typename T>
-static Error visitKnownRecord(CVTypeVisitor &Visitor, CVType &Record,
-                              TypeVisitorCallbacks &Callbacks) {
+static Error visitKnownRecord(CVType &Record, TypeVisitorCallbacks &Callbacks) {
   TypeRecordKind RK = static_cast<TypeRecordKind>(Record.Type);
   T KnownRecord(RK);
   if (auto EC = Callbacks.visitKnownRecord(Record, KnownRecord))
@@ -107,7 +106,7 @@ Error CVTypeVisitor::finishVisitation(CVType &Record) {
     break;
 #define TYPE_RECORD(EnumName, EnumVal, Name)                                   \
   case EnumName: {                                                             \
-    if (auto EC = visitKnownRecord<Name##Record>(*this, Record, Callbacks))    \
+    if (auto EC = visitKnownRecord<Name##Record>(Record, Callbacks))           \
       return EC;                                                               \
     break;                                                                     \
   }