Skip to content

Commit

Permalink
Trimmed vtables of trailing zeroes.
Browse files Browse the repository at this point in the history
This is something the format supports, but none of the builders
were doing. Can save 10-20% on FlatBuffer binary size!

Also fixed the Go tests.

Change-Id: I616c56ce9bbcfcaee23aa24f0532fcb60b6a8c75
Tested: on Linux.
  • Loading branch information
aardappel committed Aug 24, 2017
1 parent 513958e commit ac1015e
Showing 25 changed files with 129 additions and 62 deletions.
5 changes: 5 additions & 0 deletions go/builder.go
Original file line number Diff line number Diff line change
@@ -110,6 +110,11 @@ func (b *Builder) WriteVtable() (n UOffsetT) {
objectOffset := b.Offset()
existingVtable := UOffsetT(0)

// Trim vtable of trailing zeroes.
i := len(b.vtable) - 1;
for ; i >= 0 && b.vtable[i] == 0; i-- {}
b.vtable = b.vtable[:i + 1];

// Search backwards through existing vtables, because similar vtables
// are likely to have been recently appended. See
// BenchmarkVtableDeduplication for a case in which this heuristic
36 changes: 27 additions & 9 deletions include/flatbuffers/flatbuffers.h
Original file line number Diff line number Diff line change
@@ -704,8 +704,8 @@ class FlatBufferBuilder
explicit FlatBufferBuilder(size_t initial_size = 1024,
Allocator *allocator = nullptr,
bool own_allocator = false)
: buf_(initial_size, allocator, own_allocator), nested(false),
finished(false), minalign_(1), force_defaults_(false),
: buf_(initial_size, allocator, own_allocator), max_voffset_(0),
nested(false), finished(false), minalign_(1), force_defaults_(false),
dedup_vtables_(true), string_pool(nullptr) {
offsetbuf_.reserve(16); // Avoid first few reallocs.
vtables_.reserve(16);
@@ -725,7 +725,7 @@ class FlatBufferBuilder
/// to construct another buffer.
void Clear() {
buf_.clear();
offsetbuf_.clear();
ClearOffsets();
nested = false;
finished = false;
vtables_.clear();
@@ -839,6 +839,7 @@ class FlatBufferBuilder
void TrackField(voffset_t field, uoffset_t off) {
FieldLoc fl = { off, field };
offsetbuf_.push_back(fl);
max_voffset_ = (std::max)(max_voffset_, field);
}

// Like PushElement, but additionally tracks the field this represents.
@@ -899,7 +900,7 @@ class FlatBufferBuilder
// This finishes one serialized object by generating the vtable if it's a
// table, comparing it against existing vtables, and writing the
// resulting vtable offset.
uoffset_t EndTable(uoffset_t start, voffset_t numfields) {
uoffset_t EndTable(uoffset_t start) {
// If you get this assert, a corresponding StartTable wasn't called.
assert(nested);
// Write the vtable offset, which is the start of any Table.
@@ -908,11 +909,17 @@ class FlatBufferBuilder
// Write a vtable, which consists entirely of voffset_t elements.
// It starts with the number of offsets, followed by a type id, followed
// by the offsets themselves. In reverse:
buf_.fill_big(numfields * sizeof(voffset_t));
// Include space for the last offset and ensure empty tables have a
// minimum size.
max_voffset_ = (std::max)(static_cast<voffset_t>(max_voffset_ +
sizeof(voffset_t)),
FieldIndexToOffset(0));
buf_.fill_big(max_voffset_);
auto table_object_size = vtableoffsetloc - start;
assert(table_object_size < 0x10000); // Vtable use 16bit offsets.
PushElement<voffset_t>(static_cast<voffset_t>(table_object_size));
PushElement<voffset_t>(FieldIndexToOffset(numfields));
WriteScalar<voffset_t>(buf_.data() + sizeof(voffset_t),
static_cast<voffset_t>(table_object_size));
WriteScalar<voffset_t>(buf_.data(), max_voffset_);
// Write the offsets into the table
for (auto field_location = offsetbuf_.begin();
field_location != offsetbuf_.end();
@@ -922,7 +929,7 @@ class FlatBufferBuilder
assert(!ReadScalar<voffset_t>(buf_.data() + field_location->id));
WriteScalar<voffset_t>(buf_.data() + field_location->id, pos);
}
offsetbuf_.clear();
ClearOffsets();
auto vt1 = reinterpret_cast<voffset_t *>(buf_.data());
auto vt1_size = ReadScalar<voffset_t>(vt1);
auto vt_use = GetSize();
@@ -955,6 +962,11 @@ class FlatBufferBuilder
return vtableoffsetloc;
}

// DEPRECATED: call the version above instead.
uoffset_t EndTable(uoffset_t start, voffset_t /*numfields*/) {
return EndTable(start);
}

// This checks a required field has been set in a given table that has
// just been constructed.
template<typename T> void Required(Offset<T> table, voffset_t field) {
@@ -973,7 +985,10 @@ class FlatBufferBuilder

uoffset_t EndStruct() { return GetSize(); }

void ClearOffsets() { offsetbuf_.clear(); }
void ClearOffsets() {
offsetbuf_.clear();
max_voffset_ = 0;
}

// Aligns such that when "len" bytes are written, an object can be written
// after it with "alignment" without padding.
@@ -1510,6 +1525,9 @@ class FlatBufferBuilder

// Accumulating offsets of table members while it is being built.
std::vector<FieldLoc> offsetbuf_;
// Track how much of the vtable is in use, so we can output the most compact
// possible vtable.
voffset_t max_voffset_;

// Ensure objects are not nested.
bool nested;
37 changes: 30 additions & 7 deletions include/flatbuffers/reflection_generated.h
Original file line number Diff line number Diff line change
@@ -42,6 +42,29 @@ enum BaseType {
Union = 16
};

inline BaseType (&EnumValuesBaseType())[17] {
static BaseType values[] = {
None,
UType,
Bool,
Byte,
UByte,
Short,
UShort,
Int,
UInt,
Long,
ULong,
Float,
Double,
String,
Vector,
Obj,
Union
};
return values;
}

inline const char **EnumNamesBaseType() {
static const char *names[] = {
"None",
@@ -113,7 +136,7 @@ struct TypeBuilder {
}
TypeBuilder &operator=(const TypeBuilder &);
flatbuffers::Offset<Type> Finish() {
const auto end = fbb_.EndTable(start_, 3);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Type>(end);
return o;
}
@@ -173,7 +196,7 @@ struct KeyValueBuilder {
}
KeyValueBuilder &operator=(const KeyValueBuilder &);
flatbuffers::Offset<KeyValue> Finish() {
const auto end = fbb_.EndTable(start_, 2);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<KeyValue>(end);
fbb_.Required(o, KeyValue::VT_KEY);
return o;
@@ -266,7 +289,7 @@ struct EnumValBuilder {
}
EnumValBuilder &operator=(const EnumValBuilder &);
flatbuffers::Offset<EnumVal> Finish() {
const auto end = fbb_.EndTable(start_, 4);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<EnumVal>(end);
fbb_.Required(o, EnumVal::VT_NAME);
return o;
@@ -381,7 +404,7 @@ struct EnumBuilder {
}
EnumBuilder &operator=(const EnumBuilder &);
flatbuffers::Offset<Enum> Finish() {
const auto end = fbb_.EndTable(start_, 6);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Enum>(end);
fbb_.Required(o, Enum::VT_NAME);
fbb_.Required(o, Enum::VT_VALUES);
@@ -544,7 +567,7 @@ struct FieldBuilder {
}
FieldBuilder &operator=(const FieldBuilder &);
flatbuffers::Offset<Field> Finish() {
const auto end = fbb_.EndTable(start_, 11);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Field>(end);
fbb_.Required(o, Field::VT_NAME);
fbb_.Required(o, Field::VT_TYPE);
@@ -695,7 +718,7 @@ struct ObjectBuilder {
}
ObjectBuilder &operator=(const ObjectBuilder &);
flatbuffers::Offset<Object> Finish() {
const auto end = fbb_.EndTable(start_, 7);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Object>(end);
fbb_.Required(o, Object::VT_NAME);
fbb_.Required(o, Object::VT_FIELDS);
@@ -808,7 +831,7 @@ struct SchemaBuilder {
}
SchemaBuilder &operator=(const SchemaBuilder &);
flatbuffers::Offset<Schema> Finish() {
const auto end = fbb_.EndTable(start_, 5);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Schema>(end);
fbb_.Required(o, Schema::VT_OBJECTS);
fbb_.Required(o, Schema::VT_ENUMS);
12 changes: 8 additions & 4 deletions java/com/google/flatbuffers/FlatBufferBuilder.java
Original file line number Diff line number Diff line change
@@ -478,7 +478,7 @@ public <T extends Table> int createSortedVectorOfTables(T obj, int[] offsets) {
obj.sortTables(offsets, bb);
return createVectorOfTables(offsets);
}

/**
* Encode the string `s` in the buffer using UTF-8. If {@code s} is
* already a {@link CharBuffer}, this method is allocation free.
@@ -744,20 +744,24 @@ public int endObject() {
addInt(0);
int vtableloc = offset();
// Write out the current vtable.
for (int i = vtable_in_use - 1; i >= 0 ; i--) {
int i = vtable_in_use - 1;
// Trim trailing zeroes.
for (; i >= 0 && vtable[i] == 0; i--) {}
int trimmed_size = i + 1;
for (; i >= 0 ; i--) {
// Offset relative to the start of the table.
short off = (short)(vtable[i] != 0 ? vtableloc - vtable[i] : 0);
addShort(off);
}

final int standard_fields = 2; // The fields below:
addShort((short)(vtableloc - object_start));
addShort((short)((vtable_in_use + standard_fields) * SIZEOF_SHORT));
addShort((short)((trimmed_size + standard_fields) * SIZEOF_SHORT));

// Search for an existing vtable that matches the current one.
int existing_vtable = 0;
outer_loop:
for (int i = 0; i < num_vtables; i++) {
for (i = 0; i < num_vtables; i++) {
int vt1 = bb.capacity() - vtables[i];
int vt2 = space;
short len = bb.getShort(vt1);
17 changes: 11 additions & 6 deletions js/flatbuffers.js
Original file line number Diff line number Diff line change
@@ -604,23 +604,28 @@ flatbuffers.Builder.prototype.endObject = function() {
this.addInt32(0);
var vtableloc = this.offset();

// Trim trailing zeroes.
var i = this.vtable_in_use - 1;
for (; i >= 0 && this.vtable[i] == 0; i--) {}
var trimmed_size = i + 1;

// Write out the current vtable.
for (var i = this.vtable_in_use - 1; i >= 0; i--) {
for (; i >= 0; i--) {
// Offset relative to the start of the table.
this.addInt16(this.vtable[i] != 0 ? vtableloc - this.vtable[i] : 0);
}

var standard_fields = 2; // The fields below:
this.addInt16(vtableloc - this.object_start);
this.addInt16((this.vtable_in_use + standard_fields) * flatbuffers.SIZEOF_SHORT);
var len = (trimmed_size + standard_fields) * flatbuffers.SIZEOF_SHORT;
this.addInt16(len);

// Search for an existing vtable that matches the current one.
var existing_vtable = 0;
var vt1 = this.space;
outer_loop:
for (var i = 0; i < this.vtables.length; i++) {
var vt1 = this.bb.capacity() - this.vtables[i];
var vt2 = this.space;
var len = this.bb.readInt16(vt1);
for (i = 0; i < this.vtables.length; i++) {
var vt2 = this.bb.capacity() - this.vtables[i];
if (len == this.bb.readInt16(vt2)) {
for (var j = flatbuffers.SIZEOF_SHORT; j < len; j += flatbuffers.SIZEOF_SHORT) {
if (this.bb.readInt16(vt1 + j) != this.bb.readInt16(vt2 + j)) {
10 changes: 7 additions & 3 deletions net/FlatBuffers/FlatBufferBuilder.cs
Original file line number Diff line number Diff line change
@@ -500,7 +500,11 @@ public int EndObject()
AddInt((int)0);
var vtableloc = Offset;
// Write out the current vtable.
for (int i = _vtableSize - 1; i >= 0 ; i--) {
int i = _vtableSize - 1;
// Trim trailing zeroes.
for (; i >= 0 && _vtable[i] == 0; i--) {}
int trimmedSize = i + 1;
for (; i >= 0 ; i--) {
// Offset relative to the start of the table.
short off = (short)(_vtable[i] != 0
? vtableloc - _vtable[i]
@@ -513,12 +517,12 @@ public int EndObject()

const int standardFields = 2; // The fields below:
AddShort((short)(vtableloc - _objectStart));
AddShort((short)((_vtableSize + standardFields) *
AddShort((short)((trimmedSize + standardFields) *
sizeof(short)));

// Search for an existing vtable that matches the current one.
int existingVtable = 0;
for (int i = 0; i < _numVtables; i++) {
for (i = 0; i < _numVtables; i++) {
int vt1 = _bb.Length - _vtables[i];
int vt2 = _space;
short len = _bb.GetShort(vt1);
10 changes: 7 additions & 3 deletions php/FlatbufferBuilder.php
Original file line number Diff line number Diff line change
@@ -596,7 +596,7 @@ protected function is_utf8($bytes)
if (function_exists('mb_detect_encoding')) {
return (bool) mb_detect_encoding($bytes, 'UTF-8', true);
}

$len = strlen($bytes);
if ($len < 1) {
/* NOTE: always return 1 when passed string is null */
@@ -812,14 +812,18 @@ public function endObject()
$this->addInt(0);
$vtableloc = $this->offset();

for ($i = $this->vtable_in_use -1; $i >= 0; $i--) {
$i = $this->vtable_in_use -1;
// Trim trailing zeroes.
for (; $i >= 0 && $this->vtable[$i] == 0; $i--) {}
$trimmed_size = $i + 1;
for (; $i >= 0; $i--) {
$off = ($this->vtable[$i] != 0) ? $vtableloc - $this->vtable[$i] : 0;
$this->addShort($off);
}

$standard_fields = 2; // the fields below
$this->addShort($vtableloc - $this->object_start);
$this->addShort(($this->vtable_in_use + $standard_fields) * Constants::SIZEOF_SHORT);
$this->addShort(($trimmed_size + $standard_fields) * Constants::SIZEOF_SHORT);

// search for an existing vtable that matches the current one.
$existing_vtable = 0;
4 changes: 4 additions & 0 deletions python/flatbuffers/builder.py
Original file line number Diff line number Diff line change
@@ -193,6 +193,10 @@ def WriteVtable(self):
objectOffset = self.Offset()
existingVtable = None

# Trim trailing 0 offsets.
while self.current_vtable and self.current_vtable[-1] == 0:
self.current_vtable.pop()

# Search backwards through existing vtables, because similar vtables
# are likely to have been recently appended. See
# BenchmarkVtableDeduplication for a case in which this heuristic
4 changes: 2 additions & 2 deletions samples/monster_generated.h
Original file line number Diff line number Diff line change
@@ -316,7 +316,7 @@ struct MonsterBuilder {
}
MonsterBuilder &operator=(const MonsterBuilder &);
flatbuffers::Offset<Monster> Finish() {
const auto end = fbb_.EndTable(start_, 10);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Monster>(end);
return o;
}
@@ -426,7 +426,7 @@ struct WeaponBuilder {
}
WeaponBuilder &operator=(const WeaponBuilder &);
flatbuffers::Offset<Weapon> Finish() {
const auto end = fbb_.EndTable(start_, 2);
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Weapon>(end);
return o;
}
3 changes: 1 addition & 2 deletions src/idl_gen_cpp.cpp
Original file line number Diff line number Diff line change
@@ -1531,9 +1531,8 @@ class CppGenerator : public BaseGenerator {
"(const {{STRUCT_NAME}}Builder &);";

// Finish() function.
auto num_fields = NumToString(struct_def.fields.vec.size());
code_ += " flatbuffers::Offset<{{STRUCT_NAME}}> Finish() {";
code_ += " const auto end = fbb_.EndTable(start_, " + num_fields + ");";
code_ += " const auto end = fbb_.EndTable(start_);";
code_ += " auto o = flatbuffers::Offset<{{STRUCT_NAME}}>(end);";

for (auto it = struct_def.fields.vec.begin();
3 changes: 1 addition & 2 deletions src/idl_parser.cpp
Original file line number Diff line number Diff line change
@@ -1071,8 +1071,7 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
builder_.PopBytes(struct_def.bytesize);
assert(!ovalue);
} else {
auto val = builder_.EndTable(start,
static_cast<voffset_t>(struct_def.fields.vec.size()));
auto val = builder_.EndTable(start);
if (ovalue) *ovalue = val;
if (value) *value = NumToString(val);
}
2 changes: 1 addition & 1 deletion src/reflection.cpp
Original file line number Diff line number Diff line change
@@ -481,7 +481,7 @@ Offset<const Table *> CopyTable(FlatBufferBuilder &fbb,
fbb.ClearOffsets();
return fbb.EndStruct();
} else {
return fbb.EndTable(start, static_cast<voffset_t>(fielddefs->size()));
return fbb.EndTable(start);
}
}

Loading

0 comments on commit ac1015e

Please sign in to comment.