trace: add buffer limit

This applies to all span data entities that are slices: Annotations,
Links, and Message Events.

At the limit, it pops the oldest sample off prior to appending the new
sample. This should not cause an additional allocation.

Each entity shares the same WithBufferLimit option following the YAGNI
principle.

Each test ensures 4 entities are added in total -- the first 2 are
dropped and the last 2 remain. This should prevent off by 1 bugs.

trace/export.go

@@ -73,4 +73,11 @@ type SpanData struct {
 	Status
 	Links           []Link
 	HasRemoteParent bool
+
+	// The count of all span.bufferLimit overflows.
+	// See trace.WithBufferLimit for more info.
+	// TODO: This is not currently exported anywhere (e.g. tracez page).
+	DroppedAnnotations   int
+	DroppedMessageEvents int
+	DroppedLinks         int
 }

trace/trace.go

@@ -46,6 +46,7 @@ type Span struct {
 	endOnce sync.Once
 
 	executionTracerTaskEnd func() // ends the execution tracer span
+	bufferLimit            int    // limits all variable span data (slices)
 }
 
 // IsRecordingEvents returns true if events are being recorded for this span.
@@ -110,6 +111,9 @@ const (
 	SpanKindClient
 )
 
+// DefaultBufferLimit is the default value for trace.StartOptions.BufferLimit.
+const DefaultBufferLimit = 1000
+
 // StartOptions contains options concerning how a span is started.
 type StartOptions struct {
 	// Sampler to consult for this Span. If provided, it is always consulted.
@@ -125,6 +129,11 @@ type StartOptions struct {
 	// SpanKind represents the kind of a span. If none is set,
 	// SpanKindUnspecified is used.
 	SpanKind int
+
+	// BufferLimit makes new spans with a custom buffer limit.
+	// This limits variable span data: Message Events, Links, and Annotations.
+	// The default is 1000 (trace.DefaultBufferLimit) and the minimum is 1.
+	BufferLimit int
 }
 
 // StartOption apply changes to StartOptions.
@@ -145,13 +154,25 @@ func WithSampler(sampler Sampler) StartOption {
 	}
 }
 
+// WithBufferLimit makes new spans with a custom buffer limit.
+// This limits variable trace span data: Message Events, Links, and Annotations.
+// The default is 1000 (trace.DefaultBufferLimit) and the minimum is 1.
+func WithBufferLimit(limit int) StartOption {
+	return func(o *StartOptions) {
+		o.BufferLimit = limit
+		if o.BufferLimit <= 0 {
+			o.BufferLimit = 1
+		}
+	}
+}
+
 // StartSpan starts a new child span of the current span in the context. If
 // there is no span in the context, creates a new trace and span.
 //
 // Returned context contains the newly created span. You can use it to
 // propagate the returned span in process.
 func StartSpan(ctx context.Context, name string, o ...StartOption) (context.Context, *Span) {
-	var opts StartOptions
+	opts := StartOptions{BufferLimit: DefaultBufferLimit}
 	var parent SpanContext
 	if p := FromContext(ctx); p != nil {
 		parent = p.spanContext
@@ -174,7 +195,7 @@ func StartSpan(ctx context.Context, name string, o ...StartOption) (context.Cont
 // Returned context contains the newly created span. You can use it to
 // propagate the returned span in process.
 func StartSpanWithRemoteParent(ctx context.Context, name string, parent SpanContext, o ...StartOption) (context.Context, *Span) {
-	var opts StartOptions
+	opts := StartOptions{BufferLimit: DefaultBufferLimit}
 	for _, op := range o {
 		op(&opts)
 	}
@@ -185,7 +206,7 @@ func StartSpanWithRemoteParent(ctx context.Context, name string, parent SpanCont
 }
 
 func startSpanInternal(name string, hasParent bool, parent SpanContext, remoteParent bool, o StartOptions) *Span {
-	span := &Span{}
+	span := &Span{bufferLimit: o.BufferLimit}
 	span.spanContext = parent
 
 	cfg := config.Load().(*Config)
@@ -340,6 +361,10 @@ func (s *Span) lazyPrintfInternal(attributes []Attribute, format string, a ...in
 		m = make(map[string]interface{})
 		copyAttributes(m, attributes)
 	}
+	if l := len(s.data.Annotations); l > 0 && l >= s.bufferLimit {
+		s.data.Annotations = s.data.Annotations[1:]
+		s.data.DroppedAnnotations++
+	}
 	s.data.Annotations = append(s.data.Annotations, Annotation{
 		Time:       now,
 		Message:    msg,
@@ -356,6 +381,10 @@ func (s *Span) printStringInternal(attributes []Attribute, str string) {
 		a = make(map[string]interface{})
 		copyAttributes(a, attributes)
 	}
+	if l := len(s.data.Annotations); l > 0 && l >= s.bufferLimit {
+		s.data.Annotations = s.data.Annotations[1:]
+		s.data.DroppedAnnotations++
+	}
 	s.data.Annotations = append(s.data.Annotations, Annotation{
 		Time:       now,
 		Message:    str,
@@ -393,6 +422,10 @@ func (s *Span) AddMessageSendEvent(messageID, uncompressedByteSize, compressedBy
 	}
 	now := time.Now()
 	s.mu.Lock()
+	if l := len(s.data.MessageEvents); l > 0 && l >= s.bufferLimit {
+		s.data.MessageEvents = s.data.MessageEvents[1:]
+		s.data.DroppedMessageEvents++
+	}
 	s.data.MessageEvents = append(s.data.MessageEvents, MessageEvent{
 		Time:                 now,
 		EventType:            MessageEventTypeSent,
@@ -415,6 +448,10 @@ func (s *Span) AddMessageReceiveEvent(messageID, uncompressedByteSize, compresse
 	}
 	now := time.Now()
 	s.mu.Lock()
+	if l := len(s.data.MessageEvents); l > 0 && l >= s.bufferLimit {
+		s.data.MessageEvents = s.data.MessageEvents[1:]
+		s.data.DroppedMessageEvents++
+	}
 	s.data.MessageEvents = append(s.data.MessageEvents, MessageEvent{
 		Time:                 now,
 		EventType:            MessageEventTypeRecv,
@@ -431,6 +468,10 @@ func (s *Span) AddLink(l Link) {
 		return
 	}
 	s.mu.Lock()
+	if l := len(s.data.Links); l > 0 && l >= s.bufferLimit {
+		s.data.Links = s.data.Links[1:]
+		s.data.DroppedLinks++
+	}
 	s.data.Links = append(s.data.Links, l)
 	s.mu.Unlock()
 }

trace/trace_test.go

@@ -224,6 +224,10 @@ func TestStartSpanWithRemoteParent(t *testing.T) {
 
 // startSpan returns a context with a new Span that is recording events and will be exported.
 func startSpan(o StartOptions) *Span {
+	bufferLimit := DefaultBufferLimit
+	if o.BufferLimit != 0 {
+		bufferLimit = o.BufferLimit
+	}
 	_, span := StartSpanWithRemoteParent(context.Background(), "span0",
 		SpanContext{
 			TraceID:      tid,
@@ -232,6 +236,7 @@ func startSpan(o StartOptions) *Span {
 		},
 		WithSampler(o.Sampler),
 		WithSpanKind(o.SpanKind),
+		WithBufferLimit(bufferLimit),
 	)
 	return span
 }
@@ -379,7 +384,9 @@ func TestSetSpanAttributes(t *testing.T) {
 }
 
 func TestAnnotations(t *testing.T) {
-	span := startSpan(StartOptions{})
+	span := startSpan(StartOptions{BufferLimit: 2})
+	span.Annotatef([]Attribute{StringAttribute("dropped1", "dropped1")}, "%f", 123.123)
+	span.Annotate([]Attribute{StringAttribute("dropped2", "dropped2")}, "dropped2")
 	span.Annotatef([]Attribute{StringAttribute("key1", "value1")}, "%f", 1.5)
 	span.Annotate([]Attribute{StringAttribute("key2", "value2")}, "Annotate")
 	got, err := endSpan(span)
@@ -405,15 +412,19 @@ func TestAnnotations(t *testing.T) {
 			{Message: "1.500000", Attributes: map[string]interface{}{"key1": "value1"}},
 			{Message: "Annotate", Attributes: map[string]interface{}{"key2": "value2"}},
 		},
-		HasRemoteParent: true,
+		HasRemoteParent:    true,
+		DroppedAnnotations: 2,
 	}
 	if !reflect.DeepEqual(got, want) {
 		t.Errorf("exporting span: got %#v want %#v", got, want)
 	}
 }
 
 func TestMessageEvents(t *testing.T) {
-	span := startSpan(StartOptions{})
+	span := startSpan(StartOptions{BufferLimit: 2})
+	const dropped1, dropped2 = 123, 456
+	span.AddMessageReceiveEvent(dropped1, dropped1, dropped1)
+	span.AddMessageSendEvent(dropped2, dropped2, dropped2)
 	span.AddMessageReceiveEvent(3, 400, 300)
 	span.AddMessageSendEvent(1, 200, 100)
 	got, err := endSpan(span)
@@ -439,7 +450,8 @@ func TestMessageEvents(t *testing.T) {
 			{EventType: 2, MessageID: 0x3, UncompressedByteSize: 0x190, CompressedByteSize: 0x12c},
 			{EventType: 1, MessageID: 0x1, UncompressedByteSize: 0xc8, CompressedByteSize: 0x64},
 		},
-		HasRemoteParent: true,
+		HasRemoteParent:      true,
+		DroppedMessageEvents: 2,
 	}
 	if !reflect.DeepEqual(got, want) {
 		t.Errorf("exporting span: got %#v want %#v", got, want)
@@ -522,13 +534,31 @@ func TestSetSpanStatus(t *testing.T) {
 }
 
 func TestAddLink(t *testing.T) {
-	span := startSpan(StartOptions{})
+	span := startSpan(StartOptions{BufferLimit: 2})
+	span.AddLink(Link{
+		TraceID:    tid,
+		SpanID:     sid,
+		Type:       LinkTypeParent,
+		Attributes: map[string]interface{}{"dropped1": "dropped1"},
+	})
+	span.AddLink(Link{
+		TraceID:    tid,
+		SpanID:     sid,
+		Type:       LinkTypeParent,
+		Attributes: map[string]interface{}{"dropped2": "dropped2"},
+	})
 	span.AddLink(Link{
 		TraceID:    tid,
 		SpanID:     sid,
 		Type:       LinkTypeParent,
 		Attributes: map[string]interface{}{"key5": "value5"},
 	})
+	span.AddLink(Link{
+		TraceID:    tid,
+		SpanID:     sid,
+		Type:       LinkTypeParent,
+		Attributes: map[string]interface{}{"key6": "value6"},
+	})
 	got, err := endSpan(span)
 	if err != nil {
 		t.Fatal(err)
@@ -542,13 +572,22 @@ func TestAddLink(t *testing.T) {
 		},
 		ParentSpanID: sid,
 		Name:         "span0",
-		Links: []Link{{
-			TraceID:    tid,
-			SpanID:     sid,
-			Type:       2,
-			Attributes: map[string]interface{}{"key5": "value5"},
-		}},
+		Links: []Link{
+			{
+				TraceID:    tid,
+				SpanID:     sid,
+				Type:       2,
+				Attributes: map[string]interface{}{"key5": "value5"},
+			},
+			{
+				TraceID:    tid,
+				SpanID:     sid,
+				Type:       2,
+				Attributes: map[string]interface{}{"key6": "value6"},
+			},
+		},
 		HasRemoteParent: true,
+		DroppedLinks:    2,
 	}
 	if !reflect.DeepEqual(got, want) {
 		t.Errorf("exporting span: got %#v want %#v", got, want)