bump backend limit

GKLeong · Dec 9, 2021 · d52241d · d52241d
1 parent 47b8981
commit d52241d
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 10 deletions.
diff --git a/lib/backend/defaults.go b/lib/backend/defaults.go
@@ -30,7 +30,7 @@ const (
 	DefaultPollStreamPeriod = time.Second
 	// DefaultEventsTTL is a default events TTL period
 	DefaultEventsTTL = 10 * time.Minute
-	// DefaultLargeLimit is used to specify some very large limit when limit is not specified
-	// explicitly to prevent OOM
-	DefaultLargeLimit = 30000
+	// DefaultRangeLimit is used to specify some very large limit when limit is not specified
+	// explicitly to prevent OOM due to infinite loops or other issues along those lines.
+	DefaultRangeLimit = 200_000
 )
diff --git a/lib/backend/dynamo/dynamodbbk.go b/lib/backend/dynamo/dynamodbbk.go
@@ -385,13 +385,16 @@ func (b *Backend) getAllRecords(ctx context.Context, startKey []byte, endKey []b
 	var result getResult
 	// this code is being extra careful here not to introduce endless loop
 	// by some unfortunate series of events
-	for i := 0; i < backend.DefaultLargeLimit/100; i++ {
+	for i := 0; i < backend.DefaultRangeLimit/100; i++ {
 		re, err := b.getRecords(ctx, prependPrefix(startKey), prependPrefix(endKey), limit, result.lastEvaluatedKey)
 		if err != nil {
 			return nil, trace.Wrap(err)
 		}
 		result.records = append(result.records, re.records...)
 		if len(result.records) >= limit || len(re.lastEvaluatedKey) == 0 {
+			if len(result.records) == backend.DefaultRangeLimit {
+				b.Warnf("Range query hit backend limit. (this is a bug!) startKey=%q,limit=%d", startKey, backend.DefaultRangeLimit)
+			}
 			result.lastEvaluatedKey = nil
 			return &result, nil
 		}
@@ -411,7 +414,7 @@ func (b *Backend) DeleteRange(ctx context.Context, startKey, endKey []byte) erro
 	// keep fetching and deleting until no records left,
 	// keep the very large limit, just in case if someone else
 	// keeps adding records
-	for i := 0; i < backend.DefaultLargeLimit/100; i++ {
+	for i := 0; i < backend.DefaultRangeLimit/100; i++ {
 		result, err := b.getRecords(ctx, prependPrefix(startKey), prependPrefix(endKey), 100, nil)
 		if err != nil {
 			return trace.Wrap(err)

diff --git a/lib/backend/firestore/firestorebk.go b/lib/backend/firestore/firestorebk.go
@@ -357,7 +357,7 @@ func (b *Backend) getRangeDocs(ctx context.Context, startKey []byte, endKey []by
 		return nil, trace.BadParameter("missing parameter endKey")
 	}
 	if limit <= 0 {
-		limit = backend.DefaultLargeLimit
+		limit = backend.DefaultRangeLimit
 	}
 	docs, err := b.svc.Collection(b.CollectionName).
 		Where(keyDocProperty, ">=", startKey).
@@ -375,7 +375,12 @@ func (b *Backend) getRangeDocs(ctx context.Context, startKey []byte, endKey []by
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
-	return append(docs, legacyDocs...), nil
+
+	allDocs := append(docs, legacyDocs...)
+	if len(allDocs) >= backend.DefaultRangeLimit {
+		b.Warnf("Range query hit backend limit. (this is a bug!) startKey=%q,limit=%d", startKey, backend.DefaultRangeLimit)
+	}
+	return allDocs, nil
 }
 
 // GetRange returns range of elements
@@ -406,7 +411,7 @@ func (b *Backend) GetRange(ctx context.Context, startKey []byte, endKey []byte,
 
 // DeleteRange deletes range of items with keys between startKey and endKey
 func (b *Backend) DeleteRange(ctx context.Context, startKey, endKey []byte) error {
-	docSnaps, err := b.getRangeDocs(ctx, startKey, endKey, backend.DefaultLargeLimit)
+	docSnaps, err := b.getRangeDocs(ctx, startKey, endKey, backend.DefaultRangeLimit)
 	if err != nil {
 		return trace.Wrap(err)
 	}

diff --git a/lib/backend/lite/lite.go b/lib/backend/lite/lite.go
@@ -610,7 +610,7 @@ func (l *Backend) GetRange(ctx context.Context, startKey []byte, endKey []byte,
 		return nil, trace.BadParameter("missing parameter endKey")
 	}
 	if limit <= 0 {
-		limit = backend.DefaultLargeLimit
+		limit = backend.DefaultRangeLimit
 	}
 
 	// When in mirror mode, don't set the current time so the SELECT query
@@ -647,6 +647,9 @@ func (l *Backend) GetRange(ctx context.Context, startKey []byte, endKey []byte,
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
+	if len(result.Items) == backend.DefaultRangeLimit {
+		l.Warnf("Range query hit backend limit. (this is a bug!) startKey=%q,limit=%d", startKey, backend.DefaultRangeLimit)
+	}
 	return &result, nil
 }
 

diff --git a/lib/backend/memory/memory.go b/lib/backend/memory/memory.go
@@ -324,12 +324,15 @@ func (m *Memory) GetRange(ctx context.Context, startKey []byte, endKey []byte, l
 		return nil, trace.BadParameter("missing parameter endKey")
 	}
 	if limit <= 0 {
-		limit = backend.DefaultLargeLimit
+		limit = backend.DefaultRangeLimit
 	}
 	m.Lock()
 	defer m.Unlock()
 	m.removeExpired()
 	re := m.getRange(ctx, startKey, endKey, limit)
+	if len(re.Items) == backend.DefaultRangeLimit {
+		m.Warnf("Range query hit backend limit. (this is a bug!) startKey=%q,limit=%d", startKey, backend.DefaultRangeLimit)
+	}
 	return &re, nil
 }