forked from grafana/loki
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Improve metric queries by computing samples at the edges. (grafana#2293)
* First pass breaking the code appart. Wondering how we're going to achieve fast mutation of labels. Signed-off-by: Cyril Tovena <[email protected]> * Work in progress. I realize I need hash for deduping lines. going to benchmark somes. Signed-off-by: Cyril Tovena <[email protected]> * Tested some hash and decided which one to use. Signed-off-by: Cyril Tovena <[email protected]> * Wip Signed-off-by: Cyril Tovena <[email protected]> * Starting working on ingester. Signed-off-by: Cyril Tovena <[email protected]> * Trying to find a better hash function. Signed-off-by: Cyril Tovena <[email protected]> * More hash testing we have a winner. xxhash it is. Signed-off-by: Cyril Tovena <[email protected]> * Settle on xxhash Signed-off-by: Cyril Tovena <[email protected]> * Better params interfacing. Signed-off-by: Cyril Tovena <[email protected]> * Add interface for queryparams for things that exist in both type of params. Signed-off-by: Cyril Tovena <[email protected]> * Add storage sample iterator implementations. Signed-off-by: Cyril Tovena <[email protected]> * Fixing tests and verifying we don't get collions for the hashing method. Signed-off-by: Cyril Tovena <[email protected]> * Fixing ingesters tests and refactoring utility function/tests. Signed-off-by: Cyril Tovena <[email protected]> * Fixing and testing that stats are still well computed. Signed-off-by: Cyril Tovena <[email protected]> * Fixing more tests. Signed-off-by: Cyril Tovena <[email protected]> * More engine tests finished. Signed-off-by: Cyril Tovena <[email protected]> * Fixes sharding evaluator. Signed-off-by: Cyril Tovena <[email protected]> * Fixes more engine tests. Signed-off-by: Cyril Tovena <[email protected]> * Fix error tests in the engine. Signed-off-by: Cyril Tovena <[email protected]> * Finish fixing all tests. Signed-off-by: Cyril Tovena <[email protected]> * Fixes a bug where extractor was not passed in correctly. Signed-off-by: Cyril Tovena <[email protected]> * Add notes about upgrade. Signed-off-by: Cyril Tovena <[email protected]> * Renamed and fix a bug. Signed-off-by: Cyril Tovena <[email protected]> * Add memchunk tests and starting test for sampleIterator. Signed-off-by: Cyril Tovena <[email protected]> * Test heap sample iterator. Signed-off-by: Cyril Tovena <[email protected]> * working on test. Signed-off-by: Cyril Tovena <[email protected]> * Finishing testing all new iterators. Signed-off-by: Cyril Tovena <[email protected]> * Making sure all store functions are tested. Signed-off-by: Cyril Tovena <[email protected]> * Benchmark and verify everything is working well. Signed-off-by: Cyril Tovena <[email protected]> * Make the linter happy. Signed-off-by: Cyril Tovena <[email protected]> * use xxhash v2. Signed-off-by: Cyril Tovena <[email protected]> * Fix a flaky test because of map. Signed-off-by: Cyril Tovena <[email protected]> * go.mod. Signed-off-by: Cyril Tovena <[email protected]> Co-authored-by: Edward Welch <[email protected]>
- Loading branch information
1 parent
0b59960
commit 0be64fc
Showing
67 changed files
with
5,266 additions
and
1,788 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| package chunkenc | ||
|
|
||
| import ( | ||
| "hash/fnv" | ||
| "hash/maphash" | ||
| "testing" | ||
|
|
||
| "github.com/cespare/xxhash/v2" | ||
| "github.com/segmentio/fasthash/fnv1a" | ||
| "github.com/stretchr/testify/require" | ||
|
|
||
| "github.com/grafana/loki/pkg/chunkenc/testdata" | ||
| ) | ||
|
|
||
| var res uint64 | ||
|
|
||
| func Benchmark_fnv64a(b *testing.B) { | ||
| for n := 0; n < b.N; n++ { | ||
| for i := 0; i < len(testdata.LogsBytes); i++ { | ||
| h := fnv.New64a() | ||
| _, _ = h.Write(testdata.LogsBytes[i]) | ||
| res = h.Sum64() | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func Benchmark_fnv64a_third_party(b *testing.B) { | ||
| for n := 0; n < b.N; n++ { | ||
| for i := 0; i < len(testdata.LogsBytes); i++ { | ||
| res = fnv1a.HashBytes64(testdata.LogsBytes[i]) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func Benchmark_xxhash(b *testing.B) { | ||
| for n := 0; n < b.N; n++ { | ||
| for i := 0; i < len(testdata.LogsBytes); i++ { | ||
| res = xxhash.Sum64(testdata.LogsBytes[i]) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func Benchmark_hashmap(b *testing.B) { | ||
| // I discarded hashmap/map as it will compute different value on different binary for the same entry | ||
| var h maphash.Hash | ||
| for n := 0; n < b.N; n++ { | ||
| for i := 0; i < len(testdata.LogsBytes); i++ { | ||
| h.SetSeed(maphash.MakeSeed()) | ||
| _, _ = h.Write(testdata.LogsBytes[i]) | ||
| res = h.Sum64() | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func Test_xxhash_integrity(t *testing.T) { | ||
| data := []uint64{} | ||
|
|
||
| for i := 0; i < len(testdata.LogsBytes); i++ { | ||
| data = append(data, xxhash.Sum64(testdata.LogsBytes[i])) | ||
| } | ||
|
|
||
| for i := 0; i < len(testdata.LogsBytes); i++ { | ||
| require.Equal(t, data[i], xxhash.Sum64(testdata.LogsBytes[i])) | ||
| } | ||
|
|
||
| unique := map[uint64]struct{}{} | ||
| for i := 0; i < len(testdata.LogsBytes); i++ { | ||
| _, ok := unique[xxhash.Sum64(testdata.LogsBytes[i])] | ||
| require.False(t, ok, string(testdata.LogsBytes[i])) // all lines have been made unique | ||
| unique[xxhash.Sum64(testdata.LogsBytes[i])] = struct{}{} | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.