BanyanDB: fix storage and query Tag autocomplete data (apache#12432)

renovamens · Jul 11, 2024 · 641163c · 641163c
1 parent 90d5006
commit 641163c
Show file tree

Hide file tree

Showing 12 changed files with 158 additions and 18 deletions.
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
@@ -30,6 +30,7 @@
 * Support tracing topology query for debugging.
 * Fix expression of graph `Current QPS` in MySQL dashboard.
 * Support tracing logs query for debugging.
+* BanyanDB: fix Tag autocomplete data storage and query.
 
 #### UI
 * Highlight search log keywords.

diff --git a/.../server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/TimeBucket.java b/.../server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/TimeBucket.java
@@ -163,4 +163,33 @@ public static long getTimeBucket(long timestamp, DownSampling downsampling) {
                 throw new UnexpectedException("Unknown downsampling value.");
         }
     }
+
+    /**
+     * Retain the minute time bucket to day time bucket. The format of timeBucket in minute format is "yyyyMMddHHmm", `HHmm` will be set to `0000`.
+     * minuteTimeBucket `202407112218` will be converted to `202407110000`.
+     * @param minuteTimeBucket minuteTimeBucket
+     * @return minute time bucket in day precision
+     */
+    public static long retainToDay4MinuteBucket(long minuteTimeBucket) {
+        if (isMinuteBucket(minuteTimeBucket)) {
+            return minuteTimeBucket / 10000 * 10000;
+        } else {
+            throw new UnexpectedException("Current time bucket is not a minute time bucket");
+        }
+    }
+
+    /**
+     * Retain the minute time bucket to day time bucket. The format of timeBucket in minute format is "yyyyMMddHHmm", `HHmm` will be set to `2359`.
+     * minuteTimeBucket `202407112218` will be converted to `202407112359`.
+     * @param minuteTimeBucket minuteTimeBucket
+     * @return minute time bucket in day precision
+     */
+    public static long retainToDayLastMin4MinuteBucket(long minuteTimeBucket) {
+        if (isMinuteBucket(minuteTimeBucket)) {
+            return minuteTimeBucket / 10000 * 10000 + 2359;
+        } else {
+            throw new UnexpectedException("Current time bucket is not a minute time bucket");
+        }
+    }
 }
+
diff --git a/...pache/skywalking/oap/server/core/analysis/manual/searchtag/TagAutocompleteDispatcher.java b/...pache/skywalking/oap/server/core/analysis/manual/searchtag/TagAutocompleteDispatcher.java
@@ -19,6 +19,7 @@
 package org.apache.skywalking.oap.server.core.analysis.manual.searchtag;
 
 import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher;
+import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
 import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;
 import org.apache.skywalking.oap.server.core.source.TagAutocomplete;
 
@@ -30,7 +31,8 @@ public void dispatch(TagAutocomplete source) {
         autocomplete.setTagKey(source.getTagKey());
         autocomplete.setTagValue(source.getTagValue());
         autocomplete.setTagType(source.getTagType().name());
-        autocomplete.setTimeBucket(source.getTimeBucket());
+        // change the precision in Day for reduce the storage
+        autocomplete.setTimeBucket(TimeBucket.retainToDay4MinuteBucket(source.getTimeBucket()));
         MetricsStreamProcessor.getInstance().in(autocomplete);
     }
 }
diff --git a/.../server-core/src/main/java/org/apache/skywalking/oap/server/core/query/DurationUtils.java b/.../server-core/src/main/java/org/apache/skywalking/oap/server/core/query/DurationUtils.java
@@ -85,6 +85,36 @@ public long endTimeDurationToSecondTimeBucket(Step step, String dateStr) {
         throw new UnexpectedException("Unsupported step " + step.name());
     }
 
+    public long startTimeDurationToMinuteTimeBucket(Step step, String dateStr) {
+        long minTimeBucket = convertToTimeBucket(step, dateStr);
+        switch (step) {
+            case DAY:
+                return minTimeBucket * 100 * 100;
+            case HOUR:
+                return minTimeBucket * 100;
+            case MINUTE:
+                return minTimeBucket;
+            case SECOND:
+                return minTimeBucket / 100;
+        }
+        throw new UnexpectedException("Unsupported step " + step.name());
+    }
+
+    public long endTimeDurationToMinuteTimeBucket(Step step, String dateStr) {
+        long minTimeBucket = convertToTimeBucket(step, dateStr);
+        switch (step) {
+            case DAY:
+                return (minTimeBucket * 100 + 23) * 100 + 59;
+            case HOUR:
+                return minTimeBucket * 100 + 59;
+            case MINUTE:
+                return minTimeBucket;
+            case SECOND:
+                return minTimeBucket / 100;
+        }
+        throw new UnexpectedException("Unsupported step " + step.name());
+    }
+
     public List<PointOfTime> getDurationPoints(Step step, long startTimeBucket, long endTimeBucket) {
         DateTime dateTime = parseToDateTime(step, startTimeBucket);
 

diff --git a/...server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/Duration.java b/...server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/Duration.java
@@ -65,6 +65,14 @@ public long getEndTimeBucketInSec() {
         return DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(step, end);
     }
 
+    public long getStartTimeBucketInMin() {
+        return DurationUtils.INSTANCE.startTimeDurationToMinuteTimeBucket(step, start);
+    }
+
+    public long getEndTimeBucketInMin() {
+        return DurationUtils.INSTANCE.endTimeDurationToMinuteTimeBucket(step, end);
+    }
+
     /**
      * Assemble time point based on {@link #step} and {@link #start} / {@link #end}
      */

diff --git a/...ver-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/TimeBucketTest.java b/...ver-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/TimeBucketTest.java
@@ -19,6 +19,7 @@
 package org.apache.skywalking.oap.server.core.analysis;
 
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
 
@@ -64,4 +65,14 @@ public void testConversion(DownSampling downSampling) {
         }
         Assertions.assertEquals(instance.getTimeInMillis(), timestamp);
     }
+
+    @Test
+    public void testRetainToDay4MinuteBucket() {
+        Assertions.assertEquals(202407110000L, TimeBucket.retainToDay4MinuteBucket(202407112218L));
+    }
+
+    @Test
+    public void testRetainToDayLastMin4MinuteBucket() {
+        Assertions.assertEquals(202407112359L, TimeBucket.retainToDayLastMin4MinuteBucket(202407112218L));
+    }
 }
diff --git a/...r/server-core/src/test/java/org/apache/skywalking/oap/server/core/query/DurationTest.java b/...r/server-core/src/test/java/org/apache/skywalking/oap/server/core/query/DurationTest.java
@@ -91,4 +91,40 @@ public void testGetDurationPoints() {
         Assertions.assertTrue(Arrays.asList(20220910L, 20220911L, 20220912L)
                                 .equals(pointOfTimes.stream().map(PointOfTime::getPoint).collect(Collectors.toList())));
     }
+
+    @Test
+    public void testStartTimeDurationToMinuteTimeBucket() {
+        Assertions.assertEquals(
+            202209080000L, DurationUtils.INSTANCE.startTimeDurationToMinuteTimeBucket(Step.DAY, "2022-09-08"));
+        Assertions.assertEquals(
+            202209081000L, DurationUtils.INSTANCE.startTimeDurationToMinuteTimeBucket(Step.HOUR, "2022-09-08 10"));
+        Assertions.assertEquals(
+            202209081010L, DurationUtils.INSTANCE.startTimeDurationToMinuteTimeBucket(Step.MINUTE, "2022-09-08 1010"));
+        Assertions.assertEquals(
+            202209081010L, DurationUtils.INSTANCE.startTimeDurationToMinuteTimeBucket(Step.SECOND, "2022-09-08 101010"));
+        try {
+            DurationUtils.INSTANCE.startTimeDurationToMinuteTimeBucket(Step.HOUR, "2022-09-08 30");
+            Assertions.fail("Should throw IllegalArgumentException");
+        } catch (IllegalArgumentException e) {
+            Assertions.assertTrue(true);
+        }
+    }
+
+    @Test
+    public void testEndTimeDurationToMinuteTimeBucket() {
+        Assertions.assertEquals(
+            202209082359L, DurationUtils.INSTANCE.endTimeDurationToMinuteTimeBucket(Step.DAY, "2022-09-08"));
+        Assertions.assertEquals(
+            202209081059L, DurationUtils.INSTANCE.endTimeDurationToMinuteTimeBucket(Step.HOUR, "2022-09-08 10"));
+        Assertions.assertEquals(
+            202209081010L, DurationUtils.INSTANCE.endTimeDurationToMinuteTimeBucket(Step.MINUTE, "2022-09-08 1010"));
+        Assertions.assertEquals(
+            202209081010L, DurationUtils.INSTANCE.endTimeDurationToMinuteTimeBucket(Step.SECOND, "2022-09-08 101010"));
+        try {
+            DurationUtils.INSTANCE.endTimeDurationToMinuteTimeBucket(Step.HOUR, "2022-09-08 30");
+            Assertions.fail("Should throw IllegalArgumentException");
+        } catch (IllegalArgumentException e) {
+            Assertions.assertTrue(true);
+        }
+    }
 }
diff --git a/...kywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTagAutocompleteQueryDAO.java b/...kywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTagAutocompleteQueryDAO.java
@@ -54,12 +54,16 @@ public BanyanDBTagAutocompleteQueryDAO(BanyanDBStorageClient client) {
     @Override
     public Set<String> queryTagAutocompleteKeys(TagType tagType, int limit, Duration duration) throws IOException {
         MetadataRegistry.Schema schema = MetadataRegistry.INSTANCE.findMetadata(TagAutocompleteData.INDEX_NAME, DownSampling.Minute);
-        long startTB = 0;
-        long endTB = 0;
+        long startMinTB = 0;
+        long endMinTB = 0;
         if (nonNull(duration)) {
-            startTB = TimeBucket.getMinuteTimeBucket(duration.getStartTimestamp());
-            endTB = TimeBucket.getMinuteTimeBucket(duration.getEndTimestamp());
+            startMinTB = duration.getStartTimeBucketInMin();
+            endMinTB = duration.getEndTimeBucketInMin();
         }
+
+        long startTB = TimeBucket.retainToDay4MinuteBucket(startMinTB);
+        long endTB = TimeBucket.retainToDayLastMin4MinuteBucket(endMinTB);
+
         TimestampRange range = null;
         if (startTB > 0 && endTB > 0) {
             range = new TimestampRange(TimeBucket.getTimestamp(startTB), TimeBucket.getTimestamp(endTB));
@@ -91,15 +95,15 @@ protected void apply(MeasureQuery query) {
     @Override
     public Set<String> queryTagAutocompleteValues(TagType tagType, String tagKey, int limit, Duration duration) throws IOException {
         MetadataRegistry.Schema schema = MetadataRegistry.INSTANCE.findMetadata(TagAutocompleteData.INDEX_NAME, DownSampling.Minute);
-        long startSecondTB = 0;
-        long endSecondTB = 0;
+        long startMinTB = 0;
+        long endMinTB = 0;
         if (nonNull(duration)) {
-            startSecondTB = duration.getStartTimeBucketInSec();
-            endSecondTB = duration.getEndTimeBucketInSec();
+            startMinTB = duration.getStartTimeBucketInMin();
+            endMinTB = duration.getEndTimeBucketInMin();
         }
 
-        long startTB = startSecondTB / 1000000 * 10000;
-        long endTB = endSecondTB / 1000000 * 10000 + 2359;
+        long startTB = TimeBucket.retainToDay4MinuteBucket(startMinTB);
+        long endTB = TimeBucket.retainToDayLastMin4MinuteBucket(endMinTB);
 
         TimestampRange range = null;
         if (startTB > 0 && endTB > 0) {

diff --git a/...che/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAO.java b/...che/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAO.java
@@ -20,6 +20,7 @@
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
+import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
 import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagAutocompleteData;
 import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType;
 import org.apache.skywalking.oap.server.core.query.input.Duration;
@@ -125,15 +126,15 @@ private void appendTagAutocompleteCondition(final TagType tagType,
                                                 final Duration duration,
                                                 final StringBuilder sql,
                                                 final List<Object> condition) {
-        long startSecondTB = 0;
-        long endSecondTB = 0;
+        long startMinTB = 0;
+        long endMinTB = 0;
         if (nonNull(duration)) {
-            startSecondTB = duration.getStartTimeBucketInSec();
-            endSecondTB = duration.getEndTimeBucketInSec();
+            startMinTB = duration.getStartTimeBucketInMin();
+            endMinTB = duration.getEndTimeBucketInMin();
         }
 
-        long startTB = startSecondTB / 1000000 * 10000;
-        long endTB = endSecondTB / 1000000 * 10000 + 2359;
+        long startTB = TimeBucket.retainToDay4MinuteBucket(startMinTB);
+        long endTB = TimeBucket.retainToDayLastMin4MinuteBucket(endMinTB);
 
         sql.append(" and ");
         sql.append(TagAutocompleteData.TAG_TYPE).append(" = ?");

diff --git a/test/e2e-v2/cases/zipkin/expected/autocomplete-keys.yml b/test/e2e-v2/cases/zipkin/expected/autocomplete-keys.yml
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+["http.method"]
diff --git a/...v2/cases/zipkin/expected/autocomplete.yml → ...es/zipkin/expected/autocomplete-value.yml b/...v2/cases/zipkin/expected/autocomplete.yml → ...es/zipkin/expected/autocomplete-value.yml
diff --git a/test/e2e-v2/cases/zipkin/zipkin-cases.yaml b/test/e2e-v2/cases/zipkin/zipkin-cases.yaml
@@ -27,5 +27,7 @@ cases:
   - query: curl http://${oap_host}:${oap_9412}/zipkin/api/v2/traces?serviceName=frontend&remoteServiceName=backend&spanName=get&annotationQuery=wr&limit=1
     expected: expected/traces.yml
   # autocomplete
+  - query: curl http://${oap_host}:${oap_9412}/zipkin/api/v2/autocompleteKeys
+    expected: expected/autocomplete-keys.yml
   - query: curl http://${oap_host}:${oap_9412}/zipkin/api/v2/autocompleteValues?key=http.method
-    expected: expected/autocomplete.yml
+    expected: expected/autocomplete-value.yml