Skip to content

Commit

Permalink
* MQE query: make metadata not return null. (apache#12142)
Browse files Browse the repository at this point in the history 
* MQE labeled metrics Binary Operation: return empty value if the labels not match rather than report error.
  • Loading branch information
@wankai123
wankai123 authored Apr 22, 2024
1 parent d6fc848 commit 5808627
Show file tree
Hide file tree
Showing 12 changed files with 43 additions and 122 deletions.
3 changes: 2 additions & 1 deletion docs/en/api/metrics-query-expression.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,8 @@ For the result type of the expression, please refer to the following table.
### Binary Operation Rules
The following table lists if the different result types of the input expressions could do this operation and the result type after the operation.
The expression could be on the left or right side of the operator.
**Note**: If the expressions on both sides of the operator are the `TIME_SERIES_VALUES with labels`, they should have the same labels for calculation.
**Note**: If the expressions result on both sides of the operator are `with labels`, they should have the same labels for calculation.
If the labels match, will reserve left expression result labels and the calculated value. Otherwise, will return empty value.

| Expression | Expression | Yes/No | ExpressionResultType |
|-------------------------|---------------------------|--------|--------------------------|
Expand Down
2 changes: 2 additions & 0 deletions docs/en/changes/changes.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,8 @@
* Add Service Hierarchy auto matching layer relationships (upper -> lower) as following:
- ACTIVEMQ -> K8S_SERVICE
* Calculate Nginx service HTTP Latency by MQE.
* MQE query: make metadata not return `null`.
* MQE labeled metrics Binary Operation: return empty value if the labels not match rather than report error.

#### UI

Expand Down
10 changes: 1 addition & 9 deletions oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/MQEVisitorBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,17 +149,14 @@ public ExpressionResult visitAggregateLabelsOp(final MQEParser.AggregateLabelsOp
MQEParser.LabelNameListContext labelNameListContext = ctx.aggregateLabelsFunc().labelNameList();
if (null != labelNameListContext) {
for (MQEParser.LabelNameContext labelNameContext : labelNameListContext.labelName()) {
// ignore the label name that does not exist in the result
if (expResult.getResults()
.get(0)
.getMetric()
.getLabels()
.stream()
.anyMatch(label -> label.getKey().equals(labelNameContext.getText()))) {
labelNames.add(labelNameContext.getText());
} else {
expResult.setError(
"The label [" + labelNameContext.getText() + "] does not exist in the result.");
return expResult;
}
}
}
Expand Down Expand Up @@ -221,11 +218,6 @@ public ExpressionResult visitRelablesOP(MQEParser.RelablesOPContext ctx) {
}
KeyValue targetLabel = buildLabel(ctx.label());
KeyValue replaceLabel = buildLabel(ctx.replaceLabel().label());

// if (!targetLabel.getKey().equals(replaceLabel.getKey())) {
// result.setError("The target label name and replace label name must be the same.");
// return result;
// }
List<KeyValue> targetLabels = parseLabelValue(targetLabel, Const.COMMA);
List<KeyValue> replaceLabels = parseLabelValue(replaceLabel, Const.COMMA);

Expand Down
9 changes: 2 additions & 7 deletions ...server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/operation/AggregateLabelsOp.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import org.apache.skywalking.mqe.rt.operation.aggregatelabels.MaxAggregateLabelsFunc;
import org.apache.skywalking.mqe.rt.operation.aggregatelabels.MinAggregateLabelsFunc;
import org.apache.skywalking.mqe.rt.operation.aggregatelabels.SumAggregateLabelsFunc;
import org.apache.skywalking.mqe.rt.type.Metadata;
import org.apache.skywalking.oap.server.core.query.type.KeyValue;
import org.apache.skywalking.oap.server.library.util.CollectionUtils;

Expand Down Expand Up @@ -98,20 +97,16 @@ private static ExpressionResult aggregateLabeledValueResult(ExpressionResult exp
}
}
MQEValues mqeValues = new MQEValues();
Metadata metadata = new Metadata();
metadata.setLabels(labels);
mqeValues.setMetric(metadata);
mqeValues.getMetric().setLabels(labels);
mqeValues.setValues(combineTo);
expResult.getResults().add(mqeValues);
});
return expResult;
}

private static List<KeyValue> getLabels(final List<String> labelNames, final MQEValues mqeValues) {
List<KeyValue> a =
labelNames.stream().map(labelName -> mqeValues.getMetric().getLabels().stream().filter(label -> labelName.equals(label.getKey()))
return labelNames.stream().map(labelName -> mqeValues.getMetric().getLabels().stream().filter(label -> labelName.equals(label.getKey()))
.findAny().orElseGet(() -> new KeyValue(labelName, ""))).collect(toList());
return a;
}

}
51 changes: 17 additions & 34 deletions oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/operation/LROp.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,39 +107,31 @@ private static ExpressionResult single2SingleNoLabeled(ExpressionResult singleLe
} else {
double value = calculate.apply(left.getDoubleValue(), right.getDoubleValue(), opType);
mqeValue.setDoubleValue(value);
mqeValue.setEmptyValue(false);
}
return result;
}

private static ExpressionResult single2SingleLabeled(ExpressionResult singleLeft,
ExpressionResult singleRight,
int opType, LROp calculate) throws IllegalExpressionException {
Map<KeyValue, List<MQEValue>> labelMapR = new HashMap<>();
if (singleLeft.getResults().size() != singleRight.getResults().size()) {
throw new IllegalExpressionException(
"Operation between labeled metrics should have the same label.");
}
Map<Set<KeyValue>, List<MQEValue>> labelMapR = new HashMap<>();
singleRight.getResults().forEach(mqeValuesR -> {
// For now, we only have a single anonymous label named `_`
labelMapR.put(mqeValuesR.getMetric().getLabels().get(0), mqeValuesR.getValues());
labelMapR.put(new HashSet<>(mqeValuesR.getMetric().getLabels()), mqeValuesR.getValues());
});
for (MQEValues mqeValuesL : singleLeft.getResults()) {
//reserve left metric info
MQEValue valueL = mqeValuesL.getValues().get(0);
List<MQEValue> mqeValuesR = labelMapR.get(mqeValuesL.getMetric().getLabels().get(0));
List<MQEValue> mqeValuesR = labelMapR.get(new HashSet<>(mqeValuesL.getMetric().getLabels()));
if (mqeValuesR == null) {
throw new IllegalExpressionException(
"Operation between labeled metrics should have the same label.");
}
MQEValue valueR = mqeValuesR.get(0);
if (valueL.isEmptyValue() || valueR.isEmptyValue()) {
valueL.setEmptyValue(true);
valueL.setDoubleValue(0);
} else {
MQEValue valueR = mqeValuesR.get(0);
if (valueL.isEmptyValue() || valueR.isEmptyValue()) {
valueL.setEmptyValue(true);
continue;
}
double value = calculate.apply(valueL.getDoubleValue(), valueR.getDoubleValue(), opType);
valueL.setDoubleValue(value);
valueL.setEmptyValue(false);
}
}

Expand Down Expand Up @@ -193,14 +185,12 @@ private static ExpressionResult seriesNoLabeled(ExpressionResult seriesLeft,
int opType, LROp calculate) {
MQEValues mqeValuesL = seriesLeft.getResults().get(0);
MQEValues mqeValuesR = seriesRight.getResults().get(0);
mqeValuesL.setMetric(null);
for (int i = 0; i < mqeValuesL.getValues().size(); i++) {
//clean metric info
MQEValue valueL = mqeValuesL.getValues().get(i);
MQEValue valueR = mqeValuesR.getValues().get(i);
if (valueL.isEmptyValue() || valueR.isEmptyValue()) {
valueL.setEmptyValue(true);
valueL.setDoubleValue(0);
continue;
}
//time should be mapped
Expand All @@ -222,7 +212,6 @@ private static ExpressionResult seriesLabeledWithNoLabeled(ExpressionResult seri
MQEValue valueR = mqeValuesR.getValues().get(i);
if (valueL.isEmptyValue() || valueR.isEmptyValue()) {
valueL.setEmptyValue(true);
valueL.setDoubleValue(0);
continue;
}
double newValue = calculate.apply(valueL.getDoubleValue(), valueR.getDoubleValue(), opType);
Expand All @@ -244,7 +233,6 @@ private static ExpressionResult seriesNoLabeledWithLabeled(ExpressionResult seri
MQEValue valueR = mqeValuesR.getValues().get(i);
if (valueL.isEmptyValue() || valueR.isEmptyValue()) {
valueL.setEmptyValue(true);
valueL.setDoubleValue(0);
continue;
}
double newValue = calculate.apply(valueL.getDoubleValue(), valueR.getDoubleValue(), opType);
Expand All @@ -259,30 +247,25 @@ private static ExpressionResult seriesLabeledWithLabeled(ExpressionResult series
ExpressionResult seriesRight,
int opType, LROp calculate) throws IllegalExpressionException {
Map<Set<KeyValue>, List<MQEValue>> labelMapR = new HashMap<>();
if (seriesLeft.getResults().size() != seriesRight.getResults().size()) {
throw new IllegalExpressionException(
"Operation between labeled metrics should have the same label.");
}
seriesRight.getResults().forEach(mqeValuesR -> {
labelMapR.put(new HashSet<>(mqeValuesR.getMetric().getLabels()), mqeValuesR.getValues());
});
for (MQEValues mqeValuesL : seriesLeft.getResults()) {
for (int i = 0; i < mqeValuesL.getValues().size(); i++) {
//reserve left metric info
//reserve left metric info, if right metric not exist, set empty value
MQEValue valueL = mqeValuesL.getValues().get(i);
List<MQEValue> mqeValuesR = labelMapR.get(new HashSet<>(mqeValuesL.getMetric().getLabels()));
if (mqeValuesR == null) {
throw new IllegalExpressionException(
"Operation between labeled metrics should have the same label.");
}
MQEValue valueR = mqeValuesR.get(i);
if (valueL.isEmptyValue() || valueR.isEmptyValue()) {
valueL.setEmptyValue(true);
valueL.setDoubleValue(0);
continue;
} else {
MQEValue valueR = mqeValuesR.get(i);
if (valueL.isEmptyValue() || valueR.isEmptyValue()) {
valueL.setEmptyValue(true);
continue;
}
double newValue = calculate.apply(valueL.getDoubleValue(), valueR.getDoubleValue(), opType);
mqeValuesL.getValues().get(i).setDoubleValue(newValue);
}
double newValue = calculate.apply(valueL.getDoubleValue(), valueR.getDoubleValue(), opType);
mqeValuesL.getValues().get(i).setDoubleValue(newValue);
}
}

Expand Down
2 changes: 0 additions & 2 deletions ...server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/operation/LogicalFunctionOp.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import org.apache.skywalking.mqe.rt.type.ExpressionResultType;
import org.apache.skywalking.mqe.rt.type.MQEValue;
import org.apache.skywalking.mqe.rt.type.MQEValues;
import org.apache.skywalking.mqe.rt.type.Metadata;
import org.apache.skywalking.oap.server.library.util.CollectionUtils;

import java.util.Objects;
Expand Down Expand Up @@ -85,7 +84,6 @@ private static ExpressionResult isPresent(MQEParser.ExpressionListContext expres

MQEValue mqeValue = new MQEValue();
MQEValues mqeValues = new MQEValues();
mqeValues.setMetric(new Metadata());
mqeValues.getValues().add(mqeValue);
result.getResults().add(mqeValues);
mqeValue.setDoubleValue(present ? 1 : 0);
Expand Down
2 changes: 1 addition & 1 deletion oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/type/MQEValues.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@

@Data
public class MQEValues {
private Metadata metric;
private Metadata metric = new Metadata();

private List<MQEValue> values = new ArrayList<>();
}
5 changes: 5 additions & 0 deletions oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/type/Metadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,16 @@
package org.apache.skywalking.mqe.rt.type;

import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import lombok.Data;
import org.apache.skywalking.oap.server.core.query.type.KeyValue;

@Data
public class Metadata {
private List<KeyValue> labels = new ArrayList<>();

public void sortLabelsByKey(Comparator<String> comparator) {
labels.sort(Comparator.comparing(KeyValue::getKey, comparator));
}
}
29 changes: 2 additions & 27 deletions ...a/org/apache/skywalking/oap/server/core/alarm/provider/expr/rt/AlarmMQEVerifyVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,7 @@

package org.apache.skywalking.oap.server.core.alarm.provider.expr.rt;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import lombok.Getter;
Expand All @@ -31,14 +29,9 @@
import org.apache.skywalking.mqe.rt.type.ExpressionResultType;
import org.apache.skywalking.mqe.rt.type.MQEValue;
import org.apache.skywalking.mqe.rt.type.MQEValues;
import org.apache.skywalking.mqe.rt.type.Metadata;
import org.apache.skywalking.oap.server.core.query.enumeration.Step;
import org.apache.skywalking.oap.server.core.query.type.KeyValue;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
import org.apache.skywalking.oap.server.library.util.CollectionUtils;

import static org.apache.skywalking.oap.server.core.analysis.metrics.DataLabel.GENERAL_LABEL_NAME;

/**
* Used for verify the alarm expression and get the metrics name when read the alarm rules.
Expand Down Expand Up @@ -78,29 +71,11 @@ public ExpressionResult visitMetric(MQEParser.MetricContext ctx) {
MQEValue mqeValue = new MQEValue();
mqeValue.setEmptyValue(true);
mockMqeValues.getValues().add(mqeValue);
result.getResults().add(mockMqeValues);
result.setType(ExpressionResultType.TIME_SERIES_VALUES);
if (dataType == Column.ValueDataType.COMMON_VALUE) {
result.getResults().add(mockMqeValues);
result.setType(ExpressionResultType.TIME_SERIES_VALUES);
return result;
} else if (dataType == Column.ValueDataType.LABELED_VALUE) {
List<KeyValue> queryLabels = buildLabels(ctx.labelList());
ArrayList<MQEValues> mqeValuesList = new ArrayList<>();
if (CollectionUtils.isEmpty(queryLabels)) {
KeyValue label = new KeyValue(GENERAL_LABEL_NAME, GENERAL_LABEL_NAME);
Metadata metadata = new Metadata();
metadata.getLabels().add(label);
mockMqeValues.setMetric(metadata);
mqeValuesList.add(mockMqeValues);
} else {
for (KeyValue queryLabel : queryLabels) {
Metadata metadata = new Metadata();
metadata.getLabels().add(queryLabel);
mockMqeValues.setMetric(metadata);
mqeValuesList.add(mockMqeValues);
}
}
result.setType(ExpressionResultType.TIME_SERIES_VALUES);
result.setResults(mqeValuesList);
result.setLabeledResult(true);
return result;
} else {
Expand Down
34 changes: 5 additions & 29 deletions ...in/java/org/apache/skywalking/oap/server/core/alarm/provider/expr/rt/AlarmMQEVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.apache.skywalking.oap.server.core.alarm.provider.expr.rt;

import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
Expand All @@ -30,7 +31,6 @@
import org.apache.skywalking.mqe.rt.type.ExpressionResultType;
import org.apache.skywalking.mqe.rt.type.MQEValue;
import org.apache.skywalking.mqe.rt.type.MQEValues;
import org.apache.skywalking.mqe.rt.type.Metadata;
import org.apache.skywalking.oap.server.core.analysis.metrics.DataLabel;
import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
import org.apache.skywalking.oap.server.core.analysis.metrics.DoubleValueHolder;
Expand All @@ -46,7 +46,6 @@
import org.apache.skywalking.oap.server.library.util.CollectionUtils;
import org.joda.time.LocalDateTime;

import static org.apache.skywalking.oap.server.core.analysis.metrics.DataLabel.GENERAL_LABEL_NAME;
import static org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO.Util.composeLabelConditions;

@Slf4j
Expand Down Expand Up @@ -99,7 +98,7 @@ public ExpressionResult visitMetric(MQEParser.MetricContext ctx) {
List<KeyValue> queryLabels = buildLabels(ctx.labelList());
Map<String, DataTable> timeValues = labeledValuesMap.get(metricName);
if (CollectionUtils.isEmpty(timeValues)) {
mqeValuesList = buildEmptyLabeledMQEValuesList(queryLabels);
mqeValuesList = buildEmptyMQEValuesList();
} else {
mqeValuesList = buildLabledMqeValuesList(timeValues, queryLabels);
}
Expand Down Expand Up @@ -191,7 +190,6 @@ private List<MQEValues> buildMqeValuesList(Map<String, Double> timeValues) {
List<MQEValues> mqeValuesList = new ArrayList<>();
MQEValues mqeValues = new MQEValues();
for (String time : windowTimes) {
Metadata metadata = new Metadata();
Double metricValue = timeValues.get(time);
MQEValue mqeValue = new MQEValue();
//use timeBucket as id here
Expand All @@ -201,7 +199,6 @@ private List<MQEValues> buildMqeValuesList(Map<String, Double> timeValues) {
} else {
mqeValue.setEmptyValue(true);
}
mqeValues.setMetric(metadata);
mqeValues.getValues().add(mqeValue);
}
mqeValuesList.add(mqeValues);
Expand All @@ -215,7 +212,6 @@ private List<MQEValues> buildLabledMqeValuesList(Map<String, DataTable> timeValu
for (String labelCondition : labelConditions) {
MQEValues mqeValues = new MQEValues();
for (String time : windowTimes) {
Metadata metadata = new Metadata();
DataTable dataTable = timeValues.getOrDefault(time, new DataTable());
Long metricValue = dataTable.get(labelCondition);
MQEValue mqeValue = new MQEValue();
Expand All @@ -229,11 +225,12 @@ private List<MQEValues> buildLabledMqeValuesList(Map<String, DataTable> timeValu
DataLabel dataLabel = new DataLabel();
dataLabel.put(labelCondition);
for (Map.Entry<String, String> label : dataLabel.entrySet()) {
metadata.getLabels().add(new KeyValue(label.getKey(), label.getValue()));
mqeValues.getMetric().getLabels().add(new KeyValue(label.getKey(), label.getValue()));
}
mqeValues.setMetric(metadata);
mqeValues.getValues().add(mqeValue);
}
//Sort labels by key in natural order by default
mqeValues.getMetric().sortLabelsByKey(Comparator.naturalOrder());
mqeValuesList.add(mqeValues);
}
return mqeValuesList;
Expand All @@ -256,25 +253,4 @@ private ArrayList<MQEValues> buildEmptyMQEValuesList() {
mqeValuesList.add(initMQEValues());
return mqeValuesList;
}

private ArrayList<MQEValues> buildEmptyLabeledMQEValuesList(List<KeyValue> queryLabels) {
ArrayList<MQEValues> mqeValuesList = new ArrayList<>();
if (CollectionUtils.isEmpty(queryLabels)) {
MQEValues mqeValues = initMQEValues();
KeyValue label = new KeyValue(GENERAL_LABEL_NAME, GENERAL_LABEL_NAME);
Metadata metadata = new Metadata();
metadata.getLabels().add(label);
mqeValues.setMetric(metadata);
mqeValuesList.add(mqeValues);
} else {
for (KeyValue label : queryLabels) {
MQEValues mqeValues = initMQEValues();
Metadata metadata = new Metadata();
metadata.getLabels().add(label);
mqeValues.setMetric(metadata);
mqeValuesList.add(mqeValues);
}
}
return mqeValuesList;
}
}
Loading

0 comments on commit 5808627

Please sign in to comment.