diff --git a/sdk/cosmos/azure-cosmos-test/CHANGELOG.md b/sdk/cosmos/azure-cosmos-test/CHANGELOG.md index 243fcdcbc97a7..f1ca80da196ad 100644 --- a/sdk/cosmos/azure-cosmos-test/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos-test/CHANGELOG.md @@ -3,6 +3,7 @@ ### 1.0.0-beta.4 (Unreleased) #### Features Added +* Added fault injection support for Gateway connection mode - See [PR 35378](https://github.com/Azure/azure-sdk-for-java/pull/35378) #### Breaking Changes diff --git a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionCondition.java b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionCondition.java index d2e320088c8a8..ea5da4e64490f 100644 --- a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionCondition.java +++ b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionCondition.java @@ -3,6 +3,8 @@ package com.azure.cosmos.test.faultinjection; +import com.azure.cosmos.test.implementation.ImplementationBridgeHelpers; + /*** * Fault injection condition. * A fault injection rule will not be applicable if the condition mismatches. @@ -19,7 +21,7 @@ public final class FaultInjectionCondition { String region, FaultInjectionEndpoints endpoints) { this.operationType = operationType; - this.connectionType = connectionType; + this.connectionType = this.isMetadataOperationType() ? FaultInjectionConnectionType.GATEWAY : connectionType; this.region = region; this.endpoints = endpoints; } @@ -60,6 +62,18 @@ public String getRegion() { return this.region; } + boolean isMetadataOperationType() { + if (this.operationType == null) { + return false; + } + + return this.operationType == FaultInjectionOperationType.METADATA_REQUEST_PARTITION_KEY_RANGES + || this.operationType == FaultInjectionOperationType.METADATA_REQUEST_ADDRESS_REFRESH + || this.operationType == FaultInjectionOperationType.METADATA_REQUEST_CONTAINER + || this.operationType == FaultInjectionOperationType.METADATA_REQUEST_QUERY_PLAN + || this.operationType == FaultInjectionOperationType.METADATA_REQUEST_DATABASE_ACCOUNT; + } + @Override public String toString() { return String.format( @@ -69,4 +83,17 @@ public String toString() { this.connectionType, this.region); } + + /////////////////////////////////////////////////////////////////////////////////////////// + // the following helper/accessor only helps to access this class outside of this package.// + /////////////////////////////////////////////////////////////////////////////////////////// + static void initialize() { + ImplementationBridgeHelpers.FaultInjectionConditionHelper.setFaultInjectionConditionAccessor( + faultInjectionCondition -> faultInjectionCondition.isMetadataOperationType() + ); + } + + static { + initialize(); + } } diff --git a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionConnectionType.java b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionConnectionType.java index 03cbb0d332169..61ac4d66b28c8 100644 --- a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionConnectionType.java +++ b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionConnectionType.java @@ -10,5 +10,9 @@ public enum FaultInjectionConnectionType { /*** * Direct connection type. */ - DIRECT + DIRECT, + /*** + * Gateway connection type. + */ + GATEWAY } diff --git a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionOperationType.java b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionOperationType.java index 87c764d7a1aff..bf839945eb405 100644 --- a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionOperationType.java +++ b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionOperationType.java @@ -34,7 +34,25 @@ public enum FaultInjectionOperationType { /** * Patch item. */ - PATCH_ITEM - - // Add support for metadata request type + PATCH_ITEM, + /** + * Read container request. + */ + METADATA_REQUEST_CONTAINER, + /** + * Read database account request. + */ + METADATA_REQUEST_DATABASE_ACCOUNT, + /** + * Query query plan request. + */ + METADATA_REQUEST_QUERY_PLAN, + /** + * Partition key ranges request. + */ + METADATA_REQUEST_PARTITION_KEY_RANGES, + /** + * Address refresh request. + */ + METADATA_REQUEST_ADDRESS_REFRESH; } diff --git a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionRuleBuilder.java b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionRuleBuilder.java index e89a7aa862424..303d5ee933284 100644 --- a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionRuleBuilder.java +++ b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionRuleBuilder.java @@ -4,6 +4,7 @@ package com.azure.cosmos.test.faultinjection; import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; +import com.azure.cosmos.test.implementation.ImplementationBridgeHelpers; import java.time.Duration; @@ -117,10 +118,21 @@ public FaultInjectionRuleBuilder enabled(boolean enabled) { * Create a new fault injection rule. * * @return the {@link FaultInjectionRule}. + * @throws IllegalArgumentException if condition is null. + * @throws IllegalArgumentException if result is null. */ public FaultInjectionRule build() { - checkNotNull(this.condition, "Argument 'condition' can not be null"); - checkNotNull(this.result, "Argument 'result' can not be null"); + if (this.condition == null) { + throw new IllegalArgumentException("Argument 'condition' can not be null"); + } + + if (this.result == null) { + throw new IllegalArgumentException("Argument 'result' can not be null"); + } + + if (this.condition.getConnectionType() == FaultInjectionConnectionType.GATEWAY) { + this.validateRuleOnGatewayConnection(); + } return new FaultInjectionRule( this.id, @@ -131,4 +143,38 @@ public FaultInjectionRule build() { this.condition, this.result); } + + private void validateRuleOnGatewayConnection() { + if (this.result == null) { + throw new IllegalArgumentException("Argument 'result' can not be null"); + } + + if (this.result instanceof FaultInjectionConnectionErrorResult) { + throw new IllegalArgumentException("FaultInjectionConnectionError result can not be configured for rule with gateway connection type."); + } + + FaultInjectionServerErrorResult serverErrorResult = (FaultInjectionServerErrorResult) this.result; + + // Gateway service internally will retry for 410/0, so the eventual exceptions being returned to SDK is 503(SERVICE_UNAVAILABLE) instead + if (serverErrorResult.getServerErrorType() == FaultInjectionServerErrorType.GONE) { + throw new IllegalArgumentException("Gone exception can not be injected for rule with gateway connection type"); + } + + if (serverErrorResult.getServerErrorType() == FaultInjectionServerErrorType.STALED_ADDRESSES_SERVER_GONE) { + throw new IllegalArgumentException("STALED_ADDRESSES exception can not be injected for rule with gateway connection type"); + } + + // for metadata request related rule, only CONNECTION_DELAY, RESPONSE_DELAY, TOO_MANY_REQUEST error can be injected + if (ImplementationBridgeHelpers + .FaultInjectionConditionHelper + .getFaultInjectionConditionAccessor() + .isMetadataOperationType(this.condition)) { + if (serverErrorResult.getServerErrorType() != FaultInjectionServerErrorType.TOO_MANY_REQUEST + && serverErrorResult.getServerErrorType() != FaultInjectionServerErrorType.RESPONSE_DELAY + && serverErrorResult.getServerErrorType() != FaultInjectionServerErrorType.CONNECTION_DELAY) { + + throw new IllegalArgumentException("Error type " + serverErrorResult.getServerErrorType() + " is not supported for rule with metadata request"); + } + } + } } diff --git a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionServerErrorResultBuilder.java b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionServerErrorResultBuilder.java index 111c997094589..6f3a5c7f918dd 100644 --- a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionServerErrorResultBuilder.java +++ b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionServerErrorResultBuilder.java @@ -83,6 +83,12 @@ public FaultInjectionServerErrorResult build() { throw new IllegalArgumentException("Argument 'delay' is required for server error type " + this.serverErrorType); } + if (this.serverErrorType == FaultInjectionServerErrorType.STALED_ADDRESSES_SERVER_GONE) { + // for staled addresses errors, the error can only be cleared if forceRefresh address refresh request happened + // so default the times to max value + this.times = Integer.MAX_VALUE; + } + return new FaultInjectionServerErrorResult( this.serverErrorType, this.times, diff --git a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionServerErrorType.java b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionServerErrorType.java index 1c67553df27d4..c52ae512ae0a2 100644 --- a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionServerErrorType.java +++ b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionServerErrorType.java @@ -8,7 +8,7 @@ */ public enum FaultInjectionServerErrorType { - /** 410 from server */ + /** 410 from server. Only applicable for direct connection type. */ GONE, /** 449 from server */ @@ -36,5 +36,13 @@ public enum FaultInjectionServerErrorType { RESPONSE_DELAY, /** simulate high channel acquisition, when it is over connection timeout, can simulate connectionTimeoutException */ - CONNECTION_DELAY + CONNECTION_DELAY, + /** + * Simulate service unavailable(503) + */ + SERVICE_UNAVAILABLE, + /** + * simulate 410-0 due to staled addresses. The exception will only be cleared if a forceRefresh address refresh happened. + */ + STALED_ADDRESSES_SERVER_GONE } diff --git a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/ImplementationBridgeHelpers.java b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/ImplementationBridgeHelpers.java index c200b65f3e6f3..7507229774e9d 100644 --- a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/ImplementationBridgeHelpers.java +++ b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/ImplementationBridgeHelpers.java @@ -3,8 +3,9 @@ package com.azure.cosmos.test.implementation; -import com.azure.cosmos.test.implementation.faultinjection.IFaultInjectionRuleInternal; +import com.azure.cosmos.test.faultinjection.FaultInjectionCondition; import com.azure.cosmos.test.faultinjection.FaultInjectionRule; +import com.azure.cosmos.test.implementation.faultinjection.IFaultInjectionRuleInternal; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,4 +53,42 @@ public interface FaultInjectionRuleAccessor { void setEffectiveFaultInjectionRule(FaultInjectionRule rule, IFaultInjectionRuleInternal ruleInternal); } } + + public static final class FaultInjectionConditionHelper { + private static final AtomicBoolean faultInjectionConditionClassLoaded = new AtomicBoolean(false); + private static final AtomicReference accessor = new AtomicReference<>(); + + private FaultInjectionConditionHelper() { + } + + public static FaultInjectionConditionAccessor getFaultInjectionConditionAccessor() { + if (!faultInjectionConditionClassLoaded.get()) { + logger.debug("Initializing FaultInjectionConditionAccessor..."); + } + + FaultInjectionConditionAccessor snapshot = accessor.get(); + if (snapshot == null) { + logger.error("FaultInjectionConditionAccessor is not initialized yet!"); + System.exit(8701); // Using a unique status code here to help debug the issue. + } + + return snapshot; + } + + public static void setFaultInjectionConditionAccessor(final FaultInjectionConditionAccessor newAccessor) { + + assert (newAccessor != null); + + if (!accessor.compareAndSet(null, newAccessor)) { + logger.debug("FaultInjectionConditionAccessor already initialized!"); + } else { + logger.debug("Setting FaultInjectionConditionAccessor..."); + faultInjectionConditionClassLoaded.set(true); + } + } + + public interface FaultInjectionConditionAccessor { + boolean isMetadataOperationType(FaultInjectionCondition faultInjectionCondition); + } + } } diff --git a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionConditionInternal.java b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionConditionInternal.java index 81aa671e03eea..34eb612d4282c 100644 --- a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionConditionInternal.java +++ b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionConditionInternal.java @@ -4,8 +4,9 @@ package com.azure.cosmos.test.implementation.faultinjection; import com.azure.cosmos.implementation.OperationType; +import com.azure.cosmos.implementation.ResourceType; import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.implementation.directconnectivity.rntbd.RntbdRequestArgs; +import com.azure.cosmos.implementation.faultinjection.FaultInjectionRequestArgs; import java.net.URI; import java.util.ArrayList; @@ -37,6 +38,12 @@ public void setOperationType(OperationType operationType) { } } + public void setResourceType(ResourceType resourceType) { + if (resourceType != null) { + this.validators.add(new ResourceTypeValidator(resourceType)); + } + } + public void setRegionEndpoints(List regionEndpoints) { this.regionEndpoints = regionEndpoints; if (this.regionEndpoints != null) { @@ -63,7 +70,13 @@ public void setAddresses(List physicalAddresses, boolean primaryOnly) { } } - public boolean isApplicable(String ruleId, RntbdRequestArgs requestArgs) { + public void setPartitionKeyRangeIds(List partitionKeyRangeIds) { + if (partitionKeyRangeIds != null && partitionKeyRangeIds.size() > 0) { + this.validators.add(new PartitionKeyRangeIdValidator(partitionKeyRangeIds)); + } + } + + public boolean isApplicable(String ruleId, FaultInjectionRequestArgs requestArgs) { for (IFaultInjectionConditionValidator conditionValidator : this.validators) { if (!conditionValidator.isApplicable(ruleId, requestArgs)) { return false; @@ -75,7 +88,7 @@ public boolean isApplicable(String ruleId, RntbdRequestArgs requestArgs) { // region ConditionValidators interface IFaultInjectionConditionValidator { - boolean isApplicable(String ruleId, RntbdRequestArgs requestArgs); + boolean isApplicable(String ruleId, FaultInjectionRequestArgs requestArgs); } static class RegionEndpointValidator implements IFaultInjectionConditionValidator { @@ -84,16 +97,17 @@ static class RegionEndpointValidator implements IFaultInjectionConditionValidato this.regionEndpoints = regionEndpoints; } @Override - public boolean isApplicable(String ruleId, RntbdRequestArgs requestArgs) { - boolean isApplicable = this.regionEndpoints.contains(requestArgs.serviceRequest().faultInjectionRequestContext.getLocationEndpointToRoute()); + public boolean isApplicable(String ruleId, FaultInjectionRequestArgs requestArgs) { + boolean isApplicable = + this.regionEndpoints.contains(requestArgs.getServiceRequest().faultInjectionRequestContext.getLocationEndpointToRoute()); if (!isApplicable) { - requestArgs.serviceRequest().faultInjectionRequestContext - .recordFaultInjectionRuleEvaluation(requestArgs.transportRequestId(), + requestArgs.getServiceRequest().faultInjectionRequestContext + .recordFaultInjectionRuleEvaluation(requestArgs.getTransportRequestId(), String.format( "%s [RegionEndpoint mismatch: Expected [%s], Actual [%s]]", ruleId, this.regionEndpoints.stream().map(URI::toString).collect(Collectors.toList()), - requestArgs.serviceRequest().faultInjectionRequestContext.getLocationEndpointToRoute())); + requestArgs.getServiceRequest().faultInjectionRequestContext.getLocationEndpointToRoute())); } return isApplicable; @@ -107,16 +121,16 @@ static class OperationTypeValidator implements IFaultInjectionConditionValidator } @Override - public boolean isApplicable(String ruleId, RntbdRequestArgs requestArgs) { - boolean isApplicable = requestArgs.serviceRequest().getOperationType() == operationType; + public boolean isApplicable(String ruleId, FaultInjectionRequestArgs requestArgs) { + boolean isApplicable = requestArgs.getServiceRequest().getOperationType() == operationType; if (!isApplicable) { - requestArgs.serviceRequest().faultInjectionRequestContext - .recordFaultInjectionRuleEvaluation(requestArgs.transportRequestId(), + requestArgs.getServiceRequest().faultInjectionRequestContext + .recordFaultInjectionRuleEvaluation(requestArgs.getTransportRequestId(), String.format( "%s [OperationType mismatch: Expected [%s], Actual [%s]]", ruleId, operationType, - requestArgs.serviceRequest().getOperationType())); + requestArgs.getServiceRequest().getOperationType())); } return isApplicable; @@ -131,16 +145,17 @@ static class ContainerValidator implements IFaultInjectionConditionValidator { } @Override - public boolean isApplicable(String ruleId, RntbdRequestArgs requestArgs) { - boolean isApplicable = StringUtils.equals(this.containerResourceId, requestArgs.serviceRequest().requestContext.resolvedCollectionRid); + public boolean isApplicable(String ruleId, FaultInjectionRequestArgs requestArgs) { + boolean isApplicable = + StringUtils.equals(this.containerResourceId, requestArgs.getCollectionRid()); if (!isApplicable) { - requestArgs.serviceRequest().faultInjectionRequestContext - .recordFaultInjectionRuleEvaluation(requestArgs.transportRequestId(), + requestArgs.getServiceRequest().faultInjectionRequestContext + .recordFaultInjectionRuleEvaluation(requestArgs.getTransportRequestId(), String.format( "%s [ContainerRid mismatch: Expected [%s], Actual [%s]]", ruleId, containerResourceId, - requestArgs.serviceRequest().requestContext.resolvedCollectionRid)); + requestArgs.getCollectionRid())); } return isApplicable; } @@ -153,22 +168,22 @@ static class AddressValidator implements IFaultInjectionConditionValidator { } @Override - public boolean isApplicable(String ruleId, RntbdRequestArgs requestArgs) { + public boolean isApplicable(String ruleId, FaultInjectionRequestArgs requestArgs) { if (addresses != null && addresses.size() > 0) { boolean isApplicable = this.addresses .stream() - .anyMatch(address -> requestArgs.physicalAddressUri().getURIAsString().startsWith(address.toString())); + .anyMatch(address -> requestArgs.getRequestURI().toString().startsWith(address.toString())); if (!isApplicable) { - requestArgs.serviceRequest().faultInjectionRequestContext - .recordFaultInjectionRuleEvaluation(requestArgs.transportRequestId(), + requestArgs.getServiceRequest().faultInjectionRequestContext + .recordFaultInjectionRuleEvaluation(requestArgs.getTransportRequestId(), String.format( "%s [Addresses mismatch: Expected [%s], Actual [%s]]", ruleId, addresses, - requestArgs.physicalAddressUri().getURIAsString())); + requestArgs.getRequestURI().toString())); } return isApplicable; @@ -180,11 +195,11 @@ public boolean isApplicable(String ruleId, RntbdRequestArgs requestArgs) { static class PrimaryAddressValidator implements IFaultInjectionConditionValidator { @Override - public boolean isApplicable(String ruleId, RntbdRequestArgs requestArgs) { - boolean isApplicable = requestArgs.physicalAddressUri().isPrimary(); + public boolean isApplicable(String ruleId, FaultInjectionRequestArgs requestArgs) { + boolean isApplicable = requestArgs.isPrimary(); if (!isApplicable) { - requestArgs.serviceRequest().faultInjectionRequestContext - .recordFaultInjectionRuleEvaluation(requestArgs.transportRequestId(), + requestArgs.getServiceRequest().faultInjectionRequestContext + .recordFaultInjectionRuleEvaluation(requestArgs.getTransportRequestId(), String.format( "%s [NonPrimary addresses]", ruleId)); @@ -192,5 +207,58 @@ public boolean isApplicable(String ruleId, RntbdRequestArgs requestArgs) { return isApplicable; } } + + static class ResourceTypeValidator implements IFaultInjectionConditionValidator { + private ResourceType resourceType; + ResourceTypeValidator(ResourceType resourceType) { + this.resourceType = resourceType; + } + + @Override + public boolean isApplicable(String ruleId, FaultInjectionRequestArgs requestArgs) { + boolean isApplicable = + requestArgs.getServiceRequest().getResourceType() == this.resourceType + || (this.resourceType == ResourceType.Address && requestArgs.getServiceRequest().isAddressRefresh()); + + if (!isApplicable) { + requestArgs.getServiceRequest().faultInjectionRequestContext + .recordFaultInjectionRuleEvaluation(requestArgs.getTransportRequestId(), + String.format( + "%s [ResourceType mismatch: Expected [%s], Actual [%s], isAddressRefresh [%s]]", + ruleId, + resourceType, + requestArgs.getServiceRequest().getResourceType(), + requestArgs.getServiceRequest().isAddressRefresh())); + } + + return isApplicable; + } + } + + static class PartitionKeyRangeIdValidator implements IFaultInjectionConditionValidator { + private List partitionKeyRangeIdList; + PartitionKeyRangeIdValidator(List partitionKeyRangeIdList) { + this.partitionKeyRangeIdList = partitionKeyRangeIdList; + } + + @Override + public boolean isApplicable(String ruleId, FaultInjectionRequestArgs requestArgs) { + boolean isApplicable = requestArgs.getPartitionKeyRangeIds() != null + && !requestArgs.getPartitionKeyRangeIds().isEmpty() + && this.partitionKeyRangeIdList.containsAll(requestArgs.getPartitionKeyRangeIds()); + + if (!isApplicable) { + requestArgs.getServiceRequest().faultInjectionRequestContext + .recordFaultInjectionRuleEvaluation(requestArgs.getTransportRequestId(), + String.format( + "%s [PartitionKeyRangeId mismatch: Expected [%s], Actual [%s]]", + ruleId, + partitionKeyRangeIdList, + requestArgs.getPartitionKeyRangeIds())); + } + + return isApplicable; + } + } //endregion } diff --git a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionRuleProcessor.java b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionRuleProcessor.java index ab7debd26eb7b..9b41e8bdcd91c 100644 --- a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionRuleProcessor.java +++ b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionRuleProcessor.java @@ -122,6 +122,12 @@ private void validateRule(FaultInjectionRule rule) { && this.connectionMode != ConnectionMode.DIRECT) { throw new IllegalArgumentException("Direct connection type rule is not supported when client is not in direct mode."); } + + if (rule.getCondition().getOperationType() != null + && rule.getCondition().getOperationType() == FaultInjectionOperationType.METADATA_REQUEST_ADDRESS_REFRESH + && this.connectionMode != ConnectionMode.DIRECT) { + throw new IllegalArgumentException("METADATA_REQUEST_ADDRESS_REFRESH operation type is not supported when client is in gateway mode."); + } } private Mono getEffectiveRule( @@ -151,12 +157,12 @@ private Mono getEffectiveServerErrorRule( // get effective condition FaultInjectionConditionInternal effectiveCondition = new FaultInjectionConditionInternal(documentCollection.getResourceId()); - if (rule.getCondition().getOperationType() != null && canErrorLimitToOperation(errorType)) { + if ((rule.getCondition().getOperationType() != null && canErrorLimitToOperation(errorType))) { effectiveCondition.setOperationType(this.getEffectiveOperationType(rule.getCondition().getOperationType())); + effectiveCondition.setResourceType(this.getEffectiveResourceType(rule.getCondition().getOperationType())); } List regionEndpoints = this.getRegionEndpoints(rule.getCondition()); - if (StringUtils.isEmpty(rule.getCondition().getRegion())) { // if region is not specific configured, then also add the defaultEndpoint List regionEndpointsWithDefault = new ArrayList<>(regionEndpoints); @@ -166,7 +172,23 @@ private Mono getEffectiveServerErrorRule( effectiveCondition.setRegionEndpoints(regionEndpoints); } - // TODO: add handling for gateway mode + if (rule.getCondition().getConnectionType() == FaultInjectionConnectionType.GATEWAY) { + // for gateway mode, SDK does not decide which replica to send the request to + // so the most granular level it can control is by partition + if (canErrorLimitToOperation(errorType) && canRequestLimitToPartition(rule.getCondition())) { + return BackoffRetryUtility.executeRetry( + () -> this.resolvePartitionKeyRangeIds( + rule.getCondition().getEndpoints(), + documentCollection), + new FaultInjectionRuleProcessorRetryPolicy(this.retryOptions)) + .map(pkRangeIds -> { + effectiveCondition.setPartitionKeyRangeIds(pkRangeIds); + return effectiveCondition; + }); + } + + return Mono.just(effectiveCondition); + } // Direct connection mode, populate physical addresses boolean primaryAddressesOnly = this.isWriteOnly(rule.getCondition()); @@ -219,6 +241,20 @@ private boolean canErrorLimitToOperation(FaultInjectionServerErrorType errorType && errorType != FaultInjectionServerErrorType.GONE; } + private boolean canRequestLimitToPartition(FaultInjectionCondition faultInjectionCondition) { + // Some operations can be targeted for a certain partition while some can not (for example metadata requests) + if (faultInjectionCondition.getOperationType() == null + || faultInjectionCondition.getOperationType() == FaultInjectionOperationType.METADATA_REQUEST_ADDRESS_REFRESH) { + return true; + } + + // non metadata requests + return !ImplementationBridgeHelpers + .FaultInjectionConditionHelper + .getFaultInjectionConditionAccessor() + .isMetadataOperationType(faultInjectionCondition); + } + private Mono getEffectiveConnectionErrorRule( FaultInjectionRule rule, DocumentCollection documentCollection) { @@ -239,12 +275,17 @@ private Mono getEffectiveConnectionErrorRule( .collect(Collectors.toList()); FaultInjectionConnectionErrorResult result = (FaultInjectionConnectionErrorResult) rule.getResult(); + + List regionEndpointsWithDefault = new ArrayList<>(regionEndpoints); + // if region is not specific configured, then also add the defaultEndpoint + regionEndpointsWithDefault.add(this.globalEndpointManager.getDefaultEndpoint()); + return new FaultInjectionConnectionErrorRule( rule.getId(), rule.isEnabled(), rule.getStartDelay(), rule.getDuration(), - regionEndpoints, + regionEndpointsWithDefault, effectiveAddresses, rule.getCondition().getConnectionType(), result @@ -280,6 +321,9 @@ private OperationType getEffectiveOperationType(FaultInjectionOperationType faul switch (faultInjectionOperationType) { case READ_ITEM: + case METADATA_REQUEST_CONTAINER: + case METADATA_REQUEST_DATABASE_ACCOUNT: + case METADATA_REQUEST_ADDRESS_REFRESH: return OperationType.Read; case CREATE_ITEM: return OperationType.Create; @@ -293,11 +337,66 @@ private OperationType getEffectiveOperationType(FaultInjectionOperationType faul return OperationType.Delete; case PATCH_ITEM: return OperationType.Patch; + case METADATA_REQUEST_QUERY_PLAN: + return OperationType.QueryPlan; + case METADATA_REQUEST_PARTITION_KEY_RANGES: + return OperationType.ReadFeed; default: throw new IllegalStateException("FaultInjectionOperationType " + faultInjectionOperationType + " is not supported"); } } + private ResourceType getEffectiveResourceType(FaultInjectionOperationType faultInjectionOperationType) { + if (faultInjectionOperationType == null) { + return null; + } + + switch (faultInjectionOperationType) { + case READ_ITEM: + case CREATE_ITEM: + case QUERY_ITEM: + case UPSERT_ITEM: + case REPLACE_ITEM: + case DELETE_ITEM: + case PATCH_ITEM: + case METADATA_REQUEST_QUERY_PLAN: + return ResourceType.Document; + case METADATA_REQUEST_CONTAINER: + return ResourceType.DocumentCollection; + case METADATA_REQUEST_DATABASE_ACCOUNT: + return ResourceType.DatabaseAccount; + case METADATA_REQUEST_ADDRESS_REFRESH: + return ResourceType.Address; + case METADATA_REQUEST_PARTITION_KEY_RANGES: + return ResourceType.PartitionKeyRange; + default: + throw new IllegalStateException("FaultInjectionOperationType " + faultInjectionOperationType + " is not supported"); + } + } + + private Mono> resolvePartitionKeyRangeIds( + FaultInjectionEndpoints addressEndpoints, + DocumentCollection documentCollection) { + if (addressEndpoints == null) { + return Mono.just(Arrays.asList()); + } + + FeedRangeInternal feedRangeInternal = FeedRangeInternal.convert(addressEndpoints.getFeedRange()); + RxDocumentServiceRequest request = RxDocumentServiceRequest.create( + null, + OperationType.Read, + documentCollection.getResourceId(), + ResourceType.Document, + Collections.emptyMap()); + + return feedRangeInternal + .getPartitionKeyRanges( + this.partitionKeyRangeCache, + request, + Mono.just(new Utils.ValueHolder<>(documentCollection))); + } + + private Mono> resolvePhysicalAddresses( List regionEndpoints, FaultInjectionEndpoints addressEndpoints, diff --git a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionRuleStore.java b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionRuleStore.java index 557a05e2d5952..3d5b522868a2b 100644 --- a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionRuleStore.java +++ b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionRuleStore.java @@ -8,7 +8,8 @@ import com.azure.cosmos.implementation.AsyncDocumentClient; import com.azure.cosmos.implementation.ImplementationBridgeHelpers; import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.implementation.directconnectivity.rntbd.RntbdRequestArgs; +import com.azure.cosmos.implementation.faultinjection.FaultInjectionRequestArgs; +import com.azure.cosmos.implementation.faultinjection.RntbdFaultInjectionRequestArgs; import com.azure.cosmos.test.faultinjection.FaultInjectionConnectionType; import com.azure.cosmos.test.faultinjection.FaultInjectionRule; import reactor.core.publisher.Mono; @@ -18,6 +19,8 @@ import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkArgument; import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull; +import static com.azure.cosmos.test.faultinjection.FaultInjectionConnectionType.DIRECT; +import static com.azure.cosmos.test.faultinjection.FaultInjectionConnectionType.GATEWAY; public class FaultInjectionRuleStore { private final Set serverResponseDelayRuleSet = ConcurrentHashMap.newKeySet(); @@ -75,9 +78,12 @@ public Mono configureFaultInjectionRule(FaultInject }); } - public FaultInjectionServerErrorRule findRntbdServerResponseDelayRule(RntbdRequestArgs requestArgs) { + public FaultInjectionServerErrorRule findServerResponseDelayRule(FaultInjectionRequestArgs requestArgs) { + FaultInjectionConnectionType connectionType = + requestArgs instanceof RntbdFaultInjectionRequestArgs ? DIRECT : GATEWAY; + for (FaultInjectionServerErrorRule serverResponseDelayRule : this.serverResponseDelayRuleSet) { - if (serverResponseDelayRule.getConnectionType() == FaultInjectionConnectionType.DIRECT + if (serverResponseDelayRule.getConnectionType() == connectionType && serverResponseDelayRule.isApplicable(requestArgs)) { return serverResponseDelayRule; } @@ -86,9 +92,12 @@ public FaultInjectionServerErrorRule findRntbdServerResponseDelayRule(RntbdReque return null; } - public FaultInjectionServerErrorRule findRntbdServerResponseErrorRule(RntbdRequestArgs requestArgs) { + public FaultInjectionServerErrorRule findServerResponseErrorRule(FaultInjectionRequestArgs requestArgs) { + FaultInjectionConnectionType connectionType = + requestArgs instanceof RntbdFaultInjectionRequestArgs ? DIRECT : GATEWAY; + for (FaultInjectionServerErrorRule serverResponseDelayRule : this.serverResponseErrorRuleSet) { - if (serverResponseDelayRule.getConnectionType() == FaultInjectionConnectionType.DIRECT + if (serverResponseDelayRule.getConnectionType() == connectionType && serverResponseDelayRule.isApplicable(requestArgs)) { return serverResponseDelayRule; } @@ -97,9 +106,12 @@ public FaultInjectionServerErrorRule findRntbdServerResponseErrorRule(RntbdReque return null; } - public FaultInjectionServerErrorRule findRntbdServerConnectionDelayRule(RntbdRequestArgs requestArgs) { + public FaultInjectionServerErrorRule findServerConnectionDelayRule(FaultInjectionRequestArgs requestArgs) { + FaultInjectionConnectionType connectionType = + requestArgs instanceof RntbdFaultInjectionRequestArgs ? DIRECT : GATEWAY; + for (FaultInjectionServerErrorRule serverResponseDelayRule : this.serverConnectionDelayRuleSet) { - if (serverResponseDelayRule.getConnectionType() == FaultInjectionConnectionType.DIRECT + if (serverResponseDelayRule.getConnectionType() == connectionType && serverResponseDelayRule.isApplicable(requestArgs)) { return serverResponseDelayRule; } diff --git a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionServerErrorResultInternal.java b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionServerErrorResultInternal.java index 42c0b3bc4dd4c..aabf765ce0067 100644 --- a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionServerErrorResultInternal.java +++ b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionServerErrorResultInternal.java @@ -15,6 +15,7 @@ import com.azure.cosmos.implementation.RequestTimeoutException; import com.azure.cosmos.implementation.RetryWithException; import com.azure.cosmos.implementation.RxDocumentServiceRequest; +import com.azure.cosmos.implementation.ServiceUnavailableException; import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; import com.azure.cosmos.implementation.directconnectivity.WFConstants; import com.azure.cosmos.test.faultinjection.FaultInjectionServerErrorType; @@ -133,6 +134,19 @@ public CosmosException getInjectedServerError(RxDocumentServiceRequest request) cosmosException = new PartitionKeyRangeIsSplittingException(null, lsn, partitionKeyRangeId, responseHeaders); break; + case SERVICE_UNAVAILABLE: + responseHeaders.put(WFConstants.BackendHeaders.SUB_STATUS, + Integer.toString(HttpConstants.SubStatusCodes.SERVER_GENERATED_503)); + cosmosException = new ServiceUnavailableException(null, null, null, HttpConstants.SubStatusCodes.SERVER_GENERATED_503); + break; + + case STALED_ADDRESSES_SERVER_GONE: + GoneException staledAddressesException = + new GoneException(this.getErrorMessage(RMResources.Gone), HttpConstants.SubStatusCodes.SERVER_GENERATED_410); + staledAddressesException.setIsBasedOn410ResponseFromService(); + cosmosException = staledAddressesException; + break; + default: throw new IllegalArgumentException("Server error type " + this.serverErrorType + " is not supported"); } diff --git a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionServerErrorRule.java b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionServerErrorRule.java index 3c5afa34415ca..cc573f8b1b1b4 100644 --- a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionServerErrorRule.java +++ b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionServerErrorRule.java @@ -6,8 +6,9 @@ import com.azure.cosmos.CosmosException; import com.azure.cosmos.implementation.RxDocumentServiceRequest; import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.implementation.directconnectivity.rntbd.RntbdRequestArgs; +import com.azure.cosmos.implementation.faultinjection.FaultInjectionRequestArgs; import com.azure.cosmos.test.faultinjection.FaultInjectionConnectionType; +import com.azure.cosmos.test.faultinjection.FaultInjectionServerErrorType; import java.net.URI; import java.time.Duration; @@ -62,10 +63,10 @@ public FaultInjectionServerErrorRule( this.connectionType = connectionType; } - public boolean isApplicable(RntbdRequestArgs requestArgs) { + public boolean isApplicable(FaultInjectionRequestArgs requestArgs) { if (!this.isValid()) { - requestArgs.serviceRequest().faultInjectionRequestContext.recordFaultInjectionRuleEvaluation( - requestArgs.transportRequestId(), + requestArgs.getServiceRequest().faultInjectionRequestContext.recordFaultInjectionRuleEvaluation( + requestArgs.getTransportRequestId(), String.format( "%s[Disable or Duration reached. StartTime: %s, ExpireTime: %s]", this.id, @@ -81,20 +82,30 @@ public boolean isApplicable(RntbdRequestArgs requestArgs) { return false; } - if (!this.result.isApplicable(this.id, requestArgs.serviceRequest())) { - requestArgs.serviceRequest().faultInjectionRequestContext.recordFaultInjectionRuleEvaluation( - requestArgs.transportRequestId(), + if (!this.result.isApplicable(this.id, requestArgs.getServiceRequest())) { + requestArgs.getServiceRequest().faultInjectionRequestContext.recordFaultInjectionRuleEvaluation( + requestArgs.getTransportRequestId(), this.id + "[Per operation apply limit reached]" ); return false; } + if (this.result.getServerErrorType() == FaultInjectionServerErrorType.STALED_ADDRESSES_SERVER_GONE + && requestArgs.getServiceRequest().faultInjectionRequestContext.getAddressForceRefreshed()) { + requestArgs.getServiceRequest().faultInjectionRequestContext.recordFaultInjectionRuleEvaluation( + requestArgs.getTransportRequestId(), + "Address force refresh happened, STALED_ADDRESSES error is cleared." + ); + + return false; + } + long evaluationCount = this.evaluationCount.incrementAndGet(); boolean withinHitLimit = this.hitLimit == null || evaluationCount <= this.hitLimit; if (!withinHitLimit) { - requestArgs.serviceRequest().faultInjectionRequestContext.recordFaultInjectionRuleEvaluation( - requestArgs.transportRequestId(), - this.id + "[Hit Limit reached]" + requestArgs.getServiceRequest().faultInjectionRequestContext.recordFaultInjectionRuleEvaluation( + requestArgs.getTransportRequestId(), + String.format("%s [Hit Limit reached. Configured hitLimit %d, evaluationCount %d]", this.id, this.hitLimit, evaluationCount) ); return false; } else { @@ -102,7 +113,7 @@ public boolean isApplicable(RntbdRequestArgs requestArgs) { // track hit count details, key will be operationType-resourceType String name = - requestArgs.serviceRequest().getOperationType().toString() + "-" + requestArgs.serviceRequest().getResourceType().toString(); + requestArgs.getServiceRequest().getOperationType().toString() + "-" + requestArgs.getServiceRequest().getResourceType().toString(); this.hitCountDetails.compute(name, (key, count) -> { if (count == null) { count = 0L; diff --git a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectorProvider.java b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectorProvider.java index 67c49d2826034..c0ba1cc303391 100644 --- a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectorProvider.java +++ b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectorProvider.java @@ -8,7 +8,7 @@ import com.azure.cosmos.implementation.Utils; import com.azure.cosmos.implementation.directconnectivity.rntbd.RntbdEndpoint; import com.azure.cosmos.implementation.faultinjection.IFaultInjectorProvider; -import com.azure.cosmos.implementation.faultinjection.IRntbdServerErrorInjector; +import com.azure.cosmos.implementation.faultinjection.IServerErrorInjector; import com.azure.cosmos.test.faultinjection.FaultInjectionRule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,7 +25,7 @@ public class FaultInjectorProvider implements IFaultInjectorProvider { private final Logger logger = LoggerFactory.getLogger(FaultInjectorProvider.class); private final FaultInjectionRuleStore ruleStore; - private final RntbdServerErrorInjector serverErrorInjector; + private final IServerErrorInjector serverErrorInjector; private final String containerNameLink; private RntbdConnectionErrorInjector connectionErrorInjector; @@ -36,7 +36,7 @@ public FaultInjectorProvider(CosmosAsyncContainer cosmosAsyncContainer) { this.containerNameLink = Utils.trimBeginningAndEndingSlashes(BridgeInternal.extractContainerSelfLink(cosmosAsyncContainer)); this.ruleStore = new FaultInjectionRuleStore(cosmosAsyncContainer); - this.serverErrorInjector = new RntbdServerErrorInjector(this.ruleStore); + this.serverErrorInjector = new ServerErrorInjector(this.ruleStore); } public Mono configureFaultInjectionRules(List rules) { @@ -44,13 +44,15 @@ public Mono configureFaultInjectionRules(List rules) { .flatMap(rule -> this.ruleStore.configureFaultInjectionRule(rule, this.containerNameLink)) .doOnNext(effectiveRule -> { // Important step: this step will start the connection error injection task - this.connectionErrorInjector.accept(effectiveRule); + if (this.connectionErrorInjector != null) { + this.connectionErrorInjector.accept(effectiveRule); + } }) .then(); } @Override - public IRntbdServerErrorInjector getRntbdServerErrorInjector() { + public IServerErrorInjector getServerErrorInjector() { return this.serverErrorInjector; } diff --git a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/RntbdServerErrorInjector.java b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/RntbdServerErrorInjector.java deleted file mode 100644 index 0a78a205fa982..0000000000000 --- a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/RntbdServerErrorInjector.java +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.test.implementation.faultinjection; - -import com.azure.cosmos.CosmosException; -import com.azure.cosmos.implementation.directconnectivity.rntbd.IRequestRecord; -import com.azure.cosmos.implementation.directconnectivity.rntbd.RntbdRequestArgs; -import com.azure.cosmos.implementation.directconnectivity.rntbd.RntbdRequestRecord; -import com.azure.cosmos.implementation.faultinjection.IRntbdServerErrorInjector; - -import java.time.Duration; -import java.util.function.Consumer; - -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull; - -/*** - * Fault injector which can handle {@link FaultInjectionServerErrorRule} with DIRECT connection type. - */ -public class RntbdServerErrorInjector implements IRntbdServerErrorInjector { - private final FaultInjectionRuleStore ruleStore; - - public RntbdServerErrorInjector(FaultInjectionRuleStore ruleStore) { - checkNotNull(ruleStore, "Argument 'ruleStore' can not be null"); - - this.ruleStore = ruleStore; - } - - @Override - public boolean injectRntbdServerResponseDelayBeforeProcessing( - RntbdRequestRecord requestRecord, - Consumer writeRequestWithDelayConsumer) { - - RntbdRequestArgs requestArgs = requestRecord.args(); - - FaultInjectionServerErrorRule serverResponseDelayRule = this.ruleStore.findRntbdServerResponseDelayRule(requestArgs); - if (serverResponseDelayRule != null) { - - if (serverResponseDelayRule.getResult() != null - && serverResponseDelayRule.getResult().getSuppressServiceRequests() != null - && !serverResponseDelayRule.getResult().getSuppressServiceRequests()) { - - // delay will be injected after processing the request - return false; - } - - requestArgs.serviceRequest().faultInjectionRequestContext - .applyFaultInjectionRule( - requestRecord.transportRequestId(), - serverResponseDelayRule.getId()); - - writeRequestWithDelayConsumer.accept(serverResponseDelayRule.getResult().getDelay()); - return true; - } - - return false; - } - - @Override - public boolean injectRntbdServerResponseDelayAfterProcessing(RntbdRequestRecord requestRecord, - Consumer writeRequestWithDelayConsumer) { - RntbdRequestArgs requestArgs = requestRecord.args(); - - FaultInjectionServerErrorRule serverResponseDelayRule = this.ruleStore.findRntbdServerResponseDelayRule(requestArgs); - if (serverResponseDelayRule != null) { - - if (serverResponseDelayRule.getResult() == null - || serverResponseDelayRule.getResult().getSuppressServiceRequests() == null - || serverResponseDelayRule.getResult().getSuppressServiceRequests()) { - - // delay was injected before processing the request - return false; - } - - requestArgs.serviceRequest().faultInjectionRequestContext - .applyFaultInjectionRule( - requestRecord.transportRequestId(), - serverResponseDelayRule.getId()); - - writeRequestWithDelayConsumer.accept(serverResponseDelayRule.getResult().getDelay()); - return true; - } - - return false; - } - - @Override - public boolean injectRntbdServerResponseError(RntbdRequestRecord requestRecord) { - RntbdRequestArgs requestArgs = requestRecord.args(); - - FaultInjectionServerErrorRule serverResponseErrorRule = - this.ruleStore.findRntbdServerResponseErrorRule(requestArgs); - - if (serverResponseErrorRule != null) { - requestArgs.serviceRequest().faultInjectionRequestContext - .applyFaultInjectionRule( - requestRecord.transportRequestId(), - serverResponseErrorRule.getId()); - - CosmosException cause = serverResponseErrorRule.getInjectedServerError(requestArgs.serviceRequest()); - requestRecord.completeExceptionally(cause); - return true; - } - - return false; - } - - @Override - public boolean injectRntbdServerConnectionDelay( - IRequestRecord requestRecord, - Consumer openConnectionWithDelayConsumer) { - if (requestRecord == null) { - return false; - } - - RntbdRequestArgs requestArgs = requestRecord.args(); - - FaultInjectionServerErrorRule serverConnectionDelayRule = - this.ruleStore.findRntbdServerConnectionDelayRule(requestArgs); - - if (serverConnectionDelayRule != null) { - requestArgs.serviceRequest().faultInjectionRequestContext - .applyFaultInjectionRule( - requestRecord.getRequestId(), - serverConnectionDelayRule.getId()); - openConnectionWithDelayConsumer.accept(serverConnectionDelayRule.getResult().getDelay()); - return true; - } - - return false; - } -} diff --git a/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/ServerErrorInjector.java b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/ServerErrorInjector.java new file mode 100644 index 0000000000000..962115467e061 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/ServerErrorInjector.java @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.test.implementation.faultinjection; + +import com.azure.cosmos.CosmosException; +import com.azure.cosmos.implementation.Utils; +import com.azure.cosmos.implementation.faultinjection.FaultInjectionRequestArgs; +import com.azure.cosmos.implementation.faultinjection.IServerErrorInjector; + +import java.time.Duration; + +import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull; + +/*** + * Fault injector which can handle {@link FaultInjectionServerErrorRule} for both direct and gateway connection type. + */ +public class ServerErrorInjector implements IServerErrorInjector { + private final FaultInjectionRuleStore ruleStore; + + public ServerErrorInjector(FaultInjectionRuleStore ruleStore) { + checkNotNull(ruleStore, "Argument 'ruleStore' can not be null"); + + this.ruleStore = ruleStore; + } + + @Override + public boolean injectServerResponseDelayBeforeProcessing( + FaultInjectionRequestArgs faultInjectionRequestArgs, + Utils.ValueHolder injectedDelay) { + + FaultInjectionServerErrorRule serverResponseDelayRule = this.ruleStore.findServerResponseDelayRule(faultInjectionRequestArgs); + if (serverResponseDelayRule != null) { + + if (serverResponseDelayRule.getResult() != null + && serverResponseDelayRule.getResult().getSuppressServiceRequests() != null + && !serverResponseDelayRule.getResult().getSuppressServiceRequests()) { + + // delay will be injected after processing the request + return false; + } + + faultInjectionRequestArgs.getServiceRequest().faultInjectionRequestContext + .applyFaultInjectionRule( + faultInjectionRequestArgs.getTransportRequestId(), + serverResponseDelayRule.getId()); + + injectedDelay.v = serverResponseDelayRule.getResult().getDelay(); + return true; + } + + return false; + } + + @Override + public boolean injectServerResponseDelayAfterProcessing( + FaultInjectionRequestArgs faultInjectionRequestArgs, + Utils.ValueHolder injectedDelay) { + + FaultInjectionServerErrorRule serverResponseDelayRule = this.ruleStore.findServerResponseDelayRule(faultInjectionRequestArgs); + if (serverResponseDelayRule != null) { + + if (serverResponseDelayRule.getResult() == null + || serverResponseDelayRule.getResult().getSuppressServiceRequests() == null + || serverResponseDelayRule.getResult().getSuppressServiceRequests()) { + + // delay was injected before processing the request + return false; + } + + faultInjectionRequestArgs.getServiceRequest().faultInjectionRequestContext + .applyFaultInjectionRule( + faultInjectionRequestArgs.getTransportRequestId(), + serverResponseDelayRule.getId()); + + injectedDelay.v = serverResponseDelayRule.getResult().getDelay(); + return true; + } + + return false; + } + + @Override + public boolean injectServerResponseError( + FaultInjectionRequestArgs faultInjectionRequestArgs, + Utils.ValueHolder injectedException) { + + FaultInjectionServerErrorRule serverResponseErrorRule = this.ruleStore.findServerResponseErrorRule(faultInjectionRequestArgs); + + if (serverResponseErrorRule != null) { + faultInjectionRequestArgs.getServiceRequest().faultInjectionRequestContext + .applyFaultInjectionRule( + faultInjectionRequestArgs.getTransportRequestId(), + serverResponseErrorRule.getId()); + + CosmosException cause = serverResponseErrorRule.getInjectedServerError(faultInjectionRequestArgs.getServiceRequest()); + injectedException.v = cause; + return true; + } + + return false; + } + + @Override + public boolean injectServerConnectionDelay( + FaultInjectionRequestArgs faultInjectionRequestArgs, + Utils.ValueHolder injectedDelay) { + if (faultInjectionRequestArgs == null) { + return false; + } + + FaultInjectionServerErrorRule serverConnectionDelayRule = this.ruleStore.findServerConnectionDelayRule(faultInjectionRequestArgs); + + if (serverConnectionDelayRule != null) { + faultInjectionRequestArgs.getServiceRequest().faultInjectionRequestContext + .applyFaultInjectionRule( + faultInjectionRequestArgs.getTransportRequestId(), + serverConnectionDelayRule.getId()); + injectedDelay.v = serverConnectionDelayRule.getResult().getDelay(); + return true; + } + + return false; + } +} diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java index ee6c571ca436f..3a22f0f8ad11a 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java @@ -229,7 +229,7 @@ public void gatewayDiagnostics() throws Exception { assertThat(diagnostics).contains("\"connectionMode\":\"GATEWAY\""); assertThat(diagnostics).contains("\"userAgent\":\"" + this.gatewayClientUserAgent + "\""); - assertThat(diagnostics).doesNotContain(("\"gatewayStatistics\":null")); + assertThat(diagnostics).contains("gatewayStatisticsList"); assertThat(diagnostics).contains("\"operationType\":\"Create\""); assertThat(diagnostics).contains("\"metaDataName\":\"CONTAINER_LOOK_UP\""); assertThat(diagnostics).contains("\"serializationType\":\"PARTITION_KEY_FETCH_SERIALIZATION\""); @@ -261,13 +261,14 @@ public void gatewayDiagnosticsOnException() throws Exception { InternalObjectNode.class); fail("request should fail as partition key is wrong"); } catch (CosmosException exception) { + System.out.println(exception.getDiagnostics()); isValidJSON(exception.toString()); isValidJSON(exception.getMessage()); String diagnostics = exception.getDiagnostics().toString(); assertThat(exception.getStatusCode()).isEqualTo(HttpConstants.StatusCodes.NOTFOUND); assertThat(diagnostics).contains("\"connectionMode\":\"GATEWAY\""); assertThat(diagnostics).contains("\"userAgent\":\"" + this.gatewayClientUserAgent + "\""); - assertThat(diagnostics).doesNotContain(("\"gatewayStatistics\":null")); + assertThat(diagnostics).contains("gatewayStatisticsList"); assertThat(diagnostics).contains("\"statusCode\":404"); assertThat(diagnostics).contains("\"operationType\":\"Read\""); assertThat(diagnostics).contains("\"userAgent\":\"" + this.gatewayClientUserAgent + "\""); @@ -717,7 +718,7 @@ private void validateDirectModeDiagnosticsOnSuccess( String diagnostics = cosmosDiagnostics.toString(); assertThat(diagnostics).contains("\"connectionMode\":\"DIRECT\""); assertThat(diagnostics).contains("supplementalResponseStatisticsList"); - assertThat(diagnostics).contains("\"gatewayStatistics\":null"); + assertThat(diagnostics).contains("gatewayStatisticsList"); assertThat(diagnostics).contains("addressResolutionStatistics"); assertThat(diagnostics).contains("\"metaDataName\":\"CONTAINER_LOOK_UP\""); assertThat(diagnostics).contains("\"metaDataName\":\"PARTITION_KEY_RANGE_LOOK_UP\""); @@ -765,7 +766,7 @@ private void validateDirectModeQueryDiagnostics(String diagnostics, String userA assertThat(diagnostics).contains("\"connectionMode\":\"DIRECT\""); assertThat(diagnostics).contains("supplementalResponseStatisticsList"); assertThat(diagnostics).contains("responseStatisticsList"); - assertThat(diagnostics).contains("\"gatewayStatistics\":null"); + assertThat(diagnostics).contains("gatewayStatisticsList"); assertThat(diagnostics).contains("addressResolutionStatistics"); assertThat(diagnostics).contains("\"userAgent\":\"" + userAgent + "\""); assertThat(diagnostics).containsPattern("(?s).*?\"activityId\":\"[^\\s\"]+\".*"); @@ -773,7 +774,7 @@ private void validateDirectModeQueryDiagnostics(String diagnostics, String userA private void validateGatewayModeQueryDiagnostics(String diagnostics, String userAgent) { assertThat(diagnostics).contains("\"connectionMode\":\"GATEWAY\""); - assertThat(diagnostics).doesNotContain(("\"gatewayStatistics\":null")); + assertThat(diagnostics).contains(("gatewayStatisticsList")); assertThat(diagnostics).contains("\"operationType\":\"Query\""); assertThat(diagnostics).contains("\"userAgent\":\"" + userAgent + "\""); assertThat(diagnostics).containsPattern("(?s).*?\"activityId\":\"[^\\s\"]+\".*"); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosTracerTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosTracerTest.java index fcf9578e7f361..c68fbf93a7f94 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosTracerTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosTracerTest.java @@ -1768,12 +1768,12 @@ private void verifyLegacyTracerDiagnostics(CosmosDiagnostics cosmosDiagnostics, eventName, clientSideStatistics.getRequestStartTimeUTC()); } - } else if (clientSideStatistics.getGatewayStatistics() != null) { - String pkRangeId = clientSideStatistics.getGatewayStatistics().getPartitionKeyRangeId(); + } else if (clientSideStatistics.getGatewayStatisticsList() != null && clientSideStatistics.getGatewayStatisticsList().size() > 0) { + String pkRangeId = clientSideStatistics.getGatewayStatisticsList().get(0).getPartitionKeyRangeId(); if (pkRangeId != null) { String eventName = "Diagnostics for PKRange " - + clientSideStatistics.getGatewayStatistics().getPartitionKeyRangeId(); + + clientSideStatistics.getGatewayStatisticsList().get(0).getPartitionKeyRangeId(); assertEvent( mockTracer, eventName, diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExcludeRegionTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExcludeRegionTests.java index 74b8a527c43df..008d25bc71622 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExcludeRegionTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExcludeRegionTests.java @@ -9,7 +9,6 @@ import com.azure.cosmos.implementation.GlobalEndpointManager; import com.azure.cosmos.implementation.OperationType; import com.azure.cosmos.implementation.RxDocumentClientImpl; -import com.azure.cosmos.implementation.TestConfigurations; import com.azure.cosmos.implementation.directconnectivity.ReflectionUtils; import com.azure.cosmos.implementation.throughputControl.TestItem; import com.azure.cosmos.models.CosmosItemRequestOptions; diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionMetadataRequestRuleTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionMetadataRequestRuleTests.java new file mode 100644 index 0000000000000..365c1131def04 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionMetadataRequestRuleTests.java @@ -0,0 +1,560 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.faultinjection; + +import com.azure.cosmos.BridgeInternal; +import com.azure.cosmos.CosmosAsyncClient; +import com.azure.cosmos.CosmosAsyncContainer; +import com.azure.cosmos.CosmosClientBuilder; +import com.azure.cosmos.CosmosDiagnostics; +import com.azure.cosmos.CosmosException; +import com.azure.cosmos.implementation.AsyncDocumentClient; +import com.azure.cosmos.implementation.DatabaseAccount; +import com.azure.cosmos.implementation.DatabaseAccountLocation; +import com.azure.cosmos.implementation.GlobalEndpointManager; +import com.azure.cosmos.implementation.Utils; +import com.azure.cosmos.implementation.throughputControl.TestItem; +import com.azure.cosmos.models.CosmosQueryRequestOptions; +import com.azure.cosmos.models.FeedRange; +import com.azure.cosmos.models.PartitionKey; +import com.azure.cosmos.rx.TestSuiteBase; +import com.azure.cosmos.test.faultinjection.CosmosFaultInjectionHelper; +import com.azure.cosmos.test.faultinjection.FaultInjectionConditionBuilder; +import com.azure.cosmos.test.faultinjection.FaultInjectionEndpointBuilder; +import com.azure.cosmos.test.faultinjection.FaultInjectionOperationType; +import com.azure.cosmos.test.faultinjection.FaultInjectionResultBuilders; +import com.azure.cosmos.test.faultinjection.FaultInjectionRule; +import com.azure.cosmos.test.faultinjection.FaultInjectionRuleBuilder; +import com.azure.cosmos.test.faultinjection.FaultInjectionServerErrorType; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Factory; +import org.testng.annotations.Test; + +import java.time.Duration; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.testng.AssertJUnit.fail; + +public class FaultInjectionMetadataRequestRuleTests extends TestSuiteBase { + private CosmosAsyncClient client; + private CosmosAsyncContainer cosmosAsyncContainer; + private List preferredLocations; + + @Factory(dataProvider = "simpleClientBuildersWithJustDirectTcp") + public FaultInjectionMetadataRequestRuleTests(CosmosClientBuilder clientBuilder) { + super(clientBuilder); + this.subscriberValidationTimeout = TIMEOUT; + } + + @BeforeClass(groups = { "multi-region" }, timeOut = TIMEOUT) + public void beforeClass() { + this.client = getClientBuilder().buildAsyncClient(); + AsyncDocumentClient asyncDocumentClient = BridgeInternal.getContextClient(this.client); + GlobalEndpointManager globalEndpointManager = asyncDocumentClient.getGlobalEndpointManager(); + + DatabaseAccount databaseAccount = globalEndpointManager.getLatestDatabaseAccount(); + Map readRegionMap = this.getRegionMap(databaseAccount, false); + this.cosmosAsyncContainer = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(this.client); + // create a client with preferred regions + this.preferredLocations = readRegionMap.keySet().stream().collect(Collectors.toList()); + } + + @Test(groups = { "multi-region" }, timeOut = 4 * TIMEOUT) + public void faultInjectionServerErrorRuleTests_AddressRefresh_ConnectionDelay() throws JsonProcessingException { + + // Test to validate if there is http connection exception for address refresh, + // SDK will make the region unavailable and retry the request in another region. + + // We need to create a new client because client may have marked region unavailable in other tests + // which can impact the test result + CosmosAsyncClient testClient = getClientBuilder() + .contentResponseOnWriteEnabled(true) + .preferredRegions(preferredLocations) + .buildAsyncClient(); + + CosmosAsyncContainer container = + testClient + .getDatabase(this.cosmosAsyncContainer.getDatabase().getId()) + .getContainer(this.cosmosAsyncContainer.getId()); + + String addressRefreshConnectionDelay = "AddressRefresh-connectionDelay-" + UUID.randomUUID(); + FaultInjectionRule addressRefreshConnectionDelayRule = + new FaultInjectionRuleBuilder(addressRefreshConnectionDelay) + .condition( + new FaultInjectionConditionBuilder() + .region(this.preferredLocations.get(0)) + .operationType(FaultInjectionOperationType.METADATA_REQUEST_ADDRESS_REFRESH) + .build() + ) + .result( + FaultInjectionResultBuilders + .getResultBuilder(FaultInjectionServerErrorType.CONNECTION_DELAY) + .delay(Duration.ofSeconds(50)) // to simulate http connection timeout + .times(2) + .build() + ) + .duration(Duration.ofMinutes(5)) + .build(); + + FaultInjectionRule dataOperationGoneRule = + new FaultInjectionRuleBuilder("DataOperation-gone-" + UUID.randomUUID()) + .condition( + new FaultInjectionConditionBuilder() + .region(this.preferredLocations.get(0)) + .operationType(FaultInjectionOperationType.READ_ITEM) + .build()) + .result( + FaultInjectionResultBuilders + .getResultBuilder(FaultInjectionServerErrorType.GONE) + .build() + ) + .duration(Duration.ofMinutes(5)) + .build(); + try { + TestItem createdItem = TestItem.createNewItem(); + container.createItem(createdItem).block(); + + CosmosFaultInjectionHelper.configureFaultInjectionRules( + container, + Arrays.asList(addressRefreshConnectionDelayRule, dataOperationGoneRule)) + .block(); + + try { + CosmosDiagnostics cosmosDiagnostics = + container + .readItem(createdItem.getId(), new PartitionKey(createdItem.getId()), JsonNode.class) + .block() + .getDiagnostics(); + assertThat(cosmosDiagnostics.getContactedRegionNames().size()).isEqualTo(2); + validateFaultInjectionRuleAppliedForAddressResolution(cosmosDiagnostics, addressRefreshConnectionDelay, 2); + } catch (CosmosException e) { + fail("Request should be able to succeed by retrying in another region. " + e.getDiagnostics()); + } + + addressRefreshConnectionDelayRule.disable(); + dataOperationGoneRule.disable(); + + // issue another request to verify SDK has marked the first region unavailable + CosmosDiagnostics cosmosDiagnostics = + container + .readItem(createdItem.getId(), new PartitionKey(createdItem.getId()), JsonNode.class) + .block() + .getDiagnostics(); + assertThat(cosmosDiagnostics.getContactedRegionNames().size()).isEqualTo(1); + assertThat( + cosmosDiagnostics + .getContactedRegionNames() + .containsAll(Arrays.asList(this.preferredLocations.get(1).toLowerCase()))) + .isTrue(); + } finally { + addressRefreshConnectionDelayRule.disable(); + dataOperationGoneRule.disable(); + safeClose(testClient); + } + } + + @Test(groups = { "multi-region" }, timeOut = 4 * TIMEOUT) + public void faultInjectionServerErrorRuleTests_AddressRefresh_ResponseDelay() throws JsonProcessingException { + + // Test to validate if there is http request timeout for address refresh, + // SDK will retry 3 times before fail the request + + // We need to create a new client because client may have marked region unavailable in other tests + // which can impact the test result + CosmosAsyncClient testClient = getClientBuilder() + .contentResponseOnWriteEnabled(true) + .preferredRegions(preferredLocations) + .buildAsyncClient(); + + CosmosAsyncContainer container = + testClient + .getDatabase(this.cosmosAsyncContainer.getDatabase().getId()) + .getContainer(this.cosmosAsyncContainer.getId()); + + String addressRefreshResponseDelay = "AddressRefresh-responseDelay-" + UUID.randomUUID(); + FaultInjectionRule addressRefreshResponseDelayRule = + new FaultInjectionRuleBuilder(addressRefreshResponseDelay) + .condition( + new FaultInjectionConditionBuilder() + .region(this.preferredLocations.get(0)) + .operationType(FaultInjectionOperationType.METADATA_REQUEST_ADDRESS_REFRESH) + .build() + ) + .result( + FaultInjectionResultBuilders + .getResultBuilder(FaultInjectionServerErrorType.RESPONSE_DELAY) + .delay(Duration.ofSeconds(6)) // to simulate http request timeout + .times(4) + .build() + ) + .duration(Duration.ofMinutes(5)) + .build(); + + FaultInjectionRule dataOperationGoneRule = + new FaultInjectionRuleBuilder("DataOperation-gone-" + UUID.randomUUID()) + .condition( + new FaultInjectionConditionBuilder() + .region(this.preferredLocations.get(0)) + .operationType(FaultInjectionOperationType.READ_ITEM) + .build()) + .result( + FaultInjectionResultBuilders + .getResultBuilder(FaultInjectionServerErrorType.GONE) + .build() + ) + .duration(Duration.ofMinutes(5)) + .build(); + try { + + TestItem createdItem = TestItem.createNewItem(); + container.createItem(createdItem).block(); + + CosmosFaultInjectionHelper.configureFaultInjectionRules( + container, + Arrays.asList(addressRefreshResponseDelayRule, dataOperationGoneRule)) + .block(); + + try { + CosmosDiagnostics cosmosDiagnostics = + container + .readItem(createdItem.getId(), new PartitionKey(createdItem.getId()), JsonNode.class) + .block() + .getDiagnostics(); + + fail("request should have failed due to http request timeout on address resolution. " + cosmosDiagnostics); + } catch (CosmosException e) { + CosmosDiagnostics cosmosDiagnostics = e.getDiagnostics(); + assertThat(cosmosDiagnostics.getContactedRegionNames().size()).isEqualTo(1); + assertThat(cosmosDiagnostics.getContactedRegionNames().containsAll(Arrays.asList(this.preferredLocations.get(0).toLowerCase()))).isTrue(); + validateFaultInjectionRuleAppliedForAddressResolution(cosmosDiagnostics, addressRefreshResponseDelay, 4); + } + } finally { + addressRefreshResponseDelayRule.disable(); + dataOperationGoneRule.disable(); + safeClose(testClient); + } + } + + @Test(groups = { "multi-region" }, timeOut = 4 * TIMEOUT) + public void faultInjectionServerErrorRuleTests_AddressRefresh_byPartition() { + + // We need to create a new client because client may have marked region unavailable in other tests + // which can impact the test result + CosmosAsyncClient testClient = getClientBuilder() + .contentResponseOnWriteEnabled(true) + .preferredRegions(preferredLocations) + .buildAsyncClient(); + + CosmosAsyncContainer container = + testClient + .getDatabase(this.cosmosAsyncContainer.getDatabase().getId()) + .getContainer(this.cosmosAsyncContainer.getId()); + + // first create few documents + for (int i = 0; i < 10; i++) { + container.createItem(TestItem.createNewItem()).block(); + } + + List feedRanges = container.getFeedRanges().block(); + assertThat(feedRanges.size()).isGreaterThan(1); + + CosmosQueryRequestOptions cosmosQueryRequestOptions = new CosmosQueryRequestOptions(); + cosmosQueryRequestOptions.setFeedRange(feedRanges.get(0)); + String query = "select * from c"; + TestItem itemOnFeedRange1 = container.queryItems(query, cosmosQueryRequestOptions, TestItem.class) + .byPage(1) + .blockFirst() + .getResults() + .get(0); + + cosmosQueryRequestOptions.setFeedRange(feedRanges.get(1)); + TestItem itemOnFeedRange2 = container.queryItems(query, cosmosQueryRequestOptions, TestItem.class) + .byPage(1) + .blockFirst() + .getResults() + .get(0); + + // Test to validate address refresh rule can be scoped to partition + String addressRefreshResponseDelay = "AddressRefresh-responseDelay-" + UUID.randomUUID(); + FaultInjectionRule addressRefreshResponseDelayRule = + new FaultInjectionRuleBuilder(addressRefreshResponseDelay) + .condition( + new FaultInjectionConditionBuilder() + .operationType(FaultInjectionOperationType.METADATA_REQUEST_ADDRESS_REFRESH) + .endpoints(new FaultInjectionEndpointBuilder(feedRanges.get(0)).build()) + .build() + ) + .result( + FaultInjectionResultBuilders + .getResultBuilder(FaultInjectionServerErrorType.RESPONSE_DELAY) + .delay(Duration.ofSeconds(6)) // to simulate http request timeout + .times(4) + .build() + ) + .duration(Duration.ofMinutes(5)) + .build(); + + FaultInjectionRule dataOperationGoneRule = + new FaultInjectionRuleBuilder("DataOperation-gone-" + UUID.randomUUID()) + .condition( + new FaultInjectionConditionBuilder() + .operationType(FaultInjectionOperationType.READ_ITEM) + .build()) + .result( + FaultInjectionResultBuilders + .getResultBuilder(FaultInjectionServerErrorType.GONE) + .times(1) + .build() + ) + .duration(Duration.ofMinutes(5)) + .build(); + + try { + CosmosFaultInjectionHelper.configureFaultInjectionRules( + container, + Arrays.asList(addressRefreshResponseDelayRule, dataOperationGoneRule)) + .block(); + + // validate for request on feed range 0, it will fail + try { + CosmosDiagnostics cosmosDiagnostics = + container + .readItem(itemOnFeedRange1.getId(), new PartitionKey(itemOnFeedRange1.getId()), JsonNode.class) + .block() + .getDiagnostics(); + + fail("Item on feed range 1 should have failed. " + cosmosDiagnostics); + } catch (CosmosException e) { + // no-op + } + + try { + container + .readItem(itemOnFeedRange2.getId(), new PartitionKey(itemOnFeedRange2.getId()), JsonNode.class) + .block() + .getDiagnostics(); + + } catch (CosmosException e) { + fail("Item on feed range 2 should have succeeded. " + e.getDiagnostics()); + } + + } finally { + addressRefreshResponseDelayRule.disable(); + dataOperationGoneRule.disable(); + safeClose(testClient); + } + } + + @Test(groups = { "multi-region" }, timeOut = 4 * TIMEOUT) + public void faultInjectionServerErrorRuleTests_AddressRefresh_TooManyRequest() throws JsonProcessingException { + + // We need to create a new client because client may have marked region unavailable in other tests + // which can impact the test result + CosmosAsyncClient testClient = getClientBuilder() + .contentResponseOnWriteEnabled(true) + .preferredRegions(preferredLocations) + .buildAsyncClient(); + + CosmosAsyncContainer container = + testClient + .getDatabase(this.cosmosAsyncContainer.getDatabase().getId()) + .getContainer(this.cosmosAsyncContainer.getId()); + + // Test to SDK will retry 429 for address refresh + String addressRefreshTooManyRequest = "AddressRefresh-tooManyRequest-" + UUID.randomUUID(); + FaultInjectionRule addressRefreshTooManyRequestRule = + new FaultInjectionRuleBuilder(addressRefreshTooManyRequest) + .condition( + new FaultInjectionConditionBuilder() + .region(this.preferredLocations.get(0)) + .operationType(FaultInjectionOperationType.METADATA_REQUEST_ADDRESS_REFRESH) + .build() + ) + .result( + FaultInjectionResultBuilders + .getResultBuilder(FaultInjectionServerErrorType.TOO_MANY_REQUEST) + .times(1) + .build() + ) + .duration(Duration.ofMinutes(5)) + .build(); + + FaultInjectionRule dataOperationGoneRule = + new FaultInjectionRuleBuilder("DataOperation-gone-" + UUID.randomUUID()) + .condition( + new FaultInjectionConditionBuilder() + .region(this.preferredLocations.get(0)) + .operationType(FaultInjectionOperationType.READ_ITEM) + .build()) + .result( + FaultInjectionResultBuilders + .getResultBuilder(FaultInjectionServerErrorType.GONE) + .times(1) // to make sure the address refresh will be at least triggered once + .build() + ) + .duration(Duration.ofMinutes(5)) + .build(); + try { + TestItem createdItem = TestItem.createNewItem(); + container.createItem(createdItem).block(); + + CosmosFaultInjectionHelper.configureFaultInjectionRules( + container, + Arrays.asList(addressRefreshTooManyRequestRule, dataOperationGoneRule)) + .block(); + + CosmosDiagnostics cosmosDiagnostics = + container.readItem(createdItem.getId(), new PartitionKey(createdItem.getId()), JsonNode.class).block().getDiagnostics(); + + assertThat(cosmosDiagnostics.getContactedRegionNames().size()).isEqualTo(1); + assertThat(cosmosDiagnostics.getContactedRegionNames().containsAll(Arrays.asList(this.preferredLocations.get(0).toLowerCase()))).isTrue(); + validateFaultInjectionRuleAppliedForAddressResolution(cosmosDiagnostics, addressRefreshTooManyRequest, 1); + } finally { + addressRefreshTooManyRequestRule.disable(); + dataOperationGoneRule.disable(); + safeClose(testClient); + } + } + + @Test(groups = { "multi-region" }, timeOut = 40 * TIMEOUT) + public void faultInjectionServerErrorRuleTests_PartitionKeyRanges_ConnectionDelay() throws JsonProcessingException { + + // We need to create a new client because client may have marked region unavailable in other tests + // which can impact the test result + CosmosAsyncClient testClient = getClientBuilder() + .contentResponseOnWriteEnabled(true) + .preferredRegions(preferredLocations) + .buildAsyncClient(); + + CosmosAsyncContainer container = + testClient + .getDatabase(this.cosmosAsyncContainer.getDatabase().getId()) + .getContainer(this.cosmosAsyncContainer.getId()); + + // Test to validate partition key ranges request is being injected connection timeout + String pkRangesConnectionDelay = "PkRanges-connectionDelay-" + UUID.randomUUID(); + FaultInjectionRule pkRangesConnectionDelayRule = + new FaultInjectionRuleBuilder(pkRangesConnectionDelay) + .condition( + new FaultInjectionConditionBuilder() + .region(this.preferredLocations.get(0)) + .operationType(FaultInjectionOperationType.METADATA_REQUEST_PARTITION_KEY_RANGES) + .build() + ) + .result( + FaultInjectionResultBuilders + .getResultBuilder(FaultInjectionServerErrorType.CONNECTION_DELAY) + .delay(Duration.ofSeconds(50)) // to simulate http connection timeout + .times(1) + .build() + ) + .duration(Duration.ofMinutes(5)) + .build(); + + FaultInjectionRule dataOperationGoneRule = + new FaultInjectionRuleBuilder("DataOperation-gone-" + UUID.randomUUID()) + .condition( + new FaultInjectionConditionBuilder() + .operationType(FaultInjectionOperationType.CREATE_ITEM) + .region(this.preferredLocations.get(0)) + .build()) + .result( + FaultInjectionResultBuilders + .getResultBuilder(FaultInjectionServerErrorType.PARTITION_IS_SPLITTING) // using partition split to trigger routing map refresh flow + .build() + ) + .duration(Duration.ofMinutes(5)) + .build(); + try { + // create few items to first make sure the collection cache, pkRanges cache is being populated + for (int i = 0; i < 10; i++) { + container.createItem(TestItem.createNewItem()).block(); + } + + CosmosFaultInjectionHelper.configureFaultInjectionRules( + container, + Arrays.asList(pkRangesConnectionDelayRule, dataOperationGoneRule)) + .block(); + + try { + CosmosDiagnostics cosmosDiagnostics = container.createItem(TestItem.createNewItem()).block().getDiagnostics(); + fail("CreateItem should have failed. " + cosmosDiagnostics); + } catch (CosmosException cosmosException) { + CosmosDiagnostics cosmosDiagnostics = cosmosException.getDiagnostics(); + + // validate PARTITION_KEY_RANGE_LOOK_UP + ObjectNode diagnosticsNode = (ObjectNode) Utils.getSimpleObjectMapper().readTree(cosmosDiagnostics.toString()); + JsonNode metadataDiagnosticsContext = diagnosticsNode.get("metadataDiagnosticsContext"); + ArrayNode metadataDiagnosticList = (ArrayNode) metadataDiagnosticsContext.get("metadataDiagnosticList"); + + assertThat(metadataDiagnosticList.size()).isGreaterThan(0); + + JsonNode pkRangesLookup = null; + for (int i = 0; i < metadataDiagnosticList.size(); i++) { + if (metadataDiagnosticList.get(i).get("metaDataName").asText().equalsIgnoreCase("PARTITION_KEY_RANGE_LOOK_UP")) { + pkRangesLookup = metadataDiagnosticList.get(i); + break; + } + } + + assertThat(pkRangesLookup).isNotNull(); + assertThat(pkRangesLookup.get("durationinMS").asLong()).isGreaterThanOrEqualTo(45000); // the duration will be at least one timeout + } + } finally { + pkRangesConnectionDelayRule.disable(); + dataOperationGoneRule.disable(); + safeClose(testClient); + } + } + + @AfterClass(groups = {"multi-region"}, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) + public void afterClass() { + safeClose(this.client); + } + + private Map getRegionMap(DatabaseAccount databaseAccount, boolean writeOnly) { + Iterator locationIterator = + writeOnly ? databaseAccount.getWritableLocations().iterator() : databaseAccount.getReadableLocations().iterator(); + Map regionMap = new ConcurrentHashMap<>(); + + while (locationIterator.hasNext()) { + DatabaseAccountLocation accountLocation = locationIterator.next(); + regionMap.put(accountLocation.getName(), accountLocation.getEndpoint()); + } + + return regionMap; + } + + private void validateFaultInjectionRuleAppliedForAddressResolution( + CosmosDiagnostics cosmosDiagnostics, + String ruleId, + int failureInjectedExpectedCount) throws JsonProcessingException { + + ObjectNode diagnosticsNode = (ObjectNode) Utils.getSimpleObjectMapper().readTree(cosmosDiagnostics.toString()); + JsonNode addressResolutionStatistics = diagnosticsNode.get("addressResolutionStatistics"); + Iterator> addressResolutionIterator = addressResolutionStatistics.fields(); + int failureInjectedCount = 0; + while (addressResolutionIterator.hasNext()) { + JsonNode addressResolutionSingleRequest = addressResolutionIterator.next().getValue(); + if (addressResolutionSingleRequest.has("faultInjectionRuleId") + && addressResolutionSingleRequest.get("faultInjectionRuleId").asText().equalsIgnoreCase(ruleId)) { + failureInjectedCount++; + } + } + + assertThat(failureInjectedCount).isEqualTo(failureInjectedExpectedCount); + } +} diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionServerErrorRuleTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionServerErrorRuleOnDirectTests.java similarity index 94% rename from sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionServerErrorRuleTests.java rename to sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionServerErrorRuleOnDirectTests.java index 517cab164b5e1..c646d6535eaad 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionServerErrorRuleTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionServerErrorRuleOnDirectTests.java @@ -59,7 +59,7 @@ import static org.assertj.core.api.AssertionsForClassTypes.assertThat; import static org.testng.AssertJUnit.fail; -public class FaultInjectionServerErrorRuleTests extends TestSuiteBase { +public class FaultInjectionServerErrorRuleOnDirectTests extends TestSuiteBase { private static final int TIMEOUT = 60000; private static final String FAULT_INJECTION_RULE_NON_APPLICABLE_ADDRESS = "Addresses mismatch"; private static final String FAULT_INJECTION_RULE_NON_APPLICABLE_OPERATION_TYPE = "OperationType mismatch"; @@ -74,7 +74,7 @@ public class FaultInjectionServerErrorRuleTests extends TestSuiteBase { private Map writeRegionMap; @Factory(dataProvider = "simpleClientBuildersWithJustDirectTcp") - public FaultInjectionServerErrorRuleTests(CosmosClientBuilder clientBuilder) { + public FaultInjectionServerErrorRuleOnDirectTests(CosmosClientBuilder clientBuilder) { super(clientBuilder); this.subscriberValidationTimeout = TIMEOUT; } @@ -981,6 +981,46 @@ public void faultInjectionServerErrorRuleTests_includePrimary() throws JsonProce } } + @Test(groups = {"simple"}, timeOut = TIMEOUT) + public void faultInjectionServerErrorRuleTests_StaledAddressesServerGone() throws JsonProcessingException { + // Test gone error being injected and forceRefresh address refresh happens + String staledAddressesServerGoneRuleId = "staledAddressesServerGone-" + UUID.randomUUID(); + FaultInjectionRule staledAddressesServerGoneRule = + new FaultInjectionRuleBuilder(staledAddressesServerGoneRuleId) + .condition( + new FaultInjectionConditionBuilder() + .operationType(FaultInjectionOperationType.CREATE_ITEM) + .build() + ) + .result( + FaultInjectionResultBuilders + .getResultBuilder(FaultInjectionServerErrorType.STALED_ADDRESSES_SERVER_GONE) + .build() + ) + .duration(Duration.ofMinutes(5)) + .build(); + + try { + TestItem testItem = this.cosmosAsyncContainer.createItem(TestItem.createNewItem()).block().getItem(); + + CosmosFaultInjectionHelper.configureFaultInjectionRules(this.cosmosAsyncContainer, Arrays.asList(staledAddressesServerGoneRule)).block(); + + // test for create item operation, the rule will be applied + CosmosDiagnostics cosmosDiagnostics = this.performDocumentOperation(this.cosmosAsyncContainer, OperationType.Create, testItem); + this.validateFaultInjectionRuleApplied( + cosmosDiagnostics, + OperationType.Create, + HttpConstants.StatusCodes.GONE, + HttpConstants.SubStatusCodes.SERVER_GENERATED_410, + staledAddressesServerGoneRuleId, + true); + + this.validateAddressRefreshWithForceRefresh(cosmosDiagnostics); + } finally { + staledAddressesServerGoneRule.disable(); + } + } + private void validateFaultInjectionRuleApplied( CosmosDiagnostics cosmosDiagnostics, OperationType operationType, @@ -1077,4 +1117,19 @@ private void validateHitCount( assertThat(rule.getHitCountDetails().get(operationType.toString() + "-" + resourceType.toString())).isEqualTo(totalHitCount); } } + + private void validateAddressRefreshWithForceRefresh(CosmosDiagnostics cosmosDiagnostics) throws JsonProcessingException { + ObjectNode diagnosticsNode = (ObjectNode) Utils.getSimpleObjectMapper().readTree(cosmosDiagnostics.toString()); + JsonNode addressResolutionStatistics = diagnosticsNode.get("addressResolutionStatistics"); + Iterator> addressResolutionIterator = addressResolutionStatistics.fields(); + int addressRefreshWithForceRefreshCount = 0; + while (addressResolutionIterator.hasNext()) { + JsonNode addressResolutionSingleRequest = addressResolutionIterator.next().getValue(); + if (addressResolutionSingleRequest.get("forceRefresh").asBoolean()) { + addressRefreshWithForceRefreshCount++; + } + } + + assertThat(addressRefreshWithForceRefreshCount).isGreaterThanOrEqualTo(1); + } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionServerErrorRuleOnGatewayTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionServerErrorRuleOnGatewayTests.java new file mode 100644 index 0000000000000..ebe82047b1073 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionServerErrorRuleOnGatewayTests.java @@ -0,0 +1,689 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.faultinjection; + +import com.azure.cosmos.BridgeInternal; +import com.azure.cosmos.CosmosAsyncClient; +import com.azure.cosmos.CosmosAsyncContainer; +import com.azure.cosmos.CosmosClientBuilder; +import com.azure.cosmos.CosmosDiagnostics; +import com.azure.cosmos.CosmosException; +import com.azure.cosmos.DirectConnectionConfig; +import com.azure.cosmos.implementation.AsyncDocumentClient; +import com.azure.cosmos.implementation.DatabaseAccount; +import com.azure.cosmos.implementation.DatabaseAccountLocation; +import com.azure.cosmos.implementation.GlobalEndpointManager; +import com.azure.cosmos.implementation.HttpConstants; +import com.azure.cosmos.implementation.OperationType; +import com.azure.cosmos.implementation.ResourceType; +import com.azure.cosmos.implementation.TestConfigurations; +import com.azure.cosmos.implementation.Utils; +import com.azure.cosmos.implementation.throughputControl.TestItem; +import com.azure.cosmos.models.CosmosItemResponse; +import com.azure.cosmos.models.CosmosPatchOperations; +import com.azure.cosmos.models.CosmosQueryRequestOptions; +import com.azure.cosmos.models.FeedRange; +import com.azure.cosmos.models.FeedResponse; +import com.azure.cosmos.models.PartitionKey; +import com.azure.cosmos.rx.TestSuiteBase; +import com.azure.cosmos.test.faultinjection.CosmosFaultInjectionHelper; +import com.azure.cosmos.test.faultinjection.FaultInjectionConditionBuilder; +import com.azure.cosmos.test.faultinjection.FaultInjectionConnectionType; +import com.azure.cosmos.test.faultinjection.FaultInjectionEndpointBuilder; +import com.azure.cosmos.test.faultinjection.FaultInjectionOperationType; +import com.azure.cosmos.test.faultinjection.FaultInjectionResultBuilders; +import com.azure.cosmos.test.faultinjection.FaultInjectionRule; +import com.azure.cosmos.test.faultinjection.FaultInjectionRuleBuilder; +import com.azure.cosmos.test.faultinjection.FaultInjectionServerErrorType; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Factory; +import org.testng.annotations.Test; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.testng.AssertJUnit.fail; + +public class FaultInjectionServerErrorRuleOnGatewayTests extends TestSuiteBase { + + private static final String FAULT_INJECTION_RULE_NON_APPLICABLE_ADDRESS = "Addresses mismatch"; + private static final String FAULT_INJECTION_RULE_NON_APPLICABLE_REGION_ENDPOINT = "RegionEndpoint mismatch"; + private static final String FAULT_INJECTION_RULE_NON_APPLICABLE_HIT_LIMIT = "Hit Limit reached"; + + private CosmosAsyncClient client; + private CosmosAsyncContainer cosmosAsyncContainer; + private DatabaseAccount databaseAccount; + + private Map readRegionMap; + private Map writeRegionMap; + + @Factory(dataProvider = "clientBuildersWithGateway") + public FaultInjectionServerErrorRuleOnGatewayTests(CosmosClientBuilder clientBuilder) { + super(clientBuilder); + this.subscriberValidationTimeout = TIMEOUT; + } + + @BeforeClass(groups = {"multi-region", "simple"}, timeOut = TIMEOUT) + public void beforeClass() { + client = getClientBuilder().buildAsyncClient(); + AsyncDocumentClient asyncDocumentClient = BridgeInternal.getContextClient(client); + GlobalEndpointManager globalEndpointManager = asyncDocumentClient.getGlobalEndpointManager(); + + DatabaseAccount databaseAccount = globalEndpointManager.getLatestDatabaseAccount(); + this.databaseAccount = databaseAccount; + this.cosmosAsyncContainer = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(client); + this.readRegionMap = this.getRegionMap(databaseAccount, false); + this.writeRegionMap = this.getRegionMap(databaseAccount, true); + } + + @DataProvider(name = "operationTypeProvider") + public static Object[][] operationTypeProvider() { + return new Object[][]{ + { OperationType.Read, FaultInjectionOperationType.READ_ITEM }, + { OperationType.Replace, FaultInjectionOperationType.REPLACE_ITEM }, + { OperationType.Create, FaultInjectionOperationType.CREATE_ITEM }, + { OperationType.Delete, FaultInjectionOperationType.DELETE_ITEM }, + { OperationType.Query, FaultInjectionOperationType.QUERY_ITEM }, + { OperationType.Patch, FaultInjectionOperationType.PATCH_ITEM } + }; + } + + @DataProvider(name = "faultInjectionOperationTypeProvider") + public static Object[][] faultInjectionOperationTypeProvider() { + return new Object[][]{ + // fault injection operation type, primaryAddressOnly + { FaultInjectionOperationType.READ_ITEM, false }, + { FaultInjectionOperationType.REPLACE_ITEM, true }, + { FaultInjectionOperationType.CREATE_ITEM, true }, + { FaultInjectionOperationType.DELETE_ITEM, true}, + { FaultInjectionOperationType.QUERY_ITEM, false }, + { FaultInjectionOperationType.PATCH_ITEM, true } + }; + } + + @DataProvider(name = "faultInjectionServerErrorResponseProvider") + public static Object[][] faultInjectionServerErrorResponseProvider() { + return new Object[][]{ + // faultInjectionServerError, will SDK retry, errorStatusCode, errorSubStatusCode + { FaultInjectionServerErrorType.INTERNAL_SERVER_ERROR, false, 500, 0 }, + { FaultInjectionServerErrorType.RETRY_WITH, false, 449, 0 }, + { FaultInjectionServerErrorType.TOO_MANY_REQUEST, true, 429, 0 }, + { FaultInjectionServerErrorType.READ_SESSION_NOT_AVAILABLE, true, 404, 1002 }, + { FaultInjectionServerErrorType.SERVICE_UNAVAILABLE, false, 503, 21008 } + }; + } + + @Test(groups = {"multi-region"}, timeOut = TIMEOUT) + public void faultInjectionServerErrorRuleTests_Region() throws JsonProcessingException { + List preferredLocations = this.readRegionMap.keySet().stream().collect(Collectors.toList()); + + CosmosAsyncClient clientWithPreferredRegion = null; + // set local region rule + String localRegionRuleId = "ServerErrorRule-LocalRegion-" + UUID.randomUUID(); + FaultInjectionRule serverErrorRuleLocalRegion = + new FaultInjectionRuleBuilder(localRegionRuleId) + .condition( + new FaultInjectionConditionBuilder() + .connectionType(FaultInjectionConnectionType.GATEWAY) + .region(preferredLocations.get(0)) + .build() + ) + .result( + FaultInjectionResultBuilders + .getResultBuilder(FaultInjectionServerErrorType.SERVICE_UNAVAILABLE) + .times(1) + .build() + ) + .duration(Duration.ofMinutes(5)) + .build(); + + // set remote region rule + String remoteRegionRuleId = "ServerErrorRule-RemoteRegion-" + UUID.randomUUID(); + FaultInjectionRule serverErrorRuleRemoteRegion = + new FaultInjectionRuleBuilder(remoteRegionRuleId) + .condition( + new FaultInjectionConditionBuilder() + .connectionType(FaultInjectionConnectionType.GATEWAY) + .region(preferredLocations.get(1)) + .build() + ) + .result( + FaultInjectionResultBuilders + .getResultBuilder(FaultInjectionServerErrorType.SERVICE_UNAVAILABLE) + .times(1) + .build() + ) + .duration(Duration.ofMinutes(5)) + .build(); + + try { + clientWithPreferredRegion = new CosmosClientBuilder() + .endpoint(TestConfigurations.HOST) + .key(TestConfigurations.MASTER_KEY) + .contentResponseOnWriteEnabled(true) + .consistencyLevel(BridgeInternal.getContextClient(this.client).getConsistencyLevel()) + .preferredRegions(preferredLocations) + .gatewayMode() + .buildAsyncClient(); + + CosmosAsyncContainer container = + clientWithPreferredRegion + .getDatabase(this.cosmosAsyncContainer.getDatabase().getId()) + .getContainer(this.cosmosAsyncContainer.getId()); + + TestItem createdItem = TestItem.createNewItem(); + container.createItem(createdItem).block(); + + CosmosFaultInjectionHelper.configureFaultInjectionRules( + container, + Arrays.asList(serverErrorRuleLocalRegion, serverErrorRuleRemoteRegion)) + .block(); + + assertThat( + serverErrorRuleLocalRegion.getRegionEndpoints().size() == 1 + && serverErrorRuleLocalRegion.getRegionEndpoints().get(0).equals(this.readRegionMap.get(preferredLocations.get(0)))); + assertThat( + serverErrorRuleRemoteRegion.getRegionEndpoints().size() == 1 + && serverErrorRuleRemoteRegion.getRegionEndpoints().get(0).equals(this.readRegionMap.get(preferredLocations.get(1)))); + + // Validate fault injection applied in the local region + CosmosDiagnostics cosmosDiagnostics = this.performDocumentOperation(container, OperationType.Read, createdItem); + + this.validateHitCount(serverErrorRuleLocalRegion, 1, OperationType.Read, ResourceType.Document); + this.validateHitCount(serverErrorRuleRemoteRegion, 0, OperationType.Read, ResourceType.Document); + + this.validateFaultInjectionRuleApplied( + cosmosDiagnostics, + OperationType.Read, + HttpConstants.StatusCodes.SERVICE_UNAVAILABLE, + HttpConstants.SubStatusCodes.SERVER_GENERATED_503, + localRegionRuleId, + false + ); + + serverErrorRuleLocalRegion.disable(); + + cosmosDiagnostics = this.performDocumentOperation(container, OperationType.Read, createdItem); + this.validateNoFaultInjectionApplied(cosmosDiagnostics, OperationType.Read, FAULT_INJECTION_RULE_NON_APPLICABLE_REGION_ENDPOINT); + } finally { + serverErrorRuleLocalRegion.disable(); + serverErrorRuleRemoteRegion.disable(); + safeClose(clientWithPreferredRegion); + } + } + + @Test(groups = {"multi-region", "simple"}, timeOut = 4 * TIMEOUT) + public void faultInjectionServerErrorRuleTests_Partition() throws JsonProcessingException { + for (int i = 0; i < 10; i++) { + cosmosAsyncContainer.createItem(TestItem.createNewItem()).block(); + } + + // getting one item from each feedRange + List feedRanges = cosmosAsyncContainer.getFeedRanges().block(); + assertThat(feedRanges.size()).isGreaterThan(1); + + String query = "select * from c"; + CosmosQueryRequestOptions cosmosQueryRequestOptions = new CosmosQueryRequestOptions(); + cosmosQueryRequestOptions.setFeedRange(feedRanges.get(0)); + TestItem itemOnFeedRange0 = cosmosAsyncContainer.queryItems(query, cosmosQueryRequestOptions, TestItem.class).blockFirst(); + + cosmosQueryRequestOptions.setFeedRange(feedRanges.get(1)); + TestItem itemOnFeedRange1 = cosmosAsyncContainer.queryItems(query, cosmosQueryRequestOptions, TestItem.class).blockFirst(); + + // set rule by feed range + String feedRangeRuleId = "ServerErrorRule-FeedRange-" + UUID.randomUUID(); + + FaultInjectionRule serverErrorRuleByFeedRange = + new FaultInjectionRuleBuilder(feedRangeRuleId) + .condition( + new FaultInjectionConditionBuilder() + .connectionType(FaultInjectionConnectionType.GATEWAY) + .endpoints(new FaultInjectionEndpointBuilder(feedRanges.get(0)).build()) + .build() + ) + .result( + FaultInjectionResultBuilders + .getResultBuilder(FaultInjectionServerErrorType.TOO_MANY_REQUEST) + .times(1) + .build() + ) + .duration(Duration.ofMinutes(5)) + .build(); + + CosmosFaultInjectionHelper.configureFaultInjectionRules(cosmosAsyncContainer, Arrays.asList(serverErrorRuleByFeedRange)).block(); + assertThat( + serverErrorRuleByFeedRange.getRegionEndpoints().size() == this.readRegionMap.size() + && serverErrorRuleByFeedRange.getRegionEndpoints().containsAll(this.readRegionMap.keySet())); + + // Issue a read item for the same feed range as configured in the fault injection rule + CosmosDiagnostics cosmosDiagnostics = + cosmosAsyncContainer + .readItem(itemOnFeedRange0.getId(), new PartitionKey(itemOnFeedRange0.getId()), JsonNode.class) + .block() + .getDiagnostics(); + + this.validateHitCount(serverErrorRuleByFeedRange, 1, OperationType.Read, ResourceType.Document); + this.validateFaultInjectionRuleApplied( + cosmosDiagnostics, + OperationType.Read, + HttpConstants.StatusCodes.TOO_MANY_REQUESTS, + HttpConstants.SubStatusCodes.UNKNOWN, + feedRangeRuleId, + true + ); + + // Issue a read item to different feed range + try { + cosmosDiagnostics = cosmosAsyncContainer + .readItem(itemOnFeedRange1.getId(), new PartitionKey(itemOnFeedRange1.getId()), JsonNode.class) + .block() + .getDiagnostics(); + this.validateNoFaultInjectionApplied(cosmosDiagnostics, OperationType.Read, FAULT_INJECTION_RULE_NON_APPLICABLE_ADDRESS); + } finally { + serverErrorRuleByFeedRange.disable(); + } + } + + @Test(groups = {"multi-region", "simple"}, timeOut = 4 * TIMEOUT) + public void faultInjectionServerErrorRuleTests_ServerResponseDelay() throws JsonProcessingException { + // define another rule which can simulate timeout + String timeoutRuleId = "serverErrorRule-responseDelay-" + UUID.randomUUID(); + FaultInjectionRule timeoutRule = + new FaultInjectionRuleBuilder(timeoutRuleId) + .condition( + new FaultInjectionConditionBuilder() + .connectionType(FaultInjectionConnectionType.GATEWAY) + .operationType(FaultInjectionOperationType.READ_ITEM) + .build() + ) + .result( + FaultInjectionResultBuilders + .getResultBuilder(FaultInjectionServerErrorType.RESPONSE_DELAY) + .times(1) + .delay(Duration.ofSeconds(61)) // the default time out is 60s + .build() + ) + .duration(Duration.ofMinutes(5)) + .build(); + try { + DirectConnectionConfig directConnectionConfig = DirectConnectionConfig.getDefaultConfig(); + directConnectionConfig.setConnectTimeout(Duration.ofSeconds(1)); + + // create a new item to be used by read operations + TestItem createdItem = TestItem.createNewItem(); + this.cosmosAsyncContainer.createItem(createdItem).block(); + + CosmosFaultInjectionHelper.configureFaultInjectionRules(this.cosmosAsyncContainer, Arrays.asList(timeoutRule)).block(); + CosmosItemResponse itemResponse = + this.cosmosAsyncContainer.readItem(createdItem.getId(), new PartitionKey(createdItem.getId()), TestItem.class).block(); + + assertThat(timeoutRule.getHitCount()).isEqualTo(1); + this.validateHitCount(timeoutRule, 1, OperationType.Read, ResourceType.Document); + + this.validateFaultInjectionRuleApplied( + itemResponse.getDiagnostics(), + OperationType.Read, + HttpConstants.StatusCodes.REQUEST_TIMEOUT, + HttpConstants.SubStatusCodes.GATEWAY_ENDPOINT_READ_TIMEOUT, + timeoutRuleId, + true + ); + + } finally { + timeoutRule.disable(); + } + } + + @Test(groups = {"multi-region", "simple"}, timeOut = 4 * TIMEOUT) + public void faultInjectionServerErrorRuleTests_ServerConnectionDelay() throws JsonProcessingException { + // simulate high channel acquisition/connectionTimeout + String ruleId = "serverErrorRule-serverConnectionDelay-" + UUID.randomUUID(); + FaultInjectionRule serverConnectionDelayRule = + new FaultInjectionRuleBuilder(ruleId) + .condition( + new FaultInjectionConditionBuilder() + .connectionType(FaultInjectionConnectionType.GATEWAY) + .operationType(FaultInjectionOperationType.CREATE_ITEM) + .build() + ) + .result( + FaultInjectionResultBuilders + .getResultBuilder(FaultInjectionServerErrorType.CONNECTION_DELAY) + .delay(Duration.ofSeconds(46)) // default value is 45s + .times(1) + .build() + ) + .duration(Duration.ofMinutes(5)) + .build(); + + try { + CosmosFaultInjectionHelper.configureFaultInjectionRules(this.cosmosAsyncContainer, Arrays.asList(serverConnectionDelayRule)).block(); + CosmosItemResponse itemResponse = this.cosmosAsyncContainer.createItem(TestItem.createNewItem()).block(); + + assertThat(serverConnectionDelayRule.getHitCount()).isEqualTo(1l); + this.validateFaultInjectionRuleApplied( + itemResponse.getDiagnostics(), + OperationType.Create, + HttpConstants.StatusCodes.SERVICE_UNAVAILABLE, + HttpConstants.SubStatusCodes.GATEWAY_ENDPOINT_UNAVAILABLE, + ruleId, + true + ); + + } finally { + serverConnectionDelayRule.disable(); + } + } + + @Test(groups = {"multi-region", "simple"}, dataProvider = "faultInjectionServerErrorResponseProvider", timeOut = TIMEOUT) + public void faultInjectionServerErrorRuleTests_ServerErrorResponse( + FaultInjectionServerErrorType serverErrorType, + boolean canRetry, + int errorStatusCode, + int errorSubStatusCode) throws JsonProcessingException { + + // simulate high channel acquisition/connectionTimeout + String ruleId = "serverErrorRule-" + serverErrorType + "-" + UUID.randomUUID(); + FaultInjectionRule serverErrorRule = + new FaultInjectionRuleBuilder(ruleId) + .condition( + new FaultInjectionConditionBuilder() + .connectionType(FaultInjectionConnectionType.GATEWAY) + .operationType(FaultInjectionOperationType.READ_ITEM) + .build() + ) + .result( + FaultInjectionResultBuilders + .getResultBuilder(serverErrorType) + .times(1) + .build() + ) + .duration(Duration.ofMinutes(5)) + .build(); + + try { + TestItem createdItem = TestItem.createNewItem(); + cosmosAsyncContainer.createItem(createdItem).block(); + + CosmosFaultInjectionHelper.configureFaultInjectionRules(cosmosAsyncContainer, Arrays.asList(serverErrorRule)).block(); + + CosmosDiagnostics cosmosDiagnostics = null; + if (canRetry) { + try { + cosmosDiagnostics = + cosmosAsyncContainer + .readItem(createdItem.getId(), new PartitionKey(createdItem.getId()), TestItem.class) + .block() + .getDiagnostics(); + } catch (Exception exception) { + fail("Request should succeeded, but failed with " + exception); + } + } else { + try { + cosmosDiagnostics = + cosmosAsyncContainer + .readItem(createdItem.getId(), new PartitionKey(createdItem.getId()), TestItem.class) + .block() + .getDiagnostics(); + fail("Request should fail, but succeeded"); + + } catch (Exception e) { + cosmosDiagnostics = ((CosmosException)e).getDiagnostics(); + } + } + + this.validateHitCount(serverErrorRule, 1, OperationType.Read, ResourceType.Document); + this.validateFaultInjectionRuleApplied( + cosmosDiagnostics, + OperationType.Read, + errorStatusCode, + errorSubStatusCode, + ruleId, + canRetry + ); + + } finally { + serverErrorRule.disable(); + } + } + + @Test(groups = {"multi-region", "simple"}, dataProvider = "operationTypeProvider", timeOut = TIMEOUT) + public void faultInjectionServerErrorRuleTests_HitLimit( + OperationType operationType, + FaultInjectionOperationType faultInjectionOperationType) throws JsonProcessingException { + + TestItem createdItem = TestItem.createNewItem(); + cosmosAsyncContainer.createItem(createdItem).block(); + + // set rule by feed range + String hitLimitRuleId = "ServerErrorRule-hitLimit-" + UUID.randomUUID(); + + FaultInjectionRule hitLimitServerErrorRule = + new FaultInjectionRuleBuilder(hitLimitRuleId) + .condition( + new FaultInjectionConditionBuilder() + .connectionType(FaultInjectionConnectionType.GATEWAY) + .operationType(faultInjectionOperationType) + .build() + ) + .result( + FaultInjectionResultBuilders + .getResultBuilder(FaultInjectionServerErrorType.TOO_MANY_REQUEST) + .times(1) + .build() + ) + .hitLimit(2) + .build(); + + try { + CosmosFaultInjectionHelper.configureFaultInjectionRules(cosmosAsyncContainer, Arrays.asList(hitLimitServerErrorRule)).block(); + assertThat( + hitLimitServerErrorRule.getRegionEndpoints().size() == this.readRegionMap.size() + && hitLimitServerErrorRule.getRegionEndpoints().containsAll(this.readRegionMap.keySet())); + + for (int i = 1; i <= 3; i++) { + CosmosDiagnostics cosmosDiagnostics = this.performDocumentOperation(cosmosAsyncContainer, operationType, createdItem); + if (i <= 2) { + this.validateFaultInjectionRuleApplied( + cosmosDiagnostics, + operationType, + HttpConstants.StatusCodes.TOO_MANY_REQUESTS, + HttpConstants.SubStatusCodes.UNKNOWN, + hitLimitRuleId, + true + ); + } else { + // the fault injection rule will not be applied due to hitLimit + cosmosDiagnostics = this.performDocumentOperation(cosmosAsyncContainer,operationType, createdItem); + this.validateNoFaultInjectionApplied(cosmosDiagnostics, operationType, FAULT_INJECTION_RULE_NON_APPLICABLE_HIT_LIMIT); + } + } + + this.validateHitCount(hitLimitServerErrorRule, 2, operationType, ResourceType.Document); + } finally { + hitLimitServerErrorRule.disable(); + } + } + + @AfterClass(groups = {"multi-region", "simple"}, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) + public void afterClass() { + safeClose(client); + } + + private CosmosDiagnostics performDocumentOperation( + CosmosAsyncContainer cosmosAsyncContainer, + OperationType operationType, + TestItem createdItem) { + try { + if (operationType == OperationType.Query) { + CosmosQueryRequestOptions queryRequestOptions = new CosmosQueryRequestOptions(); + String query = String.format("SELECT * from c where c.id = '%s'", createdItem.getId()); + FeedResponse itemFeedResponse = + cosmosAsyncContainer.queryItems(query, queryRequestOptions, TestItem.class).byPage().blockFirst(); + + return itemFeedResponse.getCosmosDiagnostics(); + } + + if (operationType == OperationType.Read + || operationType == OperationType.Delete + || operationType == OperationType.Replace + || operationType == OperationType.Create + || operationType == OperationType.Patch + || operationType == OperationType.Upsert) { + + if (operationType == OperationType.Read) { + return cosmosAsyncContainer.readItem( + createdItem.getId(), + new PartitionKey(createdItem.getId()), + TestItem.class).block().getDiagnostics(); + } + + if (operationType == OperationType.Replace) { + return cosmosAsyncContainer.replaceItem( + createdItem, + createdItem.getId(), + new PartitionKey(createdItem.getId())).block().getDiagnostics(); + } + + if (operationType == OperationType.Delete) { + return cosmosAsyncContainer.deleteItem(createdItem, null).block().getDiagnostics(); + } + + if (operationType == OperationType.Create) { + return cosmosAsyncContainer.createItem(TestItem.createNewItem()).block().getDiagnostics(); + } + + if (operationType == OperationType.Upsert) { + return cosmosAsyncContainer.upsertItem(TestItem.createNewItem()).block().getDiagnostics(); + } + + if (operationType == OperationType.Patch) { + CosmosPatchOperations patchOperations = + CosmosPatchOperations + .create() + .add("newPath", "newPath"); + return cosmosAsyncContainer + .patchItem(createdItem.getId(), new PartitionKey(createdItem.getId()), patchOperations, TestItem.class) + .block().getDiagnostics(); + } + } + + throw new IllegalArgumentException("The operation type is not supported"); + } catch (CosmosException cosmosException) { + return cosmosException.getDiagnostics(); + } + } + + private void validateFaultInjectionRuleApplied( + CosmosDiagnostics cosmosDiagnostics, + OperationType operationType, + int statusCode, + int subStatusCode, + String ruleId, + boolean canRetryOnFaultInjectedError) throws JsonProcessingException { + + List diagnosticsNode = new ArrayList<>(); + if (operationType == OperationType.Query) { + System.out.println(cosmosDiagnostics); + int clientSideDiagnosticsIndex = cosmosDiagnostics.toString().indexOf("[{\"userAgent\""); + ArrayNode arrayNode = + (ArrayNode) Utils.getSimpleObjectMapper().readTree(cosmosDiagnostics.toString().substring(clientSideDiagnosticsIndex)); + for (JsonNode node : arrayNode) { + diagnosticsNode.add((ObjectNode) node); + } + } else { + diagnosticsNode.add((ObjectNode) Utils.getSimpleObjectMapper().readTree(cosmosDiagnostics.toString())); + } + + for (ObjectNode diagnosticNode : diagnosticsNode) { + JsonNode gatewayStatisticsList = diagnosticNode.get("gatewayStatisticsList"); + assertThat(gatewayStatisticsList.isArray()).isTrue(); + + if (canRetryOnFaultInjectedError) { + if (gatewayStatisticsList.size() != 2) { + System.out.println("FaultInjectionGatewayStatisticsList is wrong " + cosmosDiagnostics); + } + assertThat(gatewayStatisticsList.size()).isEqualTo(2); + } else { + assertThat(gatewayStatisticsList.size()).isOne(); + } + JsonNode gatewayStatistics = gatewayStatisticsList.get(0); + assertThat(gatewayStatistics).isNotNull(); + assertThat(gatewayStatistics.get("statusCode").asInt()).isEqualTo(statusCode); + assertThat(gatewayStatistics.get("subStatusCode").asInt()).isEqualTo(subStatusCode); + assertThat(gatewayStatistics.get("faultInjectionRuleId").asText()).isEqualTo(ruleId); + } + } + + private void validateNoFaultInjectionApplied( + CosmosDiagnostics cosmosDiagnostics, + OperationType operationType, + String faultInjectionNonApplicableReason) throws JsonProcessingException { + + List diagnosticsNode = new ArrayList<>(); + if (operationType == OperationType.Query) { + int clientSideDiagnosticsIndex = cosmosDiagnostics.toString().indexOf("[{\"userAgent\""); + ArrayNode arrayNode = + (ArrayNode) Utils.getSimpleObjectMapper().readTree(cosmosDiagnostics.toString().substring(clientSideDiagnosticsIndex)); + for (JsonNode node : arrayNode) { + diagnosticsNode.add((ObjectNode) node); + } + } else { + diagnosticsNode.add((ObjectNode) Utils.getSimpleObjectMapper().readTree(cosmosDiagnostics.toString())); + } + + for (ObjectNode diagnosticNode : diagnosticsNode) { + JsonNode gatewayStatisticsList = diagnosticNode.get("gatewayStatisticsList"); + assertThat(gatewayStatisticsList.isArray()).isTrue(); + + for (int i = 0; i < gatewayStatisticsList.size(); i++) { + JsonNode gatewayStatistics = gatewayStatisticsList.get(i); + assertThat(gatewayStatistics.get("faultInjectionRuleId")).isNull(); + assertThat(gatewayStatistics.get("faultInjectionEvaluationResults")).isNotNull(); + assertThat(gatewayStatistics.get("faultInjectionEvaluationResults").toString().contains(faultInjectionNonApplicableReason)); + } + assertThat(gatewayStatisticsList.size()).isOne(); + } + } + + private Map getRegionMap(DatabaseAccount databaseAccount, boolean writeOnly) { + Iterator locationIterator = + writeOnly ? databaseAccount.getWritableLocations().iterator() : databaseAccount.getReadableLocations().iterator(); + Map regionMap = new ConcurrentHashMap<>(); + + while (locationIterator.hasNext()) { + DatabaseAccountLocation accountLocation = locationIterator.next(); + regionMap.put(accountLocation.getName(), accountLocation.getEndpoint()); + } + + return regionMap; + } + + private void validateHitCount( + FaultInjectionRule rule, + long totalHitCount, + OperationType operationType, + ResourceType resourceType) { + + assertThat(rule.getHitCount()).isEqualTo(totalHitCount); + if (totalHitCount > 0) { + assertThat(rule.getHitCountDetails().size()).isEqualTo(1); + assertThat(rule.getHitCountDetails().get(operationType.toString() + "-" + resourceType.toString())).isEqualTo(totalHitCount); + } + } +} diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionUnitTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionUnitTest.java index 0984437ea7629..020f4f0e9dc6a 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionUnitTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionUnitTest.java @@ -4,6 +4,7 @@ import com.azure.cosmos.test.faultinjection.FaultInjectionCondition; import com.azure.cosmos.test.faultinjection.FaultInjectionConditionBuilder; +import com.azure.cosmos.test.faultinjection.FaultInjectionConnectionErrorType; import com.azure.cosmos.test.faultinjection.FaultInjectionConnectionType; import com.azure.cosmos.test.faultinjection.FaultInjectionOperationType; import com.azure.cosmos.test.faultinjection.FaultInjectionResultBuilders; @@ -14,6 +15,11 @@ import org.testng.annotations.Test; import java.time.Duration; +import java.util.Arrays; +import java.util.List; + +import static org.testng.AssertJUnit.assertTrue; +import static org.testng.AssertJUnit.fail; public class FaultInjectionUnitTest { @@ -39,4 +45,110 @@ public void testFaultInjectionBuilder() { Assertions.assertThat(faultInjectionRule.getDuration()).isEqualTo(Duration.ofSeconds(1)); Assertions.assertThat(faultInjectionRule.getResult()).isNotNull(); } + + @Test(groups = "unit") + public void faultInjectionRule_metadataRequestConfig() { + // validate for metadata request, only CONNECTION_DELAY, RESPONSE_DELAY, TOO_MANY_REQUEST error type supported + List metadataOperationTypes = + Arrays.asList( + FaultInjectionOperationType.METADATA_REQUEST_ADDRESS_REFRESH, + FaultInjectionOperationType.METADATA_REQUEST_CONTAINER, + FaultInjectionOperationType.METADATA_REQUEST_DATABASE_ACCOUNT, + FaultInjectionOperationType.METADATA_REQUEST_QUERY_PLAN, + FaultInjectionOperationType.METADATA_REQUEST_PARTITION_KEY_RANGES); + + List validMetadataServerErrorTypes = + Arrays.asList( + FaultInjectionServerErrorType.TOO_MANY_REQUEST, + FaultInjectionServerErrorType.CONNECTION_DELAY, + FaultInjectionServerErrorType.RESPONSE_DELAY); + + + for (FaultInjectionOperationType faultInjectionOperationTpe : FaultInjectionOperationType.values()) { + for (FaultInjectionServerErrorType faultInjectionServerErrorType : FaultInjectionServerErrorType.values()) { + + if (metadataOperationTypes.contains(faultInjectionOperationTpe) && !validMetadataServerErrorTypes.contains(faultInjectionServerErrorType)) { + try { + new FaultInjectionRuleBuilder("metadataRule") + .condition(new FaultInjectionConditionBuilder().operationType(faultInjectionOperationTpe).build()) + .result( + FaultInjectionResultBuilders + .getResultBuilder(faultInjectionServerErrorType) + .delay(Duration.ofSeconds(1)) + .build()) + .build(); + + fail(String.format( + "faultInjectionRule should have failed to create. FaultInjectionOperationType %s, FaultInjectionServerErrorType %s", + faultInjectionOperationTpe, + faultInjectionServerErrorType)); + } catch (IllegalArgumentException e) { + //no-op + } + } else { + // Validate the rule can be created successfully + new FaultInjectionRuleBuilder("metadataRule") + .condition(new FaultInjectionConditionBuilder().operationType(faultInjectionOperationTpe).build()) + .result( + FaultInjectionResultBuilders + .getResultBuilder(faultInjectionServerErrorType) + .delay(Duration.ofSeconds(1)) + .build()) + .build(); + } + } + } + } + + @Test(groups = "unit") + public void faultInjectionRule_gatewayConnectionConfig() { + // Validate no connection error type can be configured + try { + new FaultInjectionRuleBuilder("gatewayFaultInjectionRule") + .condition(new FaultInjectionConditionBuilder().connectionType(FaultInjectionConnectionType.GATEWAY).build()) + .result( + FaultInjectionResultBuilders + .getResultBuilder(FaultInjectionConnectionErrorType.CONNECTION_CLOSE) + .interval(Duration.ofSeconds(1)) + .build()) + .build(); + + fail("gatewayFaultInjection rule should have failed as no connection error is supported for gateway connection type."); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("FaultInjectionConnectionError result can not be configured for rule with gateway connection type.")); + } + + //validate no GONE exception can be configured on gateway connection + try { + new FaultInjectionRuleBuilder("gatewayFaultInjectionRule") + .condition(new FaultInjectionConditionBuilder().connectionType(FaultInjectionConnectionType.GATEWAY).build()) + .result( + FaultInjectionResultBuilders + .getResultBuilder(FaultInjectionServerErrorType.GONE) + .build()) + .build(); + + fail("gatewayFaultInjection rule should have failed as GONE error is not supported for gateway connection type."); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Gone exception can not be injected for rule with gateway connection type")); + } + + // validate STALED_ADDRESSES_SERVER_GONE error can not be injected on gateway connection + try { + new FaultInjectionRuleBuilder("gatewayFaultInjectionRule") + .condition( + new FaultInjectionConditionBuilder() + .connectionType(FaultInjectionConnectionType.GATEWAY) + .build()) + .result( + FaultInjectionResultBuilders + .getResultBuilder(FaultInjectionServerErrorType.STALED_ADDRESSES_SERVER_GONE) + .build()) + .build(); + + fail("gatewayFaultInjection rule should have failed as STALED_ADDRESSES_SERVER_GONE error is not supported for gateway connection type."); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("STALED_ADDRESSES exception can not be injected for rule with gateway connection type")); + } + } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RenameCollectionAwareClientRetryPolicyTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RenameCollectionAwareClientRetryPolicyTest.java index c51629774eed0..af040dbec646f 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RenameCollectionAwareClientRetryPolicyTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RenameCollectionAwareClientRetryPolicyTest.java @@ -14,6 +14,7 @@ import static com.azure.cosmos.implementation.ClientRetryPolicyTest.validateSuccess; import static com.azure.cosmos.implementation.TestUtils.mockDiagnosticsClientContext; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.eq; public class RenameCollectionAwareClientRetryPolicyTest { @@ -22,7 +23,7 @@ public class RenameCollectionAwareClientRetryPolicyTest { @Test(groups = "unit", timeOut = TIMEOUT) public void onBeforeSendRequestNotInvoked() { GlobalEndpointManager endpointManager = Mockito.mock(GlobalEndpointManager.class); - Mockito.doReturn(Mono.empty()).when(endpointManager).refreshLocationAsync(Mockito.eq(null), Mockito.eq(false)); + Mockito.doReturn(Mono.empty()).when(endpointManager).refreshLocationAsync(eq(null), eq(false)); IRetryPolicyFactory retryPolicyFactory = new RetryPolicy(mockDiagnosticsClientContext(), endpointManager, ConnectionPolicy.getDefaultPolicy()); RxClientCollectionCache rxClientCollectionCache = Mockito.mock(RxClientCollectionCache.class); @@ -51,7 +52,7 @@ public void onBeforeSendRequestNotInvoked() { @Test(groups = "unit", timeOut = TIMEOUT) public void shouldRetryWithNotFoundStatusCode() { GlobalEndpointManager endpointManager = Mockito.mock(GlobalEndpointManager.class); - Mockito.doReturn(Mono.empty()).when(endpointManager).refreshLocationAsync(Mockito.eq(null),Mockito.eq(false)); + Mockito.doReturn(Mono.empty()).when(endpointManager).refreshLocationAsync(eq(null), eq(false)); IRetryPolicyFactory retryPolicyFactory = new RetryPolicy(mockDiagnosticsClientContext(), endpointManager, ConnectionPolicy.getDefaultPolicy()); RxClientCollectionCache rxClientCollectionCache = Mockito.mock(RxClientCollectionCache.class); @@ -77,7 +78,7 @@ public void shouldRetryWithNotFoundStatusCode() { @Test(groups = "unit", timeOut = TIMEOUT) public void shouldRetryWithNotFoundStatusCodeAndReadSessionNotAvailableSubStatusCode() { GlobalEndpointManager endpointManager = Mockito.mock(GlobalEndpointManager.class); - Mockito.doReturn(Mono.empty()).when(endpointManager).refreshLocationAsync(Mockito.eq(null), Mockito.eq(false)); + Mockito.doReturn(Mono.empty()).when(endpointManager).refreshLocationAsync(eq(null), eq(false)); IRetryPolicyFactory retryPolicyFactory = new RetryPolicy(mockDiagnosticsClientContext(), endpointManager, ConnectionPolicy.getDefaultPolicy()); RxClientCollectionCache rxClientCollectionCache = Mockito.mock(RxClientCollectionCache.class); @@ -114,7 +115,7 @@ public void shouldRetryWithNotFoundStatusCodeAndReadSessionNotAvailableSubStatus @Test(groups = "unit", timeOut = TIMEOUT) public void shouldRetryWithGenericException() { GlobalEndpointManager endpointManager = Mockito.mock(GlobalEndpointManager.class); - Mockito.doReturn(Mono.empty()).when(endpointManager).refreshLocationAsync(Mockito.eq(null), Mockito.eq(false)); + Mockito.doReturn(Mono.empty()).when(endpointManager).refreshLocationAsync(eq(null), eq(false)); IRetryPolicyFactory retryPolicyFactory = new RetryPolicy(mockDiagnosticsClientContext(), endpointManager, ConnectionPolicy.getDefaultPolicy()); RxClientCollectionCache rxClientCollectionCache = Mockito.mock(RxClientCollectionCache.class); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RxGatewayStoreModelTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RxGatewayStoreModelTest.java index 2d7565e971ace..4b99500c971ad 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RxGatewayStoreModelTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RxGatewayStoreModelTest.java @@ -10,6 +10,7 @@ import com.azure.cosmos.implementation.http.HttpClient; import com.azure.cosmos.implementation.http.HttpHeaders; import com.azure.cosmos.implementation.http.HttpRequest; +import io.netty.channel.ConnectTimeoutException; import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.timeout.ReadTimeoutException; import io.reactivex.subscribers.TestSubscriber; @@ -22,11 +23,11 @@ import java.net.SocketException; import java.net.URI; import java.time.Duration; -import java.time.Instant; import java.util.concurrent.TimeUnit; import static com.azure.cosmos.implementation.TestUtils.mockDiagnosticsClientContext; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; public class RxGatewayStoreModelTest { @@ -246,6 +247,7 @@ public void validateApiType() throws Exception { HttpClient httpClient = Mockito.mock(HttpClient.class); ArgumentCaptor httpClientRequestCaptor = ArgumentCaptor.forClass(HttpRequest.class); + Mockito.when(httpClient.send(any(), any())).thenReturn(Mono.error(new ConnectTimeoutException())); RxGatewayStoreModel storeModel = new RxGatewayStoreModel( clientContext, @@ -263,7 +265,13 @@ public void validateApiType() throws Exception { "/fakeResourceFullName", ResourceType.Document); - storeModel.performRequest(dsr, HttpMethod.POST); + try { + storeModel.performRequest(dsr, HttpMethod.POST).block(); + fail("Request should fail"); + } catch (Exception e) { + //no-op + } + Mockito.verify(httpClient).send(httpClientRequestCaptor.capture(), any()); HttpRequest httpRequest = httpClientRequestCaptor.getValue(); HttpHeaders headers = ReflectionUtils.getHttpHeaders(httpRequest); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/GatewayAddressCacheTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/GatewayAddressCacheTest.java index 8cee9af57b8f1..91a7ef430535b 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/GatewayAddressCacheTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/GatewayAddressCacheTest.java @@ -135,6 +135,7 @@ public void getServerAddressesViaGateway(List partitionKeyRangeIds, null, null, ConnectionPolicy.getDefaultPolicy(), + null, null); for (int i = 0; i < 2; i++) { @@ -175,6 +176,7 @@ public void getMasterAddressesViaGatewayAsync(Protocol protocol) throws Exceptio null, null, ConnectionPolicy.getDefaultPolicy(), + null, null); for (int i = 0; i < 2; i++) { @@ -226,7 +228,8 @@ public void tryGetAddresses_ForDataPartitions(String partitionKeyRangeId, String null, null, ConnectionPolicy.getDefaultPolicy(), - proactiveOpenConnectionsProcessorMock); + proactiveOpenConnectionsProcessorMock, + null); RxDocumentServiceRequest req = RxDocumentServiceRequest.create(mockDiagnosticsClientContext(), OperationType.Create, ResourceType.Document, @@ -282,7 +285,8 @@ public void tryGetAddresses_ForDataPartitions_AddressCachedByOpenAsync_NoHttpReq null, null, ConnectionPolicy.getDefaultPolicy(), - proactiveOpenConnectionsProcessorMock); + proactiveOpenConnectionsProcessorMock, + null); String collectionRid = createdCollection.getResourceId(); @@ -350,7 +354,8 @@ public void tryGetAddresses_ForDataPartitions_ForceRefresh( null, null, ConnectionPolicy.getDefaultPolicy(), - proactiveOpenConnectionsProcessorMock); + proactiveOpenConnectionsProcessorMock, + null); String collectionRid = createdCollection.getResourceId(); @@ -432,7 +437,8 @@ public void tryGetAddresses_ForDataPartitions_Suboptimal_Refresh( null, null, ConnectionPolicy.getDefaultPolicy(), - proactiveOpenConnectionsProcessorMock); + proactiveOpenConnectionsProcessorMock, + null); String collectionRid = createdCollection.getResourceId(); @@ -555,7 +561,8 @@ public void tryGetAddresses_ForMasterPartition(Protocol protocol) throws Excepti null, null, null, - null); + null, + null); RxDocumentServiceRequest req = RxDocumentServiceRequest.create(mockDiagnosticsClientContext(), OperationType.Create, ResourceType.Database, @@ -606,6 +613,7 @@ public void tryGetAddresses_ForMasterPartition_MasterPartitionAddressAlreadyCach null, null, ConnectionPolicy.getDefaultPolicy(), + null, null); RxDocumentServiceRequest req = @@ -656,7 +664,8 @@ public void tryGetAddresses_ForMasterPartition_ForceRefresh() throws Exception { null, null, ConnectionPolicy.getDefaultPolicy(), - null); + null, + null); RxDocumentServiceRequest req = RxDocumentServiceRequest.create(mockDiagnosticsClientContext(), OperationType.Create, ResourceType.Database, @@ -713,6 +722,7 @@ public void tryGetAddresses_SuboptimalMasterPartition_NotStaleEnough_NoRefresh() ApiType.SQL, null, ConnectionPolicy.getDefaultPolicy(), + null, null); GatewayAddressCache spyCache = Mockito.spy(origCache); @@ -810,6 +820,7 @@ public void tryGetAddresses_SuboptimalMasterPartition_Stale_DoRefresh() throws E null, null, ConnectionPolicy.getDefaultPolicy(), + null, null); GatewayAddressCache spyCache = Mockito.spy(origCache); @@ -926,7 +937,8 @@ public void tryGetAddress_replicaValidationTests(boolean replicaValidationEnable null, null, ConnectionPolicy.getDefaultPolicy(), - proactiveOpenConnectionsProcessorMock); + proactiveOpenConnectionsProcessorMock, + null); RxDocumentServiceRequest req = RxDocumentServiceRequest.create( @@ -1082,7 +1094,8 @@ public void tryGetAddress_failedEndpointTests() throws Exception { null, null, ConnectionPolicy.getDefaultPolicy(), - proactiveOpenConnectionsProcessorMock); + proactiveOpenConnectionsProcessorMock, + null); RxDocumentServiceRequest req = RxDocumentServiceRequest.create( @@ -1143,7 +1156,8 @@ public void tryGetAddress_unhealthyStatus_forceRefresh() throws Exception { null, null, ConnectionPolicy.getDefaultPolicy(), - proactiveOpenConnectionsProcessorMock); + proactiveOpenConnectionsProcessorMock, + null); RxDocumentServiceRequest req = RxDocumentServiceRequest.create( @@ -1235,7 +1249,8 @@ public void validateReplicaAddressesTests() throws URISyntaxException, NoSuchMet null, null, ConnectionPolicy.getDefaultPolicy(), - proactiveOpenConnectionsProcessorMock); + proactiveOpenConnectionsProcessorMock, + null); Mockito.when(proactiveOpenConnectionsProcessorMock.submitOpenConnectionTaskOutsideLoop(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyInt())).thenReturn(dummyOpenConnectionsTask); @@ -1301,6 +1316,7 @@ public void mergeAddressesTests() throws URISyntaxException, NoSuchMethodExcepti null, null, ConnectionPolicy.getDefaultPolicy(), + null, null); // connected status diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/feedranges/FeedRangeTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/feedranges/FeedRangeTest.java index 9e4e89549a39f..4c479acc7d730 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/feedranges/FeedRangeTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/feedranges/FeedRangeTest.java @@ -6,7 +6,6 @@ import com.azure.cosmos.implementation.DocumentCollection; import com.azure.cosmos.implementation.HttpConstants; import com.azure.cosmos.implementation.IRoutingMapProvider; -import com.azure.cosmos.implementation.MetadataDiagnosticsContext; import com.azure.cosmos.implementation.OperationType; import com.azure.cosmos.implementation.PartitionKeyRange; import com.azure.cosmos.implementation.PartitionKeyRangeGoneException; @@ -40,7 +39,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; diff --git a/sdk/cosmos/azure-cosmos/CHANGELOG.md b/sdk/cosmos/azure-cosmos/CHANGELOG.md index ecff7cb43faa9..479e4e3288e46 100644 --- a/sdk/cosmos/azure-cosmos/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos/CHANGELOG.md @@ -9,6 +9,7 @@ #### Bugs Fixed #### Other Changes +* Added fault injection support for Gateway connection mode - See [PR 35378](https://github.com/Azure/azure-sdk-for-java/pull/35378) ### 4.47.0 (2023-06-26) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/BridgeInternal.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/BridgeInternal.java index 96aaf21164090..ebf74c17cecbb 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/BridgeInternal.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/BridgeInternal.java @@ -590,13 +590,6 @@ public static String recordAddressResolutionStart(CosmosDiagnostics cosmosDiagno forceCollectionRoutingMapRefresh); } - @Warning(value = INTERNAL_USE_ONLY_WARNING) - public static void recordAddressResolutionEnd(CosmosDiagnostics cosmosDiagnostics, - String identifier, - String errorMessage) { - cosmosDiagnostics.clientSideRequestStatistics().recordAddressResolutionEnd(identifier, errorMessage); - } - @Warning(value = INTERNAL_USE_ONLY_WARNING) public static List getContactedReplicas(CosmosDiagnostics cosmosDiagnostics) { return cosmosDiagnostics.clientSideRequestStatistics().getContactedReplicas(); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosDiagnostics.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosDiagnostics.java index b32a709854d53..20d8ade0b64ec 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosDiagnostics.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosDiagnostics.java @@ -6,6 +6,7 @@ import com.azure.cosmos.implementation.DiagnosticsClientContext; import com.azure.cosmos.implementation.FeedResponseDiagnostics; import com.azure.cosmos.implementation.ImplementationBridgeHelpers; +import com.azure.cosmos.implementation.RxDocumentServiceRequest; import com.azure.cosmos.implementation.guava25.collect.ImmutableList; import com.azure.cosmos.util.Beta; import com.fasterxml.jackson.core.JsonProcessingException; @@ -415,6 +416,26 @@ public void setSamplingRateSnapshot(CosmosDiagnostics cosmosDiagnostics, double public CosmosDiagnostics create(DiagnosticsClientContext clientContext, double samplingRate) { return new CosmosDiagnostics(clientContext).setSamplingRateSnapshot(samplingRate); } + + @Override + public void recordAddressResolutionEnd( + RxDocumentServiceRequest request, + String identifier, + String errorMessage, + long transportRequestId) { + if (request.requestContext.cosmosDiagnostics == null) { + return; + } + + request + .requestContext.cosmosDiagnostics + .clientSideRequestStatistics + .recordAddressResolutionEnd( + identifier, + errorMessage, + request.faultInjectionRequestContext.getFaultInjectionRuleId(transportRequestId), + request.faultInjectionRequestContext.getFaultInjectionRuleEvaluationResults(transportRequestId)); + } }); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosDiagnosticsContext.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosDiagnosticsContext.java index 8d13e9b707fee..21aa67d1718da 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosDiagnosticsContext.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosDiagnosticsContext.java @@ -626,28 +626,30 @@ private static void addRequestInfoForGatewayStatistics( ClientSideRequestStatistics requestStats, List requestInfo) { - ClientSideRequestStatistics.GatewayStatistics gatewayStats = requestStats.getGatewayStatistics(); + List gatewayStatsList = requestStats.getGatewayStatisticsList(); - if (gatewayStats == null) { + if (gatewayStatsList == null || gatewayStatsList.size() == 0) { return; } - CosmosDiagnosticsRequestInfo info = new CosmosDiagnosticsRequestInfo( - requestStats.getActivityId(), - null, - gatewayStats.getPartitionKeyRangeId(), - gatewayStats.getResourceType() + ":" + gatewayStats.getOperationType(), - requestStats.getRequestStartTimeUTC(), - requestStats.getDuration(), - null, - gatewayStats.getRequestCharge(), - gatewayStats.getResponsePayloadSizeInBytes(), - gatewayStats.getStatusCode(), - gatewayStats.getSubStatusCode(), - new ArrayList<>() - ); - - requestInfo.add(info); + for (ClientSideRequestStatistics.GatewayStatistics gatewayStats : gatewayStatsList) { + CosmosDiagnosticsRequestInfo info = new CosmosDiagnosticsRequestInfo( + requestStats.getActivityId(), + null, + gatewayStats.getPartitionKeyRangeId(), + gatewayStats.getResourceType() + ":" + gatewayStats.getOperationType(), + requestStats.getRequestStartTimeUTC(), + requestStats.getDuration(), + null, + gatewayStats.getRequestCharge(), + gatewayStats.getResponsePayloadSizeInBytes(), + gatewayStats.getStatusCode(), + gatewayStats.getSubStatusCode(), + new ArrayList<>() + ); + + requestInfo.add(info); + } } private static void addRequestInfoForStoreResponses( diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ClientSideRequestStatistics.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ClientSideRequestStatistics.java index 099efabf6319e..ae6298fa0adf9 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ClientSideRequestStatistics.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ClientSideRequestStatistics.java @@ -8,6 +8,7 @@ import com.azure.cosmos.implementation.directconnectivity.StoreResultDiagnostics; import com.azure.cosmos.implementation.faultinjection.FaultInjectionRequestContext; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -45,7 +46,7 @@ public class ClientSideRequestStatistics { private Set locationEndpointsContacted; private RetryContext retryContext; private FaultInjectionRequestContext requestContext; - private GatewayStatistics gatewayStatistics; + private List gatewayStatisticsList; private MetadataDiagnosticsContext metadataDiagnosticsContext; private SerializationDiagnosticsContext serializationDiagnosticsContext; private int requestPayloadSizeInBytes = 0; @@ -59,6 +60,7 @@ public ClientSideRequestStatistics(DiagnosticsClientContext diagnosticsClientCon this.requestEndTimeUTC = Instant.now(); this.responseStatisticsList = new ArrayList<>(); this.supplementalResponseStatisticsList = new ArrayList<>(); + this.gatewayStatisticsList = new ArrayList<>(); this.addressResolutionStatistics = new HashMap<>(); this.contactedReplicas = Collections.synchronizedList(new ArrayList<>()); this.failedReplicas = Collections.synchronizedSet(new HashSet<>()); @@ -78,6 +80,7 @@ public ClientSideRequestStatistics(ClientSideRequestStatistics toBeCloned) { this.requestEndTimeUTC = toBeCloned.requestEndTimeUTC; this.responseStatisticsList = new ArrayList<>(toBeCloned.responseStatisticsList); this.supplementalResponseStatisticsList = new ArrayList<>(toBeCloned.supplementalResponseStatisticsList); + this.gatewayStatisticsList = new ArrayList<>(toBeCloned.gatewayStatisticsList); this.addressResolutionStatistics = new HashMap<>(toBeCloned.addressResolutionStatistics); this.contactedReplicas = Collections.synchronizedList(new ArrayList<>(toBeCloned.contactedReplicas)); this.failedReplicas = Collections.synchronizedSet(new HashSet<>(toBeCloned.failedReplicas)); @@ -164,7 +167,10 @@ public void recordResponse(RxDocumentServiceRequest request, StoreResultDiagnost } public void recordGatewayResponse( - RxDocumentServiceRequest rxDocumentServiceRequest, StoreResponseDiagnostics storeResponseDiagnostics, GlobalEndpointManager globalEndpointManager) { + RxDocumentServiceRequest rxDocumentServiceRequest, + StoreResponseDiagnostics storeResponseDiagnostics, + GlobalEndpointManager globalEndpointManager) { + Instant responseTime = Instant.now(); synchronized (this) { @@ -183,22 +189,27 @@ public void recordGatewayResponse( this.locationEndpointsContacted.add(locationEndPoint); } - this.gatewayStatistics = new GatewayStatistics(); + GatewayStatistics gatewayStatistics = new GatewayStatistics(); if (rxDocumentServiceRequest != null) { - this.gatewayStatistics.operationType = rxDocumentServiceRequest.getOperationType(); - this.gatewayStatistics.resourceType = rxDocumentServiceRequest.getResourceType(); + gatewayStatistics.operationType = rxDocumentServiceRequest.getOperationType(); + gatewayStatistics.resourceType = rxDocumentServiceRequest.getResourceType(); this.requestPayloadSizeInBytes = rxDocumentServiceRequest.getContentLength(); } - this.gatewayStatistics.statusCode = storeResponseDiagnostics.getStatusCode(); - this.gatewayStatistics.subStatusCode = storeResponseDiagnostics.getSubStatusCode(); - this.gatewayStatistics.sessionToken = storeResponseDiagnostics.getSessionTokenAsString(); - this.gatewayStatistics.requestCharge = storeResponseDiagnostics.getRequestCharge(); - this.gatewayStatistics.requestTimeline = storeResponseDiagnostics.getRequestTimeline(); - this.gatewayStatistics.partitionKeyRangeId = storeResponseDiagnostics.getPartitionKeyRangeId(); - this.gatewayStatistics.exceptionMessage = storeResponseDiagnostics.getExceptionMessage(); - this.gatewayStatistics.exceptionResponseHeaders = storeResponseDiagnostics.getExceptionResponseHeaders(); - this.gatewayStatistics.responsePayloadSizeInBytes = storeResponseDiagnostics.getResponsePayloadLength(); + gatewayStatistics.statusCode = storeResponseDiagnostics.getStatusCode(); + gatewayStatistics.subStatusCode = storeResponseDiagnostics.getSubStatusCode(); + gatewayStatistics.sessionToken = storeResponseDiagnostics.getSessionTokenAsString(); + gatewayStatistics.requestCharge = storeResponseDiagnostics.getRequestCharge(); + gatewayStatistics.requestTimeline = storeResponseDiagnostics.getRequestTimeline(); + gatewayStatistics.partitionKeyRangeId = storeResponseDiagnostics.getPartitionKeyRangeId(); + gatewayStatistics.exceptionMessage = storeResponseDiagnostics.getExceptionMessage(); + gatewayStatistics.exceptionResponseHeaders = storeResponseDiagnostics.getExceptionResponseHeaders(); + gatewayStatistics.responsePayloadSizeInBytes = storeResponseDiagnostics.getResponsePayloadLength(); + gatewayStatistics.faultInjectionRuleId = storeResponseDiagnostics.getFaultInjectionRuleId(); + gatewayStatistics.faultInjectionEvaluationResults = storeResponseDiagnostics.getFaultInjectionEvaluationResults(); + this.activityId = storeResponseDiagnostics.getActivityId(); + + this.gatewayStatisticsList.add(gatewayStatistics); } } @@ -228,7 +239,11 @@ public String recordAddressResolutionStart( return identifier; } - public void recordAddressResolutionEnd(String identifier, String exceptionMessage) { + public void recordAddressResolutionEnd( + String identifier, + String exceptionMessage, + String faultInjectionId, + List faultInjectionEvaluationResult) { if (StringUtils.isEmpty(identifier)) { return; } @@ -248,6 +263,8 @@ public void recordAddressResolutionEnd(String identifier, String exceptionMessag resolutionStatistics.endTimeUTC = responseTime; resolutionStatistics.exceptionMessage = exceptionMessage; resolutionStatistics.inflightRequest = false; + resolutionStatistics.faultInjectionRuleId = faultInjectionId; + resolutionStatistics.faultInjectionEvaluationResults = faultInjectionEvaluationResult; } } @@ -475,11 +492,7 @@ public String getUserAgent() { public int getMaxResponsePayloadSizeInBytes() { if (responseStatisticsList == null || responseStatisticsList.isEmpty()) { - if (this.gatewayStatistics != null) { - return this.gatewayStatistics.responsePayloadSizeInBytes; - } - - return 0; + return this.getMaxResponsePayloadSizeInBytesFromGateway(); } int maxResponsePayloadSizeInBytes = 0; @@ -498,6 +511,19 @@ public int getMaxResponsePayloadSizeInBytes() { return maxResponsePayloadSizeInBytes; } + private int getMaxResponsePayloadSizeInBytesFromGateway() { + if (this.gatewayStatisticsList == null || this.gatewayStatisticsList.size() == 0) { + return 0; + } + + int maxResponsePayloadSizeInBytes = 0; + for (GatewayStatistics gatewayStatistics : this.gatewayStatisticsList) { + maxResponsePayloadSizeInBytes = Math.max(maxResponsePayloadSizeInBytes, gatewayStatistics.responsePayloadSizeInBytes); + } + + return maxResponsePayloadSizeInBytes; + } + public List getSupplementalResponseStatisticsList() { return supplementalResponseStatisticsList; } @@ -510,8 +536,8 @@ public Map getAddressResolutionStatistics() return addressResolutionStatistics; } - public GatewayStatistics getGatewayStatistics() { - return gatewayStatistics; + public List getGatewayStatisticsList() { + return this.gatewayStatisticsList; } public ClientSideRequestStatistics setSamplingRateSnapshot(double samplingRateSnapshot) { @@ -602,7 +628,7 @@ public void serialize( generator.writeObjectField("retryContext", statistics.retryContext); generator.writeObjectField("metadataDiagnosticsContext", statistics.getMetadataDiagnosticsContext()); generator.writeObjectField("serializationDiagnosticsContext", statistics.getSerializationDiagnosticsContext()); - generator.writeObjectField("gatewayStatistics", statistics.gatewayStatistics); + generator.writeObjectField("gatewayStatisticsList", statistics.gatewayStatisticsList); generator.writeObjectField("samplingRateSnapshot", statistics.samplingRateSnapshot); try { @@ -643,6 +669,10 @@ public static class AddressResolutionStatistics { private boolean forceRefresh; @JsonSerialize private boolean forceCollectionRoutingMapRefresh; + @JsonInclude(JsonInclude.Include.NON_NULL) + private String faultInjectionRuleId; + @JsonInclude(JsonInclude.Include.NON_NULL) + private List faultInjectionEvaluationResults; // If one replica return error we start address call in parallel, // on other replica valid response, we end the current user request, @@ -677,8 +707,17 @@ public boolean isForceRefresh() { public boolean isForceCollectionRoutingMapRefresh() { return forceCollectionRoutingMapRefresh; } + + public String getFaultInjectionRuleId() { + return faultInjectionRuleId; + } + + public List getFaultInjectionEvaluationResults() { + return faultInjectionEvaluationResults; + } } + @JsonSerialize(using = GatewayStatistics.GatewayStatisticsSerializer.class) public static class GatewayStatistics { private String sessionToken; private OperationType operationType; @@ -692,6 +731,8 @@ public static class GatewayStatistics { private String exceptionResponseHeaders; private int responsePayloadSizeInBytes; + private String faultInjectionRuleId; + private List faultInjectionEvaluationResults; public String getSessionToken() { return sessionToken; @@ -733,7 +774,69 @@ public String getExceptionResponseHeaders() { return exceptionResponseHeaders; } - public int getResponsePayloadSizeInBytes() { return this.responsePayloadSizeInBytes; } + public int getResponsePayloadSizeInBytes() { + return this.responsePayloadSizeInBytes; + } + + public String getFaultInjectionRuleId() { + return faultInjectionRuleId; + } + + public List getFaultInjectionEvaluationResults() { + return faultInjectionEvaluationResults; + } + + public static class GatewayStatisticsSerializer extends StdSerializer { + private static final long serialVersionUID = 1L; + + public GatewayStatisticsSerializer(){ + super(GatewayStatistics.class); + } + + @Override + public void serialize(GatewayStatistics gatewayStatistics, + JsonGenerator jsonGenerator, + SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeStartObject(); + jsonGenerator.writeStringField("sessionToken", gatewayStatistics.getSessionToken()); + jsonGenerator.writeStringField("operationType", gatewayStatistics.getOperationType().toString()); + jsonGenerator.writeStringField("resourceType", gatewayStatistics.getResourceType().toString()); + jsonGenerator.writeNumberField("statusCode", gatewayStatistics.getStatusCode()); + jsonGenerator.writeNumberField("subStatusCode", gatewayStatistics.getSubStatusCode()); + jsonGenerator.writeNumberField("requestCharge", gatewayStatistics.getRequestCharge()); + jsonGenerator.writeObjectField("requestTimeline", gatewayStatistics.getRequestTimeline()); + jsonGenerator.writeStringField("partitionKeyRangeId", gatewayStatistics.getPartitionKeyRangeId()); + jsonGenerator.writeNumberField("responsePayloadSizeInBytes", gatewayStatistics.getResponsePayloadSizeInBytes()); + this.writeNonNullStringField(jsonGenerator, "exceptionMessage", gatewayStatistics.getExceptionMessage()); + this.writeNonNullStringField(jsonGenerator, "exceptionResponseHeaders", gatewayStatistics.getExceptionResponseHeaders()); + this.writeNonNullStringField(jsonGenerator, "faultInjectionRuleId", gatewayStatistics.getFaultInjectionRuleId()); + + if (StringUtils.isEmpty(gatewayStatistics.getFaultInjectionRuleId())) { + this.writeNonEmptyStringArrayField( + jsonGenerator, + "faultInjectionEvaluationResults", + gatewayStatistics.getFaultInjectionEvaluationResults()); + } + + jsonGenerator.writeEndObject(); + } + + private void writeNonNullStringField(JsonGenerator jsonGenerator, String fieldName, String value) throws IOException { + if (value == null) { + return; + } + + jsonGenerator.writeStringField(fieldName, value); + } + + private void writeNonEmptyStringArrayField(JsonGenerator jsonGenerator, String fieldName, List values) throws IOException { + if (values == null || values.isEmpty()) { + return; + } + + jsonGenerator.writeObjectField(fieldName, values); + } + } } public static CosmosDiagnosticsSystemUsageSnapshot fetchSystemInformation() { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/DiagnosticsProvider.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/DiagnosticsProvider.java index fe994af9b7bd4..37ac4db324383 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/DiagnosticsProvider.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/DiagnosticsProvider.java @@ -914,15 +914,13 @@ private void addClientSideRequestStatisticsOnTracerEvent( } //adding gateway statistics - if (clientSideRequestStatistics.getGatewayStatistics() != null) { + for (ClientSideRequestStatistics.GatewayStatistics gatewayStats : clientSideRequestStatistics.getGatewayStatisticsList()) { attributes = new HashMap<>(); - attributes.put(JSON_STRING, - mapper.writeValueAsString(clientSideRequestStatistics.getGatewayStatistics())); + attributes.put(JSON_STRING, mapper.writeValueAsString(gatewayStats)); OffsetDateTime requestStartTime = OffsetDateTime.ofInstant(clientSideRequestStatistics.getRequestStartTimeUTC(), ZoneOffset.UTC); - if (clientSideRequestStatistics.getGatewayStatistics().getRequestTimeline() != null) { - for (RequestTimeline.Event event : - clientSideRequestStatistics.getGatewayStatistics().getRequestTimeline()) { + if (gatewayStats.getRequestTimeline() != null) { + for (RequestTimeline.Event event : gatewayStats.getRequestTimeline()) { if (event.getName().equals("created")) { requestStartTime = OffsetDateTime.ofInstant(event.getStartTime(), ZoneOffset.UTC); break; @@ -992,9 +990,9 @@ private void addClientSideRequestStatisticsOnTracerEvent( + clientSideRequestStatistics.getResponseStatisticsList().get(0).getStoreResult().getStoreResponseDiagnostics().getPartitionKeyRangeId(); this.addEvent(eventName, attributes, OffsetDateTime.ofInstant(clientSideRequestStatistics.getRequestStartTimeUTC(), ZoneOffset.UTC), context); - } else if (clientSideRequestStatistics.getGatewayStatistics() != null) { + } else if (clientSideRequestStatistics.getGatewayStatisticsList() != null && clientSideRequestStatistics.getGatewayStatisticsList().size() > 0) { String eventName = - "Diagnostics for PKRange " + clientSideRequestStatistics.getGatewayStatistics().getPartitionKeyRangeId(); + "Diagnostics for PKRange " + clientSideRequestStatistics.getGatewayStatisticsList().get(0).getPartitionKeyRangeId(); this.addEvent(eventName, attributes, OffsetDateTime.ofInstant(clientSideRequestStatistics.getRequestStartTimeUTC(), ZoneOffset.UTC), context); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java index 938452b6c4c4a..cbdc450a3b63e 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java @@ -20,9 +20,10 @@ import com.azure.cosmos.CosmosDiagnosticsThresholds; import com.azure.cosmos.CosmosEndToEndOperationLatencyPolicyConfig; import com.azure.cosmos.CosmosException; -import com.azure.cosmos.SessionRetryOptions; +import com.azure.cosmos.CosmosRegionSwitchHint; import com.azure.cosmos.DirectConnectionConfig; import com.azure.cosmos.GlobalThroughputControlConfig; +import com.azure.cosmos.SessionRetryOptions; import com.azure.cosmos.ThroughputControlGroupConfig; import com.azure.cosmos.implementation.batch.ItemBatchOperation; import com.azure.cosmos.implementation.batch.PartitionScopeThresholds; @@ -30,8 +31,8 @@ import com.azure.cosmos.implementation.clienttelemetry.CosmosMeterOptions; import com.azure.cosmos.implementation.clienttelemetry.MetricCategory; import com.azure.cosmos.implementation.clienttelemetry.TagName; -import com.azure.cosmos.implementation.directconnectivity.Uri; import com.azure.cosmos.implementation.directconnectivity.ContainerDirectConnectionMetadata; +import com.azure.cosmos.implementation.directconnectivity.Uri; import com.azure.cosmos.implementation.directconnectivity.rntbd.RntbdChannelStatistics; import com.azure.cosmos.implementation.faultinjection.IFaultInjectorProvider; import com.azure.cosmos.implementation.patch.PatchOperation; @@ -54,7 +55,6 @@ import com.azure.cosmos.models.CosmosMetricName; import com.azure.cosmos.models.CosmosPatchOperations; import com.azure.cosmos.models.CosmosQueryRequestOptions; -import com.azure.cosmos.CosmosRegionSwitchHint; import com.azure.cosmos.models.FeedResponse; import com.azure.cosmos.models.ModelBridgeInternal; import com.azure.cosmos.models.PartitionKey; @@ -734,6 +734,11 @@ void addClientSideDiagnosticsToFeed( void setSamplingRateSnapshot(CosmosDiagnostics cosmosDiagnostics, double samplingRate); CosmosDiagnostics create(DiagnosticsClientContext clientContext, double samplingRate); + void recordAddressResolutionEnd( + RxDocumentServiceRequest request, + String identifier, + String errorMessage, + long transportRequestId); } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/InvalidPartitionExceptionRetryPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/InvalidPartitionExceptionRetryPolicy.java index 0f34091db725b..48686f8835b2f 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/InvalidPartitionExceptionRetryPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/InvalidPartitionExceptionRetryPolicy.java @@ -3,8 +3,8 @@ package com.azure.cosmos.implementation; import com.azure.cosmos.BridgeInternal; -import com.azure.cosmos.implementation.caches.RxCollectionCache; import com.azure.cosmos.CosmosException; +import com.azure.cosmos.implementation.caches.RxCollectionCache; import reactor.core.publisher.Mono; import java.time.Duration; diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/PartitionKeyMismatchRetryPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/PartitionKeyMismatchRetryPolicy.java index 0b0940ac3fb1c..e3149388292d9 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/PartitionKeyMismatchRetryPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/PartitionKeyMismatchRetryPolicy.java @@ -3,8 +3,8 @@ package com.azure.cosmos.implementation; import com.azure.cosmos.BridgeInternal; -import com.azure.cosmos.implementation.caches.RxClientCollectionCache; import com.azure.cosmos.CosmosException; +import com.azure.cosmos.implementation.caches.RxClientCollectionCache; import reactor.core.publisher.Mono; import java.time.Duration; diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/PartitionKeyRangeGoneRetryPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/PartitionKeyRangeGoneRetryPolicy.java index 44de1d44a2dfa..9344eaf5e2a7f 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/PartitionKeyRangeGoneRetryPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/PartitionKeyRangeGoneRetryPolicy.java @@ -3,12 +3,10 @@ package com.azure.cosmos.implementation; import com.azure.cosmos.BridgeInternal; +import com.azure.cosmos.CosmosException; import com.azure.cosmos.implementation.caches.IPartitionKeyRangeCache; import com.azure.cosmos.implementation.caches.RxCollectionCache; import com.azure.cosmos.implementation.routing.CollectionRoutingMap; -import com.azure.cosmos.CosmosException; -import com.azure.cosmos.models.ModelBridgeInternal; -import com.azure.cosmos.models.CosmosQueryRequestOptions; import reactor.core.publisher.Mono; import java.time.Duration; diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java index 7cfe69ed31c1c..62cb25c00f120 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java @@ -11,10 +11,10 @@ import com.azure.cosmos.ConsistencyLevel; import com.azure.cosmos.CosmosContainerProactiveInitConfig; import com.azure.cosmos.CosmosDiagnostics; +import com.azure.cosmos.CosmosEndToEndOperationLatencyPolicyConfig; import com.azure.cosmos.CosmosException; -import com.azure.cosmos.SessionRetryOptions; import com.azure.cosmos.DirectConnectionConfig; -import com.azure.cosmos.CosmosEndToEndOperationLatencyPolicyConfig; +import com.azure.cosmos.SessionRetryOptions; import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; import com.azure.cosmos.implementation.apachecommons.lang.tuple.ImmutablePair; import com.azure.cosmos.implementation.batch.BatchResponseParser; @@ -4519,11 +4519,13 @@ public ConsistencyLevel getDefaultConsistencyLevelOfAccount() { @Override public void configureFaultInjectorProvider(IFaultInjectorProvider injectorProvider) { checkNotNull(injectorProvider, "Argument 'injectorProvider' can not be null"); + if (this.connectionPolicy.getConnectionMode() == ConnectionMode.DIRECT) { - this.storeModel.configureFaultInjectorProvider(injectorProvider); - } else { - throw new IllegalArgumentException("configureFaultInjectorProvider is not supported for gateway mode"); + this.storeModel.configureFaultInjectorProvider(injectorProvider, this.configs); + this.addressResolver.configureFaultInjectorProvider(injectorProvider, this.configs); } + + this.gatewayProxy.configureFaultInjectorProvider(injectorProvider, this.configs); } @Override diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java index cb07f10c5a665..2f299fa669e26 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java @@ -6,7 +6,6 @@ import com.azure.cosmos.ConsistencyLevel; import com.azure.cosmos.CosmosContainerProactiveInitConfig; import com.azure.cosmos.CosmosException; -import com.azure.cosmos.implementation.apachecommons.lang.NotImplementedException; import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; import com.azure.cosmos.implementation.caches.RxClientCollectionCache; import com.azure.cosmos.implementation.caches.RxPartitionKeyRangeCache; @@ -15,6 +14,7 @@ import com.azure.cosmos.implementation.directconnectivity.RequestHelper; import com.azure.cosmos.implementation.directconnectivity.StoreResponse; import com.azure.cosmos.implementation.directconnectivity.WebExceptionUtility; +import com.azure.cosmos.implementation.faultinjection.GatewayServerErrorInjector; import com.azure.cosmos.implementation.faultinjection.IFaultInjectorProvider; import com.azure.cosmos.implementation.http.HttpClient; import com.azure.cosmos.implementation.http.HttpHeaders; @@ -67,6 +67,7 @@ public class RxGatewayStoreModel implements RxStoreModel { private RxPartitionKeyRangeCache partitionKeyRangeCache; private GatewayServiceConfigurationReader gatewayServiceConfigurationReader; private RxClientCollectionCache collectionCache; + private GatewayServerErrorInjector gatewayServerErrorInjector; public RxGatewayStoreModel( DiagnosticsClientContext clientContext, @@ -247,6 +248,11 @@ public Mono performRequestInternal(RxDocumentServiceR } Mono httpResponseMono = this.httpClient.send(httpRequest, responseTimeout); + + if (this.gatewayServerErrorInjector != null) { + httpResponseMono = this.gatewayServerErrorInjector.injectGatewayErrors(responseTimeout, httpRequest, request, httpResponseMono); + } + return toDocumentServiceResponse(httpResponseMono, request, httpRequest); } catch (Exception e) { @@ -354,9 +360,24 @@ private Mono toDocumentServiceResponse(Mono toDocumentServiceResponse(Mono submitOpenConnectionTasksAndInitCaches(CosmosContainerProactiv } @Override - public void configureFaultInjectorProvider(IFaultInjectorProvider injectorProvider) { - throw new NotImplementedException("configureFaultInjectorProvider is not supported in RxGatewayStoreModel"); + public void configureFaultInjectorProvider(IFaultInjectorProvider injectorProvider, Configs configs) { + if (this.gatewayServerErrorInjector == null) { + this.gatewayServerErrorInjector = new GatewayServerErrorInjector(configs); + } + + this.gatewayServerErrorInjector.registerServerErrorInjector(injectorProvider.getServerErrorInjector()); } @Override diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxStoreModel.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxStoreModel.java index 3161d4595068f..6629aa860928d 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxStoreModel.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxStoreModel.java @@ -72,7 +72,7 @@ default Mono processMessage(RxDocumentServiceRequest * * @param injectorProvider the fault injector provider. */ - void configureFaultInjectorProvider(IFaultInjectorProvider injectorProvider); + void configureFaultInjectorProvider(IFaultInjectorProvider injectorProvider, Configs configs); void recordOpenConnectionsAndInitCachesCompleted(List cosmosContainerIdentities); void recordOpenConnectionsAndInitCachesStarted(List cosmosContainerIdentities); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/caches/IPartitionKeyRangeCache.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/caches/IPartitionKeyRangeCache.java index 25c63a91f3a35..cb1034c25b3ed 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/caches/IPartitionKeyRangeCache.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/caches/IPartitionKeyRangeCache.java @@ -2,13 +2,13 @@ // Licensed under the MIT License. package com.azure.cosmos.implementation.caches; +import com.azure.cosmos.implementation.ICollectionRoutingMapCache; +import com.azure.cosmos.implementation.IRoutingMapProvider; import com.azure.cosmos.implementation.MetadataDiagnosticsContext; +import com.azure.cosmos.implementation.PartitionKeyRange; import com.azure.cosmos.implementation.Utils; import com.azure.cosmos.implementation.routing.CollectionRoutingMap; import com.azure.cosmos.implementation.routing.Range; -import com.azure.cosmos.implementation.ICollectionRoutingMapCache; -import com.azure.cosmos.implementation.IRoutingMapProvider; -import com.azure.cosmos.implementation.PartitionKeyRange; import reactor.core.publisher.Mono; import java.util.List; diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/caches/RxClientCollectionCache.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/caches/RxClientCollectionCache.java index 8bf914b667032..4d8e36148be3d 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/caches/RxClientCollectionCache.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/caches/RxClientCollectionCache.java @@ -3,20 +3,20 @@ package com.azure.cosmos.implementation.caches; import com.azure.cosmos.BridgeInternal; +import com.azure.cosmos.implementation.AuthorizationTokenType; +import com.azure.cosmos.implementation.ClearingSessionContainerClientRetryPolicy; import com.azure.cosmos.implementation.DiagnosticsClientContext; -import com.azure.cosmos.implementation.RequestVerb; import com.azure.cosmos.implementation.DocumentClientRetryPolicy; import com.azure.cosmos.implementation.DocumentCollection; -import com.azure.cosmos.implementation.ISessionContainer; -import com.azure.cosmos.implementation.AuthorizationTokenType; -import com.azure.cosmos.implementation.ClearingSessionContainerClientRetryPolicy; import com.azure.cosmos.implementation.HttpConstants; import com.azure.cosmos.implementation.IAuthorizationTokenProvider; import com.azure.cosmos.implementation.IRetryPolicyFactory; +import com.azure.cosmos.implementation.ISessionContainer; import com.azure.cosmos.implementation.MetadataDiagnosticsContext; import com.azure.cosmos.implementation.ObservableHelper; import com.azure.cosmos.implementation.OperationType; import com.azure.cosmos.implementation.PathsHelper; +import com.azure.cosmos.implementation.RequestVerb; import com.azure.cosmos.implementation.ResourceType; import com.azure.cosmos.implementation.RxDocumentServiceRequest; import com.azure.cosmos.implementation.RxDocumentServiceResponse; @@ -24,12 +24,9 @@ import com.azure.cosmos.implementation.Utils; import reactor.core.publisher.Mono; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.time.Instant; -import java.time.ZoneOffset; import java.util.HashMap; import java.util.Map; diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/caches/RxCollectionCache.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/caches/RxCollectionCache.java index bc5729b903e73..5cbc046082b5b 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/caches/RxCollectionCache.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/caches/RxCollectionCache.java @@ -4,17 +4,17 @@ import com.azure.cosmos.BridgeInternal; import com.azure.cosmos.implementation.CosmosClientMetadataCachesSnapshot; -import com.azure.cosmos.implementation.MetadataDiagnosticsContext; -import com.azure.cosmos.implementation.Utils; -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.implementation.routing.PartitionKeyRangeIdentity; import com.azure.cosmos.implementation.DocumentCollection; import com.azure.cosmos.implementation.InvalidPartitionException; +import com.azure.cosmos.implementation.MetadataDiagnosticsContext; import com.azure.cosmos.implementation.NotFoundException; import com.azure.cosmos.implementation.PathsHelper; import com.azure.cosmos.implementation.RMResources; import com.azure.cosmos.implementation.ResourceId; import com.azure.cosmos.implementation.RxDocumentServiceRequest; +import com.azure.cosmos.implementation.Utils; +import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; +import com.azure.cosmos.implementation.routing.PartitionKeyRangeIdentity; import com.azure.cosmos.models.ModelBridgeInternal; import reactor.core.Exceptions; import reactor.core.publisher.Mono; diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/caches/RxPartitionKeyRangeCache.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/caches/RxPartitionKeyRangeCache.java index 67ad900c20570..efdf4d590156c 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/caches/RxPartitionKeyRangeCache.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/caches/RxPartitionKeyRangeCache.java @@ -6,7 +6,6 @@ import com.azure.cosmos.implementation.DiagnosticsClientContext; import com.azure.cosmos.implementation.DocumentCollection; import com.azure.cosmos.implementation.Exceptions; -import com.azure.cosmos.implementation.HttpConstants; import com.azure.cosmos.implementation.MetadataDiagnosticsContext; import com.azure.cosmos.implementation.NotFoundException; import com.azure.cosmos.implementation.OperationType; diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/clienttelemetry/ClientTelemetryMetrics.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/clienttelemetry/ClientTelemetryMetrics.java index 02cd31f33adc4..18bf38ed032f0 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/clienttelemetry/ClientTelemetryMetrics.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/clienttelemetry/ClientTelemetryMetrics.java @@ -456,7 +456,8 @@ public void recordOperation( recordGatewayStatistics( diagnosticsContext, cosmosAsyncClient, - requestStatistics.getDuration(), requestStatistics.getGatewayStatistics()); + requestStatistics.getDuration(), + requestStatistics.getGatewayStatisticsList()); recordAddressResolutionStatistics( diagnosticsContext, cosmosAsyncClient, @@ -958,9 +959,11 @@ private void recordGatewayStatistics( CosmosDiagnosticsContext ctx, CosmosAsyncClient client, Duration latency, - ClientSideRequestStatistics.GatewayStatistics gatewayStatistics) { + List gatewayStatisticsList) { - if (gatewayStatistics == null || !this.metricCategories.contains(MetricCategory.RequestSummary)) { + if (gatewayStatisticsList == null + || gatewayStatisticsList.size() == 0 + || !this.metricCategories.contains(MetricCategory.RequestSummary)) { return; } @@ -971,74 +974,76 @@ private void recordGatewayStatistics( metricTagNamesForGateway.remove(TagName.PartitionId); metricTagNamesForGateway.remove(TagName.ReplicaId); - Tags requestTags = operationTags.and( - createRequestTags( - metricTagNamesForGateway, - gatewayStatistics.getPartitionKeyRangeId(), - gatewayStatistics.getStatusCode(), - gatewayStatistics.getSubStatusCode(), - gatewayStatistics.getResourceType().toString(), - gatewayStatistics.getOperationType().toString(), - null, - null, - null) - ); - - CosmosMeterOptions reqOptions = clientAccessor.getMeterOptions( - client, - CosmosMetricName.REQUEST_SUMMARY_GATEWAY_REQUESTS); - if (reqOptions.isEnabled() && - (!reqOptions.isDiagnosticThresholdsFilteringEnabled() || ctx.isThresholdViolated())) { - Counter requestCounter = Counter - .builder(reqOptions.getMeterName().toString()) - .baseUnit("requests") - .description("Gateway requests") - .tags(getEffectiveTags(requestTags, reqOptions)) - .register(compositeRegistry); - requestCounter.increment(); - } + for (ClientSideRequestStatistics.GatewayStatistics gatewayStats : gatewayStatisticsList) { + Tags requestTags = operationTags.and( + createRequestTags( + metricTagNamesForGateway, + gatewayStats.getPartitionKeyRangeId(), + gatewayStats.getStatusCode(), + gatewayStats.getSubStatusCode(), + gatewayStats.getResourceType().toString(), + gatewayStats.getOperationType().toString(), + null, + null, + null) + ); - CosmosMeterOptions ruOptions = clientAccessor.getMeterOptions( - client, - CosmosMetricName.REQUEST_SUMMARY_GATEWAY_REQUEST_CHARGE); - if (ruOptions.isEnabled() && - (!ruOptions.isDiagnosticThresholdsFilteringEnabled() || ctx.isThresholdViolated())) { - double requestCharge = gatewayStatistics.getRequestCharge(); - DistributionSummary requestChargeMeter = DistributionSummary - .builder(ruOptions.getMeterName().toString()) - .baseUnit("RU (request unit)") - .description("Gateway Request RU charge") - .maximumExpectedValue(100_000d) - .publishPercentiles(ruOptions.getPercentiles()) - .publishPercentileHistogram(ruOptions.isHistogramPublishingEnabled()) - .tags(getEffectiveTags(requestTags, ruOptions)) - .register(compositeRegistry); - requestChargeMeter.record(Math.min(requestCharge, 100_000d)); - } + CosmosMeterOptions reqOptions = clientAccessor.getMeterOptions( + client, + CosmosMetricName.REQUEST_SUMMARY_GATEWAY_REQUESTS); + if (reqOptions.isEnabled() && + (!reqOptions.isDiagnosticThresholdsFilteringEnabled() || ctx.isThresholdViolated())) { + Counter requestCounter = Counter + .builder(reqOptions.getMeterName().toString()) + .baseUnit("requests") + .description("Gateway requests") + .tags(getEffectiveTags(requestTags, reqOptions)) + .register(compositeRegistry); + requestCounter.increment(); + } - if (latency != null) { - CosmosMeterOptions latencyOptions = clientAccessor.getMeterOptions( + CosmosMeterOptions ruOptions = clientAccessor.getMeterOptions( client, - CosmosMetricName.REQUEST_SUMMARY_GATEWAY_LATENCY); - if (latencyOptions.isEnabled() && - (!latencyOptions.isDiagnosticThresholdsFilteringEnabled() || ctx.isThresholdViolated())) { - Timer requestLatencyMeter = Timer - .builder(latencyOptions.getMeterName().toString()) - .description("Gateway Request latency") - .maximumExpectedValue(Duration.ofSeconds(300)) - .publishPercentiles(latencyOptions.getPercentiles()) - .publishPercentileHistogram(latencyOptions.isHistogramPublishingEnabled()) - .tags(getEffectiveTags(requestTags, latencyOptions)) + CosmosMetricName.REQUEST_SUMMARY_GATEWAY_REQUEST_CHARGE); + if (ruOptions.isEnabled() && + (!ruOptions.isDiagnosticThresholdsFilteringEnabled() || ctx.isThresholdViolated())) { + double requestCharge = gatewayStats.getRequestCharge(); + DistributionSummary requestChargeMeter = DistributionSummary + .builder(ruOptions.getMeterName().toString()) + .baseUnit("RU (request unit)") + .description("Gateway Request RU charge") + .maximumExpectedValue(100_000d) + .publishPercentiles(ruOptions.getPercentiles()) + .publishPercentileHistogram(ruOptions.isHistogramPublishingEnabled()) + .tags(getEffectiveTags(requestTags, ruOptions)) .register(compositeRegistry); - requestLatencyMeter.record(latency); + requestChargeMeter.record(Math.min(requestCharge, 100_000d)); } - } - recordRequestTimeline( - ctx, - client, - CosmosMetricName.REQUEST_DETAILS_GATEWAY_TIMELINE, - gatewayStatistics.getRequestTimeline(), requestTags); + if (latency != null) { + CosmosMeterOptions latencyOptions = clientAccessor.getMeterOptions( + client, + CosmosMetricName.REQUEST_SUMMARY_GATEWAY_LATENCY); + if (latencyOptions.isEnabled() && + (!latencyOptions.isDiagnosticThresholdsFilteringEnabled() || ctx.isThresholdViolated())) { + Timer requestLatencyMeter = Timer + .builder(latencyOptions.getMeterName().toString()) + .description("Gateway Request latency") + .maximumExpectedValue(Duration.ofSeconds(300)) + .publishPercentiles(latencyOptions.getPercentiles()) + .publishPercentileHistogram(latencyOptions.isHistogramPublishingEnabled()) + .tags(getEffectiveTags(requestTags, latencyOptions)) + .register(compositeRegistry); + requestLatencyMeter.record(latency); + } + } + + recordRequestTimeline( + ctx, + client, + CosmosMetricName.REQUEST_DETAILS_GATEWAY_TIMELINE, + gatewayStats.getRequestTimeline(), requestTags); + } } private void recordAddressResolutionStatistics( diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/AddressResolver.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/AddressResolver.java index d9ea6a5e4a511..0c5209cc3e804 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/AddressResolver.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/AddressResolver.java @@ -6,12 +6,10 @@ import com.azure.cosmos.BridgeInternal; import com.azure.cosmos.CosmosContainerProactiveInitConfig; import com.azure.cosmos.CosmosException; -import com.azure.cosmos.implementation.AsyncDocumentClient; import com.azure.cosmos.implementation.BadRequestException; import com.azure.cosmos.implementation.DocumentCollection; import com.azure.cosmos.implementation.HttpConstants; import com.azure.cosmos.implementation.ICollectionRoutingMapCache; -import com.azure.cosmos.implementation.IOpenConnectionsHandler; import com.azure.cosmos.implementation.InternalServerErrorException; import com.azure.cosmos.implementation.InvalidPartitionException; import com.azure.cosmos.implementation.NotFoundException; diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/GatewayAddressCache.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/GatewayAddressCache.java index c8bc5ab575c82..bdcdba94f2663 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/GatewayAddressCache.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/GatewayAddressCache.java @@ -18,6 +18,7 @@ import com.azure.cosmos.implementation.GlobalEndpointManager; import com.azure.cosmos.implementation.HttpConstants; import com.azure.cosmos.implementation.IAuthorizationTokenProvider; +import com.azure.cosmos.implementation.ImplementationBridgeHelpers; import com.azure.cosmos.implementation.JavaStreamUtils; import com.azure.cosmos.implementation.MetadataDiagnosticsContext; import com.azure.cosmos.implementation.MetadataDiagnosticsContext.MetadataDiagnostics; @@ -42,6 +43,7 @@ import com.azure.cosmos.implementation.caches.AsyncCacheNonBlocking; import com.azure.cosmos.implementation.directconnectivity.rntbd.OpenConnectionTask; import com.azure.cosmos.implementation.directconnectivity.rntbd.ProactiveOpenConnectionsProcessor; +import com.azure.cosmos.implementation.faultinjection.GatewayServerErrorInjector; import com.azure.cosmos.implementation.http.HttpClient; import com.azure.cosmos.implementation.http.HttpHeaders; import com.azure.cosmos.implementation.http.HttpRequest; @@ -105,6 +107,7 @@ public class GatewayAddressCache implements IAddressCache { private final ConnectionPolicy connectionPolicy; private final boolean replicaAddressValidationEnabled; private final Set replicaValidationScopes; + private GatewayServerErrorInjector gatewayServerErrorInjector; public GatewayAddressCache( DiagnosticsClientContext clientContext, @@ -117,7 +120,8 @@ public GatewayAddressCache( ApiType apiType, GlobalEndpointManager globalEndpointManager, ConnectionPolicy connectionPolicy, - ProactiveOpenConnectionsProcessor proactiveOpenConnectionsProcessor) { + ProactiveOpenConnectionsProcessor proactiveOpenConnectionsProcessor, + GatewayServerErrorInjector gatewayServerErrorInjector) { this.clientContext = clientContext; try { @@ -168,6 +172,7 @@ public GatewayAddressCache( if (this.replicaAddressValidationEnabled) { this.replicaValidationScopes.add(Uri.HealthStatus.UnhealthyPending); } + this.gatewayServerErrorInjector = gatewayServerErrorInjector; } public GatewayAddressCache( @@ -180,7 +185,8 @@ public GatewayAddressCache( ApiType apiType, GlobalEndpointManager globalEndpointManager, ConnectionPolicy connectionPolicy, - ProactiveOpenConnectionsProcessor proactiveOpenConnectionsProcessor) { + ProactiveOpenConnectionsProcessor proactiveOpenConnectionsProcessor, + GatewayServerErrorInjector gatewayServerErrorInjector) { this(clientContext, serviceEndpoint, protocol, @@ -191,7 +197,8 @@ public GatewayAddressCache( apiType, globalEndpointManager, connectionPolicy, - proactiveOpenConnectionsProcessor); + proactiveOpenConnectionsProcessor, + gatewayServerErrorInjector); } @Override @@ -321,6 +328,10 @@ public void setOpenConnectionsProcessor(ProactiveOpenConnectionsProcessor proact this.proactiveOpenConnectionsProcessor = proactiveOpenConnectionsProcessor; } + public void setGatewayServerErrorInjector(GatewayServerErrorInjector gatewayServerErrorInjector) { + this.gatewayServerErrorInjector = gatewayServerErrorInjector; + } + public Mono> getServerAddressesViaGatewayAsync( RxDocumentServiceRequest request, String collectionRid, @@ -330,6 +341,10 @@ public Mono> getServerAddressesViaGatewayAsync( logger.debug("getServerAddressesViaGatewayAsync collectionRid {}, partitionKeyRangeIds {}", collectionRid, JavaStreamUtils.toString(partitionKeyRangeIds, ",")); } + + // track address refresh has happened, this is only meant to be used for fault injection validation + request.faultInjectionRequestContext.recordAddressForceRefreshed(forceRefresh); + request.setAddressRefresh(true, forceRefresh); String entryUrl = PathsHelper.generatePath(ResourceType.Document, collectionRid, true); HashMap addressQuery = new HashMap<>(); @@ -394,15 +409,25 @@ public Mono> getServerAddressesViaGatewayAsync( Instant addressCallStartTime = Instant.now(); HttpRequest httpRequest = new HttpRequest(HttpMethod.GET, targetEndpoint, targetEndpoint.getPort(), httpHeaders); + Duration responseTimeout = Duration.ofSeconds(Configs.getAddressRefreshResponseTimeoutInSeconds()); + Mono httpResponseMono; if (tokenProvider.getAuthorizationTokenType() != AuthorizationTokenType.AadToken) { - httpResponseMono = this.httpClient.send(httpRequest, - Duration.ofSeconds(Configs.getAddressRefreshResponseTimeoutInSeconds())); + httpResponseMono = this.httpClient.send(httpRequest, responseTimeout); } else { httpResponseMono = tokenProvider .populateAuthorizationHeader(httpHeaders) - .flatMap(valueHttpHeaders -> this.httpClient.send(httpRequest, - Duration.ofSeconds(Configs.getAddressRefreshResponseTimeoutInSeconds()))); + .flatMap(valueHttpHeaders -> this.httpClient.send(httpRequest,responseTimeout)); + } + + if (this.gatewayServerErrorInjector != null) { + httpResponseMono = + this.gatewayServerErrorInjector.injectGatewayErrors( + responseTimeout, + httpRequest, + request, + httpResponseMono, + partitionKeyRangeIds); } Mono dsrObs = HttpClientUtils.parseResponseAsync(request, clientContext, httpResponseMono, httpRequest); @@ -421,11 +446,21 @@ public Mono> getServerAddressesViaGatewayAsync( if (logger.isDebugEnabled()) { logger.debug("getServerAddressesViaGatewayAsync deserializes result"); } - logAddressResolutionEnd(request, identifier, null); + logAddressResolutionEnd( + request, + identifier, + null, + httpRequest.reactorNettyRequestRecord().getTransportRequestId()); + return dsr.getQueryResponse(null, Address.class); }).onErrorResume(throwable -> { Throwable unwrappedException = reactor.core.Exceptions.unwrap(throwable); - logAddressResolutionEnd(request, identifier, unwrappedException.toString()); + logAddressResolutionEnd( + request, + identifier, + unwrappedException.toString(), + httpRequest.reactorNettyRequestRecord().getTransportRequestId()); + if (!(unwrappedException instanceof Exception)) { // fatal error logger.error("Unexpected failure {}", unwrappedException.getMessage(), unwrappedException); @@ -777,11 +812,21 @@ public Mono> getMasterAddressesViaGatewayAsync( metadataDiagnosticsContext.addMetaDataDiagnostic(metaDataDiagnostic); } - logAddressResolutionEnd(request, identifier, null); + logAddressResolutionEnd( + request, + identifier, + null, + httpRequest.reactorNettyRequestRecord().getTransportRequestId()); + return dsr.getQueryResponse(null, Address.class); }).onErrorResume(throwable -> { Throwable unwrappedException = reactor.core.Exceptions.unwrap(throwable); - logAddressResolutionEnd(request, identifier, unwrappedException.toString()); + logAddressResolutionEnd( + request, + identifier, + unwrappedException.toString(), + httpRequest.reactorNettyRequestRecord().getTransportRequestId()); + if (!(unwrappedException instanceof Exception)) { // fatal error logger.error("Unexpected failure {}", unwrappedException.getMessage(), unwrappedException); @@ -1108,9 +1153,16 @@ private static String logAddressResolutionStart( return null; } - private static void logAddressResolutionEnd(RxDocumentServiceRequest request, String identifier, String errorMessage) { + private static void logAddressResolutionEnd( + RxDocumentServiceRequest request, + String identifier, + String errorMessage, + long transportRequestId) { if (request.requestContext.cosmosDiagnostics != null) { - BridgeInternal.recordAddressResolutionEnd(request.requestContext.cosmosDiagnostics, identifier, errorMessage); + ImplementationBridgeHelpers + .CosmosDiagnosticsHelper + .getCosmosDiagnosticsAccessor() + .recordAddressResolutionEnd(request, identifier, errorMessage, transportRequestId); } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/GlobalAddressResolver.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/GlobalAddressResolver.java index f00c839633833..29ac9f9e9a9fd 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/GlobalAddressResolver.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/GlobalAddressResolver.java @@ -20,6 +20,8 @@ import com.azure.cosmos.implementation.caches.RxCollectionCache; import com.azure.cosmos.implementation.caches.RxPartitionKeyRangeCache; import com.azure.cosmos.implementation.directconnectivity.rntbd.ProactiveOpenConnectionsProcessor; +import com.azure.cosmos.implementation.faultinjection.GatewayServerErrorInjector; +import com.azure.cosmos.implementation.faultinjection.IFaultInjectorProvider; import com.azure.cosmos.implementation.http.HttpClient; import com.azure.cosmos.implementation.routing.PartitionKeyInternalHelper; import com.azure.cosmos.implementation.routing.PartitionKeyRangeIdentity; @@ -58,6 +60,7 @@ public class GlobalAddressResolver implements IAddressResolver { private HttpClient httpClient; private ProactiveOpenConnectionsProcessor proactiveOpenConnectionsProcessor; private ConnectionPolicy connectionPolicy; + private GatewayServerErrorInjector gatewayServerErrorInjector; public GlobalAddressResolver( DiagnosticsClientContext diagnosticsClientContext, @@ -252,6 +255,20 @@ public void dispose() { } } + public void configureFaultInjectorProvider(IFaultInjectorProvider faultInjectorProvider, Configs configs) { + if (this.gatewayServerErrorInjector == null) { + this.gatewayServerErrorInjector = new GatewayServerErrorInjector(configs); + + // setup gatewayServerErrorInjector for existing address cache + // For the new ones added later, the gatewayServerErrorInjector will pass through constructor + for (EndpointCache endpointCache : this.addressCacheByEndpoint.values()) { + endpointCache.addressCache.setGatewayServerErrorInjector(this.gatewayServerErrorInjector); + } + } + + this.gatewayServerErrorInjector.registerServerErrorInjector(faultInjectorProvider.getServerErrorInjector()); + } + private IAddressResolver getAddressResolver(RxDocumentServiceRequest rxDocumentServiceRequest) { URI endpoint = this.endpointManager.resolveServiceEndpoint(rxDocumentServiceRequest); return this.getOrAddEndpoint(endpoint).addressResolver; @@ -269,7 +286,8 @@ private EndpointCache getOrAddEndpoint(URI endpoint) { this.apiType, this.endpointManager, this.connectionPolicy, - this.proactiveOpenConnectionsProcessor); + this.proactiveOpenConnectionsProcessor, + this.gatewayServerErrorInjector); AddressResolver addressResolver = new AddressResolver(); addressResolver.initializeCaches(this.collectionCache, this.routingMapProvider, gatewayAddressCache); EndpointCache cache = new EndpointCache(); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClient.java index ed6c7ca5d07cc..f0b12ed6379cd 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClient.java @@ -396,7 +396,7 @@ public void configureFaultInjectorProvider(IFaultInjectorProvider injectorProvid injectorProvider.registerConnectionErrorInjector(this.endpointProvider); if (this.serverErrorInjector != null) { this.serverErrorInjector - .registerServerErrorInjector(injectorProvider.getRntbdServerErrorInjector()); + .registerServerErrorInjector(injectorProvider.getServerErrorInjector()); } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ServerStoreModel.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ServerStoreModel.java index 6e3c05acbf944..584fb1e8a1f9f 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ServerStoreModel.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ServerStoreModel.java @@ -8,6 +8,7 @@ import com.azure.cosmos.ConsistencyLevel; import com.azure.cosmos.CosmosContainerProactiveInitConfig; import com.azure.cosmos.implementation.BadRequestException; +import com.azure.cosmos.implementation.Configs; import com.azure.cosmos.implementation.HttpConstants; import com.azure.cosmos.implementation.RMResources; import com.azure.cosmos.implementation.RxDocumentServiceRequest; @@ -65,7 +66,7 @@ public Flux submitOpenConnectionTasksAndInitCaches(CosmosContainerProactiv } @Override - public void configureFaultInjectorProvider(IFaultInjectorProvider injectorProvider) { + public void configureFaultInjectorProvider(IFaultInjectorProvider injectorProvider, Configs configs) { this.storeClient.configureFaultInjectorProvider(injectorProvider); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/FaultInjectionRequestArgs.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/FaultInjectionRequestArgs.java new file mode 100644 index 0000000000000..9b45f20e8922b --- /dev/null +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/FaultInjectionRequestArgs.java @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.faultinjection; + +import com.azure.cosmos.implementation.RxDocumentServiceRequest; + +import java.net.URI; +import java.util.List; + +import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull; + +public abstract class FaultInjectionRequestArgs { + private final long transportRequestId; + private final URI requestURI; + private final RxDocumentServiceRequest serviceRequest; + private boolean isPrimary; + + public FaultInjectionRequestArgs( + long transportRequestId, + URI requestURI, + boolean isPrimary, + RxDocumentServiceRequest serviceRequest) { + + checkNotNull(requestURI, "Argument 'requestURI' can not null"); + checkNotNull(serviceRequest, "Argument 'serviceRequest' can not be null"); + + this.transportRequestId = transportRequestId; + this.requestURI = requestURI; + this.isPrimary = isPrimary; + this.serviceRequest = serviceRequest; + } + + public long getTransportRequestId() { + return this.transportRequestId; + } + + public URI getRequestURI() { + return this.requestURI; + } + + public RxDocumentServiceRequest getServiceRequest() { + return this.serviceRequest; + } + + public boolean isPrimary() { + return this.isPrimary; + } + + public abstract List getPartitionKeyRangeIds(); + public abstract String getCollectionRid(); +} diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/FaultInjectionRequestContext.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/FaultInjectionRequestContext.java index 2a3ee1b115436..a748fd033e25c 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/FaultInjectionRequestContext.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/FaultInjectionRequestContext.java @@ -8,6 +8,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; /*** * Only used in fault injection. @@ -19,6 +20,7 @@ public class FaultInjectionRequestContext { private final Map hitCountByRuleMap; private final Map transportRequestIdRuleIdMap; private final Map> transportRequestIdRuleEvaluationMap; + private final AtomicBoolean addressForceRefreshed; private volatile URI locationEndpointToRoute; @@ -33,12 +35,14 @@ public FaultInjectionRequestContext(FaultInjectionRequestContext cloneContext) { this.hitCountByRuleMap = cloneContext.hitCountByRuleMap; this.transportRequestIdRuleIdMap = new ConcurrentHashMap<>(); this.transportRequestIdRuleEvaluationMap = new ConcurrentHashMap<>(); + this.addressForceRefreshed = new AtomicBoolean(false); } public FaultInjectionRequestContext() { this.hitCountByRuleMap = new ConcurrentHashMap<>(); this.transportRequestIdRuleIdMap = new ConcurrentHashMap<>(); this.transportRequestIdRuleEvaluationMap = new ConcurrentHashMap<>(); + this.addressForceRefreshed = new AtomicBoolean(false); } public void applyFaultInjectionRule(long transportId, String ruleId) { @@ -65,11 +69,21 @@ public void recordFaultInjectionRuleEvaluation(long transportId, String ruleEval }); } + public void recordAddressForceRefreshed(boolean forceRefreshed) { + if (forceRefreshed) { + this.addressForceRefreshed.compareAndSet(false, true); + } + } + + public boolean getAddressForceRefreshed() { + return this.addressForceRefreshed.get(); + } + public int getFaultInjectionRuleApplyCount(String ruleId) { return this.hitCountByRuleMap.getOrDefault(ruleId, 0); } - public String getFaultInjectionRuleId(long transportRequesetId) { - return this.transportRequestIdRuleIdMap.getOrDefault(transportRequesetId, null); + public String getFaultInjectionRuleId(long transportRequestId) { + return this.transportRequestIdRuleIdMap.getOrDefault(transportRequestId, null); } public void setLocationEndpointToRoute(URI locationEndpointToRoute) { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/GatewayFaultInjectionRequestArgs.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/GatewayFaultInjectionRequestArgs.java new file mode 100644 index 0000000000000..20ff02fb34e85 --- /dev/null +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/GatewayFaultInjectionRequestArgs.java @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.faultinjection; + +import com.azure.cosmos.implementation.ResourceId; +import com.azure.cosmos.implementation.RxDocumentServiceRequest; +import com.azure.cosmos.implementation.Strings; +import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; + +import java.net.URI; +import java.util.List; + +public class GatewayFaultInjectionRequestArgs extends FaultInjectionRequestArgs { + private final List partitionKeyRangeIds; + + public GatewayFaultInjectionRequestArgs( + long transportRequestId, + URI requestURI, + RxDocumentServiceRequest serviceRequest, + List partitionKeyRangeIds) { + + super(transportRequestId, requestURI, false, serviceRequest); + this.partitionKeyRangeIds = partitionKeyRangeIds; + } + + @Override + public List getPartitionKeyRangeIds() { + return this.partitionKeyRangeIds; + } + + @Override + public String getCollectionRid() { + if (this.getServiceRequest().getIsNameBased()) { + return this.getServiceRequest().requestContext.resolvedCollectionRid; + } + + if (StringUtils.isNotEmpty(this.getServiceRequest().getResourceId())) { + return ResourceId.parse(this.getServiceRequest().getResourceId()).getDocumentCollectionId().toString(); + } + return Strings.Emtpy; + } +} diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/GatewayServerErrorInjector.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/GatewayServerErrorInjector.java new file mode 100644 index 0000000000000..812d405ab5ffb --- /dev/null +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/GatewayServerErrorInjector.java @@ -0,0 +1,170 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.faultinjection; + +import com.azure.cosmos.CosmosException; +import com.azure.cosmos.implementation.Configs; +import com.azure.cosmos.implementation.RxDocumentServiceRequest; +import com.azure.cosmos.implementation.Utils; +import com.azure.cosmos.implementation.http.HttpRequest; +import com.azure.cosmos.implementation.http.HttpResponse; +import com.azure.cosmos.implementation.http.ReactorNettyRequestRecord; +import io.netty.channel.ConnectTimeoutException; +import io.netty.handler.timeout.ReadTimeoutException; +import reactor.core.publisher.Mono; + +import java.net.URI; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull; + +public class GatewayServerErrorInjector { + + private final Configs configs; + + private List faultInjectors = new ArrayList<>(); + + public GatewayServerErrorInjector(Configs configs) { + checkNotNull(configs, "Argument 'configs' can not be null"); + this.configs = configs; + } + + public void registerServerErrorInjector(IServerErrorInjector serverErrorInjector) { + checkNotNull(serverErrorInjector, "Argument 'serverErrorInjector' can not be null"); + this.faultInjectors.add(serverErrorInjector); + } + + public Mono injectGatewayErrors( + Duration responseTimeout, + HttpRequest httpRequest, + RxDocumentServiceRequest serviceRequest, + Mono originalResponseMono) { + return injectGatewayErrors( + responseTimeout, + httpRequest, + serviceRequest, + originalResponseMono, + serviceRequest.requestContext.resolvedPartitionKeyRange != null + ? Arrays.asList(serviceRequest.requestContext.resolvedPartitionKeyRange.getId()) : null); + } + + public Mono injectGatewayErrors( + Duration responseTimeout, + HttpRequest httpRequest, + RxDocumentServiceRequest serviceRequest, + Mono originalResponseMono, + List partitionKeyRangeIds) { + + return Mono.just(responseTimeout) + .flatMap(effectiveResponseTimeout -> { + Utils.ValueHolder exceptionToBeInjected = new Utils.ValueHolder<>(); + Utils.ValueHolder delayToBeInjected = new Utils.ValueHolder<>(); + FaultInjectionRequestArgs faultInjectionRequestArgs = + this.createFaultInjectionRequestArgs( + httpRequest.reactorNettyRequestRecord(), + httpRequest.uri(), + serviceRequest, + partitionKeyRangeIds); + + if (this.injectGatewayServerResponseError(faultInjectionRequestArgs, exceptionToBeInjected)) { + return Mono.error(exceptionToBeInjected.v); + } + + if (this.injectGatewayServerConnectionDelay(faultInjectionRequestArgs, delayToBeInjected)) { + Duration connectionAcquireTimeout = this.configs.getConnectionAcquireTimeout(); + if (delayToBeInjected.v.toMillis() >= connectionAcquireTimeout.toMillis()) { + return Mono.delay(connectionAcquireTimeout) + .then(Mono.error(new ConnectTimeoutException())); + } else { + return Mono.delay(delayToBeInjected.v) + .then(originalResponseMono); + } + } + + if (this.injectGatewayServerResponseDelayBeforeProcessing(faultInjectionRequestArgs, delayToBeInjected)) { + if (delayToBeInjected.v.toMillis() >= effectiveResponseTimeout.toMillis()) { + return Mono.delay(effectiveResponseTimeout) + .then(Mono.error(new ReadTimeoutException())); + } else { + return Mono.delay(delayToBeInjected.v) + .then(originalResponseMono); + } + } + + if (this.injectGatewayServerResponseDelayAfterProcessing(faultInjectionRequestArgs, delayToBeInjected)) { + if (delayToBeInjected.v.toMillis() >= effectiveResponseTimeout.toMillis()) { + return originalResponseMono + .delayElement(delayToBeInjected.v) + .then(Mono.error(new ReadTimeoutException())); + } else { + return originalResponseMono + .delayElement(delayToBeInjected.v); + } + } + + return originalResponseMono; + }); + } + + private boolean injectGatewayServerResponseDelayBeforeProcessing( + FaultInjectionRequestArgs faultInjectionRequestArgs, + Utils.ValueHolder delayToBeInjected) { + + for (IServerErrorInjector serverErrorInjector : faultInjectors) { + if(serverErrorInjector.injectServerResponseDelayBeforeProcessing(faultInjectionRequestArgs, delayToBeInjected)) { + return true; + } + } + return false; + } + + private boolean injectGatewayServerResponseDelayAfterProcessing( + FaultInjectionRequestArgs faultInjectionRequestArgs, + Utils.ValueHolder delayToBeInjected) { + + for (IServerErrorInjector serverErrorInjector : faultInjectors) { + if(serverErrorInjector.injectServerResponseDelayAfterProcessing(faultInjectionRequestArgs, delayToBeInjected)) { + return true; + } + } + return false; + } + + private boolean injectGatewayServerResponseError( + FaultInjectionRequestArgs faultInjectionRequestArgs, + Utils.ValueHolder exceptionToBeInjected) { + for (IServerErrorInjector serverErrorInjector : faultInjectors) { + if(serverErrorInjector.injectServerResponseError(faultInjectionRequestArgs, exceptionToBeInjected)) { + return true; + } + } + return false; + } + + private boolean injectGatewayServerConnectionDelay( + FaultInjectionRequestArgs faultInjectionRequestArgs, + Utils.ValueHolder delayToBeInjected) { + for (IServerErrorInjector serverErrorInjector : faultInjectors) { + if(serverErrorInjector.injectServerConnectionDelay(faultInjectionRequestArgs, delayToBeInjected)) { + return true; + } + } + return false; + } + + private GatewayFaultInjectionRequestArgs createFaultInjectionRequestArgs( + ReactorNettyRequestRecord requestRecord, + URI requestUri, + RxDocumentServiceRequest serviceRequest, + List partitionKeyRangeIds) { + return new GatewayFaultInjectionRequestArgs( + requestRecord.getTransportRequestId(), + requestUri, + serviceRequest, + partitionKeyRangeIds); + } +} diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/IFaultInjectorProvider.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/IFaultInjectorProvider.java index 8b4458d901c54..45f974afbdc80 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/IFaultInjectorProvider.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/IFaultInjectorProvider.java @@ -11,10 +11,10 @@ public interface IFaultInjectorProvider { /*** - * Get the rntbd server error injector. - * @return the rntbd server error injector. + * Get the server error injector. + * @return the server error injector. */ - IRntbdServerErrorInjector getRntbdServerErrorInjector(); + IServerErrorInjector getServerErrorInjector(); /*** * Register the rntbd connection error injector. diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/IRntbdServerErrorInjector.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/IRntbdServerErrorInjector.java deleted file mode 100644 index e622e71429e96..0000000000000 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/IRntbdServerErrorInjector.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.implementation.faultinjection; - -import com.azure.cosmos.implementation.directconnectivity.rntbd.IRequestRecord; -import com.azure.cosmos.implementation.directconnectivity.rntbd.RntbdRequestRecord; - -import java.time.Duration; -import java.util.function.Consumer; - -/*** - * Rntbd server error injector. - */ -public interface IRntbdServerErrorInjector { - - /*** - * Inject server response delay before sending the request to the service - * - * @param requestRecord the request record. - * @param writeRequestWithDelayConsumer the consumer to be executed if applicable rule is found. - * @return flag to indicate whether server response delay is injected. - */ - boolean injectRntbdServerResponseDelayBeforeProcessing( - RntbdRequestRecord requestRecord, - Consumer writeRequestWithDelayConsumer); - - /*** - * Inject server response delay after sending the request to the service - * - * @param requestRecord the request record. - * @param writeRequestWithDelayConsumer the consumer to be executed if applicable rule is found. - * @return flag to indicate whether server response delay is injected. - */ - boolean injectRntbdServerResponseDelayAfterProcessing( - RntbdRequestRecord requestRecord, - Consumer writeRequestWithDelayConsumer); - - /*** - * Inject server response error. - * - * @param requestRecord the request record. - * @return flag to indicate whether server response error is injected. - */ - boolean injectRntbdServerResponseError(RntbdRequestRecord requestRecord); - - /*** - * Inject server connection delay error. - * - * @param requestRecord the request record. - * @param openConnectionWithDelayConsumer the consumer to be executed if applicable rule is found. - * @return flag to indicate whether server connection delay rule is injected. - */ - boolean injectRntbdServerConnectionDelay( - IRequestRecord requestRecord, - Consumer openConnectionWithDelayConsumer); -} diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/IServerErrorInjector.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/IServerErrorInjector.java new file mode 100644 index 0000000000000..7a9ccc4279e6d --- /dev/null +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/IServerErrorInjector.java @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.faultinjection; + +import com.azure.cosmos.CosmosException; +import com.azure.cosmos.implementation.Utils; + +import java.time.Duration; + +/*** + * Rntbd or gateway server error injector. + */ +public interface IServerErrorInjector { + + /*** + * Inject server response delay before sending the request to the service + * + * @param faultInjectionRequestArgs fault injection request args to find matched fault injection rules. + * @param injectedDelay the injected delay. The value will be null if no matched rule can be found. + * @return flag to indicate whether server response delay is injected. + */ + boolean injectServerResponseDelayBeforeProcessing( + FaultInjectionRequestArgs faultInjectionRequestArgs, + Utils.ValueHolder injectedDelay); + + /*** + * Inject server response delay after sending the request to the service + * + * @param faultInjectionRequestArgs fault injection request args to find matched fault injection rules. + * @param injectedDelay the injected delay. The value will be null if no matched rule can be found. + * @return flag to indicate whether server response delay is injected. + */ + boolean injectServerResponseDelayAfterProcessing( + FaultInjectionRequestArgs faultInjectionRequestArgs, + Utils.ValueHolder injectedDelay); + + /*** + * Inject server response error. + * + * @param faultInjectionRequestArgs fault injection request args to find matched fault injection rules. + * @param injectedException the injected exception. The value will be null if no matched rule can be found. + * @return flag to indicate whether server response error is injected. + */ + boolean injectServerResponseError( + FaultInjectionRequestArgs faultInjectionRequestArgs, + Utils.ValueHolder injectedException); + + /*** + * Inject server connection delay error. + * + * @param faultInjectionRequestArgs fault injection request args to find matched fault injection rules. + * @param injectedDelay the injected delay. Value will be null if no matched rule can be found. + * @return flag to indicate whether server connection delay rule is injected. + */ + boolean injectServerConnectionDelay( + FaultInjectionRequestArgs faultInjectionRequestArgs, + Utils.ValueHolder injectedDelay); +} diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/RntbdFaultInjectionRequestArgs.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/RntbdFaultInjectionRequestArgs.java new file mode 100644 index 0000000000000..af82b365a21ac --- /dev/null +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/RntbdFaultInjectionRequestArgs.java @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.faultinjection; + +import com.azure.cosmos.implementation.RxDocumentServiceRequest; + +import java.net.URI; +import java.util.Collections; +import java.util.List; + +public class RntbdFaultInjectionRequestArgs extends FaultInjectionRequestArgs { + public RntbdFaultInjectionRequestArgs( + long transportRequestId, + URI requestURI, + boolean isPrimary, + RxDocumentServiceRequest serviceRequest) { + super(transportRequestId, requestURI, isPrimary, serviceRequest); + } + + @Override + public List getPartitionKeyRangeIds() { + // This method is used to check whether the request meets the partition scope in fault injection + // however, for direct connection type, requestURI is being used for the above purpose + // so we will always return an empty list here as it is not really being used in direct connection type + return Collections.emptyList(); + } + + @Override + public String getCollectionRid() { + return this.getServiceRequest().requestContext.resolvedCollectionRid; + } +} diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/RntbdServerErrorInjector.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/RntbdServerErrorInjector.java index d5ca04e9aeb54..b96d51ffc5df2 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/RntbdServerErrorInjector.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/faultinjection/RntbdServerErrorInjector.java @@ -3,6 +3,8 @@ package com.azure.cosmos.implementation.faultinjection; +import com.azure.cosmos.CosmosException; +import com.azure.cosmos.implementation.Utils; import com.azure.cosmos.implementation.directconnectivity.rntbd.IRequestRecord; import com.azure.cosmos.implementation.directconnectivity.rntbd.RntbdRequestRecord; @@ -13,21 +15,24 @@ import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull; -public class RntbdServerErrorInjector implements IRntbdServerErrorInjector { - private List faultInjectors = new ArrayList<>(); +public class RntbdServerErrorInjector { + private List faultInjectors = new ArrayList<>(); - public void registerServerErrorInjector(IRntbdServerErrorInjector serverErrorInjector) { + public void registerServerErrorInjector(IServerErrorInjector serverErrorInjector) { checkNotNull(serverErrorInjector, "Argument 'serverErrorInjector' can not be null"); this.faultInjectors.add(serverErrorInjector); } - @Override public boolean injectRntbdServerResponseDelayBeforeProcessing( RntbdRequestRecord requestRecord, Consumer writeRequestWithDelayConsumer) { - for (IRntbdServerErrorInjector injector : this.faultInjectors) { - if (injector.injectRntbdServerResponseDelayBeforeProcessing(requestRecord, writeRequestWithDelayConsumer)) { + Utils.ValueHolder injectedDelay = new Utils.ValueHolder<>(); + for (IServerErrorInjector injector : this.faultInjectors) { + if (injector.injectServerResponseDelayBeforeProcessing( + this.createFaultInjectionRequestArgs(requestRecord), injectedDelay)) { + + writeRequestWithDelayConsumer.accept(injectedDelay.v); return true; } } @@ -35,11 +40,14 @@ public boolean injectRntbdServerResponseDelayBeforeProcessing( return false; } - @Override public boolean injectRntbdServerResponseDelayAfterProcessing(RntbdRequestRecord requestRecord, Consumer writeRequestWithDelayConsumer) { - for (IRntbdServerErrorInjector injector : this.faultInjectors) { - if (injector.injectRntbdServerResponseDelayAfterProcessing(requestRecord, writeRequestWithDelayConsumer)) { + Utils.ValueHolder injectedDelay = new Utils.ValueHolder<>(); + for (IServerErrorInjector injector : this.faultInjectors) { + if (injector.injectServerResponseDelayAfterProcessing( + this.createFaultInjectionRequestArgs(requestRecord), injectedDelay)) { + + writeRequestWithDelayConsumer.accept(injectedDelay.v); return true; } } @@ -47,11 +55,14 @@ public boolean injectRntbdServerResponseDelayAfterProcessing(RntbdRequestRecord return false; } - @Override public boolean injectRntbdServerResponseError(RntbdRequestRecord requestRecord) { - for (IRntbdServerErrorInjector injector : this.faultInjectors) { - if (injector.injectRntbdServerResponseError(requestRecord)) { + Utils.ValueHolder injectedException = new Utils.ValueHolder<>(); + for (IServerErrorInjector injector : this.faultInjectors) { + if (injector.injectServerResponseError( + this.createFaultInjectionRequestArgs(requestRecord), injectedException)) { + + requestRecord.completeExceptionally(injectedException.v); return true; } } @@ -59,17 +70,44 @@ public boolean injectRntbdServerResponseError(RntbdRequestRecord requestRecord) return false; } - @Override public boolean injectRntbdServerConnectionDelay( IRequestRecord requestRecord, Consumer openConnectionWithDelayConsumer) { - for (IRntbdServerErrorInjector injector : this.faultInjectors) { - if (injector.injectRntbdServerConnectionDelay(requestRecord, openConnectionWithDelayConsumer)) { + Utils.ValueHolder injectedDelay = new Utils.ValueHolder<>(); + for (IServerErrorInjector injector : this.faultInjectors) { + if (injector.injectServerConnectionDelay( + this.createFaultInjectionRequestArgs(requestRecord), injectedDelay)) { + + openConnectionWithDelayConsumer.accept(injectedDelay.v); return true; } } return false; } + + private RntbdFaultInjectionRequestArgs createFaultInjectionRequestArgs(RntbdRequestRecord requestRecord) { + if (requestRecord == null) { + return null; + } + + return new RntbdFaultInjectionRequestArgs( + requestRecord.args().transportRequestId(), + requestRecord.args().physicalAddressUri().getURI(), + requestRecord.args().physicalAddressUri().isPrimary(), + requestRecord.args().serviceRequest()); + } + + private RntbdFaultInjectionRequestArgs createFaultInjectionRequestArgs(IRequestRecord requestRecord) { + if (requestRecord == null) { + return null; + } + + return new RntbdFaultInjectionRequestArgs( + requestRecord.getRequestId(), + requestRecord.args().physicalAddressUri().getURI(), + requestRecord.args().physicalAddressUri().isPrimary(), + requestRecord.args().serviceRequest()); + } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyRequestRecord.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyRequestRecord.java index 4fa3ac5e2582d..87f68b29e8964 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyRequestRecord.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyRequestRecord.java @@ -7,6 +7,7 @@ import reactor.netty.http.client.HttpClientState; import java.time.Instant; +import java.util.concurrent.atomic.AtomicLong; /** * Represents the timeline of various events in the lifetime of a reactor netty request response. @@ -22,6 +23,7 @@ *

*/ public final class ReactorNettyRequestRecord { + private static final AtomicLong instanceCount = new AtomicLong(); private volatile Instant timeCreated; private volatile Instant timeConnected; @@ -30,6 +32,11 @@ public final class ReactorNettyRequestRecord { private volatile Instant timeSent; private volatile Instant timeReceived; private volatile Instant timeCompleted; + private final long transportRequestId; + + public ReactorNettyRequestRecord() { + this.transportRequestId = instanceCount.incrementAndGet(); + } /** * Gets request created instant. @@ -186,4 +193,8 @@ timeCreated, timeAcquired() == null ? timeCompletedOrNow : timeAcquired), timeReceived, timeCompletedOrNow)); } } + + public long getTransportRequestId() { + return transportRequestId; + } }