MaxRetryCountTest+[BUG]404/1002CyclicRegionRetry+[BUG]449RetryTooAggr…

…essively +[BUG]MissingCrossRegionRetryForWritesWithServer410 Azure#37050) (Azure#37040) * adding maxRetryCount tests for 404/1002, 500, 503, 408 * fixed 404/1002 cyclicRegionRetry * fixed 449 retry too aggressively * fixed missing cross region retry for write with server 410 * fixed missing cross region retry for server returned 503 for write operations --------- Co-authored-by: annie-mac <[email protected]>
roryp · Oct 20, 2023 · 049524b · 049524b
1 parent 2cd4878
commit 049524b
Show file tree

Hide file tree

Showing 11 changed files with 2,949 additions and 302 deletions.
diff --git a/...re/cosmos/test/implementation/faultinjection/FaultInjectionServerErrorResultInternal.java b/...re/cosmos/test/implementation/faultinjection/FaultInjectionServerErrorResultInternal.java
@@ -90,6 +90,8 @@ public CosmosException getInjectedServerError(RxDocumentServiceRequest request)
                 responseHeaders.put(
                     HttpConstants.HttpHeaders.RETRY_AFTER_IN_MILLISECONDS,
                     String.valueOf(500));
+                responseHeaders.put(WFConstants.BackendHeaders.SUB_STATUS,
+                    Integer.toString(HttpConstants.SubStatusCodes.USER_REQUEST_RATE_TOO_LARGE));
                 cosmosException = new RequestRateTooLargeException(null, lsn, partitionKeyRangeId, responseHeaders);
 
                 break;
@@ -137,7 +139,8 @@ public CosmosException getInjectedServerError(RxDocumentServiceRequest request)
             case SERVICE_UNAVAILABLE:
                 responseHeaders.put(WFConstants.BackendHeaders.SUB_STATUS,
                     Integer.toString(HttpConstants.SubStatusCodes.SERVER_GENERATED_503));
-                cosmosException = new ServiceUnavailableException(null, lsn, null, responseHeaders, HttpConstants.SubStatusCodes.SERVER_GENERATED_503);
+                cosmosException =
+                    new ServiceUnavailableException(null, lsn, null, responseHeaders, HttpConstants.SubStatusCodes.SERVER_GENERATED_503);
                 break;
 
             case STALED_ADDRESSES_SERVER_GONE:

diff --git a/...mos-tests/src/test/java/com/azure/cosmos/FaultInjectionWithAvailabilityStrategyTests.java b/...mos-tests/src/test/java/com/azure/cosmos/FaultInjectionWithAvailabilityStrategyTests.java
diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/MaxRetryCountTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/MaxRetryCountTests.java
diff --git a/.../src/test/java/com/azure/cosmos/faultinjection/ExcludedRegionWithFaultInjectionTests.java b/.../src/test/java/com/azure/cosmos/faultinjection/ExcludedRegionWithFaultInjectionTests.java
@@ -1099,9 +1099,9 @@ public Object[][] regionExclusionWriteAfterCreateTestConfigs() {
                             this.chooseFirstRegion.apply(this.preferredRegions)
                         ))
                         .withExpectedResultAfterMutation(new ExpectedResult(
-                            HttpConstants.StatusCodes.SERVICE_UNAVAILABLE,
-                            HttpConstants.SubStatusCodes.SERVER_GENERATED_503,
-                            this.chooseFirstRegion.apply(this.preferredRegions)
+                            HttpConstants.StatusCodes.CREATED,
+                            HttpConstants.SubStatusCodes.UNKNOWN,
+                            this.chooseAllRegions.apply(this.preferredRegions)
                     )),
                 },
                 {
@@ -1159,9 +1159,9 @@ public Object[][] regionExclusionWriteAfterCreateTestConfigs() {
                             this.chooseFirstRegion.apply(this.preferredRegions)
                         ))
                         .withExpectedResultAfterMutation(new ExpectedResult(
-                        HttpConstants.StatusCodes.SERVICE_UNAVAILABLE,
-                        HttpConstants.SubStatusCodes.SERVER_GENERATED_503,
-                        this.chooseFirstRegion.apply(this.preferredRegions)
+                        HttpConstants.StatusCodes.OK,
+                        HttpConstants.SubStatusCodes.UNKNOWN,
+                        this.chooseAllRegions.apply(this.preferredRegions)
                     )),
                 },
                 {
@@ -1224,9 +1224,9 @@ public Object[][] regionExclusionWriteAfterCreateTestConfigs() {
                             this.chooseFirstRegion.apply(this.preferredRegions)
                         ))
                         .withExpectedResultAfterMutation(new ExpectedResult(
-                            HttpConstants.StatusCodes.SERVICE_UNAVAILABLE,
-                            HttpConstants.SubStatusCodes.SERVER_GENERATED_503,
-                            this.chooseFirstRegion.apply(this.preferredRegions)
+                            HttpConstants.StatusCodes.OK,
+                            HttpConstants.SubStatusCodes.UNKNOWN,
+                            this.chooseAllRegions.apply(this.preferredRegions)
                     )),
                 },
                 {
@@ -1266,9 +1266,9 @@ public Object[][] regionExclusionWriteAfterCreateTestConfigs() {
                             this.chooseFirstRegion.apply(this.preferredRegions)
                         ))
                         .withExpectedResultAfterMutation(new ExpectedResult(
-                            HttpConstants.StatusCodes.SERVICE_UNAVAILABLE,
-                            HttpConstants.SubStatusCodes.SERVER_GENERATED_503,
-                            this.chooseFirstRegion.apply(this.preferredRegions)
+                            HttpConstants.StatusCodes.CREATED,
+                            HttpConstants.SubStatusCodes.UNKNOWN,
+                            this.chooseAllRegions.apply(this.preferredRegions)
                     )),
                 },
                 {
@@ -1308,9 +1308,9 @@ public Object[][] regionExclusionWriteAfterCreateTestConfigs() {
                             this.chooseFirstRegion.apply(this.preferredRegions)
                         ))
                         .withExpectedResultAfterMutation(new ExpectedResult(
-                            HttpConstants.StatusCodes.SERVICE_UNAVAILABLE,
-                            HttpConstants.SubStatusCodes.SERVER_GENERATED_503,
-                            this.chooseFirstRegion.apply(this.preferredRegions)
+                            HttpConstants.StatusCodes.NO_CONTENT,
+                            HttpConstants.SubStatusCodes.UNKNOWN,
+                            this.chooseAllRegions.apply(this.preferredRegions)
                     )),
                 },
                 {
@@ -1350,9 +1350,9 @@ public Object[][] regionExclusionWriteAfterCreateTestConfigs() {
                             this.chooseFirstRegion.apply(this.preferredRegions)
                         ))
                         .withExpectedResultAfterMutation(new ExpectedResult(
-                            HttpConstants.StatusCodes.SERVICE_UNAVAILABLE,
-                            HttpConstants.SubStatusCodes.SERVER_GENERATED_503,
-                            this.chooseFirstRegion.apply(this.preferredRegions)
+                            HttpConstants.StatusCodes.OK,
+                            HttpConstants.SubStatusCodes.UNKNOWN,
+                            this.chooseAllRegions.apply(this.preferredRegions)
                     )),
                 },
                 {
@@ -1668,9 +1668,10 @@ public Object[][] regionExclusionBatchTestConfigs() {
                             this.chooseLastRegion.apply(this.preferredRegions)
                         ))
                         .withExpectedResultAfterMutation(new ExpectedResult(
-                            HttpConstants.StatusCodes.SERVICE_UNAVAILABLE,
-                            HttpConstants.SubStatusCodes.SERVER_GENERATED_503,
-                            this.chooseFirstRegion.apply(this.preferredRegions)
+                            HttpConstants.StatusCodes.OK,
+                            HttpConstants.SubStatusCodes.UNKNOWN,
+                            2,
+                            this.chooseFirstTwoRegions.apply(this.preferredRegions)
                     ))
                 },
                 {
@@ -1689,9 +1690,10 @@ public Object[][] regionExclusionBatchTestConfigs() {
                             this.chooseLastRegion.apply(this.preferredRegions)
                         ))
                         .withExpectedResultAfterMutation(new ExpectedResult(
-                            HttpConstants.StatusCodes.SERVICE_UNAVAILABLE,
-                            HttpConstants.SubStatusCodes.SERVER_GENERATED_503,
-                            this.chooseFirstRegion.apply(this.preferredRegions)
+                            HttpConstants.StatusCodes.OK,
+                            HttpConstants.SubStatusCodes.UNKNOWN,
+                            2,
+                            this.chooseFirstTwoRegions.apply(this.preferredRegions)
                     ))
                 },
                 {
@@ -2035,9 +2037,9 @@ public Object[][] regionExclusionBulkTestConfigs() {
                             this.chooseLastRegion.apply(this.preferredRegions)
                         ))
                         .withExpectedResultAfterMutation(new ExpectedResult(
-                            HttpConstants.StatusCodes.SERVICE_UNAVAILABLE,
-                            HttpConstants.SubStatusCodes.SERVER_GENERATED_503,
-                            this.chooseFirstRegion.apply(this.preferredRegions)
+                            HttpConstants.StatusCodes.OK,
+                            HttpConstants.SubStatusCodes.UNKNOWN,
+                            this.chooseFirstTwoRegions.apply(this.preferredRegions)
                     ))
                 },
                 {
@@ -2055,9 +2057,9 @@ public Object[][] regionExclusionBulkTestConfigs() {
                             this.chooseLastRegion.apply(this.preferredRegions)
                         ))
                         .withExpectedResultAfterMutation(new ExpectedResult(
-                            HttpConstants.StatusCodes.SERVICE_UNAVAILABLE,
-                            HttpConstants.SubStatusCodes.SERVER_GENERATED_503,
-                            this.chooseFirstRegion.apply(this.preferredRegions)
+                            HttpConstants.StatusCodes.OK,
+                            HttpConstants.SubStatusCodes.UNKNOWN,
+                            this.chooseFirstTwoRegions.apply(this.preferredRegions)
                     ))
                 },
                 {

diff --git a/...test/java/com/azure/cosmos/faultinjection/FaultInjectionServerErrorRuleOnDirectTests.java b/...test/java/com/azure/cosmos/faultinjection/FaultInjectionServerErrorRuleOnDirectTests.java
@@ -124,7 +124,7 @@ public static Object[][] faultInjectionServerErrorResponseProvider() {
             { OperationType.Read, FaultInjectionOperationType.READ_ITEM, FaultInjectionServerErrorType.GONE, true, 410, HttpConstants.SubStatusCodes.SERVER_GENERATED_410 },
             { OperationType.Read, FaultInjectionOperationType.READ_ITEM, FaultInjectionServerErrorType.INTERNAL_SERVER_ERROR, false, 500, 0 },
             { OperationType.Read, FaultInjectionOperationType.READ_ITEM, FaultInjectionServerErrorType.RETRY_WITH, true, 449, 0 },
-            { OperationType.Read, FaultInjectionOperationType.READ_ITEM, FaultInjectionServerErrorType.TOO_MANY_REQUEST, true, 429, 0 },
+            { OperationType.Read, FaultInjectionOperationType.READ_ITEM, FaultInjectionServerErrorType.TOO_MANY_REQUEST, true, 429, HttpConstants.SubStatusCodes.USER_REQUEST_RATE_TOO_LARGE },
             { OperationType.Read, FaultInjectionOperationType.READ_ITEM, FaultInjectionServerErrorType.READ_SESSION_NOT_AVAILABLE, true, 404, 1002 },
             { OperationType.Read, FaultInjectionOperationType.READ_ITEM, FaultInjectionServerErrorType.TIMEOUT, true, 410, HttpConstants.SubStatusCodes.SERVER_GENERATED_408 }, // for server return 408, SDK will wrap into 410/21010
             { OperationType.Read, FaultInjectionOperationType.READ_ITEM, FaultInjectionServerErrorType.PARTITION_IS_MIGRATING, true, 410, 1008 },
@@ -133,7 +133,7 @@ public static Object[][] faultInjectionServerErrorResponseProvider() {
             { OperationType.ReadFeed, FaultInjectionOperationType.READ_FEED_ITEM, FaultInjectionServerErrorType.GONE, true, 410, HttpConstants.SubStatusCodes.SERVER_GENERATED_410 },
             { OperationType.ReadFeed, FaultInjectionOperationType.READ_FEED_ITEM, FaultInjectionServerErrorType.INTERNAL_SERVER_ERROR, false, 500, 0 },
             { OperationType.ReadFeed, FaultInjectionOperationType.READ_FEED_ITEM, FaultInjectionServerErrorType.RETRY_WITH, true, 449, 0 },
-            { OperationType.ReadFeed, FaultInjectionOperationType.READ_FEED_ITEM, FaultInjectionServerErrorType.TOO_MANY_REQUEST, true, 429, 0 },
+            { OperationType.ReadFeed, FaultInjectionOperationType.READ_FEED_ITEM, FaultInjectionServerErrorType.TOO_MANY_REQUEST, true, 429, HttpConstants.SubStatusCodes.USER_REQUEST_RATE_TOO_LARGE },
             { OperationType.ReadFeed, FaultInjectionOperationType.READ_FEED_ITEM, FaultInjectionServerErrorType.READ_SESSION_NOT_AVAILABLE, true, 404, 1002 },
             { OperationType.ReadFeed, FaultInjectionOperationType.READ_FEED_ITEM, FaultInjectionServerErrorType.TIMEOUT, true, 410, HttpConstants.SubStatusCodes.SERVER_GENERATED_408 }, // for server return 408, SDK will wrap into 410/21010
             { OperationType.ReadFeed, FaultInjectionOperationType.READ_FEED_ITEM, FaultInjectionServerErrorType.PARTITION_IS_MIGRATING, true, 410, 1008 },
@@ -142,7 +142,7 @@ public static Object[][] faultInjectionServerErrorResponseProvider() {
             { OperationType.Create, FaultInjectionOperationType.CREATE_ITEM, FaultInjectionServerErrorType.GONE, true, 410, HttpConstants.SubStatusCodes.SERVER_GENERATED_410 },
             { OperationType.Create, FaultInjectionOperationType.CREATE_ITEM, FaultInjectionServerErrorType.INTERNAL_SERVER_ERROR, false, 500, 0 },
             { OperationType.Create, FaultInjectionOperationType.CREATE_ITEM, FaultInjectionServerErrorType.RETRY_WITH, true, 449, 0 },
-            { OperationType.Create, FaultInjectionOperationType.CREATE_ITEM, FaultInjectionServerErrorType.TOO_MANY_REQUEST, true, 429, 0 },
+            { OperationType.Create, FaultInjectionOperationType.CREATE_ITEM, FaultInjectionServerErrorType.TOO_MANY_REQUEST, true, 429, HttpConstants.SubStatusCodes.USER_REQUEST_RATE_TOO_LARGE },
             { OperationType.Create, FaultInjectionOperationType.CREATE_ITEM, FaultInjectionServerErrorType.TIMEOUT, false, 410, HttpConstants.SubStatusCodes.SERVER_GENERATED_408 }, // for server return 408, SDK will wrap into 410/21010
             { OperationType.Create, FaultInjectionOperationType.CREATE_ITEM, FaultInjectionServerErrorType.PARTITION_IS_MIGRATING, true, 410, 1008 },
             { OperationType.Create, FaultInjectionOperationType.CREATE_ITEM, FaultInjectionServerErrorType.PARTITION_IS_SPLITTING, true, 410, 1007 },
@@ -220,7 +220,7 @@ public void faultInjectionServerErrorRuleTests_OperationType(OperationType opera
                     cosmosDiagnostics,
                     operationType,
                     HttpConstants.StatusCodes.TOO_MANY_REQUESTS,
-                    HttpConstants.SubStatusCodes.UNKNOWN,
+                    HttpConstants.SubStatusCodes.USER_REQUEST_RATE_TOO_LARGE,
                     tooManyRequestsRuleId,
                     true);
             } else {
@@ -494,7 +494,7 @@ public void faultInjectionServerErrorRuleTests_Partition() throws JsonProcessing
             cosmosDiagnostics,
             OperationType.Read,
             HttpConstants.StatusCodes.TOO_MANY_REQUESTS,
-            HttpConstants.SubStatusCodes.UNKNOWN,
+            HttpConstants.SubStatusCodes.USER_REQUEST_RATE_TOO_LARGE,
             feedRangeRuleId,
             true
         );

diff --git a/...est/java/com/azure/cosmos/faultinjection/FaultInjectionServerErrorRuleOnGatewayTests.java b/...est/java/com/azure/cosmos/faultinjection/FaultInjectionServerErrorRuleOnGatewayTests.java
@@ -122,7 +122,7 @@ public static Object[][] faultInjectionServerErrorResponseProvider() {
             // faultInjectionServerError, will SDK retry, errorStatusCode, errorSubStatusCode
             { FaultInjectionServerErrorType.INTERNAL_SERVER_ERROR, false, 500, 0 },
             { FaultInjectionServerErrorType.RETRY_WITH, false, 449, 0 },
-            { FaultInjectionServerErrorType.TOO_MANY_REQUEST, true, 429, 0 },
+            { FaultInjectionServerErrorType.TOO_MANY_REQUEST, true, 429, HttpConstants.SubStatusCodes.USER_REQUEST_RATE_TOO_LARGE },
             { FaultInjectionServerErrorType.READ_SESSION_NOT_AVAILABLE, true, 404, 1002 },
             { FaultInjectionServerErrorType.SERVICE_UNAVAILABLE, false, 503, 21008 }
         };
@@ -283,7 +283,7 @@ public void faultInjectionServerErrorRuleTests_Partition() throws JsonProcessing
             cosmosDiagnostics,
             OperationType.Read,
             HttpConstants.StatusCodes.TOO_MANY_REQUESTS,
-            HttpConstants.SubStatusCodes.UNKNOWN,
+            HttpConstants.SubStatusCodes.USER_REQUEST_RATE_TOO_LARGE,
             feedRangeRuleId,
             true
         );
@@ -504,7 +504,7 @@ public void faultInjectionServerErrorRuleTests_HitLimit(
                         cosmosDiagnostics,
                         operationType,
                         HttpConstants.StatusCodes.TOO_MANY_REQUESTS,
-                        HttpConstants.SubStatusCodes.UNKNOWN,
+                        HttpConstants.SubStatusCodes.USER_REQUEST_RATE_TOO_LARGE,
                         hitLimitRuleId,
                         true
                     );