VBV-2270 - [Randomly failing tests]

ContainerRemovalTaskService.testRemoveOfStaleContainerOperationFromHostRemoval
with postgres

The problem was that the test was sending delete to the container state
and right after that it was asserting that the description is null. Like
this more or less at the same time get and delete were sent to the
container description service and this issue was hit:
https://review.ec.eng.vmware.com/#/c/17003/

The fix:
Container service on delete used to first complete the operation and
after that delete the description.
- Changed the service not to complete before deleting the description in
order to avoid the case described above.

Change-Id: I60e2127636354d82e230f9c9e7c6e4106f885b03

compute/src/main/java/com/vmware/admiral/compute/container/ContainerService.java

@@ -363,16 +363,17 @@ public void handlePatch(Operation patch) {
     @Override
     public void handleDelete(Operation delete) {
         ContainerState currentState = getState(delete);
-        super.handleDelete(delete);
 
-        deleteContainerDescription(currentState, DELETE_DESCRIPTION_RETRY_COUNT);
+        deleteContainerDescription(currentState, DELETE_DESCRIPTION_RETRY_COUNT, delete);
     }
 
-    private void deleteContainerDescription(ContainerState currentState, int retries) {
+    private void deleteContainerDescription(ContainerState currentState, int retries,
+            Operation delete) {
         // do no delete THE agent container description
         if ((currentState.descriptionLink == null)
                 || (SystemContainerDescriptions.AGENT_CONTAINER_DESCRIPTION_LINK
                         .equals(currentState.descriptionLink))) {
+            delete.complete();
             return;
         }
 
@@ -398,6 +399,7 @@ private void deleteContainerDescription(ContainerState currentState, int retries
                     if (r.hasException()) {
                         logSevere("Failed to retrieve child containers for: %s - %s",
                                 containerDescriptionLink, r.getException());
+                        delete.complete();
                     } else if (r.hasResult() && r.getCount() != 0) {
                         logFine("Child containers for: %s = %s",
                                 containerDescriptionLink, r.getCount());
@@ -408,23 +410,27 @@ private void deleteContainerDescription(ContainerState currentState, int retries
                             getHost().schedule(() -> {
                                 logInfo("Additional check for containers with description %s will be performed",
                                         containerDescriptionLink);
-                                deleteContainerDescription(currentState, retries - 1);
+                                deleteContainerDescription(currentState, retries - 1, delete);
                             }, CONTAINER_DESCRIPTION_DELETE_CHECK_WAIT, TimeUnit.MILLISECONDS);
+                            delete.complete();
+                        } else {
+                            delete.complete();
                         }
                     } else {
                         logInfo("No other child containers found for: %s",
                                 containerDescriptionLink);
-                        handleNoOtherContainerStatesFound(containerDescriptionLink);
+                        handleNoOtherContainerStatesFound(containerDescriptionLink, delete);
                     }
                 });
     }
 
-    private void handleNoOtherContainerStatesFound(String containerDescriptionLink) {
+    private void handleNoOtherContainerStatesFound(String containerDescriptionLink, Operation delete) {
         sendRequest(Operation.createGet(this, containerDescriptionLink)
                 .setCompletion((o, e) -> {
                     if (e != null) {
                         logSevere("Failed to retrieve parent container description for: %s - %s",
                                 containerDescriptionLink, e);
+                        delete.complete();
                         return;
                     }
 
@@ -433,11 +439,23 @@ private void handleNoOtherContainerStatesFound(String containerDescriptionLink)
                     // do no delete the description if it has autoredeploy
                     if ((cd != null) && (cd.healthConfig != null)
                             && Boolean.TRUE.equals(cd.healthConfig.autoredeploy)) {
+                        delete.complete();
                         return;
                     }
 
                     // delete the no longer used description
-                    sendRequest(Operation.createDelete(this, containerDescriptionLink));
+                    sendRequest(Operation.createDelete(this, containerDescriptionLink)
+                            .setCompletion((oo, ee) -> {
+                                if (ee != null) {
+                                    logSevere("Failed to delete container description: %s",
+                                            containerDescriptionLink);
+                                    delete.complete();
+                                } else {
+                                    logInfo("Container description %s deleted",
+                                            containerDescriptionLink);
+                                    delete.complete();
+                                }
+                            }));
                 }));
     }