Skip to content

Commit

Permalink
[VBV-2227] Collect the logs of the old and upgraded admiral as part o…
Browse files Browse the repository at this point in the history 
…f AdmiralUpgradeIT

AdmiralUpgradeIT spins an old admiral and a latest admiral instances.
Then it tries to migrate data from the old to the new one. Finally, it
destroys both of these containers, regardless of the final state of the
test. With this, all logs are also lost and this makes it hard to
investigate pipeline issues.

With this change, the test will collect the logs of the containers and
will store them into files before destroying the containers.

Change-Id: I7047683a3de45a2f8da6f98dbad2aa5fc374c5b3
Reviewed-on: https://bellevue-ci.eng.vmware.com:8080/44196
Upgrade-Verified: jenkins <[email protected]>
Closures-Verified: jenkins <[email protected]>
CS-Verified: jenkins <[email protected]>
PG-Verified: jenkins <[email protected]>
Reviewed-by: Georgi Muleshkov <[email protected]>
Bellevue-Verified: jenkins <[email protected]>
  • Loading branch information
@shadjiiski
shadjiiski committed Sep 14, 2018
1 parent 687f136 commit 71993cf
Show file tree
Hide file tree
Showing 2 changed files with 120 additions and 14 deletions.
81 changes: 81 additions & 0 deletions test-upgrade/src/test/java/com/vmware/admiral/test/integration/AdmiralUpgradeBaseIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@

import static com.vmware.admiral.test.integration.TestPropertiesUtil.getSystemOrTestProp;

import java.io.File;
import java.io.PrintWriter;
import java.net.URI;
import java.time.Duration;
import java.util.HashSet;
import java.util.Set;

Expand All @@ -26,6 +29,7 @@
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;

import org.apache.commons.io.IOUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;

Expand All @@ -37,7 +41,9 @@
import com.vmware.admiral.compute.container.ContainerDescriptionFactoryService;
import com.vmware.admiral.compute.container.ContainerHostDataCollectionService;
import com.vmware.admiral.compute.container.ContainerHostDataCollectionService.ContainerHostDataCollectionState;
import com.vmware.admiral.compute.container.ContainerLogService;
import com.vmware.admiral.compute.container.ContainerService.ContainerState;
import com.vmware.admiral.service.common.LogService.LogServiceState;
import com.vmware.admiral.service.common.NodeHealthCheckService;
import com.vmware.admiral.service.common.NodeMigrationService;
import com.vmware.admiral.service.common.NodeMigrationService.MigrationRequest;
Expand All @@ -47,6 +53,7 @@
import com.vmware.photon.controller.model.resources.ComputeService;
import com.vmware.photon.controller.model.resources.ComputeService.ComputeState;
import com.vmware.xenon.common.Operation;
import com.vmware.xenon.common.Service;
import com.vmware.xenon.common.ServiceClient;
import com.vmware.xenon.common.UriUtils;
import com.vmware.xenon.common.Utils;
Expand All @@ -67,6 +74,12 @@ public abstract class AdmiralUpgradeBaseIT extends BaseProvisioningOnCoreOsIT {
private static final String COMPUTE_SELF_LINK = ComputeService.FACTORY_LINK + SEPARATOR
+ IntegratonTestStateFactory.DOCKER_COMPUTE_ID;

private static final String UPGRADE_CONTAINERS_LOGS_RETRIES = "upgrade.containers.logs.retires";
private static final String UPGRADE_CONTAINERS_LOGS_RETRY_DELAY_MS = "upgrade.containers.logs.retry.delay.ms";

private static final Long DEFAULT_CONTAINER_LOGS_RETRIES = 5L;
private static final Long DEFAULT_CONTAINER_LOGS_RETRY_DELAY_MS = 1000L;

private Set<String> applicationsToDelete = new HashSet<>();

protected static ServiceClient serviceClient;
Expand Down Expand Up @@ -275,6 +288,74 @@ protected void addContentToTheProvisionedAdmiral(ContainerState admiralContainer
setBaseURI(null);
}

protected void storeContainerLogs(ContainerState container, String logsDir) throws Throwable {
File logFile = new File(logsDir, generateLogNameForContainer(container));

String containerName = container.names.iterator().next();
logger.info("Trying to save logs for container %s in %s", containerName,
logFile.getAbsolutePath());
String logs = retrieveContainerLogs(container);
if (logs == null) {
logger.warning("No logs found for container %s. Nothing is saved.", containerName);
return;
}

try (PrintWriter writer = new PrintWriter(logFile)) {
IOUtils.write(logs, writer);
}
}

private String generateLogNameForContainer(ContainerState container) {
return String.format("%s-%s-%s.log",
getClass().getSimpleName(),
container.names.iterator().next(),
System.currentTimeMillis());
}

private String retrieveContainerLogs(ContainerState container) throws Throwable {
// Calling the API for container logs will trigger a call for the logs to docker. However,
// the backend is not waiting for the result to be returned from docker and gets whatever is
// currently stored in memory. In our case, since there were no previous calls for logs, no
// logs will be returned the first time. If there is some network latency and the logs are
// big enough, a few subsequent requests for logs might return nothing as well.

long retries = Long.valueOf(getSystemOrTestProp(UPGRADE_CONTAINERS_LOGS_RETRIES,
"" + DEFAULT_CONTAINER_LOGS_RETRIES));
return retrieveContainerLogs(container, retries);
}

private String retrieveContainerLogs(ContainerState container, long retries) throws Throwable {
String logs = null;

long delay = Long.valueOf(getSystemOrTestProp(UPGRADE_CONTAINERS_LOGS_RETRY_DELAY_MS,
"" + DEFAULT_CONTAINER_LOGS_RETRY_DELAY_MS));

while ((logs = doRetrieveContainerLogs(container)) == null && retries-- > 0) {
logger.warning("No logs found for container %s. Will retry %d times.",
container.names.iterator().next(), retries);
Thread.sleep(delay);
}

return logs;
}

private String doRetrieveContainerLogs(ContainerState container) throws Throwable {
String containerId = Service.getId(container.documentSelfLink);
String query = UriUtils.buildUriQuery(
"id", containerId,
"timestamps", Boolean.toString(true),
"since", "" + Duration.ofHours(1).getSeconds());
String url = ContainerLogService.SELF_LINK + UriUtils.URI_QUERY_CHAR + query;

LogServiceState logState = getDocument(url, LogServiceState.class);
if (logState.logs == null) {
return null;
}

String logs = new String(logState.logs);
return (logs.isEmpty() || "--".equals(logs)) ? null : logs;
}

protected void waitForSuccessfulHealthcheck(ContainerState admiralContainer) throws Exception {
logger.info("--- Waiting for a successful healthcheck. ---");
changeBaseURI(admiralContainer);
Expand Down
53 changes: 39 additions & 14 deletions test-upgrade/src/test/java/com/vmware/admiral/test/integration/AdmiralUpgradeIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
import java.net.URI;
import java.util.stream.Collectors;

import com.google.common.io.Files;

import org.junit.After;
import org.junit.Before;
import org.junit.Test;
Expand All @@ -27,6 +29,7 @@
import com.vmware.admiral.compute.container.ContainerService.ContainerState;
import com.vmware.admiral.request.RequestBrokerService.RequestBrokerState;
import com.vmware.xenon.common.Operation;
import com.vmware.xenon.common.Utils;

public class AdmiralUpgradeIT extends AdmiralUpgradeBaseIT {
private static final String TEMPLATE_FILE = "Admiral_master_and_0.9.1_release.yaml";
Expand All @@ -35,14 +38,18 @@ public class AdmiralUpgradeIT extends AdmiralUpgradeBaseIT {

private static final String UPGRADE_SKIP_INITIALIZE = "upgrade.skip.initialize";
private static final String UPGRADE_SKIP_VALIDATE = "upgrade.skip.validate";
private static final String UPGRADE_CONTAINERS_LOGS_DIR = "upgrade.containers.logs.dir";

private ContainerState admiralBranchContainer;
private ContainerState admiralMasterContainer;

private String compositeDescriptionLink;
private String containersLogsDir;

@Before
public void setUp() throws Exception {
containersLogsDir = getSystemOrTestProp(UPGRADE_CONTAINERS_LOGS_DIR,
Files.createTempDir().getAbsolutePath());
compositeDescriptionLink = importTemplate(serviceClient, TEMPLATE_FILE);
}

Expand Down Expand Up @@ -85,22 +92,40 @@ protected void validateAfterStart(String resourceDescLink, RequestBrokerState re
Operation.STATUS_CODE_OK);
admiralMasterContainer = getDocument(admiralMasterContainerLink, ContainerState.class);

String skipInit = getSystemOrTestProp(UPGRADE_SKIP_INITIALIZE, "false");
if (skipInit.equals(Boolean.FALSE.toString())) {
logger.info("---------- Initialize content before upgrade. --------");
addContentToTheProvisionedAdmiral(admiralBranchContainer);
} else {
logger.info("---------- Skipping content initialization. --------");
try {
String skipInit = getSystemOrTestProp(UPGRADE_SKIP_INITIALIZE, "false");
if (skipInit.equals(Boolean.FALSE.toString())) {
logger.info("---------- Initialize content before upgrade. --------");
addContentToTheProvisionedAdmiral(admiralBranchContainer);
} else {
logger.info("---------- Skipping content initialization. --------");
}

String skipValidate = getSystemOrTestProp(UPGRADE_SKIP_VALIDATE, "false");
if (skipValidate.equals(Boolean.FALSE.toString())) {
logger.info("---------- Migrate data and validate content. --------");
migrateData(admiralBranchContainer, admiralMasterContainer);
validateContent(admiralMasterContainer);
removeData(admiralMasterContainer);
} else {
logger.info("---------- Skipping content validation. --------");
}
} finally {
storeContainersLogs();
}
}

String skipValidate = getSystemOrTestProp(UPGRADE_SKIP_VALIDATE, "false");
if (skipValidate.equals(Boolean.FALSE.toString())) {
logger.info("---------- Migrate data and validate content. --------");
migrateData(admiralBranchContainer, admiralMasterContainer);
validateContent(admiralMasterContainer);
removeData(admiralMasterContainer);
} else {
logger.info("---------- Skipping content validation. --------");
private void storeContainersLogs() {
try {
storeContainerLogs(admiralBranchContainer, containersLogsDir);
} catch (Throwable e) {
logger.error("Failed to store logs for branch container: %s", Utils.toString(e));
}
try {
storeContainerLogs(admiralMasterContainer, containersLogsDir);
} catch (Throwable e) {
logger.error("Failed to store logs for master container: %s", Utils.toString(e));
}
}

}

0 comments on commit 71993cf

Please sign in to comment.