Added support for flagging new tests

Set show.history.flag=true to allow new failures to be flagged in the reports.
kaselis · Apr 8, 2017 · 006e7bf · 006e7bf
1 parent fbe656f
commit 006e7bf
Show file tree

Hide file tree

Showing 14 changed files with 379 additions and 262 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -0,0 +1,23 @@
+pipeline {
+    agent any
+
+    tools {
+        maven 'Gradle 3.4'
+        jdk 'jdk8'
+    }
+
+    stages {
+        stage('Test') {
+            steps {
+                sh './gradlew clean test'
+                sh './gradlew integrationTests'
+                sh './gradlew browserTests'
+            }
+            post {
+                always {
+                    junit 'target/surefire-reports/**/*.xml, **/build/reports/integration-tests/TEST-*.xml'
+                }
+            }
+        }
+    }
+}
diff --git a/serenity-core/src/main/java/net/serenitybdd/core/history/ClearDirectoryContents.java b/serenity-core/src/main/java/net/serenitybdd/core/history/ClearDirectoryContents.java
@@ -0,0 +1,14 @@
+package net.serenitybdd.core.history;
+
+import org.apache.commons.io.FileUtils;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+class ClearDirectoryContents implements PrepareHistoryDirectory {
+
+    @Override
+    public void prepareHistoryDirectory(Path historyDirectory) throws IOException {
+        FileUtils.cleanDirectory(historyDirectory.toFile());
+    }
+}
diff --git a/...core/src/main/java/net/serenitybdd/core/history/FileSystemTestOutcomeSummaryRecorder.java b/...core/src/main/java/net/serenitybdd/core/history/FileSystemTestOutcomeSummaryRecorder.java
@@ -0,0 +1,119 @@
+package net.serenitybdd.core.history;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableMap;
+import com.google.inject.Inject;
+import net.thucydides.core.guice.Injectors;
+import net.thucydides.core.model.ReportNamer;
+import net.thucydides.core.model.ReportType;
+import net.thucydides.core.model.TestOutcome;
+import net.thucydides.core.reports.AcceptanceTestLoader;
+import net.thucydides.core.reports.json.JSONTestOutcomeReporter;
+import net.thucydides.core.reports.json.gson.GsonPreviousOutcomeConverter;
+import net.thucydides.core.util.EnvironmentVariables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static net.thucydides.core.ThucydidesSystemProperty.SERENITY_HISTORY_DIRECTORY;
+
+public class FileSystemTestOutcomeSummaryRecorder implements TestOutcomeSummaryRecorder {
+
+    private final Path historyDirectory;
+    private final Boolean deletePreviousHistory;
+    private final AcceptanceTestLoader testOutcomeReporter = new JSONTestOutcomeReporter();
+    private final GsonPreviousOutcomeConverter previousOutcomeConverter;
+    private static final Logger LOGGER = LoggerFactory.getLogger(FileSystemTestOutcomeSummaryRecorder.class);
+
+    private static Map<Boolean, PrepareHistoryDirectory> DELETE_STRATEGY =
+            ImmutableMap.of(
+                    false, new LeaveDirectoryContents(),
+                    true, new ClearDirectoryContents()
+            );
+
+
+    @Inject
+    public FileSystemTestOutcomeSummaryRecorder(EnvironmentVariables environmentVariables) {
+        this(Paths.get(SERENITY_HISTORY_DIRECTORY.from(environmentVariables, "history"),""), false);
+    }
+
+    /**
+     * Used mainly from Maven
+     */
+    public FileSystemTestOutcomeSummaryRecorder(Path historyDirectory, Boolean deletePreviousHistory) {
+        this.historyDirectory = historyDirectory;
+        this.deletePreviousHistory = deletePreviousHistory;
+        previousOutcomeConverter = new GsonPreviousOutcomeConverter(Injectors.getInjector().getInstance(EnvironmentVariables.class));
+    }
+
+    @Override
+    public void recordOutcomeSummariesFrom(Path sourceDirectory) {
+
+        try (DirectoryStream<Path> directoryStream = Files.newDirectoryStream(sourceDirectory)) {
+
+            usingDeleteStrategyFor(deletePreviousHistory).prepareHistoryDirectory(historyDirectory);
+
+            for (Path path : directoryStream) {
+                storeOutcomesFrom(testOutcomeReporter.loadReportFrom(path).asSet());
+            }
+        } catch (IOException ex) {
+            LOGGER.warn("Unable to store test outcome for posterity", ex);
+        }
+    }
+
+    private void storeOutcomesFrom(Set<TestOutcome> testOutcomes) throws IOException {
+        for (TestOutcome testOutcome : testOutcomes) {
+            PreviousTestOutcome summary = PreviousTestOutcome.from(testOutcome);
+            File summaryFile = summaryFileFor(testOutcome);
+
+            try (OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(summaryFile))) {
+                previousOutcomeConverter.toJson(summary, outputStream);
+                outputStream.flush();
+            }
+        }
+    }
+
+    @Override
+    public List<PreviousTestOutcome> loadSummaries() {
+        List<PreviousTestOutcome> previousTestOutcomes = new ArrayList<>();
+
+        if (historyDirectory.toFile().exists()) {
+            try (DirectoryStream<Path> directoryStream = Files.newDirectoryStream(historyDirectory)) {
+                for (Path path : directoryStream) {
+                    previousTestOutcomes.addAll(previousTestOutcomesFrom(path).asSet());
+                }
+            } catch (IOException ex) {
+                LOGGER.warn("Unable to store test outcome for posterity", ex);
+            }
+        }
+
+        return previousTestOutcomes;
+    }
+
+    private Optional<PreviousTestOutcome> previousTestOutcomesFrom(Path source) {
+        try (InputStream inputStream = new BufferedInputStream(new FileInputStream(source.toFile()))) {
+            return previousOutcomeConverter.fromJson(inputStream);
+        } catch (IOException e) {
+            return Optional.absent();
+        }
+    }
+
+    private PrepareHistoryDirectory usingDeleteStrategyFor(Boolean deletePreviousHistory) {
+        return DELETE_STRATEGY.get(deletePreviousHistory);
+    }
+
+    private File summaryFileFor(TestOutcome testOutcome) {
+        String summaryFilename = ReportNamer.forReportType(ReportType.JSON).withPrefix("summary-").getNormalizedTestNameFor(testOutcome);
+        return historyDirectory.resolve(summaryFilename).toFile();
+    }
+
+}
diff --git a/serenity-core/src/main/java/net/serenitybdd/core/history/HistoricalFlagProvider.java b/serenity-core/src/main/java/net/serenitybdd/core/history/HistoricalFlagProvider.java
@@ -3,34 +3,83 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.inject.Inject;
 import net.thucydides.core.model.TestOutcome;
+import net.thucydides.core.model.TestResult;
 import net.thucydides.core.model.flags.Flag;
 import net.thucydides.core.model.flags.FlagProvider;
 import net.thucydides.core.util.EnvironmentVariables;
+import org.apache.commons.lang3.StringUtils;
 
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
-import static net.thucydides.core.ThucydidesSystemProperty.SERENITY_HISTORY_DIRECTORY;
 import static net.thucydides.core.ThucydidesSystemProperty.SHOW_HISTORY_FLAGS;
 
 public class HistoricalFlagProvider implements FlagProvider {
 
     private final EnvironmentVariables environmentVariables;
+    private final Map<String, PreviousTestOutcome> previousTestOutcomesById;
+
+    private final Set<? extends Flag> NO_FLAGS = ImmutableSet.of();
 
     @Inject
-    public HistoricalFlagProvider(EnvironmentVariables environmentVariables) {
+    public HistoricalFlagProvider(EnvironmentVariables environmentVariables,
+                                  TestOutcomeSummaryRecorder summaryRecorder) {
+
         this.environmentVariables = environmentVariables;
+
+        List<PreviousTestOutcome> previousOutcomes = summaryRecorder.loadSummaries();
+
+        previousTestOutcomesById = indexById(previousOutcomes);
+    }
+
+    private Map<String, PreviousTestOutcome> indexById(List<PreviousTestOutcome> previousOutcomes) {
+        Map<String, PreviousTestOutcome> previousTestOutcomesById = new HashMap<>();
+        for(PreviousTestOutcome previousTestOutcome : previousOutcomes) {
+            previousTestOutcomesById.put(previousTestOutcome.getId(), previousTestOutcome);
+        }
+        return previousTestOutcomesById;
     }
 
     @Override
     public Set<? extends Flag> getFlagsFor(TestOutcome testOutcome) {
         if (!historicalFlagsAreActivated()) { return ImmutableSet.of(); }
 
+        return newFailureIn(testOutcome) ? ImmutableSet.of(NewFailure.FLAG) : NO_FLAGS;
+    }
+
+    private boolean newFailureIn(TestOutcome testOutcome) {
+        if (isUnknownTest(testOutcome)) { return false; }
+        if (isNotBroken(testOutcome)) { return false; }
+
+        PreviousTestOutcome previousTestOutcome = previousTestOutcomesById.get(testOutcome.getId());
+        if (resultIsDifferent(testOutcome, previousTestOutcome)) { return true; }
+        if (causeIsDifferent(testOutcome, previousTestOutcome)) { return true; }
+
+        return false;
+    }
+
+    private boolean isUnknownTest(TestOutcome testOutcome) {
+        return (!previousTestOutcomesById.containsKey(testOutcome.getId()));
+    }
+
+    private boolean isNotBroken(TestOutcome testOutcome) {
+        return ((testOutcome.getResult() == TestResult.SUCCESS)
+                || (testOutcome.getResult() == TestResult.IGNORED)
+                || (testOutcome.getResult() == TestResult.PENDING)
+                || (testOutcome.getResult() == TestResult.SKIPPED));
+    }
+
+    private boolean causeIsDifferent(TestOutcome testOutcome, PreviousTestOutcome previousTestOutcome) {
+        return (!StringUtils.equals(testOutcome.getTestFailureSummary(), previousTestOutcome.getTestFailureSummary()));
+    }
 
-        return ImmutableSet.of(NewFailure.FLAG);
+    private boolean resultIsDifferent(TestOutcome testOutcome, PreviousTestOutcome previousTestOutcome) {
+        return (testOutcome.getResult() != previousTestOutcome.getResult());
     }
 
     private boolean historicalFlagsAreActivated() {
-        return SHOW_HISTORY_FLAGS.booleanFrom(environmentVariables, false)
-                && SERENITY_HISTORY_DIRECTORY.isDefinedIn(environmentVariables);
+        return SHOW_HISTORY_FLAGS.booleanFrom(environmentVariables, false);
     }
 }
diff --git a/serenity-core/src/main/java/net/serenitybdd/core/history/LeaveDirectoryContents.java b/serenity-core/src/main/java/net/serenitybdd/core/history/LeaveDirectoryContents.java
@@ -0,0 +1,9 @@
+package net.serenitybdd.core.history;
+
+import java.nio.file.Path;
+
+class LeaveDirectoryContents implements PrepareHistoryDirectory {
+
+        @Override
+        public void prepareHistoryDirectory(Path historyDirectory) {}
+    }
diff --git a/serenity-core/src/main/java/net/serenitybdd/core/history/PrepareHistoryDirectory.java b/serenity-core/src/main/java/net/serenitybdd/core/history/PrepareHistoryDirectory.java
@@ -0,0 +1,8 @@
+package net.serenitybdd.core.history;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+interface PrepareHistoryDirectory {
+        void prepareHistoryDirectory(Path historyDirectory) throws IOException;
+    }