first basic version of WFC Report

WebFuzzing · Dec 27, 2024 · fd53243 · fd53243
1 parent 0d7e601
commit fd53243
Show file tree

Hide file tree

Showing 12 changed files with 123 additions and 52 deletions.
diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt
@@ -594,6 +594,9 @@ class EMConfig {
         if(dockerLocalhost && !runningInDocker){
             throw ConfigProblemException("Specifying 'dockerLocalhost' only makes sense when running EvoMaster inside Docker.")
         }
+        if(writeWFCReport && !createTests){
+            throw ConfigProblemException("Cannot create a WFC Report if tests are not generated (i.e., 'createTests' is false)")
+        }
     }
 
     private fun checkPropertyConstraints(m: KMutableProperty<*>) {
@@ -1242,6 +1245,11 @@ class EMConfig {
     @FilePath
     var statisticsFile = "statistics.csv"
 
+
+    @Experimental
+    @Cfg("Output a JSON file representing statistics of the fuzzing session, written in the WFC Report format.")
+    var writeWFCReport = false
+
     @Cfg("Whether should add to an existing statistics file, instead of replacing it")
     var appendToStatisticsFile = false
 

diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt
@@ -18,6 +18,7 @@ import org.evomaster.core.output.TestSuiteSplitter
 import org.evomaster.core.output.clustering.SplitResult
 import org.evomaster.core.output.service.TestSuiteWriter
 import org.evomaster.core.problem.api.ApiWsIndividual
+import org.evomaster.core.problem.enterprise.service.WFCReportWriter
 import org.evomaster.core.problem.externalservice.httpws.service.HarvestActualHttpWsResponseHandler
 import org.evomaster.core.problem.externalservice.httpws.service.HttpWsExternalServiceHandler
 import org.evomaster.core.problem.graphql.GraphQLIndividual
@@ -257,6 +258,7 @@ class Main {
 
             writeCoveredTargets(injector, solution)
             writeTests(injector, solution, controllerInfo)
+            writeWFCReport(injector, solution)
             writeStatistics(injector, solution) //FIXME if other phases after search, might get skewed data on 100% snapshots...
 
             resetExternalServiceHandler(injector)
@@ -690,19 +692,14 @@ class Main {
 
             val writer = injector.getInstance(TestSuiteWriter::class.java)
 
-            //TODO: enable splitting for csharp. Currently not enabled due to an error while running generated tests in multiple classes (error in starting the SUT)
-            if (config.problemType == EMConfig.ProblemType.REST && !config.outputFormat.isCsharp()) {
+            if (config.problemType == EMConfig.ProblemType.REST ) {
 
-                val splitResult = TestSuiteSplitter.split(solution, config, writer.getPartialOracles())
+                val splitResult = TestSuiteSplitter.split(solution, config)
 
                 solution.clusteringTime = splitResult.clusteringTime.toInt()
-                splitResult.splitOutcome.filter { !it.individuals.isNullOrEmpty() }
+                splitResult.splitOutcome
+                    .filter { !it.individuals.isNullOrEmpty() }
                     .forEach { writer.writeTests(it, controllerInfoDto?.fullName, controllerInfoDto?.executableFullPath, snapshotTimestamp) }
-
-//                if (config.executiveSummary) {
-//                    writeExecSummary(injector, controllerInfoDto, splitResult, snapshotTimestamp)
-//                    //writeExecutiveSummary(injector, solution, controllerInfoDto, partialOracles)
-//                }
             } else {
                 /*
                     TODO refactor all the PartialOracle stuff that is meant for only REST
@@ -715,6 +712,8 @@ class Main {
         fun writeTests(injector: Injector, solution: Solution<*>, controllerInfoDto: ControllerInfoDto?,
                        snapshot: String = "") {
 
+            //TODO: the code here is quite messy. Needs to be refactored and simplified
+
             val config = injector.getInstance(EMConfig::class.java)
 
             if (!config.createTests) {
@@ -727,10 +726,9 @@ class Main {
             LoggingUtil.getInfoLogger().info("Going to save $tests to ${config.outputFolder}")
 
             val writer = injector.getInstance(TestSuiteWriter::class.java)
-            //TODO: enable splitting for csharp. Currently not enabled due to an error while running generated tests in multiple classes (error in starting the SUT)
-            if (config.problemType == EMConfig.ProblemType.REST && !config.outputFormat.isCsharp()) {
+            if (config.problemType == EMConfig.ProblemType.REST) {
 
-                val splitResult = TestSuiteSplitter.split(solution, config, writer.getPartialOracles())
+                val splitResult = TestSuiteSplitter.split(solution, config)
 
                 solution.clusteringTime = splitResult.clusteringTime.toInt()
                 splitResult.splitOutcome
@@ -741,21 +739,11 @@ class Main {
                             config.maxTestsPerTestSuite
                         )
                     }
-                    .forEach { writer.writeTests(it, controllerInfoDto?.fullName,controllerInfoDto?.executableFullPath, snapshot) }
+                    .forEach {
+                        writer.writeTests(it, controllerInfoDto?.fullName,controllerInfoDto?.executableFullPath, snapshot)
+                    }
 
-//                if (config.executiveSummary) {
-//
-//                    // Onur - if there are fault cases, executive summary makes sense
-//                    if ( splitResult.splitOutcome.any{ it.individuals.isNotEmpty()
-//                                && it.termination != Termination.SUCCESSES}) {
-//                        writeExecSummary(injector, controllerInfoDto, splitResult)
-//                    }
-//
-//                    //writeExecSummary(injector, controllerInfoDto, splitResult)
-//                    //writeExecutiveSummary(injector, solution, controllerInfoDto, partialOracles)
-//                }
-            } else
-                if (config.problemType == EMConfig.ProblemType.RPC){
+            } else if (config.problemType == EMConfig.ProblemType.RPC){
 
                 // Man: only enable for RPC as it lacks of unit tests
                 writer.writeTestsDuringSeeding(solution, controllerInfoDto?.fullName, controllerInfoDto?.executableFullPath)
@@ -766,7 +754,7 @@ class Main {
                         for RPC, just simple split based on whether there exist any exception in a test
                         TODD need to check with Andrea whether we use cluster or other type
                      */
-                    EMConfig.TestSuiteSplitType.FAULTS -> {
+                    else -> {
                         val splitResult = TestSuiteSplitter.splitRPCByException(solution as Solution<RPCIndividual>)
                         splitResult.splitOutcome
                             .filter { !it.individuals.isNullOrEmpty() }
@@ -777,21 +765,13 @@ class Main {
                                 )
                             }
                             .forEach { writer.writeTests(it, controllerInfoDto?.fullName,controllerInfoDto?.executableFullPath, snapshot) }
-
-                        // disable executiveSummary
-//                        if (config.executiveSummary) {
-//                            writeExecSummary(injector, controllerInfoDto, splitResult)
-//                        }
                     }
                 }
 
             }else if (config.problemType == EMConfig.ProblemType.GRAPHQL) {
                 when(config.testSuiteSplitType){
                     EMConfig.TestSuiteSplitType.NONE -> writer.writeTests(solution, controllerInfoDto?.fullName, controllerInfoDto?.executableFullPath,)
-                    //EMConfig.TestSuiteSplitType.CLUSTER -> throw IllegalStateException("GraphQL problem does not support splitting tests by cluster at this time")
-                    //EMConfig.TestSuiteSplitType.CODE ->
                     else -> {
-                        //throw IllegalStateException("GraphQL problem does not support splitting tests by code at this time")
                         val splitResult = TestSuiteSplitter.split(solution, config)
                         splitResult.splitOutcome
                             .filter{ !it.individuals.isNullOrEmpty() }
@@ -801,22 +781,31 @@ class Main {
                                     config.maxTestsPerTestSuite
                                 )
                             }
-                            .forEach { writer.writeTests(it, controllerInfoDto?.fullName, controllerInfoDto?.executableFullPath, snapshot ) }
+                            .forEach {
+                                writer.writeTests(it, controllerInfoDto?.fullName, controllerInfoDto?.executableFullPath, snapshot )
+                            }
                     }
-                    /*
-                      GraphQL could be split by code (where code is available and trustworthy)
-                     */
                 }
-            } else
-            {
-                /*
-                    TODO refactor all the PartialOracle stuff that is meant for only REST
-                 */
-
+            } else {
                 writer.writeTests(solution, controllerInfoDto?.fullName, controllerInfoDto?.executableFullPath,)
             }
         }
 
+        private fun writeWFCReport(injector: Injector, solution: Solution<*>){
+
+            //TODO will need to change input from Solution to the outout of generated tests
+
+            val config = injector.getInstance(EMConfig::class.java)
+
+            if (!config.writeWFCReport) {
+                return
+            }
+
+            val wfcr = injector.getInstance(WFCReportWriter::class.java)
+
+            wfcr.writeReport(solution)
+        }
+
         private fun writeStatistics(injector: Injector, solution: Solution<*>) {
 
             val config = injector.getInstance(EMConfig::class.java)

diff --git a/core/src/main/kotlin/org/evomaster/core/output/TestSuiteSplitter.kt b/core/src/main/kotlin/org/evomaster/core/output/TestSuiteSplitter.kt
@@ -78,8 +78,7 @@ object TestSuiteSplitter {
      * the original [Solution]
      */
     fun split(solution: Solution<*>,
-              config: EMConfig,
-              oracles: PartialOracles = PartialOracles()
+              config: EMConfig
     ): SplitResult {
 
         // TODO splitting support for other problem types

diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseModule.kt
@@ -0,0 +1,13 @@
+package org.evomaster.core.problem.enterprise.service
+
+import com.google.inject.AbstractModule
+
+abstract class EnterpriseModule :  AbstractModule() {
+
+    override fun configure() {
+        super.configure()
+
+        bind(WFCReportWriter::class.java)
+            .asEagerSingleton()
+    }
+}
diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/WFCReportWriter.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/WFCReportWriter.kt
@@ -0,0 +1,53 @@
+package org.evomaster.core.problem.enterprise.service
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.google.inject.Inject
+import com.webfuzzing.commons.report.Faults
+import com.webfuzzing.commons.report.RESTReport
+import org.evomaster.core.EMConfig
+import org.evomaster.core.search.Solution
+import java.nio.file.Files
+import java.nio.file.Paths
+import java.util.*
+
+class WFCReportWriter {
+
+
+    @Inject
+    private lateinit var config: EMConfig
+
+
+    fun writeReport(solution: Solution<*>) {
+
+        val report = com.webfuzzing.commons.report.Report()
+
+        report.schemaVersion = "0.0.1" //TODO
+        report.toolName = "EvoMaster"
+        //TODO tool version
+        report.creationTime = Date() // FIXME use new JDK dates
+
+        if(config.problemType == EMConfig.ProblemType.REST) {
+            val rest = RESTReport()
+            report.restReport = rest
+
+            val faults = Faults()
+            rest.faults = faults
+
+            faults.totalNumber = solution.totalNumberOfDetectedFaults()
+
+            //TODO all other entries
+        }
+        //TODO other problem types
+
+        val jackson = ObjectMapper()
+        val json = jackson.writeValueAsString(report)
+
+        val path = Paths.get(config.outputFolder, "report.json").toAbsolutePath()
+
+        Files.createDirectories(path.parent)
+        Files.deleteIfExists(path)
+        Files.createFile(path)
+
+        path.toFile().appendText(json)
+    }
+}
diff --git a/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLModule.kt
@@ -5,6 +5,7 @@ import com.google.inject.TypeLiteral
 import org.evomaster.core.output.service.GraphQLTestCaseWriter
 import org.evomaster.core.output.service.TestCaseWriter
 import org.evomaster.core.output.service.TestSuiteWriter
+import org.evomaster.core.problem.enterprise.service.EnterpriseModule
 import org.evomaster.core.problem.graphql.GraphQLIndividual
 import org.evomaster.core.problem.rest.RestIndividual
 import org.evomaster.core.remote.service.RemoteController
@@ -17,7 +18,7 @@ import org.evomaster.core.search.service.mutator.Mutator
 import org.evomaster.core.search.service.mutator.StandardMutator
 import org.evomaster.core.search.service.mutator.StructureMutator
 
-class GraphQLModule : AbstractModule() {
+class GraphQLModule : EnterpriseModule() {
 
     override fun configure() {
 

diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/RestBaseModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/RestBaseModule.kt
@@ -5,12 +5,13 @@ import com.google.inject.TypeLiteral
 import org.evomaster.core.output.service.RestTestCaseWriter
 import org.evomaster.core.output.service.TestCaseWriter
 import org.evomaster.core.output.service.TestSuiteWriter
+import org.evomaster.core.problem.enterprise.service.EnterpriseModule
 import org.evomaster.core.problem.rest.RestIndividual
 import org.evomaster.core.search.service.Archive
 import org.evomaster.core.search.service.Minimizer
 import org.evomaster.core.seeding.service.rest.PirToRest
 
-open class RestBaseModule : AbstractModule() {
+open class RestBaseModule : EnterpriseModule() {
 
     override fun configure() {
 

diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rpc/service/RPCModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/rpc/service/RPCModule.kt
@@ -5,6 +5,7 @@ import com.google.inject.TypeLiteral
 import org.evomaster.core.output.service.RPCTestCaseWriter
 import org.evomaster.core.output.service.TestCaseWriter
 import org.evomaster.core.output.service.TestSuiteWriter
+import org.evomaster.core.problem.enterprise.service.EnterpriseModule
 import org.evomaster.core.problem.rest.RestIndividual
 import org.evomaster.core.problem.rpc.RPCIndividual
 import org.evomaster.core.problem.webfrontend.WebIndividual
@@ -21,7 +22,7 @@ import org.evomaster.core.search.service.mutator.StructureMutator
 /**
  * created by manzhang on 2021/11/26
  */
-class RPCModule : AbstractModule(){
+class RPCModule : EnterpriseModule(){
 
     override fun configure() {
         bind(object : TypeLiteral<Sampler<RPCIndividual>>() {})

diff --git a/core/src/main/kotlin/org/evomaster/core/problem/webfrontend/service/WebModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/webfrontend/service/WebModule.kt
@@ -5,6 +5,7 @@ import com.google.inject.TypeLiteral
 import org.evomaster.core.output.service.TestCaseWriter
 import org.evomaster.core.output.service.TestSuiteWriter
 import org.evomaster.core.output.service.WebTestCaseWriter
+import org.evomaster.core.problem.enterprise.service.EnterpriseModule
 import org.evomaster.core.problem.graphql.GraphQLIndividual
 import org.evomaster.core.problem.rest.RestIndividual
 import org.evomaster.core.problem.webfrontend.WebIndividual
@@ -24,7 +25,7 @@ import org.evomaster.core.search.service.mutator.StructureMutator
  *
  * TODO See equivalent RestModule
  */
-class WebModule: AbstractModule() {
+class WebModule: EnterpriseModule() {
 
     override fun configure() {
         bind(object : TypeLiteral<Sampler<WebIndividual>>() {})

diff --git a/core/src/main/kotlin/org/evomaster/core/search/Solution.kt b/core/src/main/kotlin/org/evomaster/core/search/Solution.kt
@@ -87,4 +87,8 @@ where T : Individual {
             .map { it.code }
             .toSet()
     }
+
+    fun totalNumberOfDetectedFaults() : Int {
+        return DetectedFaultUtils.getDetectedFaults(this).size
+    }
 }
diff --git a/docs/options.md b/docs/options.md
@@ -279,3 +279,4 @@ There are 3 types of options:
 |`useWeightedSampling`| __Boolean__. When sampling from archive based on targets, decide whether to use weights based on properties of the targets (e.g., a target likely leading to a flag will be sampled less often). *Default value*: `false`.|
 |`wbProbabilityUseDataPool`| __Double__. Specify the probability of using the data pool when sampling test cases. This is for white-box (wb) mode. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.2`.|
 |`writeSnapshotTestsIntervalInSeconds`| __Int__. The size (in seconds) of the interval that the snapshots will be printed, if enabled. *Default value*: `3600`.|
+|`writeWFCReport`| __Boolean__. Output a JSON file representing statistics of the fuzzing session, written in the WFC Report format. *Default value*: `false`.|
diff --git a/.../src/test/java/org/evomaster/e2etests/spring/examples/splitter/TestSuiteSplitterTest.java b/.../src/test/java/org/evomaster/e2etests/spring/examples/splitter/TestSuiteSplitterTest.java
@@ -80,7 +80,7 @@ private void testRunEMMulti(EMConfig.TestSuiteSplitType splitType, Boolean execu
 
             Solution<RestIndividual> solution = initAndRun(args);
             assertTrue(solution.getIndividuals().size() >= 1);
-            SplitResult splits = TestSuiteSplitter.INSTANCE.split(solution, em, new PartialOracles());
+            SplitResult splits = TestSuiteSplitter.INSTANCE.split(solution, em);
             assertTrue(splits.splitOutcome.size() >= 1);
         };