Provide native S3 storage mercedes-benz#74

- refactored JobStorage and StorageService
  - introducing interface for job storage
  - instead file paths we use input streams now
  - divided s3 and filesystem parts
  - storage service does now check if s3 setup done,
    if not fallback to filesystem variant. If even
    filesystem setup not done initialization fails
  - SchedulerUploadService does now always store uploaded source code
    temporary on disk, validate it there and after this store content
    into shared volume or s3. So only valid content is available in
    job storage. Temporary files are always deleted after storage was
    done
- integrated minio client
- added s3mock dependency for integration tests
- add some s3 storage tests
  (currently one fails, maybe a minio problem, see
    minio/minio-java#710 )

On behalf of Daimler TSS GmbH.

sechub-adapter-checkmarx/src/main/java/com/daimler/sechub/adapter/checkmarx/CheckmarxAdapterConfig.java

@@ -1,12 +1,14 @@
 // SPDX-License-Identifier: MIT
 package com.daimler.sechub.adapter.checkmarx;
 
+import java.io.InputStream;
+
 import com.daimler.sechub.adapter.AdapterConfig;
 
 public interface CheckmarxAdapterConfig extends AdapterConfig {
 
 	String getTeamIdForNewProjects();
-	
-	String getPathToZipFile();
+
+	InputStream getSourceCodeZipFileInputStream();
 
 }

sechub-adapter-checkmarx/src/main/java/com/daimler/sechub/adapter/checkmarx/CheckmarxConfig.java

@@ -1,34 +1,37 @@
 // SPDX-License-Identifier: MIT
 package com.daimler.sechub.adapter.checkmarx;
 
+import java.io.InputStream;
+
 import com.daimler.sechub.adapter.AbstractCodeScanAdapterConfig;
 import com.daimler.sechub.adapter.AbstractCodeScanAdapterConfigBuilder;
 
 public class CheckmarxConfig extends AbstractCodeScanAdapterConfig implements CheckmarxAdapterConfig{
-	
+
 	private String teamIdForNewProjects;
-	private String pathToZipFile;
-	
+	private InputStream sourceCodeZipFileInputStream;
+
 	private CheckmarxConfig() {
 	}
 
 	@Override
 	public String getTeamIdForNewProjects() {
 		return teamIdForNewProjects;
 	}
+
 	@Override
-	public String getPathToZipFile() {
-		return pathToZipFile;
+	public InputStream getSourceCodeZipFileInputStream() {
+		return sourceCodeZipFileInputStream;
 	}
-	
+
 	public static CheckmarxConfigBuilder builder() {
 		return new CheckmarxConfigBuilder();
 	}
 
 	public static class CheckmarxConfigBuilder extends AbstractCodeScanAdapterConfigBuilder<CheckmarxConfigBuilder, CheckmarxConfig>{
 
 		private String teamIdForNewProjects;
-		private String pathToZipFile;
+		private InputStream sourceCodeZipFileInputStream;
 
 		/**
 		 * When we create a new project this is the team ID to use
@@ -39,15 +42,15 @@ public CheckmarxConfigBuilder setTeamIdForNewProjects(String teamId){
 			this.teamIdForNewProjects=teamId;
 			return this;
 		}
-		public CheckmarxConfigBuilder setPathToZipFile(String pathToZipFile){
-			this.pathToZipFile=pathToZipFile;
+		public CheckmarxConfigBuilder setSourceCodeZipFileInputStream(InputStream sourceCodeZipFileInputStream){
+			this.sourceCodeZipFileInputStream=sourceCodeZipFileInputStream;
 			return this;
 		}
-		
+
 		@Override
 		protected void customBuild(CheckmarxConfig config) {
 			config.teamIdForNewProjects=teamIdForNewProjects;
-			config.pathToZipFile=pathToZipFile;
+			config.sourceCodeZipFileInputStream=sourceCodeZipFileInputStream;
 		}
 
 		@Override
@@ -70,5 +73,5 @@ protected void assertTeamIdSet() {
 		}
 	}
 
-	
+
 }

sechub-adapter-checkmarx/src/main/java/com/daimler/sechub/adapter/checkmarx/support/CheckmarxUploadSupport.java

@@ -1,9 +1,10 @@
 // SPDX-License-Identifier: MIT
 package com.daimler.sechub.adapter.checkmarx.support;
 
-import java.io.File;
+import java.io.InputStream;
 
-import org.springframework.core.io.FileSystemResource;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.core.io.Resource;
 import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpMethod;
@@ -28,7 +29,7 @@ public void uploadZippedSourceCode(CheckmarxContext context)
 			throws AdapterException {
 		CheckmarxAdapterConfig config = context.getConfig();
 
-		FileSystemResource sourceCodeFile = fetchSystemResource(context, config);
+		Resource sourceCodeFile = fetchResource(context, config);
 
 		HttpHeaders headers = new HttpHeaders();
 		headers.setContentType(MediaType.MULTIPART_FORM_DATA);
@@ -38,23 +39,21 @@ public void uploadZippedSourceCode(CheckmarxContext context)
 		HttpEntity<MultiValueMap<String, Object>> requestEntity = new HttpEntity<>(body, headers);
 
 		String url = context.getAPIURL("projects/" + context.getSessionData().getProjectId() + "/sourceCode/attachments");
-		
+
 		RestOperations restTemplate = context.getRestOperations();
-		
+
 		ResponseEntity<String> result = restTemplate.exchange(url, HttpMethod.POST, requestEntity, String.class);
 		if (! result.getStatusCode().equals(HttpStatus.NO_CONTENT)) {
 			throw context.asAdapterException("Response HTTP status not as expected: "+result.getStatusCode(), null);
 		}
 	}
 
-	private FileSystemResource fetchSystemResource(CheckmarxAdapterContext context, CheckmarxAdapterConfig config)
+	private Resource fetchResource(CheckmarxAdapterContext context, CheckmarxAdapterConfig config)
 			throws AdapterException {
-		String pathToZipFile = config.getPathToZipFile();
-		/* currently we only provide file pathes... */
-		File file = new File(pathToZipFile);
-		if (!file.exists()) {
-			throw context.asAdapterException("File does not exist:" + pathToZipFile, null);
+		InputStream zipInputstream = config.getSourceCodeZipFileInputStream();
+		if (zipInputstream==null) {
+			throw context.asAdapterException("Input stream containing zip file is null!");
 		}
-		return new FileSystemResource(file);
+		return new InputStreamResource(zipInputstream);
 	}
 }

sechub-adapter-checkmarx/src/test/java/com/daimler/sechub/adapter/checkmarx/CheckmarxAdapterTestApplication.java

@@ -2,6 +2,7 @@
 package com.daimler.sechub.adapter.checkmarx;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileWriter;
 
 /**
@@ -14,7 +15,7 @@ public class CheckmarxAdapterTestApplication {
 	public static void main(String[] args) throws Exception {
 		System.setProperty("log4j.logger.org.apache.http","ERROR");
 		System.setProperty("org.apache.commons.logging.simplelog.log.org.apache.http","OFF");
-		
+
 		dump("https.proxyHost");
 		dump("https.proxyPort");
 		dump("https.nonProxyHosts");
@@ -23,7 +24,7 @@ public static void main(String[] args) throws Exception {
 		dump("http.nonProxyHosts");
 		dump("javax.net.ssl.keyStore");
 		dump("javax.net.ssl.trustStore");
-		
+
 		String user = System.getProperty("test.sechub.adapter.checkmarx.user");
 		if (user==null || user.isEmpty()) {
 			throw new IllegalArgumentException("user not set in system properties!");
@@ -38,25 +39,25 @@ public static void main(String[] args) throws Exception {
 		}
 		String projectname = System.getProperty("test.sechub.adapter.checkmarx.projectName");
 		String teamId = System.getProperty("test.sechub.adapter.checkmarx.teamId");
-		
+
 		String pathInOtherProject = "zipfile_contains_only_test1.txt.zip"; // leads to FAILED in queue
 		pathInOtherProject="zipfile_contains_only_one_simple_java_file.zip"; // should work
 		pathInOtherProject="zipfile_contains_sechub_doc_java.zip"; // should work
-		
+
 		File zipFile = CheckmarxTestFileSupport.getTestfileSupport().createFileFromRoot("sechub-other/testsourcecode/"+pathInOtherProject);
 		/* @formatter:off */
-		CheckmarxAdapterConfig config = 
+		CheckmarxAdapterConfig config =
 				CheckmarxConfig.builder().
 					setUser(user).
 					setProjectId(projectname).
 					setTeamIdForNewProjects(teamId).
 					setPasswordOrAPIToken(password).
-					setPathToZipFile(zipFile.getAbsolutePath()).
+					setSourceCodeZipFileInputStream(new FileInputStream(zipFile)).
 					setTrustAllCertificates(true).
 					setProductBaseUrl(baseUrl).
 				build();
 		/* @formatter:on */
-				
+
 		CheckmarxAdapterV1 adapter = new CheckmarxAdapterV1();
 		String data = adapter.start(config);
 		File file = File.createTempFile("checkmarx-adaptertest-result", ".xml");
@@ -68,9 +69,9 @@ public static void main(String[] args) throws Exception {
 		System.out.println("- RESULT:");
 		System.out.println("-----------------------------------------------------------------------------------------------------------------");
 		System.out.println(file.getAbsolutePath());
-		
+
 	}
-	
+
 
 	private static void dump(String systemPropertyName) {
 		System.out.println(systemPropertyName + "=" + System.getProperty(systemPropertyName));

sechub-adapter/src/main/java/com/daimler/sechub/adapter/AdapterContext.java

@@ -17,15 +17,19 @@ public interface AdapterContext<C extends AdapterConfig> extends TraceIdProvider
 
 	/**
 	 * Returns an id used by the product to identify the request/session
-	 * 
+	 *
 	 * @return id
 	 */
 	String getProductContextId();
 
 	RestOperations getRestOperations();
 
+	default AdapterException asAdapterException(String message) {
+		return asAdapterException(message,null);
+	}
+
 	AdapterException asAdapterException(String message, Throwable t);
-	
+
 	/**
 	 * @return configuration, never <code>null</code>
 	 */
@@ -38,16 +42,16 @@ public interface AdapterContext<C extends AdapterConfig> extends TraceIdProvider
 	boolean isTimeOut();
 
 	long getMillisecondsRun();
-	
+
 	JSONAdapterSupport json();
-	
+
 	/**
 	 * Gives back full API URL, containing of base url from configuration and given api path
 	 * @param apiPath
 	 * @return full API URL - e.g. "https://localhost/prefix/$apiPath"
 	 */
 	String getAPIURL(String apiPath);
-	
+
 	/**
 	 * Gives back full API URL, containing of base url from configuration and given api path.
 	 * At the end of the url the map entries are added

sechub-integrationtest/build.gradle

@@ -12,6 +12,9 @@ dependencies {
     compile project(':sechub-shared-kernel') // we want to have access to some shared kernel parts (e.g. trafficlight)
     compile library.springboot_starter_mail // to have easier access to email parts
 
+    // https://mvnrepository.com/artifact/com.adobe.testing/s3mock
+    testCompile group: 'com.adobe.testing', name: 's3mock', version: '2.1.16'
+    testCompile group: 'com.adobe.testing', name: 's3mock-junit4', version: '2.1.16'
 }
 
 def integrationTestServerPortValue = 8443;

sechub-integrationtest/src/test/java/com/daimler/sechub/integrationtest/storage/MinioS3JobStorageTest.java

@@ -0,0 +1,126 @@
+package com.daimler.sechub.integrationtest.storage;
+
+import static org.junit.Assert.*;
+
+import java.io.BufferedWriter;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.UUID;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import com.adobe.testing.s3mock.junit4.S3MockRule;
+import com.amazonaws.services.s3.AmazonS3;
+import com.daimler.sechub.sharedkernel.storage.s3.MinioS3JobStorage;
+
+import io.minio.MinioClient;
+
+public class MinioS3JobStorageTest {
+
+	@ClassRule
+	public static final S3MockRule S3_MOCK_RULE = S3MockRule.builder().silent().build();
+
+	private static AmazonS3 amazonTestClient;
+
+	private static MinioClient minioClient;
+
+	@BeforeClass
+	public static void beforeClass() throws Exception {
+		amazonTestClient = S3_MOCK_RULE.createS3Client();
+		minioClient = new MinioClient("http://localhost:" + S3_MOCK_RULE.getHttpPort());
+	}
+
+	@Test
+	public void a_new_storage_does_not_create_a_new_bucket() throws Exception {
+
+		/* execute */
+		new MinioS3JobStorage(minioClient, "bucket1", "projectName", UUID.randomUUID());
+
+		/* test */
+		assertTrue(amazonTestClient.listBuckets().isEmpty()); // also no bucket2 from former test execution...
+
+	}
+
+	@Test
+	public void after_store_the_inputstream_is_closed() throws Exception {
+
+		/* prepare */
+		MinioS3JobStorage storage = new MinioS3JobStorage(minioClient, "bucket2", "projectName", UUID.randomUUID());
+
+		Path tmpFile = Files.createTempFile("storage_test", ".txt");
+
+		/* execute */
+		InputStream inputStream = new FileInputStream(tmpFile.toFile());
+		InputStream inputStreamSpy = Mockito.spy(inputStream);
+		storage.store("testB", inputStreamSpy);
+
+		/* test */
+		Mockito.verify(inputStreamSpy).close(); // stream must be closed
+
+	}
+
+	@Test
+	public void store_stores_textfile_correct_and_can_be_fetched() throws Exception {
+
+		/* prepare */
+		UUID jobjUUID = UUID.randomUUID();
+		String testContent="line1\nline2";
+		MinioS3JobStorage storage = new MinioS3JobStorage(minioClient, "bucket2", "projectName", jobjUUID);
+
+		Path tmpFile = Files.createTempFile("storage_test", ".txt");
+		BufferedWriter bw = Files.newBufferedWriter(tmpFile);
+
+		bw.write(testContent);
+		bw.close();
+
+		/* execute */
+		storage.store("testA", new FileInputStream(tmpFile.toFile()));
+
+		/* test */
+		String objectName = "jobstorage/projectName/"+jobjUUID+"/testA";
+		assertTrue(amazonTestClient.doesObjectExist("bucket2", objectName)); // test location is as expected
+
+		MinioS3JobStorage storage2 = new MinioS3JobStorage(minioClient, "bucket2", "projectName", jobjUUID);
+		InputStream loadedStream = storage2.fetch("testA");
+		StringWriter writer = new StringWriter();
+		IOUtils.copy(loadedStream, writer,Charset.defaultCharset());
+
+		String loaded = writer.toString();
+
+		assertEquals(testContent,loaded); //  test content can be fetched
+
+	}
+
+	@Test
+	public void stored_object_is_deleted_by_deleteall() throws Exception {
+		/* prepare */
+		UUID jobUUID = UUID.randomUUID();
+		MinioS3JobStorage storage = new MinioS3JobStorage(minioClient, "bucket2", "projectName", jobUUID);
+
+		Path tmpFile = Files.createTempFile("storage_test", ".txt");
+
+		/* execute */
+		InputStream inputStream = new FileInputStream(tmpFile.toFile());
+		InputStream inputStreamSpy = Mockito.spy(inputStream);
+		storage.store("testC", inputStreamSpy);
+
+		String objectName = "jobstorage/projectName/"+jobUUID+"/testC";
+		assertTrue(amazonTestClient.doesObjectExist("bucket2", objectName));
+
+		/* execute */
+		storage.deleteAll();
+
+		/* test */
+		assertFalse(amazonTestClient.doesObjectExist("bucket2", objectName));
+
+	}
+
+}