Merge pull request #121 from dcanar9/master

src and pom update

.github/workflows/maven-pr-checks.yml

@@ -10,9 +10,9 @@ jobs:
 
     steps:
       - uses: actions/checkout@v1
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 1.8
+          java-version: 11
       - name: Build with Maven
         run: mvn test --file pom.xml

.travis.yml

@@ -1,6 +1,6 @@
 language: java
 jdk:
-  - openjdk8
+  - openjdk11
 services:
   - docker
 

src/main/java/com/capitalone/dashboard/Application.java

@@ -7,7 +7,7 @@
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.builder.SpringApplicationBuilder;
-import org.springframework.boot.web.support.SpringBootServletInitializer;
+import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
 
 /**
  * Application configuration and bootstrap

src/main/java/com/capitalone/dashboard/collector/DefaultSonar6Client.java

@@ -95,7 +95,7 @@ public void setServerCredentials(String username, String password, String token)
 
         if (StringUtils.isNotBlank(token)
                 && StringUtils.isNotBlank(username)) {
-            LOG.error("Only one mode of authentication is needed. Either token or username/password. " +
+            LOG.warn("Only one mode of authentication is needed. Either token or username/password. " +
                     "Both modes were detected. Using username/password");
         }
     }

src/main/java/com/capitalone/dashboard/collector/DefaultSonarClient.java

@@ -53,7 +53,7 @@ public class DefaultSonarClient implements SonarClient {
     protected static final String DATE = "date";
 
     protected final RestClient restClient;
-    protected RestUserInfo userInfo;
+    protected RestUserInfo userInfo = new RestUserInfo("","");
 
     @Autowired
     public DefaultSonarClient(RestClient restClient, SonarSettings settings) {
@@ -69,7 +69,7 @@ public DefaultSonarClient(RestClient restClient, SonarSettings settings) {
     @Override
     public void setServerCredentials(String username, String password, String token) {
         // use token when given
-        if (StringUtils.isNotBlank(token)) {
+        if (StringUtils.isNotEmpty(token)) {
             this.userInfo.setToken(token);
             this.userInfo.setUserId(null);
             this.userInfo.setPassCode(null);
@@ -83,7 +83,7 @@ public void setServerCredentials(String username, String password, String token)
         if (StringUtils.isNotBlank(token)
                 && StringUtils.isNotBlank(username)
                 && StringUtils.isNotBlank(password)) {
-            LOG.error("Only one mode of authentication is needed. Either token or username/password. " +
+            LOG.warn("Only one mode of authentication is needed. Either token or username/password. " +
                     "Both modes were detected. Using username/password");
         }
     }

src/main/java/com/capitalone/dashboard/collector/SonarClientSelector.java

@@ -43,7 +43,7 @@ public SonarClientSelector(
     }
 
     public Double getSonarVersion(String instanceUrl){
-    	Double version = 5.0;
+    	Double version = 7.9;
     	try {
     	    ResponseEntity<String> versionResponse = rest.exchange(URI.create(instanceUrl + URL_VERSION_RESOURCE), HttpMethod.GET, new HttpEntity<>(new HttpHeaders()), String.class);
     	    if(!versionResponse.getBody().isEmpty()) {
@@ -54,13 +54,13 @@ public Double getSonarVersion(String instanceUrl){
     	        }
     	    }    
     	} catch (RestClientException e) {
-    		LOG.info("Rest exception occured at fetching sonar version");
+    		LOG.info("Rest exception occurred at fetching sonar version - " + e.getMessage());
     	}
-
         return version;
     }
 
     public SonarClient getSonarClient(Double version) {
+        LOG.info(String.format("getSonarClient version=%s", version));
         if(version != null && version == 5.6){
           return sonar56Client;
         }

src/main/java/com/capitalone/dashboard/collector/SonarCollectorTask.java

@@ -117,6 +117,7 @@ public String getCron() {
     @Override
     public void collect(SonarCollector collector) {
         long start = System.currentTimeMillis();
+        LOG.info(String.format("SonarCollectorTask:collect() start=%d", start));
         int totalProjectCount = 0;
         Set<ObjectId> udId = new HashSet<>();
         udId.add(collector.getId());
@@ -136,16 +137,20 @@ public void collect(SonarCollector collector) {
                 String username = getFromListSafely(sonarSettings.getUsernames(), i);
                 String password = getFromListSafely(sonarSettings.getPasswords(), i);
                 String token = getFromListSafely(sonarSettings.getTokens(), i);
+                LOG.info(String.format("SonarCollectorTask:collect() token valid=%s", StringUtils.isNotBlank(token)));
                 sonarClient.setServerCredentials(username, password, token);
 
                 List<SonarProject> projects = sonarClient.getProjects(instanceUrl);
                 latestProjects.addAll(projects);
+                LOG.info(String.format("SonarCollectorTask:collect() latestProjects size=%d ", latestProjects.size()));
 
                 addNewProjects(projects, existingProjects, collector);
+                LOG.info(String.format("SonarCollectorTask:collect() addNewProjects() executed"));
 
                 List<SonarProject> enabledProjects = enabledProjects(collector, instanceUrl);
                 totalProjectCount = enabledProjects.size();
                 refreshData(enabledProjects, sonarClient);
+                LOG.info(String.format("SonarCollectorTask:collect() refreshData() executed"));
 
                 // Changelog apis do not exist for sonarqube versions under version 5.0
                 if (version >= 5.0) {
@@ -184,6 +189,7 @@ private String getFromListSafely(List<String> ls, int index){
     private void clean(SonarCollector collector, List<SonarProject> existingProjects) {
         // extract unique collector item IDs from components
         // (in this context collector_items are sonar projects)
+
         Set<ObjectId> uniqueIDs = StreamSupport.stream(dbComponentRepository.findAll().spliterator(),false)
                 .filter( comp -> comp.getCollectorItems() != null && !comp.getCollectorItems().isEmpty())
                 .map(comp -> comp.getCollectorItems().get(CollectorType.CodeQuality))
@@ -219,7 +225,7 @@ private void clean(SonarCollector collector, List<SonarProject> existingProjects
             }
         }
         if (!CollectionUtils.isEmpty(stateChangeJobList)) {
-            sonarProjectRepository.save(stateChangeJobList);
+            sonarProjectRepository.saveAll(stateChangeJobList);
         }
     }
 
@@ -249,7 +255,7 @@ private void deleteUnwantedJobs(List<SonarProject> latestProjects, List<SonarPro
                             c.getCollectorItems().remove(CollectorType.CodeQuality);
                         }
                     }
-                    dbComponentRepository.save(comps);
+                    dbComponentRepository.saveAll(comps);
 
                     // other collectors also delete the widget but not here
                     // should not remove the code analysis widget
@@ -260,7 +266,7 @@ private void deleteUnwantedJobs(List<SonarProject> latestProjects, List<SonarPro
             }
         }
         if (!CollectionUtils.isEmpty(deleteJobList)) {
-            sonarProjectRepository.delete(deleteJobList);
+            sonarProjectRepository.deleteAll(deleteJobList);
         }
     }
 
@@ -306,6 +312,7 @@ private void refreshData(List<SonarProject> sonarProjects, SonarClient sonarClie
     private void fetchQualityProfileConfigChanges(SonarCollector collector,String instanceUrl,SonarClient sonarClient) throws org.json.simple.parser.ParseException{
         JSONArray qualityProfiles = sonarClient.getQualityProfiles(instanceUrl);
         JSONArray sonarProfileConfigurationChanges = new JSONArray();
+        if (Objects.isNull(qualityProfiles)) return;
 
         for (Object qualityProfile : qualityProfiles ) {
             JSONObject qualityProfileJson = (JSONObject) qualityProfile;
@@ -344,7 +351,7 @@ private void addNewConfigurationChanges(SonarCollector collector,JSONArray sonar
                 profileConfigChanges.add(profileConfigChange);
             }
         }
-        sonarProfileRepostory.save(profileConfigChanges);
+        sonarProfileRepostory.saveAll(profileConfigChanges);
     }
 
     private Boolean isNewConfig(ObjectId collectorId,String authorLogin,ConfigHistOperationType operation,long timestamp) {
@@ -399,10 +406,10 @@ private void addNewProjects(List<SonarProject> projects, List<SonarProject> exis
         }
         //save all in one shot
         if (!CollectionUtils.isEmpty(newProjects)) {
-            sonarProjectRepository.save(newProjects);
+            sonarProjectRepository.saveAll(newProjects);
         }
         if (!CollectionUtils.isEmpty(updateProjects)) {
-            sonarProjectRepository.save(updateProjects);
+            sonarProjectRepository.saveAll(updateProjects);
         }
         LOG.info(String.format("addNewProjects projectsInSonar=%d existingProjects=%d new=%d updated=%d timeUsed=%d",
                 projects.size(), existingProjects.size(), newCount, updatedCount, System.currentTimeMillis()-start));

src/main/java/com/capitalone/dashboard/collector/SonarSettings.java

@@ -17,8 +17,8 @@ public class SonarSettings {
     private List<String> usernames = new ArrayList<>();
     private List<String> passwords = new ArrayList<>();
     private List<String> servers = new ArrayList<>();
-    private List<String> niceNames;
-    private List<String> tokens;
+    private List<String> niceNames = new ArrayList<>();
+    private List<String> tokens = new ArrayList<>();
     private String metrics63andAbove; // 6.3 is the sonar version
     private String metricsBefore63;
 

src/main/java/com/capitalone/dashboard/repository/SonarProfileRepostory.java

@@ -9,7 +9,7 @@
 
 public interface SonarProfileRepostory extends CollItemConfigHistoryRepository {
 
-	@Query(value="{ 'collectorItemId' : ?0, userID : ?1, operation: ?2, timestamp : ?3}")
+	@Query(value="{ 'collectorItemId' : ?0, 'userID' : ?1, operation: ?2, 'timestamp' : ?3}")
 	List<CollectorItemConfigHistory> findProfileConfigChanges(ObjectId collectorId,String authorLogin,ConfigHistOperationType operation,long timestamp);
 
 }

src/main/java/com/capitalone/dashboard/repository/SonarProjectRepository.java

@@ -8,12 +8,12 @@
 
 public interface SonarProjectRepository extends BaseCollectorItemRepository<SonarProject> {
 
-    @Query(value="{ 'collectorId' : ?0, options.instanceUrl : ?1, options.projectId : ?2}")
+    @Query(value="{ 'collectorId' : ?0, 'options.instanceUrl' : ?1, 'options.projectId' : ?2}")
     SonarProject findSonarProject(ObjectId collectorId, String instanceUrl, String projectId);
 
-    @Query(value="{ 'collectorId' : ?0, options.instanceUrl : ?1, options.projectName : ?2}")
+    @Query(value="{ 'collectorId' : ?0, 'options.instanceUrl' : ?1, 'options.projectName' : ?2}")
     List<SonarProject> findSonarProjectsByProjectName(ObjectId collectorId, String instanceUrl, String projectName);
 
-    @Query(value="{ 'collectorId' : ?0, options.instanceUrl : ?1, enabled: true}")
+    @Query(value="{ 'collectorId' : ?0, 'options.instanceUrl' : ?1, 'enabled' : true}")
     List<SonarProject> findEnabledProjects(ObjectId collectorId, String instanceUrl);
 }

src/main/resources/logback.xml

@@ -9,7 +9,7 @@
  <appender name="ROLLING" class="ch.qos.logback.core.rolling.RollingFileAppender">
     <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
       <!-- rollover daily -->
-      <fileNamePattern>logs/codequalitycollector-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+      <fileNamePattern>logs/sonar-collector-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
       <timeBasedFileNamingAndTriggeringPolicy
             class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
         <!-- or whenever the file size reaches 100MB -->

src/main/resources/sonar-collector-dev.properties

@@ -0,0 +1,19 @@
+dbname=dashboarddb
+dbusername=ENC(emn/9tgEsnNEqC34235xD4zdMfrdSr+s)
+dbpassword=ENC(qGFNjj138t1sq+RXeMPrFanPbPmUpY+n)
+dbhostport=dev-hygieia-docdb.aws-ito-dqa.cb4good.com:27017
+dbreplicaset=true
+dbssl=true
+sslInvalidHostNameAllowed=true
+dbconnecttimeout=300000
+dbsockettimeout=900000
+
+server.servlet.context-path=/sonar
+#every 2 minutes
+sonar.cron=0 0 2 * * *
+
+sonar.servers[0]=https://sonar.cloud.capitalone.com
+sonar.versions[0]=7.9
+sonar.metrics[0]=ncloc,violations,new_vulnerabilities,critical_violations,major_violations,blocker_violations,tests,test_success_density,test_errors,test_failures,coverage,line_coverage,sqale_index,alert_status,quality_gate_details
+sonar.niceNames[0]=sonarcloud
+sonar.tokens[0]=ENC(G4PdnCA7pgxtvtQhE7SOjkbq4Qjt1DrH5lydDB64AL+VtuitXIKbvBlLpLOkqnWvMN1Gv5slOo0=)

src/main/resources/sonar-collector-prode.properties

@@ -0,0 +1,22 @@
+dbname=dashboarddb
+dbusername=ENC(emn/9tgEsnNEqC34235xD4zdMfrdSr+s)
+dbpassword=ENC(qGFNjj138t1sq+RXeMPrFanPbPmUpY+n)
+dbreplicaset=true
+dbhostport=hygieia-prod-docdb.cloud.capitalone.com:27017
+dbconnecttimeout=300000
+dbsockettimeout=900000
+dbssl=true
+sslInvalidHostNameAllowed=true
+
+#every 2 minutes
+sonar.cron=0 */2 * * * *
+#Run on Random day
+#sonar.cron=0 0 0 15 10 ?
+
+server.servlet.context-path=/sonar
+
+sonar.servers[0]=https://sonar.cloud.capitalone.com
+sonar.versions[0]=7.9
+sonar.metrics[0]=ncloc,violations,new_vulnerabilities,critical_violations,major_violations,blocker_violations,tests,test_success_density,test_errors,test_failures,coverage,line_coverage,sqale_index,alert_status,quality_gate_details
+sonar.niceNames[0]=sonarcloud
+sonar.tokens[0]=ENC(G4PdnCA7pgxtvtQhE7SOjkbq4Qjt1DrH5lydDB64AL+VtuitXIKbvBlLpLOkqnWvMN1Gv5slOo0=)

src/main/resources/sonar-collector-prodw.properties

@@ -0,0 +1,22 @@
+dbname=dashboarddb
+dbusername=ENC(emn/9tgEsnNEqC34235xD4zdMfrdSr+s)
+dbpassword=ENC(qGFNjj138t1sq+RXeMPrFanPbPmUpY+n)
+dbconnecttimeout=300000
+dbsockettimeout=900000
+dbreplicaset=true
+dbhostport=hygieia-prod-docdb.cloud.capitalone.com:27017
+dbssl=true
+sslInvalidHostNameAllowed=true
+
+#every 2 minutes
+#sonar.cron=0 */2 * * * *
+#Run on Random day
+sonar.cron=0 0 0 15 10 ?
+
+server.servlet.context-path=/sonar
+
+sonar.servers[0]=https://sonar.cloud.capitalone.com
+sonar.versions[0]=7.9
+sonar.metrics[0]=ncloc,violations,new_vulnerabilities,critical_violations,major_violations,blocker_violations,tests,test_success_density,test_errors,test_failures,coverage,line_coverage,sqale_index,alert_status,quality_gate_details
+sonar.niceNames[0]=sonarcloud
+sonar.tokens[0]=ENC(G4PdnCA7pgxtvtQhE7SOjkbq4Qjt1DrH5lydDB64AL+VtuitXIKbvBlLpLOkqnWvMN1Gv5slOo0=)

src/main/resources/sonar-collector-qae.properties

@@ -0,0 +1,22 @@
+dbname=dashboarddb
+dbusername=ENC(emn/9tgEsnNEqC34235xD4zdMfrdSr+s)
+dbpassword=ENC(qGFNjj138t1sq+RXeMPrFanPbPmUpY+n)
+dbhostport=hygieia-qa-docdb.clouddqt.capitalone.com:27017
+dbreplicaset=true
+dbssl=true
+sslInvalidHostNameAllowed=true
+dbconnecttimeout=300000
+dbsockettimeout=900000
+
+#every 2 minutes
+sonar.cron=0 */2 * * * *
+#Once a day
+#sonar.cron=0 0 2 * * *
+
+server.servlet.context-path=/sonar
+
+sonar.servers[0]=https://sonar.cloud.capitalone.com
+sonar.versions[0]=7.9
+sonar.metrics[0]=ncloc,violations,new_vulnerabilities,critical_violations,major_violations,blocker_violations,tests,test_success_density,test_errors,test_failures,coverage,line_coverage,sqale_index,alert_status,quality_gate_details
+sonar.niceNames[0]=sonarcloud
+sonar.tokens[0]=ENC(G4PdnCA7pgxtvtQhE7SOjkbq4Qjt1DrH5lydDB64AL+VtuitXIKbvBlLpLOkqnWvMN1Gv5slOo0=)

src/main/resources/sonar-collector-qaw.properties

@@ -0,0 +1,25 @@
+dbname=dashboarddb
+dbusername=ENC(emn/9tgEsnNEqC34235xD4zdMfrdSr+s)
+dbpassword=ENC(qGFNjj138t1sq+RXeMPrFanPbPmUpY+n)
+dbreplicaset=true
+dbssl=true
+sslInvalidHostNameAllowed=true
+dbconnecttimeout=300000
+dbsockettimeout=900000
+
+# QA:
+dbhostport=hygieia-qa-docdb.clouddqt.capitalone.com:27017
+
+#every 2 minutes
+#sonar.cron=0 */2 * * * *
+#Once a day
+sonar.cron=0 0 2 * * *
+
+server.servlet.context-path=/sonar
+
+
+sonar.servers[0]=https://sonar.cloud.capitalone.com
+sonar.versions[0]=7.9
+sonar.metrics[0]=ncloc,violations,new_vulnerabilities,critical_violations,major_violations,blocker_violations,tests,test_success_density,test_errors,test_failures,coverage,line_coverage,sqale_index,alert_status,quality_gate_details
+sonar.niceNames[0]=sonarcloud
+sonar.tokens[0]=ENC(G4PdnCA7pgxtvtQhE7SOjkbq4Qjt1DrH5lydDB64AL+VtuitXIKbvBlLpLOkqnWvMN1Gv5slOo0=)