diff --git a/baseline.conf b/baseline.conf index 9437ee66b..c3777dca9 100755 --- a/baseline.conf +++ b/baseline.conf @@ -25,3 +25,15 @@ readonly JAVA_CPD_THRESHOLD=32 # Threshold for CPD when run for Scala readonly SCALA_CPD_THRESHOLD=0 + +# ******************* Baseline and Threshold numbers for Checkstyle ********************* +# Threshold for Checkstyle errors post which build would fail +readonly CHECKSTYLE_ERROR_THRESHOLD=1389 +# Baseline for Checkstyle warnings(build wont fail for warnings) +readonly CHECKSTYLE_WARNING_BASELINE=730 + + # ******************* Baseline and Threshold numbers for Scalastyle ********************* +# Threshold for Scalastyle errors post which build would fail +readonly SCALASTYLE_ERROR_THRESHOLD=274 +# Baseline for Scalastyle warnings(build wont fail for warnings) +readonly SCALASTYLE_WARNING_BASELINE=37 diff --git a/checkstyle.sbt b/checkstyle.sbt new file mode 100644 index 000000000..8058dcae4 --- /dev/null +++ b/checkstyle.sbt @@ -0,0 +1,27 @@ +// +// Copyright 2016 LinkedIn Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +// + +// +// sbt-checkstyle-plugin specific configurations go in this file +// + +// Path and name of checkstyle configuration file +checkstyleConfigLocation := CheckstyleConfigLocation.File("project/checkstyle-config.xml") + +// Generate HTML report in addition to default XML report by applying XSLT transformations +checkstyleXsltTransformations := { + Some(Set(CheckstyleXSLTSettings(baseDirectory(_ / "project/checkstyle-noframes-severity-sorted-modified.xsl").value, target(_ / "checkstyle-report.html").value))) +} diff --git a/common.sh b/common.sh index b6e7215b7..1a084404d 100755 --- a/common.sh +++ b/common.sh @@ -39,6 +39,14 @@ readonly CPD_REPORT_BASE_PATH=$REPORTS_BASE_PATH"cpd/" # Default path for CPD report readonly CPD_REPORT_PATH=$CPD_REPORT_BASE_PATH"cpd.xml" +# ******************* Constants for Checkstyle ********************* +# Path for Checkstyle report +readonly CHECKSTYLE_REPORT_PATH="target/checkstyle-report.xml" + +# ******************* Constants for Scalastyle ********************* +# Path for Scalastyle report +readonly SCALASTYLE_REPORT_PATH="target/scalastyle-result.xml" + # ************************ Other constants ************************** # Color coded prefixes for ERROR, WARNING, INFO and SUCCESS messages readonly ERROR_COLOR_PREFIX="[\033[0;31mERROR\033[0m]" @@ -78,7 +86,6 @@ function getCPDReportName() { ########################################################## function checkIfCPDFailed() { duplicates=`grep "//g'| awk -F" " '{printf("\t%s: %s\n", $2,$1);}'` + resultCount=`echo "${result}" | wc -l | xargs` + echo -e "$WARNING_COLOR_PREFIX $4 $1 "$3"s detected. Top $resultCount issue types with their counts are as under:" + echo "${result}" +} + +########################################################## +# This function is called when build has to be failed +# because developer has fixed CPD or Checkstyle or +# Scalastyle issues but not updated the corresponding +# threshold variable in this script. +# +# Arguments: +# arg1: Report description +# arg2: Issue count +# arg3: Issue description +# arg4: Name of variable to be updated +# Returns: +# None +########################################################## +function handleSettingThresholdVariable() { + msg="$1 Report has $2 $3" + color=$ERROR_COLOR_PREFIX + failTheBuildMsg=", hence failing the build.\n\tPlease modify" + thresholdOrBaseline="threshold" + if [ $3 = "warnings" ]; then + color=$WARNING_COLOR_PREFIX + failTheBuildMsg=".\n\tYou can modify" + thresholdOrBaseline="baseline" + fi + echo -e "$color $msg and you have fixed some of them as part of this change which is great! But you forgot to update the $thresholdOrBaseline$failTheBuildMsg"\ + "$4 variable to $2 in baseline.conf to ensure that the new $thresholdOrBaseline takes effect for subsequent builds." +} + +################################################################## +# Check if there are warnings in the report for the tool whose +# report is being processed. If warnings exist, dump top 10 issues +# grouped by issue type. Return an integer indicating whether +# number of warnings are 0, above or below baseline. +# Also print messages if baseline variable has to be updated in +# baseline.conf. Number of warnings are also returned by setting +# an argument passed to the function. +# +# Arguments: +# arg1: Indicates the tool whose report will be processed +# (Checkstyle or Scalastyle) +# arg2: Report location for the tool whose report is to be +# processed +# arg3: Warnings baseline for the tool whose report will be +# processed +# arg4: Name of the warning baseline constant for the tool +# arg5: Argument which will be set equal to number of +# warnings found in the report. +# Returns: +# 0: Success +# 1: Warnings above baseline +# 2: Warnings fixed but baseline variable not updated +################################################################ +function checkStyleToolWarnings() { + # Local variable which references to arg5. + local __numWarnings=$5 + # Check if there are any warnings in the Checkstyle or Scalastyle report + local styleWarnings=`grep 'severity="warning"' $2 | wc -l | xargs` + # Effectively sets number of warnings to arg5 + eval $__numWarnings="'$styleWarnings'" + if [ $styleWarnings -gt 0 ]; then + dumpTop10StyleIssueTypes $1 $2 "warning" $styleWarnings + # Return number of warnings only if over baseline + if [ $styleWarnings -gt $3 ]; then + return 1; + elif [ $styleWarnings -lt $3 ]; then + handleSettingThresholdVariable $1 $styleWarnings "warnings" $4 + return 2; + fi + else + echo -e "$SUCCESS_COLOR_PREFIX $1 Report has no warnings..." + fi + return 0; +} + +################################################################## +# Process checkstyle/scalastyle report after the tool has been run +# This method will find how many errors exist. +# If errors exist and they are above threshold, fail the build. +# Fail the build even if errors have been fixed but threshold +# variable has not been updated in baseline.conf +# Print top 10 issues at error severity grouped by issue +# type if errors are equal to threshold (for informational +# purposes) +# +# Arguments: +# arg1: Indicates the tool whose report will be processed +# (Checkstyle or Scalastyle) +# arg2: Report location for the tool whose report is to be +# processed +# arg3: Error threshold, above which build would fail, for +# the tool whose report will be processed +# arg4: Name of the error threshold constant for the tool +# +# Returns: +# 0: Success +# 1: Failure due to errors above threshold +# 2: Failure due to errors fixed but threshold variable not +# updated. +################################################################## +function checkStyleToolErrors() { + # Check if there are any errors in the Checkstyle or Scalastyle report and fail the build, if above threshold + styleErrors=`grep 'severity="error"' $2 | wc -l | xargs` + if [ $styleErrors -gt $3 ]; then + echo -e "$ERROR_COLOR_PREFIX Build failed as the code change has introduced $1 ERRORS. $styleErrors found (threshold: $3)" + return 1; + fi + + # Print top 10 checkstyle/scalastyle error categories if number of errors within threshold + if [ $styleErrors -gt 0 ]; then + if [ $styleErrors -gt $3 ]; then + echo -e "$ERROR_COLOR_PREFIX Build failed as this code change has introduced $1 ERRORS. $styleErrors found (threshold: $3)" + return 1; + elif [ $styleErrors -eq $3 ]; then + dumpTop10StyleIssueTypes $1 $2 "error" $styleErrors + echo -e "$WARNING_COLOR_PREFIX Note: The code change may not have introduced $1 errors as count is within threshold. Not failing"\ + "the build." + return 0; + else + handleSettingThresholdVariable $1 $styleErrors "errors" $4 + return 2; + fi + else + if [ $3 -gt 0 ]; then + handleSettingThresholdVariable $1 $styleErrors "errors" $4 + return 2; + else + echo "" + echo -e "$SUCCESS_COLOR_PREFIX $1 Report has no errors..." + return 0; + fi + fi +} + ########################################################## # Parse the findbugs report and if any bugs are found, # fail the build. @@ -189,3 +393,63 @@ function changeCPDLanguageSetting() { sed "s/$1/$2/g" cpd.sbt > cpd.sbt.bak mv cpd.sbt.bak cpd.sbt } + +############################################################ +# Generate a final scalastyle report by removing duplicate +# errors and sorting the results within a file by line +# number. +# +# Arguments: +# arg1: Report location for the tool being run +# Returns: +# None +############################################################ +function preProcessScalastyleReport() { + # Flag to indicate whether we are processing file tag i.e. we have encountered file begin tag but not the file end tag + filetag=0 + currentLineNum=0 + currentErrorTag="" + count=0 + while IFS='' read -r line || [[ -n "$line" ]]; do + if [[ $line == *"> $1.bak + filetag=1 + elif [[ $line == *""* ]]; then + # On end file tag, sort and find unique lines in tmpResult file(contains errors for a file). + # This is done to avoid duplicates + sortedResults=`cat tmpResult | sort -n -k 1 | uniq` + # Remove the line number prepended in tmpResult used for sorting errors by line number + finalResults=`echo "${sortedResults}" | sed 's/^[0-9]* / /g'` + # Copy errors for a file in sorted order and after removing duplicates + echo "${finalResults}" >> $1.bak + rm -rf tmpResult + # Copy file end tag as well + echo -e $line >> $1.bak + filetag=0 + elif [ $filetag -eq 1 ]; then + # We are processing errors inside a file + # Fetch line number from the corresponding attribute and prepend the line with line number + # This is done to ensure sorting of errors within a file by line number and removing duplicates, + # if any. Store this result in a tmpResult file + lineAttribute=`echo "$line" | sed -n 's/.* line="\([0-9.]*\).*/\1/p'` + if [[ $line == *"" ]]; then + echo -e $lineAttribute" "$line >> tmpResult + else + currentLineNum=$lineAttribute + currentErrorTag=$line + fi + elif [[ $line == *"/>" ]]; then + # Processing error tag. Encountered end of tag. + lineWithoutSpaces=`echo $line | sed 's/^[ ]*//g'` + echo -e "$currentLineNum $currentErrorTag $lineWithoutSpaces" >> tmpResult + fi + else + # Not inside file tag. Copy line as is. + echo -e $line >> $1.bak + fi + done< $1 + # Move the .bak file to the report file + mv $1.bak $1 +} diff --git a/compile.sh b/compile.sh index f826d6d61..3c09cbedb 100755 --- a/compile.sh +++ b/compile.sh @@ -16,6 +16,24 @@ # the License. # +######################################################## +# +# Global constants +# +######################################################## + +# ******************** Constants for Checkstyle ********************* +# Path for Checkstyle HTML report +readonly CHECKSTYLE_HTML_REPORT_PATH="target/checkstyle-report.html" + +# ******************** Constants for Scalastyle ********************* +# Path for Scalastyle HTML report +readonly SCALASTYLE_HTML_REPORT_PATH="target/scalastyle-result.html" +# Path for Scalastyle HTML report generation python script +readonly SCALASTYLE_XSL_FILE="project/checkstyle-noframes-severity-sorted-modified.xsl" +# Path for Scalastyle HTML report generation python script +readonly SCALASTYLE_HTML_REPORT_GEN_SCRIPT="project/scalastyle_xml_to_html.py" + function print_usage() { echo "" echo "Usage: ./compile.sh [config_file_path] [additional_options]" @@ -25,6 +43,7 @@ function print_usage() { echo -e "\tcoverage: Runs Jacoco code coverage and fails the build as per configured threshold" echo -e "\tfindbugs: Runs Findbugs for Java code" echo -e "\tcpd: Runs Copy Paste Detector(CPD) for Java and Scala code" + echo -e "\tstylechecks: Runs Checkstyle for Java and Scalastyle for Scala code" } function play_command() { @@ -46,10 +65,10 @@ function require_programs() { done if [ ! -z "$missing_programs" ]; then - echo "[ERROR] The following programs are required and are missing: $missing_programs" + echo -e "$ERROR_COLOR_PREFIX The following programs are required and are missing: $missing_programs" exit 1 else - echo "[SUCCESS] Program requirement is fulfilled!" + echo -e "$SUCCESS_COLOR_PREFIX Program requirement is fulfilled!" fi } @@ -77,9 +96,7 @@ function processCPDReportByLanguage() { result=$? if [ $result -gt 0 ]; then if [ $result -eq 2 ]; then - echo "" - echo -e "$WARNING_COLOR_PREFIX Note: Make sure your local repo is up to date with the branch you want to merge to, otherwise threshold/baseline "\ - "values to be updated in baseline.conf\n\tmight be different and that can lead to CI failure..." + echo -e $(noteForUpdatingRepo)" and that can lead to CI failure..." fi echo "" exit 1; @@ -98,19 +115,21 @@ function processCPDReportByLanguage() { # None ########################################################## function runCPD() { - echo "Running CPD for Java" + echo -e "$INFO_COLOR_PREFIX Running CPD for Java" play_command $1 cpd if [ $? -ne 0 ]; then + echo -e "$ERROR_COLOR_PREFIX CPD for Java failed" exit 1; fi processCPDReportByLanguage "Java" $JAVA_CPD_THRESHOLD "JAVA_CPD_THRESHOLD" - echo "Running CPD for Scala" + echo -e "$INFO_COLOR_PREFIX Running CPD for Scala" changeCPDLanguageSetting "Language.Java" "Language.Scala" play_command $OPTS cpd if [ $? -ne 0 ]; then # Reset language back to Java changeCPDLanguageSetting "Language.Scala" "Language.Java" + echo -e "$ERROR_COLOR_PREFIX CPD for Scala failed" exit 1; fi processCPDReportByLanguage "Scala" $SCALA_CPD_THRESHOLD "SCALA_CPD_THRESHOLD" @@ -118,6 +137,132 @@ function runCPD() { changeCPDLanguageSetting "Language.Scala" "Language.Java" } +########################################################## +# Note for updating repo before updating baseline.conf +# +# Arguments: +# None +# Returns: +# Note for updating repo +########################################################## +function noteForUpdatingRepo { + echo -e "$WARNING_COLOR_PREFIX Note: Make sure your local repo is up to date with the branch you want to merge to, otherwise threshold/baseline "\ + "values to be updated in baseline.conf\n\tmight be different" +} + +############################################################ +# Process style report based on tool for which report is +# being processed. Verifies report existence, checks for +# warning baseline, checks for error threshold breach and +# if required fail the build or print appropriate message. +# +# Arguments: +# arg1: Indicates the tool whose report will be processed +# (Checkstyle or Scalastyle) +# arg2: Report location for the tool whose report is to be +# processed +# arg3: Error threshold, above which build would fail, for +# the tool whose report will be processed +# arg4: Warnings baseline for the tool whose report will be +# processed +# arg5: Name of the error threshold constant for the tool +# and language +# arg6: Name of the warning baseline constant for the tool +# and language +# Returns: +# None +############################################################ +function processStyleReport() { + verifyStyleReportExistence $1 $2 + + # Check warnings in Checkstyle/Scalastyle report + checkStyleToolWarnings $1 $2 $4 $6 numWarnings + result=$? + if [ $result -gt 0 ]; then + if [ $result -eq 1 ]; then + msgToResetStyleReportWarning $1 $4 $6 $numWarnings + fi + echo -e $(noteForUpdatingRepo)"..." + fi + echo "" + + # Check errors in Checkstyle/Scalastyle report + checkStyleToolErrors $1 $2 $3 $5 + result=$? + if [ $result -gt 0 ]; then + if [ $result -eq 2 ]; then + echo -e $(noteForUpdatingRepo)" and that can lead to CI failure..." + fi + echo "" + exit 1; + fi + echo "" +} + +############################################################ +# Process both Checkstyle and Scalastyle XML reports. Also +# generates Scalastyle HTML report(Checkstyle HTML report is +# automatically generated by checkstyle4sbt plugin). +# Fail the build if threshold values are breached. +# +# Arguments: +# None +# Returns: +# None +############################################################ +function processCheckstyleAndScalastyleReports() { + echo "" + echo -e "$INFO_COLOR_PREFIX Checking Checkstyle report..." + echo -e "$INFO_COLOR_PREFIX Checkstyle XML report generated at path: $CHECKSTYLE_REPORT_PATH and HTML report generated at path: $CHECKSTYLE_HTML_REPORT_PATH" + processStyleReport "Checkstyle" $CHECKSTYLE_REPORT_PATH $CHECKSTYLE_ERROR_THRESHOLD $CHECKSTYLE_WARNING_BASELINE "CHECKSTYLE_ERROR_THRESHOLD" "CHECKSTYLE_WARNING_BASELINE" + + scalastyleHtmlGenMsg="" + preProcessScalastyleReport $SCALASTYLE_REPORT_PATH + pythonVersion=`python --version 2>&1` + if [ $? -ne 0 ]; then + echo -e "$WARNING_COLOR_PREFIX Cannot generate Scalastyle HTML report as Python is unavailable. Install Python and add it in PATH" + else + # Generate Scalastyle HTML Report + rm -rf $SCALASTYLE_HTML_REPORT_PATH + echo "Using $pythonVersion" + pip install lxml + if [ $? -ne 0 ]; then + echo -e "$WARNING_COLOR_PREFIX Could not install lxml module for Python. Scalastyle HTML report could not be generated" + else + python $SCALASTYLE_HTML_REPORT_GEN_SCRIPT $SCALASTYLE_REPORT_PATH $SCALASTYLE_XSL_FILE $SCALASTYLE_HTML_REPORT_PATH + if [ $? -ne 0 ]; then + echo -e "$WARNING_COLOR_PREFIX Scalastyle HTML report could not be generated" + else + scalastyleHtmlGenMsg=" and HTML report generated at path: $SCALASTYLE_HTML_REPORT_PATH" + fi + fi + fi + echo -e "$INFO_COLOR_PREFIX Checking Scalastyle report..." + echo -e "$INFO_COLOR_PREFIX Scalastyle XML report generated at path: $SCALASTYLE_REPORT_PATH"$scalastyleHtmlGenMsg + processStyleReport "Scalastyle" $SCALASTYLE_REPORT_PATH $SCALASTYLE_ERROR_THRESHOLD $SCALASTYLE_WARNING_BASELINE "SCALASTYLE_ERROR_THRESHOLD" "SCALASTYLE_WARNING_BASELINE" +} + +############################################################# +# Run Checkstyle and Scalastyle and then process the report. +# Fail the build if the command fails or if threshold values +# are breached. +# +# Arguments: +# arg1: Play command OPTS +# Returns: +# None +############################################################# +function runStyleChecks() { + echo -e "$INFO_COLOR_PREFIX Running Checkstyle and Scalastyle" + play_command $1 checkstyle scalastyle + if [ $? -ne 0 ]; then + echo -e "$ERROR_COLOR_PREFIX Either Checkstyle or Scalastyle has failed" + echo "" + exit 1; + fi + processCheckstyleAndScalastyleReports +} + require_programs zip unzip # Default configurations @@ -129,6 +274,7 @@ extra_commands="" # Indicates whether a custom configuration file is passed as first parameter. custom_config="n" run_CPD="n" +run_StyleChecks="n" # Process command line arguments while :; do if [ ! -z $1 ]; then @@ -142,6 +288,13 @@ while :; do cpd) run_CPD="y" ;; + stylechecks) + run_StyleChecks="y" + ;; + help) + print_usage + exit 0; + ;; *) # User may pass the first argument(optional) which is a path to config file if [[ -z $extra_commands && $custom_config = "n" ]]; then @@ -268,6 +421,11 @@ if [ $run_CPD = "y" ]; then runCPD $OPTS fi +# Run Checkstyle and Scalastyle if stylechecks is passed as an argument +if [ $run_StyleChecks = "y" ]; then + runStyleChecks $OPTS +fi + set -v set -x # Echo the value of pwd in the script so that it is clear what is being removed. diff --git a/project/checkstyle-config.xml b/project/checkstyle-config.xml new file mode 100644 index 000000000..1f84b1a5c --- /dev/null +++ b/project/checkstyle-config.xml @@ -0,0 +1,364 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/project/checkstyle-java.header b/project/checkstyle-java.header new file mode 100644 index 000000000..3c8eb723e --- /dev/null +++ b/project/checkstyle-java.header @@ -0,0 +1,15 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ diff --git a/project/checkstyle-noframes-severity-sorted-modified.xsl b/project/checkstyle-noframes-severity-sorted-modified.xsl new file mode 100644 index 000000000..b16a56cea --- /dev/null +++ b/project/checkstyle-noframes-severity-sorted-modified.xsl @@ -0,0 +1,269 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Scalastyle Report + Checkstyle Report + + + + + + + + + + + + + +
+ + + +

Scalastyle Audit

 +

Checkstyle Audit

 + +
Designed for use with + + ScalaStyle. + CheckStyle. + +
+
+ + + +
+ + + +
+ + + +
+ + + + + + +

Files

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameErrorsWarningsInfos
+ + + + + +

File

+ + + + + + + + + + + + + + + + + + + + + + +
SeverityError DescriptionLineColumn
+ Back to top + + + +

Summary

+ + + + + + + + + + + + + + + + + + +
FilesErrorsWarningsInfos
+ + + + + a + b + + + + + + + error + warning + a + + + diff --git a/project/checkstyle-suppressions.xml b/project/checkstyle-suppressions.xml new file mode 100644 index 000000000..e8718cc6b --- /dev/null +++ b/project/checkstyle-suppressions.xml @@ -0,0 +1,24 @@ + + + + + + + diff --git a/project/plugins.sbt b/project/plugins.sbt index 32d4f2f33..a9c495142 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -30,3 +30,9 @@ addSbtPlugin("de.johoop" % "findbugs4sbt" % "1.4.0") // Copy paste detector plugin addSbtPlugin("de.johoop" % "cpd4sbt" % "1.2.0") + +// Checkstyle plugin +addSbtPlugin("com.etsy" % "sbt-checkstyle-plugin" % "3.1.1") + +// Scalastyle plugin +addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0") diff --git a/project/scalastyle-config.xml b/project/scalastyle-config.xml new file mode 100644 index 000000000..043a30f06 --- /dev/null +++ b/project/scalastyle-config.xml @@ -0,0 +1,343 @@ + + + + Scalastyle standard configuration + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + mutable\.SynchronizedBuffer + + + + + + mutable\.SynchronizedMap + + + + + + mutable\.SynchronizedSet + + + + + + mutable\.SynchronizedQueue + + + + + + mutable\.SynchronizedPriorityQueue + + + + + + mutable\.SynchronizedStack + + + + + ^println$ + + + + JavaConversions + Instead of importing implicits in scala.collection.JavaConversions._, import + scala.collection.JavaConverters._ and use .asScala / .asJava methods + + + + throw new \w+Error\( + + + + + + + + + + + + + + + 2 + 2 + 2 + + + + + + + + + + + + + ARROW, EQUALS, ELSE, TRY, CATCH, FINALLY, LARROW, RARROW + + + + + ARROW, EQUALS, COMMA, COLON, IF, ELSE, DO, WHILE, FOR, MATCH, TRY, CATCH, FINALLY, LARROW, RARROW + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2 + + + + + + + + + + + + + + + + PatDefOrDcl,TypeDefOrDcl,FunDefOrDcl,TmplDef + false + javadoc + + + + + + + + + + + + + + + + + diff --git a/project/scalastyle_xml_to_html.py b/project/scalastyle_xml_to_html.py new file mode 100755 index 000000000..da1c6965c --- /dev/null +++ b/project/scalastyle_xml_to_html.py @@ -0,0 +1,70 @@ +# +# Copyright 2016 LinkedIn Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +# +# This script is meant to convert Scalastyle XML report to Scalastyle HTML report using +# XSLT transformations. The XSL file is typically the same as the one used to convert +# Checkstyle XML report to HTML report. +# + +import lxml.etree as ET +import os +import sys +import traceback +import os.path +from os import path + +# Takes 3 arguments: Scalastyle XML report, XSL file name and HTML report name to be generated +if len(sys.argv) < 4: + print 'Too few arguments, please specify arguments as under:\n 1st argument: Scalastyle XML report file name \n 2nd', \ + 'argument: XSL file name\n 3rd argument: Scalastyle HTML report name to be generated...' + sys.exit(1) + +print 'Generating Scalastyle HTML report' + +# Check if input Scalastyle XML report exists +xmlReportFileName = sys.argv[1] +if not path.isfile(xmlReportFileName): + print 'Scalastyle XML report {0} not found. Cannot generate HTML report!'.format(xmlReportFileName) + sys.exit(1) + +# Check if input XSL which will be used to transform XML report exists +xslFileName = sys.argv[2] +if not path.isfile(xslFileName): + print 'XSL file {0} for Scalastyle XML report conversion not found. Cannot generate HTML report!'.format(xslFileName) + sys.exit(1) + +# HTML report name which will be outputted +htmlReportFileName = sys.argv[3] + +htmlReportFD = None +try: + xmlreport_root = ET.parse(xmlReportFileName) + xslt = ET.parse(xslFileName) + transform = ET.XSLT(xslt) + # Pass reporttype to XSL to ensure scalstyle specific changes can be made while outputting HTML report. + htmlreport_root = transform(xmlreport_root, reporttype="'scalastyle'") + htmlstring = ET.tostring(htmlreport_root, pretty_print=True) + htmlReportFD = os.open(htmlReportFileName, os.O_RDWR|os.O_CREAT) + os.write(htmlReportFD, htmlstring) +except: + print 'Issue encountered during Scalastyle HTML report generation...{0} occured.'.format(sys.exc_info()[0]) + desired_trace = traceback.format_exc(sys.exc_info()) + print(desired_trace) + sys.exit(1) +finally: + if htmlReportFD is not None: + os.close(htmlReportFD) diff --git a/scalastyle.sbt b/scalastyle.sbt new file mode 100644 index 000000000..429766342 --- /dev/null +++ b/scalastyle.sbt @@ -0,0 +1,27 @@ +// +// Copyright 2016 LinkedIn Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +// + +// +// scalastyle-sbt-plugin specific configurations go in this file +// + +// Do not fail on scalastyle errors as we want to baseline error numbers till +// we fix all errors. We would fail the CI build if any new errors are introduced +// through a PR. +scalastyleFailOnError := false + +// Scalastyle config file location. +scalastyleConfig := file("project/scalastyle-config.xml") diff --git a/travis.sh b/travis.sh index 47d4d5863..06e826941 100755 --- a/travis.sh +++ b/travis.sh @@ -20,6 +20,14 @@ # Script to be used for building on Travis CI # +######################################################## +# +# Global constants +# +######################################################## +# Tab for use in sed command +readonly TAB=$'\t' + ############################################################ # Get files chnged in this PR using git commands. # @@ -98,7 +106,9 @@ function dumpCPDSummaryForChangedFiles() { ########################################################## # Check if there are duplicates in CPD report above the -# configured threshold for the language. +# configured threshold for the language. Dump CPD summary +# for changed files if checkIfCPDFailed method returns 1 +# i.e. if CPD duplicates are above threshold. # # Arguments: # arg1: Language for which CPD is run (Java or Scala) @@ -147,7 +157,232 @@ function runFindbugs() { checkFindbugsReport } -########################################################## +############################################################ +# Helper function to generate severity attribute found in +# checkstyle/scalastyle XML reports, based on the severity +# passed. Used to match severity in the XML report. +# +# Arguments: +# arg1: Severity of issues (warning or error) +# Returns: +# Severity attribute string in the form: +# severity="arg1" +############################################################ +function getSeverityStringForStyleReport() { + severityWithQuotes='"'$1'"' + echo 'severity='$severityWithQuotes +} + +############################################################ +# For the files changed in the PR and severity passed, get +# a summary of all the checkstyle or scalastyle issues in a +# printable format by extracting details from XML report. +# This function is called when warnings surpass the baseline +# or errors breach the threshold i.e. the PR introduces +# checkstyle or scalastyle errors. +# +# Arguments: +# arg1: List of files changed in the PR +# arg2: Report location for the tool whose report is being +# parsed. +# arg3: Severity of the issues for which we are dumping +# results +# Returns: +# None +############################################################ +function getStyleIssuesForChangedFiles() { + # Get severity attribute for match in style report. + severityString=$(getSeverityStringForStyleReport $3) + styleIssuesForChangedFiles=$( + # Iterate over all the files changed in this PR. + for changedFile in $1; do + # sed requires filename separators to be escaped. + changedFileEscaped=$(echo $changedFile | sed 's/\//\\\//g') + + # Extract/filter out issues based on filename and severity. + results=`sed -n "///g;s/\" /\", /g"` + echo "${finalResults}" | awk '{ + # Iterate over all the Checkstyle/Scalastyle issues for the changed file, filtered by severity + numIssues = split($0, issues, "\n") + for (issueIdx = 1; issueIdx <= numIssues; issueIdx++) { + # Convert special encoding in XML file such as " and &apos + gsub(/"/, "", issues[issueIdx]); + gsub("\\'", "'"'"'", issues[issueIdx]); + gsub("\\"", "\"", issues[issueIdx]); + # Next 4 variables hold the attribute values for line, column, message and source attributes respectively. + line="" + column="" + message="" + source="" + + # Indicates whether message attribute is being processed. + processingMessage = 0; + + # Extract different attributes for each issue by splitting the line by comma + # and for each attribute, extract its value. The attributes we are interested + # in are line, column, source and message. + # 1. Line indicates the line at which checkstyle/scalastyle issue exists + # 2. Column indicates the column at which checkstyle/scalastyle issue exists + # 3. Message is explanation about the issue. + # 4. Source is Checkstyle/Scalastyle check which led to the issue. + numAttributes = split(issues[issueIdx], attributes, ","); + for (attributeIdx = 1; attributeIdx <= numAttributes; attributeIdx++) { + lineIdx = index(attributes[attributeIdx], "line="); + if (lineIdx > 0) { + line = line "" substr(attributes[attributeIdx], lineIdx + 5); + processingMessage = 0; + continue; + } + columnIdx = index(attributes[attributeIdx], "column="); + if (columnIdx > 0) { + column = column "" substr(attributes[attributeIdx], columnIdx + 7); + processingMessage = 0; + continue; + } + sourceIdx = index(attributes[attributeIdx], "source="); + if (sourceIdx > 0) { + source = source "" substr(attributes[attributeIdx], sourceIdx + 7); + processingMessage = 0; + continue; + } + + # Extract message from message attribute. As message can contain commas as well, continue to append to message + # till another attribute is encountered. + messageIdx = index(attributes[attributeIdx], "message="); + if (messageIdx > 0) { + message = message "" substr(attributes[attributeIdx], messageIdx + 8); + processingMessage = 1; + } else if (processingMessage == 1) { + message = message "," attributes[attributeIdx]; + } + } + # Remove dot from the end of the message. + split(message, chars, ""); + len = length(message); + if (chars[len] == ".") { + message="" substr(message, 1, len - 1); + } + + # Extract last section of source string, separated by dot. + numSourceParts = split(source, sourceParts, "."); + # Print style information in the desired format + printf("\t + %s (%s) at line: %s%s\n", sourceParts[numSourceParts], message, line, ((column == "") ? "." : (" and column: " column "."))); + } + }' + fi + fi + done + ) + echo "${styleIssuesForChangedFiles}" +} + +############################################################ +# Capitalizes first character of passed string +# +# Arguments: +# arg1: String whose first character has to be capitalized +# Returns: +# String with first character captialized +############################################################ +function capitalizeFirstCharOfStr() { + echo $1 | awk '{ + split($0, chars, "") + for (i = 1; i <= length($0); i++) { + if (i == 1) { + printf("%s", toupper(chars[i])); + } else { + printf("%s", chars[i]); + } + } + }' +} + +################################################################### +# Process checkstyle/scalastyle report after the tool has been run +# This method will find out whether report has been generated and +# how many errors/warnings exist. +# If errors exist and they are above threshold, fail the build. +# Print summary of checkstyle/scalastyle warnings for changed files +# if checkStyleToolWarnings function returns 1 i.e. if warnings are +# above baseline. +# Print summary of checkstyle/scalastyle errors for changed files +# if checkStyleToolErrors function returns 1 i.e. if errors are +# above threshold. +# If warnings exist and above baseline, warn the user and print +# top 10 issues at warning severity grouped by issue type. +# Also print top 10 issues at error severity grouped by issue +# type if errors are equal to threshold (for informational +# purposes) +# +# Arguments: +# arg1: Indicates the tool whose report will be processed +# (Checkstyle or Scalastyle) +# arg2: Report location for the tool whose report is to be +# processed +# arg3: Error threshold, above which build would fail, for +# the tool whose report will be processed +# arg4: Warnings baseline for the tool whose report will be +# processed +# arg5: Name of the error threshold constant for the tool +# and language +# arg6: Name of the warning baseline constant for the tool +# and language +# arg7: List of files changed in the PR +# Returns: +# None +################################################################### +function processStyleReports() { + # Verify if style report exists for the tool whose report is being processed + verifyStyleReportExistence $1 $2 + + # Check warnings in Checkstyle/Scalastyle report + checkStyleToolWarnings $1 $2 $4 $6 numWarnings + result=$? + if [ $result -gt 0 ]; then + if [ $result -eq 1 ]; then + # If there are warnings above baseline, find all warnings for changed files + styleIssuesForChangedFiles=$(getStyleIssuesForChangedFiles "${7}" $2 "warning") + fileCnt=`echo "${styleIssuesForChangedFiles}" | grep "Failed checks for file" | wc -l | xargs` + if [ $fileCnt -gt 0 ]; then + echo -e "$WARNING_COLOR_PREFIX Note: This PR may have introduced $1 warnings (baseline: $4)" + echo -e "$WARNING_COLOR_PREFIX Listing $1 WARNINGS for the files changed in the PR:" + echo "${styleIssuesForChangedFiles}" + else + msgToResetStyleReportWarning $1 $4 $6 $numWarnings + fi + fi + fi + echo "" + + # Check errors in Checkstyle/Scalastyle report + checkStyleToolErrors $1 $2 $3 $5 + result=$? + if [ $result -gt 0 ]; then + if [ $result -eq 1 ]; then + echo -e "$ERROR_COLOR_PREFIX Listing $1 ERRORS for the files changed in the PR:" + styleIssuesForChangedFiles=$(getStyleIssuesForChangedFiles "${7}" $2 "error") + echo "${styleIssuesForChangedFiles}" + echo "" + echo -e "$ERROR_COLOR_PREFIX $1 step failed..." + fi + echo "" + exit 1; + fi + echo "" +} + +######################################################### # Run CPD for the langauge passed, check for failures and # move the final result to a separate file. # @@ -158,7 +393,7 @@ function runFindbugs() { # arg4: List of files changed in the PR # Returns: # None -########################################################## +######################################################### function runCPDForLanguage() { sbt cpd if [ $? -ne 0 ]; then @@ -195,6 +430,36 @@ function runCPD() { echo "" } +############################################################ +# Run sbt checkstyle or scalastyle command, parse the report +# and if errors are found above threshold, fail the build. +# +# Arguments: +# arg1: Command/tool to be run (checkstyle or scalastyle) +# arg2: Report location for the tool being run +# arg3: Error threshold, above which build would fail, for +# the tool being run +# arg4: Warnings baseline for the tool being run +# arg5: Name of the error threshold constant for the tool +# and language +# arg6: Name of the warning baseline constant for the tool +# and language +# arg7: List of files changed in the PR +# Returns: +# None +############################################################ +function runStylingTool() { + sbt $1 + if [ $? -ne 0 ]; then + echo -e "$ERROR_COLOR_PREFIX $1 step failed..." + exit 1; + fi + if [ $1 = "scalastyle" ]; then + preProcessScalastyleReport $SCALASTYLE_REPORT_PATH + fi + processStyleReports $(capitalizeFirstCharOfStr $1) $2 $3 $4 $5 $6 "${7}" +} + ######################################################## # # MAIN SCRIPT @@ -246,7 +511,23 @@ echo -e "$SUCCESS_COLOR_PREFIX Copy Paste Detector(CPD) step succeeded..." echo "" echo "************************************************************" -echo " 4. Run unit tests and code coverage" +echo " 4. Checkstyle for JAVA code" +echo "************************************************************" +echo -e "$INFO_COLOR_PREFIX Running Checkstyle..." +runStylingTool "checkstyle" $CHECKSTYLE_REPORT_PATH $CHECKSTYLE_ERROR_THRESHOLD $CHECKSTYLE_WARNING_BASELINE "CHECKSTYLE_ERROR_THRESHOLD" "CHECKSTYLE_WARNING_BASELINE" "${changedFilesList}" +echo -e "$SUCCESS_COLOR_PREFIX Checkstyle step succeeded..." + +echo "" +echo "************************************************************" +echo " 5. Scalastyle for Scala code" +echo "************************************************************" +echo -e "$INFO_COLOR_PREFIX Running Scalastyle..." +runStylingTool "scalastyle" $SCALASTYLE_REPORT_PATH $SCALASTYLE_ERROR_THRESHOLD $SCALASTYLE_WARNING_BASELINE "SCALASTYLE_ERROR_THRESHOLD" "SCALASTYLE_WARNING_BASELINE" "${changedFilesList}" +echo -e "$SUCCESS_COLOR_PREFIX Scalastyle step succeeded..." + +echo "" +echo "************************************************************" +echo " 6. Run unit tests and code coverage" echo "************************************************************" echo -e "$INFO_COLOR_PREFIX Running unit tests and code coverage..." sbt test jacoco:cover