Add license reporting task

Fixes elastic#9331

bin/dependencies-report

@@ -29,4 +29,9 @@ fi
 . "$(cd `dirname ${SOURCEPATH}`/..; pwd)/bin/logstash.lib.sh"
 setup
 
+SCRIPT=$(readlink -f "$0")
+SCRIPTPATH=$(dirname "$SCRIPT")
+
+
+mkdir -p build
 ruby_exec "logstash-core/lib/logstash/dependency_report_runner.rb" "$@"

bin/logstash.lib.sh

@@ -167,3 +167,10 @@ ruby_exec() {
   fi
   exec "${JRUBY_BIN}" "$@"
 }
+
+ruby_run() {
+  if [ "$DEBUG" ] ; then
+    echo "DEBUG: exec ${JRUBY_BIN} $@"
+  fi
+  $JRUBY_BIN $@
+}

build.gradle

@@ -1,9 +1,13 @@
 buildscript {
     repositories {
         mavenCentral()
+        maven {
+            url 'https://plugins.gradle.org/m2/'
+        }
     }
     dependencies {
         classpath 'org.yaml:snakeyaml:1.17'
+        classpath "gradle.plugin.com.github.jk1:gradle-license-report:0.7.1"
     }
 }
 
@@ -72,6 +76,24 @@ allprojects {
   }
 }
 
+subprojects {
+    repositories {
+        maven {
+            url 'https://plugins.gradle.org/m2/'
+        }
+    }
+    dependencies {
+        compile "gradle.plugin.com.github.jk1:gradle-license-report:0.7.1"
+    }
+
+    apply plugin: 'com.github.jk1.dependency-license-report'
+
+    licenseReport {
+        renderer = new com.github.jk1.license.render.CsvReportRenderer()
+        configurations = ['compile', 'runtime']
+    }
+}
+
 // fetch version from Logstash's master versions.yml file
 def versionMap = (Map) (new Yaml()).load(new File("${projectDir}/versions.yml").text)
 version = versionMap['logstash-core']
@@ -102,6 +124,8 @@ clean {
   delete "${projectDir}/.bundle"
   delete "${projectDir}/qa/integration/Gemfile.lock"
   delete "${projectDir}/qa/integration/.bundle"
+  delete "${projectDir}/build/licenseReportFolders.txt"
+  delete "${projectDir}/build/rubyDependencies.csv"
 }
 
 task bootstrap {}
@@ -274,6 +298,46 @@ project(":logstash-integration-tests") {
 
 task runIntegrationTests(dependsOn: [tasks.getByPath(":logstash-integration-tests:integrationTests")]) {}
 
+task generateLicenseReport(type: JavaExec) {
+    dependsOn("generateLicenseReportInputs")
+    dependsOn(":dependencies-report:assemble")
+
+    def jarFile = project('dependencies-report').getBuildDir().toString() + "/libs/dependencies-report.jar"
+
+    String licenseReportInputCSV = project.hasProperty("licenseReportInputCSV") ? project.property("licenseReportInputCSV") : "build/dependencies.csv.ruby"
+    String licenseReportOutputCSV = project.hasProperty("licenseReportOutputCSV") ? project.property("licenseReportOutputCSV") : "build/dependencies.csv"
+
+    classpath = project.files([jarFile])
+    main = "org.logstash.dependencies.Main"
+    args \
+      licenseReportInputCSV, \
+      project.getBuildDir().toString() + "/licenseReportFolders.txt", \
+      licenseReportOutputCSV
+}
+
+task generateLicenseReportInputs() {
+    dependsOn subprojects.generateLicenseReport
+
+    // write location of all license reports for subprojects containing artifacts that are distributed to single file
+    StringBuilder licenseReportFolders = new StringBuilder()
+    subprojects.findAll { s1 -> !s1.hasProperty("isDistributedArtifact") || s1.property("isDistributedArtifact") == 'true'}.each { s ->
+        s.tasks.findAll { t2 -> t2.getName() == "generateLicenseReport" }.each { t3 ->
+            licenseReportFolders.append(t3.outputs.files.asPath + "\n")
+        }
+    }
+
+    if (gradle.startParameter.taskNames.contains("generateLicenseReport")) {
+        def licenseReportPath = project.getBuildDir().toString() + "/licenseReportFolders.txt"
+        def licenseReportFolder = new File(licenseReportPath)
+        licenseReportFolder.delete()
+        licenseReportFolder = new File(licenseReportPath)
+        licenseReportFolder.createNewFile()
+        if (licenseReportFolder.canWrite()) {
+            licenseReportFolder.text = licenseReportFolders.toString()
+        }
+    }
+}
+
 // If you are running a JRuby snapshot we will skip the integrity check.
 verifyFile.onlyIf { doChecksum }
 bootstrap.dependsOn installTestGems

logstash-core/benchmarks/gradle.properties

@@ -0,0 +1,2 @@
+isDistributedArtifact=false
+

logstash-core/gradle.properties

@@ -0,0 +1,2 @@
+isDistributedArtifact=true
+

logstash-core/lib/logstash/dependency_report.rb

@@ -6,26 +6,46 @@
 require "clamp"
 require "rubygems"
 require "jars/gemspec_artifacts"
+require 'fileutils'
+require 'securerandom'
 
 class LogStash::DependencyReport < Clamp::Command
   option [ "--csv" ], "OUTPUT_PATH", "The path to write the dependency report in csv format.",
     :required => true, :attribute_name => :output_path
 
   def execute
     require "csv"
-    CSV.open(output_path, "wb", :headers => [ "name", "version", "url", "license" ], :write_headers => true) do |csv|
+
+    tmp_dir = java.lang.System.getProperty("java.io.tmpdir")
+    ruby_output_path = File.join(tmp_dir, SecureRandom.uuid)
+    # Write a CSV with just the ruby stuff
+    CSV.open(ruby_output_path, "wb", :headers => [ "name", "version", "url", "license" ], :write_headers => true) do |csv|
       puts "Finding gem dependencies"
       gems.each { |d| csv << d }
-      puts "Finding java/jar dependencies"
+      puts "Finding gem embedded java/jar dependencies"
       jars.each { |d| csv << d }
     end
+    puts "Wrote temporary ruby deps CSV to #{ruby_output_path}"
+
 
     # Copy in COPYING.csv which is a best-effort, hand-maintained file of dependency license information.
-    File.open(output_path, "a+") do |file|
+    File.open(ruby_output_path, "a+") do |file|
       extra = File.join(File.dirname(__FILE__), "..", "..", "..", "COPYING.csv")
       file.write(IO.read(extra))
     end
+
+    # Use gradle to find the rest and add to the ruby CSV
+    puts "Find gradle jar dependencies #{Dir.pwd}"
+    command = ["./gradlew", "generateLicenseReport", "-PlicenseReportInputCSV=#{ruby_output_path}", "-PlicenseReportOutputCSV=#{output_path}"]
+    puts "Executing #{command}"
+    system(*command)
+    if $?.exitstatus != 0
+      raise "Could not run gradle java deps! Exit status #{$?.exitstatus}"
+    end
+
     nil
+  ensure
+    FileUtils.rm(ruby_output_path) if ruby_output_path
   end
 
   def gems

qa/integration/gradle.properties

@@ -0,0 +1,2 @@
+isDistributedArtifact=false
+

settings.gradle

@@ -1,6 +1,7 @@
-include ':logstash-core', 'logstash-core-benchmarks', 'ingest-converter', 'benchmark-cli', 'logstash-integration-tests'
+include ':logstash-core', 'logstash-core-benchmarks', 'ingest-converter', 'benchmark-cli', 'logstash-integration-tests', 'dependencies-report'
 project(':logstash-core').projectDir = new File('./logstash-core')
 project(':logstash-core-benchmarks').projectDir = new File('./logstash-core/benchmarks')
 project(':logstash-integration-tests').projectDir = new File('./qa/integration')
 project(':ingest-converter').projectDir = new File('./tools/ingest-converter')
 project(':benchmark-cli').projectDir = new File('./tools/benchmark-cli')
+project(':dependencies-report').projectDir = new File('./tools/dependencies-report')

tools/benchmark-cli/gradle.properties

@@ -0,0 +1,2 @@
+isDistributedArtifact=false
+

tools/dependencies-report/build.gradle

@@ -0,0 +1,53 @@
+import org.yaml.snakeyaml.Yaml
+
+// fetch version from Logstash's master versions.yml file
+def versionMap = (Map) (new Yaml()).load(new File("$projectDir/../../versions.yml").text)
+
+description = """Logstash Dependency Reporting Utility"""
+version = versionMap['logstash-core']
+String jacksonVersion = versionMap['jackson']
+
+repositories {
+  mavenCentral()
+  jcenter()
+}
+
+buildscript {
+  repositories {
+    mavenCentral()
+    jcenter()
+  }
+  dependencies {
+    classpath 'org.yaml:snakeyaml:1.17'
+    classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.4'
+  }
+}
+
+dependencies {
+  compile 'commons-io:commons-io:2.6'
+  compile 'org.apache.commons:commons-csv:1.5'
+  compile "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}"
+  compile "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}"
+  compile "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}"
+  testCompile 'junit:junit:4.12'
+}
+
+javadoc {
+  enabled = false
+}
+
+test {
+  // We need to force IPV4 usage to make WireMock tests portable between *nix and Windows.
+  // For details see: https://github.com/elastic/logstash/pull/8372
+  jvmArgs '-Djava.net.preferIPv4Stack=true'
+}
+
+apply plugin: 'com.github.johnrengelman.shadow'
+
+shadowJar {
+  baseName = 'dependencies-report'
+  classifier = null
+  version = null
+}
+
+assemble.dependsOn shadowJar

tools/dependencies-report/gradle.properties

@@ -0,0 +1,2 @@
+isDistributedArtifact=false
+

tools/dependencies-report/src/main/java/org/logstash/dependencies/Dependency.java

@@ -0,0 +1,81 @@
+package org.logstash.dependencies;
+
+import org.apache.commons.csv.CSVRecord;
+
+import java.util.Objects;
+
+class Dependency implements Comparable<Dependency> {
+    public static final String RUBY_TYPE = "ruby";
+    public static final String JAVA_TYPE = "java";
+
+    String type;
+    String name;
+    String version;
+    String license;
+    String spdxLicense;
+
+    // optional
+    String licenseUrl;
+
+    public static Dependency fromRubyCsvRecord(CSVRecord record) {
+        Dependency d = new Dependency();
+
+        // name, version, url, license
+        d.type = RUBY_TYPE;
+        d.name = record.get(0);
+        d.version = record.get(1);
+        d.license = record.get(3);
+
+        return d;
+    }
+
+    public static Dependency fromJavaCsvRecord(CSVRecord record) {
+        Dependency d = new Dependency();
+
+        // artifact,moduleUrl,moduleLicense,moduleLicenseUrl
+        d.type = JAVA_TYPE;
+
+        String nameAndVersion = record.get(0);
+        int colonIndex = nameAndVersion.indexOf(':');
+        if (colonIndex == -1) {
+            String err = String.format("Could not parse java artifact name and version from '%s'",
+                    nameAndVersion);
+            throw new IllegalStateException(err);
+        }
+        colonIndex = nameAndVersion.indexOf(':', colonIndex + 1);
+        if (colonIndex == -1) {
+            String err = String.format("Could not parse java artifact name and version from '%s'",
+                    nameAndVersion);
+            throw new IllegalStateException(err);
+        }
+        d.name = nameAndVersion.substring(0, colonIndex);
+        d.version = nameAndVersion.substring(colonIndex + 1);
+
+        // We DON'T read the license info out of this CSV because it is not reliable, we want humans
+        // to use the overrides to ensure our license info is accurate
+
+        return d;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        Dependency d = (Dependency) o;
+        return Objects.equals(name, d.name) && Objects.equals(version, d.version);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, version);
+    }
+
+    @Override
+    public int compareTo(Dependency o) {
+        return (name + version).compareTo(o.name + o.version);
+    }
+}

tools/dependencies-report/src/main/java/org/logstash/dependencies/Main.java

@@ -0,0 +1,49 @@
+package org.logstash.dependencies;
+
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+
+/**
+ * Entry point for {@link ReportGenerator}.
+ */
+public class Main {
+
+    static final String LICENSE_MAPPING_PATH = "/licenseMapping.csv";
+    static final String ACCEPTABLE_LICENSES_PATH = "/acceptableLicenses.csv";
+
+    public static void main(String[] args) throws IOException {
+        if (args.length < 3) {
+            System.out.println("Usage: org.logstash.dependencies.Main <pathToRubyDependencies.csv> <pathToJavaLicenseReportFolders.txt> <output.csv>");
+            System.exit(1);
+        }
+
+        InputStream rubyDependenciesStream = new FileInputStream(args[0]);
+        List<String> javaDependencyReports = Files.readAllLines(Paths.get(args[1]));
+        InputStream[] javaDependenciesStreams = new InputStream[javaDependencyReports.size()];
+        for (int k = 0; k < javaDependencyReports.size(); k++) {
+            javaDependenciesStreams[k] = new FileInputStream(javaDependencyReports.get(k) + "/licenses.csv");
+        }
+        FileWriter outputWriter = new FileWriter(args[2]);
+
+        boolean reportResult = new ReportGenerator().generateReport(
+                getResourceAsStream(LICENSE_MAPPING_PATH),
+                getResourceAsStream(ACCEPTABLE_LICENSES_PATH),
+                rubyDependenciesStream,
+                javaDependenciesStreams,
+                outputWriter
+        );
+
+        // If there were unknown results in the report, exit with a non-zero status
+        System.exit(reportResult ? 0 : 1);
+
+    }
+
+    static InputStream getResourceAsStream(String resourcePath) {
+        return ReportGenerator.class.getResourceAsStream(resourcePath);
+    }
+}