Add license reports and checks (#8497)

Dependencies, runtime dependencies including those that are distributed as part of a fat/uber jar, must use a "compatible" license. Checking those licenses manually is good, but some automatic checks are better.

This change adds a Gradle plugin to check licenses in the runtime configurations of the Nessie distributables (see below), and to produce license reports as HTML.

Nessie distributable are:
* Nessie Quarkus Server
* Nessie Quarkus CLI
* Nessie GC tool
* Nessie Content Generator
* Nessie Spark SQL extensions
* Nessie CLI/REPL

Checks and reports are run against the `runtimeClasspath` configurations of the above.

.github/workflows/ci.yml

@@ -103,6 +103,11 @@ jobs:
       - name: Gradle / Assemble
         run: ./gradlew assemble --scan
 
+      - name: Gradle / License Check
+        uses: gradle/gradle-build-action@v2
+        with:
+          arguments: checkLicense
+
       - name: Gradle / Publish to Maven local
         run: ./gradlew publishToMavenLocal --scan
 

.github/workflows/release-publish.yml

@@ -121,6 +121,10 @@ jobs:
           ./gradlew compileAll jar testClasses 
         echo "::endgroup::"
 
+        echo "::group::Check Licenses"
+        ./gradlew aggregatedLicenseReportsZip
+        echo "::endgroup::"
+
         echo "::group::Publish to Sonatype"
         # 2 Retries - to mitigate "HTTP/502 Bad Gateway" issues
         ./gradlew publishToMavenLocal publishToSonatype closeAndReleaseSonatypeStagingRepository -Prelease -Puber-jar || \
@@ -152,6 +156,7 @@ jobs:
         echo "CLI_UBER_JAR=${ARTIFACTS}/nessie-quarkus-cli-${RELEASE_VERSION}-runner.jar" >> ${GITHUB_ENV}
         echo "GC_UBER_JAR=gc/gc-tool/build/executable/nessie-gc-${RELEASE_VERSION}.jar" >> ${GITHUB_ENV}
         echo "NESSIE_OPENAPI=api/model/build/nessie-openapi-${RELEASE_VERSION}.yaml" >> ${GITHUB_ENV}
+        echo "LICENSE_REPORTS=tools/aggregated-license-report/build/distributions/nessie-aggregated-license-report-${RELEASE_VERSION}.zip" >> ${GITHUB_ENV}
 
         echo "## Successfully released ${RELEASE_VERSION} to Sonatype" >> $GITHUB_STEP_SUMMARY
 
@@ -249,7 +254,8 @@ jobs:
           "${CLI_UBER_JAR}" \
           "${GC_UBER_JAR}" \
           "${NESSIE_OPENAPI}" \
-          "${NESSIE_HELM_CHART}"
+          "${NESSIE_HELM_CHART}" \
+          "${LICENSE_REPORTS}"
 
     - name: Update SwaggerHub
       uses: smartbear/[email protected]

NOTICE

@@ -4,3 +4,7 @@ Copyright 2015-2017 Dremio Corporation
 This product includes software developed at 
 The Apache Software Foundation (http://www.apache.org/).
 
+---
+
+The main docs page for a particular Nessie release under https://projectnessie.org/docs/ contains an
+aggregated license report (since Nessie version 0.83).

build-logic/build.gradle.kts

@@ -23,6 +23,7 @@ dependencies {
   implementation(baselibs.idea.ext)
   implementation(baselibs.shadow)
   implementation(baselibs.errorprone)
+  implementation(baselibs.license.report)
 
   testImplementation(platform(baselibs.junit.bom))
   testImplementation(baselibs.assertj.core)

build-logic/src/main/kotlin/nessie-license-report.gradle.kts

@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2023 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.github.jk1.license.filter.LicenseBundleNormalizer
+import com.github.jk1.license.render.InventoryHtmlReportRenderer
+import com.github.jk1.license.render.JsonReportRenderer
+import com.github.jk1.license.render.XmlReportRenderer
+import java.util.*
+
+plugins { id("com.github.jk1.dependency-license-report") }
+
+afterEvaluate {
+  // Need to configure after evaluation, because the spark-extensions project use a custom
+  // `buildDir`.
+  licenseReport {
+    filters =
+      arrayOf(
+        LicenseBundleNormalizer(
+          "${rootProject.projectDir}/gradle/license/normalizer-bundle.json",
+          false
+        )
+      )
+    allowedLicensesFile = rootProject.projectDir.resolve("gradle/license/allowed-licenses.json")
+    renderers =
+      arrayOf(InventoryHtmlReportRenderer("index.html"), JsonReportRenderer(), XmlReportRenderer())
+    excludeBoms = true
+    excludes =
+      arrayOf(
+        "com.google.guava:guava-parent",
+        "io.opentelemetry:opentelemetry-bom-alpha",
+        "io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom-alpha"
+      )
+    outputDir = "${project.layout.buildDirectory.get()}/reports/dependency-license"
+    excludeGroups = arrayOf("org.projectnessie.nessie", "org.projectnessie.nessie-integrations")
+  }
+}
+
+tasks.named("generateLicenseReport") {
+  inputs
+    .files(
+      rootProject.projectDir.resolve("gradle/license/normalizer-bundle.json"),
+      rootProject.projectDir.resolve("gradle/license/allowed-licenses.json")
+    )
+    .withPathSensitivity(PathSensitivity.RELATIVE)
+  inputs.property("renderersHash", Arrays.hashCode(licenseReport.renderers))
+  inputs.property("filtersHash", Arrays.hashCode(licenseReport.filters))
+  inputs.property("excludesHash", Arrays.hashCode(licenseReport.excludes))
+  inputs.property("excludeGroupsHash", Arrays.hashCode(licenseReport.excludeGroups))
+}
+
+tasks.register<Zip>("licenseReportZip") {
+  from(tasks.named("generateLicenseReport"))
+  archiveClassifier.set("license-report")
+  archiveExtension.set("zip")
+}
+
+val licenseReports by
+  configurations.creating {
+    isCanBeConsumed = true
+    isCanBeResolved = false
+    description = "License report files"
+    outgoing { artifact(tasks.named("licenseReportZip")) }
+  }
+
+plugins.withType<MavenPublishPlugin>().configureEach {
+  configure<PublishingExtension> {
+    publications { named<MavenPublication>("maven") { artifact(tasks.named("licenseReportZip")) } }
+  }
+}

cli/cli/build.gradle.kts

@@ -21,6 +21,7 @@ plugins {
   id("nessie-conventions-server")
   id("nessie-jacoco")
   id("nessie-shadow-jar")
+  id("nessie-license-report")
 }
 
 extra["maven.name"] = "Nessie - CLI"

gc/gc-tool/build.gradle.kts

@@ -21,6 +21,7 @@ plugins {
   id("nessie-conventions-iceberg")
   id("nessie-jacoco")
   id("nessie-shadow-jar")
+  id("nessie-license-report")
 }
 
 extra["maven.name"] = "Nessie - GC - Standalone command line tool"

gradle/baselibs.versions.toml

@@ -6,5 +6,6 @@ errorprone = { module = "net.ltgt.gradle:gradle-errorprone-plugin", version = "3
 idea-ext = { module = "gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext", version = "1.1.8" }
 jandex = { module = "com.github.vlsi.gradle:jandex-plugin", version = "1.90" }
 junit-bom = { module = "org.junit:junit-bom", version = "5.10.2" }
+license-report = { module = "com.github.jk1:gradle-license-report", version = "2.7" }
 shadow = { module = "com.github.johnrengelman:shadow", version = "8.1.1" }
 spotless = { module = "com.diffplug.spotless:spotless-plugin-gradle", version = "6.25.0" }

gradle/license/allowed-licenses.json

@@ -0,0 +1,88 @@
+{
+  "allowedLicenses": [
+    {
+      "moduleLicense": "Apache Software License, Version 2.0"
+    },
+    {
+      "moduleLicense": "Apache Software License, version 2.0"
+    },
+    {
+      "moduleLicense": "Apache License, Version 2.0"
+    },
+    {
+      "moduleLicense": "Apache 2"
+    },
+    {
+      "moduleLicense": "Apache 2.0"
+    },
+    {
+      "moduleLicense": "Apache-2.0"
+    },
+    {
+      "moduleLicense": "BSD-2.Clause"
+    },
+    {
+      "moduleLicense": "Revised BSD"
+    },
+    {
+      "moduleLicense": "BSD-3.Clause"
+    },
+    {
+      "moduleLicense": "BSD New license"
+    },
+    {
+      "moduleLicense": "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0"
+    },
+    {
+      "moduleLicense": "Creative Commons 1.0 Universal"
+    },
+    {
+      "moduleLicense": "Eclipse Distribution License, Version 1.0"
+    },
+    {
+      "moduleLicense": "EDL 1.0"
+    },
+    {
+      "moduleLicense": "Eclipse Public License, Version 1.0"
+    },
+    {
+      "moduleLicense": "EPL 2.0"
+    },
+    {
+      "moduleLicense": "Eclipse Public License, Version 2.0"
+    },
+    {
+      "moduleLicense": "Eclipse Public License - v 2.0"
+    },
+    {
+      "moduleLicense": "Eclipse Public License v. 2.0"
+    },
+    {
+      "moduleLicense": "GNU General Public License, Version 2 with the GNU Classpath Exception"
+    },
+    {
+      "moduleLicense": "Go License"
+    },
+    {
+      "moduleLicense": "MIT License"
+    },
+    {
+      "moduleLicense": "MIT-0"
+    },
+    {
+      "moduleLicense": "MIT"
+    },
+    {
+      "moduleLicense": "Public Domain"
+    },
+    {
+      "moduleLicense": "Public Domain, per Creative Commons CC0"
+    },
+    {
+      "moduleLicense": "Universal Permissive License, Version 1.0"
+    },
+    {
+      "moduleLicense": "Bouncy Castle Licence"
+    }
+  ]
+}

gradle/license/normalizer-bundle.json

@@ -0,0 +1,77 @@
+{
+  "bundles" : [
+    { "bundleName" : "apache2", "licenseName" : "Apache Software License, Version 2.0", "licenseUrl" : "http://www.apache.org/licenses/LICENSE-2.0" },
+    { "bundleName" : "bsd2", "licenseName" : "BSD-2-Clause", "licenseUrl" : "http://opensource.org/licenses/BSD-2-Clause" },
+    { "bundleName" : "bsd3", "licenseName" : "BSD-3-Clause", "licenseUrl" : "http://opensource.org/licenses/BSD-3-Clause" },
+    { "bundleName" : "cc0", "licenseName" : "Creative Commons 1.0 Universal", "licenseUrl" : "http://creativecommons.org/publicdomain/zero/1.0/" },
+    { "bundleName" : "cddl1", "licenseName" : "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE Version 1.0 (CDDL-1.0)", "licenseUrl" : "http://opensource.org/licenses/CDDL-1.0" },
+    { "bundleName" : "edl1", "licenseName" :  "Eclipse Distribution License, Version 1.0", "licenseUrl":  "http://www.eclipse.org/legal/edl-v10.html" },
+    { "bundleName" : "epl1", "licenseName" :  "Eclipse Public License, Version 1.0", "licenseUrl":  "http://www.eclipse.org/legal/epl-v10.html" },
+    { "bundleName" : "epl2", "licenseName" :  "Eclipse Public License, Version 2.0", "licenseUrl":  "http://www.eclipse.org/legal/epl-v20.html" },
+    { "bundleName" : "go", "licenseName" :  "Go License", "licenseUrl":  "https://golang.org/LICENSE" },
+    { "bundleName" : "gpl2", "licenseName" :  "GNU General Public License, Version 2", "licenseUrl":  "http://www.gnu.org/copyleft/gpl.html" },
+    { "bundleName" : "gpl2ce", "licenseName" :  "GNU General Public License, Version 2 with the GNU Classpath Exception", "licenseUrl": "https://www.gnu.org/software/classpath/license.html" },
+    { "bundleName" : "lgpl21", "licenseName" :  "GNU Lesser General Public License Version 2.1", "licenseUrl": "http://www.gnu.org/licenses/lgpl.html" },
+    { "bundleName" : "lgpl3", "licenseName" :  "GNU Lesser General Public License Version 3 or greater", "licenseUrl": "http://www.gnu.org/licenses/lgpl.html" },
+    { "bundleName" : "mit", "licenseName" : "MIT License", "licenseUrl": "http://www.opensource.org/licenses/mit-license.php" },
+    { "bundleName" : "mpl2", "licenseName" : "Mozilla Public License, Version 2.0", "licenseUrl": "https://www.mozilla.org/en-US/MPL/2.0/" },
+    { "bundleName" : "upl", "licenseName" : "Universal Permissive License, Version 1.0", "licenseUrl": "http://opensource.org/licenses/UPL" }
+  ],
+  "transformationRules" : [
+    { "bundleName" : "apache2", "licenseUrlPattern" : "https?://www.apache.org/licenses/LICENSE-2.0.*" },
+    { "bundleName" : "apache2", "licenseUrlPattern" : "https?://opensource.org/licenses/Apache-2.0" },
+    { "bundleName" : "apache2", "licenseUrlPattern" : "https?://repository.jboss.org/licenses/apache-2.0.*" },
+    { "bundleName" : "apache2", "licenseUrlPattern" : "https?://aws.amazon.com/apache2.0" },
+
+    { "bundleName" : "bsd2", "licenseUrlPattern" : "https?://www.opensource.org/licenses/bsd-license.php" },
+    { "bundleName" : "bsd2", "licenseUrlPattern" : "https?://opensource.org/licenses/BSD-2-Clause" },
+
+    { "bundleName" : "bsd3", "licenseNamePattern" : "BSD New license" },
+    { "bundleName" : "bsd3", "licenseNamePattern" : "The BSD License" },
+
+    { "bundleName" : "cc0", "licenseUrlPattern" : "https?://creativecommons.org/publicdomain/zero/1.0/" },
+    { "bundleName" : "cc0", "licenseUrlPattern" : "https?://repository.jboss.org/licenses/cc0-1.0.*" },
+
+    { "bundleName" : "edl1", "licenseUrlPattern" : "https?://www.eclipse.org/legal/edl-v10.*" },
+    { "bundleName" : "edl1", "licenseUrlPattern" : "https?://www.eclipse.org/org/documents/edl-v10.*" },
+
+    { "bundleName" : "epl1", "licenseUrlPattern" : "https?://www.eclipse.org/legal/epl-v10.*" },
+    { "bundleName" : "epl1", "licenseUrlPattern" : "https?://www.eclipse.org/legal/epl-1.0.*" },
+    { "bundleName" : "epl1", "licenseUrlPattern" : "https?://www.eclipse.org/org/documents/epl-1.0/EPL-1.0.*" },
+    { "bundleName" : "epl1", "licenseUrlPattern" : "https?://opensource.org/licenses/eclipse-1.0.php" },
+    { "bundleName" : "epl1", "licenseUrlPattern" : "https?://projects.eclipse.org/license/epl-1.0" },
+
+    { "bundleName" : "epl2", "licenseUrlPattern" : "https?://www.eclipse.org/legal/epl-v20.*" },
+    { "bundleName" : "epl2", "licenseUrlPattern" : "https?://www.eclipse.org/legal/epl-2.0.*" },
+    { "bundleName" : "epl2", "licenseUrlPattern" : "https?://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.*" },
+    { "bundleName" : "epl1", "licenseUrlPattern" : "https?://opensource.org/licenses/eclipse-2.0.php" },
+    { "bundleName" : "epl2", "licenseUrlPattern" : "https?://projects.eclipse.org/license/epl-2.0" },
+
+    { "bundleName" : "go", "licenseUrlPattern" : "https?://golang.org/LICENSE" },
+
+    { "bundleName" : "gpl2", "licenseUrlPattern" : "https?://www.gnu.org/licenses/gpl-2.0.html" },
+    { "bundleName" : "gpl2", "licenseUrlPattern" : "https?://www.gnu.org/copyleft/gpl.html" },
+
+    { "bundleName" : "gpl2ce", "licenseUrlPattern" : "https?://www.gnu.org/software/classpath/license.*" },
+    { "bundleName" : "gpl2ce", "licenseUrlPattern" : "https?://projects.eclipse.org/license/secondary-gpl-2.0-cp" },
+    { "bundleName" : "gpl2ce", "licenseUrlPattern" : "http://glassfish.java.net/public/CDDL+GPL_1_1.html" },
+    { "bundleName" : "gpl2ce", "licenseUrlPattern" : "https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html" },
+    { "bundleName" : "gpl2ce", "licenseUrlPattern" : "https://github.com/javaee/javax.annotation/blob/master/LICENSE" },
+
+    { "bundleName" : "lgpl21", "licenseNamePattern" : "GNU LESSER GENERAL PUBLIC LICENSE, Version 2.1" },
+    { "bundleName" : "lgpl21", "licenseNamePattern" : "LGPL, version 2.1" },
+    { "bundleName" : "lgpl21", "licenseNamePattern" : "LGPL 2.1" },
+    { "bundleName" : "lgpl21", "licenseNamePattern" : "lgpl" },
+
+    { "bundleName" : "lgpl3", "licenseNamePattern" : "Lesser General Public License, version 3 or greater" },
+
+    { "bundleName" : "mit", "licenseUrlPattern" : "https?://spdx.org/licenses/MIT.txt" },
+    { "bundleName" : "mit", "licenseUrlPattern" : "https?://spdx.org/licenses/MIT-0.*" },
+    { "bundleName" : "mit", "licenseUrlPattern" : "https?://www.opensource.org/licenses/mit-license.php" },
+    { "bundleName" : "mit", "licenseUrlPattern" : "https?://opensource.org/licenses/MIT" },
+
+    { "bundleName" : "mpl2", "licenseUrlPattern": "https?://www.mozilla.org/en-US/MPL/2.0/" },
+
+    { "bundleName" : "upl", "licenseUrlPattern": "https?://opensource.org/licenses/UPL" }
+  ]
+}

gradle/projects.main.properties

@@ -1,3 +1,4 @@
+nessie-aggregated-license-report=tools/aggregated-license-report
 nessie-azurite-testcontainer=testing/azurite-container
 nessie-bom=bom
 nessie-cli=cli/cli

integrations/spark-extensions/build.gradle.kts

@@ -21,6 +21,7 @@ plugins {
   id("nessie-conventions-spark")
   id("nessie-shadow-jar")
   id("nessie-jacoco")
+  id("nessie-license-report")
 }
 
 val sparkScala = getSparkScalaVersionsForProject()

servers/quarkus-cli/build.gradle.kts

@@ -21,6 +21,7 @@ plugins {
   alias(libs.plugins.quarkus)
   id("nessie-conventions-quarkus")
   id("nessie-jacoco")
+  id("nessie-license-report")
 }
 
 extra["maven.name"] = "Nessie - Quarkus CLI"

servers/quarkus-common/build.gradle.kts

@@ -56,7 +56,10 @@ dependencies {
   implementation(enforcedPlatform(libs.quarkus.google.cloud.services.bom))
   implementation("io.quarkiverse.googlecloudservices:quarkus-google-cloud-bigtable")
   implementation(enforcedPlatform(libs.quarkus.cassandra.bom))
-  implementation("com.datastax.oss.quarkus:cassandra-quarkus-client")
+  implementation("com.datastax.oss.quarkus:cassandra-quarkus-client") {
+    // spotbugs-annotations has only a GPL license!
+    exclude("com.github.spotbugs", "spotbugs-annotations")
+  }
 
   implementation("org.jboss.slf4j:slf4j-jboss-logmanager")
   implementation("io.opentelemetry:opentelemetry-opencensus-shim") // for Google BigTable

servers/quarkus-server/build.gradle.kts

@@ -21,6 +21,7 @@ plugins {
   alias(libs.plugins.quarkus)
   id("nessie-conventions-quarkus")
   id("nessie-jacoco")
+  id("nessie-license-report")
 }
 
 extra["maven.name"] = "Nessie - Quarkus Server"

site/in-dev/index-release.md

@@ -105,3 +105,6 @@ java -jar nessie-quarkus-cli-::NESSIE_VERSION::-runner.jar
 
 [![Maven Central](https://img.shields.io/maven-central/v/org.projectnessie.nessie/nessie?label=Maven%20Central&logo=apachemaven&color=3f6ec6&style=for-the-badge&logoColor=white)](https://search.maven.org/artifact/org.projectnessie.nessie/nessie)
 
+### License Reports
+
+License reports for this release are [available via this link (zip file)](https://github.com/projectnessie/nessie/releases/download/nessie-::NESSIE_VERSION::/nessie-aggregated-license-report-::NESSIE_VERSION::.zip).