KAFKA-17479 Allow ":jar" tasks to be cached [1/n] (apache#17066)

For several modules, we include a kafka-version.properties in the Jar file. This file includes the Git SHA of the project at the time of the build. This means that even if no source files change, the :jar task will never be UP-TO-DATE between two git commits. Ultimately, this breaks Gradle caching.

This patch marks all of the createVersionFile tasks as cacheable and also changes our Gradle invocation to override the commit ID to a dummy static value. This will allow the :jar task to be cacheable and reusable between builds.

This patch also configures the trunk build to only write to the build cache and not read from it. This will prevent any cache pollution/corruption from propagating from build to build.

Reviewers: Chia-Ping Tsai <[email protected]>

.github/actions/setup-gradle/action.yml

@@ -25,6 +25,9 @@ inputs:
   gradle-cache-read-only:
     description: "Should the Gradle cache be read-only?"
     default: "true"
+  gradle-cache-write-only:
+    description: "Should the Gradle cache be write-only?"
+    default: "false"
   develocity-access-key:
     description: "Optional access key for uploading build scans to Develocity"
     default: ""
@@ -45,6 +48,7 @@ runs:
         develocity-access-key: ${{ inputs.develocity-access-key }}
         develocity-token-expiry: 4
         cache-read-only: ${{ inputs.gradle-cache-read-only }}
+        cache-write-only: ${{ inputs.gradle-cache-write-only }}
         # Cache downloaded JDKs in addition to the default directories.
         gradle-home-cache-includes: |
           caches

.github/scripts/junit.py

@@ -55,6 +55,9 @@ class TestCase:
     def key(self) -> Tuple[str, str]:
         return self.class_name, self.test_name
 
+    def __repr__(self):
+        return f"{self.class_name} {self.test_name}"
+
 
 @dataclasses.dataclass
 class TestSuite:

.github/workflows/build.yml

@@ -22,6 +22,10 @@ on:
         description: "Should the Gradle cache be read-only?"
         default: "true"
         type: string
+      gradle-cache-write-only:
+        description: "Should the Gradle cache be write-only?"
+        default: "false"
+        type: string
       github-actions-opt-in:
         description: "Should we run the opt-in steps?"
         default: "false"
@@ -52,6 +56,7 @@ jobs:
         with:
           java-version: ${{ matrix.java }}
           gradle-cache-read-only: ${{ inputs.gradle-cache-read-only }}
+          gradle-cache-write-only: ${{ inputs.gradle-cache-write-only }}
           develocity-access-key: ${{ secrets.GE_ACCESS_TOKEN }}
       - name: Compile and validate
         # Gradle flags
@@ -94,18 +99,22 @@ jobs:
         with:
           java-version: ${{ matrix.java }}
           gradle-cache-read-only: ${{ inputs.gradle-cache-read-only }}
+          gradle-cache-write-only: ${{ inputs.gradle-cache-write-only }}
           develocity-access-key: ${{ secrets.GE_ACCESS_TOKEN }}
       - name: Test
         # Gradle flags
         # --build-cache:  Let Gradle restore the build cache
         # --scan:         Attempt to publish build scans in PRs. This will only work on PRs from apache/kafka, not public forks.
         # --continue:     Keep running even if a test fails
+        # -PcommitId      Prevent the Git SHA being written into the jar files (which breaks caching)
         timeout-minutes: 180  # 3 hours
+        continue-on-error: true
         run: |
           ./gradlew --build-cache --scan --continue \
           -PtestLoggingEvents=started,passed,skipped,failed \
-          -PignoreFailures=true -PmaxParallelForks=2 \
+          -PmaxParallelForks=2 \
           -PmaxTestRetries=1 -PmaxTestRetryFailures=10 \
+          -PcommitId=xxxxxxxxxxxxxxxx \
           test
       - name: Archive JUnit reports
         if: always()

.github/workflows/ci.yml

@@ -34,6 +34,7 @@ jobs:
     uses: ./.github/workflows/build.yml
     with:
       gradle-cache-read-only: ${{ github.ref != 'refs/heads/trunk' }}
+      gradle-cache-write-only: ${{ github.ref == 'refs/heads/trunk' }}
       github-actions-opt-in: ${{ github.ref == 'refs/heads/trunk' || contains(github.head_ref, 'gh-') }}
     secrets:
       inherit

build.gradle

@@ -252,7 +252,7 @@ if (repo != null) {
   rat.enabled = false
 }
 println("Starting build with version $version (commit id ${commitId == null ? "null" : commitId.take(8)}) using Gradle $gradleVersion, Java ${JavaVersion.current()} and Scala ${versions.scala}")
-println("Build properties: maxParallelForks=$maxTestForks, maxScalacThreads=$maxScalacThreads, maxTestRetries=$userMaxTestRetries")
+println("Build properties: ignoreFailures=$userIgnoreFailures, maxParallelForks=$maxTestForks, maxScalacThreads=$maxScalacThreads, maxTestRetries=$userMaxTestRetries")
 
 subprojects {
 
@@ -882,6 +882,7 @@ project(':server') {
     inputs.property "commitId", commitId
     inputs.property "version", version
     outputs.file receiptFile
+    outputs.cacheIf { true }
 
     doLast {
       def data = [
@@ -1366,6 +1367,7 @@ project(':group-coordinator:group-coordinator-api') {
     inputs.property "commitId", commitId
     inputs.property "version", version
     outputs.file receiptFile
+    outputs.cacheIf { true }
 
     doLast {
       def data = [
@@ -1735,6 +1737,7 @@ project(':clients') {
     inputs.property "commitId", commitId
     inputs.property "version", version
     outputs.file receiptFile
+    outputs.cacheIf { true }
 
     doLast {
       def data = [
@@ -1900,6 +1903,7 @@ project(':raft') {
     inputs.property "commitId", commitId
     inputs.property "version", version
     outputs.file receiptFile
+    outputs.cacheIf { true }
 
     doLast {
       def data = [
@@ -1993,6 +1997,7 @@ project(':server-common') {
     inputs.property "commitId", commitId
     inputs.property "version", version
     outputs.file receiptFile
+    outputs.cacheIf { true }
 
     doLast {
       def data = [
@@ -2051,6 +2056,7 @@ project(':storage:storage-api') {
     inputs.property "commitId", commitId
     inputs.property "version", version
     outputs.file receiptFile
+    outputs.cacheIf { true }
 
     doLast {
       def data = [
@@ -2141,6 +2147,7 @@ project(':storage') {
     inputs.property "commitId", commitId
     inputs.property "version", version
     outputs.file receiptFile
+    outputs.cacheIf { true }
 
     doLast {
       def data = [
@@ -2235,6 +2242,7 @@ project(':tools:tools-api') {
     inputs.property "commitId", commitId
     inputs.property "version", version
     outputs.file receiptFile
+    outputs.cacheIf { true }
 
     doLast {
       def data = [