Update avro to 1.12.0 (#190)

* Update avro-compiler to 1.12.0

* Add 1.11 in scripted tests

* Support avro 1.12 new schema parser builder

* Update CI to java 11

* Use generic AvroRuntimeException in avro compile step

---------

Co-authored-by: Michel Davit <[email protected]>

.github/workflows/ci.yml

@@ -24,22 +24,14 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         scala: [2.12.19]
-        java: [temurin@8, temurin@11]
+        java: [temurin@11]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout current branch (full)
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - name: Setup Java (temurin@8)
-        if: matrix.java == 'temurin@8'
-        uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: 8
-          cache: sbt
-
       - name: Setup Java (temurin@11)
         if: matrix.java == 'temurin@11'
         uses: actions/setup-java@v4
@@ -74,22 +66,14 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         scala: [2.12.19]
-        java: [temurin@8]
+        java: [temurin@11]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout current branch (full)
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - name: Setup Java (temurin@8)
-        if: matrix.java == 'temurin@8'
-        uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: 8
-          cache: sbt
-
       - name: Setup Java (temurin@11)
         if: matrix.java == 'temurin@11'
         uses: actions/setup-java@v4

README.md

@@ -22,7 +22,7 @@ addSbtPlugin("com.github.sbt" % "sbt-avro" % "3.4.3")
 // Java sources compiled with one version of Avro might be incompatible with a
 // different version of the Avro library. Therefore we specify the compiler
 // version here explicitly.
-libraryDependencies += "org.apache.avro" % "avro-compiler" % "1.11.3"
+libraryDependencies += "org.apache.avro" % "avro-compiler" % "1.12.0"
 ```
 
 Add the library dependency to `build.sbt`:

build.sbt

@@ -11,7 +11,7 @@ ThisBuild / githubWorkflowBuild := Seq(
   WorkflowStep.Sbt(name = Some("Build project"), commands = List("compile", "test", "scripted"))
 )
 ThisBuild / githubWorkflowTargetBranches := Seq("main")
-ThisBuild / githubWorkflowJavaVersions := Seq(JavaSpec.temurin("8"), JavaSpec.temurin("11"))
+ThisBuild / githubWorkflowJavaVersions := Seq(JavaSpec.temurin("11"))
 ThisBuild / githubWorkflowTargetTags := Seq("v*")
 ThisBuild / githubWorkflowPublishTargetBranches := Seq(RefPredicate.StartsWith(Ref.Tag("v")))
 ThisBuild / githubWorkflowPublish := Seq(WorkflowStep.Sbt(

project/Dependencies.scala

@@ -3,7 +3,7 @@ import sbt._
 object Dependencies {
 
   object Versions {
-    val Avro = "1.11.3"
+    val Avro = "1.12.0"
     val Specs2 = "4.20.4"
   }
 

src/main/java/com/github/sbt/avro/mojo/AvscFilesCompiler.java

@@ -1,5 +1,7 @@
 package com.github.sbt.avro.mojo;
 
+import org.apache.avro.AvroRuntimeException;
+import org.apache.avro.AvroTypeException;
 import org.apache.avro.Schema;
 import org.apache.avro.SchemaParseException;
 import org.apache.avro.compiler.specific.SpecificCompiler;
@@ -122,7 +124,7 @@ private boolean tryCompile(AvroFileRef src, File outputDirectory) {
     try {
       schema = schemaParser.parse(src.getFile());
       validateParsedSchema(src, schema);
-    } catch (SchemaParseException e) {
+    } catch (AvroRuntimeException e) {
       schemaParser = successfulSchemaParser;
       compileExceptions.put(src, e);
       return false;

src/main/scala/com/github/sbt/avro/DefaultSchemaParserBuilder.scala

@@ -3,32 +3,14 @@ package com.github.sbt.avro
 import com.github.sbt.avro.mojo.SchemaParserBuilder
 import org.apache.avro.Schema
 
-import scala.collection.JavaConverters._
-
-case class DefaultSchemaParserBuilder(types: Iterable[Schema],
-                                      validate: Boolean,
-                                      validateDefaults: Boolean)
-    extends SchemaParserBuilder {
-
-  override def build(): Schema.Parser = {
-    val parser = new Schema.Parser
-    parser.addTypes(types.map(el => el.getFullName() -> el).toMap.asJava)
-    parser.setValidate(validate)
-    parser.setValidateDefaults(validateDefaults)
-    parser
-  }
-}
-
 object DefaultSchemaParserBuilder {
-  def default(): DefaultSchemaParserBuilder = {
-    template(new Schema.Parser())
-  }
 
-  def template(template: Schema.Parser): DefaultSchemaParserBuilder = {
-    DefaultSchemaParserBuilder(
-      template.getTypes.values().asScala,
-      template.getValidate,
-      template.getValidateDefaults
-    )
+  def default(): SchemaParserBuilder = {
+    val Array(1, minor, _) = classOf[Schema].getPackage.getImplementationVersion.split("\\.").take(3).map(_.toInt)
+    if (minor >= 12) {
+      NameValidatorSchemaParserBuilder()
+    } else {
+      LegacySchemaParserBuilder()
+    }
   }
 }

src/main/scala/com/github/sbt/avro/LegacySchemaParserBuilder.scala

@@ -0,0 +1,46 @@
+package com.github.sbt.avro
+
+import com.github.sbt.avro.mojo.SchemaParserBuilder
+import org.apache.avro.Schema
+
+import scala.annotation.nowarn
+import scala.collection.JavaConverters.*
+
+// used until avro 1.11
+// for avro 2.12+ use NameValidatorSchemaParserBuilder
+case class LegacySchemaParserBuilder(
+  types: Iterable[Schema] = LegacySchemaParserBuilder.DefaultTypes,
+  validate: Boolean = LegacySchemaParserBuilder.DefaultValidate,
+  validateDefaults: Boolean = LegacySchemaParserBuilder.DefaultValidateDefaults)
+extends SchemaParserBuilder {
+
+  override def build(): Schema.Parser = {
+    val parser = new Schema.Parser
+    // addTypes(Map<String, Schema> types) is the only API available in 1.8
+    parser.addTypes(types.map(el => el.getFullName -> el).toMap.asJava): @nowarn
+    LegacySchemaParserBuilder.setValidate(parser)(validate)
+    parser.setValidateDefaults(validateDefaults)
+    parser
+  }
+}
+
+object LegacySchemaParserBuilder {
+  // validate hase been removed in 1.12 in favor of a NameValidator
+  private def setValidate(parser: Schema.Parser)(validate: Boolean): Schema.Parser =
+    classOf[Schema.Parser]
+      .getMethod("setValidate", classOf[Boolean])
+      .invoke(parser, validate: java.lang.Boolean)
+      .asInstanceOf[Schema.Parser]
+
+  private def getValidate(parser: Schema.Parser): Boolean =
+      classOf[Schema.Parser]
+        .getMethod("getValidate")
+        .invoke(parser)
+        .asInstanceOf[Boolean]
+
+  private val defaultParser = new Schema.Parser
+
+  private val DefaultTypes: Iterable[Schema] = defaultParser.getTypes.values().asScala
+  private val DefaultValidate: Boolean = getValidate(defaultParser)
+  private val DefaultValidateDefaults: Boolean = defaultParser.getValidateDefaults
+}

src/main/scala/com/github/sbt/avro/NameValidatorSchemaParserBuilder.scala

@@ -0,0 +1,19 @@
+package com.github.sbt.avro
+
+import com.github.sbt.avro.mojo.SchemaParserBuilder
+import org.apache.avro.{NameValidator, Schema}
+
+import scala.collection.JavaConverters._
+
+case class NameValidatorSchemaParserBuilder(
+  types: Iterable[Schema] = Iterable.empty,
+  validation: NameValidator = NameValidator.UTF_VALIDATOR,
+  validateDefaults: Boolean = true
+) extends SchemaParserBuilder {
+
+  override def build(): Schema.Parser = {
+    val parser = new Schema.Parser(validation)
+    parser.addTypes(types.asJava)
+    parser.setValidateDefaults(validateDefaults)
+  }
+}

src/sbt-test/sbt-avro/avscparser/build.sbt

@@ -4,7 +4,7 @@ import org.apache.avro.Schema
 name := "avscparser-test"
 scalaVersion := "2.13.11"
 libraryDependencies ++= Seq(
-  "org.apache.avro" % "avro" % "1.11.3",
+  "org.apache.avro" % "avro" % avroCompilerVersion,
   "org.specs2" %% "specs2-core" % "4.20.4" % Test
 )
 avroSchemaParserBuilder := AnnotateWithArtifactSchemaParser

src/sbt-test/sbt-avro/avscparser/project/plugins.sbt

@@ -4,4 +4,4 @@ sys.props.get("plugin.version") match {
                          |Specify this property using the scriptedLaunchOpts -D.""".stripMargin)
 }
 
-libraryDependencies += "org.apache.avro" % "avro-compiler" % "1.11.3"
+libraryDependencies += "org.apache.avro" % "avro-compiler" % "1.12.0"

src/sbt-test/sbt-avro/basic_1.10/project/build.properties

@@ -0,0 +1 @@
+../../basic_current/project/build.properties

src/sbt-test/sbt-avro/basic_1.10/test

@@ -1 +1 @@
-../basic_current/test
+../basic_1.11/test

src/sbt-test/sbt-avro/basic_1.11/build.sbt

@@ -0,0 +1,3 @@
+name := "basic-test"
+scalaVersion := "2.13.11"
+libraryDependencies += "org.apache.avro" % "avro" % avroCompilerVersion

src/sbt-test/sbt-avro/basic_1.11/project/build.properties

@@ -0,0 +1 @@
+../../basic_current/project/build.properties

src/sbt-test/sbt-avro/basic_1.11/project/plugins.sbt

@@ -0,0 +1,6 @@
+sys.props.get("plugin.version") match {
+  case Some(x) => addSbtPlugin("com.github.sbt" % "sbt-avro" % x)
+  case _ => sys.error("""|The system property 'plugin.version' is not defined.
+                         |Specify this property using the scriptedLaunchOpts -D.""".stripMargin)
+}
+libraryDependencies += "org.apache.avro" % "avro-compiler" % "1.11.3" // scala-steward:off

src/sbt-test/sbt-avro/basic_1.11/src

@@ -0,0 +1 @@
+../basic_current/src

src/sbt-test/sbt-avro/basic_1.11/test

@@ -0,0 +1,42 @@
+> set avroSchemaParserBuilder := com.github.sbt.avro.LegacySchemaParserBuilder(validateDefaults = false)
+> avroGenerate
+
+$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/A.java
+$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/B.java
+$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/C.java
+$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/D.java
+$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/E.java
+$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/_A.java
+$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/_B.java
+$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/_C.java
+$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/_D.java
+$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/_E.java
+
+> compile
+
+$ exists target/scala-2.13/classes/com/github/sbt/avro/test/A.class
+$ exists target/scala-2.13/classes/com/github/sbt/avro/test/B.class
+$ exists target/scala-2.13/classes/com/github/sbt/avro/test/C.class
+$ exists target/scala-2.13/classes/com/github/sbt/avro/test/D.class
+$ exists target/scala-2.13/classes/com/github/sbt/avro/test/E.class
+$ exists target/scala-2.13/classes/com/github/sbt/avro/test/_A.class
+$ exists target/scala-2.13/classes/com/github/sbt/avro/test/_B.class
+$ exists target/scala-2.13/classes/com/github/sbt/avro/test/_C.class
+$ exists target/scala-2.13/classes/com/github/sbt/avro/test/_D.class
+$ exists target/scala-2.13/classes/com/github/sbt/avro/test/_E.class
+
+> Test/compile
+
+$ exists target/scala-2.13/src_managed/compiled_avro/test/com/github/sbt/avro/test/X.java
+$ exists target/scala-2.13/src_managed/compiled_avro/test/com/github/sbt/avro/test/Y.java
+$ exists target/scala-2.13/src_managed/compiled_avro/test/com/github/sbt/avro/test/Z.java
+$ exists target/scala-2.13/test-classes/com/github/sbt/avro/test/X.class
+$ exists target/scala-2.13/test-classes/com/github/sbt/avro/test/Y.class
+$ exists target/scala-2.13/test-classes/com/github/sbt/avro/test/Z.class
+
+> clean
+
+> set avroSchemaParserBuilder := com.github.sbt.avro.LegacySchemaParserBuilder(validateDefaults = true)
+
+# should fail because f.avsc has invalid default value
+-> avroGenerate

src/sbt-test/sbt-avro/basic_1.8/build.sbt

@@ -1,3 +1,6 @@
 name := "basic-test"
 scalaVersion := "2.13.11"
-libraryDependencies += "org.apache.avro" % "avro" % avroCompilerVersion
+libraryDependencies ++= Seq(
+  "org.apache.avro" % "avro" % avroCompilerVersion,
+  "joda-time" % "joda-time" % "2.7" // marked as optional in avro pom
+)

src/sbt-test/sbt-avro/basic_1.8/project/build.properties

@@ -0,0 +1 @@
+../../basic_current/project/build.properties

src/sbt-test/sbt-avro/basic_1.8/test

@@ -1 +1 @@
-../basic_current/test
+../basic_1.11/test

src/sbt-test/sbt-avro/basic_1.9/build.sbt

@@ -1,3 +1,6 @@
 name := "basic-test"
 scalaVersion := "2.13.11"
-libraryDependencies += "org.apache.avro" % "avro" % avroCompilerVersion
+libraryDependencies ++= Seq(
+  "org.apache.avro" % "avro" % avroCompilerVersion,
+  "joda-time" % "joda-time" % "2.10.1" // marked as optional in avro pom
+)

src/sbt-test/sbt-avro/basic_1.9/project/build.properties

@@ -0,0 +1 @@
+../../basic_current/project/build.properties

src/sbt-test/sbt-avro/basic_1.9/test

@@ -1 +1 @@
-../basic_current/test
+../basic_1.11/test

src/sbt-test/sbt-avro/basic_current/project/plugins.sbt

@@ -4,4 +4,4 @@ sys.props.get("plugin.version") match {
                          |Specify this property using the scriptedLaunchOpts -D.""".stripMargin)
 }
 
-libraryDependencies += "org.apache.avro" % "avro-compiler" % "1.11.3"
+libraryDependencies += "org.apache.avro" % "avro-compiler" % "1.12.0"

src/sbt-test/sbt-avro/basic_current/src/main/avro/logicalType.avsc

@@ -0,0 +1,38 @@
+{
+  "name": "LogicalTypesTest",
+  "namespace": "org.apache.parquet.avro",
+  "doc": "Record for testing logical types",
+  "type": "record",
+  "fields": [
+    {
+      "name": "timestamp",
+      "type": {
+        "type": "long",
+        "logicalType": "timestamp-millis"
+      }
+    },
+    {
+      "name": "local_date_time",
+      "type": {
+        "name": "LocalDateTimeTest",
+        "type": "record",
+        "fields": [
+          {
+            "name": "date",
+            "type": {
+              "type": "int",
+              "logicalType": "date"
+            }
+          },
+          {
+            "name": "time",
+            "type": {
+              "type": "int",
+              "logicalType": "time-millis"
+            }
+          }
+        ]
+      }
+    }
+  ]
+}

src/sbt-test/sbt-avro/basic_current/test

@@ -1,4 +1,4 @@
-> set avroSchemaParserBuilder := com.github.sbt.avro.DefaultSchemaParserBuilder.default().copy(validateDefaults = false)
+> set avroSchemaParserBuilder := com.github.sbt.avro.NameValidatorSchemaParserBuilder(validateDefaults = false)
 > avroGenerate
 
 $ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/A.java
@@ -25,7 +25,7 @@ $ exists target/scala-2.13/classes/com/github/sbt/avro/test/_C.class
 $ exists target/scala-2.13/classes/com/github/sbt/avro/test/_D.class
 $ exists target/scala-2.13/classes/com/github/sbt/avro/test/_E.class
 
-> test:compile
+> Test/compile
 
 $ exists target/scala-2.13/src_managed/compiled_avro/test/com/github/sbt/avro/test/X.java
 $ exists target/scala-2.13/src_managed/compiled_avro/test/com/github/sbt/avro/test/Y.java
@@ -36,7 +36,7 @@ $ exists target/scala-2.13/test-classes/com/github/sbt/avro/test/Z.class
 
 > clean
 
-> set avroSchemaParserBuilder := com.github.sbt.avro.DefaultSchemaParserBuilder.default().copy(validateDefaults = true)
+> set avroSchemaParserBuilder := com.github.sbt.avro.NameValidatorSchemaParserBuilder(validateDefaults = true)
 
 # should fail because f.avsc has invalid default value
 -> avroGenerate

src/sbt-test/sbt-avro/publishing/project/plugins.sbt

@@ -4,4 +4,4 @@ sys.props.get("plugin.version") match {
                          |Specify this property using the scriptedLaunchOpts -D.""".stripMargin)
 }
 
-libraryDependencies += "org.apache.avro" % "avro-compiler" % "1.11.3"
+libraryDependencies += "org.apache.avro" % "avro-compiler" % "1.12.0"