Allow extra avsc file includes (#36)

* Allow including avsc files By default include nothing. * Remove unused import * Prepare next snapshot * Test external type * Use single avsc schema builder Co-authored-by: Michel Davit <[email protected]>
sbt · Sep 16, 2020 · 8e129ca · 8e129ca
1 parent 2857336
commit 8e129ca
Show file tree

Hide file tree

Showing 5 changed files with 49 additions and 40 deletions.
diff --git a/README.md b/README.md
@@ -40,6 +40,7 @@ libraryDependencies += "org.apache.avro" % "avro" % "1.10.0"
 | `avroUnpackDependencies` / `target`  | `sourceManaged` / `avro`                   | Source directory for schemas packaged in the dependencies |
 | `avroGenerate` / `taget`             | `sourceManaged` / `compiled_avro`          | Source directory for generated `.java` files. |
 | `avroDependencyIncludeFilter`        | `source` typed `avro` classifier artifacts | Dependencies containing avro schema to be unpacked for generation |
+| `avroIncludes`                       | `Seq()`                                    | Paths with extra `*.avsc` files to be included in compilation. |
 | `packageAvro` / `artifactClassifier` | `Some("avro")`                             | Classifier for avro artifact |
 | `packageAvro` / `publishArtifact`    | `false`                                    | Enable / Disable avro artifact publishing |
 | `avroStringType`                     | `CharSequence`                             | Type for representing strings. Possible values: `CharSequence`, `String`, `Utf8`. |

diff --git a/build.sbt b/build.sbt
@@ -11,7 +11,7 @@ val CompileOnly = config("compileonly").hide
 ThisBuild / dynverSonatypeSnapshots := true
 ThisBuild / version := {
   val orig = (ThisBuild / version).value
-  if (orig.endsWith("-SNAPSHOT")) "3.0.1-SNAPSHOT"
+  if (orig.endsWith("-SNAPSHOT")) "3.1.0-SNAPSHOT"
   else orig
 }
 

diff --git a/src/main/scala/sbtavro/SbtAvro.scala b/src/main/scala/sbtavro/SbtAvro.scala
@@ -38,6 +38,7 @@ object SbtAvro extends AutoPlugin {
     val avroFieldVisibility = settingKey[String]("Field visibility for the properties. Possible values: private, public, public_deprecated. Default: public_deprecated.")
     val avroUseNamespace = settingKey[Boolean]("Validate that directory layout reflects namespaces, i.e. src/main/avro/com/myorg/MyRecord.avsc.")
     val avroSource = settingKey[File]("Default Avro source directory.")
+    val avroIncludes = settingKey[Seq[File]]("Avro schema includes.")
     val avroSchemaParserBuilder = settingKey[SchemaParserBuilder](".avsc schema parser builder")
     val avroUnpackDependencies = taskKey[Seq[File]]("Unpack avro dependencies.")
     val avroDependencyIncludeFilter = settingKey[DependencyFilter]("Filter for including modules containing avro dependencies.")
@@ -50,6 +51,7 @@ object SbtAvro extends AutoPlugin {
 
     lazy val defaultSettings: Seq[Setting[_]] = Seq(
       avroDependencyIncludeFilter := artifactFilter(`type` = Artifact.SourceType, classifier = AvroClassifier),
+      avroIncludes := Seq(),
       // addArtifact doesn't take publishArtifact setting in account
       artifacts ++= Classpaths.artifactDefs(avroArtifactTasks).value,
       packagedArtifacts ++= Classpaths.packaged(avroArtifactTasks).value,
@@ -133,18 +135,22 @@ object SbtAvro extends AutoPlugin {
     )
   }
 
-  def compileIdl(idl: File, target: File, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean) {
-    val parser = new Idl(idl)
-    val protocol = Protocol.parse(parser.CompilationUnit.toString)
-    val compiler = new SpecificCompiler(protocol)
-    compiler.setStringType(stringType)
-    compiler.setFieldVisibility(fieldVisibility)
-    compiler.setEnableDecimalLogicalType(enableDecimalLogicalType)
-    compiler.compileToDestination(null, target)
+  def compileIdls(idls: Seq[File], target: File, log: Logger, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean) = {
+    idls.foreach { idl =>
+      log.info(s"Compiling Avro IDL $idl")
+      val parser = new Idl(idl)
+      val protocol = Protocol.parse(parser.CompilationUnit.toString)
+      val compiler = new SpecificCompiler(protocol)
+      compiler.setStringType(stringType)
+      compiler.setFieldVisibility(fieldVisibility)
+      compiler.setEnableDecimalLogicalType(enableDecimalLogicalType)
+      compiler.compileToDestination(null, target)
+    }
   }
 
-  def compileAvscs(refs: Seq[AvroFileRef], target: File, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean, useNamespace: Boolean, builder: SchemaParserBuilder) {
+  def compileAvscs(refs: Seq[AvroFileRef], target: File, log: Logger, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean, useNamespace: Boolean, builder: SchemaParserBuilder) = {
     import com.spotify.avro.mojo._
+    import scala.collection.JavaConverters._
     val compiler = new AvscFilesCompiler(builder)
     compiler.setStringType(stringType)
     compiler.setFieldVisibility(fieldVisibility)
@@ -154,50 +160,49 @@ object SbtAvro extends AutoPlugin {
     compiler.setLogCompileExceptions(true)
     compiler.setTemplateDirectory("/org/apache/avro/compiler/specific/templates/java/classic/")
 
-    import scala.collection.JavaConverters._
+    refs.foreach { avsc =>
+      log.info(s"Compiling Avro schemas $avsc")
+    }
     compiler.compileFiles(refs.toSet.asJava, target)
   }
 
-  def compileAvpr(avpr: File, target: File, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean) {
-    val protocol = Protocol.parse(avpr)
-    val compiler = new SpecificCompiler(protocol)
-    compiler.setStringType(stringType)
-    compiler.setFieldVisibility(fieldVisibility)
-    compiler.setEnableDecimalLogicalType(enableDecimalLogicalType)
-    compiler.compileToDestination(null, target)
+  def compileAvprs(avprs: Seq[File], target: File, log: Logger, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean) = {
+    avprs.foreach { avpr =>
+      log.info(s"Compiling Avro protocol $avpr")
+      val protocol = Protocol.parse(avpr)
+      val compiler = new SpecificCompiler(protocol)
+      compiler.setStringType(stringType)
+      compiler.setFieldVisibility(fieldVisibility)
+      compiler.setEnableDecimalLogicalType(enableDecimalLogicalType)
+      compiler.compileToDestination(null, target)
+    }
   }
 
-  private[this] def compileAvroSchema(srcDir: File,
+  private[this] def compileAvroSchema(srcDirs: Seq[File],
                                       target: File,
                                       log: Logger,
                                       stringType: StringType,
                                       fieldVisibility: FieldVisibility,
                                       enableDecimalLogicalType: Boolean,
                                       useNamespace: Boolean,
                                       builder: SchemaParserBuilder): Set[File] = {
-    (srcDir ** AvroAvdlFilter).get.foreach { idl =>
-      log.info(s"Compiling Avro IDL $idl")
-      compileIdl(idl, target, stringType, fieldVisibility, enableDecimalLogicalType)
-    }
-
-    val avscs = (srcDir ** AvroAvscFilter).get.map { avsc =>
-      log.info(s"Compiling Avro schemas $avsc")
-      new AvroFileRef(srcDir, avsc.relativeTo(srcDir).get.toString)
-    }
-    compileAvscs(avscs, target, stringType, fieldVisibility, enableDecimalLogicalType, useNamespace, builder)
+    val avdls = srcDirs.flatMap(d => (d ** AvroAvdlFilter).get)
+    val avscs = srcDirs.flatMap(d => (d ** AvroAvscFilter).get.map(avsc => new AvroFileRef(d, avsc.relativeTo(d).get.toString)))
+    val avprs = srcDirs.flatMap(d => (d ** AvroAvrpFilter).get)
 
-    (srcDir ** AvroAvrpFilter).get.foreach { avpr =>
-      log.info(s"Compiling Avro protocol $avpr")
-      compileAvpr(avpr, target, stringType, fieldVisibility, enableDecimalLogicalType)
-    }
+    compileIdls(avdls, target, log, stringType, fieldVisibility, enableDecimalLogicalType)
+    compileAvscs(avscs, target, log, stringType, fieldVisibility, enableDecimalLogicalType, useNamespace, builder)
+    compileAvprs(avprs, target, log, stringType, fieldVisibility, enableDecimalLogicalType)
 
     (target ** JavaFileFilter).get.toSet
   }
 
   private def sourceGeneratorTask(key: TaskKey[Seq[File]]) = Def.task {
     val out = (key / streams).value
-    val externalSrcDir = (avroUnpackDependencies / target).value
     val srcDir = avroSource.value
+    val externalSrcDir = (avroUnpackDependencies / target).value
+    val includes = avroIncludes.value
+    val srcDirs = Seq(externalSrcDir, srcDir) ++ includes
     val outDir = (key / target).value
     val strType = StringType.valueOf(avroStringType.value)
     val fieldVis = SpecificCompiler.FieldVisibility.valueOf(avroFieldVisibility.value.toUpperCase)
@@ -207,13 +212,11 @@ object SbtAvro extends AutoPlugin {
     val cachedCompile = {
       FileFunction.cached(out.cacheDirectory / "avro", FilesInfo.lastModified, FilesInfo.exists) { _ =>
         out.log.info(s"Avro compiler using stringType=$strType")
-        compileAvroSchema(externalSrcDir, outDir, out.log, strType, fieldVis, enbDecimal, useNs, builder)
-        compileAvroSchema(srcDir, outDir, out.log, strType, fieldVis, enbDecimal, useNs, builder)
-
+        compileAvroSchema(srcDirs, outDir, out.log, strType, fieldVis, enbDecimal, useNs, builder)
       }
     }
 
-    cachedCompile(((externalSrcDir +++ srcDir) ** AvroFilter).get.toSet).toSeq
+    cachedCompile((srcDirs ** AvroFilter).get.toSet).toSeq
   }
 
 }
diff --git a/src/sbt-test/sbt-avro/publishing/transitive/src/main/avro/com/cavorite/transitive/avsc.avsc b/src/sbt-test/sbt-avro/publishing/transitive/src/main/avro/com/cavorite/transitive/avsc.avsc
@@ -6,6 +6,10 @@
     {
       "name": "stringField",
       "type": "string"
+    },
+    {
+      "name": "referencedTypeField",
+      "type": "com.cavorite.external.Avsc"
     }
   ]
 }
diff --git a/src/test/scala/sbtavro/SbtAvroSpec.scala b/src/test/scala/sbtavro/SbtAvroSpec.scala
@@ -1,12 +1,12 @@
 package sbtavro
 
 import java.io.File
-import java.util.Collections
 
 import com.spotify.avro.mojo.AvroFileRef
 import org.apache.avro.compiler.specific.SpecificCompiler.FieldVisibility
 import org.apache.avro.generic.GenericData.StringType
 import org.specs2.mutable.Specification
+import sbt.util.Logger
 
 /**
   * Created by jeromewacongne on 06/08/2015.
@@ -15,6 +15,7 @@ class SbtAvroSpec extends Specification {
   val builder = DefaultSchemaParserBuilder.default()
   val sourceDir = new File(getClass.getClassLoader.getResource("avro").toURI)
   val targetDir = new File(sourceDir.getParentFile, "generated")
+  val logger = Logger.Null
 
   val fullyQualifiedNames = Seq(
     new File(sourceDir, "a.avsc"),
@@ -74,7 +75,7 @@ class SbtAvroSpec extends Specification {
     _eJavaFile.delete()
 
     val refs = sourceFiles.map(s => new AvroFileRef(sourceDir, s.getName))
-    SbtAvro.compileAvscs(refs, targetDir, StringType.CharSequence, FieldVisibility.PUBLIC_DEPRECATED, true, false, builder)
+    SbtAvro.compileAvscs(refs, targetDir, logger, StringType.CharSequence, FieldVisibility.PUBLIC_DEPRECATED, true, false, builder)
 
     aJavaFile.isFile must beTrue
     bJavaFile.isFile must beTrue