From 8e129ca97ff29c75d46bbc968bdaef80dd841b07 Mon Sep 17 00:00:00 2001 From: Oskar Jung Date: Wed, 16 Sep 2020 18:07:06 +0200 Subject: [PATCH] Allow extra `avsc` file includes (#36) * Allow including avsc files By default include nothing. * Remove unused import * Prepare next snapshot * Test external type * Use single avsc schema builder Co-authored-by: Michel Davit --- README.md | 1 + build.sbt | 2 +- src/main/scala/sbtavro/SbtAvro.scala | 77 ++++++++++--------- .../avro/com/cavorite/transitive/avsc.avsc | 4 + src/test/scala/sbtavro/SbtAvroSpec.scala | 5 +- 5 files changed, 49 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 5db744c..da04607 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ libraryDependencies += "org.apache.avro" % "avro" % "1.10.0" | `avroUnpackDependencies` / `target` | `sourceManaged` / `avro` | Source directory for schemas packaged in the dependencies | | `avroGenerate` / `taget` | `sourceManaged` / `compiled_avro` | Source directory for generated `.java` files. | | `avroDependencyIncludeFilter` | `source` typed `avro` classifier artifacts | Dependencies containing avro schema to be unpacked for generation | +| `avroIncludes` | `Seq()` | Paths with extra `*.avsc` files to be included in compilation. | | `packageAvro` / `artifactClassifier` | `Some("avro")` | Classifier for avro artifact | | `packageAvro` / `publishArtifact` | `false` | Enable / Disable avro artifact publishing | | `avroStringType` | `CharSequence` | Type for representing strings. Possible values: `CharSequence`, `String`, `Utf8`. | diff --git a/build.sbt b/build.sbt index b26b470..b395422 100644 --- a/build.sbt +++ b/build.sbt @@ -11,7 +11,7 @@ val CompileOnly = config("compileonly").hide ThisBuild / dynverSonatypeSnapshots := true ThisBuild / version := { val orig = (ThisBuild / version).value - if (orig.endsWith("-SNAPSHOT")) "3.0.1-SNAPSHOT" + if (orig.endsWith("-SNAPSHOT")) "3.1.0-SNAPSHOT" else orig } diff --git a/src/main/scala/sbtavro/SbtAvro.scala b/src/main/scala/sbtavro/SbtAvro.scala index 46ab8b1..55d1059 100644 --- a/src/main/scala/sbtavro/SbtAvro.scala +++ b/src/main/scala/sbtavro/SbtAvro.scala @@ -38,6 +38,7 @@ object SbtAvro extends AutoPlugin { val avroFieldVisibility = settingKey[String]("Field visibility for the properties. Possible values: private, public, public_deprecated. Default: public_deprecated.") val avroUseNamespace = settingKey[Boolean]("Validate that directory layout reflects namespaces, i.e. src/main/avro/com/myorg/MyRecord.avsc.") val avroSource = settingKey[File]("Default Avro source directory.") + val avroIncludes = settingKey[Seq[File]]("Avro schema includes.") val avroSchemaParserBuilder = settingKey[SchemaParserBuilder](".avsc schema parser builder") val avroUnpackDependencies = taskKey[Seq[File]]("Unpack avro dependencies.") val avroDependencyIncludeFilter = settingKey[DependencyFilter]("Filter for including modules containing avro dependencies.") @@ -50,6 +51,7 @@ object SbtAvro extends AutoPlugin { lazy val defaultSettings: Seq[Setting[_]] = Seq( avroDependencyIncludeFilter := artifactFilter(`type` = Artifact.SourceType, classifier = AvroClassifier), + avroIncludes := Seq(), // addArtifact doesn't take publishArtifact setting in account artifacts ++= Classpaths.artifactDefs(avroArtifactTasks).value, packagedArtifacts ++= Classpaths.packaged(avroArtifactTasks).value, @@ -133,18 +135,22 @@ object SbtAvro extends AutoPlugin { ) } - def compileIdl(idl: File, target: File, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean) { - val parser = new Idl(idl) - val protocol = Protocol.parse(parser.CompilationUnit.toString) - val compiler = new SpecificCompiler(protocol) - compiler.setStringType(stringType) - compiler.setFieldVisibility(fieldVisibility) - compiler.setEnableDecimalLogicalType(enableDecimalLogicalType) - compiler.compileToDestination(null, target) + def compileIdls(idls: Seq[File], target: File, log: Logger, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean) = { + idls.foreach { idl => + log.info(s"Compiling Avro IDL $idl") + val parser = new Idl(idl) + val protocol = Protocol.parse(parser.CompilationUnit.toString) + val compiler = new SpecificCompiler(protocol) + compiler.setStringType(stringType) + compiler.setFieldVisibility(fieldVisibility) + compiler.setEnableDecimalLogicalType(enableDecimalLogicalType) + compiler.compileToDestination(null, target) + } } - def compileAvscs(refs: Seq[AvroFileRef], target: File, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean, useNamespace: Boolean, builder: SchemaParserBuilder) { + def compileAvscs(refs: Seq[AvroFileRef], target: File, log: Logger, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean, useNamespace: Boolean, builder: SchemaParserBuilder) = { import com.spotify.avro.mojo._ + import scala.collection.JavaConverters._ val compiler = new AvscFilesCompiler(builder) compiler.setStringType(stringType) compiler.setFieldVisibility(fieldVisibility) @@ -154,20 +160,25 @@ object SbtAvro extends AutoPlugin { compiler.setLogCompileExceptions(true) compiler.setTemplateDirectory("/org/apache/avro/compiler/specific/templates/java/classic/") - import scala.collection.JavaConverters._ + refs.foreach { avsc => + log.info(s"Compiling Avro schemas $avsc") + } compiler.compileFiles(refs.toSet.asJava, target) } - def compileAvpr(avpr: File, target: File, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean) { - val protocol = Protocol.parse(avpr) - val compiler = new SpecificCompiler(protocol) - compiler.setStringType(stringType) - compiler.setFieldVisibility(fieldVisibility) - compiler.setEnableDecimalLogicalType(enableDecimalLogicalType) - compiler.compileToDestination(null, target) + def compileAvprs(avprs: Seq[File], target: File, log: Logger, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean) = { + avprs.foreach { avpr => + log.info(s"Compiling Avro protocol $avpr") + val protocol = Protocol.parse(avpr) + val compiler = new SpecificCompiler(protocol) + compiler.setStringType(stringType) + compiler.setFieldVisibility(fieldVisibility) + compiler.setEnableDecimalLogicalType(enableDecimalLogicalType) + compiler.compileToDestination(null, target) + } } - private[this] def compileAvroSchema(srcDir: File, + private[this] def compileAvroSchema(srcDirs: Seq[File], target: File, log: Logger, stringType: StringType, @@ -175,29 +186,23 @@ object SbtAvro extends AutoPlugin { enableDecimalLogicalType: Boolean, useNamespace: Boolean, builder: SchemaParserBuilder): Set[File] = { - (srcDir ** AvroAvdlFilter).get.foreach { idl => - log.info(s"Compiling Avro IDL $idl") - compileIdl(idl, target, stringType, fieldVisibility, enableDecimalLogicalType) - } - - val avscs = (srcDir ** AvroAvscFilter).get.map { avsc => - log.info(s"Compiling Avro schemas $avsc") - new AvroFileRef(srcDir, avsc.relativeTo(srcDir).get.toString) - } - compileAvscs(avscs, target, stringType, fieldVisibility, enableDecimalLogicalType, useNamespace, builder) + val avdls = srcDirs.flatMap(d => (d ** AvroAvdlFilter).get) + val avscs = srcDirs.flatMap(d => (d ** AvroAvscFilter).get.map(avsc => new AvroFileRef(d, avsc.relativeTo(d).get.toString))) + val avprs = srcDirs.flatMap(d => (d ** AvroAvrpFilter).get) - (srcDir ** AvroAvrpFilter).get.foreach { avpr => - log.info(s"Compiling Avro protocol $avpr") - compileAvpr(avpr, target, stringType, fieldVisibility, enableDecimalLogicalType) - } + compileIdls(avdls, target, log, stringType, fieldVisibility, enableDecimalLogicalType) + compileAvscs(avscs, target, log, stringType, fieldVisibility, enableDecimalLogicalType, useNamespace, builder) + compileAvprs(avprs, target, log, stringType, fieldVisibility, enableDecimalLogicalType) (target ** JavaFileFilter).get.toSet } private def sourceGeneratorTask(key: TaskKey[Seq[File]]) = Def.task { val out = (key / streams).value - val externalSrcDir = (avroUnpackDependencies / target).value val srcDir = avroSource.value + val externalSrcDir = (avroUnpackDependencies / target).value + val includes = avroIncludes.value + val srcDirs = Seq(externalSrcDir, srcDir) ++ includes val outDir = (key / target).value val strType = StringType.valueOf(avroStringType.value) val fieldVis = SpecificCompiler.FieldVisibility.valueOf(avroFieldVisibility.value.toUpperCase) @@ -207,13 +212,11 @@ object SbtAvro extends AutoPlugin { val cachedCompile = { FileFunction.cached(out.cacheDirectory / "avro", FilesInfo.lastModified, FilesInfo.exists) { _ => out.log.info(s"Avro compiler using stringType=$strType") - compileAvroSchema(externalSrcDir, outDir, out.log, strType, fieldVis, enbDecimal, useNs, builder) - compileAvroSchema(srcDir, outDir, out.log, strType, fieldVis, enbDecimal, useNs, builder) - + compileAvroSchema(srcDirs, outDir, out.log, strType, fieldVis, enbDecimal, useNs, builder) } } - cachedCompile(((externalSrcDir +++ srcDir) ** AvroFilter).get.toSet).toSeq + cachedCompile((srcDirs ** AvroFilter).get.toSet).toSeq } } diff --git a/src/sbt-test/sbt-avro/publishing/transitive/src/main/avro/com/cavorite/transitive/avsc.avsc b/src/sbt-test/sbt-avro/publishing/transitive/src/main/avro/com/cavorite/transitive/avsc.avsc index 26b5978..1f2304c 100644 --- a/src/sbt-test/sbt-avro/publishing/transitive/src/main/avro/com/cavorite/transitive/avsc.avsc +++ b/src/sbt-test/sbt-avro/publishing/transitive/src/main/avro/com/cavorite/transitive/avsc.avsc @@ -6,6 +6,10 @@ { "name": "stringField", "type": "string" + }, + { + "name": "referencedTypeField", + "type": "com.cavorite.external.Avsc" } ] } \ No newline at end of file diff --git a/src/test/scala/sbtavro/SbtAvroSpec.scala b/src/test/scala/sbtavro/SbtAvroSpec.scala index c83e956..4ece567 100644 --- a/src/test/scala/sbtavro/SbtAvroSpec.scala +++ b/src/test/scala/sbtavro/SbtAvroSpec.scala @@ -1,12 +1,12 @@ package sbtavro import java.io.File -import java.util.Collections import com.spotify.avro.mojo.AvroFileRef import org.apache.avro.compiler.specific.SpecificCompiler.FieldVisibility import org.apache.avro.generic.GenericData.StringType import org.specs2.mutable.Specification +import sbt.util.Logger /** * Created by jeromewacongne on 06/08/2015. @@ -15,6 +15,7 @@ class SbtAvroSpec extends Specification { val builder = DefaultSchemaParserBuilder.default() val sourceDir = new File(getClass.getClassLoader.getResource("avro").toURI) val targetDir = new File(sourceDir.getParentFile, "generated") + val logger = Logger.Null val fullyQualifiedNames = Seq( new File(sourceDir, "a.avsc"), @@ -74,7 +75,7 @@ class SbtAvroSpec extends Specification { _eJavaFile.delete() val refs = sourceFiles.map(s => new AvroFileRef(sourceDir, s.getName)) - SbtAvro.compileAvscs(refs, targetDir, StringType.CharSequence, FieldVisibility.PUBLIC_DEPRECATED, true, false, builder) + SbtAvro.compileAvscs(refs, targetDir, logger, StringType.CharSequence, FieldVisibility.PUBLIC_DEPRECATED, true, false, builder) aJavaFile.isFile must beTrue bJavaFile.isFile must beTrue