diff --git a/lang/java/archetypes/avro-service-archetype/src/main/pom/pom.xml b/lang/java/archetypes/avro-service-archetype/src/main/pom/pom.xml index 4c9558c40b9..069a367b973 100644 --- a/lang/java/archetypes/avro-service-archetype/src/main/pom/pom.xml +++ b/lang/java/archetypes/avro-service-archetype/src/main/pom/pom.xml @@ -33,8 +33,9 @@ Simple Avro Ordering Service - 1.8 - 1.8 + ${maven.compiler.source} + ${maven.compiler.target} + ${project.build.sourceEncoding} ${project.version} ${jackson-bom.version} ${junit.version} diff --git a/lang/java/compiler/pom.xml b/lang/java/compiler/pom.xml index 53719816387..3195905c03e 100644 --- a/lang/java/compiler/pom.xml +++ b/lang/java/compiler/pom.xml @@ -182,37 +182,8 @@ - - - - org.eclipse.m2e - lifecycle-mapping - 1.0.0 - - - - - - org.codehaus.mojo - exec-maven-plugin - [1.0,) - - exec - - - - - - - - - - - - - ${project.groupId} @@ -241,4 +212,42 @@ + + + m2e + + m2e.version + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + org.codehaus.mojo + exec-maven-plugin + [1.0,) + + exec + + + + + + + + + + + + + + + diff --git a/lang/java/idl/pom.xml b/lang/java/idl/pom.xml index 9ace6dc750f..6363ec150a5 100644 --- a/lang/java/idl/pom.xml +++ b/lang/java/idl/pom.xml @@ -107,37 +107,8 @@ - - - - org.eclipse.m2e - lifecycle-mapping - 1.0.0 - - - - - - org.codehaus.mojo - exec-maven-plugin - [1.0,) - - exec - - - - - - - - - - - - - ${project.groupId} @@ -159,4 +130,43 @@ jackson-databind + + + + m2e + + m2e.version + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + org.codehaus.mojo + exec-maven-plugin + [1.0,) + + exec + + + + + + + + + + + + + + + diff --git a/lang/java/idl/src/main/java/org/apache/avro/idl/IdlFile.java b/lang/java/idl/src/main/java/org/apache/avro/idl/IdlFile.java index 3cbb14807cf..56627b5821b 100644 --- a/lang/java/idl/src/main/java/org/apache/avro/idl/IdlFile.java +++ b/lang/java/idl/src/main/java/org/apache/avro/idl/IdlFile.java @@ -64,7 +64,7 @@ public List getWarnings() { public List getWarnings(String importFile) { return warnings.stream() - .map(warning -> importFile + " " + Character.toLowerCase(warning.charAt(0)) + warning.substring(1)) + .map(warning -> importFile + ' ' + Character.toLowerCase(warning.charAt(0)) + warning.substring(1)) .collect(Collectors.toList()); } @@ -96,7 +96,7 @@ public Schema getNamedSchema(String name) { return result; } if (namespace != null && !name.contains(".")) { - result = namedSchemas.get(namespace + "." + name); + result = namedSchemas.get(namespace + '.' + name); } return result; } @@ -111,9 +111,9 @@ String outputString() { } else { StringBuilder buffer = new StringBuilder(); for (Schema schema : namedSchemas.values()) { - buffer.append(",").append(schema); + buffer.append(',').append(schema); } - buffer.append("]").setCharAt(0, '['); + buffer.append(']').setCharAt(0, '['); return buffer.toString(); } } diff --git a/lang/java/idl/src/main/java/org/apache/avro/idl/Schemas.java b/lang/java/idl/src/main/java/org/apache/avro/idl/Schemas.java index 9b2ba7f21c5..da4b949d2bc 100644 --- a/lang/java/idl/src/main/java/org/apache/avro/idl/Schemas.java +++ b/lang/java/idl/src/main/java/org/apache/avro/idl/Schemas.java @@ -56,8 +56,6 @@ public static T visit(final Schema start, final SchemaVisitor visitor) { switch (action) { case CONTINUE: break; - case SKIP_SUBTREE: - throw new UnsupportedOperationException(); case SKIP_SIBLINGS: while (dq.peek() instanceof Schema) { dq.remove(); @@ -65,6 +63,7 @@ public static T visit(final Schema start, final SchemaVisitor visitor) { break; case TERMINATE: return visitor.get(); + case SKIP_SUBTREE: default: throw new UnsupportedOperationException("Invalid action " + action); } diff --git a/lang/java/idl/src/test/idl/input/forward_ref.avdl b/lang/java/idl/src/test/idl/input/forward_ref.avdl index e9317fec0a2..b75d60a4efd 100644 --- a/lang/java/idl/src/test/idl/input/forward_ref.avdl +++ b/lang/java/idl/src/test/idl/input/forward_ref.avdl @@ -15,17 +15,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - @namespace("org.foo") protocol Import { -/* Name Value record */ -record ANameValue { - /** the name */ - string name; - /** the value */ - string value; - /* is the value a json object */ - ValueType type = "PLAIN"; -} -enum ValueType {JSON, BASE64BIN, PLAIN} + /* Name Value record */ + record ANameValue { + /** the name */ + string name; + /** the value */ + string value; + /* is the value a json object */ + ValueType type = "PLAIN"; + } + + enum ValueType { + JSON, BASE64BIN, PLAIN + } } diff --git a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/AbstractAvroMojo.java b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/AbstractAvroMojo.java index d0407eb7468..a429a21e1f2 100644 --- a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/AbstractAvroMojo.java +++ b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/AbstractAvroMojo.java @@ -18,6 +18,18 @@ package org.apache.avro.mojo; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Protocol; +import org.apache.avro.Schema; +import org.apache.avro.compiler.specific.SpecificCompiler; +import org.apache.avro.generic.GenericData; +import org.apache.maven.artifact.DependencyResolutionRequiredException; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.project.MavenProject; +import org.apache.maven.shared.model.fileset.FileSet; +import org.apache.maven.shared.model.fileset.util.FileSetManager; + import java.io.File; import java.io.IOException; import java.lang.reflect.InvocationTargetException; @@ -28,15 +40,6 @@ import java.util.Arrays; import java.util.List; -import org.apache.avro.LogicalTypes; -import org.apache.avro.compiler.specific.SpecificCompiler; -import org.apache.maven.artifact.DependencyResolutionRequiredException; -import org.apache.maven.plugin.AbstractMojo; -import org.apache.maven.plugin.MojoExecutionException; -import org.apache.maven.project.MavenProject; -import org.apache.maven.shared.model.fileset.FileSet; -import org.apache.maven.shared.model.fileset.util.FileSetManager; - /** * Base for Avro Compiler Mojos. */ @@ -274,14 +277,23 @@ private String[] getIncludedFiles(String absPath, String[] excludes, String[] in } private void compileFiles(String[] files, File sourceDir, File outDir) throws MojoExecutionException { - for (String filename : files) { - try { - // Need to register custom logical type factories before schema compilation. - loadLogicalTypesFactories(); - doCompile(filename, sourceDir, outDir); - } catch (IOException e) { - throw new MojoExecutionException("Error compiling protocol file " + filename + " to " + outDir, e); + final ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(createClassLoader()); + + for (String filename : files) { + try { + // Need to register custom logical type factories before schema compilation. + loadLogicalTypesFactories(); + doCompile(filename, sourceDir, outDir); + } catch (IOException e) { + throw new MojoExecutionException("Error compiling protocol file " + filename + " to " + outDir, e); + } } + } catch (MalformedURLException | DependencyResolutionRequiredException e) { + throw new MojoExecutionException("Cannot locate classpath entries", e); + } finally { + Thread.currentThread().setContextClassLoader(contextClassLoader); } } @@ -314,7 +326,7 @@ protected List instantiateAdditionalVelocityTools() { final List velocityTools = new ArrayList<>(velocityToolsClassesNames.length); for (String velocityToolClassName : velocityToolsClassesNames) { try { - Class klass = Class.forName(velocityToolClassName); + Class klass = Class.forName(velocityToolClassName); velocityTools.add(klass.getDeclaredConstructor().newInstance()); } catch (Exception e) { throw new RuntimeException(e); @@ -325,20 +337,57 @@ protected List instantiateAdditionalVelocityTools() { protected abstract void doCompile(String filename, File sourceDirectory, File outputDirectory) throws IOException; - protected URLClassLoader createClassLoader() throws DependencyResolutionRequiredException, MalformedURLException { + protected void doCompile(File sourceFileForModificationDetection, Schema schema, File outputDirectory) + throws IOException { + doCompile(sourceFileForModificationDetection, new SpecificCompiler(schema), outputDirectory); + } + + protected void doCompile(File sourceFileForModificationDetection, Protocol protocol, File outputDirectory) + throws IOException { + doCompile(sourceFileForModificationDetection, new SpecificCompiler(protocol), outputDirectory); + } + + private void doCompile(File sourceFileForModificationDetection, SpecificCompiler compiler, File outputDirectory) + throws IOException { + compiler.setTemplateDir(templateDirectory); + compiler.setStringType(GenericData.StringType.valueOf(stringType)); + compiler.setFieldVisibility(getFieldVisibility()); + compiler.setCreateOptionalGetters(createOptionalGetters); + compiler.setGettersReturnOptional(gettersReturnOptional); + compiler.setOptionalGettersForNullableFieldsOnly(optionalGettersForNullableFieldsOnly); + compiler.setCreateSetters(createSetters); + compiler.setEnableDecimalLogicalType(enableDecimalLogicalType); + try { + final URLClassLoader classLoader = createClassLoader(); + for (String customConversion : customConversions) { + compiler.addCustomConversion(classLoader.loadClass(customConversion)); + } + } catch (ClassNotFoundException | DependencyResolutionRequiredException e) { + throw new IOException(e); + } + compiler.setOutputCharacterEncoding(project.getProperties().getProperty("project.build.sourceEncoding")); + compiler.setAdditionalVelocityTools(instantiateAdditionalVelocityTools()); + compiler.compileToDestination(sourceFileForModificationDetection, outputDirectory); + } + + protected List findClasspath() throws DependencyResolutionRequiredException, MalformedURLException { final List urls = appendElements(project.getRuntimeClasspathElements()); urls.addAll(appendElements(project.getTestClasspathElements())); + return urls; + } + + protected URLClassLoader createClassLoader() throws DependencyResolutionRequiredException, MalformedURLException { + final List urls = findClasspath(); return new URLClassLoader(urls.toArray(new URL[0]), Thread.currentThread().getContextClassLoader()); } - private List appendElements(List runtimeClasspathElements) throws MalformedURLException { + private List appendElements(List runtimeClasspathElements) throws MalformedURLException { if (runtimeClasspathElements == null) { return new ArrayList<>(); } List runtimeUrls = new ArrayList<>(runtimeClasspathElements.size()); - for (Object runtimeClasspathElement : runtimeClasspathElements) { - String element = (String) runtimeClasspathElement; - runtimeUrls.add(new File(element).toURI().toURL()); + for (String runtimeClasspathElement : runtimeClasspathElements) { + runtimeUrls.add(new File(runtimeClasspathElement).toURI().toURL()); } return runtimeUrls; } diff --git a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/IDLProtocolMojo.java b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/IDLProtocolMojo.java index 0543811400d..eceaff88ded 100644 --- a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/IDLProtocolMojo.java +++ b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/IDLProtocolMojo.java @@ -18,20 +18,16 @@ package org.apache.avro.mojo; +import org.apache.avro.Protocol; +import org.apache.avro.compiler.idl.Idl; +import org.apache.avro.compiler.idl.ParseException; +import org.apache.avro.idl.IdlFile; +import org.apache.avro.idl.IdlReader; + import java.io.File; import java.io.IOException; import java.net.URL; import java.net.URLClassLoader; -import java.util.ArrayList; -import java.util.List; - -import org.apache.avro.Protocol; -import org.apache.avro.compiler.specific.SpecificCompiler; -import org.apache.avro.generic.GenericData; - -import org.apache.avro.idl.IdlFile; -import org.apache.avro.idl.IdlReader; -import org.apache.maven.artifact.DependencyResolutionRequiredException; /** * Generate Java classes and interfaces from AvroIDL files (.avdl) @@ -42,6 +38,14 @@ * @threadSafe */ public class IDLProtocolMojo extends AbstractAvroMojo { + /** + * Use the classic JavaCC parser for .avdl files. If + * false (the default), use the new ANTLR parser instead. + * + * @parameter + */ + private boolean useJavaCC = false; + /** * A set of Ant-like inclusion patterns used to select files from the source * directory for processing. By default, the pattern **/*.avdl @@ -62,55 +66,37 @@ public class IDLProtocolMojo extends AbstractAvroMojo { @Override protected void doCompile(String filename, File sourceDirectory, File outputDirectory) throws IOException { - try { - @SuppressWarnings("rawtypes") - List runtimeClasspathElements = project.getRuntimeClasspathElements(); - - List runtimeUrls = new ArrayList<>(); + File sourceFile = new File(sourceDirectory, filename); - // Add the source directory of avro files to the classpath so that - // imports can refer to other idl files as classpath resources - runtimeUrls.add(sourceDirectory.toURI().toURL()); + ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); + URL[] extraClasspath = new URL[] { sourceDirectory.toURI().toURL() }; + ClassLoader classLoader = new URLClassLoader(extraClasspath, contextClassLoader); - // If runtimeClasspathElements is not empty values add its values to Idl path. - if (runtimeClasspathElements != null && !runtimeClasspathElements.isEmpty()) { - for (Object runtimeClasspathElement : runtimeClasspathElements) { - String element = (String) runtimeClasspathElement; - runtimeUrls.add(new File(element).toURI().toURL()); - } + Protocol protocol; + if (useJavaCC) { + try (Idl idl = new Idl(sourceFile, classLoader)) { + final Protocol p = idl.CompilationUnit(); + String json = p.toString(true); + protocol = Protocol.parse(json); + } catch (ParseException e) { + throw new IOException(e); } - - final ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); - URLClassLoader projPathLoader = new URLClassLoader(runtimeUrls.toArray(new URL[0]), contextClassLoader); - Thread.currentThread().setContextClassLoader(projPathLoader); + } else { try { + Thread.currentThread().setContextClassLoader(classLoader); + IdlReader parser = new IdlReader(); - IdlFile idlFile = parser.parse(sourceDirectory.toPath().resolve(filename)); + IdlFile idlFile = parser.parse(sourceFile.toPath()); for (String warning : idlFile.getWarnings()) { getLog().warn(warning); } - final Protocol protocol = idlFile.getProtocol(); - final SpecificCompiler compiler = new SpecificCompiler(protocol); - compiler.setStringType(GenericData.StringType.valueOf(stringType)); - compiler.setTemplateDir(templateDirectory); - compiler.setFieldVisibility(getFieldVisibility()); - compiler.setCreateOptionalGetters(createOptionalGetters); - compiler.setGettersReturnOptional(gettersReturnOptional); - compiler.setOptionalGettersForNullableFieldsOnly(optionalGettersForNullableFieldsOnly); - compiler.setCreateSetters(createSetters); - compiler.setAdditionalVelocityTools(instantiateAdditionalVelocityTools()); - compiler.setEnableDecimalLogicalType(enableDecimalLogicalType); - for (String customConversion : customConversions) { - compiler.addCustomConversion(projPathLoader.loadClass(customConversion)); - } - compiler.setOutputCharacterEncoding(project.getProperties().getProperty("project.build.sourceEncoding")); - compiler.compileToDestination(null, outputDirectory); + protocol = idlFile.getProtocol(); } finally { Thread.currentThread().setContextClassLoader(contextClassLoader); } - } catch (ClassNotFoundException | DependencyResolutionRequiredException e) { - throw new IOException(e); } + + doCompile(sourceFile, protocol, outputDirectory); } @Override diff --git a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/ProtocolMojo.java b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/ProtocolMojo.java index c5e406fc70d..ee7e4101c5d 100644 --- a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/ProtocolMojo.java +++ b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/ProtocolMojo.java @@ -18,15 +18,10 @@ package org.apache.avro.mojo; -import org.apache.avro.generic.GenericData.StringType; +import org.apache.avro.Protocol; import java.io.File; import java.io.IOException; -import java.net.URLClassLoader; - -import org.apache.avro.Protocol; -import org.apache.avro.compiler.specific.SpecificCompiler; -import org.apache.maven.artifact.DependencyResolutionRequiredException; /** * Generate Java classes and interfaces from Avro protocol files (.avpr) @@ -59,27 +54,7 @@ public class ProtocolMojo extends AbstractAvroMojo { protected void doCompile(String filename, File sourceDirectory, File outputDirectory) throws IOException { final File src = new File(sourceDirectory, filename); final Protocol protocol = Protocol.parse(src); - final SpecificCompiler compiler = new SpecificCompiler(protocol); - compiler.setTemplateDir(templateDirectory); - compiler.setStringType(StringType.valueOf(stringType)); - compiler.setFieldVisibility(getFieldVisibility()); - compiler.setCreateOptionalGetters(createOptionalGetters); - compiler.setGettersReturnOptional(gettersReturnOptional); - compiler.setOptionalGettersForNullableFieldsOnly(optionalGettersForNullableFieldsOnly); - compiler.setCreateSetters(createSetters); - compiler.setAdditionalVelocityTools(instantiateAdditionalVelocityTools()); - compiler.setEnableDecimalLogicalType(enableDecimalLogicalType); - final URLClassLoader classLoader; - try { - classLoader = createClassLoader(); - for (String customConversion : customConversions) { - compiler.addCustomConversion(classLoader.loadClass(customConversion)); - } - } catch (DependencyResolutionRequiredException | ClassNotFoundException e) { - throw new IOException(e); - } - compiler.setOutputCharacterEncoding(project.getProperties().getProperty("project.build.sourceEncoding")); - compiler.compileToDestination(src, outputDirectory); + doCompile(src, protocol, outputDirectory); } @Override diff --git a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/SchemaMojo.java b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/SchemaMojo.java index dd2ede2c12d..d8d04bf7234 100644 --- a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/SchemaMojo.java +++ b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/SchemaMojo.java @@ -18,15 +18,10 @@ package org.apache.avro.mojo; -import org.apache.avro.generic.GenericData.StringType; +import org.apache.avro.Schema; import java.io.File; import java.io.IOException; -import java.net.URLClassLoader; - -import org.apache.avro.Schema; -import org.apache.avro.compiler.specific.SpecificCompiler; -import org.apache.maven.artifact.DependencyResolutionRequiredException; /** * Generate Java classes from Avro schema files (.avsc) @@ -76,26 +71,7 @@ protected void doCompile(String filename, File sourceDirectory, File outputDirec schema = schemaParser.parse(src); } - final SpecificCompiler compiler = new SpecificCompiler(schema); - compiler.setTemplateDir(templateDirectory); - compiler.setStringType(StringType.valueOf(stringType)); - compiler.setFieldVisibility(getFieldVisibility()); - compiler.setCreateOptionalGetters(createOptionalGetters); - compiler.setGettersReturnOptional(gettersReturnOptional); - compiler.setOptionalGettersForNullableFieldsOnly(optionalGettersForNullableFieldsOnly); - compiler.setCreateSetters(createSetters); - compiler.setEnableDecimalLogicalType(enableDecimalLogicalType); - try { - final URLClassLoader classLoader = createClassLoader(); - for (String customConversion : customConversions) { - compiler.addCustomConversion(classLoader.loadClass(customConversion)); - } - } catch (ClassNotFoundException | DependencyResolutionRequiredException e) { - throw new IOException(e); - } - compiler.setOutputCharacterEncoding(project.getProperties().getProperty("project.build.sourceEncoding")); - compiler.setAdditionalVelocityTools(instantiateAdditionalVelocityTools()); - compiler.compileToDestination(src, outputDirectory); + doCompile(src, schema, outputDirectory); } @Override diff --git a/lang/java/maven-plugin/src/test/resources/unit/idl/pom-javacc.xml b/lang/java/maven-plugin/src/test/resources/unit/idl/pom-javacc.xml new file mode 100644 index 00000000000..4abd67f7bca --- /dev/null +++ b/lang/java/maven-plugin/src/test/resources/unit/idl/pom-javacc.xml @@ -0,0 +1,68 @@ + + + + 4.0.0 + + + avro-parent + org.apache.avro + 1.11.0-SNAPSHOT + ../../../../../../../../../pom.xml + + + avro-maven-plugin-test + jar + + testproject + + + + + avro-maven-plugin + + + idl + + idl-protocol + + + + + true + ${basedir}/src/test + ${basedir}/target/test-harness/idl + String + + + + + + + + org.apache.avro + avro + ${parent.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + diff --git a/lang/java/tools/src/main/java/org/apache/avro/tool/IdlToSchemataTool.java b/lang/java/tools/src/main/java/org/apache/avro/tool/IdlToSchemataTool.java index b1f671a6455..6e0901efed1 100644 --- a/lang/java/tools/src/main/java/org/apache/avro/tool/IdlToSchemataTool.java +++ b/lang/java/tools/src/main/java/org/apache/avro/tool/IdlToSchemataTool.java @@ -18,7 +18,9 @@ package org.apache.avro.tool; +import org.apache.avro.Protocol; import org.apache.avro.Schema; +import org.apache.avro.compiler.idl.Idl; import org.apache.avro.idl.IdlFile; import org.apache.avro.idl.IdlReader; @@ -36,30 +38,39 @@ public class IdlToSchemataTool implements Tool { @Override public int run(InputStream in, PrintStream out, PrintStream err, List args) throws Exception { - if (args.isEmpty() || args.size() > 2 || isRequestingHelp(args)) { - err.println("Usage: idl2schemata [idl] [outdir]"); + boolean useJavaCC = "--useJavaCC".equals(getArg(args, 0, null)); + + if (args.isEmpty() || args.size() > (useJavaCC ? 3 : 2) || isRequestingHelp(args)) { + err.println("Usage: idl2schemata [--useJavaCC] [idl [outdir]]"); err.println(); err.println("If an output directory is not specified, " + "outputs to current directory."); return -1; } - boolean pretty = true; - IdlReader parser = new IdlReader(); - IdlFile idlFile; - if (args.size() >= 1 && !"-".equals(args.get(0))) { - idlFile = parser.parse(new File(args.get(0)).toPath()); - } else { - idlFile = parser.parse(in); - } - final List warnings = idlFile.getWarnings(); - for (String warning : warnings) { - err.println("Warning: " + warning); - } - - File outputDirectory = getOutputDirectory(args); + String inputName = getArg(args, useJavaCC ? 1 : 0, "-"); + File inputFile = "-".equals(inputName) ? null : new File(inputName); + File outputDirectory = getOutputDirectory(getArg(args, useJavaCC ? 2 : 1, "")); - for (Schema schema : idlFile.getNamedSchemas().values()) { - print(schema, outputDirectory, pretty); + if (useJavaCC) { + try (Idl parser = new Idl(inputFile)) { + final Protocol protocol = parser.CompilationUnit(); + final List warnings = parser.getWarningsAfterParsing(); + for (String warning : warnings) { + err.println("Warning: " + warning); + } + for (Schema schema : protocol.getTypes()) { + print(schema, outputDirectory); + } + } + } else { + IdlReader parser = new IdlReader(); + IdlFile idlFile = inputFile == null ? parser.parse(in) : parser.parse(inputFile.toPath()); + for (String warning : idlFile.getWarnings()) { + err.println("Warning: " + warning); + } + for (Schema schema : idlFile.getNamedSchemas().values()) { + print(schema, outputDirectory); + } } return 0; @@ -69,19 +80,26 @@ private boolean isRequestingHelp(List args) { return args.size() == 1 && (args.get(0).equals("--help") || args.get(0).equals("-help")); } - private File getOutputDirectory(List args) { - String dirname = (args.size() == 2) ? args.get(1) : ""; + private String getArg(List args, int index, String defaultValue) { + if (index < args.size()) { + return args.get(index); + } else { + return defaultValue; + } + } + + private File getOutputDirectory(String dirname) { File outputDirectory = new File(dirname); outputDirectory.mkdirs(); return outputDirectory; } - private void print(Schema schema, File outputDirectory, boolean pretty) throws FileNotFoundException { + private void print(Schema schema, File outputDirectory) throws FileNotFoundException { String dirpath = outputDirectory.getAbsolutePath(); String filename = dirpath + "/" + schema.getName() + ".avsc"; FileOutputStream fileOutputStream = new FileOutputStream(filename); PrintStream printStream = new PrintStream(fileOutputStream); - printStream.println(schema.toString(pretty)); + printStream.println(schema.toString(true)); printStream.close(); } diff --git a/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java b/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java index d581e5e2c16..dfdaac966c8 100644 --- a/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java +++ b/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java @@ -19,6 +19,7 @@ package org.apache.avro.tool; import org.apache.avro.Protocol; +import org.apache.avro.compiler.idl.Idl; import org.apache.avro.idl.IdlFile; import org.apache.avro.idl.IdlReader; @@ -35,36 +36,46 @@ public class IdlTool implements Tool { @Override public int run(InputStream in, PrintStream out, PrintStream err, List args) throws Exception { - PrintStream parseOut = out; - - if (args.size() > 2 || (args.size() == 1 && (args.get(0).equals("--help") || args.get(0).equals("-help")))) { - err.println("Usage: idl [in] [out]"); + boolean useJavaCC = "--useJavaCC".equals(getArg(args, 0, null)); + if (args.size() > (useJavaCC ? 3 : 2) + || (args.size() == 1 && (args.get(0).equals("--help") || args.get(0).equals("-help")))) { + err.println("Usage: idl [--useJavaCC] [in [out]]"); err.println(); err.println("If an output path is not specified, outputs to stdout."); err.println("If no input or output is specified, takes input from"); - err.println("stdin and outputs to stdin."); + err.println("stdin and outputs to stdout."); err.println("The special path \"-\" may also be specified to refer to"); err.println("stdin and stdout."); return -1; } - IdlReader parser = new IdlReader(); - IdlFile idlFile; - if (args.size() >= 1 && !"-".equals(args.get(0))) { - idlFile = parser.parse(new File(args.get(0)).toPath()); + String inputName = getArg(args, useJavaCC ? 1 : 0, "-"); + File inputFile = "-".equals(inputName) ? null : new File(inputName); + String outputName = getArg(args, useJavaCC ? 2 : 1, "-"); + File outputFile = "-".equals(outputName) ? null : new File(outputName); + + Protocol p; + if (useJavaCC) { + try (Idl parser = new Idl(inputFile)) { + p = parser.CompilationUnit(); + for (String warning : parser.getWarningsAfterParsing()) { + err.println("Warning: " + warning); + } + } } else { - idlFile = parser.parse(in); - } - final List warnings = idlFile.getWarnings(); - for (String warning : warnings) { - err.println("Warning: " + warning); + IdlReader parser = new IdlReader(); + IdlFile idlFile = inputFile == null ? parser.parse(in) : parser.parse(inputFile.toPath()); + for (String warning : idlFile.getWarnings()) { + err.println("Warning: " + warning); + } + p = idlFile.getProtocol(); } - if (args.size() == 2 && !"-".equals(args.get(1))) { - parseOut = new PrintStream(new FileOutputStream(args.get(1))); + PrintStream parseOut = out; + if (outputFile != null) { + parseOut = new PrintStream(new FileOutputStream(outputFile)); } - Protocol p = idlFile.getProtocol(); try { parseOut.print(p.toString(true)); } finally { @@ -74,6 +85,14 @@ public int run(InputStream in, PrintStream out, PrintStream err, List ar return 0; } + private String getArg(List args, int index, String defaultValue) { + if (index < args.size()) { + return args.get(index); + } else { + return defaultValue; + } + } + @Override public String getName() { return "idl"; @@ -81,6 +100,6 @@ public String getName() { @Override public String getShortDescription() { - return "Generates a JSON schema from an Avro IDL file"; + return "Generates a JSON protocol from an Avro IDL file"; } } diff --git a/lang/java/tools/src/test/java/org/apache/avro/tool/TestIdlToSchemataTool.java b/lang/java/tools/src/test/java/org/apache/avro/tool/TestIdlToSchemataTool.java index 601e0a67d4f..04413d2789f 100644 --- a/lang/java/tools/src/test/java/org/apache/avro/tool/TestIdlToSchemataTool.java +++ b/lang/java/tools/src/test/java/org/apache/avro/tool/TestIdlToSchemataTool.java @@ -51,6 +51,37 @@ public void testSplitIdlIntoSchemata() throws Exception { + "\nDid you mean to use a multiline comment ( /* ... */ ) instead?", warnings); } + @Test + public void testSplitIdlIntoSchemataUsingJavaCC() throws Exception { + String idl = "src/test/idl/protocol.avdl"; + String outdir = "target/test-split"; + + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + List arglist = Arrays.asList("--useJavaCC", idl, outdir); + new IdlToSchemataTool().run(null, null, new PrintStream(buffer), arglist); + + String[] files = new File(outdir).list(); + assertEquals(4, files.length); + + String warnings = readPrintStreamBuffer(buffer); + assertEquals( + "Warning: Found documentation comment at line 19, column 1. Ignoring previous one at line 1, column 1: " + + "\"Licensed to the Apache Software Foundation (ASF) under one\n" + + "or more contributor license agreements. See the NOTICE file\n" + + "distributed with this work for additional information\n" + + "regarding copyright ownership. The ASF licenses this file\n" + + "to you under the Apache License, Version 2.0 (the\n" + + "\"License\"); you may not use this file except in compliance\n" + + "with the License. You may obtain a copy of the License at\n" + + "\n https://www.apache.org/licenses/LICENSE-2.0\n\n" + + "Unless required by applicable law or agreed to in writing, software\n" + + "distributed under the License is distributed on an \"AS IS\" BASIS,\n" + + "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" + + "See the License for the specific language governing permissions and\n" + + "limitations under the License.\"\nDid you mean to use a multiline comment ( /* ... */ ) instead?", + warnings); + } + private String readPrintStreamBuffer(ByteArrayOutputStream buffer) { BufferedReader reader = new BufferedReader( new InputStreamReader(new ByteArrayInputStream(buffer.toByteArray()), Charset.defaultCharset())); diff --git a/lang/java/tools/src/test/java/org/apache/avro/tool/TestIdlTool.java b/lang/java/tools/src/test/java/org/apache/avro/tool/TestIdlTool.java index ca8460cb777..de9ea1d9e1d 100644 --- a/lang/java/tools/src/test/java/org/apache/avro/tool/TestIdlTool.java +++ b/lang/java/tools/src/test/java/org/apache/avro/tool/TestIdlTool.java @@ -52,6 +52,37 @@ public void testWriteIdlAsProtocol() throws Exception { + "\nDid you mean to use a multiline comment ( /* ... */ ) instead?", warnings); } + @Test + public void testWriteIdlAsProtocolUsingJavaCC() throws Exception { + String idl = "src/test/idl/protocol.avdl"; + String protocol = "src/test/idl/protocol.avpr"; + String outfile = "target/test-protocol.avpr"; + + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + List arglist = Arrays.asList("--useJavaCC", idl, outfile); + new IdlTool().run(null, null, new PrintStream(buffer), arglist); + + assertEquals(readFileAsString(protocol), readFileAsString(outfile)); + + String warnings = readPrintStreamBuffer(buffer); + assertEquals( + "Warning: Found documentation comment at line 19, column 1. Ignoring previous one at line 1, column 1: " + + "\"Licensed to the Apache Software Foundation (ASF) under one\n" + + "or more contributor license agreements. See the NOTICE file\n" + + "distributed with this work for additional information\n" + + "regarding copyright ownership. The ASF licenses this file\n" + + "to you under the Apache License, Version 2.0 (the\n" + + "\"License\"); you may not use this file except in compliance\n" + + "with the License. You may obtain a copy of the License at\n" + + "\n https://www.apache.org/licenses/LICENSE-2.0\n\n" + + "Unless required by applicable law or agreed to in writing, software\n" + + "distributed under the License is distributed on an \"AS IS\" BASIS,\n" + + "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" + + "See the License for the specific language governing permissions and\n" + + "limitations under the License.\"\nDid you mean to use a multiline comment ( /* ... */ ) instead?", + warnings); + } + private String readFileAsString(String filePath) throws IOException { try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) { return reader.lines().collect(Collectors.joining("\n")); diff --git a/share/idl_grammar/org/apache/avro/idl/Idl.g4 b/share/idl_grammar/org/apache/avro/idl/Idl.g4 index 08f208d7d08..01017a26184 100644 --- a/share/idl_grammar/org/apache/avro/idl/Idl.g4 +++ b/share/idl_grammar/org/apache/avro/idl/Idl.g4 @@ -21,6 +21,10 @@ grammar Idl; ** design, there are no actions in this grammar: this ensures the grammar is ** usable for any language supported by ANTLR. ** +** Some names, like BTrue & BFalse for booleans, may look a bit strange, but +** that's because they can otherwise conflict with identifiers in target\ +** languages like Java, Python, etc. +** ** Implementers can implement a listener or visitor to turn a parse result into ** a protocol and/or schema. */