From 72689295add49cd8ee7d7a0036b23375ee39b53d Mon Sep 17 00:00:00 2001 From: Oscar Westra van Holthe - Kind Date: Mon, 11 Sep 2023 14:06:58 +0200 Subject: [PATCH] AVRO-3403: Create and use ANTLR to parse IDL files (#1588) This large change encompasses a new avro-idl module, a shared ANTLR grammar to allow implementations in other languages, and uses (copies of) all existing IDL tests. --- .editorconfig | 15 + .../src/main/pom/pom.xml | 3 + lang/java/avro/pom.xml | 2 +- lang/java/compiler/pom.xml | 67 +- .../compiler/specific/SpecificCompiler.java | 8 +- .../org/apache/avro/compiler/idl/idl.jj | 3 + lang/java/grpc/pom.xml | 2 +- lang/java/grpc/src/test/avro/TestService.avdl | 2 +- lang/java/idl/pom.xml | 172 +++ .../java/org/apache/avro/idl/IdlFile.java | 120 ++ .../java/org/apache/avro/idl/IdlReader.java | 1048 +++++++++++++++++ .../avro/idl/IsResolvedSchemaVisitor.java | 60 + .../org/apache/avro/idl/ResolvingVisitor.java | 192 +++ .../org/apache/avro/idl/SchemaResolver.java | 135 +++ .../org/apache/avro/idl/SchemaVisitor.java | 47 + .../apache/avro/idl/SchemaVisitorAction.java | 40 + .../java/org/apache/avro/idl/Schemas.java | 150 +++ .../idl/src/main/resources/META-INF/LICENSE | 202 ++++ .../test/idl/AnnotationOnTypeReference.avdl | 31 + lang/java/idl/src/test/idl/cycle.avdl | 42 + .../src/test/idl/extra/protocolSyntax.avdl | 9 + lang/java/idl/src/test/idl/input/bar.avpr | 2 + .../java/idl/src/test/idl/input/baseball.avdl | 23 + lang/java/idl/src/test/idl/input/baz.avsc | 3 + .../java/idl/src/test/idl/input/comments.avdl | 64 + lang/java/idl/src/test/idl/input/cycle.avdl | 42 + lang/java/idl/src/test/idl/input/echo.avdl | 32 + lang/java/idl/src/test/idl/input/foo.avsc | 3 + .../idl/src/test/idl/input/forward_ref.avdl | 33 + lang/java/idl/src/test/idl/input/import.avdl | 40 + lang/java/idl/src/test/idl/input/interop.avdl | 50 + .../idl/src/test/idl/input/mr_events.avdl | 74 ++ .../idl/src/test/idl/input/namespaces.avdl | 42 + .../idl/src/test/idl/input/nestedimport.avdl | 31 + lang/java/idl/src/test/idl/input/player.avsc | 8 + .../java/idl/src/test/idl/input/position.avsc | 3 + .../idl/src/test/idl/input/reservedwords.avdl | 27 + lang/java/idl/src/test/idl/input/simple.avdl | 87 ++ lang/java/idl/src/test/idl/input/unicode.avdl | 29 + lang/java/idl/src/test/idl/input/uuid.avdl | 46 + lang/java/idl/src/test/idl/logicalTypes.avdl | 32 + .../idl/src/test/idl/output/baseball.avpr | 31 + .../idl/src/test/idl/output/comments.avpr | 67 ++ lang/java/idl/src/test/idl/output/cycle.avpr | 55 + lang/java/idl/src/test/idl/output/echo.avpr | 37 + .../idl/src/test/idl/output/forward_ref.avpr | 16 + lang/java/idl/src/test/idl/output/import.avpr | 132 +++ .../java/idl/src/test/idl/output/interop.avpr | 94 ++ .../idl/src/test/idl/output/mr_events.avpr | 125 ++ .../idl/src/test/idl/output/namespaces.avpr | 51 + .../idl/src/test/idl/output/nestedimport.avpr | 73 ++ .../src/test/idl/output/reservedwords.avpr | 35 + lang/java/idl/src/test/idl/output/simple.avpr | 144 +++ .../java/idl/src/test/idl/output/unicode.avpr | 17 + lang/java/idl/src/test/idl/output/uuid.avpr | 29 + .../idl/putOnClassPath/OnTheClasspath.avdl | 24 + .../idl/putOnClassPath/OnTheClasspath.avpr | 11 + .../idl/putOnClassPath/OnTheClasspath.avsc | 6 + .../putOnClassPath/folder/relativePath.avdl | 22 + .../test/idl/putOnClassPath/nestedtypes.avdl | 23 + .../org/apache/avro/idl/IdlReaderTest.java | 245 ++++ .../java/org/apache/avro/idl/TestCycle.java | 87 ++ .../org/apache/avro/idl/TestLogicalTypes.java | 96 ++ .../TestReferenceAnnotationNotAllowed.java | 42 + .../apache/avro/idl/TestSchemaResolver.java | 71 ++ .../java/org/apache/avro/idl/TestSchemas.java | 195 +++ .../integration-test/codegen-test/pom.xml | 2 +- lang/java/integration-test/pom.xml | 2 +- .../test-custom-conversions/pom.xml | 2 +- lang/java/ipc-jetty/pom.xml | 2 +- lang/java/ipc-netty/pom.xml | 2 +- lang/java/ipc/pom.xml | 2 +- lang/java/mapred/pom.xml | 2 +- lang/java/maven-plugin/pom.xml | 5 + .../apache/avro/mojo/AbstractAvroMojo.java | 112 +- .../org/apache/avro/mojo/IDLProtocolMojo.java | 86 +- .../org/apache/avro/mojo/ProtocolMojo.java | 29 +- .../java/org/apache/avro/mojo/SchemaMojo.java | 74 +- .../apache/avro/mojo/TestIDLProtocolMojo.java | 19 +- .../test/resources/unit/idl/pom-javacc.xml | 68 ++ lang/java/perf/pom.xml | 2 +- lang/java/pom.xml | 1 + lang/java/protobuf/pom.xml | 2 +- lang/java/thrift/pom.xml | 2 +- lang/java/tools/pom.xml | 7 +- .../apache/avro/tool/IdlToSchemataTool.java | 59 +- .../java/org/apache/avro/tool/IdlTool.java | 55 +- .../avro/tool/TestIdlToSchemataTool.java | 29 +- .../org/apache/avro/tool/TestIdlTool.java | 29 +- lang/java/trevni/avro/pom.xml | 2 +- lang/java/trevni/core/pom.xml | 2 +- lang/java/trevni/pom.xml | 2 +- share/idl_grammar/org/apache/avro/idl/Idl.g4 | 248 ++++ share/test/schemas/contexts.avdl | 2 +- share/test/schemas/echo.avdl | 2 +- share/test/schemas/http.avdl | 2 +- share/test/schemas/nestedNullable.avdl | 2 +- share/test/schemas/schemaevolution.avdl | 2 +- share/test/schemas/social.avdl | 2 +- share/test/schemas/specialtypes.avdl | 2 +- share/test/schemas/stringables.avdl | 2 +- 101 files changed, 5342 insertions(+), 270 deletions(-) create mode 100644 lang/java/idl/pom.xml create mode 100644 lang/java/idl/src/main/java/org/apache/avro/idl/IdlFile.java create mode 100644 lang/java/idl/src/main/java/org/apache/avro/idl/IdlReader.java create mode 100644 lang/java/idl/src/main/java/org/apache/avro/idl/IsResolvedSchemaVisitor.java create mode 100644 lang/java/idl/src/main/java/org/apache/avro/idl/ResolvingVisitor.java create mode 100644 lang/java/idl/src/main/java/org/apache/avro/idl/SchemaResolver.java create mode 100644 lang/java/idl/src/main/java/org/apache/avro/idl/SchemaVisitor.java create mode 100644 lang/java/idl/src/main/java/org/apache/avro/idl/SchemaVisitorAction.java create mode 100644 lang/java/idl/src/main/java/org/apache/avro/idl/Schemas.java create mode 100644 lang/java/idl/src/main/resources/META-INF/LICENSE create mode 100644 lang/java/idl/src/test/idl/AnnotationOnTypeReference.avdl create mode 100644 lang/java/idl/src/test/idl/cycle.avdl create mode 100644 lang/java/idl/src/test/idl/extra/protocolSyntax.avdl create mode 100644 lang/java/idl/src/test/idl/input/bar.avpr create mode 100644 lang/java/idl/src/test/idl/input/baseball.avdl create mode 100644 lang/java/idl/src/test/idl/input/baz.avsc create mode 100644 lang/java/idl/src/test/idl/input/comments.avdl create mode 100644 lang/java/idl/src/test/idl/input/cycle.avdl create mode 100644 lang/java/idl/src/test/idl/input/echo.avdl create mode 100644 lang/java/idl/src/test/idl/input/foo.avsc create mode 100644 lang/java/idl/src/test/idl/input/forward_ref.avdl create mode 100644 lang/java/idl/src/test/idl/input/import.avdl create mode 100644 lang/java/idl/src/test/idl/input/interop.avdl create mode 100644 lang/java/idl/src/test/idl/input/mr_events.avdl create mode 100644 lang/java/idl/src/test/idl/input/namespaces.avdl create mode 100644 lang/java/idl/src/test/idl/input/nestedimport.avdl create mode 100644 lang/java/idl/src/test/idl/input/player.avsc create mode 100644 lang/java/idl/src/test/idl/input/position.avsc create mode 100644 lang/java/idl/src/test/idl/input/reservedwords.avdl create mode 100644 lang/java/idl/src/test/idl/input/simple.avdl create mode 100644 lang/java/idl/src/test/idl/input/unicode.avdl create mode 100644 lang/java/idl/src/test/idl/input/uuid.avdl create mode 100644 lang/java/idl/src/test/idl/logicalTypes.avdl create mode 100644 lang/java/idl/src/test/idl/output/baseball.avpr create mode 100644 lang/java/idl/src/test/idl/output/comments.avpr create mode 100644 lang/java/idl/src/test/idl/output/cycle.avpr create mode 100644 lang/java/idl/src/test/idl/output/echo.avpr create mode 100644 lang/java/idl/src/test/idl/output/forward_ref.avpr create mode 100644 lang/java/idl/src/test/idl/output/import.avpr create mode 100644 lang/java/idl/src/test/idl/output/interop.avpr create mode 100644 lang/java/idl/src/test/idl/output/mr_events.avpr create mode 100644 lang/java/idl/src/test/idl/output/namespaces.avpr create mode 100644 lang/java/idl/src/test/idl/output/nestedimport.avpr create mode 100644 lang/java/idl/src/test/idl/output/reservedwords.avpr create mode 100644 lang/java/idl/src/test/idl/output/simple.avpr create mode 100644 lang/java/idl/src/test/idl/output/unicode.avpr create mode 100644 lang/java/idl/src/test/idl/output/uuid.avpr create mode 100644 lang/java/idl/src/test/idl/putOnClassPath/OnTheClasspath.avdl create mode 100644 lang/java/idl/src/test/idl/putOnClassPath/OnTheClasspath.avpr create mode 100644 lang/java/idl/src/test/idl/putOnClassPath/OnTheClasspath.avsc create mode 100644 lang/java/idl/src/test/idl/putOnClassPath/folder/relativePath.avdl create mode 100644 lang/java/idl/src/test/idl/putOnClassPath/nestedtypes.avdl create mode 100644 lang/java/idl/src/test/java/org/apache/avro/idl/IdlReaderTest.java create mode 100644 lang/java/idl/src/test/java/org/apache/avro/idl/TestCycle.java create mode 100644 lang/java/idl/src/test/java/org/apache/avro/idl/TestLogicalTypes.java create mode 100644 lang/java/idl/src/test/java/org/apache/avro/idl/TestReferenceAnnotationNotAllowed.java create mode 100644 lang/java/idl/src/test/java/org/apache/avro/idl/TestSchemaResolver.java create mode 100644 lang/java/idl/src/test/java/org/apache/avro/idl/TestSchemas.java create mode 100644 lang/java/maven-plugin/src/test/resources/unit/idl/pom-javacc.xml create mode 100644 share/idl_grammar/org/apache/avro/idl/Idl.g4 diff --git a/.editorconfig b/.editorconfig index da154efb8a0..a2a93880be0 100644 --- a/.editorconfig +++ b/.editorconfig @@ -25,6 +25,21 @@ indent_style = space indent_size = 2 trim_trailing_whitespace=true +ij_continuation_indent_size = 4 +ij_java_wrap_comments = true +ij_any_indent_case_from_switch = false + +[*.{avsc,avpr,avdl}] +indent_style = space +indent_size = 2 +trim_trailing_whitespace=true + +ij_continuation_indent_size = 4 +ij_json_space_after_colon = true +ij_json_space_before_colon = true +ij_json_spaces_within_brackets = true +ij_any_array_initializer_wrap = off + [*.{ps1}] indent_style = space indent_size = 4 diff --git a/lang/java/archetypes/avro-service-archetype/src/main/pom/pom.xml b/lang/java/archetypes/avro-service-archetype/src/main/pom/pom.xml index 69063eb2919..399bbea940b 100644 --- a/lang/java/archetypes/avro-service-archetype/src/main/pom/pom.xml +++ b/lang/java/archetypes/avro-service-archetype/src/main/pom/pom.xml @@ -33,6 +33,9 @@ Simple Avro Ordering Service + ${maven.compiler.source} + ${maven.compiler.target} + ${project.build.sourceEncoding} ${project.version} ${maven.compiler.source} ${maven.compiler.target} diff --git a/lang/java/avro/pom.xml b/lang/java/avro/pom.xml index 541a73df8be..b96673d1851 100644 --- a/lang/java/avro/pom.xml +++ b/lang/java/avro/pom.xml @@ -24,7 +24,7 @@ avro-parent org.apache.avro 1.12.0-SNAPSHOT - ../ + ../pom.xml avro diff --git a/lang/java/compiler/pom.xml b/lang/java/compiler/pom.xml index fe7fbe404bf..2019318ee97 100644 --- a/lang/java/compiler/pom.xml +++ b/lang/java/compiler/pom.xml @@ -183,37 +183,8 @@ - - - - org.eclipse.m2e - lifecycle-mapping - 1.0.0 - - - - - - org.codehaus.mojo - exec-maven-plugin - [1.0,) - - exec - - - - - - - - - - - - - ${project.groupId} @@ -242,4 +213,42 @@ + + + m2e + + m2e.version + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + org.codehaus.mojo + exec-maven-plugin + [1.0,) + + exec + + + + + + + + + + + + + + + diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java index fdffaad32c1..f6f842035fa 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java @@ -178,8 +178,14 @@ public SpecificCompiler(Protocol protocol) { } public SpecificCompiler(Schema schema) { + this(Collections.singleton(schema)); + } + + public SpecificCompiler(Collection schemas) { this(); - enqueue(schema); + for (Schema schema : schemas) { + enqueue(schema); + } this.protocol = null; } diff --git a/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj b/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj index 3ee12fc385b..117764497e3 100644 --- a/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj +++ b/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj @@ -88,7 +88,10 @@ import org.apache.commons.text.StringEscapeUtils; * * Note: each instance is not thread-safe, but multiple separate * instances are safely independent. + * + * @deprecated Use the new org.apache.avro.idl.IdlReader from avro-idl instead. */ +@Deprecated public class Idl implements Closeable { static JsonNodeFactory FACTORY = JsonNodeFactory.instance; private static final String OPTIONAL_NULLABLE_TYPE_PROPERTY = "org.apache.avro.compiler.idl.Idl.NullableType.optional"; diff --git a/lang/java/grpc/pom.xml b/lang/java/grpc/pom.xml index f04dd468fcf..d895a1ba3a1 100644 --- a/lang/java/grpc/pom.xml +++ b/lang/java/grpc/pom.xml @@ -24,7 +24,7 @@ org.apache.avro avro-parent 1.12.0-SNAPSHOT - ../ + ../pom.xml avro-grpc diff --git a/lang/java/grpc/src/test/avro/TestService.avdl b/lang/java/grpc/src/test/avro/TestService.avdl index 9a4629a8f5c..6c5f6a038b8 100644 --- a/lang/java/grpc/src/test/avro/TestService.avdl +++ b/lang/java/grpc/src/test/avro/TestService.avdl @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/lang/java/idl/pom.xml b/lang/java/idl/pom.xml new file mode 100644 index 00000000000..6363ec150a5 --- /dev/null +++ b/lang/java/idl/pom.xml @@ -0,0 +1,172 @@ + + + + 4.0.0 + + + avro-parent + org.apache.avro + 1.12.0-SNAPSHOT + ../pom.xml + + + avro-idl + + Apache Avro IDL + bundle + https://avro.apache.org + Compilers for Avro IDL and Avro Specific Java API + + + ${project.parent.parent.basedir} + + !org.apache.avro.idl*, + org.apache.avro*;version="${project.version}", + org.apache.commons.text*, + * + + org.apache.avro.idl*;version="${project.version}" + 4.9.3 + + + + + + src/main/resources + + + + + src/test/resources + + + src/test/idl + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + org.apache.avro.idl + + + + + + prepare-test-jar + generate-test-resources + + test-jar + + + test-resource + src/test/idl/putOnClassPath + putOnClassPath + ${project.build.testOutputDirectory} + + + + + + org.antlr + antlr4-maven-plugin + ${antlr.version} + + + antlr + + antlr4 + + + + + ${project.basedir}/../../../share/idl_grammar + ${project.basedir}/../../../share/idl_grammar/imports + true + false + + + + + + + + ${project.groupId} + avro + ${project.version} + + + org.antlr + antlr4-runtime + ${antlr.version} + + + org.apache.commons + commons-text + ${commons-text.version} + + + com.fasterxml.jackson.core + jackson-databind + + + + + + m2e + + m2e.version + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + org.codehaus.mojo + exec-maven-plugin + [1.0,) + + exec + + + + + + + + + + + + + + + + diff --git a/lang/java/idl/src/main/java/org/apache/avro/idl/IdlFile.java b/lang/java/idl/src/main/java/org/apache/avro/idl/IdlFile.java new file mode 100644 index 00000000000..56627b5821b --- /dev/null +++ b/lang/java/idl/src/main/java/org/apache/avro/idl/IdlFile.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.idl; + +import org.apache.avro.Protocol; +import org.apache.avro.Schema; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * A parsed IdlFile. Provides access to the named schemas in the IDL file and + * the protocol containing the schemas. + */ +public class IdlFile { + private final Protocol protocol; + private final String namespace; + private final Map namedSchemas; + private final List warnings; + + IdlFile(Protocol protocol, List warnings) { + this(protocol.getNamespace(), protocol.getTypes(), protocol, warnings); + } + + private IdlFile(String namespace, Iterable schemas, Protocol protocol, List warnings) { + this.namespace = namespace; + this.namedSchemas = new LinkedHashMap<>(); + for (Schema namedSchema : schemas) { + this.namedSchemas.put(namedSchema.getFullName(), namedSchema); + } + this.protocol = protocol; + this.warnings = Collections.unmodifiableList(new ArrayList<>(warnings)); + } + + /** + * The protocol defined by the IDL file. + */ + public Protocol getProtocol() { + return protocol; + } + + public List getWarnings() { + return warnings; + } + + public List getWarnings(String importFile) { + return warnings.stream() + .map(warning -> importFile + ' ' + Character.toLowerCase(warning.charAt(0)) + warning.substring(1)) + .collect(Collectors.toList()); + } + + /** + * The default namespace to resolve schema names against. + */ + public String getNamespace() { + return namespace; + } + + /** + * The named schemas defined by the IDL file, mapped by their full name. + */ + public Map getNamedSchemas() { + return Collections.unmodifiableMap(namedSchemas); + } + + /** + * Get a named schema defined by the IDL file, by name. The name can be a simple + * name in the default namespace of the IDL file (e.g., the namespace of the + * protocol), or a full name. + * + * @param name the full name of the schema, or a simple name + * @return the schema, or {@code null} if it does not exist + */ + public Schema getNamedSchema(String name) { + Schema result = namedSchemas.get(name); + if (result != null) { + return result; + } + if (namespace != null && !name.contains(".")) { + result = namedSchemas.get(namespace + '.' + name); + } + return result; + } + + // Visible for testing + String outputString() { + if (protocol != null) { + return protocol.toString(); + } + if (namedSchemas.isEmpty()) { + return "[]"; + } else { + StringBuilder buffer = new StringBuilder(); + for (Schema schema : namedSchemas.values()) { + buffer.append(',').append(schema); + } + buffer.append(']').setCharAt(0, '['); + return buffer.toString(); + } + } +} diff --git a/lang/java/idl/src/main/java/org/apache/avro/idl/IdlReader.java b/lang/java/idl/src/main/java/org/apache/avro/idl/IdlReader.java new file mode 100644 index 00000000000..ec9f698819a --- /dev/null +++ b/lang/java/idl/src/main/java/org/apache/avro/idl/IdlReader.java @@ -0,0 +1,1048 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.idl; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.BooleanNode; +import com.fasterxml.jackson.databind.node.DoubleNode; +import com.fasterxml.jackson.databind.node.IntNode; +import com.fasterxml.jackson.databind.node.LongNode; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import org.antlr.v4.runtime.BaseErrorListener; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.Token; +import org.apache.avro.JsonProperties; +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Protocol; +import org.apache.avro.Schema; +import org.apache.avro.SchemaParseException; +import org.apache.avro.idl.IdlParser.ArrayTypeContext; +import org.apache.avro.idl.IdlParser.EnumDeclarationContext; +import org.apache.avro.idl.IdlParser.EnumSymbolContext; +import org.apache.avro.idl.IdlParser.FieldDeclarationContext; +import org.apache.avro.idl.IdlParser.FixedDeclarationContext; +import org.apache.avro.idl.IdlParser.FormalParameterContext; +import org.apache.avro.idl.IdlParser.FullTypeContext; +import org.apache.avro.idl.IdlParser.IdentifierContext; +import org.apache.avro.idl.IdlParser.IdlFileContext; +import org.apache.avro.idl.IdlParser.ImportStatementContext; +import org.apache.avro.idl.IdlParser.JsonArrayContext; +import org.apache.avro.idl.IdlParser.JsonLiteralContext; +import org.apache.avro.idl.IdlParser.JsonObjectContext; +import org.apache.avro.idl.IdlParser.JsonPairContext; +import org.apache.avro.idl.IdlParser.JsonValueContext; +import org.apache.avro.idl.IdlParser.MapTypeContext; +import org.apache.avro.idl.IdlParser.MessageDeclarationContext; +import org.apache.avro.idl.IdlParser.NullableTypeContext; +import org.apache.avro.idl.IdlParser.PrimitiveTypeContext; +import org.apache.avro.idl.IdlParser.ProtocolDeclarationBodyContext; +import org.apache.avro.idl.IdlParser.ProtocolDeclarationContext; +import org.apache.avro.idl.IdlParser.RecordBodyContext; +import org.apache.avro.idl.IdlParser.RecordDeclarationContext; +import org.apache.avro.idl.IdlParser.ResultTypeContext; +import org.apache.avro.idl.IdlParser.SchemaPropertyContext; +import org.apache.avro.idl.IdlParser.UnionTypeContext; +import org.apache.avro.idl.IdlParser.VariableDeclarationContext; +import org.apache.avro.util.internal.Accessor; +import org.apache.commons.text.StringEscapeUtils; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Deque; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static java.util.Collections.singleton; +import static java.util.Collections.unmodifiableMap; + +public class IdlReader { + /** + * Simple error listener. Throws a runtime exception because ANTLR does not give + * easy access to the (reasonably readable) error message elsewhere. + */ + private static final BaseErrorListener SIMPLE_AVRO_ERROR_LISTENER = new BaseErrorListener() { + @Override + public void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, int charPositionInLine, + String msg, RecognitionException e) { + throw new SchemaParseException("line " + line + ":" + charPositionInLine + " " + msg); + } + }; + private static final String OPTIONAL_NULLABLE_TYPE_PROPERTY = "org.apache.avro.idl.Idl.NullableType.optional"; + /** + * Pattern to match the common whitespace indents in a multi-line String. + * Doesn't match a single-line String, fully matches any multi-line String. + *

+ * To use: match on a {@link String#trim() trimmed} String, and then replace all + * newlines followed by the group "indent" with a newline. + */ + private static final Pattern WS_INDENT = Pattern.compile("(?U).*\\R(?\\h*).*(?:\\R\\k.*)*"); + /** + * Pattern to match the whitespace indents plus common stars (1 or 2) in a + * multi-line String. If a String fully matches, replace all occurrences of a + * newline followed by whitespace and then the group "stars" with a newline. + *

+ * Note: partial matches are invalid. + */ + private static final Pattern STAR_INDENT = Pattern.compile("(?U)(?\\*{1,2}).*(?:\\R\\h*\\k.*)*"); + /** + * Predicate to check for valid names. Should probably be delegated to the + * Schema class. + */ + private static final Predicate VALID_NAME = Pattern.compile("[_\\p{L}][_\\p{L}\\d]*").asPredicate(); + private static final Set INVALID_TYPE_NAMES = new HashSet<>(Arrays.asList("boolean", "int", "long", "float", + "double", "bytes", "string", "null", "date", "time_ms", "timestamp_ms", "localtimestamp_ms", "uuid")); + private static final String CLASSPATH_SCHEME = "classpath"; + + private final Set readLocations; + private final Map names; + + public IdlReader() { + readLocations = new HashSet<>(); + names = new LinkedHashMap<>(); + } + + public Map getTypes() { + return unmodifiableMap(names); + } + + private Schema namedSchemaOrUnresolved(String fullName) { + Schema schema = names.get(fullName); + if (schema == null) { + schema = SchemaResolver.unresolvedSchema(fullName); + } + return schema; + } + + private void setTypes(Map types) { + names.clear(); + for (Schema schema : types.values()) { + addSchema(schema); + } + } + + public void addTypes(Map types) { + for (Schema schema : types.values()) { + addSchema(schema); + } + } + + private void addSchema(Schema schema) { + String fullName = schema.getFullName(); + if (names.containsKey(fullName)) { + throw new SchemaParseException("Can't redefine: " + fullName); + } + names.put(fullName, schema); + } + + public IdlFile parse(Path location) throws IOException { + return parse(location.toUri()); + } + + IdlFile parse(URI location) throws IOException { + try (InputStream stream = location.toURL().openStream()) { + readLocations.add(location); + URI inputDir = location; + if ("jar".equals(location.getScheme())) { + String jarUriAsString = location.toString(); + String pathFromJarRoot = jarUriAsString.substring(jarUriAsString.indexOf("!/") + 2); + inputDir = URI.create(CLASSPATH_SCHEME + ":/" + pathFromJarRoot); + } + inputDir = inputDir.resolve("."); + + return parse(inputDir, CharStreams.fromStream(stream, StandardCharsets.UTF_8)); + } + } + + /** + * Parse an IDL file from a stream. This method cannot handle imports. + */ + public IdlFile parse(InputStream stream) throws IOException { + return parse(null, CharStreams.fromStream(stream, StandardCharsets.UTF_8)); + } + + private IdlFile parse(URI inputDir, CharStream charStream) { + IdlLexer lexer = new IdlLexer(charStream); + CommonTokenStream tokenStream = new CommonTokenStream(lexer); + + IdlParserListener parseListener = new IdlParserListener(inputDir, tokenStream); + + IdlParser parser = new IdlParser(tokenStream); + parser.removeErrorListeners(); + parser.addErrorListener(SIMPLE_AVRO_ERROR_LISTENER); + parser.addParseListener(parseListener); + parser.setTrace(false); + parser.setBuildParseTree(false); + + // Trigger parsing. + parser.idlFile(); + + return parseListener.getIdlFile(); + } + + /* Package private to facilitate testing */ + static String stripIndents(String docComment) { + Matcher starMatcher = STAR_INDENT.matcher(docComment); + if (starMatcher.matches()) { + return docComment.replaceAll("(?U)(?:^|(\\R)\\h*)\\Q" + starMatcher.group("stars") + "\\E\\h?", "$1"); + } + + Matcher whitespaceMatcher = WS_INDENT.matcher(docComment); + if (whitespaceMatcher.matches()) { + return docComment.replaceAll("(?U)(\\R)" + whitespaceMatcher.group("indent"), "$1"); + } + + return docComment; + } + + private static SchemaParseException error(String message, Token token) { + return error(message, token, null); + } + + private static SchemaParseException error(String message, Token token, Throwable cause) { + SchemaParseException exception = new SchemaParseException( + message + ", at line " + token.getLine() + ", column " + token.getCharPositionInLine()); + if (cause != null) { + exception.initCause(cause); + } + return exception; + } + + private class IdlParserListener extends IdlBaseListener { + private final URI inputDir; + private final CommonTokenStream tokenStream; + private int hiddenTokensProcessedIndex; + private final List warnings; + + private IdlFile result; + private Protocol protocol; + private final Deque namespaces; + private final List enumSymbols; + private String enumDefaultSymbol; + private Schema schema; + private String defaultVariableDocComment; + private final List fields; + private final Deque typeStack; + private final Deque jsonValues; + private final Deque propertiesStack; + private String messageDocComment; + + public IdlParserListener(URI inputDir, CommonTokenStream tokenStream) { + this.inputDir = inputDir; + this.tokenStream = tokenStream; + hiddenTokensProcessedIndex = -1; + warnings = new ArrayList<>(); + + result = null; + protocol = null; + namespaces = new ArrayDeque<>(); + enumSymbols = new ArrayList<>(); + enumDefaultSymbol = null; + schema = null; + defaultVariableDocComment = null; + fields = new ArrayList<>(); + typeStack = new ArrayDeque<>(); + propertiesStack = new ArrayDeque<>(); + jsonValues = new ArrayDeque<>(); + messageDocComment = null; + } + + public IdlFile getIdlFile() { + return result; + } + + private String getDocComment(ParserRuleContext ctx) { + int newHiddenTokensProcessedIndex = ctx.start.getTokenIndex(); + List docCommentTokens = tokenStream.getHiddenTokensToLeft(newHiddenTokensProcessedIndex, -1); + int searchEndIndex = newHiddenTokensProcessedIndex; + + Token docCommentToken = null; + if (docCommentTokens != null) { + // There's at least one element + docCommentToken = docCommentTokens.get(docCommentTokens.size() - 1); + searchEndIndex = docCommentToken.getTokenIndex() - 1; + } + + Set allHiddenTokens = singleton(IdlParser.DocComment); + if (searchEndIndex >= 0) { + List hiddenTokens = tokenStream.getTokens(hiddenTokensProcessedIndex + 1, searchEndIndex, + allHiddenTokens); + if (hiddenTokens != null) { + for (Token token : hiddenTokens) { + warnings.add(String.format( + "Line %d, char %d: Ignoring out-of-place documentation comment.%n" + + "Did you mean to use a multiline comment ( /* ... */ ) instead?", + token.getLine(), token.getCharPositionInLine() + 1)); + } + } + } + hiddenTokensProcessedIndex = newHiddenTokensProcessedIndex; + + if (docCommentToken == null) { + return null; + } + String comment = docCommentToken.getText(); + String text = comment.substring(3, comment.length() - 2); // Strip /** & */ + return stripIndents(text.trim()); + } + + private void pushNamespace(String namespace) { + namespaces.push(namespace == null ? "" : namespace); + } + + private String currentNamespace() { + String namespace = namespaces.element(); + return namespace.isEmpty() ? null : namespace; + } + + private void popNamespace() { + namespaces.pop(); + } + + @Override + public void exitIdlFile(IdlFileContext ctx) { + IdlFile unresolved = new IdlFile(protocol, warnings); + result = SchemaResolver.resolve(unresolved, OPTIONAL_NULLABLE_TYPE_PROPERTY); + } + + @Override + public void enterProtocolDeclaration(ProtocolDeclarationContext ctx) { + propertiesStack.push(new SchemaProperties(null, true, false, false)); + } + + @Override + public void enterProtocolDeclarationBody(ProtocolDeclarationBodyContext ctx) { + ProtocolDeclarationContext protocolCtx = (ProtocolDeclarationContext) ctx.parent; + SchemaProperties properties = propertiesStack.pop(); + String protocolIdentifier = identifier(protocolCtx.name); + pushNamespace(namespace(protocolIdentifier, properties.namespace())); + + String protocolName = name(protocolIdentifier); + String docComment = getDocComment(protocolCtx); + String protocolNamespace = currentNamespace(); + protocol = properties.copyProperties(new Protocol(protocolName, docComment, protocolNamespace)); + } + + @Override + public void exitProtocolDeclaration(ProtocolDeclarationContext ctx) { + if (protocol != null) + protocol.setTypes(getTypes().values()); + if (!namespaces.isEmpty()) + popNamespace(); + } + + @Override + public void enterSchemaProperty(SchemaPropertyContext ctx) { + assert jsonValues.isEmpty(); + } + + @Override + public void exitSchemaProperty(SchemaPropertyContext ctx) { + String name = identifier(ctx.name); + JsonNode value = jsonValues.pop(); + Token firstToken = ctx.value.start; + + propertiesStack.element().addProperty(name, value, firstToken); + super.exitSchemaProperty(ctx); + } + + @Override + public void exitImportStatement(ImportStatementContext importContext) { + String importFile = getString(importContext.location); + try { + URI importLocation = findImport(importFile); + if (!readLocations.add(importLocation)) { + // Already imported + return; + } + switch (importContext.importType.getType()) { + case IdlParser.IDL: + // Note that the parse(URI) method uses the same known schema collection + IdlFile idlFile = parse(importLocation); + if (protocol != null && idlFile.getProtocol() != null) { + protocol.getMessages().putAll(idlFile.getProtocol().getMessages()); + } + warnings.addAll(idlFile.getWarnings(importFile)); + break; + case IdlParser.Protocol: + try (InputStream stream = importLocation.toURL().openStream()) { + Protocol importProtocol = Protocol.parse(stream); + for (Schema s : importProtocol.getTypes()) { + addSchema(s); + } + if (protocol != null) { + protocol.getMessages().putAll(importProtocol.getMessages()); + } + } + break; + case IdlParser.Schema: + try (InputStream stream = importLocation.toURL().openStream()) { + Schema.Parser parser = new Schema.Parser(); + parser.addTypes(getTypes().values()); // inherit names + parser.parse(stream); + setTypes(parser.getTypes()); // update names + } + break; + } + } catch (IOException e) { + throw error("Error importing " + importFile + ": " + e, importContext.location, e); + } + } + + /** + * Best effort guess at the import file location. For locations inside jar + * files, this may result in non-existing URLs. + */ + private URI findImport(String importFile) throws IOException { + URI importLocation = inputDir.resolve(importFile); + String importLocationScheme = importLocation.getScheme(); + + if (CLASSPATH_SCHEME.equals(importLocationScheme)) { + String resourceName = importLocation.getSchemeSpecificPart().substring(1); + URI resourceLocation = findResource(resourceName); + if (resourceLocation != null) { + return resourceLocation; + } + } + + if ("file".equals(importLocationScheme) && Files.exists(Paths.get(importLocation))) { + return importLocation; + } + + // The importFile doesn't exist as file relative to the current file. Try to + // load it from the classpath. + URI resourceLocation = findResource(importFile); + if (resourceLocation != null) { + return resourceLocation; + } + + // Cannot find the import. + throw new FileNotFoundException(importFile); + } + + private URI findResource(String resourceName) { + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + URL resourceLocation; + if (classLoader == null) { + resourceLocation = ClassLoader.getSystemResource(resourceName); + } else { + resourceLocation = classLoader.getResource(resourceName); + } + return resourceLocation == null ? null : URI.create(resourceLocation.toExternalForm()); + } + + @Override + public void enterFixedDeclaration(FixedDeclarationContext ctx) { + propertiesStack.push(new SchemaProperties(currentNamespace(), true, true, false)); + } + + @Override + public void exitFixedDeclaration(FixedDeclarationContext ctx) { + SchemaProperties properties = propertiesStack.pop(); + + String doc = getDocComment(ctx); + String identifier = identifier(ctx.name); + String name = name(identifier); + String space = namespace(identifier, properties.namespace()); + int size = Integer.decode(ctx.size.getText()); + Schema schema = Schema.createFixed(name, doc, space, size); + properties.copyAliases(schema::addAlias); + properties.copyProperties(schema); + addSchema(schema); + } + + @Override + public void enterEnumDeclaration(EnumDeclarationContext ctx) { + assert enumSymbols.isEmpty(); + assert enumDefaultSymbol == null; + propertiesStack.push(new SchemaProperties(currentNamespace(), true, true, false)); + } + + @Override + public void exitEnumDeclaration(EnumDeclarationContext ctx) { + String doc = getDocComment(ctx); + SchemaProperties properties = propertiesStack.pop(); + String identifier = identifier(ctx.name); + String name = name(identifier); + String space = namespace(identifier, properties.namespace()); + + Schema schema = Schema.createEnum(name, doc, space, new ArrayList<>(enumSymbols), enumDefaultSymbol); + properties.copyAliases(schema::addAlias); + properties.copyProperties(schema); + enumSymbols.clear(); + enumDefaultSymbol = null; + + addSchema(schema); + } + + @Override + public void enterEnumSymbol(EnumSymbolContext ctx) { + propertiesStack.push(new SchemaProperties(null, false, false, false)); + } + + @Override + public void exitEnumSymbol(EnumSymbolContext ctx) { + // TODO: implement doc comment & properties for enum symbols. + propertiesStack.pop(); + + enumSymbols.add(identifier(ctx.name)); + } + + @Override + public void exitEnumDefault(IdlParser.EnumDefaultContext ctx) { + enumDefaultSymbol = identifier(ctx.defaultSymbolName); + } + + @Override + public void enterRecordDeclaration(RecordDeclarationContext ctx) { + assert schema == null; + assert fields.isEmpty(); + + propertiesStack.push(new SchemaProperties(currentNamespace(), true, true, false)); + } + + @Override + public void enterRecordBody(RecordBodyContext ctx) { + assert fields.isEmpty(); + + RecordDeclarationContext recordCtx = (RecordDeclarationContext) ctx.parent; + + SchemaProperties properties = propertiesStack.pop(); + + String doc = getDocComment(recordCtx); + String identifier = identifier(recordCtx.name); + String name = name(identifier); + pushNamespace(namespace(identifier, properties.namespace())); + boolean isError = recordCtx.recordType.getType() == IdlParser.Error; + schema = Schema.createRecord(name, doc, currentNamespace(), isError); + properties.copyAliases(schema::addAlias); + properties.copyProperties(schema); + } + + @Override + public void exitRecordDeclaration(RecordDeclarationContext ctx) { + schema.setFields(fields); + fields.clear(); + addSchema(schema); + schema = null; + + popNamespace(); + } + + @Override + public void enterFieldDeclaration(FieldDeclarationContext ctx) { + assert typeStack.isEmpty(); + defaultVariableDocComment = getDocComment(ctx); + } + + @Override + public void exitFieldDeclaration(FieldDeclarationContext ctx) { + typeStack.pop(); + defaultVariableDocComment = null; + } + + @Override + public void enterVariableDeclaration(VariableDeclarationContext ctx) { + assert jsonValues.isEmpty(); + propertiesStack.push(new SchemaProperties(currentNamespace(), false, true, true)); + } + + @Override + public void exitVariableDeclaration(VariableDeclarationContext ctx) { + String doc = Optional.ofNullable(getDocComment(ctx)).orElse(defaultVariableDocComment); + String fieldName = identifier(ctx.fieldName); + + JsonNode defaultValue = jsonValues.poll(); + Schema type = typeStack.element(); + JsonNode fieldDefault = fixDefaultValue(defaultValue, type); + Schema fieldType = fixOptionalSchema(type, fieldDefault); + + SchemaProperties properties = propertiesStack.pop(); + + boolean validate = SchemaResolver.isFullyResolvedSchema(fieldType); + Schema.Field field = Accessor.createField(fieldName, fieldType, doc, fieldDefault, validate, properties.order()); + properties.copyAliases(field::addAlias); + properties.copyProperties(field); + fields.add(field); + } + + /** + * When parsing JSON, the parser generates a LongNode or IntNode based on the + * size of the number it encounters. But this may not be expected based on the + * schema. This method fixes that. + * + * @param defaultValue the parsed default value + * @param fieldType the field schema + * @return the default value, now matching the schema + */ + private JsonNode fixDefaultValue(JsonNode defaultValue, Schema fieldType) { + if (!(defaultValue instanceof IntNode)) { + return defaultValue; + } + + if (fieldType.getType() == Schema.Type.UNION) { + for (Schema unionedType : fieldType.getTypes()) { + if (unionedType.getType() == Schema.Type.INT) { + break; + } else if (unionedType.getType() == Schema.Type.LONG) { + return new LongNode(defaultValue.longValue()); + } + } + return defaultValue; + } + + if (fieldType.getType() == Schema.Type.LONG) { + return new LongNode(defaultValue.longValue()); + } + + return defaultValue; + } + + /** + * For "optional schemas" (recognized by the marker property the NullableType + * production adds), ensure the null schema is in the right place. + * + * @param schema a schema + * @param defaultValue the intended default value + * @return the schema, or an optional schema with null in the right place + */ + private Schema fixOptionalSchema(Schema schema, JsonNode defaultValue) { + Object optionalType = schema.getObjectProp(OPTIONAL_NULLABLE_TYPE_PROPERTY); + if (optionalType != null) { + // The schema is a union schema with 2 types: "null" and a non-"null" schema + Schema nullSchema = schema.getTypes().get(0); + Schema nonNullSchema = schema.getTypes().get(1); + boolean nonNullDefault = defaultValue != null && !defaultValue.isNull(); + + // Note: the resolving visitor we'll use later drops the marker property. + if (nonNullDefault) { + return Schema.createUnion(nonNullSchema, nullSchema); + } + } + return schema; + } + + @Override + public void enterMessageDeclaration(MessageDeclarationContext ctx) { + assert typeStack.isEmpty(); + assert fields.isEmpty(); + assert messageDocComment == null; + propertiesStack.push(new SchemaProperties(currentNamespace(), false, false, false)); + messageDocComment = getDocComment(ctx); + } + + @Override + public void exitMessageDeclaration(MessageDeclarationContext ctx) { + Schema resultType = typeStack.pop(); + Map properties = propertiesStack.pop().properties; + String name = identifier(ctx.name); + + Schema request = Schema.createRecord(null, null, null, false, fields); + fields.clear(); + + Protocol.Message message; + if (ctx.oneway != null) { + if (resultType.getType() == Schema.Type.NULL) { + message = protocol.createMessage(name, messageDocComment, properties, request); + } else { + throw error("One-way message'" + name + "' must return void", ctx.returnType.start); + } + } else { + List errorSchemas = new ArrayList<>(); + errorSchemas.add(Protocol.SYSTEM_ERROR); + for (IdentifierContext errorContext : ctx.errors) { + errorSchemas.add(namedSchemaOrUnresolved(fullName(currentNamespace(), identifier(errorContext)))); + } + message = protocol.createMessage(name, messageDocComment, properties, request, resultType, + Schema.createUnion(errorSchemas)); + } + messageDocComment = null; + protocol.getMessages().put(message.getName(), message); + } + + @Override + public void enterFormalParameter(FormalParameterContext ctx) { + assert typeStack.size() == 1; // The message return type is on the stack; nothing else. + defaultVariableDocComment = getDocComment(ctx); + } + + @Override + public void exitFormalParameter(FormalParameterContext ctx) { + typeStack.pop(); + defaultVariableDocComment = null; + } + + @Override + public void exitResultType(ResultTypeContext ctx) { + if (typeStack.isEmpty()) { + // if there's no type, we've parsed 'void': use the null type + typeStack.push(Schema.create(Schema.Type.NULL)); + } + } + + @Override + public void enterFullType(FullTypeContext ctx) { + propertiesStack.push(new SchemaProperties(currentNamespace(), false, false, false)); + } + + @Override + public void exitFullType(FullTypeContext ctx) { + SchemaProperties properties = propertiesStack.pop(); + + Schema type = typeStack.element(); + if (type.getObjectProp(OPTIONAL_NULLABLE_TYPE_PROPERTY) != null) { + // Optional type: put the properties on the non-null content + properties.copyProperties(type.getTypes().get(1)); + } else { + properties.copyProperties(type); + } + } + + @Override + public void exitNullableType(NullableTypeContext ctx) { + Schema type; + if (ctx.referenceName == null) { + type = typeStack.pop(); + } else { + // propertiesStack is empty within resultType->plainType->nullableType, and + // holds our properties otherwise + if (propertiesStack.isEmpty() || propertiesStack.peek().hasProperties()) { + throw error("Type references may not be annotated", ctx.getParent().getStart()); + } + type = namedSchemaOrUnresolved(fullName(currentNamespace(), identifier(ctx.referenceName))); + } + if (ctx.optional != null) { + type = Schema.createUnion(Schema.create(Schema.Type.NULL), type); + // Add a marker property to the union (it will be removed when creating fields) + type.addProp(OPTIONAL_NULLABLE_TYPE_PROPERTY, BooleanNode.TRUE); + } + typeStack.push(type); + } + + @Override + public void exitPrimitiveType(PrimitiveTypeContext ctx) { + switch (ctx.typeName.getType()) { + case IdlParser.Boolean: + typeStack.push(Schema.create(Schema.Type.BOOLEAN)); + break; + case IdlParser.Int: + typeStack.push(Schema.create(Schema.Type.INT)); + break; + case IdlParser.Long: + typeStack.push(Schema.create(Schema.Type.LONG)); + break; + case IdlParser.Float: + typeStack.push(Schema.create(Schema.Type.FLOAT)); + break; + case IdlParser.Double: + typeStack.push(Schema.create(Schema.Type.DOUBLE)); + break; + case IdlParser.Bytes: + typeStack.push(Schema.create(Schema.Type.BYTES)); + break; + case IdlParser.String: + typeStack.push(Schema.create(Schema.Type.STRING)); + break; + case IdlParser.Null: + typeStack.push(Schema.create(Schema.Type.NULL)); + break; + case IdlParser.Date: + typeStack.push(LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT))); + break; + case IdlParser.Time: + typeStack.push(LogicalTypes.timeMillis().addToSchema(Schema.create(Schema.Type.INT))); + break; + case IdlParser.Timestamp: + typeStack.push(LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG))); + break; + case IdlParser.LocalTimestamp: + typeStack.push(LogicalTypes.localTimestampMillis().addToSchema(Schema.create(Schema.Type.LONG))); + break; + case IdlParser.UUID: + typeStack.push(LogicalTypes.uuid().addToSchema(Schema.create(Schema.Type.STRING))); + break; + default: // Only option left: decimal + int precision = Integer.decode(ctx.precision.getText()); + int scale = ctx.scale == null ? 0 : Integer.decode(ctx.scale.getText()); + typeStack.push(LogicalTypes.decimal(precision, scale).addToSchema(Schema.create(Schema.Type.BYTES))); + break; + } + } + + @Override + public void exitArrayType(ArrayTypeContext ctx) { + typeStack.push(Schema.createArray(typeStack.pop())); + } + + @Override + public void exitMapType(MapTypeContext ctx) { + typeStack.push(Schema.createMap(typeStack.pop())); + } + + @Override + public void enterUnionType(UnionTypeContext ctx) { + // push an empty marker union; we'll replace it with the real union upon exit + typeStack.push(Schema.createUnion()); + } + + @Override + public void exitUnionType(UnionTypeContext ctx) { + List types = new ArrayList<>(); + Schema type; + while ((type = typeStack.pop()).getType() != Schema.Type.UNION) { + types.add(type); + } + Collections.reverse(types); // Popping the stack works in reverse order + // type is an empty marker union; ignore (drop) it + typeStack.push(Schema.createUnion(types)); + } + + @Override + public void exitJsonValue(JsonValueContext ctx) { + if (ctx.parent instanceof JsonArrayContext) { + JsonNode value = jsonValues.pop(); + assert jsonValues.peek() instanceof ArrayNode; + ((ArrayNode) jsonValues.element()).add(value); + } + } + + @Override + public void exitJsonLiteral(JsonLiteralContext ctx) { + Token literal = ctx.literal; + switch (literal.getType()) { + case IdlParser.Null: + jsonValues.push(NullNode.getInstance()); + break; + case IdlParser.BTrue: + jsonValues.push(BooleanNode.TRUE); + break; + case IdlParser.BFalse: + jsonValues.push(BooleanNode.FALSE); + break; + case IdlParser.IntegerLiteral: + String number = literal.getText().replace("_", ""); + char lastChar = number.charAt(number.length() - 1); + boolean coerceToLong = false; + if (lastChar == 'l' || lastChar == 'L') { + coerceToLong = true; + number = number.substring(0, number.length() - 1); + } + long longNumber = Long.decode(number); + int intNumber = (int) longNumber; // Narrowing cast: if too large a number, the two are different + jsonValues.push(coerceToLong || intNumber != longNumber ? new LongNode(longNumber) : new IntNode(intNumber)); + break; + case IdlParser.FloatingPointLiteral: + jsonValues.push(new DoubleNode(Double.parseDouble(literal.getText()))); + break; + default: // StringLiteral: + jsonValues.push(new TextNode(getString(literal))); + break; + } + } + + @Override + public void enterJsonArray(JsonArrayContext ctx) { + jsonValues.push(new ArrayNode(null)); + } + + @Override + public void enterJsonObject(JsonObjectContext ctx) { + jsonValues.push(new ObjectNode(null)); + } + + @Override + public void exitJsonPair(JsonPairContext ctx) { + String name = getString(ctx.name); + JsonNode value = jsonValues.pop(); + assert jsonValues.peek() instanceof ObjectNode; + ((ObjectNode) jsonValues.element()).set(name, value); + } + + private String identifier(IdentifierContext ctx) { + return ctx.word.getText().replace("`", ""); + } + + private String name(String identifier) { + int dotPos = identifier.lastIndexOf('.'); + String name = identifier.substring(dotPos + 1); + return validateName(name, true); + } + + private String namespace(String identifier, String namespace) { + int dotPos = identifier.lastIndexOf('.'); + String ns = dotPos < 0 ? namespace : identifier.substring(0, dotPos); + if (ns == null) { + return null; + } + for (int s = 0, e = ns.indexOf('.'); e > 0; s = e + 1, e = ns.indexOf('.', s)) { + validateName(ns.substring(s, e), false); + } + return ns; + } + + private String validateName(String name, boolean isTypeName) { + if (name == null) { + throw new SchemaParseException("Null name"); + } else if (!VALID_NAME.test(name)) { + throw new SchemaParseException("Illegal name: " + name); + } + if (isTypeName && INVALID_TYPE_NAMES.contains(name)) { + throw new SchemaParseException("Illegal name: " + name); + } + return name; + } + + private String fullName(String namespace, String typeName) { + int dotPos = typeName.lastIndexOf('.'); + if (dotPos > -1) { + return typeName; + } + return namespace != null ? namespace + "." + typeName : typeName; + } + + private String getString(Token stringToken) { + String stringLiteral = stringToken.getText(); + String betweenQuotes = stringLiteral.substring(1, stringLiteral.length() - 1); + return StringEscapeUtils.unescapeJava(betweenQuotes); + } + } + + private static class SchemaProperties { + String contextNamespace; + boolean withNamespace; + String namespace; + boolean withAliases; + List aliases; + boolean withOrder; + Schema.Field.Order order; + Map properties; + + public SchemaProperties(String contextNamespace, boolean withNamespace, boolean withAliases, boolean withOrder) { + this.contextNamespace = contextNamespace; + this.withNamespace = withNamespace; + this.withAliases = withAliases; + this.aliases = Collections.emptyList(); + this.withOrder = withOrder; + this.order = Schema.Field.Order.ASCENDING; + this.properties = new LinkedHashMap<>(); + } + + public void addProperty(String name, JsonNode value, Token firstValueToken) { + if (withNamespace && "namespace".equals(name)) { + if (value.isTextual()) { + namespace = value.textValue(); + } else { + throw error("@namespace(...) must contain a String value", firstValueToken); + } + } else if (withAliases && "aliases".equals(name)) { + if (value.isArray()) { + List result = new ArrayList<>(); + Iterator elements = value.elements(); + elements.forEachRemaining(element -> { + if (element.isTextual()) { + result.add(element.textValue()); + } else { + throw error("@aliases(...) must contain an array of String values", firstValueToken); + } + }); + aliases = result; + } else { + throw error("@aliases(...) must contain an array of String values", firstValueToken); + } + } else if (withOrder && "order".equals(name)) { + if (value.isTextual()) { + String orderValue = value.textValue().toUpperCase(Locale.ROOT); + switch (orderValue) { + case "ASCENDING": + order = Schema.Field.Order.ASCENDING; + break; + case "DESCENDING": + order = Schema.Field.Order.DESCENDING; + break; + case "IGNORE": + order = Schema.Field.Order.IGNORE; + break; + default: + throw error("@order(...) must contain \"ASCENDING\", \"DESCENDING\" or \"IGNORE\"", firstValueToken); + } + } else { + throw error("@order(...) must contain a String value", firstValueToken); + } + } else { + properties.put(name, value); + } + } + + public String namespace() { + return namespace == null ? contextNamespace : namespace; + } + + public Schema.Field.Order order() { + return order; + } + + public void copyAliases(Consumer addAlias) { + aliases.forEach(addAlias); + } + + public T copyProperties(T jsonProperties) { + properties.forEach(jsonProperties::addProp); + if (jsonProperties instanceof Schema) { + Schema schema = (Schema) jsonProperties; + LogicalType logicalType = LogicalTypes.fromSchemaIgnoreInvalid(schema); + if (logicalType != null) { + logicalType.addToSchema(schema); + } + } + return jsonProperties; + } + + public boolean hasProperties() { + return !properties.isEmpty(); + } + } +} diff --git a/lang/java/idl/src/main/java/org/apache/avro/idl/IsResolvedSchemaVisitor.java b/lang/java/idl/src/main/java/org/apache/avro/idl/IsResolvedSchemaVisitor.java new file mode 100644 index 00000000000..12fd5dbff21 --- /dev/null +++ b/lang/java/idl/src/main/java/org/apache/avro/idl/IsResolvedSchemaVisitor.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.idl; + +import org.apache.avro.Schema; + +/** + * This visitor checks if the current schema is fully resolved. + */ +public final class IsResolvedSchemaVisitor implements SchemaVisitor { + boolean hasUnresolvedParts; + + IsResolvedSchemaVisitor() { + hasUnresolvedParts = false; + } + + @Override + public SchemaVisitorAction visitTerminal(Schema terminal) { + hasUnresolvedParts = SchemaResolver.isUnresolvedSchema(terminal); + return hasUnresolvedParts ? SchemaVisitorAction.TERMINATE : SchemaVisitorAction.CONTINUE; + } + + @Override + public SchemaVisitorAction visitNonTerminal(Schema nonTerminal) { + hasUnresolvedParts = SchemaResolver.isUnresolvedSchema(nonTerminal); + if (hasUnresolvedParts) { + return SchemaVisitorAction.TERMINATE; + } + if (nonTerminal.getType() == Schema.Type.RECORD && !nonTerminal.hasFields()) { + // We're still initializing the type... + return SchemaVisitorAction.SKIP_SUBTREE; + } + return SchemaVisitorAction.CONTINUE; + } + + @Override + public SchemaVisitorAction afterVisitNonTerminal(Schema nonTerminal) { + return SchemaVisitorAction.CONTINUE; + } + + @Override + public Boolean get() { + return !hasUnresolvedParts; + } +} diff --git a/lang/java/idl/src/main/java/org/apache/avro/idl/ResolvingVisitor.java b/lang/java/idl/src/main/java/org/apache/avro/idl/ResolvingVisitor.java new file mode 100644 index 00000000000..04e41f3403a --- /dev/null +++ b/lang/java/idl/src/main/java/org/apache/avro/idl/ResolvingVisitor.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.idl; + +import org.apache.avro.AvroTypeException; +import org.apache.avro.Schema; +import org.apache.avro.Schema.Field; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; + +import static org.apache.avro.Schema.Type.ARRAY; +import static org.apache.avro.Schema.Type.ENUM; +import static org.apache.avro.Schema.Type.FIXED; +import static org.apache.avro.Schema.Type.MAP; +import static org.apache.avro.Schema.Type.RECORD; +import static org.apache.avro.Schema.Type.UNION; + +/** + * This visitor creates clone of the visited Schemata, minus the specified + * schema properties, and resolves all unresolved schemas. + */ +public final class ResolvingVisitor implements SchemaVisitor { + private static final Set CONTAINER_SCHEMA_TYPES = EnumSet.of(RECORD, ARRAY, MAP, UNION); + private static final Set NAMED_SCHEMA_TYPES = EnumSet.of(RECORD, ENUM, FIXED); + + private final Function symbolTable; + private final Set schemaPropertiesToRemove; + private final IdentityHashMap replace; + + private final Schema root; + + public ResolvingVisitor(final Schema root, final Function symbolTable, + String... schemaPropertiesToRemove) { + this(root, symbolTable, new HashSet<>(Arrays.asList(schemaPropertiesToRemove))); + } + + public ResolvingVisitor(final Schema root, final Function symbolTable, + Set schemaPropertiesToRemove) { + this.replace = new IdentityHashMap<>(); + this.symbolTable = symbolTable; + this.schemaPropertiesToRemove = schemaPropertiesToRemove; + + this.root = root; + } + + public ResolvingVisitor withRoot(Schema root) { + return new ResolvingVisitor(root, symbolTable, schemaPropertiesToRemove); + } + + @Override + public SchemaVisitorAction visitTerminal(final Schema terminal) { + Schema.Type type = terminal.getType(); + Schema newSchema; + if (CONTAINER_SCHEMA_TYPES.contains(type)) { + if (!replace.containsKey(terminal)) { + throw new IllegalStateException("Schema " + terminal + " must be already processed"); + } + return SchemaVisitorAction.CONTINUE; + } else if (type == ENUM) { + newSchema = Schema.createEnum(terminal.getName(), terminal.getDoc(), terminal.getNamespace(), + terminal.getEnumSymbols(), terminal.getEnumDefault()); + } else if (type == FIXED) { + newSchema = Schema.createFixed(terminal.getName(), terminal.getDoc(), terminal.getNamespace(), + terminal.getFixedSize()); + } else { + newSchema = Schema.create(type); + } + copyProperties(terminal, newSchema); + replace.put(terminal, newSchema); + return SchemaVisitorAction.CONTINUE; + } + + public void copyProperties(final Schema first, final Schema second) { + // Logical type + Optional.ofNullable(first.getLogicalType()).ifPresent(logicalType -> logicalType.addToSchema(second)); + + // Aliases (if applicable) + if (NAMED_SCHEMA_TYPES.contains(first.getType())) { + first.getAliases().forEach(second::addAlias); + } + + // Other properties + first.getObjectProps().forEach((name, value) -> { + if (!schemaPropertiesToRemove.contains(name)) { + second.addProp(name, value); + } + }); + } + + @Override + public SchemaVisitorAction visitNonTerminal(final Schema nt) { + Schema.Type type = nt.getType(); + if (type == RECORD) { + if (SchemaResolver.isUnresolvedSchema(nt)) { + // unresolved schema will get a replacement that we already encountered, + // or we will attempt to resolve. + final String unresolvedSchemaName = SchemaResolver.getUnresolvedSchemaName(nt); + Schema resSchema = symbolTable.apply(unresolvedSchemaName); + if (resSchema == null) { + throw new AvroTypeException("Unable to resolve " + unresolvedSchemaName); + } + Schema replacement = replace.computeIfAbsent(resSchema, schema -> { + Schemas.visit(schema, this); + return replace.get(schema); + }); + replace.put(nt, replacement); + } else { + // create a fieldless clone. Fields will be added in afterVisitNonTerminal. + Schema newSchema = Schema.createRecord(nt.getName(), nt.getDoc(), nt.getNamespace(), nt.isError()); + copyProperties(nt, newSchema); + replace.put(nt, newSchema); + } + } + return SchemaVisitorAction.CONTINUE; + } + + @Override + public SchemaVisitorAction afterVisitNonTerminal(final Schema nt) { + Schema.Type type = nt.getType(); + Schema newSchema; + switch (type) { + case RECORD: + if (!SchemaResolver.isUnresolvedSchema(nt)) { + newSchema = replace.get(nt); + // Check if we've already handled the replacement schema with a + // reentrant call to visit(...) from within the visitor. + if (!newSchema.hasFields()) { + List fields = nt.getFields(); + List newFields = new ArrayList<>(fields.size()); + for (Schema.Field field : fields) { + newFields.add(new Field(field, replace.get(field.schema()))); + } + newSchema.setFields(newFields); + } + } + return SchemaVisitorAction.CONTINUE; + case UNION: + List types = nt.getTypes(); + List newTypes = new ArrayList<>(types.size()); + for (Schema sch : types) { + newTypes.add(replace.get(sch)); + } + newSchema = Schema.createUnion(newTypes); + break; + case ARRAY: + newSchema = Schema.createArray(replace.get(nt.getElementType())); + break; + case MAP: + newSchema = Schema.createMap(replace.get(nt.getValueType())); + break; + default: + throw new IllegalStateException("Illegal type " + type + ", schema " + nt); + } + copyProperties(nt, newSchema); + replace.put(nt, newSchema); + return SchemaVisitorAction.CONTINUE; + } + + @Override + public Schema get() { + return replace.get(root); + } + + @Override + public String toString() { + return "ResolvingVisitor{symbolTable=" + symbolTable + ", schemaPropertiesToRemove=" + schemaPropertiesToRemove + + ", replace=" + replace + '}'; + } +} diff --git a/lang/java/idl/src/main/java/org/apache/avro/idl/SchemaResolver.java b/lang/java/idl/src/main/java/org/apache/avro/idl/SchemaResolver.java new file mode 100644 index 00000000000..3130f5a2651 --- /dev/null +++ b/lang/java/idl/src/main/java/org/apache/avro/idl/SchemaResolver.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.idl; + +import org.apache.avro.JsonProperties; +import org.apache.avro.Protocol; +import org.apache.avro.Schema; + +import java.util.Collections; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Utility class to resolve schemas that are unavailable at the point they are + * referenced in the IDL. + */ +final class SchemaResolver { + + private SchemaResolver() { + } + + private static final String UR_SCHEMA_ATTR = "org.apache.avro.idl.unresolved.name"; + + private static final String UR_SCHEMA_NAME = "UnresolvedSchema"; + + private static final String UR_SCHEMA_NS = "org.apache.avro.compiler"; + + /** + * Create a schema to represent an "unresolved" schema. (used to represent a + * schema whose definition does not exist, yet). + * + * @param name a schema name + * @return an unresolved schema for the given name + */ + static Schema unresolvedSchema(final String name) { + Schema schema = Schema.createRecord(UR_SCHEMA_NAME, "unresolved schema", UR_SCHEMA_NS, false, + Collections.emptyList()); + schema.addProp(UR_SCHEMA_ATTR, name); + return schema; + } + + /** + * Is this an unresolved schema. + * + * @param schema a schema + * @return whether the schema is an unresolved schema + */ + static boolean isUnresolvedSchema(final Schema schema) { + return (schema.getType() == Schema.Type.RECORD && schema.getProp(UR_SCHEMA_ATTR) != null + && UR_SCHEMA_NAME.equals(schema.getName()) && UR_SCHEMA_NS.equals(schema.getNamespace())); + } + + /** + * Get the unresolved schema name. + * + * @param schema an unresolved schema + * @return the name of the unresolved schema + */ + static String getUnresolvedSchemaName(final Schema schema) { + if (!isUnresolvedSchema(schema)) { + throw new IllegalArgumentException("Not a unresolved schema: " + schema); + } + return schema.getProp(UR_SCHEMA_ATTR); + } + + /** + * Is this an unresolved schema? + */ + static boolean isFullyResolvedSchema(final Schema schema) { + if (isUnresolvedSchema(schema)) { + return false; + } else { + return Schemas.visit(schema, new IsResolvedSchemaVisitor()); + } + } + + /** + * Clone all provided schemas while resolving all unreferenced schemas. + * + * @param idlFile a parsed IDL file + * @return a copy of idlFile with all schemas resolved + */ + static IdlFile resolve(final IdlFile idlFile, String... schemaPropertiesToRemove) { + return new IdlFile(resolve(idlFile.getProtocol(), schemaPropertiesToRemove), idlFile.getWarnings()); + } + + /** + * Will clone the provided protocol while resolving all unreferenced schemas + * + * @param protocol a parsed protocol + * @return a copy of the protocol with all schemas resolved + */ + static Protocol resolve(final Protocol protocol, String... schemaPropertiesToRemove) { + // Create an empty copy of the protocol + Protocol result = new Protocol(protocol.getName(), protocol.getDoc(), protocol.getNamespace()); + protocol.getObjectProps().forEach(((JsonProperties) result)::addProp); + + ResolvingVisitor visitor = new ResolvingVisitor(null, protocol::getType, schemaPropertiesToRemove); + Function resolver = schema -> Schemas.visit(schema, visitor.withRoot(schema)); + + // Resolve all schemata in the protocol. + result.setTypes(protocol.getTypes().stream().map(resolver).collect(Collectors.toList())); + Map resultMessages = result.getMessages(); + protocol.getMessages().forEach((name, oldValue) -> { + Protocol.Message newValue; + if (oldValue.isOneWay()) { + newValue = result.createMessage(oldValue.getName(), oldValue.getDoc(), oldValue, + resolver.apply(oldValue.getRequest())); + } else { + Schema request = resolver.apply(oldValue.getRequest()); + Schema response = resolver.apply(oldValue.getResponse()); + Schema errors = resolver.apply(oldValue.getErrors()); + newValue = result.createMessage(oldValue.getName(), oldValue.getDoc(), oldValue, request, response, errors); + } + resultMessages.put(name, newValue); + }); + return result; + } +} diff --git a/lang/java/idl/src/main/java/org/apache/avro/idl/SchemaVisitor.java b/lang/java/idl/src/main/java/org/apache/avro/idl/SchemaVisitor.java new file mode 100644 index 00000000000..0f9fcae5b68 --- /dev/null +++ b/lang/java/idl/src/main/java/org/apache/avro/idl/SchemaVisitor.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.idl; + +import org.apache.avro.Schema; + +public interface SchemaVisitor { + + /** + * Invoked for schemas that do not have "child" schemas (like string, int …) or + * for a previously encountered schema with children, which will be treated as a + * terminal. (to avoid circular recursion) + */ + SchemaVisitorAction visitTerminal(Schema terminal); + + /** + * Invoked for schema with children before proceeding to visit the children. + */ + SchemaVisitorAction visitNonTerminal(Schema nonTerminal); + + /** + * Invoked for schemas with children after its children have been visited. + */ + SchemaVisitorAction afterVisitNonTerminal(Schema nonTerminal); + + /** + * Invoked when visiting is complete. + * + * @return a value that will be returned by the visit method. + */ + T get(); +} diff --git a/lang/java/idl/src/main/java/org/apache/avro/idl/SchemaVisitorAction.java b/lang/java/idl/src/main/java/org/apache/avro/idl/SchemaVisitorAction.java new file mode 100644 index 00000000000..6aed09b3d32 --- /dev/null +++ b/lang/java/idl/src/main/java/org/apache/avro/idl/SchemaVisitorAction.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.idl; + +public enum SchemaVisitorAction { + + /** + * continue visit. + */ + CONTINUE, + /** + * terminate visit. + */ + TERMINATE, + /** + * when returned from pre non terminal visit method the children of the non + * terminal are skipped. afterVisitNonTerminal for the current schema will not + * be invoked. + */ + SKIP_SUBTREE, + /** + * Skip visiting the siblings of this schema. + */ + SKIP_SIBLINGS +} diff --git a/lang/java/idl/src/main/java/org/apache/avro/idl/Schemas.java b/lang/java/idl/src/main/java/org/apache/avro/idl/Schemas.java new file mode 100644 index 00000000000..da4b949d2bc --- /dev/null +++ b/lang/java/idl/src/main/java/org/apache/avro/idl/Schemas.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.idl; + +import org.apache.avro.Schema; +import org.apache.avro.Schema.Field; + +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.Deque; +import java.util.IdentityHashMap; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * Avro Schema utilities, to traverse... + */ +public final class Schemas { + + private Schemas() { + } + + /** + * Depth first visit. + */ + public static T visit(final Schema start, final SchemaVisitor visitor) { + // Set of Visited Schemas + IdentityHashMap visited = new IdentityHashMap<>(); + // Stack that contains the Schemas to process and afterVisitNonTerminal + // functions. + // Deque>> + // Using Either<...> has a cost we want to avoid... + Deque dq = new ArrayDeque<>(); + dq.push(start); + Object current; + while ((current = dq.poll()) != null) { + if (current instanceof Supplier) { + // We are executing a non-terminal post visit. + @SuppressWarnings("unchecked") + SchemaVisitorAction action = ((Supplier) current).get(); + switch (action) { + case CONTINUE: + break; + case SKIP_SIBLINGS: + while (dq.peek() instanceof Schema) { + dq.remove(); + } + break; + case TERMINATE: + return visitor.get(); + case SKIP_SUBTREE: + default: + throw new UnsupportedOperationException("Invalid action " + action); + } + } else { + Schema schema = (Schema) current; + boolean terminate; + if (visited.containsKey(schema)) { + terminate = visitTerminal(visitor, schema, dq); + } else { + Schema.Type type = schema.getType(); + switch (type) { + case ARRAY: + terminate = visitNonTerminal(visitor, schema, dq, Collections.singleton(schema.getElementType())); + visited.put(schema, schema); + break; + case RECORD: + terminate = visitNonTerminal(visitor, schema, dq, () -> schema.getFields().stream().map(Field::schema) + .collect(Collectors.toCollection(ArrayDeque::new)).descendingIterator()); + visited.put(schema, schema); + break; + case UNION: + terminate = visitNonTerminal(visitor, schema, dq, schema.getTypes()); + visited.put(schema, schema); + break; + case MAP: + terminate = visitNonTerminal(visitor, schema, dq, Collections.singleton(schema.getValueType())); + visited.put(schema, schema); + break; + default: + terminate = visitTerminal(visitor, schema, dq); + break; + } + } + if (terminate) { + return visitor.get(); + } + } + } + return visitor.get(); + } + + private static boolean visitNonTerminal(final SchemaVisitor visitor, final Schema schema, final Deque dq, + final Iterable itSupp) { + SchemaVisitorAction action = visitor.visitNonTerminal(schema); + switch (action) { + case CONTINUE: + dq.push((Supplier) () -> visitor.afterVisitNonTerminal(schema)); + itSupp.forEach(dq::push); + break; + case SKIP_SUBTREE: + dq.push((Supplier) () -> visitor.afterVisitNonTerminal(schema)); + break; + case SKIP_SIBLINGS: + while (dq.peek() instanceof Schema) { + dq.remove(); + } + break; + case TERMINATE: + return true; + default: + throw new UnsupportedOperationException("Invalid action " + action + " for " + schema); + } + return false; + } + + private static boolean visitTerminal(final SchemaVisitor visitor, final Schema schema, final Deque dq) { + SchemaVisitorAction action = visitor.visitTerminal(schema); + switch (action) { + case CONTINUE: + break; + case SKIP_SIBLINGS: + while (dq.peek() instanceof Schema) { + dq.remove(); + } + break; + case TERMINATE: + return true; + case SKIP_SUBTREE: + default: + throw new UnsupportedOperationException("Invalid action " + action + " for " + schema); + } + return false; + } +} diff --git a/lang/java/idl/src/main/resources/META-INF/LICENSE b/lang/java/idl/src/main/resources/META-INF/LICENSE new file mode 100644 index 00000000000..62589edd12a --- /dev/null +++ b/lang/java/idl/src/main/resources/META-INF/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/lang/java/idl/src/test/idl/AnnotationOnTypeReference.avdl b/lang/java/idl/src/test/idl/AnnotationOnTypeReference.avdl new file mode 100644 index 00000000000..03f6f7c6f27 --- /dev/null +++ b/lang/java/idl/src/test/idl/AnnotationOnTypeReference.avdl @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A stripped down version of a previous `simple.avdl`, keeping the part where a type reference had an annotation (this is wrong). + */ +@namespace("org.apache.avro.test") +protocol Simple { + /** An MD5 hash. */ + fixed MD5(16); + + /** A TestRecord. */ + record TestRecord { + @foo("bar") MD5 hash = "0000000000000000"; + } +} diff --git a/lang/java/idl/src/test/idl/cycle.avdl b/lang/java/idl/src/test/idl/cycle.avdl new file mode 100644 index 00000000000..fe5bbbe4f0f --- /dev/null +++ b/lang/java/idl/src/test/idl/cycle.avdl @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@namespace("org.apache.avro.gen.test") +protocol Cycle { + + record Record1 { + string fString = ""; + Record3 rec3; + } + + record Record2 { + TestFixed fFixed; + int val; + union {null, Record1} fRec1; + } + + record Record3 { + TestEnum fEnum; + Record2 rec2; + } + + enum TestEnum { bla, blu } + + fixed TestFixed(16); + +} diff --git a/lang/java/idl/src/test/idl/extra/protocolSyntax.avdl b/lang/java/idl/src/test/idl/extra/protocolSyntax.avdl new file mode 100644 index 00000000000..8edc79876cc --- /dev/null +++ b/lang/java/idl/src/test/idl/extra/protocolSyntax.avdl @@ -0,0 +1,9 @@ +@namespace("communication") +protocol Parrot { + record Message { + string? title; + string message; + } + + Message echo(Message message); +} diff --git a/lang/java/idl/src/test/idl/input/bar.avpr b/lang/java/idl/src/test/idl/input/bar.avpr new file mode 100644 index 00000000000..5e9b194a060 --- /dev/null +++ b/lang/java/idl/src/test/idl/input/bar.avpr @@ -0,0 +1,2 @@ +{"protocol": "org.foo.Bar", + "messages": { "bar": {"request": [], "response": "null"}}} diff --git a/lang/java/idl/src/test/idl/input/baseball.avdl b/lang/java/idl/src/test/idl/input/baseball.avdl new file mode 100644 index 00000000000..e485e89ba2a --- /dev/null +++ b/lang/java/idl/src/test/idl/input/baseball.avdl @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@namespace("avro.examples.baseball") +protocol Baseball { + import schema "position.avsc"; + import schema "player.avsc"; +} diff --git a/lang/java/idl/src/test/idl/input/baz.avsc b/lang/java/idl/src/test/idl/input/baz.avsc new file mode 100644 index 00000000000..efd68d97354 --- /dev/null +++ b/lang/java/idl/src/test/idl/input/baz.avsc @@ -0,0 +1,3 @@ +{"type": "record", "name": "ns.other.schema.Baz", + "fields": [ {"name": "x", "type": "int"} ] +} diff --git a/lang/java/idl/src/test/idl/input/comments.avdl b/lang/java/idl/src/test/idl/input/comments.avdl new file mode 100644 index 00000000000..76be4e314bf --- /dev/null +++ b/lang/java/idl/src/test/idl/input/comments.avdl @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@namespace("testing") +protocol Comments { + /** Documented Enum */ + enum /** Dangling Enum1 */ DocumentedEnum /** Dangling Enum2 */ { + /** Dangling Enum3 */ A, + /** Dangling Enum4 */ B, + /** Dangling Enum5 */ C + /** Dangling Enum6 */} + /** Dangling Enum7 */= + /** Dangling Enum8 */ A + /** Dangling Enum9 */; + + enum UndocumentedEnum {D,E} + + /** Documented Fixed Type */ fixed + /** Dangling Fixed1 */ DocumentedFixed + /** Dangling Fixed2 */( + /** Dangling Fixed3 */ 16 + /** Dangling Fixed4 */) + /** Dangling Fixed5 */; + + fixed UndocumentedFixed(16); + + /** Documented Error */ error + /** Dangling Error1 */ DocumentedError + /** Dangling Field1 */{ + /** Default Doc Explanation Field */string + /** Documented Reason Field */reason, explanation + /** Dangling Field2 */; + /** Dangling Error2 */} + + record UndocumentedRecord { + string description; + } + + /** Documented Method */ void + /** Dangling Param1 */ documentedMethod + /** Dangling Param2 */( + string /** Documented Parameter */ message, + /** Default Documented Parameter */ string defMsg + /** Dangling Param3 */) + /** Dangling Method1 */ throws + /** Dangling Method2 */ DocumentedError + /** Dangling Method3 */; + + void undocumentedMethod(string message); +} diff --git a/lang/java/idl/src/test/idl/input/cycle.avdl b/lang/java/idl/src/test/idl/input/cycle.avdl new file mode 100644 index 00000000000..fc6fee954f6 --- /dev/null +++ b/lang/java/idl/src/test/idl/input/cycle.avdl @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@namespace("org.apache.avro.gen") +protocol Cycle { + + record SampleNode { + int count = 0; + array subNodes; + } + + record Method { + string @testAttribute("testValue") declaringClass; + string methodName; + } + + record SamplePair { + Method method; + SampleNode node; + } + + record SelfRef { + string something; + array subNodes = []; + } + +} diff --git a/lang/java/idl/src/test/idl/input/echo.avdl b/lang/java/idl/src/test/idl/input/echo.avdl new file mode 100644 index 00000000000..1673a125506 --- /dev/null +++ b/lang/java/idl/src/test/idl/input/echo.avdl @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@namespace("org.apache.avro.echo") +protocol Echo { + record Ping { + long timestamp = -1; + string text = ""; + } + + record Pong { + long timestamp = -1; + Ping ping; + } + + Pong ping(Ping ping); +} diff --git a/lang/java/idl/src/test/idl/input/foo.avsc b/lang/java/idl/src/test/idl/input/foo.avsc new file mode 100644 index 00000000000..83267ea4760 --- /dev/null +++ b/lang/java/idl/src/test/idl/input/foo.avsc @@ -0,0 +1,3 @@ +{"type": "record", "name": "org.foo.Foo", + "fields": [ {"name": "x", "type": "int"} ] +} diff --git a/lang/java/idl/src/test/idl/input/forward_ref.avdl b/lang/java/idl/src/test/idl/input/forward_ref.avdl new file mode 100644 index 00000000000..b75d60a4efd --- /dev/null +++ b/lang/java/idl/src/test/idl/input/forward_ref.avdl @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@namespace("org.foo") +protocol Import { + /* Name Value record */ + record ANameValue { + /** the name */ + string name; + /** the value */ + string value; + /* is the value a json object */ + ValueType type = "PLAIN"; + } + + enum ValueType { + JSON, BASE64BIN, PLAIN + } +} diff --git a/lang/java/idl/src/test/idl/input/import.avdl b/lang/java/idl/src/test/idl/input/import.avdl new file mode 100644 index 00000000000..8cd6a163759 --- /dev/null +++ b/lang/java/idl/src/test/idl/input/import.avdl @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@namespace("org.foo") +protocol Import { + import idl "reservedwords.avdl"; + import idl "nestedimport.avdl"; + + //Note that this import is resolve via the classpath, not relative path. + import idl "OnTheClasspath.avdl"; + import protocol "OnTheClasspath.avpr"; + import schema "OnTheClasspath.avsc"; + + import schema "baz.avsc"; + import schema "foo.avsc"; + import protocol "bar.avpr"; + + record Bar { + ns.other.schema.Baz baz; + Foo foo; + } + + void bazm(ns.other.schema.Baz baz); + Bar barf(Foo foo); +} diff --git a/lang/java/idl/src/test/idl/input/interop.avdl b/lang/java/idl/src/test/idl/input/interop.avdl new file mode 100644 index 00000000000..497a7902aac --- /dev/null +++ b/lang/java/idl/src/test/idl/input/interop.avdl @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Currently genavro only does Protocols. +@namespace("org.apache.avro") +protocol InteropProtocol { + record Foo { + string label; + } + + enum Kind { A, B, C } + fixed MD5(16); + + record Node { + string label; + array children = []; + } + + record Interop { + int intField = 1; + long longField = -1; + string stringField; + boolean boolField = false; + float floatField = 0.0; + double doubleField = -1.0e12; + null nullField; + array arrayField = []; + map mapField; + union { boolean, double, array } unionFIeld; + Kind enumField; + MD5 fixedField; + Node recordField; + } + +} diff --git a/lang/java/idl/src/test/idl/input/mr_events.avdl b/lang/java/idl/src/test/idl/input/mr_events.avdl new file mode 100644 index 00000000000..ffb90e97193 --- /dev/null +++ b/lang/java/idl/src/test/idl/input/mr_events.avdl @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Genavro format for a particular protocol found in Hadoop MapReduce. + * Used as a test case/example to show that we can express real-world stuff more + * succinctly. + */ +@namespace("org.apache.hadoop.mapreduce.jobhistory") +protocol Events { + record JhCounter { + string name; + string displayName; + long value; + } + + record JhCounterGroup { + string name; + string displayName; + array counts; + } + + record JhCounters { + string name; + array groups; + } + + record JobFinished { + string jobid; + timestamp_ms finishTime; + decimal(9,2) finishRatio; + int finishedMaps; + int finishedReduces; + int failedMaps; + int failedReduces; + JhCounters totalCounters; + JhCounters mapCounters; + JhCounters reduceCounters; + } + + record JobInited { + string jobid; + timestamp_ms launchTime; + int totalMaps; + int totalReduces; + string jobStatus; + } + + record JobSubmitted { + string jobid; + string jobName; + string userName; + timestamp_ms submitTime; + local_timestamp_ms submitTimeLocal; + string jobConfPath; + } + + // ... TODO continue +} diff --git a/lang/java/idl/src/test/idl/input/namespaces.avdl b/lang/java/idl/src/test/idl/input/namespaces.avdl new file mode 100644 index 00000000000..9eb2027ab0c --- /dev/null +++ b/lang/java/idl/src/test/idl/input/namespaces.avdl @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@namespace("avro.test.protocol") +protocol TestNamespace { + @namespace("avro.test.fixed") + fixed FixedInOtherNamespace(16); + + fixed FixedInThisNamespace(16); + + @namespace("avro.test.record") + record RecordInOtherNamespace {} + + @namespace("avro.test.error") + error ErrorInOtherNamespace {} + + @namespace("avro.test.enum") + enum EnumInOtherNamespace { FOO } + + record RefersToOthers { + avro.test.fixed.FixedInOtherNamespace someFixed; + avro.test.record.RecordInOtherNamespace someRecord; + avro.test.error.ErrorInOtherNamespace someError; + avro.test.enum.EnumInOtherNamespace someEnum; + FixedInThisNamespace thisFixed; + } +} diff --git a/lang/java/idl/src/test/idl/input/nestedimport.avdl b/lang/java/idl/src/test/idl/input/nestedimport.avdl new file mode 100644 index 00000000000..13c6981361a --- /dev/null +++ b/lang/java/idl/src/test/idl/input/nestedimport.avdl @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@version("1.0.5") +@namespace("org.apache.avro.ipc.specific") +protocol nestedimport { + import idl "reservedwords.avdl"; + import protocol "bar.avpr"; + import schema "position.avsc"; + import schema "player.avsc"; + + record ImportBar { + avro.examples.baseball.Player foo; + } + +} \ No newline at end of file diff --git a/lang/java/idl/src/test/idl/input/player.avsc b/lang/java/idl/src/test/idl/input/player.avsc new file mode 100644 index 00000000000..0492850ef5e --- /dev/null +++ b/lang/java/idl/src/test/idl/input/player.avsc @@ -0,0 +1,8 @@ +{"type":"record", "name":"Player", "namespace": "avro.examples.baseball", + "fields": [ + {"name": "number", "type": "int"}, + {"name": "first_name", "type": "string"}, + {"name": "last_name", "type": "string"}, + {"name": "position", "type": {"type": "array", "items": "Position"} } + ] +} diff --git a/lang/java/idl/src/test/idl/input/position.avsc b/lang/java/idl/src/test/idl/input/position.avsc new file mode 100644 index 00000000000..a47065a35f4 --- /dev/null +++ b/lang/java/idl/src/test/idl/input/position.avsc @@ -0,0 +1,3 @@ +{"type":"enum", "name": "Position", "namespace": "avro.examples.baseball", + "symbols": ["P", "C", "B1", "B2", "B3", "SS", "LF", "CF", "RF", "DH"] +} diff --git a/lang/java/idl/src/test/idl/input/reservedwords.avdl b/lang/java/idl/src/test/idl/input/reservedwords.avdl new file mode 100644 index 00000000000..f2112aae7f2 --- /dev/null +++ b/lang/java/idl/src/test/idl/input/reservedwords.avdl @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +protocol Foo { + void `error`(); + void `void`(); + void `idl`(); + void `import`(); + void `oneway`(); + void `null`(); + void `local_timestamp_ms`(); +} diff --git a/lang/java/idl/src/test/idl/input/simple.avdl b/lang/java/idl/src/test/idl/input/simple.avdl new file mode 100644 index 00000000000..27949547a1d --- /dev/null +++ b/lang/java/idl/src/test/idl/input/simple.avdl @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A simple test case. + */ +@version("1.0.5") +@namespace("org.apache.avro.test") +protocol Simple { + /** A kind of record. */ + @aliases(["org.foo.KindOf"]) + enum Kind { + FOO, + BAR, // the bar enum value + BAZ + } + + enum Status { + A, + B, + C + } = C; // C is the default value used when reading unknown values from another schema version (without it, reading throws an exception). + + /** A TestRecord. */ + @my-property({"key":3}) + record TestRecord { + // Tests that keywords can also appear in identifiers. + @avro.java.`string`("String") string @order("ignore") name = "foo"; + + /** The kind of record. */ + Kind @order("descending") kind; + + /** The status of the record. */ + Status status = "A"; + + MD5 hash = "0000000000000000"; + + // A traditional optional field + union {null, MD5} @aliases(["hash", "hsh"]) nullableHash = null; + + // These two fields parse correctly, but will brewak (be changed to strings) when serializing the protocol/schema as JSON. + double value = NaN; + float average = -Infinity; + date d = 0; + // An optional type with a non-null default value (results in a union with null last). + time_ms? t = 0; + + @foo.bar("bar.foo") long l = 0; + // Arrays (and maps) may also have properties + @foo.bar.bar("foo.bar2") array a = []; + // An optional type with a null default value (results in a union with null first). + @foo.foo.bar(42) @foo.foo.foo("3foo") string? prop = null; + } + + /** An MD5 hash. */ + fixed MD5(0x10); + + error TestError { + string message; + } + + /** method 'hello' takes @parameter 'greeting' */ + string hello(string greeting); + // The value of TestRecord also contains defaults for fields not mentioned. + TestRecord echo(TestRecord `record` = {"name":"bar","kind":"BAR"}); + /** method 'add' takes @parameter 'arg1' @parameter 'arg2' */ + @specialProp("test") + int add(int arg1, int arg2 = 0); + bytes echoBytes(bytes data); + void `error`() throws TestError; + void ping() oneway; +} diff --git a/lang/java/idl/src/test/idl/input/unicode.avdl b/lang/java/idl/src/test/idl/input/unicode.avdl new file mode 100644 index 00000000000..f16fb2f95dc --- /dev/null +++ b/lang/java/idl/src/test/idl/input/unicode.avdl @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** +* This is a test that UTF8 functions correctly. +* このテストでは、UTF - 8で正しく機能している。 +* 这是一个测试，UTF - 8的正常运行。 +*/ +protocol Протоколы { + record Структура { + string Строковый; + string 文字列; + } +} diff --git a/lang/java/idl/src/test/idl/input/uuid.avdl b/lang/java/idl/src/test/idl/input/uuid.avdl new file mode 100644 index 00000000000..49a83f45eaa --- /dev/null +++ b/lang/java/idl/src/test/idl/input/uuid.avdl @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +/** +Testing UUID fields +*/ + +@namespace("org.apache.avro") +protocol MyProtocol { + record APlaygroundEvent { + + /** + * Documentation must be provided for each attribute + */ + uuid identifier; + + /** + * A string field with a special name + */ + string `uuid`; + + + /** + * a nullable uuid field + */ + + union { null, uuid } optionalString; + } +} diff --git a/lang/java/idl/src/test/idl/logicalTypes.avdl b/lang/java/idl/src/test/idl/logicalTypes.avdl new file mode 100644 index 00000000000..9e4a284da9b --- /dev/null +++ b/lang/java/idl/src/test/idl/logicalTypes.avdl @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@version("1.0.5") +@namespace("org.apache.avro.test") +protocol LogicalTypeTest { + record LogicalTypeFields { + date aDate; + time_ms aTime; + timestamp_ms aTimestamp; + local_timestamp_ms aLocalTimestamp; + decimal(6,2) pocketMoney; + uuid identifier; + @logicalType("timestamp-micros") long anotherTimestamp; + @logicalType("decimal") @precision(6) @scale(2) bytes allowance; + @logicalType("decimal") @precision(3000000000) @scale(0) bytes byteArray; + } +} diff --git a/lang/java/idl/src/test/idl/output/baseball.avpr b/lang/java/idl/src/test/idl/output/baseball.avpr new file mode 100644 index 00000000000..715cdde8847 --- /dev/null +++ b/lang/java/idl/src/test/idl/output/baseball.avpr @@ -0,0 +1,31 @@ +{ + "protocol" : "Baseball", + "namespace" : "avro.examples.baseball", + "doc" : "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n https://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.", + "types" : [ { + "type" : "enum", + "name" : "Position", + "symbols" : [ "P", "C", "B1", "B2", "B3", "SS", "LF", "CF", "RF", "DH" ] + }, { + "type" : "record", + "name" : "Player", + "fields" : [ { + "name" : "number", + "type" : "int" + }, { + "name" : "first_name", + "type" : "string" + }, { + "name" : "last_name", + "type" : "string" + }, { + "name" : "position", + "type" : { + "type" : "array", + "items" : "Position" + } + } ] + } ], + "messages" : { + } +} diff --git a/lang/java/idl/src/test/idl/output/comments.avpr b/lang/java/idl/src/test/idl/output/comments.avpr new file mode 100644 index 00000000000..9901f8eebc8 --- /dev/null +++ b/lang/java/idl/src/test/idl/output/comments.avpr @@ -0,0 +1,67 @@ +{ + "protocol" : "Comments", + "namespace" : "testing", + "types" : [ { + "type" : "enum", + "name" : "DocumentedEnum", + "doc" : "Documented Enum", + "symbols" : [ "A", "B", "C" ], + "default" : "A" + }, { + "type" : "enum", + "name" : "UndocumentedEnum", + "symbols" : [ "D", "E" ] + }, { + "type" : "fixed", + "name" : "DocumentedFixed", + "doc" : "Documented Fixed Type", + "size" : 16 + }, { + "type" : "fixed", + "name" : "UndocumentedFixed", + "size" : 16 + }, { + "type" : "error", + "name" : "DocumentedError", + "doc" : "Documented Error", + "fields" : [ { + "name" : "reason", + "type" : "string", + "doc" : "Documented Reason Field" + }, { + "name" : "explanation", + "type" : "string", + "doc" : "Default Doc Explanation Field" + } ] + }, { + "type" : "record", + "name" : "UndocumentedRecord", + "fields" : [ { + "name" : "description", + "type" : "string" + } ] + } ], + "messages" : { + "documentedMethod" : { + "doc" : "Documented Method", + "request" : [ { + "name" : "message", + "type" : "string", + "doc" : "Documented Parameter" + }, { + "name" : "defMsg", + "type" : "string", + "doc" : "Default Documented Parameter" + } ], + "response" : "null", + "errors" : [ "DocumentedError" ] + }, + "undocumentedMethod" : { + "request" : [ { + "name" : "message", + "type" : "string" + } ], + "response" : "null" + } + } +} diff --git a/lang/java/idl/src/test/idl/output/cycle.avpr b/lang/java/idl/src/test/idl/output/cycle.avpr new file mode 100644 index 00000000000..e8b15835c7a --- /dev/null +++ b/lang/java/idl/src/test/idl/output/cycle.avpr @@ -0,0 +1,55 @@ +{ + "protocol" : "Cycle", + "namespace" : "org.apache.avro.gen", + "types" : [ { + "type" : "record", + "name" : "SampleNode", + "fields" : [ { + "name" : "count", + "type" : "int", + "default" : 0 + }, { + "name" : "subNodes", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "SamplePair", + "fields" : [ { + "name" : "method", + "type" : { + "type" : "record", + "name" : "Method", + "fields" : [ { + "name" : "declaringClass", + "type" : "string", + "testAttribute":"testValue" + }, { + "name" : "methodName", + "type" : "string" + } ] + } + }, { + "name" : "node", + "type" : "SampleNode" + } ] + } + } + } ] + }, { + "type" : "record", + "name" : "SelfRef", + "fields" : [ { + "name" : "something", + "type" : "string" + }, { + "name" : "subNodes", + "type" : { + "type" : "array", + "items" : "SelfRef" + }, + "default" : [ ] + } ] + } ], + "messages" : { } +} diff --git a/lang/java/idl/src/test/idl/output/echo.avpr b/lang/java/idl/src/test/idl/output/echo.avpr new file mode 100644 index 00000000000..dbf9b99134a --- /dev/null +++ b/lang/java/idl/src/test/idl/output/echo.avpr @@ -0,0 +1,37 @@ +{ + "protocol" : "Echo", + "namespace" : "org.apache.avro.echo", + "types" : [ { + "type" : "record", + "name" : "Ping", + "fields" : [ { + "name" : "timestamp", + "type" : "long", + "default" : -1 + }, { + "name" : "text", + "type" : "string", + "default" : "" + } ] + }, { + "type" : "record", + "name" : "Pong", + "fields" : [ { + "name" : "timestamp", + "type" : "long", + "default" : -1 + }, { + "name" : "ping", + "type" : "Ping" + } ] + } ], + "messages" : { + "ping" : { + "request" : [ { + "name" : "ping", + "type" : "Ping" + } ], + "response" : "Pong" + } + } +} \ No newline at end of file diff --git a/lang/java/idl/src/test/idl/output/forward_ref.avpr b/lang/java/idl/src/test/idl/output/forward_ref.avpr new file mode 100644 index 00000000000..a349206a37f --- /dev/null +++ b/lang/java/idl/src/test/idl/output/forward_ref.avpr @@ -0,0 +1,16 @@ +{ + "protocol": "Import", + "namespace": "org.foo", + "types": [ + { + "type": "record", + "name": "ANameValue", + "fields": [ + { "name":"name", "type": "string", "doc":"the name" }, + { "name": "value", "type": "string", "doc": "the value" }, + { "name": "type", "type": { "type": "enum", "name":"ValueType", "symbols": ["JSON","BASE64BIN","PLAIN"] }, "default": "PLAIN" } + ] + } + ], + "messages": { } +} diff --git a/lang/java/idl/src/test/idl/output/import.avpr b/lang/java/idl/src/test/idl/output/import.avpr new file mode 100644 index 00000000000..e6701ad94e1 --- /dev/null +++ b/lang/java/idl/src/test/idl/output/import.avpr @@ -0,0 +1,132 @@ +{ + "protocol" : "Import", + "namespace" : "org.foo", + "doc" : "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n https://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.", + "types" : [ { + "type" : "enum", + "name" : "Position", + "namespace" : "avro.examples.baseball", + "symbols" : [ "P", "C", "B1", "B2", "B3", "SS", "LF", "CF", "RF", "DH" ] + }, { + "type" : "record", + "name" : "Player", + "namespace" : "avro.examples.baseball", + "fields" : [ { + "name" : "number", + "type" : "int" + }, { + "name" : "first_name", + "type" : "string" + }, { + "name" : "last_name", + "type" : "string" + }, { + "name" : "position", + "type" : { + "type" : "array", + "items" : "Position" + } + } ] + }, { + "type" : "record", + "name" : "ImportBar", + "namespace" : "org.apache.avro.ipc.specific", + "fields" : [ { + "name" : "foo", + "type" : "avro.examples.baseball.Player" + } ] + }, { + "type" : "record", + "name" : "NestedType", + "namespace" : "org.on.the.classpath", + "fields" : [ ] + }, { + "type" : "record", + "name" : "FromAfar", + "namespace" : "org.on.the.classpath", + "fields" : [ ] + }, { + "type" : "record", + "name" : "VeryFar", + "namespace" : "org.on.the.classpath", + "fields" : [ ] + }, { + "type" : "record", + "name" : "FarAway", + "namespace" : "org.on.the.classpath", + "fields" : [ ] + }, { + "type" : "record", + "name" : "Baz", + "namespace" : "ns.other.schema", + "fields" : [ { + "name" : "x", + "type" : "int" + } ] + }, { + "type" : "record", + "name" : "Foo", + "fields" : [ { + "name" : "x", + "type" : "int" + } ] + }, { + "type" : "record", + "name" : "Bar", + "fields" : [ { + "name" : "baz", + "type" : "ns.other.schema.Baz" + }, { + "name" : "foo", + "type" : "Foo" + } ] + } ], + "messages" : { + "error" : { + "request" : [ ], + "response" : "null" + }, + "void" : { + "request" : [ ], + "response" : "null" + }, + "idl" : { + "request" : [ ], + "response" : "null" + }, + "import" : { + "request" : [ ], + "response" : "null" + }, + "oneway" : { + "request" : [ ], + "response" : "null" + }, + "null" : { + "request" : [ ], + "response" : "null" + }, + "local_timestamp_ms" : { + "request" : [ ], + "response" : "null" + }, + "bar" : { + "request" : [ ], + "response" : "null" + }, + "bazm" : { + "request" : [ { + "name" : "baz", + "type" : "ns.other.schema.Baz" + } ], + "response" : "null" + }, + "barf" : { + "request" : [ { + "name" : "foo", + "type" : "Foo" + } ], + "response" : "Bar" + } + } +} diff --git a/lang/java/idl/src/test/idl/output/interop.avpr b/lang/java/idl/src/test/idl/output/interop.avpr new file mode 100644 index 00000000000..6e56c7c9cee --- /dev/null +++ b/lang/java/idl/src/test/idl/output/interop.avpr @@ -0,0 +1,94 @@ +{ + "protocol" : "InteropProtocol", + "namespace" : "org.apache.avro", + "doc" : "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n https://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.", + "types" : [ { + "type" : "record", + "name" : "Foo", + "fields" : [ { + "name" : "label", + "type" : "string" + } ] + }, { + "type" : "enum", + "name" : "Kind", + "symbols" : [ "A", "B", "C" ] + }, { + "type" : "fixed", + "name" : "MD5", + "size" : 16 + }, { + "type" : "record", + "name" : "Node", + "fields" : [ { + "name" : "label", + "type" : "string" + }, { + "name" : "children", + "type" : { + "type" : "array", + "items" : "Node" + }, + "default" : [ ] + } ] + }, { + "type" : "record", + "name" : "Interop", + "fields" : [ { + "name" : "intField", + "type" : "int", + "default" : 1 + }, { + "name" : "longField", + "type" : "long", + "default" : -1 + }, { + "name" : "stringField", + "type" : "string" + }, { + "name" : "boolField", + "type" : "boolean", + "default" : false + }, { + "name" : "floatField", + "type" : "float", + "default" : 0.0 + }, { + "name" : "doubleField", + "type" : "double", + "default" : -1.0E12 + }, { + "name" : "nullField", + "type" : "null" + }, { + "name" : "arrayField", + "type" : { + "type" : "array", + "items" : "double" + }, + "default" : [ ] + }, { + "name" : "mapField", + "type" : { + "type" : "map", + "values" : "Foo" + } + }, { + "name" : "unionFIeld", + "type" : [ "boolean", "double", { + "type" : "array", + "items" : "bytes" + } ] + }, { + "name" : "enumField", + "type" : "Kind" + }, { + "name" : "fixedField", + "type" : "MD5" + }, { + "name" : "recordField", + "type" : "Node" + } ] + } ], + "messages" : { } +} diff --git a/lang/java/idl/src/test/idl/output/mr_events.avpr b/lang/java/idl/src/test/idl/output/mr_events.avpr new file mode 100644 index 00000000000..8c6343664d6 --- /dev/null +++ b/lang/java/idl/src/test/idl/output/mr_events.avpr @@ -0,0 +1,125 @@ +{ + "protocol" : "Events", + "namespace" : "org.apache.hadoop.mapreduce.jobhistory", + "doc" : "Genavro format for a particular protocol found in Hadoop MapReduce.\nUsed as a test case/example to show that we can express real-world stuff more\nsuccinctly.", + "types" : [ { + "type" : "record", + "name" : "JhCounter", + "fields" : [ { + "name" : "name", + "type" : "string" + }, { + "name" : "displayName", + "type" : "string" + }, { + "name" : "value", + "type" : "long" + } ] + }, { + "type" : "record", + "name" : "JhCounterGroup", + "fields" : [ { + "name" : "name", + "type" : "string" + }, { + "name" : "displayName", + "type" : "string" + }, { + "name" : "counts", + "type" : { + "type" : "array", + "items" : "JhCounter" + } + } ] + }, { + "type" : "record", + "name" : "JhCounters", + "fields" : [ { + "name" : "name", + "type" : "string" + }, { + "name" : "groups", + "type" : { + "type" : "array", + "items" : "JhCounterGroup" + } + } ] + }, { + "type" : "record", + "name" : "JobFinished", + "fields" : [ { + "name" : "jobid", + "type" : "string" + }, { + "name" : "finishTime", + "type" : {"type": "long", "logicalType": "timestamp-millis"} + }, { + "name" : "finishRatio", + "type" : {"type": "bytes", "logicalType": "decimal", "precision": 9, "scale": 2} + }, { + "name" : "finishedMaps", + "type" : "int" + }, { + "name" : "finishedReduces", + "type" : "int" + }, { + "name" : "failedMaps", + "type" : "int" + }, { + "name" : "failedReduces", + "type" : "int" + }, { + "name" : "totalCounters", + "type" : "JhCounters" + }, { + "name" : "mapCounters", + "type" : "JhCounters" + }, { + "name" : "reduceCounters", + "type" : "JhCounters" + } ] + }, { + "type" : "record", + "name" : "JobInited", + "fields" : [ { + "name" : "jobid", + "type" : "string" + }, { + "name" : "launchTime", + "type" : {"type": "long", "logicalType": "timestamp-millis"} + }, { + "name" : "totalMaps", + "type" : "int" + }, { + "name" : "totalReduces", + "type" : "int" + }, { + "name" : "jobStatus", + "type" : "string" + } ] + }, { + "type" : "record", + "name" : "JobSubmitted", + "fields" : [ { + "name" : "jobid", + "type" : "string" + }, { + "name" : "jobName", + "type" : "string" + }, { + "name" : "userName", + "type" : "string" + }, { + "name" : "submitTime", + "type" : {"type": "long", "logicalType": "timestamp-millis"} + }, { + "name" : "submitTimeLocal", + "type" : {"type": "long", "logicalType": "local-timestamp-millis"} + }, { + "name" : "jobConfPath", + "type" : "string" + } ] + } ], + "messages" : { + } +} diff --git a/lang/java/idl/src/test/idl/output/namespaces.avpr b/lang/java/idl/src/test/idl/output/namespaces.avpr new file mode 100644 index 00000000000..22ec760d5bf --- /dev/null +++ b/lang/java/idl/src/test/idl/output/namespaces.avpr @@ -0,0 +1,51 @@ +{ + "protocol" : "TestNamespace", + "namespace" : "avro.test.protocol", + "doc" : "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n https://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.", + "types" : [ { + "type" : "fixed", + "name" : "FixedInOtherNamespace", + "namespace" : "avro.test.fixed", + "size" : 16 + }, { + "type" : "fixed", + "name" : "FixedInThisNamespace", + "size" : 16 + }, { + "type" : "record", + "name" : "RecordInOtherNamespace", + "namespace" : "avro.test.record", + "fields" : [ ] + }, { + "type" : "error", + "name" : "ErrorInOtherNamespace", + "namespace" : "avro.test.error", + "fields" : [ ] + }, { + "type" : "enum", + "name" : "EnumInOtherNamespace", + "namespace" : "avro.test.enum", + "symbols" : [ "FOO" ] + }, { + "type" : "record", + "name" : "RefersToOthers", + "fields" : [ { + "name" : "someFixed", + "type" : "avro.test.fixed.FixedInOtherNamespace" + }, { + "name" : "someRecord", + "type" : "avro.test.record.RecordInOtherNamespace" + }, { + "name" : "someError", + "type" : "avro.test.error.ErrorInOtherNamespace" + }, { + "name" : "someEnum", + "type" : "avro.test.enum.EnumInOtherNamespace" + }, { + "name" : "thisFixed", + "type" : "FixedInThisNamespace" + } ] + } ], + "messages" : { + } +} diff --git a/lang/java/idl/src/test/idl/output/nestedimport.avpr b/lang/java/idl/src/test/idl/output/nestedimport.avpr new file mode 100644 index 00000000000..80273627109 --- /dev/null +++ b/lang/java/idl/src/test/idl/output/nestedimport.avpr @@ -0,0 +1,73 @@ +{ + "protocol" : "nestedimport", + "namespace" : "org.apache.avro.ipc.specific", + "doc" : "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n https://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.", + "version" : "1.0.5", + "types" : [ { + "type" : "enum", + "name" : "Position", + "namespace" : "avro.examples.baseball", + "symbols" : [ "P", "C", "B1", "B2", "B3", "SS", "LF", "CF", "RF", "DH" ] + }, { + "type" : "record", + "name" : "Player", + "namespace" : "avro.examples.baseball", + "fields" : [ { + "name" : "number", + "type" : "int" + }, { + "name" : "first_name", + "type" : "string" + }, { + "name" : "last_name", + "type" : "string" + }, { + "name" : "position", + "type" : { + "type" : "array", + "items" : "Position" + } + } ] + }, { + "type" : "record", + "name" : "ImportBar", + "fields" : [ { + "name" : "foo", + "type" : "avro.examples.baseball.Player" + } ] + } ], + "messages" : { + "error" : { + "request" : [ ], + "response" : "null" + }, + "void" : { + "request" : [ ], + "response" : "null" + }, + "idl" : { + "request" : [ ], + "response" : "null" + }, + "import" : { + "request" : [ ], + "response" : "null" + }, + "oneway" : { + "request" : [ ], + "response" : "null" + }, + "null" : { + "request" : [ ], + "response" : "null" + }, + "local_timestamp_ms" : { + "request" : [ ], + "response" : "null" + }, + "bar" : { + "request" : [ ], + "response" : "null" + } + } +} diff --git a/lang/java/idl/src/test/idl/output/reservedwords.avpr b/lang/java/idl/src/test/idl/output/reservedwords.avpr new file mode 100644 index 00000000000..f28a90cc8b3 --- /dev/null +++ b/lang/java/idl/src/test/idl/output/reservedwords.avpr @@ -0,0 +1,35 @@ +{ + "protocol" : "Foo", + "doc" : "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n https://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.", + "types" : [ ], + "messages" : { + "error" : { + "request" : [ ], + "response" : "null" + }, + "void" : { + "request" : [ ], + "response" : "null" + }, + "idl" : { + "request" : [ ], + "response" : "null" + }, + "import" : { + "request" : [ ], + "response" : "null" + }, + "oneway" : { + "request" : [ ], + "response" : "null" + }, + "null" : { + "request" : [ ], + "response" : "null" + }, + "local_timestamp_ms" : { + "request" : [ ], + "response" : "null" + } + } +} diff --git a/lang/java/idl/src/test/idl/output/simple.avpr b/lang/java/idl/src/test/idl/output/simple.avpr new file mode 100644 index 00000000000..0ec9edb4c24 --- /dev/null +++ b/lang/java/idl/src/test/idl/output/simple.avpr @@ -0,0 +1,144 @@ +{ + "protocol" : "Simple", + "namespace" : "org.apache.avro.test", + "doc" : "A simple test case.", + "version" : "1.0.5", + "types" : [ { + "type" : "enum", + "name" : "Kind", + "doc" : "A kind of record.", + "symbols" : [ "FOO", "BAR", "BAZ" ], + "aliases" : [ "org.foo.KindOf" ] + }, { + "type" : "enum", + "name" : "Status", + "symbols" : [ "A", "B", "C" ], + "default" : "C" + }, { + "type" : "record", + "name" : "TestRecord", + "doc" : "A TestRecord.", + "fields" : [ { + "name" : "name", + "type" : { + "type": "string", + "avro.java.string": "String" + }, + "default" : "foo", + "order" : "ignore" + }, { + "name" : "kind", + "type" : "Kind", + "doc" : "The kind of record.", + "order" : "descending" + }, { + "name" : "status", + "type" : "Status", + "doc" : "The status of the record.", + "default" : "A" + }, { + "name" : "hash", + "type" : { + "type" : "fixed", + "name" : "MD5", + "doc" : "An MD5 hash.", + "size" : 16 + }, + "default" : "0000000000000000" + }, { + "name" : "nullableHash", + "type" : [ "null", "MD5" ], + "default" : null, + "aliases" : [ "hash", "hsh" ] + }, { + "name" : "value", + "type" : "double", + "default" : "NaN" + }, { + "name" : "average", + "type" : "float", + "default" : "-Infinity" + }, { + "name": "d", + "type": {"type": "int", "logicalType": "date"}, + "default": 0 + }, { + "name": "t", + "type": [ {"type": "int", "logicalType": "time-millis"}, "null" ], + "default": 0 + } , { + "name": "l", + "type": {"type": "long", "foo.bar": "bar.foo"}, + "default": 0 + } , { + "name": "a", + "type": {"type": "array", "items": "string", "foo.bar.bar": "foo.bar2"}, + "default": [] + } , { + "name": "prop", + "type": [ "null" , {"type":"string", "foo.foo.bar": 42, "foo.foo.foo": "3foo"} ], + "default": null + }], + "my-property" : { + "key" : 3 + } + }, { + "type" : "error", + "name" : "TestError", + "fields" : [ { + "name" : "message", + "type" : "string" + } ] + } ], + "messages" : { + "hello" : { + "doc" : "method 'hello' takes @parameter 'greeting'", + "request" : [ { + "name" : "greeting", + "type" : "string" + } ], + "response" : "string" + }, + "echo" : { + "request" : [ { + "name" : "record", + "type" : "TestRecord", + "default" : { + "name" : "bar", + "kind" : "BAR" + } + } ], + "response" : "TestRecord" + }, + "add" : { + "doc" : "method 'add' takes @parameter 'arg1' @parameter 'arg2'", + "specialProp" : "test", + "request" : [ { + "name" : "arg1", + "type" : "int" + }, { + "name" : "arg2", + "type" : "int", + "default" : 0 + } ], + "response" : "int" + }, + "echoBytes" : { + "request" : [ { + "name" : "data", + "type" : "bytes" + } ], + "response" : "bytes" + }, + "error" : { + "request" : [ ], + "response" : "null", + "errors" : [ "TestError" ] + }, + "ping" : { + "request" : [ ], + "response" : "null", + "one-way" : true + } + } +} diff --git a/lang/java/idl/src/test/idl/output/unicode.avpr b/lang/java/idl/src/test/idl/output/unicode.avpr new file mode 100644 index 00000000000..c58a043003b --- /dev/null +++ b/lang/java/idl/src/test/idl/output/unicode.avpr @@ -0,0 +1,17 @@ +{ + "protocol" : "Протоколы", + "doc" : "This is a test that UTF8 functions correctly.\nこのテストでは、UTF - 8で正しく機能している。\n这是一个测试，UTF - 8的正常运行。", + "types" : [ { + "type" : "record", + "name" : "Структура", + "fields" : [ { + "name" : "Строковый", + "type" : "string" + }, { + "name" : "文字列", + "type" : "string" + } ] + } ], + "messages" : { + } +} diff --git a/lang/java/idl/src/test/idl/output/uuid.avpr b/lang/java/idl/src/test/idl/output/uuid.avpr new file mode 100644 index 00000000000..fe3d5a66c1a --- /dev/null +++ b/lang/java/idl/src/test/idl/output/uuid.avpr @@ -0,0 +1,29 @@ +{ + "protocol" : "MyProtocol", + "namespace" : "org.apache.avro", + "doc" : "Testing UUID fields", + "types" : [ { + "type" : "record", + "name" : "APlaygroundEvent", + "fields" : [ { + "name" : "identifier", + "type" : { + "type" : "string", + "logicalType" : "uuid" + }, + "doc" : "Documentation must be provided for each attribute" + }, { + "name" : "uuid", + "type" : "string", + "doc" : "A string field with a special name" + }, { + "name" : "optionalString", + "type" : [ "null", { + "type" : "string", + "logicalType" : "uuid" + } ], + "doc" : "a nullable uuid field" + } ] + } ], + "messages" : { } +} diff --git a/lang/java/idl/src/test/idl/putOnClassPath/OnTheClasspath.avdl b/lang/java/idl/src/test/idl/putOnClassPath/OnTheClasspath.avdl new file mode 100644 index 00000000000..4ee84113246 --- /dev/null +++ b/lang/java/idl/src/test/idl/putOnClassPath/OnTheClasspath.avdl @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@namespace("org.on.the.classpath") +protocol OnTheClasspath { + import idl "folder/relativePath.avdl"; + record FromAfar { + } +} diff --git a/lang/java/idl/src/test/idl/putOnClassPath/OnTheClasspath.avpr b/lang/java/idl/src/test/idl/putOnClassPath/OnTheClasspath.avpr new file mode 100644 index 00000000000..46951c2d485 --- /dev/null +++ b/lang/java/idl/src/test/idl/putOnClassPath/OnTheClasspath.avpr @@ -0,0 +1,11 @@ +{ + "protocol" : "OnTheClasspath", + "namespace" : "org.on.the.classpath", + "types" : [ { + "type" : "record", + "name" : "VeryFar", + "fields" : [ ] + } ], + "messages" : { + } +} \ No newline at end of file diff --git a/lang/java/idl/src/test/idl/putOnClassPath/OnTheClasspath.avsc b/lang/java/idl/src/test/idl/putOnClassPath/OnTheClasspath.avsc new file mode 100644 index 00000000000..40d3595e8fe --- /dev/null +++ b/lang/java/idl/src/test/idl/putOnClassPath/OnTheClasspath.avsc @@ -0,0 +1,6 @@ +{ + "type" : "record", + "name" : "FarAway", + "namespace" : "org.on.the.classpath", + "fields" : [ ] +} \ No newline at end of file diff --git a/lang/java/idl/src/test/idl/putOnClassPath/folder/relativePath.avdl b/lang/java/idl/src/test/idl/putOnClassPath/folder/relativePath.avdl new file mode 100644 index 00000000000..0343efeb90d --- /dev/null +++ b/lang/java/idl/src/test/idl/putOnClassPath/folder/relativePath.avdl @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@namespace("org.on.the.classpath") +protocol OnTheClasspathDependency { + import idl "../nestedtypes.avdl"; +} diff --git a/lang/java/idl/src/test/idl/putOnClassPath/nestedtypes.avdl b/lang/java/idl/src/test/idl/putOnClassPath/nestedtypes.avdl new file mode 100644 index 00000000000..6ef062f3d7e --- /dev/null +++ b/lang/java/idl/src/test/idl/putOnClassPath/nestedtypes.avdl @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@namespace("org.on.the.classpath") +protocol OnTheClasspathTypes { + record NestedType { + } +} diff --git a/lang/java/idl/src/test/java/org/apache/avro/idl/IdlReaderTest.java b/lang/java/idl/src/test/java/org/apache/avro/idl/IdlReaderTest.java new file mode 100644 index 00000000000..e609e1ff865 --- /dev/null +++ b/lang/java/idl/src/test/java/org/apache/avro/idl/IdlReaderTest.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.idl; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.avro.Protocol; +import org.apache.avro.Schema; +import org.junit.Before; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Simple test harness for Idl. This relies on an input/ and output/ directory. + * Inside the input/ directory are .avdl files. Each file should have a + * corresponding .avpr file in output/. When the test runs, it generates and + * stringifies each .avdl file and compares it to the expected output, failing + * if the two differ. + *

+ * To make it simpler to write these tests, you can run ant -Dtestcase=TestIdl + * -Dtest.idl.mode=write, which will *replace* all expected output. + */ +public class IdlReaderTest { + private static final File TEST_DIR = new File(System.getProperty("test.idl.dir", "src/test/idl")); + + private static final File TEST_INPUT_DIR = new File(TEST_DIR, "input").getAbsoluteFile(); + + private static final File TEST_OUTPUT_DIR = new File(TEST_DIR, "output"); + + private static final String TEST_MODE = System.getProperty("test.idl.mode", "run"); + + private static final File EXTRA_TEST_DIR = new File(TEST_DIR, "extra"); + + private List tests; + + @Before + public void loadTests() { + assertTrue(TEST_DIR.exists()); + assertTrue(TEST_INPUT_DIR.exists()); + assertTrue(TEST_OUTPUT_DIR.exists()); + + tests = new ArrayList<>(); + for (File inF : Objects.requireNonNull(TEST_INPUT_DIR.listFiles())) { + if (!inF.getName().endsWith(".avdl")) { + continue; + } + if (inF.getName().startsWith(".")) { + continue; + } + + File outF = new File(TEST_OUTPUT_DIR, + inF.getName().replaceFirst("_schema\\.avdl$", ".avsc").replaceFirst("\\.avdl$", ".avpr")); + tests.add(new GenTest(inF, outF)); + } + } + + @Test + public void validateProtocolParsingResult() throws IOException { + // runTests already tests the actual parsing; this tests the result object. + IdlFile idlFile = parseExtraIdlFile("protocolSyntax.avdl"); + + assertEquals(1, idlFile.getNamedSchemas().size()); + idlFile.getNamedSchemas().keySet().forEach(System.out::println); + assertNotNull(idlFile.getNamedSchema("communication.Message")); + assertNotNull(idlFile.getNamedSchema("Message")); + + assertNotNull(idlFile.getProtocol()); + } + + @Test + public void testDocCommentsAndWarnings() throws Exception { + final IdlFile idlFile = parseExtraIdlFile("../input/comments.avdl"); + final Protocol protocol = idlFile.getProtocol(); + final List warnings = idlFile.getWarnings(); + + assertEquals("Documented Enum", protocol.getType("testing.DocumentedEnum").getDoc()); + + assertEquals("Documented Fixed Type", protocol.getType("testing.DocumentedFixed").getDoc()); + + final Schema documentedError = protocol.getType("testing.DocumentedError"); + assertEquals("Documented Error", documentedError.getDoc()); + assertEquals("Documented Reason Field", documentedError.getField("reason").doc()); + assertEquals("Default Doc Explanation Field", documentedError.getField("explanation").doc()); + + final Map messages = protocol.getMessages(); + final Protocol.Message documentedMethod = messages.get("documentedMethod"); + assertEquals("Documented Method", documentedMethod.getDoc()); + assertEquals("Documented Parameter", documentedMethod.getRequest().getField("message").doc()); + assertEquals("Default Documented Parameter", documentedMethod.getRequest().getField("defMsg").doc()); + + assertNull(protocol.getType("testing.UndocumentedEnum").getDoc()); + assertNull(protocol.getType("testing.UndocumentedFixed").getDoc()); + assertNull(protocol.getType("testing.UndocumentedRecord").getDoc()); + assertNull(messages.get("undocumentedMethod").getDoc()); + + final String pattern = "Line %d, char %d: Ignoring out-of-place documentation comment.%n" + + "Did you mean to use a multiline comment ( /* ... */ ) instead?"; + assertEquals( + Arrays.asList(String.format(pattern, 21, 8), String.format(pattern, 21, 45), String.format(pattern, 22, 5), + String.format(pattern, 23, 5), String.format(pattern, 24, 5), String.format(pattern, 25, 5), + String.format(pattern, 26, 7), String.format(pattern, 27, 7), String.format(pattern, 28, 7), + String.format(pattern, 33, 7), String.format(pattern, 34, 7), String.format(pattern, 35, 5), + String.format(pattern, 36, 5), String.format(pattern, 37, 7), String.format(pattern, 42, 7), + String.format(pattern, 43, 7), String.format(pattern, 46, 9), String.format(pattern, 47, 5), + String.format(pattern, 54, 7), String.format(pattern, 55, 7), String.format(pattern, 58, 9), + String.format(pattern, 59, 7), String.format(pattern, 60, 11), String.format(pattern, 61, 11)), + warnings); + } + + @SuppressWarnings("SameParameterValue") + private IdlFile parseExtraIdlFile(String fileName) throws IOException { + return new IdlReader().parse(EXTRA_TEST_DIR.toPath().resolve(fileName)); + } + + @Test + public void runTests() { + if (!"run".equals(TEST_MODE)) { + return; + } + + int failed = 0; + + for (GenTest t : tests) { + try { + t.run(); + } catch (Exception e) { + failed++; + System.err.println("Failed: " + t.testName()); + e.printStackTrace(System.err); + } + } + + if (failed > 0) { + fail(failed + " tests failed"); + } + } + + @Test + public void writeTests() throws Exception { + if (!"write".equals(TEST_MODE)) { + return; + } + + for (GenTest t : tests) { + t.write(); + } + } + + /** + * An individual comparison test + */ + private static class GenTest { + private final File in, expectedOut; + + public GenTest(File in, File expectedOut) { + this.in = in; + this.expectedOut = expectedOut; + } + + private String generate() { + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + + URL[] newPathURL = new URL[] { cl.getResource("putOnClassPath-test-resource.jar") }; + URLClassLoader ucl = new URLClassLoader(newPathURL, cl); + Thread.currentThread().setContextClassLoader(ucl); + try { + IdlReader parser = new IdlReader(); + return parser.parse(in.toPath()).outputString(); + } catch (IOException e) { + throw new AssertionError(e.getMessage(), e); + } finally { + Thread.currentThread().setContextClassLoader(cl); + } + } + + public String testName() { + return this.in.getName(); + } + + public void run() throws Exception { + String output = generate(); + String slurped = slurp(expectedOut); + assertEquals(slurped.trim(), output.replace("\\r", "").trim()); + } + + public void write() throws Exception { + writeFile(expectedOut, generate()); + } + + private static String slurp(File f) throws IOException { + BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(f), StandardCharsets.UTF_8)); + + String line; + StringBuilder builder = new StringBuilder(); + while ((line = in.readLine()) != null) { + builder.append(line); + } + in.close(); + ObjectMapper mapper = new ObjectMapper(); + JsonNode json = mapper.readTree(builder.toString()); + return mapper.writer().writeValueAsString(json); + } + + private static void writeFile(File f, String s) throws IOException { + FileWriter w = new FileWriter(f); + w.write(s); + w.close(); + } + } +} diff --git a/lang/java/idl/src/test/java/org/apache/avro/idl/TestCycle.java b/lang/java/idl/src/test/java/org/apache/avro/idl/TestCycle.java new file mode 100644 index 00000000000..427de0957d6 --- /dev/null +++ b/lang/java/idl/src/test/java/org/apache/avro/idl/TestCycle.java @@ -0,0 +1,87 @@ +/* + * Copyright 2015 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.idl; + +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.GenericRecordBuilder; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Collections; + +import static java.util.Objects.requireNonNull; + +public class TestCycle { + + private static final Logger LOG = LoggerFactory.getLogger(TestCycle.class); + + @Test + public void testCycleGeneration() throws IOException, URISyntaxException { + final ClassLoader cl = Thread.currentThread().getContextClassLoader(); + IdlFile idlFile = new IdlReader().parse(requireNonNull(cl.getResource("input/cycle.avdl")).toURI()); + String json = idlFile.outputString(); + LOG.info(json); + + GenericRecordBuilder rb2 = new GenericRecordBuilder(idlFile.getNamedSchema("SampleNode")); + rb2.set("count", 10); + rb2.set("subNodes", Collections.EMPTY_LIST); + GenericData.Record node = rb2.build(); + + GenericRecordBuilder mb = new GenericRecordBuilder(idlFile.getNamedSchema("Method")); + mb.set("declaringClass", "Test"); + mb.set("methodName", "test"); + GenericData.Record method = mb.build(); + + GenericRecordBuilder spb = new GenericRecordBuilder(idlFile.getNamedSchema("SamplePair")); + spb.set("method", method); + spb.set("node", node); + GenericData.Record sp = spb.build(); + + GenericRecordBuilder rb = new GenericRecordBuilder(idlFile.getNamedSchema("SampleNode")); + rb.set("count", 10); + rb.set("subNodes", Collections.singletonList(sp)); + GenericData.Record record = rb.build(); + + serDeserRecord(record); + } + + private static void serDeserRecord(GenericData.Record data) throws IOException { + ByteArrayOutputStream bab = new ByteArrayOutputStream(); + GenericDatumWriter writer = new GenericDatumWriter<>(data.getSchema()); + final BinaryEncoder directBinaryEncoder = EncoderFactory.get().directBinaryEncoder(bab, null); + writer.write(data, directBinaryEncoder); + directBinaryEncoder.flush(); + ByteArrayInputStream bis = new ByteArrayInputStream(bab.toByteArray(), 0, bab.size()); + GenericDatumReader reader = new GenericDatumReader<>(data.getSchema()); + BinaryDecoder directBinaryDecoder = DecoderFactory.get().directBinaryDecoder(bis, null); + GenericData.Record read = (GenericData.Record) reader.read(null, directBinaryDecoder); + Assert.assertEquals(data.toString(), read.toString()); + } + +} diff --git a/lang/java/idl/src/test/java/org/apache/avro/idl/TestLogicalTypes.java b/lang/java/idl/src/test/java/org/apache/avro/idl/TestLogicalTypes.java new file mode 100644 index 00000000000..05a8dcaf2db --- /dev/null +++ b/lang/java/idl/src/test/java/org/apache/avro/idl/TestLogicalTypes.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.idl; + +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.net.URISyntaxException; + +import static java.util.Objects.requireNonNull; + +public class TestLogicalTypes { + private Schema logicalTypeFields; + + @Before + public void setup() throws IOException, URISyntaxException { + final ClassLoader cl = Thread.currentThread().getContextClassLoader(); + IdlFile idlFile = new IdlReader().parse(requireNonNull(cl.getResource("logicalTypes.avdl")).toURI()); + + logicalTypeFields = idlFile.getNamedSchema("org.apache.avro.test.LogicalTypeFields"); + } + + @Test + public void testDateBecomesLogicalType() { + Assert.assertEquals(LogicalTypes.date(), logicalTypeOfField("aDate")); + } + + @Test + public void testTimeMsBecomesLogicalType() { + Assert.assertEquals(LogicalTypes.timeMillis(), logicalTypeOfField("aTime")); + } + + @Test + public void testTimestampMsBecomesLogicalType() { + Assert.assertEquals(LogicalTypes.timestampMillis(), logicalTypeOfField("aTimestamp")); + } + + @Test + public void testLocalTimestampMsBecomesLogicalType() { + Assert.assertEquals(LogicalTypes.localTimestampMillis(), logicalTypeOfField("aLocalTimestamp")); + } + + @Test + public void testDecimalBecomesLogicalType() { + Assert.assertEquals(LogicalTypes.decimal(6, 2), logicalTypeOfField("pocketMoney")); + } + + @Test + public void testUuidBecomesLogicalType() { + Assert.assertEquals(LogicalTypes.uuid(), logicalTypeOfField("identifier")); + } + + @Test + public void testAnnotatedLongBecomesLogicalType() { + Assert.assertEquals(LogicalTypes.timestampMicros(), logicalTypeOfField("anotherTimestamp")); + } + + @Test + public void testAnnotatedBytesFieldBecomesLogicalType() { + Assert.assertEquals(LogicalTypes.decimal(6, 2), logicalTypeOfField("allowance")); + } + + @Test + public void testIncorrectlyAnnotatedBytesFieldHasNoLogicalType() { + Schema fieldSchema = logicalTypeFields.getField("byteArray").schema(); + + Assert.assertNull(fieldSchema.getLogicalType()); + Assert.assertEquals("decimal", fieldSchema.getObjectProp("logicalType")); + Assert.assertEquals(3000000000L, fieldSchema.getObjectProp("precision")); // Not an int, so not a valid precision + Assert.assertEquals(0, fieldSchema.getObjectProp("scale")); + } + + private LogicalType logicalTypeOfField(String name) { + return logicalTypeFields.getField(name).schema().getLogicalType(); + } +} diff --git a/lang/java/idl/src/test/java/org/apache/avro/idl/TestReferenceAnnotationNotAllowed.java b/lang/java/idl/src/test/java/org/apache/avro/idl/TestReferenceAnnotationNotAllowed.java new file mode 100644 index 00000000000..51b5570f661 --- /dev/null +++ b/lang/java/idl/src/test/java/org/apache/avro/idl/TestReferenceAnnotationNotAllowed.java @@ -0,0 +1,42 @@ +/* + * Copyright 2015 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.idl; + +import org.apache.avro.AvroRuntimeException; +import org.junit.Test; + +import java.io.IOException; +import java.net.URISyntaxException; + +import static java.util.Objects.requireNonNull; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class TestReferenceAnnotationNotAllowed { + + @Test + public void testReferenceAnnotationNotAllowed() throws IOException, URISyntaxException { + + final ClassLoader cl = Thread.currentThread().getContextClassLoader(); + + try { + new IdlReader().parse(requireNonNull(cl.getResource("AnnotationOnTypeReference.avdl")).toURI()); + fail("Compilation should fail: annotations on type references are not allowed."); + } catch (AvroRuntimeException e) { + assertEquals("Type references may not be annotated, at line 29, column 16", e.getMessage()); + } + } +} diff --git a/lang/java/idl/src/test/java/org/apache/avro/idl/TestSchemaResolver.java b/lang/java/idl/src/test/java/org/apache/avro/idl/TestSchemaResolver.java new file mode 100644 index 00000000000..70488232581 --- /dev/null +++ b/lang/java/idl/src/test/java/org/apache/avro/idl/TestSchemaResolver.java @@ -0,0 +1,71 @@ +/* + * Copyright 2017 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.idl; + +import org.apache.avro.Protocol; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class TestSchemaResolver { + + @Test + public void testResolving() throws IOException { + Path testIdl = Paths.get(".", "src", "test", "idl", "cycle.avdl").toAbsolutePath(); + IdlReader parser = new IdlReader(); + IdlFile idlFile = parser.parse(testIdl); + Protocol protocol = idlFile.getProtocol(); + System.out.println(protocol); + Assert.assertEquals(5, protocol.getTypes().size()); + } + + @Test(expected = IllegalArgumentException.class) + public void testIsUnresolvedSchemaError1() { + // No "org.apache.avro.idl.unresolved.name" property + Schema s = SchemaBuilder.record("R").fields().endRecord(); + SchemaResolver.getUnresolvedSchemaName(s); + } + + @Test(expected = IllegalArgumentException.class) + public void testIsUnresolvedSchemaError2() { + // No "UnresolvedSchema" property + Schema s = SchemaBuilder.record("R").prop("org.apache.avro.idl.unresolved.name", "x").fields().endRecord(); + SchemaResolver.getUnresolvedSchemaName(s); + } + + @Test(expected = IllegalArgumentException.class) + public void testIsUnresolvedSchemaError3() { + // Namespace not "org.apache.avro.compiler". + Schema s = SchemaBuilder.record("UnresolvedSchema").prop("org.apache.avro.idl.unresolved.name", "x").fields() + .endRecord(); + SchemaResolver.getUnresolvedSchemaName(s); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetUnresolvedSchemaNameError() { + Schema s = SchemaBuilder.fixed("a").size(10); + SchemaResolver.getUnresolvedSchemaName(s); + } +} diff --git a/lang/java/idl/src/test/java/org/apache/avro/idl/TestSchemas.java b/lang/java/idl/src/test/java/org/apache/avro/idl/TestSchemas.java new file mode 100644 index 00000000000..000ba20dbcb --- /dev/null +++ b/lang/java/idl/src/test/java/org/apache/avro/idl/TestSchemas.java @@ -0,0 +1,195 @@ +/* + * Copyright 2017 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.idl; + +import org.apache.avro.Schema; +import org.junit.Assert; +import org.junit.Test; + +public class TestSchemas { + + private static class TestVisitor implements SchemaVisitor { + StringBuilder sb = new StringBuilder(); + + @Override + public SchemaVisitorAction visitTerminal(Schema terminal) { + sb.append(terminal); + return SchemaVisitorAction.CONTINUE; + } + + @Override + public SchemaVisitorAction visitNonTerminal(Schema nonTerminal) { + String n = nonTerminal.getName(); + sb.append(n).append('.'); + if (n.startsWith("t")) { + return SchemaVisitorAction.TERMINATE; + } else if (n.startsWith("ss")) { + return SchemaVisitorAction.SKIP_SIBLINGS; + } else if (n.startsWith("st")) { + return SchemaVisitorAction.SKIP_SUBTREE; + } else { + return SchemaVisitorAction.CONTINUE; + } + } + + @Override + public SchemaVisitorAction afterVisitNonTerminal(Schema nonTerminal) { + sb.append("!"); + String n = nonTerminal.getName(); + if (n.startsWith("ct")) { + return SchemaVisitorAction.TERMINATE; + } else if (n.startsWith("css")) { + return SchemaVisitorAction.SKIP_SIBLINGS; + } else if (n.startsWith("cst")) { + return SchemaVisitorAction.SKIP_SUBTREE; + } else { + return SchemaVisitorAction.CONTINUE; + } + } + + @Override + public String get() { + return sb.toString(); + } + } + + @Test + public void testVisit1() { + String s1 = "{\"type\": \"record\", \"name\": \"t1\", \"fields\": [" + "{\"name\": \"f1\", \"type\": \"int\"}" + + "]}"; + Assert.assertEquals("t1.", Schemas.visit(new Schema.Parser().parse(s1), new TestVisitor())); + } + + @Test + public void testVisit2() { + String s2 = "{\"type\": \"record\", \"name\": \"c1\", \"fields\": [" + "{\"name\": \"f1\", \"type\": \"int\"}" + + "]}"; + Assert.assertEquals("c1.\"int\"!", Schemas.visit(new Schema.Parser().parse(s2), new TestVisitor())); + + } + + @Test + public void testVisit3() { + String s3 = "{\"type\": \"record\", \"name\": \"ss1\", \"fields\": [" + "{\"name\": \"f1\", \"type\": \"int\"}" + + "]}"; + Assert.assertEquals("ss1.", Schemas.visit(new Schema.Parser().parse(s3), new TestVisitor())); + + } + + @Test + public void testVisit4() { + String s4 = "{\"type\": \"record\", \"name\": \"st1\", \"fields\": [" + "{\"name\": \"f1\", \"type\": \"int\"}" + + "]}"; + Assert.assertEquals("st1.!", Schemas.visit(new Schema.Parser().parse(s4), new TestVisitor())); + + } + + @Test + public void testVisit5() { + String s5 = "{\"type\": \"record\", \"name\": \"c1\", \"fields\": [" + + "{\"name\": \"f1\", \"type\": {\"type\": \"record\", \"name\": \"c2\", \"fields\": " + + "[{\"name\": \"f11\", \"type\": \"int\"}]}}," + "{\"name\": \"f2\", \"type\": \"long\"}" + "]}"; + Assert.assertEquals("c1.c2.\"int\"!\"long\"!", Schemas.visit(new Schema.Parser().parse(s5), new TestVisitor())); + + } + + @Test + public void testVisit6() { + String s6 = "{\"type\": \"record\", \"name\": \"c1\", \"fields\": [" + + "{\"name\": \"f1\", \"type\": {\"type\": \"record\", \"name\": \"ss2\", \"fields\": " + + "[{\"name\": \"f11\", \"type\": \"int\"}]}}," + "{\"name\": \"f2\", \"type\": \"long\"}" + "]}"; + Assert.assertEquals("c1.ss2.!", Schemas.visit(new Schema.Parser().parse(s6), new TestVisitor())); + + } + + @Test + public void testVisit7() { + String s7 = "{\"type\": \"record\", \"name\": \"c1\", \"fields\": [" + + "{\"name\": \"f1\", \"type\": {\"type\": \"record\", \"name\": \"css2\", \"fields\": " + + "[{\"name\": \"f11\", \"type\": \"int\"}]}}," + "{\"name\": \"f2\", \"type\": \"long\"}" + "]}"; + Assert.assertEquals("c1.css2.\"int\"!!", Schemas.visit(new Schema.Parser().parse(s7), new TestVisitor())); + } + + @Test(expected = UnsupportedOperationException.class) + public void testVisit8() { + String s8 = "{\"type\": \"record\", \"name\": \"c1\", \"fields\": [" + + "{\"name\": \"f1\", \"type\": {\"type\": \"record\", \"name\": \"cst2\", \"fields\": " + + "[{\"name\": \"f11\", \"type\": \"int\"}]}}," + "{\"name\": \"f2\", \"type\": \"int\"}" + "]}"; + Schemas.visit(new Schema.Parser().parse(s8), new TestVisitor()); + } + + @Test + public void testVisit9() { + String s9 = "{\"type\": \"record\", \"name\": \"c1\", \"fields\": [" + + "{\"name\": \"f1\", \"type\": {\"type\": \"record\", \"name\": \"ct2\", \"fields\": " + + "[{\"name\": \"f11\", \"type\": \"int\"}]}}," + "{\"name\": \"f2\", \"type\": \"long\"}" + "]}"; + Assert.assertEquals("c1.ct2.\"int\"!", Schemas.visit(new Schema.Parser().parse(s9), new TestVisitor())); + } + + @Test(expected = UnsupportedOperationException.class) + public void testVisit10() { + String s10 = "{\"type\": \"record\", \"name\": \"c1\", \"fields\": [" + + "{\"name\": \"f1\", \"type\": {\"type\": \"record\", \"name\": \"ct2\", \"fields\": " + + "[{\"name\": \"f11\", \"type\": \"int\"}]}}," + "{\"name\": \"f2\", \"type\": \"int\"}" + "]}"; + Schemas.visit(new Schema.Parser().parse(s10), new TestVisitor() { + @Override + public SchemaVisitorAction visitTerminal(Schema terminal) { + return SchemaVisitorAction.SKIP_SUBTREE; + } + }); + } + + @Test + public void testVisit11() { + String s11 = "{\"type\": \"record\", \"name\": \"c1\", \"fields\": [" + + "{\"name\": \"f1\", \"type\": {\"type\": \"record\", \"name\": \"c2\", \"fields\": " + + "[{\"name\": \"f11\", \"type\": \"int\"},{\"name\": \"f12\", \"type\": \"double\"}" + "]}}," + + "{\"name\": \"f2\", \"type\": \"long\"}" + "]}"; + Assert.assertEquals("c1.c2.\"int\".!\"long\".!", Schemas.visit(new Schema.Parser().parse(s11), new TestVisitor() { + @Override + public SchemaVisitorAction visitTerminal(Schema terminal) { + sb.append(terminal).append('.'); + return SchemaVisitorAction.SKIP_SIBLINGS; + } + })); + } + + @Test + public void testVisit12() { + String s12 = "{\"type\": \"record\", \"name\": \"c1\", \"fields\": [" + + "{\"name\": \"f1\", \"type\": {\"type\": \"record\", \"name\": \"ct2\", \"fields\": " + + "[{\"name\": \"f11\", \"type\": \"int\"}]}}," + "{\"name\": \"f2\", \"type\": \"long\"}" + "]}"; + Assert.assertEquals("c1.ct2.\"int\".", Schemas.visit(new Schema.Parser().parse(s12), new TestVisitor() { + @Override + public SchemaVisitorAction visitTerminal(Schema terminal) { + sb.append(terminal).append('.'); + return SchemaVisitorAction.TERMINATE; + } + })); + } + + @Test + public void testVisit13() { + String s12 = "{\"type\": \"int\"}"; + Assert.assertEquals("\"int\".", Schemas.visit(new Schema.Parser().parse(s12), new TestVisitor() { + @Override + public SchemaVisitorAction visitTerminal(Schema terminal) { + sb.append(terminal).append('.'); + return SchemaVisitorAction.SKIP_SIBLINGS; + } + })); + } +} diff --git a/lang/java/integration-test/codegen-test/pom.xml b/lang/java/integration-test/codegen-test/pom.xml index e5a7ae1d962..760bea934aa 100644 --- a/lang/java/integration-test/codegen-test/pom.xml +++ b/lang/java/integration-test/codegen-test/pom.xml @@ -24,7 +24,7 @@ avro-integration-test org.apache.avro 1.12.0-SNAPSHOT - ../ + ../pom.xml avro-codegen-test diff --git a/lang/java/integration-test/pom.xml b/lang/java/integration-test/pom.xml index cc5db4c9194..bc27a788170 100644 --- a/lang/java/integration-test/pom.xml +++ b/lang/java/integration-test/pom.xml @@ -24,7 +24,7 @@ avro-parent org.apache.avro 1.12.0-SNAPSHOT - ../ + ../pom.xml avro-integration-test diff --git a/lang/java/integration-test/test-custom-conversions/pom.xml b/lang/java/integration-test/test-custom-conversions/pom.xml index 94e76b94162..9a5dc836982 100644 --- a/lang/java/integration-test/test-custom-conversions/pom.xml +++ b/lang/java/integration-test/test-custom-conversions/pom.xml @@ -24,7 +24,7 @@ avro-integration-test org.apache.avro 1.12.0-SNAPSHOT - ../ + ../pom.xml avro-test-custom-conversions diff --git a/lang/java/ipc-jetty/pom.xml b/lang/java/ipc-jetty/pom.xml index 098370404a1..2ee6248d34f 100644 --- a/lang/java/ipc-jetty/pom.xml +++ b/lang/java/ipc-jetty/pom.xml @@ -24,7 +24,7 @@ avro-parent org.apache.avro 1.12.0-SNAPSHOT - ../ + ../pom.xml avro-ipc-jetty diff --git a/lang/java/ipc-netty/pom.xml b/lang/java/ipc-netty/pom.xml index 96dd0bfaaa8..6b316c4a9b3 100644 --- a/lang/java/ipc-netty/pom.xml +++ b/lang/java/ipc-netty/pom.xml @@ -24,7 +24,7 @@ avro-parent org.apache.avro 1.12.0-SNAPSHOT - ../ + ../pom.xml avro-ipc-netty diff --git a/lang/java/ipc/pom.xml b/lang/java/ipc/pom.xml index 5878980f5bd..5a86e1393f7 100644 --- a/lang/java/ipc/pom.xml +++ b/lang/java/ipc/pom.xml @@ -24,7 +24,7 @@ avro-parent org.apache.avro 1.12.0-SNAPSHOT - ../ + ../pom.xml avro-ipc diff --git a/lang/java/mapred/pom.xml b/lang/java/mapred/pom.xml index 7a4b8a649be..8069e7df3e9 100644 --- a/lang/java/mapred/pom.xml +++ b/lang/java/mapred/pom.xml @@ -24,7 +24,7 @@ avro-parent org.apache.avro 1.12.0-SNAPSHOT - ../ + ../pom.xml avro-mapred diff --git a/lang/java/maven-plugin/pom.xml b/lang/java/maven-plugin/pom.xml index 15590348dd8..a4e9e7b6fdb 100644 --- a/lang/java/maven-plugin/pom.xml +++ b/lang/java/maven-plugin/pom.xml @@ -91,6 +91,11 @@ + + ${project.groupId} + avro-idl + ${project.version} + ${project.groupId} avro-compiler diff --git a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/AbstractAvroMojo.java b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/AbstractAvroMojo.java index e8186d42cdd..e34dd1b01fd 100644 --- a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/AbstractAvroMojo.java +++ b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/AbstractAvroMojo.java @@ -18,6 +18,18 @@ package org.apache.avro.mojo; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Protocol; +import org.apache.avro.Schema; +import org.apache.avro.compiler.specific.SpecificCompiler; +import org.apache.avro.generic.GenericData; +import org.apache.maven.artifact.DependencyResolutionRequiredException; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.project.MavenProject; +import org.apache.maven.shared.model.fileset.FileSet; +import org.apache.maven.shared.model.fileset.util.FileSetManager; + import java.io.File; import java.io.IOException; import java.lang.reflect.InvocationTargetException; @@ -26,17 +38,9 @@ import java.net.URLClassLoader; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; -import org.apache.avro.LogicalTypes; -import org.apache.avro.compiler.specific.SpecificCompiler; -import org.apache.maven.artifact.DependencyResolutionRequiredException; -import org.apache.maven.plugin.AbstractMojo; -import org.apache.maven.plugin.MojoExecutionException; -import org.apache.maven.project.MavenProject; -import org.apache.maven.shared.model.fileset.FileSet; -import org.apache.maven.shared.model.fileset.util.FileSetManager; - /** * Base for Avro Compiler Mojos. */ @@ -129,6 +133,20 @@ public abstract class AbstractAvroMojo extends AbstractMojo { */ protected String[] velocityToolsClassesNames = new String[0]; + /** + * Generated record schema classes will extend this class. + * + * @parameter property="recordSpecificClass" + */ + private String recordSpecificClass = "org.apache.avro.specific.SpecificRecordBase"; + + /** + * Generated error schema classes will extend this class. + * + * @parameter property="errorSpecificClass" + */ + private String errorSpecificClass = "org.apache.avro.specific.SpecificExceptionBase"; + /** * The createOptionalGetters parameter enables generating the getOptional... * methods that return an Optional of the requested type. This works ONLY on @@ -283,13 +301,22 @@ private String[] getIncludedFiles(String absPath, String[] excludes, String[] in } private void compileFiles(String[] files, File sourceDir, File outDir) throws MojoExecutionException { - // Need to register custom logical type factories before schema compilation. + final ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { - loadLogicalTypesFactories(); - } catch (IOException e) { - throw new MojoExecutionException("Error while loading logical types factories ", e); + Thread.currentThread().setContextClassLoader(createClassLoader()); + + // Need to register custom logical type factories before schema compilation. + try { + loadLogicalTypesFactories(); + } catch (IOException e) { + throw new MojoExecutionException("Error while loading logical types factories ", e); + } + this.doCompile(files, sourceDir, outDir); + } catch (MalformedURLException | DependencyResolutionRequiredException e) { + throw new MojoExecutionException("Cannot locate classpath entries", e); + } finally { + Thread.currentThread().setContextClassLoader(contextClassLoader); } - this.doCompile(files, sourceDir, outDir); } private void loadLogicalTypesFactories() throws IOException, MojoExecutionException { @@ -321,7 +348,7 @@ protected List instantiateAdditionalVelocityTools() { final List velocityTools = new ArrayList<>(velocityToolsClassesNames.length); for (String velocityToolClassName : velocityToolsClassesNames) { try { - Class klass = Class.forName(velocityToolClassName); + Class klass = Class.forName(velocityToolClassName); velocityTools.add(klass.getDeclaredConstructor().newInstance()); } catch (Exception e) { throw new RuntimeException(e); @@ -335,27 +362,68 @@ protected void doCompile(String[] files, File sourceDirectory, File outputDirect try { doCompile(filename, sourceDirectory, outputDirectory); } catch (IOException e) { - throw new MojoExecutionException("Error compiling protocol file " + filename + " to " + outputDirectory, e); + throw new MojoExecutionException("Error compiling file " + filename + " to " + outputDirectory, e); } } } - protected abstract void doCompile(String filename, File sourceDirectory, File outputDirectory) throws IOException; + protected void doCompile(String filename, File sourceDirectory, File outputDirectory) throws IOException { + throw new UnsupportedOperationException( + "Programmer error: AbstractAvroMojo.doCompile(String, java.io.File, java.io.File) called directly"); + }; - protected URLClassLoader createClassLoader() throws DependencyResolutionRequiredException, MalformedURLException { + protected void doCompile(File sourceFileForModificationDetection, Collection schemas, File outputDirectory) + throws IOException { + doCompile(sourceFileForModificationDetection, new SpecificCompiler(schemas), outputDirectory); + } + + protected void doCompile(File sourceFileForModificationDetection, Protocol protocol, File outputDirectory) + throws IOException { + doCompile(sourceFileForModificationDetection, new SpecificCompiler(protocol), outputDirectory); + } + + private void doCompile(File sourceFileForModificationDetection, SpecificCompiler compiler, File outputDirectory) + throws IOException { + compiler.setTemplateDir(templateDirectory); + compiler.setStringType(GenericData.StringType.valueOf(stringType)); + compiler.setFieldVisibility(getFieldVisibility()); + compiler.setCreateOptionalGetters(createOptionalGetters); + compiler.setGettersReturnOptional(gettersReturnOptional); + compiler.setOptionalGettersForNullableFieldsOnly(optionalGettersForNullableFieldsOnly); + compiler.setCreateSetters(createSetters); + compiler.setEnableDecimalLogicalType(enableDecimalLogicalType); + try { + for (String customConversion : customConversions) { + compiler.addCustomConversion(Thread.currentThread().getContextClassLoader().loadClass(customConversion)); + } + } catch (ClassNotFoundException e) { + throw new IOException(e); + } + compiler.setOutputCharacterEncoding(project.getProperties().getProperty("project.build.sourceEncoding")); + compiler.setAdditionalVelocityTools(instantiateAdditionalVelocityTools()); + compiler.setRecordSpecificClass(this.recordSpecificClass); + compiler.setErrorSpecificClass(this.errorSpecificClass); + compiler.compileToDestination(sourceFileForModificationDetection, outputDirectory); + } + + protected List findClasspath() throws DependencyResolutionRequiredException, MalformedURLException { final List urls = appendElements(project.getRuntimeClasspathElements()); urls.addAll(appendElements(project.getTestClasspathElements())); + return urls; + } + + protected URLClassLoader createClassLoader() throws DependencyResolutionRequiredException, MalformedURLException { + final List urls = findClasspath(); return new URLClassLoader(urls.toArray(new URL[0]), Thread.currentThread().getContextClassLoader()); } - private List appendElements(List runtimeClasspathElements) throws MalformedURLException { + private List appendElements(List runtimeClasspathElements) throws MalformedURLException { if (runtimeClasspathElements == null) { return new ArrayList<>(); } List runtimeUrls = new ArrayList<>(runtimeClasspathElements.size()); - for (Object runtimeClasspathElement : runtimeClasspathElements) { - String element = (String) runtimeClasspathElement; - runtimeUrls.add(new File(element).toURI().toURL()); + for (String runtimeClasspathElement : runtimeClasspathElements) { + runtimeUrls.add(new File(runtimeClasspathElement).toURI().toURL()); } return runtimeUrls; } diff --git a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/IDLProtocolMojo.java b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/IDLProtocolMojo.java index 0c9228c6133..eceaff88ded 100644 --- a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/IDLProtocolMojo.java +++ b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/IDLProtocolMojo.java @@ -18,20 +18,16 @@ package org.apache.avro.mojo; -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.ArrayList; -import java.util.List; - import org.apache.avro.Protocol; import org.apache.avro.compiler.idl.Idl; import org.apache.avro.compiler.idl.ParseException; -import org.apache.avro.compiler.specific.SpecificCompiler; -import org.apache.avro.generic.GenericData; +import org.apache.avro.idl.IdlFile; +import org.apache.avro.idl.IdlReader; -import org.apache.maven.artifact.DependencyResolutionRequiredException; +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; /** * Generate Java classes and interfaces from AvroIDL files (.avdl) @@ -42,6 +38,14 @@ * @threadSafe */ public class IDLProtocolMojo extends AbstractAvroMojo { + /** + * Use the classic JavaCC parser for .avdl files. If + * false (the default), use the new ANTLR parser instead. + * + * @parameter + */ + private boolean useJavaCC = false; + /** * A set of Ant-like inclusion patterns used to select files from the source * directory for processing. By default, the pattern **/*.avdl @@ -62,53 +66,37 @@ public class IDLProtocolMojo extends AbstractAvroMojo { @Override protected void doCompile(String filename, File sourceDirectory, File outputDirectory) throws IOException { - try { - @SuppressWarnings("rawtypes") - List runtimeClasspathElements = project.getRuntimeClasspathElements(); + File sourceFile = new File(sourceDirectory, filename); - List runtimeUrls = new ArrayList<>(); + ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); + URL[] extraClasspath = new URL[] { sourceDirectory.toURI().toURL() }; + ClassLoader classLoader = new URLClassLoader(extraClasspath, contextClassLoader); - // Add the source directory of avro files to the classpath so that - // imports can refer to other idl files as classpath resources - runtimeUrls.add(sourceDirectory.toURI().toURL()); - - // If runtimeClasspathElements is not empty values add its values to Idl path. - if (runtimeClasspathElements != null && !runtimeClasspathElements.isEmpty()) { - for (Object runtimeClasspathElement : runtimeClasspathElements) { - String element = (String) runtimeClasspathElement; - runtimeUrls.add(new File(element).toURI().toURL()); - } + Protocol protocol; + if (useJavaCC) { + try (Idl idl = new Idl(sourceFile, classLoader)) { + final Protocol p = idl.CompilationUnit(); + String json = p.toString(true); + protocol = Protocol.parse(json); + } catch (ParseException e) { + throw new IOException(e); } + } else { + try { + Thread.currentThread().setContextClassLoader(classLoader); - URLClassLoader projPathLoader = new URLClassLoader(runtimeUrls.toArray(new URL[0]), - Thread.currentThread().getContextClassLoader()); - try (Idl parser = new Idl(new File(sourceDirectory, filename), projPathLoader)) { - - Protocol p = parser.CompilationUnit(); - for (String warning : parser.getWarningsAfterParsing()) { + IdlReader parser = new IdlReader(); + IdlFile idlFile = parser.parse(sourceFile.toPath()); + for (String warning : idlFile.getWarnings()) { getLog().warn(warning); } - String json = p.toString(true); - Protocol protocol = Protocol.parse(json); - final SpecificCompiler compiler = new SpecificCompiler(protocol); - compiler.setStringType(GenericData.StringType.valueOf(stringType)); - compiler.setTemplateDir(templateDirectory); - compiler.setFieldVisibility(getFieldVisibility()); - compiler.setCreateOptionalGetters(createOptionalGetters); - compiler.setGettersReturnOptional(gettersReturnOptional); - compiler.setOptionalGettersForNullableFieldsOnly(optionalGettersForNullableFieldsOnly); - compiler.setCreateSetters(createSetters); - compiler.setAdditionalVelocityTools(instantiateAdditionalVelocityTools()); - compiler.setEnableDecimalLogicalType(enableDecimalLogicalType); - for (String customConversion : customConversions) { - compiler.addCustomConversion(projPathLoader.loadClass(customConversion)); - } - compiler.setOutputCharacterEncoding(project.getProperties().getProperty("project.build.sourceEncoding")); - compiler.compileToDestination(null, outputDirectory); + protocol = idlFile.getProtocol(); + } finally { + Thread.currentThread().setContextClassLoader(contextClassLoader); } - } catch (ParseException | ClassNotFoundException | DependencyResolutionRequiredException e) { - throw new IOException(e); } + + doCompile(sourceFile, protocol, outputDirectory); } @Override diff --git a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/ProtocolMojo.java b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/ProtocolMojo.java index c5e406fc70d..ee7e4101c5d 100644 --- a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/ProtocolMojo.java +++ b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/ProtocolMojo.java @@ -18,15 +18,10 @@ package org.apache.avro.mojo; -import org.apache.avro.generic.GenericData.StringType; +import org.apache.avro.Protocol; import java.io.File; import java.io.IOException; -import java.net.URLClassLoader; - -import org.apache.avro.Protocol; -import org.apache.avro.compiler.specific.SpecificCompiler; -import org.apache.maven.artifact.DependencyResolutionRequiredException; /** * Generate Java classes and interfaces from Avro protocol files (.avpr) @@ -59,27 +54,7 @@ public class ProtocolMojo extends AbstractAvroMojo { protected void doCompile(String filename, File sourceDirectory, File outputDirectory) throws IOException { final File src = new File(sourceDirectory, filename); final Protocol protocol = Protocol.parse(src); - final SpecificCompiler compiler = new SpecificCompiler(protocol); - compiler.setTemplateDir(templateDirectory); - compiler.setStringType(StringType.valueOf(stringType)); - compiler.setFieldVisibility(getFieldVisibility()); - compiler.setCreateOptionalGetters(createOptionalGetters); - compiler.setGettersReturnOptional(gettersReturnOptional); - compiler.setOptionalGettersForNullableFieldsOnly(optionalGettersForNullableFieldsOnly); - compiler.setCreateSetters(createSetters); - compiler.setAdditionalVelocityTools(instantiateAdditionalVelocityTools()); - compiler.setEnableDecimalLogicalType(enableDecimalLogicalType); - final URLClassLoader classLoader; - try { - classLoader = createClassLoader(); - for (String customConversion : customConversions) { - compiler.addCustomConversion(classLoader.loadClass(customConversion)); - } - } catch (DependencyResolutionRequiredException | ClassNotFoundException e) { - throw new IOException(e); - } - compiler.setOutputCharacterEncoding(project.getProperties().getProperty("project.build.sourceEncoding")); - compiler.compileToDestination(src, outputDirectory); + doCompile(src, protocol, outputDirectory); } @Override diff --git a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/SchemaMojo.java b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/SchemaMojo.java index de2b18e7f87..36a4fc4a53c 100644 --- a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/SchemaMojo.java +++ b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/SchemaMojo.java @@ -18,22 +18,17 @@ package org.apache.avro.mojo; +import org.apache.avro.Schema; import org.apache.avro.SchemaParseException; -import org.apache.avro.generic.GenericData.StringType; +import org.apache.maven.plugin.MojoExecutionException; import java.io.File; import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URLClassLoader; import java.util.Arrays; +import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; -import org.apache.avro.Schema; -import org.apache.avro.compiler.specific.SpecificCompiler; -import org.apache.maven.artifact.DependencyResolutionRequiredException; -import org.apache.maven.plugin.MojoExecutionException; - /** * Generate Java classes from Avro schema files (.avsc) * @@ -67,73 +62,32 @@ public class SchemaMojo extends AbstractAvroMojo { */ private String[] testIncludes = new String[] { "**/*.avsc" }; - /** - * generated record schema classes will extend this class. - */ - private String recordSpecificClass = "org.apache.avro.specific.SpecificRecordBase"; - - /** - * generated error schema classes will extend this class. - */ - private String errorSpecificClass = "org.apache.avro.specific.SpecificExceptionBase"; - @Override - protected void doCompile(String[] filesName, File sourceDirectory, File outputDirectory) + protected void doCompile(String[] fileNames, File sourceDirectory, File outputDirectory) throws MojoExecutionException { - final List sourceFiles = Arrays.stream(filesName) + final List sourceFiles = Arrays.stream(fileNames) .map((String filename) -> new File(sourceDirectory, filename)).collect(Collectors.toList()); + final File sourceFileForModificationDetection = sourceFiles.stream().filter(file -> file.lastModified() > 0) + .max(Comparator.comparing(File::lastModified)).orElse(null); final List schemas; - // This is necessary to maintain backward-compatibility. If there are - // no imported files then isolate the schemas from each other, otherwise - // allow them to share a single schema so reuse and sharing of schema - // is possible. try { + // This is necessary to maintain backward-compatibility. If there are + // no imported files then isolate the schemas from each other, otherwise + // allow them to share a single schema so reuse and sharing of schema + // is possible. if (imports == null) { schemas = new Schema.Parser().parse(sourceFiles); } else { schemas = schemaParser.parse(sourceFiles); } - } catch (IOException | SchemaParseException ex) { - throw new MojoExecutionException("Error compiling one file of " + sourceDirectory + " to " + outputDirectory, ex); - } - final SpecificCompiler compiler = new SpecificCompiler(schemas); - compiler.setTemplateDir(templateDirectory); - compiler.setStringType(StringType.valueOf(stringType)); - compiler.setFieldVisibility(getFieldVisibility()); - compiler.setCreateOptionalGetters(createOptionalGetters); - compiler.setGettersReturnOptional(gettersReturnOptional); - compiler.setOptionalGettersForNullableFieldsOnly(optionalGettersForNullableFieldsOnly); - compiler.setCreateSetters(createSetters); - compiler.setEnableDecimalLogicalType(enableDecimalLogicalType); - try { - final URLClassLoader classLoader = createClassLoader(); - for (String customConversion : customConversions) { - compiler.addCustomConversion(classLoader.loadClass(customConversion)); - } - } catch (ClassNotFoundException | DependencyResolutionRequiredException | MalformedURLException e) { - throw new MojoExecutionException("Compilation error: Can't add custom conversion", e); - } - compiler.setOutputCharacterEncoding(project.getProperties().getProperty("project.build.sourceEncoding")); - compiler.setAdditionalVelocityTools(instantiateAdditionalVelocityTools()); - compiler.setRecordSpecificClass(this.recordSpecificClass); - compiler.setErrorSpecificClass(this.errorSpecificClass); - for (File src : sourceFiles) { - try { - compiler.compileToDestination(src, outputDirectory); - } catch (IOException ex) { - throw new MojoExecutionException("Compilation error with file " + src + " to " + outputDirectory, ex); - } + doCompile(sourceFileForModificationDetection, schemas, outputDirectory); + } catch (IOException | SchemaParseException ex) { + throw new MojoExecutionException("Error compiling a file in " + sourceDirectory + " to " + outputDirectory, ex); } } - @Override - protected void doCompile(final String filename, final File sourceDirectory, final File outputDirectory) - throws IOException { - // Not call. - } - @Override protected String[] getIncludes() { return includes; diff --git a/lang/java/maven-plugin/src/test/java/org/apache/avro/mojo/TestIDLProtocolMojo.java b/lang/java/maven-plugin/src/test/java/org/apache/avro/mojo/TestIDLProtocolMojo.java index 226ca6de09d..c8cc22316e4 100644 --- a/lang/java/maven-plugin/src/test/java/org/apache/avro/mojo/TestIDLProtocolMojo.java +++ b/lang/java/maven-plugin/src/test/java/org/apache/avro/mojo/TestIDLProtocolMojo.java @@ -17,15 +17,16 @@ */ package org.apache.avro.mojo; -import org.codehaus.plexus.util.FileUtils; -import org.junit.Test; - import java.io.File; -import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Set; +import org.codehaus.plexus.util.FileUtils; +import org.junit.Test; + +import static java.util.Arrays.asList; + /** * Test the IDL Protocol Mojo. */ @@ -45,17 +46,15 @@ public void testIdlProtocolMojo() throws Exception { mojo.execute(); final File outputDir = new File(getBasedir(), "target/test-harness/idl/test/"); - final Set generatedFiles = new HashSet<>(Arrays.asList("IdlPrivacy.java", "IdlTest.java", "IdlUser.java", + final Set generatedFiles = new HashSet<>(asList("IdlPrivacy.java", "IdlTest.java", "IdlUser.java", "IdlUserWrapper.java", "IdlClasspathImportTest.java")); assertFilesExist(outputDir, generatedFiles); final String idlUserContent = FileUtils.fileRead(new File(outputDir, "IdlUser.java")); assertTrue(idlUserContent.contains("java.time.Instant")); - assertEquals(Collections.singletonList( - "[WARN] Found documentation comment at line 23, column 5. Ignoring previous one at line 22, column 5: \"Ignored Doc Comment\"" - + "\nDid you mean to use a multiline comment ( /* ... */ ) instead?"), - log.getLogEntries()); + assertEquals(Collections.singletonList("[WARN] Line 22, char 5: Ignoring out-of-place documentation comment.\n" + + "Did you mean to use a multiline comment ( /* ... */ ) instead?"), log.getLogEntries()); } @Test @@ -68,7 +67,7 @@ public void testSetCompilerVelocityAdditionalTools() throws Exception { mojo.execute(); final File outputDir = new File(getBasedir(), "target/test-harness/idl-inject/test"); - final Set generatedFiles = new HashSet<>(Arrays.asList("IdlPrivacy.java", "IdlTest.java", "IdlUser.java", + final Set generatedFiles = new HashSet<>(asList("IdlPrivacy.java", "IdlTest.java", "IdlUser.java", "IdlUserWrapper.java", "IdlClasspathImportTest.java")); assertFilesExist(outputDir, generatedFiles); diff --git a/lang/java/maven-plugin/src/test/resources/unit/idl/pom-javacc.xml b/lang/java/maven-plugin/src/test/resources/unit/idl/pom-javacc.xml new file mode 100644 index 00000000000..4abd67f7bca --- /dev/null +++ b/lang/java/maven-plugin/src/test/resources/unit/idl/pom-javacc.xml @@ -0,0 +1,68 @@ + + + + 4.0.0 + + + avro-parent + org.apache.avro + 1.11.0-SNAPSHOT + ../../../../../../../../../pom.xml + + + avro-maven-plugin-test + jar + + testproject + + + + + avro-maven-plugin + + + idl + + idl-protocol + + + + + true + ${basedir}/src/test + ${basedir}/target/test-harness/idl + String + + + + + + + + org.apache.avro + avro + ${parent.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + diff --git a/lang/java/perf/pom.xml b/lang/java/perf/pom.xml index 6e1d465fac3..479f9bd760c 100644 --- a/lang/java/perf/pom.xml +++ b/lang/java/perf/pom.xml @@ -24,7 +24,7 @@ avro-parent org.apache.avro 1.12.0-SNAPSHOT - ../ + ../pom.xml avro-perf diff --git a/lang/java/pom.xml b/lang/java/pom.xml index daf3167ab75..d049c42deff 100644 --- a/lang/java/pom.xml +++ b/lang/java/pom.xml @@ -73,6 +73,7 @@ android avro + idl compiler maven-plugin ipc diff --git a/lang/java/protobuf/pom.xml b/lang/java/protobuf/pom.xml index d509955f81a..0a04f8f96c6 100644 --- a/lang/java/protobuf/pom.xml +++ b/lang/java/protobuf/pom.xml @@ -24,7 +24,7 @@ avro-parent org.apache.avro 1.12.0-SNAPSHOT - ../ + ../pom.xml avro-protobuf diff --git a/lang/java/thrift/pom.xml b/lang/java/thrift/pom.xml index 64a9dfa0668..591dd255472 100644 --- a/lang/java/thrift/pom.xml +++ b/lang/java/thrift/pom.xml @@ -24,7 +24,7 @@ avro-parent org.apache.avro 1.12.0-SNAPSHOT - ../ + ../pom.xml avro-thrift diff --git a/lang/java/tools/pom.xml b/lang/java/tools/pom.xml index c64412b9591..e841c8d84fd 100644 --- a/lang/java/tools/pom.xml +++ b/lang/java/tools/pom.xml @@ -24,7 +24,7 @@ avro-parent org.apache.avro 1.12.0-SNAPSHOT - ../ + ../pom.xml avro-tools @@ -198,6 +198,11 @@ tests test + + ${project.groupId} + avro-idl + ${project.version} + ${project.groupId} avro-compiler diff --git a/lang/java/tools/src/main/java/org/apache/avro/tool/IdlToSchemataTool.java b/lang/java/tools/src/main/java/org/apache/avro/tool/IdlToSchemataTool.java index df43d6d2ac7..6e0901efed1 100644 --- a/lang/java/tools/src/main/java/org/apache/avro/tool/IdlToSchemataTool.java +++ b/lang/java/tools/src/main/java/org/apache/avro/tool/IdlToSchemataTool.java @@ -21,6 +21,8 @@ import org.apache.avro.Protocol; import org.apache.avro.Schema; import org.apache.avro.compiler.idl.Idl; +import org.apache.avro.idl.IdlFile; +import org.apache.avro.idl.IdlReader; import java.io.File; import java.io.FileNotFoundException; @@ -36,26 +38,40 @@ public class IdlToSchemataTool implements Tool { @Override public int run(InputStream in, PrintStream out, PrintStream err, List args) throws Exception { - if (args.isEmpty() || args.size() > 2 || isRequestingHelp(args)) { - err.println("Usage: idl2schemata [idl] [outdir]"); - err.println(""); + boolean useJavaCC = "--useJavaCC".equals(getArg(args, 0, null)); + + if (args.isEmpty() || args.size() > (useJavaCC ? 3 : 2) || isRequestingHelp(args)) { + err.println("Usage: idl2schemata [--useJavaCC] [idl [outdir]]"); + err.println(); err.println("If an output directory is not specified, " + "outputs to current directory."); return -1; } - boolean pretty = true; - Idl parser = new Idl(new File(args.get(0))); - File outputDirectory = getOutputDirectory(args); + String inputName = getArg(args, useJavaCC ? 1 : 0, "-"); + File inputFile = "-".equals(inputName) ? null : new File(inputName); + File outputDirectory = getOutputDirectory(getArg(args, useJavaCC ? 2 : 1, "")); - final Protocol protocol = parser.CompilationUnit(); - final List warnings = parser.getWarningsAfterParsing(); - for (String warning : warnings) { - err.println("Warning: " + warning); - } - for (Schema schema : protocol.getTypes()) { - print(schema, outputDirectory, pretty); + if (useJavaCC) { + try (Idl parser = new Idl(inputFile)) { + final Protocol protocol = parser.CompilationUnit(); + final List warnings = parser.getWarningsAfterParsing(); + for (String warning : warnings) { + err.println("Warning: " + warning); + } + for (Schema schema : protocol.getTypes()) { + print(schema, outputDirectory); + } + } + } else { + IdlReader parser = new IdlReader(); + IdlFile idlFile = inputFile == null ? parser.parse(in) : parser.parse(inputFile.toPath()); + for (String warning : idlFile.getWarnings()) { + err.println("Warning: " + warning); + } + for (Schema schema : idlFile.getNamedSchemas().values()) { + print(schema, outputDirectory); + } } - parser.close(); return 0; } @@ -64,19 +80,26 @@ private boolean isRequestingHelp(List args) { return args.size() == 1 && (args.get(0).equals("--help") || args.get(0).equals("-help")); } - private File getOutputDirectory(List args) { - String dirname = (args.size() == 2) ? args.get(1) : ""; + private String getArg(List args, int index, String defaultValue) { + if (index < args.size()) { + return args.get(index); + } else { + return defaultValue; + } + } + + private File getOutputDirectory(String dirname) { File outputDirectory = new File(dirname); outputDirectory.mkdirs(); return outputDirectory; } - private void print(Schema schema, File outputDirectory, boolean pretty) throws FileNotFoundException { + private void print(Schema schema, File outputDirectory) throws FileNotFoundException { String dirpath = outputDirectory.getAbsolutePath(); String filename = dirpath + "/" + schema.getName() + ".avsc"; FileOutputStream fileOutputStream = new FileOutputStream(filename); PrintStream printStream = new PrintStream(fileOutputStream); - printStream.println(schema.toString(pretty)); + printStream.println(schema.toString(true)); printStream.close(); } diff --git a/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java b/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java index b32b0db76a8..dfdaac966c8 100644 --- a/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java +++ b/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java @@ -20,6 +20,8 @@ import org.apache.avro.Protocol; import org.apache.avro.compiler.idl.Idl; +import org.apache.avro.idl.IdlFile; +import org.apache.avro.idl.IdlReader; import java.io.File; import java.io.FileOutputStream; @@ -34,35 +36,46 @@ public class IdlTool implements Tool { @Override public int run(InputStream in, PrintStream out, PrintStream err, List args) throws Exception { - PrintStream parseOut = out; - - if (args.size() > 2 || (args.size() == 1 && (args.get(0).equals("--help") || args.get(0).equals("-help")))) { - err.println("Usage: idl [in] [out]"); + boolean useJavaCC = "--useJavaCC".equals(getArg(args, 0, null)); + if (args.size() > (useJavaCC ? 3 : 2) + || (args.size() == 1 && (args.get(0).equals("--help") || args.get(0).equals("-help")))) { + err.println("Usage: idl [--useJavaCC] [in [out]]"); err.println(); err.println("If an output path is not specified, outputs to stdout."); err.println("If no input or output is specified, takes input from"); - err.println("stdin and outputs to stdin."); + err.println("stdin and outputs to stdout."); err.println("The special path \"-\" may also be specified to refer to"); err.println("stdin and stdout."); return -1; } - Idl parser; - if (args.size() >= 1 && !"-".equals(args.get(0))) { - parser = new Idl(new File(args.get(0))); + String inputName = getArg(args, useJavaCC ? 1 : 0, "-"); + File inputFile = "-".equals(inputName) ? null : new File(inputName); + String outputName = getArg(args, useJavaCC ? 2 : 1, "-"); + File outputFile = "-".equals(outputName) ? null : new File(outputName); + + Protocol p; + if (useJavaCC) { + try (Idl parser = new Idl(inputFile)) { + p = parser.CompilationUnit(); + for (String warning : parser.getWarningsAfterParsing()) { + err.println("Warning: " + warning); + } + } } else { - parser = new Idl(in); + IdlReader parser = new IdlReader(); + IdlFile idlFile = inputFile == null ? parser.parse(in) : parser.parse(inputFile.toPath()); + for (String warning : idlFile.getWarnings()) { + err.println("Warning: " + warning); + } + p = idlFile.getProtocol(); } - if (args.size() == 2 && !"-".equals(args.get(1))) { - parseOut = new PrintStream(new FileOutputStream(args.get(1))); + PrintStream parseOut = out; + if (outputFile != null) { + parseOut = new PrintStream(new FileOutputStream(outputFile)); } - Protocol p = parser.CompilationUnit(); - final List warnings = parser.getWarningsAfterParsing(); - for (String warning : warnings) { - err.println("Warning: " + warning); - } try { parseOut.print(p.toString(true)); } finally { @@ -72,6 +85,14 @@ public int run(InputStream in, PrintStream out, PrintStream err, List ar return 0; } + private String getArg(List args, int index, String defaultValue) { + if (index < args.size()) { + return args.get(index); + } else { + return defaultValue; + } + } + @Override public String getName() { return "idl"; @@ -79,6 +100,6 @@ public String getName() { @Override public String getShortDescription() { - return "Generates a JSON schema from an Avro IDL file"; + return "Generates a JSON protocol from an Avro IDL file"; } } diff --git a/lang/java/tools/src/test/java/org/apache/avro/tool/TestIdlToSchemataTool.java b/lang/java/tools/src/test/java/org/apache/avro/tool/TestIdlToSchemataTool.java index feb5e931a9b..cc74ba1684a 100644 --- a/lang/java/tools/src/test/java/org/apache/avro/tool/TestIdlToSchemataTool.java +++ b/lang/java/tools/src/test/java/org/apache/avro/tool/TestIdlToSchemataTool.java @@ -46,22 +46,39 @@ void splitIdlIntoSchemata() throws Exception { String[] files = new File(outdir).list(); assertEquals(4, files.length); + String warnings = readPrintStreamBuffer(buffer); + assertEquals("Warning: Line 1, char 1: Ignoring out-of-place documentation comment." + + "\nDid you mean to use a multiline comment ( /* ... */ ) instead?", warnings); + } + + @Test + public void testSplitIdlIntoSchemataUsingJavaCC() throws Exception { + String idl = "src/test/idl/protocol.avdl"; + String outdir = "target/test-split"; + + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + List arglist = Arrays.asList("--useJavaCC", idl, outdir); + new IdlToSchemataTool().run(null, null, new PrintStream(buffer), arglist); + + String[] files = new File(outdir).list(); + assertEquals(4, files.length); + String warnings = readPrintStreamBuffer(buffer); assertEquals( - "Warning: Found documentation comment at line 19, column 1. Ignoring previous one at line 1, column 1: \"" - + "Licensed to the Apache Software Foundation (ASF) under one\n" + "Warning: Found documentation comment at line 19, column 1. Ignoring previous one at line 1, column 1: " + + "\"Licensed to the Apache Software Foundation (ASF) under one\n" + "or more contributor license agreements. See the NOTICE file\n" + "distributed with this work for additional information\n" + "regarding copyright ownership. The ASF licenses this file\n" + "to you under the Apache License, Version 2.0 (the\n" + "\"License\"); you may not use this file except in compliance\n" - + "with the License. You may obtain a copy of the License at\n\n" - + " https://www.apache.org/licenses/LICENSE-2.0\n\n" + + "with the License. You may obtain a copy of the License at\n" + + "\n https://www.apache.org/licenses/LICENSE-2.0\n\n" + "Unless required by applicable law or agreed to in writing, software\n" + "distributed under the License is distributed on an \"AS IS\" BASIS,\n" + "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" - + "See the License for the specific language governing permissions and\n" + "limitations under the License." - + "\"\nDid you mean to use a multiline comment ( /* ... */ ) instead?", + + "See the License for the specific language governing permissions and\n" + + "limitations under the License.\"\nDid you mean to use a multiline comment ( /* ... */ ) instead?", warnings); } diff --git a/lang/java/tools/src/test/java/org/apache/avro/tool/TestIdlTool.java b/lang/java/tools/src/test/java/org/apache/avro/tool/TestIdlTool.java index 374a3d60562..4cfe6eddaf4 100644 --- a/lang/java/tools/src/test/java/org/apache/avro/tool/TestIdlTool.java +++ b/lang/java/tools/src/test/java/org/apache/avro/tool/TestIdlTool.java @@ -47,22 +47,39 @@ void writeIdlAsProtocol() throws Exception { assertEquals(readFileAsString(protocol), readFileAsString(outfile)); + String warnings = readPrintStreamBuffer(buffer); + assertEquals("Warning: Line 1, char 1: Ignoring out-of-place documentation comment." + + "\nDid you mean to use a multiline comment ( /* ... */ ) instead?", warnings); + } + + @Test + public void testWriteIdlAsProtocolUsingJavaCC() throws Exception { + String idl = "src/test/idl/protocol.avdl"; + String protocol = "src/test/idl/protocol.avpr"; + String outfile = "target/test-protocol.avpr"; + + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + List arglist = Arrays.asList("--useJavaCC", idl, outfile); + new IdlTool().run(null, null, new PrintStream(buffer), arglist); + + assertEquals(readFileAsString(protocol), readFileAsString(outfile)); + String warnings = readPrintStreamBuffer(buffer); assertEquals( - "Warning: Found documentation comment at line 19, column 1. Ignoring previous one at line 1, column 1: \"" - + "Licensed to the Apache Software Foundation (ASF) under one\n" + "Warning: Found documentation comment at line 19, column 1. Ignoring previous one at line 1, column 1: " + + "\"Licensed to the Apache Software Foundation (ASF) under one\n" + "or more contributor license agreements. See the NOTICE file\n" + "distributed with this work for additional information\n" + "regarding copyright ownership. The ASF licenses this file\n" + "to you under the Apache License, Version 2.0 (the\n" + "\"License\"); you may not use this file except in compliance\n" - + "with the License. You may obtain a copy of the License at\n\n" - + " https://www.apache.org/licenses/LICENSE-2.0\n\n" + + "with the License. You may obtain a copy of the License at\n" + + "\n https://www.apache.org/licenses/LICENSE-2.0\n\n" + "Unless required by applicable law or agreed to in writing, software\n" + "distributed under the License is distributed on an \"AS IS\" BASIS,\n" + "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" - + "See the License for the specific language governing permissions and\n" + "limitations under the License." - + "\"\nDid you mean to use a multiline comment ( /* ... */ ) instead?", + + "See the License for the specific language governing permissions and\n" + + "limitations under the License.\"\nDid you mean to use a multiline comment ( /* ... */ ) instead?", warnings); } diff --git a/lang/java/trevni/avro/pom.xml b/lang/java/trevni/avro/pom.xml index 5cc5f5049cb..a50cc028e5e 100644 --- a/lang/java/trevni/avro/pom.xml +++ b/lang/java/trevni/avro/pom.xml @@ -23,7 +23,7 @@ trevni-java org.apache.avro 1.12.0-SNAPSHOT - ../ + ../pom.xml trevni-avro diff --git a/lang/java/trevni/core/pom.xml b/lang/java/trevni/core/pom.xml index 99a12172673..365cf6f50f7 100644 --- a/lang/java/trevni/core/pom.xml +++ b/lang/java/trevni/core/pom.xml @@ -23,7 +23,7 @@ trevni-java org.apache.avro 1.12.0-SNAPSHOT - ../ + ../pom.xml trevni-core diff --git a/lang/java/trevni/pom.xml b/lang/java/trevni/pom.xml index 9d676bf09e8..464eee7b167 100644 --- a/lang/java/trevni/pom.xml +++ b/lang/java/trevni/pom.xml @@ -24,7 +24,7 @@ avro-parent org.apache.avro 1.12.0-SNAPSHOT - ../ + ../pom.xml trevni-java diff --git a/share/idl_grammar/org/apache/avro/idl/Idl.g4 b/share/idl_grammar/org/apache/avro/idl/Idl.g4 new file mode 100644 index 00000000000..01017a26184 --- /dev/null +++ b/share/idl_grammar/org/apache/avro/idl/Idl.g4 @@ -0,0 +1,248 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +grammar Idl; + +/* +** This file is a combined grammar to recognize the Avro IDL "language". By +** design, there are no actions in this grammar: this ensures the grammar is +** usable for any language supported by ANTLR. +** +** Some names, like BTrue & BFalse for booleans, may look a bit strange, but +** that's because they can otherwise conflict with identifiers in target\ +** languages like Java, Python, etc. +** +** Implementers can implement a listener or visitor to turn a parse result into +** a protocol and/or schema. +*/ + + +/************************************** +** ** +** Parser ** +** ** +**************************************/ + +// \u001a is the ascii 'sub'(stitute) character, used as end-of-file marker in older systems. It was also used as "end of character stream". +// Thus, accept it at end of the input and ignore anything that comes after it. (See: https://en.wikipedia.org/wiki/Substitute_character) +idlFile: protocol=protocolDeclaration ('\u001a' .*?)? EOF; + +protocolDeclaration: (doc=DocComment)? schemaProperties+=schemaProperty* Protocol name=identifier body=protocolDeclarationBody; + +protocolDeclarationBody : LBrace (imports+=importStatement|namedSchemas+=namedSchemaDeclaration|messages+=messageDeclaration)* RBrace ; + +/** + * The parser accepts anything that's not a symbol as an identifier. That is, it accepts both an IdentifierToken and all keywords. Which + * identifiers are actually allowed is context dependent and decided when transforming the parse tree. + */ +identifier: word=(IdentifierToken + | Protocol + | Import + | IDL + | Schema + | Enum + | Fixed + | Error + | Record + | Array + | Map + | Union + | Boolean + | Int + | Long + | Float + | Double + | String + | Bytes + | Null + | BTrue + | BFalse + | Decimal + | Date + | Time + | Timestamp + | LocalTimestamp + | UUID + | Void + | Oneway + | Throws +); + +schemaProperty: At name=identifier LParen value=jsonValue RParen; + +importStatement: Import importType=(Schema|Protocol|IDL) location=StringLiteral Semicolon; + +namedSchemaDeclaration: fixedDeclaration | enumDeclaration | recordDeclaration; + +fixedDeclaration: (doc=DocComment)? schemaProperties+=schemaProperty* Fixed name=identifier LParen size=IntegerLiteral RParen Semicolon; + +enumDeclaration: (doc=DocComment)? schemaProperties+=schemaProperty* Enum name=identifier + LBrace (enumSymbols+=enumSymbol (Comma enumSymbols+=enumSymbol)*)? RBrace defaultSymbol=enumDefault?; + +enumSymbol: (doc=DocComment)? schemaProperties+=schemaProperty* name=identifier; + +enumDefault : Equals defaultSymbolName=identifier Semicolon; + +recordDeclaration: (doc=DocComment)? schemaProperties+=schemaProperty* recordType=(Record|Error) name=identifier body=recordBody; + +recordBody : LBrace fields+=fieldDeclaration* RBrace; + +fieldDeclaration: (doc=DocComment)? fieldType=fullType variableDeclarations+=variableDeclaration (Comma variableDeclarations+=variableDeclaration)* Semicolon; + +variableDeclaration: (doc=DocComment)? schemaProperties+=schemaProperty* fieldName=identifier (Equals defaultValue=jsonValue)?; + +messageDeclaration: (doc=DocComment)? schemaProperties+=schemaProperty* returnType=resultType name=identifier + LParen (formalParameters+=formalParameter (Comma formalParameters+=formalParameter)*)? RParen + (oneway=Oneway | Throws errors+=identifier (Comma errors+=identifier)*)? Semicolon; + +formalParameter: (doc=DocComment)? parameterType=fullType parameter=variableDeclaration; + +resultType: Void | plainType; + +fullType: schemaProperties+=schemaProperty* plainType; + +plainType: arrayType | mapType | unionType | nullableType; + +// identifier MUST be last, as it captures any other type as well. +nullableType: (primitiveType | referenceName=identifier) optional=QuestionMark?; + +primitiveType: typeName=(Boolean | Int | Long | Float | Double | Bytes | String | Null | Date | Time | Timestamp | LocalTimestamp | UUID) + | typeName=Decimal LParen precision=IntegerLiteral ( Comma scale=IntegerLiteral )? RParen; + +arrayType: Array LT elementType=fullType GT; + +mapType: Map LT valueType=fullType GT; + +unionType: Union LBrace types+=fullType (Comma types+=fullType)* RBrace; + +jsonValue: jsonObject | jsonArray | jsonLiteral; +jsonLiteral: literal=(StringLiteral | IntegerLiteral | FloatingPointLiteral | BTrue | BFalse | Null); +jsonObject: LBrace jsonPairs+=jsonPair (Comma jsonPairs+=jsonPair)* RBrace; +jsonPair: name=StringLiteral Colon value=jsonValue; +jsonArray: LBracket (jsonValues+=jsonValue (Comma jsonValues+=jsonValue)*)? RBracket; + + +/************************************** +** ** +** Lexer ** +** ** +**************************************/ + +/* +** Comments +*/ + +// Note 1: this might be more efficient using lexer modes, but these cannot be used in a mixed file like this. +// Note 1: To do so, split this file into 'idl_lexer.g4' and 'idl_parser.g4', and import the tokens with 'options { tokenVocab=idl_lexer; }' + +// Note 2: DOC_COMMENT is now a regular token. +DocComment: '/**' .*? '*/' -> channel(HIDDEN); +EmptyComment: '/**/' -> skip; +MultiLineComment: '/*' ~[*] .*? '*/' -> skip; +SingleLineComment: '//' .*? ('\n' | '\r' '\n'?) -> skip; + +/* +** Whitespace +*/ +// Should be after the rule(s) for single-line comments, especially if rewritten to use multiple lexer modes +WS: [ \t\n\r\f] -> skip; + +/* +** Simple tokens +*/ +Protocol: 'protocol'; +Import: 'import'; +IDL: 'idl'; +Schema: 'schema'; +Enum: 'enum'; +Fixed: 'fixed'; +Error: 'error'; +Record: 'record'; +Array: 'array'; +Map: 'map'; +Union: 'union'; +Boolean: 'boolean'; +Int: 'int'; +Long: 'long'; +Float: 'float'; +Double: 'double'; +String: 'string'; +Bytes: 'bytes'; +Null: 'null'; +// The boolean values are not named True/False to avoid name conflicts with e.g. Python +BTrue: 'true'; +BFalse: 'false'; +Decimal: 'decimal'; +Date: 'date'; +Time: 'time_ms'; +Timestamp: 'timestamp_ms'; +LocalTimestamp: 'local_timestamp_ms'; +UUID: 'uuid'; +Void: 'void'; +Oneway: 'oneway'; +Throws: 'throws'; +LParen: '('; +RParen: ')'; +LBrace: '{'; +RBrace: '}'; +LBracket: '['; +RBracket: ']'; +Colon: ':'; +Semicolon: ';'; +Comma: ','; +At: '@'; +Equals: '='; +Dot: '.'; +Dash: '-'; +QuestionMark: '?'; +LT: '<'; +GT: '>'; + +/* +** Complex tokens +*/ + +// TODO: restrict to JSON string & number literals? + +// We use a reluctant qualifier, so we don't need to forbid the closing quote +StringLiteral: '"' (~[\\\n\r\b\f\t] | '\\' ([nrbft\\'"] | OctDigit OctDigit? | [0-3] OctDigit OctDigit | 'u' HexDigit HexDigit HexDigit HexDigit))*? '"'; +//StringLiteral: '"' (~[\\\u0000-\u001F] | '\\' ["\\/nrbft])*? '"'; + +IntegerLiteral: '-'? ( DecimalLiteral | HexLiteral | OctalLiteral ) [lL]?; +fragment DecimalLiteral: Digit9 Digit*; +fragment HexLiteral: '0' [xX] HexDigit+; +fragment OctalLiteral: '0' OctDigit*; + +fragment Digit9: [1-9]; +fragment Digit: '0' | Digit9; +fragment HexDigit: [0-9a-fA-F]; +fragment OctDigit: [0-7]; + +FloatingPointLiteral: [+-]? ('NaN' | 'Infinity' | DecimalFloatingPointLiteral | HexadecimalFloatingPointLiteral ); +fragment DecimalFloatingPointLiteral: (Digit+ '.' Digit* | '.' Digit+) DecimalExponent? [fFdD]? | Digit+ (DecimalExponent [fFdD]? | [fFdD]); +fragment DecimalExponent: [eE] [+\-]? Digit+; +fragment HexadecimalFloatingPointLiteral: '0' [xX] ( HexDigit+ ('.')? | HexDigit* '.' HexDigit+ ) HexadecimalExponent [fFdD]?; +fragment HexadecimalExponent: [pP] [+\-]? Digit+; + +/** + * An identifier token accepts any sequence of unicoode identifiers, optionally surrounded by backticks, separated by dots and/or dashes. + * Note that any sequence of identifier parts is an identifier token, even if an identifier part (also) matches an existing keyword. + * Also note that this token should *only* be used in the identifier grammar rule above. + */ +IdentifierToken: ( '`' IdentifierPart '`' | IdentifierPart)([.-] ( '`' IdentifierPart '`' | IdentifierPart) )*; +fragment IdentifierPart: [\p{XID_Start}] [\p{XID_Continue}]*; +// See discussion in AVRO-1022, AVRO-2659, AVRO-3115 +// fragment IdentifierPart: [A-Za-z] [A-Za-z0-9_]* diff --git a/share/test/schemas/contexts.avdl b/share/test/schemas/contexts.avdl index 245705f76df..17836c14dab 100644 --- a/share/test/schemas/contexts.avdl +++ b/share/test/schemas/contexts.avdl @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/share/test/schemas/echo.avdl b/share/test/schemas/echo.avdl index 87058e42935..14558862add 100644 --- a/share/test/schemas/echo.avdl +++ b/share/test/schemas/echo.avdl @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/share/test/schemas/http.avdl b/share/test/schemas/http.avdl index 67b6a72f4f7..13305444b43 100644 --- a/share/test/schemas/http.avdl +++ b/share/test/schemas/http.avdl @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/share/test/schemas/nestedNullable.avdl b/share/test/schemas/nestedNullable.avdl index ab2641cdf38..f31ae87571e 100644 --- a/share/test/schemas/nestedNullable.avdl +++ b/share/test/schemas/nestedNullable.avdl @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/share/test/schemas/schemaevolution.avdl b/share/test/schemas/schemaevolution.avdl index 1d8974d3428..7c77582484d 100644 --- a/share/test/schemas/schemaevolution.avdl +++ b/share/test/schemas/schemaevolution.avdl @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/share/test/schemas/social.avdl b/share/test/schemas/social.avdl index 329c9a6c930..fce2788dec3 100644 --- a/share/test/schemas/social.avdl +++ b/share/test/schemas/social.avdl @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/share/test/schemas/specialtypes.avdl b/share/test/schemas/specialtypes.avdl index a1ab7958260..3a4724b673b 100644 --- a/share/test/schemas/specialtypes.avdl +++ b/share/test/schemas/specialtypes.avdl @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/share/test/schemas/stringables.avdl b/share/test/schemas/stringables.avdl index 1214f04c49f..2fb799fdfea 100644 --- a/share/test/schemas/stringables.avdl +++ b/share/test/schemas/stringables.avdl @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information