From dd37e64a14d3e9564857ceceeff001b17d9ad179 Mon Sep 17 00:00:00 2001 From: Christophe Le Saec Date: Fri, 29 Sep 2023 09:35:41 +0200 Subject: [PATCH] AVRO-3876: Jackson util --- .../avro/util/internal/JacksonUtils.java | 26 +++++++++--- .../org/apache/avro/TestSchemaBuilder.java | 41 ++++++++++--------- .../avro/util/internal/TestJacksonUtils.java | 36 ++++++++++++++-- 3 files changed, 74 insertions(+), 29 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/util/internal/JacksonUtils.java b/lang/java/avro/src/main/java/org/apache/avro/util/internal/JacksonUtils.java index 1a822899f97..7de93079c70 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/util/internal/JacksonUtils.java +++ b/lang/java/avro/src/main/java/org/apache/avro/util/internal/JacksonUtils.java @@ -18,6 +18,7 @@ package org.apache.avro.util.internal; import java.io.IOException; +import java.io.UncheckedIOException; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.charset.StandardCharsets; @@ -75,7 +76,7 @@ static void toJson(Object datum, JsonGenerator generator) throws IOException { } generator.writeEndArray(); } else if (datum instanceof byte[]) { // bytes, fixed - generator.writeString(new String((byte[]) datum, StandardCharsets.ISO_8859_1)); + generator.writeBinary((byte[]) datum);// writeString(new String((byte[]) datum, StandardCharsets.ISO_8859_1)); } else if (datum instanceof CharSequence || datum instanceof Enum) { // string, enum generator.writeString(datum.toString()); } else if (datum instanceof Double) { // double @@ -136,10 +137,23 @@ public static Object toObject(JsonNode jsonNode, Schema schema) { return jsonNode.asDouble(); } } else if (jsonNode.isDouble() || jsonNode.isFloat()) { - if (schema == null || schema.getType().equals(Schema.Type.DOUBLE)) { - return jsonNode.asDouble(); - } else if (schema.getType().equals(Schema.Type.FLOAT)) { - return (float) jsonNode.asDouble(); + if (schema != null) { + if (schema.getType().equals(Schema.Type.DOUBLE)) { + return jsonNode.doubleValue(); + } else if (schema.getType().equals(Schema.Type.FLOAT)) { + return jsonNode.floatValue(); + } + } else if (jsonNode.isDouble()) { + return jsonNode.doubleValue(); + } else if (jsonNode.isFloat()) { + return jsonNode.floatValue(); + } + } else if (jsonNode.isBinary()) { + try { + return jsonNode.binaryValue(); + } catch (IOException ex) { + // only for TextNode, so, can't happen with binaryNode. + throw new UncheckedIOException(ex); } } else if (jsonNode.isTextual()) { if (schema == null || schema.getType().equals(Schema.Type.STRING) || schema.getType().equals(Schema.Type.ENUM)) { @@ -175,7 +189,7 @@ public static Object toObject(JsonNode jsonNode, Schema schema) { /** * Convert an object into a map - * + * * @param datum The object * @return Its Map representation */ diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaBuilder.java b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaBuilder.java index fcbaae65570..d2a234f11bf 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaBuilder.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaBuilder.java @@ -23,6 +23,7 @@ import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -111,14 +112,14 @@ void objectProps() { assertTrue(s.getObjectProp("intProp") instanceof Integer); assertTrue(s.getObjectProp("longProp") instanceof Long); assertEquals(Long.MAX_VALUE, s.getObjectProp("longProp")); - assertTrue(s.getObjectProp("floatProp") instanceof Double); + assertTrue(s.getObjectProp("floatProp") instanceof Float); // float converts to double - assertEquals(1.0d, s.getObjectProp("floatProp")); + assertEquals(1.0f, s.getObjectProp("floatProp")); assertTrue(s.getObjectProp("doubleProp") instanceof Double); assertEquals(Double.MAX_VALUE, s.getObjectProp("doubleProp")); // byte[] converts to string - assertTrue(s.getObjectProp("byteProp") instanceof String); - assertEquals("ABC", s.getObjectProp("byteProp")); + assertTrue(s.getObjectProp("byteProp") instanceof byte[]); + assertArrayEquals(new byte[] { 0x41, 0x42, 0x43 }, (byte[]) s.getObjectProp("byteProp")); assertTrue(s.getObjectProp("stringProp") instanceof String); assertEquals("abc", s.getObjectProp("stringProp")); } @@ -141,14 +142,14 @@ void fieldObjectProps() { assertTrue(f.getObjectProp("intProp") instanceof Integer); assertTrue(f.getObjectProp("longProp") instanceof Long); assertEquals(Long.MAX_VALUE, f.getObjectProp("longProp")); - assertTrue(f.getObjectProp("floatProp") instanceof Double); + assertTrue(f.getObjectProp("floatProp") instanceof Float); // float converts to double - assertEquals(1.0d, f.getObjectProp("floatProp")); + assertEquals(1.0f, f.getObjectProp("floatProp")); assertTrue(f.getObjectProp("doubleProp") instanceof Double); assertEquals(Double.MAX_VALUE, f.getObjectProp("doubleProp")); // byte[] converts to string - assertTrue(f.getObjectProp("byteProp") instanceof String); - assertEquals("ABC", f.getObjectProp("byteProp")); + assertTrue(f.getObjectProp("byteProp") instanceof byte[]); + assertArrayEquals(new byte[] { 0x41, 0x42, 0x43 }, (byte[]) f.getObjectProp("byteProp")); assertTrue(f.getObjectProp("stringProp") instanceof String); assertEquals("abc", f.getObjectProp("stringProp")); @@ -181,10 +182,10 @@ void arrayObjectProp() { assertEquals(Integer.MAX_VALUE, iter.next()); assertEquals(Long.MAX_VALUE, iter.next()); // float converts to double - assertEquals(1.0d, iter.next()); + assertEquals(1.0f, iter.next()); assertEquals(Double.MAX_VALUE, iter.next()); // byte[] converts to string - assertEquals("ABC", iter.next()); + assertArrayEquals(new byte[] { 0x41, 0x42, 0x43 }, (byte[]) iter.next()); assertEquals("abc", iter.next()); } @@ -216,10 +217,10 @@ void fieldArrayObjectProp() { assertEquals(Integer.MAX_VALUE, iter.next()); assertEquals(Long.MAX_VALUE, iter.next()); // float converts to double - assertEquals(1.0d, iter.next()); + assertEquals(1.0f, iter.next()); assertEquals(Double.MAX_VALUE, iter.next()); // byte[] converts to string - assertEquals("ABC", iter.next()); + assertArrayEquals(new byte[] { 0x41, 0x42, 0x43 }, (byte[]) iter.next()); assertEquals("abc", iter.next()); } @@ -249,13 +250,13 @@ void mapObjectProp() { assertTrue(valueMap.get("longKey") instanceof Long); assertEquals(Long.MAX_VALUE, valueMap.get("longKey")); // float converts to double - assertTrue(valueMap.get("floatKey") instanceof Double); - assertEquals(1.0d, valueMap.get("floatKey")); + assertTrue(valueMap.get("floatKey") instanceof Float); + assertEquals(1.0f, valueMap.get("floatKey")); assertTrue(valueMap.get("doubleKey") instanceof Double); assertEquals(Double.MAX_VALUE, valueMap.get("doubleKey")); // byte[] converts to string - assertTrue(valueMap.get("byteKey") instanceof String); - assertEquals("ABC", valueMap.get("byteKey")); + assertTrue(valueMap.get("byteKey") instanceof byte[]); + assertArrayEquals("ABC".getBytes(StandardCharsets.UTF_8), (byte[]) valueMap.get("byteKey")); assertTrue(valueMap.get("stringKey") instanceof String); assertEquals("abc", valueMap.get("stringKey")); } @@ -289,13 +290,13 @@ void fieldMapObjectProp() { assertTrue(valueMap.get("longKey") instanceof Long); assertEquals(Long.MAX_VALUE, valueMap.get("longKey")); // float converts to double - assertTrue(valueMap.get("floatKey") instanceof Double); - assertEquals(1.0d, valueMap.get("floatKey")); + assertTrue(valueMap.get("floatKey") instanceof Float); + assertEquals(1.0f, valueMap.get("floatKey")); assertTrue(valueMap.get("doubleKey") instanceof Double); assertEquals(Double.MAX_VALUE, valueMap.get("doubleKey")); // byte[] converts to string - assertTrue(valueMap.get("byteKey") instanceof String); - assertEquals("ABC", valueMap.get("byteKey")); + assertTrue(valueMap.get("byteKey") instanceof byte[]); + assertEquals("ABC", new String((byte[]) valueMap.get("byteKey"))); assertTrue(valueMap.get("stringKey") instanceof String); assertEquals("abc", valueMap.get("stringKey")); } diff --git a/lang/java/avro/src/test/java/org/apache/avro/util/internal/TestJacksonUtils.java b/lang/java/avro/src/test/java/org/apache/avro/util/internal/TestJacksonUtils.java index cd6fe3f74da..4a272ae5b35 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/util/internal/TestJacksonUtils.java +++ b/lang/java/avro/src/test/java/org/apache/avro/util/internal/TestJacksonUtils.java @@ -21,8 +21,10 @@ import static org.apache.avro.util.internal.JacksonUtils.toObject; import static org.junit.jupiter.api.Assertions.*; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.BigIntegerNode; +import com.fasterxml.jackson.databind.node.BinaryNode; import com.fasterxml.jackson.databind.node.BooleanNode; import com.fasterxml.jackson.databind.node.DecimalNode; import com.fasterxml.jackson.databind.node.DoubleNode; @@ -31,15 +33,24 @@ import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.LongNode; import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.NumericNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import java.math.BigDecimal; import java.math.BigInteger; +import java.nio.charset.StandardCharsets; import java.util.Collections; +import java.util.stream.Stream; + import org.apache.avro.JsonProperties; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; + +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; public class TestJacksonUtils { @@ -55,8 +66,9 @@ void testToJsonNode() { assertEquals(IntNode.valueOf(1), toJsonNode(1)); assertEquals(LongNode.valueOf(2), toJsonNode(2L)); assertEquals(FloatNode.valueOf(1.0f), toJsonNode(1.0f)); - assertEquals(DoubleNode.valueOf(2.0), toJsonNode(2.0)); - assertEquals(TextNode.valueOf("\u0001\u0002"), toJsonNode(new byte[] { 1, 2 })); + assertEquals(FloatNode.valueOf(33.33000183105469f), toJsonNode(33.33000183105469f)); + assertEquals(DoubleNode.valueOf(2.0), toJsonNode(2.0d)); + assertEquals(BinaryNode.valueOf(new byte[] { 1, 2 }), toJsonNode(new byte[] { 1, 2 })); assertEquals(TextNode.valueOf("a"), toJsonNode("a")); assertEquals(TextNode.valueOf("UP"), toJsonNode(Direction.UP)); assertEquals(BigIntegerNode.valueOf(BigInteger.ONE), toJsonNode(BigInteger.ONE)); @@ -80,7 +92,7 @@ void testToObject() { assertEquals(2L, toObject(IntNode.valueOf(2), Schema.create(Schema.Type.LONG))); assertEquals(1.0f, toObject(DoubleNode.valueOf(1.0), Schema.create(Schema.Type.FLOAT))); assertEquals(2.0, toObject(DoubleNode.valueOf(2.0))); - assertEquals(TextNode.valueOf("\u0001\u0002"), toJsonNode(new byte[] { 1, 2 })); + assertEquals(BinaryNode.valueOf(new byte[] { 1, 2 }), toJsonNode(new byte[] { 1, 2 })); assertArrayEquals(new byte[] { 1, 2 }, (byte[]) toObject(TextNode.valueOf("\u0001\u0002"), Schema.create(Schema.Type.BYTES))); assertEquals("a", toObject(TextNode.valueOf("a"))); @@ -102,4 +114,22 @@ void testToObject() { assertEquals("a", toObject(TextNode.valueOf("a"), SchemaBuilder.unionOf().stringType().and().intType().endUnion())); } + @ParameterizedTest + @MethodSource("nodes") + void cycle(JsonNode input) { + Object object = JacksonUtils.toObject(input); + JsonNode node = JacksonUtils.toJsonNode(object); + Assertions.assertEquals(input, node); + } + + public static Stream nodes() { + ObjectNode o1 = JsonNodeFactory.instance.objectNode(); + o1.put("intField", 123); + o1.put("floatField", 33.33000183105469f); + o1.put("doubleField", 33.33000183105469245d); + return Stream.of(JsonNodeFactory.instance.numberNode(33.33000183105469f), + JsonNodeFactory.instance.binaryNode("Hello".getBytes(StandardCharsets.ISO_8859_1)), + JsonNodeFactory.instance.arrayNode().add(1).add("Hello").add(o1)).map(Arguments::of); + } + }