From 66fa8b6cbba344f10534552db00946ed8ba6c367 Mon Sep 17 00:00:00 2001 From: Frank Wisniewski Date: Fri, 4 Nov 2016 12:04:47 -0700 Subject: [PATCH 01/19] Added custom types and conversion for UUID, Date, BigInteger and BigDecimal. --- project.gradle | 2 +- .../datanerds/avropatch/operation/Value.java | 2 + .../avropatch/value/ValueSchema.java | 31 +++++++ .../conversion/BigDecimalConversion.java | 56 ++++++++++++ .../conversion/BigIntegerConversion.java | 46 ++++++++++ .../value/conversion/DateConversion.java | 45 ++++++++++ .../value/conversion/UUIDConversion.java | 58 +++++++++++++ .../io/datanerds/avropatch/PatchTest.java | 6 ++ .../avropatch/operation/TestValue.java | 16 ---- .../avropatch/operation/ValueTest.java | 69 --------------- .../datanerds/avropatch/value/SchemaTest.java | 85 +++++++++++++++++++ .../conversion/BigDecimalConversionTest.java | 43 ++++++++++ .../conversion/BigIntegerConversionTest.java | 42 +++++++++ .../value/conversion/ConversionTester.java | 83 ++++++++++++++++++ .../value/conversion/DateConversionTest.java | 41 +++++++++ .../value/conversion/UUIDConversionTest.java | 40 +++++++++ 16 files changed, 579 insertions(+), 86 deletions(-) create mode 100644 src/main/java/io/datanerds/avropatch/value/ValueSchema.java create mode 100644 src/main/java/io/datanerds/avropatch/value/conversion/BigDecimalConversion.java create mode 100644 src/main/java/io/datanerds/avropatch/value/conversion/BigIntegerConversion.java create mode 100644 src/main/java/io/datanerds/avropatch/value/conversion/DateConversion.java create mode 100644 src/main/java/io/datanerds/avropatch/value/conversion/UUIDConversion.java delete mode 100644 src/test/java/io/datanerds/avropatch/operation/TestValue.java delete mode 100644 src/test/java/io/datanerds/avropatch/operation/ValueTest.java create mode 100644 src/test/java/io/datanerds/avropatch/value/SchemaTest.java create mode 100644 src/test/java/io/datanerds/avropatch/value/conversion/BigDecimalConversionTest.java create mode 100644 src/test/java/io/datanerds/avropatch/value/conversion/BigIntegerConversionTest.java create mode 100644 src/test/java/io/datanerds/avropatch/value/conversion/ConversionTester.java create mode 100644 src/test/java/io/datanerds/avropatch/value/conversion/DateConversionTest.java create mode 100644 src/test/java/io/datanerds/avropatch/value/conversion/UUIDConversionTest.java diff --git a/project.gradle b/project.gradle index ba6517a..be9d86f 100644 --- a/project.gradle +++ b/project.gradle @@ -16,7 +16,7 @@ project.ext { dependencies { compile([ - "org.apache.avro:avro:1.8.0", + "org.apache.avro:avro:1.8.1", "org.slf4j:slf4j-api:1.7.18" ]) diff --git a/src/main/java/io/datanerds/avropatch/operation/Value.java b/src/main/java/io/datanerds/avropatch/operation/Value.java index 75e9f98..958172d 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Value.java +++ b/src/main/java/io/datanerds/avropatch/operation/Value.java @@ -1,5 +1,7 @@ package io.datanerds.avropatch.operation; +import org.apache.avro.Schema; + /** * This interface holds the schema for all valid value types. */ diff --git a/src/main/java/io/datanerds/avropatch/value/ValueSchema.java b/src/main/java/io/datanerds/avropatch/value/ValueSchema.java new file mode 100644 index 0000000..fe5df6c --- /dev/null +++ b/src/main/java/io/datanerds/avropatch/value/ValueSchema.java @@ -0,0 +1,31 @@ +package io.datanerds.avropatch.value; + +import io.datanerds.avropatch.value.conversion.BigDecimalConversion; +import io.datanerds.avropatch.value.conversion.BigIntegerConversion; +import io.datanerds.avropatch.value.conversion.DateConversion; +import io.datanerds.avropatch.value.conversion.UUIDConversion; +import org.apache.avro.Schema; +import org.apache.avro.Schema.Type; + +import static org.apache.avro.Schema.*; + +public class ValueSchema { + + public static final Schema BOOLEAN = create(Type.BOOLEAN); + public static final Schema DOUBLE = create(Type.DOUBLE); + public static final Schema FLOAT = create(Type.FLOAT); + public static final Schema INTEGER = create(Type.INT); + public static final Schema LONG = create(Type.LONG); + public static final Schema NULL = create(Type.NULL); + public static final Schema STRING = create(Type.STRING); + + public static final Schema BIG_DECIMAL = BigDecimalConversion.SCHEMA; + public static final Schema BIG_INTEGER = BigIntegerConversion.SCHEMA; + public static final Schema DATE = DateConversion.SCHEMA; + public static final Schema UUID = UUIDConversion.SCHEMA; + + public static final Schema SCHEMA = + createUnion(BIG_DECIMAL, BIG_INTEGER, BOOLEAN, DATE, DOUBLE, FLOAT, INTEGER, /*LONG,*/ NULL, STRING, UUID, + createArray(createUnion(BIG_DECIMAL, BIG_INTEGER, BOOLEAN, DATE, DOUBLE, FLOAT, INTEGER, /*LONG,*/ NULL, STRING, UUID)), + createMap(createUnion(BIG_DECIMAL, BIG_INTEGER, BOOLEAN, DATE, DOUBLE, FLOAT, INTEGER, /*LONG,*/ NULL, STRING, UUID))); +} diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/BigDecimalConversion.java b/src/main/java/io/datanerds/avropatch/value/conversion/BigDecimalConversion.java new file mode 100644 index 0000000..fa973e8 --- /dev/null +++ b/src/main/java/io/datanerds/avropatch/value/conversion/BigDecimalConversion.java @@ -0,0 +1,56 @@ +package io.datanerds.avropatch.value.conversion; + +import org.apache.avro.Conversion; +import org.apache.avro.LogicalType; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.IndexedRecord; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; + +/** + * TODO + * + * @see BigIntegerConversion + */ +public class BigDecimalConversion extends Conversion { + + private static final String NAME = "big-decimal"; + private static final String DOC = "TODO"; + private static final Schema RECORD = SchemaBuilder.record("decimal").doc(DOC).fields().name("unscaledValue").type(BigIntegerConversion.SCHEMA).noDefault().name("scale").type(Schema.create(Schema.Type.INT)).noDefault().endRecord(); + public static final Schema SCHEMA = new LogicalType(NAME).addToSchema(RECORD); + + @Override + public Schema getRecommendedSchema() { + return SCHEMA; + } + + @Override + public Class getConvertedType() { + return BigDecimal.class; + } + + @Override + public String getLogicalTypeName() { + return NAME; + } + + @Override + public BigDecimal fromRecord(IndexedRecord value, Schema schema, LogicalType type) { + BigInteger unscaledValue = BigIntegerConversion.fromBytes((ByteBuffer)value.get(0)); + int scale = (int) value.get(1); + + return new BigDecimal(unscaledValue, scale); + } + + @Override + public IndexedRecord toRecord(BigDecimal value, Schema schema, LogicalType type) { + GenericData.Record record = new GenericData.Record(schema); + record.put(0, BigIntegerConversion.toBytes(value.unscaledValue())); + record.put(1, value.scale()); + return record; + } +} diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/BigIntegerConversion.java b/src/main/java/io/datanerds/avropatch/value/conversion/BigIntegerConversion.java new file mode 100644 index 0000000..af8ad76 --- /dev/null +++ b/src/main/java/io/datanerds/avropatch/value/conversion/BigIntegerConversion.java @@ -0,0 +1,46 @@ +package io.datanerds.avropatch.value.conversion; + +import org.apache.avro.Conversion; +import org.apache.avro.LogicalType; +import org.apache.avro.Schema; + +import java.math.BigInteger; +import java.nio.ByteBuffer; + +public class BigIntegerConversion extends Conversion { + private static final String NAME = "big-integer"; + public static final Schema SCHEMA = new LogicalType(NAME).addToSchema(Schema.create(Schema.Type.BYTES)); + + @Override + public Schema getRecommendedSchema() { + return SCHEMA; + } + + @Override + public Class getConvertedType() { + return BigInteger.class; + } + + @Override + public String getLogicalTypeName() { + return NAME; + } + + @Override + public BigInteger fromBytes(ByteBuffer value, Schema schema, LogicalType type) { + return fromBytes(value); + } + + @Override + public ByteBuffer toBytes(BigInteger value, Schema schema, LogicalType type) { + return toBytes(value); + } + + protected static BigInteger fromBytes(ByteBuffer value) { + return new BigInteger(value.array()); + } + + protected static ByteBuffer toBytes(BigInteger value) { + return ByteBuffer.wrap(value.toByteArray()); + } +} diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/DateConversion.java b/src/main/java/io/datanerds/avropatch/value/conversion/DateConversion.java new file mode 100644 index 0000000..482dfc8 --- /dev/null +++ b/src/main/java/io/datanerds/avropatch/value/conversion/DateConversion.java @@ -0,0 +1,45 @@ +package io.datanerds.avropatch.value.conversion; + +import org.apache.avro.Conversion; +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; + +import java.util.Date; + +/** + * This class is an implementation for Avro's built-in logical type timestamp-millis representing the + * number of milliseconds since January 1, 1970, 00:00:00 GMT. + * + * @see Conversion + * @see org.apache.avro.LogicalTypes.TimestampMillis + */ +public class DateConversion extends Conversion { + private static final String NAME = LogicalTypes.timestampMillis().getName(); + public static final Schema SCHEMA = LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG)); + + @Override + public Schema getRecommendedSchema() { + return SCHEMA; + } + + @Override + public Class getConvertedType() { + return Date.class; + } + + @Override + public String getLogicalTypeName() { + return NAME; + } + + @Override + public Long toLong(Date value, Schema schema, LogicalType type) { + return value.getTime(); + } + + @Override + public Date fromLong(Long value, Schema schema, LogicalType type) { + return new Date(value); + } +} \ No newline at end of file diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/UUIDConversion.java b/src/main/java/io/datanerds/avropatch/value/conversion/UUIDConversion.java new file mode 100644 index 0000000..0f487da --- /dev/null +++ b/src/main/java/io/datanerds/avropatch/value/conversion/UUIDConversion.java @@ -0,0 +1,58 @@ +package io.datanerds.avropatch.value.conversion; + +import org.apache.avro.Conversion; +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericFixed; + +import java.nio.ByteBuffer; +import java.util.UUID; + +public class UUIDConversion extends Conversion { + private static final String NAME = LogicalTypes.uuid().getName(); + private static final String DOC = "TODO"; + private static final int SIZE = 2 * Long.BYTES; + public static final Schema SCHEMA = LogicalTypes.uuid().addToSchema(Schema.createFixed(NAME, DOC, null, SIZE)); + + @Override + public Schema getRecommendedSchema() { + return SCHEMA; + } + + @Override + public Class getConvertedType() { + return UUID.class; + } + + @Override + public String getLogicalTypeName() { + return NAME; + } + + @Override + public GenericFixed toFixed(UUID value, Schema schema, LogicalType type) { + return new GenericFixed() { + @Override + public byte[] bytes() { + ByteBuffer buffer = ByteBuffer.allocate(SIZE); + buffer.putLong(value.getLeastSignificantBits()); + buffer.putLong(value.getMostSignificantBits()); + return buffer.array(); + } + + @Override + public Schema getSchema() { + return SCHEMA; + } + }; + } + + @Override + public UUID fromFixed(GenericFixed value, Schema schema, LogicalType type) { + ByteBuffer buffer = ByteBuffer.wrap(value.bytes()); + long leastSignificantBits = buffer.getLong(); + long mostSignificantBits = buffer.getLong(); + return new UUID(mostSignificantBits, leastSignificantBits); + } +} \ No newline at end of file diff --git a/src/test/java/io/datanerds/avropatch/PatchTest.java b/src/test/java/io/datanerds/avropatch/PatchTest.java index 80072dd..d9a45d8 100644 --- a/src/test/java/io/datanerds/avropatch/PatchTest.java +++ b/src/test/java/io/datanerds/avropatch/PatchTest.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableList; import io.datanerds.avropatch.operation.*; import org.hamcrest.MatcherAssert; +import org.junit.Ignore; import org.junit.Test; import java.io.IOException; @@ -14,6 +15,11 @@ public class PatchTest { + @Test + public void test() { + System.out.println(Patch.SCHEMA.toString()); + } + @Test public void serializesAdd() throws IOException { Patch patch = new Patch(); diff --git a/src/test/java/io/datanerds/avropatch/operation/TestValue.java b/src/test/java/io/datanerds/avropatch/operation/TestValue.java deleted file mode 100644 index 6a43ce9..0000000 --- a/src/test/java/io/datanerds/avropatch/operation/TestValue.java +++ /dev/null @@ -1,16 +0,0 @@ -package io.datanerds.avropatch.operation; - -import org.apache.avro.reflect.AvroSchema; - -public class TestValue { - @AvroSchema(Value.SCHEMA) - public final T value; - - TestValue() { - this.value = null; - } - - TestValue(T value) { - this.value = value; - } -} \ No newline at end of file diff --git a/src/test/java/io/datanerds/avropatch/operation/ValueTest.java b/src/test/java/io/datanerds/avropatch/operation/ValueTest.java deleted file mode 100644 index 84cb649..0000000 --- a/src/test/java/io/datanerds/avropatch/operation/ValueTest.java +++ /dev/null @@ -1,69 +0,0 @@ -package io.datanerds.avropatch.operation; - -import com.google.common.collect.ImmutableList; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericContainer; -import org.apache.avro.generic.GenericDatumReader; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.io.DatumReader; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.io.DecoderFactory; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.reflect.ReflectData; -import org.apache.avro.reflect.ReflectDatumReader; -import org.apache.avro.reflect.ReflectDatumWriter; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -public class ValueTest { - - private static final Schema schema = ReflectData.get().getSchema(TestValue.class); - - @Test - public void simpleValues() throws IOException { - verify(new TestValue<>(42)); - verify(new TestValue<>(42L)); - verify(new TestValue<>((float)42.1234)); - verify(new TestValue<>(42.1234)); - verify(new TestValue<>("Hello World!")); - } - - @Test - public void listValues() throws IOException { - verify(new TestValue<>(ImmutableList.of(12, 21, 42, 99))); - verify(new TestValue<>(ImmutableList.of(12L, 21L, 42L, 99L))); - verify(new TestValue<>(ImmutableList.of((float)12.34, (float)45.6789))); - verify(new TestValue<>(ImmutableList.of(12.34, 45.6789))); - verify(new TestValue<>(ImmutableList.of("Hello", "World", "!"))); - } - - private void verify(TestValue value) throws IOException { - GenericRecord record = toAvro(value, schema); - TestValue object = toObject(record, schema); - assertThat(value.value, is(equalTo(object.value))); - } - - public static R toObject(T avro, Schema schema) throws IOException { - DatumWriter writer = new GenericDatumWriter<>(schema); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - writer.write(avro, EncoderFactory.get().directBinaryEncoder(out, null)); - DatumReader reader = new ReflectDatumReader<>(schema); - return reader.read(null, DecoderFactory.get().binaryDecoder(out.toByteArray(), null)); - } - - public static R toAvro(T object, Schema schema) throws IOException { - DatumWriter writer = new ReflectDatumWriter<>(schema); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - writer.write(object, EncoderFactory.get().directBinaryEncoder(out, null)); - DatumReader reader = new GenericDatumReader<>(schema); - return reader.read(null, DecoderFactory.get().binaryDecoder(out.toByteArray(), null)); - - } -} diff --git a/src/test/java/io/datanerds/avropatch/value/SchemaTest.java b/src/test/java/io/datanerds/avropatch/value/SchemaTest.java new file mode 100644 index 0000000..39e48b1 --- /dev/null +++ b/src/test/java/io/datanerds/avropatch/value/SchemaTest.java @@ -0,0 +1,85 @@ +package io.datanerds.avropatch.value; + +import avro.shaded.com.google.common.collect.ImmutableList; +import io.datanerds.avropatch.value.conversion.BigIntegerConversion; +import io.datanerds.avropatch.value.conversion.DateConversion; +import io.datanerds.avropatch.value.conversion.UUIDConversion; +import org.apache.avro.Conversion; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.Encoder; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.reflect.ReflectDatumReader; +import org.apache.avro.reflect.ReflectDatumWriter; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.math.BigInteger; +import java.util.Collection; +import java.util.List; + +import static io.datanerds.avropatch.value.ValueSchema.SCHEMA; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +public class SchemaTest { + + @Test + public void serializesNull() throws IOException { + SerializationTester.reserializeAndAssert(null); + } + + @Test + public void serializesBigInteger() throws IOException { + SerializationTester.reserializeAndAssert(new BigInteger("364039216459034658923475606490812364908234653")); + SerializationTester.reserializeAndAssert( + new BigInteger[]{new BigInteger("364039216459034658923475606490812364908234653"), + new BigInteger("56729476543655265"), new BigInteger("987591856718456723489")}); + } + + static final class SerializationTester { + + public static final List> CONVERTERS = + ImmutableList.of(new BigIntegerConversion(), new DateConversion(), new UUIDConversion()); + + private SerializationTester() { + } + + public static void reserializeAndAssert(T value) throws IOException { + byte[] bytes = toBytes(value); + T object = toObject(bytes); + assertEquality(object, value); + } + + private static void assertEquality(T value, T object) { + if (value instanceof Collection) { + // Avro serializes collections as arrays + Collection collection = (Collection) value; + assertThat(collection.toArray(), is(equalTo(object))); + } else { + assertThat(object, is(equalTo(value))); + } + } + + private static byte[] toBytes(T value) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + Encoder binaryEncoder = EncoderFactory.get().directBinaryEncoder(outputStream, null); + + GenericDatumWriter writer = new ReflectDatumWriter<>(SCHEMA); + CONVERTERS.forEach(writer.getData()::addLogicalTypeConversion); + writer.write(value, binaryEncoder); + return outputStream.toByteArray(); + } + + private static T toObject(byte[] bytes) throws IOException { + GenericDatumReader reader = new ReflectDatumReader<>(SCHEMA); + CONVERTERS.forEach(reader.getData()::addLogicalTypeConversion); + return reader.read(null, DecoderFactory.get().binaryDecoder(bytes, null)); + } + } +} diff --git a/src/test/java/io/datanerds/avropatch/value/conversion/BigDecimalConversionTest.java b/src/test/java/io/datanerds/avropatch/value/conversion/BigDecimalConversionTest.java new file mode 100644 index 0000000..b643054 --- /dev/null +++ b/src/test/java/io/datanerds/avropatch/value/conversion/BigDecimalConversionTest.java @@ -0,0 +1,43 @@ +package io.datanerds.avropatch.value.conversion; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.avro.Schema; +import org.junit.Test; + +import java.io.IOException; +import java.math.BigDecimal; + +public class BigDecimalConversionTest { + + @Test + public void serializesSingleValue() throws IOException { + ConversionTester + .withSchemata(BigDecimalConversion.SCHEMA) + .withConverters(new BigDecimalConversion()) + .reserializeAndAssert(BigDecimal.valueOf(1234567890L)) + .reserializeAndAssert(new BigDecimal("1234567890123456.78901234567890123456789012345678901234567890")); + } + + @Test + public void serializesList() throws IOException { + ConversionTester + .withSchemata(Schema.createArray(BigDecimalConversion.SCHEMA)) + .withConverters(new BigDecimalConversion()) + .reserializeAndAssert(ImmutableList.of(BigDecimal.valueOf(1234567890L), + new BigDecimal("12345678901234567.8901234567890123456789012345678901234567890"), + new BigDecimal("9867451234560123985690347590378412095789345624401928453450891" + + "9894375093427509345702.349572349089034759349046758903758902347568902347589023467"))); + } + + @Test + public void serializesMap() throws IOException { + ConversionTester + .withSchemata(Schema.createMap(BigDecimalConversion.SCHEMA)) + .withConverters(new BigDecimalConversion()) + .reserializeAndAssert(ImmutableMap.of( + "key 1", new BigDecimal("12345678901234567.8901234567890123456789012345678901234567890"), + "key 2", new BigDecimal("9867451231209578924401928453450891.4675890375890232347589023467"))); + } + +} \ No newline at end of file diff --git a/src/test/java/io/datanerds/avropatch/value/conversion/BigIntegerConversionTest.java b/src/test/java/io/datanerds/avropatch/value/conversion/BigIntegerConversionTest.java new file mode 100644 index 0000000..08e8912 --- /dev/null +++ b/src/test/java/io/datanerds/avropatch/value/conversion/BigIntegerConversionTest.java @@ -0,0 +1,42 @@ +package io.datanerds.avropatch.value.conversion; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.avro.Schema; +import org.junit.Test; + +import java.io.IOException; +import java.math.BigInteger; + +public class BigIntegerConversionTest { + + @Test + public void serializesSingleValue() throws IOException { + ConversionTester + .withSchemata(BigIntegerConversion.SCHEMA) + .withConverters(new BigIntegerConversion()) + .reserializeAndAssert(BigInteger.valueOf(1234567890L)) + .reserializeAndAssert(new BigInteger("123456789012345678901234567890123456789012345678901234567890")); + } + + @Test + public void serializesList() throws IOException { + ConversionTester + .withSchemata(Schema.createArray(BigIntegerConversion.SCHEMA)) + .withConverters(new BigIntegerConversion()) + .reserializeAndAssert(ImmutableList.of(BigInteger.valueOf(1234567890L), + new BigInteger("123456789012345678901234567890123456789012345678901234567890"), + new BigInteger("9867451234560123985690347590378412095789345624401928453450891" + + "98943750934275093457023495723490890347590349046758903758902347568902347589023467"))); + } + + @Test + public void serializesMap() throws IOException { + ConversionTester + .withSchemata(Schema.createMap(BigIntegerConversion.SCHEMA)) + .withConverters(new BigIntegerConversion()) + .reserializeAndAssert(ImmutableMap.of( + "key 1", new BigInteger("123456789012345678901234567890123456789012345678901234567890"), + "key 2", new BigInteger("875634975613486341456634912563464598436589234658974365134"))); + } +} \ No newline at end of file diff --git a/src/test/java/io/datanerds/avropatch/value/conversion/ConversionTester.java b/src/test/java/io/datanerds/avropatch/value/conversion/ConversionTester.java new file mode 100644 index 0000000..debefc3 --- /dev/null +++ b/src/test/java/io/datanerds/avropatch/value/conversion/ConversionTester.java @@ -0,0 +1,83 @@ +package io.datanerds.avropatch.value.conversion; + +import org.apache.avro.Conversion; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.Encoder; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.reflect.ReflectDatumReader; +import org.apache.avro.reflect.ReflectDatumWriter; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Collection; +import java.util.Objects; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +public final class ConversionTester { + + private final GenericDatumWriter writer; + private final GenericDatumReader reader; + + private ConversionTester(Schema schema) { + Objects.nonNull(schema); + reader = new ReflectDatumReader<>(schema); + writer = new ReflectDatumWriter<>(schema); + } + + public static final ConversionTester withSchemata(Schema... types) { + Objects.nonNull(types); + if (types.length < 1) { + throw new IllegalArgumentException("At least one type needed for schema"); + } + + SchemaBuilder.UnionAccumulator union = SchemaBuilder.unionOf().type(types[0]); + for (int i = 1; i < types.length; i++) { + union.and().type(types[i]); + } + return new ConversionTester(union.endUnion()); + } + + public ConversionTester withConverters(Conversion... converters) { + for (Conversion conversion : converters) { + reader.getData().addLogicalTypeConversion(conversion); + writer.getData().addLogicalTypeConversion(conversion); + } + return this; + } + + public ConversionTester reserializeAndAssert(T value) throws IOException { + byte[] bytes = toBytes(value); + T object = toObject(bytes); + assertEquality(value, object); + return this; + } + + private void assertEquality(T value, T object) { + if (value instanceof Collection) { + // Avro serializes collections as arrays + Collection collection = (Collection) value; + assertThat(collection.toArray(), is(equalTo(object))); + } else { + assertThat(object, is(equalTo(value))); + } + } + + private byte[] toBytes(T value) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + Encoder binaryEncoder = EncoderFactory.get().directBinaryEncoder(outputStream, null); + + writer.write(value, binaryEncoder); + return outputStream.toByteArray(); + } + + private T toObject(byte[] bytes) throws IOException { + return reader.read(null, DecoderFactory.get().binaryDecoder(bytes, null)); + } +} \ No newline at end of file diff --git a/src/test/java/io/datanerds/avropatch/value/conversion/DateConversionTest.java b/src/test/java/io/datanerds/avropatch/value/conversion/DateConversionTest.java new file mode 100644 index 0000000..4a1848f --- /dev/null +++ b/src/test/java/io/datanerds/avropatch/value/conversion/DateConversionTest.java @@ -0,0 +1,41 @@ +package io.datanerds.avropatch.value.conversion; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.avro.Schema; +import org.junit.Test; + +import java.io.IOException; +import java.util.Date; + +public class DateConversionTest { + + @Test + public void serializesSingleValue() throws IOException { + ConversionTester + .withSchemata(DateConversion.SCHEMA) + .withConverters(new DateConversion()) + .reserializeAndAssert(new Date()) + .reserializeAndAssert(new Date()); + } + + @Test + public void serializesList() throws IOException { + ConversionTester + .withSchemata(Schema.createArray(DateConversion.SCHEMA)) + .withConverters(new DateConversion()) + .reserializeAndAssert(ImmutableList.of(new Date(), new Date(), new Date(), new Date())); + + } + + @Test + public void serializesMap() throws IOException { + ConversionTester + .withSchemata(Schema.createMap(DateConversion.SCHEMA)) + .withConverters(new DateConversion()) + .reserializeAndAssert(ImmutableMap.of( + "key 1", new Date(), + "key 2", new Date(), + "key 3", new Date())); + } +} \ No newline at end of file diff --git a/src/test/java/io/datanerds/avropatch/value/conversion/UUIDConversionTest.java b/src/test/java/io/datanerds/avropatch/value/conversion/UUIDConversionTest.java new file mode 100644 index 0000000..be75b17 --- /dev/null +++ b/src/test/java/io/datanerds/avropatch/value/conversion/UUIDConversionTest.java @@ -0,0 +1,40 @@ +package io.datanerds.avropatch.value.conversion; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.avro.Schema; +import org.junit.Test; + +import java.io.IOException; +import java.util.UUID; + +public class UUIDConversionTest { + + @Test + public void serializesSingleValue() throws IOException { + ConversionTester + .withSchemata(UUIDConversion.SCHEMA) + .withConverters(new UUIDConversion()) + .reserializeAndAssert(UUID.randomUUID()) + .reserializeAndAssert(UUID.randomUUID()); + } + + @Test + public void serializesList() throws IOException { + ConversionTester + .withSchemata(Schema.createArray(UUIDConversion.SCHEMA)) + .withConverters(new UUIDConversion()) + .reserializeAndAssert(ImmutableList.of(UUID.randomUUID(), UUID.randomUUID(), UUID.randomUUID())); + } + + @Test + public void serializesMap() throws IOException { + ConversionTester + .withSchemata(Schema.createMap(UUIDConversion.SCHEMA)) + .withConverters(new UUIDConversion()) + .reserializeAndAssert(ImmutableMap.of( + "key 1", UUID.randomUUID(), + "key 2", UUID.randomUUID(), + "key 3", UUID.randomUUID())); + } +} \ No newline at end of file From 1ff80782d319227dcf8b586e8b461d92bdba01b5 Mon Sep 17 00:00:00 2001 From: Frank Wisniewski Date: Tue, 8 Nov 2016 09:31:40 -0800 Subject: [PATCH 02/19] Fixed initialization of custom type converters. --- .../java/io/datanerds/avropatch/Patch.java | 34 ++++++-- .../datanerds/avropatch/operation/Value.java | 57 +++++++++---- .../avropatch/value/ValueSchema.java | 31 ------- .../value/conversion/AvroConversion.java | 22 +++++ .../conversion/BigDecimalConversion.java | 26 +++--- .../conversion/BigIntegerConversion.java | 25 +++--- .../value/conversion/CustomTypes.java | 51 +++++++++++ .../value/conversion/DateConversion.java | 44 +++++++--- .../value/conversion/UUIDConversion.java | 17 ++-- .../io/datanerds/avropatch/PatchTest.java | 39 +++++++-- .../datanerds/avropatch/value/SchemaTest.java | 85 ------------------- .../conversion/BigDecimalConversionTest.java | 13 +-- .../conversion/BigIntegerConversionTest.java | 7 +- .../value/conversion/ConversionTester.java | 25 +++--- .../value/conversion/DateConversionTest.java | 7 +- .../value/conversion/UUIDConversionTest.java | 7 +- 16 files changed, 261 insertions(+), 229 deletions(-) delete mode 100644 src/main/java/io/datanerds/avropatch/value/ValueSchema.java create mode 100644 src/main/java/io/datanerds/avropatch/value/conversion/AvroConversion.java create mode 100644 src/main/java/io/datanerds/avropatch/value/conversion/CustomTypes.java delete mode 100644 src/test/java/io/datanerds/avropatch/value/SchemaTest.java diff --git a/src/main/java/io/datanerds/avropatch/Patch.java b/src/main/java/io/datanerds/avropatch/Patch.java index 8aa0fdd..9a1596c 100644 --- a/src/main/java/io/datanerds/avropatch/Patch.java +++ b/src/main/java/io/datanerds/avropatch/Patch.java @@ -1,11 +1,11 @@ package io.datanerds.avropatch; +import avro.shaded.com.google.common.collect.ImmutableList; import io.datanerds.avropatch.operation.Operation; +import io.datanerds.avropatch.value.conversion.*; import org.apache.avro.Schema; import org.apache.avro.io.*; import org.apache.avro.reflect.ReflectData; -import org.apache.avro.reflect.ReflectDatumReader; -import org.apache.avro.reflect.ReflectDatumWriter; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -15,8 +15,9 @@ public class Patch { - public static final Schema SCHEMA = ReflectData.get().getSchema(Patch.class); - private static final DatumWriter writer = new ReflectDatumWriter<>(SCHEMA); + private static final PatchSerializer SERIALIZER = new PatchSerializer(); + public static final Schema SCHEMA = SERIALIZER.schema; + private final List operations; public Patch() { @@ -38,13 +39,30 @@ public List getOperations() { public byte[] toBytes() throws IOException { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); Encoder binaryEncoder = EncoderFactory.get().directBinaryEncoder(outputStream, null); - - writer.write(this, binaryEncoder); + SERIALIZER.writer.write(this, binaryEncoder); return outputStream.toByteArray(); } public static Patch of(byte[] bytes) throws IOException { - DatumReader reader = new ReflectDatumReader<>(SCHEMA); - return reader.read(null, DecoderFactory.get().binaryDecoder(bytes, null)); + return SERIALIZER.reader.read(null, DecoderFactory.get().binaryDecoder(bytes, null)); + } + + private static class PatchSerializer { + + static final List> AVRO_CONVERSIONS = + ImmutableList.of(new DateConversion(), new BigIntegerConversion(), new BigDecimalConversion(), + new UUIDConversion()); + final DatumWriter writer; + final DatumReader reader; + final Schema schema; + + private PatchSerializer() { + ReflectData data = new ReflectData(); + AVRO_CONVERSIONS.forEach(avroConversion -> avroConversion.register(data)); + schema = data.getSchema(Patch.class); + + writer = data.createDatumWriter(schema); + reader = data.createDatumReader(schema); + } } } diff --git a/src/main/java/io/datanerds/avropatch/operation/Value.java b/src/main/java/io/datanerds/avropatch/operation/Value.java index 958172d..7fb456d 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Value.java +++ b/src/main/java/io/datanerds/avropatch/operation/Value.java @@ -1,25 +1,48 @@ package io.datanerds.avropatch.operation; -import org.apache.avro.Schema; - /** * This interface holds the schema for all valid value types. */ interface Value { String SCHEMA = - "[" + - "\"boolean\"," + - "\"long\"," + - "\"int\"," + - "\"float\"," + - "\"double\"," + - "\"string\"," + - "{\"type\": \"array\", \"items\": [" + - "\"long\"," + - "\"int\"," + - "\"float\"," + - "\"double\"," + - "\"string\"" + - "]}" + - "]"; + "[{" + + " \"type\": \"record\"," + + " \"name\": \"decimal\"," + + " \"fields\": [{" + + " \"name\": \"unscaledValue\"," + + " \"type\": {" + + " \"type\": \"bytes\"," + + " \"logicalType\": \"big-integer\"" + + " }" + + " }, {" + + " \"name\": \"scale\"," + + " \"type\": \"int\"" + + " }]," + + " \"logicalType\": \"big-decimal\"" + + "}, {" + + " \"type\": \"bytes\"," + + " \"logicalType\": \"big-integer\"" + + "}, \"boolean\", {" + + " \"type\": \"fixed\"," + + " \"name\": \"timestamp\"," + + " \"size\": 8," + + " \"logicalType\": \"timestamp\"" + + "}, \"double\", \"float\", \"int\", \"long\", \"null\", \"string\", {" + + " \"type\": \"fixed\"," + + " \"name\": \"uuid\"," + + " \"size\": 16," + + " \"logicalType\": \"uuid\"" + + "}, {" + + " \"type\": \"array\"," + + " \"items\": [\"decimal\", {" + + " \"type\": \"bytes\"," + + " \"logicalType\": \"big-integer\"" + + " }, \"boolean\", \"timestamp\", \"double\", \"float\", \"int\", \"long\", \"null\", \"string\", \"uuid\"]" + + "}, {\n" + + " \"type\": \"map\"," + + " \"values\": [\"decimal\", {" + + " \"type\": \"bytes\"," + + " \"logicalType\": \"big-integer\"" + + " }, \"boolean\", \"timestamp\", \"double\", \"float\", \"int\", \"long\", \"null\", \"string\", \"uuid\"]" + + "}]"; } diff --git a/src/main/java/io/datanerds/avropatch/value/ValueSchema.java b/src/main/java/io/datanerds/avropatch/value/ValueSchema.java deleted file mode 100644 index fe5df6c..0000000 --- a/src/main/java/io/datanerds/avropatch/value/ValueSchema.java +++ /dev/null @@ -1,31 +0,0 @@ -package io.datanerds.avropatch.value; - -import io.datanerds.avropatch.value.conversion.BigDecimalConversion; -import io.datanerds.avropatch.value.conversion.BigIntegerConversion; -import io.datanerds.avropatch.value.conversion.DateConversion; -import io.datanerds.avropatch.value.conversion.UUIDConversion; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Type; - -import static org.apache.avro.Schema.*; - -public class ValueSchema { - - public static final Schema BOOLEAN = create(Type.BOOLEAN); - public static final Schema DOUBLE = create(Type.DOUBLE); - public static final Schema FLOAT = create(Type.FLOAT); - public static final Schema INTEGER = create(Type.INT); - public static final Schema LONG = create(Type.LONG); - public static final Schema NULL = create(Type.NULL); - public static final Schema STRING = create(Type.STRING); - - public static final Schema BIG_DECIMAL = BigDecimalConversion.SCHEMA; - public static final Schema BIG_INTEGER = BigIntegerConversion.SCHEMA; - public static final Schema DATE = DateConversion.SCHEMA; - public static final Schema UUID = UUIDConversion.SCHEMA; - - public static final Schema SCHEMA = - createUnion(BIG_DECIMAL, BIG_INTEGER, BOOLEAN, DATE, DOUBLE, FLOAT, INTEGER, /*LONG,*/ NULL, STRING, UUID, - createArray(createUnion(BIG_DECIMAL, BIG_INTEGER, BOOLEAN, DATE, DOUBLE, FLOAT, INTEGER, /*LONG,*/ NULL, STRING, UUID)), - createMap(createUnion(BIG_DECIMAL, BIG_INTEGER, BOOLEAN, DATE, DOUBLE, FLOAT, INTEGER, /*LONG,*/ NULL, STRING, UUID))); -} diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/AvroConversion.java b/src/main/java/io/datanerds/avropatch/value/conversion/AvroConversion.java new file mode 100644 index 0000000..fbdf28e --- /dev/null +++ b/src/main/java/io/datanerds/avropatch/value/conversion/AvroConversion.java @@ -0,0 +1,22 @@ +package io.datanerds.avropatch.value.conversion; + +import org.apache.avro.Conversion; +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.generic.GenericData; + +/** + * This abstract class serves as base class for all custom type conversions supported by this library. + * @param + */ +public abstract class AvroConversion extends Conversion { + + public void register(GenericData data) { + data.addLogicalTypeConversion(this); + } + + protected static void registerLogicalType(String name, LogicalType type) { + LogicalTypes.register(name, schema -> type); + } + +} diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/BigDecimalConversion.java b/src/main/java/io/datanerds/avropatch/value/conversion/BigDecimalConversion.java index fa973e8..9fa6735 100644 --- a/src/main/java/io/datanerds/avropatch/value/conversion/BigDecimalConversion.java +++ b/src/main/java/io/datanerds/avropatch/value/conversion/BigDecimalConversion.java @@ -1,31 +1,30 @@ package io.datanerds.avropatch.value.conversion; -import org.apache.avro.Conversion; +import io.datanerds.avropatch.value.conversion.CustomTypes.BigDecimalType; import org.apache.avro.LogicalType; import org.apache.avro.Schema; -import org.apache.avro.SchemaBuilder; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.IndexedRecord; import java.math.BigDecimal; import java.math.BigInteger; -import java.nio.ByteBuffer; /** - * TODO + * This class converts an {@link BigDecimal} value into am Avro {@link IndexedRecord} and back. It depends on + * {@link BigIntegerConversion} since it serializes the {@link BigDecimal}s unscaled value into a {@link BigInteger} and + * its scale into an {@link Integer}. * * @see BigIntegerConversion */ -public class BigDecimalConversion extends Conversion { +public class BigDecimalConversion extends AvroConversion { - private static final String NAME = "big-decimal"; - private static final String DOC = "TODO"; - private static final Schema RECORD = SchemaBuilder.record("decimal").doc(DOC).fields().name("unscaledValue").type(BigIntegerConversion.SCHEMA).noDefault().name("scale").type(Schema.create(Schema.Type.INT)).noDefault().endRecord(); - public static final Schema SCHEMA = new LogicalType(NAME).addToSchema(RECORD); + static { + registerLogicalType(BigDecimalType.NAME, BigDecimalType.LOGICAL_TYPE); + } @Override public Schema getRecommendedSchema() { - return SCHEMA; + return BigDecimalType.SCHEMA; } @Override @@ -35,21 +34,20 @@ public Class getConvertedType() { @Override public String getLogicalTypeName() { - return NAME; + return BigDecimalType.NAME; } @Override public BigDecimal fromRecord(IndexedRecord value, Schema schema, LogicalType type) { - BigInteger unscaledValue = BigIntegerConversion.fromBytes((ByteBuffer)value.get(0)); + BigInteger unscaledValue = (BigInteger)value.get(0); int scale = (int) value.get(1); - return new BigDecimal(unscaledValue, scale); } @Override public IndexedRecord toRecord(BigDecimal value, Schema schema, LogicalType type) { GenericData.Record record = new GenericData.Record(schema); - record.put(0, BigIntegerConversion.toBytes(value.unscaledValue())); + record.put(0, value.unscaledValue()); record.put(1, value.scale()); return record; } diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/BigIntegerConversion.java b/src/main/java/io/datanerds/avropatch/value/conversion/BigIntegerConversion.java index af8ad76..0de50d8 100644 --- a/src/main/java/io/datanerds/avropatch/value/conversion/BigIntegerConversion.java +++ b/src/main/java/io/datanerds/avropatch/value/conversion/BigIntegerConversion.java @@ -1,19 +1,22 @@ package io.datanerds.avropatch.value.conversion; -import org.apache.avro.Conversion; +import io.datanerds.avropatch.value.conversion.CustomTypes.BigIntegerType; import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import java.math.BigInteger; import java.nio.ByteBuffer; -public class BigIntegerConversion extends Conversion { - private static final String NAME = "big-integer"; - public static final Schema SCHEMA = new LogicalType(NAME).addToSchema(Schema.create(Schema.Type.BYTES)); +public class BigIntegerConversion extends AvroConversion { + + static { + LogicalTypes.register(BigIntegerType.NAME, schema -> BigIntegerType.LOGICAL_TYPE); + } @Override public Schema getRecommendedSchema() { - return SCHEMA; + return BigIntegerType.SCHEMA; } @Override @@ -23,24 +26,16 @@ public Class getConvertedType() { @Override public String getLogicalTypeName() { - return NAME; + return BigIntegerType.NAME; } @Override public BigInteger fromBytes(ByteBuffer value, Schema schema, LogicalType type) { - return fromBytes(value); + return new BigInteger(value.array()); } @Override public ByteBuffer toBytes(BigInteger value, Schema schema, LogicalType type) { - return toBytes(value); - } - - protected static BigInteger fromBytes(ByteBuffer value) { - return new BigInteger(value.array()); - } - - protected static ByteBuffer toBytes(BigInteger value) { return ByteBuffer.wrap(value.toByteArray()); } } diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/CustomTypes.java b/src/main/java/io/datanerds/avropatch/value/conversion/CustomTypes.java new file mode 100644 index 0000000..aac382b --- /dev/null +++ b/src/main/java/io/datanerds/avropatch/value/conversion/CustomTypes.java @@ -0,0 +1,51 @@ +package io.datanerds.avropatch.value.conversion; + +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; + +/** + * This interface holds the schema constants for all supported custom types. + * + * @see UUIDConversion + * @see DateConversion + * @see BigDecimalConversion + * @see BigIntegerConversion + */ +public interface CustomTypes { + + interface BigDecimalType { + String NAME = "big-decimal"; + String DOC = "BigDecimal value represented via it's scale and unscaled value."; + LogicalType LOGICAL_TYPE = new LogicalType(NAME); + Schema RECORD = SchemaBuilder.record("decimal").doc(BigDecimalType.DOC).fields() + .name("unscaledValue").type(BigIntegerType.SCHEMA).noDefault() + .name("scale").type(Schema.create(Schema.Type.INT)).noDefault() + .endRecord(); + + Schema SCHEMA = BigDecimalType.LOGICAL_TYPE.addToSchema(RECORD); + } + + interface BigIntegerType { + String NAME = "big-integer"; + LogicalType LOGICAL_TYPE = new LogicalType(NAME); + Schema SCHEMA = LOGICAL_TYPE.addToSchema(Schema.create(Schema.Type.BYTES)); + + } + + interface DateType { + String NAME = "timestamp"; + String DOC = "Timestamp representing the number of milliseconds since January 1, 1970, 00:00:00 GMT"; + LogicalType LOGICAL_TYPE = new LogicalType(NAME); + int SIZE = Long.BYTES; + Schema SCHEMA = LOGICAL_TYPE.addToSchema(Schema.createFixed(NAME, DOC, null, SIZE)); + } + + interface UuidType { + String NAME = LogicalTypes.uuid().getName(); + String DOC = "UUID serialized via two long values: It's most significant and least significant 64 bits."; + int SIZE = 2 * Long.BYTES; + Schema SCHEMA = LogicalTypes.uuid().addToSchema(Schema.createFixed(NAME, DOC, null, SIZE)); + } +} diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/DateConversion.java b/src/main/java/io/datanerds/avropatch/value/conversion/DateConversion.java index 482dfc8..841aef3 100644 --- a/src/main/java/io/datanerds/avropatch/value/conversion/DateConversion.java +++ b/src/main/java/io/datanerds/avropatch/value/conversion/DateConversion.java @@ -1,26 +1,32 @@ package io.datanerds.avropatch.value.conversion; +import io.datanerds.avropatch.value.conversion.CustomTypes.DateType; import org.apache.avro.Conversion; import org.apache.avro.LogicalType; -import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; +import org.apache.avro.generic.GenericFixed; +import java.nio.ByteBuffer; import java.util.Date; /** - * This class is an implementation for Avro's built-in logical type timestamp-millis representing the - * number of milliseconds since January 1, 1970, 00:00:00 GMT. + * This class is an implementation of a timestamp representing the number of milliseconds since + * January 1, 1970, 00:00:00 GMT. Avro's built-in logical type timestamp-millis is not used since it can only + * be used with an underlying long type which collides with primitive long in union for value schema. * * @see Conversion * @see org.apache.avro.LogicalTypes.TimestampMillis + * @see org.apache.avro.Schema.Type */ -public class DateConversion extends Conversion { - private static final String NAME = LogicalTypes.timestampMillis().getName(); - public static final Schema SCHEMA = LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG)); +public class DateConversion extends AvroConversion { + + static { + registerLogicalType(DateType.NAME, DateType.LOGICAL_TYPE); + } @Override public Schema getRecommendedSchema() { - return SCHEMA; + return DateType.SCHEMA; } @Override @@ -30,16 +36,30 @@ public Class getConvertedType() { @Override public String getLogicalTypeName() { - return NAME; + return DateType.NAME; } @Override - public Long toLong(Date value, Schema schema, LogicalType type) { - return value.getTime(); + public GenericFixed toFixed(Date value, Schema schema, LogicalType type) { + return new GenericFixed() { + @Override + public byte[] bytes() { + ByteBuffer buffer = ByteBuffer.allocate(DateType.SIZE); + buffer.putLong(value.getTime()); + return buffer.array(); + } + + @Override + public Schema getSchema() { + return DateType.SCHEMA; + } + }; } @Override - public Date fromLong(Long value, Schema schema, LogicalType type) { - return new Date(value); + public Date fromFixed(GenericFixed value, Schema schema, LogicalType type) { + ByteBuffer buffer = ByteBuffer.wrap(value.bytes()); + long time = buffer.getLong(); + return new Date(time); } } \ No newline at end of file diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/UUIDConversion.java b/src/main/java/io/datanerds/avropatch/value/conversion/UUIDConversion.java index 0f487da..3405410 100644 --- a/src/main/java/io/datanerds/avropatch/value/conversion/UUIDConversion.java +++ b/src/main/java/io/datanerds/avropatch/value/conversion/UUIDConversion.java @@ -1,23 +1,18 @@ package io.datanerds.avropatch.value.conversion; -import org.apache.avro.Conversion; +import io.datanerds.avropatch.value.conversion.CustomTypes.UuidType; import org.apache.avro.LogicalType; -import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import org.apache.avro.generic.GenericFixed; import java.nio.ByteBuffer; import java.util.UUID; -public class UUIDConversion extends Conversion { - private static final String NAME = LogicalTypes.uuid().getName(); - private static final String DOC = "TODO"; - private static final int SIZE = 2 * Long.BYTES; - public static final Schema SCHEMA = LogicalTypes.uuid().addToSchema(Schema.createFixed(NAME, DOC, null, SIZE)); +public class UUIDConversion extends AvroConversion { @Override public Schema getRecommendedSchema() { - return SCHEMA; + return UuidType.SCHEMA; } @Override @@ -27,7 +22,7 @@ public Class getConvertedType() { @Override public String getLogicalTypeName() { - return NAME; + return UuidType.NAME; } @Override @@ -35,7 +30,7 @@ public GenericFixed toFixed(UUID value, Schema schema, LogicalType type) { return new GenericFixed() { @Override public byte[] bytes() { - ByteBuffer buffer = ByteBuffer.allocate(SIZE); + ByteBuffer buffer = ByteBuffer.allocate(UuidType.SIZE); buffer.putLong(value.getLeastSignificantBits()); buffer.putLong(value.getMostSignificantBits()); return buffer.array(); @@ -43,7 +38,7 @@ public byte[] bytes() { @Override public Schema getSchema() { - return SCHEMA; + return UuidType.SCHEMA; } }; } diff --git a/src/test/java/io/datanerds/avropatch/PatchTest.java b/src/test/java/io/datanerds/avropatch/PatchTest.java index d9a45d8..e4ca0b5 100644 --- a/src/test/java/io/datanerds/avropatch/PatchTest.java +++ b/src/test/java/io/datanerds/avropatch/PatchTest.java @@ -3,11 +3,14 @@ import com.google.common.collect.ImmutableList; import io.datanerds.avropatch.operation.*; import org.hamcrest.MatcherAssert; -import org.junit.Ignore; import org.junit.Test; import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Date; import java.util.List; +import java.util.UUID; import static io.datanerds.avropatch.operation.matcher.OperationMatchers.hasItem; import static io.datanerds.avropatch.operation.matcher.OperationMatchers.hasItems; @@ -15,11 +18,6 @@ public class PatchTest { - @Test - public void test() { - System.out.println(Patch.SCHEMA.toString()); - } - @Test public void serializesAdd() throws IOException { Patch patch = new Patch(); @@ -105,4 +103,33 @@ public void serializesBunchOfOperations() throws IOException { new io.datanerds.avropatch.operation.Test(Path.parse("/person/number"), 42L))); } + //public static final Schema[] BASIC_TYPES = {BIG_DECIMAL, BIG_INTEGER, BOOLEAN, DATE, DOUBLE, FLOAT, INTEGER, LONG, NULL, STRING, UUID}; + @Test + public void serializesDefaultValueTypes() throws IOException { + Date date = new Date(); + UUID uuid = UUID.randomUUID(); + Patch patch = new Patch(ImmutableList.of(new Add<>(Path.of("some", "value"), "John Doe"), + new Add<>(Path.of("some", "value"), 42), + new Add<>(Path.of("some", "value"), 42L), + new Add<>(Path.of("some", "value"), uuid), + new Add<>(Path.of("some", "value"), new BigDecimal("128976548936549275.9674592348654789")), + new Add<>(Path.of("some", "value"), new BigInteger("90374692364523789623490569234562347895")), + new Add<>(Path.of("some", "value"), true), + new Add<>(Path.of("some", "value"), date), + new Add<>(Path.of("some", "value"), 4234.2345))); + + byte[] bytes = patch.toBytes(); + List operations = Patch.of(bytes).getOperations(); + MatcherAssert.assertThat(operations, hasSize(9)); + MatcherAssert.assertThat(operations, hasItems(new Add<>(Path.of("some", "value"), "John Doe"), + new Add<>(Path.of("some", "value"), 42), + new Add<>(Path.of("some", "value"), 42L), + new Add<>(Path.of("some", "value"), uuid), + new Add<>(Path.of("some", "value"), new BigDecimal("128976548936549275.9674592348654789")), + new Add<>(Path.of("some", "value"), new BigInteger("90374692364523789623490569234562347895")), + new Add<>(Path.of("some", "value"), true), + new Add<>(Path.of("some", "value"), date), + new Add<>(Path.of("some", "value"), 4234.2345))); + } + } \ No newline at end of file diff --git a/src/test/java/io/datanerds/avropatch/value/SchemaTest.java b/src/test/java/io/datanerds/avropatch/value/SchemaTest.java deleted file mode 100644 index 39e48b1..0000000 --- a/src/test/java/io/datanerds/avropatch/value/SchemaTest.java +++ /dev/null @@ -1,85 +0,0 @@ -package io.datanerds.avropatch.value; - -import avro.shaded.com.google.common.collect.ImmutableList; -import io.datanerds.avropatch.value.conversion.BigIntegerConversion; -import io.datanerds.avropatch.value.conversion.DateConversion; -import io.datanerds.avropatch.value.conversion.UUIDConversion; -import org.apache.avro.Conversion; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericDatumReader; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.io.DecoderFactory; -import org.apache.avro.io.Encoder; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.reflect.ReflectDatumReader; -import org.apache.avro.reflect.ReflectDatumWriter; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.math.BigInteger; -import java.util.Collection; -import java.util.List; - -import static io.datanerds.avropatch.value.ValueSchema.SCHEMA; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -public class SchemaTest { - - @Test - public void serializesNull() throws IOException { - SerializationTester.reserializeAndAssert(null); - } - - @Test - public void serializesBigInteger() throws IOException { - SerializationTester.reserializeAndAssert(new BigInteger("364039216459034658923475606490812364908234653")); - SerializationTester.reserializeAndAssert( - new BigInteger[]{new BigInteger("364039216459034658923475606490812364908234653"), - new BigInteger("56729476543655265"), new BigInteger("987591856718456723489")}); - } - - static final class SerializationTester { - - public static final List> CONVERTERS = - ImmutableList.of(new BigIntegerConversion(), new DateConversion(), new UUIDConversion()); - - private SerializationTester() { - } - - public static void reserializeAndAssert(T value) throws IOException { - byte[] bytes = toBytes(value); - T object = toObject(bytes); - assertEquality(object, value); - } - - private static void assertEquality(T value, T object) { - if (value instanceof Collection) { - // Avro serializes collections as arrays - Collection collection = (Collection) value; - assertThat(collection.toArray(), is(equalTo(object))); - } else { - assertThat(object, is(equalTo(value))); - } - } - - private static byte[] toBytes(T value) throws IOException { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - Encoder binaryEncoder = EncoderFactory.get().directBinaryEncoder(outputStream, null); - - GenericDatumWriter writer = new ReflectDatumWriter<>(SCHEMA); - CONVERTERS.forEach(writer.getData()::addLogicalTypeConversion); - writer.write(value, binaryEncoder); - return outputStream.toByteArray(); - } - - private static T toObject(byte[] bytes) throws IOException { - GenericDatumReader reader = new ReflectDatumReader<>(SCHEMA); - CONVERTERS.forEach(reader.getData()::addLogicalTypeConversion); - return reader.read(null, DecoderFactory.get().binaryDecoder(bytes, null)); - } - } -} diff --git a/src/test/java/io/datanerds/avropatch/value/conversion/BigDecimalConversionTest.java b/src/test/java/io/datanerds/avropatch/value/conversion/BigDecimalConversionTest.java index b643054..7809a2e 100644 --- a/src/test/java/io/datanerds/avropatch/value/conversion/BigDecimalConversionTest.java +++ b/src/test/java/io/datanerds/avropatch/value/conversion/BigDecimalConversionTest.java @@ -2,6 +2,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.datanerds.avropatch.value.conversion.CustomTypes.BigDecimalType; import org.apache.avro.Schema; import org.junit.Test; @@ -13,8 +14,8 @@ public class BigDecimalConversionTest { @Test public void serializesSingleValue() throws IOException { ConversionTester - .withSchemata(BigDecimalConversion.SCHEMA) - .withConverters(new BigDecimalConversion()) + .withSchemata(BigDecimalType.SCHEMA) + .withConverters(new BigIntegerConversion(), new BigDecimalConversion()) .reserializeAndAssert(BigDecimal.valueOf(1234567890L)) .reserializeAndAssert(new BigDecimal("1234567890123456.78901234567890123456789012345678901234567890")); } @@ -22,8 +23,8 @@ public void serializesSingleValue() throws IOException { @Test public void serializesList() throws IOException { ConversionTester - .withSchemata(Schema.createArray(BigDecimalConversion.SCHEMA)) - .withConverters(new BigDecimalConversion()) + .withSchemata(Schema.createArray(BigDecimalType.SCHEMA)) + .withConverters(new BigIntegerConversion(), new BigDecimalConversion()) .reserializeAndAssert(ImmutableList.of(BigDecimal.valueOf(1234567890L), new BigDecimal("12345678901234567.8901234567890123456789012345678901234567890"), new BigDecimal("9867451234560123985690347590378412095789345624401928453450891" @@ -33,8 +34,8 @@ public void serializesList() throws IOException { @Test public void serializesMap() throws IOException { ConversionTester - .withSchemata(Schema.createMap(BigDecimalConversion.SCHEMA)) - .withConverters(new BigDecimalConversion()) + .withSchemata(Schema.createMap(BigDecimalType.SCHEMA)) + .withConverters(new BigIntegerConversion(), new BigDecimalConversion()) .reserializeAndAssert(ImmutableMap.of( "key 1", new BigDecimal("12345678901234567.8901234567890123456789012345678901234567890"), "key 2", new BigDecimal("9867451231209578924401928453450891.4675890375890232347589023467"))); diff --git a/src/test/java/io/datanerds/avropatch/value/conversion/BigIntegerConversionTest.java b/src/test/java/io/datanerds/avropatch/value/conversion/BigIntegerConversionTest.java index 08e8912..18d611a 100644 --- a/src/test/java/io/datanerds/avropatch/value/conversion/BigIntegerConversionTest.java +++ b/src/test/java/io/datanerds/avropatch/value/conversion/BigIntegerConversionTest.java @@ -2,6 +2,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.datanerds.avropatch.value.conversion.CustomTypes.BigIntegerType; import org.apache.avro.Schema; import org.junit.Test; @@ -13,7 +14,7 @@ public class BigIntegerConversionTest { @Test public void serializesSingleValue() throws IOException { ConversionTester - .withSchemata(BigIntegerConversion.SCHEMA) + .withSchemata(BigIntegerType.SCHEMA) .withConverters(new BigIntegerConversion()) .reserializeAndAssert(BigInteger.valueOf(1234567890L)) .reserializeAndAssert(new BigInteger("123456789012345678901234567890123456789012345678901234567890")); @@ -22,7 +23,7 @@ public void serializesSingleValue() throws IOException { @Test public void serializesList() throws IOException { ConversionTester - .withSchemata(Schema.createArray(BigIntegerConversion.SCHEMA)) + .withSchemata(Schema.createArray(BigIntegerType.SCHEMA)) .withConverters(new BigIntegerConversion()) .reserializeAndAssert(ImmutableList.of(BigInteger.valueOf(1234567890L), new BigInteger("123456789012345678901234567890123456789012345678901234567890"), @@ -33,7 +34,7 @@ public void serializesList() throws IOException { @Test public void serializesMap() throws IOException { ConversionTester - .withSchemata(Schema.createMap(BigIntegerConversion.SCHEMA)) + .withSchemata(Schema.createMap(BigIntegerType.SCHEMA)) .withConverters(new BigIntegerConversion()) .reserializeAndAssert(ImmutableMap.of( "key 1", new BigInteger("123456789012345678901234567890123456789012345678901234567890"), diff --git a/src/test/java/io/datanerds/avropatch/value/conversion/ConversionTester.java b/src/test/java/io/datanerds/avropatch/value/conversion/ConversionTester.java index debefc3..8628249 100644 --- a/src/test/java/io/datanerds/avropatch/value/conversion/ConversionTester.java +++ b/src/test/java/io/datanerds/avropatch/value/conversion/ConversionTester.java @@ -3,16 +3,12 @@ import org.apache.avro.Conversion; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; -import org.apache.avro.generic.GenericDatumReader; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.io.DecoderFactory; -import org.apache.avro.io.Encoder; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.reflect.ReflectDatumReader; -import org.apache.avro.reflect.ReflectDatumWriter; +import org.apache.avro.io.*; +import org.apache.avro.reflect.ReflectData; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.Objects; @@ -22,13 +18,15 @@ public final class ConversionTester { - private final GenericDatumWriter writer; - private final GenericDatumReader reader; + private final DatumWriter writer; + private final DatumReader reader; + private final ReflectData data; private ConversionTester(Schema schema) { Objects.nonNull(schema); - reader = new ReflectDatumReader<>(schema); - writer = new ReflectDatumWriter<>(schema); + data = new ReflectData(); + reader = data.createDatumReader(schema); + writer = data.createDatumWriter(schema); } public static final ConversionTester withSchemata(Schema... types) { @@ -45,10 +43,7 @@ public static final ConversionTester withSchemata(Schema... types) { } public ConversionTester withConverters(Conversion... converters) { - for (Conversion conversion : converters) { - reader.getData().addLogicalTypeConversion(conversion); - writer.getData().addLogicalTypeConversion(conversion); - } + Arrays.asList(converters).forEach(conversion -> data.addLogicalTypeConversion(conversion)); return this; } diff --git a/src/test/java/io/datanerds/avropatch/value/conversion/DateConversionTest.java b/src/test/java/io/datanerds/avropatch/value/conversion/DateConversionTest.java index 4a1848f..60df015 100644 --- a/src/test/java/io/datanerds/avropatch/value/conversion/DateConversionTest.java +++ b/src/test/java/io/datanerds/avropatch/value/conversion/DateConversionTest.java @@ -2,6 +2,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.datanerds.avropatch.value.conversion.CustomTypes.DateType; import org.apache.avro.Schema; import org.junit.Test; @@ -13,7 +14,7 @@ public class DateConversionTest { @Test public void serializesSingleValue() throws IOException { ConversionTester - .withSchemata(DateConversion.SCHEMA) + .withSchemata(DateType.SCHEMA) .withConverters(new DateConversion()) .reserializeAndAssert(new Date()) .reserializeAndAssert(new Date()); @@ -22,7 +23,7 @@ public void serializesSingleValue() throws IOException { @Test public void serializesList() throws IOException { ConversionTester - .withSchemata(Schema.createArray(DateConversion.SCHEMA)) + .withSchemata(Schema.createArray(DateType.SCHEMA)) .withConverters(new DateConversion()) .reserializeAndAssert(ImmutableList.of(new Date(), new Date(), new Date(), new Date())); @@ -31,7 +32,7 @@ public void serializesList() throws IOException { @Test public void serializesMap() throws IOException { ConversionTester - .withSchemata(Schema.createMap(DateConversion.SCHEMA)) + .withSchemata(Schema.createMap(DateType.SCHEMA)) .withConverters(new DateConversion()) .reserializeAndAssert(ImmutableMap.of( "key 1", new Date(), diff --git a/src/test/java/io/datanerds/avropatch/value/conversion/UUIDConversionTest.java b/src/test/java/io/datanerds/avropatch/value/conversion/UUIDConversionTest.java index be75b17..5a4743d 100644 --- a/src/test/java/io/datanerds/avropatch/value/conversion/UUIDConversionTest.java +++ b/src/test/java/io/datanerds/avropatch/value/conversion/UUIDConversionTest.java @@ -2,6 +2,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.datanerds.avropatch.value.conversion.CustomTypes.UuidType; import org.apache.avro.Schema; import org.junit.Test; @@ -13,7 +14,7 @@ public class UUIDConversionTest { @Test public void serializesSingleValue() throws IOException { ConversionTester - .withSchemata(UUIDConversion.SCHEMA) + .withSchemata(UuidType.SCHEMA) .withConverters(new UUIDConversion()) .reserializeAndAssert(UUID.randomUUID()) .reserializeAndAssert(UUID.randomUUID()); @@ -22,7 +23,7 @@ public void serializesSingleValue() throws IOException { @Test public void serializesList() throws IOException { ConversionTester - .withSchemata(Schema.createArray(UUIDConversion.SCHEMA)) + .withSchemata(Schema.createArray(UuidType.SCHEMA)) .withConverters(new UUIDConversion()) .reserializeAndAssert(ImmutableList.of(UUID.randomUUID(), UUID.randomUUID(), UUID.randomUUID())); } @@ -30,7 +31,7 @@ public void serializesList() throws IOException { @Test public void serializesMap() throws IOException { ConversionTester - .withSchemata(Schema.createMap(UUIDConversion.SCHEMA)) + .withSchemata(Schema.createMap(UuidType.SCHEMA)) .withConverters(new UUIDConversion()) .reserializeAndAssert(ImmutableMap.of( "key 1", UUID.randomUUID(), From 9138f96f9f79041b2c14fd72390f77298d31b611 Mon Sep 17 00:00:00 2001 From: Frank Wisniewski Date: Thu, 10 Nov 2016 10:37:50 -0800 Subject: [PATCH 03/19] Added types for custom values and operations, fixed conversion. --- .../java/io/datanerds/avropatch/Patch.java | 21 ++- .../datanerds/avropatch/operation/Path.java | 2 - .../avropatch/schema/CustomTypes.java | 88 ++++++++++++ .../avropatch/schema/OperationTypes.java | 95 +++++++++++++ .../avropatch/schema/PrimitiveTypes.java | 20 +++ .../serialization/CustomTypeSerializer.java | 61 ++++++++ .../value/conversion/AvroConversion.java | 22 --- .../avropatch/value/conversion/AvroData.java | 24 ++++ .../conversion/BigDecimalConversion.java | 11 +- .../conversion/BigIntegerConversion.java | 5 +- .../value/conversion/CustomTypes.java | 51 ------- .../value/conversion/DateConversion.java | 7 +- .../value/conversion/UUIDConversion.java | 5 +- .../avropatch/schema/OperationTypesTest.java | 133 ++++++++++++++++++ .../avropatch/schema/SerializationTester.java | 57 ++++++++ .../CustomTypeSerializerTest.java | 26 ++++ .../conversion/BigDecimalConversionTest.java | 2 +- .../conversion/BigIntegerConversionTest.java | 2 +- .../value/conversion/DateConversionTest.java | 2 +- .../value/conversion/UUIDConversionTest.java | 2 +- 20 files changed, 534 insertions(+), 102 deletions(-) create mode 100644 src/main/java/io/datanerds/avropatch/schema/CustomTypes.java create mode 100644 src/main/java/io/datanerds/avropatch/schema/OperationTypes.java create mode 100644 src/main/java/io/datanerds/avropatch/schema/PrimitiveTypes.java create mode 100644 src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java delete mode 100644 src/main/java/io/datanerds/avropatch/value/conversion/AvroConversion.java create mode 100644 src/main/java/io/datanerds/avropatch/value/conversion/AvroData.java delete mode 100644 src/main/java/io/datanerds/avropatch/value/conversion/CustomTypes.java create mode 100644 src/test/java/io/datanerds/avropatch/schema/OperationTypesTest.java create mode 100644 src/test/java/io/datanerds/avropatch/schema/SerializationTester.java create mode 100644 src/test/java/io/datanerds/avropatch/serialization/CustomTypeSerializerTest.java diff --git a/src/main/java/io/datanerds/avropatch/Patch.java b/src/main/java/io/datanerds/avropatch/Patch.java index 9a1596c..ee333c2 100644 --- a/src/main/java/io/datanerds/avropatch/Patch.java +++ b/src/main/java/io/datanerds/avropatch/Patch.java @@ -1,11 +1,9 @@ package io.datanerds.avropatch; -import avro.shaded.com.google.common.collect.ImmutableList; import io.datanerds.avropatch.operation.Operation; import io.datanerds.avropatch.value.conversion.*; import org.apache.avro.Schema; import org.apache.avro.io.*; -import org.apache.avro.reflect.ReflectData; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -13,10 +11,14 @@ import java.util.Collections; import java.util.List; +/** + * This class represents a JSON PATCH operation holding a sequence of operations to apply to a given object. + * + * @see https://tools.ietf.org/html/rfc6902 + */ public class Patch { private static final PatchSerializer SERIALIZER = new PatchSerializer(); - public static final Schema SCHEMA = SERIALIZER.schema; private final List operations; @@ -48,21 +50,14 @@ public static Patch of(byte[] bytes) throws IOException { } private static class PatchSerializer { - - static final List> AVRO_CONVERSIONS = - ImmutableList.of(new DateConversion(), new BigIntegerConversion(), new BigDecimalConversion(), - new UUIDConversion()); final DatumWriter writer; final DatumReader reader; - final Schema schema; private PatchSerializer() { - ReflectData data = new ReflectData(); - AVRO_CONVERSIONS.forEach(avroConversion -> avroConversion.register(data)); - schema = data.getSchema(Patch.class); + Schema schema = AvroData.get().getSchema(Patch.class); - writer = data.createDatumWriter(schema); - reader = data.createDatumReader(schema); + writer = AvroData.get().createDatumWriter(schema); + reader = AvroData.get().createDatumReader(schema); } } } diff --git a/src/main/java/io/datanerds/avropatch/operation/Path.java b/src/main/java/io/datanerds/avropatch/operation/Path.java index 793184b..da30029 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Path.java +++ b/src/main/java/io/datanerds/avropatch/operation/Path.java @@ -1,7 +1,6 @@ package io.datanerds.avropatch.operation; import io.datanerds.avropatch.exception.InvalidPathException; -import org.apache.avro.reflect.AvroSchema; import java.util.Arrays; import java.util.Collections; @@ -16,7 +15,6 @@ public class Path { private static final Pattern VALID_PATTERN = Pattern.compile("[0-9a-zA-Z][0-9a-zA-Z_-]*"); - @AvroSchema("{\"type\": \"array\", \"items\": \"string\"}") private final List parts; private Path() { diff --git a/src/main/java/io/datanerds/avropatch/schema/CustomTypes.java b/src/main/java/io/datanerds/avropatch/schema/CustomTypes.java new file mode 100644 index 0000000..e9e6458 --- /dev/null +++ b/src/main/java/io/datanerds/avropatch/schema/CustomTypes.java @@ -0,0 +1,88 @@ +package io.datanerds.avropatch.schema; + +import io.datanerds.avropatch.Patch; +import io.datanerds.avropatch.value.conversion.BigDecimalConversion; +import io.datanerds.avropatch.value.conversion.BigIntegerConversion; +import io.datanerds.avropatch.value.conversion.DateConversion; +import io.datanerds.avropatch.value.conversion.UUIDConversion; +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.apache.avro.Schema.Type; +import org.apache.avro.SchemaBuilder; + +import java.util.Arrays; + +import static io.datanerds.avropatch.schema.PrimitiveTypes.*; +import static org.apache.avro.Schema.createRecord; +import static org.apache.avro.Schema.createUnion; + +/** + * This interface holds the schema constants for all supported custom types. + * + * @see UUIDConversion + * @see DateConversion + * @see BigDecimalConversion + * @see BigIntegerConversion + * @see io.datanerds.avropatch.operation.Path + */ +public interface CustomTypes { + + interface PatchType { + String NAME = Patch.class.getSimpleName(); + String DOC = "This record represents a JSON PATCH operation holding a sequence of operations to apply to a" + + " given object."; + + static Schema create(Schema valueSchema) { + Schema.Field operations = new Schema.Field("operations", + Schema.createArray(OperationTypes.Operation.create(valueSchema)), + "Sequence of operations to apply to a given object.", NO_DEFAULT); + return createRecord(NAME, DOC, Patch.class.getPackage().getName(), false, Arrays.asList(operations)); + } + } + + interface BigDecimalType { + String NAME = "big-decimal"; + String DOC = "BigDecimal value represented via it's scale and unscaled value."; + LogicalType LOGICAL_TYPE = new LogicalType(NAME); + Schema RECORD = SchemaBuilder.record("decimal").doc(DOC).fields() + .name("unscaledValue").type(BigIntegerType.SCHEMA).noDefault() + .name("scale").type(Schema.create(Type.INT)).noDefault() + .endRecord(); + + Schema SCHEMA = LOGICAL_TYPE.addToSchema(RECORD); + } + + interface BigIntegerType { + String NAME = "big-integer"; + LogicalType LOGICAL_TYPE = new LogicalType(NAME); + Schema SCHEMA = LOGICAL_TYPE.addToSchema(Schema.create(Type.BYTES)); + + } + + interface DateType { + String NAME = "timestamp"; + String DOC = "Timestamp representing the number of milliseconds since January 1, 1970, 00:00:00 GMT"; + LogicalType LOGICAL_TYPE = new LogicalType(NAME); + int SIZE = Long.BYTES; + Schema SCHEMA = LOGICAL_TYPE.addToSchema(Schema.createFixed(NAME, DOC, null, SIZE)); + } + + interface UuidType { + String NAME = LogicalTypes.uuid().getName(); + String DOC = "UUID serialized via two long values: It's most significant and least significant 64 bits."; + int SIZE = 2 * Long.BYTES; + Schema SCHEMA = LogicalTypes.uuid().addToSchema(Schema.createFixed(NAME, DOC, null, SIZE)); + } + + interface Path { + String NAME = io.datanerds.avropatch.operation.Path.class.getSimpleName(); + String DOC = "JSON Path serialized as String array holding its parts."; + Schema SCHEMA = SchemaBuilder.record(NAME).doc(DOC).fields() + .name("parts").type(Schema.createArray(Schema.create(Type.STRING))).noDefault() + .endRecord(); + } + + Schema VALUE_TYPE_UNION = createUnion(BOOLEAN, DOUBLE, FLOAT, INTEGER, LONG, NULL, STRING, + BigDecimalType.SCHEMA, BigIntegerType.SCHEMA, DateType.SCHEMA, UuidType.SCHEMA); +} diff --git a/src/main/java/io/datanerds/avropatch/schema/OperationTypes.java b/src/main/java/io/datanerds/avropatch/schema/OperationTypes.java new file mode 100644 index 0000000..daa461e --- /dev/null +++ b/src/main/java/io/datanerds/avropatch/schema/OperationTypes.java @@ -0,0 +1,95 @@ +package io.datanerds.avropatch.schema; + +import io.datanerds.avropatch.schema.CustomTypes.Path; +import org.apache.avro.Schema; + +import java.util.Arrays; + +import static io.datanerds.avropatch.schema.PrimitiveTypes.NO_DEFAULT; +import static org.apache.avro.Schema.*; +import static org.apache.avro.Schema.createRecord; + +/** + * This class holds schemata for all patch {@link io.datanerds.avropatch.operation.Operation}s. All schemata are not + * bound to any specific namespace and therefore domain independent. + * {@link io.datanerds.avropatch.Patch}/{@link io.datanerds.avropatch.schema.CustomTypes.PatchType} binds it to this + * library's domain/namespace. + */ +public interface OperationTypes { + + interface Add { + String NAME = io.datanerds.avropatch.operation.Add.class.getSimpleName(); + String DOC = "This record represents the add operation of RFC 6902 for JSON Patch'."; + + static Schema create(Schema valueSchema) { + Field path = new Field("path", Path.SCHEMA, "Path pointing out where the JSON value should be added", + NO_DEFAULT); + Field value = new Field("value", valueSchema, "Actual value to add to patched object", NO_DEFAULT); + return createRecord(NAME, DOC, null, false, Arrays.asList(path, value)); + } + } + + interface Copy { + String NAME = io.datanerds.avropatch.operation.Copy.class.getSimpleName(); + String DOC = "This record represents the copy operation of RFC 6902 for JSON Patch'."; + + static Schema create() { + Field from = new Field("from", Path.SCHEMA, "Source location of the value to be copied", NO_DEFAULT); + Field path = new Field("path", Path.SCHEMA, "Target location of value to be copied", NO_DEFAULT); + return createRecord(NAME, DOC, null, false, Arrays.asList(from, path)); + } + } + + interface Move { + String NAME = io.datanerds.avropatch.operation.Move.class.getSimpleName(); + String DOC = "This record represents the move operation of RFC 6902 for JSON Patch'."; + + static Schema create() { + Field from = new Field("from", Path.SCHEMA, "Source location of the value to be moved", NO_DEFAULT); + Field path = new Field("path", Path.SCHEMA, "Target location for the value to be moved to", NO_DEFAULT); + return createRecord(NAME, DOC, null, false, Arrays.asList(from, path)); + } + } + + interface Remove { + String NAME = io.datanerds.avropatch.operation.Remove.class.getSimpleName(); + String DOC = "This record represents the remove operation of RFC 6902 for JSON Patch'."; + + static Schema create() { + Field path = new Field("path", Path.SCHEMA, "Target location of value to be removed", NO_DEFAULT); + return createRecord(NAME, DOC, null, false, Arrays.asList(path)); + } + } + + interface Replace { + String NAME = io.datanerds.avropatch.operation.Replace.class.getSimpleName(); + String DOC = "This record represents the replace operation of RFC 6902 for JSON Patch'."; + + static Schema create(Schema valueSchema) { + Field path = new Field("path", Path.SCHEMA, "Path pointing out where the JSON value should be replaced", + NO_DEFAULT); + Field value = new Field("value", valueSchema, "Actual value to replaced in patched object", NO_DEFAULT); + return createRecord(NAME, DOC, null, false, Arrays.asList(path, value)); + } + } + + interface Test { + String NAME = io.datanerds.avropatch.operation.Test.class.getSimpleName(); + String DOC = "This record represents the test operation of RFC 6902 for JSON Patch'."; + + static Schema create(Schema valueSchema) { + Field path = new Field("path", Path.SCHEMA, "Path pointing out which JSON value should be tested against", + NO_DEFAULT); + Field value = new Field("value", valueSchema, "Actual value to test< against", NO_DEFAULT); + return createRecord(NAME, DOC, null, false, Arrays.asList(path, value)); + } + } + + interface Operation { + static Schema create(Schema valueSchema) { + return createUnion(Add.create(valueSchema), Copy.create(), Move.create(), Remove.create(), + Replace.create(valueSchema), Test.create(valueSchema)); + } + } + +} diff --git a/src/main/java/io/datanerds/avropatch/schema/PrimitiveTypes.java b/src/main/java/io/datanerds/avropatch/schema/PrimitiveTypes.java new file mode 100644 index 0000000..a52e964 --- /dev/null +++ b/src/main/java/io/datanerds/avropatch/schema/PrimitiveTypes.java @@ -0,0 +1,20 @@ +package io.datanerds.avropatch.schema; + +import org.apache.avro.Schema; + +import static org.apache.avro.Schema.create; +import static org.apache.avro.Schema.createUnion; + +public interface PrimitiveTypes { + Schema BOOLEAN = create(Schema.Type.BOOLEAN); + Schema DOUBLE = create(Schema.Type.DOUBLE); + Schema FLOAT = create(Schema.Type.FLOAT); + Schema INTEGER = create(Schema.Type.INT); + Schema LONG = create(Schema.Type.LONG); + Schema NULL = create(Schema.Type.NULL); + Schema STRING = create(Schema.Type.STRING); + + Schema UNION = createUnion(BOOLEAN, DOUBLE, FLOAT, INTEGER, LONG, NULL, STRING); + + Object NO_DEFAULT = null; +} diff --git a/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java b/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java new file mode 100644 index 0000000..13f29d0 --- /dev/null +++ b/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java @@ -0,0 +1,61 @@ +package io.datanerds.avropatch.serialization; + +import io.datanerds.avropatch.Patch; +import io.datanerds.avropatch.schema.CustomTypes; +import io.datanerds.avropatch.value.conversion.AvroData; +import org.apache.avro.Schema; +import org.apache.avro.io.*; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public final class CustomTypeSerializer { + + private final DatumWriter writer; + private final DatumReader reader; + + private CustomTypeSerializer(Schema schema) { + Objects.nonNull(schema); + writer = AvroData.get().createDatumWriter(schema); + reader = AvroData.get().createDatumReader(schema); + } + + public byte[] toBytes(Patch value) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + Encoder binaryEncoder = EncoderFactory.get().directBinaryEncoder(outputStream, null); + writer.write(value, binaryEncoder); + return outputStream.toByteArray(); + } + + public Patch toObject(byte[] bytes) throws IOException { + return reader.read(null, DecoderFactory.get().binaryDecoder(bytes, null)); + } + + public static class Builder { + private final List valueTypes = new ArrayList<>(); + + public Builder withBigDecimal() { + valueTypes.add(CustomTypes.BigDecimalType.SCHEMA); + return this; + } + + public Builder with(Type type) { + valueTypes.add(AvroData.get().getSchema(type)); + return this; + } + + public Builder with(Schema schema) { + valueTypes.add(schema); + return this; + } + + public CustomTypeSerializer build() { + Schema schema = CustomTypes.PatchType.create(Schema.createUnion(valueTypes)); + return new CustomTypeSerializer(schema); + } + } +} diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/AvroConversion.java b/src/main/java/io/datanerds/avropatch/value/conversion/AvroConversion.java deleted file mode 100644 index fbdf28e..0000000 --- a/src/main/java/io/datanerds/avropatch/value/conversion/AvroConversion.java +++ /dev/null @@ -1,22 +0,0 @@ -package io.datanerds.avropatch.value.conversion; - -import org.apache.avro.Conversion; -import org.apache.avro.LogicalType; -import org.apache.avro.LogicalTypes; -import org.apache.avro.generic.GenericData; - -/** - * This abstract class serves as base class for all custom type conversions supported by this library. - * @param - */ -public abstract class AvroConversion extends Conversion { - - public void register(GenericData data) { - data.addLogicalTypeConversion(this); - } - - protected static void registerLogicalType(String name, LogicalType type) { - LogicalTypes.register(name, schema -> type); - } - -} diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/AvroData.java b/src/main/java/io/datanerds/avropatch/value/conversion/AvroData.java new file mode 100644 index 0000000..9531f05 --- /dev/null +++ b/src/main/java/io/datanerds/avropatch/value/conversion/AvroData.java @@ -0,0 +1,24 @@ +package io.datanerds.avropatch.value.conversion; + +import avro.shaded.com.google.common.collect.ImmutableList; +import org.apache.avro.Conversion; +import org.apache.avro.reflect.ReflectData; + +import java.util.List; + +public final class AvroData extends ReflectData { + + public static List> CONVERTERS = + ImmutableList.of(new DateConversion(), new BigIntegerConversion(), new BigDecimalConversion(), + new UUIDConversion()); + + private static final AvroData INSTANCE = new AvroData(); + + private AvroData() { + CONVERTERS.forEach(this::addLogicalTypeConversion); + } + + public static AvroData get() { + return INSTANCE; + } +} diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/BigDecimalConversion.java b/src/main/java/io/datanerds/avropatch/value/conversion/BigDecimalConversion.java index 9fa6735..9e07a17 100644 --- a/src/main/java/io/datanerds/avropatch/value/conversion/BigDecimalConversion.java +++ b/src/main/java/io/datanerds/avropatch/value/conversion/BigDecimalConversion.java @@ -1,7 +1,9 @@ package io.datanerds.avropatch.value.conversion; -import io.datanerds.avropatch.value.conversion.CustomTypes.BigDecimalType; +import io.datanerds.avropatch.schema.CustomTypes.BigDecimalType; +import org.apache.avro.Conversion; import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.IndexedRecord; @@ -13,13 +15,16 @@ * This class converts an {@link BigDecimal} value into am Avro {@link IndexedRecord} and back. It depends on * {@link BigIntegerConversion} since it serializes the {@link BigDecimal}s unscaled value into a {@link BigInteger} and * its scale into an {@link Integer}. + * Also, its logical type is statically registered in {@link LogicalTypes}, because it introduces a new custom type name + * 'big-decimal'. * * @see BigIntegerConversion + * @see LogicalTypes */ -public class BigDecimalConversion extends AvroConversion { +public class BigDecimalConversion extends Conversion { static { - registerLogicalType(BigDecimalType.NAME, BigDecimalType.LOGICAL_TYPE); + LogicalTypes.register(BigDecimalType.NAME, schema -> BigDecimalType.LOGICAL_TYPE); } @Override diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/BigIntegerConversion.java b/src/main/java/io/datanerds/avropatch/value/conversion/BigIntegerConversion.java index 0de50d8..dad8723 100644 --- a/src/main/java/io/datanerds/avropatch/value/conversion/BigIntegerConversion.java +++ b/src/main/java/io/datanerds/avropatch/value/conversion/BigIntegerConversion.java @@ -1,6 +1,7 @@ package io.datanerds.avropatch.value.conversion; -import io.datanerds.avropatch.value.conversion.CustomTypes.BigIntegerType; +import io.datanerds.avropatch.schema.CustomTypes.BigIntegerType; +import org.apache.avro.Conversion; import org.apache.avro.LogicalType; import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; @@ -8,7 +9,7 @@ import java.math.BigInteger; import java.nio.ByteBuffer; -public class BigIntegerConversion extends AvroConversion { +public class BigIntegerConversion extends Conversion { static { LogicalTypes.register(BigIntegerType.NAME, schema -> BigIntegerType.LOGICAL_TYPE); diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/CustomTypes.java b/src/main/java/io/datanerds/avropatch/value/conversion/CustomTypes.java deleted file mode 100644 index aac382b..0000000 --- a/src/main/java/io/datanerds/avropatch/value/conversion/CustomTypes.java +++ /dev/null @@ -1,51 +0,0 @@ -package io.datanerds.avropatch.value.conversion; - -import org.apache.avro.LogicalType; -import org.apache.avro.LogicalTypes; -import org.apache.avro.Schema; -import org.apache.avro.SchemaBuilder; - -/** - * This interface holds the schema constants for all supported custom types. - * - * @see UUIDConversion - * @see DateConversion - * @see BigDecimalConversion - * @see BigIntegerConversion - */ -public interface CustomTypes { - - interface BigDecimalType { - String NAME = "big-decimal"; - String DOC = "BigDecimal value represented via it's scale and unscaled value."; - LogicalType LOGICAL_TYPE = new LogicalType(NAME); - Schema RECORD = SchemaBuilder.record("decimal").doc(BigDecimalType.DOC).fields() - .name("unscaledValue").type(BigIntegerType.SCHEMA).noDefault() - .name("scale").type(Schema.create(Schema.Type.INT)).noDefault() - .endRecord(); - - Schema SCHEMA = BigDecimalType.LOGICAL_TYPE.addToSchema(RECORD); - } - - interface BigIntegerType { - String NAME = "big-integer"; - LogicalType LOGICAL_TYPE = new LogicalType(NAME); - Schema SCHEMA = LOGICAL_TYPE.addToSchema(Schema.create(Schema.Type.BYTES)); - - } - - interface DateType { - String NAME = "timestamp"; - String DOC = "Timestamp representing the number of milliseconds since January 1, 1970, 00:00:00 GMT"; - LogicalType LOGICAL_TYPE = new LogicalType(NAME); - int SIZE = Long.BYTES; - Schema SCHEMA = LOGICAL_TYPE.addToSchema(Schema.createFixed(NAME, DOC, null, SIZE)); - } - - interface UuidType { - String NAME = LogicalTypes.uuid().getName(); - String DOC = "UUID serialized via two long values: It's most significant and least significant 64 bits."; - int SIZE = 2 * Long.BYTES; - Schema SCHEMA = LogicalTypes.uuid().addToSchema(Schema.createFixed(NAME, DOC, null, SIZE)); - } -} diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/DateConversion.java b/src/main/java/io/datanerds/avropatch/value/conversion/DateConversion.java index 841aef3..480328e 100644 --- a/src/main/java/io/datanerds/avropatch/value/conversion/DateConversion.java +++ b/src/main/java/io/datanerds/avropatch/value/conversion/DateConversion.java @@ -1,8 +1,9 @@ package io.datanerds.avropatch.value.conversion; -import io.datanerds.avropatch.value.conversion.CustomTypes.DateType; +import io.datanerds.avropatch.schema.CustomTypes.DateType; import org.apache.avro.Conversion; import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import org.apache.avro.generic.GenericFixed; @@ -18,10 +19,10 @@ * @see org.apache.avro.LogicalTypes.TimestampMillis * @see org.apache.avro.Schema.Type */ -public class DateConversion extends AvroConversion { +public class DateConversion extends Conversion { static { - registerLogicalType(DateType.NAME, DateType.LOGICAL_TYPE); + LogicalTypes.register(DateType.NAME, schema -> DateType.LOGICAL_TYPE); } @Override diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/UUIDConversion.java b/src/main/java/io/datanerds/avropatch/value/conversion/UUIDConversion.java index 3405410..9559f79 100644 --- a/src/main/java/io/datanerds/avropatch/value/conversion/UUIDConversion.java +++ b/src/main/java/io/datanerds/avropatch/value/conversion/UUIDConversion.java @@ -1,6 +1,7 @@ package io.datanerds.avropatch.value.conversion; -import io.datanerds.avropatch.value.conversion.CustomTypes.UuidType; +import io.datanerds.avropatch.schema.CustomTypes.UuidType; +import org.apache.avro.Conversion; import org.apache.avro.LogicalType; import org.apache.avro.Schema; import org.apache.avro.generic.GenericFixed; @@ -8,7 +9,7 @@ import java.nio.ByteBuffer; import java.util.UUID; -public class UUIDConversion extends AvroConversion { +public class UUIDConversion extends Conversion { @Override public Schema getRecommendedSchema() { diff --git a/src/test/java/io/datanerds/avropatch/schema/OperationTypesTest.java b/src/test/java/io/datanerds/avropatch/schema/OperationTypesTest.java new file mode 100644 index 0000000..0568f58 --- /dev/null +++ b/src/test/java/io/datanerds/avropatch/schema/OperationTypesTest.java @@ -0,0 +1,133 @@ +package io.datanerds.avropatch.schema; + +import io.datanerds.avropatch.operation.*; +import io.datanerds.avropatch.value.conversion.*; +import org.apache.avro.Schema; +import org.codehaus.jackson.map.ObjectMapper; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.*; +import java.util.function.Function; + +import static io.datanerds.avropatch.operation.matcher.OperationMatchers.equalTo; +import static io.datanerds.avropatch.schema.CustomTypes.VALUE_TYPE_UNION; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +public class OperationTypesTest { + + Schema.Parser parser; + + @Before + public void setup() throws IOException { + AvroData.get(); + Map types = new HashMap<>(); + types.put(Add.class.getSimpleName(), annotateNamespace(OperationTypes.Add.create(VALUE_TYPE_UNION))); + types.put(Copy.class.getSimpleName(), annotateNamespace(OperationTypes.Copy.create())); + types.put(Move.class.getSimpleName(), annotateNamespace(OperationTypes.Move.create())); + types.put(Remove.class.getSimpleName(), annotateNamespace(OperationTypes.Remove.create())); + types.put(Replace.class.getSimpleName(), annotateNamespace(OperationTypes.Replace.create(VALUE_TYPE_UNION))); + types.put(io.datanerds.avropatch.operation.Test.class.getSimpleName(), + annotateNamespace(OperationTypes.Test.create(VALUE_TYPE_UNION))); + + parser = new Schema.Parser(); + parser.addTypes(types); + } + + @Test + public void add() throws IOException { + System.out.println(OperationTypes.Add.create(VALUE_TYPE_UNION).toString()); + Schema schema = parse(Add.class); + SerializationTester tester = new SerializationTester(schema); + + List adds = createSomeOperations(value -> new Add<>(Path.of("hello", "world"), value)); + for (Operation add : adds) { + assertThat(add, is(equalTo(tester.reserialize(add)))); + } + } + + @Test + public void copy() throws IOException { + Schema schema = parse(Copy.class); + SerializationTester tester = new SerializationTester(schema); + + Copy copy = new Copy(Path.of("from", "here"), Path.of("to", "there")); + assertThat(copy, is(equalTo(tester.reserialize(copy)))); + } + + @Test + public void move() throws IOException { + Schema schema = parse(Move.class); + SerializationTester tester = new SerializationTester(schema); + + Move copy = new Move(Path.of("from", "here"), Path.of("to", "there")); + assertThat(copy, is(equalTo(tester.reserialize(copy)))); + } + + @Test + public void remove() throws IOException { + Schema schema = parse(Remove.class); + SerializationTester tester = new SerializationTester(schema); + + Remove remove = new Remove(Path.of("from", "here")); + assertThat(remove, is(equalTo(tester.reserialize(remove)))); + } + + @Test + public void replace() throws IOException { + Schema schema = parse(Replace.class); + SerializationTester tester = new SerializationTester(schema); + + List replaces = createSomeOperations(value -> new Replace<>(Path.of("hello", "world"), value)); + for (Operation replace : replaces) { + assertThat(replace, is(equalTo(tester.reserialize(replace)))); + } + } + + @Test + public void test() throws IOException { + Schema schema = parse(io.datanerds.avropatch.operation.Test.class); + SerializationTester tester = new SerializationTester(schema); + + List tests = createSomeOperations( + value -> new io.datanerds.avropatch.operation.Test<>(Path.of("hello", "world"), value)); + for (Operation test : tests) { + assertThat(test, is(equalTo(tester.reserialize(test)))); + } + } + + private Schema parse(Class clazz) { + return parser.parse(String.format("\"%s\"", clazz.getName())); + } + + private List createSomeOperations(Function operationFunction) { + List operations = new ArrayList<>(); + operations.add(createOperation(operationFunction, "hello world")); + operations.add(createOperation(operationFunction, 42)); + operations.add(createOperation(operationFunction, 42L)); + operations.add(createOperation(operationFunction, 123.456d)); + operations.add(createOperation(operationFunction, 123.456f)); + operations.add(createOperation(operationFunction, new BigInteger("83647896845639495762501378945698056348956"))); + operations.add(createOperation(operationFunction, new BigDecimal("956740578902345.56734895627895"))); + operations.add(createOperation(operationFunction, UUID.randomUUID())); + operations.add(createOperation(operationFunction, new Date())); + + return operations; + } + + private Operation createOperation(Function operationFunction, T value) { + return operationFunction.apply(value); + } + + private static Schema annotateNamespace(Schema schema) throws IOException { + ObjectMapper mappper = new ObjectMapper(); + Map jsonSchema = mappper.readValue(schema.toString(), Map.class); + jsonSchema.put("namespace", Operation.class.getPackage().getName()); + Schema.Parser parser = new Schema.Parser(); + return parser.parse(mappper.writeValueAsString(jsonSchema)); + } +} \ No newline at end of file diff --git a/src/test/java/io/datanerds/avropatch/schema/SerializationTester.java b/src/test/java/io/datanerds/avropatch/schema/SerializationTester.java new file mode 100644 index 0000000..9bf9b14 --- /dev/null +++ b/src/test/java/io/datanerds/avropatch/schema/SerializationTester.java @@ -0,0 +1,57 @@ +package io.datanerds.avropatch.schema; + +import io.datanerds.avropatch.value.conversion.AvroData; +import org.apache.avro.Schema; +import org.apache.avro.io.*; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Collection; +import java.util.Objects; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +public class SerializationTester { + + private final DatumWriter writer; + private final DatumReader reader; + + public SerializationTester(Schema schema) { + Objects.nonNull(schema); + writer = AvroData.get().createDatumWriter(schema); + reader = AvroData.get().createDatumReader(schema); + } + + public T reserialize(T value) throws IOException { + byte[] bytes = toBytes(value); + return toObject(bytes); + } + + public void reserializeAndAssert(T value) throws IOException { + T object = reserialize(value); + assertEquality(object, value); + } + + private void assertEquality(T value, T object) { + if (value instanceof Collection) { + // Avro serializes collections as arrays + Collection collection = (Collection) value; + assertThat(collection.toArray(), is(equalTo(object))); + } else { + assertThat(object, is(equalTo(value))); + } + } + + private byte[] toBytes(T value) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + Encoder binaryEncoder = EncoderFactory.get().directBinaryEncoder(outputStream, null); + writer.write(value, binaryEncoder); + return outputStream.toByteArray(); + } + + private T toObject(byte[] bytes) throws IOException { + return reader.read(null, DecoderFactory.get().binaryDecoder(bytes, null)); + } +} diff --git a/src/test/java/io/datanerds/avropatch/serialization/CustomTypeSerializerTest.java b/src/test/java/io/datanerds/avropatch/serialization/CustomTypeSerializerTest.java new file mode 100644 index 0000000..4b2d195 --- /dev/null +++ b/src/test/java/io/datanerds/avropatch/serialization/CustomTypeSerializerTest.java @@ -0,0 +1,26 @@ +package io.datanerds.avropatch.serialization; + +import avro.shaded.com.google.common.collect.ImmutableList; +import io.datanerds.avropatch.Patch; +import io.datanerds.avropatch.operation.Add; +import io.datanerds.avropatch.operation.Path; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.math.BigDecimal; + +import static org.junit.Assert.*; + +public class CustomTypeSerializerTest { + + @Ignore("work in progress") + @Test + public void test() throws IOException { + CustomTypeSerializer serializer = new CustomTypeSerializer.Builder().withBigDecimal().build(); + Patch patch = new Patch(ImmutableList.of(new Add(Path.of("hello"), new BigDecimal("23946712384.4")))); + byte[] bytes = serializer.toBytes(patch); + Patch newPatch = serializer.toObject(bytes); + } + +} \ No newline at end of file diff --git a/src/test/java/io/datanerds/avropatch/value/conversion/BigDecimalConversionTest.java b/src/test/java/io/datanerds/avropatch/value/conversion/BigDecimalConversionTest.java index 7809a2e..1c3a499 100644 --- a/src/test/java/io/datanerds/avropatch/value/conversion/BigDecimalConversionTest.java +++ b/src/test/java/io/datanerds/avropatch/value/conversion/BigDecimalConversionTest.java @@ -2,7 +2,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.datanerds.avropatch.value.conversion.CustomTypes.BigDecimalType; +import io.datanerds.avropatch.schema.CustomTypes.BigDecimalType; import org.apache.avro.Schema; import org.junit.Test; diff --git a/src/test/java/io/datanerds/avropatch/value/conversion/BigIntegerConversionTest.java b/src/test/java/io/datanerds/avropatch/value/conversion/BigIntegerConversionTest.java index 18d611a..b68046a 100644 --- a/src/test/java/io/datanerds/avropatch/value/conversion/BigIntegerConversionTest.java +++ b/src/test/java/io/datanerds/avropatch/value/conversion/BigIntegerConversionTest.java @@ -2,7 +2,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.datanerds.avropatch.value.conversion.CustomTypes.BigIntegerType; +import io.datanerds.avropatch.schema.CustomTypes.BigIntegerType; import org.apache.avro.Schema; import org.junit.Test; diff --git a/src/test/java/io/datanerds/avropatch/value/conversion/DateConversionTest.java b/src/test/java/io/datanerds/avropatch/value/conversion/DateConversionTest.java index 60df015..08dfc65 100644 --- a/src/test/java/io/datanerds/avropatch/value/conversion/DateConversionTest.java +++ b/src/test/java/io/datanerds/avropatch/value/conversion/DateConversionTest.java @@ -2,7 +2,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.datanerds.avropatch.value.conversion.CustomTypes.DateType; +import io.datanerds.avropatch.schema.CustomTypes.DateType; import org.apache.avro.Schema; import org.junit.Test; diff --git a/src/test/java/io/datanerds/avropatch/value/conversion/UUIDConversionTest.java b/src/test/java/io/datanerds/avropatch/value/conversion/UUIDConversionTest.java index 5a4743d..fd20ac3 100644 --- a/src/test/java/io/datanerds/avropatch/value/conversion/UUIDConversionTest.java +++ b/src/test/java/io/datanerds/avropatch/value/conversion/UUIDConversionTest.java @@ -2,7 +2,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.datanerds.avropatch.value.conversion.CustomTypes.UuidType; +import io.datanerds.avropatch.schema.CustomTypes.UuidType; import org.apache.avro.Schema; import org.junit.Test; From daf2c25040e3a7f073670543ad707e66e46eb3f1 Mon Sep 17 00:00:00 2001 From: Frank Wisniewski Date: Fri, 11 Nov 2016 15:16:44 -0800 Subject: [PATCH 04/19] Added CustomTypeSerializer and corresponding Builder. --- .../avropatch/schema/CustomTypes.java | 3 +- .../avropatch/schema/OperationTypes.java | 20 ++-- .../avropatch/schema/PrimitiveTypes.java | 4 - .../serialization/CustomTypeSerializer.java | 97 +++++++++++++++- .../io/datanerds/avropatch/PatchTest.java | 49 +++----- .../operation/matcher/OperationMatchers.java | 19 ++- .../operation/matcher/PatchMatcher.java | 19 +++ .../avropatch/schema/OperationTypesTest.java | 109 +++++++----------- .../avropatch/serialization/Bimmel.java | 75 ++++++++++++ .../CustomTypeSerializerTest.java | 59 ++++++++-- 10 files changed, 319 insertions(+), 135 deletions(-) create mode 100644 src/test/java/io/datanerds/avropatch/operation/matcher/PatchMatcher.java create mode 100644 src/test/java/io/datanerds/avropatch/serialization/Bimmel.java diff --git a/src/main/java/io/datanerds/avropatch/schema/CustomTypes.java b/src/main/java/io/datanerds/avropatch/schema/CustomTypes.java index e9e6458..5d44940 100644 --- a/src/main/java/io/datanerds/avropatch/schema/CustomTypes.java +++ b/src/main/java/io/datanerds/avropatch/schema/CustomTypes.java @@ -13,6 +13,7 @@ import java.util.Arrays; +import static io.datanerds.avropatch.schema.OperationTypes.NAMESPACE; import static io.datanerds.avropatch.schema.PrimitiveTypes.*; import static org.apache.avro.Schema.createRecord; import static org.apache.avro.Schema.createUnion; @@ -78,7 +79,7 @@ interface UuidType { interface Path { String NAME = io.datanerds.avropatch.operation.Path.class.getSimpleName(); String DOC = "JSON Path serialized as String array holding its parts."; - Schema SCHEMA = SchemaBuilder.record(NAME).doc(DOC).fields() + Schema SCHEMA = SchemaBuilder.record(NAME).doc(DOC).namespace(NAMESPACE).fields() .name("parts").type(Schema.createArray(Schema.create(Type.STRING))).noDefault() .endRecord(); } diff --git a/src/main/java/io/datanerds/avropatch/schema/OperationTypes.java b/src/main/java/io/datanerds/avropatch/schema/OperationTypes.java index daa461e..0f205ff 100644 --- a/src/main/java/io/datanerds/avropatch/schema/OperationTypes.java +++ b/src/main/java/io/datanerds/avropatch/schema/OperationTypes.java @@ -10,13 +10,13 @@ import static org.apache.avro.Schema.createRecord; /** - * This class holds schemata for all patch {@link io.datanerds.avropatch.operation.Operation}s. All schemata are not - * bound to any specific namespace and therefore domain independent. - * {@link io.datanerds.avropatch.Patch}/{@link io.datanerds.avropatch.schema.CustomTypes.PatchType} binds it to this - * library's domain/namespace. + * This class holds schemata for all patch {@link io.datanerds.avropatch.operation.Operation}s. All schemata are + * bound to this library's domain/namespace. */ public interface OperationTypes { + String NAMESPACE = io.datanerds.avropatch.operation.Operation.class.getPackage().getName(); + interface Add { String NAME = io.datanerds.avropatch.operation.Add.class.getSimpleName(); String DOC = "This record represents the add operation of RFC 6902 for JSON Patch'."; @@ -25,7 +25,7 @@ static Schema create(Schema valueSchema) { Field path = new Field("path", Path.SCHEMA, "Path pointing out where the JSON value should be added", NO_DEFAULT); Field value = new Field("value", valueSchema, "Actual value to add to patched object", NO_DEFAULT); - return createRecord(NAME, DOC, null, false, Arrays.asList(path, value)); + return createRecord(NAME, DOC, NAMESPACE, false, Arrays.asList(path, value)); } } @@ -36,7 +36,7 @@ interface Copy { static Schema create() { Field from = new Field("from", Path.SCHEMA, "Source location of the value to be copied", NO_DEFAULT); Field path = new Field("path", Path.SCHEMA, "Target location of value to be copied", NO_DEFAULT); - return createRecord(NAME, DOC, null, false, Arrays.asList(from, path)); + return createRecord(NAME, DOC, NAMESPACE, false, Arrays.asList(from, path)); } } @@ -47,7 +47,7 @@ interface Move { static Schema create() { Field from = new Field("from", Path.SCHEMA, "Source location of the value to be moved", NO_DEFAULT); Field path = new Field("path", Path.SCHEMA, "Target location for the value to be moved to", NO_DEFAULT); - return createRecord(NAME, DOC, null, false, Arrays.asList(from, path)); + return createRecord(NAME, DOC, NAMESPACE, false, Arrays.asList(from, path)); } } @@ -57,7 +57,7 @@ interface Remove { static Schema create() { Field path = new Field("path", Path.SCHEMA, "Target location of value to be removed", NO_DEFAULT); - return createRecord(NAME, DOC, null, false, Arrays.asList(path)); + return createRecord(NAME, DOC, NAMESPACE, false, Arrays.asList(path)); } } @@ -69,7 +69,7 @@ static Schema create(Schema valueSchema) { Field path = new Field("path", Path.SCHEMA, "Path pointing out where the JSON value should be replaced", NO_DEFAULT); Field value = new Field("value", valueSchema, "Actual value to replaced in patched object", NO_DEFAULT); - return createRecord(NAME, DOC, null, false, Arrays.asList(path, value)); + return createRecord(NAME, DOC, NAMESPACE, false, Arrays.asList(path, value)); } } @@ -81,7 +81,7 @@ static Schema create(Schema valueSchema) { Field path = new Field("path", Path.SCHEMA, "Path pointing out which JSON value should be tested against", NO_DEFAULT); Field value = new Field("value", valueSchema, "Actual value to test< against", NO_DEFAULT); - return createRecord(NAME, DOC, null, false, Arrays.asList(path, value)); + return createRecord(NAME, DOC, NAMESPACE, false, Arrays.asList(path, value)); } } diff --git a/src/main/java/io/datanerds/avropatch/schema/PrimitiveTypes.java b/src/main/java/io/datanerds/avropatch/schema/PrimitiveTypes.java index a52e964..aafc6d2 100644 --- a/src/main/java/io/datanerds/avropatch/schema/PrimitiveTypes.java +++ b/src/main/java/io/datanerds/avropatch/schema/PrimitiveTypes.java @@ -3,7 +3,6 @@ import org.apache.avro.Schema; import static org.apache.avro.Schema.create; -import static org.apache.avro.Schema.createUnion; public interface PrimitiveTypes { Schema BOOLEAN = create(Schema.Type.BOOLEAN); @@ -13,8 +12,5 @@ public interface PrimitiveTypes { Schema LONG = create(Schema.Type.LONG); Schema NULL = create(Schema.Type.NULL); Schema STRING = create(Schema.Type.STRING); - - Schema UNION = createUnion(BOOLEAN, DOUBLE, FLOAT, INTEGER, LONG, NULL, STRING); - Object NO_DEFAULT = null; } diff --git a/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java b/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java index 13f29d0..d96d28f 100644 --- a/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java +++ b/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java @@ -2,19 +2,30 @@ import io.datanerds.avropatch.Patch; import io.datanerds.avropatch.schema.CustomTypes; +import io.datanerds.avropatch.schema.CustomTypes.BigDecimalType; +import io.datanerds.avropatch.schema.CustomTypes.BigIntegerType; +import io.datanerds.avropatch.schema.CustomTypes.DateType; +import io.datanerds.avropatch.schema.CustomTypes.UuidType; import io.datanerds.avropatch.value.conversion.AvroData; import org.apache.avro.Schema; import org.apache.avro.io.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.lang.reflect.Type; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Objects; +import static io.datanerds.avropatch.schema.PrimitiveTypes.*; + public final class CustomTypeSerializer { + private static Logger logger = LoggerFactory.getLogger(CustomTypeSerializer.class); + private final DatumWriter writer; private final DatumReader reader; @@ -36,26 +47,100 @@ public Patch toObject(byte[] bytes) throws IOException { } public static class Builder { - private final List valueTypes = new ArrayList<>(); + final List types = new ArrayList<>(); + + public Builder() { + AvroData.get(); + } + + public Builder nullable() { + types.add(NULL); + return this; + } - public Builder withBigDecimal() { - valueTypes.add(CustomTypes.BigDecimalType.SCHEMA); + public Builder withPrimitives() { + types.addAll(Arrays.asList(BOOLEAN, DOUBLE, FLOAT, INTEGER, LONG, STRING)); + return this; + } + + public Builder withCustomTypes() { + types.addAll(Arrays.asList(BigDecimalType.SCHEMA, BigIntegerType.SCHEMA, DateType.SCHEMA, UuidType.SCHEMA)); return this; } public Builder with(Type type) { - valueTypes.add(AvroData.get().getSchema(type)); + types.add(AvroData.get().getSchema(type)); return this; } public Builder with(Schema schema) { - valueTypes.add(schema); + types.add(schema); return this; } public CustomTypeSerializer build() { - Schema schema = CustomTypes.PatchType.create(Schema.createUnion(valueTypes)); + Schema schema = makeSchema(); + logger.debug("Created serializer for schema {}", schema); return new CustomTypeSerializer(schema); } + + private Schema makeSchema() { + if (types.size() == 1) { + return CustomTypes.PatchType.create(types.get(0)); + } else { + return CustomTypes.PatchType.create(Schema.createUnion(types)); + } + } + + private Builder endSubBuilder(Schema schema) { + types.add(schema); + return this; + } + + public Builder.ArrayBuilder withArray() { + return new Builder.ArrayBuilder(this); + } + + public class ArrayBuilder extends Builder { + private final Builder parentBuilder; + + ArrayBuilder(Builder parentBuilder) { + this.parentBuilder = parentBuilder; + } + + public Builder endArray() { + return parentBuilder.endSubBuilder(Schema.createArray(Schema.createUnion(types))); + } + + @Override + public Builder.ArrayBuilder nullable() { + super.nullable(); + return this; + } + + @Override + public Builder.ArrayBuilder withPrimitives() { + super.withPrimitives(); + return this; + } + + @Override + public Builder.ArrayBuilder withCustomTypes() { + super.withCustomTypes(); + return this; + } + + @Override + public Builder.ArrayBuilder with(Type type) { + super.with(type); + return this; + } + + @Override + public Builder.ArrayBuilder with(Schema schema) { + super.with(schema); + return this; + } + } } } diff --git a/src/test/java/io/datanerds/avropatch/PatchTest.java b/src/test/java/io/datanerds/avropatch/PatchTest.java index e4ca0b5..fffa9b8 100644 --- a/src/test/java/io/datanerds/avropatch/PatchTest.java +++ b/src/test/java/io/datanerds/avropatch/PatchTest.java @@ -2,7 +2,6 @@ import com.google.common.collect.ImmutableList; import io.datanerds.avropatch.operation.*; -import org.hamcrest.MatcherAssert; import org.junit.Test; import java.io.IOException; @@ -13,7 +12,9 @@ import java.util.UUID; import static io.datanerds.avropatch.operation.matcher.OperationMatchers.hasItem; -import static io.datanerds.avropatch.operation.matcher.OperationMatchers.hasItems; +import static io.datanerds.avropatch.operation.matcher.PatchMatcher.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasSize; public class PatchTest { @@ -25,8 +26,8 @@ public void serializesAdd() throws IOException { byte[] bytes = patch.toBytes(); List operations = Patch.of(bytes).getOperations(); - MatcherAssert.assertThat(operations, hasSize(1)); - MatcherAssert.assertThat(operations, hasItem(new Add<>(Path.of("person", "name"), "John Doe"))); + assertThat(operations, hasSize(1)); + assertThat(operations, hasItem(new Add<>(Path.of("person", "name"), "John Doe"))); } @Test @@ -36,8 +37,8 @@ public void serializesCopy() throws IOException { byte[] bytes = patch.toBytes(); List operations = Patch.of(bytes).getOperations(); - MatcherAssert.assertThat(operations, hasSize(1)); - MatcherAssert.assertThat(operations, hasItem(new Copy(Path.parse("/person/firstName"), Path.parse("/person/lastName")))); + assertThat(operations, hasSize(1)); + assertThat(operations, hasItem(new Copy(Path.parse("/person/firstName"), Path.parse("/person/lastName")))); } @Test @@ -47,8 +48,8 @@ public void serializesMove() throws IOException { byte[] bytes = patch.toBytes(); List operations = Patch.of(bytes).getOperations(); - MatcherAssert.assertThat(operations, hasSize(1)); - MatcherAssert.assertThat(operations, hasItem(new Move(Path.parse("/person/firstName"), Path.parse("/person/lastName")))); + assertThat(operations, hasSize(1)); + assertThat(operations, hasItem(new Move(Path.parse("/person/firstName"), Path.parse("/person/lastName")))); } @Test @@ -58,8 +59,8 @@ public void serializesRemove() throws IOException { byte[] bytes = patch.toBytes(); List operations = Patch.of(bytes).getOperations(); - MatcherAssert.assertThat(operations, hasSize(1)); - MatcherAssert.assertThat(operations, hasItem(new Remove(Path.parse("/person/name")))); + assertThat(operations, hasSize(1)); + assertThat(operations, hasItem(new Remove(Path.parse("/person/name")))); } @Test @@ -69,7 +70,7 @@ public void serializesReplace() throws IOException { byte[] bytes = patch.toBytes(); List operations = Patch.of(bytes).getOperations(); - MatcherAssert.assertThat(operations, hasItem(new Replace(Path.parse("/person/number"), 42))); + assertThat(operations, hasItem(new Replace(Path.parse("/person/number"), 42))); } @Test @@ -79,8 +80,8 @@ public void serializesTest() throws IOException { byte[] bytes = patch.toBytes(); List operations = Patch.of(bytes).getOperations(); - MatcherAssert.assertThat(operations, hasSize(1)); - MatcherAssert.assertThat(operations, hasItem(new io.datanerds.avropatch.operation.Test(Path.parse("/person/number"), 42L))); + assertThat(operations, hasSize(1)); + assertThat(operations, hasItem(new io.datanerds.avropatch.operation.Test(Path.parse("/person/number"), 42L))); } @Test @@ -93,17 +94,9 @@ public void serializesBunchOfOperations() throws IOException { new io.datanerds.avropatch.operation.Test(Path.parse("/person/number"), 42L))); byte[] bytes = patch.toBytes(); - List operations = Patch.of(bytes).getOperations(); - MatcherAssert.assertThat(operations, hasSize(6)); - MatcherAssert.assertThat(operations, hasItems(new Add<>(Path.of("person", "name"), "John Doe"), - new Copy(Path.parse("/person/firstName"), Path.parse("/person/lastName")), - new Move(Path.parse("/person/firstName"), Path.parse("/person/lastName")), - new Remove(Path.parse("/person/name")), - new Replace(Path.parse("/person/number"), 42), - new io.datanerds.avropatch.operation.Test(Path.parse("/person/number"), 42L))); + assertThat(patch, is(equalTo(Patch.of(bytes)))); } - //public static final Schema[] BASIC_TYPES = {BIG_DECIMAL, BIG_INTEGER, BOOLEAN, DATE, DOUBLE, FLOAT, INTEGER, LONG, NULL, STRING, UUID}; @Test public void serializesDefaultValueTypes() throws IOException { Date date = new Date(); @@ -119,17 +112,7 @@ public void serializesDefaultValueTypes() throws IOException { new Add<>(Path.of("some", "value"), 4234.2345))); byte[] bytes = patch.toBytes(); - List operations = Patch.of(bytes).getOperations(); - MatcherAssert.assertThat(operations, hasSize(9)); - MatcherAssert.assertThat(operations, hasItems(new Add<>(Path.of("some", "value"), "John Doe"), - new Add<>(Path.of("some", "value"), 42), - new Add<>(Path.of("some", "value"), 42L), - new Add<>(Path.of("some", "value"), uuid), - new Add<>(Path.of("some", "value"), new BigDecimal("128976548936549275.9674592348654789")), - new Add<>(Path.of("some", "value"), new BigInteger("90374692364523789623490569234562347895")), - new Add<>(Path.of("some", "value"), true), - new Add<>(Path.of("some", "value"), date), - new Add<>(Path.of("some", "value"), 4234.2345))); + assertThat(patch, is(equalTo(Patch.of(bytes)))); } } \ No newline at end of file diff --git a/src/test/java/io/datanerds/avropatch/operation/matcher/OperationMatchers.java b/src/test/java/io/datanerds/avropatch/operation/matcher/OperationMatchers.java index 8c7fccc..8686160 100644 --- a/src/test/java/io/datanerds/avropatch/operation/matcher/OperationMatchers.java +++ b/src/test/java/io/datanerds/avropatch/operation/matcher/OperationMatchers.java @@ -40,7 +40,11 @@ public static Matcher> hasItem(T item) { } public static Matcher> hasItems(T... items) { - List>> all = new ArrayList>>(items.length); + return hasItems(Arrays.asList(items)); + } + + public static Matcher> hasItems(List items) { + List>> all = new ArrayList>>(items.size()); for (T element : items) { all.add(hasItem(element)); } @@ -56,7 +60,7 @@ public static Matcher equalTo(Add expected) { @Override protected boolean matchesSafely(Add item) { return item.path.equals(expected.path) - && item.value.equals(expected.value); + && compareNullable(expected.value, item.value); } }; } @@ -99,7 +103,7 @@ public static Matcher equalTo(Replace expected) { @Override protected boolean matchesSafely(Replace item) { return item.path.equals(expected.path) - && item.value.equals(expected.value); + && compareNullable(expected.value, item.value); } }; } @@ -110,10 +114,17 @@ public static Matcher equalTo(Test expected) { @Override protected boolean matchesSafely(Test item) { return item.path.equals(expected.path) - && item.value.equals(expected.value); + && compareNullable(expected.value, item.value); } }; } + + private static boolean compareNullable(T expected, V item) { + if (item == null) { + return expected == null; + } + return item.equals(expected); + } } } diff --git a/src/test/java/io/datanerds/avropatch/operation/matcher/PatchMatcher.java b/src/test/java/io/datanerds/avropatch/operation/matcher/PatchMatcher.java new file mode 100644 index 0000000..d7d3e98 --- /dev/null +++ b/src/test/java/io/datanerds/avropatch/operation/matcher/PatchMatcher.java @@ -0,0 +1,19 @@ +package io.datanerds.avropatch.operation.matcher; + +import io.datanerds.avropatch.Patch; +import org.hamcrest.CustomTypeSafeMatcher; +import org.hamcrest.Matcher; + +import static io.datanerds.avropatch.operation.matcher.OperationMatchers.hasItems; + +public class PatchMatcher { + + public static Matcher equalTo(Patch expected) { + return new CustomTypeSafeMatcher(expected.toString()) { + @Override + protected boolean matchesSafely(Patch item) { + return hasItems(item.getOperations()).matches(expected.getOperations()); + } + }; + } +} diff --git a/src/test/java/io/datanerds/avropatch/schema/OperationTypesTest.java b/src/test/java/io/datanerds/avropatch/schema/OperationTypesTest.java index 0568f58..ef3fc12 100644 --- a/src/test/java/io/datanerds/avropatch/schema/OperationTypesTest.java +++ b/src/test/java/io/datanerds/avropatch/schema/OperationTypesTest.java @@ -1,10 +1,8 @@ package io.datanerds.avropatch.schema; +import com.google.common.collect.ImmutableMap; import io.datanerds.avropatch.operation.*; -import io.datanerds.avropatch.value.conversion.*; import org.apache.avro.Schema; -import org.codehaus.jackson.map.ObjectMapper; -import org.junit.Before; import org.junit.Test; import java.io.IOException; @@ -20,31 +18,10 @@ public class OperationTypesTest { - Schema.Parser parser; - - @Before - public void setup() throws IOException { - AvroData.get(); - Map types = new HashMap<>(); - types.put(Add.class.getSimpleName(), annotateNamespace(OperationTypes.Add.create(VALUE_TYPE_UNION))); - types.put(Copy.class.getSimpleName(), annotateNamespace(OperationTypes.Copy.create())); - types.put(Move.class.getSimpleName(), annotateNamespace(OperationTypes.Move.create())); - types.put(Remove.class.getSimpleName(), annotateNamespace(OperationTypes.Remove.create())); - types.put(Replace.class.getSimpleName(), annotateNamespace(OperationTypes.Replace.create(VALUE_TYPE_UNION))); - types.put(io.datanerds.avropatch.operation.Test.class.getSimpleName(), - annotateNamespace(OperationTypes.Test.create(VALUE_TYPE_UNION))); - - parser = new Schema.Parser(); - parser.addTypes(types); - } - @Test public void add() throws IOException { - System.out.println(OperationTypes.Add.create(VALUE_TYPE_UNION).toString()); - Schema schema = parse(Add.class); - SerializationTester tester = new SerializationTester(schema); - - List adds = createSomeOperations(value -> new Add<>(Path.of("hello", "world"), value)); + OperationTester tester = new OperationTester(Add.class); + List adds = tester.createSomeOperations(value -> new Add<>(Path.of("hello", "world"), value)); for (Operation add : adds) { assertThat(add, is(equalTo(tester.reserialize(add)))); } @@ -52,37 +29,29 @@ public void add() throws IOException { @Test public void copy() throws IOException { - Schema schema = parse(Copy.class); - SerializationTester tester = new SerializationTester(schema); - + SerializationTester tester = new OperationTester(Copy.class); Copy copy = new Copy(Path.of("from", "here"), Path.of("to", "there")); assertThat(copy, is(equalTo(tester.reserialize(copy)))); } @Test public void move() throws IOException { - Schema schema = parse(Move.class); - SerializationTester tester = new SerializationTester(schema); - + SerializationTester tester = new OperationTester(Move.class); Move copy = new Move(Path.of("from", "here"), Path.of("to", "there")); assertThat(copy, is(equalTo(tester.reserialize(copy)))); } @Test public void remove() throws IOException { - Schema schema = parse(Remove.class); - SerializationTester tester = new SerializationTester(schema); - + SerializationTester tester = new OperationTester(Remove.class); Remove remove = new Remove(Path.of("from", "here")); assertThat(remove, is(equalTo(tester.reserialize(remove)))); } @Test public void replace() throws IOException { - Schema schema = parse(Replace.class); - SerializationTester tester = new SerializationTester(schema); - - List replaces = createSomeOperations(value -> new Replace<>(Path.of("hello", "world"), value)); + OperationTester tester = new OperationTester(Replace.class); + List replaces = tester.createSomeOperations(value -> new Replace<>(Path.of("hello"), value)); for (Operation replace : replaces) { assertThat(replace, is(equalTo(tester.reserialize(replace)))); } @@ -90,44 +59,46 @@ public void replace() throws IOException { @Test public void test() throws IOException { - Schema schema = parse(io.datanerds.avropatch.operation.Test.class); - SerializationTester tester = new SerializationTester(schema); - - List tests = createSomeOperations( + OperationTester tester = new OperationTester(io.datanerds.avropatch.operation.Test.class); + List tests = tester.createSomeOperations( value -> new io.datanerds.avropatch.operation.Test<>(Path.of("hello", "world"), value)); for (Operation test : tests) { assertThat(test, is(equalTo(tester.reserialize(test)))); } } - private Schema parse(Class clazz) { - return parser.parse(String.format("\"%s\"", clazz.getName())); - } + static class OperationTester extends SerializationTester { - private List createSomeOperations(Function operationFunction) { - List operations = new ArrayList<>(); - operations.add(createOperation(operationFunction, "hello world")); - operations.add(createOperation(operationFunction, 42)); - operations.add(createOperation(operationFunction, 42L)); - operations.add(createOperation(operationFunction, 123.456d)); - operations.add(createOperation(operationFunction, 123.456f)); - operations.add(createOperation(operationFunction, new BigInteger("83647896845639495762501378945698056348956"))); - operations.add(createOperation(operationFunction, new BigDecimal("956740578902345.56734895627895"))); - operations.add(createOperation(operationFunction, UUID.randomUUID())); - operations.add(createOperation(operationFunction, new Date())); - - return operations; - } + private static final Map, Schema> schemata = new ImmutableMap.Builder() + .put(Add.class, OperationTypes.Add.create(VALUE_TYPE_UNION)) + .put(Copy.class, OperationTypes.Copy.create()) + .put(Move.class, OperationTypes.Move.create()) + .put(Remove.class, OperationTypes.Remove.create()) + .put(Replace.class, OperationTypes.Replace.create(VALUE_TYPE_UNION)) + .put(io.datanerds.avropatch.operation.Test.class, OperationTypes.Test.create(VALUE_TYPE_UNION)) + .build(); - private Operation createOperation(Function operationFunction, T value) { - return operationFunction.apply(value); - } + public OperationTester(Class clazz) { + super(schemata.get(clazz)); + } - private static Schema annotateNamespace(Schema schema) throws IOException { - ObjectMapper mappper = new ObjectMapper(); - Map jsonSchema = mappper.readValue(schema.toString(), Map.class); - jsonSchema.put("namespace", Operation.class.getPackage().getName()); - Schema.Parser parser = new Schema.Parser(); - return parser.parse(mappper.writeValueAsString(jsonSchema)); + public static List createSomeOperations(Function operationFunction) { + List operations = new ArrayList<>(); + operations.add(createOperation(operationFunction, "hello world")); + operations.add(createOperation(operationFunction, 42)); + operations.add(createOperation(operationFunction, 42L)); + operations.add(createOperation(operationFunction, 123.456d)); + operations.add(createOperation(operationFunction, 123.456f)); + operations.add(createOperation(operationFunction, new BigInteger("83647896845639495762501378945698056348956"))); + operations.add(createOperation(operationFunction, new BigDecimal("956740578902345.56734895627895"))); + operations.add(createOperation(operationFunction, UUID.randomUUID())); + operations.add(createOperation(operationFunction, new Date())); + + return operations; + } + + private static Operation createOperation(Function operationFunction, T value) { + return operationFunction.apply(value); + } } } \ No newline at end of file diff --git a/src/test/java/io/datanerds/avropatch/serialization/Bimmel.java b/src/test/java/io/datanerds/avropatch/serialization/Bimmel.java new file mode 100644 index 0000000..9bcc713 --- /dev/null +++ b/src/test/java/io/datanerds/avropatch/serialization/Bimmel.java @@ -0,0 +1,75 @@ +package io.datanerds.avropatch.serialization; + +import java.util.Objects; +import java.util.UUID; + +public class Bimmel { + + public final String name; + public final int number; + public final UUID id; + public final Bommel bommel; + + private Bimmel() { + this.bommel = null; + this.name = null; + this.number = -42; + this.id = null; + } + + public Bimmel(String name, int number, UUID id, Bommel bommel) { + this.name = name; + this.number = number; + this.id = id; + this.bommel = bommel; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Bimmel bimmel = (Bimmel) o; + return number == bimmel.number && + Objects.equals(name, bimmel.name) && + Objects.equals(id, bimmel.id) && + Objects.equals(bommel, bimmel.bommel); + } + + @Override + public int hashCode() { + return Objects.hash(name, number, id, bommel); + } + + public static class Bommel { + public final String name; + + private Bommel() { + this.name = null; + } + + public Bommel(String name) { + this.name = name; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Bommel bommel = (Bommel) o; + return Objects.equals(name, bommel.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + } +} diff --git a/src/test/java/io/datanerds/avropatch/serialization/CustomTypeSerializerTest.java b/src/test/java/io/datanerds/avropatch/serialization/CustomTypeSerializerTest.java index 4b2d195..37a0fac 100644 --- a/src/test/java/io/datanerds/avropatch/serialization/CustomTypeSerializerTest.java +++ b/src/test/java/io/datanerds/avropatch/serialization/CustomTypeSerializerTest.java @@ -1,26 +1,69 @@ package io.datanerds.avropatch.serialization; -import avro.shaded.com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList; import io.datanerds.avropatch.Patch; import io.datanerds.avropatch.operation.Add; +import io.datanerds.avropatch.operation.Copy; import io.datanerds.avropatch.operation.Path; -import org.junit.Ignore; +import io.datanerds.avropatch.operation.Replace; import org.junit.Test; import java.io.IOException; import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Date; +import java.util.UUID; -import static org.junit.Assert.*; +import static io.datanerds.avropatch.operation.matcher.PatchMatcher.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; public class CustomTypeSerializerTest { - @Ignore("work in progress") @Test - public void test() throws IOException { - CustomTypeSerializer serializer = new CustomTypeSerializer.Builder().withBigDecimal().build(); - Patch patch = new Patch(ImmutableList.of(new Add(Path.of("hello"), new BigDecimal("23946712384.4")))); + public void withCustomTypes() throws IOException { + CustomTypeSerializer serializer = new CustomTypeSerializer.Builder().withCustomTypes().build(); + Patch patch = new Patch(ImmutableList.of(new Add(Path.of("hello"), new BigDecimal("23946712384.49879324")))); byte[] bytes = serializer.toBytes(patch); - Patch newPatch = serializer.toObject(bytes); + assertThat(patch, is(equalTo(serializer.toObject(bytes)))); } + @Test + public void withCustomClass() throws IOException { + CustomTypeSerializer serializer = new CustomTypeSerializer.Builder() + .withArray() + .nullable() + .withPrimitives() + .withCustomTypes() + .endArray() + .with(Bimmel.class) + .build(); + Patch patch = new Patch(ImmutableList.of( + new Replace(Path.of("hello"), new Bimmel("string", 42, UUID.randomUUID(), new Bimmel.Bommel("Gaga"))))); + byte[] bytes = serializer.toBytes(patch); + assertThat(patch, is(equalTo(serializer.toObject(bytes)))); + } + + + @Test + public void withMultipleOperations() throws IOException { + CustomTypeSerializer serializer = new CustomTypeSerializer.Builder() + .withArray() + .withPrimitives() + .withCustomTypes() + .endArray() + .nullable() + .withPrimitives() + .with(Bimmel.class) + .build(); + Patch patch = new Patch(ImmutableList.of( + new io.datanerds.avropatch.operation.Test(Path.of("hello", "world"), null), + new Add(Path.of("hello", "world"), "string"), + new Replace(Path.of("hello"), new Bimmel("string", 42, UUID.randomUUID(), new Bimmel.Bommel("Gaga"))), + new Copy(Path.of("from", "here"), Path.of("to", "there")), + new Add(Path.of("oO"), ImmutableList.of(new BigInteger("897696124"), 42, new BigInteger("4796923435"))), + new Replace(Path.of("lol"), ImmutableList.of(new Date(), new Date(), new Date())))); + byte[] bytes = serializer.toBytes(patch); + assertThat(patch, is(equalTo(serializer.toObject(bytes)))); + } } \ No newline at end of file From a47776938717f91f015a043ce93c22e10663102e Mon Sep 17 00:00:00 2001 From: Frank Wisniewski Date: Tue, 15 Nov 2016 08:28:49 -0800 Subject: [PATCH 05/19] Added headers to Patch operation. --- .../java/io/datanerds/avropatch/Patch.java | 20 ++++- .../io/datanerds/avropatch/operation/Add.java | 6 +- .../{Value.java => DefaultSchema.java} | 40 +++++++++- .../avropatch/operation/Replace.java | 4 +- .../datanerds/avropatch/operation/Test.java | 4 +- .../io/datanerds/avropatch/PatchTest.java | 19 +++++ .../operation/matcher/OperationMatchers.java | 19 +++++ .../operation/matcher/PatchMatcher.java | 76 ++++++++++++++++++- .../avropatch/schema/OperationTypesTest.java | 2 +- 9 files changed, 172 insertions(+), 18 deletions(-) rename src/main/java/io/datanerds/avropatch/operation/{Value.java => DefaultSchema.java} (51%) diff --git a/src/main/java/io/datanerds/avropatch/Patch.java b/src/main/java/io/datanerds/avropatch/Patch.java index ee333c2..835056a 100644 --- a/src/main/java/io/datanerds/avropatch/Patch.java +++ b/src/main/java/io/datanerds/avropatch/Patch.java @@ -1,15 +1,15 @@ package io.datanerds.avropatch; +import io.datanerds.avropatch.operation.DefaultSchema; import io.datanerds.avropatch.operation.Operation; import io.datanerds.avropatch.value.conversion.*; import org.apache.avro.Schema; import org.apache.avro.io.*; +import org.apache.avro.reflect.AvroSchema; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import java.util.*; /** * This class represents a JSON PATCH operation holding a sequence of operations to apply to a given object. @@ -20,14 +20,23 @@ public class Patch { private static final PatchSerializer SERIALIZER = new PatchSerializer(); + @AvroSchema(DefaultSchema.HEADERS) + private final Map headers; private final List operations; public Patch() { operations = new ArrayList<>(); + headers = new HashMap<>(); } public Patch(List operations) { this.operations = new ArrayList<>(operations); + this.headers = new HashMap<>(); + } + + public Patch(List operations, Map headers) { + this.operations = new ArrayList<>(operations); + this.headers = new HashMap<>(headers); } public boolean addOperation(Operation operation) { @@ -38,6 +47,10 @@ public List getOperations() { return Collections.unmodifiableList(operations); } + public Map getHeaders() { + return Collections.unmodifiableMap(headers); + } + public byte[] toBytes() throws IOException { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); Encoder binaryEncoder = EncoderFactory.get().directBinaryEncoder(outputStream, null); @@ -55,7 +68,6 @@ private static class PatchSerializer { private PatchSerializer() { Schema schema = AvroData.get().getSchema(Patch.class); - writer = AvroData.get().createDatumWriter(schema); reader = AvroData.get().createDatumReader(schema); } diff --git a/src/main/java/io/datanerds/avropatch/operation/Add.java b/src/main/java/io/datanerds/avropatch/operation/Add.java index 0709c3a..0c2f8fb 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Add.java +++ b/src/main/java/io/datanerds/avropatch/operation/Add.java @@ -7,13 +7,13 @@ * This class represents the "add" operation of RFC 6902 'JavaScript Object Notation (JSON) Patch'. * * @see https://tools.ietf.org/html/rfc6902#section-4.1 - * @param Value type of add operation + * @param DefaultSchema type of add operation */ public final class Add implements Operation { @AvroIgnore public static final String op = "add"; public final Path path; - @AvroSchema(Value.SCHEMA) + @AvroSchema(DefaultSchema.VALUE) public final T value; private Add() { @@ -26,7 +26,7 @@ private Add() { * @param path Path pointing out where the JSON value should be added * @param value Actual value to add to patched object * - * @see Value + * @see DefaultSchema */ public Add(Path path, T value) { this.path = path; diff --git a/src/main/java/io/datanerds/avropatch/operation/Value.java b/src/main/java/io/datanerds/avropatch/operation/DefaultSchema.java similarity index 51% rename from src/main/java/io/datanerds/avropatch/operation/Value.java rename to src/main/java/io/datanerds/avropatch/operation/DefaultSchema.java index 7fb456d..50fdfdc 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Value.java +++ b/src/main/java/io/datanerds/avropatch/operation/DefaultSchema.java @@ -3,8 +3,8 @@ /** * This interface holds the schema for all valid value types. */ -interface Value { - String SCHEMA = +public interface DefaultSchema { + String VALUE = "[{" + " \"type\": \"record\"," + " \"name\": \"decimal\"," @@ -45,4 +45,40 @@ interface Value { + " \"logicalType\": \"big-integer\"" + " }, \"boolean\", \"timestamp\", \"double\", \"float\", \"int\", \"long\", \"null\", \"string\", \"uuid\"]" + "}]"; + + String HEADERS = + "{\n" + + " \"type\": \"map\",\n" + + " \"values\": [\"boolean\", \"double\", \"float\", \"int\", \"long\", \"string\", {\n" + + " \"type\": \"record\",\n" + + " \"name\": \"decimal\",\n" + + " \"doc\": \"BigDecimal value represented via it's scale and unscaled value.\",\n" + + " \"fields\": [{\n" + + " \"name\": \"unscaledValue\",\n" + + " \"type\": {\n" + + " \"type\": \"bytes\",\n" + + " \"logicalType\": \"big-integer\"\n" + + " }\n" + + " }, {\n" + + " \"name\": \"scale\",\n" + + " \"type\": \"int\"\n" + + " }],\n" + + " \"logicalType\": \"big-decimal\"\n" + + " }, {\n" + + " \"type\": \"bytes\",\n" + + " \"logicalType\": \"big-integer\"\n" + + " }, {\n" + + " \"type\": \"fixed\",\n" + + " \"name\": \"timestamp\",\n" + + " \"doc\": \"Timestamp representing the number of milliseconds since January 1, 1970, 00:00:00 GMT\",\n" + + " \"size\": 8,\n" + + " \"logicalType\": \"timestamp\"\n" + + " }, {\n" + + " \"type\": \"fixed\",\n" + + " \"name\": \"uuid\",\n" + + " \"doc\": \"UUID serialized via two long values: It's most significant and least significant 64 bits.\",\n" + + " \"size\": 16,\n" + + " \"logicalType\": \"uuid\"\n" + + " }]\n" + + "}"; } diff --git a/src/main/java/io/datanerds/avropatch/operation/Replace.java b/src/main/java/io/datanerds/avropatch/operation/Replace.java index 0525581..b96d5c6 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Replace.java +++ b/src/main/java/io/datanerds/avropatch/operation/Replace.java @@ -12,7 +12,7 @@ public final class Replace implements Operation { @AvroIgnore public static final String op = "replace"; public final Path path; - @AvroSchema(Value.SCHEMA) + @AvroSchema(DefaultSchema.VALUE) public final T value; private Replace() { @@ -25,7 +25,7 @@ private Replace() { * @param path Path pointing out where the JSON value should be replaced * @param value Actual value to replace in patched object * - * @see Value + * @see DefaultSchema */ public Replace(Path path, T value) { this.path = path; diff --git a/src/main/java/io/datanerds/avropatch/operation/Test.java b/src/main/java/io/datanerds/avropatch/operation/Test.java index 6321ec8..d306246 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Test.java +++ b/src/main/java/io/datanerds/avropatch/operation/Test.java @@ -12,7 +12,7 @@ public final class Test implements Operation { @AvroIgnore public static final String op = "test"; public final Path path; - @AvroSchema(Value.SCHEMA) + @AvroSchema(DefaultSchema.VALUE) public final T value; private Test() { @@ -25,7 +25,7 @@ private Test() { * @param path Path pointing out which JSON value should be tested against * @param value Actual value to test against * - * @see Value + * @see DefaultSchema */ public Test(Path path, T value) { this.path = path; diff --git a/src/test/java/io/datanerds/avropatch/PatchTest.java b/src/test/java/io/datanerds/avropatch/PatchTest.java index fffa9b8..a9e8b1d 100644 --- a/src/test/java/io/datanerds/avropatch/PatchTest.java +++ b/src/test/java/io/datanerds/avropatch/PatchTest.java @@ -1,5 +1,6 @@ package io.datanerds.avropatch; +import avro.shaded.com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableList; import io.datanerds.avropatch.operation.*; import org.junit.Test; @@ -7,6 +8,7 @@ import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; +import java.util.Collections; import java.util.Date; import java.util.List; import java.util.UUID; @@ -115,4 +117,21 @@ public void serializesDefaultValueTypes() throws IOException { assertThat(patch, is(equalTo(Patch.of(bytes)))); } + @Test + public void serializesArbitraryHeaders() throws IOException { + Patch patch = new Patch(Collections.EMPTY_LIST, ImmutableMap.of("header 1", UUID.randomUUID(), + "header 2", new Date(), "header 3", 1234L, "header 4", new BigDecimal("3214123453.123512345"))); + + byte[] bytes = patch.toBytes(); + assertThat(patch, is(equalTo(Patch.of(bytes)))); + + patch = new Patch(ImmutableList.of(new Copy(Path.of("from", "here"), Path.of("to", "there")), + new Move(Path.of("from", "here"), Path.of("to", "there"))), + ImmutableMap.of("header 1", UUID.randomUUID(), + "header 2", new Date(), "header 3", 1234L, "header 4", new BigDecimal("3214123453.123512345"))); + + bytes = patch.toBytes(); + assertThat(patch, is(equalTo(Patch.of(bytes)))); + } + } \ No newline at end of file diff --git a/src/test/java/io/datanerds/avropatch/operation/matcher/OperationMatchers.java b/src/test/java/io/datanerds/avropatch/operation/matcher/OperationMatchers.java index 8686160..dd6acca 100644 --- a/src/test/java/io/datanerds/avropatch/operation/matcher/OperationMatchers.java +++ b/src/test/java/io/datanerds/avropatch/operation/matcher/OperationMatchers.java @@ -1,5 +1,6 @@ package io.datanerds.avropatch.operation.matcher; +import com.google.common.base.Joiner; import io.datanerds.avropatch.operation.*; import org.hamcrest.CustomTypeSafeMatcher; import org.hamcrest.Matcher; @@ -52,6 +53,24 @@ public static Matcher> hasItems(List items) return allOf(all); } + public static Matcher> hasItemsOrdered(List expected) { + return new CustomTypeSafeMatcher>(Joiner.on(",").join(expected)) { + @Override + protected boolean matchesSafely(List value) { + if (expected.size() != value.size()) { + return false; + } + for (int i = 0; i < expected.size(); i++) { + if (!equalTo(expected.get(i)).matches(value.get(i))) { + return false; + } + } + return true; + } + + }; + } + private static class Matchers { public static Matcher equalTo(Add expected) { diff --git a/src/test/java/io/datanerds/avropatch/operation/matcher/PatchMatcher.java b/src/test/java/io/datanerds/avropatch/operation/matcher/PatchMatcher.java index d7d3e98..1f2e7f5 100644 --- a/src/test/java/io/datanerds/avropatch/operation/matcher/PatchMatcher.java +++ b/src/test/java/io/datanerds/avropatch/operation/matcher/PatchMatcher.java @@ -1,19 +1,87 @@ package io.datanerds.avropatch.operation.matcher; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import io.datanerds.avropatch.Patch; +import io.datanerds.avropatch.operation.Copy; +import io.datanerds.avropatch.operation.Move; +import io.datanerds.avropatch.operation.Operation; +import io.datanerds.avropatch.operation.Path; import org.hamcrest.CustomTypeSafeMatcher; import org.hamcrest.Matcher; +import org.junit.Test; -import static io.datanerds.avropatch.operation.matcher.OperationMatchers.hasItems; +import java.io.IOException; +import java.util.Collections; +import java.util.UUID; + +import static io.datanerds.avropatch.operation.matcher.OperationMatchers.hasItemsOrdered; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.AllOf.allOf; public class PatchMatcher { - public static Matcher equalTo(Patch expected) { - return new CustomTypeSafeMatcher(expected.toString()) { + public static Matcher equalTo(Patch items) { + return allOf(operationsEqualTo(items), headersEqualTo(items)); + } + + private static Matcher operationsEqualTo(Patch expected) { + return new CustomTypeSafeMatcher(Joiner.on(",").join(expected.getOperations())) { @Override protected boolean matchesSafely(Patch item) { - return hasItems(item.getOperations()).matches(expected.getOperations()); + return hasItemsOrdered(item.getOperations()).matches(expected.getOperations()); } }; } + + private static Matcher headersEqualTo(Patch expected) { + return new CustomTypeSafeMatcher(Joiner.on(",").join(expected.getHeaders().keySet())) { + @Override + protected boolean matchesSafely(Patch item) { + if (expected.getHeaders().size() != item.getHeaders().size()) { + return false; + } + + for (String key : item.getHeaders().keySet()) { + if (!item.getHeaders().get(key).equals(expected.getHeaders().get(key))) { + return false; + } + } + + return true; + } + }; + } + + @Test + public void headers() throws IOException { + UUID uuid = UUID.randomUUID(); + Patch patch = new Patch(Collections.EMPTY_LIST, ImmutableMap.of("header 1", uuid)); + Patch patch2 = new Patch(Collections.EMPTY_LIST, ImmutableMap.of("header 1", uuid)); + Patch patch3 = new Patch(Collections.EMPTY_LIST, ImmutableMap.of("header 2", uuid)); + Patch patch4 = new Patch(Collections.EMPTY_LIST, ImmutableMap.of("header 1", UUID.randomUUID())); + + assertThat(patch, is(equalTo(patch2))); + assertThat(patch2, is(equalTo(patch))); + assertThat(patch, is(not(equalTo(patch3)))); + assertThat(patch, is(not(equalTo(patch4)))); + } + + @Test + public void operations() throws IOException { + Operation operation1 = new Copy(Path.of("from", "here"), Path.of("to", "there")); + Operation operation2 = new Move(Path.of("from", "here"), Path.of("to", "there")); + Patch patch1 = new Patch(ImmutableList.of(operation1, operation2), Collections.EMPTY_MAP); + Patch patch2 = new Patch(ImmutableList.of(operation1, operation2), Collections.EMPTY_MAP); + Patch patch3 = new Patch(ImmutableList.of(operation1), Collections.EMPTY_MAP); + Patch patch4 = new Patch(ImmutableList.of(operation2, operation1), Collections.EMPTY_MAP); + + assertThat(patch1, is(equalTo(patch2))); + assertThat(patch2, is(equalTo(patch1))); + assertThat(patch1, is(not(equalTo(patch3)))); + assertThat(patch1, is(not(equalTo(patch4)))); + } } diff --git a/src/test/java/io/datanerds/avropatch/schema/OperationTypesTest.java b/src/test/java/io/datanerds/avropatch/schema/OperationTypesTest.java index ef3fc12..cd7f7e7 100644 --- a/src/test/java/io/datanerds/avropatch/schema/OperationTypesTest.java +++ b/src/test/java/io/datanerds/avropatch/schema/OperationTypesTest.java @@ -89,7 +89,7 @@ public static List createSomeOperations(Function o operations.add(createOperation(operationFunction, 42L)); operations.add(createOperation(operationFunction, 123.456d)); operations.add(createOperation(operationFunction, 123.456f)); - operations.add(createOperation(operationFunction, new BigInteger("83647896845639495762501378945698056348956"))); + operations.add(createOperation(operationFunction, new BigInteger("8364789684563949576378945698056348956"))); operations.add(createOperation(operationFunction, new BigDecimal("956740578902345.56734895627895"))); operations.add(createOperation(operationFunction, UUID.randomUUID())); operations.add(createOperation(operationFunction, new Date())); From 04d8e9c50e680a6e63d08c4ed9c614661aef35fd Mon Sep 17 00:00:00 2001 From: Frank Wisniewski Date: Thu, 17 Nov 2016 13:25:38 -0800 Subject: [PATCH 06/19] Added concurrency tests for serialization. --- project.gradle | 3 +- .../serialization/CustomTypeSerializer.java | 2 + .../serialization/ConcurrencyTest.java | 213 ++++++++++++++++++ 3 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java diff --git a/project.gradle b/project.gradle index be9d86f..9fec903 100644 --- a/project.gradle +++ b/project.gradle @@ -17,7 +17,8 @@ dependencies { compile([ "org.apache.avro:avro:1.8.1", - "org.slf4j:slf4j-api:1.7.18" + "org.slf4j:slf4j-api:1.7.18", + "com.google.code.findbugs:jsr305:3.0.1" ]) testCompile([ diff --git a/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java b/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java index d96d28f..c2ad59d 100644 --- a/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java +++ b/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java @@ -12,6 +12,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.concurrent.ThreadSafe; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.lang.reflect.Type; @@ -22,6 +23,7 @@ import static io.datanerds.avropatch.schema.PrimitiveTypes.*; +@ThreadSafe public final class CustomTypeSerializer { private static Logger logger = LoggerFactory.getLogger(CustomTypeSerializer.class); diff --git a/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java b/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java new file mode 100644 index 0000000..09a780f --- /dev/null +++ b/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java @@ -0,0 +1,213 @@ +package io.datanerds.avropatch.serialization; + +import com.google.common.collect.ImmutableList; +import io.datanerds.avropatch.Patch; +import io.datanerds.avropatch.operation.*; +import org.hamcrest.CoreMatchers; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.*; +import java.util.concurrent.*; + +import static io.datanerds.avropatch.operation.matcher.PatchMatcher.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; + +public class ConcurrencyTest { + + private static Logger logger = LoggerFactory.getLogger(ConcurrencyTest.class); + private static final int NUMBER_OF_THREADS = 200; + private static final int NUMBER_OF_PATCHES = 100; + private static final int MAX_PATCH_SIZE = 50; + private static final List PATCHES = Collections.unmodifiableList(generateData()); + + private static final CustomTypeSerializer SERIALIZER = new CustomTypeSerializer.Builder() + .withArray() + .withPrimitives() + .withCustomTypes() + .nullable() + .with(Bimmel.class) + .endArray() + .withPrimitives() + .withCustomTypes() + .with(Bimmel.class) + .nullable() + .build(); + + @Test + public void customTypeSerializer() throws InterruptedException { + CountDownLatch startThreadsLatch = new CountDownLatch(1); + CountDownLatch threadsFinishedLatch = new CountDownLatch(NUMBER_OF_THREADS - 1); + ExecutorService executorService = Executors.newFixedThreadPool(NUMBER_OF_THREADS); + + List>> futures = new ArrayList<>(); + for (int i = 0; i < NUMBER_OF_THREADS; i++) { + futures.add(executorService.submit(() -> { + List output = new ArrayList<>(); + List input = ImmutableList.copyOf(PATCHES); + try { + startThreadsLatch.await(); + for (Patch patch : input) { + byte[] bytes = SERIALIZER.toBytes(patch); + output.add(SERIALIZER.toObject(bytes)); + } + } catch (Exception ex) { + logger.warn("Serialization thread failed.", ex); + } finally { + threadsFinishedLatch.countDown(); + } + return output; + })); + } + + logger.info("Releasing {} serialization threads", NUMBER_OF_THREADS); + startThreadsLatch.countDown(); + threadsFinishedLatch.await(); + executorService.shutdownNow(); + + + logger.info("All threads finished, verifying output..."); + futures.stream().forEach(future -> { + try { + verify(PATCHES, future.get()); + } catch (InterruptedException ex) { + logger.warn("Serialization thread failed.", ex); + } catch (ExecutionException ex) { + logger.warn("Serialization thread failed.", ex); + } + }); + } + + private static List generateData() { + OperationGenerator operationGenerator = new OperationGenerator(); + Random random = new Random(); + List patches = new ArrayList<>(); + for (int i = 0; i < NUMBER_OF_PATCHES; i++) { + Patch patch = new Patch(); + for (int j = 0; j < random.nextInt(MAX_PATCH_SIZE); j++) { + patch.addOperation(operationGenerator.generateOperation()); + } + patches.add(patch); + } + return patches; + } + + private void verify(List input, List output) { + assertThat(input, notNullValue()); + assertThat(output, notNullValue()); + assertThat(input.size(), is(CoreMatchers.equalTo(output.size()))); + for (int i = 0; i < input.size(); i++) { + assertThat(input.get(i), is(equalTo(output.get(i)))); + } + } + + private static class OperationGenerator { + + enum Type {ADD, COPY, MOVE, REMOVE, REPLACE, TEST} + + private final ValueGenerator valueGenerator = new ValueGenerator(); + private final Random random = new Random(); + + private Path generatePath() { + return Path.of("hello", "world"); + } + + public Operation generateOperation() { + Type type = Type.values()[random.nextInt(Type.values().length)]; + return generateOperation(type); + } + + public Operation generateOperation(Type type) { + switch (type) { + case ADD: + return new Add(generatePath(), valueGenerator.generate()); + case COPY: + return new Copy(generatePath(), generatePath()); + case MOVE: + return new Move(generatePath(), generatePath()); + case REMOVE: + return new Remove(generatePath()); + case REPLACE: + return new Replace<>(generatePath(), valueGenerator.generate()); + case TEST: + return new io.datanerds.avropatch.operation.Test(generatePath(), valueGenerator.generate()); + default: + throw new IllegalArgumentException("Unknown type"); + } + } + + } + + + private static class ValueGenerator { + enum Type {BOOLEAN, DOUBLE, FLOAT, INTEGER, LONG, NULL, STRING, BIG_INTEGER, BIG_DECIMAL, UUID, DATE, BIMMEL} + + private final Random random = new Random(); + + private static int getDigitCount(BigInteger number) { + double factor = Math.log(2) / Math.log(10); + int digitCount = (int) (factor * number.bitLength() + 1); + if (BigInteger.TEN.pow(digitCount - 1).compareTo(number) > 0) { + return digitCount - 1; + } + return digitCount; + } + + public Object generate() { + Type type = Type.values()[random.nextInt(Type.values().length)]; + if (random.nextBoolean()) { + return generate(type); + } else { + return generateList(type); + } + } + + public List generateList(Type type) { + List data = new ArrayList<>(); + for (int i = 0; i < MAX_PATCH_SIZE; i++) { + data.add(generate(type)); + } + return data; + } + + public Object generate(Type type) { + switch (type) { + case BOOLEAN: + return random.nextBoolean(); + case DOUBLE: + return random.nextDouble(); + case FLOAT: + return random.nextFloat(); + case INTEGER: + return random.nextInt(); + case LONG: + return random.nextLong(); + case NULL: + return null; + case STRING: + return UUID.randomUUID().toString(); + case BIG_INTEGER: + return new BigInteger(random.nextInt(256), random); + case BIG_DECIMAL: + BigInteger unscaledValue= new BigInteger(random.nextInt(256), random); + int numberOfDigits = getDigitCount(unscaledValue); + int scale = random.nextInt(numberOfDigits + 1); + return new BigDecimal(unscaledValue, scale); + case UUID: + return UUID.randomUUID(); + case DATE: + return new Date(random.nextLong()); + case BIMMEL: + return new Bimmel((String)generate(Type.STRING), (int)generate(Type.INTEGER), UUID.randomUUID(), + new Bimmel.Bommel((String)generate(Type.STRING))); + default: + throw new IllegalArgumentException(String.format("Unknown type %s", type)); + } + } + } +} \ No newline at end of file From 8116216657b42afe13a4286fba19b5b3604fa8ab Mon Sep 17 00:00:00 2001 From: Frank Wisniewski Date: Wed, 14 Dec 2016 14:15:04 +0100 Subject: [PATCH 07/19] PRR (@dhiller): Refactored OperationGenerator.Type to enum strategy pattern. --- .../serialization/ConcurrencyTest.java | 76 +++++++++---------- 1 file changed, 35 insertions(+), 41 deletions(-) diff --git a/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java b/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java index 09a780f..467d7d7 100644 --- a/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java +++ b/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java @@ -12,6 +12,7 @@ import java.math.BigInteger; import java.util.*; import java.util.concurrent.*; +import java.util.function.Supplier; import static io.datanerds.avropatch.operation.matcher.PatchMatcher.equalTo; import static org.hamcrest.CoreMatchers.is; @@ -39,6 +40,38 @@ public class ConcurrencyTest { .nullable() .build(); + private enum OperationType { + ADD(() -> new Add(generatePath(), generateValue())), + COPY(() -> new Copy(generatePath(), generatePath())), + MOVE(() -> new Move(generatePath(), generatePath())), + REMOVE(() -> new Remove(generatePath())), + REPLACE(() -> new Replace<>(generatePath(), generateValue())), + TEST(() -> new io.datanerds.avropatch.operation.Test(generatePath(), generateValue())); + + private static final Random random = new Random(); + private static final ValueGenerator valueGenerator = new ValueGenerator(); + private final Supplier operationSupplier; + + OperationType(Supplier operationSupplier) { + this.operationSupplier = operationSupplier; + } + + public static Operation generateOperation() { + OperationType operationType = OperationType.values()[random.nextInt(OperationType.values().length)]; + return operationType.generate(); + } + + private static Object generateValue() { return valueGenerator.generate(); } + + private static Path generatePath() { + return Path.of("hello", "world"); + } + + private Operation generate() { + return operationSupplier.get(); + } + } + @Test public void customTypeSerializer() throws InterruptedException { CountDownLatch startThreadsLatch = new CountDownLatch(1); @@ -84,13 +117,12 @@ public void customTypeSerializer() throws InterruptedException { } private static List generateData() { - OperationGenerator operationGenerator = new OperationGenerator(); Random random = new Random(); List patches = new ArrayList<>(); for (int i = 0; i < NUMBER_OF_PATCHES; i++) { Patch patch = new Patch(); for (int j = 0; j < random.nextInt(MAX_PATCH_SIZE); j++) { - patch.addOperation(operationGenerator.generateOperation()); + patch.addOperation(OperationType.generateOperation()); } patches.add(patch); } @@ -106,44 +138,6 @@ private void verify(List input, List output) { } } - private static class OperationGenerator { - - enum Type {ADD, COPY, MOVE, REMOVE, REPLACE, TEST} - - private final ValueGenerator valueGenerator = new ValueGenerator(); - private final Random random = new Random(); - - private Path generatePath() { - return Path.of("hello", "world"); - } - - public Operation generateOperation() { - Type type = Type.values()[random.nextInt(Type.values().length)]; - return generateOperation(type); - } - - public Operation generateOperation(Type type) { - switch (type) { - case ADD: - return new Add(generatePath(), valueGenerator.generate()); - case COPY: - return new Copy(generatePath(), generatePath()); - case MOVE: - return new Move(generatePath(), generatePath()); - case REMOVE: - return new Remove(generatePath()); - case REPLACE: - return new Replace<>(generatePath(), valueGenerator.generate()); - case TEST: - return new io.datanerds.avropatch.operation.Test(generatePath(), valueGenerator.generate()); - default: - throw new IllegalArgumentException("Unknown type"); - } - } - - } - - private static class ValueGenerator { enum Type {BOOLEAN, DOUBLE, FLOAT, INTEGER, LONG, NULL, STRING, BIG_INTEGER, BIG_DECIMAL, UUID, DATE, BIMMEL} @@ -206,7 +200,7 @@ public Object generate(Type type) { return new Bimmel((String)generate(Type.STRING), (int)generate(Type.INTEGER), UUID.randomUUID(), new Bimmel.Bommel((String)generate(Type.STRING))); default: - throw new IllegalArgumentException(String.format("Unknown type %s", type)); + throw new IllegalArgumentException(String.format("Unknown type %operationSupplier", type)); } } } From 22315e5b8367a54885249b102a2707637dfa8921 Mon Sep 17 00:00:00 2001 From: Frank Wisniewski Date: Wed, 14 Dec 2016 15:35:46 +0100 Subject: [PATCH 08/19] PRR (@dhiller): Refactored value generator. --- .../serialization/ConcurrencyTest.java | 105 +--------------- .../serialization/OperationGenerator.java | 116 ++++++++++++++++++ 2 files changed, 117 insertions(+), 104 deletions(-) create mode 100644 src/test/java/io/datanerds/avropatch/serialization/OperationGenerator.java diff --git a/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java b/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java index 467d7d7..091a1e0 100644 --- a/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java +++ b/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java @@ -2,17 +2,13 @@ import com.google.common.collect.ImmutableList; import io.datanerds.avropatch.Patch; -import io.datanerds.avropatch.operation.*; import org.hamcrest.CoreMatchers; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.math.BigDecimal; -import java.math.BigInteger; import java.util.*; import java.util.concurrent.*; -import java.util.function.Supplier; import static io.datanerds.avropatch.operation.matcher.PatchMatcher.equalTo; import static org.hamcrest.CoreMatchers.is; @@ -40,38 +36,6 @@ public class ConcurrencyTest { .nullable() .build(); - private enum OperationType { - ADD(() -> new Add(generatePath(), generateValue())), - COPY(() -> new Copy(generatePath(), generatePath())), - MOVE(() -> new Move(generatePath(), generatePath())), - REMOVE(() -> new Remove(generatePath())), - REPLACE(() -> new Replace<>(generatePath(), generateValue())), - TEST(() -> new io.datanerds.avropatch.operation.Test(generatePath(), generateValue())); - - private static final Random random = new Random(); - private static final ValueGenerator valueGenerator = new ValueGenerator(); - private final Supplier operationSupplier; - - OperationType(Supplier operationSupplier) { - this.operationSupplier = operationSupplier; - } - - public static Operation generateOperation() { - OperationType operationType = OperationType.values()[random.nextInt(OperationType.values().length)]; - return operationType.generate(); - } - - private static Object generateValue() { return valueGenerator.generate(); } - - private static Path generatePath() { - return Path.of("hello", "world"); - } - - private Operation generate() { - return operationSupplier.get(); - } - } - @Test public void customTypeSerializer() throws InterruptedException { CountDownLatch startThreadsLatch = new CountDownLatch(1); @@ -122,7 +86,7 @@ private static List generateData() { for (int i = 0; i < NUMBER_OF_PATCHES; i++) { Patch patch = new Patch(); for (int j = 0; j < random.nextInt(MAX_PATCH_SIZE); j++) { - patch.addOperation(OperationType.generateOperation()); + patch.addOperation(OperationGenerator.generate()); } patches.add(patch); } @@ -137,71 +101,4 @@ private void verify(List input, List output) { assertThat(input.get(i), is(equalTo(output.get(i)))); } } - - private static class ValueGenerator { - enum Type {BOOLEAN, DOUBLE, FLOAT, INTEGER, LONG, NULL, STRING, BIG_INTEGER, BIG_DECIMAL, UUID, DATE, BIMMEL} - - private final Random random = new Random(); - - private static int getDigitCount(BigInteger number) { - double factor = Math.log(2) / Math.log(10); - int digitCount = (int) (factor * number.bitLength() + 1); - if (BigInteger.TEN.pow(digitCount - 1).compareTo(number) > 0) { - return digitCount - 1; - } - return digitCount; - } - - public Object generate() { - Type type = Type.values()[random.nextInt(Type.values().length)]; - if (random.nextBoolean()) { - return generate(type); - } else { - return generateList(type); - } - } - - public List generateList(Type type) { - List data = new ArrayList<>(); - for (int i = 0; i < MAX_PATCH_SIZE; i++) { - data.add(generate(type)); - } - return data; - } - - public Object generate(Type type) { - switch (type) { - case BOOLEAN: - return random.nextBoolean(); - case DOUBLE: - return random.nextDouble(); - case FLOAT: - return random.nextFloat(); - case INTEGER: - return random.nextInt(); - case LONG: - return random.nextLong(); - case NULL: - return null; - case STRING: - return UUID.randomUUID().toString(); - case BIG_INTEGER: - return new BigInteger(random.nextInt(256), random); - case BIG_DECIMAL: - BigInteger unscaledValue= new BigInteger(random.nextInt(256), random); - int numberOfDigits = getDigitCount(unscaledValue); - int scale = random.nextInt(numberOfDigits + 1); - return new BigDecimal(unscaledValue, scale); - case UUID: - return UUID.randomUUID(); - case DATE: - return new Date(random.nextLong()); - case BIMMEL: - return new Bimmel((String)generate(Type.STRING), (int)generate(Type.INTEGER), UUID.randomUUID(), - new Bimmel.Bommel((String)generate(Type.STRING))); - default: - throw new IllegalArgumentException(String.format("Unknown type %operationSupplier", type)); - } - } - } } \ No newline at end of file diff --git a/src/test/java/io/datanerds/avropatch/serialization/OperationGenerator.java b/src/test/java/io/datanerds/avropatch/serialization/OperationGenerator.java new file mode 100644 index 0000000..6311864 --- /dev/null +++ b/src/test/java/io/datanerds/avropatch/serialization/OperationGenerator.java @@ -0,0 +1,116 @@ +package io.datanerds.avropatch.serialization; + +import io.datanerds.avropatch.operation.*; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Random; +import java.util.function.Supplier; + +enum OperationGenerator { + + @SuppressWarnings("unchecked") + ADD(() -> new Add(generatePath(), generateValue())), + COPY(() -> new Copy(generatePath(), generatePath())), + MOVE(() -> new Move(generatePath(), generatePath())), + REMOVE(() -> new Remove(generatePath())), + REPLACE(() -> new Replace<>(generatePath(), generateValue())), + @SuppressWarnings("unchecked") + TEST(() -> new io.datanerds.avropatch.operation.Test(generatePath(), generateValue())); + + private static final Random random = new Random(); + private static final int MAX_LIST_SIZE = 50; + private final Supplier operationSupplier; + + OperationGenerator(Supplier operationSupplier) { + this.operationSupplier = operationSupplier; + } + + public static Operation generate() { + OperationGenerator operationType = values()[random.nextInt(values().length)]; + return operationType.generateOperation(); + } + + private static Object generateValue() { return ValueType.generateObject(); } + + private static Path generatePath() { + return Path.of("hello", "world"); + } + + private Operation generateOperation() { + return operationSupplier.get(); + } + + enum ValueType { + BOOLEAN(random::nextBoolean), + DOUBLE(random::nextDouble), + FLOAT(random::nextFloat), + INTEGER(random::nextInt), + LONG(random::nextLong), + NULL(() -> null), + STRING(ValueType::randomString), + BIG_INTEGER(() -> new BigInteger(random.nextInt(256), random)), + BIG_DECIMAL(ValueType::randomBigDecimal), + UUID(java.util.UUID::randomUUID), + DATE(() -> new Date(random.nextLong())), + BIMMEL(ValueType::randomBimmel); + + private final Supplier valueSupplier; + + ValueType(Supplier valueSupplier) { + this.valueSupplier = valueSupplier; + } + + public static Object generateObject() { + ValueType type = ValueType.values()[random.nextInt(ValueType.values().length)]; + if (random.nextBoolean()) { + return type.generate(); + } else { + return generateList(type); + } + } + + private static List generateList(ValueType type) { + List data = new ArrayList<>(); + for (int i = 0; i < MAX_LIST_SIZE; i++) { + data.add(type.generate()); + } + return data; + } + + private Object generate() { + return valueSupplier.get(); + } + + private static String randomString() { + return java.util.UUID.randomUUID().toString(); + } + + private static BigDecimal randomBigDecimal() { + BigInteger unscaledValue= new BigInteger(random.nextInt(256), random); + int numberOfDigits = getDigitCount(unscaledValue); + int scale = random.nextInt(numberOfDigits + 1); + return new BigDecimal(unscaledValue, scale); + } + + private static int getDigitCount(BigInteger number) { + double factor = Math.log(2) / Math.log(10); + int digitCount = (int) (factor * number.bitLength() + 1); + if (BigInteger.TEN.pow(digitCount - 1).compareTo(number) > 0) { + return digitCount - 1; + } + return digitCount; + } + + private static Bimmel randomBimmel() { + return new Bimmel( + (String) STRING.generate(), + (int) INTEGER.generate(), + (java.util.UUID) UUID.generate(), + new Bimmel.Bommel(randomString())); + } + } +} \ No newline at end of file From 79d42216ffa3a5b19c8d67c147b2de0219d6b567 Mon Sep 17 00:00:00 2001 From: Frank Wisniewski Date: Thu, 15 Dec 2016 09:48:52 +0100 Subject: [PATCH 09/19] PRR (@dhiller): Some code enhancements. --- .../java/io/datanerds/avropatch/Patch.java | 3 +- .../avropatch/schema/CustomTypes.java | 4 +-- .../serialization/CustomTypeSerializer.java | 8 ++--- .../avropatch/value/conversion/AvroData.java | 9 ++--- .../conversion/BigDecimalConversion.java | 2 +- .../io/datanerds/avropatch/PatchTest.java | 35 ++++++++++++------- .../operation/matcher/OperationMatchers.java | 10 +++--- .../serialization/ConcurrencyTest.java | 4 +-- 8 files changed, 40 insertions(+), 35 deletions(-) diff --git a/src/main/java/io/datanerds/avropatch/Patch.java b/src/main/java/io/datanerds/avropatch/Patch.java index 835056a..af09860 100644 --- a/src/main/java/io/datanerds/avropatch/Patch.java +++ b/src/main/java/io/datanerds/avropatch/Patch.java @@ -25,8 +25,7 @@ public class Patch { private final List operations; public Patch() { - operations = new ArrayList<>(); - headers = new HashMap<>(); + this(new ArrayList<>(), new HashMap<>()); } public Patch(List operations) { diff --git a/src/main/java/io/datanerds/avropatch/schema/CustomTypes.java b/src/main/java/io/datanerds/avropatch/schema/CustomTypes.java index 5d44940..19f8476 100644 --- a/src/main/java/io/datanerds/avropatch/schema/CustomTypes.java +++ b/src/main/java/io/datanerds/avropatch/schema/CustomTypes.java @@ -31,9 +31,7 @@ public interface CustomTypes { interface PatchType { String NAME = Patch.class.getSimpleName(); - String DOC = "This record represents a JSON PATCH operation holding a sequence of operations to apply to a" - + " given object."; - + String DOC = "This record represents a PATCH holding a sequence of operations to apply to a given object."; static Schema create(Schema valueSchema) { Schema.Field operations = new Schema.Field("operations", Schema.createArray(OperationTypes.Operation.create(valueSchema)), diff --git a/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java b/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java index c2ad59d..54fb7b9 100644 --- a/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java +++ b/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java @@ -33,8 +33,8 @@ public final class CustomTypeSerializer { private CustomTypeSerializer(Schema schema) { Objects.nonNull(schema); - writer = AvroData.get().createDatumWriter(schema); - reader = AvroData.get().createDatumReader(schema); + this.writer = AvroData.get().createDatumWriter(schema); + this.reader = AvroData.get().createDatumReader(schema); } public byte[] toBytes(Patch value) throws IOException { @@ -49,10 +49,10 @@ public Patch toObject(byte[] bytes) throws IOException { } public static class Builder { - final List types = new ArrayList<>(); + protected final List types; public Builder() { - AvroData.get(); + this.types = new ArrayList<>(); } public Builder nullable() { diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/AvroData.java b/src/main/java/io/datanerds/avropatch/value/conversion/AvroData.java index 9531f05..5ee9aaa 100644 --- a/src/main/java/io/datanerds/avropatch/value/conversion/AvroData.java +++ b/src/main/java/io/datanerds/avropatch/value/conversion/AvroData.java @@ -8,10 +8,11 @@ public final class AvroData extends ReflectData { - public static List> CONVERTERS = - ImmutableList.of(new DateConversion(), new BigIntegerConversion(), new BigDecimalConversion(), - new UUIDConversion()); - + public static final List> CONVERTERS = ImmutableList.of( + new DateConversion(), + new BigIntegerConversion(), + new BigDecimalConversion(), + new UUIDConversion()); private static final AvroData INSTANCE = new AvroData(); private AvroData() { diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/BigDecimalConversion.java b/src/main/java/io/datanerds/avropatch/value/conversion/BigDecimalConversion.java index 9e07a17..9108300 100644 --- a/src/main/java/io/datanerds/avropatch/value/conversion/BigDecimalConversion.java +++ b/src/main/java/io/datanerds/avropatch/value/conversion/BigDecimalConversion.java @@ -12,7 +12,7 @@ import java.math.BigInteger; /** - * This class converts an {@link BigDecimal} value into am Avro {@link IndexedRecord} and back. It depends on + * This class converts a {@link BigDecimal} value into an Avro {@link IndexedRecord} and back. It depends on * {@link BigIntegerConversion} since it serializes the {@link BigDecimal}s unscaled value into a {@link BigInteger} and * its scale into an {@link Integer}. * Also, its logical type is statically registered in {@link LogicalTypes}, because it introduces a new custom type name diff --git a/src/test/java/io/datanerds/avropatch/PatchTest.java b/src/test/java/io/datanerds/avropatch/PatchTest.java index a9e8b1d..e0366da 100644 --- a/src/test/java/io/datanerds/avropatch/PatchTest.java +++ b/src/test/java/io/datanerds/avropatch/PatchTest.java @@ -88,7 +88,8 @@ public void serializesTest() throws IOException { @Test public void serializesBunchOfOperations() throws IOException { - Patch patch = new Patch(ImmutableList.of(new Add<>(Path.of("person", "name"), "John Doe"), + Patch patch = new Patch(ImmutableList.of( + new Add<>(Path.of("person", "name"), "John Doe"), new Copy(Path.parse("/person/firstName"), Path.parse("/person/lastName")), new Move(Path.parse("/person/firstName"), Path.parse("/person/lastName")), new Remove(Path.parse("/person/name")), @@ -103,7 +104,8 @@ public void serializesBunchOfOperations() throws IOException { public void serializesDefaultValueTypes() throws IOException { Date date = new Date(); UUID uuid = UUID.randomUUID(); - Patch patch = new Patch(ImmutableList.of(new Add<>(Path.of("some", "value"), "John Doe"), + Patch patch = new Patch(ImmutableList.of( + new Add<>(Path.of("some", "value"), "John Doe"), new Add<>(Path.of("some", "value"), 42), new Add<>(Path.of("some", "value"), 42L), new Add<>(Path.of("some", "value"), uuid), @@ -118,19 +120,28 @@ public void serializesDefaultValueTypes() throws IOException { } @Test - public void serializesArbitraryHeaders() throws IOException { - Patch patch = new Patch(Collections.EMPTY_LIST, ImmutableMap.of("header 1", UUID.randomUUID(), - "header 2", new Date(), "header 3", 1234L, "header 4", new BigDecimal("3214123453.123512345"))); - + public void serializesArbitraryHeadersWithoutOperations() throws IOException { + Patch patch = new Patch(Collections.EMPTY_LIST, + ImmutableMap.of( + "header 1", UUID.randomUUID(), + "header 2", new Date(), + "header 3", 1234L, + "header 4", new BigDecimal("3214123453.123512345"))); byte[] bytes = patch.toBytes(); assertThat(patch, is(equalTo(Patch.of(bytes)))); + } - patch = new Patch(ImmutableList.of(new Copy(Path.of("from", "here"), Path.of("to", "there")), - new Move(Path.of("from", "here"), Path.of("to", "there"))), - ImmutableMap.of("header 1", UUID.randomUUID(), - "header 2", new Date(), "header 3", 1234L, "header 4", new BigDecimal("3214123453.123512345"))); - - bytes = patch.toBytes(); + @Test + public void serializesArbitraryHeadersWithOperations() throws IOException { + Patch patch = new Patch(ImmutableList.of( + new Copy(Path.of("from", "here"), Path.of("to", "there")), + new Move(Path.of("from", "here"), Path.of("to", "there"))), + ImmutableMap.of( + "header 1", UUID.randomUUID(), + "header 2", new Date(), + "header 3", 1234L, + "header 4", new BigDecimal("3214123453.123512345"))); + byte[] bytes = patch.toBytes();; assertThat(patch, is(equalTo(Patch.of(bytes)))); } diff --git a/src/test/java/io/datanerds/avropatch/operation/matcher/OperationMatchers.java b/src/test/java/io/datanerds/avropatch/operation/matcher/OperationMatchers.java index dd6acca..0d544d4 100644 --- a/src/test/java/io/datanerds/avropatch/operation/matcher/OperationMatchers.java +++ b/src/test/java/io/datanerds/avropatch/operation/matcher/OperationMatchers.java @@ -8,6 +8,7 @@ import java.util.*; import java.util.function.Function; +import java.util.stream.Collectors; import static org.hamcrest.core.AllOf.allOf; @@ -45,12 +46,9 @@ public static Matcher> hasItems(T... items) { } public static Matcher> hasItems(List items) { - List>> all = new ArrayList>>(items.size()); - for (T element : items) { - all.add(hasItem(element)); - } - - return allOf(all); + return allOf(items.stream() + .map(item -> hasItem(item)) + .collect(Collectors.toList())); } public static Matcher> hasItemsOrdered(List expected) { diff --git a/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java b/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java index 091a1e0..3b9bc25 100644 --- a/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java +++ b/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java @@ -72,9 +72,7 @@ public void customTypeSerializer() throws InterruptedException { futures.stream().forEach(future -> { try { verify(PATCHES, future.get()); - } catch (InterruptedException ex) { - logger.warn("Serialization thread failed.", ex); - } catch (ExecutionException ex) { + } catch (InterruptedException | ExecutionException ex) { logger.warn("Serialization thread failed.", ex); } }); From 65adfdf1a70588ba50223e55cde262a261ee8edf Mon Sep 17 00:00:00 2001 From: Frank Wisniewski Date: Thu, 15 Dec 2016 10:45:36 +0100 Subject: [PATCH 10/19] Suppress unused warning on private Avro constructors. --- src/main/java/io/datanerds/avropatch/operation/Add.java | 4 ++-- src/main/java/io/datanerds/avropatch/operation/Copy.java | 4 ++-- src/main/java/io/datanerds/avropatch/operation/Move.java | 4 ++-- .../java/io/datanerds/avropatch/operation/Remove.java | 3 ++- .../java/io/datanerds/avropatch/operation/Replace.java | 4 ++-- src/main/java/io/datanerds/avropatch/operation/Test.java | 4 ++-- .../io/datanerds/avropatch/serialization/Bimmel.java | 9 ++++----- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/main/java/io/datanerds/avropatch/operation/Add.java b/src/main/java/io/datanerds/avropatch/operation/Add.java index 0c2f8fb..39fd8d0 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Add.java +++ b/src/main/java/io/datanerds/avropatch/operation/Add.java @@ -16,9 +16,9 @@ public final class Add implements Operation { @AvroSchema(DefaultSchema.VALUE) public final T value; + @SuppressWarnings("unused") private Add() { - this.path = null; - this.value = null; + this(null, null); } /** diff --git a/src/main/java/io/datanerds/avropatch/operation/Copy.java b/src/main/java/io/datanerds/avropatch/operation/Copy.java index 52bca07..f8e4b30 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Copy.java +++ b/src/main/java/io/datanerds/avropatch/operation/Copy.java @@ -13,9 +13,9 @@ public final class Copy implements Operation { public final Path from; public final Path path; + @SuppressWarnings("unused") private Copy() { - from = null; - path = null; + this(null, null); } /** diff --git a/src/main/java/io/datanerds/avropatch/operation/Move.java b/src/main/java/io/datanerds/avropatch/operation/Move.java index 210898c..135dc95 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Move.java +++ b/src/main/java/io/datanerds/avropatch/operation/Move.java @@ -13,9 +13,9 @@ public final class Move implements Operation { public final Path from; public final Path path; + @SuppressWarnings("unused") private Move() { - this.from = null; - this.path = null; + this(null, null); } /** diff --git a/src/main/java/io/datanerds/avropatch/operation/Remove.java b/src/main/java/io/datanerds/avropatch/operation/Remove.java index 9f17f16..424a16a 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Remove.java +++ b/src/main/java/io/datanerds/avropatch/operation/Remove.java @@ -12,8 +12,9 @@ public final class Remove implements Operation { public static final String op = "remove"; public final Path path; + @SuppressWarnings("unused") private Remove() { - this.path = null; + this(null); } /** diff --git a/src/main/java/io/datanerds/avropatch/operation/Replace.java b/src/main/java/io/datanerds/avropatch/operation/Replace.java index b96d5c6..019d52a 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Replace.java +++ b/src/main/java/io/datanerds/avropatch/operation/Replace.java @@ -15,9 +15,9 @@ public final class Replace implements Operation { @AvroSchema(DefaultSchema.VALUE) public final T value; + @SuppressWarnings("unused") private Replace() { - this.path = null; - this.value = null; + this(null, null); } /** diff --git a/src/main/java/io/datanerds/avropatch/operation/Test.java b/src/main/java/io/datanerds/avropatch/operation/Test.java index d306246..ff1ad22 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Test.java +++ b/src/main/java/io/datanerds/avropatch/operation/Test.java @@ -15,9 +15,9 @@ public final class Test implements Operation { @AvroSchema(DefaultSchema.VALUE) public final T value; + @SuppressWarnings("unused") private Test() { - this.path = null; - this.value = null; + this(null, null); } /** diff --git a/src/test/java/io/datanerds/avropatch/serialization/Bimmel.java b/src/test/java/io/datanerds/avropatch/serialization/Bimmel.java index 9bcc713..ba879a0 100644 --- a/src/test/java/io/datanerds/avropatch/serialization/Bimmel.java +++ b/src/test/java/io/datanerds/avropatch/serialization/Bimmel.java @@ -10,11 +10,9 @@ public class Bimmel { public final UUID id; public final Bommel bommel; + @SuppressWarnings("unused") private Bimmel() { - this.bommel = null; - this.name = null; - this.number = -42; - this.id = null; + this(null, -42, null, null); } public Bimmel(String name, int number, UUID id, Bommel bommel) { @@ -47,8 +45,9 @@ public int hashCode() { public static class Bommel { public final String name; + @SuppressWarnings("unused") private Bommel() { - this.name = null; + this(null); } public Bommel(String name) { From 41b78f4cde2cf63df721537f713e2785cd8b44fe Mon Sep 17 00:00:00 2001 From: Frank Wisniewski Date: Thu, 15 Dec 2016 13:34:55 +0100 Subject: [PATCH 11/19] PRR (@dhiller): Cleaned up test classes. --- .../io/datanerds/avropatch/operation/Add.java | 2 +- .../datanerds/avropatch/operation/Copy.java | 2 +- .../datanerds/avropatch/operation/Move.java | 2 +- .../datanerds/avropatch/operation/Remove.java | 2 +- .../avropatch/operation/Replace.java | 2 +- .../datanerds/avropatch/operation/Test.java | 2 +- .../operation/matcher/PatchMatcher.java | 43 ---------- .../operation/matcher/PatchMatcherTest.java | 67 +++++++++++++++ .../schema/OperationSerializationTester.java | 63 ++++++++++++++ .../avropatch/schema/OperationTypesTest.java | 85 ++++--------------- 10 files changed, 151 insertions(+), 119 deletions(-) create mode 100644 src/test/java/io/datanerds/avropatch/operation/matcher/PatchMatcherTest.java create mode 100644 src/test/java/io/datanerds/avropatch/schema/OperationSerializationTester.java diff --git a/src/main/java/io/datanerds/avropatch/operation/Add.java b/src/main/java/io/datanerds/avropatch/operation/Add.java index 39fd8d0..c8bc663 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Add.java +++ b/src/main/java/io/datanerds/avropatch/operation/Add.java @@ -16,7 +16,7 @@ public final class Add implements Operation { @AvroSchema(DefaultSchema.VALUE) public final T value; - @SuppressWarnings("unused") + @SuppressWarnings("unused") // no arg constructor needed by Avro private Add() { this(null, null); } diff --git a/src/main/java/io/datanerds/avropatch/operation/Copy.java b/src/main/java/io/datanerds/avropatch/operation/Copy.java index f8e4b30..2dc8e6f 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Copy.java +++ b/src/main/java/io/datanerds/avropatch/operation/Copy.java @@ -13,7 +13,7 @@ public final class Copy implements Operation { public final Path from; public final Path path; - @SuppressWarnings("unused") + @SuppressWarnings("unused") // no arg constructor needed by Avro private Copy() { this(null, null); } diff --git a/src/main/java/io/datanerds/avropatch/operation/Move.java b/src/main/java/io/datanerds/avropatch/operation/Move.java index 135dc95..10c2ddd 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Move.java +++ b/src/main/java/io/datanerds/avropatch/operation/Move.java @@ -13,7 +13,7 @@ public final class Move implements Operation { public final Path from; public final Path path; - @SuppressWarnings("unused") + @SuppressWarnings("unused") // no arg constructor needed by Avro private Move() { this(null, null); } diff --git a/src/main/java/io/datanerds/avropatch/operation/Remove.java b/src/main/java/io/datanerds/avropatch/operation/Remove.java index 424a16a..2d9fb43 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Remove.java +++ b/src/main/java/io/datanerds/avropatch/operation/Remove.java @@ -12,7 +12,7 @@ public final class Remove implements Operation { public static final String op = "remove"; public final Path path; - @SuppressWarnings("unused") + @SuppressWarnings("unused") // no arg constructor needed by Avro private Remove() { this(null); } diff --git a/src/main/java/io/datanerds/avropatch/operation/Replace.java b/src/main/java/io/datanerds/avropatch/operation/Replace.java index 019d52a..dfa9f39 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Replace.java +++ b/src/main/java/io/datanerds/avropatch/operation/Replace.java @@ -15,7 +15,7 @@ public final class Replace implements Operation { @AvroSchema(DefaultSchema.VALUE) public final T value; - @SuppressWarnings("unused") + @SuppressWarnings("unused") // no arg constructor needed by Avro private Replace() { this(null, null); } diff --git a/src/main/java/io/datanerds/avropatch/operation/Test.java b/src/main/java/io/datanerds/avropatch/operation/Test.java index ff1ad22..e22cdc3 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Test.java +++ b/src/main/java/io/datanerds/avropatch/operation/Test.java @@ -15,7 +15,7 @@ public final class Test implements Operation { @AvroSchema(DefaultSchema.VALUE) public final T value; - @SuppressWarnings("unused") + @SuppressWarnings("unused") // no arg constructor needed by Avro private Test() { this(null, null); } diff --git a/src/test/java/io/datanerds/avropatch/operation/matcher/PatchMatcher.java b/src/test/java/io/datanerds/avropatch/operation/matcher/PatchMatcher.java index 1f2e7f5..0538a21 100644 --- a/src/test/java/io/datanerds/avropatch/operation/matcher/PatchMatcher.java +++ b/src/test/java/io/datanerds/avropatch/operation/matcher/PatchMatcher.java @@ -1,25 +1,11 @@ package io.datanerds.avropatch.operation.matcher; import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import io.datanerds.avropatch.Patch; -import io.datanerds.avropatch.operation.Copy; -import io.datanerds.avropatch.operation.Move; -import io.datanerds.avropatch.operation.Operation; -import io.datanerds.avropatch.operation.Path; import org.hamcrest.CustomTypeSafeMatcher; import org.hamcrest.Matcher; -import org.junit.Test; - -import java.io.IOException; -import java.util.Collections; -import java.util.UUID; import static io.datanerds.avropatch.operation.matcher.OperationMatchers.hasItemsOrdered; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.AllOf.allOf; public class PatchMatcher { @@ -55,33 +41,4 @@ protected boolean matchesSafely(Patch item) { } }; } - - @Test - public void headers() throws IOException { - UUID uuid = UUID.randomUUID(); - Patch patch = new Patch(Collections.EMPTY_LIST, ImmutableMap.of("header 1", uuid)); - Patch patch2 = new Patch(Collections.EMPTY_LIST, ImmutableMap.of("header 1", uuid)); - Patch patch3 = new Patch(Collections.EMPTY_LIST, ImmutableMap.of("header 2", uuid)); - Patch patch4 = new Patch(Collections.EMPTY_LIST, ImmutableMap.of("header 1", UUID.randomUUID())); - - assertThat(patch, is(equalTo(patch2))); - assertThat(patch2, is(equalTo(patch))); - assertThat(patch, is(not(equalTo(patch3)))); - assertThat(patch, is(not(equalTo(patch4)))); - } - - @Test - public void operations() throws IOException { - Operation operation1 = new Copy(Path.of("from", "here"), Path.of("to", "there")); - Operation operation2 = new Move(Path.of("from", "here"), Path.of("to", "there")); - Patch patch1 = new Patch(ImmutableList.of(operation1, operation2), Collections.EMPTY_MAP); - Patch patch2 = new Patch(ImmutableList.of(operation1, operation2), Collections.EMPTY_MAP); - Patch patch3 = new Patch(ImmutableList.of(operation1), Collections.EMPTY_MAP); - Patch patch4 = new Patch(ImmutableList.of(operation2, operation1), Collections.EMPTY_MAP); - - assertThat(patch1, is(equalTo(patch2))); - assertThat(patch2, is(equalTo(patch1))); - assertThat(patch1, is(not(equalTo(patch3)))); - assertThat(patch1, is(not(equalTo(patch4)))); - } } diff --git a/src/test/java/io/datanerds/avropatch/operation/matcher/PatchMatcherTest.java b/src/test/java/io/datanerds/avropatch/operation/matcher/PatchMatcherTest.java new file mode 100644 index 0000000..119f362 --- /dev/null +++ b/src/test/java/io/datanerds/avropatch/operation/matcher/PatchMatcherTest.java @@ -0,0 +1,67 @@ +package io.datanerds.avropatch.operation.matcher; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.datanerds.avropatch.Patch; +import io.datanerds.avropatch.operation.Copy; +import io.datanerds.avropatch.operation.Move; +import io.datanerds.avropatch.operation.Operation; +import io.datanerds.avropatch.operation.Path; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.Collections; +import java.util.UUID; + +import static io.datanerds.avropatch.operation.matcher.PatchMatcher.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; + +@RunWith(Enclosed.class) +public class PatchMatcherTest { + + public static class Headers { + private final UUID uuid = UUID.randomUUID(); + private final Patch patchWithHeader = new Patch(Collections.EMPTY_LIST, ImmutableMap.of("header", uuid)); + private final Patch patchWithHeaderSibling = new Patch(Collections.EMPTY_LIST, ImmutableMap.of("header", uuid)); + private final Patch differentHeader = new Patch(Collections.EMPTY_LIST, ImmutableMap.of("header 2", uuid)); + private final Patch differentHeaderValue = new Patch(Collections.EMPTY_LIST, + ImmutableMap.of("header", UUID.randomUUID())); + + @Test + public void equality() throws IOException { + assertThat(patchWithHeader, is(equalTo(patchWithHeaderSibling))); + assertThat(patchWithHeaderSibling, is(equalTo(patchWithHeader))); + } + + @Test + public void inequality() throws IOException { + assertThat(patchWithHeader, is(not(equalTo(differentHeader)))); + assertThat(patchWithHeader, is(not(equalTo(differentHeaderValue)))); + } + } + + public static class Operations { + private final Operation copy = new Copy(Path.of("from", "here"), Path.of("to", "there")); + private final Operation move = new Move(Path.of("from", "here"), Path.of("to", "there")); + private final Patch copyMovePatch = new Patch(ImmutableList.of(copy, move), Collections.EMPTY_MAP); + private final Patch copyMovePatchSibling = new Patch(ImmutableList.of(copy, move), Collections.EMPTY_MAP); + private final Patch withoutMoveOperation = new Patch(ImmutableList.of(copy), Collections.EMPTY_MAP); + private final Patch reverseOperationOrder = new Patch(ImmutableList.of(move, copy), Collections.EMPTY_MAP); + + @Test + public void equality() throws IOException { + assertThat(copyMovePatch, is(equalTo(copyMovePatchSibling))); + assertThat(copyMovePatchSibling, is(equalTo(copyMovePatch))); + } + + @Test + public void inequality() throws IOException { + assertThat(copyMovePatch, is(not(equalTo(withoutMoveOperation)))); + assertThat(copyMovePatch, is(not(equalTo(reverseOperationOrder)))); + } + } +} diff --git a/src/test/java/io/datanerds/avropatch/schema/OperationSerializationTester.java b/src/test/java/io/datanerds/avropatch/schema/OperationSerializationTester.java new file mode 100644 index 0000000..dd8e7e8 --- /dev/null +++ b/src/test/java/io/datanerds/avropatch/schema/OperationSerializationTester.java @@ -0,0 +1,63 @@ +package io.datanerds.avropatch.schema; + +import com.google.common.collect.ImmutableMap; +import io.datanerds.avropatch.operation.*; +import org.apache.avro.Schema; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.*; +import java.util.function.Function; + +import static io.datanerds.avropatch.operation.matcher.OperationMatchers.equalTo; +import static io.datanerds.avropatch.schema.CustomTypes.VALUE_TYPE_UNION; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +public class OperationSerializationTester extends SerializationTester { + + private static final Map, Schema> schemata = new ImmutableMap.Builder() + .put(Add.class, OperationTypes.Add.create(VALUE_TYPE_UNION)) + .put(Copy.class, OperationTypes.Copy.create()) + .put(Move.class, OperationTypes.Move.create()) + .put(Remove.class, OperationTypes.Remove.create()) + .put(Replace.class, OperationTypes.Replace.create(VALUE_TYPE_UNION)) + .put(io.datanerds.avropatch.operation.Test.class, OperationTypes.Test.create(VALUE_TYPE_UNION)) + .build(); + + public OperationSerializationTester(Class clazz) { + super(schemata.get(clazz)); + } + + public static List createSomeOperations(Function operationFunction) { + List operations = new ArrayList<>(); + operations.add(createOperation(operationFunction, "hello world")); + operations.add(createOperation(operationFunction, 42)); + operations.add(createOperation(operationFunction, 42L)); + operations.add(createOperation(operationFunction, 123.456d)); + operations.add(createOperation(operationFunction, 123.456f)); + operations.add(createOperation(operationFunction, new BigInteger("8364789684563949576378945698056348956"))); + operations.add(createOperation(operationFunction, new BigDecimal("956740578902345.56734895627895"))); + operations.add(createOperation(operationFunction, UUID.randomUUID())); + operations.add(createOperation(operationFunction, new Date())); + + return operations; + } + + @Override + public void reserializeAndAssert(Operation value) throws IOException { + Operation operation = reserialize(value); + assertThat(value, is(equalTo(operation))); + } + + public void reserializeAndAssert(List operations) throws IOException { + for (Operation operation : operations) { + reserializeAndAssert(operation); + } + } + + private static Operation createOperation(Function operationFunction, T value) { + return operationFunction.apply(value); + } +} \ No newline at end of file diff --git a/src/test/java/io/datanerds/avropatch/schema/OperationTypesTest.java b/src/test/java/io/datanerds/avropatch/schema/OperationTypesTest.java index cd7f7e7..c319a39 100644 --- a/src/test/java/io/datanerds/avropatch/schema/OperationTypesTest.java +++ b/src/test/java/io/datanerds/avropatch/schema/OperationTypesTest.java @@ -1,104 +1,49 @@ package io.datanerds.avropatch.schema; -import com.google.common.collect.ImmutableMap; import io.datanerds.avropatch.operation.*; -import org.apache.avro.Schema; import org.junit.Test; import java.io.IOException; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.util.*; -import java.util.function.Function; -import static io.datanerds.avropatch.operation.matcher.OperationMatchers.equalTo; -import static io.datanerds.avropatch.schema.CustomTypes.VALUE_TYPE_UNION; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; +import static io.datanerds.avropatch.schema.OperationSerializationTester.createSomeOperations; public class OperationTypesTest { @Test public void add() throws IOException { - OperationTester tester = new OperationTester(Add.class); - List adds = tester.createSomeOperations(value -> new Add<>(Path.of("hello", "world"), value)); - for (Operation add : adds) { - assertThat(add, is(equalTo(tester.reserialize(add)))); - } + new OperationSerializationTester(Add.class) + .reserializeAndAssert(createSomeOperations(value -> new Add<>(Path.of("hello", "world"), value))); } @Test public void copy() throws IOException { - SerializationTester tester = new OperationTester(Copy.class); - Copy copy = new Copy(Path.of("from", "here"), Path.of("to", "there")); - assertThat(copy, is(equalTo(tester.reserialize(copy)))); + new OperationSerializationTester(Copy.class) + .reserializeAndAssert(new Copy(Path.of("from", "here"), Path.of("to", "there"))); } @Test public void move() throws IOException { - SerializationTester tester = new OperationTester(Move.class); - Move copy = new Move(Path.of("from", "here"), Path.of("to", "there")); - assertThat(copy, is(equalTo(tester.reserialize(copy)))); + new OperationSerializationTester(Move.class) + .reserializeAndAssert(new Move(Path.of("from", "here"), Path.of("to", "there"))); } @Test public void remove() throws IOException { - SerializationTester tester = new OperationTester(Remove.class); - Remove remove = new Remove(Path.of("from", "here")); - assertThat(remove, is(equalTo(tester.reserialize(remove)))); + new OperationSerializationTester(Remove.class) + .reserializeAndAssert(new Remove(Path.of("from", "here"))); } @Test public void replace() throws IOException { - OperationTester tester = new OperationTester(Replace.class); - List replaces = tester.createSomeOperations(value -> new Replace<>(Path.of("hello"), value)); - for (Operation replace : replaces) { - assertThat(replace, is(equalTo(tester.reserialize(replace)))); - } + new OperationSerializationTester(Replace.class).reserializeAndAssert( + createSomeOperations(value -> new Replace<>(Path.of("hello"), value))); } @Test public void test() throws IOException { - OperationTester tester = new OperationTester(io.datanerds.avropatch.operation.Test.class); - List tests = tester.createSomeOperations( - value -> new io.datanerds.avropatch.operation.Test<>(Path.of("hello", "world"), value)); - for (Operation test : tests) { - assertThat(test, is(equalTo(tester.reserialize(test)))); - } - } - - static class OperationTester extends SerializationTester { - - private static final Map, Schema> schemata = new ImmutableMap.Builder() - .put(Add.class, OperationTypes.Add.create(VALUE_TYPE_UNION)) - .put(Copy.class, OperationTypes.Copy.create()) - .put(Move.class, OperationTypes.Move.create()) - .put(Remove.class, OperationTypes.Remove.create()) - .put(Replace.class, OperationTypes.Replace.create(VALUE_TYPE_UNION)) - .put(io.datanerds.avropatch.operation.Test.class, OperationTypes.Test.create(VALUE_TYPE_UNION)) - .build(); - - public OperationTester(Class clazz) { - super(schemata.get(clazz)); - } - - public static List createSomeOperations(Function operationFunction) { - List operations = new ArrayList<>(); - operations.add(createOperation(operationFunction, "hello world")); - operations.add(createOperation(operationFunction, 42)); - operations.add(createOperation(operationFunction, 42L)); - operations.add(createOperation(operationFunction, 123.456d)); - operations.add(createOperation(operationFunction, 123.456f)); - operations.add(createOperation(operationFunction, new BigInteger("8364789684563949576378945698056348956"))); - operations.add(createOperation(operationFunction, new BigDecimal("956740578902345.56734895627895"))); - operations.add(createOperation(operationFunction, UUID.randomUUID())); - operations.add(createOperation(operationFunction, new Date())); - - return operations; - } - - private static Operation createOperation(Function operationFunction, T value) { - return operationFunction.apply(value); - } + new OperationSerializationTester(io.datanerds.avropatch.operation.Test.class) + .reserializeAndAssert( + createSomeOperations( + value -> new io.datanerds.avropatch.operation.Test<>(Path.of("hello", "world"), value))); } } \ No newline at end of file From f741adfa2c1705a2aa05a08e531e8c1ec2ad8bfb Mon Sep 17 00:00:00 2001 From: Frank Wisniewski Date: Thu, 15 Dec 2016 15:38:25 +0100 Subject: [PATCH 12/19] PRR (@dhiller): Incorporated some suggestions. --- .../java/io/datanerds/avropatch/Patch.java | 3 +- .../avropatch/value/conversion/AvroData.java | 4 +- .../operation/matcher/OperationMatchers.java | 8 ++-- .../schema/OperationSerializationTester.java | 4 +- .../serialization/ConcurrencyTest.java | 2 +- .../serialization/OperationGenerator.java | 43 +++++++++---------- 6 files changed, 33 insertions(+), 31 deletions(-) diff --git a/src/main/java/io/datanerds/avropatch/Patch.java b/src/main/java/io/datanerds/avropatch/Patch.java index af09860..ad2a727 100644 --- a/src/main/java/io/datanerds/avropatch/Patch.java +++ b/src/main/java/io/datanerds/avropatch/Patch.java @@ -29,8 +29,7 @@ public Patch() { } public Patch(List operations) { - this.operations = new ArrayList<>(operations); - this.headers = new HashMap<>(); + this(new ArrayList<>(operations), new HashMap<>()); } public Patch(List operations, Map headers) { diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/AvroData.java b/src/main/java/io/datanerds/avropatch/value/conversion/AvroData.java index 5ee9aaa..cd9f3a2 100644 --- a/src/main/java/io/datanerds/avropatch/value/conversion/AvroData.java +++ b/src/main/java/io/datanerds/avropatch/value/conversion/AvroData.java @@ -12,7 +12,9 @@ public final class AvroData extends ReflectData { new DateConversion(), new BigIntegerConversion(), new BigDecimalConversion(), - new UUIDConversion()); + new UUIDConversion() + ); + private static final AvroData INSTANCE = new AvroData(); private AvroData() { diff --git a/src/test/java/io/datanerds/avropatch/operation/matcher/OperationMatchers.java b/src/test/java/io/datanerds/avropatch/operation/matcher/OperationMatchers.java index 0d544d4..d473508 100644 --- a/src/test/java/io/datanerds/avropatch/operation/matcher/OperationMatchers.java +++ b/src/test/java/io/datanerds/avropatch/operation/matcher/OperationMatchers.java @@ -46,9 +46,11 @@ public static Matcher> hasItems(T... items) { } public static Matcher> hasItems(List items) { - return allOf(items.stream() - .map(item -> hasItem(item)) - .collect(Collectors.toList())); + return allOf( + items.stream() + .map(OperationMatchers::hasItem) + .collect(Collectors.toList()) + ); } public static Matcher> hasItemsOrdered(List expected) { diff --git a/src/test/java/io/datanerds/avropatch/schema/OperationSerializationTester.java b/src/test/java/io/datanerds/avropatch/schema/OperationSerializationTester.java index dd8e7e8..fa67508 100644 --- a/src/test/java/io/datanerds/avropatch/schema/OperationSerializationTester.java +++ b/src/test/java/io/datanerds/avropatch/schema/OperationSerializationTester.java @@ -17,7 +17,7 @@ public class OperationSerializationTester extends SerializationTester { - private static final Map, Schema> schemata = new ImmutableMap.Builder() + private static final Map, Schema> SCHEMATA = new ImmutableMap.Builder() .put(Add.class, OperationTypes.Add.create(VALUE_TYPE_UNION)) .put(Copy.class, OperationTypes.Copy.create()) .put(Move.class, OperationTypes.Move.create()) @@ -27,7 +27,7 @@ public class OperationSerializationTester extends SerializationTester .build(); public OperationSerializationTester(Class clazz) { - super(schemata.get(clazz)); + super(SCHEMATA.get(clazz)); } public static List createSomeOperations(Function operationFunction) { diff --git a/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java b/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java index 3b9bc25..ead1229 100644 --- a/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java +++ b/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java @@ -84,7 +84,7 @@ private static List generateData() { for (int i = 0; i < NUMBER_OF_PATCHES; i++) { Patch patch = new Patch(); for (int j = 0; j < random.nextInt(MAX_PATCH_SIZE); j++) { - patch.addOperation(OperationGenerator.generate()); + patch.addOperation(OperationGenerator.generateOperation()); } patches.add(patch); } diff --git a/src/test/java/io/datanerds/avropatch/serialization/OperationGenerator.java b/src/test/java/io/datanerds/avropatch/serialization/OperationGenerator.java index 6311864..a7a1900 100644 --- a/src/test/java/io/datanerds/avropatch/serialization/OperationGenerator.java +++ b/src/test/java/io/datanerds/avropatch/serialization/OperationGenerator.java @@ -4,22 +4,21 @@ import java.math.BigDecimal; import java.math.BigInteger; -import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Random; import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.IntStream; enum OperationGenerator { - @SuppressWarnings("unchecked") - ADD(() -> new Add(generatePath(), generateValue())), + ADD(() -> new Add<>(generatePath(), generateValue())), COPY(() -> new Copy(generatePath(), generatePath())), MOVE(() -> new Move(generatePath(), generatePath())), REMOVE(() -> new Remove(generatePath())), REPLACE(() -> new Replace<>(generatePath(), generateValue())), - @SuppressWarnings("unchecked") - TEST(() -> new io.datanerds.avropatch.operation.Test(generatePath(), generateValue())); + TEST(() -> new io.datanerds.avropatch.operation.Test<>(generatePath(), generateValue())); private static final Random random = new Random(); private static final int MAX_LIST_SIZE = 50; @@ -29,9 +28,8 @@ enum OperationGenerator { this.operationSupplier = operationSupplier; } - public static Operation generate() { - OperationGenerator operationType = values()[random.nextInt(values().length)]; - return operationType.generateOperation(); + public static Operation generateOperation() { + return values()[random.nextInt(values().length)].getOperation(); } private static Object generateValue() { return ValueType.generateObject(); } @@ -40,7 +38,7 @@ private static Path generatePath() { return Path.of("hello", "world"); } - private Operation generateOperation() { + private Operation getOperation() { return operationSupplier.get(); } @@ -65,32 +63,33 @@ enum ValueType { } public static Object generateObject() { - ValueType type = ValueType.values()[random.nextInt(ValueType.values().length)]; if (random.nextBoolean()) { - return type.generate(); - } else { - return generateList(type); + return generateList(randomValueType()); } - } - - private static List generateList(ValueType type) { - List data = new ArrayList<>(); - for (int i = 0; i < MAX_LIST_SIZE; i++) { - data.add(type.generate()); - } - return data; + return randomValueType().generate(); } private Object generate() { return valueSupplier.get(); } + private static ValueType randomValueType() { + return values()[random.nextInt(ValueType.values().length)]; + } + + private static List generateList(ValueType type) { + return IntStream + .range(0, MAX_LIST_SIZE) + .mapToObj(i -> type.generate()) + .collect(Collectors.toList()); + } + private static String randomString() { return java.util.UUID.randomUUID().toString(); } private static BigDecimal randomBigDecimal() { - BigInteger unscaledValue= new BigInteger(random.nextInt(256), random); + BigInteger unscaledValue = new BigInteger(random.nextInt(256), random); int numberOfDigits = getDigitCount(unscaledValue); int scale = random.nextInt(numberOfDigits + 1); return new BigDecimal(unscaledValue, scale); From e29f313f6ef4c7f94e37ee9042da2079410c29a5 Mon Sep 17 00:00:00 2001 From: Frank Wisniewski Date: Mon, 19 Dec 2016 11:35:30 +0100 Subject: [PATCH 13/19] PRR @abhishekjain88 & @dguenms --- .../java/io/datanerds/avropatch/Patch.java | 11 +++-- .../avropatch/schema/CustomTypes.java | 22 ++++++---- .../avropatch/schema/OperationTypes.java | 4 +- .../avropatch/value/conversion/AvroData.java | 11 ++++- .../io/datanerds/avropatch/PatchTest.java | 21 ++++------ .../serialization/ConcurrencyTest.java | 42 ++++++++++--------- .../serialization/OperationGenerator.java | 2 +- 7 files changed, 64 insertions(+), 49 deletions(-) diff --git a/src/main/java/io/datanerds/avropatch/Patch.java b/src/main/java/io/datanerds/avropatch/Patch.java index ad2a727..27d64c5 100644 --- a/src/main/java/io/datanerds/avropatch/Patch.java +++ b/src/main/java/io/datanerds/avropatch/Patch.java @@ -7,6 +7,7 @@ import org.apache.avro.io.*; import org.apache.avro.reflect.AvroSchema; +import javax.annotation.concurrent.ThreadSafe; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.*; @@ -16,6 +17,7 @@ * * @see https://tools.ietf.org/html/rfc6902 */ +@ThreadSafe public class Patch { private static final PatchSerializer SERIALIZER = new PatchSerializer(); @@ -24,12 +26,13 @@ public class Patch { private final Map headers; private final List operations; - public Patch() { + @SuppressWarnings("unused") // no arg constructor needed by Avro + private Patch() { this(new ArrayList<>(), new HashMap<>()); } public Patch(List operations) { - this(new ArrayList<>(operations), new HashMap<>()); + this(operations, Collections.EMPTY_MAP); } public Patch(List operations, Map headers) { @@ -37,10 +40,6 @@ public Patch(List operations, Map headers) { this.headers = new HashMap<>(headers); } - public boolean addOperation(Operation operation) { - return operations.add(operation); - } - public List getOperations() { return Collections.unmodifiableList(operations); } diff --git a/src/main/java/io/datanerds/avropatch/schema/CustomTypes.java b/src/main/java/io/datanerds/avropatch/schema/CustomTypes.java index 19f8476..bc8804e 100644 --- a/src/main/java/io/datanerds/avropatch/schema/CustomTypes.java +++ b/src/main/java/io/datanerds/avropatch/schema/CustomTypes.java @@ -42,11 +42,14 @@ static Schema create(Schema valueSchema) { interface BigDecimalType { String NAME = "big-decimal"; - String DOC = "BigDecimal value represented via it's scale and unscaled value."; + String DOC = "BigDecimal value represented via its scale and unscaled value."; LogicalType LOGICAL_TYPE = new LogicalType(NAME); - Schema RECORD = SchemaBuilder.record("decimal").doc(DOC).fields() - .name("unscaledValue").type(BigIntegerType.SCHEMA).noDefault() - .name("scale").type(Schema.create(Type.INT)).noDefault() + Schema RECORD = SchemaBuilder + .record("decimal") + .doc(DOC) + .fields() + .name("unscaledValue").type(BigIntegerType.SCHEMA).noDefault() + .name("scale").type(Schema.create(Type.INT)).noDefault() .endRecord(); Schema SCHEMA = LOGICAL_TYPE.addToSchema(RECORD); @@ -56,7 +59,6 @@ interface BigIntegerType { String NAME = "big-integer"; LogicalType LOGICAL_TYPE = new LogicalType(NAME); Schema SCHEMA = LOGICAL_TYPE.addToSchema(Schema.create(Type.BYTES)); - } interface DateType { @@ -69,7 +71,7 @@ interface DateType { interface UuidType { String NAME = LogicalTypes.uuid().getName(); - String DOC = "UUID serialized via two long values: It's most significant and least significant 64 bits."; + String DOC = "UUID serialized via two long values: Its most significant and least significant 64 bits."; int SIZE = 2 * Long.BYTES; Schema SCHEMA = LogicalTypes.uuid().addToSchema(Schema.createFixed(NAME, DOC, null, SIZE)); } @@ -77,8 +79,12 @@ interface UuidType { interface Path { String NAME = io.datanerds.avropatch.operation.Path.class.getSimpleName(); String DOC = "JSON Path serialized as String array holding its parts."; - Schema SCHEMA = SchemaBuilder.record(NAME).doc(DOC).namespace(NAMESPACE).fields() - .name("parts").type(Schema.createArray(Schema.create(Type.STRING))).noDefault() + Schema SCHEMA = SchemaBuilder + .record(NAME) + .doc(DOC) + .namespace(NAMESPACE) + .fields() + .name("parts").type(Schema.createArray(Schema.create(Type.STRING))).noDefault() .endRecord(); } diff --git a/src/main/java/io/datanerds/avropatch/schema/OperationTypes.java b/src/main/java/io/datanerds/avropatch/schema/OperationTypes.java index 0f205ff..b1d3610 100644 --- a/src/main/java/io/datanerds/avropatch/schema/OperationTypes.java +++ b/src/main/java/io/datanerds/avropatch/schema/OperationTypes.java @@ -68,7 +68,7 @@ interface Replace { static Schema create(Schema valueSchema) { Field path = new Field("path", Path.SCHEMA, "Path pointing out where the JSON value should be replaced", NO_DEFAULT); - Field value = new Field("value", valueSchema, "Actual value to replaced in patched object", NO_DEFAULT); + Field value = new Field("value", valueSchema, "Actual value to be replaced in patched object", NO_DEFAULT); return createRecord(NAME, DOC, NAMESPACE, false, Arrays.asList(path, value)); } } @@ -80,7 +80,7 @@ interface Test { static Schema create(Schema valueSchema) { Field path = new Field("path", Path.SCHEMA, "Path pointing out which JSON value should be tested against", NO_DEFAULT); - Field value = new Field("value", valueSchema, "Actual value to test< against", NO_DEFAULT); + Field value = new Field("value", valueSchema, "Actual value to test against", NO_DEFAULT); return createRecord(NAME, DOC, NAMESPACE, false, Arrays.asList(path, value)); } } diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/AvroData.java b/src/main/java/io/datanerds/avropatch/value/conversion/AvroData.java index cd9f3a2..78ac6ce 100644 --- a/src/main/java/io/datanerds/avropatch/value/conversion/AvroData.java +++ b/src/main/java/io/datanerds/avropatch/value/conversion/AvroData.java @@ -8,7 +8,7 @@ public final class AvroData extends ReflectData { - public static final List> CONVERTERS = ImmutableList.of( + private static final List> CONVERTERS = ImmutableList.of( new DateConversion(), new BigIntegerConversion(), new BigDecimalConversion(), @@ -21,6 +21,15 @@ private AvroData() { CONVERTERS.forEach(this::addLogicalTypeConversion); } + /** + * Hides {@link ReflectData#get()} and returns {@link ReflectData} subclass initialized with custom converters. + * @see ReflectData#get() + * @see DateConversion + * @see BigIntegerConversion + * @see BigDecimalConversion + * @see UUIDConversion + * @return singleton instance + */ public static AvroData get() { return INSTANCE; } diff --git a/src/test/java/io/datanerds/avropatch/PatchTest.java b/src/test/java/io/datanerds/avropatch/PatchTest.java index e0366da..79e1460 100644 --- a/src/test/java/io/datanerds/avropatch/PatchTest.java +++ b/src/test/java/io/datanerds/avropatch/PatchTest.java @@ -23,8 +23,7 @@ public class PatchTest { @Test public void serializesAdd() throws IOException { - Patch patch = new Patch(); - patch.addOperation(new Add<>(Path.of("person", "name"), "John Doe")); + Patch patch = patchOf(new Add<>(Path.of("person", "name"), "John Doe")); byte[] bytes = patch.toBytes(); List operations = Patch.of(bytes).getOperations(); @@ -34,8 +33,7 @@ public void serializesAdd() throws IOException { @Test public void serializesCopy() throws IOException { - Patch patch = new Patch(); - patch.addOperation(new Copy(Path.parse("/person/firstName"), Path.parse("/person/lastName"))); + Patch patch = patchOf(new Copy(Path.parse("/person/firstName"), Path.parse("/person/lastName"))); byte[] bytes = patch.toBytes(); List operations = Patch.of(bytes).getOperations(); @@ -45,8 +43,7 @@ public void serializesCopy() throws IOException { @Test public void serializesMove() throws IOException { - Patch patch = new Patch(); - patch.addOperation(new Move(Path.parse("/person/firstName"), Path.parse("/person/lastName"))); + Patch patch = patchOf(new Move(Path.parse("/person/firstName"), Path.parse("/person/lastName"))); byte[] bytes = patch.toBytes(); List operations = Patch.of(bytes).getOperations(); @@ -56,8 +53,7 @@ public void serializesMove() throws IOException { @Test public void serializesRemove() throws IOException { - Patch patch = new Patch(); - patch.addOperation(new Remove(Path.parse("/person/name"))); + Patch patch = patchOf(new Remove(Path.parse("/person/name"))); byte[] bytes = patch.toBytes(); List operations = Patch.of(bytes).getOperations(); @@ -67,8 +63,7 @@ public void serializesRemove() throws IOException { @Test public void serializesReplace() throws IOException { - Patch patch = new Patch(); - patch.addOperation(new Replace(Path.parse("/person/number"), 42)); + Patch patch = patchOf(new Replace(Path.parse("/person/number"), 42)); byte[] bytes = patch.toBytes(); List operations = Patch.of(bytes).getOperations(); @@ -77,8 +72,7 @@ public void serializesReplace() throws IOException { @Test public void serializesTest() throws IOException { - Patch patch = new Patch(); - patch.addOperation(new io.datanerds.avropatch.operation.Test(Path.parse("/person/number"), 42L)); + Patch patch = patchOf(new io.datanerds.avropatch.operation.Test(Path.parse("/person/number"), 42L)); byte[] bytes = patch.toBytes(); List operations = Patch.of(bytes).getOperations(); @@ -145,4 +139,7 @@ public void serializesArbitraryHeadersWithOperations() throws IOException { assertThat(patch, is(equalTo(Patch.of(bytes)))); } + private Patch patchOf(T operation) { + return new Patch(ImmutableList.of(operation)); + } } \ No newline at end of file diff --git a/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java b/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java index ead1229..418204c 100644 --- a/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java +++ b/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java @@ -2,6 +2,7 @@ import com.google.common.collect.ImmutableList; import io.datanerds.avropatch.Patch; +import io.datanerds.avropatch.operation.Operation; import org.hamcrest.CoreMatchers; import org.junit.Test; import org.slf4j.Logger; @@ -44,22 +45,7 @@ public void customTypeSerializer() throws InterruptedException { List>> futures = new ArrayList<>(); for (int i = 0; i < NUMBER_OF_THREADS; i++) { - futures.add(executorService.submit(() -> { - List output = new ArrayList<>(); - List input = ImmutableList.copyOf(PATCHES); - try { - startThreadsLatch.await(); - for (Patch patch : input) { - byte[] bytes = SERIALIZER.toBytes(patch); - output.add(SERIALIZER.toObject(bytes)); - } - } catch (Exception ex) { - logger.warn("Serialization thread failed.", ex); - } finally { - threadsFinishedLatch.countDown(); - } - return output; - })); + futures.add(executorService.submit(() -> reserializePatches(startThreadsLatch, threadsFinishedLatch))); } logger.info("Releasing {} serialization threads", NUMBER_OF_THREADS); @@ -78,15 +64,33 @@ public void customTypeSerializer() throws InterruptedException { }); } + private List reserializePatches(CountDownLatch startThreadsLatch, CountDownLatch threadsFinishedLatch) { + List output = new ArrayList<>(); + List input = ImmutableList.copyOf(PATCHES); + try { + startThreadsLatch.await(); + for (Patch patch : input) { + byte[] bytes = SERIALIZER.toBytes(patch); + output.add(SERIALIZER.toObject(bytes)); + } + } catch (Exception ex) { + logger.warn("Serialization thread failed.", ex); + } finally { + threadsFinishedLatch.countDown(); + } + return output; + + } + private static List generateData() { Random random = new Random(); List patches = new ArrayList<>(); for (int i = 0; i < NUMBER_OF_PATCHES; i++) { - Patch patch = new Patch(); + List operations = new ArrayList<>(); for (int j = 0; j < random.nextInt(MAX_PATCH_SIZE); j++) { - patch.addOperation(OperationGenerator.generateOperation()); + operations.add(OperationGenerator.randomOperation()); } - patches.add(patch); + patches.add(new Patch(operations)); } return patches; } diff --git a/src/test/java/io/datanerds/avropatch/serialization/OperationGenerator.java b/src/test/java/io/datanerds/avropatch/serialization/OperationGenerator.java index a7a1900..abd1131 100644 --- a/src/test/java/io/datanerds/avropatch/serialization/OperationGenerator.java +++ b/src/test/java/io/datanerds/avropatch/serialization/OperationGenerator.java @@ -28,7 +28,7 @@ enum OperationGenerator { this.operationSupplier = operationSupplier; } - public static Operation generateOperation() { + public static Operation randomOperation() { return values()[random.nextInt(values().length)].getOperation(); } From 4c0754ba64a4768b2611f6ea72997939333ceefb Mon Sep 17 00:00:00 2001 From: Frank Wisniewski Date: Thu, 5 Jan 2017 13:49:09 -0800 Subject: [PATCH 14/19] Removed operation literal for now. --- .../datanerds/avropatch/exception/AvroPatchException.java | 4 ++-- src/main/java/io/datanerds/avropatch/operation/Add.java | 5 +---- src/main/java/io/datanerds/avropatch/operation/Copy.java | 6 +----- src/main/java/io/datanerds/avropatch/operation/Move.java | 6 +----- src/main/java/io/datanerds/avropatch/operation/Remove.java | 6 +----- src/main/java/io/datanerds/avropatch/operation/Replace.java | 5 +---- src/main/java/io/datanerds/avropatch/operation/Test.java | 5 +---- 7 files changed, 8 insertions(+), 29 deletions(-) diff --git a/src/main/java/io/datanerds/avropatch/exception/AvroPatchException.java b/src/main/java/io/datanerds/avropatch/exception/AvroPatchException.java index 051d773..83a0050 100644 --- a/src/main/java/io/datanerds/avropatch/exception/AvroPatchException.java +++ b/src/main/java/io/datanerds/avropatch/exception/AvroPatchException.java @@ -1,8 +1,8 @@ package io.datanerds.avropatch.exception; -public class AvroPatchException extends RuntimeException { +public abstract class AvroPatchException extends RuntimeException { - public AvroPatchException(String message) { + AvroPatchException(String message) { super(message); } } diff --git a/src/main/java/io/datanerds/avropatch/operation/Add.java b/src/main/java/io/datanerds/avropatch/operation/Add.java index c8bc663..484155b 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Add.java +++ b/src/main/java/io/datanerds/avropatch/operation/Add.java @@ -1,6 +1,5 @@ package io.datanerds.avropatch.operation; -import org.apache.avro.reflect.AvroIgnore; import org.apache.avro.reflect.AvroSchema; /** @@ -10,13 +9,11 @@ * @param DefaultSchema type of add operation */ public final class Add implements Operation { - @AvroIgnore - public static final String op = "add"; public final Path path; @AvroSchema(DefaultSchema.VALUE) public final T value; - @SuppressWarnings("unused") // no arg constructor needed by Avro + @SuppressWarnings("unused") // no-arg constructor needed by Avro private Add() { this(null, null); } diff --git a/src/main/java/io/datanerds/avropatch/operation/Copy.java b/src/main/java/io/datanerds/avropatch/operation/Copy.java index 2dc8e6f..80c7982 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Copy.java +++ b/src/main/java/io/datanerds/avropatch/operation/Copy.java @@ -1,19 +1,15 @@ package io.datanerds.avropatch.operation; -import org.apache.avro.reflect.AvroIgnore; - /** * This class represents the "copy" operation of RFC 6902 'JavaScript Object Notation (JSON) Patch'. * * @see https://tools.ietf.org/html/rfc6902#section-4.5 */ public final class Copy implements Operation { - @AvroIgnore - public static final String op = "copy"; public final Path from; public final Path path; - @SuppressWarnings("unused") // no arg constructor needed by Avro + @SuppressWarnings("unused") // no-arg constructor needed by Avro private Copy() { this(null, null); } diff --git a/src/main/java/io/datanerds/avropatch/operation/Move.java b/src/main/java/io/datanerds/avropatch/operation/Move.java index 10c2ddd..d3e9ba8 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Move.java +++ b/src/main/java/io/datanerds/avropatch/operation/Move.java @@ -1,19 +1,15 @@ package io.datanerds.avropatch.operation; -import org.apache.avro.reflect.AvroIgnore; - /** * This class represents the "move" operation of RFC 6902 'JavaScript Object Notation (JSON) Patch'. * * @see https://tools.ietf.org/html/rfc6902#section-4.4 */ public final class Move implements Operation { - @AvroIgnore - public static final String op = "move"; public final Path from; public final Path path; - @SuppressWarnings("unused") // no arg constructor needed by Avro + @SuppressWarnings("unused") // no-arg constructor needed by Avro private Move() { this(null, null); } diff --git a/src/main/java/io/datanerds/avropatch/operation/Remove.java b/src/main/java/io/datanerds/avropatch/operation/Remove.java index 2d9fb43..174be0d 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Remove.java +++ b/src/main/java/io/datanerds/avropatch/operation/Remove.java @@ -1,18 +1,14 @@ package io.datanerds.avropatch.operation; -import org.apache.avro.reflect.AvroIgnore; - /** * This class represents the "remove" operation of RFC 6902 'JavaScript Object Notation (JSON) Patch'. * * @see https://tools.ietf.org/html/rfc6902#section-4.2 */ public final class Remove implements Operation { - @AvroIgnore - public static final String op = "remove"; public final Path path; - @SuppressWarnings("unused") // no arg constructor needed by Avro + @SuppressWarnings("unused") // no-arg constructor needed by Avro private Remove() { this(null); } diff --git a/src/main/java/io/datanerds/avropatch/operation/Replace.java b/src/main/java/io/datanerds/avropatch/operation/Replace.java index dfa9f39..9a512fd 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Replace.java +++ b/src/main/java/io/datanerds/avropatch/operation/Replace.java @@ -1,6 +1,5 @@ package io.datanerds.avropatch.operation; -import org.apache.avro.reflect.AvroIgnore; import org.apache.avro.reflect.AvroSchema; /** @@ -9,13 +8,11 @@ * @see https://tools.ietf.org/html/rfc6902#section-4.3 */ public final class Replace implements Operation { - @AvroIgnore - public static final String op = "replace"; public final Path path; @AvroSchema(DefaultSchema.VALUE) public final T value; - @SuppressWarnings("unused") // no arg constructor needed by Avro + @SuppressWarnings("unused") // no-arg constructor needed by Avro private Replace() { this(null, null); } diff --git a/src/main/java/io/datanerds/avropatch/operation/Test.java b/src/main/java/io/datanerds/avropatch/operation/Test.java index e22cdc3..faa5e15 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Test.java +++ b/src/main/java/io/datanerds/avropatch/operation/Test.java @@ -1,6 +1,5 @@ package io.datanerds.avropatch.operation; -import org.apache.avro.reflect.AvroIgnore; import org.apache.avro.reflect.AvroSchema; /** @@ -9,13 +8,11 @@ * @see https://tools.ietf.org/html/rfc6902#section-4.6 */ public final class Test implements Operation { - @AvroIgnore - public static final String op = "test"; public final Path path; @AvroSchema(DefaultSchema.VALUE) public final T value; - @SuppressWarnings("unused") // no arg constructor needed by Avro + @SuppressWarnings("unused") // no-arg constructor needed by Avro private Test() { this(null, null); } From 52f872876897e7b44140146112853beac1deb869 Mon Sep 17 00:00:00 2001 From: Frank Wisniewski Date: Fri, 6 Jan 2017 10:41:24 -0800 Subject: [PATCH 15/19] Reorganized value type schemata. --- project.gradle | 10 +- .../java/io/datanerds/avropatch/Patch.java | 18 ++-- .../io/datanerds/avropatch/operation/Add.java | 1 + .../avropatch/operation/Replace.java | 1 + .../datanerds/avropatch/operation/Test.java | 1 + .../avropatch/schema/CustomTypes.java | 93 ------------------- .../avropatch/schema/PrimitiveTypes.java | 16 ---- .../serialization/CustomTypeSerializer.java | 30 +++--- .../OperationTypes.java | 34 ++++--- .../avropatch/serialization/PatchType.java | 23 +++++ .../value/{conversion => }/AvroData.java | 6 +- .../{operation => value}/DefaultSchema.java | 2 +- .../conversion/BigDecimalConversion.java | 10 +- .../conversion/BigIntegerConversion.java | 10 +- .../value/conversion/DateConversion.java | 14 +-- .../value/conversion/UUIDConversion.java | 12 +-- .../avropatch/value/type/BigDecimalType.java | 25 +++++ .../avropatch/value/type/BigIntegerType.java | 15 +++ .../avropatch/value/type/PrimitiveType.java | 28 ++++++ .../avropatch/value/type/TimestampType.java | 17 ++++ .../avropatch/value/type/UuidType.java | 16 ++++ .../io/datanerds/avropatch/PatchTest.java | 2 +- .../schema/OperationSerializationTester.java | 12 ++- .../avropatch/schema/SerializationTester.java | 2 +- .../serialization/ConcurrencyTest.java | 4 +- .../CustomTypeSerializerTest.java | 4 +- .../conversion/BigDecimalConversionTest.java | 2 +- .../conversion/BigIntegerConversionTest.java | 2 +- .../value/conversion/DateConversionTest.java | 8 +- .../value/conversion/UUIDConversionTest.java | 2 +- 30 files changed, 234 insertions(+), 186 deletions(-) delete mode 100644 src/main/java/io/datanerds/avropatch/schema/CustomTypes.java delete mode 100644 src/main/java/io/datanerds/avropatch/schema/PrimitiveTypes.java rename src/main/java/io/datanerds/avropatch/{schema => serialization}/OperationTypes.java (67%) create mode 100644 src/main/java/io/datanerds/avropatch/serialization/PatchType.java rename src/main/java/io/datanerds/avropatch/value/{conversion => }/AvroData.java (76%) rename src/main/java/io/datanerds/avropatch/{operation => value}/DefaultSchema.java (99%) create mode 100644 src/main/java/io/datanerds/avropatch/value/type/BigDecimalType.java create mode 100644 src/main/java/io/datanerds/avropatch/value/type/BigIntegerType.java create mode 100644 src/main/java/io/datanerds/avropatch/value/type/PrimitiveType.java create mode 100644 src/main/java/io/datanerds/avropatch/value/type/TimestampType.java create mode 100644 src/main/java/io/datanerds/avropatch/value/type/UuidType.java diff --git a/project.gradle b/project.gradle index 9fec903..3ea7922 100644 --- a/project.gradle +++ b/project.gradle @@ -11,13 +11,17 @@ project.ext { developer = "Frank Wisniewski" developerId = "frankwis" developerMail = "frankwis@codekoffer.de" -}; +} + +ext { + slf4jVersion = "1.7.18" +} dependencies { compile([ "org.apache.avro:avro:1.8.1", - "org.slf4j:slf4j-api:1.7.18", + "org.slf4j:slf4j-api:$slf4jVersion", "com.google.code.findbugs:jsr305:3.0.1" ]) @@ -25,7 +29,7 @@ dependencies { "junit:junit:4.12", "org.hamcrest:hamcrest-all:1.3", "com.google.guava:guava-testlib:19.0", - "org.slf4j:slf4j-simple:1.7.18" + "org.slf4j:slf4j-simple:$slf4jVersion" ]) } diff --git a/src/main/java/io/datanerds/avropatch/Patch.java b/src/main/java/io/datanerds/avropatch/Patch.java index 27d64c5..68b761f 100644 --- a/src/main/java/io/datanerds/avropatch/Patch.java +++ b/src/main/java/io/datanerds/avropatch/Patch.java @@ -1,8 +1,8 @@ package io.datanerds.avropatch; -import io.datanerds.avropatch.operation.DefaultSchema; +import io.datanerds.avropatch.value.DefaultSchema; import io.datanerds.avropatch.operation.Operation; -import io.datanerds.avropatch.value.conversion.*; +import io.datanerds.avropatch.value.AvroData; import org.apache.avro.Schema; import org.apache.avro.io.*; import org.apache.avro.reflect.AvroSchema; @@ -26,7 +26,7 @@ public class Patch { private final Map headers; private final List operations; - @SuppressWarnings("unused") // no arg constructor needed by Avro + @SuppressWarnings("unused") // no-arg constructor needed by Avro private Patch() { this(new ArrayList<>(), new HashMap<>()); } @@ -49,10 +49,10 @@ public List getOperations() { } public byte[] toBytes() throws IOException { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - Encoder binaryEncoder = EncoderFactory.get().directBinaryEncoder(outputStream, null); - SERIALIZER.writer.write(this, binaryEncoder); - return outputStream.toByteArray(); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + Encoder encoder = EncoderFactory.get().directBinaryEncoder(output, null); + SERIALIZER.writer.write(this, encoder); + return output.toByteArray(); } public static Patch of(byte[] bytes) throws IOException { @@ -65,8 +65,8 @@ private static class PatchSerializer { private PatchSerializer() { Schema schema = AvroData.get().getSchema(Patch.class); - writer = AvroData.get().createDatumWriter(schema); - reader = AvroData.get().createDatumReader(schema); + this.writer = AvroData.get().createDatumWriter(schema); + this.reader = AvroData.get().createDatumReader(schema); } } } diff --git a/src/main/java/io/datanerds/avropatch/operation/Add.java b/src/main/java/io/datanerds/avropatch/operation/Add.java index 484155b..af2f263 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Add.java +++ b/src/main/java/io/datanerds/avropatch/operation/Add.java @@ -1,5 +1,6 @@ package io.datanerds.avropatch.operation; +import io.datanerds.avropatch.value.DefaultSchema; import org.apache.avro.reflect.AvroSchema; /** diff --git a/src/main/java/io/datanerds/avropatch/operation/Replace.java b/src/main/java/io/datanerds/avropatch/operation/Replace.java index 9a512fd..df7e101 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Replace.java +++ b/src/main/java/io/datanerds/avropatch/operation/Replace.java @@ -1,5 +1,6 @@ package io.datanerds.avropatch.operation; +import io.datanerds.avropatch.value.DefaultSchema; import org.apache.avro.reflect.AvroSchema; /** diff --git a/src/main/java/io/datanerds/avropatch/operation/Test.java b/src/main/java/io/datanerds/avropatch/operation/Test.java index faa5e15..a365ce0 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Test.java +++ b/src/main/java/io/datanerds/avropatch/operation/Test.java @@ -1,5 +1,6 @@ package io.datanerds.avropatch.operation; +import io.datanerds.avropatch.value.DefaultSchema; import org.apache.avro.reflect.AvroSchema; /** diff --git a/src/main/java/io/datanerds/avropatch/schema/CustomTypes.java b/src/main/java/io/datanerds/avropatch/schema/CustomTypes.java deleted file mode 100644 index bc8804e..0000000 --- a/src/main/java/io/datanerds/avropatch/schema/CustomTypes.java +++ /dev/null @@ -1,93 +0,0 @@ -package io.datanerds.avropatch.schema; - -import io.datanerds.avropatch.Patch; -import io.datanerds.avropatch.value.conversion.BigDecimalConversion; -import io.datanerds.avropatch.value.conversion.BigIntegerConversion; -import io.datanerds.avropatch.value.conversion.DateConversion; -import io.datanerds.avropatch.value.conversion.UUIDConversion; -import org.apache.avro.LogicalType; -import org.apache.avro.LogicalTypes; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Type; -import org.apache.avro.SchemaBuilder; - -import java.util.Arrays; - -import static io.datanerds.avropatch.schema.OperationTypes.NAMESPACE; -import static io.datanerds.avropatch.schema.PrimitiveTypes.*; -import static org.apache.avro.Schema.createRecord; -import static org.apache.avro.Schema.createUnion; - -/** - * This interface holds the schema constants for all supported custom types. - * - * @see UUIDConversion - * @see DateConversion - * @see BigDecimalConversion - * @see BigIntegerConversion - * @see io.datanerds.avropatch.operation.Path - */ -public interface CustomTypes { - - interface PatchType { - String NAME = Patch.class.getSimpleName(); - String DOC = "This record represents a PATCH holding a sequence of operations to apply to a given object."; - static Schema create(Schema valueSchema) { - Schema.Field operations = new Schema.Field("operations", - Schema.createArray(OperationTypes.Operation.create(valueSchema)), - "Sequence of operations to apply to a given object.", NO_DEFAULT); - return createRecord(NAME, DOC, Patch.class.getPackage().getName(), false, Arrays.asList(operations)); - } - } - - interface BigDecimalType { - String NAME = "big-decimal"; - String DOC = "BigDecimal value represented via its scale and unscaled value."; - LogicalType LOGICAL_TYPE = new LogicalType(NAME); - Schema RECORD = SchemaBuilder - .record("decimal") - .doc(DOC) - .fields() - .name("unscaledValue").type(BigIntegerType.SCHEMA).noDefault() - .name("scale").type(Schema.create(Type.INT)).noDefault() - .endRecord(); - - Schema SCHEMA = LOGICAL_TYPE.addToSchema(RECORD); - } - - interface BigIntegerType { - String NAME = "big-integer"; - LogicalType LOGICAL_TYPE = new LogicalType(NAME); - Schema SCHEMA = LOGICAL_TYPE.addToSchema(Schema.create(Type.BYTES)); - } - - interface DateType { - String NAME = "timestamp"; - String DOC = "Timestamp representing the number of milliseconds since January 1, 1970, 00:00:00 GMT"; - LogicalType LOGICAL_TYPE = new LogicalType(NAME); - int SIZE = Long.BYTES; - Schema SCHEMA = LOGICAL_TYPE.addToSchema(Schema.createFixed(NAME, DOC, null, SIZE)); - } - - interface UuidType { - String NAME = LogicalTypes.uuid().getName(); - String DOC = "UUID serialized via two long values: Its most significant and least significant 64 bits."; - int SIZE = 2 * Long.BYTES; - Schema SCHEMA = LogicalTypes.uuid().addToSchema(Schema.createFixed(NAME, DOC, null, SIZE)); - } - - interface Path { - String NAME = io.datanerds.avropatch.operation.Path.class.getSimpleName(); - String DOC = "JSON Path serialized as String array holding its parts."; - Schema SCHEMA = SchemaBuilder - .record(NAME) - .doc(DOC) - .namespace(NAMESPACE) - .fields() - .name("parts").type(Schema.createArray(Schema.create(Type.STRING))).noDefault() - .endRecord(); - } - - Schema VALUE_TYPE_UNION = createUnion(BOOLEAN, DOUBLE, FLOAT, INTEGER, LONG, NULL, STRING, - BigDecimalType.SCHEMA, BigIntegerType.SCHEMA, DateType.SCHEMA, UuidType.SCHEMA); -} diff --git a/src/main/java/io/datanerds/avropatch/schema/PrimitiveTypes.java b/src/main/java/io/datanerds/avropatch/schema/PrimitiveTypes.java deleted file mode 100644 index aafc6d2..0000000 --- a/src/main/java/io/datanerds/avropatch/schema/PrimitiveTypes.java +++ /dev/null @@ -1,16 +0,0 @@ -package io.datanerds.avropatch.schema; - -import org.apache.avro.Schema; - -import static org.apache.avro.Schema.create; - -public interface PrimitiveTypes { - Schema BOOLEAN = create(Schema.Type.BOOLEAN); - Schema DOUBLE = create(Schema.Type.DOUBLE); - Schema FLOAT = create(Schema.Type.FLOAT); - Schema INTEGER = create(Schema.Type.INT); - Schema LONG = create(Schema.Type.LONG); - Schema NULL = create(Schema.Type.NULL); - Schema STRING = create(Schema.Type.STRING); - Object NO_DEFAULT = null; -} diff --git a/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java b/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java index 54fb7b9..84c084b 100644 --- a/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java +++ b/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java @@ -1,12 +1,11 @@ package io.datanerds.avropatch.serialization; import io.datanerds.avropatch.Patch; -import io.datanerds.avropatch.schema.CustomTypes; -import io.datanerds.avropatch.schema.CustomTypes.BigDecimalType; -import io.datanerds.avropatch.schema.CustomTypes.BigIntegerType; -import io.datanerds.avropatch.schema.CustomTypes.DateType; -import io.datanerds.avropatch.schema.CustomTypes.UuidType; -import io.datanerds.avropatch.value.conversion.AvroData; +import io.datanerds.avropatch.value.AvroData; +import io.datanerds.avropatch.value.type.BigDecimalType; +import io.datanerds.avropatch.value.type.BigIntegerType; +import io.datanerds.avropatch.value.type.TimestampType; +import io.datanerds.avropatch.value.type.UuidType; import org.apache.avro.Schema; import org.apache.avro.io.*; import org.slf4j.Logger; @@ -21,7 +20,7 @@ import java.util.List; import java.util.Objects; -import static io.datanerds.avropatch.schema.PrimitiveTypes.*; +import static io.datanerds.avropatch.value.type.PrimitiveType.*; @ThreadSafe public final class CustomTypeSerializer { @@ -56,21 +55,22 @@ public Builder() { } public Builder nullable() { - types.add(NULL); + types.add(NULL.getSchema()); return this; } public Builder withPrimitives() { - types.addAll(Arrays.asList(BOOLEAN, DOUBLE, FLOAT, INTEGER, LONG, STRING)); + types.addAll(Arrays.asList(BOOLEAN.getSchema(), DOUBLE.getSchema(), FLOAT.getSchema(), INTEGER.getSchema(), + LONG.getSchema(), STRING.getSchema())); return this; } public Builder withCustomTypes() { - types.addAll(Arrays.asList(BigDecimalType.SCHEMA, BigIntegerType.SCHEMA, DateType.SCHEMA, UuidType.SCHEMA)); + types.addAll(Arrays.asList(BigDecimalType.SCHEMA, BigIntegerType.SCHEMA, TimestampType.SCHEMA, UuidType.SCHEMA)); return this; } - public Builder with(Type type) { + public Builder withType(Type type) { types.add(AvroData.get().getSchema(type)); return this; } @@ -88,9 +88,9 @@ public CustomTypeSerializer build() { private Schema makeSchema() { if (types.size() == 1) { - return CustomTypes.PatchType.create(types.get(0)); + return PatchType.create(types.get(0)); } else { - return CustomTypes.PatchType.create(Schema.createUnion(types)); + return PatchType.create(Schema.createUnion(types)); } } @@ -133,8 +133,8 @@ public Builder.ArrayBuilder withCustomTypes() { } @Override - public Builder.ArrayBuilder with(Type type) { - super.with(type); + public Builder.ArrayBuilder withType(Type type) { + super.withType(type); return this; } diff --git a/src/main/java/io/datanerds/avropatch/schema/OperationTypes.java b/src/main/java/io/datanerds/avropatch/serialization/OperationTypes.java similarity index 67% rename from src/main/java/io/datanerds/avropatch/schema/OperationTypes.java rename to src/main/java/io/datanerds/avropatch/serialization/OperationTypes.java index b1d3610..a849594 100644 --- a/src/main/java/io/datanerds/avropatch/schema/OperationTypes.java +++ b/src/main/java/io/datanerds/avropatch/serialization/OperationTypes.java @@ -1,11 +1,10 @@ -package io.datanerds.avropatch.schema; +package io.datanerds.avropatch.serialization; -import io.datanerds.avropatch.schema.CustomTypes.Path; import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; import java.util.Arrays; -import static io.datanerds.avropatch.schema.PrimitiveTypes.NO_DEFAULT; import static org.apache.avro.Schema.*; import static org.apache.avro.Schema.createRecord; @@ -15,14 +14,27 @@ */ public interface OperationTypes { + Object NO_DEFAULT = null; String NAMESPACE = io.datanerds.avropatch.operation.Operation.class.getPackage().getName(); + interface PathType { + String NAME = io.datanerds.avropatch.operation.Path.class.getSimpleName(); + String DOC = "JSON Path serialized as String array holding its parts."; + Schema SCHEMA = SchemaBuilder + .record(NAME) + .doc(DOC) + .namespace(NAMESPACE) + .fields() + .name("parts").type(Schema.createArray(Schema.create(Type.STRING))).noDefault() + .endRecord(); + } + interface Add { String NAME = io.datanerds.avropatch.operation.Add.class.getSimpleName(); String DOC = "This record represents the add operation of RFC 6902 for JSON Patch'."; static Schema create(Schema valueSchema) { - Field path = new Field("path", Path.SCHEMA, "Path pointing out where the JSON value should be added", + Field path = new Field("path", PathType.SCHEMA, "Path pointing out where the JSON value should be added", NO_DEFAULT); Field value = new Field("value", valueSchema, "Actual value to add to patched object", NO_DEFAULT); return createRecord(NAME, DOC, NAMESPACE, false, Arrays.asList(path, value)); @@ -34,8 +46,8 @@ interface Copy { String DOC = "This record represents the copy operation of RFC 6902 for JSON Patch'."; static Schema create() { - Field from = new Field("from", Path.SCHEMA, "Source location of the value to be copied", NO_DEFAULT); - Field path = new Field("path", Path.SCHEMA, "Target location of value to be copied", NO_DEFAULT); + Field from = new Field("from", PathType.SCHEMA, "Source location of the value to be copied", NO_DEFAULT); + Field path = new Field("path", PathType.SCHEMA, "Target location of value to be copied", NO_DEFAULT); return createRecord(NAME, DOC, NAMESPACE, false, Arrays.asList(from, path)); } } @@ -45,8 +57,8 @@ interface Move { String DOC = "This record represents the move operation of RFC 6902 for JSON Patch'."; static Schema create() { - Field from = new Field("from", Path.SCHEMA, "Source location of the value to be moved", NO_DEFAULT); - Field path = new Field("path", Path.SCHEMA, "Target location for the value to be moved to", NO_DEFAULT); + Field from = new Field("from", PathType.SCHEMA, "Source location of the value to be moved", NO_DEFAULT); + Field path = new Field("path", PathType.SCHEMA, "Target location for the value to be moved to", NO_DEFAULT); return createRecord(NAME, DOC, NAMESPACE, false, Arrays.asList(from, path)); } } @@ -56,7 +68,7 @@ interface Remove { String DOC = "This record represents the remove operation of RFC 6902 for JSON Patch'."; static Schema create() { - Field path = new Field("path", Path.SCHEMA, "Target location of value to be removed", NO_DEFAULT); + Field path = new Field("path", PathType.SCHEMA, "Target location of value to be removed", NO_DEFAULT); return createRecord(NAME, DOC, NAMESPACE, false, Arrays.asList(path)); } } @@ -66,7 +78,7 @@ interface Replace { String DOC = "This record represents the replace operation of RFC 6902 for JSON Patch'."; static Schema create(Schema valueSchema) { - Field path = new Field("path", Path.SCHEMA, "Path pointing out where the JSON value should be replaced", + Field path = new Field("path", PathType.SCHEMA, "Path pointing out where the JSON value should be replaced", NO_DEFAULT); Field value = new Field("value", valueSchema, "Actual value to be replaced in patched object", NO_DEFAULT); return createRecord(NAME, DOC, NAMESPACE, false, Arrays.asList(path, value)); @@ -78,7 +90,7 @@ interface Test { String DOC = "This record represents the test operation of RFC 6902 for JSON Patch'."; static Schema create(Schema valueSchema) { - Field path = new Field("path", Path.SCHEMA, "Path pointing out which JSON value should be tested against", + Field path = new Field("path", PathType.SCHEMA, "Path pointing out which JSON value should be tested against", NO_DEFAULT); Field value = new Field("value", valueSchema, "Actual value to test against", NO_DEFAULT); return createRecord(NAME, DOC, NAMESPACE, false, Arrays.asList(path, value)); diff --git a/src/main/java/io/datanerds/avropatch/serialization/PatchType.java b/src/main/java/io/datanerds/avropatch/serialization/PatchType.java new file mode 100644 index 0000000..2dcda6f --- /dev/null +++ b/src/main/java/io/datanerds/avropatch/serialization/PatchType.java @@ -0,0 +1,23 @@ +package io.datanerds.avropatch.serialization; + +import io.datanerds.avropatch.Patch; +import org.apache.avro.Schema; +import java.util.Arrays; + +import static org.apache.avro.Schema.createRecord; + +/** + * This interface holds the schema constants for the {@link Patch} type. + * @see Patch + */ +public interface PatchType { + Object NO_DEFAULT = null; + String NAME = Patch.class.getSimpleName(); + String DOC = "This record represents a PATCH holding a sequence of operations to apply to a given object."; + static Schema create(Schema valueSchema) { + Schema.Field operations = new Schema.Field("operations", + Schema.createArray(OperationTypes.Operation.create(valueSchema)), + "Sequence of operations to apply to a given object.", NO_DEFAULT); + return createRecord(NAME, DOC, Patch.class.getPackage().getName(), false, Arrays.asList(operations)); + } +} diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/AvroData.java b/src/main/java/io/datanerds/avropatch/value/AvroData.java similarity index 76% rename from src/main/java/io/datanerds/avropatch/value/conversion/AvroData.java rename to src/main/java/io/datanerds/avropatch/value/AvroData.java index 78ac6ce..a491149 100644 --- a/src/main/java/io/datanerds/avropatch/value/conversion/AvroData.java +++ b/src/main/java/io/datanerds/avropatch/value/AvroData.java @@ -1,6 +1,10 @@ -package io.datanerds.avropatch.value.conversion; +package io.datanerds.avropatch.value; import avro.shaded.com.google.common.collect.ImmutableList; +import io.datanerds.avropatch.value.conversion.BigDecimalConversion; +import io.datanerds.avropatch.value.conversion.BigIntegerConversion; +import io.datanerds.avropatch.value.conversion.DateConversion; +import io.datanerds.avropatch.value.conversion.UUIDConversion; import org.apache.avro.Conversion; import org.apache.avro.reflect.ReflectData; diff --git a/src/main/java/io/datanerds/avropatch/operation/DefaultSchema.java b/src/main/java/io/datanerds/avropatch/value/DefaultSchema.java similarity index 99% rename from src/main/java/io/datanerds/avropatch/operation/DefaultSchema.java rename to src/main/java/io/datanerds/avropatch/value/DefaultSchema.java index 50fdfdc..0ebb96e 100644 --- a/src/main/java/io/datanerds/avropatch/operation/DefaultSchema.java +++ b/src/main/java/io/datanerds/avropatch/value/DefaultSchema.java @@ -1,4 +1,4 @@ -package io.datanerds.avropatch.operation; +package io.datanerds.avropatch.value; /** * This interface holds the schema for all valid value types. diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/BigDecimalConversion.java b/src/main/java/io/datanerds/avropatch/value/conversion/BigDecimalConversion.java index 9108300..d910c01 100644 --- a/src/main/java/io/datanerds/avropatch/value/conversion/BigDecimalConversion.java +++ b/src/main/java/io/datanerds/avropatch/value/conversion/BigDecimalConversion.java @@ -1,6 +1,6 @@ package io.datanerds.avropatch.value.conversion; -import io.datanerds.avropatch.schema.CustomTypes.BigDecimalType; +import io.datanerds.avropatch.value.type.BigDecimalType; import org.apache.avro.Conversion; import org.apache.avro.LogicalType; import org.apache.avro.LogicalTypes; @@ -21,15 +21,15 @@ * @see BigIntegerConversion * @see LogicalTypes */ -public class BigDecimalConversion extends Conversion { +public class BigDecimalConversion extends Conversion implements BigDecimalType { static { - LogicalTypes.register(BigDecimalType.NAME, schema -> BigDecimalType.LOGICAL_TYPE); + LogicalTypes.register(NAME, schema -> LOGICAL_TYPE); } @Override public Schema getRecommendedSchema() { - return BigDecimalType.SCHEMA; + return SCHEMA; } @Override @@ -39,7 +39,7 @@ public Class getConvertedType() { @Override public String getLogicalTypeName() { - return BigDecimalType.NAME; + return NAME; } @Override diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/BigIntegerConversion.java b/src/main/java/io/datanerds/avropatch/value/conversion/BigIntegerConversion.java index dad8723..c201dc5 100644 --- a/src/main/java/io/datanerds/avropatch/value/conversion/BigIntegerConversion.java +++ b/src/main/java/io/datanerds/avropatch/value/conversion/BigIntegerConversion.java @@ -1,6 +1,6 @@ package io.datanerds.avropatch.value.conversion; -import io.datanerds.avropatch.schema.CustomTypes.BigIntegerType; +import io.datanerds.avropatch.value.type.BigIntegerType; import org.apache.avro.Conversion; import org.apache.avro.LogicalType; import org.apache.avro.LogicalTypes; @@ -9,15 +9,15 @@ import java.math.BigInteger; import java.nio.ByteBuffer; -public class BigIntegerConversion extends Conversion { +public class BigIntegerConversion extends Conversion implements BigIntegerType { static { - LogicalTypes.register(BigIntegerType.NAME, schema -> BigIntegerType.LOGICAL_TYPE); + LogicalTypes.register(NAME, schema -> LOGICAL_TYPE); } @Override public Schema getRecommendedSchema() { - return BigIntegerType.SCHEMA; + return SCHEMA; } @Override @@ -27,7 +27,7 @@ public Class getConvertedType() { @Override public String getLogicalTypeName() { - return BigIntegerType.NAME; + return NAME; } @Override diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/DateConversion.java b/src/main/java/io/datanerds/avropatch/value/conversion/DateConversion.java index 480328e..92811aa 100644 --- a/src/main/java/io/datanerds/avropatch/value/conversion/DateConversion.java +++ b/src/main/java/io/datanerds/avropatch/value/conversion/DateConversion.java @@ -1,6 +1,6 @@ package io.datanerds.avropatch.value.conversion; -import io.datanerds.avropatch.schema.CustomTypes.DateType; +import io.datanerds.avropatch.value.type.TimestampType; import org.apache.avro.Conversion; import org.apache.avro.LogicalType; import org.apache.avro.LogicalTypes; @@ -19,15 +19,15 @@ * @see org.apache.avro.LogicalTypes.TimestampMillis * @see org.apache.avro.Schema.Type */ -public class DateConversion extends Conversion { +public class DateConversion extends Conversion implements TimestampType { static { - LogicalTypes.register(DateType.NAME, schema -> DateType.LOGICAL_TYPE); + LogicalTypes.register(NAME, schema -> LOGICAL_TYPE); } @Override public Schema getRecommendedSchema() { - return DateType.SCHEMA; + return SCHEMA; } @Override @@ -37,7 +37,7 @@ public Class getConvertedType() { @Override public String getLogicalTypeName() { - return DateType.NAME; + return NAME; } @Override @@ -45,14 +45,14 @@ public GenericFixed toFixed(Date value, Schema schema, LogicalType type) { return new GenericFixed() { @Override public byte[] bytes() { - ByteBuffer buffer = ByteBuffer.allocate(DateType.SIZE); + ByteBuffer buffer = ByteBuffer.allocate(SIZE); buffer.putLong(value.getTime()); return buffer.array(); } @Override public Schema getSchema() { - return DateType.SCHEMA; + return SCHEMA; } }; } diff --git a/src/main/java/io/datanerds/avropatch/value/conversion/UUIDConversion.java b/src/main/java/io/datanerds/avropatch/value/conversion/UUIDConversion.java index 9559f79..ad1174a 100644 --- a/src/main/java/io/datanerds/avropatch/value/conversion/UUIDConversion.java +++ b/src/main/java/io/datanerds/avropatch/value/conversion/UUIDConversion.java @@ -1,6 +1,6 @@ package io.datanerds.avropatch.value.conversion; -import io.datanerds.avropatch.schema.CustomTypes.UuidType; +import io.datanerds.avropatch.value.type.UuidType; import org.apache.avro.Conversion; import org.apache.avro.LogicalType; import org.apache.avro.Schema; @@ -9,11 +9,11 @@ import java.nio.ByteBuffer; import java.util.UUID; -public class UUIDConversion extends Conversion { +public class UUIDConversion extends Conversion implements UuidType { @Override public Schema getRecommendedSchema() { - return UuidType.SCHEMA; + return SCHEMA; } @Override @@ -23,7 +23,7 @@ public Class getConvertedType() { @Override public String getLogicalTypeName() { - return UuidType.NAME; + return NAME; } @Override @@ -31,7 +31,7 @@ public GenericFixed toFixed(UUID value, Schema schema, LogicalType type) { return new GenericFixed() { @Override public byte[] bytes() { - ByteBuffer buffer = ByteBuffer.allocate(UuidType.SIZE); + ByteBuffer buffer = ByteBuffer.allocate(SIZE); buffer.putLong(value.getLeastSignificantBits()); buffer.putLong(value.getMostSignificantBits()); return buffer.array(); @@ -39,7 +39,7 @@ public byte[] bytes() { @Override public Schema getSchema() { - return UuidType.SCHEMA; + return SCHEMA; } }; } diff --git a/src/main/java/io/datanerds/avropatch/value/type/BigDecimalType.java b/src/main/java/io/datanerds/avropatch/value/type/BigDecimalType.java new file mode 100644 index 0000000..adb3f96 --- /dev/null +++ b/src/main/java/io/datanerds/avropatch/value/type/BigDecimalType.java @@ -0,0 +1,25 @@ +package io.datanerds.avropatch.value.type; + +import org.apache.avro.LogicalType; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; + +/** + * This interface holds the schema constants for the custom type representing a {@link java.math.BigDecimal} value. + * + * @see io.datanerds.avropatch.value.conversion.BigDecimalConversion + */ +public interface BigDecimalType { + String NAME = "big-decimal"; + String DOC = "BigDecimal value represented via its scale and unscaled value."; + LogicalType LOGICAL_TYPE = new LogicalType(NAME); + Schema RECORD = SchemaBuilder + .record("decimal") + .doc(DOC) + .fields() + .name("unscaledValue").type(BigIntegerType.SCHEMA).noDefault() + .name("scale").type(Schema.create(Schema.Type.INT)).noDefault() + .endRecord(); + + Schema SCHEMA = LOGICAL_TYPE.addToSchema(RECORD); +} \ No newline at end of file diff --git a/src/main/java/io/datanerds/avropatch/value/type/BigIntegerType.java b/src/main/java/io/datanerds/avropatch/value/type/BigIntegerType.java new file mode 100644 index 0000000..3ccfb6b --- /dev/null +++ b/src/main/java/io/datanerds/avropatch/value/type/BigIntegerType.java @@ -0,0 +1,15 @@ +package io.datanerds.avropatch.value.type; + +import org.apache.avro.LogicalType; +import org.apache.avro.Schema; + +/** + * This interface holds the schema constants for the custom type representing a {@link java.math.BigInteger} value. + * + * @see io.datanerds.avropatch.value.conversion.BigIntegerConversion + */ +public interface BigIntegerType { + String NAME = "big-integer"; + LogicalType LOGICAL_TYPE = new LogicalType(NAME); + Schema SCHEMA = LOGICAL_TYPE.addToSchema(Schema.create(Schema.Type.BYTES)); +} \ No newline at end of file diff --git a/src/main/java/io/datanerds/avropatch/value/type/PrimitiveType.java b/src/main/java/io/datanerds/avropatch/value/type/PrimitiveType.java new file mode 100644 index 0000000..193cd00 --- /dev/null +++ b/src/main/java/io/datanerds/avropatch/value/type/PrimitiveType.java @@ -0,0 +1,28 @@ +package io.datanerds.avropatch.value.type; + +import org.apache.avro.Schema; + +import static org.apache.avro.Schema.create; + +/** + * This enum holds the schema constants for the Avro's default value types. + */ +public enum PrimitiveType { + BOOLEAN(create(Schema.Type.BOOLEAN)), + DOUBLE(create(Schema.Type.DOUBLE)), + FLOAT(create(Schema.Type.FLOAT)), + INTEGER(create(Schema.Type.INT)), + LONG(create(Schema.Type.LONG)), + NULL(create(Schema.Type.NULL)), + STRING(create(Schema.Type.STRING)); + + private final Schema schema; + + PrimitiveType(Schema schema) { + this.schema = schema; + } + + public Schema getSchema() { + return schema; + } +} diff --git a/src/main/java/io/datanerds/avropatch/value/type/TimestampType.java b/src/main/java/io/datanerds/avropatch/value/type/TimestampType.java new file mode 100644 index 0000000..9aa2056 --- /dev/null +++ b/src/main/java/io/datanerds/avropatch/value/type/TimestampType.java @@ -0,0 +1,17 @@ +package io.datanerds.avropatch.value.type; + +import org.apache.avro.LogicalType; +import org.apache.avro.Schema; + +/** + * This interface holds the schema constants for the custom type representing a {@link java.util.Date} value. + * + * @see io.datanerds.avropatch.value.conversion.DateConversion + */ +public interface TimestampType { + String NAME = "timestamp"; + String DOC = "Timestamp representing the number of milliseconds since January 1, 1970, 00:00:00 GMT"; + LogicalType LOGICAL_TYPE = new LogicalType(NAME); + int SIZE = Long.BYTES; + Schema SCHEMA = LOGICAL_TYPE.addToSchema(Schema.createFixed(NAME, DOC, null, SIZE)); +} \ No newline at end of file diff --git a/src/main/java/io/datanerds/avropatch/value/type/UuidType.java b/src/main/java/io/datanerds/avropatch/value/type/UuidType.java new file mode 100644 index 0000000..b72aa2c --- /dev/null +++ b/src/main/java/io/datanerds/avropatch/value/type/UuidType.java @@ -0,0 +1,16 @@ +package io.datanerds.avropatch.value.type; + +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; + +/** + * This interface holds the schema constants for the custom type representing a {@link java.util.UUID} value. + * + * @see io.datanerds.avropatch.value.conversion.UUIDConversion + */ +public interface UuidType { + String NAME = LogicalTypes.uuid().getName(); + String DOC = "UUID serialized via two long values: Its most significant and least significant 64 bits."; + int SIZE = 2 * Long.BYTES; + Schema SCHEMA = LogicalTypes.uuid().addToSchema(Schema.createFixed(NAME, DOC, null, SIZE)); +} \ No newline at end of file diff --git a/src/test/java/io/datanerds/avropatch/PatchTest.java b/src/test/java/io/datanerds/avropatch/PatchTest.java index 79e1460..a69310c 100644 --- a/src/test/java/io/datanerds/avropatch/PatchTest.java +++ b/src/test/java/io/datanerds/avropatch/PatchTest.java @@ -135,7 +135,7 @@ public void serializesArbitraryHeadersWithOperations() throws IOException { "header 2", new Date(), "header 3", 1234L, "header 4", new BigDecimal("3214123453.123512345"))); - byte[] bytes = patch.toBytes();; + byte[] bytes = patch.toBytes(); assertThat(patch, is(equalTo(Patch.of(bytes)))); } diff --git a/src/test/java/io/datanerds/avropatch/schema/OperationSerializationTester.java b/src/test/java/io/datanerds/avropatch/schema/OperationSerializationTester.java index fa67508..24eb0f0 100644 --- a/src/test/java/io/datanerds/avropatch/schema/OperationSerializationTester.java +++ b/src/test/java/io/datanerds/avropatch/schema/OperationSerializationTester.java @@ -2,6 +2,11 @@ import com.google.common.collect.ImmutableMap; import io.datanerds.avropatch.operation.*; +import io.datanerds.avropatch.serialization.OperationTypes; +import io.datanerds.avropatch.value.type.BigDecimalType; +import io.datanerds.avropatch.value.type.BigIntegerType; +import io.datanerds.avropatch.value.type.TimestampType; +import io.datanerds.avropatch.value.type.UuidType; import org.apache.avro.Schema; import java.io.IOException; @@ -11,12 +16,17 @@ import java.util.function.Function; import static io.datanerds.avropatch.operation.matcher.OperationMatchers.equalTo; -import static io.datanerds.avropatch.schema.CustomTypes.VALUE_TYPE_UNION; +import static io.datanerds.avropatch.value.type.PrimitiveType.*; +import static org.apache.avro.Schema.createUnion; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; public class OperationSerializationTester extends SerializationTester { + private static final Schema VALUE_TYPE_UNION = createUnion(BOOLEAN.getSchema(), DOUBLE.getSchema(), + FLOAT.getSchema(), INTEGER.getSchema(), LONG.getSchema(), NULL.getSchema(), STRING.getSchema(), + BigDecimalType.SCHEMA, BigIntegerType.SCHEMA, TimestampType.SCHEMA, UuidType.SCHEMA); + private static final Map, Schema> SCHEMATA = new ImmutableMap.Builder() .put(Add.class, OperationTypes.Add.create(VALUE_TYPE_UNION)) .put(Copy.class, OperationTypes.Copy.create()) diff --git a/src/test/java/io/datanerds/avropatch/schema/SerializationTester.java b/src/test/java/io/datanerds/avropatch/schema/SerializationTester.java index 9bf9b14..45a50af 100644 --- a/src/test/java/io/datanerds/avropatch/schema/SerializationTester.java +++ b/src/test/java/io/datanerds/avropatch/schema/SerializationTester.java @@ -1,6 +1,6 @@ package io.datanerds.avropatch.schema; -import io.datanerds.avropatch.value.conversion.AvroData; +import io.datanerds.avropatch.value.AvroData; import org.apache.avro.Schema; import org.apache.avro.io.*; diff --git a/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java b/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java index 418204c..aa1787e 100644 --- a/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java +++ b/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java @@ -29,11 +29,11 @@ public class ConcurrencyTest { .withPrimitives() .withCustomTypes() .nullable() - .with(Bimmel.class) + .withType(Bimmel.class) .endArray() .withPrimitives() .withCustomTypes() - .with(Bimmel.class) + .withType(Bimmel.class) .nullable() .build(); diff --git a/src/test/java/io/datanerds/avropatch/serialization/CustomTypeSerializerTest.java b/src/test/java/io/datanerds/avropatch/serialization/CustomTypeSerializerTest.java index 37a0fac..e99555e 100644 --- a/src/test/java/io/datanerds/avropatch/serialization/CustomTypeSerializerTest.java +++ b/src/test/java/io/datanerds/avropatch/serialization/CustomTypeSerializerTest.java @@ -36,7 +36,7 @@ public void withCustomClass() throws IOException { .withPrimitives() .withCustomTypes() .endArray() - .with(Bimmel.class) + .withType(Bimmel.class) .build(); Patch patch = new Patch(ImmutableList.of( new Replace(Path.of("hello"), new Bimmel("string", 42, UUID.randomUUID(), new Bimmel.Bommel("Gaga"))))); @@ -54,7 +54,7 @@ public void withMultipleOperations() throws IOException { .endArray() .nullable() .withPrimitives() - .with(Bimmel.class) + .withType(Bimmel.class) .build(); Patch patch = new Patch(ImmutableList.of( new io.datanerds.avropatch.operation.Test(Path.of("hello", "world"), null), diff --git a/src/test/java/io/datanerds/avropatch/value/conversion/BigDecimalConversionTest.java b/src/test/java/io/datanerds/avropatch/value/conversion/BigDecimalConversionTest.java index 1c3a499..a972d07 100644 --- a/src/test/java/io/datanerds/avropatch/value/conversion/BigDecimalConversionTest.java +++ b/src/test/java/io/datanerds/avropatch/value/conversion/BigDecimalConversionTest.java @@ -2,7 +2,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.datanerds.avropatch.schema.CustomTypes.BigDecimalType; +import io.datanerds.avropatch.value.type.BigDecimalType; import org.apache.avro.Schema; import org.junit.Test; diff --git a/src/test/java/io/datanerds/avropatch/value/conversion/BigIntegerConversionTest.java b/src/test/java/io/datanerds/avropatch/value/conversion/BigIntegerConversionTest.java index b68046a..c08a01b 100644 --- a/src/test/java/io/datanerds/avropatch/value/conversion/BigIntegerConversionTest.java +++ b/src/test/java/io/datanerds/avropatch/value/conversion/BigIntegerConversionTest.java @@ -2,7 +2,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.datanerds.avropatch.schema.CustomTypes.BigIntegerType; +import io.datanerds.avropatch.value.type.BigIntegerType; import org.apache.avro.Schema; import org.junit.Test; diff --git a/src/test/java/io/datanerds/avropatch/value/conversion/DateConversionTest.java b/src/test/java/io/datanerds/avropatch/value/conversion/DateConversionTest.java index 08dfc65..3b458e5 100644 --- a/src/test/java/io/datanerds/avropatch/value/conversion/DateConversionTest.java +++ b/src/test/java/io/datanerds/avropatch/value/conversion/DateConversionTest.java @@ -2,7 +2,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.datanerds.avropatch.schema.CustomTypes.DateType; +import io.datanerds.avropatch.value.type.TimestampType; import org.apache.avro.Schema; import org.junit.Test; @@ -14,7 +14,7 @@ public class DateConversionTest { @Test public void serializesSingleValue() throws IOException { ConversionTester - .withSchemata(DateType.SCHEMA) + .withSchemata(TimestampType.SCHEMA) .withConverters(new DateConversion()) .reserializeAndAssert(new Date()) .reserializeAndAssert(new Date()); @@ -23,7 +23,7 @@ public void serializesSingleValue() throws IOException { @Test public void serializesList() throws IOException { ConversionTester - .withSchemata(Schema.createArray(DateType.SCHEMA)) + .withSchemata(Schema.createArray(TimestampType.SCHEMA)) .withConverters(new DateConversion()) .reserializeAndAssert(ImmutableList.of(new Date(), new Date(), new Date(), new Date())); @@ -32,7 +32,7 @@ public void serializesList() throws IOException { @Test public void serializesMap() throws IOException { ConversionTester - .withSchemata(Schema.createMap(DateType.SCHEMA)) + .withSchemata(Schema.createMap(TimestampType.SCHEMA)) .withConverters(new DateConversion()) .reserializeAndAssert(ImmutableMap.of( "key 1", new Date(), diff --git a/src/test/java/io/datanerds/avropatch/value/conversion/UUIDConversionTest.java b/src/test/java/io/datanerds/avropatch/value/conversion/UUIDConversionTest.java index fd20ac3..5f65eaa 100644 --- a/src/test/java/io/datanerds/avropatch/value/conversion/UUIDConversionTest.java +++ b/src/test/java/io/datanerds/avropatch/value/conversion/UUIDConversionTest.java @@ -2,7 +2,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.datanerds.avropatch.schema.CustomTypes.UuidType; +import io.datanerds.avropatch.value.type.UuidType; import org.apache.avro.Schema; import org.junit.Test; From c6777a6700d828d5cb52ec2a2cc3f19ed768d27d Mon Sep 17 00:00:00 2001 From: Frank Wisniewski Date: Fri, 6 Jan 2017 10:54:26 -0800 Subject: [PATCH 16/19] Seperated serialization from domain. --- .../java/io/datanerds/avropatch/Patch.java | 32 ------------- ...omTypeSerializer.java => PatchMapper.java} | 18 +++---- .../serialization/ConcurrencyTest.java | 4 +- .../DefaultPatchMapperTest.java} | 47 ++++++++++--------- ...rializerTest.java => PatchMapperTest.java} | 14 +++--- 5 files changed, 44 insertions(+), 71 deletions(-) rename src/main/java/io/datanerds/avropatch/serialization/{CustomTypeSerializer.java => PatchMapper.java} (91%) rename src/test/java/io/datanerds/avropatch/{PatchTest.java => serialization/DefaultPatchMapperTest.java} (78%) rename src/test/java/io/datanerds/avropatch/serialization/{CustomTypeSerializerTest.java => PatchMapperTest.java} (82%) diff --git a/src/main/java/io/datanerds/avropatch/Patch.java b/src/main/java/io/datanerds/avropatch/Patch.java index 68b761f..bc12259 100644 --- a/src/main/java/io/datanerds/avropatch/Patch.java +++ b/src/main/java/io/datanerds/avropatch/Patch.java @@ -2,14 +2,7 @@ import io.datanerds.avropatch.value.DefaultSchema; import io.datanerds.avropatch.operation.Operation; -import io.datanerds.avropatch.value.AvroData; -import org.apache.avro.Schema; -import org.apache.avro.io.*; import org.apache.avro.reflect.AvroSchema; - -import javax.annotation.concurrent.ThreadSafe; -import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.util.*; /** @@ -17,11 +10,8 @@ * * @see https://tools.ietf.org/html/rfc6902 */ -@ThreadSafe public class Patch { - private static final PatchSerializer SERIALIZER = new PatchSerializer(); - @AvroSchema(DefaultSchema.HEADERS) private final Map headers; private final List operations; @@ -47,26 +37,4 @@ public List getOperations() { public Map getHeaders() { return Collections.unmodifiableMap(headers); } - - public byte[] toBytes() throws IOException { - ByteArrayOutputStream output = new ByteArrayOutputStream(); - Encoder encoder = EncoderFactory.get().directBinaryEncoder(output, null); - SERIALIZER.writer.write(this, encoder); - return output.toByteArray(); - } - - public static Patch of(byte[] bytes) throws IOException { - return SERIALIZER.reader.read(null, DecoderFactory.get().binaryDecoder(bytes, null)); - } - - private static class PatchSerializer { - final DatumWriter writer; - final DatumReader reader; - - private PatchSerializer() { - Schema schema = AvroData.get().getSchema(Patch.class); - this.writer = AvroData.get().createDatumWriter(schema); - this.reader = AvroData.get().createDatumReader(schema); - } - } } diff --git a/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java b/src/main/java/io/datanerds/avropatch/serialization/PatchMapper.java similarity index 91% rename from src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java rename to src/main/java/io/datanerds/avropatch/serialization/PatchMapper.java index 84c084b..01c7aff 100644 --- a/src/main/java/io/datanerds/avropatch/serialization/CustomTypeSerializer.java +++ b/src/main/java/io/datanerds/avropatch/serialization/PatchMapper.java @@ -18,20 +18,22 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Objects; import static io.datanerds.avropatch.value.type.PrimitiveType.*; @ThreadSafe -public final class CustomTypeSerializer { +public final class PatchMapper { - private static Logger logger = LoggerFactory.getLogger(CustomTypeSerializer.class); + private static Logger logger = LoggerFactory.getLogger(PatchMapper.class); private final DatumWriter writer; private final DatumReader reader; - private CustomTypeSerializer(Schema schema) { - Objects.nonNull(schema); + public PatchMapper() { + this(AvroData.get().getSchema(Patch.class)); + } + + private PatchMapper(Schema schema) { this.writer = AvroData.get().createDatumWriter(schema); this.reader = AvroData.get().createDatumReader(schema); } @@ -43,7 +45,7 @@ public byte[] toBytes(Patch value) throws IOException { return outputStream.toByteArray(); } - public Patch toObject(byte[] bytes) throws IOException { + public Patch toPatch(byte[] bytes) throws IOException { return reader.read(null, DecoderFactory.get().binaryDecoder(bytes, null)); } @@ -80,10 +82,10 @@ public Builder with(Schema schema) { return this; } - public CustomTypeSerializer build() { + public PatchMapper build() { Schema schema = makeSchema(); logger.debug("Created serializer for schema {}", schema); - return new CustomTypeSerializer(schema); + return new PatchMapper(schema); } private Schema makeSchema() { diff --git a/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java b/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java index aa1787e..dcd2ee7 100644 --- a/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java +++ b/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java @@ -24,7 +24,7 @@ public class ConcurrencyTest { private static final int MAX_PATCH_SIZE = 50; private static final List PATCHES = Collections.unmodifiableList(generateData()); - private static final CustomTypeSerializer SERIALIZER = new CustomTypeSerializer.Builder() + private static final PatchMapper SERIALIZER = new PatchMapper.Builder() .withArray() .withPrimitives() .withCustomTypes() @@ -71,7 +71,7 @@ private List reserializePatches(CountDownLatch startThreadsLatch, CountDo startThreadsLatch.await(); for (Patch patch : input) { byte[] bytes = SERIALIZER.toBytes(patch); - output.add(SERIALIZER.toObject(bytes)); + output.add(SERIALIZER.toPatch(bytes)); } } catch (Exception ex) { logger.warn("Serialization thread failed.", ex); diff --git a/src/test/java/io/datanerds/avropatch/PatchTest.java b/src/test/java/io/datanerds/avropatch/serialization/DefaultPatchMapperTest.java similarity index 78% rename from src/test/java/io/datanerds/avropatch/PatchTest.java rename to src/test/java/io/datanerds/avropatch/serialization/DefaultPatchMapperTest.java index a69310c..dc72d93 100644 --- a/src/test/java/io/datanerds/avropatch/PatchTest.java +++ b/src/test/java/io/datanerds/avropatch/serialization/DefaultPatchMapperTest.java @@ -1,7 +1,8 @@ -package io.datanerds.avropatch; +package io.datanerds.avropatch.serialization; import avro.shaded.com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableList; +import io.datanerds.avropatch.Patch; import io.datanerds.avropatch.operation.*; import org.junit.Test; @@ -19,14 +20,16 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasSize; -public class PatchTest { +public class DefaultPatchMapperTest { + + private final PatchMapper mapper = new PatchMapper(); @Test public void serializesAdd() throws IOException { Patch patch = patchOf(new Add<>(Path.of("person", "name"), "John Doe")); - byte[] bytes = patch.toBytes(); + byte[] bytes = mapper.toBytes(patch); - List operations = Patch.of(bytes).getOperations(); + List operations = mapper.toPatch(bytes).getOperations(); assertThat(operations, hasSize(1)); assertThat(operations, hasItem(new Add<>(Path.of("person", "name"), "John Doe"))); } @@ -34,9 +37,9 @@ public void serializesAdd() throws IOException { @Test public void serializesCopy() throws IOException { Patch patch = patchOf(new Copy(Path.parse("/person/firstName"), Path.parse("/person/lastName"))); - byte[] bytes = patch.toBytes(); + byte[] bytes = mapper.toBytes(patch); - List operations = Patch.of(bytes).getOperations(); + List operations = mapper.toPatch(bytes).getOperations(); assertThat(operations, hasSize(1)); assertThat(operations, hasItem(new Copy(Path.parse("/person/firstName"), Path.parse("/person/lastName")))); } @@ -44,9 +47,9 @@ public void serializesCopy() throws IOException { @Test public void serializesMove() throws IOException { Patch patch = patchOf(new Move(Path.parse("/person/firstName"), Path.parse("/person/lastName"))); - byte[] bytes = patch.toBytes(); + byte[] bytes = mapper.toBytes(patch); - List operations = Patch.of(bytes).getOperations(); + List operations = mapper.toPatch(bytes).getOperations(); assertThat(operations, hasSize(1)); assertThat(operations, hasItem(new Move(Path.parse("/person/firstName"), Path.parse("/person/lastName")))); } @@ -54,9 +57,9 @@ public void serializesMove() throws IOException { @Test public void serializesRemove() throws IOException { Patch patch = patchOf(new Remove(Path.parse("/person/name"))); - byte[] bytes = patch.toBytes(); + byte[] bytes = mapper.toBytes(patch); - List operations = Patch.of(bytes).getOperations(); + List operations = mapper.toPatch(bytes).getOperations(); assertThat(operations, hasSize(1)); assertThat(operations, hasItem(new Remove(Path.parse("/person/name")))); } @@ -64,18 +67,18 @@ public void serializesRemove() throws IOException { @Test public void serializesReplace() throws IOException { Patch patch = patchOf(new Replace(Path.parse("/person/number"), 42)); - byte[] bytes = patch.toBytes(); + byte[] bytes = mapper.toBytes(patch); - List operations = Patch.of(bytes).getOperations(); + List operations = mapper.toPatch(bytes).getOperations(); assertThat(operations, hasItem(new Replace(Path.parse("/person/number"), 42))); } @Test public void serializesTest() throws IOException { Patch patch = patchOf(new io.datanerds.avropatch.operation.Test(Path.parse("/person/number"), 42L)); - byte[] bytes = patch.toBytes(); + byte[] bytes = mapper.toBytes(patch); - List operations = Patch.of(bytes).getOperations(); + List operations = mapper.toPatch(bytes).getOperations(); assertThat(operations, hasSize(1)); assertThat(operations, hasItem(new io.datanerds.avropatch.operation.Test(Path.parse("/person/number"), 42L))); } @@ -90,8 +93,8 @@ public void serializesBunchOfOperations() throws IOException { new Replace(Path.parse("/person/number"), 42), new io.datanerds.avropatch.operation.Test(Path.parse("/person/number"), 42L))); - byte[] bytes = patch.toBytes(); - assertThat(patch, is(equalTo(Patch.of(bytes)))); + byte[] bytes = mapper.toBytes(patch); + assertThat(patch, is(equalTo(mapper.toPatch(bytes)))); } @Test @@ -109,8 +112,8 @@ public void serializesDefaultValueTypes() throws IOException { new Add<>(Path.of("some", "value"), date), new Add<>(Path.of("some", "value"), 4234.2345))); - byte[] bytes = patch.toBytes(); - assertThat(patch, is(equalTo(Patch.of(bytes)))); + byte[] bytes = mapper.toBytes(patch); + assertThat(patch, is(equalTo(mapper.toPatch(bytes)))); } @Test @@ -121,8 +124,8 @@ public void serializesArbitraryHeadersWithoutOperations() throws IOException { "header 2", new Date(), "header 3", 1234L, "header 4", new BigDecimal("3214123453.123512345"))); - byte[] bytes = patch.toBytes(); - assertThat(patch, is(equalTo(Patch.of(bytes)))); + byte[] bytes = mapper.toBytes(patch); + assertThat(patch, is(equalTo(mapper.toPatch(bytes)))); } @Test @@ -135,8 +138,8 @@ public void serializesArbitraryHeadersWithOperations() throws IOException { "header 2", new Date(), "header 3", 1234L, "header 4", new BigDecimal("3214123453.123512345"))); - byte[] bytes = patch.toBytes(); - assertThat(patch, is(equalTo(Patch.of(bytes)))); + byte[] bytes = mapper.toBytes(patch); + assertThat(patch, is(equalTo(mapper.toPatch(bytes)))); } private Patch patchOf(T operation) { diff --git a/src/test/java/io/datanerds/avropatch/serialization/CustomTypeSerializerTest.java b/src/test/java/io/datanerds/avropatch/serialization/PatchMapperTest.java similarity index 82% rename from src/test/java/io/datanerds/avropatch/serialization/CustomTypeSerializerTest.java rename to src/test/java/io/datanerds/avropatch/serialization/PatchMapperTest.java index e99555e..1afb1d6 100644 --- a/src/test/java/io/datanerds/avropatch/serialization/CustomTypeSerializerTest.java +++ b/src/test/java/io/datanerds/avropatch/serialization/PatchMapperTest.java @@ -18,19 +18,19 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -public class CustomTypeSerializerTest { +public class PatchMapperTest { @Test public void withCustomTypes() throws IOException { - CustomTypeSerializer serializer = new CustomTypeSerializer.Builder().withCustomTypes().build(); + PatchMapper serializer = new PatchMapper.Builder().withCustomTypes().build(); Patch patch = new Patch(ImmutableList.of(new Add(Path.of("hello"), new BigDecimal("23946712384.49879324")))); byte[] bytes = serializer.toBytes(patch); - assertThat(patch, is(equalTo(serializer.toObject(bytes)))); + assertThat(patch, is(equalTo(serializer.toPatch(bytes)))); } @Test public void withCustomClass() throws IOException { - CustomTypeSerializer serializer = new CustomTypeSerializer.Builder() + PatchMapper serializer = new PatchMapper.Builder() .withArray() .nullable() .withPrimitives() @@ -41,13 +41,13 @@ public void withCustomClass() throws IOException { Patch patch = new Patch(ImmutableList.of( new Replace(Path.of("hello"), new Bimmel("string", 42, UUID.randomUUID(), new Bimmel.Bommel("Gaga"))))); byte[] bytes = serializer.toBytes(patch); - assertThat(patch, is(equalTo(serializer.toObject(bytes)))); + assertThat(patch, is(equalTo(serializer.toPatch(bytes)))); } @Test public void withMultipleOperations() throws IOException { - CustomTypeSerializer serializer = new CustomTypeSerializer.Builder() + PatchMapper serializer = new PatchMapper.Builder() .withArray() .withPrimitives() .withCustomTypes() @@ -64,6 +64,6 @@ public void withMultipleOperations() throws IOException { new Add(Path.of("oO"), ImmutableList.of(new BigInteger("897696124"), 42, new BigInteger("4796923435"))), new Replace(Path.of("lol"), ImmutableList.of(new Date(), new Date(), new Date())))); byte[] bytes = serializer.toBytes(patch); - assertThat(patch, is(equalTo(serializer.toObject(bytes)))); + assertThat(patch, is(equalTo(serializer.toPatch(bytes)))); } } \ No newline at end of file From d5189d3a12de590d6dce0141070c9bf5369505f0 Mon Sep 17 00:00:00 2001 From: Frank Wisniewski Date: Fri, 6 Jan 2017 14:39:50 -0800 Subject: [PATCH 17/19] Refactored builders for the PatchMapper. --- .../avropatch/serialization/PatchMapper.java | 134 ++++-------------- .../avropatch/value/ValueSchemaBuilder.java | 121 ++++++++++++++++ .../serialization/ConcurrencyTest.java | 24 ++-- .../serialization/PatchMapperTest.java | 26 ++-- .../value/conversion/UUIDConversionTest.java | 1 - 5 files changed, 177 insertions(+), 129 deletions(-) create mode 100644 src/main/java/io/datanerds/avropatch/value/ValueSchemaBuilder.java diff --git a/src/main/java/io/datanerds/avropatch/serialization/PatchMapper.java b/src/main/java/io/datanerds/avropatch/serialization/PatchMapper.java index 01c7aff..ece3ddb 100644 --- a/src/main/java/io/datanerds/avropatch/serialization/PatchMapper.java +++ b/src/main/java/io/datanerds/avropatch/serialization/PatchMapper.java @@ -2,37 +2,32 @@ import io.datanerds.avropatch.Patch; import io.datanerds.avropatch.value.AvroData; -import io.datanerds.avropatch.value.type.BigDecimalType; -import io.datanerds.avropatch.value.type.BigIntegerType; -import io.datanerds.avropatch.value.type.TimestampType; -import io.datanerds.avropatch.value.type.UuidType; +import io.datanerds.avropatch.value.ValueSchemaBuilder; import org.apache.avro.Schema; import org.apache.avro.io.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import javax.annotation.concurrent.ThreadSafe; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.lang.reflect.Type; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static io.datanerds.avropatch.value.type.PrimitiveType.*; +/** + * This mapper provides functionality for converting between {@link Patch} objects and {@code byte}[] facilitating Avro. + */ @ThreadSafe public final class PatchMapper { - - private static Logger logger = LoggerFactory.getLogger(PatchMapper.class); - private final DatumWriter writer; private final DatumReader reader; + /** + * Constructs the default {@link PatchMapper} with support for values of type {@link Boolean}, {@link Double}, + * {@link Integer}, {@link Long}, {@link String}, {@link java.math.BigDecimal}, {@link java.math.BigInteger}, + * {@link java.util.Date} and {@link java.util.UUID}. + */ public PatchMapper() { this(AvroData.get().getSchema(Patch.class)); } + @SuppressWarnings("unchecked") // Assignment is safe since we're always passing in a Patch schema private PatchMapper(Schema schema) { this.writer = AvroData.get().createDatumWriter(schema); this.reader = AvroData.get().createDatumReader(schema); @@ -49,102 +44,33 @@ public Patch toPatch(byte[] bytes) throws IOException { return reader.read(null, DecoderFactory.get().binaryDecoder(bytes, null)); } - public static class Builder { - protected final List types; - - public Builder() { - this.types = new ArrayList<>(); - } - - public Builder nullable() { - types.add(NULL.getSchema()); - return this; - } - - public Builder withPrimitives() { - types.addAll(Arrays.asList(BOOLEAN.getSchema(), DOUBLE.getSchema(), FLOAT.getSchema(), INTEGER.getSchema(), - LONG.getSchema(), STRING.getSchema())); - return this; - } - - public Builder withCustomTypes() { - types.addAll(Arrays.asList(BigDecimalType.SCHEMA, BigIntegerType.SCHEMA, TimestampType.SCHEMA, UuidType.SCHEMA)); - return this; - } - - public Builder withType(Type type) { - types.add(AvroData.get().getSchema(type)); - return this; - } - - public Builder with(Schema schema) { - types.add(schema); - return this; - } + public static final ValueSchemaBuilder builder() { + return new Builder(); + } - public PatchMapper build() { - Schema schema = makeSchema(); - logger.debug("Created serializer for schema {}", schema); - return new PatchMapper(schema); - } + public static final ValueSchemaBuilder arrayBuilder() { + return new ArraySchemaBuilder(); + } - private Schema makeSchema() { - if (types.size() == 1) { - return PatchType.create(types.get(0)); - } else { - return PatchType.create(Schema.createUnion(types)); - } - } + public static final ValueSchemaBuilder mapBuilder() { + return new MapSchemaBuilder(); + } - private Builder endSubBuilder(Schema schema) { - types.add(schema); - return this; + private static final class Builder extends ValueSchemaBuilder { + private Builder() { + super(schema -> new PatchMapper(PatchType.create(schema))); } + } - public Builder.ArrayBuilder withArray() { - return new Builder.ArrayBuilder(this); + private static class ArraySchemaBuilder extends ValueSchemaBuilder { + private ArraySchemaBuilder() { + super(schema -> Schema.createArray(schema)); } + } - public class ArrayBuilder extends Builder { - private final Builder parentBuilder; - - ArrayBuilder(Builder parentBuilder) { - this.parentBuilder = parentBuilder; - } - - public Builder endArray() { - return parentBuilder.endSubBuilder(Schema.createArray(Schema.createUnion(types))); - } - - @Override - public Builder.ArrayBuilder nullable() { - super.nullable(); - return this; - } - - @Override - public Builder.ArrayBuilder withPrimitives() { - super.withPrimitives(); - return this; - } - - @Override - public Builder.ArrayBuilder withCustomTypes() { - super.withCustomTypes(); - return this; - } - - @Override - public Builder.ArrayBuilder withType(Type type) { - super.withType(type); - return this; - } - - @Override - public Builder.ArrayBuilder with(Schema schema) { - super.with(schema); - return this; - } + private static class MapSchemaBuilder extends ValueSchemaBuilder { + private MapSchemaBuilder() { + super(schema -> Schema.createMap(schema)); } } } diff --git a/src/main/java/io/datanerds/avropatch/value/ValueSchemaBuilder.java b/src/main/java/io/datanerds/avropatch/value/ValueSchemaBuilder.java new file mode 100644 index 0000000..6f7a32f --- /dev/null +++ b/src/main/java/io/datanerds/avropatch/value/ValueSchemaBuilder.java @@ -0,0 +1,121 @@ +package io.datanerds.avropatch.value; + +import io.datanerds.avropatch.serialization.PatchMapper; +import io.datanerds.avropatch.value.type.BigDecimalType; +import io.datanerds.avropatch.value.type.BigIntegerType; +import io.datanerds.avropatch.value.type.TimestampType; +import io.datanerds.avropatch.value.type.UuidType; +import org.apache.avro.Schema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; + +import static io.datanerds.avropatch.value.type.PrimitiveType.*; + +/** + * This abstract class provides a builder skeleton for constructing an object depending on an Avro {@link Schema} of + * type {@code Union}. E.g., it enables defining the Avro {@link Schema} used to describe the generic type for the + * values of {@link io.datanerds.avropatch.operation.Add}, {@link io.datanerds.avropatch.operation.Replace} and + * {@link io.datanerds.avropatch.operation.Test} operations by passing in the appropriate functional to the constructor. + * Essentially, it creates a {@link org.apache.avro.Schema.UnionSchema} holding all types specified via its API and + * calls the instantiator functional with that particualr {@link Schema} instance. + * @see PatchMapper PatchMapper for implementations. + */ +public abstract class ValueSchemaBuilder { + private static Logger logger = LoggerFactory.getLogger(ValueSchemaBuilder.class); + private final Function instantiator; + private final List types; + + protected ValueSchemaBuilder(Function instantiator) { + Objects.nonNull(instantiator); + this.instantiator = instantiator; + this.types = new ArrayList<>(); + } + + public T build() { + Schema schema = makeSchema(); + logger.debug("Generated schema {}", schema); + return instantiator.apply(schema); + } + + /** + * Allows values for {@link io.datanerds.avropatch.operation.Add}, + * {@link io.datanerds.avropatch.operation.Replace} and {@link io.datanerds.avropatch.operation.Test} to be + * nullable. + * + * @return the builder instance + */ + public ValueSchemaBuilder nullable() { + types.add(NULL.getSchema()); + return this; + } + + /** + * Adds primitives supported by Avro out of the box to the {@link Schema} defining the value types for + * {@link io.datanerds.avropatch.operation.Add}, {@link io.datanerds.avropatch.operation.Replace} and + * {@link io.datanerds.avropatch.operation.Test}, namely {@link Boolean}, {@link Double}, {@link Integer}, + * {@link Long} and {@link String}. + * + * @return the builder instance + */ + public ValueSchemaBuilder withAvroPrimitives() { + types.addAll(Arrays.asList(BOOLEAN.getSchema(), DOUBLE.getSchema(), FLOAT.getSchema(), INTEGER.getSchema(), + LONG.getSchema(), STRING.getSchema())); + return this; + } + + /** + * Adds custom types to the {@link Schema} defining the value types for + * {@link io.datanerds.avropatch.operation.Add}, {@link io.datanerds.avropatch.operation.Replace} and + * {@link io.datanerds.avropatch.operation.Test} with custom converters registered in {@link PatchMapper}, + * namely {@link java.math.BigDecimal}, {@link java.math.BigInteger}, {@link java.util.Date} and + * {@link java.util.UUID}. + * + * @return the builder instance + * @see io.datanerds.avropatch.value.conversion.BigDecimalConversion + * @see io.datanerds.avropatch.value.conversion.BigIntegerConversion + * @see io.datanerds.avropatch.value.conversion.DateConversion + * @see io.datanerds.avropatch.value.conversion.UUIDConversion + */ + public ValueSchemaBuilder withCustomTypes() { + types.addAll( + Arrays.asList(BigDecimalType.SCHEMA, BigIntegerType.SCHEMA, TimestampType.SCHEMA, UuidType.SCHEMA)); + return this; + } + + /** + * Generates a {@link Schema} for the given type via reflections and adds it to set of supported types. + * @param type to be added to the {@code Union} + * @return the builder instance + */ + public ValueSchemaBuilder withType(Type type) { + return with(AvroData.get().getSchema(type)); + } + + /** + * Adds the schema to the {@link org.apache.avro.Schema.UnionSchema} defining the value types for + * {@link io.datanerds.avropatch.operation.Add}, {@link io.datanerds.avropatch.operation.Replace} and + * {@link io.datanerds.avropatch.operation.Test}. Be aware that you might need to register your + * {@link org.apache.avro.Conversion} implementation if the {@code schema} contains non default types. + * @param schema to be added to the {@code Union} + * @return the builder instance + */ + public ValueSchemaBuilder with(Schema schema) { + types.add(schema); + return this; + } + + private Schema makeSchema() { + if (types.size() == 1) { + return types.get(0); + } else { + return Schema.createUnion(types); + } + } +} \ No newline at end of file diff --git a/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java b/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java index dcd2ee7..7a19c69 100644 --- a/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java +++ b/src/test/java/io/datanerds/avropatch/serialization/ConcurrencyTest.java @@ -12,6 +12,8 @@ import java.util.concurrent.*; import static io.datanerds.avropatch.operation.matcher.PatchMatcher.equalTo; +import static io.datanerds.avropatch.serialization.PatchMapper.arrayBuilder; +import static io.datanerds.avropatch.serialization.PatchMapper.builder; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; @@ -24,17 +26,17 @@ public class ConcurrencyTest { private static final int MAX_PATCH_SIZE = 50; private static final List PATCHES = Collections.unmodifiableList(generateData()); - private static final PatchMapper SERIALIZER = new PatchMapper.Builder() - .withArray() - .withPrimitives() - .withCustomTypes() - .nullable() - .withType(Bimmel.class) - .endArray() - .withPrimitives() - .withCustomTypes() - .withType(Bimmel.class) - .nullable() + private static final PatchMapper SERIALIZER = builder() + .with(arrayBuilder() + .withAvroPrimitives() + .withCustomTypes() + .nullable() + .withType(Bimmel.class) + .build()) + .withAvroPrimitives() + .withCustomTypes() + .withType(Bimmel.class) + .nullable() .build(); @Test diff --git a/src/test/java/io/datanerds/avropatch/serialization/PatchMapperTest.java b/src/test/java/io/datanerds/avropatch/serialization/PatchMapperTest.java index 1afb1d6..a20d1c2 100644 --- a/src/test/java/io/datanerds/avropatch/serialization/PatchMapperTest.java +++ b/src/test/java/io/datanerds/avropatch/serialization/PatchMapperTest.java @@ -22,7 +22,7 @@ public class PatchMapperTest { @Test public void withCustomTypes() throws IOException { - PatchMapper serializer = new PatchMapper.Builder().withCustomTypes().build(); + PatchMapper serializer = PatchMapper.builder().withCustomTypes().build(); Patch patch = new Patch(ImmutableList.of(new Add(Path.of("hello"), new BigDecimal("23946712384.49879324")))); byte[] bytes = serializer.toBytes(patch); assertThat(patch, is(equalTo(serializer.toPatch(bytes)))); @@ -30,12 +30,12 @@ public void withCustomTypes() throws IOException { @Test public void withCustomClass() throws IOException { - PatchMapper serializer = new PatchMapper.Builder() - .withArray() - .nullable() - .withPrimitives() - .withCustomTypes() - .endArray() + PatchMapper serializer = PatchMapper.builder() + .with(PatchMapper.arrayBuilder() + .nullable() + .withAvroPrimitives() + .withCustomTypes() + .build()) .withType(Bimmel.class) .build(); Patch patch = new Patch(ImmutableList.of( @@ -47,13 +47,13 @@ public void withCustomClass() throws IOException { @Test public void withMultipleOperations() throws IOException { - PatchMapper serializer = new PatchMapper.Builder() - .withArray() - .withPrimitives() - .withCustomTypes() - .endArray() + PatchMapper serializer = PatchMapper.builder() + .with(PatchMapper.arrayBuilder() + .withAvroPrimitives() + .withCustomTypes() + .build()) .nullable() - .withPrimitives() + .withAvroPrimitives() .withType(Bimmel.class) .build(); Patch patch = new Patch(ImmutableList.of( diff --git a/src/test/java/io/datanerds/avropatch/value/conversion/UUIDConversionTest.java b/src/test/java/io/datanerds/avropatch/value/conversion/UUIDConversionTest.java index 5f65eaa..2065b5f 100644 --- a/src/test/java/io/datanerds/avropatch/value/conversion/UUIDConversionTest.java +++ b/src/test/java/io/datanerds/avropatch/value/conversion/UUIDConversionTest.java @@ -16,7 +16,6 @@ public void serializesSingleValue() throws IOException { ConversionTester .withSchemata(UuidType.SCHEMA) .withConverters(new UUIDConversion()) - .reserializeAndAssert(UUID.randomUUID()) .reserializeAndAssert(UUID.randomUUID()); } From 8a10dc22456f197860c68f54524734a5fa95d939 Mon Sep 17 00:00:00 2001 From: Frank Wisniewski Date: Mon, 9 Jan 2017 07:59:28 -0800 Subject: [PATCH 18/19] Abstraced builder for PatchMapper, array schema and map schema. --- .../avropatch/serialization/PatchMapper.java | 82 ++++++++++++++----- .../avropatch/value/ValueSchemaBuilder.java | 36 ++++---- 2 files changed, 83 insertions(+), 35 deletions(-) diff --git a/src/main/java/io/datanerds/avropatch/serialization/PatchMapper.java b/src/main/java/io/datanerds/avropatch/serialization/PatchMapper.java index ece3ddb..32da7c1 100644 --- a/src/main/java/io/datanerds/avropatch/serialization/PatchMapper.java +++ b/src/main/java/io/datanerds/avropatch/serialization/PatchMapper.java @@ -44,33 +44,73 @@ public Patch toPatch(byte[] bytes) throws IOException { return reader.read(null, DecoderFactory.get().binaryDecoder(bytes, null)); } + /** + * Instantiates a builder for creating a {@link PatchMapper} which may be used as follows. + * + * 

+     * PatchMapper mapper = PatchMapper.builder()
+     *          .nullable()
+     *          .withAvroPrimitives()
+     *          .withCustomTypes()
+     *          .build();
+     * Patch patch = new Patch(...);
+     * byte[] bytes = serializer.toBytes(patch);
+     *

+ * @see ValueSchemaBuilder + * @see PatchMapper + * @return the builder instance + */ public static final ValueSchemaBuilder builder() { - return new Builder(); + return new ValueSchemaBuilder<>(schema -> new PatchMapper(PatchType.create(schema))); } + /** + * Instantiates a builder for creating an Avro {@link Schema} of type {@code array} which may be used for the + * generic value of {@link io.datanerds.avropatch.operation.Add}, {@link io.datanerds.avropatch.operation.Replace} + * and {@link io.datanerds.avropatch.operation.Test} operations by registering the resulting {@link Schema} to a + * {@link PatchMapper}. + * + * 

+     * PatchMapper mapper = PatchMapper.builder()
+     *          .with(PatchMapper.arrayBuilder()
+     *                  .nullable()
+     *                  .withAvroPrimitives()
+     *                  .build())
+     *          .build();
+     * Patch patch = new Patch(...);
+     * byte[] bytes = serializer.toBytes(patch);
+     *

+ * @see ValueSchemaBuilder + * @see Schema + * @see #builder() + * @return the builder instance + */ public static final ValueSchemaBuilder arrayBuilder() { - return new ArraySchemaBuilder(); + return new ValueSchemaBuilder<>(schema -> Schema.createArray(schema)); } + /** + * Instantiates a builder for creating an Avro {@link Schema} of type {@code map} which may be used for the + * generic value of {@link io.datanerds.avropatch.operation.Add}, {@link io.datanerds.avropatch.operation.Replace} + * and {@link io.datanerds.avropatch.operation.Test} operations by registering the resulting {@link Schema} to a + * {@link PatchMapper}. + * + * 

+     * PatchMapper mapper = PatchMapper.builder()
+     *          .with(PatchMapper.mapBuilder()
+     *                  .nullable()
+     *                  .withAvroPrimitives()
+     *                  .build())
+     *          .build();
+     * Patch patch = new Patch(...);
+     * byte[] bytes = serializer.toBytes(patch);
+     *

+ * @see ValueSchemaBuilder + * @see Schema + * @see #builder() + * @return the builder instance + */ public static final ValueSchemaBuilder mapBuilder() { - return new MapSchemaBuilder(); - } - - private static final class Builder extends ValueSchemaBuilder { - private Builder() { - super(schema -> new PatchMapper(PatchType.create(schema))); - } - } - - private static class ArraySchemaBuilder extends ValueSchemaBuilder { - private ArraySchemaBuilder() { - super(schema -> Schema.createArray(schema)); - } - } - - private static class MapSchemaBuilder extends ValueSchemaBuilder { - private MapSchemaBuilder() { - super(schema -> Schema.createMap(schema)); - } + return new ValueSchemaBuilder<>(schema -> Schema.createMap(schema)); } } diff --git a/src/main/java/io/datanerds/avropatch/value/ValueSchemaBuilder.java b/src/main/java/io/datanerds/avropatch/value/ValueSchemaBuilder.java index 6f7a32f..6db23cc 100644 --- a/src/main/java/io/datanerds/avropatch/value/ValueSchemaBuilder.java +++ b/src/main/java/io/datanerds/avropatch/value/ValueSchemaBuilder.java @@ -1,5 +1,6 @@ package io.datanerds.avropatch.value; +import avro.shaded.com.google.common.collect.Lists; import io.datanerds.avropatch.serialization.PatchMapper; import io.datanerds.avropatch.value.type.BigDecimalType; import io.datanerds.avropatch.value.type.BigIntegerType; @@ -10,10 +11,7 @@ import org.slf4j.LoggerFactory; import java.lang.reflect.Type; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; +import java.util.*; import java.util.function.Function; import static io.datanerds.avropatch.value.type.PrimitiveType.*; @@ -23,19 +21,19 @@ * type {@code Union}. E.g., it enables defining the Avro {@link Schema} used to describe the generic type for the * values of {@link io.datanerds.avropatch.operation.Add}, {@link io.datanerds.avropatch.operation.Replace} and * {@link io.datanerds.avropatch.operation.Test} operations by passing in the appropriate functional to the constructor. - * Essentially, it creates a {@link org.apache.avro.Schema.UnionSchema} holding all types specified via its API and - * calls the instantiator functional with that particualr {@link Schema} instance. - * @see PatchMapper PatchMapper for implementations. + * Essentially, it creates a {@code UnionSchema} holding all types specified via its API and + * calls the instantiator functional with that particular {@link Schema} instance. + * @see PatchMapper PatchMapper for usage. */ -public abstract class ValueSchemaBuilder { +public class ValueSchemaBuilder { private static Logger logger = LoggerFactory.getLogger(ValueSchemaBuilder.class); private final Function instantiator; - private final List types; + private final Set types; - protected ValueSchemaBuilder(Function instantiator) { + public ValueSchemaBuilder(Function instantiator) { Objects.nonNull(instantiator); this.instantiator = instantiator; - this.types = new ArrayList<>(); + this.types = new HashSet<>(); } public T build() { @@ -89,6 +87,16 @@ public ValueSchemaBuilder withCustomTypes() { return this; } + /** + * Shortcut for {@code withAvroPrimitives().withCustomTypes()}. + * @see #withCustomTypes() + * @see #withAvroPrimitives() + * @return the builder instance + */ + public ValueSchemaBuilder withSupportedTypes() { + return withAvroPrimitives().withCustomTypes(); + } + /** * Generates a {@link Schema} for the given type via reflections and adds it to set of supported types. * @param type to be added to the {@code Union} @@ -99,7 +107,7 @@ public ValueSchemaBuilder withType(Type type) { } /** - * Adds the schema to the {@link org.apache.avro.Schema.UnionSchema} defining the value types for + * Adds the schema to the {@code UnionSchema} defining the value types for * {@link io.datanerds.avropatch.operation.Add}, {@link io.datanerds.avropatch.operation.Replace} and * {@link io.datanerds.avropatch.operation.Test}. Be aware that you might need to register your * {@link org.apache.avro.Conversion} implementation if the {@code schema} contains non default types. @@ -113,9 +121,9 @@ public ValueSchemaBuilder with(Schema schema) { private Schema makeSchema() { if (types.size() == 1) { - return types.get(0); + return types.iterator().next(); } else { - return Schema.createUnion(types); + return Schema.createUnion(Lists.newArrayList(types)); } } } \ No newline at end of file From b9f9eed77747269765ec11de50c7ba6015587aee Mon Sep 17 00:00:00 2001 From: Frank Wisniewski Date: Mon, 9 Jan 2017 15:27:35 -0800 Subject: [PATCH 19/19] Added convinience methods to Path. --- .../exception/InvalidPathException.java | 8 -- .../InvalidReferenceTokenException.java | 8 ++ .../datanerds/avropatch/operation/Path.java | 132 +++++++++++++++--- .../avropatch/serialization/PatchMapper.java | 17 +-- .../avropatch/serialization/PatchType.java | 23 --- .../{OperationTypes.java => Types.java} | 65 ++++++--- .../avropatch/operation/PathTest.java | 20 ++- .../OperationSerializationTester.java | 15 +- .../SerializationTester.java | 2 +- .../TypesTest.java} | 6 +- 10 files changed, 197 insertions(+), 99 deletions(-) delete mode 100644 src/main/java/io/datanerds/avropatch/exception/InvalidPathException.java create mode 100644 src/main/java/io/datanerds/avropatch/exception/InvalidReferenceTokenException.java delete mode 100644 src/main/java/io/datanerds/avropatch/serialization/PatchType.java rename src/main/java/io/datanerds/avropatch/serialization/{OperationTypes.java => Types.java} (50%) rename src/test/java/io/datanerds/avropatch/{schema => serialization}/OperationSerializationTester.java (84%) rename src/test/java/io/datanerds/avropatch/{schema => serialization}/SerializationTester.java (97%) rename src/test/java/io/datanerds/avropatch/{schema/OperationTypesTest.java => serialization/TypesTest.java} (89%) diff --git a/src/main/java/io/datanerds/avropatch/exception/InvalidPathException.java b/src/main/java/io/datanerds/avropatch/exception/InvalidPathException.java deleted file mode 100644 index ce4657d..0000000 --- a/src/main/java/io/datanerds/avropatch/exception/InvalidPathException.java +++ /dev/null @@ -1,8 +0,0 @@ -package io.datanerds.avropatch.exception; - -public class InvalidPathException extends AvroPatchException { - - public InvalidPathException(String message) { - super(message); - } -} diff --git a/src/main/java/io/datanerds/avropatch/exception/InvalidReferenceTokenException.java b/src/main/java/io/datanerds/avropatch/exception/InvalidReferenceTokenException.java new file mode 100644 index 0000000..dfef5f8 --- /dev/null +++ b/src/main/java/io/datanerds/avropatch/exception/InvalidReferenceTokenException.java @@ -0,0 +1,8 @@ +package io.datanerds.avropatch.exception; + +public class InvalidReferenceTokenException extends AvroPatchException { + + public InvalidReferenceTokenException(String message) { + super(message); + } +} diff --git a/src/main/java/io/datanerds/avropatch/operation/Path.java b/src/main/java/io/datanerds/avropatch/operation/Path.java index da30029..58f3536 100644 --- a/src/main/java/io/datanerds/avropatch/operation/Path.java +++ b/src/main/java/io/datanerds/avropatch/operation/Path.java @@ -1,20 +1,57 @@ package io.datanerds.avropatch.operation; -import io.datanerds.avropatch.exception.InvalidPathException; +import io.datanerds.avropatch.exception.InvalidReferenceTokenException; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.function.Function; import java.util.regex.Pattern; - +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * This class represents a JavaScript Object Notation (JSON) Pointer object as defined in RFC 6901. {@link Path} + * instances identify a specific value within a JSON document by maintaining a sequence of zero or more reference + * tokens, each prefixed by a {@code '/'} character. + *

+ * Assume the following JSON document: + * 

+ * {
+ *    "employees" : [
+ *       {
+ *          "firstName" : "John",
+ *          "lastName"  : "Doe",
+ *          "age"       : 42,
+ *          "address"   : {
+ *             "street" : "1234 Main Street",
+ *             "city"   : "San Francisco",
+ *             "zip"    : "94114"
+ *          }
+ *       },
+ *       {
+ *          "firstName" : "Jane",
+ *          "lastName"  : "Doe",
+ *          "age"       : 42,
+ *          "address"   : {
+ *             "street" : "4321 Main Street",
+ *             "city"   : "New York",
+ *             "zip"    : "10001"
+ *          }
+ *       }
+ *    ]
+ * }
+ *
+ *

+ * The JSON {@link Path} for addressing value "John" would be {@code "/employees/0/firstName"}, for the Jane's entire + * record {@code "/employees/1"} and to specify the street within the JSON blob representing her address + * {@code "/employees/1/address"}. + * @see RFC 6901: JavaScript Object Notation (JSON) Pointer + */ public class Path { - public static final String SLASH = "/"; public static final Path ROOT = new Path(); - private static final Pattern VALID_PATTERN = Pattern.compile("[0-9a-zA-Z][0-9a-zA-Z_-]*"); - private final List parts; private Path() { @@ -25,12 +62,42 @@ private Path(List parts) { this.parts = Collections.unmodifiableList(parts); } - public static Path of(String... parts) { - List paths = Arrays.asList(parts); - paths.forEach(Path::validateSubPath); + /** + * Convenient method for retrieving root path {@code "/"}. + * @return {@link #ROOT} + */ + public static Path of() { + return ROOT; + } + + /** + * Constructs a {@link Path} object for given reference tokens. + * @param referenceTokens {@link String} representation of all reference tokens for {@link Path} without prefix + * {@code "/"} + * @return JSON path assembled via given reference tokens + */ + public static Path of(String... referenceTokens) { + return new Path(validatedReferenceTokens(referenceTokens).collect(Collectors.toList())); + } + + /** + * Constructs a {@link Path} object by concatenating given paths. + * @param subpaths paths to be concatenated for new {@link Path} object + * @return JSON path assembled by concatenating given paths + */ + public static Path of(Path... subpaths) { + List paths = Stream.of(subpaths) + .flatMap(Path.partsStream()) + .collect(Collectors.toList()); return new Path(paths); } + /** + * Parses a {@link String} representation of a JSON path (e.g. {@code "/employees/0/firstName"}, + * {@code "/"} or {@code "/employees/1/address"}). + * @param path valid {@link String} representation of a JSON path + * @return JSON path instance + */ public static Path parse(String path) { verifyParsable(path); if (SLASH.equals(path)) { @@ -39,11 +106,28 @@ public static Path parse(String path) { return of(path.substring(1).split(SLASH)); } - private static void verifyParsable(String path) { - Objects.nonNull(path); - if (!path.startsWith(SLASH)) { - throw new InvalidPathException("JSON Path has to start with a slash."); - } + /** + * Constructs a new {@link Path} object by appending the given subpaths to the JSON path represented by this object. + * @param subpaths {@link Path} objects to be appended + * @return concatenated {@link Path} + */ + public Path append(Path... subpaths) { + List paths = Stream.concat(this.parts().stream(), Stream.of(subpaths).flatMap(partsStream())) + .collect(Collectors.toList()); + return new Path(paths); + } + + /** + * Constructs a new {@link Path} object by appending the JSON path specified by the given reference tokens to the + * path represented by this instance. + * @param referenceTokens {@link String} representation of all reference tokens for {@link Path} without prefix + * {@code "/"} + * @return concatenated {@link Path} + */ + public Path append(String... referenceTokens) { + List paths = Stream.concat(this.parts().stream(), validatedReferenceTokens(referenceTokens)) + .collect(Collectors.toList()); + return new Path(paths); } public Path parent() { @@ -57,9 +141,25 @@ public List parts() { return parts; } - private static void validateSubPath(String path) { + private static Function> partsStream() { + return path -> path.parts().stream(); + } + + private static Stream validatedReferenceTokens(String[] referenceTokens) { + return Stream.of(referenceTokens).map(Path::validateReferenceToken); + } + + private static String validateReferenceToken(String path) { if (!VALID_PATTERN.matcher(path).matches()) { - throw new InvalidPathException(String.format("%s is not a valid JSON path", path)); + throw new InvalidReferenceTokenException(String.format("%s is not a valid JSON path", path)); + } + return path; + } + + private static void verifyParsable(String path) { + Objects.nonNull(path); + if (!path.startsWith(SLASH)) { + throw new InvalidReferenceTokenException("JSON Path has to start with a slash."); } } diff --git a/src/main/java/io/datanerds/avropatch/serialization/PatchMapper.java b/src/main/java/io/datanerds/avropatch/serialization/PatchMapper.java index 32da7c1..af9a9f8 100644 --- a/src/main/java/io/datanerds/avropatch/serialization/PatchMapper.java +++ b/src/main/java/io/datanerds/avropatch/serialization/PatchMapper.java @@ -46,8 +46,7 @@ public Patch toPatch(byte[] bytes) throws IOException { /** * Instantiates a builder for creating a {@link PatchMapper} which may be used as follows. - * - * 

+     * 
      * PatchMapper mapper = PatchMapper.builder()
      *          .nullable()
      *          .withAvroPrimitives()
@@ -55,13 +54,13 @@ public Patch toPatch(byte[] bytes) throws IOException {
      *          .build();
      * Patch patch = new Patch(...);
      * byte[] bytes = serializer.toBytes(patch);
-     * 

+     *
* @see ValueSchemaBuilder * @see PatchMapper * @return the builder instance */ public static final ValueSchemaBuilder builder() { - return new ValueSchemaBuilder<>(schema -> new PatchMapper(PatchType.create(schema))); + return new ValueSchemaBuilder<>(schema -> new PatchMapper(Types.Patch.create(schema))); } /** @@ -69,8 +68,7 @@ public static final ValueSchemaBuilder builder() { * generic value of {@link io.datanerds.avropatch.operation.Add}, {@link io.datanerds.avropatch.operation.Replace} * and {@link io.datanerds.avropatch.operation.Test} operations by registering the resulting {@link Schema} to a * {@link PatchMapper}. - * - * 

+     * 
      * PatchMapper mapper = PatchMapper.builder()
      *          .with(PatchMapper.arrayBuilder()
      *                  .nullable()
@@ -79,7 +77,7 @@ public static final ValueSchemaBuilder builder() {
      *          .build();
      * Patch patch = new Patch(...);
      * byte[] bytes = serializer.toBytes(patch);
-     * 

+     *
* @see ValueSchemaBuilder * @see Schema * @see #builder() @@ -94,8 +92,7 @@ public static final ValueSchemaBuilder arrayBuilder() { * generic value of {@link io.datanerds.avropatch.operation.Add}, {@link io.datanerds.avropatch.operation.Replace} * and {@link io.datanerds.avropatch.operation.Test} operations by registering the resulting {@link Schema} to a * {@link PatchMapper}. - * - * 

+     * 
      * PatchMapper mapper = PatchMapper.builder()
      *          .with(PatchMapper.mapBuilder()
      *                  .nullable()
@@ -104,7 +101,7 @@ public static final ValueSchemaBuilder arrayBuilder() {
      *          .build();
      * Patch patch = new Patch(...);
      * byte[] bytes = serializer.toBytes(patch);
-     * 

+     *
* @see ValueSchemaBuilder * @see Schema * @see #builder() diff --git a/src/main/java/io/datanerds/avropatch/serialization/PatchType.java b/src/main/java/io/datanerds/avropatch/serialization/PatchType.java deleted file mode 100644 index 2dcda6f..0000000 --- a/src/main/java/io/datanerds/avropatch/serialization/PatchType.java +++ /dev/null @@ -1,23 +0,0 @@ -package io.datanerds.avropatch.serialization; - -import io.datanerds.avropatch.Patch; -import org.apache.avro.Schema; -import java.util.Arrays; - -import static org.apache.avro.Schema.createRecord; - -/** - * This interface holds the schema constants for the {@link Patch} type. - * @see Patch - */ -public interface PatchType { - Object NO_DEFAULT = null; - String NAME = Patch.class.getSimpleName(); - String DOC = "This record represents a PATCH holding a sequence of operations to apply to a given object."; - static Schema create(Schema valueSchema) { - Schema.Field operations = new Schema.Field("operations", - Schema.createArray(OperationTypes.Operation.create(valueSchema)), - "Sequence of operations to apply to a given object.", NO_DEFAULT); - return createRecord(NAME, DOC, Patch.class.getPackage().getName(), false, Arrays.asList(operations)); - } -} diff --git a/src/main/java/io/datanerds/avropatch/serialization/OperationTypes.java b/src/main/java/io/datanerds/avropatch/serialization/Types.java similarity index 50% rename from src/main/java/io/datanerds/avropatch/serialization/OperationTypes.java rename to src/main/java/io/datanerds/avropatch/serialization/Types.java index a849594..0d9ec95 100644 --- a/src/main/java/io/datanerds/avropatch/serialization/OperationTypes.java +++ b/src/main/java/io/datanerds/avropatch/serialization/Types.java @@ -9,23 +9,24 @@ import static org.apache.avro.Schema.createRecord; /** - * This class holds schemata for all patch {@link io.datanerds.avropatch.operation.Operation}s. All schemata are - * bound to this library's domain/namespace. + * This class holds schemata for all patch {@link io.datanerds.avropatch.operation.Operation}s and + * {@link io.datanerds.avropatch.Patch}. All schemata are bound to this library's domain/namespace. */ -public interface OperationTypes { +interface Types { Object NO_DEFAULT = null; - String NAMESPACE = io.datanerds.avropatch.operation.Operation.class.getPackage().getName(); + String OPERATIONS_NAMESPACE = io.datanerds.avropatch.operation.Operation.class.getPackage().getName(); + String PATCH_NAMESPACE = io.datanerds.avropatch.Patch.class.getPackage().getName(); - interface PathType { + interface Path { String NAME = io.datanerds.avropatch.operation.Path.class.getSimpleName(); String DOC = "JSON Path serialized as String array holding its parts."; Schema SCHEMA = SchemaBuilder .record(NAME) .doc(DOC) - .namespace(NAMESPACE) + .namespace(OPERATIONS_NAMESPACE) .fields() - .name("parts").type(Schema.createArray(Schema.create(Type.STRING))).noDefault() + .name("parts").type(Schema.createArray(Schema.create(Type.STRING))).noDefault() .endRecord(); } @@ -34,10 +35,10 @@ interface Add { String DOC = "This record represents the add operation of RFC 6902 for JSON Patch'."; static Schema create(Schema valueSchema) { - Field path = new Field("path", PathType.SCHEMA, "Path pointing out where the JSON value should be added", + Field path = new Field("path", Path.SCHEMA, "Path pointing out where the JSON value should be added", NO_DEFAULT); Field value = new Field("value", valueSchema, "Actual value to add to patched object", NO_DEFAULT); - return createRecord(NAME, DOC, NAMESPACE, false, Arrays.asList(path, value)); + return createRecord(NAME, DOC, OPERATIONS_NAMESPACE, false, Arrays.asList(path, value)); } } @@ -46,9 +47,9 @@ interface Copy { String DOC = "This record represents the copy operation of RFC 6902 for JSON Patch'."; static Schema create() { - Field from = new Field("from", PathType.SCHEMA, "Source location of the value to be copied", NO_DEFAULT); - Field path = new Field("path", PathType.SCHEMA, "Target location of value to be copied", NO_DEFAULT); - return createRecord(NAME, DOC, NAMESPACE, false, Arrays.asList(from, path)); + Field from = new Field("from", Path.SCHEMA, "Source location of the value to be copied", NO_DEFAULT); + Field path = new Field("path", Path.SCHEMA, "Target location of value to be copied", NO_DEFAULT); + return createRecord(NAME, DOC, OPERATIONS_NAMESPACE, false, Arrays.asList(from, path)); } } @@ -57,9 +58,9 @@ interface Move { String DOC = "This record represents the move operation of RFC 6902 for JSON Patch'."; static Schema create() { - Field from = new Field("from", PathType.SCHEMA, "Source location of the value to be moved", NO_DEFAULT); - Field path = new Field("path", PathType.SCHEMA, "Target location for the value to be moved to", NO_DEFAULT); - return createRecord(NAME, DOC, NAMESPACE, false, Arrays.asList(from, path)); + Field from = new Field("from", Path.SCHEMA, "Source location of the value to be moved", NO_DEFAULT); + Field path = new Field("path", Path.SCHEMA, "Target location for the value to be moved to", NO_DEFAULT); + return createRecord(NAME, DOC, OPERATIONS_NAMESPACE, false, Arrays.asList(from, path)); } } @@ -68,8 +69,8 @@ interface Remove { String DOC = "This record represents the remove operation of RFC 6902 for JSON Patch'."; static Schema create() { - Field path = new Field("path", PathType.SCHEMA, "Target location of value to be removed", NO_DEFAULT); - return createRecord(NAME, DOC, NAMESPACE, false, Arrays.asList(path)); + Field path = new Field("path", Path.SCHEMA, "Target location of value to be removed", NO_DEFAULT); + return createRecord(NAME, DOC, OPERATIONS_NAMESPACE, false, Arrays.asList(path)); } } @@ -78,10 +79,10 @@ interface Replace { String DOC = "This record represents the replace operation of RFC 6902 for JSON Patch'."; static Schema create(Schema valueSchema) { - Field path = new Field("path", PathType.SCHEMA, "Path pointing out where the JSON value should be replaced", + Field path = new Field("path", Path.SCHEMA, "Path pointing out where the JSON value should be replaced", NO_DEFAULT); Field value = new Field("value", valueSchema, "Actual value to be replaced in patched object", NO_DEFAULT); - return createRecord(NAME, DOC, NAMESPACE, false, Arrays.asList(path, value)); + return createRecord(NAME, DOC, OPERATIONS_NAMESPACE, false, Arrays.asList(path, value)); } } @@ -90,10 +91,11 @@ interface Test { String DOC = "This record represents the test operation of RFC 6902 for JSON Patch'."; static Schema create(Schema valueSchema) { - Field path = new Field("path", PathType.SCHEMA, "Path pointing out which JSON value should be tested against", - NO_DEFAULT); + Field path = + new Field("path", Path.SCHEMA, "Path pointing out which JSON value should be tested against", + NO_DEFAULT); Field value = new Field("value", valueSchema, "Actual value to test against", NO_DEFAULT); - return createRecord(NAME, DOC, NAMESPACE, false, Arrays.asList(path, value)); + return createRecord(NAME, DOC, OPERATIONS_NAMESPACE, false, Arrays.asList(path, value)); } } @@ -104,4 +106,21 @@ static Schema create(Schema valueSchema) { } } -} + /** + * This interface holds the schema constants for the {@link io.datanerds.avropatch.Patch} type. + * + * @see io.datanerds.avropatch.Patch + */ + interface Patch { + String NAME = io.datanerds.avropatch.Patch.class.getSimpleName(); + String DOC = "This record represents a PATCH holding a sequence of operations to apply to a given object."; + + static Schema create(Schema valueSchema) { + Schema.Field operations = new Schema.Field("operations", + Schema.createArray(Types.Operation.create(valueSchema)), + "Sequence of operations to apply to a given object.", NO_DEFAULT); + return createRecord(NAME, DOC, PATCH_NAMESPACE, false, Arrays.asList(operations)); + + } + } +} \ No newline at end of file diff --git a/src/test/java/io/datanerds/avropatch/operation/PathTest.java b/src/test/java/io/datanerds/avropatch/operation/PathTest.java index 3a5590e..efd0d39 100644 --- a/src/test/java/io/datanerds/avropatch/operation/PathTest.java +++ b/src/test/java/io/datanerds/avropatch/operation/PathTest.java @@ -2,7 +2,7 @@ import com.google.common.testing.EqualsTester; import com.google.common.testing.NullPointerTester; -import io.datanerds.avropatch.exception.InvalidPathException; +import io.datanerds.avropatch.exception.InvalidReferenceTokenException; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -23,7 +23,9 @@ public class PathTest { public void equality() { new EqualsTester() .addEqualityGroup(Path.of("hello"), Path.of("hello"), Path.parse("/hello"), Path.parse("/hello/")) - .addEqualityGroup(Path.of("hello", "world"), Path.of("hello", "world"), Path.parse("/hello/world")) + .addEqualityGroup(Path.of("hello", "world"), Path.of("hello", "world"), Path.parse("/hello/world"), + Path.of("hello").append("world"), Path.of("hello").append(Path.of("world")), + Path.of(Path.of("hello"), Path.of("world"))) .addEqualityGroup(Path.of(), Path.parse(SLASH), ROOT) .testEquals(); } @@ -41,25 +43,25 @@ public void numbersAndUnderscores() { @Test public void noLeadingUnderscores() { - expectedException.expect(InvalidPathException.class); + expectedException.expect(InvalidReferenceTokenException.class); Path.of("_hello_"); } @Test public void noLeadingDash() { - expectedException.expect(InvalidPathException.class); + expectedException.expect(InvalidReferenceTokenException.class); Path.of("-hello"); } @Test public void invalidCharacter() { - expectedException.expect(InvalidPathException.class); + expectedException.expect(InvalidReferenceTokenException.class); Path.of("hel!lo"); } @Test public void emptyPath() { - expectedException.expect(InvalidPathException.class); + expectedException.expect(InvalidReferenceTokenException.class); Path.of(""); } @@ -77,7 +79,7 @@ public void trailingSlash() { @Test public void parseWithoutSlash() { - expectedException.expect(InvalidPathException.class); + expectedException.expect(InvalidReferenceTokenException.class); expectedException.expectMessage("JSON Path has to start with a slash."); Path.parse("hello/"); } @@ -110,4 +112,8 @@ public void asString() { assertThat(ROOT.toString(), is(equalTo("/"))); } + @Test(expected = InvalidReferenceTokenException.class) + public void appendValidates() { + ROOT.append(".'!#(^((*(_$,-"); + } } \ No newline at end of file diff --git a/src/test/java/io/datanerds/avropatch/schema/OperationSerializationTester.java b/src/test/java/io/datanerds/avropatch/serialization/OperationSerializationTester.java similarity index 84% rename from src/test/java/io/datanerds/avropatch/schema/OperationSerializationTester.java rename to src/test/java/io/datanerds/avropatch/serialization/OperationSerializationTester.java index 24eb0f0..746845b 100644 --- a/src/test/java/io/datanerds/avropatch/schema/OperationSerializationTester.java +++ b/src/test/java/io/datanerds/avropatch/serialization/OperationSerializationTester.java @@ -1,8 +1,7 @@ -package io.datanerds.avropatch.schema; +package io.datanerds.avropatch.serialization; import com.google.common.collect.ImmutableMap; import io.datanerds.avropatch.operation.*; -import io.datanerds.avropatch.serialization.OperationTypes; import io.datanerds.avropatch.value.type.BigDecimalType; import io.datanerds.avropatch.value.type.BigIntegerType; import io.datanerds.avropatch.value.type.TimestampType; @@ -28,12 +27,12 @@ public class OperationSerializationTester extends SerializationTester BigDecimalType.SCHEMA, BigIntegerType.SCHEMA, TimestampType.SCHEMA, UuidType.SCHEMA); private static final Map, Schema> SCHEMATA = new ImmutableMap.Builder() - .put(Add.class, OperationTypes.Add.create(VALUE_TYPE_UNION)) - .put(Copy.class, OperationTypes.Copy.create()) - .put(Move.class, OperationTypes.Move.create()) - .put(Remove.class, OperationTypes.Remove.create()) - .put(Replace.class, OperationTypes.Replace.create(VALUE_TYPE_UNION)) - .put(io.datanerds.avropatch.operation.Test.class, OperationTypes.Test.create(VALUE_TYPE_UNION)) + .put(Add.class, Types.Add.create(VALUE_TYPE_UNION)) + .put(Copy.class, Types.Copy.create()) + .put(Move.class, Types.Move.create()) + .put(Remove.class, Types.Remove.create()) + .put(Replace.class, Types.Replace.create(VALUE_TYPE_UNION)) + .put(io.datanerds.avropatch.operation.Test.class, Types.Test.create(VALUE_TYPE_UNION)) .build(); public OperationSerializationTester(Class clazz) { diff --git a/src/test/java/io/datanerds/avropatch/schema/SerializationTester.java b/src/test/java/io/datanerds/avropatch/serialization/SerializationTester.java similarity index 97% rename from src/test/java/io/datanerds/avropatch/schema/SerializationTester.java rename to src/test/java/io/datanerds/avropatch/serialization/SerializationTester.java index 45a50af..912631a 100644 --- a/src/test/java/io/datanerds/avropatch/schema/SerializationTester.java +++ b/src/test/java/io/datanerds/avropatch/serialization/SerializationTester.java @@ -1,4 +1,4 @@ -package io.datanerds.avropatch.schema; +package io.datanerds.avropatch.serialization; import io.datanerds.avropatch.value.AvroData; import org.apache.avro.Schema; diff --git a/src/test/java/io/datanerds/avropatch/schema/OperationTypesTest.java b/src/test/java/io/datanerds/avropatch/serialization/TypesTest.java similarity index 89% rename from src/test/java/io/datanerds/avropatch/schema/OperationTypesTest.java rename to src/test/java/io/datanerds/avropatch/serialization/TypesTest.java index c319a39..c3ed8fb 100644 --- a/src/test/java/io/datanerds/avropatch/schema/OperationTypesTest.java +++ b/src/test/java/io/datanerds/avropatch/serialization/TypesTest.java @@ -1,13 +1,13 @@ -package io.datanerds.avropatch.schema; +package io.datanerds.avropatch.serialization; import io.datanerds.avropatch.operation.*; import org.junit.Test; import java.io.IOException; -import static io.datanerds.avropatch.schema.OperationSerializationTester.createSomeOperations; +import static io.datanerds.avropatch.serialization.OperationSerializationTester.createSomeOperations; -public class OperationTypesTest { +public class TypesTest { @Test public void add() throws IOException {