diff --git a/.gitignore b/.gitignore index 34df35c..f39ec2c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,37 @@ -*.iml +# Maven +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties + +# Java template *.class + +# logs +logs +*.log + +# Eclipse +.settings/ +.project +.classpath + +# Intellij +*.ipr +*.iws +.idea/ +*.iml **/.idea/ -**/target/ + +# Package Files +*.jar +*.war +*.ear + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* diff --git a/AvroConsumerExample/pom.xml b/AvroConsumerExample/pom.xml index 3d8e918..0a077c6 100644 --- a/AvroConsumerExample/pom.xml +++ b/AvroConsumerExample/pom.xml @@ -4,9 +4,15 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - AvroConsumerExample + com.shapira.examples.kafka.AvroConsumerExample ClickSessionizer - 1.0-SNAPSHOT + + + kafka-examples-parent + com.shapira.examples.kafka + 1.0-SNAPSHOT + + @@ -16,10 +22,18 @@ + + 0.10.0.1 + 2.10 + 3.2.0 + 1.8.1 + 1.8 + 1.8 UTF-8 @@ -69,6 +83,8 @@ org.apache.maven.plugins maven-shade-plugin + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + + org.apache.avro + + + avro-maven-plugin + + + [1.7.7,) + + + idl-protocol + protocol + schema + + + + + + + + + + + + - \ No newline at end of file + diff --git a/AvroConsumerExample/src/main/resources/META-INF/MANIFEST.MF b/AvroConsumerExample/src/main/resources/META-INF/MANIFEST.MF new file mode 100644 index 0000000..971c17c --- /dev/null +++ b/AvroConsumerExample/src/main/resources/META-INF/MANIFEST.MF @@ -0,0 +1 @@ +Main-Class: com.shapira.examples.consumer.avroclicks.AvroClicksSessionizer diff --git a/AvroConsumerExample/src/main/resources/log4j.properties b/AvroConsumerExample/src/main/resources/log4j.properties index c39cd11..83fe83e 100644 --- a/AvroConsumerExample/src/main/resources/log4j.properties +++ b/AvroConsumerExample/src/main/resources/log4j.properties @@ -18,4 +18,4 @@ log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n -log4j.logger.org.apache.kafka=ERROR \ No newline at end of file +log4j.logger.org.apache.kafka=INFO \ No newline at end of file diff --git a/AvroProducerExample/pom.xml b/AvroProducerExample/pom.xml index c8628cb..2438af3 100644 --- a/AvroProducerExample/pom.xml +++ b/AvroProducerExample/pom.xml @@ -4,9 +4,15 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - AvroProducerExample + com.shapira.examples.kafka.AvroProducerExample ClickstreamGenerator - 1.0-SNAPSHOT + + + kafka-examples-parent + com.shapira.examples.kafka + 1.0-SNAPSHOT + + @@ -27,11 +33,19 @@ + + 0.10.0.1 + 2.10 + 3.2.0 + 1.8.1 UTF-8 + 1.8 + 1.8 @@ -80,6 +94,8 @@ org.apache.maven.plugins maven-shade-plugin + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + + org.apache.avro + + + avro-maven-plugin + + + [1.7.7,) + + + protocol + idl-protocol + schema + + + + + + + + + + + + - \ No newline at end of file + diff --git a/AvroProducerExample/src/main/java/com/shapira/examples/producer/avroclicks/AvroClicksProducer.java b/AvroProducerExample/src/main/java/com/shapira/examples/producer/avroclicks/AvroClicksProducer.java index 065489c..44c15a6 100644 --- a/AvroProducerExample/src/main/java/com/shapira/examples/producer/avroclicks/AvroClicksProducer.java +++ b/AvroProducerExample/src/main/java/com/shapira/examples/producer/avroclicks/AvroClicksProducer.java @@ -1,16 +1,20 @@ package com.shapira.examples.producer.avroclicks; -import JavaSessionize.avro.LogLine; +import java.util.Properties; + import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; +import JavaSessionize.avro.LogLine; import java.util.Properties; import java.util.Random; import java.util.concurrent.ExecutionException; public class AvroClicksProducer { + static boolean verbose = Boolean.getBoolean("log.verbose"); + public static void main(String[] args) throws ExecutionException, InterruptedException { if (args.length != 2) { System.out.println("Please provide command line arguments: numEvents schemaRegistryUrl"); @@ -29,20 +33,30 @@ public static void main(String[] args) throws ExecutionException, InterruptedExc props.put("schema.registry.url", schemaUrl); // Hard coding topic too. String topic = "clicks"; + + System.out.println("Writing topic:" + topic); + Producer producer = new KafkaProducer(props); - Random rnd = new Random(); +// Random rnd = new Random(); for (long nEvents = 0; nEvents < events; nEvents++) { LogLine event = EventGenerator.getNext(); + if (verbose) { + System.out.print("/* >> " + nEvents + " >> */ "); + } + // Using IP as key, so events from same IP will go to same partition ProducerRecord record = new ProducerRecord(topic, event.getIp().toString(), event); producer.send(record).get(); - + if (verbose) { + System.out.println(event.toString()); + } } + producer.close(); } } diff --git a/AvroProducerExample/src/main/resources/META-INF/MANIFEST.MF b/AvroProducerExample/src/main/resources/META-INF/MANIFEST.MF new file mode 100644 index 0000000..5523057 --- /dev/null +++ b/AvroProducerExample/src/main/resources/META-INF/MANIFEST.MF @@ -0,0 +1 @@ +Main-Class: com.shapira.examples.producer.avroclicks.AvroClicksProducer diff --git a/AvroProducerExample/src/main/resources/log4j.properties b/AvroProducerExample/src/main/resources/log4j.properties new file mode 100644 index 0000000..83fe83e --- /dev/null +++ b/AvroProducerExample/src/main/resources/log4j.properties @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +log4j.rootLogger=stdout, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n + +log4j.logger.org.apache.kafka=INFO \ No newline at end of file diff --git a/FancyMovingAvg/pom.xml b/FancyMovingAvg/pom.xml index 41c0548..850f182 100644 --- a/FancyMovingAvg/pom.xml +++ b/FancyMovingAvg/pom.xml @@ -4,9 +4,17 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - ConsumerExample + com.shapira.examples.kafka.ConsumerExample FancyMovingAvg - 1.0-SNAPSHOT + + + kafka-examples-parent + com.shapira.examples.kafka + 1.0-SNAPSHOT + + + + org.apache.kafka @@ -34,4 +42,4 @@ - \ No newline at end of file + diff --git a/KafkaStreamsAvg/pom.xml b/KafkaStreamsAvg/pom.xml index cfdf594..b995da9 100644 --- a/KafkaStreamsAvg/pom.xml +++ b/KafkaStreamsAvg/pom.xml @@ -4,14 +4,29 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - KafkaStreamsExamples + com.shapira.examples.kafka.KafkaStreamsExamples StreamingAvg - 1.0-SNAPSHOT + + kafka-examples-parent + com.shapira.examples.kafka + 1.0-SNAPSHOT + + + 0.10.0.1 + 1.8.1 @@ -45,4 +60,4 @@ - \ No newline at end of file + diff --git a/KafkaStreamsAvg/src/main/java/com/shapira/examples/kstreamavg/AvgAggregator.java b/KafkaStreamsAvg/src/main/java/com/shapira/examples/kstreamavg/AvgAggregator.java index 82ed680..88a6bf3 100644 --- a/KafkaStreamsAvg/src/main/java/com/shapira/examples/kstreamavg/AvgAggregator.java +++ b/KafkaStreamsAvg/src/main/java/com/shapira/examples/kstreamavg/AvgAggregator.java @@ -1,22 +1,52 @@ package com.shapira.examples.kstreamavg; import org.apache.kafka.streams.kstream.Aggregator; +import org.apache.kafka.streams.kstream.Initializer; -public class AvgAggregator implements Aggregator { +//AvgAggregator implements Aggregator +public class AvgAggregator { + + public static class InternalAvgInitializer implements Initializer { - public AvgValue initialValue() { - return new AvgValue(0,0); - } + @Override + public AvgValue apply() { + return new AvgValue(0,0); + } + + } + + public static Initializer initializer() { + return new InternalAvgInitializer(); + } + + public static class InternalAvgAggregator implements Aggregator { - public AvgValue add(String aggKey, Integer value, AvgValue aggregate) { - return new AvgValue(aggregate.count + 1, aggregate.sum + value ); - } + @Override + public AvgValue apply(String aggKey, Integer value, AvgValue aggregate) { + return new AvgValue(aggregate.count + 1, aggregate.sum + value ); + } + + } + + public static Aggregator aggregator() { + return new InternalAvgAggregator(); + } - public AvgValue remove(String aggKey, Integer value, AvgValue aggregate) { - return new AvgValue(aggregate.count - 1, aggregate.sum - value); - } - - public AvgValue merge(AvgValue aggr1, AvgValue aggr2) { - return new AvgValue(aggr1.count + aggr2.count, aggr1.sum + aggr2.sum); - } +// +// +// public AvgValue initialValue() { +// return new AvgValue(0,0); +// } +// +// public AvgValue add(String aggKey, Integer value, AvgValue aggregate) { +// return new AvgValue(aggregate.count + 1, aggregate.sum + value ); +// } +// +// public AvgValue remove(String aggKey, Integer value, AvgValue aggregate) { +// return new AvgValue(aggregate.count - 1, aggregate.sum - value); +// } +// +// public AvgValue merge(AvgValue aggr1, AvgValue aggr2) { +// return new AvgValue(aggr1.count + aggr2.count, aggr1.sum + aggr2.sum); +// } } diff --git a/KafkaStreamsAvg/src/main/java/com/shapira/examples/kstreamavg/StreamingAvg.java b/KafkaStreamsAvg/src/main/java/com/shapira/examples/kstreamavg/StreamingAvg.java index 3c14a5e..23b97a7 100644 --- a/KafkaStreamsAvg/src/main/java/com/shapira/examples/kstreamavg/StreamingAvg.java +++ b/KafkaStreamsAvg/src/main/java/com/shapira/examples/kstreamavg/StreamingAvg.java @@ -1,41 +1,21 @@ package com.shapira.examples.kstreamavg; -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.common.serialization.IntegerDeserializer; -import org.apache.kafka.common.serialization.IntegerSerializer; -import org.apache.kafka.streams.KafkaStreaming; -import org.apache.kafka.streams.StreamingConfig; -import org.apache.kafka.streams.examples.WallclockTimestampExtractor; +import java.util.Properties; + +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.Serdes.IntegerSerde; +import org.apache.kafka.common.serialization.Serdes.StringSerde; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.KStreamBuilder; import org.apache.kafka.streams.kstream.KTable; -import org.apache.kafka.streams.kstream.KeyValue; -import org.apache.kafka.streams.kstream.TumblingWindows; +import org.apache.kafka.streams.kstream.TimeWindows; import org.apache.kafka.streams.kstream.Windowed; -import org.apache.kafka.streams.kstream.internals.TumblingWindow; +import org.apache.kafka.streams.kstream.internals.TimeWindow; +import org.apache.kafka.streams.processor.WallclockTimestampExtractor; import org.apache.log4j.Logger; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.common.serialization.StringSerializer; - -import java.util.Properties; public class StreamingAvg { static Logger log = Logger.getLogger(StreamingAvg.class.getName()); @@ -43,14 +23,15 @@ public class StreamingAvg { public static void main(String[] args) throws Exception { Properties props = new Properties(); - props.put(StreamingConfig.JOB_ID_CONFIG, "moving-avg-example"); - props.put(StreamingConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); - props.put(StreamingConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class); - props.put(StreamingConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); - props.put(StreamingConfig.KEY_DESERIALIZER_CLASS_CONFIG, IntegerDeserializer.class); - props.put(StreamingConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - props.put(StreamingConfig.TIMESTAMP_EXTRACTOR_CLASS_CONFIG, WallclockTimestampExtractor.class); - StreamingConfig config = new StreamingConfig(props); + props.put(StreamsConfig.APPLICATION_ID_CONFIG, "moving-avg-example"); + props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); + props.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, IntegerSerde.class); + props.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, StringSerde.class); + props.put(StreamsConfig.TIMESTAMP_EXTRACTOR_CLASS_CONFIG, WallclockTimestampExtractor.class); + StreamsConfig config = new StreamsConfig(props); + + final Serde stringSerde = Serdes.String(); + final Serde avgValueSerde = Serdes.serdeFrom(new AvgValueSerializer(), new AvgValueDeserializer()); KStreamBuilder builder = new KStreamBuilder(); @@ -60,15 +41,15 @@ public static void main(String[] args) throws Exception { KStream namedPrices = prices.leftJoin(names, (price, name) -> { return new NamedPrice(name, price); - }).map((ticket, namedPrice) -> new KeyValue(namedPrice.name, namedPrice.price)); + }).map((ticket, namedPrice) -> new org.apache.kafka.streams.KeyValue(namedPrice.name, namedPrice.price)); - KTable, AvgValue> tempTable = namedPrices.aggregateByKey( - () -> new AvgAggregator(), - TumblingWindows.of("avgWindow").with(10000), - new StringSerializer(), new AvgValueSerializer(), - new StringDeserializer(), new AvgValueDeserializer()); + KTable, AvgValue> tempTable = namedPrices.aggregateByKey( + AvgAggregator.initializer(), + AvgAggregator.aggregator(), + TimeWindows.of("avgWindow",10000), + stringSerde, avgValueSerde); // Should work after we implement "aggregateByKey KTable, Double> avg = tempTable.mapValues((v) -> ((double) v.sum / v.count)); @@ -76,7 +57,74 @@ public static void main(String[] args) throws Exception { avg.to("ks_avg_prices"); - KafkaStreaming kstream = new KafkaStreaming(builder, config); + KafkaStreams kstream = new KafkaStreams(builder, config); kstream.start(); } } + + +//import org.apache.avro.generic.GenericData; +//import org.apache.avro.generic.GenericRecord; +//import org.apache.kafka.common.serialization.IntegerDeserializer; +//import org.apache.kafka.common.serialization.IntegerSerializer; +//import org.apache.kafka.streams.KafkaStreaming; +//import org.apache.kafka.streams.StreamingConfig; +//import org.apache.kafka.streams.examples.WallclockTimestampExtractor; +//import org.apache.kafka.streams.kstream.KStream; +//import org.apache.kafka.streams.kstream.KStreamBuilder; +//import org.apache.kafka.streams.kstream.KTable; +//import org.apache.kafka.streams.kstream.KeyValue; +//import org.apache.kafka.streams.kstream.TumblingWindows; +//import org.apache.kafka.streams.kstream.Windowed; +//import org.apache.kafka.streams.kstream.internals.TumblingWindow; +//import org.apache.log4j.Logger; +//import org.apache.kafka.common.serialization.StringDeserializer; +//import org.apache.kafka.common.serialization.StringSerializer; +// +//import java.util.Properties; +// +//public class StreamingAvg { +// static Logger log = Logger.getLogger(StreamingAvg.class.getName()); +// +// public static void main(String[] args) throws Exception { +// Properties props = new Properties(); +// +// props.put(StreamingConfig.JOB_ID_CONFIG, "moving-avg-example"); +// props.put(StreamingConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); +// props.put(StreamingConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class); +// props.put(StreamingConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); +// props.put(StreamingConfig.KEY_DESERIALIZER_CLASS_CONFIG, IntegerDeserializer.class); +// props.put(StreamingConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); +// props.put(StreamingConfig.TIMESTAMP_EXTRACTOR_CLASS_CONFIG, WallclockTimestampExtractor.class); +// StreamingConfig config = new StreamingConfig(props); +// +// KStreamBuilder builder = new KStreamBuilder(); +// +// KStream prices = builder.stream("ks_prices"); +// +// KTable names = builder.table("ks_names"); +// +// KStream namedPrices = prices.leftJoin(names, (price, name) -> { +// return new NamedPrice(name, price); +// }).map((ticket, namedPrice) -> new KeyValue(namedPrice.name, namedPrice.price)); +// +// +// +// KTable, AvgValue> tempTable = namedPrices.aggregateByKey( +// () -> new AvgAggregator(), +// TumblingWindows.of("avgWindow").with(10000), +// new StringSerializer(), new AvgValueSerializer(), +// new StringDeserializer(), new AvgValueDeserializer()); +// +// // Should work after we implement "aggregateByKey +// KTable, Double> avg = tempTable.mapValues((v) -> ((double) v.sum / v.count)); +// +// avg.to("ks_avg_prices"); +// +// +// KafkaStreaming kstream = new KafkaStreaming(builder, config); +// kstream.start(); +// } +//} +// +// diff --git a/MirrorMakerHandler/pom.xml b/MirrorMakerHandler/pom.xml index f78192e..41be463 100644 --- a/MirrorMakerHandler/pom.xml +++ b/MirrorMakerHandler/pom.xml @@ -4,9 +4,15 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - MirrorMakerExample + com.shapira.examples.kafka.MirrorMakerExample TopicSwitchingHandler - 1.0-SNAPSHOT + + + kafka-examples-parent + com.shapira.examples.kafka + 1.0-SNAPSHOT + + @@ -16,4 +22,4 @@ - \ No newline at end of file + diff --git a/README.md b/README.md new file mode 100644 index 0000000..79971ec --- /dev/null +++ b/README.md @@ -0,0 +1,50 @@ +Apache Kafka Examples +===================== + +base: https://github.com/gwenshap/kafka-examples + + +| *Dependency* | *Version* | +| ------------- | -------- | +| Kafka | 0.10.0.1 | +| Java | 1.8 | +| Scala | 2.10 | +| Confluent | 3.2.0 | +| CDH | 5.3.0 | +| HDP | 2.5.0 | + + + + +Clients +------- + +* [SimpleCounter](SimpleCounter) +* [SimpleMovingAvg](SimpleMovingAvg) + +Balancing/Mirroring +------------------- + +* [FancyMovingAvg](FancyMovingAvg) +* [MirrorMakerHandler](MirrorMakerHandler) + + +AVRO Schema Registry +-------------------- + +* [AvroConsumerExample](AvroConsumerExample) +* [AvroProducerExample](AvroProducerExample) + + +Streams +------- + +* [KafkaStreamsAvg](KafkaStreamsAvg) +* [SparkStreamingAvg](StreamingAvg) (1) + + + +*note* +1. spark 1.2, kafka 0.8 + + diff --git a/SimpleCounter/pom.xml b/SimpleCounter/pom.xml index d0c990a..00fe204 100644 --- a/SimpleCounter/pom.xml +++ b/SimpleCounter/pom.xml @@ -4,9 +4,16 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - ProducerExample + + com.shapira.examples.kafka.ProducerExample SimpleCounter - 1.0-SNAPSHOT + + + kafka-examples-parent + com.shapira.examples.kafka + 1.0-SNAPSHOT + + org.apache.kafka @@ -25,6 +32,7 @@ org.apache.maven.plugins maven-shade-plugin + + - \ No newline at end of file + diff --git a/SimpleMovingAvg/pom.xml b/SimpleMovingAvg/pom.xml index 2b8ccd1..d78df21 100644 --- a/SimpleMovingAvg/pom.xml +++ b/SimpleMovingAvg/pom.xml @@ -4,9 +4,16 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - ConsumerExample + com.shapira.examples.kafka.ConsumerExample SimpleMovingAvg - 1.0-SNAPSHOT + + + kafka-examples-parent + com.shapira.examples.kafka + 1.0-SNAPSHOT + + + org.apache.kafka @@ -29,6 +36,7 @@ org.apache.maven.plugins maven-shade-plugin + diff --git a/StreamingAvg/pom.xml b/StreamingAvg/pom.xml index 0b6b7fc..5bd54a4 100644 --- a/StreamingAvg/pom.xml +++ b/StreamingAvg/pom.xml @@ -4,9 +4,31 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - SparkStreamingExamples - StreamingAvg - 1.0-SNAPSHOT + com.shapira.examples.kafka.SparkStreamingExamples + SparkStreamingAvg + + + kafka-examples-parent + com.shapira.examples.kafka + 1.0-SNAPSHOT + + + + + 3.4.6 + 1.2.0-cdh5.3.0 + 0.8.1.1 + 1.8.1 + @@ -20,26 +42,26 @@ org.apache.spark spark-core_2.10 - 1.2.0-cdh5.3.0 + ${spark.version} provided org.apache.spark spark-streaming_2.10 - 1.2.0-cdh5.3.0 + ${spark.version} provided org.apache.spark spark-streaming-kafka_2.10 - 1.2.0-cdh5.3.0 + ${spark.version} org.apache.kafka kafka_2.10 - 0.8.1.1 + ${kafka.version} commons-collections @@ -49,7 +71,7 @@ org.apache.zookeeper zookeeper - 3.4.6 + ${zookeeper.version} @@ -57,6 +79,10 @@ org.apache.maven.plugins maven-shade-plugin + + uber-StreamingAvg-${project.version} + + diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..1dad464 --- /dev/null +++ b/pom.xml @@ -0,0 +1,154 @@ + + + 4.0.0 + + com.shapira.examples.kafka + kafka-examples-parent + 1.0-SNAPSHOT + pom + + kafka-examples + Kafka Examples + + + AvroConsumerExample + AvroProducerExample + MirrorMakerHandler + SimpleCounter + SimpleMovingAvg + FancyMovingAvg + KafkaStreamsAvg + StreamingAvg + + + + + + https://github.com/gwenshap/kafka-examples + scm:git:git@github.com:gwenshap/kafka-examples.git + scm:git:git@github.com:gwenshap/kafka-examples.git + + + + UTF-8 + 1.8 + 1.8 + 1.8 + + + 0.10.0.1 + 2.10 + 3.2.0 + 1.8.1 + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + 1.4.1 + + + enforce-no-snapshots + + enforce + + + + + No Snapshots Allowed! + false + + com.contactlab:contactdata-proxy-api + com.contactlab:contactdata-proxy-resource + + + + true + ${enforcer.skip} + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.0.0 + + + package + + shade + + + + + + + com.shapira.examples.consumer.avroclicks.AvroClicksSessionizer + + + uber-${project.artifactId}-${project.version} + + + + + + + + + + com.fasterxml.jackson.datatype + jackson-datatype-jdk8 + 2.8.5 + + + + + + + + apache-repo + Apache Repository + https://repository.apache.org/content/repositories/releases + + true + + + false + + + + confluent + http://packages.confluent.io/maven/ + + + hortonworks-releases + hortonworks-releases + http://repo.hortonworks.com/content/groups/public/ + + true + + + false + + + + + + + + + + +