gwenshap · hute37 · Mar 30, 2017 · Mar 30, 2017 · Mar 30, 2017 · Mar 30, 2017
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,37 @@
-*.iml
+# Maven
+target/
+pom.xml.tag
+pom.xml.releaseBackup
+pom.xml.versionsBackup
+pom.xml.next
+release.properties
+dependency-reduced-pom.xml
+buildNumber.properties
+.mvn/timing.properties
+
+# Java template
 *.class
+
+# logs
+logs
+*.log
+
+# Eclipse
+.settings/
+.project
+.classpath
+
+# Intellij
+*.ipr
+*.iws
+.idea/
+*.iml
 **/.idea/
-**/target/
+
+# Package Files
+*.jar
+*.war
+*.ear
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
diff --git a/AvroConsumerExample/pom.xml b/AvroConsumerExample/pom.xml
@@ -4,9 +4,15 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
-    <groupId>AvroConsumerExample</groupId>
+    <groupId>com.shapira.examples.kafka.AvroConsumerExample</groupId>
     <artifactId>ClickSessionizer</artifactId>
-    <version>1.0-SNAPSHOT</version>
+
+    <parent>
+        <artifactId>kafka-examples-parent</artifactId>
+        <groupId>com.shapira.examples.kafka</groupId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+
 
     <repositories>
         <repository>
@@ -16,10 +22,18 @@
     </repositories>
 
     <properties>
+    <!--
         <kafka.version>0.9.0.0</kafka.version>
         <kafka.scala.version>2.11</kafka.scala.version>
         <confluent.version>2.0.0</confluent.version>
         <avro.version>1.7.7</avro.version>
+     -->
+        <kafka.version>0.10.0.1</kafka.version>
+        <kafka.scala.version>2.10</kafka.scala.version>
+        <confluent.version>3.2.0</confluent.version>
+        <avro.version>1.8.1</avro.version>
+		<maven.compiler.source>1.8</maven.compiler.source>
+		<maven.compiler.target>1.8</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     </properties>
 
@@ -69,6 +83,8 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
+                <!--
+                <version>3.0.0</version>
                 <executions>
                     <execution>
                         <phase>package</phase>
@@ -78,11 +94,54 @@
                     </execution>
                 </executions>
                 <configuration>
-                    <finalName>uber-${artifactId}-${version}</finalName>
+              		<transformers>
+	                	<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+    	             	 <mainClass>com.shapira.examples.consumer.avroclicks.AvroClicksSessionizer</mainClass>
+        	        	</transformer>
+            	  	</transformers>
+                    <finalName>uber-${project.artifactId}-${project.version}</finalName>
                 </configuration>
+              -->
             </plugin>
         </plugins>
+        <pluginManagement>
+        	<plugins>
+        		<!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->
+        		<plugin>
+        			<groupId>org.eclipse.m2e</groupId>
+        			<artifactId>lifecycle-mapping</artifactId>
+        			<version>1.0.0</version>
+        			<configuration>
+        				<lifecycleMappingMetadata>
+        					<pluginExecutions>
+        						<pluginExecution>
+        							<pluginExecutionFilter>
+        								<groupId>
+        									org.apache.avro
+        								</groupId>
+        								<artifactId>
+        									avro-maven-plugin
+        								</artifactId>
+        								<versionRange>
+        									[1.7.7,)
+        								</versionRange>
+        								<goals>
+        									<goal>idl-protocol</goal>
+        									<goal>protocol</goal>
+        									<goal>schema</goal>
+        								</goals>
+        							</pluginExecutionFilter>
+        							<action>
+        								<ignore></ignore>
+        							</action>
+        						</pluginExecution>
+        					</pluginExecutions>
+        				</lifecycleMappingMetadata>
+        			</configuration>
+        		</plugin>
+        	</plugins>
+        </pluginManagement>
     </build>
 
 
-</project>
+</project>
diff --git a/AvroConsumerExample/src/main/resources/META-INF/MANIFEST.MF b/AvroConsumerExample/src/main/resources/META-INF/MANIFEST.MF
@@ -0,0 +1 @@
+Main-Class: com.shapira.examples.consumer.avroclicks.AvroClicksSessionizer
diff --git a/AvroConsumerExample/src/main/resources/log4j.properties b/AvroConsumerExample/src/main/resources/log4j.properties
@@ -18,4 +18,4 @@ log4j.appender.stdout=org.apache.log4j.ConsoleAppender
 log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
 log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n
 
-log4j.logger.org.apache.kafka=ERROR
+log4j.logger.org.apache.kafka=INFO
diff --git a/AvroProducerExample/pom.xml b/AvroProducerExample/pom.xml
@@ -4,9 +4,15 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
-    <groupId>AvroProducerExample</groupId>
+    <groupId>com.shapira.examples.kafka.AvroProducerExample</groupId>
     <artifactId>ClickstreamGenerator</artifactId>
-    <version>1.0-SNAPSHOT</version>
+
+    <parent>
+        <artifactId>kafka-examples-parent</artifactId>
+        <groupId>com.shapira.examples.kafka</groupId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+
 
     <repositories>
         <repository>
@@ -27,11 +33,19 @@
     </repositories>
 
     <properties>
+    <!--
         <kafka.version>0.8.2.1</kafka.version>
         <kafka.scala.version>2.10</kafka.scala.version>
         <confluent.version>1.0</confluent.version>
         <avro.version>1.7.7</avro.version>
+     -->
+        <kafka.version>0.10.0.1</kafka.version>
+        <kafka.scala.version>2.10</kafka.scala.version>
+        <confluent.version>3.2.0</confluent.version>
+        <avro.version>1.8.1</avro.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+		<maven.compiler.source>1.8</maven.compiler.source>
+		<maven.compiler.target>1.8</maven.compiler.target>
     </properties>
 
 
@@ -80,6 +94,8 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
+                <!--
+                <version>3.0.0</version>
                 <executions>
                     <execution>
                         <phase>package</phase>
@@ -89,9 +105,52 @@
                     </execution>
                 </executions>
                 <configuration>
-                    <finalName>uber-${artifactId}-${version}</finalName>
+              		<transformers>
+	                	<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+    	             	 <mainClass>com.shapira.examples.producer.avroclicks.AvroClicksProducer</mainClass>
+        	        	</transformer>
+            	  	</transformers>
+                    <finalName>uber-${project.artifactId}-${project.version}</finalName>
                 </configuration>
+              -->
             </plugin>
         </plugins>
+        <pluginManagement>
+        	<plugins>
+        		<!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->
+        		<plugin>
+        			<groupId>org.eclipse.m2e</groupId>
+        			<artifactId>lifecycle-mapping</artifactId>
+        			<version>1.0.0</version>
+        			<configuration>
+        				<lifecycleMappingMetadata>
+        					<pluginExecutions>
+        						<pluginExecution>
+        							<pluginExecutionFilter>
+        								<groupId>
+        									org.apache.avro
+        								</groupId>
+        								<artifactId>
+        									avro-maven-plugin
+        								</artifactId>
+        								<versionRange>
+        									[1.7.7,)
+        								</versionRange>
+        								<goals>
+        									<goal>protocol</goal>
+        									<goal>idl-protocol</goal>
+        									<goal>schema</goal>
+        								</goals>
+        							</pluginExecutionFilter>
+        							<action>
+        								<ignore></ignore>
+        							</action>
+        						</pluginExecution>
+        					</pluginExecutions>
+        				</lifecycleMappingMetadata>
+        			</configuration>
+        		</plugin>
+        	</plugins>
+        </pluginManagement>
     </build>
-</project>
+</project>
diff --git a/...cerExample/src/main/java/com/shapira/examples/producer/avroclicks/AvroClicksProducer.java b/...cerExample/src/main/java/com/shapira/examples/producer/avroclicks/AvroClicksProducer.java
@@ -1,16 +1,20 @@
 package com.shapira.examples.producer.avroclicks;
 
-import JavaSessionize.avro.LogLine;
+import java.util.Properties;
+
 import org.apache.kafka.clients.producer.KafkaProducer;
 import org.apache.kafka.clients.producer.Producer;
 import org.apache.kafka.clients.producer.ProducerRecord;
 
+import JavaSessionize.avro.LogLine;
 import java.util.Properties;
 import java.util.Random;
 import java.util.concurrent.ExecutionException;
 
 public class AvroClicksProducer {
 
+	static boolean verbose = Boolean.getBoolean("log.verbose");
+
     public static void main(String[] args) throws ExecutionException, InterruptedException {
         if (args.length != 2) {
             System.out.println("Please provide command line arguments: numEvents schemaRegistryUrl");
@@ -29,20 +33,30 @@ public static void main(String[] args) throws ExecutionException, InterruptedExc
         props.put("schema.registry.url", schemaUrl);
         // Hard coding topic too.
         String topic = "clicks";
+
+        System.out.println("Writing topic:" + topic);
+
 
         Producer<String, LogLine> producer = new KafkaProducer<String, LogLine>(props);
 
-        Random rnd = new Random();
+//      Random rnd = new Random();
         for (long nEvents = 0; nEvents < events; nEvents++) {
             LogLine event = EventGenerator.getNext();
 
+			if (verbose) {
+				System.out.print("/* >> " + nEvents + " >> */ ");
+			}
+
             // Using IP as key, so events from same IP will go to same partition
             ProducerRecord<String, LogLine> record = new ProducerRecord<String, LogLine>(topic, event.getIp().toString(), event);
             producer.send(record).get();
 
-
+			if (verbose) {
+				System.out.println(event.toString());
+			}
 
         }
+        producer.close();
 
     }
 }

diff --git a/AvroProducerExample/src/main/resources/META-INF/MANIFEST.MF b/AvroProducerExample/src/main/resources/META-INF/MANIFEST.MF
@@ -0,0 +1 @@
+Main-Class: com.shapira.examples.producer.avroclicks.AvroClicksProducer
diff --git a/AvroProducerExample/src/main/resources/log4j.properties b/AvroProducerExample/src/main/resources/log4j.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+log4j.rootLogger=stdout, stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n
+
+log4j.logger.org.apache.kafka=INFO
diff --git a/FancyMovingAvg/pom.xml b/FancyMovingAvg/pom.xml
@@ -4,9 +4,17 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
-    <groupId>ConsumerExample</groupId>
+    <groupId>com.shapira.examples.kafka.ConsumerExample</groupId>
     <artifactId>FancyMovingAvg</artifactId>
-    <version>1.0-SNAPSHOT</version>
+
+    <parent>
+        <artifactId>kafka-examples-parent</artifactId>
+        <groupId>com.shapira.examples.kafka</groupId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+
+
+
     <dependencies>
         <dependency>
             <groupId>org.apache.kafka</groupId>
@@ -34,4 +42,4 @@
     </build>
 
 
-</project>
+</project>
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Main-Class: com.shapira.examples.consumer.avroclicks.AvroClicksSessionizer
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Main-Class: com.shapira.examples.producer.avroclicks.AvroClicksProducer