-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1 from anilkulkarni87/feature/publishtoKafka
Feature : Publish to Kafka
- Loading branch information
Showing
18 changed files
with
590 additions
and
23 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,104 @@ | ||
| <mxfile host="65bd71144e"> | ||
| <diagram id="wj3ed6IGQTsFisLiYsuj" name="Page-1"> | ||
| <mxGraphModel dx="1177" dy="1538" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" background="#818190" math="0" shadow="0"> | ||
| <root> | ||
| <mxCell id="0"/> | ||
| <mxCell id="1" parent="0"/> | ||
| <mxCell id="7" value="" style="rounded=1;whiteSpace=wrap;html=1;fillColor=none;" parent="1" vertex="1"> | ||
| <mxGeometry x="-10" y="50" width="480" height="200" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="9" style="edgeStyle=none;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="3" target="4" edge="1"> | ||
| <mxGeometry relative="1" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="3" value="Avro Schema for Data records defined via Avro IDL and class files generated." style="whiteSpace=wrap;html=1;shape=mxgraph.basic.document" parent="1" vertex="1"> | ||
| <mxGeometry x="20" y="90" width="100" height="100" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="10" style="edgeStyle=none;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="4" target="5" edge="1"> | ||
| <mxGeometry relative="1" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="4" value="Build Schema files and generate java class files" style="rounded=0;whiteSpace=wrap;html=1;" parent="1" vertex="1"> | ||
| <mxGeometry x="160" y="110" width="120" height="60" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="11" style="edgeStyle=none;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;entryPerimeter=0;" parent="1" source="5" edge="1"> | ||
| <mxGeometry relative="1" as="geometry"> | ||
| <mxPoint x="500" y="140" as="targetPoint"/> | ||
| </mxGeometry> | ||
| </mxCell> | ||
| <mxCell id="5" value="Create fake&nbsp;<br>records and<br>write producer to<br>KAFKA" style="shape=step;perimeter=stepPerimeter;whiteSpace=wrap;html=1;fixedSize=1;" parent="1" vertex="1"> | ||
| <mxGeometry x="305" y="100" width="145" height="80" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="8" value="" style="dashed=0;outlineConnect=0;html=1;align=center;labelPosition=center;verticalLabelPosition=bottom;verticalAlign=top;shape=mxgraph.weblogos.java;fillColor=none;" parent="1" vertex="1"> | ||
| <mxGeometry y="200" width="20" height="35" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="14" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" parent="1" source="12" edge="1"> | ||
| <mxGeometry relative="1" as="geometry"> | ||
| <mxPoint x="700" y="140" as="targetPoint"/> | ||
| </mxGeometry> | ||
| </mxCell> | ||
| <mxCell id="12" value="KAFKA" style="shape=process;whiteSpace=wrap;html=1;backgroundOutline=1;fillColor=default;" parent="1" vertex="1"> | ||
| <mxGeometry x="500" y="110" width="120" height="60" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="13" value="" style="endArrow=none;html=1;" parent="1" edge="1"> | ||
| <mxGeometry width="50" height="50" relative="1" as="geometry"> | ||
| <mxPoint x="700" y="280" as="sourcePoint"/> | ||
| <mxPoint x="700" y="5" as="targetPoint"/> | ||
| </mxGeometry> | ||
| </mxCell> | ||
| <mxCell id="32" style="edgeStyle=none;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;entryPerimeter=0;" parent="1" source="15" target="18" edge="1"> | ||
| <mxGeometry relative="1" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="15" value="" style="outlineConnect=0;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;shape=mxgraph.aws3.s3;fillColor=#E05243;gradientColor=none;" parent="1" vertex="1"> | ||
| <mxGeometry x="880" y="-30" width="60" height="70" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="16" value="" style="endArrow=classic;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;entryPerimeter=0;" parent="1" target="15" edge="1"> | ||
| <mxGeometry width="50" height="50" relative="1" as="geometry"> | ||
| <mxPoint x="700" y="7" as="sourcePoint"/> | ||
| <mxPoint x="930" y="160" as="targetPoint"/> | ||
| </mxGeometry> | ||
| </mxCell> | ||
| <mxCell id="17" value="Kafka S3 connect" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" parent="16" vertex="1" connectable="0"> | ||
| <mxGeometry x="0.067" y="1" relative="1" as="geometry"> | ||
| <mxPoint x="-1" as="offset"/> | ||
| </mxGeometry> | ||
| </mxCell> | ||
| <mxCell id="31" style="edgeStyle=none;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="18" target="29" edge="1"> | ||
| <mxGeometry relative="1" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="18" value="" style="shape=mxgraph.signs.nature.snowflake;html=1;pointerEvents=1;fillColor=#1ba1e2;strokeColor=#006EAF;verticalLabelPosition=bottom;verticalAlign=top;align=center;fontColor=#ffffff;" parent="1" vertex="1"> | ||
| <mxGeometry x="1080" y="110" width="60" height="78" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="19" value="" style="endArrow=classic;html=1;entryX=0.031;entryY=0.412;entryDx=0;entryDy=0;entryPerimeter=0;" parent="1" target="18" edge="1"> | ||
| <mxGeometry width="50" height="50" relative="1" as="geometry"> | ||
| <mxPoint x="700" y="140" as="sourcePoint"/> | ||
| <mxPoint x="930" y="160" as="targetPoint"/> | ||
| </mxGeometry> | ||
| </mxCell> | ||
| <mxCell id="20" value="Kafka Snowflake connect" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" parent="19" vertex="1" connectable="0"> | ||
| <mxGeometry x="0.0151" y="-3" relative="1" as="geometry"> | ||
| <mxPoint x="-1" as="offset"/> | ||
| </mxGeometry> | ||
| </mxCell> | ||
| <mxCell id="33" style="edgeStyle=none;html=1;entryX=0.5;entryY=1;entryDx=0;entryDy=0;entryPerimeter=0;" parent="1" source="21" target="18" edge="1"> | ||
| <mxGeometry relative="1" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="21" value="" style="aspect=fixed;html=1;points=[];align=center;image;fontSize=12;image=img/lib/azure2/general/Code.svg;fillColor=default;" parent="1" vertex="1"> | ||
| <mxGeometry x="887" y="250" width="64" height="52" as="geometry"/> | ||
| </mxCell> | ||
| <mxCell id="22" value="" style="endArrow=classic;html=1;" parent="1" target="21" edge="1"> | ||
| <mxGeometry width="50" height="50" relative="1" as="geometry"> | ||
| <mxPoint x="700" y="280" as="sourcePoint"/> | ||
| <mxPoint x="930" y="160" as="targetPoint"/> | ||
| </mxGeometry> | ||
| </mxCell> | ||
| <mxCell id="23" value="Spark Streaming" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" parent="22" vertex="1" connectable="0"> | ||
| <mxGeometry x="-0.2945" y="-4" relative="1" as="geometry"> | ||
| <mxPoint x="-1" as="offset"/> | ||
| </mxGeometry> | ||
| </mxCell> | ||
| <mxCell id="29" value="Superset" style="image;html=1;image=img/lib/clip_art/finance/Graph_128x128.png;fillColor=default;" parent="1" vertex="1"> | ||
| <mxGeometry x="1250" y="110" width="80" height="80" as="geometry"/> | ||
| </mxCell> | ||
| </root> | ||
| </mxGraphModel> | ||
| </diagram> | ||
| </mxfile> |
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| ## Project setup | ||
| Install the confluent kafka on your machine. This is the most easiest way as it comes packaged with all services like schema registry, connector. | ||
|
|
||
| [Installation instructions](https://docs.confluent.io/platform/current/installation/installing_cp/zip-tar.html){:target="_blank"} | ||
|
|
||
| ``` | ||
| confluent local services start | ||
| ``` | ||
|
|
||
| - Run the `ProducerDemo.java` | ||
|
|
||
| - Run the `ConsumerDemo.java` | ||
|
|
||
| Please take a look the output. | ||
|
|
||
| ``` | ||
| confluent local services stop | ||
| ``` | ||
|
|
||
| ## Things to Observe: | ||
|
|
||
| - The producer uses the java classes generated from the avdl files. | ||
| - The consumer actually decrypts the data it reads from the topic. | ||
| - If you use the kafka consumer cli you will see encrypted data. | ||
| - The custom logical types along with conversions defined make this happen. The only way to decrypt is by reading the data by using java classes generated. | ||
| - If the message on the topic is intercepted, it cannot be decrypted. |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| @namespace("com.dataanada.customer") | ||
| protocol Customer{ | ||
| record CustomerRecord { | ||
| int customerId; | ||
| @logicalType("encrypted") | ||
| string firstName; | ||
| @logicalType("encrypted") | ||
| string lastName; | ||
| @logicalType("encrypted") | ||
| string email; | ||
| string addressLine1; | ||
| string city; | ||
| string state; | ||
| int zip; | ||
| string country; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| @namespace("com.dataanada.order") | ||
| protocol Order{ | ||
| record OrderRecord { | ||
| int orderId; | ||
| union {null, int} storeId = null; | ||
| int customerId; | ||
| decimal(5,2) price; | ||
| string orderType; | ||
| } | ||
| } |
4 changes: 4 additions & 0 deletions
4
schemas/src/main/scala/com/dataanada/customer/CustomerRecord.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| /** MACHINE-GENERATED FROM AVRO SCHEMA. DO NOT EDIT DIRECTLY */ | ||
| package com.dataanada.customer | ||
|
|
||
| final case class CustomerRecord(customerId: Int, firstName: String, lastName: String, email: String, addressLine1: String, city: String, state: String, zip: Int, country: String) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| /** MACHINE-GENERATED FROM AVRO SCHEMA. DO NOT EDIT DIRECTLY */ | ||
| package com.dataanada.order | ||
|
|
||
| final case class OrderRecord(orderId: Int, storeId: Option[Int] = None, customerId: Int, price: BigDecimal, orderType: String) |
Oops, something went wrong.