Last active
February 11, 2021 09:20
-
-
Save alexvictoor/1d3937f502c60318071f to your computer and use it in GitHub Desktop.
Demo of Protobuff integration within Avro
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" | |
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> | |
<modelVersion>4.0.0</modelVersion> | |
<groupId>com.avro.is.great</groupId> | |
<artifactId>protobuff-avro-demo</artifactId> | |
<packaging>jar</packaging> | |
<version>1.0-SNAPSHOT</version> | |
<name>Demo of protobuff integration with Avro</name> | |
<build> | |
<plugins> | |
<plugin> | |
<groupId>org.codehaus.mojo</groupId> | |
<artifactId>build-helper-maven-plugin</artifactId> | |
<executions> | |
<execution> | |
<phase>generate-sources</phase> | |
<goals> | |
<goal>add-source</goal> | |
</goals> | |
<configuration> | |
<sources> | |
<source>${project.build.directory}/generated-sources</source> | |
</sources> | |
</configuration> | |
</execution> | |
</executions> | |
</plugin> | |
<plugin> | |
<artifactId>maven-antrun-plugin</artifactId> | |
<executions> | |
<execution> | |
<id>generate-sources</id> | |
<phase>generate-sources</phase> | |
<configuration> | |
<tasks> | |
<echo>**** protoc need to be in the path ****</echo> | |
<mkdir dir="target/generated-sources"/> | |
<exec executable="protoc"> | |
<arg value="--java_out=target/generated-sources" /> | |
<arg value="--proto_path=src/main/resources/protobuf" /> | |
<arg value="src/main/resources/protobuf/todo_provider.proto" /> | |
</exec> | |
</tasks> | |
<sourceRoot>>${project.build.directory}/generated-sources</sourceRoot> | |
</configuration> | |
<goals> | |
<goal>run</goal> | |
</goals> | |
</execution> | |
</executions> | |
</plugin> | |
</plugins> | |
</build> | |
<dependencies> | |
<dependency> | |
<groupId>com.google.protobuf</groupId> | |
<artifactId>protobuf-java</artifactId> | |
<version>2.6.1</version> | |
</dependency> | |
<dependency> | |
<groupId>org.apache.avro</groupId> | |
<artifactId>avro</artifactId> | |
<version>1.7.7</version> | |
</dependency> | |
<dependency> | |
<groupId>org.apache.avro</groupId> | |
<artifactId>avro-protobuf</artifactId> | |
<version>1.7.7</version> | |
</dependency> | |
</dependencies> | |
</project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package com.demo; | |
import com.avro.is.great.TodoProvider; | |
import org.apache.avro.protobuf.ProtobufDatumReader; | |
import java.io.IOException; | |
public class Proto2Avro { | |
public static void main(String[] args) throws IOException { | |
System.out.println("******************************************************************************"); | |
System.out.println("******** How to convert a proto schema to avro with 2 lines of code **********"); | |
System.out.println("******************************************************************************"); | |
System.out.println("protobuff schema description:"); | |
System.out.println(TodoProvider.Todo.getDescriptor().toProto()); | |
System.out.println("----------"); | |
System.out.println("Avro schema:"); | |
ProtobufDatumReader<TodoProvider.Todo> datumReader = new ProtobufDatumReader<TodoProvider.Todo>(TodoProvider.Todo.class); | |
System.out.println(datumReader.getSchema().toString(true)); | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package com.demo; | |
import com.avro.is.great.TodoProvider; | |
import org.apache.avro.generic.GenericDatumReader; | |
import org.apache.avro.generic.GenericRecord; | |
import org.apache.avro.io.DecoderFactory; | |
import org.apache.avro.io.Encoder; | |
import org.apache.avro.io.EncoderFactory; | |
import org.apache.avro.protobuf.ProtobufDatumReader; | |
import org.apache.avro.protobuf.ProtobufDatumWriter; | |
import java.io.ByteArrayInputStream; | |
import java.io.ByteArrayOutputStream; | |
import java.io.IOException; | |
public class ProtoWithAvro { | |
public static void main(String[] args) throws IOException { | |
System.out.println("******************************************************************************"); | |
System.out.println("******** DTO classes generated by Protobuff protoc can be used with Avro *****"); | |
System.out.println("******************************************************************************"); | |
// build a protobuff dto | |
TodoProvider.Todo.Builder builder = TodoProvider.Todo.newBuilder(); | |
builder.setTitle("coucou !"); | |
TodoProvider.Todo todo = builder.build(); | |
// serialize with avro this dto | |
ProtobufDatumWriter<TodoProvider.Todo> datumWriter = new ProtobufDatumWriter<TodoProvider.Todo>(TodoProvider.Todo.class); | |
ByteArrayOutputStream os = new ByteArrayOutputStream(); | |
Encoder e = EncoderFactory.get().binaryEncoder(os, null); | |
datumWriter.write(todo, e); | |
e.flush(); | |
// deserialize with avro the former dto | |
ProtobufDatumReader<TodoProvider.Todo> datumReader = new ProtobufDatumReader<TodoProvider.Todo>(TodoProvider.Todo.class); | |
Object o = datumReader.read | |
(null, | |
DecoderFactory.get().binaryDecoder(new ByteArrayInputStream(os.toByteArray()), null)); | |
System.out.println("Protobuff DTO deserialized by avro " + o); | |
// deserialize with avro, without using the protobuff dto | |
GenericDatumReader<GenericRecord> genericDatumReader = new GenericDatumReader<GenericRecord>(datumReader.getSchema()); | |
GenericRecord record = genericDatumReader.read(null, DecoderFactory.get().binaryDecoder(new ByteArrayInputStream(os.toByteArray()), null)); | |
System.out.println("Avro Record deserialized " + record); | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package examples; | |
option java_package = "com.avro.is.great"; | |
option java_outer_classname = "TodoProvider"; | |
message Todo { | |
required string title = 1; | |
enum Priority { | |
NORMAL = 1; | |
MEDIUM = 2; | |
HIGH = 3; | |
} | |
optional Priority priority = 2; | |
} | |
message Todos { | |
repeated Todo todos = 1; | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment