Skip to content

Instantly share code, notes, and snippets.

@filipelenfers
Last active August 23, 2024 16:04
Show Gist options
  • Save filipelenfers/80feef186407b1eb72e111e4d2501ae5 to your computer and use it in GitHub Desktop.
Save filipelenfers/80feef186407b1eb72e111e4d2501ae5 to your computer and use it in GitHub Desktop.
Generate AVRO bytes as a in-memory file, with the schema, Python3.
import io
import snappy
import avro.schema
from avro.datafile import DataFileReader, DataFileWriter
from avro.io import DatumReader, DatumWriter, BinaryEncoder
#Generate avro bytes as a in-memory file, with the schema.
def generate_avro_bytes(schema,avro_dicts, codec='snappy'):
bytes_writer = io.BytesIO()
writer = DataFileWriter(bytes_writer, DatumWriter(), schema, codec=codec)
for d in avro_dicts:
writer.append(d)
writer.flush()
bytes_value = bytes_writer.getvalue()
writer.close()
return bytes_value
#Read avro file
# file_path = '/path/to/avro/file.avro'
# reader = DataFileReader(open(file_path, "rb"), DatumReader())
# for rec in reader:
# print(rec)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment