Last active
November 4, 2021 13:15
-
-
Save jgfrancisco/318b2f5ab77c7f5acb88 to your computer and use it in GitHub Desktop.
compress protobuf
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package main | |
| import ( | |
| "bytes" | |
| "compress/flate" | |
| "compress/gzip" | |
| "compress/zlib" | |
| "errors" | |
| "fmt" | |
| "io/ioutil" | |
| "math/rand" | |
| "time" | |
| "github.com/gogo/protobuf/proto" | |
| "github.com/zvelo/adfraud/msg" | |
| ) | |
| var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") | |
| func randSeq(n int) string { | |
| b := make([]rune, n) | |
| for i := range b { | |
| b[i] = letters[rand.Intn(len(letters))] | |
| } | |
| return string(b) | |
| } | |
| func check(err error) { | |
| if err != nil { | |
| panic(err) | |
| } | |
| } | |
| func main() { | |
| var ( | |
| buf bytes.Buffer | |
| n int | |
| s time.Time | |
| elapsed time.Duration | |
| err error | |
| obj []byte | |
| test = msg.Request{ | |
| Id: randSeq(1024 * 2), | |
| } | |
| ) | |
| t, err := proto.Marshal(&test) | |
| check(err) | |
| fmt.Printf("size of protobuf: %d\n\n", len(t)) | |
| buf.Reset() | |
| fw, err := flate.NewWriter(&buf, 9) | |
| check(err) | |
| defer fw.Close() | |
| s = time.Now() | |
| n, err = fw.Write(t) | |
| check(err) | |
| fw.Close() // need to explicitly close and not just flush to prevent EOF error | |
| elapsed = time.Since(s) | |
| obj = buf.Bytes() | |
| fmt.Printf("written to flated: %d\n", n) | |
| fmt.Printf("size of flated: %d\n", len(obj)) | |
| fmt.Printf("elapsed flated: %v\n\n", elapsed) | |
| err = decompressFlate(obj, &test) | |
| check(err) | |
| buf.Reset() | |
| zw := zlib.NewWriter(&buf) | |
| defer zw.Close() | |
| s = time.Now() | |
| n, err = zw.Write(t) | |
| check(err) | |
| zw.Flush() | |
| elapsed = time.Since(s) | |
| obj = buf.Bytes() | |
| fmt.Printf("written to zipped: %d\n", n) | |
| fmt.Printf("size of zlib: %d\n", len(obj)) | |
| fmt.Printf("elapsed zlib: %v\n\n", elapsed) | |
| buf.Reset() | |
| gw := gzip.NewWriter(&buf) | |
| defer gw.Close() | |
| s = time.Now() | |
| n, err = gw.Write(t) | |
| check(err) | |
| gw.Flush() | |
| elapsed = time.Since(s) | |
| obj = buf.Bytes() | |
| fmt.Printf("written to gzipped: %d\n", n) | |
| fmt.Printf("size of gzip: %d\n", len(obj)) | |
| fmt.Printf("elapsed gzip: %v\n\n", elapsed) | |
| } | |
| func decompressFlate(obj []byte, expected *msg.Request) error { | |
| fmt.Println("Decompressing") | |
| data := bytes.NewReader(obj) | |
| r := flate.NewReader(data) | |
| enflate, err := ioutil.ReadAll(r) | |
| if err != nil { | |
| panic(err) | |
| } | |
| result := msg.Request{} | |
| proto.Unmarshal(enflate, &result) | |
| if result.Id != expected.Id { | |
| fmt.Printf("RESULT[%s] != EXPECTED[%s]\n", result.Id, expected.Id) | |
| return errors.New("not equal") | |
| } | |
| fmt.Println("DECOMPRESSION EQUAL") | |
| return nil | |
| } |
size of protobuf: 2051
written to zipped: 2051
size of zlib: 1510
elapsed zlib: 1.390776ms
written to gzipped: 2051
size of gzip: 1518
elapsed gzip: 1.489147ms
written to flated: 2051
size of flated: 1508
elapsed flated: 685.489µs
size of protobuf: 2051
written to flated: 2051
size of flated: 1508
elapsed flated: 619.604µs
written to zipped: 2051
size of zlib: 1510
elapsed zlib: 1.521965ms
written to gzipped: 2051
size of gzip: 1518
elapsed gzip: 931.558µs
gzip time improves when running after flate
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
size of protobuf: 2051
written to zipped: 2051
size of zlib: 1510
written to gzipped: 2051
size of gzip: 1518