From 3dde4cf20b53cae139c9571ef73d762e86b6fb01 Mon Sep 17 00:00:00 2001 From: "Pascal S. de Kloe" Date: Mon, 13 Aug 2018 15:19:28 +0200 Subject: [PATCH] Compare with Colfer. --- README.md | 39 ++- bench/Colfer.go | 720 +++++++++++++++++++++++++++++++++++++++++++ bench/all.bat | 1 + bench/colfer_test.go | 135 ++++++++ bench/fixed.colf | 8 + bench/test.colf | 12 + 6 files changed, 899 insertions(+), 16 deletions(-) create mode 100644 bench/Colfer.go create mode 100644 bench/colfer_test.go create mode 100644 bench/fixed.colf create mode 100644 bench/test.colf diff --git a/README.md b/README.md index 57d9dad..43f457c 100644 --- a/README.md +++ b/README.md @@ -117,24 +117,31 @@ The Request field will be declared of type Command, which must be an interface t Gencode encodes to smaller amounts of data, and does so very fast. Some benchmarks (using schemas and test files located in the bench folder): ``` +Colfer encoded size: 62 Gencode encoded size: 48 -GOB encoded size: 182 +GOB encoded size: 183 GOB Stream encoded size: 62 JSON encoded size: 138 MSGP encoded size: 115 -PASS -BenchmarkFixedBinarySerialize-8 2000000 894 ns/op -BenchmarkFixedBinaryDeserialize-8 3000000 539 ns/op -BenchmarkGencodeSerialize-8 10000000 174 ns/op -BenchmarkGencodeDeserialize-8 10000000 219 ns/op -BenchmarkFixedGencodeSerialize-8 20000000 75.7 ns/op -BenchmarkFixedGencodeDeserialize-8 100000000 20.7 ns/op -BenchmarkGobSerialize-8 200000 9370 ns/op -BenchmarkGobDeserialize-8 30000 40337 ns/op -BenchmarkGobStreamSerialize-8 1000000 1694 ns/op -BenchmarkGobStreamDeserialize-8 1000000 2125 ns/op -BenchmarkJSONSerialize-8 500000 2780 ns/op -BenchmarkJSONDeserialize-8 300000 5263 ns/op -BenchmarkMSGPSerialize-8 5000000 277 ns/op -BenchmarkMSGPDeserialize-8 2000000 608 ns/op +goos: darwin +goarch: amd64 +pkg: github.com/andyleap/gencode/bench +BenchmarkFixedBinarySerialize-12 3000000 569 ns/op +BenchmarkFixedBinaryDeserialize-12 5000000 358 ns/op +BenchmarkColferSerialize-12 30000000 45.2 ns/op 0 B/op 0 allocs/op +BenchmarkColferDeserialize-12 10000000 232 ns/op 144 B/op 6 allocs/op +BenchmarkFixedColferSerialize-12 200000000 9.10 ns/op +BenchmarkFixedColferDeserialize-12 200000000 9.62 ns/op +BenchmarkGencodeSerialize-12 20000000 90.3 ns/op 48 B/op 1 allocs/op +BenchmarkGencodeDeserialize-12 20000000 97.5 ns/op 16 B/op 4 allocs/op +BenchmarkFixedGencodeSerialize-12 100000000 11.4 ns/op +BenchmarkFixedGencodeDeserialize-12 200000000 7.79 ns/op +BenchmarkGobSerialize-12 200000 6737 ns/op +BenchmarkGobDeserialize-12 50000 28500 ns/op +BenchmarkGobStreamSerialize-12 1000000 1211 ns/op +BenchmarkGobStreamDeserialize-12 1000000 1420 ns/op +BenchmarkJSONSerialize-12 1000000 1847 ns/op +BenchmarkJSONDeserialize-12 300000 5087 ns/op +BenchmarkMSGPSerialize-12 10000000 132 ns/op 144 B/op 1 allocs/op +BenchmarkMSGPDeserialize-12 5000000 286 ns/op 16 B/op 4 allocs/op ``` diff --git a/bench/Colfer.go b/bench/Colfer.go new file mode 100644 index 0000000..947a39a --- /dev/null +++ b/bench/Colfer.go @@ -0,0 +1,720 @@ +package bench + +// Code generated by colf(1); DO NOT EDIT. +// The compiler used schema file test.colf and fixed.colf. + +import ( + "encoding/binary" + "fmt" + "io" + "math" +) + +var intconv = binary.BigEndian + +// Colfer configuration attributes +var ( + // ColferSizeMax is the upper limit for serial byte sizes. + ColferSizeMax = 16 * 1024 * 1024 + // ColferListMax is the upper limit for the number of elements in a list. + ColferListMax = 64 * 1024 +) + +// ColferMax signals an upper limit breach. +type ColferMax string + +// Error honors the error interface. +func (m ColferMax) Error() string { return string(m) } + +// ColferError signals a data mismatch as as a byte index. +type ColferError int + +// Error honors the error interface. +func (i ColferError) Error() string { + return fmt.Sprintf("colfer: unknown header at byte %d", i) +} + +// ColferTail signals data continuation as a byte index. +type ColferTail int + +// Error honors the error interface. +func (i ColferTail) Error() string { + return fmt.Sprintf("colfer: data continuation at byte %d", i) +} + +type ColferFixed struct { + A int64 + + B uint32 + + C float32 + + D float64 +} + +// MarshalTo encodes o as Colfer into buf and returns the number of bytes written. +// If the buffer is too small, MarshalTo will panic. +func (o *ColferFixed) MarshalTo(buf []byte) int { + var i int + + if v := o.A; v != 0 { + x := uint64(v) + if v >= 0 { + buf[i] = 0 + } else { + x = ^x + 1 + buf[i] = 0 | 0x80 + } + i++ + for n := 0; x >= 0x80 && n < 8; n++ { + buf[i] = byte(x | 0x80) + x >>= 7 + i++ + } + buf[i] = byte(x) + i++ + } + + if x := o.B; x >= 1<<21 { + buf[i] = 1 | 0x80 + intconv.PutUint32(buf[i+1:], x) + i += 5 + } else if x != 0 { + buf[i] = 1 + i++ + for x >= 0x80 { + buf[i] = byte(x | 0x80) + x >>= 7 + i++ + } + buf[i] = byte(x) + i++ + } + + if v := o.C; v != 0 { + buf[i] = 2 + intconv.PutUint32(buf[i+1:], math.Float32bits(v)) + i += 5 + } + + if v := o.D; v != 0 { + buf[i] = 3 + intconv.PutUint64(buf[i+1:], math.Float64bits(v)) + i += 9 + } + + buf[i] = 0x7f + i++ + return i +} + +// MarshalLen returns the Colfer serial byte size. +// The error return option is bench.ColferMax. +func (o *ColferFixed) MarshalLen() (int, error) { + l := 1 + + if v := o.A; v != 0 { + l += 2 + x := uint64(v) + if v < 0 { + x = ^x + 1 + } + for n := 0; x >= 0x80 && n < 8; n++ { + x >>= 7 + l++ + } + } + + if x := o.B; x >= 1<<21 { + l += 5 + } else if x != 0 { + for l += 2; x >= 0x80; l++ { + x >>= 7 + } + } + + if o.C != 0 { + l += 5 + } + + if o.D != 0 { + l += 9 + } + + if l > ColferSizeMax { + return l, ColferMax(fmt.Sprintf("colfer: struct bench.colferFixed exceeds %d bytes", ColferSizeMax)) + } + return l, nil +} + +// MarshalBinary encodes o as Colfer conform encoding.BinaryMarshaler. +// The error return option is bench.ColferMax. +func (o *ColferFixed) MarshalBinary() (data []byte, err error) { + l, err := o.MarshalLen() + if err != nil { + return nil, err + } + data = make([]byte, l) + o.MarshalTo(data) + return data, nil +} + +// Unmarshal decodes data as Colfer and returns the number of bytes read. +// The error return options are io.EOF, bench.ColferError and bench.ColferMax. +func (o *ColferFixed) Unmarshal(data []byte) (int, error) { + if len(data) == 0 { + return 0, io.EOF + } + header := data[0] + i := 1 + + if header == 0 { + if i+1 >= len(data) { + i++ + goto eof + } + x := uint64(data[i]) + i++ + + if x >= 0x80 { + x &= 0x7f + for shift := uint(7); ; shift += 7 { + b := uint64(data[i]) + i++ + if i >= len(data) { + goto eof + } + + if b < 0x80 || shift == 56 { + x |= b << shift + break + } + x |= (b & 0x7f) << shift + } + } + o.A = int64(x) + + header = data[i] + i++ + } else if header == 0|0x80 { + if i+1 >= len(data) { + i++ + goto eof + } + x := uint64(data[i]) + i++ + + if x >= 0x80 { + x &= 0x7f + for shift := uint(7); ; shift += 7 { + b := uint64(data[i]) + i++ + if i >= len(data) { + goto eof + } + + if b < 0x80 || shift == 56 { + x |= b << shift + break + } + x |= (b & 0x7f) << shift + } + } + o.A = int64(^x + 1) + + header = data[i] + i++ + } + + if header == 1 { + start := i + i++ + if i >= len(data) { + goto eof + } + x := uint32(data[start]) + + if x >= 0x80 { + x &= 0x7f + for shift := uint(7); ; shift += 7 { + b := uint32(data[i]) + i++ + if i >= len(data) { + goto eof + } + + if b < 0x80 { + x |= b << shift + break + } + x |= (b & 0x7f) << shift + } + } + o.B = x + + header = data[i] + i++ + } else if header == 1|0x80 { + start := i + i += 4 + if i >= len(data) { + goto eof + } + o.B = intconv.Uint32(data[start:]) + header = data[i] + i++ + } + + if header == 2 { + start := i + i += 4 + if i >= len(data) { + goto eof + } + o.C = math.Float32frombits(intconv.Uint32(data[start:])) + header = data[i] + i++ + } + + if header == 3 { + start := i + i += 8 + if i >= len(data) { + goto eof + } + o.D = math.Float64frombits(intconv.Uint64(data[start:])) + header = data[i] + i++ + } + + if header != 0x7f { + return 0, ColferError(i - 1) + } + if i < ColferSizeMax { + return i, nil + } +eof: + if i >= ColferSizeMax { + return 0, ColferMax(fmt.Sprintf("colfer: struct bench.colferFixed size exceeds %d bytes", ColferSizeMax)) + } + return 0, io.EOF +} + +// UnmarshalBinary decodes data as Colfer conform encoding.BinaryUnmarshaler. +// The error return options are io.EOF, bench.ColferError, bench.ColferTail and bench.ColferMax. +func (o *ColferFixed) UnmarshalBinary(data []byte) error { + i, err := o.Unmarshal(data) + if i < len(data) && err == nil { + return ColferTail(i) + } + return err +} + +type ColferPerson struct { + Name string + + Age uint8 + + Height float64 +} + +// MarshalTo encodes o as Colfer into buf and returns the number of bytes written. +// If the buffer is too small, MarshalTo will panic. +func (o *ColferPerson) MarshalTo(buf []byte) int { + var i int + + if l := len(o.Name); l != 0 { + buf[i] = 0 + i++ + x := uint(l) + for x >= 0x80 { + buf[i] = byte(x | 0x80) + x >>= 7 + i++ + } + buf[i] = byte(x) + i++ + i += copy(buf[i:], o.Name) + } + + if x := o.Age; x != 0 { + buf[i] = 1 + i++ + buf[i] = x + i++ + } + + if v := o.Height; v != 0 { + buf[i] = 2 + intconv.PutUint64(buf[i+1:], math.Float64bits(v)) + i += 9 + } + + buf[i] = 0x7f + i++ + return i +} + +// MarshalLen returns the Colfer serial byte size. +// The error return option is bench.ColferMax. +func (o *ColferPerson) MarshalLen() (int, error) { + l := 1 + + if x := len(o.Name); x != 0 { + if x > ColferSizeMax { + return 0, ColferMax(fmt.Sprintf("colfer: field bench.colferPerson.name exceeds %d bytes", ColferSizeMax)) + } + for l += x + 2; x >= 0x80; l++ { + x >>= 7 + } + } + + if x := o.Age; x != 0 { + l += 2 + } + + if o.Height != 0 { + l += 9 + } + + if l > ColferSizeMax { + return l, ColferMax(fmt.Sprintf("colfer: struct bench.colferPerson exceeds %d bytes", ColferSizeMax)) + } + return l, nil +} + +// MarshalBinary encodes o as Colfer conform encoding.BinaryMarshaler. +// The error return option is bench.ColferMax. +func (o *ColferPerson) MarshalBinary() (data []byte, err error) { + l, err := o.MarshalLen() + if err != nil { + return nil, err + } + data = make([]byte, l) + o.MarshalTo(data) + return data, nil +} + +// Unmarshal decodes data as Colfer and returns the number of bytes read. +// The error return options are io.EOF, bench.ColferError and bench.ColferMax. +func (o *ColferPerson) Unmarshal(data []byte) (int, error) { + if len(data) == 0 { + return 0, io.EOF + } + header := data[0] + i := 1 + + if header == 0 { + if i >= len(data) { + goto eof + } + x := uint(data[i]) + i++ + + if x >= 0x80 { + x &= 0x7f + for shift := uint(7); ; shift += 7 { + if i >= len(data) { + goto eof + } + b := uint(data[i]) + i++ + + if b < 0x80 { + x |= b << shift + break + } + x |= (b & 0x7f) << shift + } + } + + if x > uint(ColferSizeMax) { + return 0, ColferMax(fmt.Sprintf("colfer: bench.colferPerson.name size %d exceeds %d bytes", x, ColferSizeMax)) + } + + start := i + i += int(x) + if i >= len(data) { + goto eof + } + o.Name = string(data[start:i]) + + header = data[i] + i++ + } + + if header == 1 { + start := i + i++ + if i >= len(data) { + goto eof + } + o.Age = data[start] + header = data[i] + i++ + } + + if header == 2 { + start := i + i += 8 + if i >= len(data) { + goto eof + } + o.Height = math.Float64frombits(intconv.Uint64(data[start:])) + header = data[i] + i++ + } + + if header != 0x7f { + return 0, ColferError(i - 1) + } + if i < ColferSizeMax { + return i, nil + } +eof: + if i >= ColferSizeMax { + return 0, ColferMax(fmt.Sprintf("colfer: struct bench.colferPerson size exceeds %d bytes", ColferSizeMax)) + } + return 0, io.EOF +} + +// UnmarshalBinary decodes data as Colfer conform encoding.BinaryUnmarshaler. +// The error return options are io.EOF, bench.ColferError, bench.ColferTail and bench.ColferMax. +func (o *ColferPerson) UnmarshalBinary(data []byte) error { + i, err := o.Unmarshal(data) + if i < len(data) && err == nil { + return ColferTail(i) + } + return err +} + +type ColferGroup struct { + Name string + + Members []*ColferPerson +} + +// MarshalTo encodes o as Colfer into buf and returns the number of bytes written. +// If the buffer is too small, MarshalTo will panic. +// All nil entries in o.Members will be replaced with a new value. +func (o *ColferGroup) MarshalTo(buf []byte) int { + var i int + + if l := len(o.Name); l != 0 { + buf[i] = 0 + i++ + x := uint(l) + for x >= 0x80 { + buf[i] = byte(x | 0x80) + x >>= 7 + i++ + } + buf[i] = byte(x) + i++ + i += copy(buf[i:], o.Name) + } + + if l := len(o.Members); l != 0 { + buf[i] = 1 + i++ + x := uint(l) + for x >= 0x80 { + buf[i] = byte(x | 0x80) + x >>= 7 + i++ + } + buf[i] = byte(x) + i++ + for vi, v := range o.Members { + if v == nil { + v = new(ColferPerson) + o.Members[vi] = v + } + i += v.MarshalTo(buf[i:]) + } + } + + buf[i] = 0x7f + i++ + return i +} + +// MarshalLen returns the Colfer serial byte size. +// The error return option is bench.ColferMax. +func (o *ColferGroup) MarshalLen() (int, error) { + l := 1 + + if x := len(o.Name); x != 0 { + if x > ColferSizeMax { + return 0, ColferMax(fmt.Sprintf("colfer: field bench.colferGroup.name exceeds %d bytes", ColferSizeMax)) + } + for l += x + 2; x >= 0x80; l++ { + x >>= 7 + } + } + + if x := len(o.Members); x != 0 { + if x > ColferListMax { + return 0, ColferMax(fmt.Sprintf("colfer: field bench.colferGroup.members exceeds %d elements", ColferListMax)) + } + for l += 2; x >= 0x80; l++ { + x >>= 7 + } + for _, v := range o.Members { + if v == nil { + l++ + continue + } + vl, err := v.MarshalLen() + if err != nil { + return 0, err + } + l += vl + } + if l > ColferSizeMax { + return 0, ColferMax(fmt.Sprintf("colfer: struct bench.colferGroup size exceeds %d bytes", ColferSizeMax)) + } + } + + if l > ColferSizeMax { + return l, ColferMax(fmt.Sprintf("colfer: struct bench.colferGroup exceeds %d bytes", ColferSizeMax)) + } + return l, nil +} + +// MarshalBinary encodes o as Colfer conform encoding.BinaryMarshaler. +// All nil entries in o.Members will be replaced with a new value. +// The error return option is bench.ColferMax. +func (o *ColferGroup) MarshalBinary() (data []byte, err error) { + l, err := o.MarshalLen() + if err != nil { + return nil, err + } + data = make([]byte, l) + o.MarshalTo(data) + return data, nil +} + +// Unmarshal decodes data as Colfer and returns the number of bytes read. +// The error return options are io.EOF, bench.ColferError and bench.ColferMax. +func (o *ColferGroup) Unmarshal(data []byte) (int, error) { + if len(data) == 0 { + return 0, io.EOF + } + header := data[0] + i := 1 + + if header == 0 { + if i >= len(data) { + goto eof + } + x := uint(data[i]) + i++ + + if x >= 0x80 { + x &= 0x7f + for shift := uint(7); ; shift += 7 { + if i >= len(data) { + goto eof + } + b := uint(data[i]) + i++ + + if b < 0x80 { + x |= b << shift + break + } + x |= (b & 0x7f) << shift + } + } + + if x > uint(ColferSizeMax) { + return 0, ColferMax(fmt.Sprintf("colfer: bench.colferGroup.name size %d exceeds %d bytes", x, ColferSizeMax)) + } + + start := i + i += int(x) + if i >= len(data) { + goto eof + } + o.Name = string(data[start:i]) + + header = data[i] + i++ + } + + if header == 1 { + if i >= len(data) { + goto eof + } + x := uint(data[i]) + i++ + + if x >= 0x80 { + x &= 0x7f + for shift := uint(7); ; shift += 7 { + if i >= len(data) { + goto eof + } + b := uint(data[i]) + i++ + + if b < 0x80 { + x |= b << shift + break + } + x |= (b & 0x7f) << shift + } + } + + if x > uint(ColferListMax) { + return 0, ColferMax(fmt.Sprintf("colfer: bench.colferGroup.members length %d exceeds %d elements", x, ColferListMax)) + } + + l := int(x) + a := make([]*ColferPerson, l) + malloc := make([]ColferPerson, l) + for ai := range a { + v := &malloc[ai] + a[ai] = v + + n, err := v.Unmarshal(data[i:]) + if err != nil { + if err == io.EOF && len(data) >= ColferSizeMax { + return 0, ColferMax(fmt.Sprintf("colfer: bench.colferGroup size exceeds %d bytes", ColferSizeMax)) + } + return 0, err + } + i += n + } + o.Members = a + + if i >= len(data) { + goto eof + } + header = data[i] + i++ + } + + if header != 0x7f { + return 0, ColferError(i - 1) + } + if i < ColferSizeMax { + return i, nil + } +eof: + if i >= ColferSizeMax { + return 0, ColferMax(fmt.Sprintf("colfer: struct bench.colferGroup size exceeds %d bytes", ColferSizeMax)) + } + return 0, io.EOF +} + +// UnmarshalBinary decodes data as Colfer conform encoding.BinaryUnmarshaler. +// The error return options are io.EOF, bench.ColferError, bench.ColferTail and bench.ColferMax. +func (o *ColferGroup) UnmarshalBinary(data []byte) error { + i, err := o.Unmarshal(data) + if i < len(data) && err == nil { + return ColferTail(i) + } + return err +} diff --git a/bench/all.bat b/bench/all.bat index f172d7c..ab25d63 100644 --- a/bench/all.bat +++ b/bench/all.bat @@ -1,3 +1,4 @@ ..\gencode go -schema=test.schema ..\gencode go -schema=fixed.schema +REM colf -f -b .. go go test -bench=. \ No newline at end of file diff --git a/bench/colfer_test.go b/bench/colfer_test.go new file mode 100644 index 0000000..525f5f5 --- /dev/null +++ b/bench/colfer_test.go @@ -0,0 +1,135 @@ +package bench + +import ( + "fmt" + "testing" +) + +func TestColferSize(t *testing.T) { + p := ColferGroup{ + Name: "test", + Members: []*ColferPerson{ + { + Name: "John", + Age: 21, + Height: 5.9, + }, + { + Name: "Tom", + Age: 23, + Height: 5.8, + }, + { + Name: "Alan", + Age: 24, + Height: 6, + }, + }, + } + n, err := p.MarshalLen() + if err != nil { + t.Fatal(err) + } + fmt.Printf("Colfer encoded size: %v\n", n) +} + +func BenchmarkColferSerialize(b *testing.B) { + p := ColferGroup{ + Name: "test", + Members: []*ColferPerson{ + { + Name: "John", + Age: 21, + Height: 5.9, + }, + { + Name: "Tom", + Age: 23, + Height: 5.8, + }, + { + Name: "Alan", + Age: 24, + Height: 6, + }, + }, + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + var buf [100]byte + p.MarshalTo(buf[:]) + } +} + +func BenchmarkColferDeserialize(b *testing.B) { + p := ColferGroup{ + Name: "test", + Members: []*ColferPerson{ + { + Name: "John", + Age: 21, + Height: 5.9, + }, + { + Name: "Tom", + Age: 23, + Height: 5.8, + }, + { + Name: "Alan", + Age: 24, + Height: 6, + }, + }, + } + buf, err := p.MarshalBinary() + if err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := p.Unmarshal(buf) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkFixedColferSerialize(b *testing.B) { + p := ColferFixed{ + A: -5, + B: 6, + C: 6.7, + D: 12.65, + } + buf, err := p.MarshalBinary() + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + p.MarshalTo(buf) + } +} + +func BenchmarkFixedColferDeserialize(b *testing.B) { + p := ColferFixed{ + A: -5, + B: 6, + C: 6.7, + D: 12.65, + } + buf, err := p.MarshalBinary() + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := p.Unmarshal(buf) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/bench/fixed.colf b/bench/fixed.colf new file mode 100644 index 0000000..7b40a61 --- /dev/null +++ b/bench/fixed.colf @@ -0,0 +1,8 @@ +package bench + +type colferFixed struct { + a int64 + b uint32 + c float32 + d float64 +} diff --git a/bench/test.colf b/bench/test.colf new file mode 100644 index 0000000..45bbd9d --- /dev/null +++ b/bench/test.colf @@ -0,0 +1,12 @@ +package bench + +type colferPerson struct { + name text + age uint8 + height float64 +} + +type colferGroup struct { + name text + members []colferPerson +}