pquerna
Slower than encoding/json at all levels of GOMAXPROCS (8 cores, i7 macbook pro 16gb ram)
All, my results of running benchmarks against ffjson and encoding/json show that ffjson does not meet the claims of being 2-3x faster than encoding/json; and, in-fact, is much slower.
Should the documentation be revised, or did I do something 'wrong'?
Results (GOMAXPROCS 1):
MyComp:gotestffjson mo.omer$ ffjson ./main.go
./main_ffjson.go
# Erm, imagine a build in here somewhere; just wanted to emphasize that I'd created the ffjson file.
MyComp:gotestffjson mo.omer$ ./main
2015/04/27 11:34:35 GOMAXPROCS: 1
2015/04/27 11:34:41 encoding/json Marshal Function: 1000000 1297 ns/op
2015/04/27 11:34:41 encoding/json Encoder Function: 1000000 1121 ns/op
2015/04/27 11:34:41 ffjson Marshal Function: 1000000 1385 ns/op
2015/04/27 11:34:41 ffjson Encoder Function: 1000000 1888 ns/op
Results (GOMAXPROCS: 8):
MyComp:gotestffjson mo.omer$ ./main
2015/04/27 11:30:18 GOMAXPROCS: 8
2015/04/27 11:30:24 encoding/json Marshal Function: 1000000 1024 ns/op
2015/04/27 11:30:24 encoding/json Encoder Function: 2000000 973 ns/op
2015/04/27 11:30:24 ffjson Marshal Function: 1000000 1093 ns/op
2015/04/27 11:30:24 ffjson Encoder Function: 1000000 1488 ns/op
main.go:
package main
import (
"encoding/json"
"github.com/pquerna/ffjson/ffjson"
"io"
"log"
"net/http/httptest"
"runtime"
"testing"
)
const helloWorldString = "Hello, World!"
type Message struct {
Message string `json:"message"`
}
func Encode(item interface{}, out io.Writer) {
// Encode
buf, err := ffjson.Marshal(&item)
if err != nil {
log.Fatalf("Error: %+v\n", err)
}
// Write the buffer
_, _ = out.Write(buf)
// We are now no longer need the buffer so we pool it.
ffjson.Pool(buf)
}
func encoderFunction(b *testing.B) {
for i := 0; i < b.N; i++ {
w := httptest.NewRecorder()
_ = json.NewEncoder(w).Encode(&Message{helloWorldString})
}
}
func ffencoderFunction(b *testing.B) {
for i := 0; i < b.N; i++ {
w := httptest.NewRecorder()
Encode(&Message{helloWorldString}, w)
}
}
func marshalFunction(b *testing.B) {
for i := 0; i < b.N; i++ {
w := httptest.NewRecorder()
a, _ := json.Marshal(&Message{helloWorldString})
w.Write(a)
}
}
func ffmarshalFunction(b *testing.B) {
for i := 0; i < b.N; i++ {
w := httptest.NewRecorder()
a, _ := ffjson.Marshal(&Message{helloWorldString})
w.Write(a)
}
}
func main() {
// numProcs := runtime.NumCPU()
numProcs := 1
runtime.GOMAXPROCS(numProcs)
log.Println("GOMAXPROCS: ", numProcs)
ffmf := testing.Benchmark(ffmarshalFunction)
ffef := testing.Benchmark(ffencoderFunction)
mf := testing.Benchmark(marshalFunction)
ef := testing.Benchmark(encoderFunction)
log.Println("encoding/json Marshal Function: ", mf)
log.Println("encoding/json Encoder Function: ", ef)
log.Println("ffjson Marshal Function: ", ffmf)
log.Println("ffjson Encoder Function: ", ffef)
}
main_ffjson.go:
// DO NOT EDIT!
// Code generated by ffjson <https://github.com/pquerna/ffjson>
// source: ./main.go
// DO NOT EDIT!
package main
import (
"bytes"
"fmt"
fflib "github.com/pquerna/ffjson/fflib/v1"
)
func (mj *Message) MarshalJSON() ([]byte, error) {
var buf fflib.Buffer
err := mj.MarshalJSONBuf(&buf)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func (mj *Message) MarshalJSONBuf(buf fflib.EncodingBuffer) error {
var err error
var obj []byte
_ = obj
_ = err
buf.WriteString(`{"message":`)
fflib.WriteJsonString(buf, string(mj.Message))
buf.WriteByte('}')
return nil
}
const (
ffj_t_Messagebase = iota
ffj_t_Messageno_such_key
ffj_t_Message_Message
)
var ffj_key_Message_Message = []byte("message")
func (uj *Message) UnmarshalJSON(input []byte) error {
fs := fflib.NewFFLexer(input)
return uj.UnmarshalJSONFFLexer(fs, fflib.FFParse_map_start)
}
func (uj *Message) UnmarshalJSONFFLexer(fs *fflib.FFLexer, state fflib.FFParseState) error {
var err error = nil
currentKey := ffj_t_Messagebase
_ = currentKey
tok := fflib.FFTok_init
wantedTok := fflib.FFTok_init
mainparse:
for {
tok = fs.Scan()
// println(fmt.Sprintf("debug: tok: %v state: %v", tok, state))
if tok == fflib.FFTok_error {
goto tokerror
}
switch state {
case fflib.FFParse_map_start:
if tok != fflib.FFTok_left_bracket {
wantedTok = fflib.FFTok_left_bracket
goto wrongtokenerror
}
state = fflib.FFParse_want_key
continue
case fflib.FFParse_after_value:
if tok == fflib.FFTok_comma {
state = fflib.FFParse_want_key
} else if tok == fflib.FFTok_right_bracket {
goto done
} else {
wantedTok = fflib.FFTok_comma
goto wrongtokenerror
}
case fflib.FFParse_want_key:
// json {} ended. goto exit. woo.
if tok == fflib.FFTok_right_bracket {
goto done
}
if tok != fflib.FFTok_string {
wantedTok = fflib.FFTok_string
goto wrongtokenerror
}
kn := fs.Output.Bytes()
if len(kn) <= 0 {
// "" case. hrm.
currentKey = ffj_t_Messageno_such_key
state = fflib.FFParse_want_colon
goto mainparse
} else {
switch kn[0] {
case 'm':
if bytes.Equal(ffj_key_Message_Message, kn) {
currentKey = ffj_t_Message_Message
state = fflib.FFParse_want_colon
goto mainparse
}
}
if fflib.EqualFoldRight(ffj_key_Message_Message, kn) {
currentKey = ffj_t_Message_Message
state = fflib.FFParse_want_colon
goto mainparse
}
currentKey = ffj_t_Messageno_such_key
state = fflib.FFParse_want_colon
goto mainparse
}
case fflib.FFParse_want_colon:
if tok != fflib.FFTok_colon {
wantedTok = fflib.FFTok_colon
goto wrongtokenerror
}
state = fflib.FFParse_want_value
continue
case fflib.FFParse_want_value:
if tok == fflib.FFTok_left_brace || tok == fflib.FFTok_left_bracket || tok == fflib.FFTok_integer || tok == fflib.FFTok_double || tok == fflib.FFTok_string || tok == fflib.FFTok_bool || tok == fflib.FFTok_null {
switch currentKey {
case ffj_t_Message_Message:
goto handle_Message
case ffj_t_Messageno_such_key:
err = fs.SkipField(tok)
if err != nil {
return fs.WrapErr(err)
}
state = fflib.FFParse_after_value
goto mainparse
}
} else {
goto wantedvalue
}
}
}
handle_Message:
/* handler: uj.Message type=string kind=string */
{
{
if tok != fflib.FFTok_string && tok != fflib.FFTok_null {
return fs.WrapErr(fmt.Errorf("cannot unmarshal %s into Go value for string", tok))
}
}
if tok == fflib.FFTok_null {
} else {
uj.Message = string(fs.Output.String())
}
}
state = fflib.FFParse_after_value
goto mainparse
wantedvalue:
return fs.WrapErr(fmt.Errorf("wanted value token, but got token: %v", tok))
wrongtokenerror:
return fs.WrapErr(fmt.Errorf("ffjson: wanted token: %v, but got token: %v output=%s", wantedTok, tok, fs.Output.String()))
tokerror:
if fs.BigError != nil {
return fs.WrapErr(fs.BigError)
}
err = fs.Error.ToError()
if err != nil {
return fs.WrapErr(err)
}
panic("ffjson-generated: unreachable, please report bug.")
done:
return nil
}
"much slower" is rather an overstatement. First of all use real-world data, a single string isn't going to tell you much.
Also, I don't think the ffjson code is actually included in your build. The ffjson must be generated before you build your executable.
I revised the encoder a bit: https://gist.github.com/klauspost/0aacfc0cbb9eee99b9db
- Don't use ffjson generated types for testing encoding/json.
- Reduce allocations, so it doesn't give noise to the benchmark
- Use the encoder function
- Check errors
- Use the "Fast" functions. Performance will be the same, but you will get an error if ffjson isn't generated.
With that, I get this on my machine:
encoding/json Marshal Function: 1000000 2687 ns/op
ffjson Marshal Function: 1000000 1937 ns/op
encoding/json Encoder Function: 1000000 2031 ns/op
ffjson Encoder Function: 500000 2531 ns/op
The ffjson code was generated before the build, of course, though I'd neglected to copy/pasta it into the example; my apologies for the misleading paste. I just wanted to emphasize that it was being used. Good catch on the usage of the ffjson methods/type in the encoding/json benchmarks, though!
I didn't test the ffjson Encoder, but rather the README Encode func, but it seems that the Encoder func you used closed the performance gap from ~50% slower to ~25% slower.
MarshalFast seems like the way to go, as it's around 38% faster than the encoding/json Marshal on your machine.
ffjson is a really interesting and fun project; however, the claims are "2-3x" faster than encoding/json, which are false. Additionally, the tuned encoder methods are still 25% slower.
Thoughts on either addressing the performance or documentation?
Maybe I'll run some further tests on more complex structs to see if the gap widens in ffjson's favor. In which case, that should probably be mentioned somewhere.
The performance numbers are inline for 'real world' structures, eg goser:
https://github.com/pquerna/ffjson/blob/master/tests/goser/ff/goser.go
and go.stripe:
https://github.com/pquerna/ffjson/blob/master/tests/go.stripe/ff/customer.go
I'm happy to add more structures -- ideally ones used in real world projects -- to the test suite to keep up to date benchmarks.
Ah yep, I'd just amended the comment above; I'll play around with those this evening. Maybe that should be mentioned in the docs, though.
Right now, the README mentions that we should ffjson all the things and shows an example which is similar to mine.
The ffjson code was generated before the build, of course,
Then something else is wrong, because using your code yields:
encoding/json Marshal Function: 200000 5000 ns/op
ffjson Marshal Function: 1000000 2312 ns/op
encoding/json Encoder Function: 300000 4635 ns/op
ffjson Encoder Function: 500000 2593 ns/op
But that would be dishonest, since it disfavours encoding/json (therefore the XMessage struct).
MarshalFast seems like the way to go, as it's around 38% faster than the encoding/json Marshal on your machine.
As I wrote, and as written in the docs, MarshalFast is the same speed as Marshal, it only checks if the struct can be marshalled fast and then calls Marshal.
Right now, the README mentions that we should ffjson all the things and shows an example which is similar to mine.
No. You encode a string in a struct. The example encodes a struct in a struct.
Don't know what to tell you about the code and the results disparity; I ran my bench a couple times today and got very similar results. Will post again with the results set and your XMessage example soon.
For what it's worth, here's what I get running @klauspost's version with current master:
2015/05/31 18:12:08 GOMAXPROCS: 1
encoding/json Marshal Function: 1000000 1994 ns/op
ffjson Marshal Function: 1000000 1630 ns/op
encoding/json Encoder Function: 1000000 1876 ns/op
ffjson Encoder Function: 500000 2198 ns/op
I agree though that the unqualified claims of 2-3x speed up are a bit misleading.
oldie% ./tests
2015/09/08 14:19:49 GOMAXPROCS: 1
2015/09/08 14:19:56 encoding/json Marshal Function: 1000000 1477 ns/op
2015/09/08 14:19:56 encoding/json Encoder Function: 1000000 1452 ns/op
2015/09/08 14:19:56 ffjson Marshal Function: 2000000 714 ns/op
2015/09/08 14:19:56 ffjson Encoder Function: 1000000 1634 ns/op
oldie [./tests] INS| ~/tmp/tests
oldie%
There is a bug in the Encode function, you pass a *interface{} to ffjson.Marshal which probably triggers a fallback to a standard json.Marshal. This is why the Encoder results are so slow.
You could easily see it if you used ffjson.FastMarshal instead of ffjson.Marshal.
Anyway, the fixed Encode functions looks as follows:
func Encode(item interface{}, out io.Writer) {
// Encode
buf, err := ffjson.MarshalFast(item) // <--- passing the interface as is
if err != nil {
log.Fatalf("Error: %+v\n", err)
}
// Write the buffer
_, _ = out.Write(buf)
// We are now no longer need the buffer so we pool it.
ffjson.Pool(buf)
}
And the results I get are:
$ go run main.go
2016/02/02 09:47:04 GOMAXPROCS: 1
2016/02/02 09:47:11 encoding/json Marshal Function: 1000000 1459 ns/op
2016/02/02 09:47:11 encoding/json Encoder Function: 1000000 1453 ns/op
2016/02/02 09:47:11 ffjson Marshal Function: 2000000 714 ns/op
2016/02/02 09:47:11 ffjson Encoder Function: 2000000 662 ns/op
I'm also observing better performance on newer versions of Go (tested on go1.6.2 darwin/amd64):
This is a simple testing loop result on ffjson:
Interface: 1000000 1237 ns/op
Struct: 2000000 930 ns/op
This is with 1.6 built-in encoding/json:
Interface: 1000000 1193 ns/op
Struct: 2000000 899 ns/op
Maybe this is something that happened over time from many tweaks Google themselves have done to the JSON-decoder? The differences aren't dramatic, but if the 2-3x claim was true 2 years ago I can imagine that maybe back then it was valid, but over time Go's core code has caught up.