diff --git a/jsonapi.go b/jsonapi.go index 73b0a12..14015ed 100644 --- a/jsonapi.go +++ b/jsonapi.go @@ -15,19 +15,42 @@ type resourceObject struct { Relationships map[string]*document `json:"relationships,omitempty"` Meta any `json:"meta,omitempty"` Links *Link `json:"links,omitempty"` + + // useNumber keeps the attributes as received in rawAttributes, so UnmarshalUseNumber can decode them + // without a float64 round-trip; otherwise they are decoded into Attributes as before. + useNumber bool + rawAttributes rawJSON +} + +// rawJSON keeps a reference to the bytes it is decoded from instead of copying them like json.RawMessage. +// It is only read while Unmarshal runs, so the input is still valid. +type rawJSON []byte + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (r *rawJSON) UnmarshalJSON(data []byte) error { + *r = data + return nil } // UnmarshalJSON implements the json.Unmarshaler interface. func (ro *resourceObject) UnmarshalJSON(data []byte) error { type alias resourceObject - auxRaw := &struct { + type aux struct { Rels map[string]json.RawMessage `json:"relationships,omitempty"` *alias - }{ - alias: (*alias)(ro), } - if err := json.Unmarshal(data, &auxRaw); err != nil { + auxRaw := &aux{alias: (*alias)(ro)} + if ro.useNumber { + withRaw := &struct { + Attrs rawJSON `json:"attributes,omitempty"` + *aux + }{aux: auxRaw} + if err := json.Unmarshal(data, withRaw); err != nil { + return err + } + ro.rawAttributes = withRaw.Attrs + } else if err := json.Unmarshal(data, auxRaw); err != nil { return err } @@ -35,7 +58,7 @@ func (ro *resourceObject) UnmarshalJSON(data []byte) error { for name, raw := range auxRaw.Rels { // mark the created sub-documents as relationships so that the document Unmarshaler // can handle their different member requirements - d := document{isRelationship: true} + d := document{isRelationship: true, useNumber: ro.useNumber} if err := json.Unmarshal(raw, &d); err != nil { return err } @@ -143,6 +166,9 @@ type document struct { // isRelationship marks a document as a relationship sub-document (within primary data) isRelationship bool `json:"-"` + // useNumber is passed to the document's resource objects (see resourceObject.useNumber) + useNumber bool `json:"-"` + // Meta is Meta Information as defined by https://jsonapi.org/format/1.0/#document-meta. Meta any `json:"meta,omitempty"` @@ -199,14 +225,27 @@ func (d *document) MarshalJSON() ([]byte, error) { // UnmarshalJSON implements the json.Unmarshaler interface. func (d *document) UnmarshalJSON(data []byte) error { type alias document - - auxRaw := &struct { + type aux struct { Data json.RawMessage `json:"data,omitempty"` *alias - }{ - alias: (*alias)(d), } - if err := json.Unmarshal(data, &auxRaw); err != nil { + + auxRaw := &aux{alias: (*alias)(d)} + if d.useNumber { + // decode included resources one by one so they carry useNumber + withRaw := &struct { + Included []rawJSON `json:"included,omitempty"` + *aux + }{aux: auxRaw} + if err := json.Unmarshal(data, withRaw); err != nil { + return err + } + included, err := d.unmarshalResourceObjects(withRaw.Included) + if err != nil { + return err + } + d.Included = included + } else if err := json.Unmarshal(data, auxRaw); err != nil { return err } @@ -231,11 +270,41 @@ func (d *document) UnmarshalJSON(data []byte) error { if auxRaw.Data[0] == '[' { d.hasMany = true - return json.Unmarshal(auxRaw.Data, &auxRaw.DataMany) + if !d.useNumber { + return json.Unmarshal(auxRaw.Data, &auxRaw.DataMany) + } + var raws []rawJSON + if err := json.Unmarshal(auxRaw.Data, &raws); err != nil { + return err + } + many, err := d.unmarshalResourceObjects(raws) + if err != nil { + return err + } + d.DataMany = many + return nil + } + if d.useNumber { + d.DataOne = &resourceObject{useNumber: true} } return json.Unmarshal(auxRaw.Data, &auxRaw.DataOne) } +// unmarshalResourceObjects decodes each raw resource object with d.useNumber. +func (d *document) unmarshalResourceObjects(raws []rawJSON) ([]*resourceObject, error) { + if raws == nil { + return nil, nil + } + ros := make([]*resourceObject, len(raws)) + for i, raw := range raws { + ros[i] = &resourceObject{useNumber: d.useNumber} + if err := json.Unmarshal(raw, ros[i]); err != nil { + return nil, err + } + } + return ros, nil +} + // isEmpty returns true if there is no primary data in the given document (i.e. null or []). func (d *document) isEmpty() bool { return len(d.DataMany) == 0 && d.DataOne == nil diff --git a/unmarshal.go b/unmarshal.go index 92eafb3..8979e4c 100644 --- a/unmarshal.go +++ b/unmarshal.go @@ -1,6 +1,7 @@ package jsonapi import ( + "bytes" "encoding" "encoding/json" "errors" @@ -13,6 +14,7 @@ type Unmarshaler struct { unmarshalMeta bool unmarshalLinks bool checkUniqueness bool + useNumber bool meta any links *Link memberNameValidationMode MemberNameValidationMode @@ -44,6 +46,15 @@ func UnmarshalCheckUniqueness() UnmarshalOption { } } +// UnmarshalUseNumber decodes attributes directly with json.Number, so integers above 2^53 keep every digit +// in typed fields and interface{} values (e.g. map[string]any) hold json.Number instead of float64. +// Non-integer literals such as 100.0 no longer decode into integer fields. +func UnmarshalUseNumber() UnmarshalOption { + return func(m *Unmarshaler) { + m.useNumber = true + } +} + // UnmarshalSetNameValidation enables a given level of document member name validation. func UnmarshalSetNameValidation(mode MemberNameValidationMode) UnmarshalOption { return func(m *Unmarshaler) { @@ -82,7 +93,7 @@ func Unmarshal(data []byte, v any, opts ...UnmarshalOption) (err error) { return } - var d document + d := document{useNumber: m.useNumber} if err = json.Unmarshal(data, &d); err != nil { return } @@ -339,6 +350,15 @@ func (ro *resourceObject) unmarshalFields(v any, rv reflect.Value, rt reflect.Ty } func (ro *resourceObject) unmarshalAttributes(v any) error { + if ro.useNumber { + if len(ro.rawAttributes) == 0 || bytes.Equal(ro.rawAttributes, []byte("null")) { + return nil + } + d := json.NewDecoder(bytes.NewReader(ro.rawAttributes)) + d.UseNumber() + return d.Decode(v) + } + if len(ro.Attributes) == 0 { return nil } diff --git a/unmarshal_test.go b/unmarshal_test.go index 91b1ad0..0aaea2e 100644 --- a/unmarshal_test.go +++ b/unmarshal_test.go @@ -4,6 +4,8 @@ import ( "encoding/json" "fmt" "reflect" + "strconv" + "strings" "testing" "github.com/DataDog/jsonapi/internal/is" @@ -764,6 +766,120 @@ func TestUnmarshalLinks(t *testing.T) { } } +func TestUnmarshalNumbers(t *testing.T) { + t.Parallel() + + type nested struct { + Max int64 `json:"max"` + } + type numbers struct { + ID string `jsonapi:"primary,numbers"` + Int int64 `jsonapi:"attribute" json:"int,omitempty"` + Uint uint64 `jsonapi:"attribute" json:"uint,omitempty"` + Float float64 `jsonapi:"attribute" json:"float,omitempty"` + Nested *nested `jsonapi:"attribute" json:"nested,omitempty"` + List []int64 `jsonapi:"attribute" json:"list,omitempty"` + Values map[string]any `jsonapi:"attribute" json:"values,omitempty"` + } + type item struct { + ID string `jsonapi:"primary,items"` + Int int64 `jsonapi:"attribute" json:"int,omitempty"` + Related *item `jsonapi:"relationship" json:"related,omitempty"` + } + + unmarshalNumbers := func(attributes string, opts ...UnmarshalOption) (any, error) { + var n numbers + err := Unmarshal([]byte(`{"data":{"type":"numbers","id":"1","attributes":`+attributes+`}}`), &n, opts...) + return &n, err + } + itemsWithIncludedBody := `{"data":[{"type":"items","id":"1","attributes":{"int":9007199254740993},"relationships":{"related":{"data":{"type":"items","id":"2"}}}}],` + + `"included":[{"type":"items","id":"2","attributes":{"int":9007199254740995}}]}` + itemWithRelationshipAttributesBody := `{"data":{"type":"items","id":"1","relationships":{"related":{"data":{"type":"items","id":"2","attributes":{"int":9007199254740993}}}}}}` + + tests := []struct { + description string + do func() (any, error) + expect any + expectError error + }{ + { + description: "int64 above 2^53 rounds", + do: func() (any, error) { return unmarshalNumbers(`{"int":9007199254740993}`) }, + expect: &numbers{ID: "1", Int: 9007199254740992}, + }, { + description: "non-integer literals with integer values decode into int64", + do: func() (any, error) { return unmarshalNumbers(`{"int":100.0,"list":[1e2]}`) }, + expect: &numbers{ID: "1", Int: 100, List: []int64{100}}, + }, { + description: "map[string]any values are float64", + do: func() (any, error) { return unmarshalNumbers(`{"values":{"int":9007199254740993,"float":1.5}}`) }, + expect: &numbers{ID: "1", Values: map[string]any{"int": float64(9007199254740992), "float": 1.5}}, + }, { + description: "int64 above 2^53 with UnmarshalUseNumber", + do: func() (any, error) { return unmarshalNumbers(`{"int":9007199254740993}`, UnmarshalUseNumber()) }, + expect: &numbers{ID: "1", Int: 9007199254740993}, + }, { + description: "uint64 max with UnmarshalUseNumber", + do: func() (any, error) { return unmarshalNumbers(`{"uint":18446744073709551615}`, UnmarshalUseNumber()) }, + expect: &numbers{ID: "1", Uint: 18446744073709551615}, + }, { + description: "int64 min and max in nested struct and slice with UnmarshalUseNumber", + do: func() (any, error) { + return unmarshalNumbers(`{"nested":{"max":9223372036854775807},"list":[-9223372036854775808,9007199254740993]}`, UnmarshalUseNumber()) + }, + expect: &numbers{ID: "1", Nested: &nested{Max: 9223372036854775807}, List: []int64{-9223372036854775808, 9007199254740993}}, + }, { + description: "integer above 2^53 into float64 rounds with UnmarshalUseNumber", + do: func() (any, error) { return unmarshalNumbers(`{"float":9007199254740993}`, UnmarshalUseNumber()) }, + expect: &numbers{ID: "1", Float: 9007199254740992}, + }, { + description: "map[string]any values are json.Number with UnmarshalUseNumber", + do: func() (any, error) { + return unmarshalNumbers(`{"values":{"int":9007199254740993,"float":1.5}}`, UnmarshalUseNumber()) + }, + expect: &numbers{ID: "1", Values: map[string]any{"int": json.Number("9007199254740993"), "float": json.Number("1.5")}}, + }, { + description: "non-integer literal into int64 errors with UnmarshalUseNumber", + do: func() (any, error) { return unmarshalNumbers(`{"int":100.0}`, UnmarshalUseNumber()) }, + expect: &numbers{ID: "1"}, + expectError: &json.UnmarshalTypeError{Value: "number 100.0", Type: reflect.TypeOf(int64(0)), Struct: "numbers", Field: "int"}, + }, { + description: "[]*item with included resource with UnmarshalUseNumber", + do: func() (any, error) { + var items []*item + err := Unmarshal([]byte(itemsWithIncludedBody), &items, UnmarshalUseNumber()) + return items, err + }, + expect: []*item{{ID: "1", Int: 9007199254740993, Related: &item{ID: "2", Int: 9007199254740995}}}, + }, { + description: "*item with attributes in relationship data with UnmarshalUseNumber", + do: func() (any, error) { + var i item + err := Unmarshal([]byte(itemWithRelationshipAttributesBody), &i, UnmarshalUseNumber()) + return &i, err + }, + expect: &item{ID: "1", Related: &item{ID: "2", Int: 9007199254740993}}, + }, + } + + for i, tc := range tests { + tc := tc + t.Run(fmt.Sprintf("%02d", i), func(t *testing.T) { + t.Parallel() + t.Log(tc.description) + + actual, err := tc.do() + if tc.expectError != nil { + is.EqualError(t, tc.expectError, err) + is.Equal(t, tc.expect, actual) + return + } + is.MustNoError(t, err) + is.Equal(t, tc.expect, actual) + }) + } +} + // TestUnmarshalMemberNameValidation collects tests which verify that invalid member names are // caught during unmarshaling, no matter where they're placed. This test does not exhaustively test // every possible invalid name. @@ -920,43 +1036,42 @@ func TestUnmarshalMemberNameValidation(t *testing.T) { } } +var benchmarkSink any + func BenchmarkUnmarshal(b *testing.B) { benchmarks := []struct { - name string - data string - target any - opts []UnmarshalOption + name string + data string + newTarget func() any + opts func() []UnmarshalOption }{ { - name: "ArticleSimple", - data: articleABody, - target: Article{}, - opts: nil, - }, { - name: "ArticleSimpleWithToplevelMeta", - data: articleAToplevelMetaBody, - target: Article{}, - opts: []UnmarshalOption{UnmarshalMeta(map[string]any{"foo": "bar"})}, - }, { - name: "ArticleComplex", - data: articleRelatedCommentsNestedWithIncludeBody, - target: ArticleRelated{}, - opts: nil, - }, { - name: "ArticleComplexDisableNameValidation", - data: articleRelatedCommentsNestedWithIncludeBody, - target: ArticleRelated{}, - opts: []UnmarshalOption{UnmarshalSetNameValidation(DisableValidation)}, - }, { - name: "ArticlesComplex", - data: articlesRelatedComplexBody, - target: []*ArticleRelated{}, - opts: nil, - }, { - name: "ArticlesComplexDisableNameValidation", - data: articlesRelatedComplexBody, - target: []*ArticleRelated{}, - opts: []UnmarshalOption{UnmarshalSetNameValidation(DisableValidation)}, + name: "ArticleSimple", + data: articleABody, + newTarget: func() any { return new(Article) }, + }, { + name: "ArticleSimpleWithToplevelMeta", + data: articleAToplevelMetaBody, + newTarget: func() any { return new(Article) }, + opts: func() []UnmarshalOption { return []UnmarshalOption{UnmarshalMeta(&map[string]any{})} }, + }, { + name: "ArticleComplex", + data: articleRelatedCommentsNestedWithIncludeBody, + newTarget: func() any { return new(ArticleRelated) }, + }, { + name: "ArticleComplexDisableNameValidation", + data: articleRelatedCommentsNestedWithIncludeBody, + newTarget: func() any { return new(ArticleRelated) }, + opts: func() []UnmarshalOption { return []UnmarshalOption{UnmarshalSetNameValidation(DisableValidation)} }, + }, { + name: "ArticlesComplex", + data: articlesRelatedComplexBody, + newTarget: func() any { return new([]*ArticleRelated) }, + }, { + name: "ArticlesComplexDisableNameValidation", + data: articlesRelatedComplexBody, + newTarget: func() any { return new([]*ArticleRelated) }, + opts: func() []UnmarshalOption { return []UnmarshalOption{UnmarshalSetNameValidation(DisableValidation)} }, }, } @@ -966,8 +1081,75 @@ func BenchmarkUnmarshal(b *testing.B) { b.Run(bm.name, func(b *testing.B) { b.ReportAllocs() for n := 0; n < b.N; n++ { - _ = Unmarshal(data, &bm.target, bm.opts...) + var opts []UnmarshalOption + if bm.opts != nil { + opts = bm.opts() + } + target := bm.newTarget() + if err := Unmarshal(data, target, opts...); err != nil { + b.Fatal(err) + } + benchmarkSink = target } }) } } + +func BenchmarkUnmarshalNumbers(b *testing.B) { + type item struct { + ID string `jsonapi:"primary,items"` + Blob string `jsonapi:"attribute" json:"blob,omitempty"` + Numbers []int64 `jsonapi:"attribute" json:"numbers,omitempty"` + Related *item `jsonapi:"relationship" json:"related,omitempty"` + } + + blob, _ := json.Marshal(strings.Repeat("a", 4096)) + numbers := make([]string, 1000) + for i := range numbers { + numbers[i] = strconv.Itoa(i) + } + shared := make([]string, 32) + for i := range shared { + shared[i] = `{"type":"items","id":"` + strconv.Itoa(i) + `","relationships":{"related":{"data":{"type":"items","id":"shared"}}}}` + } + + benchmarks := []struct { + name string + data []byte + many bool + }{ + { + name: "String4KiB", + data: []byte(`{"data":{"type":"items","id":"1","attributes":{"blob":` + string(blob) + `}}}`), + }, { + name: "Numbers1000", + data: []byte(`{"data":{"type":"items","id":"1","attributes":{"numbers":[` + strings.Join(numbers, ",") + `]}}}`), + }, { + name: "SharedIncluded32", + data: []byte(`{"data":[` + strings.Join(shared, ",") + `],"included":[{"type":"items","id":"shared","attributes":{"blob":` + string(blob) + `}}]}`), + many: true, + }, + } + + for _, mode := range []struct { + name string + opts []UnmarshalOption + }{{"Default", nil}, {"UseNumber", []UnmarshalOption{UnmarshalUseNumber()}}} { + for _, bm := range benchmarks { + mode, bm := mode, bm + b.Run(bm.name+"/"+mode.name, func(b *testing.B) { + b.ReportAllocs() + for n := 0; n < b.N; n++ { + var target any = new(item) + if bm.many { + target = new([]*item) + } + if err := Unmarshal(bm.data, target, mode.opts...); err != nil { + b.Fatal(err) + } + benchmarkSink = target + } + }) + } + } +}