JSON Serialization: Change Libraries (prebid#3225)

account/account.go

@@ -2,7 +2,6 @@ package account
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 
 	"github.com/prebid/go-gdpr/consentconstants"
@@ -13,6 +12,7 @@ import (
 	"github.com/prebid/prebid-server/openrtb_ext"
 	"github.com/prebid/prebid-server/stored_requests"
 	"github.com/prebid/prebid-server/util/iputil"
+	"github.com/prebid/prebid-server/util/jsonutil"
 )
 
 // GetAccount looks up the config.Account object referenced by the given accountID, with access rules applied
@@ -44,17 +44,12 @@ func GetAccount(ctx context.Context, cfg *config.Configuration, fetcher stored_r
 	} else {
 		// accountID resolved to a valid account, merge with AccountDefaults for a complete config
 		account = &config.Account{}
-		err := json.Unmarshal(accountJSON, account)
-		if _, ok := err.(*json.UnmarshalTypeError); ok {
+		if err := jsonutil.UnmarshalValid(accountJSON, account); err != nil {
 			return nil, []error{&errortypes.MalformedAcct{
 				Message: fmt.Sprintf("The prebid-server account config for account id \"%s\" is malformed. Please reach out to the prebid server host.", accountID),
 			}}
 		}
 
-		if err != nil {
-			errs = append(errs, err)
-			return nil, errs
-		}
 		// Fill in ID if needed, so it can be left out of account definition
 		if len(account.ID) == 0 {
 			account.ID = accountID

adservertargeting/adservertargeting_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/prebid/openrtb/v19/openrtb2"
 	"github.com/prebid/openrtb/v19/openrtb3"
 	"github.com/prebid/prebid-server/openrtb_ext"
+	"github.com/prebid/prebid-server/util/jsonutil"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -42,7 +43,7 @@ func TestExtractAdServerTargeting(t *testing.T) {
 	p := "https://www.test-url.com?ampkey=testAmpKey&data-override-height=400"
 	u, _ := url.Parse(p)
 	params := u.Query()
-	reqBytes, err := json.Marshal(r)
+	reqBytes, err := jsonutil.Marshal(r)
 	assert.NoError(t, err, "unexpected req marshal error")
 
 	res, warnings := collect(rw, reqBytes, params)
@@ -248,7 +249,7 @@ func TestProcessAdServerTargetingFull(t *testing.T) {
 
 	bidResponseExt := &openrtb_ext.ExtBidResponse{Warnings: make(map[openrtb_ext.BidderName][]openrtb_ext.ExtBidderMessage)}
 
-	reqBytes, err := json.Marshal(r)
+	reqBytes, err := jsonutil.Marshal(r)
 	assert.NoError(t, err, "unexpected req marshal error")
 	targetingKeyLen := 0
 	resResp := Apply(rw, reqBytes, resp, params, bidResponseExt, &targetingKeyLen)
@@ -331,7 +332,7 @@ func TestProcessAdServerTargetingWarnings(t *testing.T) {
 
 	bidResponseExt := &openrtb_ext.ExtBidResponse{Warnings: make(map[openrtb_ext.BidderName][]openrtb_ext.ExtBidderMessage)}
 
-	reqBytes, err := json.Marshal(r)
+	reqBytes, err := jsonutil.Marshal(r)
 	assert.NoError(t, err, "unexpected req marshal error")
 	resResp := Apply(rw, reqBytes, resp, params, bidResponseExt, nil)
 	assert.Len(t, resResp.SeatBid, 2, "Incorrect response: seat bid number")

adservertargeting/reqcache.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/buger/jsonparser"
 	"github.com/prebid/openrtb/v19/openrtb2"
+	"github.com/prebid/prebid-server/util/jsonutil"
 )
 
 type requestCache struct {
@@ -24,7 +25,7 @@ func (reqImpCache *requestCache) GetImpsData() ([]json.RawMessage, error) {
 		}
 		var impsData []json.RawMessage
 
-		err = json.Unmarshal(imps, &impsData)
+		err = jsonutil.Unmarshal(imps, &impsData)
 		if err != nil {
 			return nil, err
 		}
@@ -48,7 +49,7 @@ func (bidsCache *bidsCache) GetBid(bidderName, bidId string, bid openrtb2.Bid) (
 	}
 	_, bidExists := bidsCache.bids[bidderName][bidId]
 	if !bidExists {
-		bidBytes, err := json.Marshal(bid)
+		bidBytes, err := jsonutil.Marshal(bid)
 		if err != nil {
 			return nil, err
 		}

adservertargeting/respdataprocessor.go

@@ -8,6 +8,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prebid/openrtb/v19/openrtb2"
 	"github.com/prebid/prebid-server/openrtb_ext"
+	"github.com/prebid/prebid-server/util/jsonutil"
 	jsonpatch "gopkg.in/evanphx/json-patch.v4"
 )
 
@@ -79,7 +80,7 @@ func buildBidExt(targetingData map[string]string,
 			Targeting: targetingDataTruncated,
 		},
 	}
-	bidExtTargeting, err := json.Marshal(bidExtTargetingData)
+	bidExtTargeting, err := jsonutil.Marshal(bidExtTargetingData)
 	if err != nil {
 		warnings = append(warnings, createWarning(err.Error()))
 		return nil

analytics/filesystem/file_module.go

@@ -2,12 +2,12 @@ package filesystem
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 
 	"github.com/chasex/glog"
 	"github.com/prebid/openrtb/v19/openrtb2"
 	"github.com/prebid/prebid-server/analytics"
+	"github.com/prebid/prebid-server/util/jsonutil"
 )
 
 type RequestType string
@@ -120,7 +120,7 @@ func jsonifyAuctionObject(ao *analytics.AuctionObject) string {
 		}
 	}
 
-	b, err := json.Marshal(&struct {
+	b, err := jsonutil.Marshal(&struct {
 		Type RequestType `json:"type"`
 		*logAuction
 	}{
@@ -153,7 +153,7 @@ func jsonifyVideoObject(vo *analytics.VideoObject) string {
 		}
 	}
 
-	b, err := json.Marshal(&struct {
+	b, err := jsonutil.Marshal(&struct {
 		Type RequestType `json:"type"`
 		*logVideo
 	}{
@@ -178,7 +178,7 @@ func jsonifyCookieSync(cso *analytics.CookieSyncObject) string {
 		}
 	}
 
-	b, err := json.Marshal(&struct {
+	b, err := jsonutil.Marshal(&struct {
 		Type RequestType `json:"type"`
 		*logUserSync
 	}{
@@ -205,7 +205,7 @@ func jsonifySetUIDObject(so *analytics.SetUIDObject) string {
 		}
 	}
 
-	b, err := json.Marshal(&struct {
+	b, err := jsonutil.Marshal(&struct {
 		Type RequestType `json:"type"`
 		*logSetUID
 	}{
@@ -239,7 +239,7 @@ func jsonifyAmpObject(ao *analytics.AmpObject) string {
 		}
 	}
 
-	b, err := json.Marshal(&struct {
+	b, err := jsonutil.Marshal(&struct {
 		Type RequestType `json:"type"`
 		*logAMP
 	}{
@@ -263,7 +263,7 @@ func jsonifyNotificationEventObject(ne *analytics.NotificationEvent) string {
 		}
 	}
 
-	b, err := json.Marshal(&struct {
+	b, err := jsonutil.Marshal(&struct {
 		Type RequestType `json:"type"`
 		*logNotificationEvent
 	}{

analytics/pubstack/helpers/json.go

@@ -1,11 +1,11 @@
 package helpers
 
 import (
-	"encoding/json"
 	"fmt"
 
 	"github.com/prebid/openrtb/v19/openrtb2"
 	"github.com/prebid/prebid-server/analytics"
+	"github.com/prebid/prebid-server/util/jsonutil"
 )
 
 func JsonifyAuctionObject(ao *analytics.AuctionObject, scope string) ([]byte, error) {
@@ -26,7 +26,7 @@ func JsonifyAuctionObject(ao *analytics.AuctionObject, scope string) ([]byte, er
 		}
 	}
 
-	b, err := json.Marshal(&struct {
+	b, err := jsonutil.Marshal(&struct {
 		Scope string `json:"scope"`
 		*logAuction
 	}{
@@ -59,7 +59,7 @@ func JsonifyVideoObject(vo *analytics.VideoObject, scope string) ([]byte, error)
 		}
 	}
 
-	b, err := json.Marshal(&struct {
+	b, err := jsonutil.Marshal(&struct {
 		Scope string `json:"scope"`
 		*logVideo
 	}{
@@ -84,7 +84,7 @@ func JsonifyCookieSync(cso *analytics.CookieSyncObject, scope string) ([]byte, e
 		}
 	}
 
-	b, err := json.Marshal(&struct {
+	b, err := jsonutil.Marshal(&struct {
 		Scope string `json:"scope"`
 		*logUserSync
 	}{
@@ -111,7 +111,7 @@ func JsonifySetUIDObject(so *analytics.SetUIDObject, scope string) ([]byte, erro
 		}
 	}
 
-	b, err := json.Marshal(&struct {
+	b, err := jsonutil.Marshal(&struct {
 		Scope string `json:"scope"`
 		*logSetUID
 	}{
@@ -145,7 +145,7 @@ func JsonifyAmpObject(ao *analytics.AmpObject, scope string) ([]byte, error) {
 		}
 	}
 
-	b, err := json.Marshal(&struct {
+	b, err := jsonutil.Marshal(&struct {
 		Scope string `json:"scope"`
 		*logAMP
 	}{

config/config.go

@@ -14,6 +14,7 @@ import (
 	"github.com/prebid/openrtb/v19/openrtb2"
 	"github.com/prebid/prebid-server/errortypes"
 	"github.com/prebid/prebid-server/openrtb_ext"
+	"github.com/prebid/prebid-server/util/jsonutil"
 	"github.com/spf13/viper"
 )
 
@@ -775,7 +776,7 @@ func New(v *viper.Viper, bidderInfos BidderInfos, normalizeBidderName func(strin
 // MarshalAccountDefaults compiles AccountDefaults into the JSON format used for merge patch
 func (cfg *Configuration) MarshalAccountDefaults() error {
 	var err error
-	if cfg.accountDefaultsJSON, err = json.Marshal(cfg.AccountDefaults); err != nil {
+	if cfg.accountDefaultsJSON, err = jsonutil.Marshal(cfg.AccountDefaults); err != nil {
 		glog.Warningf("converting %+v to json: %v", cfg.AccountDefaults, err)
 	}
 	return err

currency/rate_converter.go

@@ -1,7 +1,6 @@
 package currency
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -10,6 +9,7 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/prebid/prebid-server/errortypes"
+	"github.com/prebid/prebid-server/util/jsonutil"
 	"github.com/prebid/prebid-server/util/timeutil"
 )
 
@@ -66,7 +66,7 @@ func (rc *RateConverter) fetch() (*Rates, error) {
 	}
 
 	updatedRates := &Rates{}
-	err = json.Unmarshal(bytesJSON, updatedRates)
+	err = jsonutil.UnmarshalValid(bytesJSON, updatedRates)
 	if err != nil {
 		return nil, err
 	}

currency/rates_test.go

@@ -1,11 +1,12 @@
 package currency
 
 import (
-	"encoding/json"
 	"errors"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	"github.com/prebid/prebid-server/util/jsonutil"
 )
 
 func TestUnMarshallRates(t *testing.T) {
@@ -22,7 +23,7 @@ func TestUnMarshallRates(t *testing.T) {
 			ratesJSON:     `malformed`,
 			expectedRates: Rates{},
 			expectsError:  true,
-			expectedError: errors.New("invalid character 'm' looking for beginning of value"),
+			expectedError: errors.New("expect { or n, but found m"),
 		},
 		{
 			desc: "Valid JSON field defining valid conversion object. Expect no error",
@@ -50,33 +51,39 @@ func TestUnMarshallRates(t *testing.T) {
 			expectedError: nil,
 		},
 		{
-			desc: "Valid JSON field defines a conversions map with repeated entries, expect error",
+			desc: "Valid JSON field defines a conversions map with repeated entries, last one wins",
 			ratesJSON: `{
 				"conversions":{
 					"USD":{
 						"GBP":0.7662523901,
 						"MXN":20.00
 					},
 					"USD":{
-						"GBP":0.7662523901
-					},
+						"GBP":0.4815162342
+					}
 				}
 			}`,
-			expectedRates: Rates{},
-			expectsError:  true,
-			expectedError: errors.New("invalid character '}' looking for beginning of object key string"),
+			expectedRates: Rates{
+				Conversions: map[string]map[string]float64{
+					"USD": {
+						"GBP": 0.4815162342,
+					},
+				},
+			},
+			expectsError:  false,
+			expectedError: nil,
 		},
 	}
 
 	for _, tc := range testCases {
 		// Execute:
 		updatedRates := Rates{}
-		err := json.Unmarshal([]byte(tc.ratesJSON), &updatedRates)
+		err := jsonutil.UnmarshalValid([]byte(tc.ratesJSON), &updatedRates)
 
 		// Verify:
 		assert.Equal(t, err != nil, tc.expectsError, tc.desc)
 		if tc.expectsError {
-			assert.Equal(t, err.Error(), tc.expectedError.Error(), tc.desc)
+			assert.Equal(t, tc.expectedError.Error(), err.Error(), tc.desc)
 		}
 		assert.Equal(t, tc.expectedRates, updatedRates, tc.desc)
 	}

endpoints/cookie_sync.go

@@ -27,6 +27,7 @@ import (
 	gppPrivacy "github.com/prebid/prebid-server/privacy/gpp"
 	"github.com/prebid/prebid-server/stored_requests"
 	"github.com/prebid/prebid-server/usersync"
+	"github.com/prebid/prebid-server/util/jsonutil"
 	stringutil "github.com/prebid/prebid-server/util/stringutil"
 )
 
@@ -119,7 +120,7 @@ func (c *cookieSyncEndpoint) parseRequest(r *http.Request) (usersync.Request, ma
 	}
 
 	request := cookieSyncRequest{}
-	if err := json.Unmarshal(body, &request); err != nil {
+	if err := jsonutil.UnmarshalValid(body, &request); err != nil {
 		return usersync.Request{}, macros.UserSyncPrivacy{}, fmt.Errorf("JSON parsing failed: %s", err.Error())
 	}
 

endpoints/cookie_sync_test.go

@@ -190,14 +190,14 @@ func TestCookieSyncHandle(t *testing.T) {
 				SyncersChosen:    []usersync.SyncerChoice{{Bidder: "a", Syncer: &syncer}},
 			},
 			expectedStatusCode: 400,
-			expectedBody:       `JSON parsing failed: invalid character 'm' looking for beginning of value` + "\n",
+			expectedBody:       `JSON parsing failed: expect { or n, but found m` + "\n",
 			setMetricsExpectations: func(m *metrics.MetricsEngineMock) {
 				m.On("RecordCookieSync", metrics.CookieSyncBadRequest).Once()
 			},
 			setAnalyticsExpectations: func(a *MockAnalyticsRunner) {
 				expected := analytics.CookieSyncObject{
 					Status:       400,
-					Errors:       []error{errors.New("JSON parsing failed: invalid character 'm' looking for beginning of value")},
+					Errors:       []error{errors.New("JSON parsing failed: expect { or n, but found m")},
 					BidderStatus: []*analytics.CookieSyncBidder{},
 				}
 				a.On("LogCookieSyncObject", &expected).Once()
@@ -805,7 +805,7 @@ func TestCookieSyncParseRequest(t *testing.T) {
 			givenBody:        strings.NewReader(`malformed`),
 			givenGDPRConfig:  config.GDPR{Enabled: true, DefaultValue: "0"},
 			givenCCPAEnabled: true,
-			expectedError:    "JSON parsing failed: invalid character 'm' looking for beginning of value",
+			expectedError:    "JSON parsing failed: expect { or n, but found m",
 		},
 		{
 			description:      "Invalid Type Filter",