JSON Serialization: Change Libraries

account/account.go

@@ -2,7 +2,6 @@ package account
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 
 	"github.com/prebid/go-gdpr/consentconstants"
@@ -13,6 +12,7 @@ import (
 	"github.com/prebid/prebid-server/openrtb_ext"
 	"github.com/prebid/prebid-server/stored_requests"
 	"github.com/prebid/prebid-server/util/iputil"
+	"github.com/prebid/prebid-server/util/jsonutil"
 )
 
 // GetAccount looks up the config.Account object referenced by the given accountID, with access rules applied
@@ -44,17 +44,12 @@ func GetAccount(ctx context.Context, cfg *config.Configuration, fetcher stored_r
 	} else {
 		// accountID resolved to a valid account, merge with AccountDefaults for a complete config
 		account = &config.Account{}
-		err := json.Unmarshal(accountJSON, account)
-		if _, ok := err.(*json.UnmarshalTypeError); ok {
+		if err := jsonutil.UnmarshalValid(accountJSON, account); err != nil {
 			return nil, []error{&errortypes.MalformedAcct{
 				Message: fmt.Sprintf("The prebid-server account config for account id \"%s\" is malformed. Please reach out to the prebid server host.", accountID),
 			}}
 		}
 
-		if err != nil {
-			errs = append(errs, err)
-			return nil, errs
-		}
 		// Fill in ID if needed, so it can be left out of account definition
 		if len(account.ID) == 0 {
 			account.ID = accountID

adservertargeting/reqcache.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/buger/jsonparser"
 	"github.com/prebid/openrtb/v19/openrtb2"
+	"github.com/prebid/prebid-server/util/jsonutil"
 )
 
 type requestCache struct {
@@ -24,7 +25,7 @@ func (reqImpCache *requestCache) GetImpsData() ([]json.RawMessage, error) {
 		}
 		var impsData []json.RawMessage
 
-		err = json.Unmarshal(imps, &impsData)
+		err = jsonutil.Unmarshal(imps, &impsData)
 		if err != nil {
 			return nil, err
 		}
@@ -48,7 +49,7 @@ func (bidsCache *bidsCache) GetBid(bidderName, bidId string, bid openrtb2.Bid) (
 	}
 	_, bidExists := bidsCache.bids[bidderName][bidId]
 	if !bidExists {
-		bidBytes, err := json.Marshal(bid)
+		bidBytes, err := jsonutil.Marshal(bid)
 		if err != nil {
 			return nil, err
 		}

adservertargeting/respdataprocessor.go

@@ -8,6 +8,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prebid/openrtb/v19/openrtb2"
 	"github.com/prebid/prebid-server/openrtb_ext"
+	"github.com/prebid/prebid-server/util/jsonutil"
 	jsonpatch "gopkg.in/evanphx/json-patch.v4"
 )
 
@@ -79,7 +80,7 @@ func buildBidExt(targetingData map[string]string,
 			Targeting: targetingDataTruncated,
 		},
 	}
-	bidExtTargeting, err := json.Marshal(bidExtTargetingData)
+	bidExtTargeting, err := jsonutil.Marshal(bidExtTargetingData)
 	if err != nil {
 		warnings = append(warnings, createWarning(err.Error()))
 		return nil

currency/rate_converter.go

@@ -1,7 +1,6 @@
 package currency
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -10,6 +9,7 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/prebid/prebid-server/errortypes"
+	"github.com/prebid/prebid-server/util/jsonutil"
 	"github.com/prebid/prebid-server/util/timeutil"
 )
 
@@ -66,7 +66,7 @@ func (rc *RateConverter) fetch() (*Rates, error) {
 	}
 
 	updatedRates := &Rates{}
-	err = json.Unmarshal(bytesJSON, updatedRates)
+	err = jsonutil.UnmarshalValid(bytesJSON, updatedRates)
 	if err != nil {
 		return nil, err
 	}

endpoints/openrtb2/amp_auction.go

@@ -37,6 +37,7 @@ import (
 	"github.com/prebid/prebid-server/stored_responses"
 	"github.com/prebid/prebid-server/usersync"
 	"github.com/prebid/prebid-server/util/iputil"
+	"github.com/prebid/prebid-server/util/jsonutil"
 	"github.com/prebid/prebid-server/version"
 )
 
@@ -341,7 +342,7 @@ func sendAmpResponse(
 					// but this is a very unlikely corner case. Doing this so we can catch "hb_cache_id"
 					// and "hb_cache_id_{deal}", which allows for deal support in AMP.
 					bidExt := &openrtb_ext.ExtBid{}
-					err := json.Unmarshal(bid.Ext, bidExt)
+					err := jsonutil.Unmarshal(bid.Ext, bidExt)
 					if err != nil {
 						w.WriteHeader(http.StatusInternalServerError)
 						fmt.Fprintf(w, "Critical error while unpacking AMP targets: %v", err)
@@ -360,7 +361,7 @@ func sendAmpResponse(
 
 	// Extract global targeting
 	var extResponse openrtb_ext.ExtBidResponse
-	eRErr := json.Unmarshal(response.Ext, &extResponse)
+	eRErr := jsonutil.Unmarshal(response.Ext, &extResponse)
 	if eRErr != nil {
 		ao.Errors = append(ao.Errors, fmt.Errorf("AMP response: failed to unpack OpenRTB response.ext, debug info cannot be forwarded: %v", eRErr))
 	}
@@ -409,7 +410,7 @@ func getExtBidResponse(
 	}
 	// Extract any errors
 	var extResponse openrtb_ext.ExtBidResponse
-	eRErr := json.Unmarshal(response.Ext, &extResponse)
+	eRErr := jsonutil.Unmarshal(response.Ext, &extResponse)
 	if eRErr != nil {
 		ao.Errors = append(ao.Errors, fmt.Errorf("AMP response: failed to unpack OpenRTB response.ext, debug info cannot be forwarded: %v", eRErr))
 	}
@@ -524,7 +525,7 @@ func (deps *endpointDeps) loadRequestJSONForAmp(httpRequest *http.Request) (req
 
 	// The fetched config becomes the entire OpenRTB request
 	requestJSON := storedRequests[ampParams.StoredRequestID]
-	if err := json.Unmarshal(requestJSON, req); err != nil {
+	if err := jsonutil.UnmarshalValid(requestJSON, req); err != nil {
 		errs = []error{err}
 		return
 	}
@@ -824,7 +825,7 @@ func setTrace(req *openrtb2.BidRequest, value string) error {
 		return nil
 	}
 
-	ext, err := json.Marshal(map[string]map[string]string{"prebid": {"trace": value}})
+	ext, err := jsonutil.Marshal(map[string]map[string]string{"prebid": {"trace": value}})
 	if err != nil {
 		return err
 	}

endpoints/openrtb2/amp_auction_test.go

@@ -516,7 +516,7 @@ func TestOverrideWithParams(t *testing.T) {
 					Site: &openrtb2.Site{Ext: json.RawMessage(`{"amp":1}`)},
 					User: &openrtb2.User{Ext: json.RawMessage(`malformed`)},
 				},
-				errorMsgs:         []string{"invalid character 'm' looking for beginning of value"},
+				errorMsgs:         []string{"ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|..."},
 				expectFatalErrors: true,
 			},
 		},
@@ -564,7 +564,7 @@ func TestOverrideWithParams(t *testing.T) {
 					User: &openrtb2.User{Ext: json.RawMessage(`{"prebid":{malformed}}`)},
 					Site: &openrtb2.Site{Ext: json.RawMessage(`{"amp":1}`)},
 				},
-				errorMsgs:         []string{"invalid character 'm' looking for beginning of object key string"},
+				errorMsgs:         []string{"ReadObjectCB: expect \" after {, but found m, error found in #10 byte of ...|prebid\":{malformed}}|..., bigger context ...|{\"prebid\":{malformed}}|..."},
 				expectFatalErrors: true,
 			},
 		},
@@ -1131,7 +1131,7 @@ func TestInitAmpTargetingAndCache(t *testing.T) {
 		{
 			name:         "malformed",
 			request:      &openrtb2.BidRequest{Ext: json.RawMessage("malformed")},
-			expectedErrs: []string{"invalid character 'm' looking for beginning of value"},
+			expectedErrs: []string{"ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|..."},
 		},
 		{
 			name:           "nil",
@@ -2245,7 +2245,7 @@ func TestSendAmpResponse_LogsErrors(t *testing.T) {
 		{
 			description: "Error logged when bid.ext unmarshal fails",
 			expectedErrors: []error{
-				errors.New("Critical error while unpacking AMP targets: unexpected end of JSON input"),
+				errors.New("Critical error while unpacking AMP targets: readObjectStart: expect { or n, but found \", error found in #1 byte of ...|\"hb_cache_i|..., bigger context ...|\"hb_cache_id|..."),
 			},
 			expectedStatus: http.StatusInternalServerError,
 			writer:         httptest.NewRecorder(),
@@ -2322,7 +2322,7 @@ func TestSendAmpResponse_LogsErrors(t *testing.T) {
 
 			_, ao = sendAmpResponse(test.writer, test.hookExecutor, &exchange.AuctionResponse{BidResponse: test.response}, &reqWrapper, account, labels, ao, nil)
 
-			assert.Equal(t, ao.Errors, test.expectedErrors, "Invalid errors.")
+			assert.Equal(t, test.expectedErrors, ao.Errors, "Invalid errors.")
 			assert.Equal(t, test.expectedStatus, ao.Status, "Invalid HTTP response status.")
 		})
 	}

endpoints/openrtb2/auction.go

@@ -52,6 +52,7 @@ import (
 	"github.com/prebid/prebid-server/usersync"
 	"github.com/prebid/prebid-server/util/httputil"
 	"github.com/prebid/prebid-server/util/iputil"
+	"github.com/prebid/prebid-server/util/jsonutil"
 	"github.com/prebid/prebid-server/util/uuidutil"
 	"github.com/prebid/prebid-server/version"
 )
@@ -308,11 +309,11 @@ func setSeatNonBidRaw(request *openrtb_ext.RequestWrapper, auctionResponse *exch
 	// by HoldAuction
 	response := auctionResponse.BidResponse
 	respExt := &openrtb_ext.ExtBidResponse{}
-	if err := json.Unmarshal(response.Ext, &respExt); err != nil {
+	if err := jsonutil.Unmarshal(response.Ext, &respExt); err != nil {
 		return err
 	}
 	if setSeatNonBid(respExt, request, auctionResponse) {
-		if respExtJson, err := json.Marshal(respExt); err == nil {
+		if respExtJson, err := jsonutil.Marshal(respExt); err == nil {
 			response.Ext = respExtJson
 			return nil
 		} else {
@@ -445,7 +446,7 @@ func (deps *endpointDeps) parseRequest(httpRequest *http.Request, labels *metric
 	requestJson, rejectErr := hookExecutor.ExecuteEntrypointStage(httpRequest, requestJson)
 	if rejectErr != nil {
 		errs = []error{rejectErr}
-		if err = json.Unmarshal(requestJson, req.BidRequest); err != nil {
+		if err = jsonutil.UnmarshalValid(requestJson, req.BidRequest); err != nil {
 			glog.Errorf("Failed to unmarshal BidRequest during entrypoint rejection: %s", err)
 		}
 		return
@@ -493,7 +494,7 @@ func (deps *endpointDeps) parseRequest(httpRequest *http.Request, labels *metric
 	requestJson, rejectErr = hookExecutor.ExecuteRawAuctionStage(requestJson)
 	if rejectErr != nil {
 		errs = []error{rejectErr}
-		if err = json.Unmarshal(requestJson, req.BidRequest); err != nil {
+		if err = jsonutil.UnmarshalValid(requestJson, req.BidRequest); err != nil {
 			glog.Errorf("Failed to unmarshal BidRequest during raw auction stage rejection: %s", err)
 		}
 		return
@@ -522,7 +523,7 @@ func (deps *endpointDeps) parseRequest(httpRequest *http.Request, labels *metric
 		return nil, nil, nil, nil, nil, nil, errs
 	}
 
-	if err := json.Unmarshal(requestJson, req.BidRequest); err != nil {
+	if err := jsonutil.UnmarshalValid(requestJson, req.BidRequest); err != nil {
 		errs = []error{err}
 		return
 	}
@@ -621,7 +622,7 @@ func mergeBidderParams(req *openrtb_ext.RequestWrapper) error {
 	}
 
 	bidderParams := map[string]map[string]json.RawMessage{}
-	if err := json.Unmarshal(bidderParamsJson, &bidderParams); err != nil {
+	if err := jsonutil.Unmarshal(bidderParamsJson, &bidderParams); err != nil {
 		return nil
 	}
 
@@ -664,7 +665,7 @@ func mergeBidderParamsImpExt(impExt *openrtb_ext.ImpExt, reqExtParams map[string
 
 		impExtBidderMap := map[string]json.RawMessage{}
 		if len(impExtBidder) > 0 {
-			if err := json.Unmarshal(impExtBidder, &impExtBidderMap); err != nil {
+			if err := jsonutil.Unmarshal(impExtBidder, &impExtBidderMap); err != nil {
 				continue
 			}
 		}
@@ -678,7 +679,7 @@ func mergeBidderParamsImpExt(impExt *openrtb_ext.ImpExt, reqExtParams map[string
 		}
 
 		if modified {
-			impExtBidderJson, err := json.Marshal(impExtBidderMap)
+			impExtBidderJson, err := jsonutil.Marshal(impExtBidderMap)
 			if err != nil {
 				return fmt.Errorf("error marshalling ext.BIDDER: %s", err.Error())
 			}
@@ -712,7 +713,7 @@ func mergeBidderParamsImpExtPrebid(impExt *openrtb_ext.ImpExt, reqExtParams map[
 
 		impExtPrebidBidderMap := map[string]json.RawMessage{}
 		if len(impExtPrebidBidder) > 0 {
-			if err := json.Unmarshal(impExtPrebidBidder, &impExtPrebidBidderMap); err != nil {
+			if err := jsonutil.Unmarshal(impExtPrebidBidder, &impExtPrebidBidderMap); err != nil {
 				continue
 			}
 		}
@@ -726,7 +727,7 @@ func mergeBidderParamsImpExtPrebid(impExt *openrtb_ext.ImpExt, reqExtParams map[
 		}
 
 		if modified {
-			impExtPrebidBidderJson, err := json.Marshal(impExtPrebidBidderMap)
+			impExtPrebidBidderJson, err := jsonutil.Marshal(impExtPrebidBidderMap)
 			if err != nil {
 				return fmt.Errorf("error marshalling ext.prebid.bidder.BIDDER: %s", err.Error())
 			}
@@ -1188,7 +1189,7 @@ func fillAndValidateNative(n *openrtb2.Native, impIndex int) error {
 		return fmt.Errorf("request.imp[%d].native missing required property \"request\"", impIndex)
 	}
 	var nativePayload nativeRequests.Request
-	if err := json.Unmarshal(json.RawMessage(n.Request), &nativePayload); err != nil {
+	if err := jsonutil.UnmarshalValid(json.RawMessage(n.Request), &nativePayload); err != nil {
 		return err
 	}
 
@@ -1205,7 +1206,7 @@ func fillAndValidateNative(n *openrtb2.Native, impIndex int) error {
 		return err
 	}
 
-	serialized, err := json.Marshal(nativePayload)
+	serialized, err := jsonutil.Marshal(nativePayload)
 	if err != nil {
 		return err
 	}
@@ -2077,7 +2078,7 @@ func getJsonSyntaxError(testJSON []byte) (bool, string) {
 	}
 	type jNode map[string]*JsonNode
 	docErrdoc := &jNode{}
-	docErr := json.Unmarshal(testJSON, docErrdoc)
+	docErr := jsonutil.UnmarshalValid(testJSON, docErrdoc)
 	if uerror, ok := docErr.(*json.SyntaxError); ok {
 		err := fmt.Sprintf("%s at offset %v", uerror.Error(), uerror.Offset)
 		return true, err
@@ -2226,7 +2227,7 @@ func (deps *endpointDeps) processStoredRequests(requestJson []byte, impInfo []Im
 		}
 	}
 	if len(resolvedImps) > 0 {
-		newImpJson, err := json.Marshal(resolvedImps)
+		newImpJson, err := jsonutil.Marshal(resolvedImps)
 		if err != nil {
 			return nil, nil, []error{err}
 		}
@@ -2246,7 +2247,7 @@ func parseImpInfo(requestJson []byte) (impData []ImpExtPrebidData, errs []error)
 			impExtData, _, _, err := jsonparser.Get(imp, "ext", "prebid")
 			var impExtPrebid openrtb_ext.ExtImpPrebid
 			if impExtData != nil {
-				if err := json.Unmarshal(impExtData, &impExtPrebid); err != nil {
+				if err := jsonutil.Unmarshal(impExtData, &impExtPrebid); err != nil {
 					errs = append(errs, err)
 				}
 			}
@@ -2375,7 +2376,7 @@ func getAccountID(pub *openrtb2.Publisher) string {
 	if pub != nil {
 		if pub.Ext != nil {
 			var pubExt openrtb_ext.ExtPublisher
-			err := json.Unmarshal(pub.Ext, &pubExt)
+			err := jsonutil.Unmarshal(pub.Ext, &pubExt)
 			if err == nil && pubExt.Prebid != nil && pubExt.Prebid.ParentAccount != nil && *pubExt.Prebid.ParentAccount != "" {
 				return *pubExt.Prebid.ParentAccount
 			}

endpoints/openrtb2/auction_test.go

@@ -192,7 +192,7 @@ func runEndToEndTest(t *testing.T, auctionEndpointHandler httprouter.Handle, tes
 
 	// Either assert bid response or expected error
 	if len(test.ExpectedErrorMessage) > 0 {
-		assert.True(t, strings.HasPrefix(actualJsonBidResponse, test.ExpectedErrorMessage), "Actual: %s \nExpected: %s. Filename: %s \n", actualJsonBidResponse, test.ExpectedErrorMessage, testFile)
+		assert.Contains(t, actualJsonBidResponse, test.ExpectedErrorMessage, "Actual: %s \nExpected: %s. Filename: %s \n", actualJsonBidResponse, test.ExpectedErrorMessage, testFile)
 	}
 
 	if len(test.ExpectedBidResponse) > 0 {
@@ -476,8 +476,8 @@ func TestExplicitUserId(t *testing.T) {
 // processes aliases before it processes stored imps.  Changing that order
 // would probably cause this test to fail.
 func TestBadAliasRequests(t *testing.T) {
-	doBadAliasRequest(t, "sample-requests/invalid-stored/bad_stored_imp.json", "Invalid request: Invalid JSON in Default Request Settings: invalid character '\"' after object key:value pair at offset 51\n")
-	doBadAliasRequest(t, "sample-requests/invalid-stored/bad_incoming_imp.json", "Invalid request: Invalid JSON in Incoming Request: invalid character '\"' after object key:value pair at offset 230\n")
+	doBadAliasRequest(t, "sample-requests/invalid-stored/bad_stored_imp.json", "Invalid request: Invalid JSON Document\n")
+	doBadAliasRequest(t, "sample-requests/invalid-stored/bad_incoming_imp.json", "Invalid request: Invalid JSON Document\n")
 }
 
 // doBadAliasRequest() is a customized variation of doRequest(), above
@@ -1907,7 +1907,7 @@ func TestValidateRequestExt(t *testing.T) {
 		{
 			description:     "prebid cache - bids - wrong type",
 			givenRequestExt: json.RawMessage(`{"prebid":{"cache":{"bids":true}}}`),
-			expectedErrors:  []string{`json: cannot unmarshal bool into Go struct field ExtRequestPrebidCache.cache.bids of type openrtb_ext.ExtRequestPrebidCacheBids`},
+			expectedErrors:  []string{"openrtb_ext.ExtRequestPrebid.Cache: openrtb_ext.ExtRequestPrebidCache.Bids: readObjectStart: expect { or n, but found t, error found in #10 byte of ...|:{\"bids\":true}}|..., bigger context ...|{\"cache\":{\"bids\":true}}|..."},
 		},
 		{
 			description:     "prebid cache - bids - provided",
@@ -1921,7 +1921,7 @@ func TestValidateRequestExt(t *testing.T) {
 		{
 			description:     "prebid cache - vastxml - wrong type",
 			givenRequestExt: json.RawMessage(`{"prebid":{"cache":{"vastxml":true}}}`),
-			expectedErrors:  []string{`json: cannot unmarshal bool into Go struct field ExtRequestPrebidCache.cache.vastxml of type openrtb_ext.ExtRequestPrebidCacheVAST`},
+			expectedErrors:  []string{"openrtb_ext.ExtRequestPrebid.Cache: openrtb_ext.ExtRequestPrebidCache.VastXML: readObjectStart: expect { or n, but found t, error found in #10 byte of ...|vastxml\":true}}|..., bigger context ...|{\"cache\":{\"vastxml\":true}}|..."},
 		},
 		{
 			description:     "prebid cache - vastxml - provided",
@@ -4196,7 +4196,7 @@ func TestParseRequestParseImpInfoError(t *testing.T) {
 	assert.Nil(t, resReq, "Result request should be nil due to incorrect imp")
 	assert.Nil(t, impExtInfoMap, "Impression info map should be nil due to incorrect imp")
 	assert.Len(t, errL, 1, "One error should be returned")
-	assert.Contains(t, errL[0].Error(), "echovideoattrs of type bool", "Incorrect error message")
+	assert.Contains(t, errL[0].Error(), "openrtb_ext.Options.EchoVideoAttrs: ReadBool", "Incorrect error message")
 }
 
 func TestParseGzipedRequest(t *testing.T) {