From 859c3a4e206bc812039d83883d8d3bd3168dd76f Mon Sep 17 00:00:00 2001 From: bsardo <1168933+bsardo@users.noreply.github.com> Date: Mon, 16 Oct 2023 12:54:14 -0400 Subject: [PATCH 01/10] Add json-iterator lib with unmarshal helper functions and call throughout main auction flow --- account/account.go | 9 +- adservertargeting/reqcache.go | 3 +- currency/rate_converter.go | 4 +- endpoints/openrtb2/amp_auction.go | 9 +- endpoints/openrtb2/amp_auction_test.go | 10 +- endpoints/openrtb2/auction.go | 23 ++-- endpoints/openrtb2/auction_test.go | 12 +- .../invalid-stored/bad_incoming_1.json | 2 +- .../invalid-stored/bad_incoming_imp.json | 2 +- .../invalid-stored/bad_stored_imp.json | 2 +- .../invalid-stored/bad_stored_req.json | 2 +- .../invalid-whole/interstital-bad-perc.json | 2 +- .../invalid-whole/invalid-source.json | 2 +- .../invalid-whole/malformed-bid-request.json | 2 +- .../invalid-whole/regs-ext-malformed.json | 2 +- .../invalid-whole/user-ext-consent-int.json | 2 +- .../user-gdpr-consent-invalid.json | 2 +- endpoints/setuid_test.go | 4 +- errortypes/code.go | 1 + errortypes/errortypes.go | 17 +++ exchange/bidder.go | 5 +- exchange/exchange.go | 9 +- exchange/exchange_test.go | 5 +- exchange/utils.go | 25 ++-- exchange/utils_test.go | 20 +-- firstpartydata/first_party_data.go | 7 +- firstpartydata/first_party_data_test.go | 116 +++++++++--------- floors/rule_test.go | 2 +- go.mod | 3 + go.sum | 3 + modules/prebid/ortb2blocking/config.go | 3 +- modules/prebid/ortb2blocking/module_test.go | 2 +- openrtb_ext/convert_down_test.go | 99 +++++++-------- openrtb_ext/convert_up_test.go | 12 +- openrtb_ext/deal_tier.go | 5 +- openrtb_ext/deal_tier_test.go | 21 ++-- openrtb_ext/request.go | 5 +- openrtb_ext/request_wrapper.go | 55 +++++---- openrtb_ext/request_wrapper_test.go | 25 ++-- ortb/default_test.go | 4 +- privacy/scrubber.go | 5 +- router/router.go | 3 +- .../backends/file_fetcher/fetcher.go | 3 +- .../backends/http_fetcher/fetcher.go | 10 +- stored_requests/events/api/api.go | 6 +- stored_requests/events/http/http.go | 3 +- stored_responses/stored_responses.go | 3 +- util/jsonutil/jsonutil.go | 30 +++++ 48 files changed, 334 insertions(+), 267 deletions(-) diff --git a/account/account.go b/account/account.go index 7b5f9435a11..02cb5ac5c4d 100644 --- a/account/account.go +++ b/account/account.go @@ -2,7 +2,6 @@ package account import ( "context" - "encoding/json" "fmt" "github.com/prebid/go-gdpr/consentconstants" "github.com/prebid/prebid-server/config" @@ -11,6 +10,7 @@ import ( "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/stored_requests" "github.com/prebid/prebid-server/util/iputil" + "github.com/prebid/prebid-server/util/jsonutil" ) // GetAccount looks up the config.Account object referenced by the given accountID, with access rules applied @@ -47,8 +47,7 @@ func GetAccount(ctx context.Context, cfg *config.Configuration, fetcher stored_r } else { // accountID resolved to a valid account, merge with AccountDefaults for a complete config account = &config.Account{} - err := json.Unmarshal(accountJSON, account) - if _, ok := err.(*json.UnmarshalTypeError); ok { + if err := jsonutil.UnmarshalValid(accountJSON, account); err != nil { return nil, []error{&errortypes.MalformedAcct{ Message: fmt.Sprintf("The prebid-server account config for account id \"%s\" is malformed. Please reach out to the prebid server host.", accountID), }} @@ -66,10 +65,6 @@ func GetAccount(ctx context.Context, cfg *config.Configuration, fetcher stored_r me.RecordAccountUpgradeStatus(accountID) } - if err != nil { - errs = append(errs, err) - return nil, errs - } // Fill in ID if needed, so it can be left out of account definition if len(account.ID) == 0 { account.ID = accountID diff --git a/adservertargeting/reqcache.go b/adservertargeting/reqcache.go index cb2edac9e4a..7ff2aca13f7 100644 --- a/adservertargeting/reqcache.go +++ b/adservertargeting/reqcache.go @@ -5,6 +5,7 @@ import ( "github.com/buger/jsonparser" "github.com/prebid/openrtb/v19/openrtb2" + "github.com/prebid/prebid-server/util/jsonutil" ) type requestCache struct { @@ -24,7 +25,7 @@ func (reqImpCache *requestCache) GetImpsData() ([]json.RawMessage, error) { } var impsData []json.RawMessage - err = json.Unmarshal(imps, &impsData) + err = jsonutil.Unmarshal(imps, &impsData) if err != nil { return nil, err } diff --git a/currency/rate_converter.go b/currency/rate_converter.go index f28807701ae..cda8d763048 100644 --- a/currency/rate_converter.go +++ b/currency/rate_converter.go @@ -1,7 +1,6 @@ package currency import ( - "encoding/json" "fmt" "io" "net/http" @@ -10,6 +9,7 @@ import ( "github.com/golang/glog" "github.com/prebid/prebid-server/errortypes" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/util/timeutil" ) @@ -66,7 +66,7 @@ func (rc *RateConverter) fetch() (*Rates, error) { } updatedRates := &Rates{} - err = json.Unmarshal(bytesJSON, updatedRates) + err = jsonutil.UnmarshalValid(bytesJSON, updatedRates) if err != nil { return nil, err } diff --git a/endpoints/openrtb2/amp_auction.go b/endpoints/openrtb2/amp_auction.go index 2879610d8e4..6f933e9c888 100644 --- a/endpoints/openrtb2/amp_auction.go +++ b/endpoints/openrtb2/amp_auction.go @@ -38,6 +38,7 @@ import ( "github.com/prebid/prebid-server/stored_responses" "github.com/prebid/prebid-server/usersync" "github.com/prebid/prebid-server/util/iputil" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/version" ) @@ -340,7 +341,7 @@ func sendAmpResponse( // but this is a very unlikely corner case. Doing this so we can catch "hb_cache_id" // and "hb_cache_id_{deal}", which allows for deal support in AMP. bidExt := &openrtb_ext.ExtBid{} - err := json.Unmarshal(bid.Ext, bidExt) + err := jsonutil.Unmarshal(bid.Ext, bidExt) if err != nil { w.WriteHeader(http.StatusInternalServerError) fmt.Fprintf(w, "Critical error while unpacking AMP targets: %v", err) @@ -359,7 +360,7 @@ func sendAmpResponse( // Extract global targeting var extResponse openrtb_ext.ExtBidResponse - eRErr := json.Unmarshal(response.Ext, &extResponse) + eRErr := jsonutil.Unmarshal(response.Ext, &extResponse) if eRErr != nil { ao.Errors = append(ao.Errors, fmt.Errorf("AMP response: failed to unpack OpenRTB response.ext, debug info cannot be forwarded: %v", eRErr)) } @@ -408,7 +409,7 @@ func getExtBidResponse( } // Extract any errors var extResponse openrtb_ext.ExtBidResponse - eRErr := json.Unmarshal(response.Ext, &extResponse) + eRErr := jsonutil.Unmarshal(response.Ext, &extResponse) if eRErr != nil { ao.Errors = append(ao.Errors, fmt.Errorf("AMP response: failed to unpack OpenRTB response.ext, debug info cannot be forwarded: %v", eRErr)) } @@ -523,7 +524,7 @@ func (deps *endpointDeps) loadRequestJSONForAmp(httpRequest *http.Request) (req // The fetched config becomes the entire OpenRTB request requestJSON := storedRequests[ampParams.StoredRequestID] - if err := json.Unmarshal(requestJSON, req); err != nil { + if err := jsonutil.UnmarshalValid(requestJSON, req); err != nil { errs = []error{err} return } diff --git a/endpoints/openrtb2/amp_auction_test.go b/endpoints/openrtb2/amp_auction_test.go index 87033f71b64..d72c9aff1e8 100644 --- a/endpoints/openrtb2/amp_auction_test.go +++ b/endpoints/openrtb2/amp_auction_test.go @@ -524,7 +524,7 @@ func TestOverrideWithParams(t *testing.T) { Site: &openrtb2.Site{Ext: json.RawMessage(`{"amp":1}`)}, User: &openrtb2.User{Ext: json.RawMessage(`malformed`)}, }, - errorMsgs: []string{"invalid character 'm' looking for beginning of value"}, + errorMsgs: []string{"ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|..."}, expectFatalErrors: true, }, }, @@ -572,7 +572,7 @@ func TestOverrideWithParams(t *testing.T) { User: &openrtb2.User{Ext: json.RawMessage(`{"prebid":{malformed}}`)}, Site: &openrtb2.Site{Ext: json.RawMessage(`{"amp":1}`)}, }, - errorMsgs: []string{"invalid character 'm' looking for beginning of object key string"}, + errorMsgs: []string{"ReadObjectCB: expect \" after {, but found m, error found in #10 byte of ...|prebid\":{malformed}}|..., bigger context ...|{\"prebid\":{malformed}}|..."}, expectFatalErrors: true, }, }, @@ -1139,7 +1139,7 @@ func TestInitAmpTargetingAndCache(t *testing.T) { { name: "malformed", request: &openrtb2.BidRequest{Ext: json.RawMessage("malformed")}, - expectedErrs: []string{"invalid character 'm' looking for beginning of value"}, + expectedErrs: []string{"ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|..."}, }, { name: "nil", @@ -2253,7 +2253,7 @@ func TestSendAmpResponse_LogsErrors(t *testing.T) { { description: "Error logged when bid.ext unmarshal fails", expectedErrors: []error{ - errors.New("Critical error while unpacking AMP targets: unexpected end of JSON input"), + errors.New("Critical error while unpacking AMP targets: readObjectStart: expect { or n, but found \", error found in #1 byte of ...|\"hb_cache_i|..., bigger context ...|\"hb_cache_id|..."), }, expectedStatus: http.StatusInternalServerError, writer: httptest.NewRecorder(), @@ -2330,7 +2330,7 @@ func TestSendAmpResponse_LogsErrors(t *testing.T) { labels, ao = sendAmpResponse(test.writer, test.hookExecutor, &exchange.AuctionResponse{BidResponse: test.response}, &reqWrapper, account, labels, ao, nil) - assert.Equal(t, ao.Errors, test.expectedErrors, "Invalid errors.") + assert.Equal(t, test.expectedErrors, ao.Errors, "Invalid errors.") assert.Equal(t, test.expectedStatus, ao.Status, "Invalid HTTP response status.") }) } diff --git a/endpoints/openrtb2/auction.go b/endpoints/openrtb2/auction.go index 810741982ef..8982fb464fa 100644 --- a/endpoints/openrtb2/auction.go +++ b/endpoints/openrtb2/auction.go @@ -53,6 +53,7 @@ import ( "github.com/prebid/prebid-server/usersync" "github.com/prebid/prebid-server/util/httputil" "github.com/prebid/prebid-server/util/iputil" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/util/uuidutil" "github.com/prebid/prebid-server/version" ) @@ -303,7 +304,7 @@ func setSeatNonBidRaw(request *openrtb_ext.RequestWrapper, auctionResponse *exch // by HoldAuction response := auctionResponse.BidResponse respExt := &openrtb_ext.ExtBidResponse{} - if err := json.Unmarshal(response.Ext, &respExt); err != nil { + if err := jsonutil.Unmarshal(response.Ext, &respExt); err != nil { return err } if setSeatNonBid(respExt, request, auctionResponse) { @@ -440,7 +441,7 @@ func (deps *endpointDeps) parseRequest(httpRequest *http.Request, labels *metric requestJson, rejectErr := hookExecutor.ExecuteEntrypointStage(httpRequest, requestJson) if rejectErr != nil { errs = []error{rejectErr} - if err = json.Unmarshal(requestJson, req.BidRequest); err != nil { + if err = jsonutil.UnmarshalValid(requestJson, req.BidRequest); err != nil { glog.Errorf("Failed to unmarshal BidRequest during entrypoint rejection: %s", err) } return @@ -488,7 +489,7 @@ func (deps *endpointDeps) parseRequest(httpRequest *http.Request, labels *metric requestJson, rejectErr = hookExecutor.ExecuteRawAuctionStage(requestJson) if rejectErr != nil { errs = []error{rejectErr} - if err = json.Unmarshal(requestJson, req.BidRequest); err != nil { + if err = jsonutil.UnmarshalValid(requestJson, req.BidRequest); err != nil { glog.Errorf("Failed to unmarshal BidRequest during raw auction stage rejection: %s", err) } return @@ -517,7 +518,7 @@ func (deps *endpointDeps) parseRequest(httpRequest *http.Request, labels *metric return nil, nil, nil, nil, nil, nil, errs } - if err := json.Unmarshal(requestJson, req.BidRequest); err != nil { + if err := jsonutil.UnmarshalValid(requestJson, req.BidRequest); err != nil { errs = []error{err} return } @@ -616,7 +617,7 @@ func mergeBidderParams(req *openrtb_ext.RequestWrapper) error { } bidderParams := map[string]map[string]json.RawMessage{} - if err := json.Unmarshal(bidderParamsJson, &bidderParams); err != nil { + if err := jsonutil.Unmarshal(bidderParamsJson, &bidderParams); err != nil { return nil } @@ -659,7 +660,7 @@ func mergeBidderParamsImpExt(impExt *openrtb_ext.ImpExt, reqExtParams map[string impExtBidderMap := map[string]json.RawMessage{} if len(impExtBidder) > 0 { - if err := json.Unmarshal(impExtBidder, &impExtBidderMap); err != nil { + if err := jsonutil.Unmarshal(impExtBidder, &impExtBidderMap); err != nil { continue } } @@ -707,7 +708,7 @@ func mergeBidderParamsImpExtPrebid(impExt *openrtb_ext.ImpExt, reqExtParams map[ impExtPrebidBidderMap := map[string]json.RawMessage{} if len(impExtPrebidBidder) > 0 { - if err := json.Unmarshal(impExtPrebidBidder, &impExtPrebidBidderMap); err != nil { + if err := jsonutil.Unmarshal(impExtPrebidBidder, &impExtPrebidBidderMap); err != nil { continue } } @@ -1187,7 +1188,7 @@ func fillAndValidateNative(n *openrtb2.Native, impIndex int) error { return fmt.Errorf("request.imp[%d].native missing required property \"request\"", impIndex) } var nativePayload nativeRequests.Request - if err := json.Unmarshal(json.RawMessage(n.Request), &nativePayload); err != nil { + if err := jsonutil.UnmarshalValid(json.RawMessage(n.Request), &nativePayload); err != nil { return err } @@ -2076,7 +2077,7 @@ func getJsonSyntaxError(testJSON []byte) (bool, string) { } type jNode map[string]*JsonNode docErrdoc := &jNode{} - docErr := json.Unmarshal(testJSON, docErrdoc) + docErr := jsonutil.UnmarshalValid(testJSON, docErrdoc) if uerror, ok := docErr.(*json.SyntaxError); ok { err := fmt.Sprintf("%s at offset %v", uerror.Error(), uerror.Offset) return true, err @@ -2245,7 +2246,7 @@ func parseImpInfo(requestJson []byte) (impData []ImpExtPrebidData, errs []error) impExtData, _, _, err := jsonparser.Get(imp, "ext", "prebid") var impExtPrebid openrtb_ext.ExtImpPrebid if impExtData != nil { - if err := json.Unmarshal(impExtData, &impExtPrebid); err != nil { + if err := jsonutil.Unmarshal(impExtData, &impExtPrebid); err != nil { errs = append(errs, err) } } @@ -2374,7 +2375,7 @@ func getAccountID(pub *openrtb2.Publisher) string { if pub != nil { if pub.Ext != nil { var pubExt openrtb_ext.ExtPublisher - err := json.Unmarshal(pub.Ext, &pubExt) + err := jsonutil.Unmarshal(pub.Ext, &pubExt) if err == nil && pubExt.Prebid != nil && pubExt.Prebid.ParentAccount != nil && *pubExt.Prebid.ParentAccount != "" { return *pubExt.Prebid.ParentAccount } diff --git a/endpoints/openrtb2/auction_test.go b/endpoints/openrtb2/auction_test.go index c071d6b0950..93d17d45bc9 100644 --- a/endpoints/openrtb2/auction_test.go +++ b/endpoints/openrtb2/auction_test.go @@ -194,7 +194,7 @@ func runEndToEndTest(t *testing.T, auctionEndpointHandler httprouter.Handle, tes // Either assert bid response or expected error if len(test.ExpectedErrorMessage) > 0 { - assert.True(t, strings.HasPrefix(actualJsonBidResponse, test.ExpectedErrorMessage), "Actual: %s \nExpected: %s. Filename: %s \n", actualJsonBidResponse, test.ExpectedErrorMessage, testFile) + assert.Contains(t, actualJsonBidResponse, test.ExpectedErrorMessage, "Actual: %s \nExpected: %s. Filename: %s \n", actualJsonBidResponse, test.ExpectedErrorMessage, testFile) } if len(test.ExpectedBidResponse) > 0 { @@ -478,8 +478,8 @@ func TestExplicitUserId(t *testing.T) { // processes aliases before it processes stored imps. Changing that order // would probably cause this test to fail. func TestBadAliasRequests(t *testing.T) { - doBadAliasRequest(t, "sample-requests/invalid-stored/bad_stored_imp.json", "Invalid request: Invalid JSON in Default Request Settings: invalid character '\"' after object key:value pair at offset 51\n") - doBadAliasRequest(t, "sample-requests/invalid-stored/bad_incoming_imp.json", "Invalid request: Invalid JSON in Incoming Request: invalid character '\"' after object key:value pair at offset 230\n") + doBadAliasRequest(t, "sample-requests/invalid-stored/bad_stored_imp.json", "Invalid request: Invalid JSON Document\n") + doBadAliasRequest(t, "sample-requests/invalid-stored/bad_incoming_imp.json", "Invalid request: Invalid JSON Document\n") } // doBadAliasRequest() is a customized variation of doRequest(), above @@ -1907,7 +1907,7 @@ func TestValidateRequestExt(t *testing.T) { { description: "prebid cache - bids - wrong type", givenRequestExt: json.RawMessage(`{"prebid":{"cache":{"bids":true}}}`), - expectedErrors: []string{`json: cannot unmarshal bool into Go struct field ExtRequestPrebidCache.cache.bids of type openrtb_ext.ExtRequestPrebidCacheBids`}, + expectedErrors: []string{"openrtb_ext.ExtRequestPrebid.Cache: openrtb_ext.ExtRequestPrebidCache.Bids: readObjectStart: expect { or n, but found t, error found in #10 byte of ...|:{\"bids\":true}}|..., bigger context ...|{\"cache\":{\"bids\":true}}|..."}, }, { description: "prebid cache - bids - provided", @@ -1921,7 +1921,7 @@ func TestValidateRequestExt(t *testing.T) { { description: "prebid cache - vastxml - wrong type", givenRequestExt: json.RawMessage(`{"prebid":{"cache":{"vastxml":true}}}`), - expectedErrors: []string{`json: cannot unmarshal bool into Go struct field ExtRequestPrebidCache.cache.vastxml of type openrtb_ext.ExtRequestPrebidCacheVAST`}, + expectedErrors: []string{"openrtb_ext.ExtRequestPrebid.Cache: openrtb_ext.ExtRequestPrebidCache.VastXML: readObjectStart: expect { or n, but found t, error found in #10 byte of ...|vastxml\":true}}|..., bigger context ...|{\"cache\":{\"vastxml\":true}}|..."}, }, { description: "prebid cache - vastxml - provided", @@ -4161,7 +4161,7 @@ func TestParseRequestParseImpInfoError(t *testing.T) { assert.Nil(t, resReq, "Result request should be nil due to incorrect imp") assert.Nil(t, impExtInfoMap, "Impression info map should be nil due to incorrect imp") assert.Len(t, errL, 1, "One error should be returned") - assert.Contains(t, errL[0].Error(), "echovideoattrs of type bool", "Incorrect error message") + assert.Contains(t, errL[0].Error(), "openrtb_ext.Options.EchoVideoAttrs: ReadBool", "Incorrect error message") } func TestParseGzipedRequest(t *testing.T) { diff --git a/endpoints/openrtb2/sample-requests/invalid-stored/bad_incoming_1.json b/endpoints/openrtb2/sample-requests/invalid-stored/bad_incoming_1.json index 812f0664fbb..4803ec7f605 100644 --- a/endpoints/openrtb2/sample-requests/invalid-stored/bad_incoming_1.json +++ b/endpoints/openrtb2/sample-requests/invalid-stored/bad_incoming_1.json @@ -11,5 +11,5 @@ } }, "expectedReturnCode": 400, - "expectedErrorMessage": "Invalid request: Invalid JSON in Incoming Request: invalid character ']' looking for beginning of value at offset" + "expectedErrorMessage": "Invalid request: readObjectStart: expect { or n, but found" } diff --git a/endpoints/openrtb2/sample-requests/invalid-stored/bad_incoming_imp.json b/endpoints/openrtb2/sample-requests/invalid-stored/bad_incoming_imp.json index e3960b17399..fb0dc866d10 100644 --- a/endpoints/openrtb2/sample-requests/invalid-stored/bad_incoming_imp.json +++ b/endpoints/openrtb2/sample-requests/invalid-stored/bad_incoming_imp.json @@ -37,5 +37,5 @@ "tmax": 500 }, "expectedReturnCode": 400, - "expectedErrorMessage": "Invalid request: Invalid JSON in Imp[0] of Incoming Request: invalid character '\"' after object key:value pair at offset 132\n" + "expectedErrorMessage": "Invalid request: Invalid JSON Document" } diff --git a/endpoints/openrtb2/sample-requests/invalid-stored/bad_stored_imp.json b/endpoints/openrtb2/sample-requests/invalid-stored/bad_stored_imp.json index 0fed6c32adf..ea19855e75a 100644 --- a/endpoints/openrtb2/sample-requests/invalid-stored/bad_stored_imp.json +++ b/endpoints/openrtb2/sample-requests/invalid-stored/bad_stored_imp.json @@ -33,5 +33,5 @@ "tmax": 500 }, "expectedReturnCode": 400, - "expectedErrorMessage": "Invalid request: imp.ext.prebid.storedrequest.id 7: Stored Imp has Invalid JSON" + "expectedErrorMessage": "Invalid request: Invalid JSON Document" } diff --git a/endpoints/openrtb2/sample-requests/invalid-stored/bad_stored_req.json b/endpoints/openrtb2/sample-requests/invalid-stored/bad_stored_req.json index 4e9d7f03352..50a000110b1 100644 --- a/endpoints/openrtb2/sample-requests/invalid-stored/bad_stored_req.json +++ b/endpoints/openrtb2/sample-requests/invalid-stored/bad_stored_req.json @@ -23,5 +23,5 @@ } }, "expectedReturnCode": 400, - "expectedErrorMessage": "Invalid request: ext.prebid.storedrequest.id refers to Stored Request 3 which contains Invalid JSON" + "expectedErrorMessage": "Invalid request: readObjectStart: expect { or n, but found" } diff --git a/endpoints/openrtb2/sample-requests/invalid-whole/interstital-bad-perc.json b/endpoints/openrtb2/sample-requests/invalid-whole/interstital-bad-perc.json index 302372b5e5d..9042b9a6098 100644 --- a/endpoints/openrtb2/sample-requests/invalid-whole/interstital-bad-perc.json +++ b/endpoints/openrtb2/sample-requests/invalid-whole/interstital-bad-perc.json @@ -49,5 +49,5 @@ } }, "expectedReturnCode": 400, - "expectedErrorMessage": "Invalid request: request.device.ext.prebid.interstitial.minwidthperc must be a number between 0 and 100\n" + "expectedErrorMessage": "Invalid request: openrtb_ext.ExtDevicePrebid.Interstitial: unmarshalerDecoder: request.device.ext.prebid.interstitial.minwidthperc must be a number between 0 and 100, error found in #10 byte of ...| }\n }|..., bigger context ...|: 120,\n \"minheightperc\": 60\n }\n }|...\n" } diff --git a/endpoints/openrtb2/sample-requests/invalid-whole/invalid-source.json b/endpoints/openrtb2/sample-requests/invalid-whole/invalid-source.json index 5aa7fd4dea1..fd78849e316 100644 --- a/endpoints/openrtb2/sample-requests/invalid-whole/invalid-source.json +++ b/endpoints/openrtb2/sample-requests/invalid-whole/invalid-source.json @@ -39,5 +39,5 @@ ] }, "expectedReturnCode": 400, - "expectedErrorMessage": "Invalid request: json: cannot unmarshal object into Go struct field ExtAppPrebid.source of type string" + "expectedErrorMessage": "Invalid request: openrtb_ext.ExtAppPrebid.Source: ReadString: expects \" or n, but found {" } diff --git a/endpoints/openrtb2/sample-requests/invalid-whole/malformed-bid-request.json b/endpoints/openrtb2/sample-requests/invalid-whole/malformed-bid-request.json index c6fd52304a7..7dd6b24ede1 100644 --- a/endpoints/openrtb2/sample-requests/invalid-whole/malformed-bid-request.json +++ b/endpoints/openrtb2/sample-requests/invalid-whole/malformed-bid-request.json @@ -2,5 +2,5 @@ "description": "Malformed bid request throws an error", "mockBidRequest": "malformed", "expectedReturnCode": 400, - "expectedErrorMessage": "Invalid request: invalid character 'm' looking for beginning of value\n" + "expectedErrorMessage": "Invalid request: readObjectStart: expect { or n, but found m" } diff --git a/endpoints/openrtb2/sample-requests/invalid-whole/regs-ext-malformed.json b/endpoints/openrtb2/sample-requests/invalid-whole/regs-ext-malformed.json index ab44e3e2428..ebe0bc3b105 100644 --- a/endpoints/openrtb2/sample-requests/invalid-whole/regs-ext-malformed.json +++ b/endpoints/openrtb2/sample-requests/invalid-whole/regs-ext-malformed.json @@ -42,5 +42,5 @@ } }, "expectedReturnCode": 400, - "expectedErrorMessage": "Invalid request: request.regs.ext is invalid: json: cannot unmarshal string into Go value of type map[string]json.RawMessage\n" + "expectedErrorMessage": "Invalid request: request.regs.ext is invalid: ReadMapCB: expect { or n, but found " } diff --git a/endpoints/openrtb2/sample-requests/invalid-whole/user-ext-consent-int.json b/endpoints/openrtb2/sample-requests/invalid-whole/user-ext-consent-int.json index a26db8a5695..4b5fb997897 100644 --- a/endpoints/openrtb2/sample-requests/invalid-whole/user-ext-consent-int.json +++ b/endpoints/openrtb2/sample-requests/invalid-whole/user-ext-consent-int.json @@ -46,5 +46,5 @@ } }, "expectedReturnCode": 400, - "expectedErrorMessage": "Invalid request: request.user.ext object is not valid: json: cannot unmarshal number into Go value of type string\n" + "expectedErrorMessage": "Invalid request: request.user.ext object is not valid: ReadString: expects \" or n, but found 1" } diff --git a/endpoints/openrtb2/sample-requests/invalid-whole/user-gdpr-consent-invalid.json b/endpoints/openrtb2/sample-requests/invalid-whole/user-gdpr-consent-invalid.json index c4646550dd2..57630c7cfbb 100644 --- a/endpoints/openrtb2/sample-requests/invalid-whole/user-gdpr-consent-invalid.json +++ b/endpoints/openrtb2/sample-requests/invalid-whole/user-gdpr-consent-invalid.json @@ -41,5 +41,5 @@ } }, "expectedReturnCode": 400, - "expectedErrorMessage": "Invalid request: request.user.ext object is not valid: json: cannot unmarshal number into Go value of type string" + "expectedErrorMessage": "Invalid request: request.user.ext object is not valid: ReadString: expects \" or n, but found 2" } diff --git a/endpoints/setuid_test.go b/endpoints/setuid_test.go index 897538c35e5..b4311dbd239 100644 --- a/endpoints/setuid_test.go +++ b/endpoints/setuid_test.go @@ -1318,14 +1318,14 @@ func TestSetUIDEndpointMetrics(t *testing.T) { cfgAccountRequired: true, expectedResponseCode: 400, expectedMetrics: func(m *metrics.MetricsEngineMock) { - m.On("RecordSetUid", metrics.SetUidBadRequest).Once() + m.On("RecordSetUid", metrics.SetUidAccountConfigMalformed).Once() }, expectedAnalytics: func(a *MockAnalytics) { expected := analytics.SetUIDObject{ Status: 400, Bidder: "pubmatic", UID: "", - Errors: []error{errors.New("unexpected end of JSON input")}, + Errors: []error{errCookieSyncAccountConfigMalformed}, Success: false, } a.On("LogSetUIDObject", &expected).Once() diff --git a/errortypes/code.go b/errortypes/code.go index ef8287f2d7d..1dbd5c0e996 100644 --- a/errortypes/code.go +++ b/errortypes/code.go @@ -15,6 +15,7 @@ const ( MalformedAcctErrorCode ModuleRejectionErrorCode TmaxTimeoutErrorCode + FailedToUnmarshalErrorCode ) // Defines numeric codes for well-known warnings. diff --git a/errortypes/errortypes.go b/errortypes/errortypes.go index 01a5cca4af7..58221d7bec7 100644 --- a/errortypes/errortypes.go +++ b/errortypes/errortypes.go @@ -220,3 +220,20 @@ func (err *Warning) Code() int { func (err *Warning) Severity() Severity { return SeverityWarning } + +// FailedToUnmarshal should be used to represent errors that occur when unmarshaling raw json. +type FailedToUnmarshal struct { + Message string +} + +func (err *FailedToUnmarshal) Error() string { + return err.Message +} + +func (err *FailedToUnmarshal) Code() int { + return FailedToUnmarshalErrorCode +} + +func (err *FailedToUnmarshal) Severity() Severity { + return SeverityFatal +} diff --git a/exchange/bidder.go b/exchange/bidder.go index dfdb7cdf85f..718768d2df4 100644 --- a/exchange/bidder.go +++ b/exchange/bidder.go @@ -33,6 +33,7 @@ import ( "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/metrics" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/util/jsonutil" "golang.org/x/net/context/ctxhttp" ) @@ -406,7 +407,7 @@ func (bidder *bidderAdapter) requestBid(ctx context.Context, bidderRequest Bidde func addNativeTypes(bid *openrtb2.Bid, request *openrtb2.BidRequest) (*nativeResponse.Response, []error) { var errs []error var nativeMarkup *nativeResponse.Response - if err := json.Unmarshal(json.RawMessage(bid.AdM), &nativeMarkup); err != nil || len(nativeMarkup.Assets) == 0 { + if err := jsonutil.UnmarshalValid(json.RawMessage(bid.AdM), &nativeMarkup); err != nil || len(nativeMarkup.Assets) == 0 { // Some bidders are returning non-IAB compliant native markup. In this case Prebid server will not be able to add types. E.g Facebook return nil, errs } @@ -418,7 +419,7 @@ func addNativeTypes(bid *openrtb2.Bid, request *openrtb2.BidRequest) (*nativeRes } var nativePayload nativeRequests.Request - if err := json.Unmarshal(json.RawMessage((*nativeImp).Request), &nativePayload); err != nil { + if err := jsonutil.UnmarshalValid(json.RawMessage((*nativeImp).Request), &nativePayload); err != nil { errs = append(errs, err) } diff --git a/exchange/exchange.go b/exchange/exchange.go index 7544b4a7020..5c9bbfed721 100644 --- a/exchange/exchange.go +++ b/exchange/exchange.go @@ -35,6 +35,7 @@ import ( "github.com/prebid/prebid-server/stored_requests" "github.com/prebid/prebid-server/stored_responses" "github.com/prebid/prebid-server/usersync" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/util/maputil" "github.com/buger/jsonparser" @@ -1287,7 +1288,7 @@ func makeBidExtJSON(ext json.RawMessage, prebid *openrtb_ext.ExtBidPrebid, impEx var extMap map[string]interface{} if len(ext) != 0 { - if err := json.Unmarshal(ext, &extMap); err != nil { + if err := jsonutil.Unmarshal(ext, &extMap); err != nil { return nil, err } } else { @@ -1311,7 +1312,7 @@ func makeBidExtJSON(ext json.RawMessage, prebid *openrtb_ext.ExtBidPrebid, impEx Meta openrtb_ext.ExtBidPrebidMeta `json:"meta"` } `json:"prebid"` }{} - if err := json.Unmarshal(ext, &metaContainer); err != nil { + if err := jsonutil.Unmarshal(ext, &metaContainer); err != nil { return nil, fmt.Errorf("error validaing response from server, %s", err) } prebid.Meta = &metaContainer.Prebid.Meta @@ -1436,7 +1437,7 @@ func buildStoredAuctionResponse(storedAuctionResponses map[string]json.RawMessag for impId, storedResp := range storedAuctionResponses { var seatBids []openrtb2.SeatBid - if err := json.Unmarshal(storedResp, &seatBids); err != nil { + if err := jsonutil.UnmarshalValid(storedResp, &seatBids); err != nil { return nil, nil, nil, err } for _, seat := range seatBids { @@ -1455,7 +1456,7 @@ func buildStoredAuctionResponse(storedAuctionResponses map[string]json.RawMessag if seat.Ext != nil { var seatExt openrtb_ext.ExtBidResponse - if err := json.Unmarshal(seat.Ext, &seatExt); err != nil { + if err := jsonutil.Unmarshal(seat.Ext, &seatExt); err != nil { return nil, nil, nil, err } // add in FLEDGE response with impId substituted diff --git a/exchange/exchange_test.go b/exchange/exchange_test.go index 6e6e8afe76e..9e34debdfd6 100644 --- a/exchange/exchange_test.go +++ b/exchange/exchange_test.go @@ -4177,7 +4177,7 @@ func TestMakeBidExtJSON(t *testing.T) { extBidPrebid: openrtb_ext.ExtBidPrebid{Type: openrtb_ext.BidType("video")}, impExtInfo: map[string]ImpExtInfo{"test_imp_id": {true, []byte(`{"video":{"h":480,"mimes":["video/mp4"]}}`), json.RawMessage(`"prebid": {"passthrough": {"imp_passthrough_val": some_val}}"`)}}, expectedBidExt: ``, - expectedErrMessage: "invalid character", + expectedErrMessage: "ReadString: expects \" or n, but found i, error found in #2 byte of ...|{invalid jso|..., bigger context ...|{invalid json}|...", }, { description: "Valid extension, empty extBidPrebid and invalid imp ext info", @@ -4192,9 +4192,8 @@ func TestMakeBidExtJSON(t *testing.T) { ext: json.RawMessage(`{"prebid":{"meta":{"brandId":"foo"}}}`), // brandId should be an int, but is a string in this test case extBidPrebid: openrtb_ext.ExtBidPrebid{Type: openrtb_ext.BidType("banner")}, impExtInfo: nil, - expectedErrMessage: "error validaing response from server, json: cannot unmarshal string into Go struct field ExtBidPrebidMeta.prebid.meta.brandId of type int", + expectedErrMessage: "error validaing response from server, Prebid: Meta: openrtb_ext.ExtBidPrebidMeta.BrandID: readUint64: unexpected character: \xff, error found in #10 byte of ...|brandId\":\"foo\"}}}|..., bigger context ...|{\"prebid\":{\"meta\":{\"brandId\":\"foo\"}}}|...", }, - // add invalid } for _, test := range testCases { diff --git a/exchange/utils.go b/exchange/utils.go index 0a38e487632..4bf6ae7da3a 100644 --- a/exchange/utils.go +++ b/exchange/utils.go @@ -25,6 +25,7 @@ import ( "github.com/prebid/prebid-server/privacy/lmt" "github.com/prebid/prebid-server/schain" "github.com/prebid/prebid-server/stored_responses" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/util/ptrutil" ) @@ -283,7 +284,7 @@ func ExtractReqExtBidderParamsMap(bidRequest *openrtb2.BidRequest) (map[string]j reqExt := &openrtb_ext.ExtRequest{} if len(bidRequest.Ext) > 0 { - err := json.Unmarshal(bidRequest.Ext, &reqExt) + err := jsonutil.Unmarshal(bidRequest.Ext, &reqExt) if err != nil { return nil, fmt.Errorf("error decoding Request.ext : %s", err.Error()) } @@ -294,7 +295,7 @@ func ExtractReqExtBidderParamsMap(bidRequest *openrtb2.BidRequest) (map[string]j } var bidderParams map[string]json.RawMessage - err := json.Unmarshal(reqExt.Prebid.BidderParams, &bidderParams) + err := jsonutil.Unmarshal(reqExt.Prebid.BidderParams, &bidderParams) if err != nil { return nil, err } @@ -415,7 +416,7 @@ func buildRequestExtForBidder(bidder string, requestExt json.RawMessage, request // Parse Existing Ext extMap := make(map[string]json.RawMessage) if len(requestExt) != 0 { - if err := json.Unmarshal(requestExt, &extMap); err != nil { + if err := jsonutil.Unmarshal(requestExt, &extMap); err != nil { return nil, err } } @@ -512,7 +513,7 @@ func extractBuyerUIDs(user *openrtb2.User) (map[string]string, error) { } var userExt openrtb_ext.ExtUser - if err := json.Unmarshal(user.Ext, &userExt); err != nil { + if err := jsonutil.Unmarshal(user.Ext, &userExt); err != nil { return nil, err } if userExt.Prebid == nil { @@ -550,20 +551,20 @@ func splitImps(imps []openrtb2.Imp) (map[string][]openrtb2.Imp, error) { for i, imp := range imps { var impExt map[string]json.RawMessage - if err := json.Unmarshal(imp.Ext, &impExt); err != nil { + if err := jsonutil.Unmarshal(imp.Ext, &impExt); err != nil { return nil, fmt.Errorf("invalid json for imp[%d]: %v", i, err) } var impExtPrebid map[string]json.RawMessage if impExtPrebidJSON, exists := impExt[openrtb_ext.PrebidExtKey]; exists { // validation already performed by impExt unmarshal. no error is possible here, proven by tests. - json.Unmarshal(impExtPrebidJSON, &impExtPrebid) + jsonutil.Unmarshal(impExtPrebidJSON, &impExtPrebid) } var impExtPrebidBidder map[string]json.RawMessage if impExtPrebidBidderJSON, exists := impExtPrebid[openrtb_ext.PrebidExtBidderKey]; exists { // validation already performed by impExt unmarshal. no error is possible here, proven by tests. - json.Unmarshal(impExtPrebidBidderJSON, &impExtPrebidBidder) + jsonutil.Unmarshal(impExtPrebidBidderJSON, &impExtPrebidBidder) } sanitizedImpExt, err := createSanitizedImpExt(impExt, impExtPrebid) @@ -680,7 +681,7 @@ func removeUnpermissionedEids(request *openrtb2.BidRequest, bidder string, reque // low level unmarshal to preserve other request.user.ext values. prebid server is non-destructive. var userExt map[string]json.RawMessage - if err := json.Unmarshal(request.User.Ext, &userExt); err != nil { + if err := jsonutil.Unmarshal(request.User.Ext, &userExt); err != nil { return err } @@ -690,7 +691,7 @@ func removeUnpermissionedEids(request *openrtb2.BidRequest, bidder string, reque } var eids []openrtb2.EID - if err := json.Unmarshal(eidsJSON, &eids); err != nil { + if err := jsonutil.Unmarshal(eidsJSON, &eids); err != nil { return err } @@ -773,7 +774,7 @@ func resolveBidder(bidder string, aliases map[string]string) openrtb_ext.BidderN func parseAliases(orig *openrtb2.BidRequest) (map[string]string, []error) { var aliases map[string]string if value, dataType, _, err := jsonparser.Get(orig.Ext, openrtb_ext.PrebidExtKey, "aliases"); dataType == jsonparser.Object && err == nil { - if err := json.Unmarshal(value, &aliases); err != nil { + if err := jsonutil.Unmarshal(value, &aliases); err != nil { return nil, []error{err} } } else if dataType != jsonparser.NotExist && err != jsonparser.KeyPathNotFoundError { @@ -786,7 +787,7 @@ func parseAliases(orig *openrtb2.BidRequest) (map[string]string, []error) { func parseAliasesGVLIDs(orig *openrtb2.BidRequest) (map[string]uint16, []error) { var aliasesGVLIDs map[string]uint16 if value, dataType, _, err := jsonparser.Get(orig.Ext, openrtb_ext.PrebidExtKey, "aliasgvlids"); dataType == jsonparser.Object && err == nil { - if err := json.Unmarshal(value, &aliasesGVLIDs); err != nil { + if err := jsonutil.Unmarshal(value, &aliasesGVLIDs); err != nil { return nil, []error{err} } } else if dataType != jsonparser.NotExist && err != jsonparser.KeyPathNotFoundError { @@ -1060,7 +1061,7 @@ func getPrebidMediaTypeForBid(bid openrtb2.Bid) (openrtb_ext.BidType, error) { if bid.Ext != nil { var bidExt openrtb_ext.ExtBid - err = json.Unmarshal(bid.Ext, &bidExt) + err = jsonutil.Unmarshal(bid.Ext, &bidExt) if err == nil && bidExt.Prebid != nil { if bidType, err = openrtb_ext.ParseBidType(string(bidExt.Prebid.Type)); err == nil { return bidType, nil diff --git a/exchange/utils_test.go b/exchange/utils_test.go index ee6dad1ad96..1add2fc4f96 100644 --- a/exchange/utils_test.go +++ b/exchange/utils_test.go @@ -187,21 +187,21 @@ func TestSplitImps(t *testing.T) { givenImps: []openrtb2.Imp{ {ID: "imp1", Ext: json.RawMessage(`malformed`)}, }, - expectedError: "invalid json for imp[0]: invalid character 'm' looking for beginning of value", + expectedError: "invalid json for imp[0]: ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...", }, { description: "Malformed imp.ext.prebid", givenImps: []openrtb2.Imp{ {ID: "imp1", Ext: json.RawMessage(`{"prebid": malformed}`)}, }, - expectedError: "invalid json for imp[0]: invalid character 'm' looking for beginning of value", + expectedError: "invalid json for imp[0]: Skip: do not know how to skip: 109, error found in #10 byte of ...|prebid\": malformed}|..., bigger context ...|{\"prebid\": malformed}|...", }, { description: "Malformed imp.ext.prebid.bidder", givenImps: []openrtb2.Imp{ {ID: "imp1", Ext: json.RawMessage(`{"prebid": {"bidder": malformed}}`)}, }, - expectedError: "invalid json for imp[0]: invalid character 'm' looking for beginning of value", + expectedError: "invalid json for imp[0]: Skip: do not know how to skip: 109, error found in #10 byte of ...|bidder\": malformed}}|..., bigger context ...|{\"prebid\": {\"bidder\": malformed}}|...", }, } @@ -592,7 +592,7 @@ func TestExtractAdapterReqBidderParamsMap(t *testing.T) { name: "malformed req.ext", givenBidRequest: &openrtb2.BidRequest{Ext: json.RawMessage("malformed")}, want: nil, - wantErr: errors.New("error decoding Request.ext : invalid character 'm' looking for beginning of value"), + wantErr: errors.New("error decoding Request.ext : readObjectStart: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|..."), }, { name: "extract bidder params from req.Ext for input request in adapter code", @@ -2528,7 +2528,7 @@ func TestBuildRequestExtForBidder_RequestExtMalformed(t *testing.T) { actualJson, actualErr := buildRequestExtForBidder(bidder, requestExt, requestExtParsed, bidderParams, alternateBidderCodes) assert.Equal(t, json.RawMessage(nil), actualJson) - assert.EqualError(t, actualErr, "invalid character 'm' looking for beginning of value") + assert.EqualError(t, actualErr, "ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...") } // newAdapterAliasBidRequest builds a BidRequest with aliases @@ -2851,17 +2851,17 @@ func TestRemoveUnpermissionedEidsUnmarshalErrors(t *testing.T) { { description: "Malformed Ext", userExt: json.RawMessage(`malformed`), - expectedErr: "invalid character 'm' looking for beginning of value", + expectedErr: "ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...", }, { description: "Malformed Eid Array Type", userExt: json.RawMessage(`{"eids":[42]}`), - expectedErr: "json: cannot unmarshal number into Go value of type openrtb2.EID", + expectedErr: "[]openrtb2.EID: readObjectStart: expect { or n, but found 4, error found in #2 byte of ...|[42]|..., bigger context ...|[42]|...", }, { description: "Malformed Eid Item Type", userExt: json.RawMessage(`{"eids":[{"source":42,"id":"anyID"}]}`), - expectedErr: "json: cannot unmarshal number into Go struct field EID.source of type string", + expectedErr: "[]openrtb2.EID: openrtb2.EID.Source: ReadString: expects \" or n, but found 4, error found in #10 byte of ...|\"source\":42,\"id\":\"an|..., bigger context ...|[{\"source\":42,\"id\":\"anyID\"}]|...", }, } @@ -4298,7 +4298,7 @@ func TestGetPrebidMediaTypeForBid(t *testing.T) { { description: "Invalid bid ext", inputBid: openrtb2.Bid{ID: "bidId", ImpID: "impId", Ext: json.RawMessage(`[true`)}, - expectedError: "Failed to parse bid mediatype for impression \"impId\", unexpected end of JSON input", + expectedError: "Failed to parse bid mediatype for impression \"impId\", readObjectStart: expect { or n, but found [, error found in #1 byte of ...|[true|..., bigger context ...|[true|...", }, { description: "Bid ext is nil", @@ -4339,7 +4339,7 @@ func TestGetMediaTypeForBid(t *testing.T) { { description: "invalid bid ext", inputBid: openrtb2.Bid{ID: "bidId", ImpID: "impId", Ext: json.RawMessage(`{"prebid"`)}, - expectedError: "Failed to parse bid mediatype for impression \"impId\", unexpected end of JSON input", + expectedError: "Failed to parse bid mediatype for impression \"impId\", openrtb_ext.ExtBid.readFieldHash: expect :, but found \x00, error found in #9 byte of ...|{\"prebid\"|..., bigger context ...|{\"prebid\"|...", }, { description: "Valid bid ext with mtype native", diff --git a/firstpartydata/first_party_data.go b/firstpartydata/first_party_data.go index 0fde931d445..b3bc20ddb77 100644 --- a/firstpartydata/first_party_data.go +++ b/firstpartydata/first_party_data.go @@ -10,6 +10,7 @@ import ( "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/ortb" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/util/ptrutil" ) @@ -213,7 +214,7 @@ func mergeUser(v *openrtb2.User, overrideJSON json.RawMessage) error { } // Merge - if err := json.Unmarshal(overrideJSON, &v); err != nil { + if err := jsonutil.Unmarshal(overrideJSON, &v); err != nil { return err } @@ -307,7 +308,7 @@ func mergeSite(v *openrtb2.Site, overrideJSON json.RawMessage, bidderName string } // Merge - if err := json.Unmarshal(overrideJSON, &v); err != nil { + if err := jsonutil.Unmarshal(overrideJSON, &v); err != nil { return err } @@ -424,7 +425,7 @@ func mergeApp(v *openrtb2.App, overrideJSON json.RawMessage) error { } // Merge - if err := json.Unmarshal(overrideJSON, &v); err != nil { + if err := jsonutil.Unmarshal(overrideJSON, &v); err != nil { return err } diff --git a/firstpartydata/first_party_data_test.go b/firstpartydata/first_party_data_test.go index 4c9cd7ad5e8..9ef9e310eff 100644 --- a/firstpartydata/first_party_data_test.go +++ b/firstpartydata/first_party_data_test.go @@ -751,7 +751,7 @@ func TestResolveUser(t *testing.T) { globalFPD map[string][]byte openRtbGlobalFPD map[string][]openrtb2.Data expectedUser *openrtb2.User - expectedError string + expectError bool }{ { description: "FPD config and bid request user are not specified", @@ -794,7 +794,7 @@ func TestResolveUser(t *testing.T) { fpdConfig: &openrtb_ext.ORTB2{User: json.RawMessage(`{"id": "test1"}`)}, bidRequestUser: &openrtb2.User{ID: "test2", Ext: json.RawMessage(`{"data":{"inputFPDUserData":"inputFPDUserDataValue"}}`)}, globalFPD: map[string][]byte{userKey: []byte(`malformed`)}, - expectedError: "Invalid JSON Patch", + expectError: true, }, { description: "bid request and openrtb global fpd user are specified, no input user ext", @@ -863,18 +863,18 @@ func TestResolveUser(t *testing.T) { }, Ext: json.RawMessage(`{"key":"value","test":1}`), }, - expectedError: "invalid character 'm' looking for beginning of object key string", + expectError: true, }, } for _, test := range testCases { t.Run(test.description, func(t *testing.T) { resultUser, err := resolveUser(test.fpdConfig, test.bidRequestUser, test.globalFPD, test.openRtbGlobalFPD, "bidderA") - if test.expectedError == "" { + if test.expectError { + assert.Error(t, err, "expected error incorrect") + } else { assert.NoError(t, err, "unexpected error returned") assert.Equal(t, test.expectedUser, resultUser, "Result user is incorrect") - } else { - assert.EqualError(t, err, test.expectedError, "expected error incorrect") } }) } @@ -888,16 +888,16 @@ func TestResolveSite(t *testing.T) { globalFPD map[string][]byte openRtbGlobalFPD map[string][]openrtb2.Data expectedSite *openrtb2.Site - expectedError string + expectError bool }{ { description: "FPD config and bid request site are not specified", expectedSite: nil, }, { - description: "FPD config site only is specified", - fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id": "test"}`)}, - expectedError: "incorrect First Party Data for bidder bidderA: Site object is not defined in request, but defined in FPD config", + description: "FPD config site only is specified", + fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id": "test"}`)}, + expectError: true, }, { description: "FPD config and bid request site are specified", @@ -931,7 +931,7 @@ func TestResolveSite(t *testing.T) { fpdConfig: &openrtb_ext.ORTB2{Site: json.RawMessage(`{"id": "test1"}`)}, bidRequestSite: &openrtb2.Site{ID: "test2", Ext: json.RawMessage(`{"data":{"inputFPDSiteData":"inputFPDSiteDataValue"}}`)}, globalFPD: map[string][]byte{siteKey: []byte(`malformed`)}, - expectedError: "Invalid JSON Patch", + expectError: true, }, { description: "bid request and openrtb global fpd site are specified, no input site ext", @@ -1023,18 +1023,18 @@ func TestResolveSite(t *testing.T) { }}, Ext: json.RawMessage(`{"key":"value","test":1}`), }, - expectedError: "invalid character 'm' looking for beginning of object key string", + expectError: true, }, } for _, test := range testCases { t.Run(test.description, func(t *testing.T) { resultSite, err := resolveSite(test.fpdConfig, test.bidRequestSite, test.globalFPD, test.openRtbGlobalFPD, "bidderA") - if test.expectedError == "" { + if test.expectError { + assert.Error(t, err) + } else { assert.NoError(t, err, "unexpected error returned") assert.Equal(t, test.expectedSite, resultSite, "Result site is incorrect") - } else { - assert.EqualError(t, err, test.expectedError, "expected error incorrect") } }) } @@ -1048,16 +1048,16 @@ func TestResolveApp(t *testing.T) { globalFPD map[string][]byte openRtbGlobalFPD map[string][]openrtb2.Data expectedApp *openrtb2.App - expectedError string + expectError bool }{ { description: "FPD config and bid request app are not specified", expectedApp: nil, }, { - description: "FPD config app only is specified", - fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id": "test"}`)}, - expectedError: "incorrect First Party Data for bidder bidderA: App object is not defined in request, but defined in FPD config", + description: "FPD config app only is specified", + fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id": "test"}`)}, + expectError: true, }, { description: "FPD config and bid request app are specified", @@ -1091,7 +1091,7 @@ func TestResolveApp(t *testing.T) { fpdConfig: &openrtb_ext.ORTB2{App: json.RawMessage(`{"id": "test1"}`)}, bidRequestApp: &openrtb2.App{ID: "test2", Ext: json.RawMessage(`{"data":{"inputFPDAppData":"inputFPDAppDataValue"}}`)}, globalFPD: map[string][]byte{appKey: []byte(`malformed`)}, - expectedError: "Invalid JSON Patch", + expectError: true, }, { description: "bid request and openrtb global fpd app are specified, no input app ext", @@ -1183,18 +1183,18 @@ func TestResolveApp(t *testing.T) { }}, Ext: json.RawMessage(`{"key":"value","test":1}`), }, - expectedError: "invalid character 'm' looking for beginning of object key string", + expectError: true, }, } for _, test := range testCases { t.Run(test.description, func(t *testing.T) { resultApp, err := resolveApp(test.fpdConfig, test.bidRequestApp, test.globalFPD, test.openRtbGlobalFPD, "bidderA") - if test.expectedError == "" { - assert.NoError(t, err, "unexpected error returned") - assert.Equal(t, test.expectedApp, resultApp, "Result app is incorrect") + if test.expectError { + assert.Error(t, err) } else { - assert.EqualError(t, err, test.expectedError, "expected error incorrect") + assert.NoError(t, err) + assert.Equal(t, test.expectedApp, resultApp, "Result app is incorrect") } }) } @@ -1245,7 +1245,7 @@ func TestMergeUser(t *testing.T) { givenUser openrtb2.User givenFPD json.RawMessage expectedUser openrtb2.User - expectedErr string + expectError bool }{ { name: "empty", @@ -1269,7 +1269,7 @@ func TestMergeUser(t *testing.T) { name: "toplevel-ext-err", givenUser: openrtb2.User{ID: "1", Ext: []byte(`malformed`)}, givenFPD: []byte(`{"id":"2"}`), - expectedErr: "invalid request ext", + expectError: true, }, { name: "nested-geo", @@ -1293,13 +1293,13 @@ func TestMergeUser(t *testing.T) { name: "nested-geo-ext-err", givenUser: openrtb2.User{Geo: &openrtb2.Geo{Ext: []byte(`malformed`)}}, givenFPD: []byte(`{"geo":{"ext":{"b":100,"c":3}}}`), - expectedErr: "invalid request ext", + expectError: true, }, { name: "fpd-err", givenUser: openrtb2.User{ID: "1", Ext: []byte(`{"a":1}`)}, givenFPD: []byte(`malformed`), - expectedErr: "invalid character 'm' looking for beginning of value", + expectError: true, }, } @@ -1307,11 +1307,11 @@ func TestMergeUser(t *testing.T) { t.Run(test.name, func(t *testing.T) { err := mergeUser(&test.givenUser, test.givenFPD) - if test.expectedErr == "" { - assert.NoError(t, err, "unexpected error returned") - assert.Equal(t, test.expectedUser, test.givenUser, "result user is incorrect") + if test.expectError { + assert.Error(t, err) } else { - assert.EqualError(t, err, test.expectedErr, "expected error incorrect") + assert.NoError(t, err) + assert.Equal(t, test.expectedUser, test.givenUser, "result user is incorrect") } }) } @@ -1323,7 +1323,7 @@ func TestMergeApp(t *testing.T) { givenApp openrtb2.App givenFPD json.RawMessage expectedApp openrtb2.App - expectedErr string + expectError bool }{ { name: "empty", @@ -1347,7 +1347,7 @@ func TestMergeApp(t *testing.T) { name: "toplevel-ext-err", givenApp: openrtb2.App{ID: "1", Ext: []byte(`malformed`)}, givenFPD: []byte(`{"id":"2"}`), - expectedErr: "invalid request ext", + expectError: true, }, { name: "nested-publisher", @@ -1443,37 +1443,37 @@ func TestMergeApp(t *testing.T) { name: "nested-publisher-ext-err", givenApp: openrtb2.App{Publisher: &openrtb2.Publisher{Ext: []byte(`malformed`)}}, givenFPD: []byte(`{"publisher":{"ext":{"b":100,"c":3}}}`), - expectedErr: "invalid request ext", + expectError: true, }, { name: "nested-content-ext-err", givenApp: openrtb2.App{Content: &openrtb2.Content{Ext: []byte(`malformed`)}}, givenFPD: []byte(`{"content":{"ext":{"b":100,"c":3}}}`), - expectedErr: "invalid request ext", + expectError: true, }, { name: "nested-content-producer-ext-err", givenApp: openrtb2.App{Content: &openrtb2.Content{Producer: &openrtb2.Producer{Ext: []byte(`malformed`)}}}, givenFPD: []byte(`{"content":{"producer": {"ext":{"b":100,"c":3}}}}`), - expectedErr: "invalid request ext", + expectError: true, }, { name: "nested-content-network-ext-err", givenApp: openrtb2.App{Content: &openrtb2.Content{Network: &openrtb2.Network{Ext: []byte(`malformed`)}}}, givenFPD: []byte(`{"content":{"network": {"ext":{"b":100,"c":3}}}}`), - expectedErr: "invalid request ext", + expectError: true, }, { name: "nested-content-channel-ext-err", givenApp: openrtb2.App{Content: &openrtb2.Content{Channel: &openrtb2.Channel{Ext: []byte(`malformed`)}}}, givenFPD: []byte(`{"content":{"channelx": {"ext":{"b":100,"c":3}}}}`), - expectedErr: "invalid request ext", + expectError: true, }, { name: "fpd-err", givenApp: openrtb2.App{ID: "1", Ext: []byte(`{"a":1}`)}, givenFPD: []byte(`malformed`), - expectedErr: "invalid character 'm' looking for beginning of value", + expectError: true, }, } @@ -1481,11 +1481,11 @@ func TestMergeApp(t *testing.T) { t.Run(test.name, func(t *testing.T) { err := mergeApp(&test.givenApp, test.givenFPD) - if test.expectedErr == "" { - assert.NoError(t, err, "unexpected error returned") - assert.Equal(t, test.expectedApp, test.givenApp, " result app is incorrect") + if test.expectError { + assert.Error(t, err) } else { - assert.EqualError(t, err, test.expectedErr, "expected error incorrect") + assert.NoError(t, err) + assert.Equal(t, test.expectedApp, test.givenApp, " result app is incorrect") } }) } @@ -1497,13 +1497,13 @@ func TestMergeSite(t *testing.T) { givenSite openrtb2.Site givenFPD json.RawMessage expectedSite openrtb2.Site - expectedErr string + expectError bool }{ { name: "empty", givenSite: openrtb2.Site{}, givenFPD: []byte(`{}`), - expectedErr: "incorrect First Party Data for bidder BidderA: Site object cannot set empty page if req.site.id is empty", + expectError: true, }, { name: "toplevel", @@ -1521,7 +1521,7 @@ func TestMergeSite(t *testing.T) { name: "toplevel-ext-err", givenSite: openrtb2.Site{ID: "1", Ext: []byte(`malformed`)}, givenFPD: []byte(`{"id":"2"}`), - expectedErr: "invalid request ext", + expectError: true, }, { name: "nested-publisher", @@ -1617,37 +1617,37 @@ func TestMergeSite(t *testing.T) { name: "nested-publisher-ext-err", givenSite: openrtb2.Site{ID: "1", Publisher: &openrtb2.Publisher{Ext: []byte(`malformed`)}}, givenFPD: []byte(`{"publisher":{"ext":{"b":100,"c":3}}}`), - expectedErr: "invalid request ext", + expectError: true, }, { name: "nested-content-ext-err", givenSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Ext: []byte(`malformed`)}}, givenFPD: []byte(`{"content":{"ext":{"b":100,"c":3}}}`), - expectedErr: "invalid request ext", + expectError: true, }, { name: "nested-content-producer-ext-err", givenSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Producer: &openrtb2.Producer{Ext: []byte(`malformed`)}}}, givenFPD: []byte(`{"content":{"producer": {"ext":{"b":100,"c":3}}}}`), - expectedErr: "invalid request ext", + expectError: true, }, { name: "nested-content-network-ext-err", givenSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Network: &openrtb2.Network{Ext: []byte(`malformed`)}}}, givenFPD: []byte(`{"content":{"network": {"ext":{"b":100,"c":3}}}}`), - expectedErr: "invalid request ext", + expectError: true, }, { name: "nested-content-channel-ext-err", givenSite: openrtb2.Site{ID: "1", Content: &openrtb2.Content{Channel: &openrtb2.Channel{Ext: []byte(`malformed`)}}}, givenFPD: []byte(`{"content":{"channelx": {"ext":{"b":100,"c":3}}}}`), - expectedErr: "invalid request ext", + expectError: true, }, { name: "fpd-err", givenSite: openrtb2.Site{ID: "1", Ext: []byte(`{"a":1}`)}, givenFPD: []byte(`malformed`), - expectedErr: "invalid character 'm' looking for beginning of value", + expectError: true, }, } @@ -1655,11 +1655,11 @@ func TestMergeSite(t *testing.T) { t.Run(test.name, func(t *testing.T) { err := mergeSite(&test.givenSite, test.givenFPD, "BidderA") - if test.expectedErr == "" { - assert.NoError(t, err, "unexpected error returned") - assert.Equal(t, test.expectedSite, test.givenSite, " result Site is incorrect") + if test.expectError { + assert.Error(t, err) } else { - assert.EqualError(t, err, test.expectedErr, "expected error incorrect") + assert.NoError(t, err) + assert.Equal(t, test.expectedSite, test.givenSite, " result Site is incorrect") } }) } diff --git a/floors/rule_test.go b/floors/rule_test.go index 27be6ef0dd2..4de06dc80a2 100644 --- a/floors/rule_test.go +++ b/floors/rule_test.go @@ -760,7 +760,7 @@ func TestGetMinFloorValue(t *testing.T) { }, want: 0.0, want1: "", - wantErr: errors.New("Error in getting FloorMin value : 'unexpected end of JSON input'"), + wantErr: errors.New("Error in getting FloorMin value : 'ReadString: expects \" or n, but found \x00, error found in #1 byte of ...|{|..., bigger context ...|{|...'"), }, } for _, tc := range testCases { diff --git a/go.mod b/go.mod index 6731c177ecf..e52946fbe1e 100644 --- a/go.mod +++ b/go.mod @@ -46,10 +46,13 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index 1f76be7af31..eefe0228261 100644 --- a/go.sum +++ b/go.sum @@ -296,6 +296,7 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -356,9 +357,11 @@ github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= diff --git a/modules/prebid/ortb2blocking/config.go b/modules/prebid/ortb2blocking/config.go index cefd436b402..31bc9196c13 100644 --- a/modules/prebid/ortb2blocking/config.go +++ b/modules/prebid/ortb2blocking/config.go @@ -5,11 +5,12 @@ import ( "fmt" "github.com/prebid/openrtb/v19/adcom1" + "github.com/prebid/prebid-server/util/jsonutil" ) func newConfig(data json.RawMessage) (config, error) { var cfg config - if err := json.Unmarshal(data, &cfg); err != nil { + if err := jsonutil.UnmarshalValid(data, &cfg); err != nil { return cfg, fmt.Errorf("failed to parse config: %s", err) } return cfg, nil diff --git a/modules/prebid/ortb2blocking/module_test.go b/modules/prebid/ortb2blocking/module_test.go index b47f9f58a02..23a8fc35bdc 100644 --- a/modules/prebid/ortb2blocking/module_test.go +++ b/modules/prebid/ortb2blocking/module_test.go @@ -488,7 +488,7 @@ func TestHandleBidderRequestHook(t *testing.T) { bidRequest: &openrtb2.BidRequest{}, expectedBidRequest: &openrtb2.BidRequest{}, expectedHookResult: hookstage.HookResult[hookstage.BidderRequestPayload]{}, - expectedError: errors.New("failed to parse config: invalid character '.' looking for beginning of value"), + expectedError: errors.New("failed to parse config: readObjectStart: expect { or n, but found ., error found in #1 byte of ...|...|..., bigger context ...|...|..."), }, { description: "Expect error if nil BidRequest provided", diff --git a/openrtb_ext/convert_down_test.go b/openrtb_ext/convert_down_test.go index ea21468e737..3b78337df8b 100644 --- a/openrtb_ext/convert_down_test.go +++ b/openrtb_ext/convert_down_test.go @@ -6,6 +6,7 @@ import ( "github.com/prebid/openrtb/v19/adcom1" "github.com/prebid/openrtb/v19/openrtb2" + "github.com/prebid/prebid-server/errortypes" "github.com/stretchr/testify/assert" ) @@ -14,7 +15,7 @@ func TestConvertDownTo25(t *testing.T) { name string givenRequest openrtb2.BidRequest expectedRequest openrtb2.BidRequest - expectedErr string + expectedErrType error }{ { name: "2.6-to-2.5", @@ -87,12 +88,12 @@ func TestConvertDownTo25(t *testing.T) { }, }, { - name: "malformed-shhain", + name: "malformed-schain", givenRequest: openrtb2.BidRequest{ ID: "anyID", Source: &openrtb2.Source{SChain: &openrtb2.SupplyChain{Complete: 1, Nodes: []openrtb2.SupplyChainNode{}, Ver: "2"}, Ext: json.RawMessage(`malformed`)}, }, - expectedErr: "invalid character 'm' looking for beginning of value", + expectedErrType: &errortypes.FailedToUnmarshal{}, }, { name: "malformed-gdpr", @@ -100,7 +101,7 @@ func TestConvertDownTo25(t *testing.T) { ID: "anyID", Regs: &openrtb2.Regs{GDPR: openrtb2.Int8Ptr(1), Ext: json.RawMessage(`malformed`)}, }, - expectedErr: "invalid character 'm' looking for beginning of value", + expectedErrType: &errortypes.FailedToUnmarshal{}, }, { name: "malformed-consent", @@ -108,7 +109,7 @@ func TestConvertDownTo25(t *testing.T) { ID: "anyID", User: &openrtb2.User{Consent: "1", Ext: json.RawMessage(`malformed`)}, }, - expectedErr: "invalid character 'm' looking for beginning of value", + expectedErrType: &errortypes.FailedToUnmarshal{}, }, { name: "malformed-usprivacy", @@ -116,7 +117,7 @@ func TestConvertDownTo25(t *testing.T) { ID: "anyID", Regs: &openrtb2.Regs{USPrivacy: "3", Ext: json.RawMessage(`malformed`)}, }, - expectedErr: "invalid character 'm' looking for beginning of value", + expectedErrType: &errortypes.FailedToUnmarshal{}, }, { name: "malformed-eid", @@ -124,7 +125,7 @@ func TestConvertDownTo25(t *testing.T) { ID: "anyID", User: &openrtb2.User{EIDs: []openrtb2.EID{{Source: "42"}}, Ext: json.RawMessage(`malformed`)}, }, - expectedErr: "invalid character 'm' looking for beginning of value", + expectedErrType: &errortypes.FailedToUnmarshal{}, }, { name: "malformed-imp", @@ -132,7 +133,7 @@ func TestConvertDownTo25(t *testing.T) { ID: "anyID", Imp: []openrtb2.Imp{{Rwdd: 1, Ext: json.RawMessage(`malformed`)}}, }, - expectedErr: "invalid character 'm' looking for beginning of value", + expectedErrType: &errortypes.FailedToUnmarshal{}, }, } @@ -141,8 +142,8 @@ func TestConvertDownTo25(t *testing.T) { w := &RequestWrapper{BidRequest: &test.givenRequest} err := ConvertDownTo25(w) - if len(test.expectedErr) > 0 { - assert.EqualError(t, err, test.expectedErr, "error") + if test.expectedErrType != nil { + assert.IsType(t, test.expectedErrType, err) } else { assert.NoError(t, w.RebuildRequest(), "error") assert.Equal(t, test.expectedRequest, *w.BidRequest, "result") @@ -162,7 +163,7 @@ func TestMoveSupplyChainFrom26To25(t *testing.T) { name string givenRequest openrtb2.BidRequest expectedRequest openrtb2.BidRequest - expectedErr string + expectedErrType error }{ { name: "notpresent-source", @@ -185,9 +186,9 @@ func TestMoveSupplyChainFrom26To25(t *testing.T) { expectedRequest: openrtb2.BidRequest{Source: &openrtb2.Source{Ext: schain1Json}}, }, { - name: "malformed", - givenRequest: openrtb2.BidRequest{Source: &openrtb2.Source{SChain: schain1, Ext: json.RawMessage(`malformed`)}}, - expectedErr: "invalid character 'm' looking for beginning of value", + name: "malformed", + givenRequest: openrtb2.BidRequest{Source: &openrtb2.Source{SChain: schain1, Ext: json.RawMessage(`malformed`)}}, + expectedErrType: &errortypes.FailedToUnmarshal{}, }, } @@ -196,8 +197,8 @@ func TestMoveSupplyChainFrom26To25(t *testing.T) { w := &RequestWrapper{BidRequest: &test.givenRequest} err := moveSupplyChainFrom26To25(w) - if len(test.expectedErr) > 0 { - assert.EqualError(t, err, test.expectedErr, "error") + if test.expectedErrType != nil { + assert.IsType(t, test.expectedErrType, err) } else { assert.NoError(t, w.RebuildRequest(), "error") assert.Equal(t, test.expectedRequest, *w.BidRequest, "result") @@ -211,7 +212,7 @@ func TestMoveGDPRFrom26To25(t *testing.T) { name string givenRequest openrtb2.BidRequest expectedRequest openrtb2.BidRequest - expectedErr string + expectedErrType error }{ { name: "notpresent-regs", @@ -234,9 +235,9 @@ func TestMoveGDPRFrom26To25(t *testing.T) { expectedRequest: openrtb2.BidRequest{Regs: &openrtb2.Regs{Ext: json.RawMessage(`{"gdpr":0}`)}}, }, { - name: "malformed", - givenRequest: openrtb2.BidRequest{Regs: &openrtb2.Regs{GDPR: openrtb2.Int8Ptr(0), Ext: json.RawMessage(`malformed`)}}, - expectedErr: "invalid character 'm' looking for beginning of value", + name: "malformed", + givenRequest: openrtb2.BidRequest{Regs: &openrtb2.Regs{GDPR: openrtb2.Int8Ptr(0), Ext: json.RawMessage(`malformed`)}}, + expectedErrType: &errortypes.FailedToUnmarshal{}, }, } @@ -245,8 +246,8 @@ func TestMoveGDPRFrom26To25(t *testing.T) { w := &RequestWrapper{BidRequest: &test.givenRequest} err := moveGDPRFrom26To25(w) - if len(test.expectedErr) > 0 { - assert.EqualError(t, err, test.expectedErr, "error") + if test.expectedErrType != nil { + assert.IsType(t, test.expectedErrType, err) } else { assert.NoError(t, w.RebuildRequest(), "error") assert.Equal(t, test.expectedRequest, *w.BidRequest, "result") @@ -260,7 +261,7 @@ func TestMoveConsentFrom26To25(t *testing.T) { name string givenRequest openrtb2.BidRequest expectedRequest openrtb2.BidRequest - expectedErr string + expectedErrType error }{ { name: "notpresent-user", @@ -283,9 +284,9 @@ func TestMoveConsentFrom26To25(t *testing.T) { expectedRequest: openrtb2.BidRequest{User: &openrtb2.User{Ext: json.RawMessage(`{"consent":"1"}`)}}, }, { - name: "malformed", - givenRequest: openrtb2.BidRequest{User: &openrtb2.User{Consent: "1", Ext: json.RawMessage(`malformed`)}}, - expectedErr: "invalid character 'm' looking for beginning of value", + name: "malformed", + givenRequest: openrtb2.BidRequest{User: &openrtb2.User{Consent: "1", Ext: json.RawMessage(`malformed`)}}, + expectedErrType: &errortypes.FailedToUnmarshal{}, }, } @@ -294,8 +295,8 @@ func TestMoveConsentFrom26To25(t *testing.T) { w := &RequestWrapper{BidRequest: &test.givenRequest} err := moveConsentFrom26To25(w) - if len(test.expectedErr) > 0 { - assert.EqualError(t, err, test.expectedErr, "error") + if test.expectedErrType != nil { + assert.IsType(t, test.expectedErrType, err) } else { assert.NoError(t, w.RebuildRequest(), "error") assert.Equal(t, test.expectedRequest, *w.BidRequest, "result") @@ -309,7 +310,7 @@ func TestMoveUSPrivacyFrom26To25(t *testing.T) { name string givenRequest openrtb2.BidRequest expectedRequest openrtb2.BidRequest - expectedErr string + expectedErrType error }{ { name: "notpresent-regs", @@ -332,9 +333,9 @@ func TestMoveUSPrivacyFrom26To25(t *testing.T) { expectedRequest: openrtb2.BidRequest{Regs: &openrtb2.Regs{Ext: json.RawMessage(`{"us_privacy":"1"}`)}}, }, { - name: "malformed", - givenRequest: openrtb2.BidRequest{Regs: &openrtb2.Regs{USPrivacy: "1", Ext: json.RawMessage(`malformed`)}}, - expectedErr: "invalid character 'm' looking for beginning of value", + name: "malformed", + givenRequest: openrtb2.BidRequest{Regs: &openrtb2.Regs{USPrivacy: "1", Ext: json.RawMessage(`malformed`)}}, + expectedErrType: &errortypes.FailedToUnmarshal{}, }, } @@ -343,8 +344,8 @@ func TestMoveUSPrivacyFrom26To25(t *testing.T) { w := &RequestWrapper{BidRequest: &test.givenRequest} err := moveUSPrivacyFrom26To25(w) - if len(test.expectedErr) > 0 { - assert.EqualError(t, err, test.expectedErr, "error") + if test.expectedErrType != nil { + assert.IsType(t, test.expectedErrType, err) } else { assert.NoError(t, w.RebuildRequest(), "error") assert.Equal(t, test.expectedRequest, *w.BidRequest, "result") @@ -364,7 +365,7 @@ func TestMoveEIDFrom26To25(t *testing.T) { name string givenRequest openrtb2.BidRequest expectedRequest openrtb2.BidRequest - expectedErr string + expectedErrType error }{ { name: "notpresent-user", @@ -392,9 +393,9 @@ func TestMoveEIDFrom26To25(t *testing.T) { expectedRequest: openrtb2.BidRequest{User: &openrtb2.User{Ext: eid1Json}}, }, { - name: "malformed", - givenRequest: openrtb2.BidRequest{User: &openrtb2.User{EIDs: eid1, Ext: json.RawMessage(`malformed`)}}, - expectedErr: "invalid character 'm' looking for beginning of value", + name: "malformed", + givenRequest: openrtb2.BidRequest{User: &openrtb2.User{EIDs: eid1, Ext: json.RawMessage(`malformed`)}}, + expectedErrType: &errortypes.FailedToUnmarshal{}, }, } @@ -403,8 +404,8 @@ func TestMoveEIDFrom26To25(t *testing.T) { w := &RequestWrapper{BidRequest: &test.givenRequest} err := moveEIDFrom26To25(w) - if len(test.expectedErr) > 0 { - assert.EqualError(t, err, test.expectedErr, "error") + if test.expectedErrType != nil { + assert.IsType(t, test.expectedErrType, err) } else { assert.NoError(t, w.RebuildRequest(), "error") assert.Equal(t, test.expectedRequest, *w.BidRequest, "result") @@ -415,10 +416,10 @@ func TestMoveEIDFrom26To25(t *testing.T) { func TestMoveRewardedFrom26ToPrebidExt(t *testing.T) { testCases := []struct { - name string - givenImp openrtb2.Imp - expectedImp openrtb2.Imp - expectedErr string + name string + givenImp openrtb2.Imp + expectedImp openrtb2.Imp + expectedErrType error }{ { name: "notpresent-prebid", @@ -436,9 +437,9 @@ func TestMoveRewardedFrom26ToPrebidExt(t *testing.T) { expectedImp: openrtb2.Imp{Ext: json.RawMessage(`{"prebid":{"is_rewarded_inventory":1}}`)}, }, { - name: "Malformed", - givenImp: openrtb2.Imp{Rwdd: 1, Ext: json.RawMessage(`malformed`)}, - expectedErr: "invalid character 'm' looking for beginning of value", + name: "Malformed", + givenImp: openrtb2.Imp{Rwdd: 1, Ext: json.RawMessage(`malformed`)}, + expectedErrType: &errortypes.FailedToUnmarshal{}, }, } @@ -447,8 +448,8 @@ func TestMoveRewardedFrom26ToPrebidExt(t *testing.T) { w := &ImpWrapper{Imp: &test.givenImp} err := moveRewardedFrom26ToPrebidExt(w) - if len(test.expectedErr) > 0 { - assert.EqualError(t, err, test.expectedErr, "error") + if test.expectedErrType != nil { + assert.IsType(t, test.expectedErrType, err) } else { assert.NoError(t, w.RebuildImp(), "error") assert.Equal(t, test.expectedImp, *w.Imp, "result") diff --git a/openrtb_ext/convert_up_test.go b/openrtb_ext/convert_up_test.go index d3ba034d07e..78cd675d89e 100644 --- a/openrtb_ext/convert_up_test.go +++ b/openrtb_ext/convert_up_test.go @@ -20,7 +20,7 @@ func TestConvertUpTo26(t *testing.T) { givenRequest: openrtb2.BidRequest{ Ext: json.RawMessage(`malformed`), }, - expectedErr: "req.ext is invalid: invalid character 'm' looking for beginning of value", + expectedErr: "req.ext is invalid: ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...", }, { description: "2.4 -> 2.6", @@ -120,27 +120,27 @@ func TestConvertUpEnsureExt(t *testing.T) { { description: "Ext", givenRequest: openrtb2.BidRequest{Ext: json.RawMessage("malformed")}, - expectedErr: "req.ext is invalid: invalid character 'm' looking for beginning of value", + expectedErr: "req.ext is invalid: ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...", }, { description: "Source.Ext", givenRequest: openrtb2.BidRequest{Source: &openrtb2.Source{Ext: json.RawMessage("malformed")}}, - expectedErr: "req.source.ext is invalid: invalid character 'm' looking for beginning of value", + expectedErr: "req.source.ext is invalid: ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...", }, { description: "Regs.Ext", givenRequest: openrtb2.BidRequest{Regs: &openrtb2.Regs{Ext: json.RawMessage("malformed")}}, - expectedErr: "req.regs.ext is invalid: invalid character 'm' looking for beginning of value", + expectedErr: "req.regs.ext is invalid: ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...", }, { description: "User.Ext", givenRequest: openrtb2.BidRequest{User: &openrtb2.User{Ext: json.RawMessage("malformed")}}, - expectedErr: "req.user.ext is invalid: invalid character 'm' looking for beginning of value", + expectedErr: "req.user.ext is invalid: ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...", }, { description: "Imp.Ext", givenRequest: openrtb2.BidRequest{Imp: []openrtb2.Imp{{Ext: json.RawMessage("malformed")}}}, - expectedErr: "imp[0].imp.ext is invalid: invalid character 'm' looking for beginning of value", + expectedErr: "imp[0].imp.ext is invalid: ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...", }, } diff --git a/openrtb_ext/deal_tier.go b/openrtb_ext/deal_tier.go index 45285d21663..df386916b77 100644 --- a/openrtb_ext/deal_tier.go +++ b/openrtb_ext/deal_tier.go @@ -1,9 +1,8 @@ package openrtb_ext import ( - "encoding/json" - "github.com/prebid/openrtb/v19/openrtb2" + "github.com/prebid/prebid-server/util/jsonutil" ) // DealTier defines the configuration of a deal tier. @@ -34,7 +33,7 @@ func ReadDealTiersFromImp(imp openrtb2.Imp) (DealTierBidderMap, error) { } `json:"bidder"` } `json:"prebid"` } - if err := json.Unmarshal(imp.Ext, &impPrebidExt); err != nil { + if err := jsonutil.Unmarshal(imp.Ext, &impPrebidExt); err != nil { return nil, err } for bidder, param := range impPrebidExt.Prebid.Bidders { diff --git a/openrtb_ext/deal_tier_test.go b/openrtb_ext/deal_tier_test.go index 0046b788ece..dabecf6a9e7 100644 --- a/openrtb_ext/deal_tier_test.go +++ b/openrtb_ext/deal_tier_test.go @@ -5,15 +5,16 @@ import ( "testing" "github.com/prebid/openrtb/v19/openrtb2" + "github.com/prebid/prebid-server/errortypes" "github.com/stretchr/testify/assert" ) func TestReadDealTiersFromImp(t *testing.T) { testCases := []struct { - description string - impExt json.RawMessage - expectedResult DealTierBidderMap - expectedError string + description string + impExt json.RawMessage + expectedResult DealTierBidderMap + expectedErrorType error }{ { description: "Nil", @@ -71,9 +72,9 @@ func TestReadDealTiersFromImp(t *testing.T) { expectedResult: DealTierBidderMap{}, }, { - description: "imp.ext.prebid.bidder - error", - impExt: json.RawMessage(`{"prebid": {"bidder": {"appnexus": {"dealTier": "wrong type", "placementId": 12345}}}}`), - expectedError: "json: cannot unmarshal string into Go struct field .prebid.bidder.dealTier of type openrtb_ext.DealTier", + description: "imp.ext.prebid.bidder - error", + impExt: json.RawMessage(`{"prebid": {"bidder": {"appnexus": {"dealTier": "wrong type", "placementId": 12345}}}}`), + expectedErrorType: &errortypes.FailedToUnmarshal{}, }, } @@ -84,10 +85,10 @@ func TestReadDealTiersFromImp(t *testing.T) { assert.Equal(t, test.expectedResult, result, test.description+":result") - if len(test.expectedError) == 0 { - assert.NoError(t, err, test.description+":error") + if test.expectedErrorType != nil { + assert.IsType(t, test.expectedErrorType, err) } else { - assert.EqualError(t, err, test.expectedError, test.description+":error") + assert.NoError(t, err, test.description+":error") } } } diff --git a/openrtb_ext/request.go b/openrtb_ext/request.go index f5418ba4d1f..3a41fdf0dbb 100644 --- a/openrtb_ext/request.go +++ b/openrtb_ext/request.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/prebid/openrtb/v19/openrtb2" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/util/maputil" "github.com/prebid/prebid-server/util/ptrutil" "github.com/prebid/prebid-server/util/sliceutil" @@ -234,7 +235,7 @@ func (pg *PriceGranularity) UnmarshalJSON(b []byte) error { // price granularity used to be a string referencing a predefined value, try to parse // and map the legacy string before falling back to the modern custom model. legacyID := "" - if err := json.Unmarshal(b, &legacyID); err == nil { + if err := jsonutil.Unmarshal(b, &legacyID); err == nil { if legacyValue, ok := NewPriceGranularityFromLegacyID(legacyID); ok { *pg = legacyValue return nil @@ -243,7 +244,7 @@ func (pg *PriceGranularity) UnmarshalJSON(b []byte) error { // use a type-alias to avoid calling back into this UnmarshalJSON implementation modernValue := PriceGranularityRaw{} - err := json.Unmarshal(b, &modernValue) + err := jsonutil.Unmarshal(b, &modernValue) if err == nil { *pg = (PriceGranularity)(modernValue) } diff --git a/openrtb_ext/request_wrapper.go b/openrtb_ext/request_wrapper.go index f4ef69b0523..1f93d4c0d0e 100644 --- a/openrtb_ext/request_wrapper.go +++ b/openrtb_ext/request_wrapper.go @@ -5,6 +5,7 @@ import ( "errors" "github.com/prebid/openrtb/v19/openrtb2" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/util/maputil" "github.com/prebid/prebid-server/util/ptrutil" "github.com/prebid/prebid-server/util/sliceutil" @@ -440,13 +441,13 @@ func (ue *UserExt) unmarshal(extJson json.RawMessage) error { return nil } - if err := json.Unmarshal(extJson, &ue.ext); err != nil { + if err := jsonutil.Unmarshal(extJson, &ue.ext); err != nil { return err } consentJson, hasConsent := ue.ext[consentKey] if hasConsent { - if err := json.Unmarshal(consentJson, &ue.consent); err != nil { + if err := jsonutil.Unmarshal(consentJson, &ue.consent); err != nil { return err } } @@ -454,7 +455,7 @@ func (ue *UserExt) unmarshal(extJson json.RawMessage) error { prebidJson, hasPrebid := ue.ext[prebidKey] if hasPrebid { ue.prebid = &ExtUserPrebid{} - if err := json.Unmarshal(prebidJson, ue.prebid); err != nil { + if err := jsonutil.Unmarshal(prebidJson, ue.prebid); err != nil { return err } } @@ -462,21 +463,21 @@ func (ue *UserExt) unmarshal(extJson json.RawMessage) error { eidsJson, hasEids := ue.ext[eidsKey] if hasEids { ue.eids = &[]openrtb2.EID{} - if err := json.Unmarshal(eidsJson, ue.eids); err != nil { + if err := jsonutil.Unmarshal(eidsJson, ue.eids); err != nil { return err } } if consentedProviderSettingsJson, hasCPSettings := ue.ext[consentedProvidersSettingsStringKey]; hasCPSettings { ue.consentedProvidersSettingsIn = &ConsentedProvidersSettingsIn{} - if err := json.Unmarshal(consentedProviderSettingsJson, ue.consentedProvidersSettingsIn); err != nil { + if err := jsonutil.Unmarshal(consentedProviderSettingsJson, ue.consentedProvidersSettingsIn); err != nil { return err } } if consentedProviderSettingsJson, hasCPSettings := ue.ext[consentedProvidersSettingsListKey]; hasCPSettings { ue.consentedProvidersSettingsOut = &ConsentedProvidersSettingsOut{} - if err := json.Unmarshal(consentedProviderSettingsJson, ue.consentedProvidersSettingsOut); err != nil { + if err := jsonutil.Unmarshal(consentedProviderSettingsJson, ue.consentedProvidersSettingsOut); err != nil { return err } } @@ -726,14 +727,14 @@ func (re *RequestExt) unmarshal(extJson json.RawMessage) error { return nil } - if err := json.Unmarshal(extJson, &re.ext); err != nil { + if err := jsonutil.Unmarshal(extJson, &re.ext); err != nil { return err } prebidJson, hasPrebid := re.ext[prebidKey] if hasPrebid { re.prebid = &ExtRequestPrebid{} - if err := json.Unmarshal(prebidJson, re.prebid); err != nil { + if err := jsonutil.Unmarshal(prebidJson, re.prebid); err != nil { return err } } @@ -741,7 +742,7 @@ func (re *RequestExt) unmarshal(extJson json.RawMessage) error { schainJson, hasSChain := re.ext[schainKey] if hasSChain { re.schain = &openrtb2.SupplyChain{} - if err := json.Unmarshal(schainJson, re.schain); err != nil { + if err := jsonutil.Unmarshal(schainJson, re.schain); err != nil { return err } } @@ -880,14 +881,14 @@ func (de *DeviceExt) unmarshal(extJson json.RawMessage) error { return nil } - if err := json.Unmarshal(extJson, &de.ext); err != nil { + if err := jsonutil.Unmarshal(extJson, &de.ext); err != nil { return err } prebidJson, hasPrebid := de.ext[prebidKey] if hasPrebid { de.prebid = &ExtDevicePrebid{} - if err := json.Unmarshal(prebidJson, de.prebid); err != nil { + if err := jsonutil.Unmarshal(prebidJson, de.prebid); err != nil { return err } } @@ -992,14 +993,14 @@ func (ae *AppExt) unmarshal(extJson json.RawMessage) error { return nil } - if err := json.Unmarshal(extJson, &ae.ext); err != nil { + if err := jsonutil.Unmarshal(extJson, &ae.ext); err != nil { return err } prebidJson, hasPrebid := ae.ext[prebidKey] if hasPrebid { ae.prebid = &ExtAppPrebid{} - if err := json.Unmarshal(prebidJson, ae.prebid); err != nil { + if err := jsonutil.Unmarshal(prebidJson, ae.prebid); err != nil { return err } } @@ -1096,7 +1097,7 @@ func (de *DOOHExt) unmarshal(extJson json.RawMessage) error { return nil } - if err := json.Unmarshal(extJson, &de.ext); err != nil { + if err := jsonutil.Unmarshal(extJson, &de.ext); err != nil { return err } @@ -1163,20 +1164,20 @@ func (re *RegExt) unmarshal(extJson json.RawMessage) error { return nil } - if err := json.Unmarshal(extJson, &re.ext); err != nil { + if err := jsonutil.Unmarshal(extJson, &re.ext); err != nil { return err } gdprJson, hasGDPR := re.ext[gdprKey] - if hasGDPR { - if err := json.Unmarshal(gdprJson, &re.gdpr); err != nil { + if hasGDPR && gdprJson != nil { + if err := jsonutil.Unmarshal(gdprJson, &re.gdpr); err != nil { return errors.New("gdpr must be an integer") } } uspJson, hasUsp := re.ext[us_privacyKey] if hasUsp { - if err := json.Unmarshal(uspJson, &re.usPrivacy); err != nil { + if err := jsonutil.Unmarshal(uspJson, &re.usPrivacy); err != nil { return err } } @@ -1290,13 +1291,13 @@ func (se *SiteExt) unmarshal(extJson json.RawMessage) error { return nil } - if err := json.Unmarshal(extJson, &se.ext); err != nil { + if err := jsonutil.Unmarshal(extJson, &se.ext); err != nil { return err } ampJson, hasAmp := se.ext[ampKey] if hasAmp { - if err := json.Unmarshal(ampJson, &se.amp); err != nil { + if err := jsonutil.Unmarshal(ampJson, &se.amp); err != nil { return errors.New(`request.site.ext.amp must be either 1, 0, or undefined`) } } @@ -1385,13 +1386,13 @@ func (se *SourceExt) unmarshal(extJson json.RawMessage) error { return nil } - if err := json.Unmarshal(extJson, &se.ext); err != nil { + if err := jsonutil.Unmarshal(extJson, &se.ext); err != nil { return err } schainJson, hasSChain := se.ext[schainKey] if hasSChain { - if err := json.Unmarshal(schainJson, &se.schain); err != nil { + if err := jsonutil.Unmarshal(schainJson, &se.schain); err != nil { return err } } @@ -1550,14 +1551,14 @@ func (e *ImpExt) unmarshal(extJson json.RawMessage) error { return nil } - if err := json.Unmarshal(extJson, &e.ext); err != nil { + if err := jsonutil.Unmarshal(extJson, &e.ext); err != nil { return err } prebidJson, hasPrebid := e.ext[prebidKey] if hasPrebid { e.prebid = &ExtImpPrebid{} - if err := json.Unmarshal(prebidJson, e.prebid); err != nil { + if err := jsonutil.Unmarshal(prebidJson, e.prebid); err != nil { return err } } @@ -1565,21 +1566,21 @@ func (e *ImpExt) unmarshal(extJson json.RawMessage) error { dataJson, hasData := e.ext[dataKey] if hasData { e.data = &ExtImpData{} - if err := json.Unmarshal(dataJson, e.data); err != nil { + if err := jsonutil.Unmarshal(dataJson, e.data); err != nil { return err } } tidJson, hasTid := e.ext["tid"] if hasTid { - if err := json.Unmarshal(tidJson, &e.tid); err != nil { + if err := jsonutil.Unmarshal(tidJson, &e.tid); err != nil { return err } } gpIdJson, hasGpId := e.ext["gpid"] if hasGpId { - if err := json.Unmarshal(gpIdJson, &e.gpId); err != nil { + if err := jsonutil.Unmarshal(gpIdJson, &e.gpId); err != nil { return err } } diff --git a/openrtb_ext/request_wrapper_test.go b/openrtb_ext/request_wrapper_test.go index afa047e6909..0127c545274 100644 --- a/openrtb_ext/request_wrapper_test.go +++ b/openrtb_ext/request_wrapper_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/prebid/openrtb/v19/openrtb2" + "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/util/ptrutil" "github.com/stretchr/testify/assert" ) @@ -1792,10 +1793,10 @@ func TestImpWrapperGetImpExt(t *testing.T) { var isRewardedInventoryOne int8 = 1 testCases := []struct { - description string - givenWrapper ImpWrapper - expectedImpExt ImpExt - expectedError string + description string + givenWrapper ImpWrapper + expectedImpExt ImpExt + expectedErrorType error }{ { description: "Empty", @@ -1833,21 +1834,21 @@ func TestImpWrapperGetImpExt(t *testing.T) { expectedImpExt: ImpExt{ext: map[string]json.RawMessage{"foo": json.RawMessage("bar")}}, }, { - description: "Error - Ext", - givenWrapper: ImpWrapper{Imp: &openrtb2.Imp{Ext: json.RawMessage(`malformed`)}}, - expectedError: "invalid character 'm' looking for beginning of value", + description: "Error - Ext", + givenWrapper: ImpWrapper{Imp: &openrtb2.Imp{Ext: json.RawMessage(`malformed`)}}, + expectedErrorType: &errortypes.FailedToUnmarshal{}, }, { - description: "Error - Ext - Prebid", - givenWrapper: ImpWrapper{Imp: &openrtb2.Imp{Ext: json.RawMessage(`{"prebid":malformed}`)}}, - expectedError: "invalid character 'm' looking for beginning of value", + description: "Error - Ext - Prebid", + givenWrapper: ImpWrapper{Imp: &openrtb2.Imp{Ext: json.RawMessage(`{"prebid":malformed}`)}}, + expectedErrorType: &errortypes.FailedToUnmarshal{}, }, } for _, test := range testCases { impExt, err := test.givenWrapper.GetImpExt() - if test.expectedError != "" { - assert.EqualError(t, err, test.expectedError, test.description) + if test.expectedErrorType != nil { + assert.IsType(t, test.expectedErrorType, err) } else { assert.NoError(t, err, test.description) assert.Equal(t, test.expectedImpExt, *impExt, test.description) diff --git a/ortb/default_test.go b/ortb/default_test.go index 04eeeebdcb6..330434fba27 100644 --- a/ortb/default_test.go +++ b/ortb/default_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/util/ptrutil" ) @@ -31,7 +32,7 @@ func TestSetDefaults(t *testing.T) { name: "malformed request.ext", givenRequest: openrtb2.BidRequest{Ext: json.RawMessage(`malformed`)}, expectedRequest: openrtb2.BidRequest{Ext: json.RawMessage(`malformed`)}, - expectedErr: "invalid character 'm' looking for beginning of value", + expectedErr: "ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...", }, { name: "targeting", // tests integration with setDefaultsTargeting @@ -55,6 +56,7 @@ func TestSetDefaults(t *testing.T) { // assert error if len(test.expectedErr) > 0 { assert.EqualError(t, err, test.expectedErr, "Error") + assert.IsType(t, &errortypes.FailedToUnmarshal{}, err) } // rebuild request diff --git a/privacy/scrubber.go b/privacy/scrubber.go index 59b74a2532b..8ed3ad0e211 100644 --- a/privacy/scrubber.go +++ b/privacy/scrubber.go @@ -6,6 +6,7 @@ import ( "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/util/iputil" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/util/ptrutil" "github.com/prebid/openrtb/v19/openrtb2" @@ -103,7 +104,7 @@ func (s scrubber) ScrubRequest(bidRequest *openrtb2.BidRequest, enforcement Enfo if userCopy != nil && (enforcement.UFPD || enforcement.Eids) { if len(userCopy.Ext) != 0 { - json.Unmarshal(userCopy.Ext, &userExtParsed) + jsonutil.Unmarshal(userCopy.Ext, &userExtParsed) } } @@ -283,7 +284,7 @@ func scrubExtIDs(ext json.RawMessage, fieldName string) json.RawMessage { } var userExtParsed map[string]json.RawMessage - err := json.Unmarshal(ext, &userExtParsed) + err := jsonutil.Unmarshal(ext, &userExtParsed) if err != nil { return ext } diff --git a/router/router.go b/router/router.go index f2b1f7e7447..7bb8d7e8e46 100644 --- a/router/router.go +++ b/router/router.go @@ -34,6 +34,7 @@ import ( "github.com/prebid/prebid-server/server/ssl" storedRequestsConf "github.com/prebid/prebid-server/stored_requests/config" "github.com/prebid/prebid-server/usersync" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/util/uuidutil" "github.com/prebid/prebid-server/version" @@ -358,7 +359,7 @@ func readDefaultRequest(defReqConfig config.DefReqConfig) (map[string]string, [] return aliases, []byte{} } - if err := json.Unmarshal(defReqJSON, defReq); err != nil { + if err := jsonutil.UnmarshalValid(defReqJSON, defReq); err != nil { // we might not have aliases defined, but will atleast show that the JSON file is parsable. glog.Fatalf("error parsing alias json in file %s: %v", defReqConfig.FileSystem.FileName, err) return aliases, []byte{} diff --git a/stored_requests/backends/file_fetcher/fetcher.go b/stored_requests/backends/file_fetcher/fetcher.go index 56f5bdf853c..a9bbe919dcf 100644 --- a/stored_requests/backends/file_fetcher/fetcher.go +++ b/stored_requests/backends/file_fetcher/fetcher.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/prebid/prebid-server/stored_requests" + "github.com/prebid/prebid-server/util/jsonutil" jsonpatch "gopkg.in/evanphx/json-patch.v4" ) @@ -78,7 +79,7 @@ func (fetcher *eagerFetcher) FetchCategories(ctx context.Context, primaryAdServe tmp := make(map[string]stored_requests.Category) - if err := json.Unmarshal(file, &tmp); err != nil { + if err := jsonutil.UnmarshalValid(file, &tmp); err != nil { return "", fmt.Errorf("Unable to unmarshal categories for adserver: '%s', publisherId: '%s'", primaryAdServer, publisherId) } fetcher.Categories[fileName] = tmp diff --git a/stored_requests/backends/http_fetcher/fetcher.go b/stored_requests/backends/http_fetcher/fetcher.go index 326b63fce71..88afa39fb1d 100644 --- a/stored_requests/backends/http_fetcher/fetcher.go +++ b/stored_requests/backends/http_fetcher/fetcher.go @@ -1,7 +1,6 @@ package http_fetcher import ( - "bytes" "context" "encoding/json" "fmt" @@ -11,6 +10,7 @@ import ( "strings" "github.com/prebid/prebid-server/stored_requests" + "github.com/prebid/prebid-server/util/jsonutil" jsonpatch "gopkg.in/evanphx/json-patch.v4" "github.com/golang/glog" @@ -142,7 +142,7 @@ func (fetcher *HttpFetcher) FetchAccounts(ctx context.Context, accountIDs []stri } } var responseData accountsResponseContract - if err = json.Unmarshal(respBytes, &responseData); err != nil { + if err = jsonutil.UnmarshalValid(respBytes, &responseData); err != nil { return nil, []error{ fmt.Errorf(`Error fetching accounts %v via http: failed to parse response: %v`, accountIDs, err), } @@ -209,7 +209,7 @@ func (fetcher *HttpFetcher) FetchCategories(ctx context.Context, primaryAdServer respBytes, err := io.ReadAll(httpResp.Body) tmp := make(map[string]stored_requests.Category) - if err := json.Unmarshal(respBytes, &tmp); err != nil { + if err := jsonutil.UnmarshalValid(respBytes, &tmp); err != nil { return "", fmt.Errorf("Unable to unmarshal categories for adserver: '%s', publisherId: '%s'", primaryAdServer, publisherId) } fetcher.Categories[dataName] = tmp @@ -240,7 +240,7 @@ func unpackResponse(resp *http.Response) (requestData map[string]json.RawMessage if resp.StatusCode == http.StatusOK { var responseObj responseContract - if err := json.Unmarshal(respBytes, &responseObj); err != nil { + if err := jsonutil.UnmarshalValid(respBytes, &responseObj); err != nil { errs = append(errs, err) return } @@ -260,7 +260,7 @@ func unpackResponse(resp *http.Response) (requestData map[string]json.RawMessage func convertNullsToErrs(m map[string]json.RawMessage, dataType string, errs []error) []error { for id, val := range m { - if bytes.Equal(val, []byte("null")) { + if val == nil { delete(m, id) errs = append(errs, stored_requests.NotFoundError{ ID: id, diff --git a/stored_requests/events/api/api.go b/stored_requests/events/api/api.go index bf8edd2d849..30778f0e11a 100644 --- a/stored_requests/events/api/api.go +++ b/stored_requests/events/api/api.go @@ -1,12 +1,12 @@ package api import ( - "encoding/json" "io" "net/http" "github.com/julienschmidt/httprouter" "github.com/prebid/prebid-server/stored_requests/events" + "github.com/prebid/prebid-server/util/jsonutil" ) type eventsAPI struct { @@ -43,7 +43,7 @@ func (api *eventsAPI) HandleEvent(w http.ResponseWriter, r *http.Request, _ http } var save events.Save - if err := json.Unmarshal(body, &save); err != nil { + if err := jsonutil.UnmarshalValid(body, &save); err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte("Invalid update.\n")) return @@ -59,7 +59,7 @@ func (api *eventsAPI) HandleEvent(w http.ResponseWriter, r *http.Request, _ http } var invalidation events.Invalidation - if err := json.Unmarshal(body, &invalidation); err != nil { + if err := jsonutil.UnmarshalValid(body, &invalidation); err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte("Invalid invalidation.\n")) return diff --git a/stored_requests/events/http/http.go b/stored_requests/events/http/http.go index 1c4f8fdff73..be5b85a03a6 100644 --- a/stored_requests/events/http/http.go +++ b/stored_requests/events/http/http.go @@ -13,6 +13,7 @@ import ( "github.com/buger/jsonparser" "github.com/prebid/prebid-server/stored_requests/events" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/golang/glog" ) @@ -184,7 +185,7 @@ func (e *HTTPEvents) parse(endpoint string, resp *httpCore.Response, err error) } var respObj responseContract - if err := json.Unmarshal(respBytes, &respObj); err != nil { + if err := jsonutil.UnmarshalValid(respBytes, &respObj); err != nil { glog.Errorf("Failed to unmarshal body of GET %s for Stored Requests: %v", endpoint, err) return nil, false } diff --git a/stored_responses/stored_responses.go b/stored_responses/stored_responses.go index 6a5da01deba..ef8fe50619d 100644 --- a/stored_responses/stored_responses.go +++ b/stored_responses/stored_responses.go @@ -9,6 +9,7 @@ import ( "github.com/prebid/openrtb/v19/openrtb2" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/stored_requests" + "github.com/prebid/prebid-server/util/jsonutil" ) type ImpsWithAuctionResponseIDs map[string]string @@ -210,7 +211,7 @@ func parseImpInfo(requestJson []byte) (impData []ImpExtPrebidData, errs []error) impExtData, _, _, err := jsonparser.Get(imp, "ext", "prebid") var impExtPrebid openrtb_ext.ExtImpPrebid if impExtData != nil { - if err := json.Unmarshal(impExtData, &impExtPrebid); err != nil { + if err := jsonutil.Unmarshal(impExtData, &impExtPrebid); err != nil { errs = append(errs, err) } } diff --git a/util/jsonutil/jsonutil.go b/util/jsonutil/jsonutil.go index 3b468731cad..8e4327e82ed 100644 --- a/util/jsonutil/jsonutil.go +++ b/util/jsonutil/jsonutil.go @@ -3,6 +3,8 @@ package jsonutil import ( "bytes" "encoding/json" + jsoniter "github.com/json-iterator/go" + "github.com/prebid/prebid-server/errortypes" "io" ) @@ -110,3 +112,31 @@ func DropElement(extension []byte, elementNames ...string) ([]byte, error) { } return extension, nil } + +// Unmarshal unmarshals a byte slice into the specified data structure without performing +// any validation on the data. An unmarshal error is returned if a non-validation error occurs. +func Unmarshal(data []byte, v interface{}) error { + err := jsoniter.Config{ + EscapeHTML: true, + SortMapKeys: true, + ValidateJsonRawMessage: false, + }.Froze().Unmarshal(data, v) + + if err != nil { + return &errortypes.FailedToUnmarshal{ + Message: err.Error(), + } + } + return nil +} + +// UnmarshalValid validates and unmarshals a byte slice into the specified data structure +// returning an error if validation fails +func UnmarshalValid(data []byte, v interface{}) error { + if err := jsoniter.ConfigCompatibleWithStandardLibrary.Unmarshal(data, v); err != nil { + return &errortypes.FailedToUnmarshal{ + Message: err.Error(), + } + } + return nil +} From 0fcd2f8fc84899cc3161d45b5dcf8f901518637c Mon Sep 17 00:00:00 2001 From: bsardo <1168933+bsardo@users.noreply.github.com> Date: Mon, 16 Oct 2023 12:56:56 -0400 Subject: [PATCH 02/10] Add marshal error type and marshal helper function that wraps json-iterator marshal call --- errortypes/code.go | 1 + errortypes/errortypes.go | 17 +++++++++++++++++ util/jsonutil/jsonutil.go | 12 ++++++++++++ 3 files changed, 30 insertions(+) diff --git a/errortypes/code.go b/errortypes/code.go index 1dbd5c0e996..27657a35429 100644 --- a/errortypes/code.go +++ b/errortypes/code.go @@ -15,6 +15,7 @@ const ( MalformedAcctErrorCode ModuleRejectionErrorCode TmaxTimeoutErrorCode + FailedToMarshalErrorCode FailedToUnmarshalErrorCode ) diff --git a/errortypes/errortypes.go b/errortypes/errortypes.go index 58221d7bec7..969ef30796c 100644 --- a/errortypes/errortypes.go +++ b/errortypes/errortypes.go @@ -237,3 +237,20 @@ func (err *FailedToUnmarshal) Code() int { func (err *FailedToUnmarshal) Severity() Severity { return SeverityFatal } + +// FailedToMarshal should be used to represent errors that occur when marshaling to a byte slice. +type FailedToMarshal struct { + Message string +} + +func (err *FailedToMarshal) Error() string { + return err.Message +} + +func (err *FailedToMarshal) Code() int { + return FailedToMarshalErrorCode +} + +func (err *FailedToMarshal) Severity() Severity { + return SeverityFatal +} diff --git a/util/jsonutil/jsonutil.go b/util/jsonutil/jsonutil.go index 8e4327e82ed..1d9e1827826 100644 --- a/util/jsonutil/jsonutil.go +++ b/util/jsonutil/jsonutil.go @@ -140,3 +140,15 @@ func UnmarshalValid(data []byte, v interface{}) error { } return nil } + +// Marshal marshals a data structure into a byte slice without performing any validation +// on the data. A marshal error is returned if a non-validation error occurs. +func Marshal(v interface{}) ([]byte, error) { + data, err := jsoniter.ConfigCompatibleWithStandardLibrary.Marshal(v) + if err != nil { + return nil, &errortypes.FailedToMarshal{ + Message: err.Error(), + } + } + return data, nil +} From 1090b3224944be4e57de70d6591f51b1f2bc30bc Mon Sep 17 00:00:00 2001 From: bsardo <1168933+bsardo@users.noreply.github.com> Date: Mon, 16 Oct 2023 13:01:54 -0400 Subject: [PATCH 03/10] Call marshal helper function throughout main auction flow --- adservertargeting/reqcache.go | 2 +- adservertargeting/respdataprocessor.go | 3 ++- endpoints/openrtb2/amp_auction.go | 2 +- endpoints/openrtb2/auction.go | 10 +++++----- exchange/auction.go | 10 +++++----- exchange/bidder.go | 4 ++-- exchange/events.go | 4 ++-- exchange/exchange.go | 8 ++++---- exchange/exchange_test.go | 27 ++++++++++---------------- exchange/utils.go | 14 ++++++------- exchange/utils_test.go | 20 ++----------------- hooks/hookexecution/enricher.go | 5 +++-- modules/modules.go | 3 ++- modules/modules_test.go | 2 +- privacy/scrubber.go | 4 ++-- schain/schainwriter.go | 5 ++--- 16 files changed, 51 insertions(+), 72 deletions(-) diff --git a/adservertargeting/reqcache.go b/adservertargeting/reqcache.go index 7ff2aca13f7..b8b147af84e 100644 --- a/adservertargeting/reqcache.go +++ b/adservertargeting/reqcache.go @@ -49,7 +49,7 @@ func (bidsCache *bidsCache) GetBid(bidderName, bidId string, bid openrtb2.Bid) ( } _, bidExists := bidsCache.bids[bidderName][bidId] if !bidExists { - bidBytes, err := json.Marshal(bid) + bidBytes, err := jsonutil.Marshal(bid) if err != nil { return nil, err } diff --git a/adservertargeting/respdataprocessor.go b/adservertargeting/respdataprocessor.go index 94a391d08bb..649f802f6e2 100644 --- a/adservertargeting/respdataprocessor.go +++ b/adservertargeting/respdataprocessor.go @@ -8,6 +8,7 @@ import ( "github.com/pkg/errors" "github.com/prebid/openrtb/v19/openrtb2" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/util/jsonutil" jsonpatch "gopkg.in/evanphx/json-patch.v4" ) @@ -79,7 +80,7 @@ func buildBidExt(targetingData map[string]string, Targeting: targetingDataTruncated, }, } - bidExtTargeting, err := json.Marshal(bidExtTargetingData) + bidExtTargeting, err := jsonutil.Marshal(bidExtTargetingData) if err != nil { warnings = append(warnings, createWarning(err.Error())) return nil diff --git a/endpoints/openrtb2/amp_auction.go b/endpoints/openrtb2/amp_auction.go index 6f933e9c888..3ac482e1ec3 100644 --- a/endpoints/openrtb2/amp_auction.go +++ b/endpoints/openrtb2/amp_auction.go @@ -824,7 +824,7 @@ func setTrace(req *openrtb2.BidRequest, value string) error { return nil } - ext, err := json.Marshal(map[string]map[string]string{"prebid": {"trace": value}}) + ext, err := jsonutil.Marshal(map[string]map[string]string{"prebid": {"trace": value}}) if err != nil { return err } diff --git a/endpoints/openrtb2/auction.go b/endpoints/openrtb2/auction.go index 8982fb464fa..7f6d33ca6d0 100644 --- a/endpoints/openrtb2/auction.go +++ b/endpoints/openrtb2/auction.go @@ -308,7 +308,7 @@ func setSeatNonBidRaw(request *openrtb_ext.RequestWrapper, auctionResponse *exch return err } if setSeatNonBid(respExt, request, auctionResponse) { - if respExtJson, err := json.Marshal(respExt); err == nil { + if respExtJson, err := jsonutil.Marshal(respExt); err == nil { response.Ext = respExtJson return nil } else { @@ -674,7 +674,7 @@ func mergeBidderParamsImpExt(impExt *openrtb_ext.ImpExt, reqExtParams map[string } if modified { - impExtBidderJson, err := json.Marshal(impExtBidderMap) + impExtBidderJson, err := jsonutil.Marshal(impExtBidderMap) if err != nil { return fmt.Errorf("error marshalling ext.BIDDER: %s", err.Error()) } @@ -722,7 +722,7 @@ func mergeBidderParamsImpExtPrebid(impExt *openrtb_ext.ImpExt, reqExtParams map[ } if modified { - impExtPrebidBidderJson, err := json.Marshal(impExtPrebidBidderMap) + impExtPrebidBidderJson, err := jsonutil.Marshal(impExtPrebidBidderMap) if err != nil { return fmt.Errorf("error marshalling ext.prebid.bidder.BIDDER: %s", err.Error()) } @@ -1205,7 +1205,7 @@ func fillAndValidateNative(n *openrtb2.Native, impIndex int) error { return err } - serialized, err := json.Marshal(nativePayload) + serialized, err := jsonutil.Marshal(nativePayload) if err != nil { return err } @@ -2226,7 +2226,7 @@ func (deps *endpointDeps) processStoredRequests(requestJson []byte, impInfo []Im } } if len(resolvedImps) > 0 { - newImpJson, err := json.Marshal(resolvedImps) + newImpJson, err := jsonutil.Marshal(resolvedImps) if err != nil { return nil, nil, []error{err} } diff --git a/exchange/auction.go b/exchange/auction.go index b2d6eb571ca..abbf56ba799 100644 --- a/exchange/auction.go +++ b/exchange/auction.go @@ -2,7 +2,6 @@ package exchange import ( "context" - "encoding/json" "encoding/xml" "errors" "fmt" @@ -17,6 +16,7 @@ import ( "github.com/prebid/prebid-server/exchange/entities" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/prebid_cache_client" + "github.com/prebid/prebid-server/util/jsonutil" ) const ( @@ -83,7 +83,7 @@ func (d *DebugLog) PutDebugLogError(cache prebid_cache_client.Client, timeout in d.CacheKey = rawUUID.String() } - data, err := json.Marshal(d.CacheString) + data, err := jsonutil.Marshal(d.CacheString) if err != nil { return err } @@ -244,7 +244,7 @@ func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client, } } if bids { - if jsonBytes, err := json.Marshal(topBid.Bid); err == nil { + if jsonBytes, err := jsonutil.Marshal(topBid.Bid); err == nil { jsonBytes, err = evTracking.modifyBidJSON(topBid, bidderName, jsonBytes) if err != nil { errs = append(errs, err) @@ -265,7 +265,7 @@ func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client, } if vast && topBid.BidType == openrtb_ext.BidTypeVideo { vastXML := makeVAST(topBid.Bid) - if jsonBytes, err := json.Marshal(vastXML); err == nil { + if jsonBytes, err := jsonutil.Marshal(vastXML); err == nil { if useCustomCacheKey { toCache = append(toCache, prebid_cache_client.Cacheable{ Type: prebid_cache_client.TypeXML, @@ -292,7 +292,7 @@ func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client, if len(toCache) > 0 && debugLog != nil && debugLog.DebugEnabledOrOverridden { debugLog.CacheKey = hbCacheID debugLog.BuildCacheString() - if jsonBytes, err := json.Marshal(debugLog.CacheString); err == nil { + if jsonBytes, err := jsonutil.Marshal(debugLog.CacheString); err == nil { toCache = append(toCache, prebid_cache_client.Cacheable{ Type: debugLog.CacheType, Data: jsonBytes, diff --git a/exchange/bidder.go b/exchange/bidder.go index 718768d2df4..1b9ed923138 100644 --- a/exchange/bidder.go +++ b/exchange/bidder.go @@ -288,7 +288,7 @@ func (bidder *bidderAdapter) requestBid(ctx context.Context, bidderRequest Bidde errs = append(errs, moreErrs...) if nativeMarkup != nil { - markup, err := json.Marshal(*nativeMarkup) + markup, err := jsonutil.Marshal(*nativeMarkup) if err != nil { errs = append(errs, err) } else { @@ -637,7 +637,7 @@ func (bidder *bidderAdapter) doTimeoutNotification(timeoutBidder adapters.Timeou } } } else if bidder.config.Debug.TimeoutNotification.Log { - reqJSON, err := json.Marshal(req) + reqJSON, err := jsonutil.Marshal(req) var msg string if err == nil { msg = fmt.Sprintf("TimeoutNotification: Failed to generate timeout request: error(%s), bidder request(%s)", errL[0].Error(), string(reqJSON)) diff --git a/exchange/events.go b/exchange/events.go index e4f6b0d503f..fb535f6f70e 100644 --- a/exchange/events.go +++ b/exchange/events.go @@ -1,7 +1,6 @@ package exchange import ( - "encoding/json" "time" "github.com/prebid/prebid-server/exchange/entities" @@ -11,6 +10,7 @@ import ( "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/endpoints/events" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/util/jsonutil" ) // eventTracking has configuration fields needed for adding event tracking to an auction response @@ -91,7 +91,7 @@ func (ev *eventTracking) modifyBidJSON(pbsBid *entities.PbsOrtbBid, bidderName o winEventURL = ev.makeEventURL(analytics.Win, pbsBid, bidderName) } // wurl attribute is not in the schema, so we have to patch - patch, err := json.Marshal(map[string]string{"wurl": winEventURL}) + patch, err := jsonutil.Marshal(map[string]string{"wurl": winEventURL}) if err != nil { return jsonBytes, err } diff --git a/exchange/exchange.go b/exchange/exchange.go index 5c9bbfed721..3446ebc78e2 100644 --- a/exchange/exchange.go +++ b/exchange/exchange.go @@ -278,7 +278,7 @@ func (e *exchange) HoldAuction(ctx context.Context, r *AuctionRequest, debugLog if err := r.BidRequestWrapper.RebuildRequest(); err != nil { return nil, err } - resolvedBidReq, err := json.Marshal(r.BidRequestWrapper.BidRequest) + resolvedBidReq, err := jsonutil.Marshal(r.BidRequestWrapper.BidRequest) if err != nil { return nil, err } @@ -442,7 +442,7 @@ func (e *exchange) HoldAuction(ctx context.Context, r *AuctionRequest, debugLog bidResponseExt = e.makeExtBidResponse(adapterBids, adapterExtra, *r, responseDebugAllow, requestExtPrebid.Passthrough, fledge, errs) if debugLog.DebugEnabledOrOverridden { - if bidRespExtBytes, err := json.Marshal(bidResponseExt); err == nil { + if bidRespExtBytes, err := jsonutil.Marshal(bidResponseExt); err == nil { debugLog.Data.Response = string(bidRespExtBytes) } else { debugLog.Data.Response = "Unable to marshal response ext for debugging" @@ -463,7 +463,7 @@ func (e *exchange) HoldAuction(ctx context.Context, r *AuctionRequest, debugLog if debugLog.DebugEnabledOrOverridden { - if bidRespExtBytes, err := json.Marshal(bidResponseExt); err == nil { + if bidRespExtBytes, err := jsonutil.Marshal(bidResponseExt); err == nil { debugLog.Data.Response = string(bidRespExtBytes) } else { debugLog.Data.Response = "Unable to marshal response ext for debugging" @@ -1333,7 +1333,7 @@ func makeBidExtJSON(ext json.RawMessage, prebid *openrtb_ext.ExtBidPrebid, impEx } } extMap[openrtb_ext.PrebidExtKey] = prebid - return json.Marshal(extMap) + return jsonutil.Marshal(extMap) } // If bid got cached inside `(a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client, targData *targetData, bidRequest *openrtb2.BidRequest, ttlBuffer int64, defaultTTLs *config.DefaultTTLs, bidCategory map[string]string)`, diff --git a/exchange/exchange_test.go b/exchange/exchange_test.go index 9e34debdfd6..0b3af663e12 100644 --- a/exchange/exchange_test.go +++ b/exchange/exchange_test.go @@ -1966,7 +1966,7 @@ func TestBidResponseImpExtInfo(t *testing.T) { impExtInfo["some-impression-id"] = ImpExtInfo{ true, []byte(`{"video":{"h":480,"mimes":["video/mp4"]}}`), - json.RawMessage(`{"imp_passthrough_val": 1}`)} + json.RawMessage(`{"imp_passthrough_val":1}`)} expectedBidResponseExt := `{"origbidcpm":0,"prebid":{"type":"video","passthrough":{"imp_passthrough_val":1}},"storedrequestattributes":{"h":480,"mimes":["video/mp4"]}}` @@ -4175,18 +4175,9 @@ func TestMakeBidExtJSON(t *testing.T) { description: "Invalid extension, valid extBidPrebid and valid imp ext info", ext: json.RawMessage(`{invalid json}`), extBidPrebid: openrtb_ext.ExtBidPrebid{Type: openrtb_ext.BidType("video")}, - impExtInfo: map[string]ImpExtInfo{"test_imp_id": {true, []byte(`{"video":{"h":480,"mimes":["video/mp4"]}}`), json.RawMessage(`"prebid": {"passthrough": {"imp_passthrough_val": some_val}}"`)}}, expectedBidExt: ``, expectedErrMessage: "ReadString: expects \" or n, but found i, error found in #2 byte of ...|{invalid jso|..., bigger context ...|{invalid json}|...", }, - { - description: "Valid extension, empty extBidPrebid and invalid imp ext info", - ext: json.RawMessage(`{"video":{"h":100}}`), - extBidPrebid: openrtb_ext.ExtBidPrebid{}, - impExtInfo: map[string]ImpExtInfo{"test_imp_id": {true, []byte(`{"video":{!}}`), nil}}, - expectedBidExt: ``, - expectedErrMessage: "invalid character", - }, { description: "Meta - Invalid", ext: json.RawMessage(`{"prebid":{"meta":{"brandId":"foo"}}}`), // brandId should be an int, but is a string in this test case @@ -4197,14 +4188,16 @@ func TestMakeBidExtJSON(t *testing.T) { } for _, test := range testCases { - result, err := makeBidExtJSON(test.ext, &test.extBidPrebid, test.impExtInfo, "test_imp_id", test.origbidcpm, test.origbidcur) + t.Run(test.description, func(t *testing.T) { + result, err := makeBidExtJSON(test.ext, &test.extBidPrebid, test.impExtInfo, "test_imp_id", test.origbidcpm, test.origbidcur) - if test.expectedErrMessage == "" { - assert.JSONEq(t, test.expectedBidExt, string(result), "Incorrect result") - assert.NoError(t, err, "Error should not be returned") - } else { - assert.Contains(t, err.Error(), test.expectedErrMessage, "incorrect error message") - } + if test.expectedErrMessage == "" { + assert.JSONEq(t, test.expectedBidExt, string(result), "Incorrect result") + assert.NoError(t, err, "Error should not be returned") + } else { + assert.Contains(t, err.Error(), test.expectedErrMessage, "incorrect error message") + } + }) } } diff --git a/exchange/utils.go b/exchange/utils.go index 4bf6ae7da3a..f9164d91516 100644 --- a/exchange/utils.go +++ b/exchange/utils.go @@ -408,7 +408,7 @@ func buildRequestExtForBidder(bidder string, requestExt json.RawMessage, request } // Marshal New Prebid Object - prebidJson, err := json.Marshal(prebid) + prebidJson, err := jsonutil.Marshal(prebid) if err != nil { return nil, err } @@ -429,7 +429,7 @@ func buildRequestExtForBidder(bidder string, requestExt json.RawMessage, request } if len(extMap) > 0 { - return json.Marshal(extMap) + return jsonutil.Marshal(extMap) } else { return nil, nil } @@ -527,7 +527,7 @@ func extractBuyerUIDs(user *openrtb2.User) (map[string]string, error) { // Remarshal (instead of removing) if the ext has other known fields if userExt.Consent != "" || len(userExt.Eids) > 0 { - if newUserExtBytes, err := json.Marshal(userExt); err != nil { + if newUserExtBytes, err := jsonutil.Marshal(userExt); err != nil { return nil, err } else { user.Ext = newUserExtBytes @@ -577,7 +577,7 @@ func splitImps(imps []openrtb2.Imp) (map[string][]openrtb2.Imp, error) { sanitizedImpExt[openrtb_ext.PrebidExtBidderKey] = bidderExt - impExtJSON, err := json.Marshal(sanitizedImpExt) + impExtJSON, err := jsonutil.Marshal(sanitizedImpExt) if err != nil { return nil, fmt.Errorf("unable to remove other bidder fields for imp[%d]: cannot marshal ext: %v", i, err) } @@ -617,7 +617,7 @@ func createSanitizedImpExt(impExt, impExtPrebid map[string]json.RawMessage) (map // marshal sanitized imp[].ext.prebid if len(sanitizedImpPrebidExt) > 0 { - if impExtPrebidJSON, err := json.Marshal(sanitizedImpPrebidExt); err == nil { + if impExtPrebidJSON, err := jsonutil.Marshal(sanitizedImpPrebidExt); err == nil { sanitizedImpExt[openrtb_ext.PrebidExtKey] = impExtPrebidJSON } else { return nil, fmt.Errorf("cannot marshal ext.prebid: %v", err) @@ -734,7 +734,7 @@ func removeUnpermissionedEids(request *openrtb2.BidRequest, bidder string, reque if len(eidsAllowed) == 0 { delete(userExt, "eids") } else { - eidsRaw, err := json.Marshal(eidsAllowed) + eidsRaw, err := jsonutil.Marshal(eidsAllowed) if err != nil { return err } @@ -747,7 +747,7 @@ func removeUnpermissionedEids(request *openrtb2.BidRequest, bidder string, reque return nil } - userExtJSON, err := json.Marshal(userExt) + userExtJSON, err := jsonutil.Marshal(userExt) if err != nil { return err } diff --git a/exchange/utils_test.go b/exchange/utils_test.go index 1add2fc4f96..1b9aaacf156 100644 --- a/exchange/utils_test.go +++ b/exchange/utils_test.go @@ -403,22 +403,6 @@ func TestCreateSanitizedImpExt(t *testing.T) { }, expectedError: "", }, - { - description: "Marshal Error - imp.ext.prebid", - givenImpExt: map[string]json.RawMessage{ - "prebid": json.RawMessage(`"ignoredInFavorOfSeparatelyUnmarshalledImpExtPrebid"`), - "data": json.RawMessage(`"anyData"`), - "context": json.RawMessage(`"anyContext"`), - "skadn": json.RawMessage(`"anySKAdNetwork"`), - "gpid": json.RawMessage(`"anyGPID"`), - "tid": json.RawMessage(`"anyTID"`), - }, - givenImpExtPrebid: map[string]json.RawMessage{ - "options": json.RawMessage(`malformed`), // String value without quotes. - }, - expected: nil, - expectedError: "cannot marshal ext.prebid: json: error calling MarshalJSON for type json.RawMessage: invalid character 'm' looking for beginning of value", - }, } for _, test := range testCases { @@ -3159,7 +3143,7 @@ func TestCleanOpenRTBRequestsBidAdjustment(t *testing.T) { H: 600, }}, }, - Ext: json.RawMessage(`{"bidder":{"placementId":1}}`), + Ext: json.RawMessage(`{"bidder":{"placementId": 1}}`), }}, }, { @@ -3187,7 +3171,7 @@ func TestCleanOpenRTBRequestsBidAdjustment(t *testing.T) { H: 600, }}, }, - Ext: json.RawMessage(`{"bidder":{"placementId":1}}`), + Ext: json.RawMessage(`{"bidder":{"placementId": 1}}`), }}, }, } diff --git a/hooks/hookexecution/enricher.go b/hooks/hookexecution/enricher.go index 2978c21957d..c3dd5a23339 100644 --- a/hooks/hookexecution/enricher.go +++ b/hooks/hookexecution/enricher.go @@ -6,6 +6,7 @@ import ( "github.com/buger/jsonparser" "github.com/prebid/openrtb/v19/openrtb2" "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/util/jsonutil" jsonpatch "gopkg.in/evanphx/json-patch.v4" ) @@ -52,7 +53,7 @@ func EnrichExtBidResponse( return ext, warnings, err } - response, err := json.Marshal(extPrebid{Prebid: extModules{Modules: modules}}) + response, err := jsonutil.Marshal(extPrebid{Prebid: extModules{Modules: modules}}) if err != nil { return ext, warnings, err } @@ -83,7 +84,7 @@ func GetModulesJSON( return nil, warnings, nil } - data, err := json.Marshal(modulesOutcome) + data, err := jsonutil.Marshal(modulesOutcome) return data, warnings, err } diff --git a/modules/modules.go b/modules/modules.go index ac60ec58082..4e60a9eda32 100644 --- a/modules/modules.go +++ b/modules/modules.go @@ -8,6 +8,7 @@ import ( "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/hooks" "github.com/prebid/prebid-server/modules/moduledeps" + "github.com/prebid/prebid-server/util/jsonutil" ) //go:generate go run ./generator/buildergen.go @@ -58,7 +59,7 @@ func (m *builder) Build( id := fmt.Sprintf("%s.%s", vendor, moduleName) if data, ok := cfg[vendor][moduleName]; ok { - if conf, err = json.Marshal(data); err != nil { + if conf, err = jsonutil.Marshal(data); err != nil { return nil, nil, fmt.Errorf(`failed to marshal "%s" module config: %s`, id, err) } diff --git a/modules/modules_test.go b/modules/modules_test.go index 1c70ce4badf..f88a48a2c44 100644 --- a/modules/modules_test.go +++ b/modules/modules_test.go @@ -85,7 +85,7 @@ func TestModuleBuilderBuild(t *testing.T) { givenConfig: map[string]map[string]interface{}{vendor: {moduleName: math.Inf(1)}}, expectedHookRepo: nil, expectedModulesStages: nil, - expectedErr: fmt.Errorf(`failed to marshal "%s.%s" module config: json: unsupported value: +Inf`, vendor, moduleName), + expectedErr: fmt.Errorf(`failed to marshal "%s.%s" module config: unsupported value: +Inf`, vendor, moduleName), }, } diff --git a/privacy/scrubber.go b/privacy/scrubber.go index 8ed3ad0e211..0cfc7cdd7f4 100644 --- a/privacy/scrubber.go +++ b/privacy/scrubber.go @@ -149,7 +149,7 @@ func (s scrubber) ScrubRequest(bidRequest *openrtb2.BidRequest, enforcement Enfo } if userExtModified { - userExt, _ := json.Marshal(userExtParsed) + userExt, _ := jsonutil.Marshal(userExtParsed) userCopy.Ext = userExt } @@ -292,7 +292,7 @@ func scrubExtIDs(ext json.RawMessage, fieldName string) json.RawMessage { _, hasField := userExtParsed[fieldName] if hasField { delete(userExtParsed, fieldName) - result, err := json.Marshal(userExtParsed) + result, err := jsonutil.Marshal(userExtParsed) if err == nil { return result } diff --git a/schain/schainwriter.go b/schain/schainwriter.go index 7e2161adb3b..e7c9dd4ce72 100644 --- a/schain/schainwriter.go +++ b/schain/schainwriter.go @@ -1,10 +1,9 @@ package schain import ( - "encoding/json" - "github.com/prebid/openrtb/v19/openrtb2" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/util/jsonutil" ) // NewSChainWriter creates an ORTB 2.5 schain writer instance @@ -70,7 +69,7 @@ func (w SChainWriter) Write(req *openrtb2.BidRequest, bidder string) { schain.SChain.Nodes = append(schain.SChain.Nodes, *w.hostSChainNode) } - sourceExt, err := json.Marshal(schain) + sourceExt, err := jsonutil.Marshal(schain) if err == nil { req.Source.Ext = sourceExt } From 31d7abe49ec01f4271749d7711739041a1f1ffa8 Mon Sep 17 00:00:00 2001 From: bsardo <1168933+bsardo@users.noreply.github.com> Date: Mon, 16 Oct 2023 16:26:56 -0400 Subject: [PATCH 04/10] Only unmarshal in request wrapper if body is not null --- openrtb_ext/request_wrapper.go | 90 +++++++++++++++++++++------------- 1 file changed, 56 insertions(+), 34 deletions(-) diff --git a/openrtb_ext/request_wrapper.go b/openrtb_ext/request_wrapper.go index 1f93d4c0d0e..09321e5b0b3 100644 --- a/openrtb_ext/request_wrapper.go +++ b/openrtb_ext/request_wrapper.go @@ -446,7 +446,7 @@ func (ue *UserExt) unmarshal(extJson json.RawMessage) error { } consentJson, hasConsent := ue.ext[consentKey] - if hasConsent { + if hasConsent && consentJson != nil { if err := jsonutil.Unmarshal(consentJson, &ue.consent); err != nil { return err } @@ -455,6 +455,8 @@ func (ue *UserExt) unmarshal(extJson json.RawMessage) error { prebidJson, hasPrebid := ue.ext[prebidKey] if hasPrebid { ue.prebid = &ExtUserPrebid{} + } + if prebidJson != nil { if err := jsonutil.Unmarshal(prebidJson, ue.prebid); err != nil { return err } @@ -463,21 +465,29 @@ func (ue *UserExt) unmarshal(extJson json.RawMessage) error { eidsJson, hasEids := ue.ext[eidsKey] if hasEids { ue.eids = &[]openrtb2.EID{} + } + if eidsJson != nil { if err := jsonutil.Unmarshal(eidsJson, ue.eids); err != nil { return err } } - if consentedProviderSettingsJson, hasCPSettings := ue.ext[consentedProvidersSettingsStringKey]; hasCPSettings { + consentedProviderSettingsInJson, hasCPSettingsIn := ue.ext[consentedProvidersSettingsStringKey] + if hasCPSettingsIn { ue.consentedProvidersSettingsIn = &ConsentedProvidersSettingsIn{} - if err := jsonutil.Unmarshal(consentedProviderSettingsJson, ue.consentedProvidersSettingsIn); err != nil { + } + if consentedProviderSettingsInJson != nil { + if err := jsonutil.Unmarshal(consentedProviderSettingsInJson, ue.consentedProvidersSettingsIn); err != nil { return err } } - if consentedProviderSettingsJson, hasCPSettings := ue.ext[consentedProvidersSettingsListKey]; hasCPSettings { + consentedProviderSettingsOutJson, hasCPSettingsOut := ue.ext[consentedProvidersSettingsListKey] + if hasCPSettingsOut { ue.consentedProvidersSettingsOut = &ConsentedProvidersSettingsOut{} - if err := jsonutil.Unmarshal(consentedProviderSettingsJson, ue.consentedProvidersSettingsOut); err != nil { + } + if consentedProviderSettingsOutJson != nil { + if err := jsonutil.Unmarshal(consentedProviderSettingsOutJson, ue.consentedProvidersSettingsOut); err != nil { return err } } @@ -488,7 +498,7 @@ func (ue *UserExt) unmarshal(extJson json.RawMessage) error { func (ue *UserExt) marshal() (json.RawMessage, error) { if ue.consentDirty { if ue.consent != nil && len(*ue.consent) > 0 { - consentJson, err := json.Marshal(ue.consent) + consentJson, err := jsonutil.Marshal(ue.consent) if err != nil { return nil, err } @@ -501,7 +511,7 @@ func (ue *UserExt) marshal() (json.RawMessage, error) { if ue.prebidDirty { if ue.prebid != nil { - prebidJson, err := json.Marshal(ue.prebid) + prebidJson, err := jsonutil.Marshal(ue.prebid) if err != nil { return nil, err } @@ -518,7 +528,7 @@ func (ue *UserExt) marshal() (json.RawMessage, error) { if ue.consentedProvidersSettingsInDirty { if ue.consentedProvidersSettingsIn != nil { - cpSettingsJson, err := json.Marshal(ue.consentedProvidersSettingsIn) + cpSettingsJson, err := jsonutil.Marshal(ue.consentedProvidersSettingsIn) if err != nil { return nil, err } @@ -535,7 +545,7 @@ func (ue *UserExt) marshal() (json.RawMessage, error) { if ue.consentedProvidersSettingsOutDirty { if ue.consentedProvidersSettingsOut != nil { - cpSettingsJson, err := json.Marshal(ue.consentedProvidersSettingsOut) + cpSettingsJson, err := jsonutil.Marshal(ue.consentedProvidersSettingsOut) if err != nil { return nil, err } @@ -552,7 +562,7 @@ func (ue *UserExt) marshal() (json.RawMessage, error) { if ue.eidsDirty { if ue.eids != nil && len(*ue.eids) > 0 { - eidsJson, err := json.Marshal(ue.eids) + eidsJson, err := jsonutil.Marshal(ue.eids) if err != nil { return nil, err } @@ -567,7 +577,7 @@ func (ue *UserExt) marshal() (json.RawMessage, error) { if len(ue.ext) == 0 { return nil, nil } - return json.Marshal(ue.ext) + return jsonutil.Marshal(ue.ext) } func (ue *UserExt) Dirty() bool { @@ -734,6 +744,8 @@ func (re *RequestExt) unmarshal(extJson json.RawMessage) error { prebidJson, hasPrebid := re.ext[prebidKey] if hasPrebid { re.prebid = &ExtRequestPrebid{} + } + if prebidJson != nil { if err := jsonutil.Unmarshal(prebidJson, re.prebid); err != nil { return err } @@ -742,6 +754,8 @@ func (re *RequestExt) unmarshal(extJson json.RawMessage) error { schainJson, hasSChain := re.ext[schainKey] if hasSChain { re.schain = &openrtb2.SupplyChain{} + } + if schainJson != nil { if err := jsonutil.Unmarshal(schainJson, re.schain); err != nil { return err } @@ -753,7 +767,7 @@ func (re *RequestExt) unmarshal(extJson json.RawMessage) error { func (re *RequestExt) marshal() (json.RawMessage, error) { if re.prebidDirty { if re.prebid != nil { - prebidJson, err := json.Marshal(re.prebid) + prebidJson, err := jsonutil.Marshal(re.prebid) if err != nil { return nil, err } @@ -770,7 +784,7 @@ func (re *RequestExt) marshal() (json.RawMessage, error) { if re.schainDirty { if re.schain != nil { - schainJson, err := json.Marshal(re.schain) + schainJson, err := jsonutil.Marshal(re.schain) if err != nil { return nil, err } @@ -789,7 +803,7 @@ func (re *RequestExt) marshal() (json.RawMessage, error) { if len(re.ext) == 0 { return nil, nil } - return json.Marshal(re.ext) + return jsonutil.Marshal(re.ext) } func (re *RequestExt) Dirty() bool { @@ -888,6 +902,8 @@ func (de *DeviceExt) unmarshal(extJson json.RawMessage) error { prebidJson, hasPrebid := de.ext[prebidKey] if hasPrebid { de.prebid = &ExtDevicePrebid{} + } + if prebidJson != nil { if err := jsonutil.Unmarshal(prebidJson, de.prebid); err != nil { return err } @@ -899,7 +915,7 @@ func (de *DeviceExt) unmarshal(extJson json.RawMessage) error { func (de *DeviceExt) marshal() (json.RawMessage, error) { if de.prebidDirty { if de.prebid != nil { - prebidJson, err := json.Marshal(de.prebid) + prebidJson, err := jsonutil.Marshal(de.prebid) if err != nil { return nil, err } @@ -918,7 +934,7 @@ func (de *DeviceExt) marshal() (json.RawMessage, error) { if len(de.ext) == 0 { return nil, nil } - return json.Marshal(de.ext) + return jsonutil.Marshal(de.ext) } func (de *DeviceExt) Dirty() bool { @@ -1000,6 +1016,8 @@ func (ae *AppExt) unmarshal(extJson json.RawMessage) error { prebidJson, hasPrebid := ae.ext[prebidKey] if hasPrebid { ae.prebid = &ExtAppPrebid{} + } + if prebidJson != nil { if err := jsonutil.Unmarshal(prebidJson, ae.prebid); err != nil { return err } @@ -1011,7 +1029,7 @@ func (ae *AppExt) unmarshal(extJson json.RawMessage) error { func (ae *AppExt) marshal() (json.RawMessage, error) { if ae.prebidDirty { if ae.prebid != nil { - prebidJson, err := json.Marshal(ae.prebid) + prebidJson, err := jsonutil.Marshal(ae.prebid) if err != nil { return nil, err } @@ -1030,7 +1048,7 @@ func (ae *AppExt) marshal() (json.RawMessage, error) { if len(ae.ext) == 0 { return nil, nil } - return json.Marshal(ae.ext) + return jsonutil.Marshal(ae.ext) } func (ae *AppExt) Dirty() bool { @@ -1109,7 +1127,7 @@ func (de *DOOHExt) marshal() (json.RawMessage, error) { if len(de.ext) == 0 { return nil, nil } - return json.Marshal(de.ext) + return jsonutil.Marshal(de.ext) } func (de *DOOHExt) Dirty() bool { @@ -1176,7 +1194,7 @@ func (re *RegExt) unmarshal(extJson json.RawMessage) error { } uspJson, hasUsp := re.ext[us_privacyKey] - if hasUsp { + if hasUsp && uspJson != nil { if err := jsonutil.Unmarshal(uspJson, &re.usPrivacy); err != nil { return err } @@ -1188,7 +1206,7 @@ func (re *RegExt) unmarshal(extJson json.RawMessage) error { func (re *RegExt) marshal() (json.RawMessage, error) { if re.gdprDirty { if re.gdpr != nil { - rawjson, err := json.Marshal(re.gdpr) + rawjson, err := jsonutil.Marshal(re.gdpr) if err != nil { return nil, err } @@ -1201,7 +1219,7 @@ func (re *RegExt) marshal() (json.RawMessage, error) { if re.usPrivacyDirty { if len(re.usPrivacy) > 0 { - rawjson, err := json.Marshal(re.usPrivacy) + rawjson, err := jsonutil.Marshal(re.usPrivacy) if err != nil { return nil, err } @@ -1216,7 +1234,7 @@ func (re *RegExt) marshal() (json.RawMessage, error) { if len(re.ext) == 0 { return nil, nil } - return json.Marshal(re.ext) + return jsonutil.Marshal(re.ext) } func (re *RegExt) Dirty() bool { @@ -1296,7 +1314,7 @@ func (se *SiteExt) unmarshal(extJson json.RawMessage) error { } ampJson, hasAmp := se.ext[ampKey] - if hasAmp { + if hasAmp && ampJson != nil { if err := jsonutil.Unmarshal(ampJson, &se.amp); err != nil { return errors.New(`request.site.ext.amp must be either 1, 0, or undefined`) } @@ -1308,7 +1326,7 @@ func (se *SiteExt) unmarshal(extJson json.RawMessage) error { func (se *SiteExt) marshal() (json.RawMessage, error) { if se.ampDirty { if se.amp != nil { - ampJson, err := json.Marshal(se.amp) + ampJson, err := jsonutil.Marshal(se.amp) if err != nil { return nil, err } @@ -1323,7 +1341,7 @@ func (se *SiteExt) marshal() (json.RawMessage, error) { if len(se.ext) == 0 { return nil, nil } - return json.Marshal(se.ext) + return jsonutil.Marshal(se.ext) } func (se *SiteExt) Dirty() bool { @@ -1391,7 +1409,7 @@ func (se *SourceExt) unmarshal(extJson json.RawMessage) error { } schainJson, hasSChain := se.ext[schainKey] - if hasSChain { + if hasSChain && schainJson != nil { if err := jsonutil.Unmarshal(schainJson, &se.schain); err != nil { return err } @@ -1403,7 +1421,7 @@ func (se *SourceExt) unmarshal(extJson json.RawMessage) error { func (se *SourceExt) marshal() (json.RawMessage, error) { if se.schainDirty { if se.schain != nil { - schainJson, err := json.Marshal(se.schain) + schainJson, err := jsonutil.Marshal(se.schain) if err != nil { return nil, err } @@ -1422,7 +1440,7 @@ func (se *SourceExt) marshal() (json.RawMessage, error) { if len(se.ext) == 0 { return nil, nil } - return json.Marshal(se.ext) + return jsonutil.Marshal(se.ext) } func (se *SourceExt) Dirty() bool { @@ -1558,6 +1576,8 @@ func (e *ImpExt) unmarshal(extJson json.RawMessage) error { prebidJson, hasPrebid := e.ext[prebidKey] if hasPrebid { e.prebid = &ExtImpPrebid{} + } + if prebidJson != nil { if err := jsonutil.Unmarshal(prebidJson, e.prebid); err != nil { return err } @@ -1566,20 +1586,22 @@ func (e *ImpExt) unmarshal(extJson json.RawMessage) error { dataJson, hasData := e.ext[dataKey] if hasData { e.data = &ExtImpData{} + } + if dataJson != nil { if err := jsonutil.Unmarshal(dataJson, e.data); err != nil { return err } } tidJson, hasTid := e.ext["tid"] - if hasTid { + if hasTid && tidJson != nil { if err := jsonutil.Unmarshal(tidJson, &e.tid); err != nil { return err } } gpIdJson, hasGpId := e.ext["gpid"] - if hasGpId { + if hasGpId && gpIdJson != nil { if err := jsonutil.Unmarshal(gpIdJson, &e.gpId); err != nil { return err } @@ -1591,7 +1613,7 @@ func (e *ImpExt) unmarshal(extJson json.RawMessage) error { func (e *ImpExt) marshal() (json.RawMessage, error) { if e.prebidDirty { if e.prebid != nil { - prebidJson, err := json.Marshal(e.prebid) + prebidJson, err := jsonutil.Marshal(e.prebid) if err != nil { return nil, err } @@ -1608,7 +1630,7 @@ func (e *ImpExt) marshal() (json.RawMessage, error) { if e.tidDirty { if len(e.tid) > 0 { - tidJson, err := json.Marshal(e.tid) + tidJson, err := jsonutil.Marshal(e.tid) if err != nil { return nil, err } @@ -1623,7 +1645,7 @@ func (e *ImpExt) marshal() (json.RawMessage, error) { if len(e.ext) == 0 { return nil, nil } - return json.Marshal(e.ext) + return jsonutil.Marshal(e.ext) } func (e *ImpExt) Dirty() bool { From 15054619c179ef27d41834c659f343ac9ef7e9a6 Mon Sep 17 00:00:00 2001 From: bsardo <1168933+bsardo@users.noreply.github.com> Date: Mon, 16 Oct 2023 17:17:52 -0400 Subject: [PATCH 05/10] Move json-iterator initialization to private package variables --- util/jsonutil/jsonutil.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/util/jsonutil/jsonutil.go b/util/jsonutil/jsonutil.go index 1d9e1827826..5c54a017b64 100644 --- a/util/jsonutil/jsonutil.go +++ b/util/jsonutil/jsonutil.go @@ -113,14 +113,21 @@ func DropElement(extension []byte, elementNames ...string) ([]byte, error) { return extension, nil } +// jsonConfigValidationOn attempts to maintain compatibility with the standard library which +// includes enabling validation +var jsonConfigValidationOn = jsoniter.ConfigCompatibleWithStandardLibrary + +// jsonConfigValidationOff disables validation +var jsonConfigValidationOff = jsoniter.Config{ + EscapeHTML: true, + SortMapKeys: true, + ValidateJsonRawMessage: false, +}.Froze() + // Unmarshal unmarshals a byte slice into the specified data structure without performing // any validation on the data. An unmarshal error is returned if a non-validation error occurs. func Unmarshal(data []byte, v interface{}) error { - err := jsoniter.Config{ - EscapeHTML: true, - SortMapKeys: true, - ValidateJsonRawMessage: false, - }.Froze().Unmarshal(data, v) + err := jsonConfigValidationOff.Unmarshal(data, v) if err != nil { return &errortypes.FailedToUnmarshal{ @@ -133,7 +140,7 @@ func Unmarshal(data []byte, v interface{}) error { // UnmarshalValid validates and unmarshals a byte slice into the specified data structure // returning an error if validation fails func UnmarshalValid(data []byte, v interface{}) error { - if err := jsoniter.ConfigCompatibleWithStandardLibrary.Unmarshal(data, v); err != nil { + if err := jsonConfigValidationOn.Unmarshal(data, v); err != nil { return &errortypes.FailedToUnmarshal{ Message: err.Error(), } @@ -144,7 +151,7 @@ func UnmarshalValid(data []byte, v interface{}) error { // Marshal marshals a data structure into a byte slice without performing any validation // on the data. A marshal error is returned if a non-validation error occurs. func Marshal(v interface{}) ([]byte, error) { - data, err := jsoniter.ConfigCompatibleWithStandardLibrary.Marshal(v) + data, err := jsonConfigValidationOn.Marshal(v) if err != nil { return nil, &errortypes.FailedToMarshal{ Message: err.Error(), From 138909a368121ebe161a54ed073e646fcc44bbb8 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 17 Oct 2023 12:42:00 -0400 Subject: [PATCH 06/10] Cookie Sync --- endpoints/cookie_sync.go | 3 ++- endpoints/cookie_sync_test.go | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/endpoints/cookie_sync.go b/endpoints/cookie_sync.go index 24530efa56a..170e81a997c 100644 --- a/endpoints/cookie_sync.go +++ b/endpoints/cookie_sync.go @@ -27,6 +27,7 @@ import ( gppPrivacy "github.com/prebid/prebid-server/privacy/gpp" "github.com/prebid/prebid-server/stored_requests" "github.com/prebid/prebid-server/usersync" + "github.com/prebid/prebid-server/util/jsonutil" stringutil "github.com/prebid/prebid-server/util/stringutil" ) @@ -119,7 +120,7 @@ func (c *cookieSyncEndpoint) parseRequest(r *http.Request) (usersync.Request, ma } request := cookieSyncRequest{} - if err := json.Unmarshal(body, &request); err != nil { + if err := jsonutil.Unmarshal(body, &request); err != nil { return usersync.Request{}, macros.UserSyncPrivacy{}, fmt.Errorf("JSON parsing failed: %s", err.Error()) } diff --git a/endpoints/cookie_sync_test.go b/endpoints/cookie_sync_test.go index 45f7a75da0c..096842e0671 100644 --- a/endpoints/cookie_sync_test.go +++ b/endpoints/cookie_sync_test.go @@ -190,14 +190,14 @@ func TestCookieSyncHandle(t *testing.T) { SyncersChosen: []usersync.SyncerChoice{{Bidder: "a", Syncer: &syncer}}, }, expectedStatusCode: 400, - expectedBody: `JSON parsing failed: invalid character 'm' looking for beginning of value` + "\n", + expectedBody: `JSON parsing failed: readObjectStart: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...` + "\n", setMetricsExpectations: func(m *metrics.MetricsEngineMock) { m.On("RecordCookieSync", metrics.CookieSyncBadRequest).Once() }, setAnalyticsExpectations: func(a *MockAnalyticsRunner) { expected := analytics.CookieSyncObject{ Status: 400, - Errors: []error{errors.New("JSON parsing failed: invalid character 'm' looking for beginning of value")}, + Errors: []error{errors.New("JSON parsing failed: readObjectStart: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...")}, BidderStatus: []*analytics.CookieSyncBidder{}, } a.On("LogCookieSyncObject", &expected).Once() @@ -805,7 +805,7 @@ func TestCookieSyncParseRequest(t *testing.T) { givenBody: strings.NewReader(`malformed`), givenGDPRConfig: config.GDPR{Enabled: true, DefaultValue: "0"}, givenCCPAEnabled: true, - expectedError: "JSON parsing failed: invalid character 'm' looking for beginning of value", + expectedError: "JSON parsing failed: readObjectStart: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...", }, { description: "Invalid Type Filter", From 230efba69d167fb756b7183314b408c44d4ab5ff Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 17 Oct 2023 13:13:01 -0400 Subject: [PATCH 07/10] Remaining JSON Un/Marshal For Non-Adapter Non-Test --- analytics/filesystem/file_module.go | 14 +++++++------- analytics/pubstack/helpers/json.go | 12 ++++++------ config/config.go | 3 ++- endpoints/currency_rates.go | 4 ++-- endpoints/events/vtrack.go | 7 ++++--- endpoints/events/vtrack_test.go | 5 +++-- endpoints/info/bidders.go | 11 +++++------ endpoints/info/bidders_detail.go | 5 +++-- endpoints/openrtb2/video_auction.go | 18 ++++++++---------- endpoints/version.go | 3 ++- modules/prebid/ortb2blocking/config.go | 2 +- openrtb_ext/imp_appnexus.go | 6 +++--- router/router.go | 2 +- usersync/cookie.go | 6 +++--- usersync/decoder.go | 8 ++++---- usersync/encoder.go | 5 +++-- 16 files changed, 57 insertions(+), 54 deletions(-) diff --git a/analytics/filesystem/file_module.go b/analytics/filesystem/file_module.go index f055e4470b1..24b7be6b599 100644 --- a/analytics/filesystem/file_module.go +++ b/analytics/filesystem/file_module.go @@ -2,12 +2,12 @@ package filesystem import ( "bytes" - "encoding/json" "fmt" "github.com/chasex/glog" "github.com/prebid/openrtb/v19/openrtb2" "github.com/prebid/prebid-server/analytics" + "github.com/prebid/prebid-server/util/jsonutil" ) type RequestType string @@ -120,7 +120,7 @@ func jsonifyAuctionObject(ao *analytics.AuctionObject) string { } } - b, err := json.Marshal(&struct { + b, err := jsonutil.Marshal(&struct { Type RequestType `json:"type"` *logAuction }{ @@ -153,7 +153,7 @@ func jsonifyVideoObject(vo *analytics.VideoObject) string { } } - b, err := json.Marshal(&struct { + b, err := jsonutil.Marshal(&struct { Type RequestType `json:"type"` *logVideo }{ @@ -178,7 +178,7 @@ func jsonifyCookieSync(cso *analytics.CookieSyncObject) string { } } - b, err := json.Marshal(&struct { + b, err := jsonutil.Marshal(&struct { Type RequestType `json:"type"` *logUserSync }{ @@ -205,7 +205,7 @@ func jsonifySetUIDObject(so *analytics.SetUIDObject) string { } } - b, err := json.Marshal(&struct { + b, err := jsonutil.Marshal(&struct { Type RequestType `json:"type"` *logSetUID }{ @@ -239,7 +239,7 @@ func jsonifyAmpObject(ao *analytics.AmpObject) string { } } - b, err := json.Marshal(&struct { + b, err := jsonutil.Marshal(&struct { Type RequestType `json:"type"` *logAMP }{ @@ -263,7 +263,7 @@ func jsonifyNotificationEventObject(ne *analytics.NotificationEvent) string { } } - b, err := json.Marshal(&struct { + b, err := jsonutil.Marshal(&struct { Type RequestType `json:"type"` *logNotificationEvent }{ diff --git a/analytics/pubstack/helpers/json.go b/analytics/pubstack/helpers/json.go index 368c79e3f6a..f56a4c6194b 100644 --- a/analytics/pubstack/helpers/json.go +++ b/analytics/pubstack/helpers/json.go @@ -1,11 +1,11 @@ package helpers import ( - "encoding/json" "fmt" "github.com/prebid/openrtb/v19/openrtb2" "github.com/prebid/prebid-server/analytics" + "github.com/prebid/prebid-server/util/jsonutil" ) func JsonifyAuctionObject(ao *analytics.AuctionObject, scope string) ([]byte, error) { @@ -26,7 +26,7 @@ func JsonifyAuctionObject(ao *analytics.AuctionObject, scope string) ([]byte, er } } - b, err := json.Marshal(&struct { + b, err := jsonutil.Marshal(&struct { Scope string `json:"scope"` *logAuction }{ @@ -59,7 +59,7 @@ func JsonifyVideoObject(vo *analytics.VideoObject, scope string) ([]byte, error) } } - b, err := json.Marshal(&struct { + b, err := jsonutil.Marshal(&struct { Scope string `json:"scope"` *logVideo }{ @@ -84,7 +84,7 @@ func JsonifyCookieSync(cso *analytics.CookieSyncObject, scope string) ([]byte, e } } - b, err := json.Marshal(&struct { + b, err := jsonutil.Marshal(&struct { Scope string `json:"scope"` *logUserSync }{ @@ -111,7 +111,7 @@ func JsonifySetUIDObject(so *analytics.SetUIDObject, scope string) ([]byte, erro } } - b, err := json.Marshal(&struct { + b, err := jsonutil.Marshal(&struct { Scope string `json:"scope"` *logSetUID }{ @@ -145,7 +145,7 @@ func JsonifyAmpObject(ao *analytics.AmpObject, scope string) ([]byte, error) { } } - b, err := json.Marshal(&struct { + b, err := jsonutil.Marshal(&struct { Scope string `json:"scope"` *logAMP }{ diff --git a/config/config.go b/config/config.go index cf3591dd564..4bd7f807002 100644 --- a/config/config.go +++ b/config/config.go @@ -14,6 +14,7 @@ import ( "github.com/prebid/openrtb/v19/openrtb2" "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/spf13/viper" ) @@ -775,7 +776,7 @@ func New(v *viper.Viper, bidderInfos BidderInfos, normalizeBidderName func(strin // MarshalAccountDefaults compiles AccountDefaults into the JSON format used for merge patch func (cfg *Configuration) MarshalAccountDefaults() error { var err error - if cfg.accountDefaultsJSON, err = json.Marshal(cfg.AccountDefaults); err != nil { + if cfg.accountDefaultsJSON, err = jsonutil.Marshal(cfg.AccountDefaults); err != nil { glog.Warningf("converting %+v to json: %v", cfg.AccountDefaults, err) } return err diff --git a/endpoints/currency_rates.go b/endpoints/currency_rates.go index d35cb74cea4..49ae9963cd9 100644 --- a/endpoints/currency_rates.go +++ b/endpoints/currency_rates.go @@ -1,12 +1,12 @@ package endpoints import ( - "encoding/json" "net/http" "time" "github.com/golang/glog" "github.com/prebid/prebid-server/currency" + "github.com/prebid/prebid-server/util/jsonutil" ) // currencyRatesInfo holds currency rates information. @@ -60,7 +60,7 @@ func NewCurrencyRatesEndpoint(rateConverter rateConverter, fetchingInterval time currencyRateInfo := newCurrencyRatesInfo(rateConverter, fetchingInterval) return func(w http.ResponseWriter, _ *http.Request) { - jsonOutput, err := json.Marshal(currencyRateInfo) + jsonOutput, err := jsonutil.Marshal(currencyRateInfo) if err != nil { glog.Errorf("/currency/rates Critical error when trying to marshal currencyRateInfo: %v", err) w.WriteHeader(http.StatusInternalServerError) diff --git a/endpoints/events/vtrack.go b/endpoints/events/vtrack.go index e0bf3fd4be8..eeb409e24ae 100644 --- a/endpoints/events/vtrack.go +++ b/endpoints/events/vtrack.go @@ -19,6 +19,7 @@ import ( "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/prebid_cache_client" "github.com/prebid/prebid-server/stored_requests" + "github.com/prebid/prebid-server/util/jsonutil" ) const ( @@ -123,7 +124,7 @@ func (v *vtrackEndpoint) Handle(w http.ResponseWriter, r *http.Request, _ httpro } } - d, err := json.Marshal(*cachingResponse) + d, err := jsonutil.Marshal(*cachingResponse) if err != nil { w.WriteHeader(http.StatusInternalServerError) @@ -187,7 +188,7 @@ func ParseVTrackRequest(httpRequest *http.Request, maxRequestSize int64) (req *B return req, err } - if err := json.Unmarshal(requestJson, req); err != nil { + if err := jsonutil.UnmarshalValid(requestJson, req); err != nil { return req, err } @@ -319,7 +320,7 @@ func ModifyVastXmlString(externalUrl, vast, bidid, bidder, accountID string, tim // ModifyVastXmlJSON modifies BidCacheRequest element Vast XML data func ModifyVastXmlJSON(externalUrl string, data json.RawMessage, bidid, bidder, accountId string, timestamp int64, integrationType string) json.RawMessage { var vast string - if err := json.Unmarshal(data, &vast); err != nil { + if err := jsonutil.Unmarshal(data, &vast); err != nil { // failed to decode json, fall back to string vast = string(data) } diff --git a/endpoints/events/vtrack_test.go b/endpoints/events/vtrack_test.go index 1e36a9e627f..d950a443afe 100644 --- a/endpoints/events/vtrack_test.go +++ b/endpoints/events/vtrack_test.go @@ -16,6 +16,7 @@ import ( "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/prebid_cache_client" "github.com/prebid/prebid-server/stored_requests" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/stretchr/testify/assert" ) @@ -186,7 +187,7 @@ func TestShouldRespondWithBadRequestWhenBidIdIsMissing(t *testing.T) { }, } - reqData, err := json.Marshal(data) + reqData, err := jsonutil.Marshal(data) if err != nil { t.Fatal(err) } @@ -239,7 +240,7 @@ func TestShouldRespondWithBadRequestWhenBidderIsMissing(t *testing.T) { }, } - reqData, err := json.Marshal(data) + reqData, err := jsonutil.Marshal(data) if err != nil { t.Fatal(err) } diff --git a/endpoints/info/bidders.go b/endpoints/info/bidders.go index 9984abe216d..bd6d078b3ba 100644 --- a/endpoints/info/bidders.go +++ b/endpoints/info/bidders.go @@ -1,7 +1,6 @@ package info import ( - "encoding/json" "net/http" "sort" "strings" @@ -9,6 +8,7 @@ import ( "github.com/golang/glog" "github.com/julienschmidt/httprouter" "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/util/jsonutil" ) var invalidEnabledOnlyMsg = []byte(`Invalid value for 'enabledonly' query param, must be of boolean type`) @@ -103,7 +103,7 @@ func prepareBiddersResponseAll(bidders config.BidderInfos, aliases map[string]st } sort.Strings(bidderNames) - return json.Marshal(bidderNames) + return jsonutil.Marshal(bidderNames) } func prepareBiddersResponseAllBaseOnly(bidders config.BidderInfos) ([]byte, error) { @@ -116,7 +116,7 @@ func prepareBiddersResponseAllBaseOnly(bidders config.BidderInfos) ([]byte, erro } sort.Strings(bidderNames) - return json.Marshal(bidderNames) + return jsonutil.Marshal(bidderNames) } func prepareBiddersResponseEnabledOnly(bidders config.BidderInfos, aliases map[string]string) ([]byte, error) { @@ -135,8 +135,7 @@ func prepareBiddersResponseEnabledOnly(bidders config.BidderInfos, aliases map[s } sort.Strings(bidderNames) - - return json.Marshal(bidderNames) + return jsonutil.Marshal(bidderNames) } func prepareBiddersResponseEnabledOnlyBaseOnly(bidders config.BidderInfos) ([]byte, error) { @@ -149,7 +148,7 @@ func prepareBiddersResponseEnabledOnlyBaseOnly(bidders config.BidderInfos) ([]by } sort.Strings(bidderNames) - return json.Marshal(bidderNames) + return jsonutil.Marshal(bidderNames) } func writeBadRequest(w http.ResponseWriter, data []byte) { diff --git a/endpoints/info/bidders_detail.go b/endpoints/info/bidders_detail.go index d9dc776f50d..34d14efde1b 100644 --- a/endpoints/info/bidders_detail.go +++ b/endpoints/info/bidders_detail.go @@ -10,6 +10,7 @@ import ( "github.com/julienschmidt/httprouter" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/util/jsonutil" ) const ( @@ -101,7 +102,7 @@ func marshalDetailsResponse(details map[string]bidderDetail) (map[string][]byte, responses := map[string][]byte{} for bidder, detail := range details { - json, err := json.Marshal(detail) + json, err := jsonutil.Marshal(detail) if err != nil { return nil, fmt.Errorf("unable to marshal info for bidder %s: %v", bidder, err) } @@ -118,7 +119,7 @@ func marshalAllResponse(responses map[string][]byte) ([]byte, error) { responsesJSON[k] = json.RawMessage(v) } - json, err := json.Marshal(responsesJSON) + json, err := jsonutil.Marshal(responsesJSON) if err != nil { return nil, fmt.Errorf("unable to marshal info for bidder all: %v", err) } diff --git a/endpoints/openrtb2/video_auction.go b/endpoints/openrtb2/video_auction.go index 8faf15543d3..94455e0cb52 100644 --- a/endpoints/openrtb2/video_auction.go +++ b/endpoints/openrtb2/video_auction.go @@ -2,7 +2,6 @@ package openrtb2 import ( "context" - "encoding/json" "errors" "fmt" "io" @@ -36,6 +35,7 @@ import ( "github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher" "github.com/prebid/prebid-server/usersync" "github.com/prebid/prebid-server/util/iputil" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/util/ptrutil" "github.com/prebid/prebid-server/util/uuidutil" "github.com/prebid/prebid-server/version" @@ -179,7 +179,7 @@ func (deps *endpointDeps) VideoAuctionEndpoint(w http.ResponseWriter, r *http.Re resolvedRequest := requestJson if debugLog.DebugEnabledOrOverridden { debugLog.Data.Request = string(requestJson) - if headerBytes, err := json.Marshal(r.Header); err == nil { + if headerBytes, err := jsonutil.Marshal(r.Header); err == nil { debugLog.Data.Headers = string(headerBytes) } else { debugLog.Data.Headers = fmt.Sprintf("Unable to marshal headers data: %s", err.Error()) @@ -219,7 +219,7 @@ func (deps *endpointDeps) VideoAuctionEndpoint(w http.ResponseWriter, r *http.Re var bidReq = &openrtb2.BidRequest{} if deps.defaultRequest { - if err := json.Unmarshal(deps.defReqJSON, bidReq); err != nil { + if err := jsonutil.UnmarshalValid(deps.defReqJSON, bidReq); err != nil { err = fmt.Errorf("Invalid JSON in Default Request Settings: %s", err) handleError(&labels, w, []error{err}, &vo, &debugLog) return @@ -374,8 +374,7 @@ func (deps *endpointDeps) VideoAuctionEndpoint(w http.ResponseWriter, r *http.Re vo.VideoResponse = bidResp - resp, err := json.Marshal(bidResp) - //resp, err := json.Marshal(response) + resp, err := jsonutil.Marshal(bidResp) if err != nil { errL := []error{err} handleError(&labels, w, errL, &vo, &debugLog) @@ -384,7 +383,6 @@ func (deps *endpointDeps) VideoAuctionEndpoint(w http.ResponseWriter, r *http.Re w.Header().Set("Content-Type", "application/json") w.Write(resp) - } func cleanupVideoBidRequest(videoReq *openrtb_ext.BidRequestVideo, podErrors []PodError) *openrtb_ext.BidRequestVideo { @@ -507,7 +505,7 @@ func (deps *endpointDeps) loadStoredImp(storedImpId string) (openrtb2.Imp, []err return impr, err } - if err := json.Unmarshal(imp[storedImpId], &impr); err != nil { + if err := jsonutil.UnmarshalValid(imp[storedImpId], &impr); err != nil { return impr, []error{err} } return impr, nil @@ -536,7 +534,7 @@ func buildVideoResponse(bidresponse *openrtb2.BidResponse, podErrors []PodError) anyBidsReturned = true var tempRespBidExt openrtb_ext.ExtBid - if err := json.Unmarshal(bid.Ext, &tempRespBidExt); err != nil { + if err := jsonutil.UnmarshalValid(bid.Ext, &tempRespBidExt); err != nil { return nil, err } if tempRespBidExt.Prebid.Targeting[formatTargetingKey(openrtb_ext.HbVastCacheKey, seatBid.Seat)] == "" { @@ -705,13 +703,13 @@ func createBidExtension(videoRequest *openrtb_ext.BidRequestVideo) ([]byte, erro } extReq := openrtb_ext.ExtRequest{Prebid: prebid} - return json.Marshal(extReq) + return jsonutil.Marshal(extReq) } func (deps *endpointDeps) parseVideoRequest(request []byte, headers http.Header) (req *openrtb_ext.BidRequestVideo, errs []error, podErrors []PodError) { req = &openrtb_ext.BidRequestVideo{} - if err := json.Unmarshal(request, &req); err != nil { + if err := jsonutil.UnmarshalValid(request, &req); err != nil { errs = []error{err} return } diff --git a/endpoints/version.go b/endpoints/version.go index 00d894963e6..0a837fd8bc3 100644 --- a/endpoints/version.go +++ b/endpoints/version.go @@ -5,6 +5,7 @@ import ( "net/http" "github.com/golang/glog" + "github.com/prebid/prebid-server/util/jsonutil" ) const versionEndpointValueNotSet = "not-set" @@ -29,7 +30,7 @@ func prepareVersionEndpointResponse(version, revision string) (json.RawMessage, revision = versionEndpointValueNotSet } - return json.Marshal(struct { + return jsonutil.Marshal(struct { Revision string `json:"revision"` Version string `json:"version"` }{ diff --git a/modules/prebid/ortb2blocking/config.go b/modules/prebid/ortb2blocking/config.go index 31bc9196c13..74841eb5b33 100644 --- a/modules/prebid/ortb2blocking/config.go +++ b/modules/prebid/ortb2blocking/config.go @@ -113,7 +113,7 @@ type Override struct { func (o *Override) UnmarshalJSON(bytes []byte) error { var overrideData interface{} - if err := json.Unmarshal(bytes, &overrideData); err != nil { + if err := jsonutil.UnmarshalValid(bytes, &overrideData); err != nil { return err } diff --git a/openrtb_ext/imp_appnexus.go b/openrtb_ext/imp_appnexus.go index d9549e74750..02476d6bcf0 100644 --- a/openrtb_ext/imp_appnexus.go +++ b/openrtb_ext/imp_appnexus.go @@ -45,7 +45,7 @@ func (ks *ExtImpAppnexusKeywords) UnmarshalJSON(b []byte) error { switch b[0] { case '{': var results map[string][]string - if err := json.Unmarshal(b, &results); err != nil { + if err := jsonutil.UnmarshalValid(b, &results); err != nil { return err } @@ -64,7 +64,7 @@ func (ks *ExtImpAppnexusKeywords) UnmarshalJSON(b []byte) error { } case '[': var results []extImpAppnexusKeyVal - if err := json.Unmarshal(b, &results); err != nil { + if err := jsonutil.UnmarshalValid(b, &results); err != nil { return err } var kvs strings.Builder @@ -82,7 +82,7 @@ func (ks *ExtImpAppnexusKeywords) UnmarshalJSON(b []byte) error { } case '"': var keywords string - if err := json.Unmarshal(b, &keywords); err != nil { + if err := jsonutil.UnmarshalValid(b, &keywords); err != nil { return err } *ks = ExtImpAppnexusKeywords(keywords) diff --git a/router/router.go b/router/router.go index ed1fb4647a2..70b7860d661 100644 --- a/router/router.go +++ b/router/router.go @@ -92,7 +92,7 @@ func newJsonDirectoryServer(schemaDirectory string, validator openrtb_ext.Bidder data[aliasName] = bidderData } - response, err := json.Marshal(data) + response, err := jsonutil.Marshal(data) if err != nil { glog.Fatalf("Failed to marshal bidder param JSON-schema: %v", err) } diff --git a/usersync/cookie.go b/usersync/cookie.go index 94ada94ed75..88524018c49 100644 --- a/usersync/cookie.go +++ b/usersync/cookie.go @@ -1,13 +1,13 @@ package usersync import ( - "encoding/json" "errors" "net/http" "time" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/util/jsonutil" ) const uidCookieName = "uids" @@ -226,7 +226,7 @@ type cookieJson struct { } func (cookie *Cookie) MarshalJSON() ([]byte, error) { - return json.Marshal(cookieJson{ + return jsonutil.Marshal(cookieJson{ UIDs: cookie.uids, OptOut: cookie.optOut, }) @@ -234,7 +234,7 @@ func (cookie *Cookie) MarshalJSON() ([]byte, error) { func (cookie *Cookie) UnmarshalJSON(b []byte) error { var cookieContract cookieJson - if err := json.Unmarshal(b, &cookieContract); err != nil { + if err := jsonutil.Unmarshal(b, &cookieContract); err != nil { return err } diff --git a/usersync/decoder.go b/usersync/decoder.go index 3ff13aa3242..c803fbe2a52 100644 --- a/usersync/decoder.go +++ b/usersync/decoder.go @@ -2,12 +2,12 @@ package usersync import ( "encoding/base64" - "encoding/json" + + "github.com/prebid/prebid-server/util/jsonutil" ) type Decoder interface { - // Decode takes an encoded string and decodes it into a cookie - Decode(v string) *Cookie + Decode(encodedValue string) *Cookie } type Base64Decoder struct{} @@ -19,7 +19,7 @@ func (d Base64Decoder) Decode(encodedValue string) *Cookie { } var cookie Cookie - if err = json.Unmarshal(jsonValue, &cookie); err != nil { + if err = jsonutil.UnmarshalValid(jsonValue, &cookie); err != nil { return NewCookie() } diff --git a/usersync/encoder.go b/usersync/encoder.go index eef7e2ef34f..74472f23307 100644 --- a/usersync/encoder.go +++ b/usersync/encoder.go @@ -2,7 +2,8 @@ package usersync import ( "encoding/base64" - "encoding/json" + + "github.com/prebid/prebid-server/util/jsonutil" ) type Encoder interface { @@ -13,7 +14,7 @@ type Encoder interface { type Base64Encoder struct{} func (e Base64Encoder) Encode(c *Cookie) (string, error) { - j, err := json.Marshal(c) + j, err := jsonutil.Marshal(c) if err != nil { return "", err } From 72092f2d6d81bf8501a5082b2ea0b1be0e11fc8b Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 17 Oct 2023 13:14:08 -0400 Subject: [PATCH 08/10] Update Cookie To UnmarshalValid --- endpoints/cookie_sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/endpoints/cookie_sync.go b/endpoints/cookie_sync.go index 170e81a997c..ff801de988f 100644 --- a/endpoints/cookie_sync.go +++ b/endpoints/cookie_sync.go @@ -120,7 +120,7 @@ func (c *cookieSyncEndpoint) parseRequest(r *http.Request) (usersync.Request, ma } request := cookieSyncRequest{} - if err := jsonutil.Unmarshal(body, &request); err != nil { + if err := jsonutil.UnmarshalValid(body, &request); err != nil { return usersync.Request{}, macros.UserSyncPrivacy{}, fmt.Errorf("JSON parsing failed: %s", err.Error()) } From 5a2f3b19dbc624454f10ca57127d01fe3dd66696 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Wed, 18 Oct 2023 01:43:42 -0400 Subject: [PATCH 09/10] Extended To Tests --- adservertargeting/adservertargeting_test.go | 7 +- currency/rates_test.go | 27 ++++--- endpoints/cookie_sync_test.go | 6 +- endpoints/openrtb2/amp_auction_test.go | 59 +++++++-------- endpoints/openrtb2/auction_test.go | 37 +++++----- .../invalid-stored/bad_incoming_1.json | 2 +- .../invalid-stored/bad_stored_req.json | 2 +- .../invalid-whole/interstital-bad-perc.json | 2 +- .../invalid-whole/invalid-source.json | 2 +- .../invalid-whole/malformed-bid-request.json | 2 +- .../invalid-whole/regs-ext-malformed.json | 2 +- .../invalid-whole/user-ext-consent-int.json | 2 +- .../user-gdpr-consent-invalid.json | 2 +- endpoints/openrtb2/test_utils.go | 15 ++-- endpoints/openrtb2/video_auction_test.go | 21 +++--- exchange/auction_test.go | 10 +-- exchange/bidder_test.go | 7 +- exchange/exchange_test.go | 41 +++++------ exchange/targeting_test.go | 5 +- exchange/utils.go | 2 +- exchange/utils_test.go | 33 ++++----- firstpartydata/first_party_data_test.go | 21 +++--- floors/floors_test.go | 3 +- floors/rule_test.go | 2 +- gdpr/full_enforcement_test.go | 4 +- gdpr/vendorlist-fetching_test.go | 4 +- hooks/hookanalytics/analytics_test.go | 4 +- hooks/hookexecution/enricher_test.go | 5 +- hooks/hookexecution/test_utils.go | 5 +- hooks/plan_test.go | 18 ++--- modules/prebid/ortb2blocking/module_test.go | 2 +- openrtb_ext/convert_up_test.go | 12 ++-- openrtb_ext/device_test.go | 25 +++---- openrtb_ext/imp_appnexus_test.go | 6 +- openrtb_ext/request_test.go | 3 +- openrtb_ext/site_test.go | 18 ++--- ortb/default_test.go | 7 +- prebid_cache_client/client_test.go | 3 +- router/router_test.go | 3 +- schain/schainwriter_test.go | 3 +- .../backends/file_fetcher/fetcher_test.go | 7 +- .../backends/http_fetcher/fetcher_test.go | 7 +- stored_requests/events/http/http_test.go | 6 +- util/jsonutil/jsonutil.go | 72 ++++++++++++++++--- util/jsonutil/jsonutil_test.go | 61 +++++++++++++++- util/jsonutil/stringInt_test.go | 11 ++- version/xprebidheader_test.go | 6 +- 47 files changed, 370 insertions(+), 234 deletions(-) diff --git a/adservertargeting/adservertargeting_test.go b/adservertargeting/adservertargeting_test.go index 4a651fdd2be..0da3635d9b7 100644 --- a/adservertargeting/adservertargeting_test.go +++ b/adservertargeting/adservertargeting_test.go @@ -8,6 +8,7 @@ import ( "github.com/prebid/openrtb/v19/openrtb2" "github.com/prebid/openrtb/v19/openrtb3" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/stretchr/testify/assert" ) @@ -42,7 +43,7 @@ func TestExtractAdServerTargeting(t *testing.T) { p := "https://www.test-url.com?ampkey=testAmpKey&data-override-height=400" u, _ := url.Parse(p) params := u.Query() - reqBytes, err := json.Marshal(r) + reqBytes, err := jsonutil.Marshal(r) assert.NoError(t, err, "unexpected req marshal error") res, warnings := collect(rw, reqBytes, params) @@ -248,7 +249,7 @@ func TestProcessAdServerTargetingFull(t *testing.T) { bidResponseExt := &openrtb_ext.ExtBidResponse{Warnings: make(map[openrtb_ext.BidderName][]openrtb_ext.ExtBidderMessage)} - reqBytes, err := json.Marshal(r) + reqBytes, err := jsonutil.Marshal(r) assert.NoError(t, err, "unexpected req marshal error") targetingKeyLen := 0 resResp := Apply(rw, reqBytes, resp, params, bidResponseExt, &targetingKeyLen) @@ -331,7 +332,7 @@ func TestProcessAdServerTargetingWarnings(t *testing.T) { bidResponseExt := &openrtb_ext.ExtBidResponse{Warnings: make(map[openrtb_ext.BidderName][]openrtb_ext.ExtBidderMessage)} - reqBytes, err := json.Marshal(r) + reqBytes, err := jsonutil.Marshal(r) assert.NoError(t, err, "unexpected req marshal error") resResp := Apply(rw, reqBytes, resp, params, bidResponseExt, nil) assert.Len(t, resResp.SeatBid, 2, "Incorrect response: seat bid number") diff --git a/currency/rates_test.go b/currency/rates_test.go index 23226dce8fb..86c25d14ac0 100644 --- a/currency/rates_test.go +++ b/currency/rates_test.go @@ -1,11 +1,12 @@ package currency import ( - "encoding/json" "errors" "testing" "github.com/stretchr/testify/assert" + + "github.com/prebid/prebid-server/util/jsonutil" ) func TestUnMarshallRates(t *testing.T) { @@ -22,7 +23,7 @@ func TestUnMarshallRates(t *testing.T) { ratesJSON: `malformed`, expectedRates: Rates{}, expectsError: true, - expectedError: errors.New("invalid character 'm' looking for beginning of value"), + expectedError: errors.New("expect { or n, but found m"), }, { desc: "Valid JSON field defining valid conversion object. Expect no error", @@ -50,7 +51,7 @@ func TestUnMarshallRates(t *testing.T) { expectedError: nil, }, { - desc: "Valid JSON field defines a conversions map with repeated entries, expect error", + desc: "Valid JSON field defines a conversions map with repeated entries, last one wins", ratesJSON: `{ "conversions":{ "USD":{ @@ -58,25 +59,31 @@ func TestUnMarshallRates(t *testing.T) { "MXN":20.00 }, "USD":{ - "GBP":0.7662523901 - }, + "GBP":0.4815162342 + } } }`, - expectedRates: Rates{}, - expectsError: true, - expectedError: errors.New("invalid character '}' looking for beginning of object key string"), + expectedRates: Rates{ + Conversions: map[string]map[string]float64{ + "USD": { + "GBP": 0.4815162342, + }, + }, + }, + expectsError: false, + expectedError: nil, }, } for _, tc := range testCases { // Execute: updatedRates := Rates{} - err := json.Unmarshal([]byte(tc.ratesJSON), &updatedRates) + err := jsonutil.UnmarshalValid([]byte(tc.ratesJSON), &updatedRates) // Verify: assert.Equal(t, err != nil, tc.expectsError, tc.desc) if tc.expectsError { - assert.Equal(t, err.Error(), tc.expectedError.Error(), tc.desc) + assert.Equal(t, tc.expectedError.Error(), err.Error(), tc.desc) } assert.Equal(t, tc.expectedRates, updatedRates, tc.desc) } diff --git a/endpoints/cookie_sync_test.go b/endpoints/cookie_sync_test.go index 096842e0671..ed3ada59625 100644 --- a/endpoints/cookie_sync_test.go +++ b/endpoints/cookie_sync_test.go @@ -190,14 +190,14 @@ func TestCookieSyncHandle(t *testing.T) { SyncersChosen: []usersync.SyncerChoice{{Bidder: "a", Syncer: &syncer}}, }, expectedStatusCode: 400, - expectedBody: `JSON parsing failed: readObjectStart: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...` + "\n", + expectedBody: `JSON parsing failed: expect { or n, but found m` + "\n", setMetricsExpectations: func(m *metrics.MetricsEngineMock) { m.On("RecordCookieSync", metrics.CookieSyncBadRequest).Once() }, setAnalyticsExpectations: func(a *MockAnalyticsRunner) { expected := analytics.CookieSyncObject{ Status: 400, - Errors: []error{errors.New("JSON parsing failed: readObjectStart: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...")}, + Errors: []error{errors.New("JSON parsing failed: expect { or n, but found m")}, BidderStatus: []*analytics.CookieSyncBidder{}, } a.On("LogCookieSyncObject", &expected).Once() @@ -805,7 +805,7 @@ func TestCookieSyncParseRequest(t *testing.T) { givenBody: strings.NewReader(`malformed`), givenGDPRConfig: config.GDPR{Enabled: true, DefaultValue: "0"}, givenCCPAEnabled: true, - expectedError: "JSON parsing failed: readObjectStart: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...", + expectedError: "JSON parsing failed: expect { or n, but found m", }, { description: "Invalid Type Filter", diff --git a/endpoints/openrtb2/amp_auction_test.go b/endpoints/openrtb2/amp_auction_test.go index 77d56803d2f..e8d8816a900 100644 --- a/endpoints/openrtb2/amp_auction_test.go +++ b/endpoints/openrtb2/amp_auction_test.go @@ -32,6 +32,7 @@ import ( "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/privacy" "github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher" + "github.com/prebid/prebid-server/util/jsonutil" ) // TestGoodRequests makes sure that the auction runs properly-formatted stored bids correctly. @@ -74,7 +75,7 @@ func TestGoodAmpRequests(t *testing.T) { } test := testCase{} - if !assert.NoError(t, json.Unmarshal(fileJsonData, &test), "Failed to unmarshal data from file: %s. Error: %v", filename, err) { + if !assert.NoError(t, jsonutil.UnmarshalValid(fileJsonData, &test), "Failed to unmarshal data from file: %s. Error: %v", filename, err) { continue } @@ -127,7 +128,7 @@ func TestGoodAmpRequests(t *testing.T) { } if test.ExpectedValidatedBidReq != nil { // compare as json to ignore whitespace and ext field ordering - actualJson, err := json.Marshal(ex.actualValidatedBidReq) + actualJson, err := jsonutil.Marshal(ex.actualValidatedBidReq) if assert.NoError(t, err, "Error converting actual bid request to json. Test file: %s", filename) { assert.JSONEq(t, string(test.ExpectedValidatedBidReq), string(actualJson), "Not the expected validated request. Test file: %s", filename) } @@ -156,7 +157,7 @@ func TestAccountErrors(t *testing.T) { } test := testCase{} - if !assert.NoError(t, json.Unmarshal(fileJsonData, &test), "Failed to unmarshal data from file: %s. Error: %v", tt.filename, err) { + if !assert.NoError(t, jsonutil.UnmarshalValid(fileJsonData, &test), "Failed to unmarshal data from file: %s. Error: %v", tt.filename, err) { continue } test.StoredRequest = map[string]json.RawMessage{tt.storedReqID: test.BidRequest} @@ -322,7 +323,7 @@ func TestGDPRConsent(t *testing.T) { // Parse Response var response AmpResponse - if err := json.Unmarshal(responseRecorder.Body.Bytes(), &response); err != nil { + if err := jsonutil.UnmarshalValid(responseRecorder.Body.Bytes(), &response); err != nil { t.Fatalf("Error unmarshalling response: %s", err.Error()) } @@ -338,7 +339,7 @@ func TestGDPRConsent(t *testing.T) { return } var ue openrtb_ext.ExtUser - err = json.Unmarshal(result.User.Ext, &ue) + err = jsonutil.UnmarshalValid(result.User.Ext, &ue) if !assert.NoError(t, err, test.description+":deserialize") { return } @@ -353,7 +354,7 @@ func TestGDPRConsent(t *testing.T) { // Parse Resonse var responseLegacy AmpResponse - if err := json.Unmarshal(responseRecorderLegacy.Body.Bytes(), &responseLegacy); err != nil { + if err := jsonutil.UnmarshalValid(responseRecorderLegacy.Body.Bytes(), &responseLegacy); err != nil { t.Fatalf("Error unmarshalling response: %s", err.Error()) } @@ -369,7 +370,7 @@ func TestGDPRConsent(t *testing.T) { return } var ueLegacy openrtb_ext.ExtUser - err = json.Unmarshal(resultLegacy.User.Ext, &ueLegacy) + err = jsonutil.UnmarshalValid(resultLegacy.User.Ext, &ueLegacy) if !assert.NoError(t, err, test.description+":legacy:deserialize") { return } @@ -516,7 +517,7 @@ func TestOverrideWithParams(t *testing.T) { Site: &openrtb2.Site{Ext: json.RawMessage(`{"amp":1}`)}, User: &openrtb2.User{Ext: json.RawMessage(`malformed`)}, }, - errorMsgs: []string{"ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|..."}, + errorMsgs: []string{"expect { or n, but found m"}, expectFatalErrors: true, }, }, @@ -564,7 +565,7 @@ func TestOverrideWithParams(t *testing.T) { User: &openrtb2.User{Ext: json.RawMessage(`{"prebid":{malformed}}`)}, Site: &openrtb2.Site{Ext: json.RawMessage(`{"amp":1}`)}, }, - errorMsgs: []string{"ReadObjectCB: expect \" after {, but found m, error found in #10 byte of ...|prebid\":{malformed}}|..., bigger context ...|{\"prebid\":{malformed}}|..."}, + errorMsgs: []string{"expect \" after {, but found m"}, expectFatalErrors: true, }, }, @@ -746,7 +747,7 @@ func TestCCPAConsent(t *testing.T) { // Parse Response var response AmpResponse - if err := json.Unmarshal(responseRecorder.Body.Bytes(), &response); err != nil { + if err := jsonutil.UnmarshalValid(responseRecorder.Body.Bytes(), &response); err != nil { t.Fatalf("Error unmarshalling response: %s", err.Error()) } @@ -762,7 +763,7 @@ func TestCCPAConsent(t *testing.T) { return } var re openrtb_ext.ExtRegs - err = json.Unmarshal(result.Regs.Ext, &re) + err = jsonutil.UnmarshalValid(result.Regs.Ext, &re) if !assert.NoError(t, err, test.description+":deserialize") { return } @@ -868,7 +869,7 @@ func TestConsentWarnings(t *testing.T) { // Parse Response var response AmpResponse - if err := json.Unmarshal(responseRecorder.Body.Bytes(), &response); err != nil { + if err := jsonutil.UnmarshalValid(responseRecorder.Body.Bytes(), &response); err != nil { t.Fatalf("Error unmarshalling response: %s", err.Error()) } @@ -959,7 +960,7 @@ func TestNewAndLegacyConsentBothProvided(t *testing.T) { // Parse Response var response AmpResponse - if err := json.Unmarshal(responseRecorder.Body.Bytes(), &response); err != nil { + if err := jsonutil.UnmarshalValid(responseRecorder.Body.Bytes(), &response); err != nil { t.Fatalf("Error unmarshalling response: %s", err.Error()) } @@ -975,7 +976,7 @@ func TestNewAndLegacyConsentBothProvided(t *testing.T) { return } var ue openrtb_ext.ExtUser - err = json.Unmarshal(result.User.Ext, &ue) + err = jsonutil.UnmarshalValid(result.User.Ext, &ue) if !assert.NoError(t, err, test.description+":deserialize") { return } @@ -1096,7 +1097,7 @@ func TestAmpDebug(t *testing.T) { } var response AmpResponse - if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + if err := jsonutil.UnmarshalValid(recorder.Body.Bytes(), &response); err != nil { t.Fatalf("Error unmarshalling response: %s", err.Error()) } @@ -1131,7 +1132,7 @@ func TestInitAmpTargetingAndCache(t *testing.T) { { name: "malformed", request: &openrtb2.BidRequest{Ext: json.RawMessage("malformed")}, - expectedErrs: []string{"ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|..."}, + expectedErrs: []string{"expect { or n, but found m"}, }, { name: "nil", @@ -1237,12 +1238,12 @@ func TestQueryParamOverrides(t *testing.T) { } var response AmpResponse - if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + if err := jsonutil.UnmarshalValid(recorder.Body.Bytes(), &response); err != nil { t.Fatalf("Error unmarshalling response: %s", err.Error()) } var resolvedRequest openrtb2.BidRequest - err := json.Unmarshal(response.ORTB2.Ext.Debug.ResolvedRequest, &resolvedRequest) + err := jsonutil.UnmarshalValid(response.ORTB2.Ext.Debug.ResolvedRequest, &resolvedRequest) assert.NoError(t, err, "resolved request should have a correct format") if resolvedRequest.TMax != timeout { t.Errorf("Expected TMax to equal timeout (%d), got: %d", timeout, resolvedRequest.TMax) @@ -1388,11 +1389,11 @@ func (s formatOverrideSpec) execute(t *testing.T) { t.Errorf("Request was: %s", string(requests["1"])) } var response AmpResponse - if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + if err := jsonutil.UnmarshalValid(recorder.Body.Bytes(), &response); err != nil { t.Fatalf("Error unmarshalling response: %s", err.Error()) } var resolvedRequest openrtb2.BidRequest - err := json.Unmarshal(response.ORTB2.Ext.Debug.ResolvedRequest, &resolvedRequest) + err := jsonutil.UnmarshalValid(response.ORTB2.Ext.Debug.ResolvedRequest, &resolvedRequest) assert.NoError(t, err, "resolved request should have the correct format") formats := resolvedRequest.Imp[0].Banner.Format if len(formats) != len(s.expect) { @@ -1442,14 +1443,14 @@ func (m *mockAmpExchange) HoldAuction(ctx context.Context, auctionRequest *excha if len(auctionRequest.StoredAuctionResponses) > 0 { var seatBids []openrtb2.SeatBid - if err := json.Unmarshal(auctionRequest.StoredAuctionResponses[r.BidRequest.Imp[0].ID], &seatBids); err != nil { + if err := jsonutil.UnmarshalValid(auctionRequest.StoredAuctionResponses[r.BidRequest.Imp[0].ID], &seatBids); err != nil { return nil, err } response.SeatBid = seatBids } if r.BidRequest.Test == 1 { - resolvedRequest, err := json.Marshal(r.BidRequest) + resolvedRequest, err := jsonutil.Marshal(r.BidRequest) if err != nil { resolvedRequest = json.RawMessage("{}") } @@ -1508,7 +1509,7 @@ func getTestBidRequest(nilUser bool, userExt *openrtb_ext.ExtUser, nilRegs bool, var userExtData []byte if userExt != nil { var err error - userExtData, err = json.Marshal(userExt) + userExtData, err = jsonutil.Marshal(userExt) if err != nil { return nil, err } @@ -1525,7 +1526,7 @@ func getTestBidRequest(nilUser bool, userExt *openrtb_ext.ExtUser, nilRegs bool, var regsExtData []byte if regsExt != nil { var err error - regsExtData, err = json.Marshal(regsExt) + regsExtData, err = jsonutil.Marshal(regsExt) if err != nil { return nil, err } @@ -1537,7 +1538,7 @@ func getTestBidRequest(nilUser bool, userExt *openrtb_ext.ExtUser, nilRegs bool, Ext: regsExtData, } } - return json.Marshal(bidRequest) + return jsonutil.Marshal(bidRequest) } func TestSetEffectiveAmpPubID(t *testing.T) { @@ -2188,7 +2189,7 @@ func TestValidAmpResponseWhenRequestRejected(t *testing.T) { assert.NoError(t, err, "Failed to read test file.") test := testCase{} - assert.NoError(t, json.Unmarshal(fileData, &test), "Failed to parse test file.") + assert.NoError(t, jsonutil.UnmarshalValid(fileData, &test), "Failed to parse test file.") request := httptest.NewRequest("GET", fmt.Sprintf("/openrtb2/auction/amp?%s", test.Query), nil) recorder := httptest.NewRecorder() @@ -2208,8 +2209,8 @@ func TestValidAmpResponseWhenRequestRejected(t *testing.T) { var actualAmpResp AmpResponse var expectedAmpResp AmpResponse - assert.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &actualAmpResp), "Unable to unmarshal actual AmpResponse.") - assert.NoError(t, json.Unmarshal(test.ExpectedAmpResponse, &expectedAmpResp), "Unable to unmarshal expected AmpResponse.") + assert.NoError(t, jsonutil.UnmarshalValid(recorder.Body.Bytes(), &actualAmpResp), "Unable to unmarshal actual AmpResponse.") + assert.NoError(t, jsonutil.UnmarshalValid(test.ExpectedAmpResponse, &expectedAmpResp), "Unable to unmarshal expected AmpResponse.") // validate modules data separately, because it has dynamic data if expectedAmpResp.ORTB2.Ext.Prebid == nil { @@ -2245,7 +2246,7 @@ func TestSendAmpResponse_LogsErrors(t *testing.T) { { description: "Error logged when bid.ext unmarshal fails", expectedErrors: []error{ - errors.New("Critical error while unpacking AMP targets: readObjectStart: expect { or n, but found \", error found in #1 byte of ...|\"hb_cache_i|..., bigger context ...|\"hb_cache_id|..."), + errors.New("Critical error while unpacking AMP targets: expect { or n, but found \""), }, expectedStatus: http.StatusInternalServerError, writer: httptest.NewRecorder(), diff --git a/endpoints/openrtb2/auction_test.go b/endpoints/openrtb2/auction_test.go index 78d934bc1f4..530d197e2ae 100644 --- a/endpoints/openrtb2/auction_test.go +++ b/endpoints/openrtb2/auction_test.go @@ -36,6 +36,7 @@ import ( "github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher" "github.com/prebid/prebid-server/stored_responses" "github.com/prebid/prebid-server/util/iputil" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/util/ptrutil" "github.com/stretchr/testify/assert" ) @@ -200,9 +201,9 @@ func runEndToEndTest(t *testing.T, auctionEndpointHandler httprouter.Handle, tes var actualBidResponse openrtb2.BidResponse var err error - err = json.Unmarshal(test.ExpectedBidResponse, &expectedBidResponse) + err = jsonutil.Unmarshal(test.ExpectedBidResponse, &expectedBidResponse) if assert.NoError(t, err, "Could not unmarshal expected bidResponse taken from test file.\n Test file: %s\n Error:%s\n", testFile, err) { - err = json.Unmarshal([]byte(actualJsonBidResponse), &actualBidResponse) + err = jsonutil.UnmarshalValid([]byte(actualJsonBidResponse), &actualBidResponse) if assert.NoError(t, err, "Could not unmarshal actual bidResponse from auction.\n Test file: %s\n Error:%s\n actualJsonBidResponse: %s", testFile, err, actualJsonBidResponse) { assertBidResponseEqual(t, testFile, expectedBidResponse, actualBidResponse) } @@ -222,13 +223,13 @@ func compareWarnings(t *testing.T, expectedBidResponseExt, actualBidResponseExt } var expectedWarn []openrtb_ext.ExtBidderMessage - err = json.Unmarshal(expectedWarnings, &expectedWarn) + err = jsonutil.UnmarshalValid(expectedWarnings, &expectedWarn) if err != nil { assert.Fail(t, "error unmarshalling expected warnings data from response extension") } var actualWarn []openrtb_ext.ExtBidderMessage - err = json.Unmarshal(actualWarnings, &actualWarn) + err = jsonutil.UnmarshalValid(actualWarnings, &actualWarn) if err != nil { assert.Fail(t, "error unmarshalling actual warnings data from response extension") } @@ -1907,7 +1908,7 @@ func TestValidateRequestExt(t *testing.T) { { description: "prebid cache - bids - wrong type", givenRequestExt: json.RawMessage(`{"prebid":{"cache":{"bids":true}}}`), - expectedErrors: []string{"openrtb_ext.ExtRequestPrebid.Cache: openrtb_ext.ExtRequestPrebidCache.Bids: readObjectStart: expect { or n, but found t, error found in #10 byte of ...|:{\"bids\":true}}|..., bigger context ...|{\"cache\":{\"bids\":true}}|..."}, + expectedErrors: []string{"cannot unmarshal openrtb_ext.ExtRequestPrebidCache.Bids: expect { or n, but found t"}, }, { description: "prebid cache - bids - provided", @@ -1921,7 +1922,7 @@ func TestValidateRequestExt(t *testing.T) { { description: "prebid cache - vastxml - wrong type", givenRequestExt: json.RawMessage(`{"prebid":{"cache":{"vastxml":true}}}`), - expectedErrors: []string{"openrtb_ext.ExtRequestPrebid.Cache: openrtb_ext.ExtRequestPrebidCache.VastXML: readObjectStart: expect { or n, but found t, error found in #10 byte of ...|vastxml\":true}}|..., bigger context ...|{\"cache\":{\"vastxml\":true}}|..."}, + expectedErrors: []string{"cannot unmarshal openrtb_ext.ExtRequestPrebidCache.VastXML: expect { or n, but found t"}, }, { description: "prebid cache - vastxml - provided", @@ -2555,7 +2556,7 @@ func TestStoredRequestGenerateUuid(t *testing.T) { newRequest, _, errList := deps.processStoredRequests(json.RawMessage(test.givenRawData), impInfo, storedRequests, storedImps, storedBidRequestId, hasStoredBidRequest) assert.Empty(t, errList, test.description) - if err := json.Unmarshal(newRequest, req); err != nil { + if err := jsonutil.UnmarshalValid(newRequest, req); err != nil { t.Errorf("processStoredRequests Error: %s", err.Error()) } if test.expectedCur != "" { @@ -3464,7 +3465,7 @@ func TestGetAccountID(t *testing.T) { ParentAccount: &testParentAccount, }, } - testPubExtJSON, err := json.Marshal(testPubExt) + testPubExtJSON, err := jsonutil.Marshal(testPubExt) assert.NoError(t, err) testCases := []struct { @@ -4196,7 +4197,7 @@ func TestParseRequestParseImpInfoError(t *testing.T) { assert.Nil(t, resReq, "Result request should be nil due to incorrect imp") assert.Nil(t, impExtInfoMap, "Impression info map should be nil due to incorrect imp") assert.Len(t, errL, 1, "One error should be returned") - assert.Contains(t, errL[0].Error(), "openrtb_ext.Options.EchoVideoAttrs: ReadBool", "Incorrect error message") + assert.Contains(t, errL[0].Error(), "cannot unmarshal openrtb_ext.Options.EchoVideoAttrs", "Incorrect error message") } func TestParseGzipedRequest(t *testing.T) { @@ -4878,20 +4879,20 @@ func TestParseRequestMergeBidderParams(t *testing.T) { assert.NoError(t, resReq.RebuildRequest()) var expIExt, iExt map[string]interface{} - err := json.Unmarshal(test.expectedImpExt, &expIExt) + err := jsonutil.UnmarshalValid(test.expectedImpExt, &expIExt) assert.Nil(t, err, "unmarshal() should return nil error") assert.NotNil(t, resReq.BidRequest.Imp[0].Ext, "imp[0].Ext should not be nil") - err = json.Unmarshal(resReq.BidRequest.Imp[0].Ext, &iExt) + err = jsonutil.UnmarshalValid(resReq.BidRequest.Imp[0].Ext, &iExt) assert.Nil(t, err, "unmarshal() should return nil error") assert.Equal(t, expIExt, iExt, "bidderparams in imp[].Ext should match") var eReqE, reqE map[string]interface{} - err = json.Unmarshal(test.expectedReqExt, &eReqE) + err = jsonutil.UnmarshalValid(test.expectedReqExt, &eReqE) assert.Nil(t, err, "unmarshal() should return nil error") - err = json.Unmarshal(resReq.BidRequest.Ext, &reqE) + err = jsonutil.UnmarshalValid(resReq.BidRequest.Ext, &reqE) assert.Nil(t, err, "unmarshal() should return nil error") assert.Equal(t, eReqE, reqE, "req.Ext should match") @@ -5753,11 +5754,11 @@ func TestValidResponseAfterExecutingStages(t *testing.T) { var actualExt openrtb_ext.ExtBidResponse var expectedExt openrtb_ext.ExtBidResponse - assert.NoError(t, json.Unmarshal(test.ExpectedBidResponse, &expectedResp), "Unable to unmarshal expected BidResponse.") - assert.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &actualResp), "Unable to unmarshal actual BidResponse.") + assert.NoError(t, jsonutil.UnmarshalValid(test.ExpectedBidResponse, &expectedResp), "Unable to unmarshal expected BidResponse.") + assert.NoError(t, jsonutil.UnmarshalValid(recorder.Body.Bytes(), &actualResp), "Unable to unmarshal actual BidResponse.") if expectedResp.Ext != nil { - assert.NoError(t, json.Unmarshal(expectedResp.Ext, &expectedExt), "Unable to unmarshal expected ExtBidResponse.") - assert.NoError(t, json.Unmarshal(actualResp.Ext, &actualExt), "Unable to unmarshal actual ExtBidResponse.") + assert.NoError(t, jsonutil.UnmarshalValid(expectedResp.Ext, &expectedExt), "Unable to unmarshal expected ExtBidResponse.") + assert.NoError(t, jsonutil.UnmarshalValid(actualResp.Ext, &actualExt), "Unable to unmarshal actual ExtBidResponse.") } assertBidResponseEqual(t, tc.file, expectedResp, actualResp) @@ -5959,7 +5960,7 @@ func getObject(t *testing.T, filename, key string) json.RawMessage { assert.NoError(t, err, "Error jsonparsing root.mockBidRequest from file %s. Desc: %v.", filename, err) var obj json.RawMessage - err = json.Unmarshal(testBidRequest, &obj) + err = jsonutil.UnmarshalValid(testBidRequest, &obj) if err != nil { t.Fatalf("Failed to fetch object with key '%s' ... got error: %v", key, err) } diff --git a/endpoints/openrtb2/sample-requests/invalid-stored/bad_incoming_1.json b/endpoints/openrtb2/sample-requests/invalid-stored/bad_incoming_1.json index 4803ec7f605..cf699fea0c9 100644 --- a/endpoints/openrtb2/sample-requests/invalid-stored/bad_incoming_1.json +++ b/endpoints/openrtb2/sample-requests/invalid-stored/bad_incoming_1.json @@ -11,5 +11,5 @@ } }, "expectedReturnCode": 400, - "expectedErrorMessage": "Invalid request: readObjectStart: expect { or n, but found" + "expectedErrorMessage": "Invalid request: expect { or n, but found" } diff --git a/endpoints/openrtb2/sample-requests/invalid-stored/bad_stored_req.json b/endpoints/openrtb2/sample-requests/invalid-stored/bad_stored_req.json index 50a000110b1..235eb26179b 100644 --- a/endpoints/openrtb2/sample-requests/invalid-stored/bad_stored_req.json +++ b/endpoints/openrtb2/sample-requests/invalid-stored/bad_stored_req.json @@ -23,5 +23,5 @@ } }, "expectedReturnCode": 400, - "expectedErrorMessage": "Invalid request: readObjectStart: expect { or n, but found" + "expectedErrorMessage": "Invalid request: expect { or n, but found" } diff --git a/endpoints/openrtb2/sample-requests/invalid-whole/interstital-bad-perc.json b/endpoints/openrtb2/sample-requests/invalid-whole/interstital-bad-perc.json index 9042b9a6098..e5031d4597d 100644 --- a/endpoints/openrtb2/sample-requests/invalid-whole/interstital-bad-perc.json +++ b/endpoints/openrtb2/sample-requests/invalid-whole/interstital-bad-perc.json @@ -49,5 +49,5 @@ } }, "expectedReturnCode": 400, - "expectedErrorMessage": "Invalid request: openrtb_ext.ExtDevicePrebid.Interstitial: unmarshalerDecoder: request.device.ext.prebid.interstitial.minwidthperc must be a number between 0 and 100, error found in #10 byte of ...| }\n }|..., bigger context ...|: 120,\n \"minheightperc\": 60\n }\n }|...\n" + "expectedErrorMessage": "Invalid request: request.device.ext.prebid.interstitial.minwidthperc must be a number between 0 and 100" } diff --git a/endpoints/openrtb2/sample-requests/invalid-whole/invalid-source.json b/endpoints/openrtb2/sample-requests/invalid-whole/invalid-source.json index fd78849e316..8db15b82a75 100644 --- a/endpoints/openrtb2/sample-requests/invalid-whole/invalid-source.json +++ b/endpoints/openrtb2/sample-requests/invalid-whole/invalid-source.json @@ -39,5 +39,5 @@ ] }, "expectedReturnCode": 400, - "expectedErrorMessage": "Invalid request: openrtb_ext.ExtAppPrebid.Source: ReadString: expects \" or n, but found {" + "expectedErrorMessage": "Invalid request: cannot unmarshal openrtb_ext.ExtAppPrebid.Source: expects \" or n, but found {" } diff --git a/endpoints/openrtb2/sample-requests/invalid-whole/malformed-bid-request.json b/endpoints/openrtb2/sample-requests/invalid-whole/malformed-bid-request.json index 7dd6b24ede1..482af257d62 100644 --- a/endpoints/openrtb2/sample-requests/invalid-whole/malformed-bid-request.json +++ b/endpoints/openrtb2/sample-requests/invalid-whole/malformed-bid-request.json @@ -2,5 +2,5 @@ "description": "Malformed bid request throws an error", "mockBidRequest": "malformed", "expectedReturnCode": 400, - "expectedErrorMessage": "Invalid request: readObjectStart: expect { or n, but found m" + "expectedErrorMessage": "Invalid request: expect { or n, but found m" } diff --git a/endpoints/openrtb2/sample-requests/invalid-whole/regs-ext-malformed.json b/endpoints/openrtb2/sample-requests/invalid-whole/regs-ext-malformed.json index ebe0bc3b105..7ab2631b701 100644 --- a/endpoints/openrtb2/sample-requests/invalid-whole/regs-ext-malformed.json +++ b/endpoints/openrtb2/sample-requests/invalid-whole/regs-ext-malformed.json @@ -42,5 +42,5 @@ } }, "expectedReturnCode": 400, - "expectedErrorMessage": "Invalid request: request.regs.ext is invalid: ReadMapCB: expect { or n, but found " + "expectedErrorMessage": "Invalid request: request.regs.ext is invalid: expect { or n, but found " } diff --git a/endpoints/openrtb2/sample-requests/invalid-whole/user-ext-consent-int.json b/endpoints/openrtb2/sample-requests/invalid-whole/user-ext-consent-int.json index 4b5fb997897..af04627c3a9 100644 --- a/endpoints/openrtb2/sample-requests/invalid-whole/user-ext-consent-int.json +++ b/endpoints/openrtb2/sample-requests/invalid-whole/user-ext-consent-int.json @@ -46,5 +46,5 @@ } }, "expectedReturnCode": 400, - "expectedErrorMessage": "Invalid request: request.user.ext object is not valid: ReadString: expects \" or n, but found 1" + "expectedErrorMessage": "Invalid request: request.user.ext object is not valid: expects \" or n, but found 1" } diff --git a/endpoints/openrtb2/sample-requests/invalid-whole/user-gdpr-consent-invalid.json b/endpoints/openrtb2/sample-requests/invalid-whole/user-gdpr-consent-invalid.json index 57630c7cfbb..b710d589ea5 100644 --- a/endpoints/openrtb2/sample-requests/invalid-whole/user-gdpr-consent-invalid.json +++ b/endpoints/openrtb2/sample-requests/invalid-whole/user-gdpr-consent-invalid.json @@ -41,5 +41,5 @@ } }, "expectedReturnCode": 400, - "expectedErrorMessage": "Invalid request: request.user.ext object is not valid: ReadString: expects \" or n, but found 2" + "expectedErrorMessage": "Invalid request: request.user.ext object is not valid: expects \" or n, but found 2" } diff --git a/endpoints/openrtb2/test_utils.go b/endpoints/openrtb2/test_utils.go index 365f480eb4c..e8506813eef 100644 --- a/endpoints/openrtb2/test_utils.go +++ b/endpoints/openrtb2/test_utils.go @@ -38,6 +38,7 @@ import ( "github.com/prebid/prebid-server/stored_requests" "github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher" "github.com/prebid/prebid-server/util/iputil" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/util/uuidutil" jsonpatch "gopkg.in/evanphx/json-patch.v4" ) @@ -925,7 +926,7 @@ func (s mockCurrencyRatesClient) handle(w http.ResponseWriter, req *http.Request s.data.DataAsOfRaw = "2018-09-12" // Marshal the response and http write it - currencyServerJsonResponse, err := json.Marshal(&s.data) + currencyServerJsonResponse, err := jsonutil.Marshal(&s.data) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -949,13 +950,13 @@ func (b mockBidderHandler) bid(w http.ResponseWriter, req *http.Request) { // Unmarshal exit if error var openrtb2Request openrtb2.BidRequest - if err := json.Unmarshal(buf.Bytes(), &openrtb2Request); err != nil { + if err := jsonutil.UnmarshalValid(buf.Bytes(), &openrtb2Request); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } var openrtb2ImpExt map[string]json.RawMessage - if err := json.Unmarshal(openrtb2Request.Imp[0].Ext, &openrtb2ImpExt); err != nil { + if err := jsonutil.UnmarshalValid(openrtb2Request.Imp[0].Ext, &openrtb2ImpExt); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -986,7 +987,7 @@ func (b mockBidderHandler) bid(w http.ResponseWriter, req *http.Request) { } // Marshal the response and http write it - serverJsonResponse, err := json.Marshal(&serverResponseObject) + serverJsonResponse, err := jsonutil.Marshal(&serverResponseObject) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -1016,7 +1017,7 @@ func (a mockAdapter) MakeRequests(request *openrtb2.BidRequest, requestInfo *ada for _, imp := range request.Imp { requestCopy.Imp = []openrtb2.Imp{imp} - requestJSON, err := json.Marshal(request) + requestJSON, err := jsonutil.Marshal(request) if err != nil { errors = append(errors, err) continue @@ -1049,7 +1050,7 @@ func (a mockAdapter) MakeBids(request *openrtb2.BidRequest, requestData *adapter } var publisherResponse openrtb2.BidResponse - if err := json.Unmarshal(responseData.Body, &publisherResponse); err != nil { + if err := jsonutil.UnmarshalValid(responseData.Body, &publisherResponse); err != nil { return nil, []error{err} } @@ -1113,7 +1114,7 @@ func parseTestData(fileData []byte, testFile string) (testCase, error) { jsonTestConfig, _, _, err = jsonparser.Get(fileData, "config") if err == nil { - if err = json.Unmarshal(jsonTestConfig, parsedTestData.Config); err != nil { + if err = jsonutil.UnmarshalValid(jsonTestConfig, parsedTestData.Config); err != nil { return parsedTestData, fmt.Errorf("Error unmarshaling root.config from file %s. Desc: %v.", testFile, err) } } diff --git a/endpoints/openrtb2/video_auction_test.go b/endpoints/openrtb2/video_auction_test.go index 19af821f205..281c80b204d 100644 --- a/endpoints/openrtb2/video_auction_test.go +++ b/endpoints/openrtb2/video_auction_test.go @@ -23,6 +23,7 @@ import ( "github.com/prebid/prebid-server/prebid_cache_client" "github.com/prebid/prebid-server/privacy" "github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/util/ptrutil" "github.com/prebid/openrtb/v19/adcom1" @@ -47,7 +48,7 @@ func TestVideoEndpointImpressionsNumber(t *testing.T) { respBytes := recorder.Body.Bytes() resp := &openrtb_ext.BidResponseVideo{} - if err := json.Unmarshal(respBytes, resp); err != nil { + if err := jsonutil.UnmarshalValid(respBytes, resp); err != nil { t.Fatalf("Unable to unmarshal response.") } @@ -80,7 +81,7 @@ func TestVideoEndpointImpressionsDuration(t *testing.T) { } var extData openrtb_ext.ExtRequest - json.Unmarshal(ex.lastRequest.Ext, &extData) + jsonutil.UnmarshalValid(ex.lastRequest.Ext, &extData) assert.NotNil(t, extData.Prebid.Targeting.IncludeBidderKeys, "Request ext incorrect: IncludeBidderKeys should be true ") assert.True(t, *extData.Prebid.Targeting.IncludeBidderKeys, "Request ext incorrect: IncludeBidderKeys should be true ") @@ -135,7 +136,7 @@ func TestCreateBidExtension(t *testing.T) { resExt := &openrtb_ext.ExtRequest{} - if err := json.Unmarshal(res, &resExt); err != nil { + if err := jsonutil.UnmarshalValid(res, &resExt); err != nil { assert.Fail(t, "Unable to unmarshal bid extension") } assert.Equal(t, resExt.Prebid.Targeting.DurationRangeSec, durationRange, "Duration range seconds is incorrect") @@ -178,7 +179,7 @@ func TestVideoEndpointDebugQueryTrue(t *testing.T) { respBytes := recorder.Body.Bytes() resp := &openrtb_ext.BidResponseVideo{} - if err := json.Unmarshal(respBytes, resp); err != nil { + if err := jsonutil.UnmarshalValid(respBytes, resp); err != nil { t.Fatalf("Unable to unmarshal response.") } @@ -216,7 +217,7 @@ func TestVideoEndpointDebugQueryFalse(t *testing.T) { respBytes := recorder.Body.Bytes() resp := &openrtb_ext.BidResponseVideo{} - if err := json.Unmarshal(respBytes, resp); err != nil { + if err := jsonutil.UnmarshalValid(respBytes, resp); err != nil { t.Fatalf("Unable to unmarshal response.") } @@ -272,7 +273,7 @@ func TestVideoEndpointDebugNoAdPods(t *testing.T) { respBytes := recorder.Body.Bytes() resp := &openrtb_ext.BidResponseVideo{} - if err := json.Unmarshal(respBytes, resp); err != nil { + if err := jsonutil.UnmarshalValid(respBytes, resp); err != nil { t.Fatalf("Unable to unmarshal response.") } @@ -1091,7 +1092,7 @@ func TestCCPA(t *testing.T) { t.Fatalf("%s: The request never made it into the exchange.", test.description) } extRegs := &openrtb_ext.ExtRegs{} - if err := json.Unmarshal(ex.lastRequest.Regs.Ext, extRegs); err != nil { + if err := jsonutil.UnmarshalValid(ex.lastRequest.Regs.Ext, extRegs); err != nil { t.Fatalf("%s: Failed to unmarshal reg.ext in request to the exchange: %v", test.description, err) } if test.expectConsentString { @@ -1103,7 +1104,7 @@ func TestCCPA(t *testing.T) { // Validate HTTP Response responseBytes := httpResponseRecorder.Body.Bytes() response := &openrtb_ext.BidResponseVideo{} - if err := json.Unmarshal(responseBytes, response); err != nil { + if err := jsonutil.UnmarshalValid(responseBytes, response); err != nil { t.Fatalf("%s: Unable to unmarshal response.", test.description) } assert.Len(t, ex.lastRequest.Imp, 11, test.description+":imps") @@ -1125,12 +1126,12 @@ func TestVideoEndpointAppendBidderNames(t *testing.T) { } var extData openrtb_ext.ExtRequest - json.Unmarshal(ex.lastRequest.Ext, &extData) + jsonutil.UnmarshalValid(ex.lastRequest.Ext, &extData) assert.True(t, extData.Prebid.Targeting.AppendBidderNames, "Request ext incorrect: AppendBidderNames should be true ") respBytes := recorder.Body.Bytes() resp := &openrtb_ext.BidResponseVideo{} - if err := json.Unmarshal(respBytes, resp); err != nil { + if err := jsonutil.UnmarshalValid(respBytes, resp); err != nil { t.Fatalf("Unable to unmarshal response.") } diff --git a/exchange/auction_test.go b/exchange/auction_test.go index ad0f8d4a72c..199b2abba77 100644 --- a/exchange/auction_test.go +++ b/exchange/auction_test.go @@ -2,7 +2,6 @@ package exchange import ( "context" - "encoding/json" "encoding/xml" "fmt" "os" @@ -17,6 +16,7 @@ import ( "github.com/prebid/prebid-server/exchange/entities" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/prebid_cache_client" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/util/ptrutil" "github.com/stretchr/testify/assert" @@ -179,7 +179,7 @@ func loadCacheSpec(filename string) (*cacheSpec, error) { } var spec cacheSpec - if err := json.Unmarshal(specData, &spec); err != nil { + if err := jsonutil.UnmarshalValid(specData, &spec); err != nil { return nil, fmt.Errorf("Failed to unmarshal JSON from file: %v", err) } @@ -286,18 +286,18 @@ func runCacheSpec(t *testing.T, fileDisplayName string, specData *cacheSpec) { for i, expectedCacheable := range specData.ExpectedCacheables { found := false var expectedData interface{} - if err := json.Unmarshal(expectedCacheable.Data, &expectedData); err != nil { + if err := jsonutil.UnmarshalValid(expectedCacheable.Data, &expectedData); err != nil { t.Fatalf("Failed to decode expectedCacheables[%d].value: %v", i, err) } if s, ok := expectedData.(string); ok && expectedCacheable.Type == prebid_cache_client.TypeJSON { // decode again if we have pre-encoded json string values - if err := json.Unmarshal([]byte(s), &expectedData); err != nil { + if err := jsonutil.UnmarshalValid([]byte(s), &expectedData); err != nil { t.Fatalf("Failed to re-decode expectedCacheables[%d].value :%v", i, err) } } for j, cachedItem := range cache.items { var actualData interface{} - if err := json.Unmarshal(cachedItem.Data, &actualData); err != nil { + if err := jsonutil.UnmarshalValid(cachedItem.Data, &actualData); err != nil { t.Fatalf("Failed to decode actual cache[%d].value: %s", j, err) } if assert.ObjectsAreEqual(expectedData, actualData) && diff --git a/exchange/bidder_test.go b/exchange/bidder_test.go index 65f6b795247..a0bee708b60 100644 --- a/exchange/bidder_test.go +++ b/exchange/bidder_test.go @@ -31,6 +31,7 @@ import ( "github.com/prebid/prebid-server/metrics" metricsConfig "github.com/prebid/prebid-server/metrics/config" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/version" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -856,7 +857,7 @@ func TestMultiCurrencies(t *testing.T) { mockedHTTPServer := httptest.NewServer(http.HandlerFunc( func(rw http.ResponseWriter, req *http.Request) { - b, err := json.Marshal(tc.rates) + b, err := jsonutil.Marshal(tc.rates) if err == nil { rw.WriteHeader(http.StatusOK) rw.Write(b) @@ -1177,7 +1178,7 @@ func TestMultiCurrencies_RequestCurrencyPick(t *testing.T) { mockedHTTPServer := httptest.NewServer(http.HandlerFunc( func(rw http.ResponseWriter, req *http.Request) { - b, err := json.Marshal(tc.rates) + b, err := jsonutil.Marshal(tc.rates) if err == nil { rw.WriteHeader(http.StatusOK) rw.Write(b) @@ -2344,7 +2345,7 @@ func (bidder *goodSingleBidderWithStoredBidResp) MakeRequests(request *openrtb2. func (bidder *goodSingleBidderWithStoredBidResp) MakeBids(internalRequest *openrtb2.BidRequest, externalRequest *adapters.RequestData, response *adapters.ResponseData) (*adapters.BidderResponse, []error) { var bidResp openrtb2.BidResponse - if err := json.Unmarshal(response.Body, &bidResp); err != nil { + if err := jsonutil.UnmarshalValid(response.Body, &bidResp); err != nil { return nil, []error{err} } bidResponse := adapters.NewBidderResponseWithBidsCapacity(5) diff --git a/exchange/exchange_test.go b/exchange/exchange_test.go index 0b3af663e12..4da418b254c 100644 --- a/exchange/exchange_test.go +++ b/exchange/exchange_test.go @@ -41,6 +41,7 @@ import ( "github.com/prebid/prebid-server/stored_requests" "github.com/prebid/prebid-server/stored_requests/backends/file_fetcher" "github.com/prebid/prebid-server/usersync" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/util/ptrutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -379,7 +380,7 @@ func TestDebugBehaviour(t *testing.T) { assert.NotNilf(t, outBidResponse.Ext, "%s. outBidResponse.Ext should not be nil \n", test.desc) assert.False(t, auctionRequest.BidderResponseStartTime.IsZero()) actualExt := &openrtb_ext.ExtBidResponse{} - err = json.Unmarshal(outBidResponse.Ext, actualExt) + err = jsonutil.UnmarshalValid(outBidResponse.Ext, actualExt) assert.NoErrorf(t, err, "%s. \"ext\" JSON field could not be unmarshaled. err: \"%v\" \n outBidResponse.Ext: \"%s\" \n", test.desc, err, outBidResponse.Ext) assert.NotEmpty(t, actualExt.Prebid, "%s. ext.prebid should not be empty") @@ -541,7 +542,7 @@ func TestTwoBiddersDebugDisabledAndEnabled(t *testing.T) { assert.False(t, auctionRequest.BidderResponseStartTime.IsZero()) actualExt := &openrtb_ext.ExtBidResponse{} - err = json.Unmarshal(outBidResponse.Ext, actualExt) + err = jsonutil.UnmarshalValid(outBidResponse.Ext, actualExt) assert.NoErrorf(t, err, "JSON field unmarshaling err. ") assert.NotEmpty(t, actualExt.Prebid, "ext.prebid should not be empty") @@ -1143,7 +1144,7 @@ func TestGetAuctionCurrencyRates(t *testing.T) { for _, tc := range testCases { // Test setup: - jsonPbsRates, err := json.Marshal(tc.given.pbsRates) + jsonPbsRates, err := jsonutil.Marshal(tc.given.pbsRates) if err != nil { t.Fatalf("Failed to marshal PBS rates: %v", err) } @@ -1627,7 +1628,7 @@ func TestBidReturnsCreative(t *testing.T) { assert.Equal(t, test.expectedCreativeMarkup, resultingBids[0].AdM, "%s. Ad markup string doesn't match expected \n", test.description) var bidExt openrtb_ext.ExtBid - json.Unmarshal(resultingBids[0].Ext, &bidExt) + jsonutil.UnmarshalValid(resultingBids[0].Ext, &bidExt) assert.Equal(t, 0, bidExt.Prebid.DealPriority, "%s. Test should have DealPriority set to 0", test.description) assert.Equal(t, false, bidExt.Prebid.DealTierSatisfied, "%s. Test should have DealTierSatisfied set to false", test.description) } @@ -2279,7 +2280,7 @@ func loadFile(filename string) (*exchangeSpec, error) { } var spec exchangeSpec - if err := json.Unmarshal(specData, &spec); err != nil { + if err := jsonutil.UnmarshalValid(specData, &spec); err != nil { return nil, fmt.Errorf("Failed to unmarshal JSON from file: %v", err) } @@ -2376,7 +2377,7 @@ func runSpec(t *testing.T, filename string, spec *exchangeSpec) { auctionRequest.Account.DefaultBidLimit = spec.MultiBid.AccountMaxBid requestExt := &openrtb_ext.ExtRequest{} - err := json.Unmarshal(spec.IncomingRequest.OrtbRequest.Ext, requestExt) + err := jsonutil.UnmarshalValid(spec.IncomingRequest.OrtbRequest.Ext, requestExt) assert.NoError(t, err, "invalid request ext") validatedMultiBids, errs := openrtb_ext.ValidateAndBuildExtMultiBid(&requestExt.Prebid) for _, err := range errs { // same as in validateRequestExt(). @@ -2387,7 +2388,7 @@ func runSpec(t *testing.T, filename string, spec *exchangeSpec) { } requestExt.Prebid.MultiBid = validatedMultiBids - updateReqExt, err := json.Marshal(requestExt) + updateReqExt, err := jsonutil.Marshal(requestExt) assert.NoError(t, err, "invalid request ext") auctionRequest.BidRequestWrapper.Ext = updateReqExt } @@ -2450,7 +2451,7 @@ func runSpec(t *testing.T, filename string, spec *exchangeSpec) { actualPassthrough := "" actualBidRespExt := &openrtb_ext.ExtBidResponse{} if bid.Ext != nil { - if err := json.Unmarshal(bid.Ext, actualBidRespExt); err != nil { + if err := jsonutil.UnmarshalValid(bid.Ext, actualBidRespExt); err != nil { assert.NoError(t, err, fmt.Sprintf("Error when unmarshalling: %s", err)) } if actualBidRespExt.Prebid != nil { @@ -2459,7 +2460,7 @@ func runSpec(t *testing.T, filename string, spec *exchangeSpec) { } expectedBidRespExt := &openrtb_ext.ExtBidResponse{} if spec.Response.Ext != nil { - if err := json.Unmarshal(spec.Response.Ext, expectedBidRespExt); err != nil { + if err := jsonutil.UnmarshalValid(spec.Response.Ext, expectedBidRespExt); err != nil { assert.NoError(t, err, fmt.Sprintf("Error when unmarshalling: %s", err)) } if expectedBidRespExt.Prebid != nil { @@ -2490,11 +2491,11 @@ func runSpec(t *testing.T, filename string, spec *exchangeSpec) { actualBidRespExt := &openrtb_ext.ExtBidResponse{} expectedBidRespExt := &openrtb_ext.ExtBidResponse{} if bid.Ext != nil { - if err := json.Unmarshal(bid.Ext, actualBidRespExt); err != nil { + if err := jsonutil.UnmarshalValid(bid.Ext, actualBidRespExt); err != nil { assert.NoError(t, err, fmt.Sprintf("Error when unmarshalling: %s", err)) } } - if err := json.Unmarshal(spec.Response.Ext, expectedBidRespExt); err != nil { + if err := jsonutil.UnmarshalValid(spec.Response.Ext, expectedBidRespExt); err != nil { assert.NoError(t, err, fmt.Sprintf("Error when unmarshalling: %s", err)) } @@ -2524,7 +2525,7 @@ func extractResponseTimes(t *testing.T, context string, bid *openrtb2.BidRespons return nil } else { responseTimes := make(map[string]int) - if err := json.Unmarshal(data, &responseTimes); err != nil { + if err := jsonutil.UnmarshalValid(data, &responseTimes); err != nil { t.Errorf("%s: Failed to unmarshal ext.responsetimemillis into map[string]int: %v", context, err) return nil } @@ -4176,14 +4177,14 @@ func TestMakeBidExtJSON(t *testing.T) { ext: json.RawMessage(`{invalid json}`), extBidPrebid: openrtb_ext.ExtBidPrebid{Type: openrtb_ext.BidType("video")}, expectedBidExt: ``, - expectedErrMessage: "ReadString: expects \" or n, but found i, error found in #2 byte of ...|{invalid jso|..., bigger context ...|{invalid json}|...", + expectedErrMessage: "expects \" or n, but found i", }, { description: "Meta - Invalid", ext: json.RawMessage(`{"prebid":{"meta":{"brandId":"foo"}}}`), // brandId should be an int, but is a string in this test case extBidPrebid: openrtb_ext.ExtBidPrebid{Type: openrtb_ext.BidType("banner")}, impExtInfo: nil, - expectedErrMessage: "error validaing response from server, Prebid: Meta: openrtb_ext.ExtBidPrebidMeta.BrandID: readUint64: unexpected character: \xff, error found in #10 byte of ...|brandId\":\"foo\"}}}|..., bigger context ...|{\"prebid\":{\"meta\":{\"brandId\":\"foo\"}}}|...", + expectedErrMessage: "error validaing response from server, cannot unmarshal openrtb_ext.ExtBidPrebidMeta.BrandID: unexpected character: \xff", }, } @@ -5330,12 +5331,12 @@ func (b *capturingRequestBidder) requestBid(ctx context.Context, bidderRequest B func diffOrtbRequests(t *testing.T, description string, expected *openrtb2.BidRequest, actual *openrtb2.BidRequest) { t.Helper() - actualJSON, err := json.Marshal(actual) + actualJSON, err := jsonutil.Marshal(actual) if err != nil { t.Fatalf("%s failed to marshal actual BidRequest into JSON. %v", description, err) } - expectedJSON, err := json.Marshal(expected) + expectedJSON, err := jsonutil.Marshal(expected) if err != nil { t.Fatalf("%s failed to marshal expected BidRequest into JSON. %v", description, err) } @@ -5353,12 +5354,12 @@ func diffOrtbResponses(t *testing.T, description string, expected *openrtb2.BidR // this implementation detail, I'm cutting a corner and ignoring it here. actualSeats := mapifySeatBids(t, description, actual.SeatBid) expectedSeats := mapifySeatBids(t, description, expected.SeatBid) - actualJSON, err := json.Marshal(actualSeats) + actualJSON, err := jsonutil.Marshal(actualSeats) if err != nil { t.Fatalf("%s failed to marshal actual BidResponse into JSON. %v", description, err) } - expectedJSON, err := json.Marshal(expectedSeats) + expectedJSON, err := jsonutil.Marshal(expectedSeats) if err != nil { t.Fatalf("%s failed to marshal expected BidResponse into JSON. %v", description, err) } @@ -5488,13 +5489,13 @@ func getInfoFromImp(req *openrtb_ext.RequestWrapper) (json.RawMessage, string, e impID := imp.ID var bidderExts map[string]json.RawMessage - if err := json.Unmarshal(imp.Ext, &bidderExts); err != nil { + if err := jsonutil.UnmarshalValid(imp.Ext, &bidderExts); err != nil { return nil, "", err } var extPrebid openrtb_ext.ExtImpPrebid if bidderExts[openrtb_ext.PrebidExtKey] != nil { - if err := json.Unmarshal(bidderExts[openrtb_ext.PrebidExtKey], &extPrebid); err != nil { + if err := jsonutil.UnmarshalValid(bidderExts[openrtb_ext.PrebidExtKey], &extPrebid); err != nil { return nil, "", err } } diff --git a/exchange/targeting_test.go b/exchange/targeting_test.go index a5f49689349..25c6ea1714a 100644 --- a/exchange/targeting_test.go +++ b/exchange/targeting_test.go @@ -16,6 +16,7 @@ import ( "github.com/prebid/prebid-server/hooks/hookexecution" metricsConfig "github.com/prebid/prebid-server/metrics/config" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/util/ptrutil" "github.com/prebid/openrtb/v19/openrtb2" @@ -182,7 +183,7 @@ func buildParams(t *testing.T, mockBids map[openrtb_ext.BidderName][]*openrtb2.B paramsPrebid["bidder"] = paramsPrebidBidders params["prebid"] = paramsPrebid - ext, err := json.Marshal(params) + ext, err := jsonutil.Marshal(params) if err != nil { t.Fatalf("Failed to make imp exts: %v", err) } @@ -224,7 +225,7 @@ func buildBidMap(seatBids []openrtb2.SeatBid, numBids int) map[string]*openrtb2. func parseTargets(t *testing.T, bid *openrtb2.Bid) map[string]string { t.Helper() var parsed openrtb_ext.ExtBid - if err := json.Unmarshal(bid.Ext, &parsed); err != nil { + if err := jsonutil.UnmarshalValid(bid.Ext, &parsed); err != nil { t.Fatalf("Unexpected error parsing targeting params: %v", err) } return parsed.Prebid.Targeting diff --git a/exchange/utils.go b/exchange/utils.go index 9fbaeeefd5a..c7a938432ac 100644 --- a/exchange/utils.go +++ b/exchange/utils.go @@ -552,7 +552,7 @@ func splitImps(imps []openrtb2.Imp) (map[string][]openrtb2.Imp, error) { for i, imp := range imps { var impExt map[string]json.RawMessage - if err := jsonutil.Unmarshal(imp.Ext, &impExt); err != nil { + if err := jsonutil.UnmarshalValid(imp.Ext, &impExt); err != nil { return nil, fmt.Errorf("invalid json for imp[%d]: %v", i, err) } diff --git a/exchange/utils_test.go b/exchange/utils_test.go index 77c00506f27..4380d648cb1 100644 --- a/exchange/utils_test.go +++ b/exchange/utils_test.go @@ -18,6 +18,7 @@ import ( "github.com/prebid/prebid-server/metrics" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/privacy" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/util/ptrutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -187,21 +188,21 @@ func TestSplitImps(t *testing.T) { givenImps: []openrtb2.Imp{ {ID: "imp1", Ext: json.RawMessage(`malformed`)}, }, - expectedError: "invalid json for imp[0]: ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...", + expectedError: "invalid json for imp[0]: expect { or n, but found m", }, { description: "Malformed imp.ext.prebid", givenImps: []openrtb2.Imp{ {ID: "imp1", Ext: json.RawMessage(`{"prebid": malformed}`)}, }, - expectedError: "invalid json for imp[0]: Skip: do not know how to skip: 109, error found in #10 byte of ...|prebid\": malformed}|..., bigger context ...|{\"prebid\": malformed}|...", + expectedError: "invalid json for imp[0]: do not know how to skip: 109", }, { description: "Malformed imp.ext.prebid.bidder", givenImps: []openrtb2.Imp{ {ID: "imp1", Ext: json.RawMessage(`{"prebid": {"bidder": malformed}}`)}, }, - expectedError: "invalid json for imp[0]: Skip: do not know how to skip: 109, error found in #10 byte of ...|bidder\": malformed}}|..., bigger context ...|{\"prebid\": {\"bidder\": malformed}}|...", + expectedError: "invalid json for imp[0]: do not know how to skip: 109", }, } @@ -576,7 +577,7 @@ func TestExtractAdapterReqBidderParamsMap(t *testing.T) { name: "malformed req.ext", givenBidRequest: &openrtb2.BidRequest{Ext: json.RawMessage("malformed")}, want: nil, - wantErr: errors.New("error decoding Request.ext : readObjectStart: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|..."), + wantErr: errors.New("error decoding Request.ext : expect { or n, but found m"), }, { name: "extract bidder params from req.Ext for input request in adapter code", @@ -1065,7 +1066,7 @@ func TestCleanOpenRTBRequestsCCPAErrors(t *testing.T) { req.Regs = &openrtb2.Regs{Ext: test.reqRegsExt} var reqExtStruct openrtb_ext.ExtRequest - err := json.Unmarshal(req.Ext, &reqExtStruct) + err := jsonutil.UnmarshalValid(req.Ext, &reqExtStruct) assert.NoError(t, err, test.description+":marshal_ext") auctionReq := AuctionRequest{ @@ -1224,7 +1225,7 @@ func TestCleanOpenRTBRequestsSChain(t *testing.T) { if test.inExt != nil { req.Ext = test.inExt extRequest = &openrtb_ext.ExtRequest{} - err := json.Unmarshal(req.Ext, extRequest) + err := jsonutil.UnmarshalValid(req.Ext, extRequest) assert.NoErrorf(t, err, test.description+":Error unmarshaling inExt") } @@ -1295,7 +1296,7 @@ func TestCleanOpenRTBRequestsBidderParams(t *testing.T) { if test.inExt != nil { req.Ext = test.inExt extRequest = &openrtb_ext.ExtRequest{} - err := json.Unmarshal(req.Ext, extRequest) + err := jsonutil.UnmarshalValid(req.Ext, extRequest) assert.NoErrorf(t, err, test.description+":Error unmarshaling inExt") } @@ -2471,7 +2472,7 @@ func TestBuildRequestExtForBidder(t *testing.T) { for _, test := range testCases { requestExtParsed := &openrtb_ext.ExtRequest{} if test.requestExt != nil { - err := json.Unmarshal(test.requestExt, requestExtParsed) + err := jsonutil.UnmarshalValid(test.requestExt, requestExtParsed) if !assert.NoError(t, err, test.description+":parse_ext") { continue } @@ -2512,7 +2513,7 @@ func TestBuildRequestExtForBidder_RequestExtMalformed(t *testing.T) { actualJson, actualErr := buildRequestExtForBidder(bidder, requestExt, requestExtParsed, bidderParams, alternateBidderCodes) assert.Equal(t, json.RawMessage(nil), actualJson) - assert.EqualError(t, actualErr, "ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...") + assert.EqualError(t, actualErr, "expect { or n, but found m") } // newAdapterAliasBidRequest builds a BidRequest with aliases @@ -2843,17 +2844,17 @@ func TestRemoveUnpermissionedEidsUnmarshalErrors(t *testing.T) { { description: "Malformed Ext", userExt: json.RawMessage(`malformed`), - expectedErr: "ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...", + expectedErr: "expect { or n, but found m", }, { description: "Malformed Eid Array Type", userExt: json.RawMessage(`{"eids":[42]}`), - expectedErr: "[]openrtb2.EID: readObjectStart: expect { or n, but found 4, error found in #2 byte of ...|[42]|..., bigger context ...|[42]|...", + expectedErr: "cannot unmarshal []openrtb2.EID: expect { or n, but found 4", }, { description: "Malformed Eid Item Type", userExt: json.RawMessage(`{"eids":[{"source":42,"id":"anyID"}]}`), - expectedErr: "[]openrtb2.EID: openrtb2.EID.Source: ReadString: expects \" or n, but found 4, error found in #10 byte of ...|\"source\":42,\"id\":\"an|..., bigger context ...|[{\"source\":42,\"id\":\"anyID\"}]|...", + expectedErr: "cannot unmarshal openrtb2.EID.Source: expects \" or n, but found 4", }, } @@ -3067,7 +3068,7 @@ func TestCleanOpenRTBRequestsSChainMultipleBidders(t *testing.T) { } extRequest := &openrtb_ext.ExtRequest{} - err := json.Unmarshal(req.Ext, extRequest) + err := jsonutil.UnmarshalValid(req.Ext, extRequest) assert.NoErrorf(t, err, "Error unmarshaling inExt") auctionReq := AuctionRequest{ @@ -3529,7 +3530,7 @@ func TestCleanOpenRTBRequestsFilterBidderRequestExt(t *testing.T) { if test.inExt != nil { req.Ext = test.inExt extRequest = &openrtb_ext.ExtRequest{} - err := json.Unmarshal(req.Ext, extRequest) + err := jsonutil.UnmarshalValid(req.Ext, extRequest) assert.NoErrorf(t, err, test.desc+":Error unmarshaling inExt") } @@ -4290,7 +4291,7 @@ func TestGetPrebidMediaTypeForBid(t *testing.T) { { description: "Invalid bid ext", inputBid: openrtb2.Bid{ID: "bidId", ImpID: "impId", Ext: json.RawMessage(`[true`)}, - expectedError: "Failed to parse bid mediatype for impression \"impId\", readObjectStart: expect { or n, but found [, error found in #1 byte of ...|[true|..., bigger context ...|[true|...", + expectedError: "Failed to parse bid mediatype for impression \"impId\", expect { or n, but found [", }, { description: "Bid ext is nil", @@ -4331,7 +4332,7 @@ func TestGetMediaTypeForBid(t *testing.T) { { description: "invalid bid ext", inputBid: openrtb2.Bid{ID: "bidId", ImpID: "impId", Ext: json.RawMessage(`{"prebid"`)}, - expectedError: "Failed to parse bid mediatype for impression \"impId\", openrtb_ext.ExtBid.readFieldHash: expect :, but found \x00, error found in #9 byte of ...|{\"prebid\"|..., bigger context ...|{\"prebid\"|...", + expectedError: "Failed to parse bid mediatype for impression \"impId\", expect :, but found \x00", }, { description: "Valid bid ext with mtype native", diff --git a/firstpartydata/first_party_data_test.go b/firstpartydata/first_party_data_test.go index 9ef9e310eff..182cd1d274b 100644 --- a/firstpartydata/first_party_data_test.go +++ b/firstpartydata/first_party_data_test.go @@ -9,6 +9,7 @@ import ( "github.com/prebid/openrtb/v19/openrtb2" "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -480,7 +481,7 @@ func TestExtractBidderConfigFPD(t *testing.T) { require.NoError(t, err, "Cannot Load Test") givenRequestExtPrebid := &openrtb_ext.ExtRequestPrebid{} - err = json.Unmarshal(fpdFile.InputRequestData, givenRequestExtPrebid) + err = jsonutil.UnmarshalValid(fpdFile.InputRequestData, givenRequestExtPrebid) require.NoError(t, err, "Cannot Load Test Conditions") testRequest := &openrtb_ext.RequestExt{} @@ -535,15 +536,15 @@ func TestResolveFPD(t *testing.T) { require.NoError(t, err, "Cannot Load Test") request := &openrtb2.BidRequest{} - err = json.Unmarshal(fpdFile.InputRequestData, &request) + err = jsonutil.UnmarshalValid(fpdFile.InputRequestData, &request) require.NoError(t, err, "Cannot Load Request") originalRequest := &openrtb2.BidRequest{} - err = json.Unmarshal(fpdFile.InputRequestData, &originalRequest) + err = jsonutil.UnmarshalValid(fpdFile.InputRequestData, &originalRequest) require.NoError(t, err, "Cannot Load Request") outputReq := &openrtb2.BidRequest{} - err = json.Unmarshal(fpdFile.OutputRequestData, &outputReq) + err = jsonutil.UnmarshalValid(fpdFile.OutputRequestData, &outputReq) require.NoError(t, err, "Cannot Load Output Request") reqExtFPD := make(map[string][]byte) @@ -556,7 +557,7 @@ func TestResolveFPD(t *testing.T) { reqFPDSiteContentData := fpdFile.GlobalFPD[siteContentDataKey] if len(reqFPDSiteContentData) > 0 { var siteConData []openrtb2.Data - err = json.Unmarshal(reqFPDSiteContentData, &siteConData) + err = jsonutil.UnmarshalValid(reqFPDSiteContentData, &siteConData) if err != nil { t.Errorf("Unable to unmarshal site.content.data:") } @@ -566,7 +567,7 @@ func TestResolveFPD(t *testing.T) { reqFPDAppContentData := fpdFile.GlobalFPD[appContentDataKey] if len(reqFPDAppContentData) > 0 { var appConData []openrtb2.Data - err = json.Unmarshal(reqFPDAppContentData, &appConData) + err = jsonutil.UnmarshalValid(reqFPDAppContentData, &appConData) if err != nil { t.Errorf("Unable to unmarshal app.content.data: ") } @@ -576,7 +577,7 @@ func TestResolveFPD(t *testing.T) { reqFPDUserData := fpdFile.GlobalFPD[userDataKey] if len(reqFPDUserData) > 0 { var userData []openrtb2.Data - err = json.Unmarshal(reqFPDUserData, &userData) + err = jsonutil.UnmarshalValid(reqFPDUserData, &userData) if err != nil { t.Errorf("Unable to unmarshal app.content.data: ") } @@ -642,14 +643,14 @@ func TestExtractFPDForBidders(t *testing.T) { } var expectedRequest openrtb2.BidRequest - err = json.Unmarshal(fpdFile.OutputRequestData, &expectedRequest) + err = jsonutil.UnmarshalValid(fpdFile.OutputRequestData, &expectedRequest) if err != nil { t.Errorf("Unable to unmarshal input request: %s", fileName) } resultRequest := &openrtb_ext.RequestWrapper{} resultRequest.BidRequest = &openrtb2.BidRequest{} - err = json.Unmarshal(fpdFile.InputRequestData, resultRequest.BidRequest) + err = jsonutil.UnmarshalValid(fpdFile.InputRequestData, resultRequest.BidRequest) assert.NoError(t, err, "Error should be nil") resultFPD, errL := ExtractFPDForBidders(resultRequest) @@ -726,7 +727,7 @@ func loadFpdFile(filename string) (fpdFile, error) { if err != nil { return fileData, err } - err = json.Unmarshal(fileContents, &fileData) + err = jsonutil.UnmarshalValid(fileContents, &fileData) if err != nil { return fileData, err } diff --git a/floors/floors_test.go b/floors/floors_test.go index 7f27e1a26d5..b3002c2f155 100644 --- a/floors/floors_test.go +++ b/floors/floors_test.go @@ -9,6 +9,7 @@ import ( "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/currency" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/stretchr/testify/assert" ) @@ -447,7 +448,7 @@ func TestResolveFloors(t *testing.T) { } func printFloors(floors *openrtb_ext.PriceFloorRules) string { - fbytes, _ := json.Marshal(floors) + fbytes, _ := jsonutil.Marshal(floors) return string(fbytes) } diff --git a/floors/rule_test.go b/floors/rule_test.go index 4de06dc80a2..7c484a7c95f 100644 --- a/floors/rule_test.go +++ b/floors/rule_test.go @@ -760,7 +760,7 @@ func TestGetMinFloorValue(t *testing.T) { }, want: 0.0, want1: "", - wantErr: errors.New("Error in getting FloorMin value : 'ReadString: expects \" or n, but found \x00, error found in #1 byte of ...|{|..., bigger context ...|{|...'"), + wantErr: errors.New("Error in getting FloorMin value : 'expects \" or n, but found \x00'"), }, } for _, tc := range testCases { diff --git a/gdpr/full_enforcement_test.go b/gdpr/full_enforcement_test.go index 61f6e8b8520..4a859ecaabb 100644 --- a/gdpr/full_enforcement_test.go +++ b/gdpr/full_enforcement_test.go @@ -1,7 +1,6 @@ package gdpr import ( - "encoding/json" "testing" "github.com/prebid/go-gdpr/consentconstants" @@ -10,6 +9,7 @@ import ( "github.com/prebid/go-gdpr/vendorlist" "github.com/prebid/go-gdpr/vendorlist2" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/stretchr/testify/assert" ) @@ -961,7 +961,7 @@ func TestLegalBasisWithoutVendor(t *testing.T) { func getVendorList(t *testing.T) vendorlist.VendorList { GVL := makeVendorList() - marshaledGVL, err := json.Marshal(GVL) + marshaledGVL, err := jsonutil.Marshal(GVL) if err != nil { t.Fatalf("Failed to marshal GVL") } diff --git a/gdpr/vendorlist-fetching_test.go b/gdpr/vendorlist-fetching_test.go index a1dfb7fefb8..b8c73bbf9a4 100644 --- a/gdpr/vendorlist-fetching_test.go +++ b/gdpr/vendorlist-fetching_test.go @@ -2,7 +2,6 @@ package gdpr import ( "context" - "encoding/json" "net/http" "net/http/httptest" "strconv" @@ -13,6 +12,7 @@ import ( "github.com/prebid/go-gdpr/api" "github.com/prebid/go-gdpr/consentconstants" "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/util/jsonutil" ) func TestFetcherDynamicLoadListExists(t *testing.T) { @@ -305,7 +305,7 @@ type vendor struct { } func MarshalVendorList(vendorList vendorList) string { - json, _ := json.Marshal(vendorList) + json, _ := jsonutil.Marshal(vendorList) return string(json) } diff --git a/hooks/hookanalytics/analytics_test.go b/hooks/hookanalytics/analytics_test.go index 177a9335da9..abf5e7f2ddc 100644 --- a/hooks/hookanalytics/analytics_test.go +++ b/hooks/hookanalytics/analytics_test.go @@ -1,9 +1,9 @@ package hookanalytics import ( - "encoding/json" "testing" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/stretchr/testify/assert" ) @@ -50,7 +50,7 @@ func TestAnalytics(t *testing.T) { Activity{Name: "define-blocks", Status: ActivityStatusError}, ) - gotAnalytics, err := json.Marshal(analytics) + gotAnalytics, err := jsonutil.Marshal(analytics) assert.NoError(t, err, "Failed to marshal analytics: %s", err) assert.JSONEq(t, string(expectedAnalytics), string(gotAnalytics)) } diff --git a/hooks/hookexecution/enricher_test.go b/hooks/hookexecution/enricher_test.go index 7bb19c2ecf2..24f0f1c1d45 100644 --- a/hooks/hookexecution/enricher_test.go +++ b/hooks/hookexecution/enricher_test.go @@ -10,6 +10,7 @@ import ( "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/hooks/hookanalytics" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -258,7 +259,7 @@ func TestGetModulesJSON(t *testing.T) { assert.Empty(t, expectedResponse) } else { var expectedExtBidResponse openrtb_ext.ExtBidResponse - err := json.Unmarshal(expectedResponse, &expectedExtBidResponse) + err := jsonutil.UnmarshalValid(expectedResponse, &expectedExtBidResponse) assert.NoError(t, err, "Failed to unmarshal prebid response extension") assert.JSONEq(t, string(expectedExtBidResponse.Prebid.Modules), string(modules)) } @@ -271,7 +272,7 @@ func getStageOutcomes(t *testing.T, file string) []StageOutcome { var stageOutcomesTest []StageOutcomeTest data := readFile(t, file) - err := json.Unmarshal(data, &stageOutcomesTest) + err := jsonutil.UnmarshalValid(data, &stageOutcomesTest) require.NoError(t, err, "Failed to unmarshal stage outcomes: %s", err) for _, stageT := range stageOutcomesTest { diff --git a/hooks/hookexecution/test_utils.go b/hooks/hookexecution/test_utils.go index bd94d5778b4..2a604d851f5 100644 --- a/hooks/hookexecution/test_utils.go +++ b/hooks/hookexecution/test_utils.go @@ -4,6 +4,7 @@ import ( "encoding/json" "testing" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/stretchr/testify/assert" ) @@ -16,8 +17,8 @@ func AssertEqualModulesData(t *testing.T, expectedData, actualData json.RawMessa var expectedModulesOutcome ModulesOutcome var actualModulesOutcome ModulesOutcome - assert.NoError(t, json.Unmarshal(expectedData, &expectedModulesOutcome), "Failed to unmarshal expected modules data.") - assert.NoError(t, json.Unmarshal(actualData, &actualModulesOutcome), "Failed to unmarshal actual modules data.") + assert.NoError(t, jsonutil.UnmarshalValid(expectedData, &expectedModulesOutcome), "Failed to unmarshal expected modules data.") + assert.NoError(t, jsonutil.UnmarshalValid(actualData, &actualModulesOutcome), "Failed to unmarshal actual modules data.") assert.Equal(t, expectedModulesOutcome.Errors, actualModulesOutcome.Errors, "Invalid error messages.") assert.Equal(t, expectedModulesOutcome.Warnings, actualModulesOutcome.Warnings, "Invalid warning messages.") diff --git a/hooks/plan_test.go b/hooks/plan_test.go index 5d2a504f0d1..8af49b42e17 100644 --- a/hooks/plan_test.go +++ b/hooks/plan_test.go @@ -2,12 +2,12 @@ package hooks import ( "context" - "encoding/json" "testing" "time" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/hooks/hookstage" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/stretchr/testify/assert" ) @@ -227,7 +227,7 @@ func TestPlanForRawAuctionStage(t *testing.T) { for name, test := range testCases { t.Run(name, func(t *testing.T) { account := new(config.Account) - if err := json.Unmarshal(test.giveAccountPlanData, &account.Hooks); err != nil { + if err := jsonutil.UnmarshalValid(test.giveAccountPlanData, &account.Hooks); err != nil { t.Fatal(err) } @@ -333,7 +333,7 @@ func TestPlanForProcessedAuctionStage(t *testing.T) { for name, test := range testCases { t.Run(name, func(t *testing.T) { account := new(config.Account) - if err := json.Unmarshal(test.giveAccountPlanData, &account.Hooks); err != nil { + if err := jsonutil.UnmarshalValid(test.giveAccountPlanData, &account.Hooks); err != nil { t.Fatal(err) } @@ -439,7 +439,7 @@ func TestPlanForBidderRequestStage(t *testing.T) { for name, test := range testCases { t.Run(name, func(t *testing.T) { account := new(config.Account) - if err := json.Unmarshal(test.giveAccountPlanData, &account.Hooks); err != nil { + if err := jsonutil.UnmarshalValid(test.giveAccountPlanData, &account.Hooks); err != nil { t.Fatal(err) } @@ -545,7 +545,7 @@ func TestPlanForRawBidderResponseStage(t *testing.T) { for name, test := range testCases { t.Run(name, func(t *testing.T) { account := new(config.Account) - if err := json.Unmarshal(test.giveAccountPlanData, &account.Hooks); err != nil { + if err := jsonutil.UnmarshalValid(test.giveAccountPlanData, &account.Hooks); err != nil { t.Fatal(err) } @@ -651,7 +651,7 @@ func TestPlanForAllProcessedBidResponsesStage(t *testing.T) { for name, test := range testCases { t.Run(name, func(t *testing.T) { account := new(config.Account) - if err := json.Unmarshal(test.giveAccountPlanData, &account.Hooks); err != nil { + if err := jsonutil.UnmarshalValid(test.giveAccountPlanData, &account.Hooks); err != nil { t.Fatal(err) } @@ -757,7 +757,7 @@ func TestPlanForAuctionResponseStage(t *testing.T) { for name, test := range testCases { t.Run(name, func(t *testing.T) { account := new(config.Account) - if err := json.Unmarshal(test.giveAccountPlanData, &account.Hooks); err != nil { + if err := jsonutil.UnmarshalValid(test.giveAccountPlanData, &account.Hooks); err != nil { t.Fatal(err) } @@ -779,12 +779,12 @@ func getPlanBuilder( var hostPlan config.HookExecutionPlan var defaultAccountPlan config.HookExecutionPlan - err = json.Unmarshal(hostPlanData, &hostPlan) + err = jsonutil.UnmarshalValid(hostPlanData, &hostPlan) if err != nil { return nil, err } - err = json.Unmarshal(accountPlanData, &defaultAccountPlan) + err = jsonutil.UnmarshalValid(accountPlanData, &defaultAccountPlan) if err != nil { return nil, err } diff --git a/modules/prebid/ortb2blocking/module_test.go b/modules/prebid/ortb2blocking/module_test.go index 23a8fc35bdc..ecae5ca8c0a 100644 --- a/modules/prebid/ortb2blocking/module_test.go +++ b/modules/prebid/ortb2blocking/module_test.go @@ -488,7 +488,7 @@ func TestHandleBidderRequestHook(t *testing.T) { bidRequest: &openrtb2.BidRequest{}, expectedBidRequest: &openrtb2.BidRequest{}, expectedHookResult: hookstage.HookResult[hookstage.BidderRequestPayload]{}, - expectedError: errors.New("failed to parse config: readObjectStart: expect { or n, but found ., error found in #1 byte of ...|...|..., bigger context ...|...|..."), + expectedError: errors.New("failed to parse config: expect { or n, but found ."), }, { description: "Expect error if nil BidRequest provided", diff --git a/openrtb_ext/convert_up_test.go b/openrtb_ext/convert_up_test.go index 78cd675d89e..3cafe8c1612 100644 --- a/openrtb_ext/convert_up_test.go +++ b/openrtb_ext/convert_up_test.go @@ -20,7 +20,7 @@ func TestConvertUpTo26(t *testing.T) { givenRequest: openrtb2.BidRequest{ Ext: json.RawMessage(`malformed`), }, - expectedErr: "req.ext is invalid: ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...", + expectedErr: "req.ext is invalid: expect { or n, but found m", }, { description: "2.4 -> 2.6", @@ -120,27 +120,27 @@ func TestConvertUpEnsureExt(t *testing.T) { { description: "Ext", givenRequest: openrtb2.BidRequest{Ext: json.RawMessage("malformed")}, - expectedErr: "req.ext is invalid: ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...", + expectedErr: "req.ext is invalid: expect { or n, but found m", }, { description: "Source.Ext", givenRequest: openrtb2.BidRequest{Source: &openrtb2.Source{Ext: json.RawMessage("malformed")}}, - expectedErr: "req.source.ext is invalid: ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...", + expectedErr: "req.source.ext is invalid: expect { or n, but found m", }, { description: "Regs.Ext", givenRequest: openrtb2.BidRequest{Regs: &openrtb2.Regs{Ext: json.RawMessage("malformed")}}, - expectedErr: "req.regs.ext is invalid: ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...", + expectedErr: "req.regs.ext is invalid: expect { or n, but found m", }, { description: "User.Ext", givenRequest: openrtb2.BidRequest{User: &openrtb2.User{Ext: json.RawMessage("malformed")}}, - expectedErr: "req.user.ext is invalid: ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...", + expectedErr: "req.user.ext is invalid: expect { or n, but found m", }, { description: "Imp.Ext", givenRequest: openrtb2.BidRequest{Imp: []openrtb2.Imp{{Ext: json.RawMessage("malformed")}}}, - expectedErr: "imp[0].imp.ext is invalid: ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...", + expectedErr: "imp[0].imp.ext is invalid: expect { or n, but found m", }, } diff --git a/openrtb_ext/device_test.go b/openrtb_ext/device_test.go index 1a3dbe8e2f4..86a0e1d7ff2 100644 --- a/openrtb_ext/device_test.go +++ b/openrtb_ext/device_test.go @@ -4,30 +4,31 @@ import ( "encoding/json" "testing" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/stretchr/testify/assert" ) func TestInvalidDeviceExt(t *testing.T) { var s ExtDevice - assert.EqualError(t, json.Unmarshal([]byte(`{"prebid":{"interstitial":{"minheightperc":0}}}`), &s), "request.device.ext.prebid.interstitial.minwidthperc must be a number between 0 and 100") - assert.EqualError(t, json.Unmarshal([]byte(`{"prebid":{"interstitial":{"minwidthperc":105}}}`), &s), "request.device.ext.prebid.interstitial.minwidthperc must be a number between 0 and 100") - assert.EqualError(t, json.Unmarshal([]byte(`{"prebid":{"interstitial":{"minwidthperc":true,"minheightperc":0}}}`), &s), "request.device.ext.prebid.interstitial.minwidthperc must be a number between 0 and 100") - assert.EqualError(t, json.Unmarshal([]byte(`{"prebid":{"interstitial":{"minwidthperc":null,"minheightperc":0}}}`), &s), "request.device.ext.prebid.interstitial.minwidthperc must be a number between 0 and 100") - assert.EqualError(t, json.Unmarshal([]byte(`{"prebid":{"interstitial":{"minwidthperc":"75","minheightperc":0}}}`), &s), "request.device.ext.prebid.interstitial.minwidthperc must be a number between 0 and 100") + assert.EqualError(t, jsonutil.UnmarshalValid([]byte(`{"prebid":{"interstitial":{"minheightperc":0}}}`), &s), "request.device.ext.prebid.interstitial.minwidthperc must be a number between 0 and 100") + assert.EqualError(t, jsonutil.UnmarshalValid([]byte(`{"prebid":{"interstitial":{"minwidthperc":105}}}`), &s), "request.device.ext.prebid.interstitial.minwidthperc must be a number between 0 and 100") + assert.EqualError(t, jsonutil.UnmarshalValid([]byte(`{"prebid":{"interstitial":{"minwidthperc":true,"minheightperc":0}}}`), &s), "request.device.ext.prebid.interstitial.minwidthperc must be a number between 0 and 100") + assert.EqualError(t, jsonutil.UnmarshalValid([]byte(`{"prebid":{"interstitial":{"minwidthperc":null,"minheightperc":0}}}`), &s), "request.device.ext.prebid.interstitial.minwidthperc must be a number between 0 and 100") + assert.EqualError(t, jsonutil.UnmarshalValid([]byte(`{"prebid":{"interstitial":{"minwidthperc":"75","minheightperc":0}}}`), &s), "request.device.ext.prebid.interstitial.minwidthperc must be a number between 0 and 100") - assert.EqualError(t, json.Unmarshal([]byte(`{"prebid":{"interstitial":{"minwidthperc":85}}}`), &s), "request.device.ext.prebid.interstitial.minheightperc must be a number between 0 and 100") - assert.EqualError(t, json.Unmarshal([]byte(`{"prebid":{"interstitial":{"minwidthperc":85,"minheightperc":-5}}}`), &s), "request.device.ext.prebid.interstitial.minheightperc must be a number between 0 and 100") - assert.EqualError(t, json.Unmarshal([]byte(`{"prebid":{"interstitial":{"minwidthperc":85,"minheightperc":false}}}`), &s), "request.device.ext.prebid.interstitial.minheightperc must be a number between 0 and 100") - assert.EqualError(t, json.Unmarshal([]byte(`{"prebid":{"interstitial":{"minwidthperc":85,"minheightperc":"75"}}}`), &s), "request.device.ext.prebid.interstitial.minheightperc must be a number between 0 and 100") + assert.EqualError(t, jsonutil.UnmarshalValid([]byte(`{"prebid":{"interstitial":{"minwidthperc":85}}}`), &s), "request.device.ext.prebid.interstitial.minheightperc must be a number between 0 and 100") + assert.EqualError(t, jsonutil.UnmarshalValid([]byte(`{"prebid":{"interstitial":{"minwidthperc":85,"minheightperc":-5}}}`), &s), "request.device.ext.prebid.interstitial.minheightperc must be a number between 0 and 100") + assert.EqualError(t, jsonutil.UnmarshalValid([]byte(`{"prebid":{"interstitial":{"minwidthperc":85,"minheightperc":false}}}`), &s), "request.device.ext.prebid.interstitial.minheightperc must be a number between 0 and 100") + assert.EqualError(t, jsonutil.UnmarshalValid([]byte(`{"prebid":{"interstitial":{"minwidthperc":85,"minheightperc":"75"}}}`), &s), "request.device.ext.prebid.interstitial.minheightperc must be a number between 0 and 100") } func TestValidDeviceExt(t *testing.T) { var s ExtDevice - assert.NoError(t, json.Unmarshal([]byte(`{"prebid":{}}`), &s)) + assert.NoError(t, jsonutil.UnmarshalValid([]byte(`{"prebid":{}}`), &s)) assert.Nil(t, s.Prebid.Interstitial) - assert.NoError(t, json.Unmarshal([]byte(`{}`), &s)) + assert.NoError(t, jsonutil.UnmarshalValid([]byte(`{}`), &s)) assert.Nil(t, s.Prebid.Interstitial) - assert.NoError(t, json.Unmarshal([]byte(`{"prebid":{"interstitial":{"minwidthperc":75,"minheightperc":60}}}`), &s)) + assert.NoError(t, jsonutil.UnmarshalValid([]byte(`{"prebid":{"interstitial":{"minwidthperc":75,"minheightperc":60}}}`), &s)) assert.EqualValues(t, 75, s.Prebid.Interstitial.MinWidthPerc) assert.EqualValues(t, 60, s.Prebid.Interstitial.MinHeightPerc) } diff --git a/openrtb_ext/imp_appnexus_test.go b/openrtb_ext/imp_appnexus_test.go index cbd6779d5da..a226c0d8410 100644 --- a/openrtb_ext/imp_appnexus_test.go +++ b/openrtb_ext/imp_appnexus_test.go @@ -1,9 +1,9 @@ package openrtb_ext import ( - "encoding/json" "testing" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/stretchr/testify/assert" ) @@ -31,7 +31,7 @@ func TestKeywordsUnmarshalJSON(t *testing.T) { for _, test := range validTestCases { var keywords keywords - assert.NoError(t, json.Unmarshal(test.input, &keywords), test.desc) + assert.NoError(t, jsonutil.UnmarshalValid(test.input, &keywords), test.desc) assert.Equal(t, test.expected, keywords.Keywords.String()) } @@ -42,6 +42,6 @@ func TestKeywordsUnmarshalJSON(t *testing.T) { for _, test := range invalidTestCases { var keywords keywords - assert.Error(t, json.Unmarshal(test.input, &keywords), test.desc) + assert.Error(t, jsonutil.UnmarshalValid(test.input, &keywords), test.desc) } } diff --git a/openrtb_ext/request_test.go b/openrtb_ext/request_test.go index a05bae3a6bf..ad6a655b022 100644 --- a/openrtb_ext/request_test.go +++ b/openrtb_ext/request_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/prebid/openrtb/v19/openrtb2" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/util/ptrutil" "github.com/stretchr/testify/assert" ) @@ -37,7 +38,7 @@ func TestGranularityUnmarshal(t *testing.T) { for _, tg := range testGroups { for i, tc := range tg.in { var resolved PriceGranularity - err := json.Unmarshal(tc.json, &resolved) + err := jsonutil.UnmarshalValid(tc.json, &resolved) // Assert validation error if tg.expectError && !assert.Errorf(t, err, "%s test case %d", tg.desc, i) { diff --git a/openrtb_ext/site_test.go b/openrtb_ext/site_test.go index 67ec6cc4f99..7a7140282f2 100644 --- a/openrtb_ext/site_test.go +++ b/openrtb_ext/site_test.go @@ -1,28 +1,28 @@ package openrtb_ext_test import ( - "encoding/json" "testing" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/stretchr/testify/assert" ) func TestInvalidSiteExt(t *testing.T) { var s openrtb_ext.ExtSite - assert.EqualError(t, json.Unmarshal([]byte(`{"amp":-1}`), &s), "request.site.ext.amp must be either 1, 0, or undefined") - assert.EqualError(t, json.Unmarshal([]byte(`{"amp":2}`), &s), "request.site.ext.amp must be either 1, 0, or undefined") - assert.EqualError(t, json.Unmarshal([]byte(`{"amp":true}`), &s), "request.site.ext.amp must be either 1, 0, or undefined") - assert.EqualError(t, json.Unmarshal([]byte(`{"amp":null}`), &s), "request.site.ext.amp must be either 1, 0, or undefined") - assert.EqualError(t, json.Unmarshal([]byte(`{"amp":"1"}`), &s), "request.site.ext.amp must be either 1, 0, or undefined") + assert.EqualError(t, jsonutil.UnmarshalValid([]byte(`{"amp":-1}`), &s), "request.site.ext.amp must be either 1, 0, or undefined") + assert.EqualError(t, jsonutil.UnmarshalValid([]byte(`{"amp":2}`), &s), "request.site.ext.amp must be either 1, 0, or undefined") + assert.EqualError(t, jsonutil.UnmarshalValid([]byte(`{"amp":true}`), &s), "request.site.ext.amp must be either 1, 0, or undefined") + assert.EqualError(t, jsonutil.UnmarshalValid([]byte(`{"amp":null}`), &s), "request.site.ext.amp must be either 1, 0, or undefined") + assert.EqualError(t, jsonutil.UnmarshalValid([]byte(`{"amp":"1"}`), &s), "request.site.ext.amp must be either 1, 0, or undefined") } func TestValidSiteExt(t *testing.T) { var s openrtb_ext.ExtSite - assert.NoError(t, json.Unmarshal([]byte(`{"amp":0}`), &s)) + assert.NoError(t, jsonutil.UnmarshalValid([]byte(`{"amp":0}`), &s)) assert.EqualValues(t, 0, s.AMP) - assert.NoError(t, json.Unmarshal([]byte(`{"amp":1}`), &s)) + assert.NoError(t, jsonutil.UnmarshalValid([]byte(`{"amp":1}`), &s)) assert.EqualValues(t, 1, s.AMP) - assert.NoError(t, json.Unmarshal([]byte(`{"amp": 1 }`), &s)) + assert.NoError(t, jsonutil.UnmarshalValid([]byte(`{"amp": 1 }`), &s)) assert.EqualValues(t, 1, s.AMP) } diff --git a/ortb/default_test.go b/ortb/default_test.go index 330434fba27..8bda02ef4f5 100644 --- a/ortb/default_test.go +++ b/ortb/default_test.go @@ -10,6 +10,7 @@ import ( "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/prebid/prebid-server/util/ptrutil" ) @@ -32,7 +33,7 @@ func TestSetDefaults(t *testing.T) { name: "malformed request.ext", givenRequest: openrtb2.BidRequest{Ext: json.RawMessage(`malformed`)}, expectedRequest: openrtb2.BidRequest{Ext: json.RawMessage(`malformed`)}, - expectedErr: "ReadMapCB: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|...", + expectedErr: "expect { or n, but found m", }, { name: "targeting", // tests integration with setDefaultsTargeting @@ -68,10 +69,10 @@ func TestSetDefaults(t *testing.T) { assert.Equal(t, &test.expectedRequest, wrapper.BidRequest, "Request") } else { // assert request as json to ignore order in ext fields - expectedRequestJSON, err := json.Marshal(test.expectedRequest) + expectedRequestJSON, err := jsonutil.Marshal(test.expectedRequest) require.NoError(t, err, "Marshal Expected Request") - actualRequestJSON, err := json.Marshal(wrapper.BidRequest) + actualRequestJSON, err := jsonutil.Marshal(wrapper.BidRequest) require.NoError(t, err, "Marshal Actual Request") assert.JSONEq(t, string(expectedRequestJSON), string(actualRequestJSON), "Request") diff --git a/prebid_cache_client/client_test.go b/prebid_cache_client/client_test.go index ec390364849..f20d3c7829f 100644 --- a/prebid_cache_client/client_test.go +++ b/prebid_cache_client/client_test.go @@ -13,6 +13,7 @@ import ( "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/metrics" metricsConf "github.com/prebid/prebid-server/metrics/config" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -297,7 +298,7 @@ func newHandler(numResponses int) http.HandlerFunc { resp.Responses[i].UUID = strconv.Itoa(i) } - respBytes, _ := json.Marshal(resp) + respBytes, _ := jsonutil.Marshal(resp) w.Write(respBytes) }) } diff --git a/router/router_test.go b/router/router_test.go index 41c2a724c91..f4f7715e6c3 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -9,6 +9,7 @@ import ( "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/stretchr/testify/assert" ) @@ -45,7 +46,7 @@ func TestNewJsonDirectoryServer(t *testing.T) { handler(recorder, request, nil) var data map[string]json.RawMessage - json.Unmarshal(recorder.Body.Bytes(), &data) + jsonutil.UnmarshalValid(recorder.Body.Bytes(), &data) // Make sure that every adapter has a json schema by the same name associated with it. adapterFiles, err := os.ReadDir(adapterDirectory) diff --git a/schain/schainwriter_test.go b/schain/schainwriter_test.go index e98c962b4fa..9288b531d56 100644 --- a/schain/schainwriter_test.go +++ b/schain/schainwriter_test.go @@ -6,6 +6,7 @@ import ( "github.com/prebid/openrtb/v19/openrtb2" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/stretchr/testify/assert" ) @@ -197,7 +198,7 @@ func TestSChainWriter(t *testing.T) { var reqExt *openrtb_ext.ExtRequest if tt.giveRequest.Ext != nil { reqExt = &openrtb_ext.ExtRequest{} - err := json.Unmarshal(tt.giveRequest.Ext, reqExt) + err := jsonutil.UnmarshalValid(tt.giveRequest.Ext, reqExt) if err != nil { t.Error("Unable to unmarshal request.ext") } diff --git a/stored_requests/backends/file_fetcher/fetcher_test.go b/stored_requests/backends/file_fetcher/fetcher_test.go index 3c585f9f456..a69945641f1 100644 --- a/stored_requests/backends/file_fetcher/fetcher_test.go +++ b/stored_requests/backends/file_fetcher/fetcher_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/prebid/prebid-server/stored_requests" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/stretchr/testify/assert" ) @@ -58,7 +59,7 @@ func validateStoredReqOne(t *testing.T, storedRequests map[string]json.RawMessag } var req1Val map[string]string - if err := json.Unmarshal(value, &req1Val); err != nil { + if err := jsonutil.UnmarshalValid(value, &req1Val); err != nil { t.Errorf("Failed to unmarshal 1: %v", err) } if len(req1Val) != 1 { @@ -80,7 +81,7 @@ func validateStoredReqTwo(t *testing.T, storedRequests map[string]json.RawMessag } var req2Val string - if err := json.Unmarshal(value, &req2Val); err != nil { + if err := jsonutil.UnmarshalValid(value, &req2Val); err != nil { t.Errorf("Failed to unmarshal %d: %v", 2, err) } if req2Val != `esca"ped` { @@ -95,7 +96,7 @@ func validateImp(t *testing.T, storedImps map[string]json.RawMessage) { } var impVal map[string]bool - if err := json.Unmarshal(value, &impVal); err != nil { + if err := jsonutil.UnmarshalValid(value, &impVal); err != nil { t.Errorf("Failed to unmarshal some-imp: %v", err) } if len(impVal) != 1 { diff --git a/stored_requests/backends/http_fetcher/fetcher_test.go b/stored_requests/backends/http_fetcher/fetcher_test.go index 3c8ca7cb070..80be6918ad8 100644 --- a/stored_requests/backends/http_fetcher/fetcher_test.go +++ b/stored_requests/backends/http_fetcher/fetcher_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/stretchr/testify/assert" ) @@ -233,7 +234,7 @@ func newHandler(t *testing.T, expectReqIDs []string, expectImpIDs []string, json Imps: impIDResponse, } - if respBytes, err := json.Marshal(respObj); err != nil { + if respBytes, err := jsonutil.Marshal(respObj); err != nil { t.Errorf("failed to marshal responseContract in test: %v", err) w.WriteHeader(http.StatusInternalServerError) } else { @@ -267,7 +268,7 @@ func newAccountHandler(t *testing.T, expectAccIDs []string) func(w http.Response Accounts: accIDResponse, } - if respBytes, err := json.Marshal(respObj); err != nil { + if respBytes, err := jsonutil.Marshal(respObj); err != nil { t.Errorf("failed to marshal responseContract in test: %v", err) w.WriteHeader(http.StatusInternalServerError) } else { @@ -327,7 +328,7 @@ func richSplit(queryVal string) []string { } func jsonifyID(id string) json.RawMessage { - if b, err := json.Marshal(id); err != nil { + if b, err := jsonutil.Marshal(id); err != nil { return json.RawMessage([]byte("\"error encoding ID=" + id + "\"")) } else { return json.RawMessage(b) diff --git a/stored_requests/events/http/http_test.go b/stored_requests/events/http/http_test.go index a185a3c2360..663d8cd193f 100644 --- a/stored_requests/events/http/http_test.go +++ b/stored_requests/events/http/http_test.go @@ -2,13 +2,13 @@ package http import ( "context" - "encoding/json" "fmt" httpCore "net/http" "net/http/httptest" "testing" "time" + "github.com/prebid/prebid-server/util/jsonutil" "github.com/stretchr/testify/assert" ) @@ -151,14 +151,14 @@ func TestStartup(t *testing.T) { t.Run(fmt.Sprintf("Step %d", i+1), func(t *testing.T) { // Check expected Saves if len(test.saves) > 0 { - saves, err := json.Marshal(<-ev.Saves()) + saves, err := jsonutil.Marshal(<-ev.Saves()) assert.NoError(t, err, `Failed to marshal event.Save object: %v`, err) assert.JSONEq(t, test.saves, string(saves)) } assert.Empty(t, ev.Saves(), "Unexpected additional messages in save channel") // Check expected Invalidations if len(test.invalidations) > 0 { - invalidations, err := json.Marshal(<-ev.Invalidations()) + invalidations, err := jsonutil.Marshal(<-ev.Invalidations()) assert.NoError(t, err, `Failed to marshal event.Invalidation object: %v`, err) assert.JSONEq(t, test.invalidations, string(invalidations)) } diff --git a/util/jsonutil/jsonutil.go b/util/jsonutil/jsonutil.go index 5c54a017b64..a8981477dc6 100644 --- a/util/jsonutil/jsonutil.go +++ b/util/jsonutil/jsonutil.go @@ -3,17 +3,17 @@ package jsonutil import ( "bytes" "encoding/json" + "io" + "strings" + jsoniter "github.com/json-iterator/go" "github.com/prebid/prebid-server/errortypes" - "io" ) -var comma = []byte(",")[0] -var colon = []byte(":")[0] -var sqBracket = []byte("]")[0] -var openCurlyBracket = []byte("{")[0] -var closingCurlyBracket = []byte("}")[0] -var quote = []byte(`"`)[0] +var comma = byte(',') +var colon = byte(':') +var sqBracket = byte(']') +var closingCurlyBracket = byte('}') // Finds element in json byte array with any level of nesting func FindElement(extension []byte, elementNames ...string) (bool, int64, int64, error) { @@ -128,10 +128,9 @@ var jsonConfigValidationOff = jsoniter.Config{ // any validation on the data. An unmarshal error is returned if a non-validation error occurs. func Unmarshal(data []byte, v interface{}) error { err := jsonConfigValidationOff.Unmarshal(data, v) - if err != nil { return &errortypes.FailedToUnmarshal{ - Message: err.Error(), + Message: tryExtractErrorMessage(err), } } return nil @@ -142,7 +141,7 @@ func Unmarshal(data []byte, v interface{}) error { func UnmarshalValid(data []byte, v interface{}) error { if err := jsonConfigValidationOn.Unmarshal(data, v); err != nil { return &errortypes.FailedToUnmarshal{ - Message: err.Error(), + Message: tryExtractErrorMessage(err), } } return nil @@ -159,3 +158,56 @@ func Marshal(v interface{}) ([]byte, error) { } return data, nil } + +// tryExtractErrorMessage attempts to extract a sane error message from the json-iter package. The errors +// returned from that library are not types and include a lot of extra information we don't want to respond with. +// This is hacky, but it's the only downside to the json-iter library. +func tryExtractErrorMessage(err error) string { + msg := err.Error() + + msgEndIndex := strings.LastIndex(msg, ", error found in #") + if msgEndIndex == -1 { + return msg + } + + msgStartIndex := strings.Index(msg, ": ") + if msgStartIndex == -1 { + return msg + } + + operationStack := []string{msg[0:msgStartIndex]} + for { + msgStartIndexNext := strings.Index(msg[msgStartIndex+2:], ": ") + + // no more matches + if msgStartIndexNext == -1 { + break + } + + // matches occur after the end message marker (sanity check) + if (msgStartIndex + msgStartIndexNext) >= msgEndIndex { + break + } + + // match should not contain a space, indicates operation is really an error message + match := msg[msgStartIndex+2 : msgStartIndex+2+msgStartIndexNext] + if strings.Contains(match, " ") { + break + } + + operationStack = append(operationStack, match) + msgStartIndex += msgStartIndexNext + 2 + } + + if len(operationStack) > 1 && isLikelyDetailedErrorMessage(msg[msgStartIndex+2:]) { + return "cannot unmarshal " + operationStack[len(operationStack)-2] + ": " + msg[msgStartIndex+2:msgEndIndex] + } + + return msg[msgStartIndex+2 : msgEndIndex] +} + +// isLikelyDetailedErrorMessage checks if the json unmarshal error contains enough information such +// that the caller clearly understands the context, where the structure name is not needed. +func isLikelyDetailedErrorMessage(msg string) bool { + return !strings.HasPrefix(msg, "request.") +} diff --git a/util/jsonutil/jsonutil_test.go b/util/jsonutil/jsonutil_test.go index 12b1fd5e803..09fb6727309 100644 --- a/util/jsonutil/jsonutil_test.go +++ b/util/jsonutil/jsonutil_test.go @@ -1,13 +1,14 @@ package jsonutil import ( - "github.com/stretchr/testify/assert" + "errors" "strings" "testing" + + "github.com/stretchr/testify/assert" ) func TestDropElement(t *testing.T) { - tests := []struct { description string input []byte @@ -183,3 +184,59 @@ func TestDropElement(t *testing.T) { } } } + +func TestTryExtractErrorMessage(t *testing.T) { + tests := []struct { + name string + givenErr string + expectedMsg string + }{ + { + name: "level-1", + givenErr: "readObjectStart: expect { or n, but found m, error found in #1 byte of ...|malformed|..., bigger context ...|malformed|..", + expectedMsg: "expect { or n, but found m", + }, + { + name: "level-2", + givenErr: "openrtb_ext.ExtRequestPrebidCache.Bids: readObjectStart: expect { or n, but found t, error found in #10 byte of ...|:{\"bids\":true}}|..., bigger context ...|{\"cache\":{\"bids\":true}}|...", + expectedMsg: "cannot unmarshal openrtb_ext.ExtRequestPrebidCache.Bids: expect { or n, but found t", + }, + { + name: "level-3+", + givenErr: "openrtb_ext.ExtRequestPrebid.Cache: openrtb_ext.ExtRequestPrebidCache.Bids: readObjectStart: expect { or n, but found t, error found in #10 byte of ...|:{\"bids\":true}}|..., bigger context ...|{\"cache\":{\"bids\":true}}|...", + expectedMsg: "cannot unmarshal openrtb_ext.ExtRequestPrebidCache.Bids: expect { or n, but found t", + }, + { + name: "error-msg", + givenErr: "Skip: do not know how to skip: 109, error found in #10 byte of ...|prebid\": malformed}|..., bigger context ...|{\"prebid\": malformed}|...", + expectedMsg: "do not know how to skip: 109", + }, + { + name: "specific", + givenErr: "openrtb_ext.ExtDevicePrebid.Interstitial: unmarshalerDecoder: request.device.ext.prebid.interstitial.minwidthperc must be a number between 0 and 100, error found in #10 byte of ...| }\n }|..., bigger context ...|: 120,\n \"minheightperc\": 60\n }\n }|...", + expectedMsg: "request.device.ext.prebid.interstitial.minwidthperc must be a number between 0 and 100", + }, + { + name: "normal", + givenErr: "normal error message", + expectedMsg: "normal error message", + }, + { + name: "norma-false-start", + givenErr: "false: normal error message", + expectedMsg: "false: normal error message", + }, + { + name: "norma-false-end", + givenErr: "normal error message, error found in #10 but doesn't follow format", + expectedMsg: "normal error message, error found in #10 but doesn't follow format", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := tryExtractErrorMessage(errors.New(test.givenErr)) + assert.Equal(t, test.expectedMsg, result) + }) + } +} diff --git a/util/jsonutil/stringInt_test.go b/util/jsonutil/stringInt_test.go index e8639c7acee..cd2f3476e46 100644 --- a/util/jsonutil/stringInt_test.go +++ b/util/jsonutil/stringInt_test.go @@ -1,7 +1,6 @@ package jsonutil import ( - "encoding/json" "testing" "github.com/buger/jsonparser" @@ -16,27 +15,27 @@ func TestStringIntUnmarshalJSON(t *testing.T) { t.Run("string", func(t *testing.T) { jsonData := []byte(`{"item_id":"30"}`) var item Item - assert.NoError(t, json.Unmarshal(jsonData, &item)) + assert.NoError(t, UnmarshalValid(jsonData, &item)) assert.Equal(t, 30, int(item.ItemId)) }) t.Run("int", func(t *testing.T) { jsonData := []byte(`{"item_id":30}`) var item Item - assert.NoError(t, json.Unmarshal(jsonData, &item)) + assert.NoError(t, UnmarshalValid(jsonData, &item)) assert.Equal(t, 30, int(item.ItemId)) }) t.Run("empty_id", func(t *testing.T) { jsonData := []byte(`{"item_id": ""}`) var item Item - assert.NoError(t, json.Unmarshal(jsonData, &item)) + assert.NoError(t, UnmarshalValid(jsonData, &item)) }) t.Run("invalid_input", func(t *testing.T) { jsonData := []byte(`{"item_id":true}`) var item Item - err := json.Unmarshal(jsonData, &item) - assert.Equal(t, jsonparser.MalformedValueError, err) + err := UnmarshalValid(jsonData, &item) + assert.EqualError(t, err, "cannot unmarshal jsonutil.Item.ItemId: "+jsonparser.MalformedValueError.Error()) }) } diff --git a/version/xprebidheader_test.go b/version/xprebidheader_test.go index 90aef6cb420..a1c7b355bb8 100644 --- a/version/xprebidheader_test.go +++ b/version/xprebidheader_test.go @@ -1,13 +1,13 @@ package version import ( - "encoding/json" "testing" "github.com/prebid/openrtb/v19/openrtb2" "github.com/stretchr/testify/assert" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/util/jsonutil" ) func TestBuildXPrebidHeader(t *testing.T) { @@ -134,12 +134,12 @@ func TestBuildXPrebidHeaderForRequest(t *testing.T) { for _, test := range testCases { req := &openrtb2.BidRequest{} if test.requestExt != nil { - reqExt, err := json.Marshal(test.requestExt) + reqExt, err := jsonutil.Marshal(test.requestExt) assert.NoError(t, err, test.description+":err marshalling reqExt") req.Ext = reqExt } if test.requestAppExt != nil { - reqAppExt, err := json.Marshal(test.requestAppExt) + reqAppExt, err := jsonutil.Marshal(test.requestAppExt) assert.NoError(t, err, test.description+":err marshalling reqAppExt") req.App = &openrtb2.App{Ext: reqAppExt} } From 240a7ce6fddb14a66b8320fee1fce089fee7f55f Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Wed, 18 Oct 2023 17:04:19 -0400 Subject: [PATCH 10/10] Fix Merge Conflicts --- firstpartydata/first_party_data_test.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/firstpartydata/first_party_data_test.go b/firstpartydata/first_party_data_test.go index 37d2117254f..61369738bf3 100644 --- a/firstpartydata/first_party_data_test.go +++ b/firstpartydata/first_party_data_test.go @@ -482,7 +482,7 @@ func TestExtractBidderConfigFPD(t *testing.T) { require.NoError(t, err, "Load Test File") givenRequestExtPrebid := &openrtb_ext.ExtRequestPrebid{} - err = jsonutil.UnmarshalValid(fpdFile.InputRequestData, givenRequestExtPrebid) + err = jsonutil.UnmarshalValid(testFile.InputRequestData, givenRequestExtPrebid) require.NoError(t, err, "Cannot Load Test Conditions") testRequest := &openrtb_ext.RequestExt{} @@ -538,17 +538,13 @@ func TestResolveFPD(t *testing.T) { require.NoError(t, err, "Load Test File") request := &openrtb2.BidRequest{} - err = jsonutil.UnmarshalValid(fpdFile.InputRequestData, &request) + err = jsonutil.UnmarshalValid(testFile.InputRequestData, &request) require.NoError(t, err, "Cannot Load Request") originalRequest := &openrtb2.BidRequest{} - err = jsonutil.UnmarshalValid(fpdFile.InputRequestData, &originalRequest) + err = jsonutil.UnmarshalValid(testFile.InputRequestData, &originalRequest) require.NoError(t, err, "Cannot Load Request") - outputReq := &openrtb2.BidRequest{} - err = jsonutil.UnmarshalValid(fpdFile.OutputRequestData, &outputReq) - require.NoError(t, err, "Cannot Load Output Request") - reqExtFPD := make(map[string][]byte) reqExtFPD["site"] = testFile.GlobalFPD["site"] reqExtFPD["app"] = testFile.GlobalFPD["app"] @@ -645,14 +641,14 @@ func TestExtractFPDForBidders(t *testing.T) { require.NoError(t, err, "Load Test File") var expectedRequest openrtb2.BidRequest - err = jsonutil.UnmarshalValid(fpdFile.OutputRequestData, &expectedRequest) + err = jsonutil.UnmarshalValid(testFile.OutputRequestData, &expectedRequest) if err != nil { t.Errorf("Unable to unmarshal input request: %s", path) } resultRequest := &openrtb_ext.RequestWrapper{} resultRequest.BidRequest = &openrtb2.BidRequest{} - err = jsonutil.UnmarshalValid(fpdFile.InputRequestData, resultRequest.BidRequest) + err = jsonutil.UnmarshalValid(testFile.InputRequestData, resultRequest.BidRequest) assert.NoError(t, err, "Error should be nil") resultFPD, errL := ExtractFPDForBidders(resultRequest)