From 08488d2c8b25b3effd7ed0e382dbc674536023f9 Mon Sep 17 00:00:00 2001 From: Mansi Nahar Date: Mon, 18 Oct 2021 13:34:22 -0400 Subject: [PATCH] Revert "Remove /auction Endpoint (#2033)" (#2048) This reverts commit fe211a9010bc264e43bfa7a396a3078c19de5ebf. --- adapters/adform/adform.go | 173 +++- adapters/adform/adform_test.go | 226 ++++- adapters/adhese/adhese.go | 2 +- adapters/adman/adman.go | 4 + adapters/appnexus/appnexus.go | 230 ++++- adapters/appnexus/appnexus_test.go | 421 ++++++++- adapters/connectad/connectad.go | 4 + adapters/consumable/instant.go | 2 +- adapters/consumable/utils.go | 20 + adapters/conversant/cnvr_legacy.go | 291 ++++++ adapters/conversant/cnvr_legacy_test.go | 853 ++++++++++++++++++ adapters/deepintent/deepintent.go | 4 + adapters/dmx/dmx_test.go | 4 + adapters/infoawarebidder_test.go | 1 + adapters/ix/ix.go | 248 ++++- adapters/ix/ix_test.go | 703 ++++++++++++++- adapters/legacy.go | 97 ++ adapters/openrtb_util.go | 174 ++++ adapters/openrtb_util_test.go | 543 +++++++++++ adapters/pubmatic/pubmatic.go | 315 ++++++- adapters/pubmatic/pubmatic_test.go | 673 +++++++++++++- adapters/pulsepoint/pulsepoint.go | 201 ++++- adapters/pulsepoint/pulsepoint_test.go | 294 +++++- adapters/rubicon/rubicon.go | 335 ++++++- adapters/rubicon/rubicon_test.go | 658 +++++++++++++- adapters/sharethrough/utils_test.go | 4 + adapters/sonobi/sonobi.go | 10 + adapters/sovrn/sovrn.go | 167 +++- adapters/sovrn/sovrn_test.go | 278 +++++- analytics/filesystem/file_module.go | 2 + analytics/pubstack/pubstack_module_test.go | 44 + cache/dummycache/dummycache.go | 65 ++ cache/dummycache/dummycache_test.go | 31 + cache/filecache/filecache.go | 123 +++ cache/filecache/filecache_test.go | 79 ++ cache/legacy.go | 33 + cache/postgrescache/postgrescache.go | 139 +++ cache/postgrescache/postgrescache_test.go | 94 ++ config/config.go | 17 + config/config_test.go | 13 + config/stored_requests.go | 1 + config/structlog.go | 4 +- endpoints/auction.go | 513 +++++++++++ endpoints/auction_test.go | 654 ++++++++++++++ endpoints/openrtb2/auction.go | 9 + endpoints/openrtb2/auction_test.go | 16 + endpoints/openrtb2/video_auction_test.go | 16 + exchange/auction.go | 7 + exchange/auction_test.go | 6 + exchange/bidder_test.go | 1 + exchange/exchange_test.go | 2 +- exchange/targeting_test.go | 19 +- go.mod | 3 + go.sum | 4 + main.go | 3 + metrics/config/metrics.go | 11 + metrics/config/metrics_test.go | 2 + metrics/go_metrics.go | 10 + metrics/go_metrics_test.go | 4 + metrics/metrics.go | 1 + metrics/metrics_mock.go | 5 + metrics/prometheus/prometheus.go | 4 + metrics/prometheus/prometheus_test.go | 22 + metrics/prometheus/type_conversion.go | 9 + openrtb_ext/bidders.go | 2 + openrtb_ext/imp_beachfront.go | 6 +- openrtb_ext/imp_nanointeractive.go | 8 +- pbs/pbsrequest.go | 403 +++++++++ pbs/pbsrequest_test.go | 735 +++++++++++++++ pbs/pbsresponse.go | 84 ++ pbs/pbsresponse_test.go | 88 ++ pbs/usersync.go | 26 +- prebid_cache_client/client.go | 2 +- prebid_cache_client/client_test.go | 12 +- prebid_cache_client/prebid_cache.go | 122 +++ prebid_cache_client/prebid_cache_test.go | 150 +++ privacy/ccpa/parsedpolicy_test.go | 11 + router/router.go | 94 +- router/router_test.go | 45 + server/prometheus.go | 5 +- .../backends/file_fetcher/fetcher_test.go | 2 +- .../backends/http_fetcher/fetcher.go | 1 + .../backends/http_fetcher/fetcher_test.go | 38 + stored_requests/caches/nil_cache/nil_cache.go | 2 + stored_requests/config/config.go | 14 +- stored_requests/events/events.go | 2 +- usersync/cookie_test.go | 2 +- 87 files changed, 10648 insertions(+), 107 deletions(-) create mode 100644 adapters/consumable/utils.go create mode 100644 adapters/conversant/cnvr_legacy.go create mode 100644 adapters/conversant/cnvr_legacy_test.go create mode 100644 adapters/legacy.go create mode 100644 adapters/openrtb_util.go create mode 100644 adapters/openrtb_util_test.go create mode 100644 cache/dummycache/dummycache.go create mode 100644 cache/dummycache/dummycache_test.go create mode 100644 cache/filecache/filecache.go create mode 100644 cache/filecache/filecache_test.go create mode 100644 cache/legacy.go create mode 100644 cache/postgrescache/postgrescache.go create mode 100644 cache/postgrescache/postgrescache_test.go create mode 100644 endpoints/auction.go create mode 100644 endpoints/auction_test.go create mode 100644 pbs/pbsrequest.go create mode 100644 pbs/pbsrequest_test.go create mode 100644 pbs/pbsresponse.go create mode 100644 pbs/pbsresponse_test.go create mode 100644 prebid_cache_client/prebid_cache.go create mode 100644 prebid_cache_client/prebid_cache_test.go diff --git a/adapters/adform/adform.go b/adapters/adform/adform.go index a432bb075b3..225c7af35d4 100644 --- a/adapters/adform/adform.go +++ b/adapters/adform/adform.go @@ -2,27 +2,31 @@ package adform import ( "bytes" + "context" "encoding/base64" "encoding/json" "errors" "fmt" + "io/ioutil" "net/http" "net/url" "strconv" "strings" + "github.com/buger/jsonparser" + "github.com/mxmCherry/openrtb/v15/openrtb2" "github.com/prebid/prebid-server/adapters" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/openrtb_ext" - - "github.com/buger/jsonparser" - "github.com/mxmCherry/openrtb/v15/openrtb2" + "github.com/prebid/prebid-server/pbs" + "golang.org/x/net/context/ctxhttp" ) const version = "0.1.3" type AdformAdapter struct { + http *adapters.HTTPAdapter URL *url.URL version string } @@ -96,6 +100,155 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters return bidder, nil } +// used for cookies and such +func (a *AdformAdapter) Name() string { + return "adform" +} + +func (a *AdformAdapter) SkipNoCookies() bool { + return false +} + +func (a *AdformAdapter) Call(ctx context.Context, request *pbs.PBSRequest, bidder *pbs.PBSBidder) (pbs.PBSBidSlice, error) { + adformRequest, err := pbsRequestToAdformRequest(a, request, bidder) + if err != nil { + return nil, err + } + + uri := adformRequest.buildAdformUrl(a) + + debug := &pbs.BidderDebug{RequestURI: uri} + if request.IsDebug { + bidder.Debug = append(bidder.Debug, debug) + } + + httpRequest, err := http.NewRequest("GET", uri, nil) + if err != nil { + return nil, err + } + + httpRequest.Header = adformRequest.buildAdformHeaders(a) + + response, err := ctxhttp.Do(ctx, a.http.Client, httpRequest) + if err != nil { + return nil, err + } + + debug.StatusCode = response.StatusCode + + if response.StatusCode == 204 { + return nil, nil + } + + defer response.Body.Close() + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return nil, err + } + responseBody := string(body) + + if response.StatusCode == http.StatusBadRequest { + return nil, &errortypes.BadInput{ + Message: fmt.Sprintf("HTTP status %d; body: %s", response.StatusCode, responseBody), + } + } + + if response.StatusCode != 200 { + return nil, &errortypes.BadServerResponse{ + Message: fmt.Sprintf("HTTP status %d; body: %s", response.StatusCode, responseBody), + } + } + + if request.IsDebug { + debug.ResponseBody = responseBody + } + + adformBids, err := parseAdformBids(body) + if err != nil { + return nil, err + } + + bids := toPBSBidSlice(adformBids, adformRequest) + + return bids, nil +} + +func pbsRequestToAdformRequest(a *AdformAdapter, request *pbs.PBSRequest, bidder *pbs.PBSBidder) (*adformRequest, error) { + adUnits := make([]*adformAdUnit, 0, len(bidder.AdUnits)) + for _, adUnit := range bidder.AdUnits { + var adformAdUnit adformAdUnit + if err := json.Unmarshal(adUnit.Params, &adformAdUnit); err != nil { + return nil, err + } + mid, err := adformAdUnit.MasterTagId.Int64() + if err != nil { + return nil, &errortypes.BadInput{ + Message: err.Error(), + } + } + if mid <= 0 { + return nil, &errortypes.BadInput{ + Message: fmt.Sprintf("master tag(placement) id is invalid=%s", adformAdUnit.MasterTagId), + } + } + adformAdUnit.bidId = adUnit.BidID + adformAdUnit.adUnitCode = adUnit.Code + adUnits = append(adUnits, &adformAdUnit) + } + + userId, _, _ := request.Cookie.GetUID(a.Name()) + + gdprApplies := request.ParseGDPR() + if gdprApplies != "0" && gdprApplies != "1" { + gdprApplies = "" + } + consent := request.ParseConsent() + + return &adformRequest{ + adUnits: adUnits, + ip: request.Device.IP, + advertisingId: request.Device.IFA, + userAgent: request.Device.UA, + bidderCode: bidder.BidderCode, + isSecure: request.Secure == 1, + referer: request.Url, + userId: userId, + tid: request.Tid, + gdprApplies: gdprApplies, + consent: consent, + currency: defaultCurrency, + }, nil +} + +func toPBSBidSlice(adformBids []*adformBid, r *adformRequest) pbs.PBSBidSlice { + bids := make(pbs.PBSBidSlice, 0) + + for i, bid := range adformBids { + adm, bidType := getAdAndType(bid) + if adm == "" { + continue + } + pbsBid := pbs.PBSBid{ + BidID: r.adUnits[i].bidId, + AdUnitCode: r.adUnits[i].adUnitCode, + BidderCode: r.bidderCode, + Price: bid.Price, + Adm: adm, + Width: int64(bid.Width), + Height: int64(bid.Height), + DealId: bid.DealId, + Creative_id: bid.CreativeId, + CreativeMediaType: string(bidType), + } + + bids = append(bids, &pbsBid) + } + + return bids +} + +// COMMON + func (r *adformRequest) buildAdformUrl(a *AdformAdapter) string { parameters := url.Values{} @@ -206,6 +359,20 @@ func parseAdformBids(response []byte) ([]*adformBid, error) { // BIDDER Interface +func NewAdformLegacyAdapter(httpConfig *adapters.HTTPAdapterConfig, endpointURL string) *AdformAdapter { + var uriObj *url.URL + uriObj, err := url.Parse(endpointURL) + if err != nil { + panic(fmt.Sprintf("Incorrect Adform request url %s, check the configuration, please.", endpointURL)) + } + + return &AdformAdapter{ + http: adapters.NewHTTPAdapter(httpConfig), + URL: uriObj, + version: version, + } +} + func (a *AdformAdapter) MakeRequests(request *openrtb2.BidRequest, reqInfo *adapters.ExtraRequestInfo) ([]*adapters.RequestData, []error) { adformRequest, errors := openRtbToAdformRequest(request) if len(adformRequest.adUnits) == 0 { diff --git a/adapters/adform/adform_test.go b/adapters/adform/adform_test.go index 53a658f9715..53219f4c4c0 100644 --- a/adapters/adform/adform_test.go +++ b/adapters/adform/adform_test.go @@ -2,18 +2,26 @@ package adform import ( "bytes" + "context" "encoding/json" - "fmt" "net/http" + "net/http/httptest" "strconv" "testing" + "time" - "github.com/prebid/prebid-server/adapters" + "github.com/mxmCherry/openrtb/v15/openrtb2" "github.com/prebid/prebid-server/adapters/adapterstest" + "github.com/prebid/prebid-server/cache/dummycache" + "github.com/prebid/prebid-server/pbs" + "github.com/prebid/prebid-server/usersync" + + "fmt" + + "github.com/prebid/prebid-server/adapters" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/openrtb_ext" - "github.com/mxmCherry/openrtb/v15/openrtb2" "github.com/stretchr/testify/assert" ) @@ -63,6 +71,31 @@ type aBidInfo struct { buyerUID string secure bool currency string + delay time.Duration +} + +var adformTestData aBidInfo + +// Legacy auction tests + +func DummyAdformServer(w http.ResponseWriter, r *http.Request) { + errorString := assertAdformServerRequest(adformTestData, r, false) + if errorString != nil { + http.Error(w, *errorString, http.StatusInternalServerError) + return + } + + if adformTestData.delay > 0 { + <-time.After(adformTestData.delay) + } + + adformServerResponse, err := createAdformServerResponse(adformTestData) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.Write(adformServerResponse) } func createAdformServerResponse(testData aBidInfo) ([]byte, error) { @@ -103,6 +136,168 @@ func createAdformServerResponse(testData aBidInfo) ([]byte, error) { return adformServerResponse, err } +func TestAdformBasicResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(DummyAdformServer)) + defer server.Close() + + adapter, ctx, prebidRequest := initTestData(server, t) + + bids, err := adapter.Call(ctx, prebidRequest, prebidRequest.Bidders[0]) + + if err != nil { + t.Fatalf("Should not have gotten adapter error: %v", err) + } + if len(bids) != 3 { + t.Fatalf("Received %d bids instead of 3", len(bids)) + } + expectedTypes := []openrtb_ext.BidType{ + openrtb_ext.BidTypeBanner, + openrtb_ext.BidTypeBanner, + openrtb_ext.BidTypeVideo, + } + + for i, bid := range bids { + + if bid.CreativeMediaType != string(expectedTypes[i]) { + t.Errorf("Expected a %s bid. Got: %s", expectedTypes[i], bid.CreativeMediaType) + } + + matched := false + for _, tag := range adformTestData.tags { + if bid.AdUnitCode == tag.code { + matched = true + if bid.BidderCode != "adform" { + t.Errorf("Incorrect BidderCode '%s'", bid.BidderCode) + } + if bid.Price != tag.price { + t.Errorf("Incorrect bid price '%.2f' expected '%.2f'", bid.Price, tag.price) + } + if bid.Width != int64(adformTestData.width) || bid.Height != int64(adformTestData.height) { + t.Errorf("Incorrect bid size %dx%d, expected %dx%d", bid.Width, bid.Height, adformTestData.width, adformTestData.height) + } + if bid.Adm != tag.content { + t.Errorf("Incorrect bid markup '%s' expected '%s'", bid.Adm, tag.content) + } + if bid.DealId != tag.dealId { + t.Errorf("Incorrect deal id '%s' expected '%s'", bid.DealId, tag.dealId) + } + if bid.Creative_id != tag.creativeId { + t.Errorf("Incorrect creative id '%s' expected '%s'", bid.Creative_id, tag.creativeId) + } + } + } + if !matched { + t.Errorf("Received bid for unknown ad unit '%s'", bid.AdUnitCode) + } + } + + // same test but with request timing out + adformTestData.delay = 5 * time.Millisecond + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) + defer cancel() + + bids, err = adapter.Call(ctx, prebidRequest, prebidRequest.Bidders[0]) + if err == nil { + t.Fatalf("Should have gotten a timeout error: %v", err) + } +} + +func initTestData(server *httptest.Server, t *testing.T) (*AdformAdapter, context.Context, *pbs.PBSRequest) { + adformTestData = createTestData(false) + + // prepare adapter + conf := *adapters.DefaultHTTPAdapterConfig + adapter := NewAdformLegacyAdapter(&conf, server.URL) + + prebidRequest := preparePrebidRequest(server.URL, t) + ctx := context.TODO() + + return adapter, ctx, prebidRequest +} + +func preparePrebidRequest(serverUrl string, t *testing.T) *pbs.PBSRequest { + body := preparePrebidRequestBody(adformTestData, t) + prebidHttpRequest := httptest.NewRequest("POST", serverUrl, body) + prebidHttpRequest.Header.Add("User-Agent", adformTestData.deviceUA) + prebidHttpRequest.Header.Add("Referer", adformTestData.referrer) + prebidHttpRequest.Header.Add("X-Real-IP", adformTestData.deviceIP) + + pbsCookie := usersync.ParseCookieFromRequest(prebidHttpRequest, &config.HostCookie{}) + pbsCookie.TrySync("adform", adformTestData.buyerUID) + fakeWriter := httptest.NewRecorder() + + pbsCookie.SetCookieOnResponse(fakeWriter, false, &config.HostCookie{Domain: ""}, time.Minute) + prebidHttpRequest.Header.Add("Cookie", fakeWriter.Header().Get("Set-Cookie")) + + cacheClient, _ := dummycache.New() + r, err := pbs.ParsePBSRequest(prebidHttpRequest, &config.AuctionTimeouts{ + Default: 2000, + Max: 2000, + }, cacheClient, &config.HostCookie{}) + if err != nil { + t.Fatalf("ParsePBSRequest failed: %v", err) + } + if len(r.Bidders) != 1 { + t.Fatalf("ParsePBSRequest returned %d bidders instead of 1", len(r.Bidders)) + } + if r.Bidders[0].BidderCode != "adform" { + t.Fatalf("ParsePBSRequest returned invalid bidder") + } + + // can't be set in preparePrebidRequestBody as will be lost during json serialization and deserialization + // for the adapters which don't support OpenRTB requests the old PBSRequest is created from OpenRTB request + // so User and Regs are copied from OpenRTB request, see legacy.go -> toLegacyRequest + regs := getRegs() + r.Regs = ®s + user := openrtb2.User{ + Ext: getUserExt(), + } + r.User = &user + + return r +} + +func preparePrebidRequestBody(requestData aBidInfo, t *testing.T) *bytes.Buffer { + prebidRequest := pbs.PBSRequest{ + AdUnits: make([]pbs.AdUnit, 4), + Device: &openrtb2.Device{ + UA: requestData.deviceUA, + IP: requestData.deviceIP, + IFA: requestData.deviceIFA, + }, + Tid: requestData.tid, + Secure: 0, + } + for i, tag := range requestData.tags { + prebidRequest.AdUnits[i] = pbs.AdUnit{ + Code: tag.code, + Sizes: []openrtb2.Format{ + { + W: int64(requestData.width), + H: int64(requestData.height), + }, + }, + Bids: []pbs.Bids{ + { + BidderCode: "adform", + BidID: fmt.Sprintf("random-id-from-pbjs-%d", i), + Params: json.RawMessage(formatAdUnitJson(tag)), + }, + }, + } + } + + body := new(bytes.Buffer) + err := json.NewEncoder(body).Encode(prebidRequest) + if err != nil { + t.Fatalf("Json encoding failed: %v", err) + } + fmt.Println("body", body) + return body +} + +// OpenRTB auction tests + func TestOpenRTBRequest(t *testing.T) { bidder, buildErr := Builder(openrtb_ext.BidderAdform, config.Adapter{ Endpoint: "https://adx.adform.net"}) @@ -129,7 +324,7 @@ func TestOpenRTBRequest(t *testing.T) { } r.Header = httpRequests[0].Headers - errorString := assertAdformServerRequest(testData, r) + errorString := assertAdformServerRequest(testData, r, true) if errorString != nil { t.Errorf("Request error: %s", *errorString) } @@ -304,6 +499,16 @@ func TestOpenRTBSurpriseResponse(t *testing.T) { } } +// Properties tests + +func TestAdformProperties(t *testing.T) { + adapter := NewAdformLegacyAdapter(adapters.DefaultHTTPAdapterConfig, "adx.adform.net/adx") + + if adapter.SkipNoCookies() != false { + t.Fatalf("should have been false") + } +} + // helpers func getRegs() openrtb2.Regs { @@ -383,7 +588,7 @@ func formatAdUnitParam(fieldName string, fieldValue string) string { return "" } -func assertAdformServerRequest(testData aBidInfo, r *http.Request) *string { +func assertAdformServerRequest(testData aBidInfo, r *http.Request, isOpenRtb bool) *string { if ok, err := equal("GET", r.Method, "HTTP method"); !ok { return err } @@ -393,8 +598,15 @@ func assertAdformServerRequest(testData aBidInfo, r *http.Request) *string { } } - midsWithCurrency := "bWlkPTMyMzQ0JnJjdXI9RVVSJm1rdj1jb2xvcjpyZWQsYWdlOjMwLTQwJm1rdz1yZWQsYmx1ZSZjZGltcz0zMDB4MzAwLDQwMHgyMDA&bWlkPTMyMzQ1JnJjdXI9RVVSJmNkaW1zPTMwMHgyMDAmbWlucD0yMy4xMA&bWlkPTMyMzQ2JnJjdXI9RVVS&bWlkPTMyMzQ3JnJjdXI9RVVS" - queryString := "CC=1&adid=6D92078A-8246-4BA4-AE5B-76104861E7DC&eids=eyJ0ZXN0LmNvbSI6eyJvdGhlcl91c2VyX2lkIjpbMF0sInNvbWVfdXNlcl9pZCI6WzFdfSwidGVzdDIub3JnIjp7Im90aGVyX3VzZXJfaWQiOlsyXX19&fd=1&gdpr=1&gdpr_consent=abc&ip=111.111.111.111&pt=gross&rp=4&stid=transaction-id&url=https%3A%2F%2Fadform.com%3Fa%3Db&" + midsWithCurrency + var midsWithCurrency = "" + var queryString = "" + if isOpenRtb { + midsWithCurrency = "bWlkPTMyMzQ0JnJjdXI9RVVSJm1rdj1jb2xvcjpyZWQsYWdlOjMwLTQwJm1rdz1yZWQsYmx1ZSZjZGltcz0zMDB4MzAwLDQwMHgyMDA&bWlkPTMyMzQ1JnJjdXI9RVVSJmNkaW1zPTMwMHgyMDAmbWlucD0yMy4xMA&bWlkPTMyMzQ2JnJjdXI9RVVS&bWlkPTMyMzQ3JnJjdXI9RVVS" + queryString = "CC=1&adid=6D92078A-8246-4BA4-AE5B-76104861E7DC&eids=eyJ0ZXN0LmNvbSI6eyJvdGhlcl91c2VyX2lkIjpbMF0sInNvbWVfdXNlcl9pZCI6WzFdfSwidGVzdDIub3JnIjp7Im90aGVyX3VzZXJfaWQiOlsyXX19&fd=1&gdpr=1&gdpr_consent=abc&ip=111.111.111.111&pt=gross&rp=4&stid=transaction-id&url=https%3A%2F%2Fadform.com%3Fa%3Db&" + midsWithCurrency + } else { + midsWithCurrency = "bWlkPTMyMzQ0JnJjdXI9VVNEJm1rdj1jb2xvcjpyZWQsYWdlOjMwLTQwJm1rdz1yZWQsYmx1ZSZjZGltcz0zMDB4MzAwLDQwMHgyMDA&bWlkPTMyMzQ1JnJjdXI9VVNEJmNkaW1zPTMwMHgyMDAmbWlucD0yMy4xMA&bWlkPTMyMzQ2JnJjdXI9VVNE&bWlkPTMyMzQ3JnJjdXI9VVNE" // no way to pass currency in legacy adapter + queryString = "CC=1&adid=6D92078A-8246-4BA4-AE5B-76104861E7DC&fd=1&gdpr=1&gdpr_consent=abc&ip=111.111.111.111&pt=gross&rp=4&stid=transaction-id&" + midsWithCurrency + } if ok, err := equal(queryString, r.URL.RawQuery, "Query string"); !ok { return err diff --git a/adapters/adhese/adhese.go b/adapters/adhese/adhese.go index b23b49b3774..6fc12e3df5e 100644 --- a/adapters/adhese/adhese.go +++ b/adapters/adhese/adhese.go @@ -109,7 +109,7 @@ func (a *AdheseAdapter) MakeRequests(request *openrtb2.BidRequest, reqInfo *adap // Compose url endpointParams := macros.EndpointTemplateParams{AccountID: params.Account} - host, err := macros.ResolveMacros(a.endpointTemplate, endpointParams) + host, err := macros.ResolveMacros(*&a.endpointTemplate, endpointParams) if err != nil { errs = append(errs, WrapReqError("Could not compose url from template and request account val: "+err.Error())) return nil, errs diff --git a/adapters/adman/adman.go b/adapters/adman/adman.go index fd31bc9d14f..808951d3aba 100644 --- a/adapters/adman/adman.go +++ b/adapters/adman/adman.go @@ -25,6 +25,10 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters return bidder, nil } +type admanParams struct { + TagID string `json:"TagID"` +} + // MakeRequests create bid request for adman demand func (a *AdmanAdapter) MakeRequests(request *openrtb2.BidRequest, reqInfo *adapters.ExtraRequestInfo) ([]*adapters.RequestData, []error) { var errs []error diff --git a/adapters/appnexus/appnexus.go b/adapters/appnexus/appnexus.go index 3695f541532..b1004601774 100644 --- a/adapters/appnexus/appnexus.go +++ b/adapters/appnexus/appnexus.go @@ -1,6 +1,8 @@ package appnexus import ( + "bytes" + "context" "encoding/json" "errors" "fmt" @@ -13,6 +15,9 @@ import ( "github.com/buger/jsonparser" "github.com/mxmCherry/openrtb/v15/openrtb2" "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/pbs" + + "golang.org/x/net/context/ctxhttp" "github.com/prebid/prebid-server/adapters" "github.com/prebid/prebid-server/errortypes" @@ -22,12 +27,22 @@ import ( const defaultPlatformID int = 5 -type adapter struct { +type AppNexusAdapter struct { + http *adapters.HTTPAdapter URI string iabCategoryMap map[string]string hbSource int } +// used for cookies and such +func (a *AppNexusAdapter) Name() string { + return "adnxs" +} + +func (a *AppNexusAdapter) SkipNoCookies() bool { + return false +} + type KeyVal struct { Key string `json:"key,omitempty"` Values []string `json:"value,omitempty"` @@ -37,6 +52,21 @@ type appnexusAdapterOptions struct { IabCategories map[string]string `json:"iab_categories"` } +type appnexusParams struct { + LegacyPlacementId int `json:"placementId"` + LegacyInvCode string `json:"invCode"` + LegacyTrafficSourceCode string `json:"trafficSourceCode"` + PlacementId int `json:"placement_id"` + InvCode string `json:"inv_code"` + Member string `json:"member"` + Keywords []KeyVal `json:"keywords"` + TrafficSourceCode string `json:"traffic_source_code"` + Reserve float64 `json:"reserve"` + Position string `json:"position"` + UsePmtRule *bool `json:"use_pmt_rule"` + PrivateSizes json.RawMessage `json:"private_sizes"` +} + type appnexusImpExtAppnexus struct { PlacementID int `json:"placement_id,omitempty"` Keywords string `json:"keywords,omitempty"` @@ -85,7 +115,181 @@ type appnexusReqExt struct { var maxImpsPerReq = 10 -func (a *adapter) MakeRequests(request *openrtb2.BidRequest, reqInfo *adapters.ExtraRequestInfo) ([]*adapters.RequestData, []error) { +func (a *AppNexusAdapter) Call(ctx context.Context, req *pbs.PBSRequest, bidder *pbs.PBSBidder) (pbs.PBSBidSlice, error) { + supportedMediaTypes := []pbs.MediaType{pbs.MEDIA_TYPE_BANNER, pbs.MEDIA_TYPE_VIDEO} + anReq, err := adapters.MakeOpenRTBGeneric(req, bidder, a.Name(), supportedMediaTypes) + + if err != nil { + return nil, err + } + uri := a.URI + for i, unit := range bidder.AdUnits { + var params appnexusParams + err := json.Unmarshal(unit.Params, ¶ms) + if err != nil { + return nil, err + } + // Accept legacy Appnexus parameters if we don't have modern ones + // Don't worry if both is set as validation rules should prevent, and this is temporary anyway. + if params.PlacementId == 0 && params.LegacyPlacementId != 0 { + params.PlacementId = params.LegacyPlacementId + } + if params.InvCode == "" && params.LegacyInvCode != "" { + params.InvCode = params.LegacyInvCode + } + if params.TrafficSourceCode == "" && params.LegacyTrafficSourceCode != "" { + params.TrafficSourceCode = params.LegacyTrafficSourceCode + } + + if params.PlacementId == 0 && (params.InvCode == "" || params.Member == "") { + return nil, &errortypes.BadInput{ + Message: "No placement or member+invcode provided", + } + } + + // Fixes some segfaults. Since this is legacy code, I'm not looking into it too deeply + if len(anReq.Imp) <= i { + break + } + if params.InvCode != "" { + anReq.Imp[i].TagID = params.InvCode + if params.Member != "" { + // this assumes that the same member ID is used across all tags, which should be the case + uri = appendMemberId(a.URI, params.Member) + } + + } + if params.Reserve > 0 { + anReq.Imp[i].BidFloor = params.Reserve // TODO: we need to factor in currency here if non-USD + } + if anReq.Imp[i].Banner != nil && params.Position != "" { + if params.Position == "above" { + anReq.Imp[i].Banner.Pos = openrtb2.AdPositionAboveTheFold.Ptr() + } else if params.Position == "below" { + anReq.Imp[i].Banner.Pos = openrtb2.AdPositionBelowTheFold.Ptr() + } + } + + kvs := make([]string, 0, len(params.Keywords)*2) + for _, kv := range params.Keywords { + if len(kv.Values) == 0 { + kvs = append(kvs, kv.Key) + } else { + for _, val := range kv.Values { + kvs = append(kvs, fmt.Sprintf("%s=%s", kv.Key, val)) + } + + } + } + + keywordStr := strings.Join(kvs, ",") + + impExt := appnexusImpExt{Appnexus: appnexusImpExtAppnexus{ + PlacementID: params.PlacementId, + TrafficSourceCode: params.TrafficSourceCode, + Keywords: keywordStr, + UsePmtRule: params.UsePmtRule, + PrivateSizes: params.PrivateSizes, + }} + anReq.Imp[i].Ext, err = json.Marshal(&impExt) + } + + reqJSON, err := json.Marshal(anReq) + if err != nil { + return nil, err + } + + debug := &pbs.BidderDebug{ + RequestURI: uri, + } + + if req.IsDebug { + debug.RequestBody = string(reqJSON) + bidder.Debug = append(bidder.Debug, debug) + } + + httpReq, err := http.NewRequest("POST", uri, bytes.NewBuffer(reqJSON)) + httpReq.Header.Add("Content-Type", "application/json;charset=utf-8") + httpReq.Header.Add("Accept", "application/json") + + anResp, err := ctxhttp.Do(ctx, a.http.Client, httpReq) + if err != nil { + return nil, err + } + + debug.StatusCode = anResp.StatusCode + + if anResp.StatusCode == 204 { + return nil, nil + } + + defer anResp.Body.Close() + body, err := ioutil.ReadAll(anResp.Body) + if err != nil { + return nil, err + } + responseBody := string(body) + + if anResp.StatusCode == http.StatusBadRequest { + return nil, &errortypes.BadInput{ + Message: fmt.Sprintf("HTTP status %d; body: %s", anResp.StatusCode, responseBody), + } + } + + if anResp.StatusCode != http.StatusOK { + return nil, &errortypes.BadServerResponse{ + Message: fmt.Sprintf("HTTP status %d; body: %s", anResp.StatusCode, responseBody), + } + } + + if req.IsDebug { + debug.ResponseBody = responseBody + } + + var bidResp openrtb2.BidResponse + err = json.Unmarshal(body, &bidResp) + if err != nil { + return nil, err + } + + bids := make(pbs.PBSBidSlice, 0) + + for _, sb := range bidResp.SeatBid { + for _, bid := range sb.Bid { + bidID := bidder.LookupBidID(bid.ImpID) + if bidID == "" { + return nil, &errortypes.BadServerResponse{ + Message: fmt.Sprintf("Unknown ad unit code '%s'", bid.ImpID), + } + } + + pbid := pbs.PBSBid{ + BidID: bidID, + AdUnitCode: bid.ImpID, + BidderCode: bidder.BidderCode, + Price: bid.Price, + Adm: bid.AdM, + Creative_id: bid.CrID, + Width: bid.W, + Height: bid.H, + DealId: bid.DealID, + NURL: bid.NURL, + } + + var impExt appnexusBidExt + if err := json.Unmarshal(bid.Ext, &impExt); err == nil { + if mediaType, err := getMediaTypeForBid(&impExt); err == nil { + pbid.CreativeMediaType = string(mediaType) + bids = append(bids, &pbid) + } + } + } + } + + return bids, nil +} + +func (a *AppNexusAdapter) MakeRequests(request *openrtb2.BidRequest, reqInfo *adapters.ExtraRequestInfo) ([]*adapters.RequestData, []error) { memberIds := make(map[string]bool) errs := make([]error, 0, len(request.Imp)) @@ -179,7 +383,7 @@ func (a *adapter) MakeRequests(request *openrtb2.BidRequest, reqInfo *adapters.E requests := make([]*adapters.RequestData, 0, len(podImps)) for _, podImps := range podImps { - reqExt.Appnexus.AdPodId = generatePodID() + reqExt.Appnexus.AdPodId = generatePodId() reqs, errors := splitRequests(podImps, request, reqExt, thisURI, errs) requests = append(requests, reqs...) @@ -191,7 +395,7 @@ func (a *adapter) MakeRequests(request *openrtb2.BidRequest, reqInfo *adapters.E return splitRequests(imps, request, reqExt, thisURI, errs) } -func generatePodID() string { +func generatePodId() string { val := rand.Int63() return fmt.Sprint(val) } @@ -357,7 +561,7 @@ func makeKeywordStr(keywords []*openrtb_ext.ExtImpAppnexusKeyVal) string { return strings.Join(kvs, ",") } -func (a *adapter) MakeBids(internalRequest *openrtb2.BidRequest, externalRequest *adapters.RequestData, response *adapters.ResponseData) (*adapters.BidderResponse, []error) { +func (a *AppNexusAdapter) MakeBids(internalRequest *openrtb2.BidRequest, externalRequest *adapters.RequestData, response *adapters.ResponseData) (*adapters.BidderResponse, []error) { if response.StatusCode == http.StatusNoContent { return nil, nil } @@ -392,7 +596,7 @@ func (a *adapter) MakeBids(internalRequest *openrtb2.BidRequest, externalRequest bid.Cat = []string{iabCategory} } else if len(bid.Cat) > 1 { //create empty categories array to force bid to be rejected - bid.Cat = make([]string, 0) + bid.Cat = make([]string, 0, 0) } impVideo := &openrtb_ext.ExtBidPrebidVideo{ @@ -434,7 +638,7 @@ func getMediaTypeForBid(bid *appnexusBidExt) (openrtb_ext.BidType, error) { } // getIabCategoryForBid maps an appnexus brand id to an IAB category. -func (a *adapter) getIabCategoryForBid(bid *appnexusBidExt) (string, error) { +func (a *AppNexusAdapter) getIabCategoryForBid(bid *appnexusBidExt) (string, error) { brandIDString := strconv.Itoa(bid.Appnexus.BrandCategory) if iabCategory, ok := a.iabCategoryMap[brandIDString]; ok { return iabCategory, nil @@ -453,7 +657,7 @@ func appendMemberId(uri string, memberId string) string { // Builder builds a new instance of the AppNexus adapter for the given bidder with the given config. func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters.Bidder, error) { - bidder := &adapter{ + bidder := &AppNexusAdapter{ URI: config.Endpoint, iabCategoryMap: loadCategoryMapFromFileSystem(), hbSource: resolvePlatformID(config.PlatformID), @@ -461,6 +665,16 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters return bidder, nil } +// NewAppNexusLegacyAdapter builds a legacy version of the AppNexus adapter. +func NewAppNexusLegacyAdapter(httpConfig *adapters.HTTPAdapterConfig, endpoint, platformID string) *AppNexusAdapter { + return &AppNexusAdapter{ + http: adapters.NewHTTPAdapter(httpConfig), + URI: endpoint, + iabCategoryMap: loadCategoryMapFromFileSystem(), + hbSource: resolvePlatformID(platformID), + } +} + func resolvePlatformID(platformID string) int { if len(platformID) > 0 { if val, err := strconv.Atoi(platformID); err == nil { diff --git a/adapters/appnexus/appnexus_test.go b/adapters/appnexus/appnexus_test.go index 6399be1a5bb..cad52348134 100644 --- a/adapters/appnexus/appnexus_test.go +++ b/adapters/appnexus/appnexus_test.go @@ -1,17 +1,29 @@ package appnexus import ( + "bytes" + "context" "encoding/json" + "io/ioutil" + "net/http" + "net/http/httptest" "regexp" "testing" + "time" + + "github.com/mxmCherry/openrtb/v15/openrtb2" + "github.com/stretchr/testify/assert" + + "github.com/prebid/prebid-server/cache/dummycache" + "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/pbs" + "github.com/prebid/prebid-server/usersync" + + "fmt" "github.com/prebid/prebid-server/adapters" "github.com/prebid/prebid-server/adapters/adapterstest" "github.com/prebid/prebid-server/config" - "github.com/prebid/prebid-server/openrtb_ext" - - "github.com/mxmCherry/openrtb/v15/openrtb2" - "github.com/stretchr/testify/assert" ) func TestJsonSamples(t *testing.T) { @@ -46,7 +58,7 @@ func TestMemberQueryParam(t *testing.T) { } func TestVideoSinglePod(t *testing.T) { - var a adapter + var a AppNexusAdapter a.URI = "http://test.com/openrtb2" a.hbSource = 5 @@ -84,7 +96,7 @@ func TestVideoSinglePod(t *testing.T) { } func TestVideoSinglePodManyImps(t *testing.T) { - var a adapter + var a AppNexusAdapter a.URI = "http://test.com/openrtb2" a.hbSource = 5 @@ -142,7 +154,7 @@ func TestVideoSinglePodManyImps(t *testing.T) { } func TestVideoTwoPods(t *testing.T) { - var a adapter + var a AppNexusAdapter a.URI = "http://test.com/openrtb2" a.hbSource = 5 @@ -194,7 +206,7 @@ func TestVideoTwoPods(t *testing.T) { } func TestVideoTwoPodsManyImps(t *testing.T) { - var a adapter + var a AppNexusAdapter a.URI = "http://test.com/openrtb2" a.hbSource = 5 @@ -269,3 +281,396 @@ func TestVideoTwoPodsManyImps(t *testing.T) { assert.Len(t, podIds, 2, "Incorrect number of unique pod ids") } + +// ---------------------------------------------------------------------------- +// Code below this line tests the legacy, non-openrtb code flow. It can be deleted after we +// clean up the existing code and make everything openrtb2. + +type anTagInfo struct { + code string + invCode string + placementID int + trafficSourceCode string + in_keywords string + out_keywords string + reserve float64 + position string + bid float64 + content string + mediaType string +} + +type anBidInfo struct { + memberID string + domain string + page string + accountID int + siteID int + tags []anTagInfo + deviceIP string + deviceUA string + buyerUID string + delay time.Duration +} + +var andata anBidInfo + +func DummyAppNexusServer(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + var breq openrtb2.BidRequest + err = json.Unmarshal(body, &breq) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + memberID := r.FormValue("member_id") + if memberID != andata.memberID { + http.Error(w, fmt.Sprintf("Member ID '%s' doesn't match '%s", memberID, andata.memberID), http.StatusInternalServerError) + return + } + + resp := openrtb2.BidResponse{ + ID: breq.ID, + BidID: "a-random-id", + Cur: "USD", + SeatBid: []openrtb2.SeatBid{ + { + Seat: "Buyer Member ID", + Bid: make([]openrtb2.Bid, 0, 2), + }, + }, + } + + for i, imp := range breq.Imp { + var aix appnexusImpExt + err = json.Unmarshal(imp.Ext, &aix) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Either placementID or member+invCode must be specified + has_placement := false + if aix.Appnexus.PlacementID != 0 { + if aix.Appnexus.PlacementID != andata.tags[i].placementID { + http.Error(w, fmt.Sprintf("Placement ID '%d' doesn't match '%d", aix.Appnexus.PlacementID, + andata.tags[i].placementID), http.StatusInternalServerError) + return + } + has_placement = true + } + if memberID != "" && imp.TagID != "" { + if imp.TagID != andata.tags[i].invCode { + http.Error(w, fmt.Sprintf("Inv Code '%s' doesn't match '%s", imp.TagID, + andata.tags[i].invCode), http.StatusInternalServerError) + return + } + has_placement = true + } + if !has_placement { + http.Error(w, fmt.Sprintf("Either placement or member+inv not present"), http.StatusInternalServerError) + return + } + + if aix.Appnexus.Keywords != andata.tags[i].out_keywords { + http.Error(w, fmt.Sprintf("Keywords '%s' doesn't match '%s", aix.Appnexus.Keywords, + andata.tags[i].out_keywords), http.StatusInternalServerError) + return + } + + if aix.Appnexus.TrafficSourceCode != andata.tags[i].trafficSourceCode { + http.Error(w, fmt.Sprintf("Traffic source code '%s' doesn't match '%s", aix.Appnexus.TrafficSourceCode, + andata.tags[i].trafficSourceCode), http.StatusInternalServerError) + return + } + if imp.BidFloor != andata.tags[i].reserve { + http.Error(w, fmt.Sprintf("Bid floor '%.2f' doesn't match '%.2f", imp.BidFloor, + andata.tags[i].reserve), http.StatusInternalServerError) + return + } + if imp.Banner == nil && imp.Video == nil { + http.Error(w, fmt.Sprintf("No banner or app object sent"), http.StatusInternalServerError) + return + } + if (imp.Banner == nil && andata.tags[i].mediaType == "banner") || (imp.Banner != nil && andata.tags[i].mediaType != "banner") { + http.Error(w, fmt.Sprintf("Invalid impression type - banner"), http.StatusInternalServerError) + return + } + if (imp.Video == nil && andata.tags[i].mediaType == "video") || (imp.Video != nil && andata.tags[i].mediaType != "video") { + http.Error(w, fmt.Sprintf("Invalid impression type - video"), http.StatusInternalServerError) + return + } + + if imp.Banner != nil { + if len(imp.Banner.Format) == 0 { + http.Error(w, fmt.Sprintf("Empty imp.banner.format array"), http.StatusInternalServerError) + return + } + if andata.tags[i].position == "above" && *imp.Banner.Pos != openrtb2.AdPosition(1) { + http.Error(w, fmt.Sprintf("Mismatch in position - expected 1 for atf"), http.StatusInternalServerError) + return + } + if andata.tags[i].position == "below" && *imp.Banner.Pos != openrtb2.AdPosition(3) { + http.Error(w, fmt.Sprintf("Mismatch in position - expected 3 for btf"), http.StatusInternalServerError) + return + } + } + if imp.Video != nil { + // TODO: add more validations + if len(imp.Video.MIMEs) == 0 { + http.Error(w, fmt.Sprintf("Empty imp.video.mimes array"), http.StatusInternalServerError) + return + } + if len(imp.Video.Protocols) == 0 { + http.Error(w, fmt.Sprintf("Empty imp.video.protocols array"), http.StatusInternalServerError) + return + } + for _, protocol := range imp.Video.Protocols { + if protocol < 1 || protocol > 8 { + http.Error(w, fmt.Sprintf("Invalid video protocol %d", protocol), http.StatusInternalServerError) + return + } + } + } + + resBid := openrtb2.Bid{ + ID: "random-id", + ImpID: imp.ID, + Price: andata.tags[i].bid, + AdM: andata.tags[i].content, + Ext: json.RawMessage(fmt.Sprintf(`{"appnexus":{"bid_ad_type":%d}}`, bidTypeToInt(andata.tags[i].mediaType))), + } + + if imp.Video != nil { + resBid.Attr = []openrtb2.CreativeAttribute{openrtb2.CreativeAttribute(6)} + } + resp.SeatBid[0].Bid = append(resp.SeatBid[0].Bid, resBid) + } + + // TODO: are all of these valid for app? + if breq.Site == nil { + http.Error(w, fmt.Sprintf("No site object sent"), http.StatusInternalServerError) + return + } + if breq.Site.Domain != andata.domain { + http.Error(w, fmt.Sprintf("Domain '%s' doesn't match '%s", breq.Site.Domain, andata.domain), http.StatusInternalServerError) + return + } + if breq.Site.Page != andata.page { + http.Error(w, fmt.Sprintf("Page '%s' doesn't match '%s", breq.Site.Page, andata.page), http.StatusInternalServerError) + return + } + if breq.Device.UA != andata.deviceUA { + http.Error(w, fmt.Sprintf("UA '%s' doesn't match '%s", breq.Device.UA, andata.deviceUA), http.StatusInternalServerError) + return + } + if breq.Device.IP != andata.deviceIP { + http.Error(w, fmt.Sprintf("IP '%s' doesn't match '%s", breq.Device.IP, andata.deviceIP), http.StatusInternalServerError) + return + } + if breq.User.BuyerUID != andata.buyerUID { + http.Error(w, fmt.Sprintf("User ID '%s' doesn't match '%s", breq.User.BuyerUID, andata.buyerUID), http.StatusInternalServerError) + return + } + + if andata.delay > 0 { + <-time.After(andata.delay) + } + + js, err := json.Marshal(resp) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.Write(js) +} + +func bidTypeToInt(bidType string) int { + switch bidType { + case "banner": + return 0 + case "video": + return 1 + case "audio": + return 2 + case "native": + return 3 + default: + return -1 + } +} +func TestAppNexusLegacyBasicResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(DummyAppNexusServer)) + defer server.Close() + + andata = anBidInfo{ + domain: "nytimes.com", + page: "https://www.nytimes.com/2017/05/04/movies/guardians-of-the-galaxy-2-review-chris-pratt.html?hpw&rref=movies&action=click&pgtype=Homepage&module=well-region®ion=bottom-well&WT.nav=bottom-well&_r=0", + tags: make([]anTagInfo, 2), + deviceIP: "25.91.96.36", + deviceUA: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.1 Safari/603.1.30", + buyerUID: "23482348223", + memberID: "958", + } + andata.tags[0] = anTagInfo{ + code: "first-tag", + placementID: 8394, + bid: 1.67, + trafficSourceCode: "ppc-exchange", + content: "huh", + in_keywords: "[{ \"key\": \"genre\", \"value\": [\"jazz\", \"pop\"] }, {\"key\": \"myEmptyKey\", \"value\": []}]", + out_keywords: "genre=jazz,genre=pop,myEmptyKey", + reserve: 1.50, + position: "below", + mediaType: "banner", + } + andata.tags[1] = anTagInfo{ + code: "second-tag", + invCode: "leftbottom", + bid: 3.22, + trafficSourceCode: "taboola", + content: "yow!", + in_keywords: "[{ \"key\": \"genre\", \"value\": [\"rock\", \"pop\"] }, {\"key\": \"myKey\", \"value\": [\"myVal\"]}]", + out_keywords: "genre=rock,genre=pop,myKey=myVal", + reserve: 0.75, + position: "above", + mediaType: "video", + } + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewAppNexusLegacyAdapter(&conf, server.URL, "") + + pbin := pbs.PBSRequest{ + AdUnits: make([]pbs.AdUnit, 2), + } + for i, tag := range andata.tags { + var params json.RawMessage + if tag.placementID > 0 { + params = json.RawMessage(fmt.Sprintf("{\"placementId\": %d, \"member\": \"%s\", \"keywords\": %s, "+ + "\"trafficSourceCode\": \"%s\", \"reserve\": %.3f, \"position\": \"%s\"}", + tag.placementID, andata.memberID, tag.in_keywords, tag.trafficSourceCode, tag.reserve, tag.position)) + } else { + params = json.RawMessage(fmt.Sprintf("{\"invCode\": \"%s\", \"member\": \"%s\", \"keywords\": %s, "+ + "\"trafficSourceCode\": \"%s\", \"reserve\": %.3f, \"position\": \"%s\"}", + tag.invCode, andata.memberID, tag.in_keywords, tag.trafficSourceCode, tag.reserve, tag.position)) + } + + pbin.AdUnits[i] = pbs.AdUnit{ + Code: tag.code, + MediaTypes: []string{tag.mediaType}, + Sizes: []openrtb2.Format{ + { + W: 300, + H: 600, + }, + { + W: 300, + H: 250, + }, + }, + Bids: []pbs.Bids{ + { + BidderCode: "appnexus", + BidID: fmt.Sprintf("random-id-from-pbjs-%d", i), + Params: params, + }, + }, + } + if tag.mediaType == "video" { + pbin.AdUnits[i].Video = pbs.PBSVideo{ + Mimes: []string{"video/mp4"}, + Minduration: 15, + Maxduration: 30, + Startdelay: 5, + Skippable: 0, + PlaybackMethod: 1, + Protocols: []int8{2, 3}, + } + } + } + + body := new(bytes.Buffer) + err := json.NewEncoder(body).Encode(pbin) + if err != nil { + t.Fatalf("Json encoding failed: %v", err) + } + + req := httptest.NewRequest("POST", server.URL, body) + req.Header.Add("Referer", andata.page) + req.Header.Add("User-Agent", andata.deviceUA) + req.Header.Add("X-Real-IP", andata.deviceIP) + + pc := usersync.ParseCookieFromRequest(req, &config.HostCookie{}) + pc.TrySync("adnxs", andata.buyerUID) + fakewriter := httptest.NewRecorder() + + pc.SetCookieOnResponse(fakewriter, false, &config.HostCookie{Domain: ""}, 90*24*time.Hour) + req.Header.Add("Cookie", fakewriter.Header().Get("Set-Cookie")) + + cacheClient, _ := dummycache.New() + hcc := config.HostCookie{} + + pbReq, err := pbs.ParsePBSRequest(req, &config.AuctionTimeouts{ + Default: 2000, + Max: 2000, + }, cacheClient, &hcc) + if err != nil { + t.Fatalf("ParsePBSRequest failed: %v", err) + } + if len(pbReq.Bidders) != 1 { + t.Fatalf("ParsePBSRequest returned %d bidders instead of 1", len(pbReq.Bidders)) + } + if pbReq.Bidders[0].BidderCode != "appnexus" { + t.Fatalf("ParsePBSRequest returned invalid bidder") + } + + ctx := context.TODO() + bids, err := an.Call(ctx, pbReq, pbReq.Bidders[0]) + if err != nil { + t.Fatalf("Should not have gotten an error: %v", err) + } + if len(bids) != 2 { + t.Fatalf("Received %d bids instead of 2", len(bids)) + } + for _, bid := range bids { + matched := false + for _, tag := range andata.tags { + if bid.AdUnitCode == tag.code { + matched = true + if bid.CreativeMediaType != tag.mediaType { + t.Errorf("Incorrect Creative MediaType '%s'. Expected '%s'", bid.CreativeMediaType, tag.mediaType) + } + if bid.BidderCode != "appnexus" { + t.Errorf("Incorrect BidderCode '%s'", bid.BidderCode) + } + if bid.Price != tag.bid { + t.Errorf("Incorrect bid price '%.2f' expected '%.2f'", bid.Price, tag.bid) + } + if bid.Adm != tag.content { + t.Errorf("Incorrect bid markup '%s' expected '%s'", bid.Adm, tag.content) + } + } + } + if !matched { + t.Errorf("Received bid for unknown ad unit '%s'", bid.AdUnitCode) + } + } + + // same test but with request timing out + andata.delay = 5 * time.Millisecond + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) + defer cancel() + + bids, err = an.Call(ctx, pbReq, pbReq.Bidders[0]) + if err == nil { + t.Fatalf("Should have gotten a timeout error: %v", err) + } +} diff --git a/adapters/connectad/connectad.go b/adapters/connectad/connectad.go index 5c30e3a6adc..9827ebcea7b 100644 --- a/adapters/connectad/connectad.go +++ b/adapters/connectad/connectad.go @@ -18,6 +18,10 @@ type ConnectAdAdapter struct { endpoint string } +type connectadImpExt struct { + ConnectAd openrtb_ext.ExtImpConnectAd `json:"connectad"` +} + // Builder builds a new instance of the ConnectAd adapter for the given bidder with the given config. func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters.Bidder, error) { bidder := &ConnectAdAdapter{ diff --git a/adapters/consumable/instant.go b/adapters/consumable/instant.go index a6162d44e22..5a32fef8837 100644 --- a/adapters/consumable/instant.go +++ b/adapters/consumable/instant.go @@ -9,7 +9,7 @@ type instant interface { // Send a real instance when you construct it in adapter_map.go type realInstant struct{} -func (realInstant) Now() time.Time { +func (_ realInstant) Now() time.Time { return time.Now() } diff --git a/adapters/consumable/utils.go b/adapters/consumable/utils.go new file mode 100644 index 00000000000..64e4872c619 --- /dev/null +++ b/adapters/consumable/utils.go @@ -0,0 +1,20 @@ +package consumable + +import ( + netUrl "net/url" +) + +/** + * Creates a snippet of HTML that retrieves the specified `url` + * Returns HTML snippet that contains the img src = set to `url` + */ +func createTrackPixelHtml(url *string) string { + if url == nil { + return "" + } + + escapedUrl := netUrl.QueryEscape(*url) + img := "
" + + "
" + return img +} diff --git a/adapters/conversant/cnvr_legacy.go b/adapters/conversant/cnvr_legacy.go new file mode 100644 index 00000000000..eff1afc5d32 --- /dev/null +++ b/adapters/conversant/cnvr_legacy.go @@ -0,0 +1,291 @@ +package conversant + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + "github.com/mxmCherry/openrtb/v15/openrtb2" + "github.com/prebid/prebid-server/adapters" + "github.com/prebid/prebid-server/errortypes" + "github.com/prebid/prebid-server/pbs" + "golang.org/x/net/context/ctxhttp" +) + +type ConversantLegacyAdapter struct { + http *adapters.HTTPAdapter + URI string +} + +// Corresponds to the bidder name in cookies and requests +func (a *ConversantLegacyAdapter) Name() string { + return "conversant" +} + +// Return true so no request will be sent unless user has been sync'ed. +func (a *ConversantLegacyAdapter) SkipNoCookies() bool { + return true +} + +type conversantParams struct { + SiteID string `json:"site_id"` + Secure *int8 `json:"secure"` + TagID string `json:"tag_id"` + Position *int8 `json:"position"` + BidFloor float64 `json:"bidfloor"` + Mobile *int8 `json:"mobile"` + MIMEs []string `json:"mimes"` + API []int8 `json:"api"` + Protocols []int8 `json:"protocols"` + MaxDuration *int64 `json:"maxduration"` +} + +func (a *ConversantLegacyAdapter) Call(ctx context.Context, req *pbs.PBSRequest, bidder *pbs.PBSBidder) (pbs.PBSBidSlice, error) { + mediaTypes := []pbs.MediaType{pbs.MEDIA_TYPE_BANNER, pbs.MEDIA_TYPE_VIDEO} + cnvrReq, err := adapters.MakeOpenRTBGeneric(req, bidder, a.Name(), mediaTypes) + + if err != nil { + return nil, err + } + + // Create a map of impression objects for both request creation + // and response parsing. + + impMap := make(map[string]*openrtb2.Imp, len(cnvrReq.Imp)) + for idx := range cnvrReq.Imp { + impMap[cnvrReq.Imp[idx].ID] = &cnvrReq.Imp[idx] + } + + // Fill in additional info from custom params + + for _, unit := range bidder.AdUnits { + var params conversantParams + + imp := impMap[unit.Code] + if imp == nil { + // Skip ad units that do not have corresponding impressions. + continue + } + + err := json.Unmarshal(unit.Params, ¶ms) + if err != nil { + return nil, &errortypes.BadInput{ + Message: err.Error(), + } + } + + // Fill in additional Site info + if params.SiteID != "" { + if cnvrReq.Site != nil { + cnvrReq.Site.ID = params.SiteID + } + if cnvrReq.App != nil { + cnvrReq.App.ID = params.SiteID + } + } + + if params.Mobile != nil && !(cnvrReq.Site == nil) { + cnvrReq.Site.Mobile = *params.Mobile + } + + // Fill in additional impression info + + imp.DisplayManager = "prebid-s2s" + imp.DisplayManagerVer = "1.0.1" + imp.BidFloor = params.BidFloor + imp.TagID = params.TagID + + var position *openrtb2.AdPosition + if params.Position != nil { + position = openrtb2.AdPosition(*params.Position).Ptr() + } + + if imp.Banner != nil { + imp.Banner.Pos = position + } else if imp.Video != nil { + imp.Video.Pos = position + + if len(params.API) > 0 { + imp.Video.API = make([]openrtb2.APIFramework, 0, len(params.API)) + for _, api := range params.API { + imp.Video.API = append(imp.Video.API, openrtb2.APIFramework(api)) + } + } + + // Include protocols, mimes, and max duration if specified + // These properties can also be specified in ad unit's video object, + // but are overridden if the custom params object also contains them. + + if len(params.Protocols) > 0 { + imp.Video.Protocols = make([]openrtb2.Protocol, 0, len(params.Protocols)) + for _, protocol := range params.Protocols { + imp.Video.Protocols = append(imp.Video.Protocols, openrtb2.Protocol(protocol)) + } + } + + if len(params.MIMEs) > 0 { + imp.Video.MIMEs = make([]string, len(params.MIMEs)) + copy(imp.Video.MIMEs, params.MIMEs) + } + + if params.MaxDuration != nil { + imp.Video.MaxDuration = *params.MaxDuration + } + } + + // Take care not to override the global secure flag + + if (imp.Secure == nil || *imp.Secure == 0) && params.Secure != nil { + imp.Secure = params.Secure + } + } + + // Do a quick check on required parameters + + if cnvrReq.Site != nil && cnvrReq.Site.ID == "" { + return nil, &errortypes.BadInput{ + Message: "Missing site id", + } + } + + if cnvrReq.App != nil && cnvrReq.App.ID == "" { + return nil, &errortypes.BadInput{ + Message: "Missing app id", + } + } + + // Start capturing debug info + + debug := &pbs.BidderDebug{ + RequestURI: a.URI, + } + + if cnvrReq.Device == nil { + cnvrReq.Device = &openrtb2.Device{} + } + + // Convert request to json to be sent over http + + j, _ := json.Marshal(cnvrReq) + + if req.IsDebug { + debug.RequestBody = string(j) + bidder.Debug = append(bidder.Debug, debug) + } + + httpReq, err := http.NewRequest("POST", a.URI, bytes.NewBuffer(j)) + httpReq.Header.Add("Content-Type", "application/json") + httpReq.Header.Add("Accept", "application/json") + + resp, err := ctxhttp.Do(ctx, a.http.Client, httpReq) + if err != nil { + return nil, err + } + + defer resp.Body.Close() + + if req.IsDebug { + debug.StatusCode = resp.StatusCode + } + + if resp.StatusCode == 204 { + return nil, nil + } + + body, err := ioutil.ReadAll(resp.Body) + + if err != nil { + return nil, err + } + + if resp.StatusCode == http.StatusBadRequest { + return nil, &errortypes.BadInput{ + Message: fmt.Sprintf("HTTP status: %d, body: %s", resp.StatusCode, string(body)), + } + } + + if resp.StatusCode != 200 { + return nil, &errortypes.BadServerResponse{ + Message: fmt.Sprintf("HTTP status: %d, body: %s", resp.StatusCode, string(body)), + } + } + + if req.IsDebug { + debug.ResponseBody = string(body) + } + + var bidResp openrtb2.BidResponse + + err = json.Unmarshal(body, &bidResp) + if err != nil { + return nil, &errortypes.BadServerResponse{ + Message: err.Error(), + } + } + + bids := make(pbs.PBSBidSlice, 0) + + for _, seatbid := range bidResp.SeatBid { + for _, bid := range seatbid.Bid { + if bid.Price <= 0 { + continue + } + + imp := impMap[bid.ImpID] + if imp == nil { + // All returned bids should have a matching impression + return nil, &errortypes.BadServerResponse{ + Message: fmt.Sprintf("Unknown impression id '%s'", bid.ImpID), + } + } + + bidID := bidder.LookupBidID(bid.ImpID) + if bidID == "" { + return nil, &errortypes.BadServerResponse{ + Message: fmt.Sprintf("Unknown ad unit code '%s'", bid.ImpID), + } + } + + pbsBid := pbs.PBSBid{ + BidID: bidID, + AdUnitCode: bid.ImpID, + Price: bid.Price, + Creative_id: bid.CrID, + BidderCode: bidder.BidderCode, + } + + if imp.Video != nil { + pbsBid.CreativeMediaType = "video" + pbsBid.NURL = bid.AdM // Assign to NURL so it'll be interpreted as a vastUrl + pbsBid.Width = imp.Video.W + pbsBid.Height = imp.Video.H + } else { + pbsBid.CreativeMediaType = "banner" + pbsBid.NURL = bid.NURL + pbsBid.Adm = bid.AdM + pbsBid.Width = bid.W + pbsBid.Height = bid.H + } + + bids = append(bids, &pbsBid) + } + } + + if len(bids) == 0 { + return nil, nil + } + + return bids, nil +} + +func NewConversantLegacyAdapter(config *adapters.HTTPAdapterConfig, uri string) *ConversantLegacyAdapter { + a := adapters.NewHTTPAdapter(config) + + return &ConversantLegacyAdapter{ + http: a, + URI: uri, + } +} diff --git a/adapters/conversant/cnvr_legacy_test.go b/adapters/conversant/cnvr_legacy_test.go new file mode 100644 index 00000000000..fc34a93fae2 --- /dev/null +++ b/adapters/conversant/cnvr_legacy_test.go @@ -0,0 +1,853 @@ +package conversant + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/mxmCherry/openrtb/v15/openrtb2" + "github.com/prebid/prebid-server/adapters" + "github.com/prebid/prebid-server/cache/dummycache" + "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/pbs" + "github.com/prebid/prebid-server/usersync" +) + +// Constants + +const ExpectedSiteID string = "12345" +const ExpectedDisplayManager string = "prebid-s2s" +const ExpectedBuyerUID string = "AQECT_o7M1FLbQJK8QFmAQEBAQE" +const ExpectedNURL string = "http://test.dotomi.com" +const ExpectedAdM string = "" +const ExpectedCrID string = "98765" + +const DefaultParam = `{"site_id": "12345"}` + +// Test properties of Adapter interface + +func TestConversantProperties(t *testing.T) { + an := NewConversantLegacyAdapter(adapters.DefaultHTTPAdapterConfig, "someUrl") + + assertNotEqual(t, an.Name(), "", "Missing family name") + assertTrue(t, an.SkipNoCookies(), "SkipNoCookies should be true") +} + +// Test empty bid requests + +func TestConversantEmptyBid(t *testing.T) { + an := NewConversantLegacyAdapter(adapters.DefaultHTTPAdapterConfig, "someUrl") + + ctx := context.TODO() + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{} + _, err := an.Call(ctx, &pbReq, &pbBidder) + assertTrue(t, err != nil, "No error received for an invalid request") +} + +// Test required parameters, which is just the site id for now + +func TestConversantRequiredParameters(t *testing.T) { + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, http.StatusText(http.StatusNoContent), http.StatusNoContent) + }), + ) + defer server.Close() + + an := NewConversantLegacyAdapter(adapters.DefaultHTTPAdapterConfig, server.URL) + ctx := context.TODO() + + testParams := func(params ...string) (pbs.PBSBidSlice, error) { + req, err := CreateBannerRequest(params...) + if err != nil { + return nil, err + } + return an.Call(ctx, req, req.Bidders[0]) + } + + var err error + + if _, err = testParams(`{}`); err == nil { + t.Fatal("Failed to catch missing site id") + } +} + +// Test handling of 404 + +func TestConversantBadStatus(t *testing.T) { + // Create a test http server that returns after 2 milliseconds + + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + }), + ) + defer server.Close() + + // Create a adapter to test + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewConversantLegacyAdapter(&conf, server.URL) + + ctx := context.TODO() + pbReq, err := CreateBannerRequest(DefaultParam) + if err != nil { + t.Fatal("Failed to create a banner request", err) + } + + _, err = an.Call(ctx, pbReq, pbReq.Bidders[0]) + assertTrue(t, err != nil, "Failed to catch 404 error") +} + +// Test handling of HTTP timeout + +func TestConversantTimeout(t *testing.T) { + // Create a test http server that returns after 2 milliseconds + + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-time.After(2 * time.Millisecond) + }), + ) + defer server.Close() + + // Create a adapter to test + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewConversantLegacyAdapter(&conf, server.URL) + + // Create a context that expires before http returns + + ctx, cancel := context.WithTimeout(context.Background(), 0) + defer cancel() + + // Create a basic request + pbReq, err := CreateBannerRequest(DefaultParam) + if err != nil { + t.Fatal("Failed to create a banner request", err) + } + + // Attempt to process the request, which should hit a timeout + // immediately + + _, err = an.Call(ctx, pbReq, pbReq.Bidders[0]) + if err == nil || err != context.DeadlineExceeded { + t.Fatal("No timeout recevied for timed out request", err) + } +} + +// Test handling of 204 + +func TestConversantNoBid(t *testing.T) { + // Create a test http server that returns after 2 milliseconds + + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, http.StatusText(http.StatusNoContent), http.StatusNoContent) + }), + ) + defer server.Close() + + // Create a adapter to test + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewConversantLegacyAdapter(&conf, server.URL) + + ctx := context.TODO() + pbReq, err := CreateBannerRequest(DefaultParam) + if err != nil { + t.Fatal("Failed to create a banner request", err) + } + + resp, err := an.Call(ctx, pbReq, pbReq.Bidders[0]) + if resp != nil || err != nil { + t.Fatal("Failed to handle empty bid", err) + } +} + +// Verify an outgoing openrtp request is created correctly + +func TestConversantRequestDefault(t *testing.T) { + server, lastReq := CreateServer() + if server == nil { + t.Fatal("server not created") + } + + defer server.Close() + + // Create a adapter to test + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewConversantLegacyAdapter(&conf, server.URL) + + ctx := context.TODO() + pbReq, err := CreateBannerRequest(DefaultParam) + if err != nil { + t.Fatal("Failed to create a banner request", err) + } + + _, err = an.Call(ctx, pbReq, pbReq.Bidders[0]) + if err != nil { + t.Fatal("Failed to retrieve bids", err) + } + + assertEqual(t, len(lastReq.Imp), 1, "Request number of impressions") + imp := &lastReq.Imp[0] + + assertEqual(t, imp.DisplayManager, ExpectedDisplayManager, "Request display manager value") + assertEqual(t, lastReq.Site.ID, ExpectedSiteID, "Request site id") + assertEqual(t, int(lastReq.Site.Mobile), 0, "Request site mobile flag") + assertEqual(t, lastReq.User.BuyerUID, ExpectedBuyerUID, "Request buyeruid") + assertTrue(t, imp.Video == nil, "Request video should be nil") + assertEqual(t, int(*imp.Secure), 0, "Request secure") + assertEqual(t, imp.BidFloor, 0.0, "Request bid floor") + assertEqual(t, imp.TagID, "", "Request tag id") + assertTrue(t, imp.Banner.Pos == nil, "Request pos") + assertEqual(t, int(*imp.Banner.W), 300, "Request width") + assertEqual(t, int(*imp.Banner.H), 250, "Request height") +} + +// Verify inapp video request +func TestConversantInappVideoRequest(t *testing.T) { + server, lastReq := CreateServer() + if server == nil { + t.Fatal("server not created") + } + + defer server.Close() + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewConversantLegacyAdapter(&conf, server.URL) + + requestParam := `{"secure": 1, "site_id": "12345"}` + appParam := `{ "bundle": "com.naver.linewebtoon" }` + videoParam := `{ "mimes": ["video/x-ms-wmv"], + "protocols": [1, 2], + "maxduration": 90 }` + + ctx := context.TODO() + pbReq := CreateRequest(requestParam) + pbReq, err := ConvertToVideoRequest(pbReq, videoParam) + if err != nil { + t.Fatal("failed to parse request") + } + pbReq, err = ConvertToAppRequest(pbReq, appParam) + if err != nil { + t.Fatal("failed to parse request") + } + pbReq, err = ParseRequest(pbReq) + if err != nil { + t.Fatal("failed to parse request") + } + + _, err = an.Call(ctx, pbReq, pbReq.Bidders[0]) + + imp := &lastReq.Imp[0] + assertEqual(t, int(imp.Video.W), 300, "Request width") + assertEqual(t, int(imp.Video.H), 250, "Request height") + assertEqual(t, lastReq.App.ID, "12345", "App Id") +} + +// Verify inapp video request +func TestConversantInappBannerRequest(t *testing.T) { + server, lastReq := CreateServer() + if server == nil { + t.Fatal("server not created") + } + + defer server.Close() + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewConversantLegacyAdapter(&conf, server.URL) + + param := `{ "secure": 1, + "site_id": "12345", + "tag_id": "top", + "position": 2, + "bidfloor": 1.01 }` + appParam := `{ "bundle": "com.naver.linewebtoon" }` + + ctx := context.TODO() + pbReq, _ := CreateBannerRequest(param) + pbReq, err := ConvertToAppRequest(pbReq, appParam) + if err != nil { + t.Fatal("failed to parse request") + } + + _, err = an.Call(ctx, pbReq, pbReq.Bidders[0]) + + imp := &lastReq.Imp[0] + assertEqual(t, lastReq.App.ID, "12345", "App Id") + assertEqual(t, int(*imp.Banner.W), 300, "Request width") + assertEqual(t, int(*imp.Banner.H), 250, "Request height") +} + +// Verify an outgoing openrtp request with additional conversant parameters is +// processed correctly + +func TestConversantRequest(t *testing.T) { + server, lastReq := CreateServer() + if server == nil { + t.Fatal("server not created") + } + + defer server.Close() + + // Create a adapter to test + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewConversantLegacyAdapter(&conf, server.URL) + + param := `{ "site_id": "12345", + "secure": 1, + "tag_id": "top", + "position": 2, + "bidfloor": 1.01, + "mobile": 1 }` + + ctx := context.TODO() + pbReq, err := CreateBannerRequest(param) + if err != nil { + t.Fatal("Failed to create a banner request", err) + } + + _, err = an.Call(ctx, pbReq, pbReq.Bidders[0]) + if err != nil { + t.Fatal("Failed to retrieve bids", err) + } + + assertEqual(t, len(lastReq.Imp), 1, "Request number of impressions") + imp := &lastReq.Imp[0] + + assertEqual(t, imp.DisplayManager, ExpectedDisplayManager, "Request display manager value") + assertEqual(t, lastReq.Site.ID, ExpectedSiteID, "Request site id") + assertEqual(t, int(lastReq.Site.Mobile), 1, "Request site mobile flag") + assertEqual(t, lastReq.User.BuyerUID, ExpectedBuyerUID, "Request buyeruid") + assertTrue(t, imp.Video == nil, "Request video should be nil") + assertEqual(t, int(*imp.Secure), 1, "Request secure") + assertEqual(t, imp.BidFloor, 1.01, "Request bid floor") + assertEqual(t, imp.TagID, "top", "Request tag id") + assertEqual(t, int(*imp.Banner.Pos), 2, "Request pos") + assertEqual(t, int(*imp.Banner.W), 300, "Request width") + assertEqual(t, int(*imp.Banner.H), 250, "Request height") +} + +// Verify openrtp responses are converted correctly + +func TestConversantResponse(t *testing.T) { + prices := []float64{0.01, 0.0, 2.01} + server, lastReq := CreateServer(prices...) + if server == nil { + t.Fatal("server not created") + } + + defer server.Close() + + // Create a adapter to test + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewConversantLegacyAdapter(&conf, server.URL) + + param := `{ "site_id": "12345", + "secure": 1, + "tag_id": "top", + "position": 2, + "bidfloor": 1.01, + "mobile" : 1}` + + ctx := context.TODO() + pbReq, err := CreateBannerRequest(param, param, param) + if err != nil { + t.Fatal("Failed to create a banner request", err) + } + + resp, err := an.Call(ctx, pbReq, pbReq.Bidders[0]) + if err != nil { + t.Fatal("Failed to retrieve bids", err) + } + + prices, imps := FilterZeroPrices(prices, lastReq.Imp) + + assertEqual(t, len(resp), len(prices), "Bad number of responses") + + for i, bid := range resp { + assertEqual(t, bid.Price, prices[i], "Bad price in response") + assertEqual(t, bid.AdUnitCode, imps[i].ID, "Bad bid id in response") + + if bid.Price > 0 { + assertEqual(t, bid.Adm, ExpectedAdM, "Bad ad markup in response") + assertEqual(t, bid.NURL, ExpectedNURL, "Bad notification url in response") + assertEqual(t, bid.Creative_id, ExpectedCrID, "Bad creative id in response") + assertEqual(t, bid.Width, *imps[i].Banner.W, "Bad width in response") + assertEqual(t, bid.Height, *imps[i].Banner.H, "Bad height in response") + } + } +} + +// Test video request + +func TestConversantBasicVideoRequest(t *testing.T) { + server, lastReq := CreateServer() + if server == nil { + t.Fatal("server not created") + } + + defer server.Close() + + // Create a adapter to test + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewConversantLegacyAdapter(&conf, server.URL) + + param := `{ "site_id": "12345", + "tag_id": "bottom left", + "position": 3, + "bidfloor": 1.01 }` + + ctx := context.TODO() + pbReq, err := CreateVideoRequest(param) + if err != nil { + t.Fatal("Failed to create a video request", err) + } + + _, err = an.Call(ctx, pbReq, pbReq.Bidders[0]) + if err != nil { + t.Fatal("Failed to retrieve bids", err) + } + + assertEqual(t, len(lastReq.Imp), 1, "Request number of impressions") + imp := &lastReq.Imp[0] + + assertEqual(t, imp.DisplayManager, ExpectedDisplayManager, "Request display manager value") + assertEqual(t, lastReq.Site.ID, ExpectedSiteID, "Request site id") + assertEqual(t, int(lastReq.Site.Mobile), 0, "Request site mobile flag") + assertEqual(t, lastReq.User.BuyerUID, ExpectedBuyerUID, "Request buyeruid") + assertTrue(t, imp.Banner == nil, "Request banner should be nil") + assertEqual(t, int(*imp.Secure), 0, "Request secure") + assertEqual(t, imp.BidFloor, 1.01, "Request bid floor") + assertEqual(t, imp.TagID, "bottom left", "Request tag id") + assertEqual(t, int(*imp.Video.Pos), 3, "Request pos") + assertEqual(t, int(imp.Video.W), 300, "Request width") + assertEqual(t, int(imp.Video.H), 250, "Request height") + + assertEqual(t, len(imp.Video.MIMEs), 1, "Request video MIMEs entries") + assertEqual(t, imp.Video.MIMEs[0], "video/mp4", "Requst video MIMEs type") + assertTrue(t, imp.Video.Protocols == nil, "Request video protocols") + assertEqual(t, imp.Video.MaxDuration, int64(0), "Request video 0 max duration") + assertTrue(t, imp.Video.API == nil, "Request video api should be nil") +} + +// Test video request with parameters in custom params object + +func TestConversantVideoRequestWithParams(t *testing.T) { + server, lastReq := CreateServer() + if server == nil { + t.Fatal("server not created") + } + + defer server.Close() + + // Create a adapter to test + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewConversantLegacyAdapter(&conf, server.URL) + + param := `{ "site_id": "12345", + "tag_id": "bottom left", + "position": 3, + "bidfloor": 1.01, + "mimes": ["video/x-ms-wmv"], + "protocols": [1, 2], + "api": [1, 2], + "maxduration": 90 }` + + ctx := context.TODO() + pbReq, err := CreateVideoRequest(param) + if err != nil { + t.Fatal("Failed to create a video request", err) + } + + _, err = an.Call(ctx, pbReq, pbReq.Bidders[0]) + if err != nil { + t.Fatal("Failed to retrieve bids", err) + } + + assertEqual(t, len(lastReq.Imp), 1, "Request number of impressions") + imp := &lastReq.Imp[0] + + assertEqual(t, imp.DisplayManager, ExpectedDisplayManager, "Request display manager value") + assertEqual(t, lastReq.Site.ID, ExpectedSiteID, "Request site id") + assertEqual(t, int(lastReq.Site.Mobile), 0, "Request site mobile flag") + assertEqual(t, lastReq.User.BuyerUID, ExpectedBuyerUID, "Request buyeruid") + assertTrue(t, imp.Banner == nil, "Request banner should be nil") + assertEqual(t, int(*imp.Secure), 0, "Request secure") + assertEqual(t, imp.BidFloor, 1.01, "Request bid floor") + assertEqual(t, imp.TagID, "bottom left", "Request tag id") + assertEqual(t, int(*imp.Video.Pos), 3, "Request pos") + assertEqual(t, int(imp.Video.W), 300, "Request width") + assertEqual(t, int(imp.Video.H), 250, "Request height") + + assertEqual(t, len(imp.Video.MIMEs), 1, "Request video MIMEs entries") + assertEqual(t, imp.Video.MIMEs[0], "video/x-ms-wmv", "Requst video MIMEs type") + assertEqual(t, len(imp.Video.Protocols), 2, "Request video protocols") + assertEqual(t, imp.Video.Protocols[0], openrtb2.Protocol(1), "Request video protocols 1") + assertEqual(t, imp.Video.Protocols[1], openrtb2.Protocol(2), "Request video protocols 2") + assertEqual(t, imp.Video.MaxDuration, int64(90), "Request video 0 max duration") + assertEqual(t, len(imp.Video.API), 2, "Request video api should be nil") + assertEqual(t, imp.Video.API[0], openrtb2.APIFramework(1), "Request video api 1") + assertEqual(t, imp.Video.API[1], openrtb2.APIFramework(2), "Request video api 2") +} + +// Test video request with parameters in the video object + +func TestConversantVideoRequestWithParams2(t *testing.T) { + server, lastReq := CreateServer() + if server == nil { + t.Fatal("server not created") + } + + defer server.Close() + + // Create a adapter to test + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewConversantLegacyAdapter(&conf, server.URL) + + param := `{ "site_id": "12345" }` + videoParam := `{ "mimes": ["video/x-ms-wmv"], + "protocols": [1, 2], + "maxduration": 90 }` + + ctx := context.TODO() + pbReq := CreateRequest(param) + pbReq, err := ConvertToVideoRequest(pbReq, videoParam) + if err != nil { + t.Fatal("Failed to convert to a video request", err) + } + pbReq, err = ParseRequest(pbReq) + if err != nil { + t.Fatal("Failed to parse video request", err) + } + + _, err = an.Call(ctx, pbReq, pbReq.Bidders[0]) + if err != nil { + t.Fatal("Failed to retrieve bids", err) + } + + assertEqual(t, len(lastReq.Imp), 1, "Request number of impressions") + imp := &lastReq.Imp[0] + + assertEqual(t, imp.DisplayManager, ExpectedDisplayManager, "Request display manager value") + assertEqual(t, lastReq.Site.ID, ExpectedSiteID, "Request site id") + assertEqual(t, int(lastReq.Site.Mobile), 0, "Request site mobile flag") + assertEqual(t, lastReq.User.BuyerUID, ExpectedBuyerUID, "Request buyeruid") + assertTrue(t, imp.Banner == nil, "Request banner should be nil") + assertEqual(t, int(*imp.Secure), 0, "Request secure") + assertEqual(t, imp.BidFloor, 0.0, "Request bid floor") + assertEqual(t, int(imp.Video.W), 300, "Request width") + assertEqual(t, int(imp.Video.H), 250, "Request height") + + assertEqual(t, len(imp.Video.MIMEs), 1, "Request video MIMEs entries") + assertEqual(t, imp.Video.MIMEs[0], "video/x-ms-wmv", "Requst video MIMEs type") + assertEqual(t, len(imp.Video.Protocols), 2, "Request video protocols") + assertEqual(t, imp.Video.Protocols[0], openrtb2.Protocol(1), "Request video protocols 1") + assertEqual(t, imp.Video.Protocols[1], openrtb2.Protocol(2), "Request video protocols 2") + assertEqual(t, imp.Video.MaxDuration, int64(90), "Request video 0 max duration") +} + +// Test video responses + +func TestConversantVideoResponse(t *testing.T) { + prices := []float64{0.01, 0.0, 2.01} + server, lastReq := CreateServer(prices...) + if server == nil { + t.Fatal("server not created") + } + + defer server.Close() + + // Create a adapter to test + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewConversantLegacyAdapter(&conf, server.URL) + + param := `{ "site_id": "12345", + "secure": 1, + "tag_id": "top", + "position": 2, + "bidfloor": 1.01, + "mobile" : 1}` + + ctx := context.TODO() + pbReq, err := CreateVideoRequest(param, param, param) + if err != nil { + t.Fatal("Failed to create a video request", err) + } + + resp, err := an.Call(ctx, pbReq, pbReq.Bidders[0]) + if err != nil { + t.Fatal("Failed to retrieve bids", err) + } + + prices, imps := FilterZeroPrices(prices, lastReq.Imp) + + assertEqual(t, len(resp), len(prices), "Bad number of responses") + + for i, bid := range resp { + assertEqual(t, bid.Price, prices[i], "Bad price in response") + assertEqual(t, bid.AdUnitCode, imps[i].ID, "Bad bid id in response") + + if bid.Price > 0 { + assertEqual(t, bid.Adm, "", "Bad ad markup in response") + assertEqual(t, bid.NURL, ExpectedAdM, "Bad notification url in response") + assertEqual(t, bid.Creative_id, ExpectedCrID, "Bad creative id in response") + assertEqual(t, bid.Width, imps[i].Video.W, "Bad width in response") + assertEqual(t, bid.Height, imps[i].Video.H, "Bad height in response") + } + } +} + +// Helpers to create a banner and video requests + +func CreateRequest(params ...string) *pbs.PBSRequest { + num := len(params) + + req := pbs.PBSRequest{ + Tid: "t-000", + AccountID: "1", + AdUnits: make([]pbs.AdUnit, num), + } + + for i := 0; i < num; i++ { + req.AdUnits[i] = pbs.AdUnit{ + Code: fmt.Sprintf("au-%03d", i), + Sizes: []openrtb2.Format{ + { + W: 300, + H: 250, + }, + }, + Bids: []pbs.Bids{ + { + BidderCode: "conversant", + BidID: fmt.Sprintf("b-%03d", i), + Params: json.RawMessage(params[i]), + }, + }, + } + } + + return &req +} + +// Convert a request to a video request by adding required properties + +func ConvertToVideoRequest(req *pbs.PBSRequest, videoParams ...string) (*pbs.PBSRequest, error) { + for i := 0; i < len(req.AdUnits); i++ { + video := pbs.PBSVideo{} + if i < len(videoParams) { + err := json.Unmarshal([]byte(videoParams[i]), &video) + if err != nil { + return nil, err + } + } + + if video.Mimes == nil { + video.Mimes = []string{"video/mp4"} + } + + req.AdUnits[i].Video = video + req.AdUnits[i].MediaTypes = []string{"video"} + } + + return req, nil +} + +// Convert a request to an app request by adding required properties +func ConvertToAppRequest(req *pbs.PBSRequest, appParams string) (*pbs.PBSRequest, error) { + app := new(openrtb2.App) + err := json.Unmarshal([]byte(appParams), &app) + if err == nil { + req.App = app + } + + return req, nil +} + +// Feed the request thru the prebid parser so user id and +// other private properties are defined + +func ParseRequest(req *pbs.PBSRequest) (*pbs.PBSRequest, error) { + body := new(bytes.Buffer) + _ = json.NewEncoder(body).Encode(req) + + // Need to pass the conversant user id thru uid cookie + + httpReq := httptest.NewRequest("POST", "/foo", body) + cookie := usersync.NewCookie() + _ = cookie.TrySync("conversant", ExpectedBuyerUID) + httpReq.Header.Set("Cookie", cookie.ToHTTPCookie(90*24*time.Hour).String()) + httpReq.Header.Add("Referer", "http://example.com") + cache, _ := dummycache.New() + hcc := config.HostCookie{} + + parsedReq, err := pbs.ParsePBSRequest(httpReq, &config.AuctionTimeouts{ + Default: 2000, + Max: 2000, + }, cache, &hcc) + + return parsedReq, err +} + +// A helper to create a banner request + +func CreateBannerRequest(params ...string) (*pbs.PBSRequest, error) { + req := CreateRequest(params...) + req, err := ParseRequest(req) + return req, err +} + +// A helper to create a video request + +func CreateVideoRequest(params ...string) (*pbs.PBSRequest, error) { + req := CreateRequest(params...) + req, err := ConvertToVideoRequest(req) + if err != nil { + return nil, err + } + req, err = ParseRequest(req) + return req, err +} + +// Helper to create a test http server that receives and generate openrtb requests and responses + +func CreateServer(prices ...float64) (*httptest.Server, *openrtb2.BidRequest) { + var lastBidRequest openrtb2.BidRequest + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + var bidReq openrtb2.BidRequest + var price float64 + var bids []openrtb2.Bid + var bid openrtb2.Bid + + err = json.Unmarshal(body, &bidReq) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + + lastBidRequest = bidReq + + for i, imp := range bidReq.Imp { + if i < len(prices) { + price = prices[i] + } else { + price = 0 + } + + if price > 0 { + bid = openrtb2.Bid{ + ID: imp.ID, + ImpID: imp.ID, + Price: price, + NURL: ExpectedNURL, + AdM: ExpectedAdM, + CrID: ExpectedCrID, + } + + if imp.Banner != nil { + bid.W = *imp.Banner.W + bid.H = *imp.Banner.H + } else if imp.Video != nil { + bid.W = imp.Video.W + bid.H = imp.Video.H + } + } else { + bid = openrtb2.Bid{ + ID: imp.ID, + ImpID: imp.ID, + Price: 0, + } + } + + bids = append(bids, bid) + } + + if len(bids) == 0 { + w.WriteHeader(http.StatusNoContent) + } else { + js, _ := json.Marshal(openrtb2.BidResponse{ + ID: bidReq.ID, + SeatBid: []openrtb2.SeatBid{ + { + Bid: bids, + }, + }, + }) + + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(js) + } + }), + ) + + return server, &lastBidRequest +} + +// Helper to remove impressions with $0 bids + +func FilterZeroPrices(prices []float64, imps []openrtb2.Imp) ([]float64, []openrtb2.Imp) { + prices2 := make([]float64, 0) + imps2 := make([]openrtb2.Imp, 0) + + for i := range prices { + if prices[i] > 0 { + prices2 = append(prices2, prices[i]) + imps2 = append(imps2, imps[i]) + } + } + + return prices2, imps2 +} + +// Helpers to test equality + +func assertEqual(t *testing.T, actual interface{}, expected interface{}, msg string) { + if expected != actual { + msg = fmt.Sprintf("%s: act(%v) != exp(%v)", msg, actual, expected) + t.Fatal(msg) + } +} + +func assertNotEqual(t *testing.T, actual interface{}, expected interface{}, msg string) { + if expected == actual { + msg = fmt.Sprintf("%s: act(%v) == exp(%v)", msg, actual, expected) + t.Fatal(msg) + } +} + +func assertTrue(t *testing.T, val bool, msg string) { + if val == false { + msg = fmt.Sprintf("%s: is false but should be true", msg) + t.Fatal(msg) + } +} + +func assertFalse(t *testing.T, val bool, msg string) { + if val == true { + msg = fmt.Sprintf("%s: is true but should be false", msg) + t.Fatal(msg) + } +} diff --git a/adapters/deepintent/deepintent.go b/adapters/deepintent/deepintent.go index 0853bb8b405..b5b0fd54c5d 100644 --- a/adapters/deepintent/deepintent.go +++ b/adapters/deepintent/deepintent.go @@ -20,6 +20,10 @@ type DeepintentAdapter struct { URI string } +type deepintentParams struct { + tagId string `json:"tagId"` +} + // Builder builds a new instance of the Deepintent adapter for the given bidder with the given config. func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters.Bidder, error) { bidder := &DeepintentAdapter{ diff --git a/adapters/dmx/dmx_test.go b/adapters/dmx/dmx_test.go index aa4a6f79053..409290c110d 100644 --- a/adapters/dmx/dmx_test.go +++ b/adapters/dmx/dmx_test.go @@ -13,6 +13,10 @@ import ( "github.com/prebid/prebid-server/adapters/adapterstest" ) +var ( + bidRequest string +) + func TestFetchParams(t *testing.T) { var w, h int = 300, 250 diff --git a/adapters/infoawarebidder_test.go b/adapters/infoawarebidder_test.go index 62bcc08d7cb..375248137ad 100644 --- a/adapters/infoawarebidder_test.go +++ b/adapters/infoawarebidder_test.go @@ -178,6 +178,7 @@ func TestImpFiltering(t *testing.T) { } type mockBidder struct { + gotRequest *openrtb2.BidRequest } func (m *mockBidder) MakeRequests(request *openrtb2.BidRequest, reqInfo *adapters.ExtraRequestInfo) ([]*adapters.RequestData, []error) { diff --git a/adapters/ix/ix.go b/adapters/ix/ix.go index 1cfec69322d..c79eda31040 100644 --- a/adapters/ix/ix.go +++ b/adapters/ix/ix.go @@ -1,27 +1,259 @@ package ix import ( + "bytes" + "context" "encoding/json" "fmt" + "io/ioutil" "net/http" "sort" "strings" + "github.com/mxmCherry/openrtb/v15/native1" + native1response "github.com/mxmCherry/openrtb/v15/native1/response" + "github.com/mxmCherry/openrtb/v15/openrtb2" "github.com/prebid/prebid-server/adapters" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/openrtb_ext" - - "github.com/mxmCherry/openrtb/v15/native1" - native1response "github.com/mxmCherry/openrtb/v15/native1/response" - "github.com/mxmCherry/openrtb/v15/openrtb2" + "github.com/prebid/prebid-server/pbs" + "golang.org/x/net/context/ctxhttp" ) type IxAdapter struct { + http *adapters.HTTPAdapter URI string maxRequests int } +func (a *IxAdapter) Name() string { + return string(openrtb_ext.BidderIx) +} + +func (a *IxAdapter) SkipNoCookies() bool { + return false +} + +type indexParams struct { + SiteID string `json:"siteId"` +} + +type ixBidResult struct { + Request *callOneObject + StatusCode int + ResponseBody string + Bid *pbs.PBSBid + Error error +} + +type callOneObject struct { + requestJSON bytes.Buffer + width int64 + height int64 + bidType string +} + +func (a *IxAdapter) Call(ctx context.Context, req *pbs.PBSRequest, bidder *pbs.PBSBidder) (pbs.PBSBidSlice, error) { + var prioritizedRequests, requests []callOneObject + + mediaTypes := []pbs.MediaType{pbs.MEDIA_TYPE_BANNER, pbs.MEDIA_TYPE_VIDEO} + indexReq, err := adapters.MakeOpenRTBGeneric(req, bidder, a.Name(), mediaTypes) + if err != nil { + return nil, err + } + + indexReqImp := indexReq.Imp + for i, unit := range bidder.AdUnits { + // Supposedly fixes some segfaults + if len(indexReqImp) <= i { + break + } + + var params indexParams + err := json.Unmarshal(unit.Params, ¶ms) + if err != nil { + return nil, &errortypes.BadInput{ + Message: fmt.Sprintf("unmarshal params '%s' failed: %v", unit.Params, err), + } + } + + if params.SiteID == "" { + return nil, &errortypes.BadInput{ + Message: "Missing siteId param", + } + } + + for sizeIndex, format := range unit.Sizes { + // Only grab this ad unit. Not supporting multi-media-type adunit yet. + thisImp := indexReqImp[i] + + thisImp.TagID = unit.Code + if thisImp.Banner != nil { + thisImp.Banner.Format = []openrtb2.Format{format} + thisImp.Banner.W = &format.W + thisImp.Banner.H = &format.H + } + indexReq.Imp = []openrtb2.Imp{thisImp} + // Index spec says "adunit path representing ad server inventory" but we don't have this + // ext is DFP div ID and KV pairs if avail + //indexReq.Imp[i].Ext = json.RawMessage("{}") + + if indexReq.Site != nil { + // Any objects pointed to by indexReq *must not be mutated*, or we will get race conditions. + siteCopy := *indexReq.Site + siteCopy.Publisher = &openrtb2.Publisher{ID: params.SiteID} + indexReq.Site = &siteCopy + } + + bidType := "" + if thisImp.Banner != nil { + bidType = string(openrtb_ext.BidTypeBanner) + } else if thisImp.Video != nil { + bidType = string(openrtb_ext.BidTypeVideo) + } + j, _ := json.Marshal(indexReq) + request := callOneObject{requestJSON: *bytes.NewBuffer(j), width: format.W, height: format.H, bidType: bidType} + + // prioritize slots over sizes + if sizeIndex == 0 { + prioritizedRequests = append(prioritizedRequests, request) + } else { + requests = append(requests, request) + } + } + } + + // cap the number of requests to maxRequests + requests = append(prioritizedRequests, requests...) + if len(requests) > a.maxRequests { + requests = requests[:a.maxRequests] + } + + if len(requests) == 0 { + return nil, &errortypes.BadInput{ + Message: "Invalid ad unit/imp/size", + } + } + + ch := make(chan ixBidResult) + for _, request := range requests { + go func(bidder *pbs.PBSBidder, request callOneObject) { + result, err := a.callOne(ctx, request.requestJSON) + result.Request = &request + result.Error = err + if result.Bid != nil { + result.Bid.BidderCode = bidder.BidderCode + result.Bid.BidID = bidder.LookupBidID(result.Bid.AdUnitCode) + result.Bid.Width = request.width + result.Bid.Height = request.height + result.Bid.CreativeMediaType = request.bidType + + if result.Bid.BidID == "" { + result.Error = &errortypes.BadServerResponse{ + Message: fmt.Sprintf("Unknown ad unit code '%s'", result.Bid.AdUnitCode), + } + result.Bid = nil + } + } + ch <- result + }(bidder, request) + } + + bids := make(pbs.PBSBidSlice, 0) + for i := 0; i < len(requests); i++ { + result := <-ch + if result.Bid != nil && result.Bid.Price != 0 { + bids = append(bids, result.Bid) + } + + if req.IsDebug { + debug := &pbs.BidderDebug{ + RequestURI: a.URI, + RequestBody: result.Request.requestJSON.String(), + StatusCode: result.StatusCode, + ResponseBody: result.ResponseBody, + } + bidder.Debug = append(bidder.Debug, debug) + } + if result.Error != nil { + err = result.Error + } + } + + if len(bids) == 0 { + return nil, err + } + return bids, nil +} + +func (a *IxAdapter) callOne(ctx context.Context, reqJSON bytes.Buffer) (ixBidResult, error) { + var result ixBidResult + + httpReq, _ := http.NewRequest("POST", a.URI, &reqJSON) + httpReq.Header.Add("Content-Type", "application/json;charset=utf-8") + httpReq.Header.Add("Accept", "application/json") + + ixResp, err := ctxhttp.Do(ctx, a.http.Client, httpReq) + if err != nil { + return result, err + } + + result.StatusCode = ixResp.StatusCode + + if ixResp.StatusCode == http.StatusNoContent { + return result, nil + } + + if ixResp.StatusCode == http.StatusBadRequest { + return result, &errortypes.BadInput{ + Message: fmt.Sprintf("HTTP status: %d", ixResp.StatusCode), + } + } + + if ixResp.StatusCode != http.StatusOK { + return result, &errortypes.BadServerResponse{ + Message: fmt.Sprintf("HTTP status: %d", ixResp.StatusCode), + } + } + + defer ixResp.Body.Close() + body, err := ioutil.ReadAll(ixResp.Body) + if err != nil { + return result, err + } + result.ResponseBody = string(body) + + var bidResp openrtb2.BidResponse + err = json.Unmarshal(body, &bidResp) + if err != nil { + return result, &errortypes.BadServerResponse{ + Message: fmt.Sprintf("Error parsing response: %v", err), + } + } + + if len(bidResp.SeatBid) == 0 { + return result, nil + } + if len(bidResp.SeatBid[0].Bid) == 0 { + return result, nil + } + bid := bidResp.SeatBid[0].Bid[0] + + pbid := pbs.PBSBid{ + AdUnitCode: bid.ImpID, + Price: bid.Price, + Adm: bid.AdM, + Creative_id: bid.CrID, + Width: bid.W, + Height: bid.H, + DealId: bid.DealID, + } + + result.Bid = &pbid + return result, nil +} + func (a *IxAdapter) MakeRequests(request *openrtb2.BidRequest, reqInfo *adapters.ExtraRequestInfo) ([]*adapters.RequestData, []error) { nImp := len(request.Imp) if nImp > a.maxRequests { @@ -222,6 +454,14 @@ func (a *IxAdapter) MakeBids(internalRequest *openrtb2.BidRequest, externalReque return bidderResponse, errs } +func NewIxLegacyAdapter(config *adapters.HTTPAdapterConfig, endpoint string) *IxAdapter { + return &IxAdapter{ + http: adapters.NewHTTPAdapter(config), + URI: endpoint, + maxRequests: 20, + } +} + // Builder builds a new instance of the Ix adapter for the given bidder with the given config. func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters.Bidder, error) { bidder := &IxAdapter{ diff --git a/adapters/ix/ix_test.go b/adapters/ix/ix_test.go index fc1d0f9a0a2..d292273a92c 100644 --- a/adapters/ix/ix_test.go +++ b/adapters/ix/ix_test.go @@ -1,15 +1,22 @@ package ix import ( + "context" "encoding/json" + "fmt" + "io/ioutil" + "math/rand" + "net/http" + "net/http/httptest" "testing" + "time" + "github.com/mxmCherry/openrtb/v15/openrtb2" "github.com/prebid/prebid-server/adapters" "github.com/prebid/prebid-server/adapters/adapterstest" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/openrtb_ext" - - "github.com/mxmCherry/openrtb/v15/openrtb2" + "github.com/prebid/prebid-server/pbs" ) const endpoint string = "http://host/endpoint" @@ -24,6 +31,698 @@ func TestJsonSamples(t *testing.T) { } } +// Tests for the legacy, non-openrtb code. +// They can be removed after the legacy interface is deprecated. + +func getAdUnit() pbs.PBSAdUnit { + return pbs.PBSAdUnit{ + Code: "unitCode", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + BidID: "bidid", + Sizes: []openrtb2.Format{ + { + W: 10, + H: 12, + }, + }, + Params: json.RawMessage("{\"siteId\":\"12\"}"), + } +} + +func getVideoAdUnit() pbs.PBSAdUnit { + return pbs.PBSAdUnit{ + Code: "unitCodeVideo", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_VIDEO}, + BidID: "bididvideo", + Sizes: []openrtb2.Format{ + { + W: 100, + H: 75, + }, + }, + Video: pbs.PBSVideo{ + Mimes: []string{"video/mp4"}, + Minduration: 15, + Maxduration: 30, + Startdelay: 5, + Skippable: 0, + PlaybackMethod: 1, + Protocols: []int8{2, 3}, + }, + Params: json.RawMessage("{\"siteId\":\"12\"}"), + } +} + +func getOpenRTBBid(i openrtb2.Imp) openrtb2.Bid { + return openrtb2.Bid{ + ID: fmt.Sprintf("%d", rand.Intn(1000)), + ImpID: i.ID, + Price: 1.0, + AdM: "Content", + } +} + +func newAdapter(endpoint string) *IxAdapter { + return NewIxLegacyAdapter(adapters.DefaultHTTPAdapterConfig, endpoint) +} + +func dummyIXServer(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + var breq openrtb2.BidRequest + err = json.Unmarshal(body, &breq) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + impression := breq.Imp[0] + + resp := openrtb2.BidResponse{ + SeatBid: []openrtb2.SeatBid{ + { + Bid: []openrtb2.Bid{ + getOpenRTBBid(impression), + }, + }, + }, + } + + js, err := json.Marshal(resp) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.Write(js) +} + +func TestIxSkipNoCookies(t *testing.T) { + if newAdapter(endpoint).SkipNoCookies() { + t.Fatalf("SkipNoCookies must return false") + } +} + +func TestIxInvalidCall(t *testing.T) { + ctx := context.TODO() + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{} + _, err := newAdapter(endpoint).Call(ctx, &pbReq, &pbBidder) + if err == nil { + t.Fatalf("No error received for invalid request") + } +} + +func TestIxInvalidCallReqAppNil(t *testing.T) { + ctx := context.TODO() + pbReq := pbs.PBSRequest{ + App: &openrtb2.App{}, + } + pbBidder := pbs.PBSBidder{} + + _, err := newAdapter(endpoint).Call(ctx, &pbReq, &pbBidder) + if err == nil { + t.Fatalf("No error received for invalid request") + } +} + +func TestIxInvalidCallMissingSiteID(t *testing.T) { + ctx := context.TODO() + pbReq := pbs.PBSRequest{} + adUnit := getAdUnit() + adUnit.Params = json.RawMessage("{}") + + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + adUnit, + }, + } + _, err := newAdapter(endpoint).Call(ctx, &pbReq, &pbBidder) + if err == nil { + t.Fatalf("No error received for request with missing siteId") + } +} + +func TestIxTimeout(t *testing.T) { + + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-time.After(2 * time.Millisecond) + }), + ) + defer server.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 0) + defer cancel() + + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + getAdUnit(), + }, + } + _, err := newAdapter(server.URL).Call(ctx, &pbReq, &pbBidder) + if err == nil || err != context.DeadlineExceeded { + t.Fatalf("Invalid timeout error received") + } +} + +func TestIxTimeoutMultipleSlots(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + body, err := ioutil.ReadAll(r.Body) + + var breq openrtb2.BidRequest + err = json.Unmarshal(body, &breq) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + impression := breq.Imp[0] + + resp := openrtb2.BidResponse{ + SeatBid: []openrtb2.SeatBid{ + { + Bid: []openrtb2.Bid{ + getOpenRTBBid(impression), + }, + }, + }, + } + + js, err := json.Marshal(resp) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // cancel the request before 2nd impression is returned + // delay to let 1st impression return successfully + if impression.ID == "unitCode2" { + <-time.After(10 * time.Millisecond) + cancel() + <-r.Context().Done() + } + + w.Header().Set("Content-Type", "application/json") + w.Write(js) + }), + ) + defer server.Close() + + pbReq := pbs.PBSRequest{} + + adUnit1 := getAdUnit() + adUnit2 := getAdUnit() + adUnit2.Code = "unitCode2" + adUnit2.Sizes = []openrtb2.Format{ + { + W: 8, + H: 10, + }, + } + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + adUnit1, + adUnit2, + }, + } + bids, err := newAdapter(server.URL).Call(ctx, &pbReq, &pbBidder) + + if err != nil { + t.Fatalf("Should not have gotten an error: %v", err) + } + + if len(bids) != 1 { + t.Fatalf("Should have received one bid") + } + + bid := findBidByAdUnitCode(bids, adUnit1.Code) + if adUnit1.Sizes[0].H != bid.Height || adUnit1.Sizes[0].W != bid.Width { + t.Fatalf("Received the wrong size") + } +} + +func TestIxInvalidJsonResponse(t *testing.T) { + + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "Blah") + }), + ) + defer server.Close() + + ctx := context.TODO() + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + getAdUnit(), + }, + } + _, err := newAdapter(server.URL).Call(ctx, &pbReq, &pbBidder) + if err == nil { + t.Fatalf("No error received for invalid request") + } +} + +func TestIxInvalidStatusCode(t *testing.T) { + + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Send 404 + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + }), + ) + defer server.Close() + + ctx := context.TODO() + pbReq := pbs.PBSRequest{IsDebug: true} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + getAdUnit(), + }, + } + _, err := newAdapter(server.URL).Call(ctx, &pbReq, &pbBidder) + if err == nil { + t.Fatalf("No error received for invalid request") + } +} + +func TestIxBadRequest(t *testing.T) { + + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Send 400 + http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) + }), + ) + defer server.Close() + + ctx := context.TODO() + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + getAdUnit(), + }, + } + _, err := newAdapter(server.URL).Call(ctx, &pbReq, &pbBidder) + if err == nil { + t.Fatalf("No error received for bad request") + } +} + +func TestIxNoContent(t *testing.T) { + + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Send 204 + http.Error(w, http.StatusText(http.StatusNoContent), http.StatusNoContent) + }), + ) + defer server.Close() + + ctx := context.TODO() + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + getAdUnit(), + }, + } + + bids, err := newAdapter(server.URL).Call(ctx, &pbReq, &pbBidder) + if err != nil || bids != nil { + t.Fatalf("Must return nil for no content") + } +} + +func TestIxInvalidCallMissingSize(t *testing.T) { + + server := httptest.NewServer( + http.HandlerFunc(dummyIXServer), + ) + defer server.Close() + + ctx := context.TODO() + pbReq := pbs.PBSRequest{} + adUnit := getAdUnit() + adUnit.Sizes = []openrtb2.Format{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + adUnit, + }, + } + if _, err := newAdapter(server.URL).Call(ctx, &pbReq, &pbBidder); err == nil { + t.Fatalf("Should not have gotten an error for missing/invalid size: %v", err) + } +} + +func TestIxInvalidCallEmptyBidIDResponse(t *testing.T) { + + server := httptest.NewServer( + http.HandlerFunc(dummyIXServer), + ) + defer server.Close() + + ctx := context.TODO() + pbReq := pbs.PBSRequest{} + adUnit := getAdUnit() + adUnit.BidID = "" + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + adUnit, + }, + } + if _, err := newAdapter(server.URL).Call(ctx, &pbReq, &pbBidder); err == nil { + t.Fatalf("Should have gotten an error for unknown adunit code") + } +} + +func TestIxMismatchUnitCode(t *testing.T) { + + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + body, err := ioutil.ReadAll(r.Body) + + var breq openrtb2.BidRequest + err = json.Unmarshal(body, &breq) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + resp := openrtb2.BidResponse{ + SeatBid: []openrtb2.SeatBid{ + { + Bid: []openrtb2.Bid{ + { + ID: fmt.Sprintf("%d", rand.Intn(1000)), + ImpID: "unitCode_bogus", + Price: 1.0, + AdM: "Content", + W: 10, + H: 12, + }, + }, + }, + }, + } + + js, err := json.Marshal(resp) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.Write(js) + }), + ) + defer server.Close() + + ctx := context.TODO() + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + getAdUnit(), + }, + } + if _, err := newAdapter(server.URL).Call(ctx, &pbReq, &pbBidder); err == nil { + t.Fatalf("Should have gotten an error for unknown adunit code") + } +} + +func TestNoSeatBid(t *testing.T) { + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + body, err := ioutil.ReadAll(r.Body) + + var breq openrtb2.BidRequest + err = json.Unmarshal(body, &breq) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + resp := openrtb2.BidResponse{} + + js, err := json.Marshal(resp) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.Write(js) + }), + ) + defer server.Close() + + ctx := context.TODO() + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + getAdUnit(), + }, + } + if _, err := newAdapter(server.URL).Call(ctx, &pbReq, &pbBidder); err != nil { + t.Fatalf("Should not have gotten an error: %v", err) + } +} + +func TestNoSeatBidBid(t *testing.T) { + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + body, err := ioutil.ReadAll(r.Body) + + var breq openrtb2.BidRequest + err = json.Unmarshal(body, &breq) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + resp := openrtb2.BidResponse{ + SeatBid: []openrtb2.SeatBid{ + {}, + }, + } + + js, err := json.Marshal(resp) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.Write(js) + }), + ) + defer server.Close() + + ctx := context.TODO() + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + getAdUnit(), + }, + } + if _, err := newAdapter(server.URL).Call(ctx, &pbReq, &pbBidder); err != nil { + t.Fatalf("Should not have gotten an error: %v", err) + } +} + +func TestIxInvalidParam(t *testing.T) { + + server := httptest.NewServer( + http.HandlerFunc(dummyIXServer), + ) + defer server.Close() + + ctx := context.TODO() + pbReq := pbs.PBSRequest{} + adUnit := getAdUnit() + adUnit.Params = json.RawMessage("Bogus invalid input") + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + adUnit, + }, + } + if _, err := newAdapter(server.URL).Call(ctx, &pbReq, &pbBidder); err == nil { + t.Fatalf("Should have gotten an error for unrecognized params") + } +} + +func TestIxSingleSlotSingleValidSize(t *testing.T) { + + server := httptest.NewServer( + http.HandlerFunc(dummyIXServer), + ) + defer server.Close() + + ctx := context.TODO() + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + getAdUnit(), + }, + } + bids, err := newAdapter(server.URL).Call(ctx, &pbReq, &pbBidder) + if err != nil { + t.Fatalf("Should not have gotten an error: %v", err) + } + + if len(bids) != 1 { + t.Fatalf("Should have received one bid") + } +} + +func TestIxTwoSlotValidSize(t *testing.T) { + + server := httptest.NewServer( + http.HandlerFunc(dummyIXServer), + ) + defer server.Close() + + ctx := context.TODO() + pbReq := pbs.PBSRequest{} + adUnit1 := getAdUnit() + adUnit2 := getVideoAdUnit() + adUnit2.Params = json.RawMessage("{\"siteId\":\"1111\"}") + + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + adUnit1, + adUnit2, + }, + } + bids, err := newAdapter(server.URL).Call(ctx, &pbReq, &pbBidder) + if err != nil { + t.Fatalf("Should not have gotten an error: %v", err) + } + + if len(bids) != 2 { + t.Fatalf("Should have received two bid") + } + + bid := findBidByAdUnitCode(bids, adUnit1.Code) + if adUnit1.Sizes[0].H != bid.Height || adUnit1.Sizes[0].W != bid.Width { + t.Fatalf("Received the wrong size") + } + + bid = findBidByAdUnitCode(bids, adUnit2.Code) + if adUnit2.Sizes[0].H != bid.Height || adUnit2.Sizes[0].W != bid.Width { + t.Fatalf("Received the wrong size") + } +} + +func TestIxTwoSlotMultiSizeOnlyValidIXSizeResponse(t *testing.T) { + + server := httptest.NewServer( + http.HandlerFunc(dummyIXServer), + ) + defer server.Close() + + ctx := context.TODO() + pbReq := pbs.PBSRequest{} + adUnit := getAdUnit() + adUnit.Sizes = append(adUnit.Sizes, openrtb2.Format{W: 20, H: 22}) + + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + adUnit, + }, + } + bids, err := newAdapter(server.URL).Call(ctx, &pbReq, &pbBidder) + if err != nil { + t.Fatalf("Should not have gotten an error: %v", err) + } + + if len(bids) != 2 { + t.Fatalf("Should have received 2 bids") + } + + for _, size := range adUnit.Sizes { + if !bidResponseForSizeExist(bids, size.H, size.W) { + t.Fatalf("Missing bid for specified size %d and %d", size.W, size.H) + } + } +} + +func bidResponseForSizeExist(bids pbs.PBSBidSlice, h, w int64) bool { + for _, v := range bids { + if v.Height == h && v.Width == w { + return true + } + } + return false +} + +func findBidByAdUnitCode(bids pbs.PBSBidSlice, c string) *pbs.PBSBid { + for _, v := range bids { + if v.AdUnitCode == c { + return v + } + } + return &pbs.PBSBid{} +} + +func TestIxMaxRequests(t *testing.T) { + + server := httptest.NewServer( + http.HandlerFunc(dummyIXServer), + ) + defer server.Close() + + adapter := newAdapter(server.URL) + ctx := context.TODO() + pbReq := pbs.PBSRequest{} + adUnits := []pbs.PBSAdUnit{} + + for i := 0; i < adapter.maxRequests+1; i++ { + adUnits = append(adUnits, getAdUnit()) + } + + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: adUnits, + } + + bids, err := adapter.Call(ctx, &pbReq, &pbBidder) + if err != nil { + t.Fatalf("Should not have gotten an error: %v", err) + } + + if len(bids) != adapter.maxRequests { + t.Fatalf("Should have received %d bid", adapter.maxRequests) + } +} + func TestIxMakeBidsWithCategoryDuration(t *testing.T) { bidder := &IxAdapter{} diff --git a/adapters/legacy.go b/adapters/legacy.go new file mode 100644 index 00000000000..8b2221fe0ca --- /dev/null +++ b/adapters/legacy.go @@ -0,0 +1,97 @@ +package adapters + +import ( + "context" + "crypto/tls" + "net/http" + "time" + + "github.com/prebid/prebid-server/pbs" + "github.com/prebid/prebid-server/server/ssl" +) + +// This file contains some deprecated, legacy types. +// +// These support the `/auction` endpoint, but will be replaced by `/openrtb2/auction`. +// New demand partners should ignore this file, and implement the Bidder interface. + +// Adapter is a deprecated interface which connects prebid-server to a demand partner. +// PBS is currently being rewritten to use Bidder, and this will be removed after. +// Their primary purpose is to produce bids in response to Auction requests. +type Adapter interface { + // Name must be identical to the BidderName. + Name() string + // Determines whether this adapter should get callouts if there is not a synched user ID. + SkipNoCookies() bool + // Call produces bids which should be considered, given the auction params. + // + // In practice, implementations almost always make one call to an external server here. + // However, that is not a requirement for satisfying this interface. + // + // An error here will cause all bids to be ignored. If the error was caused by bad user input, + // this should return a BadInputError. If it was caused by bad server behavior + // (e.g. 500, unexpected response format, etc), this should return a BadServerResponseError. + Call(ctx context.Context, req *pbs.PBSRequest, bidder *pbs.PBSBidder) (pbs.PBSBidSlice, error) +} + +// HTTPAdapterConfig groups options which control how HTTP requests are made by adapters. +type HTTPAdapterConfig struct { + // See IdleConnTimeout on https://golang.org/pkg/net/http/#Transport + IdleConnTimeout time.Duration + // See MaxIdleConns on https://golang.org/pkg/net/http/#Transport + MaxConns int + // See MaxIdleConnsPerHost on https://golang.org/pkg/net/http/#Transport + MaxConnsPerHost int +} + +type HTTPAdapter struct { + Client *http.Client +} + +// DefaultHTTPAdapterConfig is an HTTPAdapterConfig that chooses sensible default values. +var DefaultHTTPAdapterConfig = &HTTPAdapterConfig{ + MaxConns: 50, + MaxConnsPerHost: 10, + IdleConnTimeout: 60 * time.Second, +} + +// NewHTTPAdapter creates an HTTPAdapter which obeys the rules given by the config, and +// has all the available SSL certs available in the project. +func NewHTTPAdapter(c *HTTPAdapterConfig) *HTTPAdapter { + ts := &http.Transport{ + MaxIdleConns: c.MaxConns, + MaxIdleConnsPerHost: c.MaxConnsPerHost, + IdleConnTimeout: c.IdleConnTimeout, + TLSClientConfig: &tls.Config{RootCAs: ssl.GetRootCAPool()}, + } + + return &HTTPAdapter{ + Client: &http.Client{ + Transport: ts, + }, + } +} + +// used for callOne (possibly pull all of the shared code here) +type CallOneResult struct { + StatusCode int + ResponseBody string + Bid *pbs.PBSBid + Error error +} + +type MisconfiguredAdapter struct { + TheName string + Err error +} + +func (b *MisconfiguredAdapter) Name() string { + return b.TheName +} +func (b *MisconfiguredAdapter) SkipNoCookies() bool { + return false +} + +func (b *MisconfiguredAdapter) Call(ctx context.Context, req *pbs.PBSRequest, bidder *pbs.PBSBidder) (pbs.PBSBidSlice, error) { + return nil, b.Err +} diff --git a/adapters/openrtb_util.go b/adapters/openrtb_util.go new file mode 100644 index 00000000000..6aa07c6b764 --- /dev/null +++ b/adapters/openrtb_util.go @@ -0,0 +1,174 @@ +package adapters + +import ( + "encoding/json" + + "github.com/mxmCherry/openrtb/v15/openrtb2" + "github.com/prebid/prebid-server/errortypes" + "github.com/prebid/prebid-server/pbs" +) + +func min(x, y int) int { + if x < y { + return x + } + return y +} + +func mediaTypeInSlice(t pbs.MediaType, list []pbs.MediaType) bool { + for _, b := range list { + if b == t { + return true + } + } + return false +} + +func commonMediaTypes(l1 []pbs.MediaType, l2 []pbs.MediaType) []pbs.MediaType { + res := make([]pbs.MediaType, min(len(l1), len(l2))) + i := 0 + for _, b := range l1 { + if mediaTypeInSlice(b, l2) { + res[i] = b + i = i + 1 + } + } + return res[:i] +} + +func makeBanner(unit pbs.PBSAdUnit) *openrtb2.Banner { + return &openrtb2.Banner{ + W: openrtb2.Int64Ptr(unit.Sizes[0].W), + H: openrtb2.Int64Ptr(unit.Sizes[0].H), + Format: copyFormats(unit.Sizes), // defensive copy because adapters may mutate Imps, and this is shared data + TopFrame: unit.TopFrame, + } +} + +func makeVideo(unit pbs.PBSAdUnit) *openrtb2.Video { + // empty mimes array is a sign of uninitialized Video object + if len(unit.Video.Mimes) < 1 { + return nil + } + mimes := make([]string, len(unit.Video.Mimes)) + copy(mimes, unit.Video.Mimes) + pbm := make([]openrtb2.PlaybackMethod, 1) + //this will become int8 soon, so we only care about the first index in the array + pbm[0] = openrtb2.PlaybackMethod(unit.Video.PlaybackMethod) + + protocols := make([]openrtb2.Protocol, 0, len(unit.Video.Protocols)) + for _, protocol := range unit.Video.Protocols { + protocols = append(protocols, openrtb2.Protocol(protocol)) + } + return &openrtb2.Video{ + MIMEs: mimes, + MinDuration: unit.Video.Minduration, + MaxDuration: unit.Video.Maxduration, + W: unit.Sizes[0].W, + H: unit.Sizes[0].H, + StartDelay: openrtb2.StartDelay(unit.Video.Startdelay).Ptr(), + PlaybackMethod: pbm, + Protocols: protocols, + } +} + +// adapters.MakeOpenRTBGeneric makes an openRTB request from the PBS-specific structs. +// +// Any objects pointed to by the returned BidRequest *must not be mutated*, or we will get race conditions. +// The only exception is the Imp property, whose objects will be created new by this method and can be mutated freely. +func MakeOpenRTBGeneric(req *pbs.PBSRequest, bidder *pbs.PBSBidder, bidderFamily string, allowedMediatypes []pbs.MediaType) (openrtb2.BidRequest, error) { + imps := make([]openrtb2.Imp, 0, len(bidder.AdUnits)*len(allowedMediatypes)) + for _, unit := range bidder.AdUnits { + if len(unit.Sizes) <= 0 { + continue + } + unitMediaTypes := commonMediaTypes(unit.MediaTypes, allowedMediatypes) + if len(unitMediaTypes) == 0 { + continue + } + + newImp := openrtb2.Imp{ + ID: unit.Code, + Secure: &req.Secure, + Instl: unit.Instl, + } + for _, mType := range unitMediaTypes { + switch mType { + case pbs.MEDIA_TYPE_BANNER: + newImp.Banner = makeBanner(unit) + case pbs.MEDIA_TYPE_VIDEO: + newImp.Video = makeVideo(unit) + // It's strange to error here... but preserves legacy behavior in legacy code. See #603. + if newImp.Video == nil { + return openrtb2.BidRequest{}, &errortypes.BadInput{ + Message: "Invalid AdUnit: VIDEO media type with no video data", + } + } + } + } + if newImp.Banner != nil || newImp.Video != nil { + imps = append(imps, newImp) + } + } + + if len(imps) < 1 { + return openrtb2.BidRequest{}, &errortypes.BadInput{ + Message: "openRTB bids need at least one Imp", + } + } + + if req.App != nil { + return openrtb2.BidRequest{ + ID: req.Tid, + Imp: imps, + App: req.App, + Device: req.Device, + User: req.User, + Source: &openrtb2.Source{ + TID: req.Tid, + }, + AT: 1, + TMax: req.TimeoutMillis, + Regs: req.Regs, + }, nil + } + + buyerUID, _, _ := req.Cookie.GetUID(bidderFamily) + id, _, _ := req.Cookie.GetUID("adnxs") + + var userExt json.RawMessage + if req.User != nil { + userExt = req.User.Ext + } + + return openrtb2.BidRequest{ + ID: req.Tid, + Imp: imps, + Site: &openrtb2.Site{ + Domain: req.Domain, + Page: req.Url, + }, + Device: req.Device, + User: &openrtb2.User{ + BuyerUID: buyerUID, + ID: id, + Ext: userExt, + }, + Source: &openrtb2.Source{ + FD: 1, // upstream, aka header + TID: req.Tid, + }, + AT: 1, + TMax: req.TimeoutMillis, + Regs: req.Regs, + }, nil +} + +func copyFormats(sizes []openrtb2.Format) []openrtb2.Format { + sizesCopy := make([]openrtb2.Format, len(sizes)) + for i := 0; i < len(sizes); i++ { + sizesCopy[i] = sizes[i] + sizesCopy[i].Ext = append([]byte(nil), sizes[i].Ext...) + } + return sizesCopy +} diff --git a/adapters/openrtb_util_test.go b/adapters/openrtb_util_test.go new file mode 100644 index 00000000000..035f4d9b679 --- /dev/null +++ b/adapters/openrtb_util_test.go @@ -0,0 +1,543 @@ +package adapters + +import ( + "testing" + + "encoding/json" + + "github.com/mxmCherry/openrtb/v15/openrtb2" + "github.com/prebid/prebid-server/pbs" + "github.com/prebid/prebid-server/usersync" + "github.com/stretchr/testify/assert" +) + +func TestCommonMediaTypes(t *testing.T) { + mt1 := []pbs.MediaType{pbs.MEDIA_TYPE_BANNER} + mt2 := []pbs.MediaType{pbs.MEDIA_TYPE_BANNER, pbs.MEDIA_TYPE_VIDEO} + common := commonMediaTypes(mt1, mt2) + assert.Equal(t, len(common), 1) + assert.Equal(t, common[0], pbs.MEDIA_TYPE_BANNER) + + common2 := commonMediaTypes(mt2, mt1) + assert.Equal(t, len(common2), 1) + assert.Equal(t, common2[0], pbs.MEDIA_TYPE_BANNER) + + mt3 := []pbs.MediaType{pbs.MEDIA_TYPE_BANNER, pbs.MEDIA_TYPE_VIDEO} + mt4 := []pbs.MediaType{pbs.MEDIA_TYPE_BANNER, pbs.MEDIA_TYPE_VIDEO} + common3 := commonMediaTypes(mt3, mt4) + assert.Equal(t, len(common3), 2) + + mt5 := []pbs.MediaType{pbs.MEDIA_TYPE_BANNER} + mt6 := []pbs.MediaType{pbs.MEDIA_TYPE_VIDEO} + common4 := commonMediaTypes(mt5, mt6) + assert.Equal(t, len(common4), 0) +} + +func TestOpenRTB(t *testing.T) { + + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + Sizes: []openrtb2.Format{ + { + W: 10, + H: 12, + }, + }, + Instl: 1, + }, + }, + } + resp, err := MakeOpenRTBGeneric(&pbReq, &pbBidder, "test", []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}) + + assert.Equal(t, err, nil) + assert.Equal(t, resp.Imp[0].ID, "unitCode") + assert.EqualValues(t, *resp.Imp[0].Banner.W, 10) + assert.EqualValues(t, *resp.Imp[0].Banner.H, 12) + assert.EqualValues(t, resp.Imp[0].Instl, 1) + + assert.Nil(t, resp.User.Ext) + assert.Nil(t, resp.Regs) +} + +func TestOpenRTBVideo(t *testing.T) { + + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_VIDEO}, + Sizes: []openrtb2.Format{ + { + W: 10, + H: 12, + }, + }, + Video: pbs.PBSVideo{ + Mimes: []string{"video/mp4"}, + Minduration: 15, + Maxduration: 30, + Startdelay: 5, + Skippable: 0, + PlaybackMethod: 1, + }, + }, + }, + } + resp, err := MakeOpenRTBGeneric(&pbReq, &pbBidder, "test", []pbs.MediaType{pbs.MEDIA_TYPE_VIDEO}) + + assert.Equal(t, err, nil) + assert.Equal(t, resp.Imp[0].ID, "unitCode") + assert.EqualValues(t, resp.Imp[0].Video.MaxDuration, 30) + assert.EqualValues(t, resp.Imp[0].Video.MinDuration, 15) + assert.EqualValues(t, *resp.Imp[0].Video.StartDelay, openrtb2.StartDelay(5)) + assert.EqualValues(t, resp.Imp[0].Video.PlaybackMethod, []openrtb2.PlaybackMethod{openrtb2.PlaybackMethod(1)}) + assert.EqualValues(t, resp.Imp[0].Video.MIMEs, []string{"video/mp4"}) +} + +func TestOpenRTBVideoNoVideoData(t *testing.T) { + + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_VIDEO}, + Sizes: []openrtb2.Format{ + { + W: 10, + H: 12, + }, + }, + }, + }, + } + _, err := MakeOpenRTBGeneric(&pbReq, &pbBidder, "test", []pbs.MediaType{pbs.MEDIA_TYPE_VIDEO}) + + assert.NotEqual(t, err, nil) + +} + +func TestOpenRTBVideoFilteredOut(t *testing.T) { + + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_VIDEO}, + Sizes: []openrtb2.Format{ + { + W: 10, + H: 12, + }, + }, + Video: pbs.PBSVideo{ + Mimes: []string{"video/mp4"}, + Minduration: 15, + Maxduration: 30, + Startdelay: 5, + Skippable: 0, + PlaybackMethod: 1, + }, + }, + { + Code: "unitCode2", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + Sizes: []openrtb2.Format{ + { + W: 10, + H: 12, + }, + }, + }, + }, + } + resp, err := MakeOpenRTBGeneric(&pbReq, &pbBidder, "test", []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}) + assert.Equal(t, err, nil) + for i := 0; i < len(resp.Imp); i++ { + if resp.Imp[i].Video != nil { + t.Errorf("No video impressions should exist.") + } + } +} + +func TestOpenRTBMultiMediaImp(t *testing.T) { + + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_VIDEO, pbs.MEDIA_TYPE_BANNER}, + Sizes: []openrtb2.Format{ + { + W: 10, + H: 12, + }, + }, + Video: pbs.PBSVideo{ + Mimes: []string{"video/mp4"}, + Minduration: 15, + Maxduration: 30, + Startdelay: 5, + Skippable: 0, + PlaybackMethod: 1, + }, + }, + }, + } + resp, err := MakeOpenRTBGeneric(&pbReq, &pbBidder, "test", []pbs.MediaType{pbs.MEDIA_TYPE_VIDEO, pbs.MEDIA_TYPE_BANNER}) + assert.Equal(t, err, nil) + assert.Equal(t, len(resp.Imp), 1) + assert.Equal(t, resp.Imp[0].ID, "unitCode") + assert.EqualValues(t, *resp.Imp[0].Banner.W, 10) + assert.EqualValues(t, resp.Imp[0].Video.W, 10) + assert.EqualValues(t, resp.Imp[0].Video.MaxDuration, 30) + assert.EqualValues(t, resp.Imp[0].Video.MinDuration, 15) +} + +func TestOpenRTBMultiMediaImpFiltered(t *testing.T) { + + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_VIDEO, pbs.MEDIA_TYPE_BANNER}, + Sizes: []openrtb2.Format{ + { + W: 10, + H: 12, + }, + }, + Video: pbs.PBSVideo{ + Mimes: []string{"video/mp4"}, + Minduration: 15, + Maxduration: 30, + Startdelay: 5, + Skippable: 0, + PlaybackMethod: 1, + }, + }, + }, + } + resp, err := MakeOpenRTBGeneric(&pbReq, &pbBidder, "test", []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}) + assert.Equal(t, err, nil) + assert.Equal(t, len(resp.Imp), 1) + assert.Equal(t, resp.Imp[0].ID, "unitCode") + assert.EqualValues(t, *resp.Imp[0].Banner.W, 10) + assert.EqualValues(t, resp.Imp[0].Video, (*openrtb2.Video)(nil)) +} + +func TestOpenRTBNoSize(t *testing.T) { + + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + }, + }, + } + _, err := MakeOpenRTBGeneric(&pbReq, &pbBidder, "test", []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}) + if err == nil { + t.Errorf("Bids without impressions should not be allowed.") + } +} + +func TestOpenRTBMobile(t *testing.T) { + pbReq := pbs.PBSRequest{ + AccountID: "test_account_id", + Tid: "test_tid", + CacheMarkup: 1, + SortBids: 1, + MaxKeyLength: 20, + Secure: 1, + TimeoutMillis: 1000, + App: &openrtb2.App{ + Bundle: "AppNexus.PrebidMobileDemo", + Publisher: &openrtb2.Publisher{ + ID: "1995257847363113", + }, + }, + Device: &openrtb2.Device{ + UA: "test_ua", + IP: "test_ip", + Make: "test_make", + Model: "test_model", + IFA: "test_ifa", + }, + User: &openrtb2.User{ + BuyerUID: "test_buyeruid", + }, + } + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + Sizes: []openrtb2.Format{ + { + W: 300, + H: 250, + }, + }, + }, + }, + } + resp, err := MakeOpenRTBGeneric(&pbReq, &pbBidder, "test", []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}) + assert.Equal(t, err, nil) + assert.Equal(t, resp.Imp[0].ID, "unitCode") + assert.EqualValues(t, *resp.Imp[0].Banner.W, 300) + assert.EqualValues(t, *resp.Imp[0].Banner.H, 250) + + assert.EqualValues(t, resp.App.Bundle, "AppNexus.PrebidMobileDemo") + assert.EqualValues(t, resp.App.Publisher.ID, "1995257847363113") + assert.EqualValues(t, resp.User.BuyerUID, "test_buyeruid") + + assert.EqualValues(t, resp.Device.UA, "test_ua") + assert.EqualValues(t, resp.Device.IP, "test_ip") + assert.EqualValues(t, resp.Device.Make, "test_make") + assert.EqualValues(t, resp.Device.Model, "test_model") + assert.EqualValues(t, resp.Device.IFA, "test_ifa") +} + +func TestOpenRTBEmptyUser(t *testing.T) { + pbReq := pbs.PBSRequest{ + User: &openrtb2.User{}, + } + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode2", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + Sizes: []openrtb2.Format{ + { + W: 10, + H: 12, + }, + }, + }, + }, + } + resp, err := MakeOpenRTBGeneric(&pbReq, &pbBidder, "test", []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}) + assert.Equal(t, err, nil) + assert.EqualValues(t, resp.User, &openrtb2.User{}) +} + +func TestOpenRTBUserWithCookie(t *testing.T) { + pbsCookie := usersync.NewCookie() + pbsCookie.TrySync("test", "abcde") + pbReq := pbs.PBSRequest{ + User: &openrtb2.User{}, + } + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + Sizes: []openrtb2.Format{ + { + W: 300, + H: 250, + }, + }, + }, + }, + } + pbReq.Cookie = pbsCookie + resp, err := MakeOpenRTBGeneric(&pbReq, &pbBidder, "test", []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}) + assert.Equal(t, err, nil) + assert.EqualValues(t, resp.User.BuyerUID, "abcde") +} + +func TestSizesCopy(t *testing.T) { + formats := []openrtb2.Format{ + { + W: 10, + }, + { + Ext: []byte{0x5}, + }, + } + clone := copyFormats(formats) + + if len(clone) != 2 { + t.Error("The copy should have 2 elements") + } + if clone[0].W != 10 { + t.Error("The Format's width should be preserved.") + } + if len(clone[1].Ext) != 1 || clone[1].Ext[0] != 0x5 { + t.Error("The Format's Ext should be preserved.") + } + if &formats[0] == &clone[0] || &formats[1] == &clone[1] { + t.Error("The Format elements should not point to the same instance") + } + if &formats[0] == &clone[0] || &formats[1] == &clone[1] { + t.Error("The Format elements should not point to the same instance") + } + if &formats[1].Ext[0] == &clone[1].Ext[0] { + t.Error("The Format.Ext property should point to two different instances") + } +} + +func TestMakeVideo(t *testing.T) { + adUnit := pbs.PBSAdUnit{ + Code: "unitCode", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_VIDEO}, + Sizes: []openrtb2.Format{ + { + W: 10, + H: 12, + }, + }, + Video: pbs.PBSVideo{ + Mimes: []string{"video/mp4"}, + Minduration: 15, + Maxduration: 30, + Startdelay: 5, + Skippable: 0, + PlaybackMethod: 1, + Protocols: []int8{1, 2, 5, 6}, + }, + } + video := makeVideo(adUnit) + assert.EqualValues(t, video.MinDuration, 15) + assert.EqualValues(t, video.MaxDuration, 30) + assert.EqualValues(t, *video.StartDelay, openrtb2.StartDelay(5)) + assert.EqualValues(t, len(video.PlaybackMethod), 1) + assert.EqualValues(t, len(video.Protocols), 4) +} + +func TestGDPR(t *testing.T) { + + rawUserExt := json.RawMessage(`{"consent": "12345"}`) + userExt, _ := json.Marshal(rawUserExt) + + rawRegsExt := json.RawMessage(`{"gdpr": 1}`) + regsExt, _ := json.Marshal(rawRegsExt) + + pbReq := pbs.PBSRequest{ + User: &openrtb2.User{ + Ext: userExt, + }, + Regs: &openrtb2.Regs{ + Ext: regsExt, + }, + } + + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + Sizes: []openrtb2.Format{ + { + W: 10, + H: 12, + }, + }, + Instl: 1, + }, + }, + } + resp, err := MakeOpenRTBGeneric(&pbReq, &pbBidder, "test", []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}) + + assert.Equal(t, err, nil) + assert.Equal(t, resp.Imp[0].ID, "unitCode") + assert.EqualValues(t, *resp.Imp[0].Banner.W, 10) + assert.EqualValues(t, *resp.Imp[0].Banner.H, 12) + assert.EqualValues(t, resp.Imp[0].Instl, 1) + + assert.EqualValues(t, resp.User.Ext, userExt) + assert.EqualValues(t, resp.Regs.Ext, regsExt) +} + +func TestGDPRMobile(t *testing.T) { + rawUserExt := json.RawMessage(`{"consent": "12345"}`) + userExt, _ := json.Marshal(rawUserExt) + + rawRegsExt := json.RawMessage(`{"gdpr": 1}`) + regsExt, _ := json.Marshal(rawRegsExt) + + pbReq := pbs.PBSRequest{ + AccountID: "test_account_id", + Tid: "test_tid", + CacheMarkup: 1, + SortBids: 1, + MaxKeyLength: 20, + Secure: 1, + TimeoutMillis: 1000, + App: &openrtb2.App{ + Bundle: "AppNexus.PrebidMobileDemo", + Publisher: &openrtb2.Publisher{ + ID: "1995257847363113", + }, + }, + Device: &openrtb2.Device{ + UA: "test_ua", + IP: "test_ip", + Make: "test_make", + Model: "test_model", + IFA: "test_ifa", + }, + User: &openrtb2.User{ + BuyerUID: "test_buyeruid", + Ext: userExt, + }, + Regs: &openrtb2.Regs{ + Ext: regsExt, + }, + } + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + Sizes: []openrtb2.Format{ + { + W: 300, + H: 250, + }, + }, + }, + }, + } + resp, err := MakeOpenRTBGeneric(&pbReq, &pbBidder, "test", []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}) + assert.Equal(t, err, nil) + assert.Equal(t, resp.Imp[0].ID, "unitCode") + assert.EqualValues(t, *resp.Imp[0].Banner.W, 300) + assert.EqualValues(t, *resp.Imp[0].Banner.H, 250) + + assert.EqualValues(t, resp.App.Bundle, "AppNexus.PrebidMobileDemo") + assert.EqualValues(t, resp.App.Publisher.ID, "1995257847363113") + assert.EqualValues(t, resp.User.BuyerUID, "test_buyeruid") + + assert.EqualValues(t, resp.Device.UA, "test_ua") + assert.EqualValues(t, resp.Device.IP, "test_ip") + assert.EqualValues(t, resp.Device.Make, "test_make") + assert.EqualValues(t, resp.Device.Model, "test_model") + assert.EqualValues(t, resp.Device.IFA, "test_ifa") + + assert.EqualValues(t, resp.User.Ext, userExt) + assert.EqualValues(t, resp.Regs.Ext, regsExt) +} diff --git a/adapters/pubmatic/pubmatic.go b/adapters/pubmatic/pubmatic.go index 19024f4a123..c2e9fffa0fe 100644 --- a/adapters/pubmatic/pubmatic.go +++ b/adapters/pubmatic/pubmatic.go @@ -1,8 +1,11 @@ package pubmatic import ( + "bytes" + "context" "encoding/json" "fmt" + "io/ioutil" "net/http" "strconv" "strings" @@ -13,23 +16,46 @@ import ( "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/pbs" + "golang.org/x/net/context/ctxhttp" ) const MAX_IMPRESSIONS_PUBMATIC = 30 type PubmaticAdapter struct { - URI string + http *adapters.HTTPAdapter + URI string } -type pubmaticBidExt struct { - BidType *int `json:"BidType,omitempty"` - VideoCreativeInfo *pubmaticBidExtVideo `json:"video,omitempty"` +// used for cookies and such +func (a *PubmaticAdapter) Name() string { + return "pubmatic" +} + +func (a *PubmaticAdapter) SkipNoCookies() bool { + return false +} + +// Below is bidder specific parameters for pubmatic adaptor, +// PublisherId and adSlot are mandatory parameters, others are optional parameters +// Keywords is bid specific parameter, +// WrapExt needs to be sent once per bid request +type pubmaticParams struct { + PublisherId string `json:"publisherId"` + AdSlot string `json:"adSlot"` + WrapExt json.RawMessage `json:"wrapper,omitempty"` + Keywords map[string]string `json:"keywords,omitempty"` } type pubmaticBidExtVideo struct { Duration *int `json:"duration,omitempty"` } +type pubmaticBidExt struct { + BidType *int `json:"BidType,omitempty"` + VideoCreativeInfo *pubmaticBidExtVideo `json:"video,omitempty"` +} + type ExtImpBidderPubmatic struct { adapters.ExtImpBidder Data *ExtData `json:"data,omitempty"` @@ -46,6 +72,16 @@ type ExtAdServer struct { } const ( + INVALID_PARAMS = "Invalid BidParam" + MISSING_PUBID = "Missing PubID" + MISSING_ADSLOT = "Missing AdSlot" + INVALID_WRAPEXT = "Invalid WrapperExt" + INVALID_ADSIZE = "Invalid AdSize" + INVALID_WIDTH = "Invalid Width" + INVALID_HEIGHT = "Invalid Height" + INVALID_MEDIATYPE = "Invalid MediaType" + INVALID_ADSLOT = "Invalid AdSlot" + dctrKeyName = "key_val" pmZoneIDKeyName = "pmZoneId" pmZoneIDKeyNameOld = "pmZoneID" @@ -53,6 +89,251 @@ const ( AdServerGAM = "gam" ) +func PrepareLogMessage(tID, pubId, adUnitId, bidID, details string, args ...interface{}) string { + return fmt.Sprintf("[PUBMATIC] ReqID [%s] PubID [%s] AdUnit [%s] BidID [%s] %s \n", + tID, pubId, adUnitId, bidID, details) +} + +func (a *PubmaticAdapter) Call(ctx context.Context, req *pbs.PBSRequest, bidder *pbs.PBSBidder) (pbs.PBSBidSlice, error) { + mediaTypes := []pbs.MediaType{pbs.MEDIA_TYPE_BANNER, pbs.MEDIA_TYPE_VIDEO} + pbReq, err := adapters.MakeOpenRTBGeneric(req, bidder, a.Name(), mediaTypes) + + if err != nil { + logf("[PUBMATIC] Failed to make ortb request for request id [%s] \n", pbReq.ID) + return nil, err + } + + var errState []string + adSlotFlag := false + pubId := "" + wrapExt := "" + if len(bidder.AdUnits) > MAX_IMPRESSIONS_PUBMATIC { + logf("[PUBMATIC] First %d impressions will be considered from request tid %s\n", + MAX_IMPRESSIONS_PUBMATIC, pbReq.ID) + } + + for i, unit := range bidder.AdUnits { + var params pubmaticParams + err := json.Unmarshal(unit.Params, ¶ms) + if err != nil { + errState = append(errState, fmt.Sprintf("BidID:%s;Error:%s;param:%s", unit.BidID, INVALID_PARAMS, unit.Params)) + logf(PrepareLogMessage(pbReq.ID, params.PublisherId, unit.Code, unit.BidID, + fmt.Sprintf("Ignored bid: invalid JSON [%s] err [%s]", unit.Params, err.Error()))) + continue + } + + if params.PublisherId == "" { + errState = append(errState, fmt.Sprintf("BidID:%s;Error:%s;param:%s", unit.BidID, MISSING_PUBID, unit.Params)) + logf(PrepareLogMessage(pbReq.ID, params.PublisherId, unit.Code, unit.BidID, + fmt.Sprintf("Ignored bid: Publisher Id missing"))) + continue + } + pubId = params.PublisherId + + if params.AdSlot == "" { + errState = append(errState, fmt.Sprintf("BidID:%s;Error:%s;param:%s", unit.BidID, MISSING_ADSLOT, unit.Params)) + logf(PrepareLogMessage(pbReq.ID, params.PublisherId, unit.Code, unit.BidID, + fmt.Sprintf("Ignored bid: adSlot missing"))) + continue + } + + // Parse Wrapper Extension i.e. ProfileID and VersionID only once per request + if wrapExt == "" && len(params.WrapExt) != 0 { + var wrapExtMap map[string]int + err := json.Unmarshal([]byte(params.WrapExt), &wrapExtMap) + if err != nil { + errState = append(errState, fmt.Sprintf("BidID:%s;Error:%s;param:%s", unit.BidID, INVALID_WRAPEXT, unit.Params)) + logf(PrepareLogMessage(pbReq.ID, params.PublisherId, unit.Code, unit.BidID, + fmt.Sprintf("Ignored bid: Wrapper Extension Invalid"))) + continue + } + wrapExt = string(params.WrapExt) + } + + adSlotStr := strings.TrimSpace(params.AdSlot) + adSlot := strings.Split(adSlotStr, "@") + if len(adSlot) == 2 && adSlot[0] != "" && adSlot[1] != "" { + // Fixes some segfaults. Since this is legacy code, I'm not looking into it too deeply + if len(pbReq.Imp) <= i { + break + } + if pbReq.Imp[i].Banner != nil { + adSize := strings.Split(strings.ToLower(strings.TrimSpace(adSlot[1])), "x") + if len(adSize) == 2 { + width, err := strconv.Atoi(strings.TrimSpace(adSize[0])) + if err != nil { + errState = append(errState, fmt.Sprintf("BidID:%s;Error:%s;param:%s", unit.BidID, INVALID_WIDTH, unit.Params)) + logf(PrepareLogMessage(pbReq.ID, params.PublisherId, unit.Code, unit.BidID, + fmt.Sprintf("Ignored bid: invalid adSlot width [%s]", adSize[0]))) + continue + } + + heightStr := strings.Split(strings.TrimSpace(adSize[1]), ":") + height, err := strconv.Atoi(strings.TrimSpace(heightStr[0])) + if err != nil { + errState = append(errState, fmt.Sprintf("BidID:%s;Error:%s;param:%s", unit.BidID, INVALID_HEIGHT, unit.Params)) + logf(PrepareLogMessage(pbReq.ID, params.PublisherId, unit.Code, unit.BidID, + fmt.Sprintf("Ignored bid: invalid adSlot height [%s]", heightStr[0]))) + continue + } + + pbReq.Imp[i].TagID = strings.TrimSpace(adSlot[0]) + pbReq.Imp[i].Banner.W = openrtb2.Int64Ptr(int64(width)) + pbReq.Imp[i].Banner.H = openrtb2.Int64Ptr(int64(height)) + + if len(params.Keywords) != 0 { + kvstr := prepareImpressionExt(params.Keywords) + pbReq.Imp[i].Ext = json.RawMessage([]byte(kvstr)) + } else { + pbReq.Imp[i].Ext = nil + } + + adSlotFlag = true + } else { + errState = append(errState, fmt.Sprintf("BidID:%s;Error:%s;param:%s", unit.BidID, INVALID_ADSIZE, unit.Params)) + logf(PrepareLogMessage(pbReq.ID, params.PublisherId, unit.Code, unit.BidID, + fmt.Sprintf("Ignored bid: invalid adSize [%s]", adSize))) + continue + } + } else { + errState = append(errState, fmt.Sprintf("BidID:%s;Error:%s;param:%s", unit.BidID, INVALID_MEDIATYPE, unit.Params)) + logf(PrepareLogMessage(pbReq.ID, params.PublisherId, unit.Code, unit.BidID, + fmt.Sprintf("Ignored bid: invalid Media Type"))) + continue + } + } else { + errState = append(errState, fmt.Sprintf("BidID:%s;Error:%s;param:%s", unit.BidID, INVALID_ADSLOT, unit.Params)) + logf(PrepareLogMessage(pbReq.ID, params.PublisherId, unit.Code, unit.BidID, + fmt.Sprintf("Ignored bid: invalid adSlot [%s]", params.AdSlot))) + continue + } + + if pbReq.Site != nil { + siteCopy := *pbReq.Site + siteCopy.Publisher = &openrtb2.Publisher{ID: params.PublisherId, Domain: req.Domain} + pbReq.Site = &siteCopy + } + if pbReq.App != nil { + appCopy := *pbReq.App + appCopy.Publisher = &openrtb2.Publisher{ID: params.PublisherId, Domain: req.Domain} + pbReq.App = &appCopy + } + } + + if !(adSlotFlag) { + return nil, &errortypes.BadInput{ + Message: "Incorrect adSlot / Publisher params, Error list: [" + strings.Join(errState, ",") + "]", + } + } + + if wrapExt != "" { + rawExt := fmt.Sprintf("{\"wrapper\": %s}", wrapExt) + pbReq.Ext = json.RawMessage(rawExt) + } + + reqJSON, err := json.Marshal(pbReq) + + debug := &pbs.BidderDebug{ + RequestURI: a.URI, + } + + if req.IsDebug { + debug.RequestBody = string(reqJSON) + bidder.Debug = append(bidder.Debug, debug) + } + + userId, _, _ := req.Cookie.GetUID(a.Name()) + httpReq, err := http.NewRequest("POST", a.URI, bytes.NewBuffer(reqJSON)) + httpReq.Header.Add("Content-Type", "application/json;charset=utf-8") + httpReq.Header.Add("Accept", "application/json") + httpReq.AddCookie(&http.Cookie{ + Name: "KADUSERCOOKIE", + Value: userId, + }) + + pbResp, err := ctxhttp.Do(ctx, a.http.Client, httpReq) + if err != nil { + return nil, err + } + + debug.StatusCode = pbResp.StatusCode + + if pbResp.StatusCode == http.StatusNoContent { + return nil, nil + } + + if pbResp.StatusCode == http.StatusBadRequest { + return nil, &errortypes.BadInput{ + Message: fmt.Sprintf("HTTP status: %d", pbResp.StatusCode), + } + } + + if pbResp.StatusCode != http.StatusOK { + return nil, &errortypes.BadServerResponse{ + Message: fmt.Sprintf("HTTP status: %d", pbResp.StatusCode), + } + } + + defer pbResp.Body.Close() + body, err := ioutil.ReadAll(pbResp.Body) + if err != nil { + return nil, err + } + + if req.IsDebug { + debug.ResponseBody = string(body) + } + + var bidResp openrtb2.BidResponse + err = json.Unmarshal(body, &bidResp) + if err != nil { + return nil, &errortypes.BadServerResponse{ + Message: fmt.Sprintf("HTTP status: %d", pbResp.StatusCode), + } + } + + bids := make(pbs.PBSBidSlice, 0) + + numBids := 0 + for _, sb := range bidResp.SeatBid { + for _, bid := range sb.Bid { + numBids++ + + bidID := bidder.LookupBidID(bid.ImpID) + if bidID == "" { + return nil, &errortypes.BadServerResponse{ + Message: fmt.Sprintf("Unknown ad unit code '%s'", bid.ImpID), + } + } + + pbid := pbs.PBSBid{ + BidID: bidID, + AdUnitCode: bid.ImpID, + BidderCode: bidder.BidderCode, + Price: bid.Price, + Adm: bid.AdM, + Creative_id: bid.CrID, + Width: bid.W, + Height: bid.H, + DealId: bid.DealID, + } + + var bidExt pubmaticBidExt + mediaType := openrtb_ext.BidTypeBanner + if err := json.Unmarshal(bid.Ext, &bidExt); err == nil { + mediaType = getBidType(&bidExt) + } + pbid.CreativeMediaType = string(mediaType) + + bids = append(bids, &pbid) + logf("[PUBMATIC] Returned Bid for PubID [%s] AdUnit [%s] BidID [%s] Size [%dx%d] Price [%f] \n", + pubId, pbid.AdUnitCode, pbid.BidID, pbid.Width, pbid.Height, pbid.Price) + } + } + + return bids, nil +} + func (a *PubmaticAdapter) MakeRequests(request *openrtb2.BidRequest, reqInfo *adapters.ExtraRequestInfo) ([]*adapters.RequestData, []error) { errs := make([]error, 0, len(request.Imp)) @@ -254,6 +535,7 @@ func parseImpressionObject(imp *openrtb2.Imp, wrapExt *string, pubID *string) er } return nil + } func addKeywordsToExt(keywords []*openrtb_ext.ExtImpPubmaticKeyVal, extMap map[string]interface{}) { @@ -271,6 +553,22 @@ func addKeywordsToExt(keywords []*openrtb_ext.ExtImpPubmaticKeyVal, extMap map[s } } +func prepareImpressionExt(keywords map[string]string) string { + + eachKv := make([]string, 0, len(keywords)) + for key, val := range keywords { + if len(val) == 0 { + logf("No values present for key = %s", key) + continue + } else { + eachKv = append(eachKv, fmt.Sprintf("\"%s\":\"%s\"", key, val)) + } + } + + kvStr := "{" + strings.Join(eachKv, ",") + "}" + return kvStr +} + func (a *PubmaticAdapter) MakeBids(internalRequest *openrtb2.BidRequest, externalRequest *adapters.RequestData, response *adapters.ResponseData) (*adapters.BidderResponse, []error) { if response.StatusCode == http.StatusNoContent { return nil, nil @@ -348,6 +646,15 @@ func logf(msg string, args ...interface{}) { } } +func NewPubmaticLegacyAdapter(config *adapters.HTTPAdapterConfig, uri string) *PubmaticAdapter { + a := adapters.NewHTTPAdapter(config) + + return &PubmaticAdapter{ + http: a, + URI: uri, + } +} + // Builder builds a new instance of the Pubmatic adapter for the given bidder with the given config. func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters.Bidder, error) { bidder := &PubmaticAdapter{ diff --git a/adapters/pubmatic/pubmatic_test.go b/adapters/pubmatic/pubmatic_test.go index ac7dbdb711f..2e8a6804850 100644 --- a/adapters/pubmatic/pubmatic_test.go +++ b/adapters/pubmatic/pubmatic_test.go @@ -1,11 +1,25 @@ package pubmatic import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "math/rand" + "net/http" + "net/http/httptest" "testing" + "time" + "github.com/mxmCherry/openrtb/v15/openrtb2" + "github.com/prebid/prebid-server/adapters" "github.com/prebid/prebid-server/adapters/adapterstest" + "github.com/prebid/prebid-server/cache/dummycache" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/pbs" + "github.com/prebid/prebid-server/usersync" ) func TestJsonSamples(t *testing.T) { @@ -19,8 +33,655 @@ func TestJsonSamples(t *testing.T) { adapterstest.RunJSONBidderTest(t, "pubmatictest", bidder) } +// ---------------------------------------------------------------------------- +// Code below this line tests the legacy, non-openrtb code flow. It can be deleted after we +// clean up the existing code and make everything openrtb2. + +func CompareStringValue(val1 string, val2 string, t *testing.T) { + if val1 != val2 { + t.Fatalf(fmt.Sprintf("Expected = %s , Actual = %s", val2, val1)) + } +} + +func DummyPubMaticServer(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + var breq openrtb2.BidRequest + err = json.Unmarshal(body, &breq) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + resp := openrtb2.BidResponse{ + ID: breq.ID, + BidID: "bidResponse_ID", + Cur: "USD", + SeatBid: []openrtb2.SeatBid{ + { + Seat: "pubmatic", + Bid: make([]openrtb2.Bid, 0), + }, + }, + } + rand.Seed(int64(time.Now().UnixNano())) + var bids []openrtb2.Bid + + for i, imp := range breq.Imp { + bids = append(bids, openrtb2.Bid{ + ID: fmt.Sprintf("SeatID_%d", i), + ImpID: imp.ID, + Price: float64(int(rand.Float64()*1000)) / 100, + AdID: fmt.Sprintf("adID-%d", i), + AdM: "AdContent", + CrID: fmt.Sprintf("creative-%d", i), + W: *imp.Banner.W, + H: *imp.Banner.H, + DealID: fmt.Sprintf("DealID_%d", i), + }) + } + resp.SeatBid[0].Bid = bids + + js, err := json.Marshal(resp) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.Write(js) +} + +func TestPubmaticInvalidCall(t *testing.T) { + + an := NewPubmaticLegacyAdapter(adapters.DefaultHTTPAdapterConfig, "blah") + + ctx := context.Background() + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{} + _, err := an.Call(ctx, &pbReq, &pbBidder) + if err == nil { + t.Fatalf("No error received for invalid request") + } +} + +func TestPubmaticTimeout(t *testing.T) { + + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-time.After(2 * time.Millisecond) + }), + ) + defer server.Close() + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewPubmaticLegacyAdapter(&conf, server.URL) + ctx, cancel := context.WithTimeout(context.Background(), 0) + defer cancel() + + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + Sizes: []openrtb2.Format{ + { + W: 120, + H: 240, + }, + }, + Params: json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@120x240\"}"), + }, + }, + } + _, err := an.Call(ctx, &pbReq, &pbBidder) + if err == nil || err != context.DeadlineExceeded { + t.Fatalf("No timeout received for timed out request: %v", err) + } +} + +func TestPubmaticInvalidJson(t *testing.T) { + + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "Blah") + }), + ) + defer server.Close() + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewPubmaticLegacyAdapter(&conf, server.URL) + ctx := context.Background() + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + Sizes: []openrtb2.Format{ + { + W: 120, + H: 240, + }, + }, + Params: json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@120x240\"}"), + }, + }, + } + _, err := an.Call(ctx, &pbReq, &pbBidder) + if err == nil { + t.Fatalf("No error received for invalid request") + } +} + +func TestPubmaticInvalidStatusCode(t *testing.T) { + + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Send 404 + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + }), + ) + defer server.Close() + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewPubmaticLegacyAdapter(&conf, server.URL) + ctx := context.Background() + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + Sizes: []openrtb2.Format{ + { + W: 120, + H: 240, + }, + }, + Params: json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@120x240\"}"), + }, + }, + } + _, err := an.Call(ctx, &pbReq, &pbBidder) + if err == nil { + t.Fatalf("No error received for invalid request") + } +} + +func TestPubmaticInvalidInputParameters(t *testing.T) { + + server := httptest.NewServer(http.HandlerFunc(DummyPubMaticServer)) + defer server.Close() + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewPubmaticLegacyAdapter(&conf, server.URL) + ctx := context.Background() + + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + BidID: "bidid", + Sizes: []openrtb2.Format{ + { + W: 120, + H: 240, + }, + }, + }, + }, + } + + pbReq.IsDebug = true + inValidPubmaticParams := []json.RawMessage{ + // Invalid Request JSON + json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@120x240\""), + // Missing adSlot in AdUnits.Params + json.RawMessage("{\"publisherId\": \"10\"}"), + // Missing publisher ID + json.RawMessage("{\"adSlot\": \"slot@120x240\"}"), + // Missing slot name in AdUnits.Params + json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"@120x240\"}"), + // Invalid adSize in AdUnits.Params + json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@120-240\"}"), + // Missing impression width and height in AdUnits.Params + json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@\"}"), + // Missing height in AdUnits.Params + json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@120\"}"), + // Missing width in AdUnits.Params + json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@x120\"}"), + // Incorrect width param in AdUnits.Params + json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@valx120\"}"), + // Incorrect height param in AdUnits.Params + json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@120xval\"}"), + // Empty slot name in AdUnits.Params, + json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \" @120x240\"}"), + // Empty width in AdUnits.Params + json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@ x240\"}"), + // Empty height in AdUnits.Params + json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@120x \"}"), + // Empty height in AdUnits.Params + json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \" @120x \"}"), + // Invalid Keywords + json.RawMessage(`{"publisherId": "640", "adSlot": "slot1@336x280","keywords":{"pmZoneId":1},"wrapper":{"version":2,"profile":595}}`), + // Invalid Wrapper ext + json.RawMessage(`{"publisherId": "640", "adSlot": "slot1@336x280","keywords":{"pmZoneId":"Zone1,Zone2"},"wrapper":{"version":"2","profile":595}}`), + } + + for _, param := range inValidPubmaticParams { + pbBidder.AdUnits[0].Params = param + _, err := an.Call(ctx, &pbReq, &pbBidder) + if err == nil { + t.Fatalf("Should get errors for params = %v", string(param)) + } + } + +} + +func TestPubmaticBasicResponse_MandatoryParams(t *testing.T) { + + server := httptest.NewServer(http.HandlerFunc(DummyPubMaticServer)) + defer server.Close() + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewPubmaticLegacyAdapter(&conf, server.URL) + ctx := context.Background() + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode", + BidID: "bidid", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + Sizes: []openrtb2.Format{ + { + W: 336, + H: 280, + }, + }, + Params: json.RawMessage("{\"publisherId\": \"640\", \"adSlot\": \"slot1@336x280\"}"), + }, + }, + } + pbReq.IsDebug = true + bids, err := an.Call(ctx, &pbReq, &pbBidder) + if err != nil { + t.Fatalf("Should not have gotten an error: %v", err) + } + if len(bids) != 1 { + t.Fatalf("Should have received one bid") + } +} + +func TestPubmaticBasicResponse_AllParams(t *testing.T) { + + server := httptest.NewServer(http.HandlerFunc(DummyPubMaticServer)) + defer server.Close() + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewPubmaticLegacyAdapter(&conf, server.URL) + ctx := context.Background() + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode", + BidID: "bidid", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + Sizes: []openrtb2.Format{ + { + W: 336, + H: 280, + }, + }, + Params: json.RawMessage(`{"publisherId": "640", + "adSlot": "slot1@336x280", + "keywords":{ + "pmZoneId": "Zone1,Zone2" + }, + "wrapper": + {"version":2, + "profile":595} + }`), + }, + }, + } + pbReq.IsDebug = true + bids, err := an.Call(ctx, &pbReq, &pbBidder) + if err != nil { + t.Fatalf("Should not have gotten an error: %v", err) + } + if len(bids) != 1 { + t.Fatalf("Should have received one bid") + } +} + +func TestPubmaticMultiImpressionResponse(t *testing.T) { + + server := httptest.NewServer(http.HandlerFunc(DummyPubMaticServer)) + defer server.Close() + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewPubmaticLegacyAdapter(&conf, server.URL) + + ctx := context.Background() + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode1", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + BidID: "bidid", + Sizes: []openrtb2.Format{ + { + W: 336, + H: 280, + }, + }, + Params: json.RawMessage("{\"publisherId\": \"640\", \"adSlot\": \"slot1@336x280\"}"), + }, + { + Code: "unitCode1", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + BidID: "bidid", + Sizes: []openrtb2.Format{ + { + W: 800, + H: 200, + }, + }, + Params: json.RawMessage("{\"publisherId\": \"640\", \"adSlot\": \"slot1@800x200\"}"), + }, + }, + } + bids, err := an.Call(ctx, &pbReq, &pbBidder) + if err != nil { + t.Fatalf("Should not have gotten an error: %v", err) + } + if len(bids) != 2 { + t.Fatalf("Should have received two bids") + } +} + +func TestPubmaticMultiAdUnitResponse(t *testing.T) { + + server := httptest.NewServer(http.HandlerFunc(DummyPubMaticServer)) + defer server.Close() + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewPubmaticLegacyAdapter(&conf, server.URL) + + ctx := context.Background() + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode1", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + BidID: "bidid", + Sizes: []openrtb2.Format{ + { + W: 336, + H: 280, + }, + }, + Params: json.RawMessage("{\"publisherId\": \"640\", \"adSlot\": \"slot1@336x280\"}"), + }, + { + Code: "unitCode2", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + BidID: "bidid", + Sizes: []openrtb2.Format{ + { + W: 800, + H: 200, + }, + }, + Params: json.RawMessage("{\"publisherId\": \"640\", \"adSlot\": \"slot1@800x200\"}"), + }, + }, + } + bids, err := an.Call(ctx, &pbReq, &pbBidder) + if err != nil { + t.Fatalf("Should not have gotten an error: %v", err) + } + if len(bids) != 2 { + t.Fatalf("Should have received one bid") + } + +} + +func TestPubmaticMobileResponse(t *testing.T) { + + server := httptest.NewServer(http.HandlerFunc(DummyPubMaticServer)) + defer server.Close() + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewPubmaticLegacyAdapter(&conf, server.URL) + + ctx := context.Background() + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode", + BidID: "bidid", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + Sizes: []openrtb2.Format{ + { + W: 336, + H: 280, + }, + }, + Params: json.RawMessage("{\"publisherId\": \"640\", \"adSlot\": \"slot1@336x280\"}"), + }, + }, + } + + pbReq.App = &openrtb2.App{ + ID: "com.test", + Name: "testApp", + } + + bids, err := an.Call(ctx, &pbReq, &pbBidder) + if err != nil { + t.Fatalf("Should not have gotten an error: %v", err) + } + if len(bids) != 1 { + t.Fatalf("Should have received one bid") + } +} +func TestPubmaticInvalidLookupBidIDParameter(t *testing.T) { + + server := httptest.NewServer(http.HandlerFunc(DummyPubMaticServer)) + defer server.Close() + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewPubmaticLegacyAdapter(&conf, server.URL) + + ctx := context.Background() + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + Sizes: []openrtb2.Format{ + { + W: 120, + H: 240, + }, + }, + }, + }, + } + + pbBidder.AdUnits[0].Params = json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@120x240\"}") + _, err := an.Call(ctx, &pbReq, &pbBidder) + + CompareStringValue(err.Error(), "Unknown ad unit code 'unitCode'", t) +} + +func TestPubmaticAdSlotParams(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(DummyPubMaticServer)) + defer server.Close() + + conf := *adapters.DefaultHTTPAdapterConfig + an := NewPubmaticLegacyAdapter(&conf, server.URL) + + ctx := context.Background() + pbReq := pbs.PBSRequest{} + pbBidder := pbs.PBSBidder{ + BidderCode: "bannerCode", + AdUnits: []pbs.PBSAdUnit{ + { + Code: "unitCode", + BidID: "bidid", + MediaTypes: []pbs.MediaType{pbs.MEDIA_TYPE_BANNER}, + Sizes: []openrtb2.Format{ + { + W: 120, + H: 240, + }, + }, + }, + }, + } + pbBidder.AdUnits[0].Params = json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \" slot@120x240\"}") + bids, err := an.Call(ctx, &pbReq, &pbBidder) + if err != nil && len(bids) != 1 { + t.Fatalf("Should not return err") + } + + pbBidder.AdUnits[0].Params = json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot @120x240\"}") + bids, err = an.Call(ctx, &pbReq, &pbBidder) + if err != nil && len(bids) != 1 { + t.Fatalf("Should not return err") + } + + pbBidder.AdUnits[0].Params = json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@120x240 \"}") + bids, err = an.Call(ctx, &pbReq, &pbBidder) + if err != nil && len(bids) != 1 { + t.Fatalf("Should not return err") + } + + pbBidder.AdUnits[0].Params = json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@ 120x240\"}") + bids, err = an.Call(ctx, &pbReq, &pbBidder) + if err != nil && len(bids) != 1 { + t.Fatalf("Should not return err") + } + + pbBidder.AdUnits[0].Params = json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@220 x240\"}") + bids, err = an.Call(ctx, &pbReq, &pbBidder) + if err != nil && len(bids) != 1 { + t.Fatalf("Should not return err") + } + + pbBidder.AdUnits[0].Params = json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@120x 240\"}") + bids, err = an.Call(ctx, &pbReq, &pbBidder) + if err != nil && len(bids) != 1 { + t.Fatalf("Should not return err") + } + + pbBidder.AdUnits[0].Params = json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@120x240:1\"}") + bids, err = an.Call(ctx, &pbReq, &pbBidder) + if err != nil && len(bids) != 1 { + t.Fatalf("Should not return err") + } + + pbBidder.AdUnits[0].Params = json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@120x 240:1\"}") + bids, err = an.Call(ctx, &pbReq, &pbBidder) + if err != nil && len(bids) != 1 { + t.Fatalf("Should not return err") + } + + pbBidder.AdUnits[0].Params = json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@120x240 :1\"}") + bids, err = an.Call(ctx, &pbReq, &pbBidder) + if err != nil && len(bids) != 1 { + t.Fatalf("Should not return err") + } + + pbBidder.AdUnits[0].Params = json.RawMessage("{\"publisherId\": \"10\", \"adSlot\": \"slot@120x240: 1\"}") + bids, err = an.Call(ctx, &pbReq, &pbBidder) + if err != nil && len(bids) != 1 { + t.Fatalf("Should not return err") + } +} + +func TestPubmaticSampleRequest(t *testing.T) { + + server := httptest.NewServer(http.HandlerFunc(DummyPubMaticServer)) + defer server.Close() + + pbReq := pbs.PBSRequest{ + AdUnits: make([]pbs.AdUnit, 1), + } + pbReq.AdUnits[0] = pbs.AdUnit{ + Code: "adUnit_1", + Sizes: []openrtb2.Format{ + { + W: 100, + H: 120, + }, + }, + Bids: []pbs.Bids{ + { + BidderCode: "pubmatic", + BidID: "BidID", + Params: json.RawMessage("{\"publisherId\": \"640\", \"adSlot\": \"slot1@100x120\"}"), + }, + }, + } + + pbReq.IsDebug = true + + body := new(bytes.Buffer) + err := json.NewEncoder(body).Encode(pbReq) + if err != nil { + t.Fatalf("Error when serializing request") + } + + httpReq := httptest.NewRequest("POST", server.URL, body) + httpReq.Header.Add("Referer", "http://test.com/sports") + pc := usersync.ParseCookieFromRequest(httpReq, &config.HostCookie{}) + pc.TrySync("pubmatic", "12345") + fakewriter := httptest.NewRecorder() + + pc.SetCookieOnResponse(fakewriter, false, &config.HostCookie{Domain: ""}, 90*24*time.Hour) + httpReq.Header.Add("Cookie", fakewriter.Header().Get("Set-Cookie")) + + cacheClient, _ := dummycache.New() + hcs := config.HostCookie{} + + _, err = pbs.ParsePBSRequest(httpReq, &config.AuctionTimeouts{ + Default: 2000, + Max: 2000, + }, cacheClient, &hcs) + if err != nil { + t.Fatalf("Error when parsing request: %v", err) + } +} + func TestGetBidTypeVideo(t *testing.T) { - pubmaticExt := &pubmaticBidExt{} + pubmaticExt := new(pubmaticBidExt) pubmaticExt.BidType = new(int) *pubmaticExt.BidType = 1 actualBidTypeValue := getBidType(pubmaticExt) @@ -30,8 +691,8 @@ func TestGetBidTypeVideo(t *testing.T) { } func TestGetBidTypeForMissingBidTypeExt(t *testing.T) { - pubmaticExt := &pubmaticBidExt{} - actualBidTypeValue := getBidType(pubmaticExt) + pubmaticExt := pubmaticBidExt{} + actualBidTypeValue := getBidType(&pubmaticExt) // banner is the default bid type when no bidType key is present in the bid.ext if actualBidTypeValue != "banner" { t.Errorf("Expected Bid Type value was: banner, actual value is: %v", actualBidTypeValue) @@ -39,7 +700,7 @@ func TestGetBidTypeForMissingBidTypeExt(t *testing.T) { } func TestGetBidTypeBanner(t *testing.T) { - pubmaticExt := &pubmaticBidExt{} + pubmaticExt := new(pubmaticBidExt) pubmaticExt.BidType = new(int) *pubmaticExt.BidType = 0 actualBidTypeValue := getBidType(pubmaticExt) @@ -49,7 +710,7 @@ func TestGetBidTypeBanner(t *testing.T) { } func TestGetBidTypeNative(t *testing.T) { - pubmaticExt := &pubmaticBidExt{} + pubmaticExt := new(pubmaticBidExt) pubmaticExt.BidType = new(int) *pubmaticExt.BidType = 2 actualBidTypeValue := getBidType(pubmaticExt) @@ -59,7 +720,7 @@ func TestGetBidTypeNative(t *testing.T) { } func TestGetBidTypeForUnsupportedCode(t *testing.T) { - pubmaticExt := &pubmaticBidExt{} + pubmaticExt := new(pubmaticBidExt) pubmaticExt.BidType = new(int) *pubmaticExt.BidType = 99 actualBidTypeValue := getBidType(pubmaticExt) diff --git a/adapters/pulsepoint/pulsepoint.go b/adapters/pulsepoint/pulsepoint.go index f756d5dd31a..6b6b4305607 100644 --- a/adapters/pulsepoint/pulsepoint.go +++ b/adapters/pulsepoint/pulsepoint.go @@ -1,21 +1,27 @@ package pulsepoint import ( + "bytes" + "context" "encoding/json" "fmt" + "io/ioutil" "net/http" "strconv" + "strings" + "github.com/mxmCherry/openrtb/v15/openrtb2" "github.com/prebid/prebid-server/adapters" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/openrtb_ext" - - "github.com/mxmCherry/openrtb/v15/openrtb2" + "github.com/prebid/prebid-server/pbs" + "golang.org/x/net/context/ctxhttp" ) type PulsePointAdapter struct { - URI string + http *adapters.HTTPAdapter + URI string } // Builds an instance of PulsePointAdapter @@ -162,3 +168,192 @@ func getBidType(imp openrtb2.Imp) openrtb_ext.BidType { } return "" } + +///////////////////////////////// +// Legacy implementation: Start +///////////////////////////////// + +func NewPulsePointLegacyAdapter(config *adapters.HTTPAdapterConfig, uri string) *PulsePointAdapter { + a := adapters.NewHTTPAdapter(config) + + return &PulsePointAdapter{ + http: a, + URI: uri, + } +} + +// used for cookies and such +func (a *PulsePointAdapter) Name() string { + return "pulsepoint" +} + +// parameters for pulsepoint adapter. +type PulsepointParams struct { + PublisherId int `json:"cp"` + TagId int `json:"ct"` + AdSize string `json:"cf"` +} + +func (a *PulsePointAdapter) SkipNoCookies() bool { + return false +} + +func (a *PulsePointAdapter) Call(ctx context.Context, req *pbs.PBSRequest, bidder *pbs.PBSBidder) (pbs.PBSBidSlice, error) { + mediaTypes := []pbs.MediaType{pbs.MEDIA_TYPE_BANNER} + ppReq, err := adapters.MakeOpenRTBGeneric(req, bidder, a.Name(), mediaTypes) + + if err != nil { + return nil, err + } + + for i, unit := range bidder.AdUnits { + var params PulsepointParams + err := json.Unmarshal(unit.Params, ¶ms) + if err != nil { + return nil, &errortypes.BadInput{ + Message: err.Error(), + } + } + if params.PublisherId == 0 { + return nil, &errortypes.BadInput{ + Message: "Missing PublisherId param cp", + } + } + if params.TagId == 0 { + return nil, &errortypes.BadInput{ + Message: "Missing TagId param ct", + } + } + if params.AdSize == "" { + return nil, &errortypes.BadInput{ + Message: "Missing AdSize param cf", + } + } + // Fixes some segfaults. Since this is legacy code, I'm not looking into it too deeply + if len(ppReq.Imp) <= i { + break + } + ppReq.Imp[i].TagID = strconv.Itoa(params.TagId) + publisher := &openrtb2.Publisher{ID: strconv.Itoa(params.PublisherId)} + if ppReq.Site != nil { + siteCopy := *ppReq.Site + siteCopy.Publisher = publisher + ppReq.Site = &siteCopy + } else { + appCopy := *ppReq.App + appCopy.Publisher = publisher + ppReq.App = &appCopy + } + if ppReq.Imp[i].Banner != nil { + var size = strings.Split(strings.ToLower(params.AdSize), "x") + if len(size) == 2 { + width, err := strconv.Atoi(size[0]) + if err == nil { + ppReq.Imp[i].Banner.W = openrtb2.Int64Ptr(int64(width)) + } else { + return nil, &errortypes.BadInput{ + Message: fmt.Sprintf("Invalid Width param %s", size[0]), + } + } + height, err := strconv.Atoi(size[1]) + if err == nil { + ppReq.Imp[i].Banner.H = openrtb2.Int64Ptr(int64(height)) + } else { + return nil, &errortypes.BadInput{ + Message: fmt.Sprintf("Invalid Height param %s", size[1]), + } + } + } else { + return nil, &errortypes.BadInput{ + Message: fmt.Sprintf("Invalid AdSize param %s", params.AdSize), + } + } + } + } + reqJSON, err := json.Marshal(ppReq) + debug := &pbs.BidderDebug{ + RequestURI: a.URI, + } + + if req.IsDebug { + debug.RequestBody = string(reqJSON) + bidder.Debug = append(bidder.Debug, debug) + } + + httpReq, err := http.NewRequest("POST", a.URI, bytes.NewBuffer(reqJSON)) + httpReq.Header.Add("Content-Type", "application/json;charset=utf-8") + httpReq.Header.Add("Accept", "application/json") + + ppResp, err := ctxhttp.Do(ctx, a.http.Client, httpReq) + if err != nil { + return nil, err + } + + debug.StatusCode = ppResp.StatusCode + + if ppResp.StatusCode == http.StatusNoContent { + return nil, nil + } + + if ppResp.StatusCode == http.StatusBadRequest { + return nil, &errortypes.BadInput{ + Message: fmt.Sprintf("HTTP status: %d", ppResp.StatusCode), + } + } + + if ppResp.StatusCode != http.StatusOK { + return nil, &errortypes.BadServerResponse{ + Message: fmt.Sprintf("HTTP status: %d", ppResp.StatusCode), + } + } + + defer ppResp.Body.Close() + body, err := ioutil.ReadAll(ppResp.Body) + if err != nil { + return nil, err + } + + if req.IsDebug { + debug.ResponseBody = string(body) + } + + var bidResp openrtb2.BidResponse + err = json.Unmarshal(body, &bidResp) + if err != nil { + return nil, &errortypes.BadServerResponse{ + Message: err.Error(), + } + } + + bids := make(pbs.PBSBidSlice, 0) + + for _, sb := range bidResp.SeatBid { + for _, bid := range sb.Bid { + bidID := bidder.LookupBidID(bid.ImpID) + if bidID == "" { + return nil, &errortypes.BadServerResponse{ + Message: fmt.Sprintf("Unknown ad unit code '%s'", bid.ImpID), + } + } + + pbid := pbs.PBSBid{ + BidID: bidID, + AdUnitCode: bid.ImpID, + BidderCode: bidder.BidderCode, + Price: bid.Price, + Adm: bid.AdM, + Creative_id: bid.CrID, + Width: bid.W, + Height: bid.H, + CreativeMediaType: string(openrtb_ext.BidTypeBanner), + } + bids = append(bids, &pbid) + } + } + + return bids, nil +} + +///////////////////////////////// +// Legacy implementation: End +///////////////////////////////// diff --git a/adapters/pulsepoint/pulsepoint_test.go b/adapters/pulsepoint/pulsepoint_test.go index 8929898522a..a4e20b04859 100644 --- a/adapters/pulsepoint/pulsepoint_test.go +++ b/adapters/pulsepoint/pulsepoint_test.go @@ -1,11 +1,26 @@ package pulsepoint import ( + "encoding/json" + "net/http" "testing" + "github.com/mxmCherry/openrtb/v15/openrtb2" + "github.com/prebid/prebid-server/adapters" + "github.com/prebid/prebid-server/openrtb_ext" + + "bytes" + "context" + "fmt" + "io/ioutil" + "net/http/httptest" + "time" + "github.com/prebid/prebid-server/adapters/adapterstest" + "github.com/prebid/prebid-server/cache/dummycache" "github.com/prebid/prebid-server/config" - "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/pbs" + "github.com/prebid/prebid-server/usersync" ) func TestJsonSamples(t *testing.T) { @@ -18,3 +33,280 @@ func TestJsonSamples(t *testing.T) { adapterstest.RunJSONBidderTest(t, "pulsepointtest", bidder) } + +///////////////////////////////// +// Legacy implementation: Start +///////////////////////////////// + +/** + * Verify adapter names are setup correctly. + */ +func TestPulsePointAdapterNames(t *testing.T) { + adapter := NewPulsePointLegacyAdapter(adapters.DefaultHTTPAdapterConfig, "http://localhost/bid") + adapterstest.VerifyStringValue(adapter.Name(), "pulsepoint", t) +} + +/** + * Test required parameters not sent + */ +func TestPulsePointRequiredBidParameters(t *testing.T) { + adapter := NewPulsePointLegacyAdapter(adapters.DefaultHTTPAdapterConfig, "http://localhost/bid") + ctx := context.TODO() + req := SampleRequest(1, t) + bidder := req.Bidders[0] + // remove "ct" param and verify error message. + bidder.AdUnits[0].Params = json.RawMessage("{\"cp\": 2001, \"cf\": \"728X90\"}") + _, errTag := adapter.Call(ctx, req, bidder) + adapterstest.VerifyStringValue(errTag.Error(), "Missing TagId param ct", t) + // remove "cp" param and verify error message. + bidder.AdUnits[0].Params = json.RawMessage("{\"ct\": 1001, \"cf\": \"728X90\"}") + _, errPub := adapter.Call(ctx, req, bidder) + adapterstest.VerifyStringValue(errPub.Error(), "Missing PublisherId param cp", t) + // remove "cf" param and verify error message. + bidder.AdUnits[0].Params = json.RawMessage("{\"cp\": 2001, \"ct\": 1001}") + _, errSize := adapter.Call(ctx, req, bidder) + adapterstest.VerifyStringValue(errSize.Error(), "Missing AdSize param cf", t) + // invalid width parameter value for cf + bidder.AdUnits[0].Params = json.RawMessage("{\"ct\": 1001, \"cp\": 2001, \"cf\": \"aXb\"}") + _, errWidth := adapter.Call(ctx, req, bidder) + adapterstest.VerifyStringValue(errWidth.Error(), "Invalid Width param a", t) + // invalid parameter values for cf + bidder.AdUnits[0].Params = json.RawMessage("{\"ct\": 1001, \"cp\": 2001, \"cf\": \"12Xb\"}") + _, errHeight := adapter.Call(ctx, req, bidder) + adapterstest.VerifyStringValue(errHeight.Error(), "Invalid Height param b", t) + // invalid parameter values for cf + bidder.AdUnits[0].Params = json.RawMessage("{\"ct\": 1001, \"cp\": 2001, \"cf\": \"12-20\"}") + _, errAdSizeValue := adapter.Call(ctx, req, bidder) + adapterstest.VerifyStringValue(errAdSizeValue.Error(), "Invalid AdSize param 12-20", t) +} + +/** + * Verify the openrtb request sent to Pulsepoint endpoint. + * Ensure the ct, cp, cf params are transformed and sent alright. + */ +func TestPulsePointOpenRTBRequest(t *testing.T) { + service := CreateService(adapterstest.BidOnTags("")) + server := service.Server + ctx := context.TODO() + req := SampleRequest(1, t) + bidder := req.Bidders[0] + adapter := NewPulsePointLegacyAdapter(adapters.DefaultHTTPAdapterConfig, server.URL) + adapter.Call(ctx, req, bidder) + adapterstest.VerifyIntValue(len(service.LastBidRequest.Imp), 1, t) + adapterstest.VerifyStringValue(service.LastBidRequest.Imp[0].TagID, "1001", t) + adapterstest.VerifyStringValue(service.LastBidRequest.Site.Publisher.ID, "2001", t) + adapterstest.VerifyBannerSize(service.LastBidRequest.Imp[0].Banner, 728, 90, t) +} + +/** + * Verify bidding behavior. + */ +func TestPulsePointBiddingBehavior(t *testing.T) { + // setup server endpoint to return bid. + server := CreateService(adapterstest.BidOnTags("1001")).Server + ctx := context.TODO() + req := SampleRequest(1, t) + bidder := req.Bidders[0] + adapter := NewPulsePointLegacyAdapter(adapters.DefaultHTTPAdapterConfig, server.URL) + bids, _ := adapter.Call(ctx, req, bidder) + // number of bids should be 1 + adapterstest.VerifyIntValue(len(bids), 1, t) + adapterstest.VerifyStringValue(bids[0].AdUnitCode, "div-adunit-1", t) + adapterstest.VerifyStringValue(bids[0].BidderCode, "pulsepoint", t) + adapterstest.VerifyStringValue(bids[0].Adm, "
This is an Ad
", t) + adapterstest.VerifyStringValue(bids[0].Creative_id, "Cr-234", t) + adapterstest.VerifyIntValue(int(bids[0].Width), 728, t) + adapterstest.VerifyIntValue(int(bids[0].Height), 90, t) + adapterstest.VerifyIntValue(int(bids[0].Price*100), 210, t) + adapterstest.VerifyStringValue(bids[0].CreativeMediaType, string(openrtb_ext.BidTypeBanner), t) +} + +/** + * Verify bidding behavior on multiple impressions, some impressions make a bid + */ +func TestPulsePointMultiImpPartialBidding(t *testing.T) { + // setup server endpoint to return bid. + service := CreateService(adapterstest.BidOnTags("1001")) + server := service.Server + ctx := context.TODO() + req := SampleRequest(2, t) + bidder := req.Bidders[0] + adapter := NewPulsePointLegacyAdapter(adapters.DefaultHTTPAdapterConfig, server.URL) + bids, _ := adapter.Call(ctx, req, bidder) + // two impressions sent. + // number of bids should be 1 + adapterstest.VerifyIntValue(len(service.LastBidRequest.Imp), 2, t) + adapterstest.VerifyIntValue(len(bids), 1, t) + adapterstest.VerifyStringValue(bids[0].AdUnitCode, "div-adunit-1", t) +} + +/** + * Verify bidding behavior on multiple impressions, all impressions passed back. + */ +func TestPulsePointMultiImpPassback(t *testing.T) { + // setup server endpoint to return bid. + service := CreateService(adapterstest.BidOnTags("")) + server := service.Server + ctx := context.TODO() + req := SampleRequest(2, t) + bidder := req.Bidders[0] + adapter := NewPulsePointLegacyAdapter(adapters.DefaultHTTPAdapterConfig, server.URL) + bids, _ := adapter.Call(ctx, req, bidder) + // two impressions sent. + // number of bids should be 1 + adapterstest.VerifyIntValue(len(service.LastBidRequest.Imp), 2, t) + adapterstest.VerifyIntValue(len(bids), 0, t) +} + +/** + * Verify bidding behavior on multiple impressions, all impressions passed back. + */ +func TestPulsePointMultiImpAllBid(t *testing.T) { + // setup server endpoint to return bid. + service := CreateService(adapterstest.BidOnTags("1001,1002")) + server := service.Server + ctx := context.TODO() + req := SampleRequest(2, t) + bidder := req.Bidders[0] + adapter := NewPulsePointLegacyAdapter(adapters.DefaultHTTPAdapterConfig, server.URL) + bids, _ := adapter.Call(ctx, req, bidder) + // two impressions sent. + // number of bids should be 1 + adapterstest.VerifyIntValue(len(service.LastBidRequest.Imp), 2, t) + adapterstest.VerifyIntValue(len(bids), 2, t) + adapterstest.VerifyStringValue(bids[0].AdUnitCode, "div-adunit-1", t) + adapterstest.VerifyStringValue(bids[1].AdUnitCode, "div-adunit-2", t) +} + +/** + * Verify bidding behavior on mobile app requests + */ +func TestMobileAppRequest(t *testing.T) { + // setup server endpoint to return bid. + service := CreateService(adapterstest.BidOnTags("1001")) + server := service.Server + ctx := context.TODO() + req := SampleRequest(1, t) + req.App = &openrtb2.App{ + ID: "com.facebook.katana", + Name: "facebook", + } + bidder := req.Bidders[0] + adapter := NewPulsePointLegacyAdapter(adapters.DefaultHTTPAdapterConfig, server.URL) + bids, _ := adapter.Call(ctx, req, bidder) + // one mobile app impression sent. + // verify appropriate fields are sent to pulsepoint endpoint. + adapterstest.VerifyIntValue(len(service.LastBidRequest.Imp), 1, t) + adapterstest.VerifyStringValue(service.LastBidRequest.App.ID, "com.facebook.katana", t) + adapterstest.VerifyIntValue(len(bids), 1, t) + adapterstest.VerifyStringValue(bids[0].AdUnitCode, "div-adunit-1", t) +} + +/** + * Produces a sample PBSRequest, for the impressions given. + */ +func SampleRequest(numberOfImpressions int, t *testing.T) *pbs.PBSRequest { + // create a request object + req := pbs.PBSRequest{ + AdUnits: make([]pbs.AdUnit, 2), + } + req.AccountID = "1" + tagId := 1001 + for i := 0; i < numberOfImpressions; i++ { + req.AdUnits[i] = pbs.AdUnit{ + Code: fmt.Sprintf("div-adunit-%d", i+1), + Sizes: []openrtb2.Format{ + { + W: 10, + H: 12, + }, + }, + Bids: []pbs.Bids{ + { + BidderCode: "pulsepoint", + BidID: fmt.Sprintf("Bid-%d", i+1), + Params: json.RawMessage(fmt.Sprintf("{\"ct\": %d, \"cp\": 2001, \"cf\": \"728X90\"}", tagId+i)), + }, + }, + } + } + // serialize the request to json + body := new(bytes.Buffer) + err := json.NewEncoder(body).Encode(req) + if err != nil { + t.Fatalf("Error when serializing request") + } + // setup a http request + httpReq := httptest.NewRequest("POST", CreateService(adapterstest.BidOnTags("")).Server.URL, body) + httpReq.Header.Add("Referer", "http://news.pub/topnews") + pc := usersync.ParseCookieFromRequest(httpReq, &config.HostCookie{}) + pc.TrySync("pulsepoint", "pulsepointUser123") + fakewriter := httptest.NewRecorder() + + pc.SetCookieOnResponse(fakewriter, false, &config.HostCookie{Domain: ""}, 90*24*time.Hour) + httpReq.Header.Add("Cookie", fakewriter.Header().Get("Set-Cookie")) + // parse the http request + cacheClient, _ := dummycache.New() + hcs := config.HostCookie{} + + parsedReq, err := pbs.ParsePBSRequest(httpReq, &config.AuctionTimeouts{ + Default: 2000, + Max: 2000, + }, cacheClient, &hcs) + if err != nil { + t.Fatalf("Error when parsing request: %v", err) + } + return parsedReq +} + +/** + * Represents a mock ORTB endpoint of PulsePoint. Would return a bid + * for TagId 1001 and passback for 1002 as the default behavior. + */ +func CreateService(tagsToBid map[string]bool) adapterstest.OrtbMockService { + service := adapterstest.OrtbMockService{} + var lastBidRequest openrtb2.BidRequest + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + var breq openrtb2.BidRequest + err = json.Unmarshal(body, &breq) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + lastBidRequest = breq + var bids []openrtb2.Bid + for i, imp := range breq.Imp { + if tagsToBid[imp.TagID] { + bids = append(bids, adapterstest.SampleBid(imp.Banner.W, imp.Banner.H, imp.ID, i+1)) + } + } + // no bids were produced, pulsepoint service returns 204 + if len(bids) == 0 { + w.WriteHeader(204) + } else { + // serialize the bids to openrtb2.BidResponse + js, _ := json.Marshal(openrtb2.BidResponse{ + SeatBid: []openrtb2.SeatBid{ + { + Bid: bids, + }, + }, + }) + w.Header().Set("Content-Type", "application/json") + w.Write(js) + } + })) + service.Server = server + service.LastBidRequest = &lastBidRequest + return service +} + +///////////////////////////////// +// Legacy implementation: End +///////////////////////////////// diff --git a/adapters/rubicon/rubicon.go b/adapters/rubicon/rubicon.go index ace1bfaa12d..80c62df16a1 100644 --- a/adapters/rubicon/rubicon.go +++ b/adapters/rubicon/rubicon.go @@ -1,30 +1,53 @@ package rubicon import ( + "bytes" + "context" "encoding/json" "fmt" + "github.com/buger/jsonparser" + "io/ioutil" "net/http" "net/url" "strconv" "strings" + "github.com/golang/glog" + "github.com/mxmCherry/openrtb/v15/openrtb2" "github.com/prebid/prebid-server/adapters" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/openrtb_ext" - - "github.com/buger/jsonparser" - "github.com/mxmCherry/openrtb/v15/openrtb2" + "github.com/prebid/prebid-server/pbs" + "golang.org/x/net/context/ctxhttp" ) const badvLimitSize = 50 type RubiconAdapter struct { + http *adapters.HTTPAdapter URI string XAPIUsername string XAPIPassword string } +func (a *RubiconAdapter) Name() string { + return "rubicon" +} + +func (a *RubiconAdapter) SkipNoCookies() bool { + return false +} + +type rubiconParams struct { + AccountId int `json:"accountId"` + SiteId int `json:"siteId"` + ZoneId int `json:"zoneId"` + Inventory json.RawMessage `json:"inventory,omitempty"` + Visitor json.RawMessage `json:"visitor,omitempty"` + Video rubiconVideoParams `json:"video"` +} + type bidRequestExt struct { Prebid bidRequestExtPrebid `json:"prebid"` } @@ -111,6 +134,15 @@ type rubiconBannerExt struct { } // ***** Video Extension ***** +type rubiconVideoParams struct { + Language string `json:"language,omitempty"` + PlayerHeight int `json:"playerHeight,omitempty"` + PlayerWidth int `json:"playerWidth,omitempty"` + VideoSizeID int `json:"size_id,omitempty"` + Skip int `json:"skip,omitempty"` + SkipDelay int `json:"skipdelay,omitempty"` +} + type rubiconVideoExt struct { Skip int `json:"skip,omitempty"` SkipDelay int `json:"skipdelay,omitempty"` @@ -122,6 +154,19 @@ type rubiconVideoExtRP struct { SizeID int `json:"size_id,omitempty"` } +type rubiconTargetingExt struct { + RP rubiconTargetingExtRP `json:"rp"` +} + +type rubiconTargetingExtRP struct { + Targeting []rubiconTargetingObj `json:"targeting"` +} + +type rubiconTargetingObj struct { + Key string `json:"key"` + Values []string `json:"values"` +} + type rubiconDeviceExtRP struct { PixelRatio float64 `json:"pixelratio"` } @@ -130,6 +175,10 @@ type rubiconDeviceExt struct { RP rubiconDeviceExtRP `json:"rp"` } +type rubiconUser struct { + Language string `json:"language"` +} + type rubiconBidResponse struct { openrtb2.BidResponse SeatBid []rubiconSeatBid `json:"seatbid,omitempty"` @@ -304,6 +353,273 @@ func parseRubiconSizes(sizes []openrtb2.Format) (primary int, alt []int, err err return } +func (a *RubiconAdapter) callOne(ctx context.Context, reqJSON bytes.Buffer) (result adapters.CallOneResult, err error) { + httpReq, err := http.NewRequest("POST", a.URI, &reqJSON) + httpReq.Header.Add("Content-Type", "application/json;charset=utf-8") + httpReq.Header.Add("Accept", "application/json") + httpReq.Header.Add("User-Agent", "prebid-server/1.0") + httpReq.SetBasicAuth(a.XAPIUsername, a.XAPIPassword) + + rubiResp, e := ctxhttp.Do(ctx, a.http.Client, httpReq) + if e != nil { + err = e + return + } + + defer rubiResp.Body.Close() + body, _ := ioutil.ReadAll(rubiResp.Body) + result.ResponseBody = string(body) + + result.StatusCode = rubiResp.StatusCode + + if rubiResp.StatusCode == 204 { + return + } + + if rubiResp.StatusCode == http.StatusBadRequest { + err = &errortypes.BadInput{ + Message: fmt.Sprintf("HTTP status %d; body: %s", rubiResp.StatusCode, result.ResponseBody), + } + } + + if rubiResp.StatusCode != http.StatusOK { + err = &errortypes.BadServerResponse{ + Message: fmt.Sprintf("HTTP status %d; body: %s", rubiResp.StatusCode, result.ResponseBody), + } + return + } + + var bidResp openrtb2.BidResponse + err = json.Unmarshal(body, &bidResp) + if err != nil { + err = &errortypes.BadServerResponse{ + Message: err.Error(), + } + return + } + if len(bidResp.SeatBid) == 0 { + return + } + if len(bidResp.SeatBid[0].Bid) == 0 { + return + } + bid := bidResp.SeatBid[0].Bid[0] + + result.Bid = &pbs.PBSBid{ + AdUnitCode: bid.ImpID, + Price: bid.Price, + Adm: bid.AdM, + Creative_id: bid.CrID, + // for video, the width and height are undefined as there's no corresponding return value from XAPI + Width: bid.W, + Height: bid.H, + DealId: bid.DealID, + } + + // Pull out any server-side determined targeting + var rpExtTrg rubiconTargetingExt + + if err := json.Unmarshal([]byte(bid.Ext), &rpExtTrg); err == nil { + // Converting string => array(string) to string => string + targeting := make(map[string]string) + + // Only pick off the first for now + for _, target := range rpExtTrg.RP.Targeting { + targeting[target.Key] = target.Values[0] + } + + result.Bid.AdServerTargeting = targeting + } + + return +} + +type callOneObject struct { + requestJson bytes.Buffer + mediaType pbs.MediaType +} + +func (a *RubiconAdapter) Call(ctx context.Context, req *pbs.PBSRequest, bidder *pbs.PBSBidder) (pbs.PBSBidSlice, error) { + callOneObjects := make([]callOneObject, 0, len(bidder.AdUnits)) + supportedMediaTypes := []pbs.MediaType{pbs.MEDIA_TYPE_BANNER, pbs.MEDIA_TYPE_VIDEO} + + rubiReq, err := adapters.MakeOpenRTBGeneric(req, bidder, a.Name(), supportedMediaTypes) + if err != nil { + return nil, err + } + + rubiReqImpCopy := rubiReq.Imp + + for i, unit := range bidder.AdUnits { + // Fixes some segfaults. Since this is legacy code, I'm not looking into it too deeply + if len(rubiReqImpCopy) <= i { + break + } + // Only grab this ad unit + // Not supporting multi-media-type add-unit yet + thisImp := rubiReqImpCopy[i] + + // Amend it with RP-specific information + var params rubiconParams + err = json.Unmarshal(unit.Params, ¶ms) + if err != nil { + return nil, &errortypes.BadInput{ + Message: err.Error(), + } + } + + var mint, mintVersion string + mint = "prebid" + mintVersion = req.SDK.Source + "_" + req.SDK.Platform + "_" + req.SDK.Version + track := rubiconImpExtRPTrack{Mint: mint, MintVersion: mintVersion} + + impExt := rubiconImpExt{RP: rubiconImpExtRP{ + ZoneID: params.ZoneId, + Target: params.Inventory, + Track: track, + }} + thisImp.Ext, err = json.Marshal(&impExt) + if err != nil { + continue + } + + // Copy the $.user object and amend with $.user.ext.rp.target + // Copy avoids race condition since it points to ref & shared with other adapters + userCopy := *rubiReq.User + userExt := rubiconUserExt{RP: rubiconUserExtRP{Target: params.Visitor}} + userCopy.Ext, err = json.Marshal(&userExt) + // Assign back our copy + rubiReq.User = &userCopy + + deviceCopy := *rubiReq.Device + deviceExt := rubiconDeviceExt{RP: rubiconDeviceExtRP{PixelRatio: rubiReq.Device.PxRatio}} + deviceCopy.Ext, err = json.Marshal(&deviceExt) + rubiReq.Device = &deviceCopy + + if thisImp.Video != nil { + + videoSizeId := params.Video.VideoSizeID + if videoSizeId == 0 { + resolvedSizeId, err := resolveVideoSizeId(thisImp.Video.Placement, thisImp.Instl, thisImp.ID) + if err == nil { + videoSizeId = resolvedSizeId + } else { + continue + } + } + + videoExt := rubiconVideoExt{Skip: params.Video.Skip, SkipDelay: params.Video.SkipDelay, RP: rubiconVideoExtRP{SizeID: videoSizeId}} + thisImp.Video.Ext, err = json.Marshal(&videoExt) + } else { + primarySizeID, altSizeIDs, err := parseRubiconSizes(unit.Sizes) + if err != nil { + continue + } + bannerExt := rubiconBannerExt{RP: rubiconBannerExtRP{SizeID: primarySizeID, AltSizeIDs: altSizeIDs, MIME: "text/html"}} + thisImp.Banner.Ext, err = json.Marshal(&bannerExt) + } + + siteExt := rubiconSiteExt{RP: rubiconSiteExtRP{SiteID: params.SiteId}} + pubExt := rubiconPubExt{RP: rubiconPubExtRP{AccountID: params.AccountId}} + var rubiconUser rubiconUser + err = json.Unmarshal(req.PBSUser, &rubiconUser) + + if rubiReq.Site != nil { + siteCopy := *rubiReq.Site + siteCopy.Ext, err = json.Marshal(&siteExt) + siteCopy.Publisher = &openrtb2.Publisher{} + siteCopy.Publisher.Ext, err = json.Marshal(&pubExt) + siteCopy.Content = &openrtb2.Content{} + siteCopy.Content.Language = rubiconUser.Language + rubiReq.Site = &siteCopy + } else { + site := &openrtb2.Site{} + site.Content = &openrtb2.Content{} + site.Content.Language = rubiconUser.Language + rubiReq.Site = site + } + + if rubiReq.App != nil { + appCopy := *rubiReq.App + appCopy.Ext, err = json.Marshal(&siteExt) + appCopy.Publisher = &openrtb2.Publisher{} + appCopy.Publisher.Ext, err = json.Marshal(&pubExt) + rubiReq.App = &appCopy + } + + rubiReq.Imp = []openrtb2.Imp{thisImp} + + var reqBuffer bytes.Buffer + err = json.NewEncoder(&reqBuffer).Encode(rubiReq) + if err != nil { + return nil, err + } + callOneObjects = append(callOneObjects, callOneObject{reqBuffer, unit.MediaTypes[0]}) + } + if len(callOneObjects) == 0 { + return nil, &errortypes.BadInput{ + Message: "Invalid ad unit/imp", + } + } + + ch := make(chan adapters.CallOneResult) + for _, obj := range callOneObjects { + go func(bidder *pbs.PBSBidder, reqJSON bytes.Buffer, mediaType pbs.MediaType) { + result, err := a.callOne(ctx, reqJSON) + result.Error = err + if result.Bid != nil { + result.Bid.BidderCode = bidder.BidderCode + result.Bid.BidID = bidder.LookupBidID(result.Bid.AdUnitCode) + if result.Bid.BidID == "" { + result.Error = &errortypes.BadServerResponse{ + Message: fmt.Sprintf("Unknown ad unit code '%s'", result.Bid.AdUnitCode), + } + result.Bid = nil + } else { + // no need to check whether mediaTypes is nil or length of zero, pbs.ParsePBSRequest will cover + // these cases. + // for media types other than banner and video, pbs.ParseMediaType will throw error. + // we may want to create a map/switch cases to support more media types in the future. + if mediaType == pbs.MEDIA_TYPE_VIDEO { + result.Bid.CreativeMediaType = string(openrtb_ext.BidTypeVideo) + } else { + result.Bid.CreativeMediaType = string(openrtb_ext.BidTypeBanner) + } + } + } + ch <- result + }(bidder, obj.requestJson, obj.mediaType) + } + + bids := make(pbs.PBSBidSlice, 0) + for i := 0; i < len(callOneObjects); i++ { + result := <-ch + if result.Bid != nil && result.Bid.Price != 0 { + bids = append(bids, result.Bid) + } + if req.IsDebug { + debug := &pbs.BidderDebug{ + RequestURI: a.URI, + RequestBody: callOneObjects[i].requestJson.String(), + StatusCode: result.StatusCode, + ResponseBody: result.ResponseBody, + } + bidder.Debug = append(bidder.Debug, debug) + } + if result.Error != nil { + if glog.V(2) { + glog.Infof("Error from rubicon adapter: %v", result.Error) + } + err = result.Error + } + } + + if len(bids) == 0 { + return nil, err + } + return bids, nil +} + func resolveVideoSizeId(placement openrtb2.VideoPlacementType, instl int8, impId string) (sizeID int, err error) { if placement != 0 { if placement == 1 { @@ -349,6 +665,19 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters return bidder, nil } +func NewRubiconLegacyAdapter(httpConfig *adapters.HTTPAdapterConfig, uri string, xuser string, xpass string, tracker string) *RubiconAdapter { + a := adapters.NewHTTPAdapter(httpConfig) + + uri = appendTrackerToUrl(uri, tracker) + + return &RubiconAdapter{ + http: a, + URI: uri, + XAPIUsername: xuser, + XAPIPassword: xpass, + } +} + func (a *RubiconAdapter) MakeRequests(request *openrtb2.BidRequest, reqInfo *adapters.ExtraRequestInfo) ([]*adapters.RequestData, []error) { numRequests := len(request.Imp) errs := make([]error, 0, len(request.Imp)) diff --git a/adapters/rubicon/rubicon_test.go b/adapters/rubicon/rubicon_test.go index 0d88937b6da..9bfa04fa78f 100644 --- a/adapters/rubicon/rubicon_test.go +++ b/adapters/rubicon/rubicon_test.go @@ -1,17 +1,27 @@ package rubicon import ( + "bytes" + "context" "encoding/json" "errors" + "fmt" + "io/ioutil" "net/http" + "net/http/httptest" "strconv" + "strings" "testing" + "time" "github.com/prebid/prebid-server/adapters" "github.com/prebid/prebid-server/adapters/adapterstest" + "github.com/prebid/prebid-server/cache/dummycache" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/pbs" + "github.com/prebid/prebid-server/usersync" "github.com/buger/jsonparser" "github.com/mxmCherry/openrtb/v15/openrtb2" @@ -38,17 +48,291 @@ type rubiSetNetworkIdTestScenario struct { isNetworkIdSet bool } +type rubiTagInfo struct { + code string + zoneID int + bid float64 + content string + adServerTargeting map[string]string + mediaType string +} + type rubiBidInfo struct { - domain string - page string - deviceIP string - deviceUA string - buyerUID string - devicePxRatio float64 + domain string + page string + accountID int + siteID int + tags []rubiTagInfo + deviceIP string + deviceUA string + buyerUID string + xapiuser string + xapipass string + delay time.Duration + visitorTargeting string + inventoryTargeting string + sdkVersion string + sdkPlatform string + sdkSource string + devicePxRatio float64 } var rubidata rubiBidInfo +func DummyRubiconServer(w http.ResponseWriter, r *http.Request) { + defer func() { + err := r.Body.Close() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + }() + + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + var breq openrtb2.BidRequest + err = json.Unmarshal(body, &breq) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + if len(breq.Imp) > 1 { + http.Error(w, "Rubicon adapter only supports one Imp per request", http.StatusInternalServerError) + return + } + imp := breq.Imp[0] + var rix rubiconImpExt + err = json.Unmarshal(imp.Ext, &rix) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + impTargetingString, _ := json.Marshal(&rix.RP.Target) + if string(impTargetingString) != rubidata.inventoryTargeting { + http.Error(w, fmt.Sprintf("Inventory FPD targeting '%s' doesn't match '%s'", string(impTargetingString), rubidata.inventoryTargeting), http.StatusInternalServerError) + return + } + if rix.RP.Track.Mint != "prebid" { + http.Error(w, fmt.Sprintf("Track mint '%s' doesn't match '%s'", rix.RP.Track.Mint, "prebid"), http.StatusInternalServerError) + return + } + mintVersionString := rubidata.sdkSource + "_" + rubidata.sdkPlatform + "_" + rubidata.sdkVersion + if rix.RP.Track.MintVersion != mintVersionString { + http.Error(w, fmt.Sprintf("Track mint version '%s' doesn't match '%s'", rix.RP.Track.MintVersion, mintVersionString), http.StatusInternalServerError) + return + } + + ix := -1 + + for i, tag := range rubidata.tags { + if rix.RP.ZoneID == tag.zoneID { + ix = i + } + } + if ix == -1 { + http.Error(w, fmt.Sprintf("Zone %d not found", rix.RP.ZoneID), http.StatusInternalServerError) + return + } + + resp := openrtb2.BidResponse{ + ID: "test-response-id", + BidID: "test-bid-id", + Cur: "USD", + SeatBid: []openrtb2.SeatBid{ + { + Seat: "RUBICON", + Bid: make([]openrtb2.Bid, 2), + }, + }, + } + + if imp.Banner != nil { + var bix rubiconBannerExt + err = json.Unmarshal(imp.Banner.Ext, &bix) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if bix.RP.SizeID != 15 { // 300x250 + http.Error(w, fmt.Sprintf("Primary size ID isn't 15"), http.StatusInternalServerError) + return + } + if len(bix.RP.AltSizeIDs) != 1 || bix.RP.AltSizeIDs[0] != 10 { // 300x600 + http.Error(w, fmt.Sprintf("Alt size ID isn't 10"), http.StatusInternalServerError) + return + } + if bix.RP.MIME != "text/html" { + http.Error(w, fmt.Sprintf("MIME isn't text/html"), http.StatusInternalServerError) + return + } + } + + if imp.Video != nil { + var vix rubiconVideoExt + err = json.Unmarshal(imp.Video.Ext, &vix) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if len(imp.Video.MIMEs) == 0 { + http.Error(w, fmt.Sprintf("Empty imp.video.mimes array"), http.StatusInternalServerError) + return + } + if len(imp.Video.Protocols) == 0 { + http.Error(w, fmt.Sprintf("Empty imp.video.protocols array"), http.StatusInternalServerError) + return + } + for _, protocol := range imp.Video.Protocols { + if protocol < 1 || protocol > 8 { + http.Error(w, fmt.Sprintf("Invalid video protocol %d", protocol), http.StatusInternalServerError) + return + } + } + } + + targeting := "{\"rp\":{\"targeting\":[{\"key\":\"key1\",\"values\":[\"value1\"]},{\"key\":\"key2\",\"values\":[\"value2\"]}]}}" + rawTargeting := json.RawMessage(targeting) + + resp.SeatBid[0].Bid[0] = openrtb2.Bid{ + ID: "random-id", + ImpID: imp.ID, + Price: rubidata.tags[ix].bid, + AdM: rubidata.tags[ix].content, + Ext: rawTargeting, + } + + if breq.Site == nil { + http.Error(w, fmt.Sprintf("No site object sent"), http.StatusInternalServerError) + return + } + if breq.Site.Domain != rubidata.domain { + http.Error(w, fmt.Sprintf("Domain '%s' doesn't match '%s", breq.Site.Domain, rubidata.domain), http.StatusInternalServerError) + return + } + if breq.Site.Page != rubidata.page { + http.Error(w, fmt.Sprintf("Page '%s' doesn't match '%s", breq.Site.Page, rubidata.page), http.StatusInternalServerError) + return + } + var rsx rubiconSiteExt + err = json.Unmarshal(breq.Site.Ext, &rsx) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if rsx.RP.SiteID != rubidata.siteID { + http.Error(w, fmt.Sprintf("SiteID '%d' doesn't match '%d", rsx.RP.SiteID, rubidata.siteID), http.StatusInternalServerError) + return + } + if breq.Site.Publisher == nil { + http.Error(w, fmt.Sprintf("No site.publisher object sent"), http.StatusInternalServerError) + return + } + var rpx rubiconPubExt + err = json.Unmarshal(breq.Site.Publisher.Ext, &rpx) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if rpx.RP.AccountID != rubidata.accountID { + http.Error(w, fmt.Sprintf("AccountID '%d' doesn't match '%d'", rpx.RP.AccountID, rubidata.accountID), http.StatusInternalServerError) + return + } + if breq.Device.UA != rubidata.deviceUA { + http.Error(w, fmt.Sprintf("UA '%s' doesn't match '%s'", breq.Device.UA, rubidata.deviceUA), http.StatusInternalServerError) + return + } + if breq.Device.IP != rubidata.deviceIP { + http.Error(w, fmt.Sprintf("IP '%s' doesn't match '%s'", breq.Device.IP, rubidata.deviceIP), http.StatusInternalServerError) + return + } + if breq.Device.PxRatio != rubidata.devicePxRatio { + http.Error(w, fmt.Sprintf("Pixel ratio '%f' doesn't match '%f'", breq.Device.PxRatio, rubidata.devicePxRatio), http.StatusInternalServerError) + return + } + if breq.User.BuyerUID != rubidata.buyerUID { + http.Error(w, fmt.Sprintf("User ID '%s' doesn't match '%s'", breq.User.BuyerUID, rubidata.buyerUID), http.StatusInternalServerError) + return + } + + var rux rubiconUserExt + err = json.Unmarshal(breq.User.Ext, &rux) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + userTargetingString, _ := json.Marshal(&rux.RP.Target) + if string(userTargetingString) != rubidata.visitorTargeting { + http.Error(w, fmt.Sprintf("User FPD targeting '%s' doesn't match '%s'", string(userTargetingString), rubidata.visitorTargeting), http.StatusInternalServerError) + return + } + + if rubidata.delay > 0 { + <-time.After(rubidata.delay) + } + + js, err := json.Marshal(resp) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.Write(js) +} + +func TestRubiconBasicResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(DummyRubiconServer)) + defer server.Close() + + an, ctx, pbReq := CreatePrebidRequest(server, t) + + bids, err := an.Call(ctx, pbReq, pbReq.Bidders[0]) + assert.Nil(t, err, "Should not have gotten an error: %v", err) + assert.Equal(t, 2, len(bids), "Received %d bids instead of 3", len(bids)) + + for _, bid := range bids { + matched := false + for _, tag := range rubidata.tags { + if bid.AdUnitCode == tag.code { + matched = true + + assert.Equal(t, "rubicon", bid.BidderCode, "Incorrect BidderCode '%s'", bid.BidderCode) + + assert.Equal(t, tag.bid, bid.Price, "Incorrect bid price '%.2f' expected '%.2f'", bid.Price, tag.bid) + + assert.Equal(t, tag.content, bid.Adm, "Incorrect bid markup '%s' expected '%s'", bid.Adm, tag.content) + + assert.Equal(t, bid.AdServerTargeting, tag.adServerTargeting, + "Incorrect targeting '%+v' expected '%+v'", bid.AdServerTargeting, tag.adServerTargeting) + + assert.Equal(t, tag.mediaType, bid.CreativeMediaType, "Incorrect media type '%s' expected '%s'", bid.CreativeMediaType, tag.mediaType) + } + } + assert.True(t, matched, "Received bid for unknown ad unit '%s'", bid.AdUnitCode) + } + + // same test but with request timing out + rubidata.delay = 20 * time.Millisecond + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) + defer cancel() + + bids, err = an.Call(ctx, pbReq, pbReq.Bidders[0]) + assert.NotNil(t, err, "Should have gotten a timeout error: %v", err) +} + +func TestRubiconUserSyncInfo(t *testing.T) { + conf := *adapters.DefaultHTTPAdapterConfig + an := NewRubiconLegacyAdapter(&conf, "uri", "xuser", "xpass", "pbs-test-tracker") + + assert.Equal(t, "rubicon", an.Name(), "Name '%s' != 'rubicon'", an.Name()) + + assert.False(t, an.SkipNoCookies(), "SkipNoCookies should be false") +} + func getTestSizes() map[int]openrtb2.Format { return map[int]openrtb2.Format{ 15: {W: 300, H: 250}, @@ -419,6 +703,368 @@ func (m mockCurrencyConversion) GetRates() *map[string]map[string]float64 { return args.Get(0).(*map[string]map[string]float64) } +func TestNoContentResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + an, ctx, pbReq := CreatePrebidRequest(server, t) + bids, err := an.Call(ctx, pbReq, pbReq.Bidders[0]) + + assert.Empty(t, bids, "Length of bids should be 0 instead of: %v", len(bids)) + + assert.Equal(t, 204, pbReq.Bidders[0].Debug[0].StatusCode, + "StatusCode should be 204 instead of: %v", pbReq.Bidders[0].Debug[0].StatusCode) + + assert.Nil(t, err, "Should not have gotten an error: %v", err) +} + +func TestNotFoundResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + an, ctx, pbReq := CreatePrebidRequest(server, t) + _, err := an.Call(ctx, pbReq, pbReq.Bidders[0]) + + assert.Equal(t, 404, pbReq.Bidders[0].Debug[0].StatusCode, + "StatusCode should be 404 instead of: %v", pbReq.Bidders[0].Debug[0].StatusCode) + + assert.NotNil(t, err, "Should have gotten an error: %v", err) + + assert.True(t, strings.HasPrefix(err.Error(), "HTTP status 404"), + "Should start with 'HTTP status' instead of: %v", err.Error()) +} + +func TestWrongFormatResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("This is text.")) + })) + defer server.Close() + + an, ctx, pbReq := CreatePrebidRequest(server, t) + _, err := an.Call(ctx, pbReq, pbReq.Bidders[0]) + + assert.Equal(t, 200, pbReq.Bidders[0].Debug[0].StatusCode, + "StatusCode should be 200 instead of: %v", pbReq.Bidders[0].Debug[0].StatusCode) + + assert.NotNil(t, err, "Should have gotten an error: %v", err) + + assert.True(t, strings.HasPrefix(err.Error(), "invalid character"), + "Should start with 'invalid character' instead of: %v", err) +} + +func TestZeroSeatBidResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resp := openrtb2.BidResponse{ + ID: "test-response-id", + BidID: "test-bid-id", + Cur: "USD", + SeatBid: []openrtb2.SeatBid{}, + } + js, _ := json.Marshal(resp) + w.Write(js) + })) + defer server.Close() + + an, ctx, pbReq := CreatePrebidRequest(server, t) + bids, err := an.Call(ctx, pbReq, pbReq.Bidders[0]) + + assert.Empty(t, bids, "Length of bids should be 0 instead of: %v", len(bids)) + + assert.Nil(t, err, "Should not have gotten an error: %v", err) +} + +func TestEmptyBidResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resp := openrtb2.BidResponse{ + ID: "test-response-id", + BidID: "test-bid-id", + Cur: "USD", + SeatBid: []openrtb2.SeatBid{ + { + Seat: "RUBICON", + Bid: make([]openrtb2.Bid, 0), + }, + }, + } + js, _ := json.Marshal(resp) + w.Write(js) + })) + defer server.Close() + + an, ctx, pbReq := CreatePrebidRequest(server, t) + bids, err := an.Call(ctx, pbReq, pbReq.Bidders[0]) + + assert.Empty(t, bids, "Length of bids should be 0 instead of: %v", len(bids)) + + assert.Nil(t, err, "Should not have gotten an error: %v", err) +} + +func TestWrongBidIdResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resp := openrtb2.BidResponse{ + ID: "test-response-id", + BidID: "test-bid-id", + Cur: "USD", + SeatBid: []openrtb2.SeatBid{ + { + Seat: "RUBICON", + Bid: make([]openrtb2.Bid, 2), + }, + }, + } + resp.SeatBid[0].Bid[0] = openrtb2.Bid{ + ID: "random-id", + ImpID: "zma", + Price: 1.67, + AdM: "zma", + Ext: json.RawMessage("{\"rp\":{\"targeting\":[{\"key\":\"key1\",\"values\":[\"value1\"]},{\"key\":\"key2\",\"values\":[\"value2\"]}]}}"), + } + js, _ := json.Marshal(resp) + w.Write(js) + })) + defer server.Close() + + an, ctx, pbReq := CreatePrebidRequest(server, t) + bids, err := an.Call(ctx, pbReq, pbReq.Bidders[0]) + + assert.Empty(t, bids, "Length of bids should be 0 instead of: %v", len(bids)) + + assert.NotNil(t, err, "Should not have gotten an error: %v", err) + + assert.True(t, strings.HasPrefix(err.Error(), "Unknown ad unit code"), + "Should start with 'Unknown ad unit code' instead of: %v", err) +} + +func TestZeroPriceBidResponse(t *testing.T) { + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resp := openrtb2.BidResponse{ + ID: "test-response-id", + BidID: "test-bid-id", + Cur: "USD", + SeatBid: []openrtb2.SeatBid{ + { + Seat: "RUBICON", + Bid: make([]openrtb2.Bid, 1), + }, + }, + } + resp.SeatBid[0].Bid[0] = openrtb2.Bid{ + ID: "test-bid-id", + ImpID: "first-tag", + Price: 0, + AdM: "zma", + Ext: json.RawMessage("{\"rp\":{\"targeting\":[{\"key\":\"key1\",\"values\":[\"value1\"]},{\"key\":\"key2\",\"values\":[\"value2\"]}]}}"), + } + js, _ := json.Marshal(resp) + w.Write(js) + })) + defer server.Close() + + an, ctx, pbReq := CreatePrebidRequest(server, t) + b, err := an.Call(ctx, pbReq, pbReq.Bidders[0]) + + assert.Nil(t, b, "\n\n\n0 price bids are being included %d, err : %v", len(b), err) +} + +func TestDifferentRequest(t *testing.T) { + SIZE_ID := getTestSizes() + server := httptest.NewServer(http.HandlerFunc(DummyRubiconServer)) + defer server.Close() + + an, ctx, pbReq := CreatePrebidRequest(server, t) + + // test app not nil + pbReq.App = &openrtb2.App{ + ID: "com.test", + Name: "testApp", + } + + _, err := an.Call(ctx, pbReq, pbReq.Bidders[0]) + assert.NotNil(t, err, "Should have gotten an error: %v", err) + + // set app back to normal + pbReq.App = nil + + // test video media type + pbReq.Bidders[0].AdUnits[0].MediaTypes = []pbs.MediaType{pbs.MEDIA_TYPE_VIDEO} + _, err = an.Call(ctx, pbReq, pbReq.Bidders[0]) + assert.NotNil(t, err, "Should have gotten an error: %v", err) + + // set media back to normal + pbReq.Bidders[0].AdUnits[0].MediaTypes = []pbs.MediaType{pbs.MEDIA_TYPE_BANNER} + + // test wrong params + pbReq.Bidders[0].AdUnits[0].Params = json.RawMessage(fmt.Sprintf("{\"zoneId\": %s, \"siteId\": %d, \"visitor\": %s, \"inventory\": %s}", "zma", rubidata.siteID, rubidata.visitorTargeting, rubidata.inventoryTargeting)) + _, err = an.Call(ctx, pbReq, pbReq.Bidders[0]) + assert.NotNil(t, err, "Should have gotten an error: %v", err) + + // set params back to normal + pbReq.Bidders[0].AdUnits[0].Params = json.RawMessage(fmt.Sprintf("{\"zoneId\": %d, \"siteId\": %d, \"accountId\": %d, \"visitor\": %s, \"inventory\": %s}", 8394, rubidata.siteID, rubidata.accountID, rubidata.visitorTargeting, rubidata.inventoryTargeting)) + + // test invalid size + pbReq.Bidders[0].AdUnits[0].Sizes = []openrtb2.Format{ + { + W: 2222, + H: 333, + }, + } + pbReq.Bidders[0].AdUnits[1].Sizes = []openrtb2.Format{ + { + W: 222, + H: 3333, + }, + { + W: 350, + H: 270, + }, + } + pbReq.Bidders[0].AdUnits = pbReq.Bidders[0].AdUnits[:len(pbReq.Bidders[0].AdUnits)-1] + b, err := an.Call(ctx, pbReq, pbReq.Bidders[0]) + assert.NotNil(t, err, "Should have gotten an error: %v", err) + + pbReq.Bidders[0].AdUnits[1].Sizes = []openrtb2.Format{ + { + W: 222, + H: 3333, + }, + SIZE_ID[10], + SIZE_ID[15], + } + b, err = an.Call(ctx, pbReq, pbReq.Bidders[0]) + assert.Nil(t, err, "Should have not gotten an error: %v", err) + + assert.Equal(t, 1, len(b), + "Filtering bids based on ad unit sizes failed. Got %d bids instead of 1, error = %v", len(b), err) +} + +func CreatePrebidRequest(server *httptest.Server, t *testing.T) (an *RubiconAdapter, ctx context.Context, pbReq *pbs.PBSRequest) { + SIZE_ID := getTestSizes() + rubidata = rubiBidInfo{ + domain: "nytimes.com", + page: "https://www.nytimes.com/2017/05/04/movies/guardians-of-the-galaxy-2-review-chris-pratt.html?hpw&rref=movies&action=click&pgtype=Homepage&module=well-region®ion=bottom-well&WT.nav=bottom-well&_r=0", + accountID: 7891, + siteID: 283282, + tags: make([]rubiTagInfo, 3), + deviceIP: "25.91.96.36", + deviceUA: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.1 Safari/603.1.30", + buyerUID: "need-an-actual-rp-id", + visitorTargeting: "[\"v1\",\"v2\"]", + inventoryTargeting: "[\"i1\",\"i2\"]", + sdkVersion: "2.0.0", + sdkPlatform: "iOS", + sdkSource: "some-sdk", + devicePxRatio: 4.0, + } + + targeting := make(map[string]string, 2) + targeting["key1"] = "value1" + targeting["key2"] = "value2" + + rubidata.tags[0] = rubiTagInfo{ + code: "first-tag", + zoneID: 8394, + bid: 1.67, + adServerTargeting: targeting, + mediaType: "banner", + } + rubidata.tags[1] = rubiTagInfo{ + code: "second-tag", + zoneID: 8395, + bid: 3.22, + adServerTargeting: targeting, + mediaType: "banner", + } + rubidata.tags[2] = rubiTagInfo{ + code: "video-tag", + zoneID: 7780, + bid: 23.12, + adServerTargeting: targeting, + mediaType: "video", + } + + conf := *adapters.DefaultHTTPAdapterConfig + an = NewRubiconLegacyAdapter(&conf, "uri", rubidata.xapiuser, rubidata.xapipass, "pbs-test-tracker") + an.URI = server.URL + + pbin := pbs.PBSRequest{ + AdUnits: make([]pbs.AdUnit, 3), + Device: &openrtb2.Device{PxRatio: rubidata.devicePxRatio}, + SDK: &pbs.SDK{Source: rubidata.sdkSource, Platform: rubidata.sdkPlatform, Version: rubidata.sdkVersion}, + } + + for i, tag := range rubidata.tags { + pbin.AdUnits[i] = pbs.AdUnit{ + Code: tag.code, + MediaTypes: []string{tag.mediaType}, + Sizes: []openrtb2.Format{ + SIZE_ID[10], + SIZE_ID[15], + }, + Bids: []pbs.Bids{ + { + BidderCode: "rubicon", + BidID: fmt.Sprintf("random-id-from-pbjs-%d", i), + Params: json.RawMessage(fmt.Sprintf("{\"zoneId\": %d, \"siteId\": %d, \"accountId\": %d, \"visitor\": %s, \"inventory\": %s}", tag.zoneID, rubidata.siteID, rubidata.accountID, rubidata.visitorTargeting, rubidata.inventoryTargeting)), + }, + }, + } + if tag.mediaType == "video" { + pbin.AdUnits[i].Video = pbs.PBSVideo{ + Mimes: []string{"video/mp4"}, + Minduration: 15, + Maxduration: 30, + Startdelay: 5, + Skippable: 0, + PlaybackMethod: 1, + Protocols: []int8{1, 2, 3, 4, 5}, + } + } + } + + body := new(bytes.Buffer) + err := json.NewEncoder(body).Encode(pbin) + if err != nil { + t.Fatalf("Json encoding failed: %v", err) + } + + req := httptest.NewRequest("POST", server.URL, body) + req.Header.Add("Referer", rubidata.page) + req.Header.Add("User-Agent", rubidata.deviceUA) + req.Header.Add("X-Real-IP", rubidata.deviceIP) + + pc := usersync.ParseCookieFromRequest(req, &config.HostCookie{}) + pc.TrySync("rubicon", rubidata.buyerUID) + fakewriter := httptest.NewRecorder() + + pc.SetCookieOnResponse(fakewriter, false, &config.HostCookie{Domain: ""}, 90*24*time.Hour) + req.Header.Add("Cookie", fakewriter.Header().Get("Set-Cookie")) + + cacheClient, _ := dummycache.New() + hcc := config.HostCookie{} + + pbReq, err = pbs.ParsePBSRequest(req, &config.AuctionTimeouts{ + Default: 2000, + Max: 2000, + }, cacheClient, &hcc) + pbReq.IsDebug = true + + assert.Nil(t, err, "ParsePBSRequest failed: %v", err) + + assert.Equal(t, 1, len(pbReq.Bidders), + "ParsePBSRequest returned %d bidders instead of 1", len(pbReq.Bidders)) + + assert.Equal(t, "rubicon", pbReq.Bidders[0].BidderCode, + "ParsePBSRequest returned invalid bidder") + + ctx = context.TODO() + return +} + func TestOpenRTBRequest(t *testing.T) { SIZE_ID := getTestSizes() bidder := new(RubiconAdapter) diff --git a/adapters/sharethrough/utils_test.go b/adapters/sharethrough/utils_test.go index 70e97947880..b842cf0b0c0 100644 --- a/adapters/sharethrough/utils_test.go +++ b/adapters/sharethrough/utils_test.go @@ -151,6 +151,10 @@ type userAgentTest struct { expected bool } +type userAgentFailureTest struct { + input string +} + func runUserAgentTests(tests map[string]userAgentTest, fn func(string) bool, t *testing.T) { for testName, test := range tests { t.Logf("Test case: %s\n", testName) diff --git a/adapters/sonobi/sonobi.go b/adapters/sonobi/sonobi.go index 0ff71cdb0e5..690d5f59f67 100644 --- a/adapters/sonobi/sonobi.go +++ b/adapters/sonobi/sonobi.go @@ -25,6 +25,10 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters return bidder, nil } +type sonobiParams struct { + TagID string `json:"TagID"` +} + // MakeRequests Makes the OpenRTB request payload func (a *SonobiAdapter) MakeRequests(request *openrtb2.BidRequest, reqInfo *adapters.ExtraRequestInfo) ([]*adapters.RequestData, []error) { var errs []error @@ -148,3 +152,9 @@ func getMediaTypeForImp(impID string, imps []openrtb2.Imp) (openrtb_ext.BidType, Message: fmt.Sprintf("Failed to find impression \"%s\" ", impID), } } + +func addHeaderIfNonEmpty(headers http.Header, headerName string, headerValue string) { + if len(headerValue) > 0 { + headers.Add(headerName, headerValue) + } +} diff --git a/adapters/sovrn/sovrn.go b/adapters/sovrn/sovrn.go index 98264ce3a1b..40969d3638e 100644 --- a/adapters/sovrn/sovrn.go +++ b/adapters/sovrn/sovrn.go @@ -1,23 +1,176 @@ package sovrn import ( + "bytes" + "context" "encoding/json" "fmt" + "io/ioutil" "net/http" "net/url" + "sort" "strconv" "strings" + "github.com/mxmCherry/openrtb/v15/openrtb2" "github.com/prebid/prebid-server/adapters" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/openrtb_ext" - - "github.com/mxmCherry/openrtb/v15/openrtb2" + "github.com/prebid/prebid-server/pbs" + "golang.org/x/net/context/ctxhttp" ) type SovrnAdapter struct { - URI string + http *adapters.HTTPAdapter + URI string +} + +// Name - export adapter name */ +func (s *SovrnAdapter) Name() string { + return "sovrn" +} + +// FamilyName used for cookies and such +func (s *SovrnAdapter) FamilyName() string { + return "sovrn" +} + +func (s *SovrnAdapter) SkipNoCookies() bool { + return false +} + +// Call send bid requests to sovrn and receive responses +func (s *SovrnAdapter) Call(ctx context.Context, req *pbs.PBSRequest, bidder *pbs.PBSBidder) (pbs.PBSBidSlice, error) { + supportedMediaTypes := []pbs.MediaType{pbs.MEDIA_TYPE_BANNER} + sReq, err := adapters.MakeOpenRTBGeneric(req, bidder, s.FamilyName(), supportedMediaTypes) + + if err != nil { + return nil, err + } + + sovrnReq := openrtb2.BidRequest{ + ID: sReq.ID, + Imp: sReq.Imp, + Site: sReq.Site, + User: sReq.User, + Regs: sReq.Regs, + } + + // add tag ids to impressions + for i, unit := range bidder.AdUnits { + var params openrtb_ext.ExtImpSovrn + err = json.Unmarshal(unit.Params, ¶ms) + if err != nil { + return nil, err + } + + // Fixes some segfaults. Since this is legacy code, I'm not looking into it too deeply + if len(sovrnReq.Imp) <= i { + break + } + sovrnReq.Imp[i].TagID = getTagid(params) + } + + reqJSON, err := json.Marshal(sovrnReq) + if err != nil { + return nil, err + } + + debug := &pbs.BidderDebug{ + RequestURI: s.URI, + } + + httpReq, _ := http.NewRequest("POST", s.URI, bytes.NewReader(reqJSON)) + httpReq.Header.Set("Content-Type", "application/json") + if sReq.Device != nil { + addHeaderIfNonEmpty(httpReq.Header, "User-Agent", sReq.Device.UA) + addHeaderIfNonEmpty(httpReq.Header, "X-Forwarded-For", sReq.Device.IP) + addHeaderIfNonEmpty(httpReq.Header, "Accept-Language", sReq.Device.Language) + if sReq.Device.DNT != nil { + addHeaderIfNonEmpty(httpReq.Header, "DNT", strconv.Itoa(int(*sReq.Device.DNT))) + } + } + if sReq.User != nil { + userID := strings.TrimSpace(sReq.User.BuyerUID) + if len(userID) > 0 { + httpReq.AddCookie(&http.Cookie{Name: "ljt_reader", Value: userID}) + } + } + sResp, err := ctxhttp.Do(ctx, s.http.Client, httpReq) + if err != nil { + return nil, err + } + defer sResp.Body.Close() + + debug.StatusCode = sResp.StatusCode + + if sResp.StatusCode == http.StatusNoContent { + return nil, nil + } + + body, err := ioutil.ReadAll(sResp.Body) + if err != nil { + return nil, err + } + responseBody := string(body) + + if sResp.StatusCode == http.StatusBadRequest { + return nil, &errortypes.BadInput{ + Message: fmt.Sprintf("HTTP status %d; body: %s", sResp.StatusCode, responseBody), + } + } + + if sResp.StatusCode != http.StatusOK { + return nil, &errortypes.BadServerResponse{ + Message: fmt.Sprintf("HTTP status %d; body: %s", sResp.StatusCode, responseBody), + } + } + + if req.IsDebug { + debug.RequestBody = string(reqJSON) + bidder.Debug = append(bidder.Debug, debug) + debug.ResponseBody = responseBody + } + + var bidResp openrtb2.BidResponse + err = json.Unmarshal(body, &bidResp) + if err != nil { + return nil, &errortypes.BadServerResponse{ + Message: err.Error(), + } + } + + bids := make(pbs.PBSBidSlice, 0) + + for _, sb := range bidResp.SeatBid { + for _, bid := range sb.Bid { + bidID := bidder.LookupBidID(bid.ImpID) + if bidID == "" { + return nil, &errortypes.BadServerResponse{ + Message: fmt.Sprintf("Unknown ad unit code '%s'", bid.ImpID), + } + } + + adm, _ := url.QueryUnescape(bid.AdM) + pbid := pbs.PBSBid{ + BidID: bidID, + AdUnitCode: bid.ImpID, + BidderCode: bidder.BidderCode, + Price: bid.Price, + Adm: adm, + Creative_id: bid.CrID, + Width: bid.W, + Height: bid.H, + DealId: bid.DealID, + NURL: bid.NURL, + } + bids = append(bids, &pbid) + } + } + + sort.Sort(bids) + return bids, nil } func (s *SovrnAdapter) MakeRequests(request *openrtb2.BidRequest, reqInfo *adapters.ExtraRequestInfo) ([]*adapters.RequestData, []error) { @@ -150,6 +303,14 @@ func getTagid(sovrnExt openrtb_ext.ExtImpSovrn) string { } } +// NewSovrnLegacyAdapter create a new SovrnAdapter instance +func NewSovrnLegacyAdapter(config *adapters.HTTPAdapterConfig, endpoint string) *SovrnAdapter { + return &SovrnAdapter{ + http: adapters.NewHTTPAdapter(config), + URI: endpoint, + } +} + // Builder builds a new instance of the Sovrn adapter for the given bidder with the given config. func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters.Bidder, error) { bidder := &SovrnAdapter{ diff --git a/adapters/sovrn/sovrn_test.go b/adapters/sovrn/sovrn_test.go index 49ed52844f3..407c505437a 100644 --- a/adapters/sovrn/sovrn_test.go +++ b/adapters/sovrn/sovrn_test.go @@ -1,11 +1,28 @@ package sovrn import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http/httptest" "testing" + "github.com/mxmCherry/openrtb/v15/openrtb2" + "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/pbs" + "github.com/prebid/prebid-server/usersync" + + "context" + "net/http" + + "strconv" + "time" + + "github.com/prebid/prebid-server/adapters" "github.com/prebid/prebid-server/adapters/adapterstest" + "github.com/prebid/prebid-server/cache/dummycache" "github.com/prebid/prebid-server/config" - "github.com/prebid/prebid-server/openrtb_ext" ) func TestJsonSamples(t *testing.T) { @@ -18,3 +35,262 @@ func TestJsonSamples(t *testing.T) { adapterstest.RunJSONBidderTest(t, "sovrntest", bidder) } + +// ---------------------------------------------------------------------------- +// Code below this line tests the legacy, non-openrtb code flow. It can be deleted after we +// clean up the existing code and make everything openrtb2. + +var testSovrnUserId = "SovrnUser123" +var testUserAgent = "user-agent-test" +var testUrl = "http://news.pub/topnews" +var testIp = "123.123.123.123" + +func TestSovrnAdapterNames(t *testing.T) { + adapter := NewSovrnLegacyAdapter(adapters.DefaultHTTPAdapterConfig, "http://sovrn/rtb/bid") + adapterstest.VerifyStringValue(adapter.Name(), "sovrn", t) + adapterstest.VerifyStringValue(adapter.FamilyName(), "sovrn", t) +} + +func TestSovrnAdapter_SkipNoCookies(t *testing.T) { + adapter := NewSovrnLegacyAdapter(adapters.DefaultHTTPAdapterConfig, "http://sovrn/rtb/bid") + adapterstest.VerifyBoolValue(adapter.SkipNoCookies(), false, t) +} + +func TestSovrnOpenRtbRequest(t *testing.T) { + service := CreateSovrnService(adapterstest.BidOnTags("")) + server := service.Server + ctx := context.Background() + req := SampleSovrnRequest(1, t) + bidder := req.Bidders[0] + adapter := NewSovrnLegacyAdapter(adapters.DefaultHTTPAdapterConfig, server.URL) + adapter.Call(ctx, req, bidder) + + adapterstest.VerifyIntValue(len(service.LastBidRequest.Imp), 1, t) + adapterstest.VerifyStringValue(service.LastBidRequest.Imp[0].TagID, "123456", t) + adapterstest.VerifyBannerSize(service.LastBidRequest.Imp[0].Banner, 728, 90, t) + checkHttpRequest(*service.LastHttpRequest, t) +} + +func TestSovrnBiddingBehavior(t *testing.T) { + service := CreateSovrnService(adapterstest.BidOnTags("123456")) + server := service.Server + ctx := context.TODO() + req := SampleSovrnRequest(1, t) + bidder := req.Bidders[0] + adapter := NewSovrnLegacyAdapter(adapters.DefaultHTTPAdapterConfig, server.URL) + bids, _ := adapter.Call(ctx, req, bidder) + + adapterstest.VerifyIntValue(len(bids), 1, t) + adapterstest.VerifyStringValue(bids[0].AdUnitCode, "div-adunit-1", t) + adapterstest.VerifyStringValue(bids[0].BidderCode, "sovrn", t) + adapterstest.VerifyStringValue(bids[0].Adm, "
This is an Ad
", t) + adapterstest.VerifyStringValue(bids[0].Creative_id, "Cr-234", t) + adapterstest.VerifyIntValue(int(bids[0].Width), 728, t) + adapterstest.VerifyIntValue(int(bids[0].Height), 90, t) + adapterstest.VerifyIntValue(int(bids[0].Price*100), 210, t) + checkHttpRequest(*service.LastHttpRequest, t) +} + +/** + * Verify bidding behavior on multiple impressions, some impressions make a bid + */ +func TestSovrntMultiImpPartialBidding(t *testing.T) { + // setup server endpoint to return bid. + service := CreateSovrnService(adapterstest.BidOnTags("123456")) + server := service.Server + ctx := context.TODO() + req := SampleSovrnRequest(2, t) + bidder := req.Bidders[0] + adapter := NewSovrnLegacyAdapter(adapters.DefaultHTTPAdapterConfig, server.URL) + bids, _ := adapter.Call(ctx, req, bidder) + // two impressions sent. + // number of bids should be 1 + adapterstest.VerifyIntValue(len(service.LastBidRequest.Imp), 2, t) + adapterstest.VerifyIntValue(len(bids), 1, t) + adapterstest.VerifyStringValue(bids[0].AdUnitCode, "div-adunit-1", t) + checkHttpRequest(*service.LastHttpRequest, t) +} + +/** + * Verify bidding behavior on multiple impressions, all impressions passed back. + */ +func TestSovrnMultiImpAllBid(t *testing.T) { + // setup server endpoint to return bid. + service := CreateSovrnService(adapterstest.BidOnTags("123456,123457")) + server := service.Server + ctx := context.TODO() + req := SampleSovrnRequest(2, t) + bidder := req.Bidders[0] + adapter := NewSovrnLegacyAdapter(adapters.DefaultHTTPAdapterConfig, server.URL) + bids, _ := adapter.Call(ctx, req, bidder) + // two impressions sent. + // number of bids should be 1 + adapterstest.VerifyIntValue(len(service.LastBidRequest.Imp), 2, t) + adapterstest.VerifyIntValue(len(bids), 2, t) + adapterstest.VerifyStringValue(bids[0].AdUnitCode, "div-adunit-1", t) + adapterstest.VerifyStringValue(bids[1].AdUnitCode, "div-adunit-2", t) + checkHttpRequest(*service.LastHttpRequest, t) +} + +func checkHttpRequest(req http.Request, t *testing.T) { + adapterstest.VerifyStringValue(req.Header.Get("Accept-Language"), "murican", t) + var cookie, _ = req.Cookie("ljt_reader") + adapterstest.VerifyStringValue((*cookie).Value, testSovrnUserId, t) + adapterstest.VerifyStringValue(req.Header.Get("User-Agent"), testUserAgent, t) + adapterstest.VerifyStringValue(req.Header.Get("Content-Type"), "application/json", t) + adapterstest.VerifyStringValue(req.Header.Get("X-Forwarded-For"), testIp, t) + adapterstest.VerifyStringValue(req.Header.Get("DNT"), "0", t) +} + +func SampleSovrnRequest(numberOfImpressions int, t *testing.T) *pbs.PBSRequest { + dnt := int8(0) + device := openrtb2.Device{ + Language: "murican", + DNT: &dnt, + } + + user := openrtb2.User{ + ID: testSovrnUserId, + } + + req := pbs.PBSRequest{ + AccountID: "1", + AdUnits: make([]pbs.AdUnit, 2), + Device: &device, + User: &user, + } + + tagID := 123456 + + for i := 0; i < numberOfImpressions; i++ { + req.AdUnits[i] = pbs.AdUnit{ + Code: fmt.Sprintf("div-adunit-%d", i+1), + Sizes: []openrtb2.Format{ + { + W: 728, + H: 90, + }, + }, + Bids: []pbs.Bids{ + { + BidderCode: "sovrn", + BidID: fmt.Sprintf("Bid-%d", i+1), + Params: json.RawMessage(fmt.Sprintf("{\"tagid\": \"%s\" }", strconv.Itoa(tagID+i))), + }, + }, + } + + } + + body := new(bytes.Buffer) + err := json.NewEncoder(body).Encode(req) + if err != nil { + t.Fatalf("Error when serializing request") + } + + httpReq := httptest.NewRequest("POST", CreateSovrnService(adapterstest.BidOnTags("")).Server.URL, body) + httpReq.Header.Add("Referer", testUrl) + httpReq.Header.Add("User-Agent", testUserAgent) + httpReq.Header.Add("X-Forwarded-For", testIp) + pc := usersync.ParseCookieFromRequest(httpReq, &config.HostCookie{}) + pc.TrySync("sovrn", testSovrnUserId) + fakewriter := httptest.NewRecorder() + + pc.SetCookieOnResponse(fakewriter, false, &config.HostCookie{Domain: ""}, 90*24*time.Hour) + httpReq.Header.Add("Cookie", fakewriter.Header().Get("Set-Cookie")) + // parse the http request + cacheClient, _ := dummycache.New() + hcc := config.HostCookie{} + + parsedReq, err := pbs.ParsePBSRequest(httpReq, &config.AuctionTimeouts{ + Default: 2000, + Max: 2000, + }, cacheClient, &hcc) + if err != nil { + t.Fatalf("Error when parsing request: %v", err) + } + return parsedReq + +} + +func TestNoContentResponse(t *testing.T) { + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + ctx := context.TODO() + req := SampleSovrnRequest(1, t) + bidder := req.Bidders[0] + adapter := NewSovrnLegacyAdapter(adapters.DefaultHTTPAdapterConfig, server.URL) + _, err := adapter.Call(ctx, req, bidder) + + if err != nil { + t.Fatalf("Should not have gotten an error: %v", err) + } + +} + +func TestNotFoundResponse(t *testing.T) { + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + ctx := context.TODO() + req := SampleSovrnRequest(1, t) + bidder := req.Bidders[0] + adapter := NewSovrnLegacyAdapter(adapters.DefaultHTTPAdapterConfig, server.URL) + _, err := adapter.Call(ctx, req, bidder) + + adapterstest.VerifyStringValue(err.Error(), "HTTP status 404; body: ", t) + +} + +func CreateSovrnService(tagsToBid map[string]bool) adapterstest.OrtbMockService { + service := adapterstest.OrtbMockService{} + var lastBidRequest openrtb2.BidRequest + var lastHttpReq http.Request + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + lastHttpReq = *r + defer r.Body.Close() + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + var breq openrtb2.BidRequest + err = json.Unmarshal(body, &breq) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + lastBidRequest = breq + var bids []openrtb2.Bid + for i, imp := range breq.Imp { + if tagsToBid[imp.TagID] { + bids = append(bids, adapterstest.SampleBid(imp.Banner.W, imp.Banner.H, imp.ID, i+1)) + } + } + + // serialize the bids to openrtb2.BidResponse + js, _ := json.Marshal(openrtb2.BidResponse{ + SeatBid: []openrtb2.SeatBid{ + { + Bid: bids, + }, + }, + }) + w.Header().Set("Content-Type", "application/json") + w.Write(js) + })) + + service.Server = server + service.LastBidRequest = &lastBidRequest + service.LastHttpRequest = &lastHttpReq + + return service +} diff --git a/analytics/filesystem/file_module.go b/analytics/filesystem/file_module.go index 43853382354..a0721d98a2a 100644 --- a/analytics/filesystem/file_module.go +++ b/analytics/filesystem/file_module.go @@ -102,6 +102,8 @@ func NewFileLogger(filename string) (analytics.PBSAnalyticsModule, error) { } } +type fileAuctionObject analytics.AuctionObject + func jsonifyAuctionObject(ao *analytics.AuctionObject) string { type alias analytics.AuctionObject b, err := json.Marshal(&struct { diff --git a/analytics/pubstack/pubstack_module_test.go b/analytics/pubstack/pubstack_module_test.go index 0e0b3634508..cb8f088d0bf 100644 --- a/analytics/pubstack/pubstack_module_test.go +++ b/analytics/pubstack/pubstack_module_test.go @@ -2,15 +2,59 @@ package pubstack import ( "encoding/json" + "io/ioutil" "net/http" "net/http/httptest" + "os" "testing" "time" + "github.com/mxmCherry/openrtb/v15/openrtb2" "github.com/prebid/prebid-server/analytics" "github.com/stretchr/testify/assert" ) +func loadJSONFromFile() (*analytics.AuctionObject, error) { + req, err := os.Open("mocks/mock_openrtb_request.json") + if err != nil { + return nil, err + } + defer req.Close() + + reqCtn := openrtb2.BidRequest{} + reqPayload, err := ioutil.ReadAll(req) + if err != nil { + return nil, err + } + + err = json.Unmarshal(reqPayload, &reqCtn) + if err != nil { + return nil, err + } + + res, err := os.Open("mocks/mock_openrtb_response.json") + if err != nil { + return nil, err + } + defer res.Close() + + resCtn := openrtb2.BidResponse{} + resPayload, err := ioutil.ReadAll(res) + if err != nil { + return nil, err + } + + err = json.Unmarshal(resPayload, &resCtn) + if err != nil { + return nil, err + } + + return &analytics.AuctionObject{ + Request: &reqCtn, + Response: &resCtn, + }, nil +} + func TestPubstackModuleErrors(t *testing.T) { tests := []struct { description string diff --git a/cache/dummycache/dummycache.go b/cache/dummycache/dummycache.go new file mode 100644 index 00000000000..02fe726d043 --- /dev/null +++ b/cache/dummycache/dummycache.go @@ -0,0 +1,65 @@ +package dummycache + +import ( + "fmt" + + "github.com/prebid/prebid-server/cache" +) + +// Cache dummy config that will echo back results +type Cache struct { + accounts *accountService + config *configService +} + +// New creates new dummy.Cache +func New() (*Cache, error) { + return &Cache{ + accounts: &accountService{}, + config: &configService{}, + }, nil +} + +func (c *Cache) Accounts() cache.AccountsService { + return c.accounts +} +func (c *Cache) Config() cache.ConfigService { + return c.config +} + +// AccountService handles the account information +type accountService struct { +} + +// Get echos back the account +func (s *accountService) Get(id string) (*cache.Account, error) { + return &cache.Account{ + ID: id, + }, nil +} + +// ConfigService not supported, always returns an error +type configService struct { + c string +} + +// Get not supported, always returns an error +func (s *configService) Get(id string) (string, error) { + if s.c == "" { + return s.c, fmt.Errorf("No configuration provided") + } + return s.c, nil +} + +// Set will set a string in memory as the configuration +// this is so we can use it in tests such as pbs/pbsrequest_test.go +// it will ignore the id so this will pass tests +func (s *configService) Set(id, val string) error { + s.c = val + return nil +} + +// Close will always return nil +func (c *Cache) Close() error { + return nil +} diff --git a/cache/dummycache/dummycache_test.go b/cache/dummycache/dummycache_test.go new file mode 100644 index 00000000000..74004feaa38 --- /dev/null +++ b/cache/dummycache/dummycache_test.go @@ -0,0 +1,31 @@ +package dummycache + +import "testing" + +func TestDummyCache(t *testing.T) { + + c, _ := New() + + account, err := c.Accounts().Get("account1") + if err != nil { + t.Fatal(err) + } + + if account.ID != "account1" { + t.Error("Wrong account returned") + } + + if err := c.Config().Set("config", "abc123"); err != nil { + t.Errorf("Dummy config should return nil") + } + + cfg, err := c.Config().Get("config") + if err != nil { + t.Error("Dummy configs should be supported") + } + + if cfg != "abc123" { + t.Error("Dummy config did not return back expected string") + } + +} diff --git a/cache/filecache/filecache.go b/cache/filecache/filecache.go new file mode 100644 index 00000000000..7bc4bea43f0 --- /dev/null +++ b/cache/filecache/filecache.go @@ -0,0 +1,123 @@ +package filecache + +import ( + "fmt" + "io/ioutil" + + "github.com/golang/glog" + "github.com/prebid/prebid-server/cache" + "gopkg.in/yaml.v2" +) + +type shared struct { + Configs map[string]string + Accounts map[string]bool +} + +// Cache is a file backed cache +type Cache struct { + shared *shared + accounts *accountService + config *configService +} + +type fileConfig struct { + ID string `yaml:"id"` + Config string `yaml:"config"` +} + +type fileCacheFile struct { + Configs []fileConfig `yaml:"configs"` + Accounts []string `yaml:"accounts"` +} + +// New will load the file into memory +func New(filename string) (*Cache, error) { + if glog.V(2) { + glog.Infof("Reading inventory urls from %s", filename) + } + + b, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + + if glog.V(2) { + glog.Infof("Parsing filecache YAML") + } + + var u fileCacheFile + if err = yaml.Unmarshal(b, &u); err != nil { + return nil, err + } + + if glog.V(2) { + glog.Infof("Building URL map") + } + + s := &shared{} + + s.Configs = make(map[string]string, len(u.Configs)) + for _, config := range u.Configs { + s.Configs[config.ID] = config.Config + } + glog.Infof("Loaded %d configs", len(u.Configs)) + + s.Accounts = make(map[string]bool, len(u.Accounts)) + for _, Account := range u.Accounts { + s.Accounts[Account] = true + } + glog.Infof("Loaded %d accounts", len(u.Accounts)) + + return &Cache{ + shared: s, + accounts: &accountService{s}, + config: &configService{s}, + }, nil +} + +// This empty function exists so the Cache struct implements the Cache interface defined in cache/legacy.go +func (c *Cache) Close() error { + return nil +} + +func (c *Cache) Accounts() cache.AccountsService { + return c.accounts +} +func (c *Cache) Config() cache.ConfigService { + return c.config +} + +// AccountService handles the account information +type accountService struct { + shared *shared +} + +// Get will return Account from memory if it exists +func (s *accountService) Get(id string) (*cache.Account, error) { + if _, ok := s.shared.Accounts[id]; !ok { + return nil, fmt.Errorf("Not found") + } + return &cache.Account{ + ID: id, + }, nil +} + +// ConfigService not supported, always returns an error +type configService struct { + shared *shared +} + +// Get will return config from memory if it exists +func (s *configService) Get(id string) (string, error) { + cfg, ok := s.shared.Configs[id] + if !ok { + return "", fmt.Errorf("Not found") + } + return cfg, nil +} + +// Set not supported, always returns an error +func (s *configService) Set(id, value string) error { + return fmt.Errorf("Not supported") +} diff --git a/cache/filecache/filecache_test.go b/cache/filecache/filecache_test.go new file mode 100644 index 00000000000..80a72803bbe --- /dev/null +++ b/cache/filecache/filecache_test.go @@ -0,0 +1,79 @@ +package filecache + +import ( + "io/ioutil" + "os" + "testing" + + yaml "gopkg.in/yaml.v2" +) + +func TestFileCache(t *testing.T) { + fcf := fileCacheFile{ + Accounts: []string{"account1", "account2", "account3"}, + Configs: []fileConfig{ + { + ID: "one", + Config: "config1", + }, { + ID: "two", + Config: "config2", + }, { + ID: "three", + Config: "config3", + }, + }, + } + + bytes, err := yaml.Marshal(&fcf) + if err != nil { + t.Fatal(err) + } + + tmpfile, err := ioutil.TempFile("", "filecache") + if err != nil { + t.Fatal(err) + } + defer os.Remove(tmpfile.Name()) + + if _, err := tmpfile.Write(bytes); err != nil { + t.Fatal(err) + } + + if err := tmpfile.Close(); err != nil { + t.Fatal(err) + } + + dataCache, err := New(tmpfile.Name()) + if err != nil { + t.Fatal(err) + } + + a, err := dataCache.Accounts().Get("account1") + if err != nil { + t.Fatal(err) + } + + if a.ID != "account1" { + t.Error("fetched invalid account") + } + + a, err = dataCache.Accounts().Get("abc123") + if err == nil { + t.Error("account should not exist in cache") + } + + c, err := dataCache.Config().Get("one") + if err != nil { + t.Fatal(err) + } + + if c != "config1" { + t.Error("fetched invalid config") + } + + c, err = dataCache.Config().Get("abc123") + if err == nil { + t.Error("config should not exist in cache") + } +} diff --git a/cache/legacy.go b/cache/legacy.go new file mode 100644 index 00000000000..19c5ae5a4fe --- /dev/null +++ b/cache/legacy.go @@ -0,0 +1,33 @@ +package cache + +type Domain struct { + Domain string `json:"domain"` +} + +type App struct { + Bundle string `json:"bundle"` +} + +type Account struct { + ID string `json:"id"` + PriceGranularity string `json:"price_granularity"` +} + +type Configuration struct { + Type string `json:"type"` // required +} + +type Cache interface { + Close() error + Accounts() AccountsService + Config() ConfigService +} + +type AccountsService interface { + Get(string) (*Account, error) +} + +type ConfigService interface { + Get(string) (string, error) + Set(string, string) error +} diff --git a/cache/postgrescache/postgrescache.go b/cache/postgrescache/postgrescache.go new file mode 100644 index 00000000000..2333e08269e --- /dev/null +++ b/cache/postgrescache/postgrescache.go @@ -0,0 +1,139 @@ +package postgrescache + +import ( + "bytes" + "context" + "database/sql" + "encoding/gob" + "time" + + "github.com/prebid/prebid-server/stored_requests" + + "github.com/coocood/freecache" + "github.com/lib/pq" + "github.com/prebid/prebid-server/cache" +) + +type CacheConfig struct { + TTL int + Size int +} + +// shared configuration that get used by all of the services +type shared struct { + db *sql.DB + lru *freecache.Cache + ttlSeconds int +} + +// Cache postgres +type Cache struct { + shared *shared + accounts *accountService + config *configService +} + +// New creates new postgres.Cache +func New(db *sql.DB, cfg CacheConfig) *Cache { + shared := &shared{ + db: db, + lru: freecache.NewCache(cfg.Size), + ttlSeconds: cfg.TTL, + } + return &Cache{ + shared: shared, + accounts: &accountService{shared: shared}, + config: &configService{shared: shared}, + } +} + +func (c *Cache) Accounts() cache.AccountsService { + return c.accounts +} +func (c *Cache) Config() cache.ConfigService { + return c.config +} + +func (c *Cache) Close() error { + return c.shared.db.Close() +} + +// AccountService handles the account information +type accountService struct { + shared *shared +} + +// Get echos back the account +func (s *accountService) Get(key string) (*cache.Account, error) { + var account cache.Account + + b, err := s.shared.lru.Get([]byte(key)) + if err == nil { + return decodeAccount(b), nil + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(50)*time.Millisecond) + defer cancel() + var id string + var priceGranularity sql.NullString + if err := s.shared.db.QueryRowContext(ctx, "SELECT uuid, price_granularity FROM accounts_account where uuid = $1 LIMIT 1", key).Scan(&id, &priceGranularity); err != nil { + /* TODO -- We should store failed attempts in the LRU as well to stop from hitting to DB */ + return nil, err + } + + account.ID = id + if priceGranularity.Valid { + account.PriceGranularity = priceGranularity.String + } + + buf := bytes.Buffer{} + if err := gob.NewEncoder(&buf).Encode(&account); err != nil { + panic(err) + } + + s.shared.lru.Set([]byte(key), buf.Bytes(), s.shared.ttlSeconds) + return &account, nil +} + +func decodeAccount(b []byte) *cache.Account { + var account cache.Account + buf := bytes.NewReader(b) + if err := gob.NewDecoder(buf).Decode(&account); err != nil { + panic(err) + } + return &account +} + +// ConfigService +type configService struct { + shared *shared +} + +func (s *configService) Set(id, value string) error { + return nil +} + +func (s *configService) Get(key string) (string, error) { + if b, err := s.shared.lru.Get([]byte(key)); err == nil { + return string(b), nil + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(50)*time.Millisecond) + defer cancel() + var config string + if err := s.shared.db.QueryRowContext(ctx, "SELECT config FROM s2sconfig_config where uuid = $1 LIMIT 1", key).Scan(&config); err != nil { + // TODO -- We should store failed attempts in the LRU as well to stop from hitting to DB + + // If the user didn't give us a UUID, the query fails with this error. Wrap it so that we don't + // pollute the app logs with bad user input. + if pqErr, ok := err.(*pq.Error); ok && string(pqErr.Code) == "22P02" { + err = &stored_requests.NotFoundError{ + ID: key, + DataType: "Legacy Config", + } + } + return "", err + } + s.shared.lru.Set([]byte(key), []byte(config), s.shared.ttlSeconds) + return config, nil +} diff --git a/cache/postgrescache/postgrescache_test.go b/cache/postgrescache/postgrescache_test.go new file mode 100644 index 00000000000..bab96f1a11e --- /dev/null +++ b/cache/postgrescache/postgrescache_test.go @@ -0,0 +1,94 @@ +package postgrescache + +import ( + "database/sql" + "testing" + + "github.com/coocood/freecache" + "github.com/erikstmartin/go-testdb" +) + +type StubCache struct { + shared *shared + accounts *accountService + config *configService +} + +// New creates new postgres.Cache +func StubNew(cfg CacheConfig) *Cache { + shared := stubnewShared(cfg) + return &Cache{ + shared: shared, + accounts: &accountService{shared: shared}, + config: &configService{shared: shared}, + } +} + +func stubnewShared(conf CacheConfig) *shared { + db, _ := sql.Open("testdb", "") + + s := &shared{ + db: db, + lru: freecache.NewCache(conf.Size), + ttlSeconds: 0, + } + return s +} + +func TestPostgresDbPriceGranularity(t *testing.T) { + defer testdb.Reset() + + sql := "SELECT uuid, price_granularity FROM accounts_account where uuid = $1 LIMIT 1" + columns := []string{"uuid", "price_granularity"} + result := ` + bdc928ef-f725-4688-8171-c104cc715bdf,med + ` + testdb.StubQuery(sql, testdb.RowsFromCSVString(columns, result)) + + conf := CacheConfig{ + TTL: 3434, + Size: 100, + } + dataCache := StubNew(conf) + + account, err := dataCache.Accounts().Get("bdc928ef-f725-4688-8171-c104cc715bdf") + if err != nil { + t.Fatalf("test postgres db errored: %v", err) + } + + if account.ID != "bdc928ef-f725-4688-8171-c104cc715bdf" { + t.Error("Expected bdc928ef-f725-4688-8171-c104cc715bdf") + } + if account.PriceGranularity != "med" { + t.Error("Expected med") + } +} + +func TestPostgresDbNullPriceGranularity(t *testing.T) { + defer testdb.Reset() + + sql := "SELECT uuid, price_granularity FROM accounts_account where uuid = $1 LIMIT 1" + columns := []string{"uuid", "price_granularity"} + result := ` + bdc928ef-f725-4688-8171-c104cc715bdf + ` + testdb.StubQuery(sql, testdb.RowsFromCSVString(columns, result)) + + conf := CacheConfig{ + TTL: 3434, + Size: 100, + } + dataCache := StubNew(conf) + + account, err := dataCache.Accounts().Get("bdc928ef-f725-4688-8171-c104cc715bdf") + if err != nil { + t.Fatalf("test postgres db errored: %v", err) + } + + if account.ID != "bdc928ef-f725-4688-8171-c104cc715bdf" { + t.Error("Expected bdc928ef-f725-4688-8171-c104cc715bdf") + } + if account.PriceGranularity != "" { + t.Error("Expected null string") + } +} diff --git a/config/config.go b/config/config.go index 179a30af1cc..597ac1b41a7 100644 --- a/config/config.go +++ b/config/config.go @@ -33,6 +33,7 @@ type Configuration struct { RecaptchaSecret string `mapstructure:"recaptcha_secret"` HostCookie HostCookie `mapstructure:"host_cookie"` Metrics Metrics `mapstructure:"metrics"` + DataCache DataCache `mapstructure:"datacache"` StoredRequests StoredRequests `mapstructure:"stored_requests"` StoredRequestsAMP StoredRequests `mapstructure:"stored_amp_req"` CategoryMapping StoredRequests `mapstructure:"category_mapping"` @@ -84,6 +85,10 @@ type Configuration struct { GenerateBidID bool `mapstructure:"generate_bid_id"` // GenerateRequestID overrides the bidrequest.id in an AMP Request or an App Stored Request with a generated UUID if set to true. The default is false. GenerateRequestID bool `mapstructure:"generate_request_id"` + + // EnableLegacyAuction specifies if the original /auction endpoint with a custom PBS data model is allowed + // by the host. + EnableLegacyAuction bool `mapstructure:"enable_legacy_auction"` } const MIN_COOKIE_SIZE_BYTES = 500 @@ -396,6 +401,13 @@ func (m *PrometheusMetrics) Timeout() time.Duration { return time.Duration(m.TimeoutMillisRaw) * time.Millisecond } +type DataCache struct { + Type string `mapstructure:"type"` + Filename string `mapstructure:"filename"` + CacheSize int `mapstructure:"cache_size"` + TTLSeconds int `mapstructure:"ttl_seconds"` +} + // ExternalCache configures the externally accessible cache url. type ExternalCache struct { Scheme string `mapstructure:"scheme"` @@ -648,6 +660,10 @@ func SetupViper(v *viper.Viper, filename string) { v.SetDefault("metrics.prometheus.namespace", "") v.SetDefault("metrics.prometheus.subsystem", "") v.SetDefault("metrics.prometheus.timeout_ms", 10000) + v.SetDefault("datacache.type", "dummy") + v.SetDefault("datacache.filename", "") + v.SetDefault("datacache.cache_size", 0) + v.SetDefault("datacache.ttl_seconds", 0) v.SetDefault("category_mapping.filesystem.enabled", true) v.SetDefault("category_mapping.filesystem.directorypath", "./static/category-mapping") v.SetDefault("category_mapping.http.endpoint", "") @@ -938,6 +954,7 @@ func SetupViper(v *viper.Viper, filename string) { v.SetDefault("auto_gen_source_tid", true) v.SetDefault("generate_bid_id", false) v.SetDefault("generate_request_id", false) + v.SetDefault("enable_legacy_auction", false) v.SetDefault("request_timeout_headers.request_time_in_queue", "") v.SetDefault("request_timeout_headers.request_timeout_in_queue", "") diff --git a/config/config_test.go b/config/config_test.go index 4477d127f63..819eb21f819 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -130,6 +130,7 @@ func TestDefaults(t *testing.T) { cmpInts(t, "max_request_size", int(cfg.MaxRequestSize), 1024*256) cmpInts(t, "host_cookie.ttl_days", int(cfg.HostCookie.TTL), 90) cmpInts(t, "host_cookie.max_cookie_size_bytes", cfg.HostCookie.MaxCookieSizeBytes, 0) + cmpStrings(t, "datacache.type", cfg.DataCache.Type, "dummy") cmpStrings(t, "adapters.pubmatic.endpoint", cfg.Adapters[string(openrtb_ext.BidderPubmatic)].Endpoint, "https://hbopenbid.pubmatic.com/translator?source=prebid-server") cmpInts(t, "currency_converter.fetch_interval_seconds", cfg.CurrencyConverter.FetchIntervalSeconds, 1800) cmpStrings(t, "currency_converter.fetch_url", cfg.CurrencyConverter.FetchURL, "https://cdn.jsdelivr.net/gh/prebid/currency-file@1/latest.json") @@ -143,6 +144,7 @@ func TestDefaults(t *testing.T) { cmpStrings(t, "stored_requests.filesystem.directorypath", "./stored_requests/data/by_id", cfg.StoredRequests.Files.Path) cmpBools(t, "auto_gen_source_tid", cfg.AutoGenSourceTID, true) cmpBools(t, "generate_bid_id", cfg.GenerateBidID, false) + cmpBools(t, "enable_legacy_auction", cfg.EnableLegacyAuction, false) //Assert purpose VendorExceptionMap hash tables were built correctly expectedTCF2 := TCF2{ @@ -314,6 +316,11 @@ metrics: account_adapter_details: true adapter_connections_metrics: true adapter_gdpr_request_blocked: true +datacache: + type: postgres + filename: /usr/db/db.db + cache_size: 10000000 + ttl_seconds: 3600 adapters: appnexus: endpoint: http://ib.adnxs.com/some/endpoint @@ -347,6 +354,7 @@ request_validation: ipv4_private_networks: ["1.1.1.0/24"] ipv6_private_networks: ["1111::/16", "2222::/16"] generate_bid_id: true +enable_legacy_auction: true `) var adapterExtraInfoConfig = []byte(` @@ -537,6 +545,10 @@ func TestFullConfig(t *testing.T) { cmpStrings(t, "metrics.influxdb.username", cfg.Metrics.Influxdb.Username, "admin") cmpStrings(t, "metrics.influxdb.password", cfg.Metrics.Influxdb.Password, "admin1324") cmpInts(t, "metrics.influxdb.metric_send_interval", cfg.Metrics.Influxdb.MetricSendInterval, 30) + cmpStrings(t, "datacache.type", cfg.DataCache.Type, "postgres") + cmpStrings(t, "datacache.filename", cfg.DataCache.Filename, "/usr/db/db.db") + cmpInts(t, "datacache.cache_size", cfg.DataCache.CacheSize, 10000000) + cmpInts(t, "datacache.ttl_seconds", cfg.DataCache.TTLSeconds, 3600) cmpStrings(t, "", cfg.CacheURL.GetBaseURL(), "http://prebidcache.net") cmpStrings(t, "", cfg.GetCachedAssetURL("a0eebc99-9c0b-4ef8-bb00-6bb9bd380a11"), "http://prebidcache.net/cache?uuid=a0eebc99-9c0b-4ef8-bb00-6bb9bd380a11") cmpStrings(t, "adapters.appnexus.endpoint", cfg.Adapters[string(openrtb_ext.BidderAppnexus)].Endpoint, "http://ib.adnxs.com/some/endpoint") @@ -567,6 +579,7 @@ func TestFullConfig(t *testing.T) { cmpStrings(t, "request_validation.ipv6_private_networks", cfg.RequestValidation.IPv6PrivateNetworks[1], "2222::/16") cmpBools(t, "generate_bid_id", cfg.GenerateBidID, true) cmpStrings(t, "debug.override_token", cfg.Debug.OverrideToken, "") + cmpBools(t, "enable_legacy_auction", cfg.EnableLegacyAuction, true) } func TestUnmarshalAdapterExtraInfo(t *testing.T) { diff --git a/config/stored_requests.go b/config/stored_requests.go index e752e9e4d9d..ee78179eb65 100644 --- a/config/stored_requests.go +++ b/config/stored_requests.go @@ -133,6 +133,7 @@ func resolvedStoredRequestsConfig(cfg *Configuration) { cfg.StoredVideo.dataType = VideoDataType cfg.CategoryMapping.dataType = CategoryDataType cfg.Accounts.dataType = AccountDataType + return } func (cfg *StoredRequests) validate(errs []error) []error { diff --git a/config/structlog.go b/config/structlog.go index 911f475717d..a91e5ab857e 100644 --- a/config/structlog.go +++ b/config/structlog.go @@ -12,7 +12,7 @@ import ( type logMsg func(string, ...interface{}) var mapregex = regexp.MustCompile(`mapstructure:"([^"]+)"`) -var blocklistregexp = []*regexp.Regexp{ +var blacklistregexp = []*regexp.Regexp{ regexp.MustCompile("password"), } @@ -84,7 +84,7 @@ func fieldNameByTag(f reflect.StructField) string { } func allowedName(name string) bool { - for _, r := range blocklistregexp { + for _, r := range blacklistregexp { if r.MatchString(name) { return false } diff --git a/endpoints/auction.go b/endpoints/auction.go new file mode 100644 index 00000000000..10d2ced6c37 --- /dev/null +++ b/endpoints/auction.go @@ -0,0 +1,513 @@ +package endpoints + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "runtime/debug" + "sort" + "strconv" + "time" + + "github.com/golang/glog" + "github.com/julienschmidt/httprouter" + "github.com/prebid/prebid-server/adapters" + "github.com/prebid/prebid-server/cache" + "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/errortypes" + "github.com/prebid/prebid-server/exchange" + "github.com/prebid/prebid-server/gdpr" + "github.com/prebid/prebid-server/metrics" + "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/pbs" + pbc "github.com/prebid/prebid-server/prebid_cache_client" + "github.com/prebid/prebid-server/privacy" + gdprPrivacy "github.com/prebid/prebid-server/privacy/gdpr" + "github.com/prebid/prebid-server/usersync" +) + +var allSyncTypes []usersync.SyncType = []usersync.SyncType{usersync.SyncTypeIFrame, usersync.SyncTypeRedirect} + +type bidResult struct { + bidder *pbs.PBSBidder + bidList pbs.PBSBidSlice +} + +const defaultPriceGranularity = "med" + +func min(x, y int) int { + if x < y { + return x + } + return y +} + +func writeAuctionError(w http.ResponseWriter, s string, err error) { + var resp pbs.PBSResponse + if err != nil { + resp.Status = fmt.Sprintf("%s: %v", s, err) + } else { + resp.Status = s + } + b, err := json.Marshal(&resp) + if err != nil { + glog.Errorf("Failed to marshal auction error JSON: %s", err) + } else { + w.Write(b) + } +} + +type auction struct { + cfg *config.Configuration + syncersByBidder map[string]usersync.Syncer + gdprPerms gdpr.Permissions + metricsEngine metrics.MetricsEngine + dataCache cache.Cache + exchanges map[string]adapters.Adapter +} + +func Auction(cfg *config.Configuration, syncersByBidder map[string]usersync.Syncer, gdprPerms gdpr.Permissions, metricsEngine metrics.MetricsEngine, dataCache cache.Cache, exchanges map[string]adapters.Adapter) httprouter.Handle { + a := &auction{ + cfg: cfg, + syncersByBidder: syncersByBidder, + gdprPerms: gdprPerms, + metricsEngine: metricsEngine, + dataCache: dataCache, + exchanges: exchanges, + } + return a.auction +} + +func (a *auction) auction(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.Header().Add("Content-Type", "application/json") + var labels = getDefaultLabels(r) + req, err := pbs.ParsePBSRequest(r, &a.cfg.AuctionTimeouts, a.dataCache, &(a.cfg.HostCookie)) + + defer a.recordMetrics(req, labels) + + if err != nil { + if glog.V(2) { + glog.Infof("Failed to parse /auction request: %v", err) + } + writeAuctionError(w, "Error parsing request", err) + labels.RequestStatus = metrics.RequestStatusBadInput + return + } + status := "OK" + setLabelSource(&labels, req, &status) + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*time.Duration(req.TimeoutMillis)) + defer cancel() + account, err := a.dataCache.Accounts().Get(req.AccountID) + if err != nil { + if glog.V(2) { + glog.Infof("Invalid account id: %v", err) + } + writeAuctionError(w, "Unknown account id", fmt.Errorf("Unknown account")) + labels.RequestStatus = metrics.RequestStatusBadInput + return + } + labels.PubID = req.AccountID + resp := pbs.PBSResponse{ + Status: status, + TID: req.Tid, + BidderStatus: req.Bidders, + } + ch := make(chan bidResult) + sentBids := 0 + for _, bidder := range req.Bidders { + if ex, ok := a.exchanges[bidder.BidderCode]; ok { + // Make sure we have an independent label struct for each bidder. We don't want to run into issues with the goroutine below. + blabels := metrics.AdapterLabels{ + Source: labels.Source, + RType: labels.RType, + Adapter: openrtb_ext.BidderName(bidder.BidderCode), + PubID: labels.PubID, + CookieFlag: labels.CookieFlag, + AdapterBids: metrics.AdapterBidPresent, + } + if skip := a.processUserSync(req, bidder, blabels, ex, &ctx); skip == true { + continue + } + sentBids++ + bidderRunner := a.recoverSafely(func(bidder *pbs.PBSBidder, aLabels metrics.AdapterLabels) { + + start := time.Now() + bidList, err := ex.Call(ctx, req, bidder) + a.metricsEngine.RecordAdapterTime(aLabels, time.Since(start)) + bidder.ResponseTime = int(time.Since(start) / time.Millisecond) + processBidResult(bidList, bidder, &aLabels, a.metricsEngine, err) + + ch <- bidResult{ + bidder: bidder, + bidList: bidList, + // Bidder done, record bidder metrics + } + a.metricsEngine.RecordAdapterRequest(aLabels) + }) + + go bidderRunner(bidder, blabels) + + } else if bidder.BidderCode == "lifestreet" { + bidder.Error = "Bidder is no longer available" + } else { + bidder.Error = "Unsupported bidder" + } + } + for i := 0; i < sentBids; i++ { + result := <-ch + for _, bid := range result.bidList { + resp.Bids = append(resp.Bids, bid) + } + } + if err := cacheAccordingToMarkup(req, &resp, ctx, a, &labels); err != nil { + writeAuctionError(w, "Prebid cache failed", err) + labels.RequestStatus = metrics.RequestStatusErr + return + } + if req.SortBids == 1 { + sortBidsAddKeywordsMobile(resp.Bids, req, account.PriceGranularity) + } + if glog.V(2) { + glog.Infof("Request for %d ad units on url %s by account %s got %d bids", len(req.AdUnits), req.Url, req.AccountID, len(resp.Bids)) + } + enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) + enc.Encode(resp) +} + +func (a *auction) recoverSafely(inner func(*pbs.PBSBidder, metrics.AdapterLabels)) func(*pbs.PBSBidder, metrics.AdapterLabels) { + return func(bidder *pbs.PBSBidder, labels metrics.AdapterLabels) { + defer func() { + if r := recover(); r != nil { + if bidder == nil { + glog.Errorf("Legacy auction recovered panic: %v. Stack trace is: %v", r, string(debug.Stack())) + } else { + glog.Errorf("Legacy auction recovered panic from Bidder %s: %v. Stack trace is: %v", bidder.BidderCode, r, string(debug.Stack())) + } + a.metricsEngine.RecordAdapterPanic(labels) + } + }() + inner(bidder, labels) + } +} + +func (a *auction) shouldUsersync(ctx context.Context, bidder openrtb_ext.BidderName, gdprPrivacyPolicy gdprPrivacy.Policy) bool { + gdprSignal := gdpr.SignalAmbiguous + if signal, err := gdpr.SignalParse(gdprPrivacyPolicy.Signal); err != nil { + gdprSignal = signal + } + + if canSync, err := a.gdprPerms.HostCookiesAllowed(ctx, gdprSignal, gdprPrivacyPolicy.Consent); err != nil || !canSync { + return false + } + canSync, err := a.gdprPerms.BidderSyncAllowed(ctx, bidder, gdprSignal, gdprPrivacyPolicy.Consent) + return canSync && err == nil +} + +// cache video bids only for Web +func cacheVideoOnly(bids pbs.PBSBidSlice, ctx context.Context, deps *auction, labels *metrics.Labels) error { + var cobjs []*pbc.CacheObject + for _, bid := range bids { + if bid.CreativeMediaType == "video" { + cobjs = append(cobjs, &pbc.CacheObject{ + Value: bid.Adm, + IsVideo: true, + }) + } + } + err := pbc.Put(ctx, cobjs) + if err != nil { + return err + } + videoIndex := 0 + for _, bid := range bids { + if bid.CreativeMediaType == "video" { + bid.CacheID = cobjs[videoIndex].UUID + bid.CacheURL = deps.cfg.GetCachedAssetURL(bid.CacheID) + bid.NURL = "" + bid.Adm = "" + videoIndex++ + } + } + return nil +} + +// checkForValidBidSize goes through list of bids & find those which are banner mediaType and with height or width not defined +// determine the num of ad unit sizes that were used in corresponding bid request +// if num_adunit_sizes == 1, assign the height and/or width to bid's height/width +// if num_adunit_sizes > 1, reject the bid (remove from list) and return an error +// return updated bid list object for next steps in auction +func checkForValidBidSize(bids pbs.PBSBidSlice, bidder *pbs.PBSBidder) pbs.PBSBidSlice { + finalValidBids := make([]*pbs.PBSBid, len(bids)) + finalBidCounter := 0 +bidLoop: + for _, bid := range bids { + if isUndimensionedBanner(bid) { + for _, adunit := range bidder.AdUnits { + if copyBannerDimensions(&adunit, bid, finalValidBids, &finalBidCounter) { + continue bidLoop + } + } + } else { + finalValidBids[finalBidCounter] = bid + finalBidCounter = finalBidCounter + 1 + } + } + return finalValidBids[:finalBidCounter] +} + +func isUndimensionedBanner(bid *pbs.PBSBid) bool { + return bid.CreativeMediaType == "banner" && (bid.Height == 0 || bid.Width == 0) +} + +func copyBannerDimensions(adunit *pbs.PBSAdUnit, bid *pbs.PBSBid, finalValidBids []*pbs.PBSBid, finalBidCounter *int) bool { + var bidIDEqualsCode bool = false + + if adunit.BidID == bid.BidID && adunit.Code == bid.AdUnitCode && adunit.Sizes != nil { + if len(adunit.Sizes) == 1 { + bid.Width, bid.Height = adunit.Sizes[0].W, adunit.Sizes[0].H + finalValidBids[*finalBidCounter] = bid + *finalBidCounter += 1 + } else if len(adunit.Sizes) > 1 { + glog.Warningf("Bid was rejected for bidder %s because no size was defined", bid.BidderCode) + } + bidIDEqualsCode = true + } + + return bidIDEqualsCode +} + +// sortBidsAddKeywordsMobile sorts the bids and adds ad server targeting keywords to each bid. +// The bids are sorted by cpm to find the highest bid. +// The ad server targeting keywords are added to all bids, with specific keywords for the highest bid. +func sortBidsAddKeywordsMobile(bids pbs.PBSBidSlice, pbs_req *pbs.PBSRequest, priceGranularitySetting string) { + if priceGranularitySetting == "" { + priceGranularitySetting = defaultPriceGranularity + } + + // record bids by ad unit code for sorting + code_bids := make(map[string]pbs.PBSBidSlice, len(bids)) + for _, bid := range bids { + code_bids[bid.AdUnitCode] = append(code_bids[bid.AdUnitCode], bid) + } + + // loop through ad units to find top bid + for _, unit := range pbs_req.AdUnits { + bar := code_bids[unit.Code] + + if len(bar) == 0 { + if glog.V(3) { + glog.Infof("No bids for ad unit '%s'", unit.Code) + } + continue + } + sort.Sort(bar) + + // after sorting we need to add the ad targeting keywords + for i, bid := range bar { + // We should eventually check for the error and do something. + roundedCpm := exchange.GetPriceBucket(bid.Price, openrtb_ext.PriceGranularityFromString(priceGranularitySetting)) + + hbSize := "" + if bid.Width != 0 && bid.Height != 0 { + width := strconv.FormatInt(bid.Width, 10) + height := strconv.FormatInt(bid.Height, 10) + hbSize = width + "x" + height + } + + hbPbBidderKey := string(openrtb_ext.HbpbConstantKey) + "_" + bid.BidderCode + hbBidderBidderKey := string(openrtb_ext.HbBidderConstantKey) + "_" + bid.BidderCode + hbCacheIDBidderKey := string(openrtb_ext.HbCacheKey) + "_" + bid.BidderCode + hbDealIDBidderKey := string(openrtb_ext.HbDealIDConstantKey) + "_" + bid.BidderCode + hbSizeBidderKey := string(openrtb_ext.HbSizeConstantKey) + "_" + bid.BidderCode + if pbs_req.MaxKeyLength != 0 { + hbPbBidderKey = hbPbBidderKey[:min(len(hbPbBidderKey), int(pbs_req.MaxKeyLength))] + hbBidderBidderKey = hbBidderBidderKey[:min(len(hbBidderBidderKey), int(pbs_req.MaxKeyLength))] + hbCacheIDBidderKey = hbCacheIDBidderKey[:min(len(hbCacheIDBidderKey), int(pbs_req.MaxKeyLength))] + hbDealIDBidderKey = hbDealIDBidderKey[:min(len(hbDealIDBidderKey), int(pbs_req.MaxKeyLength))] + hbSizeBidderKey = hbSizeBidderKey[:min(len(hbSizeBidderKey), int(pbs_req.MaxKeyLength))] + } + + // fixes #288 where map was being overwritten instead of updated + if bid.AdServerTargeting == nil { + bid.AdServerTargeting = make(map[string]string) + } + kvs := bid.AdServerTargeting + + kvs[hbPbBidderKey] = roundedCpm + kvs[hbBidderBidderKey] = bid.BidderCode + kvs[hbCacheIDBidderKey] = bid.CacheID + + if hbSize != "" { + kvs[hbSizeBidderKey] = hbSize + } + if bid.DealId != "" { + kvs[hbDealIDBidderKey] = bid.DealId + } + // For the top bid, we want to add the following additional keys + if i == 0 { + kvs[string(openrtb_ext.HbpbConstantKey)] = roundedCpm + kvs[string(openrtb_ext.HbBidderConstantKey)] = bid.BidderCode + kvs[string(openrtb_ext.HbCacheKey)] = bid.CacheID + if bid.DealId != "" { + kvs[string(openrtb_ext.HbDealIDConstantKey)] = bid.DealId + } + if hbSize != "" { + kvs[string(openrtb_ext.HbSizeConstantKey)] = hbSize + } + } + } + } +} + +func getDefaultLabels(r *http.Request) metrics.Labels { + return metrics.Labels{ + Source: metrics.DemandUnknown, + RType: metrics.ReqTypeLegacy, + PubID: "", + CookieFlag: metrics.CookieFlagUnknown, + RequestStatus: metrics.RequestStatusOK, + } +} + +func setLabelSource(labels *metrics.Labels, req *pbs.PBSRequest, status *string) { + if req.App != nil { + labels.Source = metrics.DemandApp + } else { + labels.Source = metrics.DemandWeb + if req.Cookie.HasAnyLiveSyncs() { + labels.CookieFlag = metrics.CookieFlagYes + } else { + labels.CookieFlag = metrics.CookieFlagNo + *status = "no_cookie" + } + } +} + +func cacheAccordingToMarkup(req *pbs.PBSRequest, resp *pbs.PBSResponse, ctx context.Context, a *auction, labels *metrics.Labels) error { + if req.CacheMarkup == 1 { + cobjs := make([]*pbc.CacheObject, len(resp.Bids)) + for i, bid := range resp.Bids { + if bid.CreativeMediaType == "video" { + cobjs[i] = &pbc.CacheObject{ + Value: bid.Adm, + IsVideo: true, + } + } else { + cobjs[i] = &pbc.CacheObject{ + Value: &pbc.BidCache{ + Adm: bid.Adm, + NURL: bid.NURL, + Width: bid.Width, + Height: bid.Height, + }, + IsVideo: false, + } + } + } + if err := pbc.Put(ctx, cobjs); err != nil { + return err + } + for i, bid := range resp.Bids { + bid.CacheID = cobjs[i].UUID + bid.CacheURL = a.cfg.GetCachedAssetURL(bid.CacheID) + bid.NURL = "" + bid.Adm = "" + } + } else if req.CacheMarkup == 2 { + return cacheVideoOnly(resp.Bids, ctx, a, labels) + } + return nil +} + +func processBidResult(bidList pbs.PBSBidSlice, bidder *pbs.PBSBidder, aLabels *metrics.AdapterLabels, metricsEngine metrics.MetricsEngine, err error) { + if err != nil { + var s struct{} + if err == context.DeadlineExceeded { + aLabels.AdapterErrors = map[metrics.AdapterError]struct{}{metrics.AdapterErrorTimeout: s} + bidder.Error = "Timed out" + } else if err != context.Canceled { + bidder.Error = err.Error() + switch err.(type) { + case *errortypes.BadInput: + aLabels.AdapterErrors = map[metrics.AdapterError]struct{}{metrics.AdapterErrorBadInput: s} + case *errortypes.BadServerResponse: + aLabels.AdapterErrors = map[metrics.AdapterError]struct{}{metrics.AdapterErrorBadServerResponse: s} + default: + glog.Warningf("Error from bidder %v. Ignoring all bids: %v", bidder.BidderCode, err) + aLabels.AdapterErrors = map[metrics.AdapterError]struct{}{metrics.AdapterErrorUnknown: s} + } + } + } else if bidList != nil { + bidList = checkForValidBidSize(bidList, bidder) + bidder.NumBids = len(bidList) + for _, bid := range bidList { + var cpm = float64(bid.Price * 1000) + metricsEngine.RecordAdapterPrice(*aLabels, cpm) + switch bid.CreativeMediaType { + case "banner": + metricsEngine.RecordAdapterBidReceived(*aLabels, openrtb_ext.BidTypeBanner, bid.Adm != "") + case "video": + metricsEngine.RecordAdapterBidReceived(*aLabels, openrtb_ext.BidTypeVideo, bid.Adm != "") + } + bid.ResponseTime = bidder.ResponseTime + } + } else { + bidder.NoBid = true + aLabels.AdapterBids = metrics.AdapterBidNone + } +} + +func (a *auction) recordMetrics(req *pbs.PBSRequest, labels metrics.Labels) { + a.metricsEngine.RecordRequest(labels) + if req == nil { + a.metricsEngine.RecordLegacyImps(labels, 0) + return + } + a.metricsEngine.RecordLegacyImps(labels, len(req.AdUnits)) + a.metricsEngine.RecordRequestTime(labels, time.Since(req.Start)) +} + +func (a *auction) processUserSync(req *pbs.PBSRequest, bidder *pbs.PBSBidder, blabels metrics.AdapterLabels, ex adapters.Adapter, ctx *context.Context) bool { + var skip bool = false + if req.App != nil { + return skip + } + // If exchanges[bidderCode] exists, then a.syncers[bidderCode] exists *except for districtm*. + // OpenRTB handles aliases differently, so this hack will keep legacy code working. For all other + // bidderCodes, a.syncers[bidderCode] will exist if exchanges[bidderCode] also does. + // This is guaranteed by the TestSyncers unit test inside usersync/usersync_test.go, which compares these maps to the (source of truth) openrtb_ext.BidderMap: + syncerCode := bidder.BidderCode + if syncerCode == "districtm" { + syncerCode = "appnexus" + } + syncer := a.syncersByBidder[syncerCode] + uid, _, _ := req.Cookie.GetUID(syncer.Key()) + if uid == "" { + bidder.NoCookie = true + privacyPolicies := privacy.Policies{ + GDPR: gdprPrivacy.Policy{ + Signal: req.ParseGDPR(), + Consent: req.ParseConsent(), + }, + } + if a.shouldUsersync(*ctx, openrtb_ext.BidderName(syncerCode), privacyPolicies.GDPR) { + sync, err := syncer.GetSync(allSyncTypes, privacyPolicies) + if err == nil { + bidder.UsersyncInfo = &pbs.UsersyncInfo{ + URL: sync.URL, + Type: string(sync.Type), + SupportCORS: sync.SupportCORS, + } + } else { + glog.Errorf("Failed to get usersync info for %s: %v", syncerCode, err) + } + } + blabels.CookieFlag = metrics.CookieFlagNo + if ex.SkipNoCookies() { + skip = true + } + } + return skip +} diff --git a/endpoints/auction_test.go b/endpoints/auction_test.go new file mode 100644 index 00000000000..1a30e025faa --- /dev/null +++ b/endpoints/auction_test.go @@ -0,0 +1,654 @@ +package endpoints + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + + "github.com/mxmCherry/openrtb/v15/openrtb2" + "github.com/prebid/prebid-server/cache/dummycache" + "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/gdpr" + "github.com/prebid/prebid-server/metrics" + metricsConf "github.com/prebid/prebid-server/metrics/config" + "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/pbs" + "github.com/prebid/prebid-server/prebid_cache_client" + gdprPolicy "github.com/prebid/prebid-server/privacy/gdpr" + "github.com/prebid/prebid-server/usersync" + "github.com/spf13/viper" + + "github.com/stretchr/testify/assert" +) + +func TestSortBidsAndAddKeywordsForMobile(t *testing.T) { + body := []byte(`{ + "max_key_length":20, + "user":{ + "gender":"F", + "buyeruid":"test_buyeruid", + "yob":2000, + "id":"testid" + }, + "prebid_version":"0.21.0-pre", + "sort_bids":1, + "ad_units":[ + { + "sizes":[ + { + "w":300, + "h":250 + } + ], + "config_id":"ad5ffb41-3492-40f3-9c25-ade093eb4e5f", + "code":"test_adunitcode" + } + ], + "cache_markup":1, + "app":{ + "bundle":"AppNexus.PrebidMobileDemo", + "ver":"0.0.1" + }, + "sdk":{ + "version":"0.0.1", + "platform":"iOS", + "source":"prebid-mobile" + }, + "device":{ + "ifa":"test_device_ifa", + "osv":"9.3.5", + "os":"iOS", + "make":"Apple", + "model":"iPhone6,1" + }, + "tid":"abcd", + "account_id":"aecd6ef7-b992-4e99-9bb8-65e2d984e1dd" + } + `) + r := httptest.NewRequest("POST", "/auction", bytes.NewBuffer(body)) + d, _ := dummycache.New() + hcc := config.HostCookie{} + + pbs_req, err := pbs.ParsePBSRequest(r, &config.AuctionTimeouts{ + Default: 2000, + Max: 2000, + }, d, &hcc) + if err != nil { + t.Errorf("Unexpected error on parsing %v", err) + } + + bids := make(pbs.PBSBidSlice, 0) + + fb_bid := pbs.PBSBid{ + BidID: "test_bidid", + AdUnitCode: "test_adunitcode", + BidderCode: "audienceNetwork", + Price: 2.00, + Adm: "test_adm", + Width: 300, + Height: 250, + CacheID: "test_cache_id1", + DealId: "2345", + } + bids = append(bids, &fb_bid) + an_bid := pbs.PBSBid{ + BidID: "test_bidid2", + AdUnitCode: "test_adunitcode", + BidderCode: "appnexus", + Price: 1.00, + Adm: "test_adm", + Width: 320, + Height: 50, + CacheID: "test_cache_id2", + DealId: "1234", + } + bids = append(bids, &an_bid) + rb_bid := pbs.PBSBid{ + BidID: "test_bidid2", + AdUnitCode: "test_adunitcode", + BidderCode: "rubicon", + Price: 1.00, + Adm: "test_adm", + Width: 300, + Height: 250, + CacheID: "test_cache_id2", + DealId: "7890", + } + rb_bid.AdServerTargeting = map[string]string{ + "rpfl_1001": "15_tier0100", + } + bids = append(bids, &rb_bid) + nosize_bid := pbs.PBSBid{ + BidID: "test_bidid2", + AdUnitCode: "test_adunitcode", + BidderCode: "nosizebidder", + Price: 1.00, + Adm: "test_adm", + CacheID: "test_cache_id2", + } + bids = append(bids, &nosize_bid) + nodeal_bid := pbs.PBSBid{ + BidID: "test_bidid2", + AdUnitCode: "test_adunitcode", + BidderCode: "nodeal", + Price: 1.00, + Adm: "test_adm", + CacheID: "test_cache_id2", + } + bids = append(bids, &nodeal_bid) + pbs_resp := pbs.PBSResponse{ + Bids: bids, + } + sortBidsAddKeywordsMobile(pbs_resp.Bids, pbs_req, "") + + for _, bid := range bids { + if bid.AdServerTargeting == nil { + t.Error("Ad server targeting should not be nil") + } + if bid.BidderCode == "audienceNetwork" { + if bid.AdServerTargeting[string(openrtb_ext.HbSizeConstantKey)] != "300x250" { + t.Error(string(openrtb_ext.HbSizeConstantKey) + " key was not parsed correctly") + } + if bid.AdServerTargeting[string(openrtb_ext.HbpbConstantKey)] != "2.00" { + t.Error(string(openrtb_ext.HbpbConstantKey)+" key was not parsed correctly ", bid.AdServerTargeting[string(openrtb_ext.HbpbConstantKey)]) + } + + if bid.AdServerTargeting[string(openrtb_ext.HbCacheKey)] != "test_cache_id1" { + t.Error(string(openrtb_ext.HbCacheKey) + " key was not parsed correctly") + } + if bid.AdServerTargeting[string(openrtb_ext.HbBidderConstantKey)] != "audienceNetwork" { + t.Error(string(openrtb_ext.HbBidderConstantKey) + " key was not parsed correctly") + } + if bid.AdServerTargeting[string(openrtb_ext.HbDealIDConstantKey)] != "2345" { + t.Error(string(openrtb_ext.HbDealIDConstantKey) + " key was not parsed correctly ") + } + } + if bid.BidderCode == "appnexus" { + if bid.AdServerTargeting[string(openrtb_ext.HbSizeConstantKey)+"_appnexus"] != "320x50" { + t.Error(string(openrtb_ext.HbSizeConstantKey) + " key for appnexus bidder was not parsed correctly") + } + if bid.AdServerTargeting[string(openrtb_ext.HbCacheKey)+"_appnexus"] != "test_cache_id2" { + t.Error(string(openrtb_ext.HbCacheKey) + " key for appnexus bidder was not parsed correctly") + } + if bid.AdServerTargeting[string(openrtb_ext.HbBidderConstantKey)+"_appnexus"] != "appnexus" { + t.Error(string(openrtb_ext.HbBidderConstantKey) + " key for appnexus bidder was not parsed correctly") + } + if bid.AdServerTargeting[string(openrtb_ext.HbpbConstantKey)+"_appnexus"] != "1.00" { + t.Error(string(openrtb_ext.HbpbConstantKey) + " key for appnexus bidder was not parsed correctly") + } + if bid.AdServerTargeting[string(openrtb_ext.HbpbConstantKey)] != "" { + t.Error(string(openrtb_ext.HbpbConstantKey) + " key was parsed for two bidders") + } + if bid.AdServerTargeting[string(openrtb_ext.HbDealIDConstantKey)+"_appnexus"] != "1234" { + t.Errorf(string(openrtb_ext.HbDealIDConstantKey)+"_appnexus was not parsed correctly %v", bid.AdServerTargeting[string(openrtb_ext.HbDealIDConstantKey)+"_appnexus"]) + } + } + if bid.BidderCode == string(openrtb_ext.BidderRubicon) { + if bid.AdServerTargeting["rpfl_1001"] != "15_tier0100" { + t.Error("custom ad_server_targeting KVPs from adapter were not preserved") + } + } + if bid.BidderCode == "nosizebidder" { + if _, exists := bid.AdServerTargeting[string(openrtb_ext.HbSizeConstantKey)+"_nosizebidder"]; exists { + t.Error(string(openrtb_ext.HbSizeConstantKey)+" key for nosize bidder was not parsed correctly", bid.AdServerTargeting) + } + } + if bid.BidderCode == "nodeal" { + if _, exists := bid.AdServerTargeting[string(openrtb_ext.HbDealIDConstantKey)+"_nodeal"]; exists { + t.Error(string(openrtb_ext.HbDealIDConstantKey) + " key for nodeal bidder was not parsed correctly") + } + } + } +} + +var ( + MaxValueLength = 1024 * 10 + MaxNumValues = 10 +) + +type responseObject struct { + UUID string `json:"uuid"` +} + +type response struct { + Responses []responseObject `json:"responses"` +} + +type putAnyObject struct { + Type string `json:"type"` + Value json.RawMessage `json:"value"` +} + +type putAnyRequest struct { + Puts []putAnyObject `json:"puts"` +} + +func DummyPrebidCacheServer(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, "Failed to read the request body.", http.StatusBadRequest) + return + } + defer r.Body.Close() + var put putAnyRequest + + err = json.Unmarshal(body, &put) + if err != nil { + http.Error(w, "Request body "+string(body)+" is not valid JSON.", http.StatusBadRequest) + return + } + + if len(put.Puts) > MaxNumValues { + http.Error(w, fmt.Sprintf("More keys than allowed: %d", MaxNumValues), http.StatusBadRequest) + return + } + + resp := response{ + Responses: make([]responseObject, len(put.Puts)), + } + for i, p := range put.Puts { + resp.Responses[i].UUID = fmt.Sprintf("UUID-%d", i+1) // deterministic for testing + if len(p.Value) > MaxValueLength { + http.Error(w, fmt.Sprintf("Value is larger than allowed size: %d", MaxValueLength), http.StatusBadRequest) + return + } + if len(p.Value) == 0 { + http.Error(w, "Missing value.", http.StatusBadRequest) + return + } + if p.Type != "xml" && p.Type != "json" { + http.Error(w, fmt.Sprintf("Type must be one of [\"json\", \"xml\"]. Found %v", p.Type), http.StatusBadRequest) + return + } + } + + b, err := json.Marshal(&resp) + if err != nil { + http.Error(w, "Failed to serialize UUIDs into JSON.", http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Write(b) +} + +func TestCacheVideoOnly(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(DummyPrebidCacheServer)) + defer server.Close() + + bids := make(pbs.PBSBidSlice, 0) + fbBid := pbs.PBSBid{ + BidID: "test_bidid0", + AdUnitCode: "test_adunitcode0", + BidderCode: "audienceNetwork", + Price: 2.00, + Adm: "fb_test_adm", + Width: 300, + Height: 250, + DealId: "2345", + CreativeMediaType: "video", + } + bids = append(bids, &fbBid) + anBid := pbs.PBSBid{ + BidID: "test_bidid1", + AdUnitCode: "test_adunitcode1", + BidderCode: "appnexus", + Price: 1.00, + Adm: "an_test_adm", + Width: 320, + Height: 50, + DealId: "1234", + CreativeMediaType: "banner", + } + bids = append(bids, &anBid) + rbBannerBid := pbs.PBSBid{ + BidID: "test_bidid2", + AdUnitCode: "test_adunitcode2", + BidderCode: "rubicon", + Price: 1.00, + Adm: "rb_banner_test_adm", + Width: 300, + Height: 250, + DealId: "7890", + CreativeMediaType: "banner", + } + bids = append(bids, &rbBannerBid) + rbVideoBid1 := pbs.PBSBid{ + BidID: "test_bidid3", + AdUnitCode: "test_adunitcode3", + BidderCode: "rubicon", + Price: 1.00, + Adm: "rb_video_test_adm1", + Width: 300, + Height: 250, + DealId: "7890", + CreativeMediaType: "video", + } + bids = append(bids, &rbVideoBid1) + rbVideoBid2 := pbs.PBSBid{ + BidID: "test_bidid4", + AdUnitCode: "test_adunitcode4", + BidderCode: "rubicon", + Price: 1.00, + Adm: "rb_video_test_adm2", + Width: 300, + Height: 250, + DealId: "7890", + CreativeMediaType: "video", + } + bids = append(bids, &rbVideoBid2) + + ctx := context.TODO() + v := viper.New() + config.SetupViper(v, "") + v.Set("gdpr.default_value", "0") + cfg, err := config.New(v) + if err != nil { + t.Fatal(err.Error()) + } + syncersByBidder := map[string]usersync.Syncer{} + gdprPerms := gdpr.NewPermissions(context.Background(), config.GDPR{ + HostVendorID: 0, + }, nil, nil) + prebid_cache_client.InitPrebidCache(server.URL) + var labels = &metrics.Labels{} + if err := cacheVideoOnly(bids, ctx, &auction{cfg: cfg, syncersByBidder: syncersByBidder, gdprPerms: gdprPerms, metricsEngine: &metricsConf.NilMetricsEngine{}}, labels); err != nil { + t.Errorf("Prebid cache failed: %v \n", err) + return + } + if bids[0].CacheID != "UUID-1" { + t.Errorf("UUID was '%s', should have been 'UUID-1'", bids[0].CacheID) + } + if bids[1].CacheID != "" { + t.Errorf("UUID was '%s', should have been empty", bids[1].CacheID) + } + if bids[2].CacheID != "" { + t.Errorf("UUID was '%s', should have been empty", bids[2].CacheID) + } + if bids[3].CacheID != "UUID-2" { + t.Errorf("First object UUID was '%s', should have been 'UUID-2'", bids[3].CacheID) + } + if bids[4].CacheID != "UUID-3" { + t.Errorf("Second object UUID was '%s', should have been 'UUID-3'", bids[4].CacheID) + } +} + +func TestShouldUsersync(t *testing.T) { + tests := []struct { + description string + signal string + allowHostCookies bool + allowBidderSync bool + wantAllow bool + }{ + { + description: "Don't sync - GDPR on, host cookies disallows and bidder sync disallows", + signal: "1", + allowHostCookies: false, + allowBidderSync: false, + wantAllow: false, + }, + { + description: "Don't sync - GDPR on, host cookies disallows and bidder sync allows", + signal: "1", + allowHostCookies: false, + allowBidderSync: true, + wantAllow: false, + }, + { + description: "Don't sync - GDPR on, host cookies allows and bidder sync disallows", + signal: "1", + allowHostCookies: true, + allowBidderSync: false, + wantAllow: false, + }, + { + description: "Sync - GDPR on, host cookies allows and bidder sync allows", + signal: "1", + allowHostCookies: true, + allowBidderSync: true, + wantAllow: true, + }, + { + description: "Don't sync - invalid GDPR signal, host cookies disallows and bidder sync disallows", + signal: "2", + allowHostCookies: false, + allowBidderSync: false, + wantAllow: false, + }, + } + + for _, tt := range tests { + deps := auction{ + gdprPerms: &auctionMockPermissions{ + allowBidderSync: tt.allowBidderSync, + allowHostCookies: tt.allowHostCookies, + }, + } + gdprPrivacyPolicy := gdprPolicy.Policy{ + Signal: tt.signal, + } + + allow := deps.shouldUsersync(context.Background(), openrtb_ext.BidderAdform, gdprPrivacyPolicy) + assert.Equal(t, tt.wantAllow, allow, tt.description) + } +} + +type auctionMockPermissions struct { + allowBidderSync bool + allowHostCookies bool + allowBidRequest bool + passGeo bool + passID bool +} + +func (m *auctionMockPermissions) HostCookiesAllowed(ctx context.Context, gdprSignal gdpr.Signal, consent string) (bool, error) { + return m.allowHostCookies, nil +} + +func (m *auctionMockPermissions) BidderSyncAllowed(ctx context.Context, bidder openrtb_ext.BidderName, gdprSignal gdpr.Signal, consent string) (bool, error) { + return m.allowBidderSync, nil +} + +func (m *auctionMockPermissions) AuctionActivitiesAllowed(ctx context.Context, bidder openrtb_ext.BidderName, PublisherID string, gdprSignal gdpr.Signal, consent string, weakVendorEnforcement bool) (allowBidRequest bool, passGeo bool, passID bool, err error) { + return m.allowBidRequest, m.passGeo, m.passID, nil +} + +func TestBidSizeValidate(t *testing.T) { + bids := make(pbs.PBSBidSlice, 0) + // bid1 will be rejected due to undefined size when adunit has multiple sizes + bid1 := pbs.PBSBid{ + BidID: "test_bidid1", + AdUnitCode: "test_adunitcode1", + BidderCode: "randNetwork", + Price: 1.05, + Adm: "test_adm", + // Width: 100, + // Height: 100, + CreativeMediaType: "banner", + } + bids = append(bids, &bid1) + // bid2 will be considered a normal ideal banner bid + bid2 := pbs.PBSBid{ + BidID: "test_bidid2", + AdUnitCode: "test_adunitcode2", + BidderCode: "randNetwork", + Price: 1.05, + Adm: "test_adm", + Width: 100, + Height: 100, + CreativeMediaType: "banner", + } + bids = append(bids, &bid2) + // bid3 will have it's dimensions set based on sizes defined in request + bid3 := pbs.PBSBid{ + BidID: "test_bidid3", + AdUnitCode: "test_adunitcode3", + BidderCode: "randNetwork", + Price: 1.05, + Adm: "test_adm", + //Width: 200, + //Height: 200, + CreativeMediaType: "banner", + } + + bids = append(bids, &bid3) + + // bid4 will be ignored as it's a video creative type + bid4 := pbs.PBSBid{ + BidID: "test_bidid_video", + AdUnitCode: "test_adunitcode_video", + BidderCode: "randNetwork", + Price: 1.05, + Adm: "test_adm", + //Width: 400, + //Height: 400, + CreativeMediaType: "video", + } + + bids = append(bids, &bid4) + + mybidder := pbs.PBSBidder{ + BidderCode: "randNetwork", + AdUnitCode: "test_adunitcode", + AdUnits: []pbs.PBSAdUnit{ + { + BidID: "test_bidid1", + Sizes: []openrtb2.Format{ + { + W: 350, + H: 250, + }, + { + W: 300, + H: 50, + }, + }, + Code: "test_adunitcode1", + MediaTypes: []pbs.MediaType{ + pbs.MEDIA_TYPE_BANNER, + }, + }, + { + BidID: "test_bidid2", + Sizes: []openrtb2.Format{ + { + W: 100, + H: 100, + }, + }, + Code: "test_adunitcode2", + MediaTypes: []pbs.MediaType{ + pbs.MEDIA_TYPE_BANNER, + }, + }, + { + BidID: "test_bidid3", + Sizes: []openrtb2.Format{ + { + W: 200, + H: 200, + }, + }, + Code: "test_adunitcode3", + MediaTypes: []pbs.MediaType{ + pbs.MEDIA_TYPE_BANNER, + }, + }, + { + BidID: "test_bidid_video", + Sizes: []openrtb2.Format{ + { + W: 400, + H: 400, + }, + }, + Code: "test_adunitcode_video", + MediaTypes: []pbs.MediaType{ + pbs.MEDIA_TYPE_VIDEO, + }, + }, + { + BidID: "test_bidid3", + Sizes: []openrtb2.Format{ + { + W: 150, + H: 150, + }, + }, + Code: "test_adunitcode_x", + MediaTypes: []pbs.MediaType{ + pbs.MEDIA_TYPE_BANNER, + }, + }, + { + BidID: "test_bidid_y", + Sizes: []openrtb2.Format{ + { + W: 150, + H: 150, + }, + }, + Code: "test_adunitcode_3", + MediaTypes: []pbs.MediaType{ + pbs.MEDIA_TYPE_BANNER, + }, + }, + }, + } + + bids = checkForValidBidSize(bids, &mybidder) + + testdata, _ := json.MarshalIndent(bids, "", " ") + if len(bids) != 3 { + t.Errorf("Detected returned bid list did not contain only 3 bid objects as expected.\nBelow is the contents of the bid list\n%v", string(testdata)) + } + + for _, bid := range bids { + if bid.BidID == "test_bidid3" { + if bid.Width == 0 && bid.Height == 0 { + t.Errorf("Detected the Width & Height attributes in test bidID %v were not set to the dimensions used from the mybidder object", bid.BidID) + } + } + } +} + +func TestWriteAuctionError(t *testing.T) { + recorder := httptest.NewRecorder() + writeAuctionError(recorder, "some error message", nil) + var resp pbs.PBSResponse + json.Unmarshal(recorder.Body.Bytes(), &resp) + + if len(resp.Bids) != 0 { + t.Error("Error responses should return no bids.") + } + if resp.Status != "some error message" { + t.Errorf("The response status should be the error message. Got: %s", resp.Status) + } + + if len(resp.BidderStatus) != 0 { + t.Errorf("Error responses shouldn't have any BidderStatus elements. Got %d", len(resp.BidderStatus)) + } +} + +func TestPanicRecovery(t *testing.T) { + testAuction := auction{ + cfg: nil, + syncersByBidder: nil, + gdprPerms: &auctionMockPermissions{ + allowBidderSync: false, + allowHostCookies: false, + }, + metricsEngine: &metricsConf.NilMetricsEngine{}, + } + panicker := func(bidder *pbs.PBSBidder, blables metrics.AdapterLabels) { + panic("panic!") + } + recovered := testAuction.recoverSafely(panicker) + recovered(nil, metrics.AdapterLabels{}) +} diff --git a/endpoints/openrtb2/auction.go b/endpoints/openrtb2/auction.go index eb6f17f2359..790a8d6482d 100644 --- a/endpoints/openrtb2/auction.go +++ b/endpoints/openrtb2/auction.go @@ -1572,6 +1572,15 @@ func setDoNotTrackImplicitly(httpReq *http.Request, bidReq *openrtb2.BidRequest) } } +// parseUserID gets this user's ID for the host machine, if it exists. +func parseUserID(cfg *config.Configuration, httpReq *http.Request) (string, bool) { + if hostCookie, err := httpReq.Cookie(cfg.HostCookie.CookieName); hostCookie != nil && err == nil { + return hostCookie.Value, true + } else { + return "", false + } +} + // Write(return) errors to the client, if any. Returns true if errors were found. func writeError(errs []error, w http.ResponseWriter, labels *metrics.Labels) bool { var rc bool = false diff --git a/endpoints/openrtb2/auction_test.go b/endpoints/openrtb2/auction_test.go index 20fdd56e74b..a259719ba8a 100644 --- a/endpoints/openrtb2/auction_test.go +++ b/endpoints/openrtb2/auction_test.go @@ -25,6 +25,7 @@ import ( "github.com/prebid/prebid-server/metrics" metricsConfig "github.com/prebid/prebid-server/metrics/config" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/stored_requests" "github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher" "github.com/prebid/prebid-server/util/iputil" @@ -3724,6 +3725,21 @@ func (cf mockStoredReqFetcher) FetchRequests(ctx context.Context, requestIDs []s return testStoredRequestData, testStoredImpData, nil } +var mockAccountData = map[string]json.RawMessage{ + "valid_acct": json.RawMessage(`{"disabled":false}`), +} + +type mockAccountFetcher struct { +} + +func (af mockAccountFetcher) FetchAccount(ctx context.Context, accountID string) (json.RawMessage, []error) { + if account, ok := mockAccountData[accountID]; ok { + return account, nil + } else { + return nil, []error{stored_requests.NotFoundError{accountID, "Account"}} + } +} + type mockExchange struct { lastRequest *openrtb2.BidRequest } diff --git a/endpoints/openrtb2/video_auction_test.go b/endpoints/openrtb2/video_auction_test.go index b7eb0b27c0b..3163cd9d323 100644 --- a/endpoints/openrtb2/video_auction_test.go +++ b/endpoints/openrtb2/video_auction_test.go @@ -1435,3 +1435,19 @@ var testVideoStoredImpData = map[string]json.RawMessage{ var testVideoStoredRequestData = map[string]json.RawMessage{ "80ce30c53c16e6ede735f123ef6e32361bfc7b22": json.RawMessage(`{"accountid": "11223344", "site": {"page": "mygame.foo.com"}}`), } + +func loadValidRequest(t *testing.T) *openrtb_ext.BidRequestVideo { + reqData, err := ioutil.ReadFile("sample-requests/video/video_valid_sample.json") + if err != nil { + t.Fatalf("Failed to fetch a valid request: %v", err) + } + + reqBody := getRequestPayload(t, reqData) + + reqVideo := &openrtb_ext.BidRequestVideo{} + if err := json.Unmarshal(reqBody, reqVideo); err != nil { + t.Fatalf("Failed to unmarshal the request: %v", err) + } + + return reqVideo +} diff --git a/exchange/auction.go b/exchange/auction.go index c8aff684e41..94e808801d9 100644 --- a/exchange/auction.go +++ b/exchange/auction.go @@ -311,6 +311,13 @@ func valOrZero(useVal bool, val int) int { return 0 } +func maybeMake(shouldMake bool, capacity int) []prebid_cache_client.Cacheable { + if shouldMake { + return make([]prebid_cache_client.Cacheable, 0, capacity) + } + return nil +} + func cacheTTL(impTTL int64, bidTTL int64, defTTL int64, buffer int64) (ttl int64) { if impTTL <= 0 && bidTTL <= 0 { // Only use default if there is no imp nor bid TTL provided. We don't want the default diff --git a/exchange/auction_test.go b/exchange/auction_test.go index 455ae5018e8..ee064fcb6f1 100644 --- a/exchange/auction_test.go +++ b/exchange/auction_test.go @@ -555,6 +555,12 @@ type pbsBid struct { Bidder openrtb_ext.BidderName `json:"bidder"` } +type cacheComparator struct { + freq int + expectedKeys []string + actualKeys []string +} + type mockCache struct { scheme string host string diff --git a/exchange/bidder_test.go b/exchange/bidder_test.go index 82f058514f7..da31658e32d 100644 --- a/exchange/bidder_test.go +++ b/exchange/bidder_test.go @@ -1795,6 +1795,7 @@ func (bidder *mixedMultiBidder) MakeBids(internalRequest *openrtb2.BidRequest, e } type bidRejector struct { + httpRequest *adapters.RequestData httpResponse *adapters.ResponseData } diff --git a/exchange/exchange_test.go b/exchange/exchange_test.go index b4fd270d023..9dcf9d66a7f 100644 --- a/exchange/exchange_test.go +++ b/exchange/exchange_test.go @@ -2997,7 +2997,7 @@ func TestCategoryMappingTwoBiddersOneBidEachNoCategorySamePrice(t *testing.T) { adapterBids[bidderNameApn1] = &seatBidApn1 adapterBids[bidderNameApn2] = &seatBidApn2 - bidCategory, _, rejections, err := applyCategoryMapping(nil, &requestExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) + bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &requestExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) assert.NoError(t, err, "Category mapping error should be empty") assert.Len(t, rejections, 1, "There should be 1 bid rejection message") diff --git a/exchange/targeting_test.go b/exchange/targeting_test.go index 48e054c7bda..8991a116624 100644 --- a/exchange/targeting_test.go +++ b/exchange/targeting_test.go @@ -8,14 +8,17 @@ import ( "testing" "time" - "github.com/prebid/prebid-server/adapters" + "github.com/mxmCherry/openrtb/v15/openrtb2" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/currency" + "github.com/prebid/prebid-server/gdpr" + + metricsConf "github.com/prebid/prebid-server/metrics/config" metricsConfig "github.com/prebid/prebid-server/metrics/config" - "github.com/prebid/prebid-server/openrtb_ext" - "github.com/mxmCherry/openrtb/v15/openrtb2" + "github.com/prebid/prebid-server/adapters" + "github.com/prebid/prebid-server/openrtb_ext" "github.com/stretchr/testify/assert" ) @@ -85,7 +88,7 @@ func runTargetingAuction(t *testing.T, mockBids map[openrtb_ext.BidderName][]*op ex := &exchange{ adapterMap: buildAdapterMap(mockBids, server.URL, server.Client()), - me: &metricsConfig.NilMetricsEngine{}, + me: &metricsConf.NilMetricsEngine{}, cache: &wellBehavedCache{}, cacheTime: time.Duration(0), gDPR: gdpr.AlwaysAllow{}, @@ -126,6 +129,14 @@ func runTargetingAuction(t *testing.T, mockBids map[openrtb_ext.BidderName][]*op return buildBidMap(bidResp.SeatBid, len(mockBids)) } +func buildBidderList(bids map[openrtb_ext.BidderName][]*openrtb2.Bid) []openrtb_ext.BidderName { + bidders := make([]openrtb_ext.BidderName, 0, len(bids)) + for name := range bids { + bidders = append(bidders, name) + } + return bidders +} + func buildAdapterMap(bids map[openrtb_ext.BidderName][]*openrtb2.Bid, mockServerURL string, client *http.Client) map[openrtb_ext.BidderName]adaptedBidder { adapterMap := make(map[openrtb_ext.BidderName]adaptedBidder, len(bids)) for bidder, bids := range bids { diff --git a/go.mod b/go.mod index e673d2218c7..347f36bcf86 100644 --- a/go.mod +++ b/go.mod @@ -8,17 +8,20 @@ require ( github.com/OneOfOne/xxhash v1.2.5 // indirect github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect + github.com/blang/semver v3.5.1+incompatible github.com/buger/jsonparser v1.1.1 github.com/cespare/xxhash v1.0.0 // indirect github.com/chasex/glog v0.0.0-20160217080310-c62392af379c github.com/coocood/freecache v1.0.1 github.com/docker/go-units v0.4.0 + github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 github.com/evanphx/json-patch v0.0.0-20180720181644-f195058310bd github.com/gofrs/uuid v3.2.0+incompatible github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/influxdata/influxdb v1.6.1 github.com/julienschmidt/httprouter v1.1.0 github.com/lib/pq v1.0.0 + github.com/magiconair/properties v1.8.5 github.com/mattn/go-colorable v0.1.2 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/mitchellh/copystructure v1.1.2 diff --git a/go.sum b/go.sum index bcab99e4ebf..31e6b8fc93f 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLM github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -85,6 +87,8 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= +github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/evanphx/json-patch v0.0.0-20180720181644-f195058310bd h1:biTJQdqouE5by89AAffXG8++TY+9Fsdrg5rinbt3tHk= github.com/evanphx/json-patch v0.0.0-20180720181644-f195058310bd/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= diff --git a/main.go b/main.go index 76fa64f77ef..6087c3d69dd 100644 --- a/main.go +++ b/main.go @@ -8,6 +8,7 @@ import ( "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/currency" + pbc "github.com/prebid/prebid-server/prebid_cache_client" "github.com/prebid/prebid-server/router" "github.com/prebid/prebid-server/server" "github.com/prebid/prebid-server/util/task" @@ -55,6 +56,8 @@ func serve(cfg *config.Configuration) error { return err } + pbc.InitPrebidCache(cfg.CacheURL.GetBaseURL()) + corsRouter := router.SupportCORS(r) server.Listen(cfg, router.NoCache{Handler: corsRouter}, router.Admin(currencyConverter, fetchingInterval), r.MetricsEngine) diff --git a/metrics/config/metrics.go b/metrics/config/metrics.go index 66849db5864..51ba8cafe2f 100644 --- a/metrics/config/metrics.go +++ b/metrics/config/metrics.go @@ -90,6 +90,13 @@ func (me *MultiMetricsEngine) RecordImps(implabels metrics.ImpLabels) { } } +// RecordImps for the legacy endpoint +func (me *MultiMetricsEngine) RecordLegacyImps(labels metrics.Labels, numImps int) { + for _, thisME := range *me { + thisME.RecordLegacyImps(labels, numImps) + } +} + // RecordRequestTime across all engines func (me *MultiMetricsEngine) RecordRequestTime(labels metrics.Labels, length time.Duration) { for _, thisME := range *me { @@ -271,6 +278,10 @@ func (me *NilMetricsEngine) RecordConnectionClose(success bool) { func (me *NilMetricsEngine) RecordImps(implabels metrics.ImpLabels) { } +// RecordLegacyImps as a noop +func (me *NilMetricsEngine) RecordLegacyImps(labels metrics.Labels, numImps int) { +} + // RecordRequestTime as a noop func (me *NilMetricsEngine) RecordRequestTime(labels metrics.Labels, length time.Duration) { } diff --git a/metrics/config/metrics_test.go b/metrics/config/metrics_test.go index e4227afda77..0d6bcdb922d 100644 --- a/metrics/config/metrics_test.go +++ b/metrics/config/metrics_test.go @@ -78,6 +78,7 @@ func TestMultiMetricsEngine(t *testing.T) { for i := 0; i < 5; i++ { metricsEngine.RecordRequest(labels) metricsEngine.RecordImps(impTypeLabels) + metricsEngine.RecordLegacyImps(labels, 2) metricsEngine.RecordRequestTime(labels, time.Millisecond*20) metricsEngine.RecordAdapterRequest(pubLabels) metricsEngine.RecordAdapterRequest(apnLabels) @@ -146,6 +147,7 @@ func TestMultiMetricsEngine(t *testing.T) { VerifyMetrics(t, "Request", goEngine.RequestStatuses[metrics.ReqTypeORTB2Web][metrics.RequestStatusOK].Count(), 5) VerifyMetrics(t, "ImpMeter", goEngine.ImpMeter.Count(), 8) + VerifyMetrics(t, "LegacyImpMeter", goEngine.LegacyImpMeter.Count(), 10) VerifyMetrics(t, "NoCookieMeter", goEngine.NoCookieMeter.Count(), 0) VerifyMetrics(t, "AdapterMetrics.Pubmatic.GotBidsMeter", goEngine.AdapterMetrics[openrtb_ext.BidderPubmatic].GotBidsMeter.Count(), 5) VerifyMetrics(t, "AdapterMetrics.Pubmatic.NoBidMeter", goEngine.AdapterMetrics[openrtb_ext.BidderPubmatic].NoBidMeter.Count(), 0) diff --git a/metrics/go_metrics.go b/metrics/go_metrics.go index c93f10602c7..615b83b8be9 100644 --- a/metrics/go_metrics.go +++ b/metrics/go_metrics.go @@ -18,6 +18,7 @@ type Metrics struct { ConnectionAcceptErrorMeter metrics.Meter ConnectionCloseErrorMeter metrics.Meter ImpMeter metrics.Meter + LegacyImpMeter metrics.Meter AppRequestMeter metrics.Meter NoCookieMeter metrics.Meter RequestTimer metrics.Timer @@ -100,6 +101,9 @@ type accountMetrics struct { adapterMetrics map[openrtb_ext.BidderName]*AdapterMetrics } +// Defining an "unknown" bidder +const unknownBidder openrtb_ext.BidderName = "unknown" + // NewBlankMetrics creates a new Metrics object with all blank metrics object. This may also be useful for // testing routines to ensure that no metrics are written anywhere. // @@ -118,6 +122,7 @@ func NewBlankMetrics(registry metrics.Registry, exchanges []openrtb_ext.BidderNa ConnectionAcceptErrorMeter: blankMeter, ConnectionCloseErrorMeter: blankMeter, ImpMeter: blankMeter, + LegacyImpMeter: blankMeter, AppRequestMeter: blankMeter, NoCookieMeter: blankMeter, RequestTimer: blankTimer, @@ -211,6 +216,7 @@ func NewMetrics(registry metrics.Registry, exchanges []openrtb_ext.BidderName, d newMetrics.ConnectionAcceptErrorMeter = metrics.GetOrRegisterMeter("connection_accept_errors", registry) newMetrics.ConnectionCloseErrorMeter = metrics.GetOrRegisterMeter("connection_close_errors", registry) newMetrics.ImpMeter = metrics.GetOrRegisterMeter("imps_requested", registry) + newMetrics.LegacyImpMeter = metrics.GetOrRegisterMeter("legacy_imps_requested", registry) newMetrics.ImpsTypeBanner = metrics.GetOrRegisterMeter("imp_banner", registry) newMetrics.ImpsTypeVideo = metrics.GetOrRegisterMeter("imp_video", registry) @@ -446,6 +452,10 @@ func (me *Metrics) RecordImps(labels ImpLabels) { } } +func (me *Metrics) RecordLegacyImps(labels Labels, numImps int) { + me.LegacyImpMeter.Mark(int64(numImps)) +} + func (me *Metrics) RecordConnectionAccept(success bool) { if success { me.ConnectionCounter.Inc(1) diff --git a/metrics/go_metrics_test.go b/metrics/go_metrics_test.go index dd2430d6b74..346a64a737f 100644 --- a/metrics/go_metrics_test.go +++ b/metrics/go_metrics_test.go @@ -36,6 +36,10 @@ func TestNewMetrics(t *testing.T) { ensureContains(t, registry, "prebid_cache_request_time.ok", m.PrebidCacheRequestTimerSuccess) ensureContains(t, registry, "prebid_cache_request_time.err", m.PrebidCacheRequestTimerError) + ensureContains(t, registry, "requests.ok.legacy", m.RequestStatuses[ReqTypeLegacy][RequestStatusOK]) + ensureContains(t, registry, "requests.badinput.legacy", m.RequestStatuses[ReqTypeLegacy][RequestStatusBadInput]) + ensureContains(t, registry, "requests.err.legacy", m.RequestStatuses[ReqTypeLegacy][RequestStatusErr]) + ensureContains(t, registry, "requests.networkerr.legacy", m.RequestStatuses[ReqTypeLegacy][RequestStatusNetworkErr]) ensureContains(t, registry, "requests.ok.openrtb2-web", m.RequestStatuses[ReqTypeORTB2Web][RequestStatusOK]) ensureContains(t, registry, "requests.badinput.openrtb2-web", m.RequestStatuses[ReqTypeORTB2Web][RequestStatusBadInput]) ensureContains(t, registry, "requests.err.openrtb2-web", m.RequestStatuses[ReqTypeORTB2Web][RequestStatusErr]) diff --git a/metrics/metrics.go b/metrics/metrics.go index 9d16143f0d4..af45f9b4f5a 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -382,6 +382,7 @@ type MetricsEngine interface { RecordConnectionClose(success bool) RecordRequest(labels Labels) // ignores adapter. only statusOk and statusErr fom status RecordImps(labels ImpLabels) // RecordImps across openRTB2 engines that support the 'Native' Imp Type + RecordLegacyImps(labels Labels, numImps int) // RecordImps for the legacy engine RecordRequestTime(labels Labels, length time.Duration) // ignores adapter. only statusOk and statusErr fom status RecordAdapterRequest(labels AdapterLabels) RecordAdapterConnections(adapterName openrtb_ext.BidderName, connWasReused bool, connWaitTime time.Duration) diff --git a/metrics/metrics_mock.go b/metrics/metrics_mock.go index c8d5311c3a4..b8ab23b768a 100644 --- a/metrics/metrics_mock.go +++ b/metrics/metrics_mock.go @@ -32,6 +32,11 @@ func (me *MetricsEngineMock) RecordImps(labels ImpLabels) { me.Called(labels) } +// RecordLegacyImps mock +func (me *MetricsEngineMock) RecordLegacyImps(labels Labels, numImps int) { + me.Called(labels, numImps) +} + // RecordRequestTime mock func (me *MetricsEngineMock) RecordRequestTime(labels Labels, length time.Duration) { me.Called(labels, length) diff --git a/metrics/prometheus/prometheus.go b/metrics/prometheus/prometheus.go index 53a5afabd53..52470369094 100644 --- a/metrics/prometheus/prometheus.go +++ b/metrics/prometheus/prometheus.go @@ -476,6 +476,10 @@ func (m *Metrics) RecordImps(labels metrics.ImpLabels) { }).Inc() } +func (m *Metrics) RecordLegacyImps(labels metrics.Labels, numImps int) { + m.impressionsLegacy.Add(float64(numImps)) +} + func (m *Metrics) RecordRequestTime(labels metrics.Labels, length time.Duration) { if labels.RequestStatus == metrics.RequestStatusOK { m.requestsTimer.With(prometheus.Labels{ diff --git a/metrics/prometheus/prometheus_test.go b/metrics/prometheus/prometheus_test.go index fc8abdb9d04..0fe852b81df 100644 --- a/metrics/prometheus/prometheus_test.go +++ b/metrics/prometheus/prometheus_test.go @@ -355,6 +355,16 @@ func TestImpressionsMetric(t *testing.T) { } } +func TestLegacyImpressionsMetric(t *testing.T) { + m := createMetricsForTesting() + + m.RecordLegacyImps(metrics.Labels{}, 42) + + expectedCount := float64(42) + assertCounterValue(t, "", "impressionsLegacy", m.impressionsLegacy, + expectedCount) +} + func TestRequestTimeMetric(t *testing.T) { requestType := metrics.ReqTypeORTB2Web performTest := func(m *Metrics, requestStatus metrics.RequestStatus, timeInMs float64) { @@ -1152,6 +1162,18 @@ func TestPrebidCacheRequestTimeMetric(t *testing.T) { assertHistogram(t, "Error", errorResult, errorExpectedCount, errorExpectedSum) } +func TestMetricAccumulationSpotCheck(t *testing.T) { + m := createMetricsForTesting() + + m.RecordLegacyImps(metrics.Labels{}, 1) + m.RecordLegacyImps(metrics.Labels{}, 2) + m.RecordLegacyImps(metrics.Labels{}, 3) + + expectedValue := float64(1 + 2 + 3) + assertCounterValue(t, "", "impressionsLegacy", m.impressionsLegacy, + expectedValue) +} + func TestRecordRequestQueueTimeMetric(t *testing.T) { performTest := func(m *Metrics, requestStatus bool, requestType metrics.RequestType, timeInSec float64) { m.RecordRequestQueueTime(requestStatus, requestType, time.Duration(timeInSec*float64(time.Second))) diff --git a/metrics/prometheus/type_conversion.go b/metrics/prometheus/type_conversion.go index d99f2f6f39b..4e2d1ff5ba1 100644 --- a/metrics/prometheus/type_conversion.go +++ b/metrics/prometheus/type_conversion.go @@ -104,6 +104,15 @@ func setUidStatusesAsString() []string { return valuesAsString } +func storedDataTypesAsString() []string { + values := metrics.StoredDataTypes() + valuesAsString := make([]string, len(values)) + for i, v := range values { + valuesAsString[i] = string(v) + } + return valuesAsString +} + func storedDataFetchTypesAsString() []string { values := metrics.StoredDataFetchTypes() valuesAsString := make([]string, len(values)) diff --git a/openrtb_ext/bidders.go b/openrtb_ext/bidders.go index 14a261e314e..7976451877c 100644 --- a/openrtb_ext/bidders.go +++ b/openrtb_ext/bidders.go @@ -12,6 +12,8 @@ import ( "github.com/xeipuuv/gojsonschema" ) +const schemaDirectory = "static/bidder-params" + // BidderName refers to a core bidder id or an alias id. type BidderName string diff --git a/openrtb_ext/imp_beachfront.go b/openrtb_ext/imp_beachfront.go index 9e65d54b82b..104ca8f9fb7 100644 --- a/openrtb_ext/imp_beachfront.go +++ b/openrtb_ext/imp_beachfront.go @@ -4,10 +4,10 @@ type ExtImpBeachfront struct { AppId string `json:"appId"` AppIds ExtImpBeachfrontAppIds `json:"appIds"` BidFloor float64 `json:"bidfloor"` - VideoResponseType string `json:"videoResponseType,omitempty"` + VideoResponseType string `json:"videoResponseType, omitempty"` } type ExtImpBeachfrontAppIds struct { - Video string `json:"video,omitempty"` - Banner string `json:"banner,omitempty"` + Video string `json:"video, omitempty"` + Banner string `json:"banner, omitempty"` } diff --git a/openrtb_ext/imp_nanointeractive.go b/openrtb_ext/imp_nanointeractive.go index 77e386237ac..28db5be0d07 100644 --- a/openrtb_ext/imp_nanointeractive.go +++ b/openrtb_ext/imp_nanointeractive.go @@ -3,8 +3,8 @@ package openrtb_ext // ExtImpNanoInteractive defines the contract for bidrequest.imp[i].ext.nanointeractive type ExtImpNanoInteractive struct { Pid string `json:"pid"` - Nq []string `json:"nq,omitempty"` - Category string `json:"category,omitempty"` - SubId string `json:"subId,omitempty"` - Ref string `json:"ref,omitempty"` + Nq []string `json:"nq, omitempty"` + Category string `json:"category, omitempty"` + SubId string `json:"subId, omitempty"` + Ref string `json:"ref, omitempty"` } diff --git a/pbs/pbsrequest.go b/pbs/pbsrequest.go new file mode 100644 index 00000000000..c05b9a8c00d --- /dev/null +++ b/pbs/pbsrequest.go @@ -0,0 +1,403 @@ +package pbs + +import ( + "encoding/json" + "fmt" + "math/rand" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/prebid/prebid-server/cache" + "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/stored_requests" + "github.com/prebid/prebid-server/usersync" + "github.com/prebid/prebid-server/util/httputil" + "github.com/prebid/prebid-server/util/iputil" + + "github.com/blang/semver" + "github.com/buger/jsonparser" + "github.com/golang/glog" + "github.com/mxmCherry/openrtb/v15/openrtb2" + "golang.org/x/net/publicsuffix" +) + +const MAX_BIDDERS = 8 + +type MediaType byte + +const ( + MEDIA_TYPE_BANNER MediaType = iota + MEDIA_TYPE_VIDEO +) + +type ConfigCache interface { + LoadConfig(string) ([]Bids, error) +} + +type Bids struct { + BidderCode string `json:"bidder"` + BidID string `json:"bid_id"` + Params json.RawMessage `json:"params"` +} + +// Structure for holding video-specific information +type PBSVideo struct { + //Content MIME types supported. Popular MIME types may include “video/x-ms-wmv” for Windows Media and “video/x-flv” for Flash Video. + Mimes []string `json:"mimes,omitempty"` + + //Minimum video ad duration in seconds. + Minduration int64 `json:"minduration,omitempty"` + + // Maximum video ad duration in seconds. + Maxduration int64 `json:"maxduration,omitempty"` + + //Indicates the start delay in seconds for pre-roll, mid-roll, or post-roll ad placements. + Startdelay int64 `json:"startdelay,omitempty"` + + // Indicates if the player will allow the video to be skipped ( 0 = no, 1 = yes). + Skippable int `json:"skippable,omitempty"` + + // Playback method code Description + // 1 - Initiates on Page Load with Sound On + // 2 - Initiates on Page Load with Sound Off by Default + // 3 - Initiates on Click with Sound On + // 4 - Initiates on Mouse-Over with Sound On + // 5 - Initiates on Entering Viewport with Sound On + // 6 - Initiates on Entering Viewport with Sound Off by Default + PlaybackMethod int8 `json:"playback_method,omitempty"` + + //protocols as specified in ORTB 5.8 + // 1 VAST 1.0 + // 2 VAST 2.0 + // 3 VAST 3.0 + // 4 VAST 1.0 Wrapper + // 5 VAST 2.0 Wrapper + // 6 VAST 3.0 Wrapper + // 7 VAST 4.0 + // 8 VAST 4.0 Wrapper + // 9 DAAST 1.0 + // 10 DAAST 1.0 Wrapper + Protocols []int8 `json:"protocols,omitempty"` +} + +type AdUnit struct { + Code string `json:"code"` + TopFrame int8 `json:"is_top_frame"` + Sizes []openrtb2.Format `json:"sizes"` + Bids []Bids `json:"bids"` + ConfigID string `json:"config_id"` + MediaTypes []string `json:"media_types"` + Instl int8 `json:"instl"` + Video PBSVideo `json:"video"` +} + +type PBSAdUnit struct { + Sizes []openrtb2.Format + TopFrame int8 + Code string + BidID string + Params json.RawMessage + Video PBSVideo + MediaTypes []MediaType + Instl int8 +} + +func ParseMediaType(s string) (MediaType, error) { + mediaTypes := map[string]MediaType{"BANNER": MEDIA_TYPE_BANNER, "VIDEO": MEDIA_TYPE_VIDEO} + t, ok := mediaTypes[strings.ToUpper(s)] + if !ok { + return 0, fmt.Errorf("Invalid MediaType %s", s) + } + return t, nil +} + +type SDK struct { + Version string `json:"version"` + Source string `json:"source"` + Platform string `json:"platform"` +} + +type PBSBidder struct { + BidderCode string `json:"bidder"` + AdUnitCode string `json:"ad_unit,omitempty"` // for index to dedup responses + ResponseTime int `json:"response_time_ms,omitempty"` + NumBids int `json:"num_bids,omitempty"` + Error string `json:"error,omitempty"` + NoCookie bool `json:"no_cookie,omitempty"` + NoBid bool `json:"no_bid,omitempty"` + UsersyncInfo *UsersyncInfo `json:"usersync,omitempty"` + Debug []*BidderDebug `json:"debug,omitempty"` + + AdUnits []PBSAdUnit `json:"-"` +} + +type UsersyncInfo struct { + URL string `json:"url,omitempty"` + Type string `json:"type,omitempty"` + SupportCORS bool `json:"supportCORS,omitempty"` +} + +func (bidder *PBSBidder) LookupBidID(Code string) string { + for _, unit := range bidder.AdUnits { + if unit.Code == Code { + return unit.BidID + } + } + return "" +} + +func (bidder *PBSBidder) LookupAdUnit(Code string) (unit *PBSAdUnit) { + for _, unit := range bidder.AdUnits { + if unit.Code == Code { + return &unit + } + } + return nil +} + +type PBSRequest struct { + AccountID string `json:"account_id"` + Tid string `json:"tid"` + CacheMarkup int8 `json:"cache_markup"` + SortBids int8 `json:"sort_bids"` + MaxKeyLength int8 `json:"max_key_length"` + Secure int8 `json:"secure"` + TimeoutMillis int64 `json:"timeout_millis"` + AdUnits []AdUnit `json:"ad_units"` + IsDebug bool `json:"is_debug"` + App *openrtb2.App `json:"app"` + Device *openrtb2.Device `json:"device"` + PBSUser json.RawMessage `json:"user"` + SDK *SDK `json:"sdk"` + + // internal + Bidders []*PBSBidder `json:"-"` + User *openrtb2.User `json:"-"` + Cookie *usersync.Cookie `json:"-"` + Url string `json:"-"` + Domain string `json:"-"` + Regs *openrtb2.Regs `json:"-"` + Start time.Time +} + +func ConfigGet(cache cache.Cache, id string) ([]Bids, error) { + conf, err := cache.Config().Get(id) + if err != nil { + return nil, err + } + + bids := make([]Bids, 0) + err = json.Unmarshal([]byte(conf), &bids) + if err != nil { + return nil, err + } + + return bids, nil +} + +func ParseMediaTypes(types []string) []MediaType { + var mtypes []MediaType + mtmap := make(map[MediaType]bool) + + if types == nil { + mtypes = append(mtypes, MEDIA_TYPE_BANNER) + } else { + for _, t := range types { + mt, er := ParseMediaType(t) + if er != nil { + glog.Infof("Invalid media type: %s", er) + } else { + if !mtmap[mt] { + mtypes = append(mtypes, mt) + mtmap[mt] = true + } + } + } + if len(mtypes) == 0 { + mtypes = append(mtypes, MEDIA_TYPE_BANNER) + } + } + return mtypes +} + +var ipv4Validator iputil.IPValidator = iputil.VersionIPValidator{Version: iputil.IPv4} + +func ParsePBSRequest(r *http.Request, cfg *config.AuctionTimeouts, cache cache.Cache, hostCookieConfig *config.HostCookie) (*PBSRequest, error) { + defer r.Body.Close() + + pbsReq := &PBSRequest{} + err := json.NewDecoder(r.Body).Decode(&pbsReq) + if err != nil { + return nil, err + } + pbsReq.Start = time.Now() + + if len(pbsReq.AdUnits) == 0 { + return nil, fmt.Errorf("No ad units specified") + } + + pbsReq.TimeoutMillis = int64(cfg.LimitAuctionTimeout(time.Duration(pbsReq.TimeoutMillis)*time.Millisecond) / time.Millisecond) + + if pbsReq.Device == nil { + pbsReq.Device = &openrtb2.Device{} + } + if ip, _ := httputil.FindIP(r, ipv4Validator); ip != nil { + pbsReq.Device.IP = ip.String() + } + + if pbsReq.SDK == nil { + pbsReq.SDK = &SDK{} + } + + // Early versions of prebid mobile are sending requests with gender indicated by numbers, + // those traffic can't be parsed by latest Prebid Server after the change of gender to use string so clients using early versions can't be monetized. + // To handle those traffic, adding a check here to ignore the sent gender for versions lower than 0.0.2. + v1, err := semver.Make(pbsReq.SDK.Version) + v2, err := semver.Make("0.0.2") + if v1.Compare(v2) >= 0 && pbsReq.PBSUser != nil { + err = json.Unmarshal([]byte(pbsReq.PBSUser), &pbsReq.User) + if err != nil { + return nil, err + } + } + + if pbsReq.User == nil { + pbsReq.User = &openrtb2.User{} + } + + // use client-side data for web requests + if pbsReq.App == nil { + pbsReq.Cookie = usersync.ParseCookieFromRequest(r, hostCookieConfig) + + pbsReq.Device.UA = r.Header.Get("User-Agent") + + pbsReq.Url = r.Header.Get("Referer") // must be specified in the header + // TODO: this should explicitly put us in test mode + if r.FormValue("url_override") != "" { + pbsReq.Url = r.FormValue("url_override") + } + if strings.Index(pbsReq.Url, "http") == -1 { + pbsReq.Url = fmt.Sprintf("http://%s", pbsReq.Url) + } + + url, err := url.Parse(pbsReq.Url) + if err != nil { + return nil, fmt.Errorf("Invalid URL '%s': %v", pbsReq.Url, err) + } + + if url.Host == "" { + return nil, fmt.Errorf("Host not found from URL '%v'", url) + } + + pbsReq.Domain, err = publicsuffix.EffectiveTLDPlusOne(url.Host) + if err != nil { + return nil, fmt.Errorf("Invalid URL '%s': %v", url.Host, err) + } + } + + if r.FormValue("debug") == "1" { + pbsReq.IsDebug = true + } + + if httputil.IsSecure(r) { + pbsReq.Secure = 1 + } + + pbsReq.Bidders = make([]*PBSBidder, 0, MAX_BIDDERS) + + for _, unit := range pbsReq.AdUnits { + bidders := unit.Bids + if unit.ConfigID != "" { + bidders, err = ConfigGet(cache, unit.ConfigID) + if err != nil { + if _, notFound := err.(*stored_requests.NotFoundError); !notFound { + glog.Warningf("Failed to load config '%s' from cache: %v", unit.ConfigID, err) + } + // proceed with other ad units + continue + } + } + + if glog.V(2) { + glog.Infof("Ad unit %s has %d bidders for %d sizes", unit.Code, len(bidders), len(unit.Sizes)) + } + + mtypes := ParseMediaTypes(unit.MediaTypes) + for _, b := range bidders { + var bidder *PBSBidder + for _, pb := range pbsReq.Bidders { + if pb.BidderCode == b.BidderCode { + bidder = pb + } + } + + if bidder == nil { + bidder = &PBSBidder{BidderCode: b.BidderCode} + pbsReq.Bidders = append(pbsReq.Bidders, bidder) + } + if b.BidID == "" { + b.BidID = fmt.Sprintf("%d", rand.Int63()) + } + + pau := PBSAdUnit{ + Sizes: unit.Sizes, + TopFrame: unit.TopFrame, + Code: unit.Code, + Instl: unit.Instl, + Params: b.Params, + BidID: b.BidID, + MediaTypes: mtypes, + Video: unit.Video, + } + + bidder.AdUnits = append(bidder.AdUnits, pau) + } + } + + return pbsReq, nil +} + +func (req PBSRequest) Elapsed() int { + return int(time.Since(req.Start) / 1000000) +} + +func (p PBSRequest) String() string { + b, _ := json.MarshalIndent(p, "", " ") + return string(b) +} + +// parses the "Regs.ext.gdpr" from the request, if it exists. Otherwise returns an empty string. +func (req *PBSRequest) ParseGDPR() string { + if req == nil || req.Regs == nil || len(req.Regs.Ext) == 0 { + return "" + } + val, err := jsonparser.GetInt(req.Regs.Ext, "gdpr") + if err != nil { + return "" + } + gdpr := strconv.Itoa(int(val)) + + return gdpr +} + +// parses the "User.ext.consent" from the request, if it exists. Otherwise returns an empty string. +func (req *PBSRequest) ParseConsent() string { + if req == nil || req.User == nil { + return "" + } + return parseString(req.User.Ext, "consent") +} + +func parseString(data []byte, key string) string { + if len(data) == 0 { + return "" + } + val, err := jsonparser.GetString(data, key) + if err != nil { + return "" + } + return val +} diff --git a/pbs/pbsrequest_test.go b/pbs/pbsrequest_test.go new file mode 100644 index 00000000000..52cd6153323 --- /dev/null +++ b/pbs/pbsrequest_test.go @@ -0,0 +1,735 @@ +package pbs + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/magiconair/properties/assert" + "github.com/prebid/prebid-server/cache/dummycache" + "github.com/prebid/prebid-server/config" +) + +const mimeVideoMp4 = "video/mp4" +const mimeVideoFlv = "video/x-flv" + +func TestParseMediaTypes(t *testing.T) { + types1 := []string{"Banner"} + t1 := ParseMediaTypes(types1) + assert.Equal(t, len(t1), 1) + assert.Equal(t, t1[0], MEDIA_TYPE_BANNER) + + types2 := []string{"Banner", "Video"} + t2 := ParseMediaTypes(types2) + assert.Equal(t, len(t2), 2) + assert.Equal(t, t2[0], MEDIA_TYPE_BANNER) + assert.Equal(t, t2[1], MEDIA_TYPE_VIDEO) + + types3 := []string{"Banner", "Vo"} + t3 := ParseMediaTypes(types3) + assert.Equal(t, len(t3), 1) + assert.Equal(t, t3[0], MEDIA_TYPE_BANNER) +} + +func TestParseSimpleRequest(t *testing.T) { + body := []byte(`{ + "tid": "abcd", + "ad_units": [ + { + "code": "first", + "sizes": [{"w": 300, "h": 250}], + "bids": [ + { + "bidder": "ix" + }, + { + "bidder": "appnexus" + } + ] + }, + { + "code": "second", + "sizes": [{"w": 728, "h": 90}], + "media_types" :["banner", "video"], + "video" : { + "mimes" : ["video/mp4", "video/x-flv"] + }, + "bids": [ + { + "bidder": "ix" + }, + { + "bidder": "appnexus" + } + ] + } + + ] + } + `) + r := httptest.NewRequest("POST", "/auction", bytes.NewBuffer(body)) + r.Header.Add("Referer", "http://nytimes.com/cool.html") + d, _ := dummycache.New() + hcc := config.HostCookie{} + + pbs_req, err := ParsePBSRequest(r, &config.AuctionTimeouts{ + Default: 2000, + Max: 2000, + }, d, &hcc) + if err != nil { + t.Fatalf("Parse simple request failed: %v", err) + } + if pbs_req.Tid != "abcd" { + t.Errorf("Parse TID failed") + } + if len(pbs_req.AdUnits) != 2 { + t.Errorf("Parse ad units failed") + } + + // see if our internal representation is intact + if len(pbs_req.Bidders) != 2 { + t.Fatalf("Should have two bidders not %d", len(pbs_req.Bidders)) + } + if pbs_req.Bidders[0].BidderCode != "ix" { + t.Errorf("First bidder not index") + } + if len(pbs_req.Bidders[0].AdUnits) != 2 { + t.Errorf("Index bidder should have 2 ad unit") + } + if pbs_req.Bidders[1].BidderCode != "appnexus" { + t.Errorf("Second bidder not appnexus") + } + if len(pbs_req.Bidders[1].AdUnits) != 2 { + t.Errorf("AppNexus bidder should have 2 ad unit") + } + if pbs_req.Bidders[1].AdUnits[0].BidID == "" { + t.Errorf("ID should have been generated for empty BidID") + } + if pbs_req.AdUnits[1].MediaTypes[0] != "banner" { + t.Errorf("Instead of banner MediaType received %s", pbs_req.AdUnits[1].MediaTypes[0]) + } + if pbs_req.AdUnits[1].MediaTypes[1] != "video" { + t.Errorf("Instead of video MediaType received %s", pbs_req.AdUnits[1].MediaTypes[0]) + } + if pbs_req.AdUnits[1].Video.Mimes[0] != mimeVideoMp4 { + t.Errorf("Instead of video/mp4 mimes received %s", pbs_req.AdUnits[1].Video.Mimes) + } + if pbs_req.AdUnits[1].Video.Mimes[1] != mimeVideoFlv { + t.Errorf("Instead of video/flv mimes received %s", pbs_req.AdUnits[1].Video.Mimes) + } + +} + +func TestHeaderParsing(t *testing.T) { + body := []byte(`{ + "tid": "abcd", + "ad_units": [ + { + "code": "first", + "sizes": [{"w": 300, "h": 250}], + "bidders": [ + { + "bidder": "ix", + "params": { + "id": "417", + "siteID": "test-site" + } + } + ] + } + ] + } + `) + r := httptest.NewRequest("POST", "/auction", bytes.NewBuffer(body)) + r.Header.Add("Referer", "http://nytimes.com/cool.html") + r.Header.Add("User-Agent", "Mozilla/") + d, _ := dummycache.New() + hcc := config.HostCookie{} + + d.Config().Set("dummy", dummyConfig) + + pbs_req, err := ParsePBSRequest(r, &config.AuctionTimeouts{ + Default: 2000, + Max: 2000, + }, d, &hcc) + if err != nil { + t.Fatalf("Parse simple request failed") + } + if pbs_req.Url != "http://nytimes.com/cool.html" { + t.Errorf("Failed to pull URL from referrer") + } + if pbs_req.Domain != "nytimes.com" { + t.Errorf("Failed to parse TLD from referrer: %s not nytimes.com", pbs_req.Domain) + } + if pbs_req.Device.UA != "Mozilla/" { + t.Errorf("Failed to pull User-Agent from referrer") + } +} + +var dummyConfig = ` +[ + { + "bidder": "ix", + "bid_id": "22222222", + "params": { + "id": "4", + "siteID": "186774", + "timeout": "10000" + } + + }, + { + "bidder": "audienceNetwork", + "bid_id": "22222225", + "params": { + } + }, + { + "bidder": "pubmatic", + "bid_id": "22222223", + "params": { + "publisherId": "156009", + "adSlot": "39620189@728x90" + } + }, + { + "bidder": "appnexus", + "bid_id": "22222224", + "params": { + "placementId": "1" + } + } + ] + ` + +func TestParseConfig(t *testing.T) { + body := []byte(`{ + "tid": "abcd", + "ad_units": [ + { + "code": "first", + "sizes": [{"w": 300, "h": 250}], + "bids": [ + { + "bidder": "ix" + }, + { + "bidder": "appnexus" + } + ] + }, + { + "code": "second", + "sizes": [{"w": 728, "h": 90}], + "config_id": "abcd" + } + ] + } + `) + r := httptest.NewRequest("POST", "/auction", bytes.NewBuffer(body)) + r.Header.Add("Referer", "http://nytimes.com/cool.html") + d, _ := dummycache.New() + hcc := config.HostCookie{} + + d.Config().Set("dummy", dummyConfig) + + pbs_req, err := ParsePBSRequest(r, &config.AuctionTimeouts{ + Default: 2000, + Max: 2000, + }, d, &hcc) + if err != nil { + t.Fatalf("Parse simple request failed: %v", err) + } + if pbs_req.Tid != "abcd" { + t.Errorf("Parse TID failed") + } + if len(pbs_req.AdUnits) != 2 { + t.Errorf("Parse ad units failed") + } + + // see if our internal representation is intact + if len(pbs_req.Bidders) != 4 { + t.Fatalf("Should have 4 bidders not %d", len(pbs_req.Bidders)) + } + if pbs_req.Bidders[0].BidderCode != "ix" { + t.Errorf("First bidder not index") + } + if len(pbs_req.Bidders[0].AdUnits) != 2 { + t.Errorf("Index bidder should have 1 ad unit") + } + if pbs_req.Bidders[1].BidderCode != "appnexus" { + t.Errorf("Second bidder not appnexus") + } + if len(pbs_req.Bidders[1].AdUnits) != 2 { + t.Errorf("AppNexus bidder should have 2 ad unit") + } +} + +func TestParseMobileRequestFirstVersion(t *testing.T) { + body := []byte(`{ + "max_key_length":20, + "user":{ + "gender":0, + "buyeruid":"test_buyeruid" + }, + "prebid_version":"0.21.0-pre", + "sort_bids":1, + "ad_units":[ + { + "sizes":[ + { + "w":300, + "h":250 + } + ], + "config_id":"ad5ffb41-3492-40f3-9c25-ade093eb4e5f", + "code":"5d748364ee9c46a2b112892fc3551b6f" + } + ], + "cache_markup":1, + "app":{ + "bundle":"AppNexus.PrebidMobileDemo", + "ver":"0.0.1" + }, + "sdk":{ + "version":"0.0.1", + "platform":"iOS", + "source":"prebid-mobile" + }, + "device":{ + "ifa":"test_device_ifa", + "osv":"9.3.5", + "os":"iOS", + "make":"Apple", + "model":"iPhone6,1" + }, + "tid":"abcd", + "account_id":"aecd6ef7-b992-4e99-9bb8-65e2d984e1dd" + } + `) + r := httptest.NewRequest("POST", "/auction", bytes.NewBuffer(body)) + d, _ := dummycache.New() + hcc := config.HostCookie{} + + pbs_req, err := ParsePBSRequest(r, &config.AuctionTimeouts{ + Default: 2000, + Max: 2000, + }, d, &hcc) + if err != nil { + t.Fatalf("Parse simple request failed: %v", err) + } + if pbs_req.Tid != "abcd" { + t.Errorf("Parse TID failed") + } + if len(pbs_req.AdUnits) != 1 { + t.Errorf("Parse ad units failed") + } + // We are expecting all user fields to be nil. We don't parse user on v0.0.1 of prebid mobile + if pbs_req.User.BuyerUID != "" { + t.Errorf("Parse user buyeruid failed %s", pbs_req.User.BuyerUID) + } + if pbs_req.User.Gender != "" { + t.Errorf("Parse user gender failed %s", pbs_req.User.Gender) + } + if pbs_req.User.Yob != 0 { + t.Errorf("Parse user year of birth failed %d", pbs_req.User.Yob) + } + if pbs_req.User.ID != "" { + t.Errorf("Parse user id failed %s", pbs_req.User.ID) + } + + if pbs_req.App.Bundle != "AppNexus.PrebidMobileDemo" { + t.Errorf("Parse app bundle failed") + } + if pbs_req.App.Ver != "0.0.1" { + t.Errorf("Parse app version failed") + } + + if pbs_req.Device.IFA != "test_device_ifa" { + t.Errorf("Parse device ifa failed") + } + if pbs_req.Device.OSV != "9.3.5" { + t.Errorf("Parse device osv failed") + } + if pbs_req.Device.OS != "iOS" { + t.Errorf("Parse device os failed") + } + if pbs_req.Device.Make != "Apple" { + t.Errorf("Parse device make failed") + } + if pbs_req.Device.Model != "iPhone6,1" { + t.Errorf("Parse device model failed") + } +} + +func TestParseMobileRequest(t *testing.T) { + body := []byte(`{ + "max_key_length":20, + "user":{ + "gender":"F", + "buyeruid":"test_buyeruid", + "yob":2000, + "id":"testid" + }, + "prebid_version":"0.21.0-pre", + "sort_bids":1, + "ad_units":[ + { + "sizes":[ + { + "w":300, + "h":250 + } + ], + "config_id":"ad5ffb41-3492-40f3-9c25-ade093eb4e5f", + "code":"5d748364ee9c46a2b112892fc3551b6f" + } + ], + "cache_markup":1, + "app":{ + "bundle":"AppNexus.PrebidMobileDemo", + "ver":"0.0.2" + }, + "sdk":{ + "version":"0.0.2", + "platform":"iOS", + "source":"prebid-mobile" + }, + "device":{ + "ifa":"test_device_ifa", + "osv":"9.3.5", + "os":"iOS", + "make":"Apple", + "model":"iPhone6,1" + }, + "tid":"abcd", + "account_id":"aecd6ef7-b992-4e99-9bb8-65e2d984e1dd" + } + `) + r := httptest.NewRequest("POST", "/auction", bytes.NewBuffer(body)) + d, _ := dummycache.New() + hcc := config.HostCookie{} + + pbs_req, err := ParsePBSRequest(r, &config.AuctionTimeouts{ + Default: 2000, + Max: 2000, + }, d, &hcc) + if err != nil { + t.Fatalf("Parse simple request failed: %v", err) + } + if pbs_req.Tid != "abcd" { + t.Errorf("Parse TID failed") + } + if len(pbs_req.AdUnits) != 1 { + t.Errorf("Parse ad units failed") + } + + if pbs_req.User.BuyerUID != "test_buyeruid" { + t.Errorf("Parse user buyeruid failed") + } + if pbs_req.User.Gender != "F" { + t.Errorf("Parse user gender failed") + } + if pbs_req.User.Yob != 2000 { + t.Errorf("Parse user year of birth failed") + } + if pbs_req.User.ID != "testid" { + t.Errorf("Parse user id failed") + } + if pbs_req.App.Bundle != "AppNexus.PrebidMobileDemo" { + t.Errorf("Parse app bundle failed") + } + if pbs_req.App.Ver != "0.0.2" { + t.Errorf("Parse app version failed") + } + + if pbs_req.Device.IFA != "test_device_ifa" { + t.Errorf("Parse device ifa failed") + } + if pbs_req.Device.OSV != "9.3.5" { + t.Errorf("Parse device osv failed") + } + if pbs_req.Device.OS != "iOS" { + t.Errorf("Parse device os failed") + } + if pbs_req.Device.Make != "Apple" { + t.Errorf("Parse device make failed") + } + if pbs_req.Device.Model != "iPhone6,1" { + t.Errorf("Parse device model failed") + } + if pbs_req.SDK.Version != "0.0.2" { + t.Errorf("Parse sdk version failed") + } + if pbs_req.SDK.Source != "prebid-mobile" { + t.Errorf("Parse sdk source failed") + } + if pbs_req.SDK.Platform != "iOS" { + t.Errorf("Parse sdk platform failed") + } + if pbs_req.Device.IP == "" { + t.Errorf("Parse device ip failed %s", pbs_req.Device.IP) + } +} + +func TestParseMalformedMobileRequest(t *testing.T) { + body := []byte(`{ + "max_key_length":20, + "user":{ + "gender":0, + "buyeruid":"test_buyeruid" + }, + "prebid_version":"0.21.0-pre", + "sort_bids":1, + "ad_units":[ + { + "sizes":[ + { + "w":300, + "h":250 + } + ], + "config_id":"ad5ffb41-3492-40f3-9c25-ade093eb4e5f", + "code":"5d748364ee9c46a2b112892fc3551b6f" + } + ], + "cache_markup":1, + "app":{ + "bundle":"AppNexus.PrebidMobileDemo", + "ver":"0.0.1" + }, + "device":{ + "ifa":"test_device_ifa", + "osv":"9.3.5", + "os":"iOS", + "make":"Apple", + "model":"iPhone6,1" + }, + "tid":"abcd", + "account_id":"aecd6ef7-b992-4e99-9bb8-65e2d984e1dd" + } + `) + r := httptest.NewRequest("POST", "/auction", bytes.NewBuffer(body)) + d, _ := dummycache.New() + hcc := config.HostCookie{} + + pbs_req, err := ParsePBSRequest(r, &config.AuctionTimeouts{ + Default: 2000, + Max: 2000, + }, d, &hcc) + if err != nil { + t.Fatalf("Parse simple request failed: %v", err) + } + if pbs_req.Tid != "abcd" { + t.Errorf("Parse TID failed") + } + if len(pbs_req.AdUnits) != 1 { + t.Errorf("Parse ad units failed") + } + // We are expecting all user fields to be nil. Since no SDK version is passed in + if pbs_req.User.BuyerUID != "" { + t.Errorf("Parse user buyeruid failed %s", pbs_req.User.BuyerUID) + } + if pbs_req.User.Gender != "" { + t.Errorf("Parse user gender failed %s", pbs_req.User.Gender) + } + if pbs_req.User.Yob != 0 { + t.Errorf("Parse user year of birth failed %d", pbs_req.User.Yob) + } + if pbs_req.User.ID != "" { + t.Errorf("Parse user id failed %s", pbs_req.User.ID) + } + + if pbs_req.App.Bundle != "AppNexus.PrebidMobileDemo" { + t.Errorf("Parse app bundle failed") + } + if pbs_req.App.Ver != "0.0.1" { + t.Errorf("Parse app version failed") + } + + if pbs_req.Device.IFA != "test_device_ifa" { + t.Errorf("Parse device ifa failed") + } + if pbs_req.Device.OSV != "9.3.5" { + t.Errorf("Parse device osv failed") + } + if pbs_req.Device.OS != "iOS" { + t.Errorf("Parse device os failed") + } + if pbs_req.Device.Make != "Apple" { + t.Errorf("Parse device make failed") + } + if pbs_req.Device.Model != "iPhone6,1" { + t.Errorf("Parse device model failed") + } +} + +func TestParseRequestWithInstl(t *testing.T) { + body := []byte(`{ + "max_key_length":20, + "user":{ + "gender":"F", + "buyeruid":"test_buyeruid", + "yob":2000, + "id":"testid" + }, + "prebid_version":"0.21.0-pre", + "sort_bids":1, + "ad_units":[ + { + "sizes":[ + { + "w":300, + "h":250 + } + ], + "bids": [ + { + "bidder": "ix" + }, + { + "bidder": "appnexus" + } + ], + "code":"5d748364ee9c46a2b112892fc3551b6f", + "instl": 1 + } + ], + "cache_markup":1, + "app":{ + "bundle":"AppNexus.PrebidMobileDemo", + "ver":"0.0.2" + }, + "sdk":{ + "version":"0.0.2", + "platform":"iOS", + "source":"prebid-mobile" + }, + "device":{ + "ifa":"test_device_ifa", + "osv":"9.3.5", + "os":"iOS", + "make":"Apple", + "model":"iPhone6,1" + }, + "tid":"abcd", + "account_id":"aecd6ef7-b992-4e99-9bb8-65e2d984e1dd" + } + `) + r := httptest.NewRequest("POST", "/auction", bytes.NewBuffer(body)) + d, _ := dummycache.New() + hcc := config.HostCookie{} + + pbs_req, err := ParsePBSRequest(r, &config.AuctionTimeouts{ + Default: 2000, + Max: 2000, + }, d, &hcc) + if err != nil { + t.Fatalf("Parse simple request failed: %v", err) + } + if len(pbs_req.Bidders) != 2 { + t.Errorf("Should have 2 bidders. ") + } + if pbs_req.Bidders[0].AdUnits[0].Instl != 1 { + t.Errorf("Parse instl failed.") + } + if pbs_req.Bidders[1].AdUnits[0].Instl != 1 { + t.Errorf("Parse instl failed.") + } + +} + +func TestTimeouts(t *testing.T) { + doTimeoutTest(t, 10, 15, 10, 0) + doTimeoutTest(t, 10, 0, 10, 0) + doTimeoutTest(t, 5, 5, 10, 0) + doTimeoutTest(t, 15, 15, 0, 0) + doTimeoutTest(t, 15, 0, 20, 15) +} + +func doTimeoutTest(t *testing.T, expected int, requested int, max uint64, def uint64) { + t.Helper() + cfg := &config.AuctionTimeouts{ + Default: def, + Max: max, + } + body := fmt.Sprintf(`{ + "tid": "abcd", + "timeout_millis": %d, + "app":{ + "bundle":"AppNexus.PrebidMobileDemo", + "ver":"0.0.2" + }, + "ad_units": [ + { + "code": "first", + "sizes": [{"w": 300, "h": 250}], + "bids": [ + { + "bidder": "ix" + } + ] + } + ] +}`, requested) + r := httptest.NewRequest("POST", "/auction", strings.NewReader(body)) + d, _ := dummycache.New() + parsed, err := ParsePBSRequest(r, cfg, d, &config.HostCookie{}) + if err != nil { + t.Fatalf("Unexpected err: %v", err) + } + if parsed.TimeoutMillis != int64(expected) { + t.Errorf("Expected %dms timeout, got %dms", expected, parsed.TimeoutMillis) + } +} + +func TestParsePBSRequestUsesHostCookie(t *testing.T) { + body := []byte(`{ + "tid": "abcd", + "ad_units": [ + { + "code": "first", + "sizes": [{"w": 300, "h": 250}], + "bidders": [ + { + "bidder": "bidder1", + "params": { + "id": "417", + "siteID": "test-site" + } + } + ] + } + ] + } + `) + r, err := http.NewRequest("POST", "/auction", bytes.NewBuffer(body)) + r.Header.Add("Referer", "http://nytimes.com/cool.html") + if err != nil { + t.Fatalf("new request failed") + } + r.AddCookie(&http.Cookie{Name: "key", Value: "testcookie"}) + d, _ := dummycache.New() + hcc := config.HostCookie{ + CookieName: "key", + Family: "family", + OptOutCookie: config.Cookie{ + Name: "trp_optout", + Value: "true", + }, + } + + pbs_req, err2 := ParsePBSRequest(r, &config.AuctionTimeouts{ + Default: 2000, + Max: 2000, + }, d, &hcc) + if err2 != nil { + t.Fatalf("Parse simple request failed %v", err2) + } + if uid, _, _ := pbs_req.Cookie.GetUID("family"); uid != "testcookie" { + t.Errorf("Failed to leverage host cookie space for user identifier") + } +} diff --git a/pbs/pbsresponse.go b/pbs/pbsresponse.go new file mode 100644 index 00000000000..b8cf2c19ff7 --- /dev/null +++ b/pbs/pbsresponse.go @@ -0,0 +1,84 @@ +package pbs + +// PBSBid is a bid from the auction. These are produced by Adapters, and target a particular Ad Unit. +// +// This JSON format is a contract with both Prebid.js and Prebid-mobile. +// All changes *must* be backwards compatible, since clients cannot be forced to update their code. +type PBSBid struct { + // BidID identifies the Bid Request within the Ad Unit which this Bid targets. It should match one of + // the values inside PBSRequest.AdUnits[i].Bids[j].BidID. + BidID string `json:"bid_id"` + // AdUnitCode identifies the AdUnit which this Bid targets. + // It should match one of PBSRequest.AdUnits[i].Code, where "i" matches the AdUnit used in + // as BidID. + AdUnitCode string `json:"code"` + // Creative_id uniquely identifies the creative being served. It is not used by prebid-server, but + // it helps publishers and bidders identify and communicate about malicious or inappropriate ads. + // This project simply passes it along with the bid. + Creative_id string `json:"creative_id,omitempty"` + // CreativeMediaType shows whether the creative is a video or banner. + CreativeMediaType string `json:"media_type,omitempty"` + // BidderCode is the PBSBidder.BidderCode of the PBSBidder who made this bid. + BidderCode string `json:"bidder"` + // BidHash is the hash of the bidder's unique bid identifier for blockchain. It should not be sent to browser. + BidHash string `json:"-"` + // Price is the cpm, in US Dollars, which the bidder is willing to pay if this bid is chosen. + // TODO: Add support for other currencies someday. + Price float64 `json:"price"` + // NURL is a URL which returns ad markup, and should be called if the bid wins. + // If NURL and Adm are both defined, then Adm takes precedence. + NURL string `json:"nurl,omitempty"` + // Adm is the ad markup which should be used to deliver the ad, if this bid is chosen. + // If NURL and Adm are both defined, then Adm takes precedence. + Adm string `json:"adm,omitempty"` + // Width is the intended width which Adm should be shown, in pixels. + Width int64 `json:"width,omitempty"` + // Height is the intended width which Adm should be shown, in pixels. + Height int64 `json:"height,omitempty"` + // DealId is not used by prebid-server, but may be used by buyers and sellers who make special + // deals with each other. We simply pass this information along with the bid. + DealId string `json:"deal_id,omitempty"` + // CacheId is an ID in prebid-cache which can be used to fetch this ad's content. + // This supports prebid-mobile, which requires that the content be available from a URL. + CacheID string `json:"cache_id,omitempty"` + // Complete cache url returned from the prebid-cache. + // more flexible than a design that assumes the UUID is always appended to the end of the URL. + CacheURL string `json:"cache_url,omitempty"` + // ResponseTime is the number of milliseconds it took for the adapter to return a bid. + ResponseTime int `json:"response_time_ms,omitempty"` + AdServerTargeting map[string]string `json:"ad_server_targeting,omitempty"` +} + +// PBSBidSlice attaches the methods of sort.Interface to []PBSBid, ordering them by price. +// If two prices are equal, then the response time will be used as a tiebreaker. +// For more information, see https://golang.org/pkg/sort/#Interface +type PBSBidSlice []*PBSBid + +func (bids PBSBidSlice) Len() int { + return len(bids) +} + +func (bids PBSBidSlice) Less(i, j int) bool { + bidiResponseTimeInTerras := (float64(bids[i].ResponseTime) / 1000000000.0) + bidjResponseTimeInTerras := (float64(bids[j].ResponseTime) / 1000000000.0) + return bids[i].Price-bidiResponseTimeInTerras > bids[j].Price-bidjResponseTimeInTerras +} + +func (bids PBSBidSlice) Swap(i, j int) { + bids[i], bids[j] = bids[j], bids[i] +} + +type BidderDebug struct { + RequestURI string `json:"request_uri,omitempty"` + RequestBody string `json:"request_body,omitempty"` + ResponseBody string `json:"response_body,omitempty"` + StatusCode int `json:"status_code,omitempty"` +} + +type PBSResponse struct { + TID string `json:"tid,omitempty"` + Status string `json:"status,omitempty"` + BidderStatus []*PBSBidder `json:"bidder_status,omitempty"` + Bids PBSBidSlice `json:"bids,omitempty"` + BUrl string `json:"burl,omitempty"` +} diff --git a/pbs/pbsresponse_test.go b/pbs/pbsresponse_test.go new file mode 100644 index 00000000000..0e51120cdf4 --- /dev/null +++ b/pbs/pbsresponse_test.go @@ -0,0 +1,88 @@ +package pbs + +import ( + "sort" + "testing" +) + +func TestSortBids(t *testing.T) { + bid1 := PBSBid{ + BidID: "testBidId", + AdUnitCode: "testAdUnitCode", + BidderCode: "testBidderCode", + Price: 0.0, + } + bid2 := PBSBid{ + BidID: "testBidId", + AdUnitCode: "testAdUnitCode", + BidderCode: "testBidderCode", + Price: 4.0, + } + bid3 := PBSBid{ + BidID: "testBidId", + AdUnitCode: "testAdUnitCode", + BidderCode: "testBidderCode", + Price: 2.0, + } + bid4 := PBSBid{ + BidID: "testBidId", + AdUnitCode: "testAdUnitCode", + BidderCode: "testBidderCode", + Price: 0.50, + } + + bids := make(PBSBidSlice, 0) + bids = append(bids, &bid1, &bid2, &bid3, &bid4) + + sort.Sort(bids) + if bids[0].Price != 4.0 { + t.Error("Expected 4.00 to be highest price") + } + if bids[1].Price != 2.0 { + t.Error("Expected 2.00 to be second highest price") + } + if bids[2].Price != 0.5 { + t.Error("Expected 0.50 to be third highest price") + } + if bids[3].Price != 0.0 { + t.Error("Expected 0.00 to be lowest price") + } +} + +func TestSortBidsWithResponseTimes(t *testing.T) { + bid1 := PBSBid{ + BidID: "testBidId", + AdUnitCode: "testAdUnitCode", + BidderCode: "testBidderCode", + Price: 1.0, + ResponseTime: 70, + } + bid2 := PBSBid{ + BidID: "testBidId", + AdUnitCode: "testAdUnitCode", + BidderCode: "testBidderCode", + Price: 1.0, + ResponseTime: 20, + } + bid3 := PBSBid{ + BidID: "testBidId", + AdUnitCode: "testAdUnitCode", + BidderCode: "testBidderCode", + Price: 1.0, + ResponseTime: 99, + } + + bids := make(PBSBidSlice, 0) + bids = append(bids, &bid1, &bid2, &bid3) + + sort.Sort(bids) + if bids[0] != &bid2 { + t.Error("Expected bid 2 to win") + } + if bids[1] != &bid1 { + t.Error("Expected bid 1 to be second") + } + if bids[2] != &bid3 { + t.Error("Expected bid 3 to be last") + } +} diff --git a/pbs/usersync.go b/pbs/usersync.go index d5043b8c13f..85b55f42aeb 100644 --- a/pbs/usersync.go +++ b/pbs/usersync.go @@ -7,10 +7,13 @@ import ( "net/http" "net/url" "strings" + "time" "github.com/golang/glog" "github.com/julienschmidt/httprouter" + "github.com/prebid/prebid-server/analytics" "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/metrics" "github.com/prebid/prebid-server/server/ssl" "github.com/prebid/prebid-server/usersync" ) @@ -18,10 +21,27 @@ import ( // Recaptcha code from https://github.com/haisum/recaptcha/blob/master/recaptcha.go const RECAPTCHA_URL = "https://www.google.com/recaptcha/api/siteverify" +const ( + USERSYNC_OPT_OUT = "usersync.opt_outs" + USERSYNC_BAD_REQUEST = "usersync.bad_requests" + USERSYNC_SUCCESS = "usersync.%s.sets" +) + +// uidWithExpiry bundles the UID with an Expiration date. +// After the expiration, the UID is no longer valid. +type uidWithExpiry struct { + // UID is the ID given to a user by a particular bidder + UID string `json:"uid"` + // Expires is the time at which this UID should no longer apply. + Expires time.Time `json:"expires"` +} + type UserSyncDeps struct { ExternalUrl string RecaptchaSecret string HostCookieConfig *config.HostCookie + MetricsEngine metrics.MetricsEngine + PBSAnalytics analytics.PBSAnalyticsModule } // Struct for parsing json in google's response @@ -59,7 +79,7 @@ func (deps *UserSyncDeps) OptOut(w http.ResponseWriter, r *http.Request, _ httpr rr := r.FormValue("g-recaptcha-response") if rr == "" { - http.Redirect(w, r, fmt.Sprintf("%s/static/optout.html", deps.ExternalUrl), http.StatusMovedPermanently) + http.Redirect(w, r, fmt.Sprintf("%s/static/optout.html", deps.ExternalUrl), 301) return } @@ -78,8 +98,8 @@ func (deps *UserSyncDeps) OptOut(w http.ResponseWriter, r *http.Request, _ httpr pc.SetCookieOnResponse(w, false, deps.HostCookieConfig, deps.HostCookieConfig.TTLDuration()) if optout == "" { - http.Redirect(w, r, deps.HostCookieConfig.OptInURL, http.StatusMovedPermanently) + http.Redirect(w, r, deps.HostCookieConfig.OptInURL, 301) } else { - http.Redirect(w, r, deps.HostCookieConfig.OptOutURL, http.StatusMovedPermanently) + http.Redirect(w, r, deps.HostCookieConfig.OptOutURL, 301) } } diff --git a/prebid_cache_client/client.go b/prebid_cache_client/client.go index a24a139ea1d..730d54b0acb 100644 --- a/prebid_cache_client/client.go +++ b/prebid_cache_client/client.go @@ -120,7 +120,7 @@ func (c *clientImpl) PutJson(ctx context.Context, values []Cacheable) (uuids []s responseBody, err := ioutil.ReadAll(anResp.Body) if anResp.StatusCode != 200 { - logError(&errs, "Prebid Cache call to %s returned %d: %s", c.putUrl, anResp.StatusCode, responseBody) + logError(&errs, "Prebid Cache call to %s returned %d: %s", putURL, anResp.StatusCode, responseBody) return uuidsToReturn, errs } diff --git a/prebid_cache_client/client_test.go b/prebid_cache_client/client_test.go index ec390364849..60237bbbb27 100644 --- a/prebid_cache_client/client_test.go +++ b/prebid_cache_client/client_test.go @@ -280,18 +280,10 @@ func assertStringEqual(t *testing.T, expected, actual string) { } } -type handlerResponseObject struct { - UUID string `json:"uuid"` -} - -type handlerResponse struct { - Responses []handlerResponseObject `json:"responses"` -} - func newHandler(numResponses int) http.HandlerFunc { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - resp := handlerResponse{ - Responses: make([]handlerResponseObject, numResponses), + resp := response{ + Responses: make([]responseObject, numResponses), } for i := 0; i < numResponses; i++ { resp.Responses[i].UUID = strconv.Itoa(i) diff --git a/prebid_cache_client/prebid_cache.go b/prebid_cache_client/prebid_cache.go new file mode 100644 index 00000000000..cde7ec8d951 --- /dev/null +++ b/prebid_cache_client/prebid_cache.go @@ -0,0 +1,122 @@ +package prebid_cache_client + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + + "golang.org/x/net/context/ctxhttp" +) + +// This file is deprecated, and is only used to cache things for the legacy (/auction) endpoint. +// For /openrtb2/auction cache, see client.go in this package. + +type CacheObject struct { + Value interface{} + UUID string + IsVideo bool +} + +type BidCache struct { + Adm string `json:"adm,omitempty"` + NURL string `json:"nurl,omitempty"` + Width int64 `json:"width,omitempty"` + Height int64 `json:"height,omitempty"` +} + +// internal protocol objects +type putObject struct { + Type string `json:"type"` + Value interface{} `json:"value"` +} + +type putRequest struct { + Puts []putObject `json:"puts"` +} + +type responseObject struct { + UUID string `json:"uuid"` +} +type response struct { + Responses []responseObject `json:"responses"` +} + +var ( + client *http.Client + baseURL string + putURL string +) + +// InitPrebidCache setup the global prebid cache +func InitPrebidCache(baseurl string) { + baseURL = baseurl + putURL = fmt.Sprintf("%s/cache", baseURL) + + ts := &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 65, + } + + client = &http.Client{ + Transport: ts, + } +} + +// Put will send the array of objs and update each with a UUID +func Put(ctx context.Context, objs []*CacheObject) error { + // Fixes #197 + if len(objs) == 0 { + return nil + } + pr := putRequest{Puts: make([]putObject, len(objs))} + for i, obj := range objs { + if obj.IsVideo { + pr.Puts[i].Type = "xml" + } else { + pr.Puts[i].Type = "json" + } + pr.Puts[i].Value = obj.Value + } + // Don't want to escape the HTML for adm and nurl + buf := new(bytes.Buffer) + enc := json.NewEncoder(buf) + enc.SetEscapeHTML(false) + err := enc.Encode(pr) + if err != nil { + return err + } + + httpReq, err := http.NewRequest("POST", putURL, buf) + if err != nil { + return err + } + httpReq.Header.Add("Content-Type", "application/json;charset=utf-8") + httpReq.Header.Add("Accept", "application/json") + + anResp, err := ctxhttp.Do(ctx, client, httpReq) + if err != nil { + return err + } + defer anResp.Body.Close() + + if anResp.StatusCode != 200 { + return fmt.Errorf("HTTP status code %d", anResp.StatusCode) + } + + var resp response + if err := json.NewDecoder(anResp.Body).Decode(&resp); err != nil { + return err + } + + if len(resp.Responses) != len(objs) { + return fmt.Errorf("Put response length didn't match") + } + + for i, r := range resp.Responses { + objs[i].UUID = r.UUID + } + + return nil +} diff --git a/prebid_cache_client/prebid_cache_test.go b/prebid_cache_client/prebid_cache_test.go new file mode 100644 index 00000000000..65688789fd0 --- /dev/null +++ b/prebid_cache_client/prebid_cache_test.go @@ -0,0 +1,150 @@ +package prebid_cache_client + +import ( + "context" + "encoding/json" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + "time" + + "fmt" +) + +var delay time.Duration +var ( + MaxValueLength = 1024 * 10 + MaxNumValues = 10 +) + +type putAnyObject struct { + Type string `json:"type"` + Value json.RawMessage `json:"value"` +} + +type putAnyRequest struct { + Puts []putAnyObject `json:"puts"` +} + +func DummyPrebidCacheServer(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, "Failed to read the request body.", http.StatusBadRequest) + return + } + defer r.Body.Close() + var put putAnyRequest + + err = json.Unmarshal(body, &put) + if err != nil { + http.Error(w, "Request body "+string(body)+" is not valid JSON.", http.StatusBadRequest) + return + } + + if len(put.Puts) > MaxNumValues { + http.Error(w, fmt.Sprintf("More keys than allowed: %d", MaxNumValues), http.StatusBadRequest) + return + } + + resp := response{ + Responses: make([]responseObject, len(put.Puts)), + } + for i, p := range put.Puts { + resp.Responses[i].UUID = fmt.Sprintf("UUID-%d", i+1) // deterministic for testing + if len(p.Value) > MaxValueLength { + http.Error(w, fmt.Sprintf("Value is larger than allowed size: %d", MaxValueLength), http.StatusBadRequest) + return + } + if len(p.Value) == 0 { + http.Error(w, "Missing value.", http.StatusBadRequest) + return + } + if p.Type != "xml" && p.Type != "json" { + http.Error(w, fmt.Sprintf("Type must be one of [\"json\", \"xml\"]. Found %v", p.Type), http.StatusBadRequest) + return + } + } + + bytes, err := json.Marshal(&resp) + if err != nil { + http.Error(w, "Failed to serialize UUIDs into JSON.", http.StatusInternalServerError) + return + } + if delay > 0 { + <-time.After(delay) + } + w.Header().Set("Content-Type", "application/json") + w.Write(bytes) +} + +func TestPrebidClient(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(DummyPrebidCacheServer)) + defer server.Close() + + cobj := make([]*CacheObject, 3) + + // example bids + cobj[0] = &CacheObject{ + IsVideo: false, + Value: &BidCache{ + Adm: "{\"type\":\"ID\",\"bid_id\":\"8255649814109237089\",\"placement_id\":\"1995257847363113_1997038003851764\",\"resolved_placement_id\":\"1995257847363113_1997038003851764\",\"sdk_version\":\"4.25.0-appnexus.bidding\",\"device_id\":\"87ECBA49-908A-428F-9DE7-4B9CED4F486C\",\"template\":7,\"payload\":\"null\"}", + NURL: "https://www.facebook.com/audiencenetwork/nurl/?partner=442648859414574&app=1995257847363113&placement=1997038003851764&auction=d3013e9e-ca55-4a86-9baa-d44e31355e1d&impression=bannerad1&request=7187783259538616534&bid=3832427901228167009&ortb_loss_code=0", + Width: 300, + Height: 250, + }, + } + cobj[1] = &CacheObject{ + IsVideo: false, + Value: &BidCache{ + Adm: "", + Width: 300, + Height: 250, + }, + } + cobj[2] = &CacheObject{ + IsVideo: true, + Value: "", + } + InitPrebidCache(server.URL) + + ctx := context.TODO() + err := Put(ctx, cobj) + if err != nil { + t.Fatalf("pbc put failed: %v", err) + } + + if cobj[0].UUID != "UUID-1" { + t.Errorf("First object UUID was '%s', should have been 'UUID-1'", cobj[0].UUID) + } + if cobj[1].UUID != "UUID-2" { + t.Errorf("Second object UUID was '%s', should have been 'UUID-2'", cobj[1].UUID) + } + if cobj[2].UUID != "UUID-3" { + t.Errorf("Third object UUID was '%s', should have been 'UUID-3'", cobj[2].UUID) + } + + delay = 5 * time.Millisecond + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) + defer cancel() + + err = Put(ctx, cobj) + if err == nil { + t.Fatalf("pbc put succeeded but should have timed out") + } +} + +// Prevents #197 +func TestEmptyBids(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("The server should not be called.") + }) + server := httptest.NewServer(handler) + defer server.Close() + + InitPrebidCache(server.URL) + + if err := Put(context.Background(), []*CacheObject{}); err != nil { + t.Errorf("Error on Put: %v", err) + } +} diff --git a/privacy/ccpa/parsedpolicy_test.go b/privacy/ccpa/parsedpolicy_test.go index 4bbb4cbfd0f..33563b50567 100644 --- a/privacy/ccpa/parsedpolicy_test.go +++ b/privacy/ccpa/parsedpolicy_test.go @@ -3,7 +3,9 @@ package ccpa import ( "testing" + "github.com/mxmCherry/openrtb/v15/openrtb2" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" ) func TestValidateConsent(t *testing.T) { @@ -378,3 +380,12 @@ func TestShouldEnforce(t *testing.T) { assert.Equal(t, test.expected, result, test.description) } } + +type mockPolicWriter struct { + mock.Mock +} + +func (m *mockPolicWriter) Write(req *openrtb2.BidRequest) error { + args := m.Called(req) + return args.Error(0) +} diff --git a/router/router.go b/router/router.go index 81623f13838..90074753a5b 100644 --- a/router/router.go +++ b/router/router.go @@ -3,6 +3,7 @@ package router import ( "context" "crypto/tls" + "database/sql" "encoding/json" "fmt" "io/ioutil" @@ -11,17 +12,34 @@ import ( "strings" "time" + "github.com/prebid/prebid-server/currency" + "github.com/prebid/prebid-server/endpoints/events" + "github.com/prebid/prebid-server/errortypes" + "github.com/prebid/prebid-server/util/uuidutil" + "github.com/prebid/prebid-server/version" + + "github.com/prebid/prebid-server/metrics" + + "github.com/prebid/prebid-server/adapters" + "github.com/prebid/prebid-server/adapters/adform" + "github.com/prebid/prebid-server/adapters/appnexus" + "github.com/prebid/prebid-server/adapters/conversant" + "github.com/prebid/prebid-server/adapters/ix" + "github.com/prebid/prebid-server/adapters/pubmatic" + "github.com/prebid/prebid-server/adapters/pulsepoint" + "github.com/prebid/prebid-server/adapters/rubicon" + "github.com/prebid/prebid-server/adapters/sovrn" analyticsConf "github.com/prebid/prebid-server/analytics/config" + "github.com/prebid/prebid-server/cache" + "github.com/prebid/prebid-server/cache/dummycache" + "github.com/prebid/prebid-server/cache/filecache" + "github.com/prebid/prebid-server/cache/postgrescache" "github.com/prebid/prebid-server/config" - "github.com/prebid/prebid-server/currency" "github.com/prebid/prebid-server/endpoints" - "github.com/prebid/prebid-server/endpoints/events" infoEndpoints "github.com/prebid/prebid-server/endpoints/info" "github.com/prebid/prebid-server/endpoints/openrtb2" - "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/exchange" "github.com/prebid/prebid-server/gdpr" - "github.com/prebid/prebid-server/metrics" metricsConf "github.com/prebid/prebid-server/metrics/config" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/pbs" @@ -31,8 +49,6 @@ import ( storedRequestsConf "github.com/prebid/prebid-server/stored_requests/config" "github.com/prebid/prebid-server/usersync" "github.com/prebid/prebid-server/util/sliceutil" - "github.com/prebid/prebid-server/util/uuidutil" - "github.com/prebid/prebid-server/version" "github.com/golang/glog" "github.com/julienschmidt/httprouter" @@ -40,6 +56,9 @@ import ( "github.com/rs/cors" ) +var dataCache cache.Cache +var exchanges map[string]adapters.Adapter + // NewJsonDirectoryServer is used to serve .json files from a directory as a single blob. For example, // given a directory containing the files "a.json" and "b.json", this returns a Handle which serves JSON like: // @@ -104,6 +123,51 @@ func (m NoCache) ServeHTTP(w http.ResponseWriter, r *http.Request) { m.Handler.ServeHTTP(w, r) } +func loadDataCache(cfg *config.Configuration, db *sql.DB) (err error) { + switch cfg.DataCache.Type { + case "dummy": + dataCache, err = dummycache.New() + if err != nil { + glog.Fatalf("Dummy cache not configured: %s", err.Error()) + } + + case "postgres": + if db == nil { + return fmt.Errorf("Nil db cannot connect to postgres. Did you forget to set the config.stored_requests.postgres values?") + } + dataCache = postgrescache.New(db, postgrescache.CacheConfig{ + Size: cfg.DataCache.CacheSize, + TTL: cfg.DataCache.TTLSeconds, + }) + return nil + case "filecache": + dataCache, err = filecache.New(cfg.DataCache.Filename) + if err != nil { + return fmt.Errorf("FileCache Error: %s", err.Error()) + } + + default: + return fmt.Errorf("Unknown datacache.type: %s", cfg.DataCache.Type) + } + return nil +} + +func newExchangeMap(cfg *config.Configuration) map[string]adapters.Adapter { + // These keys _must_ coincide with the bidder code in Prebid.js, if the adapter exists in both projects + return map[string]adapters.Adapter{ + "appnexus": appnexus.NewAppNexusLegacyAdapter(adapters.DefaultHTTPAdapterConfig, cfg.Adapters[string(openrtb_ext.BidderAppnexus)].Endpoint, cfg.Adapters[string(openrtb_ext.BidderAppnexus)].PlatformID), + "districtm": appnexus.NewAppNexusLegacyAdapter(adapters.DefaultHTTPAdapterConfig, cfg.Adapters[string(openrtb_ext.BidderAppnexus)].Endpoint, cfg.Adapters[string(openrtb_ext.BidderAppnexus)].PlatformID), + "ix": ix.NewIxLegacyAdapter(adapters.DefaultHTTPAdapterConfig, cfg.Adapters[strings.ToLower(string(openrtb_ext.BidderIx))].Endpoint), + "pubmatic": pubmatic.NewPubmaticLegacyAdapter(adapters.DefaultHTTPAdapterConfig, cfg.Adapters[string(openrtb_ext.BidderPubmatic)].Endpoint), + "pulsepoint": pulsepoint.NewPulsePointLegacyAdapter(adapters.DefaultHTTPAdapterConfig, cfg.Adapters[string(openrtb_ext.BidderPulsepoint)].Endpoint), + "rubicon": rubicon.NewRubiconLegacyAdapter(adapters.DefaultHTTPAdapterConfig, cfg.Adapters[string(openrtb_ext.BidderRubicon)].Endpoint, + cfg.Adapters[string(openrtb_ext.BidderRubicon)].XAPI.Username, cfg.Adapters[string(openrtb_ext.BidderRubicon)].XAPI.Password, cfg.Adapters[string(openrtb_ext.BidderRubicon)].XAPI.Tracker), + "conversant": conversant.NewConversantLegacyAdapter(adapters.DefaultHTTPAdapterConfig, cfg.Adapters[string(openrtb_ext.BidderConversant)].Endpoint), + "adform": adform.NewAdformLegacyAdapter(adapters.DefaultHTTPAdapterConfig, cfg.Adapters[string(openrtb_ext.BidderAdform)].Endpoint), + "sovrn": sovrn.NewSovrnLegacyAdapter(adapters.DefaultHTTPAdapterConfig, cfg.Adapters[string(openrtb_ext.BidderSovrn)].Endpoint), + } +} + type Router struct { *httprouter.Router MetricsEngine *metricsConf.DetailedMetricsEngine @@ -147,6 +211,10 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R }, } + // Hack because of how legacy handles districtm + legacyBidderList := openrtb_ext.CoreBidderNames() + legacyBidderList = append(legacyBidderList, openrtb_ext.BidderName("districtm")) + p, _ := filepath.Abs(infoDirectory) bidderInfos, err := config.LoadBidderInfoFromDisk(p, cfg.Adapters, openrtb_ext.BuildBidderStringSlice()) if err != nil { @@ -176,10 +244,13 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R } // Metrics engine - r.MetricsEngine = metricsConf.NewMetricsEngine(cfg, openrtb_ext.CoreBidderNames(), syncerKeys) - shutdown, fetcher, ampFetcher, accounts, categoriesFetcher, videoFetcher := storedRequestsConf.NewStoredRequests(cfg, r.MetricsEngine, generalHttpClient, r.Router) + r.MetricsEngine = metricsConf.NewMetricsEngine(cfg, legacyBidderList, syncerKeys) + db, shutdown, fetcher, ampFetcher, accounts, categoriesFetcher, videoFetcher := storedRequestsConf.NewStoredRequests(cfg, r.MetricsEngine, generalHttpClient, r.Router) // todo(zachbadgett): better shutdown r.Shutdown = shutdown + if err := loadDataCache(cfg, db); err != nil { + return nil, fmt.Errorf("Prebid Server could not load data cache: %v", err) + } pbsAnalytics := analyticsConf.NewPBSAnalytics(&cfg.Analytics) @@ -199,6 +270,7 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R gvlVendorIDs := bidderInfos.ToGVLVendorIDMap() gdprPerms := gdpr.NewPermissions(context.Background(), cfg.GDPR, gvlVendorIDs, generalHttpClient) + exchanges = newExchangeMap(cfg) cacheClient := pbc.NewClient(cacheHttpClient, &cfg.CacheURL, &cfg.ExtCacheURL, r.MetricsEngine) adapters, adaptersErrs := exchange.BuildAdapters(generalHttpClient, cfg, bidderInfos, r.MetricsEngine) @@ -229,6 +301,10 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R videoEndpoint = aspects.QueuedRequestTimeout(videoEndpoint, cfg.RequestTimeoutHeaders, r.MetricsEngine, metrics.ReqTypeVideo) } + if cfg.EnableLegacyAuction { + r.POST("/auction", endpoints.Auction(cfg, syncersByBidder, gdprPerms, r.MetricsEngine, dataCache, exchanges)) + } + r.POST("/openrtb2/auction", openrtbEndpoint) r.POST("/openrtb2/video", videoEndpoint) r.GET("/openrtb2/amp", ampEndpoint) @@ -255,6 +331,8 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R HostCookieConfig: &(cfg.HostCookie), ExternalUrl: cfg.ExternalURL, RecaptchaSecret: cfg.RecaptchaSecret, + MetricsEngine: r.MetricsEngine, + PBSAnalytics: pbsAnalytics, } r.GET("/setuid", endpoints.NewSetUIDEndpoint(cfg.HostCookie, syncersByBidder, gdprPerms, pbsAnalytics, r.MetricsEngine)) diff --git a/router/router_test.go b/router/router_test.go index b4ceaff16a9..24a7709c365 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "net/http" "net/http/httptest" + "os" "testing" "github.com/prebid/prebid-server/config" @@ -61,6 +62,18 @@ func TestNewJsonDirectoryServer(t *testing.T) { ensureHasKey(t, data, "aliastest") } +func TestExchangeMap(t *testing.T) { + exchanges := newExchangeMap(&config.Configuration{}) + bidderMap := openrtb_ext.BuildBidderMap() + for bidderName := range exchanges { + // OpenRTB doesn't support hardcoded aliases... so this test skips districtm, + // which was the only alias in the legacy adapter map. + if _, ok := bidderMap[bidderName]; bidderName != "districtm" && !ok { + t.Errorf("Bidder %s exists in exchange, but is not a part of the BidderMap.", bidderName) + } + } +} + func TestApplyBidderInfoConfigOverrides(t *testing.T) { var testCases = []struct { description string @@ -285,6 +298,38 @@ func TestNoCache(t *testing.T) { } } +func TestLoadDataCache(t *testing.T) { + // Test dummy + if err := loadDataCache(&config.Configuration{ + DataCache: config.DataCache{ + Type: "dummy", + }, + }, nil); err != nil { + t.Errorf("data cache: dummy: %s", err) + } + // Test postgres error + if err := loadDataCache(&config.Configuration{ + DataCache: config.DataCache{ + Type: "postgres", + }, + }, nil); err == nil { + t.Errorf("data cache: postgres: db nil should return error") + } + // Test file + d, _ := ioutil.TempDir("", "pbs-filecache") + defer os.RemoveAll(d) + f, _ := ioutil.TempFile(d, "file") + defer f.Close() + if err := loadDataCache(&config.Configuration{ + DataCache: config.DataCache{ + Type: "filecache", + Filename: f.Name(), + }, + }, nil); err != nil { + t.Errorf("data cache: filecache: %s", err) + } +} + var testDefReqConfig = config.DefReqConfig{ Type: "file", FileSystem: config.DefReqFiles{ diff --git a/server/prometheus.go b/server/prometheus.go index 6f0a0b2df45..4b9f7037d0a 100644 --- a/server/prometheus.go +++ b/server/prometheus.go @@ -9,10 +9,13 @@ import ( "github.com/prebid/prebid-server/config" metricsconfig "github.com/prebid/prebid-server/metrics/config" + prometheusMetrics "github.com/prebid/prebid-server/metrics/prometheus" ) func newPrometheusServer(cfg *config.Configuration, metrics *metricsconfig.DetailedMetricsEngine) *http.Server { - proMetrics := metrics.PrometheusMetrics + var proMetrics *prometheusMetrics.Metrics + + proMetrics = metrics.PrometheusMetrics if proMetrics == nil { glog.Fatal("Prometheus metrics configured, but a Prometheus metrics engine was not found. Cannot set up a Prometheus listener.") diff --git a/stored_requests/backends/file_fetcher/fetcher_test.go b/stored_requests/backends/file_fetcher/fetcher_test.go index 7d23f942d56..a145a3b43a2 100644 --- a/stored_requests/backends/file_fetcher/fetcher_test.go +++ b/stored_requests/backends/file_fetcher/fetcher_test.go @@ -32,7 +32,7 @@ func TestAccountFetcher(t *testing.T) { assertErrorCount(t, 0, errs) assert.JSONEq(t, `{"disabled":false, "id":"valid"}`, string(account)) - _, errs = fetcher.FetchAccount(context.Background(), "nonexistent") + account, errs = fetcher.FetchAccount(context.Background(), "nonexistent") assertErrorCount(t, 1, errs) assert.Error(t, errs[0]) assert.Equal(t, stored_requests.NotFoundError{"nonexistent", "Account"}, errs[0]) diff --git a/stored_requests/backends/http_fetcher/fetcher.go b/stored_requests/backends/http_fetcher/fetcher.go index 75a92e3f331..bc12caecb98 100644 --- a/stored_requests/backends/http_fetcher/fetcher.go +++ b/stored_requests/backends/http_fetcher/fetcher.go @@ -74,6 +74,7 @@ func NewFetcher(client *http.Client, endpoint string) *HttpFetcher { type HttpFetcher struct { client *http.Client Endpoint string + hasQuery bool Categories map[string]map[string]stored_requests.Category } diff --git a/stored_requests/backends/http_fetcher/fetcher_test.go b/stored_requests/backends/http_fetcher/fetcher_test.go index 10d3984a818..30933181e1d 100644 --- a/stored_requests/backends/http_fetcher/fetcher_test.go +++ b/stored_requests/backends/http_fetcher/fetcher_test.go @@ -1,8 +1,10 @@ package http_fetcher import ( + "bytes" "context" "encoding/json" + "io" "net/http" "net/http/httptest" "strings" @@ -166,6 +168,42 @@ func TestErrResponse(t *testing.T) { assert.Len(t, errs, 1) } +func assertSameContents(t *testing.T, expected map[string]json.RawMessage, actual map[string]json.RawMessage) { + if len(expected) != len(actual) { + t.Errorf("Wrong counts. Expected %d, actual %d", len(expected), len(actual)) + return + } + for expectedKey, expectedVal := range expected { + if actualVal, ok := actual[expectedKey]; ok { + if !bytes.Equal(expectedVal, actualVal) { + t.Errorf("actual[%s] value %s does not match expected: %s", expectedKey, string(actualVal), string(actualVal)) + } + } else { + t.Errorf("actual map missing expected key %s", expectedKey) + } + } +} + +func assertSameErrMsgs(t *testing.T, expected []string, actual []error) { + if len(expected) != len(actual) { + t.Errorf("Wrong error counts. Expected %d, actual %d", len(expected), len(actual)) + return + } + for i, expectedErr := range expected { + if actual[i].Error() != expectedErr { + t.Errorf("Wrong error[%d]. Expected %s, got %s", i, expectedErr, actual[i].Error()) + } + } +} + +type closeWrapper struct { + io.Reader +} + +func (w closeWrapper) Close() error { + return nil +} + func newFetcherBrokenBackend() (fetcher *HttpFetcher, closer func()) { handler := func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) diff --git a/stored_requests/caches/nil_cache/nil_cache.go b/stored_requests/caches/nil_cache/nil_cache.go index 88bbd404674..d043ae55c96 100644 --- a/stored_requests/caches/nil_cache/nil_cache.go +++ b/stored_requests/caches/nil_cache/nil_cache.go @@ -13,7 +13,9 @@ func (c *NilCache) Get(ctx context.Context, ids []string) map[string]json.RawMes } func (c *NilCache) Save(ctx context.Context, data map[string]json.RawMessage) { + return } func (c *NilCache) Invalidate(ctx context.Context, ids []string) { + return } diff --git a/stored_requests/config/config.go b/stored_requests/config/config.go index 89022582ace..f682ff932f4 100644 --- a/stored_requests/config/config.go +++ b/stored_requests/config/config.go @@ -93,12 +93,12 @@ func CreateStoredRequests(cfg *config.StoredRequests, metricsEngine metrics.Metr return } -// NewStoredRequests returns: +// NewStoredRequests returns five things: // -// 1. A function which should be called on shutdown for graceful cleanups. -// 2. A Fetcher which can be used to get Stored Requests for /openrtb2/auction -// 3. A Fetcher which can be used to get Stored Requests for /openrtb2/amp -// 4. A Fetcher which can be used to get Account data +// 1. A DB connection, if one was created. This may be nil. +// 2. A function which should be called on shutdown for graceful cleanups. +// 3. A Fetcher which can be used to get Stored Requests for /openrtb2/auction +// 4. A Fetcher which can be used to get Stored Requests for /openrtb2/amp // 5. A Fetcher which can be used to get Category Mapping data // 6. A Fetcher which can be used to get Stored Requests for /openrtb2/video // @@ -107,7 +107,7 @@ func CreateStoredRequests(cfg *config.StoredRequests, metricsEngine metrics.Metr // // As a side-effect, it will add some endpoints to the router if the config calls for it. // In the future we should look for ways to simplify this so that it's not doing two things. -func NewStoredRequests(cfg *config.Configuration, metricsEngine metrics.MetricsEngine, client *http.Client, router *httprouter.Router) (shutdown func(), fetcher stored_requests.Fetcher, ampFetcher stored_requests.Fetcher, accountsFetcher stored_requests.AccountFetcher, categoriesFetcher stored_requests.CategoryFetcher, videoFetcher stored_requests.Fetcher) { +func NewStoredRequests(cfg *config.Configuration, metricsEngine metrics.MetricsEngine, client *http.Client, router *httprouter.Router) (db *sql.DB, shutdown func(), fetcher stored_requests.Fetcher, ampFetcher stored_requests.Fetcher, accountsFetcher stored_requests.AccountFetcher, categoriesFetcher stored_requests.CategoryFetcher, videoFetcher stored_requests.Fetcher) { // TODO: Switch this to be set in config defaults //if cfg.CategoryMapping.CacheEvents.Enabled && cfg.CategoryMapping.CacheEvents.Endpoint == "" { // cfg.CategoryMapping.CacheEvents.Endpoint = "/storedrequest/categorymapping" @@ -121,6 +121,8 @@ func NewStoredRequests(cfg *config.Configuration, metricsEngine metrics.MetricsE fetcher4, shutdown4 := CreateStoredRequests(&cfg.StoredVideo, metricsEngine, client, router, &dbc) fetcher5, shutdown5 := CreateStoredRequests(&cfg.Accounts, metricsEngine, client, router, &dbc) + db = dbc.db + fetcher = fetcher1.(stored_requests.Fetcher) ampFetcher = fetcher2.(stored_requests.Fetcher) categoriesFetcher = fetcher3.(stored_requests.CategoryFetcher) diff --git a/stored_requests/events/events.go b/stored_requests/events/events.go index 7cb8f4b9b6d..5b89943572f 100644 --- a/stored_requests/events/events.go +++ b/stored_requests/events/events.go @@ -77,7 +77,7 @@ func (e *EventListener) Listen(cache stored_requests.Cache, events EventProducer e.onInvalidate() } case <-e.stop: - return + break } } } diff --git a/usersync/cookie_test.go b/usersync/cookie_test.go index 4e87db5dd0a..4792db1969f 100644 --- a/usersync/cookie_test.go +++ b/usersync/cookie_test.go @@ -33,7 +33,7 @@ func TestEmptyOptOutCookie(t *testing.T) { func TestEmptyCookie(t *testing.T) { cookie := &Cookie{ - uids: make(map[string]uidWithExpiry), + uids: make(map[string]uidWithExpiry, 0), optOut: false, birthday: timestamp(), }