Merge branch 'datacratic-upstream_16_07_2015'

jml/utils/exception_ptr.cc

@@ -0,0 +1,51 @@
+/* exception_ptr.cc
+   Wolfgang Sourdeau, July 2015
+   Copyright (c) 2015 Datacratic.  All rights reserved.
+
+*/
+
+#include "exception_ptr.h"
+
+
+using namespace std;
+using namespace ML;
+
+
+/****************************************************************************/
+/* EXCEPTIONPTR HANDLER                                                     */
+/****************************************************************************/
+
+bool
+ExceptionPtrHandler::
+hasException()
+{
+    std::unique_lock<mutex> guard(excLock_);
+    return bool(excPtr_);
+}
+
+void
+ExceptionPtrHandler::
+takeException(std::exception_ptr newPtr)
+{
+    std::unique_lock<mutex> guard(excLock_);
+    excPtr_ = newPtr;
+}
+
+void
+ExceptionPtrHandler::
+takeCurrentException()
+{
+    takeException(std::current_exception());
+}
+
+void
+ExceptionPtrHandler::
+rethrowIfSet()
+{
+    std::unique_lock<mutex> guard(excLock_);
+    if (excPtr_) {
+        std::exception_ptr ptr = excPtr_;
+        excPtr_ = nullptr;
+        std::rethrow_exception(ptr);
+    }
+}

jml/utils/exception_ptr.h

@@ -0,0 +1,33 @@
+/* exception_ptr.h                                                 -*- C++ -*-
+   Wolfgang Sourdeau, July 2015
+   Copyright (c) 2015 Datacratic.  All rights reserved.
+
+*/
+
+#pragma once
+
+#include <exception>
+#include <mutex>
+
+
+namespace ML {
+
+/****************************************************************************/
+/* EXCEPTIONPTR HANDLER                                                     */
+/****************************************************************************/
+
+/* This class provides thread-safe handling of exception-ptr. */
+struct ExceptionPtrHandler {
+    bool hasException();
+    void takeException(std::exception_ptr newPtr);
+    void takeCurrentException();
+    void rethrowIfSet();
+    void clear()
+    { takeException(nullptr); }
+
+private:
+    std::mutex excLock_;
+    std::exception_ptr excPtr_;
+};
+
+} // namespace Datacratic

jml/utils/utils.mk

@@ -1,6 +1,7 @@
 
 LIBUTILS_SOURCES := \
         environment.cc \
+        exception_ptr.cc \
         file_functions.cc \
         filter_streams.cc \
         string_functions.cc \

rtbkit/common/plugin_interface.h

@@ -10,9 +10,25 @@
 
 namespace RTBKIT {
 
+namespace details {
+    template<typename Plugin>
+    struct has_factory {
+        template<typename T>
+        static std::true_type test(typename T::Factory* = 0);
+
+        template<typename T>
+        static std::false_type test(...);
+
+        static constexpr bool value
+            = std::is_same<decltype(test<Plugin>(nullptr)), std::true_type>::value;
+    };
+}
+
 template<class T>
 struct PluginInterface
 {
+  static_assert(details::has_factory<T>::value, "The plugin must provide a Factory type");
+
   static void registerPlugin(const std::string& name,
 			     typename T::Factory functor);
   static typename T::Factory& getPlugin(const std::string& name);

rtbkit/core/router/filters/creative_filters.h

@@ -45,13 +45,16 @@ struct CreativeFormatFilter : public CreativeFilter<CreativeFormatFilter>
     void filterImpression(
             FilterState& state, unsigned impIndex, const AdSpot& imp) const
     {
-        // The 0x0 format means: match anything.
-        CreativeMatrix creatives = get(Format(0,0));
+        if(!(imp.formats.empty()))
+        {
+            // The 0x0 format means: match anything.
+            CreativeMatrix creatives = get(Format(0,0));
 
-        for (const auto& format : imp.formats)
-            creatives |= get(format);
+            for (const auto& format : imp.formats)
+                creatives |= get(format);
 
-        state.narrowCreativesForImp(impIndex, creatives);
+            state.narrowCreativesForImp(impIndex, creatives);
+        }
     }
 
 

rtbkit/core/router/filters/testing/creative_filters_test.cc

@@ -48,6 +48,8 @@ BOOST_AUTO_TEST_CASE( testFormatFilter )
     addImp(r0, OpenRTB::AdPosition::ABOVE, { {300, 300} });
     addImp(r0, OpenRTB::AdPosition::ABOVE, { {400, 400} });
 
+    BidRequest r1;
+    addImp(r1, OpenRTB::AdPosition::ABOVE, {}); //Empty format bid request
 
     title("format-1");
     addConfig(filter, 0, c0, creatives);
@@ -59,6 +61,9 @@ BOOST_AUTO_TEST_CASE( testFormatFilter )
     check(filter, r0, creatives, 3, { {0},    {},    {0} });
     check(filter, r0, creatives, 4, { {0}                });
 
+    //Test empty format bid request
+    //Empty format is a wild card, all filters should pass.
+    check(filter, r1, creatives, 0, { {0, 1}, {0,1}, {0} });
 
     title("format-2");
     removeConfig(filter, 0, c0, creatives);

rtbkit/plugins/bidder_interface/http_bidder_interface.cc

@@ -635,12 +635,15 @@ void HttpBidderInterface::tagRequest(OpenRTB::BidRequest &request,
             auto &externalIds = imp.ext["external-ids"];
             externalIds.append(agentConfig->externalId);
 
-            auto& creativesExtField = imp.ext["creative-indexes"];
+            auto& creativesExtField = imp.ext["creative-ids"];
 
 
             auto &creativesList = creativesExtField[std::to_string(agentConfig->externalId)];
+            const auto& creatives = agentConfig->creatives;
             for (int index: creativeIndexes) {
-                creativesList.append(index);
+                ExcAssert(index < creatives.size());
+
+                creativesList.append(creatives[index].id);
             }
         }
 

rtbkit/plugins/exchange/bidswitch_exchange_connector.h

@@ -43,8 +43,14 @@ struct BidSwitchExchangeConnector: public OpenRTBExchangeConnector {
     getTimeAvailableMs(HttpAuctionHandler & connection,
                        const HttpHeader & header,
                        const std::string & payload) {
-        // TODO: check that is at it seems
-        return 200.0;
+        // Scan the payload quickly for the tmax parameter.
+        static const std::string toFind = "\"tmax\":";
+        std::string::size_type pos = payload.find(toFind);
+        if (pos == std::string::npos)
+            return 30.0; //ms as specified by the SLA 
+
+        int tmax = atoi(payload.c_str() + pos + toFind.length());
+        return (absoluteTimeMax < tmax) ? absoluteTimeMax : tmax;
     }
 
     /** This is the information that the BidSwitch exchange needs to keep

rtbkit/plugins/exchange/rtbkit_exchange_connector.h

@@ -10,6 +10,71 @@
 
 namespace RTBKIT {
 
+namespace {
+
+    /* Function to compare a plain int (signed or unsigned) to a string
+
+       We use this function to avoid allocating too much in the filter below
+       when we compare ids that are in the json
+
+       The function will throw if the @str parameter is not a valid number
+       Note that this function is not safe at all as it does not do any
+       overflow checking
+    */
+
+    bool string_int_equals(const char* str, int value) {
+        if (!str) return false;
+        if (str[0] == 0) return false;
+
+        /* Fast path, kinda */
+
+        if (value < 0 && str[0] != '-') return false;
+        if (value > 0 && str[0] == '-') return false;
+
+        /* Slow path */
+
+        /* Compute the length and check if there is any non-digit character
+           at the same time
+        */
+        const size_t len = [&]() {
+            size_t index = 0;
+
+            int c;
+            while ((c = str[index]) != 0) {
+                if (index == 0 && c != '-' && !isdigit(c))
+                    throw std::invalid_argument("Not a valid number");
+
+                ++index;
+            }
+
+            return index;
+        }();
+
+        if (value < 0)
+            value = -value;
+
+        size_t i = len - 1;
+
+        auto to_int = [](char c) {
+            return c - '0';
+        };
+
+        do {
+            const char digitChar = str[i];
+
+            const int currentDigit = value % 10;
+            if (to_int(digitChar) != currentDigit) return false;
+
+            value /= 10;
+            if (i == 0) break;
+            else --i;
+        } while (value > 0);
+
+        return value == 0 && i == 0;
+    }
+}
+
+
 struct CreativeIdsExchangeFilter
     : public IterativeCreativeFilter<CreativeIdsExchangeFilter>
 {
@@ -18,42 +83,54 @@ struct CreativeIdsExchangeFilter
     bool filterCreative(FilterState &state, const AdSpot &spot,
                         const AgentConfig &config, const Creative &creative) const
     {
+        if (spot.ext.isMember("creative-ids")) {
+            return filterCreativeIds(config, creative, spot.ext["creative-ids"]);
+        } else if (spot.ext.isMember("external-ids")) {
+            return filterExternalIds(config, spot.ext["external-ids"]);
+        }
+
+        return false;
+    }
+
+private:
+    bool filterExternalIds(
+            const AgentConfig& config, const Json::Value& extIds) const
+    {
+        ExcAssert(extIds.isArray());
 
-        auto doFilter = [&](const Json::Value& value) -> bool {
-            using std::find_if;  using std::begin;  using std::end;
-            using std::stoi;
-            if (value.isArray()) {
-                return find_if(begin(value), end(value), [&](const Json::Value &val) {
-                     return val.isIntegral() && val.asInt() == config.externalId;
-                }) != end(value);
+        using std::find_if;
+        using std::begin;   using std::end;
+
+        return find_if(
+            begin(extIds), end(extIds),
+            [&](const Json::Value &extId) {
+                 return extId.isIntegral() && extId.asInt() == config.externalId;
             }
-            else if (value.isObject()) {
-                char id[32];
-                snprintf(id, sizeof(id)-1, "%d", (int) config.externalId);
-                for (auto it = value.begin(), end = value.end(); it != end; ++it) {
-                    auto key = it.memberNameC();
-                    try {
-                        if (strcmp(id, key) == 0) {
-                            const auto& crids = *it;
-                            return find_if(crids.begin(), crids.end(), [&](const Json::Value& value) {
-                                return value.isIntegral() && value.asInt() == creative.id;
-                            }) != crids.end();
-                        }
-                    } catch (const std::invalid_argument&) {
-                        return false;
-                    }
+        ) != end(extIds);
+
+    }
+
+    bool filterCreativeIds(
+            const AgentConfig& config, const Creative& creative,
+            const Json::Value& crIds) const
+    {
+        ExcAssert(crIds.isObject());
+
+        using std::find_if;
+        using std::begin;   using std::end;
+
+        for (auto it = crIds.begin(), end = crIds.end(); it != end; ++it) {
+            auto key = it.memberNameC();
+            try {
+                if (string_int_equals(key, config.externalId)) {
+                    const auto& crids = *it;
+                    return find_if(crids.begin(), crids.end(), [&](const Json::Value& value) {
+                        return value.isIntegral() && value.asInt() == creative.id;
+                    }) != crids.end();
                 }
+            } catch (const std::invalid_argument&) {
+                return false;
             }
-            else {
-                ExcCheck(false, "Invalid code path");
-            }
-            return false;
-        };
-
-        if (spot.ext.isMember("creative-ids")) {
-            return doFilter(spot.ext["creative-ids"]);
-        } else if (spot.ext.isMember("external-ids")) {
-            return doFilter(spot.ext["external-ids"]);
         }
 
         return false;

rtbkit/plugins/exchange/spotx_exchange_connector.cc

@@ -68,6 +68,18 @@ SpotXExchangeConnector::initCreativeConfiguration()
 
             return true;
     }).required();
+
+    creativeConfig.addField(
+        "adid",
+        [](const Json::Value& value, CreativeInfo& info) {
+            Datacratic::jsonDecode(value, info.adid);
+            if (info.adid.empty()) {
+                throw std::invalid_argument("adid is required");
+            }
+
+            return true;
+    }).required();
+
 }
 
 ExchangeConnector::ExchangeCompatibility
@@ -191,6 +203,7 @@ SpotXExchangeConnector::setSeatBid(
     bid.price.val = USD_CPM(resp.price.maxPrice);
 
     bid.adomain = creativeInfo->adomain;
+    bid.adid = Id(creativeInfo->adid);
     bid.adm = creativeConfig.expand(creativeInfo->adm, context);
 }
 

rtbkit/plugins/exchange/spotx_exchange_connector.h

@@ -52,6 +52,7 @@ struct SpotXExchangeConnector : public OpenRTBExchangeConnector {
     struct CreativeInfo {
         std::string adm;
         std::vector<std::string> adomain;
+        std::string adid;
     };
 
     static Logging::Category print;