Add specific config stuff of plugin being set in the settings file.

pablin87 · pablin87 · commit 64451149d3ed · 2014-04-03T12:26:40.000-03:00
Some minor changes for the ParameterPlugin interface to make it
receive a config parameter. This config is get from the settings
file and is specific of each plug in(can have anything).
Also make the openrtb plugin independent of the mopub exchange. The
next step will be to make rubicon from this one.
Aslo add initial Readme file.
Add mopub specific settings for the openrtb plugin under
settings/mopub_config.py
Also click notifications for the openrtb plugin.
diff --git a/README b/README
@@ -1,8 +1,8 @@
 Requirements
 ============
 
-- libev (from source, > 4.15)
+- libev (from source, > 4.15) put to make an easy install ./configure --prefix=/usr
 - pyev
 - http_parser
-
+- mako
 
diff --git a/exchange/exchange.py b/exchange/exchange.py
@@ -14,7 +14,7 @@
 from settings import MAX_CONNS, MAX_EVENT_CONNS, CHECK_CONNS_TO, CHECK_PENDING_TO, \
                     TEMPLATE_FILENAME, EVENT_ENDPOINT, PARAMETER_PLUGIN, \
                     KEEP_ALIVE_HTTP_REQUEST, EVENT_CONN_KEEP_ALIVE_TO, \
-                    PLUGIN_DO_TO, REPORT_WINS
+                    PLUGIN_DO_TO, REPORT_WINS, PLUGIN_CONFIG
 
 from rtb import RTBRequestFactory
 from adserver import AdServer
@@ -78,7 +78,9 @@ def __init__(self, dsp_endpoints, event_endpoint, balance_conn_timeout):
                                     TEMPLATE_FILENAME)
         self.adserver = AdServer(self.loop)
         self.request_fact.initialize()
-        self.request_fact.set_parameter_plug(PARAMETER_PLUGIN, self.adserver)
+        self.request_fact.set_parameter_plug(PARAMETER_PLUGIN, 
+                                             self.adserver,
+                                             PLUGIN_CONFIG)
         if PLUGIN_DO_TO:
             self.watchers.append(pyev.Timer(
                                     PLUGIN_DO_TO, 
diff --git a/plugin/datacratic_plugin.py b/plugin/datacratic_plugin.py
@@ -15,7 +15,7 @@ class DatacraticPlugin(ParameterPlugin):
     def __init__(self):
         self.aid = None
 
-    def initialize(self, adserver):
+    def initialize(self, adserver, config):
         self.adserver = adserver
 
     def get_request(self):
diff --git a/plugin/openrtb_plugin.py b/plugin/openrtb_plugin.py
@@ -5,17 +5,6 @@
 import logging
 from urlparse import urlparse
 
-# CONFIG (see to put this in any other part; aka not global)
-from render_utils import incrementor
-RENDER_MAP = {
-              'auction_id' : incrementor(12345678)}
-BODY_TEMPLATES = ['plugin/mopub/mopub_body1.tmpl', 
-                  'plugin/mopub/mopub_body2.tmpl']
-HEH_ENDPOINT_TMPL = "http://localhost:8080/impression/${exchange}/${AUCTION_ID}/${AUCTION_PRICE}?impid=${AUCTION_IMP_ID}"
-AD_SERVER_ENDPOINT_TMPL = "http://localhost:8080/events?ev=imp&aid=${AUCTION_ID}&apr=${AUCTION_PRICE}&sptid=${AUCTION_IMP_ID}"
-USE_HEH_ENDPOINT = False
-HTTP_RESOURCE = 'mopub'
-
 
 class OpenRTBPlugin(ParameterPlugin):
     '''
@@ -24,34 +13,42 @@ class OpenRTBPlugin(ParameterPlugin):
     def __init__(self):
         self.request_body_templates = []
         self.render_map = {}
-        self.tmpl_notif = ''
+        self.tmpl_imp_notif = ''
     
-    def initialize(self, adserver):
+    def initialize(self, adserver, config):
         
         self.adserver = adserver
+        self.http_resource = config['http_resource']
+        self.exchange = config['exchange']
         
-        self.render_map = RENDER_MAP
-        self.request_body_templates_files = BODY_TEMPLATES
+        # Map to render the bid request body templates
+        self.render_map = config['render_map']
         
-        # Create templates...
+        # Create templates fot the bid requests...
+        self.request_body_templates_files = config['req_body_templates']
         for filename in self.request_body_templates_files:
             with open(filename) as f:
                 logging.info('Using file template %s' % filename)
                 tmpl = Template(''.join(f.readlines()))
                 self.request_body_templates.append(tmpl)
         
-        # Template for notification endpoint
-        if USE_HEH_ENDPOINT :
-            self.tmpl_notif_file = HEH_ENDPOINT_TMPL
+        # Templates for notification endpoint
+        if config['use_heh_endpoint'] :
+            self.tmpl_imp_notif_file = config['heh_endpt_imp_tmpl']
+            self.tmpl_click_notif_file = config['heh_endpt_click_tmpl']
         else :
-            self.tmpl_notif_file = AD_SERVER_ENDPOINT_TMPL
-        self.tmpl_notif = Template(self.tmpl_notif_file)
+            self.tmpl_imp_notif_file = config['adserver_endpt_imp_tmpl']
+            self.tmpl_click_notif_file = config['adserver_endpt_click_tmpl']
+        
+        # Create the templates for the notifications
+        self.tmpl_imp_notif = Template(self.tmpl_imp_notif_file)
+        self.tmpl_click_notif = Template(self.tmpl_click_notif_file)
 
     def get_request(self):
-        # We need to return a request line, a map of headers and a body string
+        # We need to return a request line, a map of headers and a body
         
         # Create the request line
-        req_line = 'POST /%s HTTP/1.1' % HTTP_RESOURCE
+        req_line = 'POST /%s HTTP/1.1' % self.http_resource
         
         # Set the headers
         headers = {}
@@ -81,19 +78,53 @@ def receive_response(self, status_code, headers, body):
         js = json.loads(body)
         logging.debug("Response received :")
         logging.debug(str(js))
-        price = js['seatbid'][0]['bid'][0]['price']
-        auction_id = js['id']
-        spot_id = js['seatbid'][0]['bid'][0]['impid']
+        price = self.get_auction_price(js)
+        auction_id = self.get_auction_id(js)
+        spot_id = self.get_auction_impression_id(js)
         
         # With that data, create the notification...
         notif_render = { 'AUCTION_PRICE' : price,
                          'AUCTION_ID' : auction_id,
                          'AUCTION_IMP_ID' : spot_id,
-                         'exchange' : 'mopub'}
-        url = self.tmpl_notif.substitute(notif_render)
+                         'exchange' : self.exchange }
+        
+        # Win notification...
+        url = self.tmpl_imp_notif.substitute(notif_render)
         self.__send_impression_notification(url)
+        
+        # Click notification... 
+        click_url = self.tmpl_click_notif.substitute(notif_render)
+        self.__send_click_notification(click_url)
+        
         return (False, '', {}, '')
         
+    def get_auction_price(self, json_response):
+        return json_response['seatbid'][0]['bid'][0]['price']
+
+    def get_auction_id(self, json_response):
+        return json_response['id']
+
+    def get_auction_impression_id(self, json_response):
+        return json_response['seatbid'][0]['bid'][0]['impid']
+
+    def __send_click_notification(self, url):
+        
+        # Only generate clicks for the 2% of the cases
+        if random.random < 0.98:
+            return
+        
+        parsed_url = urlparse(url)
+        req_line = 'GET %s?%s HTTP/1.1' % (parsed_url.path,
+                                           parsed_url.query)
+        headers = {}
+        headers['Host'] = 'localhost'
+        heads = self.__headers_to_str(headers)
+        buf = '%s\r\n%s\r\n' % (req_line, heads)
+        
+        # send the impression event in 0.1 secs
+        logging.debug("Sending click notification :")
+        logging.debug(buf)
+        self.adserver.send_event(buf, 0.1)
         
     def __send_impression_notification(self, url):
         parsed_url = urlparse(url)
@@ -105,7 +136,7 @@ def __send_impression_notification(self, url):
         buf = '%s\r\n%s\r\n' % (req_line, heads)
         
         # send the impression event in 0.1 secs
-        logging.debug("Sending notification :")
+        logging.debug("Sending impression notification :")
         logging.debug(buf)
         self.adserver.send_event(buf, 0.1)
         
@@ -115,9 +146,8 @@ def __headers_to_str(self, headers):
             heads += '%s: %s\r\n' % (k, v)
         return heads
 
-
     def receive_win_response(self, status_code, headers, body):
         logging.debug('received_win_response')
 
     def do(self, watcher, revents):
-        logging.info('doing...')
+        logging.debug('doing...')
diff --git a/plugin/parameter_plugin.py b/plugin/parameter_plugin.py
@@ -5,7 +5,7 @@ class ParameterPlugin(object):
     def __init__(self):
         pass
     
-    def initialize(self, adserver):
+    def initialize(self, adserver, config):
         raise NotImplementedError('initialize must be implemeted')
 
     def get_request(self):
diff --git a/plugin/rubicon_plugin.py b/plugin/rubicon_plugin.py
@@ -45,7 +45,7 @@ def __init__(self):
         self.request_templates = []
         self.aid = None 
 
-    def initialize(self, adserver):
+    def initialize(self, adserver, config):
         self.adserver = adserver
         for name in REQUEST_FILES:
             with open(os.path.join(BASE_PATH, name), 'rb') as f:
diff --git a/rtb/request_factory.py b/rtb/request_factory.py
@@ -18,13 +18,13 @@ def initialize(self):
         # read the template file        
         self.template = Template(filename=self.template_file)
     
-    def set_parameter_plug(self, plugin, adserver):
+    def set_parameter_plug(self, plugin, adserver, config):
         '''
             Set a param class plugin, this must implement
             the ParameterPlugin interface
         '''
         self.plugin_instance = plugin()
-        self.plugin_instance.initialize(adserver)
+        self.plugin_instance.initialize(adserver, config)
 
     def create_request(self, set_aid=True):
         '''
diff --git a/settings/mopub_config.py b/settings/mopub_config.py
@@ -0,0 +1,42 @@
+from render_utils import incrementor
+
+conf = {
+        
+        # Name of the exchange being simulated (used to generate notifications)
+        # in the correct path
+        'exchange' : 'mopub',
+        
+        # This define the body templates that are being sent to the rtbkit
+        # exchange connector endpoint (files).
+        'req_body_templates' : [ 
+                                'plugin/mopub/mopub_body1.tmpl', 
+                                'plugin/mopub/mopub_body2.tmpl'
+                                ],
+        
+        # This map defines a list of functions that when being called return a 
+        # value to be rendered in the templates defined in the 
+        # 'req_body_templates' field. For example to return a constant value 
+        # 'auction_id' : lambda : 5   (will return always five)
+        'render_map' : {
+                        'auction_id' : incrementor(12345678)
+                        },
+        
+        # Definde what resource of the exchange connector endpoint is hitted
+        # in order to place a bid request. 
+        'http_resource' : 'mopub',
+
+        # Define the url where the notifications of impressions and clicks will
+        # be send. By this, the adm field is not have in count. The ip and
+        # port set it here are replaced by global parameters of the endpoint 
+        # settings. 
+        'adserver_endpt_imp_tmpl' : "http://localhost:8080/events?ev=imp&aid=${AUCTION_ID}&apr=${AUCTION_PRICE}&sptid=${AUCTION_IMP_ID}",
+        'adserver_endpt_click_tmpl' : "http://localhost:8080/events?ev=cli&aid=${AUCTION_ID}&sptid=${AUCTION_IMP_ID}",
+        
+        # If set to true, instead of using the 'adserver_endpt_imp_tmpl' and 
+        # 'adserver_endpt_click_tmpl', it will use the following heh templates
+        # in order to hit heh instead of the ad server connector directly.
+        'use_heh_endpoint' : False, 
+        'heh_endpt_imp_tmpl' : "http://localhost:8080/impression/${exchange}/${AUCTION_ID}/${AUCTION_PRICE}?impid=${AUCTION_IMP_ID}",
+        'heh_endpt_click_tmpl' : "http://localhost:8080/click/${exchange}/${AUCTION_ID}?impid=${AUCTION_IMP_ID}"
+        
+        }
diff --git a/settings/render_utils.py b/settings/render_utils.py
diff --git a/settings/settings.py b/settings/settings.py
@@ -3,6 +3,7 @@
 # Plugin imports
 from plugin.rubicon_plugin import RubiconPlugin
 from plugin.datacratic_plugin import DatacraticPlugin
+from plugin.openrtb_plugin import OpenRTBPlugin
 
 # Max connections for bid requests allowed for the process
 MAX_CONNS = 1
@@ -13,12 +14,12 @@
 #  - endpoint should be a string 'host:port'
 #  - expected_qps is the amount of qps expected for the endpoint
 ENDPOINT_LIST = [
-    ('localhost:12339', 100),
+    ('10.0.2.21:12340', 20),
 ]
 
 # Event endpoint :
 # - endpoint should be a string 'host:port'
-EVENT_ENDPOINT = 'localhost:12340'
+EVENT_ENDPOINT = '10.0.2.21:12350'
 
 # Balance time out indicating the period in seconds 
 # to balance connections
@@ -33,7 +34,7 @@
 
 # Timeout in seconds to periodically invoke the plugin.do method
 # set to None if it does not need to be invoked
-PLUGIN_DO_TO = 2
+PLUGIN_DO_TO = 10
 
 # Report wps statistics
 REPORT_WINS = False
@@ -53,10 +54,15 @@
 # - logging.INFO
 # - logging.WARNING
 # - logging.ERROR
-LOG_LEVEL = logging.INFO
+LOG_LEVEL = logging.DEBUG
 
 # Parameter plugin
-PARAMETER_PLUGIN = RubiconPlugin
+#PARAMETER_PLUGIN = RubiconPlugin
+PARAMETER_PLUGIN = OpenRTBPlugin
+
+# Configuration map that will be passed in the initialize 
+import mopub_config 
+PLUGIN_CONFIG = mopub_config.conf
 
 # RTB request template filename
 TEMPLATE_FILENAME = 'templates/request.template'
