Merge pull request facebookresearch#40 from facebookresearch/mturk-new

added more documentation for mturk

parlai/mturk/run_mturk.py

@@ -29,7 +29,7 @@
 	task_module_name=task_module_name,
 	bot=Agent(opt=argparser.parse_args()), 
 	num_hits=2, # Number of HITs you want to create for this task
-	hit_reward=0.05, # in US dollars
-	is_sandbox=True,
+	hit_reward=0.05, # In US dollars
+	is_sandbox=True, # We suggest that you run it in MTurk sandbox mode to test first before moving to live site
 	verbose=True
 )

parlai/mturk/tasks/model_evaluator/agents.py

@@ -11,31 +11,45 @@
 
 class ModelEvaluatorAgent(Agent):
     """
-    MTurk agent for evaluating bots performance given a context.
+    MTurk agent for evaluating a dialog model's performance given a context.
     Assumes the context is a context from a given task, e.g.
     from SQuAD, CBT, etc.
     """
     def __init__(self, opt, shared=None):
         self.opt = copy.deepcopy(opt)
         self.id = 'Model Evaluator'
-        # The bot we will evaluate.
+
+        # The dialog model we will evaluate
         agent_opt = {}
         agent_opt['model'] = 'ir_baseline'
         agent = create_agent(agent_opt)
-        # The task that we will be collecting evaluation over.
+
+        # The task that we will evaluate the dialog model on
         task_opt = {}
         task_opt['datatype'] = 'test'
         task_opt['datapath'] = opt['datapath']
         task_opt['task'] = '#MovieDD-Reddit'
         self.world = create_task(task_opt, agent)
 
+    def observe(self, observation):
+        self.observation = observation
+
+        # The rating given by turker
+        # Because we only have one turn in this conversation, we don't need to track turn_index
+        # print(self.observation)
+        pass
+
     def act(self):
+        # All agents act once in the world
         self.world.parley()
+
         ad = {}
+        # Show the dialog model's response to the context, and ask the turker to rate the response
         ad['text'] = (
-            self.world.query['text'] + "\n\n" +
+            self.world.get_acts()[0]['text'] + "\n\n" +
             "How would you rate the following response (from 0 to 10):\n\n" +
-            self.world.reply['text'])
+            self.world.get_acts()[1]['text'])
+
         # TODO: deal with multi-turn dialogs, for now we will just deal
         # with 1-turn dialogs in this task.
         ad['episode_done'] = True  # self.world.episode_done()

parlai/mturk/tasks/model_evaluator/task_config.py

@@ -6,17 +6,35 @@
 
 task_config = {}
 
-# MTurk config related
+
+"""A short and descriptive title about the kind of task the HIT contains.
+On the Amazon Mechanical Turk web site, the HIT title appears in search results,
+and everywhere the HIT is mentioned.
+"""
 task_config['hit_title'] = 'Give a rating to a dialog between two people'
+
+
+"""A description includes detailed information about the kind of task the HIT contains.
+On the Amazon Mechanical Turk web site, the HIT description appears in the expanded
+view of search results, and in the HIT and assignment screens.
+"""
 task_config['hit_description'] = 'Give a rating to a dialog between two people.'
+
+
+"""One or more words or phrases that describe the HIT, separated by commas.
+On MTurk website, these words are used in searches to find HITs.
+"""
 task_config['hit_keywords'] = 'chat,dialog,rating'
 
-# Task specific
-task_config['teacher_agent_id'] = 'Teacher'
-task_config['worker_agent_id'] = task_config['teacher_agent_id']
 
-# Required for all tasks
-# Task description shown on the HIT task preview page and the left side of the HIT chat page
+"""A short name indicating the turker's role in the conversation.
+"""
+task_config['worker_agent_id'] = 'Teacher'
+
+
+"""A detailed task description that will be shown on the HIT task preview page
+and on the left side of the chat page. Supports HTML formatting.
+"""
 task_config['task_description'] = \
 '''\'\'\'
 In this task, you are going to read a dialog between two people, and you will need to give a rating on how good the response is.<br><br>

parlai/mturk/tasks/qa_data_collection/agents.py

@@ -10,7 +10,7 @@
 
 class QADataCollectionAgent(Agent):
     """
-    MTurk agent for recording questions and answers given a context.
+    MTurk agent for recording a turker's question and answer given a context.
     Assumes the context is a random context from a given task, e.g.
     from SQuAD, CBT, etc.
     """
@@ -19,6 +19,7 @@ def __init__(self, opt, shared=None):
         self.id = 'QA Collector'
         self.turn_index = -1
 
+        # Initialize a SQuAD teacher agent, which we will later get context from
         module_name = 'parlai.tasks.squad.agents'
         class_name = 'DefaultTeacher'
         my_module = importlib.import_module(module_name)
@@ -28,17 +29,42 @@ def __init__(self, opt, shared=None):
         task_opt['datapath'] = opt['datapath']
         self.task = task_class(task_opt)
 
+    def observe(self, observation):
+        self.observation = observation
+
+        if self.turn_index == 0:
+            # Turker's question, from the first turn
+            # print(self.observation)
+            pass
+        elif self.turn_index == 1:
+            # Turker's answer, from the second turn
+            # print(self.observation)
+            pass
+
     def act(self):
-        self.turn_index = (self.turn_index + 1) % 2;
+        self.turn_index = (self.turn_index + 1) % 2; # Each turn starts from the QA Collector agent
         ad = { 'episode_done': False }
         ad['id'] = self.id
+
         if self.turn_index == 0:
-            # get context
+            # At the first turn, the QA Collector agent provides the context and
+            # prompts the turker to ask a question regarding the context
+
+            # Get context from SQuAD teacher agent
             qa = self.task.act()
             context = '\n'.join(qa['text'].split('\n')[:-1])
+
+            # Wrap the context with a prompt telling the turker what to do next
             ad['text'] = (context + 
                         '\n\nPlease provide a question given this context.')
+
         if self.turn_index == 1:
+            # At the second turn, the QA Collector collects the turker's question from the first turn,
+            # and then prompts the turker to provide the answer
+
+            # A prompt telling the turker what to do next
             ad['text'] = 'Thanks. And what is the answer to your question?'
+
             ad['episode_done'] = True  # end of episode
+
         return ad

parlai/mturk/tasks/qa_data_collection/task_config.py

@@ -6,17 +6,35 @@
 
 task_config = {}
 
-# MTurk config related
+
+"""A short and descriptive title about the kind of task the HIT contains.
+On the Amazon Mechanical Turk web site, the HIT title appears in search results,
+and everywhere the HIT is mentioned.
+"""
 task_config['hit_title'] = 'Ask and answer a question about a paragraph'
+
+
+"""A description includes detailed information about the kind of task the HIT contains.
+On the Amazon Mechanical Turk web site, the HIT description appears in the expanded
+view of search results, and in the HIT and assignment screens.
+"""
 task_config['hit_description'] = 'Ask and answer a question about a paragraph.'
+
+
+"""One or more words or phrases that describe the HIT, separated by commas.
+On MTurk website, these words are used in searches to find HITs.
+"""
 task_config['hit_keywords'] = 'chat,question,answer'
 
-# Task specific
-task_config['teacher_agent_id'] = 'Teacher'
-task_config['worker_agent_id'] = task_config['teacher_agent_id']
 
-# Required for all tasks
-# Task description shown on the left side of the HIT chat page
+"""A short name indicating the turker's role in the conversation.
+"""
+task_config['worker_agent_id'] = 'Teacher'
+
+
+"""A detailed task description that will be shown on the HIT task preview page
+and on the left side of the chat page. Supports HTML formatting.
+"""
 task_config['task_description'] = \
 '''\'\'\'
 In this task, you will need to ask a question about a paragraph, and then provide your own answer to it.<br><br>