Skip to content

Commit

Permalink
Refactor MTurk html to make it more modular
Browse files Browse the repository at this point in the history
  • Loading branch information
Will Feng committed Jul 6, 2017
1 parent c5fd1eb commit 2b413f2
Show file tree
Hide file tree
Showing 12 changed files with 467 additions and 301 deletions.
3 changes: 2 additions & 1 deletion docs/source/mturk.rst
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,8 @@ A few things to keep in mind:

1. To end a conversation, you should send a message with ``episode_done = True`` from the first non-MTurk agent, and the conversation is ended after all MTurk agents respond.
2. In ``run.py``, You can use ``hit_index`` and ``assignment_index`` to differentiate between different HITs and assignments, and change the content of the task accordingly.
3. Make sure to test your dialog task using MTurk's sandbox mode before pushing it live, by using the ``--sandbox`` flag (enabled by default) when running ``run.py``.
3. If you want to show a custom webpage for any of your MTurk agents, you can create an ``html`` folder within your task directory, and then create the ``<mturk_agent_id>_cover_page.html`` and ``<mturk_agent_id>_index.html`` files within the ``html`` directory. In those files, you can extend from ``core.html`` (please look at `parlai/mturk/core/html/mturk_index.html <https://github.com/facebookresearch/ParlAI/blob/master/parlai/mturk/core/html/mturk_index.html>`__ as an example) and override any code blocks that you want to change. These agent-specific templates will automatically be shown to the Turkers in the next run.
4. Make sure to test your dialog task using MTurk's sandbox mode before pushing it live, by using the ``--sandbox`` flag (enabled by default) when running ``run.py``.


Running a Task
Expand Down
15 changes: 12 additions & 3 deletions parlai/mturk/core/agents.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import json
import requests
from parlai.core.agents import create_agent_from_shared
from .setup_aws import setup_aws, check_mturk_balance, create_hit_type, create_hit_with_hit_type, setup_aws_credentials
from .setup_aws import setup_aws, check_mturk_balance, create_hit_type, create_hit_with_hit_type, setup_aws_credentials, create_hit_config
import threading
from .data_model import Base, Message
from .data_model import get_new_messages as _get_new_messages
Expand All @@ -31,6 +31,7 @@
polling_interval = 1 # in seconds
create_hit_type_lock = threading.Lock()
local_db_lock = threading.Lock()
task_dir = os.getcwd()

class MTurkManager():
def __init__(self):
Expand All @@ -44,6 +45,7 @@ def __init__(self):
self.run_id = None
self.mturk_agent_ids = None
self.all_agent_ids = None
self.task_files_to_copy = None

def init_aws(self, opt):
self.run_id = str(int(time.time()))
Expand All @@ -58,7 +60,14 @@ def init_aws(self, opt):
return

print('Setting up MTurk backend...')
html_api_endpoint_url, json_api_endpoint_url, requester_key_gt = setup_aws(task_description=opt['task_description'], num_hits=opt['num_hits'], num_assignments=opt['num_assignments'], is_sandbox=opt['is_sandbox'])
create_hit_config(task_description=opt['task_description'], num_hits=opt['num_hits'], num_assignments=opt['num_assignments'], is_sandbox=opt['is_sandbox'])
if not self.task_files_to_copy:
self.task_files_to_copy = []
for mturk_agent_id in self.mturk_agent_ids:
self.task_files_to_copy.append(os.path.join(task_dir, 'html', mturk_agent_id+'_cover_page.html'))
self.task_files_to_copy.append(os.path.join(task_dir, 'html', mturk_agent_id+'_index.html'))
self.task_files_to_copy.append(os.path.join(task_dir, 'html', 'approval_index.html'))
html_api_endpoint_url, json_api_endpoint_url, requester_key_gt = setup_aws(task_files_to_copy = self.task_files_to_copy)
self.html_api_endpoint_url = html_api_endpoint_url
self.json_api_endpoint_url = json_api_endpoint_url
self.requester_key_gt = requester_key_gt
Expand Down Expand Up @@ -180,7 +189,7 @@ def create_hits(self, opt):
is_sandbox=opt['is_sandbox']
)
all_agent_ids_string = str(self.all_agent_ids).replace("'", '''"''')
mturk_chat_url = self.html_api_endpoint_url + "?method_name=chat_index&task_group_id="+str(self.task_group_id)+"&all_agent_ids="+all_agent_ids_string+"&cur_agent_id="+str(mturk_agent_id)+"&task_additional_info="+str(opt.get('task_additional_info', ''))
mturk_chat_url = self.html_api_endpoint_url + "?method_name=chat_index&task_group_id="+str(self.task_group_id)+"&all_agent_ids="+all_agent_ids_string+"&cur_agent_id="+str(mturk_agent_id)
mturk_page_url = create_hit_with_hit_type(
page_url=mturk_chat_url,
hit_type_id=hit_type_id,
Expand Down
35 changes: 20 additions & 15 deletions parlai/mturk/core/handler_template.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import data_model

# Dynamically generated code begin
# Expects mturk_submit_url, frame_height, rds_host, rds_db_name, rds_username, rds_password, task_description, requester_key_gt, num_hits, num_assignments, is_sandbox
# Expects mturk_submit_url, frame_height, rds_host, rds_db_name, rds_username, rds_password, requester_key_gt
# {{block_task_config}}
# Dynamically generated code end

Expand Down Expand Up @@ -56,27 +56,35 @@ def chat_index(event, context):
all_agent_ids = event['query']['all_agent_ids']
cur_agent_id = event['query']['cur_agent_id']
assignment_id = event['query']['assignmentId'] # from mturk
task_additional_info = event['query'].get('task_additional_info', '') # Maximum length: 1000 characters

if assignment_id == 'ASSIGNMENT_ID_NOT_AVAILABLE':
template_context['task_description'] = task_description
template_context['is_cover_page'] = True
custom_cover_page = cur_agent_id + '_cover_page.html'
if os.path.exists(custom_cover_page):
return _render_template(template_context, custom_cover_page)
else:
return _render_template(template_context, 'cover_page.html')
else:
template_context['task_group_id'] = task_group_id
template_context['hit_index'] = hit_index
template_context['assignment_index'] = assignment_index
template_context['cur_agent_id'] = cur_agent_id
template_context['all_agent_ids'] = all_agent_ids
template_context['task_description'] = task_description.replace('{{task_additional_info}}', task_additional_info)
template_context['mturk_submit_url'] = mturk_submit_url
template_context['is_cover_page'] = False
template_context['frame_height'] = frame_height

return _render_template(template_context, 'mturk_index.html')
custom_index_page = cur_agent_id + '_index.html'
if os.path.exists(custom_index_page):
return _render_template(template_context, custom_index_page)
else:
return _render_template(template_context, 'mturk_index.html')

except KeyError:
raise Exception('400')

def get_hit_config(event, context):
if event['method'] == 'GET':
with open('hit_config.json', 'r') as hit_config_file:
return json.loads(hit_config_file.read().replace('\n', ''))

def save_hit_info(event, context):
if event['method'] == 'POST':
"""
Expand All @@ -89,6 +97,7 @@ def save_hit_info(event, context):
assignment_id = params['assignmentId']
hit_id = params['hitId']
worker_id = params['workerId']
is_sandbox = params['is_sandbox']

data_model.set_hit_info(
db_session = db_session,
Expand Down Expand Up @@ -181,12 +190,13 @@ def get_hit_index_and_assignment_index(event, context):
try:
task_group_id = event['query']['task_group_id']
agent_id = event['query']['agent_id']
num_assignments = event['query']['num_assignments']

return data_model.get_hit_index_and_assignment_index(
db_session=db_session,
task_group_id=task_group_id,
agent_id=agent_id,
num_assignments=num_assignments
num_assignments=int(num_assignments)
)
except KeyError:
raise Exception('400')
Expand All @@ -212,14 +222,9 @@ def approval_index(event, context):
template_context['hit_index'] = hit_index
template_context['assignment_index'] = assignment_index
template_context['mturk_agent_ids'] = mturk_agent_ids
template_context['task_description'] = task_description
template_context['is_cover_page'] = False
template_context['is_approval_page'] = True
template_context['num_hits'] = int(num_hits)
template_context['num_assignments'] = int(num_assignments)
template_context['frame_height'] = frame_height

return _render_template(template_context, 'mturk_index.html')
return _render_template(template_context, 'approval_index.html')

except KeyError:
raise Exception('400')
Expand Down
150 changes: 150 additions & 0 deletions parlai/mturk/core/html/approval_core.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
<!--
Copyright (c) 2017-present, Facebook, Inc.
All rights reserved.
This source code is licensed under the BSD-style license found in the
LICENSE file in the root directory of this source tree. An additional grant
of patent rights can be found in the PATENTS file in the same directory.
-->
{% extends "core.html" %}

{% block right_pane %}
<div id="right-pane" style="min-height: 100%; display: flex; flex-direction: column; justify-content: space-between;">
<div id="right-top-pane" style="width: 100%; height: 570px; padding-top: 60px; padding-left: 20px; padding-right: 20px; padding-bottom: 20px; overflow:scroll; ">
<div id="message_thread" style="width: 100%">
</div>
</div>

<div id="right-bottom-pane" style="width: 100%; background-color: #eee">
<div id="approval" style="padding-left: 35px; padding-top: 30px; padding-bottom: 30px; padding-right: 35px; float: left">
<div id="approval-prompt-text" style="width: 100%; display: block; float: left">
<span class="prompt-text" style="width: 100%; float: left; font-size: 16px">Do you approve this work?</span>
</div>
<div id="approval-buttons-group" style="width: 100%; display: block; float: left; margin-top: 30px;">
<button id="approve-button" type="button" class="btn btn-primary btn-lg" style="width: 200px">
<span class="glyphicon glyphicon-ok" aria-hidden="true"></span> Approve
</button>
<button id="reject-button" type="button" class="btn btn-danger btn-lg" style="margin-left: 20px; width: 200px">
<span class="glyphicon glyphicon-remove" aria-hidden="true"></span> Reject
</button>
</div>
</div>
</div>
</div>
{% endblock %}

{% block handle_new_messages %}
<script type="text/javascript">
function handle_new_messages(new_messages) {
for (var i = 0; i < new_messages.length; i++) {
var message = new_messages[i];
var agent_id = message['id'];
var message_id = parseInt(message['message_id']);
var message_text = message['text'].replace(/(?:\r\n|\r|\n)/g, '<br />');
var reward = null;
var received_message_from_other_agents = false;
if ('reward' in message) {
reward = message['reward']
}

if (!(message_id in messages_shown)) {
if ($.inArray(agent_id, mturk_agent_ids) == -1) {
$('div#message_thread').append(`
<div class="row" style="margin-left: 0; margin-right: 0">
<div class="alert alert-warning" role="alert" style="float: left; display:table">
<span style="font-size: 16px"><b>`+agent_id+`</b>: `+message_text+`</span>
</div>
</div>
`);
received_message_from_other_agents = true;
} else {
if (reward) {
var reward_text = reward;
if (reward > 0) {
reward_text = '+' + reward_text;
}
$('div#message_thread').append(`
<div class="row" style="margin-left: 0; margin-right: 0">
<div class="alert alert-info" role="alert" style="float: right; display:table">
<span style="font-size: 16px"><b>`+agent_id+`</b>: `+reward_text+`</span>
</div>
</div>
`);
} else {
$('div#message_thread').append(`
<div class="row" style="margin-left: 0; margin-right: 0">
<div class="alert alert-info" role="alert" style="float: right; display:table">
<span style="font-size: 16px"><b>`+agent_id+`</b>: `+message_text+`</span>
</div>
</div>
`);
}
}
messages_shown[message_id] = true;
}
}
if (new_messages.length > 0) {
$("div#message_thread").css("display", "");
scroll_conversation_to_bottom();
}
}
</script>
{% endblock %}

{% block additional_scripts %}
<script type="text/javascript">
function send_approval_and_update_UI(action) {
$("div#message_thread").css("display", "none");
$("div#approval-prompt-text span.prompt-text").text("Submitting to MTurk...");
$("div#approval-buttons-group").css("display", "none");

var post_data_dict = {
requester_key: get_url_parameter('requester_key'),
task_group_id: task_group_id,
conversation_id: conversation_id,
action: action
};
post_data_dict['method_name'] = 'review_hit';
$.ajax({
url: '/prod/json',
type: "POST",
data: JSON.stringify(post_data_dict),
contentType: "application/json",
success: function(data){
if (verbose_log) console.log(data);

$("div#approval-prompt-text span.prompt-text").text("Do you approve this work?");
$("div#approval-buttons-group").css("display", "");

hit_index = parseInt(hit_index);
assignment_index = parseInt(assignment_index);

if (assignment_index < num_assignments) {
window.location.href = window.location.href.replace('hit_index='+hit_index+'&assignment_index='+assignment_index, 'hit_index='+hit_index+'&assignment_index='+(assignment_index+1));
} else if (hit_index < num_hits) {
window.location.href = window.location.href.replace('hit_index='+hit_index+'&assignment_index='+assignment_index, 'hit_index='+(hit_index+1)+'&assignment_index=1');
} else {
$("div#message_thread").css("display", "none");
$("div#approval-prompt-text span.prompt-text").text("All done! Please feel free to close the window.");
$("div#approval-buttons-group").css("display", "none");
}
}
});
}

$("button#approve-button").click(function() {
send_approval_and_update_UI('approve');
});

$("button#reject-button").click(function() {
send_approval_and_update_UI('reject');
});
</script>
{% endblock %}

{% block additional_init_scripts %}
<script type="text/javascript">
function init_additional() {
$("div#left-pane").css("min-height", $(window).height());
}
</script>
{% endblock %}
8 changes: 8 additions & 0 deletions parlai/mturk/core/html/approval_index.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
<!--
Copyright (c) 2017-present, Facebook, Inc.
All rights reserved.
This source code is licensed under the BSD-style license found in the
LICENSE file in the root directory of this source tree. An additional grant
of patent rights can be found in the PATENTS file in the same directory.
-->
{% extends "approval_core.html" %}
Loading

0 comments on commit 2b413f2

Please sign in to comment.