feat: add dd-trace to edxapp workers (edx#6)

This PR adds dd-trace to the worker boxes so we can monitor them (and celery tasks). It does not require dd to be enabled.
---------

Co-authored-by: Tim McCormack <[email protected]>

Author: connorhaugh

playbooks/roles/edxapp/templates/edx/app/edxapp/worker.sh.j2

@@ -4,19 +4,31 @@
 
 {% set edxapp_venv_bin = edxapp_venv_dir + "/bin" %}
 source {{ edxapp_app_dir }}/edxapp_env
+{% set executable = edxapp_venv_bin + '/celery' %}
+
 {% if COMMON_ENABLE_NEWRELIC_APP %}
-{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/celery' %}
+{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + executable %}
 
 export NEW_RELIC_CONFIG_FILE="{{ edxapp_app_dir }}/newrelic.ini"
 if command -v ec2metadata >/dev/null 2>&1; then
   INSTANCEID=$(ec2metadata --instance-id);
   HOSTNAME=$(hostname)
   export NEW_RELIC_PROCESS_HOST_DISPLAY_NAME="$HOSTNAME-$INSTANCEID"
 fi
-{% else %}
-{% set executable = edxapp_venv_bin + '/celery' %}
 {% endif %}
 
+{% if EDXAPP_DATADOG_ENABLE %}
+{% set executable = edxapp_venv_bin + '/ddtrace-run ' + executable %}
+export DD_TAGS="service:edx-edxapp-workers-{{ SERVICE_VARIANT }},queue:{{QUEUE_NAME}}"
+export DD_DJANGO_USE_LEGACY_RESOURCE_FORMAT=true
+# Copied from edx_django_service playbook for consistency; Datadog
+# trace debug logging issue doesn't actually affect edxapp for some
+# reason.
+export DD_TRACE_LOG_STREAM_HANDLER=false
+# Datadog's instrumentation breaks pymongo: https://github.com/edx/edx-arch-experiments/issues/580
+export DD_TRACE_PYMONGO_ENABLED=false
+{% endif -%}
+
 # We exec so that celery is the child of supervisor and can be managed properly
 
 exec {{ executable }} $@

playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2

@@ -1,7 +1,7 @@
 {% for w in edxapp_workers %}
 [program:{{ w.service_variant }}_{{ w.queue }}_{{ w.concurrency }}]
 
-environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ EDXAPP_NEWRELIC_WORKERS_APPNAME }}-{{ w.service_variant }},NEW_RELIC_DISTRIBUTED_TRACING_ENABLED={{ EDXAPP_WORKERS_ENABLE_NEWRELIC_DISTRIBUTED_TRACING }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}CONCURRENCY={{ w.concurrency }},LOGLEVEL=info,DJANGO_SETTINGS_MODULE={{ w.service_variant }}.envs.{{ worker_django_settings_module }},LANG={{ EDXAPP_LANG }},PYTHONPATH={{ edxapp_code_dir }},SERVICE_VARIANT={{ w.service_variant }},BOTO_CONFIG="{{ edxapp_app_dir }}/.boto",EDX_REST_API_CLIENT_NAME=edx.{{ w.service_variant }}.core.{{ w.queue }}
+environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ EDXAPP_NEWRELIC_WORKERS_APPNAME }}-{{ w.service_variant }},NEW_RELIC_DISTRIBUTED_TRACING_ENABLED={{ EDXAPP_WORKERS_ENABLE_NEWRELIC_DISTRIBUTED_TRACING }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}CONCURRENCY={{ w.concurrency }},LOGLEVEL=info,DJANGO_SETTINGS_MODULE={{ w.service_variant }}.envs.{{ worker_django_settings_module }},LANG={{ EDXAPP_LANG }},PYTHONPATH={{ edxapp_code_dir }},SERVICE_VARIANT={{ w.service_variant }},BOTO_CONFIG="{{ edxapp_app_dir }}/.boto",EDX_REST_API_CLIENT_NAME=edx.{{ w.service_variant }}.core.{{ w.queue }},QUEUE_NAME={{ w.service_variant }}.core.{{ w.queue }}
 user={{ common_web_user }}
 directory={{ edxapp_code_dir }}
 stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log