feat: add analytics data api (openedx-unsupported#847)

Makefile

@@ -442,6 +442,9 @@ dev.shell.studio_watcher:
 dev.shell.xqueue_consumer:
 	docker-compose exec xqueue_consumer env TERM=$(TERM) /edx/app/xqueue/devstack.sh open
 
+dev.shell.analyticsapi:
+	docker exec -it edx.devstack.analyticsapi env TERM=$(TERM) /edx/app/analytics_api/devstack.sh open
+
 dev.shell.insights:
 	docker-compose exec insights env TERM=$(TERM) bash -c 'eval $$(source /edx/app/insights/insights_env; echo PATH="$$PATH";) && /edx/app/insights/devstack.sh open'
 

README.rst

@@ -322,6 +322,8 @@ Instead of a service name or list, you can also run commands like ``make dev.pro
 +------------------------------------+-------------------------------------+----------------+--------------+
 | `insights`                         | http://localhost:18110              | Python/Django  | Extra        |
 +------------------------------------+-------------------------------------+----------------+--------------+
+| `analyticsapi`                     | http://localhost:19001              | Python/Django  | Extra        |
++------------------------------------+-------------------------------------+----------------+--------------+
 | `frontend-app-ora-grading`         | http://localhost:1993               | MFE (React.js) | Extra        |
 +------------------------------------+-------------------------------------+----------------+--------------+
 
@@ -351,6 +353,8 @@ Some common service combinations include:
 .. _xqueue: https://github.com/edx/xqueue
 .. _coursegraph: https://github.com/edx/edx-platform/tree/master/openedx/core/djangoapps/coursegraph
 .. _frontend-app-ora-grading: https://github.com/edx/frontend-app-ora-grading
+.. _insights: https://github.com/edx/edx-analytics-dashboard
+.. _analyticsapi: https://github.com/edx/edx-analytics-data-api
 
 
 Known Issues

check.sh

@@ -115,6 +115,12 @@ if should_check insights; then
         "curl --fail -L http://localhost:18110/health/"
 fi
 
+if should_check analyticsapi; then
+    echo "Running Analytics Data API Devstack tests: "
+    run_check analyticsapi_heartbeat analyticsapi \
+        "curl --fail -L http://localhost:19001/health/"
+fi
+
 echo "Successful checks:${succeeded:- NONE}"
 echo "Failed checks:${failed:- NONE}"
 if [[ "$succeeded" ]]; then

docker-compose-host.yml

@@ -53,6 +53,9 @@ services:
     volumes:
       - ${DEVSTACK_WORKSPACE}/edx-analytics-dashboard:/edx/app/insights/insights
       - insights_node_modules:/edx/app/insights/insights/node_modules
+  analyticsapi:
+    volumes:
+      - ${DEVSTACK_WORKSPACE}/edx-analytics-data-api:/edx/app/analytics_api/analytics_api
 
   # Note that frontends mount `src` to /edx/app/src instead of /edx/src.
   # See ADR #5 for rationale.

docker-compose.yml

@@ -356,6 +356,7 @@ services:
     container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.insights"
     hostname: insights.devstack.edx
     depends_on:
+      - analyticsapi
       - mysql57
       - lms
       - memcached
@@ -366,8 +367,8 @@ services:
       DB_HOST: edx.devstack.mysql57
       DB_NAME: dashboard
       DB_PORT: 3306
-      DB_USER: rozencrantz
-      DB_PASSWORD: secret
+      DB_USER: analytics001
+      DB_PASSWORD: password
       LMS_HOST: http://localhost:18000
       DJANGO_SETTINGS_MODULE: analytics_dashboard.settings.devstack
     image: edxops/insights:${OPENEDX_RELEASE:-latest}
@@ -380,6 +381,27 @@ services:
     volumes:
       - /edx/var/insights/
 
+  analyticsapi:
+    image: edxops/analytics_api:${OPENEDX_RELEASE:-latest}
+    container_name: edx.devstack.analyticsapi
+    hostname: analyticsapi
+    depends_on:
+      - mysql57
+      - elasticsearch710
+    command: bash -c 'source /edx/app/analytics_api/analytics_api_env && while true;  do python /edx/app/analytics_api/analytics_api/manage.py runserver 0.0.0.0:19001 --settings analyticsdataserver.settings.devstack; sleep 2; done'
+    stdin_open: true
+    tty: true
+    environment:
+      DB_HOST: edx.devstack.mysql57
+      DB_PORT: 3306
+      DB_USER: analytics001
+      DB_PASSWORD: password
+      ELASTICSEARCH_LEARNERS_HOST: edx.devstack.elasticsearch710
+    ports:
+      - "19001:19001"
+    volumes:
+      - /edx/var/analyticsapi
+
   registrar:
     command: bash -c 'source /edx/app/registrar/registrar_env && while true; do python /edx/app/registrar/registrar/manage.py runserver 0.0.0.0:18734; sleep 2; done'
     container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.registrar"

options.mk

@@ -67,7 +67,7 @@ credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-gradebook+front
 # Separated by plus signs.
 # Separated by plus signs. Listed in alphabetical order for clarity.
 EDX_SERVICES ?= \
-credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-account+frontend-app-course-authoring+frontend-app-gradebook+frontend-app-ora-grading+frontend-app-learning+frontend-app-library-authoring+frontend-app-payment+frontend-app-program-console+frontend-app-publisher+insights+lms+lms_watcher+registrar+registrar-worker+studio+studio_watcher+xqueue+xqueue_consumer
+analyticsapi+credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-account+frontend-app-course-authoring+frontend-app-gradebook+frontend-app-ora-grading+frontend-app-learning+frontend-app-library-authoring+frontend-app-payment+frontend-app-program-console+frontend-app-publisher+insights+lms+lms_watcher+registrar+registrar-worker+studio+studio_watcher+xqueue+xqueue_consumer
 
 # Services with database migrations.
 # Should be a subset of $(EDX_SERVICES).

provision-analyticsapi.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -eu -o pipefail
+
+. scripts/colors.sh
+set -x
+
+name=analyticsapi
+port=19001
+
+docker-compose up -d ${name}
+
+echo -e "${GREEN}Installing requirements for ${name}...${NC}"
+docker-compose exec -T ${name}  bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && make develop' -- ${name}
+
+echo -e "${GREEN}Running migrations for ${name}...${NC}"
+docker-compose exec -T ${name}  bash -e -c 'source /edx/app/analytics_api/analytics_api_env && export DJANGO_SETTINGS_MODULE="analyticsdataserver.settings.devstack" && cd /edx/app/analytics_api/analytics_api && make migrate-all' -- ${name}
+
+echo -e "${GREEN}Creating default user and authentication token for ${name}...${NC}"
+docker-compose exec -T ${name}  bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && python manage.py set_api_key edx edx' -- ${name}
+
+echo -e "${GREEN}Loading test data for ${name}...${NC}"
+docker-compose exec -T ${name}  bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && make loaddata' -- ${name}
+
+echo -e "${GREEN}Populating elasticsearch for ${name}...${NC}"
+docker-compose exec -T ${name}  bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && make create_indices' -- ${name}

provision-insights.sh

@@ -5,7 +5,7 @@ set -eu -o pipefail
 set -x
 
 name=insights
-port=8011
+port=18011
 
 docker-compose up -d insights
 

provision.sh

@@ -49,6 +49,7 @@ registrar \
 xqueue \
 coursegraph \
 insights \
+analyticsapi \
  "
 
 # What should we provision?

provision.sql

@@ -22,6 +22,13 @@ GRANT ALL ON edxapp.* TO 'edxapp001'@'%' IDENTIFIED BY 'password';
 GRANT ALL ON edxapp_csmh.* TO 'edxapp001'@'%';
 
 CREATE DATABASE IF NOT EXISTS `dashboard`;
-GRANT ALL PRIVILEGES ON `dashboard`.* TO 'rozencrantz'@'%' IDENTIFIED BY 'secret';
+GRANT ALL ON `dashboard`.* TO 'analytics001'@'%' IDENTIFIED BY 'password';
+
+CREATE DATABASE IF NOT EXISTS `analytics-api`;
+GRANT ALL ON `analytics-api`.* TO 'analytics001'@'%' IDENTIFIED BY 'password';
+
+CREATE DATABASE IF NOT EXISTS `reports`;
+GRANT ALL ON `reports`.* TO 'analytics001'@'%' IDENTIFIED BY 'password';
+
 
 FLUSH PRIVILEGES;

repo.sh

@@ -33,6 +33,8 @@ repos=(
     "https://github.com/edx/frontend-app-gradebook.git"
     "https://github.com/edx/frontend-app-payment.git"
     "https://github.com/edx/frontend-app-publisher.git"
+    "https://github.com/edx/edx-analytics-dashboard.git"
+    "https://github.com/edx/edx-analytics-data-api.git"
 )
 
 non_release_repos=(
@@ -58,6 +60,8 @@ ssh_repos=(
     "[email protected]:edx/frontend-app-gradebook.git"
     "[email protected]:edx/frontend-app-payment.git"
     "[email protected]:edx/frontend-app-publisher.git"
+    "[email protected]:edx/edx-analytics-dashboard.git"
+    "[email protected]:edx/edx-analytics-data-api.git"
 )
 
 non_release_ssh_repos=(