Extend admin statistics page to contain log search hedyorg#665 (hedyo…

…rg#1630)

app.py

@@ -36,6 +36,7 @@
 import hedy_content
 import hedyweb
 from website import querylog, aws_helpers, jsonbin, translating, ab_proxying, cdn, database, achievements
+from website.log_fetcher import log_fetcher
 import quiz
 
 # Set the current directory to the root Hedy folder
@@ -705,45 +706,89 @@ def get_program_stats():
         return utils.page_403(ui_message='unauthorized')
 
     data = DATABASE.get_all_program_stats(start_date, end_date)
-    processed_data = _program_runs_stats(data)
-    response = [{'level': k, 'data': v} for k, v in processed_data.items()]
-    response.sort(key=lambda el: el['level'])
-    for r in response:
-        r['level'] = str(r['level'])
+    per_level_data = _to_dict_on_key(data, lambda e: e["level"])
+    per_week_data = _to_dict_on_key(data, lambda e: f'{e["week"]}#{e["level"]}')
+
+    response = {'per_level': _per_level_to_response(per_level_data),
+                'per_week': _per_week_to_response(per_week_data)}
     return jsonify(response)
 
-def _program_runs_stats(data):
+
+def _per_level_to_response(data):
+    res = [{'level': level, 'data': data} for level, data in data.items()]
+    res.sort(key=lambda el: el['level'])
+    return [{'level': f"L{entry['level']}", 'data': entry['data']} for entry in res]
+
+
+def _per_week_to_response(data):
+    res = {}
+    for e in [{'week': k.split('#')[0], 'level': int(k.split('#')[1]), 'data': v} for k, v in data.items()]:
+        week = e['week']
+        level_name = 'level' + str(e['level'])
+        if week not in res.keys():
+            res[week] = {'successful_runs': {}, 'failed_runs': {}}
+        res[week]['successful_runs'][level_name] = e['data']['successful_runs']
+        res[week]['failed_runs'][level_name] = e['data']['failed_runs']
+        _add_exception_data(res[week], e['data'])
+    result = [{'week': k, 'data': v} for k, v in res.items()]
+    result.sort(key=lambda el: el['week'])
+    return result
+
+
+def _to_dict_on_key(data, key_selector):
     result = {}
-    for rec in data:
-        level = rec['level']
-        result[level] = _process_program_runs_rec(result.get(level), rec)
-    _add_error_rate(result)
+    for record in data:
+        key = key_selector(record)
+        result[key] = _add_program_run_data(result.get(key), record)
     return result
 
-def _process_program_runs_rec(data, rec):
+
+def _add_program_run_data(data, rec):
     if not data:
         data = {'failed_runs': 0, 'successful_runs': 0}
+    data['successful_runs'] += rec.get('successful_runs') or 0
+    _add_exception_data(data, rec, True)
+    return data
 
-    successes = rec.get('successful_runs')
-    if successes:
-        data['successful_runs'] += successes
 
-    for k, v in _filter_exceptions(rec).items():
-        if not data.get(k):
-            data[k] = 0
-        data[k] += v
-        data['failed_runs'] += v
+def _add_exception_data(entry, data, include_failed_runs=False):
+    exceptions = {k: v for k, v in data.items() if k.lower().endswith('exception')}
+    for k, v in exceptions.items():
+        if not entry.get(k):
+            entry[k] = 0
+        entry[k] += v
+        if include_failed_runs:
+            entry['failed_runs'] += v
 
-    return data
 
-def _filter_exceptions(s):
-    return {k: v for k, v in s.items() if k.lower().endswith('exception')}
+@app.route('/logs/query', methods=['POST'])
+def query_logs():
+    user = current_user()
+    if not is_admin(user):
+        return utils.page_403(ui_message='unauthorized')
+
+    body = request.json
+    if body is not None and not isinstance(body, dict):
+        return 'body must be an object', 400
+
+    (exec_id, status) = log_fetcher.query(body)
+    response = {'query_status': status, 'query_execution_id': exec_id}
+    return jsonify(response)
+
+
+@app.route('/logs/results', methods=['GET'])
+def get_log_results():
+    query_execution_id = request.args.get('query_execution_id', default=None, type=str)
+    next_token = request.args.get('next_token', default=None, type=str)
+
+    user = current_user()
+    if not is_admin(user):
+        return utils.page_403(ui_message='unauthorized')
+
+    data, next_token = log_fetcher.get_query_results(query_execution_id, next_token)
+    response = {'data': data, 'next_token': next_token}
+    return jsonify(response)
 
-def _add_error_rate(data):
-    for k, v in data.items():
-        fails = v['failed_runs']
-        successes = v['successful_runs']
-        v['error_rate'] = fails / (successes + fails)
 
 def get_user_formatted_age(now, date):
     texts = TRANSLATIONS.get_translations(g.lang, 'Programs')

build-tools/heroku/tailwind/styles.css

@@ -311,4 +311,46 @@ p.close-dialog {
   text-decoration: none;
   cursor: default;
   font-weight: normal;
-}
+}
+
+/* Tab content - closed */
+.accordion .tab-content {
+  max-height: 0;
+  -webkit-transition: max-height .35s;
+  -o-transition: max-height .35s;
+  transition: max-height .35s;
+}
+
+/* Icon */
+.accordion .accordion-tab label::before {
+  float: right;
+  right: 0;
+  top: 0;
+  display: block;
+  width: 1.5em;
+  height: 1.5em;
+  line-height: 1.5;
+  font-size: 1.25rem;
+  font-weight: bolder;
+  text-align: center;
+  -webkit-transition: all .35s;
+  -o-transition: all .35s;
+  transition: all .35s;
+}
+
+/* :checked - resize to full height */
+.accordion .accordion-tab input:checked ~ .tab-content {
+  max-height: 3000vh;
+}
+
+/* Icon formatting - closed */
+.accordion .accordion-tab input[type=checkbox] + label::before {
+  content: ">";
+  transform: rotate(90deg);
+  margin-right: 5px;
+}
+
+/* Icon formatting - open */
+.accordion .accordion-tab input[type=checkbox]:checked + label::before {
+  transform: rotate(270deg);
+}

config.py

@@ -34,6 +34,14 @@
         'postfix': ('-' + dyno if dyno else '') + '-' + str(os.getpid()),
         'region': 'eu-west-1'
     },
+    'athena': {
+        'region': 'eu-west-1',
+        'database': 'hedy-logs',
+        'table': 'hedy-alpha',
+        'prepare_statement': 'query',
+        's3_output': 's3://hedy-query-outputs/',
+        'max_results': 50
+    },
     #enables the quiz environment by setting the config variable on True
     'quiz-enabled': True,
 }

requirements.txt

@@ -16,5 +16,6 @@ awscli>=1.19.88
 PySumTypes==0.0.1
 beautifulsoup4==4.9.3
 regex==2021.8.28
+retrying==1.3.3
 pytest==6.2.5
 parameterized==0.8.1