fix: log some more information into query logs

app.py

@@ -255,6 +255,10 @@ def check_language():
 def before_request_begin_logging():
     querylog.begin_global_log_record(path=request.path, method=request.method)
 
+@app.after_request
+def after_request_log_status(response):
+    querylog.log_value(http_code=response.status_code)
+    return response
 
 @app.teardown_request
 def teardown_request_finish_logging(exc):
@@ -280,11 +284,11 @@ def parse():
     # reason.
     lang = body.get('lang', requested_lang())
 
-    querylog.log_value(level=level, lang=lang)
-
     response = {}
     username = current_user(request) ['username'] or None
 
+    querylog.log_value(level=level, lang=lang, session_id=session_id(), username=username)
+
     # Check if user sent code
     if not code:
         response["Error"] = "no code found, please send code."
@@ -319,6 +323,7 @@ def parse():
             traceback.print_exc()
             print(f"error transpiling {code}")
             response["Error"] = str(E)
+    querylog.log_value(server_error=response.get('Error'))
     logger.log ({
         'session': session_id(),
         'date': str(datetime.datetime.now()),

doc/logs.md

@@ -12,7 +12,7 @@ aws_access_key_id = AKIA**********
 aws_secret_access_key = ***********
 ```
 
-Install The Log File Navigator (`lnav`) using one of the methods described [here](http://lnav.org/downloads).
+Install RecordStream (`recs`) using one of the methods described [here](https://metacpan.org/pod/App::RecordStream).
 `lnav` is not the *best* tool, but it's usable for now.
 
 Install the [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv1.html).
@@ -30,8 +30,5 @@ $ tools/view-logs hedy-beta 2021-05-10
 
 NOTE: Time stamps will be in UTC.
 
-In lnav, you can use some limited form of SQL on columns we have defined. Example:
+The `view-logs` tool will give you a RecordStream command line to paste into shell.
 
-```
-;SELECT * FROM request WHERE duration_ms > 1000
-```

querylog.py

@@ -6,7 +6,10 @@
 import os
 import datetime
 import traceback
+import logging
+import utils
 
+logger = logging.getLogger('querylog')
 
 class LogRecord:
     """A log record."""
@@ -167,8 +170,8 @@ def add(self, record):
         bucket = div_clip(time.time(), self.batch_window_s)
         data = record.as_data()
 
-        if self.do_print:
-            print(data)
+        if self.do_print or utils.is_debug_mode():
+            logging.debug(repr(data))
 
         with self.mutex:
             self.records_queue[bucket].append(data)

tools/view-logs

@@ -2,11 +2,6 @@
 set -eu
 scriptdir=$(cd $(dirname $0) && pwd)
 
-if ! type lnav > /dev/null; then
-    echo "Install 'lnav' before running this script! (http://lnav.org)" >&2
-    exit 1
-fi
-
 if ! type aws > /dev/null; then
     echo "Install the AWS CLI before running this script." >&2
     exit 1
@@ -26,11 +21,9 @@ bucket=hedy-query-logs
 aws s3 sync s3://${bucket}/${hedy_env}/${day} $cache_dir
 
 # Need to make sure every file ends in a newline
-sed -i -e '$a\' $cache_dir/*
-
-#finalfile=$cache_root/$day.jsonl
-#rm -f $finalfile
-#for f in $cache_dir/*; do (cat "${f}"; echo) >> $finalfile; done
+finalfile=$cache_root/$day.jsonl
+sed -e '$a\' $cache_dir/* > $finalfile
 
-lnav -i $scriptdir/jsonlines_fmt.json
-lnav -r $cache_dir
+echo "Now run a command like:"
+echo ""
+echo "   cat $finalfile | recs grep '{{duration_ms}} > 10000' | recs totable -k start_time,duration_ms,method,path,'!_ms\$!' | less -S"