Merge branch 'main' into test

.gitignore

@@ -146,3 +146,6 @@ logs.txt
 *.swo
 .local
 node_modules
+
+# Ignore things that start with underscores
+_*

app.py

@@ -187,8 +187,10 @@ def before_request_https():
 
 Compress(app)
 Commonmark(app)
-logger = jsonbin.JsonBinLogger.from_env_vars()
-querylog.LOG_QUEUE.set_transmitter(aws_helpers.s3_transmitter_from_env())
+parse_logger = jsonbin.MultiParseLogger(
+    jsonbin.JsonBinLogger.from_env_vars(),
+    jsonbin.S3ParseLogger.from_env_vars())
+querylog.LOG_QUEUE.set_transmitter(aws_helpers.s3_querylog_transmitter_from_env())
 
 # Check that requested language is supported, otherwise return 404
 @app.before_request
@@ -309,7 +311,7 @@ def parse():
             session ['code'] = code
 
     querylog.log_value(server_error=response.get('Error'))
-    logger.log ({
+    parse_logger.log ({
         'session': session_id(),
         'date': str(datetime.datetime.now()),
         'level': level,
@@ -472,7 +474,7 @@ def initialize_gfi_session(language):
 def report_error():
     post_body = request.json
 
-    logger.log ({
+    parse_logger.log ({
         'session': session_id(),
         'date': str(datetime.datetime.now()),
         'level': post_body.get('level'),

config.py

@@ -27,4 +27,11 @@
         'postfix': ('-' + dyno if dyno else '') + '-' + str(os.getpid()),
         'region': 'eu-west-1'
     },
+    's3-parse-logs': {
+        'bucket': 'hedy-parse-logs',
+        'prefix': app_name + '/',
+        # Make logs from different instances/processes unique
+        'postfix': ('-' + dyno if dyno else '') + '-' + str(os.getpid()),
+        'region': 'eu-west-1'
+    },
 }

coursedata/texts/de.yaml

@@ -127,6 +127,9 @@ Auth:
     unsaved_changes: "Du hast ein nicht gespeichertes Programm. Möchtest du es verwerfen, ohne zu speichern?"
     save_success: "Erfolgreich"
     save_success_detail: "Programm erfolgreich gespeichert"
+    share_success_detail: "Program shared successfully"
+    unshare_success_detail: "Program unshared successfully"
+    copy_clipboard: "Successfully copied to clipboard"
     answer_question: "Du kannst das Programm erst ausführen, nachdem du die Frage beantwortet hast."
 Programs:
     recent: "Zuletzt verwendete Programme"
@@ -139,5 +142,10 @@ Programs:
     open: "Öffnen"
     delete: "Löschen"
     delete_confirm: "Bist du sicher, dass du das Programm löschen möchtest?"
+    share: "Share"
+    share_confirm: "Are you sure you want to make the program public?"
+    unshare: "Unshare"
+    unshare_confirm: "Are you sure you want to make the program private?"
+    copy_link_to_share: "Copy link to share"
     no_programs: "Du hast noch kein Programm."
     write_first: "Schreibe dein erstes Programm!"

coursedata/texts/el.yaml

@@ -125,6 +125,9 @@ Auth:
     unsaved_changes: 'Έχεις ένα μη αποθηκευμένο πρόγραμμα. Θέλεις να αποχωρήσεις χωρίς να το αποθηκεύσεις;'
     save_success: 'Επιτυχία'
     save_success_detail: 'Το πρόγραμμα αποθηκεύτηκε επιτυχώς'
+    share_success_detail: "Program shared successfully"
+    unshare_success_detail: "Program unshared successfully"
+    copy_clipboard: "Successfully copied to clipboard"
     answer_question: 'Δεν μπορείς να εκτελέσεις το πρόγραμμα πριν να απαντήσεις την ερώτηση'
     login_long: Συνδέσου στο λογαριασμό σου
 Programs:
@@ -138,5 +141,10 @@ Programs:
     open: 'Άνοιγμα'
     delete: 'Διαγραφή'
     delete_confirm: 'Είσαι σίγουρος/η ότι θέλεις να διαγράψεις το πρόγραμμα;'
+    share: "Share"
+    share_confirm: "Are you sure you want to make the program public?"
+    unshare: "Unshare"
+    unshare_confirm: "Are you sure you want to make the program private?"
+    copy_link_to_share: "Copy link to share"
     no_programs: 'Δεν έχεις προγράμματα ακόμα.'
     write_first: 'Γράψε το πρώτο σου πρόγραμμα!'

coursedata/texts/en.yaml

@@ -172,6 +172,8 @@ Auth:
     save_success: "Success"
     save_success_detail: "Program saved successfully"
     share_success_detail: "Program shared successfully"
+    unshare_success_detail: "Program unshared successfully"
+    copy_clipboard: "Successfully copied to clipboard"
     answer_question: "You can't run the program until you answer the question first"
 Programs:
     recent: "My recent programs"
@@ -188,5 +190,6 @@ Programs:
     share_confirm: "Are you sure you want to make the program public?"
     unshare: "Unshare"
     unshare_confirm: "Are you sure you want to make the program private?"
+    copy_link_to_share: "Copy link to share"
     no_programs: "You have no programs yet."
     write_first: "Write your first program!"

coursedata/texts/es.yaml

@@ -166,6 +166,9 @@ Auth:
     unsaved_changes: "Tu programa no se ha salvado. ¿Deseas irte sin salvarlo?"
     save_success: "Éxito"
     save_success_detail: "Tu programa se ha salvado exitosamente"
+    share_success_detail: "Tu programa es ahora público"
+    unshare_success_detail: "Tu programa es ahora privado"
+    copy_clipboard: "Copiado al portapapeles"
     answer_question: "No puedes ejecutar el programa hasta que hayas contestado la pregunta"
 Programs:
     recent: "Mis programas recientes"
@@ -178,5 +181,10 @@ Programs:
     open: "Abrir"
     delete: "Borrar"
     delete_confirm: "Estás seguro de querer borrar este programa?"
+    share: "Compartir públicamente"
+    share_confirm: "Estás seguro/a de querer volver público tu programa?"
+    unshare: "Dejar de compartir"
+    unshare_confirm: "Estás seguro/a de querer volver privado tu programa?"
+    copy_link_to_share: "Copiar vínculo para compartir"
     no_programs: "Todavía no tienes programas guardados."
     write_first: "¡Escribe tu primer programa!"

coursedata/texts/fr.yaml

@@ -166,6 +166,9 @@ Auth:
     unsaved_changes: "Le programme en cours n'a pas été sauvegardé. Souhaites-tu vraiment quitter sans le sauvegarder ?"
     save_success: "Succès"
     save_success_detail: "Le programme a été sauvegardé avec succès"
+    share_success_detail: "Program shared successfully"
+    unshare_success_detail: "Program unshared successfully"
+    copy_clipboard: "Successfully copied to clipboard"
     answer_question: "Tu ne peux pas exécuter le programme avant d'avoir répondu à la question."
 Programs:
     recent: "Mes programmes récents"
@@ -178,5 +181,10 @@ Programs:
     open: "Ouvrir"
     delete: "Supprimer"
     delete_confirm: "Es-tu sûr de vouloir supprimer ce programme ?"
+    share: "Share"
+    share_confirm: "Are you sure you want to make the program public?"
+    unshare: "Unshare"
+    unshare_confirm: "Are you sure you want to make the program private?"
+    copy_link_to_share: "Copy link to share"
     no_programs: "Tu n'as pas encore de programme."
     write_first: "Écris ton premier programme !"

coursedata/texts/hu.yaml

@@ -122,6 +122,9 @@ Auth:
     unsaved_changes: "Nem mentett programod van. El akarsz menni mentés nélkül?"
     save_success: "Siker"
     save_success_detail: "A programot sikeresen mentetted"
+    share_success_detail: "Program shared successfully"
+    unshare_success_detail: "Program unshared successfully"
+    copy_clipboard: "Successfully copied to clipboard"
     answer_question: "Addig nem futtathatod a programot, amíg nem válaszolsz a kérdésre "
 Programs:
     recent: "Legutóbbi programjaim "
@@ -134,5 +137,10 @@ Programs:
     open: "Nyitva"
     delete: "Törlés"
     delete_confirm: "Biztosan törlöd a programot?"
+    share: "Share"
+    share_confirm: "Are you sure you want to make the program public?"
+    unshare: "Unshare"
+    unshare_confirm: "Are you sure you want to make the program private?"
+    copy_link_to_share: "Copy link to share"
     no_programs: "Még nincs programod."
     write_first: "Írd meg az első programot!"

coursedata/texts/it.yaml

@@ -126,6 +126,9 @@ Auth:
     unsaved_changes: "Hai un programma non salvato. Vuoi uscire senza salvare?"
     save_success: "Successo"
     save_success_detail: "Programma salvato con successo"
+    share_success_detail: "Program shared successfully"
+    unshare_success_detail: "Program unshared successfully"
+    copy_clipboard: "Successfully copied to clipboard"
     answer_question: "Non puoi eseguire il programma prima di rispondere alla domanda"
 Programs:
     recent: "I miei programmi recenti"
@@ -138,5 +141,10 @@ Programs:
     open: "Apri"
     delete: "Elimina"
     delete_confirm: "Sei sicura\\o di voler eliminare questo programma?"
+    share: "Share"
+    share_confirm: "Are you sure you want to make the program public?"
+    unshare: "Unshare"
+    unshare_confirm: "Are you sure you want to make the program private?"
+    copy_link_to_share: "Copy link to share"
     no_programs: "Non hai ancora nessun programma."
     write_first: "Scrivi il tuo primo programma!"

coursedata/texts/nl.yaml

@@ -171,6 +171,9 @@ Auth:
     unsaved_changes: "Jouw programma is niet opgeslagen. Wil je weggaan zonder het op te slaan?"
     save_success: "Gelukt!"
     save_success_detail: "Je programma is opgeslagen"
+    share_success_detail: "Program shared successfully"
+    unshare_success_detail: "Program unshared successfully"
+    copy_clipboard: "Successfully copied to clipboard"
     answer_question: "Je kunt het programma pas uitvoeren als je de vraag hebt beantwoord."
 Programs:
     recent: "Mijn programma's"
@@ -183,5 +186,10 @@ Programs:
     open: "Openen"
     delete: "Verwijderen"
     delete_confirm: "Weet je zeker dat je het programma wil verwijderen?"
+    share: "Share"
+    share_confirm: "Are you sure you want to make the program public?"
+    unshare: "Unshare"
+    unshare_confirm: "Are you sure you want to make the program private?"
+    copy_link_to_share: "Copy link to share"
     no_programs: "Nog geen programma's."
     write_first: "Schrijf je eerste programma!"

coursedata/texts/pt_br.yaml

@@ -166,6 +166,9 @@ Auth:
     unsaved_changes: "You have an unsaved program. Do you want to leave without saving it?"
     save_success: "Success"
     save_success_detail: "Program saved successfully"
+    share_success_detail: "Program shared successfully"
+    unshare_success_detail: "Program unshared successfully"
+    copy_clipboard: "Successfully copied to clipboard"
     answer_question: "You can't run the program until you answer the question first"
 Programs:
     recent: "My recent programs"
@@ -178,5 +181,10 @@ Programs:
     open: "Open"
     delete: "Delete"
     delete_confirm: "Are you sure you want to delete the program?"
+    share: "Share"
+    share_confirm: "Are you sure you want to make the program public?"
+    unshare: "Unshare"
+    unshare_confirm: "Are you sure you want to make the program private?"
+    copy_link_to_share: "Copy link to share"
     no_programs: "You have no programs yet."
     write_first: "Write your first program!"

coursedata/texts/sw.yaml

@@ -114,6 +114,9 @@ Auth:
   unsaved_changes: "Una programu ambao haujahifadhiwa. Je! Unataka kuondoka bila kuiokoa?"
   save_success: "Fanikio"
   save_success_detail: "Programu imehifadhiwa kwa mafanikio"
+  share_success_detail: "Program shared successfully"
+  unshare_success_detail: "Program unshared successfully"
+  copy_clipboard: "Successfully copied to clipboard"
   answer_question: "Huwezi kuendesha programu hadi ujibu swali kwanza"
 Programs:
   recent: "Programu zangu za hivi karibuni"
@@ -126,5 +129,10 @@ Programs:
   open: "Fungua"
   delete: "Futa"
   delete_confirm: "Je! Una uhakika unataka kufuta programu?"
+  share: "Share"
+  share_confirm: "Are you sure you want to make the program public?"
+  unshare: "Unshare"
+  unshare_confirm: "Are you sure you want to make the program private?"
+  copy_link_to_share: "Copy link to share"
   no_programs: "Bado hauna programu"
-  write_first: "Andika programu yako ya kwanza."
+  write_first: "Andika programu yako ya kwanza."

coursedata/texts/zh.yaml

@@ -162,6 +162,9 @@ Auth:
     unsaved_changes: "你有一个未保存的程序. 你想不保存就离开吗?"
     save_success: "成功"
     save_success_detail: "程序保存成功"
+    share_success_detail: "Program shared successfully"
+    unshare_success_detail: "Program unshared successfully"
+    copy_clipboard: "Successfully copied to clipboard"
     answer_question: "你要先回答问题，才能运行程序."
 Programs:
     recent: "我最近的程序"
@@ -174,5 +177,10 @@ Programs:
     open: "打开"
     delete: "删除"
     delete_confirm: "你确定要删除这个程序吗?"
+    share: "Share"
+    share_confirm: "Are you sure you want to make the program public?"
+    unshare: "Unshare"
+    unshare_confirm: "Are you sure you want to make the program private?"
+    copy_link_to_share: "Copy link to share"
     no_programs: "你还没有程序."
     write_first: "编写你的第一个程序!"

gunicorn.conf.py

@@ -4,5 +4,6 @@
 def worker_exit(server, worker):
     # When the worker is being exited (perhaps because of a timeout),
     # give the query_log handler a chance to flush to disk.
-    from website import querylog
+    from website import querylog, jsonbin
     querylog.emergency_shutdown()
+    jsonbin.emergency_shutdown()

requirements.txt

@@ -8,7 +8,8 @@ PyYAML==5.4
 attrs==19.3.0
 Flask-Commonmark==0.8
 bcrypt==3.2.0
-boto3==1.16.50
 levenshtein==0.12.0
+boto3>=1.16.50
 ruamel.yaml==0.17.4
 pylint==2.8.2
+awscli>=1.19.88

static/js/app.js

@@ -376,16 +376,40 @@ window.share_program = function share_program (id, Public, reload) {
     contentType: 'application/json',
     dataType: 'json'
   }).done(function(response) {
-    $ ('#okbox').show ();
-    $ ('#okbox .caption').html (window.auth.texts.save_success);
-    $ ('#okbox .details').html (window.auth.texts.share_success_detail);
-    if (reload) location.reload ();
+    if ($ ('#okbox') && $ ('#okbox').length) {
+      $ ('#okbox').show ();
+      $ ('#okbox .caption').html (window.auth.texts.save_success);
+      $ ('#okbox .details').html (Public ? window.auth.texts.share_success_detail : window.auth.texts.unshare_success_detail);
+    }
+    else {
+      alert (Public ? window.auth.texts.share_success_detail : window.auth.texts.unshare_success_detail);
+    }
+    if (reload) setTimeout (function () {location.reload ()}, 1000);
   }).fail(function(err) {
     console.error(err);
     error.show(ErrorMessages.Connection_error, JSON.stringify(err));
   });
 }
 
+window.copy_to_clipboard = function copy_to_clipboard (string) {
+  // https://hackernoon.com/copying-text-to-clipboard-with-javascript-df4d4988697f
+  var el = document.createElement ('textarea');
+  el.value = string;
+  el.setAttribute ('readonly', '');
+  el.style.position = 'absolute';
+  el.style.left = '-9999px';
+  document.body.appendChild (el);
+  var selected = document.getSelection ().rangeCount > 0 ? document.getSelection ().getRangeAt (0) : false;
+  el.select ();
+  document.execCommand ('copy');
+  document.body.removeChild (el);
+  if (selected) {
+     document.getSelection ().removeAllRanges ();
+     document.getSelection ().addRange (selected);
+  }
+  alert (window.auth.texts.copy_clipboard);
+}
+
 /**
  * Do a POST with the error to the server so we can log it
  */

templates/programs.html

@@ -20,6 +20,7 @@
       <a href="#" onclick="if (confirm ('{{ texts.delete_confirm }}')) window.open ('/programs/delete/{{program.id}}', '_self')">{{ texts.delete }}</a>
       {% if program.public %}
         <a href="#" onclick="if (confirm ('{{ texts.unshare_confirm }}')) window.share_program ('{{program.id}}', false, true)">{{ texts.unshare}}</a>
+        <a href="#" onclick="window.copy_to_clipboard (window.location.origin + '{{'/hedy/' + program.level|string + '/' + program.id}}')">{{ texts.copy_link_to_share }}</a>
       {% else %}
         <a href="#" onclick="if (confirm ('{{ texts.share_confirm }}')) window.share_program ('{{program.id}}', true, true)">{{ texts.share}}</a>
       {% endif %}

tests/test_querylog.py

@@ -1,4 +1,4 @@
-from website import querylog
+from website import querylog, log_queue
 import unittest
 
 class TestQueryLog(unittest.TestCase):
@@ -25,7 +25,7 @@ def test_emergency_recovery(self):
 
     querylog.emergency_shutdown()
 
-    recovered_queue = querylog.LogQueue(batch_window_s=300)
+    recovered_queue = log_queue.LogQueue('querylog', batch_window_s=300)
     recovered_queue.try_load_emergency_saves()
     recovered_queue.set_transmitter(self._fake_transmitter)
 

tools/view-logs → tools/download-logs

@@ -26,4 +26,4 @@ sed -e '$a\' $cache_dir/* > $finalfile
 
 echo "Now run a command like:"
 echo ""
-echo "   cat $finalfile | recs grep '{{duration_ms}} > 10000' | recs totable -k start_time,duration_ms,method,path,'!_ms\$!' | less -S"
+echo "   cat $finalfile | recs grep '{{duration_ms}} > 1000' | tee _lastquery.jsonl | recs totable -k start_time,duration_ms,method,path,'!_ms\$!' | less -S"