Skip to content

Commit

Permalink
Merge pull request harvard-lil#3644 from rebeccacremona/less-reporting
Browse files Browse the repository at this point in the history 
Disable some metrics, to hopefully improve performance.
  • Loading branch information
@rebeccacremona
rebeccacremona authored Oct 31, 2024
2 parents 30ff2db + 1d33584 commit 27a348a
Show file tree
Hide file tree
Showing 2 changed files with 32 additions and 56 deletions.
24 changes: 0 additions & 24 deletions perma_web/perma/templates/admin-stats.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -361,11 +361,7 @@ <h4>Last hour</h4>
<dt>Completed:</dt><dd>{{ this.last_hour.completed }} ({{ this.last_hour.completed_percent}}%)</dd>
<dt>Failed:</dt><dd>{{ this.last_hour.failed }} ({{ this.last_hour.failed_percent}}%)</dd>
<dt>Killed by Celery:</dt><dd>{{ this.last_hour.celery_timeout }} ({{ this.last_hour.celery_timeout_percent}}%)</dd>
<dt>Proxy error:</dt><dd>{{ this.last_hour.proxy_error }} ({{ this.last_hour.proxy_error_percent}}%)</dd>
<dt>Blocklist error:</dt><dd>{{ this.last_hour.blocklist_error }} ({{ this.last_hour.blocklist_error_percent}}%)</dd>
<dt>Playwright error:</dt><dd>{{ this.last_hour.playwright_error }} ({{ this.last_hour.playwright_error_percent}}%)</dd>
<dt>Scoop timeout:</dt><dd>{{ this.last_hour.timeout }} ({{ this.last_hour.timeout_percent}}%)</dd>
<dt>URL didn't load:</dt><dd>{{ this.last_hour.didnt_load }} ({{ this.last_hour.didnt_load_percent}}%)</dd>
</dl>
</div>

Expand All @@ -375,11 +371,7 @@ <h4>Last 3 hours</h4>
<dt>Completed:</dt><dd>{{ this.last_3_hrs.completed }} ({{ this.last_3_hrs.completed_percent}}%)</dd>
<dt>Failed:</dt><dd>{{ this.last_3_hrs.failed }} ({{ this.last_3_hrs.failed_percent}}%)</dd>
<dt>Killed by Celery:</dt><dd>{{ this.last_3_hrs.celery_timeout }} ({{ this.last_3_hrs.celery_timeout_percent}}%)</dd>
<dt>Proxy error:</dt><dd>{{ this.last_3_hrs.proxy_error }} ({{ this.last_3_hrs.proxy_error_percent}}%)</dd>
<dt>Blocklist error:</dt><dd>{{ this.last_3_hrs.blocklist_error }} ({{ this.last_3_hrs.blocklist_error_percent}}%)</dd>
<dt>Playwright error:</dt><dd>{{ this.last_3_hrs.playwright_error }} ({{ this.last_3_hrs.playwright_error_percent}}%)</dd>
<dt>Scoop timeout:</dt><dd>{{ this.last_3_hrs.timeout }} ({{ this.last_3_hrs.timeout_percent}}%)</dd>
<dt>URL didn't load:</dt><dd>{{ this.last_3_hrs.didnt_load }} ({{ this.last_3_hrs.didnt_load_percent}}%)</dd>
</dl>
</div>

Expand All @@ -389,11 +381,7 @@ <h4>Last 24 hrs</h4>
<dt>Completed:</dt><dd>{{ this.last_24_hrs.completed }} ({{ this.last_24_hrs.completed_percent}}%)</dd>
<dt>Failed:</dt><dd>{{ this.last_24_hrs.failed }} ({{ this.last_24_hrs.failed_percent}}%)</dd>
<dt>Killed by Celery:</dt><dd>{{ this.last_24_hrs.celery_timeout }} ({{ this.last_24_hrs.celery_timeout_percent}}%)</dd>
<dt>Proxy error:</dt><dd>{{ this.last_24_hrs.proxy_error }} ({{ this.last_24_hrs.proxy_error_percent}}%)</dd>
<dt>Blocklist error:</dt><dd>{{ this.last_24_hrs.blocklist_error }} ({{ this.last_24_hrs.blocklist_error_percent}}%)</dd>
<dt>Playwright error:</dt><dd>{{ this.last_24_hrs.playwright_error }} ({{ this.last_24_hrs.playwright_error_percent}}%)</dd>
<dt>Scoop timeout:</dt><dd>{{ this.last_24_hrs.timeout }} ({{ this.last_24_hrs.timeout_percent}}%)</dd>
<dt>URL didn't load:</dt><dd>{{ this.last_24_hrs.didnt_load }} ({{ this.last_24_hrs.didnt_load_percent}}%)</dd>
</dl>
</div>
</div>
Expand All @@ -406,11 +394,7 @@ <h4>Last hour, previous day</h4>
<dt>Completed:</dt><dd>{{ this.last_hour_on_previous_day.completed }} ({{ this.last_hour_on_previous_day.completed_percent}}%)</dd>
<dt>Failed:</dt><dd>{{ this.last_hour_on_previous_day.failed }} ({{ this.last_hour_on_previous_day.failed_percent}}%)</dd>
<dt>Killed by Celery:</dt><dd>{{ this.last_hour_on_previous_day.celery_timeout }} ({{ this.last_hour_on_previous_day.celery_timeout_percent}}%)</dd>
<dt>Proxy error:</dt><dd>{{ this.last_hour_on_previous_day.proxy_error }} ({{ this.last_hour_on_previous_day.proxy_error_percent}}%)</dd>
<dt>Blocklist error:</dt><dd>{{ this.last_hour_on_previous_day.blocklist_error }} ({{ this.last_hour_on_previous_day.blocklist_error_percent}}%)</dd>
<dt>Playwright error:</dt><dd>{{ this.last_hour_on_previous_day.playwright_error }} ({{ this.last_hour_on_previous_day.playwright_error_percent}}%)</dd>
<dt>Scoop timeout:</dt><dd>{{ this.last_hour_on_previous_day.timeout }} ({{ this.last_hour_on_previous_day.timeout_percent}}%)</dd>
<dt>URL didn't load:</dt><dd>{{ this.last_hour_on_previous_day.didnt_load }} ({{ this.last_hour_on_previous_day.didnt_load_percent}}%)</dd>
</dl>
</div>

Expand All @@ -420,11 +404,7 @@ <h4>Last 3 hours, previous day</h4>
<dt>Completed:</dt><dd>{{ this.last_3_hrs_on_previous_day.completed }} ({{ this.last_3_hrs_on_previous_day.completed_percent}}%)</dd>
<dt>Failed:</dt><dd>{{ this.last_3_hrs_on_previous_day.failed }} ({{ this.last_3_hrs_on_previous_day.failed_percent}}%)</dd>
<dt>Killed by Celery:</dt><dd>{{ this.last_3_hrs_on_previous_day.celery_timeout }} ({{ this.last_3_hrs_on_previous_day.celery_timeout_percent}}%)</dd>
<dt>Proxy error:</dt><dd>{{ this.last_3_hrs_on_previous_day.proxy_error }} ({{ this.last_3_hrs_on_previous_day.proxy_error_percent}}%)</dd>
<dt>Blocklist error:</dt><dd>{{ this.last_3_hrs_on_previous_day.blocklist_error }} ({{ this.last_3_hrs_on_previous_day.blocklist_error_percent}}%)</dd>
<dt>Playwright error:</dt><dd>{{ this.last_3_hrs_on_previous_day.playwright_error }} ({{ this.last_3_hrs_on_previous_day.playwright_error_percent}}%)</dd>
<dt>Scoop timeout:</dt><dd>{{ this.last_3_hrs_on_previous_day.timeout }} ({{ this.last_3_hrs_on_previous_day.timeout_percent}}%)</dd>
<dt>URL didn't load:</dt><dd>{{ this.last_3_hrs_on_previous_day.didnt_load }} ({{ this.last_3_hrs_on_previous_day.didnt_load_percent}}%)</dd>
</dl>
</div>

Expand All @@ -435,11 +415,7 @@ <h4>Previous 24 hrs</h4>
<dt>Completed:</dt><dd>{{ this.previous_24_hrs.completed }} ({{ this.previous_24_hrs.completed_percent}}%)</dd>
<dt>Failed:</dt><dd>{{ this.previous_24_hrs.failed }} ({{ this.previous_24_hrs.failed_percent}}%)</dd>
<dt>Killed by Celery:</dt><dd>{{ this.previous_24_hrs.celery_timeout }} ({{ this.previous_24_hrs.celery_timeout_percent}}%)</dd>
<dt>Proxy error:</dt><dd>{{ this.previous_24_hrs.proxy_error }} ({{ this.previous_24_hrs.proxy_error_percent}}%)</dd>
<dt>Blocklist error:</dt><dd>{{ this.previous_24_hrs.blocklist_error }} ({{ this.previous_24_hrs.blocklist_error_percent}}%)</dd>
<dt>Playwright error:</dt><dd>{{ this.previous_24_hrs.playwright_error }} ({{ this.previous_24_hrs.playwright_error_percent}}%)</dd>
<dt>Scoop timeout:</dt><dd>{{ this.previous_24_hrs.timeout }} ({{ this.previous_24_hrs.timeout_percent}}%)</dd>
<dt>URL didn't load:</dt><dd>{{ this.previous_24_hrs.didnt_load }} ({{ this.previous_24_hrs.didnt_load_percent}}%)</dd>
</dl>
</div>
</script>
Expand Down
64 changes: 32 additions & 32 deletions perma_web/perma/views/admin_stats.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,30 +240,30 @@ def stats(request, stat_type=None):
creation_timestamp__range=range_tuple
).count()

proxy_error = Link.objects.all_with_deleted().filter(
tags__name__in=['scoop-proxy-failure'],
creation_timestamp__range=range_tuple
).count()
# proxy_error = Link.objects.all_with_deleted().filter(
# tags__name__in=['scoop-proxy-failure'],
# creation_timestamp__range=range_tuple
# ).count()

blocklist_error = Link.objects.all_with_deleted().filter(
tags__name__in=['scoop-blocklist-failure'],
creation_timestamp__range=range_tuple
).count()
# blocklist_error = Link.objects.all_with_deleted().filter(
# tags__name__in=['scoop-blocklist-failure'],
# creation_timestamp__range=range_tuple
# ).count()

playwright_error = Link.objects.all_with_deleted().filter(
tags__name__in=['scoop-playwright-failure'],
creation_timestamp__range=range_tuple
).count()
# playwright_error = Link.objects.all_with_deleted().filter(
# tags__name__in=['scoop-playwright-failure'],
# creation_timestamp__range=range_tuple
# ).count()

timeout = Link.objects.all_with_deleted().filter(
tags__name__in=['scoop-silent-failure'],
creation_timestamp__range=range_tuple
).count()

didnt_load = Link.objects.all_with_deleted().filter(
tags__name__in=['scoop-load-failure'],
creation_timestamp__range=range_tuple
).count()
# didnt_load = Link.objects.all_with_deleted().filter(
# tags__name__in=['scoop-load-failure'],
# creation_timestamp__range=range_tuple
# ).count()

out[range_name] = {
"completed": completed,
Expand All @@ -272,16 +272,16 @@ def stats(request, stat_type=None):
"failed_percent": round(failed/denominator * 100, 1),
"celery_timeout": celery_timeout,
"celery_timeout_percent": round(celery_timeout/denominator * 100, 1),
"proxy_error": proxy_error,
"proxy_error_percent": round(proxy_error/denominator * 100, 1),
"blocklist_error": blocklist_error,
"blocklist_error_percent": round(blocklist_error/denominator * 100, 1),
"playwright_error": playwright_error,
"playwright_error_percent": round(playwright_error/denominator * 100, 1),
# "proxy_error": proxy_error,
# "proxy_error_percent": round(proxy_error/denominator * 100, 1),
# "blocklist_error": blocklist_error,
# "blocklist_error_percent": round(blocklist_error/denominator * 100, 1),
# "playwright_error": playwright_error,
# "playwright_error_percent": round(playwright_error/denominator * 100, 1),
"timeout": timeout,
"timeout_percent": round(timeout/denominator * 100, 1),
"didnt_load": didnt_load,
"didnt_load_percent": round(didnt_load/denominator * 100, 1),
# "didnt_load": didnt_load,
# "didnt_load_percent": round(didnt_load/denominator * 100, 1),
}

else:
Expand All @@ -293,16 +293,16 @@ def stats(request, stat_type=None):
"failed_percent": 0,
"celery_timeout": 0,
"celery_timeout_percent": 0,
"proxy_error": 0,
"proxy_error_percent": 0,
"blocklist_error": 0,
"blocklist_error_percent": 0,
"playwright_error": 0,
"playwright_error_percent": 0,
# "proxy_error": 0,
# "proxy_error_percent": 0,
# "blocklist_error": 0,
# "blocklist_error_percent": 0,
# "playwright_error": 0,
# "playwright_error_percent": 0,
"timeout": 0,
"timeout_percent": 0,
"didnt_load": 0,
"didnt_load_percent": 0,
# "didnt_load": 0,
# "didnt_load_percent": 0,
}

if out:
Expand Down

0 comments on commit 27a348a

Please sign in to comment.