Skip to content

Commit

Permalink
ref: Performance improvements when serializing events (getsentry#368)
Browse files Browse the repository at this point in the history
A customer was experiencing performance issues when serializing frames with large local vars. Instead of recursing through very large data that we're going to go trim later anyway, do everything in one go.
  • Loading branch information
untitaker authored May 17, 2019
1 parent fa5971d commit b181dfa
Show file tree
Hide file tree
Showing 18 changed files with 430 additions and 372 deletions.
2 changes: 1 addition & 1 deletion .flake8
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[flake8]
ignore = E203, E266, E501, W503, E402, E731
ignore = E203, E266, E501, W503, E402, E731, C901
max-line-length = 80
max-complexity = 18
select = B,C,E,F,W,T4,B9
Expand Down
1 change: 1 addition & 0 deletions pytest.ini
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[pytest]
DJANGO_SETTINGS_MODULE = tests.integrations.django.myapp.settings
addopts = --boxed --tb=short
markers = tests_internal_exceptions
12 changes: 4 additions & 8 deletions sentry_sdk/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,15 @@
import random
from datetime import datetime

from sentry_sdk._compat import string_types, text_type
from sentry_sdk._compat import string_types, text_type, iteritems
from sentry_sdk.utils import (
strip_event_mut,
flatten_metadata,
convert_types,
handle_in_app,
get_type_name,
capture_internal_exceptions,
current_stacktrace,
logger,
)
from sentry_sdk.serializer import Serializer
from sentry_sdk.transport import make_transport
from sentry_sdk.consts import DEFAULT_OPTIONS, SDK_INFO
from sentry_sdk.integrations import setup_integrations
Expand Down Expand Up @@ -43,7 +41,7 @@ def get_options(*args, **kwargs):
if dsn is not None and options.get("dsn") is None:
options["dsn"] = dsn # type: ignore

for key, value in options.items():
for key, value in iteritems(options):
if key not in rv:
raise TypeError("Unknown option %r" % (key,))
rv[key] = value # type: ignore
Expand Down Expand Up @@ -146,9 +144,7 @@ def _prepare_event(
# Postprocess the event here so that annotated types do
# generally not surface in before_send
if event is not None:
event = convert_types(event)
strip_event_mut(event)
event = flatten_metadata(event)
event = Serializer().serialize_event(event)

before_send = self.options["before_send"]
if before_send is not None:
Expand Down
8 changes: 4 additions & 4 deletions sentry_sdk/integrations/_wsgi_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from sentry_sdk.hub import Hub, _should_send_default_pii
from sentry_sdk.utils import AnnotatedValue
from sentry_sdk._compat import text_type
from sentry_sdk._compat import text_type, iteritems

if False:
from typing import Any
Expand Down Expand Up @@ -75,8 +75,8 @@ def parsed_body(self):
form = self.form()
files = self.files()
if form or files:
data = dict(form.items())
for k, v in files.items():
data = dict(iteritems(form))
for k, v in iteritems(files):
size = self.size_of_file(v)
data[k] = AnnotatedValue(
"", {"len": size, "rem": [["!raw", "x", 0, size]]}
Expand Down Expand Up @@ -130,6 +130,6 @@ def _filter_headers(headers):

return {
k: v
for k, v in headers.items()
for k, v in iteritems(headers)
if k.lower().replace("_", "-") not in ("set-cookie", "cookie", "authorization")
}
2 changes: 1 addition & 1 deletion sentry_sdk/integrations/django/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@
from sentry_sdk import Hub
from sentry_sdk.hub import _should_send_default_pii
from sentry_sdk.scope import add_global_event_processor
from sentry_sdk.serializer import add_global_repr_processor
from sentry_sdk.utils import (
add_global_repr_processor,
capture_internal_exceptions,
event_from_exception,
safe_repr,
Expand Down
3 changes: 2 additions & 1 deletion sentry_sdk/integrations/logging.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
capture_internal_exceptions,
)
from sentry_sdk.integrations import Integration
from sentry_sdk._compat import iteritems

if False:
from logging import LogRecord
Expand Down Expand Up @@ -135,7 +136,7 @@ def _extra_from_record(record):
# type: (LogRecord) -> Dict[str, None]
return {
k: v
for k, v in vars(record).items()
for k, v in iteritems(vars(record))
if k not in COMMON_RECORD_ATTRS and not k.startswith("_")
}

Expand Down
6 changes: 3 additions & 3 deletions sentry_sdk/integrations/pyramid.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

from sentry_sdk.hub import Hub, _should_send_default_pii
from sentry_sdk.utils import capture_internal_exceptions, event_from_exception
from sentry_sdk._compat import reraise
from sentry_sdk._compat import reraise, iteritems

from sentry_sdk.integrations import Integration
from sentry_sdk.integrations._wsgi_common import RequestExtractor
Expand Down Expand Up @@ -135,15 +135,15 @@ def form(self):
# type: () -> Dict[str, str]
return {
key: value
for key, value in self.request.POST.items()
for key, value in iteritems(self.request.POST)
if not getattr(value, "filename", None)
}

def files(self):
# type: () -> Dict[str, cgi_FieldStorage]
return {
key: value
for key, value in self.request.POST.items()
for key, value in iteritems(self.request.POST)
if getattr(value, "filename", None)
}

Expand Down
7 changes: 4 additions & 3 deletions sentry_sdk/integrations/tornado.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
_is_json_content_type,
)
from sentry_sdk.integrations.logging import ignore_logger
from sentry_sdk._compat import iteritems

from tornado.web import RequestHandler, HTTPError # type: ignore
from tornado.gen import coroutine # type: ignore
Expand Down Expand Up @@ -166,7 +167,7 @@ def content_length(self):

def cookies(self):
# type: () -> Dict
return {k: v.value for k, v in self.request.cookies.items()}
return {k: v.value for k, v in iteritems(self.request.cookies)}

def raw_data(self):
# type: () -> bytes
Expand All @@ -176,7 +177,7 @@ def form(self):
# type: () -> Optional[Any]
return {
k: [v.decode("latin1", "replace") for v in vs]
for k, vs in self.request.body_arguments.items()
for k, vs in iteritems(self.request.body_arguments)
}

def is_json(self):
Expand All @@ -185,7 +186,7 @@ def is_json(self):

def files(self):
# type: () -> Dict
return {k: v[0] for k, v in self.request.files.items() if v}
return {k: v[0] for k, v in iteritems(self.request.files) if v}

def size_of_file(self, file):
return len(file.body or ())
4 changes: 2 additions & 2 deletions sentry_sdk/integrations/wsgi.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from sentry_sdk.hub import Hub, _should_send_default_pii
from sentry_sdk.utils import capture_internal_exceptions, event_from_exception
from sentry_sdk._compat import PY2, reraise
from sentry_sdk._compat import PY2, reraise, iteritems
from sentry_sdk.tracing import SpanContext
from sentry_sdk.integrations._wsgi_common import _filter_headers

Expand Down Expand Up @@ -118,7 +118,7 @@ def _get_headers(environ):
Returns only proper HTTP headers.
"""
for key, value in environ.items():
for key, value in iteritems(environ):
key = str(key)
if key.startswith("HTTP_") and key not in (
"HTTP_CONTENT_TYPE",
Expand Down
4 changes: 2 additions & 2 deletions sentry_sdk/scope.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from functools import wraps
from itertools import chain

from sentry_sdk.utils import logger, capture_internal_exceptions, object_to_json
from sentry_sdk.utils import logger, capture_internal_exceptions

if False:
from typing import Any
Expand Down Expand Up @@ -192,7 +192,7 @@ def _drop(event, cause, ty):
event["fingerprint"] = self._fingerprint

if self._extras:
event.setdefault("extra", {}).update(object_to_json(self._extras))
event.setdefault("extra", {}).update(self._extras)

if self._tags:
event.setdefault("tags", {}).update(self._tags)
Expand Down
Loading

0 comments on commit b181dfa

Please sign in to comment.