requirements: Upgrade Python requirements.

Signed-off-by: Anders Kaseorg <[email protected]>

pyproject.toml

@@ -135,6 +135,7 @@ ignore = [
     "E731", # Do not assign a lambda expression, use a def
     "N802", # Function name should be lowercase
     "N806", # Variable in function should be lowercase
+    "PLC1901", # `s == ""` can be simplified to `not s` as an empty string is falsey
     "PLR0911", # Too many return statements
     "PLR0912", # Too many branches
     "PLR0913", # Too many arguments to function call
@@ -152,6 +153,8 @@ ignore = [
     "S107", # Possible hardcoded password
     "S110", # `try`-`except`-`pass` detected, consider logging the exception
     "S113", # Probable use of requests call without timeout
+    "S310", # Audit URL open for permitted schemes. Allowing use of `file:` or custom schemes is often unexpected.
+    "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes
     "S324", # Probable use of insecure hash functions in `hashlib`
     "SIM103", # Return the condition directly
     "SIM108", # Use ternary operator `action = "[commented]" if action == "created" else f"{action} a [comment]"` instead of if-else-block

requirements/dev.in

@@ -21,7 +21,7 @@ coverage<6.4.3  # undiagnosed bug
 fakeldap
 
 # For testing mock http requests
-https://github.com/getsentry/responses/archive/b30c13fe1c9a2e60e7e2de7a92322b4d4ad57562.zip#egg=responses==0.22.0+git  # https://github.com/getsentry/responses/pull/593
+responses
 
 # For doing highly usable Python profiling
 line-profiler

requirements/pip.txt

@@ -7,17 +7,17 @@
 #
 # For details, see requirements/README.md .
 #
-wheel==0.38.4 \
-    --hash=sha256:965f5259b566725405b05e7cf774052044b1ed30119b5d586b2703aafe8719ac \
-    --hash=sha256:b60533f3f5d530e971d6737ca6d58681ee434818fab630c83a734bb10c083ce8
+wheel==0.40.0 \
+    --hash=sha256:cd1196f3faee2b31968d626e1731c94f99cbdb67cf5a46e4f5656cbee7738873 \
+    --hash=sha256:d236b20e7cb522daf2390fa84c55eea81c5c30190f90f29ae2ca1ad8355bf247
     # via -r requirements/pip.in
 
 # The following packages are considered to be unsafe in a requirements file:
 pip==20.3.4 \
     --hash=sha256:217ae5161a0e08c0fb873858806e3478c9775caffce5168b50ec885e358c199d \
     --hash=sha256:6773934e5f5fc3eaa8c5a44949b5b924fc122daa0a8aa9f80c835b4ca2a543fc
     # via -r requirements/pip.in
-setuptools==67.4.0 \
-    --hash=sha256:e5fd0a713141a4a105412233c63dc4e17ba0090c8e8334594ac790ec97792330 \
-    --hash=sha256:f106dee1b506dee5102cc3f3e9e68137bbad6d47b616be7991714b0c62204251
+setuptools==67.6.1 \
+    --hash=sha256:257de92a9d50a60b8e22abfcbb771571fde0dbf3ec234463212027a4eeecbe9a \
+    --hash=sha256:e728ca814a823bf7bf60162daf9db95b93d532948c4c0bea762ce62f60189078
     # via -r requirements/pip.in

version.py

@@ -48,4 +48,4 @@
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = (228, 0)
+PROVISION_VERSION = (229, 0)

zerver/apps.py

@@ -27,9 +27,7 @@ def ready(self) -> None:
         # needs to be here (rather than e.g. at top-of-file) to avoid
         # running that code too early in Django's setup process, but
         # in any case, this is an intentionally unused import.
-        import zerver.signals
-
-        zerver.signals
+        import zerver.signals  # noqa: F401
 
         if settings.POST_MIGRATION_CACHE_FLUSHING:
             post_migrate.connect(flush_cache, sender=self)

zerver/lib/email_mirror.py

@@ -3,7 +3,7 @@
 import secrets
 from email.headerregistry import Address, AddressHeader
 from email.message import EmailMessage
-from typing import Dict, List, Match, Optional, Tuple
+from typing import Dict, List, Match, Optional, Tuple, cast
 
 from django.conf import settings
 
@@ -231,15 +231,18 @@ def send_mm_reply_to_stream(
 
 
 def get_message_part_by_type(message: EmailMessage, content_type: str) -> Optional[str]:
-    charsets = message.get_charsets()
+    charsets = cast(  # https://github.com/python/typeshed/pull/9944
+        List[Optional[str]], message.get_charsets()
+    )
 
     for idx, part in enumerate(message.walk()):
         if part.get_content_type() == content_type:
             content = part.get_payload(decode=True)
             assert isinstance(content, bytes)
-            if charsets[idx]:
+            charset = charsets[idx]
+            if charset is not None:
                 try:
-                    return content.decode(charsets[idx], errors="ignore")
+                    return content.decode(charset, errors="ignore")
                 except LookupError:
                     # The RFCs do not define how to handle unknown
                     # charsets, but treating as US-ASCII seems

zerver/lib/singleton_bmemcached.py

@@ -7,7 +7,7 @@
 
 @lru_cache(None)
 def _get_bmemcached(location: str, params: bytes) -> BMemcached:
-    return BMemcached(location, pickle.loads(params))
+    return BMemcached(location, pickle.loads(params))  # noqa: S301
 
 
 def SingletonBMemcached(location: str, params: Dict[str, Any]) -> BMemcached:

zerver/lib/templates.py

@@ -172,9 +172,9 @@ def render_markdown_path(
 
     html = md_engine.convert(markdown_string)
     if context is None:
-        return mark_safe(html)
+        return mark_safe(html)  # noqa: S308
 
-    return mark_safe(jinja.from_string(html).render(context))
+    return mark_safe(jinja.from_string(html).render(context))  # noqa: S308
 
 
 def webpack_entry(entrypoint: str) -> List[str]:

zerver/openapi/openapi.py

@@ -11,12 +11,9 @@
 from typing import Any, Dict, List, Mapping, Optional, Set, Tuple, Union
 
 import orjson
-from jsonschema.exceptions import ValidationError as JsonSchemaValidationError
-from openapi_core import Spec
+from openapi_core import Spec, openapi_request_validator, openapi_response_validator
 from openapi_core.testing import MockRequest, MockResponse
-from openapi_core.unmarshalling.schemas.exceptions import InvalidSchemaValue
-from openapi_core.validation.request import openapi_request_validator
-from openapi_core.validation.response import openapi_response_validator
+from openapi_core.validation.exceptions import ValidationError as OpenAPIValidationError
 
 OPENAPI_SPEC_PATH = os.path.abspath(
     os.path.join(os.path.dirname(__file__), "../openapi/zulip.yaml")
@@ -441,48 +438,9 @@ def validate_against_openapi_schema(
     result = openapi_response_validator.validate(openapi_spec.spec(), mock_request, mock_response)
     try:
         result.raise_for_errors()
-    except InvalidSchemaValue as isv:
-        schema_errors = list(isv.schema_errors)
-        message = f"{len(schema_errors)} response validation error(s) at {method} /api/v1{path} ({status_code}):"
-        for error in schema_errors:
-            if display_brief_error and isinstance(error, JsonSchemaValidationError):
-                # display_brief_error is designed to avoid printing 1000 lines
-                # of output when the schema to validate is extremely large
-                # (E.g. the several dozen format variants for individual
-                # events returned by GET /events) and instead just display the
-                # specific variant we expect to match the response.
-                brief_error_validator_value = [
-                    validator_value
-                    for validator_value in error.validator_value
-                    if not prune_schema_by_type(validator_value, error.instance["type"])
-                ]
-                brief_error_display_schema = error.schema.copy()
-                if "oneOf" in brief_error_display_schema:
-                    brief_error_display_schema["oneOf"] = [
-                        i_schema
-                        for i_schema in error.schema["oneOf"]
-                        if not prune_schema_by_type(i_schema, error.instance["type"])
-                    ]
-
-                # Field list from https://python-jsonschema.readthedocs.io/en/stable/errors/
-                error = JsonSchemaValidationError(
-                    message=error.message,
-                    validator=error.validator,
-                    path=error.path,
-                    instance=error.instance,
-                    schema_path=error.schema_path,
-                    schema=brief_error_display_schema,
-                    validator_value=brief_error_validator_value,
-                    cause=error.cause,
-                )
-            # Some endpoints have long, descriptive OpenAPI schemas
-            # which, when printed to the console, do not assist with
-            # debugging, so we omit some of the error information.
-            if path in ["/register"] and isinstance(error, JsonSchemaValidationError):
-                error.schema = "OpenAPI schema omitted due to length of output."
-                if len(error.instance) > 100:
-                    error.instance = "Error instance omitted due to length of output."
-            message += f"\n\n{type(error).__name__}: {error}"
+    except OpenAPIValidationError as error:
+        message = f"Response validation error at {method} /api/v1{path} ({status_code}):"
+        message += f"\n\n{type(error).__name__}: {error}"
         message += (
             "\n\nFor help debugging these errors see: "
             "https://zulip.readthedocs.io/en/latest/documentation/api.html#debugging-schema-validation-errors"

zerver/tests/test_upload.py

@@ -15,6 +15,7 @@
 from django.utils.timezone import now as timezone_now
 from PIL import Image
 from urllib3 import encode_multipart_formdata
+from urllib3.fields import RequestField
 
 import zerver.lib.upload
 from zerver.actions.create_realm import do_create_realm
@@ -34,21 +35,9 @@
 from zerver.lib.realm_logo import get_realm_logo_url
 from zerver.lib.retention import clean_archived_data
 from zerver.lib.test_classes import UploadSerializeMixin, ZulipTestCase
-from zerver.lib.test_helpers import (
-    avatar_disk_path,
-    get_test_image_file,
-    read_test_image_file,
-)
-from zerver.lib.upload import (
-    delete_message_attachment,
-    upload_message_attachment,
-)
-from zerver.lib.upload.base import (
-    BadImageError,
-    ZulipUploadBackend,
-    resize_emoji,
-    sanitize_name,
-)
+from zerver.lib.test_helpers import avatar_disk_path, get_test_image_file, read_test_image_file
+from zerver.lib.upload import delete_message_attachment, upload_message_attachment
+from zerver.lib.upload.base import BadImageError, ZulipUploadBackend, resize_emoji, sanitize_name
 from zerver.lib.upload.local import LocalUploadBackend
 from zerver.lib.upload.s3 import S3UploadBackend
 from zerver.lib.users import get_api_key
@@ -163,13 +152,17 @@ def test_guess_content_type_from_filename(self) -> None:
         Test coverage for files without content-type in the metadata;
         in which case we try to guess the content-type from the filename.
         """
-        data, content_type = encode_multipart_formdata({"file": ("somefile", b"zulip!", None)})
+        field = RequestField("file", b"zulip!", filename="somefile")
+        field.make_multipart()
+        data, content_type = encode_multipart_formdata([field])
         result = self.api_post(
             self.example_user("hamlet"), "/api/v1/user_uploads", data, content_type=content_type
         )
         self.assert_json_success(result)
 
-        data, content_type = encode_multipart_formdata({"file": ("somefile.txt", b"zulip!", None)})
+        field = RequestField("file", b"zulip!", filename="somefile.txt")
+        field.make_multipart()
+        data, content_type = encode_multipart_formdata([field])
         result = self.api_post(
             self.example_user("hamlet"), "/api/v1/user_uploads", data, content_type=content_type
         )

zerver/views/realm.py

@@ -179,10 +179,11 @@ def update_realm(
         if not user_profile.is_realm_owner:
             raise OrganizationOwnerRequiredError
         realm.ensure_not_on_limited_plan()
-        message_retention_days = parse_message_retention_days(
-            message_retention_days_raw, Realm.MESSAGE_RETENTION_SPECIAL_VALUES_MAP
+        message_retention_days = (  # noqa: F841 # used by locals() below
+            parse_message_retention_days(
+                message_retention_days_raw, Realm.MESSAGE_RETENTION_SPECIAL_VALUES_MAP
+            )
         )
-    message_retention_days  # used by locals() below
 
     if (
         invite_to_realm_policy is not None or invite_required is not None