Merge branch 'release-v0.18.4' of github.com:matrix-org/synapse

CHANGES.rst

@@ -1,3 +1,31 @@
+Changes in synapse v0.18.4 (2016-11-22)
+=======================================
+
+Bug fixes:
+
+* Add workaround for buggy clients that the fail to register (PR #1632)
+
+
+Changes in synapse v0.18.4-rc1 (2016-11-14)
+===========================================
+
+Changes:
+
+* Various database efficiency improvements (PR #1188, #1192)
+* Update default config to blacklist more internal IPs, thanks to Euan Kemp (PR
+  #1198)
+* Allow specifying duration in minutes in config, thanks to Daniel Dent (PR
+  #1625)
+
+
+Bug fixes:
+
+* Fix media repo to set CORs headers on responses (PR #1190)
+* Fix registration to not error on non-ascii passwords (PR #1191)
+* Fix create event code to limit the number of prev_events (PR #1615)
+* Fix bug in transaction ID deduplication (PR #1624)
+
+
 Changes in synapse v0.18.3 (2016-11-08)
 =======================================
 

docs/metrics-howto.rst

@@ -51,9 +51,9 @@ python_gc_counts            reactor_gc_counts
 
 The twisted-specific reactor metrics have been renamed.
 
-==================================== =================
+==================================== =====================
 New name                             Old name
------------------------------------- -----------------
-python_twisted_reactor_pending_calls reactor_tick_time
+------------------------------------ ---------------------
+python_twisted_reactor_pending_calls reactor_pending_calls
 python_twisted_reactor_tick_time     reactor_tick_time
-==================================== =================
+==================================== =====================

synapse/__init__.py

@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.18.3"
+__version__ = "0.18.4"

synapse/app/appservice.py

@@ -34,6 +34,8 @@
 from synapse.util.rlimit import change_resource_limit
 from synapse.util.versionstring import get_version_string
 
+from synapse import events
+
 from twisted.internet import reactor, defer
 from twisted.web.resource import Resource
 
@@ -151,6 +153,8 @@ def start(config_options):
 
     setup_logging(config.worker_log_config, config.worker_log_file)
 
+    events.USE_FROZEN_DICTS = config.use_frozen_dicts
+
     database_engine = create_engine(config.database_config)
 
     if config.notify_appservices:

synapse/app/client_reader.py

@@ -41,6 +41,8 @@
 from synapse.util.versionstring import get_version_string
 from synapse.crypto import context_factory
 
+from synapse import events
+
 
 from twisted.internet import reactor, defer
 from twisted.web.resource import Resource
@@ -165,6 +167,8 @@ def start(config_options):
 
     setup_logging(config.worker_log_config, config.worker_log_file)
 
+    events.USE_FROZEN_DICTS = config.use_frozen_dicts
+
     database_engine = create_engine(config.database_config)
 
     tls_server_context_factory = context_factory.ServerContextFactory(config)

synapse/app/federation_reader.py

@@ -39,6 +39,8 @@
 from synapse.federation.transport.server import TransportLayerServer
 from synapse.crypto import context_factory
 
+from synapse import events
+
 
 from twisted.internet import reactor, defer
 from twisted.web.resource import Resource
@@ -156,6 +158,8 @@ def start(config_options):
 
     setup_logging(config.worker_log_config, config.worker_log_file)
 
+    events.USE_FROZEN_DICTS = config.use_frozen_dicts
+
     database_engine = create_engine(config.database_config)
 
     tls_server_context_factory = context_factory.ServerContextFactory(config)

synapse/app/media_repository.py

@@ -41,6 +41,8 @@
 )
 from synapse.crypto import context_factory
 
+from synapse import events
+
 
 from twisted.internet import reactor, defer
 from twisted.web.resource import Resource
@@ -162,6 +164,8 @@ def start(config_options):
 
     setup_logging(config.worker_log_config, config.worker_log_file)
 
+    events.USE_FROZEN_DICTS = config.use_frozen_dicts
+
     database_engine = create_engine(config.database_config)
 
     tls_server_context_factory = context_factory.ServerContextFactory(config)

synapse/app/pusher.py

@@ -36,6 +36,8 @@
 from synapse.util.rlimit import change_resource_limit
 from synapse.util.versionstring import get_version_string
 
+from synapse import events
+
 from twisted.internet import reactor, defer
 from twisted.web.resource import Resource
 
@@ -239,6 +241,8 @@ def start(config_options):
 
     setup_logging(config.worker_log_config, config.worker_log_file)
 
+    events.USE_FROZEN_DICTS = config.use_frozen_dicts
+
     if config.start_pushers:
         sys.stderr.write(
             "\nThe pushers must be disabled in the main synapse process"

synapse/app/synchrotron.py

@@ -446,6 +446,8 @@ def start(config_options):
 
     setup_logging(config.worker_log_config, config.worker_log_file)
 
+    synapse.events.USE_FROZEN_DICTS = config.use_frozen_dicts
+
     database_engine = create_engine(config.database_config)
 
     ss = SynchrotronServer(

synapse/config/_base.py

@@ -64,11 +64,12 @@ def parse_duration(value):
         if isinstance(value, int) or isinstance(value, long):
             return value
         second = 1000
-        hour = 60 * 60 * second
+        minute = 60 * second
+        hour = 60 * minute
         day = 24 * hour
         week = 7 * day
         year = 365 * day
-        sizes = {"s": second, "h": hour, "d": day, "w": week, "y": year}
+        sizes = {"s": second, "m": minute, "h": hour, "d": day, "w": week, "y": year}
         size = 1
         suffix = value[-1]
         if suffix in sizes:

synapse/config/password_auth_providers.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ._base import Config
+from ._base import Config, ConfigError
 
 import importlib
 
@@ -39,7 +39,12 @@ def read_config(self, config):
             module = importlib.import_module(module)
             provider_class = getattr(module, clz)
 
-            provider_config = provider_class.parse_config(provider["config"])
+            try:
+                provider_config = provider_class.parse_config(provider["config"])
+            except Exception as e:
+                raise ConfigError(
+                    "Failed to parse config for %r: %r" % (provider['module'], e)
+                )
             self.password_providers.append((provider_class, provider_config))
 
     def default_config(self, **kwargs):

synapse/config/repository.py

@@ -167,6 +167,8 @@ def default_config(self, **kwargs):
         # - '10.0.0.0/8'
         # - '172.16.0.0/12'
         # - '192.168.0.0/16'
+        # - '100.64.0.0/10'
+        # - '169.254.0.0/16'
         #
         # List of IP address CIDR ranges that the URL preview spider is allowed
         # to access even if they are specified in url_preview_ip_range_blacklist.

synapse/handlers/auth.py

@@ -653,7 +653,7 @@ def hash(self, password):
         Returns:
             Hashed password (str).
         """
-        return bcrypt.hashpw(password + self.hs.config.password_pepper,
+        return bcrypt.hashpw(password.encode('utf8') + self.hs.config.password_pepper,
                              bcrypt.gensalt(self.bcrypt_rounds))
 
     def validate_hash(self, password, stored_hash):

synapse/handlers/message.py

@@ -34,6 +34,7 @@
 from canonicaljson import encode_canonical_json
 
 import logging
+import random
 
 logger = logging.getLogger(__name__)
 
@@ -415,6 +416,20 @@ def _create_new_client_event(self, builder, prev_event_ids=None):
                 builder.room_id,
             )
 
+            # We want to limit the max number of prev events we point to in our
+            # new event
+            if len(latest_ret) > 10:
+                # Sort by reverse depth, so we point to the most recent.
+                latest_ret.sort(key=lambda a: -a[2])
+                new_latest_ret = latest_ret[:5]
+
+                # We also randomly point to some of the older events, to make
+                # sure that we don't completely ignore the older events.
+                if latest_ret[5:]:
+                    sample_size = min(5, len(latest_ret[5:]))
+                    new_latest_ret.extend(random.sample(latest_ret[5:], sample_size))
+                latest_ret = new_latest_ret
+
             if latest_ret:
                 depth = max([d for _, _, d in latest_ret]) + 1
             else:

synapse/http/server.py

@@ -392,17 +392,30 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False,
     request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),))
 
     if send_cors:
-        request.setHeader("Access-Control-Allow-Origin", "*")
-        request.setHeader("Access-Control-Allow-Methods",
-                          "GET, POST, PUT, DELETE, OPTIONS")
-        request.setHeader("Access-Control-Allow-Headers",
-                          "Origin, X-Requested-With, Content-Type, Accept")
+        set_cors_headers(request)
 
     request.write(json_bytes)
     finish_request(request)
     return NOT_DONE_YET
 
 
+def set_cors_headers(request):
+    """Set the CORs headers so that javascript running in a web browsers can
+    use this API
+
+    Args:
+        request (twisted.web.http.Request): The http request to add CORs to.
+    """
+    request.setHeader("Access-Control-Allow-Origin", "*")
+    request.setHeader(
+        "Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS"
+    )
+    request.setHeader(
+        "Access-Control-Allow-Headers",
+        "Origin, X-Requested-With, Content-Type, Accept"
+    )
+
+
 def finish_request(request):
     """ Finish writing the response to the request.
 

synapse/metrics/__init__.py

@@ -111,18 +111,20 @@ def render_all():
     return "\n".join(strs)
 
 
-reactor_metrics = get_metrics_for("reactor")
-tick_time = reactor_metrics.register_distribution("tick_time")
-pending_calls_metric = reactor_metrics.register_distribution("pending_calls")
+register_process_collector(get_metrics_for("process"))
+
 
-gc_time = reactor_metrics.register_distribution("gc_time", labels=["gen"])
-gc_unreachable = reactor_metrics.register_counter("gc_unreachable", labels=["gen"])
+python_metrics = get_metrics_for("python")
 
-reactor_metrics.register_callback(
+gc_time = python_metrics.register_distribution("gc_time", labels=["gen"])
+gc_unreachable = python_metrics.register_counter("gc_unreachable_total", labels=["gen"])
+python_metrics.register_callback(
     "gc_counts", lambda: {(i,): v for i, v in enumerate(gc.get_count())}, labels=["gen"]
 )
 
-register_process_collector(get_metrics_for("process"))
+reactor_metrics = get_metrics_for("python.twisted.reactor")
+tick_time = reactor_metrics.register_distribution("tick_time")
+pending_calls_metric = reactor_metrics.register_distribution("pending_calls")
 
 
 def runUntilCurrentTimer(func):