Skip to content

Commit

Permalink
Merge pull request matrix-org#3764 from matrix-org/rav/close_db_conn_…
Browse files Browse the repository at this point in the history
…after_init

Make sure that we close db connections opened during init
  • Loading branch information
richvdh authored Aug 30, 2018
2 parents 7f03995 + 414fa36 commit 475253a
Showing 16 changed files with 41 additions and 54 deletions.
1 change: 1 addition & 0 deletions changelog.d/3764.misc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Make sure that we close db connections opened during init
5 changes: 1 addition & 4 deletions synapse/app/appservice.py
Original file line number Diff line number Diff line change
@@ -51,10 +51,7 @@ class AppserviceSlaveStore(


class AppserviceServer(HomeServer):
def setup(self):
logger.info("Setting up.")
self.datastore = AppserviceSlaveStore(self.get_db_conn(), self)
logger.info("Finished setting up.")
DATASTORE_CLASS = AppserviceSlaveStore

def _listen_http(self, listener_config):
port = listener_config["port"]
5 changes: 1 addition & 4 deletions synapse/app/client_reader.py
Original file line number Diff line number Diff line change
@@ -74,10 +74,7 @@ class ClientReaderSlavedStore(


class ClientReaderServer(HomeServer):
def setup(self):
logger.info("Setting up.")
self.datastore = ClientReaderSlavedStore(self.get_db_conn(), self)
logger.info("Finished setting up.")
DATASTORE_CLASS = ClientReaderSlavedStore

def _listen_http(self, listener_config):
port = listener_config["port"]
5 changes: 1 addition & 4 deletions synapse/app/event_creator.py
Original file line number Diff line number Diff line change
@@ -90,10 +90,7 @@ class EventCreatorSlavedStore(


class EventCreatorServer(HomeServer):
def setup(self):
logger.info("Setting up.")
self.datastore = EventCreatorSlavedStore(self.get_db_conn(), self)
logger.info("Finished setting up.")
DATASTORE_CLASS = EventCreatorSlavedStore

def _listen_http(self, listener_config):
port = listener_config["port"]
5 changes: 1 addition & 4 deletions synapse/app/federation_reader.py
Original file line number Diff line number Diff line change
@@ -72,10 +72,7 @@ class FederationReaderSlavedStore(


class FederationReaderServer(HomeServer):
def setup(self):
logger.info("Setting up.")
self.datastore = FederationReaderSlavedStore(self.get_db_conn(), self)
logger.info("Finished setting up.")
DATASTORE_CLASS = FederationReaderSlavedStore

def _listen_http(self, listener_config):
port = listener_config["port"]
5 changes: 1 addition & 4 deletions synapse/app/federation_sender.py
Original file line number Diff line number Diff line change
@@ -78,10 +78,7 @@ def _get_federation_out_pos(self, db_conn):


class FederationSenderServer(HomeServer):
def setup(self):
logger.info("Setting up.")
self.datastore = FederationSenderSlaveStore(self.get_db_conn(), self)
logger.info("Finished setting up.")
DATASTORE_CLASS = FederationSenderSlaveStore

def _listen_http(self, listener_config):
port = listener_config["port"]
5 changes: 1 addition & 4 deletions synapse/app/frontend_proxy.py
Original file line number Diff line number Diff line change
@@ -148,10 +148,7 @@ class FrontendProxySlavedStore(


class FrontendProxyServer(HomeServer):
def setup(self):
logger.info("Setting up.")
self.datastore = FrontendProxySlavedStore(self.get_db_conn(), self)
logger.info("Finished setting up.")
DATASTORE_CLASS = FrontendProxySlavedStore

def _listen_http(self, listener_config):
port = listener_config["port"]
14 changes: 8 additions & 6 deletions synapse/app/homeserver.py
Original file line number Diff line number Diff line change
@@ -62,7 +62,7 @@
from synapse.rest.key.v2 import KeyApiV2Resource
from synapse.rest.media.v0.content_repository import ContentRepoResource
from synapse.server import HomeServer
from synapse.storage import are_all_users_on_domain
from synapse.storage import DataStore, are_all_users_on_domain
from synapse.storage.engines import IncorrectDatabaseSetup, create_engine
from synapse.storage.prepare_database import UpgradeDatabaseException, prepare_database
from synapse.util.caches import CACHE_SIZE_FACTOR
@@ -111,6 +111,8 @@ def build_resource_for_web_client(hs):


class SynapseHomeServer(HomeServer):
DATASTORE_CLASS = DataStore

def _listener_http(self, config, listener_config):
port = listener_config["port"]
bind_addresses = listener_config["bind_addresses"]
@@ -356,13 +358,13 @@ def setup(config_options):
logger.info("Preparing database: %s...", config.database_config['name'])

try:
db_conn = hs.get_db_conn(run_new_connection=False)
prepare_database(db_conn, database_engine, config=config)
database_engine.on_new_connection(db_conn)
with hs.get_db_conn(run_new_connection=False) as db_conn:
prepare_database(db_conn, database_engine, config=config)
database_engine.on_new_connection(db_conn)

hs.run_startup_checks(db_conn, database_engine)
hs.run_startup_checks(db_conn, database_engine)

db_conn.commit()
db_conn.commit()
except UpgradeDatabaseException:
sys.stderr.write(
"\nFailed to upgrade database.\n"
5 changes: 1 addition & 4 deletions synapse/app/media_repository.py
Original file line number Diff line number Diff line change
@@ -60,10 +60,7 @@ class MediaRepositorySlavedStore(


class MediaRepositoryServer(HomeServer):
def setup(self):
logger.info("Setting up.")
self.datastore = MediaRepositorySlavedStore(self.get_db_conn(), self)
logger.info("Finished setting up.")
DATASTORE_CLASS = MediaRepositorySlavedStore

def _listen_http(self, listener_config):
port = listener_config["port"]
5 changes: 1 addition & 4 deletions synapse/app/pusher.py
Original file line number Diff line number Diff line change
@@ -78,10 +78,7 @@ class PusherSlaveStore(


class PusherServer(HomeServer):
def setup(self):
logger.info("Setting up.")
self.datastore = PusherSlaveStore(self.get_db_conn(), self)
logger.info("Finished setting up.")
DATASTORE_CLASS = PusherSlaveStore

def remove_pusher(self, app_id, push_key, user_id):
self.get_tcp_replication().send_remove_pusher(app_id, push_key, user_id)
5 changes: 1 addition & 4 deletions synapse/app/synchrotron.py
Original file line number Diff line number Diff line change
@@ -249,10 +249,7 @@ def notify_interested_services(self, event):


class SynchrotronServer(HomeServer):
def setup(self):
logger.info("Setting up.")
self.datastore = SynchrotronSlavedStore(self.get_db_conn(), self)
logger.info("Finished setting up.")
DATASTORE_CLASS = SynchrotronSlavedStore

def _listen_http(self, listener_config):
port = listener_config["port"]
5 changes: 1 addition & 4 deletions synapse/app/user_dir.py
Original file line number Diff line number Diff line change
@@ -94,10 +94,7 @@ def process_replication_rows(self, stream_name, token, rows):


class UserDirectoryServer(HomeServer):
def setup(self):
logger.info("Setting up.")
self.datastore = UserDirectorySlaveStore(self.get_db_conn(), self)
logger.info("Finished setting up.")
DATASTORE_CLASS = UserDirectorySlaveStore

def _listen_http(self, listener_config):
port = listener_config["port"]
14 changes: 12 additions & 2 deletions synapse/server.py
Original file line number Diff line number Diff line change
@@ -19,6 +19,7 @@
# partial one for unit test mocking.

# Imports required for the default HomeServer() implementation
import abc
import logging

from twisted.enterprise import adbapi
@@ -81,7 +82,6 @@
from synapse.server_notices.server_notices_sender import ServerNoticesSender
from synapse.server_notices.worker_server_notices_sender import WorkerServerNoticesSender
from synapse.state import StateHandler, StateResolutionHandler
from synapse.storage import DataStore
from synapse.streams.events import EventSources
from synapse.util import Clock
from synapse.util.distributor import Distributor
@@ -111,6 +111,8 @@ def build_DEPENDENCY(self)
config (synapse.config.homeserver.HomeserverConfig):
"""

__metaclass__ = abc.ABCMeta

DEPENDENCIES = [
'http_client',
'db_pool',
@@ -172,6 +174,11 @@ def build_DEPENDENCY(self)
'room_context_handler',
]

# This is overridden in derived application classes
# (such as synapse.app.homeserver.SynapseHomeServer) and gives the class to be
# instantiated during setup() for future return by get_datastore()
DATASTORE_CLASS = abc.abstractproperty()

def __init__(self, hostname, reactor=None, **kwargs):
"""
Args:
@@ -188,13 +195,16 @@ def __init__(self, hostname, reactor=None, **kwargs):
self.distributor = Distributor()
self.ratelimiter = Ratelimiter()

self.datastore = None

# Other kwargs are explicit dependencies
for depname in kwargs:
setattr(self, depname, kwargs[depname])

def setup(self):
logger.info("Setting up.")
self.datastore = DataStore(self.get_db_conn(), self)
with self.get_db_conn() as conn:
self.datastore = self.DATASTORE_CLASS(conn, self)
logger.info("Finished setting up.")

def get_reactor(self):
4 changes: 2 additions & 2 deletions tests/storage/test_base.py
Original file line number Diff line number Diff line change
@@ -20,11 +20,11 @@

from twisted.internet import defer

from synapse.server import HomeServer
from synapse.storage._base import SQLBaseStore
from synapse.storage.engines import create_engine

from tests import unittest
from tests.utils import TestHomeServer


class SQLBaseStoreTestCase(unittest.TestCase):
@@ -51,7 +51,7 @@ def runWithConnection(func, *args, **kwargs):
config = Mock()
config.event_cache_size = 1
config.database_config = {"name": "sqlite3"}
hs = HomeServer(
hs = TestHomeServer(
"test",
db_pool=self.db_pool,
config=config,
4 changes: 2 additions & 2 deletions tests/test_types.py
Original file line number Diff line number Diff line change
@@ -14,12 +14,12 @@
# limitations under the License.

from synapse.api.errors import SynapseError
from synapse.server import HomeServer
from synapse.types import GroupID, RoomAlias, UserID

from tests import unittest
from tests.utils import TestHomeServer

mock_homeserver = HomeServer(hostname="my.domain")
mock_homeserver = TestHomeServer(hostname="my.domain")


class UserIDTestCase(unittest.TestCase):
8 changes: 6 additions & 2 deletions tests/utils.py
Original file line number Diff line number Diff line change
@@ -29,7 +29,7 @@
from synapse.federation.transport import server
from synapse.http.server import HttpServer
from synapse.server import HomeServer
from synapse.storage import PostgresEngine
from synapse.storage import DataStore, PostgresEngine
from synapse.storage.engines import create_engine
from synapse.storage.prepare_database import (
_get_or_create_schema_state,
@@ -92,10 +92,14 @@ def _cleanup():
atexit.register(_cleanup)


class TestHomeServer(HomeServer):
DATASTORE_CLASS = DataStore


@defer.inlineCallbacks
def setup_test_homeserver(
cleanup_func, name="test", datastore=None, config=None, reactor=None,
homeserverToUse=HomeServer, **kargs
homeserverToUse=TestHomeServer, **kargs
):
"""
Setup a homeserver suitable for running tests against. Keyword arguments

0 comments on commit 475253a

Please sign in to comment.