From 29d19687c32e5aad22e9c7023dad4fec25a1fd8d Mon Sep 17 00:00:00 2001 From: Simon Li Date: Wed, 2 Dec 2020 20:27:28 +0000 Subject: [PATCH] admin rewrite: templates can be modified to enable/disable servers --- src/omero/plugins/admin.py | 13 +++- src/omero/util/_process_defaultxml.py | 42 ++++++------ test/unit/clitest/test_admin.py | 92 ++++++++++++++++++++++++++- 3 files changed, 119 insertions(+), 28 deletions(-) diff --git a/src/omero/plugins/admin.py b/src/omero/plugins/admin.py index ed682d53c..ac16c02e2 100755 --- a/src/omero/plugins/admin.py +++ b/src/omero/plugins/admin.py @@ -57,6 +57,8 @@ from omero_ext.argparse import FileType from omero_version import ice_compatibility +from ..util._process_defaultxml import _process_xml + try: import pywintypes @@ -1186,9 +1188,10 @@ def clear_tail(elem): for clienttp in client_transports.split(',')] substitutions['@omero.client.endpoints@'] = ':'.join(client_endpoints) - def copy_template(input_file, output_dir): - """Replace templates""" + node_descriptors = config.get('omero.server.nodedescriptors', '') + def copy_template(input_file, output_dir, post_process=None): + """Replace templates""" with open(input_file) as template: data = template.read() output_file = path(old_div(output_dir, @@ -1198,6 +1201,8 @@ def copy_template(input_file, output_dir): with open(output_file, 'w') as f: for key, value in substitutions.items(): data = re.sub(key, value, data) + if post_process: + data = post_process(data) f.write(data) # Regenerate various configuration files from templates @@ -1205,7 +1210,9 @@ def copy_template(input_file, output_dir): copy_template(cfg_file, self._get_etc_dir()) for xml_file in glob( self._get_templates_dir() / "grid" / "*default.xml"): - copy_template(xml_file, old_div(self._get_etc_dir(), "grid")) + copy_template( + xml_file, old_div(self._get_etc_dir(), "grid"), + lambda xml: _process_xml(xml, node_descriptors)) ice_config = old_div(self._get_templates_dir(), "ice.config") substitutions['@omero.master.host@'] = config.get( 'omero.master.host', config.get('Ice.Default.Host', 'localhost')) diff --git a/src/omero/util/_process_defaultxml.py b/src/omero/util/_process_defaultxml.py index 580370c90..8b0ec8f7d 100644 --- a/src/omero/util/_process_defaultxml.py +++ b/src/omero/util/_process_defaultxml.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - """ Convert the nodes and server-instances in default.xml to a multi-node configuration @@ -23,7 +21,7 @@ import sys -def getnodes(nodedescs): +def _getnodes(nodedescs): nodes = {} for nd in nodedescs: s = '' @@ -47,27 +45,23 @@ def getnodes(nodedescs): return nodes -with open(sys.argv[1]) as f: - xml = f.read() - -pattern = r'\\s*\]*\>(.*?\)' -m = re.search(pattern, xml, re.DOTALL) -assert m +def _process_xml(xml, nodedescs): + pattern = r'\\s*\]*\>(.*?\)' + m = re.search(pattern, xml, re.DOTALL) + assert m -nodedescs = sys.argv[2:] + master = '\n \n' + slaves = '' + nodes = _getnodes(nodedescs.split()) + for nodename in sorted(nodes.keys()): + servers = nodes[nodename] + if nodename == 'master': + master = '%s%s' % (servers, master) + else: + slaves += ' \n%s \n' % (nodename, servers) -master = '\n \n' -slaves = '' -nodes = getnodes(nodedescs) -for nodename in sorted(nodes.keys()): - servers = nodes[nodename] - if nodename == 'master': - master = '%s%s' % (servers, master) + if nodes: + xmlout = xml[:m.start(1)] + master + slaves + xml[m.end(1):] else: - slaves += ' \n%s \n' % (nodename, servers) - -if nodes: - xmlout = xml[:m.start(1)] + master + slaves + xml[m.end(1):] -else: - xmlout = xml -print xmlout + xmlout = xml + return xmlout diff --git a/test/unit/clitest/test_admin.py b/test/unit/clitest/test_admin.py index 66077a26e..4238ef504 100644 --- a/test/unit/clitest/test_admin.py +++ b/test/unit/clitest/test_admin.py @@ -48,8 +48,12 @@ OMERODIR = os.environ.get('OMERODIR') +def _squashWhiteSpace(s): + # Attempt to make the XML comparison more robust + return ' '.join(s.split()) + @pytest.fixture(autouse=True) -def tmpadmindir(tmpdir): +def tmpadmindir(tmpdir, monkeypatch): etc_dir = tmpdir.mkdir('etc') etc_dir.mkdir('grid') tmpdir.mkdir('var') @@ -67,6 +71,9 @@ def tmpadmindir(tmpdir): for f in glob(os.path.join(old_templates_dir, "grid", "*.xml")): path(f).copy(path(old_div(templates_dir, "grid"))) path(os.path.join(old_templates_dir, "ice.config")).copy(path(templates_dir)) + # The OMERODIR env-var is directly reference in other omero components so + # we need to override it + monkeypatch.setenv('OMERODIR', str(tmpdir)) return path(tmpdir) @@ -329,6 +336,72 @@ def testInvalidJvmCfgStrategy(self, suffix, tmpdir): self.cli.invoke(self.args, strict=True) + +DEFAULT_XML_PARAMS = { + # "": """\ + # + # + # + # + # + # + # + # + # + # + # + # + # """, + # "master:Blitz-0,Indexer-0,DropBox,MonitorServer,FileServer,Storm,PixelData-0,Tables-0,TestDropBox slave:Processor-0": """ + # + # + # + # + # + # + # + # + # + # + # + # + + # + # + # + # """, + "master:Blitz-0,Indexer-0,DropBox,MonitorServer,FileServer,Storm,Tables-0,TestDropBox worker-1:Processor-0,PixelData-0 worker-2:Processor-1,PixelData-1": """ + + + + + + + + + + + + + + + + + + + + + + """, +} + + @pytest.mark.skipif(OMERODIR is False, reason="We need $OMERODIR") class TestRewrite(object): """Test template files regeneration""" @@ -446,3 +519,20 @@ def testGlacier2Icessl(self, monkeypatch): strict=True) self.cli.invoke(self.args, strict=True) check_templates_xml(self.cli.dir, glacier2) + + @pytest.mark.parametrize('descriptor', DEFAULT_XML_PARAMS.keys()) + def testNodeDescriptors(self, descriptor, monkeypatch): + """ + Test omero.server.nodedescriptors for configuring the available + services in master and other nodes + """ + self.cli.invoke( + ["config", "set", "omero.server.nodedescriptors", descriptor], + strict=True) + self.cli.invoke(self.args, strict=True) + + defaultxml = (self.cli.dir / "etc" / "grid" / "default.xml").text() + # print(defaultxml) + expected = DEFAULT_XML_PARAMS[descriptor] + + assert _squashWhiteSpace(expected) in _squashWhiteSpace(defaultxml)