Skip to content

Commit

Permalink
admin rewrite: templates can be modified to enable/disable servers
Browse files Browse the repository at this point in the history
  • Loading branch information
@manics
manics committed Dec 2, 2020
1 parent 5425f76 commit 29d1968
Show file tree
Hide file tree
Showing 3 changed files with 119 additions and 28 deletions.
13 changes: 10 additions & 3 deletions src/omero/plugins/admin.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@
from omero_ext.argparse import FileType
from omero_version import ice_compatibility

from ..util._process_defaultxml import _process_xml


try:
import pywintypes
Expand Down Expand Up @@ -1186,9 +1188,10 @@ def clear_tail(elem):
for clienttp in client_transports.split(',')]
substitutions['@omero.client.endpoints@'] = ':'.join(client_endpoints)

def copy_template(input_file, output_dir):
"""Replace templates"""
node_descriptors = config.get('omero.server.nodedescriptors', '')

def copy_template(input_file, output_dir, post_process=None):
"""Replace templates"""
with open(input_file) as template:
data = template.read()
output_file = path(old_div(output_dir,
Expand All @@ -1198,14 +1201,18 @@ def copy_template(input_file, output_dir):
with open(output_file, 'w') as f:
for key, value in substitutions.items():
data = re.sub(key, value, data)
if post_process:
data = post_process(data)
f.write(data)

# Regenerate various configuration files from templates
for cfg_file in glob(old_div(self._get_templates_dir(), "*.cfg")):
copy_template(cfg_file, self._get_etc_dir())
for xml_file in glob(
self._get_templates_dir() / "grid" / "*default.xml"):
copy_template(xml_file, old_div(self._get_etc_dir(), "grid"))
copy_template(
xml_file, old_div(self._get_etc_dir(), "grid"),
lambda xml: _process_xml(xml, node_descriptors))
ice_config = old_div(self._get_templates_dir(), "ice.config")
substitutions['@omero.master.host@'] = config.get(
'omero.master.host', config.get('Ice.Default.Host', 'localhost'))
Expand Down
42 changes: 18 additions & 24 deletions src/omero/util/_process_defaultxml.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
#!/usr/bin/env python

"""
Convert the nodes and server-instances in default.xml to a multi-node
configuration
Expand All @@ -23,7 +21,7 @@
import sys


def getnodes(nodedescs):
def _getnodes(nodedescs):
nodes = {}
for nd in nodedescs:
s = ''
Expand All @@ -47,27 +45,23 @@ def getnodes(nodedescs):
return nodes


with open(sys.argv[1]) as f:
xml = f.read()

pattern = r'\<node name="master"\>\s*\<server-instance[^\>]*\>(.*?\</node\>)'
m = re.search(pattern, xml, re.DOTALL)
assert m
def _process_xml(xml, nodedescs):
pattern = r'\<node name="master"\>\s*\<server-instance[^\>]*\>(.*?\</node\>)'
m = re.search(pattern, xml, re.DOTALL)
assert m

nodedescs = sys.argv[2:]
master = '\n </node>\n'
slaves = ''
nodes = _getnodes(nodedescs.split())
for nodename in sorted(nodes.keys()):
servers = nodes[nodename]
if nodename == 'master':
master = '%s%s' % (servers, master)
else:
slaves += ' <node name="%s">\n%s </node>\n' % (nodename, servers)

master = '\n </node>\n'
slaves = ''
nodes = getnodes(nodedescs)
for nodename in sorted(nodes.keys()):
servers = nodes[nodename]
if nodename == 'master':
master = '%s%s' % (servers, master)
if nodes:
xmlout = xml[:m.start(1)] + master + slaves + xml[m.end(1):]
else:
slaves += ' <node name="%s">\n%s </node>\n' % (nodename, servers)

if nodes:
xmlout = xml[:m.start(1)] + master + slaves + xml[m.end(1):]
else:
xmlout = xml
print xmlout
xmlout = xml
return xmlout
92 changes: 91 additions & 1 deletion test/unit/clitest/test_admin.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,12 @@
OMERODIR = os.environ.get('OMERODIR')


def _squashWhiteSpace(s):
# Attempt to make the XML comparison more robust
return ' '.join(s.split())

@pytest.fixture(autouse=True)
def tmpadmindir(tmpdir):
def tmpadmindir(tmpdir, monkeypatch):
etc_dir = tmpdir.mkdir('etc')
etc_dir.mkdir('grid')
tmpdir.mkdir('var')
Expand All @@ -67,6 +71,9 @@ def tmpadmindir(tmpdir):
for f in glob(os.path.join(old_templates_dir, "grid", "*.xml")):
path(f).copy(path(old_div(templates_dir, "grid")))
path(os.path.join(old_templates_dir, "ice.config")).copy(path(templates_dir))
# The OMERODIR env-var is directly reference in other omero components so
# we need to override it
monkeypatch.setenv('OMERODIR', str(tmpdir))

return path(tmpdir)

Expand Down Expand Up @@ -329,6 +336,72 @@ def testInvalidJvmCfgStrategy(self, suffix, tmpdir):
self.cli.invoke(self.args, strict=True)



DEFAULT_XML_PARAMS = {
# "": """\
# <node name="master">
# <server-instance template="Glacier2Template"
# client-endpoints="ssl -p 4064:tcp -p 4063"
# server-endpoints="tcp -h 127.0.0.1"/>
# <server-instance template="BlitzTemplate" index="0" config="default"/>
# <server-instance template="IndexerTemplate" index="0"/>
# <server-instance template="DropBoxTemplate"/>
# <server-instance template="MonitorServerTemplate"/>
# <server-instance template="FileServerTemplate"/>
# <server-instance template="StormTemplate"/>
# <server-instance template="PixelDataTemplate" index="0" dir=""/><!-- assumes legacy -->
# <server-instance template="ProcessorTemplate" index="0" dir=""/><!-- assumes legacy -->
# <server-instance template="TablesTemplate" index="0" dir=""/><!-- assumes legacy -->
# <server-instance template="TestDropBoxTemplate"/>
# </node>""",
# "master:Blitz-0,Indexer-0,DropBox,MonitorServer,FileServer,Storm,PixelData-0,Tables-0,TestDropBox slave:Processor-0": """
# <node name="master">
# <server-instance template="Glacier2Template"
# client-endpoints="@omero.client.endpoints@"
# server-endpoints="tcp -h @omero.master.host@"/>
# <server-instance template="BlitzTemplate" index="0" config="default"/>
# <server-instance template="IndexerTemplate" index="0"/>
# <server-instance template="DropBoxTemplate"/>
# <server-instance template="MonitorServerTemplate"/>
# <server-instance template="FileServerTemplate"/>
# <server-instance template="StormTemplate"/>
# <server-instance template="PixelDataTemplate" index="0" dir=""/>
# <server-instance template="TablesTemplate" index="0" dir=""/>
# <server-instance template="TestDropBoxTemplate"/>
# </node>

# <node name="worker">
# <server-instance template="ProcessorTemplate" index="0" dir=""/>
# </node>
# """,
"master:Blitz-0,Indexer-0,DropBox,MonitorServer,FileServer,Storm,Tables-0,TestDropBox worker-1:Processor-0,PixelData-0 worker-2:Processor-1,PixelData-1": """
<node name="master">
<server-instance template="Glacier2Template"
client-endpoints="ssl -p 4064:tcp -p 4063"
server-endpoints="tcp -h 127.0.0.1"/>
<server-instance template="BlitzTemplate" index="0" config="default"/>
<server-instance template="IndexerTemplate" index="0"/>
<server-instance template="DropBoxTemplate"/>
<server-instance template="MonitorServerTemplate"/>
<server-instance template="FileServerTemplate"/>
<server-instance template="StormTemplate"/>
<server-instance template="TablesTemplate" index="0" dir=""/>
<server-instance template="TestDropBoxTemplate"/>
</node>
<node name="worker-1">
<server-instance template="ProcessorTemplate" index="0" dir=""/>
<server-instance template="PixelDataTemplate" index="0" dir=""/>
</node>
<node name="worker-2">
<server-instance template="ProcessorTemplate" index="1" dir=""/>
<server-instance template="PixelDataTemplate" index="1" dir=""/>
</node>
""",
}


@pytest.mark.skipif(OMERODIR is False, reason="We need $OMERODIR")
class TestRewrite(object):
"""Test template files regeneration"""
Expand Down Expand Up @@ -446,3 +519,20 @@ def testGlacier2Icessl(self, monkeypatch):
strict=True)
self.cli.invoke(self.args, strict=True)
check_templates_xml(self.cli.dir, glacier2)

@pytest.mark.parametrize('descriptor', DEFAULT_XML_PARAMS.keys())
def testNodeDescriptors(self, descriptor, monkeypatch):
"""
Test omero.server.nodedescriptors for configuring the available
services in master and other nodes
"""
self.cli.invoke(
["config", "set", "omero.server.nodedescriptors", descriptor],
strict=True)
self.cli.invoke(self.args, strict=True)

defaultxml = (self.cli.dir / "etc" / "grid" / "default.xml").text()
# print(defaultxml)
expected = DEFAULT_XML_PARAMS[descriptor]

assert _squashWhiteSpace(expected) in _squashWhiteSpace(defaultxml)

0 comments on commit 29d1968

Please sign in to comment.