Skip to content

Commit

Permalink
add --manual_run option for running interop tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@jtattermusch
jtattermusch committed Mar 10, 2017
1 parent d159af3 commit 7134a31
Showing 1 changed file with 94 additions and 24 deletions.
118 changes: 94 additions & 24 deletions tools/run_tests/run_interop_tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -501,6 +501,28 @@ def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None):
return docker_cmdline


def manual_cmdline(docker_cmdline):
"""Returns docker cmdline adjusted for manual invocation."""
print_cmdline = []
for item in docker_cmdline:
if item.startswith('--name='):
continue
# add quotes when necessary
if any(character.isspace() for character in item):
item = "\"%s\"" % item
print_cmdline.append(item)
return ' '.join(print_cmdline)


def write_cmdlog_maybe(cmdlog, filename):
"""Returns docker cmdline adjusted for manual invocation."""
if cmdlog:
with open(filename, 'w') as logfile:
logfile.write('#!/bin/bash\n')
logfile.writelines("%s\n" % line for line in cmdlog)
print('Command log written to file %s' % filename)


def bash_cmdline(cmdline):
"""Creates bash -c cmdline from args list."""
# Use login shell:
Expand Down Expand Up @@ -551,7 +573,8 @@ def _job_kill_handler(job):


def cloud_to_prod_jobspec(language, test_case, server_host_name,
server_host_detail, docker_image=None, auth=False):
server_host_detail, docker_image=None, auth=False,
manual_cmd_log=None):
"""Creates jobspec for cloud-to-prod interop test"""
container_name = None
cmdargs = [
Expand All @@ -576,7 +599,9 @@ def cloud_to_prod_jobspec(language, test_case, server_host_name,
cwd=cwd,
environ=environ,
docker_args=['--net=host',
'--name', container_name])
'--name=%s' % container_name])
if manual_cmd_log is not None:
manual_cmd_log.append(manual_cmdline(cmdline))
cwd = None
environ = None

Expand All @@ -597,7 +622,8 @@ def cloud_to_prod_jobspec(language, test_case, server_host_name,


def cloud_to_cloud_jobspec(language, test_case, server_name, server_host,
server_port, docker_image=None, insecure=False):
server_port, docker_image=None, insecure=False,
manual_cmd_log=None):
"""Creates jobspec for cloud-to-cloud interop test"""
interop_only_options = [
'--server_host_override=foo.test.google.fr',
Expand Down Expand Up @@ -628,7 +654,9 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host,
environ=environ,
cwd=cwd,
docker_args=['--net=host',
'--name', container_name])
'--name=%s' % container_name])
if manual_cmd_log is not None:
manual_cmd_log.append(manual_cmdline(cmdline))
cwd = None

test_job = jobset.JobSpec(
Expand All @@ -646,7 +674,7 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host,
return test_job


def server_jobspec(language, docker_image, insecure=False):
def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None):
"""Create jobspec for running a server"""
container_name = dockerjob.random_name('interop_server_%s' % language.safename)
cmdline = bash_cmdline(
Expand Down Expand Up @@ -676,7 +704,9 @@ def server_jobspec(language, docker_image, insecure=False):
cwd=language.server_cwd,
environ=environ,
docker_args=port_args +
['--name', container_name])
['--name=%s' % container_name])
if manual_cmd_log is not None:
manual_cmd_log.append(manual_cmdline(docker_cmdline))
server_job = jobset.JobSpec(
cmdline=docker_cmdline,
environ=environ,
Expand Down Expand Up @@ -806,6 +836,14 @@ def aggregate_http2_results(stdout):
action='store_const',
const=True,
help='Allow flaky tests to show as passing (re-runs failed tests up to five times)')
argp.add_argument('--manual_run',
default=False,
action='store_const',
const=True,
help='Prepare things for running interop tests manually. ' +
'Preserve docker images after building them and skip '
'actually running the tests. Only print commands to run by ' +
'hand.')
argp.add_argument('--http2_interop',
default=False,
action='store_const',
Expand Down Expand Up @@ -837,6 +875,10 @@ def aggregate_http2_results(stdout):
print('copied to the docker environment.')
time.sleep(5)

if args.manual_run and not args.use_docker:
print('--manual_run is only supported with --use_docker option enabled.')
sys.exit(1)

if not args.use_docker and servers:
print('Running interop servers is only supported with --use_docker option enabled.')
sys.exit(1)
Expand Down Expand Up @@ -887,24 +929,36 @@ def aggregate_http2_results(stdout):
dockerjob.remove_image(image, skip_nonexistent=True)
sys.exit(1)

server_manual_cmd_log = [] if args.manual_run else None
client_manual_cmd_log = [] if args.manual_run else None

# Start interop servers.
server_jobs={}
server_addresses={}
server_jobs = {}
server_addresses = {}
try:
for s in servers:
lang = str(s)
spec = server_jobspec(_LANGUAGES[lang], docker_images.get(lang),
args.insecure)
job = dockerjob.DockerJob(spec)
server_jobs[lang] = job
server_addresses[lang] = ('localhost', job.mapped_port(_DEFAULT_SERVER_PORT))
args.insecure, manual_cmd_log=server_manual_cmd_log)
if not args.manual_run:
job = dockerjob.DockerJob(spec)
server_jobs[lang] = job
server_addresses[lang] = ('localhost', job.mapped_port(_DEFAULT_SERVER_PORT))
else:
# don't run the server, set server port to a placeholder value
server_addresses[lang] = ('localhost', '${SERVER_PORT}')

if args.http2_badserver_interop:
# launch a HTTP2 server emulator that creates edge cases
lang = str(http2InteropServer)
spec = server_jobspec(http2InteropServer, docker_images.get(lang))
job = dockerjob.DockerJob(spec)
server_jobs[lang] = job
spec = server_jobspec(http2InteropServer, docker_images.get(lang),
manual_cmd_log=server_manual_cmd_log)
if not args.manual_run:
job = dockerjob.DockerJob(spec)
server_jobs[lang] = job
else:
# don't run the server, set server port to a placeholder value
server_addresses[lang] = ('localhost', '${SERVER_PORT}')

jobs = []
if args.cloud_to_prod:
Expand All @@ -918,15 +972,17 @@ def aggregate_http2_results(stdout):
test_job = cloud_to_prod_jobspec(
language, test_case, server_host_name,
prod_servers[server_host_name],
docker_image=docker_images.get(str(language)))
docker_image=docker_images.get(str(language)),
manual_cmd_log=client_manual_cmd_log)
jobs.append(test_job)

if args.http2_interop:
for test_case in _HTTP2_TEST_CASES:
test_job = cloud_to_prod_jobspec(
http2Interop, test_case, server_host_name,
prod_servers[server_host_name],
docker_image=docker_images.get(str(http2Interop)))
docker_image=docker_images.get(str(http2Interop)),
manual_cmd_log=client_manual_cmd_log)
jobs.append(test_job)

if args.cloud_to_prod_auth:
Expand All @@ -939,7 +995,8 @@ def aggregate_http2_results(stdout):
test_job = cloud_to_prod_jobspec(
language, test_case, server_host_name,
prod_servers[server_host_name],
docker_image=docker_images.get(str(language)), auth=True)
docker_image=docker_images.get(str(language)), auth=True,
manual_cmd_log=client_manual_cmd_log)
jobs.append(test_job)

for server in args.override_server:
Expand All @@ -963,7 +1020,8 @@ def aggregate_http2_results(stdout):
server_host,
server_port,
docker_image=docker_images.get(str(language)),
insecure=args.insecure)
insecure=args.insecure,
manual_cmd_log=client_manual_cmd_log)
jobs.append(test_job)

if args.http2_interop:
Expand All @@ -977,7 +1035,8 @@ def aggregate_http2_results(stdout):
server_host,
server_port,
docker_image=docker_images.get(str(http2Interop)),
insecure=args.insecure)
insecure=args.insecure,
manual_cmd_log=client_manual_cmd_log)
jobs.append(test_job)

if args.http2_badserver_interop:
Expand All @@ -988,7 +1047,8 @@ def aggregate_http2_results(stdout):
str(http2InteropServer),
'localhost',
_DEFAULT_SERVER_PORT,
docker_image=docker_images.get(str(language)))
docker_image=docker_images.get(str(language)),
manual_cmd_log=client_manual_cmd_log)
jobs.append(test_job)

if not jobs:
Expand All @@ -997,13 +1057,20 @@ def aggregate_http2_results(stdout):
dockerjob.remove_image(image, skip_nonexistent=True)
sys.exit(1)

if args.manual_run:
print('All tests will skipped --manual_run option is active.')

num_failures, resultset = jobset.run(jobs, newline_on_success=True,
maxjobs=args.jobs)
maxjobs=args.jobs,
skip_jobs=args.manual_run)
if num_failures:
jobset.message('FAILED', 'Some tests failed', do_newline=True)
else:
jobset.message('SUCCESS', 'All tests passed', do_newline=True)

write_cmdlog_maybe(server_manual_cmd_log, 'interop_server_cmds.sh')
write_cmdlog_maybe(client_manual_cmd_log, 'interop_client_cmds.sh')

report_utils.render_junit_xml_report(resultset, 'report.xml')

for name, job in resultset.items():
Expand All @@ -1029,5 +1096,8 @@ def aggregate_http2_results(stdout):
dockerjob.finish_jobs([j for j in server_jobs.itervalues()])

for image in docker_images.itervalues():
print('Removing docker image %s' % image)
dockerjob.remove_image(image)
if not args.manual_run:
print('Removing docker image %s' % image)
dockerjob.remove_image(image)
else:
print('Preserving docker image: %s' % image)

0 comments on commit 7134a31

Please sign in to comment.