Skip to content

Commit

Permalink
SERVER-48203 add precious and link-type install actions to ninja
Browse files Browse the repository at this point in the history
  • Loading branch information
@dmoody256
dmoody256 authored and Evergreen Agent committed Aug 24, 2022
1 parent 11eb8bd commit 0054d39
Show file tree
Hide file tree
Showing 2 changed files with 189 additions and 64 deletions.
49 changes: 46 additions & 3 deletions SConstruct
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ add_option(
add_option(
'install-action',
choices=([*install_actions.available_actions] + ['default']),
default='default',
default='hardlink',
help=
'select mechanism to use to install files (advanced option to reduce disk IO and utilization)',
nargs=1,
Expand Down Expand Up @@ -1631,8 +1631,6 @@ unknown_vars = env_vars.UnknownVariables()
if unknown_vars:
env.FatalError("Unknown variables specified: {0}", ", ".join(list(unknown_vars.keys())))

if get_option('install-action') != 'default' and get_option('ninja') != "disabled":
env.FatalError("Cannot use non-default install actions when generating Ninja.")
install_actions.setup(env, get_option('install-action'))


Expand Down Expand Up @@ -5336,6 +5334,51 @@ if get_option('ninja') != 'disabled':
return dependencies

env['NINJA_REGENERATE_DEPS'] = ninja_generate_deps
if env.GetOption('install-action') == 'hardlink':
if env.TargetOSIs('windows'):
install_cmd = "cmd.exe /c mklink /h $out $in 1>nul"
else:
install_cmd = "ln $in $out"

elif env.GetOption('install-action') == 'symlink':

# macOS's ln and Windows mklink command do not support relpaths
# out of the box so we will precompute during generation in a
# custom handler.
def symlink_install_action_function(_env, node):
# should only be one output and input for this case
output_file = _env.NinjaGetOutputs(node)[0]
input_file = _env.NinjaGetDependencies(node)[0]
try:
relpath = os.path.relpath(input_file, os.path.dirname(output_file))
except ValueError:
relpath = os.path.abspath(input_file)

return {
"outputs": [output_file],
"rule": "INSTALL",
"inputs": [input_file],
"implicit": _env.NinjaGetDependencies(node),
"variables": {"precious": node.precious, "relpath": relpath},
}

env.NinjaRegisterFunctionHandler("installFunc", symlink_install_action_function)

if env.TargetOSIs('windows'):
install_cmd = "cmd.exe /c mklink $out $relpath 1>nul"
else:
install_cmd = "ln -s $relpath $out"

else:
if env.TargetOSIs('windows'):
# The /b option here will make sure that windows updates the mtime
# when copying the file. This allows to not need to use restat for windows
# copy commands.
install_cmd = "cmd.exe /c copy /b $in $out 1>NUL"
else:
install_cmd = "install $in $out"

env.NinjaRule("INSTALL", install_cmd, description="Installed $out", pool="install_pool")

if env.TargetOSIs("windows"):
# This is a workaround on windows for SERVER-48691 where the line length
Expand Down
204 changes: 143 additions & 61 deletions site_scons/site_tools/ninja.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,24 @@
SCons.Action.CommandGeneratorAction,
)

ninja_compdb_adjust = """\
import json
import sys
compdb = {}
with open(sys.argv[1]) as f:
compdb = json.load(f)
for command in compdb:
if command['output'].endswith('.compdb'):
command['output'] = command['output'][:-(len('.compdb'))]
else:
print(f"compdb entry does not contain '.compdb': {command['output']}")
with open(sys.argv[1], 'w') as f:
json.dump(compdb, f, indent=2)
"""


def _install_action_function(_env, node):
"""Install files using the install or copy commands"""
Expand All @@ -65,6 +83,7 @@ def _install_action_function(_env, node):
"rule": "INSTALL",
"inputs": [get_path(src_file(s)) for s in node.sources],
"implicit": get_dependencies(node),
"variables": {"precious": node.precious},
}


Expand All @@ -83,6 +102,7 @@ def _mkdir_action_function(env, node):
"mkdir {args}".format(
args=' '.join(get_outputs(node)) + " & exit /b 0"
if env["PLATFORM"] == "win32" else "-p " + ' '.join(get_outputs(node)), ),
"variables": {"precious": node.precious},
},
}

Expand All @@ -102,6 +122,7 @@ def _lib_symlink_action_function(_env, node):
"inputs": inputs,
"rule": "SYMLINK",
"implicit": get_dependencies(node),
"variables": {"precious": node.precious},
}


Expand Down Expand Up @@ -312,6 +333,11 @@ def action_to_ninja_build(self, node, action=None):
if callable(node_callback):
node_callback(env, node, build)

if build is not None and node.precious:
if not build.get('variables'):
build['variables'] = {}
build['variables']['precious'] = node.precious

return build

def handle_func_action(self, node, action):
Expand Down Expand Up @@ -414,14 +440,6 @@ def handle_list_action(self, node, action):
"implicit": dependencies,
}

elif results[0]["rule"] == "INSTALL":
return {
"outputs": all_outputs,
"rule": "INSTALL",
"inputs": [get_path(src_file(s)) for s in node.sources],
"implicit": dependencies,
}

raise Exception("Unhandled list action with rule: " + results[0]["rule"])


Expand Down Expand Up @@ -451,8 +469,11 @@ def __init__(self, env, ninja_syntax):
scons_escape = env.get("ESCAPE", lambda x: x)

self.variables = {
# The /b option here will make sure that windows updates the mtime
# when copying the file. This allows to not need to use restat for windows
# copy commands.
"COPY":
"cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp",
"cmd.exe /c 1>NUL copy /b" if sys.platform == "win32" else "cp",
"NOOP":
"cmd.exe /c 1>NUL echo 0" if sys.platform == "win32" else "echo 0 >/dev/null",
"SCONS_INVOCATION":
Expand Down Expand Up @@ -498,31 +519,31 @@ def __init__(self, env, ninja_syntax):
"rspfile": "$out.rsp",
"rspfile_content": "$rspc",
},
"COMPDB_CC": {
"command": "$CC @$out.rsp",
"description": "Compiling $out",
"rspfile": "$out.rsp",
"rspfile_content": "$rspc",
},
"COMPDB_CXX": {
"command": "$CXX @$out.rsp",
"description": "Compiling $out",
"rspfile": "$out.rsp",
"rspfile_content": "$rspc",
},
"LINK": {
"command": "$env$LINK @$out.rsp",
"description": "Linked $out",
"rspfile": "$out.rsp",
"rspfile_content": "$rspc",
"pool": "local_pool",
},
# Ninja does not automatically delete the archive before
# invoking ar. The ar utility will append to an existing archive, which
# can cause duplicate symbols if the symbols moved between object files.
# Native SCons will perform this operation so we need to force ninja
# to do the same. See related for more info:
# https://jira.mongodb.org/browse/SERVER-49457
"AR": {
"command":
"{}$env$AR @$out.rsp".format('' if sys.platform == "win32" else "rm -f $out && "
),
"description":
"Archived $out",
"rspfile":
"$out.rsp",
"rspfile_content":
"$rspc",
"pool":
"local_pool",
"command": "$env$AR @$out.rsp",
"description": "Archived $out",
"rspfile": "$out.rsp",
"rspfile_content": "$rspc",
"pool": "local_pool",
},
"SYMLINK": {
"command": (
Expand All @@ -538,15 +559,6 @@ def __init__(self, env, ninja_syntax):
"command": "$COPY $in $out",
"description": "Installed $out",
"pool": "install_pool",
# On Windows cmd.exe /c copy does not always correctly
# update the timestamp on the output file. This leads
# to a stuck constant timestamp in the Ninja database
# and needless rebuilds.
#
# Adding restat here ensures that Ninja always checks
# the copy updated the timestamp and that Ninja has
# the correct information.
"restat": 1,
},
"TEMPLATE": {
"command": "$SCONS_INVOCATION $out",
Expand Down Expand Up @@ -667,10 +679,34 @@ def generate(self, ninja_file):
for var, val in self.variables.items():
ninja.variable(var, val)

# This is the command that is used to clean a target before building it,
# excluding precious targets.
if sys.platform == "win32":
rm_cmd = f'cmd.exe /c del /q $rm_outs >nul 2>&1 &'
else:
rm_cmd = 'rm -f $rm_outs;'

precious_rule_suffix = "_PRECIOUS"

# Make two sets of rules to honor scons Precious setting. The build nodes themselves
# will then reselect their rule according to the precious being set for that node.
precious_rules = {}
for rule, kwargs in self.rules.items():
if self.env.get('NINJA_MAX_JOBS') is not None and 'pool' not in kwargs:
kwargs['pool'] = 'local_pool'
ninja.rule(rule, **kwargs)
# Do not worry about precious for commands that don't have targets (phony)
# or that will callback to scons (which maintains its own precious).
if rule not in ['phony', 'TEMPLATE', 'REGENERATE', 'COMPDB_CC', 'COMPDB_CXX']:
precious_rule = rule + precious_rule_suffix
precious_rules[precious_rule] = kwargs.copy()
ninja.rule(precious_rule, **precious_rules[precious_rule])

kwargs['command'] = f"{rm_cmd} " + kwargs['command']
ninja.rule(rule, **kwargs)
else:

ninja.rule(rule, **kwargs)
self.rules.update(precious_rules)

# If the user supplied an alias to determine generated sources, use that, otherwise
# determine what the generated sources are dynamically.
Expand Down Expand Up @@ -722,6 +758,45 @@ def check_generated_source_deps(build):

template_builders = []

# If we ever change the name/s of the rules that include
# compile commands (i.e. something like CC) we will need to
# update this build to reflect that complete list.
compile_commands = "compile_commands.json"
compdb_expand = '-x ' if self.env.get('NINJA_COMPDB_EXPAND') else ''
adjust_script_out = os.path.join(
get_path(self.env['NINJA_BUILDDIR']), 'ninja_compdb_adjust.py')
os.makedirs(os.path.dirname(adjust_script_out), exist_ok=True)
with open(adjust_script_out, 'w') as f:
f.write(ninja_compdb_adjust)
self.builds[compile_commands] = {
'rule': "CMD",
'outputs': [compile_commands],
'pool': "console",
'implicit': [ninja_file],
'variables': {
"cmd":
f"ninja -f {ninja_file} -t compdb {compdb_expand}COMPDB_CC COMPDB_CXX > {compile_commands};"
+ f"{sys.executable} {adjust_script_out} {compile_commands}"
},
}
self.builds["compiledb"] = {
'rule': "phony",
"outputs": ["compiledb"],
'implicit': [compile_commands],
}

# Now for all build nodes, we want to select the precious rule or not.
# If it's not precious, we need to save all the outputs into a variable
# on that node. Later we will be removing outputs and switching them to
# phonies so that we can generate response and depfiles correctly.
for build, kwargs in self.builds.items():
if kwargs.get('variables') and kwargs['variables'].get('precious'):
kwargs['rule'] = kwargs['rule'] + precious_rule_suffix
elif kwargs['rule'] not in ['phony', 'TEMPLATE', 'REGENERATE']:
if not kwargs.get('variables'):
kwargs['variables'] = {}
kwargs['variables']['rm_outs'] = kwargs['outputs'].copy()

for build in [self.builds[key] for key in sorted(self.builds.keys())]:
if build["rule"] == "TEMPLATE":
template_builders.append(build)
Expand Down Expand Up @@ -813,6 +888,20 @@ def check_generated_source_deps(build):

ninja_sorted_build(ninja, **build)

for build, kwargs in self.builds.items():
if kwargs['rule'] in [
'CC', f'CC{precious_rule_suffix}', 'CXX', f'CXX{precious_rule_suffix}'
]:
rule = kwargs['rule'].replace(
precious_rule_suffix
) if precious_rule_suffix in kwargs['rule'] else kwargs['rule']
rule = "COMPDB_" + rule
compdb_build = kwargs.copy()

compdb_build['rule'] = rule
compdb_build['outputs'] = [kwargs['outputs'] + ".compdb"]
ninja.build(**compdb_build)

template_builds = {'rule': "TEMPLATE"}
for template_builder in template_builders:

Expand Down Expand Up @@ -876,30 +965,6 @@ def check_generated_source_deps(build):
implicit=[__file__],
)

# If we ever change the name/s of the rules that include
# compile commands (i.e. something like CC) we will need to
# update this build to reflect that complete list.
ninja_sorted_build(
ninja,
outputs="compile_commands.json",
rule="CMD",
pool="console",
implicit=[ninja_file],
variables={
"cmd":
"ninja -f {} -t compdb {}CC CXX > compile_commands.json".format(
ninja_file, '-x ' if self.env.get('NINJA_COMPDB_EXPAND') else '')
},
order_only=[generated_sources_alias],
)

ninja_sorted_build(
ninja,
outputs="compiledb",
rule="phony",
implicit=["compile_commands.json"],
)

# Look in SCons's list of DEFAULT_TARGETS, find the ones that
# we generated a ninja build rule for.
scons_default_targets = [
Expand Down Expand Up @@ -1297,7 +1362,8 @@ def register_custom_rule_mapping(env, pre_subst_string, rule):


def register_custom_rule(env, rule, command, description="", deps=None, pool=None,
use_depfile=False, use_response_file=False, response_file_content="$rspc"):
use_depfile=False, use_response_file=False, response_file_content="$rspc",
restat=False):
"""Allows specification of Ninja rules from inside SCons files."""
rule_obj = {
"command": command,
Expand All @@ -1315,8 +1381,15 @@ def register_custom_rule(env, rule, command, description="", deps=None, pool=Non

if use_response_file:
rule_obj["rspfile"] = "$out.rsp"
if rule_obj["rspfile"] not in command:
raise Exception(
f'Bad Ninja Custom Rule: response file requested, but {rule_obj["rspfile"]} not in in command: {command}'
)
rule_obj["rspfile_content"] = response_file_content

if restat:
rule_obj["restat"] = 1

env[NINJA_RULES][rule] = rule_obj


Expand Down Expand Up @@ -1527,6 +1600,15 @@ def ninja_generate_deps(env):
env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider")
env.AddMethod(set_build_node_callback, "NinjaSetBuildNodeCallback")

# Expose ninja node path converstion functions to make writing
# custom function action handlers easier.
env.AddMethod(lambda _env, node: get_outputs(node), "NinjaGetOutputs")
env.AddMethod(lambda _env, node, skip_unknown_types=False: get_inputs(node, skip_unknown_types),
"NinjaGetInputs")
env.AddMethod(lambda _env, node, skip_sources=False: get_dependencies(node),
"NinjaGetDependencies")
env.AddMethod(lambda _env, node: get_order_only(node), "NinjaGetOrderOnly")

# Provides a way for users to handle custom FunctionActions they
# want to translate to Ninja.
env[NINJA_CUSTOM_HANDLERS] = {}
Expand Down

0 comments on commit 0054d39

Please sign in to comment.