Skip to content

Commit

Permalink
SERVER-50476 created gdb_index scons tool to generate gdb_index
Browse files Browse the repository at this point in the history
  • Loading branch information
dmoody256 authored and Evergreen Agent committed Jan 17, 2023
1 parent e7cf956 commit 15e9bc6
Show file tree
Hide file tree
Showing 7 changed files with 313 additions and 40 deletions.
42 changes: 27 additions & 15 deletions SConstruct
Original file line number Diff line number Diff line change
Expand Up @@ -856,13 +856,13 @@ def variable_arch_converter(val):
return val


def split_dwarf_converter(val):
def bool_var_converter(val, var):
try:
return to_boolean(val)
except ValueError as exc:
if val.lower() != "auto":
raise ValueError(
f'Invalid SPLIT_DWARF value {s}, must be a boolean-like string or "auto"') from exc
f'Invalid {var} value {s}, must be a boolean-like string or "auto"') from exc
return "auto"


Expand Down Expand Up @@ -1390,7 +1390,20 @@ env_vars.Add(
'SPLIT_DWARF',
help=
'Set the boolean (auto, on/off true/false 1/0) to enable gsplit-dwarf (non-Windows). Incompatible with DWARF_VERSION=5',
converter=split_dwarf_converter,
converter=functools.partial(bool_var_converter, var='SPLIT_DWARF'),
default="auto",
)

env_vars.Add(
'GDB',
help="Configures the path to the 'gdb' debugger binary.",
)

env_vars.Add(
'GDB_INDEX',
help=
'Set the boolean (auto, on/off true/false 1/0) to enable creation of a gdb_index in binaries.',
converter=functools.partial(bool_var_converter, var='GDB_INDEX'),
default="auto",
)

Expand Down Expand Up @@ -4343,10 +4356,6 @@ def doConfigure(myenv):
myenv.AddToCCFLAGSIfSupported('-fno-limit-debug-info')

if myenv.ToolchainIs('gcc', 'clang'):
# Usually, --gdb-index is too expensive in big static binaries, but for dynamic
# builds it works well.
if link_model.startswith("dynamic"):
myenv.AddToLINKFLAGSIfSupported('-Wl,--gdb-index')

# Pass -gdwarf{32,64} if an explicit value was selected
# or defaulted. Fail the build if we can't honor the
Expand Down Expand Up @@ -5717,13 +5726,18 @@ if get_option('ninja') != 'disabled':

env['NINJA_GENERATED_SOURCE_ALIAS_NAME'] = 'generated-sources'

if get_option('separate-debug') == "on" or env.TargetOSIs("windows"):
gdb_index = env.get('GDB_INDEX')
if gdb_index == 'auto' and link_model == 'dynamic':
gdb_index = True

# The current ninja builder can't handle --separate-debug on non-Windows platforms
# like linux or macOS, because they depend on adding extra actions to the link step,
# which cannot be translated into the ninja bulider.
if not env.TargetOSIs("windows") and get_option('ninja') != 'disabled':
env.FatalError("Cannot use --separate-debug with Ninja on non-Windows platforms.")
if gdb_index == True:
gdb_index = Tool('gdb_index')
if gdb_index.exists(env):
gdb_index.generate(env)
elif env.get('GDB_INDEX') != 'auto':
env.FatalError('Could not enable explicit request for gdb index generation.')

if get_option('separate-debug') == "on" or env.TargetOSIs("windows"):

separate_debug = Tool('separate_debug')
if not separate_debug.exists(env):
Expand All @@ -5750,8 +5764,6 @@ if env['SPLIT_DWARF']:
env.FatalError(
'Running split dwarf outside of DWARF4 has shown compilation issues when using DWARF5 and gdb index. Disabling this functionality for now. Use SPLIT_DWARF=0 to disable building with split dwarf or use DWARF_VERSION=4 to pin to DWARF version 4.'
)
if env.ToolchainIs('gcc', 'clang'):
env.AddToLINKFLAGSIfSupported('-Wl,--gdb-index')
env.Tool('split_dwarf')

env["AUTO_ARCHIVE_TARBALL_SUFFIX"] = "tgz"
Expand Down
83 changes: 83 additions & 0 deletions buildscripts/setup_spawnhost_coredump
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,89 @@ fi' >> .bash_profile
# harmless. gdb expects the .debug files to live adjacent to the physical binary.
find bin -type f -perm -o=x -exec ln -s {} . \;

# This script checks a bin for the dwarf version and then generates an index if the bin does not already have an index. eu-readelf is used in place of readelf as it is much faster.
cat > add_index.sh <<EOF
#!/bin/bash
set -o pipefail
target_dir="\$(dirname \${1})"
dwarf_version="\$($TOOLCHAIN_ROOT/bin/eu-readelf --debug-dump=info \$1 | grep --line-buffered -E '^\s+Version:' | head -1 | awk -F, '{print(\$1)}' | awk '{print(\$2)}')"
if [[ \$dwarf_version == 5 ]]
then
$TOOLCHAIN_ROOT/bin/gdb --batch-silent --quiet --nx --eval-command "save gdb-index -dwarf-5 \$target_dir" \$1
if [ -f \${1}.debug_names ]
then
$TOOLCHAIN_ROOT/bin/objcopy --dump-section .debug_str=\${1}.debug_str.new \$1
cat \${1}.debug_str >>\${1}.debug_str.new
$TOOLCHAIN_ROOT/bin/objcopy --add-section .debug_names=\${1}.debug_names --set-section-flags .debug_names=readonly --update-section .debug_str=\${1}.debug_str.new \${1} \${1}
rm -f \${1}.debug_names \${1}.debug_str.new \${1}.debug_str
fi
elif [[ \$dwarf_version == 4 ]]
then
$TOOLCHAIN_ROOT/bin/gdb --batch-silent --quiet --nx --eval-command "save gdb-index \$target_dir" \$1
if [ -f \${1}.gdb-index ]
then
$TOOLCHAIN_ROOT/bin/objcopy --add-section .gdb_index=\${1}.gdb-index --set-section-flags .gdb_index=readonly \${1} \${1}
rm -f \${1}.gdb-index
fi
else
echo "Can't determine debug info for \$1"
fi
EOF

# After creating the index file in a separate debug file, the debuglink CRC
# is no longer value, this will simply recreate the debuglink and therefore
# update the CRC to match.
cat > recalc_debuglink.sh <<EOF
#!/bin/bash
set -o pipefail
debuglink="\$($TOOLCHAIN_ROOT/bin/eu-readelf -S \$1 | grep '.gnu_debuglink')"
if [ ! -z "\$debuglink" ]
then
$TOOLCHAIN_ROOT/bin/objcopy --remove-section ".gnu_debuglink" \$1
$TOOLCHAIN_ROOT/bin/objcopy --add-gnu-debuglink "$(basename \$1).debug" \$1
fi
EOF

# this script creates a symlink in the toolchain lib/debug directory which is in the
# build-id format. This allows gdb to load the separate debug file and skip CRC
# checking.
cat > create_build_id_links.sh <<EOF
#!/bin/bash
set -o pipefail
build_id="\$($TOOLCHAIN_ROOT/bin/eu-readelf -n \$1 | grep 'Build ID:' | awk -F: '{print \$2}' | sed 's/ *//')"
gdb_debug_dir="\$(readlink $TOOLCHAIN_ROOT/bin/gdb)"
gdb_debug_dir="\$(dirname \$gdb_debug_dir)"
gdb_debug_dir="\$(dirname \$gdb_debug_dir)/lib/debug/.build-id/\${build_id:0:2}"
gdb_debug_file="\${build_id:2}.debug"
mkdir -p \$gdb_debug_dir
ln -s \$PWD/\$1 \$gdb_debug_dir/\$gdb_debug_file
EOF

chmod +x ./add_index.sh
chmod +x ./recalc_debuglink.sh
chmod +x ./create_build_id_links.sh
cpus=$(getconf _NPROCESSORS_ONLN)

# notice we don't search lib directory as we assume dynamic builds build the index during
# the build.
find bin -type f -perm -o=x | xargs --max-args=1 --max-procs=$cpus ./add_index.sh
find bin -type f -perm -o=x | xargs --max-args=1 --max-procs=$cpus ./recalc_debuglink.sh

# This script constructs symblinks based off the build-id so GDB can skip the crc check
# normally performed during .gnu_debuglink loading.
find bin lib -name "*.debug" -type f -perm -o=x | xargs --max-args=1 --max-procs=$cpus ./create_build_id_links.sh

# Boost-Pretty-Printer supports auto-detection for the boost version but relies on the system
# installed version of boost. To avoid this behavior we explicitly specify the boost_version.
# Moreover, the most recent version of boost that Boost-Pretty-Printer verifies it supports is
Expand Down
1 change: 1 addition & 0 deletions etc/scons/mongodbtoolchain_v4_clang.vars
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ CC = os.path.join(toolchain_bindir, 'clang')
CXX = os.path.join(toolchain_bindir, 'clang++')
DWP = os.path.join(toolchain_bindir, 'dwp')
READELF = os.path.join(toolchain_bindir, 'readelf')
GDB = os.path.join(toolchain_bindir, 'gdb')

try:
AR = subprocess.check_output([CXX, '-print-prog-name=ar']).decode('utf-8').strip()
Expand Down
1 change: 1 addition & 0 deletions etc/scons/mongodbtoolchain_v4_gcc.vars
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ CC = os.path.join(toolchain_bindir, 'gcc')
CXX = os.path.join(toolchain_bindir, 'g++')
DWP = os.path.join(toolchain_bindir, 'dwp')
READELF = os.path.join(toolchain_bindir, 'readelf')
GDB = os.path.join(toolchain_bindir, 'gdb')

try:
AR = subprocess.check_output([CXX, '-print-prog-name=ar']).decode('utf-8').strip()
Expand Down
134 changes: 134 additions & 0 deletions site_scons/site_tools/gdb_index.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
# Copyright 2020 MongoDB Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#

import SCons


def _update_builder(env, builder):

verbose = '' if env.Verbose() else None

base_action = builder.action
if not isinstance(base_action, SCons.Action.ListAction):
base_action = SCons.Action.ListAction([base_action])

# There are cases were a gdb-index file is NOT generated from gdb 'save gdb-index' command,
# mostly shim libraries where there is no code in the library, and the following Actions would
# then fail. The files are created make sure there is always a file to operate on, if its an
# empty file then the following actions are basically NOOP, and its cleaner then writing
# conditions into each action.

# Because this is all taking under one task, the list action will always run all actions if
# the target is out of date. So the gdb-index files would always be regenerated, and there is
# no value in keeping them around, it will just waste disk space. Therefore they should be
# removed as if they never existed from the task. The build system doesn't need to know about
# them.
if env.get('DWARF_VERSION') <= 4:
base_action.list.extend([
SCons.Action.Action(
'touch ${TARGET}.gdb-index',
verbose,
),
SCons.Action.Action(
'$GDB --batch-silent --quiet --nx --eval-command "save gdb-index ${TARGET.dir}" $TARGET',
"$GDB_INDEX_GEN_INDEX_STR",
),
SCons.Action.Action(
'$OBJCOPY --add-section .gdb_index=${TARGET}.gdb-index --set-section-flags .gdb_index=readonly ${TARGET} ${TARGET}',
"$GDB_INDEX_ADD_SECTION_STR",
),
SCons.Action.Action(
'rm -f ${TARGET}.gdb-index',
verbose,
),
])
else:
base_action.list.extend([
SCons.Action.Action(
'touch ${TARGET}.debug_names ${TARGET}.debug_str',
verbose,
),
SCons.Action.Action(
'$GDB --batch-silent --quiet --nx --eval-command "save gdb-index -dwarf-5 ${TARGET.dir}" $TARGET',
"$GDB_INDEX_GEN_INDEX_STR",
),
SCons.Action.Action(
'$OBJCOPY --dump-section .debug_str=${TARGET}.debug_str.new $TARGET',
verbose,
),
SCons.Action.Action(
'cat ${TARGET}.debug_str >>${TARGET}.debug_str.new',
verbose,
),
SCons.Action.Action(
'$OBJCOPY --add-section .debug_names=${TARGET}.debug_names --set-section-flags .debug_names=readonly --update-section .debug_str=${TARGET}.debug_str.new ${TARGET} ${TARGET}',
"$GDB_INDEX_ADD_SECTION_STR",
),
SCons.Action.Action(
'rm -f ${TARGET}.debug_names ${TARGET}.debug_str.new ${TARGET}.debug_str',
verbose,
),
])

builder.action = base_action


def generate(env):
if env.get("OBJCOPY", None) is None:
env["OBJCOPY"] = env.WhereIs("objcopy")
if env.get("GDB", None) is None:
env["GDB"] = env.WhereIs("gdb")

if not env.Verbose():
env.Append(
GDB_INDEX_GEN_INDEX_STR="Using $GDB to generate index for $TARGET",
GDB_INDEX_ADD_SECTION_STR="Adding index sections into $TARGET",
)

for builder in ["Program", "SharedLibrary", "LoadableModule"]:
_update_builder(env, env["BUILDERS"][builder])


def exists(env):
result = False
if env.TargetOSIs("posix"):
objcopy = env.get("OBJCOPY", None) or env.WhereIs("objcopy")
gdb = env.get("GDB", None) or env.WhereIs("gdb")
try:
dwarf_version = int(env.get('DWARF_VERSION'))
except ValueError:
dwarf_version = None

unset_vars = []
if not objcopy:
unset_vars += ['OBJCOPY']
if not gdb:
unset_vars += ['GDB']
if not dwarf_version:
unset_vars += ['DWARF_VERSION']

if not unset_vars:
print("Enabled generation of gdb index into binaries.")
result = True
else:
print(f"Disabled generation gdb index because {', '.join(unset_vars)} were not set.")
return result
Loading

0 comments on commit 15e9bc6

Please sign in to comment.