diff --git a/configure.py b/configure.py index 47b566a9c0fe7b..9d18cf5e70dad4 100644 --- a/configure.py +++ b/configure.py @@ -492,7 +492,8 @@ def set_tf_cuda_clang(environ_cp): question=question, yes_reply=yes_reply, no_reply=no_reply, - bazel_config_name='cuda_clang') + bazel_config_name='cuda_clang', + ) def set_tf_download_clang(environ_cp): @@ -751,17 +752,21 @@ def get_ndk_api_level(environ_cp, android_ndk_home_path): def valid_api_level(api_level): return os.path.exists( - os.path.join(android_ndk_home_path, 'platforms', - 'android-' + api_level)) + os.path.join(android_ndk_home_path, 'platforms', 'android-' + api_level) + ) android_ndk_api_level = prompt_loop_or_load_from_env( environ_cp, var_name='ANDROID_NDK_API_LEVEL', var_default='26', # 26 is required to support AHardwareBuffer. - ask_for_var=('Please specify the (min) Android NDK API level to use. ' - '[Available levels: %s]') % api_levels, + ask_for_var=( + 'Please specify the (min) Android NDK API level to use. ' + '[Available levels: %s]' + ) + % api_levels, check_success=valid_api_level, - error_msg='Android-%s is not present in the NDK path.') + error_msg='Android-%s is not present in the NDK path.', + ) return android_ndk_api_level @@ -789,6 +794,91 @@ def set_gcc_host_compiler_path(environ_cp): write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH', gcc_host_compiler_path) +def choose_compiler(environ_cp): + question = 'Do you want to use Clang to build TensorFlow?' + yes_reply = 'Clang will be used to compile TensorFlow.' + no_reply = 'GCC will be used to compile TensorFlow.' + var = int( + get_var( + environ_cp, 'TF_NEED_CLANG', None, True, question, yes_reply, no_reply + ) + ) + return var + + +def set_clang_compiler_path(environ_cp): + """Set CLANG_COMPILER_PATH and environment variables. + + Loop over user prompts for clang path until receiving a valid response. + Default is used if no input is given. Set CLANG_COMPILER_PATH and write + environment variables CC and BAZEL_COMPILER to .bazelrc. + + Args: + environ_cp: (Dict) copy of the os.environ. + + Returns: + string value for clang_compiler_path. + """ + # Default path if clang-16 is installed by using apt-get install + default_clang_path = '/usr/lib/llvm-16/bin/clang' + if not os.path.exists(default_clang_path): + default_clang_path = which('clang') or '' + + clang_compiler_path = prompt_loop_or_load_from_env( + environ_cp, + var_name='CLANG_COMPILER_PATH', + var_default=default_clang_path, + ask_for_var='Please specify the path to clang executable.', + check_success=os.path.exists, + resolve_symlinks=True, + error_msg='Invalid clang path. %s cannot be found.', + ) + + write_action_env_to_bazelrc('CLANG_COMPILER_PATH', clang_compiler_path) + write_to_bazelrc('build --repo_env=CC=%s' % clang_compiler_path) + write_to_bazelrc('build --repo_env=BAZEL_COMPILER=%s' % clang_compiler_path) + + return clang_compiler_path + + +def retrieve_clang_version(clang_executable): + """Retrieve installed clang version. + + Args: + clang_executable: (String) path to clang executable + + Returns: + The clang version detected. + """ + stderr = open(os.devnull, 'wb') + curr_version = run_shell([clang_executable, '--version'], + allow_non_zero=True, + stderr=stderr) + + curr_version_split = curr_version.lower().split('clang version ') + if len(curr_version_split) > 1: + curr_version = curr_version_split[1].split()[0] + + curr_version_int = convert_version_to_int(curr_version) + # Check if current clang version can be detected properly. + if not curr_version_int: + print('WARNING: current clang installation is not a release version.\n') + return None + + print('You have Clang %s installed.\n' % curr_version) + return curr_version + + +# Disable clang extension that rejects type definitions within offsetof. +# This was added in clang-16 by https://reviews.llvm.org/D133574. +# Can be removed once upb is updated, since a type definition is used within +# offset of in the current version of ubp. See +# https://github.com/protocolbuffers/upb/blob/9effcbcb27f0a665f9f345030188c0b291e32482/upb/upb.c#L183. +def disable_clang16_offsetof_extension(clang_version): + if int(clang_version.split('.')[0]) == 16: + write_to_bazelrc('build --copt=-Wno-gnu-offsetof-extensions') + + def set_tf_cuda_paths(environ_cp): """Set TF_CUDA_PATHS.""" ask_cuda_paths = ( @@ -949,8 +1039,9 @@ def set_tf_cuda_compute_capabilities(environ_cp): # Set TF_CUDA_COMPUTE_CAPABILITIES environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = tf_cuda_compute_capabilities - write_action_env_to_bazelrc('TF_CUDA_COMPUTE_CAPABILITIES', - tf_cuda_compute_capabilities) + write_action_env_to_bazelrc( + 'TF_CUDA_COMPUTE_CAPABILITIES', tf_cuda_compute_capabilities + ) def set_other_cuda_vars(environ_cp): @@ -1308,9 +1399,13 @@ def main(): set_gcc_host_compiler_path(environ_cp) set_other_cuda_vars(environ_cp) else: - # CUDA not required. Ask whether we should download the clang toolchain and - # use it for the CPU build. - set_tf_download_clang(environ_cp) + # CUDA not required. Ask whether we should use clang for the CPU build. + if is_linux(): + environ_cp['TF_NEED_CLANG'] = str(choose_compiler(environ_cp)) + if environ_cp.get('TF_NEED_CLANG') == '1': + clang_compiler_path = set_clang_compiler_path(environ_cp) + clang_version = retrieve_clang_version(clang_compiler_path) + disable_clang16_offsetof_extension(clang_version) # ROCm / CUDA are mutually exclusive. # At most 1 GPU platform can be configured. diff --git a/tensorflow/tools/ci_build/builds/android.sh b/tensorflow/tools/ci_build/builds/android.sh index 3b49f7a8b45a22..d7c87e9ee62ef3 100755 --- a/tensorflow/tools/ci_build/builds/android.sh +++ b/tensorflow/tools/ci_build/builds/android.sh @@ -16,6 +16,11 @@ set -e +# Install Clang +wget https://apt.llvm.org/llvm.sh +chmod +x llvm.sh +sudo ./llvm.sh 16 + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${SCRIPT_DIR}/builds_common.sh" # To setup Android via `configure` script.