From b51ad2408b5050ee497a3af602d4bbd950f89973 Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Fri, 16 Nov 2018 14:16:23 -0800 Subject: [PATCH 1/4] Add option to disable nccl --- configure.py | 1 + tensorflow/BUILD | 6 +++ tensorflow/core/BUILD | 111 +++++++++++++++++++++------------------- tensorflow/python/BUILD | 1 + tools/bazel.rc | 2 + 5 files changed, 67 insertions(+), 54 deletions(-) diff --git a/configure.py b/configure.py index 234561d94a46f5..f663f6df6f1d26 100644 --- a/configure.py +++ b/configure.py @@ -1694,6 +1694,7 @@ def main(): config_info_line('nohdfs', 'Disable HDFS support.') config_info_line('noignite', 'Disable Apacha Ignite support.') config_info_line('nokafka', 'Disable Apache Kafka support.') + config_info_line('nonccl', 'Disable NVIDIA NCCL support.') if __name__ == '__main__': diff --git a/tensorflow/BUILD b/tensorflow/BUILD index 2dc70c359cabbe..ab704860022b80 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -246,6 +246,12 @@ config_setting( visibility = ["//visibility:public"], ) +config_setting( + name = "no_nccl_support", + define_values = {"no_nccl_support": "true"}, + visibility = ["//visibility:public"], +) + # Crosses between platforms and file system libraries not supported on those # platforms due to limitations in nested select() statements. config_setting( diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 73e8db58a835d2..3237c34a99acdb 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -1349,60 +1349,63 @@ cc_library( name = "all_kernels_statically_linked", visibility = ["//visibility:private"], deps = [ - "//tensorflow/core/kernels:array", - "//tensorflow/core/kernels:audio", - "//tensorflow/core/kernels:batch_kernels", - "//tensorflow/core/kernels:bincount_op", - "//tensorflow/core/kernels:boosted_trees_ops", - "//tensorflow/core/kernels:candidate_sampler_ops", - "//tensorflow/core/kernels:checkpoint_ops", - "//tensorflow/core/kernels:collective_ops", - "//tensorflow/core/kernels:control_flow_ops", - "//tensorflow/core/kernels:ctc_ops", - "//tensorflow/core/kernels:cudnn_rnn_kernels", - "//tensorflow/core/kernels:data_flow", - "//tensorflow/core/kernels:dataset_ops", - "//tensorflow/core/kernels:decode_proto_op", - "//tensorflow/core/kernels:encode_proto_op", - "//tensorflow/core/kernels:fake_quant_ops", - "//tensorflow/core/kernels:function_ops", - "//tensorflow/core/kernels:functional_ops", - "//tensorflow/core/kernels:grappler", - "//tensorflow/core/kernels:histogram_op", - "//tensorflow/core/kernels:image", - "//tensorflow/core/kernels:io", - "//tensorflow/core/kernels:linalg", - "//tensorflow/core/kernels:list_kernels", - "//tensorflow/core/kernels:lookup", - "//tensorflow/core/kernels:logging", - "//tensorflow/core/kernels:manip", - "//tensorflow/core/kernels:math", - "//tensorflow/core/kernels:multinomial_op", - "//tensorflow/core/kernels:nn", - "//tensorflow/core/kernels:parameterized_truncated_normal_op", - "//tensorflow/core/kernels:parsing", - "//tensorflow/core/kernels:partitioned_function_ops", - "//tensorflow/core/kernels:ragged_ops", - "//tensorflow/core/kernels:random_ops", - "//tensorflow/core/kernels:random_poisson_op", - "//tensorflow/core/kernels:remote_fused_graph_ops", - "//tensorflow/core/kernels:required", - "//tensorflow/core/kernels:resource_variable_ops", - "//tensorflow/core/kernels:rpc_op", - "//tensorflow/core/kernels:scoped_allocator_ops", - "//tensorflow/core/kernels:sdca_ops", - "//tensorflow/core/kernels:searchsorted_op", - "//tensorflow/core/kernels:set_kernels", - "//tensorflow/core/kernels:sparse", - "//tensorflow/core/kernels:state", - "//tensorflow/core/kernels:stateless_random_ops", - "//tensorflow/core/kernels:string", - "//tensorflow/core/kernels:summary_kernels", - "//tensorflow/core/kernels:training_ops", - "//tensorflow/core/kernels:word2vec_kernels", - ] + tf_additional_cloud_kernel_deps() + if_not_tx2_llvm_or_windows_cuda([ - "//tensorflow/core/kernels:nccl_kernels", - ]) + if_not_windows([ + "//tensorflow/core/kernels:array", + "//tensorflow/core/kernels:audio", + "//tensorflow/core/kernels:batch_kernels", + "//tensorflow/core/kernels:bincount_op", + "//tensorflow/core/kernels:boosted_trees_ops", + "//tensorflow/core/kernels:candidate_sampler_ops", + "//tensorflow/core/kernels:checkpoint_ops", + "//tensorflow/core/kernels:collective_ops", + "//tensorflow/core/kernels:control_flow_ops", + "//tensorflow/core/kernels:ctc_ops", + "//tensorflow/core/kernels:cudnn_rnn_kernels", + "//tensorflow/core/kernels:data_flow", + "//tensorflow/core/kernels:dataset_ops", + "//tensorflow/core/kernels:decode_proto_op", + "//tensorflow/core/kernels:encode_proto_op", + "//tensorflow/core/kernels:fake_quant_ops", + "//tensorflow/core/kernels:function_ops", + "//tensorflow/core/kernels:functional_ops", + "//tensorflow/core/kernels:grappler", + "//tensorflow/core/kernels:histogram_op", + "//tensorflow/core/kernels:image", + "//tensorflow/core/kernels:io", + "//tensorflow/core/kernels:linalg", + "//tensorflow/core/kernels:list_kernels", + "//tensorflow/core/kernels:lookup", + "//tensorflow/core/kernels:logging", + "//tensorflow/core/kernels:manip", + "//tensorflow/core/kernels:math", + "//tensorflow/core/kernels:multinomial_op", + "//tensorflow/core/kernels:nn", + "//tensorflow/core/kernels:parameterized_truncated_normal_op", + "//tensorflow/core/kernels:parsing", + "//tensorflow/core/kernels:partitioned_function_ops", + "//tensorflow/core/kernels:ragged_ops", + "//tensorflow/core/kernels:random_ops", + "//tensorflow/core/kernels:random_poisson_op", + "//tensorflow/core/kernels:remote_fused_graph_ops", + "//tensorflow/core/kernels:required", + "//tensorflow/core/kernels:resource_variable_ops", + "//tensorflow/core/kernels:rpc_op", + "//tensorflow/core/kernels:scoped_allocator_ops", + "//tensorflow/core/kernels:sdca_ops", + "//tensorflow/core/kernels:searchsorted_op", + "//tensorflow/core/kernels:set_kernels", + "//tensorflow/core/kernels:sparse", + "//tensorflow/core/kernels:state", + "//tensorflow/core/kernels:stateless_random_ops", + "//tensorflow/core/kernels:string", + "//tensorflow/core/kernels:summary_kernels", + "//tensorflow/core/kernels:training_ops", + "//tensorflow/core/kernels:word2vec_kernels", + ] + tf_additional_cloud_kernel_deps() + + select({ + "//tensorflow:no_nccl_support": [], + "//tensorflow:with_cuda_support_windows_override": [], + "//conditions:default": ["//tensorflow/core/kernels:nccl_kernels"], + }) + if_not_windows([ "//tensorflow/core/kernels:fact_op", "//tensorflow/core/kernels:array_not_windows", "//tensorflow/core/kernels:math_not_windows", diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index e55b2a0e9251c9..4d9ed5eea436ec 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -5778,6 +5778,7 @@ cuda_py_test( "no_oss", "noguitar", "notap", + "no_nccl" ], ) diff --git a/tools/bazel.rc b/tools/bazel.rc index 8c2052ee8ad6e6..76da3470dc9599 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -72,6 +72,8 @@ build:nogcp --define=no_gcp_support=true build:nohdfs --define=no_hdfs_support=true build:nokafka --define=no_kafka_support=true build:noignite --define=no_ignite_support=true +build:nonccl --define=no_nccl_support=true +test:nonccl --define=no_nccl_support=true --test_tag_filters=-no_nccl build --define=use_fast_cpp_protos=true build --define=allow_oversize_protos=true From 6ee836b86245a8dc989a7e66bd39a96b2d5b26e4 Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Mon, 19 Nov 2018 09:55:37 -0800 Subject: [PATCH 2/4] Review changes --- tensorflow/python/BUILD | 1 - tools/bazel.rc | 1 - 2 files changed, 2 deletions(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 4d9ed5eea436ec..e55b2a0e9251c9 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -5778,7 +5778,6 @@ cuda_py_test( "no_oss", "noguitar", "notap", - "no_nccl" ], ) diff --git a/tools/bazel.rc b/tools/bazel.rc index 76da3470dc9599..1fdf51f53e29c7 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -73,7 +73,6 @@ build:nohdfs --define=no_hdfs_support=true build:nokafka --define=no_kafka_support=true build:noignite --define=no_ignite_support=true build:nonccl --define=no_nccl_support=true -test:nonccl --define=no_nccl_support=true --test_tag_filters=-no_nccl build --define=use_fast_cpp_protos=true build --define=allow_oversize_protos=true From b64c750d37d6d59d853e0f75e89fdae498dffb03 Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Tue, 20 Nov 2018 14:24:48 -0800 Subject: [PATCH 3/4] Revert buildifier changes and exclude nccl kernels for windows builds --- tensorflow/core/BUILD | 114 +++++++++++++++++++++--------------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 3237c34a99acdb..41d3f6eab8a96b 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -1349,63 +1349,63 @@ cc_library( name = "all_kernels_statically_linked", visibility = ["//visibility:private"], deps = [ - "//tensorflow/core/kernels:array", - "//tensorflow/core/kernels:audio", - "//tensorflow/core/kernels:batch_kernels", - "//tensorflow/core/kernels:bincount_op", - "//tensorflow/core/kernels:boosted_trees_ops", - "//tensorflow/core/kernels:candidate_sampler_ops", - "//tensorflow/core/kernels:checkpoint_ops", - "//tensorflow/core/kernels:collective_ops", - "//tensorflow/core/kernels:control_flow_ops", - "//tensorflow/core/kernels:ctc_ops", - "//tensorflow/core/kernels:cudnn_rnn_kernels", - "//tensorflow/core/kernels:data_flow", - "//tensorflow/core/kernels:dataset_ops", - "//tensorflow/core/kernels:decode_proto_op", - "//tensorflow/core/kernels:encode_proto_op", - "//tensorflow/core/kernels:fake_quant_ops", - "//tensorflow/core/kernels:function_ops", - "//tensorflow/core/kernels:functional_ops", - "//tensorflow/core/kernels:grappler", - "//tensorflow/core/kernels:histogram_op", - "//tensorflow/core/kernels:image", - "//tensorflow/core/kernels:io", - "//tensorflow/core/kernels:linalg", - "//tensorflow/core/kernels:list_kernels", - "//tensorflow/core/kernels:lookup", - "//tensorflow/core/kernels:logging", - "//tensorflow/core/kernels:manip", - "//tensorflow/core/kernels:math", - "//tensorflow/core/kernels:multinomial_op", - "//tensorflow/core/kernels:nn", - "//tensorflow/core/kernels:parameterized_truncated_normal_op", - "//tensorflow/core/kernels:parsing", - "//tensorflow/core/kernels:partitioned_function_ops", - "//tensorflow/core/kernels:ragged_ops", - "//tensorflow/core/kernels:random_ops", - "//tensorflow/core/kernels:random_poisson_op", - "//tensorflow/core/kernels:remote_fused_graph_ops", - "//tensorflow/core/kernels:required", - "//tensorflow/core/kernels:resource_variable_ops", - "//tensorflow/core/kernels:rpc_op", - "//tensorflow/core/kernels:scoped_allocator_ops", - "//tensorflow/core/kernels:sdca_ops", - "//tensorflow/core/kernels:searchsorted_op", - "//tensorflow/core/kernels:set_kernels", - "//tensorflow/core/kernels:sparse", - "//tensorflow/core/kernels:state", - "//tensorflow/core/kernels:stateless_random_ops", - "//tensorflow/core/kernels:string", - "//tensorflow/core/kernels:summary_kernels", - "//tensorflow/core/kernels:training_ops", - "//tensorflow/core/kernels:word2vec_kernels", - ] + tf_additional_cloud_kernel_deps() + - select({ - "//tensorflow:no_nccl_support": [], - "//tensorflow:with_cuda_support_windows_override": [], - "//conditions:default": ["//tensorflow/core/kernels:nccl_kernels"], - }) + if_not_windows([ + "//tensorflow/core/kernels:array", + "//tensorflow/core/kernels:audio", + "//tensorflow/core/kernels:batch_kernels", + "//tensorflow/core/kernels:bincount_op", + "//tensorflow/core/kernels:boosted_trees_ops", + "//tensorflow/core/kernels:candidate_sampler_ops", + "//tensorflow/core/kernels:checkpoint_ops", + "//tensorflow/core/kernels:collective_ops", + "//tensorflow/core/kernels:control_flow_ops", + "//tensorflow/core/kernels:ctc_ops", + "//tensorflow/core/kernels:cudnn_rnn_kernels", + "//tensorflow/core/kernels:data_flow", + "//tensorflow/core/kernels:dataset_ops", + "//tensorflow/core/kernels:decode_proto_op", + "//tensorflow/core/kernels:encode_proto_op", + "//tensorflow/core/kernels:fake_quant_ops", + "//tensorflow/core/kernels:function_ops", + "//tensorflow/core/kernels:functional_ops", + "//tensorflow/core/kernels:grappler", + "//tensorflow/core/kernels:histogram_op", + "//tensorflow/core/kernels:image", + "//tensorflow/core/kernels:io", + "//tensorflow/core/kernels:linalg", + "//tensorflow/core/kernels:list_kernels", + "//tensorflow/core/kernels:lookup", + "//tensorflow/core/kernels:logging", + "//tensorflow/core/kernels:manip", + "//tensorflow/core/kernels:math", + "//tensorflow/core/kernels:multinomial_op", + "//tensorflow/core/kernels:nn", + "//tensorflow/core/kernels:parameterized_truncated_normal_op", + "//tensorflow/core/kernels:parsing", + "//tensorflow/core/kernels:partitioned_function_ops", + "//tensorflow/core/kernels:ragged_ops", + "//tensorflow/core/kernels:random_ops", + "//tensorflow/core/kernels:random_poisson_op", + "//tensorflow/core/kernels:remote_fused_graph_ops", + "//tensorflow/core/kernels:required", + "//tensorflow/core/kernels:resource_variable_ops", + "//tensorflow/core/kernels:rpc_op", + "//tensorflow/core/kernels:scoped_allocator_ops", + "//tensorflow/core/kernels:sdca_ops", + "//tensorflow/core/kernels:searchsorted_op", + "//tensorflow/core/kernels:set_kernels", + "//tensorflow/core/kernels:sparse", + "//tensorflow/core/kernels:state", + "//tensorflow/core/kernels:stateless_random_ops", + "//tensorflow/core/kernels:string", + "//tensorflow/core/kernels:summary_kernels", + "//tensorflow/core/kernels:training_ops", + "//tensorflow/core/kernels:word2vec_kernels", + ] + tf_additional_cloud_kernel_deps() + + select({ + "//tensorflow:no_nccl_support": [], + "//tensorflow:windows": [], + "//conditions:default": ["//tensorflow/core/kernels:nccl_kernels"], + }) + if_not_windows([ "//tensorflow/core/kernels:fact_op", "//tensorflow/core/kernels:array_not_windows", "//tensorflow/core/kernels:math_not_windows", From dd3b98867f9809023ce975bcd3e96dba4021d7bb Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Wed, 21 Nov 2018 13:19:46 -0800 Subject: [PATCH 4/4] Re-apply buildifier --- tensorflow/core/BUILD | 114 +++++++++++++++++++++--------------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 41d3f6eab8a96b..52d3f8f2b98c1e 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -1349,63 +1349,63 @@ cc_library( name = "all_kernels_statically_linked", visibility = ["//visibility:private"], deps = [ - "//tensorflow/core/kernels:array", - "//tensorflow/core/kernels:audio", - "//tensorflow/core/kernels:batch_kernels", - "//tensorflow/core/kernels:bincount_op", - "//tensorflow/core/kernels:boosted_trees_ops", - "//tensorflow/core/kernels:candidate_sampler_ops", - "//tensorflow/core/kernels:checkpoint_ops", - "//tensorflow/core/kernels:collective_ops", - "//tensorflow/core/kernels:control_flow_ops", - "//tensorflow/core/kernels:ctc_ops", - "//tensorflow/core/kernels:cudnn_rnn_kernels", - "//tensorflow/core/kernels:data_flow", - "//tensorflow/core/kernels:dataset_ops", - "//tensorflow/core/kernels:decode_proto_op", - "//tensorflow/core/kernels:encode_proto_op", - "//tensorflow/core/kernels:fake_quant_ops", - "//tensorflow/core/kernels:function_ops", - "//tensorflow/core/kernels:functional_ops", - "//tensorflow/core/kernels:grappler", - "//tensorflow/core/kernels:histogram_op", - "//tensorflow/core/kernels:image", - "//tensorflow/core/kernels:io", - "//tensorflow/core/kernels:linalg", - "//tensorflow/core/kernels:list_kernels", - "//tensorflow/core/kernels:lookup", - "//tensorflow/core/kernels:logging", - "//tensorflow/core/kernels:manip", - "//tensorflow/core/kernels:math", - "//tensorflow/core/kernels:multinomial_op", - "//tensorflow/core/kernels:nn", - "//tensorflow/core/kernels:parameterized_truncated_normal_op", - "//tensorflow/core/kernels:parsing", - "//tensorflow/core/kernels:partitioned_function_ops", - "//tensorflow/core/kernels:ragged_ops", - "//tensorflow/core/kernels:random_ops", - "//tensorflow/core/kernels:random_poisson_op", - "//tensorflow/core/kernels:remote_fused_graph_ops", - "//tensorflow/core/kernels:required", - "//tensorflow/core/kernels:resource_variable_ops", - "//tensorflow/core/kernels:rpc_op", - "//tensorflow/core/kernels:scoped_allocator_ops", - "//tensorflow/core/kernels:sdca_ops", - "//tensorflow/core/kernels:searchsorted_op", - "//tensorflow/core/kernels:set_kernels", - "//tensorflow/core/kernels:sparse", - "//tensorflow/core/kernels:state", - "//tensorflow/core/kernels:stateless_random_ops", - "//tensorflow/core/kernels:string", - "//tensorflow/core/kernels:summary_kernels", - "//tensorflow/core/kernels:training_ops", - "//tensorflow/core/kernels:word2vec_kernels", - ] + tf_additional_cloud_kernel_deps() + - select({ - "//tensorflow:no_nccl_support": [], - "//tensorflow:windows": [], - "//conditions:default": ["//tensorflow/core/kernels:nccl_kernels"], - }) + if_not_windows([ + "//tensorflow/core/kernels:array", + "//tensorflow/core/kernels:audio", + "//tensorflow/core/kernels:batch_kernels", + "//tensorflow/core/kernels:bincount_op", + "//tensorflow/core/kernels:boosted_trees_ops", + "//tensorflow/core/kernels:candidate_sampler_ops", + "//tensorflow/core/kernels:checkpoint_ops", + "//tensorflow/core/kernels:collective_ops", + "//tensorflow/core/kernels:control_flow_ops", + "//tensorflow/core/kernels:ctc_ops", + "//tensorflow/core/kernels:cudnn_rnn_kernels", + "//tensorflow/core/kernels:data_flow", + "//tensorflow/core/kernels:dataset_ops", + "//tensorflow/core/kernels:decode_proto_op", + "//tensorflow/core/kernels:encode_proto_op", + "//tensorflow/core/kernels:fake_quant_ops", + "//tensorflow/core/kernels:function_ops", + "//tensorflow/core/kernels:functional_ops", + "//tensorflow/core/kernels:grappler", + "//tensorflow/core/kernels:histogram_op", + "//tensorflow/core/kernels:image", + "//tensorflow/core/kernels:io", + "//tensorflow/core/kernels:linalg", + "//tensorflow/core/kernels:list_kernels", + "//tensorflow/core/kernels:lookup", + "//tensorflow/core/kernels:logging", + "//tensorflow/core/kernels:manip", + "//tensorflow/core/kernels:math", + "//tensorflow/core/kernels:multinomial_op", + "//tensorflow/core/kernels:nn", + "//tensorflow/core/kernels:parameterized_truncated_normal_op", + "//tensorflow/core/kernels:parsing", + "//tensorflow/core/kernels:partitioned_function_ops", + "//tensorflow/core/kernels:ragged_ops", + "//tensorflow/core/kernels:random_ops", + "//tensorflow/core/kernels:random_poisson_op", + "//tensorflow/core/kernels:remote_fused_graph_ops", + "//tensorflow/core/kernels:required", + "//tensorflow/core/kernels:resource_variable_ops", + "//tensorflow/core/kernels:rpc_op", + "//tensorflow/core/kernels:scoped_allocator_ops", + "//tensorflow/core/kernels:sdca_ops", + "//tensorflow/core/kernels:searchsorted_op", + "//tensorflow/core/kernels:set_kernels", + "//tensorflow/core/kernels:sparse", + "//tensorflow/core/kernels:state", + "//tensorflow/core/kernels:stateless_random_ops", + "//tensorflow/core/kernels:string", + "//tensorflow/core/kernels:summary_kernels", + "//tensorflow/core/kernels:training_ops", + "//tensorflow/core/kernels:word2vec_kernels", + ] + tf_additional_cloud_kernel_deps() + + select({ + "//tensorflow:no_nccl_support": [], + "//tensorflow:windows": [], + "//conditions:default": ["//tensorflow/core/kernels:nccl_kernels"], + }) + if_not_windows([ "//tensorflow/core/kernels:fact_op", "//tensorflow/core/kernels:array_not_windows", "//tensorflow/core/kernels:math_not_windows",