Revert D20783179: [pytorch][PR] Bazel build of pytorch

Test Plan: revert-hammer Differential Revision: D20783179 Original commit changeset: b160908a3e10 fbshipit-source-id: 5b7b36305525e7ccc49540b48991149cf0a759f4
huaxz1986 · Apr 4, 2020 · 6be9c77 · 6be9c77
1 parent fced0c9
commit 6be9c77
Show file tree

Hide file tree

Showing 31 changed files with 53 additions and 4,309 deletions.
diff --git a/.bazelrc b/.bazelrc
diff --git a/.bazelversion b/.bazelversion
diff --git a/.circleci/cimodel/data/pytorch_build_definitions.py b/.circleci/cimodel/data/pytorch_build_definitions.py
@@ -187,9 +187,9 @@ def instantiate_configs():
 
     root = get_root()
     found_configs = conf_tree.dfs(root)
+    restrict_phases = None
     for fc in found_configs:
 
-        restrict_phases = None
         distro_name = fc.find_prop("distro_name")
         compiler_name = fc.find_prop("compiler_name")
         compiler_version = fc.find_prop("compiler_version")

diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1557,77 +1557,6 @@ jobs:
             cd ${PROJ_ROOT}/ios/TestApp
             instruments -s -devices
             fastlane scan
-  pytorch_linux_bazel_build:
-    <<: *pytorch_params
-    machine:
-      image: ubuntu-1604:201903-01
-    steps:
-    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - attach_scripts
-    - setup_linux_system_environment
-    - checkout
-    - setup_ci_environment
-    - run:
-        name: Bazel Build
-        no_output_timeout: "1h"
-        command: |
-          set -e
-          # Pull Docker image and run build
-          echo "DOCKER_IMAGE: "${DOCKER_IMAGE}
-          time docker pull ${DOCKER_IMAGE} >/dev/null
-          export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
-
-          echo "Do NOT merge master branch into $CIRCLE_BRANCH in environment $BUILD_ENVIRONMENT"
-
-          git submodule sync && git submodule update -q --init --recursive
-
-          docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
-
-          export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/build.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
-
-          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
-
-          # Push intermediate Docker image for next phase to use
-          if [ -z "${BUILD_ONLY}" ]; then
-            # Augment our output image name with bazel to avoid collisions
-            output_image=${DOCKER_IMAGE}-bazel-${CIRCLE_SHA1}
-            export COMMIT_DOCKER_IMAGE=$output_image
-            docker commit "$id" ${COMMIT_DOCKER_IMAGE}
-            time docker push ${COMMIT_DOCKER_IMAGE}
-          fi
-
-  pytorch_linux_bazel_test:
-    <<: *pytorch_params
-    machine:
-      image: ubuntu-1604:201903-01
-    steps:
-    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - attach_scripts
-    - setup_linux_system_environment
-    - setup_ci_environment
-    - run:
-        name: Test
-        no_output_timeout: "90m"
-        command: |
-          set -e
-          output_image=${DOCKER_IMAGE}-bazel-${CIRCLE_SHA1}
-          export COMMIT_DOCKER_IMAGE=$output_image
-          echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
-
-          time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
-
-          if [ -n "${USE_CUDA_DOCKER_RUNTIME}" ]; then
-            export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
-          else
-            export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
-          fi
-
-          if [[ ${BUILD_ENVIRONMENT} == *"multigpu"* ]]; then
-            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/multigpu-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
-          else
-            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export CIRCLE_PULL_REQUEST=${CIRCLE_PULL_REQUEST}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
-          fi
-          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
   promote_s3:
     <<: *promote_common
@@ -2476,20 +2405,6 @@ workflows:
           docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7:f990c76a-a798-42bb-852f-5be5006f8026"
           use_cuda_docker_runtime: "1"
           resource_class: gpu.medium
-      - pytorch_linux_bazel_build:
-          name: pytorch_bazel_build
-          requires:
-            - setup
-          build_environment: "pytorch-linux-xenial-py3.6-gcc7-bazel-build"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7:f990c76a-a798-42bb-852f-5be5006f8026"
-          resource_class: large
-      - pytorch_linux_bazel_test:
-          name: pytorch_bazel_test
-          requires:
-            - setup
-            - pytorch_bazel_build
-          build_environment: "pytorch-linux-xenial-py3.6-gcc7-bazel-test"
-          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7:f990c76a-a798-42bb-852f-5be5006f8026"
       - caffe2_linux_build:
           name: caffe2_onnx_main_py3_6_clang7_ubuntu16_04_build
           requires:

diff --git a/.circleci/generate_config_yml.py b/.circleci/generate_config_yml.py
@@ -101,7 +101,6 @@ def write(self, output_filehandle):
     File("workflows-pytorch-ios-builds.yml"),
     File("workflows-pytorch-mobile-builds.yml"),
     File("workflows-pytorch-ge-config-tests.yml"),
-    File("workflows-pytorch-bazel-builds.yml"),
     Listgen(caffe2_build_definitions.get_workflow_jobs, 3),
     File("workflows-binary-builds-smoke-subset.yml"),
     Listgen(binary_build_definitions.get_binary_smoke_test_jobs, 3),

diff --git a/.circleci/verbatim-sources/job-specs-custom.yml b/.circleci/verbatim-sources/job-specs-custom.yml
@@ -440,74 +440,3 @@
             cd ${PROJ_ROOT}/ios/TestApp
             instruments -s -devices
             fastlane scan
-  pytorch_linux_bazel_build:
-    <<: *pytorch_params
-    machine:
-      image: ubuntu-1604:201903-01
-    steps:
-    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - attach_scripts
-    - setup_linux_system_environment
-    - checkout
-    - setup_ci_environment
-    - run:
-        name: Bazel Build
-        no_output_timeout: "1h"
-        command: |
-          set -e
-          # Pull Docker image and run build
-          echo "DOCKER_IMAGE: "${DOCKER_IMAGE}
-          time docker pull ${DOCKER_IMAGE} >/dev/null
-          export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
-
-          echo "Do NOT merge master branch into $CIRCLE_BRANCH in environment $BUILD_ENVIRONMENT"
-
-          git submodule sync && git submodule update -q --init --recursive
-
-          docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
-
-          export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/build.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
-
-          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
-
-          # Push intermediate Docker image for next phase to use
-          if [ -z "${BUILD_ONLY}" ]; then
-            # Augment our output image name with bazel to avoid collisions
-            output_image=${DOCKER_IMAGE}-bazel-${CIRCLE_SHA1}
-            export COMMIT_DOCKER_IMAGE=$output_image
-            docker commit "$id" ${COMMIT_DOCKER_IMAGE}
-            time docker push ${COMMIT_DOCKER_IMAGE}
-          fi
-
-  pytorch_linux_bazel_test:
-    <<: *pytorch_params
-    machine:
-      image: ubuntu-1604:201903-01
-    steps:
-    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - attach_scripts
-    - setup_linux_system_environment
-    - setup_ci_environment
-    - run:
-        name: Test
-        no_output_timeout: "90m"
-        command: |
-          set -e
-          output_image=${DOCKER_IMAGE}-bazel-${CIRCLE_SHA1}
-          export COMMIT_DOCKER_IMAGE=$output_image
-          echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
-
-          time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
-
-          if [ -n "${USE_CUDA_DOCKER_RUNTIME}" ]; then
-            export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
-          else
-            export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
-          fi
-
-          if [[ ${BUILD_ENVIRONMENT} == *"multigpu"* ]]; then
-            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/multigpu-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
-          else
-            export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export CIRCLE_PULL_REQUEST=${CIRCLE_PULL_REQUEST}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
-          fi
-          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
diff --git a/.circleci/verbatim-sources/workflows-pytorch-bazel-builds.yml b/.circleci/verbatim-sources/workflows-pytorch-bazel-builds.yml
diff --git a/.gitignore b/.gitignore
@@ -255,6 +255,3 @@ TAGS
 
 # clangd background index
 .clangd/
-
-# bazel symlinks
-bazel-*
diff --git a/.jenkins/pytorch/build.sh b/.jenkins/pytorch/build.sh
@@ -176,72 +176,65 @@ if [[ "${BUILD_ENVIRONMENT}" == *clang* ]]; then
   export CXX=clang++
 fi
 
-if [[ "$BUILD_ENVIRONMENT" == *-bazel-* ]]; then
-  set -e
 
-  get_bazel
+# check that setup.py would fail with bad arguments
+echo "The next three invocations are expected to fail with invalid command error messages."
+( ! get_exit_code python setup.py bad_argument )
+( ! get_exit_code python setup.py clean] )
+( ! get_exit_code python setup.py clean bad_argument )
 
-  tools/bazel build :torch
-else
-  # check that setup.py would fail with bad arguments
-  echo "The next three invocations are expected to fail with invalid command error messages."
-  ( ! get_exit_code python setup.py bad_argument )
-  ( ! get_exit_code python setup.py clean] )
-  ( ! get_exit_code python setup.py clean bad_argument )
-
-  if [[ "$BUILD_ENVIRONMENT" != *libtorch* ]]; then
-
-    # ppc64le build fails when WERROR=1
-    # set only when building other architectures
-    # only use for "python setup.py install" line
-    if [[ "$BUILD_ENVIRONMENT" != *ppc64le*  && "$BUILD_ENVIRONMENT" != *clang* ]]; then
-      WERROR=1 python setup.py install
-    else
-      python setup.py install
-    fi
+if [[ "$BUILD_ENVIRONMENT" != *libtorch* ]]; then
 
-    # TODO: I'm not sure why, but somehow we lose verbose commands
-    set -x
+  # ppc64le build fails when WERROR=1
+  # set only when building other architectures
+  # only use for "python setup.py install" line
+  if [[ "$BUILD_ENVIRONMENT" != *ppc64le*  && "$BUILD_ENVIRONMENT" != *clang* ]]; then
+    WERROR=1 python setup.py install
+  else
+    python setup.py install
+  fi
 
-    if which sccache > /dev/null; then
-      echo 'PyTorch Build Statistics'
-      sccache --show-stats
-    fi
+  # TODO: I'm not sure why, but somehow we lose verbose commands
+  set -x
 
-    assert_git_not_dirty
+  if which sccache > /dev/null; then
+    echo 'PyTorch Build Statistics'
+    sccache --show-stats
+  fi
 
-    # Build custom operator tests.
-    CUSTOM_OP_BUILD="$PWD/../custom-op-build"
-    CUSTOM_OP_TEST="$PWD/test/custom_operator"
-    python --version
-    SITE_PACKAGES="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')"
-    mkdir "$CUSTOM_OP_BUILD"
-    pushd "$CUSTOM_OP_BUILD"
-    cmake "$CUSTOM_OP_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" -DPYTHON_EXECUTABLE="$(which python)"
-    make VERBOSE=1
-    popd
-    assert_git_not_dirty
-  else
-    # Test standalone c10 build
-    if [[ "$BUILD_ENVIRONMENT" == *xenial-cuda10.1-cudnn7-py3* ]]; then
-      mkdir -p c10/build
-      pushd c10/build
-      cmake ..
-      make -j
-      popd
-      assert_git_not_dirty
-    fi
+  assert_git_not_dirty
 
-    # Test no-Python build
-    echo "Building libtorch"
-    # NB: Install outside of source directory (at the same level as the root
-    # pytorch folder) so that it doesn't get cleaned away prior to docker push.
-    BUILD_LIBTORCH_PY=$PWD/tools/build_libtorch.py
-    mkdir -p ../cpp-build/caffe2
-    pushd ../cpp-build/caffe2
-    WERROR=1 VERBOSE=1 DEBUG=1 python $BUILD_LIBTORCH_PY
+  # Build custom operator tests.
+  CUSTOM_OP_BUILD="$PWD/../custom-op-build"
+  CUSTOM_OP_TEST="$PWD/test/custom_operator"
+  python --version
+  SITE_PACKAGES="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')"
+  mkdir "$CUSTOM_OP_BUILD"
+  pushd "$CUSTOM_OP_BUILD"
+  cmake "$CUSTOM_OP_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" -DPYTHON_EXECUTABLE="$(which python)"
+  make VERBOSE=1
+  popd
+  assert_git_not_dirty
+else
+  # Test standalone c10 build
+  if [[ "$BUILD_ENVIRONMENT" == *xenial-cuda10.1-cudnn7-py3* ]]; then
+    mkdir -p c10/build
+    pushd c10/build
+    cmake ..
+    make -j
     popd
+    assert_git_not_dirty
   fi
+
+  # Test no-Python build
+  echo "Building libtorch"
+  # NB: Install outside of source directory (at the same level as the root
+  # pytorch folder) so that it doesn't get cleaned away prior to docker push.
+  BUILD_LIBTORCH_PY=$PWD/tools/build_libtorch.py
+  mkdir -p ../cpp-build/caffe2
+  pushd ../cpp-build/caffe2
+  WERROR=1 VERBOSE=1 DEBUG=1 python $BUILD_LIBTORCH_PY
+  popd
 fi
 
 # Test XLA build

diff --git a/.jenkins/pytorch/common.sh b/.jenkins/pytorch/common.sh
@@ -187,12 +187,3 @@ function file_diff_from_base() {
   set -e
   git diff --name-only "$(git merge-base origin master HEAD)" > "$1"
 }
-
-function get_bazel() {
-  # download bazel version
-  wget https://github.com/bazelbuild/bazel/releases/download/2.2.0/bazel-2.2.0-linux-x86_64 -O tools/bazel
-  # verify content
-  echo 'b2f002ea0e6194a181af6ac84cd94bd8dc797722eb2354690bebac92dda233ff tools/bazel' | sha256sum --quiet -c
-
-  chmod +x tools/bazel
-}
diff --git a/.jenkins/pytorch/test.sh b/.jenkins/pytorch/test.sh
@@ -42,7 +42,7 @@ if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
 fi
 
 # --user breaks ppc64le builds and these packages are already in ppc64le docker
-if [[ "$BUILD_ENVIRONMENT" != *ppc64le* ]] && [[ "$BUILD_ENVIRONMENT" != *-bazel-* ]] ; then
+if [[ "$BUILD_ENVIRONMENT" != *ppc64le* ]]; then
   # JIT C++ extensions require ninja.
   pip_install --user ninja
   # ninja is installed in /var/lib/jenkins/.local/bin
@@ -251,15 +251,7 @@ test_backward_compatibility() {
   assert_git_not_dirty
 }
 
-test_bazel() {
-  set -e
-
-  get_bazel
-
-  tools/bazel test --test_tag_filters=-gpu-required --test_filter=-*_CUDA :all_tests
-}
-
-if ! [[ "${BUILD_ENVIRONMENT}" == *libtorch* || "${BUILD_ENVIRONMENT}" == *-bazel-* ]]; then
+if ! [[ "${BUILD_ENVIRONMENT}" == *libtorch* ]]; then
   (cd test && python -c "import torch; print(torch.__config__.show())")
   (cd test && python -c "import torch; print(torch.__config__.parallel_info())")
 fi
@@ -285,8 +277,6 @@ elif [[ "${BUILD_ENVIRONMENT}" == *-test2 || "${JOB_BASE_NAME}" == *-test2 ]]; t
   test_aten
   test_libtorch
   test_custom_script_ops
-elif [[ "${BUILD_ENVIRONMENT}" == *-bazel-* ]]; then
-  test_bazel
 else
   test_torchvision
   test_python_nn