Revert D15499918: Reduce set of build/tests which run on PRs.

Differential Revision:
D15499918

Original commit changeset: 992e3f91f95d

fbshipit-source-id: 86fc43d3da7ea3e3a32e95fc4f4f3de6cbd5d49b

.circleci/README.md

@@ -1,23 +1,3 @@
-Structure of CI
-===============
-
-setup job:
-1. Does a git checkout
-2. Persists CircleCI scripts (everything in `.circleci`) into a workspace.  Why?
-   We don't always do a Git checkout on all subjobs, but we usually
-   still want to be able to call scripts one way or another in a subjob.
-   Persisting files this way lets us have access to them without doing a
-   checkout.  This workspace is conventionally mounted on `~/workspace`
-   (this is distinguished from `~/project`, which is the conventional
-   working directory that CircleCI will default to starting your jobs
-   in.)
-3. Write out the commit message to `.circleci/COMMIT_MSG`.  This is so
-   we can determine in subjobs if we should actually run the jobs or
-   not, even if there isn't a Git checkout.
-
-
-
-
 CircleCI configuration generator
 ================================
 
@@ -55,4 +35,4 @@ Future direction
 See comment [here](https://github.com/pytorch/pytorch/pull/17323#pullrequestreview-206945747):
 
 In contrast with a full recursive tree traversal of configuration dimensions,
-> in the future future I think we actually want to decrease our matrix somewhat and have only a few mostly-orthogonal builds that taste as many different features as possible on PRs, plus a more complete suite on every PR and maybe an almost full suite nightly/weekly (we don't have this yet). Specifying PR jobs in the future might be easier to read with an explicit list when we come to this.
+> in the future future I think we actually want to decrease our matrix somewhat and have only a few mostly-orthogonal builds that taste as many different features as possible on PRs, plus a more complete suite on every PR and maybe an almost full suite nightly/weekly (we don't have this yet). Specifying PR jobs in the future might be easier to read with an explicit list when we come to this.

.circleci/config.yml

@@ -130,16 +130,12 @@ install_doc_push_script: &install_doc_push_script
     EOL
     chmod +x /home/circleci/project/doc_push_script.sh
 
-# NB: This (and the command below) must be run after attaching
-# ~/workspace.  This is NOT the default working directory (that's
-# ~/project); this workspace is generated by the setup job.
-should_run_job: &should_run_job
-  name: Should Run Job After attach_workspace
-  no_output_timeout: "2m"
-  command: ~/workspace/.circleci/scripts/should_run_job.sh
-
+# `setup_ci_environment` has to be run **after** the ``checkout`` step because
+# it writes into the checkout directory and otherwise git will complain
+# that
+#   Directory (/home/circleci/project) you are trying to checkout to is not empty and not git repository
 setup_ci_environment: &setup_ci_environment
-  name: Set Up CI Environment After attach_workspace
+  name: Set Up CI Environment After Checkout
   no_output_timeout: "1h"
   command: ~/workspace/.circleci/scripts/setup_ci_environment.sh
 
@@ -174,11 +170,8 @@ pytorch_linux_build_defaults: &pytorch_linux_build_defaults
   machine:
     image: ubuntu-1604:201903-01
   steps:
-  # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
   - attach_workspace:
       at: ~/workspace
-  - run:
-      <<: *should_run_job
   - run:
       <<: *setup_linux_system_environment
   - checkout
@@ -225,13 +218,11 @@ pytorch_linux_test_defaults: &pytorch_linux_test_defaults
   machine:
     image: ubuntu-1604:201903-01
   steps:
-  # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
   - attach_workspace:
       at: ~/workspace
-  - run:
-      <<: *should_run_job
   - run:
       <<: *setup_linux_system_environment
+  - checkout
   - run:
       <<: *setup_ci_environment
   - run:
@@ -265,11 +256,8 @@ caffe2_linux_build_defaults: &caffe2_linux_build_defaults
   machine:
     image: ubuntu-1604:201903-01
   steps:
-  # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
   - attach_workspace:
       at: ~/workspace
-  - run:
-      <<: *should_run_job
   - run:
       <<: *setup_linux_system_environment
   - checkout
@@ -329,13 +317,10 @@ caffe2_linux_test_defaults: &caffe2_linux_test_defaults
   machine:
     image: ubuntu-1604:201903-01
   steps:
-  # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
   - attach_workspace:
       at: ~/workspace
   - run:
       <<: *setup_linux_system_environment
-  - run:
-      <<: *should_run_job
   - run:
       <<: *setup_ci_environment
   - run:
@@ -401,11 +386,6 @@ caffe2_macos_build_defaults: &caffe2_macos_build_defaults
   macos:
     xcode: "9.0"
   steps:
-    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-    - attach_workspace:
-        at: ~/workspace
-    - run:
-        <<: *should_run_job
     - checkout
     - run:
         <<: *macos_brew_update
@@ -556,11 +536,8 @@ binary_macos_brew_update: &binary_macos_brew_update
 binary_linux_build: &binary_linux_build
   resource_class: 2xlarge+
   steps:
-  # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
   - attach_workspace:
       at: ~/workspace
-  - run:
-      <<: *should_run_job
   - run:
       <<: *binary_checkout
   - run:
@@ -609,14 +586,10 @@ binary_linux_test: &binary_linux_test
   machine:
     image: ubuntu-1604:201903-01
   steps:
-  # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
   - attach_workspace:
       at: ~/workspace
-  # TODO: We shouldn't attach the workspace multiple times
   - attach_workspace:
       at: /home/circleci/project
-  - run:
-      <<: *should_run_job
   - run:
       <<: *setup_linux_system_environment
   - run:
@@ -636,11 +609,8 @@ binary_linux_upload: &binary_linux_upload
   machine:
     image: ubuntu-1604:201903-01
   steps:
-  # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
   - attach_workspace:
       at: ~/workspace
-  - run:
-      <<: *should_run_job
   - run:
       <<: *setup_linux_system_environment
   - run:
@@ -665,11 +635,8 @@ binary_mac_build: &binary_mac_build
   macos:
     xcode: "9.0"
   steps:
-  # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
   - attach_workspace:
       at: ~/workspace
-  - run:
-      <<: *should_run_job
   - run:
       <<: *binary_checkout
   - run:
@@ -705,11 +672,8 @@ binary_mac_upload: &binary_mac_upload
   macos:
     xcode: "9.0"
   steps:
-  # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
   - attach_workspace:
       at: ~/workspace
-  - run:
-      <<: *should_run_job
   - run:
       <<: *binary_checkout
   - run:
@@ -1046,30 +1010,11 @@ jobs:
           name: Ensure config is up to date
           command: ./ensure-consistency.py
           working_directory: .circleci
-      - run:
-          name: Save commit message
-          command: git log --format='%B' -n 1 HEAD > .circleci/scripts/COMMIT_MSG
-      # Note [Workspace for CircleCI scripts]
-      # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-      # In the beginning, you wrote your CI scripts in a
-      # .circleci/config.yml file, and life was good.  Your CI
-      # configurations flourished and multiplied.
-      #
-      # Then one day, CircleCI cometh down high and say, "Your YAML file
-      # is too biggeth, it stresses our servers so."  And thus they
-      # asketh us to smite the scripts in the yml file.
-      #
-      # But you can't just put the scripts in the .circleci folder,
-      # because in some jobs, you don't ever actually checkout the
-      # source repository.  Where you gonna get the scripts from?
-      #
-      # Here's how you do it: you persist .circleci/scripts into a
-      # workspace, attach the workspace in your subjobs, and run all
-      # your scripts from there.
       - persist_to_workspace:
           root: .
           paths: .circleci/scripts
 
+
   pytorch_short_perf_test_gpu:
     environment:
       BUILD_ENVIRONMENT: pytorch-short-perf-test-gpu
@@ -1080,11 +1025,8 @@ jobs:
     machine:
       image: ubuntu-1604:201903-01
     steps:
-    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
     - attach_workspace:
         at: ~/workspace
-    - run:
-        <<: *should_run_job
     - run:
         <<: *setup_linux_system_environment
     - run:
@@ -1118,11 +1060,8 @@ jobs:
     machine:
       image: ubuntu-1604:201903-01
     steps:
-    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
     - attach_workspace:
         at: ~/workspace
-    - run:
-        <<: *should_run_job
     - run:
         <<: *setup_linux_system_environment
     - run:
@@ -1165,21 +1104,16 @@ jobs:
           docker push ${DEBUG_COMMIT_DOCKER_IMAGE}
 
   pytorch_macos_10_13_py3_build:
-    environment:
-      BUILD_ENVIRONMENT: pytorch-macos-10.13-py3-build
     macos:
       xcode: "9.0"
     steps:
-      # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-      - attach_workspace:
-          at: ~/workspace
-      - run:
-          <<: *should_run_job
       - checkout
       - run:
           <<: *macos_brew_update
       - run:
           name: Build
+          environment:
+            BUILD_ENVIRONMENT: pytorch-macos-10.13-py3-build
           no_output_timeout: "1h"
           command: |
             set -e
@@ -1210,50 +1144,44 @@ jobs:
             - "*"
 
   pytorch_macos_10_13_py3_test:
-    environment:
-      BUILD_ENVIRONMENT: pytorch-macos-10.13-py3-test
     macos:
       xcode: "9.0"
     steps:
-      # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-      # This workspace also carries binaries from the build job
-      - attach_workspace:
-          at: ~/workspace
       - run:
-          <<: *should_run_job
+          name: Prepare workspace
+          command: |
+            sudo mkdir -p /Users/distiller/pytorch-ci-env
+            sudo chmod -R 777 /Users/distiller/pytorch-ci-env
+      - attach_workspace:
+          at: /Users/distiller/pytorch-ci-env
       - run:
           <<: *macos_brew_update
       - run:
           name: Test
+          environment:
+            BUILD_ENVIRONMENT: pytorch-macos-10.13-py3-test
           no_output_timeout: "1h"
           command: |
             set -e
             export IN_CIRCLECI=1
 
             # copy with -a to preserve relative structure (e.g., symlinks), and be recursive
-            # TODO: I'm not sure why we can't just run our job in
-            # ~/workspace and call it a day
-            cp -a ~/workspace/. /Users/distiller/project
+            cp -a /Users/distiller/pytorch-ci-env/workspace/. /Users/distiller/project
 
             chmod a+x .jenkins/pytorch/macos-test.sh
             unbuffer .jenkins/pytorch/macos-test.sh 2>&1 | ts
 
   pytorch_macos_10_13_cuda9_2_cudnn7_py3_build:
-    environment:
-      BUILD_ENVIRONMENT: pytorch-macos-10.13-cuda9.2-cudnn7-py3-build
     macos:
       xcode: "9.0"
     steps:
-      # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
-      - attach_workspace:
-          at: ~/workspace
-      - run:
-          <<: *should_run_job
       - checkout
       - run:
           <<: *macos_brew_update
       - run:
           name: Build
+          environment:
+            BUILD_ENVIRONMENT: pytorch-macos-10.13-cuda9.2-cudnn7-py3-build
           no_output_timeout: "1h"
           command: |
             set -e
@@ -1289,6 +1217,7 @@ jobs:
             git submodule sync && git submodule update -q --init --recursive
             chmod a+x .jenkins/pytorch/macos-build.sh
             unbuffer .jenkins/pytorch/macos-build.sh 2>&1 | ts
+
   caffe2_py2_gcc4_8_ubuntu14_04_build:
     environment:
       BUILD_ENVIRONMENT: "caffe2-py2-gcc4.8-ubuntu14.04-build"

.circleci/scripts/setup_ci_environment.sh

@@ -1,6 +1,46 @@
 #!/usr/bin/env bash
 set -ex -o pipefail
 
+# Check if we should actually run
+echo "BUILD_ENVIRONMENT: ${BUILD_ENVIRONMENT}"
+echo "CIRCLE_PULL_REQUEST: ${CIRCLE_PULL_REQUEST:-}"
+if [[ "${BUILD_ENVIRONMENT}" == *-slow-* ]]; then
+  if ! [ -z "${CIRCLE_PULL_REQUEST:-}" ]; then
+    # It's a PR; test for [slow ci] tag on the TOPMOST commit
+    topmost_commit=$(git log --format='%B' -n 1 HEAD)
+    if !(echo $topmost_commit | grep -q -e '\[slow ci\]' -e '\[ci slow\]' -e '\[test slow\]' -e '\[slow test\]'); then
+      circleci step halt
+      exit
+    fi
+  fi
+fi
+if [[ "${BUILD_ENVIRONMENT}" == *xla* ]]; then
+  if ! [ -z "${CIRCLE_PULL_REQUEST:-}" ]; then
+    # It's a PR; test for [xla ci] tag on the TOPMOST commit
+    topmost_commit=$(git log --format='%B' -n 1 HEAD)
+    if !(echo $topmost_commit | grep -q -e '\[xla ci\]' -e '\[ci xla\]' -e '\[test xla\]' -e '\[xla test\]'); then
+      # NB: This doesn't halt everything, just this job.  So
+      # the rest of the workflow will keep going and you need
+      # to make sure you halt there too.  Blegh.
+      circleci step halt
+      exit
+    fi
+  fi
+fi
+if [[ "${BUILD_ENVIRONMENT}" == *namedtensor* ]]; then
+  if ! [ -z "${CIRCLE_PULL_REQUEST:-}" ]; then
+    # It's a PR; test for [namedtensor] tag on the TOPMOST commit
+    topmost_commit=$(git log --format='%B' -n 1 HEAD)
+    if !(echo $topmost_commit | grep -q -e '\[namedtensor\]' -e '\[ci namedtensor\]' -e '\[namedtensor ci\]'); then
+      # NB: This doesn't halt everything, just this job.  So
+      # the rest of the workflow will keep going and you need
+      # to make sure you halt there too.  Blegh.
+      circleci step halt
+      exit
+    fi
+  fi
+fi
+
 # Set up NVIDIA docker repo
 curl -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
 echo "deb https://nvidia.github.io/libnvidia-container/ubuntu16.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list