Run sccache in background mode and save logs to file (#7594)

Running sccache in foreground mode seems to uniformly slow down the builds and causes virtual memory exhausted errors for gcc7.2 builds. This PR moves sccache to background mode instead and print the compilation log at the end of the build.

.jenkins/pytorch/common.sh

@@ -62,15 +62,18 @@ declare -f -t trap_add
 trap_add cleanup EXIT
 
 if which sccache > /dev/null; then
-  # Start sccache server in foreground mode, so that we can see better logging
+  # Save sccache logs to file
   sccache --stop-server || true
-  SCCACHE_NO_DAEMON=1 RUST_LOG=sccache::server=error sccache --start-server
+  SCCACHE_ERROR_LOG=~/sccache_error.log RUST_LOG=sccache::server=error sccache --start-server
 
   # Report sccache stats for easier debugging
   sccache --zero-stats
-  sccache --show-stats
   function sccache_epilogue() {
-     sccache --show-stats
+    echo '=================== sccache compilation log ==================='
+    python $(dirname "${BASH_SOURCE[0]}")/print_sccache_log.py ~/sccache_error.log
+    echo '=========== If your build fails, please take a look at the log above for possible reasons ==========='
+    sccache --show-stats
+    sccache --stop-server || true
   }
   trap_add sccache_epilogue EXIT
 fi

.jenkins/pytorch/print_sccache_log.py

@@ -0,0 +1,11 @@
+import sys
+
+log_file_path = sys.argv[1]
+
+with open(log_file_path) as f:
+    lines = f.readlines()
+
+for line in lines:
+    # Ignore errors from CPU instruction set testing
+    if 'CMakeFiles/CMakeTmp/src.c' not in line:
+        print(line)

.jenkins/pytorch/short-perf-test-gpu.sh

@@ -3,7 +3,7 @@
 COMPACT_JOB_NAME="short-perf-test-gpu"
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
-cd .jenkins/pytorch/perf_test
+pushd .jenkins/pytorch/perf_test
 
 echo "Running GPU perf test for PyTorch..."
 
@@ -64,3 +64,5 @@ if [[ "$COMMIT_SOURCE" == master ]]; then
     # but the chance of them executing this line at the same time is low.
     aws s3 cp new_gpu_runtime.json s3://ossci-perf-test/pytorch/gpu_runtime/${MASTER_COMMIT_ID}.json --acl public-read
 fi
+
+popd