[FLINK-19664][e2e] Upload logs before tests time out

This closes apache#13655

tools/azure-pipelines/e2e_uploading_watchdog.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file has the following tasks
+# a) It reads the e2e timeout from the configuration file
+# b) It prints a warning if the test has reached 80% of it's execution time
+# c) N minutes before the end of the execution time, it will start uploading the current output as azure artifacts
+
+COMMAND=$1
+
+HERE="`dirname \"$0\"`"             # relative
+HERE="`( cd \"$HERE\" && pwd )`"    # absolutized and normalized
+if [ -z "$HERE" ] ; then
+	exit 1
+fi
+
+OUTPUT_FILE=/tmp/_e2e_watchdog.output
+# Start uploading 11 minutes before the timeout imposed by Azure (11 minutes because the upload happens 
+# every 5 minutes, so we should ideally get 2 uploads and then the operation gets killed)
+START_LOG_UPLOAD_SECONDS_FROM_END=$((11*60))
+
+DEFINED_TIMEOUT_MINUTES=`cat $HERE/jobs-template.yml | grep "timeoutInMinutes" | tail -n 1 | cut -d ":" -f 2 | tr -d '[:space:]'`
+DEFINED_TIMEOUT_SECONDS=$(($DEFINED_TIMEOUT_MINUTES*60))
+
+echo "Running command '$COMMAND' with a timeout of $DEFINED_TIMEOUT_MINUTES minutes."
+
+function warning_watchdog {
+	SLEEP_TIME=$(echo "scale=0; $DEFINED_TIMEOUT_SECONDS*0.8/1" | bc)
+	sleep $SLEEP_TIME
+	echo "=========================================================================================="
+	echo "=== WARNING: This E2E Run took already 80% of the allocated time budget of $DEFINED_TIMEOUT_MINUTES minutes ==="
+	echo "=========================================================================================="
+}
+
+function log_upload_watchdog {
+	SLEEP_TIME=$(($DEFINED_TIMEOUT_SECONDS-$START_LOG_UPLOAD_SECONDS_FROM_END))
+	sleep $SLEEP_TIME
+	echo "======================================================================================================"
+	echo "=== WARNING: This E2E Run will time out in the next few minutes. Starting to upload the log output ==="
+	echo "======================================================================================================"
+
+	INDEX=0
+	while true; do
+		cp $OUTPUT_FILE "$OUTPUT_FILE.$INDEX"
+		echo "##vso[artifact.upload containerfolder=e2e-timeout-logs;artifactname=log_upload_watchdog.output;]$OUTPUT_FILE.$INDEX"
+		INDEX=$(($INDEX+1))
+		sleep 300
+	done
+}
+
+warning_watchdog &
+log_upload_watchdog &
+
+# ts from moreutils prepends the time to each line
+( $COMMAND & PID=$! ; wait $PID ) | ts | tee $OUTPUT_FILE

tools/azure-pipelines/jobs-template.yml

@@ -170,7 +170,7 @@ jobs:
   #condition: or(eq(variables['MODE'], 'e2e'), eq(${{parameters.run_end_to_end}}, 'true'))
   # We are running this in a separate pool
   pool: ${{parameters.e2e_pool_definition}}
-  timeoutInMinutes: 260
+  timeoutInMinutes: 310
   cancelTimeoutInMinutes: 1
   workspace:
     clean: all
@@ -216,13 +216,14 @@ jobs:
         ./tools/azure-pipelines/free_disk_space.sh
 
         echo "Installing required software"
-        sudo apt-get install -y bc
+        sudo apt-get install -y bc moreutils
       displayName: Prepare E2E run
       condition: not(eq(variables['SKIP'], '1'))
     - script: ${{parameters.environment}} ./tools/ci/compile.sh
       displayName: Build Flink
       condition: not(eq(variables['SKIP'], '1'))
-    - script: ${{parameters.environment}} FLINK_DIR=`pwd`/build-target flink-end-to-end-tests/run-nightly-tests.sh
+    - script: ${{parameters.environment}} FLINK_DIR=`pwd`/build-target ./tools/azure-pipelines/e2e_uploading_watchdog.sh flink-end-to-end-tests/run-nightly-tests.sh
+      timeoutInMinutes: 250
       displayName: Run e2e tests
       env:
         IT_CASE_S3_BUCKET: $(SECRET_S3_BUCKET)