[FLINK-16633][AZP] Fix builds without s3 credentials

Problem: Builds on Azure where failing if S3 credentails were not provided, because the syntax in the pipeline definition was incorrect. Solution: Use a way to access secret variables that leads to an empty value (instead of the variable name). Also, extend the check in the unit tests to "variable empty" instead of just "variable not set".
dmcnelis · Mar 20, 2020 · b226cbd · b226cbd
1 parent 0bc38ba
commit b226cbd
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 9 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -40,13 +40,19 @@ resources:
   - container: flink-build-container
     image: rmetzger/flink-ci:ubuntu-amd64-3528acd
 
-# See tools/azure-pipelines/jobs-template.yml for a short summary of the caching
+# Define variables:
+# - See tools/azure-pipelines/jobs-template.yml for a short summary of the caching
+# - See https://stackoverflow.com/questions/60742105/how-can-i-access-a-secret-value-from-an-azure-pipelines-expression
+#   to understand why the secrets are handled like this
 variables:
   MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
   MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)'
   CACHE_KEY: maven | $(Agent.OS) | **/pom.xml, !**/target/**
   CACHE_FALLBACK_KEY: maven | $(Agent.OS)
   CACHE_FLINK_DIR: $(Pipeline.Workspace)/flink_cache
+  SECRET_S3_BUCKET: $[variables.IT_CASE_S3_BUCKET]
+  SECRET_S3_ACCESS_KEY: $[variables.IT_CASE_S3_ACCESS_KEY]
+  SECRET_S3_SECRET_KEY: $[variables.IT_CASE_S3_SECRET_KEY]
 
 
 jobs:

diff --git a/...flink-test-utils-junit/src/main/java/org/apache/flink/testutils/s3/S3TestCredentials.java b/...flink-test-utils-junit/src/main/java/org/apache/flink/testutils/s3/S3TestCredentials.java
@@ -43,8 +43,15 @@ public class S3TestCredentials {
 	 * Checks whether S3 test credentials are available in the environment variables
 	 * of this JVM.
 	 */
-	public static boolean credentialsAvailable() {
-		return S3_TEST_BUCKET != null && S3_TEST_ACCESS_KEY != null && S3_TEST_SECRET_KEY != null;
+	private static boolean credentialsAvailable() {
+		return isNotEmpty(S3_TEST_BUCKET) && isNotEmpty(S3_TEST_ACCESS_KEY) && isNotEmpty(S3_TEST_SECRET_KEY);
+	}
+
+	/**
+	 * Checks if a String is not null and not empty.
+	 */
+	private static boolean isNotEmpty(@Nullable String str) {
+		return str != null && !str.isEmpty();
 	}
 
 	/**

diff --git a/tools/azure-pipelines/build-apache-repo.yml b/tools/azure-pipelines/build-apache-repo.yml
@@ -46,6 +46,9 @@ variables:
   CACHE_KEY: maven | $(Agent.OS) | **/pom.xml, !**/target/**
   CACHE_FALLBACK_KEY: maven | $(Agent.OS)
   CACHE_FLINK_DIR: $(Pipeline.Workspace)/flink_cache
+  SECRET_S3_BUCKET: $[variables.IT_CASE_S3_BUCKET]
+  SECRET_S3_ACCESS_KEY: $[variables.IT_CASE_S3_ACCESS_KEY]
+  SECRET_S3_SECRET_KEY: $[variables.IT_CASE_S3_SECRET_KEY]
 
 stages:
   # CI / PR triggered stage:

diff --git a/tools/azure-pipelines/jobs-template.yml b/tools/azure-pipelines/jobs-template.yml
@@ -123,9 +123,9 @@ jobs:
   - script: STAGE=test ${{parameters.environment}} ./tools/azure_controller.sh $(module)
     displayName: Test - $(module)
     env:
-      IT_CASE_S3_BUCKET: $(IT_CASE_S3_BUCKET)
-      IT_CASE_S3_ACCESS_KEY: $(IT_CASE_S3_ACCESS_KEY)
-      IT_CASE_S3_SECRET_KEY: $(IT_CASE_S3_SECRET_KEY)
+      IT_CASE_S3_BUCKET: $(SECRET_S3_BUCKET)
+      IT_CASE_S3_ACCESS_KEY: $(SECRET_S3_ACCESS_KEY)
+      IT_CASE_S3_SECRET_KEY: $(SECRET_S3_SECRET_KEY)
 
   - task: PublishTestResults@2
     inputs:
@@ -175,9 +175,9 @@ jobs:
     - script: ${{parameters.environment}} FLINK_DIR=`pwd`/build-target flink-end-to-end-tests/run-nightly-tests.sh
       displayName: Run e2e tests
       env:
-        IT_CASE_S3_BUCKET: $(IT_CASE_S3_BUCKET)
-        IT_CASE_S3_ACCESS_KEY: $(IT_CASE_S3_ACCESS_KEY)
-        IT_CASE_S3_SECRET_KEY: $(IT_CASE_S3_SECRET_KEY)
+        IT_CASE_S3_BUCKET: $(SECRET_S3_BUCKET)
+        IT_CASE_S3_ACCESS_KEY: $(SECRET_S3_ACCESS_KEY)
+        IT_CASE_S3_SECRET_KEY: $(SECRET_S3_SECRET_KEY)
       # upload debug artifacts
     - task: PublishPipelineArtifact@1
       condition: and(succeededOrFailed(), not(eq('$(ARTIFACT_DIR)', '')))