Skip to content

Commit

Permalink
[FLINK-15053][runtime] Escape all dynamic property values for taskman…
Browse files Browse the repository at this point in the history
…ager

For unix-like OS(Linux, MacOS, FREE_BSD, etc.), each value will put in single quotes. This works for all chars except single quote itself. To escape the single quote, close the quoting before it, insert the escaped single quote, and then re-open the quoting. For example, the value is foo'bar and the escaped value is 'foo'\''bar'.

For windows OS, each value will be surrounded with double quotes. The double quote itself needs to be escaped with back slash. Also the caret symbol need to be escaped with double carets since the window command uses it to escape characters.

This closes apache#10532.
  • Loading branch information
wangyang0918 authored and tillrohrmann committed Jan 7, 2020
1 parent f0fe365 commit 2ff1de7
Show file tree
Hide file tree
Showing 6 changed files with 343 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,7 @@ private List<String> getTaskManagerStartCommand() {

final String mainClassArgs = "--" + CommandLineOptions.CONFIG_DIR_OPTION.getLongOpt() + " " +
flinkConfig.getString(KubernetesConfigOptions.FLINK_CONF_DIR) + " " +
BootstrapTools.getDynamicProperties(flinkClientConfig, flinkConfig);
BootstrapTools.getDynamicPropertiesAsString(flinkClientConfig, flinkConfig);

final String command = KubernetesUtils.getTaskManagerStartCommand(
flinkConfig,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,10 @@
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.entrypoint.parser.CommandLineOptions;
import org.apache.flink.util.NetUtils;
import org.apache.flink.util.OperatingSystem;

import org.apache.flink.shaded.guava18.com.google.common.escape.Escaper;
import org.apache.flink.shaded.guava18.com.google.common.escape.Escapers;
import org.apache.flink.shaded.netty4.io.netty.channel.ChannelException;

import akka.actor.ActorSystem;
Expand Down Expand Up @@ -69,6 +72,15 @@ public class BootstrapTools {

private static final Logger LOG = LoggerFactory.getLogger(BootstrapTools.class);

private static final Escaper UNIX_SINGLE_QUOTE_ESCAPER = Escapers.builder()
.addEscape('\'', "'\\''")
.build();

private static final Escaper WINDOWS_DOUBLE_QUOTE_ESCAPER = Escapers.builder()
.addEscape('"', "\\\"")
.addEscape('^', "\"^^\"")
.build();

/**
* Starts an ActorSystem with the given configuration listening at the address/ports.
* @param configuration The Flink configuration
Expand Down Expand Up @@ -624,7 +636,7 @@ public Config getAkkaConfig() {
* @param targetConfig The target configuration.
* @return Dynamic properties as string, separated by whitespace.
*/
public static String getDynamicProperties(Configuration baseConfig, Configuration targetConfig) {
public static String getDynamicPropertiesAsString(Configuration baseConfig, Configuration targetConfig) {

String[] newAddedConfigs = targetConfig.keySet().stream().flatMap(
(String key) -> {
Expand All @@ -633,7 +645,7 @@ public static String getDynamicProperties(Configuration baseConfig, Configuratio

if (!baseConfig.keySet().contains(key) || !baseValue.equals(targetValue)) {
return Stream.of("-" + CommandLineOptions.DYNAMIC_PROPERTY_OPTION.getOpt() + key +
CommandLineOptions.DYNAMIC_PROPERTY_OPTION.getValueSeparator() + targetValue);
CommandLineOptions.DYNAMIC_PROPERTY_OPTION.getValueSeparator() + escapeForDifferentOS(targetValue));
} else {
return Stream.empty();
}
Expand All @@ -642,6 +654,38 @@ public static String getDynamicProperties(Configuration baseConfig, Configuratio
return String.join(" ", newAddedConfigs);
}

/**
* Escape all the dynamic property values.
* For Unix-like OS(Linux, MacOS, FREE_BSD, etc.), each value will be surrounded with single quotes. This works for
* all chars except single quote itself. To escape the single quote, close the quoting before it, insert the escaped
* single quote, and then re-open the quoting. For example, the value is foo'bar and the escaped value is
* 'foo'\''bar'. See <a href="https://stackoverflow.com/questions/15783701/which-characters-need-to-be-escaped-when-using-bash">https://stackoverflow.com/questions/15783701/which-characters-need-to-be-escaped-when-using-bash</a>
* for more information about Unix escaping.
*
* <p>For Windows OS, each value will be surrounded with double quotes. The double quote itself needs to be escaped with
* back slash. Also the caret symbol need to be escaped with double carets since Windows uses it to escape characters.
* See <a href="https://en.wikibooks.org/wiki/Windows_Batch_Scripting">https://en.wikibooks.org/wiki/Windows_Batch_Scripting</a>
* for more information about Windows escaping.
*
* @param value value to be escaped
* @return escaped value
*/
public static String escapeForDifferentOS(String value) {
if (OperatingSystem.isWindows()) {
return escapeWithDoubleQuote(value);
} else {
return escapeWithSingleQuote(value);
}
}

public static String escapeWithSingleQuote(String value) {
return "'" + UNIX_SINGLE_QUOTE_ESCAPER.escape(value) + "'";
}

public static String escapeWithDoubleQuote(String value) {
return "\"" + WINDOWS_DOUBLE_QUOTE_ESCAPER.escape(value) + "\"";
}

/**
* Calculate heap size after cut-off. The heap size after cut-off will be used to set -Xms and -Xmx for jobmanager
* start command.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.flink.runtime.concurrent.FutureUtils;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.ExecutorUtils;
import org.apache.flink.util.OperatingSystem;
import org.apache.flink.util.TestLogger;
import org.apache.flink.util.function.CheckedSupplier;

Expand Down Expand Up @@ -413,17 +414,48 @@ public void testActorSystemInstantiationFailureWhenPortOccupied() throws Excepti
}

@Test
public void testGetDynamicProperties() {
Configuration baseConfig = new Configuration();
public void testGetDynamicPropertiesAsString() {
final Configuration baseConfig = new Configuration();
baseConfig.setString("key.a", "a");
baseConfig.setString("key.b", "b1");

Configuration targetConfig = new Configuration();
final Configuration targetConfig = new Configuration();
targetConfig.setString("key.b", "b2");
targetConfig.setString("key.c", "c");

String dynamicProperties = BootstrapTools.getDynamicProperties(baseConfig, targetConfig);
assertEquals("-Dkey.b=b2 -Dkey.c=c", dynamicProperties);
final String dynamicProperties = BootstrapTools.getDynamicPropertiesAsString(baseConfig, targetConfig);
if (OperatingSystem.isWindows()) {
assertEquals("-Dkey.b=\"b2\" -Dkey.c=\"c\"", dynamicProperties);
} else {
assertEquals("-Dkey.b='b2' -Dkey.c='c'", dynamicProperties);
}
}

@Test
public void testEscapeDynamicPropertyValueWithSingleQuote() {
final String value1 = "#a,b&c^d*e@f(g!h";
assertEquals("'" + value1 + "'", BootstrapTools.escapeWithSingleQuote(value1));

final String value2 = "'foobar";
assertEquals("''\\''foobar'", BootstrapTools.escapeWithSingleQuote(value2));

final String value3 = "foo''bar";
assertEquals("'foo'\\'''\\''bar'", BootstrapTools.escapeWithSingleQuote(value3));

final String value4 = "'foo' 'bar'";
assertEquals("''\\''foo'\\'' '\\''bar'\\'''", BootstrapTools.escapeWithSingleQuote(value4));
}

@Test
public void testEscapeDynamicPropertyValueWithDoubleQuote() {
final String value1 = "#a,b&c^d*e@f(g!h";
assertEquals("\"#a,b&c\"^^\"d*e@f(g!h\"", BootstrapTools.escapeWithDoubleQuote(value1));

final String value2 = "foo\"bar'";
assertEquals("\"foo\\\"bar'\"", BootstrapTools.escapeWithDoubleQuote(value2));

final String value3 = "\"foo\" \"bar\"";
assertEquals("\"\\\"foo\\\" \\\"bar\\\"\"", BootstrapTools.escapeWithDoubleQuote(value3));
}

@Test
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.flink.test.util;

import org.apache.flink.util.OperatingSystem;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.List;

/**
* ShellScript for creating shell script on linux and windows.
*/
public class ShellScript {

public static ShellScriptBuilder createShellScriptBuilder() {
if (OperatingSystem.isWindows()) {
return new WindowsShellScriptBuilder();
}
return new UnixShellScriptBuilder();
}

public static String getScriptExtension() {
return OperatingSystem.isWindows() ? ".cmd" : ".sh";
}

/**
* Builder to create shell script.
*/
public abstract static class ShellScriptBuilder {
private static final String LINE_SEPARATOR = System.getProperty("line.separator");
private final StringBuilder sb = new StringBuilder();

void line(String... command) {
for (String s : command) {
sb.append(s);
}
sb.append(LINE_SEPARATOR);
}

public void write(File file) throws IOException {
try (FileWriter fwrt = new FileWriter(file); PrintWriter out = new PrintWriter(fwrt)) {
out.append(sb);
}
file.setExecutable(true, false);
}

public abstract void command(List<String> command);

public abstract void env(String key, String value);
}

private static final class UnixShellScriptBuilder extends ShellScriptBuilder {

UnixShellScriptBuilder(){
line("#!/usr/bin/env bash");
line();
}

public void command(List<String> command) {
line("exec ", String.join(" ", command));
}

public void env(String key, String value) {
line("export ", key, "=\"", value, "\"");
}
}

private static final class WindowsShellScriptBuilder extends ShellScriptBuilder {

WindowsShellScriptBuilder() {
line("@setlocal");
line();
}

public void command(List<String> command) {
line("@call ", String.join(" ", command));
}

public void env(String key, String value) {
line("@set ", key, "=", value);
}
}
}
Loading

0 comments on commit 2ff1de7

Please sign in to comment.