[FLINK-3296] Remove 'flushing' behavior of the OutputFormat support o…

…f the DataStream API

This closes apache#1563

docs/apis/streaming/index.md

@@ -1819,14 +1819,14 @@ of each element on the standard out / strandard error stream. Optionally, a pref
 prepended to the output. This can help to distinguish between different calls to *print*. If the parallelism is
 greater than 1, the output will also be prepended with the identifier of the task which produced the output.
 
-- `write()` / `FileOutputFormat` - Method and base class for custom file outputs. Supports
+- `writeUsingOutputFormat()` / `FileOutputFormat` - Method and base class for custom file outputs. Supports
   custom object-to-bytes conversion.
 
 - `writeToSocket` - Writes elements to a socket according to a `SerializationSchema`
 
 - `addSink` - Invokes a custom sink function. Flink comes bundled with connectors to other systems (such as
     Apache Kafka) that are implemented as sink functions.
-
+    
 </div>
 <div data-lang="scala" markdown="1">
 
@@ -1847,7 +1847,7 @@ of each element on the standard out / strandard error stream. Optionally, a pref
 prepended to the output. This can help to distinguish between different calls to *print*. If the parallelism is
 greater than 1, the output will also be prepended with the identifier of the task which produced the output.
 
-- `write()` / `FileOutputFormat` - Method and base class for custom file outputs. Supports
+- `writeUsingOutputFormat()` / `FileOutputFormat` - Method and base class for custom file outputs. Supports
   custom object-to-bytes conversion.
 
 - `writeToSocket` - Writes elements to a socket according to a `SerializationSchema`
@@ -1858,6 +1858,17 @@ greater than 1, the output will also be prepended with the identifier of the tas
 </div>
 </div>
 
+Note that the `write*()` methods on `DataStream` are mainly intended for debugging purposes.
+They are not participating in Flink's checkpointing, this means these functions usually have 
+at-least-once semantics. The data flushing to the target system depends on the implementation of the 
+OutputFormat. This means that not all elements send to the OutputFormat are immediately showing up 
+in the target system. Also, in failure cases, those records might be lost.
+
+For reliable, exactly-once delivery of a stream into a file system, use the `flink-connector-filesystem`.
+Also, custom implementations through the `.addSink(...)` method can partiticpate in Flink's checkpointing
+for exactly-once semantics.
+
+
 
 {% top %}
 

flink-batch-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/example/HBaseWriteStreamExample.java

@@ -41,7 +41,7 @@
  */
 public class HBaseWriteStreamExample {
 
-	public static void main(String[] args) {
+	public static void main(String[] args) throws Exception {
 		final StreamExecutionEnvironment env = StreamExecutionEnvironment
 				.getExecutionEnvironment();
 
@@ -64,14 +64,9 @@ public void cancel() {
 				isRunning = false;
 			}
 		});
-		dataStream.write(new HBaseOutputFormat(), 0L);
+		dataStream.writeUsingOutputFormat(new HBaseOutputFormat());
 
-		try {
-			env.execute();
-		} catch (Exception e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
-		}
+		env.execute();
 	}
 
 	/**

flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java

@@ -90,7 +90,7 @@ public static void main(String[] args) throws Exception {
 
 		// emit results
 		if (fileOutput) {
-			numbers.writeAsText(outputPath, 1);
+			numbers.writeAsText(outputPath);
 		} else {
 			numbers.print();
 		}

flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java

@@ -87,7 +87,7 @@ public static void main(String[] args) throws Exception {
 
 		// emit result
 		if (fileOutput) {
-			joinedStream.writeAsText(outputPath, 1);
+			joinedStream.writeAsText(outputPath);
 		} else {
 			joinedStream.print();
 		}

flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java

@@ -50,8 +50,6 @@
  */
 public class IncrementalLearningSkeleton {
 
-	private static DataStream<Integer> trainingData = null;
-	private static DataStream<Integer> newData = null;
 
 	// *************************************************************************
 	// PROGRAM
@@ -66,8 +64,8 @@ public static void main(String[] args) throws Exception {
 		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 
-		trainingData = env.addSource(new FiniteTrainingDataSource());
-		newData = env.addSource(new FiniteNewDataSource());
+		DataStream<Integer> trainingData = env.addSource(new FiniteTrainingDataSource());
+		DataStream<Integer> newData = env.addSource(new FiniteNewDataSource());
 
 		// build new model on every second of new data
 		DataStream<Double[]> model = trainingData
@@ -80,7 +78,7 @@ public static void main(String[] args) throws Exception {
 
 		// emit result
 		if (fileOutput) {
-			prediction.writeAsText(outputPath, 1);
+			prediction.writeAsText(outputPath);
 		} else {
 			prediction.print();
 		}

flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java

@@ -67,7 +67,7 @@ public static void main(String[] args) throws Exception {
 						.sum(1);
 
 		if (fileOutput) {
-			counts.writeAsText(outputPath, 1);
+			counts.writeAsText(outputPath);
 		} else {
 			counts.print();
 		}

flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketTextStreamWordCount.scala

@@ -61,7 +61,7 @@ object SocketTextStreamWordCount {
       .sum(1)
 
     if (fileOutput) {
-      counts.writeAsText(outputPath, 1)
+      counts.writeAsText(outputPath)
     } else {
       counts print
     }

flink-java8/src/main/java/org/apache/flink/streaming/examples/java8/wordcount/WordCount.java

@@ -78,7 +78,7 @@ public static void main(String[] args) throws Exception {
 
 		// emit result
 		if(fileOutput) {
-			counts.writeAsCsv(outputPath, 1);
+			counts.writeAsCsv(outputPath);
 		} else {
 			counts.print();
 		}

flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java

@@ -114,7 +114,7 @@ public void invoke() throws Exception {
 
 		ExecutionConfig executionConfig;
 		try {
-			ExecutionConfig c = (ExecutionConfig) InstantiationUtil.readObjectFromConfig(
+			ExecutionConfig c = InstantiationUtil.readObjectFromConfig(
 					getJobConfiguration(),
 					ExecutionConfig.CONFIG_KEY,
 					getUserCodeClassLoader());
@@ -130,7 +130,6 @@ public void invoke() throws Exception {
 		boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled();
 
 		try {
-
 			// initialize local strategies
 			MutableObjectIterator<IT> input1;
 			switch (this.config.getInputLocalStrategy(0)) {