diff --git a/docs/dev/api_concepts.md b/docs/dev/api_concepts.md index c421507468325..bd7ca5aff9027 100644 --- a/docs/dev/api_concepts.md +++ b/docs/dev/api_concepts.md @@ -510,7 +510,7 @@ data.map(new MapFunction () { #### Java 8 Lambdas -Flink also supports Java 8 Lambdas in the Java API. Please see the full [Java 8 Guide]({{ site.baseurl }}/dev/java8.html). +Flink also supports Java 8 Lambdas in the Java API. {% highlight java %} data.filter(s -> s.startsWith("http://")); diff --git a/docs/dev/batch/examples.md b/docs/dev/batch/examples.md index 90e372dfd60e8..fe2bd8d3bdc78 100644 --- a/docs/dev/batch/examples.md +++ b/docs/dev/batch/examples.md @@ -27,8 +27,7 @@ The following example programs showcase different applications of Flink from simple word counting to graph algorithms. The code samples illustrate the use of [Flink's DataSet API]({{ site.baseurl }}/dev/batch/index.html). -The full source code of the following and more examples can be found in the __flink-examples-batch__ -or __flink-examples-streaming__ module of the Flink source repository. +The full source code of the following and more examples can be found in the {% gh_link flink-examples/flink-examples-batch "flink-examples-batch" %} module of the Flink source repository. * This will be replaced by the TOC {:toc} @@ -420,102 +419,4 @@ Input files are plain text files and must be formatted as follows: - Edges are represented as pairs for vertex IDs which are separated by space characters. Edges are separated by new-line characters: * For example `"1 2\n2 12\n1 12\n42 63\n"` gives four (undirected) links (1)-(2), (2)-(12), (1)-(12), and (42)-(63). -## Relational Query - -The Relational Query example assumes two tables, one with `orders` and the other with `lineitems` as specified by the [TPC-H decision support benchmark](http://www.tpc.org/tpch/). TPC-H is a standard benchmark in the database industry. See below for instructions how to generate the input data. - -The example implements the following SQL query. - -{% highlight sql %} -SELECT l_orderkey, o_shippriority, sum(l_extendedprice) as revenue - FROM orders, lineitem -WHERE l_orderkey = o_orderkey - AND o_orderstatus = "F" - AND YEAR(o_orderdate) > 1993 - AND o_orderpriority LIKE "5%" -GROUP BY l_orderkey, o_shippriority; -{% endhighlight %} - -The Flink program, which implements the above query looks as follows. - -
-
- -{% highlight java %} -// get orders data set: (orderkey, orderstatus, orderdate, orderpriority, shippriority) -DataSet> orders = getOrdersDataSet(env); -// get lineitem data set: (orderkey, extendedprice) -DataSet> lineitems = getLineitemDataSet(env); - -// orders filtered by year: (orderkey, custkey) -DataSet> ordersFilteredByYear = - // filter orders - orders.filter( - new FilterFunction>() { - @Override - public boolean filter(Tuple5 t) { - // status filter - if(!t.f1.equals(STATUS_FILTER)) { - return false; - // year filter - } else if(Integer.parseInt(t.f2.substring(0, 4)) <= YEAR_FILTER) { - return false; - // order priority filter - } else if(!t.f3.startsWith(OPRIO_FILTER)) { - return false; - } - return true; - } - }) - // project fields out that are no longer required - .project(0,4).types(Integer.class, Integer.class); - -// join orders with lineitems: (orderkey, shippriority, extendedprice) -DataSet> lineitemsOfOrders = - ordersFilteredByYear.joinWithHuge(lineitems) - .where(0).equalTo(0) - .projectFirst(0,1).projectSecond(1) - .types(Integer.class, Integer.class, Double.class); - -// extendedprice sums: (orderkey, shippriority, sum(extendedprice)) -DataSet> priceSums = - // group by order and sum extendedprice - lineitemsOfOrders.groupBy(0,1).aggregate(Aggregations.SUM, 2); - -// emit result -priceSums.writeAsCsv(outputPath); -{% endhighlight %} - -The {% gh_link /flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java/relational/TPCHQuery10.java "Relational Query program" %} implements the above query. It requires the following parameters to run: `--orders --lineitem --output `. - -
-
-Coming soon... - -The {% gh_link /flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/relational/TPCHQuery3.scala "Relational Query program" %} implements the above query. It requires the following parameters to run: `--orders --lineitem --output `. - -
-
- -The orders and lineitem files can be generated using the [TPC-H benchmark](http://www.tpc.org/tpch/) suite's data generator tool (DBGEN). -Take the following steps to generate arbitrary large input files for the provided Flink programs: - -1. Download and unpack DBGEN -2. Make a copy of *makefile.suite* called *Makefile* and perform the following changes: - -{% highlight bash %} -DATABASE = DB2 -MACHINE = LINUX -WORKLOAD = TPCH -CC = gcc -{% endhighlight %} - -1. Build DBGEN using *make* -2. Generate lineitem and orders relations using dbgen. A scale factor - (-s) of 1 results in a generated data set with about 1 GB size. - -{% highlight bash %} -./dbgen -T o -s 1 -{% endhighlight %} - {% top %} diff --git a/docs/dev/best_practices.md b/docs/dev/best_practices.md index e8dee306a55a3..daf4aaf73d946 100644 --- a/docs/dev/best_practices.md +++ b/docs/dev/best_practices.md @@ -192,7 +192,7 @@ public class MyClass implements MapFunction { In all cases were classes are executed with a classpath created by a dependency manager such as Maven, Flink will pull log4j into the classpath. -Therefore, you will need to exclude log4j from Flink's dependencies. The following description will assume a Maven project created from a [Flink quickstart](../quickstart/java_api_quickstart.html). +Therefore, you will need to exclude log4j from Flink's dependencies. The following description will assume a Maven project created from a [Flink quickstart](./projectsetup/java_api_quickstart.html). Change your projects `pom.xml` file like this: diff --git a/docs/dev/index.md b/docs/dev/index.md index 8b96672eeed14..b58120471b371 100644 --- a/docs/dev/index.md +++ b/docs/dev/index.md @@ -4,6 +4,7 @@ nav-id: dev nav-title: ' Application Development' nav-parent_id: root nav-pos: 5 +section-break: true --- diff --git a/docs/quickstart/java_api_quickstart.md b/docs/dev/projectsetup/java_api_quickstart.md similarity index 96% rename from docs/quickstart/java_api_quickstart.md rename to docs/dev/projectsetup/java_api_quickstart.md index 0d14d840056e5..03d75c130720b 100644 --- a/docs/quickstart/java_api_quickstart.md +++ b/docs/dev/projectsetup/java_api_quickstart.md @@ -1,7 +1,7 @@ --- title: "Project Template for Java" nav-title: Project Template for Java -nav-parent_id: start +nav-parent_id: projectsetup nav-pos: 0 --- -[Sample Project in Java]({{ site.baseurl }}/quickstart/java_api_quickstart.html) and [Sample Project in Scala]({{ site.baseurl }}/quickstart/scala_api_quickstart.html) are guides to setting up Maven and SBT projects and include simple implementations of a word count application. -[Monitoring Wikipedia Edits]({{ site.baseurl }}/quickstart/run_example_quickstart.html) is a more complete example of a streaming analytics application. +## Bundled Examples -[Building real-time dashboard applications with Apache Flink, Elasticsearch, and Kibana](https://www.elastic.co/blog/building-real-time-dashboard-applications-with-apache-flink-elasticsearch-and-kibana) is a blog post at elastic.co showing how to build a real-time dashboard solution for streaming data analytics using Apache Flink, Elasticsearch, and Kibana. +The Flink sources include many examples for Flink's different APIs: -The [Flink training website](http://training.data-artisans.com/) from data Artisans has a number of examples. See the hands-on sections, and the exercises. +* DataStream applications ({% gh_link flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples "Java" %} / {% gh_link flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples "Scala" %}) +* DataSet applications ({% gh_link flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java "Java" %} / {% gh_link flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala "Scala" %}) +* Table API / SQL queries ({% gh_link flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java "Java" %} / {% gh_link flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala "Scala" %}) -## Bundled Examples +These [instructions]({{ site.baseurl }}/dev/batch/examples.html#running-an-example) explain how to run the examples. + +## Examples on the Web + +There are also a few blog posts published online that discuss example applications: + +* [How to build stateful streaming applications with Apache Flink +](https://www.infoworld.com/article/3293426/big-data/how-to-build-stateful-streaming-applications-with-apache-flink.html) presents an event-driven application implemented with the DataStream API and two SQL queries for streaming analytics. -The Flink sources include a number of examples for both **streaming** ( [java](https://github.com/apache/flink/tree/master/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples) / [scala](https://github.com/apache/flink/tree/master/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples) ) and **batch** ( [java](https://github.com/apache/flink/tree/master/flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java) / [scala](https://github.com/apache/flink/tree/master/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala) ). These [instructions]({{ site.baseurl }}/dev/batch/examples.html#running-an-example) explain how to run the examples. +* [Building real-time dashboard applications with Apache Flink, Elasticsearch, and Kibana](https://www.elastic.co/blog/building-real-time-dashboard-applications-with-apache-flink-elasticsearch-and-kibana) is a blog post at elastic.co showing how to build a real-time dashboard solution for streaming data analytics using Apache Flink, Elasticsearch, and Kibana. +* The [Flink training website](http://training.data-artisans.com/) from data Artisans has a number of examples. Check out the hands-on sections and the exercises. \ No newline at end of file diff --git a/docs/start/building.md b/docs/flinkDev/building.md similarity index 99% rename from docs/start/building.md rename to docs/flinkDev/building.md index 654f00bc4db55..679f44416a337 100644 --- a/docs/start/building.md +++ b/docs/flinkDev/building.md @@ -1,6 +1,6 @@ --- title: Building Flink from Source -nav-parent_id: start +nav-parent_id: flinkdev nav-pos: 20 --- diff --git a/docs/redirects/example_quickstart.md b/docs/redirects/example_quickstart.md new file mode 100644 index 0000000000000..8795bedcb9fff --- /dev/null +++ b/docs/redirects/example_quickstart.md @@ -0,0 +1,24 @@ +--- +title: "DataStream API Tutorial" +layout: redirect +redirect: /tutorials/datastream_api.html +permalink: /quickstart/run_example_quickstart.html +--- + diff --git a/docs/redirects/ide_setup.md b/docs/redirects/ide_setup.md new file mode 100644 index 0000000000000..dfe747f5e4d37 --- /dev/null +++ b/docs/redirects/ide_setup.md @@ -0,0 +1,24 @@ +--- +title: "Importing Flink into an IDE" +layout: redirect +redirect: /flinkDev/ide_setup.html +permalink: /internals/ide_setup.html +--- + diff --git a/docs/redirects/java_quickstart.md b/docs/redirects/java_quickstart.md new file mode 100644 index 0000000000000..1b78e7518dc4e --- /dev/null +++ b/docs/redirects/java_quickstart.md @@ -0,0 +1,24 @@ +--- +title: "Project Template for Java" +layout: redirect +redirect: /dev/projectsetup/java_api_quickstart.html +permalink: /quickstart/java_api_quickstart.html +--- + diff --git a/docs/redirects/linking_with_flink.md b/docs/redirects/linking_with_flink.md index 1289487c9ff22..719a1d812f5a9 100644 --- a/docs/redirects/linking_with_flink.md +++ b/docs/redirects/linking_with_flink.md @@ -1,7 +1,7 @@ --- title: "Linking with Flink" layout: redirect -redirect: /start/dependencies.html +redirect: /dev/projectsetup/dependencies.html permalink: /dev/linking_with_flink.html --- diff --git a/docs/redirects/scala_shell.md b/docs/redirects/scala_shell.md index c17b263bc0823..8caa7daba90e0 100644 --- a/docs/redirects/scala_shell.md +++ b/docs/redirects/scala_shell.md @@ -1,7 +1,7 @@ --- title: "Scala Shell" layout: redirect -redirect: /dev/scala_shell.html +redirect: /ops/scala_shell.html permalink: /apis/scala_shell.html --- diff --git a/docs/redirects/windows.md b/docs/redirects/windows.md new file mode 100644 index 0000000000000..b769552268f8a --- /dev/null +++ b/docs/redirects/windows.md @@ -0,0 +1,24 @@ +--- +title: "Running Flink on Windows" +layout: redirect +redirect: /tutorials/flink_on_windows.html +permalink: /start/flink_on_windows.html +--- + diff --git a/docs/tutorials/api_tutorials.md b/docs/tutorials/api_tutorials.md new file mode 100644 index 0000000000000..2592e7b67f6e9 --- /dev/null +++ b/docs/tutorials/api_tutorials.md @@ -0,0 +1,25 @@ +--- +title: "API Tutorials" +nav-id: apitutorials +nav-title: 'API Tutorials' +nav-parent_id: tutorials +nav-pos: 10 +--- + \ No newline at end of file diff --git a/docs/quickstart/run_example_quickstart.md b/docs/tutorials/datastream_api.md similarity index 98% rename from docs/quickstart/run_example_quickstart.md rename to docs/tutorials/datastream_api.md index 1a793382edbd2..4f93646c6fe45 100644 --- a/docs/quickstart/run_example_quickstart.md +++ b/docs/tutorials/datastream_api.md @@ -1,7 +1,7 @@ --- -title: "Monitoring the Wikipedia Edit Stream" -nav-title: Monitoring Wikipedia Edits -nav-parent_id: examples +title: "DataStream API Tutorial" +nav-title: DataStream API +nav-parent_id: apitutorials nav-pos: 10 --- diff --git a/docs/quickstart/setup_quickstart.md b/docs/tutorials/local_setup.md similarity index 97% rename from docs/quickstart/setup_quickstart.md rename to docs/tutorials/local_setup.md index f862b4a2addb9..4442b546871f2 100644 --- a/docs/quickstart/setup_quickstart.md +++ b/docs/tutorials/local_setup.md @@ -1,8 +1,8 @@ --- -title: "Quickstart" -nav-title: ' Quickstart' -nav-parent_id: root -nav-pos: 2 +title: "Local Setup Tutorial" +nav-title: 'Local Setup' +nav-parent_id: setuptutorials +nav-pos: 10 --- \ No newline at end of file