Moved WebInfoServer and WebInterfaceServer resources into resource fo…

…lders of respective projects. Jetty uses the jar as base directory.

DEPENDENCIES

@@ -165,7 +165,7 @@ The Apache Flink project packages the following fonts under the
 Open Font License (OFT) - http://scripts.sil.org/OFL/
 
  - Font Awesome (http://fortawesome.github.io/Font-Awesome/) - Created by Dave Gandy
-   -> fonts in "flink-runtime/resources/web-docs-infoserver/font-awesome/fonts"
+   -> fonts in "flink-runtime/src/main/resources/web-docs-infoserver/font-awesome/fonts"
 
 
 

LICENSE

@@ -293,5 +293,5 @@ The Apache Flink project bundles the following fonts under the
 Open Font License (OFT) - http://scripts.sil.org/OFL/
 
  - Font Awesome (http://fortawesome.github.io/Font-Awesome/) - Created by Dave Gandy
-   -> fonts in "flink-runtime/resources/web-docs-infoserver/font-awesome/fonts"
+   -> fonts in "flink-runtime/src/main/resources/web-docs-infoserver/font-awesome/fonts"
 

flink-addons/flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationMaster.scala

@@ -69,8 +69,6 @@ object ApplicationMaster{
           val taskManagerCount = env.get(Client.ENV_TM_COUNT).toInt
           val slots = env.get(Client.ENV_SLOTS).toInt
           val dynamicPropertiesEncodedString = env.get(Client.ENV_DYNAMIC_PROPERTIES)
-          val localWebInterfaceDir =
-            s"$currDir/resources/${ConfigConstants.DEFAULT_JOB_MANAGER_WEB_PATH_NAME}"
 
           val appNumber = env.get(Client.ENV_APP_NUMBER).toInt
 
@@ -84,8 +82,8 @@ object ApplicationMaster{
           val jobManagerWebPort = GlobalConfiguration.getInteger(ConfigConstants
             .JOB_MANAGER_WEB_PORT_KEY, ConfigConstants.DEFAULT_JOB_MANAGER_WEB_FRONTEND_PORT)
 
-          generateConfigurationFile(currDir, ownHostname, jobManagerPort, localWebInterfaceDir,
-            jobManagerWebPort, logDirs, slots, taskManagerCount, dynamicPropertiesEncodedString)
+          generateConfigurationFile(currDir, ownHostname, jobManagerPort, jobManagerWebPort,
+            logDirs, slots, taskManagerCount, dynamicPropertiesEncodedString)
 
           val (system, actor) = startJobManager(currDir)
 
@@ -114,8 +112,9 @@ object ApplicationMaster{
   }
 
   def generateConfigurationFile(currDir: String, ownHostname: String, jobManagerPort: Int,
-                                localWebInterfaceDir: String, jobManagerWebPort: Int, logDirs:
-  String, slots: Int, taskManagerCount: Int, dynamicPropertiesEncodedString: String): Unit = {
+                               jobManagerWebPort: Int, logDirs: String, slots: Int,
+                               taskManagerCount: Int, dynamicPropertiesEncodedString: String)
+  : Unit = {
     val output = new PrintWriter(new BufferedWriter(new FileWriter(s"$currDir/flink-conf-modified" +
       s".yaml")))
 
@@ -128,7 +127,6 @@ object ApplicationMaster{
 
     output.println(s"${ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY}: $ownHostname")
     output.println(s"${ConfigConstants.JOB_MANAGER_IPC_PORT_KEY}: $jobManagerPort")
-    output.println(s"${ConfigConstants.JOB_MANAGER_WEB_ROOT_PATH_KEY}: $localWebInterfaceDir")
     output.println(s"${ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY}: $logDirs")
     output.println(s"${ConfigConstants.JOB_MANAGER_WEB_PORT_KEY}: $jobManagerWebPort")
 
@@ -151,8 +149,6 @@ object ApplicationMaster{
   }
 
   def startJobManager(currDir: String): (ActorSystem, ActorRef) = {
-//    Utils.copyJarContents(s"resources/${ConfigConstants.DEFAULT_JOB_MANAGER_WEB_PATH_NAME}",
-//      classOf[YarnMaster].getProtectionDomain.getCodeSource.getLocation.getPath)
 
     val pathToConfig = s"$currDir/flink-conf.modified.yaml"
     val args = Array[String]("--configDir", pathToConfig)

flink-addons/pom.xml

@@ -34,11 +34,11 @@ under the License.
 	<packaging>pom</packaging>
 
 	<modules>
-		<!--<module>flink-avro</module>-->
-		<!--<module>flink-jdbc</module>-->
-		<!--<module>flink-spargel</module>-->
-		<!--<module>flink-hadoop-compatibility</module>-->
-		<!--<module>flink-streaming</module>-->
+		<module>flink-avro</module>
+		<module>flink-jdbc</module>
+		<module>flink-spargel</module>
+		<module>flink-hadoop-compatibility</module>
+		<module>flink-streaming</module>
 		<module>flink-hbase</module>
 	</modules>
 

flink-clients/src/main/java/org/apache/flink/client/web/WebInterfaceServer.java

@@ -22,6 +22,7 @@
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.net.URL;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -45,6 +46,9 @@
  * configures an embedded jetty server.
  */
 public class WebInterfaceServer {
+
+	private static final String WEB_ROOT_DIR = "web-docs";
+
 	/**
 	 * The log for this class.
 	 */
@@ -81,20 +85,15 @@ public WebInterfaceServer(Configuration nepheleConfig, int port)
 		// get base path of Flink installation
 		String basePath = nepheleConfig.getString(ConfigConstants.FLINK_BASE_DIR_PATH_KEY,"");
 
-		File webDir;
 		File tmpDir;
 		File uploadDir;
 		File planDumpDir;
 
-		String webDirPath = config.getString(ConfigConstants.WEB_ROOT_PATH_KEY,
-			ConfigConstants.DEFAULT_WEB_ROOT_DIR);
-
-		if(webDirPath.startsWith("/")) {
-			// absolute path
-			webDir = new File(webDirPath);
-		} else {
-			// path relative to base dir
-			webDir = new File(basePath+"/"+webDirPath);
+		URL webRootDir = this.getClass().getClassLoader().getResource(WEB_ROOT_DIR);
+
+		if(webRootDir == null){
+			throw new FileNotFoundException("Cannot start web interface server because the web " +
+					"root dir " + WEB_ROOT_DIR + " is not included in the jar.");
 		}
 
 		String tmpDirPath = config.getString(ConfigConstants.WEB_TMP_DIR_KEY,
@@ -131,7 +130,8 @@ public WebInterfaceServer(Configuration nepheleConfig, int port)
 		}
 
 		if (LOG.isInfoEnabled()) {
-			LOG.info("Setting up web frontend server, using web-root directory '" + webDir.getAbsolutePath() + "'.");
+			LOG.info("Setting up web frontend server, using web-root directory '" +
+					webRootDir.toExternalForm()	+ "'.");
 			LOG.info("Web frontend server will store temporary files in '" + tmpDir.getAbsolutePath()
 				+ "', uploaded jobs in '" + uploadDir.getAbsolutePath() + "', plan-json-dumps in '"
 				+ planDumpDir.getAbsolutePath() + "'.");
@@ -144,12 +144,6 @@ public WebInterfaceServer(Configuration nepheleConfig, int port)
 
 		server = new Server(port);
 
-		// ensure that the directory with the web documents exists
-		if (!webDir.exists()) {
-			throw new FileNotFoundException("The directory containing the web documents does not exist: "
-				+ webDir.getAbsolutePath());
-		}
-
 		// ensure, that all the directories exist
 		checkAndCreateDirectories(tmpDir, true);
 		checkAndCreateDirectories(uploadDir, true);
@@ -178,7 +172,7 @@ public WebInterfaceServer(Configuration nepheleConfig, int port)
 		// ----- the handler serving all the static files -----
 		ResourceHandler resourceHandler = new ResourceHandler();
 		resourceHandler.setDirectoriesListed(false);
-		resourceHandler.setResourceBase(webDir.getAbsolutePath());
+		resourceHandler.setResourceBase(webRootDir.toExternalForm());
 
 		// ----- add the handlers to the list handler -----
 		HandlerList handlers = new HandlerList();

flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java

@@ -249,11 +249,6 @@ public final class ConfigConstants {
 	 */
 	public static final String JOB_MANAGER_WEB_PORT_KEY = "jobmanager.web.port";
 
-	/**
-	 * The parameter defining the directory containing the web documents for the jobmanager web frontend.
-	 */
-	public static final String JOB_MANAGER_WEB_ROOT_PATH_KEY = "jobmanager.web.rootpath";
-
 	/**
 	 * The config parameter defining the path to the htaccess file protecting the web frontend.
 	 */
@@ -274,11 +269,6 @@ public final class ConfigConstants {
 	 */
 	public static final String WEB_FRONTEND_PORT_KEY = "webclient.port";
 
-	/**
-	 * The config parameter defining the directory containing the web documents.
-	 */
-	public static final String WEB_ROOT_PATH_KEY = "webclient.rootpath";
-
 	/**
 	 * The config parameter defining the temporary data directory for the web client.
 	 */
@@ -299,9 +289,6 @@ public final class ConfigConstants {
 	 */
 	public static final String WEB_ACCESS_FILE_KEY = "webclient.access";
 
-	// ----------------------------- YARN Client ----------------------------
-
-	public static final String YARN_AM_PRC_PORT = "yarn.am.rpc.port";
 
 	// ------------------------------ AKKA ------------------------------------
 
@@ -549,16 +536,6 @@ public final class ConfigConstants {
 	 */
 	public static final int DEFAULT_JOB_MANAGER_WEB_FRONTEND_PORT = 8081;
 
-	/**
-	 * The default directory name of the info server
-	 */
-	public static final String DEFAULT_JOB_MANAGER_WEB_PATH_NAME = "web-docs-infoserver";
-
-	/**
-	 * The default path of the directory for info server containing the web documents.
-	 */
-	public static final String DEFAULT_JOB_MANAGER_WEB_ROOT_PATH = "./resources/"+DEFAULT_JOB_MANAGER_WEB_PATH_NAME+"/";
-
 	/**
 	 * The default number of archived jobs for the jobmanager
 	 */
@@ -572,11 +549,6 @@ public final class ConfigConstants {
 	 */
 	public static final int DEFAULT_WEBCLIENT_PORT = 8080;
 
-	/**
-	 * The default path of the directory containing the web documents.
-	 */
-	public static final String DEFAULT_WEB_ROOT_DIR = "./resources/web-docs/";
-
 	/**
 	 * The default directory to store temporary objects (e.g. during file uploads).
 	 */
@@ -599,10 +571,6 @@ public final class ConfigConstants {
 	 */
 	public static final String DEFAULT_WEB_ACCESS_FILE_PATH = null;
 
-	// ----------------------------- YARN ----------------------------
-
-	public static final int DEFAULT_YARN_AM_RPC_PORT = 10245;
-
 	// ------------------------------ Akka Values ------------------------------
 
 	public static String DEFAULT_AKKA_TRANSPORT_HEARTBEAT_INTERVAL = "1000 ms";
@@ -635,19 +603,6 @@ public final class ConfigConstants {
 	 */
 	public static final String LOCAL_INSTANCE_MANAGER_NUMBER_TASK_MANAGER = "localinstancemanager.numtaskmanager";
 
-
-	// ----------------------------- Deprecated --------------------------------
-
-	/**
-	 * The default definition for an instance type, if no other configuration is provided.
-	 */
-	public static final String DEFAULT_INSTANCE_TYPE = "default,1,1,1,1,0"; // minimalistic instance type until "cloud" model is fully removed.
-
-	/**
-	 * The default index for the default instance type.
-	 */
-	public static final int DEFAULT_DEFAULT_INSTANCE_TYPE_INDEX = 1;
-
 	// ------------------------------------------------------------------------
 
 	/**

flink-dist/pom.xml

@@ -31,7 +31,7 @@ under the License.
 
 	<artifactId>flink-dist</artifactId>
 	<name>flink-dist</name>
-	<packaging>pom</packaging>
+	<packaging>jar</packaging>
 
 	<dependencies>
 		<!-- BINARIES -->
@@ -162,35 +162,71 @@ under the License.
 			<build>
 				<plugins>
 					<plugin>
-						<artifactId>maven-assembly-plugin</artifactId>
-						<version>2.4</version><!--$NO-MVN-MAN-VER$-->
+						<groupId>org.apache.maven.plugins</groupId>
+						<artifactId>maven-shade-plugin</artifactId>
+						<version>2.3</version>
 						<executions>
-							<!--  Uber-jar -->
 							<execution>
-								<id>uber-jar</id>
 								<phase>package</phase>
 								<goals>
-									<goal>single</goal>
+									<goal>shade</goal>
 								</goals>
-								<configuration>
-									<archiverConfig>
-										<!-- https://jira.codehaus.org/browse/MASSEMBLY-449 -->
-										<fileMode>420</fileMode> <!-- 420(dec) = 644(oct) -->
-										<directoryMode>493</directoryMode> <!-- 493(dec) = 755(oct) -->
-										<defaultDirectoryMode>493</defaultDirectoryMode>
-									</archiverConfig>
-									<archive>
-										<manifest>
-											<mainClass>org.apache.flink.yarn.Client</mainClass>
-											<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
-											<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
-										</manifest>
-									</archive>
-									<descriptors>
-										<descriptor>src/main/assemblies/yarn-uberjar.xml</descriptor>
-									</descriptors>
+								<configuration combine.self="override">
+									<shadedArtifactAttached>false</shadedArtifactAttached>
+									<finalName>${project.artifactId}-${project.version}-yarn-uberjar</finalName>
+									<artifactSet>
+										<excludes>
+											<exclude>org.apache.flink:flink-java-examples</exclude>
+											<exclude>org.apache.flink:flink-scala-examples</exclude>
+											<exclude>org.apache.flink:flink-streaming-examples</exclude>
+										</excludes>
+									</artifactSet>
+									<transformers>
+										<transformer
+												implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+											<resource>reference.conf</resource>
+										</transformer>
+										<transformer
+												implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+											<manifestEntries>
+												<Main-Class>org.apache.flink.yarn.Client</Main-Class>
+											</manifestEntries>
+										</transformer>
+									</transformers>
 								</configuration>
 							</execution>
+						</executions>
+					</plugin>
+					<plugin>
+						<artifactId>maven-assembly-plugin</artifactId>
+						<version>2.4</version>
+						<executions>
+							<!--&lt;!&ndash;  Uber-jar &ndash;&gt;-->
+							<!--<execution>-->
+								<!--<id>uber-jar</id>-->
+								<!--<phase>package</phase>-->
+								<!--<goals>-->
+									<!--<goal>single</goal>-->
+								<!--</goals>-->
+								<!--<configuration>-->
+									<!--<archiverConfig>-->
+										<!--&lt;!&ndash; https://jira.codehaus.org/browse/MASSEMBLY-449 &ndash;&gt;-->
+										<!--<fileMode>420</fileMode> &lt;!&ndash; 420(dec) = 644(oct) &ndash;&gt;-->
+										<!--<directoryMode>493</directoryMode> &lt;!&ndash; 493(dec) = 755(oct) &ndash;&gt;-->
+										<!--<defaultDirectoryMode>493</defaultDirectoryMode>-->
+									<!--</archiverConfig>-->
+									<!--<archive>-->
+										<!--<manifest>-->
+											<!--<mainClass>org.apache.flink.yarn.Client</mainClass>-->
+											<!--<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>-->
+											<!--<addDefaultImplementationEntries>true</addDefaultImplementationEntries>-->
+										<!--</manifest>-->
+									<!--</archive>-->
+									<!--<descriptors>-->
+										<!--<descriptor>src/main/assemblies/yarn-uberjar.xml</descriptor>-->
+									<!--</descriptors>-->
+								<!--</configuration>-->
+							<!--</execution>-->
 
 							<!-- yarn bin directory -->
 							<execution>

flink-dist/src/main/assemblies/bin.xml

@@ -103,26 +103,6 @@ under the License.
 			<fileMode>0644</fileMode>
 		</fileSet>
 
-		<fileSet>
-			<!-- copy files for compiler web frontend -->
-			<directory>../flink-clients/resources</directory>
-			<outputDirectory>resources</outputDirectory>
-			<fileMode>0644</fileMode>
-			<excludes>
-				<exclude>*etc/users</exclude>
-			</excludes>
-		</fileSet>
-
-		<fileSet>
-			<!-- copy files for Jobmanager web frontend -->
-			<directory>../flink-runtime/resources</directory>
-			<outputDirectory>resources</outputDirectory>
-			<fileMode>0644</fileMode>
-			<excludes>
-				<exclude>*etc/users</exclude>
-			</excludes>
-		</fileSet>
-
 		<!-- copy the tools -->
 		<fileSet>
 			<directory>src/main/flink-bin/tools</directory>

flink-dist/src/main/assemblies/yarn-uberjar.xml

@@ -54,16 +54,6 @@ under the License.
 	</files>
 
 	<fileSets>
-		<!-- copy files for Jobmanager web frontend -->
-		<fileSet>
-			<directory>../nephele/nephele-server/resources</directory>
-			<outputDirectory>resources</outputDirectory>
-			<fileMode>0644</fileMode>
-			<excludes>
-				<exclude>*etc/users</exclude>
-			</excludes>
-		</fileSet>
-
 		<!-- copy files for Jobmanager web frontend -->
 		<fileSet>
 			<directory>../flink-runtime/resources</directory>