Skip to content

Commit

Permalink
Change -base_port to -baseport.
Browse files Browse the repository at this point in the history
Added -ea and -baseport options to hadoop driver.
  • Loading branch information
tomkraljevic committed Jan 19, 2014
1 parent 0d6df0a commit d4ee039
Show file tree
Hide file tree
Showing 4 changed files with 31 additions and 5 deletions.
2 changes: 1 addition & 1 deletion R/tests/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ def start(self):
"-ea",
"-jar", self.h2o_jar,
"-name", self.cloud_name,
"-base_port", str(self.my_base_port)]
"-baseport", str(self.my_base_port)]
self.output_file_name = \
os.path.join(self.output_dir, "java_" + str(self.cloud_num) + "_" + str(self.node_num) + ".out")
f = open(self.output_file_name, "w")
Expand Down
19 changes: 18 additions & 1 deletion hadoop/src/main/java/water/hadoop/h2odriver.java
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,9 @@ public class h2odriver extends Configured implements Tool {
static String clusterReadyFileName = null;
static int cloudFormationTimeoutSeconds = DEFAULT_CLOUD_FORMATION_TIMEOUT_SECONDS;
static int nthreads = -1;
static int basePort = -1;
static boolean beta = false;
static boolean enableExceptions = false;

// Runtime state that might be touched by different threads.
volatile ServerSocket driverCallbackSocket = null;
Expand Down Expand Up @@ -367,6 +369,8 @@ static void usage() {
" [-extramempercent <0 to 20>]\n" +
" -n | -nodes <number of H2O nodes (i.e. mappers) to create>\n" +
" [-nthreads <maximum typical worker threads, i.e. cpus to use>]\n" +
" [-baseport <starting HTTP port for H2O nodes; default is 54321>]\n" +
" [-ea]\n" +
" -o | -output <hdfs output dir>\n" +
"\n" +
"Notes:\n" +
Expand Down Expand Up @@ -494,9 +498,19 @@ else if (s.equals("-nthreads")) {
i++; if (i >= args.length) { usage(); }
nthreads = Integer.parseInt(args[i]);
}
else if (s.equals("-baseport")) {
i++; if (i >= args.length) { usage(); }
basePort = Integer.parseInt(args[i]);
if ((basePort < 0) || (basePort > 65535)) {
error("Base port must be between 1 and 65535");
}
}
else if (s.equals("-beta")) {
beta = true;
}
else if (s.equals("-ea")) {
enableExceptions = true;
}
else {
error("Unrecognized option " + s);
}
Expand Down Expand Up @@ -701,7 +715,7 @@ private int run2(String[] args) throws Exception {
conf.set("mapreduce.map.memory.mb", mapreduceMapMemoryMb);

// MRv1 standard options, but also required for YARN.
String mapChildJavaOpts = "-Xms" + mapperXmx + " -Xmx" + mapperXmx;
String mapChildJavaOpts = "-Xms" + mapperXmx + " -Xmx" + mapperXmx + (enableExceptions ? " -ea" : "");
conf.set("mapred.child.java.opts", mapChildJavaOpts);
conf.set("mapred.map.child.java.opts", mapChildJavaOpts); // MapR 2.x requires this.

Expand Down Expand Up @@ -744,6 +758,9 @@ private int run2(String[] args) throws Exception {
if (nthreads >= 0) {
conf.set(h2omapper.H2O_NTHREADS_KEY, Integer.toString(nthreads));
}
if (basePort >= 0) {
conf.set(h2omapper.H2O_BASE_PORT_KEY, Integer.toString(basePort));
}
if (beta) {
conf.set(h2omapper.H2O_BETA_KEY, "-beta");
}
Expand Down
9 changes: 9 additions & 0 deletions hadoop/src/main/java/water/hadoop/h2omapper.java
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ public class h2omapper extends Mapper<Text, Text, Text, Text> {
final static public String H2O_NETWORK_KEY = "h2o.network";
final static public String H2O_BETA_KEY = "h2o.beta";
final static public String H2O_NTHREADS_KEY = "h2o.nthreads";
final static public String H2O_BASE_PORT_KEY = "h2o.baseport";

static EmbeddedH2OConfig _embeddedH2OConfig;

Expand Down Expand Up @@ -345,6 +346,7 @@ private int run2(Context context) throws IOException, InterruptedException {
String driverPortString = conf.get(H2O_DRIVER_PORT_KEY);
String network = conf.get(H2O_NETWORK_KEY);
String nthreadsString = conf.get(H2O_NTHREADS_KEY);
String basePortString = conf.get(H2O_BASE_PORT_KEY);
String betaString = conf.get(H2O_BETA_KEY);

ServerSocket ss = new ServerSocket();
Expand Down Expand Up @@ -373,6 +375,13 @@ private int run2(Context context) throws IOException, InterruptedException {
argsList.add(Integer.toString(nthreads));
}
}
if (basePortString != null) {
if (basePortString.length() > 0) {
argsList.add("-baseport");
int basePort = Integer.parseInt(basePortString);
argsList.add(Integer.toString(basePort));
}
}
if (betaString != null) {
if (betaString.length() > 0) {
argsList.add(betaString);
Expand Down
6 changes: 3 additions & 3 deletions src/main/java/water/H2O.java
Original file line number Diff line number Diff line change
Expand Up @@ -736,7 +736,7 @@ public static class H2OEmptyCompleter extends H2OCountedCompleter{
public static class OptArgs extends Arguments.Opt {
public String name; // set_cloud_name_and_mcast()
public String flatfile; // set_cloud_name_and_mcast()
public int base_port; // starting number to search for open ports
public int baseport; // starting number to search for open ports
public int port; // set_cloud_name_and_mcast()
public String ip; // Named IP4/IP6 address instead of the default
public String network; // Network specification for acceptable interfaces to bind to.
Expand Down Expand Up @@ -905,8 +905,8 @@ public static void main( String[] args ) {

printAndLogVersion();

if (OPT_ARGS.base_port != 0) {
DEFAULT_PORT = OPT_ARGS.base_port;
if (OPT_ARGS.baseport != 0) {
DEFAULT_PORT = OPT_ARGS.baseport;
}

// Get ice path before loading Log or Persist class
Expand Down

0 comments on commit d4ee039

Please sign in to comment.