Skip to content

Commit 9af96d4

Browse files
author
Giovanni Matteo Fumarola
committed
HADOOP-15707. Add IsActiveServlet to be used for Load Balancers. Contributed by Lukas Majercak.
1 parent e780556 commit 9af96d4

File tree

12 files changed

+307
-2
lines changed

12 files changed

+307
-2
lines changed

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/IsActiveServlet.java

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.http;
19+
20+
import javax.servlet.http.HttpServlet;
21+
import javax.servlet.http.HttpServletRequest;
22+
import javax.servlet.http.HttpServletResponse;
23+
import java.io.IOException;
24+
25+
/**
26+
* Used by Load Balancers to detect the active NameNode/ResourceManager/Router.
27+
*/
28+
public abstract class IsActiveServlet extends HttpServlet {
29+
30+
/** Default serial identifier. */
31+
private static final long serialVersionUID = 1L;
32+
33+
public static final String SERVLET_NAME = "isActive";
34+
public static final String PATH_SPEC = "/isActive";
35+
36+
public static final String RESPONSE_ACTIVE =
37+
"I am Active!";
38+
39+
public static final String RESPONSE_NOT_ACTIVE =
40+
"I am not Active!";
41+
42+
/**
43+
* Check whether this instance is the Active one.
44+
* @param req HTTP request
45+
* @param resp HTTP response to write to
46+
*/
47+
@Override
48+
public void doGet(
49+
final HttpServletRequest req, final HttpServletResponse resp)
50+
throws IOException {
51+
52+
// By default requests are persistent. We don't want long-lived connections
53+
// on server side.
54+
resp.addHeader("Connection", "close");
55+
56+
if (!isActive()) {
57+
// Report not SC_OK
58+
resp.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED,
59+
RESPONSE_NOT_ACTIVE);
60+
return;
61+
}
62+
resp.setStatus(HttpServletResponse.SC_OK);
63+
resp.getWriter().write(RESPONSE_ACTIVE);
64+
resp.getWriter().flush();
65+
}
66+
67+
/**
68+
* @return true if this instance is in Active HA state.
69+
*/
70+
protected abstract boolean isActive();
71+
}

‎hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestIsActiveServlet.java

+95
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.http;
19+
20+
21+
import org.junit.Before;
22+
import org.junit.Test;
23+
24+
import javax.servlet.http.HttpServletRequest;
25+
import javax.servlet.http.HttpServletResponse;
26+
27+
import java.io.ByteArrayOutputStream;
28+
import java.io.IOException;
29+
import java.io.PrintWriter;
30+
31+
import static org.junit.Assert.assertEquals;
32+
import static org.mockito.Matchers.anyInt;
33+
import static org.mockito.Matchers.anyString;
34+
import static org.mockito.Matchers.eq;
35+
import static org.mockito.Mockito.atLeastOnce;
36+
import static org.mockito.Mockito.mock;
37+
import static org.mockito.Mockito.never;
38+
import static org.mockito.Mockito.verify;
39+
import static org.mockito.Mockito.when;
40+
41+
42+
/**
43+
* Test if the {@link IsActiveServlet} returns the right answer if the
44+
* underlying service is active.
45+
*/
46+
public class TestIsActiveServlet {
47+
48+
private IsActiveServlet servlet;
49+
private HttpServletRequest req;
50+
private HttpServletResponse resp;
51+
private ByteArrayOutputStream respOut;
52+
53+
@Before
54+
public void setUp() throws Exception {
55+
req = mock(HttpServletRequest.class);
56+
resp = mock(HttpServletResponse.class);
57+
respOut = new ByteArrayOutputStream();
58+
PrintWriter writer = new PrintWriter(respOut);
59+
when(resp.getWriter()).thenReturn(writer);
60+
}
61+
62+
@Test
63+
public void testSucceedsOnActive() throws IOException {
64+
servlet = new IsActiveServlet() {
65+
@Override
66+
protected boolean isActive() {
67+
return true;
68+
}
69+
};
70+
71+
String response = doGet();
72+
verify(resp, never()).sendError(anyInt(), anyString());
73+
assertEquals(IsActiveServlet.RESPONSE_ACTIVE, response);
74+
}
75+
76+
@Test
77+
public void testFailsOnInactive() throws IOException {
78+
servlet = new IsActiveServlet() {
79+
@Override
80+
protected boolean isActive() {
81+
return false;
82+
}
83+
};
84+
85+
doGet();
86+
verify(resp, atLeastOnce()).sendError(
87+
eq(HttpServletResponse.SC_METHOD_NOT_ALLOWED),
88+
eq(IsActiveServlet.RESPONSE_NOT_ACTIVE));
89+
}
90+
91+
private String doGet() throws IOException {
92+
servlet.doGet(req, resp);
93+
return new String(respOut.toByteArray(), "UTF-8");
94+
}
95+
}

‎hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/IsRouterActiveServlet.java

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.hdfs.server.federation.router;
19+
20+
import org.apache.hadoop.http.IsActiveServlet;
21+
22+
import javax.servlet.ServletContext;
23+
24+
/**
25+
* Detect if the Router is active and ready to serve requests.
26+
*/
27+
public class IsRouterActiveServlet extends IsActiveServlet {
28+
29+
@Override
30+
protected boolean isActive() {
31+
final ServletContext context = getServletContext();
32+
final Router router = RouterHttpServer.getRouterFromContext(context);
33+
final RouterServiceState routerState = router.getRouterState();
34+
35+
return routerState == RouterServiceState.RUNNING;
36+
}
37+
}

‎hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java

+9
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@
2727
import org.apache.hadoop.http.HttpServer2;
2828
import org.apache.hadoop.service.AbstractService;
2929

30+
import javax.servlet.ServletContext;
31+
3032
/**
3133
* Web interface for the {@link Router}. It exposes the Web UI and the WebHDFS
3234
* methods from {@link RouterWebHdfsMethods}.
@@ -116,6 +118,9 @@ protected void serviceStop() throws Exception {
116118
private static void setupServlets(
117119
HttpServer2 httpServer, Configuration conf) {
118120
// TODO Add servlets for FSCK, etc
121+
httpServer.addInternalServlet(IsRouterActiveServlet.SERVLET_NAME,
122+
IsRouterActiveServlet.PATH_SPEC,
123+
IsRouterActiveServlet.class);
119124
}
120125

121126
public InetSocketAddress getHttpAddress() {
@@ -125,4 +130,8 @@ public InetSocketAddress getHttpAddress() {
125130
public InetSocketAddress getHttpsAddress() {
126131
return this.httpsAddress;
127132
}
133+
134+
public static Router getRouterFromContext(ServletContext context) {
135+
return (Router)context.getAttribute(NAMENODE_ATTRIBUTE_KEY);
136+
}
128137
}

‎hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ Each Router has two roles:
6262
#### Federated interface
6363
The Router receives a client request, checks the State Store for the correct subcluster, and forwards the request to the active NameNode of that subcluster.
6464
The reply from the NameNode then flows in the opposite direction.
65-
The Routers are stateless and can be behind a load balancer.
65+
The Routers are stateless and can be behind a load balancer. For health checking, you can use /isActive endpoint as a health probe (e.g. http://ROUTER_HOSTNAME:ROUTER_PORT/isActive).
6666
For performance, the Router also caches remote mount table entries and the state of the subclusters.
6767
To make sure that changes have been propagated to all Routers, each Router heartbeats its state to the State Store.
6868

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/IsNameNodeActiveServlet.java

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.hdfs.server.namenode;
19+
20+
import org.apache.hadoop.http.IsActiveServlet;
21+
22+
/**
23+
* Used by Load Balancers to find the active NameNode.
24+
*/
25+
public class IsNameNodeActiveServlet extends IsActiveServlet {
26+
27+
@Override
28+
protected boolean isActive() {
29+
NameNode namenode = NameNodeHttpServer.getNameNodeFromContext(
30+
getServletContext());
31+
return namenode.isActiveState();
32+
}
33+
}

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java

+3
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,9 @@ private static void setupServlets(HttpServer2 httpServer, Configuration conf) {
294294
true);
295295
httpServer.addInternalServlet("imagetransfer", ImageServlet.PATH_SPEC,
296296
ImageServlet.class, true);
297+
httpServer.addInternalServlet(IsNameNodeActiveServlet.SERVLET_NAME,
298+
IsNameNodeActiveServlet.PATH_SPEC,
299+
IsNameNodeActiveServlet.class);
297300
}
298301

299302
static FSImage getFsImageFromContext(ServletContext context) {

‎hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md

+8
Original file line numberDiff line numberDiff line change
@@ -423,6 +423,14 @@ This guide describes high-level uses of each of these subcommands. For specific
423423
**Note:** This is not yet implemented, and at present will always return
424424
success, unless the given NameNode is completely down.
425425

426+
427+
### Load Balancer Setup
428+
429+
If you are running a set of NameNodes behind a Load Balancer (e.g. [Azure](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-custom-probe-overview) or [AWS](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-healthchecks.html) ) and would like the Load Balancer to point to the active NN, you can use the /isActive HTTP endpoint as a health probe.
430+
http://NN_HOSTNAME/isActive will return a 200 status code response if the NN is in Active HA State, 405 otherwise.
431+
432+
433+
426434
Automatic Failover
427435
------------------
428436

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/IsResourceManagerActiveServlet.java

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.yarn.server.resourcemanager;
19+
20+
import org.apache.hadoop.http.IsActiveServlet;
21+
import org.apache.hadoop.ha.HAServiceProtocol;
22+
23+
/**
24+
* Used by Load Balancers to find the active ResourceManager.
25+
*/
26+
public class IsResourceManagerActiveServlet extends IsActiveServlet {
27+
28+
public static final String RM_ATTRIBUTE = "rm";
29+
30+
@Override
31+
protected boolean isActive() {
32+
ResourceManager rm = (ResourceManager)
33+
getServletContext().getAttribute(RM_ATTRIBUTE);
34+
RMContext rmContext = rm.getRMContext();
35+
HAServiceProtocol.HAServiceState state = rmContext.getHAServiceState();
36+
return state == HAServiceProtocol.HAServiceState.ACTIVE;
37+
}
38+
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java

+5
Original file line numberDiff line numberDiff line change
@@ -1206,6 +1206,11 @@ protected void startWepApp() {
12061206
}
12071207
}
12081208

1209+
builder.withAttribute(IsResourceManagerActiveServlet.RM_ATTRIBUTE, this);
1210+
builder.withServlet(IsResourceManagerActiveServlet.SERVLET_NAME,
1211+
IsResourceManagerActiveServlet.PATH_SPEC,
1212+
IsResourceManagerActiveServlet.class);
1213+
12091214
webApp = builder.start(new RMWebApp(this), uiWebAppContext);
12101215
}
12111216

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737

3838
import org.apache.hadoop.conf.Configuration;
3939
import org.apache.hadoop.http.HtmlQuoting;
40+
import org.apache.hadoop.http.IsActiveServlet;
4041
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
4142
import org.apache.hadoop.yarn.api.records.ApplicationId;
4243
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -68,7 +69,7 @@ public class RMWebAppFilter extends GuiceContainer {
6869

6970
// define a set of URIs which do not need to do redirection
7071
private static final Set<String> NON_REDIRECTED_URIS = Sets.newHashSet(
71-
"/conf", "/stacks", "/logLevel", "/logs");
72+
"/conf", "/stacks", "/logLevel", "/logs", IsActiveServlet.PATH_SPEC);
7273
private String path;
7374
private boolean ahsEnabled;
7475
private String ahsPageURLPrefix;

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerHA.md

+5
Original file line numberDiff line numberDiff line change
@@ -144,3 +144,8 @@ Assuming a standby RM is up and running, the Standby automatically redirects all
144144
### Web Services
145145

146146
Assuming a standby RM is up and running, RM web-services described at [ResourceManager REST APIs](./ResourceManagerRest.html) when invoked on a standby RM are automatically redirected to the Active RM.
147+
148+
### Load Balancer Setup
149+
150+
If you are running a set of ResourceManagers behind a Load Balancer (e.g. [Azure](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-custom-probe-overview) or [AWS](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-healthchecks.html) ) and would like the Load Balancer to point to the active RM, you can use the /isActive HTTP endpoint as a health probe.
151+
http://RM_HOSTNAME/isActive will return a 200 status code response if the RM is in Active HA State, 405 otherwise.

0 commit comments

Comments
 (0)