Skip to content

Commit

Permalink
HIVE-16389: Allow HookContext to access SQLOperationDisplay (Sahil Ta…
Browse files Browse the repository at this point in the history 
…kiar, reviewed by Sergio Pena)
  • Loading branch information
@sahilTakiar
sahilTakiar authored and Sergio Pena committed May 5, 2017
1 parent 44804d8 commit 1fecb81
Show file tree
Hide file tree
Showing 11 changed files with 262 additions and 139 deletions.
40 changes: 20 additions & 20 deletions itests/hive-unit/src/test/java/org/apache/hive/service/cli/session/TestQueryDisplay.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@

import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.QueryDisplay;
import org.apache.hadoop.hive.ql.QueryInfo;
import org.apache.hadoop.hive.ql.plan.api.StageType;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.service.cli.OperationHandle;
import org.apache.hive.service.cli.SessionHandle;
import org.apache.hive.service.cli.operation.SQLOperationDisplay;
import org.apache.hive.service.rpc.thrift.TProtocolVersion;
import org.apache.hive.service.server.HiveServer2;
import org.apache.hive.tmpl.QueryProfileTmpl;
Expand Down Expand Up @@ -69,25 +69,25 @@ public void testQueryDisplay() throws Exception {
OperationHandle opHandle2 = session.executeStatement("show tables", null);


List<SQLOperationDisplay> liveSqlOperations, historicSqlOperations;
liveSqlOperations = sessionManager.getOperationManager().getLiveSqlOperations();
historicSqlOperations = sessionManager.getOperationManager().getHistoricalSQLOperations();
List<QueryInfo> liveSqlOperations, historicSqlOperations;
liveSqlOperations = sessionManager.getOperationManager().getLiveQueryInfos();
historicSqlOperations = sessionManager.getOperationManager().getHistoricalQueryInfos();
Assert.assertEquals(liveSqlOperations.size(), 2);
Assert.assertEquals(historicSqlOperations.size(), 0);
verifyDDL(liveSqlOperations.get(0), "show databases", opHandle1.getHandleIdentifier().toString(), false);
verifyDDL(liveSqlOperations.get(1),"show tables", opHandle2.getHandleIdentifier().toString(), false);

session.closeOperation(opHandle1);
liveSqlOperations = sessionManager.getOperationManager().getLiveSqlOperations();
historicSqlOperations = sessionManager.getOperationManager().getHistoricalSQLOperations();
liveSqlOperations = sessionManager.getOperationManager().getLiveQueryInfos();
historicSqlOperations = sessionManager.getOperationManager().getHistoricalQueryInfos();
Assert.assertEquals(liveSqlOperations.size(), 1);
Assert.assertEquals(historicSqlOperations.size(), 1);
verifyDDL(historicSqlOperations.get(0),"show databases", opHandle1.getHandleIdentifier().toString(), true);
verifyDDL(liveSqlOperations.get(0),"show tables", opHandle2.getHandleIdentifier().toString(), false);

session.closeOperation(opHandle2);
liveSqlOperations = sessionManager.getOperationManager().getLiveSqlOperations();
historicSqlOperations = sessionManager.getOperationManager().getHistoricalSQLOperations();
liveSqlOperations = sessionManager.getOperationManager().getLiveQueryInfos();
historicSqlOperations = sessionManager.getOperationManager().getHistoricalQueryInfos();
Assert.assertEquals(liveSqlOperations.size(), 0);
Assert.assertEquals(historicSqlOperations.size(), 2);
verifyDDL(historicSqlOperations.get(1),"show databases", opHandle1.getHandleIdentifier().toString(), true);
Expand Down Expand Up @@ -123,23 +123,23 @@ public void testWebUI() throws Exception {
session.close();
}

private void verifyDDL(SQLOperationDisplay display, String stmt, String handle, boolean finished) {
private void verifyDDL(QueryInfo queryInfo, String stmt, String handle, boolean finished) {

Assert.assertEquals(display.getUserName(), "testuser");
Assert.assertEquals(display.getExecutionEngine(), "mr");
Assert.assertEquals(display.getOperationId(), handle);
Assert.assertTrue(display.getBeginTime() > 0 && display.getBeginTime() <= System.currentTimeMillis());
Assert.assertEquals(queryInfo.getUserName(), "testuser");
Assert.assertEquals(queryInfo.getExecutionEngine(), "mr");
Assert.assertEquals(queryInfo.getOperationId(), handle);
Assert.assertTrue(queryInfo.getBeginTime() > 0 && queryInfo.getBeginTime() <= System.currentTimeMillis());

if (finished) {
Assert.assertTrue(display.getEndTime() > 0 && display.getEndTime() >= display.getBeginTime()
&& display.getEndTime() <= System.currentTimeMillis());
Assert.assertTrue(display.getRuntime() > 0);
Assert.assertTrue(queryInfo.getEndTime() > 0 && queryInfo.getEndTime() >= queryInfo.getBeginTime()
&& queryInfo.getEndTime() <= System.currentTimeMillis());
Assert.assertTrue(queryInfo.getRuntime() > 0);
} else {
Assert.assertNull(display.getEndTime());
Assert.assertNull(queryInfo.getEndTime());
//For runtime, query may have finished.
}

QueryDisplay qDisplay1 = display.getQueryDisplay();
QueryDisplay qDisplay1 = queryInfo.getQueryDisplay();
Assert.assertNotNull(qDisplay1);
Assert.assertEquals(qDisplay1.getQueryString(), stmt);
Assert.assertNotNull(qDisplay1.getExplainPlan());
Expand Down Expand Up @@ -170,9 +170,9 @@ private void verifyDDL(SQLOperationDisplay display, String stmt, String handle,
*/
private void verifyDDLHtml(String stmt, String opHandle) throws Exception {
StringWriter sw = new StringWriter();
SQLOperationDisplay sod = sessionManager.getOperationManager().getSQLOperationDisplay(
QueryInfo queryInfo = sessionManager.getOperationManager().getQueryInfo(
opHandle);
new QueryProfileTmpl().render(sw, sod);
new QueryProfileTmpl().render(sw, queryInfo);
String html = sw.toString();

Assert.assertTrue(html.contains(stmt));
Expand Down
16 changes: 11 additions & 5 deletions ql/src/java/org/apache/hadoop/hive/ql/Driver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ public class Driver implements CommandProcessor {
private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME);
static final private LogHelper console = new LogHelper(LOG);
static final int SHUTDOWN_HOOK_PRIORITY = 0;
private final QueryInfo queryInfo;
private Runnable shutdownRunner = null;

private int maxRows = 100;
Expand Down Expand Up @@ -360,25 +361,30 @@ public Driver(HiveConf conf, Context ctx) {
}

public Driver(HiveConf conf, String userName) {
this(new QueryState(conf), userName);
this(new QueryState(conf), userName, null);
}

public Driver(QueryState queryState, String userName) {
this(queryState, userName, new HooksLoader(queryState.getConf()));
this(queryState, userName, new HooksLoader(queryState.getConf()), null);
}

public Driver(HiveConf conf, HooksLoader hooksLoader) {
this(new QueryState(conf), null, hooksLoader);
this(new QueryState(conf), null, hooksLoader, null);
}

private Driver(QueryState queryState, String userName, HooksLoader hooksLoader) {
public Driver(QueryState queryState, String userName, QueryInfo queryInfo) {
this(queryState, userName, new HooksLoader(queryState.getConf()), queryInfo);
}

public Driver(QueryState queryState, String userName, HooksLoader hooksLoader, QueryInfo queryInfo) {
this.queryState = queryState;
this.conf = queryState.getConf();
isParallelEnabled = (conf != null)
&& HiveConf.getBoolVar(conf, ConfVars.HIVE_SERVER2_PARALLEL_COMPILATION);
this.userName = userName;
this.hooksLoader = hooksLoader;
this.queryLifeTimeHookRunner = new QueryLifeTimeHookRunner(conf, hooksLoader, console);
this.queryInfo = queryInfo;
}

/**
Expand Down Expand Up @@ -1736,7 +1742,7 @@ public int execute(boolean deferClose) throws CommandNeedRetryException {

hookContext = new HookContext(plan, queryState, ctx.getPathToCS(), ss.getUserFromAuthenticator(),
ss.getUserIpAddress(), InetAddress.getLocalHost().getHostAddress(), operationId,
ss.getSessionId(), Thread.currentThread().getName(), ss.isHiveServerQuery(), perfLogger);
ss.getSessionId(), Thread.currentThread().getName(), ss.isHiveServerQuery(), perfLogger, queryInfo);
hookContext.setHookType(HookContext.HookType.PRE_EXEC_HOOK);

for (Hook peh : hooksLoader.getHooks(HiveConf.ConfVars.PREEXECHOOKS, console)) {
Expand Down
48 changes: 21 additions & 27 deletions ...ce/cli/operation/SQLOperationDisplay.java → .../org/apache/hadoop/hive/ql/QueryInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,35 +15,29 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hive.service.cli.operation;

import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.QueryDisplay;
import org.apache.hive.service.cli.HiveSQLException;
import org.apache.hive.service.cli.OperationState;
package org.apache.hadoop.hive.ql;

/**
* Used to display some info in the HS2 WebUI.
*
* The class is synchronized, as WebUI may access information about a running query.
*/
public class SQLOperationDisplay {
public final String userName;
public final String executionEngine;
public final long beginTime;
public final String operationId;
public Long runtime; //tracks only running portion of the query.

public Long endTime;
public OperationState state;
public QueryDisplay queryDisplay;

public SQLOperationDisplay(SQLOperation sqlOperation) throws HiveSQLException {
this.state = sqlOperation.getState();
this.userName = sqlOperation.getParentSession().getUserName();
this.executionEngine = sqlOperation.getExecutionEngine();
public class QueryInfo {

private final String userName;
private final String executionEngine;
private final long beginTime;
private final String operationId;
private Long runtime; // tracks only running portion of the query.

private Long endTime;
private String state;
private QueryDisplay queryDisplay;

public QueryInfo(String state, String userName, String executionEngine, String operationId) {
this.state = state;
this.userName = userName;
this.executionEngine = executionEngine;
this.beginTime = System.currentTimeMillis();
this.operationId = sqlOperation.getHandle().getHandleIdentifier().toString();
this.operationId = operationId;
}

public synchronized long getElapsedTime() {
Expand Down Expand Up @@ -74,7 +68,7 @@ public String getExecutionEngine() {
return executionEngine;
}

public synchronized OperationState getState() {
public synchronized String getState() {
return state;
}

Expand All @@ -86,15 +80,15 @@ public synchronized Long getEndTime() {
return endTime;
}

public synchronized void updateState(OperationState state) {
public synchronized void updateState(String state) {
this.state = state;
}

public String getOperationId() {
return operationId;
}

public synchronized void closed() {
public synchronized void setEndTime() {
this.endTime = System.currentTimeMillis();
}

Expand Down
10 changes: 9 additions & 1 deletion ql/src/java/org/apache/hadoop/hive/ql/hooks/HookContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,11 @@

import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.QueryInfo;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.TaskRunner;
import org.apache.hadoop.hive.ql.history.HiveHistory;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.optimizer.lineage.LineageCtx.Index;
import org.apache.hadoop.hive.ql.session.SessionState;
Expand Down Expand Up @@ -68,11 +70,12 @@ static public enum HookType {
private final String threadId;
private final boolean isHiveServerQuery;
private final PerfLogger perfLogger;
private final QueryInfo queryInfo;

public HookContext(QueryPlan queryPlan, QueryState queryState,
Map<String, ContentSummary> inputPathToContentSummary, String userName, String ipAddress,
String hiveInstanceAddress, String operationId, String sessionId, String threadId,
boolean isHiveServerQuery, PerfLogger perfLogger) throws Exception {
boolean isHiveServerQuery, PerfLogger perfLogger, QueryInfo queryInfo) throws Exception {
this.queryPlan = queryPlan;
this.queryState = queryState;
this.conf = queryState.getConf();
Expand All @@ -95,6 +98,7 @@ public HookContext(QueryPlan queryPlan, QueryState queryState,
this.threadId = threadId;
this.isHiveServerQuery = isHiveServerQuery;
this.perfLogger = perfLogger;
this.queryInfo = queryInfo;
}

public QueryPlan getQueryPlan() {
Expand Down Expand Up @@ -232,4 +236,8 @@ public boolean isHiveServerQuery() {
public PerfLogger getPerfLogger() {
return perfLogger;
}

public QueryInfo getQueryInfo() {
return queryInfo;
}
}
Loading

0 comments on commit 1fecb81

Please sign in to comment.