diff --git a/server/src/test/scala/org/apache/livy/utils/SparkYarnAppSpec.scala b/server/src/test/scala/org/apache/livy/utils/SparkYarnAppSpec.scala
index 672444f7f..064bb77b5 100644
--- a/server/src/test/scala/org/apache/livy/utils/SparkYarnAppSpec.scala
+++ b/server/src/test/scala/org/apache/livy/utils/SparkYarnAppSpec.scala
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.util.ConverterUtils
 import org.mockito.Mockito._
 import org.mockito.invocation.InvocationOnMock
 import org.mockito.stubbing.Answer
+import org.scalatest.concurrent.Eventually
 import org.scalatest.FunSpec
 import org.scalatest.mock.MockitoSugar.mock
 
@@ -200,10 +201,19 @@ class SparkYarnAppSpec extends FunSpec with LivyBaseUnitTestSuite {
 
     it("can kill spark-submit while it's running") {
       Clock.withSleepMethod(mockSleep) {
+        val diag = "DIAG"
         val livyConf = new LivyConf()
         livyConf.set(LivyConf.YARN_APP_LOOKUP_TIMEOUT, "0")
 
+        val mockAppReport = mock[ApplicationReport]
+        when(mockAppReport.getApplicationId).thenReturn(appId)
+        when(mockAppReport.getDiagnostics).thenReturn(diag)
+        when(mockAppReport.getFinalApplicationStatus).thenReturn(FinalApplicationStatus.SUCCEEDED)
+        when(mockAppReport.getYarnApplicationState).thenReturn(RUNNING)
+
         val mockYarnClient = mock[YarnClient]
+        when(mockYarnClient.getApplicationReport(appId)).thenReturn(mockAppReport)
+
         val mockSparkSubmit = mock[LineBufferedProcess]
 
         val sparkSubmitRunningLatch = new CountDownLatch(1)
@@ -222,10 +232,14 @@ class SparkYarnAppSpec extends FunSpec with LivyBaseUnitTestSuite {
           None,
           livyConf,
           mockYarnClient)
-        cleanupThread(app.yarnAppMonitorThread) {
-          app.kill()
-          verify(mockSparkSubmit, times(1)).destroy()
-          sparkSubmitRunningLatch.countDown()
+
+        Eventually.eventually(Eventually.timeout(10 seconds), Eventually.interval(100 millis)) {
+          assert(app.isRunning)
+          cleanupThread(app.yarnAppMonitorThread) {
+            app.kill()
+            verify(mockSparkSubmit, times(1)).destroy()
+            sparkSubmitRunningLatch.countDown()
+          }
         }
       }
     }