From 00a8c85a442402ebfd1bed0e381a2a42d713856a Mon Sep 17 00:00:00 2001 From: Tom van Bussel Date: Mon, 27 May 2019 17:36:22 +0200 Subject: [PATCH] [SPARK-27071][CORE] Expose additional metrics in status.api.v1.StageData ## What changes were proposed in this pull request? This PR exposes additional metrics in `status.api.v1.StageData`. These metrics were already computed for `LiveStage`, but they were never exposed to the user. This includes extra metrics about the JVM GC, executor (de)serialization times, shuffle reads and writes, and more. ## How was this patch tested? Existing tests. cc hvanhovell Closes #24011 from tomvanbussel/SPARK-27071. Authored-by: Tom van Bussel Signed-off-by: herman --- .../apache/spark/status/AppStatusStore.scala | 81 +++++++++------- .../org/apache/spark/status/LiveEntity.scala | 92 +++++++++++-------- .../org/apache/spark/status/api/v1/api.scala | 21 ++++- .../org/apache/spark/ui/jobs/JobPage.scala | 65 ++++++++++--- .../blacklisting_for_stage_expectation.json | 21 ++++- ...acklisting_node_for_stage_expectation.json | 23 ++++- .../complete_stage_list_json_expectation.json | 63 ++++++++++--- .../failed_stage_list_json_expectation.json | 21 ++++- .../one_stage_attempt_json_expectation.json | 21 ++++- .../one_stage_json_expectation.json | 21 ++++- .../stage_list_json_expectation.json | 84 +++++++++++++---- ...ist_with_accumulable_json_expectation.json | 21 ++++- ...age_with_accumulable_json_expectation.json | 21 ++++- .../org/apache/spark/ui/StagePageSuite.scala | 21 ++++- 14 files changed, 424 insertions(+), 152 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala index 0487f2f07c097..62402564c2556 100644 --- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala +++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala @@ -462,40 +462,53 @@ private[spark] class AppStatusStore( .toMap new v1.StageData( - stage.status, - stage.stageId, - stage.attemptId, - stage.numTasks, - stage.numActiveTasks, - stage.numCompleteTasks, - stage.numFailedTasks, - stage.numKilledTasks, - stage.numCompletedIndices, - stage.executorRunTime, - stage.executorCpuTime, - stage.submissionTime, - stage.firstTaskLaunchedTime, - stage.completionTime, - stage.failureReason, - stage.inputBytes, - stage.inputRecords, - stage.outputBytes, - stage.outputRecords, - stage.shuffleReadBytes, - stage.shuffleReadRecords, - stage.shuffleWriteBytes, - stage.shuffleWriteRecords, - stage.memoryBytesSpilled, - stage.diskBytesSpilled, - stage.name, - stage.description, - stage.details, - stage.schedulingPool, - stage.rddIds, - stage.accumulatorUpdates, - Some(tasks), - Some(executorSummary(stage.stageId, stage.attemptId)), - stage.killedTasksSummary) + status = stage.status, + stageId = stage.stageId, + attemptId = stage.attemptId, + numTasks = stage.numTasks, + numActiveTasks = stage.numActiveTasks, + numCompleteTasks = stage.numCompleteTasks, + numFailedTasks = stage.numFailedTasks, + numKilledTasks = stage.numKilledTasks, + numCompletedIndices = stage.numCompletedIndices, + submissionTime = stage.submissionTime, + firstTaskLaunchedTime = stage.firstTaskLaunchedTime, + completionTime = stage.completionTime, + failureReason = stage.failureReason, + executorDeserializeTime = stage.executorDeserializeTime, + executorDeserializeCpuTime = stage.executorDeserializeCpuTime, + executorRunTime = stage.executorRunTime, + executorCpuTime = stage.executorCpuTime, + resultSize = stage.resultSize, + jvmGcTime = stage.jvmGcTime, + resultSerializationTime = stage.resultSerializationTime, + memoryBytesSpilled = stage.memoryBytesSpilled, + diskBytesSpilled = stage.diskBytesSpilled, + peakExecutionMemory = stage.peakExecutionMemory, + inputBytes = stage.inputBytes, + inputRecords = stage.inputRecords, + outputBytes = stage.outputBytes, + outputRecords = stage.outputRecords, + shuffleRemoteBlocksFetched = stage.shuffleRemoteBlocksFetched, + shuffleLocalBlocksFetched = stage.shuffleLocalBlocksFetched, + shuffleFetchWaitTime = stage.shuffleFetchWaitTime, + shuffleRemoteBytesRead = stage.shuffleRemoteBytesRead, + shuffleRemoteBytesReadToDisk = stage.shuffleRemoteBytesReadToDisk, + shuffleLocalBytesRead = stage.shuffleLocalBytesRead, + shuffleReadBytes = stage.shuffleReadBytes, + shuffleReadRecords = stage.shuffleReadRecords, + shuffleWriteBytes = stage.shuffleWriteBytes, + shuffleWriteTime = stage.shuffleWriteTime, + shuffleWriteRecords = stage.shuffleWriteRecords, + name = stage.name, + description = stage.description, + details = stage.details, + schedulingPool = stage.schedulingPool, + rddIds = stage.rddIds, + accumulatorUpdates = stage.accumulatorUpdates, + tasks = Some(tasks), + executorSummary = Some(executorSummary(stage.stageId, stage.attemptId)), + killedTasksSummary = stage.killedTasksSummary) } def rdd(rddId: Int): v1.RDDStorageInfo = { diff --git a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala index 6d7b34ae979f2..c5a233f14aa6d 100644 --- a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala +++ b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala @@ -394,45 +394,59 @@ private class LiveStage extends LiveEntity { def toApi(): v1.StageData = { new v1.StageData( - status, - info.stageId, - info.attemptNumber, - - info.numTasks, - activeTasks, - completedTasks, - failedTasks, - killedTasks, - completedIndices.size, - - metrics.executorRunTime, - metrics.executorCpuTime, - info.submissionTime.map(new Date(_)), - if (firstLaunchTime < Long.MaxValue) Some(new Date(firstLaunchTime)) else None, - info.completionTime.map(new Date(_)), - info.failureReason, - - metrics.inputMetrics.bytesRead, - metrics.inputMetrics.recordsRead, - metrics.outputMetrics.bytesWritten, - metrics.outputMetrics.recordsWritten, - metrics.shuffleReadMetrics.localBytesRead + metrics.shuffleReadMetrics.remoteBytesRead, - metrics.shuffleReadMetrics.recordsRead, - metrics.shuffleWriteMetrics.bytesWritten, - metrics.shuffleWriteMetrics.recordsWritten, - metrics.memoryBytesSpilled, - metrics.diskBytesSpilled, - - info.name, - description, - info.details, - schedulingPool, - - info.rddInfos.map(_.id), - newAccumulatorInfos(info.accumulables.values), - None, - None, - killedSummary) + status = status, + stageId = info.stageId, + attemptId = info.attemptNumber, + numTasks = info.numTasks, + numActiveTasks = activeTasks, + numCompleteTasks = completedTasks, + numFailedTasks = failedTasks, + numKilledTasks = killedTasks, + numCompletedIndices = completedIndices.size, + + submissionTime = info.submissionTime.map(new Date(_)), + firstTaskLaunchedTime = + if (firstLaunchTime < Long.MaxValue) Some(new Date(firstLaunchTime)) else None, + completionTime = info.completionTime.map(new Date(_)), + failureReason = info.failureReason, + + executorDeserializeTime = metrics.executorDeserializeTime, + executorDeserializeCpuTime = metrics.executorDeserializeCpuTime, + executorRunTime = metrics.executorRunTime, + executorCpuTime = metrics.executorCpuTime, + resultSize = metrics.resultSize, + jvmGcTime = metrics.jvmGcTime, + resultSerializationTime = metrics.resultSerializationTime, + memoryBytesSpilled = metrics.memoryBytesSpilled, + diskBytesSpilled = metrics.diskBytesSpilled, + peakExecutionMemory = metrics.peakExecutionMemory, + inputBytes = metrics.inputMetrics.bytesRead, + inputRecords = metrics.inputMetrics.recordsRead, + outputBytes = metrics.outputMetrics.bytesWritten, + outputRecords = metrics.outputMetrics.recordsWritten, + shuffleRemoteBlocksFetched = metrics.shuffleReadMetrics.remoteBlocksFetched, + shuffleLocalBlocksFetched = metrics.shuffleReadMetrics.localBlocksFetched, + shuffleFetchWaitTime = metrics.shuffleReadMetrics.fetchWaitTime, + shuffleRemoteBytesRead = metrics.shuffleReadMetrics.remoteBytesRead, + shuffleRemoteBytesReadToDisk = metrics.shuffleReadMetrics.remoteBytesReadToDisk, + shuffleLocalBytesRead = metrics.shuffleReadMetrics.localBytesRead, + shuffleReadBytes = + metrics.shuffleReadMetrics.localBytesRead + metrics.shuffleReadMetrics.remoteBytesRead, + shuffleReadRecords = metrics.shuffleReadMetrics.recordsRead, + shuffleWriteBytes = metrics.shuffleWriteMetrics.bytesWritten, + shuffleWriteTime = metrics.shuffleWriteMetrics.writeTime, + shuffleWriteRecords = metrics.shuffleWriteMetrics.recordsWritten, + + name = info.name, + description = description, + details = info.details, + schedulingPool = schedulingPool, + + rddIds = info.rddInfos.map(_.id), + accumulatorUpdates = newAccumulatorInfos(info.accumulables.values), + tasks = None, + executorSummary = None, + killedTasksSummary = killedSummary) } override protected def doUpdate(): Any = { diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala index 60b8bf8921814..7b3636fdc5b47 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -210,23 +210,36 @@ class StageData private[spark]( val numKilledTasks: Int, val numCompletedIndices: Int, - val executorRunTime: Long, - val executorCpuTime: Long, val submissionTime: Option[Date], val firstTaskLaunchedTime: Option[Date], val completionTime: Option[Date], val failureReason: Option[String], + val executorDeserializeTime: Long, + val executorDeserializeCpuTime: Long, + val executorRunTime: Long, + val executorCpuTime: Long, + val resultSize: Long, + val jvmGcTime: Long, + val resultSerializationTime: Long, + val memoryBytesSpilled: Long, + val diskBytesSpilled: Long, + val peakExecutionMemory: Long, val inputBytes: Long, val inputRecords: Long, val outputBytes: Long, val outputRecords: Long, + val shuffleRemoteBlocksFetched: Long, + val shuffleLocalBlocksFetched: Long, + val shuffleFetchWaitTime: Long, + val shuffleRemoteBytesRead: Long, + val shuffleRemoteBytesReadToDisk: Long, + val shuffleLocalBytesRead: Long, val shuffleReadBytes: Long, val shuffleReadRecords: Long, val shuffleWriteBytes: Long, + val shuffleWriteTime: Long, val shuffleWriteRecords: Long, - val memoryBytesSpilled: Long, - val diskBytesSpilled: Long, val name: String, val description: Option[String], diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala index 46295e73e086b..6e373cfddeb52 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala @@ -203,20 +203,57 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP // stage or if the stage information has been garbage collected store.asOption(store.lastStageAttempt(stageId)).getOrElse { new v1.StageData( - v1.StageStatus.PENDING, - stageId, - 0, 0, 0, 0, 0, 0, 0, - 0L, 0L, None, None, None, None, - 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, - "Unknown", - None, - "Unknown", - null, - Nil, - Nil, - None, - None, - Map()) + status = v1.StageStatus.PENDING, + stageId = stageId, + attemptId = 0, + numTasks = 0, + numActiveTasks = 0, + numCompleteTasks = 0, + numFailedTasks = 0, + numKilledTasks = 0, + numCompletedIndices = 0, + + submissionTime = None, + firstTaskLaunchedTime = None, + completionTime = None, + failureReason = None, + + executorDeserializeTime = 0L, + executorDeserializeCpuTime = 0L, + executorRunTime = 0L, + executorCpuTime = 0L, + resultSize = 0L, + jvmGcTime = 0L, + resultSerializationTime = 0L, + memoryBytesSpilled = 0L, + diskBytesSpilled = 0L, + peakExecutionMemory = 0L, + inputBytes = 0L, + inputRecords = 0L, + outputBytes = 0L, + outputRecords = 0L, + shuffleRemoteBlocksFetched = 0L, + shuffleLocalBlocksFetched = 0L, + shuffleFetchWaitTime = 0L, + shuffleRemoteBytesRead = 0L, + shuffleRemoteBytesReadToDisk = 0L, + shuffleLocalBytesRead = 0L, + shuffleReadBytes = 0L, + shuffleReadRecords = 0L, + shuffleWriteBytes = 0L, + shuffleWriteTime = 0L, + shuffleWriteRecords = 0L, + + name = "Unknown", + description = None, + details = "Unknown", + schedulingPool = null, + + rddIds = Nil, + accumulatorUpdates = Nil, + tasks = None, + executorSummary = None, + killedTasksSummary = Map()) } } diff --git a/core/src/test/resources/HistoryServerExpectations/blacklisting_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/blacklisting_for_stage_expectation.json index 62e5c123fd3d4..b18b19f7eeffb 100644 --- a/core/src/test/resources/HistoryServerExpectations/blacklisting_for_stage_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/blacklisting_for_stage_expectation.json @@ -8,21 +8,34 @@ "numFailedTasks" : 2, "numKilledTasks" : 0, "numCompletedIndices" : 10, - "executorRunTime" : 761, - "executorCpuTime" : 269916000, "submissionTime" : "2018-01-09T10:21:18.152GMT", "firstTaskLaunchedTime" : "2018-01-09T10:21:18.347GMT", "completionTime" : "2018-01-09T10:21:19.062GMT", + "executorDeserializeTime" : 327, + "executorDeserializeCpuTime" : 225900000, + "executorRunTime" : 761, + "executorCpuTime" : 269916000, + "resultSize" : 10376, + "jvmGcTime" : 27, + "resultSerializationTime" : 1, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "peakExecutionMemory" : 0, "inputBytes" : 0, "inputRecords" : 0, "outputBytes" : 0, "outputRecords" : 0, + "shuffleRemoteBlocksFetched" : 0, + "shuffleLocalBlocksFetched" : 0, + "shuffleFetchWaitTime" : 0, + "shuffleRemoteBytesRead" : 0, + "shuffleRemoteBytesReadToDisk" : 0, + "shuffleLocalBytesRead" : 0, "shuffleReadBytes" : 0, "shuffleReadRecords" : 0, "shuffleWriteBytes" : 460, + "shuffleWriteTime" : 8711515, "shuffleWriteRecords" : 10, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, "name" : "map at :26", "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:370)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:26)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:34)\n$line17.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:36)\n$line17.$read$$iw$$iw$$iw$$iw$$iw.(:38)\n$line17.$read$$iw$$iw$$iw$$iw.(:40)\n$line17.$read$$iw$$iw$$iw.(:42)\n$line17.$read$$iw$$iw.(:44)\n$line17.$read$$iw.(:46)\n$line17.$read.(:48)\n$line17.$read$.(:52)\n$line17.$read$.()\n$line17.$eval$.$print$lzycompute(:7)\n$line17.$eval$.$print(:6)\n$line17.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)", "schedulingPool" : "default", diff --git a/core/src/test/resources/HistoryServerExpectations/blacklisting_node_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/blacklisting_node_for_stage_expectation.json index 6e46c881b2a21..8d11081247913 100644 --- a/core/src/test/resources/HistoryServerExpectations/blacklisting_node_for_stage_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/blacklisting_node_for_stage_expectation.json @@ -8,21 +8,34 @@ "numFailedTasks" : 4, "numKilledTasks" : 0, "numCompletedIndices" : 10, - "executorRunTime" : 5080, - "executorCpuTime" : 1163210819, "submissionTime" : "2018-01-18T18:33:12.658GMT", "firstTaskLaunchedTime" : "2018-01-18T18:33:12.816GMT", "completionTime" : "2018-01-18T18:33:15.279GMT", + "executorDeserializeTime" : 3679, + "executorDeserializeCpuTime" : 1029819716, + "executorRunTime" : 5080, + "executorCpuTime" : 1163210819, + "resultSize" : 10824, + "jvmGcTime" : 370, + "resultSerializationTime" : 5, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "peakExecutionMemory" : 0, "inputBytes" : 0, "inputRecords" : 0, "outputBytes" : 0, "outputRecords" : 0, + "shuffleRemoteBlocksFetched" : 0, + "shuffleLocalBlocksFetched" : 0, + "shuffleFetchWaitTime" : 0, + "shuffleRemoteBytesRead" : 0, + "shuffleRemoteBytesReadToDisk" : 0, + "shuffleLocalBytesRead" : 0, "shuffleReadBytes" : 0, "shuffleReadRecords" : 0, "shuffleWriteBytes" : 1461, + "shuffleWriteTime" : 33251697, "shuffleWriteRecords" : 30, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, "name" : "map at :27", "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:370)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:27)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.(:35)\n$line15.$read$$iw$$iw$$iw$$iw$$iw$$iw.(:37)\n$line15.$read$$iw$$iw$$iw$$iw$$iw.(:39)\n$line15.$read$$iw$$iw$$iw$$iw.(:41)\n$line15.$read$$iw$$iw$$iw.(:43)\n$line15.$read$$iw$$iw.(:45)\n$line15.$read$$iw.(:47)\n$line15.$read.(:49)\n$line15.$read$.(:53)\n$line15.$read$.()\n$line15.$eval$.$print$lzycompute(:7)\n$line15.$eval$.$print(:6)\n$line15.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\nscala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)", "schedulingPool" : "default", @@ -864,4 +877,4 @@ } }, "killedTasksSummary" : { } -} \ No newline at end of file +} diff --git a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json index 37b7d7269059f..a47cd26ed102b 100644 --- a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json @@ -8,21 +8,34 @@ "numFailedTasks" : 0, "numKilledTasks" : 0, "numCompletedIndices" : 8, - "executorRunTime" : 162, - "executorCpuTime" : 0, "submissionTime" : "2015-02-03T16:43:07.191GMT", "firstTaskLaunchedTime" : "2015-02-03T16:43:07.191GMT", "completionTime" : "2015-02-03T16:43:07.226GMT", + "executorDeserializeTime" : 36, + "executorDeserializeCpuTime" : 0, + "executorRunTime" : 162, + "executorCpuTime" : 0, + "resultSize" : 14496, + "jvmGcTime" : 0, + "resultSerializationTime" : 1, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "peakExecutionMemory" : 0, "inputBytes" : 160, "inputRecords" : 0, "outputBytes" : 0, "outputRecords" : 0, + "shuffleRemoteBlocksFetched" : 0, + "shuffleLocalBlocksFetched" : 0, + "shuffleFetchWaitTime" : 0, + "shuffleRemoteBytesRead" : 0, + "shuffleRemoteBytesReadToDisk" : 0, + "shuffleLocalBytesRead" : 0, "shuffleReadBytes" : 0, "shuffleReadRecords" : 0, "shuffleWriteBytes" : 0, + "shuffleWriteTime" : 0, "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, "name" : "count at :17", "details" : "org.apache.spark.rdd.RDD.count(RDD.scala:910)\n$line19.$read$$iwC$$iwC$$iwC$$iwC.(:17)\n$line19.$read$$iwC$$iwC$$iwC.(:22)\n$line19.$read$$iwC$$iwC.(:24)\n$line19.$read$$iwC.(:26)\n$line19.$read.(:28)\n$line19.$read$.(:32)\n$line19.$read$.()\n$line19.$eval$.(:7)\n$line19.$eval$.()\n$line19.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", "schedulingPool" : "default", @@ -39,21 +52,34 @@ "numFailedTasks" : 0, "numKilledTasks" : 0, "numCompletedIndices" : 8, - "executorRunTime" : 3476, - "executorCpuTime" : 0, "submissionTime" : "2015-02-03T16:43:05.829GMT", "firstTaskLaunchedTime" : "2015-02-03T16:43:05.829GMT", "completionTime" : "2015-02-03T16:43:06.286GMT", + "executorDeserializeTime" : 13, + "executorDeserializeCpuTime" : 0, + "executorRunTime" : 3476, + "executorCpuTime" : 0, + "resultSize" : 15216, + "jvmGcTime" : 152, + "resultSerializationTime" : 9, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "peakExecutionMemory" : 0, "inputBytes" : 28000128, "inputRecords" : 0, "outputBytes" : 0, "outputRecords" : 0, + "shuffleRemoteBlocksFetched" : 0, + "shuffleLocalBlocksFetched" : 0, + "shuffleFetchWaitTime" : 0, + "shuffleRemoteBytesRead" : 0, + "shuffleRemoteBytesReadToDisk" : 0, + "shuffleLocalBytesRead" : 0, "shuffleReadBytes" : 0, "shuffleReadRecords" : 0, "shuffleWriteBytes" : 13180, + "shuffleWriteTime" : 692000, "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, "name" : "map at :14", "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:271)\n$line10.$read$$iwC$$iwC$$iwC$$iwC.(:14)\n$line10.$read$$iwC$$iwC$$iwC.(:19)\n$line10.$read$$iwC$$iwC.(:21)\n$line10.$read$$iwC.(:23)\n$line10.$read.(:25)\n$line10.$read$.(:29)\n$line10.$read$.()\n$line10.$eval$.(:7)\n$line10.$eval$.()\n$line10.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", "schedulingPool" : "default", @@ -70,21 +96,34 @@ "numFailedTasks" : 0, "numKilledTasks" : 0, "numCompletedIndices" : 8, - "executorRunTime" : 4338, - "executorCpuTime" : 0, "submissionTime" : "2015-02-03T16:43:04.228GMT", "firstTaskLaunchedTime" : "2015-02-03T16:43:04.234GMT", "completionTime" : "2015-02-03T16:43:04.819GMT", + "executorDeserializeTime" : 91, + "executorDeserializeCpuTime" : 0, + "executorRunTime" : 4338, + "executorCpuTime" : 0, + "resultSize" : 10144, + "jvmGcTime" : 200, + "resultSerializationTime" : 5, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "peakExecutionMemory" : 0, "inputBytes" : 0, "inputRecords" : 0, "outputBytes" : 0, "outputRecords" : 0, + "shuffleRemoteBlocksFetched" : 0, + "shuffleLocalBlocksFetched" : 0, + "shuffleFetchWaitTime" : 0, + "shuffleRemoteBytesRead" : 0, + "shuffleRemoteBytesReadToDisk" : 0, + "shuffleLocalBytesRead" : 0, "shuffleReadBytes" : 0, "shuffleReadRecords" : 0, "shuffleWriteBytes" : 0, + "shuffleWriteTime" : 0, "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, "name" : "count at :15", "details" : "org.apache.spark.rdd.RDD.count(RDD.scala:910)\n$line9.$read$$iwC$$iwC$$iwC$$iwC.(:15)\n$line9.$read$$iwC$$iwC$$iwC.(:20)\n$line9.$read$$iwC$$iwC.(:22)\n$line9.$read$$iwC.(:24)\n$line9.$read.(:26)\n$line9.$read$.(:30)\n$line9.$read$.()\n$line9.$eval$.(:7)\n$line9.$eval$.()\n$line9.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", "schedulingPool" : "default", diff --git a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json index 2fd55666fa018..da26271e66bc4 100644 --- a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json @@ -8,22 +8,35 @@ "numFailedTasks" : 1, "numKilledTasks" : 0, "numCompletedIndices" : 7, - "executorRunTime" : 278, - "executorCpuTime" : 0, "submissionTime" : "2015-02-03T16:43:06.296GMT", "firstTaskLaunchedTime" : "2015-02-03T16:43:06.296GMT", "completionTime" : "2015-02-03T16:43:06.347GMT", "failureReason" : "Job aborted due to stage failure: Task 3 in stage 2.0 failed 1 times, most recent failure: Lost task 3.0 in stage 2.0 (TID 19, localhost): java.lang.RuntimeException: got a 3, failing\n\tat $line11.$read$$iwC$$iwC$$iwC$$iwC$$anonfun$1.apply(:18)\n\tat $line11.$read$$iwC$$iwC$$iwC$$iwC$$anonfun$1.apply(:17)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:328)\n\tat org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1311)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:910)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:910)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$4.apply(SparkContext.scala:1314)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$4.apply(SparkContext.scala:1314)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:61)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:56)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:196)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\tat java.lang.Thread.run(Thread.java:745)\n\nDriver stacktrace:", + "executorDeserializeTime" : 10, + "executorDeserializeCpuTime" : 0, + "executorRunTime" : 278, + "executorCpuTime" : 0, + "resultSize" : 6034, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "peakExecutionMemory" : 0, "inputBytes" : 0, "inputRecords" : 0, "outputBytes" : 0, "outputRecords" : 0, + "shuffleRemoteBlocksFetched" : 0, + "shuffleLocalBlocksFetched" : 64, + "shuffleFetchWaitTime" : 1, + "shuffleRemoteBytesRead" : 0, + "shuffleRemoteBytesReadToDisk" : 0, + "shuffleLocalBytesRead" : 0, "shuffleReadBytes" : 0, "shuffleReadRecords" : 0, "shuffleWriteBytes" : 0, + "shuffleWriteTime" : 0, "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, "name" : "count at :20", "details" : "org.apache.spark.rdd.RDD.count(RDD.scala:910)\n$line11.$read$$iwC$$iwC$$iwC$$iwC.(:20)\n$line11.$read$$iwC$$iwC$$iwC.(:25)\n$line11.$read$$iwC$$iwC.(:27)\n$line11.$read$$iwC.(:29)\n$line11.$read.(:31)\n$line11.$read$.(:35)\n$line11.$read$.()\n$line11.$eval$.(:7)\n$line11.$eval$.()\n$line11.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", "schedulingPool" : "default", diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json index dfa7385d3c066..791907045e500 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json @@ -8,21 +8,34 @@ "numFailedTasks" : 0, "numKilledTasks" : 0, "numCompletedIndices" : 8, - "executorRunTime" : 3476, - "executorCpuTime" : 0, "submissionTime" : "2015-02-03T16:43:05.829GMT", "firstTaskLaunchedTime" : "2015-02-03T16:43:05.829GMT", "completionTime" : "2015-02-03T16:43:06.286GMT", + "executorDeserializeTime" : 13, + "executorDeserializeCpuTime" : 0, + "executorRunTime" : 3476, + "executorCpuTime" : 0, + "resultSize" : 15216, + "jvmGcTime" : 152, + "resultSerializationTime" : 9, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "peakExecutionMemory" : 0, "inputBytes" : 28000128, "inputRecords" : 0, "outputBytes" : 0, "outputRecords" : 0, + "shuffleRemoteBlocksFetched" : 0, + "shuffleLocalBlocksFetched" : 0, + "shuffleFetchWaitTime" : 0, + "shuffleRemoteBytesRead" : 0, + "shuffleRemoteBytesReadToDisk" : 0, + "shuffleLocalBytesRead" : 0, "shuffleReadBytes" : 0, "shuffleReadRecords" : 0, "shuffleWriteBytes" : 13180, + "shuffleWriteTime" : 692000, "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, "name" : "map at :14", "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:271)\n$line10.$read$$iwC$$iwC$$iwC$$iwC.(:14)\n$line10.$read$$iwC$$iwC$$iwC.(:19)\n$line10.$read$$iwC$$iwC.(:21)\n$line10.$read$$iwC.(:23)\n$line10.$read.(:25)\n$line10.$read$.(:29)\n$line10.$read$.()\n$line10.$eval$.(:7)\n$line10.$eval$.()\n$line10.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", "schedulingPool" : "default", diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json index e036627dd79c2..50d3f74ae775f 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json @@ -8,21 +8,34 @@ "numFailedTasks" : 0, "numKilledTasks" : 0, "numCompletedIndices" : 8, - "executorRunTime" : 3476, - "executorCpuTime" : 0, "submissionTime" : "2015-02-03T16:43:05.829GMT", "firstTaskLaunchedTime" : "2015-02-03T16:43:05.829GMT", "completionTime" : "2015-02-03T16:43:06.286GMT", + "executorDeserializeTime" : 13, + "executorDeserializeCpuTime" : 0, + "executorRunTime" : 3476, + "executorCpuTime" : 0, + "resultSize" : 15216, + "jvmGcTime" : 152, + "resultSerializationTime" : 9, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "peakExecutionMemory" : 0, "inputBytes" : 28000128, "inputRecords" : 0, "outputBytes" : 0, "outputRecords" : 0, + "shuffleRemoteBlocksFetched" : 0, + "shuffleLocalBlocksFetched" : 0, + "shuffleFetchWaitTime" : 0, + "shuffleRemoteBytesRead" : 0, + "shuffleRemoteBytesReadToDisk" : 0, + "shuffleLocalBytesRead" : 0, "shuffleReadBytes" : 0, "shuffleReadRecords" : 0, "shuffleWriteBytes" : 13180, + "shuffleWriteTime" : 692000, "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, "name" : "map at :14", "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:271)\n$line10.$read$$iwC$$iwC$$iwC$$iwC.(:14)\n$line10.$read$$iwC$$iwC$$iwC.(:19)\n$line10.$read$$iwC$$iwC.(:21)\n$line10.$read$$iwC.(:23)\n$line10.$read.(:25)\n$line10.$read$.(:29)\n$line10.$read$.()\n$line10.$eval$.(:7)\n$line10.$eval$.()\n$line10.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", "schedulingPool" : "default", diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json index 1e6fb40d60284..edbac7127039d 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json @@ -8,21 +8,34 @@ "numFailedTasks" : 0, "numKilledTasks" : 0, "numCompletedIndices" : 8, - "executorRunTime" : 162, - "executorCpuTime" : 0, "submissionTime" : "2015-02-03T16:43:07.191GMT", "firstTaskLaunchedTime" : "2015-02-03T16:43:07.191GMT", "completionTime" : "2015-02-03T16:43:07.226GMT", + "executorDeserializeTime" : 36, + "executorDeserializeCpuTime" : 0, + "executorRunTime" : 162, + "executorCpuTime" : 0, + "resultSize" : 14496, + "jvmGcTime" : 0, + "resultSerializationTime" : 1, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "peakExecutionMemory" : 0, "inputBytes" : 160, "inputRecords" : 0, "outputBytes" : 0, "outputRecords" : 0, + "shuffleRemoteBlocksFetched" : 0, + "shuffleLocalBlocksFetched" : 0, + "shuffleFetchWaitTime" : 0, + "shuffleRemoteBytesRead" : 0, + "shuffleRemoteBytesReadToDisk" : 0, + "shuffleLocalBytesRead" : 0, "shuffleReadBytes" : 0, "shuffleReadRecords" : 0, "shuffleWriteBytes" : 0, + "shuffleWriteTime" : 0, "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, "name" : "count at :17", "details" : "org.apache.spark.rdd.RDD.count(RDD.scala:910)\n$line19.$read$$iwC$$iwC$$iwC$$iwC.(:17)\n$line19.$read$$iwC$$iwC$$iwC.(:22)\n$line19.$read$$iwC$$iwC.(:24)\n$line19.$read$$iwC.(:26)\n$line19.$read.(:28)\n$line19.$read$.(:32)\n$line19.$read$.()\n$line19.$eval$.(:7)\n$line19.$eval$.()\n$line19.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", "schedulingPool" : "default", @@ -39,22 +52,35 @@ "numFailedTasks" : 1, "numKilledTasks" : 0, "numCompletedIndices" : 7, - "executorRunTime" : 278, - "executorCpuTime" : 0, "submissionTime" : "2015-02-03T16:43:06.296GMT", "firstTaskLaunchedTime" : "2015-02-03T16:43:06.296GMT", "completionTime" : "2015-02-03T16:43:06.347GMT", "failureReason" : "Job aborted due to stage failure: Task 3 in stage 2.0 failed 1 times, most recent failure: Lost task 3.0 in stage 2.0 (TID 19, localhost): java.lang.RuntimeException: got a 3, failing\n\tat $line11.$read$$iwC$$iwC$$iwC$$iwC$$anonfun$1.apply(:18)\n\tat $line11.$read$$iwC$$iwC$$iwC$$iwC$$anonfun$1.apply(:17)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:328)\n\tat org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1311)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:910)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:910)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$4.apply(SparkContext.scala:1314)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$4.apply(SparkContext.scala:1314)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:61)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:56)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:196)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\tat java.lang.Thread.run(Thread.java:745)\n\nDriver stacktrace:", + "executorDeserializeTime" : 10, + "executorDeserializeCpuTime" : 0, + "executorRunTime" : 278, + "executorCpuTime" : 0, + "resultSize" : 6034, + "jvmGcTime" : 0, + "resultSerializationTime" : 0, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "peakExecutionMemory" : 0, "inputBytes" : 0, "inputRecords" : 0, "outputBytes" : 0, "outputRecords" : 0, + "shuffleRemoteBlocksFetched" : 0, + "shuffleLocalBlocksFetched" : 64, + "shuffleFetchWaitTime" : 1, + "shuffleRemoteBytesRead" : 0, + "shuffleRemoteBytesReadToDisk" : 0, + "shuffleLocalBytesRead" : 0, "shuffleReadBytes" : 0, "shuffleReadRecords" : 0, "shuffleWriteBytes" : 0, + "shuffleWriteTime" : 0, "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, "name" : "count at :20", "details" : "org.apache.spark.rdd.RDD.count(RDD.scala:910)\n$line11.$read$$iwC$$iwC$$iwC$$iwC.(:20)\n$line11.$read$$iwC$$iwC$$iwC.(:25)\n$line11.$read$$iwC$$iwC.(:27)\n$line11.$read$$iwC.(:29)\n$line11.$read.(:31)\n$line11.$read$.(:35)\n$line11.$read$.()\n$line11.$eval$.(:7)\n$line11.$eval$.()\n$line11.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", "schedulingPool" : "default", @@ -71,21 +97,34 @@ "numFailedTasks" : 0, "numKilledTasks" : 0, "numCompletedIndices" : 8, - "executorRunTime" : 3476, - "executorCpuTime" : 0, "submissionTime" : "2015-02-03T16:43:05.829GMT", "firstTaskLaunchedTime" : "2015-02-03T16:43:05.829GMT", "completionTime" : "2015-02-03T16:43:06.286GMT", + "executorDeserializeTime" : 13, + "executorDeserializeCpuTime" : 0, + "executorRunTime" : 3476, + "executorCpuTime" : 0, + "resultSize" : 15216, + "jvmGcTime" : 152, + "resultSerializationTime" : 9, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "peakExecutionMemory" : 0, "inputBytes" : 28000128, "inputRecords" : 0, "outputBytes" : 0, "outputRecords" : 0, + "shuffleRemoteBlocksFetched" : 0, + "shuffleLocalBlocksFetched" : 0, + "shuffleFetchWaitTime" : 0, + "shuffleRemoteBytesRead" : 0, + "shuffleRemoteBytesReadToDisk" : 0, + "shuffleLocalBytesRead" : 0, "shuffleReadBytes" : 0, "shuffleReadRecords" : 0, "shuffleWriteBytes" : 13180, + "shuffleWriteTime" : 692000, "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, "name" : "map at :14", "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:271)\n$line10.$read$$iwC$$iwC$$iwC$$iwC.(:14)\n$line10.$read$$iwC$$iwC$$iwC.(:19)\n$line10.$read$$iwC$$iwC.(:21)\n$line10.$read$$iwC.(:23)\n$line10.$read.(:25)\n$line10.$read$.(:29)\n$line10.$read$.()\n$line10.$eval$.(:7)\n$line10.$eval$.()\n$line10.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", "schedulingPool" : "default", @@ -102,21 +141,34 @@ "numFailedTasks" : 0, "numKilledTasks" : 0, "numCompletedIndices" : 8, - "executorRunTime" : 4338, - "executorCpuTime" : 0, "submissionTime" : "2015-02-03T16:43:04.228GMT", "firstTaskLaunchedTime" : "2015-02-03T16:43:04.234GMT", "completionTime" : "2015-02-03T16:43:04.819GMT", + "executorDeserializeTime" : 91, + "executorDeserializeCpuTime" : 0, + "executorRunTime" : 4338, + "executorCpuTime" : 0, + "resultSize" : 10144, + "jvmGcTime" : 200, + "resultSerializationTime" : 5, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "peakExecutionMemory" : 0, "inputBytes" : 0, "inputRecords" : 0, "outputBytes" : 0, "outputRecords" : 0, + "shuffleRemoteBlocksFetched" : 0, + "shuffleLocalBlocksFetched" : 0, + "shuffleFetchWaitTime" : 0, + "shuffleRemoteBytesRead" : 0, + "shuffleRemoteBytesReadToDisk" : 0, + "shuffleLocalBytesRead" : 0, "shuffleReadBytes" : 0, "shuffleReadRecords" : 0, "shuffleWriteBytes" : 0, + "shuffleWriteTime" : 0, "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, "name" : "count at :15", "details" : "org.apache.spark.rdd.RDD.count(RDD.scala:910)\n$line9.$read$$iwC$$iwC$$iwC$$iwC.(:15)\n$line9.$read$$iwC$$iwC$$iwC.(:20)\n$line9.$read$$iwC$$iwC.(:22)\n$line9.$read$$iwC.(:24)\n$line9.$read.(:26)\n$line9.$read$.(:30)\n$line9.$read$.()\n$line9.$eval$.(:7)\n$line9.$eval$.()\n$line9.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", "schedulingPool" : "default", diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json index e6284ccf9b73d..836f2cb095097 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json @@ -8,21 +8,34 @@ "numFailedTasks" : 0, "numKilledTasks" : 0, "numCompletedIndices" : 8, - "executorRunTime" : 120, - "executorCpuTime" : 0, "submissionTime" : "2015-03-16T19:25:36.103GMT", "firstTaskLaunchedTime" : "2015-03-16T19:25:36.515GMT", "completionTime" : "2015-03-16T19:25:36.579GMT", + "executorDeserializeTime" : 102, + "executorDeserializeCpuTime" : 0, + "executorRunTime" : 120, + "executorCpuTime" : 0, + "resultSize" : 5576, + "jvmGcTime" : 0, + "resultSerializationTime" : 15, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "peakExecutionMemory" : 0, "inputBytes" : 0, "inputRecords" : 0, "outputBytes" : 0, "outputRecords" : 0, + "shuffleRemoteBlocksFetched" : 0, + "shuffleLocalBlocksFetched" : 0, + "shuffleFetchWaitTime" : 0, + "shuffleRemoteBytesRead" : 0, + "shuffleRemoteBytesReadToDisk" : 0, + "shuffleLocalBytesRead" : 0, "shuffleReadBytes" : 0, "shuffleReadRecords" : 0, "shuffleWriteBytes" : 0, + "shuffleWriteTime" : 0, "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, "name" : "foreach at :15", "details" : "org.apache.spark.rdd.RDD.foreach(RDD.scala:765)\n$line9.$read$$iwC$$iwC$$iwC$$iwC.(:15)\n$line9.$read$$iwC$$iwC$$iwC.(:20)\n$line9.$read$$iwC$$iwC.(:22)\n$line9.$read$$iwC.(:24)\n$line9.$read.(:26)\n$line9.$read$.(:30)\n$line9.$read$.()\n$line9.$eval$.(:7)\n$line9.$eval$.()\n$line9.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:483)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", "schedulingPool" : "default", diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json index b79f8a5baa85e..735a8257fc343 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json @@ -8,21 +8,34 @@ "numFailedTasks" : 0, "numKilledTasks" : 0, "numCompletedIndices" : 8, - "executorRunTime" : 120, - "executorCpuTime" : 0, "submissionTime" : "2015-03-16T19:25:36.103GMT", "firstTaskLaunchedTime" : "2015-03-16T19:25:36.515GMT", "completionTime" : "2015-03-16T19:25:36.579GMT", + "executorDeserializeTime" : 102, + "executorDeserializeCpuTime" : 0, + "executorRunTime" : 120, + "executorCpuTime" : 0, + "resultSize" : 5576, + "jvmGcTime" : 0, + "resultSerializationTime" : 15, + "memoryBytesSpilled" : 0, + "diskBytesSpilled" : 0, + "peakExecutionMemory" : 0, "inputBytes" : 0, "inputRecords" : 0, "outputBytes" : 0, "outputRecords" : 0, + "shuffleRemoteBlocksFetched" : 0, + "shuffleLocalBlocksFetched" : 0, + "shuffleFetchWaitTime" : 0, + "shuffleRemoteBytesRead" : 0, + "shuffleRemoteBytesReadToDisk" : 0, + "shuffleLocalBytesRead" : 0, "shuffleReadBytes" : 0, "shuffleReadRecords" : 0, "shuffleWriteBytes" : 0, + "shuffleWriteTime" : 0, "shuffleWriteRecords" : 0, - "memoryBytesSpilled" : 0, - "diskBytesSpilled" : 0, "name" : "foreach at :15", "details" : "org.apache.spark.rdd.RDD.foreach(RDD.scala:765)\n$line9.$read$$iwC$$iwC$$iwC$$iwC.(:15)\n$line9.$read$$iwC$$iwC$$iwC.(:20)\n$line9.$read$$iwC$$iwC.(:22)\n$line9.$read$$iwC.(:24)\n$line9.$read.(:26)\n$line9.$read$.(:30)\n$line9.$read$.()\n$line9.$eval$.(:7)\n$line9.$eval$.()\n$line9.$eval.$print()\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:483)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)", "schedulingPool" : "default", diff --git a/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala b/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala index 5e976ae4e91da..f4896983d565f 100644 --- a/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala @@ -51,23 +51,36 @@ class StagePageSuite extends SparkFunSuite with LocalSparkContext { numKilledTasks = 1, numCompletedIndices = 1, - executorRunTime = 1L, - executorCpuTime = 1L, submissionTime = None, firstTaskLaunchedTime = None, completionTime = None, failureReason = None, + executorDeserializeTime = 1L, + executorDeserializeCpuTime = 1L, + executorRunTime = 1L, + executorCpuTime = 1L, + resultSize = 1L, + jvmGcTime = 1L, + resultSerializationTime = 1L, + memoryBytesSpilled = 1L, + diskBytesSpilled = 1L, + peakExecutionMemory = 1L, inputBytes = 1L, inputRecords = 1L, outputBytes = 1L, outputRecords = 1L, + shuffleRemoteBlocksFetched = 1L, + shuffleLocalBlocksFetched = 1L, + shuffleFetchWaitTime = 1L, + shuffleRemoteBytesRead = 1L, + shuffleRemoteBytesReadToDisk = 1L, + shuffleLocalBytesRead = 1L, shuffleReadBytes = 1L, shuffleReadRecords = 1L, shuffleWriteBytes = 1L, + shuffleWriteTime = 1L, shuffleWriteRecords = 1L, - memoryBytesSpilled = 1L, - diskBytesSpilled = 1L, name = "stage1", description = Some("description"),