diff --git a/core/src/test/scala/org/apache/spark/util/logging/DriverLoggerSuite.scala b/core/src/test/scala/org/apache/spark/util/logging/DriverLoggerSuite.scala index f5c47df94a2b4..97cd5caaac907 100644 --- a/core/src/test/scala/org/apache/spark/util/logging/DriverLoggerSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/logging/DriverLoggerSuite.scala @@ -97,7 +97,9 @@ class DriverLoggerSuite extends SparkFunSuite with LocalSparkContext { test("SPARK-44214: DriverLogger.apply returns None when only spark.driver.log.localDir exists") { val sparkConf = new SparkConf() assert(DriverLogger(sparkConf).isEmpty) - assert(DriverLogger(sparkConf.set(DRIVER_LOG_LOCAL_DIR, "file://tmp/")).isEmpty) + withTempDir { dir => + assert(DriverLogger(sparkConf.set(DRIVER_LOG_LOCAL_DIR, dir.getCanonicalPath)).isEmpty) + } } private def getSparkContext(): SparkContext = { diff --git a/docs/configuration.md b/docs/configuration.md index dfded480c99d3..6f7e12555e8e8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -430,6 +430,14 @@ of the most common options to set are: 1.3.0 + + spark.driver.log.localDir + (none) + + Specifies a local directory to write driver logs and enable Driver Log UI Tab. + + 4.0.0 + spark.driver.log.dfsDir (none) @@ -460,7 +468,7 @@ of the most common options to set are: spark.driver.log.layout %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex - The layout for the driver logs that are synced to spark.driver.log.dfsDir. If this is not configured, + The layout for the driver logs that are synced to spark.driver.log.localDir and spark.driver.log.dfsDir. If this is not configured, it uses the layout for the first appender defined in log4j2.properties. If that is also not configured, driver logs use the default layout. diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md index 707a76196f3ab..16e28f1f2297f 100644 --- a/docs/running-on-kubernetes.md +++ b/docs/running-on-kubernetes.md @@ -441,6 +441,19 @@ $ kubectl port-forward 4040:4040 Then, the Spark driver UI can be accessed on `http://localhost:4040`. +Since Apache Spark 4.0.0, Driver UI provides a way to see driver logs via a new configuration. + +``` +spark.driver.log.localDir=/tmp +``` + +Then, the Spark driver UI can be accessed on `http://localhost:4040/logs/`. +Optionally, the layout of log is configured by the following. + +``` +spark.driver.log.layout="%m%n%ex" +``` + ### Debugging There may be several kinds of failures. If the Kubernetes API server rejects the request made from spark-submit, or the