spark.driver.block_manager.disk.usage | sum | int | MBy | development | yes | | Disk space used by the BlockManager. |
spark.driver.block_manager.memory.usage | sum | int | MBy | development | yes | location (string), state (string) | Memory usage for the driver's BlockManager. |
spark.driver.code_generator.compilation.average_time | gauge | double | ms | development | yes | | Average time spent during CodeGenerator source code compilation operations. |
spark.driver.code_generator.compilation.count | sum (monotonic) | int | { compilation } | development | yes | | Number of source code compilation operations performed by the CodeGenerator. |
spark.driver.code_generator.generated_class.average_size | gauge | double | By | development | yes | | Average class size of the classes generated by the CodeGenerator. |
spark.driver.code_generator.generated_class.count | sum (monotonic) | int | { class } | development | yes | | Number of classes generated by the CodeGenerator. |
spark.driver.code_generator.generated_method.average_size | gauge | double | By | development | yes | | Average method size of the classes generated by the CodeGenerator. |
spark.driver.code_generator.generated_method.count | sum (monotonic) | int | { method } | development | yes | | Number of methods generated by the CodeGenerator. |
spark.driver.code_generator.source_code.average_size | gauge | double | By | development | yes | | Average size of the source code generated by a CodeGenerator code generation operation. |
spark.driver.code_generator.source_code.operations | sum (monotonic) | int | { operation } | development | yes | | Number of source code generation operations performed by the CodeGenerator. |
spark.driver.dag_scheduler.job.active | sum | int | { job } | development | yes | | Number of active jobs currently being processed by the DAGScheduler. |
spark.driver.dag_scheduler.job.count | sum (monotonic) | int | { job } | development | yes | | Number of jobs that have been submitted to the DAGScheduler. |
spark.driver.dag_scheduler.stage.count | sum | int | { stage } | development | yes | scheduler_status (string) | Number of stages the DAGScheduler is either running or needs to run. |
spark.driver.dag_scheduler.stage.failed | sum (monotonic) | int | { stage } | development | yes | | Number of failed stages run by the DAGScheduler. |
spark.driver.executor.gc.operations | sum (monotonic) | int | { gc_operation } | development | yes | gc_type (string) | Number of garbage collection operations performed by the driver. |
spark.driver.executor.gc.time | sum (monotonic) | int | ms | development | yes | gc_type (string) | Total elapsed time during garbage collection operations performed by the driver. |
spark.driver.executor.memory.execution | sum | int | By | development | yes | location (string) | Amount of execution memory currently used by the driver. |
spark.driver.executor.memory.jvm | sum | int | By | development | yes | location (string) | Amount of memory used by the driver's JVM. |
spark.driver.executor.memory.pool | sum | int | By | development | yes | pool_memory_type (string) | Amount of pool memory currently used by the driver. |
spark.driver.executor.memory.storage | sum | int | By | development | yes | location (string) | Amount of storage memory currently used by the driver. |
spark.driver.hive_external_catalog.file_cache_hits | sum (monotonic) | int | { hit } | development | yes | | Number of file cache hits on the HiveExternalCatalog. |
spark.driver.hive_external_catalog.files_discovered | sum (monotonic) | int | { file } | development | yes | | Number of files discovered while listing the partitions of a table in the Hive metastore |
spark.driver.hive_external_catalog.hive_client_calls | sum (monotonic) | int | { call } | development | yes | | Number of calls to the underlying Hive Metastore client made by the Spark application. |
spark.driver.hive_external_catalog.parallel_listing_jobs | sum (monotonic) | int | { listing_job } | development | yes | | Number of parallel listing jobs initiated by the HiveExternalCatalog when listing partitions of a table. |
spark.driver.hive_external_catalog.partitions_fetched | sum (monotonic) | int | { partition } | development | yes | | Table partitions fetched by the HiveExternalCatalog. |
spark.driver.jvm_cpu_time | sum (monotonic) | int | ns | development | yes | | Current CPU time taken by the Spark driver. |
spark.driver.live_listener_bus.dropped | sum (monotonic) | int | { event } | development | yes | | Number of events that have been dropped by the LiveListenerBus. |
spark.driver.live_listener_bus.posted | sum (monotonic) | int | { event } | development | yes | | Number of events that have been posted on the LiveListenerBus. |
spark.driver.live_listener_bus.processing_time.average | gauge | double | ms | development | yes | | Average time taken for the LiveListenerBus to process an event posted to it. |
spark.driver.live_listener_bus.queue_size | sum | int | { event } | development | yes | | Number of events currently waiting to be processed by the LiveListenerBus. |
spark.executor.disk.usage | sum | int | By | development | yes | | Disk space used by this executor for RDD storage. |
spark.executor.gc_time | sum (monotonic) | int | ms | development | yes | | Elapsed time the JVM spent in garbage collection in this executor. |
spark.executor.input_size | sum (monotonic) | int | By | development | yes | | Amount of data input for this executor. |
spark.executor.memory.usage | sum | int | By | development | yes | | Storage memory used by this executor. |
spark.executor.shuffle.io.size | sum (monotonic) | int | By | development | yes | direction (string) | Amount of data written and read during shuffle operations for this executor. |
spark.executor.storage_memory.usage | sum | int | By | development | yes | location (string), state (string) | The executor's storage memory usage. |
spark.executor.task.active | sum | int | { task } | development | yes | | Number of tasks currently running in this executor. |
spark.executor.task.limit | sum | int | { task } | development | yes | | Maximum number of tasks that can run concurrently in this executor. |
spark.executor.task.result | sum (monotonic) | int | { task } | development | yes | executor_task_result (string) | Number of tasks with a specific result in this executor. |
spark.executor.time | sum (monotonic) | int | ms | development | yes | | Elapsed time the JVM spent executing tasks in this executor. |
spark.job.stage.active | sum | int | { stage } | development | yes | | Number of active stages in this job. |
spark.job.stage.result | sum (monotonic) | int | { stage } | development | yes | job_result (string) | Number of stages with a specific result in this job. |
spark.job.task.active | sum | int | { task } | development | yes | | Number of active tasks in this job. |
spark.job.task.result | sum (monotonic) | int | { task } | development | yes | job_result (string) | Number of tasks with a specific result in this job. |
spark.stage.disk.spilled | sum (monotonic) | int | By | development | yes | | The amount of disk space used for storing portions of overly large data chunks that couldn't fit in memory in this stage. |
spark.stage.executor.cpu_time | sum (monotonic) | int | ns | development | yes | | CPU time spent by the executor in this stage. |
spark.stage.executor.run_time | sum (monotonic) | int | ms | development | yes | | Amount of time spent by the executor in this stage. |
spark.stage.io.records | sum (monotonic) | int | { record } | development | yes | direction (string) | Number of records written and read in this stage. |
spark.stage.io.size | sum (monotonic) | int | By | development | yes | direction (string) | Amount of data written and read at this stage. |
spark.stage.jvm_gc_time | sum (monotonic) | int | ms | development | yes | | The amount of time the JVM spent on garbage collection in this stage. |
spark.stage.memory.peak | sum (monotonic) | int | By | development | yes | | Peak memory used by internal data structures created during shuffles, aggregations and joins in this stage. |
spark.stage.memory.spilled | sum (monotonic) | int | By | development | yes | | The amount of memory moved to disk due to size constraints (spilled) in this stage. |
spark.stage.shuffle.blocks_fetched | sum (monotonic) | int | { block } | development | yes | source (string) | Number of blocks fetched in shuffle operations in this stage. |
spark.stage.shuffle.fetch_wait_time | sum (monotonic) | int | ms | development | yes | | Time spent in this stage waiting for remote shuffle blocks. |
spark.stage.shuffle.io.disk | sum (monotonic) | int | By | development | yes | | Amount of data read to disk in shuffle operations (sometimes required for large blocks, as opposed to the default behavior of reading into memory). |
spark.stage.shuffle.io.read.size | sum (monotonic) | int | By | development | yes | source (string) | Amount of data read in shuffle operations in this stage. |
spark.stage.shuffle.io.records | sum (monotonic) | int | { record } | development | yes | direction (string) | Number of records written or read in shuffle operations in this stage. |
spark.stage.shuffle.io.write.size | sum (monotonic) | int | By | development | yes | | Amount of data written in shuffle operations in this stage. |
spark.stage.shuffle.write_time | sum (monotonic) | int | ns | development | yes | | Time spent blocking on writes to disk or buffer cache in this stage. |
spark.stage.status | sum | int | { status } | development | yes | stage_active (bool), stage_complete (bool), stage_failed (bool), stage_pending (bool) | A one-hot encoding representing the status of this stage. |
spark.stage.task.active | sum | int | { task } | development | yes | | Number of active tasks in this stage. |
spark.stage.task.result | sum (monotonic) | int | { task } | development | yes | stage_task_result (string) | Number of tasks with a specific result in this stage. |
spark.stage.task.result_size | sum (monotonic) | int | By | development | yes | | The amount of data transmitted back to the driver by all the tasks in this stage. |