digraph G {
0 [id="node0" labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>" tooltip="AdaptiveSparkPlan isFinalPlan=true"];
subgraph cluster1 {
isCluster="true";
id="cluster1";
label="WholeStageCodegen (2)\n \nduration: 12 ms";
tooltip="WholeStageCodegen (2)";
2 [id="node2" labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build: 12 ms<br>number of output rows: 1" tooltip="HashAggregate(keys=[], functions=[sum((_extract_numRecords#4803L - coalesce(_extract_cardinality#4802L, 0))), count(CASE WHEN isnull(_extract_numRecords#4803L) THEN 1 END), count(CASE WHEN (_extract_numRecords#4803L > 0) THEN 1 END)])"];
}
3 [id="node3" labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 50<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 50<br>local bytes read: 1932.0 B<br>merged fetch fallback count: 0<br>local blocks read: 31<br>remote merged chunks fetched: 0<br>remote blocks read: 19<br>data size total (min, med, max (stageId: taskId))<br>1600.0 B (32.0 B, 32.0 B, 32.0 B (stage 108.0: task 203))<br>local merged bytes read: 0.0 B<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>38 ms (0 ms, 0 ms, 2 ms (stage 108.0: task 224))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>fetch wait time: 43 ms<br>remote bytes read: 1102.0 B<br>number of partitions: 1<br>remote reqs duration: 68 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>3.0 KiB (58.0 B, 58.0 B, 68.0 B (stage 108.0: task 215))" tooltip="Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=195]"];
subgraph cluster4 {
isCluster="true";
id="cluster4";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n20 ms (0 ms, 0 ms, 6 ms (stage 108.0: task 204))";
tooltip="WholeStageCodegen (1)";
5 [id="node5" labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build total (min, med, max (stageId: taskId))<br>2 ms (0 ms, 0 ms, 1 ms (stage 108.0: task 223))<br>number of output rows: 50" tooltip="HashAggregate(keys=[], functions=[partial_sum((_extract_numRecords#4803L - coalesce(_extract_cardinality#4802L, 0))), partial_count(CASE WHEN isnull(_extract_numRecords#4803L) THEN 1 END), partial_count(CASE WHEN (_extract_numRecords#4803L > 0) THEN 1 END)])"];
6 [id="node6" labelType="html" label="<br><b>Project</b><br><br>" tooltip="Project [stats#4587.numRecords AS _extract_numRecords#4803L, deletionVector#4583.cardinality AS _extract_cardinality#4802L]"];
7 [id="node7" labelType="html" label="<b>Scan ExistingRDD Delta Table State with Stats #2 - file:/home/jovyan/spark-warehouse/yellow_taxi.db/trips/_delta_log</b><br><br>number of output rows: 17" tooltip="Scan ExistingRDD Delta Table State with Stats #2 - file:/home/jovyan/spark-warehouse/yellow_taxi.db/trips/_delta_log[path#4576,partitionValues#4577,size#4578L,modificationTime#4579L,dataChange#4580,stats#4587,tags#4582,deletionVector#4583,baseRowId#4584L,defaultRowCommitVersion#4585L,clusteringProvider#4586]"];
}
2->0;
3->2;
5->3;
6->5;
7->6;
}
== Physical Plan ==
AdaptiveSparkPlan (10)
+- == Final Plan ==
ResultQueryStage (7), Statistics(sizeInBytes=8.0 EiB)
+- * HashAggregate (6)
+- ShuffleQueryStage (5), Statistics(sizeInBytes=1600.0 B, rowCount=50)
+- Exchange (4)
+- * HashAggregate (3)
+- * Project (2)
+- * Scan ExistingRDD Delta Table State with Stats #2 - file:/home/jovyan/spark-warehouse/yellow_taxi.db/trips/_delta_log (1)
+- == Initial Plan ==
HashAggregate (9)
+- Exchange (8)
+- HashAggregate (3)
+- Project (2)
+- Scan ExistingRDD Delta Table State with Stats #2 - file:/home/jovyan/spark-warehouse/yellow_taxi.db/trips/_delta_log (1)
(1) Scan ExistingRDD Delta Table State with Stats #2 - file:/home/jovyan/spark-warehouse/yellow_taxi.db/trips/_delta_log [codegen id : 1]
Output [11]: [path#4576, partitionValues#4577, size#4578L, modificationTime#4579L, dataChange#4580, stats#4587, tags#4582, deletionVector#4583, baseRowId#4584L, defaultRowCommitVersion#4585L, clusteringProvider#4586]
Arguments: [path#4576, partitionValues#4577, size#4578L, modificationTime#4579L, dataChange#4580, stats#4587, tags#4582, deletionVector#4583, baseRowId#4584L, defaultRowCommitVersion#4585L, clusteringProvider#4586], Delta Table State with Stats #2 - file:/home/jovyan/spark-warehouse/yellow_taxi.db/trips/_delta_log MapPartitionsRDD[209] at Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
root {
common {
plan_id: 112
}
limit {
in..., ExistingRDD, UnknownPartitioning(0)
(2) Project [codegen id : 1]
Output [2]: [stats#4587.numRecords AS _extract_numRecords#4803L, deletionVector#4583.cardinality AS _extract_cardinality#4802L]
Input [11]: [path#4576, partitionValues#4577, size#4578L, modificationTime#4579L, dataChange#4580, stats#4587, tags#4582, deletionVector#4583, baseRowId#4584L, defaultRowCommitVersion#4585L, clusteringProvider#4586]
(3) HashAggregate [codegen id : 1]
Input [2]: [_extract_numRecords#4803L, _extract_cardinality#4802L]
Keys: []
Functions [3]: [partial_sum((_extract_numRecords#4803L - coalesce(_extract_cardinality#4802L, 0))), partial_count(CASE WHEN isnull(_extract_numRecords#4803L) THEN 1 END), partial_count(CASE WHEN (_extract_numRecords#4803L > 0) THEN 1 END)]
Aggregate Attributes [3]: [sum#4804L, count#4805L, count#4806L]
Results [3]: [sum#4807L, count#4808L, count#4809L]
(4) Exchange
Input [3]: [sum#4807L, count#4808L, count#4809L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=195]
(5) ShuffleQueryStage
Output [3]: [sum#4807L, count#4808L, count#4809L]
Arguments: 0
(6) HashAggregate [codegen id : 2]
Input [3]: [sum#4807L, count#4808L, count#4809L]
Keys: []
Functions [3]: [sum((_extract_numRecords#4803L - coalesce(_extract_cardinality#4802L, 0))), count(CASE WHEN isnull(_extract_numRecords#4803L) THEN 1 END), count(CASE WHEN (_extract_numRecords#4803L > 0) THEN 1 END)]
Aggregate Attributes [3]: [sum((_extract_numRecords#4803L - coalesce(_extract_cardinality#4802L, 0)))#4800L, count(CASE WHEN isnull(_extract_numRecords#4803L) THEN 1 END)#4799L, count(CASE WHEN (_extract_numRecords#4803L > 0) THEN 1 END)#4801L]
Results [3]: [sum((_extract_numRecords#4803L - coalesce(_extract_cardinality#4802L, 0)))#4800L AS numLogicalRecords#4792L, count(CASE WHEN isnull(_extract_numRecords#4803L) THEN 1 END)#4799L AS missingNumRecords#4793L, count(CASE WHEN (_extract_numRecords#4803L > 0) THEN 1 END)#4801L AS countNonEmptyFiles#4794L]
(7) ResultQueryStage
Output [3]: [numLogicalRecords#4792L, missingNumRecords#4793L, countNonEmptyFiles#4794L]
Arguments: 1
(8) Exchange
Input [3]: [sum#4807L, count#4808L, count#4809L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=185]
(9) HashAggregate
Input [3]: [sum#4807L, count#4808L, count#4809L]
Keys: []
Functions [3]: [sum((_extract_numRecords#4803L - coalesce(_extract_cardinality#4802L, 0))), count(CASE WHEN isnull(_extract_numRecords#4803L) THEN 1 END), count(CASE WHEN (_extract_numRecords#4803L > 0) THEN 1 END)]
Aggregate Attributes [3]: [sum((_extract_numRecords#4803L - coalesce(_extract_cardinality#4802L, 0)))#4800L, count(CASE WHEN isnull(_extract_numRecords#4803L) THEN 1 END)#4799L, count(CASE WHEN (_extract_numRecords#4803L > 0) THEN 1 END)#4801L]
Results [3]: [sum((_extract_numRecords#4803L - coalesce(_extract_cardinality#4802L, 0)))#4800L AS numLogicalRecords#4792L, count(CASE WHEN isnull(_extract_numRecords#4803L) THEN 1 END)#4799L AS missingNumRecords#4793L, count(CASE WHEN (_extract_numRecords#4803L > 0) THEN 1 END)#4801L AS countNonEmptyFiles#4794L]
(10) AdaptiveSparkPlan
Output [3]: [numLogicalRecords#4792L, missingNumRecords#4793L, countNonEmptyFiles#4794L]
Arguments: isFinalPlan=true