digraph G {
0 [id="node0" labelType="html" label="<br><b>Project</b><br><br>" tooltip="Project [add#4492.path AS path#4576, add#4492.partitionValues AS partitionValues#4577, add#4492.size AS size#4578L, add#4492.modificationTime AS modificationTime#4579L, add#4492.dataChange AS dataChange#4580, from_json(StructField(numRecords,LongType,true), StructField(minValues,StructType(StructField(VendorID,LongType,true),StructField(tpep_pickup_datetime,TimestampType,true),StructField(tpep_dropoff_datetime,TimestampType,true),StructField(passenger_count,LongType,true),StructField(trip_distance,DoubleType,true),StructField(RatecodeID,LongType,true),StructField(store_and_fwd_flag,StringType,true),StructField(PULocationID,LongType,true),StructField(DOLocationID,LongType,true),StructField(payment_type,LongType,true),StructField(fare_amount,DoubleType,true),StructField(extra,DoubleType,true),StructField(mta_tax,DoubleType,true),StructField(tip_amount,DoubleType,true),StructField(tolls_amount,DoubleType,true),StructField(improvement_surcharge,DoubleType,true),StructField(total_amount,DoubleType,true),StructField(congestion_surcharge,DoubleType,true),StructField(airport_fee,DoubleType,true),StructField(load_timestamp,TimestampType,true),StructField(day,IntegerType,true)),true), StructField(maxValues,StructType(StructField(VendorID,LongType,true),StructField(tpep_pickup_datetime,TimestampType,true),StructField(tpep_dropoff_datetime,TimestampType,true),StructField(passenger_count,LongType,true),StructField(trip_distance,DoubleType,true),StructField(RatecodeID,LongType,true),StructField(store_and_fwd_flag,StringType,true),StructField(PULocationID,LongType,true),StructField(DOLocationID,LongType,true),StructField(payment_type,LongType,true),StructField(fare_amount,DoubleType,true),StructField(extra,DoubleType,true),StructField(mta_tax,DoubleType,true),StructField(tip_amount,DoubleType,true),StructField(tolls_amount,DoubleType,true),StructField(improvement_surcharge,DoubleType,true),StructField(total_amount,DoubleType,true),StructField(congestion_surcharge,DoubleType,true),StructField(airport_fee,DoubleType,true),StructField(load_timestamp,TimestampType,true),StructField(day,IntegerType,true)),true), StructField(nullCount,StructType(StructField(VendorID,LongType,true),StructField(tpep_pickup_datetime,LongType,true),StructField(tpep_dropoff_datetime,LongType,true),StructField(passenger_count,LongType,true),StructField(trip_distance,LongType,true),StructField(RatecodeID,LongType,true),StructField(store_and_fwd_flag,LongType,true),StructField(PULocationID,LongType,true),StructField(DOLocationID,LongType,true),StructField(payment_type,LongType,true),StructField(fare_amount,LongType,true),StructField(extra,LongType,true),StructField(mta_tax,LongType,true),StructField(tip_amount,LongType,true),StructField(tolls_amount,LongType,true),StructField(improvement_surcharge,LongType,true),StructField(total_amount,LongType,true),StructField(congestion_surcharge,LongType,true),StructField(airport_fee,LongType,true),StructField(load_timestamp,LongType,true),StructField(day,LongType,true)),true), add#4492.stats, Some(Etc/UTC), false) AS stats#4587, add#4492.tags AS tags#4582, add#4492.deletionVector AS deletionVector#4583, add#4492.baseRowId AS baseRowId#4584L, add#4492.defaultRowCommitVersion AS defaultRowCommitVersion#4585L, add#4492.clusteringProvider AS clusteringProvider#4586]"];
subgraph cluster1 {
isCluster="true";
id="cluster1";
label="WholeStageCodegen (1)";
tooltip="WholeStageCodegen (1)";
2 [id="node2" labelType="html" label="<br><b>Filter</b><br><br>" tooltip="Filter isnotnull(add#4492)"];
3 [id="node3" labelType="html" label="<br><b>Scan ExistingRDD Delta Table State #2 - file:/home/jovyan/spark-warehouse/yellow_taxi.db/trips/_delta_log</b><br><br>" tooltip="Scan ExistingRDD Delta Table State #2 - file:/home/jovyan/spark-warehouse/yellow_taxi.db/trips/_delta_log[txn#4491,add#4492,remove#4493,metaData#4494,protocol#4495,cdc#4496,checkpointMetadata#4497,sidecar#4498,domainMetadata#4499,commitInfo#4500]"];
}
2->0;
3->2;
}
== Physical Plan ==
Project (3)
+- * Filter (2)
+- * Scan ExistingRDD Delta Table State #2 - file:/home/jovyan/spark-warehouse/yellow_taxi.db/trips/_delta_log (1)
(1) Scan ExistingRDD Delta Table State #2 - file:/home/jovyan/spark-warehouse/yellow_taxi.db/trips/_delta_log [codegen id : 1]
Output [10]: [txn#4491, add#4492, remove#4493, metaData#4494, protocol#4495, cdc#4496, checkpointMetadata#4497, sidecar#4498, domainMetadata#4499, commitInfo#4500]
Arguments: [txn#4491, add#4492, remove#4493, metaData#4494, protocol#4495, cdc#4496, checkpointMetadata#4497, sidecar#4498, domainMetadata#4499, commitInfo#4500], Delta Table State #2 - file:/home/jovyan/spark-warehouse/yellow_taxi.db/trips/_delta_log MapPartitionsRDD[205] at Spark Connect - session_id: "29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0"
plan {
root {
common {
plan_id: 112
}
limit {
in..., ExistingRDD, UnknownPartitioning(0)
(2) Filter [codegen id : 1]
Input [10]: [txn#4491, add#4492, remove#4493, metaData#4494, protocol#4495, cdc#4496, checkpointMetadata#4497, sidecar#4498, domainMetadata#4499, commitInfo#4500]
Condition : isnotnull(add#4492)
(3) Project
Output [11]: [add#4492.path AS path#4576, add#4492.partitionValues AS partitionValues#4577, add#4492.size AS size#4578L, add#4492.modificationTime AS modificationTime#4579L, add#4492.dataChange AS dataChange#4580, from_json(StructField(numRecords,LongType,true), StructField(minValues,StructType(StructField(VendorID,LongType,true),StructField(tpep_pickup_datetime,TimestampType,true),StructField(tpep_dropoff_datetime,TimestampType,true),StructField(passenger_count,LongType,true),StructField(trip_distance,DoubleType,true),StructField(RatecodeID,LongType,true),StructField(store_and_fwd_flag,StringType,true),StructField(PULocationID,LongType,true),StructField(DOLocationID,LongType,true),StructField(payment_type,LongType,true),StructField(fare_amount,DoubleType,true),StructField(extra,DoubleType,true),StructField(mta_tax,DoubleType,true),StructField(tip_amount,DoubleType,true),StructField(tolls_amount,DoubleType,true),StructField(improvement_surcharge,DoubleType,true),StructField(total_amount,DoubleType,true),StructField(congestion_surcharge,DoubleType,true),StructField(airport_fee,DoubleType,true),StructField(load_timestamp,TimestampType,true),StructField(day,IntegerType,true)),true), StructField(maxValues,StructType(StructField(VendorID,LongType,true),StructField(tpep_pickup_datetime,TimestampType,true),StructField(tpep_dropoff_datetime,TimestampType,true),StructField(passenger_count,LongType,true),StructField(trip_distance,DoubleType,true),StructField(RatecodeID,LongType,true),StructField(store_and_fwd_flag,StringType,true),StructField(PULocationID,LongType,true),StructField(DOLocationID,LongType,true),StructField(payment_type,LongType,true),StructField(fare_amount,DoubleType,true),StructField(extra,DoubleType,true),StructField(mta_tax,DoubleType,true),StructField(tip_amount,DoubleType,true),StructField(tolls_amount,DoubleType,true),StructField(improvement_surcharge,DoubleType,true),StructField(total_amount,DoubleType,true),StructField(congestion_surcharge,DoubleType,true),StructField(airport_fee,DoubleType,true),StructField(load_timestamp,TimestampType,true),StructField(day,IntegerType,true)),true), StructField(nullCount,StructType(StructField(VendorID,LongType,true),StructField(tpep_pickup_datetime,LongType,true),StructField(tpep_dropoff_datetime,LongType,true),StructField(passenger_count,LongType,true),StructField(trip_distance,LongType,true),StructField(RatecodeID,LongType,true),StructField(store_and_fwd_flag,LongType,true),StructField(PULocationID,LongType,true),StructField(DOLocationID,LongType,true),StructField(payment_type,LongType,true),StructField(fare_amount,LongType,true),StructField(extra,LongType,true),StructField(mta_tax,LongType,true),StructField(tip_amount,LongType,true),StructField(tolls_amount,LongType,true),StructField(improvement_surcharge,LongType,true),StructField(total_amount,LongType,true),StructField(congestion_surcharge,LongType,true),StructField(airport_fee,LongType,true),StructField(load_timestamp,LongType,true),StructField(day,LongType,true)),true), add#4492.stats, Some(Etc/UTC), false) AS stats#4587, add#4492.tags AS tags#4582, add#4492.deletionVector AS deletionVector#4583, add#4492.baseRowId AS baseRowId#4584L, add#4492.defaultRowCommitVersion AS defaultRowCommitVersion#4585L, add#4492.clusteringProvider AS clusteringProvider#4586]
Input [10]: [txn#4491, add#4492, remove#4493, metaData#4494, protocol#4495, cdc#4496, checkpointMetadata#4497, sidecar#4498, domainMetadata#4499, commitInfo#4500]