digraph G {
0 [id="node0" labelType="html" label="<b>CollectLimit</b><br><br>shuffle records written: 1,004<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 1,000<br>local bytes read: 53.9 KiB<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 1<br>local merged bytes read: 0.0 B<br>remote reqs duration: 4 ms<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>3 ms (1 ms, 1 ms, 1 ms (stage 105.0: task 200))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>fetch wait time: 0 ms<br>remote bytes read: 443.0 B<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>54.4 KiB (443.0 B, 53.9 KiB, 53.9 KiB (stage 105.0: task 200))" tooltip="CollectLimit 1000"];
subgraph cluster1 {
isCluster="true";
id="cluster1";
label="WholeStageCodegen (1)\n \nduration: 1.1 s";
tooltip="WholeStageCodegen (1)";
2 [id="node2" labelType="html" label="<br><b>Project</b><br><br>" tooltip="Project [VendorID#4366L, tpep_pickup_datetime#4367, tpep_dropoff_datetime#4368, passenger_count#4369L, trip_distance#4370, RatecodeID#4371L, store_and_fwd_flag#4372, PULocationID#4373L, DOLocationID#4374L, payment_type#4375L, fare_amount#4376, extra#4377, mta_tax#4378, tip_amount#4379, tolls_amount#4380, improvement_surcharge#4381, total_amount#4382, congestion_surcharge#4383, airport_fee#4384, load_timestamp#4385, year#4386, month#4387, day#4388]"];
3 [id="node3" labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 4,100<br>number of input batches: 3" tooltip="ColumnarToRow"];
}
4 [id="node4" labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 3<br>scan time total (min, med, max (stageId: taskId))<br>2.5 s (1.1 s, 1.4 s, 1.4 s (stage 105.0: task 200))<br>dynamic partition pruning time: 0 ms<br>metadata time: 4 ms<br>size of files read: 25.8 MiB<br>number of output rows: 4,100<br>number of partitions read: 3" tooltip="FileScan parquet [VendorID#4366L,tpep_pickup_datetime#4367,tpep_dropoff_datetime#4368,passenger_count#4369L,trip_distance#4370,RatecodeID#4371L,store_and_fwd_flag#4372,PULocationID#4373L,DOLocationID#4374L,payment_type#4375L,fare_amount#4376,extra#4377,mta_tax#4378,tip_amount#4379,tolls_amount#4380,improvement_surcharge#4381,total_amount#4382,congestion_surcharge#4383,airport_fee#4384,load_timestamp#4385,day#4388,year#4386,month#4387] Batched: true, DataFilters: [], Format: Parquet, Location: PreparedDeltaFileIndex(1 paths)[file:/home/jovyan/spark-warehouse/yellow_taxi.db/trips], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<VendorID:bigint,tpep_pickup_datetime:timestamp,tpep_dropoff_datetime:timestamp,passenger_c..."];
2->0;
3->2;
4->3;
}
== Physical Plan ==
CollectLimit (4)
+- * Project (3)
+- * ColumnarToRow (2)
+- Scan parquet (1)
(1) Scan parquet
Output [23]: [VendorID#4366L, tpep_pickup_datetime#4367, tpep_dropoff_datetime#4368, passenger_count#4369L, trip_distance#4370, RatecodeID#4371L, store_and_fwd_flag#4372, PULocationID#4373L, DOLocationID#4374L, payment_type#4375L, fare_amount#4376, extra#4377, mta_tax#4378, tip_amount#4379, tolls_amount#4380, improvement_surcharge#4381, total_amount#4382, congestion_surcharge#4383, airport_fee#4384, load_timestamp#4385, day#4388, year#4386, month#4387]
Batched: true
Location: PreparedDeltaFileIndex [file:/home/jovyan/spark-warehouse/yellow_taxi.db/trips]
ReadSchema: struct<VendorID:bigint,tpep_pickup_datetime:timestamp,tpep_dropoff_datetime:timestamp,passenger_count:bigint,trip_distance:double,RatecodeID:bigint,store_and_fwd_flag:string,PULocationID:bigint,DOLocationID:bigint,payment_type:bigint,fare_amount:double,extra:double,mta_tax:double,tip_amount:double,tolls_amount:double,improvement_surcharge:double,total_amount:double,congestion_surcharge:double,airport_fee:double,load_timestamp:timestamp,day:int>
(2) ColumnarToRow [codegen id : 1]
Input [23]: [VendorID#4366L, tpep_pickup_datetime#4367, tpep_dropoff_datetime#4368, passenger_count#4369L, trip_distance#4370, RatecodeID#4371L, store_and_fwd_flag#4372, PULocationID#4373L, DOLocationID#4374L, payment_type#4375L, fare_amount#4376, extra#4377, mta_tax#4378, tip_amount#4379, tolls_amount#4380, improvement_surcharge#4381, total_amount#4382, congestion_surcharge#4383, airport_fee#4384, load_timestamp#4385, day#4388, year#4386, month#4387]
(3) Project [codegen id : 1]
Output [23]: [VendorID#4366L, tpep_pickup_datetime#4367, tpep_dropoff_datetime#4368, passenger_count#4369L, trip_distance#4370, RatecodeID#4371L, store_and_fwd_flag#4372, PULocationID#4373L, DOLocationID#4374L, payment_type#4375L, fare_amount#4376, extra#4377, mta_tax#4378, tip_amount#4379, tolls_amount#4380, improvement_surcharge#4381, total_amount#4382, congestion_surcharge#4383, airport_fee#4384, load_timestamp#4385, year#4386, month#4387, day#4388]
Input [23]: [VendorID#4366L, tpep_pickup_datetime#4367, tpep_dropoff_datetime#4368, passenger_count#4369L, trip_distance#4370, RatecodeID#4371L, store_and_fwd_flag#4372, PULocationID#4373L, DOLocationID#4374L, payment_type#4375L, fare_amount#4376, extra#4377, mta_tax#4378, tip_amount#4379, tolls_amount#4380, improvement_surcharge#4381, total_amount#4382, congestion_surcharge#4383, airport_fee#4384, load_timestamp#4385, day#4388, year#4386, month#4387]
(4) CollectLimit
Input [23]: [VendorID#4366L, tpep_pickup_datetime#4367, tpep_dropoff_datetime#4368, passenger_count#4369L, trip_distance#4370, RatecodeID#4371L, store_and_fwd_flag#4372, PULocationID#4373L, DOLocationID#4374L, payment_type#4375L, fare_amount#4376, extra#4377, mta_tax#4378, tip_amount#4379, tolls_amount#4380, improvement_surcharge#4381, total_amount#4382, congestion_surcharge#4383, airport_fee#4384, load_timestamp#4385, year#4386, month#4387, day#4388]
Arguments: 1000