|
|
[209]
|
[215]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
171 ms
|
175 ms
|
write_operation {
input {
common {
plan_id: 94
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-07.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
00f8c9d7-c11a-411f-b7be-c0be0e2f29b0
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_00f8c9d7-c11a-411f-b7be-c0be0e2f29b0
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[169][170]
|
[175][176]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:07
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
25 seconds 362 ms
|
25 seconds 366 ms
|
write_operation {
input {
common {
plan_id: 16
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
035259f2-8534-4df9-9603-193cb3db6fe8
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_035259f2-8534-4df9-9603-193cb3db6fe8
|
|
|
|
|
[186]
|
[192]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
81 ms
|
86 ms
|
write_operation {
input {
common {
plan_id: 48
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
038056f6-65ae-4842-828e-83013290f29d
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_038056f6-65ae-4842-828e-83013290f29d
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'passenger_count' and 'passenger_count'
|
|
|
[242]
|
[248]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
79 ms
|
82 ms
|
write_operation {
input {
common {
plan_id: 158
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-11.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
04c2c4d4-48cd-41f1-911a-34f31a74a4dd
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_04c2c4d4-48cd-41f1-911a-34f31a74a4dd
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[173]
|
[179]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
93 ms
|
98 ms
|
write_operation {
input {
common {
plan_id: 22
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
070b7388-8ec5-4d50-b7fa-defa8b2fc8d7
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_070b7388-8ec5-4d50-b7fa-defa8b2fc8d7
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[214]
|
[220]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
2026/09/03 16:13:39
|
91 ms
|
98 ms
|
write_operation {
input {
common {
plan_id: 104
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
0cc0f910-7a61-4785-aaf6-54f3223e0dab
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_0cc0f910-7a61-4785-aaf6-54f3223e0dab
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[225]
|
[231]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
86 ms
|
90 ms
|
write_operation {
input {
common {
plan_id: 124
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-10.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
0ee9b11f-8e57-463c-86a9-723e65186262
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_0ee9b11f-8e57-463c-86a9-723e65186262
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[227]
|
[233]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
102 ms
|
106 ms
|
write_operation {
input {
common {
plan_id: 128
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-01.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
0f6391dc-4186-438a-a313-ba7c32f0b0e6
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_0f6391dc-4186-438a-a313-ba7c32f0b0e6
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[193]
|
[199]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
136 ms
|
141 ms
|
write_operation {
input {
common {
plan_id: 62
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-09.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
11505f12-b6ea-4784-8780-c85845c387a8
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_11505f12-b6ea-4784-8780-c85845c387a8
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[250]
|
[256]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
158 ms
|
162 ms
|
write_operation {
input {
common {
plan_id: 174
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
12b1ae76-d166-4c27-9c07-8187996ee91e
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_12b1ae76-d166-4c27-9c07-8187996ee91e
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'passenger_count' and 'passenger_count'
|
|
|
[256]
|
[262]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
143 ms
|
146 ms
|
write_operation {
input {
common {
plan_id: 186
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-02.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
131986c8-8cfc-4db3-a6cc-6201942f62e0
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_131986c8-8cfc-4db3-a6cc-6201942f62e0
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[207]
|
[213]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
127 ms
|
131 ms
|
write_operation {
input {
common {
plan_id: 90
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-03.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
14c8aa2c-e9b7-4654-8609-907121af310f
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_14c8aa2c-e9b7-4654-8609-907121af310f
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'passenger_count' and 'passenger_count'
|
|
|
[211]
|
[217]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
95 ms
|
99 ms
|
write_operation {
input {
common {
plan_id: 98
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
15644fc0-5219-436f-978a-262f2851cff8
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_15644fc0-5219-436f-978a-262f2851cff8
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[195]
|
[201]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
107 ms
|
111 ms
|
write_operation {
input {
common {
plan_id: 66
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
1842e87c-6411-4588-a7cc-b4a675f5ab04
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_1842e87c-6411-4588-a7cc-b4a675f5ab04
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[212]
|
[218]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
97 ms
|
100 ms
|
write_operation {
input {
common {
plan_id: 100
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-03.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
193c6253-94c4-4d1c-b0b0-d48c996d246e
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_193c6253-94c4-4d1c-b0b0-d48c996d246e
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[181]
|
[187]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
116 ms
|
121 ms
|
write_operation {
input {
common {
plan_id: 38
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-07.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
19df5b34-f378-494e-81db-e3371ab2c257
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_19df5b34-f378-494e-81db-e3371ab2c257
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[196]
|
[202]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
87 ms
|
94 ms
|
write_operation {
input {
common {
plan_id: 68
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
1a81ea2a-525d-4e8d-9b12-741b7aa7c3f9
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_1a81ea2a-525d-4e8d-9b12-741b7aa7c3f9
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[218]
|
[224]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:39
|
2026/09/03 16:13:39
|
2026/09/03 16:13:39
|
86 ms
|
91 ms
|
write_operation {
input {
common {
plan_id: 112
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
1b892df8-440a-4e11-8f2a-a8e9f5a0d381
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_1b892df8-440a-4e11-8f2a-a8e9f5a0d381
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[206]
|
[212]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:37
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
116 ms
|
121 ms
|
write_operation {
input {
common {
plan_id: 88
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-02.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
1f032c4e-d4a8-4748-b54f-a53e0013de78
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_1f032c4e-d4a8-4748-b54f-a53e0013de78
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[219][220]
|
[225][226]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:39
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
26 seconds 854 ms
|
26 seconds 857 ms
|
write_operation {
input {
common {
plan_id: 114
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-07.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
21211c3d-a004-4101-a1bb-488c89fddc2f
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_21211c3d-a004-4101-a1bb-488c89fddc2f
|
|
|
|
|
[215]
|
[221]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:39
|
2026/09/03 16:13:39
|
2026/09/03 16:13:39
|
106 ms
|
110 ms
|
write_operation {
input {
common {
plan_id: 106
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-04.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
29dbea9c-4898-40e0-8360-9ebb3400571e
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_29dbea9c-4898-40e0-8360-9ebb3400571e
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'passenger_count' and 'passenger_count'
|
|
|
[230]
|
[236]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
134 ms
|
137 ms
|
write_operation {
input {
common {
plan_id: 134
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-09.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
2b2caa6f-19ec-4b1d-a859-3e6126e4c68b
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_2b2caa6f-19ec-4b1d-a859-3e6126e4c68b
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[192]
|
[198]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:35
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
279 ms
|
284 ms
|
write_operation {
input {
common {
plan_id: 60
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-04.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
2c86dd1a-1570-4f98-9e2e-9b81a80d2006
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_2c86dd1a-1570-4f98-9e2e-9b81a80d2006
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[240]
|
[246]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
91 ms
|
94 ms
|
write_operation {
input {
common {
plan_id: 154
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-07.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
2cd0c3f1-7085-448d-af0e-31d580b5d84f
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_2cd0c3f1-7085-448d-af0e-31d580b5d84f
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[241]
|
[247]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
82 ms
|
85 ms
|
write_operation {
input {
common {
plan_id: 156
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-07.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
2d69db4e-6414-4518-b9f7-a4f2a54548ab
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_2d69db4e-6414-4518-b9f7-a4f2a54548ab
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[200]
|
[206]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
92 ms
|
98 ms
|
write_operation {
input {
common {
plan_id: 76
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-01.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
31428a5a-88f9-453a-b45c-c7644ee6ce7e
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_31428a5a-88f9-453a-b45c-c7644ee6ce7e
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[183]
|
[189]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:34
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
105 ms
|
111 ms
|
write_operation {
input {
common {
plan_id: 42
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
320e82c2-abf9-41e7-831c-5b4e4e6469f9
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_320e82c2-abf9-41e7-831c-5b4e4e6469f9
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[185]
|
[191]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
83 ms
|
88 ms
|
write_operation {
input {
common {
plan_id: 46
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
3a121330-0197-4c65-82ea-66cd99480a68
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_3a121330-0197-4c65-82ea-66cd99480a68
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[190]
|
[196]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
79 ms
|
82 ms
|
write_operation {
input {
common {
plan_id: 56
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
3a1b4a58-3756-4684-a865-1c18f8afce94
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_3a1b4a58-3756-4684-a865-1c18f8afce94
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[210]
|
[216]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
141 ms
|
145 ms
|
write_operation {
input {
common {
plan_id: 96
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-09.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
3c9481a0-0f2d-428a-9746-fcd72062da9f
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_3c9481a0-0f2d-428a-9746-fcd72062da9f
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[243]
|
[249]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
85 ms
|
88 ms
|
write_operation {
input {
common {
plan_id: 160
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
425e9ce3-23ef-448b-be7a-da8b0336f908
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_425e9ce3-23ef-448b-be7a-da8b0336f908
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[175]
|
[181]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
110 ms
|
118 ms
|
write_operation {
input {
common {
plan_id: 26
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
43c883ae-8661-4fae-9072-fff947596643
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_43c883ae-8661-4fae-9072-fff947596643
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[197]
|
[203]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
99 ms
|
105 ms
|
write_operation {
input {
common {
plan_id: 70
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
43d0c529-3218-4ae3-a405-0e00f1eb6600
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_43d0c529-3218-4ae3-a405-0e00f1eb6600
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[187]
|
[193]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
87 ms
|
93 ms
|
write_operation {
input {
common {
plan_id: 50
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-04.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
453243e3-12f6-43ff-b295-28633ad30539
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_453243e3-12f6-43ff-b295-28633ad30539
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[163]
|
[169]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:12:39
|
2026/09/03 16:12:39
|
2026/09/03 16:12:39
|
100 ms
|
103 ms
|
write_operation {
input {
common {
plan_id: 6
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-10.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
49321596-692d-4bbb-9aa6-ac0b49869aae
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_49321596-692d-4bbb-9aa6-ac0b49869aae
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[248]
|
[254]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
75 ms
|
78 ms
|
write_operation {
input {
common {
plan_id: 170
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
49bf026a-31fb-4e3b-a127-7952a0a67f41
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_49bf026a-31fb-4e3b-a127-7952a0a67f41
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[164][165]
|
[170][171]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:12:39
|
2026/09/03 16:13:06
|
2026/09/03 16:13:06
|
27 seconds 620 ms
|
27 seconds 625 ms
|
write_operation {
input {
common {
plan_id: 8
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
4a0e6fa4-11f8-45c5-9138-12142a2c4d93
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_4a0e6fa4-11f8-45c5-9138-12142a2c4d93
|
|
|
|
|
[251]
|
[257]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:09
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
93 ms
|
97 ms
|
write_operation {
input {
common {
plan_id: 176
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-09.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
4aa96cc8-eac5-47dd-a15d-7d6ec84e4631
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_4aa96cc8-eac5-47dd-a15d-7d6ec84e4631
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[198]
|
[204]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
93 ms
|
98 ms
|
write_operation {
input {
common {
plan_id: 72
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
4fae2a35-37ef-4ee1-a1b5-ff7757bf2cff
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_4fae2a35-37ef-4ee1-a1b5-ff7757bf2cff
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[213]
|
[219]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
98 ms
|
102 ms
|
write_operation {
input {
common {
plan_id: 102
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-04.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
50badd40-73b6-413c-bf7f-913fa37c99a1
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_50badd40-73b6-413c-bf7f-913fa37c99a1
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[232]
|
[238]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
110 ms
|
161 ms
|
write_operation {
input {
common {
plan_id: 138
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-10.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
52a96a13-df5e-4a95-af5a-163a6252c863
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_52a96a13-df5e-4a95-af5a-163a6252c863
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[231]
|
[237]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
88 ms
|
91 ms
|
write_operation {
input {
common {
plan_id: 136
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
57ae2e76-b04c-45fc-bed4-787052fe1f0b
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_57ae2e76-b04c-45fc-bed4-787052fe1f0b
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[223]
|
[229]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
156 ms
|
160 ms
|
write_operation {
input {
common {
plan_id: 120
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-11.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
581465f6-3066-4298-b9b9-5ec1adcbc575
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_581465f6-3066-4298-b9b9-5ec1adcbc575
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[179]
|
[185]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
250 ms
|
255 ms
|
write_operation {
input {
common {
plan_id: 34
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
58782597-3c4f-4429-82f1-d7be3d74c34e
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_58782597-3c4f-4429-82f1-d7be3d74c34e
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[217]
|
[223]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:39
|
2026/09/03 16:13:39
|
2026/09/03 16:13:39
|
89 ms
|
93 ms
|
write_operation {
input {
common {
plan_id: 110
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
5fac896b-bc68-4d5c-bcc2-abfc406e750f
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_5fac896b-bc68-4d5c-bcc2-abfc406e750f
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[245]
|
[251]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
230 ms
|
236 ms
|
write_operation {
input {
common {
plan_id: 164
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-02.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
68456b2c-116e-405c-9494-0c45481a30f9
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_68456b2c-116e-405c-9494-0c45481a30f9
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[204]
|
[210]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
143 ms
|
148 ms
|
write_operation {
input {
common {
plan_id: 84
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-03.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
6bc41597-674a-4fee-9ab3-55277c54abe3
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_6bc41597-674a-4fee-9ab3-55277c54abe3
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[158][159][160]
|
[164][165][166]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:12:07
|
2026/09/03 16:12:38
|
2026/09/03 16:12:38
|
31 seconds 764 ms
|
31 seconds 769 ms
|
write_operation {
input {
common {
plan_id: 0
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-10.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
72c531e6-b91e-430c-a90f-107332e492cb
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_72c531e6-b91e-430c-a90f-107332e492cb
|
|
|
|
|
[199]
|
[205]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
2026/09/03 16:13:37
|
113 ms
|
117 ms
|
write_operation {
input {
common {
plan_id: 74
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-01.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
7338ba55-d94b-45e1-998b-8100ac497b10
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_7338ba55-d94b-45e1-998b-8100ac497b10
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[184]
|
[190]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
143 ms
|
150 ms
|
write_operation {
input {
common {
plan_id: 44
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
7a5df2e0-b57e-470b-94e5-badc2b56aa44
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_7a5df2e0-b57e-470b-94e5-badc2b56aa44
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[252]
|
[258]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
88 ms
|
93 ms
|
write_operation {
input {
common {
plan_id: 178
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-04.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
7a710e17-02ae-4a00-94f4-26950f47b03f
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_7a710e17-02ae-4a00-94f4-26950f47b03f
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[162]
|
[168]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:12:39
|
2026/09/03 16:12:39
|
2026/09/03 16:12:39
|
83 ms
|
87 ms
|
write_operation {
input {
common {
plan_id: 4
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-10.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
7eb4ab4c-6e7e-4cda-971f-5b6294638da5
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_7eb4ab4c-6e7e-4cda-971f-5b6294638da5
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[216]
|
[222]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:39
|
2026/09/03 16:13:39
|
2026/09/03 16:13:39
|
119 ms
|
128 ms
|
write_operation {
input {
common {
plan_id: 108
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
821d0b0d-41ea-48a1-b8fa-a3c7cf284237
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_821d0b0d-41ea-48a1-b8fa-a3c7cf284237
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[257]
|
[263]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
99 ms
|
118 ms
|
write_operation {
input {
common {
plan_id: 188
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
8251ea1b-2aa4-4e33-9d45-236b851177ae
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_8251ea1b-2aa4-4e33-9d45-236b851177ae
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[188]
|
[194]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
116 ms
|
123 ms
|
write_operation {
input {
common {
plan_id: 52
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
82e7c2d4-fb99-40f7-b2c3-701d13a1e7b3
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_82e7c2d4-fb99-40f7-b2c3-701d13a1e7b3
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[180]
|
[186]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
76 ms
|
80 ms
|
write_operation {
input {
common {
plan_id: 36
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
858e3b2b-185a-49ad-8d3f-1c664581c01e
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_858e3b2b-185a-49ad-8d3f-1c664581c01e
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[234]
|
[240]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:07
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
98 ms
|
101 ms
|
write_operation {
input {
common {
plan_id: 142
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
86f21334-439e-47d0-a875-fd51c6187d5c
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_86f21334-439e-47d0-a875-fd51c6187d5c
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[237]
|
[243]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
109 ms
|
114 ms
|
write_operation {
input {
common {
plan_id: 148
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-03.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
88dfc4ad-bfe4-4e22-a465-84d5a27f6a7d
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_88dfc4ad-bfe4-4e22-a465-84d5a27f6a7d
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[191]
|
[197]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
110 ms
|
115 ms
|
write_operation {
input {
common {
plan_id: 58
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-04.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
89c328c3-476f-4d77-ae7a-980f234b092b
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_89c328c3-476f-4d77-ae7a-980f234b092b
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[177]
|
[183]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
112 ms
|
117 ms
|
write_operation {
input {
common {
plan_id: 30
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-07.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
8bcbc6a0-14e8-4a77-9380-2b94ecc9e95b
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_8bcbc6a0-14e8-4a77-9380-2b94ecc9e95b
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[233]
|
[239]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
209 ms
|
212 ms
|
write_operation {
input {
common {
plan_id: 140
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-01.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
8d6ace22-b5ab-4313-a159-d5f59b0f7b07
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_8d6ace22-b5ab-4313-a159-d5f59b0f7b07
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[166]
|
[172]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:06
|
2026/09/03 16:13:07
|
2026/09/03 16:13:07
|
176 ms
|
183 ms
|
write_operation {
input {
common {
plan_id: 10
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
95614555-3777-4dfe-826f-dd28d83359e8
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_95614555-3777-4dfe-826f-dd28d83359e8
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[172]
|
[178]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
140 ms
|
151 ms
|
write_operation {
input {
common {
plan_id: 20
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-02.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
993baa0e-ddcb-4699-b50a-50e6c4c21fde
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_993baa0e-ddcb-4699-b50a-50e6c4c21fde
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[247]
|
[253]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
87 ms
|
91 ms
|
write_operation {
input {
common {
plan_id: 168
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-02.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
9d08ab95-a13f-4b32-af42-1a9b9a8db562
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_9d08ab95-a13f-4b32-af42-1a9b9a8db562
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'passenger_count' and 'passenger_count'
|
|
|
[236]
|
[242]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
121 ms
|
127 ms
|
write_operation {
input {
common {
plan_id: 146
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-11.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
a096a92e-2cf9-4190-bd0c-07e3dea8fab7
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_a096a92e-2cf9-4190-bd0c-07e3dea8fab7
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[254]
|
[260]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
87 ms
|
90 ms
|
write_operation {
input {
common {
plan_id: 182
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-09.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
a2301a18-0e5c-483a-b9c5-4256bfc59c7f
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_a2301a18-0e5c-483a-b9c5-4256bfc59c7f
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[228]
|
[234]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
75 ms
|
80 ms
|
write_operation {
input {
common {
plan_id: 130
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-03.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
a4ec91a3-2e3c-4564-828a-1af8b644b1ba
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_a4ec91a3-2e3c-4564-828a-1af8b644b1ba
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[189]
|
[195]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
102 ms
|
105 ms
|
write_operation {
input {
common {
plan_id: 54
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
a71ebdcc-81b1-4f0f-8ba8-05eb45a4bb71
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_a71ebdcc-81b1-4f0f-8ba8-05eb45a4bb71
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[182]
|
[188]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
139 ms
|
143 ms
|
write_operation {
input {
common {
plan_id: 40
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-07.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
a9b43a10-8ee3-4cdc-b188-b967035b5a2e
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_a9b43a10-8ee3-4cdc-b188-b967035b5a2e
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[176]
|
[182]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
2026/09/03 16:13:34
|
97 ms
|
100 ms
|
write_operation {
input {
common {
plan_id: 28
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
b041af04-e936-43f4-ae5f-b9c73544bfdd
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_b041af04-e936-43f4-ae5f-b9c73544bfdd
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[239]
|
[245]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
81 ms
|
84 ms
|
write_operation {
input {
common {
plan_id: 152
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-11.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
b452b4b9-2c1d-401d-856d-69a829ad0aaa
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_b452b4b9-2c1d-401d-856d-69a829ad0aaa
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[238]
|
[244]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
88 ms
|
91 ms
|
write_operation {
input {
common {
plan_id: 150
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-10.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
b5b19e34-af5f-43f4-8dae-a7a494428649
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_b5b19e34-af5f-43f4-8dae-a7a494428649
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[203]
|
[209]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
142 ms
|
148 ms
|
write_operation {
input {
common {
plan_id: 82
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-10.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
b69fd70b-bb76-4198-91e3-0f7223de0693
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_b69fd70b-bb76-4198-91e3-0f7223de0693
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[201]
|
[207]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
140 ms
|
144 ms
|
write_operation {
input {
common {
plan_id: 78
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-03.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
be22460e-a3d6-4c1d-98e9-0f8c38a34805
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_be22460e-a3d6-4c1d-98e9-0f8c38a34805
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[253]
|
[259]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
84 ms
|
88 ms
|
write_operation {
input {
common {
plan_id: 180
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
bf000549-c1aa-4eb9-924e-d87ff0d8d33f
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_bf000549-c1aa-4eb9-924e-d87ff0d8d33f
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[229]
|
[235]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
86 ms
|
89 ms
|
write_operation {
input {
common {
plan_id: 132
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-11.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
c647fcb0-13e5-47e7-a2c6-e0920af818ee
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_c647fcb0-13e5-47e7-a2c6-e0920af818ee
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[246]
|
[252]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
107 ms
|
113 ms
|
write_operation {
input {
common {
plan_id: 166
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-03.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
c9f6afc9-1002-4e61-a0c6-75a6e13466d1
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_c9f6afc9-1002-4e61-a0c6-75a6e13466d1
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[167]
|
[173]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:07
|
2026/09/03 16:13:07
|
2026/09/03 16:13:07
|
304 ms
|
310 ms
|
write_operation {
input {
common {
plan_id: 12
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-07.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
ccc38014-821f-42c5-b1f5-65d5700fbadf
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_ccc38014-821f-42c5-b1f5-65d5700fbadf
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[171]
|
[177]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
156 ms
|
159 ms
|
write_operation {
input {
common {
plan_id: 18
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-01.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
ce551e70-2c11-40d6-bee9-c12c33e80063
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_ce551e70-2c11-40d6-bee9-c12c33e80063
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[168]
|
[174]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:07
|
2026/09/03 16:13:07
|
2026/09/03 16:13:07
|
499 ms
|
503 ms
|
write_operation {
input {
common {
plan_id: 14
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-03.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
d2e00546-45c8-4a1f-890f-63c0315ed1cc
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_d2e00546-45c8-4a1f-890f-63c0315ed1cc
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[226]
|
[232]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:06
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
88 ms
|
92 ms
|
write_operation {
input {
common {
plan_id: 126
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-02.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
d30ab6dc-8509-490d-9183-1f392faaeab6
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_d30ab6dc-8509-490d-9183-1f392faaeab6
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[194]
|
[200]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
100 ms
|
104 ms
|
write_operation {
input {
common {
plan_id: 64
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-09.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
d3b188fb-9986-4e7b-a6e2-4c40c4e69317
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_d3b188fb-9986-4e7b-a6e2-4c40c4e69317
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[202]
|
[208]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
124 ms
|
133 ms
|
write_operation {
input {
common {
plan_id: 80
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-04.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
d3e4ca73-208a-48c3-bbdf-5076a7d99caa
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_d3e4ca73-208a-48c3-bbdf-5076a7d99caa
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[205]
|
[211]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
159 ms
|
170 ms
|
write_operation {
input {
common {
plan_id: 86
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-09.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
d7ae4bfa-6ae1-4130-a89f-a2a3062ce403
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_d7ae4bfa-6ae1-4130-a89f-a2a3062ce403
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[255]
|
[261]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
77 ms
|
81 ms
|
write_operation {
input {
common {
plan_id: 184
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-04.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
d90ee069-8ce2-49c2-8ac7-5b3b13c0da5d
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_d90ee069-8ce2-49c2-8ac7-5b3b13c0da5d
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[221]
|
[227]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
112 ms
|
118 ms
|
write_operation {
input {
common {
plan_id: 116
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-11.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
dc8a1ecd-9dcf-4368-b9f1-539b5cd080ee
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_dc8a1ecd-9dcf-4368-b9f1-539b5cd080ee
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[249]
|
[255]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
98 ms
|
104 ms
|
write_operation {
input {
common {
plan_id: 172
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
dd6f855b-4b32-4fb2-8d11-f133ff6422b0
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_dd6f855b-4b32-4fb2-8d11-f133ff6422b0
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[208]
|
[214]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
97 ms
|
104 ms
|
write_operation {
input {
common {
plan_id: 92
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-01.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
e22ae106-151d-445f-92a5-48264e354cc9
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_e22ae106-151d-445f-92a5-48264e354cc9
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[222]
|
[228]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
151 ms
|
165 ms
|
write_operation {
input {
common {
plan_id: 118
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-10.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
e95632ff-dda1-4fe8-90ec-9a939137983b
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_e95632ff-dda1-4fe8-90ec-9a939137983b
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[235]
|
[241]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
113 ms
|
119 ms
|
write_operation {
input {
common {
plan_id: 144
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-01.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
e9d4efb1-75f2-497f-9d0f-d75b7833153b
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_e9d4efb1-75f2-497f-9d0f-d75b7833153b
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[178]
|
[184]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
164 ms
|
169 ms
|
write_operation {
input {
common {
plan_id: 32
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-11.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
ed081ed8-4ab9-4e4d-a9d8-f25faeb6c9ed
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_ed081ed8-4ab9-4e4d-a9d8-f25faeb6c9ed
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[161]
|
[167]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:12:38
|
2026/09/03 16:12:39
|
2026/09/03 16:12:39
|
101 ms
|
104 ms
|
write_operation {
input {
common {
plan_id: 2
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-11.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
f1a22686-1c05-430b-bd21-db360e87205f
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_f1a22686-1c05-430b-bd21-db360e87205f
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[224]
|
[230]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
97 ms
|
100 ms
|
write_operation {
input {
common {
plan_id: 122
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-09.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
f1c57fb8-ee34-47e9-9a07-720cfd1c5e8e
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_f1c57fb8-ee34-47e9-9a07-720cfd1c5e8e
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[258]
|
[264]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
193 ms
|
196 ms
|
write_operation {
input {
common {
plan_id: 190
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-02.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
f51353ee-b876-4820-8de9-8e595ad43048
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_f51353ee-b876-4820-8de9-8e595ad43048
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[174]
|
[180]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
93 ms
|
97 ms
|
write_operation {
input {
common {
plan_id: 24
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-02.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
f8ae7c29-a8ea-4205-a0a3-f1dd0000d6a4
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_f8ae7c29-a8ea-4205-a0a3-f1dd0000d6a4
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[244]
|
[250]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
169 ms
|
179 ms
|
write_operation {
input {
common {
plan_id: 162
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-01.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
f988164d-2a30-4c6c-812b-4ad65094c526
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_f988164d-2a30-4c6c-812b-4ad65094c526
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'passenger_count' and 'passenger_count'
|
|
|
[70]
|
[70]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:55
|
2026/09/03 15:55:55
|
2026/09/03 15:55:55
|
103 ms
|
108 ms
|
sql_command {
input {
common {
plan_id: 71
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-10.parquet`\n "
}
}
}
|
CLOSED
|
07292414-b820-462e-b17d-cdeb39d1bfac
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_07292414-b820-462e-b17d-cdeb39d1bfac
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3067L as bigint) AS VendorID#3088L, try_cast(tpep_pickup_datetime#3068 as timestamp) AS tpep_pickup_datetime#3089, try_cast(tpep_dropoff_datetime#3069 as timestamp) AS tpep_dropoff_datetime#3090, try_cast(passenger_count#3070 as bigint) AS passenger_count#3091L, try_cast(trip_distance#3071 as double) AS trip_distance#3092, try_cast(RatecodeID#3072 as bigint) AS RatecodeID#3093L, try_cast(store_and_fwd_flag#3073 as string) AS store_and_fwd_flag#3094, try_cast(PULocationID#3074L as bigint) AS PULocationID#3095L, try_cast(DOLocationID#3075L as bigint) AS DOLocationID#3096L, try_cast(payment_type#3076L as bigint) AS payment_type#3097L, try_cast(fare_amount#3077 as double) AS fare_amount#3098, try_cast(extra#3078 as double) AS extra#3099, try_cast(mta_tax#3079 as double) AS mta_tax#3100, try_cast(tip_amount#3080 as double) AS tip_amount#3101, try_cast(tolls_amount#3081 as double) AS tolls_amount#3102, try_cast(improvement_surcharge#3082 as double) AS improvement_surcharge#3103, try_cast(total_amount#3083 as double) AS total_amount#3104, try_cast(congestion_surcharge#3084 as double) AS congestion_surcharge#3105, try_cast(airport_fee#3085 as double) AS airport_fee#3106, current_timestamp() AS current_timestamp()#3087, coalesce(year(cast(tpep_pickup_datetime#3068 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3107, coalesce(month(cast(tpep_pickup_datetime#3068 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3108, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[122]
|
[130][131]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:06
|
|
|
8 weeks 3 days 2 hours
|
8 weeks 3 days 2 hours
|
write_operation {
input {
common {
plan_id: 141
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-02.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
RUNNING
|
0736c6a9-8500-4c2b-a735-a0089e70f6bc
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_0736c6a9-8500-4c2b-a735-a0089e70f6bc
|
|
|
|
|
[81]
|
[81]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:56
|
2026/09/03 15:55:56
|
2026/09/03 15:55:56
|
102 ms
|
108 ms
|
sql_command {
input {
common {
plan_id: 82
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-05.parquet`\n "
}
}
}
|
CLOSED
|
08247687-c8b5-4a36-bc65-7818df1a7d86
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_08247687-c8b5-4a36-bc65-7818df1a7d86
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3552L as bigint) AS VendorID#3573L, try_cast(tpep_pickup_datetime#3553 as timestamp) AS tpep_pickup_datetime#3574, try_cast(tpep_dropoff_datetime#3554 as timestamp) AS tpep_dropoff_datetime#3575, try_cast(passenger_count#3555 as bigint) AS passenger_count#3576L, try_cast(trip_distance#3556 as double) AS trip_distance#3577, try_cast(RatecodeID#3557 as bigint) AS RatecodeID#3578L, try_cast(store_and_fwd_flag#3558 as string) AS store_and_fwd_flag#3579, try_cast(PULocationID#3559L as bigint) AS PULocationID#3580L, try_cast(DOLocationID#3560L as bigint) AS DOLocationID#3581L, try_cast(payment_type#3561L as bigint) AS payment_type#3582L, try_cast(fare_amount#3562 as double) AS fare_amount#3583, try_cast(extra#3563 as double) AS extra#3584, try_cast(mta_tax#3564 as double) AS mta_tax#3585, try_cast(tip_amount#3565 as double) AS tip_amount#3586, try_cast(tolls_amount#3566 as double) AS tolls_amount#3587, try_cast(improvement_surcharge#3567 as double) AS improvement_surcharge#3588, try_cast(total_amount#3568 as double) AS total_amount#3589, try_cast(congestion_surcharge#3569 as double) AS congestion_surcharge#3590, try_cast(airport_fee#3570 as double) AS airport_fee#3591, current_timestamp() AS current_timestamp()#3572, coalesce(year(cast(tpep_pickup_datetime#3553 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3592, coalesce(month(cast(tpep_pickup_datetime#3553 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3593, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[155][156][157]
|
[161][162][163]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:26
|
2026/09/03 16:10:39
|
2026/09/03 16:17:54
|
12 seconds 670 ms
|
7 minutes 27 seconds
|
write_operation {
input {
common {
plan_id: 189
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CLOSED
|
0d73ec80-efc3-457f-b7a0-ecf03792de7e
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_0d73ec80-efc3-457f-b7a0-ecf03792de7e
|
|
|