|
|
|
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:59:02
|
2026/09/03 15:59:02
|
2026/09/03 15:59:02
|
11 ms
|
24 ms
|
common {
plan_id: 107
}
limit {
input {
common {
plan_id: 106
}
local_relation {
data: "\377\377\377\377@\000\000\000[truncated(size=152)]"
}
}
limit: 1000
}
|
CLOSED
|
21f9f48f-f295-462d-88d7-9aec4bdd3753
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_21f9f48f-f295-462d-88d7-9aec4bdd3753
|
|
|
|
|
|
[104]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:00:30
|
2026/09/03 16:00:30
|
2026/09/03 16:00:30
|
11 ms
|
14 ms
|
sql_command {
input {
common {
plan_id: 110
}
sql {
query: "describe details spark_catalog.yellow_taxi.trips"
}
}
}
|
CLOSED
|
729b26e6-c29d-4972-9c76-e77b5ade0a53
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_729b26e6-c29d-4972-9c76-e77b5ade0a53
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `details` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 9;
'DescribeColumn 'spark_catalog.yellow_taxi.trips, false, [info_name#4363, info_value#4364]
+- 'UnresolvedTableOrView [details], DESCRIBE TABLE, true
|
|
|
|
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:58:44
|
2026/09/03 15:58:44
|
2026/09/03 15:58:44
|
15 ms
|
22 ms
|
sql_command {
input {
common {
plan_id: 104
}
sql {
query: "\ncreate table if not exists spark_catalog.yellow_taxi.trips(\n VendorID long,\n tpep_pickup_datetime timestamp,\n tpep_dropoff_datetime timestamp,\n passenger_count long,\n trip_distance double,\n RatecodeID long,\n store_and_fwd_flag string,\n PULocationID long,\n DOLocationID long,\n payment_type long,\n fare_amount double,\n extra double,\n mta_tax double,\n tip_amount double,\n tolls_amount double,\n improvement_surcharge double,\n total_amount double,\n congestion_surcharge double,\n airport_fee double,\n load_timestamp timestamp,\n year int,\n month int,\n day int\n)\nusing delta\npartitioned by (year,month)\nlocation \'/home/jovyan/spark-warehouse/yellow_taxi.db/trips\n"
}
}
}
|
CLOSED
|
bf5ad8eb-5982-421f-a63e-3b0aece0293b
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_bf5ad8eb-5982-421f-a63e-3b0aece0293b
|
|
PARSE_SYNTAX_ERROR
+details
INTERNAL:
[PARSE_SYNTAX_ERROR] Syntax error at or near '''. SQLSTATE: 42601 (line 29, pos 9)
== SQL ==
create table if not exists spark_catalog.yellow_taxi.trips(
VendorID long,
tpep_pickup_datetime timestamp,
tpep_dropoff_datetime timestamp,
passenger_count long,
trip_distance double,
RatecodeID long,
store_and_fwd_flag string,
PULocationID long,
DOLocationID long,
payment_type long,
fare_amount double,
extra double,
mta_tax double,
tip_amount double,
tolls_amount double,
improvement_surcharge double,
total_amount double,
congestion_surcharge double,
airport_fee double,
load_timestamp timestamp,
year int,
month int,
day int
)
using delta
partitioned by (year,month)
location '/home/jovyan/spark-warehouse/yellow_taxi.db/trips
---------^^^
|
|
|
|
[102]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:00:08
|
2026/09/03 16:00:08
|
2026/09/03 16:00:08
|
19 ms
|
25 ms
|
sql_command {
input {
common {
plan_id: 108
}
sql {
query: "describe details spark_catalog.yellow_taxi.trips"
}
}
}
|
CLOSED
|
e917eaaf-733e-4196-be3a-45f85a4d60ba
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_e917eaaf-733e-4196-be3a-45f85a4d60ba
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `details` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 9;
'DescribeColumn 'spark_catalog.yellow_taxi.trips, false, [info_name#4336, info_value#4337]
+- 'UnresolvedTableOrView [details], DESCRIBE TABLE, true
|
|
|
|
[103]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:00:22
|
2026/09/03 16:00:22
|
2026/09/03 16:00:22
|
25 ms
|
30 ms
|
sql_command {
input {
common {
plan_id: 109
}
sql {
query: "\ncreate table spark_catalog.yellow_taxi.trips(\n VendorID long,\n tpep_pickup_datetime timestamp,\n tpep_dropoff_datetime timestamp,\n passenger_count long,\n trip_distance double,\n RatecodeID long,\n store_and_fwd_flag string,\n PULocationID long,\n DOLocationID long,\n payment_type long,\n fare_amount double,\n extra double,\n mta_tax double,\n tip_amount double,\n tolls_amount double,\n improvement_surcharge double,\n total_amount double,\n congestion_surcharge double,\n airport_fee double,\n load_timestamp timestamp,\n year int,\n month int,\n day int\n)\nusing delta\npartitioned by (year,month)\nlocation \'/home/jovyan/spark-warehouse/yellow_taxi.db/trips\'\n"
}
}
}
|
CLOSED
|
269f6447-c3d5-480a-826a-f446bdec940f
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_269f6447-c3d5-480a-826a-f446bdec940f
|
|
TABLE_OR_VIEW_ALREADY_EXISTS
+details
INTERNAL: [TABLE_OR_VIEW_ALREADY_EXISTS] Cannot create table or view `yellow_taxi`.`trips` because it already exists.
Choose a different name, drop or replace the existing object, or add the IF NOT EXISTS clause to tolerate pre-existing objects. SQLSTATE: 42P07
|
|
|
|
[100]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:56:49
|
2026/09/03 15:56:49
|
2026/09/03 15:56:49
|
27 ms
|
35 ms
|
sql_command {
input {
common {
plan_id: 103
}
sql {
query: "drop table spark_catalog.yellow_taxi.trips"
}
}
}
|
CLOSED
|
bdb9942f-a378-4f47-8872-09aee772ba34
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_bdb9942f-a378-4f47-8872-09aee772ba34
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `spark_catalog`.`yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01
|
|
|
[248]
|
[254]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
75 ms
|
78 ms
|
write_operation {
input {
common {
plan_id: 170
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
49bf026a-31fb-4e3b-a127-7952a0a67f41
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_49bf026a-31fb-4e3b-a127-7952a0a67f41
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[228]
|
[234]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
75 ms
|
80 ms
|
write_operation {
input {
common {
plan_id: 130
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-03.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
a4ec91a3-2e3c-4564-828a-1af8b644b1ba
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_a4ec91a3-2e3c-4564-828a-1af8b644b1ba
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[180]
|
[186]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
76 ms
|
80 ms
|
write_operation {
input {
common {
plan_id: 36
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
858e3b2b-185a-49ad-8d3f-1c664581c01e
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_858e3b2b-185a-49ad-8d3f-1c664581c01e
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[255]
|
[261]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
77 ms
|
81 ms
|
write_operation {
input {
common {
plan_id: 184
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-04.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
d90ee069-8ce2-49c2-8ac7-5b3b13c0da5d
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_d90ee069-8ce2-49c2-8ac7-5b3b13c0da5d
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[242]
|
[248]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
79 ms
|
82 ms
|
write_operation {
input {
common {
plan_id: 158
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-11.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
04c2c4d4-48cd-41f1-911a-34f31a74a4dd
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_04c2c4d4-48cd-41f1-911a-34f31a74a4dd
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[190]
|
[196]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
79 ms
|
82 ms
|
write_operation {
input {
common {
plan_id: 56
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
3a1b4a58-3756-4684-a865-1c18f8afce94
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_3a1b4a58-3756-4684-a865-1c18f8afce94
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[186]
|
[192]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
81 ms
|
86 ms
|
write_operation {
input {
common {
plan_id: 48
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
038056f6-65ae-4842-828e-83013290f29d
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_038056f6-65ae-4842-828e-83013290f29d
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'passenger_count' and 'passenger_count'
|
|
|
[239]
|
[245]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
81 ms
|
84 ms
|
write_operation {
input {
common {
plan_id: 152
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-11.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
b452b4b9-2c1d-401d-856d-69a829ad0aaa
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_b452b4b9-2c1d-401d-856d-69a829ad0aaa
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[241]
|
[247]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
82 ms
|
85 ms
|
write_operation {
input {
common {
plan_id: 156
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-07.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
2d69db4e-6414-4518-b9f7-a4f2a54548ab
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_2d69db4e-6414-4518-b9f7-a4f2a54548ab
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[185]
|
[191]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
83 ms
|
88 ms
|
write_operation {
input {
common {
plan_id: 46
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
3a121330-0197-4c65-82ea-66cd99480a68
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_3a121330-0197-4c65-82ea-66cd99480a68
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[162]
|
[168]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:12:39
|
2026/09/03 16:12:39
|
2026/09/03 16:12:39
|
83 ms
|
87 ms
|
write_operation {
input {
common {
plan_id: 4
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-10.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
7eb4ab4c-6e7e-4cda-971f-5b6294638da5
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_7eb4ab4c-6e7e-4cda-971f-5b6294638da5
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[253]
|
[259]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
84 ms
|
88 ms
|
write_operation {
input {
common {
plan_id: 180
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
bf000549-c1aa-4eb9-924e-d87ff0d8d33f
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_bf000549-c1aa-4eb9-924e-d87ff0d8d33f
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[243]
|
[249]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
85 ms
|
88 ms
|
write_operation {
input {
common {
plan_id: 160
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
425e9ce3-23ef-448b-be7a-da8b0336f908
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_425e9ce3-23ef-448b-be7a-da8b0336f908
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[225]
|
[231]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
86 ms
|
90 ms
|
write_operation {
input {
common {
plan_id: 124
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-10.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
0ee9b11f-8e57-463c-86a9-723e65186262
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_0ee9b11f-8e57-463c-86a9-723e65186262
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[218]
|
[224]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:39
|
2026/09/03 16:13:39
|
2026/09/03 16:13:39
|
86 ms
|
91 ms
|
write_operation {
input {
common {
plan_id: 112
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
1b892df8-440a-4e11-8f2a-a8e9f5a0d381
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_1b892df8-440a-4e11-8f2a-a8e9f5a0d381
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[229]
|
[235]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
86 ms
|
89 ms
|
write_operation {
input {
common {
plan_id: 132
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-11.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
c647fcb0-13e5-47e7-a2c6-e0920af818ee
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_c647fcb0-13e5-47e7-a2c6-e0920af818ee
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[196]
|
[202]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
87 ms
|
94 ms
|
write_operation {
input {
common {
plan_id: 68
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
1a81ea2a-525d-4e8d-9b12-741b7aa7c3f9
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_1a81ea2a-525d-4e8d-9b12-741b7aa7c3f9
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[187]
|
[193]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
87 ms
|
93 ms
|
write_operation {
input {
common {
plan_id: 50
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-04.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
453243e3-12f6-43ff-b295-28633ad30539
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_453243e3-12f6-43ff-b295-28633ad30539
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[247]
|
[253]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
87 ms
|
91 ms
|
write_operation {
input {
common {
plan_id: 168
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-02.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
9d08ab95-a13f-4b32-af42-1a9b9a8db562
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_9d08ab95-a13f-4b32-af42-1a9b9a8db562
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'passenger_count' and 'passenger_count'
|
|
|
[254]
|
[260]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
87 ms
|
90 ms
|
write_operation {
input {
common {
plan_id: 182
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-09.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
a2301a18-0e5c-483a-b9c5-4256bfc59c7f
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_a2301a18-0e5c-483a-b9c5-4256bfc59c7f
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[231]
|
[237]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
88 ms
|
91 ms
|
write_operation {
input {
common {
plan_id: 136
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
57ae2e76-b04c-45fc-bed4-787052fe1f0b
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_57ae2e76-b04c-45fc-bed4-787052fe1f0b
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[252]
|
[258]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
88 ms
|
93 ms
|
write_operation {
input {
common {
plan_id: 178
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-04.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
7a710e17-02ae-4a00-94f4-26950f47b03f
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_7a710e17-02ae-4a00-94f4-26950f47b03f
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[238]
|
[244]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
88 ms
|
91 ms
|
write_operation {
input {
common {
plan_id: 150
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-10.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
b5b19e34-af5f-43f4-8dae-a7a494428649
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_b5b19e34-af5f-43f4-8dae-a7a494428649
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[226]
|
[232]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:06
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
88 ms
|
92 ms
|
write_operation {
input {
common {
plan_id: 126
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-02.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
d30ab6dc-8509-490d-9183-1f392faaeab6
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_d30ab6dc-8509-490d-9183-1f392faaeab6
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[217]
|
[223]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:39
|
2026/09/03 16:13:39
|
2026/09/03 16:13:39
|
89 ms
|
93 ms
|
write_operation {
input {
common {
plan_id: 110
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
5fac896b-bc68-4d5c-bcc2-abfc406e750f
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_5fac896b-bc68-4d5c-bcc2-abfc406e750f
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[89]
|
[89]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:57
|
2026/09/03 15:55:58
|
2026/09/03 15:55:58
|
91 ms
|
97 ms
|
sql_command {
input {
common {
plan_id: 90
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-09.parquet`\n "
}
}
}
|
CLOSED
|
f5e9f4cb-cb12-435a-9e06-23a296f9126b
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_f5e9f4cb-cb12-435a-9e06-23a296f9126b
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3904L as bigint) AS VendorID#3925L, try_cast(tpep_pickup_datetime#3905 as timestamp) AS tpep_pickup_datetime#3926, try_cast(tpep_dropoff_datetime#3906 as timestamp) AS tpep_dropoff_datetime#3927, try_cast(passenger_count#3907 as bigint) AS passenger_count#3928L, try_cast(trip_distance#3908 as double) AS trip_distance#3929, try_cast(RatecodeID#3909 as bigint) AS RatecodeID#3930L, try_cast(store_and_fwd_flag#3910 as string) AS store_and_fwd_flag#3931, try_cast(PULocationID#3911L as bigint) AS PULocationID#3932L, try_cast(DOLocationID#3912L as bigint) AS DOLocationID#3933L, try_cast(payment_type#3913L as bigint) AS payment_type#3934L, try_cast(fare_amount#3914 as double) AS fare_amount#3935, try_cast(extra#3915 as double) AS extra#3936, try_cast(mta_tax#3916 as double) AS mta_tax#3937, try_cast(tip_amount#3917 as double) AS tip_amount#3938, try_cast(tolls_amount#3918 as double) AS tolls_amount#3939, try_cast(improvement_surcharge#3919 as double) AS improvement_surcharge#3940, try_cast(total_amount#3920 as double) AS total_amount#3941, try_cast(congestion_surcharge#3921 as double) AS congestion_surcharge#3942, try_cast(airport_fee#3922 as double) AS airport_fee#3943, current_timestamp() AS current_timestamp()#3924, coalesce(year(cast(tpep_pickup_datetime#3905 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3944, coalesce(month(cast(tpep_pickup_datetime#3905 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3945, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[214]
|
[220]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
2026/09/03 16:13:39
|
91 ms
|
98 ms
|
write_operation {
input {
common {
plan_id: 104
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
0cc0f910-7a61-4785-aaf6-54f3223e0dab
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_0cc0f910-7a61-4785-aaf6-54f3223e0dab
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[240]
|
[246]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
91 ms
|
94 ms
|
write_operation {
input {
common {
plan_id: 154
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-07.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
2cd0c3f1-7085-448d-af0e-31d580b5d84f
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_2cd0c3f1-7085-448d-af0e-31d580b5d84f
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[200]
|
[206]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
92 ms
|
98 ms
|
write_operation {
input {
common {
plan_id: 76
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-01.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
31428a5a-88f9-453a-b45c-c7644ee6ce7e
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_31428a5a-88f9-453a-b45c-c7644ee6ce7e
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[173]
|
[179]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
93 ms
|
98 ms
|
write_operation {
input {
common {
plan_id: 22
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
070b7388-8ec5-4d50-b7fa-defa8b2fc8d7
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_070b7388-8ec5-4d50-b7fa-defa8b2fc8d7
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[251]
|
[257]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:09
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
93 ms
|
97 ms
|
write_operation {
input {
common {
plan_id: 176
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-09.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
4aa96cc8-eac5-47dd-a15d-7d6ec84e4631
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_4aa96cc8-eac5-47dd-a15d-7d6ec84e4631
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[198]
|
[204]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
93 ms
|
98 ms
|
write_operation {
input {
common {
plan_id: 72
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
4fae2a35-37ef-4ee1-a1b5-ff7757bf2cff
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_4fae2a35-37ef-4ee1-a1b5-ff7757bf2cff
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[174]
|
[180]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
93 ms
|
97 ms
|
write_operation {
input {
common {
plan_id: 24
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-02.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
f8ae7c29-a8ea-4205-a0a3-f1dd0000d6a4
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_f8ae7c29-a8ea-4205-a0a3-f1dd0000d6a4
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[69]
|
[69]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:55
|
2026/09/03 15:55:55
|
2026/09/03 15:55:55
|
94 ms
|
100 ms
|
sql_command {
input {
common {
plan_id: 70
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-06.parquet`\n "
}
}
}
|
CLOSED
|
ae4dab30-e630-45cc-8cee-de828d2d1ea5
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_ae4dab30-e630-45cc-8cee-de828d2d1ea5
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3023L as bigint) AS VendorID#3044L, try_cast(tpep_pickup_datetime#3024 as timestamp) AS tpep_pickup_datetime#3045, try_cast(tpep_dropoff_datetime#3025 as timestamp) AS tpep_dropoff_datetime#3046, try_cast(passenger_count#3026 as bigint) AS passenger_count#3047L, try_cast(trip_distance#3027 as double) AS trip_distance#3048, try_cast(RatecodeID#3028 as bigint) AS RatecodeID#3049L, try_cast(store_and_fwd_flag#3029 as string) AS store_and_fwd_flag#3050, try_cast(PULocationID#3030L as bigint) AS PULocationID#3051L, try_cast(DOLocationID#3031L as bigint) AS DOLocationID#3052L, try_cast(payment_type#3032L as bigint) AS payment_type#3053L, try_cast(fare_amount#3033 as double) AS fare_amount#3054, try_cast(extra#3034 as double) AS extra#3055, try_cast(mta_tax#3035 as double) AS mta_tax#3056, try_cast(tip_amount#3036 as double) AS tip_amount#3057, try_cast(tolls_amount#3037 as double) AS tolls_amount#3058, try_cast(improvement_surcharge#3038 as double) AS improvement_surcharge#3059, try_cast(total_amount#3039 as double) AS total_amount#3060, try_cast(congestion_surcharge#3040 as double) AS congestion_surcharge#3061, try_cast(airport_fee#3041 as double) AS airport_fee#3062, current_timestamp() AS current_timestamp()#3043, coalesce(year(cast(tpep_pickup_datetime#3024 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3063, coalesce(month(cast(tpep_pickup_datetime#3024 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3064, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[211]
|
[217]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
95 ms
|
99 ms
|
write_operation {
input {
common {
plan_id: 98
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
15644fc0-5219-436f-978a-262f2851cff8
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_15644fc0-5219-436f-978a-262f2851cff8
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[65]
|
[65]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:54
|
2026/09/03 15:55:54
|
2026/09/03 15:55:54
|
96 ms
|
99 ms
|
sql_command {
input {
common {
plan_id: 66
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-01.parquet`\n "
}
}
}
|
CLOSED
|
ceac7c9c-a170-42e5-b966-6eaa281f3f27
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_ceac7c9c-a170-42e5-b966-6eaa281f3f27
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#2847L as bigint) AS VendorID#2868L, try_cast(tpep_pickup_datetime#2848 as timestamp) AS tpep_pickup_datetime#2869, try_cast(tpep_dropoff_datetime#2849 as timestamp) AS tpep_dropoff_datetime#2870, try_cast(passenger_count#2850 as bigint) AS passenger_count#2871L, try_cast(trip_distance#2851 as double) AS trip_distance#2872, try_cast(RatecodeID#2852 as bigint) AS RatecodeID#2873L, try_cast(store_and_fwd_flag#2853 as string) AS store_and_fwd_flag#2874, try_cast(PULocationID#2854L as bigint) AS PULocationID#2875L, try_cast(DOLocationID#2855L as bigint) AS DOLocationID#2876L, try_cast(payment_type#2856L as bigint) AS payment_type#2877L, try_cast(fare_amount#2857 as double) AS fare_amount#2878, try_cast(extra#2858 as double) AS extra#2879, try_cast(mta_tax#2859 as double) AS mta_tax#2880, try_cast(tip_amount#2860 as double) AS tip_amount#2881, try_cast(tolls_amount#2861 as double) AS tolls_amount#2882, try_cast(improvement_surcharge#2862 as double) AS improvement_surcharge#2883, try_cast(total_amount#2863 as double) AS total_amount#2884, try_cast(congestion_surcharge#2864 as double) AS congestion_surcharge#2885, try_cast(airport_fee#2865 as double) AS airport_fee#2886, current_timestamp() AS current_timestamp()#2867, coalesce(year(cast(tpep_pickup_datetime#2848 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#2887, coalesce(month(cast(tpep_pickup_datetime#2848 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#2888, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[212]
|
[218]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
97 ms
|
100 ms
|
write_operation {
input {
common {
plan_id: 100
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-03.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
193c6253-94c4-4d1c-b0b0-d48c996d246e
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_193c6253-94c4-4d1c-b0b0-d48c996d246e
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[176]
|
[182]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
2026/09/03 16:13:34
|
97 ms
|
100 ms
|
write_operation {
input {
common {
plan_id: 28
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
b041af04-e936-43f4-ae5f-b9c73544bfdd
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_b041af04-e936-43f4-ae5f-b9c73544bfdd
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[208]
|
[214]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
97 ms
|
104 ms
|
write_operation {
input {
common {
plan_id: 92
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-01.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
e22ae106-151d-445f-92a5-48264e354cc9
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_e22ae106-151d-445f-92a5-48264e354cc9
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[224]
|
[230]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
97 ms
|
100 ms
|
write_operation {
input {
common {
plan_id: 122
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-09.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
f1c57fb8-ee34-47e9-9a07-720cfd1c5e8e
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_f1c57fb8-ee34-47e9-9a07-720cfd1c5e8e
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[213]
|
[219]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
98 ms
|
102 ms
|
write_operation {
input {
common {
plan_id: 102
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-04.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
50badd40-73b6-413c-bf7f-913fa37c99a1
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_50badd40-73b6-413c-bf7f-913fa37c99a1
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[234]
|
[240]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:07
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
98 ms
|
101 ms
|
write_operation {
input {
common {
plan_id: 142
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
86f21334-439e-47d0-a875-fd51c6187d5c
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_86f21334-439e-47d0-a875-fd51c6187d5c
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[249]
|
[255]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
98 ms
|
104 ms
|
write_operation {
input {
common {
plan_id: 172
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
dd6f855b-4b32-4fb2-8d11-f133ff6422b0
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_dd6f855b-4b32-4fb2-8d11-f133ff6422b0
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[197]
|
[203]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
99 ms
|
105 ms
|
write_operation {
input {
common {
plan_id: 70
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
43d0c529-3218-4ae3-a405-0e00f1eb6600
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_43d0c529-3218-4ae3-a405-0e00f1eb6600
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[257]
|
[263]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
2026/09/03 16:14:10
|
99 ms
|
118 ms
|
write_operation {
input {
common {
plan_id: 188
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-05.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
8251ea1b-2aa4-4e33-9d45-236b851177ae
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_8251ea1b-2aa4-4e33-9d45-236b851177ae
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[163]
|
[169]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:12:39
|
2026/09/03 16:12:39
|
2026/09/03 16:12:39
|
100 ms
|
103 ms
|
write_operation {
input {
common {
plan_id: 6
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-10.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
49321596-692d-4bbb-9aa6-ac0b49869aae
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_49321596-692d-4bbb-9aa6-ac0b49869aae
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[194]
|
[200]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
100 ms
|
104 ms
|
write_operation {
input {
common {
plan_id: 64
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-09.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
d3b188fb-9986-4e7b-a6e2-4c40c4e69317
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_d3b188fb-9986-4e7b-a6e2-4c40c4e69317
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[90]
|
[90]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:58
|
2026/09/03 15:55:58
|
2026/09/03 15:55:58
|
101 ms
|
106 ms
|
sql_command {
input {
common {
plan_id: 91
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-04.parquet`\n "
}
}
}
|
CLOSED
|
50a40a08-5b4b-450b-ac7d-e821cbbf4955
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_50a40a08-5b4b-450b-ac7d-e821cbbf4955
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3948L as bigint) AS VendorID#3969L, try_cast(tpep_pickup_datetime#3949 as timestamp) AS tpep_pickup_datetime#3970, try_cast(tpep_dropoff_datetime#3950 as timestamp) AS tpep_dropoff_datetime#3971, try_cast(passenger_count#3951 as bigint) AS passenger_count#3972L, try_cast(trip_distance#3952 as double) AS trip_distance#3973, try_cast(RatecodeID#3953 as bigint) AS RatecodeID#3974L, try_cast(store_and_fwd_flag#3954 as string) AS store_and_fwd_flag#3975, try_cast(PULocationID#3955L as bigint) AS PULocationID#3976L, try_cast(DOLocationID#3956L as bigint) AS DOLocationID#3977L, try_cast(payment_type#3957L as bigint) AS payment_type#3978L, try_cast(fare_amount#3958 as double) AS fare_amount#3979, try_cast(extra#3959 as double) AS extra#3980, try_cast(mta_tax#3960 as double) AS mta_tax#3981, try_cast(tip_amount#3961 as double) AS tip_amount#3982, try_cast(tolls_amount#3962 as double) AS tolls_amount#3983, try_cast(improvement_surcharge#3963 as double) AS improvement_surcharge#3984, try_cast(total_amount#3964 as double) AS total_amount#3985, try_cast(congestion_surcharge#3965 as double) AS congestion_surcharge#3986, try_cast(airport_fee#3966 as double) AS airport_fee#3987, current_timestamp() AS current_timestamp()#3968, coalesce(year(cast(tpep_pickup_datetime#3949 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3988, coalesce(month(cast(tpep_pickup_datetime#3949 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3989, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[161]
|
[167]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:12:38
|
2026/09/03 16:12:39
|
2026/09/03 16:12:39
|
101 ms
|
104 ms
|
write_operation {
input {
common {
plan_id: 2
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-11.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
f1a22686-1c05-430b-bd21-db360e87205f
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_f1a22686-1c05-430b-bd21-db360e87205f
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[81]
|
[81]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:56
|
2026/09/03 15:55:56
|
2026/09/03 15:55:56
|
102 ms
|
108 ms
|
sql_command {
input {
common {
plan_id: 82
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-05.parquet`\n "
}
}
}
|
CLOSED
|
08247687-c8b5-4a36-bc65-7818df1a7d86
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_08247687-c8b5-4a36-bc65-7818df1a7d86
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3552L as bigint) AS VendorID#3573L, try_cast(tpep_pickup_datetime#3553 as timestamp) AS tpep_pickup_datetime#3574, try_cast(tpep_dropoff_datetime#3554 as timestamp) AS tpep_dropoff_datetime#3575, try_cast(passenger_count#3555 as bigint) AS passenger_count#3576L, try_cast(trip_distance#3556 as double) AS trip_distance#3577, try_cast(RatecodeID#3557 as bigint) AS RatecodeID#3578L, try_cast(store_and_fwd_flag#3558 as string) AS store_and_fwd_flag#3579, try_cast(PULocationID#3559L as bigint) AS PULocationID#3580L, try_cast(DOLocationID#3560L as bigint) AS DOLocationID#3581L, try_cast(payment_type#3561L as bigint) AS payment_type#3582L, try_cast(fare_amount#3562 as double) AS fare_amount#3583, try_cast(extra#3563 as double) AS extra#3584, try_cast(mta_tax#3564 as double) AS mta_tax#3585, try_cast(tip_amount#3565 as double) AS tip_amount#3586, try_cast(tolls_amount#3566 as double) AS tolls_amount#3587, try_cast(improvement_surcharge#3567 as double) AS improvement_surcharge#3588, try_cast(total_amount#3568 as double) AS total_amount#3589, try_cast(congestion_surcharge#3569 as double) AS congestion_surcharge#3590, try_cast(airport_fee#3570 as double) AS airport_fee#3591, current_timestamp() AS current_timestamp()#3572, coalesce(year(cast(tpep_pickup_datetime#3553 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3592, coalesce(month(cast(tpep_pickup_datetime#3553 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3593, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[227]
|
[233]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
102 ms
|
106 ms
|
write_operation {
input {
common {
plan_id: 128
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-01.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
0f6391dc-4186-438a-a313-ba7c32f0b0e6
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_0f6391dc-4186-438a-a313-ba7c32f0b0e6
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[189]
|
[195]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
102 ms
|
105 ms
|
write_operation {
input {
common {
plan_id: 54
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-08.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
a71ebdcc-81b1-4f0f-8ba8-05eb45a4bb71
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_a71ebdcc-81b1-4f0f-8ba8-05eb45a4bb71
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[70]
|
[70]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:55
|
2026/09/03 15:55:55
|
2026/09/03 15:55:55
|
103 ms
|
108 ms
|
sql_command {
input {
common {
plan_id: 71
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-10.parquet`\n "
}
}
}
|
CLOSED
|
07292414-b820-462e-b17d-cdeb39d1bfac
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_07292414-b820-462e-b17d-cdeb39d1bfac
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3067L as bigint) AS VendorID#3088L, try_cast(tpep_pickup_datetime#3068 as timestamp) AS tpep_pickup_datetime#3089, try_cast(tpep_dropoff_datetime#3069 as timestamp) AS tpep_dropoff_datetime#3090, try_cast(passenger_count#3070 as bigint) AS passenger_count#3091L, try_cast(trip_distance#3071 as double) AS trip_distance#3092, try_cast(RatecodeID#3072 as bigint) AS RatecodeID#3093L, try_cast(store_and_fwd_flag#3073 as string) AS store_and_fwd_flag#3094, try_cast(PULocationID#3074L as bigint) AS PULocationID#3095L, try_cast(DOLocationID#3075L as bigint) AS DOLocationID#3096L, try_cast(payment_type#3076L as bigint) AS payment_type#3097L, try_cast(fare_amount#3077 as double) AS fare_amount#3098, try_cast(extra#3078 as double) AS extra#3099, try_cast(mta_tax#3079 as double) AS mta_tax#3100, try_cast(tip_amount#3080 as double) AS tip_amount#3101, try_cast(tolls_amount#3081 as double) AS tolls_amount#3102, try_cast(improvement_surcharge#3082 as double) AS improvement_surcharge#3103, try_cast(total_amount#3083 as double) AS total_amount#3104, try_cast(congestion_surcharge#3084 as double) AS congestion_surcharge#3105, try_cast(airport_fee#3085 as double) AS airport_fee#3106, current_timestamp() AS current_timestamp()#3087, coalesce(year(cast(tpep_pickup_datetime#3068 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3107, coalesce(month(cast(tpep_pickup_datetime#3068 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3108, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[74]
|
[74]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:55
|
2026/09/03 15:55:55
|
2026/09/03 15:55:55
|
104 ms
|
113 ms
|
sql_command {
input {
common {
plan_id: 75
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-11.parquet`\n "
}
}
}
|
CLOSED
|
9e2a560d-f642-487d-af0a-834f2ed8f66b
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_9e2a560d-f642-487d-af0a-834f2ed8f66b
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3243 as bigint) AS VendorID#3264L, try_cast(tpep_pickup_datetime#3244 as timestamp) AS tpep_pickup_datetime#3265, try_cast(tpep_dropoff_datetime#3245 as timestamp) AS tpep_dropoff_datetime#3266, try_cast(passenger_count#3246L as bigint) AS passenger_count#3267L, try_cast(trip_distance#3247 as double) AS trip_distance#3268, try_cast(RatecodeID#3248L as bigint) AS RatecodeID#3269L, try_cast(store_and_fwd_flag#3249 as string) AS store_and_fwd_flag#3270, try_cast(PULocationID#3250 as bigint) AS PULocationID#3271L, try_cast(DOLocationID#3251 as bigint) AS DOLocationID#3272L, try_cast(payment_type#3252L as bigint) AS payment_type#3273L, try_cast(fare_amount#3253 as double) AS fare_amount#3274, try_cast(extra#3254 as double) AS extra#3275, try_cast(mta_tax#3255 as double) AS mta_tax#3276, try_cast(tip_amount#3256 as double) AS tip_amount#3277, try_cast(tolls_amount#3257 as double) AS tolls_amount#3278, try_cast(improvement_surcharge#3258 as double) AS improvement_surcharge#3279, try_cast(total_amount#3259 as double) AS total_amount#3280, try_cast(congestion_surcharge#3260 as double) AS congestion_surcharge#3281, try_cast(airport_fee#3261 as double) AS airport_fee#3282, current_timestamp() AS current_timestamp()#3263, coalesce(year(cast(tpep_pickup_datetime#3244 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3283, coalesce(month(cast(tpep_pickup_datetime#3244 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3284, coalesce(day(cast(tpep_pickup_date...
|
|
|
[92]
|
[92]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:58
|
2026/09/03 15:55:58
|
2026/09/03 15:55:58
|
104 ms
|
109 ms
|
sql_command {
input {
common {
plan_id: 93
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-09.parquet`\n "
}
}
}
|
CLOSED
|
c5a2e2e9-125a-471d-bfb7-037f249e2cc1
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_c5a2e2e9-125a-471d-bfb7-037f249e2cc1
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#4037L as bigint) AS VendorID#4058L, try_cast(tpep_pickup_datetime#4038 as timestamp) AS tpep_pickup_datetime#4059, try_cast(tpep_dropoff_datetime#4039 as timestamp) AS tpep_dropoff_datetime#4060, try_cast(passenger_count#4040 as bigint) AS passenger_count#4061L, try_cast(trip_distance#4041 as double) AS trip_distance#4062, try_cast(RatecodeID#4042 as bigint) AS RatecodeID#4063L, try_cast(store_and_fwd_flag#4043 as string) AS store_and_fwd_flag#4064, try_cast(PULocationID#4044L as bigint) AS PULocationID#4065L, try_cast(DOLocationID#4045L as bigint) AS DOLocationID#4066L, try_cast(payment_type#4046L as bigint) AS payment_type#4067L, try_cast(fare_amount#4047 as double) AS fare_amount#4068, try_cast(extra#4048 as double) AS extra#4069, try_cast(mta_tax#4049 as double) AS mta_tax#4070, try_cast(tip_amount#4050 as double) AS tip_amount#4071, try_cast(tolls_amount#4051 as double) AS tolls_amount#4072, try_cast(improvement_surcharge#4052 as double) AS improvement_surcharge#4073, try_cast(total_amount#4053 as double) AS total_amount#4074, try_cast(congestion_surcharge#4054 as double) AS congestion_surcharge#4075, try_cast(airport_fee#4055 as double) AS airport_fee#4076, current_timestamp() AS current_timestamp()#4057, coalesce(year(cast(tpep_pickup_datetime#4038 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#4077, coalesce(month(cast(tpep_pickup_datetime#4038 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#4078, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[93]
|
[93]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:58
|
2026/09/03 15:55:58
|
2026/09/03 15:55:58
|
105 ms
|
125 ms
|
sql_command {
input {
common {
plan_id: 94
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-04.parquet`\n "
}
}
}
|
CLOSED
|
4f6df773-6d3f-428e-9bfc-d72f445d2d47
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_4f6df773-6d3f-428e-9bfc-d72f445d2d47
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#4081L as bigint) AS VendorID#4102L, try_cast(tpep_pickup_datetime#4082 as timestamp) AS tpep_pickup_datetime#4103, try_cast(tpep_dropoff_datetime#4083 as timestamp) AS tpep_dropoff_datetime#4104, try_cast(passenger_count#4084 as bigint) AS passenger_count#4105L, try_cast(trip_distance#4085 as double) AS trip_distance#4106, try_cast(RatecodeID#4086 as bigint) AS RatecodeID#4107L, try_cast(store_and_fwd_flag#4087 as string) AS store_and_fwd_flag#4108, try_cast(PULocationID#4088L as bigint) AS PULocationID#4109L, try_cast(DOLocationID#4089L as bigint) AS DOLocationID#4110L, try_cast(payment_type#4090L as bigint) AS payment_type#4111L, try_cast(fare_amount#4091 as double) AS fare_amount#4112, try_cast(extra#4092 as double) AS extra#4113, try_cast(mta_tax#4093 as double) AS mta_tax#4114, try_cast(tip_amount#4094 as double) AS tip_amount#4115, try_cast(tolls_amount#4095 as double) AS tolls_amount#4116, try_cast(improvement_surcharge#4096 as double) AS improvement_surcharge#4117, try_cast(total_amount#4097 as double) AS total_amount#4118, try_cast(congestion_surcharge#4098 as double) AS congestion_surcharge#4119, try_cast(airport_fee#4099 as double) AS airport_fee#4120, current_timestamp() AS current_timestamp()#4101, coalesce(year(cast(tpep_pickup_datetime#4082 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#4121, coalesce(month(cast(tpep_pickup_datetime#4082 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#4122, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[183]
|
[189]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:34
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
105 ms
|
111 ms
|
write_operation {
input {
common {
plan_id: 42
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
320e82c2-abf9-41e7-831c-5b4e4e6469f9
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_320e82c2-abf9-41e7-831c-5b4e4e6469f9
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[82]
|
[82]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:56
|
2026/09/03 15:55:57
|
2026/09/03 15:55:57
|
106 ms
|
111 ms
|
sql_command {
input {
common {
plan_id: 83
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-01.parquet`\n "
}
}
}
|
CLOSED
|
dc5e456b-00f0-45ab-85cd-f56432823392
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_dc5e456b-00f0-45ab-85cd-f56432823392
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3596L as bigint) AS VendorID#3617L, try_cast(tpep_pickup_datetime#3597 as timestamp) AS tpep_pickup_datetime#3618, try_cast(tpep_dropoff_datetime#3598 as timestamp) AS tpep_dropoff_datetime#3619, try_cast(passenger_count#3599L as bigint) AS passenger_count#3620L, try_cast(trip_distance#3600 as double) AS trip_distance#3621, try_cast(RatecodeID#3601L as bigint) AS RatecodeID#3622L, try_cast(store_and_fwd_flag#3602 as string) AS store_and_fwd_flag#3623, try_cast(PULocationID#3603L as bigint) AS PULocationID#3624L, try_cast(DOLocationID#3604L as bigint) AS DOLocationID#3625L, try_cast(payment_type#3605L as bigint) AS payment_type#3626L, try_cast(fare_amount#3606 as double) AS fare_amount#3627, try_cast(extra#3607 as double) AS extra#3628, try_cast(mta_tax#3608 as double) AS mta_tax#3629, try_cast(tip_amount#3609 as double) AS tip_amount#3630, try_cast(tolls_amount#3610 as double) AS tolls_amount#3631, try_cast(improvement_surcharge#3611 as double) AS improvement_surcharge#3632, try_cast(total_amount#3612 as double) AS total_amount#3633, try_cast(congestion_surcharge#3613 as double) AS congestion_surcharge#3634, try_cast(airport_fee#3614 as double) AS airport_fee#3635, current_timestamp() AS current_timestamp()#3616, coalesce(year(cast(tpep_pickup_datetime#3597 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3636, coalesce(month(cast(tpep_pickup_datetime#3597 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3637, coalesce(day(cast(tpep_pickup_d...
|
|
|
[215]
|
[221]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:39
|
2026/09/03 16:13:39
|
2026/09/03 16:13:39
|
106 ms
|
110 ms
|
write_operation {
input {
common {
plan_id: 106
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-04.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
29dbea9c-4898-40e0-8360-9ebb3400571e
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_29dbea9c-4898-40e0-8360-9ebb3400571e
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'passenger_count' and 'passenger_count'
|
|
|
[73]
|
[73]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:55
|
2026/09/03 15:55:55
|
2026/09/03 15:55:55
|
107 ms
|
113 ms
|
sql_command {
input {
common {
plan_id: 74
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-01.parquet`\n "
}
}
}
|
CLOSED
|
8dd5b860-1ca6-43e0-a824-37fcfcef8b48
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_8dd5b860-1ca6-43e0-a824-37fcfcef8b48
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3199 as bigint) AS VendorID#3220L, try_cast(tpep_pickup_datetime#3200 as timestamp) AS tpep_pickup_datetime#3221, try_cast(tpep_dropoff_datetime#3201 as timestamp) AS tpep_dropoff_datetime#3222, try_cast(passenger_count#3202L as bigint) AS passenger_count#3223L, try_cast(trip_distance#3203 as double) AS trip_distance#3224, try_cast(RatecodeID#3204L as bigint) AS RatecodeID#3225L, try_cast(store_and_fwd_flag#3205 as string) AS store_and_fwd_flag#3226, try_cast(PULocationID#3206 as bigint) AS PULocationID#3227L, try_cast(DOLocationID#3207 as bigint) AS DOLocationID#3228L, try_cast(payment_type#3208L as bigint) AS payment_type#3229L, try_cast(fare_amount#3209 as double) AS fare_amount#3230, try_cast(extra#3210 as double) AS extra#3231, try_cast(mta_tax#3211 as double) AS mta_tax#3232, try_cast(tip_amount#3212 as double) AS tip_amount#3233, try_cast(tolls_amount#3213 as double) AS tolls_amount#3234, try_cast(improvement_surcharge#3214 as double) AS improvement_surcharge#3235, try_cast(total_amount#3215 as double) AS total_amount#3236, try_cast(congestion_surcharge#3216 as double) AS congestion_surcharge#3237, try_cast(airport_fee#3217 as double) AS airport_fee#3238, current_timestamp() AS current_timestamp()#3219, coalesce(year(cast(tpep_pickup_datetime#3200 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3239, coalesce(month(cast(tpep_pickup_datetime#3200 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3240, coalesce(day(cast(tpep_pickup_date...
|
|
|
[195]
|
[201]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
107 ms
|
111 ms
|
write_operation {
input {
common {
plan_id: 66
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-12.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
1842e87c-6411-4588-a7cc-b4a675f5ab04
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_1842e87c-6411-4588-a7cc-b4a675f5ab04
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[246]
|
[252]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
2026/09/03 16:14:09
|
107 ms
|
113 ms
|
write_operation {
input {
common {
plan_id: 166
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-03.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
c9f6afc9-1002-4e61-a0c6-75a6e13466d1
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_c9f6afc9-1002-4e61-a0c6-75a6e13466d1
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
|
[101]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:59:02
|
2026/09/03 15:59:02
|
2026/09/03 15:59:02
|
109 ms
|
118 ms
|
sql_command {
input {
common {
plan_id: 105
}
sql {
query: "\ncreate table if not exists spark_catalog.yellow_taxi.trips(\n VendorID long,\n tpep_pickup_datetime timestamp,\n tpep_dropoff_datetime timestamp,\n passenger_count long,\n trip_distance double,\n RatecodeID long,\n store_and_fwd_flag string,\n PULocationID long,\n DOLocationID long,\n payment_type long,\n fare_amount double,\n extra double,\n mta_tax double,\n tip_amount double,\n tolls_amount double,\n improvement_surcharge double,\n total_amount double,\n congestion_surcharge double,\n airport_fee double,\n load_timestamp timestamp,\n year int,\n month int,\n day int\n)\nusing delta\npartitioned by (year,month)\nlocation \'/home/jovyan/spark-warehouse/yellow_taxi.db/trips\'\n"
}
}
}
|
CLOSED
|
4e3e69c5-e8a8-4c00-a30a-7571f47e1126
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_4e3e69c5-e8a8-4c00-a30a-7571f47e1126
|
|
|
|
|
[71]
|
[71]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:55
|
2026/09/03 15:55:55
|
2026/09/03 15:55:55
|
109 ms
|
115 ms
|
sql_command {
input {
common {
plan_id: 72
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-01.parquet`\n "
}
}
}
|
CLOSED
|
d2864b9d-8834-461b-bba7-f064f608aac9
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_d2864b9d-8834-461b-bba7-f064f608aac9
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3111L as bigint) AS VendorID#3132L, try_cast(tpep_pickup_datetime#3112 as timestamp) AS tpep_pickup_datetime#3133, try_cast(tpep_dropoff_datetime#3113 as timestamp) AS tpep_dropoff_datetime#3134, try_cast(passenger_count#3114 as bigint) AS passenger_count#3135L, try_cast(trip_distance#3115 as double) AS trip_distance#3136, try_cast(RatecodeID#3116 as bigint) AS RatecodeID#3137L, try_cast(store_and_fwd_flag#3117 as string) AS store_and_fwd_flag#3138, try_cast(PULocationID#3118L as bigint) AS PULocationID#3139L, try_cast(DOLocationID#3119L as bigint) AS DOLocationID#3140L, try_cast(payment_type#3120L as bigint) AS payment_type#3141L, try_cast(fare_amount#3121 as double) AS fare_amount#3142, try_cast(extra#3122 as double) AS extra#3143, try_cast(mta_tax#3123 as double) AS mta_tax#3144, try_cast(tip_amount#3124 as double) AS tip_amount#3145, try_cast(tolls_amount#3125 as double) AS tolls_amount#3146, try_cast(improvement_surcharge#3126 as double) AS improvement_surcharge#3147, try_cast(total_amount#3127 as double) AS total_amount#3148, try_cast(congestion_surcharge#3128 as double) AS congestion_surcharge#3149, try_cast(airport_fee#3129 as double) AS airport_fee#3150, current_timestamp() AS current_timestamp()#3131, coalesce(year(cast(tpep_pickup_datetime#3112 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3151, coalesce(month(cast(tpep_pickup_datetime#3112 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3152, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[237]
|
[243]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
109 ms
|
114 ms
|
write_operation {
input {
common {
plan_id: 148
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-03.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
88dfc4ad-bfe4-4e22-a465-84d5a27f6a7d
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_88dfc4ad-bfe4-4e22-a465-84d5a27f6a7d
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[175]
|
[181]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
2026/09/03 16:13:33
|
110 ms
|
118 ms
|
write_operation {
input {
common {
plan_id: 26
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2023-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
43c883ae-8661-4fae-9072-fff947596643
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_43c883ae-8661-4fae-9072-fff947596643
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[232]
|
[238]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
2026/09/03 16:14:07
|
110 ms
|
161 ms
|
write_operation {
input {
common {
plan_id: 138
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-10.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
52a96a13-df5e-4a95-af5a-163a6252c863
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_52a96a13-df5e-4a95-af5a-163a6252c863
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[191]
|
[197]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
110 ms
|
115 ms
|
write_operation {
input {
common {
plan_id: 58
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-04.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
89c328c3-476f-4d77-ae7a-980f234b092b
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_89c328c3-476f-4d77-ae7a-980f234b092b
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[83]
|
[83]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:57
|
2026/09/03 15:55:57
|
2026/09/03 15:55:57
|
111 ms
|
117 ms
|
sql_command {
input {
common {
plan_id: 84
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-02.parquet`\n "
}
}
}
|
CLOSED
|
52a9dfb4-11be-42bb-8c91-43f0c9bbb04b
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_52a9dfb4-11be-42bb-8c91-43f0c9bbb04b
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3640L as bigint) AS VendorID#3661L, try_cast(tpep_pickup_datetime#3641 as timestamp) AS tpep_pickup_datetime#3662, try_cast(tpep_dropoff_datetime#3642 as timestamp) AS tpep_dropoff_datetime#3663, try_cast(passenger_count#3643 as bigint) AS passenger_count#3664L, try_cast(trip_distance#3644 as double) AS trip_distance#3665, try_cast(RatecodeID#3645 as bigint) AS RatecodeID#3666L, try_cast(store_and_fwd_flag#3646 as string) AS store_and_fwd_flag#3667, try_cast(PULocationID#3647L as bigint) AS PULocationID#3668L, try_cast(DOLocationID#3648L as bigint) AS DOLocationID#3669L, try_cast(payment_type#3649L as bigint) AS payment_type#3670L, try_cast(fare_amount#3650 as double) AS fare_amount#3671, try_cast(extra#3651 as double) AS extra#3672, try_cast(mta_tax#3652 as double) AS mta_tax#3673, try_cast(tip_amount#3653 as double) AS tip_amount#3674, try_cast(tolls_amount#3654 as double) AS tolls_amount#3675, try_cast(improvement_surcharge#3655 as double) AS improvement_surcharge#3676, try_cast(total_amount#3656 as double) AS total_amount#3677, try_cast(congestion_surcharge#3657 as double) AS congestion_surcharge#3678, try_cast(airport_fee#3658 as double) AS airport_fee#3679, current_timestamp() AS current_timestamp()#3660, coalesce(year(cast(tpep_pickup_datetime#3641 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3680, coalesce(month(cast(tpep_pickup_datetime#3641 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3681, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[91]
|
[91]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:58
|
2026/09/03 15:55:58
|
2026/09/03 15:55:58
|
111 ms
|
121 ms
|
sql_command {
input {
common {
plan_id: 92
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-08.parquet`\n "
}
}
}
|
CLOSED
|
f3cc306b-69c3-42b0-950b-af75f9752b53
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_f3cc306b-69c3-42b0-950b-af75f9752b53
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3992 as bigint) AS VendorID#4014L, try_cast(tpep_pickup_datetime#3993 as timestamp) AS tpep_pickup_datetime#4015, try_cast(tpep_dropoff_datetime#3994 as timestamp) AS tpep_dropoff_datetime#4016, try_cast(passenger_count#3995L as bigint) AS passenger_count#4017L, try_cast(trip_distance#3996 as double) AS trip_distance#4018, try_cast(RatecodeID#3997L as bigint) AS RatecodeID#4019L, try_cast(store_and_fwd_flag#3998 as string) AS store_and_fwd_flag#4020, try_cast(PULocationID#3999 as bigint) AS PULocationID#4021L, try_cast(DOLocationID#4000 as bigint) AS DOLocationID#4022L, try_cast(payment_type#4001L as bigint) AS payment_type#4023L, try_cast(fare_amount#4002 as double) AS fare_amount#4024, try_cast(extra#4003 as double) AS extra#4025, try_cast(mta_tax#4004 as double) AS mta_tax#4026, try_cast(tip_amount#4005 as double) AS tip_amount#4027, try_cast(tolls_amount#4006 as double) AS tolls_amount#4028, try_cast(improvement_surcharge#4007 as double) AS improvement_surcharge#4029, try_cast(total_amount#4008 as double) AS total_amount#4030, try_cast(congestion_surcharge#4009 as double) AS congestion_surcharge#4031, try_cast(airport_fee#4010 as double) AS airport_fee#4032, current_timestamp() AS current_timestamp()#4013, coalesce(year(cast(tpep_pickup_datetime#3993 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#4033, coalesce(month(cast(tpep_pickup_datetime#3993 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#4034, coalesce(day(cast(tpep_pickup_date...
|
|
|
[79]
|
[79]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:56
|
2026/09/03 15:55:56
|
2026/09/03 15:55:56
|
112 ms
|
118 ms
|
sql_command {
input {
common {
plan_id: 80
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-07.parquet`\n "
}
}
}
|
CLOSED
|
562cc198-1aba-4541-973d-7c1250cf4270
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_562cc198-1aba-4541-973d-7c1250cf4270
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3464 as bigint) AS VendorID#3485L, try_cast(tpep_pickup_datetime#3465 as timestamp) AS tpep_pickup_datetime#3486, try_cast(tpep_dropoff_datetime#3466 as timestamp) AS tpep_dropoff_datetime#3487, try_cast(passenger_count#3467L as bigint) AS passenger_count#3488L, try_cast(trip_distance#3468 as double) AS trip_distance#3489, try_cast(RatecodeID#3469L as bigint) AS RatecodeID#3490L, try_cast(store_and_fwd_flag#3470 as string) AS store_and_fwd_flag#3491, try_cast(PULocationID#3471 as bigint) AS PULocationID#3492L, try_cast(DOLocationID#3472 as bigint) AS DOLocationID#3493L, try_cast(payment_type#3473L as bigint) AS payment_type#3494L, try_cast(fare_amount#3474 as double) AS fare_amount#3495, try_cast(extra#3475 as double) AS extra#3496, try_cast(mta_tax#3476 as double) AS mta_tax#3497, try_cast(tip_amount#3477 as double) AS tip_amount#3498, try_cast(tolls_amount#3478 as double) AS tolls_amount#3499, try_cast(improvement_surcharge#3479 as double) AS improvement_surcharge#3500, try_cast(total_amount#3480 as double) AS total_amount#3501, try_cast(congestion_surcharge#3481 as double) AS congestion_surcharge#3502, try_cast(airport_fee#3482 as double) AS airport_fee#3503, current_timestamp() AS current_timestamp()#3484, coalesce(year(cast(tpep_pickup_datetime#3465 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3504, coalesce(month(cast(tpep_pickup_datetime#3465 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3505, coalesce(day(cast(tpep_pickup_date...
|
|
|
[84]
|
[84]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:57
|
2026/09/03 15:55:57
|
2026/09/03 15:55:57
|
112 ms
|
124 ms
|
sql_command {
input {
common {
plan_id: 85
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-03.parquet`\n "
}
}
}
|
CLOSED
|
abff517f-610b-4643-a428-3d861cc6fa42
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_abff517f-610b-4643-a428-3d861cc6fa42
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3684L as bigint) AS VendorID#3705L, try_cast(tpep_pickup_datetime#3685 as timestamp) AS tpep_pickup_datetime#3706, try_cast(tpep_dropoff_datetime#3686 as timestamp) AS tpep_dropoff_datetime#3707, try_cast(passenger_count#3687 as bigint) AS passenger_count#3708L, try_cast(trip_distance#3688 as double) AS trip_distance#3709, try_cast(RatecodeID#3689 as bigint) AS RatecodeID#3710L, try_cast(store_and_fwd_flag#3690 as string) AS store_and_fwd_flag#3711, try_cast(PULocationID#3691L as bigint) AS PULocationID#3712L, try_cast(DOLocationID#3692L as bigint) AS DOLocationID#3713L, try_cast(payment_type#3693L as bigint) AS payment_type#3714L, try_cast(fare_amount#3694 as double) AS fare_amount#3715, try_cast(extra#3695 as double) AS extra#3716, try_cast(mta_tax#3696 as double) AS mta_tax#3717, try_cast(tip_amount#3697 as double) AS tip_amount#3718, try_cast(tolls_amount#3698 as double) AS tolls_amount#3719, try_cast(improvement_surcharge#3699 as double) AS improvement_surcharge#3720, try_cast(total_amount#3700 as double) AS total_amount#3721, try_cast(congestion_surcharge#3701 as double) AS congestion_surcharge#3722, try_cast(airport_fee#3702 as double) AS airport_fee#3723, current_timestamp() AS current_timestamp()#3704, coalesce(year(cast(tpep_pickup_datetime#3685 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3724, coalesce(month(cast(tpep_pickup_datetime#3685 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3725, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[177]
|
[183]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
112 ms
|
117 ms
|
write_operation {
input {
common {
plan_id: 30
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-07.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
8bcbc6a0-14e8-4a77-9380-2b94ecc9e95b
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_8bcbc6a0-14e8-4a77-9380-2b94ecc9e95b
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[221]
|
[227]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
2026/09/03 16:14:06
|
112 ms
|
118 ms
|
write_operation {
input {
common {
plan_id: 116
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-11.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
dc8a1ecd-9dcf-4368-b9f1-539b5cd080ee
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_dc8a1ecd-9dcf-4368-b9f1-539b5cd080ee
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[62]
|
[62]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:54
|
2026/09/03 15:55:54
|
2026/09/03 15:55:54
|
113 ms
|
122 ms
|
sql_command {
input {
common {
plan_id: 63
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-09.parquet`\n "
}
}
}
|
CLOSED
|
2df7501f-cf45-447d-8e5b-5abb0eb1e7dc
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_2df7501f-cf45-447d-8e5b-5abb0eb1e7dc
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#2715 as bigint) AS VendorID#2736L, try_cast(tpep_pickup_datetime#2716 as timestamp) AS tpep_pickup_datetime#2737, try_cast(tpep_dropoff_datetime#2717 as timestamp) AS tpep_dropoff_datetime#2738, try_cast(passenger_count#2718L as bigint) AS passenger_count#2739L, try_cast(trip_distance#2719 as double) AS trip_distance#2740, try_cast(RatecodeID#2720L as bigint) AS RatecodeID#2741L, try_cast(store_and_fwd_flag#2721 as string) AS store_and_fwd_flag#2742, try_cast(PULocationID#2722 as bigint) AS PULocationID#2743L, try_cast(DOLocationID#2723 as bigint) AS DOLocationID#2744L, try_cast(payment_type#2724L as bigint) AS payment_type#2745L, try_cast(fare_amount#2725 as double) AS fare_amount#2746, try_cast(extra#2726 as double) AS extra#2747, try_cast(mta_tax#2727 as double) AS mta_tax#2748, try_cast(tip_amount#2728 as double) AS tip_amount#2749, try_cast(tolls_amount#2729 as double) AS tolls_amount#2750, try_cast(improvement_surcharge#2730 as double) AS improvement_surcharge#2751, try_cast(total_amount#2731 as double) AS total_amount#2752, try_cast(congestion_surcharge#2732 as double) AS congestion_surcharge#2753, try_cast(airport_fee#2733 as double) AS airport_fee#2754, current_timestamp() AS current_timestamp()#2735, coalesce(year(cast(tpep_pickup_datetime#2716 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#2755, coalesce(month(cast(tpep_pickup_datetime#2716 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#2756, coalesce(day(cast(tpep_pickup_date...
|
|
|
[88]
|
[88]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:57
|
2026/09/03 15:55:57
|
2026/09/03 15:55:57
|
113 ms
|
121 ms
|
sql_command {
input {
common {
plan_id: 89
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-06.parquet`\n "
}
}
}
|
CLOSED
|
5b82384b-d3f7-41dc-ae98-91690da5f528
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_5b82384b-d3f7-41dc-ae98-91690da5f528
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3860L as bigint) AS VendorID#3881L, try_cast(tpep_pickup_datetime#3861 as timestamp) AS tpep_pickup_datetime#3882, try_cast(tpep_dropoff_datetime#3862 as timestamp) AS tpep_dropoff_datetime#3883, try_cast(passenger_count#3863L as bigint) AS passenger_count#3884L, try_cast(trip_distance#3864 as double) AS trip_distance#3885, try_cast(RatecodeID#3865L as bigint) AS RatecodeID#3886L, try_cast(store_and_fwd_flag#3866 as string) AS store_and_fwd_flag#3887, try_cast(PULocationID#3867L as bigint) AS PULocationID#3888L, try_cast(DOLocationID#3868L as bigint) AS DOLocationID#3889L, try_cast(payment_type#3869L as bigint) AS payment_type#3890L, try_cast(fare_amount#3870 as double) AS fare_amount#3891, try_cast(extra#3871 as double) AS extra#3892, try_cast(mta_tax#3872 as double) AS mta_tax#3893, try_cast(tip_amount#3873 as double) AS tip_amount#3894, try_cast(tolls_amount#3874 as double) AS tolls_amount#3895, try_cast(improvement_surcharge#3875 as double) AS improvement_surcharge#3896, try_cast(total_amount#3876 as double) AS total_amount#3897, try_cast(congestion_surcharge#3877 as double) AS congestion_surcharge#3898, try_cast(airport_fee#3878 as double) AS airport_fee#3899, current_timestamp() AS current_timestamp()#3880, coalesce(year(cast(tpep_pickup_datetime#3861 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3900, coalesce(month(cast(tpep_pickup_datetime#3861 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3901, coalesce(day(cast(tpep_pickup_d...
|
|
|
[199]
|
[205]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:36
|
2026/09/03 16:13:36
|
2026/09/03 16:13:37
|
113 ms
|
117 ms
|
write_operation {
input {
common {
plan_id: 74
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-01.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
7338ba55-d94b-45e1-998b-8100ac497b10
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_7338ba55-d94b-45e1-998b-8100ac497b10
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[235]
|
[241]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
113 ms
|
119 ms
|
write_operation {
input {
common {
plan_id: 144
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-01.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
e9d4efb1-75f2-497f-9d0f-d75b7833153b
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_e9d4efb1-75f2-497f-9d0f-d75b7833153b
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[75]
|
[75]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:55
|
2026/09/03 15:55:56
|
2026/09/03 15:55:56
|
114 ms
|
121 ms
|
sql_command {
input {
common {
plan_id: 76
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-03.parquet`\n "
}
}
}
|
CLOSED
|
a24285a8-f4de-40a8-bb1a-495bfafe95d9
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_a24285a8-f4de-40a8-bb1a-495bfafe95d9
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3287 as bigint) AS VendorID#3308L, try_cast(tpep_pickup_datetime#3288 as timestamp) AS tpep_pickup_datetime#3309, try_cast(tpep_dropoff_datetime#3289 as timestamp) AS tpep_dropoff_datetime#3310, try_cast(passenger_count#3290L as bigint) AS passenger_count#3311L, try_cast(trip_distance#3291 as double) AS trip_distance#3312, try_cast(RatecodeID#3292L as bigint) AS RatecodeID#3313L, try_cast(store_and_fwd_flag#3293 as string) AS store_and_fwd_flag#3314, try_cast(PULocationID#3294 as bigint) AS PULocationID#3315L, try_cast(DOLocationID#3295 as bigint) AS DOLocationID#3316L, try_cast(payment_type#3296L as bigint) AS payment_type#3317L, try_cast(fare_amount#3297 as double) AS fare_amount#3318, try_cast(extra#3298 as double) AS extra#3319, try_cast(mta_tax#3299 as double) AS mta_tax#3320, try_cast(tip_amount#3300 as double) AS tip_amount#3321, try_cast(tolls_amount#3301 as double) AS tolls_amount#3322, try_cast(improvement_surcharge#3302 as double) AS improvement_surcharge#3323, try_cast(total_amount#3303 as double) AS total_amount#3324, try_cast(congestion_surcharge#3304 as double) AS congestion_surcharge#3325, try_cast(airport_fee#3305 as double) AS airport_fee#3326, current_timestamp() AS current_timestamp()#3307, coalesce(year(cast(tpep_pickup_datetime#3288 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3327, coalesce(month(cast(tpep_pickup_datetime#3288 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3328, coalesce(day(cast(tpep_pickup_date...
|
|
|
[66]
|
[66]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:54
|
2026/09/03 15:55:54
|
2026/09/03 15:55:54
|
114 ms
|
118 ms
|
sql_command {
input {
common {
plan_id: 67
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2021-03.parquet`\n "
}
}
}
|
CLOSED
|
feb15988-28ea-4d06-912b-78a90866264b
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_feb15988-28ea-4d06-912b-78a90866264b
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#2891L as bigint) AS VendorID#2912L, try_cast(tpep_pickup_datetime#2892 as timestamp) AS tpep_pickup_datetime#2913, try_cast(tpep_dropoff_datetime#2893 as timestamp) AS tpep_dropoff_datetime#2914, try_cast(passenger_count#2894 as bigint) AS passenger_count#2915L, try_cast(trip_distance#2895 as double) AS trip_distance#2916, try_cast(RatecodeID#2896 as bigint) AS RatecodeID#2917L, try_cast(store_and_fwd_flag#2897 as string) AS store_and_fwd_flag#2918, try_cast(PULocationID#2898L as bigint) AS PULocationID#2919L, try_cast(DOLocationID#2899L as bigint) AS DOLocationID#2920L, try_cast(payment_type#2900L as bigint) AS payment_type#2921L, try_cast(fare_amount#2901 as double) AS fare_amount#2922, try_cast(extra#2902 as double) AS extra#2923, try_cast(mta_tax#2903 as double) AS mta_tax#2924, try_cast(tip_amount#2904 as double) AS tip_amount#2925, try_cast(tolls_amount#2905 as double) AS tolls_amount#2926, try_cast(improvement_surcharge#2906 as double) AS improvement_surcharge#2927, try_cast(total_amount#2907 as double) AS total_amount#2928, try_cast(congestion_surcharge#2908 as double) AS congestion_surcharge#2929, try_cast(airport_fee#2909 as double) AS airport_fee#2930, current_timestamp() AS current_timestamp()#2911, coalesce(year(cast(tpep_pickup_datetime#2892 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#2931, coalesce(month(cast(tpep_pickup_datetime#2892 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#2932, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[181]
|
[187]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
2026/09/03 16:13:34
|
116 ms
|
121 ms
|
write_operation {
input {
common {
plan_id: 38
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-07.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
19df5b34-f378-494e-81db-e3371ab2c257
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_19df5b34-f378-494e-81db-e3371ab2c257
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[206]
|
[212]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:37
|
2026/09/03 16:13:38
|
2026/09/03 16:13:38
|
116 ms
|
121 ms
|
write_operation {
input {
common {
plan_id: 88
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2025-02.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
1f032c4e-d4a8-4748-b54f-a53e0013de78
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_1f032c4e-d4a8-4748-b54f-a53e0013de78
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[188]
|
[194]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
2026/09/03 16:13:35
|
116 ms
|
123 ms
|
write_operation {
input {
common {
plan_id: 52
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
82e7c2d4-fb99-40f7-b2c3-701d13a1e7b3
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_82e7c2d4-fb99-40f7-b2c3-701d13a1e7b3
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[86]
|
[86]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:57
|
2026/09/03 15:55:57
|
2026/09/03 15:55:57
|
117 ms
|
125 ms
|
sql_command {
input {
common {
plan_id: 87
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-08.parquet`\n "
}
}
}
|
CLOSED
|
b67a5f83-a293-4cb0-a08f-01c601ab8aab
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_b67a5f83-a293-4cb0-a08f-01c601ab8aab
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3772L as bigint) AS VendorID#3793L, try_cast(tpep_pickup_datetime#3773 as timestamp) AS tpep_pickup_datetime#3794, try_cast(tpep_dropoff_datetime#3774 as timestamp) AS tpep_dropoff_datetime#3795, try_cast(passenger_count#3775 as bigint) AS passenger_count#3796L, try_cast(trip_distance#3776 as double) AS trip_distance#3797, try_cast(RatecodeID#3777 as bigint) AS RatecodeID#3798L, try_cast(store_and_fwd_flag#3778 as string) AS store_and_fwd_flag#3799, try_cast(PULocationID#3779L as bigint) AS PULocationID#3800L, try_cast(DOLocationID#3780L as bigint) AS DOLocationID#3801L, try_cast(payment_type#3781L as bigint) AS payment_type#3802L, try_cast(fare_amount#3782 as double) AS fare_amount#3803, try_cast(extra#3783 as double) AS extra#3804, try_cast(mta_tax#3784 as double) AS mta_tax#3805, try_cast(tip_amount#3785 as double) AS tip_amount#3806, try_cast(tolls_amount#3786 as double) AS tolls_amount#3807, try_cast(improvement_surcharge#3787 as double) AS improvement_surcharge#3808, try_cast(total_amount#3788 as double) AS total_amount#3809, try_cast(congestion_surcharge#3789 as double) AS congestion_surcharge#3810, try_cast(airport_fee#3790 as double) AS airport_fee#3811, current_timestamp() AS current_timestamp()#3792, coalesce(year(cast(tpep_pickup_datetime#3773 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3812, coalesce(month(cast(tpep_pickup_datetime#3773 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3813, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[63]
|
[63]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:54
|
2026/09/03 15:55:54
|
2026/09/03 15:55:54
|
117 ms
|
127 ms
|
sql_command {
input {
common {
plan_id: 64
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-10.parquet`\n "
}
}
}
|
CLOSED
|
da61d58d-5f1c-4128-bd01-85135b94f575
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_da61d58d-5f1c-4128-bd01-85135b94f575
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#2759L as bigint) AS VendorID#2780L, try_cast(tpep_pickup_datetime#2760 as timestamp) AS tpep_pickup_datetime#2781, try_cast(tpep_dropoff_datetime#2761 as timestamp) AS tpep_dropoff_datetime#2782, try_cast(passenger_count#2762 as bigint) AS passenger_count#2783L, try_cast(trip_distance#2763 as double) AS trip_distance#2784, try_cast(RatecodeID#2764 as bigint) AS RatecodeID#2785L, try_cast(store_and_fwd_flag#2765 as string) AS store_and_fwd_flag#2786, try_cast(PULocationID#2766L as bigint) AS PULocationID#2787L, try_cast(DOLocationID#2767L as bigint) AS DOLocationID#2788L, try_cast(payment_type#2768L as bigint) AS payment_type#2789L, try_cast(fare_amount#2769 as double) AS fare_amount#2790, try_cast(extra#2770 as double) AS extra#2791, try_cast(mta_tax#2771 as double) AS mta_tax#2792, try_cast(tip_amount#2772 as double) AS tip_amount#2793, try_cast(tolls_amount#2773 as double) AS tolls_amount#2794, try_cast(improvement_surcharge#2774 as double) AS improvement_surcharge#2795, try_cast(total_amount#2775 as double) AS total_amount#2796, try_cast(congestion_surcharge#2776 as double) AS congestion_surcharge#2797, try_cast(airport_fee#2777 as double) AS airport_fee#2798, current_timestamp() AS current_timestamp()#2779, coalesce(year(cast(tpep_pickup_datetime#2760 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#2799, coalesce(month(cast(tpep_pickup_datetime#2760 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#2800, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[67]
|
[67]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:54
|
2026/09/03 15:55:55
|
2026/09/03 15:55:55
|
119 ms
|
127 ms
|
sql_command {
input {
common {
plan_id: 68
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-11.parquet`\n "
}
}
}
|
CLOSED
|
3eadc714-e301-4dca-9d15-ca05eb9c2b8a
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_3eadc714-e301-4dca-9d15-ca05eb9c2b8a
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#2935L as bigint) AS VendorID#2956L, try_cast(tpep_pickup_datetime#2936 as timestamp) AS tpep_pickup_datetime#2957, try_cast(tpep_dropoff_datetime#2937 as timestamp) AS tpep_dropoff_datetime#2958, try_cast(passenger_count#2938 as bigint) AS passenger_count#2959L, try_cast(trip_distance#2939 as double) AS trip_distance#2960, try_cast(RatecodeID#2940 as bigint) AS RatecodeID#2961L, try_cast(store_and_fwd_flag#2941 as string) AS store_and_fwd_flag#2962, try_cast(PULocationID#2942L as bigint) AS PULocationID#2963L, try_cast(DOLocationID#2943L as bigint) AS DOLocationID#2964L, try_cast(payment_type#2944L as bigint) AS payment_type#2965L, try_cast(fare_amount#2945 as double) AS fare_amount#2966, try_cast(extra#2946 as double) AS extra#2967, try_cast(mta_tax#2947 as double) AS mta_tax#2968, try_cast(tip_amount#2948 as double) AS tip_amount#2969, try_cast(tolls_amount#2949 as double) AS tolls_amount#2970, try_cast(improvement_surcharge#2950 as double) AS improvement_surcharge#2971, try_cast(total_amount#2951 as double) AS total_amount#2972, try_cast(congestion_surcharge#2952 as double) AS congestion_surcharge#2973, try_cast(airport_fee#2953 as double) AS airport_fee#2974, current_timestamp() AS current_timestamp()#2955, coalesce(year(cast(tpep_pickup_datetime#2936 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#2975, coalesce(month(cast(tpep_pickup_datetime#2936 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#2976, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[216]
|
[222]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:39
|
2026/09/03 16:13:39
|
2026/09/03 16:13:39
|
119 ms
|
128 ms
|
write_operation {
input {
common {
plan_id: 108
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-06.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
821d0b0d-41ea-48a1-b8fa-a3c7cf284237
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_821d0b0d-41ea-48a1-b8fa-a3c7cf284237
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[61]
|
[61]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:54
|
2026/09/03 15:55:54
|
2026/09/03 15:55:54
|
120 ms
|
127 ms
|
sql_command {
input {
common {
plan_id: 62
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-11.parquet`\n "
}
}
}
|
CLOSED
|
5d5b0d8c-a3a4-4b14-9ec4-126ac9ef9b6c
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_5d5b0d8c-a3a4-4b14-9ec4-126ac9ef9b6c
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#2671L as bigint) AS VendorID#2692L, try_cast(tpep_pickup_datetime#2672 as timestamp) AS tpep_pickup_datetime#2693, try_cast(tpep_dropoff_datetime#2673 as timestamp) AS tpep_dropoff_datetime#2694, try_cast(passenger_count#2674 as bigint) AS passenger_count#2695L, try_cast(trip_distance#2675 as double) AS trip_distance#2696, try_cast(RatecodeID#2676 as bigint) AS RatecodeID#2697L, try_cast(store_and_fwd_flag#2677 as string) AS store_and_fwd_flag#2698, try_cast(PULocationID#2678L as bigint) AS PULocationID#2699L, try_cast(DOLocationID#2679L as bigint) AS DOLocationID#2700L, try_cast(payment_type#2680L as bigint) AS payment_type#2701L, try_cast(fare_amount#2681 as double) AS fare_amount#2702, try_cast(extra#2682 as double) AS extra#2703, try_cast(mta_tax#2683 as double) AS mta_tax#2704, try_cast(tip_amount#2684 as double) AS tip_amount#2705, try_cast(tolls_amount#2685 as double) AS tolls_amount#2706, try_cast(improvement_surcharge#2686 as double) AS improvement_surcharge#2707, try_cast(total_amount#2687 as double) AS total_amount#2708, try_cast(congestion_surcharge#2688 as double) AS congestion_surcharge#2709, try_cast(airport_fee#2689 as double) AS airport_fee#2710, current_timestamp() AS current_timestamp()#2691, coalesce(year(cast(tpep_pickup_datetime#2672 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#2711, coalesce(month(cast(tpep_pickup_datetime#2672 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#2712, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[87]
|
[87]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:57
|
2026/09/03 15:55:57
|
2026/09/03 15:55:57
|
120 ms
|
125 ms
|
sql_command {
input {
common {
plan_id: 88
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-12.parquet`\n "
}
}
}
|
CLOSED
|
b2bd0011-a204-4aa2-bb89-081f8b518c8b
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_b2bd0011-a204-4aa2-bb89-081f8b518c8b
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3816L as bigint) AS VendorID#3837L, try_cast(tpep_pickup_datetime#3817 as timestamp) AS tpep_pickup_datetime#3838, try_cast(tpep_dropoff_datetime#3818 as timestamp) AS tpep_dropoff_datetime#3839, try_cast(passenger_count#3819 as bigint) AS passenger_count#3840L, try_cast(trip_distance#3820 as double) AS trip_distance#3841, try_cast(RatecodeID#3821 as bigint) AS RatecodeID#3842L, try_cast(store_and_fwd_flag#3822 as string) AS store_and_fwd_flag#3843, try_cast(PULocationID#3823L as bigint) AS PULocationID#3844L, try_cast(DOLocationID#3824L as bigint) AS DOLocationID#3845L, try_cast(payment_type#3825L as bigint) AS payment_type#3846L, try_cast(fare_amount#3826 as double) AS fare_amount#3847, try_cast(extra#3827 as double) AS extra#3848, try_cast(mta_tax#3828 as double) AS mta_tax#3849, try_cast(tip_amount#3829 as double) AS tip_amount#3850, try_cast(tolls_amount#3830 as double) AS tolls_amount#3851, try_cast(improvement_surcharge#3831 as double) AS improvement_surcharge#3852, try_cast(total_amount#3832 as double) AS total_amount#3853, try_cast(congestion_surcharge#3833 as double) AS congestion_surcharge#3854, try_cast(airport_fee#3834 as double) AS airport_fee#3855, current_timestamp() AS current_timestamp()#3836, coalesce(year(cast(tpep_pickup_datetime#3817 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3856, coalesce(month(cast(tpep_pickup_datetime#3817 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3857, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[236]
|
[242]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
2026/09/03 16:14:08
|
121 ms
|
127 ms
|
write_operation {
input {
common {
plan_id: 146
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2024-11.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
a096a92e-2cf9-4190-bd0c-07e3dea8fab7
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_a096a92e-2cf9-4190-bd0c-07e3dea8fab7
|
|
DELTA_FAILED_TO_MERGE_FIELDS
+details
INTERNAL: [DELTA_FAILED_TO_MERGE_FIELDS] Failed to merge fields 'VendorID' and 'VendorID'
|
|
|
[78]
|
[78]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:56
|
2026/09/03 15:55:56
|
2026/09/03 15:55:56
|
122 ms
|
127 ms
|
sql_command {
input {
common {
plan_id: 79
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2019-07.parquet`\n "
}
}
}
|
CLOSED
|
dae7f573-38d6-40b9-a0b7-62773b7ed2d8
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_dae7f573-38d6-40b9-a0b7-62773b7ed2d8
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#3420L as bigint) AS VendorID#3441L, try_cast(tpep_pickup_datetime#3421 as timestamp) AS tpep_pickup_datetime#3442, try_cast(tpep_dropoff_datetime#3422 as timestamp) AS tpep_dropoff_datetime#3443, try_cast(passenger_count#3423 as bigint) AS passenger_count#3444L, try_cast(trip_distance#3424 as double) AS trip_distance#3445, try_cast(RatecodeID#3425 as bigint) AS RatecodeID#3446L, try_cast(store_and_fwd_flag#3426 as string) AS store_and_fwd_flag#3447, try_cast(PULocationID#3427L as bigint) AS PULocationID#3448L, try_cast(DOLocationID#3428L as bigint) AS DOLocationID#3449L, try_cast(payment_type#3429L as bigint) AS payment_type#3450L, try_cast(fare_amount#3430 as double) AS fare_amount#3451, try_cast(extra#3431 as double) AS extra#3452, try_cast(mta_tax#3432 as double) AS mta_tax#3453, try_cast(tip_amount#3433 as double) AS tip_amount#3454, try_cast(tolls_amount#3434 as double) AS tolls_amount#3455, try_cast(improvement_surcharge#3435 as double) AS improvement_surcharge#3456, try_cast(total_amount#3436 as double) AS total_amount#3457, try_cast(congestion_surcharge#3437 as double) AS congestion_surcharge#3458, try_cast(airport_fee#3438 as double) AS airport_fee#3459, current_timestamp() AS current_timestamp()#3440, coalesce(year(cast(tpep_pickup_datetime#3421 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3460, coalesce(month(cast(tpep_pickup_datetime#3421 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3461, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[202]
|
[208]
|
50cf85d5-ae27-4707-b30c-2f106b0fbf42
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
2026/09/03 16:13:37
|
124 ms
|
133 ms
|
write_operation {
input {
common {
plan_id: 80
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2022-04.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw_ingest"
mode: SAVE_MODE_APPEND
}
|
CLOSED
|
d3e4ca73-208a-48c3-bbdf-5076a7d99caa
|
SparkConnect_OperationTag_User__Session_50cf85d5-ae27-4707-b30c-2f106b0fbf42_Operation_d3e4ca73-208a-48c3-bbdf-5076a7d99caa
|
|
DELTA_METADATA_MISMATCH
+details
INTERNAL: [DELTA_METADATA_MISMATCH] A metadata mismatch was detected when writing to the Delta table.
- A schema mismatch detected when writing to the Delta table (Table ID: 28b4cb71-f282-476d-b38f-4c5b4063764f).
To enable schema migration using DataFrameWriter or DataStreamWriter, please set: '.option("mergeSchema", "true")'.
For other operations, set the session configuration spark.databricks.delta.schema.autoMerge.enabled to "true". See the documentation specific to the operation for details.
Table schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double (nullable = true)
|-- improvement_surcharge: double (nullable = true)
|-- total_amount: double (nullable = true)
|-- congestion_surcharge: void (nullable = true)
|-- airport_fee: void (nullable = true)
Data schema:
root
|-- VendorID: long (nullable = true)
|-- tpep_pickup_datetime: timestamp_ntz (nullable = true)
|-- tpep_dropoff_datetime: timestamp_ntz (nullable = true)
|-- passenger_count: double (nullable = true)
|-- trip_distance: double (nullable = true)
|-- RatecodeID: double (nullable = true)
|-- store_and_fwd_flag: string (nullable = true)
|-- PULocationID: long (nullable = true)
|-- DOLocationID: long (nullable = true)
|-- payment_type: long (nullable = true)
|-- fare_amount: double (nullable = true)
|-- extra: double (nullable = true)
|-- mta_tax: double (nullable = true)
|-- tip_amount: double (nullable = true)
|-- tolls_amount: double ...
|
|
|
[68]
|
[68]
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 15:55:55
|
2026/09/03 15:55:55
|
2026/09/03 15:55:55
|
125 ms
|
130 ms
|
sql_command {
input {
common {
plan_id: 69
}
sql {
query: "insert into yellow_taxi.trips\n select \n try_cast(VendorID as long),\n try_cast(tpep_pickup_datetime as timestamp),\n try_cast(tpep_dropoff_datetime as timestamp),\n try_cast(passenger_count as long),\n try_cast(trip_distance as double),\n try_cast(RatecodeID as long),\n try_cast(store_and_fwd_flag as string),\n try_cast(PULocationID as long),\n try_cast(DOLocationID as long),\n try_cast(payment_type as long),\n try_cast(fare_amount as double),\n try_cast(extra as double),\n try_cast(mta_tax as double),\n try_cast(tip_amount as double),\n try_cast(tolls_amount as double),\n try_cast(improvement_surcharge as double),\n try_cast(total_amount as double),\n try_cast(congestion_surcharge as double),\n try_cast(airport_fee as double),\n current_timestamp,\n coalesce(year(tpep_pickup_datetime),2026),\n coalesce(month(tpep_pickup_datetime),9),\n coalesce(day(tpep_pickup_datetime),3)\n from\n parquet.`/home/jovyan/data/yellow-taxi/yellow_tripdata_2018-09.parquet`\n "
}
}
}
|
CLOSED
|
63857746-50c6-4e6f-b0a0-3726dd1a630a
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_63857746-50c6-4e6f-b0a0-3726dd1a630a
|
|
TABLE_OR_VIEW_NOT_FOUND
+details
INTERNAL: [TABLE_OR_VIEW_NOT_FOUND] The table or view `yellow_taxi`.`trips` cannot be found. Verify the spelling and correctness of the schema and catalog.
If you did not qualify the name with a schema, verify the current_schema() output, or qualify the name with the correct schema and catalog.
To tolerate the error on drop use DROP VIEW IF EXISTS or DROP TABLE IF EXISTS. SQLSTATE: 42P01; line 1 pos 12;
'InsertIntoStatement 'UnresolvedRelation [yellow_taxi, trips], [__required_write_privileges__=INSERT], false, false, false, false
+- Project [try_cast(VendorID#2979L as bigint) AS VendorID#3000L, try_cast(tpep_pickup_datetime#2980 as timestamp) AS tpep_pickup_datetime#3001, try_cast(tpep_dropoff_datetime#2981 as timestamp) AS tpep_dropoff_datetime#3002, try_cast(passenger_count#2982 as bigint) AS passenger_count#3003L, try_cast(trip_distance#2983 as double) AS trip_distance#3004, try_cast(RatecodeID#2984 as bigint) AS RatecodeID#3005L, try_cast(store_and_fwd_flag#2985 as string) AS store_and_fwd_flag#3006, try_cast(PULocationID#2986L as bigint) AS PULocationID#3007L, try_cast(DOLocationID#2987L as bigint) AS DOLocationID#3008L, try_cast(payment_type#2988L as bigint) AS payment_type#3009L, try_cast(fare_amount#2989 as double) AS fare_amount#3010, try_cast(extra#2990 as double) AS extra#3011, try_cast(mta_tax#2991 as double) AS mta_tax#3012, try_cast(tip_amount#2992 as double) AS tip_amount#3013, try_cast(tolls_amount#2993 as double) AS tolls_amount#3014, try_cast(improvement_surcharge#2994 as double) AS improvement_surcharge#3015, try_cast(total_amount#2995 as double) AS total_amount#3016, try_cast(congestion_surcharge#2996 as double) AS congestion_surcharge#3017, try_cast(airport_fee#2997 as double) AS airport_fee#3018, current_timestamp() AS current_timestamp()#2999, coalesce(year(cast(tpep_pickup_datetime#2980 as date)), 2026) AS coalesce(year(tpep_pickup_datetime), 2026)#3019, coalesce(month(cast(tpep_pickup_datetime#2980 as date)), 9) AS coalesce(month(tpep_pickup_datetime), 9)#3020, coalesce(day(cast(tpep_pickup_dat...
|
|
|
[127]
|
|
29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0
|
2026/09/03 16:10:07
|
2026/09/03 16:10:07
|
|
127 ms
|
8 weeks 3 days 3 hours
|
write_operation {
input {
common {
plan_id: 149
}
read {
data_source {
format: "parquet"
schema: ""
paths: "/home/jovyan/data/yellow-taxi/yellow_tripdata_2020-11.parquet"
}
}
}
source: "delta"
path: "/home/jovyan/spark-warehouse/yellow_taxi.db/trips_raw"
}
|
CANCELED
|
7865d33d-b14d-45dd-80a0-07c104be9353
|
SparkConnect_OperationTag_User__Session_29cbf5ee-3bc4-44da-a6fe-5b90efeec9f0_Operation_7865d33d-b14d-45dd-80a0-07c104be9353
|
|
|